diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index aca3ccfa9c5..e6fcdecf460 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -144,12 +144,16 @@ jobs: # Cross-stack generation contracts: the frontend writes these, Python validates them # against the invocation and architecture registries. A frontend-only change that # regenerates one would otherwise pass vitest while skipping the backend half. - # Shared Generation numerical fixtures are read by Python tests, so fixture-only changes - # must trigger this suite. + # `__fixtures__/` does not exist yet -- it arrives with the graph-builder fixtures. + # A pattern matching nothing costs nothing here, and listing it now means the branch + # that adds the directory does not also have to remember to add this line. - 'invokeai/frontend/webv2/src/features/generation/core/__snapshots__/**' - 'invokeai/frontend/webv2/src/features/generation/core/__fixtures__/**' - 'invokeai/frontend/webv2/src/features/video/core/__snapshots__/**' - 'invokeai/frontend/webv2/src/features/upscale/core/__snapshots__/**' + # test_shared_numerical_fixtures.py checks the Python image adjustments against the + # canvas spatial-adjustment kernels' shared fixtures. + - 'invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/__fixtures__/**' # test_ideogram4_frontend_bounds.py reads the Ideogram 4 control bounds from here. - 'invokeai/frontend/webv2/src/features/generation/core/settings.ts' - 'tests/**' diff --git a/docs/src/content/docs/Users Guide/02.Projects/introduction.mdx b/docs/src/content/docs/Users Guide/02.Projects/introduction.mdx index 91d42441d97..27e270c3b1f 100644 --- a/docs/src/content/docs/Users Guide/02.Projects/introduction.mdx +++ b/docs/src/content/docs/Users Guide/02.Projects/introduction.mdx @@ -170,6 +170,6 @@ In the Workflow Library dialog, **This project** lists the project's own workflo - **Projects belong to one account.** With [multiple users](/configuration/multi-user-mode/user-guide/) enabled, each person sees only their own projects, and there is no project sharing. Export and import a `.invk` file to pass a project to someone else. - **Undo history is not saved** with the project; it is cleared when you reload or close it. - **Invoke needs the server to open projects.** If it cannot reach the server, a **Projects are temporarily unavailable** screen offers to retry and to export any unsaved work your browser is holding. -- **Newer projects need a newer Invoke.** A project saved by a newer version of Invoke shows **Requires a newer Invoke** and cannot be opened until you upgrade. Some features raise this requirement for good: once a project has had a Noise adjustment on the canvas, versions of Invoke without Noise adjustments refuse to open it, even after you remove the adjustment. +- **Newer projects need a newer Invoke.** A project saved by a newer version of Invoke shows **Requires a newer Invoke** and cannot be opened until you upgrade. Some features raise this requirement for good: once a project has had a Noise, Texture, Clarity or Dehaze adjustment on the canvas, versions of Invoke without those adjustments refuse to open it, even after you remove the adjustment. - **Size limits.** A project file can be up to 2 GB, with up to 20,000 files. Export reports a project that is too large to package. - **Clear saved data…** under **Data & workspace** in Settings deletes *all* of your projects and settings at once, and cannot be undone. diff --git a/docs/src/content/docs/Users Guide/07.Canvas/03.layers-panel.mdx b/docs/src/content/docs/Users Guide/07.Canvas/03.layers-panel.mdx index 65bb138e11e..84d61ac098a 100644 --- a/docs/src/content/docs/Users Guide/07.Canvas/03.layers-panel.mdx +++ b/docs/src/content/docs/Users Guide/07.Canvas/03.layers-panel.mdx @@ -86,11 +86,11 @@ The icons at the top **Arrange** the layer (move it to the front, forward, backw * **Delete layer**. * **Raster layers:** * **Transform**. - * **Add adjustment** — Brightness/Contrast, Exposure, Levels, Curves, Saturation, Hue, Invert or Noise. Adjustments are non-destructive and appear as child rows. Noise adds film grain. + * **Add adjustment** — Brightness/Contrast, Exposure, Levels, Curves, Saturation, Hue, Invert, Texture, Clarity, Dehaze or Noise. Adjustments are non-destructive and appear as child rows. Texture, Clarity and Dehaze change detail and contrast; Noise adds film grain. * **Add regenerate region** — treats the layer's own shape as an inpaint mask. * **Filter**, **Select object** and **Run workflow** — process the layer; see the [Canvas guide](/users-guide/canvas/canvas-guide/#filters-and-object-selection). * **Boolean operations** — combine the layer with the one below: intersect, cut out, cut away or exclude. - * **Convert to** — make it a control layer, inpaint mask or regional guidance layer. Text, shape and gradient layers can also be rasterized into plain pixels. + * **Convert to** — make it a control layer, inpaint mask or regional guidance layer. Text, shape and gradient layers can also be rasterized into plain pixels. Control layers have no adjustments, so converting or copying a layer with adjustments to a control layer bakes them, with the layer's scaling and rotation, into its pixels; undo restores the raster layer with its adjustments. * **Control layers:** **Transform**, **Filter**, **Select object**, **Run workflow**, **Enable transparent preview**, and **Convert to raster layer**. * **Inpaint masks:** **Add noise**, **Add denoise limit**, **Extract masked area** (copy the masked part of the image to a new layer), and **Hide layer**. * **Regional guidance:** **Enable auto-negative**, **Add reference image**, and **Hide layer**. @@ -137,6 +137,27 @@ Things to know about Noise: * **Baking.** Rasterizing a text, shape or gradient layer whose Noise adjustment is enabled bakes its whole adjustment stack into the pixels and removes it, so the grain stays where it was. **Merge down** bakes each layer's adjustments the same way. * **Older versions of Invoke.** Once a project has had a Noise adjustment, it needs a version of Invoke with Noise adjustments, even after you remove the adjustment; older versions refuse to open it. +**Texture**, **Clarity** and **Dehaze** work on detail and contrast across an area rather than pixel by pixel. They match the [Clarity, Texture and Contrast Dehaze workflow nodes](/users-guide/workflows/image-adjustments/): + +* **Texture** — fine detail, such as skin, fabric or foliage. Positive amounts bring it out; negative amounts smooth it. +* **Clarity** — midtone contrast in larger shapes. Positive amounts add punch; negative amounts soften the image. +* **Dehaze** — broad contrast and color intensity. Positive amounts increase both, which can lift a flat, washed-out image; negative amounts reduce both. It is an aesthetic approximation, not physical haze removal: it does not estimate real haze or recover lost detail, and strong settings can clip tones or colors. + +Their settings: + +* **Amount** — from −200 to 200. The default, 0, changes nothing. Double-click the field or press **Backspace** in it to reset it. +* **Size** (Texture and Clarity only) — how large the detail is that the adjustment affects: 1 to 100 px for Texture (8 by default) and 4 to 1000 px for Clarity (40 by default). It is measured in the layer's own pixels, before the layer is scaled, so resizing a layer does not change which detail is affected. On a group, it is measured in canvas pixels. + +Things to know about Texture, Clarity and Dehaze: + +* **They are computed in the background.** Each one looks at the whole layer, so the canvas computes it in the background and stays responsive. Meanwhile, the canvas shows the previous result, or the layer without them after you paint on it. A small spinner appears on the layer's row when the work takes more than a moment. +* **Size limit.** They cannot run on content larger than 4096 × 4096 px (16.8 megapixels, measured over the area the layer's visible content spans, in the layer's own pixels before any scaling; for a group, at document resolution). A warning icon then appears on the layer's or group's row, the canvas leaves them out, and anything that uses the layer's pixels with them — generating, exporting, merging, rasterizing, filtering, copying or converting to a control layer, saving to PSD, painting on an image layer — is refused with a message naming the layer or group. Disable or remove them, or crop the content. The same warning appears when there is not enough canvas memory or the computation fails; hover over it for what to do, or select the adjustment to read it in Properties. +* **What you see is what you get.** Generating, exporting, merging, rasterizing and saving to PSD use the same result as the canvas, computing it first if the canvas is still working on it. One exception: PSD has no group adjustments, so a group's Texture, Clarity and Dehaze are applied to each of its layers separately, and the PSD can look different from the canvas. +* **While you drag a selection**, the dragged pixels show without them; once placed, the layer is computed again with them. +* **Thumbnails** of layers the canvas has not shown yet can leave them out. +* **Baking.** Rasterizing a text, shape or gradient layer with one of them enabled bakes its whole adjustment stack into the pixels and removes it. **Merge down** bakes each layer's adjustments the same way. +* **Older versions of Invoke.** Like Noise, once a project has had one of these adjustments, older versions of Invoke refuse to open it, even after you remove the adjustment. + ### Control layer * **Adapter type** — ControlNet, T2I Adapter, Control LoRA, Z-Image ControlNet or Anima ControlNet-LLLite, depending on the model selected in the Generate panel. diff --git a/docs/src/content/docs/development/Front End/project-files.mdx b/docs/src/content/docs/development/Front End/project-files.mdx index e1f6311bd9f..5dcf56162fb 100644 --- a/docs/src/content/docs/development/Front End/project-files.mdx +++ b/docs/src/content/docs/development/Front End/project-files.mdx @@ -41,8 +41,9 @@ a maximum of v2 is refused before it can overwrite that document. A canvas adopts a newer schema only when its content needs one, that is, when an older reader would drop part of it on load and lose it on the next autosave. Schema 4 marks custom-font -typography (a text source with `fontRef`, `fontStyle` or `fontVariations`). Schema 5 marks a noise -adjustment, enabled or not, on any layer or group at any depth; an older reader would drop the +typography (a text source with `fontRef`, `fontStyle` or `fontVariations`). Schema 5 marks a noise, +clarity, texture or dehaze adjustment, enabled or not, on any layer or group at any depth; an older +reader would drop the whole adjustment stack holding it. The live document and each saved canvas snapshot are versioned this way, and the canvas state takes the newest of them. Adoption is sticky: removing the feature does not lower the version, as the server floor cannot be lowered either. A canvas with neither diff --git a/docs/src/content/docs/development/Guides/image-manipulation-provenance.mdx b/docs/src/content/docs/development/Guides/image-manipulation-provenance.mdx index 328f0f61a02..009a94d2adf 100644 --- a/docs/src/content/docs/development/Guides/image-manipulation-provenance.mdx +++ b/docs/src/content/docs/development/Guides/image-manipulation-provenance.mdx @@ -8,11 +8,11 @@ This record covers the image manipulation invocation and its shared numerical he ## Sources and implementation The image color-space conversion follows the published [Oklab model and reference equations](https://bottosson.github.io/posts/oklab/). -The scalar edge-aware smoothing follows the guided-filter formulation of He, Sun, and Tang, [Guided Image Filtering (ECCV 2010)](https://doi.org/10.1007/978-3-642-15549-9_1), using normalized box means and reflected boundaries. The sRGB transfer function follows the [CSS Color Module Level 4 definition](https://www.w3.org/TR/css-color-4/#predefined-sRGB). +The scalar edge-aware smoothing follows the guided-filter formulation of He, Sun, and Tang, [Guided Image Filtering (ECCV 2010)](https://doi.org/10.1007/978-3-642-15549-9_1), using normalized box means and reflected boundaries. The sRGB transfer function follows the [CSS Color Module Level 4 definition](https://www.w3.org/TR/css-color-4/#predefined-sRGB). The canvas fill of fully transparent pixels follows the chamfer distance transform of Borgefors, [Distance transformations in digital images (1986)](https://doi.org/10.1016/S0734-189X(86)80047-0). The clarity, texture, and contrast-dehaze controls are first-party image adjustments built from these documented equations and standard multiscale image-processing operations. Contrast Dehaze is an aesthetic approximation: it is not a physical haze estimator or scene-recovery method. The candidate contains no Dark Channel Prior (DCP) implementation or DCP control. -The references below identify mathematical and specification foundations only. They are not code sources. +The references below identify mathematical and specification foundations; apart from the gamut-clipping reference noted for `numerics.ts` below, they are not code sources. | Implementation file | Purpose and algorithm/source basis | Libraries and owner | | --- | --- | --- | @@ -23,11 +23,18 @@ The references below identify mathematical and specification foundations only. T | `invokeai/backend/image_util/guided_filter.py` | Scalar guided image filter; He, Sun, and Tang's guided-filter formulation, implemented with normalized OpenCV box filters. | NumPy, OpenCV core APIs (`cv2`). | | `invokeai/backend/image_util/multiscale.py` | Difference between two self-guided smooths; built on the guided-filter formulation. | NumPy; delegates to `guided_filter.py`. | | `invokeai/backend/image_util/texture.py` | Fine-scale detail adjustment with a local RMS gate; guided-filter bands and Oklab lightness. | NumPy, Pillow, OpenCV core APIs (`cv2`); delegates to the `color.py` and `multiscale.py` helpers. | -| `invokeai/frontend/webv2/src/features/generation/core/colorConversion.ts` | sRGB-to-Oklab conversion; published Oklab equations and sRGB transfer function. | TypeScript and standard ECMAScript numeric arrays/math. | -| `invokeai/frontend/webv2/src/features/generation/core/guidedFilter.ts` | Scalar guided filter with reflected boundaries; He, Sun, and Tang's guided-filter formulation. | TypeScript and standard ECMAScript numeric arrays/math. | -| `invokeai/frontend/webv2/src/features/generation/core/multiscale.ts` | Difference between guided-filter smooths; guided-filter formulation. | TypeScript and standard ECMAScript numeric arrays/math; delegates to `guidedFilter.ts`. | +| `invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/alphaFill.ts` | Nearest-visible-pixel labels for hidden-pixel fill; reproduces the labels of OpenCV's 5×5 chamfer distance transform (`distanceTransformWithLabels` with `DIST_L2` and `DIST_LABEL_PIXEL`), a chamfer transform after Borgefors. | TypeScript and standard ECMAScript typed arrays. | +| `invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/evaluateSpatialSteps.ts` | Canvas evaluation of Clarity, Texture, and Contrast Dehaze over a layer's alpha bounds; mirrors the first-party backend helpers above. | TypeScript and standard ECMAScript typed arrays/math; delegates to the other files in this directory and to the canvas engine's `alphaBounds.ts`. | +| `invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/guidedFilter.ts` | Self-guided filter and edge-aware bands with reflected boundaries and float64 box sums; He, Sun, and Tang's guided-filter formulation. | TypeScript and standard ECMAScript typed arrays/math. | +| `invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/numerics.ts` | sRGB transfer, Oklab conversion, Oklab gamut clipping, and Python-compatible half-to-even rounding; published Oklab equations, sRGB transfer function, and a scalar port of the backend's `gamut_clip_tensor` (`invokeai/backend/image_util/color_conversion.py`), which follows Björn Ottosson's [sRGB gamut clipping](https://bottosson.github.io/posts/gamutclipping/) reference implementation. | TypeScript and standard ECMAScript numeric arrays/math. Gamut-clipping portion `MIT`, Copyright (c) 2021 Björn Ottosson; the notice is reproduced in the file. | +| `invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialSteps.ts` | Step and result types shared with the canvas worker; no algorithm. | TypeScript. | -For **every file in the table**, the implementation was authored for InvokeAI, with no individual author asserted. No code was copied or adapted from the cited works, their reference implementations, or other external code. Each file is licensed under `Apache-2.0` (SPDX identifier in the source); no third-party code or third-party notices are included in these files. The Python modules call OpenCV core APIs only. The project already declares the `opencv-contrib-python` distribution, but this feature uses no contrib-only API and adds no contrib requirement or other dependency. +For every file in the table except the two noted below, the implementation was authored for InvokeAI, with no individual author asserted, and no code was copied or adapted from the cited works, their reference implementations, or other external code. + +- `numerics.ts` adapts Björn Ottosson's MIT-licensed gamut-clipping reference code (maximum-saturation polynomial and its coefficients, cusp search, gamut intersection, and the adaptive lightness anchor) through the backend's `gamut_clip_tensor`. It carries Ottosson's MIT copyright and permission notice and the SPDX expression `Apache-2.0 AND MIT`. The backend's `color_conversion.py`, which predates this feature and is outside this manifest, implements the same reference but does not carry that notice; `composition.py` carries a one-line Ottosson attribution. +- `alphaFill.ts`: to return the same labels as the backend, its mask weights, neighbor visiting order, and tie rule were taken from OpenCV 4.11.0's `distanceTransformEx_5x5` (`modules/imgproc/src/distransform.cpp`; OpenCV is distributed under `Apache-2.0`, and that file keeps its earlier Intel and OpenCV Foundation BSD-style notice). The TypeScript is an independent implementation of that behavior; no OpenCV source text is reproduced. + +Each file is licensed under `Apache-2.0` (SPDX identifier in the source), except the MIT-licensed gamut-clipping portion of `numerics.ts` described above; no other third-party code or notices are included in these files. The Python modules call OpenCV core APIs only. The project already declares the `opencv-contrib-python` distribution, but this feature uses no contrib-only API and adds no contrib requirement or other dependency. ## License and patent status diff --git a/invokeai/frontend/webv2/performance/architecture-baseline.json b/invokeai/frontend/webv2/performance/architecture-baseline.json index 0e6c8ec4b7a..201a7296cb6 100644 --- a/invokeai/frontend/webv2/performance/architecture-baseline.json +++ b/invokeai/frontend/webv2/performance/architecture-baseline.json @@ -855,6 +855,7 @@ "source:src/workbench/canvas-engine/render/adjustments.ts", "source:src/workbench/canvas-engine/render/compositor.ts", "source:src/workbench/canvas-engine/render/controlTransparency.ts", + "source:src/workbench/canvas-engine/render/exactAdjustments.ts", "source:src/workbench/canvas-engine/render/groupCompositeScopes.ts", "source:src/workbench/canvas-engine/render/maskFill.ts", "source:src/workbench/canvas-engine/render/rasterComposite.ts", diff --git a/invokeai/frontend/webv2/performance/browser-baseline.json b/invokeai/frontend/webv2/performance/browser-baseline.json index bc55fde72b5..ac6d601e260 100644 --- a/invokeai/frontend/webv2/performance/browser-baseline.json +++ b/invokeai/frontend/webv2/performance/browser-baseline.json @@ -29,12 +29,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 0, - "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "largestAssetRawBytes": 752431, + "otherRawBytes": 235365, "requestCount": 29, - "scriptRawBytes": 1882471, + "scriptRawBytes": 1897694, "scriptRequestCount": 25, - "totalRawBytes": 2139551 + "totalRawBytes": 2158882 }, "routeReadyMedianMs": 215.30000019073486, "scriptSourceOwnerSet": "launchpad-static", @@ -67,12 +67,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 0, - "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "largestAssetRawBytes": 752431, + "otherRawBytes": 235365, "requestCount": 29, - "scriptRawBytes": 1882471, + "scriptRawBytes": 1897694, "scriptRequestCount": 25, - "totalRawBytes": 2139551 + "totalRawBytes": 2158882 }, "routeReadyMedianMs": 225.39999961853027, "scriptSourceOwnerSet": "launchpad-static", @@ -105,12 +105,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "largestAssetRawBytes": 752431, + "otherRawBytes": 235365, "requestCount": 87, - "scriptRawBytes": 4250746, + "scriptRawBytes": 4312524, "scriptRequestCount": 82, - "totalRawBytes": 4552224 + "totalRawBytes": 4618110 }, "routeReadyMedianMs": 360.90000009536743, "scriptSourceOwnerSet": "editor-minimal-static", @@ -143,12 +143,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "largestAssetRawBytes": 752431, + "otherRawBytes": 235365, "requestCount": 87, - "scriptRawBytes": 4250746, + "scriptRawBytes": 4312524, "scriptRequestCount": 82, - "totalRawBytes": 4552224 + "totalRawBytes": 4618110 }, "routeReadyMedianMs": 367.80000019073486, "scriptSourceOwnerSet": "editor-minimal-static", @@ -159,12 +159,12 @@ "cssRawBytes": 0, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 311154, + "largestAssetRawBytes": 317101, "otherRawBytes": 0, "requestCount": 9, - "scriptRawBytes": 529607, + "scriptRawBytes": 540817, "scriptRequestCount": 9, - "totalRawBytes": 529607 + "totalRawBytes": 540817 }, "domContentLoadedMedianMs": 100.40000009536743, "id": "editor-canvas", @@ -181,12 +181,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "largestAssetRawBytes": 752431, + "otherRawBytes": 235365, "requestCount": 96, - "scriptRawBytes": 4823800, + "scriptRawBytes": 4853620, "scriptRequestCount": 91, - "totalRawBytes": 5126937 + "totalRawBytes": 5159206 }, "routeReadyMedianMs": 1095.3000001907349, "scriptSourceOwnerSet": "editor-canvas-static", @@ -197,12 +197,12 @@ "cssRawBytes": 0, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 311154, + "largestAssetRawBytes": 317101, "otherRawBytes": 0, "requestCount": 9, - "scriptRawBytes": 529607, + "scriptRawBytes": 540817, "scriptRequestCount": 9, - "totalRawBytes": 529607 + "totalRawBytes": 540817 }, "domContentLoadedMedianMs": 97.7000002861023, "id": "editor-canvas", @@ -219,12 +219,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "largestAssetRawBytes": 752431, + "otherRawBytes": 235365, "requestCount": 96, - "scriptRawBytes": 4823800, + "scriptRawBytes": 4853620, "scriptRequestCount": 91, - "totalRawBytes": 5126937 + "totalRawBytes": 5159206 }, "routeReadyMedianMs": 1042.5, "scriptSourceOwnerSet": "editor-canvas-static", @@ -235,12 +235,12 @@ "cssRawBytes": 15413, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 208822, + "largestAssetRawBytes": 208808, "otherRawBytes": 0, "requestCount": 7, - "scriptRawBytes": 451936, + "scriptRawBytes": 452154, "scriptRequestCount": 6, - "totalRawBytes": 467349 + "totalRawBytes": 467567 }, "domContentLoadedMedianMs": 101.40000009536743, "id": "editor-workflow", @@ -257,12 +257,12 @@ "cssRawBytes": 17572, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "largestAssetRawBytes": 752431, + "otherRawBytes": 235365, "requestCount": 94, - "scriptRawBytes": 4750033, + "scriptRawBytes": 4772177, "scriptRequestCount": 88, - "totalRawBytes": 5019779 + "totalRawBytes": 5093176 }, "routeReadyMedianMs": 1093.0999999046326, "scriptSourceOwnerSet": "editor-workflow-static", @@ -273,12 +273,12 @@ "cssRawBytes": 15413, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 208822, + "largestAssetRawBytes": 208808, "otherRawBytes": 0, "requestCount": 7, - "scriptRawBytes": 451936, + "scriptRawBytes": 452154, "scriptRequestCount": 6, - "totalRawBytes": 467349 + "totalRawBytes": 467567 }, "domContentLoadedMedianMs": 99.89999961853027, "id": "editor-workflow", @@ -295,12 +295,12 @@ "cssRawBytes": 17572, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "largestAssetRawBytes": 752431, + "otherRawBytes": 235365, "requestCount": 94, - "scriptRawBytes": 4750033, + "scriptRawBytes": 4772177, "scriptRequestCount": 88, - "totalRawBytes": 5019779 + "totalRawBytes": 5093176 }, "routeReadyMedianMs": 1105.0999999046326, "scriptSourceOwnerSet": "editor-workflow-static", @@ -333,12 +333,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "largestAssetRawBytes": 752431, + "otherRawBytes": 235365, "requestCount": 87, - "scriptRawBytes": 4250746, + "scriptRawBytes": 4312524, "scriptRequestCount": 82, - "totalRawBytes": 4552224 + "totalRawBytes": 4618110 }, "routeReadyMedianMs": 1324.5, "scriptSourceOwnerSet": "editor-minimal-static", @@ -371,12 +371,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "largestAssetRawBytes": 752431, + "otherRawBytes": 235365, "requestCount": 87, - "scriptRawBytes": 4250746, + "scriptRawBytes": 4312524, "scriptRequestCount": 82, - "totalRawBytes": 4552224 + "totalRawBytes": 4618110 }, "routeReadyMedianMs": 1301.5, "scriptSourceOwnerSet": "editor-minimal-static", @@ -1465,6 +1465,7 @@ "source:src/workbench/canvas-engine/render/controlTransparency.ts", "source:src/workbench/canvas-engine/render/decodedBitmapPool.ts", "source:src/workbench/canvas-engine/render/derivedSurfaceCache.ts", + "source:src/workbench/canvas-engine/render/exactAdjustments.ts", "source:src/workbench/canvas-engine/render/floatingSelectionFrame.ts", "source:src/workbench/canvas-engine/render/fontLoader.ts", "source:src/workbench/canvas-engine/render/frameDemand.ts", @@ -1485,6 +1486,8 @@ "source:src/workbench/canvas-engine/render/rasterizers/shapeRasterizer.ts", "source:src/workbench/canvas-engine/render/rasterizers/textRasterizer.ts", "source:src/workbench/canvas-engine/render/scheduler.ts", + "source:src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.ts", + "source:src/workbench/canvas-engine/render/spatialDisplay.ts", "source:src/workbench/canvas-engine/render/thumbnail.ts", "source:src/workbench/canvas-engine/samCoordinates.ts", "source:src/workbench/canvas-engine/selectObjectBridge.ts", @@ -2575,6 +2578,7 @@ "source:src/workbench/canvas-engine/render/controlTransparency.ts", "source:src/workbench/canvas-engine/render/decodedBitmapPool.ts", "source:src/workbench/canvas-engine/render/derivedSurfaceCache.ts", + "source:src/workbench/canvas-engine/render/exactAdjustments.ts", "source:src/workbench/canvas-engine/render/floatingSelectionFrame.ts", "source:src/workbench/canvas-engine/render/fontLoader.ts", "source:src/workbench/canvas-engine/render/frameDemand.ts", @@ -2595,6 +2599,8 @@ "source:src/workbench/canvas-engine/render/rasterizers/shapeRasterizer.ts", "source:src/workbench/canvas-engine/render/rasterizers/textRasterizer.ts", "source:src/workbench/canvas-engine/render/scheduler.ts", + "source:src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.ts", + "source:src/workbench/canvas-engine/render/spatialDisplay.ts", "source:src/workbench/canvas-engine/render/thumbnail.ts", "source:src/workbench/canvas-engine/samCoordinates.ts", "source:src/workbench/canvas-engine/selectObjectBridge.ts", @@ -2984,6 +2990,7 @@ "source:src/workbench/widgets/image-map/manifest.ts", "source:src/workbench/widgets/image-map/settingsContribution.ts", "source:src/workbench/widgets/layers/AdjustmentSettings.tsx", + "source:src/workbench/widgets/layers/AdjustmentStatusIcon.tsx", "source:src/workbench/widgets/layers/ControlLayerSettings.tsx", "source:src/workbench/widgets/layers/ControlLayerWarningIcon.tsx", "source:src/workbench/widgets/layers/InpaintMaskSettings.tsx", @@ -3908,6 +3915,7 @@ "source:src/workbench/canvas-engine/render/controlTransparency.ts", "source:src/workbench/canvas-engine/render/decodedBitmapPool.ts", "source:src/workbench/canvas-engine/render/derivedSurfaceCache.ts", + "source:src/workbench/canvas-engine/render/exactAdjustments.ts", "source:src/workbench/canvas-engine/render/floatingSelectionFrame.ts", "source:src/workbench/canvas-engine/render/fontLoader.ts", "source:src/workbench/canvas-engine/render/frameDemand.ts", @@ -3928,6 +3936,8 @@ "source:src/workbench/canvas-engine/render/rasterizers/shapeRasterizer.ts", "source:src/workbench/canvas-engine/render/rasterizers/textRasterizer.ts", "source:src/workbench/canvas-engine/render/scheduler.ts", + "source:src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.ts", + "source:src/workbench/canvas-engine/render/spatialDisplay.ts", "source:src/workbench/canvas-engine/render/thumbnail.ts", "source:src/workbench/canvas-engine/samCoordinates.ts", "source:src/workbench/canvas-engine/selectObjectBridge.ts", diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index 4167765928f..668e35f542f 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -4131,6 +4131,9 @@ "processFailed": "The filter could not run.", "replace": "Replace", "running": "Running filter…", + "sourceAdjustmentError": "Clarity, Texture or Dehaze failed on this layer, so it was not filtered. Change a setting or disable them, then try again.", + "sourceAdjustmentOverBudget": "Not enough canvas memory to apply this layer’s Clarity, Texture or Dehaze before filtering it. Delete layers you no longer need, then try again.", + "sourceAdjustmentTooLarge": "Clarity, Texture and Dehaze can’t run on this layer: its content is larger than 16.8 megapixels (4096 × 4096 px). Disable or remove them, or crop the layer, then try again.", "sourceOverBudget": "Not enough memory is free to read this layer for the filter.", "statusCommitting": "Applying filter…", "statusError": "Filter needs attention.", @@ -4148,6 +4151,9 @@ "errorInvalid": "Add an object prompt, point, or bounding box before processing.", "errorLocked": "Select Object cannot finish while canvas editing is locked.", "errorNoOutput": "Select Object finished without producing an image.", + "errorAdjustment": "Clarity, Texture or Dehaze failed on this layer, so it was not used as the source. Change a setting or disable them, then try again.", + "errorAdjustmentOverBudget": "Not enough canvas memory to apply this layer’s Clarity, Texture or Dehaze to the source. Delete layers you no longer need, then try again.", + "errorAdjustmentTooLarge": "Clarity, Texture and Dehaze can’t run on this layer: its content is larger than 16.8 megapixels (4096 × 4096 px). Disable or remove them, or crop the layer, then try again.", "errorNotReady": "The layer source is not ready. Try again when it finishes loading.", "errorOutputDimension": "The Select Object result dimensions do not match the layer source.", "errorQueue": "Select Object processing failed on the backend.", @@ -4226,7 +4232,9 @@ }, "addLayer": "Add layer", "adjustments": { + "amount": "Amount", "brightness": "Brightness", + "clarity": "Clarity", "channel": "Channel", "channels": { "rgb": "RGB", @@ -4236,6 +4244,8 @@ }, "contrast": "Contrast", "curves": "Curves", + "dehaze": "Dehaze", + "dehazeHint": "An aesthetic contrast and color adjustment, not physical haze removal.", "exposure": "Exposure", "curvesHint": "Drag points to bend the curve. Double-click empty space to add a point; double-click a point to remove it.", "levels": "Levels", @@ -4269,7 +4279,22 @@ }, "noiseSize": "Size", "newNoisePattern": "New pattern", + "refusal": { + "error": "Clarity, Texture or Dehaze failed on “{{name}}”. Change a setting or disable them, then try again.", + "overBudget": "Not enough canvas memory to apply Clarity, Texture or Dehaze to “{{name}}”. Delete layers you no longer need, then try again.", + "tooLarge": "Clarity, Texture and Dehaze can’t run on “{{name}}”: its content is larger than 16.8 megapixels (4096 × 4096 px). Disable or remove them, or crop the content, then try again." + }, "saturation": "Saturation", + "size": "Size", + "sizeHintGroup": "In canvas pixels.", + "sizeHintLayer": "In the layer’s own pixels, before it is scaled.", + "status": { + "computing": "Applying Clarity, Texture or Dehaze…", + "error": "Clarity, Texture or Dehaze failed, so the canvas leaves them out, and generating or exporting with them is refused. Change a setting to try again, or disable them.", + "overBudget": "Not enough canvas memory for Clarity, Texture or Dehaze, so the canvas leaves them out, and generating or exporting with them is refused. Delete layers you no longer need, then change a setting to try again.", + "tooLarge": "Clarity, Texture and Dehaze can’t run on content larger than 16.8 megapixels (4096 × 4096 px), so the canvas leaves them out, and generating or exporting with them is refused. Disable or remove them, or crop the content." + }, + "texture": "Texture", "transparencyLock": "Lock transparency" }, "blendModes": { diff --git a/invokeai/frontend/webv2/src/features/generation/core/colorConversion.ts b/invokeai/frontend/webv2/src/features/generation/core/colorConversion.ts deleted file mode 100644 index b45d16d44a3..00000000000 --- a/invokeai/frontend/webv2/src/features/generation/core/colorConversion.ts +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -export type RgbImage = { - readonly width: number; - readonly height: number; - /** Interleaved byte-valued red, green, blue samples in row-major pixel order. */ - readonly data: Uint8Array; -}; - -export type OklabImage = { - readonly width: number; - readonly height: number; - /** Interleaved lightness, a, b samples in row-major pixel order. */ - readonly data: Float64Array; -}; - -const decodeSrgb = (channel: number): number => { - const normalized = channel / 255; - return normalized <= 0.04045 ? normalized / 12.92 : ((normalized + 0.055) / 1.055) ** 2.4; -}; - -/** Converts interleaved byte-valued sRGB pixels to compact interleaved Oklab samples. */ -export const oklabFromSrgb = (rgb: RgbImage): OklabImage => { - const { width, height, data: input } = rgb; - if ( - !Number.isInteger(width) || - width < 1 || - !Number.isInteger(height) || - height < 1 || - !(input instanceof Uint8Array) || - input.length !== width * height * 3 - ) { - throw new Error('rgb must be a non-empty image with three interleaved byte channels per pixel'); - } - - const data = new Float64Array(input.length); - for (let i = 0; i < input.length; i += 3) { - const r = decodeSrgb(input[i]); - const g = decodeSrgb(input[i + 1]); - const b = decodeSrgb(input[i + 2]); - const l = Math.cbrt(0.4122214708 * r + 0.5363325363 * g + 0.0514459929 * b); - const m = Math.cbrt(0.2119034982 * r + 0.6806995451 * g + 0.1073969566 * b); - const s = Math.cbrt(0.0883024619 * r + 0.2817188376 * g + 0.6299787005 * b); - data[i] = 0.2104542553 * l + 0.793617785 * m - 0.0040720468 * s; - data[i + 1] = 1.9779984951 * l - 2.428592205 * m + 0.4505937099 * s; - data[i + 2] = 0.0259040371 * l + 0.7827717662 * m - 0.808675766 * s; - } - return { width, height, data }; -}; diff --git a/invokeai/frontend/webv2/src/features/generation/core/guidedFilter.ts b/invokeai/frontend/webv2/src/features/generation/core/guidedFilter.ts deleted file mode 100644 index 980348dadc5..00000000000 --- a/invokeai/frontend/webv2/src/features/generation/core/guidedFilter.ts +++ /dev/null @@ -1,118 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -export type ScalarImage = { - readonly width: number; - readonly height: number; - readonly data: Float64Array; -}; - -const validateImage = (image: ScalarImage, name: string): void => { - if ( - !Number.isInteger(image.width) || - image.width < 1 || - !Number.isInteger(image.height) || - image.height < 1 || - !(image.data instanceof Float64Array) || - image.data.length !== image.width * image.height - ) { - throw new TypeError(`${name} must be a non-empty image with width * height Float64 samples`); - } - for (let i = 0; i < image.data.length; i++) { - if (!Number.isFinite(image.data[i])) { - throw new TypeError(`${name} must contain finite samples`); - } - } -}; - -const reflect101 = (index: number, length: number): number => { - if (length <= 1) { - return 0; - } - const period = 2 * length - 2; - const wrapped = ((index % period) + period) % period; - return wrapped < length ? wrapped : period - wrapped; -}; - -/** Computes a normalized square box mean with OpenCV BORDER_REFLECT_101 boundaries. */ -const boxMean = ( - first: Float64Array, - width: number, - height: number, - radius: number, - second?: Float64Array, - square = false -): Float64Array => { - const valueAt = (index: number) => - square ? first[index] * first[index] : second ? first[index] * second[index] : first[index]; - const windowSize = radius * 2 + 1; - const horizontal = new Float64Array(first.length); - const output = new Float64Array(first.length); - for (let y = 0; y < height; y++) { - const rowOffset = y * width; - let sum = 0; - for (let offset = -radius; offset <= radius; offset++) { - const index = rowOffset + reflect101(offset, width); - sum += valueAt(index); - } - for (let x = 0; x < width; x++) { - horizontal[rowOffset + x] = sum / windowSize; - const leaving = rowOffset + reflect101(x - radius, width); - const entering = rowOffset + reflect101(x + radius + 1, width); - sum -= valueAt(leaving); - sum += valueAt(entering); - } - } - for (let x = 0; x < width; x++) { - let sum = 0; - for (let offset = -radius; offset <= radius; offset++) { - const index = reflect101(offset, height) * width + x; - sum += horizontal[index]; - } - for (let y = 0; y < height; y++) { - const index = y * width + x; - output[index] = sum / windowSize; - sum -= horizontal[reflect101(y - radius, height) * width + x]; - sum += horizontal[reflect101(y + radius + 1, height) * width + x]; - } - } - return output; -}; - -/** Applies the classic single-channel guided filter to equally sized row-major images. */ -export const guidedFilter = (guide: ScalarImage, signal: ScalarImage, radius: number, epsilon: number): ScalarImage => { - validateImage(guide, 'guide'); - validateImage(signal, 'signal'); - if (guide.width !== signal.width || guide.height !== signal.height) { - throw new Error('guide and signal must have the same dimensions'); - } - if (!Number.isInteger(radius) || radius < 0) { - throw new Error('radius must be a non-negative integer'); - } - if (!Number.isFinite(epsilon) || epsilon <= 0) { - throw new Error('epsilon must be a finite positive number'); - } - const { width, height } = guide; - const count = guide.data.length; - if (radius === 0) { - return { width, height, data: signal.data.slice() }; - } - - const meanGuide = boxMean(guide.data, width, height, radius); - const meanSignal = boxMean(signal.data, width, height, radius); - const meanGuideSquared = boxMean(guide.data, width, height, radius, undefined, true); - const meanGuideSignal = boxMean(guide.data, width, height, radius, signal.data); - const a = new Float64Array(count); - const b = new Float64Array(count); - for (let i = 0; i < count; i++) { - const variance = Math.max(0, meanGuideSquared[i] - meanGuide[i] * meanGuide[i]); - const covariance = meanGuideSignal[i] - meanGuide[i] * meanSignal[i]; - a[i] = covariance / (variance + epsilon); - b[i] = meanSignal[i] - a[i] * meanGuide[i]; - } - const meanA = boxMean(a, width, height, radius); - const meanB = boxMean(b, width, height, radius); - const data = new Float64Array(count); - for (let i = 0; i < count; i++) { - data[i] = meanA[i] * guide.data[i] + meanB[i]; - } - return { width, height, data }; -}; diff --git a/invokeai/frontend/webv2/src/features/generation/core/multiscale.ts b/invokeai/frontend/webv2/src/features/generation/core/multiscale.ts deleted file mode 100644 index 0741036e287..00000000000 --- a/invokeai/frontend/webv2/src/features/generation/core/multiscale.ts +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -import { guidedFilter, type ScalarImage } from './guidedFilter'; - -/** Returns the detail band between two guided-filter scales of the same lightness image. */ -export const edgeAwareBand = ( - lightness: ScalarImage, - innerRadius: number, - outerRadius: number, - epsilon: number -): ScalarImage => { - if (!Number.isInteger(innerRadius) || innerRadius < 0) { - throw new Error('innerRadius must be a non-negative integer'); - } - if (!Number.isInteger(outerRadius) || outerRadius <= innerRadius) { - throw new Error('outerRadius must be greater than innerRadius'); - } - const inner = innerRadius === 0 ? lightness.data : guidedFilter(lightness, lightness, innerRadius, epsilon).data; - const outer = guidedFilter(lightness, lightness, outerRadius, epsilon).data; - const data = new Float64Array(lightness.data.length); - for (let i = 0; i < data.length; i++) { - data[i] = inner[i] - outer[i]; - } - return { width: lightness.width, height: lightness.height, data }; -}; diff --git a/invokeai/frontend/webv2/src/features/generation/core/sharedImageManipulationNumerics.test.ts b/invokeai/frontend/webv2/src/features/generation/core/sharedImageManipulationNumerics.test.ts deleted file mode 100644 index baa67ada400..00000000000 --- a/invokeai/frontend/webv2/src/features/generation/core/sharedImageManipulationNumerics.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import fixtures from './__fixtures__/shared_image_manipulation_numerics.json'; -import { oklabFromSrgb } from './colorConversion'; -import { guidedFilter, type ScalarImage } from './guidedFilter'; -import { edgeAwareBand } from './multiscale'; - -const scalarImage = (rows: number[][]): ScalarImage => ({ - width: rows[0].length, - height: rows.length, - data: new Float64Array(rows.flat()), -}); - -const rgbImage = (rows: number[][][]): { width: number; height: number; data: Uint8Array } => ({ - width: rows[0].length, - height: rows.length, - data: new Uint8Array(rows.flat(2)), -}); - -/** Compare compact buffers against independently derived float64 references, with no relative tolerance. */ -const expectImageClose = (actual: ScalarImage, expected: number[][], atol: number) => { - expect(actual.width).toBe(expected[0].length); - expect(actual.height).toBe(expected.length); - expect(actual.data).toHaveLength(expected.length * expected[0].length); - expected.flat().forEach((value, i) => expect(Math.abs(actual.data[i] - value)).toBeLessThanOrEqual(atol)); -}; - -describe('shared image manipulation numerical fixtures', () => { - it('matches the independently derived guided-filter fixture and treats radius zero as identity', () => { - const { guide, signal, radius, epsilon, expected, atol } = fixtures.guided_filter; - expectImageClose(guidedFilter(scalarImage(guide), scalarImage(signal), radius, epsilon), expected, atol); - - const singleton = scalarImage([[0.375]]); - const result = guidedFilter(scalarImage([[1]]), singleton, 0, epsilon); - expect(Array.from(result.data)).toEqual([0.375]); - expect(result.data).not.toBe(singleton.data); - }); - - it('keeps a constant guide numerically constant for near-zero regularization', () => { - const guide = scalarImage(Array.from({ length: 3 }, () => Array(3).fill(0.1))); - const signal = scalarImage([ - [0, 1, 0], - [1, 0, 1], - [0, 1, 0], - ]); - const expected = [ - [40 / 81, 41 / 81, 40 / 81], - [41 / 81, 40 / 81, 41 / 81], - [40 / 81, 41 / 81, 40 / 81], - ]; - const actual = guidedFilter(guide, signal, 1, 1e-18); - - expect(Array.from(actual.data).every(Number.isFinite)).toBe(true); - expectImageClose(actual, expected, 1e-10); - }); - - it('matches the edge-aware band fixture where the fine scale has radius zero', () => { - const { lightness, inner_radius, outer_radius, epsilon, expected, atol } = fixtures.multiscale_band; - expectImageClose(edgeAwareBand(scalarImage(lightness), inner_radius, outer_radius, epsilon), expected, atol); - - expect(() => edgeAwareBand(scalarImage(lightness), 1, 1, epsilon)).toThrow(); - }); - - it('matches Oklab values for sRGB primaries and white, including black and channel endpoints', () => { - const { rgb, expected_oklab, atol } = fixtures.color_adapter; - const image = oklabFromSrgb(rgbImage(rgb)); - expect(image).toMatchObject({ width: expected_oklab[0].length, height: expected_oklab.length }); - expect(image.data).toHaveLength(expected_oklab.length * expected_oklab[0].length * 3); - expected_oklab.flat(2).forEach((value, i) => expect(Math.abs(image.data[i] - value)).toBeLessThanOrEqual(atol)); - - expect(Array.from(oklabFromSrgb({ width: 1, height: 1, data: new Uint8Array([0, 0, 0]) }).data)).toEqual([0, 0, 0]); - }); -}); - -describe('compact image buffer contract', () => { - it('uses row-major scalar buffers and preserves radius-zero identity values', () => { - const input = { width: 2, height: 2, data: new Float64Array([0.1, 0.2, 0.3, 0.4]) }; - const result = guidedFilter(input, input, 0, 0.1); - - expect(result).toMatchObject({ width: 2, height: 2 }); - expect(result.data).toBeInstanceOf(Float64Array); - expect(Array.from(result.data)).toEqual(Array.from(input.data)); - expect(result.data).not.toBe(input.data); - }); - - it('stores RGB channels interleaved in row-major order and returns compact Oklab data', () => { - const rgb = { width: 2, height: 1, data: new Uint8Array([255, 0, 0, 0, 255, 0]) }; - const result = oklabFromSrgb(rgb); - - expect(result).toMatchObject({ width: 2, height: 1 }); - expect(result.data).toBeInstanceOf(Float64Array); - expect(result.data).toHaveLength(6); - expect(Math.abs(result.data[0] - 0.62795536)).toBeLessThanOrEqual(0.00002); - expect(Math.abs(result.data[3] - 0.86643961)).toBeLessThanOrEqual(0.00002); - }); - - it('returns compact bands and rejects malformed buffer dimensions', () => { - const lightness = { width: 2, height: 2, data: new Float64Array([0.1, 0.2, 0.3, 0.4]) }; - const result = edgeAwareBand(lightness, 0, 1, 0.1); - expect(result.data).toBeInstanceOf(Float64Array); - expect(result.data).toHaveLength(4); - expect(() => guidedFilter({ ...lightness, data: new Float64Array(3) }, lightness, 1, 0.1)).toThrow(); - expect(() => oklabFromSrgb({ width: 2, height: 1, data: new Uint8Array(5) })).toThrow(); - }); -}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/ARCHITECTURE.md b/invokeai/frontend/webv2/src/workbench/canvas-engine/ARCHITECTURE.md index 8945fe6fc77..63f9e74342f 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/ARCHITECTURE.md +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/ARCHITECTURE.md @@ -63,7 +63,7 @@ Controllers talk through constructor-injected ports or public capabilities. They Current extracted ownership includes: -- `RasterController`: base layer caches, decoded bitmap leases, derived, adjusted and group surfaces, rasterization jobs, invalidation, residency pins, working-set eviction, unified raster-memory controller. +- `RasterController`: base layer caches, decoded bitmap leases, derived, adjusted and group surfaces, the spatial adjustment worker service, rasterization jobs, invalidation, residency pins, working-set eviction, unified raster-memory controller. - `RasterMemoryBudgetController`: base, derived, group, decoded, detached, reserved and overage byte accounting plus generation-scoped reservations and operation-owned reservations and pins. - `RenderController`: render targets, scheduler construction, previews, render lifecycle. - `HistoryController`: engine-owned history, guarded async replay with failure reporting, inactive byte trimming. @@ -142,11 +142,27 @@ Staged results use a separate compressed-Blob cache instead of retaining a decod Damage is explicit: `FrameDamage` is full repaint, no visible repaint, or layer-local regions. Scheduler widens to full on any invalidation without a region, a viewport change or a forced repaint. Compositor resolves regions through frame's matrices into one padded screen rect: regions entirely off target draw nothing; bounded repaint clips to the rect and culls every leaf, group scope and staged preview whose screen bounds miss it. Unknown layers or non-finite geometry repaint everything. -Compositing always renders through canonical `RasterBackend`; no approximate backend-free path. Filter previews and floating selection of a grouped member compose inside group's isolated surface, under group's opacity, blend and adjustments, exactly as member's committed pixels would. Committed content composes with same semantics for display, sampling and export; previews and floats are display-only. Layer's own adjustments apply to its layer-local pixels before resampling: export draws display's memoized adjusted copy of a leased live cache when one is current, never building one (a layer-sized copy would exceed the export's reservation), and otherwise adjusts only the layer region the output samples (plus one pixel of resampling margin). When a downscaled layer samples more pixels than output holds and its stack is position independent (color only), a detached copy of it is adjusted after resampling instead, keeping export work bounded by output size; composite reservations count each such scratch buffer at output size. A position-dependent stack (noise) never takes that shortcut: resampling must average its grain exactly as on screen, so its layer-local sampled region is adjusted and reserved even when larger than the output. Composite reservations plan that region from estimated content bounds; when a layer's rasterized pixels (measured text) sample more, rendering reserves the excess before allocating it and refuses as over-budget when it does not fit. World-space bakes (merges, boolean merges, crops, copies) and PSD always adjust the layer's own stack on a layer-local copy, or draw the display's current adjusted copy, and then resample it. Merge Down bakes both layers' own stacks the same way before the upper layer resamples into the lower layer's space, so the merged layer carries no stack; its undo snapshots stay raw and undo restores both layers with their stacks. Rasterize bakes a position-dependent stack (all of it) into the layer-local source render before the transform and drops it from the paint contract, since kept it would grain the baked pixel grid; a color-only stack stays on the contract. Thumbnails downscale the display's current adjusted copy when one exists. +Compositing always renders through canonical `RasterBackend`; no approximate backend-free path. Filter previews and floating selection of a grouped member compose inside group's isolated surface, under group's opacity, blend and adjustments, exactly as member's committed pixels would. Committed content composes with same semantics for display, sampling and export; previews and floats are display-only. Layer's own adjustments apply to its layer-local pixels before resampling: export draws display's memoized adjusted copy of a leased live cache when one is current and exact, never building one (a layer-sized copy would exceed the export's reservation), and otherwise adjusts only the layer region the output samples (plus one pixel of resampling margin). When a downscaled layer samples more pixels than output holds and its stack is position independent (color only), a detached copy of it is adjusted after resampling instead, keeping export work bounded by output size; composite reservations count each such scratch buffer at output size. A position-dependent stack (noise) never takes that shortcut: resampling must average its grain exactly as on screen, so its layer-local sampled region is adjusted and reserved even when larger than the output. Composite reservations plan that region from estimated content bounds; when a layer's rasterized pixels (measured text) sample more, rendering reserves the excess before allocating it and refuses as over-budget when it does not fit. World-space bakes (merges, boolean merges, crops, copies) and PSD always adjust the layer's own stack on a layer-local copy, or draw the display's exact adjusted copy, and then resample it. Merge Down bakes both layers' own stacks the same way before the upper layer resamples into the lower layer's space, so the merged layer carries no stack; its undo snapshots stay raw and undo restores both layers with their stacks. Control layers carry no stack, so a raster layer whose stack changes pixels reaches the control stack only baked: copy to control (`CopyLayerController`) and convert to control (`LayerMutationController.convertToControl`) draw the layer's exact baked export at an identity transform, conversion in place as one step whose undo restores the raster contract, its stack and its original cache; a layer whose stack changes nothing, or an empty layer with no pixels to bake, takes the plain contract clone of source and transform. Mask conversions read alpha alone, which no adjustment writes. Rasterize bakes a position-dependent or spatial stack (all of it) into the layer-local source render before the transform and drops it from the paint contract, since kept it would grain, or measure detail on, the baked pixel grid; a color-only stack stays on the contract. Thumbnails downscale the display's current adjusted copy when one exists. -Adjustments are position dependent but region independent. `applyAdjustments` takes the pixel of its ImageData's top-left in the stack's reference space; adjusting any sub-rectangle at its own origin yields exactly the pixels a whole-surface pass gives there, which damage refreshes, sampled regions and group-region repaints rely on. Color entries ignore position. Noise adds a field that is a pure function of seed, integer pixel coordinate and channel (a stable integer hash into an equiprobable quantile table, no sequential state), scaled per pixel by a function of that pixel alone: 1 for `flat` response, 4·Y′·(1 − Y′) of the incoming Rec. 709 luma for `midtones`. Size 1 adds the white samples directly. Larger sizes blur them with a separable Gaussian of σ √(size² − 1)/2 truncated at 3σ (R taps each side), renormalized so squared taps sum to one (σ unchanged); a kernel whose outer taps all stay under 10⁻³ (sizes under about 1.14) runs as size 1. Samples beyond the ImageData come from the hash, never from pixels, and each output sums the same float64 values in the same order wherever the region starts, so region independence stays exact. The blur streams rows through a ring of 2R + 1 horizontally filtered rows: transient JS heap of O(R × width) per channel, not counted in the raster budget. Cost is O(pixels × R × channels) on the main thread, paid once per changed preview frame while a slider scrubs. Monochrome shares one field across R, G, B; color draws three; chroma draws two, added to OKLab a and b with lightness kept, then converts back to sRGB clamped in linear light. A layer's own stack works in layer-local pixels (cache surface (0,0) is `entry.rect`'s origin, possibly negative) and is always applied before resampling when it contains noise, so display, generation, exports, bakes and PSD agree in grain pattern and strength, and the grain moves with the layer under transforms. A group stack works in document pixels: group surfaces, composite group buffers and PSD's baked group stacks all sit on the document grid at scale 1. The one exception is a floating selection: its display copy is baked at its lift origin and moves with the drag; committing recomputes the layer's grain at the new position. Thumbnails without a current display copy adjust their own downscaled pixels, shrinking grain to size × scale (at least 1) and scaling σ by min(1, scale × grain span) to approximate the averaging a downscale applies; span is (Σ taps)², 1 at size 1. +Per-pixel adjustments (every type except the spatial ones below) are position dependent but region independent. `applyAdjustments` takes the pixel of its ImageData's top-left in the stack's reference space; adjusting any sub-rectangle at its own origin yields exactly the pixels a whole-surface pass gives there, which damage refreshes, sampled regions and group-region repaints rely on. Color entries ignore position. Noise adds a field that is a pure function of seed, integer pixel coordinate and channel (a stable integer hash into an equiprobable quantile table, no sequential state), scaled per pixel by a function of that pixel alone: 1 for `flat` response, 4·Y′·(1 − Y′) of the incoming Rec. 709 luma for `midtones`. Size 1 adds the white samples directly. Larger sizes blur them with a separable Gaussian of σ √(size² − 1)/2 truncated at 3σ (R taps each side), renormalized so squared taps sum to one (σ unchanged); a kernel whose outer taps all stay under 10⁻³ (sizes under about 1.14) runs as size 1. Samples beyond the ImageData come from the hash, never from pixels, and each output sums the same float64 values in the same order wherever the region starts, so region independence stays exact. The blur streams rows through a ring of 2R + 1 horizontally filtered rows: transient JS heap of O(R × width) per channel, not counted in the raster budget. Cost is O(pixels × R × channels) on the main thread, paid once per changed preview frame while a slider scrubs. Monochrome shares one field across R, G, B; color draws three; chroma draws two, added to OKLab a and b with lightness kept, then converts back to sRGB clamped in linear light. A layer's own stack works in layer-local pixels (cache surface (0,0) is `entry.rect`'s origin, possibly negative) and is always applied before resampling when it contains noise, so display, generation, exports, bakes and PSD agree in grain pattern and strength, and the grain moves with the layer under transforms. A group stack works in document pixels: group surfaces, composite group buffers and PSD's baked group stacks all sit on the document grid at scale 1. The one exception is a floating selection: its display copy is baked at its lift origin and moves with the drag; committing recomputes the layer's grain at the new position. Thumbnails without a current display copy adjust their own downscaled pixels, shrinking grain to size × scale (at least 1) and scaling σ by min(1, scale × grain span) to approximate the averaging a downscale applies; span is (Σ taps)², 1 at size 1. -Group surface is keyed by everything except its members' pixels and float's placement (scope shape, member appearance and matrices, in-scope previews, float's surface). While that key and bounds hold, cache refreshes in place: it unions damage each member's cache recorded since the version it last drew, padded by one source pixel before member's matrix maps it (bilinear reach at any scale), plus old and new landing of a moved float, clears that region with one pixel of antialiasing margin and redraws every contributor there, then applies group's adjustments to the region alone, at the region's document origin. Unknown member damage, changed member set or new bounds redraw whole surface, reallocating only when its size changed; new key reuses least-recently-used of a group's two slots once both are taken. +### Spatial adjustments + +Clarity, texture and dehaze are spatial: an output pixel depends on OKLab lightness across a neighborhood of the owner's source, so they are neither per-pixel, region independent, nor resample-commuting. Their domain is the source's alpha bounds, computed inside the kernel (`render/spatial/evaluateSpatialSteps.ts`): hidden pixels borrow the nearest visible pixel's lightness, and nothing outside the bounds, no hidden pixel and no alpha is written, so any buffer holding all the content gives identical pixels. They therefore evaluate only on the owner's full source in owner pixel space (a layer's cache surface before its transform, a group's document-resolution composite), never on a sampled, cropped or downscaled region. `applyAdjustments` skips them; `toSpatialPlan` splits a stack into ordered runs, per-pixel runs applied on the main thread at the owner's origin and each run of consecutive spatial entries sent as one job to `SpatialAdjustmentService`, a module worker evaluating one job at a time. + +Display is asynchronous, run by one state machine per owner (`render/spatialDisplay.ts`) that layers (`render/adjustedSurfaceCache.ts`) and spatial group scopes (below) share. While a layer's job is pending its adjusted surface shows an interim: a parameter change keeps the shown pixels; a content edit with known damage re-patches that rect with the per-pixel part only; unknown damage, a new source or size, or a rebuild after eviction shows a whole per-pixel-only interim. A layer has at most one job in flight and a later target waits for it, so scrubbing reads the source once per landed result. A content change starts the next job after 120 ms of quiet, anything else as soon as the running job ends; a placeholder cache with no published pixels starts none. A result installs only onto the same source surface, from a version at or before the shown one whose later damage is known (re-patched per-pixel after the install), and only when newer than the result shown, so an older parameter's result can land as an interim while scrubbing. The copy is exact only when the installed result's version and key both match the current ones. `peek` returns exact copies only (export reuse, merges, control-pixel materialization, which refuses `not-ready` meanwhile); `peekDisplayed` returns whatever is shown (thumbnails). Installing calls the engine's settle handler, which recomposites the layer, steps its thumbnail display token and the pixel epoch (overview) and marks ancestor group surfaces for a full redraw, since no cache version changed; it never drops the surface that landed. A status leaving `computing` without a landing (failure, eviction) also redraws ancestor groups, which may be waiting on it. After `too-large`, `over-budget` or `error` the owner shows the target's per-pixel part only (repainted when that part differs from what is shown, then settled like a landing) and retries only after the target changes: `too-large` depends on the content's alpha bounds alone, so only new content retries it, never a parameter scrub; the others also retry on a parameter change. The keyed `adjustmentStatus` store (`interaction.getLayerAdjustmentStatus`) reports those and `computing`; absence means exact. Control-pixel materialization refuses `not-ready` while the owner computes and an `AdjustmentRefusal` naming it once it cannot run. A floating selection previews the per-pixel part only and committing re-evaluates the layer. Layer deletion, base-cache eviction and disposal abandon spatial work; the engine also drops the spatial state of layers and groups that no longer draw a spatial stack, drawn or not (`retainSpatial`, visiting only owners that hold such state), so no status outlives its stack. Cooldown abandons only the display's running and scheduled jobs, dropping their interims; exact copies and failures keep their state, and pixel-writing operations' jobs stay governed by their own signal, since their detached snapshots outlive cooldown. + +Each job reserves its 4 B/px pixel buffer as an operation-owned `spatial-adjustment` reservation until the buffer returns from the worker; `over-budget` refuses the job. Worker scratch (up to `SPATIAL_SCRATCH_BYTES_PER_PIXEL` per alpha-bounds pixel, at most `MAX_SPATIAL_PIXELS`, about 400 MB at the cap) stays outside the raster budget: the budget accounts surfaces resident in the page's heap, while scratch lives in the worker's, one job at a time per engine (each engine owns its service and worker), freed when the job ends or the idle worker terminates after 30 s. Counting it would evict display caches for memory the page never holds. + +Pixel-writing consumers evaluate spatial stacks exactly or refuse; none leaves a spatial entry out. `render/exactAdjustments.ts` applies a stack's per-pixel runs on the calling thread and sends its spatial runs to the shared service at `now` priority, one slot per job so consumers never supersede each other. `too-large`, `over-budget` or `error` becomes an `AdjustmentRefusal` naming the owner and cause, returned through each operation's result: generation (its notice), exports, layer bakes and everything built on them (filter and Select Object sources, crop, copy, boolean merges, Run Workflow, clipboard and board saves), Merge Down, Visible and Selected, Rasterize, conversion to a control layer, PSD and extraction. A consumer reuses a layer's display copy only when exact; groups have no exact peek, so a spatial group scope always runs its own job. + +The raster composite (generation, export, extraction) evaluates a spatial leaf on all its leased pixels in layer-local space, then resamples it, never through the sampled-region or post-resample paths. A spatial group scope leases its members first, composites them over the union of their actual bounds (the display's group surface domain), unclipped by the bbox, evaluates its stack there in document pixels and lands at its offset; nested scopes draw into that frame. Pixels inside a bbox therefore do not depend on the bbox. Composite reservations count a spatial leaf's whole planned content and a spatial scope's planned member extent, each with its ImageData; the job's buffer is the service's reservation, and larger actual extents (measured text) reserve the excess while rendering. Extraction composites its leased contributors through this path instead of the display's providers, so it never reads an interim or takes a group slot. + +Layer bakes (`RasterExportController`) adjust the whole layer-local copy and then recheck the lease guard, since a job yields. Merge Down and Rasterize are asynchronous: the transaction stays open across the job and lands only if the edit revision, the caches and their versions, and any display copy drawn are unchanged, otherwise `not-ready`. Merge Visible evaluates a spatial group stack on its merged-rect buffer, which holds every member, and lands only while every contributor's guard holds. Colour sampling, the overview and thumbnails mirror the display and may show an interim. + +Group surface is keyed by everything except its members' pixels and float's placement (scope shape, member appearance and matrices, in-scope previews, float's surface). While that key and bounds hold, cache refreshes in place: it unions damage each member's cache recorded since the version it last drew, padded by one source pixel before member's matrix maps it (bilinear reach at any scale), plus old and new landing of a moved float, clears that region with one pixel of antialiasing margin and redraws every contributor there, then applies group's adjustments to the region alone, at the region's document origin. Unknown member damage, a member's landed spatial result (`invalidate`), changed member set or new bounds redraw whole surface, reallocating only when its size changed; new key reuses least-recently-used of a group's two slots once both are taken. + +A scope whose own stack is spatial keeps its slot surface raw (its own stack key stays out of its shape key, so a parameter edit repaints nothing; a parent still keys it as a child). Each raw repaint bumps the slot's raw version and records its surface-local damage: unknown for a whole rebuild, and only member content damage for an `invalidate` redraw, so the adjusted copy keeps its pixels until the group's own result lands. It tracks each member's and nested group's look (stack, opacity, blend, matrix) apart from its shape: a look change repaints that look's old and new bounds in a same-shaped slot with the same bounds. A spatial stack edited into another spatial stack is redrawn only in the raw composite, not patched in the adjusted copy, since its shown pixels stay until it lands. A move that changes the group's bounds repaints the whole slot. A same-size adjusted surface, counted in group bytes and evicted with its slot, follows the layer rules on that raw composite: the domain is the document-resolution union of member bounds, whose transparent margin the kernel's alpha-bounds domain ignores, and per-pixel runs work at the slot's document origin. Each slot runs its own jobs; the group reports, under its id in `adjustmentStatus`, the status of the slot the canvas frame read last (else of its most recently read slot), so reads by the overview or picking under other keys never flip its row or redraw its ancestors. While a filter preview, float or transform session gives the frame and the overview different keys, each runs its own job on its own inputs: a known cost, not a correctness issue. A raw repaint trail longer than 16 entries folds its oldest entries together, so damage since an older version stays known, only coarser. A job waits (reporting `computing`) while any member's or nested group's displayed pixels are still pending; a member that cannot finish does not block it. Every read returns `state`, `exact`, `pending` or `inexact`, combining the group's own state with its inputs'. The group's settle handler steps the pixel epoch and redraws ancestor groups and the frame, never the landed group. The engine drops the spatial slots of groups that no longer draw a spatial stack (`retainSpatial`), clearing their status. Overlay renders in CSS pixels. Receives document→CSS view, CSS viewport size and device-pixel ratio, clears backing store in device pixels, then draws every handle, nub, dash and cursor under a DPR base transform (nested image and path transforms compose with it), so chrome keeps its CSS size at any ratio and matches CSS-pixel hit tests. Viewport owns the ratio (capped at `MAX_DPR`, sub-one ratios kept); backing-store size and composite view matrix both derive from it. @@ -154,7 +170,7 @@ Color picker shows a loupe of pixels under the pointer. Anchors to pipeline's la ## Derived surfaces and invalidation -Every display-only pixel effect uses `DerivedSurfaceCache`. Slot identified by layer ID and effect kind, guarded by source surface identity, monotonic source version, and a deterministic parameter key. Reuse safe only when every guard matches. So source replacement cannot publish or reuse a surface produced for an older preview. +Every display-only pixel effect uses `DerivedSurfaceCache`. Slot identified by layer ID and effect kind, guarded by source surface identity, monotonic source version, and a deterministic parameter key. Reuse safe only when every guard matches. So source replacement cannot publish or reuse a surface produced for an older preview. A spatial stack's adjusted surface also records whether its pixels are exact or an interim (see Spatial adjustments). Invalidate only affected effect when parameters change. Delete every derived slot when a layer is removed, replaced, or its base cache evicted. Frame-demand culling happens before derived lookup or construction. @@ -188,9 +204,11 @@ Undo removes accepted layer and restores prior selection. Redo restores identica PSD export is a planner/executor transaction over immutable snapshots. `PsdExportController` captures one `CanvasDocumentSnapshot`, walks raster tree into planner inputs (groups as pass-through folders, exportable leaves with their captured content rects), then obtains a `CanvasRasterSnapshot` including disabled layers. `planPsdExport` reverses every level of top-first tree into ag-psd's bottom-to-top order, writes each node's own visibility flag (disabled group is a hidden folder), flattens merged preview from the leaves that contribute (enabled with every ancestor enabled), which is what Photoshop renders for the same flags. Folders whose subtree exports nothing are dropped. Execution reads only detached surfaces; cannot mix live layer caches from different document generations. +Each leaf bakes its own stack exactly (spatial entries included), layer-local, before the transform. PSD has no group adjustments, so each enclosing group's stack, innermost first and named per group for refusals, is baked into each leaf's world pixels: under member opacity and blending, and for spatial entries, which see that leaf alone instead of the group composite, this approximates the canvas. + Controller keeps 30,000-pixel side cap and also derives a pixel-area limit from currently available reserved bytes. Accounts for flattened composite plus each transformed layer surface, budgets eight bytes per required pixel, reserves full execution allocation before creating PSD surfaces. Final freshness check happens before that reservation; once execution starts it uses only frozen pixels. Capture and execution reservations release in `finally`; cancellation aborts before next allocation or side effect. -Capability returns `exported`, `nothing`, `too-large`, `not-ready`, `over-budget`, `stale`, or `aborted`. Layers widget awaits the promise and maps every non-success result to a user-visible notice. +Capability returns `exported`, `nothing`, `too-large`, `not-ready`, `over-budget`, `stale`, `aborted`, or an `AdjustmentRefusal`. Layers widget awaits the promise and maps every non-success result to a user-visible notice. ## Input ownership and frame scheduling @@ -200,7 +218,7 @@ Pointer pipeline (`input/pointerPipeline.ts`) owns gesture capture and temporary ## Lifecycle -Lifecycle states: `active`, `cooling`, `cool`, `disposed`. Losing final registry reference immediately banks a live floating selection, detaches input/render targets, cancels background PSD work and rasterization, releases old generation's reservations, starts a dirty-pixel flush. Successful, still-current flushes release every derived and group surface and each base cache that no read lease pins and no session holds, then trim history. Failed flushes keep dirty pixels and stay retryable; registry keeps the engine and retries after another grace period, disposing it only after a `cooled` result. Reacquisition increments lifecycle generation so late flush or grace-period work cannot clean up or dispose a live engine. Operation-owned reservations and pins survive these generation transitions until their owning operation settles. +Lifecycle states: `active`, `cooling`, `cool`, `disposed`. Losing final registry reference immediately banks a live floating selection, detaches input/render targets, cancels background PSD work, rasterization and the display's spatial adjustment jobs, releases old generation's reservations, starts a dirty-pixel flush. Successful, still-current flushes release every derived and group surface and each base cache that no read lease pins and no session holds, then trim history. Failed flushes keep dirty pixels and stay retryable; registry keeps the engine and retries after another grace period, disposing it only after a `cooled` result. Reacquisition increments lifecycle generation so late flush or grace-period work cannot clean up or dispose a live engine. Operation-owned reservations and pins survive these generation transitions until their owning operation settles. Operations that write an open project's document elsewhere (close, export, duplicate) first cross its engine's paint barrier through open-project handle and Workbench's `LiveCanvasEngines` registry; close crosses it again after the push so pixels painted meanwhile are not lost. These barriers do not wait on an edit that holds dirty pixels (session, gesture or suspension): they fail fast and ask user to finish or cancel it, and if unsaved pixels cannot be persisted the project stays open with its engine state intact. Rename keeps its tolerant flush. Invocation waits for short-lived holds. Disposal cancels outstanding work and releases every resource category: caches, decoded bitmaps, reservations, pins, outstanding snapshots. diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/applicationHost.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/applicationHost.ts index 593ede1aa32..72fecad528e 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/applicationHost.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/applicationHost.ts @@ -16,6 +16,7 @@ import type { CommitMaskImageResultOptions, } from '@workbench/canvas-engine/controllers/maskResultController'; import type { CanvasCompositeExecutorDeps } from '@workbench/canvas-engine/rasterTransactions'; +import type { AdjustmentRefusal } from '@workbench/canvas-engine/render/exactAdjustments'; import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { SamInteractionState, SamVisualInput } from '@workbench/canvas-engine/samInteraction'; import type { Rect } from '@workbench/canvas-engine/types'; @@ -48,6 +49,7 @@ export interface CanvasApplicationHost { ): Promise< | { status: 'ok'; surface: RasterSurface; rect: Rect; guard: LayerExportGuard; release(): void } | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal >; getCompositeExecutorDeps(): CanvasCompositeExecutorDeps; getDocument(): CanvasDocumentContractV3 | null; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/capabilities.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/capabilities.ts index 6e13dcf1537..97cc9ec71d4 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/capabilities.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/capabilities.ts @@ -12,9 +12,10 @@ import type { StrokeCommittedEvent } from '@workbench/canvas-engine/tools/tool'; import type { LayerTransform } from '@workbench/canvas-engine/transform/transformMath'; import type { BooleanRasterResult } from './controllers/booleanMergeController'; -import type { CopyLayerToRasterResult } from './controllers/copyLayerController'; +import type { CopyLayerResult } from './controllers/copyLayerController'; import type { CropLayerResult } from './controllers/cropLayerController'; import type { ExtractMaskedAreaResult } from './controllers/extractMaskedAreaController'; +import type { ConvertLayerToControlResult } from './controllers/layerMutationController'; import type { MergeDownResult, MergeVisibleResult } from './controllers/mergeLayerController'; import type { NewRasterLayerResult } from './controllers/newRasterLayerController'; import type { RasterizeLayerResult } from './controllers/rasterizeLayerController'; @@ -34,6 +35,7 @@ import type { EraserOptions, GradientToolOptions, LassoToolOptions, + LayerAdjustmentStatus, LayerQuickSwitchState, LayerThumbnailStatus, MarqueeToolOptions, @@ -49,6 +51,7 @@ import type { CanvasFontReplacementTarget, } from './fontReferences'; import type { CanvasLayerPreviewMutation, CanvasProjectMutation } from './mutationContracts'; +import type { AdjustmentRefusal } from './render/exactAdjustments'; import type { Rect, ToolId, Vec2 } from './types'; import type { Viewport } from './viewport'; @@ -178,6 +181,9 @@ export interface CanvasInteractionStateCapability { getLayerThumbnailVersion(layerId: string): number | undefined; subscribeLayerThumbnailStatus(layerId: string, listener: () => void): () => void; subscribeLayerThumbnailVersion(layerId: string, listener: () => void): () => void; + /** Whether the canvas shows a layer's or raster group's spatial adjustments exactly (`idle`), or why not. */ + getLayerAdjustmentStatus(layerId: string): LayerAdjustmentStatus | 'idle'; + subscribeLayerAdjustmentStatus(layerId: string, listener: () => void): () => void; } /** @@ -234,6 +240,7 @@ export interface CanvasDocumentSnapshot { } export type PsdExportResult = + | AdjustmentRefusal | 'exported' | 'nothing' | 'too-large' @@ -281,11 +288,13 @@ export interface CanvasHistoryCapability { stepBy(offset: number): Promise; } -/** Why the engine refused an edit the user started without a caller to report to (a stroke, a shape). */ -export type CanvasEditRefusal = SubsetOf< - CanvasTransactionOutcome, - 'busy' | 'gesture-active' | 'not-ready' | 'over-budget' ->; +/** + * Why the engine refused an edit the user started without a caller to report to (a stroke, a shape). An + * `AdjustmentRefusal` names a layer whose spatial adjustments cannot run, so its pixels cannot be baked for painting. + */ +export type CanvasEditRefusal = + | SubsetOf + | AdjustmentRefusal; export type LayerThumbnailRequestResult = | 'ready' @@ -337,7 +346,8 @@ export type ExportBakedLayerBlobResult = | 'disabled' | 'empty' | 'over-budget'; - }; + } + | AdjustmentRefusal; export interface CanvasSelectionCapability { deselect(): void; @@ -515,7 +525,8 @@ export type DuplicateLayersResult = */ export type { BooleanRasterResult, - CopyLayerToRasterResult, + ConvertLayerToControlResult, + CopyLayerResult, CropLayerResult, ExtractMaskedAreaResult, MergeDownResult, @@ -594,17 +605,29 @@ export interface CanvasEngineLayerCapability extends CanvasLayerCapability { commitRasterFilterResult(options: CommitRasterFilterOptions): Promise; /** `null` when there was nothing to commit: an empty creation or an unchanged edit. */ commitTextEdit(content: string, styleChanges?: Partial): StructuralCommitResult | null; - copyLayerToRaster(layerId: string): Promise; + /** + * Converts a raster image or paint layer to a control layer in place, baking its own adjustments (exactly, or + * refused) and transform into paint pixels; undo restores the raster layer with its adjustments and pixels. An empty + * layer converts as a plain clone of its source and transform. A layer whose stack changes nothing can use the + * cheaper `commitLayerConversion`, which keeps its source and transform. + */ + convertLayerToControl(label: string, expectedLiveLayer: CanvasLayerContract): Promise; + /** + * Adds a control-layer copy holding the layer's baked pixels, as `copyLayerToRaster` does for raster copies; an empty + * raster layer's copy is a plain clone of its source and transform. + */ + copyLayerToControl(layerId: string): Promise; + copyLayerToRaster(layerId: string): Promise; cropLayerToBbox(layerId: string): Promise; duplicateLayers(layerIds: readonly string[]): Promise; - mergeLayerDown(upperLayerId: string): MergeDownResult; + mergeLayerDown(upperLayerId: string): Promise; mergeSelectedRasterLayers(layerIds: readonly string[]): Promise; mergeVisibleRasterLayers(): Promise; /** `dispatch-rejected` also covers a selection with nothing eligible to move. */ nudgeSelectedLayer(dx: number, dy: number): StructuralCommitResult; openTextCreate(docPoint: Vec2): void; openTextEdit(layerId: string): void; - rasterizeLayer(layerId: string): RasterizeLayerResult; + rasterizeLayer(layerId: string): Promise; setTextEditContentReader(reader: (() => string) | null): void; updateTextEditStyle(patch: TextStylePatch): void; updateTransformSession(transform: LayerTransform): void; @@ -666,6 +689,7 @@ export type { GradientStop, GradientToolOptions, LassoToolOptions, + LayerAdjustmentStatus, LayerQuickSwitchState, LayerThumbnailStatus, MarqueeToolOptions, @@ -693,7 +717,8 @@ export { export type { LayerTransform } from './transform/transformMath'; export type { ImageResolver } from './render/rasterizers'; export type { Rect, SelectionOp, ToolId, Vec2 } from './types'; -export { adjustmentsKey, buildCurveLut } from './render/adjustments'; +export { adjustmentsKey, buildCurveLut, isIdentityAdjustments } from './render/adjustments'; +export { isAdjustmentRefusal, type AdjustmentRefusal, type AdjustmentRefusalReason } from './render/exactAdjustments'; export { DEFAULT_CHECKER_COLORS } from './render/compositor'; export { getBaseRasterContentBounds, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts index bf3f3d8fffd..fcc3a86471d 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts @@ -266,6 +266,29 @@ export type CanvasAdjustmentEntry = * reference space (layer-local pixels for a layer's own stack, document pixels for a group's) and the channel. */ seed: number; + }) + /** + * Spatial entries match the backend's Clarity, Texture and Contrast Dehaze nodes. Each reads a neighborhood of + * OKLab lightness across the owner's whole surface (layer-local pixels for a layer's stack, document pixels for a + * group's) within its alpha bounds, so unlike the entries above it is neither per-pixel nor region independent. + * `amount` is a percentage, −200 to 200 (0 is identity); out-of-range values are clamped when evaluated. + */ + | (CanvasAdjustmentEntryBase & { + /** Local contrast of mid-size detail, midtones weighted. `size`: feature size in pixels, 4–1000. */ + type: 'clarity'; + amount: number; + size: number; + }) + | (CanvasAdjustmentEntryBase & { + /** Local contrast of fine detail; positive amounts spare noise-level flat areas. `size`: pixels, 1–100. */ + type: 'texture'; + amount: number; + size: number; + }) + | (CanvasAdjustmentEntryBase & { + /** Removes (positive) or adds (negative) haze; its radius follows the alpha bounds' shorter side. */ + type: 'dehaze'; + amount: number; }); /** A raster layer's ordered adjustment stack, applied top to bottom. */ @@ -423,10 +446,10 @@ export const CANVAS_MAX_NODE_COUNT = 10_000; export interface CanvasDocumentContractV3 { /** - * Schema this document needs: 3 for plain content, 4 for custom-font typography, 5 for noise adjustments. Each - * document mutation raises it to the newer of its content's need and the canvas state's version. A restored - * snapshot's document keeps its own, possibly lower, version until the next mutation; the state's version and the - * project's compatibility floor stay put meanwhile. + * Schema this document needs: 3 for plain content, 4 for custom-font typography, 5 for noise or spatial + * (clarity, texture, dehaze) adjustments. Each document mutation raises it to the newer of its content's need and + * the canvas state's version. A restored snapshot's document keeps its own, possibly lower, version until the next + * mutation; the state's version and the project's compatibility floor stay put meanwhile. */ version: 3 | 4 | 5; width: number; @@ -452,7 +475,8 @@ export interface CanvasStagingAreaContractV2 extends CanvasStagingAreaContract { export interface CanvasStateContractV3 { /** * Newest schema the state needs: the newest of its previous version, the live document's and every snapshot's - * (3 plain, 4 custom-font typography, 5 noise adjustments). Sticky: removing that content never lowers it. + * (3 plain, 4 custom-font typography, 5 noise or spatial adjustments). Sticky: removing that content never lowers + * it. */ version: 3 | 4 | 5; document: CanvasDocumentContractV3; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/bakedControlLayer.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/bakedControlLayer.browser.test.ts new file mode 100644 index 00000000000..5b4d15cd5c4 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/bakedControlLayer.browser.test.ts @@ -0,0 +1,266 @@ +/** Real-canvas checks that copying or converting a raster layer to a control layer keeps its displayed pixels. */ + +import type { + CanvasAdjustmentsContract, + CanvasDocumentContractV3, + CanvasRasterLayerContractV2, +} from '@workbench/canvas-engine/contracts'; +import type { SpatialRunner } from '@workbench/canvas-engine/render/exactAdjustments'; +import type { Rect } from '@workbench/canvas-engine/types'; + +import { areJsonValuesStructurallyEqual } from '@platform/core/json'; +import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; +import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { getDocumentLayer } from '@workbench/canvas-engine/document/documentIndex'; +import { fromTRS } from '@workbench/canvas-engine/math/mat2d'; +import { createAdjustedSurfaceCache } from '@workbench/canvas-engine/render/adjustedSurfaceCache'; +import { createDomRasterBackend } from '@workbench/canvas-engine/render/raster'; +import { createSpatialAdjustmentService } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import { createDefaultControlAdapter } from '@workbench/controlAdapters'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { CopyLayerController, type CopyLayerControllerOptions } from './copyLayerController'; +import { LayerMutationController } from './layerMutationController'; +import { createLayerOperationHarness } from './layerOperationHarness.testing'; +import { RasterExportController } from './rasterExportController'; + +const ADJUSTMENTS: CanvasAdjustmentsContract = [ + { amount: 90, id: 'clarity', isEnabled: true, size: 8, type: 'clarity' }, + { brightness: 0.1, contrast: 0.2, id: 'bc', isEnabled: true, type: 'brightness-contrast' }, + { amount: 50, id: 'dehaze', isEnabled: true, type: 'dehaze' }, +]; +const TRANSFORM = { rotation: Math.PI / 9, scaleX: 0.6, scaleY: 1.3, x: 11, y: 5 }; +const CONTENT: Rect = { height: 30, width: 40, x: -6, y: -4 }; +const CONTROL_MODEL = { base: 'sdxl', model: 'union-model' } as const; + +const LAYER: CanvasRasterLayerContractV2 = { + adjustments: ADJUSTMENTS, + blendMode: 'multiply', + id: 'layer', + isEnabled: false, + isLocked: false, + name: 'Layer', + opacity: 0.7, + source: { bitmap: { height: 30, imageName: 'layer.png', width: 40 }, offset: { x: -6, y: -4 }, type: 'paint' }, + transform: TRANSFORM, + type: 'raster', +}; + +const services: { dispose(): void }[] = []; +afterEach(() => { + for (const service of services.splice(0)) { + service.dispose(); + } +}); + +/** A rotated, scaled raster layer with a spatial stack, its controllers over a real export bake, and its display. */ +const scene = (runSpatial?: SpatialRunner) => { + const backend = createDomRasterBackend(); + const document: CanvasDocumentContractV3 = { + background: 'transparent', + bbox: { height: 64, width: 64, x: 0, y: 0 }, + height: 64, + selectedLayerId: 'layer', + stacks: stacksFrom([LAYER]), + version: 3, + width: 64, + }; + const h = createLayerOperationHarness(document, { backend }); + const entry = h.layers.getOrCreateRect('layer', CONTENT); + for (let x = 0; x < CONTENT.width; x += 1) { + entry.surface.ctx.fillStyle = `rgba(${60 + x * 3}, ${x % 5 === 0 ? 20 : 128}, ${200 - x * 3}, ${x % 7 ? 1 : 0.5})`; + entry.surface.ctx.fillRect(x, 2, 1, 26); + } + h.layers.publishPixels('layer'); + const pixelsOf = (layerId: string): number[] => { + const cached = h.layers.get(layerId)!; + return [...cached.surface.ctx.getImageData(0, 0, cached.rect.width, cached.rect.height).data]; + }; + const raw = pixelsOf('layer'); + + const memory = new RasterMemoryBudgetController({ budgetBytes: 64 * 1024 * 1024 }); + const service = createSpatialAdjustmentService({ + reserve: (bytes) => memory.reserveOperation(bytes, { purpose: 'spatial-adjustment' }), + }); + services.push(service); + const run: SpatialRunner = runSpatial ?? ((request) => service.run(request)); + const isGuardCurrent = (guard: { layer: unknown; layerId: string; cacheVersion: number }) => + getDocumentLayer(h.document(), guard.layerId) === guard.layer && + h.layers.get(guard.layerId)?.version === guard.cacheVersion; + const exporter = new RasterExportController({ + backend, + captureGuard: (layer, cached) => ({ + cacheVersion: cached.version, + documentGeneration: 1, + layer, + layerId: layer.id, + projectId: 'p', + }), + getDocument: () => h.document(), + getOrStartRasterization: () => Promise.resolve('published'), + isGuardCurrent, + isRasterizing: () => false, + isSupportedSource: () => true, + layers: h.layers, + pin: () => ({ release: () => undefined }), + runSpatial: run, + }); + const exportBaked: CopyLayerControllerOptions['exportBaked'] = async (layerId) => { + const result = await exporter.baked(layerId, { includeDisabled: true }); + if (result.status === 'ok' || result.status === 'adjustment-refused') { + return result; + } + return { status: result.status === 'aborted' ? 'not-ready' : result.status }; + }; + const copy = new CopyLayerController({ + backend, + commitLayerCopy: () => { + throw new Error('a layer with pixels is never copied as a plain clone'); + }, + ctx: h.ctx, + exportBaked, + getControlModel: () => CONTROL_MODEL, + isGuardCurrent, + }); + const unused = () => { + throw new Error('not used by conversion'); + }; + const mutation = new LayerMutationController({ + backend, + captureCache: (layer) => { + const cached = h.layers.get(layer.id); + if (!cached) { + return null; + } + const pixels = backend.createSurface(cached.rect.width, cached.rect.height); + pixels.ctx.drawImage(cached.surface.canvas, 0, 0); + return { pixels, rect: { ...cached.rect } }; + }, + ctx: h.ctx, + discardPersisted: () => undefined, + exportBaked, + getControlModel: () => CONTROL_MODEL, + getDuplicateRasterPlan: unused, + getSelectedLayerIds: unused, + hasPendingPixelWork: () => false, + isGuardCurrent, + needsPixelPersistence: (layer) => 'source' in layer && layer.source.type === 'paint', + pinDuplicateRasterSources: unused, + prepareDuplicateRasterSource: unused, + publishSelectedLayerIds: unused, + sameContract: (candidate, layer) => + areJsonValuesStructurallyEqual(getDocumentLayer(candidate, layer.id) ?? null, layer), + scheduleDuplicateRasterization: unused, + trackDetached: unused, + }); + + /** The settled display: the layer's exact adjusted cache through its transform, over `rect`. */ + const displayed = async (rect: Rect): Promise => { + const display = createAdjustedSurfaceCache(backend, undefined, { spatial: { run } }); + await vi.waitFor( + () => { + display.get('layer', entry, ADJUSTMENTS); + expect(display.peek('layer', entry, ADJUSTMENTS)).not.toBeNull(); + }, + { interval: 20, timeout: 10_000 } + ); + const surface = backend.createSurface(rect.width, rect.height); + const matrix = fromTRS({ x: TRANSFORM.x, y: TRANSFORM.y }, TRANSFORM.rotation, TRANSFORM.scaleX, TRANSFORM.scaleY); + surface.ctx.setTransform(matrix.a, matrix.b, matrix.c, matrix.d, matrix.e - rect.x, matrix.f - rect.y); + surface.ctx.drawImage(display.peek('layer', entry, ADJUSTMENTS)!.canvas, CONTENT.x, CONTENT.y); + return [...surface.ctx.getImageData(0, 0, rect.width, rect.height).data]; + }; + return { copy, displayed, h, mutation, pixelsOf, raw }; +}; + +const bakedControl = (id: string, name: string, rect: Rect) => ({ + adapter: createDefaultControlAdapter(CONTROL_MODEL.base, CONTROL_MODEL.model), + blendMode: LAYER.blendMode, + id, + isEnabled: LAYER.isEnabled, + isLocked: LAYER.isLocked, + name, + opacity: LAYER.opacity, + source: { bitmap: null, offset: { x: rect.x, y: rect.y }, type: 'paint' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'control', + withTransparencyEffect: true, +}); + +describe('copying a raster layer with a spatial stack to a control layer', () => { + it('adds a control layer holding the settled display, evaluated in the spatial worker', async () => { + const { copy, displayed, h, pixelsOf } = scene(); + + const result = await copy.copy('layer', 'control'); + + expect(result).toEqual({ layerId: 'new-1', status: 'copied' }); + const rect = h.layers.get('new-1')!.rect; + expect(getDocumentLayer(h.document(), 'new-1')).toEqual(bakedControl('new-1', 'Layer copy', rect)); + expect(pixelsOf('new-1')).toEqual(await displayed(rect)); + expect(getDocumentLayer(h.document(), 'layer')).toEqual(LAYER); + }); + + it('refuses with the layer name when the stack cannot run, adding nothing', async () => { + const { copy, h } = scene(() => Promise.resolve({ status: 'too-large' })); + const before = h.document(); + + expect(await copy.copy('layer', 'control')).toEqual({ + ownerId: 'layer', + ownerName: 'Layer', + reason: 'too-large', + status: 'adjustment-refused', + }); + expect(h.document()).toBe(before); + expect(h.history.canUndo()).toBe(false); + }); +}); + +describe('converting a raster layer with a spatial stack to a control layer', () => { + it('bakes the settled display into the control pixels; undo restores the stack and raw pixels', async () => { + const { displayed, h, mutation, pixelsOf, raw } = scene(); + const live = getDocumentLayer(h.document(), 'layer')!; + + expect(await mutation.convertToControl('Convert to control', live)).toBe('converted'); + + const rect = h.layers.get('layer')!.rect; + const expected = await displayed(rect); + expect(getDocumentLayer(h.document(), 'layer')).toEqual(bakedControl('layer', 'Layer', rect)); + expect(pixelsOf('layer')).toEqual(expected); + expect(h.history.entries().past).toEqual(['Convert to control']); + + await expect(h.history.undo()).resolves.toEqual({ status: 'applied' }); + expect(getDocumentLayer(h.document(), 'layer')).toEqual(LAYER); + expect(h.layers.get('layer')!.rect).toEqual(CONTENT); + expect(pixelsOf('layer')).toEqual(raw); + + await expect(h.history.redo()).resolves.toEqual({ status: 'applied' }); + expect(getDocumentLayer(h.document(), 'layer')).toEqual(bakedControl('layer', 'Layer', rect)); + expect(pixelsOf('layer')).toEqual(expected); + }); + + it('refuses with the layer name when the stack cannot run, leaving the raster layer', async () => { + const { h, mutation, pixelsOf, raw } = scene(() => Promise.resolve({ status: 'over-budget' })); + const before = h.document(); + + expect(await mutation.convertToControl('Convert to control', getDocumentLayer(before, 'layer')!)).toEqual({ + ownerId: 'layer', + ownerName: 'Layer', + reason: 'over-budget', + status: 'adjustment-refused', + }); + expect(h.document()).toBe(before); + expect(pixelsOf('layer')).toEqual(raw); + expect(h.history.canUndo()).toBe(false); + }); + + it('lands nothing when the layer changed while its stack was evaluated', async () => { + const { h, mutation } = scene(); + const live = getDocumentLayer(h.document(), 'layer')!; + const converting = mutation.convertToControl('Convert to control', live); + h.ctx.dispatch({ id: 'layer', patch: { opacity: 0.2 }, type: 'updateCanvasLayer' }); + + expect(await converting).toBe('not-ready'); + expect(getDocumentLayer(h.document(), 'layer')).toMatchObject({ opacity: 0.2, type: 'raster' }); + expect(h.history.canUndo()).toBe(false); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/booleanMergeController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/booleanMergeController.ts index 9ad70e2643a..85bddcbf1c8 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/booleanMergeController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/booleanMergeController.ts @@ -1,5 +1,6 @@ import type { LayerExportGuard } from '@workbench/canvas-engine/capabilities'; import type { CanvasDocumentContractV3, CanvasLayerContract } from '@workbench/canvas-engine/contracts'; +import type { AdjustmentRefusal } from '@workbench/canvas-engine/render/exactAdjustments'; import type { RasterBackend, RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Rect } from '@workbench/canvas-engine/types'; @@ -21,11 +22,19 @@ import { } from './editSteps'; export type BooleanRasterOperation = 'intersect' | 'cutout' | 'cutaway' | 'exclude'; -export type BooleanRasterResult = 'merged' | 'missing' | 'unsupported' | 'empty' | 'failed' | LayerEditRefusal; +export type BooleanRasterResult = + | 'merged' + | 'missing' + | 'unsupported' + | 'empty' + | 'failed' + | LayerEditRefusal + | AdjustmentRefusal; type ExportResult = | { status: 'ok'; surface: RasterSurface; rect: Rect; guard: LayerExportGuard; release(): void } - | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' }; + | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal; export interface BooleanMergeControllerOptions { readonly ctx: LayerStepContext & @@ -101,6 +110,9 @@ export class BooleanMergeController { return 'busy'; } if (upperPixels.status !== 'ok' || belowPixels.status !== 'ok') { + if (upperPixels.status === 'adjustment-refused' || belowPixels.status === 'adjustment-refused') { + return upperPixels.status === 'adjustment-refused' ? upperPixels : (belowPixels as AdjustmentRefusal); + } if (upperPixels.status === 'over-budget' || belowPixels.status === 'over-budget') { return 'over-budget'; } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlPixelController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlPixelController.test.ts index 8e084928dc1..69827efcce4 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlPixelController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlPixelController.test.ts @@ -1,5 +1,7 @@ import type { CanvasLayerContract } from '@workbench/canvas-engine/contracts'; import type { CanvasProjectMutation } from '@workbench/canvas-engine/mutationContracts'; +import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; +import type { AdjustedSurfaceStatus } from '@workbench/canvas-engine/render/spatialDisplay'; import { createCanvasMutationContext } from '@workbench/canvas-engine/controllers/mutationContext'; import { documentFrom, layerContract } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; @@ -30,7 +32,13 @@ const image = layerContract('image', 'raster', { /** The controller over a real reducer, mutation context and history. */ const createHarness = ( layer: CanvasLayerContract, - options: { byteBudget?: number; gestureActive?: boolean; notifyPainted?: () => void } = {} + options: { + byteBudget?: number; + gestureActive?: boolean; + notifyPainted?: () => void; + exactAdjustedSurface?: () => RasterSurface | null; + adjustmentStatus?: AdjustedSurfaceStatus; + } = {} ) => { let project = applyCanvasProjectMutation(createInitialWorkbenchState().projects[0]!, { document: documentFrom([layer], layer.id), @@ -81,8 +89,10 @@ const createHarness = ( deleteDerived: vi.fn(), getActiveProjectId: () => project.id, getAdjustedSurface: () => null, + getAdjustmentStatus: () => options.adjustmentStatus, getDocument: () => project.canvas.document, getTransformSession: () => null, + peekAdjustedSurface: options.exactAdjustedSurface ?? (() => null), installPrepared: calls.installPrepared, invalidate: vi.fn(), isCacheReady: () => true, @@ -173,6 +183,38 @@ describe('PixelEditController', () => { expect(h.calls.installPrepared).not.toHaveBeenCalled(); }); + const clarity = { amount: 50, id: 'c', isEnabled: true, size: 40, type: 'clarity' } as const; + const adjusted = { ...image, adjustments: [clarity], name: 'Sky' } as CanvasLayerContract; + + it('refuses to materialize a spatially adjusted image as not-ready until its display copy is exact', () => { + const pending = createHarness(adjusted, { adjustmentStatus: 'computing' }); + const surface = pending.layers.get('image')!.surface; + + expect(pending.controller.begin('image')).toBeNull(); + expect(pending.calls.reportRefusal).toHaveBeenCalledWith('not-ready'); + expect(pending.layers.get('image')!.surface).toBe(surface); + expect(pending.calls.suspendLayer).not.toHaveBeenCalled(); + + const exact = createHarness(adjusted, { exactAdjustedSurface: () => surface }); + expect(exact.controller.begin('image')).not.toBeNull(); + expect(exact.calls.reportRefusal).not.toHaveBeenCalled(); + }); + + it('names the layer and cause when its spatial adjustments cannot run, since waiting will not help', () => { + const h = createHarness(adjusted, { adjustmentStatus: 'too-large' }); + const surface = h.layers.get('image')!.surface; + + expect(h.controller.begin('image')).toBeNull(); + expect(h.calls.reportRefusal).toHaveBeenCalledWith({ + ownerId: 'image', + ownerName: 'Sky', + reason: 'too-large', + status: 'adjustment-refused', + }); + expect(h.layers.get('image')!.surface).toBe(surface); + expect(h.calls.suspendLayer).not.toHaveBeenCalled(); + }); + it('refuses a materialization too large to undo before baking anything', () => { const h = createHarness(image, { byteBudget: 4 }); const surface = h.layers.get('image')!.surface; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlPixelController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlPixelController.ts index d15f9df641e..a74a9f86496 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlPixelController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlPixelController.ts @@ -15,6 +15,7 @@ import type { PreparedLayerCacheReplacement, } from '@workbench/canvas-engine/render/layerCache'; import type { RasterBackend, RasterSurface } from '@workbench/canvas-engine/render/raster'; +import type { AdjustedSurfaceStatus } from '@workbench/canvas-engine/render/spatialDisplay'; import type { PixelEditTransaction, PixelEditPatch, StrokeCommittedEvent } from '@workbench/canvas-engine/tools/tool'; import type { LayerTransform } from '@workbench/canvas-engine/transform/transformMath'; import type { Rect } from '@workbench/canvas-engine/types'; @@ -33,6 +34,8 @@ import { HISTORY_ENTRY_OVERHEAD_BYTES } from '@workbench/canvas-engine/history/h import { createImagePatchEntry } from '@workbench/canvas-engine/history/imagePatch'; import { createLayerSnapshotEntry } from '@workbench/canvas-engine/history/layerSnapshot'; import { isEmpty } from '@workbench/canvas-engine/math/rect'; +import { hasSpatialAdjustments } from '@workbench/canvas-engine/render/adjustments'; +import { adjustmentRefusal } from '@workbench/canvas-engine/render/exactAdjustments'; import { strokeCommitLabel } from '@workbench/canvas-engine/strokeCommit'; import type { CanvasMutationContext, EditStep, EditTransaction } from './mutationContext'; @@ -46,6 +49,10 @@ export interface PixelEditControllerOptions { readonly deleteDerived: (layerId: string) => void; readonly getActiveProjectId: () => string | null; readonly getAdjustedSurface: (layer: PixelEditableLayer, entry: LayerCacheEntry) => RasterSurface | null; + /** The display's adjusted copy only when it is exact (spatial results landed); never builds one. */ + readonly peekAdjustedSurface: (layer: PixelEditableLayer, entry: LayerCacheEntry) => RasterSurface | null; + /** Why a layer's displayed spatial adjustments are not exact; undefined while exact. */ + readonly getAdjustmentStatus: (layerId: string) => AdjustedSurfaceStatus | undefined; readonly getDocument: () => CanvasDocumentContractV3 | null; readonly getTransformSession: () => unknown; readonly installPrepared: (prepared: PreparedLayerCacheReplacement, persist?: boolean) => void; @@ -149,6 +156,22 @@ export class PixelEditController { return null; } const entry = o.layers.get(layerId); + // Materializing bakes what the display shows; a spatial stack's interim is not that layer's pixels. It may still + // become exact while its job computes; a stack that cannot run never will, so that refusal names the cause. + if ( + decision.status !== 'direct' && + layer.type === 'raster' && + hasSpatialAdjustments(layer.adjustments) && + entry && + !isEmpty(entry.rect) && + !o.peekAdjustedSurface(layer, entry) + ) { + const status = o.getAdjustmentStatus(layerId); + o.reportRefusal( + status === undefined || status === 'computing' ? 'not-ready' : adjustmentRefusal(document, layerId, status) + ); + return null; + } const beforeBytes = entry && !isEmpty(entry.rect) ? entry.rect.width * entry.rect.height * 4 : 0; // A materialized edit records the whole layer before and after. const historyBytes = diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/copyLayerController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/copyLayerController.ts index 167fa0feabc..db1dcd484a7 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/copyLayerController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/copyLayerController.ts @@ -1,10 +1,14 @@ import type { LayerExportGuard } from '@workbench/canvas-engine/capabilities'; +import type { CanvasLayerContract } from '@workbench/canvas-engine/contracts'; import type { CanvasCommandRefusal } from '@workbench/canvas-engine/document/commandRefusal'; +import type { CanvasNodeInsertionAnchor } from '@workbench/canvas-engine/document/insertionAnchors'; import type { SubsetOf } from '@workbench/canvas-engine/editConcurrency'; +import type { AdjustmentRefusal } from '@workbench/canvas-engine/render/exactAdjustments'; import type { RasterBackend, RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Rect } from '@workbench/canvas-engine/types'; import { getDocumentLayer } from '@workbench/canvas-engine/document/documentIndex'; +import { createControlLayerFrom } from '@workbench/canvas-engine/document/layerFactories'; import type { CanvasMutationContext } from './mutationContext'; @@ -21,11 +25,16 @@ import { type ExportResult = | { status: 'ok'; surface: RasterSurface; rect: Rect; guard: LayerExportGuard; release(): void } - | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' }; + | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal; -export type CopyLayerToRasterResult = +/** The stack a baked copy lands in: a raster paint layer, or a control layer with a default adapter. */ +export type CopyLayerTarget = 'raster' | 'control'; + +export type CopyLayerResult = | { status: 'copied'; layerId: string } - | { status: LayerEditRefusal | SubsetOf | 'empty' | 'failed' }; + | { status: LayerEditRefusal | SubsetOf | 'empty' | 'failed' } + | AdjustmentRefusal; export interface CopyLayerControllerOptions { readonly ctx: LayerStepContext & @@ -40,16 +49,35 @@ export interface CopyLayerControllerOptions { | 'isPermitCurrent' >; readonly backend: RasterBackend; + /** Adds `layer` as a plain contract copy of the source, with the source's live cache if it has one. */ + readonly commitLayerCopy: ( + label: string, + sourceLayerId: string, + layer: CanvasLayerContract, + anchor: CanvasNodeInsertionAnchor + ) => boolean; readonly exportBaked: (layerId: string) => Promise; + /** The main model's base and its default control model, for a control copy's adapter. */ + readonly getControlModel: () => { readonly base: string | null; readonly model: string | null }; readonly isGuardCurrent: (guard: LayerExportGuard) => boolean; } -/** Owns guarded baked copies into new raster paint layers. */ +const COPY_LABELS: Record = { + control: 'Copy layer to control', + raster: 'Copy layer to raster', +}; + +/** + * Owns guarded baked copies into new paint layers. The copy holds the layer's pixels as displayed: its own adjustment + * stack evaluated exactly (spatial entries included, or refused) and its transform, at an identity transform. An + * empty raster image or paint layer has no pixels to bake, so its control copy is a plain clone of its source and + * transform, without the stack. + */ export class CopyLayerController { private disposed = false; constructor(private readonly deps: CopyLayerControllerOptions) {} - async copyToRaster(layerId: string): Promise { + async copy(layerId: string, target: CopyLayerTarget): Promise { const { ctx } = this.deps; const permit = ctx.capturePermit(); if (this.disposed || !permit) { @@ -60,6 +88,33 @@ export class CopyLayerController { return { status: 'missing' }; } const baked = await this.deps.exportBaked(layerId); + if (baked.status === 'adjustment-refused') { + return baked; + } + if ( + baked.status === 'empty' && + target === 'control' && + sourceLayer.type === 'raster' && + (sourceLayer.source.type === 'image' || sourceLayer.source.type === 'paint') + ) { + if (!ctx.isPermitCurrent(permit)) { + return { status: 'busy' }; + } + // A preview-tolerant recheck, as for baked copies below. + if (getDocumentLayer(ctx.getReducerDocument(), layerId) !== sourceLayer) { + return { status: 'not-ready' }; + } + const layer = createControlLayerFrom(sourceLayer, { + ...this.deps.getControlModel(), + content: { source: sourceLayer.source }, + id: ctx.createLayerId(), + name: `${sourceLayer.name} copy`, + }); + const anchor = ctx.captureInsertionAnchor('control', layerId); + return this.deps.commitLayerCopy(COPY_LABELS.control, layerId, layer, anchor) + ? { layerId: layer.id, status: 'copied' } + : { status: 'not-ready' }; + } if (baked.status !== 'ok') { return { status: baked.status === 'disabled' ? 'not-ready' : baked.status }; } @@ -88,11 +143,21 @@ export class CopyLayerController { } const pixels = this.deps.backend.createSurface(baked.rect.width, baked.rect.height); pixels.ctx.drawImage(baked.surface.canvas, 0, 0); - const layer = paintLayerAt(ctx.createLayerId(), `${sourceLayer.name} copy`, baked.rect); + const id = ctx.createLayerId(); + const name = `${sourceLayer.name} copy`; + const layer = + target === 'control' + ? createControlLayerFrom(sourceLayer, { + ...this.deps.getControlModel(), + content: { rect: baked.rect }, + id, + name, + }) + : paintLayerAt(id, name, baked.rect); const status = layerOperationStatus( publishAddedRasterLayer(ctx, txn, { - anchor: ctx.captureInsertionAnchor('raster', layerId), - label: 'Copy layer to raster', + anchor: ctx.captureInsertionAnchor(target, layerId), + label: COPY_LABELS[target], layer, pixels, rect: baked.rect, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/cropLayerController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/cropLayerController.ts index e5b62f7fc75..ff89084cd5d 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/cropLayerController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/cropLayerController.ts @@ -4,6 +4,7 @@ import type { CanvasLayerContract, CanvasLayerSourceContract, } from '@workbench/canvas-engine/contracts'; +import type { AdjustmentRefusal } from '@workbench/canvas-engine/render/exactAdjustments'; import type { RasterBackend, RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Rect } from '@workbench/canvas-engine/types'; @@ -27,11 +28,13 @@ import { export type CropLayerResult = | { status: 'cropped' } | { status: 'missing' | 'locked' | 'unsupported' | 'empty' | LayerEditRefusal } - | { status: 'failed'; message: string }; + | { status: 'failed'; message: string } + | AdjustmentRefusal; type ExportResult = | { status: 'ok'; surface: RasterSurface; rect: Rect; guard: LayerExportGuard; release(): void } - | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' }; + | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal; export interface CropLayerControllerOptions { readonly ctx: Pick< @@ -85,6 +88,9 @@ export class CropLayerController { } try { const exported = await this.deps.exportBaked(layerId); + if (exported.status === 'adjustment-refused') { + return exported; + } if (exported.status !== 'ok') { return { status: exported.status === 'disabled' ? 'not-ready' : exported.status }; } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/editSteps.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/editSteps.ts index dc14c3c2516..b1125c94002 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/editSteps.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/editSteps.ts @@ -198,7 +198,7 @@ type EnabledUpdates = readonly { readonly id: string; readonly isEnabled: boolea /** A new paint layer whose pixels the undo entry owns, inserted and selected in one step. */ export interface AddedRasterLayerEdit { readonly label: string; - /** A raster paint layer whose source offset places `pixels` at `rect`. */ + /** A raster or control paint layer whose source offset places `pixels` at `rect`. */ readonly layer: CanvasLayerContract; readonly rect: Rect; readonly pixels: RasterSurface; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/extractMaskedAreaController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/extractMaskedAreaController.ts index 89f545c179b..940f034280f 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/extractMaskedAreaController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/extractMaskedAreaController.ts @@ -1,28 +1,37 @@ import type { LayerExportGuard } from '@workbench/canvas-engine/capabilities'; import type { + CanvasAdjustmentsContract, CanvasDocumentContractV3, CanvasLayerContract, - CanvasNodeContract, + CanvasRasterLayerContractV2, } from '@workbench/canvas-engine/contracts'; -import type { CanvasDiagnostics } from '@workbench/canvas-engine/diagnostics'; +import type { SemanticLeaf } from '@workbench/canvas-engine/document-model/semanticLeaf'; import type { CanvasCommandRefusal } from '@workbench/canvas-engine/document/commandRefusal'; import type { SubsetOf } from '@workbench/canvas-engine/editConcurrency'; -import type { CompositeOptions } from '@workbench/canvas-engine/render/compositor'; -import type { DerivedSurfaceCache } from '@workbench/canvas-engine/render/derivedSurfaceCache'; -import type { LayerCacheEntry, LayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; +import type { LayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; import type { RasterBackend, RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Rect } from '@workbench/canvas-engine/types'; import { compileDocumentLeaves } from '@workbench/canvas-engine/document-model/documentModel'; import { getDocumentLeaves } from '@workbench/canvas-engine/document/documentIndex'; -import { createEmptyStacks, isGroupNode } from '@workbench/canvas-engine/document/documentTree'; import { getSourceContentRect } from '@workbench/canvas-engine/document/sources'; import { isEmpty } from '@workbench/canvas-engine/math/rect'; -import { compositeDocument } from '@workbench/canvas-engine/render/compositor'; +import { + type AdjustmentRefusal, + AdjustmentRefusedError, + adjustmentRefusal, + type SpatialRunner, +} from '@workbench/canvas-engine/render/exactAdjustments'; import { collectCompositedGroups, planGroupCompositeScopes, } from '@workbench/canvas-engine/render/groupCompositeScopes'; +import { + compositeScratchPixels, + planRasterComposite, + RasterCompositeOverBudgetError, + renderRasterComposite, +} from '@workbench/canvas-engine/render/rasterComposite'; import type { CanvasMutationContext } from './mutationContext'; @@ -39,11 +48,15 @@ import { export type ExtractMaskedAreaResult = | { status: 'extracted'; layerId: string } - | { status: LayerEditRefusal | SubsetOf | 'empty' | 'failed' }; + | { status: LayerEditRefusal | SubsetOf | 'empty' | 'failed' } + | AdjustmentRefusal; + +type OwnedExport = Extract; type ExportResult = | { status: 'ok'; surface: RasterSurface; rect: Rect; guard: LayerExportGuard; release(): void } - | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' }; + | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal; export interface ExtractMaskedAreaControllerOptions { readonly ctx: LayerStepContext & @@ -59,28 +72,21 @@ export interface ExtractMaskedAreaControllerOptions { >; readonly backend: RasterBackend; readonly layers: LayerCacheStore; - readonly derived: DerivedSurfaceCache; - readonly diagnostics: CanvasDiagnostics; readonly isCacheReady: (layer: CanvasLayerContract, document: CanvasDocumentContractV3) => boolean; readonly hasExportableContent: (layerId: string) => boolean; readonly exportBaked: (layerId: string, includeDisabled?: boolean) => Promise; readonly rasterize: (layerId: string) => Promise; readonly isGuardCurrent: (guard: LayerExportGuard) => boolean; - readonly getAdjustedSurface: (layer: CanvasLayerContract, entry: LayerCacheEntry) => RasterSurface | null; - readonly getGroupSurface?: CompositeOptions['groupSurface']; - readonly getMaskPattern: (style: string, color: string) => RasterSurface | null; + /** The display's exact adjusted copy of a layer's live cache `surface`, or null; never builds one. */ + readonly peekAdjustedCacheSurface?: ( + layerId: string, + surface: RasterSurface, + adjustments: CanvasAdjustmentsContract + ) => RasterSurface | null; + /** Evaluates spatial adjustment runs; without it a spatial stack refuses `error`. */ + readonly runSpatial?: SpatialRunner; } -/** The raster forest reduced to `keep` leaves, groups intact so their stacks still apply. */ -const filterRasterForest = (nodes: readonly CanvasNodeContract[], keep: ReadonlySet): CanvasNodeContract[] => - nodes.flatMap((node): CanvasNodeContract[] => { - if (isGroupNode(node)) { - const children = filterRasterForest(node.children, keep); - return children.length > 0 ? [{ ...node, children }] : []; - } - return keep.has(node.id) ? [node] : []; - }); - /** The group compositing `layers` pass through (ancestor opacity, blend and adjustments), for comparison. */ const groupScopesKey = (document: CanvasDocumentContractV3, layers: readonly CanvasLayerContract[]): string => { const ids = new Set(layers.map((layer) => layer.id)); @@ -88,13 +94,18 @@ const groupScopesKey = (document: CanvasDocumentContractV3, layers: readonly Can return JSON.stringify(planGroupCompositeScopes(leaves, collectCompositedGroups(document))); }; +type RasterLeaf = SemanticLeaf & { layer: CanvasRasterLayerContractV2 }; + +const contributingLeaves = (document: CanvasDocumentContractV3, hasContent: (layerId: string) => boolean) => + compileDocumentLeaves(document).filter( + (leaf): leaf is RasterLeaf => + leaf.stack === 'raster' && leaf.layer.type === 'raster' && leaf.contributionEnabled && hasContent(leaf.id) + ); + const contributingRasters = ( document: CanvasDocumentContractV3, hasContent: (layerId: string) => boolean -): CanvasLayerContract[] => - compileDocumentLeaves(document) - .filter((leaf) => leaf.stack === 'raster' && leaf.contributionEnabled && hasContent(leaf.id)) - .map((leaf) => leaf.layer); +): CanvasLayerContract[] => contributingLeaves(document, hasContent).map((leaf) => leaf.layer); /** Owns guarded extraction of raster content through an inpaint mask. */ export class ExtractMaskedAreaController { @@ -209,30 +220,58 @@ export class ExtractMaskedAreaController { return { status: layerEditRefusal(txn.status) }; } try { - if (!txn.reserveRaster(rgbaBytes(rect))) { + // The contributors composite as generation and export do: exact adjustments (spatial ones on whole sources, + // never a display interim), from the leased pixels. + const leased = new Map( + contributorPixels.map((leasedPixels, index) => [contributors[index]!.id, leasedPixels as OwnedExport]) + ); + const entry = planRasterComposite( + document, + contributingLeaves(document, this.deps.hasExportableContent), + rect, + new Map([...leased].map(([id, owned]) => [id, owned.rect])) + ); + // The output plus each scratch buffer and its ImageData, until the transaction ends. + if (!txn.reserveRaster(rgbaBytes(rect) + compositeScratchPixels(entry) * 2 * 4)) { return { status: 'over-budget' }; } - const pixels = this.deps.backend.createSurface(rect.width, rect.height); - compositeDocument( - pixels, - { - ...document, - stacks: { - ...createEmptyStacks(), - raster: filterRasterForest(document.stacks.raster, new Set(contributors.map((layer) => layer.id))), - }, - }, - this.deps.layers, - { a: 1, b: 0, c: 0, d: 1, e: -rect.x, f: -rect.y }, - { - adjustedSurface: this.deps.getAdjustedSurface, - groupSurface: this.deps.getGroupSurface, + let pixels: RasterSurface; + try { + pixels = await renderRasterComposite(entry, { + adjustedSurface: this.deps.peekAdjustedCacheSurface, backend: this.deps.backend, - derivedSurfaces: this.deps.derived, - diagnostics: this.deps.diagnostics, - maskPatternTile: this.deps.getMaskPattern, + getLayerSurface: (layerId) => { + const owned = leased.get(layerId); + return owned + ? Promise.resolve({ rect: owned.rect, release: () => undefined, surface: owned.surface }) + : Promise.reject(new Error(`Extraction did not lease layer ${layerId}.`)); + }, + reserve: (bytes) => + txn.reserveRaster(bytes) + ? { lease: { release: () => undefined }, status: 'ok' } + : { availableBytes: 0, requestedBytes: bytes, status: 'over-budget' }, + runSpatial: this.deps.runSpatial, + }); + } catch (error) { + if (error instanceof AdjustmentRefusedError) { + return adjustmentRefusal(document, error.ownerId, error.reason); } - ); + if (error instanceof RasterCompositeOverBudgetError) { + return { status: 'over-budget' }; + } + if (error instanceof DOMException && error.name === 'AbortError') { + return { status: 'not-ready' }; + } + throw error; + } + // Spatial jobs yield: the extraction stands only for pixels that still describe their layers. + if ( + this.disposed || + !this.deps.isGuardCurrent(maskPixels.guard) || + [...leased.values()].some((owned) => !this.deps.isGuardCurrent(owned.guard)) + ) { + return { status: 'not-ready' }; + } pixels.ctx.setTransform(1, 0, 0, 1, 0, 0); pixels.ctx.globalAlpha = 1; pixels.ctx.globalCompositeOperation = 'destination-in'; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/floatingSelectionController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/floatingSelectionController.ts index c22c85c8c3e..0915df843c1 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/floatingSelectionController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/floatingSelectionController.ts @@ -74,7 +74,7 @@ const commitHistoryBytes = ( }; type PublishedStatus = EditPublishResult['status']; -const isRefusal = (status: PublishedStatus): status is CanvasEditRefusal => +const isRefusal = (status: PublishedStatus): status is Extract => status === 'busy' || status === 'gesture-active' || status === 'not-ready' || status === 'over-budget'; /** diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerController.test.ts index 8911c82edd5..9205d03d0e8 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerController.test.ts @@ -27,7 +27,7 @@ describe('LayerController', () => { getMaskPattern: () => null, isDisposed: () => false, isSupportedSource: () => true, - peekAdjustedSurface: () => null, + peekDisplayedSurface: () => null, projectId: 'p1', rasterize: vi.fn(), reportError: vi.fn(), @@ -90,7 +90,14 @@ describe('LayerController', () => { isGuardCurrent: () => true, isSupportedSource: () => true, }; - const copy = { backend: {} as never, ctx: {} as never, exportBaked: vi.fn(), isGuardCurrent: () => true }; + const copy = { + backend: {} as never, + commitLayerCopy: () => false, + ctx: {} as never, + exportBaked: vi.fn(), + getControlModel: () => ({ base: null, model: null }), + isGuardCurrent: () => true, + }; const newRasterLayer = { backend: {} as never, ctx: {} as never, layers: {} as never, selection: {} as never }; it('exposes only declared layer and preview ports', async () => { const forward: CanvasProjectMutation = { id: 'layer', type: 'setCanvasSelectedLayer' }; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerMutationController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerMutationController.ts index 3f5aa013d35..80cb3a35f4a 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerMutationController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerMutationController.ts @@ -1,8 +1,12 @@ -import type { DuplicateLayersResult } from '@workbench/canvas-engine/capabilities'; +import type { DuplicateLayersResult, LayerExportGuard } from '@workbench/canvas-engine/capabilities'; import type { CanvasDocumentContractV3, CanvasLayerContract } from '@workbench/canvas-engine/contracts'; +import type { CanvasCommandRefusal } from '@workbench/canvas-engine/document/commandRefusal'; import type { CanvasNodeInsertionAnchor } from '@workbench/canvas-engine/document/insertionAnchors'; +import type { SubsetOf } from '@workbench/canvas-engine/editConcurrency'; import type { CanvasProjectMutation } from '@workbench/canvas-engine/mutationContracts'; +import type { AdjustmentRefusal } from '@workbench/canvas-engine/render/exactAdjustments'; import type { PreparedLayerCacheReplacement } from '@workbench/canvas-engine/render/layerCache'; +import type { RasterBackend, RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Rect } from '@workbench/canvas-engine/types'; import { lookupDocumentLeaf } from '@workbench/canvas-engine/document-model/documentModel'; @@ -16,14 +20,17 @@ import { } from '@workbench/canvas-engine/document/documentIndex'; import { cloneSubtree, collectSubtreeLeaves } from '@workbench/canvas-engine/document/documentTree'; import { insertNodesAtAnchor } from '@workbench/canvas-engine/document/insertionAnchors'; +import { createControlLayerFrom } from '@workbench/canvas-engine/document/layerFactories'; import { haveSameStructure } from '@workbench/canvas-engine/document/layerStacks'; import { collectHistoryMediaRefs, HISTORY_ENTRY_OVERHEAD_BYTES } from '@workbench/canvas-engine/history/history'; -import type { CanvasMutationContext, EditStep } from './mutationContext'; +import type { CanvasMutationContext, EditPublishResult, EditStep, EditTransaction } from './mutationContext'; import { guardedResultRefusal, layerEditRefusal, + layerOperationStatus, + type LayerEditRefusal, type LayerPixels, rgbaBytes, withReplayReservation, @@ -31,6 +38,18 @@ import { export type CapturedLayerCache = LayerPixels | null | 'not-ready' | 'over-budget'; +export type ConvertLayerToControlResult = + | 'converted' + | SubsetOf + | LayerEditRefusal + | 'failed' + | AdjustmentRefusal; + +type BakedExportResult = + | { status: 'ok'; surface: RasterSurface; rect: Rect; guard: LayerExportGuard; release(): void } + | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal; + export type DuplicateLayerRasterPlan = | { readonly captureBytes: number; @@ -66,6 +85,12 @@ export interface LayerMutationControllerOptions { | 'preparePixels' | 'reserveRaster' >; + readonly backend: RasterBackend; + /** A layer's pixels with its own adjustments and transform baked in, disabled layers included. */ + readonly exportBaked: (layerId: string) => Promise; + /** The main model's base and its default control model, for a converted control layer's adapter. */ + readonly getControlModel: () => { readonly base: string | null; readonly model: string | null }; + readonly isGuardCurrent: (guard: LayerExportGuard) => boolean; /** Copies a layer's live cache; `admit` accounts its bytes first and refuses the copy as `over-budget`. */ readonly captureCache: ( layer: CanvasLayerContract, @@ -422,19 +447,13 @@ export class LayerMutationController { } convert(label: string, expected: CanvasLayerContract, after: CanvasLayerContract): boolean { - const { ctx, ...o } = this.options; + const { ctx } = this.options; if (expected.id !== after.id || expected.type === after.type) { return false; } const document = ctx.getDocument(); const current = getDocumentLayer(document, expected.id); - if ( - !document || - !current || - current !== expected || - current.type !== expected.type || - lookupDocumentLeaf(document, current.id)?.effectiveLocked !== false - ) { + if (!document || !current || current !== expected || !this.isUnlocked(document, current.id)) { return false; } const txn = ctx.begin({ historyBytes: HISTORY_ENTRY_OVERHEAD_BYTES }); @@ -442,54 +461,171 @@ export class LayerMutationController { return false; } try { - const captured = o.captureCache( - current, - document, - (rect) => txn.growHistory(rgbaBytes(rect)) && txn.reserveRaster(rgbaBytes(rect)) - ); - if (captured === 'not-ready' || captured === 'over-budget') { - return false; - } - // A conversion changes stacks, so undo carries the leaf back to its captured place. - const restoreAnchor = ctx.captureRestoreAnchor(current.id) ?? undefined; - const before = structuredClone(current); - const converted = ( - layer: CanvasLayerContract, - restore: CanvasLayerContract, - anchor?: CanvasNodeInsertionAnchor - ): EditStep => { - const prepared = captured ? ctx.preparePixels(layer.id, captured.rect, captured.pixels) : null; - return { - accepted: (candidate) => o.sameContract(candidate, layer), - install: () => { - try { - o.discardPersisted(layer.id); - } catch { - /* Ancillary after reducer acceptance. */ - } - if (prepared) { - ctx.installPrepared(prepared, o.needsPixelPersistence(layer)); - } - }, - mutation: { anchor, id: layer.id, layer, targetType: layer.type, type: 'convertCanvasLayer' }, - rollback: { - mutation: { id: layer.id, layer: restore, targetType: restore.type, type: 'convertCanvasLayer' }, - restored: (candidate) => o.sameContract(candidate, restore), - }, - }; - }; - const result = txn.publish(label, converted(after, current), { - bytes: (captured ? rgbaBytes(captured.rect) : 0) + HISTORY_ENTRY_OVERHEAD_BYTES, - heldAssetRefs: collectHistoryMediaRefs(before, after), - redo: () => replayPrepared(ctx, captured, () => ctx.applyStep(converted(after, before))), - undo: () => replayPrepared(ctx, captured, () => ctx.applyStep(converted(before, after, restoreAnchor))), - }); - return result.status === 'committed'; + return this.publishConversion(txn, label, document, current, after, null).status === 'committed'; } finally { txn.end(); } } + /** + * Converts a raster image or paint layer into a control layer in place, as one undo step. Control layers carry no + * adjustments, so the layer's pixels as displayed — its own stack evaluated exactly (spatial entries included, or + * refused) and its transform — become the control layer's paint pixels at an identity transform. Undo restores the + * raster layer with its stack and original pixels; redo reinstalls the baked pixels. An empty layer has no pixels + * to bake, so it converts as a plain clone of its source and transform, without the stack. + */ + async convertToControl(label: string, expected: CanvasLayerContract): Promise { + const { ctx, ...o } = this.options; + const permit = ctx.capturePermit(); + if (!permit) { + return 'busy'; + } + const document = ctx.getDocument(); + const current = getDocumentLayer(document, expected.id); + if (!document || !current) { + return 'missing'; + } + if (current !== expected) { + return 'not-ready'; + } + if (current.type !== 'raster' || (current.source.type !== 'image' && current.source.type !== 'paint')) { + return 'unsupported'; + } + if (!this.isUnlocked(document, current.id)) { + return 'locked'; + } + const baked = await o.exportBaked(current.id); + if (baked.status === 'adjustment-refused') { + return baked; + } + if (baked.status === 'empty') { + const plain = createControlLayerFrom(current, { + ...o.getControlModel(), + content: { source: current.source }, + id: current.id, + name: current.name, + }); + return ctx.isPermitCurrent(permit) && this.convert(label, current, plain) ? 'converted' : 'not-ready'; + } + if (baked.status !== 'ok') { + return baked.status === 'disabled' ? 'not-ready' : baked.status; + } + try { + if (!ctx.isPermitCurrent(permit)) { + return 'busy'; + } + // A preview-tolerant recheck; see `CanvasMutationContext.getReducerDocument`. + const liveDocument = ctx.getReducerDocument(); + if ( + getDocumentLayer(liveDocument, current.id) !== current || + baked.guard.layer !== current || + !o.isGuardCurrent(baked.guard) + ) { + return 'not-ready'; + } + const txn = ctx.begin({ historyBytes: rgbaBytes(baked.rect) + HISTORY_ENTRY_OVERHEAD_BYTES }); + if (!('publish' in txn)) { + return layerEditRefusal(txn.status); + } + try { + // Admission ended any open preview; the conversion starts from the document it left. + const startDocument = ctx.getDocument(); + if (!startDocument || getDocumentLayer(startDocument, current.id) !== current) { + return 'not-ready'; + } + if (!this.isUnlocked(startDocument, current.id)) { + return 'locked'; + } + // The entry's copy of the baked pixels (the export's surface is released below) and its prepared install. + if (!txn.reserveRaster(rgbaBytes(baked.rect) * 2)) { + return 'over-budget'; + } + const pixels = o.backend.createSurface(baked.rect.width, baked.rect.height); + pixels.ctx.drawImage(baked.surface.canvas, 0, 0); + const after = createControlLayerFrom(current, { + ...o.getControlModel(), + content: { rect: baked.rect }, + id: current.id, + name: current.name, + }); + const status = layerOperationStatus( + this.publishConversion(txn, label, startDocument, current, after, { pixels, rect: baked.rect }) + ); + return status === 'committed' ? 'converted' : status; + } finally { + txn.end(); + } + } finally { + baked.release(); + } + } + + private isUnlocked(document: CanvasDocumentContractV3, layerId: string): boolean { + return lookupDocumentLeaf(document, layerId)?.effectiveLocked === false; + } + + /** + * Publishes `current`'s conversion to `after`. `afterPixels` are installed with the converted layer, else its own + * live cache is; undo always reinstalls that cache, so a paint layer gets its original pixels back. + */ + private publishConversion( + txn: EditTransaction, + label: string, + document: CanvasDocumentContractV3, + current: CanvasLayerContract, + after: CanvasLayerContract, + afterPixels: LayerPixels | null + ): EditPublishResult { + const { ctx, ...o } = this.options; + const captured = o.captureCache( + current, + document, + (rect) => txn.growHistory(rgbaBytes(rect)) && txn.reserveRaster(rgbaBytes(rect)) + ); + if (captured === 'not-ready' || captured === 'over-budget') { + return { status: captured }; + } + const forwardPixels = afterPixels ?? captured; + // A conversion changes stacks, so undo carries the leaf back to its captured place. + const restoreAnchor = ctx.captureRestoreAnchor(current.id) ?? undefined; + const before = structuredClone(current); + const converted = ( + layer: CanvasLayerContract, + restore: CanvasLayerContract, + pixels: LayerPixels | null, + anchor?: CanvasNodeInsertionAnchor + ): EditStep => { + const prepared = pixels ? ctx.preparePixels(layer.id, pixels.rect, pixels.pixels) : null; + return { + accepted: (candidate) => o.sameContract(candidate, layer), + install: () => { + try { + o.discardPersisted(layer.id); + } catch { + /* Ancillary after reducer acceptance. */ + } + if (prepared) { + ctx.installPrepared(prepared, o.needsPixelPersistence(layer)); + } + }, + mutation: { anchor, id: layer.id, layer, targetType: layer.type, type: 'convertCanvasLayer' }, + rollback: { + mutation: { id: layer.id, layer: restore, targetType: restore.type, type: 'convertCanvasLayer' }, + restored: (candidate) => o.sameContract(candidate, restore), + }, + }; + }; + return txn.publish(label, converted(after, current, forwardPixels), { + bytes: + (captured ? rgbaBytes(captured.rect) : 0) + + (afterPixels ? rgbaBytes(afterPixels.rect) : 0) + + HISTORY_ENTRY_OVERHEAD_BYTES, + heldAssetRefs: collectHistoryMediaRefs(before, after), + redo: () => replayPrepared(ctx, forwardPixels, () => ctx.applyStep(converted(after, before, forwardPixels))), + undo: () => replayPrepared(ctx, captured, () => ctx.applyStep(converted(before, after, captured, restoreAnchor))), + }); + } + dispose(): void {} } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.browser.test.ts index b7d6700da06..ee9d6286ae6 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.browser.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.browser.test.ts @@ -7,12 +7,17 @@ import type { } from '@workbench/canvas-engine/contracts'; import type { Rect } from '@workbench/canvas-engine/types'; +import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { getDocumentLayer, getDocumentLeaves } from '@workbench/canvas-engine/document/documentIndex'; import { fromTRS } from '@workbench/canvas-engine/math/mat2d'; import { createAdjustedSurfaceCache } from '@workbench/canvas-engine/render/adjustedSurfaceCache'; import { createDomRasterBackend } from '@workbench/canvas-engine/render/raster'; -import { describe, expect, it } from 'vitest'; +import { + createSpatialAdjustmentService, + type SpatialAdjustmentService, +} from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { createLayerOperationHarness } from './layerOperationHarness.testing'; import { MergeLayerController } from './mergeLayerController'; @@ -60,7 +65,7 @@ interface Scene { upperTransform?: CanvasRasterLayerContractV2['transform']; } -const mergeScene = (scene: Scene, displayed: boolean) => { +const mergeScene = (scene: Scene, displayed: boolean, service?: SpatialAdjustmentService) => { const backend = createDomRasterBackend(); const upper = layer('upper', UPPER_RECT, scene.upperAdjustments, scene.upperTransform); const lower = layer('lower', LOWER_RECT, scene.lowerAdjustments); @@ -84,7 +89,11 @@ const mergeScene = (scene: Scene, displayed: boolean) => { }; fill('upper', UPPER_RECT, (x) => `rgba(${40 + x * 8}, 90, 200, ${x % 3 === 0 ? 0.6 : 1})`); fill('lower', LOWER_RECT, (x) => `rgb(200, ${50 + x * 9}, 70)`); - const display = createAdjustedSurfaceCache(backend); + const display = createAdjustedSurfaceCache( + backend, + undefined, + service ? { spatial: { run: (request) => service.run(request) } } : {} + ); const contracts = { lower: getDocumentLayer(h.document(), 'lower')!, upper: getDocumentLayer(h.document(), 'upper')!, @@ -116,10 +125,23 @@ const mergeScene = (scene: Scene, displayed: boolean) => { layers: h.layers, needsPixelPersistence: () => false, peekAdjustedSurface: (contract, entry) => - displayed && contract.type === 'raster' ? display.get(contract.id, entry, contract.adjustments) : null, + displayed && contract.type === 'raster' ? display.peek(contract.id, entry, contract.adjustments) : null, publishSelectedLayerIds: () => undefined, + runSpatial: service ? (request) => service.run(request) : undefined, }); - return { ...h, canvasPixels, contracts, controller, rawPixels }; + /** Shows the canvas until every spatial result has landed, as frames would. */ + const settle = () => + vi.waitFor( + () => { + canvasPixels(); + for (const contract of getDocumentLeaves(h.document())) { + const adjustments = contract.type === 'raster' ? contract.adjustments : undefined; + expect(display.state(contract.id, h.layers.get(contract.id)!, adjustments)).toBe('exact'); + } + }, + { interval: 20, timeout: 10_000 } + ); + return { ...h, canvasPixels, contracts, controller, rawPixels, settle }; }; describe('Merge Down with adjustment stacks', () => { @@ -143,7 +165,7 @@ describe('Merge Down with adjustment stacks', () => { const raw = { lower: h.rawPixels('lower'), upper: h.rawPixels('upper') }; const shown = h.canvasPixels(); - expect(h.controller.mergeDown('upper')).toBe('merged'); + expect(await h.controller.mergeDown('upper')).toBe('merged'); // The merged layer has no stack: its baked pixels show what both layers showed. expect(getDocumentLayer(h.document(), 'lower')).not.toHaveProperty('adjustments'); @@ -157,3 +179,45 @@ describe('Merge Down with adjustment stacks', () => { }); }); }); + +describe('Merge Down with spatial stacks', () => { + const services: SpatialAdjustmentService[] = []; + afterEach(() => { + for (const service of services.splice(0)) { + service.dispose(); + } + }); + const spatialService = () => { + const memory = new RasterMemoryBudgetController({ budgetBytes: 64 * 1024 * 1024 }); + const service = createSpatialAdjustmentService({ + reserve: (bytes) => memory.reserveOperation(bytes, { purpose: 'spatial-adjustment' }), + }); + services.push(service); + return service; + }; + const scene: Scene = { + lowerAdjustments: [{ amount: 80, id: 'texture', isEnabled: true, size: 3, type: 'texture' }], + upperAdjustments: [ + { amount: 90, id: 'clarity', isEnabled: true, size: 8, type: 'clarity' }, + { id: 'invert', isEnabled: true, type: 'invert' }, + ], + upperTransform: { rotation: Math.PI / 7, scaleX: 0.7, scaleY: 0.55, x: 9, y: 4 }, + }; + + it.each([ + ['evaluating each stack in the spatial worker', false], + ["drawing the display's exact copies", true], + ])('keeps the settled display by %s; undo restores both stacks', async (_label, displayed) => { + const h = mergeScene(scene, displayed, spatialService()); + await h.settle(); + const shown = h.canvasPixels(); + + expect(await h.controller.mergeDown('upper')).toBe('merged'); + + expect(getDocumentLayer(h.document(), 'lower')).not.toHaveProperty('adjustments'); + expect(h.canvasPixels()).toEqual(shown); + await expect(h.history.undo()).resolves.toEqual({ status: 'applied' }); + expect(getDocumentLayer(h.document(), 'upper')).toBe(h.contracts.upper); + expect(getDocumentLayer(h.document(), 'lower')).toBe(h.contracts.lower); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.test.ts index 96e4f874f83..c864e10343c 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.test.ts @@ -1,11 +1,16 @@ -import type { CanvasDocumentContractV3, CanvasLayerContract } from '@workbench/canvas-engine/contracts'; +import type { + CanvasAdjustmentsContract, + CanvasDocumentContractV3, + CanvasLayerContract, +} from '@workbench/canvas-engine/contracts'; +import type { SpatialRunner } from '@workbench/canvas-engine/render/exactAdjustments'; import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Rect } from '@workbench/canvas-engine/types'; -import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { groupContract, stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { getDocumentLayer, getDocumentLeaves } from '@workbench/canvas-engine/document/documentIndex'; import { createTestStubRasterBackend } from '@workbench/canvas-engine/render/raster.testStub'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { createLayerOperationHarness } from './layerOperationHarness.testing'; import { MergeLayerController } from './mergeLayerController'; @@ -28,12 +33,23 @@ const MERGED_RECT: Rect = { height: 15, width: 15, x: 0, y: 0 }; const createHarness = ( historyBytes?: number, - { lowerAdjusted = false, displayed = null }: { lowerAdjusted?: boolean; displayed?: RasterSurface | null } = {} + { + lowerAdjusted = false, + lowerAdjustments, + displayed = null, + runSpatial, + }: { + lowerAdjusted?: boolean; + lowerAdjustments?: CanvasAdjustmentsContract; + displayed?: RasterSurface | null; + runSpatial?: SpatialRunner; + } = {} ) => { const upper = paintLayer('upper'); const lower: CanvasLayerContract = { ...paintLayer('lower'), ...(lowerAdjusted ? { adjustments: [{ id: 'invert', isEnabled: true, type: 'invert' }] } : {}), + ...(lowerAdjustments ? { adjustments: lowerAdjustments } : {}), }; const document: CanvasDocumentContractV3 = { background: 'transparent', @@ -57,6 +73,7 @@ const createHarness = ( needsPixelPersistence: () => true, peekAdjustedSurface: () => displayed, publishSelectedLayerIds: () => undefined, + runSpatial, }); // The reducer's own contracts, which merge and undo must restore by identity. const originals = { @@ -70,10 +87,10 @@ const leafIds = (document: CanvasDocumentContractV3 | null): string[] => getDocumentLeaves(document).map((layer) => layer.id); describe('MergeLayerController: merge down', () => { - it('bakes the upper layer into the lower one as one undo step', () => { + it('bakes the upper layer into the lower one as one undo step', async () => { const h = createHarness(); - expect(h.controller.mergeDown('upper')).toBe('merged'); + expect(await h.controller.mergeDown('upper')).toBe('merged'); expect(leafIds(h.document())).toEqual(['lower']); expect(getDocumentLayer(h.document(), 'lower')).toMatchObject({ source: { bitmap: null, offset: { x: 0, y: 0 }, type: 'paint' }, @@ -84,7 +101,7 @@ describe('MergeLayerController: merge down', () => { it('undo restores both layers, their pixels and the selection; redo merges again', async () => { const h = createHarness(); - h.controller.mergeDown('upper'); + await h.controller.mergeDown('upper'); await expect(h.history.undo()).resolves.toEqual({ status: 'applied' }); expect(leafIds(h.document())).toEqual(['upper', 'lower']); @@ -101,7 +118,7 @@ describe('MergeLayerController: merge down', () => { it('puts the merge back and keeps the step when the upper layer cannot be reinserted', async () => { const h = createHarness(); - h.controller.mergeDown('upper'); + await h.controller.mergeDown('upper'); const merged = h.document(); h.state.refuse = (mutation) => mutation.type === 'applyCanvasLayerStackMutation'; @@ -113,48 +130,197 @@ describe('MergeLayerController: merge down', () => { expect(h.history.canRedo()).toBe(false); }); - it('refuses a merge it could never undo before anything changes', () => { + it('refuses a merge it could never undo before anything changes', async () => { const h = createHarness(1024); const before = h.document(); - expect(h.controller.mergeDown('upper')).toBe('over-budget'); + expect(await h.controller.mergeDown('upper')).toBe('over-budget'); expect(h.document()).toBe(before); expect(h.dispatched).toEqual([]); expect(h.layers.get('upper')?.rect).toEqual(UPPER_RECT); }); - it('refuses when its pixels cannot be reserved', () => { + it('refuses when its pixels cannot be reserved', async () => { const h = createHarness(); h.state.rasterBytes = 1024; - expect(h.controller.mergeDown('upper')).toBe('over-budget'); + expect(await h.controller.mergeDown('upper')).toBe('over-budget'); expect(h.dispatched).toEqual([]); expect(h.history.canUndo()).toBe(false); }); - it('reserves a layer-local adjusted copy only when the display has no current one', () => { + it('reserves a layer-local adjusted copy only when the display has no current one', async () => { // Raw undo snapshots plus the merged pixels and their prepared replacement; a copy adds a surface and ImageData. const baseBytes = (100 + 100) * 4 + 225 * 4 * 2; const copying = createHarness(undefined, { lowerAdjusted: true }); copying.state.rasterBytes = baseBytes; - expect(copying.controller.mergeDown('upper')).toBe('over-budget'); + expect(await copying.controller.mergeDown('upper')).toBe('over-budget'); copying.state.rasterBytes = baseBytes + 100 * 4 * 2; - expect(copying.controller.mergeDown('upper')).toBe('merged'); + expect(await copying.controller.mergeDown('upper')).toBe('merged'); const displayed = createHarness(undefined, { displayed: createTestStubRasterBackend().createSurface(10, 10), lowerAdjusted: true, }); displayed.state.rasterBytes = baseBytes; - expect(displayed.controller.mergeDown('upper')).toBe('merged'); + expect(await displayed.controller.mergeDown('upper')).toBe('merged'); }); - it('refuses the bottom-most layer and a busy canvas', () => { + it('refuses the bottom-most layer and a busy canvas', async () => { const h = createHarness(); - expect(h.controller.mergeDown('lower')).toBe('unsupported'); - expect(h.controller.mergeDown('missing')).toBe('missing'); + expect(await h.controller.mergeDown('lower')).toBe('unsupported'); + expect(await h.controller.mergeDown('missing')).toBe('missing'); h.state.gestureActive = true; - expect(h.controller.mergeDown('upper')).toBe('busy'); + expect(await h.controller.mergeDown('upper')).toBe('busy'); expect(h.dispatched).toEqual([]); }); + + describe('with a spatial stack', () => { + const CLARITY: CanvasAdjustmentsContract = [ + { amount: 60, id: 'clarity', isEnabled: true, size: 20, type: 'clarity' }, + ]; + const passThrough = () => + vi.fn((request) => Promise.resolve({ pixels: request.pixels, status: 'ok' })); + + it('evaluates the layer-local stack in the spatial service before merging, and drops it', async () => { + const runSpatial = passThrough(); + const h = createHarness(undefined, { lowerAdjustments: CLARITY, runSpatial }); + + expect(await h.controller.mergeDown('upper')).toBe('merged'); + + expect(runSpatial).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ height: LOWER_RECT.height, priority: 'now', width: LOWER_RECT.width }) + ); + expect(getDocumentLayer(h.document(), 'lower')).not.toHaveProperty('adjustments'); + }); + + it("reuses the display's exact copy without a job", async () => { + const runSpatial = passThrough(); + const h = createHarness(undefined, { + displayed: createTestStubRasterBackend().createSurface(10, 10), + lowerAdjustments: CLARITY, + runSpatial, + }); + + expect(await h.controller.mergeDown('upper')).toBe('merged'); + expect(runSpatial).not.toHaveBeenCalled(); + }); + + it.each(['too-large', 'over-budget', 'error'] as const)( + 'refuses %s with the layer name before anything changes', + async (reason) => { + const h = createHarness(undefined, { + lowerAdjustments: CLARITY, + runSpatial: () => Promise.resolve({ status: reason }), + }); + + expect(await h.controller.mergeDown('upper')).toEqual({ + ownerId: 'lower', + ownerName: 'lower', + reason, + status: 'adjustment-refused', + }); + expect(h.dispatched).toEqual([]); + expect(h.history.canUndo()).toBe(false); + expect(h.reservedBytes()).toBe(0); + } + ); + + it('refuses without a spatial runner rather than merge without the spatial entries', async () => { + const h = createHarness(undefined, { lowerAdjustments: CLARITY }); + + expect(await h.controller.mergeDown('upper')).toMatchObject({ reason: 'error', status: 'adjustment-refused' }); + expect(h.dispatched).toEqual([]); + }); + + it('lands nothing when a layer changed while the job ran', async () => { + const h = createHarness(undefined, { + lowerAdjustments: CLARITY, + runSpatial: (request) => { + h.layers.publishPixels('upper'); + return Promise.resolve({ pixels: request.pixels, status: 'ok' }); + }, + }); + + expect(await h.controller.mergeDown('upper')).toBe('not-ready'); + expect(h.dispatched).toEqual([]); + expect(h.history.canUndo()).toBe(false); + }); + }); +}); + +describe('MergeLayerController: merge visible with a spatial group stack', () => { + const scene = (runSpatial?: SpatialRunner) => { + const first = { ...paintLayer('first'), name: 'First' }; + const second = { ...paintLayer('second'), name: 'Second' }; + const document: CanvasDocumentContractV3 = { + background: 'transparent', + bbox: { height: 100, width: 100, x: 0, y: 0 }, + height: 100, + selectedLayerId: null, + stacks: stacksFrom([ + groupContract('group', [first, second], { + adjustments: [{ amount: 30, id: 'dehaze', isEnabled: true, type: 'dehaze' }], + name: 'Haze', + }), + ]), + version: 3, + width: 100, + }; + const h = createLayerOperationHarness(document); + const rects: Record = { first: UPPER_RECT, second: LOWER_RECT }; + const controller = new MergeLayerController({ + backend: h.backend, + ctx: h.ctx, + exportBaked: (layerId) => + Promise.resolve({ + guard: { + cacheVersion: 1, + documentGeneration: 1, + layer: getDocumentLayer(h.document(), layerId)!, + layerId, + projectId: 'p', + }, + rect: rects[layerId]!, + release: () => undefined, + status: 'ok', + surface: h.backend.createSurface(rects[layerId]!.width, rects[layerId]!.height), + }), + hasExportableContent: () => true, + isCacheReady: () => true, + layers: h.layers, + needsPixelPersistence: () => false, + peekAdjustedSurface: () => null, + publishSelectedLayerIds: () => undefined, + runSpatial, + }); + return { ...h, controller }; + }; + + it("evaluates the group's spatial stack on the merged members", async () => { + const runSpatial = vi.fn((request) => Promise.resolve({ pixels: request.pixels, status: 'ok' })); + const h = scene(runSpatial); + + expect(await h.controller.mergeVisible()).toBe('merged'); + // The buffer holds every member: the union of their baked rects. + expect(runSpatial).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + height: MERGED_RECT.height, + steps: [{ amount: 30, kind: 'dehaze' }], + width: MERGED_RECT.width, + }) + ); + }); + + it('refuses with the group name when its stack cannot run', async () => { + const h = scene(() => Promise.resolve({ status: 'too-large' })); + + expect(await h.controller.mergeVisible()).toEqual({ + ownerId: 'group', + ownerName: 'Haze', + reason: 'too-large', + status: 'adjustment-refused', + }); + expect(h.history.canUndo()).toBe(false); + }); }); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.ts index 26b7f95f5d7..4cfd5d5f640 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.ts @@ -20,8 +20,21 @@ import { mergeDownMatrix } from '@workbench/canvas-engine/document/mergeDown'; import { canMergeSelectedRasters, getMergeVisibleRasterLeaves } from '@workbench/canvas-engine/document/mergeVisible'; import { collectHistoryMediaRefs, HISTORY_ENTRY_OVERHEAD_BYTES } from '@workbench/canvas-engine/history/history'; import { isEmpty, roundOut, transformBounds, union } from '@workbench/canvas-engine/math/rect'; -import { applyAdjustments, isIdentityAdjustments } from '@workbench/canvas-engine/render/adjustments'; +import { + applyAdjustments, + hasSpatialAdjustments, + isIdentityAdjustments, +} from '@workbench/canvas-engine/render/adjustments'; import { blendToComposite } from '@workbench/canvas-engine/render/compositor'; +import { + adjustExactly, + type AdjustmentRefusal, + adjustmentRefusal, + adjustSurfaceExactly, + imageDataOf, + isAdjustmentRefusal, + type SpatialRunner, +} from '@workbench/canvas-engine/render/exactAdjustments'; import { collectCompositedGroups, planGroupCompositeScopes, @@ -43,17 +56,19 @@ import { type LayerPixels, } from './editSteps'; -export type MergeVisibleResult = 'merged' | LayerEditRefusal | 'nothing' | 'failed'; +export type MergeVisibleResult = 'merged' | LayerEditRefusal | 'nothing' | 'failed' | AdjustmentRefusal; export type MergeDownResult = | 'merged' | SubsetOf | LayerEditRefusal - | 'failed'; + | 'failed' + | AdjustmentRefusal; type ExportResult = | { status: 'ok'; surface: RasterSurface; rect: Rect; guard: LayerExportGuard; release(): void } - | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' }; + | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal; export interface MergeLayerControllerOptions { readonly backend: RasterBackend; @@ -63,8 +78,10 @@ export interface MergeLayerControllerOptions { readonly hasExportableContent: (layerId: string) => boolean; readonly exportBaked: (layerId: string) => Promise; readonly needsPixelPersistence: (layer: CanvasLayerContract) => boolean; - /** The display's current adjusted copy of a raster layer's live cache, or null; never builds one. */ + /** The display's current, exact adjusted copy of a raster layer's live cache, or null; never builds one. */ readonly peekAdjustedSurface: (layer: CanvasLayerContract, entry: LayerCacheEntry) => RasterSurface | null; + /** Evaluates spatial adjustment runs; without it a spatial stack refuses `error`. */ + readonly runSpatial?: SpatialRunner; readonly publishSelectedLayerIds: (primaryId: string | null, selectedIds: readonly string[]) => void; } @@ -83,9 +100,10 @@ export class MergeLayerController { * Bakes the upper layer into the one below as one undo step: the lower layer keeps its id and takes the merged * pixels, with each layer's own adjustment stack baked in layer-local pixels before the upper one is resampled, so * the merged layer (whose contract has no stack) shows what the canvas showed. Undo restores the lower contract and - * raw pixels, then reinserts the upper layer with its own. + * raw pixels, then reinserts the upper layer with its own. A spatial stack without an exact display copy is + * evaluated in the spatial worker first; the merge then lands only if neither layer nor the document changed. */ - mergeDown(upperLayerId: string): MergeDownResult { + async mergeDown(upperLayerId: string): Promise { const { ctx } = this.deps; if (this.disposed) { return 'not-ready'; @@ -129,7 +147,7 @@ export class MergeLayerController { : transformBounds(matrix, upperRect) ); const snapshotBytes = rgbaBytes(upperRect) + rgbaBytes(belowRect); - // Each stack applies to its layer's raw pixels before compositing: the display's current adjusted copy when it + // Each stack applies to its layer's raw pixels before compositing: the display's exact adjusted copy when it // has one, else a layer-local copy (surface plus ImageData) adjusted here. const ownStack = (layer: CanvasLayerContract, cache: LayerCacheEntry | undefined) => cache && layer.type === 'raster' && layer.adjustments && !isIdentityAdjustments(layer.adjustments) @@ -160,23 +178,53 @@ export class MergeLayerController { }; const upperPixels = snapshot(upperRect, upper.id); const belowPixels = snapshot(belowRect, below.id); - const adjusted = (pixels: LayerPixels, stack: ReturnType): RasterSurface => { - if (!stack) { - return pixels.pixels; - } - if (stack.displayed) { - return stack.displayed; + const revision = ctx.getEditRevision(); + const versions = [upperCache?.version, belowCache?.version]; + // A layer's stack works in layer-local pixels; surface (0,0) sits at its cache rect's origin. + const adjusted = async ( + layer: CanvasLayerContract, + pixels: LayerPixels, + stack: ReturnType + ): Promise => { + if (!stack || stack.displayed) { + return stack?.displayed ?? pixels.pixels; } - const { height, width } = pixels.rect; - const imageData = pixels.pixels.ctx.getImageData(0, 0, width, height); - // A layer's stack works in layer-local pixels; surface (0,0) sits at its cache rect's origin. - applyAdjustments(imageData, stack.adjustments, pixels.rect); - const copy = this.deps.backend.createSurface(width, height); - copy.ctx.putImageData(imageData, 0, 0); - return copy; + const result = await adjustSurfaceExactly( + (width, height) => this.deps.backend.createSurface(width, height), + pixels.pixels, + pixels.rect, + stack.adjustments, + { run: this.deps.runSpatial } + ); + return result.status === 'ok' + ? result.surface + : result.status === 'aborted' + ? 'aborted' + : adjustmentRefusal(document, layer.id, result.status); }; - const upperDrawn = adjusted(upperPixels, upperStack); - const belowDrawn = adjusted(belowPixels, belowStack); + const upperDrawn = await adjusted(upper, upperPixels, upperStack); + if (upperDrawn === 'aborted' || isAdjustmentRefusal(upperDrawn)) { + return upperDrawn === 'aborted' ? 'not-ready' : upperDrawn; + } + const belowDrawn = await adjusted(below, belowPixels, belowStack); + if (belowDrawn === 'aborted' || isAdjustmentRefusal(belowDrawn)) { + return belowDrawn === 'aborted' ? 'not-ready' : belowDrawn; + } + // Evaluation yields; the merge stands only for the document and pixels it read, and the display copies drawn + // below are still those pixels' exact copies. + if ( + this.disposed || + !txn.isCurrent() || + ctx.getEditRevision() !== revision || + this.deps.layers.peek(upper.id) !== upperCache || + this.deps.layers.peek(below.id) !== belowCache || + upperCache?.version !== versions[0] || + belowCache?.version !== versions[1] || + (upperStack?.displayed && this.deps.peekAdjustedSurface(upper, upperCache!) !== upperStack.displayed) || + (belowStack?.displayed && this.deps.peekAdjustedSurface(below, belowCache!) !== belowStack.displayed) + ) { + return 'not-ready'; + } const merged = this.deps.backend.createSurface(mergedRect.width, mergedRect.height); const context = merged.ctx; context.setTransform(1, 0, 0, 1, 0, 0); @@ -297,6 +345,10 @@ export class MergeLayerController { if (!ctx.isPermitCurrent(permit)) { return 'busy'; } + const refused = exports.find(isAdjustmentRefusal); + if (refused) { + return refused; + } if (exports.some((result) => result.status === 'over-budget')) { return 'over-budget'; } @@ -358,13 +410,15 @@ export class MergeLayerController { const context = pixels.ctx; context.setTransform(1, 0, 0, 1, 0, 0); context.clearRect(0, 0, rect.width, rect.height); - // Composite the group to preserve member opacity and blending in merged pixels. - const drawMerged = ( + // Composite the group to preserve member opacity and blending in merged pixels. A group stack works in + // document pixels; the buffer is `rect` at scale 1. `rect` holds every member, so a spatial stack sees all + // of its group's content (its alpha-bounds domain ignores the transparent margin). + const drawMerged = async ( target: RasterSurface['ctx'], from: number, to: number, range: readonly GroupCompositeScope[] - ): void => { + ): Promise => { let scopeIndex = range.length - 1; for (let index = to - 1; index >= from;) { const scope = scopeIndex >= 0 ? range[scopeIndex]! : null; @@ -372,10 +426,19 @@ export class MergeLayerController { const buffer = this.deps.backend.createSurface(rect.width, rect.height); buffer.ctx.setTransform(1, 0, 0, 1, 0, 0); buffer.ctx.clearRect(0, 0, rect.width, rect.height); - drawMerged(buffer.ctx, scope.start, scope.end, scope.children); - if (!isIdentityAdjustments(scope.adjustments)) { + const inner = await drawMerged(buffer.ctx, scope.start, scope.end, scope.children); + if (inner) { + return inner; + } + if (hasSpatialAdjustments(scope.adjustments)) { + const scoped = buffer.ctx.getImageData(0, 0, rect.width, rect.height); + const result = await adjustExactly(scoped, scope.adjustments, rect, { run: this.deps.runSpatial }); + if (result.status !== 'ok') { + return result.status === 'aborted' ? 'aborted' : adjustmentRefusal(document, scope.id, result.status); + } + buffer.ctx.putImageData(imageDataOf(result.data, rect.width, rect.height), 0, 0); + } else if (!isIdentityAdjustments(scope.adjustments)) { const scoped = buffer.ctx.getImageData(0, 0, rect.width, rect.height); - // Group stacks work in document pixels; the buffer is `rect` at scale 1. applyAdjustments(scoped, scope.adjustments, rect); buffer.ctx.putImageData(scoped, 0, 0); } @@ -393,8 +456,16 @@ export class MergeLayerController { target.drawImage(exported.surface.canvas, exported.rect.x - rect.x, exported.rect.y - rect.y); index -= 1; } + return null; }; - drawMerged(context, 0, successful.length, scopes); + const refusal = await drawMerged(context, 0, successful.length, scopes); + if (refusal) { + return refusal === 'aborted' ? 'not-ready' : refusal; + } + // A group job yields: the merged layer stands only for contributors that still look as they were read. + if (this.disposed || successful.some((exported) => !ctx.isGuardCurrent(exported.guard))) { + return 'not-ready'; + } context.globalAlpha = 1; context.globalCompositeOperation = 'source-over'; @@ -455,6 +526,10 @@ export class MergeLayerController { if (!ctx.isPermitCurrent(permit)) { return 'busy'; } + const refused = exports.find(isAdjustmentRefusal); + if (refused) { + return refused; + } if (exports.some((result) => result.status === 'over-budget')) { return 'over-budget'; } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/psdExportController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/psdExportController.test.ts index 9a79a7eb74f..d6659b22822 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/psdExportController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/psdExportController.test.ts @@ -466,3 +466,83 @@ describe('PsdExportController — raster tree', () => { ]); }); }); + +describe('PsdExportController — spatial adjustments', () => { + const leaf: CanvasRasterLayerContractV2 = { + adjustments: [{ amount: 30, id: 'texture', isEnabled: true, size: 4, type: 'texture' }], + blendMode: 'normal', + id: 'leaf', + isEnabled: true, + isLocked: false, + name: 'Leaf', + opacity: 1, + source: { image: { height: 20, imageName: 'leaf.png', width: 20 }, type: 'image' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'raster', + }; + const CLARITY = [{ amount: 50, id: 'clarity', isEnabled: true, size: 10, type: 'clarity' as const }]; + const spatialDocument: CanvasDocumentContractV3 = { + ...document, + stacks: stacksFrom([ + groupContract('outer', [groupContract('inner', [leaf], { adjustments: CLARITY, name: 'Inner' })], { + adjustments: [{ amount: 20, id: 'dehaze', isEnabled: true, type: 'dehaze' }], + name: 'Outer', + }), + ]), + }; + const controllerFor = (options: Partial[0]>) => { + const backend = createTestStubRasterBackend(); + const snapshot: CanvasDocumentSnapshot = { canvas: canvasWithDocument(spatialDocument), documentGeneration: 1 }; + return new PsdExportController({ + backend, + captureDocumentSnapshot: () => snapshot, + captureRasterSnapshot: () => + Promise.resolve({ + snapshot: { + ...snapshot, + emptyLayerIds: new Set(), + layerSurfaces: new Map([ + ['leaf', { rect: { height: 20, width: 20, x: 0, y: 0 }, surface: backend.createSurface(20, 20) }], + ]), + release: vi.fn(), + }, + status: 'ok' as const, + }), + getAvailableBytes: () => 10_000_000, + isDocumentSnapshotCurrent: () => true, + reserve: () => ({ lease: { release: vi.fn() }, status: 'ok' }), + ...options, + }); + }; + + it('bakes enclosing group stacks into each leaf, innermost first, named by group', async () => { + let planned: PsdExportPlan | null = null; + const execute = vi.fn((plan: PsdExportPlan) => { + planned = plan; + return Promise.resolve(); + }); + + await expect(controllerFor({ execute }).export('layers.psd')).resolves.toBe('exported'); + + const plan = planned as PsdExportPlan | null; + expect(plan?.status === 'ok' && plan.layers[0]!.groupStacks?.map((stack) => stack.id)).toEqual(['inner', 'outer']); + }); + + it("refuses with the group's name when its spatial stack cannot run on a leaf", async () => { + // The leaf's own Texture runs; the inner group's Clarity cannot. + const runSpatial = vi + .fn() + .mockImplementationOnce((request: { pixels: Uint8ClampedArray }) => + Promise.resolve({ pixels: request.pixels, status: 'ok' }) + ) + .mockResolvedValue({ status: 'too-large' }); + + await expect(controllerFor({ runSpatial }).export('layers.psd')).resolves.toEqual({ + ownerId: 'inner', + ownerName: 'Inner', + reason: 'too-large', + status: 'adjustment-refused', + }); + expect(runSpatial).toHaveBeenCalledTimes(2); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/psdExportController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/psdExportController.ts index c0b08fe0b00..0afd797aab8 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/psdExportController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/psdExportController.ts @@ -1,5 +1,5 @@ import type { CanvasDocumentSnapshot, PsdExportResult } from '@workbench/canvas-engine/capabilities'; -import type { CanvasAdjustmentsContract, CanvasNodeContract } from '@workbench/canvas-engine/contracts'; +import type { CanvasNodeContract } from '@workbench/canvas-engine/contracts'; import type { CanvasDetachedLayerSurface, CaptureRasterSnapshotResult, @@ -14,10 +14,16 @@ import { type ExecutePsdExportDeps, type PsdExportNodeInput, type PsdExportPlan, + type PsdGroupStack, type PsdPlanNode, } from '@workbench/canvas-engine/export/psdExport'; import { isExportableRasterLayer } from '@workbench/canvas-engine/layerExportGuards'; import { isIdentityAdjustments } from '@workbench/canvas-engine/render/adjustments'; +import { + AdjustmentRefusedError, + adjustmentRefusal, + type SpatialRunner, +} from '@workbench/canvas-engine/render/exactAdjustments'; type RasterMemoryReservationResult = | { status: 'ok'; lease: { release(): void } } @@ -35,6 +41,8 @@ export interface PsdExportControllerOptions { readonly getAvailableBytes: () => number; readonly isDocumentSnapshotCurrent: (snapshot: CanvasDocumentSnapshot) => boolean; readonly reserve: (bytes: number) => RasterMemoryReservationResult; + /** Evaluates spatial adjustment runs; without it a spatial stack refuses `error`. */ + readonly runSpatial?: SpatialRunner; } export const PSD_ALLOCATION_BYTES_PER_PIXEL = 8; @@ -112,16 +120,18 @@ export class PsdExportController { let missing: string | null = null; // PSD cannot represent a group stack: bake enclosing stacks into each // leaf (own first, then innermost outward). Approximate under member - // opacity/blending; the merged preview flattens the same baked pixels. + // opacity/blending, and for spatial entries (clarity, texture, dehaze), + // which see the leaf's pixels alone rather than the group composite; + // the merged preview flattens the same baked pixels. const toInputs = ( nodes: readonly CanvasNodeContract[], - ancestorStacks: readonly CanvasAdjustmentsContract[] = [] + ancestorStacks: readonly PsdGroupStack[] = [] ): PsdExportNodeInput[] => nodes.flatMap((node): PsdExportNodeInput[] => { if (isGroupNode(node)) { const stacks = isIdentityAdjustments(node.adjustments) ? ancestorStacks - : [...ancestorStacks, node.adjustments!]; + : [...ancestorStacks, { adjustments: node.adjustments!, id: node.id }]; return [ { blendMode: node.blendMode, @@ -146,12 +156,11 @@ export class PsdExportController { return []; } const own = node.type === 'raster' ? node.adjustments : undefined; - const enclosing = [...ancestorStacks].reverse().flat(); return [ { // Identity-aware: an emptied stack must not trigger the executor's bake writeback. adjustments: isIdentityAdjustments(own) ? undefined : own, - groupAdjustments: isIdentityAdjustments(enclosing) ? undefined : enclosing, + groupStacks: ancestorStacks.length > 0 ? [...ancestorStacks].reverse() : undefined, blendMode: node.blendMode, colorLabel: node.colorLabel, contentRect: detached.rect, @@ -195,12 +204,20 @@ export class PsdExportController { await (this.deps.execute ?? executePsdExport)( plan, /\.psd$/i.test(fileName) ? fileName : `${fileName}.psd`, - { backend: this.deps.backend, getLayerSurface, signal: abortController.signal } + { + backend: this.deps.backend, + getLayerSurface, + runSpatial: this.deps.runSpatial, + signal: abortController.signal, + } ); } catch (error) { if (abortController.signal.aborted || (error instanceof DOMException && error.name === 'AbortError')) { return 'aborted'; } + if (error instanceof AdjustmentRefusedError) { + return adjustmentRefusal(document, error.ownerId, error.reason); + } throw error; } return abortController.signal.aborted ? 'aborted' : 'exported'; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.test.ts index 0326436a9f2..aa7aeecdf55 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.test.ts @@ -5,7 +5,9 @@ import type { GroupCompositeScope, GroupSurfaceContent } from '@workbench/canvas import { createCanvasDiagnostics } from '@workbench/canvas-engine/diagnostics'; import { createBitmapStore } from '@workbench/canvas-engine/document/bitmapStore'; import { identity } from '@workbench/canvas-engine/math/mat2d'; +import { adjustExactly } from '@workbench/canvas-engine/render/exactAdjustments'; import { createTestStubRasterBackend } from '@workbench/canvas-engine/render/raster.testStub'; +import { createInlineSpatialExecutor } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService.testing'; import { describe, expect, it, vi } from 'vitest'; import { RasterController } from './rasterController'; @@ -259,3 +261,142 @@ describe('RasterController', () => { store.dispose(); }); }); + +describe('RasterController spatial adjustments', () => { + const spatialLayer = (id: string) => + ({ + adjustments: [{ amount: 50, id: 'adj-cl', isEnabled: true, size: 40, type: 'clarity' as const }], + blendMode: 'normal', + id, + opacity: 1, + type: 'raster', + }) as unknown as CanvasRasterLayerContractV2; + + const flush = async () => { + for (let i = 0; i < 10; i++) { + await Promise.resolve(); + } + }; + + const createSpatialController = (budgetBytes = 1 << 20) => { + const settled: string[] = []; + const statuses: [string, string | null][] = []; + const diagnostics = createCanvasDiagnostics(true); + const controller = new RasterController({ + backend: createTestStubRasterBackend({ readbackAlpha: 255 }), + budgetBytes, + createSpatialExecutor: createInlineSpatialExecutor, + diagnostics, + onAdjustedSurfaceSettled: (layerId) => settled.push(layerId), + onAdjustmentStatus: (layerId, status) => statuses.push([layerId, status]), + }); + return { controller, diagnostics, settled, statuses }; + }; + + it('redraws an invalidated group in full, since a landed result changes no member cache version', async () => { + const { controller, diagnostics } = createSpatialController(); + const entry = publish(controller, 'a'); + const drawGroup = () => + controller.groups.get( + groupScope, + [{ id: 'a', layer: spatialLayer('a') } as unknown as SemanticLeaf], + [identity()], + NO_GROUP_CONTENT + ); + const rebuilds = () => diagnostics.snapshot().groupSurfaceRebuilds; + const allocations = () => diagnostics.snapshot().groupSurfaceAllocations; + + drawGroup(); + await flush(); + drawGroup(); + expect([allocations(), rebuilds()]).toEqual([1, 0]); + + controller.groups.invalidate(['group']); + drawGroup(); + expect([allocations(), rebuilds()]).toEqual([1, 1]); + expect(controller.peekAdjustedSurface(spatialLayer('a'), entry)).not.toBeNull(); + }); + + it('reserves each job under the raster budget and settles to an exact copy export may reuse', async () => { + const { controller, settled, statuses } = createSpatialController(); + const entry = publish(controller, 'layer'); + + const shown = controller.getAdjustedSurface(spatialLayer('layer'), entry); + expect(controller.memory.snapshot().reservedBytes).toBe(SURFACE_BYTES); + expect(controller.peekAdjustedSurface(spatialLayer('layer'), entry)).toBeNull(); + expect(controller.peekDisplayedAdjustedSurface(spatialLayer('layer'), entry)).toBe(shown); + + await flush(); + expect(controller.memory.snapshot().reservedBytes).toBe(0); + expect(settled).toEqual(['layer']); + expect(statuses).toEqual([ + ['layer', 'computing'], + ['layer', null], + ]); + expect(controller.peekAdjustedSurface(spatialLayer('layer'), entry)).toBe(shown); + }); + + it('reports over-budget when the job cannot reserve its pixels', async () => { + const { controller, statuses } = createSpatialController(SURFACE_BYTES); + const entry = publish(controller, 'layer'); + + controller.getAdjustedSurface(spatialLayer('layer'), entry); + await flush(); + expect(statuses.at(-1)).toEqual(['layer', 'over-budget']); + }); + + it("abandons only the display's work on cooldown: exact jobs and exact display copies survive", async () => { + const { controller, settled, statuses } = createSpatialController(); + const exactEntry = publish(controller, 'exact'); + const exactShown = controller.getAdjustedSurface(spatialLayer('exact'), exactEntry); + await flush(); + const pendingEntry = publish(controller, 'pending'); + controller.getAdjustedSurface(spatialLayer('pending'), pendingEntry); + // A generation composite evaluating a detached snapshot, as a pixel-writing operation does. + const composite = adjustExactly( + { data: new Uint8ClampedArray(SURFACE_BYTES).fill(255), height: RECT.height, width: RECT.width }, + spatialLayer('detached').adjustments, + RECT, + { run: (request) => controller.spatial.run(request) } + ); + + controller.cancelSpatialAdjustments(); + expect(statuses.at(-1)).toEqual(['pending', null]); + expect(controller.peekAdjustedSurface(spatialLayer('exact'), exactEntry)).toBe(exactShown); + controller.releaseReconstructible(); + expect((await composite).status).toBe('ok'); + expect(settled).toEqual(['exact']); + }); + + it('abandons pending work on cooldown release and disposal, clearing each status', async () => { + const { controller, settled, statuses } = createSpatialController(); + const entry = publish(controller, 'layer'); + + controller.getAdjustedSurface(spatialLayer('layer'), entry); + controller.releaseReconstructible(); + expect(statuses).toEqual([ + ['layer', 'computing'], + ['layer', null], + ]); + await flush(); + expect(settled).toEqual([]); + expect(controller.memory.snapshot().derivedBytes).toBe(0); + + const again = controller.layers.peek('layer') ?? publish(controller, 'layer'); + controller.getAdjustedSurface(spatialLayer('layer'), again); + controller.dispose(); + expect(statuses.at(-1)).toEqual(['layer', null]); + await flush(); + expect(settled).toEqual([]); + await expect( + controller.spatial.run({ + height: 1, + pixels: new Uint8ClampedArray(4), + priority: 'now', + slotKey: 'late', + steps: [], + width: 1, + }) + ).resolves.toEqual({ status: 'aborted' }); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.ts index 8738e08b7af..512fa452b0b 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.ts @@ -8,6 +8,7 @@ import type { import type { CanvasDiagnostics } from '@workbench/canvas-engine/diagnostics'; import type { LayerCacheEntry, LayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; import type { RasterBackend, RasterSurface } from '@workbench/canvas-engine/render/raster'; +import type { AdjustedSurfaceStatus } from '@workbench/canvas-engine/render/spatialDisplay'; export type DecodeImageResult = | { status: 'ok'; surface: RasterSurface; decodedWidth: number; decodedHeight: number } @@ -33,6 +34,11 @@ import { } from '@workbench/canvas-engine/render/derivedSurfaceCache'; import { createGroupSurfaceCache, type GroupSurfaceCache } from '@workbench/canvas-engine/render/groupSurfaceCache'; import { createLayerCacheStore, DEFAULT_CACHE_BUDGET_BYTES } from '@workbench/canvas-engine/render/layerCache'; +import { + createSpatialAdjustmentService, + type SpatialAdjustmentExecutor, + type SpatialAdjustmentService, +} from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; import { RasterMemoryBudgetController } from './rasterMemoryBudgetController'; @@ -47,6 +53,14 @@ export interface RasterControllerOptions { readonly isLayerHeld?: (layerId: string) => boolean; /** Layers whose live cache already includes their adjustments (an open pixel transaction), drawn raw. */ readonly isAdjustmentBaked?: (layerId: string) => boolean; + /** A spatial adjustment result changed a layer's displayed pixels; its cache version did not change. */ + readonly onAdjustedSurfaceSettled?: (layerId: string) => void; + /** A spatial adjustment result changed a group surface's displayed pixels; no member changed. */ + readonly onGroupSurfaceSettled?: (groupId: string) => void; + /** A layer's or group's displayed spatial adjustments became inexact for `status`, or exact again (null). */ + readonly onAdjustmentStatus?: (ownerId: string, status: AdjustedSurfaceStatus | null) => void; + /** Spatial worker factory; Node tests pass `createInlineSpatialExecutor`. Defaults to a module worker. */ + readonly createSpatialExecutor?: () => SpatialAdjustmentExecutor; } /** Access clocks captured before a frame draws; artifacts read after them are that frame's working set. */ @@ -68,6 +82,8 @@ export class RasterController { readonly groups: GroupSurfaceCache; readonly memory: RasterMemoryBudgetController; readonly bitmaps: DecodedBitmapPool; + /** Evaluates spatial adjustments off the main thread; each job reserves its pixel buffer here. */ + readonly spatial: SpatialAdjustmentService; private readonly diagnostics: CanvasDiagnostics; private readonly isLayerHeld: (layerId: string) => boolean; private readonly isAdjustmentBaked: (layerId: string) => boolean; @@ -95,16 +111,33 @@ export class RasterController { this.getDocument = options.getDocument ?? (() => null); this.imageResolver = options.imageResolver ?? null; this.derived = createDerivedSurfaceCache(options.diagnostics, (bytes) => memory.setCategoryBytes('derived', bytes)); - this.adjustments = createAdjustedSurfaceCache(options.backend, this.derived, (layerId, version) => - this.layers.damageSince(layerId, version) - ); + this.spatial = createSpatialAdjustmentService({ + createExecutor: options.createSpatialExecutor, + reserve: (bytes) => memory.reserveOperation(bytes, { purpose: 'spatial-adjustment' }), + }); + this.adjustments = createAdjustedSurfaceCache(options.backend, this.derived, { + damageSince: (layerId, version) => this.layers.damageSince(layerId, version), + spatial: { + onSettle: options.onAdjustedSurfaceSettled, + onStatus: options.onAdjustmentStatus, + run: (request) => this.spatial.run(request), + }, + }); this.groups = createGroupSurfaceCache({ createSurface: (width, height) => options.backend.createSurface(width, height), damageSince: (layerId, version) => this.layers.damageSince(layerId, version), diagnostics: options.diagnostics, - getAdjustedSurface: (layer, entry) => this.getAdjustedSurface(layer, entry), + getAdjustedSurface: (layer, entry) => { + const surface = this.getAdjustedSurface(layer, entry); + return { state: surface ? this.adjustments.state(layer.id, entry, layer.adjustments) : 'exact', surface }; + }, getCacheEntry: (layerId) => this.layers.get(layerId), onBytesChange: (bytes) => memory.setCategoryBytes('group', bytes), + spatial: { + onSettle: options.onGroupSurfaceSettled, + onStatus: options.onAdjustmentStatus, + run: (request) => this.spatial.run(request), + }, }); } @@ -150,9 +183,20 @@ export class RasterController { return { evictedBaseLayerIds, overageBytes }; } - /** Drops every derived and group surface and each base cache nothing pins or holds. */ + /** + * Abandons the display's queued and running spatial jobs, dropping the interims they would have replaced. Exact jobs + * of pixel-writing operations keep running: their own operation's signal governs them, and detached snapshots stay + * valid through cooldown. + */ + cancelSpatialAdjustments(): void { + this.adjustments.cancelSpatial(); + this.groups.cancelSpatial(); + } + + /** Drops every derived and group surface, the display's spatial work and each base cache nothing pins or holds. */ releaseReconstructible(): void { - this.derived.dispose(); + // The adjusted-surface cache stores its surfaces in `derived`: disposing it releases both. + this.adjustments.dispose(); this.groups.clear(); for (const layerId of this.layers.evict((candidate) => this.isProtected(candidate), 0)) { this.deleteDerivedSurfaces(layerId); @@ -230,13 +274,20 @@ export class RasterController { : null; } - /** The display's current adjusted copy of a raster layer's cache, without building one on a miss. */ + /** The display's current, exact adjusted copy of a raster layer's cache, without building one on a miss. */ peekAdjustedSurface(layer: CanvasLayerContract, entry: LayerCacheEntry): RasterSurface | null { return layer.type === 'raster' && !this.isAdjustmentBaked(layer.id) ? this.adjustments.peek(layer.id, entry, layer.adjustments) : null; } + /** What the display shows for a raster layer's cache, a spatial interim included; never builds one. */ + peekDisplayedAdjustedSurface(layer: CanvasLayerContract, entry: LayerCacheEntry): RasterSurface | null { + return layer.type === 'raster' && !this.isAdjustmentBaked(layer.id) + ? this.adjustments.peekDisplayed(layer.id, entry, layer.adjustments) + : null; + } + /** The display's current adjusted copy of a layer's live cache, when `surface` is that cache; never builds one. */ peekAdjustedCacheSurface( layerId: string, @@ -332,7 +383,8 @@ export class RasterController { this.cancelAllRasterization(); this.clearThumbnailKeys(); this.layers.dispose(); - this.derived.dispose(); + this.adjustments.dispose(); + this.spatial.dispose(); this.groups.clear(); this.bitmaps.dispose(); this.memory.dispose(); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.browser.test.ts index b8ef92470c1..6e2aecbf6c0 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.browser.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.browser.test.ts @@ -3,11 +3,13 @@ import type { CanvasDocumentContractV3, CanvasRasterLayerContractV2 } from '@workbench/canvas-engine/contracts'; import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; +import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { fromTRS } from '@workbench/canvas-engine/math/mat2d'; import { createAdjustedSurfaceCache } from '@workbench/canvas-engine/render/adjustedSurfaceCache'; import { createLayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; import { createDomRasterBackend } from '@workbench/canvas-engine/render/raster'; +import { createSpatialAdjustmentService } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; import { describe, expect, it, vi } from 'vitest'; import { RasterExportController } from './rasterExportController'; @@ -118,3 +120,104 @@ describe('baked raster export', () => { baked.release(); }); }); + +describe('baked raster export with a spatial stack', () => { + it("bakes a rotated, downscaled layer's settled display, evaluated in the spatial worker", async () => { + const backend = createDomRasterBackend(); + const layers = createLayerCacheStore(backend); + const entry = layers.growToRect('layer', { height: 30, width: 40, x: -6, y: -4 }); + for (let x = 0; x < 40; x += 1) { + entry.surface.ctx.fillStyle = `rgb(${60 + x * 3}, ${x % 5 === 0 ? 20 : 128}, ${200 - x * 3})`; + entry.surface.ctx.fillRect(x, 2, 1, 26); + } + layers.publishPixels('layer'); + const layer: CanvasRasterLayerContractV2 = { + adjustments: [ + { amount: 90, id: 'clarity', isEnabled: true, size: 8, type: 'clarity' }, + { amount: 50, id: 'dehaze', isEnabled: true, type: 'dehaze' }, + ], + blendMode: 'normal', + id: 'layer', + isEnabled: true, + isLocked: false, + name: 'Layer', + opacity: 1, + source: { bitmap: { height: 30, imageName: 'layer.png', width: 40 }, offset: { x: -6, y: -4 }, type: 'paint' }, + transform: TRANSFORM, + type: 'raster', + }; + const document: CanvasDocumentContractV3 = { + background: 'transparent', + bbox: { height: 64, width: 64, x: 0, y: 0 }, + height: 64, + stacks: stacksFrom([layer]), + selectedLayerId: null, + version: 3, + width: 64, + }; + const memory = new RasterMemoryBudgetController({ budgetBytes: 64 * 1024 * 1024 }); + const service = createSpatialAdjustmentService({ + reserve: (bytes) => memory.reserveOperation(bytes, { purpose: 'spatial-adjustment' }), + }); + try { + const display = createAdjustedSurfaceCache(backend, undefined, { + spatial: { run: (request) => service.run(request) }, + }); + await vi.waitFor( + () => { + display.get('layer', entry, layer.adjustments); + expect(display.peek('layer', entry, layer.adjustments)).not.toBeNull(); + }, + { interval: 20, timeout: 10_000 } + ); + const controller = new RasterExportController({ + backend, + captureGuard: (captured) => ({ + cacheVersion: 1, + documentGeneration: 1, + layer: captured, + layerId: captured.id, + projectId: 'p', + }), + getDocument: () => document, + getOrStartRasterization: () => Promise.resolve('published'), + isGuardCurrent: () => true, + isRasterizing: () => false, + isSupportedSource: () => true, + layers, + pin: () => ({ release: () => undefined }), + runSpatial: (request) => service.run(request), + }); + + const baked = await controller.baked('layer'); + + expect(baked.status).toBe('ok'); + if (baked.status !== 'ok') { + return; + } + const { height, width } = baked.rect; + const reference = backend.createSurface(width, height); + const matrix = fromTRS( + { x: TRANSFORM.x, y: TRANSFORM.y }, + TRANSFORM.rotation, + TRANSFORM.scaleX, + TRANSFORM.scaleY + ); + reference.ctx.setTransform( + matrix.a, + matrix.b, + matrix.c, + matrix.d, + matrix.e - baked.rect.x, + matrix.f - baked.rect.y + ); + reference.ctx.drawImage(display.peek('layer', entry, layer.adjustments)!.canvas, -6, -4); + expect([...baked.surface.ctx.getImageData(0, 0, width, height).data]).toEqual([ + ...reference.ctx.getImageData(0, 0, width, height).data, + ]); + baked.release(); + } finally { + service.dispose(); + } + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.test.ts index 68ce664ac31..bd8d70b667b 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.test.ts @@ -521,3 +521,99 @@ describe('RasterExportController budget', () => { expect(memory.snapshot().reservedBytes).toBe(0); }); }); + +describe('RasterExportController spatial adjustments', () => { + const spatialScene = (options: { isGuardCurrent?: () => boolean; displayed?: boolean } = {}) => { + const backend = createTestStubRasterBackend(); + const layers = createLayerCacheStore(backend); + const entry = layers.getOrCreate('layer', 40, 30); + entry.hasPublishedPixels = true; + entry.stale = false; + const layer = { + adjustments: [{ amount: 40, id: 'texture', isEnabled: true, size: 8, type: 'texture' as const }], + blendMode: 'normal' as const, + id: 'layer', + isEnabled: true, + isLocked: false, + name: 'Fabric', + opacity: 1, + source: { image: { height: 30, imageName: 'layer.png', width: 40 }, type: 'image' as const }, + transform: { rotation: 0, scaleX: 0.5, scaleY: 0.5, x: 3, y: 2 }, + type: 'raster' as const, + }; + const document: CanvasDocumentContractV3 = { + background: 'transparent', + bbox: { height: 100, width: 100, x: 0, y: 0 }, + height: 100, + stacks: stacksFrom([layer]), + selectedLayerId: null, + version: 3, + width: 100, + }; + const runSpatial = vi.fn((request: { pixels: Uint8ClampedArray }) => + Promise.resolve({ pixels: request.pixels, status: 'ok' as const }) + ); + const display = backend.createSurface(40, 30); + const controller = new RasterExportController({ + backend, + captureGuard: () => ({ cacheVersion: 1, documentGeneration: 1, layer, layerId: 'layer', projectId: 'p' }), + getDocument: () => document, + getOrStartRasterization: () => Promise.resolve('published'), + isGuardCurrent: options.isGuardCurrent ?? (() => true), + isRasterizing: () => false, + isSupportedSource: () => true, + layers, + peekAdjustedCacheSurface: () => (options.displayed ? display : null), + pin, + runSpatial, + }); + return { controller, display, runSpatial }; + }; + + it('evaluates the whole layer-local surface in the spatial service when the display has no exact copy', async () => { + const { controller, runSpatial } = spatialScene(); + + const baked = await controller.baked('layer'); + + expect(baked.status).toBe('ok'); + expect(runSpatial).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + height: 30, + priority: 'now', + steps: [{ amount: 40, kind: 'texture', size: 8 }], + width: 40, + }) + ); + }); + + it("draws the display's exact copy instead of running a job", async () => { + const { controller, runSpatial } = spatialScene({ displayed: true }); + + await expect(controller.rasterize('layer', { applyAdjustments: true })).resolves.toMatchObject({ status: 'ok' }); + expect(runSpatial).not.toHaveBeenCalled(); + }); + + it.each(['too-large', 'over-budget', 'error'] as const)( + 'refuses %s with the layer name, for bakes and adjusted reads alike', + async (reason) => { + const { controller, runSpatial } = spatialScene(); + runSpatial.mockResolvedValue({ status: reason } as never); + const refusal = { ownerId: 'layer', ownerName: 'Fabric', reason, status: 'adjustment-refused' }; + + await expect(controller.baked('layer')).resolves.toEqual(refusal); + await expect(controller.rasterize('layer', { applyAdjustments: true })).resolves.toEqual(refusal); + await expect(controller.blob('layer')).resolves.toEqual(refusal); + } + ); + + it('reports pixels replaced while the job ran as not ready', async () => { + let current = true; + const { controller, runSpatial } = spatialScene({ isGuardCurrent: () => current }); + runSpatial.mockImplementation((request) => { + current = false; + return Promise.resolve({ pixels: request.pixels, status: 'ok' as const }); + }); + + await expect(controller.rasterize('layer', { applyAdjustments: true })).resolves.toEqual({ status: 'not-ready' }); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.ts index 02bc1ed9b8f..f07356c0bb4 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.ts @@ -19,11 +19,18 @@ import { lookupDocumentLayer, lookupDocumentLeaf } from '@workbench/canvas-engin import { getSourceContentRect, renderableSourceOf } from '@workbench/canvas-engine/document/sources'; import { fromTRS } from '@workbench/canvas-engine/math/mat2d'; import { isEmpty, roundOut, transformBounds } from '@workbench/canvas-engine/math/rect'; -import { applyAdjustments, isIdentityAdjustments } from '@workbench/canvas-engine/render/adjustments'; +import { hasSpatialAdjustments, isIdentityAdjustments } from '@workbench/canvas-engine/render/adjustments'; +import { + type AdjustmentRefusal, + adjustmentRefusal, + adjustSurfaceExactly, + type SpatialRunner, +} from '@workbench/canvas-engine/render/exactAdjustments'; export type ExportLayerPixelsResult = | ({ status: 'ok' } & RasterReadLease) - | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' | 'aborted' }; + | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' | 'aborted' } + | AdjustmentRefusal; export interface RasterExportControllerOptions { readonly backend: RasterBackend; @@ -44,14 +51,16 @@ export interface RasterExportControllerOptions { | { status: 'ok'; lease: { release(): void } } | { status: 'over-budget'; requestedBytes: number; availableBytes: number }; /** - * The display's current adjusted copy of `surface` when it is the layer's live cache, or null; never builds one. - * Bakes draw it instead of reserving and adjusting a layer-sized copy. + * The display's current, exact adjusted copy of `surface` when it is the layer's live cache, or null; never builds + * one. Bakes draw it instead of reserving and adjusting a layer-sized copy. */ readonly peekAdjustedCacheSurface?: ( layerId: string, surface: RasterSurface, adjustments: CanvasAdjustmentsContract ) => RasterSurface | null; + /** Evaluates spatial adjustment runs; without it a spatial stack refuses `error`. */ + readonly runSpatial?: SpatialRunner; /** Keeps a layer resident and untrimmed for the lifetime of a read lease. */ readonly pin: (layerId: string) => { release(): void }; /** Resolves custom font bytes before an operation whose pixels leave the editor. */ @@ -74,10 +83,16 @@ const noReservedPixels = (result: ExportLayerPixelsResult): ReservedExportLayerP export class RasterExportController { constructor(private readonly options: RasterExportControllerOptions) {} - private applyAdjustments( + /** + * An owned, layer-local adjusted copy of the leased pixels, releasing the lease. A spatial stack is evaluated + * exactly (reusing the display's copy when it is exact for these pixels) or refused; its job yields, so the lease's + * guard is checked again afterwards. + */ + private async adjustLease( result: Extract, - shouldApply: boolean - ): ExportLayerPixelsResult { + shouldApply: boolean, + signal?: AbortSignal + ): Promise { const layer = result.guard.layer; // Identity-aware: an emptied or all-disabled stack must not reserve or copy. if (!shouldApply || layer.type !== 'raster' || isIdentityAdjustments(layer.adjustments)) { @@ -99,14 +114,36 @@ export class RasterExportController { } }; try { - const surface = this.options.backend.createSurface(result.rect.width, result.rect.height); - const ctx = surface.ctx; - ctx.setTransform(1, 0, 0, 1, 0, 0); - ctx.clearRect(0, 0, result.rect.width, result.rect.height); - ctx.drawImage(result.surface.canvas, 0, 0); - const imageData = ctx.getImageData(0, 0, result.rect.width, result.rect.height); - applyAdjustments(imageData, layer.adjustments, result.rect); - ctx.putImageData(imageData, 0, 0); + const { backend } = this.options; + const displayed = hasSpatialAdjustments(layer.adjustments) + ? (this.options.peekAdjustedCacheSurface?.(layer.id, result.surface, layer.adjustments!) ?? null) + : null; + let surface: RasterSurface; + if (displayed) { + // A copy: the display repaints its own surface in place when the stack changes. + surface = backend.createSurface(result.rect.width, result.rect.height); + surface.ctx.setTransform(1, 0, 0, 1, 0, 0); + surface.ctx.drawImage(displayed.canvas, 0, 0); + } else { + const adjusted = await adjustSurfaceExactly( + (width, height) => backend.createSurface(width, height), + result.surface, + result.rect, + layer.adjustments, + { run: this.options.runSpatial, signal } + ); + if (adjusted.status !== 'ok') { + release(); + return adjusted.status === 'aborted' + ? { status: 'aborted' } + : adjustmentRefusal(this.options.getDocument(), layer.id, adjusted.status); + } + surface = adjusted.surface; + } + if (!this.options.isGuardCurrent(result.guard)) { + release(); + return { status: 'not-ready' }; + } return { ...result, release, surface }; } catch (error) { release(); @@ -153,7 +190,7 @@ export class RasterExportController { } const liveEntry = this.options.layers.get(layerId); if (liveEntry && !liveEntry.stale && !this.options.isRasterizing(layer) && !isEmpty(liveEntry.rect)) { - return this.applyAdjustments(this.lease(layer, liveEntry), options.applyAdjustments === true); + return this.adjustLease(this.lease(layer, liveEntry), options.applyAdjustments === true, options.signal); } const contentRect = getSourceContentRect(layer, document); if (isEmpty(contentRect)) { @@ -192,7 +229,7 @@ export class RasterExportController { } const lease = this.lease(currentLayer, entry, pin); leased = true; - return this.applyAdjustments(lease, options.applyAdjustments === true); + return this.adjustLease(lease, options.applyAdjustments === true, options.signal); } finally { if (!leased) { pin.release(); @@ -236,11 +273,12 @@ export class RasterExportController { return noReservedPixels({ status: 'empty' }); } // The layer's own stack applies to layer-local pixels before the transform resamples them, as the display does, - // so bakes keep the displayed grain and color for scaled and rotated layers too. The display's current adjusted - // copy of the leased live cache is drawn as is; otherwise a layer-local copy is reserved and adjusted. + // so bakes keep the displayed grain, detail and color for scaled and rotated layers too. The display's exact + // adjusted copy of the leased live cache is drawn as is (before anything yields); otherwise a layer-local copy is + // reserved and adjusted, spatial entries included, or the bake refuses. if (options.applyAdjustments !== false && layer.type === 'raster' && !isIdentityAdjustments(layer.adjustments)) { const displayed = this.options.peekAdjustedCacheSurface?.(layer.id, raw.surface, layer.adjustments!) ?? null; - const adjusted = displayed ? { ...raw, surface: displayed } : this.applyAdjustments(raw, true); + const adjusted = displayed ? { ...raw, surface: displayed } : await this.adjustLease(raw, true, options.signal); if (adjusted.status !== 'ok') { return noReservedPixels(adjusted); } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterMemoryBudgetController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterMemoryBudgetController.ts index 56e3ca67caa..06325cb3096 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterMemoryBudgetController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterMemoryBudgetController.ts @@ -6,7 +6,8 @@ export type RasterBackgroundPurpose = | 'layer-operation' | 'thumbnail' | 'raster-export' - | 'psd-export'; + | 'psd-export' + | 'spatial-adjustment'; export interface RasterMemoryLease { release(): void; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.browser.test.ts index 890057f7bab..f79c8a2d786 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.browser.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.browser.test.ts @@ -3,6 +3,7 @@ import type { CanvasDocumentContractV3, CanvasRasterLayerContractV2 } from '@workbench/canvas-engine/contracts'; import type { Rect } from '@workbench/canvas-engine/types'; +import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { getDocumentLayer } from '@workbench/canvas-engine/document/documentIndex'; import { getSourceContentRect } from '@workbench/canvas-engine/document/sources'; @@ -11,7 +12,8 @@ import { createAdjustedSurfaceCache } from '@workbench/canvas-engine/render/adju import { createLayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; import { createDomRasterBackend } from '@workbench/canvas-engine/render/raster'; import { rasterizeSource, type RasterizeDeps } from '@workbench/canvas-engine/render/rasterizers'; -import { describe, expect, it } from 'vitest'; +import { createSpatialAdjustmentService } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import { describe, expect, it, vi } from 'vitest'; import { createLayerOperationHarness } from './layerOperationHarness.testing'; import { RasterizeLayerController } from './rasterizeLayerController'; @@ -99,7 +101,7 @@ describe('RasterizeLayerController with a position-dependent stack', () => { return [...surface.ctx.getImageData(0, 0, rect.width, rect.height).data]; }; - expect(controller.rasterize('layer')).toBe('rasterized'); + expect(await controller.rasterize('layer')).toBe('rasterized'); const paint = getDocumentLayer(h.document(), 'layer'); expect(paint).toMatchObject({ source: { type: 'paint' }, transform: { rotation: 0, scaleX: 1, scaleY: 1 } }); @@ -112,3 +114,89 @@ describe('RasterizeLayerController with a position-dependent stack', () => { expect(getDocumentLayer(h.document(), 'layer')).toEqual(layer); }); }); + +describe('RasterizeLayerController with a spatial stack', () => { + it('bakes the settled display of a rotated, scaled shape layer and drops the stack until undo', async () => { + const backend = createDomRasterBackend(); + const layer: CanvasRasterLayerContractV2 = { + ...noisyLayer({ + fill: '#808080', + height: 30, + kind: 'ellipse', + stroke: '#202020', + strokeWidth: 6, + type: 'shape', + width: 40, + }), + adjustments: [ + { amount: 120, id: 'clarity', isEnabled: true, size: 6, type: 'clarity' }, + { brightness: 0.1, contrast: 0.2, id: 'bc', isEnabled: true, type: 'brightness-contrast' }, + { amount: 80, id: 'dehaze', isEnabled: true, type: 'dehaze' }, + ], + }; + const document: CanvasDocumentContractV3 = { + background: 'transparent', + bbox: { height: 64, width: 64, x: 0, y: 0 }, + height: 64, + selectedLayerId: 'layer', + stacks: stacksFrom([layer]), + version: 3, + width: 64, + }; + const memory = new RasterMemoryBudgetController({ budgetBytes: 64 * 1024 * 1024 }); + const service = createSpatialAdjustmentService({ + reserve: (bytes) => memory.reserveOperation(bytes, { purpose: 'spatial-adjustment' }), + }); + try { + const h = createLayerOperationHarness(document, { backend }); + const rasterizeDeps = (): RasterizeDeps => ({ + backend, + documentSize: { height: 64, width: 64 }, + resolver: () => Promise.reject(new Error('parametric sources need no images')), + store: h.layers, + }); + const controller = new RasterizeLayerController({ + backend, + ctx: h.ctx, + rasterizeDeps, + runSpatial: (request) => service.run(request), + }); + + // The settled display: the layer's rasterized cache with its spatial result landed, through the transform. + const rendered = await rasterizeSource(layer.source, rasterizeDeps()); + const displayCache = createLayerCacheStore(backend); + const entry = displayCache.publishRasterized('layer', rendered.rect, rendered.surface); + const display = createAdjustedSurfaceCache(backend, undefined, { + spatial: { run: (request) => service.run(request) }, + }); + await vi.waitFor( + () => { + display.get('layer', entry, layer.adjustments); + expect(display.peek('layer', entry, layer.adjustments)).not.toBeNull(); + }, + { interval: 20, timeout: 10_000 } + ); + const adjusted = display.peek('layer', entry, layer.adjustments)!; + const displayed = (rect: Rect): number[] => { + const surface = backend.createSurface(rect.width, rect.height); + const { rotation, scaleX, scaleY, x, y } = TRANSFORM; + const matrix = fromTRS({ x, y }, rotation, scaleX, scaleY); + surface.ctx.setTransform(matrix.a, matrix.b, matrix.c, matrix.d, matrix.e - rect.x, matrix.f - rect.y); + surface.ctx.drawImage(adjusted.canvas, entry.rect.x, entry.rect.y); + return [...surface.ctx.getImageData(0, 0, rect.width, rect.height).data]; + }; + + expect(await controller.rasterize('layer')).toBe('rasterized'); + + expect(getDocumentLayer(h.document(), 'layer')).not.toHaveProperty('adjustments'); + const baked = h.layers.get('layer')!; + expect([...baked.surface.ctx.getImageData(0, 0, baked.rect.width, baked.rect.height).data]).toEqual( + displayed(baked.rect) + ); + await expect(h.history.undo()).resolves.toEqual({ status: 'applied' }); + expect(getDocumentLayer(h.document(), 'layer')).toEqual(layer); + } finally { + service.dispose(); + } + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.test.ts index 00ea624a211..9ce0357d000 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.test.ts @@ -1,8 +1,9 @@ import type { CanvasDocumentContractV3, CanvasLayerContract } from '@workbench/canvas-engine/contracts'; +import type { SpatialRunner } from '@workbench/canvas-engine/render/exactAdjustments'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { getDocumentLayer } from '@workbench/canvas-engine/document/documentIndex'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { createLayerOperationHarness } from './layerOperationHarness.testing'; import { RasterizeLayerController } from './rasterizeLayerController'; @@ -19,7 +20,7 @@ const shapeLayer: CanvasLayerContract = { type: 'raster', }; -const createHarness = (historyBytes?: number, layer: CanvasLayerContract = shapeLayer) => { +const createHarness = (historyBytes?: number, layer: CanvasLayerContract = shapeLayer, runSpatial?: SpatialRunner) => { const document: CanvasDocumentContractV3 = { background: 'transparent', bbox: { height: 100, width: 100, x: 0, y: 0 }, @@ -39,6 +40,7 @@ const createHarness = (historyBytes?: number, layer: CanvasLayerContract = shape resolver: () => Promise.reject(new Error('shapes need no images')), store: h.layers, }), + runSpatial, }); return { ...h, controller }; }; @@ -46,10 +48,10 @@ const createHarness = (historyBytes?: number, layer: CanvasLayerContract = shape const BAKED_RECT = { height: 40, width: 120, x: 10, y: 20 }; describe('RasterizeLayerController', () => { - it('bakes source and transform into paint pixels as one undo step', () => { + it('bakes source and transform into paint pixels as one undo step', async () => { const h = createHarness(); - expect(h.controller.rasterize('shape')).toBe('rasterized'); + expect(await h.controller.rasterize('shape')).toBe('rasterized'); expect(getDocumentLayer(h.document(), 'shape')).toMatchObject({ source: { bitmap: null, offset: { x: 10, y: 20 }, type: 'paint' }, transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, @@ -60,7 +62,7 @@ describe('RasterizeLayerController', () => { it('undo restores the parametric layer; redo reinstalls the same baked pixels without rerendering', async () => { const h = createHarness(); - h.controller.rasterize('shape'); + await h.controller.rasterize('shape'); const baked = h.installed[0]!; await expect(h.history.undo()).resolves.toEqual({ status: 'applied' }); @@ -72,33 +74,33 @@ describe('RasterizeLayerController', () => { expect(h.installed[1]!.rect).toEqual(baked.rect); }); - it('refuses a conversion it could never undo, leaving the layer parametric', () => { + it('refuses a conversion it could never undo, leaving the layer parametric', async () => { const h = createHarness(1024); const before = h.document(); - expect(h.controller.rasterize('shape')).toBe('over-budget'); + expect(await h.controller.rasterize('shape')).toBe('over-budget'); expect(h.document()).toBe(before); expect(h.dispatched).toEqual([]); }); - it('reports a refused conversion as failed and installs nothing', () => { + it('reports a refused conversion as failed and installs nothing', async () => { const h = createHarness(); h.state.refuse = () => true; - expect(h.controller.rasterize('shape')).toBe('failed'); + expect(await h.controller.rasterize('shape')).toBe('failed'); expect(h.installed).toEqual([]); expect(h.history.canUndo()).toBe(false); }); - it('keeps a color-only stack on the paint contract', () => { + it('keeps a color-only stack on the paint contract', async () => { const adjustments = [{ id: 'invert', isEnabled: true, type: 'invert' }] as const; const h = createHarness(undefined, { ...shapeLayer, adjustments }); - expect(h.controller.rasterize('shape')).toBe('rasterized'); + expect(await h.controller.rasterize('shape')).toBe('rasterized'); expect(getDocumentLayer(h.document(), 'shape')).toMatchObject({ adjustments, source: { type: 'paint' } }); }); - it('reserves the readback that bakes a noisy stack', () => { + it('reserves the readback that bakes a noisy stack', async () => { const noise = { amount: 0.2, distribution: 'uniform', @@ -114,17 +116,74 @@ describe('RasterizeLayerController', () => { // The 60×40 source render, plus the 120×40 baked pixels and their prepared replacement. h.state.rasterBytes = (60 * 40 + 120 * 40 * 2) * 4; - expect(h.controller.rasterize('shape')).toBe('over-budget'); + expect(await h.controller.rasterize('shape')).toBe('over-budget'); expect(h.dispatched).toEqual([]); h.state.rasterBytes += 60 * 40 * 4; - expect(h.controller.rasterize('shape')).toBe('rasterized'); + expect(await h.controller.rasterize('shape')).toBe('rasterized'); expect(getDocumentLayer(h.document(), 'shape')).not.toHaveProperty('adjustments'); }); - it('refuses missing, locked and already-painted layers', () => { + it('refuses missing, locked and already-painted layers', async () => { const h = createHarness(); - expect(h.controller.rasterize('nope')).toBe('missing'); - h.controller.rasterize('shape'); - expect(h.controller.rasterize('shape')).toBe('unsupported'); + expect(await h.controller.rasterize('nope')).toBe('missing'); + await h.controller.rasterize('shape'); + expect(await h.controller.rasterize('shape')).toBe('unsupported'); + }); + + describe('with a spatial stack', () => { + const spatial = { + ...shapeLayer, + adjustments: [ + { amount: 50, id: 'texture', isEnabled: true, size: 6, type: 'texture' }, + { id: 'invert', isEnabled: true, type: 'invert' }, + ], + } as const satisfies CanvasLayerContract; + + it('bakes the whole stack on the layer-local render in the spatial service and drops it', async () => { + const runSpatial = vi.fn((request) => Promise.resolve({ pixels: request.pixels, status: 'ok' })); + const h = createHarness(undefined, spatial, runSpatial); + + expect(await h.controller.rasterize('shape')).toBe('rasterized'); + + // The 60×40 source render, before the transform scales it. + expect(runSpatial).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ height: 40, priority: 'now', width: 60 }) + ); + expect(getDocumentLayer(h.document(), 'shape')).not.toHaveProperty('adjustments'); + }); + + it.each(['too-large', 'over-budget', 'error'] as const)( + 'refuses %s with the layer name, leaving the layer parametric', + async (reason) => { + const h = createHarness(undefined, spatial, () => Promise.resolve({ status: reason })); + + expect(await h.controller.rasterize('shape')).toEqual({ + ownerId: 'shape', + ownerName: 'Shape', + reason, + status: 'adjustment-refused', + }); + expect(h.dispatched).toEqual([]); + expect(h.reservedBytes()).toBe(0); + } + ); + + it('refuses without a spatial runner rather than bake without the spatial entries', async () => { + const h = createHarness(undefined, spatial); + + expect(await h.controller.rasterize('shape')).toMatchObject({ reason: 'error', status: 'adjustment-refused' }); + expect(h.dispatched).toEqual([]); + }); + + it('lands nothing when an edit landed while the job ran', async () => { + const h = createHarness(undefined, spatial, (request) => { + h.ctx.dispatch({ id: null, type: 'setCanvasSelectedLayer' }); + return Promise.resolve({ pixels: request.pixels, status: 'ok' }); + }); + + expect(await h.controller.rasterize('shape')).toBe('not-ready'); + expect(getDocumentLayer(h.document(), 'shape')).toMatchObject({ source: { type: 'shape' } }); + expect(h.history.canUndo()).toBe(false); + }); }); }); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.ts index 8a2cfd292c4..68ba5da7e2e 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.ts @@ -14,7 +14,14 @@ import { getDocumentLayer } from '@workbench/canvas-engine/document/documentInde import { getSourceContentRect, isEmptyPolygonShape } from '@workbench/canvas-engine/document/sources'; import { collectHistoryMediaRefs, HISTORY_ENTRY_OVERHEAD_BYTES } from '@workbench/canvas-engine/history/history'; import { roundOut, transformBounds } from '@workbench/canvas-engine/math/rect'; -import { applyAdjustments, isPositionDependentAdjustments } from '@workbench/canvas-engine/render/adjustments'; +import { hasSpatialAdjustments, isPositionDependentAdjustments } from '@workbench/canvas-engine/render/adjustments'; +import { + type AdjustmentRefusal, + adjustExactly, + adjustmentRefusal, + imageDataOf, + type SpatialRunner, +} from '@workbench/canvas-engine/render/exactAdjustments'; import { rasterizeSource } from '@workbench/canvas-engine/render/rasterizers'; import { bakeMatrix } from '@workbench/canvas-engine/transform/transformMath'; @@ -32,14 +39,17 @@ export type RasterizeLayerResult = | 'rasterized' | SubsetOf | LayerEditRefusal - | 'failed'; + | 'failed' + | AdjustmentRefusal; export interface RasterizeLayerControllerOptions { readonly ctx: Pick< CanvasMutationContext, - 'applyStep' | 'begin' | 'getDocument' | 'installPrepared' | 'preparePixels' | 'reserveRaster' + 'applyStep' | 'begin' | 'getDocument' | 'getEditRevision' | 'installPrepared' | 'preparePixels' | 'reserveRaster' >; readonly backend: RasterBackend; + /** Evaluates spatial adjustment runs; without it a spatial stack refuses `error`. */ + readonly runSpatial?: SpatialRunner; readonly rasterizeDeps: (document: CanvasDocumentContractV3) => RasterizeDeps; } @@ -53,11 +63,13 @@ export class RasterizeLayerController { * Bakes the layer's source and transform into paint pixels as one undo step. Undo restores the parametric * contract, which rerasterizes from its source; redo reinstalls the baked pixels, never a rerender. * - * A position-dependent stack (noise) is baked too, on the layer-local render before the transform resamples it as - * the display does, and leaves the contract: kept, it would place its grain on the baked pixel grid. A color-only - * stack stays on the contract and keeps applying to the baked pixels. + * A position-dependent stack (noise) or a spatial one (clarity, texture, dehaze) is baked too, all of it, on the + * layer-local render before the transform resamples it as the display does, and leaves the contract: kept, it would + * place its grain on, or measure its detail in, the baked pixel grid. A color-only stack stays on the contract and + * keeps applying to the baked pixels. A spatial bake runs in the worker; the result lands only if no edit landed + * meanwhile, and a stack that cannot run refuses. */ - rasterize(layerId: string): RasterizeLayerResult { + async rasterize(layerId: string): Promise { const { ctx } = this.deps; if (this.disposed) { return 'not-ready'; @@ -81,7 +93,9 @@ export class RasterizeLayerController { return 'unsupported'; } const contentRect = getSourceContentRect(layer, document); - const bakesAdjustments = isPositionDependentAdjustments(layer.adjustments); + const bakesAdjustments = + isPositionDependentAdjustments(layer.adjustments) || hasSpatialAdjustments(layer.adjustments); + const revision = ctx.getEditRevision(); const matrix = bakeMatrix(layer.transform); const bakedRect = roundOut(transformBounds(matrix, contentRect)); const txn = ctx.begin({ historyBytes: rgbaBytes(bakedRect) + HISTORY_ENTRY_OVERHEAD_BYTES }); @@ -103,8 +117,18 @@ export class RasterizeLayerController { return 'over-budget'; } const imageData = scratch.ctx.getImageData(0, 0, width, height); - applyAdjustments(imageData, layer.adjustments, contentRect); - scratch.ctx.putImageData(imageData, 0, 0); + const adjusted = await adjustExactly(imageData, layer.adjustments, contentRect, { run: this.deps.runSpatial }); + if (adjusted.status !== 'ok') { + return adjusted.status === 'aborted' ? 'not-ready' : adjustmentRefusal(document, layerId, adjusted.status); + } + if (this.disposed || !txn.isCurrent() || ctx.getEditRevision() !== revision) { + return 'not-ready'; + } + scratch.ctx.putImageData( + adjusted.data === imageData.data ? imageData : imageDataOf(adjusted.data, width, height), + 0, + 0 + ); } const baked = this.deps.backend.createSurface(bakedRect.width, bakedRect.height); baked.ctx.setTransform(1, 0, 0, 1, 0, 0); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.test.ts index 9a732b619b9..9ffd7f2eaa4 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.test.ts @@ -46,7 +46,7 @@ describe('ThumbnailController budget', () => { getMaskPattern: () => null, isDisposed: () => false, isSupportedSource: () => true, - peekAdjustedSurface: () => null, + peekDisplayedSurface: () => null, projectId: 'p', rasterize, reportError: vi.fn(), @@ -120,7 +120,7 @@ describe('ThumbnailController adjusted rasters', () => { getMaskPattern: () => null, isDisposed: () => false, isSupportedSource: () => true, - peekAdjustedSurface: (_layer, peeked) => peek(peeked.surface), + peekDisplayedSurface: (_layer, peeked) => peek(peeked.surface), projectId: 'p', rasterize: vi.fn(), reportError: vi.fn(), diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.ts index a0c3a3f18c0..c96d71b5d4a 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.ts @@ -25,8 +25,11 @@ export interface ThumbnailControllerOptions { readonly getMaskPattern: (style: string, color: string) => RasterSurface | null; readonly isDisposed: () => boolean; readonly isSupportedSource: (source: CanvasLayerSourceContract) => boolean; - /** The display's current adjusted copy of a raster layer's cache, or null; never builds a layer-sized surface. */ - readonly peekAdjustedSurface: (layer: CanvasLayerContract, entry: LayerCacheEntry) => RasterSurface | null; + /** + * What the display shows for a raster layer's cache (a spatial interim included), or null; never builds a + * layer-sized surface. + */ + readonly peekDisplayedSurface: (layer: CanvasLayerContract, entry: LayerCacheEntry) => RasterSurface | null; readonly rasterize: ( layer: CanvasLayerContract, document: CanvasDocumentContractV3 @@ -42,7 +45,8 @@ export interface ThumbnailControllerOptions { } /** - * Bounded fallback without a display copy: the stack adjusts the downscaled pixels, its noise on the thumbnail's own + * Bounded fallback without a display copy: the stack's per-pixel part adjusts the downscaled pixels (spatial entries + * read whole-layer neighborhoods a thumbnail cannot reproduce, so it leaves them out), its noise on the thumbnail's own * grid. A downscale box-averages about 1/scale × 1/scale pixels, which leaves grain spanning `noiseGrainSpan` pixels * `scale × span` of its σ once blocks are larger than the grain (exact for white noise), and keeps it when they are * smaller; the grain itself shrinks to `size × scale` thumbnail pixels, at least one. An approximation: exact only in @@ -97,8 +101,9 @@ export class ThumbnailController { thumbnail.ctx.globalAlpha = 1; thumbnail.ctx.globalCompositeOperation = 'source-over'; const adjusted = layer.type === 'raster' && !isIdentityAdjustments(layer.adjustments) ? layer.adjustments : null; - // Downscale the display's adjusted copy when it is current, so the thumbnail shows the canvas's grain and color. - const displayed = adjusted ? this.deps.peekAdjustedSurface(layer, entry) : null; + // Downscale what the display shows when it is current, so the thumbnail matches the canvas's grain, color and + // spatial adjustments; a spatial result landing republishes the thumbnail. + const displayed = adjusted ? this.deps.peekDisplayedSurface(layer, entry) : null; thumbnail.ctx.drawImage((displayed ?? entry.surface).canvas, 0, 0, width, height); const checker = context.createPattern(this.deps.getCheckerboard().canvas as CanvasImageSource, 'repeat'); if (checker) { diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/transformEditingController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/transformEditingController.ts index c02cf110e6c..3708e9ef7a2 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/transformEditingController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/transformEditingController.ts @@ -42,7 +42,7 @@ export interface TransformEditingControllerOptions { const IDENTITY: LayerTransform = { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }; -const isRefusal = (status: string): status is CanvasEditRefusal => +const isRefusal = (status: string): status is Extract => status === 'busy' || status === 'gesture-active' || status === 'not-ready' || status === 'over-budget'; const unchanged = (left: LayerTransform, right: LayerTransform): boolean => diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerChangeDecision.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerChangeDecision.ts index a23dfa65658..59926c7945b 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerChangeDecision.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerChangeDecision.ts @@ -5,6 +5,14 @@ import { getLayerThumbnailDisplayKey } from '@workbench/canvas-engine/render/thu /** The display token given to a thumbnail whose appearance changed without its pixels. */ const FIRST_DISPLAY_TOKEN = -1; +/** + * The thumbnail version to publish when a layer's appearance changed without a cache publication. Cache versions + * are positive, so a negative display token can never collide with the next cache publication and suppress its + * redraw. Each further change steps further negative rather than resetting. + */ +export const nextThumbnailDisplayToken = (current: number | undefined): number => + current !== undefined && current < 0 ? current - 1 : FIRST_DISPLAY_TOKEN; + export interface LayerChangeInput { /** The layer as it now stands, or `undefined` when it is no longer in the document. */ readonly layer: CanvasLayerContract | undefined; @@ -61,16 +69,9 @@ export const decideLayerChange = (input: LayerChangeInput): LayerChangeDecision if (key === input.currentThumbnailKey) { return { kind: 'appearance-only', thumbnailDisplay: null }; } - // Cache versions are positive, so a negative display token can never collide - // with the next cache publication and suppress its redraw. Each further - // appearance change steps further negative rather than resetting. - const current = input.currentThumbnailVersion; return { kind: 'appearance-only', - thumbnailDisplay: { - key, - version: current !== undefined && current < 0 ? current - 1 : FIRST_DISPLAY_TOKEN, - }, + thumbnailDisplay: { key, version: nextThumbnailDisplayToken(input.currentThumbnailVersion) }, }; } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerFactories.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerFactories.ts index 6e9fa753d04..539418f77a3 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerFactories.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerFactories.ts @@ -2,6 +2,8 @@ import type { CanvasControlLayerContract, CanvasImageRef, CanvasInpaintMaskLayerContract, + CanvasLayerBaseContract, + CanvasLayerSourceContract, CanvasMaskFillContract, CanvasRegionalGuidanceLayerContract, } from '@workbench/canvas-engine/contracts'; @@ -108,3 +110,28 @@ export const createControlLayer = ( withTransparencyEffect: true, }; }; + +/** + * A control layer made from another layer, keeping its appearance and flags with a default adapter. `content` either + * places baked pixels at `rect` under an identity transform, or reuses a source and transform as they are. + */ +export const createControlLayerFrom = ( + from: CanvasLayerBaseContract, + options: { + id: string; + name: string; + base: string | null; + model: string | null; + content: { readonly rect: Rect } | { readonly source: CanvasLayerSourceContract }; + } +): CanvasControlLayerContract => ({ + ...createControlLayer(options.name, options.id, options.base, options.model), + blendMode: from.blendMode, + ...(from.colorLabel ? { colorLabel: from.colorLabel } : {}), + isEnabled: from.isEnabled, + isLocked: from.isLocked, + opacity: from.opacity, + ...('rect' in options.content + ? { source: { bitmap: null, offset: { x: options.content.rect.x, y: options.content.rect.y }, type: 'paint' } } + : { source: structuredClone(options.content.source), transform: { ...from.transform } }), +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/editing/controlPixelEdit.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/editing/controlPixelEdit.browser.test.ts index ff701f02ae0..e4fbdb24836 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/editing/controlPixelEdit.browser.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/editing/controlPixelEdit.browser.test.ts @@ -113,8 +113,11 @@ const setup = ( getActiveProjectId: () => 'project', getAdjustedSurface: (candidate, candidateEntry) => candidate.type === 'raster' ? adjusted.get(candidate.id, candidateEntry, candidate.adjustments) : null, + getAdjustmentStatus: () => undefined, getDocument: () => document, getTransformSession: () => null, + peekAdjustedSurface: (candidate, candidateEntry) => + candidate.type === 'raster' ? adjusted.peek(candidate.id, candidateEntry, candidate.adjustments) : null, installPrepared: (prepared) => { layers.installReplacement(prepared); }, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.test.ts index cb314554736..4699a6065b2 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.test.ts @@ -1,4 +1,6 @@ +import type { CanvasEditRefusal } from '@workbench/canvas-engine/capabilities'; import type { + CanvasAdjustmentsContract, CanvasControlLayerContract, CanvasDocumentContractV3, CanvasInpaintMaskLayerContract, @@ -30,12 +32,15 @@ import { type EngineTestAction = WorkbenchAction | CanvasProjectMutation; -import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; -import { getDocumentLeaves } from '@workbench/canvas-engine/document/documentIndex'; +import type { SpatialAdjustmentExecutor } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; + +import { groupContract, stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { getDocumentLayer, getDocumentLeaves } from '@workbench/canvas-engine/document/documentIndex'; import { stackTopAnchor } from '@workbench/canvas-engine/document/insertionAnchors.testStub'; import { haveSameStructure } from '@workbench/canvas-engine/document/layerStacks'; import { DEFAULT_CHECKER_COLORS } from '@workbench/canvas-engine/render/compositor'; import { createTestStubRasterBackend } from '@workbench/canvas-engine/render/raster.testStub'; +import { createInlineSpatialExecutor } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService.testing'; import { canvasApplicationPort } from '@workbench/canvas-operations/applicationPort'; import { createCanvasEngine as createApplicationCanvasEngine, @@ -172,8 +177,8 @@ vi.mock('@workbench/canvas-engine/render/adjustedSurfaceCache', async (importOri const actual = await importOriginal(); return { ...actual, - createAdjustedSurfaceCache: (backend: Parameters[0]) => { - const cache = actual.createAdjustedSurfaceCache(backend); + createAdjustedSurfaceCache: (...args: Parameters) => { + const cache = actual.createAdjustedSurfaceCache(...args); return { ...cache, delete: (layerId: string) => { @@ -4618,6 +4623,122 @@ describe('drawLayerThumbnail', () => { engine.lifecycle.dispose(); }); + it("publishes a spatial layer's adjustment status and redraws its thumbnail when the result lands", async () => { + const raf = createControllableRaf(); + vi.stubGlobal('requestAnimationFrame', raf.requestFrame); + vi.stubGlobal('cancelAnimationFrame', raf.cancelFrame); + const layer = { + ...rasterLayer('a'), + adjustments: [{ amount: 40, id: 'adj-cl', isEnabled: true, size: 40, type: 'clarity' as const }], + } as CanvasRasterLayerContractV2; + const { store } = createReactiveStore({ ...makeDoc(), stacks: stacksFrom([layer]) }); + const engine = createCanvasEngine({ + backend: createTestStubRasterBackend(), + createSpatialExecutor: createInlineSpatialExecutor, + imageResolver: () => Promise.resolve(new Blob()), + projectId: 'p1', + store, + }); + engine.surface.attach(createInputCanvas().element, createInputCanvas().element); + raf.flush(); + await flushMicrotasks(); + const statuses: string[] = []; + engine.interaction.subscribeLayerAdjustmentStatus('a', () => + statuses.push(engine.interaction.getLayerAdjustmentStatus('a')) + ); + const thumbnailBefore = engine.interaction.getLayerThumbnailVersion('a'); + raf.flush(); + expect(engine.interaction.getLayerAdjustmentStatus('a')).toBe('computing'); + + await flushMicrotasks(); + expect(statuses).toEqual(['computing', 'idle']); + const thumbnailAfter = engine.interaction.getLayerThumbnailVersion('a'); + expect(thumbnailAfter).not.toBe(thumbnailBefore); + expect(thumbnailAfter).toBeLessThan(0); + engine.lifecycle.dispose(); + }); + + it("clears a layer's status once its stack loses its spatial entries, though nothing draws it", async () => { + const raf = createControllableRaf(); + vi.stubGlobal('requestAnimationFrame', raf.requestFrame); + vi.stubGlobal('cancelAnimationFrame', raf.cancelFrame); + const layer = { + ...rasterLayer('a'), + adjustments: [{ amount: 40, id: 'adj-cl', isEnabled: true, size: 40, type: 'clarity' as const }], + } as CanvasRasterLayerContractV2; + const reactive = createReactiveStore({ ...makeDoc(), stacks: stacksFrom([layer]) }); + const engine = createCanvasEngine({ + backend: createTestStubRasterBackend(), + createSpatialExecutor: createInlineSpatialExecutor, + imageResolver: () => Promise.resolve(new Blob()), + projectId: 'p1', + store: reactive.store, + }); + engine.surface.attach(createInputCanvas().element, createInputCanvas().element); + raf.flush(); + await flushMicrotasks(); + raf.flush(); + expect(engine.interaction.getLayerAdjustmentStatus('a')).toBe('computing'); + + reactive.setDocument({ ...makeDoc(), stacks: stacksFrom([{ ...layer, adjustments: [] }]) }); + expect(engine.interaction.getLayerAdjustmentStatus('a')).toBe('idle'); + engine.lifecycle.dispose(); + }); + + describe('spatial group adjustments', () => { + const clarityGroup = (adjustments: CanvasAdjustmentsContract) => + stacksFrom([groupContract('g', [rasterLayer('a')], { adjustments })]); + const CLARITY: CanvasAdjustmentsContract = [ + { amount: 40, id: 'adj-cl', isEnabled: true, size: 40, type: 'clarity' }, + ]; + + const startEngine = async () => { + const raf = createControllableRaf(); + vi.stubGlobal('requestAnimationFrame', raf.requestFrame); + vi.stubGlobal('cancelAnimationFrame', raf.cancelFrame); + const reactive = createReactiveStore({ ...makeDoc(), stacks: clarityGroup(CLARITY) }); + const engine = createCanvasEngine({ + backend: createTestStubRasterBackend(), + createSpatialExecutor: createInlineSpatialExecutor, + imageResolver: () => Promise.resolve(new Blob()), + projectId: 'p1', + store: reactive.store, + }); + engine.surface.attach(createInputCanvas().element, createInputCanvas().element); + raf.flush(); + await flushMicrotasks(); + const statuses: string[] = []; + engine.interaction.subscribeLayerAdjustmentStatus('g', () => + statuses.push(engine.interaction.getLayerAdjustmentStatus('g')) + ); + return { engine, raf, reactive, statuses }; + }; + + it('reports status by group id and redraws the overview when the result lands', async () => { + const { engine, raf, statuses } = await startEngine(); + const epoch = engine.interaction.get('layerPixelEpoch'); + raf.flush(); + expect(engine.interaction.getLayerAdjustmentStatus('g')).toBe('computing'); + + await flushMicrotasks(); + expect(statuses).toEqual(['computing', 'idle']); + expect(engine.interaction.get('layerPixelEpoch')).toBeGreaterThan(epoch); + engine.lifecycle.dispose(); + }); + + it('clears the status of a group whose stack loses its spatial entries before its result lands', async () => { + const { engine, raf, reactive, statuses } = await startEngine(); + raf.flush(); + expect(engine.interaction.getLayerAdjustmentStatus('g')).toBe('computing'); + + reactive.setDocument({ ...makeDoc(), stacks: clarityGroup([]) }); + expect(engine.interaction.getLayerAdjustmentStatus('g')).toBe('idle'); + await flushMicrotasks(); + expect(statuses).toEqual(['computing', 'idle']); + engine.lifecycle.dispose(); + }); + }); + it.each([ ['control transparency', { ...controlLayerForThumbnail('a'), withTransparencyEffect: true }, 'getImageData'], ['inpaint mask fill', thumbnailMaskLayer('a', 'inpaint_mask'), 'source-in'], @@ -5357,7 +5478,7 @@ describe('mergeLayerDown', () => { raf.flush(); const surfacesBeforeMerge = surfaces.length; - expect(engine.layers.mergeLayerDown('upper')).toBe('merged'); + expect(await engine.layers.mergeLayerDown('upper')).toBe('merged'); // The reducer is asked to collapse the two layers into a paint layer. const mergeCall = dispatch.mock.calls @@ -5420,7 +5541,7 @@ describe('mergeLayerDown', () => { engine.lifecycle.dispose(); }); - it('is a no-op for the bottom-most layer (nothing below)', () => { + it('is a no-op for the bottom-most layer (nothing below)', async () => { const raf = createControllableRaf(); vi.stubGlobal('requestAnimationFrame', raf.requestFrame); vi.stubGlobal('cancelAnimationFrame', raf.cancelFrame); @@ -5438,7 +5559,7 @@ describe('mergeLayerDown', () => { engine.surface.attach(screen.element, overlay.element); raf.flush(); - expect(engine.layers.mergeLayerDown('below')).toBe('unsupported'); + expect(await engine.layers.mergeLayerDown('below')).toBe('unsupported'); expect(dispatch.mock.calls.some((call) => (call[0] as EngineTestAction).type === 'mergeCanvasLayersDown')).toBe( false ); @@ -5448,7 +5569,7 @@ describe('mergeLayerDown', () => { it.each(['upper', 'below'] as const)( "is a no-op when the %s layer is locked (merge is not undoable; must mirror the paint tool's locked-target refusal)", - (lockedId) => { + async (lockedId) => { const raf = createControllableRaf(); vi.stubGlobal('requestAnimationFrame', raf.requestFrame); vi.stubGlobal('cancelAnimationFrame', raf.cancelFrame); @@ -5473,7 +5594,7 @@ describe('mergeLayerDown', () => { engine.surface.attach(screen.element, overlay.element); raf.flush(); - expect(engine.layers.mergeLayerDown('upper')).toBe('unsupported'); + expect(await engine.layers.mergeLayerDown('upper')).toBe('unsupported'); expect(dispatch.mock.calls.some((call) => (call[0] as EngineTestAction).type === 'mergeCanvasLayersDown')).toBe( false ); @@ -5534,7 +5655,7 @@ describe('mergeLayerDown', () => { raf.flush(); const surfacesBeforeMerge = surfaces.length; - expect(engine.layers.mergeLayerDown('upper')).toBe('merged'); + expect(await engine.layers.mergeLayerDown('upper')).toBe('merged'); // Single-dispatch collapse still happens (undo semantics unchanged). expect(dispatch.mock.calls.some((call) => (call[0] as EngineTestAction).type === 'mergeCanvasLayersDown')).toBe( @@ -5605,7 +5726,7 @@ describe('mergeLayerDown', () => { raf.flush(); const surfacesBeforeMerge = surfaces.length; - expect(engine.layers.mergeLayerDown('upper')).toBe('merged'); + expect(await engine.layers.mergeLayerDown('upper')).toBe('merged'); // The collapse still dispatches (the upper layer is removed). expect(dispatch.mock.calls.some((call) => (call[0] as EngineTestAction).type === 'mergeCanvasLayersDown')).toBe( @@ -5627,7 +5748,7 @@ describe('mergeLayerDown', () => { it.each([ { label: 'mask above a raster layer', maskId: 'upper' as const }, { label: 'a raster layer above a mask', maskId: 'below' as const }, - ])('is a no-op for $label (document unchanged, no dispatch)', ({ maskId }) => { + ])('is a no-op for $label (document unchanged, no dispatch)', async ({ maskId }) => { const raf = createControllableRaf(); vi.stubGlobal('requestAnimationFrame', raf.requestFrame); vi.stubGlobal('cancelAnimationFrame', raf.cancelFrame); @@ -5650,7 +5771,7 @@ describe('mergeLayerDown', () => { engine.surface.attach(screen.element, overlay.element); raf.flush(); - expect(engine.layers.mergeLayerDown('upper')).toBe('unsupported'); + expect(await engine.layers.mergeLayerDown('upper')).toBe('unsupported'); expect(dispatch.mock.calls.some((call) => (call[0] as EngineTestAction).type === 'mergeCanvasLayersDown')).toBe( false ); @@ -5661,7 +5782,7 @@ describe('mergeLayerDown', () => { engine.lifecycle.dispose(); }); - it('is a no-op for mask-above-mask (document unchanged, no dispatch)', () => { + it('is a no-op for mask-above-mask (document unchanged, no dispatch)', async () => { const raf = createControllableRaf(); vi.stubGlobal('requestAnimationFrame', raf.requestFrame); vi.stubGlobal('cancelAnimationFrame', raf.cancelFrame); @@ -5684,7 +5805,7 @@ describe('mergeLayerDown', () => { engine.surface.attach(screen.element, overlay.element); raf.flush(); - expect(engine.layers.mergeLayerDown('upper')).toBe('unsupported'); + expect(await engine.layers.mergeLayerDown('upper')).toBe('unsupported'); expect(dispatch.mock.calls.some((call) => (call[0] as EngineTestAction).type === 'mergeCanvasLayersDown')).toBe( false ); @@ -6165,6 +6286,62 @@ describe('extract masked canvas area', () => { engine.lifecycle.dispose(); }); + describe('with a spatial stack', () => { + const spatialSetup = (createSpatialExecutor: () => SpatialAdjustmentExecutor) => { + const raf = createControllableRaf(); + vi.stubGlobal('requestAnimationFrame', raf.requestFrame); + vi.stubGlobal('cancelAnimationFrame', raf.cancelFrame); + const document = maskedDoc(); + const leaves = getDocumentLeaves(document); + document.stacks = stacksFrom( + leaves.map((layer) => + layer.id === 'upper' + ? { ...layer, adjustments: [{ amount: 50, id: 'clarity', isEnabled: true, size: 10, type: 'clarity' }] } + : layer + ) + ); + const { projectId, store } = createReducerBackedStore(document); + const engine = createCanvasEngine({ + backend: createTestStubRasterBackend(), + createSpatialExecutor, + imageResolver: () => Promise.resolve(new Blob()), + projectId, + store, + }); + engine.surface.attach(createFakeCanvas().element, createFakeCanvas().element); + return { engine, leafCount: leaves.length, raf }; + }; + + it('extracts a layer whose spatial stack runs', async () => { + const { engine, raf } = spatialSetup(createInlineSpatialExecutor); + raf.flush(); + await flushMicrotasks(); + raf.flush(); + + expect((await engine.exports.extractMaskedArea('mask')).status).toBe('extracted'); + engine.lifecycle.dispose(); + }); + + it('refuses with the layer name when the stack cannot run, extracting nothing', async () => { + const { engine, leafCount, raf } = spatialSetup(() => ({ + dispose: () => undefined, + run: (job) => Promise.resolve({ buffer: job.buffer, status: 'too-large' }), + })); + raf.flush(); + await flushMicrotasks(); + raf.flush(); + + expect(await engine.exports.extractMaskedArea('mask')).toEqual({ + ownerId: 'upper', + ownerName: expect.any(String), + reason: 'too-large', + status: 'adjustment-refused', + }); + expect(getDocumentLeaves(engine.document.getDocument()!)).toHaveLength(leafCount); + engine.lifecycle.dispose(); + }); + }); + it('does not extract stale contributor pixels after a source change during await', async () => { const raf = createControllableRaf(); vi.stubGlobal('requestAnimationFrame', raf.requestFrame); @@ -7348,7 +7525,7 @@ describe('mergeVisibleRasterLayers', () => { raf.flush(); expect(blendModeOf('upper')).toBe('screen'); - expect(engine.layers.mergeLayerDown('upper')).toBe('merged'); + expect(await engine.layers.mergeLayerDown('upper')).toBe('merged'); expect(session.isActive()).toBe(false); expect(await engine.history.undo()).toBe('applied'); @@ -8179,11 +8356,11 @@ describe('rasterizeLayer (parametric → paint)', () => { const convertCalls = (dispatch: Mock): EngineTestAction[] => dispatch.mock.calls.map((call) => call[0] as EngineTestAction).filter((a) => a.type === 'convertCanvasLayer'); - it('bakes to a CONTENT-sized paint layer at identity and dispatches convertCanvasLayer with the offset', () => { + it('bakes to a CONTENT-sized paint layer at identity and dispatches convertCanvasLayer with the offset', async () => { const { dispatch, engine, surfaces } = setup(shapeLayerDoc()); const before = surfaces.length; - expect(engine.layers.rasterizeLayer('shape1')).toBe('rasterized'); + expect(await engine.layers.rasterizeLayer('shape1')).toBe('rasterized'); const converts = convertCalls(dispatch); expect(converts).toHaveLength(1); @@ -8210,7 +8387,7 @@ describe('rasterizeLayer (parametric → paint)', () => { it('undo re-converts to the ORIGINAL parametric source (no pixel snapshot)', async () => { const { dispatch, engine } = setup(shapeLayerDoc()); - engine.layers.rasterizeLayer('shape1'); + await engine.layers.rasterizeLayer('shape1'); await engine.history.undo(); const converts = convertCalls(dispatch); @@ -8247,7 +8424,7 @@ describe('rasterizeLayer (parametric → paint)', () => { // memory budget. const { engine, surfaces } = setup(shapeLayerDoc()); - expect(engine.layers.rasterizeLayer('shape1')).toBe('rasterized'); + expect(await engine.layers.rasterizeLayer('shape1')).toBe('rasterized'); const afterApply = surfaces.length; await engine.history.undo(); @@ -8264,22 +8441,22 @@ describe('rasterizeLayer (parametric → paint)', () => { engine.lifecycle.dispose(); }); - it('rasterizes a gradient layer', () => { + it('rasterizes a gradient layer', async () => { const doc = shapeLayerDoc(); doc.stacks.raster[0] = { ...(doc.stacks.raster[0] as CanvasLayerContract), source: { angle: 45, kind: 'linear', stops: [{ color: '#000', offset: 0 }], type: 'gradient' }, } as CanvasLayerContract; const { dispatch, engine } = setup(doc); - expect(engine.layers.rasterizeLayer('shape1')).toBe('rasterized'); + expect(await engine.layers.rasterizeLayer('shape1')).toBe('rasterized'); expect(convertCalls(dispatch)).toHaveLength(1); engine.lifecycle.dispose(); }); - it('is a no-op for a locked layer, a missing layer, and a non-parametric (paint) layer', () => { + it('is a no-op for a locked layer, a missing layer, and a non-parametric (paint) layer', async () => { const { dispatch, engine } = setup(shapeLayerDoc({ isLocked: true })); - expect(engine.layers.rasterizeLayer('shape1')).toBe('locked'); - expect(engine.layers.rasterizeLayer('nope')).toBe('missing'); + expect(await engine.layers.rasterizeLayer('shape1')).toBe('locked'); + expect(await engine.layers.rasterizeLayer('nope')).toBe('missing'); expect(convertCalls(dispatch)).toHaveLength(0); engine.lifecycle.dispose(); @@ -8289,7 +8466,7 @@ describe('rasterizeLayer (parametric → paint)', () => { source: { bitmap: null, type: 'paint' }, } as CanvasLayerContract; const paint = setup(paintDoc); - expect(paint.engine.layers.rasterizeLayer('shape1')).toBe('unsupported'); + expect(await paint.engine.layers.rasterizeLayer('shape1')).toBe('unsupported'); paint.engine.lifecycle.dispose(); }); @@ -8368,7 +8545,7 @@ describe('rasterizeLayer (parametric → paint)', () => { const harness = setupWithRealBitmapStore(doc); const { dispatch, engine, raf, setDocument } = harness; - expect(engine.layers.rasterizeLayer('shape1')).toBe('rasterized'); + expect(await engine.layers.rasterizeLayer('shape1')).toBe('rasterized'); doc = applyLastConvert(doc, dispatch); setDocument(doc); raf.flush(); @@ -8425,7 +8602,7 @@ describe('rasterizeLayer (parametric → paint)', () => { const { dispatch, engine, raf, setDocument, uploadImage } = setupWithRealBitmapStore(doc); // Rasterize → bake to paint → flush lands img-x. - expect(engine.layers.rasterizeLayer('shape1')).toBe('rasterized'); + expect(await engine.layers.rasterizeLayer('shape1')).toBe('rasterized'); doc = applyLastConvert(doc, dispatch); setDocument(doc); raf.flush(); @@ -8498,7 +8675,7 @@ describe('rasterizeLayer (parametric → paint)', () => { let doc = shapeLayerDoc(); const { dispatch, engine, raf, setDocument, uploadImage } = setupWithRealBitmapStore(doc); - expect(engine.layers.rasterizeLayer('shape1')).toBe('rasterized'); + expect(await engine.layers.rasterizeLayer('shape1')).toBe('rasterized'); doc = applyLastConvert(doc, dispatch); setDocument(doc); raf.flush(); @@ -14217,6 +14394,168 @@ describe('document mirror wiring: prop vs source change (paint-pixel survival)', engine.lifecycle.dispose(); }); + describe('raster layers whose adjustments reach the control stack', () => { + const CLARITY: CanvasAdjustmentsContract = [ + { amount: 50, id: 'clarity', isEnabled: true, size: 10, type: 'clarity' }, + ]; + const adjustedEngine = (createSpatialExecutor: () => SpatialAdjustmentExecutor) => { + // Disabled: conversions and copies act on the layer's own pixels whether or not it contributes. + const layer = { ...rasterLayer('a'), adjustments: CLARITY, isEnabled: false } as CanvasRasterLayerContractV2; + const { dispatch, projectId, store } = createReducerBackedStore({ ...makeDoc(), stacks: stacksFrom([layer]) }); + const engine = createCanvasEngine({ + backend: createTestStubRasterBackend(), + createSpatialExecutor, + getDefaultControlModel: (base) => `${base}-control`, + getMainModelBase: () => 'sdxl', + imageResolver: () => Promise.resolve(new Blob()), + projectId, + store, + }); + return { dispatch, engine, layer: getDocumentLayer(engine.document.getDocument(), 'a')! }; + }; + const tooLarge = () => ({ + dispose: () => undefined, + run: (job: { buffer: ArrayBuffer }) => Promise.resolve({ buffer: job.buffer, status: 'too-large' as const }), + }); + + it('converts in place with baked pixels at an identity transform; undo restores the stack', async () => { + const { engine, layer } = adjustedEngine(createInlineSpatialExecutor); + + expect(await engine.layers.convertLayerToControl('Convert to control', layer)).toBe('converted'); + expect(getDocumentLayer(engine.document.getDocument(), 'a')).toMatchObject({ + adapter: { model: 'sdxl-control' }, + isEnabled: false, + source: { bitmap: null, offset: { x: 0, y: 0 }, type: 'paint' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'control', + }); + expect(engine.history.getEntries().past.at(-1)).toBe('Convert to control'); + + await engine.history.undo(); + expect(getDocumentLayer(engine.document.getDocument(), 'a')).toEqual(layer); + engine.lifecycle.dispose(); + }); + + it('copies to a new control layer with baked pixels', async () => { + const { engine } = adjustedEngine(createInlineSpatialExecutor); + + const copied = await engine.layers.copyLayerToControl('a'); + + expect(copied.status).toBe('copied'); + const copyId = copied.status === 'copied' ? copied.layerId : ''; + expect(getDocumentLayer(engine.document.getDocument(), copyId)).toMatchObject({ + adapter: { model: 'sdxl-control' }, + name: 'a copy', + source: { bitmap: null, type: 'paint' }, + type: 'control', + }); + engine.lifecycle.dispose(); + }); + + it('converts and copies an empty layer as a plain clone, since there are no pixels to bake', async () => { + const transform = { rotation: 0.3, scaleX: 2, scaleY: 1, x: 7, y: -4 }; + const empty = { + ...rasterLayer('a'), + adjustments: CLARITY, + source: { bitmap: null, type: 'paint' }, + transform, + } as CanvasRasterLayerContractV2; + const { projectId, store } = createReducerBackedStore({ ...makeDoc(), stacks: stacksFrom([empty]) }); + const engine = createCanvasEngine({ + backend: createTestStubRasterBackend(), + createSpatialExecutor: tooLarge, + getMainModelBase: () => 'sdxl', + imageResolver: () => Promise.resolve(new Blob()), + projectId, + store, + }); + const plain = { source: { bitmap: null, type: 'paint' }, transform, type: 'control' }; + + const copied = await engine.layers.copyLayerToControl('a'); + expect(copied.status).toBe('copied'); + const copy = getDocumentLayer(engine.document.getDocument(), copied.status === 'copied' ? copied.layerId : ''); + expect(copy).toMatchObject({ ...plain, name: 'a copy' }); + expect(copy).not.toHaveProperty('adjustments'); + + const live = getDocumentLayer(engine.document.getDocument(), 'a')!; + expect(await engine.layers.convertLayerToControl('Convert to control', live)).toBe('converted'); + expect(getDocumentLayer(engine.document.getDocument(), 'a')).toMatchObject(plain); + expect(getDocumentLayer(engine.document.getDocument(), 'a')).not.toHaveProperty('adjustments'); + + await engine.history.undo(); + expect(getDocumentLayer(engine.document.getDocument(), 'a')).toEqual(empty); + engine.lifecycle.dispose(); + }); + + it('copies an empty layer only while it is unchanged across the export', async () => { + const empty = { ...rasterLayer('a'), adjustments: CLARITY, source: { bitmap: null, type: 'paint' } }; + const { dispatch, projectId, store } = createReducerBackedStore({ + ...makeDoc(), + stacks: stacksFrom([empty as CanvasRasterLayerContractV2]), + }); + const engine = createCanvasEngine({ + backend: createTestStubRasterBackend(), + imageResolver: () => Promise.resolve(new Blob()), + projectId, + store, + }); + + const copying = engine.layers.copyLayerToControl('a'); + dispatch({ id: 'a', patch: { opacity: 0.3 }, type: 'updateCanvasLayer' }); + + expect(await copying).toEqual({ status: 'not-ready' }); + expect(getDocumentLeaves(engine.document.getDocument()!)).toHaveLength(1); + expect(engine.stores.canUndo.get()).toBe(false); + engine.lifecycle.dispose(); + }); + + it('converts an image layer with baked pixels; undo restores the image contract without persisting', async () => { + const image = { + ...rasterLayer('a', { imageName: 'held-source.png' }), + adjustments: [{ id: 'invert', isEnabled: true, type: 'invert' }], + } as CanvasRasterLayerContractV2; + const { projectId, store } = createReducerBackedStore({ ...makeDoc(), stacks: stacksFrom([image]) }); + const bitmapStore = createSpyBitmapStore(); + const engine = createCanvasEngine({ + backend: createTestStubRasterBackend(), + bitmapStore, + imageResolver: () => Promise.resolve(new Blob()), + projectId, + store, + }); + const live = getDocumentLayer(engine.document.getDocument(), 'a')!; + + expect(await engine.layers.convertLayerToControl('Convert to control', live)).toBe('converted'); + // The baked control pixels are the layer's own now, so they persist. + expect(bitmapStore.markLayerDirty).toHaveBeenCalledExactlyOnceWith('a'); + expect(engine.history.getHeldAssetRefs().images).toContain('held-source.png'); + bitmapStore.markLayerDirty.mockClear(); + vi.mocked(bitmapStore.discardLayer).mockClear(); + + await engine.history.undo(); + + expect(getDocumentLayer(engine.document.getDocument(), 'a')).toEqual(image); + // The baked pixels' pending upload is dropped; the image's reinstalled cache is never persisted. + expect(bitmapStore.discardLayer).toHaveBeenCalledWith('a'); + expect(bitmapStore.markLayerDirty).not.toHaveBeenCalled(); + expect((await engine.exports.exportLayerPixels('a')).status).toBe('ok'); + expect(engine.history.getHeldAssetRefs().images).toContain('held-source.png'); + engine.lifecycle.dispose(); + }); + + it('refuses both, naming the layer, when its stack cannot run', async () => { + const { dispatch, engine, layer } = adjustedEngine(tooLarge); + dispatch.mockClear(); + const refusal = { ownerId: 'a', ownerName: 'a', reason: 'too-large', status: 'adjustment-refused' }; + + expect(await engine.layers.copyLayerToControl('a')).toEqual(refusal); + expect(await engine.layers.convertLayerToControl('Convert to control', layer)).toEqual(refusal); + expect(dispatch).not.toHaveBeenCalled(); + expect(getDocumentLayer(engine.document.getDocument(), 'a')).toBe(layer); + engine.lifecycle.dispose(); + }); + }); + it('flushing after a prop-only change persists the painted (non-blank) surface', async () => { const { bitmapStore, engine, paintCache, raf, setDocument } = await paintOneStroke(); const clearsBefore = fullClearCount(paintCache); @@ -14653,14 +14992,14 @@ describe('gesture guard: nudge / commitStructural mid-stroke', () => { overlay.fire('pointermove', pointerAt(30, 30)); // Mid-gesture merge is refused before anything changes, like commitStructural/nudge. - expect(engine.layers.mergeLayerDown('upper')).toBe('busy'); + expect(await engine.layers.mergeLayerDown('upper')).toBe('busy'); expect(dispatch.mock.calls.some((call) => (call[0] as EngineTestAction).type === 'mergeCanvasLayersDown')).toBe( false ); // After the gesture ends, the merge lands. overlay.fire('pointerup', pointerAt(30, 30, { buttons: 0 })); - expect(engine.layers.mergeLayerDown('upper')).toBe('merged'); + expect(await engine.layers.mergeLayerDown('upper')).toBe('merged'); expect(dispatch.mock.calls.some((call) => (call[0] as EngineTestAction).type === 'mergeCanvasLayersDown')).toBe( true ); @@ -18867,7 +19206,7 @@ describe('refused strokes', () => { projectId: 'p1', store, }); - const refusals: string[] = []; + const refusals: CanvasEditRefusal[] = []; engine.tools.onEditRefused((refusal) => refusals.push(refusal)); const overlay = createInputCanvas(); engine.surface.attach(createInputCanvas().element, overlay.element); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.ts index 28d9e30f253..ac9460a57fb 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.ts @@ -20,7 +20,7 @@ import type { CanvasViewportCapability, BooleanRasterOperation, BooleanRasterResult, - CopyLayerToRasterResult, + CopyLayerResult, CropLayerResult, ExportBakedLayerPixelsOptions, ExportLayerPixelsOptions, @@ -82,10 +82,12 @@ import type { } from '@workbench/canvas-engine/contracts'; import type { CreatePath2D } from '@workbench/canvas-engine/freehand'; import type { ColorSampleProviders, ColorSampler } from '@workbench/canvas-engine/render/colorSample'; +import type { AdjustmentRefusal } from '@workbench/canvas-engine/render/exactAdjustments'; import type { CanvasFontRuntime, CanvasTextSource, FontLoadApi } from '@workbench/canvas-engine/render/fontLoader'; import type { LayerCacheEntry, LayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; import type { OverlayCursor } from '@workbench/canvas-engine/render/overlayRenderer'; import type { RenderScheduler } from '@workbench/canvas-engine/render/scheduler'; +import type { SpatialAdjustmentExecutor } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; import type { SamVisualInput } from '@workbench/canvas-engine/samInteraction'; import type { LayerTransform } from '@workbench/canvas-engine/transform/transformMath'; import type { Rect, RenderFlags, ToolId, Vec2 } from '@workbench/canvas-engine/types'; @@ -135,6 +137,7 @@ import { import { createEngineStores, type EngineStores, + type LayerAdjustmentStatus, type ScalarStore, type TextStylePatch, type TextToolOptions, @@ -147,6 +150,7 @@ import { import { createPointerPipeline, type PointerPipeline } from '@workbench/canvas-engine/input/pointerPipeline'; import { createWheelHandler } from '@workbench/canvas-engine/input/wheel'; import { isEmpty, union } from '@workbench/canvas-engine/math/rect'; +import { hasSpatialAdjustments } from '@workbench/canvas-engine/render/adjustments'; import { createColorSampler } from '@workbench/canvas-engine/render/colorSample'; import { compositeDocument, @@ -188,7 +192,7 @@ import type { StrokeCommittedEvent, Tool, ToolContext } from './tools/tool'; import { createBitmapStore, type BitmapStore } from './document/bitmapStore'; import { createDocumentMirror, type DocumentMirror } from './document/documentMirror'; -import { decideLayerChange } from './document/layerChangeDecision'; +import { decideLayerChange, nextThumbnailDisplayToken } from './document/layerChangeDecision'; import { getSourceBounds, getSourceContentRect, isRenderableLayer, renderableSourceOf } from './document/sources'; import { collectCanvasFontReferences, replaceCanvasFontReferences } from './fontReferences'; import { createLayerExportGuards, isSupportedExportSource } from './layerExportGuards'; @@ -287,6 +291,8 @@ export interface CanvasEngineOptions { fonts?: FontLoadApi | CanvasFontRuntime | null; /** Enables deterministic raster/render counters. Disabled by default. */ enableDiagnostics?: boolean; + /** Spatial adjustment executor factory; defaults to a module worker. Node tests pass an inline one. */ + createSpatialExecutor?: () => SpatialAdjustmentExecutor; } export interface CanvasEngineSelectionCapability extends CanvasSelectionCapability {} @@ -353,7 +359,11 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC // the contract keeps them until commit; applying them again would double them and snap back on pointer-up. isAdjustmentBaked: (layerId) => pixelEditController?.isOpenFor([layerId]) === true, isLayerHeld: (layerId) => isLayerHeldBySession(layerId) || bitmapStore.hasPendingWork(layerId), + onAdjustedSurfaceSettled: (layerId) => onAdjustedSurfaceSettled(layerId), + onAdjustmentStatus: (ownerId, status) => onAdjustmentStatus(ownerId, status), + onGroupSurfaceSettled: (groupId) => onGroupSurfaceSettled(groupId), onVersionChange: (layerId) => editingController?.invalidateLayer(layerId), + createSpatialExecutor: opts.createSpatialExecutor, }); const layerCache = rasterController.layers; const stores = createEngineStores(); @@ -404,6 +414,8 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC subscribe: (key, listener) => interactionStores[key].subscribe(listener), subscribeLayerThumbnailStatus: (layerId, listener) => stores.thumbnailStatus.subscribeKey(layerId, listener), subscribeLayerThumbnailVersion: (layerId, listener) => stores.thumbnailVersion.subscribeKey(layerId, listener), + getLayerAdjustmentStatus: (layerId) => stores.adjustmentStatus.get(layerId) ?? 'idle', + subscribeLayerAdjustmentStatus: (layerId, listener) => stores.adjustmentStatus.subscribeKey(layerId, listener), }; // Undefined uses browser font readiness; explicit null disables it. Loaded fonts trigger text rerasterization. const fontLoader = createFontLoader(opts.fonts === undefined ? domFontLoadApi() : opts.fonts); @@ -754,6 +766,84 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC scheduler.invalidate({ layers: [layerId] }); }; + /** Marks every group surface holding `ownerId` for a full redraw; false when no group holds it. */ + const invalidateAncestorGroups = (ownerId: string): boolean => { + const document = mirror.getDocument(); + const ancestors = document ? (getDocumentIndex(document).byId.get(ownerId)?.path ?? []) : []; + if (ancestors.length === 0) { + return false; + } + groupSurfaces.invalidate(ancestors); + return true; + }; + + /** + * A spatial adjustment result changed what a layer displays, with no cache publication: recomposite the layer, + * redraw its thumbnail, the overview and the group surfaces holding it. Its own adjusted surface is current, so it + * stays. + */ + const onAdjustedSurfaceSettled = (layerId: string): void => { + if (disposed) { + return; + } + stores.thumbnailVersion.set(layerId, nextThumbnailDisplayToken(stores.thumbnailVersion.get(layerId))); + stores.layerPixelEpoch.set(stores.layerPixelEpoch.get() + 1); + invalidateAncestorGroups(layerId); + scheduler.invalidate({ layers: [layerId] }); + }; + + /** + * A group's spatial result landed: recomposite it, redraw the overview (groups have no row thumbnail) and the + * surfaces of the groups holding it. Never the landed group's own, or every landing would start its next job. + */ + const onGroupSurfaceSettled = (groupId: string): void => { + if (disposed) { + return; + } + stores.layerPixelEpoch.set(stores.layerPixelEpoch.get() + 1); + invalidateAncestorGroups(groupId); + scheduler.invalidate({ layers: [groupId] }); + }; + + /** + * Publishes a layer's or group's spatial status. A job that ended without landing (a failure, or a surface evicted + * meanwhile) changes no pixels, but groups holding the owner wait for it: redraw them so they see it finished. + */ + const onAdjustmentStatus = (ownerId: string, status: LayerAdjustmentStatus | null): void => { + const wasComputing = stores.adjustmentStatus.get(ownerId) === 'computing'; + if (status) { + stores.adjustmentStatus.set(ownerId, status); + } else { + stores.adjustmentStatus.delete(ownerId); + } + if (wasComputing && status !== 'computing' && !disposed && invalidateAncestorGroups(ownerId)) { + scheduler.invalidate({ layers: [ownerId] }); + } + }; + + /** + * Drops the spatial display state of layers and groups that no longer draw a spatial stack (removed, edited or, for + * groups, disabled), clearing their status even while nothing draws them. Visits only owners holding that state. + */ + const retainSpatialSurfaces = (): void => { + const document = mirror.getDocument(); + const byId = document ? getDocumentIndex(document).byId : null; + rasterController.adjustments.retainSpatial((layerId) => { + const node = byId?.get(layerId)?.node; + return node?.type === 'raster' && hasSpatialAdjustments(node.adjustments); + }); + groupSurfaces.retainSpatial((groupId) => { + const entry = byId?.get(groupId); + return ( + entry?.stack === 'raster' && + entry.node.type === 'group' && + entry.node.isEnabled && + entry.ancestorsEnabled && + hasSpatialAdjustments(entry.node.adjustments) + ); + }); + }; + /** Invalidates cached pixels and drops only previews tied to that exact cache version. */ const invalidateLayerCache = (layerId: string): void => { cancelLayerRasterization(layerId); @@ -1259,6 +1349,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC pin: (layerId) => rasterController.memory.pin(layerId), reserve: (bytes) => rasterController.memory.reserve(bytes, { generation: lifecycleGeneration, purpose: 'raster-export' }), + runSpatial: (request) => rasterController.spatial.run(request), waitForFont: fontLoader.waitForReady, }); const rasterizeLayerPixels = rasterExportController.rasterize.bind(rasterExportController); @@ -1287,12 +1378,13 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC const exportBakedLayerBlob = rasterExportController.blob.bind(rasterExportController); type StructuralExportLayerPixelsResult = | Extract - | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' }; + | { status: 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal; const normalizeStructuralExport = async ( result: Promise ): Promise => { const resolved = await result; - if (resolved.status === 'ok') { + if (resolved.status === 'ok' || resolved.status === 'adjustment-refused') { return resolved; } return { status: resolved.status === 'aborted' ? 'not-ready' : resolved.status }; @@ -1315,8 +1407,9 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC const cropLayerToBbox = (layerId: string): Promise => layerController.crop.crop(layerId); - const copyLayerToRaster = (layerId: string): Promise => - layerController.copy.copyToRaster(layerId); + const copyLayerToRaster = (layerId: string): Promise => layerController.copy.copy(layerId, 'raster'); + const copyLayerToControl = (layerId: string): Promise => + layerController.copy.copy(layerId, 'control'); /** * Leases rasterized layer pixels for a composite export. Missing, ineligible or unsupported layers throw so @@ -1404,7 +1497,9 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC derivedSurfaceCache, diagnostics, getAdjustedSurface, - getGroupSurface, + // The frame's reads decide each group's spatial status, so the overview's other keys never flip its row. + getGroupSurface: (scope, members, matrices, content) => + groupSurfaces.get(scope, members, matrices, content, { frame: true }), getCheckerboardTile, getMaskPatternTile, layerCache, @@ -1530,6 +1625,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC scheduler.invalidate({ all: true }); }, onLayersRecomposite: (ids) => { + retainSpatialSurfaces(); scheduler.invalidate({ layers: ids }); }, onLayersChanged: (ids, sourceChangedIds, restructured) => { @@ -1539,6 +1635,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC const doc = mirror.getDocument(); cleanup.run(() => groupSurfaces.prune(doc ? new Set(getDocumentIndex(doc).byId.keys()) : new Set())); } + cleanup.run(retainSpatialSurfaces); const floatLayerId = floatingSelection.get()?.layerId; if (floatLayerId && ids.includes(floatLayerId)) { // The float's layer was replaced or removed. Drop the float rather than @@ -1967,6 +2064,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC clearStagedPreviewCache(); detach(); cancelAllLayerRasterizations(); + rasterController.cancelSpatialAdjustments(); cooldownPromise = persistenceController.flush().then( () => { if (disposed || lifecycleState !== 'cooling' || lifecycleGeneration !== generation) { @@ -2083,6 +2181,10 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC const getDefaultControlModel = (base: string | null): string | null => { return opts.getDefaultControlModel?.(base) ?? null; }; + const getControlModel = (): { base: string | null; model: string | null } => { + const base = getMainModelBase(); + return { base, model: getDefaultControlModel(base) }; + }; /** Conversion reducers clone contracts, so their publication postcondition compares by value. */ const documentHasLayerContract = ( @@ -2102,6 +2204,8 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC deleteDerived: deleteDerivedSurfaces, getActiveProjectId: () => projectId, getAdjustedSurface, + getAdjustmentStatus: (layerId) => stores.adjustmentStatus.get(layerId), + peekAdjustedSurface: (layer, entry) => rasterController.peekAdjustedSurface(layer, entry), getDocument: () => mirror.getDocument(), getTransformSession: () => stores.transformSession.get(), installPrepared: installGeneratedPaintCache, @@ -2198,12 +2302,16 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC renderableSourceOf(layer)?.type === 'paint'; const layerMutationController = new LayerMutationController({ + backend, captureCache: captureLayerCache, ctx: mutationContext, discardPersisted: (layerId) => bitmapStore.discardLayer(layerId), + exportBaked: (layerId) => exportBakedLayerPixelsForStructural(layerId, { includeDisabled: true }), + getControlModel, getDuplicateRasterPlan, getSelectedLayerIds: resolveSelectedLayerIds, hasPendingPixelWork: (layerId) => bitmapStore.hasPendingWork(layerId), + isGuardCurrent: isLayerExportGuardCurrent, needsPixelPersistence: layerNeedsPixelPersistence, prepareDuplicateRasterSource: prepareLayerRasterCache, pinDuplicateRasterSources: (layerIds) => { @@ -2223,6 +2331,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC }); const commitLayerCopy = layerMutationController.copy.bind(layerMutationController); const commitLayerConversion = layerMutationController.convert.bind(layerMutationController); + const convertLayerToControl = layerMutationController.convertToControl.bind(layerMutationController); const duplicateLayers = layerMutationController.duplicate.bind(layerMutationController); const replaceSelectionFromImage = editingController.selectionImage.replace.bind(editingController.selectionImage); @@ -2269,7 +2378,8 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC const extractMaskedArea = (maskLayerId: string): Promise => layerController.extractMaskedArea.extract(maskLayerId); - const mergeLayerDown = (upperLayerId: string): MergeDownResult => layerController.merge.mergeDown(upperLayerId); + const mergeLayerDown = (upperLayerId: string): Promise => + layerController.merge.mergeDown(upperLayerId); const mergeSelectedRasterLayers = (layerIds: readonly string[]): Promise => layerController.merge.mergeSelected(layerIds); const mergeVisibleRasterLayers = (): Promise => layerController.merge.mergeVisible(); @@ -2295,10 +2405,12 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC isDocumentSnapshotCurrent, reserve: (bytes) => rasterController.memory.reserve(bytes, { generation: lifecycleGeneration, purpose: 'psd-export' }), + runSpatial: (request) => rasterController.spatial.run(request), }); const exportRasterLayersToPsd = (fileName: string): Promise => psdExportController.export(fileName); - const rasterizeLayer = (layerId: string): RasterizeLayerResult => layerController.rasterize.rasterize(layerId); + const rasterizeLayer = (layerId: string): Promise => + layerController.rasterize.rasterize(layerId); // Transform sessions span gestures. Apply records one undoable parameter edit for image layers or pixel bake for // paint; Cancel drops previews. Tools and public UI share the session API. @@ -2537,6 +2649,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC const getCompositeExecutorDeps = (): CanvasCompositeExecutorDeps => ({ backend, reserve: (bytes) => rasterController.memory.reserveOperation(bytes, { purpose: 'invocation-composite' }), + runSpatial: (request) => rasterController.spatial.run(request), // Unreferenced generation inputs upload as reclaimable intermediates. uploadImage: (blob) => opts.uploadIntermediateImage(blob), }); @@ -2549,7 +2662,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC documentGeneration: rasterController.getDocumentGeneration(), lifecycleGeneration, }), - // Never builds the display's copy: a miss adjusts only the sampled region the export reserved. + // Never builds the display's copy, and reuses only an exact one: a miss adjusts what the export reserved. adjustedSurface: (layerId, surface, adjustments) => rasterController.peekAdjustedCacheSurface(layerId, surface, adjustments), getLayerSurface: requireLayerSurfaceForExport, @@ -2571,6 +2684,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC }; }, reserve: (bytes) => rasterController.memory.reserveOperation(bytes, { purpose: 'background-snapshot' }), + runSpatial: (request) => rasterController.spatial.run(request), }); const surface: CanvasSurfaceCapability = { attach, detach, resize }; const viewportCapability: CanvasViewportCapability = { fitToView, fitToViewOnFirstShow, getViewport, setBboxGrid }; @@ -2618,24 +2732,24 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC }, copy: { backend, + commitLayerCopy, ctx: mutationContext, exportBaked: (layerId) => exportBakedLayerPixelsForStructural(layerId, { includeDisabled: true }), + getControlModel, isGuardCurrent: isLayerExportGuardCurrent, }, extractMaskedArea: { backend, ctx: mutationContext, - derived: derivedSurfaceCache, - diagnostics, exportBaked: (layerId, includeDisabled) => exportBakedLayerPixelsForStructural(layerId, { includeDisabled }), - getAdjustedSurface, - getGroupSurface, - getMaskPattern: getMaskPatternTile, hasExportableContent: hasExportableLayerContent, isCacheReady: isLayerCacheReadyForOp, isGuardCurrent: isLayerExportGuardCurrent, layers: layerCache, + peekAdjustedCacheSurface: (layerId, surface, adjustments) => + rasterController.peekAdjustedCacheSurface(layerId, surface, adjustments), rasterize: (layerId) => rasterizeLayerPixelsForStructural(layerId), + runSpatial: (request) => rasterController.spatial.run(request), }, commitGeneratedImageResult, newRasterLayer: { @@ -2665,11 +2779,13 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC needsPixelPersistence: layerNeedsPixelPersistence, peekAdjustedSurface: (layer, entry) => rasterController.peekAdjustedSurface(layer, entry), publishSelectedLayerIds: (primaryId, selectedIds) => opts.setSelectedLayerIds?.(primaryId, selectedIds), + runSpatial: (request) => rasterController.spatial.run(request), }, rasterize: { backend, ctx: mutationContext, rasterizeDeps: (document) => rasterizeDeps(document), + runSpatial: (request) => rasterController.spatial.run(request), }, structural: structuralController, thumbnail: { @@ -2681,7 +2797,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC getMaskPattern: getMaskPatternTile, isDisposed: () => disposed, isSupportedSource: isSupportedExportSource, - peekAdjustedSurface: (layer, entry) => rasterController.peekAdjustedSurface(layer, entry), + peekDisplayedSurface: (layer, entry) => rasterController.peekDisplayedAdjustedSurface(layer, entry), pin: (layerId) => rasterController.memory.pin(layerId), projectId, rasterize: rasterizeLayerForThumbnail, @@ -2784,6 +2900,8 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC commitRasterFilterResult, commitStagedImage, commitTextEdit, + convertLayerToControl, + copyLayerToControl, copyLayerToRaster, cropLayerToBbox, duplicateLayers, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/engineStores.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/engineStores.ts index 78890b98504..734eef51393 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/engineStores.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/engineStores.ts @@ -10,6 +10,7 @@ import type { CanvasTextFontVariations, ParametricShapeKind, } from '@workbench/canvas-engine/contracts'; +import type { AdjustedSurfaceStatus } from '@workbench/canvas-engine/render/spatialDisplay'; import type { SamInteractionState } from '@workbench/canvas-engine/samInteraction'; import type { LayerTransform } from '@workbench/canvas-engine/transform/transformMath'; import type { Rect, SelectionOp, ToolId, Vec2 } from '@workbench/canvas-engine/types'; @@ -336,10 +337,13 @@ export interface KeyedVersionStore { export type LayerThumbnailStatus = 'loading' | 'ready' | 'error'; -/** A per-layer thumbnail request state; absence represents `idle`. */ -export interface KeyedThumbnailStatusStore { - get(key: string): LayerThumbnailStatus | undefined; - set(key: string, value: LayerThumbnailStatus): void; +/** Why a layer's displayed spatial adjustments are not exact yet; absence means they are (or it has none). */ +export type LayerAdjustmentStatus = AdjustedSurfaceStatus; + +/** A per-layer status; absence represents `idle`. */ +export interface KeyedStatusStore { + get(key: string): T | undefined; + set(key: string, value: T): void; delete(key: string): void; clear(): void; subscribeKey(key: string, listener: () => void): () => void; @@ -393,8 +397,8 @@ const createKeyedVersionStore = (): KeyedVersionStore => { }; }; -const createKeyedThumbnailStatusStore = (): KeyedThumbnailStatusStore => { - const values = new Map(); +const createKeyedStatusStore = (): KeyedStatusStore => { + const values = new Map(); const keyedListeners = new Map void>>(); const notify = (key: string): void => { for (const listener of keyedListeners.get(key) ?? []) { @@ -444,7 +448,8 @@ export interface EngineStores { viewportReady: ScalarStore; cursor: ScalarStore; thumbnailVersion: KeyedVersionStore; - thumbnailStatus: KeyedThumbnailStatusStore; + thumbnailStatus: KeyedStatusStore; + adjustmentStatus: KeyedStatusStore; /** Monotonic aggregate version for consumers whose eligibility depends on live layer content. */ layerPixelEpoch: ScalarStore; /** Monotonic version of the mirrored document, for consumers that read the engine's model during render. */ @@ -725,7 +730,8 @@ export const createEngineStores = (initialTool: ToolId = 'view'): EngineStores = textEditSession: createScalarStore(null, textEditSessionEqual), textOptions: createScalarStore({ ...DEFAULT_TEXT_OPTIONS }, textOptionsEqual), thumbnailVersion: createKeyedVersionStore(), - thumbnailStatus: createKeyedThumbnailStatusStore(), + thumbnailStatus: createKeyedStatusStore(), + adjustmentStatus: createKeyedStatusStore(), transformSession: createScalarStore(null, transformSessionEqual), viewportReady: createScalarStore(false), zoom: createScalarStore(1), diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.browser.test.ts new file mode 100644 index 00000000000..c8f64a129a2 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.browser.test.ts @@ -0,0 +1,96 @@ +/** Real-worker check that a PSD leaf carries what the settled display shows for its own spatial stack. */ + +import type { CanvasAdjustmentsContract } from '@workbench/canvas-engine/contracts'; +import type { Rect } from '@workbench/canvas-engine/types'; +import type { Psd } from 'ag-psd'; + +import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; +import { executePsdExport, planPsdExport } from '@workbench/canvas-engine/export/psdExport'; +import { fromTRS } from '@workbench/canvas-engine/math/mat2d'; +import { createAdjustedSurfaceCache } from '@workbench/canvas-engine/render/adjustedSurfaceCache'; +import { createLayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; +import { createDomRasterBackend } from '@workbench/canvas-engine/render/raster'; +import { createSpatialAdjustmentService } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import { describe, expect, it, vi } from 'vitest'; + +const CONTENT: Rect = { height: 30, width: 40, x: -6, y: -4 }; +const TRANSFORM = { rotation: Math.PI / 9, scaleX: 0.6, scaleY: 0.8, x: 11, y: 5 }; +const STACK: CanvasAdjustmentsContract = [ + { amount: 100, id: 'texture', isEnabled: true, size: 3, type: 'texture' }, + { id: 'invert', isEnabled: true, type: 'invert' }, + { amount: 70, id: 'clarity', isEnabled: true, size: 10, type: 'clarity' }, +]; + +describe('executePsdExport with a spatial stack', () => { + it("bakes a rotated, scaled leaf's settled display, layer-local before the transform", async () => { + const backend = createDomRasterBackend(); + const layers = createLayerCacheStore(backend); + const entry = layers.growToRect('layer', CONTENT); + for (let x = 0; x < CONTENT.width; x += 1) { + entry.surface.ctx.fillStyle = `rgb(${40 + x * 5}, ${x % 7 === 0 ? 30 : 140}, ${220 - x * 4})`; + entry.surface.ctx.fillRect(x, 3, 1, CONTENT.height - 6); + } + layers.publishPixels('layer'); + const memory = new RasterMemoryBudgetController({ budgetBytes: 64 * 1024 * 1024 }); + const service = createSpatialAdjustmentService({ + reserve: (bytes) => memory.reserveOperation(bytes, { purpose: 'spatial-adjustment' }), + }); + try { + const display = createAdjustedSurfaceCache(backend, undefined, { + spatial: { run: (request) => service.run(request) }, + }); + await vi.waitFor( + () => { + display.get('layer', entry, STACK); + expect(display.peek('layer', entry, STACK)).not.toBeNull(); + }, + { interval: 20, timeout: 10_000 } + ); + const plan = planPsdExport([ + { + adjustments: STACK, + blendMode: 'normal', + contentRect: CONTENT, + id: 'layer', + isEnabled: true, + name: 'Layer', + opacity: 1, + transform: TRANSFORM, + }, + ]); + expect(plan.status).toBe('ok'); + let written: Psd | null = null; + + await executePsdExport(plan, 'layer.psd', { + backend, + download: () => undefined, + getLayerSurface: () => Promise.resolve({ rect: entry.rect, surface: entry.surface }), + runSpatial: (request) => service.run(request), + writePsd: (psd) => { + written = psd; + return Promise.resolve(new ArrayBuffer(0)); + }, + }); + + // What the canvas shows: the display's exact adjusted surface resampled through the transform. + const worldRect = plan.status === 'ok' ? plan.layers[0]!.worldRect : CONTENT; + const reference = backend.createSurface(worldRect.width, worldRect.height); + const { rotation, scaleX, scaleY, x, y } = TRANSFORM; + const matrix = fromTRS({ x, y }, rotation, scaleX, scaleY); + reference.ctx.setTransform( + matrix.a, + matrix.b, + matrix.c, + matrix.d, + matrix.e - worldRect.x, + matrix.f - worldRect.y + ); + reference.ctx.drawImage(display.peek('layer', entry, STACK)!.canvas, CONTENT.x, CONTENT.y); + const expected = reference.ctx.getImageData(0, 0, worldRect.width, worldRect.height).data; + expect([...written!.children![0]!.imageData!.data]).toEqual([...expected]); + expect(memory.snapshot().reservedBytes).toBe(0); + } finally { + service.dispose(); + } + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.test.ts index 4cd1ed145d3..d8723a339c9 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.test.ts @@ -393,7 +393,7 @@ describe('executePsdExport — adjustments', () => { layer({ adjustments: own, contentRect, - groupAdjustments: enclosing, + groupStacks: [{ adjustments: enclosing, id: 'g' }], transform: { rotation: 0, scaleX: 0.5, scaleY: 0.5, x: 9, y: 6 }, }), ]); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.ts index 232b818ff27..9ece83eea5f 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.ts @@ -16,8 +16,13 @@ import type { BlendMode, Layer as AgPsdLayer, Psd } from 'ag-psd'; import { downloadBlob } from '@platform/browser/downloadBlob'; import { fromTRS } from '@workbench/canvas-engine/math/mat2d'; import { isEmpty, roundOut, transformBounds, union } from '@workbench/canvas-engine/math/rect'; -import { applyAdjustments } from '@workbench/canvas-engine/render/adjustments'; import { blendToComposite } from '@workbench/canvas-engine/render/compositor'; +import { + adjustExactly, + AdjustmentRefusedError, + imageDataOf, + type SpatialRunner, +} from '@workbench/canvas-engine/render/exactAdjustments'; /** Cap exports at the legacy PSD 30000px side limit to bound unbounded-canvas allocations. */ export const PSD_MAX_DIMENSION = 30000; @@ -75,9 +80,15 @@ export interface PsdExportLayerInput { adjustments?: CanvasAdjustmentsContract; /** * Enclosing group stacks, innermost first, baked after the layer's own: PSD cannot represent them. They work in - * document pixels, as the groups do. + * document pixels, as the groups do, but on this layer's pixels alone: an approximation of the group composite. */ - groupAdjustments?: CanvasAdjustmentsContract; + groupStacks?: readonly PsdGroupStack[]; +} + +/** One enclosing group's own stack, named for a refusal. */ +export interface PsdGroupStack { + readonly id: string; + readonly adjustments: CanvasAdjustmentsContract; } /** A raster group: a folder in the PSD, children top-first. */ @@ -127,7 +138,7 @@ export interface PsdPlanLayer { contributes: boolean; colorLabel?: CanvasColorLabel; adjustments?: CanvasAdjustmentsContract; - groupAdjustments?: CanvasAdjustmentsContract; + groupStacks?: readonly PsdGroupStack[]; } /** A planned PSD folder; `children` are in ag-psd bottom-to-top order. */ @@ -239,7 +250,7 @@ export const planPsdExport = ( const top = worldRect.y - canvasRect.y; const layer: PsdPlanLayer = { adjustments: input.adjustments, - groupAdjustments: input.groupAdjustments, + groupStacks: input.groupStacks, blendMode: mapped ?? 'normal', bottom: top + worldRect.height, colorLabel: input.colorLabel, @@ -343,6 +354,11 @@ export interface ExecutePsdExportDeps { backend: { createSurface(width: number, height: number): RasterSurface }; /** Cancels before the next background allocation or side effect. */ signal?: AbortSignal; + /** + * Evaluates spatial adjustment runs; without it a spatial stack throws {@link AdjustmentRefusedError} rather than + * bake without its spatial entries. + */ + runSpatial?: SpatialRunner; /** Returns rasterized pixels and local content bounds, using live paint caches when available. */ getLayerSurface(layerId: string): Promise<{ surface: RasterSurface; rect: Rect }>; /** Reads a surface region's pixels (default `getImageData`). */ @@ -366,11 +382,29 @@ const throwIfAborted = (signal?: AbortSignal): void => { } }; +/** The whole stack on `pixels` at `origin`; throws when `ownerId`'s stack cannot run or the export was aborted. */ +const evaluate = async ( + pixels: ImageData, + adjustments: CanvasAdjustmentsContract, + origin: Rect, + ownerId: string, + deps: ExecutePsdExportDeps +): Promise => { + const result = await adjustExactly(pixels, adjustments, origin, { run: deps.runSpatial, signal: deps.signal }); + if (result.status === 'aborted') { + throw new DOMException('PSD export was aborted.', 'AbortError'); + } + if (result.status !== 'ok') { + throw new AdjustmentRefusedError(ownerId, result.status); + } + return result.data === pixels.data ? pixels : imageDataOf(result.data, pixels.width, pixels.height); +}; + /** * Bakes transformed pixels and adjustments into world-AABB bounds, returning surface and straight-alpha ImageData. - * The layer's own stack adjusts layer-local pixels before the transform resamples them, as the display does; the - * enclosing group stacks then adjust the world pixels, which are document pixels at scale 1. Opacity/blend remain - * PSD properties. + * The layer's own stack adjusts all its layer-local pixels before the transform resamples them, as the display does, + * spatial entries included; the enclosing group stacks then adjust the world pixels, which are document pixels at + * scale 1. Opacity/blend remain PSD properties. */ const bakeLayer = async ( planLayer: PsdPlanLayer, @@ -395,8 +429,7 @@ const bakeLayer = async ( setTransform(source.ctx, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }); source.ctx.drawImage(cache.canvas, 0, 0); const localPixels = read(source, { height: rect.height, width: rect.width, x: 0, y: 0 }); - applyAdjustments(localPixels, planLayer.adjustments, rect); - write(source, localPixels, 0, 0); + write(source, await evaluate(localPixels, planLayer.adjustments, rect, planLayer.id, deps), 0, 0); } // local→world then shift into AABB-local (translation only affects e/f). const local = layerMatrix(planLayer.transform); @@ -407,9 +440,11 @@ const bakeLayer = async ( } const fullRect: Rect = { height, width, x: 0, y: 0 }; - const imageData = read(surface, fullRect); - if (planLayer.groupAdjustments) { - applyAdjustments(imageData, planLayer.groupAdjustments, worldRect); + let imageData = read(surface, fullRect); + if (planLayer.groupStacks && planLayer.groupStacks.length > 0) { + for (const stack of planLayer.groupStacks) { + imageData = await evaluate(imageData, stack.adjustments, worldRect, stack.id, deps); + } // Write the adjusted pixels back so the flattened composite (below) reuses // this surface directly. write(surface, imageData, 0, 0); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.test.ts index 13e308438ae..5ba65fb596c 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.test.ts @@ -1,9 +1,14 @@ -import type { CanvasDocumentContractV3, CanvasImageRef, CanvasLayerContract } from '@workbench/canvas-engine/contracts'; +import type { + CanvasDocumentContractV3, + CanvasImageRef, + CanvasLayerContract, + CanvasRasterLayerContractV2, +} from '@workbench/canvas-engine/contracts'; import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Rect } from '@workbench/canvas-engine/types'; import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; -import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { groupContract, stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { createTestStubRasterBackend } from '@workbench/canvas-engine/render/raster.testStub'; import { MEASURED_NOISY_TEXT } from '@workbench/canvas-engine/render/rasterComposite.testing'; import { describe, expect, it, vi } from 'vitest'; @@ -48,7 +53,7 @@ const controlLayer = (id: string): CanvasLayerContract => ({ withTransparencyEffect: false, }); -const makeDoc = (layers: CanvasLayerContract[]): CanvasDocumentContractV3 => ({ +const makeDoc = (layers: Parameters[0]): CanvasDocumentContractV3 => ({ background: 'transparent', bbox: { height: 128, width: 128, x: 0, y: 0 }, height: 128, @@ -264,6 +269,102 @@ describe('exportRasterComposite', () => { }); }); + describe('spatial adjustments', () => { + const CLARITY = [{ amount: 50, id: 'clarity', isEnabled: true, size: 10, type: 'clarity' as const }]; + /** A 64×32 image layer at (-4, 8); a 4×4 bbox samples a sliver of it. */ + const spatialLayer = (adjustments = CLARITY): CanvasLayerContract => ({ + ...(rasterLayer('portrait') as CanvasRasterLayerContractV2), + adjustments, + }); + const SLIVER: Rect = { height: 4, width: 4, x: 0, y: 10 }; + const withReserve = (deps: ExportRasterCompositeDeps) => { + const reserve = vi.fn((_bytes: number) => ({ lease: { release: vi.fn() }, status: 'ok' as const })); + deps.reserve = reserve; + return reserve; + }; + + it("reserves a spatial layer's whole surface and its ImageData, whatever the bbox samples", async () => { + const { deps } = makeDeps(makeDoc([spatialLayer()])); + const reserve = withReserve(deps); + const runSpatial = vi.fn((request: { pixels: Uint8ClampedArray }) => + Promise.resolve({ pixels: request.pixels, status: 'ok' as const }) + ); + deps.runSpatial = runSpatial; + + await expect(exportRasterComposite({ bounds: 'rect', rect: SLIVER }, deps)).resolves.toMatchObject({ + status: 'ok', + }); + + expect(reserve).toHaveBeenCalledExactlyOnceWith((4 * 4 + 64 * 32 * 2) * 4); + // The job reads all of the layer, not the sampled region. + expect(runSpatial).toHaveBeenCalledOnce(); + expect(runSpatial.mock.calls[0]![0]).toMatchObject({ height: 32, priority: 'now', width: 64 }); + }); + + it("reserves a spatial group's buffer over its members' bounds, not the bbox", async () => { + const layer = rasterLayer('member'); + const { deps } = makeDeps(makeDoc([groupContract('group', [layer], { adjustments: CLARITY })])); + const reserve = withReserve(deps); + const runSpatial = vi.fn((request: { pixels: Uint8ClampedArray }) => + Promise.resolve({ pixels: request.pixels, status: 'ok' as const }) + ); + deps.runSpatial = runSpatial; + + await expect(exportRasterComposite({ bounds: 'rect', rect: SLIVER }, deps)).resolves.toMatchObject({ + status: 'ok', + }); + + expect(reserve).toHaveBeenCalledExactlyOnceWith((4 * 4 + 64 * 32 * 2) * 4); + expect(runSpatial.mock.calls[0]![0]).toMatchObject({ height: 32, width: 64 }); + }); + + it.each(['too-large', 'over-budget', 'error'] as const)( + 'refuses %s with the name of the layer whose stack could not run', + async (reason) => { + const { deps, encodeSurface } = makeDeps(makeDoc([spatialLayer()])); + deps.runSpatial = () => Promise.resolve({ status: reason }); + + await expect(exportRasterComposite({ bounds: 'content' }, deps)).resolves.toEqual({ + ownerId: 'portrait', + ownerName: 'portrait', + reason, + status: 'adjustment-refused', + }); + expect(encodeSurface).not.toHaveBeenCalled(); + } + ); + + it('names the group whose stack could not run', async () => { + const { deps } = makeDeps( + makeDoc([groupContract('group', [rasterLayer('member')], { adjustments: CLARITY, name: 'Sky' })]) + ); + deps.runSpatial = () => Promise.resolve({ status: 'too-large' }); + + await expect(exportRasterComposite({ bounds: 'content' }, deps)).resolves.toMatchObject({ + ownerId: 'group', + ownerName: 'Sky', + status: 'adjustment-refused', + }); + }); + + it('never exports a spatial stack without its spatial entries', async () => { + const { deps, encodeSurface } = makeDeps(makeDoc([spatialLayer()])); + + await expect(exportRasterComposite({ bounds: 'content' }, deps)).resolves.toMatchObject({ + reason: 'error', + status: 'adjustment-refused', + }); + expect(encodeSurface).not.toHaveBeenCalled(); + }); + + it('reports a job cancelled by cooldown or cache release as stale', async () => { + const { deps } = makeDeps(makeDoc([spatialLayer()])); + deps.runSpatial = () => Promise.resolve({ status: 'aborted' }); + + await expect(exportRasterComposite({ bounds: 'content' }, deps)).resolves.toEqual({ status: 'stale' }); + }); + }); + it('discards an encoded blob if the document changes during export', async () => { const { deps } = makeDeps(makeDoc([rasterLayer('raster')])); vi.mocked(deps.isSnapshotCurrent).mockReturnValueOnce(true).mockReturnValue(false); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.ts index 8d83553498e..da754b1d603 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.ts @@ -3,6 +3,11 @@ import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Rect } from '@workbench/canvas-engine/types'; import { isEmpty } from '@workbench/canvas-engine/math/rect'; +import { + type AdjustmentRefusal, + AdjustmentRefusedError, + adjustmentRefusal, +} from '@workbench/canvas-engine/render/exactAdjustments'; import { compositeScratchPixels, getBaseRasterContentBounds, @@ -16,7 +21,8 @@ export type RasterCompositeExportRequest = { bounds: 'content' } | { bounds: 're export type RasterCompositeExportResult = | { status: 'ok'; blob: Blob; rect: Rect } - | { status: 'empty' | 'stale' | 'not-ready' | 'over-budget' }; + | { status: 'empty' | 'stale' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal; export interface RasterCompositeExportSnapshot { directPixelEpoch: number; @@ -69,7 +75,11 @@ export const exportRasterComposite = async ( if (error instanceof RasterCompositeOverBudgetError) { return { status: 'over-budget' }; } - if (!deps.isSnapshotCurrent(snapshot)) { + if (error instanceof AdjustmentRefusedError) { + return adjustmentRefusal(document, error.ownerId, error.reason); + } + // Spatial work is cancelled only by cooldown, cache release or disposal; the export can simply be retried. + if (!deps.isSnapshotCurrent(snapshot) || (error instanceof DOMException && error.name === 'AbortError')) { return { status: 'stale' }; } throw error; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/rasterTransactions.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/rasterTransactions.ts index 0d092698566..9f528f4746a 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/rasterTransactions.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/rasterTransactions.ts @@ -1,5 +1,6 @@ import type { LayerExportGuard } from './capabilities'; import type { CanvasStateContractV3 } from './contracts'; +import type { SpatialRunner } from './render/exactAdjustments'; import type { RasterSurface } from './render/raster'; import type { Rect } from './types'; @@ -43,4 +44,6 @@ export interface CanvasCompositeExecutorDeps { | { status: 'ok'; lease: { release(): void } } | { status: 'over-budget'; requestedBytes: number; availableBytes: number }; uploadImage(blob: Blob): Promise<{ imageName: string; width: number; height: number }>; + /** Evaluates spatial adjustment runs exactly for the composites (`SpatialAdjustmentService.run`). */ + runSpatial?: SpatialRunner; } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceCache.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceCache.test.ts index dbdccc50e31..f93599c63cc 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceCache.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceCache.test.ts @@ -1,12 +1,22 @@ import type { CanvasAdjustmentsContract } from '@workbench/canvas-engine/contracts'; +import type { + SpatialAdjustmentExecutor, + SpatialExecutorJob, + SpatialExecutorResult, +} from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; -import { describe, expect, it } from 'vitest'; +import { createSpatialAdjustmentService } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import { createInlineSpatialExecutor } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService.testing'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import type { LayerCacheEntry } from './layerCache'; import type { StubRasterSurface } from './raster.testStub'; import { createAdjustedSurfaceCache } from './adjustedSurfaceCache'; +import { createDerivedSurfaceCache } from './derivedSurfaceCache'; +import { createLayerCacheStore } from './layerCache'; import { createTestStubRasterBackend } from './raster.testStub'; +import { type AdjustedSurfaceStatus, SPATIAL_CONTENT_DEBOUNCE_MS } from './spatialDisplay'; const BRIGHTEN: CanvasAdjustmentsContract = [ { brightness: 0.5, contrast: 0, id: 'b', isEnabled: true, type: 'brightness-contrast' }, @@ -109,3 +119,408 @@ describe('createAdjustedSurfaceCache', () => { expect(cache.byteSize()).toBe(0); }); }); + +describe('spatial stacks', () => { + const W = 8; + const H = 6; + const clarity = (amount: number): CanvasAdjustmentsContract => [ + { amount, id: 'cl', isEnabled: true, size: 40, type: 'clarity' }, + ]; + + /** An executor whose jobs settle when the test says; an `ok` job's pixels all become its 1-based job number. */ + const createManualExecutor = () => { + const jobs: { steps: SpatialExecutorJob['steps']; settle: (status: SpatialExecutorResult['status']) => void }[] = + []; + const executor: SpatialAdjustmentExecutor = { + dispose: () => undefined, + run: (job) => + new Promise((resolve) => { + const tag = jobs.length + 1; + jobs.push({ + settle: (status) => { + new Uint8ClampedArray(job.buffer).fill(status === 'ok' ? tag : 0); + resolve({ buffer: job.buffer, status }); + }, + steps: job.steps, + }); + }), + }; + return { executor, jobs }; + }; + + const flush = async () => { + for (let i = 0; i < 10; i++) { + await Promise.resolve(); + } + }; + + const setup = (options: { executor?: SpatialAdjustmentExecutor; overBudget?: boolean } = {}) => { + const manual = createManualExecutor(); + const backend = createTestStubRasterBackend({ readbackAlpha: 255 }); + const layers = createLayerCacheStore(backend); + layers.getOrCreate('L', W, H); + layers.publishPixels('L'); + const derived = createDerivedSurfaceCache(); + const service = createSpatialAdjustmentService({ + createExecutor: () => options.executor ?? manual.executor, + reserve: (bytes) => + options.overBudget + ? { availableBytes: 0, requestedBytes: bytes, status: 'over-budget' } + : { lease: { release: () => undefined }, status: 'ok' }, + }); + const settled: string[] = []; + const statuses: (AdjustedSurfaceStatus | null)[] = []; + const cache = createAdjustedSurfaceCache(backend, derived, { + damageSince: (layerId, version) => layers.damageSince(layerId, version), + spatial: { + onSettle: (layerId) => settled.push(layerId), + onStatus: (_layerId, status) => statuses.push(status), + run: (request) => service.run(request), + }, + }); + const entry = () => layers.get('L')!; + const show = (adjustments: CanvasAdjustmentsContract) => cache.get('L', entry(), adjustments) as StubRasterSurface; + return { cache, derived, entry, jobs: manual.jobs, layers, service, settled, show, statuses }; + }; + + /** The first byte of the last whole-surface write: 0 for a per-pixel interim, a job number for a result. */ + const shownTag = (surface: StubRasterSurface): number => { + const writes = surface.callLog.filter( + (call) => call.op === 'putImageData' && (call.args[0] as ImageData).width === W && call.args[1] === 0 + ); + return (writes[writes.length - 1]!.args[0] as ImageData).data[0]!; + }; + + const sourceReads = (surface: StubRasterSurface) => + surface.callLog.filter((call) => call.op === 'drawImage').map((call) => call.args.slice(1, 5)); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('shows a per-pixel interim at once, then the landed result, exactly and without further work', async () => { + const h = setup(); + const surface = h.show(clarity(50)); + + expect(shownTag(surface)).toBe(0); + expect(h.cache.peek('L', h.entry(), clarity(50))).toBeNull(); + expect(h.cache.peekDisplayed('L', h.entry(), clarity(50))).toBe(surface); + expect(h.statuses).toEqual(['computing']); + expect(h.jobs.map((job) => job.steps)).toEqual([[{ amount: 50, kind: 'clarity', size: 40 }]]); + + h.jobs[0]!.settle('ok'); + await flush(); + expect(h.settled).toEqual(['L']); + expect(h.statuses).toEqual(['computing', null]); + expect(shownTag(surface)).toBe(1); + expect(h.cache.peek('L', h.entry(), clarity(50))).toBe(surface); + + const calls = surface.callLog.length; + expect(h.show(clarity(50))).toBe(surface); + expect(surface.callLog).toHaveLength(calls); + expect(h.jobs).toHaveLength(1); + }); + + it('keeps the shown pixels through a parameter change and starts its job at once', async () => { + const h = setup(); + const surface = h.show(clarity(50)); + h.jobs[0]!.settle('ok'); + await flush(); + const reads = sourceReads(surface).length; + + expect(h.show(clarity(80))).toBe(surface); + expect(sourceReads(surface)).toHaveLength(reads); + expect(shownTag(surface)).toBe(1); + expect(h.cache.peek('L', h.entry(), clarity(80))).toBeNull(); + expect(h.jobs).toHaveLength(2); + + h.jobs[1]!.settle('ok'); + await flush(); + expect(shownTag(surface)).toBe(2); + expect(h.cache.peek('L', h.entry(), clarity(80))).toBe(surface); + }); + + it('re-patches known damage per-pixel and starts a job only after 120 ms without edits', async () => { + vi.useFakeTimers(); + const h = setup(); + const surface = h.show(clarity(50)); + h.jobs[0]!.settle('ok'); + await flush(); + + h.layers.publishPixels('L', { height: 2, width: 3, x: 1, y: 1 }); + h.show(clarity(50)); + expect(sourceReads(surface).at(-1)).toEqual([1, 1, 3, 2]); + expect(shownTag(surface)).toBe(1); + expect(h.cache.peek('L', h.entry(), clarity(50))).toBeNull(); + expect(h.statuses.at(-1)).toBe('computing'); + + await vi.advanceTimersByTimeAsync(SPATIAL_CONTENT_DEBOUNCE_MS - 1); + h.layers.publishPixels('L', { height: 1, width: 1, x: 5, y: 4 }); + h.show(clarity(50)); + await vi.advanceTimersByTimeAsync(SPATIAL_CONTENT_DEBOUNCE_MS - 1); + expect(h.jobs).toHaveLength(1); + await vi.advanceTimersByTimeAsync(1); + expect(h.jobs).toHaveLength(2); + + h.jobs[1]!.settle('ok'); + await flush(); + expect(shownTag(surface)).toBe(2); + expect(h.cache.peek('L', h.entry(), clarity(50))).toBe(surface); + }); + + it('falls back to a whole per-pixel interim after a write of unknown extent', async () => { + vi.useFakeTimers(); + const h = setup(); + const surface = h.show(clarity(50)); + h.jobs[0]!.settle('ok'); + await flush(); + + h.layers.publishPixels('L'); + h.show(clarity(50)); + expect(sourceReads(surface).at(-1)).toEqual([0, 0, W, H]); + expect(shownTag(surface)).toBe(0); + await vi.advanceTimersByTimeAsync(SPATIAL_CONTENT_DEBOUNCE_MS); + expect(h.jobs).toHaveLength(2); + }); + + it('runs one job per layer while scrubbing: the running result lands as an interim, then the latest', async () => { + const h = setup(); + const surface = h.show(clarity(10)); + h.show(clarity(20)); + h.show(clarity(30)); + expect(h.jobs).toHaveLength(1); + + h.jobs[0]!.settle('ok'); + await flush(); + expect(shownTag(surface)).toBe(1); + expect(h.settled).toEqual(['L']); + expect(h.cache.peek('L', h.entry(), clarity(30))).toBeNull(); + expect(h.jobs.map((job) => job.steps[0]!.amount)).toEqual([10, 30]); + + h.jobs[1]!.settle('ok'); + await flush(); + expect(shownTag(surface)).toBe(2); + expect(h.cache.peek('L', h.entry(), clarity(30))).toBe(surface); + }); + + it('lands a result from before a known edit, re-patching that edit, and stays inexact until the next job', async () => { + vi.useFakeTimers(); + const h = setup(); + const surface = h.show(clarity(50)); + h.layers.publishPixels('L', { height: 2, width: 2, x: 4, y: 2 }); + h.show(clarity(50)); + + h.jobs[0]!.settle('ok'); + await flush(); + expect(h.settled).toEqual(['L']); + expect(shownTag(surface)).toBe(1); + expect(sourceReads(surface).at(-1)).toEqual([4, 2, 2, 2]); + expect(h.cache.peek('L', h.entry(), clarity(50))).toBeNull(); + expect(h.statuses.at(-1)).toBe('computing'); + + await vi.advanceTimersByTimeAsync(SPATIAL_CONTENT_DEBOUNCE_MS); + h.jobs[1]!.settle('ok'); + await flush(); + expect(h.cache.peek('L', h.entry(), clarity(50))).toBe(surface); + }); + + it('discards a result whose source changed in ways it cannot re-patch', async () => { + vi.useFakeTimers(); + const h = setup(); + const surface = h.show(clarity(50)); + h.layers.publishPixels('L'); + h.show(clarity(50)); + + h.jobs[0]!.settle('ok'); + await flush(); + expect(h.settled).toEqual([]); + expect(shownTag(surface)).toBe(0); + expect(h.statuses.at(-1)).toBe('computing'); + + await vi.advanceTimersByTimeAsync(SPATIAL_CONTENT_DEBOUNCE_MS); + h.jobs[1]!.settle('ok'); + await flush(); + expect(shownTag(surface)).toBe(2); + expect(h.cache.peek('L', h.entry(), clarity(50))).toBe(surface); + }); + + it('is exact again on returning to the shown parameters, abandoning the job for the others', async () => { + const h = setup(); + const surface = h.show(clarity(50)); + h.jobs[0]!.settle('ok'); + await flush(); + h.show(clarity(80)); + expect(h.statuses.at(-1)).toBe('computing'); + + h.show(clarity(50)); + expect(h.cache.peek('L', h.entry(), clarity(50))).toBe(surface); + expect(h.statuses.at(-1)).toBeNull(); + h.jobs[1]!.settle('ok'); + await flush(); + expect(shownTag(surface)).toBe(1); + expect(h.settled).toEqual(['L']); + }); + + it('rebuilds an evicted surface as an interim and recomputes it; a result for an evicted surface is dropped', async () => { + const h = setup(); + h.show(clarity(50)); + h.derived.evict(0, () => false); + h.jobs[0]!.settle('ok'); + await flush(); + expect(h.settled).toEqual([]); + expect(h.statuses.at(-1)).toBeNull(); + + const rebuilt = h.show(clarity(50)); + expect(shownTag(rebuilt)).toBe(0); + expect(h.statuses.at(-1)).toBe('computing'); + h.jobs[1]!.settle('ok'); + await flush(); + expect(shownTag(rebuilt)).toBe(2); + expect(h.cache.peek('L', h.entry(), clarity(50))).toBe(rebuilt); + }); + + it('reports too-large, error and over-budget, keeps the interim and retries only after a change', async () => { + vi.useFakeTimers(); + const h = setup(); + const surface = h.show(clarity(50)); + h.jobs[0]!.settle('too-large'); + await flush(); + expect(h.statuses.at(-1)).toBe('too-large'); + expect(shownTag(surface)).toBe(0); + h.show(clarity(50)); + expect(h.jobs).toHaveLength(1); + + // Too large depends on the content alone, so scrubbing the stack never reads the source back for another job. + const reads = sourceReads(surface).length; + h.show(clarity(60)); + h.show(clarity(70)); + expect(h.jobs).toHaveLength(1); + expect(sourceReads(surface)).toHaveLength(reads); + expect(h.statuses.at(-1)).toBe('too-large'); + + // New content may fit. + h.layers.publishPixels('L'); + h.show(clarity(70)); + expect(h.statuses.at(-1)).toBe('computing'); + await vi.advanceTimersByTimeAsync(SPATIAL_CONTENT_DEBOUNCE_MS); + expect(h.jobs).toHaveLength(2); + h.jobs[1]!.settle('error'); + await flush(); + expect(h.statuses.at(-1)).toBe('error'); + expect(h.settled).toEqual([]); + // An error belongs to its stack: a parameter change retries at once. + h.show(clarity(80)); + expect(h.jobs).toHaveLength(3); + + const starved = setup({ overBudget: true }); + starved.show(clarity(50)); + await flush(); + expect(starved.statuses).toEqual(['computing', 'over-budget']); + expect(starved.jobs).toHaveLength(0); + }); + + it('shows only the per-pixel part of a stack that cannot run, repainting it when that part changes', async () => { + const h = setup(); + const stack = (brightness: number, amount = 50): CanvasAdjustmentsContract => [ + { brightness, contrast: 0, id: 'b', isEnabled: true, type: 'brightness-contrast' }, + ...clarity(amount), + ]; + const surface = h.show(stack(0.2)); + // Edited while the job runs: the shown pixels wait for a result that will not come. + h.show(stack(0.4)); + const reads = sourceReads(surface).length; + h.jobs[0]!.settle('too-large'); + await flush(); + expect(h.jobs).toHaveLength(1); + expect(sourceReads(surface).slice(reads)).toEqual([[0, 0, W, H]]); + expect(h.settled).toEqual(['L']); + + // A per-pixel edit repaints at once; a spatial one changes nothing. + h.show(stack(0.6)); + expect(sourceReads(surface).slice(reads)).toEqual([ + [0, 0, W, H], + [0, 0, W, H], + ]); + h.show(stack(0.6, 90)); + expect(sourceReads(surface)).toHaveLength(reads + 2); + expect(h.jobs).toHaveLength(1); + expect(h.statuses.at(-1)).toBe('too-large'); + }); + + it('keeps exact surfaces through a cancel and drops the interims of the jobs it abandons', async () => { + const h = setup(); + h.layers.getOrCreate('M', W, H); + h.layers.publishPixels('M'); + const exact = h.show(clarity(50)); + h.jobs[0]!.settle('ok'); + await flush(); + h.cache.get('M', h.layers.get('M')!, clarity(50)); + expect(h.statuses.at(-1)).toBe('computing'); + + h.cache.cancelSpatial(); + expect(h.statuses.at(-1)).toBeNull(); + expect(h.cache.peek('L', h.entry(), clarity(50))).toBe(exact); + expect(h.cache.peekDisplayed('M', h.layers.get('M')!, clarity(50))).toBeNull(); + h.jobs[1]!.settle('ok'); + await flush(); + expect(h.settled).toEqual(['L']); + + expect(h.show(clarity(50))).toBe(exact); + expect(h.jobs).toHaveLength(2); + }); + + it('drops the spatial state of a layer its owner no longer keeps, though nothing draws it', async () => { + const h = setup(); + h.show(clarity(50)); + h.jobs[0]!.settle('too-large'); + await flush(); + + h.cache.retainSpatial(() => true); + expect(h.statuses.at(-1)).toBe('too-large'); + h.cache.retainSpatial((layerId) => layerId !== 'L'); + expect(h.statuses.at(-1)).toBeNull(); + expect(h.cache.size()).toBe(0); + }); + + it('abandons spatial work when the layer is dropped or its stack loses its spatial entries', async () => { + const h = setup(); + h.show(clarity(50)); + h.cache.delete('L'); + expect(h.statuses).toEqual(['computing', null]); + h.jobs[0]!.settle('ok'); + await flush(); + expect(h.settled).toEqual([]); + expect(h.cache.size()).toBe(0); + + h.show(clarity(50)); + const perPixel = h.show([{ brightness: 0.3, contrast: 0, id: 'b', isEnabled: true, type: 'brightness-contrast' }]); + expect(h.statuses.at(-1)).toBeNull(); + expect( + h.cache.peek('L', h.entry(), [ + { brightness: 0.3, contrast: 0, id: 'b', isEnabled: true, type: 'brightness-contrast' }, + ]) + ).toBe(perPixel); + }); + + it('starts no job for a placeholder cache, and one at once for its first published pixels', () => { + const h = setup(); + h.layers.delete('L'); + h.layers.getOrCreate('L', W, H); + h.show(clarity(50)); + expect(h.jobs).toHaveLength(0); + expect(h.statuses).toEqual([]); + + h.layers.publishPixels('L'); + h.show(clarity(50)); + expect(h.jobs).toHaveLength(1); + expect(h.statuses).toEqual(['computing']); + }); + + it('settles to the kernel result through the real service and inline executor', async () => { + const h = setup({ executor: createInlineSpatialExecutor() }); + const surface = h.show(clarity(50)); + await flush(); + expect(h.settled).toEqual(['L']); + expect(h.cache.peek('L', h.entry(), clarity(50))).toBe(surface); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceCache.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceCache.ts index 11e5b7b2890..ba936f7b884 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceCache.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceCache.ts @@ -1,40 +1,75 @@ import type { CanvasAdjustmentsContract } from '@workbench/canvas-engine/contracts'; import type { Rect } from '@workbench/canvas-engine/types'; -import { intersect, roundOut } from '@workbench/canvas-engine/math/rect'; - import type { LayerCacheEntry } from './layerCache'; import type { RasterBackend, RasterSurface } from './raster'; -import { adjustmentsKey, applyAdjustments, isIdentityAdjustments } from './adjustments'; +import { adjustmentsKey, hasSpatialAdjustments, isIdentityAdjustments } from './adjustments'; import { createDerivedSurfaceCache, type DerivedSurfaceCache } from './derivedSurfaceCache'; +import { + clampDamage, + createSpatialDisplay, + type DisplayedPixelsState, + paintAdjustedRegion, + type SpatialDisplayOptions, + type SpatialDisplayTarget, +} from './spatialDisplay'; + +export interface AdjustedSurfaceCacheOptions { + /** Surface-local {@link LayerCacheStore.damageSince}; without it every content change rebuilds whole surfaces. */ + readonly damageSince?: (layerId: string, version: number) => Rect | null; + /** Without it, spatial entries are left out of the display (the per-pixel part only). */ + readonly spatial?: SpatialDisplayOptions; +} /** The imperative adjusted-surface store returned by {@link createAdjustedSurfaceCache}. */ export interface AdjustedSurfaceCache { /** - * Returns a surface holding `entry`'s pixels with `adjustments` applied, or - * `null` when the adjustments are identity (the caller should draw the original - * cache surface). The returned surface is memoized: an unchanged - * `(entry.version, adjustmentsKey)` reuses it with zero pixel work. + * Returns the surface the display shows for `entry`'s pixels with `adjustments`, or `null` when the adjustments are + * identity (the caller should draw the original cache surface). Memoized: an unchanged + * `(entry.version, adjustmentsKey)` reuses it with zero pixel work. A stack with spatial entries shows an interim + * until its asynchronous evaluation lands (see `SpatialDisplayOptions.onSettle`). */ get( layerId: string, entry: LayerCacheEntry, adjustments: CanvasAdjustmentsContract | undefined ): RasterSurface | null; - /** The memoized adjusted surface when it is current for `entry` and `adjustments`; never builds one. */ + /** The memoized adjusted surface when it is current and exact for `entry` and `adjustments`; never builds one. */ peek( layerId: string, entry: LayerCacheEntry, adjustments: CanvasAdjustmentsContract | undefined ): RasterSurface | null; - /** Drops a layer's memoized adjusted surface (e.g. on layer delete). */ + /** What the display currently shows for `entry` and `adjustments`, exact or interim; never builds one. */ + peekDisplayed( + layerId: string, + entry: LayerCacheEntry, + adjustments: CanvasAdjustmentsContract | undefined + ): RasterSurface | null; + /** How final what `get` shows for `entry` and `adjustments` is; a stack without spatial entries is exact. */ + state( + layerId: string, + entry: LayerCacheEntry, + adjustments: CanvasAdjustmentsContract | undefined + ): DisplayedPixelsState; + /** Drops a layer's memoized adjusted surface and abandons its spatial work (e.g. on layer delete). */ delete(layerId: string): void; + /** + * Abandons running and scheduled spatial jobs, dropping the interims they would have replaced; exact and failed + * spatial surfaces keep their pixels and state. + */ + cancelSpatial(): void; + /** + * Drops the spatial state and surface of every layer `keep` rejects, e.g. one whose stack lost its spatial entries + * or that was removed while not drawn, so no stale status outlives its stack. + */ + retainSpatial(keep: (layerId: string) => boolean): void; /** Number of memoized adjusted surfaces (for tests / accounting). */ size(): number; /** Bytes held by adjusted surfaces. */ byteSize(): number; - /** Releases all memoized surfaces. */ + /** Releases every memoized derived surface and abandons all spatial work; the cache stays usable. */ dispose(): void; } @@ -42,33 +77,16 @@ export interface AdjustedSurfaceCache { export const createAdjustedSurfaceCache = ( backend: RasterBackend, cache: DerivedSurfaceCache = createDerivedSurfaceCache(), - /** Optional surface-local {@link LayerCacheStore.damageSince} lookup; callers without it rebuild whole surfaces. */ - layerDamageSince?: (layerId: string, version: number) => Rect | null + options: AdjustedSurfaceCacheOptions = {} ): AdjustedSurfaceCache => { - /** The damaged region clamped to the surface, or `null` to rebuild in full. */ - const damageSince = (layerId: string, version: number, width: number, height: number): Rect | null => { - const damaged = layerDamageSince?.(layerId, version) ?? null; - if (!damaged) { - return null; - } - const clamped = intersect(roundOut(damaged), { height, width, x: 0, y: 0 }); - return clamped && clamped.width > 0 && clamped.height > 0 ? clamped : null; - }; + const display = options.spatial ? createSpatialDisplay(options.spatial, 'adjusted:') : null; - const get = ( + const getPerPixel = ( layerId: string, entry: LayerCacheEntry, adjustments: CanvasAdjustmentsContract | undefined - ): RasterSurface | null => { - if (isIdentityAdjustments(adjustments)) { - cache.delete(layerId, 'adjustments'); - return null; - } + ): RasterSurface => { const { height, width } = entry.surface; - if (width <= 0 || height <= 0) { - return null; - } - const key = adjustmentsKey(adjustments); return cache.get({ create: (target, reusableFromVersion) => { const surface = target ?? backend.createSurface(width, height); @@ -79,57 +97,151 @@ export const createAdjustedSurfaceCache = ( // Source versions change every stroke tick. Refresh only reported damage to avoid full-surface readback // and adjustment work; resized surfaces always rebuild. const refresh = - resized || reusableFromVersion === null ? null : damageSince(layerId, reusableFromVersion, width, height); - const region = refresh ?? { height, width, x: 0, y: 0 }; - const ctx = surface.ctx; - ctx.setTransform(1, 0, 0, 1, 0, 0); - ctx.clearRect(region.x, region.y, region.width, region.height); - ctx.globalAlpha = 1; - ctx.globalCompositeOperation = 'source-over'; - ctx.drawImage( - entry.surface.canvas, - region.x, - region.y, - region.width, - region.height, - region.x, - region.y, - region.width, - region.height - ); - const imageData = ctx.getImageData(region.x, region.y, region.width, region.height); - // A layer's stack works in layer-local pixels; surface (0,0) sits at `entry.rect`'s origin. - applyAdjustments(imageData, adjustments, { x: entry.rect.x + region.x, y: entry.rect.y + region.y }); - ctx.putImageData(imageData, region.x, region.y); + resized || reusableFromVersion === null + ? null + : clampDamage(options.damageSince?.(layerId, reusableFromVersion) ?? null, width, height); + if (refresh !== 'none') { + // A layer's stack works in layer-local pixels; surface (0,0) sits at `entry.rect`'s origin. + paintAdjustedRegion( + surface, + entry.surface, + refresh ?? { height, width, x: 0, y: 0 }, + adjustments, + entry.rect + ); + } return surface; }, kind: 'adjustments', layerId, - paramsKey: key, + paramsKey: adjustmentsKey(adjustments), source: entry.surface, sourceVersion: entry.version, }); }; + /** Forgets a layer's spatial state with the surface it describes, so no later request reuses that surface blind. */ + const dropSlot = (layerId: string): void => { + if (display?.drop(layerId)) { + cache.delete(layerId, 'adjustments'); + } + }; + + const dropSlots = (layerIds: readonly string[]): void => { + for (const layerId of layerIds) { + dropSlot(layerId); + } + }; + + /** A spatial stack: an interim now, the exact pixels when the worker lands them. */ + const getSpatial = ( + spatial: NonNullable, + layerId: string, + entry: LayerCacheEntry, + adjustments: CanvasAdjustmentsContract, + key: string + ): RasterSurface => { + const source = entry.surface; + const version = entry.version; + const target: SpatialDisplayTarget = { + adjustments, + damageSince: (from) => options.damageSince?.(layerId, from) ?? null, + hold: entry.hasPublishedPixels ? null : 'placeholder', + key, + liveVersion: () => (entry.surface === source ? entry.version : null), + origin: { x: entry.rect.x, y: entry.rect.y }, + shown: () => cache.peek({ kind: 'adjustments', layerId, paramsKey: key, source, sourceVersion: version }), + source, + version, + }; + const { height, width } = source; + let rebuilt = false; + const surface = cache.get({ + create: (reusable) => { + const surface = reusable ?? backend.createSurface(width, height); + const resized = surface.width !== width || surface.height !== height; + if (resized) { + surface.resize(width, height); + } + rebuilt = spatial.paintInterim(layerId, surface, reusable !== null && !resized, target); + return surface; + }, + kind: 'adjustments', + layerId, + paramsKey: key, + source, + sourceVersion: version, + }); + spatial.request(layerId, target, rebuilt); + return surface; + }; + + const get = ( + layerId: string, + entry: LayerCacheEntry, + adjustments: CanvasAdjustmentsContract | undefined + ): RasterSurface | null => { + if (isIdentityAdjustments(adjustments)) { + dropSlot(layerId); + cache.delete(layerId, 'adjustments'); + return null; + } + if (entry.surface.width <= 0 || entry.surface.height <= 0) { + return null; + } + if (display && hasSpatialAdjustments(adjustments)) { + return getSpatial(display, layerId, entry, adjustments!, adjustmentsKey(adjustments)); + } + dropSlot(layerId); + return getPerPixel(layerId, entry, adjustments); + }; + + const peekDisplayed = ( + layerId: string, + entry: LayerCacheEntry, + adjustments: CanvasAdjustmentsContract | undefined + ): RasterSurface | null => + isIdentityAdjustments(adjustments) + ? null + : cache.peek({ + kind: 'adjustments', + layerId, + paramsKey: adjustmentsKey(adjustments), + source: entry.surface, + sourceVersion: entry.version, + }); + + const isExact = (layerId: string, entry: LayerCacheEntry, adjustments: CanvasAdjustmentsContract | undefined) => + !display || + !hasSpatialAdjustments(adjustments) || + display.isExact(layerId, { key: adjustmentsKey(adjustments), source: entry.surface, version: entry.version }); + return { byteSize: cache.byteSize, delete: (layerId) => { + dropSlot(layerId); cache.delete(layerId, 'adjustments'); }, + cancelSpatial: () => { + for (const layerId of display?.cancel() ?? []) { + cache.delete(layerId, 'adjustments'); + } + }, dispose: () => { + dropSlots(display?.ownerIds() ?? []); cache.dispose(); }, + retainSpatial: (keep) => dropSlots(display?.ownerIds().filter((layerId) => !keep(layerId)) ?? []), get, - peek: (layerId, entry, adjustments) => - isIdentityAdjustments(adjustments) - ? null - : cache.peek({ - kind: 'adjustments', - layerId, - paramsKey: adjustmentsKey(adjustments), - source: entry.surface, - sourceVersion: entry.version, - }), + peek: (layerId, entry, adjustments) => { + const displayed = peekDisplayed(layerId, entry, adjustments); + return displayed && isExact(layerId, entry, adjustments) ? displayed : null; + }, + peekDisplayed, size: cache.size, + state: (layerId, entry, adjustments) => + display && hasSpatialAdjustments(adjustments) + ? display.state(layerId, { key: adjustmentsKey(adjustments), source: entry.surface, version: entry.version }) + : 'exact', }; }; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceDamage.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceDamage.browser.test.ts index 357ed2f95fc..b5d2598400f 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceDamage.browser.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceDamage.browser.test.ts @@ -1,13 +1,23 @@ -/** Real-canvas tests require damage-only adjusted-surface refreshes to match full rebuilds pixel for pixel. */ +/** + * Real-canvas tests require damage-only adjusted-surface refreshes to match full rebuilds pixel for pixel, and + * spatial stacks to settle on the worker's exact result. + */ import type { CanvasAdjustmentsContract } from '@workbench/canvas-engine/contracts'; import type { Rect } from '@workbench/canvas-engine/types'; +import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; import { createAdjustedSurfaceCache } from '@workbench/canvas-engine/render/adjustedSurfaceCache'; +import { applyAdjustments, perPixelAdjustments, toSpatialPlan } from '@workbench/canvas-engine/render/adjustments'; import { createDerivedSurfaceCache } from '@workbench/canvas-engine/render/derivedSurfaceCache'; import { createLayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; import { createDomRasterBackend } from '@workbench/canvas-engine/render/raster'; -import { describe, expect, it } from 'vitest'; +import { evaluateSpatialSteps } from '@workbench/canvas-engine/render/spatial/evaluateSpatialSteps'; +import { + createSpatialAdjustmentService, + type SpatialAdjustmentService, +} from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import { afterEach, describe, expect, it } from 'vitest'; const WIDTH = 320; const HEIGHT = 240; @@ -40,11 +50,9 @@ const setup = (withDamageTracking: boolean) => { entry.surface.ctx.fillStyle = gradient; entry.surface.ctx.fillRect(0, 0, WIDTH, HEIGHT); layers.publishPixels('L'); - const adjusted = createAdjustedSurfaceCache( - backend, - createDerivedSurfaceCache(), - withDamageTracking ? (layerId, version) => layers.damageSince(layerId, version) : undefined - ); + const adjusted = createAdjustedSurfaceCache(backend, createDerivedSurfaceCache(), { + damageSince: withDamageTracking ? (layerId, version) => layers.damageSince(layerId, version) : undefined, + }); return { adjusted, entry, layers }; }; @@ -176,3 +184,114 @@ describe('partially-refreshed adjusted surfaces match a full rebuild', () => { expect(worstDifference(grown.ctx.getImageData(30, 40, WIDTH, HEIGHT).data, before)).toBe(0); }); }); + +describe('spatially adjusted surfaces', () => { + // Per-pixel runs on both sides of the spatial step exercise the stack's order across the worker boundary. + const STACK: CanvasAdjustmentsContract = [ + { brightness: 0.1, contrast: 0.05, id: 'bc', isEnabled: true, type: 'brightness-contrast' }, + { amount: 70, id: 'clarity', isEnabled: true, size: 24, type: 'clarity' }, + { id: 'sat', isEnabled: true, saturation: -0.2, type: 'hsl' }, + ]; + + const services: SpatialAdjustmentService[] = []; + afterEach(() => { + for (const service of services.splice(0)) { + service.dispose(); + } + }); + + /** A gradient layer with a transparent border, so the kernel's alpha-bounds domain is a strict sub-rectangle. */ + const setupSpatial = () => { + const backend = createDomRasterBackend(); + const layers = createLayerCacheStore(backend); + layers.getOrCreate('L', WIDTH, HEIGHT); + const entry = layers.get('L')!; + const gradient = entry.surface.ctx.createLinearGradient(0, 0, WIDTH, HEIGHT); + gradient.addColorStop(0, '#1d4ed8'); + gradient.addColorStop(0.5, '#e2e8f0'); + gradient.addColorStop(1, '#f59e0b'); + entry.surface.ctx.fillStyle = gradient; + entry.surface.ctx.fillRect(16, 12, WIDTH - 40, HEIGHT - 30); + // A dark block gives the clarity step edges to work on. + entry.surface.ctx.fillStyle = '#0f172a'; + entry.surface.ctx.fillRect(150, 40, 60, 50); + layers.publishPixels('L'); + const budget = new RasterMemoryBudgetController({ budgetBytes: 64 * 1024 * 1024 }); + const service = createSpatialAdjustmentService({ + reserve: (bytes) => budget.reserveOperation(bytes, { purpose: 'spatial-adjustment' }), + }); + services.push(service); + let settled: (() => void) | null = null; + const adjusted = createAdjustedSurfaceCache(backend, createDerivedSurfaceCache(), { + damageSince: (layerId, version) => layers.damageSince(layerId, version), + spatial: { onSettle: () => settled?.(), run: (request) => service.run(request) }, + }); + const nextSettle = () => + new Promise((resolve) => { + settled = resolve; + }); + const show = () => adjusted.get('L', layers.get('L')!, STACK)!; + return { adjusted, backend, layers, nextSettle, show }; + }; + + /** Reads back through a canvas as the display does, so unpremultiplied values round like the shown pixels. */ + const roundTrip = (backend: ReturnType, pixels: Uint8ClampedArray) => { + const scratch = backend.createSurface(WIDTH, HEIGHT); + const image = scratch.ctx.createImageData(WIDTH, HEIGHT); + image.data.set(pixels); + scratch.ctx.putImageData(image, 0, 0); + return pixelsOf(scratch); + }; + + /** The whole stack evaluated on the main thread: what the worker path must land exactly. */ + const evaluateStack = (source: Uint8ClampedArray): Uint8ClampedArray => { + const pixels = source.slice(); + for (const run of toSpatialPlan(STACK)) { + if (run.kind === 'per-pixel') { + applyAdjustments({ data: pixels, height: HEIGHT, width: WIDTH }, run.adjustments, { x: 0, y: 0 }); + } else { + expect(evaluateSpatialSteps(pixels, WIDTH, HEIGHT, run.steps)).toEqual({ status: 'ok' }); + } + } + return pixels; + }; + + it('shows the per-pixel interim under a stroke, then settles to exactly the evaluated stack', async () => { + const scene = setupSpatial(); + const firstSettle = scene.nextSettle(); + const shown = scene.show(); + expect(scene.adjusted.peek('L', scene.layers.get('L')!, STACK)).toBeNull(); + await firstSettle; + const settledPixels = pixelsOf(shown); + const source = () => pixelsOf(scene.layers.get('L')!.surface); + const perPixelOnly = () => { + const pixels = source(); + applyAdjustments({ data: pixels, height: HEIGHT, width: WIDTH }, perPixelAdjustments(STACK), { x: 0, y: 0 }); + return pixels; + }; + expect(worstDifference(settledPixels, roundTrip(scene.backend, evaluateStack(source())))).toBe(0); + // The clarity step really changed pixels, so the comparisons below can tell interim from result. + expect(worstDifference(settledPixels, roundTrip(scene.backend, perPixelOnly()))).toBeGreaterThan(0); + expect(scene.adjusted.peek('L', scene.layers.get('L')!, STACK)).toBe(shown); + + const stroke = { height: 30, width: 50, x: 60, y: 70 }; + const secondSettle = scene.nextSettle(); + paint(scene.layers, stroke, '#22c55e', true); + expect(scene.show()).toBe(shown); + + // Inside the stroke: the new pixels with the per-pixel part only. Outside: the settled result, untouched. + const interim = pixelsOf(shown); + const perPixel = perPixelOnly(); + const expectedInterim = settledPixels.slice(); + for (let y = stroke.y; y < stroke.y + stroke.height; y++) { + const start = (y * WIDTH + stroke.x) * 4; + expectedInterim.set(perPixel.subarray(start, start + stroke.width * 4), start); + } + expect(worstDifference(interim, roundTrip(scene.backend, expectedInterim))).toBe(0); + expect(scene.adjusted.peek('L', scene.layers.get('L')!, STACK)).toBeNull(); + + await secondSettle; + expect(worstDifference(pixelsOf(shown), roundTrip(scene.backend, evaluateStack(source())))).toBe(0); + expect(scene.adjusted.peek('L', scene.layers.get('L')!, STACK)).toBe(shown); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.test.ts index 55ea14df38c..e9e9a8a102b 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.test.ts @@ -7,11 +7,15 @@ import { applyAdjustments, buildCurveLut, compileAdjustments, + hasSpatialAdjustments, isIdentityAdjustmentEntry, isIdentityAdjustments, isPositionDependentAdjustments, + isSpatialAdjustmentEntry, NOISE_FULL_SCALE_SIGMA, noiseGrainSpan, + perPixelAdjustments, + toSpatialPlan, } from './adjustments'; /** Builds an ImageData-like object (node has no DOM ImageData; the shape is enough). */ @@ -786,3 +790,81 @@ describe('isPositionDependentAdjustments', () => { expect(isPositionDependentAdjustments([noise(0.1, { isEnabled: false })])).toBe(false); }); }); + +const clarity = (amount: number, size = 40, overrides: EntryOverrides = {}) => + ({ amount, id: 'cl', isEnabled: true, size, type: 'clarity', ...overrides }) as const; +const texture = (amount: number, size = 8, overrides: EntryOverrides = {}) => + ({ amount, id: 'tx', isEnabled: true, size, type: 'texture', ...overrides }) as const; +const dehaze = (amount: number, overrides: EntryOverrides = {}) => + ({ amount, id: 'dh', isEnabled: true, type: 'dehaze', ...overrides }) as const; + +describe('spatial entries', () => { + it('are identity at amount 0 and key every parameter', () => { + for (const entry of [clarity(0), texture(0), dehaze(0)]) { + expect(isIdentityAdjustmentEntry(entry)).toBe(true); + expect(adjustmentsKey([entry])).toBe('identity'); + } + expect(adjustmentsKey([clarity(25, 60)])).toBe('cl:25,60'); + expect(adjustmentsKey([texture(-10, 4)])).toBe('tx:-10,4'); + expect(adjustmentsKey([dehaze(30)])).toBe('dh:30'); + expect(adjustmentsKey([clarity(25, 60), bc(0.1, 0)])).not.toBe(adjustmentsKey([bc(0.1, 0), clarity(25, 60)])); + expect(adjustmentsKey([clarity(25, 60)])).not.toBe(adjustmentsKey([texture(25, 60)])); + }); + + it('are skipped by the per-pixel pass, which equals the stack without them', () => { + const pixels = [40, 120, 200, 255, 230, 10, 90, 128]; + const stack = [clarity(80), bc(0.2, 0.1), dehaze(-40), hsl(-0.5), texture(50)]; + expect(compileAdjustments([clarity(80), texture(-50), dehaze(10)])).toEqual([]); + expect(isIdentityAdjustments(perPixelAdjustments([clarity(80)]))).toBe(true); + + const withSpatial = imageData(pixels); + applyAdjustments(withSpatial, stack, { x: 0, y: 0 }); + const without = imageData(pixels); + applyAdjustments(without, [bc(0.2, 0.1), hsl(-0.5)], { x: 0, y: 0 }); + expect([...withSpatial.data]).toEqual([...without.data]); + expect(perPixelAdjustments(stack)).toEqual([bc(0.2, 0.1), hsl(-0.5)]); + }); + + it('count as spatial only while one contributes, and are not position dependent', () => { + expect([clarity(1), texture(1), dehaze(1)].every(isSpatialAdjustmentEntry)).toBe(true); + expect(isSpatialAdjustmentEntry(noise(0.1))).toBe(false); + expect(hasSpatialAdjustments(undefined)).toBe(false); + expect(hasSpatialAdjustments([bc(0.2, 0), noise(0.1)])).toBe(false); + expect(hasSpatialAdjustments([clarity(0), dehaze(20, { isEnabled: false })])).toBe(false); + expect(hasSpatialAdjustments([bc(0.2, 0), texture(-20)])).toBe(true); + expect(isPositionDependentAdjustments([clarity(50), texture(50), dehaze(50)])).toBe(false); + }); + + it('plan contributing entries as ordered runs, merging spatial neighbours across inert entries', () => { + expect(toSpatialPlan([])).toEqual([]); + expect(toSpatialPlan([bc(0.2, 0), hsl(0.1)])).toEqual([{ adjustments: [bc(0.2, 0), hsl(0.1)], kind: 'per-pixel' }]); + expect( + toSpatialPlan([ + bc(0.2, 0), + clarity(30, 50), + hsl(0.4, { isEnabled: false }), + texture(-20, 6), + exposure(0), + dehaze(15), + hsl(-0.3), + clarity(0), + ]) + ).toEqual([ + { adjustments: [bc(0.2, 0)], kind: 'per-pixel' }, + { + kind: 'spatial', + steps: [ + { amount: 30, kind: 'clarity', size: 50 }, + { amount: -20, kind: 'texture', size: 6 }, + { amount: 15, kind: 'dehaze' }, + ], + }, + { adjustments: [hsl(-0.3)], kind: 'per-pixel' }, + ]); + expect(toSpatialPlan([dehaze(10), bc(0.1, 0), dehaze(20)]).map((run) => run.kind)).toEqual([ + 'spatial', + 'per-pixel', + 'spatial', + ]); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.ts index 8466504145e..1d687a1e76a 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.ts @@ -1,4 +1,5 @@ import type { CanvasAdjustmentEntry, CanvasAdjustmentsContract } from '@workbench/canvas-engine/contracts'; +import type { SpatialAdjustmentStep } from '@workbench/canvas-engine/render/spatial/spatialSteps'; const LUT_SIZE = 256; @@ -137,6 +138,9 @@ export const isIdentityAdjustmentEntry = (entry: CanvasAdjustmentEntry): boolean case 'invert': return false; case 'noise': + case 'clarity': + case 'texture': + case 'dehaze': return entry.amount === 0; case 'curves': return isIdentityCurve(entry.curves.r) && isIdentityCurve(entry.curves.g) && isIdentityCurve(entry.curves.b); @@ -156,6 +160,60 @@ export const isIdentityAdjustments = (adjustments: CanvasAdjustmentsContract | u export const isPositionDependentAdjustments = (adjustments: CanvasAdjustmentsContract | undefined): boolean => adjustments?.some((entry) => entry.type === 'noise' && contributes(entry)) ?? false; +type SpatialAdjustmentEntry = Extract; + +/** + * Spatial entries read a neighborhood across the owner's whole surface (its alpha bounds), so they are neither + * region independent nor resample-commuting. `compileAdjustments` and `applyAdjustments` skip them: they run in the + * spatial worker on the owner's full source. + */ +export const isSpatialAdjustmentEntry = (entry: CanvasAdjustmentEntry): entry is SpatialAdjustmentEntry => + entry.type === 'clarity' || entry.type === 'texture' || entry.type === 'dehaze'; + +/** True when any contributing entry is spatial. */ +export const hasSpatialAdjustments = (adjustments: CanvasAdjustmentsContract | undefined): boolean => + adjustments?.some((entry) => isSpatialAdjustmentEntry(entry) && contributes(entry)) ?? false; + +/** The stack without its spatial entries: what `applyAdjustments` evaluates, and the display's interim. */ +export const perPixelAdjustments = (adjustments: CanvasAdjustmentsContract | undefined): CanvasAdjustmentsContract => + (adjustments ?? []).filter((entry) => !isSpatialAdjustmentEntry(entry)); + +/** One run of a {@link toSpatialPlan}: per-pixel entries for `applyAdjustments`, or steps for one spatial job. */ +export type SpatialPlanRun = + | { readonly kind: 'per-pixel'; readonly adjustments: CanvasAdjustmentsContract } + | { readonly kind: 'spatial'; readonly steps: readonly SpatialAdjustmentStep[] }; + +const spatialStep = (entry: SpatialAdjustmentEntry): SpatialAdjustmentStep => + entry.type === 'dehaze' + ? { amount: entry.amount, kind: 'dehaze' } + : { amount: entry.amount, kind: entry.type, size: entry.size }; + +/** + * Splits the contributing entries into ordered alternating runs, so evaluating them in turn equals the stack in list + * order. Consecutive spatial entries share one run (one worker job); empty means identity. + */ +export const toSpatialPlan = (adjustments: CanvasAdjustmentsContract | undefined): SpatialPlanRun[] => { + const runs: SpatialPlanRun[] = []; + for (const entry of adjustments ?? []) { + if (!contributes(entry)) { + continue; + } + const last = runs[runs.length - 1]; + if (isSpatialAdjustmentEntry(entry)) { + if (last?.kind === 'spatial') { + runs[runs.length - 1] = { kind: 'spatial', steps: [...last.steps, spatialStep(entry)] }; + } else { + runs.push({ kind: 'spatial', steps: [spatialStep(entry)] }); + } + } else if (last?.kind === 'per-pixel') { + runs[runs.length - 1] = { adjustments: [...last.adjustments, entry], kind: 'per-pixel' }; + } else { + runs.push({ adjustments: [entry], kind: 'per-pixel' }); + } + } + return runs; +}; + /** * One step of the compiled pixel pass. Per-channel entries fold into LUT * segments (adjacent ones compose into ONE lut); saturation and hue are @@ -423,13 +481,13 @@ const entryLuts = ( }; /** - * Compiles enabled nonidentity entries in order, folding adjacent per-channel operations into LUT trios. Empty - * means identity. + * Compiles enabled nonidentity per-pixel entries in order (spatial entries are skipped), folding adjacent per-channel + * operations into LUT trios. Empty means identity. */ export const compileAdjustments = (adjustments: CanvasAdjustmentsContract | undefined): CompiledAdjustmentSegment[] => { const segments: CompiledAdjustmentSegment[] = []; for (const entry of adjustments ?? []) { - if (!contributes(entry)) { + if (!contributes(entry) || isSpatialAdjustmentEntry(entry)) { continue; } if (entry.type === 'hsl') { @@ -486,6 +544,12 @@ export const adjustmentsKey = (adjustments: CanvasAdjustmentsContract | undefine return 'inv'; case 'noise': return `nz:${entry.distribution},${entry.grain},${entry.response},${entry.size},${entry.amount},${entry.seed}`; + case 'clarity': + return `cl:${entry.amount},${entry.size}`; + case 'texture': + return `tx:${entry.amount},${entry.size}`; + case 'dehaze': + return `dh:${entry.amount}`; case 'curves': return `cv:${curveKey(entry.curves.r)}|${curveKey(entry.curves.g)}|${curveKey(entry.curves.b)}`; } @@ -891,13 +955,21 @@ const applyNoise = ( ); }; +/** Unpremultiplied RGBA8 pixels, row-major: an `ImageData` or a worker result viewed the same way. */ +export interface AdjustablePixels { + readonly data: Uint8ClampedArray; + readonly width: number; + readonly height: number; +} + /** * Applies the compiled stack in place, preserving alpha; identity is a no-op. `origin` places the pixels in the - * stack's reference space. Every entry is region independent: adjusting a sub-rectangle with its matching origin - * yields exactly the pixels a whole-surface pass would there. + * stack's reference space. Spatial entries are skipped (see {@link isSpatialAdjustmentEntry}); every entry applied is + * region independent: adjusting a sub-rectangle with its matching origin yields exactly the pixels a whole-surface + * pass would there. */ export const applyAdjustments = ( - imageData: ImageData, + imageData: AdjustablePixels, adjustments: CanvasAdjustmentsContract | undefined, origin: AdjustmentOrigin ): void => { diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/exactAdjustments.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/exactAdjustments.test.ts new file mode 100644 index 00000000000..a1c3ee6d92e --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/exactAdjustments.test.ts @@ -0,0 +1,164 @@ +import type { CanvasAdjustmentsContract } from '@workbench/canvas-engine/contracts'; + +import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; +import { groupContract, stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { applyAdjustments } from '@workbench/canvas-engine/render/adjustments'; +import { adjustExactly, adjustmentRefusal } from '@workbench/canvas-engine/render/exactAdjustments'; +import { evaluateSpatialSteps } from '@workbench/canvas-engine/render/spatial/evaluateSpatialSteps'; +import { createSpatialAdjustmentService } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import { createInlineSpatialExecutor } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService.testing'; +import { describe, expect, it, vi } from 'vitest'; + +const WIDTH = 24; +const HEIGHT = 16; +const ORIGIN = { x: -3, y: 5 }; + +/** A diagonal gradient with a dark block and a transparent border: edges for the kernels, and a sub-domain. */ +const source = (): Uint8ClampedArray => { + const data = new Uint8ClampedArray(WIDTH * HEIGHT * 4); + for (let y = 2; y < HEIGHT - 2; y += 1) { + for (let x = 2; x < WIDTH - 2; x += 1) { + const i = (y * WIDTH + x) * 4; + const dark = x > 8 && x < 14 && y > 5 && y < 10; + data[i] = dark ? 20 : 60 + x * 6; + data[i + 1] = dark ? 25 : 90 + y * 5; + data[i + 2] = dark ? 30 : 200 - x * 4; + data[i + 3] = 255; + } + } + return data; +}; + +const BRIGHTEN = { brightness: 0.2, contrast: 0.1, id: 'bc', isEnabled: true, type: 'brightness-contrast' } as const; +const CLARITY = { amount: 80, id: 'cl', isEnabled: true, size: 6, type: 'clarity' } as const; +const GRAIN = { + amount: 0.2, + distribution: 'gaussian', + grain: 'color', + id: 'grain', + isEnabled: true, + response: 'flat', + seed: 3, + size: 1, + type: 'noise', +} as const; +const STACK: CanvasAdjustmentsContract = [BRIGHTEN, CLARITY, GRAIN]; + +const inlineRunner = () => { + const memory = new RasterMemoryBudgetController({ budgetBytes: 64 * 1024 * 1024 }); + const service = createSpatialAdjustmentService({ + createExecutor: createInlineSpatialExecutor, + reserve: (bytes) => memory.reserveOperation(bytes, { purpose: 'spatial-adjustment' }), + }); + return { memory, run: vi.fn(service.run) }; +}; + +describe('adjustExactly', () => { + it('evaluates the whole stack in list order, per-pixel runs at the origin and spatial runs in the service', async () => { + const { memory, run } = inlineRunner(); + + const result = await adjustExactly({ data: source(), height: HEIGHT, width: WIDTH }, STACK, ORIGIN, { run }); + + const expected = source(); + applyAdjustments({ data: expected, height: HEIGHT, width: WIDTH }, [BRIGHTEN], ORIGIN); + expect(evaluateSpatialSteps(expected, WIDTH, HEIGHT, [{ amount: 80, kind: 'clarity', size: 6 }])).toEqual({ + status: 'ok', + }); + applyAdjustments({ data: expected, height: HEIGHT, width: WIDTH }, [GRAIN], ORIGIN); + expect(result.status).toBe('ok'); + expect(result.status === 'ok' && [...result.data]).toEqual([...expected]); + expect(run).toHaveBeenCalledOnce(); + expect(run.mock.calls[0]![0]).toMatchObject({ priority: 'now' }); + expect(memory.snapshot().reservedBytes).toBe(0); + }); + + it('changes the pixels a per-pixel-only pass leaves, so a spatial entry is never dropped', async () => { + const { run } = inlineRunner(); + const exact = await adjustExactly({ data: source(), height: HEIGHT, width: WIDTH }, STACK, ORIGIN, { run }); + const perPixel = source(); + applyAdjustments({ data: perPixel, height: HEIGHT, width: WIDTH }, STACK, ORIGIN); + + expect(exact.status === 'ok' && [...exact.data]).not.toEqual([...perPixel]); + }); + + it('applies a stack without spatial entries in place, needing no runner', async () => { + const data = source(); + const result = await adjustExactly({ data, height: HEIGHT, width: WIDTH }, [BRIGHTEN, GRAIN], ORIGIN, {}); + + expect(result).toEqual({ data, status: 'ok' }); + }); + + it('refuses a spatial stack without a runner instead of leaving its spatial entries out', async () => { + await expect(adjustExactly({ data: source(), height: HEIGHT, width: WIDTH }, STACK, ORIGIN, {})).resolves.toEqual({ + status: 'error', + }); + }); + + it.each(['too-large', 'over-budget', 'error', 'aborted'] as const)('passes a %s job through', async (status) => { + const run = vi.fn(() => Promise.resolve({ status })); + + await expect( + adjustExactly({ data: source(), height: HEIGHT, width: WIDTH }, STACK, ORIGIN, { run }) + ).resolves.toEqual({ status }); + }); + + it('reports a job the service ran out of budget for as over-budget', async () => { + const memory = new RasterMemoryBudgetController({ budgetBytes: WIDTH * HEIGHT * 4 - 1 }); + const service = createSpatialAdjustmentService({ + createExecutor: createInlineSpatialExecutor, + reserve: (bytes) => memory.reserveOperation(bytes, { purpose: 'spatial-adjustment' }), + }); + + await expect( + adjustExactly({ data: source(), height: HEIGHT, width: WIDTH }, STACK, ORIGIN, { run: service.run }) + ).resolves.toEqual({ status: 'over-budget' }); + }); + + it('starts no job once its signal is aborted', async () => { + const run = vi.fn(); + const controller = new AbortController(); + controller.abort(); + + await expect( + adjustExactly({ data: source(), height: HEIGHT, width: WIDTH }, STACK, ORIGIN, { + run, + signal: controller.signal, + }) + ).resolves.toEqual({ status: 'aborted' }); + expect(run).not.toHaveBeenCalled(); + }); +}); + +describe('adjustmentRefusal', () => { + const layer = { + blendMode: 'normal', + id: 'leaf', + isEnabled: true, + isLocked: false, + name: 'Portrait', + opacity: 1, + source: { bitmap: null, offset: { x: 0, y: 0 }, type: 'paint' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'raster', + } as const; + const document = { + background: 'transparent' as const, + bbox: { height: 8, width: 8, x: 0, y: 0 }, + height: 8, + selectedLayerId: null, + stacks: stacksFrom([groupContract('group', [layer], { name: 'Background group' })]), + version: 3 as const, + width: 8, + }; + + it('names the layer or group whose stack could not run', () => { + expect(adjustmentRefusal(document, 'leaf', 'too-large')).toEqual({ + ownerId: 'leaf', + ownerName: 'Portrait', + reason: 'too-large', + status: 'adjustment-refused', + }); + expect(adjustmentRefusal(document, 'group', 'error')).toMatchObject({ ownerName: 'Background group' }); + expect(adjustmentRefusal(null, 'gone', 'over-budget')).toMatchObject({ ownerName: 'gone' }); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/exactAdjustments.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/exactAdjustments.ts new file mode 100644 index 00000000000..b9159546c40 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/exactAdjustments.ts @@ -0,0 +1,190 @@ +/** + * Exact adjustment evaluation for pixel-writing consumers (generation, exports, bakes, PSD, filter input, + * extraction). Per-pixel runs apply on the calling thread; spatial runs go to the shared spatial service at `now` + * priority. A stack is never evaluated with its spatial entries left out: when they cannot run, the consumer refuses + * with an {@link AdjustmentRefusal} naming the owner and the cause. + */ + +import type { CanvasAdjustmentsContract, CanvasDocumentContractV3 } from '@workbench/canvas-engine/contracts'; +import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; +import type { + SpatialRunPriority, + SpatialRunRequest, + SpatialRunResult, +} from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import type { Rect } from '@workbench/canvas-engine/types'; + +import { getDocumentIndex } from '@workbench/canvas-engine/document/documentIndex'; +import { + type AdjustablePixels, + type AdjustmentOrigin, + applyAdjustments, + toSpatialPlan, +} from '@workbench/canvas-engine/render/adjustments'; + +/** Why an owner's spatial adjustments could not run, as the spatial service reports it. */ +export type AdjustmentRefusalReason = 'too-large' | 'over-budget' | 'error'; + +/** A pixel-writing operation refused because one owner's spatial adjustments could not be evaluated exactly. */ +export interface AdjustmentRefusal { + readonly status: 'adjustment-refused'; + /** The layer or group whose stack could not run. */ + readonly ownerId: string; + readonly ownerName: string; + readonly reason: AdjustmentRefusalReason; +} + +/** Evaluates spatial steps off the main thread (`SpatialAdjustmentService.run`). */ +export type SpatialRunner = (request: SpatialRunRequest) => Promise; + +export interface ExactAdjustmentOptions { + /** Without a runner a spatial stack refuses `error` rather than leave its spatial entries out. */ + readonly run?: SpatialRunner; + readonly signal?: AbortSignal; +} + +export type ExactAdjustmentResult = + | { readonly status: 'ok'; readonly data: Uint8ClampedArray } + | { readonly status: 'aborted' | AdjustmentRefusalReason }; + +/** + * Thrown by renderers that report through exceptions (the raster composite) when an owner's stack cannot run; the + * operation boundary turns it into an {@link AdjustmentRefusal} with the owner's name. + */ +export class AdjustmentRefusedError extends Error { + constructor( + readonly ownerId: string, + readonly reason: AdjustmentRefusalReason + ) { + super(`The adjustments of ${ownerId} could not be applied (${reason}).`); + this.name = 'AdjustmentRefusedError'; + } +} + +/** The refusal for `ownerId`, named from `document` (the id when the owner is gone). */ +export const adjustmentRefusal = ( + document: CanvasDocumentContractV3 | null, + ownerId: string, + reason: AdjustmentRefusalReason +): AdjustmentRefusal => ({ + ownerId, + ownerName: (document && getDocumentIndex(document).byId.get(ownerId)?.node.name) || ownerId, + reason, + status: 'adjustment-refused', +}); + +/** Type guard for results that mix string statuses with an {@link AdjustmentRefusal}. */ +export const isAdjustmentRefusal = (value: unknown): value is AdjustmentRefusal => + typeof value === 'object' && value !== null && (value as { status?: unknown }).status === 'adjustment-refused'; + +export interface AdjustmentStackOptions { + /** Without a runner a stack with spatial entries resolves `error`. */ + readonly run?: SpatialRunner; + readonly signal?: AbortSignal; + readonly priority: SpatialRunPriority; + /** The service slot; a later request for the same slot supersedes a queued one. */ + readonly slotKey: string; +} + +/** + * Evaluates `adjustments` in list order on `pixels`, whose (0,0) sits at `origin` in the stack's reference space: + * per-pixel runs on this thread, each run of spatial entries as one service job. Spatial runs need `pixels` to be the + * owner's whole source (their domain is its alpha bounds) and `pixels.data` to own its entire buffer, which is + * transferred to the worker: afterwards only the returned pixels may be used. A stack without spatial entries is + * applied in place. Reports the service's outcome unchanged, `superseded` included. + */ +export const evaluateAdjustmentStack = async ( + pixels: AdjustablePixels, + adjustments: CanvasAdjustmentsContract | undefined, + origin: AdjustmentOrigin, + options: AdjustmentStackOptions +): Promise => { + const { height, width } = pixels; + let data = pixels.data; + for (const run of toSpatialPlan(adjustments)) { + if (run.kind === 'per-pixel') { + applyAdjustments({ data, height, width }, run.adjustments, origin); + continue; + } + if (!options.run) { + return { status: 'error' }; + } + if (options.signal?.aborted) { + return { status: 'aborted' }; + } + const result = await options.run({ + height, + pixels: data, + priority: options.priority, + signal: options.signal, + slotKey: options.slotKey, + steps: run.steps, + width, + }); + if (result.status !== 'ok') { + return result; + } + data = result.pixels; + } + return { pixels: data, status: 'ok' }; +}; + +// Every exact job gets its own slot, so concurrent consumers never supersede each other in the shared service. +let nextJob = 0; + +/** + * {@link evaluateAdjustmentStack} for pixel-writing consumers: spatial runs go at `now` priority, each job in its own + * service slot. + */ +export const adjustExactly = async ( + pixels: AdjustablePixels, + adjustments: CanvasAdjustmentsContract | undefined, + origin: AdjustmentOrigin, + options: ExactAdjustmentOptions +): Promise => { + const result = await evaluateAdjustmentStack(pixels, adjustments, origin, { + ...options, + priority: 'now', + slotKey: `exact:${(nextJob += 1)}`, + }); + if (result.status === 'ok') { + return { data: result.pixels, status: 'ok' }; + } + // Slots are unique, so nothing supersedes an exact job; treat it as a failure should it ever happen. + return { status: result.status === 'superseded' ? 'error' : result.status }; +}; + +/** `ImageData` is absent on the Node raster stub; a structural stand-in carries the same fields. */ +export const imageDataOf = (data: Uint8ClampedArray, width: number, height: number): ImageData => + typeof ImageData === 'undefined' + ? ({ colorSpace: 'srgb', data, height, width } as ImageData) + : new ImageData(data as Uint8ClampedArray, width, height); + +/** + * A new `rect`-sized surface holding `source`'s pixels at (0,0) with `adjustments` evaluated exactly; `rect`'s origin + * places it in the stack's reference space. Allocates the surface and one ImageData buffer (8 B/px), which the + * caller reserves; a spatial job reserves its in-flight buffer separately. + */ +export const adjustSurfaceExactly = async ( + createSurface: (width: number, height: number) => RasterSurface, + source: RasterSurface, + rect: Rect, + adjustments: CanvasAdjustmentsContract | undefined, + options: ExactAdjustmentOptions +): Promise< + { readonly status: 'ok'; readonly surface: RasterSurface } | Exclude +> => { + const { height, width } = rect; + const surface = createSurface(width, height); + const ctx = surface.ctx; + ctx.setTransform(1, 0, 0, 1, 0, 0); + ctx.clearRect(0, 0, width, height); + ctx.drawImage(source.canvas, 0, 0); + const image = ctx.getImageData(0, 0, width, height); + const result = await adjustExactly(image, adjustments, rect, options); + if (result.status !== 'ok') { + return result; + } + ctx.putImageData(result.data === image.data ? image : imageDataOf(result.data, width, height), 0, 0); + return { status: 'ok', surface }; +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupAdjustmentComposite.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupAdjustmentComposite.browser.test.ts index c8554fb6145..5bc2ed3ebce 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupAdjustmentComposite.browser.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupAdjustmentComposite.browser.test.ts @@ -1,14 +1,18 @@ import type { CanvasAdjustmentsContract, CanvasRasterLayerContractV2 } from '@workbench/canvas-engine/contracts'; +import type { GroupSurfaceResult } from '@workbench/canvas-engine/render/groupSurfaceCache'; import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Rect } from '@workbench/canvas-engine/types'; +import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; import { createCanvasDiagnostics } from '@workbench/canvas-engine/diagnostics'; import { groupContract, stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { identity } from '@workbench/canvas-engine/math/mat2d'; import { roundOut } from '@workbench/canvas-engine/math/rect'; import { createAdjustedSurfaceCache } from '@workbench/canvas-engine/render/adjustedSurfaceCache'; +import { applyAdjustments, toSpatialPlan } from '@workbench/canvas-engine/render/adjustments'; import { createColorSampler } from '@workbench/canvas-engine/render/colorSample'; import { compositeDocument } from '@workbench/canvas-engine/render/compositor'; +import { createDerivedSurfaceCache } from '@workbench/canvas-engine/render/derivedSurfaceCache'; import { createGroupSurfaceCache } from '@workbench/canvas-engine/render/groupSurfaceCache'; import { createLayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; import { createDomRasterBackend } from '@workbench/canvas-engine/render/raster'; @@ -17,7 +21,12 @@ import { planBaseRasterComposite, renderRasterComposite, } from '@workbench/canvas-engine/render/rasterComposite'; -import { describe, expect, it } from 'vitest'; +import { evaluateSpatialSteps } from '@workbench/canvas-engine/render/spatial/evaluateSpatialSteps'; +import { + createSpatialAdjustmentService, + type SpatialAdjustmentService, +} from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import { afterEach, describe, expect, it, vi } from 'vitest'; const WIDTH = 64; const HEIGHT = 64; @@ -56,6 +65,9 @@ const noiseStack = (id: string): CanvasAdjustmentsContract => [ type: 'noise', }, ]; +/** Members drawn from their raw caches, which are always final. */ +const RAW_MEMBERS = () => ({ state: 'exact' as const, surface: null }); + const worstDifference = (a: Uint8ClampedArray, b: Uint8ClampedArray): number => { let worst = 0; for (let i = 0; i < a.length; i += 1) { @@ -145,7 +157,7 @@ describe('group adjustment composite', () => { const scene = sceneFor({ red: '#ff4040', under: '#2040c0', white: '#c0c0c0' }); const groupSurfaces = createGroupSurfaceCache({ createSurface: (w, h) => scene.backend.createSurface(w, h), - getAdjustedSurface: () => null, + getAdjustedSurface: RAW_MEMBERS, damageSince: (id, version) => scene.caches.damageSince(id, version), getCacheEntry: (id) => scene.caches.get(id), }); @@ -182,7 +194,7 @@ describe('group adjustment composite', () => { const groupSurfaces = createGroupSurfaceCache({ createSurface: (w, h) => scene.backend.createSurface(w, h), damageSince: (id, version) => scene.caches.damageSince(id, version), - getAdjustedSurface: () => null, + getAdjustedSurface: RAW_MEMBERS, getCacheEntry: (id) => scene.caches.get(id), }); const screen = scene.backend.createSurface(WIDTH, HEIGHT); @@ -217,7 +229,7 @@ describe('group adjustment composite', () => { const groupSurfaces = createGroupSurfaceCache({ createSurface: (w, h) => scene.backend.createSurface(w, h), - getAdjustedSurface: () => null, + getAdjustedSurface: RAW_MEMBERS, damageSince: (id, version) => scene.caches.damageSince(id, version), getCacheEntry: (id) => scene.caches.get(id), }); @@ -234,7 +246,7 @@ describe('group adjustment composite', () => { const scene = sceneFor({ gray: '#404040' }); const groupSurfaces = createGroupSurfaceCache({ createSurface: (w, h) => scene.backend.createSurface(w, h), - getAdjustedSurface: () => null, + getAdjustedSurface: RAW_MEMBERS, damageSince: (id, version) => scene.caches.damageSince(id, version), getCacheEntry: (id) => scene.caches.get(id), }); @@ -265,7 +277,7 @@ describe('group adjustment composite', () => { const groupSurfaces = createGroupSurfaceCache({ createSurface: (w, h) => scene.backend.createSurface(w, h), - getAdjustedSurface: () => null, + getAdjustedSurface: RAW_MEMBERS, damageSince: (id, version) => scene.caches.damageSince(id, version), getCacheEntry: (id) => scene.caches.get(id), }); @@ -301,7 +313,7 @@ describe('group adjustment composite', () => { const groupSurfaces = createGroupSurfaceCache({ createSurface: (w, h) => nestedScene.backend.createSurface(w, h), - getAdjustedSurface: () => null, + getAdjustedSurface: RAW_MEMBERS, damageSince: (id, version) => nestedScene.caches.damageSince(id, version), getCacheEntry: (id) => nestedScene.caches.get(id), }); @@ -325,7 +337,7 @@ describe('group adjustment composite', () => { builds += 1; return scene.backend.createSurface(w, h); }, - getAdjustedSurface: () => null, + getAdjustedSurface: RAW_MEMBERS, damageSince: (id, version) => scene.caches.damageSince(id, version), getCacheEntry: (id) => scene.caches.get(id), }); @@ -356,7 +368,7 @@ describe('group adjustment composite', () => { createSurface: (w, h) => scene.backend.createSurface(w, h), damageSince: (id, version) => scene.caches.damageSince(id, version), diagnostics, - getAdjustedSurface: () => null, + getAdjustedSurface: RAW_MEMBERS, getCacheEntry: (id) => scene.caches.get(id), }); const docAt = (x: number) => @@ -443,7 +455,7 @@ describe('group-scoped previews and partial refresh', () => { createSurface: (w, h) => scene.backend.createSurface(w, h), damageSince: (id, version) => scene.caches.damageSince(id, version), diagnostics, - getAdjustedSurface: () => null, + getAdjustedSurface: RAW_MEMBERS, getCacheEntry: (id) => scene.caches.get(id), }); const memberNode = raster('member', member); @@ -728,3 +740,329 @@ describe('layer adjustments in export', () => { ]); }); }); + +describe('spatial group display', () => { + const MEMBER: Rect = { height: 30, width: 40, x: 0, y: 0 }; + const AT = { x: 9, y: 7 }; + const translated = { rotation: 0, scaleX: 1, scaleY: 1, ...AT }; + const services: SpatialAdjustmentService[] = []; + afterEach(() => { + for (const service of services.splice(0)) { + service.dispose(); + } + }); + + /** A gradient member with a transparent border and a dark block, so the kernels have edges and a sub-domain. */ + const spatialScene = () => { + const scene = sceneFor({}); + scene.caches.growToRect('m', MEMBER); + const ctx = scene.caches.get('m')!.surface.ctx; + const gradient = ctx.createLinearGradient(0, 0, MEMBER.width, MEMBER.height); + gradient.addColorStop(0, '#1d4ed8'); + gradient.addColorStop(0.5, '#e2e8f0'); + gradient.addColorStop(1, '#f59e0b'); + ctx.fillStyle = gradient; + ctx.fillRect(3, 2, MEMBER.width - 7, MEMBER.height - 5); + ctx.fillStyle = '#0f172a'; + ctx.fillRect(14, 9, 10, 8); + scene.caches.publishPixels('m'); + const budget = new RasterMemoryBudgetController({ budgetBytes: 64 * 1024 * 1024 }); + const service = createSpatialAdjustmentService({ + reserve: (bytes) => budget.reserveOperation(bytes, { purpose: 'spatial-adjustment' }), + }); + services.push(service); + return { scene, service }; + }; + + const pixelsOf = (surface: RasterSurface): Uint8ClampedArray => + surface.ctx.getImageData(0, 0, surface.width, surface.height).data; + + /** Writes pixels through a canvas and reads them back, rounding like the display. */ + const roundTrip = ( + backend: ReturnType, + pixels: Uint8ClampedArray, + width: number, + height: number + ) => { + const scratch = backend.createSurface(width, height); + const image = scratch.ctx.createImageData(width, height); + image.data.set(pixels); + scratch.ctx.putImageData(image, 0, 0); + return scratch; + }; + + /** The whole stack on the main thread, per-pixel runs at `origin`. */ + const evaluateStack = ( + pixels: Uint8ClampedArray, + width: number, + height: number, + stack: CanvasAdjustmentsContract, + origin = { x: 0, y: 0 } + ): Uint8ClampedArray => { + const out = pixels.slice(); + for (const run of toSpatialPlan(stack)) { + if (run.kind === 'per-pixel') { + applyAdjustments({ data: out, height, width }, run.adjustments, origin); + } else { + expect(evaluateSpatialSteps(out, width, height, run.steps)).toEqual({ status: 'ok' }); + } + } + return out; + }; + + /** The member's raw pixels placed on a transparent document-sized buffer: a larger domain than the group's. */ + const memberOnDocument = ( + scene: ReturnType, + member: RasterSurface = scene.caches.get('m')!.surface + ) => { + const document = scene.backend.createSurface(WIDTH, HEIGHT); + document.ctx.drawImage(member.canvas, AT.x, AT.y); + return document; + }; + + /** Composites until `done` holds, as frames would while results land; the content debounce runs in real time. */ + const settle = async (composite: () => void, done: () => boolean) => { + await vi.waitFor( + () => { + composite(); + expect(done()).toBe(true); + }, + { interval: 20, timeout: 10_000 } + ); + }; + + it('settles to the stack evaluated on the raw composite, independent of the buffer holding it', async () => { + // Per-pixel runs on both sides of the spatial step; noise places its grain by document pixel. + const stack: CanvasAdjustmentsContract = [ + { brightness: 0.1, contrast: 0.05, id: 'bc', isEnabled: true, type: 'brightness-contrast' }, + { amount: 70, id: 'clarity', isEnabled: true, size: 12, type: 'clarity' }, + ...noiseStack('grain'), + ]; + const { scene, service } = spatialScene(); + const document = docWith([groupContract('g', [raster('m', { transform: translated })], { adjustments: stack })]); + const results = new Map(); + const groupSurfaces = createGroupSurfaceCache({ + createSurface: (w, h) => scene.backend.createSurface(w, h), + damageSince: (id, version) => scene.caches.damageSince(id, version), + getAdjustedSurface: RAW_MEMBERS, + getCacheEntry: (id) => scene.caches.get(id), + spatial: { run: (request) => service.run(request) }, + }); + const screen = scene.backend.createSurface(WIDTH, HEIGHT); + const composite = () => { + screen.ctx.clearRect(0, 0, WIDTH, HEIGHT); + compositeDocument(screen, document, scene.caches, identity(), { + backend: scene.backend, + groupSurface: (scope, members, matrices, content) => { + const result = groupSurfaces.get(scope, members, matrices, content); + results.set(scope.id, result); + return result; + }, + }); + }; + + composite(); + expect(results.get('g')).toMatchObject({ rect: { height: 30, width: 40, ...AT }, state: 'pending' }); + await settle(composite, () => results.get('g')?.state === 'exact'); + + const raw = pixelsOf(memberOnDocument(scene)); + const expected = pixelsOf(roundTrip(scene.backend, evaluateStack(raw, WIDTH, HEIGHT, stack), WIDTH, HEIGHT)); + expect(worstDifference(pixelsOf(screen), expected)).toBe(0); + // The spatial step changed pixels, so the comparison above tells it apart from the per-pixel interim. + const perPixel = evaluateStack( + raw, + WIDTH, + HEIGHT, + stack.filter((entry) => entry.type !== 'clarity') + ); + expect(worstDifference(expected, pixelsOf(roundTrip(scene.backend, perPixel, WIDTH, HEIGHT)))).toBeGreaterThan(0); + }); + + it("repaints a group when a member's spatial result lands", async () => { + const memberStack: CanvasAdjustmentsContract = [ + { amount: 80, id: 'clarity', isEnabled: true, size: 12, type: 'clarity' }, + ]; + const member = raster('m', { adjustments: memberStack, transform: translated }); + const document = docWith([groupContract('g', [member], { adjustments: invertStack('inv') })]); + const { scene, service } = spatialScene(); + let landed = false; + // Wired as the engine does: a member's landing redraws the groups holding it. + const adjusted = createAdjustedSurfaceCache(scene.backend, createDerivedSurfaceCache(), { + damageSince: (id, version) => scene.caches.damageSince(id, version), + spatial: { + onSettle: () => { + landed = true; + groupSurfaces.invalidate(['g']); + }, + run: (request) => service.run(request), + }, + }); + const groupSurfaces = createGroupSurfaceCache({ + createSurface: (w, h) => scene.backend.createSurface(w, h), + damageSince: (id, version) => scene.caches.damageSince(id, version), + getAdjustedSurface: (layer, entry) => { + const surface = adjusted.get(layer.id, entry, layer.adjustments); + return { state: surface ? adjusted.state(layer.id, entry, layer.adjustments) : 'exact', surface }; + }, + getCacheEntry: (id) => scene.caches.get(id), + }); + const screen = scene.backend.createSurface(WIDTH, HEIGHT); + let state: GroupSurfaceResult['state'] | undefined; + const composite = () => { + screen.ctx.clearRect(0, 0, WIDTH, HEIGHT); + compositeDocument(screen, document, scene.caches, identity(), { + backend: scene.backend, + groupSurface: (scope, members, matrices, content) => { + const result = groupSurfaces.get(scope, members, matrices, content); + state = result?.state; + return result; + }, + }); + }; + const inverted = (pixels: Uint8ClampedArray) => { + const out = pixels.slice(); + applyAdjustments({ data: out, height: HEIGHT, width: WIDTH }, invertStack('inv'), { x: 0, y: 0 }); + return pixelsOf(roundTrip(scene.backend, out, WIDTH, HEIGHT)); + }; + + composite(); + expect(state).toBe('pending'); + const interim = pixelsOf(screen); + expect(worstDifference(interim, inverted(pixelsOf(memberOnDocument(scene))))).toBe(0); + await vi.waitFor(() => expect(landed).toBe(true), { timeout: 10_000 }); + composite(); + + expect(state).toBe('exact'); + const source = scene.caches.get('m')!.surface; + const memberResult = roundTrip( + scene.backend, + evaluateStack(pixelsOf(source), source.width, source.height, memberStack), + source.width, + source.height + ); + const expected = inverted(pixelsOf(memberOnDocument(scene, memberResult))); + expect(worstDifference(pixelsOf(screen), expected)).toBe(0); + expect(worstDifference(interim, expected)).toBeGreaterThan(0); + }); + + it('refreshes an edited and moved member in place to the pixels a fresh build settles to', async () => { + const { scene, service } = spatialScene(); + scene.caches.growToRect('b', { height: 8, width: 8, x: 0, y: 0 }); + const ctx = scene.caches.get('b')!.surface.ctx; + ctx.fillStyle = '#22c55e'; + ctx.fillRect(1, 1, 6, 6); + scene.caches.publishPixels('b'); + const stack: CanvasAdjustmentsContract = [ + { amount: 70, id: 'clarity', isEnabled: true, size: 12, type: 'clarity' }, + ]; + // `b` stays inside `m`'s bounds, so the group surface keeps its size and origin. + const documentFor = (opacity: number, x: number) => + docWith([ + groupContract( + 'g', + [ + raster('b', { opacity, transform: { rotation: 0, scaleX: 1, scaleY: 1, x, y: 15 } }), + raster('m', { transform: translated }), + ], + { adjustments: stack } + ), + ]); + const display = () => { + const diagnostics = createCanvasDiagnostics(true); + const groupSurfaces = createGroupSurfaceCache({ + createSurface: (w, h) => scene.backend.createSurface(w, h), + damageSince: (id, version) => scene.caches.damageSince(id, version), + diagnostics, + getAdjustedSurface: RAW_MEMBERS, + getCacheEntry: (id) => scene.caches.get(id), + spatial: { run: (request) => service.run(request) }, + }); + const screen = scene.backend.createSurface(WIDTH, HEIGHT); + let state: GroupSurfaceResult['state'] | undefined; + const show = (document: ReturnType) => + settle( + () => { + screen.ctx.clearRect(0, 0, WIDTH, HEIGHT); + compositeDocument(screen, document, scene.caches, identity(), { + backend: scene.backend, + groupSurface: (scope, members, matrices, content) => { + const result = groupSurfaces.get(scope, members, matrices, content); + state = result?.state; + return result; + }, + }); + }, + () => state === 'exact' + ); + return { diagnostics, screen, show }; + }; + + const edited = display(); + await edited.show(documentFor(1, 20)); + await edited.show(documentFor(0.5, 23)); + expect(edited.diagnostics.snapshot()).toMatchObject({ groupSurfaceAllocations: 1 }); + expect(edited.diagnostics.snapshot().groupSurfaceRefreshes).toBeGreaterThan(0); + + const fresh = display(); + await fresh.show(documentFor(0.5, 23)); + expect(worstDifference(pixelsOf(edited.screen), pixelsOf(fresh.screen))).toBe(0); + }); + + it('converges nested spatial groups to the inner result evaluated again by the outer stack', async () => { + const innerStack: CanvasAdjustmentsContract = [ + { amount: 90, id: 'texture', isEnabled: true, size: 4, type: 'texture' }, + ]; + const outerStack: CanvasAdjustmentsContract = [ + { amount: 60, id: 'clarity', isEnabled: true, size: 12, type: 'clarity' }, + ]; + const document = docWith([ + groupContract( + 'outer', + [groupContract('inner', [raster('m', { transform: translated })], { adjustments: innerStack })], + { adjustments: outerStack } + ), + ]); + const { scene, service } = spatialScene(); + const settled: string[] = []; + const groupSurfaces = createGroupSurfaceCache({ + createSurface: (w, h) => scene.backend.createSurface(w, h), + damageSince: (id, version) => scene.caches.damageSince(id, version), + getAdjustedSurface: RAW_MEMBERS, + getCacheEntry: (id) => scene.caches.get(id), + spatial: { + // Wired as the engine does: a landing redraws the groups holding the owner, never the owner itself. + onSettle: (groupId) => { + settled.push(groupId); + if (groupId === 'inner') { + groupSurfaces.invalidate(['outer']); + } + }, + run: (request) => service.run(request), + }, + }); + const screen = scene.backend.createSurface(WIDTH, HEIGHT); + let outerState: GroupSurfaceResult['state'] | undefined; + const composite = () => { + screen.ctx.clearRect(0, 0, WIDTH, HEIGHT); + compositeDocument(screen, document, scene.caches, identity(), { + backend: scene.backend, + groupSurface: (scope, members, matrices, content) => { + const result = groupSurfaces.get(scope, members, matrices, content); + outerState = scope.id === 'outer' ? result?.state : outerState; + return result; + }, + }); + }; + + composite(); + expect(outerState).toBe('pending'); + await settle(composite, () => outerState === 'exact'); + expect(settled).toEqual(['inner', 'outer']); + + const raw = pixelsOf(memberOnDocument(scene)); + const inner = pixelsOf(roundTrip(scene.backend, evaluateStack(raw, WIDTH, HEIGHT, innerStack), WIDTH, HEIGHT)); + const expected = pixelsOf(roundTrip(scene.backend, evaluateStack(inner, WIDTH, HEIGHT, outerStack), WIDTH, HEIGHT)); + expect(worstDifference(pixelsOf(screen), expected)).toBe(0); + expect(worstDifference(inner, expected)).toBeGreaterThan(0); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupSurfaceCache.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupSurfaceCache.test.ts new file mode 100644 index 00000000000..ffba33bf377 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupSurfaceCache.test.ts @@ -0,0 +1,509 @@ +import type { CanvasAdjustmentsContract, CanvasRasterLayerContractV2 } from '@workbench/canvas-engine/contracts'; +import type { SemanticLeaf } from '@workbench/canvas-engine/document-model/semanticLeaf'; +import type { + SpatialAdjustmentExecutor, + SpatialExecutorJob, + SpatialExecutorResult, +} from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import type { Mat2d } from '@workbench/canvas-engine/types'; + +import { createCanvasDiagnostics } from '@workbench/canvas-engine/diagnostics'; +import { identity } from '@workbench/canvas-engine/math/mat2d'; +import { createSpatialAdjustmentService } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import type { GroupCompositeScope, GroupSurfaceContent } from './groupCompositeScopes'; +import type { StubRasterSurface } from './raster.testStub'; +import type { AdjustedSurfaceStatus, DisplayedPixelsState } from './spatialDisplay'; + +import { createGroupSurfaceCache } from './groupSurfaceCache'; +import { createLayerCacheStore } from './layerCache'; +import { createTestStubRasterBackend } from './raster.testStub'; +import { SPATIAL_CONTENT_DEBOUNCE_MS } from './spatialDisplay'; + +const W = 8; +const H = 6; +const NO_CONTENT: GroupSurfaceContent = { excludeIds: new Set(), float: null, previews: null }; + +const clarity = (amount: number): CanvasAdjustmentsContract => [ + { amount, id: 'cl', isEnabled: true, size: 40, type: 'clarity' }, +]; + +const leaf = (id: string, layer: Partial = {}) => + ({ + id, + layer: { + adjustments: [], + blendMode: 'normal', + id, + opacity: 1, + type: 'raster', + ...layer, + } as unknown as CanvasRasterLayerContractV2, + }) as unknown as SemanticLeaf; + +const scope = ( + id: string, + adjustments: CanvasAdjustmentsContract, + children: readonly GroupCompositeScope[] = [] +): GroupCompositeScope => ({ adjustments, blendMode: 'normal', children, end: 1, id, opacity: 1, start: 0 }); + +/** An executor whose jobs settle when the test says; an `ok` job's pixels all become its 1-based job number. */ +const createManualExecutor = () => { + const jobs: { settle: (status: SpatialExecutorResult['status']) => void; steps: SpatialExecutorJob['steps'] }[] = []; + const executor: SpatialAdjustmentExecutor = { + dispose: () => undefined, + run: (job) => + new Promise((resolve) => { + const tag = jobs.length + 1; + jobs.push({ + settle: (status) => { + new Uint8ClampedArray(job.buffer).fill(status === 'ok' ? tag : 0); + resolve({ buffer: job.buffer, status }); + }, + steps: job.steps, + }); + }), + }; + return { executor, jobs }; +}; + +const flush = async () => { + for (let i = 0; i < 10; i++) { + await Promise.resolve(); + } +}; + +/** Member `b`'s cache: inside `a`'s, so editing or nudging it leaves the group bounds alone. */ +const B_RECT = { height: 2, width: 3, x: 2, y: 1 }; + +/** Member `a` in group `g`, plus `b` for `showPair`; nested setups put `inner` around `a` inside `outer`. */ +const setup = (ancestors: Readonly> = {}) => { + const { executor, jobs } = createManualExecutor(); + const backend = createTestStubRasterBackend({ readbackAlpha: 255 }); + const layers = createLayerCacheStore(backend); + layers.getOrCreateRect('a', { height: H, width: W, x: 0, y: 0 }); + layers.publishPixels('a'); + layers.getOrCreateRect('b', B_RECT); + layers.publishPixels('b'); + const service = createSpatialAdjustmentService({ + createExecutor: () => executor, + reserve: () => ({ lease: { release: () => undefined }, status: 'ok' }), + }); + const diagnostics = createCanvasDiagnostics(true); + const memberStates = new Map(); + /** The stack each member was last read with. */ + const memberStacks = new Map(); + const settled: string[] = []; + const statuses: [string, AdjustedSurfaceStatus | null][] = []; + const surfaces: StubRasterSurface[] = []; + const groups = createGroupSurfaceCache({ + createSurface: (width, height) => { + const surface = backend.createSurface(width, height) as StubRasterSurface; + surfaces.push(surface); + return surface; + }, + damageSince: (layerId, version) => layers.damageSince(layerId, version), + diagnostics, + getAdjustedSurface: (layer) => { + memberStacks.set(layer.id, layer.adjustments); + return { state: memberStates.get(layer.id) ?? 'exact', surface: null }; + }, + getCacheEntry: (layerId) => layers.get(layerId), + spatial: { + // Mirrors the engine: a landing redraws the groups holding the owner, never the owner itself. + onSettle: (groupId) => { + settled.push(groupId); + groups.invalidate(ancestors[groupId] ?? []); + }, + onStatus: (groupId, status) => statuses.push([groupId, status]), + run: (request) => service.run(request), + }, + }); + const show = (target: GroupCompositeScope) => groups.get(target, [leaf('a')], [identity()], NO_CONTENT)!; + /** Draws `a` under `b` in a two-member group. */ + const showPair = (stack: CanvasAdjustmentsContract, b: SemanticLeaf, bMatrix: Mat2d = identity()) => + groups.get({ ...scope('g', stack), end: 2 }, [leaf('a'), b], [identity(), bMatrix], NO_CONTENT)!; + const rawPaints = () => { + const { groupSurfaceRebuilds, groupSurfaceRefreshes } = diagnostics.snapshot(); + return groupSurfaceRebuilds + groupSurfaceRefreshes; + }; + return { + groups, + jobs, + layers, + memberStacks, + memberStates, + rawPaints, + settled, + show, + showPair, + statuses, + surfaces, + }; +}; + +/** The first byte of the last whole-surface write: 0 for a per-pixel interim, a job number for a result. */ +const shownTag = (surface: StubRasterSurface): number => { + const writes = surface.callLog.filter( + (call) => call.op === 'putImageData' && (call.args[0] as ImageData).width === W && call.args[1] === 0 + ); + return (writes[writes.length - 1]!.args[0] as ImageData).data[0]!; +}; + +/** Source regions drawn into a surface, as [x, y, width, height]. */ +const sourceReads = (surface: StubRasterSurface) => + surface.callLog + .filter((call) => call.op === 'drawImage' && call.args.length === 9) + .map((call) => call.args.slice(1, 5)); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('spatial group surfaces', () => { + it('keeps the raw composite and shows the evaluated stack on a separate surface', async () => { + const h = setup(); + const first = h.show(scope('g', clarity(50))); + const [raw, adjusted] = h.surfaces; + + expect(first.surface).toBe(adjusted); + expect(first.surface).not.toBe(raw); + expect(first.state).toBe('pending'); + expect(shownTag(adjusted!)).toBe(0); + expect(h.statuses).toEqual([['g', 'computing']]); + expect(h.jobs.map((job) => job.steps)).toEqual([[{ amount: 50, kind: 'clarity', size: 40 }]]); + expect(h.groups.byteSize()).toBe(2 * W * H * 4); + + h.jobs[0]!.settle('ok'); + await flush(); + expect(h.settled).toEqual(['g']); + expect(h.statuses.at(-1)).toEqual(['g', null]); + const paints = h.rawPaints(); + const again = h.show(scope('g', clarity(50))); + expect(again.state).toBe('exact'); + expect(again.surface).toBe(adjusted); + expect(shownTag(adjusted!)).toBe(1); + expect(h.rawPaints()).toBe(paints); + expect(h.jobs).toHaveLength(1); + }); + + it('keeps the raw composite and the shown pixels through a parameter edit, starting its job at once', async () => { + const h = setup(); + h.show(scope('g', clarity(50))); + h.jobs[0]!.settle('ok'); + await flush(); + const adjusted = h.surfaces[1]!; + const paints = h.rawPaints(); + const reads = sourceReads(adjusted).length; + + expect(h.show(scope('g', clarity(80))).state).toBe('pending'); + expect(h.rawPaints()).toBe(paints); + expect(h.surfaces).toHaveLength(2); + expect(sourceReads(adjusted)).toHaveLength(reads); + expect(shownTag(adjusted)).toBe(1); + expect(h.jobs).toHaveLength(2); + + h.jobs[1]!.settle('ok'); + await flush(); + expect(h.show(scope('g', clarity(80))).state).toBe('exact'); + expect(shownTag(adjusted)).toBe(2); + }); + + it('repaints damaged raw pixels, re-patches them per-pixel and re-runs the job after the content debounce', async () => { + vi.useFakeTimers(); + const h = setup(); + h.show(scope('g', clarity(50))); + h.jobs[0]!.settle('ok'); + await flush(); + const adjusted = h.surfaces[1]!; + const paints = h.rawPaints(); + + h.layers.publishPixels('a', { height: 1, width: 1, x: 3, y: 2 }); + expect(h.show(scope('g', clarity(50))).state).toBe('pending'); + expect(h.rawPaints()).toBe(paints + 1); + // The member's one-pixel damage reaches one source pixel further, plus one pixel of antialiasing. + expect(sourceReads(adjusted).at(-1)).toEqual([1, 0, 5, 5]); + expect(shownTag(adjusted)).toBe(1); + + await vi.advanceTimersByTimeAsync(SPATIAL_CONTENT_DEBOUNCE_MS - 1); + expect(h.jobs).toHaveLength(1); + await vi.advanceTimersByTimeAsync(1); + expect(h.jobs).toHaveLength(2); + h.jobs[1]!.settle('ok'); + await flush(); + expect(h.show(scope('g', clarity(50))).state).toBe('exact'); + expect(shownTag(adjusted)).toBe(2); + }); + + it('waits for members still computing, then starts at once and keeps its pixels while their results land', async () => { + const h = setup(); + h.memberStates.set('a', 'pending'); + expect(h.show(scope('g', clarity(50))).state).toBe('pending'); + expect(h.statuses).toEqual([['g', 'computing']]); + expect(h.jobs).toHaveLength(0); + const adjusted = h.surfaces[1]!; + const reads = sourceReads(adjusted).length; + + // The member's result landed: the engine redraws the group without any member version change. + h.memberStates.set('a', 'exact'); + h.groups.invalidate(['g']); + expect(h.show(scope('g', clarity(50))).state).toBe('pending'); + expect(sourceReads(adjusted)).toHaveLength(reads); + expect(h.jobs).toHaveLength(1); + + h.jobs[0]!.settle('ok'); + await flush(); + expect(h.show(scope('g', clarity(50))).state).toBe('exact'); + }); + + it('runs on a member that cannot finish, reporting the composite as inexact', async () => { + const h = setup(); + h.memberStates.set('a', 'inexact'); + h.show(scope('g', clarity(50))); + expect(h.jobs).toHaveLength(1); + h.jobs[0]!.settle('ok'); + await flush(); + expect(h.show(scope('g', clarity(50))).state).toBe('inexact'); + expect(h.statuses.at(-1)).toEqual(['g', null]); + }); + + it('converges nested spatial groups: the outer job waits for the inner result', async () => { + const h = setup({ inner: ['outer'] }); + const outer = scope('outer', clarity(30), [scope('inner', clarity(60))]); + + expect(h.show(outer).state).toBe('pending'); + expect(h.jobs.map((job) => job.steps[0]!.amount)).toEqual([60]); + expect(h.statuses).toContainEqual(['outer', 'computing']); + + h.jobs[0]!.settle('ok'); + await flush(); + expect(h.settled).toEqual(['inner']); + expect(h.show(outer).state).toBe('pending'); + expect(h.jobs.map((job) => job.steps[0]!.amount)).toEqual([60, 30]); + + h.jobs[1]!.settle('ok'); + await flush(); + expect(h.settled).toEqual(['inner', 'outer']); + expect(h.show(outer).state).toBe('exact'); + expect(h.jobs).toHaveLength(2); + expect(h.statuses.filter(([, status]) => status === null).map(([id]) => id)).toEqual(['inner', 'outer']); + }); + + it('evicts the adjusted surface with its slot and abandons the slot job', async () => { + const h = setup(); + h.show(scope('g', clarity(50))); + h.groups.evict(0, h.groups.tick()); + + expect(h.groups.byteSize()).toBe(0); + expect(h.statuses.at(-1)).toEqual(['g', null]); + h.jobs[0]!.settle('ok'); + await flush(); + expect(h.settled).toEqual([]); + }); + + it('clears the status and releases the adjusted surface once the group stops drawing a spatial stack', async () => { + const h = setup(); + h.show(scope('g', clarity(50))); + h.jobs[0]!.settle('too-large'); + await flush(); + expect(h.statuses.at(-1)).toEqual(['g', 'too-large']); + expect(h.show(scope('g', clarity(50))).state).toBe('inexact'); + expect(h.jobs).toHaveLength(1); + + // Edited to a per-pixel stack: the group reports the slot it now draws, which has no adjusted surface. + const perPixel = h.show(scope('g', [{ id: 'inv', isEnabled: true, type: 'invert' }])); + expect(perPixel.state).toBe('exact'); + expect(h.statuses.at(-1)).toEqual(['g', null]); + expect(h.groups.byteSize()).toBe(3 * W * H * 4); + + // Undone: the warm spatial slot still knows its target cannot run. + h.show(scope('g', clarity(50))); + expect(h.statuses.at(-1)).toEqual(['g', 'too-large']); + // No longer drawn at all (disabled, or its stack reset): only `retainSpatial` can tell. + h.groups.retainSpatial(() => false); + expect(h.statuses.at(-1)).toEqual(['g', null]); + expect(h.groups.byteSize()).toBe(W * H * 4); + }); + + it("reports the frame's slot, so another consumer's key never flips the group's status", async () => { + const h = setup(); + const preview = { + rect: { height: H, width: W, x: 0, y: 0 }, + surface: createTestStubRasterBackend().createSurface(W, H), + }; + const previewed: GroupSurfaceContent = { ...NO_CONTENT, previews: new Map([['a', preview]]) }; + const target = scope('g', clarity(50)); + const frame = () => h.groups.get(target, [leaf('a')], [identity()], previewed, { frame: true })!; + + // The frame shows a filter preview on `a`; the overview composites the committed pixels in another slot. + frame(); + h.show(target); + expect(h.statuses).toEqual([['g', 'computing']]); + h.jobs[0]!.settle('ok'); + await flush(); + expect(h.statuses.at(-1)).toEqual(['g', null]); + + h.show(target); + frame(); + h.show(target); + expect(h.show(target).state).toBe('pending'); + h.jobs[1]!.settle('ok'); + await flush(); + expect(h.statuses).toEqual([ + ['g', 'computing'], + ['g', null], + ]); + }); + + it('re-patches a result that lands after more raw repaints than the trail records apart', async () => { + vi.useFakeTimers(); + const h = setup(); + h.show(scope('g', clarity(50))); + const adjusted = h.surfaces[1]!; + for (let i = 0; i < 20; i++) { + h.layers.publishPixels('a', { height: 1, width: 1, x: 3 + (i % 2), y: 2 }); + h.show(scope('g', clarity(50))); + } + + h.jobs[0]!.settle('ok'); + await flush(); + expect(h.settled).toEqual(['g']); + expect(shownTag(adjusted)).toBe(1); + }); + + it('keeps an exact group through a cancel; one still computing shows a whole interim and starts again', async () => { + const h = setup(); + h.show(scope('g', clarity(50))); + h.jobs[0]!.settle('ok'); + await flush(); + h.groups.cancelSpatial(); + expect(h.show(scope('g', clarity(50))).state).toBe('exact'); + expect(h.jobs).toHaveLength(1); + + h.show(scope('g', clarity(60))); + h.groups.cancelSpatial(); + expect(h.statuses.at(-1)).toEqual(['g', null]); + const adjusted = h.surfaces[1]!; + expect(h.show(scope('g', clarity(60))).state).toBe('pending'); + expect(shownTag(adjusted)).toBe(0); + h.jobs[1]!.settle('ok'); + await flush(); + h.jobs[2]!.settle('ok'); + await flush(); + expect(h.show(scope('g', clarity(60))).state).toBe('exact'); + expect(shownTag(adjusted)).toBe(3); + }); + + describe('member looks under a spatial group', () => { + const brighten = (brightness: number): CanvasAdjustmentsContract => [ + { brightness, contrast: 0, id: 'bc', isEnabled: true, type: 'brightness-contrast' }, + ]; + /** `b`'s bounds plus one pixel of antialiasing, clamped to the group: what its edits repaint. */ + const B_REGION = [1, 0, 5, 4]; + + const settledPair = async (b: SemanticLeaf = leaf('b')) => { + const h = setup(); + h.showPair(clarity(50), b); + h.jobs[0]!.settle('ok'); + await flush(); + expect(h.showPair(clarity(50), b).state).toBe('exact'); + return { ...h, adjusted: h.surfaces[1]!, paints: h.rawPaints() }; + }; + + it('repaints only an edited member, keeps the adjusted pixels elsewhere and is exact only after a later job', async () => { + vi.useFakeTimers(); + const h = await settledPair(); + + expect(h.showPair(clarity(50), leaf('b', { adjustments: brighten(0.2) })).state).toBe('pending'); + expect(h.surfaces).toHaveLength(2); + expect(h.rawPaints()).toBe(h.paints + 1); + expect(sourceReads(h.adjusted).at(-1)).toEqual(B_REGION); + expect(shownTag(h.adjusted)).toBe(1); + expect(h.jobs).toHaveLength(1); + await vi.advanceTimersByTimeAsync(SPATIAL_CONTENT_DEBOUNCE_MS); + expect(h.jobs).toHaveLength(2); + + // Changed again while that job runs: its result lands as an interim, never as exact. + const faded = leaf('b', { adjustments: brighten(0.2), opacity: 0.5 }); + h.showPair(clarity(50), faded); + expect(sourceReads(h.adjusted).at(-1)).toEqual(B_REGION); + h.jobs[1]!.settle('ok'); + await flush(); + expect(h.showPair(clarity(50), faded).state).toBe('pending'); + expect(shownTag(h.adjusted)).toBe(2); + + await vi.advanceTimersByTimeAsync(SPATIAL_CONTENT_DEBOUNCE_MS); + h.jobs[2]!.settle('ok'); + await flush(); + expect(h.showPair(clarity(50), faded).state).toBe('exact'); + expect(h.surfaces).toHaveLength(2); + }); + + it('refreshes a member moved within the group bounds by its old and new bounds', async () => { + vi.useFakeTimers(); + const h = await settledPair(); + const nudged: Mat2d = { a: 1, b: 0, c: 0, d: 1, e: 1, f: 0 }; + + expect(h.showPair(clarity(50), leaf('b'), nudged).state).toBe('pending'); + expect(h.surfaces).toHaveLength(2); + expect(sourceReads(h.adjusted).at(-1)).toEqual([1, 0, 6, 4]); + expect(shownTag(h.adjusted)).toBe(1); + + await vi.advanceTimersByTimeAsync(SPATIAL_CONTENT_DEBOUNCE_MS); + h.jobs[1]!.settle('ok'); + await flush(); + expect(h.showPair(clarity(50), leaf('b'), nudged).state).toBe('exact'); + }); + + it("keeps the shown pixels through a spatial member's edit, rereading the member until its result lands", async () => { + const h = await settledPair(leaf('b', { adjustments: clarity(10) })); + const reads = sourceReads(h.adjusted).length; + + h.memberStates.set('b', 'pending'); + expect(h.showPair(clarity(50), leaf('b', { adjustments: clarity(20) })).state).toBe('pending'); + expect(h.memberStacks.get('b')).toEqual(clarity(20)); + expect(h.rawPaints()).toBe(h.paints + 1); + expect(sourceReads(h.adjusted)).toHaveLength(reads); + expect(shownTag(h.adjusted)).toBe(1); + expect(h.jobs).toHaveLength(1); + + h.memberStates.set('b', 'exact'); + h.groups.invalidate(['g']); + h.showPair(clarity(50), leaf('b', { adjustments: clarity(20) })); + expect(sourceReads(h.adjusted)).toHaveLength(reads); + expect(h.jobs).toHaveLength(2); + h.jobs[1]!.settle('ok'); + await flush(); + expect(h.showPair(clarity(50), leaf('b', { adjustments: clarity(20) })).state).toBe('exact'); + }); + + it("keeps the outer group's pixels through a nested spatial group's edit", async () => { + const h = setup({ inner: ['outer'] }); + const nested = (amount: number) => scope('outer', clarity(30), [scope('inner', clarity(amount))]); + h.show(nested(60)); + h.jobs[0]!.settle('ok'); + await flush(); + h.show(nested(60)); + h.jobs[1]!.settle('ok'); + await flush(); + expect(h.show(nested(60)).state).toBe('exact'); + const outerAdjusted = h.surfaces[3]!; + const reads = sourceReads(outerAdjusted).length; + + expect(h.show(nested(70)).state).toBe('pending'); + expect(h.surfaces).toHaveLength(4); + expect(sourceReads(outerAdjusted)).toHaveLength(reads); + expect(shownTag(outerAdjusted)).toBe(2); + expect(h.jobs.map((job) => job.steps[0]!.amount)).toEqual([60, 30, 70]); + + h.jobs[2]!.settle('ok'); + await flush(); + h.show(nested(70)); + h.jobs[3]!.settle('ok'); + await flush(); + expect(h.show(nested(70)).state).toBe('exact'); + expect(sourceReads(outerAdjusted)).toHaveLength(reads); + }); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupSurfaceCache.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupSurfaceCache.ts index 738039e8c0e..2d06021d4c5 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupSurfaceCache.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupSurfaceCache.ts @@ -2,7 +2,9 @@ * Document-space group composites keyed by scope shape, member appearance, effective matrices and in-scope * previews. Own opacity/blend apply at draw time. A stable slot refreshes in place: only the region its members * damaged since the last draw is cleared and redrawn, and geometry changes redraw without reallocating when the - * size holds. Document-resolution buffers resample transformed members twice and upscale above 100% zoom. + * size holds. Document-resolution buffers resample transformed members twice and upscale above 100% zoom. A scope + * whose own stack is spatial keeps its raw composite and shows a separate adjusted surface, evaluated + * asynchronously by the shared spatial display rules (`spatialDisplay.ts`). */ import type { CanvasRasterLayerContractV2 } from '@workbench/canvas-engine/contracts'; @@ -14,8 +16,22 @@ import type { Mat2d, Rect } from '@workbench/canvas-engine/types'; import { multiply } from '@workbench/canvas-engine/math/mat2d'; import { expand, intersect, isEmpty, roundOut, transformBounds, union } from '@workbench/canvas-engine/math/rect'; -import { adjustmentsKey, applyAdjustments, isIdentityAdjustments } from '@workbench/canvas-engine/render/adjustments'; +import { + adjustmentsKey, + applyAdjustments, + hasSpatialAdjustments, + isIdentityAdjustments, +} from '@workbench/canvas-engine/render/adjustments'; import { blendToComposite } from '@workbench/canvas-engine/render/compositor'; +import { + createSpatialDisplay, + type AdjustedSurfaceStatus, + type DisplayedPixelsState, + type SpatialDisplay, + type SpatialDisplayOptions, + type SpatialDisplayTarget, + worseDisplayedState, +} from '@workbench/canvas-engine/render/spatialDisplay'; import type { GroupCompositeScope, GroupSurfaceContent } from './groupCompositeScopes'; @@ -23,6 +39,14 @@ export interface GroupSurfaceResult { readonly surface: RasterSurface; /** Document-space placement of the surface. */ readonly rect: Rect; + /** How final the pixels are: the group's own spatial stack and every member's and nested group's included. */ + readonly state: DisplayedPixelsState; +} + +/** A member's displayed pixels: its own adjusted surface (null for raw) and how final they are. */ +export interface MemberDisplayedPixels { + readonly surface: RasterSurface | null; + readonly state: DisplayedPixelsState; } export interface GroupSurfaceDeps { @@ -32,22 +56,46 @@ export interface GroupSurfaceDeps { getCacheEntry(layerId: string): LayerCacheEntry | undefined; /** Surface-local damage a member's cache received since `version`, or null when unknown. */ damageSince(layerId: string, version: number): Rect | null; - /** The member's own adjusted pixels (its personal stack), or null for raw. */ - getAdjustedSurface(layer: CanvasRasterLayerContractV2, entry: LayerCacheEntry): RasterSurface | null; + /** The member's own displayed pixels (its personal stack applied). */ + getAdjustedSurface(layer: CanvasRasterLayerContractV2, entry: LayerCacheEntry): MemberDisplayedPixels; diagnostics?: CanvasDiagnostics; + /** + * Evaluates group stacks with spatial entries, reporting by group id. Without it those scopes show the per-pixel + * part only. + */ + spatial?: SpatialDisplayOptions; } export interface GroupSurfaceCache { /** Total RGBA bytes the cached group surfaces hold, for the memory budget. */ byteSize(): number; + /** + * `frame` marks the canvas frame's reads: a group reports, under its id in `adjustmentStatus`, the status of the + * slot the frame read last (else of its most recently read slot), so reads by other consumers with other keys + * (the overview, picking) never flip it. + */ get( scope: GroupCompositeScope, members: readonly SemanticLeaf[], memberMatrices: readonly Mat2d[], - content: GroupSurfaceContent + content: GroupSurfaceContent, + options?: { readonly frame?: boolean } ): GroupSurfaceResult | null; /** Drops every cached group not named; call when document structure changes. */ prune(liveGroupIds: ReadonlySet): void; + /** + * Makes the named groups redraw in full on their next read, keeping their surfaces: a member's displayed pixels + * changed without a cache version (a spatial adjustment landed). A spatial group's adjusted surface keeps its + * pixels outside member content damage until its own job lands. + */ + invalidate(groupIds: Iterable): void; + /** Drops the spatial slots of every group `keep` rejects, e.g. a group whose stack lost its spatial entries. */ + retainSpatial(keep: (groupId: string) => boolean): void; + /** + * Abandons running and scheduled group jobs; those slots show a whole per-pixel interim on their next read and start + * again. Exact and failed slots keep their state. + */ + cancelSpatial(): void; /** The access clock; a slot read at or after a captured tick is in use by that frame. */ tick(): number; /** Evicts slots least-recently-used first until within budget, keeping slots read after `sinceTick`. */ @@ -70,7 +118,41 @@ const ANTIALIAS_MARGIN = 1; // slots stop those consumers evicting each other every tick. const SLOTS_PER_GROUP = 2; +/** Raw repaints a spatial slot records apart; older ones fold into the oldest entry, coarser but still known. */ +const RAW_DAMAGE_TRAIL_LIMIT = 16; + +const NO_AREA: Rect = { height: 0, width: 0, x: 0, y: 0 }; + +/** A spatial scope's raw composite history and the adjusted surface the display shows. */ +interface SpatialGroupState { + /** Bumped by every raw repaint. */ + rawVersion: number; + /** Surface-local damage of the raw repaints from `first` to `version`, `null` when unknown; oldest first. */ + readonly trail: { readonly first: number; readonly version: number; readonly damage: Rect | null }[]; + readonly adjusted: RasterSurface; + /** Raw version and stack key `adjusted` was last brought to; null forces a whole per-pixel interim. */ + adjustedFor: { readonly version: number; readonly key: string } | null; +} + +/** + * How a spatial scope's member or nested group draws into the raw composite. A change repaints only its bounds, + * instead of moving the scope to another slot whose adjusted copy would restart from a whole per-pixel interim. + */ +interface DrawnAppearance { + /** Opacity and blend, plus the matrix for a member. */ + readonly placement: string; + /** `adjustmentsKey` of its own stack. */ + readonly stack: string; + /** A spatial stack's display keeps its pixels through a parameter edit until its result lands (`invalidate`). */ + readonly spatial: boolean; + /** Document-space bounds it draws into. */ + readonly bounds: Rect | null; +} + interface GroupSlot { + readonly groupId: string; + /** This slot's spatial display owner id; each slot runs its own jobs. */ + readonly owner: string; shapeKey: string; rect: Rect; readonly surface: RasterSurface; @@ -79,6 +161,15 @@ interface GroupSlot { /** Placement of the floating selection the surface shows, so a drag refreshes in place. */ float: DrawnFloat | null; lastUsed: number; + /** Member pixels changed in a way damage tracking cannot see; the next read redraws everything. */ + stale: boolean; + /** How final the members and nested groups drawn at the last repaint were. */ + inputs: DisplayedPixelsState; + /** Present while the slot holds a spatial scope's raw composite. */ + spatial: SpatialGroupState | null; + /** A spatial scope's shape without its members' looks, and those looks; null for other scopes. */ + structureKey: string | null; + appearances: ReadonlyMap | null; } interface DrawnFloat { @@ -87,6 +178,14 @@ interface DrawnFloat { readonly landing: Rect; } +/** What changed in a slot's members since its last draw (document space). */ +interface Damage { + /** Pixels that changed. */ + readonly region: Rect | null; + /** Looks to redraw whose pixels did not change. */ + readonly reread: Rect | null; +} + interface DrawnEntry { readonly leaf: SemanticLeaf; readonly matrix: Mat2d; @@ -95,12 +194,41 @@ interface DrawnEntry { readonly float: DrawnFloat | null; } +const unionOf = (a: Rect | null, b: Rect | null): Rect | null => (a && b ? union(a, b) : (a ?? b)); + const surfaceBytes = (surface: RasterSurface): number => surface.width * surface.height * 4; +const slotBytes = (slot: GroupSlot): number => + surfaceBytes(slot.surface) + (slot.spatial ? surfaceBytes(slot.spatial.adjusted) : 0); + +/** Union of the raw damage since `version`, or null when unknown or older than the trail. */ +const rawDamageSince = (spatial: SpatialGroupState, version: number): Rect | null => { + const { trail } = spatial; + if (trail.length === 0 || trail[0]!.first > version + 1) { + return null; + } + let damaged = NO_AREA; + for (const step of trail) { + if (step.version <= version) { + continue; + } + if (!step.damage) { + return null; + } + damaged = union(damaged, step.damage); + } + return damaged; +}; export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCache => { const cache = new Map(); const objectIds = new WeakMap(); + const slotsByOwner = new Map(); + /** The slot each group's latest frame read used; its status is the one the group reports. */ + const frameSlots = new Map(); + /** Last status reported per group. */ + const reported = new Map(); let nextObjectId = 1; + let nextSlot = 1; let clock = 0; let totalBytes = 0; @@ -121,20 +249,86 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac deps.onBytesChange?.(totalBytes); }; + const reportStatus = (groupId: string): void => { + const reporting = frameSlots.get(groupId) ?? cache.get(groupId)?.[0]; + const status = reporting?.spatial ? (display?.status(reporting.owner) ?? null) : null; + if ((reported.get(groupId) ?? null) === status) { + return; + } + if (status) { + reported.set(groupId, status); + } else { + reported.delete(groupId); + } + deps.spatial?.onStatus?.(groupId, status); + }; + + const display = deps.spatial + ? createSpatialDisplay( + { + onSettle: (owner) => { + const slot = slotsByOwner.get(owner); + if (slot) { + deps.spatial?.onSettle?.(slot.groupId); + } + }, + onStatus: (owner) => { + const slot = slotsByOwner.get(owner); + if (slot) { + reportStatus(slot.groupId); + } + }, + run: deps.spatial.run, + }, + 'group:' + ) + : null; + + /** Releases a spatial slot's adjusted surface and abandons its work, keeping the raw composite. */ + const releaseSpatial = (slot: GroupSlot): void => { + if (slot.spatial) { + adjustBytes(-surfaceBytes(slot.spatial.adjusted)); + slot.spatial = null; + display?.drop(slot.owner); + } + }; + + /** Forgets slots already removed from their group's list, then reports each group's status. */ const dropSlots = (slots: readonly GroupSlot[]): void => { - adjustBytes(-slots.reduce((bytes, slot) => bytes + surfaceBytes(slot.surface), 0)); + adjustBytes(-slots.reduce((bytes, slot) => bytes + slotBytes(slot), 0)); + for (const slot of slots) { + slotsByOwner.delete(slot.owner); + if (frameSlots.get(slot.groupId) === slot) { + frameSlots.delete(slot.groupId); + } + display?.drop(slot.owner); + } + for (const groupId of new Set(slots.map((slot) => slot.groupId))) { + reportStatus(groupId); + } }; // A scope's OWN opacity/blend are applied by the consumer when the surface // lands, so they stay out of its shape key (an opacity scrub reuses the // surface); a CHILD scope's opacity/blend are baked in here, so children key // with them. + // + // A spatial scope's own stack applies to a separate adjusted surface, so its own key stays out too and a + // parameter edit reuses the raw composite; a parent still keys a spatial child with it. const childScopeKey = (scope: GroupCompositeScope): string => - `${scopeShapeKey(scope)}:${scope.opacity}:${scope.blendMode}`; - const scopeShapeKey = (scope: GroupCompositeScope): string => - `${scope.id}@${scope.start}-${scope.end}:${adjustmentsKey(scope.adjustments)}(${scope.children - .map(childScopeKey) - .join(',')})`; + `${scopeShapeKey(scope, adjustmentsKey(scope.adjustments))}:${scope.opacity}:${scope.blendMode}`; + const scopeShapeKey = (scope: GroupCompositeScope, ownKey: string): string => + `${scope.id}@${scope.start}-${scope.end}:${ownKey}(${scope.children.map(childScopeKey).join(',')})`; + const scopeStructureKey = (scope: GroupCompositeScope): string => + `${scope.id}@${scope.start}-${scope.end}(${scope.children.map(scopeStructureKey).join(',')})`; + + /** Identity of what replaces or rides on a member's committed pixels: its filter preview and its float. */ + const contentKeyOf = (leafId: string, content: GroupSurfaceContent): string => { + const preview = content.previews?.get(leafId); + const float = content.float?.layerId === leafId ? content.float : null; + const previewKey = preview ? `:p${objectId(preview.surface)}@${rectKey(preview.rect)}` : ''; + return float ? `${previewKey}:f${objectId(float.surface)}` : previewKey; + }; /** Everything a surface depends on except member pixels and float placement, which are tracked as damage. */ const shapeKeyOf = ( @@ -142,7 +336,8 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac members: readonly SemanticLeaf[], memberMatrices: readonly Mat2d[], content: GroupSurfaceContent, - baseIndex: number + baseIndex: number, + spatial: boolean ): string => { const memberKeys: string[] = []; for (let i = scope.start; i < scope.end; i += 1) { @@ -153,14 +348,67 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac } const { layer } = leaf; const own = layer.type === 'raster' ? adjustmentsKey(layer.adjustments) : '-'; - const preview = content.previews?.get(leaf.id); - const float = content.float?.layerId === leaf.id ? content.float : null; - const previewKey = preview ? `:p${objectId(preview.surface)}@${rectKey(preview.rect)}` : ''; - const floatKey = float ? `:f${objectId(float.surface)}` : ''; const appearance = `${layer.opacity}:${layer.blendMode}:${matKey(memberMatrices[i - baseIndex]!)}:${own}`; - memberKeys.push(`${leaf.id}:${appearance}${previewKey}${floatKey}`); + memberKeys.push(`${leaf.id}:${appearance}${contentKeyOf(leaf.id, content)}`); + } + return `${scopeShapeKey(scope, spatial ? '~spatial' : adjustmentsKey(scope.adjustments))}|${memberKeys.join('|')}`; + }; + + /** A spatial scope's shape key without its members' and nested groups' looks, which {@link appearancesOf} tracks. */ + const structureKeyOf = ( + scope: GroupCompositeScope, + members: readonly SemanticLeaf[], + content: GroupSurfaceContent, + baseIndex: number + ): string => { + const memberKeys: string[] = []; + for (let i = scope.start; i < scope.end; i += 1) { + const { id } = members[i - baseIndex]!; + memberKeys.push(content.excludeIds.has(id) ? `${id}:x` : `${id}${contentKeyOf(id, content)}`); + } + return `~spatial:${scopeStructureKey(scope)}|${memberKeys.join('|')}`; + }; + + /** The looks of a spatial scope's drawn members and nested groups, keyed by member id or `scope:`. */ + const appearancesOf = ( + scope: GroupCompositeScope, + members: readonly SemanticLeaf[], + baseIndex: number, + drawn: readonly DrawnEntry[] + ): ReadonlyMap => { + const appearances = new Map(); + const boundsById = new Map(); + for (const { bounds, leaf, matrix } of drawn) { + const { layer } = leaf; + const adjustments = layer.type === 'raster' ? layer.adjustments : undefined; + boundsById.set(leaf.id, bounds); + appearances.set(leaf.id, { + bounds, + placement: `${layer.opacity}:${layer.blendMode}:${matKey(matrix)}`, + spatial: hasSpatialAdjustments(adjustments), + stack: adjustmentsKey(adjustments), + }); } - return `${scopeShapeKey(scope)}|${memberKeys.join('|')}`; + const addScopes = (scopes: readonly GroupCompositeScope[]): void => { + for (const child of scopes) { + let bounds: Rect | null = null; + for (let i = child.start; i < child.end; i += 1) { + const leafBounds = boundsById.get(members[i - baseIndex]!.id); + if (leafBounds) { + bounds = bounds ? union(bounds, leafBounds) : leafBounds; + } + } + appearances.set(`scope:${child.id}`, { + bounds, + placement: `${child.opacity}:${child.blendMode}`, + spatial: hasSpatialAdjustments(child.adjustments), + stack: adjustmentsKey(child.adjustments), + }); + addScopes(child.children); + } + }; + addScopes(scope.children); + return appearances; }; /** Members whose committed cache pixels the surface shows, with their document-space bounds. */ @@ -198,7 +446,10 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac return drawn; }; - /** Draws `[from, to)` (absolute plan indices, bottom first); `baseIndex` = absolute index of `members[0]`. */ + /** + * Draws `[from, to)` (absolute plan indices, bottom first); `baseIndex` = absolute index of `members[0]`. Returns + * how final the drawn members and nested groups are. + */ const drawRange = ( ctx: RasterSurface['ctx'], view: Mat2d, @@ -208,14 +459,17 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac baseIndex: number, from: number, to: number, - children: readonly GroupCompositeScope[] - ): void => { + children: readonly GroupCompositeScope[], + frame: boolean + ): DisplayedPixelsState => { + let inputs: DisplayedPixelsState = 'exact'; let childIndex = 0; for (let i = from; i < to;) { const child = childIndex < children.length ? children[childIndex]! : null; if (child && i >= child.start && i < child.end) { - const nested = resolve(child, members, memberMatrices, content, baseIndex); + const nested = resolve(child, members, memberMatrices, content, baseIndex, frame); if (nested) { + inputs = worseDisplayedState(inputs, nested.state); ctx.save(); ctx.globalAlpha = child.opacity; ctx.globalCompositeOperation = blendToComposite(child.blendMode); @@ -243,8 +497,9 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac if (preview) { ctx.drawImage(preview.surface.canvas, preview.rect.x, preview.rect.y); } else if (entry && !isEmpty(entry.rect)) { - const adjusted = deps.getAdjustedSurface(leaf.layer, entry); - ctx.drawImage((adjusted ?? entry.surface).canvas, entry.rect.x, entry.rect.y); + const displayed = deps.getAdjustedSurface(leaf.layer, entry); + inputs = worseDisplayedState(inputs, displayed.state); + ctx.drawImage((displayed.surface ?? entry.surface).canvas, entry.rect.x, entry.rect.y); } const float = content.float?.layerId === leaf.id ? content.float : null; if (float) { @@ -254,9 +509,13 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac } ctx.restore(); } + return inputs; }; - /** Clears and redraws `region` (surface-local) of a slot, then applies the scope's own adjustments there. */ + /** + * Clears and redraws `region` (surface-local) of a slot, then applies the scope's own adjustments there unless they + * are spatial (a spatial scope's surface stays raw). Records how final the drawn inputs were. + */ const paint = ( slot: GroupSlot, region: Rect, @@ -264,7 +523,9 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac members: readonly SemanticLeaf[], memberMatrices: readonly Mat2d[], content: GroupSurfaceContent, - baseIndex: number + baseIndex: number, + spatial: boolean, + frame: boolean ): void => { const ctx = slot.surface.ctx; ctx.save(); @@ -274,23 +535,67 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac ctx.clip(); ctx.clearRect(region.x, region.y, region.width, region.height); const view: Mat2d = { a: 1, b: 0, c: 0, d: 1, e: -slot.rect.x, f: -slot.rect.y }; - drawRange(ctx, view, members, memberMatrices, content, baseIndex, scope.start, scope.end, scope.children); + slot.inputs = drawRange( + ctx, + view, + members, + memberMatrices, + content, + baseIndex, + scope.start, + scope.end, + scope.children, + frame + ); ctx.restore(); - // Adjustments are region independent and group stacks work in document pixels, so a region-limited pass at - // the region's document origin equals a whole-surface one there. - if (!isIdentityAdjustments(scope.adjustments)) { + // Per-pixel adjustments are region independent and group stacks work in document pixels, so a region-limited + // pass at the region's document origin equals a whole-surface one there. + if (!spatial && !isIdentityAdjustments(scope.adjustments)) { const pixels = ctx.getImageData(region.x, region.y, region.width, region.height); applyAdjustments(pixels, scope.adjustments, { x: slot.rect.x + region.x, y: slot.rect.y + region.y }); ctx.putImageData(pixels, region.x, region.y); } }; - /** Document-space region the members changed since the slot was drawn; null when only a full redraw is safe. */ - const damagedRegion = (slot: GroupSlot, drawn: readonly DrawnEntry[]): { region: Rect | null } | null => { + /** + * Document-space region the members changed since the slot was drawn; null when only a full redraw is safe. With + * `appearances` (a spatial scope), changed looks add their old and new bounds. A spatial stack edited into another + * spatial stack keeps its shown pixels until its result lands, so its bounds go to `reread` instead: redrawn so + * the edit reaches its owner and its pending state is seen, but not damage for the adjusted copy. + */ + const damagedRegion = ( + slot: GroupSlot, + drawn: readonly DrawnEntry[], + appearances: ReadonlyMap | null + ): Damage | null => { if (drawn.filter((item) => item.entry).length !== slot.versions.size) { return null; } let region: Rect | null = null; + let reread: Rect | null = null; + if (appearances) { + const before = slot.appearances; + if (!before || before.size !== appearances.size) { + return null; + } + for (const [id, next] of appearances) { + const previous = before.get(id); + if (!previous) { + return null; + } + if (previous.placement === next.placement && previous.stack === next.stack) { + continue; + } + const kept = previous.placement === next.placement && previous.spatial && next.spatial; + for (const bounds of [previous.bounds, next.bounds]) { + if (bounds && kept) { + reread = reread ? union(reread, bounds) : bounds; + } else if (bounds) { + region = region ? union(region, bounds) : bounds; + } + } + } + } const float = drawn.find((item) => item.float)?.float ?? null; if (float?.placement !== slot.float?.placement) { for (const landing of [slot.float?.landing, float?.landing]) { @@ -323,18 +628,88 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac ); region = region ? union(region, changed) : changed; } - return { region }; + return { region, reread }; }; const versionsOf = (drawn: readonly DrawnEntry[]): ReadonlyMap => new Map(drawn.flatMap(({ entry, leaf }) => (entry ? [[leaf.id, entry.version] as const] : []))); + /** + * Shows a spatial scope's adjusted surface for the slot's raw composite: an interim now (`painted` is the raw + * damage this read drew, `undefined` when none), the evaluated stack when its job lands. Jobs wait while an input + * is still pending. + */ + const showSpatial = ( + display: SpatialDisplay, + slot: GroupSlot, + scope: GroupCompositeScope, + painted: Rect | null | undefined, + published: boolean + ): { readonly surface: RasterSurface; readonly own: DisplayedPixelsState } => { + const raw = slot.surface; + let spatial = slot.spatial; + if (!spatial) { + spatial = { adjusted: deps.createSurface(raw.width, raw.height), adjustedFor: null, rawVersion: 0, trail: [] }; + adjustBytes(surfaceBytes(spatial.adjusted)); + slot.spatial = spatial; + } + const state = spatial; + if (painted !== undefined) { + state.rawVersion += 1; + state.trail.push({ damage: painted, first: state.rawVersion, version: state.rawVersion }); + if (state.trail.length > RAW_DAMAGE_TRAIL_LIMIT) { + // Fold the oldest repaint into the next, so an adjusted copy from a long stroke ago still re-patches. An + // unknown one already leaves every earlier version unknown: dropping it loses nothing. + const oldest = state.trail.shift()!; + const next = state.trail[0]!; + if (oldest.damage) { + state.trail[0] = { + damage: next.damage && union(oldest.damage, next.damage), + first: oldest.first, + version: next.version, + }; + } + } + } + let reusable = state.adjustedFor !== null; + if (state.adjusted.width !== raw.width || state.adjusted.height !== raw.height) { + const before = surfaceBytes(state.adjusted); + state.adjusted.resize(raw.width, raw.height); + adjustBytes(surfaceBytes(state.adjusted) - before); + reusable = false; + } + const key = adjustmentsKey(scope.adjustments); + const version = state.rawVersion; + const live = (): boolean => slotsByOwner.get(slot.owner) === slot && slot.spatial === state; + const target: SpatialDisplayTarget = { + adjustments: scope.adjustments, + damageSince: (from) => rawDamageSince(state, from), + hold: slot.inputs === 'pending' ? 'inputs' : published ? null : 'placeholder', + key, + liveVersion: () => (live() ? state.rawVersion : null), + // Group stacks work in document pixels; the surface's (0,0) sits at the slot rect's origin. + origin: { x: slot.rect.x, y: slot.rect.y }, + shown: () => + live() && state.adjustedFor?.version === version && state.adjustedFor.key === key ? state.adjusted : null, + source: raw, + version, + }; + let rebuilt = false; + if (state.adjustedFor?.version !== version || state.adjustedFor.key !== key) { + rebuilt = display.paintInterim(slot.owner, state.adjusted, reusable, target); + state.adjustedFor = { key, version }; + } + display.request(slot.owner, target, rebuilt); + return { own: display.state(slot.owner, target), surface: state.adjusted }; + }; + function resolve( scope: GroupCompositeScope, members: readonly SemanticLeaf[], memberMatrices: readonly Mat2d[], content: GroupSurfaceContent, - baseIndex: number + baseIndex: number, + frame: boolean ): GroupSurfaceResult | null { clock += 1; const drawn = drawnEntries(scope, members, memberMatrices, content, baseIndex); @@ -352,27 +727,59 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac if (rect.width <= 0 || rect.height <= 0) { return null; } - const shapeKey = shapeKeyOf(scope, members, memberMatrices, content, baseIndex); + const spatial = display !== null && hasSpatialAdjustments(scope.adjustments); + const shapeKey = shapeKeyOf(scope, members, memberMatrices, content, baseIndex, spatial); + const structureKey = spatial ? structureKeyOf(scope, members, content, baseIndex) : null; + const appearances = spatial ? appearancesOf(scope, members, baseIndex, drawn) : null; const slots = cache.get(scope.id) ?? []; - const matching = slots.find((candidate) => candidate.shapeKey === shapeKey) ?? null; + let matching = slots.find((candidate) => candidate.shapeKey === shapeKey) ?? null; + let damage: Damage | null | undefined; + if (structureKey !== null && !(matching && sameRect(matching.rect, rect))) { + // Only looks changed (a member's stack, opacity, blend or matrix, a nested group's stack, opacity or blend): + // refresh the same-shaped slot needing the least repaint in place, so its adjusted copy keeps its pixels + // outside the changed bounds. Frame and overview keys still settle into one slot each. + let bestArea = Infinity; + for (const candidate of slots) { + if (candidate.structureKey !== structureKey || !sameRect(candidate.rect, rect)) { + continue; + } + const candidateDamage = damagedRegion(candidate, drawn, appearances); + const redrawn = candidateDamage && unionOf(candidateDamage.region, candidateDamage.reread); + const area = !candidateDamage ? Infinity : redrawn ? redrawn.width * redrawn.height : 0; + if (damage === undefined || area < bestArea) { + matching = candidate; + damage = candidateDamage; + bestArea = area; + } + } + if (damage !== undefined) { + matching!.shapeKey = shapeKey; + } + } const whole: Rect = { height: rect.height, width: rect.width, x: 0, y: 0 }; + const toLocal = (region: Rect | null): Rect | null => + region && + intersect(roundOut(expand({ ...region, x: region.x - rect.x, y: region.y - rect.y }, ANTIALIAS_MARGIN)), whole); let slot: GroupSlot; + /** Raw damage this read drew: a surface-local rect, null when unknown, undefined when nothing was drawn. */ + let painted: Rect | null | undefined; if (matching && sameRect(matching.rect, rect)) { slot = matching; - const damage = damagedRegion(slot, drawn); - const region = damage - ? damage.region && - intersect( - roundOut( - expand({ ...damage.region, x: damage.region.x - rect.x, y: damage.region.y - rect.y }, ANTIALIAS_MARGIN) - ), - whole - ) - : whole; - if (region && !isEmpty(region)) { - paint(slot, region, scope, members, memberMatrices, content, baseIndex); - deps.diagnostics?.increment(damage ? 'groupSurfaceRefreshes' : 'groupSurfaceRebuilds'); + damage = damage === undefined ? damagedRegion(slot, drawn, appearances) : damage; + if (slot.stale) { + // A member's displayed pixels changed without a version: redraw everything, but report only the content + // damage, so a spatial scope keeps its adjusted pixels elsewhere until its own job lands. + paint(slot, whole, scope, members, memberMatrices, content, baseIndex, spatial, frame); + painted = damage ? (toLocal(damage.region) ?? NO_AREA) : null; + deps.diagnostics?.increment('groupSurfaceRebuilds'); + } else { + const region = damage ? toLocal(unionOf(damage.region, damage.reread)) : whole; + if (region && !isEmpty(region)) { + paint(slot, region, scope, members, memberMatrices, content, baseIndex, spatial, frame); + painted = damage ? (toLocal(damage.region) ?? NO_AREA) : null; + deps.diagnostics?.increment(damage ? 'groupSurfaceRefreshes' : 'groupSurfaceRebuilds'); + } } } else if (matching || slots.length >= SLOTS_PER_GROUP) { // New bounds or content reuse a slot's backing store, reallocating only when the size changed. @@ -385,41 +792,92 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac } slot.rect = rect; slot.shapeKey = shapeKey; - paint(slot, whole, scope, members, memberMatrices, content, baseIndex); + paint(slot, whole, scope, members, memberMatrices, content, baseIndex, spatial, frame); + painted = null; deps.diagnostics?.increment('groupSurfaceRebuilds'); } else { slot = { + float: null, + groupId: scope.id, + inputs: 'exact', lastUsed: clock, + owner: `${scope.id}#${nextSlot++}`, rect, shapeKey, - float: null, + appearances: null, + spatial: null, + stale: false, + structureKey: null, surface: deps.createSurface(rect.width, rect.height), versions: new Map(), }; adjustBytes(surfaceBytes(slot.surface)); + slotsByOwner.set(slot.owner, slot); deps.diagnostics?.increment('groupSurfaceAllocations'); - paint(slot, whole, scope, members, memberMatrices, content, baseIndex); + paint(slot, whole, scope, members, memberMatrices, content, baseIndex, spatial, frame); + painted = null; slots.unshift(slot); cache.set(scope.id, slots); } slot.versions = versionsOf(drawn); + slot.stale = false; + slot.structureKey = structureKey; + slot.appearances = appearances; slot.float = drawn.find((item) => item.float)?.float ?? null; slot.lastUsed = clock; + if (frame) { + frameSlots.set(scope.id, slot); + } const index = slots.indexOf(slot); if (index > 0) { slots.splice(index, 1); slots.unshift(slot); } - return { rect: slot.rect, surface: slot.surface }; + let result: GroupSurfaceResult; + if (spatial) { + // A placeholder composite (no member has real pixels yet) is never worth a job. + const published = drawn.some( + (item) => item.float !== null || (item.entry ? item.entry.hasPublishedPixels : item.bounds !== null) + ); + const shown = showSpatial(display!, slot, scope, painted, published); + result = { rect: slot.rect, state: worseDisplayedState(slot.inputs, shown.own), surface: shown.surface }; + } else { + releaseSpatial(slot); + result = { rect: slot.rect, state: slot.inputs, surface: slot.surface }; + } + reportStatus(scope.id); + return result; } + /** Removes `slots` from their group's list (and the group once empty), then forgets them. */ + const removeSlots = (groupId: string, removed: readonly GroupSlot[]): void => { + const slots = cache.get(groupId); + if (!slots) { + return; + } + const kept = slots.filter((slot) => !removed.includes(slot)); + if (kept.length === 0) { + cache.delete(groupId); + } else { + cache.set(groupId, kept); + } + dropSlots(removed); + }; + return { byteSize: () => totalBytes, - clear: () => { - for (const slots of cache.values()) { - dropSlots(slots); + cancelSpatial: () => { + for (const owner of display?.cancel() ?? []) { + const spatial = slotsByOwner.get(owner)?.spatial; + if (spatial) { + spatial.adjustedFor = null; + } } + }, + clear: () => { + const all = [...cache.values()].flat(); cache.clear(); + dropSlots(all); }, evict: (budgetBytes, sinceTick) => { if (totalBytes <= budgetBytes) { @@ -434,22 +892,32 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac if (totalBytes <= budgetBytes) { break; } - const slots = cache.get(groupId)!; - slots.splice(slots.indexOf(slot), 1); - if (slots.length === 0) { - cache.delete(groupId); - } - dropSlots([slot]); + removeSlots(groupId, [slot]); evicted += 1; } return evicted; }, - get: (scope, members, memberMatrices, content) => resolve(scope, members, memberMatrices, content, scope.start), + get: (scope, members, memberMatrices, content, options) => + resolve(scope, members, memberMatrices, content, scope.start, options?.frame ?? false), + invalidate: (groupIds) => { + for (const groupId of groupIds) { + for (const slot of cache.get(groupId) ?? []) { + slot.stale = true; + } + } + }, prune: (liveGroupIds) => { for (const [id, slots] of cache) { if (!liveGroupIds.has(id)) { - dropSlots(slots); - cache.delete(id); + removeSlots(id, slots); + } + } + }, + retainSpatial: (keep) => { + for (const [id, slots] of cache) { + const spatialSlots = slots.filter((slot) => slot.spatial); + if (spatialSlots.length > 0 && !keep(id)) { + removeSlots(id, spatialSlots); } } }, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.test.ts index 2a2668ccc8c..5b6052eeb69 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.test.ts @@ -108,6 +108,11 @@ describe('renderLayerDisplayEffect', () => { expect(renderLayerDisplayEffect(backend, controlLayer(true), backend.createSurface(0, 0), ORIGIN)).toBeNull(); }); + it("previews only a raster stack's per-pixel part: a spatial-only stack needs no copy", () => { + const spatialOnly = rasterLayer([{ amount: 60, id: 'c', isEnabled: true, size: 40, type: 'clarity' }]); + expect(renderLayerDisplayEffect(backend, spatialOnly, backend.createSurface(4, 4), ORIGIN)).toBeNull(); + }); + it('returns a copy for a raster layer with adjustments', () => { const out = renderLayerDisplayEffect( backend, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.ts index 89e879c70da..ec19dfcf73a 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.ts @@ -1,7 +1,9 @@ /** - * Floating selections need the same raster adjustments/control transparency as their layer. These effects are + * Floating selections need the same raster adjustments/control transparency as their layer. Per-pixel effects are * region independent, so a display copy baked once at lift, at the cut's layer-local origin, matches the pixels it - * left (noise included). Transform changes do not affect it; appearance changes cancel the float. + * left (noise included). Spatial adjustments are not: they depend on the whole layer the float will land in, so the + * float previews the per-pixel part only and committing re-evaluates the layer. Transform changes do not affect it; + * appearance changes cancel the float. */ import type { CanvasLayerContract } from '@workbench/canvas-engine/contracts'; @@ -10,6 +12,7 @@ import { type AdjustmentOrigin, applyAdjustments, isIdentityAdjustments, + perPixelAdjustments, } from '@workbench/canvas-engine/render/adjustments'; import { applyLightnessToAlpha } from '@workbench/canvas-engine/render/controlTransparency'; @@ -33,7 +36,9 @@ export const renderLayerDisplayEffect = ( source: RasterSurface, origin: AdjustmentOrigin ): RasterSurface | null => { - if (!hasLayerDisplayEffect(layer) || source.width <= 0 || source.height <= 0) { + const adjustments = layer.type === 'raster' ? perPixelAdjustments(layer.adjustments) : undefined; + const hasEffect = layer.type === 'raster' ? !isIdentityAdjustments(adjustments) : hasLayerDisplayEffect(layer); + if (!hasEffect || source.width <= 0 || source.height <= 0) { return null; } const out = backend.createSurface(source.width, source.height); @@ -46,7 +51,7 @@ export const renderLayerDisplayEffect = ( const imageData = ctx.getImageData(0, 0, source.width, source.height); if (layer.type === 'raster') { - applyAdjustments(imageData, layer.adjustments, origin); + applyAdjustments(imageData, adjustments, origin); } else { applyLightnessToAlpha(imageData.data); } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/rasterComposite.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/rasterComposite.ts index c1c812b57a9..975f07c3cfa 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/rasterComposite.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/rasterComposite.ts @@ -18,10 +18,17 @@ import { expand, intersect, roundOut, transformBounds, union } from '@workbench/ import { adjustmentsKey, applyAdjustments, + hasSpatialAdjustments, isIdentityAdjustments, isPositionDependentAdjustments, } from '@workbench/canvas-engine/render/adjustments'; import { blendToComposite } from '@workbench/canvas-engine/render/compositor'; +import { + adjustExactly, + AdjustmentRefusedError, + imageDataOf, + type SpatialRunner, +} from '@workbench/canvas-engine/render/exactAdjustments'; import { collectCompositedGroups, planGroupCompositeScopes, @@ -62,8 +69,8 @@ export interface RenderRasterCompositeDeps { /** Owned access to a layer's pixels, released once they are drawn. */ getLayerSurface(layerId: string): Promise<{ surface: RasterSurface; rect: Rect; release(): void }>; /** - * The display's memoized layer-local adjusted copy of `surface`, or null on a miss or when `surface` is not a live - * cache. Never builds one: {@link compositeScratchPixels} covers only the sampled region a miss adjusts. + * The display's memoized layer-local adjusted copy of `surface` when it is exact, or null on a miss or when + * `surface` is not a live cache. Never builds one: {@link compositeScratchPixels} covers only what a miss adjusts. */ adjustedSurface?( layerId: string, @@ -72,6 +79,13 @@ export interface RenderRasterCompositeDeps { ): RasterSurface | null; readImageData?(surface: RasterSurface, rect: Rect): ImageData; writeImageData?(surface: RasterSurface, imageData: ImageData, x: number, y: number): void; + /** + * Evaluates spatial adjustment runs. Without it, a layer or group whose stack has a contributing spatial entry + * throws {@link AdjustmentRefusedError} rather than render without it. + */ + runSpatial?: SpatialRunner; + /** Cancels spatial jobs; an aborted render throws an `AbortError`. */ + signal?: AbortSignal; /** * Reserves transient raster bytes. The caller reserves {@link compositeScratchPixels} up front; rendering reserves * here only what a layer's actual pixels need beyond that estimate, and refuses when it does not fit. @@ -218,17 +232,44 @@ export const getCompositeLayerBounds = (layers: readonly CompositeLayerRef[]): R const scopeKey = (scope: GroupCompositeScope): string => `${scope.id}@${scope.start}-${scope.end}:${adjustmentsKey(scope.adjustments)}:${scope.opacity}:${scope.blendMode}(${scope.children.map(scopeKey).join(',')})`; +/** + * Plans `leaves` (drawn raster leaves, top first) over a document-space rectangle with the group scopes enclosing + * them. `contentRects` replaces a leaf's planned content rect with the extent of the pixels it will draw. + */ +export const planRasterComposite = ( + document: CanvasDocumentContractV3, + leaves: readonly (SemanticLeaf & { layer: CanvasRasterLayerContractV2 })[], + rect: Rect, + contentRects?: ReadonlyMap +): CompositeEntry => { + const layers = leaves.map((leaf) => { + const ref = toLayerRef(leaf.layer, document); + const content = contentRects?.get(leaf.id); + return content + ? { + ...ref, + contentOffset: { x: content.x, y: content.y }, + contentSize: { height: content.height, width: content.width }, + } + : ref; + }); + const groupScopes = planGroupCompositeScopes(leaves, collectCompositedGroups(document)); + return { bbox: rect, layers, ...(groupScopes.length > 0 ? { groupScopes } : {}) }; +}; + /** Plans the enabled base-raster layers over an exact document-space rectangle. */ export const planBaseRasterComposite = (document: CanvasDocumentContractV3, rect: Rect): BaseRasterCompositeEntry => { - const drawn = compileDocumentLeaves(document).filter(isBaseRasterLeaf); - const layers = drawn.map((leaf) => toLayerRef(leaf.layer, document)); - const groupScopes = planGroupCompositeScopes(drawn, collectCompositedGroups(document)); + const { layers, groupScopes } = planRasterComposite( + document, + compileDocumentLeaves(document).filter(isBaseRasterLeaf), + rect + ); return { bbox: rect, - key: `base-raster|${rectKey(rect)}|${layers.map(layerKey).join('|')}|${groupScopes.map(scopeKey).join('|')}`, + key: `base-raster|${rectKey(rect)}|${layers.map(layerKey).join('|')}|${(groupScopes ?? []).map(scopeKey).join('|')}`, kind: 'base-raster', - layers, - ...(groupScopes.length > 0 ? { groupScopes } : {}), + layers: [...layers], + ...(groupScopes ? { groupScopes } : {}), }; }; @@ -255,26 +296,26 @@ const layerMatrix = (ref: CompositeLayerRef): Mat2d => ref.transform.scaleY ); -/** The layer-local region of `content` an output over `bbox` samples, with one pixel of resampling margin. */ -const sampledLayerRegion = (ref: CompositeLayerRef, bbox: Rect, content: Rect): Rect | null => { - const inverse = invert(multiply(bboxView(bbox), layerMatrix(ref))); - const output: Rect = { height: Math.max(0, bbox.height), width: Math.max(0, bbox.width), x: 0, y: 0 }; +/** The layer-local region of `content` an output over `frame` samples, with one pixel of resampling margin. */ +const sampledLayerRegion = (ref: CompositeLayerRef, frame: Rect, content: Rect): Rect | null => { + const inverse = invert(multiply(bboxView(frame), layerMatrix(ref))); + const output: Rect = { height: Math.max(0, frame.height), width: Math.max(0, frame.width), x: 0, y: 0 }; return inverse ? intersect(roundOut(expand(transformBounds(inverse, output), 1)), content) : content; }; -const countGroupScopes = (scopes: CompositeEntry['groupScopes']): number => - (scopes ?? []).reduce((total, scope) => total + 1 + countGroupScopes(scope.children), 0); - -const outputPixelsOf = (bbox: Rect): number => Math.max(0, bbox.width) * Math.max(0, bbox.height); +const areaOf = (rect: Rect | null): number => (rect ? Math.max(0, rect.width) * Math.max(0, rect.height) : 0); /** - * Scratch pixels an adjusted layer whose pixels cover `content` needs: output-sized, except that a position-dependent - * stack adjusts the whole layer-local region the output samples, which exceeds the output when the layer is - * downscaled. + * Scratch pixels an adjusted layer whose pixels cover `content` needs in an output over `frame`: output-sized, + * except that a position-dependent stack adjusts the whole layer-local region the output samples, which exceeds the + * output when the layer is downscaled, and a spatial stack adjusts all of `content`. */ -const layerScratchPixels = (ref: CompositeLayerRef, bbox: Rect, content: Rect): number => { - const sampled = isPositionDependentAdjustments(ref.adjustments) ? sampledLayerRegion(ref, bbox, content) : null; - return Math.max(outputPixelsOf(bbox), sampled ? sampled.width * sampled.height : 0); +const layerScratchPixels = (ref: CompositeLayerRef, frame: Rect, content: Rect): number => { + if (hasSpatialAdjustments(ref.adjustments)) { + return areaOf(content); + } + const sampled = isPositionDependentAdjustments(ref.adjustments) ? sampledLayerRegion(ref, frame, content) : null; + return Math.max(areaOf(frame), areaOf(sampled)); }; const plannedContent = (ref: CompositeLayerRef): Rect => ({ @@ -283,41 +324,105 @@ const plannedContent = (ref: CompositeLayerRef): Rect => ({ y: ref.contentOffset.y, }); +/** The document-space bounds of `rects` placed by their layers, rounded out; null when nothing has area. */ +const placedExtent = (placed: readonly { readonly ref: CompositeLayerRef; readonly rect: Rect }[]): Rect | null => { + let extent: Rect | null = null; + for (const { rect, ref } of placed) { + if (rect.width > 0 && rect.height > 0) { + const bounds = transformBounds(layerMatrix(ref), rect); + extent = extent ? union(extent, bounds) : bounds; + } + } + return extent ? roundOut(extent) : null; +}; + +const plannedScopeExtent = (entry: CompositeEntry, scope: GroupCompositeScope): Rect | null => + placedExtent(entry.layers.slice(scope.start, scope.end).map((ref) => ({ rect: plannedContent(ref), ref }))); + /** * Pixels of the scratch buffers {@link renderRasterComposite} may allocate beyond its output, each also read back - * once as ImageData: one per group scope and adjusted layer. Estimated from planned content bounds; a layer whose - * rasterized pixels exceed them (measured text) reserves the difference through `deps.reserve` while it draws. + * once as ImageData: one per group scope and adjusted layer. A group scope's buffer covers its frame, except that a + * spatial scope's covers the union of its members' bounds, which is also the frame of everything inside it. + * Estimated from planned content bounds; a layer or spatial scope whose rasterized pixels exceed them (measured text) + * reserves the difference through `deps.reserve` while it draws. */ export const compositeScratchPixels = (entry: CompositeEntry): number => { - let pixels = countGroupScopes(entry.groupScopes) * outputPixelsOf(entry.bbox); - for (const ref of entry.layers) { - if (ref.adjustments) { - pixels += layerScratchPixels(ref, entry.bbox, plannedContent(ref)); + const scratchOf = (start: number, end: number, scopes: readonly GroupCompositeScope[], frame: Rect): number => { + let pixels = 0; + let index = start; + for (const scope of scopes) { + for (; index < scope.start; index += 1) { + pixels += leafScratch(index, frame); + } + const inner = hasSpatialAdjustments(scope.adjustments) ? plannedScopeExtent(entry, scope) : frame; + if (inner) { + pixels += areaOf(inner) + scratchOf(scope.start, scope.end, scope.children, inner); + } + index = scope.end; } - } - return pixels; + for (; index < end; index += 1) { + pixels += leafScratch(index, frame); + } + return pixels; + }; + const leafScratch = (index: number, frame: Rect): number => { + const ref = entry.layers[index]!; + return ref.adjustments ? layerScratchPixels(ref, frame, plannedContent(ref)) : 0; + }; + return scratchOf(0, entry.layers.length, entry.groupScopes ?? [], entry.bbox); }; -/** Composites an entry's layers, in z-order, onto a fresh bbox-sized surface. */ +type LeasedLayerSurface = Awaited>; + +const IDENTITY: Mat2d = { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }; + +/** + * Composites an entry's layers, in z-order, onto a fresh bbox-sized surface. Spatial stacks evaluate on their owner's + * whole source whatever the bbox: a layer's on all its leased pixels before resampling, a group's on the union of its + * members' bounds, landed at its offset. Any bbox over the same content therefore draws the same pixels there. + */ export const renderRasterComposite = async ( entry: CompositeEntry, deps: RenderRasterCompositeDeps ): Promise => { const { bbox } = entry; - const width = Math.max(0, bbox.width); - const height = Math.max(0, bbox.height); const readImageData = deps.readImageData ?? defaultReadImageData; const writeImageData = deps.writeImageData ?? defaultWriteImageData; - const view = bboxView(bbox); - const fullRect: Rect = { height, width, x: 0, y: 0 }; + const fullRectOf = (rect: Rect): Rect => ({ + height: Math.max(0, rect.height), + width: Math.max(0, rect.width), + x: 0, + y: 0, + }); + + /** The whole stack on `pixels`, whose (0,0) sits at `origin`; throws when `ownerId`'s stack cannot run. */ + const evaluate = async ( + pixels: ImageData, + adjustments: CanvasAdjustmentsContract, + origin: Rect, + ownerId: string + ): Promise => { + const result = await adjustExactly(pixels, adjustments, origin, { run: deps.runSpatial, signal: deps.signal }); + if (result.status === 'aborted') { + throw new DOMException('The raster composite was aborted.', 'AbortError'); + } + if (result.status !== 'ok') { + throw new AdjustmentRefusedError(ownerId, result.status); + } + return result.data === pixels.data ? pixels : imageDataOf(result.data, pixels.width, pixels.height); + }; - const drawRef = async (ctx: Ctx, ref: CompositeLayerRef): Promise => { - const layerSurface = await deps.getLayerSurface(ref.id); - try { - drawLayer(ctx, ref, layerSurface); - } finally { - layerSurface.release(); + /** + * The caller reserved scratch from planned content bounds; rasterized pixels can exceed them (measured text), so + * the excess surface and ImageData are reserved before they are allocated. + */ + const reserveExcess = (actualPixels: number, plannedPixels: number): { release(): void } | null => { + const excess = actualPixels - plannedPixels; + const reservation = excess > 0 ? deps.reserve?.(excess * 2 * 4) : undefined; + if (reservation?.status === 'over-budget') { + throw new RasterCompositeOverBudgetError(); } + return reservation?.lease ?? null; }; /** @@ -326,63 +431,76 @@ export const renderRasterComposite = async ( */ const drawAdjustedAtOutput = ( ctx: Ctx, + frame: Rect, ref: CompositeLayerRef, source: RasterSurface, origin: Rect, placed: Mat2d, adjustments: CanvasAdjustmentsContract ): void => { - const temp = deps.backend.createSurface(width, height); - setTransform(temp.ctx, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }); - temp.ctx.clearRect(0, 0, width, height); + const output = fullRectOf(frame); + const temp = deps.backend.createSurface(output.width, output.height); + setTransform(temp.ctx, IDENTITY); + temp.ctx.clearRect(0, 0, output.width, output.height); setTransform(temp.ctx, placed); temp.ctx.drawImage(source.canvas, origin.x, origin.y); - const pixels = readImageData(temp, fullRect); - applyAdjustments(pixels, adjustments, bbox); + const pixels = readImageData(temp, output); + applyAdjustments(pixels, adjustments, frame); writeImageData(temp, pixels, 0, 0); ctx.save(); - setTransform(ctx, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }); + setTransform(ctx, IDENTITY); ctx.globalAlpha = ref.opacity; ctx.globalCompositeOperation = blendToComposite(ref.blendMode); ctx.drawImage(temp.canvas, 0, 0); ctx.restore(); }; - const drawLayer = (ctx: Ctx, ref: CompositeLayerRef, layerSurface: { surface: RasterSurface; rect: Rect }): void => { + /** Draws a layer into `ctx`, whose (0,0) is `frame`'s document-space origin. */ + const drawLayer = async ( + ctx: Ctx, + frame: Rect, + ref: CompositeLayerRef, + layerSurface: { surface: RasterSurface; rect: Rect } + ): Promise => { let extra: { release(): void } | null = null; try { - const placed = multiply(view, layerMatrix(ref)); + const placed = multiply(bboxView(frame), layerMatrix(ref)); let source = layerSurface.surface; let origin: Rect = layerSurface.rect; if (ref.adjustments) { - // Adjust layer-local pixels before resampling, as the display does: reuse the display's adjusted copy, else - // adjust only the region the output samples. A downscaled layer samples more pixels than the output holds; - // a color-only stack then adjusts after resampling, keeping the work bounded by the output. Noise must stay - // layer-local so resampling averages it exactly as the display does, so its work is the sampled region. + // Adjust layer-local pixels before resampling, as the display does: reuse the display's exact adjusted copy, + // else adjust all pixels when the stack is spatial (it reads the whole source), and otherwise only the region + // the output samples. A downscaled layer samples more pixels than the output holds; a color-only stack then + // adjusts after resampling, keeping the work bounded by the output. Noise must stay layer-local so + // resampling averages it exactly as the display does, so its work is the sampled region. const shared = deps.adjustedSurface?.(ref.id, source, ref.adjustments) ?? null; if (shared) { source = shared; + } else if (hasSpatialAdjustments(ref.adjustments)) { + if (areaOf(origin) === 0) { + return; + } + extra = reserveExcess(areaOf(origin), layerScratchPixels(ref, frame, plannedContent(ref))); + const local = deps.backend.createSurface(origin.width, origin.height); + setTransform(local.ctx, IDENTITY); + local.ctx.drawImage(source.canvas, 0, 0); + const pixels = readImageData(local, fullRectOf(origin)); + writeImageData(local, await evaluate(pixels, ref.adjustments, origin, ref.id), 0, 0); + source = local; } else { - const sampled = sampledLayerRegion(ref, bbox, origin); + const sampled = sampledLayerRegion(ref, frame, origin); if (!sampled) { return; } - if (sampled.width * sampled.height > width * height && !isPositionDependentAdjustments(ref.adjustments)) { - drawAdjustedAtOutput(ctx, ref, source, origin, placed, ref.adjustments); + if (areaOf(sampled) > areaOf(frame) && !isPositionDependentAdjustments(ref.adjustments)) { + drawAdjustedAtOutput(ctx, frame, ref, source, origin, placed, ref.adjustments); return; } - // The caller reserved this scratch from planned content bounds; rasterized pixels can exceed them - // (measured text), so the excess surface and ImageData are reserved before they are allocated. - const excess = sampled.width * sampled.height - layerScratchPixels(ref, bbox, plannedContent(ref)); - const reservation = excess > 0 ? deps.reserve?.(excess * 2 * 4) : undefined; - if (reservation?.status === 'over-budget') { - throw new RasterCompositeOverBudgetError(); - } - extra = reservation?.lease ?? null; + extra = reserveExcess(areaOf(sampled), layerScratchPixels(ref, frame, plannedContent(ref))); const local = deps.backend.createSurface(sampled.width, sampled.height); - setTransform(local.ctx, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }); + setTransform(local.ctx, IDENTITY); local.ctx.drawImage(source.canvas, origin.x - sampled.x, origin.y - sampled.y); - const pixels = readImageData(local, { height: sampled.height, width: sampled.width, x: 0, y: 0 }); + const pixels = readImageData(local, fullRectOf(sampled)); applyAdjustments(pixels, ref.adjustments, sampled); writeImageData(local, pixels, 0, 0); source = local; @@ -401,48 +519,131 @@ export const renderRasterComposite = async ( } }; - /** Draws `[start, end)` bottom→top; a scope isolates into a buffer, applies its stack, lands source-over. */ + /** Draws layer `index`, through its already leased pixels when a spatial scope holds them. */ + const drawRef = async ( + ctx: Ctx, + frame: Rect, + index: number, + leased: ReadonlyMap + ): Promise => { + const ref = entry.layers[index]; + if (!ref) { + return; + } + const held = leased.get(index); + const layerSurface = held ?? (await deps.getLayerSurface(ref.id)); + try { + await drawLayer(ctx, frame, ref, layerSurface); + } finally { + if (!held) { + layerSurface.release(); + } + } + }; + + /** Lands an isolated scope buffer whose (0,0) sits at document `at` into `ctx` over `frame`. */ + const landScope = (ctx: Ctx, frame: Rect, scope: GroupCompositeScope, buffer: RasterSurface, at: Rect): void => { + ctx.save(); + setTransform(ctx, IDENTITY); + ctx.globalAlpha = scope.opacity; + ctx.globalCompositeOperation = blendToComposite(scope.blendMode); + ctx.drawImage(buffer.canvas, at.x - frame.x, at.y - frame.y); + ctx.restore(); + }; + + /** + * A spatial scope composites its members over the union of their bounds (the display's group surface), never + * clipped to the output, then evaluates its stack there in document pixels. Its members' pixels are leased first, + * since that union comes from their actual extents; nested scopes draw from the same leases. + */ + const drawSpatialScope = async ( + ctx: Ctx, + frame: Rect, + scope: GroupCompositeScope, + leased: ReadonlyMap + ): Promise => { + const held = new Map(leased); + const acquired: LeasedLayerSurface[] = []; + try { + for (let index = scope.start; index < scope.end; index += 1) { + if (!held.has(index)) { + const layerSurface = await deps.getLayerSurface(entry.layers[index]!.id); + acquired.push(layerSurface); + held.set(index, layerSurface); + } + } + const members: { ref: CompositeLayerRef; rect: Rect }[] = []; + for (let index = scope.start; index < scope.end; index += 1) { + members.push({ rect: held.get(index)!.rect, ref: entry.layers[index]! }); + } + const extent = placedExtent(members); + if (!extent) { + return; + } + const extra = reserveExcess(areaOf(extent), areaOf(plannedScopeExtent(entry, scope))); + try { + const buffer = deps.backend.createSurface(extent.width, extent.height); + setTransform(buffer.ctx, IDENTITY); + buffer.ctx.clearRect(0, 0, extent.width, extent.height); + await renderRange(buffer.ctx, extent, scope.start, scope.end, scope.children, held); + const pixels = readImageData(buffer, fullRectOf(extent)); + writeImageData(buffer, await evaluate(pixels, scope.adjustments, extent, scope.id), 0, 0); + landScope(ctx, frame, scope, buffer, extent); + } finally { + extra?.release(); + } + } finally { + for (const layerSurface of acquired) { + layerSurface.release(); + } + } + }; + + /** + * Draws `[start, end)` bottom→top into `ctx`, whose (0,0) is `frame`'s document-space origin at scale 1. A scope + * isolates into a buffer, applies its stack, lands source-over. + */ const renderRange = async ( ctx: Ctx, + frame: Rect, start: number, end: number, - scopes: readonly GroupCompositeScope[] + scopes: readonly GroupCompositeScope[], + leased: ReadonlyMap ): Promise => { let scopeIndex = scopes.length - 1; for (let i = end - 1; i >= start;) { const scope = scopeIndex >= 0 ? scopes[scopeIndex]! : null; if (scope && i >= scope.start && i < scope.end) { - const buffer = deps.backend.createSurface(width, height); - setTransform(buffer.ctx, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }); - buffer.ctx.clearRect(0, 0, width, height); - await renderRange(buffer.ctx, scope.start, scope.end, scope.children); - if (!isIdentityAdjustments(scope.adjustments)) { - const pixels = readImageData(buffer, fullRect); - // Group stacks work in document pixels; the buffer is the bbox at scale 1. - applyAdjustments(pixels, scope.adjustments, bbox); - writeImageData(buffer, pixels, 0, 0); + if (hasSpatialAdjustments(scope.adjustments)) { + await drawSpatialScope(ctx, frame, scope, leased); + } else { + const output = fullRectOf(frame); + const buffer = deps.backend.createSurface(output.width, output.height); + setTransform(buffer.ctx, IDENTITY); + buffer.ctx.clearRect(0, 0, output.width, output.height); + await renderRange(buffer.ctx, frame, scope.start, scope.end, scope.children, leased); + if (!isIdentityAdjustments(scope.adjustments)) { + const pixels = readImageData(buffer, output); + // Group stacks work in document pixels; the buffer is the frame at scale 1. + applyAdjustments(pixels, scope.adjustments, frame); + writeImageData(buffer, pixels, 0, 0); + } + landScope(ctx, frame, scope, buffer, frame); } - ctx.save(); - setTransform(ctx, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }); - ctx.globalAlpha = scope.opacity; - ctx.globalCompositeOperation = blendToComposite(scope.blendMode); - ctx.drawImage(buffer.canvas, 0, 0); - ctx.restore(); i = scope.start - 1; scopeIndex -= 1; continue; } - const ref = entry.layers[i]; - if (ref) { - await drawRef(ctx, ref); - } + await drawRef(ctx, frame, i, leased); i -= 1; } }; - const surface = deps.backend.createSurface(width, height); - setTransform(surface.ctx, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }); + const output = fullRectOf(bbox); + const surface = deps.backend.createSurface(output.width, output.height); + setTransform(surface.ctx, IDENTITY); surface.ctx.clearRect(0, 0, surface.width, surface.height); - await renderRange(surface.ctx, 0, entry.layers.length, entry.groupScopes ?? []); + await renderRange(surface.ctx, bbox, 0, entry.layers.length, entry.groupScopes ?? [], new Map()); return surface; }; diff --git a/invokeai/frontend/webv2/src/features/generation/core/__fixtures__/shared_image_manipulation_numerics.json b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/__fixtures__/shared_image_manipulation_numerics.json similarity index 100% rename from invokeai/frontend/webv2/src/features/generation/core/__fixtures__/shared_image_manipulation_numerics.json rename to invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/__fixtures__/shared_image_manipulation_numerics.json diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/__fixtures__/spatial_adjustment_parity.json b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/__fixtures__/spatial_adjustment_parity.json new file mode 100644 index 00000000000..efb109abd3e --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/__fixtures__/spatial_adjustment_parity.json @@ -0,0 +1,299 @@ +{ + "description": "Written by tests/backend/image_util/test_shared_numerical_fixtures.py. Labels: base64 uint8 hidden masks and base64 little-endian int32 visible-pixel raster indices from cv2.distanceTransformWithLabels. Adjustments: base64 RGBA8 inputs and the Python nodes' outputs, steps applied in order to the alpha bounds.", + "opencv": "4.11.0", + "labels": [ + { + "name": "equidistant_corners", + "width": 9, + "height": 7, + "hidden": "AAEBAQEBAQEAAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAAEBAQEBAQEA", + "labels": "AAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAQAAAAEAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAQAAAAEAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAQAAAAEAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAQAAAAEAAAABAAAAAgAAAAIAAAACAAAAAgAAAAMAAAADAAAAAwAAAAMAAAADAAAAAgAAAAIAAAACAAAAAgAAAAMAAAADAAAAAwAAAAMAAAADAAAAAgAAAAIAAAACAAAAAgAAAAIAAAADAAAAAwAAAAMAAAADAAAA" + }, + { + "name": "single_centre", + "width": 5, + "height": 5, + "hidden": "AQEBAQEBAQEBAQEBAAEBAQEBAQEBAQEBAQ==", + "labels": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "name": "hollow_ring", + "width": 11, + "height": 11, + "hidden": "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAAAAAAAAAAEBAQEAAQEBAQEAAQEBAQABAQEBAQABAQEBAAEBAQEBAAEBAQEAAQEBAQEAAQEBAQABAQEBAQABAQEBAAAAAAAAAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQ==", + "labels": "AAAAAAAAAAAAAAAAAQAAAAIAAAADAAAABAAAAAUAAAAGAAAABgAAAAYAAAAAAAAAAAAAAAAAAAABAAAAAgAAAAMAAAAEAAAABQAAAAYAAAAGAAAABgAAAAAAAAAAAAAAAAAAAAEAAAACAAAAAwAAAAQAAAAFAAAABgAAAAYAAAAGAAAABwAAAAcAAAAHAAAAAQAAAAIAAAADAAAABAAAAAUAAAAIAAAACAAAAAgAAAAJAAAACQAAAAkAAAAJAAAAAgAAAAMAAAAEAAAACgAAAAoAAAAKAAAACgAAAAsAAAALAAAACwAAAAsAAAALAAAAAwAAAAwAAAAMAAAADAAAAAwAAAAMAAAADQAAAA0AAAANAAAADQAAAA0AAAAUAAAAFQAAAA4AAAAOAAAADgAAAA4AAAAPAAAADwAAAA8AAAAPAAAAEwAAABQAAAAVAAAAFgAAABAAAAAQAAAAEAAAABEAAAARAAAAEQAAABIAAAATAAAAFAAAABUAAAAWAAAAFwAAABcAAAAXAAAAEQAAABEAAAARAAAAEgAAABMAAAAUAAAAFQAAABYAAAAXAAAAFwAAABcAAAARAAAAEQAAABEAAAASAAAAEwAAABQAAAAVAAAAFgAAABcAAAAXAAAAFwAAAA==" + }, + { + "name": "row", + "width": 12, + "height": 1, + "hidden": "AQEBAAEBAQEAAQEB", + "labels": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA" + }, + { + "name": "column", + "width": 1, + "height": 12, + "hidden": "AQEAAQEBAQEBAAEB", + "labels": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAA" + }, + { + "name": "random_sparse", + "width": 17, + "height": 13, + "hidden": "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEAAQEBAQEBAQEBAQEBAQEBAQEBAQEBAAABAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQABAQEBAQEBAQEBAQEAAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQABAQEBAQEBAQEBAQEBAQEBAQEBAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEAAQEBAQE=", + "labels": "AwAAAAMAAAADAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAABAAAAAQAAAAIAAAADAAAAAwAAAAMAAAADAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAEAAAABAAAAAgAAAAMAAAADAAAAAwAAAAMAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAQAAAAEAAAACAAAAAwAAAAMAAAADAAAAAwAAAAMAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAABAAAAAQAAAAIAAAADAAAAAwAAAAMAAAADAAAAAwAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAEAAAABAAAAAgAAAAMAAAADAAAAAwAAAAMAAAADAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAEAAAABAAAAAUAAAAFAAAAAwAAAAMAAAADAAAAAwAAAAMAAAADAAAABgAAAAYAAAAGAAAABgAAAAYAAAAEAAAABAAAAAQAAAAEAAAABQAAAAUAAAADAAAAAwAAAAMAAAADAAAAAwAAAAMAAAAGAAAABgAAAAYAAAAGAAAABgAAAAYAAAAEAAAABAAAAAQAAAAFAAAABQAAAAMAAAADAAAAAwAAAAMAAAADAAAAAwAAAAYAAAAGAAAABgAAAAYAAAAGAAAABgAAAAcAAAAHAAAABAAAAAUAAAAFAAAAAwAAAAMAAAADAAAAAwAAAAMAAAADAAAABgAAAAYAAAAGAAAABgAAAAYAAAAGAAAABwAAAAcAAAAHAAAABwAAAAUAAAADAAAAAwAAAAMAAAADAAAAAwAAAAYAAAAGAAAABgAAAAYAAAAGAAAABgAAAAYAAAAHAAAABwAAAAcAAAAHAAAABwAAAAMAAAADAAAAAwAAAAMAAAADAAAABgAAAAYAAAAGAAAABgAAAAYAAAAIAAAACAAAAAcAAAAHAAAABwAAAAcAAAAHAAAAAwAAAAMAAAADAAAAAwAAAAMAAAAGAAAABgAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAcAAAAHAAAABwAAAAcAAAA=" + }, + { + "name": "random_half", + "width": 23, + "height": 19, + "hidden": "AQABAAAAAQABAAEAAQEBAQEAAAEAAQEAAAEAAQABAAABAQAAAAABAQAAAQEAAQAAAQEBAAABAAABAAEAAAEAAQEBAQAAAAEBAAEAAAABAAEBAAEAAAAAAQEBAQABAAAAAAAAAAEBAAEBAQABAAEBAAEAAAEBAQEBAQEAAQABAQAAAQAAAQEBAAEBAQEAAAEAAQEBAQAAAQEAAAEAAQAAAAABAQEAAQABAAABAQABAQEAAQEBAQABAQEAAAEBAQEAAAEAAQEAAQAAAAEAAAEAAQEBAAAAAAEAAAABAQEAAAAAAAEBAAAAAQABAQEAAQABAAABAQABAQEAAQABAAEBAQEAAQAAAQABAQEBAQABAAAAAAEBAQEAAQEBAQEBAQABAAEBAQABAQABAAEAAQEBAQABAAABAAABAQEBAQEBAQABAQAAAAEAAQABAQEBAAABAAAAAAEBAAABAQABAQEAAQABAQAAAAEAAQAAAAAAAQABAAAAAAEBAAAAAQEAAAEBAQEAAQAAAAEBAAAAAAABAAEBAQEAAQABAAABAAAAAQEBAAEAAAABAAEAAAEBAAEBAAEAAQA=", + "labels": "CgAAAAAAAAAAAAAAAQAAAAIAAAADAAAAAwAAAAQAAAAEAAAABQAAAAUAAAAGAAAABgAAABIAAAATAAAAEwAAAAcAAAAHAAAACAAAAAgAAAAJAAAACQAAABYAAAAKAAAACwAAAAsAAAAMAAAAAgAAAA0AAAANAAAADgAAAA8AAAAFAAAAEAAAABAAAAARAAAAEgAAABMAAAATAAAAIAAAABQAAAAVAAAAFQAAAAkAAAAWAAAAFgAAABcAAAAYAAAAGAAAAAwAAAAZAAAAGQAAABoAAAAOAAAAGwAAABwAAAAcAAAAHQAAABEAAAAeAAAAHwAAAB8AAAAgAAAAFAAAABUAAAAVAAAAIQAAACEAAAAiAAAAIwAAABgAAAAwAAAAJAAAACQAAAAlAAAAJgAAACcAAAAbAAAAKAAAACgAAAAdAAAAKQAAAB4AAAAqAAAAKwAAACwAAAAtAAAALQAAADkAAAAhAAAAIQAAAC4AAAAjAAAALwAAADAAAAAxAAAAMgAAADMAAAA0AAAANQAAADUAAAAoAAAANgAAADYAAAApAAAAPwAAADcAAAArAAAAOAAAAC0AAAA5AAAAOQAAADkAAAA6AAAAOwAAAC8AAAAvAAAAMAAAADEAAAAyAAAAMwAAADQAAAA8AAAAPAAAAD0AAAA2AAAARwAAAD4AAAA/AAAANwAAAEAAAABBAAAAQQAAADkAAAA5AAAAQgAAADoAAAA7AAAAQwAAAEMAAABDAAAARAAAAEQAAABFAAAARQAAADwAAABSAAAAPQAAAEYAAABHAAAAPgAAAD8AAABIAAAASQAAAEEAAABKAAAASgAAAEsAAABMAAAATQAAAE4AAABWAAAAVgAAAEMAAABPAAAATwAAAFAAAABQAAAAUQAAAFIAAABSAAAARgAAAFMAAABTAAAAWwAAAEgAAABUAAAAVAAAAEoAAABKAAAASwAAAFUAAABNAAAATgAAAFYAAABWAAAAVwAAAE8AAABjAAAAUAAAAGUAAABYAAAAWQAAAFkAAABaAAAAUwAAAFsAAABbAAAAWwAAAFwAAABdAAAAXgAAAF4AAABfAAAAYAAAAGAAAABhAAAAcAAAAFYAAABXAAAAYgAAAGMAAABkAAAAZQAAAFgAAABmAAAAZwAAAGgAAABoAAAAWwAAAFsAAABpAAAAagAAAGsAAABsAAAAbQAAAF8AAABgAAAAbgAAAG8AAABwAAAAcAAAAHEAAABiAAAAYwAAAGQAAAByAAAAcgAAAHMAAABnAAAAdAAAAHUAAAB1AAAAdgAAAHYAAABqAAAAawAAAGwAAAB3AAAAdwAAAHgAAABuAAAAeQAAAHAAAABwAAAAcQAAAHoAAAB6AAAAegAAAHsAAAB8AAAAcwAAAH0AAAB0AAAAdQAAAIUAAAB2AAAAdgAAAH4AAAB+AAAAfwAAAIAAAACBAAAAggAAAIIAAAB5AAAAiQAAAIMAAACDAAAAgwAAAHoAAACKAAAAewAAAHwAAACMAAAAfQAAAIQAAACEAAAAhQAAAIUAAAB+AAAAfgAAAIYAAAB/AAAAgAAAAIcAAACCAAAAiAAAAIgAAACJAAAAiQAAAIMAAACDAAAAkwAAAIoAAACKAAAAiwAAAIwAAACMAAAAjQAAAI4AAACFAAAAhQAAAJcAAACYAAAAhgAAAJoAAACHAAAAhwAAAI8AAACIAAAAiAAAAJAAAACRAAAAkgAAAJIAAACTAAAAigAAAJQAAACLAAAAjAAAAIwAAACNAAAAlQAAAJYAAACWAAAAlwAAAJgAAACZAAAAmgAAAJoAAACnAAAAmwAAAJwAAACcAAAAkAAAAJ0AAACSAAAArAAAAJMAAACeAAAAlAAAAJ8AAACfAAAAsAAAAKAAAAChAAAAogAAAKIAAACjAAAAmAAAAKQAAAClAAAApgAAAKcAAACoAAAAnAAAAKkAAACqAAAAqgAAAKsAAACsAAAArQAAAJ4AAAC7AAAArgAAAK8AAACwAAAAoAAAAKEAAACxAAAAsgAAAKMAAAC+AAAApAAAAKUAAACzAAAApwAAALQAAAC1AAAAtgAAAKoAAACqAAAAtwAAALgAAAC5AAAAugAAALsAAACuAAAAvAAAALAAAACwAAAAyQAAALEAAAC9AAAAvQAAAL4AAAC+AAAAvwAAAMAAAADAAAAAwQAAAMIAAADDAAAAqgAAALcAAAC3AAAAxAAAALkAAADFAAAAxgAAAMcAAAC8AAAAyAAAAMgAAADJAAAAygAAAL0AAADLAAAAywAAAMsAAAC/AAAAzAAAAMwAAADNAAAAwgAAAM4AAAA=" + }, + { + "name": "random_dense", + "width": 31, + "height": 8, + "hidden": "AQEBAQEBAQABAAEBAQEBAQABAAABAQEBAQEBAQEBAQEBAQEBAQEBAQABAQEBAQABAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQABAQEBAQEBAQEBAQEBAQAAAQEBAQABAQEBAQEBAQEBAQEBAQEBAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQABAQEBAQEAAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEAAQEBAQEBAQEBAQE=", + "labels": "BwAAAAcAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAEAAAABAAAAAQAAAAYAAAAGAAAABgAAAAIAAAACAAAAAwAAAAQAAAAEAAAABAAAAAQAAAAEAAAACgAAAAsAAAALAAAACwAAAAsAAAALAAAACwAAAAcAAAAHAAAABwAAAAcAAAAAAAAAAAAAAAAAAAAAAAAABQAAAAUAAAAFAAAABQAAAAUAAAAGAAAABgAAAAYAAAACAAAAAgAAAAMAAAAEAAAABAAAAAQAAAAKAAAACgAAAAoAAAALAAAACwAAAAsAAAALAAAACwAAAAsAAAAHAAAABwAAAAcAAAAHAAAACAAAAAgAAAAIAAAAAAAAAAUAAAAFAAAABQAAAAUAAAAFAAAABgAAAAYAAAAGAAAABgAAAAIAAAADAAAABAAAAAQAAAAJAAAACgAAAAoAAAAKAAAACwAAAAsAAAALAAAACwAAAAsAAAALAAAABwAAAAcAAAAHAAAABwAAAAgAAAAIAAAACAAAAAgAAAAFAAAABQAAAAUAAAAFAAAADAAAAAwAAAAGAAAABgAAAAYAAAAGAAAAAwAAAAkAAAAJAAAACQAAAAoAAAAKAAAACgAAAAsAAAALAAAACwAAAAsAAAALAAAACwAAAAcAAAAHAAAABwAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAUAAAAFAAAADAAAAAwAAAAMAAAADAAAAAwAAAAGAAAADQAAAA0AAAANAAAACQAAAAkAAAAKAAAACgAAAAoAAAALAAAACwAAAAsAAAALAAAACwAAAAsAAAAHAAAABwAAAAcAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAADAAAAAwAAAAMAAAADAAAAAwAAAAMAAAADAAAAA0AAAANAAAADQAAAAkAAAAJAAAACgAAAAoAAAAKAAAADgAAAA4AAAALAAAACwAAAAsAAAALAAAABwAAAAcAAAAHAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAwAAAAMAAAADAAAAAwAAAAMAAAADAAAAA0AAAANAAAADQAAAA0AAAANAAAACQAAAAoAAAAKAAAADgAAAA4AAAAOAAAADgAAAA4AAAALAAAACwAAAAcAAAAHAAAABwAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAMAAAADAAAAAwAAAAMAAAADAAAAAwAAAAPAAAADwAAAA8AAAAPAAAADwAAAA8AAAAKAAAACgAAAA4AAAAOAAAADgAAAA4AAAAOAAAADgAAAA4AAAA=" + } + ], + "adjustments": [ + { + "name": "clarity_default_size", + "width": 24, + "height": 18, + "steps": [ + { + "kind": "clarity", + "amount": 80, + "size": 40 + } + ], + "rgba": "AH4A/yGsAP8mxh7/NrYA/y/oAGVA6gj/OsgD/1OgAP9deQdxXk4A/4VTAP9rJAD/jRwA/5UZCf+TIQD/tVAC/5iTEf+jeFYA8MYA/+H0Df/14AX/6+cIxuW8AP//kw//AHUA/y6JAv8VsBH/Id0N0D/iEf8a3RP/Z8Ab/1eSEf9rjAH/YEEbVodBAP+XEwv/fTMC/44WAP+XTAr/tUMD/IeeAHGonxL/yrYa/+LgLf/P9Af/9ssA//++H33/own/AI4T/xKsFv8C1Sf/tGdgAGUJAwAjn9AAZKMeAAvDZQB1aAD/UVs2/3xNHft+KDj/iDEc/5w1H/+kNh7/nW4e/xGAaQChpzX/uNwo/7u9Hv/kwE//aenVAP+bF///pSrLAHod/wivEzsCtRb/9azEAELJ1wCMQVIAam3PAGABrQBctD4AdmA2/3hZLv9mJyf/mTQy/5pcMf+ZSTB/l30TPbaASv+wfif/qL0j/9ziJP+qpk8A8KwX//+eMP//b0D/FKcxzQCwLv8dqDb/DSPGACgB4QBmNGcARIwPAIwvTgBRmUocRoEz/0zCigCAUj//a2cQ8qJSOJKQaET/uHIh/6afLP/Knkf/yatY/9rGKP//ujz/27o1//6ba///kz//q5KGAAuAOf8onTn/JUIqALmkyAA9v10AZQg8ADQapwBfkmD/eG4+/1h8X/9nhl3/fHlO/5ZiSv+gbTb/tIZF/89hX//Nhz8Q041F/9lsW+P1g0v/1IpQ/+t/P///hGj/AINt/xOPVP8sd03/PW5E/zeYTv8qf4D/JGxh/1hrbP84gVT/DJ3MAHGkT/+fkU///wAA/wD/AP8AAP//+hQK/x4U+v/18BSMy3Rr/7dsWv/jTGD/0XRv//95QZT/fVr/HH1m/wBnjf8ZU4DeAFN8/xlLfv8wRFX/WXaB/2ViS/9JfVr/aHx2/1mxhf9iqF1B/wAA/wD/AP8AAP//+hQK/x4U+v/18BT/6VZR/9xnXf//Y0j/5DRh/9cOxQD/cXr/AIF0/wxek0oNVX7/HUOa/zdXjv8yXpT/O0WS/05jd/9xg1z/Tq1jVHG4fyOCx3T//wAA/yaX4wAAAP9T+hQK/x4U+v/18BT/wESG/8BPiv/QPYL/2Edz//9Wd//3W3v/IY2F/xJdcv8AEab/wD2BAAARpv8AEab/S06e/0Vahf9oiWz/gqJ5/5m1h/9x1Zj/YNiD/5nPhf+Ip4b/la5+/6Buov/HQYv/1hxr/586QADzMY7/8jmR//9Ml///Woz/A46IvSBJ8QAAEab/ABGm/wARpv8AEab/SjuJ/19Xpf9zhpP/d6yH/2C3g/+A04//keOv/3bdm/+qt4j/sJSK/9FvhP/KTqn/dju9AOcek/+C4A4A5S2G//9Oiv//ZJL/AIK1/wBujv8AEab/ABGm/wARpv8AEab/UDW3/01XnP8/esL/Wq6q/3q/rP93wrD/dtfQ/43Auf+Xs5f/qpSo/9Fgqf+uT73/zTq6/+4Vtv/cHKz/8ja8/wFVnAD/Vo3AAGTF/wFutf8TNLv/LxTTEiINx/+P5gEAVUaq/11mrf9TjrL/cLqn/yj8jgB83qUpZOWt/7G+tf+U16r/HXX/ALN0vf/NUb3/tEC1/9BBov/lGJ//CQL2AP9plv/6dI3/BWay/wBPyv+Aa4QAIz/N/zgiy/9EI7/bUje3/0lq10lwkLH/Yo3L/33W0/9zybf/kuHV/67nyf+YqrP/pZCoH9WJyv/fbc6hz0m2/7M3xf/mG8z/503K//84wP/3db7/AIP3/wBa8f8AUbb/NE/FUgYv4/80NOj/WEWp/1933v9dcsSPXp/Z/36jyv9vx8P/iNTX/4Gk6/+Emt//npzS/7eO2v/gX/YGvlnK/+hA1f/AQ73/52Tc+PNV1P//Z7YJAIXw/w1vzv8TTur/LWb9/xhv1f+/Y/AAPUDizlJr4/9gesj/Ypzc/3utxP94teP/gJ3j/3yN5P+Zf///u5Tn/7Fw1P9HsvwAvF/v/+Ry+P/GQtr/32Dt/+ts5///cOL/GGLj/wpf5f8MnuL/Dv86AB185v9GiMf/P3r1/y99//9ekOxfanbg/xDRewCCtf//sUzzAI2F/P+Gbv//qY/d74E9rgDSVOH/rnDmyeRw///ugPb/+Yz/2vqN7v//Z/r/AHPe/yl///8Aqu3/QHP4/x6q//85h///P4v9/zl5//9bYP//YW/h/1BT5v+Obf//mmb//4ld/v+mif//oKLi/7t7///JhNn/1Jzm/8+H///1mP//8q3//++d///8kOn/", + "expected": "AH0A/yGrAP8kxRz/NrUA/y/nAGU/6QX/OccA/1OfAP9deQZxXk4A/4VTAP9rJAD/jh0B/5YaCv+UIgH/tlEE/5mUE/+jeFYA8MYB/+H0Df/14Ab/6+cJxuW8Av//lBf/AHQA/y2IAP8Srw7/H9wJ0D7hDv8Y3BH/Zr8a/1aREP9rjAD/YEEbVodBAP+XEwv/fjMD/48XAf+YTQv/tkQF/IifAnGpoBT/yrYb/+LgLf/P9Af/9ssB//++I33/pBL/AI0T/w+rFP8A1Cb/tGdgAGUJAwAjn9AAZKMeAAvDZQB1aAD/UVs2/3xNHft+KDj/iTId/502IP+lNx//nm8f/xGAaQCiqDb/uNwo/7u9H//kwE//aenVAP+cHP//pi3LAHkd/wSuEDsAtBX/9azEAELJ1wCMQVIAam3PAGABrQBctD4AdmA2/3hZLv9mJyf/mjUy/5tdMv+aSjF/mH4UPbeBS/+xfyj/qL0k/9ziJP+qpk8A8KwY//+fMv//cUL/EaYwzQCvLv8bpzX/DSPGACgB4QBmNGcARIwPAIwvTgBRmUocRoEz/0zCigCAUj//a2gR8qNTOZKRaUX/uXMi/6afLf/Knkf/yatY/9rGKP//uj3/27o1//6ba///lEH/q5KGAAh/OP8mnDj/JUIqALmkyAA9v10AZQg8ADQapwBfkmD/eG4+/1h8X/9nhl3/fHlO/5djS/+hbjf/tYdG/9BiYP/OiEAQ1I5G/9ptXOP2hEz/1YtR/+yAQP//hWn/AIJs/xCOU/8rdkz/PG1D/zaXTf8pfn//I2tg/1hrbP84gVT/DJ3MAHGkT/+fkU///wQC/wD/AP8ADv//+xYM/x4Z/P/18BOMzHVs/7htW//kTWH/0nVw//96Q5T/flv/Gnxl/wBmjP8YUn/eAFJ7/xhKff8vQ1T/WHWA/2ViS/9JfVr/aHx2/1mxhf9iqF1B/wQC/wD/AP8AD///+xYM/x8a/P/18BP/6ldS/91oXv//ZUr/5jZi/9cOxQD/cnv/AIBz/wpdkkoLVH3/HEKZ/zZWjf8xXZP/O0SR/05jd/9xg1z/Tq1jVHG4fyOCx3T//wQC/yaX4wAAEP9T+xYM/x8a/f/18BP/wkaI/8JRjP/SP4T/2kl1//9YeP/4XHz/H4yE/xBccf8AD6X/wD2BAAAPpf8AEKX/S02d/0Vahf9oiWz/gqJ5/5m1h/9x1Zj/YNiD/5nPhf+Ip4b/la5+/6Fvo//JQ4z/2B9t/586QAD1M4//9DuS//9PmP//XI3/AI2HvSBJ8QAAD6X/AA+l/wAPpf8AEKX/SjuJ/19Xpf9zhpP/d6yH/2C3g/+A04//keOv/3bdm/+qt4j/sJSK/9Jwhf/LT6r/dju9AOkhlf+C4A4A5zCI//9Ri///ZpP/AIGz/wBtjf8AD6X/AA+l/wAPpf8AEKX/UDS2/01XnP8/esL/Wa6q/3q/rP93wrD/dtfQ/43Auf+Xs5f/qpSo/9Jhqv+wUb//zzy8//AZt//eIK7/8zi9/wFVnAD/WI7AAGPD/wBttP8SM7r/LxLSEiIKxv+P5gEAVUWp/11mrf9TjrL/b7mm/yj8jgB73aQpY+St/7G+tf+U16r/HXX/ALR1vv/OUr7/tkK3/9JDpP/nHKH/CQL2AP9ql//7dI3/A2Wx/wBOyP+Aa4QAIj7M/zghyv9EIr7bUje3/0lq1klwkLH/Yo3L/3zV0v9zybf/keDU/63myP+YqrP/pZCoH9WJyv/gbs+h0Uu4/7U6x//oHs7/6E7L//88wP/3db7/AIL1/wBZ7/8AULT/M07EUgYu4v80M+f/WEWp/1923f9dcsSPXp/Z/36jyv9vx8P/h9PW/4Gk6/+Emt//npzS/7eO2v/hYPcGwFrM/+lB1v/CRb//6GXd+PRW1f//aLYJAITu/wpuzf8STen/LGX8/xdu1P+/Y/AAPT/hzlJr4v9gesj/Ypzc/3utxP94teP/gJ3j/3yN5P+ZgP//u5Tn/7Jx1f9HsvwAvWDw/+Ry+P/IRNz/4GHu/+ts5///cOL/F2Hi/whe5P8IneH/Dv86ABx75f9Fh8b/Pnn0/y58/v9ej+tfanbg/xDRewCCtf//sUzzAI2F/P+Hb///qY/d74E9rgDTVeL/r3HnyeRx///ugPb/+Yz/2vqN7v//Z/r/AHLc/yh+/v8Aqev/P3L3/xyp/v84hv7/Por8/zh4/v9bYP//YW/h/1BU5/+Obf//mmf//4pe//+mif//oKLi/7t8///JhNn/1Jzm/8+H///1mP//8az+/++d///8kOn/" + }, + { + "name": "clarity_size_10_half_even_radius", + "width": 24, + "height": 18, + "steps": [ + { + "kind": "clarity", + "amount": 120, + "size": 10 + } + ], + "rgba": "FooA/wDNAP+9oQ4AGM8A/yH7GnEe4wD/Ur8F/2SeGP9ReBb/V10P/3hFFP9tIgD/hyUA/3o9Av+TRSCzjo0aDLGFAP+8rgT/weIS/+DbAP/22xb/38cU/9jcCP/ejgD/AJ8V/wzDDf8Rqyn/IeQP/zfxFP80xiL/Hc0A/1WsGf9DhAz/XkYU/25VLP9/RAz/aR4u/6wWFvCIYhvZrFwz/5+KE/+stwn/sboA/962AP+4tyla38Q3//W1Gf/xmQrHCmw0/1shSwATuRT/A2mqAPmulQBgF8wA1kQIAJKLlABOexL/cnQr/1pgBf97PQf/hi0J/4U6OKaqRzL/iUgZ/6agLv9OLbwAwbw8/+TSAIzWxBv/861K//+yCP//dwD/L3we/ySqJ/8SziD/Bf9YAM440ACOtGsADZx4ANTM7ABObTz/F6j6AGJTMv+BVS3/fTZB/3k1MKKCYFD/RNGyAKukJP/Cmyv/wZ4K/8+qR//Muj7/6rsj//iHHv//uBP/AJct/w6VLP8AfSL/6bKYAJY9UgCSHhEAlpTGAFzZOgBrfCf/cnwr/3VpP/9lbjn/hl0n/4l9O/+nWib/sp9E/6CeQP+8kjT/t6c6//fATf/ju07/7LQ9reapUf//mC7/AI1L/xGXVP8Wlln/XEmQAPwbMwCbd/sAkiePAA0pmgBDdlj/Z4dw/2x3Pf94aEL/j5hj/7hoXv+CcFj/mp5N/8lkS//CkjD/7nkz/9ioSf/dkV1O3Z1fZ/+XSv/7lUT/FKZI/wBnK/8bj1j/AFJ3/0ZVYzdMjkv/qzJkAGSPNf9bgEn/gHxJ/4eBSP9jlGHm/wAA/wD/AP8AAP//+hQK/x4U+v/18BT/uI94/+1rPP/3djb//15W/+uJS///hWj/AH54/wZ7X/8STWr/GnN//zhzcf84T2fwLVRu/02CZv9ciny7YH1n/45vgv97p5D//wAA/wD/AP8AAP//+hQK/x4U+v/18BT/uXxc/9JEVf/uQXj/6WRb/9VopP//dVv/OC8qABNsYYYOUH3/Dlt//zcuYsFDRIX/QkCA/1hqg/9TcGb/TKyK/36bc/+CzZP//wAA/wD/AP8AAP//+hQK/x4U+v/18BRknD9//9JqjP+7OpT/51Sa/9Jhb//xW4v/DoiQ/wBEdP8+yAQAABGm/wARpvQAEab/V2N8/0homP9niqL/Z7Zw/2PAmv+L6or/gM2f/4H7l/+FsX7/maWL//ocxQCnV47/wx2Q/+ocf//jQZT/3DZ3/99Jh///UIL/An+F/xpUqv8AEaYhABGm/wARpv8AEab/MVeT/0ddmP9hhoz/UJ6j/36xof9v25n/ft6W/5O7jquoqMP/q5ik/6Z8j//TQ4j/8TqF/9svkP/TH5th1jiX//9bpv//Vpv/AzfFAAhmmv8AEab/ABGm/wARpv9VF6wAPj6L/1pVkf9KhLr/Xqmw/3fDf5t17Kv/ifig/4bjqf+sr5f/f6Gb/6GMqf+pSpv/7E6Y/8o8n//nOab/602g/+U/of/0d5r/BX2k/wA/xP8lIbn/HTDO/0I3xv9ZKbv/LnKe/01Ll/9lccP/Wbaq/2Kzqf9d1Kj/iNm5/4vLtP+IxZ7/oouq/752uv/HX87/xhe8/9FYyv/aH6v/7De20v8nvf//T9D/HZLQ/ylatf8jNsz/Ni3YpUIr3f9TLsb/P1v8/1lT1P9IfcaI1aQ3AIjKwLt/usD/ktDH/4/1z/+Nr5j/v4no/65twv+xS8D/zD/F/9lBuf+oJNL/5RPb//9Xuf//lpz/nHcJACBr0/8DO8r/8yTaAAZS1P87OMA/P4LR/1dY3P9cgdP/Sprh/5eRwf+NqNM3Z9HJ/6O64P++qb7/pZPh/5R70f+7Lf7/xT3L/+ZK2//jTNj/1ie6/+5xqP//ac//DHHm/wBi6YMiUuX/Mkns/y5P3/8taNn/MGDr/1hk5P9aYMr/eJrG/1G16v+JlOn/k63d/6i10v+Ittf/pYTU/6qZ6//ElNf/tlnO/85x8f/EVc3/7k7w/91u+P//hNf/EXn//wiM1P8Ie/b/HWjkgCGD//8njeT/Rl7t/zZ6//9ejeiMgZX0/2KK6NxLfv//hpn//4Sf//+5mPL/vYj8jouF/P/FU///yGr4/8Zs7f/gf87/53blotuO+f//g/yrAJTo/w6L3f85kfA0AHXv/zaP7P88jf//MJb0/2F1//9ai///f2b//4Fw92Jzdv//lWD//6J2//+mhP//pY7+/6JV///Fgv//vZH//7+H9v//pP///6X///yX////hP//", + "expected": "IpIS/wnPB/+9oQ4AIdMO/yH7GnEh5AX/VMEN/2SeGP9PdhP/VVsL/3VCEP9wJQP/hCQA/3Y7AP+OQRuziogRDLKGAv++sA3/wOEO/9/aAP/z2Av/4MgY/9baAP/vnSf/EqQe/xXGFP8gsjH/I+US/zjxFf85ySj/H84E/1SrF/9AgQX/X0cV/2pRKP97QAX/bCEw/6UJDvCCXRPZqVkw/6CLFf+uuQ//tb4Q/+G5Df+/vjRa4MU5//e3Hv/7oiLHGXQ8/1shSwAcvBv/A2mqAPmulQBgF8wA1kQIAJKLlABNeQ//bW8l/1ddAP96PAX/hCsG/4E3NaalQi3/iEgY/6agLv9OLbwAw74//+LRAIzXxR7/9rBN//+zF///j03/NIEk/yesKf8Rzh//Bf9YAM440ACOtGsADZx4ANTM7ABPbj3/F6j6AGJTMv99USn/fDVA/3c0L6J+XU3/RNGyAKukI//FnS7/xqMY/9OuTP/PvUH/6rsk//+SOP/+tw//AJct/w+VLP8HgCX/6bKYAJY9UgCSHhEAlpTGAFzZOgBpeiX/b3kn/3NnPf9iazb/g1oj/4R4Nv+kVyL/sJ1C/6CeQP+/lTf/uqo+//a/TP/ju07/7LQ9reirU///nD3/AItK/wmUUf8Qk1f/XEmQAPwbMwCbd/sAkiePAA0pmgBEd1n/Y4Ns/2p1O/93Z0H/ipNe/7VlW/+Ablb/mZ1M/8lkS//ElDL/8Xw2/9mpSv/hlWFO4KBiZ/+XSv/8lkX/AJ9C/wFnK/8RilP/AVJ3/0VUYjdHiUb/qzJkAGCLMP9af0j/f3tI/4V/Rv9hkl/m/AAA/wD+AP8AAPr/+RII/x4P+P/18BX/uZB5//BuP//5eDj//2Zd/+yKTP/+hGf/AHhz/wB2Wv8STWr/EG15/zJta/83TmbwLlVv/0p/Y/9Zh3m7YX5o/49wg/96po///woF/xz/GP8AAPv/+hMJ/x0L9//18Rb/uXxc/9ZIWP/xRHr/62Zd/9hrp//8cln/OC8qAAhnXIYKTnr/BVZ6/zcuY8FBQYL/QkCA/1Zogf9TcGb/SqqI/3+cdP+CzZP//yAW/yf/I/8AAP//+hUL/x4N+P/18BRknkGB/9Fpi//BQJr/51Sa/9Zkcv/wWor/AH2F/wBDcv8+yAQAABCm/wAPpfQAEKX/Ul52/0Vllf9liKD/Z7Zw/2TBm/+M64v/gs+h/4L8mP+Hs4D/mqaM//ocxQCnV47/xSCS/+ocf//jQZT/4Tt7/+JMif/7TH//AHV6/xJNo/8AEaYhABCl/wAPpf8AD6X/LVOO/0Valf9ghYv/UqCl/4G0pP9x3Zv/gOCY/5e/kqurq8b/rJml/6Z8j//TQ4j/7jaC/9wwkf/XJp9h2j2b//dTn//5UJb/AzfFAABekP8AEqf/ABGm/wARpv9VF6wAPDyJ/1hTj/9Jg7n/YKuy/3rGgpt27az/ifig/4fkqv+vspr/gaOd/6GMqf+uTp//50qU/80/ov/lN6T/6Eqd/+ZAov/scJP/AHSZ/wA9vv8kH7j/GyvK/z4xwP9WJLf/KGyY/01Ll/9lccP/Wrer/2a3rf9g1qr/idq6/43Ntv+Jxp//po+u/752uv/GXs3/zCLC/85Vx//eJa7/6za10v0ku//6Ssz/A4jF/yZWsf8iNMr/NSrVpUAn2v9SLMT/O1b2/1lT1P9JfseI1aQ3AIjKwLuCvcP/k9HI/43zzf+Rs5z/wIrp/7Jxxv+3Ucf/z0LI/9pCuv+yM93/5xnd//xUt//5kJb/nHcJABtnzv8FPs3/8yTaAARR0/89O8M/On3M/1hZ3v9dgtT/TJzj/5uVxf+Qq9Y3Z9HJ/6O64P++qb7/ppTj/5mA1/++P///y0TR/+RI2f/jTNj/3jLC/+5xqP/+aM7/BW7j/wBh54MjU+b/M0rt/y9R4f8sZ9j/MWHs/1lm5v9fZtH/eJrG/1C06f+NmO3/lK7e/6e00f+Ittf/qonZ/6iX6f/AkNP/vV/V/81w8P/KW9P/8FDy/99w+v/9gtX/B3X6/wCHzf8Ad/D/HWjkgBd8+P8eh97/SGDv/zV5/v9djOaMf5Py/2aO7dxZif//iZz//4eh//+5mPL/vYj8jo+J///HXP//yWv5/8hu7//gf87/6HfmotuO+f/+gvurAI3d/wCG1/8xieg0AHPr/y6I5f81hvj/KJDt/2B0/v9Yif3/g23//4Z2/mJ7gP//nXH//6eA//+pif//po///6lo///DgP3/uo78/7+H9v/7oPv//KL8//qW/f//hP//" + }, + { + "name": "clarity_negative", + "width": 24, + "height": 18, + "steps": [ + { + "kind": "clarity", + "amount": -150, + "size": 25 + } + ], + "rgba": "AGUO/weaAP8cqhH/B+AP/yjOAP8w6gD/PLcD/0nHF/9biAD/Z1EA/1JFAP+AMRb/lDQN/6klAP+yIAD/sTKwAKWhLP9T164Ap7QT/87mAP/rwAD/wsgY/1g8IgD/jAL/AoMI/waQJv9wRdgAIdw0/zLhBf9K9wD/O7sm/1KaCP9PWxb/VHsQ/3M9Bf+JKg3/gh4AwYk4F/+lTQD/A6isAK2UFBXAxRD/sMwA/zIIpgDy5CLGzvAK//+tBAT/yhv/CY0h/wCJGv8ptjPBrSyIAOazSQCgeDoAZ2POADT3HABJci7/iFMm/25fHv9WOg//iT4m/404Hv+qSBEVqG0L/8N9tQDCnBz/uO0W/9PtLzPRxTH/+9Ml//GtFVjxuyL/B5Ea/wCTMZgDvELI/S5IAFS8MADg6V4AOoAjAFA3hgCEgzn/gThJ/35aK/9qTTj/lUcS/4EzIP+uYTv/rm0x/6qcJv/Urh//x6wc/9SaPf/wqDv/5KYS//qLQ///hT3/DoMa/w96LP8Lq1T/3FeZAAMHEwAcQCsA3O0oAKIqCwBSWxr/aGBP/3h7R7N7ayX/c24j/4JfPf+NXTv/smMv/5KVN//MlUv/3cw/UO29Wv+7tS//6H1XR96dRv/jpDr/AIoz/wB9V/8aejf/LMtcAOhNTgA/E9UAV5L/AOBb3wBPfkr/YotT/0B3av9/fDr/kmRH/6p8Q/+Gakb/gF89cdOGR//OiE3/1ng6/8aMLP/qh0//3Zhl//+Daf/skFT/BoJr/wV3UP8Ob2b/E1Na/yBgRP9DhVf/JJhf/06Eav9HmFv/VG5j/26Edf94kzr//wAA/wD/AP8AAP//+hQK/x4U+v/18BT/04RgsgOS4wDhklX/8mpg//NvYv/4n3D/IltS/xBidv8aYUf/HFlq/09NW/8uWW3/QUlg/2RXW1lLhEb/W4RS/32FadKWr2T//wAA/wD/AP8AAP//+hQK/x4U+hD18BT/pVt4/89GWP/eQmztkGTvAPdvZf//iXv/F491/wBaeP8XdX//KVF+/zExkv83XKL/hbC8AFZkiVtiVWX/RpaL/7kK8ABsxHX//wAA/wD/AP8AAP//+hQKwB4U+v8KEQkAwk59/xW0UgDDLlv/5kN//6ZOBgD3WHz/AHSm/wRZcf8AEab/ABGm/wARpv8AEab/OjmF/zhkgf9qjYb/dKqZ/2zThNWLz4b/i+J2/4jefP92u5L/qHpv/6GImv/VYZ//riV49cIzh//MTXr/R0wbAPvMgQD/bon/IGiV/yZYpf8AEab/ABGm/wARpv8AEab/RB+W/2BXkv9LgqH/Za6L/2SozP9pyq3/nvWO/4m1if+9tbr/oL19/65wkvvKN4n/qCSh/7g2k//eHoD//xyA//BJl///VJ//EVux//c9KQAAEab/xElJAAARpv8AEab/PTKp/1Ncq01Yjpz/cMrRN3fFpf+E26X/hsvD/5/Qov+Iw5//qKyt/6JurP/JWYz/s0yW/+Yiw/+1F5T/9FKb//Rax///Y6v/AJa1/wx/vf8ZOcD/Sjmc/y8vzP8mI6v/NkSw/1lOs/9qc7blVqfF/5t2OABuu7v/ivq//47Csf+lvJ//qrG+/7Rng/+tRLv/wjjB//8mtP/mFsr/3w/D//RKn//2SpX/EGvE/wBE0nMbabT0MDjDoCQ9yf8cPdv/InC6/zJmsv9qhtv/GKt5AGDQ5f+OrM//i9za/67evP+/pan/0J3agatcxf/KVL3/wDy4/+Ueyf/2Ucf/+jyb//pDyv/rXb7/AIDX/7rqnAA2VdL/F0zL/ycx5/8iSs9LPC3F/zBl7/9Dl+P/SbnS/32txP9jlsr/cs7M/6C96ZGZpdL/nIz98LF50kalasP/zlnM/9lP1P/rT8n/5S/g/95S4v/ta7P/AJHc/wCDzv8Yd/X/KkvS/zli//9CSdX/PXHe/z6exv9UoPP/jZDd/3aN6P9umuP/iaT4/4iZ0v+VjdT/tI7A/8Ru3f/CgeH/43j9//o/wv/gVd//6oTf/95d1f//be//BIrL/y98//8AcPP/IHjn/yZb8f8sdMT/NG3oBWmN5v97gOT/RHnm/3Cl//9+iv//hJ///4mF9P+Zk97/tmPu/5qC///wE08A0HD//9Rq5f/Qeu//64L///9j6f/3k/z/AIT//xuo7v8yeP//G33sZwRf//9Ebf//W2///zx0//9nh/9Ga4///42D//+Dav//j3Px/x56XwCtiOz/qJH//7KD//+2g///wIv5/9N8///Orf//7o/1//ee////gf//", + "expected": "A2YQ/w6cBv8grBb/EeIV/yzQCP8z7Aj/P7kL/0vJHP9digX/aFIC/1NGAf+BMhf/lTUO/6klAP+xIAD/sTKwAKSgKv9T164ApbIM/83lAP/pvwD/wMYT/1g8IgD7igD/B4UM/w2SKP9wRdgAJd43/zXjDv9M+Qj/Pr0p/1ScDv9QXBj/VX0T/3Q+B/+KKw7/gx8BwYk4F/+kTQD/A6isAKuSEBW/xAz/r8sA/zIIpgDy5CHGzu8H//2rAAT+yRj/D48k/waLHf8tuTbBrSyIAOazSQCgeDoAZ2POADT3HABKdDD/iVQn/29gH/9XOxD/iT4m/404Hv+pRxAVp2wH/8N9tQDAmhj/uO0V/9PtLzPQxC//+tIk/++rD1jwuh//D5Qe/wmWNJgQv0XI/S5IAFS8MADg6V4AOoAjAFA3hgCGhTv/gjlK/39bLP9rTjn/lUcS/4EzIP+tYDr/rGwv/6mbJP/TrR3/xaoY/9GYOv/upjn/4qQL//eJQP/8gjr/FIYd/xR8Lv8Vrlf/3FeZAAMHEwAcQCsA3O0oAKIqCwBTXBv/aWFQ/3l8SLN8bCb/c24j/4FePP+MXDr/sGEt/5CTNf/Kk0n/3cw/UO29Wv+5syz/5XpUR9ybRP/hojj/Co02/wiAWv8efTr/LMtcAOhNTgA/E9UAV5L/AOBb3wBRgEz/Y4xU/0F4a/9/fDr/kmRH/6p8Q/+EaEX/fVw6cdGFRv/Mhkv/03U3/8OJKP/ohU3/25Zj//2BZ//qjlL/EIVu/w16U/8Tcmn/FlVc/yJiRv9GiFr/KJti/1CGbP9Jml3/VW9k/2+Fdv94kzr//wAA/xP/EP8AAPj/+A0F/x0A8//28Rf/0IJesgOS4wDfkFP/72dd//BsX//3nm//JV1U/xVlef8dY0n/H1ts/1FPXf8wW2//Qkti/2ZZXVlNhkj/XIVT/36GatKXsGX//gAA/xX/Ev8AAPf/9wwE/xwA8RD28Rf/n1Zz/8lAU//YPGftkGTvAPRtY//+iHr/H5R5/wVde/8deYP/K1OB/zMzlP85X6X/hbC8AFhmi1tjVmb/R5eM/7kK8ABtxXb//gAA/xn/Fv8AAPT/9wsDwBwA7/8KEQkAvEl4/xW0UgC6JFT/4T57/6ZOBgD0VXn/DHiq/wpcdP8AFKj/ABSo/wAUqP8AFKj/OzqH/zllg/9sj4j/daua/23UhdWM0If/jON3/4nfff92u5L/pXds/56GmP/SXpz/pBlw9boqgP/FR3T/R0wbAPvMgQD+bYj/JWyZ/ylbqf8AFKj/ABSo/wAUqP8AFKf/RSCX/2FYk/9Mg6L/Zq+M/2Wpzf9qy67/n/aP/4m1if++trv/oL19/6psjvvDL4P/nhWX/64siv/WDXn/+hB8/+xFk//9Up3/Fl+1//c9KQAAE6f/xElJAAATp/8AE6f/PjOq/1RdrE1Zj53/cszTN3jGpv+F3Kb/h8zE/6DRo/+Iw5//qKyt/51pp//DVIf/qkSO/+AXvv+nAIj/8U+Y//JYxf/+Yqr/FZy7/xiEw/8bPMP/TDue/zAyz/8nJKz/N0Wx/1pPtP9rdLflV6jG/5t2OABuu7v/i/vA/4/Dsv+mvaD/q7K//65ifv+kO7L/ui65//sgsf/gAcT/1gC7//BGnP/yRpL/GHDJ/wNI1nMgbbn0MjvGoCZAzP8dQN7/JXK9/zNns/9rh9z/GKt5AGLS5/+OrM//jd7c/7Dgvv+/pan/0Z7bgaVWvv/ET7j/tzKv/98Pw//0T8X/9jeY//hAyP/pW7z/EYbe/7rqnAA5Wdf/Gk/O/yg16v8kTNJLPC7G/zFm8f9FmeX/S7rU/36uxf9ilcn/c8/N/6G+6pGZpdL/m4v88K52z0afZLz/yVTH/9RK0P/oTMb/4Cjb/9tP3//sarL/Fpjj/xKJ1f8gffz/LU/W/z9p//9ETNj/QHTh/0Ghyf9WovX/jpHe/3aM5/9tmeL/iaT4/4eX0P+Ti9L/so2+/8Fr2v/BgOD/5Hn+//c8wP/dUtz/7Ibh/9ta0v//c+//FpDS/z2E//8Odvn/J37t/ypf9v8weMj/N3DrBWyQ6f98geX/Q3jl/3Gm//99if7/hJ///4eD8f+Xkdz/sl/q/5iA/P/wE08Az2/+/9Jo4//Qeu//7If///9n6f/6mf//Joz//yqw9v8+gP//I4PyZxRm//9Kc///X3T//z93//9pif9GbJD//42D//+BZ/z/jHDu/x56XwCsh+r/qJD+/7GC/v+1gv7/wIv5/9N9///Rs///8pP5//in////iv//" + }, + { + "name": "texture_positive_gated", + "width": 24, + "height": 18, + "steps": [ + { + "kind": "texture", + "amount": 150, + "size": 8 + } + ], + "rgba": "GX8A/w2cAf8ovgAPJNgg/zfjCv8Y5AD/QdMIPTGNAP9QaAD/hDwA/3kdDf91JyP/uGDJAJcoEf+bRwD/oEcA/7FaIf+uyAT/5fIUUJnPAP/r6A3/+/0f//+yE//skQH/AJka/wCaB/8k1x//F/UA/7kmXwAzxif/McYL/yuYAv9TZAb/YWML/1ciDP+ZBxb/nTsL/x2u9AClUiH/mXYApteJG//Hphz/ysIC//C7JP/vAO4A8tgD/+O4E/809ekAPmwuABZ+AP8rnRj/vN3pAN0cJgBM9HsAzmYCAK+PuQBVbAD/QlEB/44uE/+CPiz/tBIl/44PJ/+XYDn/sGUq/7CGEP+7ny7/5MQU/9vGMP89hGgA/7AiTuWuG0Smf90AE4o2lgCILP8PsC7/d5ldAAG9BAB4wyAAbYpyAHOv+ABfcz//ZEAw/1xVJf+YShz/hzsX/5g+Pv+uVg7/g4ItM7dcNP/Yuyf/usQ6/86tJqn/4iL/xbAH//iUNf8zbvcACKBM/wCMJf8mklL/J3wCAI72YQC54oQAIk6TAOIr2QB8kiv/aHdA/3NxOv9xbEP/mVVB/4mCMP+Ub0P/mltK/+xnvADAhjLOvpAv/8+vQ//JqkZP561v/+60Sv/vgzThEp9J/wCQPP8kmUHA0XQTAMZ9TwC97yMAKjOpAAzjQQB5kzr/ZHNalnJtXv+Ue1H/mHUt/49fVf+wVUL/tpA3/7tgQf/HhDn/0ZtE//uGav/mkiv/3n4q//6eHv/MnTT/Dodc/xx0Zf8AgGP/JmdR/xuCVf8yfVuEMW1o/05vTf/VQ1AAU3lS/2huaENiokv//wAA/wD/AKcAAP//+hQK/x4U+v/18BT/soNf/8KSN//mdkH/8Wxb//dcfv//VWT/DGeI/wBkY/8GaVu3DUJ1/zNLV/8zSnz/UGlr/8HsHwBJe1v/b4Rh/26JZv9ovHv//wAA/wD/AP8AAP//+hQK/x4U+v/18BT/3lqR/99tZf/ccmhs2Hpd/91rh///iHX/AHt0/yhFZqgZcYL/KD6e/zZDgP8eQW7/JDB//09JjP9Prn7/dbVR/2WtcP975V///wAA/wD/AP8AAP//+hQK/x4U+t718BT/z0Jr/9Q6ef/NKXcArahXAP9gbP//anD/AJR7/wCDbv8AEab/ABGm/wARpv8AEab/S0Nm/2lelv9rbX3/h4yL/2mngf9lcEwAqOJy/6PEh/+Eopr/oK6L/8Jxd//GeYv/w1tt/85Hdf/YLnj/4TChufRZkP/6c2z/EF2i/wRXkv8AEab/ABGmqwARpv8AEab/OkSi/znmAABje6H/cbN//4TWk/+F0KL/hPCG/7fFhP+cs6D/p5KI/65etf+9PqT/tzK9/7IbikPaOXuZ2QS1/+FNjf//W4n3AHqzBxtkif8AEab/ABGm/wARpv8AEab/V1Cu/2VYp/ep46cASbmBI2jNgP9/zKf/a8yu/w7V6gCu1rX/onyX/8Nipv+qUpX/xi66/9Qjqv/1MN//2hKq/+1Yrf/tZqUUAGSjXgdBxf8DZan/Hw2R/wQfsrNBG6z/PTql/1Fgrv9tdMT/YLSu/33Ruf9Tvq7/W8qo/5G7qv+Tt7r/l6Ck/6tbmP/mSqf/sVKwzdArrv/YJ8f/+RaQ//E2lP/yY7z/AIrY/xRduv8vQ8r/CS3Q/zgfx/89M9L/SXa2/11l2f9chbHNbqm9H2i8vP941Mr/guvE/8qIqAC5rL3/oY+m/5+Hz/+4bbf/ujzi/7gW1v/fAYUA703C//tDztP/Psf/AnXO/wl7vf8TUMr/MorT/yFjuP8vY+X/NUjS/3Jwv/9Nbt3/Z43R/2ggBgBZwK3/icK7/4uy3P+Tn9X/tLDj/6xq6//PfcH/0UHK/85Zov/uMtP/+lHq//9i6P//dNj/GI/l/wNv7l8pcfT/K2ny/x5Y7v80PuaOS2LwJ0SR9v9LbfH/TYjw/2+58P+ax/P/oq3V/4+n9Fi2ttr/vnrq/6Oc4f+aa8z/xl7i/9Yl0///VeH/6l3S//9n7f/7X+T/FYb/owhl+P8idO7/Mk///ySS//8ueeX/Wnj//yeG+P9qbvT/a4n//22g//9te/r/aYfr/3iG4f+PjP9jwpHw/17KNAC+n+f/tmTW//Bg///vc///5Vvs/92B////kuD/AKz6/wOK4/8AkO//HaLr/z+f//8slv//SHj/pVYDUACEfvv/fmLt/2lv9/9/c///hVz//5JY/eq2GJkAqWv3/7yB/Y2itP//s6H5/9uU4//Jqd7/3Kfz/99o9f//aP//", + "expected": "GoAC/w6dAv8pvgEPJNgg/zfjCv8Y5AD/QdMIPTGNAP9QaAD/hDwA/3kdDf91JyP/uGDJAJcoEf+bRwD/oEcA/7FaIf+uyAX/5fITUJrQBv/q5wj/9/kR//+1Lv/zlxb/AZkb/wKaCP8k1x//F/UA/7kmXwAzxif/McYL/yuYAv9TZAb/YWMK/1gjDf+ZBhb/nDoJ/x2u9ACkUiD/mXYApteJG//Hphz/ysIE//G8Jf/vAO4A8dcA/+a7G/809ekAPmwuABd/Af8rnRn/vN3pAN0cJgBM9HsAzmYCAK+PuQBVbAD/QlEB/44uE/+CPiz/sxAk/44PJ/+XYDn/sGUq/7CGEP+7ny7/5MQU/9vGMf89hGgA/7ItTuixIUSmf90AFIo2lgGILP8PsC7/d5ldAAG9BAB4wyAAbYpyAHOv+ABfcz//ZEAw/1xVJf+XShv/hzsX/5g+Pv+uVg3/g4ItM7dcNP/Yuyf/usQ6/8+uJ6n94Bz/x7IR//uXOP8zbvcAB6BM/wCMJf8mklL/J3wCAI72YQC54oQAIk6TAOIr2QB8kiv/aHdA/3NxOv9xbEP/mVVB/4mCL/+Ub0P/mltK/+xnvADAhjLOvpAv/8+vQ//Kq0dP561v/+2zSf/yhjfhEZ9J/wCQPP8kmUHA0XQTAMZ9TwC97yMAKjOpAAzjQQB5kzr/ZHNalnJtXv+Ue1H/mHUt/49fVf+wVUL/tpA3/7tgQf/HhDn/0ZtE//uGav/mkiz/4IAt//ycG//NnjX/DIZb/xx0Zf8AgGP/JmdR/xuCVf8yfVuEMW1o/05vTf/VQ1AAU3lS/2huaENiokv//wAA/wD/AKcAAP//+hQK/x4U+v/18BT/soNf/8KSN//nd0L/8m1c//hdf///WWf/C2eI/wBkY/8FaVu3DUJ1/zNLV/8zSnz/UGlr/8HsHwBJe1v/b4Rh/26JZv9ovHv//wAA/wL/Af8AAP//+hQK/x4U+v/18BT/3lqR/99tZf/ccmhs2Hpd/95siP/8hXL/AHly/yhFZqgWb4D/KD2d/zZDgP8eQW7/JDB//09JjP9Prn7/dbVR/2WtcP975V///wEA/wL/Af8AAP//+hQK/x4U+t718BT/z0Jr/9Q6ef/NKXcArahXAP1eav/9aG7/AJF5/wCAbP8AEab/ABGm/wARpv8AEab/S0Nm/2lelv9rbX3/h4yL/2mngf9lcEwAqOJy/6PEh/+Eopr/oK6L/8Jxd//GeYv/w1tt/85Hdf/ZMHn/4jGiufJXj//3cGn/Dlyh/wJWkf8AEab/ABGmqwARpv8AEab/OkSi/znmAABje6H/cbN//4TWk/+F0KL/hPCG/7fFhP+cs6D/p5KI/65etf+9PqT/tzK9/7Mci0PbOnuZ2wq2/+FNjf/9WYf3AHiwBxljh/8AEab/ABGm/wARpv8AEab/V1Cu/2VYp/ep46cASbmBI2jNgP9/zKf/a8yu/w7V6gCu1rX/onyX/8Nipv+qUpX/xi66/9Qjqv/zLd3/3Bar/+tXrP/rZKMUAGOiXgdBxf8AZKj/Hw6R/wQfsrNBG6z/PTql/1Fgrv9tdMT/YLSu/33Ruf9Tvq7/W8qo/5G7qv+Tt7r/l6Ck/6xcmf/mSqf/sVKwzdAsrv/YJ8f/+RaQ//I3lP/wYbr/AInW/xRduv8vQ8r/CS3Q/zgfx/89M9L/SXa2/11l2f9chbHNbqm9H2i8vP941Mr/gerD/8qIqAC5rL3/opCn/5+Hz/+4bbf/uz3j/7kZ1//fAYUA7kzB//pCzdP/Psf/AXXO/wd6vP8UUcv/MIjR/yFjuP8vY+X/NUnT/3Jwv/9Nbt3/aI7S/2ggBgBawa7/icK7/4uz3f+UoNb/s6/i/61r7P/OfMD/0kLL/89ao//uM9P/+VDp//5h5//9ctb/FY7j/wNv7l8pcfT/K2ny/x5Y7v81P+eOS2LwJ0SR9v9LbfL/Tonx/2647/+YxfH/oq3V/4+n9Fi0tNj/v3vr/6Oc4f+cbc7/x1/j/9go1f/+VOD/617T//5m7P/7X+T/E4X+owpm+f8idO7/M1H//yOR/v8ueeX/Wnj//yeG+P9qbvT/a4n//22g//9vffz/a4nu/3qJ5P+Rjv9jwpHv/17KNAC9nub/uGbY//Bg///ucv7/513u/92B///9kN7/AKn1/wGJ4v8Aj+3/GaDp/z6e/v8qlf7/SHj/pVYDUACEfvv/f2Pu/2px+f+Bdv//h2H//5Vd/+q2GJkArG76/7yB/Y2hsv3/sqD4/9qT4v/IqN3/2qXx/+Fq9///aP//" + }, + { + "name": "texture_negative", + "width": 24, + "height": 18, + "steps": [ + { + "kind": "texture", + "amount": -120, + "size": 6 + } + ], + "rgba": "AGcI/wCwAP8G0QD/NOAS/x7cAP85xwD/LMYJJ1+eCP9idAD/VU4cslAzCv99JAr/hkQA/5suAP+OOQD/rGMB/6SNAP+0qwj/398A/9zPB2j/7AD/9bgO/+zCDf/9pAD/AIsA/wmvHtsX0RH/Iv8q/zi6H/9Gzwb/VrcC/4V1SgBacDd6Z1UG/2sdAP9zMQD/iikA/4gjEf+xaSf/q3gt/5afHf+rugWf0MgA/8nmOv/p1Qf/++wD/9EkjgB157AAAGNE/wCVAP8Iph//PP6fAAJxKQAAs9UAjeF1AH+EIQBPdjMeX1IzNXU1Mf9nTwv/kS0XhI5IL/+8Txb/omQW/8uRO/+2pQT/zZZT//nXJP/p3j7/48wd//+8JZb7kij/DYNC/xOfEv8D0iT/lEgrAFxLhQDZ0tsAEejuAImz1QB0cif/W14qQHZRMf90Sir/gkUf/6EwLv+pWCD/sXIL/5uXOf/Bwx7/7pQt/+KqR//zwDX/6cAf//+qI///oT8PHplD/wByN/8OjDH/FAenALgzgADLE1MAUFS6AIJeBwBllCL/RHdm/2ldSv98STv/dkI3/5lbHv85EHAAm0gR/5CCLf+2djWqvZRB/82rRL7Qokj/b9TGAKq2RQD5nGH/EqA3/wB2NtWynToA38KQALBYlADAl30Agg9GANLVwQBIdVf/aIJW/2CCT/98bk+piWli/2NTRf+Udz7/rG9Y/7ZoQ//dkzf/7YtV4MuOPWHhrVX/44Uo//93QP//el1yAGtQ/wBvj/8pa0IeJJNt/yeKVeM5b2//SVdg/0ulUv9al3T/XIBX/3uKZ/95k1j//wAA/wD/AP8AAP//+hQK/x4U+v/18BT/w6Bn/+lRXiXgcWD/3W9M/y0tKgD/bTn/pv8CABhuXf8Ob2T/HTF7/zlaZf9NSlz/XFlJ/35/Vv9XfniqfYZN/2mSYf+GuXX//wAA+AD/AP8AAP//+hQKCR4U+v9JRkIAv5Fk/8xudv/yMH3/53lT/9Vxaf//iVj/BHNs/y9zjP8KSYP/QE55/zEvcv8/R3L/Rk5w/4lpfP93cXr/YJqN/2urg/+S13f//wAA/wD/AP8AAP//+hQK/x4U+v/18BT/0VGO/9wzav/nPpD//DWf//9Jdv//eV3/AI6K/wBnof8AEaYWABGm/wARpv8AEab/VVSX/zRbc/+AlI7/XJh//4G2mP90y3P/leFxsY/Wcf8Bc/4Ap4ub/7srDADVOxoA5jOCjsQ3kf/rJYT//xKX/9lSh//df3P/DXGi/wpgnP8AEab/ABGm/wARpv8AEab/UkSb/zpcgf9cgY3/dc2V/23Tqjt+3Jv/fuWX/2/in/+1o3z/tHWe/8p8iv9RRHMAzCW+Xcogm//lFI7/7hKP//ZIpf//a4X/BoKe/xZqnP8AEab/ABGm/wARpv8AEab/Tz2//1punQ0ei5f/W5eh/5l/iABy6YX/ZP+s/53nrP+8TX8ArJ+l/6F0qf/YZaz/w81DAO8xtv/kDcv/5B2m/+5BsAX/QJT/C3ve/xFf0f8tQKX/LjHl/047uv8ZM8L/SkrD/zpP0q+Ld8n/YpvUzX6/yP9027X/eO+7/5zKnf+mtrn/xbKccqxytv8Xkz0AzTmg/8cmrVDXSp3/8y21/+lXkSD/jq/ZAHbS/ylSuv8DO+r/ECfN/zc4yP/3+dUARkS1/1Bs1v9IkMz/dZDC/2C9v/+N1ab/pLnB/3njyf+Lycr/rJHZ/8pu3/+kP8r/slDI/+YkxrL5M7z/2ju7/+4tvyLgW77/AITW/xBqyf8XWb//CES3/0c/3f8jRcr/RkO7/2Jh0v9GhtH/dJjU/1DNzv9ss9f/a9C2/5TKyf+aneP/sZvY/6p50v+rXtH/1TLU/8RM8P/2T+P//1nJ//5Xvf/YXLL/FmDh/ypUyP8AUOP/FFLz/yxN//8keez/MUDi/0V84/9hjun/SojT/3KXv/+Mw9L/dK7L/7yE5v+vl97/oYr3/61w0UYwsN8Aymrn/7o/1f/NW+zBw1r4//9P3f//cOb/DZPt/yqO/v8Rev//AIfx/yiC3v94Zs7/SmDr/8Gi5wBVi+j/bHH9/191//9bqP//hYj//4CN//+Dlf+Ci3Xc/5pz0G3Uef//vZfZ/71w///QhOD/5Gf///GI0//wf/P/EoD//wCI8v+j9b8AJXj//zqb//8gfv//N4n3/0uG//9ci/z/X3ru/3eE/2mCXv//hFf//4Bl1v+ghf//qV38/6Oc+P/GjNb/v5b0/+B28P/RlP//4Jb2//xp3P/Ujvf/", + "expected": "AGUI/wCuAP8G0AD/Mt8O/x7bAP84xQD/K8UGJ1+eCf9jdQP/VU4csk0wB/98Iwn/iEYE/50wA/+OOQD/rWQD/6SNAP+zqgT/4OAD/9vOAWj/7kX/8bUA/+m/AP/1nwD/AIkA/wKtG9sU0A3/Jv8t/zW4G/9FzgL/VbYA/4V1SgBbcTh6aFYI/2ccAP9xMAD/iikA/4ciEP+0bCv/rXkv/5afHf+ruQOfz8cA/8nmO//p1Qb///Aj/9EkjgB157AAAGFC/wCTAP8BpBz/PP6fAAJxKQAAs9UAjeF1AH+EIQBQdzQeX1IzNXQ1Mf9oUAz/ki4YhJBKMf++URn/o2UX/8ySPP+1pAH/ypNQ//rYJv/q30D/4ssb//67I5b0ixz/CoJB/xGeEP8F0iX/lEgrAFxLhQDZ0tsAEejuAImz1QB1cyj/W14qQHdSMv90Sir/g0Yg/6IxL/+rWiL/snMO/5uXOf/Bwx//7ZMr/+CoRf/zwDX/6L8e//2oIP/8njwPH5pE/wBxNv8OjDH/FAenALgzgADLE1MAUFS6AIJeBwBnliX/RXhn/2ldSv98STv/dkI3/5tdIP85EHAAm0gR/5CCLf+1dTSqvJM//8yqQ77OoEb/b9TGAKq2RQD4m2D/F6I5/wB2NtWynToA38KQALBYlADAl30Agg9GANLVwQBIdVf/aYNX/2GDUP99b1Cpimpj/2NTRf+VeD//rXBZ/7ZoQ//dkzf/7YtV4MqNPGHirlb/4YMl//12P//+eVxyAGtQ/wJwkP8pa0IeKJZv/ymMV+M6cHD/SFZf/02nVP9cmXb/XIBX/3yLaP96lFn//w0H/x//HP8AC///+xYM/x4Y/P/07xD/w6Bn/+hQXSXfcF//2m1K/y0tKgD/bTn/pv8CABpvXv8ScWb/HDB6/zpbZv9NSlz/XFlJ/3+AV/9XfniqfYZN/2mSYf+Hunb//wAA+AD+AP8ADP//+xYMCR8a/P9JRkIAwJJl/8xudv/wLnz/6HpU/9NvaP//kmb/CXVu/zJ2j/8KSYP/QU96/zEvcv9ASHP/Rk5x/4pqff93cXr/YJqN/2urg/+S13f//QAA/wD9AP8AC///+hUL/x8b/f/z7gv/0VGO/9syaf/mPY///Tag//9NeP//g2j/DpKO/wZqpP8AEKUWABGm/wARpv8AEab/V1aZ/zRbc/+BlY//W5d+/4C1l/9zynL/lOBwsY3Ub/8Bc/4Ap4ub/7srDADVOxoA5zSDjsM1kP/qJIP//xiY/9dQhf/hg3b/E3Sl/w9in/8AEKX/ABGm/wARpv8AEab/VEad/ztdgv9cgY3/dc2V/2zSqTt925r/feSW/23gnf+zoXr/s3Sd/8p8iv9RRHMAzCW+Xcgdmf/kEI3/7A2O//lLqP//d43/EIai/xptn/8AD6X/AA+l/wAQpv8AEKb/UT/B/1tvng0ei5f/WZWf/5l/iABx6IT/ZP+s/5zmq/+8TX8Aq56k/6F0qf/ZZq3/w81DAPI0uP/kD8v/4hmk//BDsQX/R5b/E37i/xRh1P8tQab/LzTo/1A9vP8aNMP/S0zF/zpP0q+Ld8n/YZrSzXy9xv9z2rT/eO+7/5vInP+ltbj/xbKccqxytv8Xkz0AzDif/8Ujq1DXSp3/9C+2/+pYkiD/nbjZBnjU/ylSuv8EPOv/ECbM/zc5yf/3+dUARkS1/1Ft1/9IkMz/dI/A/1+8vv+N1ab/ore//3rkyv+Lycr/q5DY/8tv4P+hPMf/sU/H/+clx7L7Nr7/2Dm5/+0rviLgW77/BobY/xFqyf8XWb//BkK1/0c/3f8iRMn/RUG5/2Jh0v9FhdD/dJjT/1HOz/9rstb/a9C2/5XLyv+ZnOL/spzZ/6l40f+pXM//1DDT/8RL8P/5Uub//1/K//9Yvv/UWa//FV/g/yhSxv8AT+D/E1Hy/yxM/v8meu7/Lzze/0V84/9ij+r/SIbR/3GWvf+OxdT/c63K/7uD5f+vl97/oYr3/6xvz0YwsN8Ay2vo/7Y60f/MWurBwFf1//5O3f//dOb/EpXv/y6Q//8Ve///BYnz/ymD3/94Zs7/SV/q/8Gi5wBVi+j/a2/7/11y/P9dqf//hIf9/3+M/v+Dlf+CiHLZ/5hxzW3Vff//v5nb/71w///QhOD/42b+//KJ1P/xgPT/FID//wKJ8/+j9b8AJnn//0Of//8jf///OYv5/02I//9djP3/Xnjs/3aD/ml/Wvv/gFL6/3tg0P+ghf//p1r5/6Wf+//Ijtj/wpn3/+F38f/Sl///4pj4//to2//Ujvf/" + }, + { + "name": "dehaze_positive", + "width": 24, + "height": 18, + "steps": [ + { + "kind": "dehaze", + "amount": 90 + } + ], + "rgba": "AJYR/yG0FP9M9gD/NNQV/1bZE/891QD/P98A/0+sF/9tcw//ZV8A/303AP9zKQD/iQAA/34oAP+qNQD/kl4h/7KTE//EeugA1+AA/7OsAljv+hFV2sgA//DPAP/8rQD/AHAm/w27Df8Q1Qf/GtkD/zC5IVSjIyUAQ8Iy/1mXEf9OdAL/IonjAFklFf9qGCz/xN1AAHoMC/+eJxL/p1cax6WWAf+RsAT/6Mwp/93ZEv/70gH//7Ie//XPA///iQj/AJMb/wmQHv8Hsi5nWb6dACuejQAQUx0AESDDAPWssABvhjf/Xk4+/3I3G/9eICv/mCIP/4YjHv+KOA3/s3MA/7uLF9nCvCf/s9gj/8POFv/Lzxn/9NcO/86lNP/6ghf/Bo4//wCQG/8ilh//7B1yAN3V5AA95lAAtb/nACpRhQBMWCT/Z1FG/3JXI/9wUBj/cDdB/4U1Qv+3SkPAs3lB/6CJJP+pmAD/7akFANKzL//XriX/59Mz//+xL1H/iy3/AoIn/wCNS8snpEj/tRlKAB5k+AD9voQAj4NgAF+RigBYkVD/chbLAH1rMP+ESEb/bWpH/45UHv+97KUAZRSVAMaMMv+/jSL/2bw6f9KRL//3rFb/0pc94f+LJv/mhVP/AX1C/wB4V/8Nskj/xyAwAMiilQCdvbwAq64oACPJ4wBVdUD/cH1K/2yNQ/9wdGP/gmo9/4peOf+Uf1D/ooRR/6d1Rf/GhiT/vGxG/9WLUP/QgFn/8JMn/99+X//yeEj/AGRg/wxzS/85bnJeGo9L/x96bf9jhWz/JItE/26hbxNrZUb/a4Js13uMbP9gkjeO/wAA/wD/AP8AAP//+hQKfyNbWQD18BT/vnhb/+dYV//7fFL/129c/xdP2ADnfWX/G3tg/wBybf8JaH//MVdg/0RlfP83WFn/NXxTr1hnW/9LmYj/Oopu/26oW/9il4H//wAA/wD/AP8AAP//+hQK/x4U+v/18BT/rlCC/9pNcP/nWWDc5GJL//Bma///aXT/AGxu/yl0gP86PZT/HFp5/xJGaf8lMn3/OlOZ/1VknP9rZmL/T6Fw/22tdP9+xnv//wAA/wD/AP8AAP//+hQK/x4U+v/bjYwAuDWF/9pMi//WVZP/4kqLCv9Sg///aHL/CYh8/whmlBEAEab/ABGm/wARpvUAEab/Wl1r/05Ri/9XipX/UZpv/0vDc/984Ir/kcpp/5/Mpf+btn3/lbKM/5ROlv+wVnP/zyaPvfkkef/+Epf/4y6m/+Bafv/1iJj/AIx8/y9afP8AEab/ABGmOQARppMAEab/ZFGK/1x3nv9jgqT/VK2N/0zIof9szbr/cfCy/3HKnP99nYf/u6eZ/7Blq/+YQpH/2mWb/9oJq///J5L/1z+i//9fn///cobAAIOu/yJvlkQAEab/ABGm/4alWwAAEab/SjWH/1ZywORaiar/UqmW/3DZof9y4LX/nNGt/1u4tBaUmLH/sYZ//7GAvHSrRZH/wiqS/8QTkf/UCqX/9h+V//9To///Xaf/AIOy/xNbpv8ALcX/NiKy/yEfs/9ZMLrXRxqg/11XjP9Zr6VNTa/R/3m41v9p8qb/aNXMDonwvv+uyqP/mqXM/8RjtuW6P7j/vzqz/+MwwP/JKb7/8yjD/+VvxP//dMP/AYHb/yFbxf8UVer/ITfG/ywOwv9PL6b/QEe9/1B6yv9oe8T/ZrC5/4u9yv9z0bX/k+PQ/4a83uqSmbr/pI6c/5p0tBraZtH/vi/A/79Gyf/dOb0c9yu6Nf9GwP/9ctL/AIPO/yJwxP8XQs3/TLFyACcu1f8pVM//WljR/z+IyP9Tg8//bBMyAF644P+Ju97/Z6a6/463xP+Lrs//mK3P/8GK26S/b9H/yl/e/9464mLZQbtd8V/o/9Ne2zr/b8//AmxhAANr6v8ZwGsAADvFPABQ1/83iNb/KHHu/0SFy/9Tb/Kff4YNAGGZx1Fyrcn/gZ3g/5WY9v+WldT/maDV/6+U5v+uk9f/1FTk/9hc//+qDcoA9yrl/+9f4///gdf/B3jn/wqE//8Edvr/DlL3/xBx8P9Lce7/RILt/7wxhwBbeNP/YZb4/3Vv/P+EhfH/ko/+/22A9/+ge+X/jpfv/559//+1Zf//zWXu/91+9//YW9z/4Xb6/+6L6f/fbub/BoL9/zN2//8RlPD/PZr//zpt8/8eg+dFPH///16c7v9Ibf//XX///2lx//95W/3/iXX//5N82f+eifX5oYXm/6x9+P+cgP9W6TDmAKYPTgDgou7/4ar6/+uS////hv//", + "expected": "AJcR/xbAAP+p/5f/MuwA/17zAP9F7AD/SPsA/1G6AP9vdQD/ZV8A/3YzAP9lIwD/ewAA/3IjAP+rNQD/mFsA/7+dAP/EeugA8/0A/7qzAFj1/4tV79sA///mcP//wWH/AGUh/wDKA/8L7QD/HPIA/x/HAFSjIyUAMtcU/1yfAP9OdAD/IonjAEsRAP9dAB//xN1AAGkAA/+eGgD/rVQAx6+gAP+auwD//+RR//n0AP//6Zr//8l4///mhv//lTv/AJQb/wCQG/8AvjBnWb6dACuejQAQUx0AESDDAPWssAB0jyv/Wkc1/28qAf9SBRv/lxYA/4UAB/+FMQD/vHkA/8iTANnXzwD/x/IA/9jkAP/h5gD//++V/9yrAP//iSn/AJA//wCRG/8AmwL/7B1yAN3V5AA95lAAtb/nACpRhQBDUAv/ZEs+/3RUDP9vSwD/bCo3/4YmOv/GQjzAw343/6qPAP+yoAD/7akFAOjDAP/rvAD//+5t///MhlH/nVn/AIAm/wCQTcsAskX/tRlKAB5k+AD9voQAj4NgAF+RigBXnE7/chbLAINtIf+GQD//b2tB/5NQAP+97KUAZRSVANqUF//NlAD/9dEif+SXAP//xor/5J0l4f+iX//6hkj/AHtB/wB1Vf8Ax0//xyAwAMiilQCdvbwAq64oACPJ4wBQdjX/c4JE/3CXPP9yd2P/h2oz/49bLf+ehUv/ropM/7F2Ov/VjAD/xmg5/+mRR//ggVH//6VH//N/Wv//gVD/AFxY/wBwSP8rbnNeAJhM/wB8bv9ljnD/AJI//3WzdxNpYTz/bIhu14GVb/9fmyeO/zos/6n/ov8AAOz//ywffyNbWQD//oX/zHlW//tNUP//lHL/52tV/xdP2AD/gmb/AIBi/wBybf8AZn7/IlFc/z1lgP8qUlP/JYFQr1NlV/9Fp5L/J5Bv/3K4Wv9goYf//zMm/6b/nv8AAOv//yse/xwA7f///oX/tUWD/+o9b//7T1zc91pA//9uc///h4z/AGtt/xR6if8zL5j/AFd6/wA8X/8ZHnn/MUyg/1Jipf9pY17/R65z/2+9eP+G3YL//yof/6T/nP8AAOj//ygb/xwA7P/bjYwAvxWG/+w+kP/mSpn/8zaOCv9wk///iIz/AJSH/wBolxEBAIv/AQCM/wEAjvUBAI3/WVxt/0hKjv9Tkp//SKNw/z3Xd/+E/pb/nOFq/67ktf+nx4P/ocOW/5dDmv+5TnP/1wCSvf86gf//QJ//7wCs//JQgP//rLb/AJmI/yNYgf8BAIz/AQCNOQEAjpMBAI7/ZU6S/1x8qv9ih7D/TLuV/z3er/9s5M7/sv/V/3LfqP9/pYv/z7em/7xitv+ZMJL/8mWo/94Arv//UJz/5CGo//+FsP//l6LAAI+9/wR1pEQBAIv/AQCN/4alWwABAI3/RCWI/1V10+RWkLf/SLaf/3P1sv92/sv/q+vA/1LHwhabn73/vYqC/8CGznSvM5L/xQCS/70Ai//RAKL//yGb//93sP//gbX/AI7B/wBZq/8BKrP/MQCw/x0Aq/9aFcnXPgCV/1pRkP9WwbVNQb7n/33K7v+0/9D/aPDlDsv/4//D5bb/p7Tj/9New+XCJMH/yBi7//cA0P/MAMH//y3N//922f//n9L/AI3u/wxX1/8EUv//HBnU/yYAr/9LEqz/OznJ/0x+3v9ofdT/ZcHM/5bS4v9368r/u//u/4/R+eqYoMj/rpOk/55xvRrzZun/wADD/8kv1f/uDsoc/znBNf9px///mtz/AI3e/wBz2P8JK9v/TLFyACQA3P8dSuD/WVDj/zOP3P9Nh+L/bBMyAFzN/f+V0vz/YrDI/5rL2/+WwOf/qMHq/9eV96TPbuT/3Fj0//IN92LnIsVd/3X1/+NU7jr/ldj/AmxhAABv9P8ZwGsAADSvPABNzv8rk/H/H3P//zqL3/9Sbv+ff4YNAF6i2VF2vuD/h6j4/6On//+fnen/p6/v/8Kk///BoPL/5kb6/912//+qDcoA/zrt//9p8v//qeH/AH71/zuT//8ggP//Akz//wJ1//9Ndv//SIv//7wxhwBXd+T/dKX//3dx//+Ki///n5///2x///+nefj/m6T//6WI//+4bv//22j//+mb///pT+//6o////+m+v/0a/3/M4///0SC//81pP//aq///zxy//8Ai/5FTov//3Sw//9Kb///Zoj//2t0//91Uf//jHr//5Z56P+plf/5q4j9/7WJ//+jjP9W6TDmAKYPTgD0xP//8ND///K3////s/3/" + }, + { + "name": "dehaze_negative", + "width": 24, + "height": 18, + "steps": [ + { + "kind": "dehaze", + "amount": -140 + } + ], + "rgba": "AI0D/w2zAP8qtBL/DckA/z/vAAoqxi//T6sR/0ihCv9tVwD/V1QA/2ZQIP99JQD/dyoR/3kaAP96KQT/k00E/+CNAP/CpQP/rrAB/9/jBP/RyQAX4LwO/+zMFP/anBX/D15H/wCmFf8IwD//CdsZ/z3bAP9wtgr/O68E/zihB/9dezT/WmkARm8xK/+JFA3/lxIM/3g9Gf+rOhELmE8Fm5teAP+4tBf/IsceANXYG//5yBhJ/9sA/+bZCf//jxr/Fnw1/wOGGP8V1wD/De4fALR9awDlk68AKBCBACYWVgCYjgD/Vlse/4BEOP+VNhf/rEf6AJYlMf+XIwz/nGse/7uOEP+5rDr/6agA/73JLv/P3wD/6+Yj///UL//9lwD/DJo8/wCFTP8MwEL/briKAO60YwChYboA+yTxAGSGQgBPnBr/jUY2/4hOJv9vaTsAgj43/5I3Of+YWigtC7h/AL58JP/JrD3/x809/8rbIv/fxS//9sA2//+1KP//miL/AH0k/wKBJv8Uu1j/NIuVAIDwpQByYjwAU0GWABjTlQBllUj/b3g1/11QU/9wVUv/jnUp/51bWQCydDb/oHZd/6qURP/ClUf/zbNN/8+iJ3nMmTv/2YFE/+yHMf//jy//AKhQ0wuGZv8AjUP/V4ePAIa+BwA8XDEAPzWUAI9W4gBPkkL/VXVh/4Z3Qv8IkncAaohS/6dwRf+KhEv/vXRQ7bV8O/+yekf/y55i//OmTP/Ijkv/6ZxG/++dMf//glP/AHtU/wthR/8WY07/LmdW/zV0aP80WU//Qm9V/6WvewBTk2z/SIlS/1GSLv92hGD//wAATAD/AP8AAP+v+hQK/x4U+v/18BQvy19o/91neP/kbesA8G1c//OhPf//clz/AHmGigBqbf+wG6kAHnlv/yk9X/88e3H/WGh7/09zSf9Aek3/XZNh/1+krwBcmGD//wAA/wD/AP8AAP//+hQK/x4U+v/18BT/u1aN/+1ob//fZ3D/911c/+t5ZP/3YG1kAGl5/w5vhf8NUXMTAD1+/zQ9fP9DV3z/RFR//zh0iP9Rg37/WKhk/1mfef96sXb//wAA/2N7AQAAAP//+hQK/x4U+v/18BT/oFBj//JDi//ISYT/3V11//9oi//+eH//Ao+F/yFnnf8AEab/Ctl7AODOvAAAEab/akKb/0dVkf9qoJj/U7qR/33Ma9mOs57/cMB9/3qtkP+mqIH/mqCT/8h6ev++a5L/ozuQ/9ksflXpLHH//0aN//DozgDtWXz/F0iX/xQ8f/8AEab/ABGm/wARpnkAEaZ7PzKmOGpYhf9Kh5v/UaKN/0G1mP9dxg4Ak7ep/5vPpf+RwYj/oaGh/5mRnzDAP5f/Ji7uAMxBlP/mKJD/9gaz//86tM//aZb/AIyD/wBgp/8AEaZ/ABGm/wARpv8AEab/PDmk/0plmv9ahJr/XMWR/2nGz/9/5pz/fs+5/3j1of+i3d/8r5SvobJwiP/ATJ//HVM8AOAruv/kKMD/3Ti70P9Zp///cJn/BJKZ/wBQ2P8hK7f/DCWX/yUPt/9oGusA/2aUAENt1f9ddKr/c5HL/4a+u/9f77j/iNnm/4jksf8X/JoAp5LG/6xTuv/RVKD/yT/AMNouwv/MKsf/SKEAAP80vv8CHBYAComO/xJ4y/8LOZeAKD3p/5UCAwBBXc7/RS+3/0iHs/8eaM3/b4bH/2Xd0f95vuH/g7bB/5rRuP+Cr8P/x2zTfZJxwP+9RcD/c8fgAOQ0wv/vH9H/50LG//2FuP//hLD/AHOl/wBs0v8fW9FTD1Hn/yY5s/8iYdP/L0m8/1Rm3f9hkdbIaLHE/3mgzv9xw9D/rsXFOYu/tv+xkd//tGvM/8Jt1//fds//om/Q/+csyv/8Ndb/21Tn/yD3FgDiY+D/EGby/5WFEQANZ+D/P0Do/zF03v9EozMAS2ze/z500/9Qkf7/S7nx/4mh0f+Htu//a5DR/6e73P+Lneb/on/f/5B7+MSrcuif1l/x/9tl5P/Xd97/cGndAOJv2v/ue9j/AI38/xZ/6f8Eg+P/HoTY/zl34P9Bh87/PmHr/zOU9P85gfP/TZ74/1uF7/9/n/H/r5v/u5qD2/+FiOH/sZjf/6aj/91j6sYA1Hno/9OF9f/sj///5YT5//Ro5f//cvD/AHf//wCs8P8xjv//P5T/Aj+I4P8nif//XIn//0+F9f9gc/D/YpT/5Wt5//+Ojv//Rmb//4Zi5f+Wlfn/qXj/f7KB//+8Xv//13Ho/+iX+v/nh///33z///ae///zY///", + "expected": "PYg5/0CcOf9Jnj7/Qqk7/0m3NApHpkX/WZc9/1KOOP9yYzP/YWAx/2lZOf+DRDD/f0c2/4RBMf+BRjH/i1gy/7d9Nv+mkz7/nJ9D/6+zO/+wrEMXv6dI/8CrRP/JnVL/P21c/z+WPv9Ao07/PrA8/0yvOP9rnDv/Tpk6/0qPN/9gdUX/XWkuRnlLRf+MPjT/kTsx/3pPN/+YSTALjloym5NnN/+cm0D/IsceAKyvQv/GpUJJxKw5/7iwP//gkVD/QH1L/z6EP/8/rTf/De4fALR9awDlk68AKBCBACYWVgCAei7/YGQ7/31RSP+LSDP/rEf6AI9BQ/+SQzP/jGk6/5+AOv+elk3/wZQ//6GqS/+msjz/tLI+/8apRf/akkj/PYxL/zyBWP89oEz/briKAO60YwChYboA+yTxAGSGQgBRhTX/iFVI/4FXPf9vaTsAgU9J/4tKSf+KXj0tC7h/AKV2QP+olE3/o6hM/6KvQf+1pEj/xJ9K/8yZRv/Vjkn/OXxA/zp+Qf87mlb/NIuVAIDwpQByYjwAU0GWABjTlQBegEv/anFE/2lfYf9yXlf/fWw7/51bWQCYbEP/j3Ff/5SFUP+nh1T/qplW/7GSSHmzj1P/w4Rc/8yFT//ThEr/NIxQ0zh7ZP82gEz/V4ePAIa+BwA8XDEAPzWUAI9W4gBTgUr/XHNl/3pwTP8IkncAZntV/5JrTv99eVL/o29W7Z92TP+geFX/q4xj/8ONU/+xiFv/v4pS/8KJR//Se1v/M3RX/zdoVP85aFj/Qmhc/0RuZf9JYlv/T25b/6WvewBVgGT/U39Y/1eEQv9xe2H/00EzTDvCNv8ZTt6v0kY3/y1Q2/+7uTwvtGds/8Bsdv/kbesAzXFk/72FRf/TcWD/MG53ijFoav+wG6kANm1m/0JSa/9Fb2f/W2Z0/1dxU/9QeFj/XoNg/1+krwBgiWL/1kQ2/z/FOf8aT9//00Y4/y1P2/+7uTz/qWGG/8dpbP/BbHD/0mZi/8R0ZP/SaG5kL2Zx/zBndv8yWnMTLVGA/0VOfP9MWnX/T1t7/0dvff9Yenb/WpBg/16Ncv9xl27/2kc5/2N7AQAbUeH/1Ec5/y1Q3P+5tzr/ml9r/8xVgP+6YIb/w2l2/8xkef/Ha27/KnVt/zRhh/8eRKH/Ctl7AODOvAAeRKH/Z02L/1Jeif9jiIL/V5p9/2+lY9l/mYr/bKN0/3aZhf+Sk3j/iY2E/6tzc/+laoX/n1WP/8JPfVXNUHP/01V///DozgDLZHj/MlSM/zJPf/8dQqD/HUKg/x1Cn3kdQqB7SUmZOGldff9UfYv/W5CB/1Seif9dxg4AhJ2T/4aqjP+Co3v/jY2N/4iCjDCvVY//Ji7uALhXjf/KTon/00eg/9JQm8/KYn//LHVu/ytekP8dQqB/HUKg/x1Cn/8dQqD/R0yX/1JmjP9ee4v/XqN//2ekqv9wtYL/dKub/2q9g/+Fra78mIWYoaBxgf+uXZT/HVM8AMFMpP/FTKn/xFap0M1bjf/IZX//Lnl9/ydYtf8ySaf/KkeV/zNBqf9oGusA/2aUAEtrs/9ic5r/cYav/3qgnv9cuZP/dqy1/3Wzj/8X/JoAk4Sp/6JkrP+2X5H/tFesML9PrP+8VLf/SKEAANRQpP8CHBYAMnd7/zFtpv8wUpSAM1HD/5UCAwBGXqz/UEyo/1B6mf8/cLf/cIGv/1+upf9xoLj/e56m/4Knlv96maf/rm+2fY13r/+wXrH/c8fgAMVRqv/MS7T/yFmv/8Jvkf/FcI3/NHCT/zBrsv83YrRTLl3E/z1Uqf85ZbT/Q1yt/1lqvv9hgrPIZZaj/3WQsf9qoar/jp6eOXufmf+ahLv/pXO3/6luuP+za6f/mHW5/8pSs//RULT/v2PG/yD3FgDCar//M2rL/5WFEQA0a7//SlrL/0BvuP9EozMAUm29/051uP9Ugs7/UJe+/32Or/92lr7/co27/4iWrf9/jL//lH3A/4d808SbdceftWXI/7lov/+0crn/cGndAMFxuv/CcrL/L3nG/zd1v/8zd7r/Oney/0Zxu/9Keav/TGvL/0OCxP9Mfs3/VYvK/2SEz/91jMb/koXLu5CBv/+BhcT/lIO0/4aExN1j6sYAsHG9/6x2w/+1dcL/t3XF/8trv//La8D/MXLQ/yyGtP89ecf/QnrEAkd4tf86ecr/V3jJ/1R7yP9kds3/YYXQ5W592v+Bg9L/Xn3o/4t30/+EhMr/l3fVf5h4z/+pa9r/tG7A/694vP+2dcf/uHTO/7d7vf/KadL/" + }, + { + "name": "dehaze_short_side_20_half_even_radius", + "width": 28, + "height": 20, + "steps": [ + { + "kind": "dehaze", + "amount": 70 + } + ], + "rgba": "AJcB/wCTABgO4QD/BNgI/wHKANsd/xT/IbYA/yagAP8/iwD/TlQA/0saCP9rMB//XCkA/4seAP91RQD/l1gA/7B+AP+QwAr/sMwAZq7gD/+s2AD/Y1h7AMu/AHbTqQz/4YUiBeteAHnyLiFV/yIA/5SeAwAkqw7/CNAD/wDWGP8x3wD/O88A/yrDNv9JqQD/TXoA/0TkywBdVAr/cC0H/3wzAP+QGS7/ZVEX/5dxIf+bjif/lzqSAJ/LAP+v+AwWoOwC/7TyJf/Uvy3/4KAa//9pAf/SNQD/6zAA//8wCv8Pbi3/AKgP/xOyBv8DG1cA8HY2AIpbmAD7YpEAgm+uAF98I/9nWib/g0sq/2w0AP9kHR//oBck/51NPkGAZxf/oYkWnbueC/+lsxv/v9MniKzqBv+yvBL/vKYh/9Z2F//oZSUf/2k2//0pTP/zSyQEAH5Q/wmzMSwn6hUAjrnEAEeCcgCzU/EAgfvqAEOZWgBIfDD/SntD/0FME/9rJkP/dCE8/6VbWP+DcyP/oF89/5CFIP+Prgn/tMQA/6TEHv+TyDD/vMAm/8bBNv/VjjH/0ngg/+RzG//yTCn//0sX/wBoPv8kfzv/Eos3gaMD3ADDgwAAGzKtAOIAiQAqt7IAP3k1/zplOv9rVlr/b306/31WP+h/bUL/gHdE/2xpRf+IqTX/wY9X/6eqJv+toDX/w6Il/8K6P//FqSf/2YEy//lqPf/ykjD/50Eo/8wIqAAAdEv/OYsy/yeEVP+2ykoAFjWSAG+UrwAC+SYAFykRAEmHRP9Xekj/WnVc/14mvgBbXGH/b4A6/wuVrwCDmFr/gJUt/598VP+tgDT/rolD/xcSTwDbfV//wYNH/9eAPv/qfTT/6oxR//95Ov/+hD//AGhb/wNwT4wAZj3/M39G/xh4Uf8zflL/N4dB/zqaT5xXegcAZm1R/zgOrwB4kmL/ZJQ0/2+TM///AAD/AP8A/wAA///6FAr/HhT6//XwFP+/hlb/z2tQ/7N9Udvw8KoAyHJW/+GhQ///flf//5Bq/yJ0ZP8XeFn52EzbAB5JS/8TQGj/LVpP/y5tPf9OaXH/SoVn/0elXf9Dl33/Val7/36eZ/9swVz/osbgAAD/AP8AAP//+hQK/x4U+qb18BT/rW10/9daWP/OWk7/5n6F/7x78gDgkGv/0a5f/+mFa/8Ac3b/FWVvmBZYbv8RSmf/LkRG/0QiaP9OYlr/MW9w/zCGd/9pnXewWKhl8G64aP9l03f/eKxO//8AAP8A/wD/AAD///oUCsoeFPr/9fAU/5F0cP+6QID/yE1O/+B8c//sjVD/8Klt//aubGjwu0n/CHh4/wFdgv8AEab/ABGm/wARpv8AEab/NDZY/1l2Zf9Jekn/a71b/6zl6ABZ0WT/WdR4/3XRb/8onT0AgYRg/6NSnP+iLGT/q0OC/8MtegTCH3f/xyOE/8VAif/Gb23/6YSG//++fP/txJMg77eS/w2EhP8AYo//ABGm/wARpv8AEab/ABGm/z9Dkv9Mboz/TJ6I/0+fi/9hx3rFRuCG/1Pdqf918az/jMeR/5WYi/+oeYv/vUqM/7M1lFiSDpf/rTOL/8FKm/+8N4D/3pCJ/+WcjP/myI3//86I/+DGjf8On5//BVeL/wARpv8AEab/ABGmCQARpv8fTLD/TWGN/1aRg4dhxI//cda2/13ZhP9p+qL/cOKgPZy2k5GJmI3/lWqN/7lnqf+hHX7/whWH/8AAnf+yNHf/4Eucg9JVm//edHz/Gw2JAPPPdv//1of/I1yqAKAo7gAlR6T/Jzas/3fpjgAmLJj/WDqw/wlasf+HR78AV7ud/07ew/9zv7r/dcSH+Zz5o/9trrH/jKuBqK94j/+OZ5D/pSGo/6g/o//OE6H/nRmtwd1Fj//BcJxg5qSp//Gxh///3q3//8mn/xOM5f8HbrixLjK+/x4Aqv8zJ6//SiunwCBejP85X6//8UbdAF2hqUppvLH/cM6M/3vjtf9ysLn/hMO6qZt5kf+QZKz/oxvA/65ivf+/N7eb9Cq3/75Pqv9V+RAAqFn3APCMrv/jtan//Li8/+fXqxgZhrL/AHfK/wRbt/8APblZMDjA/zxQuP80TqH/Mm/L/1xi5f9XmbT/aKyW/1qqt/9Vu9H/f7S0/2Ccqf9zmbz/mHjN/40tz//AMsb/jFPV/88clv+8Xtv/yQAtAPd3wN7ElrT//6e2t/vAvv//tr//AIvJZQBnz7EcZLf/HWSw/x5Xs/8pXtf/TlDy/2pnyP9IjLv/4gV6AGZ/zUNKosz/jLKr/5qjyf+Eirb/gJDUKWS6+gCPZLj/qDy5/7Btxv/bR+P/0l6t/72SxVbZV9P//Xba//+yy//tmb3//6ew/wBv0P8Scsv/JGzI/zFu2P8eYuH/M3np/0CByv9Gc8f/V4Py/z+I7v9sg9v/UKLj/26RrP9+m+//hZPb/56K4f+lksf/ppTC/46O+P+0atL/+FvU//B26v/hZMT/93zX//+X0v/umOj//5/f//x76P8OtM3/CoLx/xqU+f8il9L/GY7Y/x2v8EJKg/dRm1ubAB2G//9mber/UnLrhGZg6v8b0OwAj2D//5x+vv+NgP3/lXbQ/6mA7v+vjvn/sWng/7ek3//fg/D/04PR/+Rp0v/yoe3/2Wf3/+Jd5///etT/Am78/wCe5/80j/f/IJHv/yKi//9q2AoAOKDu/1iH5P9LdPv/ZHrQ/1xV//9gWev/b0v5/wehGgCLZ+mGsFbY/5xz+/+Xmu3isp3//6Cf//+9men/1pr//8av4/+7ff//4mHp//9b4P//duL//zju9haB/f8chP//Lqrz/ym3+P8LxP7/GJz//ymS9/8be///PHP//1Re8/ZqPf//Vkb/2Ik3//+ENe7/lWL//7SF//+pcv9ciZvr/7PK8f++p///1M3+/6jC///wwf//xLP3/9hr//3/Qfr//VX///9M5f8=", + "expected": "AJgB/wCSABgR9wD/AOoG/wHXANuh/5n/I74A/yimAP9BjwD/SU8A/zkNAP9nJBD/TCEA/4IbAP9vQQD/l1gA/7aDAP+azQD/vtwAZr71AP+76gD/Y1h7ANvOAHbjtgD/84sABfRiAHn+BABV/zYd/5SeAwAfsgD/BeAA/wDnG/828wD/QN4A/wDSKP9NsgD/TnsA/0TkywBaUQD/ZyYA/3QvAP+OACb/YkwA/59zBP+jlAf/lzqSAKvaAP/T/5oWt/9P/9L/jP/o0Aj/8qsA//98PP/IMgD/6jAA//9DJv8AZib/AK4Q/w+6AP8DG1cA8HY2AIpbmAD7YpEAgm+uAF+ADP9mVxf/h0cf/2EuAP9bBA//oAAd/6RIOEGBZgD/qY4AncanAP+uvQD/0ecAiML/Sf++yAD/x64A/9x2AP/2YAAf/4BX//81Uf//RhcEAHtO/wC/Mywn6hUAjrnEAEeCcgCzU/EAgfvqAEOZWgBCfyT/Rn8+/zhDAP9kFDr/bwky/7BbWf+Idgz/p101/5WJAP+XuAD/wtQA/6/SAP+b2A//zNAA/9rUJf/lkxn/3HgA//J1AP//QRT//1wz/wBhOf8CgTL/AI41gaMD3ADDgwAAGzKtAOIAiQAqt7IANnoq/y9hMP9pUlb/c4Mz/39TOOiDbjz/hHo+/2tnPv+PtSX/z5ZU/7G0AP+3px//0KsA/9TKM//VtQD/6IMb//92S///pFD/7yYA/8wIqAAAckr/MJMo/w6KUv+2ykoAFjWSAG+UrwAC+SYAFykRAESNP/9UfUP/WHda/14mvgBXWF3/cYQx/wuVrwCJoVn/hJwY/6Z+T/+2giP/t4w4/xcSTwDrf1z/zIQ8/+aCMP/+fyH//5RP//+OXP//mmb/AGRY/wBvTowAYTr/KIVD/wB6UP8ng1D/LI08/zClTpxXegcAZGxM/zgOrwB9mmP/ZJsm/3GaJP//MCT/mv+S/wAA7///Ixf/HADw///+if/MilL/22hJ/7t9Sdvw8KoA0G5O//etO///knH//6iK/wp4Zv8AfVv52EzbAAlBQ/8ANmH/IVdK/yBtNv9JaHH/RIpo/0GxXv85n4H/UbWA/4KnZ/9u0lv/osbgAJf/j/8AAPD//yUZ/x0A8qb//on/tWx1/+VUVP/YUkb/+oKL/7x78gDylmv/47td//qHaf8Adnn/AGVwmABVbf8AQ2H/Iz0//zsOYv9LYln/I3Bx/x2Kev9qp3uwVLNl8HDGaf9l6H3/erZH//8mG/+X/4//AADw//8nG8odAPL///6J/5Z1cP/BMoH/0UJH//J/df//lU///7l+///BiWj/zWj/AH19/wBcgf8BAJP/AQCU/wEAlf8BAJX/KitR/1h6Zv9Ce0P/bc1a/6zl6ABV5GT/VOh8/3jkcf8onT0Ag4dd/6lMov+lFGH/sjmF/80RfATHAHf/zQCF/84xjP/Rbmz//IiL///Wrv//16gg/8ej/wCNjf8AY5D/AQCU/wEAlf8BAJb/AQCV/zs8l/9JcJL/RqiP/0enkP9f2H7FOPeO/0vzt/+l/8n/lNma/5ygkf+yfJH/yEKR/7sjmViKAI//sh6N/81Bov/DIoD/8piR//mlk//925b//+TA//XWlP8Ar6//AFWJ/wEAlP8BAJX/AQCVCQEAlP8TRrv/SmCS/1OYiIdg1Zj/dOvH/1nui/+1/8z/c/qtPabEnJGOoJP/mmmR/8dotf+dAHn/xACH/78AnP+3H3b/8UKlg95Nof/scnz/Gw2JAP/inf//68X/I1yqAKAo7gAcQKz/Iie0/3fpjgAgFpr/WC66/wBXsP+HR78AUsqo/0b21/90zsj/dtON+dP/1f9tub3/lLiHqLp7lv+SZZT/pACo/60xqf/TAKX/lwCowew4lf/Mb6Jg+6+1//+/lf//8+L//+HP/wCX+/8Acb+xKxvH/xgAkP8wAbX/RxOtwAlckP8xXLj/8UbdAFqqtEppzL//c+KV/4P+yf9zu8b/jNbMqaB5lf+TYLP/oAC+/7piy//KIsGb/z/B/8hHsv9V+RAAqFn3AP+Vuf/3wrT//8/S///uvhgAjb7/AHzS/wBZtv8AOKxZLCXJ/zdJwv8rRaX/Jm/Z/1td9f9TocD/aLif/1W1xP9Qy+X/hcPD/1yisf91ocr/oHrc/4oB0v/KE9L/jkjg/9IAlv/IWez/yQAtAP+Iyt7Qm77//7/Jt//a2P//0NX/AJPUZQBnz7EAYb//AGG4/wdQu/8dWeb/Tkn//2tl1f9Bksn/4gV6AGSA2kNCrN3/lsO6/6aw3P+HjsD/hZflKWS6+gCRXr//qia+/7tt1P/sOfb/31i2/8ya1VblS9///4vf///P3v//oMn//7/F/wBv0P8ActD/CmnT/yVs5/8EXPD/KHv9/ziF2f8+ctL/XIr//zyO//9uh+3/Tq/7/26Xtv+KqP//jJzv/6mR9v+ynNn/s57T/5iZ//+9Z9///27c//+E+f/wXs///47g//+03f//p/n//73n//+P7P8AxuL/AIb7/zSe//8An+L/AJTk/0C//0JQiv9Rm1ubADaP//9ma/z/T3D8hGRX+P8b0OwAkmb//6SBy/+Vi///mnXc/7OG//+6nP//uGPt/8ex9f/tjv//4oXg//Jj3///uvr/4Gf//+1Q8///ktn/AG37/wCp9/8+l///D5j//0+v//9q2AoAQq3//1eN+f9PeP//ZHzf/1pS//9dTfj/bT7//wehGgCPYvmGt0zk/6F6//+ipf/iv7D//62u///Lof7/3rD//9m++/++hf//71f3//9w4v//kOX//0vu9iKG//8tiv//Srn//2DH//911f//Q6f//zib//8mgP//QXf//1FV//ZnMP//Ujn/2Igx//+CAff/mGn//7+a//+ufv9ckaT//9Hi///Mu///6+j//8PV///44v//1cj//9p2//3/Wfn//m7///9f5v8=" + }, + { + "name": "dehaze_short_side_36_half_even_radius", + "width": 40, + "height": 36, + "steps": [ + { + "kind": "dehaze", + "amount": 200 + } + ], + "rgba": "F7UO/we3AAkSwAD/K9gA/xHZEf8H2AD/LskA/0aNAP9FaRn/YF4O/zBHBf9ZGwb/bgIA/30MAAoC+JgAe14n/3ttAP9kxAD/bb0AjGXiDP9i1hH/j8IA/5XeAv+YhgD/h3AK/6xmAP+LETcAsjIA/6kLAP+/Lw3/zzoWw9J/B7DRmgb/2a8A/9HfAP/f1S3/ydcA/+noAP//tAAt5Z8c/wCMEP8KoQD/ANsI/wDcCP9A1Qj/NNYa2UG8AP98yRwAKGQAICJTB/80MgD/RiYQ/0kzIP9sCgT/ZDMACUx2AP96nwD/bIgB/5PVBv9l3Az/ftkA/6PCAHhtpAD/i6kJ/4NpAP+xWhD/rBoX/6ZMAP+bEhT/0CcA/7RPAv/ZgACF0pcD/+KfGv/m2Aj/2+QU///sAv/n0AD//8YA/+ptAP8Olgr/P6MbnQC5JP8s5HUACZkLAJWaMADPPtsAj5CBADiEFZpLXxn/SEIO/0U4Aggtt7sAZjAA/2tLAP9dRCD/bHEl/3rRAP9lpRf/d7kM/5DYAP+FvgD/kawP/46kK/8LWv4Ay0YH/8NCC//DLDr/wDoL/9BkKf/HeoMAvGAF//B5Hv/2xgH/48AV/+u6CSPusBH/6cwD//6oA//ZiQz/BHUV/xGQAP8kpBb/DgoNAEqyvgAbp74ABiUpAOzbXwBAixz/IlUc/zQqDv9XJiT/QEtC/ztqOv93QRr/b5QW/2aMAP+EvQ7/Ub0e/1zMDf+R1QD/gcYOTTVR8wCSkSH/foQp/4FVAP9m5ksAm0In/5pPEP/TVCj/uFsG/9aDE//hbCD/phhsANSpFf/gwQ3/7rMuKv+pEf/+gxXf/30A/wCSIf8itA7/AI0V/1VmDADMAVsAtlhfAGywhgCg8SQAI3wl/0dmATE5XiqJOGQk/15MIf9FRR7/THYX/2iQP/9XbQX/TnUc/2WRFZl8tQP/hJwr/5egAP94mRH/npJA/7xeL/+lWhf/lHwO/61ACv+hTCf/wV8z68BsD/+2iCL/3p0K/+2lJ/+/lxz/btHoAOaGM//zjB3//5EJ//+FJv8AiyP/AHEy/wqaHl4PeZcAcoqOAL3tJQBpfa8AW2iFACyFFv9LiwT/MEcc/01wHf9AfDf/V2Fi/1KLENJudhD/cXMm/2VuIf9khwD/jW4Z/5GBFv+shxQwj4kt/5JsNP+cYSv/uIQa/8JcPf+sajf/wHAr/8B+Lv/RVDf/0Isk//9xD//Agyi12Lcv/+eDKP/ukSv/4ItB/+uPD//pej7/AGkk/wBvMf8QaDj/F4RH/y10H/9AYhn/J2sd/y9kNP8keyn/UZMc/0dtOf9lhSD/NbYv/0h3HP9tqR5iP4cQ/2h9O/94chX/fVUj/25aM/+AXiv/gvK1AONcVwCtVzz/oIIa/42QLv+0d0z/opsI/8CKMf+2iQ3/030z/1k2uQDSeFL/+IQl/8J8I//TdC//3IQr/8R5L///hi///4JF/wKTLP8AfDv/FDoo/xNnNf8mW0X/JGQn/0tmO/8rcS7/Nm9H/y2bK/8XNZ8AMq9I/z22H/9axz7/WZo4/2RrNf93azqpX2g5/2tVRHhsTDE3xFQt/5c9Tv+NbyvthltF/7uvMv+3mxr/rrEl/5XDOf+uqRr/pa9D/8CeSUajfTH/ma29APRpMefpTEH/3Vk7/9A7JP/2dz7/92pB//95Qf8fcir/AG4o/wA7QaVNge0AETZD/xtAVf82MTa/OnU6/y51Nv8qlCn/WoIz/0K9Sv9kuC3lXag//0eqOf9HlTb/aHFJ/49+Qv9/Sib/h0ZT/489QP+AWzj/qDVC45NmRM2oaCb/q8xC/6GdUl+82C//uN5I/9WKQv+toE3/yH8//8+AMuHnUk7/XayGANZGCP/jOUr/7k81/+JfKBH/czAZAJBN/wJqSP8XUER6GyQx/xYuTv8VIxj/MUU//wo1Ov9EhVD/Q6JA/zrVMf9dzEr/Ocgh/2i1Sq00tUD/al5D/2lrQLZ0T0T/bTlF/6BCUP+HOSj/gkdK/5A+Qf+sbSv/tINP/6mrMv+70ST/udgwkqnbVv/Z0jCkqsAc/9F9Tf/Zciux0ktj/8wySP/kNVr/7zY6/+8/Qf/+QUT//4hD/wCaSv8PREj/AjpY/yQdSiYELk7/QCh1/y4wUP8xcE7/PHgp/1DFSv85o2b/Puk0/0LLVf9Zym0TWdEy/1idef+GcDgW5bSlAIo/VP+HJWJHixxb/3X62wChVFH/hopr/8B9Of/Hr0L/rN9g/73kMP+/4TP/rN1B/9TVUf9jf2EAv3U9/+tUJv/eGET/6SZk//06Pv/yOl3/7Flc//9lQP8AcWn/AF9a/wAkT/8YMU3/Fi9i/w0fPP8CJlf/CmBT/yylUP8dulaOa+FG/yWyRv9PzWT/LL5L/2LRTv9jhj+ZWXI8/3ZLWP9YP0j/fjJX/4oJRv+MMVn/wE9Y/3NUSv+fkEjqprdz/6r6Yf+n2yP/rtxQ8bGvR/+7snT/up9JKtWBU+LUO1j/2B01//MfX//rGE3/6DxL//9PUf//ql3/CoFW/wU6Wv8YWmL/CTJC/xYnRf8IIEj/MUxD/0N3XP8kgDP/VL1G/0rLZ/9F81X/W9tV/0W2Of9axFj/DTMgAIRhZ/95blP/djdV/3MwVv/I7McAAP8A/wAA///6FApCHhT6//XwFP+wuGf/ytlJ/6DSZv/D3UP/qCqfALegTP/Fe2z/+mdf/8UrN//qB1f/JC5zAP8bRf//WWMFW9sZAABuYP8TVGn/srGYAAgta/8WUl//Sjdo/xpAev87XDr/YiqjADe0T/85uVn/FT1tAD3YWv9LuGn/Yq8t/12WVv/OSsQAbEdi/5UoZ/9qMEP//wAA/wD/AP8AAP//+hQK/x4U+v/18BT/pb1d/8zAXqmlwlf/sblF/7utS/+1jT3/zHNg/8I4Rf/PIm3/7hNw/+oaOP/QLXn/4E6A//hzYf8Armz/CEdw/w1UaP8gJWf/LS1Z/yZFS/88Qlv/KYFX/zN8Sf85p3r/OJl5bSzQbf8xz3z/PclZ/1CkWP9hqHv/Zmlp/0R6PQBuSVv/ckl6//8AAP8A/wD/AAD///oUCv8eFPr/9fAU/8atZP/KyYf/m8E3/7DVcP/HikX/Lv1yANZpZP//xFkA5ElY//8mXP/xNV7//09a//9dbP//nmT/AHp4/xWLbP8AEaabABGm/wARpv8AEab/KGVj/yBrYf8rc2H/NZNO/1eLYv9ekFX/JbB1/2KfZv9ulHf/W61QWIOAXf97iWf/bnZr/4ZPfv+PP2j/pmp9/5lqbv+Wimb/kZF6/6SJgP+UqX//r8Vt/7ecfv/GrFv/z4CK/61iff/ilGfI+Wxt/9lJgf+9Z3/S/1h7//9SXP/we17//4Bn/xRyZP8Ai3H/ABGm/wARpv8AEab/ABGm/yyIZv8ed2f/NHOA/ztmYfhvHaEAV4d2/z6LYf9HkHD/MpRy/2ySeKZ6gIT/coWB/2+Ob/9ddHr/fIpj/2wuogCel5f/kHRx/5OIhhi3nnH/jJtz/6p+gP/NgW3/xXRll6p6fau5gE7/vJCD/9qSef/zaYv/1llv/+BmT//teoD/5ntR//RdQv8Af1T/AJdOGBUqwQAAEab/ABGm/wARpv8oh5D/PZuZbS+Gjf9Ve4D/V4N8/z12jv9LX4SzamVr/1Vyh/9Td2P/YoiI/2qchv9tfG7/gJFq/4aUh/91mpL/gmyE/4ODev+hf4Zix4yG/8CHd/+ggH3/rVh+/717fP/aiaP/tJOJ/6TjRQDSpWb/xJ9e/8yjh8frlpH//5Gd/+l/df/wfnP/AIlwMgehgf8IvXL/GoiD/0ikof8/vYiwI5yj/12In//Py+0AOnNw/1NdlP8mT3v/R1eFV3NqfP9LeJPOfWpi/5+z4ACFnIL/YLeT/3m4rf97rHb/hn56/62CkP+ejnX/q2qN/6x5gv+qQnH/yU6O/9hwg/+5VpP/zGaS/8p7iv/XdX//6YWH/+Wvkf/1sIL//7KD/9qIk//8oID/9J6q/wBxhf8Ai4n/AKKP/xbghf8epZ//LKCb/ymfkz0qlIP/MXWP/0FkkpdBPnf/fEqP/204qf9PSLH/eluJ/2eJqP9ZiG4rip5z/3q5e/9fvIb/r8dz/5Gud/93lXz/cpmU/5WDhf+SYnD/rTmp/6s8lP+uQoj/5v+XAL1rev++baD/362nmq+0jKnXuZf/ysOiVP6/i///sJD/97Ry/+2Ekf8ekrX/EKCJ/yG2pv8czI25Ec6M/zy4ef8fuKr/Jo2f/0J8of9MW3f/WDV6/0U7mv9UT7n/ajOT/1k1pVFYLJh/Vmmq/3+ijP9ks4f/e+iv/5DtlP+Gp4f/I31ZAHmHdv+FfH7/sFeQ/7Y/oymuF4L/pCyZ/7ofhP+zNpr/y2SB/8GLcP8YzXwA1t2S/97Imf/mqZz/9dqf//+pp///iaf/Inue/wCHqP8AxZn/BdDF/y3bjZYa75X/KsB6/yumhv9bbZZjLFmv/0omiRYkDsL/YBGF/0gYsv9cSoD/bXas/1qPsf9+t5v/bqi0/3j/lGOY47D/p+qt/4jIqP+WfpH/r3mP/6pPov+XKpX/mSWf/7BMqf+6Oaf/202J/8WBlv/Np4f/3bCW/+Xxgv//4Y///fCb///EhP//s33/9KOW/yF+ov8HtLD/Layt/wLes/8t1o3/Dr2n/xrFq/8zsLP/LH6Z/0Rwe/9JN5z/PCO5/1YPg/9GGqf/aVWp/1p/nP+AaZ7/gLGIsWTDpP99yqj/feKB/4jkr/+Rm6T/m6it/0e6jgCxG0oAsDCk/7ITq/+jFJ3/7S6U/81EBAC1aJT/0Jyd/+LMkv/S1Yz/0+2Yf//TmUzhzKX/+eGl/96Xqv8Aarj/AIi//y2wtf8A0bj/GdrEkwvesv8cobL/LpOiI0dasv81aa3/LzSP/0MktP9pLbb/WkGf/2U7sv+TZ7H/e2Ol/1mpsP9+3amEdfm8tozXoP+TzqP/fqKk/4qgvaPMepNarl+y/6kguP/NIsf/uhGK/7wov87GPJD/uWef/8qHtcj9yJ3/3das/+Hgm//u2KT/+cK4//rMgv//kbA1AI/MfgCpu/8ey8j/GN+m/zPMt/8sy4j/Pbas/y2KuP8lkrb/Fly2/1NDwv9LI7z/RE2m/1gYu/9US5j/41IvAG2Eqf9ioq1MasSeBHrVqf+I5MX/muLB/5DFsv+ZoLT/hnu2/8Bjzv+/Op7/qh60/8E3wf+lLqX/t1rV/+Rrnf+vhqD/LQHHANfEoP/5x7r/29+q//Pbvv/crrP/+XO6/wCCvw0AmKz/ALrB/xauyv8A47H/Pam5/zqswPExg7//ajDQAF55vP9MTLX/QEap/2w6vf9GNdT/Wm7Z/1dgRABSicZ6wB2zAHjHsf+LucH/kqes/36Yvv+nl73/lJvK/597uf9rXqr/ykDD/6kttv+UKcD/vjzP/61Qrf+nc7f/13yv/966tv/Uuse37cvA/72Zu///tbD/9p2d//+DlP8emcX/AIzC/8vgCQAAntdxAJOd/zavp/9MMz0AOn2Y/z50w/87ocH/VHi8/1A8vv9SY5n/Rkqz/05ky/9aUqb/YITM/5crrQCJsJn/j53h/2eioP+MrcH/labC/5N9v/94cpQSqIjF/45etv+das7/01Ws/81O2f+pbKn/woDI/8uVxP/bjczIsqbO/+qD1P/WuLP/9562//+grv//ebXZAJLL/wCQrv8UjcP/JIrM/wCoyf8Qrb//FGvWc0F7vv8iftD/MpnB/15vxv9WcND/b3Ky/x12nv9ufsX/jWjX/2xxsP9U9pcAcpjD/3+Ov/9vnNH/hJa1/5Klyv93hPL/hYS5/7KJvf+yg8P/ulzW/7h/5f/GbNH/tGm8/+eJv//TgOL/2o2v/918yP/YlaX/+H/T//+Q0/80uDMA/5a4bgCeyf8Ak7r/AGPe/xV+3f8ZdNv/jULvABxj6Tsves+yL5PZ/y96zP9Affr/G6LPkV6HuP9BgtP/SZ+dW3N/xFpXxKYAX1nT/2Z8qv+KZ9H/j2zUu4d4yrt8hs7/vXC8/6Ou3P+cgKv/vn7g3cJizP+tm8j/wI++/8mHyf/Kb9T/5Zq4/75/wf/YeLv/3IDh//d6327pV8X//W3R/+ha1/8Mfvn/zHTPAB5p0v8AZ7v/FlPZ/yJL0f8OT8jPKXTN/yp8wMBGqd//HYfs/1KJv/89obn/ELvu/2WK6f9ZiPX/U3DK/4OHzAVhat//iFXG/5ROy/+7JuMAeGXy/52FvP+gitf/lpH0/7qv2/+FrL7/w7m4/8ektv/Rhbr/tnPUNOFY1BLNb+z/12Xj/1okIgDmRdhe7FPc//9B3bL/juP/Jmbs/wAr3v8bPeT/IDXl/xJK7v8fRPf/OWjY/5flJQAqh/r/R6K//0fLxf9M0Lr/cI+AAFm/5v9drv//UYrr/2aQvv9qfs7/hGPn/2It0cdjSNj/jDfs/5ZxxP+MU+f/ln7b/4eb2P+mstz/u8fV1t7D1v/PvO3/s5/O/x+g5gDSjvL/52To/+tFxv/UHcb/0Wi9//pE0P/tYOX//0+z5gB88f8Pb+7/IEDl/wYj5yMsKuz/Oy/3/ycz3/8sfPL/MnTg/zSm6f9GpcbOSMLx/1jy2f9NzO3/ZbTuVWm0qf9wden/ZD3c/49Tvf9yH9H/gg7p/50m8v+OX9P/oHPf/6+k8/+0yMYOl63D/6HS5f/S4OU5u9Lq/8SU0/+9nOeku3z1/8ZC/P/YLNj/6zvd/9VI9f/zZb4A903t/+KE/xYAivH/E0Dh/yc013AgE/9FIxz//x405P8rNNf/KmHm/0GE+r9VidT/RuLY/0nY2/9TzeT/aXoOAFG45P9cmtz/U1Ts/3hj//97Kuz/Zxi8/4Yp5v+YLOIGn0nhjpdd1/+dh9f/mkhVAI9CfgCh7v/Fv/ri/8nV/wrJyN3/yHXC//Nl4v/dTN3/sCP//+gA8P/zKuX/7TrW//xE/v//cPL/AKDmvBVo9f8AO+b/FS7o/yCotgAOLef/KyvW/zdb0P9Dkt7/VLfi/1nR5P9C3fTkRvzz/1jy+P9Jvf//FVJLAHiD//9UVP//dCz0y4cW7f+PJPEUeyvu/3Ih7f+Zc+H/tI7lzKu5/v+tue3/s+zH/67K+P+y4dP/wMPd/82u+v/pY/T/zyb/nOko///gLf3/4DXyw/Ep1f/9Xs7//2Po/wBc/P8AYvL/K0fx/zQs5f8mDf//Gizq/xda6Uk7cvf/JXT9/0qU//8zvdL/NNDc/1/8//9duv//Wqjo/1Kb2/9ifP3/j1L//38j+maGJOj/gDT0/73hhACWMv//loj//3dsAwCaxez/u8v//7O92//A3uv/xaT//9Gi///elfP/3zj//9FA///lMen/0TP8//E4////Iv//5jX//9+I+BMAevn/Ekfg/zZI//8GKf9KGCj//4XgQQAMUu7/N1rv6kCJ//84sd3/LKz//y/J+/9G1Ob/T7Ln/1Kq+v9fyPr/Ym3//3Np/f9MRuj/bsiyAHQt//+sOeL/oFT8/7gLRQCwn///k6//ZK+99f+nsP//xeD7/7nJ+aRaoAkAgzMOANZa2f+/OvM90FH///Uf///rNf///Df///9S+P//dv//", + "expected": "DtMA/wrWAAkY5QD/f/9v/33/df94/2//O/gA/02aAP8+ZAD/W1gA/x8wAP8xCAD/RAEA/1sGAAoC+JgAflkA/3xuAP988QD/heUAjL//p/+f/3X/s/EA/9X/qf+nlAD/hm4A/7JqAP+LETcAoCwA/4YHAP+1JgD/1DEAw/CRALD3tgD//9ZT//L/hP//+ZH/8v+I//7/ZP//5Lot/7tP/wCNEP8MsQD/hP99/4n/gf9+/2P/fv9u2VDjAP98yRwAIlgAIBY/AP8XFgD/IwwA/zIYAP9DAQD/RiIACUx2AP+PugD/dJIA/8r/h/+y/5P/v/+Q/830AHh+vQD/p8sA/3tjAP+1WQD/kwAG/5pGAP9zAAb/zCYA/61LAP/7lQCF97IA///CYf//+JH/9v99///1kf//9KL//+y+//FxAP8DoAD/P7kAnQDaLP8s5HUACZkLAJWaMADPPtsAj5CBADaNAJpCVwD/My4A/yohAAgtt7sASCAA/1xAAP9MMAD/bXIA/6v/ZP9zvwD/kOAA/83/l/+l6gD/r9AA/6nFAP8LWv4A0kcA/8U/AP/HAC3/uzQA/+9mAP/HeoMAvmEA//+bXP//7br//+yi///mpiP/25n///Kr///Zqf/rlAD/AGoQ/xKXAP8bugD/DgoNAEqyvgAbp74ABiUpAOzbXwA+mgD/BkQA/xMMAP9AAAX/LDsv/yBpIv9rMgD/fqoA/3CaAP+k6gD/XOYA/3L+AP/L/4z/ovgATTVR8wCnpQD/ho0A/2xHAP9m5ksAkSkA/4xEAP/tTQD/uVoA//mXAP/+cwD/phhsAP/LCP//7Zz//+GqKv/cs///q3Tf/6Nq/wCdJP8i1wD/AJQX/1VmDADMAVsAtlhfAGywhgCg8SQAAIAP/0JfADEhVgCJJl0A/1I9AP8wMAD/SXcA/22mI/9SZwD/SHIA/26gAJmX3AD/l7YA/7K9AP+HrQD/vKkd/9RYAP+lVQD/nYMA/583AP+cOgD/2VUA685yAP/OlwD//8JZ///UmP/aqwD/btHoAP+dTf//snH//8KR//+zhP8AlCb/AGou/wCuH14PeZcAcoqOAL3tJQBpfa8AW2iFACWOAP9SmAD/GTAA/0hwAP8mhRT/Tlxe/1iYANJyegD/dHYA/2JrAP9rkAD/k3AA/5+NAP/GmgAwoZkA/51pAP+jWgD/1JYA/+FMFv/BZQD/2XQA/9+JAP/uOwD/9qAA//+idf/XjQC1/+WH//+dTv//unv//6NV//+yY///jlL/AGAg/wBqLv8AYDH/AI5J/xt1AP82WgD/EWYA/wBfGf8Afxb/VqYA/zZtHv9skgD/CN8A/0N4AP+AywBiP44A/2mGGv97dAD/d0kA/2lNCf+AVQD/gvK1AONcVwDCRhr/so8A/56hAP/Sezj/u7MA/+ScAP/NmgD/+4gA/1k2uQD7ej///7SD/9WDAP/vdwD//5Yk/9d7AP//vZf//7ma/wCoM/8Agj7/AB0P/wBhLv8AUjj/AF4P/z9lJv8AchX/C3E4/wCyC/8XNZ8AANNF/z3cAP9U/wP/VLQA/2BpDf98ahupWGMa/2VHL3hjNwo34ksA/50SPf+WcADtiU4u/+vZAP/ZtwD/1dgA/7X0AP/QygD/yNUU/+27LEavfwD/ma29AP+LYOf/Qzn//j4A/8gfAP//o33//5Fy//+xkv8Adx7/AGwn/wAfI6VNge0AABok/wAqP/8dFh2/Gnsh/wB3If8Apgz/VY0A/wDsOP9w4gDlWcsY/yzNAP8xqgD/Z3M4/6SLK/94OQD/iS5F/5IXKf+DUBf/rgAx46BhLM21aAD/2P9l/722QV/p/53/5P+o//+dN//PuzX/7oUC//OLAOH/Tkz/XayGANFCAP/yAD3//1c7//1cABH/q4kZAKlc/wBpR/8AQTZ6AAQT/wASLv8AAwH/FjMs/wAWGf8slEf/HMAg/37/dP9x/1n/NvgA/2zgNq0A3DX/ZVQt/2dpJ7ZwPi//YBgu/64hQ/+AGQD/gTA4/5IYKf/BcAD/1pA//8vNAP/p/3//5/+fkt//tP//+42k0+8A//yDNf/5eACx7AlW/8UAN//zAFT//yc0//8+Qv//bGb//8qv/wC7XP8ALzP/ACI3/wkAJSYAEiX/LwBn/xUTPv8Ac0X/KnwA/zr6Nf8Av2z/v/+4/zb/XP9l/4MTjP9t/0+6hv+SchAW5bSlAIwfR/93AFJHdgBK/3X62wCyRUT/lZtu/+WHAP/61gH/3/+3/+P/n//m/57/3/+s//3/j/9jf2EA2nIA//9ZJf/UAD3/8wBj//9iXP//Rmb//2xt//+dhf8AeG//AFlV/wAIG/8AFjH/ABJJ/wADD/8ACyT/AFVJ/wDDVP8A4WSOxP+0/wDSSv9l/3//AOhP/5j/hv9hkiGZT3Ie/3I3S/9FJDH/dgBI/2oAM/+HAEz/4DhR/25ENv+5oy7qyeF8/9T/tf/c/6H/4P+w8djTIP/q3YD/47srKv+OTeLdAEv/ygAr//8XYv/tAEr//wo///+CfP//6df/AJJh/wAjOv8AVFz/ABYg/wAIJf8ABBf/FD4y/y6AWv8Ahij/Re0r/0z/ev/A/7//r/+o/yXeAP9S+VT/DTMgAItbZP9/b0j/bRBE/2YARP/I7McAuv+z/wAA1///TDtCGADT///9b//Y4mf/8f+a/9D/n//q/5//qCqfAN+8M//of2r//5WM/7EAJP/oAFX/JC5zAP9LWv//kJAFW9sZAABxY/8ASl//srGYAAAVQv8ARlP/PB1j/wArav8jVCT/YiqjAADbTf8A41z/FT1tAJT/nv8r5HD/a9MA/1itTv/OSsQAYjBW/4wAXP9ZAir//1pJ/7r/s/8AANb//0s6/xcA0////W//yOpV///wd6nK80r/2+UW/+jTMP/TmwD/8HFX/7oAL//DAGL/9QBy/+oAM//MAHH//zaG//+nmP8A2on/ADda/wBJXP8MAEz/FgtK/wAzO/8oL1D/AItX/wCDPf8AyIv/ALCFbWD/lf9h/6T/Dv9e/z3DUv9gzIv/YWVl/0R6PQBmNE7/bTN5//9YR/+6/7P/AADU//9JOf8XANH///1v//jTY////rX/vO8A/+X/v//wmB7/Lv1yAP1fXf//xFkA/yxQ//9Sb///NGL//35+//+Rlf//4dD/AIOB/wCeev8AAHWbAAB0/wAAc/8AAHL/AGFf/wBnXP8AdF7/AKZI/06cYv9apE3/ANOJ/2G9av90rIL/VdNEWJCMWf+Gmmn/b3pr/4o8gf+SGmD/vmmG/6pobv+rmmX/paSD/76Yi/+syo3/3Pt2/963i//601X//Iqb/7tRe///u5XI/5iV//cAg//TWX/S/4ma//99fv//oYr//7qr/wB0Zf8Am37/AABy/wAAcv8AAHH/AABw/wCXbP8AeWf/AHaJ/xthWvhvHaEATpd//w6bX/8rpHb/AKd7/3GphKaEjJL/eJON/3OgdP9UdX7/hplh/2wuogC8sbH/nXZy/6aWlBjhvHr/nrN5/8SGif/5jHD/6HRgl8B8gavXhTb/4aKQ//+wlf//jaX/9Dlo//9eQv//m57//49k//9wVv8Ag1f/AKlYGBUqwQAAAG7/AABu/wAAbf8AlaD/ALi2bQCUnf9MhYz/UJKI/xl7nf86VomzaGFp/0p0kv9Fel7/X5eY/224mP9tgm7/jKVs/5aqmP98s6f/h2iK/42NgP+5iZNi9qGZ/+mYgf+2iYX/vUN//9+ChP//qcT/16ia/6TjRQD/yHv/7rhU//zBmMf/wr3//8zP//+WjP//nZL/AJB2MgC4k/8A5Yr/AJKM/ybFwv8A7KKwALW+/1qbvP/Py+0AEXZy/0lUof8AO27/NUiIV3drhP85fqTOg2le/5+z4ACYuZP/WuOv/4bl1v+Hz4D/kIV//8yPpP+4oX3/w2aa/8iAjf+zCWr/6Sya//98lf/QP57/8l6i//KBmf//d4j//6an///gz///5dT//+rd//+Tov//1Mb//9HW/wBpfP8AkpD/ALii/6n/yP8AwLn/ALq0/wC5qj0AqJP/AHib/yten5csHm7/gDab/2kAs/9HI8X/f1KT/2qcyP9PlnErn7x9/4jmi/9W6Zr/3f+E/6rUg/99qYX/d7Cp/6iOkf+YVmv/tACx/7EAl/+6AIr/5v+XANpmfv/daLT//+DcmtPaoKn/58v///XFVP/t3v//593//+XN//+bpv8AoMj/ALOZ/wDaxv8A/a65Hv+v/wDhiv8A4dD/AJ2z/yeFuv88UHf/SAVz/zcAn/9PNdT/YACQ/0wAolFEAIJ/TGPA/47Cov9g2pv/zv/i/9L/0v+Xx5n/I31ZAH2Ref+NgIP/xEOb/8YAsCmaAHL/mgCP/68Aev+1AJr/7ViI/+WYcP8YzXwA+P+9///03f//0sj///PY///g3v//tsb/AHif/wCKrP8A8Lv/W//z/57/x5bA/9f/AO2T/wDFnP9VbqhjAEbE/zEAaRYWAIr/QABc/zMAiP9QNH//cn3K/1Oj1P+Q5bz/csnb/8j/z2PY/+P/2v/d/6z/1v+nhaD/zYCg/7owsf+EAIP/hgCN/8QnvP/GALD//zKV/+2Mq///yZz//9zI//T/qP//8s7///fB///t2v//5dT//87G/wB8o/8A1dD/AMrM/63/5f+I/8D/AObL/wD21v8A2d3/AIWn/yxxgf86AJj/LACX/zMAUv8vAHr/aUW+/1SKs/+IZrL/ktqfsWH2yf+Y/9L/z//P/9P/4/+ntsL/vM/W/0e6jgCxG0oArwCj/6YAoP+MAIf//0Ci/81EBADMXqD//7m6///01//8/7v/7P/Df//v3Ez/9eL///PX//+owP8AXqX/AI3G/wDS2P9n/+X/qv/wk7P/5f8Au8//AKa6IzlIyP8JZMX/GwB7/zIAlP9gALH/URiq/18Avf+nZtH/gFu5/03M1v/S/+SEyf/ittP/3f/K/9f/i8DD/6DA6qP7hataxVHM/58Arv/bANX/qgB9/8AAxM7VAJT/0lyw//OT18j/7uH///ra///+vP//9Nr//+/s///v1f//wNA1AJrbfgDE2P8z//v/tf/f/1T/5v8h/6n/AOLV/wCY0/8AptP/AFGu/04A2/86AJ//NjG0/0QAmP9IMJ//41IvAHCRxv9ewNBMbfjBBML/3//T/+7/2f/s/7D/4/+3wd7/kH/T/+VZ+v/NAKb/ngCp/9EA0/+bAJz/00H+//9/s//IjrP/LQHHAP/z3f//8Oz/+//M///06P//1Nj//5DI/wCEwg0Aprz/AOPs/wDQ8v/B/+f/AMzj/wDS7vEAjdv/ajDQAFyB4f9CKsn/Mhy1/2kByv8+ANL/W2///1dgRABEmPB6wB2zAIz/4P+l6fX/q8rR/4qv5v/Jsun/r7j8/7aB2/9hR7P/5gDf/6QAs/+FALH/zwDm/7srvf+6b9H//4XL///u7f//6/W3//Pv/9un2P//7Or//8vK//+stP8AruL/AJbQ/8vgCQAAt/lxAJ+q/wDVy/9MMz0AAIKo/yB05v8AwvD/S37f/0YAxP9FWqb/OSLE/0RX7v9POLL/XZD4/5crrQCg2rn/sL7//2W+vP+l1fP/ssvz/6WD4v91bJwSyJnx/5RKyv+vYfP/+jnG/+4v//+7Y7z/6Ijx//qs8P//qe7I1MP8//+n6///6OT//9Xf///b3///p8rZAKTk/wCdvv8AnNn/AJjo/wDJ8P8A0ef/AGnacyKA4P8Ahuf/ALLn/1tt6/9Qbvr/c3TQ/wB0oP90iO7/nmb//2xwyf9U9pcAerHv/4yg5/96t///lK3a/67J//+JmP//kI3Z/9aa5v/Uj+z/10X//9GY///rZfz/y13X//+03f/wm////6XN//+K5v//rsH//7Tl///R6v80uDMA/9TgbgC97/8Ap9P/AGDX/wCI9P8AeOz/jULvAABd+TsAf+6yJKn//wB+6f9Wj///AL71kVmV2/8hiv//KLu5W3qI61pXxKYAWzvz/2KAwf+XXfn/n2b+u5R78buGkfn/4XHh/9Ha//+vh8T/2pX/3eJR8f/Qtfb/6KLm//SU9f/uZ/7//8re/9yB4f//edn/+pz///+z7m7/U9b//6Xf//9c7P9Dlf//zHTPAABn4f8AYrL/AETX/xAA7f8AQLPPAHXk/wCB1MBwyv//LJb//0KW5P8Avd7/jN3//3if//9ynv//R2ns/5SX+wVgY///kDnk/6Am6/+7JuMAfWj//7WS4f+6nf//sK///+Xd//+Uzef/+Ono//jF3//+kN7/z2v5NP9Q8BLgg///8Wv//1okIgD/I/Be/2Tu//974rL/2vT/JWz//wEAuv8UAOf/GgDb/wAr//8YK///JWD//5flJQBRnP//IcHr/2L/+P+I/+n/cI+AAKfl//+n0v//Ypz//2aj5v9shfj/jmX//1EAu8deAO//jgH7/6hw6P+SQP//qIf//5qz///Q2///8/n/1v/z+//18P//17r+/x+g5gDpvP///4D///9A1v/HALn/8FXW//9x2f//hvb//3/B5j6V//8kfv//GQDz/wUAwiMlANr/NwD1/x8A0f8/jP//HHX//2nD//8Tw/TOquX//8X/8v+77///p9j/VW3g0P96f///XADh/5k52P9gALb/dgHV/6EB/P+bTfn/tH3//9XQ///o//0Or8/v/+b4///0/P857/f//+uq///cwf+ky5X//8lG///bANz//zfw/99R///zZb4A/4f1//LI/xZir///CQDt/x4AyXAbAOFFHwDp/xgA2/8iAMr/F1f//16Z/79Nlv//xP/5/7P+//+y8f//aXoOAJHc//9ss///TTr//4Fz//91Aen/TACQ/4EB5f+ZAeoGrS7/jqZH/v+0lf//mkhVAI9CfgDd+f/F3v/x/+/z/wr39///52/g//+U7//9D///oQHs//EA+f//SvD//zvm//+U/f//xvf/jNH/vDF7//8BM8j/EQDW/yCotgALANP/IQC7/yRC8v9NqP//kdz//771///P+P/kv//6/8X9//+v3///FVJLAJKf//9TUf//bwHzy4AB4/+PAfYUdQHs/2YA2v+qfP//0K3/zN/l///c4///3//p/+vz///i//b/9PX//+7k///3jf//zyn/nO1a///lTf//7Dr/w/8v4v//ot7//7bw/yBs//8Xb///Jzn//y0A1/8gAOH/FQDY/wBT+ElKgf//OID//3eu//855f//lPb//8L+//+x3P//g8b//1yz//91kP//lmT//3wB+WaBAeX/gRP//73hhACXOf//sKr//3dsAwDb7v//7fH//+Ho///t+f//4dL//+rW///zyv//41j//9RU///5BP//1Dj///Z5////g/z/62b///TP/xNVnf//AQD//zpR//8HAPVKFgD1/4XgQQAAS/D/MlP/6mih//9j0///g8j//7Ho//+39v//gtH//5LJ///N7f//bHr//355//9GAPv/bsiyAHEZ//+9Dv//qWr//7gLRQDSzP//xNT/ZOLo///R1///7vb//+7y/6RaoAkAgzMOAPcx/v/AAfo91m3///lp///wbf///4z+//+r+P//1f3/" + }, + { + "name": "clarity_then_dehaze", + "width": 24, + "height": 18, + "steps": [ + { + "kind": "clarity", + "amount": 60, + "size": 12 + }, + { + "kind": "dehaze", + "amount": 50 + } + ], + "rgba": "IY8c/wabAP8uwwD/QfIX/0LcBP867wv/N8cA/0aEG/89hAD/fGMAjX8aH/+HCgn/oCcWKodDAv/BOAD/w3AA/6t+AP+ntwH/ycIA/8HNDf/34AD/4eoA//K8D//1lA//AH0j/xGSAP8Ktgf/H9UA/zbdEf8ezRb/OdIY/2urA7l4eAL/c18dv1slEP+FFxf/xJitAJI9Hj+nLTT/3HMT/8Z+Ef+jphv/1JIA/8biGv/T5C7//8Yj//+5Df//nw7zAIkT/xeWI/8LqwD/DMFsAAugpgA1kg8A3beLACEfoQBkaCb/bkcA/3A+D/+UMzT/hTUX/4JAFv+DQQD/m3oAMqZxAP+0tyj/1bQN/8euHf/11Ap+5MkZ/9yeOeX+oiX/EoEn/yF6K/8YnAX/JMJvAM5muwBOJVwAbo47AG95hwBPk0z/XmEj/1teMP9VURr/fDE4/6EqJQOkOjSTln80/7d/Nf/QkT0burBN/93LM//Dzzj/5LoS//a5Kv//niz/AHks/xmROP8cpkL/aXvmAPI3LACpwzoArffDABOivQBOk0AoaUs+/2VpR/9jWR//kk1I/5lfTf+TYSjemo5E/7x+Qv+9kTT/0bVj/92zSf++mif/9Kw0CP+XNA/5oCH/AIRv/wx5Rf8AlT//nvk3AN15ZACzq1IAft5EAHI1qwBZhFf/cHQ1/2t4Rf9xeiT/d1BM/5tnQ/+Sd0T/sZo2/7lsSv+1iDz/qTzdANyqRv/Jo2X/+JhR//+JPv//hj7/A6pt/wCaVvUWjlH/SF5F/z1zVP9dbHr/RXZH/1h2X/95gG3/XXtZ/4qLSmFvh0H//wAA/wD/AP8AAP//+hQK/x4U+v/18BT/z11c/8dhU//3R0v/5XZc//aAVP//cVaSDYZh/x5Mav8AhWz/dp3NADpnVv8jYHX/Rldw/02HT/9kgmlBb4aZ/9aszgBZjGL//wAA/3pfVAAAAP//+hQK/x4U+v/18BT/zX9N/9BYWf/iZVj/zHNm//B2jP//iFj3A3aF/xFnhP8jS2b/P0l1/yNNd/8oS3P/VmNr/2NrS/9si1r/Sqhe/3uxc/+QsHT//wAA/wD/AP8AAP//+hQK/x4U+v/18BT/qmJz/+w8bP/IJWT/LU6AAP9rZ7nxfoX/AIW8/yZVkP8AEaZ7ABGm//rXHwAAEab/KlCO/19fof9iepz/fpV6/3G1mP9g14r/bNxy/5m+aKWjpYH/wbOR7sF2m/+sa47/q0SBXd9AgP/ZLIP//yiV//dDev//b53/DIic/x55nf8AEab/ABGm/wARpt0AEab/OkSC/0pHrmhUeIyBVamR/3repY9s2H3/ad6V/57Owf+f3oz/lZKt/653qP/GfWn/0DKW/9E4jt3GJa3/+CeX//9Ml//2aZP/AYq6/wBajP8AEab/ABGm/wARpv8AEab/TjemK0RWp/9Dk6//daKd/23BqP+BbZsAnP+o/5zBuv+evbv/vJ+A/7aZmv/RY6z/vzeG/8s1hP/cLLD/3RnQ/9NYov//WrZ9EGSu/wBZtP8JOJ7/Diie/y8lrVY7C7L/DIx/AEBjvP9Zm6urWcK7/3fns/+U/dYAW9LN/5XbxP+Rwce7mY+i/6x/pv+dT5vclTYvAMNaxP/bGJz/7Bq1/9lBxP//StD/EIq//ytdvv8ZQeGsIibL/zA0rf9PT63/PWva/y5xzP9UcctIX4jMLmS41P+Ayqb/kdfH/5i9z/+Wxuz/pK3t/7Nkxf+fU9KImQXBGscp1v/6PcL//1y///Ij0//9U8T/AG2w9kR7t/8mW8n/H0vn/zA/x/8tR7b/SkO+/1xfxP9plJj/VZLE/1+X1v+KuK0DfsD/4JCwyP+8tev/v63g/6SQ1f+qbrT/smLY/8Y/wf/fSM3/71z6//Re4///c9D/AFv0/xVu9v8tcNr/PM3pABxG5v9QQNT/WHDW/2GV94ZQMqcAQoDo/32H9P+ap8n/fJnM/6K05P+Co/X/oJPs/6t1+v+sdcL/vmj5/+lJ1f/UYO3//EXK//NW3yv/hOf/JXfa/wB+5P8Xbu3/KGf9/xJf//88YOH/V4r8/0mN7f9Xh/7/YoL+/2h5//+Bgu//k238/wspgwB4gP//iWn3/5V68v+zc97/2WztG81o///hdv///FjrlfZt///3YuT/CIbi/xqc//8Ae+n/JqP//4xzWgBGeP9WUXr//1GF//9iiv//aVr//3Zh//9pgf3/mHf//5+H+MGbgfb/p47//6h29f/Mg///zo///8+N///oj///7YX//9aE9v/yj///", + "expected": "CJMB/waeAP8xzQD/Yv9H/0bqAP9J/yb/OtAA/0CFAP89hAD/e2IAjXsAFP9+AAL/ohUBKoNAAP/BOAD/x3MA/66BAP+wwQD/1c4A/83aAP//73T/8/0A///HH///mxj/AHwj/xKVAP8CvQD/IeAA/zTrAP8M2AD/M98A/2+xALl5eQD/dF0Lv1MYAP+AAAn/xJitAJM1Dj+qHiz/5nYA/86BAP+qrQD/3JgA/9XzAP/k9hv//9Z2///IY///rUzzAIsT/wCaGv8NsAD/DMFsAAugpgA1kg8A3beLACEfoQBiZhn/aEMA/2s3AP+VKS3/hCsE/4E4Af98PQD/nXwAMqdyAP+/whH/4r8A/9K2AP//411+9dcA/+mkLeX/sFX/AIEh/wt6If8WoAD/JMJvAM5muwBOJVwAbo47AG95hwBLl0j/W14V/1hbJ/9QSwf/eSYw/6MaGQOnMCyTmoEq/7+BK//cljUbxblI/+3ZJP/R3iz/88UA///HVP//rVr/AHcr/wCVNP8ArT//aXvmAPI3LACpwzoArffDABOivQBLlzooZkU3/2RoQv9gVQ//lEhE/55eSv+WXxveoJI+/8WAPP/Hlin/4MBk/+2+RP/IoA///7lKCP+mXA//rEX/AIRv/wB4Q/8AmED/nvk3AN15ZACzq1IAft5EAHI1qwBXh1X/cHQs/2t5QP9yexT/dUtH/59mPv+WeD7/u6Et/8FrRf++jDT/qTzdAOy0Qf/Vq2X//6Vl//+YXP//lVr/ALFy/wCdWPUAkVD/Q1tA/zZzUf9bbHv/P3ZC/1V2Xf97g27/WnxW/46PRmFviTv//ygd/4b/fv8AAPT//xsQ/x0A9v///Xf/2Vpa/9FfUP//SEz/83hb//+JXv//gGeSAIZh/w9GZ/8Ahm3/dp3NADNmU/8UXXT/QFJu/0qLTf9jhWlBcIme/9aszgBWj2H//yYb/3pfVAAAAPX//xkO/x0A9////Xf/2YNK/9pVWP/uY1b/1nRm//95kv//mXH3AHWE/wBlg/8XRWP/OUR0/xdIdv8eRXH/VGJr/2JqR/9tj1n/RK9d/3+7dv+WuXf//yMZ/4H/ef8BAPX//xwR/x0A9////Xz/sGF0//kwbf/PB2L/LU6AAP97dbn/hIz/AIa+/xtRkv8BAJp7AQCa//rXHwABAJn/IUuQ/15dpv9he6H/gZp8/3K/nv9e5pD/but0/6DJaaWrrYX/zb2X7st5ov+zbJP/sD2DXes2g//kFob//0Ga//9Hfv//gqf/AIme/wF6ov8BAJv/AQCb/wEAmt0BAJr/ND6C/0ZAsmhReY+BUbGW/37vr49t54L/au6d/6fdzv+o7pP/mpe1/7Z5sP/RgGr/2iOb/9ssk93OB7T//y6c//9fnv//cZv/AIy9/wBWh/8BAJr/AQCa/wEAmv8BAJr/Sy2qKz9Sq/89mLj/dqmj/27Nsf+BbZsAzf/R/6TNxf+mycf/x6eE/8Cgof/dY7X/xyyI/9QoiP/oFbj/5gDY/91UqP//brt9AGGu/wBWr/8AMJv/CRSf/ywMsFY3AKn/DIx/ADtgw/9XorSrVs7G/3v6wP+U/dYAWOHa/53s0v+XzNO7n5Sp/7SCrf+hSp/clTYvAM1Yz//iAKH/9wC9/+Q2zv//YNP/AI3E/yJZxf8TNOusIQPS/ywnsf9NSrP/OWrl/yNw1f9ScdVIXovWLmPC4f+E16//mejW/5/I3P+e1P7/rbf+/7tjz/+lTt2ImADAGtEP4v//Tcj//3DF//4A3f//ZMj/AGut9j58v/8cV9H/GELy/yw0zv8nP7v/RzrE/1pby/9qmp7/UpfO/12c4/+Pw7YDlsv/4Ja61P/IwP3/zLjw/62W4/+ycL3/u2Hk/9A1y//sQNn/9Wr///9i7f//h9X/AFrx/wNt//8jb+X/PM3pABU68P9ONNz/Vm/h/2ec/4ZQMqcAPYL2/4KM//+isNb/f5/Y/6y/9P+KrP//qJn8/7B8//+0dsz/xG////dB4f/gXfz//1XO//9U6iv/mur/Enfk/wB/5v8AbPf/KGj//xNg//84W+v/X5H//0WR/P9fjv//aYn//21///+Fhf7/l3L//wspgwB+h///jGv//5t8//+8dOz/52z9G9B1///kh////2fulfh/////a+z/AInn/zqk//8Afez/Rav//4xzWgBLff9WVn///1mL//9skv//alz//3hl//9uhv//nX///6WO/8Ghh///rpn//697///Rkf//1aD//9Sb///sof//8Jn//96L///1o///" + }, + { + "name": "flat_zero_channel_clarity", + "width": 6, + "height": 5, + "steps": [ + { + "kind": "clarity", + "amount": 100, + "size": 40 + } + ], + "rgba": "ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/", + "expected": "ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/ABGm/wARpv8AEab/" + }, + { + "name": "flat_pure_blue_dehaze", + "width": 6, + "height": 5, + "steps": [ + { + "kind": "dehaze", + "amount": 50 + } + ], + "rgba": "AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//", + "expected": "AAD8/wAA/P8AAPz/AAD8/wAA/P8AAPz/AAD8/wAA/P8AAPz/AAD8/wAA/P8AAPz/AAD8/wAA/P8AAPz/AAD8/wAA/P8AAPz/AAD8/wAA/P8AAPz/AAD8/wAA/P8AAPz/AAD8/wAA/P8AAPz/AAD8/wAA/P8AAPz/" + }, + { + "name": "flat_pure_red_dehaze", + "width": 6, + "height": 5, + "steps": [ + { + "kind": "dehaze", + "amount": 120 + } + ], + "rgba": "/wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD//wAA//8AAP//AAD/", + "expected": "/z4w//8+MP//PjD//z4w//8+MP//PjD//z4w//8+MP//PjD//z4w//8+MP//PjD//z4w//8+MP//PjD//z4w//8+MP//PjD//z4w//8+MP//PjD//z4w//8+MP//PjD//z4w//8+MP//PjD//z4w//8+MP//PjD/" + }, + { + "name": "cropped_alpha_bounds", + "width": 31, + "height": 22, + "steps": [ + { + "kind": "clarity", + "amount": 90, + "size": 15 + }, + { + "kind": "dehaze", + "amount": 60 + } + ], + "rgba": "eGeXAKzMxwDQBpcA9dQiAG+V1QAaI3MAe/xWAAifdQDzFPwA1HYYAC3N3AB/8nUAJsfjAG6gHgAoNBUARQ4HAKr6/gAuxP4A6B8eAA3nRACnjBYAOsiqAA3thwArt/oAbtUDAC/MSAAxDtYAWj/FADZR8wDMyOUAITogAEOhrQCNbiAAsxT5AJc5pQDdnL0AGWI7AKp4UQDUbC8Agym5AGMioQCwFhgARcuwAONEOwDSCd4Ac4EeAK13LgAjFWEA5FwKADEy2QBcVh8A0EReALJOZQCho8wAL/kFALAbmwD6C1IA91AdACdetgA0yHYAUnY3ACb2UQCaiRUAaL91AKdbwADtE1oAj4aHAFX8zgAFvDMAOOriAGYZAACSCdEAoDxdAKlV+gAS+SYAh8AWAAQ+6wClDVEA0me+AHCT5wBZIU0AY/G6AIgHawCrAUMAlbxWAPzFGgAPf1kAfB3hAK9imgAfl/IAjRRsAF4cfwC0x04A5V27ACRfwADjS2cAazzLAM6HogCSY+0ABXUH/ya/AP8A4QD/GuQF/03QEP9H+xasUMAB/1O2AP9rZgD7bloA/45QGP+QJgb/pRgA/6tNAP+qSgCk4I8A/2y3OwDotgD//7ob//j9AP/yD+AAwTOxAJJZ7gBjEmUAj7DQAKPMFQChvfcAdA5+AEcSRgDEoxcAuTeAABaFH/8Auh//MdYG/wTfGf9G5TD/VuQr/2HJAYxXkxj/0O6IAItLDP+VYBT/mUQA/5ggFf+/MyX/rUMd/8JmAP/JkRb/5bgkAcvdLDv6vQD/QR1DAOuLXQB9g6kA7MS+ADb/mwAyXVoALwBWAL46WQD/UyMA+j78ADa86QAAiij/Das41grAIv/WERwA1jd3AFzpswBCvgYAGoneAI2GBv98VRP/aFhBdooUNP+nRyP/BKCoALAaF//Ybin/0o0g/+icCv//yRX//+VK/2rZSgDY5r0ApQrWAI+lcQB8N0wA0nVmAN30GgDILwYAWE5qANWZRgBX2PoAG4xJ/webLv8dxD//mBGIAILsWAADXf0AiwGDAA6ilwBwcir/b2pW/4xkOLiYTVf/vTFD/7E0HP/DRTz/22w5/9NxKf/jnCH/47xH//+7O//hWksAjU+3ACUZFgDzpf8AiJhdAHoH5ACrMH0ABno1AIUilQBswV4AZUheAACHZf8SpEf/8czkANnxZQARGO0AY8aNAF2vAQCL2NUAZIFN/2yPR///AAD/AP8A/2MNcAD6FAr/HhT6//XwFGPObEj/54tJ5uZ/TZ38oVT//LhPAM3vZACBSLQASU4uAC1H8wA2l6oAOMwqANDLfwCFyy8Aq0U8AN7nYwAAaWL/I45f/ziTdv8Pr6EAo+4wADyidwBUCGEAOhs7AGtsYf+HZH3//wAA/wD/AP8AAP//+hQK/x4U+v/18BRd7pNd//WJYP/soYX/+Jdi/9Vg9wB/2vwAxRnZAOZTLAAjjJ0ArqGWAAscyQAfESsAqGR5APmtLwCkiSoAFoxv/w+Bfv8lX4L/N4+M/0xxb/9hbmb/UkqV/3dvlv9WTGL/h2Zt//8AAP8A/wD/AAD/kvoUCv8eFPr/9fAU/yIe6ABbJgEA1p14//9nfCotC5MAzrh6AKEC1ADsLcIAUKFZAOmN0gCuk6cAQ2HtAMsZHwCa6EMA5WbTABaDif8Aenf/ABGmZwARpv8AEaaGABGm/1BrZ/9GXbb/YXiK/62PlP9jm5H/kL2M/07EBgC9t3f/pMF/CseLeP+6ipH/4GOo/+5Lm//yZp7/hVb+AOcYZADGiU0A64hxAF+0DwDf8uMA/oEpAPCZCwC8zi4AFuuQAMYBMwAAhar/Wn8jAAARpv8AEab/ABGm/wARpv9NR6z/WXqG/02IuP+EqoH/gKGU/4qwn/+GqpH/yp69/9GymP/Ll5CN2Iye//Jeqf/RQcT//zp6VVjH+QBkcREAzhoZACNThQDKZxUA4FLCAEPtngCZ4kYA7b7/AI9kNwBiehUAAIOq/xNMvBQAEabHABGm/wARpv8AEab/Z2TT/05tn+CJhrr/g7Cz/5XNlv/RWpMAucPGxMHYtf+9rdb/xY69/+pWof/9a6f//0XK//cjtf9peu4AMN8zAOEAswAZsU8AyK/5AN9VygC8cNQAq47HAG/NRwDPxYAAmIFHAABkuP8WfzsAPj3H/8fdQgAtEM//Pha6/1ZL0/9RWdX/l5C9/4C/vf99xbv/nefA/5XP3f+c7NqqprTP/7KI4P/vfcH/4VK3//0juiT/EL//SZKeAJSQlAAUwGcAZOd2AJFglwClGtcAo4n2AB4QdwBgwSkAjOIlAPm8NwAMluEEBmS//yAx4/8PJtT/GxrY/0Mb8P9BP8//UFWu/2ee2v9OstcPks/R/4TO3v+Q2/P/stnU/67G7f/Imf//8XrmD+dhyf/1KMX//zLp/6Z55wAr/x0AmL0FALb1ygBOIxoAm0CMAPMwaQAvOIcA5ZoDACYAjQAPVukAAIHY/ylf//8AOPP/QkXu/0oj7/85FOL/XSns/5EiKgBiluf/oKX+/2jVPQCN5dT/tdzr/7nk///TteIasJXc/8iB3P/5S99k60r1//8N7f+6wLIAxs1jAAlP6QC4MdcAcN3wAOmFlwDvjO8ACoHwAF1OEgBABHsA2WIGAACI//8cdf//+L0yADZK//8wFv//Wir//2JljgB0Zv93YoLR/6Cd//+Oruv/kL3//67N4/+3xP7/+TpQAMd29//kcu7//3n//wZ5qgDyPP7/6orIAODaogB4TUUA3sQbAN82YgD16ZMAjBxXAGm8PACbskoA5V4OALPDsABTQxYAz6qjAJVZGAAMc8EAIlkEAP05wwDSKBUAfw3jADOTEAB670AAouqCABsUlQBxZRoAsSlhAGGRBAAqWuEAqWEbAIsThwA1DuIAL5kMAJwO0ADybq8Ax/cNAEvi/wCzjdIAr5cxAOOOfwCUAW4AbQYwANb1QgD15YIAwSJdANbyUgCZEjAAz1tzAAWmNAAzn1oAroXsAAqO5AA845UAJteBANhT3gBJAEsAOE2yAJ7FkgALhLIAG5MPAAKe/QCdafgANZNvAGNFpADOcGIAi+l+ALq4FgAJ6kwAAfdLAA4qGQAQJ+IAo7YrAI4lMQDy+CIAPD/qAG0bSQAODYAAn9B8AAbdMwB8QIoAIXbqAPWArwCBNr4AIlLlAA9rGQDSW44A34KRAESqsQADquQAtoUjAA/rmgB2qy0ABkBvAKxLggDaqqQAmQdgABfJfgBVBisA/GO3ANdErAAG8K8A3tEyAP+jZwDvBmkABwYaAPVTkAC7DrwAbzVbAPk79gDqrOIA5S+EADcgAADAzJYAW6biALERNAA/NHsAT7RbALrJrQC+AuUAvr0cAM+gsABzePsABN3vAMsivwDfc9oARlJMAFofgwBndDYAI1FSAC9WqwDKnNYATxC5AE+MBADOCbUABbexAGEuOQBQkd4AoqCLABHrUQC9j3EAUo4UAIeb9ABuuqQAJkyyAK6IIgBFfKIAH21tAJSNMwBqSdoAEmjwAKCHGAA69nYACSM4AA5pNwB+pMQA1gSYAA8/wADMVjEAM2QEAGl8TgBLKEoAAeoOAA==", + "expected": "eGeXAKzMxwDQBpcA9dQiAG+V1QAaI3MAe/xWAAifdQDzFPwA1HYYAC3N3AB/8nUAJsfjAG6gHgAoNBUARQ4HAKr6/gAuxP4A6B8eAA3nRACnjBYAOsiqAA3thwArt/oAbtUDAC/MSAAxDtYAWj/FADZR8wDMyOUAITogAEOhrQCNbiAAsxT5AJc5pQDdnL0AGWI7AKp4UQDUbC8Agym5AGMioQCwFhgARcuwAONEOwDSCd4Ac4EeAK13LgAjFWEA5FwKADEy2QBcVh8A0EReALJOZQCho8wAL/kFALAbmwD6C1IA91AdACdetgA0yHYAUnY3ACb2UQCaiRUAaL91AKdbwADtE1oAj4aHAFX8zgAFvDMAOOriAGYZAACSCdEAoDxdAKlV+gAS+SYAh8AWAAQ+6wClDVEA0me+AHCT5wBZIU0AY/G6AIgHawCrAUMAlbxWAPzFGgAPf1kAfB3hAK9imgAfl/IAjRRsAF4cfwC0x04A5V27ACRfwADjS2cAazzLAM6HogCSY+0AAHAD/yjIAP8A8QD/GfQA/07bAP+P/3ysU8gA/1fAAP9pZAD7alcA/5FMAP+HIAD/nhYA/61OAP+rSgCk7pgA/2y3OwD4wgD//8xw//n/if/yD+AAwTOxAJJZ7gBjEmUAj7DQAKPMFQChvfcAdA5+AEcSRgDEoxcAuTeAAACFFP8AwiD/M+MA/wDtGf859xb/UPcC/2bUAIxVlgD/0O6IAIlHAP+aYQD/l0MA/5YGAP/JJBf/tDsG/8lpAP/UmAD/98QAAdzwBTv/zV//QR1DAOuLXQB9g6kA7MS+ADb/mwAyXVoALwBWAL46WQD/UyMA+j78ADa86QAAiSj/ALA31gDJIf/WERwA1jd3AFzpswBCvgYAGoneAJKKAP97UQD/ZlQ6doQALP+uQRP/BKCoALMACP/nbxb/35IA//WlAP//23////Ou/2rZSgDY5r0ApQrWAI+lcQB8N0wA0nVmAN30GgDILwYAWE5qANWZRgBX2PoAAI5G/wCdLf8Azzz/mBGIAILsWAADXf0AiwGDAA6ilwBxcxz/cGpT/5FkMbidSFX/xiE+/7gnBf/OPTb/6mwv/+BxE//zpAD/9sk+///Of//hWksAjU+3ACUZFgDzpf8AiJhdAHoH5ACrMH0ABno1AIUilQBswV4AZUheAACHZf8AqUf/8czkANnxZQARGO0AY8aNAF2vAQCL2NUAY4NI/22UQf//MCP/kv+K/2MNcAD/KBv/HgD7///9g2Pba0L/949C5vWBR53/tHj//LhPAM3vZACBSLQASU4uAC1H8wA2l6oAOMwqANDLfwCFyy8Aq0U8AN7nYwAAZF7/AJJd/yqXd/8Pr6EAo+4wADyidwBUCGEAOhs7AGtsYP+KY3///ywg/4z/hP8BAPj//yUZ/x4A/P///YJd/5xh//+UbP//rI7//6d5/9Vg9wB/2vwAxRnZAOZTLAAjjJ0ArqGWAAscyQAfESsAqGR5APmtLwCkiSoAAI1w/wCBfv8UW4P/KpWR/0hycP9gb2b/TkOX/3lwm/9RRl7/imVt//8tIf+K/4H/AQD4kv8mGv8eAP3///6F/yIe6ABbJgEA56d9//95iSotC5MAzrh6AKEC1ADsLcIAUKFZAOmN0gCuk6cAQ2HtAMsZHwCa6EMA5WbTAACEi/8AeXb/AQCXZwEAl/8BAJeGAQCX/0xrZv9BWb3/YHqO/7iWnP9iopf/l8qS/07EBgDKw3r/rs+DCtWRfP/HkZn/72Kx//5Cov//baf/hVb+AOcYZADGiU0A64hxAF+0DwDf8uMA/oEpAPCZCwC8zi4AFuuQAMYBMwAAhqv/Wn8jAAEAl/8BAJf/AQCX/wEAl/9KP7L/V3yK/0mNwv+JtIb/hKqb/5C7qP+Ks5f/2KbK/+G+oP/anpaN6ZSo//9ls//dNND//1KDVVjH+QBkcREAzhoZACNThQDKZxUA4FLCAEPtngCZ4kYA7b7/AI9kNwBiehUAAISq/wBEwBQBAJbHAQCX/wEAl/8BAJf/aGPh/0pspeCPi8T/iLu+/53dnv/RWpMAxtHVxNHrw//Jt+b/0pTJ//xRqv//grL//2XO//8zvP9peu4AMN8zAOEAswAZsU8AyK/5AN9VygC8cNQAq47HAG/NRwDPxYAAmIFHAABgs/8WfzsAOi7P/8fdQgApAMX/OQCz/1ZC3/9PU+H/npXI/4TMyv9/0sf/qP3Q/5zf7/+x/+2qsL/e/72N8P//hc7/8k7D//9GvyT/Q8P/SZKeAJSQlAAUwGcAZOd2AJFglwClGtcAo4n2AB4QdwBgwSkAjOIlAPm8NwAAmugEAGG7/x8K7f8OANX/GgDR/0IA8f8+MNj/TE+0/2al6f9Gu+UPmd/h/4jd7/+p6v//v+vm/7rU///PqP///4f0D/ph2f//M83//13q/6Z55wAr/x0AmL0FALb1ygBOIxoAm0CMAPMwaQAvOIcA5ZoDACYAjQAPVukAAIHX/ydd//8BN+X/QDb7/0kA8/82ANr/XgD3/5EiKgBinPj/q7D//2jVPQCV+uf/x/D//97y///iwfMauZvs/9aG7f//YOZk9Vn///9M7f+6wLIAxs1jAAlP6QC4MdcAcN3wAOmFlwDvjO8ACoHwAF1OEgBABHsA2WIGAA6K//8ddf//+L0yADVI//8vAPv/WSj//2JljgB4bP93YYTd/6mn//+Stvv/osj//7nc9f/I0///+TpQAM59///0d////5r+/wZ5qgD2Xv//6orIAODaogB4TUUA3sQbAN82YgD16ZMAjBxXAGm8PACbskoA5V4OALPDsABTQxYAz6qjAJVZGAAMc8EAIlkEAP05wwDSKBUAfw3jADOTEAB670AAouqCABsUlQBxZRoAsSlhAGGRBAAqWuEAqWEbAIsThwA1DuIAL5kMAJwO0ADybq8Ax/cNAEvi/wCzjdIAr5cxAOOOfwCUAW4AbQYwANb1QgD15YIAwSJdANbyUgCZEjAAz1tzAAWmNAAzn1oAroXsAAqO5AA845UAJteBANhT3gBJAEsAOE2yAJ7FkgALhLIAG5MPAAKe/QCdafgANZNvAGNFpADOcGIAi+l+ALq4FgAJ6kwAAfdLAA4qGQAQJ+IAo7YrAI4lMQDy+CIAPD/qAG0bSQAODYAAn9B8AAbdMwB8QIoAIXbqAPWArwCBNr4AIlLlAA9rGQDSW44A34KRAESqsQADquQAtoUjAA/rmgB2qy0ABkBvAKxLggDaqqQAmQdgABfJfgBVBisA/GO3ANdErAAG8K8A3tEyAP+jZwDvBmkABwYaAPVTkAC7DrwAbzVbAPk79gDqrOIA5S+EADcgAADAzJYAW6biALERNAA/NHsAT7RbALrJrQC+AuUAvr0cAM+gsABzePsABN3vAMsivwDfc9oARlJMAFofgwBndDYAI1FSAC9WqwDKnNYATxC5AE+MBADOCbUABbexAGEuOQBQkd4AoqCLABHrUQC9j3EAUo4UAIeb9ABuuqQAJkyyAK6IIgBFfKIAH21tAJSNMwBqSdoAEmjwAKCHGAA69nYACSM4AA5pNwB+pMQA1gSYAA8/wADMVjEAM2QEAGl8TgBLKEoAAeoOAA==" + }, + { + "name": "single_row_alpha_bounds", + "width": 19, + "height": 7, + "steps": [ + { + "kind": "clarity", + "amount": 150, + "size": 8 + }, + { + "kind": "texture", + "amount": 120, + "size": 4 + } + ], + "rgba": "R3iOAMyx6gAlb80A46TIADN5AgAVsrkAQnTAAL8xsgAiAekAP+VCAMBJSQCCugIAjMWhAP01aABY0ycApgyBAM5fsQBtWZQApxTzAPeoKADjTTAAOMWuAEqHAADolYcA4dpaAKqykQAIsGkA+jLnAFWy8wBD/IoAmirRAO+gagDWmF0AI4XEAHFyvgAizCUAQbhgAPtfagA1AZsAy4nAAKK26gDjqsAAc4rWAJF1ywAGQekAZbvfAJtxKgAlVNoAxkIdACX5jgA/VfEAyYT7AIUSkwBo50kAmxUkABLQ0QCidK4ARcprAFgXEgDjQVgAJAvEADL3OQDpIoAAMKnuAG2hLACq2O0AzI9KAAHrwwBMpv8AeWuxABmZQQB261kAV+25AMVfvwAh0WgAmOyKAPTz4wCcz3cAAH4C/xqkAP8ozgD/VN0O/yLxAP9CuQP/V6sA/261AP+hdAX/nUoA/9s/AP/IAAD/1iAA/+dMCP/8KRT/59BLAPtOVAA2cj0A4zARAAqcPADDYL4Ajq5pAK6q0QCKU+MA1K52AMhVqAAG+lcATAtVAP5S1gBz+o4ANbD1AA+qVgDbdwoAr0hgAHoUXgBNkpsA53HqAHBcvQApanUAqihpAPoDvQDMhVEAlQFaAKJu4gBO+cgA0VogANEuQgAZQGgAjlsmAFQbiQCzHgwA79/EACRysQBHsFUAZnI0AA==", + "expected": "R3iOAMyx6gAlb80A46TIADN5AgAVsrkAQnTAAL8xsgAiAekAP+VCAMBJSQCCugIAjMWhAP01aABY0ycApgyBAM5fsQBtWZQApxTzAPeoKADjTTAAOMWuAEqHAADolYcA4dpaAKqykQAIsGkA+jLnAFWy8wBD/IoAmirRAO+gagDWmF0AI4XEAHFyvgAizCUAQbhgAPtfagA1AZsAy4nAAKK26gDjqsAAc4rWAJF1ywAGQekAZbvfAJtxKgAlVNoAxkIdACX5jgA/VfEAyYT7AIUSkwBo50kAmxUkABLQ0QCidK4ARcprAFgXEgDjQVgAJAvEADL3OQDpIoAAMKnuAG2hLACq2O0AzI9KAAHrwwBMpv8AeWuxABmZQQB261kAV+25AMVfvwAh0WgAmOyKAPTz4wCcz3cAAH4C/x6mBv8s0Av/V98W/yTxBP9JwBf/XLES/261AP+dcgD/nksD/8k5AP/KBQP/1CEB/9tGAP/yGAD/59BLAPtOVAA2cj0A4zARAAqcPADDYL4Ajq5pAK6q0QCKU+MA1K52AMhVqAAG+lcATAtVAP5S1gBz+o4ANbD1AA+qVgDbdwoAr0hgAHoUXgBNkpsA53HqAHBcvQApanUAqihpAPoDvQDMhVEAlQFaAKJu4gBO+cgA0VogANEuQgAZQGgAjlsmAFQbiQCzHgwA79/EACRysQBHsFUAZnI0AA==" + }, + { + "name": "single_column_alpha_bounds", + "width": 6, + "height": 16, + "steps": [ + { + "kind": "dehaze", + "amount": 80 + }, + { + "kind": "texture", + "amount": -90, + "size": 3 + } + ], + "rgba": "WVwlACwLZwBfOIIA7ZaVAHBlgQBg6BwAGKCaAGBf+gAFxKIAAHcA/7ORAQAtER0Af64hABk5JwAwQ3QAAJYp/xSUCwCqycEAD5dGACeZjQD2pNEAIYRT/02tOQAWahMAytcxAOJf+gAcjiEAFZcy/8TRCAAWjDwAWWBeAP3yRAAdvkMAAJtX/xBhjAAzVhcA0ONiAE83TADHuLsAAY5O/wa7XQDbZlMAOk5iAI5gbQAk9xAAPWOdACTgCgCCWqYAHaPTADHw2gDVRr4AAFZ0/1sHZAAvGiYA8ixhAAyUtAC89lIAAHq+/2vxkABpYywAetZyAGDZ5wBuxEAAAG25/204ggBfI/cAO1k6AE1ndAC4GOwADV/K/9yN/AAgKtQAn09gALHDGgDycSQAAHzv/7mPJABQMQwAg6Z7AIg2UABg2GkABojq/6bR8ACAxsMAoGY4AEKWNwA9Ub8AR6+yANfNFwBABV4AZV6VAHoPqQCpdcIAvIobANqkygDX4ekA", + "expected": "WVwlACwLZwBfOIIA7ZaVAHBlgQBg6BwAGKCaAGBf+gAFxKIABm4F/7ORAQAtER0Af64hABk5JwAwQ3QAA6At/xSUCwCqycEAD5dGACeZjQD2pNEAAIJN/02tOQAWahMAytcxAOJf+gAcjiEAAKEw/8TRCAAWjDwAWWBeAP3yRAAdvkMABqhf/xBhjAAzVhcA0ONiAE83TADHuLsAAI9P/wa7XQDbZlMAOk5iAI5gbQAk9xAAPWOdACTgCgCCWqYAHaPTADHw2gDVRr4AA0tl/1sHZAAvGiYA8ixhAAyUtAC89lIAB4HI/2vxkABpYywAetZyAGDZ5wBuxEAAAW25/204ggBfI/cAO1k6AE1ndAC4GOwAAFnF/9yN/AAgKtQAn09gALHDGgDycSQAAIP8/7mPJABQMQwAg6Z7AIg2UABg2GkABJT//6bR8ACAxsMAoGY4AEKWNwA9Ub8AR6+yANfNFwBABV4AZV6VAHoPqQCpdcIAvIobANqkygDX4ekA" + } + ] +} diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/alphaFill.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/alphaFill.test.ts new file mode 100644 index 00000000000..e4fe80d2f22 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/alphaFill.test.ts @@ -0,0 +1,27 @@ +import { propagateNearestVisibleLabels } from '@workbench/canvas-engine/render/spatial/alphaFill'; +import { describe, expect, it } from 'vitest'; + +import parity from './__fixtures__/spatial_adjustment_parity.json'; + +const bytes = (text: string): Uint8Array => Uint8Array.from(atob(text), (char) => char.charCodeAt(0)); + +describe('propagateNearestVisibleLabels', () => { + it.each(parity.labels)('picks the visible pixel OpenCV labels, ties included, for $name', (entry) => { + const { height, width } = entry; + const hidden = bytes(entry.hidden); + const labels = new Int32Array(width * height); + const visibleRank = new Int32Array(width * height); + let rank = 0; + hidden.forEach((isHidden, index) => { + labels[index] = isHidden ? -1 : index; + visibleRank[index] = isHidden ? -1 : rank++; + }); + + propagateNearestVisibleLabels(labels, width, height, new Uint32Array(width * height)); + + const expected = new DataView(bytes(entry.labels).buffer); + expect(Array.from(labels, (label) => visibleRank[label])).toEqual( + Array.from({ length: width * height }, (_, index) => expected.getInt32(index * 4, true)) + ); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/alphaFill.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/alphaFill.ts new file mode 100644 index 00000000000..3c226f75d92 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/alphaFill.ts @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Nearest-visible-pixel labels with the exact assignment the backend gets from + * `cv2.distanceTransformWithLabels(hidden, DIST_L2, 5, labelType=DIST_LABEL_PIXEL)`. With labels OpenCV runs its + * two-pass 5x5 chamfer transform in 16.16 fixed point rather than an exact Euclidean transform, so ties and the + * chamfer's approximation decide which visible color a hidden pixel borrows; matching them keeps the band filters' + * input, and so the visible result, identical to the node's. + */ + +// OpenCV's L2 5x5 mask weights 1, 1.4 and 2.1969, rounded to 16.16 fixed point. +const ORTHOGONAL = 65536; +const DIAGONAL = 91750; +const KNIGHT = 143976; +/** OpenCV's initial and out-of-image distance; adding any weight to it can never win a strict comparison. */ +const UNREACHED = 0xffffffff - KNIGHT; + +// Forward-pass neighbours in OpenCV's comparison order. The backward pass visits their mirror images in the same +// order. A neighbour replaces the current best only when strictly closer, so ties keep the earlier one. +const NEIGHBOUR_DY = [-2, -2, -1, -1, -1, -1, -1, 0] as const; +const NEIGHBOUR_DX = [-1, 1, -2, -1, 0, 1, 2, -1] as const; +const NEIGHBOUR_WEIGHT = [KNIGHT, KNIGHT, KNIGHT, DIAGONAL, ORTHOGONAL, DIAGONAL, KNIGHT, ORTHOGONAL] as const; + +/** + * Resolves `labels` in place. On entry a visible pixel holds its own index and a hidden pixel a negative value; on + * return every hidden pixel holds the index of the visible pixel whose color it takes. Needs at least one visible + * pixel. `distances` is caller-owned scratch of at least `width * height` entries. + */ +export const propagateNearestVisibleLabels = ( + labels: Int32Array, + width: number, + height: number, + distances: Uint32Array +): void => { + for (let y = 0; y < height; y++) { + const interiorRow = y >= 2; + for (let x = 0; x < width; x++) { + const index = y * width + x; + if (labels[index] >= 0) { + distances[index] = 0; + continue; + } + const interior = interiorRow && x >= 2 && x < width - 2; + let best = UNREACHED; + let bestLabel = -1; + for (let k = 0; k < 8; k++) { + const nx = x + NEIGHBOUR_DX[k]; + if (!interior && (y + NEIGHBOUR_DY[k] < 0 || nx < 0 || nx >= width)) { + continue; + } + const neighbour = index + NEIGHBOUR_DY[k] * width + NEIGHBOUR_DX[k]; + const distance = distances[neighbour] + NEIGHBOUR_WEIGHT[k]; + if (distance < best) { + best = distance; + bestLabel = labels[neighbour]; + } + } + distances[index] = best; + labels[index] = bestLabel; + } + } + + for (let y = height - 1; y >= 0; y--) { + const interiorRow = y < height - 2; + for (let x = width - 1; x >= 0; x--) { + const index = y * width + x; + let best = distances[index]; + if (best <= ORTHOGONAL) { + continue; + } + const interior = interiorRow && x >= 2 && x < width - 2; + let bestLabel = labels[index]; + for (let k = 0; k < 8; k++) { + const nx = x - NEIGHBOUR_DX[k]; + if (!interior && (y - NEIGHBOUR_DY[k] >= height || nx < 0 || nx >= width)) { + continue; + } + const neighbour = index - NEIGHBOUR_DY[k] * width - NEIGHBOUR_DX[k]; + const distance = distances[neighbour] + NEIGHBOUR_WEIGHT[k]; + if (distance < best) { + best = distance; + bestLabel = labels[neighbour]; + } + } + distances[index] = best; + labels[index] = bestLabel; + } + } +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/evaluateSpatialSteps.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/evaluateSpatialSteps.test.ts new file mode 100644 index 00000000000..b4c95dad2eb --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/evaluateSpatialSteps.test.ts @@ -0,0 +1,153 @@ +import type { SpatialAdjustmentStep } from '@workbench/canvas-engine/render/spatial/spatialSteps'; + +import { evaluateSpatialSteps } from '@workbench/canvas-engine/render/spatial/evaluateSpatialSteps'; +import { + MAX_SPATIAL_PIXELS, + SPATIAL_SCRATCH_BYTES_PER_PIXEL, +} from '@workbench/canvas-engine/render/spatial/spatialSteps'; +import { describe, expect, it, vi } from 'vitest'; + +import parity from './__fixtures__/spatial_adjustment_parity.json'; + +const decode = (text: string): Uint8ClampedArray => Uint8ClampedArray.from(atob(text), (char) => char.charCodeAt(0)); + +const cases = parity.adjustments.map((entry) => ({ + ...entry, + steps: entry.steps as SpatialAdjustmentStep[], +})); + +/** + * Counts output channels off the contract: visible color equals the Python node's output byte for byte, while alpha + * and the color of fully transparent pixels stay as they came in. + */ +const compareWithPython = (actual: Uint8ClampedArray, input: Uint8ClampedArray, expected: Uint8ClampedArray) => { + let alphaChanged = 0; + let hiddenChanged = 0; + let visibleDiffering = 0; + for (let p = 0; p < actual.length; p += 4) { + if (actual[p + 3] !== input[p + 3]) { + alphaChanged++; + } + for (let c = 0; c < 3; c++) { + if (input[p + 3] === 0) { + hiddenChanged += actual[p + c] === input[p + c] ? 0 : 1; + } else { + visibleDiffering += actual[p + c] === expected[p + c] ? 0 : 1; + } + } + } + return { alphaChanged, hiddenChanged, visibleDiffering }; +}; + +const evaluate = (pixels: Uint8ClampedArray, width: number, height: number, steps: SpatialAdjustmentStep[]) => { + const output = pixels.slice(); + expect(evaluateSpatialSteps(output, width, height, steps)).toEqual({ status: 'ok' }); + return output; +}; + +describe('evaluateSpatialSteps', () => { + it.each(cases)('matches the Python node output for $name', ({ expected, height, rgba, steps, width }) => { + const input = decode(rgba); + const result = compareWithPython(evaluate(input, width, height, steps), input, decode(expected)); + + expect(result).toEqual({ alphaChanged: 0, hiddenChanged: 0, visibleDiffering: 0 }); + }); + + it('gives content identical results whatever transparent padding surrounds it', () => { + const { height, rgba, steps, width } = cases.find((entry) => entry.name === 'clarity_then_dehaze')!; + const content = decode(rgba); + const padded = new Uint8ClampedArray((width + 11) * (height + 6) * 4); + for (let i = 0; i < padded.length; i += 4) { + // Hidden padding colors are arbitrary and must survive untouched. + padded[i] = i % 251; + padded[i + 1] = 7; + padded[i + 2] = 200; + } + const left = 7; + const top = 4; + const paddedWidth = width + 11; + for (let y = 0; y < height; y++) { + padded.set(content.subarray(y * width * 4, (y + 1) * width * 4), ((top + y) * paddedWidth + left) * 4); + } + + const alone = evaluate(content, width, height, steps); + const surrounded = evaluate(padded, paddedWidth, height + 6, steps); + + const expected = padded.slice(); + for (let y = 0; y < height; y++) { + expected.set(alone.subarray(y * width * 4, (y + 1) * width * 4), ((top + y) * paddedWidth + left) * 4); + } + expect(surrounded).toEqual(expected); + }); + + it('clamps out-of-range amounts and sizes to the node limits', () => { + const { height, rgba, width } = cases[0]; + const input = decode(rgba); + + expect(evaluate(input, width, height, [{ amount: 500, kind: 'clarity', size: 4000 }])).toEqual( + evaluate(input, width, height, [{ amount: 200, kind: 'clarity', size: 1000 }]) + ); + expect(evaluate(input, width, height, [{ amount: -900, kind: 'texture', size: 0.2 }])).toEqual( + evaluate(input, width, height, [{ amount: -200, kind: 'texture', size: 1 }]) + ); + expect(evaluate(input, width, height, [{ amount: 1e9, kind: 'dehaze' }])).toEqual( + evaluate(input, width, height, [{ amount: 200, kind: 'dehaze' }]) + ); + }); + + it('leaves pixels untouched for zero or non-finite steps and for fully transparent layers', () => { + const { height, rgba, width } = cases[0]; + const input = decode(rgba); + const identitySteps: SpatialAdjustmentStep[] = [ + { amount: 0, kind: 'clarity', size: 40 }, + { amount: Number.NaN, kind: 'texture', size: 8 }, + { amount: 50, kind: 'texture', size: Number.POSITIVE_INFINITY }, + { amount: Number.NEGATIVE_INFINITY, kind: 'dehaze' }, + ]; + expect(evaluate(input, width, height, identitySteps)).toEqual(input); + + const transparent = input.map((value, index) => (index % 4 === 3 ? 0 : value)); + expect(evaluate(transparent, width, height, [{ amount: 100, kind: 'dehaze' }])).toEqual(transparent); + }); + + it('allocates the per-pixel scratch the service reserves, less the labels when nothing is hidden', () => { + const allocations: number[] = []; + vi.stubGlobal( + 'ArrayBuffer', + class extends ArrayBuffer { + constructor(length: number) { + super(length); + allocations.push(length); + } + } + ); + const withHidden = cases.find((entry) => entry.name === 'clarity_default_size')!; + const opaque = cases.find((entry) => entry.name === 'flat_zero_channel_clarity')!; + try { + evaluate(decode(withHidden.rgba), withHidden.width, withHidden.height, withHidden.steps); + evaluate(decode(opaque.rgba), opaque.width, opaque.height, opaque.steps); + } finally { + vi.unstubAllGlobals(); + } + + expect(allocations).toEqual([ + withHidden.width * withHidden.height * SPATIAL_SCRATCH_BYTES_PER_PIXEL, + opaque.width * opaque.height * (SPATIAL_SCRATCH_BYTES_PER_PIXEL - 4), + ]); + }); + + it('refuses alpha bounds larger than the pixel limit without touching pixels', () => { + const width = 4097; + const height = MAX_SPATIAL_PIXELS / 4096; + const pixels = new Uint8ClampedArray(width * height * 4); + const last = pixels.length - 4; + pixels.set([10, 20, 30, 255], 0); + pixels.set([40, 50, 60, 255], last); + + expect(evaluateSpatialSteps(pixels, width, height, [{ amount: 100, kind: 'dehaze' }])).toEqual({ + status: 'too-large', + }); + expect(Array.from(pixels.subarray(0, 4))).toEqual([10, 20, 30, 255]); + expect(Array.from(pixels.subarray(last))).toEqual([40, 50, 60, 255]); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/evaluateSpatialSteps.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/evaluateSpatialSteps.ts new file mode 100644 index 00000000000..dc12b4f8a53 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/evaluateSpatialSteps.ts @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Canvas evaluation of the backend's Clarity, Texture and Contrast Dehaze nodes (`invokeai/backend/image_util/`). + * Each step equals the node run on the layer cropped to its alpha bounds: transparent padding never changes the + * result, and pixels outside those bounds, hidden (alpha 0) pixels and alpha itself are never written. + */ + +import type { Rect } from '@workbench/canvas-engine/types'; + +import { alphaBounds } from '@workbench/canvas-engine/render/alphaBounds'; +import { propagateNearestVisibleLabels } from '@workbench/canvas-engine/render/spatial/alphaFill'; +import { boxMean, edgeAwareBand, type FilterScratch } from '@workbench/canvas-engine/render/spatial/guidedFilter'; +import { + clipToSrgbGamut, + linearSrgbFromOklab, + oklabFromSrgb8, + roundHalfEven, + srgb8FromLinear, +} from '@workbench/canvas-engine/render/spatial/numerics'; +import { + MAX_SPATIAL_PIXELS, + SPATIAL_SCRATCH_BYTES_PER_PIXEL, + type SpatialAdjustmentStep, + type SpatialEvaluation, +} from '@workbench/canvas-engine/render/spatial/spatialSteps'; + +const CLARITY_EPSILON = 0.002; +const TEXTURE_EPSILON = 0.0008; +const DEHAZE_EPSILON = 0.002; + +const OK: SpatialEvaluation = { status: 'ok' }; +const TOO_LARGE: SpatialEvaluation = { status: 'too-large' }; + +const clamp = (value: number, min: number, max: number): number => (value < min ? min : value > max ? max : value); + +/** Clamps a step into the node's accepted ranges; null when it cannot change pixels. */ +const resolveStep = (step: SpatialAdjustmentStep): SpatialAdjustmentStep | null => { + if (!Number.isFinite(step.amount) || step.amount === 0) { + return null; + } + const amount = clamp(step.amount, -200, 200); + switch (step.kind) { + case 'clarity': + return Number.isFinite(step.size) ? { amount, kind: 'clarity', size: clamp(step.size, 4, 1000) } : null; + case 'texture': + return Number.isFinite(step.size) ? { amount, kind: 'texture', size: clamp(step.size, 1, 100) } : null; + case 'dehaze': + return { amount, kind: 'dehaze' }; + } +}; + +/** One evaluation's view of the layer: its alpha bounds and the bounds-sized planes every step reuses. */ +type Workspace = { + readonly pixels: Uint8ClampedArray; + readonly stride: number; + readonly bounds: Rect; + /** Visible pixels label themselves; hidden ones name the visible pixel whose lightness they borrow. Null when + * nothing inside the bounds is hidden. */ + readonly labels: Int32Array | null; + readonly lightness: Float32Array; + /** The step's band, then its adjusted lightness. */ + readonly adjusted: Float32Array; + readonly scratch: FilterScratch; + readonly color: Float64Array; +}; + +const LABEL_BYTES_PER_PIXEL = 4; + +const hasHiddenPixels = (pixels: Uint8ClampedArray, stride: number, bounds: Rect): boolean => { + for (let y = 0; y < bounds.height; y++) { + const alphaOffset = ((bounds.y + y) * stride + bounds.x) * 4 + 3; + for (let x = 0; x < bounds.width; x++) { + if (pixels[alphaOffset + x * 4] === 0) { + return true; + } + } + } + return false; +}; + +/** Visible pixels label themselves; hidden pixels get the visible pixel the backend's fill copies from. */ +const fillLabels = ( + pixels: Uint8ClampedArray, + stride: number, + bounds: Rect, + labels: Int32Array, + distances: Uint32Array +) => { + for (let y = 0; y < bounds.height; y++) { + const rowOffset = y * bounds.width; + const alphaOffset = ((bounds.y + y) * stride + bounds.x) * 4 + 3; + for (let x = 0; x < bounds.width; x++) { + labels[rowOffset + x] = pixels[alphaOffset + x * 4] === 0 ? -1 : rowOffset + x; + } + } + propagateNearestVisibleLabels(labels, bounds.width, bounds.height, distances); +}; + +/** + * Every per-pixel plane comes from one buffer sized by {@link SPATIAL_SCRATCH_BYTES_PER_PIXEL}, so the service's + * memory reservation and the actual allocation cannot drift apart. + */ +const createWorkspace = (pixels: Uint8ClampedArray, stride: number, bounds: Rect): Workspace => { + const count = bounds.width * bounds.height; + const hidden = hasHiddenPixels(pixels, stride, bounds); + const bytesPerPixel = SPATIAL_SCRATCH_BYTES_PER_PIXEL - (hidden ? 0 : LABEL_BYTES_PER_PIXEL); + const buffer = new ArrayBuffer(count * bytesPerPixel); + let offset = 0; + const plane = (): Float32Array => { + const view = new Float32Array(buffer, offset, count); + offset += view.byteLength; + return view; + }; + const scratch: FilterScratch = { + columnSums: new Float64Array(bounds.width), + first: plane(), + second: plane(), + third: plane(), + }; + const lightness = plane(); + const adjusted = plane(); + let labels: Int32Array | null = null; + if (hidden) { + labels = new Int32Array(buffer, offset, count); + offset += labels.byteLength; + // The distances are dead before any filter runs, so they borrow a filter plane's memory. + fillLabels(pixels, stride, bounds, labels, new Uint32Array(buffer, scratch.first.byteOffset, count)); + } + if (offset !== buffer.byteLength) { + throw new Error('Spatial scratch layout does not match SPATIAL_SCRATCH_BYTES_PER_PIXEL'); + } + return { adjusted, bounds, color: new Float64Array(3), labels, lightness, pixels, scratch, stride }; +}; + +/** Oklab lightness of the current pixels; hidden pixels take their labelled visible pixel's lightness. */ +const loadLightness = ({ bounds, color, labels, lightness, pixels, stride }: Workspace): void => { + for (let y = 0; y < bounds.height; y++) { + const rowOffset = y * bounds.width; + const pixelOffset = ((bounds.y + y) * stride + bounds.x) * 4; + for (let x = 0; x < bounds.width; x++) { + const p = pixelOffset + x * 4; + if (pixels[p + 3] !== 0) { + oklabFromSrgb8(pixels[p], pixels[p + 1], pixels[p + 2], color); + lightness[rowOffset + x] = color[0]; + } + } + } + if (labels) { + for (let i = 0; i < labels.length; i++) { + if (labels[i] !== i) { + lightness[i] = lightness[labels[i]]; + } + } + } +}; + +/** Writes visible pixels back with `adjusted` lightness and their own chroma scaled by float32 `chromaScale`. */ +const storeAdjusted = ({ adjusted, bounds, color, pixels, stride }: Workspace, chromaScale: number): void => { + for (let y = 0; y < bounds.height; y++) { + const rowOffset = y * bounds.width; + const pixelOffset = ((bounds.y + y) * stride + bounds.x) * 4; + for (let x = 0; x < bounds.width; x++) { + const p = pixelOffset + x * 4; + if (pixels[p + 3] === 0) { + continue; + } + oklabFromSrgb8(pixels[p], pixels[p + 1], pixels[p + 2], color); + // The backend stores Oklab as float32 and scales chroma in float32. + const a = Math.fround(Math.fround(color[1]) * chromaScale); + const b = Math.fround(Math.fround(color[2]) * chromaScale); + linearSrgbFromOklab(clamp(adjusted[rowOffset + x], 0, 1), a, b, color); + clipToSrgbGamut(color); + pixels[p] = srgb8FromLinear(color[0]); + pixels[p + 1] = srgb8FromLinear(color[1]); + pixels[p + 2] = srgb8FromLinear(color[2]); + } + } +}; + +/** Inner and outer band radii shared by Clarity and Texture. */ +const bandRadii = (size: number): readonly [number, number] => { + const inner = Math.max(1, roundHalfEven(size / 4)); + return [inner, Math.max(inner + 1, roundHalfEven(size))]; +}; + +const applyClarity = (workspace: Workspace, amount: number, size: number): void => { + const { adjusted, bounds, lightness, scratch } = workspace; + const [inner, outer] = bandRadii(size); + edgeAwareBand(lightness, adjusted, bounds.width, bounds.height, inner, outer, CLARITY_EPSILON, scratch); + const strength = amount / 100; + for (let i = 0; i < adjusted.length; i++) { + const value = lightness[i]; + const clipped = clamp(value, 0, 1); + adjusted[i] = value + strength * (4 * clipped * (1 - clipped)) * adjusted[i]; + } + storeAdjusted(workspace, 1); +}; + +const applyTexture = (workspace: Workspace, amount: number, size: number): void => { + const { adjusted, bounds, lightness, scratch } = workspace; + const [inner, outer] = bandRadii(size); + edgeAwareBand(lightness, adjusted, bounds.width, bounds.height, inner, outer, TEXTURE_EPSILON, scratch); + if (amount > 0) { + // Fade sharpening out where the local band RMS is noise-level flat. + const meanSquare = scratch.first; + const rmsRadius = Math.max(2, roundHalfEven(size / 3)); + boxMean(adjusted, meanSquare, bounds.width, bounds.height, rmsRadius, true, scratch.columnSums); + for (let i = 0; i < adjusted.length; i++) { + const weight = clamp((Math.sqrt(Math.max(meanSquare[i], 0)) - 0.002) / (0.01 - 0.002), 0, 1); + adjusted[i] *= weight * weight * (3 - 2 * weight); + } + } + const strength = amount / 100; + for (let i = 0; i < adjusted.length; i++) { + adjusted[i] = lightness[i] + strength * adjusted[i]; + } + storeAdjusted(workspace, 1); +}; + +const applyDehaze = (workspace: Workspace, amount: number): void => { + const { adjusted, bounds, lightness, scratch } = workspace; + const radius = Math.max(2, Math.min(1000, roundHalfEven(Math.min(bounds.width, bounds.height) / 8))); + edgeAwareBand(lightness, adjusted, bounds.width, bounds.height, 0, radius, DEHAZE_EPSILON, scratch); + const strength = amount / 100; + const contrast = 1 + 0.2 * strength; + for (let i = 0; i < adjusted.length; i++) { + adjusted[i] = 0.5 + contrast * (lightness[i] - 0.5) + strength * adjusted[i]; + } + storeAdjusted(workspace, Math.fround(1 + 0.2 * strength)); +}; + +/** Applies `steps` in order to unpremultiplied RGBA8 `pixels` (row-major, width*height*4) in place. */ +export const evaluateSpatialSteps = ( + pixels: Uint8ClampedArray, + width: number, + height: number, + steps: readonly SpatialAdjustmentStep[] +): SpatialEvaluation => { + const effective: SpatialAdjustmentStep[] = []; + for (const step of steps) { + const resolved = resolveStep(step); + if (resolved) { + effective.push(resolved); + } + } + if (effective.length === 0) { + return OK; + } + const bounds = alphaBounds({ data: pixels, height, width }); + if (bounds.width === 0 || bounds.height === 0) { + return OK; + } + if (bounds.width * bounds.height > MAX_SPATIAL_PIXELS) { + return TOO_LARGE; + } + const workspace = createWorkspace(pixels, width, bounds); + for (const step of effective) { + loadLightness(workspace); + switch (step.kind) { + case 'clarity': + applyClarity(workspace, step.amount, step.size); + break; + case 'texture': + applyTexture(workspace, step.amount, step.size); + break; + case 'dehaze': + applyDehaze(workspace, step.amount); + break; + } + } + return OK; +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/guidedFilter.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/guidedFilter.test.ts new file mode 100644 index 00000000000..6ec4736d6c5 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/guidedFilter.test.ts @@ -0,0 +1,80 @@ +import { boxMean, edgeAwareBand } from '@workbench/canvas-engine/render/spatial/guidedFilter'; +import { describe, expect, it } from 'vitest'; + +import fixtures from './__fixtures__/shared_image_manipulation_numerics.json'; + +const plane = (length: number) => new Float32Array(length); + +/** OpenCV's BORDER_REFLECT_101 by repeated mirroring, independent of the kernels' periodic form. */ +const mirror = (index: number, length: number): number => { + if (length === 1) { + return 0; + } + let position = index; + while (position < 0 || position >= length) { + position = position < 0 ? -position : 2 * (length - 1) - position; + } + return position; +}; + +const directBoxMean = (values: Float32Array, width: number, height: number, radius: number, squared: boolean) => + Array.from({ length: width * height }, (_, index) => { + const x = index % width; + const y = Math.floor(index / width); + let sum = 0; + for (let dy = -radius; dy <= radius; dy++) { + for (let dx = -radius; dx <= radius; dx++) { + const value = values[mirror(y + dy, height) * width + mirror(x + dx, width)]; + sum += squared ? value * value : value; + } + } + return sum / (2 * radius + 1) ** 2; + }); + +describe('boxMean', () => { + it.each([ + { height: 5, radius: 2, width: 6 }, + { height: 5, radius: 7, width: 4 }, + { height: 1, radius: 9, width: 6 }, + { height: 3, radius: 4, width: 1 }, + ])('matches reflected direct sums for $width x $height at radius $radius', ({ height, radius, width }) => { + const source = Float32Array.from({ length: width * height }, (_, index) => Math.sin(index * 1.7) * 0.5 + 0.5); + const columnSums = new Float64Array(width); + for (const squared of [false, true]) { + const target = plane(width * height); + boxMean(source, target, width, height, radius, squared, columnSums); + directBoxMean(source, width, height, radius, squared).forEach((expected, index) => { + expect(target[index]).toBeCloseTo(expected, 6); + }); + } + }); +}); + +describe('edgeAwareBand', () => { + it('matches the independently derived band whose fine scale is the input itself', () => { + const { atol, epsilon, expected, inner_radius, lightness, outer_radius } = fixtures.multiscale_band; + const width = lightness[0].length; + const height = lightness.length; + const count = width * height; + const detail = plane(count); + const scratch = { + columnSums: new Float64Array(width), + first: plane(count), + second: plane(count), + third: plane(count), + }; + + edgeAwareBand( + Float32Array.from(lightness.flat()), + detail, + width, + height, + inner_radius, + outer_radius, + epsilon, + scratch + ); + + expected.flat().forEach((value, index) => expect(Math.abs(detail[index] - value)).toBeLessThanOrEqual(atol)); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/guidedFilter.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/guidedFilter.ts new file mode 100644 index 00000000000..f2d6e92e56c --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/guidedFilter.ts @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Self-guided filtering and edge-aware bands with the backend's numerics (`guided_filter.py`, `multiscale.py`): + * normalized box means with OpenCV `BORDER_REFLECT_101` borders at any radius, accumulated in float64 as OpenCV's + * float32 box filter does. Work is O(width × height) for every radius. + */ + +/** Scratch shared by the filters: three planes of the image's size and one float64 row. */ +export type FilterScratch = { + readonly first: Float32Array; + readonly second: Float32Array; + readonly third: Float32Array; + readonly columnSums: Float64Array; +}; + +const reflect101 = (index: number, length: number): number => { + if (length === 1) { + return 0; + } + const period = 2 * length - 2; + const wrapped = ((index % period) + period) % period; + return wrapped < length ? wrapped : period - wrapped; +}; + +/** + * Reflect-101 bookkeeping along one axis: the distinct indices of the first window (centered on 0) with their + * multiplicities, and the indices entering and leaving as the window moves from each position to the next. + */ +type AxisWindow = { + readonly firstIndices: Int32Array; + readonly firstCounts: Int32Array; + readonly entering: Int32Array; + readonly leaving: Int32Array; +}; + +const axisWindow = (length: number, radius: number): AxisWindow => { + const counts = new Int32Array(length); + for (let offset = -radius; offset <= radius; offset++) { + counts[reflect101(offset, length)]++; + } + let distinct = 0; + for (let index = 0; index < length; index++) { + if (counts[index] !== 0) { + distinct++; + } + } + const firstIndices = new Int32Array(distinct); + const firstCounts = new Int32Array(distinct); + for (let index = 0, slot = 0; index < length; index++) { + if (counts[index] !== 0) { + firstIndices[slot] = index; + firstCounts[slot] = counts[index]; + slot++; + } + } + const entering = new Int32Array(length); + const leaving = new Int32Array(length); + for (let position = 0; position < length; position++) { + entering[position] = reflect101(position + radius + 1, length); + leaving[position] = reflect101(position - radius, length); + } + return { entering, firstCounts, firstIndices, leaving }; +}; + +/** + * Writes the normalized (2r+1)² box mean of `source`, or of its squares, to `target` (a different plane). Columns are + * summed while streaming rows, then each row slides a window across the column sums. + */ +export const boxMean = ( + source: Float32Array, + target: Float32Array, + width: number, + height: number, + radius: number, + squared: boolean, + columnSums: Float64Array +): void => { + const rows = axisWindow(height, radius); + const columns = axisWindow(width, radius); + const scale = 1 / ((2 * radius + 1) * (2 * radius + 1)); + + columnSums.fill(0, 0, width); + for (let slot = 0; slot < rows.firstIndices.length; slot++) { + const offset = rows.firstIndices[slot] * width; + const count = rows.firstCounts[slot]; + for (let x = 0; x < width; x++) { + const value = source[offset + x]; + columnSums[x] += count * (squared ? value * value : value); + } + } + + for (let y = 0; y < height; y++) { + let sum = 0; + for (let slot = 0; slot < columns.firstIndices.length; slot++) { + sum += columns.firstCounts[slot] * columnSums[columns.firstIndices[slot]]; + } + const rowOffset = y * width; + for (let x = 0; x < width; x++) { + target[rowOffset + x] = sum * scale; + sum += columnSums[columns.entering[x]] - columnSums[columns.leaving[x]]; + } + if (y + 1 === height) { + break; + } + const entering = rows.entering[y] * width; + const leaving = rows.leaving[y] * width; + if (squared) { + for (let x = 0; x < width; x++) { + const enteringValue = source[entering + x]; + const leavingValue = source[leaving + x]; + columnSums[x] += enteringValue * enteringValue - leavingValue * leavingValue; + } + } else { + for (let x = 0; x < width; x++) { + columnSums[x] += source[entering + x] - source[leaving + x]; + } + } + } +}; + +/** + * Writes the guided filter of `signal` guided by itself to `target`. Uses `scratch.first` and `scratch.second`; + * `target` must be neither of them nor `signal`. Radius must be at least 1. + */ +export const selfGuidedFilter = ( + signal: Float32Array, + target: Float32Array, + width: number, + height: number, + radius: number, + epsilon: number, + scratch: FilterScratch +): void => { + const { columnSums, first, second } = scratch; + const count = width * height; + boxMean(signal, first, width, height, radius, false, columnSums); + boxMean(signal, second, width, height, radius, true, columnSums); + for (let i = 0; i < count; i++) { + const mean = first[i]; + // Covariance equals the unclamped variance when the guide is the signal; only the denominator is clamped. + const variance = second[i] - mean * mean; + const coefficient = variance / (Math.max(variance, 0) + epsilon); + second[i] = coefficient; + first[i] = mean - coefficient * mean; + } + boxMean(second, target, width, height, radius, false, columnSums); + boxMean(first, second, width, height, radius, false, columnSums); + for (let i = 0; i < count; i++) { + target[i] = target[i] * signal[i] + second[i]; + } +}; + +/** + * Writes the detail between the inner- and outer-radius self-guided smooths of `signal` to `detail`. Inner radius + * zero means the signal itself. Uses all of `scratch`; `detail` must not be one of its planes or `signal`. + */ +export const edgeAwareBand = ( + signal: Float32Array, + detail: Float32Array, + width: number, + height: number, + innerRadius: number, + outerRadius: number, + epsilon: number, + scratch: FilterScratch +): void => { + const fine = innerRadius === 0 ? signal : detail; + if (innerRadius > 0) { + selfGuidedFilter(signal, detail, width, height, innerRadius, epsilon, scratch); + } + const coarse = scratch.third; + selfGuidedFilter(signal, coarse, width, height, outerRadius, epsilon, scratch); + const count = width * height; + for (let i = 0; i < count; i++) { + detail[i] = fine[i] - coarse[i]; + } +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/numerics.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/numerics.test.ts new file mode 100644 index 00000000000..e22ce98e21c --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/numerics.test.ts @@ -0,0 +1,53 @@ +import { oklabFromSrgb8, roundHalfEven, srgb8FromLinear } from '@workbench/canvas-engine/render/spatial/numerics'; +import { describe, expect, it } from 'vitest'; + +import fixtures from './__fixtures__/shared_image_manipulation_numerics.json'; + +describe('roundHalfEven', () => { + it.each([ + [2.5, 2], + [3.5, 4], + [-2.5, -2], + [-3.5, -4], + [0.5, 0], + [2.4999999, 2], + [2.5000001, 3], + ])('rounds %d to %d as Python does', (value, expected) => { + expect(roundHalfEven(value)).toBe(expected); + }); +}); + +describe('oklabFromSrgb8', () => { + it('matches independently derived Oklab values for the sRGB primaries and white', () => { + const { atol, expected_oklab, rgb } = fixtures.color_adapter; + const out = new Float64Array(3); + rgb.flat().forEach(([red, green, blue], pixel) => { + oklabFromSrgb8(red, green, blue, out); + expected_oklab.flat()[pixel].forEach((value, channel) => { + expect(Math.abs(out[channel] - value)).toBeLessThanOrEqual(atol); + }); + }); + }); +}); + +describe('srgb8FromLinear', () => { + const direct = (linear: number) => { + const clamped = Math.min(1, Math.max(0, linear)); + const encoded = clamped <= 0.0031308 ? clamped * 12.92 : 1.055 * clamped ** (1 / 2.4) - 0.055; + return roundHalfEven(encoded * 255); + }; + + it('equals clamping, gamma encoding and half-even rounding, including next to every byte boundary', () => { + const samples = [-1, -0, 0, 1e-300, 1, 1.5, Number.POSITIVE_INFINITY]; + for (let i = 0; i <= 200_000; i++) { + samples.push(i / 200_000); + } + for (let byte = 1; byte < 256; byte++) { + const encoded = (byte - 0.5) / 255; + const boundary = encoded <= 0.04045 ? encoded / 12.92 : ((encoded + 0.055) / 1.055) ** 2.4; + samples.push(boundary, boundary * (1 - 1e-15), boundary * (1 + 1e-15), boundary * (1 - 1e-9)); + } + const mismatches = samples.filter((linear) => srgb8FromLinear(linear) !== direct(linear)); + expect(mismatches).toEqual([]); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/numerics.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/numerics.ts new file mode 100644 index 00000000000..2d992005eba --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/numerics.ts @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: Apache-2.0 AND MIT +/** + * Scalar ports of the backend's sRGB, Oklab and gamut-clip conversions + * (`invokeai/backend/image_util/color_conversion.py`, `color.py`), with its constants, so canvas previews match + * the nodes. + * + * The gamut clip (maximum-saturation polynomial, cusp, gamut intersection and adaptive lightness anchor) ports + * Björn Ottosson's reference implementation, https://bottosson.github.io/posts/gamutclipping/, under this notice: + * + * Copyright (c) 2021 Björn Ottosson + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +const SRGB_TO_LINEAR_THRESHOLD = 0.0404482362771082; +const LINEAR_TO_SRGB_THRESHOLD = 0.0031308; +/** torch.finfo(torch.float32).max, the backend's "no intersection" step. */ +const MAX_FLOAT32 = 3.4028234663852886e38; +const GAMUT_CLIP_ALPHA = 0.05; +const GAMUT_CLIP_MIN_CHROMA = 0.00001; +/** The backend's `_GAMUT_TOLERANCE`: smaller overshoots are conversion noise and are clamped, not clipped. */ +const GAMUT_TOLERANCE = 1e-5; + +/** Python's `round()` and `torch.round`: ties go to the even integer. */ +export const roundHalfEven = (value: number): number => { + const rounded = Math.round(value); + return rounded - value === 0.5 && rounded % 2 !== 0 ? rounded - 1 : rounded; +}; + +const buildLinearFromSrgb8 = (): Float64Array => { + const table = new Float64Array(256); + for (let value = 0; value < 256; value++) { + const normalized = value / 255; + table[value] = normalized <= SRGB_TO_LINEAR_THRESHOLD ? normalized / 12.92 : ((normalized + 0.055) / 1.055) ** 2.4; + } + return table; +}; + +const LINEAR_FROM_SRGB8 = buildLinearFromSrgb8(); + +/** The backend's 8-bit encoding: clamp to [0, 1], gamma-encode, scale and round half to even. */ +const srgb8FromLinearDirect = (linear: number): number => { + const clamped = linear < 0 ? 0 : linear > 1 ? 1 : linear; + const encoded = clamped <= LINEAR_TO_SRGB_THRESHOLD ? clamped * 12.92 : 1.055 * clamped ** (1 / 2.4) - 0.055; + return roundHalfEven(encoded * 255); +}; + +const SRGB8_BUCKETS = 4096; + +/** + * Exact replacement for the per-channel power in {@link srgb8FromLinearDirect}: `thresholds[k]` is the smallest + * double that encodes to byte k or above (found by bisection over the direct encoding, which is monotone), and each + * of the uniform buckets records the byte at its lower edge, leaving at most a couple of threshold steps per lookup. + */ +const buildSrgb8Encoding = (): { readonly thresholds: Float64Array; readonly bucketBytes: Uint8Array } => { + const thresholds = new Float64Array(257); + thresholds[256] = Number.POSITIVE_INFINITY; + for (let byte = 1; byte < 256; byte++) { + let below = 0; + let above = 1; + for (;;) { + const middle = (below + above) / 2; + if (middle === below || middle === above) { + break; + } + if (srgb8FromLinearDirect(middle) >= byte) { + above = middle; + } else { + below = middle; + } + } + thresholds[byte] = above; + } + const bucketBytes = new Uint8Array(SRGB8_BUCKETS); + for (let bucket = 0, byte = 0; bucket < SRGB8_BUCKETS; bucket++) { + while (bucket / SRGB8_BUCKETS >= thresholds[byte + 1]) { + byte++; + } + bucketBytes[bucket] = byte; + } + return { bucketBytes, thresholds }; +}; + +const SRGB8_ENCODING = buildSrgb8Encoding(); + +/** Encodes one linear channel to the byte the backend stores for it. */ +export const srgb8FromLinear = (linear: number): number => { + if (!(linear > 0)) { + return 0; + } + if (linear >= 1) { + return 255; + } + const { bucketBytes, thresholds } = SRGB8_ENCODING; + let byte = bucketBytes[Math.floor(linear * SRGB8_BUCKETS)]; + while (linear >= thresholds[byte + 1]) { + byte++; + } + return byte; +}; + +/** Writes Oklab (L, a, b) for linear sRGB into `out`. */ +export const oklabFromLinearSrgb = (red: number, green: number, blue: number, out: Float64Array): void => { + const l = Math.cbrt(0.4122214708 * red + 0.5363325363 * green + 0.0514459929 * blue); + const m = Math.cbrt(0.2119034982 * red + 0.6806995451 * green + 0.1073969566 * blue); + const s = Math.cbrt(0.0883024619 * red + 0.2817188376 * green + 0.6299787005 * blue); + out[0] = 0.2104542553 * l + 0.793617785 * m - 0.0040720468 * s; + out[1] = 1.9779984951 * l - 2.428592205 * m + 0.4505937099 * s; + out[2] = 0.0259040371 * l + 0.7827717662 * m - 0.808675766 * s; +}; + +/** Writes Oklab (L, a, b) for byte-valued sRGB into `out`. */ +export const oklabFromSrgb8 = (red: number, green: number, blue: number, out: Float64Array): void => { + oklabFromLinearSrgb(LINEAR_FROM_SRGB8[red], LINEAR_FROM_SRGB8[green], LINEAR_FROM_SRGB8[blue], out); +}; + +/** Writes unclamped linear sRGB for Oklab (L, a, b) into `out`. */ +export const linearSrgbFromOklab = (lightness: number, a: number, b: number, out: Float64Array): void => { + const lRoot = lightness + 0.3963377774 * a + 0.2158037573 * b; + const mRoot = lightness - 0.1055613458 * a - 0.0638541728 * b; + const sRoot = lightness - 0.0894841775 * a - 1.291485548 * b; + const l = lRoot * lRoot * lRoot; + const m = mRoot * mRoot * mRoot; + const s = sRoot * sRoot * sRoot; + out[0] = 4.0767416621 * l - 3.3077115913 * m + 0.2309699292 * s; + out[1] = -1.2684380046 * l + 2.6097574011 * m - 0.3413193965 * s; + out[2] = -0.0041960863 * l - 0.7034186147 * m + 1.707614701 * s; +}; + +/** + * Maximum saturation (C / L) inside sRGB along the unit hue (a, b), refined with one Halley step, as + * `_max_srgb_saturation_tensor(steps=1)`. + */ +const maxSrgbSaturation = (a: number, b: number): number => { + // Polynomial fit per limiting channel (red, green, else blue) and that channel's LMS-to-linear-sRGB row. + let saturation: number; + let wl: number; + let wm: number; + let ws: number; + if (-1.88170328 * a - 0.80936493 * b > 1) { + saturation = 1.19086277 + 1.76576728 * a + 0.59662641 * b + 0.75515197 * a * a + 0.56771245 * a * b; + wl = 4.0767416621; + wm = -3.3077115913; + ws = 0.2309699292; + } else if (1.81444104 * a - 1.19445276 * b > 1) { + saturation = 0.73956515 - 0.45954494 * a + 0.08285427 * b + 0.1254107 * a * a + 0.14503204 * a * b; + wl = -1.2684380046; + wm = 2.6097574011; + ws = -0.3413193965; + } else { + saturation = 1.35733652 - 0.00915799 * a - 1.1513021 * b - 0.50559606 * a * a + 0.00692167 * a * b; + wl = -0.0041960863; + wm = -0.7034186147; + ws = 1.707614701; + } + const kl = 0.3963377774 * a + 0.2158037573 * b; + const km = -0.1055613458 * a - 0.0638541728 * b; + const ks = -0.0894841775 * a - 1.291485548 * b; + const lRoot = 1 + saturation * kl; + const mRoot = 1 + saturation * km; + const sRoot = 1 + saturation * ks; + const f = wl * lRoot * lRoot * lRoot + wm * mRoot * mRoot * mRoot + ws * sRoot * sRoot * sRoot; + const f1 = wl * 3 * kl * lRoot * lRoot + wm * 3 * km * mRoot * mRoot + ws * 3 * ks * sRoot * sRoot; + const f2 = wl * 6 * kl * kl * lRoot + wm * 6 * km * km * mRoot + ws * 6 * ks * ks * sRoot; + return saturation - (f * f1) / (f1 * f1 - 0.5 * f * f2); +}; + +/** Newton-Halley step length towards one channel's gamut boundary; MAX_FLOAT32 when it moves away. */ +const channelStep = (value: number, first: number, second: number): number => { + const u = first / (first * first - 0.5 * value * second); + return u < 0 ? MAX_FLOAT32 : -value * u; +}; + +const scratch = new Float64Array(3); + +/** + * Projects out-of-gamut linear sRGB in `rgb` towards an adaptive lightness anchor, as the backend's restore does: + * `gamut_clip_tensor` (alpha 0.05, one refinement step) in float64, applied only to colors more than + * {@link GAMUT_TOLERANCE} outside [0, 1]. The clip moves colors that sit on the gamut boundary, so the tolerance + * keeps float32 round-trip noise on an in-gamut color from reaching it. Other colors are left unchanged. + */ +export const clipToSrgbGamut = (rgb: Float64Array): void => { + const red = rgb[0]; + const green = rgb[1]; + const blue = rgb[2]; + if (!(Math.max(red, green, blue) > 1 + GAMUT_TOLERANCE || Math.min(red, green, blue) < -GAMUT_TOLERANCE)) { + return; + } + oklabFromLinearSrgb(red, green, blue, scratch); + const lightness = scratch[0]; + let chroma = Math.sqrt(scratch[1] * scratch[1] + scratch[2] * scratch[2]); + if (chroma < GAMUT_CLIP_MIN_CHROMA) { + chroma = GAMUT_CLIP_MIN_CHROMA; + } + const unitA = scratch[1] / chroma; + const unitB = scratch[2] / chroma; + const lightnessOffset = lightness - 0.5; + const absOffset = Math.abs(lightnessOffset); + const e1 = absOffset + chroma * GAMUT_CLIP_ALPHA + 0.5; + const anchor = (Math.sign(lightnessOffset) * (e1 - Math.sqrt(e1 * e1 - 2 * absOffset)) + 1) * 0.5; + + // Cusp of the hue slice: the most saturated sRGB color along (unitA, unitB). + const cuspSaturation = maxSrgbSaturation(unitA, unitB); + linearSrgbFromOklab(1, cuspSaturation * unitA, cuspSaturation * unitB, scratch); + const cuspLightness = (1 / Math.max(scratch[0], scratch[1], scratch[2])) ** (1 / 3); + const cuspChroma = cuspLightness * cuspSaturation; + + // Intersection of the line from (lightness, chroma) to (anchor, 0) with the gamut boundary. + const upper = (lightness - anchor) * cuspChroma - (cuspLightness - anchor) * chroma > 0; + let t = upper + ? (cuspChroma * (anchor - 1)) / (chroma * (cuspLightness - 1) + cuspChroma * (anchor - lightness)) + : (cuspChroma * anchor) / (chroma * cuspLightness + cuspChroma * (anchor - lightness)); + if (upper) { + const kl = 0.3963377774 * unitA + 0.2158037573 * unitB; + const km = -0.1055613458 * unitA - 0.0638541728 * unitB; + const ks = -0.0894841775 * unitA - 1.291485548 * unitB; + const dLightness = lightness - anchor; + const lDt = kl * chroma + dLightness; + const mDt = km * chroma + dLightness; + const sDt = ks * chroma + dLightness; + const stepLightness = anchor * (1 - t) + t * lightness; + const stepChroma = t * chroma; + const lRoot = kl * stepChroma + stepLightness; + const mRoot = km * stepChroma + stepLightness; + const sRoot = ks * stepChroma + stepLightness; + const l = lRoot * lRoot * lRoot; + const m = mRoot * mRoot * mRoot; + const s = sRoot * sRoot * sRoot; + const l1 = lRoot * lRoot * lDt * 3; + const m1 = mRoot * mRoot * mDt * 3; + const s1 = sRoot * sRoot * sDt * 3; + const l2 = lDt * lDt * lRoot * 6; + const m2 = mDt * mDt * mRoot * 6; + const s2 = sDt * sDt * sRoot * 6; + const redStep = channelStep( + 4.0767416621 * l - 3.3077115913 * m + 0.2309699292 * s - 1, + 4.0767416621 * l1 - 3.3077115913 * m1 + 0.2309699292 * s1, + 4.0767416621 * l2 - 3.3077115913 * m2 + 0.2309699292 * s2 + ); + const greenStep = channelStep( + -1.2684380046 * l + 2.6097574011 * m - 0.3413193965 * s - 1, + -1.2684380046 * l1 + 2.6097574011 * m1 - 0.3413193965 * s1, + -1.2684380046 * l2 + 2.6097574011 * m2 - 0.3413193965 * s2 + ); + const blueStep = channelStep( + -0.0041960863 * l - 0.7034186147 * m + 1.707614701 * s - 1, + -0.0041960863 * l1 - 0.7034186147 * m1 + 1.707614701 * s1, + -0.0041960863 * l2 - 0.7034186147 * m2 + 1.707614701 * s2 + ); + t += Math.min(redStep, greenStep, blueStep); + } + const clippedChroma = t * chroma; + linearSrgbFromOklab(anchor * (1 - t) + t * lightness, clippedChroma * unitA, clippedChroma * unitB, rgb); +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustment.worker.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustment.worker.ts new file mode 100644 index 00000000000..50159772c03 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustment.worker.ts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 +/** Module worker behind `spatialAdjustmentService`: evaluates one job per message and transfers the buffer back. */ + +import type { + SpatialExecutorResult, + SpatialWorkerRequest, + SpatialWorkerResponse, +} from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; + +import { evaluateSpatialSteps } from '@workbench/canvas-engine/render/spatial/evaluateSpatialSteps'; + +const evaluate = ({ buffer, height, steps, width }: SpatialWorkerRequest): SpatialExecutorResult => { + try { + return { buffer, status: evaluateSpatialSteps(new Uint8ClampedArray(buffer), width, height, steps).status }; + } catch (error) { + return { buffer, message: error instanceof Error ? error.message : String(error), status: 'error' }; + } +}; + +self.addEventListener('message', (event: MessageEvent) => { + const response: SpatialWorkerResponse = { ...evaluate(event.data), id: event.data.id }; + self.postMessage(response, { transfer: [response.buffer] }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.browser.test.ts new file mode 100644 index 00000000000..a59d3b628e8 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.browser.test.ts @@ -0,0 +1,108 @@ +import type { SpatialAdjustmentStep } from '@workbench/canvas-engine/render/spatial/spatialSteps'; + +import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; +import { evaluateSpatialSteps } from '@workbench/canvas-engine/render/spatial/evaluateSpatialSteps'; +import { + createSpatialAdjustmentService, + createWorkerExecutor, + type SpatialAdjustmentExecutor, + type SpatialAdjustmentService, +} from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import { afterEach, describe, expect, it } from 'vitest'; + +/** Deterministic mulberry32 so the image is identical on every run. */ +const seeded = (seed: number) => () => { + seed = (seed + 0x6d2b79f5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; +}; + +/** Smooth gradients plus noise, with a transparent border, a transparent hole and soft alpha in between. */ +const createImage = (width: number, height: number): Uint8ClampedArray => { + const random = seeded(0x5eed); + const pixels = new Uint8ClampedArray(width * height * 4); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const p = (y * width + x) * 4; + const noise = random() * 48 - 24; + pixels[p] = (x / width) * 200 + noise + 20; + pixels[p + 1] = (y / height) * 180 + Math.sin(x / 9) * 30 + noise + 30; + pixels[p + 2] = 120 + Math.cos((x + y) / 13) * 60 + noise; + const border = x < 12 || y < 10 || x >= width - 20 || y >= height - 8; + const hole = (x - width / 2) ** 2 + (y - height / 2) ** 2 < 30 ** 2; + pixels[p + 3] = border || hole ? 0 : x % 37 < 4 ? 96 : 255; + } + } + return pixels; +}; + +const services: SpatialAdjustmentService[] = []; + +const createService = (createExecutor?: () => SpatialAdjustmentExecutor) => { + const budget = new RasterMemoryBudgetController({ budgetBytes: 512 * 1024 * 1024 }); + const service = createSpatialAdjustmentService({ + createExecutor, + reserve: (bytes) => budget.reserveOperation(bytes, { purpose: 'layer-operation' }), + }); + services.push(service); + return { budget, service }; +}; + +afterEach(() => { + for (const service of services.splice(0)) { + service.dispose(); + } +}); + +describe('spatial adjustment worker', () => { + it('returns pixels bit-identical to evaluating on the main thread', async () => { + const { budget, service } = createService(); + const width = 256; + const height = 192; + const steps: SpatialAdjustmentStep[] = [ + { amount: 70, kind: 'clarity', size: 24 }, + { amount: 45, kind: 'texture', size: 6 }, + { amount: 35, kind: 'dehaze' }, + ]; + const input = createImage(width, height); + const expected = input.slice(); + expect(evaluateSpatialSteps(expected, width, height, steps)).toEqual({ status: 'ok' }); + + const transferred = input.slice(); + const result = await service.run({ height, pixels: transferred, priority: 'now', slotKey: 'layer', steps, width }); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') { + return; + } + // Zero copy: the caller's buffer moved to the worker and a different one came back. + expect(transferred.byteLength).toBe(0); + expect(result.pixels).toEqual(expected); + expect(result.pixels).not.toEqual(input); + expect(budget.snapshot().reservedBytes).toBe(0); + }); + + it('reports a job whose buffer cannot be transferred as an error and keeps the same worker', async () => { + let workers = 0; + const { service } = createService(() => { + workers++; + return createWorkerExecutor(); + }); + const steps: SpatialAdjustmentStep[] = [{ amount: 50, kind: 'clarity', size: 8 }]; + const job = (slotKey: string, pixels: Uint8ClampedArray) => + service.run({ height: 32, pixels, priority: 'now', slotKey, steps, width: 32 }); + + const first = job('first', createImage(32, 32)); + const detached = createImage(32, 32); + const broken = job('broken', detached); + // Contrary to the contract, the caller transfers the queued job's buffer elsewhere. + structuredClone(detached.buffer, { transfer: [detached.buffer] }); + const after = job('after', createImage(32, 32)); + + expect((await first).status).toBe('ok'); + expect(await broken).toEqual({ status: 'error' }); + expect((await after).status).toBe('ok'); + expect(workers).toBe(1); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.test.ts new file mode 100644 index 00000000000..83954801a6f --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.test.ts @@ -0,0 +1,361 @@ +import type { + SpatialAdjustmentExecutor, + SpatialExecutorJob, + SpatialExecutorResult, + SpatialRunPriority, + SpatialRunResult, +} from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import type { SpatialAdjustmentStep } from '@workbench/canvas-engine/render/spatial/spatialSteps'; + +import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; +import { evaluateSpatialSteps } from '@workbench/canvas-engine/render/spatial/evaluateSpatialSteps'; +import { + createSpatialAdjustmentService, + SPATIAL_WORKER_IDLE_MS, +} from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import { createInlineSpatialExecutor } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService.testing'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +const WIDTH = 4; +const HEIGHT = 2; +const PIXEL_BYTES = WIDTH * HEIGHT * 4; +const STEPS: readonly SpatialAdjustmentStep[] = [{ amount: 50, kind: 'clarity', size: 8 }]; + +/** Pixels whose first byte names the job, so the executor's view shows which job it got. */ +const tagged = (tag: number): Uint8ClampedArray => { + const pixels = new Uint8ClampedArray(PIXEL_BYTES); + pixels[0] = tag; + return pixels; +}; + +/** An executor whose jobs settle only when the test says so. */ +const createManualExecutors = () => { + const executors: { disposed: boolean; started: number[] }[] = []; + const pending: { job: SpatialExecutorJob; settle: (status: SpatialExecutorResult['status'] | 'crash') => void }[] = + []; + const create = (): SpatialAdjustmentExecutor => { + const state = { disposed: false, started: [] as number[] }; + executors.push(state); + const own = new Set<(error: Error) => void>(); + return { + dispose: () => { + state.disposed = true; + for (const reject of own) { + reject(new Error('terminated')); + } + }, + run: (job) => + new Promise((resolve, reject) => { + state.started.push(new Uint8ClampedArray(job.buffer)[0]); + own.add(reject); + pending.push({ + job, + settle: (status) => { + own.delete(reject); + if (status === 'crash') { + reject(new Error('worker crashed')); + } else { + const pixels = new Uint8ClampedArray(job.buffer); + pixels[1] = 255; + resolve({ buffer: job.buffer, status }); + } + }, + }); + }), + }; + }; + /** Settles the oldest unsettled job, then lets the service start the next one. */ + const settleNext = async (status: SpatialExecutorResult['status'] | 'crash' = 'ok') => { + const next = pending.shift(); + if (!next) { + throw new Error('no job is running'); + } + next.settle(status); + await flush(); + }; + const started = () => executors.flatMap((executor) => executor.started); + return { create, executors, settleNext, started }; +}; + +const flush = async () => { + for (let i = 0; i < 5; i++) { + await Promise.resolve(); + } +}; + +const setup = (budgetBytes = 1 << 20) => { + const budget = new RasterMemoryBudgetController({ budgetBytes }); + const manual = createManualExecutors(); + const service = createSpatialAdjustmentService({ + createExecutor: manual.create, + reserve: (bytes) => budget.reserveOperation(bytes, { purpose: 'layer-operation' }), + }); + const reserved = () => budget.snapshot().reservedBytes; + const run = (tag: number, options: { slotKey?: string; priority?: SpatialRunPriority; signal?: AbortSignal } = {}) => + service.run({ + height: HEIGHT, + pixels: tagged(tag), + priority: options.priority ?? 'display', + signal: options.signal, + slotKey: options.slotKey ?? `slot-${tag}`, + steps: STEPS, + width: WIDTH, + }); + return { ...manual, reserved, run, service }; +}; + +/** Records results as they settle so tests can assert resolution without awaiting the job. */ +const track = (promise: Promise) => { + const state: { result: SpatialRunResult | null } = { result: null }; + void promise.then((result) => { + state.result = result; + }); + return state; +}; + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('createSpatialAdjustmentService', () => { + it('returns the kernels’ output for the transferred pixels and releases the reservation', async () => { + const budget = new RasterMemoryBudgetController({ budgetBytes: 1 << 20 }); + const service = createSpatialAdjustmentService({ + createExecutor: createInlineSpatialExecutor, + reserve: (bytes) => budget.reserveOperation(bytes, { purpose: 'layer-operation' }), + }); + const width = 24; + const height = 16; + const input = new Uint8ClampedArray(width * height * 4); + for (let i = 0; i < input.length; i++) { + input[i] = (i * 37 + (i >> 5) * 11) % 256; + } + const steps: SpatialAdjustmentStep[] = [ + { amount: 60, kind: 'clarity', size: 6 }, + { amount: -40, kind: 'dehaze' }, + ]; + const expected = input.slice(); + evaluateSpatialSteps(expected, width, height, steps); + + const pending = service.run({ height, pixels: input.slice(), priority: 'now', slotKey: 'a', steps, width }); + expect(budget.snapshot().reservedBytes).toBe(width * height * 4); + const result = await pending; + + expect(result.status).toBe('ok'); + expect(result.status === 'ok' && result.pixels).toEqual(expected); + expect(expected).not.toEqual(input); + expect(budget.snapshot().reservedBytes).toBe(0); + }); + + it('runs one job at a time, `now` before `display`, FIFO within a priority', async () => { + const { run, settleNext, started } = setup(); + const results = [run(1), run(2), run(3, { priority: 'now' }), run(4), run(5, { priority: 'now' })]; + await flush(); + expect(started()).toEqual([1]); + + for (let i = 0; i < 5; i++) { + await settleNext(); + } + + expect(started()).toEqual([1, 3, 5, 2, 4]); + for (const result of await Promise.all(results)) { + expect(result.status).toBe('ok'); + expect(result.status === 'ok' && result.pixels[1]).toBe(255); + } + }); + + it('replaces a queued job for the same slot in its queue position, but never the running one', async () => { + const { reserved, run, settleNext, started } = setup(); + const running = run(1, { slotKey: 'layer' }); + const replaced = track(run(2, { slotKey: 'layer' })); + const other = run(3, { slotKey: 'other' }); + const replacement = run(4, { slotKey: 'layer' }); + await flush(); + + expect(replaced.result).toEqual({ status: 'superseded' }); + expect(reserved()).toBe(3 * PIXEL_BYTES); + + for (let i = 0; i < 3; i++) { + await settleNext(); + } + + expect(started()).toEqual([1, 4, 3]); + expect((await running).status).toBe('ok'); + expect((await other).status).toBe('ok'); + expect((await replacement).status).toBe('ok'); + expect(reserved()).toBe(0); + }); + + it('moves a replaced slot to the replacement’s priority', async () => { + const { run, settleNext, started } = setup(); + void run(1); + void run(2, { slotKey: 'layer' }); + void run(3); + void run(4, { priority: 'now', slotKey: 'layer' }); + for (let i = 0; i < 3; i++) { + await settleNext(); + } + + expect(started()).toEqual([1, 4, 3]); + }); + + it('resolves an already-aborted request without reserving or running', async () => { + const { reserved, run, started } = setup(); + const controller = new AbortController(); + controller.abort(); + + expect(await run(1, { signal: controller.signal })).toEqual({ status: 'aborted' }); + await flush(); + expect(started()).toEqual([]); + expect(reserved()).toBe(0); + }); + + it('drops a queued job when its signal aborts', async () => { + const { reserved, run, settleNext, started } = setup(); + void run(1); + const controller = new AbortController(); + const queued = run(2, { signal: controller.signal }); + await flush(); + + controller.abort(); + expect(await queued).toEqual({ status: 'aborted' }); + expect(reserved()).toBe(PIXEL_BYTES); + + await settleNext(); + expect(started()).toEqual([1]); + expect(reserved()).toBe(0); + }); + + it('answers an aborted running job at once but holds its reservation and slot until the worker returns', async () => { + const { reserved, run, settleNext, started } = setup(); + const controller = new AbortController(); + const running = run(1, { signal: controller.signal }); + const next = run(2); + await flush(); + + controller.abort(); + expect(await running).toEqual({ status: 'aborted' }); + expect(reserved()).toBe(2 * PIXEL_BYTES); + expect(started()).toEqual([1]); + + await settleNext(); + expect(reserved()).toBe(PIXEL_BYTES); + expect(started()).toEqual([1, 2]); + await settleNext(); + expect((await next).status).toBe('ok'); + expect(reserved()).toBe(0); + }); + + it('refuses a job whose pixels do not fit the raster budget and keeps serving others', async () => { + const { reserved, run, settleNext, started } = setup(PIXEL_BYTES * 2); + const first = run(1); + const second = run(2); + + expect(await run(3)).toEqual({ status: 'over-budget' }); + + await settleNext(); + await settleNext(); + expect((await first).status).toBe('ok'); + expect((await second).status).toBe('ok'); + expect(started()).toEqual([1, 2]); + expect(reserved()).toBe(0); + }); + + it('maps executor outcomes and replaces a crashed executor for the rest of the queue', async () => { + const { executors, reserved, run, settleNext, started } = setup(); + const tooLarge = run(1); + const failed = run(2); + const crashed = run(3); + const after = run(4); + + await settleNext('too-large'); + await settleNext('error'); + expect(executors).toHaveLength(1); + await settleNext('crash'); + await settleNext(); + + expect(await tooLarge).toEqual({ status: 'too-large' }); + expect(await failed).toEqual({ status: 'error' }); + expect(await crashed).toEqual({ status: 'error' }); + expect((await after).status).toBe('ok'); + expect(executors.map((executor) => executor.disposed)).toEqual([true, false]); + expect(started()).toEqual([1, 2, 3, 4]); + expect(reserved()).toBe(0); + }); + + it('reports an executor that cannot be created as an error and tries again for the next job', async () => { + const budget = new RasterMemoryBudgetController({ budgetBytes: 1 << 20 }); + const manual = createManualExecutors(); + let attempts = 0; + const service = createSpatialAdjustmentService({ + createExecutor: () => { + attempts++; + if (attempts === 1) { + throw new Error('workers unavailable'); + } + return manual.create(); + }, + reserve: (bytes) => budget.reserveOperation(bytes, { purpose: 'layer-operation' }), + }); + const request = { height: HEIGHT, priority: 'now', slotKey: 'a', steps: STEPS, width: WIDTH } as const; + + expect(await service.run({ ...request, pixels: tagged(1) })).toEqual({ status: 'error' }); + const second = service.run({ ...request, pixels: tagged(2) }); + await flush(); + await manual.settleNext(); + + expect((await second).status).toBe('ok'); + expect(budget.snapshot().reservedBytes).toBe(0); + }); + + it('dispose aborts everything, terminates the executor, releases every reservation and refuses later work', async () => { + const { executors, reserved, run, service, started } = setup(); + const running = run(1); + const queued = run(2); + await flush(); + + service.dispose(); + expect(await running).toEqual({ status: 'aborted' }); + expect(await queued).toEqual({ status: 'aborted' }); + await flush(); + expect(executors.map((executor) => executor.disposed)).toEqual([true]); + expect(reserved()).toBe(0); + + expect(await run(3)).toEqual({ status: 'aborted' }); + await flush(); + expect(started()).toEqual([1]); + expect(executors).toHaveLength(1); + expect(reserved()).toBe(0); + }); + + it('terminates the executor after an idle period and creates a fresh one on demand', async () => { + vi.useFakeTimers(); + const { executors, run, settleNext } = setup(); + void run(1); + await settleNext(); + + await vi.advanceTimersByTimeAsync(SPATIAL_WORKER_IDLE_MS - 1); + void run(2); + await settleNext(); + await vi.advanceTimersByTimeAsync(SPATIAL_WORKER_IDLE_MS - 1); + expect(executors.map((executor) => executor.disposed)).toEqual([false]); + + await vi.advanceTimersByTimeAsync(1); + expect(executors.map((executor) => executor.disposed)).toEqual([true]); + + const revived = run(3); + await settleNext(); + expect((await revived).status).toBe('ok'); + expect(executors.map((executor) => executor.disposed)).toEqual([true, false]); + }); + + it('rejects pixels that do not span exactly one whole buffer', () => { + const { service } = setup(); + const request = { height: HEIGHT, priority: 'now', slotKey: 'a', steps: STEPS, width: WIDTH } as const; + + expect(() => + service.run({ ...request, pixels: new Uint8ClampedArray(PIXEL_BYTES * 2).subarray(0, PIXEL_BYTES) }) + ).toThrow(RangeError); + expect(() => service.run({ ...request, pixels: new Uint8ClampedArray(PIXEL_BYTES + 4) })).toThrow(RangeError); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.testing.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.testing.ts new file mode 100644 index 00000000000..0dcd46a3e0d --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.testing.ts @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +import type { SpatialAdjustmentExecutor } from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; + +import { evaluateSpatialSteps } from '@workbench/canvas-engine/render/spatial/evaluateSpatialSteps'; + +/** Evaluates on the calling thread after a microtask, so Node tests keep the worker's asynchronous boundary. */ +export const createInlineSpatialExecutor = (): SpatialAdjustmentExecutor => ({ + dispose: () => undefined, + run: async ({ buffer, height, steps, width }) => { + await Promise.resolve(); + try { + return { buffer, status: evaluateSpatialSteps(new Uint8ClampedArray(buffer), width, height, steps).status }; + } catch (error) { + return { buffer, message: error instanceof Error ? error.message : String(error), status: 'error' }; + } + }, +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.ts new file mode 100644 index 00000000000..f8589686503 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialAdjustmentService.ts @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Runs spatial adjustment evaluations off the main thread, one at a time, with raster-memory accounting. This is + * the only spatial module production code imports: the kernels load inside the worker, never in a page chunk. + */ + +import type { RasterMemoryReservationResult } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; +import type { SpatialAdjustmentStep } from '@workbench/canvas-engine/render/spatial/spatialSteps'; + +/** One evaluation handed to an executor; `buffer` holds `width * height` unpremultiplied RGBA8 pixels. */ +export interface SpatialExecutorJob { + readonly width: number; + readonly height: number; + readonly buffer: ArrayBuffer; + readonly steps: readonly SpatialAdjustmentStep[]; +} + +/** `error` reports a job that failed to send or threw while evaluating; the executor itself stays usable. */ +export interface SpatialExecutorResult { + readonly status: 'ok' | 'too-large' | 'error'; + readonly buffer: ArrayBuffer; + readonly message?: string; +} + +/** + * Evaluates one job at a time. `run` rejects only when the executor broke (worker crash, undeliverable reply); the + * service then disposes it and continues on a fresh one. `dispose` rejects any job still pending. + */ +export interface SpatialAdjustmentExecutor { + run(job: SpatialExecutorJob): Promise; + dispose(): void; +} + +export type SpatialWorkerRequest = SpatialExecutorJob & { readonly id: number }; +export type SpatialWorkerResponse = SpatialExecutorResult & { readonly id: number }; + +export type SpatialRunPriority = 'now' | 'display'; + +export interface SpatialRunRequest { + /** Identifies the surface this result is for; a newer request for the same slot replaces a queued one. */ + readonly slotKey: string; + readonly width: number; + readonly height: number; + /** + * Unpremultiplied RGBA8, exactly `width * height * 4` bytes spanning a whole `ArrayBuffer` (`run` throws a + * `RangeError` otherwise). The buffer is transferred to the worker: callers must not use this array, or any other + * view of its buffer, after calling `run`, whatever the result. + */ + readonly pixels: Uint8ClampedArray; + readonly steps: readonly SpatialAdjustmentStep[]; + /** `now` jobs run before every queued `display` job. */ + readonly priority: SpatialRunPriority; + readonly signal?: AbortSignal; +} + +export type SpatialRunResult = + | { readonly status: 'ok'; readonly pixels: Uint8ClampedArray } + | { readonly status: 'superseded' | 'aborted' | 'too-large' | 'over-budget' | 'error' }; + +export interface SpatialAdjustmentService { + run(request: SpatialRunRequest): Promise; + /** Cancels everything and terminates the worker; later `run` calls resolve `aborted`. */ + dispose(): void; +} + +export interface SpatialAdjustmentServiceOptions { + /** Reserves raster memory for one job's pixel buffer; the caller binds the reservation purpose. */ + readonly reserve: (bytes: number) => RasterMemoryReservationResult; + readonly createExecutor?: () => SpatialAdjustmentExecutor; +} + +/** An idle worker holds its module and heap; recreating one costs a module load, so keep it through short pauses. */ +export const SPATIAL_WORKER_IDLE_MS = 30_000; + +/** The production executor: one module worker running the kernels. */ +export const createWorkerExecutor = (): SpatialAdjustmentExecutor => { + const worker = new Worker(new URL('./spatialAdjustment.worker.ts', import.meta.url), { type: 'module' }); + const pending = new Map< + number, + { resolve: (result: SpatialExecutorResult) => void; reject: (error: Error) => void } + >(); + let nextId = 0; + + const rejectPending = (error: Error) => { + const settlers = [...pending.values()]; + pending.clear(); + for (const { reject } of settlers) { + reject(error); + } + }; + + worker.addEventListener('message', (event: MessageEvent) => { + const { id, ...result } = event.data; + const settler = pending.get(id); + pending.delete(id); + settler?.resolve(result); + }); + worker.addEventListener('error', (event) => { + rejectPending(new Error(event.message || 'Spatial adjustment worker failed.')); + }); + worker.addEventListener('messageerror', () => { + rejectPending(new Error('Spatial adjustment worker reply could not be deserialized.')); + }); + + return { + dispose: () => { + worker.terminate(); + rejectPending(new Error('Spatial adjustment worker was terminated.')); + }, + run: (job) => + new Promise((resolve, reject) => { + const id = nextId++; + const request: SpatialWorkerRequest = { ...job, id }; + pending.set(id, { reject, resolve }); + try { + worker.postMessage(request, [job.buffer]); + } catch (error) { + // The message never left (e.g. a detached buffer cannot be transferred); the worker is still healthy. + pending.delete(id); + resolve({ + buffer: job.buffer, + message: error instanceof Error ? error.message : String(error), + status: 'error', + }); + } + }), + }; +}; + +interface Job { + readonly request: SpatialRunRequest; + readonly buffer: ArrayBuffer; + readonly release: () => void; + readonly resolve: (result: SpatialRunResult) => void; + /** Set once the caller has its result; a running job's late executor result is then discarded. */ + answered: boolean; +} + +const ABORTED: SpatialRunResult = { status: 'aborted' }; +const SUPERSEDED: SpatialRunResult = { status: 'superseded' }; +const OVER_BUDGET: SpatialRunResult = { status: 'over-budget' }; +const TOO_LARGE: SpatialRunResult = { status: 'too-large' }; +const ERROR: SpatialRunResult = { status: 'error' }; + +/** + * One job runs at a time. Queued jobs run `now` before `display`, FIFO within a priority. Each accepted job holds a + * reservation for its pixel buffer until the buffer comes back from the executor (or the executor is disposed), so an + * aborted running job keeps its reservation, and the single run slot, until the worker finishes. + */ +export const createSpatialAdjustmentService = ({ + createExecutor = createWorkerExecutor, + reserve, +}: SpatialAdjustmentServiceOptions): SpatialAdjustmentService => { + const queues: Record = { display: [], now: [] }; + let running: Job | null = null; + let executor: SpatialAdjustmentExecutor | null = null; + let idleTimer: ReturnType | null = null; + let disposed = false; + + const answer = (job: Job, result: SpatialRunResult) => { + if (!job.answered) { + job.answered = true; + job.resolve(result); + } + }; + + /** Ends a job that never reached the executor. */ + const finishQueued = (job: Job, result: SpatialRunResult) => { + answer(job, result); + job.release(); + }; + + const removeQueued = (job: Job): boolean => { + const queue = queues[job.request.priority]; + const index = queue.indexOf(job); + if (index < 0) { + return false; + } + queue.splice(index, 1); + return true; + }; + + const dropExecutor = () => { + const current = executor; + executor = null; + current?.dispose(); + }; + + const scheduleIdleTermination = () => { + idleTimer = setTimeout(() => { + idleTimer = null; + dropExecutor(); + }, SPATIAL_WORKER_IDLE_MS); + }; + + const execute = async (job: Job) => { + const { height, steps, width } = job.request; + try { + executor ??= createExecutor(); + const result = await executor.run({ buffer: job.buffer, height, steps, width }); + answer( + job, + result.status === 'ok' + ? { pixels: new Uint8ClampedArray(result.buffer), status: 'ok' } + : result.status === 'too-large' + ? TOO_LARGE + : ERROR + ); + } catch { + answer(job, ERROR); + if (!disposed) { + dropExecutor(); + } + } finally { + job.release(); + running = null; + pump(); + } + }; + + const pump = () => { + if (disposed || running) { + return; + } + const next = queues.now.shift() ?? queues.display.shift(); + if (!next) { + if (executor && !idleTimer) { + scheduleIdleTermination(); + } + return; + } + if (idleTimer) { + clearTimeout(idleTimer); + idleTimer = null; + } + running = next; + void execute(next); + }; + + const cancelAll = () => { + const queued = [...queues.now, ...queues.display]; + queues.now.length = 0; + queues.display.length = 0; + for (const job of queued) { + finishQueued(job, ABORTED); + } + if (running) { + answer(running, ABORTED); + } + }; + + const run = (request: SpatialRunRequest): Promise => { + const { height, pixels, signal, width } = request; + const { buffer } = pixels; + // Transfer moves the whole buffer: a partial view would detach memory its caller still uses. + if ( + !(buffer instanceof ArrayBuffer) || + pixels.length !== width * height * 4 || + pixels.byteLength !== buffer.byteLength + ) { + throw new RangeError('Spatial adjustment pixels must be width * height RGBA8 bytes owning an ArrayBuffer.'); + } + if (disposed || signal?.aborted) { + return Promise.resolve(ABORTED); + } + // The raster budget accounts canvas surfaces resident on the main thread. Worker scratch is transient and bounded + // separately: one job at a time, at most MAX_SPATIAL_PIXELS * SPATIAL_SCRATCH_BYTES_PER_PIXEL (about 400 MB at the + // cap), freed when the job ends or the worker idles out. + const reservation = reserve(width * height * 4); + if (reservation.status !== 'ok') { + return Promise.resolve(OVER_BUDGET); + } + + return new Promise((resolve) => { + const onAbort = () => { + if (removeQueued(job)) { + finishQueued(job, ABORTED); + } else { + answer(job, ABORTED); + } + }; + const job: Job = { + answered: false, + buffer, + release: () => { + signal?.removeEventListener('abort', onAbort); + reservation.lease.release(); + }, + request, + resolve, + }; + signal?.addEventListener('abort', onAbort, { once: true }); + + const queue = queues[request.priority]; + const replaced = [...queues.now, ...queues.display].find((queued) => queued.request.slotKey === request.slotKey); + if (replaced && replaced.request.priority === request.priority) { + // Take over the replaced job's place so a slot that keeps changing is not pushed behind later slots. + queue.splice(queue.indexOf(replaced), 1, job); + } else { + if (replaced) { + removeQueued(replaced); + } + queue.push(job); + } + if (replaced) { + finishQueued(replaced, SUPERSEDED); + } + pump(); + }); + }; + + return { + dispose: () => { + if (disposed) { + return; + } + disposed = true; + cancelAll(); + if (idleTimer) { + clearTimeout(idleTimer); + idleTimer = null; + } + dropExecutor(); + }, + run, + }; +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialSteps.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialSteps.ts new file mode 100644 index 00000000000..6a29eb2c49f --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/spatialSteps.ts @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: Apache-2.0 +export type SpatialAdjustmentStep = + | { readonly kind: 'clarity'; readonly amount: number; readonly size: number } + | { readonly kind: 'texture'; readonly amount: number; readonly size: number } + | { readonly kind: 'dehaze'; readonly amount: number }; + +/** Largest alpha-bounds area one evaluation accepts; bounds worker scratch memory. */ +export const MAX_SPATIAL_PIXELS = 4096 * 4096; + +/** + * Worst-case scratch one evaluation allocates per alpha-bounds pixel: five float32 planes (lightness, adjusted + * lightness and three filter planes) plus int32 fill labels when the bounds contain hidden pixels. + */ +export const SPATIAL_SCRATCH_BYTES_PER_PIXEL = 24; + +export type SpatialEvaluation = { readonly status: 'ok' } | { readonly status: 'too-large' }; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatialComposite.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatialComposite.browser.test.ts new file mode 100644 index 00000000000..468e04af04d --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatialComposite.browser.test.ts @@ -0,0 +1,337 @@ +/** + * Real-worker checks that generation and export composites draw exactly what the settled display shows for spatial + * stacks, layers and groups alike, whatever the bbox. + */ + +import type { CanvasAdjustmentsContract, CanvasRasterLayerContractV2 } from '@workbench/canvas-engine/contracts'; +import type { GroupSurfaceResult } from '@workbench/canvas-engine/render/groupSurfaceCache'; +import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; +import type { Rect } from '@workbench/canvas-engine/types'; + +import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; +import { groupContract, stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { identity } from '@workbench/canvas-engine/math/mat2d'; +import { createAdjustedSurfaceCache } from '@workbench/canvas-engine/render/adjustedSurfaceCache'; +import { compositeDocument } from '@workbench/canvas-engine/render/compositor'; +import { createGroupSurfaceCache } from '@workbench/canvas-engine/render/groupSurfaceCache'; +import { createLayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; +import { createDomRasterBackend } from '@workbench/canvas-engine/render/raster'; +import { planBaseRasterComposite, renderRasterComposite } from '@workbench/canvas-engine/render/rasterComposite'; +import { + createSpatialAdjustmentService, + type SpatialAdjustmentService, +} from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import { createCompositeDedupeCache, executeCompositePlan } from '@workbench/canvas-operations/compositeForGeneration'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +const WIDTH = 64; +const HEIGHT = 64; +const BBOX: Rect = { height: HEIGHT, width: WIDTH, x: 0, y: 0 }; +const MEMBER: Rect = { height: 30, width: 40, x: 0, y: 0 }; +const TRANSLATED = { rotation: 0, scaleX: 1, scaleY: 1, x: 9, y: 7 }; +const ROTATED_DOWNSCALED = { rotation: Math.PI / 7, scaleX: 0.6, scaleY: 0.5, x: 22, y: 4 }; + +/** Per-pixel runs around two spatial runs; noise places its grain by pixel, so a misplaced origin shows. */ +const STACK: CanvasAdjustmentsContract = [ + { brightness: 0.1, contrast: 0.05, id: 'bc', isEnabled: true, type: 'brightness-contrast' }, + { amount: 70, id: 'clarity', isEnabled: true, size: 12, type: 'clarity' }, + { + amount: 0.15, + distribution: 'gaussian', + grain: 'color', + id: 'grain', + isEnabled: true, + response: 'flat', + seed: 7, + size: 1, + type: 'noise', + }, + { amount: 60, id: 'texture', isEnabled: true, size: 4, type: 'texture' }, +]; +const GROUP_STACK: CanvasAdjustmentsContract = [ + { amount: 80, id: 'gclarity', isEnabled: true, size: 10, type: 'clarity' }, + { amount: 40, id: 'gdehaze', isEnabled: true, type: 'dehaze' }, +]; + +const raster = (id: string, overrides: Partial = {}): CanvasRasterLayerContractV2 => ({ + blendMode: 'normal', + id, + isEnabled: true, + isLocked: false, + name: id, + opacity: 1, + source: { bitmap: { height: MEMBER.height, imageName: id, width: MEMBER.width }, type: 'paint' }, + transform: TRANSLATED, + type: 'raster', + ...overrides, +}); + +const docWith = (nodes: Parameters[0]) => ({ + background: 'transparent' as const, + bbox: BBOX, + height: HEIGHT, + selectedLayerId: null, + stacks: stacksFrom(nodes), + version: 3 as const, + width: WIDTH, +}); + +const worstDifference = (a: Uint8ClampedArray, b: Uint8ClampedArray): number => { + expect(a.length).toBe(b.length); + let worst = 0; + for (let i = 0; i < a.length; i += 1) { + worst = Math.max(worst, Math.abs(a[i]! - b[i]!)); + } + return worst; +}; + +const pixelsOf = (surface: RasterSurface, rect: Rect = { ...BBOX, height: surface.height, width: surface.width }) => + surface.ctx.getImageData(rect.x, rect.y, rect.width, rect.height).data; + +const services: SpatialAdjustmentService[] = []; +afterEach(() => { + for (const service of services.splice(0)) { + service.dispose(); + } +}); + +/** Layers `ids` hold a gradient with a transparent border and a dark block: edges for the kernels, and a sub-domain. */ +const sceneFor = (ids: readonly string[]) => { + const backend = createDomRasterBackend(); + const caches = createLayerCacheStore(backend); + for (const [index, id] of ids.entries()) { + const ctx = caches.growToRect(id, MEMBER).surface.ctx; + const gradient = ctx.createLinearGradient(0, 0, MEMBER.width, MEMBER.height); + gradient.addColorStop(0, index === 0 ? '#1d4ed8' : '#15803d'); + gradient.addColorStop(0.5, '#e2e8f0'); + gradient.addColorStop(1, '#f59e0b'); + ctx.fillStyle = gradient; + ctx.fillRect(3, 2, MEMBER.width - 7, MEMBER.height - 5); + ctx.fillStyle = '#0f172a'; + ctx.fillRect(14, 9, 10, 8); + caches.publishPixels(id); + } + const memory = new RasterMemoryBudgetController({ budgetBytes: 64 * 1024 * 1024 }); + const service = createSpatialAdjustmentService({ + reserve: (bytes) => memory.reserveOperation(bytes, { purpose: 'spatial-adjustment' }), + }); + services.push(service); + const getLayerSurface = (layerId: string) => { + const entry = caches.get(layerId)!; + return Promise.resolve({ rect: entry.rect, release: () => undefined, surface: entry.surface }); + }; + /** The export composite: no display copy to reuse, so every spatial stack runs a job. */ + const exported = (document: ReturnType, bbox: Rect) => + renderRasterComposite(planBaseRasterComposite(document, bbox), { + backend, + getLayerSurface, + runSpatial: (request) => service.run(request), + }); + return { backend, caches, exported, getLayerSurface, memory, service }; +}; + +/** Composites until `done` holds, as frames would while results land. */ +const settle = async (composite: () => void, done: () => boolean) => { + await vi.waitFor( + () => { + composite(); + expect(done()).toBe(true); + }, + { interval: 20, timeout: 10_000 } + ); +}; + +/** The display of a document whose layers carry their own stacks, once every spatial result has landed. */ +const settledLayerDisplay = async (scene: ReturnType, document: ReturnType) => { + const display = createAdjustedSurfaceCache(scene.backend, undefined, { + spatial: { run: (request) => scene.service.run(request) }, + }); + const layers = document.stacks.raster as CanvasRasterLayerContractV2[]; + const screen = scene.backend.createSurface(WIDTH, HEIGHT); + const composite = () => { + screen.ctx.clearRect(0, 0, WIDTH, HEIGHT); + compositeDocument(screen, document, scene.caches, identity(), { + adjustedSurface: (layer, entry) => + layer.type === 'raster' ? display.get(layer.id, entry, layer.adjustments) : null, + backend: scene.backend, + }); + }; + await settle(composite, () => + layers.every((layer) => display.peek(layer.id, scene.caches.get(layer.id)!, layer.adjustments) !== null) + ); + return { display, screen }; +}; + +/** The display of spatial groups over raw members, once the outermost group's pixels are exact. */ +const settledGroupDisplay = async ( + scene: ReturnType, + document: ReturnType, + outermost: string, + inner: readonly string[] = [] +) => { + const groupSurfaces = createGroupSurfaceCache({ + createSurface: (w, h) => scene.backend.createSurface(w, h), + damageSince: (id, version) => scene.caches.damageSince(id, version), + getAdjustedSurface: () => ({ state: 'exact', surface: null }), + getCacheEntry: (id) => scene.caches.get(id), + spatial: { + // Wired as the engine does: a landing redraws the groups holding the owner. + onSettle: (groupId) => { + if (inner.includes(groupId)) { + groupSurfaces.invalidate([outermost]); + } + }, + run: (request) => scene.service.run(request), + }, + }); + const screen = scene.backend.createSurface(WIDTH, HEIGHT); + let state: GroupSurfaceResult['state'] | undefined; + const composite = () => { + screen.ctx.clearRect(0, 0, WIDTH, HEIGHT); + compositeDocument(screen, document, scene.caches, identity(), { + backend: scene.backend, + groupSurface: (scope, members, matrices, content) => { + const result = groupSurfaces.get(scope, members, matrices, content); + state = scope.id === outermost ? result?.state : state; + return result; + }, + }); + }; + await settle(composite, () => state === 'exact'); + return screen; +}; + +describe('spatial layer composites', () => { + it.each([ + ['translated', TRANSLATED], + ['rotated and downscaled', ROTATED_DOWNSCALED], + ])('exports a %s layer as the settled display shows it', async (_, transform) => { + const scene = sceneFor(['m']); + const layer = raster('m', { adjustments: STACK, transform }); + const document = docWith([layer]); + const { display, screen } = await settledLayerDisplay(scene, document); + + const fromJob = await scene.exported(document, BBOX); + const fromDisplayCopy = await renderRasterComposite(planBaseRasterComposite(document, BBOX), { + adjustedSurface: (layerId, surface, adjustments) => { + const entry = scene.caches.get(layerId)!; + return entry.surface === surface ? display.peek(layerId, entry, adjustments) : null; + }, + backend: scene.backend, + getLayerSurface: scene.getLayerSurface, + }); + + expect(worstDifference(pixelsOf(fromJob), pixelsOf(screen))).toBeLessThanOrEqual(1); + expect(worstDifference(pixelsOf(fromDisplayCopy), pixelsOf(screen))).toBe(0); + // Leaving the spatial entries out would be visible: the per-pixel interim differs from what landed. + const interim = scene.backend.createSurface(WIDTH, HEIGHT); + const perPixel = createAdjustedSurfaceCache(scene.backend); + compositeDocument(interim, document, scene.caches, identity(), { + adjustedSurface: (_layer, entry) => perPixel.get('m', entry, STACK), + backend: scene.backend, + }); + expect(worstDifference(pixelsOf(fromJob), pixelsOf(interim))).toBeGreaterThan(8); + expect(scene.memory.snapshot().reservedBytes).toBe(0); + }); + + it.each([ + ['translated', TRANSLATED], + ['rotated and downscaled', ROTATED_DOWNSCALED], + ])('draws a %s layer identically over any bbox that crops it', async (_, transform) => { + const scene = sceneFor(['m']); + const document = docWith([raster('m', { adjustments: STACK, transform })]); + const crop: Rect = { height: 14, width: 19, x: 17, y: 12 }; + + const whole = await scene.exported(document, BBOX); + const cropped = await scene.exported(document, crop); + + expect(worstDifference(pixelsOf(cropped), pixelsOf(whole, crop))).toBe(0); + }); +}); + +describe('spatial group composites', () => { + /** Two members, one partly outside a cropping bbox, under a spatial group. */ + const groupDocument = () => + docWith([ + groupContract('g', [raster('m'), raster('n', { opacity: 0.7, transform: { ...TRANSLATED, x: 21, y: 26 } })], { + adjustments: GROUP_STACK, + }), + ]); + + it('exports a group as the settled display shows it: its stack on the union of its members', async () => { + const scene = sceneFor(['m', 'n']); + const document = groupDocument(); + const screen = await settledGroupDisplay(scene, document, 'g'); + + expect(worstDifference(pixelsOf(await scene.exported(document, BBOX)), pixelsOf(screen))).toBe(0); + }); + + it('draws a group identically over any bbox that crops it', async () => { + const scene = sceneFor(['m', 'n']); + const document = groupDocument(); + const crop: Rect = { height: 20, width: 24, x: 18, y: 20 }; + + const whole = await scene.exported(document, BBOX); + const cropped = await scene.exported(document, crop); + + expect(worstDifference(pixelsOf(cropped), pixelsOf(whole, crop))).toBe(0); + }); + + it('exports nested spatial groups as the settled display shows them', async () => { + const scene = sceneFor(['m', 'n']); + const document = docWith([ + groupContract( + 'outer', + [ + groupContract('inner', [raster('m')], { + adjustments: [{ amount: 90, id: 'texture', isEnabled: true, size: 4, type: 'texture' }], + }), + raster('n', { transform: { ...TRANSLATED, x: 24, y: 30 } }), + ], + { adjustments: GROUP_STACK } + ), + ]); + const screen = await settledGroupDisplay(scene, document, 'outer', ['inner']); + + expect(worstDifference(pixelsOf(await scene.exported(document, BBOX)), pixelsOf(screen))).toBe(0); + }); +}); + +describe('generation composite', () => { + it('uploads the pixels the export composite draws', async () => { + const scene = sceneFor(['m', 'n']); + const document = docWith([ + raster('m', { adjustments: STACK }), + groupContract('g', [raster('n', { transform: { ...TRANSLATED, x: 20, y: 24 } })], { adjustments: GROUP_STACK }), + ]); + const entry = planBaseRasterComposite(document, BBOX); + let uploaded: Blob | null = null; + + await executeCompositePlan( + { bbox: BBOX, entries: [entry] }, + { + backend: { + createSurface: (width, height) => scene.backend.createSurface(width, height), + encodeSurface: (surface) => scene.backend.encodeSurface(surface, 'image/png'), + }, + dedupe: createCompositeDedupeCache(), + // Generation composites detached copies, never the live caches the display adjusts. + getLayerSurface: async (layerId) => { + const live = await scene.getLayerSurface(layerId); + const detached = scene.backend.createSurface(live.rect.width, live.rect.height); + detached.ctx.drawImage(live.surface.canvas, 0, 0); + return { ...live, surface: detached }; + }, + runSpatial: (request) => scene.service.run(request), + uploadImage: (blob) => { + uploaded = blob; + return Promise.resolve({ height: HEIGHT, imageName: 'composite.png', width: WIDTH }); + }, + } + ); + + const decoded = scene.backend.createSurface(WIDTH, HEIGHT); + decoded.ctx.drawImage(await createImageBitmap(uploaded!), 0, 0); + expect(worstDifference(pixelsOf(decoded), pixelsOf(await scene.exported(document, BBOX)))).toBe(0); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatialDisplay.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatialDisplay.ts new file mode 100644 index 00000000000..890dcc24157 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatialDisplay.ts @@ -0,0 +1,484 @@ +/** + * The asynchronous display of spatially adjusted pixels, shared by layers (`adjustedSurfaceCache.ts`) and groups + * (`groupSurfaceCache.ts`). Each owner shows an interim until its stack's evaluation lands from the spatial worker; + * the canvas-engine ARCHITECTURE "Spatial adjustments" section states the interim, install and scheduling rules. + */ + +import type { CanvasAdjustmentsContract } from '@workbench/canvas-engine/contracts'; +import type { + SpatialRunRequest, + SpatialRunResult, +} from '@workbench/canvas-engine/render/spatial/spatialAdjustmentService'; +import type { Rect } from '@workbench/canvas-engine/types'; + +import { intersect, roundOut } from '@workbench/canvas-engine/math/rect'; + +import type { RasterSurface } from './raster'; + +import { type AdjustmentOrigin, adjustmentsKey, applyAdjustments, perPixelAdjustments } from './adjustments'; +import { evaluateAdjustmentStack } from './exactAdjustments'; + +/** Why an owner's displayed spatial adjustments are not exact; absent means they are (or there are none). */ +export type AdjustedSurfaceStatus = 'computing' | 'too-large' | 'over-budget' | 'error'; + +/** + * How final displayed pixels are: `exact`; `pending` while a spatial job will still change them; `inexact` when none + * can run until the owner's target changes (a failure, or a placeholder source). + */ +export type DisplayedPixelsState = 'exact' | 'pending' | 'inexact'; + +const STATE_RANK: Record = { exact: 0, inexact: 1, pending: 2 }; + +/** The less final of two states: pending outranks inexact, which outranks exact. */ +export const worseDisplayedState = (a: DisplayedPixelsState, b: DisplayedPixelsState): DisplayedPixelsState => + STATE_RANK[a] >= STATE_RANK[b] ? a : b; + +/** Content edits start a spatial job only after this much quiet, so a stroke does not queue one per tick. */ +export const SPATIAL_CONTENT_DEBOUNCE_MS = 120; + +export interface SpatialDisplayOptions { + /** Evaluates spatial steps off the main thread (`SpatialAdjustmentService.run`). */ + readonly run: (request: SpatialRunRequest) => Promise; + /** A spatial result changed the owner's displayed pixels without a source version change. */ + readonly onSettle?: (ownerId: string) => void; + /** The owner's spatial status changed; null when exact or no longer spatial. */ + readonly onStatus?: (ownerId: string, status: AdjustedSurfaceStatus | null) => void; +} + +/** What one display request asked for; a job snapshots it. */ +export interface SpatialDisplayTarget { + /** The owner's unadjusted pixels; a job reads all of them. */ + readonly source: RasterSurface; + readonly version: number; + /** `adjustmentsKey` of the stack. */ + readonly key: string; + readonly adjustments: CanvasAdjustmentsContract; + /** Where the source's (0,0) sits in the stack's reference space. */ + readonly origin: AdjustmentOrigin; + /** + * Why no job may start yet: a `placeholder` source with no real pixels (reported idle), or `inputs` whose own + * spatial results are still computing (reported computing). + */ + readonly hold: 'placeholder' | 'inputs' | null; + /** The source's current version, or null once the owner no longer draws from `source`. */ + readonly liveVersion: () => number | null; + /** Source-local damage since `version`, or null when unknown. */ + readonly damageSince: (version: number) => Rect | null; + /** The display surface currently guarded for this target, or null once it was evicted or replaced. */ + readonly shown: () => RasterSurface | null; +} + +interface TargetKey { + readonly source: RasterSurface; + readonly version: number; + readonly key: string; +} + +interface SpatialJob extends TargetKey { + readonly width: number; + readonly height: number; + readonly seq: number; + readonly controller: AbortController; +} + +interface SpatialSlot { + /** The latest display request. */ + target: SpatialDisplayTarget | null; + /** Source version the surface's pixels reflect; null until this slot painted the surface. */ + shownVersion: number | null; + /** Provenance of the installed spatial result; null while the surface is a per-pixel-only interim. */ + shown: { readonly seq: number; readonly version: number; readonly key: string } | null; + /** At most one job per owner: a later target waits for it, so no readback is wasted while it runs. */ + job: SpatialJob | null; + timer: ReturnType | null; + /** `adjustmentsKey` of the per-pixel part a whole interim painted, while `shown` is null and no patch differs. */ + interimKey: string | null; + /** The last terminal outcome, kept so an unchanged target is not retried every frame. */ + failure: SpatialFailure | null; + status: AdjustedSurfaceStatus | null; +} + +interface SpatialFailure extends TargetKey { + readonly status: Exclude; +} + +/** Per-owner spatial display state returned by {@link createSpatialDisplay}. */ +export interface SpatialDisplay { + /** + * Brings `surface` to the interim for `target`: a parameter change keeps the shown pixels, known damage is + * re-patched with the per-pixel part only, and anything else (or `reusable` false) repaints the whole surface + * per-pixel. Returns whether it repainted the whole surface. + */ + paintInterim(ownerId: string, surface: RasterSurface, reusable: boolean, target: SpatialDisplayTarget): boolean; + /** Records `target` as the owner's latest request and schedules a job toward it. */ + request(ownerId: string, target: SpatialDisplayTarget, rebuilt: boolean): void; + /** Whether the owner shows the installed result for exactly `target`. */ + isExact(ownerId: string, target: TargetKey): boolean; + /** How final the owner's displayed pixels are for `target`. */ + state(ownerId: string, target: TargetKey): DisplayedPixelsState; + status(ownerId: string): AdjustedSurfaceStatus | null; + /** Forgets an owner, abandoning its job and clearing its status; false when it had no state. */ + drop(ownerId: string): boolean; + /** + * Forgets every owner with a job running or scheduled, returning their ids so callers drop the interims those jobs + * would have replaced. Exact, failed and waiting owners keep their state. + */ + cancel(): string[]; + ownerIds(): string[]; +} + +const sameTarget = (a: TargetKey, b: TargetKey): boolean => + a.source === b.source && a.version === b.version && a.key === b.key; + +/** Whether a failure still applies to `target`: too-large depends only on the content's alpha bounds, not the stack. */ +const failureHolds = (failure: SpatialFailure, target: TargetKey): boolean => + failure.source === target.source && + failure.version === target.version && + (failure.status === 'too-large' || failure.key === target.key); + +const interimKeyOf = (target: SpatialDisplayTarget): string => adjustmentsKey(perPixelAdjustments(target.adjustments)); + +/** Damage clamped to a `width` × `height` surface: `null` when unknown, `'none'` when it misses the surface. */ +export const clampDamage = (damaged: Rect | null, width: number, height: number): Rect | 'none' | null => { + if (!damaged) { + return null; + } + const clamped = intersect(roundOut(damaged), { height, width, x: 0, y: 0 }); + return clamped && clamped.width > 0 && clamped.height > 0 ? clamped : 'none'; +}; + +const resetContext = (surface: RasterSurface): RasterSurface['ctx'] => { + const ctx = surface.ctx; + ctx.setTransform(1, 0, 0, 1, 0, 0); + ctx.globalAlpha = 1; + ctx.globalCompositeOperation = 'source-over'; + return ctx; +}; + +/** Redraws `region` of `surface` from the same region of `source`, with `adjustments` applied at its place. */ +export const paintAdjustedRegion = ( + surface: RasterSurface, + source: RasterSurface, + region: Rect, + adjustments: CanvasAdjustmentsContract | undefined, + origin: AdjustmentOrigin +): void => { + const ctx = resetContext(surface); + ctx.clearRect(region.x, region.y, region.width, region.height); + ctx.drawImage( + source.canvas, + region.x, + region.y, + region.width, + region.height, + region.x, + region.y, + region.width, + region.height + ); + const imageData = ctx.getImageData(region.x, region.y, region.width, region.height); + applyAdjustments(imageData, adjustments, { x: origin.x + region.x, y: origin.y + region.y }); + ctx.putImageData(imageData, region.x, region.y); +}; + +/** + * Creates the state machine; `slotKeyPrefix` keeps owners of different kinds from superseding each other's queued + * jobs in the shared service. + */ +export const createSpatialDisplay = (options: SpatialDisplayOptions, slotKeyPrefix: string): SpatialDisplay => { + const slots = new Map(); + let nextSeq = 0; + + const slotFor = (ownerId: string): SpatialSlot => { + let slot = slots.get(ownerId); + if (!slot) { + slot = { + failure: null, + interimKey: null, + job: null, + shown: null, + shownVersion: null, + status: null, + target: null, + timer: null, + }; + slots.set(ownerId, slot); + } + return slot; + }; + + /** Repaints all of `surface` with `target`'s per-pixel part only. */ + const paintWholeInterim = (slot: SpatialSlot, surface: RasterSurface, target: SpatialDisplayTarget): void => { + const { height, width } = surface; + paintAdjustedRegion( + surface, + target.source, + { height, width, x: 0, y: 0 }, + perPixelAdjustments(target.adjustments), + target.origin + ); + slot.shownVersion = target.version; + slot.shown = null; + slot.interimKey = interimKeyOf(target); + }; + + /** + * A target that cannot run shows its per-pixel part alone, as no result will replace what a parameter change kept. + * Returns whether it repainted. + */ + const showPerPixelOnly = (slot: SpatialSlot): boolean => { + const target = slot.target!; + if (slot.shown === null && slot.interimKey === interimKeyOf(target)) { + return false; + } + const surface = target.shown(); + if (!surface || surface.width !== target.source.width || surface.height !== target.source.height) { + return false; + } + paintWholeInterim(slot, surface, target); + return true; + }; + + const setStatus = (ownerId: string, slot: SpatialSlot, status: AdjustedSurfaceStatus | null): void => { + if (slot.status !== status) { + slot.status = status; + options.onStatus?.(ownerId, status); + } + }; + + const clearTimer = (slot: SpatialSlot): void => { + if (slot.timer !== null) { + clearTimeout(slot.timer); + slot.timer = null; + } + }; + + const abandonJob = (slot: SpatialSlot): void => { + slot.job?.controller.abort(); + slot.job = null; + }; + + const drop = (ownerId: string): boolean => { + const slot = slots.get(ownerId); + if (!slot) { + return false; + } + slots.delete(ownerId); + clearTimer(slot); + abandonJob(slot); + setStatus(ownerId, slot, null); + return true; + }; + + const isExact = (slot: SpatialSlot): boolean => + slot.shown !== null && + slot.target !== null && + slot.shownVersion === slot.target.version && + slot.shown.version === slot.target.version && + slot.shown.key === slot.target.key; + + /** + * Writes a result only when it is newer than the shown one, came from the same source surface and from a version + * at or before the shown one whose later damage is known; that damage is then re-patched per-pixel. + */ + const install = (slot: SpatialSlot, job: SpatialJob, pixels: Uint8ClampedArray): boolean => { + const target = slot.target; + const live = target?.liveVersion() ?? null; + if ( + !target || + live === null || + target.source !== job.source || + slot.shownVersion === null || + job.version > slot.shownVersion || + (slot.shown !== null && job.seq <= slot.shown.seq) + ) { + return false; + } + const surface = target.shown(); + const { height, width } = job; + if ( + !surface || + surface.width !== width || + surface.height !== height || + job.source.width !== width || + job.source.height !== height + ) { + return false; + } + const patch = live === job.version ? 'none' : clampDamage(target.damageSince(job.version), width, height); + if (patch === null) { + return false; + } + const ctx = resetContext(surface); + const image = ctx.createImageData(width, height); + image.data.set(pixels); + ctx.putImageData(image, 0, 0); + if (patch !== 'none') { + paintAdjustedRegion(surface, target.source, patch, perPixelAdjustments(target.adjustments), target.origin); + } + slot.shown = { key: job.key, seq: job.seq, version: job.version }; + return true; + }; + + const finish = (ownerId: string, slot: SpatialSlot, job: SpatialJob, outcome: SpatialRunResult): void => { + if (slots.get(ownerId) !== slot || slot.job !== job) { + return; + } + slot.job = null; + if (outcome.status === 'ok') { + if (install(slot, job, outcome.pixels)) { + options.onSettle?.(ownerId); + } else if (slot.target && !slot.target.shown()) { + // Evicted while computing: the next display request rebuilds and reschedules. + setStatus(ownerId, slot, null); + return; + } + } else if (outcome.status !== 'aborted' && outcome.status !== 'superseded') { + slot.failure = { key: job.key, source: job.source, status: outcome.status, version: job.version }; + if (slot.target && failureHolds(slot.failure, slot.target) && showPerPixelOnly(slot)) { + options.onSettle?.(ownerId); + } + } + if (slot.timer === null) { + schedule(ownerId, slot, true); + } + }; + + const start = (ownerId: string, slot: SpatialSlot): void => { + const target = slot.target!; + // The source moved on without a display request: that request will reschedule. + if (target.liveVersion() !== target.version) { + setStatus(ownerId, slot, null); + return; + } + const { height, width } = target.source; + const job: SpatialJob = { + controller: new AbortController(), + height, + key: target.key, + seq: (nextSeq += 1), + source: target.source, + version: target.version, + width, + }; + slot.job = job; + let outcome: Promise; + try { + const { data } = target.source.ctx.getImageData(0, 0, width, height); + // Display jobs share one service slot per owner, so a newer target supersedes a queued older one. + outcome = evaluateAdjustmentStack({ data, height, width }, target.adjustments, target.origin, { + priority: 'display', + run: options.run, + signal: job.controller.signal, + slotKey: `${slotKeyPrefix}${ownerId}`, + }); + } catch { + outcome = Promise.resolve({ status: 'error' }); + } + void outcome.then( + (result) => finish(ownerId, slot, job, result), + () => finish(ownerId, slot, job, { status: 'error' }) + ); + }; + + /** Brings the slot toward its target: idle when exact, else one job now or after the content debounce. */ + const schedule = (ownerId: string, slot: SpatialSlot, immediate: boolean): void => { + const target = slot.target!; + if (isExact(slot)) { + clearTimer(slot); + abandonJob(slot); + slot.failure = null; + setStatus(ownerId, slot, null); + return; + } + if (target.hold) { + // A placeholder is never worth a job, and inputs still computing would waste one; a later request starts it. + clearTimer(slot); + setStatus(ownerId, slot, target.hold === 'inputs' ? 'computing' : null); + return; + } + if (slot.failure) { + if (failureHolds(slot.failure, target)) { + // No job will replace what this request shows, so it shows the per-pixel part now, as the caller draws it. + clearTimer(slot); + showPerPixelOnly(slot); + setStatus(ownerId, slot, slot.failure.status); + return; + } + slot.failure = null; + } + setStatus(ownerId, slot, 'computing'); + if (slot.job && sameTarget(slot.job, target)) { + clearTimer(slot); + return; + } + clearTimer(slot); + if (immediate) { + if (!slot.job) { + start(ownerId, slot); + } + return; + } + slot.timer = setTimeout(() => { + slot.timer = null; + if (!slot.job && slots.get(ownerId) === slot) { + schedule(ownerId, slot, true); + } + }, SPATIAL_CONTENT_DEBOUNCE_MS); + }; + + const isExactFor = (ownerId: string, target: TargetKey): boolean => { + const slot = slots.get(ownerId); + return slot?.target !== null && slot?.target !== undefined && sameTarget(slot.target, target) && isExact(slot); + }; + + return { + cancel: () => { + const cancelled = [...slots].filter(([, slot]) => slot.job || slot.timer !== null).map(([ownerId]) => ownerId); + for (const ownerId of cancelled) { + drop(ownerId); + } + return cancelled; + }, + drop, + isExact: isExactFor, + ownerIds: () => [...slots.keys()], + paintInterim: (ownerId, surface, reusable, target) => { + const slot = slotFor(ownerId); + const { height, width } = surface; + const from = reusable ? slot.shownVersion : null; + const damage = + from === null ? null : from === target.version ? 'none' : clampDamage(target.damageSince(from), width, height); + if (damage === null) { + paintWholeInterim(slot, surface, target); + return true; + } + slot.shownVersion = target.version; + if (damage !== 'none') { + paintAdjustedRegion(surface, target.source, damage, perPixelAdjustments(target.adjustments), target.origin); + if (slot.interimKey !== interimKeyOf(target)) { + slot.interimKey = null; + } + } + return false; + }, + request: (ownerId, target, rebuilt) => { + const slot = slotFor(ownerId); + const previous = slot.target; + slot.target = target; + if (!previous || rebuilt || !sameTarget(previous, target) || previous.hold !== target.hold) { + // Content edits wait for quiet; a parameter change, first pixels, released inputs, a new slot or a rebuilt + // surface start at once. + const contentOnly = + previous !== null && + previous.hold === null && + previous.key === target.key && + (previous.source !== target.source || previous.version !== target.version); + schedule(ownerId, slot, !contentOnly); + } + }, + state: (ownerId, target) => + isExactFor(ownerId, target) ? 'exact' : slots.get(ownerId)?.status === 'computing' ? 'pending' : 'inexact', + status: (ownerId) => slots.get(ownerId)?.status ?? null, + }; +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/compositeForGeneration.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/compositeForGeneration.ts index d9859d152e5..c539f17b70f 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/compositeForGeneration.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/compositeForGeneration.ts @@ -6,6 +6,7 @@ */ import type { CanvasImageUploadResult } from '@workbench/canvas-engine/document/imageUpload'; +import type { SpatialRunner } from '@workbench/canvas-engine/render/exactAdjustments'; import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Mat2d, Rect } from '@workbench/canvas-engine/types'; import type { @@ -120,6 +121,13 @@ export interface ExecuteCompositePlanDeps { | { status: 'over-budget'; requestedBytes: number; availableBytes: number }; /** Writes pixels back to a surface (default `putImageData`; injectable for tests). */ writeImageData?(surface: RasterSurface, imageData: ImageData, x: number, y: number): void; + /** + * Evaluates spatial adjustment runs. Without it a spatial stack refuses with `AdjustmentRefusedError` instead of + * compositing without its spatial entries. + */ + runSpatial?: SpatialRunner; + /** Cancels spatial jobs; an aborted composite throws an `AbortError`. */ + signal?: AbortSignal; } const reserveComposite = ( diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/createFromBbox.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/createFromBbox.ts index 498dac2526b..2b1245c0a51 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/createFromBbox.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/createFromBbox.ts @@ -1,5 +1,5 @@ import type { GalleryImage } from '@features/gallery'; -import type { CanvasLayerCapability } from '@workbench/canvas-engine/api'; +import type { AdjustmentRefusal, CanvasLayerCapability } from '@workbench/canvas-engine/api'; import type { CanvasImageUploadResult } from '@workbench/canvas-engine/document/imageUpload'; import type { CanvasProjectMutation } from '@workbench/canvasProjectMutations'; import type { Project } from '@workbench/projectContracts'; @@ -29,6 +29,7 @@ export type CreateFromBboxResult = | { status: 'created'; destination: CreateFromBboxLayerDestination; layerIds: string[] } | { status: 'uploaded'; image: CanvasImageUploadResult } | { status: 'empty' | 'stale' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal | { status: 'blocked' | 'stale-document' | 'stale-project' }; type CreateFromBboxEngine = CanvasCompositeExportEngine & { diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/createFromBboxNotice.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/createFromBboxNotice.test.ts index 441be9ff0b9..4bd383497e5 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/createFromBboxNotice.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/createFromBboxNotice.test.ts @@ -26,4 +26,9 @@ describe('getCreateFromBboxNotice', () => { ] as const)('maps %s to a %s notice', (status, kind, titleKey) => { expect(getCreateFromBboxNotice({ status })).toEqual({ kind, titleKey }); }); + + it('passes an adjustment refusal on as an error for the view to word', () => { + const refusal = { ownerId: 'g', ownerName: 'Sky', reason: 'too-large', status: 'adjustment-refused' } as const; + expect(getCreateFromBboxNotice(refusal)).toEqual({ kind: 'error', refusal }); + }); }); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/createFromBboxNotice.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/createFromBboxNotice.ts index 876a34f5a83..dd9349b2e10 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/createFromBboxNotice.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/createFromBboxNotice.ts @@ -1,19 +1,24 @@ +import type { AdjustmentRefusal } from '@workbench/canvas-engine/api'; + import type { CanvasImportNotice } from './canvasImportNotice'; import type { CreateFromBboxLayerDestination, CreateFromBboxResult } from './createFromBbox'; import { getCanvasImportNotice } from './canvasImportNotice'; -export interface CreateFromBboxNotice { - kind: 'error' | 'info' | 'success'; - titleKey: - | CanvasImportNotice['titleKey'] - | 'widgets.canvas.contextMenu.createdFromBbox' - | 'widgets.canvas.contextMenu.empty' - | 'widgets.canvas.contextMenu.notReady' - | 'widgets.canvas.contextMenu.stale'; - /** `destination` is itself a translation key naming the created entity. */ - options?: CanvasImportNotice['options'] | { destination: string }; -} +/** A translated notice, or a spatial adjustment refusal the view words like every other one. */ +export type CreateFromBboxNotice = + | { + kind: 'error' | 'info' | 'success'; + titleKey: + | CanvasImportNotice['titleKey'] + | 'widgets.canvas.contextMenu.createdFromBbox' + | 'widgets.canvas.contextMenu.empty' + | 'widgets.canvas.contextMenu.notReady' + | 'widgets.canvas.contextMenu.stale'; + /** `destination` is itself a translation key naming the created entity. */ + options?: CanvasImportNotice['options'] | { destination: string }; + } + | { kind: 'error'; refusal: AdjustmentRefusal }; const DESTINATION_LABEL_KEYS: Record = { control: 'widgets.canvas.import.control', @@ -42,6 +47,8 @@ export const getCreateFromBboxNotice = ( case 'not-ready': case 'over-budget': return { kind: 'info', titleKey: 'widgets.canvas.contextMenu.notReady' }; + case 'adjustment-refused': + return { kind: 'error', refusal: result }; case 'blocked': case 'stale-document': case 'stale-project': diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/exportCanvasComposite.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/exportCanvasComposite.ts index 927f15b56d4..f2f5bbf91b6 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/exportCanvasComposite.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/exportCanvasComposite.ts @@ -1,4 +1,5 @@ import type { + AdjustmentRefusal, CanvasDocumentCapability, CanvasExportCapability, CanvasLifecycleCapability, @@ -9,7 +10,8 @@ export type CanvasCompositeRegion = 'canvas' | 'bbox'; export type ExportCanvasCompositeResult = | { status: 'ok'; blob: Blob; rect: Rect } - | { status: 'empty' | 'stale' | 'not-ready' | 'over-budget' }; + | { status: 'empty' | 'stale' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal; export type CanvasCompositeExportEngine = { readonly document: CanvasDocumentCapability; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/filterOperationSession.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/filterOperationSession.test.ts index c25cf6208ec..9c71a35d550 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/filterOperationSession.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/filterOperationSession.test.ts @@ -274,6 +274,23 @@ describe('createFilterOperationSession', () => { expect(deps.runFilter).not.toHaveBeenCalled(); }); + it.each(['too-large', 'over-budget', 'error'] as const)( + "publishes a filter source whose Clarity, Texture or Dehaze refused %s as the source's adjustment code", + async (reason) => { + const deps = createDeps({ + exportPixels: vi.fn(() => + Promise.resolve({ ownerId: 'layer', ownerName: 'Layer', reason, status: 'adjustment-refused' as const }) + ), + }); + const session = createFilterOperationSession({ deps, guard, initialFilter: layer.filter!, layerType: 'raster' })!; + + expect(await session.process()).toBe('error'); + + expect(session.getSnapshot()).toMatchObject({ error: { code: `source-adjustment-${reason}` }, status: 'error' }); + expect(deps.runFilter).not.toHaveBeenCalled(); + } + ); + it('publishes a filter source the guard no longer matches as stale', async () => { const deps = createDeps({ isGuardCurrent: vi.fn(() => false) }); const session = createFilterOperationSession({ deps, guard, initialFilter: layer.filter!, layerType: 'raster' })!; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/filterOperationSession.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/filterOperationSession.ts index 54fdc986ac4..e10ac2f322e 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/filterOperationSession.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/filterOperationSession.ts @@ -180,11 +180,13 @@ export const createFilterOperationSession = ( const exported = await deps.exportPixels(); if (exported.status !== 'ok') { refusal = - exported.status === 'over-budget' - ? 'source-over-budget' - : exported.status === 'aborted' - ? 'not-ready' - : exported.status; + exported.status === 'adjustment-refused' + ? `source-adjustment-${exported.reason}` + : exported.status === 'over-budget' + ? 'source-over-budget' + : exported.status === 'aborted' + ? 'not-ready' + : exported.status; throw new Error(`The filter source is ${exported.status}.`); } try { diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/generationComposite.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/generationComposite.test.ts index 2d44ae27b51..07787c3e188 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/generationComposite.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/generationComposite.test.ts @@ -579,6 +579,33 @@ describe('composeForGeneration', () => { } }); + it("refuses with the layer's name when its spatial adjustments cannot run, uploading nothing", async () => { + const layer = { + ...rasterLayer('base'), + adjustments: [{ amount: 60, id: 'dehaze', isEnabled: true, type: 'dehaze' as const }], + name: 'Portrait', + }; + const harness = makeHost(makeDoc([layer])); + const executorDeps = harness.host.getCompositeExecutorDeps(); + const runSpatial = vi.fn(() => Promise.resolve({ status: 'too-large' as const })); + harness.host.getCompositeExecutorDeps = () => ({ ...executorDeps, runSpatial }); + const signal = new AbortController().signal; + + const result = await compose(harness.host, { signal }); + + expect(result).toEqual({ + ownerId: 'base', + ownerName: 'Portrait', + reason: 'too-large', + status: 'adjustment-refused', + }); + // The job is cancellable with the invocation. + expect(runSpatial).toHaveBeenCalledWith(expect.objectContaining({ priority: 'now', signal })); + expect(harness.uploadImage).not.toHaveBeenCalled(); + expect(harness.release).toHaveBeenCalledOnce(); + expect(harness.host.dedupe.byKey.size).toBe(0); + }); + it('releases the raster snapshot on success', async () => { const harness = makeHost(makeDoc([rasterLayer('base')])); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/generationComposite.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/generationComposite.ts index 847e8af19ea..d1778beddd9 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/generationComposite.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/generationComposite.ts @@ -20,6 +20,11 @@ import type { import { lookupDocumentLayer } from '@workbench/canvas-engine/document-model/documentModel'; import { intersect } from '@workbench/canvas-engine/math/rect'; +import { + type AdjustmentRefusal, + AdjustmentRefusedError, + adjustmentRefusal, +} from '@workbench/canvas-engine/render/exactAdjustments'; import { computeCompositeContentBounds, executeCompositePlan, @@ -123,7 +128,8 @@ export interface GenerationComposites { export type ComposeForGenerationResult = | { status: 'ok'; composites: GenerationComposites; dedupeCommit: GenerationCompositeDedupeCommit } - | { status: 'no-document' | 'stale' | 'aborted' | 'not-ready' | 'over-budget' }; + | { status: 'no-document' | 'stale' | 'aborted' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal; /** * Runs the complete canvas → generation composite pipeline against `host`. @@ -186,6 +192,7 @@ export const composeForGeneration = async ( const deps: ExecuteCompositePlanDeps = { ...executorDeps, dedupe: operationDedupe, + signal: options.signal, getLayerSurface: (layerId) => { const detached = rasterSnapshot.layerSurfaces.get(layerId); // The raster snapshot owns these detached pixels until the whole operation releases it. @@ -305,6 +312,11 @@ export const composeForGeneration = async ( dedupeCommit, status: 'ok', }; + } catch (error) { + if (error instanceof AdjustmentRefusedError) { + return adjustmentRefusal(document, error.ownerId, error.reason); + } + throw error; } finally { rasterSnapshot.release(); } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/layerImageResult.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/layerImageResult.ts index 92470579d55..957f1a8ba44 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/layerImageResult.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/layerImageResult.ts @@ -1,4 +1,5 @@ import type { RunUtilityGraphOptions, UtilityGraphResult } from '@features/queue/utility'; +import type { AdjustmentRefusal } from '@workbench/canvas-engine/api'; import type { CanvasImageRef } from '@workbench/canvas-engine/contracts'; import type { ExportBakedLayerBlobResult, LayerExportGuard } from '@workbench/canvas-engine/engine'; import type { Rect } from '@workbench/canvas-engine/types'; @@ -18,6 +19,7 @@ export interface SelectObjectReadyResult { export type SelectObjectRunResult = | SelectObjectReadyResult | { status: 'aborted' | 'missing' | 'disabled' | 'unsupported' | 'empty' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal | { status: 'invalid-input' } | { status: 'dimension-mismatch'; message: string } | { status: 'failed'; message: string; code: SamSessionErrorCode }; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/operationTypes.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/operationTypes.ts index fec8922ea85..d31988fcbff 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/operationTypes.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/operationTypes.ts @@ -13,6 +13,10 @@ export type SamPointLabel = 'include' | 'exclude'; export type SamSessionErrorCode = | 'invalid' | 'not-ready' + /** The layer's Clarity, Texture or Dehaze could not run on the source, so it was not used without them. */ + | 'adjustment-too-large' + | 'adjustment-over-budget' + | 'adjustment-error' | 'empty' | 'upload' | 'queue' @@ -73,6 +77,10 @@ export type FilterSessionErrorCode = | 'disabled' | 'empty' | 'source-over-budget' + /** The layer's Clarity, Texture or Dehaze could not run on the source, so it was not filtered without them. */ + | 'source-adjustment-too-large' + | 'source-adjustment-over-budget' + | 'source-adjustment-error' | 'process-failed' | 'apply-failed'; export interface FilterSessionError { diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/saveCanvasToGallery.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/saveCanvasToGallery.ts index 8f34f0616a1..c7cd7e5cba6 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/saveCanvasToGallery.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/saveCanvasToGallery.ts @@ -1,3 +1,4 @@ +import type { AdjustmentRefusal } from '@workbench/canvas-engine/api'; import type { Rect } from '@workbench/canvas-engine/types'; import type { Project } from '@workbench/projectContracts'; @@ -26,7 +27,9 @@ export type CanvasGallerySaveRegion = CanvasCompositeRegion; export type SaveCanvasToGalleryResult = /** `boardId` is where the image went: a board id, or `none` for Uncategorized. */ - { status: 'saved'; boardId: string; imageName: string } | { status: 'empty' | 'stale' | 'not-ready' | 'over-budget' }; + | { status: 'saved'; boardId: string; imageName: string } + | { status: 'empty' | 'stale' | 'not-ready' | 'over-budget' } + | AdjustmentRefusal; const buildCanvasSaveMetadata = (project: Project, rect: Rect): Record => { const generateValues = normalizeGenerateWidgetValues(getProjectWidgetValues(project, 'generate')); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/selectObjectSession.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/selectObjectSession.test.ts index 2e4d48eb14c..08ff77b7cf0 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/selectObjectSession.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/selectObjectSession.test.ts @@ -284,6 +284,19 @@ describe('createSelectObjectSession', () => { notReady.setCurrentGuard(null); await expect(notReady.session.process()).resolves.toBe('error'); expect(notReady.session.getSnapshot().error).toEqual({ code: 'not-ready' }); + + const refused = createHarness({ + exportSource: vi.fn(() => + Promise.resolve({ + ownerId: 'layer', + ownerName: 'Layer', + reason: 'too-large', + status: 'adjustment-refused', + } as const) + ), + }); + await expect(refused.session.process()).resolves.toBe('error'); + expect(refused.session.getSnapshot().error).toEqual({ code: 'adjustment-too-large' }); }); it('maps output dimensions and decode failures to typed errors', async () => { diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/selectObjectSession.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/selectObjectSession.ts index ef80c6b04ce..8af42d818d7 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/selectObjectSession.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/selectObjectSession.ts @@ -131,6 +131,9 @@ const getPreparedSourceError = ( if (result.status === 'dimension-mismatch') { return withDetail('output-dimension', result.message); } + if (result.status === 'adjustment-refused') { + return { code: `adjustment-${result.reason}` }; + } return { code: 'not-ready' }; }; diff --git a/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts b/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts index a05b7cb14c1..9e271c9b5fc 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts @@ -792,6 +792,56 @@ describe('loadCanvasState', () => { }); }); + describe('spatial adjustments adopt schema 5', () => { + const spatialEntries = (isEnabled: boolean) => [ + { amount: 40, id: `clarity-${String(isEnabled)}`, isEnabled, size: 40, type: 'clarity' }, + { amount: -25, id: `texture-${String(isEnabled)}`, isEnabled, size: 8, type: 'texture' }, + { amount: 0, id: `dehaze-${String(isEnabled)}`, isEnabled, type: 'dehaze' }, + ]; + const cases = [true, false].flatMap((isEnabled) => + spatialEntries(isEnabled).map( + (entry) => [`${entry.type} (${isEnabled ? 'enabled' : 'disabled'})`, entry] as const + ) + ); + + it.each(cases)('writes a v3 or v4 canvas holding %s on a layer or a group as version 5', (_label, entry) => { + const onLayer = [{ ...createEmptyPaintLayer('Detail', 'detail'), adjustments: [entry] }]; + const onGroup = [groupContract('graded', [], { adjustments: [entry as never] })]; + for (const nodes of [onLayer, onGroup]) { + for (const declared of [3, 4]) { + const canvas = withNodes(nodes); + const loaded = load({ ...canvas, document: { ...canvas.document, version: declared }, version: declared }); + + expect(loaded.version).toBe(5); + expect(loaded.document.version).toBe(5); + } + } + }); + + it('keeps out-of-range finite values for evaluation to clamp, and drops a stack whose entry is not finite', () => { + const loadStack = (adjustments: unknown[]) => { + const layer = load(withNodes([{ ...createEmptyPaintLayer('Detail', 'detail'), adjustments }])).document.stacks + .raster[0]; + return layer?.type === 'raster' ? layer.adjustments : null; + }; + const extremes = [ + { amount: 500, id: 'c', isEnabled: true, size: 0, type: 'clarity' }, + { amount: -500, id: 't', isEnabled: true, size: 5000, type: 'texture' }, + { amount: 1e6, id: 'd', isEnabled: false, type: 'dehaze' }, + ]; + expect(loadStack(extremes)).toEqual(extremes); + + for (const invalid of [ + { amount: Number.NaN, id: 'c', isEnabled: true, size: 40, type: 'clarity' }, + { amount: 10, id: 't', isEnabled: true, size: Number.POSITIVE_INFINITY, type: 'texture' }, + { amount: 10, id: 't', isEnabled: true, type: 'texture' }, + { id: 'd', isEnabled: true, type: 'dehaze' }, + ]) { + expect(loadStack([invalid])).toBeUndefined(); + } + }); + }); + it('refuses a custom-font reference whose content hash is not a lowercase SHA-256', () => { const current = createEmptyCanvasState(); const textLayer = { diff --git a/invokeai/frontend/webv2/src/workbench/canvasMigration.ts b/invokeai/frontend/webv2/src/workbench/canvasMigration.ts index a28c48cd57b..fa555671db8 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasMigration.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasMigration.ts @@ -133,6 +133,10 @@ const zAdjustmentEntry = z.discriminatedUnion('type', [ size: zFiniteNumber.min(1).max(8), type: z.literal('noise'), }), + // Spatial entries accept any finite value: evaluation clamps, and a refine failure would drop the whole stack. + z.object({ ...zAdjustmentBase, amount: zFiniteNumber, size: zFiniteNumber, type: z.literal('clarity') }), + z.object({ ...zAdjustmentBase, amount: zFiniteNumber, size: zFiniteNumber, type: z.literal('texture') }), + z.object({ ...zAdjustmentBase, amount: zFiniteNumber, type: z.literal('dehaze') }), ]); const zAdjustments = z.array(zAdjustmentEntry); const zControlAdapter = z @@ -393,15 +397,19 @@ const textSourceRequiresV4 = (source: unknown): boolean => source.type === 'text' && ('fontRef' in source || 'fontStyle' in source || 'fontVariations' in source); -const hasNoiseAdjustment = (node: Record): boolean => - Array.isArray(node.adjustments) && node.adjustments.some((entry) => isRecord(entry) && entry.type === 'noise'); +/** Adjustment types a v4 reader cannot parse; it would drop the whole stack holding one. */ +const V5_ADJUSTMENT_TYPES: ReadonlySet = new Set(['noise', 'clarity', 'texture', 'dehaze']); -/** v4: custom-font typography. v5: a noise adjustment, enabled or not, which a v4 reader would drop with its stack. */ +const hasV5Adjustment = (node: Record): boolean => + Array.isArray(node.adjustments) && + node.adjustments.some((entry) => isRecord(entry) && V5_ADJUSTMENT_TYPES.has(entry.type)); + +/** v4: custom-font typography. v5: a noise or spatial adjustment, enabled or not, which a v4 reader would drop. */ const nodeSchemaRequirement = (node: unknown): CanvasSchemaVersion => { if (!isRecord(node)) { return 3; } - if (hasNoiseAdjustment(node)) { + if (hasV5Adjustment(node)) { return 5; } let required: CanvasSchemaVersion = textSourceRequiresV4(node.source) ? 4 : 3; diff --git a/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.test.ts b/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.test.ts index 1048582f214..be20926bd42 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.test.ts @@ -46,4 +46,29 @@ describe('canvas schema adoption', () => { expect(cleared.canvas.version).toBe(5); expect(cleared.canvas.document.version).toBe(5); }); + + it.each([ + { amount: 30, id: 'clarity', isEnabled: false, size: 40, type: 'clarity' }, + { amount: 30, id: 'texture', isEnabled: false, size: 8, type: 'texture' }, + { amount: 30, id: 'dehaze', isEnabled: false, type: 'dehaze' }, + ])('moves a v4 canvas to schema 5 when a disabled $type adjustment is added', (entry) => { + const project = createDraftProject([]); + const v4 = { + ...project, + canvas: { ...project.canvas, document: { ...project.canvas.document, version: 4 as const }, version: 4 as const }, + }; + const added = applyCanvasProjectMutation(v4, { + anchor: stackTopAnchor(project.id), + layer: createEmptyPaintLayer('Detail', 'detail'), + type: 'addCanvasLayer', + }); + + const adjusted = applyCanvasProjectMutation(added, { + config: { adjustments: [entry], layerType: 'raster' }, + id: 'detail', + type: 'updateCanvasLayerConfig', + }); + expect(adjusted.canvas.version).toBe(5); + expect(adjusted.canvas.document.version).toBe(5); + }); }); diff --git a/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts b/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts index ef26990d861..b40b2068e9a 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts @@ -4,7 +4,7 @@ export const MIN_SUPPORTED_CANVAS_SCHEMA_VERSION = 3; /** * Newest canvas document schema this build can safely read, edit, and write. A document adopts a newer schema only * when it carries something older builds would drop on load and then lose on save: 4 for custom-font typography, 5 - * for noise adjustments. Older builds refuse the newer schema instead. + * for noise, clarity, texture or dehaze adjustments. Older builds refuse the newer schema instead. */ export const MAX_SUPPORTED_CANVAS_SCHEMA_VERSION = 5; diff --git a/invokeai/frontend/webv2/src/workbench/projects/README.md b/invokeai/frontend/webv2/src/workbench/projects/README.md index 5d201713584..ea0c88aefa4 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/README.md +++ b/invokeai/frontend/webv2/src/workbench/projects/README.md @@ -74,7 +74,7 @@ Cold-start offline editing not supported. When backend cannot load, unavailable New value in a closed canvas set, such as a control layer's adapter kind (`anima_lllite`), is added without raising canvas schema version or project's compatibility floor. Builds that know the value load and save it unchanged; builds that do not refuse the canvas instead of rewriting it, since coercing an unknown kind to a known one would replace the user's adapter on next autosave. In an older build the project is not opened and user sees that it "contains canvas data this version of Invoke cannot read"; saved copy is untouched and exportable, and a build that knows the value opens it as saved. Raise schema version instead when older builds could misread a document without refusing it. -Adjustment stacks are such a case: load drops a stack holding any entry it cannot parse, so an older build would load a document with a new adjustment type minus that layer's whole stack and autosave the loss. A new adjustment type therefore raises the canvas schema for documents that use it. A noise adjustment, enabled or not, on any layer or group at any depth of the live document or a saved snapshot, makes the canvas schema 5 (`canvasMigration.ts`, `writableCanvasSchemaVersion`); a document without one keeps its version. Adoption is sticky: removing the last noise entry leaves the document at 5, matching the server's monotonic floor. A build whose maximum is 4 refuses the project at the server and the canvas at load as an unsupported version. +Adjustment stacks are such a case: load drops a stack holding any entry it cannot parse, so an older build would load a document with a new adjustment type minus that layer's whole stack and autosave the loss. A new adjustment type therefore raises the canvas schema for documents that use it. A noise, clarity, texture or dehaze adjustment, enabled or not, on any layer or group at any depth of the live document or a saved snapshot, makes the canvas schema 5 (`canvasMigration.ts`, `writableCanvasSchemaVersion`); a document without one keeps its version. Adoption is sticky: removing the last such entry leaves the document at 5, matching the server's monotonic floor. A build whose maximum is 4 refuses the project at the server and the canvas at load as an unsupported version. Load does not reinterpret a known value either. Anima control layer saved as ControlNet before ControlNet-LLLite was supported keeps its kind and model; layer settings name the adapter type to switch to and offer a one-step switch that keeps model, weight and step range where the new kind can use them. diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/CanvasEditRefusalNotices.tsx b/invokeai/frontend/webv2/src/workbench/widgets/canvas/CanvasEditRefusalNotices.tsx index 60d0fe56b3c..ca10d21db3c 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/CanvasEditRefusalNotices.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/CanvasEditRefusalNotices.tsx @@ -2,7 +2,7 @@ import type { CanvasEngineHandle } from '@workbench/canvas-operations/react'; import { useMountEffect } from '@platform/react/useMountEffect'; import { useNotify } from '@workbench/useNotify'; -import { reportStructuralCommit } from '@workbench/widgets/canvas/useStructuralCommit'; +import { describeAdjustmentRefusal, reportStructuralCommit } from '@workbench/widgets/canvas/useStructuralCommit'; import { useTranslation } from 'react-i18next'; /** Explains edits the engine refused without a caller to report to, such as a stroke too large to undo. Keyed by engine. */ @@ -10,12 +10,16 @@ export const CanvasEditRefusalNotices = ({ engine }: { engine: Pick - engine.tools.onEditRefused((status) => - // Elsewhere `busy` stays silent because its controls are disabled; a stroke can still meet it mid-gesture. - status === 'busy' - ? notify.error(t('widgets.canvas.structural.failed'), t('widgets.canvas.structural.busy')) - : reportStructuralCommit({ status }, notify.error, t) - ) + engine.tools.onEditRefused((refusal) => { + if (typeof refusal !== 'string') { + notify.error(t('widgets.canvas.structural.failed'), describeAdjustmentRefusal(refusal, t)); + } else if (refusal === 'busy') { + // Elsewhere `busy` stays silent because its controls are disabled; a stroke can still meet it mid-gesture. + notify.error(t('widgets.canvas.structural.failed'), t('widgets.canvas.structural.busy')); + } else { + reportStructuralCommit({ status: refusal }, notify.error, t); + } + }) ); return null; }; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/canvasHotkeyCommands.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/canvasHotkeyCommands.test.ts index 777d98baf11..266efaf4a2c 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/canvasHotkeyCommands.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/canvasHotkeyCommands.test.ts @@ -78,7 +78,7 @@ const createEngine = (interaction: Partial = {}) => { status: 'duplicated', }) ), - mergeLayerDown: vi.fn((): MergeDownResult => 'merged'), + mergeLayerDown: vi.fn((): Promise => Promise.resolve('merged')), nudgeSelectedLayer: vi.fn(() => ({ status: 'committed' as const })), }, selection: { @@ -586,12 +586,12 @@ describe('mergeDown', () => { expect(engine.layers.mergeLayerDown).not.toHaveBeenCalled(); }); - it('reports a merge the engine refused', () => { + it('reports a merge the engine refused', async () => { const engine = createEngine(); - engine.layers.mergeLayerDown.mockReturnValue('over-budget'); + engine.layers.mergeLayerDown.mockResolvedValue('over-budget'); const reportLayerOperation = vi.fn(); executeCanvasHotkeyCommand('canvas.mergeDown', contextOf({ engine, reportLayerOperation })); - expect(reportLayerOperation).toHaveBeenCalledExactlyOnceWith('over-budget'); + await vi.waitFor(() => expect(reportLayerOperation).toHaveBeenCalledExactlyOnceWith('over-budget')); }); }); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/canvasHotkeyCommands.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/canvasHotkeyCommands.ts index 835f17437fb..25d676abf22 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/canvasHotkeyCommands.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/canvasHotkeyCommands.ts @@ -1,4 +1,5 @@ import type { + AdjustmentRefusal, CanvasDocumentContractV3, CanvasEngine, LayerStackMoveKind, @@ -105,7 +106,7 @@ export interface CanvasHotkeyContext { readonly notifyLayerDuplicateFailed: () => void; readonly reportStructuralCommit: (result: StructuralCommitResult) => void; readonly reportPreparedCommit: (outcome: PreparedCommitOutcome) => void; - readonly reportLayerOperation: (refusal: LayerOperationRefusal) => void; + readonly reportLayerOperation: (refusal: LayerOperationRefusal | AdjustmentRefusal) => void; readonly t: (key: string) => string; } @@ -354,10 +355,14 @@ export const executeCanvasHotkeyCommand = (commandId: string, ctx: CanvasHotkeyC } else if (commandId === 'canvas.mergeDown') { // Share canMergeLayerDown with the layer menu so shortcuts respect identical eligibility. if (engine && selectedLayer && canMergeLayerDown(document, selectedLayer.id, true)) { - const result = engine.layers.mergeLayerDown(selectedLayer.id); - if (result !== 'merged') { - ctx.reportLayerOperation(result); - } + void engine.layers.mergeLayerDown(selectedLayer.id).then( + (result) => { + if (result !== 'merged') { + ctx.reportLayerOperation(result); + } + }, + () => ctx.reportLayerOperation('failed') + ); } } }; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts index 165e844be82..c13a1a6bbc7 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts @@ -690,6 +690,19 @@ describe('runCanvasInvocation', () => { expect(harness.notices()).toHaveLength(0); }); + it('names the layer or group whose Clarity, Texture or Dehaze refused generation, and dispatches nothing', async () => { + const harness = makeHarness(); + harness.deps.composeForGeneration = () => + Promise.resolve({ ownerId: 'g', ownerName: 'Sky', reason: 'too-large', status: 'adjustment-refused' }); + + await runCanvasInvocation(harness.deps); + + expect(harness.submittedGraphs()).toHaveLength(0); + expect(harness.notices().map((notice) => notice.message)).toEqual([ + 'Clarity, Texture and Dehaze can’t run on “Sky”: its content is larger than 16.8 megapixels (4096 × 4096 px). Disable or remove them, or crop the content, then invoke again.', + ]); + }); + it('uses the frozen canvas and detached pixels after edits at composite, upload, and queue-dispatch boundaries', async () => { let liveDocument = makeDoc([rasterLayer('frozen-layer')], 96); liveDocument.bbox = { height: 64, width: 64, x: 7, y: 11 }; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts index 0ddb5f308fc..ced828a3c9c 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts @@ -295,6 +295,16 @@ const createRegionalGuidanceCollector = ( }; }; +/** Why generation refused a layer's or group's Clarity, Texture or Dehaze, which it never leaves out. */ +const ADJUSTMENT_REFUSAL_NOTICES: Record<'too-large' | 'over-budget' | 'error', (name: string) => string> = { + error: (name) => + `Clarity, Texture or Dehaze failed on “${name}”. Change a setting or disable them, then invoke again.`, + 'over-budget': (name) => + `Not enough canvas memory to apply Clarity, Texture or Dehaze to “${name}”. Delete layers you no longer need, then invoke again.`, + 'too-large': (name) => + `Clarity, Texture and Dehaze can’t run on “${name}”: its content is larger than 16.8 megapixels (4096 × 4096 px). Disable or remove them, or crop the content, then invoke again.`, +}; + /** * Runs the canvas-invoke pipeline with injected dependencies. Never throws: * every failure is turned into a notice, and the in-flight guard is always @@ -348,7 +358,9 @@ export const runCanvasInvocation = async (deps: RunCanvasInvocationDeps): Promis ? 'The canvas has no active document to generate from.' : composed.status === 'stale' ? 'The canvas changed while generation was being prepared. Please invoke again.' - : `The canvas could not be captured for generation (${composed.status}).` + : composed.status === 'adjustment-refused' + ? ADJUSTMENT_REFUSAL_NOTICES[composed.reason](composed.ownerName) + : `The canvas could not be captured for generation (${composed.status}).` ); return; } diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/FilterOptions.test.tsx b/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/FilterOptions.test.tsx index 6ffc40f0bdb..1622dbab0bd 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/FilterOptions.test.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/FilterOptions.test.tsx @@ -156,6 +156,9 @@ describe('getFilterErrorTranslationKey', () => { 'not-ready': true, 'over-budget': true, 'process-failed': true, + 'source-adjustment-error': true, + 'source-adjustment-over-budget': true, + 'source-adjustment-too-large': true, 'source-over-budget': true, stale: true, unsupported: true, diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/FilterOptions.tsx b/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/FilterOptions.tsx index 4157443afd7..5d66bfc5f95 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/FilterOptions.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/FilterOptions.tsx @@ -84,6 +84,9 @@ const FILTER_ERROR_TRANSLATION_KEYS: Record = { 'not-ready': 'widgets.layers.rasterFilter.notReady', 'over-budget': 'widgets.layers.rasterFilter.overBudget', 'process-failed': 'widgets.layers.rasterFilter.processFailed', + 'source-adjustment-error': 'widgets.layers.rasterFilter.sourceAdjustmentError', + 'source-adjustment-over-budget': 'widgets.layers.rasterFilter.sourceAdjustmentOverBudget', + 'source-adjustment-too-large': 'widgets.layers.rasterFilter.sourceAdjustmentTooLarge', 'source-over-budget': 'widgets.layers.rasterFilter.sourceOverBudget', stale: 'widgets.layers.rasterFilter.stale', unsupported: 'widgets.layers.rasterFilter.unsupported', diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/SamOptions.tsx b/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/SamOptions.tsx index f43a199302e..ee3ac9a9db1 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/SamOptions.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/SamOptions.tsx @@ -73,6 +73,9 @@ const SAM_STATUS_TRANSLATION_KEYS: Record }; const SAM_ERROR_TRANSLATION_KEYS: Record = { + 'adjustment-error': 'widgets.layers.selectObject.errorAdjustment', + 'adjustment-over-budget': 'widgets.layers.selectObject.errorAdjustmentOverBudget', + 'adjustment-too-large': 'widgets.layers.selectObject.errorAdjustmentTooLarge', decode: 'widgets.layers.selectObject.errorDecode', empty: 'widgets.layers.selectObject.errorEmpty', invalid: 'widgets.layers.selectObject.errorInvalid', diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/useCanvasGallerySave.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/useCanvasGallerySave.ts index f7c7c4d370a..7fff4fbafdd 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/useCanvasGallerySave.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/useCanvasGallerySave.ts @@ -20,6 +20,7 @@ import { useTranslation } from 'react-i18next'; import { getCanvasGallerySaveErrorAction, withMatchingCanvasProject } from './canvasGallerySaveState'; import { ACTION_TOAST_DURATION_MS, announce, describeBoard } from './gallerySaveToast'; +import { describeAdjustmentRefusal } from './useStructuralCommit'; type CanvasGallerySaveEngine = Pick; @@ -87,6 +88,8 @@ export const useCanvasGallerySave = ( notify.info(t('widgets.canvas.contextMenu.empty')); } else if (result.status === 'stale') { notify.info(t('widgets.canvas.contextMenu.stale')); + } else if (result.status === 'adjustment-refused') { + notify.error(t('widgets.canvas.contextMenu.saveError'), describeAdjustmentRefusal(result, t)); } else { notify.info(t('widgets.canvas.contextMenu.notReady')); } diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/useCreateFromBbox.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/useCreateFromBbox.ts index f8eff914e5a..13af337f1fc 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/useCreateFromBbox.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/useCreateFromBbox.ts @@ -16,6 +16,7 @@ import { useCallback, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { getCanvasGallerySaveErrorAction, withMatchingCanvasProject } from './canvasGallerySaveState'; +import { describeAdjustmentRefusal } from './useStructuralCommit'; type CreateFromBboxEngine = Pick; @@ -82,6 +83,10 @@ export const useCreateFromBbox = ( } const notice = getCreateFromBboxNotice(result); + if ('refusal' in notice) { + notify.error(t('widgets.canvas.contextMenu.createError'), describeAdjustmentRefusal(notice.refusal, t)); + return; + } const options = notice.options && 'destination' in notice.options ? { destination: t(notice.options.destination) } diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/useStructuralCommit.browser.test.tsx b/invokeai/frontend/webv2/src/workbench/widgets/canvas/useStructuralCommit.browser.test.tsx index 62c194b4f49..42bb24e23d8 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/useStructuralCommit.browser.test.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/useStructuralCommit.browser.test.tsx @@ -21,7 +21,11 @@ import { const notify = vi.hoisted(() => ({ error: vi.fn(), info: vi.fn(), success: vi.fn() })); vi.mock('@workbench/useNotify', () => ({ useNotify: () => notify })); -vi.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string) => key }) })); +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ + t: (key: string, options?: { name?: string }) => (options?.name ? `${key}:${options.name}` : key), + }), +})); (globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; @@ -266,4 +270,15 @@ describe('CanvasEditRefusalNotices', () => { emit(refusal); expect(notify.error).toHaveBeenCalledWith('widgets.canvas.structural.failed', message); }); + + it('names the layer whose spatial adjustments keep an edit from baking its pixels', () => { + const { emit, engine } = createRefusalEngine(); + render(); + + emit({ ownerId: 'a', ownerName: 'Sky', reason: 'too-large', status: 'adjustment-refused' }); + expect(notify.error).toHaveBeenCalledWith( + 'widgets.canvas.structural.failed', + 'widgets.layers.adjustments.refusal.tooLarge:Sky' + ); + }); }); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/useStructuralCommit.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/useStructuralCommit.test.ts index 6ec662bca1f..adcd4e98865 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/useStructuralCommit.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/useStructuralCommit.test.ts @@ -12,6 +12,7 @@ import { baselineConfig, baselinePatch, commitPreparedEdit, + reportLayerOperation, reportPreparedCommit, reportStructuralCommit, } from './useStructuralCommit'; @@ -44,6 +45,25 @@ describe('reportStructuralCommit', () => { }); }); +describe('reportLayerOperation', () => { + it.each([ + ['too-large', 'widgets.layers.adjustments.refusal.tooLarge'], + ['over-budget', 'widgets.layers.adjustments.refusal.overBudget'], + ['error', 'widgets.layers.adjustments.refusal.error'], + ] as const)('names the layer or group whose adjustments refused %s', (reason, key) => { + const report = vi.fn(); + const translate = vi.fn((key: string, options?: { name?: string }) => `${key}:${options?.name ?? ''}`); + + reportLayerOperation( + { ownerId: 'g', ownerName: 'Sky', reason, status: 'adjustment-refused' }, + report, + translate as unknown as TFunction + ); + + expect(report).toHaveBeenCalledWith('widgets.layers.actions.actionFailed:', `${key}:Sky`); + }); +}); + describe('commitPreparedEdit', () => { const layer = createEmptyPaintLayer('Layer', 'a'); const document = { ...createEmptyCanvasDocument(), stacks: stacksFrom([layer]), selectedLayerId: 'a' }; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/useStructuralCommit.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/useStructuralCommit.ts index 23305fa8144..70a6a55fbaa 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/useStructuralCommit.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/useStructuralCommit.ts @@ -1,4 +1,6 @@ import type { + AdjustmentRefusal, + AdjustmentRefusalReason, CanvasDocumentCapability, CanvasLayerBasePatch, CanvasLayerCapability, @@ -41,12 +43,26 @@ const LAYER_OPERATION_KEYS: Record = { unsupported: 'widgets.layers.actions.unsupported', }; +const ADJUSTMENT_REFUSAL_KEYS: Record = { + error: 'widgets.layers.adjustments.refusal.error', + 'over-budget': 'widgets.layers.adjustments.refusal.overBudget', + 'too-large': 'widgets.layers.adjustments.refusal.tooLarge', +}; + +/** Names the layer or group whose Clarity, Texture or Dehaze could not run, and why. */ +export const describeAdjustmentRefusal = (refusal: AdjustmentRefusal, t: TFunction): string => + t(ADJUSTMENT_REFUSAL_KEYS[refusal.reason], { name: refusal.ownerName }); + /** Explains a refused layer pixel operation; nothing changed in the document, pixels or history. */ export const reportLayerOperation = ( - refusal: LayerOperationRefusal, + refusal: LayerOperationRefusal | AdjustmentRefusal, reportError: (title: string, message: string) => void, t: TFunction -): void => reportError(t('widgets.layers.actions.actionFailed'), t(LAYER_OPERATION_KEYS[refusal])); +): void => + reportError( + t('widgets.layers.actions.actionFailed'), + typeof refusal === 'string' ? t(LAYER_OPERATION_KEYS[refusal]) : describeAdjustmentRefusal(refusal, t) + ); /** Explains a refused mask clear or invert; `busy` and a mask with nothing to change stay silent. */ export const reportMaskEdit = ( diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.browser.test.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.browser.test.tsx index 5e41487dea5..b54e7716853 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.browser.test.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.browser.test.tsx @@ -1,4 +1,9 @@ -import type { CanvasAdjustmentEntry, CanvasNodeContract, PreparedDocumentEdit } from '@workbench/canvas-engine/api'; +import type { + CanvasAdjustmentEntry, + CanvasGroupContract, + CanvasNodeContract, + PreparedDocumentEdit, +} from '@workbench/canvas-engine/api'; import type { StructuralEngineStub } from '@workbench/canvas-engine/controllers/structuralEngine.testStub'; /* oxlint-disable react-perf/jsx-no-new-function-as-prop */ @@ -10,14 +15,16 @@ import { groupContract, layerContract } from '@workbench/canvas-engine/document- import { getDocumentIndex } from '@workbench/canvas-engine/document/documentIndex'; import { commitPreparedEdit } from '@workbench/widgets/canvas/useStructuralCommit'; import { createInstance } from 'i18next'; -import { act, useSyncExternalStore } from 'react'; +import { act, type ReactElement, useSyncExternalStore } from 'react'; import { createRoot, type Root } from 'react-dom/client'; import { I18nextProvider, initReactI18next } from 'react-i18next'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { userEvent } from 'vitest/browser'; import { type AdjustmentOwner, type AdjustmentsEngine, AdjustmentSettings } from './AdjustmentSettings'; +import { createFakeAdjustmentStatus } from './AdjustmentStatusIcon.testing'; import { CURVE_SIZE } from './curveEditorMath'; +import { LayerGroupContextMenu, type LayerGroupContextMenuEngine } from './LayerGroupContextMenu'; const i18n = createInstance(); void i18n.use(initReactI18next).init({ fallbackLng: 'en', initAsync: false, lng: 'en', resources: {} }); @@ -28,6 +35,12 @@ vi.mock('@workbench/useCanvasProjectMutationDispatch', () => ({ })); const notify = vi.hoisted(() => ({ error: vi.fn(), info: vi.fn(), success: vi.fn() })); vi.mock('@workbench/useNotify', () => ({ useNotify: () => notify })); +// The group menu reads the active project's document; here that is the stub engine's. +vi.mock('@workbench/WorkbenchContext', () => ({ + useActiveProjectId: () => 'project-1', + useActiveProjectSelector: (select: (project: { canvas: { document: unknown } }) => T): T => + select({ canvas: { document: stub.document() } }), +})); (globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; @@ -62,6 +75,7 @@ const ownerNode = (owner: Owner, adjustments: CanvasAdjustmentEntry[]): CanvasNo /** The structural engine over the real reducer and history; previews flush synchronously, as a settled frame would. */ let stub: StructuralEngineStub; +let adjustmentStatus = createFakeAdjustmentStatus(); let ownerId = OWNER_ID.raster; const documentOwner = (): AdjustmentOwner => getDocumentIndex(stub.document()).byId.get(ownerId)!.node as unknown as AdjustmentOwner; @@ -100,12 +114,8 @@ const settle = (action: () => void): Promise => }); }); -const render = async (entryId = 'cv1', owner: Owner = 'raster', adjustments = initialEntries(), withEngine = true) => { - applyThemeToRoot('classic'); - host = document.createElement('div'); - host.style.width = '260px'; - document.body.append(host); - root = createRoot(host); +/** A fresh stub engine over one owner, reporting spatial status from a fake store the test drives. */ +const setUp = (owner: Owner, adjustments: CanvasAdjustmentEntry[]): void => { ownerId = OWNER_ID[owner]; stub = createStructuralEngineStub({ layers: [ownerNode(owner, adjustments)], @@ -114,18 +124,29 @@ const render = async (entryId = 'cv1', owner: Owner = 'raster', adjustments = in return () => undefined; }, }); + adjustmentStatus = createFakeAdjustmentStatus(); + Object.assign(stub.engine, { interaction: adjustmentStatus.interaction }); +}; +const mount = async (element: ReactElement): Promise => { + applyThemeToRoot('classic'); + host = document.createElement('div'); + host.style.width = '260px'; + document.body.append(host); + root = createRoot(host); await settle(() => { root?.render( - - - + {element} ); }); +}; - return host.querySelector(`svg[viewBox="0 0 ${CURVE_SIZE} ${CURVE_SIZE}"]`)!; +const render = async (entryId = 'cv1', owner: Owner = 'raster', adjustments = initialEntries(), withEngine = true) => { + setUp(owner, adjustments); + await mount(); + return host!.querySelector(`svg[viewBox="0 0 ${CURVE_SIZE} ${CURVE_SIZE}"]`)!; }; const handles = (svg: SVGSVGElement): SVGCircleElement[] => Array.from(svg.querySelectorAll('circle')); @@ -577,3 +598,107 @@ describe('curves entry editor', () => { expect(added.x).toBeCloseTo(targetX, 0); }); }); + +const MENU_ANCHOR = { height: 1, width: 1, x: 300, y: 10 }; +const closeMenu = (): void => undefined; + +describe('spatial adjustment editors', () => { + type SpatialEntry = Extract; + + /** The group's menu over the editor of whichever entry the group's stack ends with. */ + const MenuHarness = () => { + const owner = useSyncExternalStore(stub.subscribe, documentOwner); + const last = owner.adjustments?.at(-1); + return ( + <> + + {last ? ( + + ) : null} + + ); + }; + + const sliderOf = (label: string): HTMLElement => scrubber(label).querySelector('[role="slider"]')!; + + it('adds Clarity from the menu at identity, then scrubs and resets its amount as undoable steps', async () => { + setUp('group', []); + await mount(); + + const trigger = document.querySelector('[aria-label="widgets.layers.menu.addAdjustment"]')!; + await act(() => userEvent.click(trigger)); + const clarityItem = () => document.querySelector('[role="menuitem"][data-value="add-clarity"]'); + await expect.poll(clarityItem).not.toBeNull(); + const clarity = clarityItem()!; + expect(clarity.textContent).toBe('widgets.layers.adjustments.clarity'); + await act(() => userEvent.click(clarity)); + await settle(() => undefined); + + expect(commits()).toHaveLength(1); + const [added] = latestAdjustments() as SpatialEntry[]; + expect(added).toEqual({ amount: 0, id: expect.any(String), isEnabled: true, size: 40, type: 'clarity' }); + expect(sliderOf('widgets.layers.adjustments.amount').getAttribute('aria-valuetext')).toBe('0'); + expect(sliderOf('widgets.layers.adjustments.size').getAttribute('aria-valuetext')).toBe('40 px'); + + await scrub(scrubber('widgets.layers.adjustments.amount'), [0.1, 0.25]); + expect(commits()).toHaveLength(2); + expect(latestAdjustments()).toEqual([{ ...added, amount: 100 }]); + expect(sliderOf('widgets.layers.adjustments.amount').getAttribute('aria-valuetext')).toBe('+100'); + + // Backspace restores the default, recorded like any other edit. + await act(async () => { + sliderOf('widgets.layers.adjustments.amount').focus(); + await userEvent.keyboard('{Backspace}'); + }); + expect(commits()).toHaveLength(3); + expect(latestAdjustments()).toEqual([added]); + + await stub.engine.history.undo(); + expect(latestAdjustments()).toEqual([{ ...added, amount: 100 }]); + }); + + it("sizes Texture in the layer's whole pixels on a log track, keeping the amount", async () => { + await render('tx1', 'raster', [{ amount: -30, id: 'tx1', isEnabled: true, size: 8, type: 'texture' }]); + expect(host!.textContent).toContain('widgets.layers.adjustments.sizeHintLayer'); + expect(sliderOf('widgets.layers.adjustments.amount').getAttribute('aria-valuetext')).toBe('-30'); + + await act(async () => { + sliderOf('widgets.layers.adjustments.size').focus(); + await userEvent.keyboard('2.6{Enter}'); + }); + expect(latestAdjustments()).toEqual([{ amount: -30, id: 'tx1', isEnabled: true, size: 3, type: 'texture' }]); + + await act(async () => { + await userEvent.keyboard('{End}'); + }); + expect((latestAdjustments()[0] as SpatialEntry & { size: number }).size).toBe(100); + expect(commits()).toHaveLength(2); + }); + + it('gives Dehaze only an amount, says it approximates, and explains when the canvas cannot apply it', async () => { + await render('dh1', 'group', [{ amount: 50, id: 'dh1', isEnabled: true, type: 'dehaze' }]); + expect(() => scrubber('widgets.layers.adjustments.size')).toThrow(); + expect(host!.textContent).toContain('widgets.layers.adjustments.dehazeHint'); + // One live region stays mounted, so a warning that appears in it is announced. + const region = host!.querySelector('[role="status"]')!; + expect(region.textContent).toBe(''); + + await settle(() => adjustmentStatus.set('group-1', 'computing')); + expect(region.textContent).toBe(''); + + await settle(() => adjustmentStatus.set('group-1', 'too-large')); + expect(host!.querySelector('[role="status"]')).toBe(region); + expect(region.textContent).toBe('widgets.layers.adjustments.status.tooLarge'); + + await settle(() => adjustmentStatus.set('group-1', 'idle')); + expect([...host!.querySelectorAll('[role="status"]')]).toEqual([region]); + expect(region.textContent).toBe(''); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.tsx index e98435c0dff..59501496061 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.tsx @@ -21,6 +21,9 @@ import { DicesIcon } from 'lucide-react'; import { useCallback, useMemo, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; +import type { AdjustmentStatusEngine } from './AdjustmentStatusIcon'; + +import { AdjustmentStatusNotice } from './AdjustmentStatusIcon'; import { CURVE_PADDING, CURVE_SIZE, @@ -29,14 +32,20 @@ import { finishCurveDragResult, getCurveGridCoordinates, } from './curveEditorMath'; -import { createNoiseSeed, DEFAULT_NOISE_AMOUNT, DEFAULT_NOISE_SIZE } from './layerOps'; +import { + createNoiseSeed, + DEFAULT_CLARITY_SIZE, + DEFAULT_NOISE_AMOUNT, + DEFAULT_NOISE_SIZE, + DEFAULT_TEXTURE_SIZE, +} from './layerOps'; /** * Edit adjustment subrows through whole-stack patch-config. Preview gestures live and commit once; tree rows own * enable/remove/reorder. */ -export type AdjustmentsEngine = CanvasStructuralEngine & CanvasPreparedEngine; +export type AdjustmentsEngine = CanvasStructuralEngine & CanvasPreparedEngine & AdjustmentStatusEngine; const SELECT_POSITIONING = { placement: 'bottom-end', sameWidth: true } as const; @@ -169,6 +178,10 @@ const AdjustmentEntryEditor = ({ (patch: Partial) => commitEntry(t('widgets.layers.adjustments.noise'), patch), [commitEntry, t] ); + const handleSpatialCommit = useCallback( + (patch: Partial) => commitEntry(t(`widgets.layers.adjustments.${entry.type}`), patch), + [commitEntry, entry.type, t] + ); const handleCurvesLive = useCallback((curves: CanvasAdjustmentCurves) => patchLive({ curves }), [patchLive]); const handleCurvesCommit = useCallback( (curves: CanvasAdjustmentCurves) => commitEntry(t('widgets.layers.adjustments.curves'), { curves }), @@ -225,6 +238,19 @@ const AdjustmentEntryEditor = ({ ); case 'noise': return ; + case 'clarity': + case 'texture': + case 'dehaze': + return ( + + ); case 'curves': return ( ; type NoiseEntry = Extract; +type SpatialEntry = Extract; const formatSigned = (value: number): string => `${value > 0 ? '+' : ''}${value}`; const formatDegrees = (value: number): string => `${Math.round(value)}°`; @@ -602,6 +629,81 @@ const NoiseEditor = ({ ); }; +/** The spatial amount's range, as on the backend nodes; 0 is identity. */ +const SPATIAL_AMOUNT_LIMIT = 200; +/** Feature-size ranges in owner pixels, as on the backend nodes; Dehaze has none (it follows the content's size). */ +const SPATIAL_SIZES = { + clarity: { defaultValue: DEFAULT_CLARITY_SIZE, max: 1000, min: 4 }, + texture: { defaultValue: DEFAULT_TEXTURE_SIZE, max: 100, min: 1 }, +} as const; + +const formatWholePixels = (value: number): string => `${Math.round(value)} px`; + +/** Clarity, Texture and Dehaze: a signed amount, a feature size for the first two, and why the canvas skips them. */ +const SpatialEditor = ({ + engine, + entry, + onCommit, + onLive, + ownerId, + ownerType, +}: { + engine: AdjustmentStatusEngine | null; + entry: SpatialEntry; + ownerId: string; + ownerType: AdjustmentOwner['type']; + onLive: (patch: Partial) => void; + onCommit: (patch: Partial) => void; +}) => { + const { t } = useTranslation(); + const handleAmountChange = useCallback((amount: number) => onLive({ amount }), [onLive]); + const handleAmountCommit = useCallback((amount: number) => onCommit({ amount }), [onCommit]); + // Whole pixels: a typed fraction would show one size and store another. + const handleSizeChange = useCallback((size: number) => onLive({ size: Math.round(size) }), [onLive]); + const handleSizeCommit = useCallback((size: number) => onCommit({ size: Math.round(size) }), [onCommit]); + const size = entry.type === 'dehaze' ? null : { ...SPATIAL_SIZES[entry.type], value: entry.size }; + + return ( + + + + + {size ? ( + + ) : null} + + + ); +}; + const sameChannel = (left: CanvasAdjustmentCurves, right: CanvasAdjustmentCurves, channel: CurveChannel): boolean => JSON.stringify(left[channel] ?? IDENTITY_CURVE) === JSON.stringify(right[channel] ?? IDENTITY_CURVE); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentStatusIcon.testing.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentStatusIcon.testing.ts new file mode 100644 index 00000000000..8f7b0fa795a --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentStatusIcon.testing.ts @@ -0,0 +1,24 @@ +import type { AdjustmentOwnerStatus, AdjustmentStatusEngine } from './AdjustmentStatusIcon'; + +/** A keyed status store standing in for the engine's, which tests drive with `set`. */ +export const createFakeAdjustmentStatus = (): AdjustmentStatusEngine & { + set(ownerId: string, status: AdjustmentOwnerStatus): void; +} => { + const statuses = new Map(); + const listeners = new Map void>>(); + return { + interaction: { + getLayerAdjustmentStatus: (ownerId) => statuses.get(ownerId) ?? 'idle', + subscribeLayerAdjustmentStatus: (ownerId, listener) => { + const keyed = listeners.get(ownerId) ?? new Set(); + listeners.set(ownerId, keyed); + keyed.add(listener); + return () => keyed.delete(listener); + }, + }, + set: (ownerId, status) => { + statuses.set(ownerId, status); + listeners.get(ownerId)?.forEach((listener) => listener()); + }, + }; +}; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentStatusIcon.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentStatusIcon.tsx new file mode 100644 index 00000000000..0be0e2ff696 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentStatusIcon.tsx @@ -0,0 +1,131 @@ +import type { CanvasInteractionStateCapability, LayerAdjustmentStatus } from '@workbench/canvas-engine/api'; +import type { TFunction } from 'i18next'; + +import { Box, HStack, Icon, Spinner, Text } from '@chakra-ui/react'; +import { useMountEffect } from '@platform/react/useMountEffect'; +import { Tooltip } from '@platform/ui/Tooltip'; +import { TriangleAlertIcon } from 'lucide-react'; +import { useCallback, useState, useSyncExternalStore } from 'react'; +import { useTranslation } from 'react-i18next'; + +/** The engine slice reporting whether an owner's spatial adjustments (Clarity, Texture, Dehaze) are shown exactly. */ +export type AdjustmentStatusEngine = { + readonly interaction: Pick< + CanvasInteractionStateCapability, + 'getLayerAdjustmentStatus' | 'subscribeLayerAdjustmentStatus' + >; +}; + +export type AdjustmentOwnerStatus = LayerAdjustmentStatus | 'idle'; + +/** A job shorter than this never shows the spinner, so quick updates while scrubbing do not flash. */ +export const ADJUSTMENT_SPINNER_DELAY_MS = 250; + +/** One owner's status, layer or group alike (the engine keys both by node id); idle without an engine. */ +export const useAdjustmentStatus = (engine: AdjustmentStatusEngine | null, ownerId: string): AdjustmentOwnerStatus => { + const subscribe = useCallback( + (listener: () => void) => + engine?.interaction.subscribeLayerAdjustmentStatus(ownerId, listener) ?? (() => undefined), + [engine, ownerId] + ); + const getSnapshot = useCallback( + () => engine?.interaction.getLayerAdjustmentStatus(ownerId) ?? 'idle', + [engine, ownerId] + ); + return useSyncExternalStore(subscribe, getSnapshot, getSnapshot); +}; + +/** Why the canvas leaves an owner's spatial adjustments out and what to do, or null while they apply. */ +export const adjustmentWarningMessage = (t: TFunction, status: AdjustmentOwnerStatus): string | null => { + switch (status) { + case 'too-large': + return t('widgets.layers.adjustments.status.tooLarge'); + case 'over-budget': + return t('widgets.layers.adjustments.status.overBudget'); + case 'error': + return t('widgets.layers.adjustments.status.error'); + case 'computing': + case 'idle': + return null; + } +}; + +/** Mounted for one run of `computing`; it appears only once the job has outlasted the delay. */ +const DelayedSpinner = () => { + const { t } = useTranslation(); + const [revealed, setRevealed] = useState(false); + useMountEffect(() => { + const timer = window.setTimeout(() => setRevealed(true), ADJUSTMENT_SPINNER_DELAY_MS); + return () => window.clearTimeout(timer); + }); + if (!revealed) { + return null; + } + const label = t('widgets.layers.adjustments.status.computing'); + return ( + + + + ); +}; + +/** + * A layer or group row's spatial adjustment indicator: a delayed spinner while a job runs, a warning when the canvas + * cannot apply them. `descriptionId` lets the row point its accessible description at the warning, since the row's + * controls are pointer-only. + */ +export const AdjustmentStatusIcon = ({ + descriptionId, + status, +}: { + descriptionId: string; + status: AdjustmentOwnerStatus; +}) => { + const { t } = useTranslation(); + if (status === 'computing') { + return ; + } + const message = adjustmentWarningMessage(t, status); + if (!message) { + return null; + } + return ( + + + + ); +}; + +/** + * The same warning, in words, above a spatial adjustment's settings, where the keyboard reaches it. The empty live + * region stays mounted so screen readers announce a warning when it appears. + */ +export const AdjustmentStatusNotice = ({ + engine, + ownerId, +}: { + engine: AdjustmentStatusEngine | null; + ownerId: string; +}) => { + const { t } = useTranslation(); + const message = adjustmentWarningMessage(t, useAdjustmentStatus(engine, ownerId)); + // Spaced below by its own padding, since an empty region must not add a gap. + return ( + + {message ? ( + + + {message} + + ) : null} + + ); +}; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerChildRow.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerChildRow.tsx index df77b82103c..9e8a9f5f894 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerChildRow.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerChildRow.tsx @@ -8,7 +8,10 @@ import { ContrastIcon, DropletIcon, FilmIcon, + FingerprintPatternIcon, + FocusIcon, GaugeIcon, + HazeIcon, ImageIcon, RainbowIcon, SlidersVerticalIcon, @@ -35,13 +38,16 @@ const stopPropagation = (event: { stopPropagation: () => void }): void => event. const CHILD_ROW_GLYPHS: Record = { 'adjustment-brightness-contrast': SunMediumIcon, + 'adjustment-clarity': FocusIcon, 'adjustment-curves': SplineIcon, + 'adjustment-dehaze': HazeIcon, 'adjustment-exposure': ApertureIcon, 'adjustment-hsl': DropletIcon, 'adjustment-hue': RainbowIcon, 'adjustment-invert': ContrastIcon, 'adjustment-levels': SlidersVerticalIcon, 'adjustment-noise': FilmIcon, + 'adjustment-texture': FingerprintPatternIcon, 'layer-region': WandSparklesIcon, 'mask-denoise': GaugeIcon, 'mask-noise': WavesIcon, diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerContextMenu.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerContextMenu.tsx index 696ae935204..241c4906109 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerContextMenu.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerContextMenu.tsx @@ -1,4 +1,5 @@ import type { + AdjustmentRefusal, CanvasAdjustmentEntry, CanvasColorLabel, CanvasDocumentContractV3, @@ -26,6 +27,7 @@ import { getDocumentIndex, getDocumentLayer, getSourceContentRect, + isAdjustmentRefusal, isGroupNode, isHideableLayer, isNodeHidden, @@ -45,7 +47,11 @@ import { useCanvasEngineRead, useLayerQuickSwitch, } from '@workbench/widgets/canvas/engineStoreHooks'; -import { reportLayerOperation, usePreparedCommit } from '@workbench/widgets/canvas/useStructuralCommit'; +import { + describeAdjustmentRefusal, + reportLayerOperation, + usePreparedCommit, +} from '@workbench/widgets/canvas/useStructuralCommit'; import { useActiveProjectId, useActiveProjectSelector, useWorkbenchCommands } from '@workbench/WorkbenchContext'; import { ArrowRightLeftIcon, @@ -115,6 +121,7 @@ import { getControlTransparencyEffectPatch, getRegionalGuidanceAutoNegativePatch, MASK_MODIFIER_DEFAULTS, + needsBakedControlPixels, } from './layerOps'; import { requestLayerProperties } from './layerPropertiesRequestStore'; import { LayerSaveToBoardSubmenu } from './LayerSaveToBoardSubmenu'; @@ -356,7 +363,8 @@ const LayerMenu = ({ ); const makeStatusError = useCallback( - (status: LayerActionErrorStatus): Error => new Error(t(LAYER_ACTION_ERROR_KEYS[status])), + (status: LayerActionErrorStatus | AdjustmentRefusal): Error => + new Error(typeof status === 'string' ? t(LAYER_ACTION_ERROR_KEYS[status]) : describeAdjustmentRefusal(status, t)), [t] ); @@ -410,7 +418,9 @@ const LayerMenu = ({ return; } void engine.layers.mergeSelectedRasterLayers(actionTargets({ layer, selectedIds })).then((result) => { - if (result === 'not-ready') { + if (isAdjustmentRefusal(result)) { + notify.error(t('widgets.layers.actions.actionFailed'), describeAdjustmentRefusal(result, t)); + } else if (result === 'not-ready') { notify.error(t('widgets.layers.actions.actionFailed'), t('widgets.layers.groupActions.mergeNotReady')); } else if (result === 'over-budget') { notify.error(t('widgets.layers.actions.actionFailed'), t('widgets.layers.groupActions.mergeOverBudget')); @@ -420,21 +430,21 @@ const LayerMenu = ({ }); }, [engine, layer, notify, selectedIds, t]); - const handleMerge = useCallback(() => { + const handleMerge = useCallback(async () => { if (!engine) { throw makeStatusError('not-ready'); } - const result = engine.layers.mergeLayerDown(layer.id); + const result = await engine.layers.mergeLayerDown(layer.id); if (result !== 'merged') { throw makeStatusError(result); } }, [engine, layer.id, makeStatusError]); - const handleRasterize = useCallback(() => { + const handleRasterize = useCallback(async () => { if (!engine) { throw makeStatusError('not-ready'); } - const result = engine.layers.rasterizeLayer(layer.id); + const result = await engine.layers.rasterizeLayer(layer.id); if (result !== 'rasterized') { throw makeStatusError(result); } @@ -460,7 +470,17 @@ const LayerMenu = ({ ); const convert = useCallback( - (targetType: CanvasLayerContract['type'], label: string) => { + async (targetType: CanvasLayerContract['type'], label: string) => { + if (targetType === 'control' && needsBakedControlPixels(layer)) { + if (!engine) { + throw makeStatusError('not-ready'); + } + const result = await engine.layers.convertLayerToControl(label, layer); + if (result !== 'converted') { + throw makeStatusError(result); + } + return; + } const converted = layer.type === 'raster' && targetType === 'control' ? convertRasterToControl(layer, base, defaultControlModel) @@ -662,6 +682,8 @@ const LayerMenu = ({ case 'not-ready': case 'over-budget': throw makeStatusError(result.status); + case 'adjustment-refused': + throw makeStatusError(result); case 'busy': throw new Error(t('widgets.layers.actions.cropBusy')); case 'unsupported': @@ -679,7 +701,7 @@ const LayerMenu = ({ } const result = await engine.exports.extractMaskedArea(layer.id); if (result.status !== 'extracted') { - throw makeStatusError(result.status); + throw makeStatusError(isAdjustmentRefusal(result) ? result : result.status); } }, [engine, layer.id, makeStatusError]); @@ -712,18 +734,29 @@ const LayerMenu = ({ } const result = await engine.layers.copyLayerToRaster(layer.id); if (result.status !== 'copied') { - throw makeStatusError(result.status); + throw makeStatusError(isAdjustmentRefusal(result) ? result : result.status); } }, [addCopy, engine, getActionLabel, layer, makeStatusError]); - const handleCopyToControl = useCallback(() => { - if (layer.type === 'raster') { + const handleCopyToControl = useCallback(async () => { + if (layer.type !== 'raster') { + return; + } + if (!needsBakedControlPixels(layer)) { addCopy( copyRasterToControl(layer, createLayerId(), base, defaultControlModel), getActionLabel('copy-to-control') ); + return; + } + if (!engine) { + throw makeStatusError('not-ready'); + } + const result = await engine.layers.copyLayerToControl(layer.id); + if (result.status !== 'copied') { + throw makeStatusError(isAdjustmentRefusal(result) ? result : result.status); } - }, [addCopy, base, defaultControlModel, getActionLabel, layer]); + }, [addCopy, base, defaultControlModel, engine, getActionLabel, layer, makeStatusError]); const handleCopyToInpaintMask = useCallback(() => { const id = createLayerId(); @@ -884,7 +917,7 @@ const LayerMenu = ({ : target === 'inpaint_mask' ? 'convert-to-inpaint-mask' : 'convert-to-regional-guidance'; - convert(target, getActionLabel(actionId)); + return convert(target, getActionLabel(actionId)); }, copyTo: handleCopyTo, copyToClipboard: handleCopyToClipboard, diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerRow.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerRow.tsx index 28906b73d1a..e25de492df2 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerRow.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerRow.tsx @@ -16,12 +16,13 @@ import { LockIcon, LockOpenIcon, } from 'lucide-react'; -import { memo, useCallback, useLayoutEffect, useMemo, useRef } from 'react'; +import { memo, useCallback, useId, useLayoutEffect, useMemo, useRef } from 'react'; import { useTranslation } from 'react-i18next'; import type { LayerRowCommands } from './layerRowCommands'; import type { LayerTreeRow } from './layerTreeRows'; +import { AdjustmentStatusIcon, useAdjustmentStatus } from './AdjustmentStatusIcon'; import { colorLabelHex } from './colorLabels'; import { ControlLayerWarningIcon } from './ControlLayerWarningIcon'; import { LayerActiveDot, TREE_ROW_FOCUS } from './LayerActiveDot'; @@ -114,6 +115,11 @@ const LayerRowComponent = ({ }, [setDragRef, setDropRef] ); + // Keyed by node id, so a row re-renders only for its own owner's status. + const adjustmentStatus = useAdjustmentStatus(engine, node.id); + const adjustmentStatusId = useId(); + // The warning icon is pointer-only like every row control; focusing the row reads its explanation instead. + const adjustmentWarned = adjustmentStatus !== 'idle' && adjustmentStatus !== 'computing'; const nameInput = useRef(null); // Escape abandons the draft; the blur that follows refocusing the row must not commit it. const renameCancelled = useRef(false); @@ -286,6 +292,7 @@ const LayerRowComponent = ({ ref={setRowRef} {...dragListeners} aria-current={primary ? 'true' : undefined} + aria-describedby={adjustmentWarned ? adjustmentStatusId : undefined} aria-expanded={group ? row.expanded : childCount > 0 ? childrenExpanded : undefined} aria-label={node.name} aria-level={vm.depth + 2} @@ -434,6 +441,7 @@ const LayerRowComponent = ({ /> )} + {layer ? : null} {/* One control cluster on the same rhythm as the stack header; slots a row cannot use are held open. */} diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayersTree.browser.test.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayersTree.browser.test.tsx index 72febccb004..ca98afc1212 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayersTree.browser.test.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayersTree.browser.test.tsx @@ -40,6 +40,8 @@ import { userEvent } from 'vitest/browser'; import type { ProjectedChildRow } from './layerChildRows'; import type { LayerRowCommands } from './layerRowCommands'; +import { ADJUSTMENT_SPINNER_DELAY_MS } from './AdjustmentStatusIcon'; +import { createFakeAdjustmentStatus } from './AdjustmentStatusIcon.testing'; import { clearLayerChildSelection, getLayerChildSelection } from './layerChildSelection'; import { getLayerRowCommits, resetLayerRowCommits } from './layerPanelDiagnostics'; import { LAYER_PANEL_DEGRADE_THRESHOLD, LAYER_ROW_HEIGHT_PX } from './layerPanelRows'; @@ -190,6 +192,7 @@ const withOperations = (engine: LayersTreeEngine): LayersTreeEngine => { attachCanvasOperations(engine, operationHarness as never); return engine; }; +const adjustmentStatus = createFakeAdjustmentStatus(); // The real engine handle is stable across document changes; this one reads the harness's latest document. let harnessDocument = createEmptyCanvasDocument(); const engine = withOperations({ @@ -206,7 +209,7 @@ const engine = withOperations({ }, }, exports: { exportBakedLayerBlob: exportBakedLayerBlob, hasExportableLayerContent: () => false }, - interaction: { get: () => false }, + interaction: { get: () => false, ...adjustmentStatus.interaction }, layers: { commitPrepared: (_label: string, edit: PreparedDocumentEdit) => { committedEdits.push(edit); @@ -487,6 +490,54 @@ describe('LayersTree keyboard and accessibility', () => { expect(group).toHaveAttribute('aria-selected', 'false'); }); + it('shows a delayed spinner while adjustments compute and a described warning when they cannot run', async () => { + await renderTree(nested()); + const top = treeitem('Top'); + const statusIcon = (row: HTMLElement, key: string): HTMLElement | null => + row.querySelector(`[role="img"][aria-label="widgets.layers.adjustments.status.${key}"]`); + const advance = (ms: number) => act(() => vi.advanceTimersByTime(ms)); + + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + // A job that ends before the delay never shows the spinner. + await act(() => adjustmentStatus.set('top', 'computing')); + await advance(ADJUSTMENT_SPINNER_DELAY_MS - 1); + expect(statusIcon(top, 'computing')).toBeNull(); + await act(() => adjustmentStatus.set('top', 'idle')); + await advance(ADJUSTMENT_SPINNER_DELAY_MS); + expect(statusIcon(top, 'computing')).toBeNull(); + + await act(() => adjustmentStatus.set('top', 'computing')); + await advance(ADJUSTMENT_SPINNER_DELAY_MS - 1); + expect(statusIcon(top, 'computing')).toBeNull(); + await advance(1); + expect(statusIcon(top, 'computing')).not.toBeNull(); + expect(top).not.toHaveAttribute('aria-describedby'); + } finally { + vi.useRealTimers(); + } + + await act(() => adjustmentStatus.set('top', 'too-large')); + expect(statusIcon(top, 'computing')).toBeNull(); + const warning = statusIcon(top, 'tooLarge')!; + expect(warning).not.toBeNull(); + // The icon is pointer-only, so focusing the row is how the keyboard reaches the explanation. + expect(top.getAttribute('aria-describedby')).toBe(warning.id); + expect(tabStops()).toHaveLength(1); + + // A group reports through the same store, by its own id; other rows stay clear. + await act(() => adjustmentStatus.set('g', 'over-budget')); + expect(statusIcon(treeitem('Group'), 'overBudget')).not.toBeNull(); + expect(treeitem('Bottom').querySelector('[role="img"]')).toBeNull(); + + await act(() => { + adjustmentStatus.set('top', 'idle'); + adjustmentStatus.set('g', 'idle'); + }); + expect(top.querySelector('[role="img"]')).toBeNull(); + expect(top).not.toHaveAttribute('aria-describedby'); + }); + it('walks rows with the arrow keys, opens and enters groups, climbs to the parent, and jumps with End', async () => { await renderTree(nested()); treeitem('Top').focus(); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayersTree.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayersTree.tsx index 2d97f886bfa..c4dc5ee43b6 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayersTree.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayersTree.tsx @@ -11,7 +11,12 @@ import { restrictToVerticalAxis } from '@dnd-kit/modifiers'; import { useMountEffect } from '@platform/react/useMountEffect'; import { toaster } from '@platform/ui'; import { Scrollable } from '@platform/ui/Scrollable'; -import { getDocumentIndex, getDocumentNode, lookupDocumentNodeState } from '@workbench/canvas-engine/api'; +import { + getDocumentIndex, + getDocumentNode, + isAdjustmentRefusal, + lookupDocumentNodeState, +} from '@workbench/canvas-engine/api'; import { publishLayerPanelSelection, readLayerPanelState, @@ -22,7 +27,7 @@ import { toggleLayerStackCollapsed, } from '@workbench/layerPanelState'; import { useCanvasOperation } from '@workbench/widgets/canvas/engineStoreHooks'; -import { usePreparedCommit } from '@workbench/widgets/canvas/useStructuralCommit'; +import { describeAdjustmentRefusal, usePreparedCommit } from '@workbench/widgets/canvas/useStructuralCommit'; import { useCallback, useEffectEvent, useLayoutEffect, useMemo, useRef, useState } from 'react'; import { defaultRangeExtractor, useVirtualizer, type Range } from 'react-hook-tanstack-virtual'; import { useTranslation } from 'react-i18next'; @@ -504,9 +509,16 @@ export const LayersTree = ({ event.preventDefault(); const { engine: currentEngine } = latest.current; if (currentEngine) { - void copyLayerToClipboard(key, { exportLayer: currentEngine.exports.exportBakedLayerBlob }) - .then((status) => status === 'ok' || Promise.reject(new Error(status))) - .catch(() => toaster.create({ title: t('widgets.layers.actions.copyFailed'), type: 'warning' })); + const copyFailed = (description?: string) => + toaster.create({ description, title: t('widgets.layers.actions.copyFailed'), type: 'warning' }); + void copyLayerToClipboard(key, { exportLayer: currentEngine.exports.exportBakedLayerBlob }).then( + (status) => { + if (status !== 'ok') { + copyFailed(isAdjustmentRefusal(status) ? describeAdjustmentRefusal(status, t) : undefined); + } + }, + () => copyFailed() + ); } return; } diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/RunLayerWorkflowDialog.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/RunLayerWorkflowDialog.tsx index 0474f9714de..9f2fdcb2280 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/RunLayerWorkflowDialog.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/RunLayerWorkflowDialog.tsx @@ -28,6 +28,7 @@ import { useQueryClient } from '@tanstack/react-query'; import { getCanvasOperations } from '@workbench/canvas-operations/api'; import { getActiveProjectGraph } from '@workbench/projectWorkflows'; import { useNotify } from '@workbench/useNotify'; +import { describeAdjustmentRefusal } from '@workbench/widgets/canvas/useStructuralCommit'; import { useActiveProjectSelector, useWorkbenchCommands } from '@workbench/WorkbenchContext'; import { useCallback, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; @@ -164,7 +165,10 @@ const SUCCESS_KEYS: Record = { staging: 'widgets.layers.runWorkflow.stagingSuccess', }; -const FAILURE_KEYS: Record, string> = { +const FAILURE_KEYS: Record< + Exclude, + string +> = { aborted: 'widgets.layers.runWorkflow.aborted', busy: 'widgets.layers.runWorkflow.busy', disabled: 'widgets.layers.runWorkflow.disabled', @@ -433,7 +437,9 @@ export const RunLayerWorkflowDialog = ({ return; } - setError(t(FAILURE_KEYS[result.status])); + setError( + result.status === 'adjustment-refused' ? describeAdjustmentRefusal(result, t) : t(FAILURE_KEYS[result.status]) + ); } catch (error) { if (session.isCurrent(request.token)) { setError(t('widgets.layers.runWorkflow.failed', { message: messageOf(error) })); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.test.ts index 2989544ec48..5a9c1892a85 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.test.ts @@ -340,6 +340,25 @@ describe('adjustment rows', () => { expect(layerChildRemoveLabelKey('adjustment-noise')).toBe('widgets.layers.modifiers.removeAdjustment'); }); + it('projects clarity, texture and dehaze entries with their signed amount as a plain strength', () => { + const rows = projectLayerChildRows( + rasterRow( + documentFrom([ + rasterWith([ + { amount: 35.4, id: 'c1', isEnabled: true, size: 40, type: 'clarity' }, + { amount: -120, id: 't1', isEnabled: false, size: 8, type: 'texture' }, + { amount: 0, id: 'd1', isEnabled: true, type: 'dehaze' }, + ]), + ]) + ).vm + ); + expect(rows.map((row) => [row.kind, row.itemId, row.isEnabled, row.detail])).toEqual([ + ['adjustment-clarity', 'c1', true, '+35'], + ['adjustment-texture', 't1', false, '-120'], + ['adjustment-dehaze', 'd1', true, '0'], + ]); + }); + it('renames an entry, clears the name with null, and refuses no-op renames and non-adjustment kinds', () => { const document = documentFrom([rasterWith(), regionalWith([referenceImage('ref1')])]); const named = layerChildRowCommand(document, { itemId: 'a1', layerId: 'r1' }, { name: 'Warm up', type: 'rename' }); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.ts index b0eab663f3d..ac18ed37881 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.ts @@ -30,7 +30,10 @@ export type LayerChildRowKind = | 'adjustment-hsl' | 'adjustment-hue' | 'adjustment-invert' - | 'adjustment-noise'; + | 'adjustment-noise' + | 'adjustment-clarity' + | 'adjustment-texture' + | 'adjustment-dehaze'; type AdjustmentChildKind = Extract; @@ -40,24 +43,30 @@ export const adjustmentChildKind = (type: CanvasAdjustmentEntry['type']): Adjust const ADJUSTMENT_KIND_OF: Record = { 'brightness-contrast': 'adjustment-brightness-contrast', + clarity: 'adjustment-clarity', curves: 'adjustment-curves', + dehaze: 'adjustment-dehaze', exposure: 'adjustment-exposure', hsl: 'adjustment-hsl', hue: 'adjustment-hue', invert: 'adjustment-invert', levels: 'adjustment-levels', noise: 'adjustment-noise', + texture: 'adjustment-texture', }; const CHILD_ROW_NAME_KEYS: Record, string> = { 'adjustment-brightness-contrast': 'widgets.layers.modifiers.brightnessContrast', + 'adjustment-clarity': 'widgets.layers.adjustments.clarity', 'adjustment-curves': 'widgets.layers.adjustments.curves', + 'adjustment-dehaze': 'widgets.layers.adjustments.dehaze', 'adjustment-exposure': 'widgets.layers.adjustments.exposure', 'adjustment-hsl': 'widgets.layers.adjustments.saturation', 'adjustment-hue': 'widgets.layers.adjustments.hue', 'adjustment-invert': 'widgets.layers.adjustments.invert', 'adjustment-levels': 'widgets.layers.adjustments.levels', 'adjustment-noise': 'widgets.layers.adjustments.noise', + 'adjustment-texture': 'widgets.layers.adjustments.texture', 'layer-region': 'widgets.layers.modifiers.regenerateRegion', 'mask-denoise': 'widgets.layers.modifiers.denoise', 'mask-noise': 'widgets.layers.modifiers.noise', @@ -84,6 +93,13 @@ const adjustmentDetail = (entry: CanvasAdjustmentEntry): string | null => { return entry.channel && entry.channel !== 'rgb' ? entry.channel.toUpperCase() : null; case 'noise': return `${Math.round(entry.amount * 100)}%`; + case 'clarity': + case 'texture': + case 'dehaze': { + // A signed strength, as on the backend nodes; it is not a percentage of any visible change. + const amount = Math.round(entry.amount); + return `${amount > 0 ? '+' : ''}${amount}`; + } case 'brightness-contrast': case 'curves': case 'invert': @@ -349,6 +365,9 @@ export const layerChildRemoveLabelKey = (kind: LayerChildRowKind): string => { case 'adjustment-hue': case 'adjustment-invert': case 'adjustment-noise': + case 'adjustment-clarity': + case 'adjustment-texture': + case 'adjustment-dehaze': return 'widgets.layers.modifiers.removeAdjustment'; } }; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.test.ts index 0782d6f96e0..dd1ba1c19ef 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.test.ts @@ -107,7 +107,7 @@ const makeEffects = (): LayerContextActionEffects => ({ extractMaskedArea: vi.fn(() => Promise.resolve()), fitToBbox: vi.fn(), group: vi.fn(), - mergeDown: vi.fn(), + mergeDown: vi.fn(() => Promise.resolve()), mergeSelected: vi.fn(), openProperties: vi.fn(), openRename: vi.fn(), @@ -115,7 +115,7 @@ const makeEffects = (): LayerContextActionEffects => ({ startSelectObject: vi.fn(), startFilter: vi.fn(), patchConfig: vi.fn(), - rasterize: vi.fn(), + rasterize: vi.fn(() => Promise.resolve()), reorder: vi.fn(), setColorLabel: vi.fn(), saveToBoard: vi.fn(() => Promise.resolve()), @@ -320,6 +320,7 @@ describe('layer context action registry', () => { getLayerContextActionDefinition('move-to-front').handler(context); getLayerContextActionDefinition('add-curves').handler(context); getLayerContextActionDefinition('add-noise-adjustment').handler(context); + getLayerContextActionDefinition('add-clarity').handler(context); getLayerContextActionDefinition('filter').handler(context); getLayerContextActionDefinition('intersect').handler(context); getLayerContextActionDefinition('copy-to-control').handler(context); @@ -329,6 +330,7 @@ describe('layer context action registry', () => { expect(effects.reorder).toHaveBeenCalledWith('front', 'move-to-front'); expect(effects.addAdjustment).toHaveBeenCalledWith('curves'); expect(effects.addAdjustment).toHaveBeenCalledWith('noise'); + expect(effects.addAdjustment).toHaveBeenCalledWith('clarity'); expect(effects.startFilter).toHaveBeenCalledWith(rasterLayer.id); expect(effects.booleanMerge).toHaveBeenCalledWith('intersect'); expect(effects.copyTo).toHaveBeenCalledWith('control'); @@ -482,6 +484,9 @@ describe('getLayerContextActions', () => { 'add-hsl', 'add-hue', 'add-invert', + 'add-texture', + 'add-clarity', + 'add-dehaze', 'add-noise-adjustment', 'add-regenerate-region', 'save-to-board', @@ -632,6 +637,9 @@ describe('getLayerContextActions', () => { 'add-hsl', 'add-hue', 'add-invert', + 'add-texture', + 'add-clarity', + 'add-dehaze', 'add-noise-adjustment', 'add-regenerate-region', 'filter', @@ -713,6 +721,9 @@ describe('getLayerContextActions', () => { 'add-hsl', 'add-hue', 'add-invert', + 'add-texture', + 'add-clarity', + 'add-dehaze', 'add-noise-adjustment', 'add-regenerate-region', 'save-to-board', diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.ts index 6b399766ba6..ad12fdf4ae3 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.ts @@ -33,9 +33,12 @@ import { DropletIcon, EyeIcon, FilmIcon, + FingerprintPatternIcon, + FocusIcon, EyeOffIcon, FolderPlusIcon, GaugeIcon, + HazeIcon, ImageIcon, ImagePlusIcon, LockIcon, @@ -70,6 +73,9 @@ export type LayerContextActionId = | 'add-hsl' | 'add-hue' | 'add-invert' + | 'add-texture' + | 'add-clarity' + | 'add-dehaze' | 'add-noise-adjustment' | 'merge-selected' | 'move-to-front' @@ -158,15 +164,15 @@ export interface LayerContextActionEffects { extractMaskedArea(): Promise; booleanMerge(operation: BooleanRasterOperation): Promise; copyTo(target: LayerType): void | Promise; - rasterize(): void; - convertTo(target: LayerType): void; + rasterize(): Promise; + convertTo(target: LayerType): void | Promise; patchConfig(kind: LayerConfigPatchKind): void; addReferenceImage(): void; addMaskModifier(field: 'noise' | 'denoise'): void; addAdjustment(type: CanvasAdjustmentEntry['type']): void; addLayerRegion(): void; setColorLabel(label: CanvasColorLabel | null): void; - mergeDown(): void; + mergeDown(): Promise; toggleVisibility(): void; toggleHidden(): void; toggleLock(): void; @@ -261,6 +267,15 @@ export const ADJUSTMENT_ADD_ITEMS: readonly AdjustmentAddItem[] = [ { defaultLabel: 'Saturation', icon: DropletIcon, labelKey: 'widgets.layers.adjustments.saturation', type: 'hsl' }, { defaultLabel: 'Hue', icon: RainbowIcon, labelKey: 'widgets.layers.adjustments.hue', type: 'hue' }, { defaultLabel: 'Invert', icon: ContrastIcon, labelKey: 'widgets.layers.adjustments.invert', type: 'invert' }, + // Detail adjustments, finest scale first. + { + defaultLabel: 'Texture', + icon: FingerprintPatternIcon, + labelKey: 'widgets.layers.adjustments.texture', + type: 'texture', + }, + { defaultLabel: 'Clarity', icon: FocusIcon, labelKey: 'widgets.layers.adjustments.clarity', type: 'clarity' }, + { defaultLabel: 'Dehaze', icon: HazeIcon, labelKey: 'widgets.layers.adjustments.dehaze', type: 'dehaze' }, { defaultLabel: 'Noise', icon: FilmIcon, labelKey: 'widgets.layers.adjustments.noise', type: 'noise' }, ]; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextMenuLayout.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextMenuLayout.test.ts index c2c872deb22..aacc8630544 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextMenuLayout.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextMenuLayout.test.ts @@ -88,7 +88,7 @@ describe('getLayerContextMenuLayout', () => { 'transform', 'rename', 'fit-to-bbox', - 'add-adjustment(add-brightness-contrast,add-exposure,add-levels,add-curves,add-hsl,add-hue,add-invert,add-noise-adjustment)', + 'add-adjustment(add-brightness-contrast,add-exposure,add-levels,add-curves,add-hsl,add-hue,add-invert,add-texture,add-clarity,add-dehaze,add-noise-adjustment)', 'add-regenerate-region', 'filter', 'select-object', diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerExportActions.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerExportActions.ts index 97546259b4b..a3aa9ed3dfa 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerExportActions.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerExportActions.ts @@ -1,5 +1,5 @@ import type { QueryClient } from '@tanstack/react-query'; -import type { ExportBakedLayerBlobResult } from '@workbench/canvas-engine/api'; +import type { AdjustmentRefusal, ExportBakedLayerBlobResult } from '@workbench/canvas-engine/api'; import { canOnBoard } from '@features/gallery/contracts'; import { @@ -17,8 +17,11 @@ import { isAccountScopeCurrent, } from '@platform/state/accountLifecycle'; -type LayerAssetExportFailureStatus = Exclude; -type LayerAssetExportResult = { status: 'ok'; blob: Blob } | { status: LayerAssetExportFailureStatus }; +type LayerAssetExportFailureStatus = Exclude; +type LayerAssetExportResult = + | { status: 'ok'; blob: Blob } + | { status: LayerAssetExportFailureStatus } + | AdjustmentRefusal; interface CopyLayerToClipboardDeps extends ClipboardAccess { exportLayer(layerId: string, options: { includeDisabled: boolean }): Promise; @@ -28,8 +31,11 @@ interface CopyLayerToClipboardDeps extends ClipboardAccess { export const copyLayerToClipboard = async ( layerId: string, { exportLayer, ...clipboard }: CopyLayerToClipboardDeps -): Promise<'ok' | LayerAssetExportFailureStatus> => { +): Promise<'ok' | LayerAssetExportFailureStatus | AdjustmentRefusal> => { const result = await exportLayer(layerId, { includeDisabled: true }); + if (result.status === 'adjustment-refused') { + return result; + } if (result.status !== 'ok') { return result.status; } @@ -51,7 +57,8 @@ export type SaveLayerToBoardResult = | 'stale' | 'board-view-only' | 'board-unavailable' - | LayerAssetExportFailureStatus; + | LayerAssetExportFailureStatus + | AdjustmentRefusal; /** * Owns the complete local-export → gallery-upload boundary: the baked layer is uploaded to `boardId` (`'none'` for @@ -85,6 +92,9 @@ export const saveLayerToBoard = async ( const result = await exportLayer(layerId, { includeDisabled: true, signal: owner.signal }); assertAccountScopeCurrent(owner); + if (result.status === 'adjustment-refused') { + return result; + } if (result.status !== 'ok') { return result.status; } diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.test.ts index 2b0c14e79ce..6908a605eb7 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.test.ts @@ -655,6 +655,29 @@ describe('layer copy and conversion parity', () => { }); }); + it('clones into a control layer only while the stack changes no pixel; masks keep reading alpha', () => { + const unchanged = { + ...raster, + adjustments: [ + { amount: 0, id: 'clarity', isEnabled: true, size: 40, type: 'clarity' as const }, + { id: 'invert', isEnabled: false, type: 'invert' as const }, + ], + }; + expect(convertRasterToControl(unchanged)).toMatchObject({ source: raster.source, transform }); + expect(copyRasterToControl(unchanged, 'c-copy')).toMatchObject({ source: raster.source, transform }); + + for (const entry of [ + { brightness: 0.2, contrast: 0, id: 'bc', isEnabled: true, type: 'brightness-contrast' as const }, + { amount: 30, id: 'dehaze', isEnabled: true, type: 'dehaze' as const }, + ]) { + const adjusted = { ...raster, adjustments: [entry] }; + expect(copyRasterToControl(adjusted, 'c-copy')).toBeNull(); + expect(convertRasterToControl(adjusted)).toBeNull(); + expect(convertRasterControlLayer(adjusted, 'control')).toBeNull(); + expect(convertRasterToInpaintMask(adjusted)?.mask.bitmap).toEqual(raster.source.bitmap); + } + }); + it('rejects parametric raster sources that cannot be represented as masks or controls', () => { const text = { ...raster, diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.ts index 5d40db00ca3..5f3c545d997 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.ts @@ -20,7 +20,12 @@ export type { CanvasStructuralEngine } from '@workbench/canvas-engine/api'; import { getRegionalGuidanceSupport } from '@features/generation/graph'; import { createItemId } from '@platform/browser/randomUuid'; -import { getSourceContentRect, isMergeableRasterLayer, mergeDownEligibility } from '@workbench/canvas-engine/api'; +import { + getSourceContentRect, + isIdentityAdjustments, + isMergeableRasterLayer, + mergeDownEligibility, +} from '@workbench/canvas-engine/api'; import { CONTROL_ADAPTER_DEFAULTS, createDefaultControlAdapter } from '@workbench/controlAdapters'; type LayerTransform = CanvasLayerBaseContract['transform']; @@ -232,6 +237,10 @@ export const DEFAULT_NOISE_AMOUNT = 0.1; /** A new noise entry's grain size: one independent sample per pixel, the finest grain. */ export const DEFAULT_NOISE_SIZE = 1; +/** Clarity's and Texture's default feature sizes in owner pixels, matching the backend nodes' defaults. */ +export const DEFAULT_CLARITY_SIZE = 40; +export const DEFAULT_TEXTURE_SIZE = 8; + /** A fresh uint32 pattern seed for a noise entry. */ export const createNoiseSeed = (): number => crypto.getRandomValues(new Uint32Array(1))[0]!; @@ -269,6 +278,12 @@ export const createAdjustmentEntry = (type: CanvasAdjustmentEntry['type']): Canv size: DEFAULT_NOISE_SIZE, type, }; + case 'clarity': + return { amount: 0, id, isEnabled: true, size: DEFAULT_CLARITY_SIZE, type }; + case 'texture': + return { amount: 0, id, isEnabled: true, size: DEFAULT_TEXTURE_SIZE, type }; + case 'dehaze': + return { amount: 0, id, isEnabled: true, type }; } }; @@ -413,6 +428,14 @@ export const canConvertRasterControl = (layer: CanvasLayerContract): boolean => }; type PixelLayer = CanvasRasterLayerContractV2 | CanvasControlLayerContract; + +/** + * Whether a raster layer's adjustments change its pixels. Control layers carry no adjustments, so such a layer reaches + * the control stack only with its pixels baked by the engine (`copyLayerToControl`, `convertLayerToControl`); its + * alpha, all a mask conversion reads, no adjustment changes. + */ +export const needsBakedControlPixels = (layer: CanvasLayerContract): boolean => + layer.type === 'raster' && !isIdentityAdjustments(layer.adjustments); type PixelSource = Extract; const cloneTransform = (layer: CanvasLayerContract): LayerTransform => ({ ...layer.transform }); @@ -463,6 +486,7 @@ const destinationBase = (layer: CanvasLayerContract, id: string, isCopy: boolean transform: cloneTransform(layer), }); +/** Clones the source and transform; null for a layer {@link needsBakedControlPixels}, which no clone represents. */ const pixelLayerToControl = ( layer: PixelLayer, id: string, @@ -470,7 +494,7 @@ const pixelLayerToControl = ( base?: string | null, model?: string | null ): CanvasControlLayerContract | null => { - const source = clonePixelSource(layer); + const source = needsBakedControlPixels(layer) ? null : clonePixelSource(layer); if (!source) { return null; } diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/psdExportNotice.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/psdExportNotice.ts index 3a9de715b3c..e07dd3104fd 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/psdExportNotice.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/psdExportNotice.ts @@ -1,4 +1,4 @@ -import type { PsdExportResult } from '@workbench/canvas-engine/api'; +import type { AdjustmentRefusal, PsdExportResult } from '@workbench/canvas-engine/api'; export type PsdExportNoticeKey = | 'widgets.layers.groupActions.exportNotReady' @@ -8,7 +8,10 @@ export type PsdExportNoticeKey = | 'widgets.layers.groupActions.exportStale' | 'widgets.layers.groupActions.exportAborted'; -export const getPsdExportNoticeKey = (result: PsdExportResult): PsdExportNoticeKey | null => { +/** The notice for a non-refusal result; an `AdjustmentRefusal` is worded by `describeAdjustmentRefusal`. */ +export const getPsdExportNoticeKey = ( + result: Exclude +): PsdExportNoticeKey | null => { switch (result) { case 'exported': return null; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/runLayerWorkflow.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/runLayerWorkflow.ts index 803000dfd3b..2bb1ec48776 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/runLayerWorkflow.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/runLayerWorkflow.ts @@ -10,6 +10,7 @@ import type { } from '@features/workflow/graph'; import type { InvocationTemplatesSnapshot } from '@features/workflow/react'; import type { + AdjustmentRefusal, CanvasStagingCandidateContract, CommitGeneratedImageOptions, CommitGeneratedImageResult, @@ -46,7 +47,8 @@ export type RunLayerWorkflowResult = | 'stale' | 'busy'; } - | { status: 'failed'; message: string; stage: LayerWorkflowFailureStage }; + | { status: 'failed'; message: string; stage: LayerWorkflowFailureStage } + | AdjustmentRefusal; export interface RunLayerWorkflowDeps { /** Plans this run's seeds from the project graph; see `reserveLayerWorkflowSeeds`. */ diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/useLayerStackActions.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/useLayerStackActions.ts index b9a7e9e9e72..5f2fa825fac 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/useLayerStackActions.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/useLayerStackActions.ts @@ -3,9 +3,14 @@ import type { CanvasEngineHandle } from '@workbench/canvas-operations/react'; import type { LucideIcon } from 'lucide-react'; import { toaster } from '@platform/ui'; -import { canMergeVisibleRasters, compileDocumentNodes, getDocumentLeaves } from '@workbench/canvas-engine/api'; +import { + canMergeVisibleRasters, + compileDocumentNodes, + getDocumentLeaves, + isAdjustmentRefusal, +} from '@workbench/canvas-engine/api'; import { useCanvasEngineRead } from '@workbench/widgets/canvas/engineStoreHooks'; -import { usePreparedCommit } from '@workbench/widgets/canvas/useStructuralCommit'; +import { describeAdjustmentRefusal, usePreparedCommit } from '@workbench/widgets/canvas/useStructuralCommit'; import { useActiveProjectName } from '@workbench/WorkbenchContext'; import { EyeIcon, EyeOffIcon, FileDownIcon, LayersIcon, PlusIcon } from 'lucide-react'; import { useMemo } from 'react'; @@ -80,6 +85,14 @@ export const useLayerStackActions = ( void engine.exports .exportRasterLayersToPsd(projectName) .then((result) => { + if (isAdjustmentRefusal(result)) { + toaster.create({ + description: describeAdjustmentRefusal(result, t), + title: t('widgets.layers.groupActions.exportFailed'), + type: 'warning', + }); + return; + } const noticeKey = getPsdExportNoticeKey(result); if (noticeKey) { toaster.create({ title: t(noticeKey), type: 'warning' }); @@ -98,7 +111,13 @@ export const useLayerStackActions = ( return; } void engine.layers.mergeVisibleRasterLayers().then((result) => { - if (result === 'not-ready') { + if (isAdjustmentRefusal(result)) { + toaster.create({ + description: describeAdjustmentRefusal(result, t), + title: t('widgets.layers.actions.actionFailed'), + type: 'warning', + }); + } else if (result === 'not-ready') { toaster.create({ title: t('widgets.layers.groupActions.mergeNotReady'), type: 'warning' }); } else if (result === 'over-budget') { toaster.create({ title: t('widgets.layers.groupActions.mergeOverBudget'), type: 'warning' }); diff --git a/tests/backend/image_util/test_shared_numerical_fixtures.py b/tests/backend/image_util/test_shared_numerical_fixtures.py index 94d3ec21936..a125eb1143c 100644 --- a/tests/backend/image_util/test_shared_numerical_fixtures.py +++ b/tests/backend/image_util/test_shared_numerical_fixtures.py @@ -1,19 +1,250 @@ +"""Numerical contracts shared with the canvas spatial-adjustment kernels. + +webv2 previews Clarity, Texture and Contrast Dehaze on the canvas with its own TypeScript ports +(`workbench/canvas-engine/render/spatial/`). Its tests read the fixtures in that directory's +`__fixtures__/`; these tests pin the same files to the Python implementations, so a backend change that +moves the numerics fails here until the parity fixture is regenerated and the ports follow. +""" + +import base64 import json +import os +from collections.abc import Callable from pathlib import Path +from typing import Any +import cv2 import numpy as np import torch from PIL import Image +from invokeai.backend.image_util.clarity import clarity from invokeai.backend.image_util.color import prepare_lightness +from invokeai.backend.image_util.contrast_dehaze import contrast_dehaze from invokeai.backend.image_util.guided_filter import guided_filter from invokeai.backend.image_util.multiscale import edge_aware_band +from invokeai.backend.image_util.texture import texture -REPO_ROOT = Path(__file__).resolve().parents[3] -FIXTURE_PATH = ( - REPO_ROOT - / "invokeai/frontend/webv2/src/features/generation/core/__fixtures__/shared_image_manipulation_numerics.json" +FIXTURE_DIR = ( + Path(__file__).resolve().parents[3] + / "invokeai/frontend/webv2/src/workbench/canvas-engine/render/spatial/__fixtures__" ) +FIXTURE_PATH = FIXTURE_DIR / "shared_image_manipulation_numerics.json" +PARITY_FIXTURE_PATH = FIXTURE_DIR / "spatial_adjustment_parity.json" + +REGEN_HINT = ( + "Regenerate it with:\n" + f" REGEN_SPATIAL_PARITY_FIXTURE=1 pytest {Path(__file__).name}\n" + "then make webv2's spatial kernels pass `evaluateSpatialSteps.test.ts` against it." +) + +# The parity fixture holds only objects, strings and integers, so `json.dumps(indent=2)` is already +# oxfmt's output and webv2's `format:check` accepts the regenerated file. Keep number arrays out of it. + +_STEPS: dict[str, Callable[..., Image.Image]] = { + "clarity": lambda image, step: clarity(image, step["amount"], step["size"]), + "texture": lambda image, step: texture(image, step["amount"], step["size"]), + "dehaze": lambda image, step: contrast_dehaze(image, step["amount"]), +} + + +def _encode(array: np.ndarray) -> str: + return base64.b64encode(np.ascontiguousarray(array).tobytes()).decode("ascii") + + +def _decode(text: str, dtype: type[np.generic], shape: tuple[int, ...]) -> np.ndarray: + return np.frombuffer(base64.b64decode(text), dtype=dtype).reshape(shape) + + +def _visible_raster_labels(hidden: np.ndarray) -> np.ndarray: + """cv2's nearest-visible labels, as each pixel's visible pixel's index among visible pixels in raster order.""" + _, labels = cv2.distanceTransformWithLabels( + np.ascontiguousarray(hidden, dtype=np.uint8), cv2.DIST_L2, 5, labelType=cv2.DIST_LABEL_PIXEL + ) + # DIST_LABEL_PIXEL numbers the zero (visible) pixels from 1 in raster order. + return (labels - 1).astype(" np.ndarray: + """Runs the nodes on the alpha bounds, as the canvas does, leaving everything outside them untouched.""" + visible_rows, visible_columns = np.nonzero(rgba[..., 3]) + if visible_rows.size == 0: + return rgba.copy() + bounds = ( + slice(visible_rows.min(), visible_rows.max() + 1), + slice(visible_columns.min(), visible_columns.max() + 1), + ) + image = Image.fromarray(np.ascontiguousarray(rgba[bounds]), mode="RGBA") + for step in steps: + image = _STEPS[step["kind"]](image, step) + result = rgba.copy() + result[bounds] = np.asarray(image, dtype=np.uint8) + return result + + +def _label_masks(rng: np.random.Generator) -> dict[str, np.ndarray]: + corners = np.ones((7, 9), dtype=bool) + corners[[0, 0, -1, -1], [0, -1, 0, -1]] = False + centre = np.ones((5, 5), dtype=bool) + centre[2, 2] = False + ring = np.ones((11, 11), dtype=bool) + ring[2, 2:9] = ring[8, 2:9] = ring[2:9, 2] = ring[2:9, 8] = False + row = np.ones((1, 12), dtype=bool) + row[0, [3, 8]] = False + column = np.ones((12, 1), dtype=bool) + column[[2, 9], 0] = False + masks = {"equidistant_corners": corners, "single_centre": centre, "hollow_ring": ring, "row": row, "column": column} + for name, shape, hidden_share in ( + ("random_sparse", (13, 17), 0.97), + ("random_half", (19, 23), 0.5), + ("random_dense", (8, 31), 0.9), + ): + mask = rng.random(shape) < hidden_share + mask[rng.integers(shape[0]), rng.integers(shape[1])] = False + masks[name] = mask + return masks + + +def _parity_image(rng: np.random.Generator, width: int, height: int) -> np.ndarray: + """Detailed RGBA content whose alpha bounds are the whole image, with holes, partial alpha and saturated color.""" + y, x = np.mgrid[0:height, 0:width].astype(np.float64) + rgb = np.stack( + ( + 255 * x / max(width - 1, 1), + 128 + 100 * np.sin(x / 2.5) * np.cos(y / 3.5), + 255 * y / max(height - 1, 1), + ), + axis=-1, + ) + rgb += rng.normal(0, 18, rgb.shape) + # Saturated colors leave the sRGB gamut once adjusted. Exact primaries sit on the boundaries between the gamut + # clip's cusp approximations, and colors with a 0 or 255 channel sit on the gamut boundary itself, where + # float32 round-trip noise must not decide whether the clip runs; both sides must agree on them. + if width >= 12 and height >= 9: + rgb[height // 3 : height // 3 + 3, width // 2 : width // 2 + 6] = np.array( + [[255, 0, 0], [0, 255, 0], [0, 0, 255], [250, 20, 10], [30, 20, 250], [245, 240, 20]] + )[None] + rgb[height // 3 + 3 : height // 3 + 6, 2:6] = (0, 17, 166) + rgba = np.empty((height, width, 4), dtype=np.uint8) + rgba[..., :3] = np.clip(np.rint(rgb), 0, 255).astype(np.uint8) + alpha = np.full((height, width), 255, dtype=np.uint8) + partial = rng.random((height, width)) < 0.12 + alpha[partial] = rng.integers(1, 255, int(partial.sum()), dtype=np.uint8) + alpha[rng.random((height, width)) < 0.06] = 0 + alpha[2:6, 3:8] = 0 + alpha[[0, 0, -1, -1], [0, -1, 0, -1]] = 255 + rgba[..., 3] = alpha + # Hidden pixels keep arbitrary color bytes, which the adjustments must neither read nor rewrite. + hidden = alpha == 0 + rgba[hidden, :3] = rng.integers(0, 256, (int(hidden.sum()), 3), dtype=np.uint8) + return rgba + + +def _flat_image(width: int, height: int, rgb: tuple[int, int, int]) -> np.ndarray: + rgba = np.full((height, width, 4), 255, dtype=np.uint8) + rgba[..., :3] = rgb + return rgba + + +def _padded(rng: np.random.Generator, content: np.ndarray, width: int, height: int, left: int, top: int) -> np.ndarray: + """Content inside a larger transparent buffer whose hidden colors are arbitrary.""" + rgba = np.zeros((height, width, 4), dtype=np.uint8) + rgba[..., :3] = rng.integers(0, 256, (height, width, 3), dtype=np.uint8) + rgba[top : top + content.shape[0], left : left + content.shape[1]] = content + return rgba + + +_PARITY_CASES: tuple[tuple[str, Callable[[np.random.Generator], np.ndarray], list[dict[str, Any]]], ...] = ( + ("clarity_default_size", lambda rng: _parity_image(rng, 24, 18), [{"kind": "clarity", "amount": 80, "size": 40}]), + ( + "clarity_size_10_half_even_radius", + lambda rng: _parity_image(rng, 24, 18), + [{"kind": "clarity", "amount": 120, "size": 10}], + ), + ("clarity_negative", lambda rng: _parity_image(rng, 24, 18), [{"kind": "clarity", "amount": -150, "size": 25}]), + ("texture_positive_gated", lambda rng: _parity_image(rng, 24, 18), [{"kind": "texture", "amount": 150, "size": 8}]), + ("texture_negative", lambda rng: _parity_image(rng, 24, 18), [{"kind": "texture", "amount": -120, "size": 6}]), + ("dehaze_positive", lambda rng: _parity_image(rng, 24, 18), [{"kind": "dehaze", "amount": 90}]), + ("dehaze_negative", lambda rng: _parity_image(rng, 24, 18), [{"kind": "dehaze", "amount": -140}]), + ( + "dehaze_short_side_20_half_even_radius", + lambda rng: _parity_image(rng, 28, 20), + [{"kind": "dehaze", "amount": 70}], + ), + ( + "dehaze_short_side_36_half_even_radius", + lambda rng: _parity_image(rng, 40, 36), + [{"kind": "dehaze", "amount": 200}], + ), + ( + "clarity_then_dehaze", + lambda rng: _parity_image(rng, 24, 18), + [{"kind": "clarity", "amount": 60, "size": 12}, {"kind": "dehaze", "amount": 50}], + ), + ( + "flat_zero_channel_clarity", + lambda _: _flat_image(6, 5, (0, 17, 166)), + [{"kind": "clarity", "amount": 100, "size": 40}], + ), + ("flat_pure_blue_dehaze", lambda _: _flat_image(6, 5, (0, 0, 255)), [{"kind": "dehaze", "amount": 50}]), + ("flat_pure_red_dehaze", lambda _: _flat_image(6, 5, (255, 0, 0)), [{"kind": "dehaze", "amount": 120}]), + ( + "cropped_alpha_bounds", + lambda rng: _padded(rng, _parity_image(rng, 20, 14), 31, 22, 6, 3), + [{"kind": "clarity", "amount": 90, "size": 15}, {"kind": "dehaze", "amount": 60}], + ), + ( + "single_row_alpha_bounds", + lambda rng: _padded(rng, _parity_image(rng, 15, 1), 19, 7, 2, 4), + [{"kind": "clarity", "amount": 150, "size": 8}, {"kind": "texture", "amount": 120, "size": 4}], + ), + ( + "single_column_alpha_bounds", + lambda rng: _padded(rng, _parity_image(rng, 1, 13), 6, 16, 3, 1), + [{"kind": "dehaze", "amount": 80}, {"kind": "texture", "amount": -90, "size": 3}], + ), +) + + +def _render_parity_fixture() -> dict[str, Any]: + rng = np.random.default_rng(9891) + labels = [] + for name, hidden in _label_masks(rng).items(): + height, width = hidden.shape + labels.append( + { + "name": name, + "width": width, + "height": height, + "hidden": _encode(hidden.astype(np.uint8)), + "labels": _encode(_visible_raster_labels(hidden)), + } + ) + adjustments = [] + for name, build, steps in _PARITY_CASES: + rgba = build(rng) + height, width = rgba.shape[:2] + adjustments.append( + { + "name": name, + "width": width, + "height": height, + "steps": steps, + "rgba": _encode(rgba), + "expected": _encode(_apply_steps(rgba, steps)), + } + ) + return { + "description": ( + "Written by tests/backend/image_util/test_shared_numerical_fixtures.py. Labels: base64 uint8 hidden " + "masks and base64 little-endian int32 visible-pixel raster indices from cv2.distanceTransformWithLabels. " + "Adjustments: base64 RGBA8 inputs and the Python nodes' outputs, steps applied in order to the alpha " + "bounds." + ), + "opencv": cv2.__version__, + "labels": labels, + "adjustments": adjustments, + } def test_shared_numerical_fixtures_match_python_cores() -> None: @@ -46,3 +277,29 @@ def test_shared_numerical_fixtures_match_python_cores() -> None: ) np.testing.assert_allclose(lightness, np.asarray(adapter["expected_oklab"])[:, :, 0], atol=adapter["atol"], rtol=0) assert color_tensor.dtype == torch.float32 + + +def test_spatial_parity_fixture_matches_the_python_adjustments() -> None: + if os.environ.get("REGEN_SPATIAL_PARITY_FIXTURE"): + PARITY_FIXTURE_PATH.write_text( + json.dumps(_render_parity_fixture(), indent=2) + "\n", encoding="utf-8", newline="\n" + ) + + assert PARITY_FIXTURE_PATH.exists(), f"{PARITY_FIXTURE_PATH} is missing. {REGEN_HINT}" + fixture = json.loads(PARITY_FIXTURE_PATH.read_text(encoding="utf-8")) + + for case in fixture["labels"]: + shape = (case["height"], case["width"]) + hidden = _decode(case["hidden"], np.uint8, shape).astype(bool) + expected = _decode(case["labels"], np.dtype("