diff --git a/docs/src/content/docs/Users Guide/02.Projects/introduction.mdx b/docs/src/content/docs/Users Guide/02.Projects/introduction.mdx index 1c4eda91cb4..91d42441d97 100644 --- a/docs/src/content/docs/Users Guide/02.Projects/introduction.mdx +++ b/docs/src/content/docs/Users Guide/02.Projects/introduction.mdx @@ -170,6 +170,6 @@ In the Workflow Library dialog, **This project** lists the project's own workflo - **Projects belong to one account.** With [multiple users](/configuration/multi-user-mode/user-guide/) enabled, each person sees only their own projects, and there is no project sharing. Export and import a `.invk` file to pass a project to someone else. - **Undo history is not saved** with the project; it is cleared when you reload or close it. - **Invoke needs the server to open projects.** If it cannot reach the server, a **Projects are temporarily unavailable** screen offers to retry and to export any unsaved work your browser is holding. -- **Newer projects need a newer Invoke.** A project saved by a newer version of Invoke shows **Requires a newer Invoke** and cannot be opened until you upgrade. +- **Newer projects need a newer Invoke.** A project saved by a newer version of Invoke shows **Requires a newer Invoke** and cannot be opened until you upgrade. Some features raise this requirement for good: once a project has had a Noise adjustment on the canvas, versions of Invoke without Noise adjustments refuse to open it, even after you remove the adjustment. - **Size limits.** A project file can be up to 2 GB, with up to 20,000 files. Export reports a project that is too large to package. - **Clear saved data…** under **Data & workspace** in Settings deletes *all* of your projects and settings at once, and cannot be undone. diff --git a/docs/src/content/docs/Users Guide/07.Canvas/03.layers-panel.mdx b/docs/src/content/docs/Users Guide/07.Canvas/03.layers-panel.mdx index 9e2caf58c41..65bb138e11e 100644 --- a/docs/src/content/docs/Users Guide/07.Canvas/03.layers-panel.mdx +++ b/docs/src/content/docs/Users Guide/07.Canvas/03.layers-panel.mdx @@ -79,14 +79,14 @@ The icons at the top **Arrange** the layer (move it to the front, forward, backw * **For every layer:** * **Rename layer** and **Fit to bbox** (scale the layer to fit inside the generation frame). - * **Merge down**. + * **Merge down** — combines the layer with the one below. Each layer's adjustments are baked into the merged pixels; undo restores both layers with their adjustments. * **Copy to** — including **Copy layer to clipboard**, and copies to other layer types. * **Crop layer to bbox** and **Save layer to board** (save it to the Gallery as an upload, in Uncategorized or a board you pick from its submenu). * **Disable layer**, **Lock layer**, **Bookmark for quick switch** and **Color label**. A bookmarked layer is where **Q** switches on the canvas; press **Q** again to return. The bookmark is temporary and is not saved in the project. It can reset when the canvas is released, for example after switching away from its project, and resets on reload. * **Delete layer**. * **Raster layers:** * **Transform**. - * **Add adjustment** — Brightness/Contrast, Exposure, Levels, Curves, Saturation, Hue or Invert. Adjustments are non-destructive and appear as child rows. + * **Add adjustment** — Brightness/Contrast, Exposure, Levels, Curves, Saturation, Hue, Invert or Noise. Adjustments are non-destructive and appear as child rows. Noise adds film grain. * **Add regenerate region** — treats the layer's own shape as an inpaint mask. * **Filter**, **Select object** and **Run workflow** — process the layer; see the [Canvas guide](/users-guide/canvas/canvas-guide/#filters-and-object-selection). * **Boolean operations** — combine the layer with the one below: intersect, cut out, cut away or exclude. @@ -120,6 +120,23 @@ The Layer section is dimmed while results are being staged or a canvas generatio Adjustment child rows each have their own settings, such as **Brightness** and **Contrast**, **Exposure**, **Levels** and **Curves** with a channel selector, **Saturation**, and **Hue**. A regenerate region has a **Fill color** and **Fill style**. +**Noise** adds film grain with these settings: + +* **Distribution** — **Gaussian** for natural, film-like grain, or **Uniform** for an even spread. Both have the same strength at the same amount; larger grain sizes look gaussian either way. +* **Grain** — **Mono** changes brightness only; **Color** varies each color channel separately; **Chroma** varies only hue and colorfulness and leaves brightness alone, like the color speckle of a digital sensor. +* **Tonal response** — **Midtones** (the default) gives full grain in the midtones and fades it to none in pure black and white, as film does; **Even** gives the same grain at every tone. +* **Size** — how large each grain is, from 1 px (every pixel varies on its own, the default) to 8 px. Larger grain is softer and clumpier but keeps the same strength. +* **Amount** — the grain's strength. 2–5% gives subtle grain, about 10% (the default) clearly visible grain, and high values very heavy noise. +* **New pattern** (dice button) — picks a different random grain pattern. + +Things to know about Noise: + +* **The pattern stays put.** Each Noise adjustment keeps its grain pattern until you pick a new one, and reloading the project keeps it. Duplicating a Noise adjustment gives the copy a new pattern. +* **The grain you see is the grain you get.** Generating, exporting, merging and saving to PSD all use the same pattern and strength. One exception: PSD has no group adjustments, so a group's Noise is baked onto each of its layers separately, and where semi-transparent layers in the group overlap, the PSD can look slightly different. +* **Grain follows its layer.** It moves, scales and rotates with the layer; on a group, it stays fixed to the canvas instead. While you drag a selection, the dragged pixels keep their grain; once placed, they take the layer's grain at their new position. +* **Baking.** Rasterizing a text, shape or gradient layer whose Noise adjustment is enabled bakes its whole adjustment stack into the pixels and removes it, so the grain stays where it was. **Merge down** bakes each layer's adjustments the same way. +* **Older versions of Invoke.** Once a project has had a Noise adjustment, it needs a version of Invoke with Noise adjustments, even after you remove the adjustment; older versions refuse to open it. + ### Control layer * **Adapter type** — ControlNet, T2I Adapter, Control LoRA, Z-Image ControlNet or Anima ControlNet-LLLite, depending on the model selected in the Generate panel. diff --git a/docs/src/content/docs/development/Front End/project-files.mdx b/docs/src/content/docs/development/Front End/project-files.mdx index 664b00d8c01..e1f6311bd9f 100644 --- a/docs/src/content/docs/development/Front End/project-files.mdx +++ b/docs/src/content/docs/development/Front End/project-files.mdx @@ -39,10 +39,21 @@ canvas v3 document rides this seam: every save sends the document together with `minimum_canvas_schema_version` it requires (3 for any v3 canvas), so a client that still declares a maximum of v2 is refused before it can overwrite that document. -The frontend derives the requested floor from the highest declared version among the live canvas -and every queued canvas snapshot. It also retains the server's monotonic floor in the offline sync -map, so importing, duplicating, conflict recovery, and deletion recovery cannot accidentally create -a copy with weaker compatibility metadata than the document or its source project requires. +A canvas adopts a newer schema only when its content needs one, that is, when an older reader +would drop part of it on load and lose it on the next autosave. Schema 4 marks custom-font +typography (a text source with `fontRef`, `fontStyle` or `fontVariations`). Schema 5 marks a noise +adjustment, enabled or not, on any layer or group at any depth; an older reader would drop the +whole adjustment stack holding it. The live document and each saved canvas snapshot are versioned +this way, and the canvas state takes the newest of them. Adoption is sticky: removing the feature +does not lower the version, as the server floor cannot be lowered either. A canvas with neither +stays at 3, so older builds keep opening it. A build refuses a canvas that declares a schema above +its maximum as an unsupported version before parsing it, and once the floor is raised the server +refuses the project to a client declaring a lower maximum. + +The frontend derives the requested floor from the live canvas state's version. It also retains the +server's monotonic floor in the offline sync map, so importing, duplicating, conflict recovery, and +deletion recovery cannot accidentally create a copy with weaker compatibility metadata than the +document or its source project requires. If another client raises the floor while a project is already open, the next save becomes a terminal per-project schema refusal instead of a generic network failure. The local document still diff --git a/invokeai/frontend/webv2/performance/architecture-baseline.json b/invokeai/frontend/webv2/performance/architecture-baseline.json index 285496f99a2..0e6c8ec4b7a 100644 --- a/invokeai/frontend/webv2/performance/architecture-baseline.json +++ b/invokeai/frontend/webv2/performance/architecture-baseline.json @@ -2,15 +2,15 @@ "build": { "launchpad": { "baseline": { - "brotliBytes": 802380, + "brotliBytes": 804953, "cssRawBytes": 2159, "fontRawBytes": 219480, - "gzipBytes": 801520, + "gzipBytes": 804296, "imageRawBytes": 44398, - "initialRawBytes": 2066324, + "initialRawBytes": 2074195, "largestAssetRawBytes": 752422, "otherAssetRawBytes": 0, - "ownedRawBytes": 101787, + "ownedRawBytes": 103111, "requestCount": 35, "scriptRequestCount": 19, "sourceOwners": [ @@ -350,15 +350,15 @@ }, "editor": { "baseline": { - "brotliBytes": 1201065, + "brotliBytes": 1213758, "cssRawBytes": 2159, "fontRawBytes": 219480, - "gzipBytes": 1194189, + "gzipBytes": 1206518, "imageRawBytes": 44398, - "initialRawBytes": 3306361, + "initialRawBytes": 3341503, "largestAssetRawBytes": 752422, "otherAssetRawBytes": 0, - "ownedRawBytes": 155629, + "ownedRawBytes": 156608, "requestCount": 71, "scriptRequestCount": 55, "sourceOwners": [ @@ -859,6 +859,7 @@ "source:src/workbench/canvas-engine/render/maskFill.ts", "source:src/workbench/canvas-engine/render/rasterComposite.ts", "source:src/workbench/canvas-engine/render/rasterizers/textRasterizer.ts", + "source:src/workbench/canvas-engine/render/thumbnail.ts", "source:src/workbench/canvas-engine/transform/transformMath.ts", "source:src/workbench/canvasDimsSync.ts", "source:src/workbench/canvasLayerContent.ts", @@ -1111,7 +1112,7 @@ "source": "src/app/WorkbenchApp.tsx" } }, - "capturedAt": "2026-10-20", + "capturedAt": "2026-10-23", "developmentInvalidation": { "platformUiBarrel": { "specifier": "@platform/ui", diff --git a/invokeai/frontend/webv2/performance/browser-baseline.json b/invokeai/frontend/webv2/performance/browser-baseline.json index 0f02b90cdfa..bc55fde72b5 100644 --- a/invokeai/frontend/webv2/performance/browser-baseline.json +++ b/invokeai/frontend/webv2/performance/browser-baseline.json @@ -159,12 +159,12 @@ "cssRawBytes": 0, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 305523, + "largestAssetRawBytes": 311154, "otherRawBytes": 0, "requestCount": 9, - "scriptRawBytes": 524052, + "scriptRawBytes": 529607, "scriptRequestCount": 9, - "totalRawBytes": 524052 + "totalRawBytes": 529607 }, "domContentLoadedMedianMs": 100.40000009536743, "id": "editor-canvas", @@ -184,9 +184,9 @@ "largestAssetRawBytes": 752422, "otherRawBytes": 231257, "requestCount": 96, - "scriptRawBytes": 4775202, + "scriptRawBytes": 4823800, "scriptRequestCount": 91, - "totalRawBytes": 5076680 + "totalRawBytes": 5126937 }, "routeReadyMedianMs": 1095.3000001907349, "scriptSourceOwnerSet": "editor-canvas-static", @@ -197,12 +197,12 @@ "cssRawBytes": 0, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 305523, + "largestAssetRawBytes": 311154, "otherRawBytes": 0, "requestCount": 9, - "scriptRawBytes": 524052, + "scriptRawBytes": 529607, "scriptRequestCount": 9, - "totalRawBytes": 524052 + "totalRawBytes": 529607 }, "domContentLoadedMedianMs": 97.7000002861023, "id": "editor-canvas", @@ -222,9 +222,9 @@ "largestAssetRawBytes": 752422, "otherRawBytes": 231257, "requestCount": 96, - "scriptRawBytes": 4775202, + "scriptRawBytes": 4823800, "scriptRequestCount": 91, - "totalRawBytes": 5076680 + "totalRawBytes": 5126937 }, "routeReadyMedianMs": 1042.5, "scriptSourceOwnerSet": "editor-canvas-static", @@ -260,7 +260,7 @@ "largestAssetRawBytes": 752422, "otherRawBytes": 231257, "requestCount": 94, - "scriptRawBytes": 4702888, + "scriptRawBytes": 4750033, "scriptRequestCount": 88, "totalRawBytes": 5019779 }, @@ -298,7 +298,7 @@ "largestAssetRawBytes": 752422, "otherRawBytes": 231257, "requestCount": 94, - "scriptRawBytes": 4702888, + "scriptRawBytes": 4750033, "scriptRequestCount": 88, "totalRawBytes": 5019779 }, diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index f8846ef359c..58f58f6d92e 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -4249,6 +4249,26 @@ "hue": "Hue", "invert": "Invert", "invertHint": "Inverts the layer’s colors. This adjustment has no options.", + "noise": "Noise", + "noiseAmount": "Amount", + "noiseDistribution": "Distribution", + "noiseDistributions": { + "gaussian": "Gaussian", + "uniform": "Uniform" + }, + "noiseGrain": "Grain", + "noiseGrains": { + "monochrome": "Mono", + "color": "Color", + "chroma": "Chroma" + }, + "noiseResponse": "Tonal response", + "noiseResponses": { + "midtones": "Midtones", + "flat": "Even" + }, + "noiseSize": "Size", + "newNoisePattern": "New pattern", "saturation": "Saturation", "transparencyLock": "Lock transparency" }, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/ARCHITECTURE.md b/invokeai/frontend/webv2/src/workbench/canvas-engine/ARCHITECTURE.md index 95ef03401b0..8945fe6fc77 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/ARCHITECTURE.md +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/ARCHITECTURE.md @@ -142,9 +142,11 @@ Staged results use a separate compressed-Blob cache instead of retaining a decod Damage is explicit: `FrameDamage` is full repaint, no visible repaint, or layer-local regions. Scheduler widens to full on any invalidation without a region, a viewport change or a forced repaint. Compositor resolves regions through frame's matrices into one padded screen rect: regions entirely off target draw nothing; bounded repaint clips to the rect and culls every leaf, group scope and staged preview whose screen bounds miss it. Unknown layers or non-finite geometry repaint everything. -Compositing always renders through canonical `RasterBackend`; no approximate backend-free path. Filter previews and floating selection of a grouped member compose inside group's isolated surface, under group's opacity, blend and adjustments, exactly as member's committed pixels would. Committed content composes with same semantics for display, sampling and export; previews and floats are display-only. Layer's own adjustments apply to its layer-local pixels before resampling: export draws display's memoized adjusted copy of a leased live cache, and otherwise adjusts only the layer region the output samples (plus one pixel of resampling margin). Downscaled layer samples more pixels than output holds, so a detached copy of it is adjusted after resampling instead; export work stays bounded by output size. +Compositing always renders through canonical `RasterBackend`; no approximate backend-free path. Filter previews and floating selection of a grouped member compose inside group's isolated surface, under group's opacity, blend and adjustments, exactly as member's committed pixels would. Committed content composes with same semantics for display, sampling and export; previews and floats are display-only. Layer's own adjustments apply to its layer-local pixels before resampling: export draws display's memoized adjusted copy of a leased live cache when one is current, never building one (a layer-sized copy would exceed the export's reservation), and otherwise adjusts only the layer region the output samples (plus one pixel of resampling margin). When a downscaled layer samples more pixels than output holds and its stack is position independent (color only), a detached copy of it is adjusted after resampling instead, keeping export work bounded by output size; composite reservations count each such scratch buffer at output size. A position-dependent stack (noise) never takes that shortcut: resampling must average its grain exactly as on screen, so its layer-local sampled region is adjusted and reserved even when larger than the output. Composite reservations plan that region from estimated content bounds; when a layer's rasterized pixels (measured text) sample more, rendering reserves the excess before allocating it and refuses as over-budget when it does not fit. World-space bakes (merges, boolean merges, crops, copies) and PSD always adjust the layer's own stack on a layer-local copy, or draw the display's current adjusted copy, and then resample it. Merge Down bakes both layers' own stacks the same way before the upper layer resamples into the lower layer's space, so the merged layer carries no stack; its undo snapshots stay raw and undo restores both layers with their stacks. Rasterize bakes a position-dependent stack (all of it) into the layer-local source render before the transform and drops it from the paint contract, since kept it would grain the baked pixel grid; a color-only stack stays on the contract. Thumbnails downscale the display's current adjusted copy when one exists. -Group surface is keyed by everything except its members' pixels and float's placement (scope shape, member appearance and matrices, in-scope previews, float's surface). While that key and bounds hold, cache refreshes in place: it unions damage each member's cache recorded since the version it last drew, padded by one source pixel before member's matrix maps it (bilinear reach at any scale), plus old and new landing of a moved float, clears that region with one pixel of antialiasing margin and redraws every contributor there, then applies group's per-pixel adjustments to the region alone. Unknown member damage, changed member set or new bounds redraw whole surface, reallocating only when its size changed; new key reuses least-recently-used of a group's two slots once both are taken. +Adjustments are position dependent but region independent. `applyAdjustments` takes the pixel of its ImageData's top-left in the stack's reference space; adjusting any sub-rectangle at its own origin yields exactly the pixels a whole-surface pass gives there, which damage refreshes, sampled regions and group-region repaints rely on. Color entries ignore position. Noise adds a field that is a pure function of seed, integer pixel coordinate and channel (a stable integer hash into an equiprobable quantile table, no sequential state), scaled per pixel by a function of that pixel alone: 1 for `flat` response, 4·Y′·(1 − Y′) of the incoming Rec. 709 luma for `midtones`. Size 1 adds the white samples directly. Larger sizes blur them with a separable Gaussian of σ √(size² − 1)/2 truncated at 3σ (R taps each side), renormalized so squared taps sum to one (σ unchanged); a kernel whose outer taps all stay under 10⁻³ (sizes under about 1.14) runs as size 1. Samples beyond the ImageData come from the hash, never from pixels, and each output sums the same float64 values in the same order wherever the region starts, so region independence stays exact. The blur streams rows through a ring of 2R + 1 horizontally filtered rows: transient JS heap of O(R × width) per channel, not counted in the raster budget. Cost is O(pixels × R × channels) on the main thread, paid once per changed preview frame while a slider scrubs. Monochrome shares one field across R, G, B; color draws three; chroma draws two, added to OKLab a and b with lightness kept, then converts back to sRGB clamped in linear light. A layer's own stack works in layer-local pixels (cache surface (0,0) is `entry.rect`'s origin, possibly negative) and is always applied before resampling when it contains noise, so display, generation, exports, bakes and PSD agree in grain pattern and strength, and the grain moves with the layer under transforms. A group stack works in document pixels: group surfaces, composite group buffers and PSD's baked group stacks all sit on the document grid at scale 1. The one exception is a floating selection: its display copy is baked at its lift origin and moves with the drag; committing recomputes the layer's grain at the new position. Thumbnails without a current display copy adjust their own downscaled pixels, shrinking grain to size × scale (at least 1) and scaling σ by min(1, scale × grain span) to approximate the averaging a downscale applies; span is (Σ taps)², 1 at size 1. + +Group surface is keyed by everything except its members' pixels and float's placement (scope shape, member appearance and matrices, in-scope previews, float's surface). While that key and bounds hold, cache refreshes in place: it unions damage each member's cache recorded since the version it last drew, padded by one source pixel before member's matrix maps it (bilinear reach at any scale), plus old and new landing of a moved float, clears that region with one pixel of antialiasing margin and redraws every contributor there, then applies group's adjustments to the region alone, at the region's document origin. Unknown member damage, changed member set or new bounds redraw whole surface, reallocating only when its size changed; new key reuses least-recently-used of a group's two slots once both are taken. Overlay renders in CSS pixels. Receives document→CSS view, CSS viewport size and device-pixel ratio, clears backing store in device pixels, then draws every handle, nub, dash and cursor under a DPR base transform (nested image and path transforms compose with it), so chrome keeps its CSS size at any ratio and matches CSS-pixel hit tests. Viewport owns the ratio (capped at `MAX_DPR`, sub-one ratios kept); backing-store size and composite view matrix both derive from it. diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts index fe73166ecaa..bf3f3d8fffd 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts @@ -228,7 +228,45 @@ export type CanvasAdjustmentEntry = /** Rotation around the color wheel in degrees, -180 to 180. */ rotation: number; }) - | (CanvasAdjustmentEntryBase & { type: 'invert' }); + | (CanvasAdjustmentEntryBase & { type: 'invert' }) + | (CanvasAdjustmentEntryBase & { + /** Film grain; alpha is untouched and fully transparent pixels stay transparent. */ + type: 'noise'; + /** + * Sample shape. Both share the standard deviation `amount` sets, so switching keeps perceived strength. Grain + * larger than one pixel sums many samples, so both tend toward gaussian as `size` grows. + */ + distribution: 'gaussian' | 'uniform'; + /** + * `monochrome`: one sample shared by R, G and B (luminance grain). `color`: an independent sample per channel. + * `chroma`: independent samples on OKLab a and b, keeping OKLab lightness, so only hue and colorfulness vary. + */ + grain: 'monochrome' | 'color' | 'chroma'; + /** + * `flat`: the same strength at every tone. `midtones`: strength scales by 4·Y′·(1 − Y′), with Y′ the Rec. 709 luma + * (0–1) of the pixel entering the noise pass, so grain is full at mid-grey and fades to none at black and white. + */ + response: 'flat' | 'midtones'; + /** + * Grain size in reference-space pixels, finite, 1–8: grains about `size` pixels across. 1 is an independent + * sample per pixel; larger sizes blur that sample field with a Gaussian of standard deviation `√(size² − 1) / 2` + * pixels (squared widths add: one pixel's 1 plus the blur's 4σ²), renormalized so strength stays the standard + * deviation `amount` sets. Sizes under about 1.14, whose blur no 8-bit pixel would show, render as 1. + */ + size: number; + /** + * Strength 0–1, the standard deviation at full response. Monochrome and color: `amount × 128` in 0–255 channel + * units, so 100% is half the channel range (very heavy, mostly clipped), 1–5% subtle grain or dither and 10% + * clearly visible grain. Chroma: `amount × 0.32` on OKLab a and b each, about the total OKLab spread color grain + * of the same amount gives mid-grey, so equal amounts look comparably strong. 0 is identity. + */ + amount: number; + /** + * Pattern seed, uint32. Noise is a pure function of the seed, the integer pixel coordinate in the stack's + * reference space (layer-local pixels for a layer's own stack, document pixels for a group's) and the channel. + */ + seed: number; + }); /** A raster layer's ordered adjustment stack, applied top to bottom. */ export type CanvasAdjustmentsContract = readonly CanvasAdjustmentEntry[]; @@ -384,8 +422,13 @@ export const CANVAS_MAX_NODE_DEPTH = 10; export const CANVAS_MAX_NODE_COUNT = 10_000; export interface CanvasDocumentContractV3 { - /** v3 is read for compatibility; v4 is the current writable schema. */ - version: 3 | 4; + /** + * Schema this document needs: 3 for plain content, 4 for custom-font typography, 5 for noise adjustments. Each + * document mutation raises it to the newer of its content's need and the canvas state's version. A restored + * snapshot's document keeps its own, possibly lower, version until the next mutation; the state's version and the + * project's compatibility floor stay put meanwhile. + */ + version: 3 | 4 | 5; width: number; height: number; background: 'transparent' | { color: string }; @@ -395,9 +438,6 @@ export interface CanvasDocumentContractV3 { selectedLayerId: string | null; } -/** Current writable document contract. v3 remains accepted at the load boundary. */ -export type CanvasDocumentContractV4 = Omit & { version: 4 }; - export interface CanvasSnapshotContract { id: string; name: string; @@ -410,8 +450,11 @@ export interface CanvasStagingAreaContractV2 extends CanvasStagingAreaContract { } export interface CanvasStateContractV3 { - /** v3 is read for compatibility; v4 is the current writable schema. */ - version: 3 | 4; + /** + * Newest schema the state needs: the newest of its previous version, the live document's and every snapshot's + * (3 plain, 4 custom-font typography, 5 noise adjustments). Sticky: removing that content never lowers it. + */ + version: 3 | 4 | 5; document: CanvasDocumentContractV3; /** * Monotonic wholesale-replacement counter. The mirror clears pixel history whenever it changes, including @@ -421,6 +464,3 @@ export interface CanvasStateContractV3 { snapshots: CanvasSnapshotContract[]; stagingArea: CanvasStagingAreaContractV2; } - -/** Current writable state contract. v3 remains accepted at the load boundary. */ -export type CanvasStateContractV4 = Omit & { version: 4 }; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/floatingSelectionController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/floatingSelectionController.ts index 79eb42854df..c22c85c8c3e 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/floatingSelectionController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/floatingSelectionController.ts @@ -164,7 +164,7 @@ export class FloatingSelectionController { return 'unavailable'; } // Bake darkness dropout once so the float matches the rendered control map. - const display = renderLayerDisplayEffect(this.deps.backend, layer, lifted.pixels.surface); + const display = renderLayerDisplayEffect(this.deps.backend, layer, lifted.pixels.surface, lifted.pixels.rect); before = entry.surface.ctx.getImageData( region.x - entry.rect.x, region.y - entry.rect.y, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerController.test.ts index 867c3413789..8911c82edd5 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerController.test.ts @@ -27,6 +27,7 @@ describe('LayerController', () => { getMaskPattern: () => null, isDisposed: () => false, isSupportedSource: () => true, + peekAdjustedSurface: () => null, projectId: 'p1', rasterize: vi.fn(), reportError: vi.fn(), @@ -56,6 +57,7 @@ describe('LayerController', () => { isCacheReady: () => true, layers: {} as never, needsPixelPersistence: () => false, + peekAdjustedSurface: () => null, publishSelectedLayerIds: vi.fn(), }; const booleanMerge = { diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerOperationHarness.testing.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerOperationHarness.testing.ts index 63168d97423..7272c725c14 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerOperationHarness.testing.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerOperationHarness.testing.ts @@ -1,6 +1,7 @@ import type { CanvasDocumentContractV3 } from '@workbench/canvas-engine/contracts'; import type { CanvasProjectMutation } from '@workbench/canvas-engine/mutationContracts'; import type { PreparedLayerCacheReplacement } from '@workbench/canvas-engine/render/layerCache'; +import type { RasterBackend } from '@workbench/canvas-engine/render/raster'; import { createHistory } from '@workbench/canvas-engine/history/history'; import { createLayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; @@ -14,6 +15,8 @@ export interface LayerOperationHarnessOptions { /** History byte budget; an edit whose entry can never fit is refused. */ readonly historyBytes?: number; readonly gestureActive?: boolean; + /** Real canvases for pixel checks in browser tests; the stub backend otherwise. */ + readonly backend?: RasterBackend; } /** @@ -25,7 +28,7 @@ export const createLayerOperationHarness = ( document: CanvasDocumentContractV3, options: LayerOperationHarnessOptions = {} ) => { - const backend = createTestStubRasterBackend(); + const backend = options.backend ?? createTestStubRasterBackend(); const layers = createLayerCacheStore(backend); const history = createHistory({ byteBudget: options.historyBytes }); let project = applyCanvasProjectMutation(createInitialWorkbenchState().projects[0]!, { diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.browser.test.ts new file mode 100644 index 00000000000..b7d6700da06 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.browser.test.ts @@ -0,0 +1,159 @@ +/** Real-canvas checks that Merge Down bakes each layer's own adjustment stack as the canvas displayed it. */ + +import type { + CanvasAdjustmentsContract, + CanvasDocumentContractV3, + CanvasRasterLayerContractV2, +} from '@workbench/canvas-engine/contracts'; +import type { Rect } from '@workbench/canvas-engine/types'; + +import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { getDocumentLayer, getDocumentLeaves } from '@workbench/canvas-engine/document/documentIndex'; +import { fromTRS } from '@workbench/canvas-engine/math/mat2d'; +import { createAdjustedSurfaceCache } from '@workbench/canvas-engine/render/adjustedSurfaceCache'; +import { createDomRasterBackend } from '@workbench/canvas-engine/render/raster'; +import { describe, expect, it } from 'vitest'; + +import { createLayerOperationHarness } from './layerOperationHarness.testing'; +import { MergeLayerController } from './mergeLayerController'; + +const NOISE: CanvasAdjustmentsContract = [ + { + amount: 0.25, + distribution: 'gaussian', + grain: 'monochrome', + id: 'grain', + isEnabled: true, + response: 'flat', + seed: 21, + size: 1, + type: 'noise', + }, +]; +const IDENTITY = { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }; +const UPPER_RECT: Rect = { height: 18, width: 24, x: -3, y: 2 }; +const LOWER_RECT: Rect = { height: 20, width: 20, x: 6, y: -2 }; +/** Document region covering both layers before and after the merge. */ +const VIEW: Rect = { height: 48, width: 48, x: -8, y: -8 }; + +const layer = ( + id: string, + rect: Rect, + adjustments: CanvasAdjustmentsContract | undefined, + transform = IDENTITY +): CanvasRasterLayerContractV2 => ({ + ...(adjustments ? { adjustments } : {}), + blendMode: 'normal', + id, + isEnabled: true, + isLocked: false, + name: id, + opacity: 1, + source: { bitmap: { height: rect.height, imageName: `${id}.png`, width: rect.width }, offset: rect, type: 'paint' }, + transform, + type: 'raster', +}); + +interface Scene { + upperAdjustments?: CanvasAdjustmentsContract; + lowerAdjustments?: CanvasAdjustmentsContract; + upperTransform?: CanvasRasterLayerContractV2['transform']; +} + +const mergeScene = (scene: Scene, displayed: boolean) => { + const backend = createDomRasterBackend(); + const upper = layer('upper', UPPER_RECT, scene.upperAdjustments, scene.upperTransform); + const lower = layer('lower', LOWER_RECT, scene.lowerAdjustments); + const document: CanvasDocumentContractV3 = { + background: 'transparent', + bbox: { height: 64, width: 64, x: 0, y: 0 }, + height: 64, + selectedLayerId: 'upper', + stacks: stacksFrom([upper, lower]), + version: 3, + width: 64, + }; + const h = createLayerOperationHarness(document, { backend }); + const fill = (id: string, rect: Rect, color: (x: number) => string) => { + const entry = h.layers.getOrCreateRect(id, rect); + for (let x = 0; x < rect.width; x += 1) { + entry.surface.ctx.fillStyle = color(x); + entry.surface.ctx.fillRect(x, 0, 1, rect.height); + } + h.layers.publishPixels(id); + }; + fill('upper', UPPER_RECT, (x) => `rgba(${40 + x * 8}, 90, 200, ${x % 3 === 0 ? 0.6 : 1})`); + fill('lower', LOWER_RECT, (x) => `rgb(200, ${50 + x * 9}, 70)`); + const display = createAdjustedSurfaceCache(backend); + const contracts = { + lower: getDocumentLayer(h.document(), 'lower')!, + upper: getDocumentLayer(h.document(), 'upper')!, + }; + /** What the canvas shows over {@link VIEW}: each layer's display copy (adjusted when it has a stack) through its transform. */ + const canvasPixels = (): number[] => { + const surface = backend.createSurface(VIEW.width, VIEW.height); + for (const contract of [...getDocumentLeaves(h.document())].reverse()) { + const entry = h.layers.get(contract.id)!; + const adjustments = contract.type === 'raster' ? contract.adjustments : undefined; + const pixels = display.get(contract.id, entry, adjustments) ?? entry.surface; + const { rotation, scaleX, scaleY, x, y } = contract.transform; + const matrix = fromTRS({ x, y }, rotation, scaleX, scaleY); + surface.ctx.setTransform(matrix.a, matrix.b, matrix.c, matrix.d, matrix.e - VIEW.x, matrix.f - VIEW.y); + surface.ctx.drawImage(pixels.canvas, entry.rect.x, entry.rect.y); + } + return [...surface.ctx.getImageData(0, 0, VIEW.width, VIEW.height).data]; + }; + const rawPixels = (id: string): number[] => { + const entry = h.layers.get(id)!; + return [...entry.surface.ctx.getImageData(0, 0, entry.rect.width, entry.rect.height).data]; + }; + const controller = new MergeLayerController({ + backend, + ctx: h.ctx, + exportBaked: () => Promise.reject(new Error('merge down reads live caches')), + hasExportableContent: (layerId) => (h.layers.get(layerId)?.rect.width ?? 0) > 0, + isCacheReady: () => true, + layers: h.layers, + needsPixelPersistence: () => false, + peekAdjustedSurface: (contract, entry) => + displayed && contract.type === 'raster' ? display.get(contract.id, entry, contract.adjustments) : null, + publishSelectedLayerIds: () => undefined, + }); + return { ...h, canvasPixels, contracts, controller, rawPixels }; +}; + +describe('Merge Down with adjustment stacks', () => { + describe.each([ + ['noise on the upper layer', { upperAdjustments: NOISE }], + ['noise on the layer below', { lowerAdjustments: NOISE }], + [ + 'noise on a rotated, downscaled upper layer above a color-adjusted one', + { + lowerAdjustments: [{ id: 'invert', isEnabled: true, type: 'invert' }], + upperAdjustments: NOISE, + upperTransform: { rotation: Math.PI / 7, scaleX: 0.7, scaleY: 0.55, x: 9, y: 4 }, + }, + ], + ] satisfies [string, Scene][])('%s', (_, scene) => { + it.each([ + ['adjusting layer-local copies', false], + ["drawing the display's current adjusted copies", true], + ])('keeps the displayed pixels by %s; undo restores both raw layers and stacks', async (_label, displayed) => { + const h = mergeScene(scene, displayed); + const raw = { lower: h.rawPixels('lower'), upper: h.rawPixels('upper') }; + const shown = h.canvasPixels(); + + expect(h.controller.mergeDown('upper')).toBe('merged'); + + // The merged layer has no stack: its baked pixels show what both layers showed. + expect(getDocumentLayer(h.document(), 'lower')).not.toHaveProperty('adjustments'); + expect(h.canvasPixels()).toEqual(shown); + + await expect(h.history.undo()).resolves.toEqual({ status: 'applied' }); + expect(getDocumentLayer(h.document(), 'upper')).toBe(h.contracts.upper); + expect(getDocumentLayer(h.document(), 'lower')).toBe(h.contracts.lower); + expect(h.rawPixels('upper')).toEqual(raw.upper); + expect(h.rawPixels('lower')).toEqual(raw.lower); + }); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.test.ts index f4d6e47db4b..96e4f874f83 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.test.ts @@ -1,8 +1,10 @@ import type { CanvasDocumentContractV3, CanvasLayerContract } from '@workbench/canvas-engine/contracts'; +import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Rect } from '@workbench/canvas-engine/types'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { getDocumentLayer, getDocumentLeaves } from '@workbench/canvas-engine/document/documentIndex'; +import { createTestStubRasterBackend } from '@workbench/canvas-engine/render/raster.testStub'; import { describe, expect, it } from 'vitest'; import { createLayerOperationHarness } from './layerOperationHarness.testing'; @@ -24,9 +26,15 @@ const UPPER_RECT: Rect = { height: 10, width: 10, x: 0, y: 0 }; const LOWER_RECT: Rect = { height: 10, width: 10, x: 5, y: 5 }; const MERGED_RECT: Rect = { height: 15, width: 15, x: 0, y: 0 }; -const createHarness = (historyBytes?: number) => { +const createHarness = ( + historyBytes?: number, + { lowerAdjusted = false, displayed = null }: { lowerAdjusted?: boolean; displayed?: RasterSurface | null } = {} +) => { const upper = paintLayer('upper'); - const lower = paintLayer('lower'); + const lower: CanvasLayerContract = { + ...paintLayer('lower'), + ...(lowerAdjusted ? { adjustments: [{ id: 'invert', isEnabled: true, type: 'invert' }] } : {}), + }; const document: CanvasDocumentContractV3 = { background: 'transparent', bbox: { height: 100, width: 100, x: 0, y: 0 }, @@ -47,6 +55,7 @@ const createHarness = (historyBytes?: number) => { isCacheReady: () => true, layers: h.layers, needsPixelPersistence: () => true, + peekAdjustedSurface: () => displayed, publishSelectedLayerIds: () => undefined, }); // The reducer's own contracts, which merge and undo must restore by identity. @@ -123,6 +132,23 @@ describe('MergeLayerController: merge down', () => { expect(h.history.canUndo()).toBe(false); }); + it('reserves a layer-local adjusted copy only when the display has no current one', () => { + // Raw undo snapshots plus the merged pixels and their prepared replacement; a copy adds a surface and ImageData. + const baseBytes = (100 + 100) * 4 + 225 * 4 * 2; + const copying = createHarness(undefined, { lowerAdjusted: true }); + copying.state.rasterBytes = baseBytes; + expect(copying.controller.mergeDown('upper')).toBe('over-budget'); + copying.state.rasterBytes = baseBytes + 100 * 4 * 2; + expect(copying.controller.mergeDown('upper')).toBe('merged'); + + const displayed = createHarness(undefined, { + displayed: createTestStubRasterBackend().createSurface(10, 10), + lowerAdjusted: true, + }); + displayed.state.rasterBytes = baseBytes; + expect(displayed.controller.mergeDown('upper')).toBe('merged'); + }); + it('refuses the bottom-most layer and a busy canvas', () => { const h = createHarness(); expect(h.controller.mergeDown('lower')).toBe('unsupported'); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.ts index ece8ae5060b..26b7f95f5d7 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/mergeLayerController.ts @@ -3,7 +3,7 @@ import type { CanvasDocumentContractV3, CanvasLayerContract } from '@workbench/c import type { CanvasCommandRefusal } from '@workbench/canvas-engine/document/commandRefusal'; import type { SubsetOf } from '@workbench/canvas-engine/editConcurrency'; import type { CanvasProjectMutation } from '@workbench/canvas-engine/mutationContracts'; -import type { LayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; +import type { LayerCacheEntry, LayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; import type { RasterBackend, RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Rect } from '@workbench/canvas-engine/types'; @@ -63,6 +63,8 @@ export interface MergeLayerControllerOptions { readonly hasExportableContent: (layerId: string) => boolean; readonly exportBaked: (layerId: string) => Promise; readonly needsPixelPersistence: (layer: CanvasLayerContract) => boolean; + /** The display's current adjusted copy of a raster layer's live cache, or null; never builds one. */ + readonly peekAdjustedSurface: (layer: CanvasLayerContract, entry: LayerCacheEntry) => RasterSurface | null; readonly publishSelectedLayerIds: (primaryId: string | null, selectedIds: readonly string[]) => void; } @@ -79,7 +81,9 @@ export class MergeLayerController { /** * Bakes the upper layer into the one below as one undo step: the lower layer keeps its id and takes the merged - * pixels. Undo restores the lower contract and pixels, then reinserts the upper layer with its own. + * pixels, with each layer's own adjustment stack baked in layer-local pixels before the upper one is resampled, so + * the merged layer (whose contract has no stack) shows what the canvas showed. Undo restores the lower contract and + * raw pixels, then reinserts the upper layer with its own. */ mergeDown(upperLayerId: string): MergeDownResult { const { ctx } = this.deps; @@ -125,6 +129,17 @@ export class MergeLayerController { : transformBounds(matrix, upperRect) ); const snapshotBytes = rgbaBytes(upperRect) + rgbaBytes(belowRect); + // Each stack applies to its layer's raw pixels before compositing: the display's current adjusted copy when it + // has one, else a layer-local copy (surface plus ImageData) adjusted here. + const ownStack = (layer: CanvasLayerContract, cache: LayerCacheEntry | undefined) => + cache && layer.type === 'raster' && layer.adjustments && !isIdentityAdjustments(layer.adjustments) + ? { adjustments: layer.adjustments, displayed: this.deps.peekAdjustedSurface(layer, cache) } + : null; + const upperStack = upperHasContent ? ownStack(upper, upperCache) : null; + const belowStack = belowHasContent ? ownStack(below, belowCache) : null; + const adjustedCopyBytes = + (upperStack && !upperStack.displayed ? rgbaBytes(upperRect) * 2 : 0) + + (belowStack && !belowStack.displayed ? rgbaBytes(belowRect) * 2 : 0); const txn = ctx.begin({ historyBytes: snapshotBytes + rgbaBytes(mergedRect) + HISTORY_ENTRY_OVERHEAD_BYTES, }); @@ -132,8 +147,8 @@ export class MergeLayerController { return layerEditRefusal(txn.status); } try { - // Undo snapshots, the merged pixels and their prepared cache replacement. - if (!txn.reserveRaster(snapshotBytes + rgbaBytes(mergedRect) * 2)) { + // Undo snapshots, adjusted copies, the merged pixels and their prepared cache replacement. + if (!txn.reserveRaster(snapshotBytes + adjustedCopyBytes + rgbaBytes(mergedRect) * 2)) { return 'over-budget'; } const snapshot = (rect: Rect, layerId: string): LayerPixels => { @@ -145,18 +160,35 @@ export class MergeLayerController { }; const upperPixels = snapshot(upperRect, upper.id); const belowPixels = snapshot(belowRect, below.id); + const adjusted = (pixels: LayerPixels, stack: ReturnType): RasterSurface => { + if (!stack) { + return pixels.pixels; + } + if (stack.displayed) { + return stack.displayed; + } + const { height, width } = pixels.rect; + const imageData = pixels.pixels.ctx.getImageData(0, 0, width, height); + // A layer's stack works in layer-local pixels; surface (0,0) sits at its cache rect's origin. + applyAdjustments(imageData, stack.adjustments, pixels.rect); + const copy = this.deps.backend.createSurface(width, height); + copy.ctx.putImageData(imageData, 0, 0); + return copy; + }; + const upperDrawn = adjusted(upperPixels, upperStack); + const belowDrawn = adjusted(belowPixels, belowStack); const merged = this.deps.backend.createSurface(mergedRect.width, mergedRect.height); const context = merged.ctx; context.setTransform(1, 0, 0, 1, 0, 0); context.clearRect(0, 0, mergedRect.width, mergedRect.height); if (belowHasContent) { - context.drawImage(belowPixels.pixels.canvas, belowRect.x - mergedRect.x, belowRect.y - mergedRect.y); + context.drawImage(belowDrawn.canvas, belowRect.x - mergedRect.x, belowRect.y - mergedRect.y); } if (upperHasContent) { context.setTransform(matrix.a, matrix.b, matrix.c, matrix.d, matrix.e - mergedRect.x, matrix.f - mergedRect.y); context.globalAlpha = upper.opacity; context.globalCompositeOperation = blendToComposite(upper.blendMode); - context.drawImage(upperPixels.pixels.canvas, upperRect.x, upperRect.y); + context.drawImage(upperDrawn.canvas, upperRect.x, upperRect.y); context.setTransform(1, 0, 0, 1, 0, 0); context.globalAlpha = 1; context.globalCompositeOperation = 'source-over'; @@ -343,7 +375,8 @@ export class MergeLayerController { drawMerged(buffer.ctx, scope.start, scope.end, scope.children); if (!isIdentityAdjustments(scope.adjustments)) { const scoped = buffer.ctx.getImageData(0, 0, rect.width, rect.height); - applyAdjustments(scoped, scope.adjustments); + // Group stacks work in document pixels; the buffer is `rect` at scale 1. + applyAdjustments(scoped, scope.adjustments, rect); buffer.ctx.putImageData(scoped, 0, 0); } target.globalAlpha = scope.opacity; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/psdExportController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/psdExportController.ts index 156caaac1bb..c0b08fe0b00 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/psdExportController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/psdExportController.ts @@ -53,9 +53,15 @@ const countIsolatedFolders = (nodes: readonly PsdPlanNode[]): number => 0 ); +// Layers bake one at a time; an adjusted layer's transient layer-local copy is counted once, at the largest. const getRequiredAllocationPixelArea = (plan: Extract): number => plan.width * plan.height * (1 + countIsolatedFolders(plan.tree)) + - plan.layers.reduce((total, layer) => total + layer.worldRect.width * layer.worldRect.height, 0); + plan.layers.reduce((total, layer) => total + layer.worldRect.width * layer.worldRect.height, 0) + + plan.layers.reduce( + (largest, layer) => + layer.adjustments ? Math.max(largest, layer.contentRect.width * layer.contentRect.height) : largest, + 0 + ); /** Owns immutable PSD snapshot capture, budget reservation, execution, and cancellation. */ export class PsdExportController { @@ -139,14 +145,13 @@ export class PsdExportController { } return []; } - const combined = [ - ...(node.type === 'raster' ? (node.adjustments ?? []) : []), - ...[...ancestorStacks].reverse().flat(), - ]; + const own = node.type === 'raster' ? node.adjustments : undefined; + const enclosing = [...ancestorStacks].reverse().flat(); return [ { // Identity-aware: an emptied stack must not trigger the executor's bake writeback. - adjustments: isIdentityAdjustments(combined) ? undefined : combined, + adjustments: isIdentityAdjustments(own) ? undefined : own, + groupAdjustments: isIdentityAdjustments(enclosing) ? undefined : enclosing, blendMode: node.blendMode, colorLabel: node.colorLabel, contentRect: detached.rect, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.test.ts index 8fa441c3e2a..0326436a9f2 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.test.ts @@ -60,14 +60,28 @@ describe('RasterController', () => { const entry = publish(controller, 'layer'); const { adjustments } = adjustedLayer('layer'); - const shared = controller.getAdjustedCacheSurface('layer', entry.surface, adjustments!); - expect(shared).not.toBeNull(); - expect(shared).toBe(controller.getAdjustedSurface(adjustedLayer('layer'), entry)); + // Export only reuses the display's copy; it never builds one outside its reservation. + expect(controller.peekAdjustedCacheSurface('layer', entry.surface, adjustments!)).toBeNull(); + expect(controller.memory.snapshot().derivedBytes).toBe(0); + const displayed = controller.getAdjustedSurface(adjustedLayer('layer'), entry); + expect(displayed).not.toBeNull(); + expect(controller.peekAdjustedCacheSurface('layer', entry.surface, adjustments!)).toBe(displayed); expect( - controller.getAdjustedCacheSurface('layer', createTestStubRasterBackend().createSurface(1, 1), adjustments!) + controller.peekAdjustedCacheSurface('layer', createTestStubRasterBackend().createSurface(1, 1), adjustments!) + ).toBeNull(); + // A copy made for other adjustment parameters is not this stack's. + expect( + controller.peekAdjustedCacheSurface('layer', entry.surface, [ + { brightness: 0.25, contrast: 0, id: 'adj-bc', isEnabled: true, type: 'brightness-contrast' }, + ]) ).toBeNull(); baked = true; - expect(controller.getAdjustedCacheSurface('layer', entry.surface, adjustments!)).toBeNull(); + expect(controller.peekAdjustedCacheSurface('layer', entry.surface, adjustments!)).toBeNull(); + baked = false; + expect(controller.peekAdjustedCacheSurface('layer', entry.surface, adjustments!)).toBe(displayed); + // New source pixels make the copy stale; export adjusts its own sampled region instead of drawing it. + controller.layers.publishPixels('layer'); + expect(controller.peekAdjustedCacheSurface('layer', entry.surface, adjustments!)).toBeNull(); }); it('accounts base, derived and group surfaces as they are allocated and released', () => { diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.ts index 530e3887dff..8738e08b7af 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterController.ts @@ -230,15 +230,22 @@ export class RasterController { : null; } - /** The adjusted copy of a layer's live cache (built and memoized on a miss), when `surface` is that cache and unbaked. */ - getAdjustedCacheSurface( + /** The display's current adjusted copy of a raster layer's cache, without building one on a miss. */ + peekAdjustedSurface(layer: CanvasLayerContract, entry: LayerCacheEntry): RasterSurface | null { + return layer.type === 'raster' && !this.isAdjustmentBaked(layer.id) + ? this.adjustments.peek(layer.id, entry, layer.adjustments) + : null; + } + + /** The display's current adjusted copy of a layer's live cache, when `surface` is that cache; never builds one. */ + peekAdjustedCacheSurface( layerId: string, surface: RasterSurface, adjustments: CanvasAdjustmentsContract ): RasterSurface | null { const entry = this.layers.peek(layerId); return entry?.surface === surface && !this.isAdjustmentBaked(layerId) - ? this.adjustments.get(layerId, entry, adjustments) + ? this.adjustments.peek(layerId, entry, adjustments) : null; } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.browser.test.ts new file mode 100644 index 00000000000..b8ef92470c1 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.browser.test.ts @@ -0,0 +1,120 @@ +/** Real-canvas checks that world-space bakes resample the display's layer-local adjusted pixels. */ + +import type { CanvasDocumentContractV3, CanvasRasterLayerContractV2 } from '@workbench/canvas-engine/contracts'; +import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; + +import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { fromTRS } from '@workbench/canvas-engine/math/mat2d'; +import { createAdjustedSurfaceCache } from '@workbench/canvas-engine/render/adjustedSurfaceCache'; +import { createLayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; +import { createDomRasterBackend } from '@workbench/canvas-engine/render/raster'; +import { describe, expect, it, vi } from 'vitest'; + +import { RasterExportController } from './rasterExportController'; + +const TRANSFORM = { rotation: Math.PI / 9, scaleX: 0.6, scaleY: 0.6, x: 11, y: 5 }; + +/** A rotated, downscaled noisy layer with off-origin content: its stack works in layer-local pixels. */ +const bakeScene = (peekDisplayed: boolean) => { + const backend = createDomRasterBackend(); + const layers = createLayerCacheStore(backend); + const entry = layers.growToRect('layer', { height: 30, width: 40, x: -6, y: -4 }); + for (let x = 0; x < 40; x += 1) { + entry.surface.ctx.fillStyle = `rgb(${60 + x * 3}, 128, ${200 - x * 3})`; + entry.surface.ctx.fillRect(x, 0, 1, 30); + } + layers.publishPixels('layer'); + const layer: CanvasRasterLayerContractV2 = { + adjustments: [ + { + amount: 0.2, + distribution: 'gaussian', + id: 'n', + isEnabled: true, + grain: 'color', + response: 'flat', + size: 1, + seed: 9, + type: 'noise', + }, + { id: 'i', isEnabled: true, type: 'invert' }, + ], + blendMode: 'normal', + id: 'layer', + isEnabled: true, + isLocked: false, + name: 'Layer', + opacity: 1, + source: { bitmap: { height: 30, imageName: 'layer.png', width: 40 }, offset: { x: -6, y: -4 }, type: 'paint' }, + transform: TRANSFORM, + type: 'raster', + }; + const document: CanvasDocumentContractV3 = { + background: 'transparent', + bbox: { height: 64, width: 64, x: 0, y: 0 }, + height: 64, + stacks: stacksFrom([layer]), + selectedLayerId: null, + version: 3, + width: 64, + }; + const displayed = createAdjustedSurfaceCache(backend); + const reserve = vi.fn((_bytes: number) => ({ lease: { release: () => undefined }, status: 'ok' as const })); + const controller = new RasterExportController({ + backend, + captureGuard: (captured) => ({ + cacheVersion: 1, + documentGeneration: 1, + layer: captured, + layerId: captured.id, + projectId: 'p', + }), + getDocument: () => document, + getOrStartRasterization: () => Promise.resolve('published'), + isGuardCurrent: () => true, + isRasterizing: () => false, + isSupportedSource: () => true, + layers, + ...(peekDisplayed + ? { + peekAdjustedCacheSurface: (layerId: string, surface: RasterSurface) => + surface === layers.get(layerId)!.surface + ? displayed.get(layerId, layers.get(layerId)!, layer.adjustments) + : null, + } + : {}), + pin: () => ({ release: () => undefined }), + reserve, + }); + /** The display's adjusted surface drawn through the layer transform: what the canvas shows. */ + const reference = (rect: { x: number; y: number; width: number; height: number }) => { + const surface = backend.createSurface(rect.width, rect.height); + const matrix = fromTRS({ x: TRANSFORM.x, y: TRANSFORM.y }, TRANSFORM.rotation, TRANSFORM.scaleX, TRANSFORM.scaleY); + surface.ctx.setTransform(matrix.a, matrix.b, matrix.c, matrix.d, matrix.e - rect.x, matrix.f - rect.y); + surface.ctx.drawImage(displayed.get('layer', layers.get('layer')!, layer.adjustments)!.canvas, -6, -4); + return [...surface.ctx.getImageData(0, 0, rect.width, rect.height).data]; + }; + return { controller, reference, reserve }; +}; + +describe('baked raster export', () => { + it.each([ + ['adjusting a reserved layer-local copy', false], + ["drawing the display's current adjusted copy", true], + ])("keeps a rotated, downscaled noisy layer's displayed grain by %s", async (_, peekDisplayed) => { + const { controller, reference, reserve } = bakeScene(peekDisplayed); + + const baked = await controller.baked('layer'); + + expect(baked.status).toBe('ok'); + if (baked.status !== 'ok') { + return; + } + const { height, width } = baked.rect; + expect([...baked.surface.ctx.getImageData(0, 0, width, height).data]).toEqual(reference(baked.rect)); + // The world surface always; a layer-local copy (surface plus ImageData) only without a current display copy. + const world = width * height * 4; + expect(reserve.mock.calls).toEqual(peekDisplayed ? [[world]] : [[40 * 30 * 8], [world]]); + baked.release(); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.test.ts index 89f430b56da..68ce664ac31 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.test.ts @@ -92,8 +92,13 @@ describe('RasterExportController budget', () => { version: 3, width: 100, }; - const release = vi.fn(); - const reserve = vi.fn(() => ({ lease: { release }, status: 'ok' as const })); + // One lease per reservation: the layer-local adjusted copy's, then the baked surface's. + const leases: ReturnType[] = []; + const reserve = vi.fn((_bytes: number) => { + const release = vi.fn(); + leases.push(release); + return { lease: { release }, status: 'ok' as const }; + }); const controller = new RasterExportController({ backend, captureGuard: () => ({ cacheVersion: 1, documentGeneration: 1, layer, layerId: 'layer', projectId: 'p' }), @@ -109,11 +114,13 @@ describe('RasterExportController budget', () => { const pending = controller.blob('layer'); await vi.waitFor(() => expect(encodeSurface).toHaveBeenCalledOnce()); - expect(release).not.toHaveBeenCalled(); + const [adjustedCopy, baked] = leases; + expect(adjustedCopy).toHaveBeenCalledOnce(); + expect(baked).not.toHaveBeenCalled(); resolveEncode(new Blob(['png'])); await expect(pending).resolves.toMatchObject({ status: 'ok' }); - expect(release).toHaveBeenCalledOnce(); + expect(baked).toHaveBeenCalledOnce(); }); it('transfers the baked-surface reservation to the caller until idempotent release', async () => { @@ -145,8 +152,13 @@ describe('RasterExportController budget', () => { version: 3, width: 100, }; - const release = vi.fn(); - const reserve = vi.fn(() => ({ lease: { release }, status: 'ok' as const })); + // One lease per reservation: the layer-local adjusted copy's, then the baked surface's. + const leases: ReturnType[] = []; + const reserve = vi.fn((_bytes: number) => { + const release = vi.fn(); + leases.push(release); + return { lease: { release }, status: 'ok' as const }; + }); const controller = new RasterExportController({ backend, captureGuard: () => ({ cacheVersion: 1, documentGeneration: 1, layer, layerId: 'layer', projectId: 'p' }), @@ -162,13 +174,16 @@ describe('RasterExportController budget', () => { const result = await controller.baked('layer'); expect(result.status).toBe('ok'); - expect(reserve).toHaveBeenCalledWith(80_000); - expect(release).not.toHaveBeenCalled(); + // The stack adjusts a layer-local copy (surface + ImageData) before the transform draws the baked surface. + expect(reserve.mock.calls).toEqual([[80_000], [40_000]]); + const [adjustedCopy, baked] = leases; + expect(adjustedCopy).toHaveBeenCalledOnce(); + expect(baked).not.toHaveBeenCalled(); if (result.status === 'ok') { result.release(); result.release(); } - expect(release).toHaveBeenCalledOnce(); + expect(baked).toHaveBeenCalledOnce(); }); it('waits for custom font readiness before returning cached text pixels', async () => { diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.ts index 4b498b639fd..02bc1ed9b8f 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterExportController.ts @@ -5,6 +5,7 @@ import type { LayerExportGuard, } from '@workbench/canvas-engine/capabilities'; import type { + CanvasAdjustmentsContract, CanvasDocumentContractV3, CanvasLayerContract, CanvasLayerSourceContract, @@ -12,7 +13,7 @@ import type { import type { RasterReadLease } from '@workbench/canvas-engine/rasterTransactions'; import type { CanvasTextSource } from '@workbench/canvas-engine/render/fontLoader'; import type { LayerCacheEntry, LayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; -import type { RasterBackend } from '@workbench/canvas-engine/render/raster'; +import type { RasterBackend, RasterSurface } from '@workbench/canvas-engine/render/raster'; import { lookupDocumentLayer, lookupDocumentLeaf } from '@workbench/canvas-engine/document-model/documentModel'; import { getSourceContentRect, renderableSourceOf } from '@workbench/canvas-engine/document/sources'; @@ -42,6 +43,15 @@ export interface RasterExportControllerOptions { ) => | { status: 'ok'; lease: { release(): void } } | { status: 'over-budget'; requestedBytes: number; availableBytes: number }; + /** + * The display's current adjusted copy of `surface` when it is the layer's live cache, or null; never builds one. + * Bakes draw it instead of reserving and adjusting a layer-sized copy. + */ + readonly peekAdjustedCacheSurface?: ( + layerId: string, + surface: RasterSurface, + adjustments: CanvasAdjustmentsContract + ) => RasterSurface | null; /** Keeps a layer resident and untrimmed for the lifetime of a read lease. */ readonly pin: (layerId: string) => { release(): void }; /** Resolves custom font bytes before an operation whose pixels leave the editor. */ @@ -95,7 +105,7 @@ export class RasterExportController { ctx.clearRect(0, 0, result.rect.width, result.rect.height); ctx.drawImage(result.surface.canvas, 0, 0); const imageData = ctx.getImageData(0, 0, result.rect.width, result.rect.height); - applyAdjustments(imageData, layer.adjustments); + applyAdjustments(imageData, layer.adjustments, result.rect); ctx.putImageData(imageData, 0, 0); return { ...result, release, surface }; } catch (error) { @@ -208,10 +218,11 @@ export class RasterExportController { layerId: string, options: ExportBakedLayerPixelsOptions = {} ): Promise { - const raw = await this.rasterize(layerId, { ...options, applyAdjustments: false }); - if (raw.status !== 'ok') { - return noReservedPixels(raw); + const leased = await this.rasterize(layerId, { ...options, applyAdjustments: false }); + if (leased.status !== 'ok') { + return noReservedPixels(leased); } + let raw = leased; const layer = raw.guard.layer; const matrix = fromTRS( { x: layer.transform.x, y: layer.transform.y }, @@ -224,9 +235,18 @@ export class RasterExportController { raw.release(); return noReservedPixels({ status: 'empty' }); } - const appliesAdjustments = - options.applyAdjustments !== false && layer.type === 'raster' && !isIdentityAdjustments(layer.adjustments); - const reservation = this.options.reserve?.(rect.width * rect.height * (appliesAdjustments ? 8 : 4)); + // The layer's own stack applies to layer-local pixels before the transform resamples them, as the display does, + // so bakes keep the displayed grain and color for scaled and rotated layers too. The display's current adjusted + // copy of the leased live cache is drawn as is; otherwise a layer-local copy is reserved and adjusted. + if (options.applyAdjustments !== false && layer.type === 'raster' && !isIdentityAdjustments(layer.adjustments)) { + const displayed = this.options.peekAdjustedCacheSurface?.(layer.id, raw.surface, layer.adjustments!) ?? null; + const adjusted = displayed ? { ...raw, surface: displayed } : this.applyAdjustments(raw, true); + if (adjusted.status !== 'ok') { + return noReservedPixels(adjusted); + } + raw = adjusted; + } + const reservation = this.options.reserve?.(rect.width * rect.height * 4); if (reservation?.status === 'over-budget') { raw.release(); return noReservedPixels({ status: 'over-budget' }); @@ -248,11 +268,6 @@ export class RasterExportController { ctx.clearRect(0, 0, rect.width, rect.height); ctx.setTransform(matrix.a, matrix.b, matrix.c, matrix.d, matrix.e - rect.x, matrix.f - rect.y); ctx.drawImage(raw.surface.canvas, raw.rect.x, raw.rect.y); - if (appliesAdjustments && layer.type === 'raster' && layer.adjustments) { - const imageData = ctx.getImageData(0, 0, rect.width, rect.height); - applyAdjustments(imageData, layer.adjustments); - ctx.putImageData(imageData, 0, 0); - } return { release, result: { guard: raw.guard, rect, release, status: 'ok', surface } }; } catch (error) { release(); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.browser.test.ts new file mode 100644 index 00000000000..890057f7bab --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.browser.test.ts @@ -0,0 +1,114 @@ +/** Real-canvas checks that rasterizing keeps a noisy layer's displayed grain. */ + +import type { CanvasDocumentContractV3, CanvasRasterLayerContractV2 } from '@workbench/canvas-engine/contracts'; +import type { Rect } from '@workbench/canvas-engine/types'; + +import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { getDocumentLayer } from '@workbench/canvas-engine/document/documentIndex'; +import { getSourceContentRect } from '@workbench/canvas-engine/document/sources'; +import { fromTRS } from '@workbench/canvas-engine/math/mat2d'; +import { createAdjustedSurfaceCache } from '@workbench/canvas-engine/render/adjustedSurfaceCache'; +import { createLayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; +import { createDomRasterBackend } from '@workbench/canvas-engine/render/raster'; +import { rasterizeSource, type RasterizeDeps } from '@workbench/canvas-engine/render/rasterizers'; +import { describe, expect, it } from 'vitest'; + +import { createLayerOperationHarness } from './layerOperationHarness.testing'; +import { RasterizeLayerController } from './rasterizeLayerController'; + +const NOISE = { + amount: 0.3, + distribution: 'gaussian', + grain: 'color', + id: 'grain', + isEnabled: true, + response: 'flat', + seed: 13, + size: 1, + type: 'noise', +} as const; +const TRANSFORM = { rotation: Math.PI / 9, scaleX: 0.6, scaleY: 1.3, x: 11, y: 5 }; + +const noisyLayer = (source: CanvasRasterLayerContractV2['source']): CanvasRasterLayerContractV2 => ({ + adjustments: [NOISE, { brightness: 0.1, contrast: 0.2, id: 'bc', isEnabled: true, type: 'brightness-contrast' }], + blendMode: 'normal', + id: 'layer', + isEnabled: true, + isLocked: false, + name: 'Layer', + opacity: 1, + source, + transform: TRANSFORM, + type: 'raster', +}); + +describe('RasterizeLayerController with a position-dependent stack', () => { + it.each([ + [ + 'shape', + noisyLayer({ fill: '#808080', height: 30, kind: 'rect', stroke: null, strokeWidth: 0, type: 'shape', width: 40 }), + ], + // Text resizes its scratch to measured bounds; this text fits its estimate, so nothing is clipped. + [ + 'text', + noisyLayer({ + align: 'left', + color: '#808080', + content: 'Iii\nill', + fontFamily: 'sans-serif', + fontSize: 28, + fontWeight: 700, + lineHeight: 1.2, + type: 'text', + }), + ], + ])('bakes the displayed grain of a rotated, scaled %s layer and drops the stack until undo', async (_, layer) => { + const backend = createDomRasterBackend(); + const document: CanvasDocumentContractV3 = { + background: 'transparent', + bbox: { height: 64, width: 64, x: 0, y: 0 }, + height: 64, + selectedLayerId: 'layer', + stacks: stacksFrom([layer]), + version: 3, + width: 64, + }; + const h = createLayerOperationHarness(document, { backend }); + const rasterizeDeps = (): RasterizeDeps => ({ + backend, + documentSize: { height: 64, width: 64 }, + resolver: () => Promise.reject(new Error('parametric sources need no images')), + store: h.layers, + }); + const controller = new RasterizeLayerController({ backend, ctx: h.ctx, rasterizeDeps }); + + // The display: the layer's rasterized cache, adjusted by the display's cache, resampled through the transform. + const rendered = await rasterizeSource(layer.source, rasterizeDeps()); + const displayCache = createLayerCacheStore(backend); + const entry = displayCache.publishRasterized('layer', rendered.rect, rendered.surface); + if (layer.source.type === 'text') { + expect(entry.rect.width).toBeLessThan(getSourceContentRect(layer, document).width); + } + const adjusted = createAdjustedSurfaceCache(backend).get('layer', entry, layer.adjustments)!; + const displayed = (rect: Rect): number[] => { + const surface = backend.createSurface(rect.width, rect.height); + const { rotation, scaleX, scaleY, x, y } = TRANSFORM; + const matrix = fromTRS({ x, y }, rotation, scaleX, scaleY); + surface.ctx.setTransform(matrix.a, matrix.b, matrix.c, matrix.d, matrix.e - rect.x, matrix.f - rect.y); + surface.ctx.drawImage(adjusted.canvas, entry.rect.x, entry.rect.y); + return [...surface.ctx.getImageData(0, 0, rect.width, rect.height).data]; + }; + + expect(controller.rasterize('layer')).toBe('rasterized'); + + const paint = getDocumentLayer(h.document(), 'layer'); + expect(paint).toMatchObject({ source: { type: 'paint' }, transform: { rotation: 0, scaleX: 1, scaleY: 1 } }); + expect(paint).not.toHaveProperty('adjustments'); + const baked = h.layers.get('layer')!; + const bakedPixels = [...baked.surface.ctx.getImageData(0, 0, baked.rect.width, baked.rect.height).data]; + expect(bakedPixels).toEqual(displayed(baked.rect)); + + await expect(h.history.undo()).resolves.toEqual({ status: 'applied' }); + expect(getDocumentLayer(h.document(), 'layer')).toEqual(layer); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.test.ts index 9886b9c732d..00ea624a211 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.test.ts @@ -19,13 +19,13 @@ const shapeLayer: CanvasLayerContract = { type: 'raster', }; -const createHarness = (historyBytes?: number) => { +const createHarness = (historyBytes?: number, layer: CanvasLayerContract = shapeLayer) => { const document: CanvasDocumentContractV3 = { background: 'transparent', bbox: { height: 100, width: 100, x: 0, y: 0 }, height: 100, selectedLayerId: 'shape', - stacks: stacksFrom([shapeLayer]), + stacks: stacksFrom([layer]), version: 3, width: 100, }; @@ -90,6 +90,37 @@ describe('RasterizeLayerController', () => { expect(h.history.canUndo()).toBe(false); }); + it('keeps a color-only stack on the paint contract', () => { + const adjustments = [{ id: 'invert', isEnabled: true, type: 'invert' }] as const; + const h = createHarness(undefined, { ...shapeLayer, adjustments }); + + expect(h.controller.rasterize('shape')).toBe('rasterized'); + expect(getDocumentLayer(h.document(), 'shape')).toMatchObject({ adjustments, source: { type: 'paint' } }); + }); + + it('reserves the readback that bakes a noisy stack', () => { + const noise = { + amount: 0.2, + distribution: 'uniform', + grain: 'monochrome', + id: 'grain', + isEnabled: true, + response: 'flat', + seed: 1, + size: 1, + type: 'noise', + } as const; + const h = createHarness(undefined, { ...shapeLayer, adjustments: [noise] }); + // The 60×40 source render, plus the 120×40 baked pixels and their prepared replacement. + h.state.rasterBytes = (60 * 40 + 120 * 40 * 2) * 4; + + expect(h.controller.rasterize('shape')).toBe('over-budget'); + expect(h.dispatched).toEqual([]); + h.state.rasterBytes += 60 * 40 * 4; + expect(h.controller.rasterize('shape')).toBe('rasterized'); + expect(getDocumentLayer(h.document(), 'shape')).not.toHaveProperty('adjustments'); + }); + it('refuses missing, locked and already-painted layers', () => { const h = createHarness(); expect(h.controller.rasterize('nope')).toBe('missing'); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.ts index cceffe29786..8a2cfd292c4 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/rasterizeLayerController.ts @@ -1,4 +1,8 @@ -import type { CanvasDocumentContractV3, CanvasLayerContract } from '@workbench/canvas-engine/contracts'; +import type { + CanvasDocumentContractV3, + CanvasLayerContract, + CanvasRasterLayerContractV2, +} from '@workbench/canvas-engine/contracts'; import type { CanvasCommandRefusal } from '@workbench/canvas-engine/document/commandRefusal'; import type { SubsetOf } from '@workbench/canvas-engine/editConcurrency'; import type { RasterBackend, RasterSurface } from '@workbench/canvas-engine/render/raster'; @@ -10,6 +14,7 @@ import { getDocumentLayer } from '@workbench/canvas-engine/document/documentInde import { getSourceContentRect, isEmptyPolygonShape } from '@workbench/canvas-engine/document/sources'; import { collectHistoryMediaRefs, HISTORY_ENTRY_OVERHEAD_BYTES } from '@workbench/canvas-engine/history/history'; import { roundOut, transformBounds } from '@workbench/canvas-engine/math/rect'; +import { applyAdjustments, isPositionDependentAdjustments } from '@workbench/canvas-engine/render/adjustments'; import { rasterizeSource } from '@workbench/canvas-engine/render/rasterizers'; import { bakeMatrix } from '@workbench/canvas-engine/transform/transformMath'; @@ -47,6 +52,10 @@ export class RasterizeLayerController { /** * Bakes the layer's source and transform into paint pixels as one undo step. Undo restores the parametric * contract, which rerasterizes from its source; redo reinstalls the baked pixels, never a rerender. + * + * A position-dependent stack (noise) is baked too, on the layer-local render before the transform resamples it as + * the display does, and leaves the contract: kept, it would place its grain on the baked pixel grid. A color-only + * stack stays on the contract and keeps applying to the baked pixels. */ rasterize(layerId: string): RasterizeLayerResult { const { ctx } = this.deps; @@ -72,6 +81,7 @@ export class RasterizeLayerController { return 'unsupported'; } const contentRect = getSourceContentRect(layer, document); + const bakesAdjustments = isPositionDependentAdjustments(layer.adjustments); const matrix = bakeMatrix(layer.transform); const bakedRect = roundOut(transformBounds(matrix, contentRect)); const txn = ctx.begin({ historyBytes: rgbaBytes(bakedRect) + HISTORY_ENTRY_OVERHEAD_BYTES }); @@ -85,6 +95,17 @@ export class RasterizeLayerController { } const scratch = this.deps.backend.createSurface(contentRect.width, contentRect.height); void rasterizeSource(source, this.deps.rasterizeDeps(document), scratch); + if (bakesAdjustments) { + // The render's own size: text resizes the scratch to its measured bounds. Its pixel (0,0) sits at the + // layer-local content origin, where the display's cache places it. + const { height, width } = scratch; + if (!txn.reserveRaster(width * height * 4)) { + return 'over-budget'; + } + const imageData = scratch.ctx.getImageData(0, 0, width, height); + applyAdjustments(imageData, layer.adjustments, contentRect); + scratch.ctx.putImageData(imageData, 0, 0); + } const baked = this.deps.backend.createSurface(bakedRect.width, bakedRect.height); baked.ctx.setTransform(1, 0, 0, 1, 0, 0); baked.ctx.clearRect(0, 0, bakedRect.width, bakedRect.height); @@ -93,11 +114,14 @@ export class RasterizeLayerController { baked.ctx.drawImage(scratch.canvas, contentRect.x, contentRect.y); baked.ctx.setTransform(1, 0, 0, 1, 0, 0); const parametric = structuredClone(layer); - const paint: CanvasLayerContract = { + const paint: CanvasRasterLayerContractV2 = { ...parametric, source: { bitmap: null, offset: { x: bakedRect.x, y: bakedRect.y }, type: 'paint' }, transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, }; + if (bakesAdjustments) { + delete paint.adjustments; + } const status = layerOperationStatus( txn.publish('Rasterize layer', this.convertStep(layerId, paint, parametric, bakedRect, baked), { bytes: rgbaBytes(bakedRect) + HISTORY_ENTRY_OVERHEAD_BYTES, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.test.ts index 50c8ebbef3c..9a732b619b9 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.test.ts @@ -1,6 +1,14 @@ -import type { CanvasDocumentContractV3 } from '@workbench/canvas-engine/contracts'; +import type { + CanvasAdjustmentEntry, + CanvasDocumentContractV3, + CanvasRasterLayerContractV2, +} from '@workbench/canvas-engine/contracts'; +import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; +import type { StubRasterSurface } from '@workbench/canvas-engine/render/raster.testStub'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { applyAdjustments } from '@workbench/canvas-engine/render/adjustments'; +import { createLayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; import { createTestStubRasterBackend } from '@workbench/canvas-engine/render/raster.testStub'; import { describe, expect, it, vi } from 'vitest'; @@ -38,6 +46,7 @@ describe('ThumbnailController budget', () => { getMaskPattern: () => null, isDisposed: () => false, isSupportedSource: () => true, + peekAdjustedSurface: () => null, projectId: 'p', rasterize, reportError: vi.fn(), @@ -49,3 +58,137 @@ describe('ThumbnailController budget', () => { expect(rasterize).not.toHaveBeenCalled(); }); }); + +describe('ThumbnailController adjusted rasters', () => { + const grain: CanvasAdjustmentEntry = { + amount: 0.4, + distribution: 'gaussian', + id: 'n', + isEnabled: true, + grain: 'color', + response: 'flat', + size: 1, + seed: 3, + type: 'noise', + }; + const invert: CanvasAdjustmentEntry = { id: 'i', isEnabled: true, type: 'invert' }; + + /** A 200×100 cache drawn into a 50×25 thumbnail (scale ¼), with the given display-copy peek. */ + const drawThumbnail = ( + peek: (surface: RasterSurface) => RasterSurface | null, + noise: CanvasAdjustmentEntry = grain + ) => { + const backend = createTestStubRasterBackend(); + const created: StubRasterSurface[] = []; + const layers = createLayerCacheStore(backend); + const entry = layers.getOrCreate('layer', 200, 100); + entry.hasPublishedPixels = true; + const layer: CanvasRasterLayerContractV2 = { + adjustments: [invert, noise], + blendMode: 'normal', + id: 'layer', + isEnabled: true, + isLocked: false, + name: 'Layer', + opacity: 1, + source: { image: { height: 100, imageName: 'layer.png', width: 200 }, type: 'image' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'raster', + }; + const document: CanvasDocumentContractV3 = { + background: 'transparent', + bbox: { height: 100, width: 200, x: 0, y: 0 }, + height: 100, + stacks: stacksFrom([layer]), + selectedLayerId: null, + version: 3, + width: 200, + }; + const controller = new ThumbnailController({ + backend: { + ...backend, + createSurface: (width, height) => { + const surface = backend.createSurface(width, height); + created.push(surface); + return surface; + }, + }, + getActiveProjectId: () => 'p', + getCheckerboard: () => backend.createSurface(8, 8), + getDocument: () => document, + getEntry: () => entry, + getMaskPattern: () => null, + isDisposed: () => false, + isSupportedSource: () => true, + peekAdjustedSurface: (_layer, peeked) => peek(peeked.surface), + projectId: 'p', + rasterize: vi.fn(), + reportError: vi.fn(), + setStatus: vi.fn(), + }); + const ctx = { clearRect: vi.fn(), createPattern: () => null, drawImage: vi.fn(), globalAlpha: 1 }; + const target = { getContext: () => ctx, height: 0, width: 0 } as unknown as HTMLCanvasElement; + expect(controller.draw('layer', target, 50)).toBe(true); + const thumbnail = created.find((surface) => surface.width === 50 && surface.height === 25)!; + return { cache: entry.surface, thumbnail }; + }; + + it("downscales the display's current adjusted copy without applying the stack again", () => { + const displayed = createTestStubRasterBackend().createSurface(200, 100); + const { thumbnail } = drawThumbnail(() => displayed); + + const drawn = thumbnail.callLog.find((call) => call.op === 'drawImage'); + expect(drawn?.args).toEqual([displayed.canvas, 0, 0, 50, 25]); + expect(thumbnail.callLog.some((call) => call.op === 'getImageData' || call.op === 'putImageData')).toBe(false); + }); + + it('without a display copy adjusts the thumbnail itself, scaling noise σ by the thumbnail scale', () => { + const { cache, thumbnail } = drawThumbnail(() => null); + + expect(thumbnail.callLog.find((call) => call.op === 'drawImage')?.args[0]).toBe(cache.canvas); + const written = thumbnail.callLog.find((call) => call.op === 'putImageData')!.args[0] as ImageData; + // The stub reads back opaque black; a quarter-size thumbnail carries a quarter of the grain's σ. + const expected = { data: new Uint8ClampedArray(50 * 25 * 4), height: 25, width: 50 } as ImageData; + for (let i = 3; i < expected.data.length; i += 4) { + expected.data[i] = 255; + } + applyAdjustments(expected, [invert, { ...grain, amount: 0.1 } as CanvasAdjustmentEntry], { x: 0, y: 0 }); + expect([...written.data]).toEqual([...expected.data]); + }); + + it("weakens grain narrower than the downscale blocks by the scale times the grain's span", () => { + // The grain kernel by its definition: a Gaussian of σ √(size² − 1) / 2 truncated at 3σ. Box-averaging B×B pixels + // leaves σ × (Σ taps)² / (Σ taps²) / B: about 3.07 / 4 for size 2 at a quarter scale. + const size = 2; + const sigma = Math.sqrt(size * size - 1) / 2; + const radius = Math.ceil(3 * sigma); + const taps = Array.from({ length: 2 * radius + 1 }, (_, i) => Math.exp(-((i - radius) ** 2) / (2 * sigma * sigma))); + const span = taps.reduce((sum, tap) => sum + tap, 0) ** 2 / taps.reduce((sum, tap) => sum + tap * tap, 0); + const { thumbnail } = drawThumbnail(() => null, { ...grain, size } as CanvasAdjustmentEntry); + + const written = thumbnail.callLog.find((call) => call.op === 'putImageData')!.args[0] as ImageData; + const expected = { data: new Uint8ClampedArray(50 * 25 * 4), height: 25, width: 50 } as ImageData; + for (let i = 3; i < expected.data.length; i += 4) { + expected.data[i] = 255; + } + const scaled = { ...grain, amount: 0.4 * (span / 4), size: 1 } as CanvasAdjustmentEntry; + applyAdjustments(expected, [invert, scaled], { x: 0, y: 0 }); + expect(span / 4).toBeLessThan(1); + expect([...written.data]).toEqual([...expected.data]); + }); + + it('keeps the σ of grain wider than the downscale blocks and shrinks the grain to thumbnail pixels', () => { + // Size 4 grain (blur σ √15 / 2 ≈ 1.9) spans about seven source pixels, wider than the 4×4 blocks a quarter-size + // thumbnail averages, so averaging barely weakens it; on the thumbnail grid it is one pixel across. + const coarse = { ...grain, size: 4 } as CanvasAdjustmentEntry; + const { thumbnail } = drawThumbnail(() => null, coarse); + + const written = thumbnail.callLog.find((call) => call.op === 'putImageData')!.args[0] as ImageData; + const expected = { data: new Uint8ClampedArray(50 * 25 * 4), height: 25, width: 50 } as ImageData; + for (let i = 3; i < expected.data.length; i += 4) { + expected.data[i] = 255; + } + applyAdjustments(expected, [invert, { ...grain, size: 1 } as CanvasAdjustmentEntry], { x: 0, y: 0 }); + expect([...written.data]).toEqual([...expected.data]); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.ts index fb3a5988483..a0c3a3f18c0 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/thumbnailController.ts @@ -1,5 +1,6 @@ import type { LayerThumbnailRequestResult } from '@workbench/canvas-engine/capabilities'; import type { + CanvasAdjustmentsContract, CanvasDocumentContractV3, CanvasLayerContract, CanvasLayerSourceContract, @@ -9,7 +10,7 @@ import type { RasterBackend, RasterSurface } from '@workbench/canvas-engine/rend import { lookupDocumentLayer } from '@workbench/canvas-engine/document-model/documentModel'; import { getSourceContentRect, renderableSourceOf } from '@workbench/canvas-engine/document/sources'; -import { applyAdjustments, isIdentityAdjustments } from '@workbench/canvas-engine/render/adjustments'; +import { applyAdjustments, isIdentityAdjustments, noiseGrainSpan } from '@workbench/canvas-engine/render/adjustments'; import { renderControlTransparency } from '@workbench/canvas-engine/render/controlTransparency'; import { colorizeMask } from '@workbench/canvas-engine/render/maskFill'; import { fitThumbnailSize } from '@workbench/canvas-engine/render/thumbnail'; @@ -24,6 +25,8 @@ export interface ThumbnailControllerOptions { readonly getMaskPattern: (style: string, color: string) => RasterSurface | null; readonly isDisposed: () => boolean; readonly isSupportedSource: (source: CanvasLayerSourceContract) => boolean; + /** The display's current adjusted copy of a raster layer's cache, or null; never builds a layer-sized surface. */ + readonly peekAdjustedSurface: (layer: CanvasLayerContract, entry: LayerCacheEntry) => RasterSurface | null; readonly rasterize: ( layer: CanvasLayerContract, document: CanvasDocumentContractV3 @@ -38,6 +41,26 @@ export interface ThumbnailControllerOptions { readonly pin?: (layerId: string) => { release(): void }; } +/** + * Bounded fallback without a display copy: the stack adjusts the downscaled pixels, its noise on the thumbnail's own + * grid. A downscale box-averages about 1/scale × 1/scale pixels, which leaves grain spanning `noiseGrainSpan` pixels + * `scale × span` of its σ once blocks are larger than the grain (exact for white noise), and keeps it when they are + * smaller; the grain itself shrinks to `size × scale` thumbnail pixels, at least one. An approximation: exact only in + * those limits, and it ignores how resampling actually weights pixels. + */ +const atThumbnailScale = (adjustments: CanvasAdjustmentsContract, scale: number): CanvasAdjustmentsContract => + scale >= 1 + ? adjustments + : adjustments.map((entry) => + entry.type === 'noise' + ? { + ...entry, + amount: entry.amount * Math.min(1, scale * noiseGrainSpan(entry.size)), + size: Math.max(1, entry.size * scale), + } + : entry + ); + /** Owns bounded thumbnail rendering and lazy rasterization status. */ export class ThumbnailController { private disposed = false; @@ -73,16 +96,19 @@ export class ThumbnailController { thumbnail.ctx.clearRect(0, 0, width, height); thumbnail.ctx.globalAlpha = 1; thumbnail.ctx.globalCompositeOperation = 'source-over'; - thumbnail.ctx.drawImage(entry.surface.canvas, 0, 0, width, height); + const adjusted = layer.type === 'raster' && !isIdentityAdjustments(layer.adjustments) ? layer.adjustments : null; + // Downscale the display's adjusted copy when it is current, so the thumbnail shows the canvas's grain and color. + const displayed = adjusted ? this.deps.peekAdjustedSurface(layer, entry) : null; + thumbnail.ctx.drawImage((displayed ?? entry.surface).canvas, 0, 0, width, height); const checker = context.createPattern(this.deps.getCheckerboard().canvas as CanvasImageSource, 'repeat'); if (checker) { context.fillStyle = checker; context.fillRect(0, 0, width, height); } let display = thumbnail; - if (layer.type === 'raster' && !isIdentityAdjustments(layer.adjustments)) { + if (adjusted && !displayed) { const pixels = thumbnail.ctx.getImageData(0, 0, width, height); - applyAdjustments(pixels, layer.adjustments); + applyAdjustments(pixels, atThumbnailScale(adjusted, width / entry.surface.width), { x: 0, y: 0 }); thumbnail.ctx.putImageData(pixels, 0, 0); } else if (layer.type === 'control' && layer.withTransparencyEffect) { display = renderControlTransparency(this.deps.backend, thumbnail, width, height); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/document-model/documentModel.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/document-model/documentModel.test.ts index ff7c6f5cf04..e8afd9bbd4d 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/document-model/documentModel.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/document-model/documentModel.test.ts @@ -335,6 +335,27 @@ describe('createDocumentModel', () => { expect(after.canvas.document.stacks.raster[1]).toMatchObject({ name: 'G copy', type: 'group' }); }); + it('keeps adjustment entries on a duplicated layer, noise seeds included, so the copy looks identical', () => { + const adjustments = [ + { + amount: 0.1, + distribution: 'gaussian', + id: 'n', + isEnabled: true, + grain: 'monochrome', + response: 'flat', + size: 1, + seed: 1234, + type: 'noise', + }, + ] as const; + const project = projectWith([layer('r1', 'raster', { adjustments: [...adjustments] })], 'r1'); + const { after } = roundTrip(project, { createId: () => 'd0', ids: ['r1'], type: 'duplicate' }); + + const copy = after.canvas.document.stacks.raster.find((node) => node.id === 'd0'); + expect(copy).toMatchObject({ adjustments }); + }); + it('moves within sibling lists, keeping a selected group and its descendants together', () => { const project = projectWith(tree(), 'r3'); const moved = roundTrip(project, { ids: ['G', 'r3'], kind: 'front', type: 'move' }); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.resources.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.resources.test.ts index 1e3334a9a7a..70bca061feb 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.resources.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.resources.test.ts @@ -225,6 +225,109 @@ describe('engine raster resource lifecycle', () => { }); }); +describe('engine raster composite export', () => { + // A 1000×1000 noisy layer at 0.1 scale: a 10×10 export samples only its 101×101 layer-local corner (with margin). + const LAYER_RECT = { height: 1_000, width: 1_000, x: 0, y: 0 }; + const EXPORT_RECT = { height: 10, width: 10, x: 0, y: 0 }; + const SAMPLED_SIZE: [number, number] = [101, 101]; + + const createNoisyEngine = () => { + const noisy = layerContract('noisy', 'raster', { + adjustments: [ + { + amount: 0.2, + distribution: 'gaussian', + grain: 'monochrome', + id: 'grain', + isEnabled: true, + response: 'flat', + seed: 5, + size: 1, + type: 'noise', + }, + ], + source: { bitmap: { height: 1_000, imageName: 'noisy', width: 1_000 }, offset: { x: 0, y: 0 }, type: 'paint' }, + transform: { rotation: 0, scaleX: 0.1, scaleY: 0.1, x: 0, y: 0 }, + }); + const project = applyCanvasProjectMutation(createInitialWorkbenchState().projects[0]!, { + document: documentFrom([noisy]), + type: 'replaceCanvasDocument', + }); + const stub = createTestStubRasterBackend(); + const allocated: [number, number][] = []; + let recording = false; + const { engine } = createCanvasEngine({ + backend: { + ...stub, + createSurface: (width, height) => { + if (recording) { + allocated.push([width, height]); + } + return stub.createSurface(width, height); + }, + }, + fonts: null, + imageResolver: () => Promise.resolve(new Blob()), + mutationPort: { + commitEdit: () => undefined, + dispatch: () => false, + getCanvasState: () => project.canvas, + subscribe: () => () => undefined, + } as unknown as CanvasProjectMutationPort, + projectId: 'project', + reportError: () => undefined, + uploadImage: () => Promise.resolve({ height: 10, imageName: 'unused', width: 10 }), + uploadIntermediateImage: () => Promise.resolve({ height: 10, imageName: 'unused', width: 10 }), + }); + const raster = owners.raster!; + raster.layers.getOrCreateRect('noisy', LAYER_RECT); + raster.layers.publishPixels('noisy'); + const reserved: number[] = []; + const reserveOperation = raster.memory.reserveOperation.bind(raster.memory); + vi.spyOn(raster.memory, 'reserveOperation').mockImplementation((bytes, options) => { + reserved.push(bytes); + return reserveOperation(bytes, options); + }); + const exportRect = async () => { + recording = true; + try { + return await engine.exports.exportRasterComposite({ bounds: 'rect', rect: EXPORT_RECT }); + } finally { + recording = false; + } + }; + return { allocated, engine, exportRect, noisy, raster, reserved }; + }; + + it('adjusts only the reserved sampled region of a downscaled noisy layer when the display has no adjusted copy', async () => { + const { allocated, engine, exportRect, raster, reserved } = createNoisyEngine(); + + await expect(exportRect()).resolves.toMatchObject({ status: 'ok' }); + + // The output and the sampled layer-local scratch, never a layer-sized adjusted copy outside the reservation. + expect(allocated).toEqual([[EXPORT_RECT.width, EXPORT_RECT.height], SAMPLED_SIZE]); + const [sampledWidth, sampledHeight] = SAMPLED_SIZE; + // Output surface, plus the scratch surface and its ImageData readback. + expect(reserved[0]).toBeGreaterThanOrEqual( + (EXPORT_RECT.width * EXPORT_RECT.height + sampledWidth * sampledHeight * 2) * 4 + ); + expect(raster.memory.snapshot()).toMatchObject({ derivedBytes: 0, reservedBytes: 0 }); + engine.lifecycle.dispose(); + }); + + it("draws the display's memoized adjusted copy without adjusting again", async () => { + const { allocated, engine, exportRect, noisy, raster } = createNoisyEngine(); + raster.getAdjustedSurface(noisy, raster.layers.peek('noisy')!); + const derivedBytes = raster.memory.snapshot().derivedBytes; + + await expect(exportRect()).resolves.toMatchObject({ status: 'ok' }); + + expect(allocated).toEqual([[EXPORT_RECT.width, EXPORT_RECT.height]]); + expect(raster.memory.snapshot().derivedBytes).toBe(derivedBytes); + engine.lifecycle.dispose(); + }); +}); + describe('engine group surfaces', () => { it('releases a deleted group surface', () => { const { apply, engine, raster } = createEngine(); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.test.ts index 327c87a54f8..cb314554736 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.test.ts @@ -1560,7 +1560,7 @@ describe('createCanvasEngine', () => { engine.lifecycle.dispose(); }); - it('exportBakedLayerPixels bakes raster adjustments into the exported surface only', async () => { + it('exportBakedLayerPixels adjusts a layer-local copy before the transform, leaving the cache untouched', async () => { const layer = { ...rasterLayer('a'), adjustments: [ @@ -1568,8 +1568,17 @@ describe('createCanvasEngine', () => { ], }; const { store } = createFakeStore({ ...makeDoc(), stacks: stacksFrom([layer]) }); + const base = createTestStubRasterBackend(); + const surfaces: StubRasterSurface[] = []; const engine = createCanvasEngine({ - backend: createTestStubRasterBackend(), + backend: { + ...base, + createSurface: (width, height) => { + const surface = base.createSurface(width, height); + surfaces.push(surface); + return surface; + }, + }, imageResolver: () => Promise.resolve(new Blob()), projectId: 'p1', store, @@ -1583,7 +1592,13 @@ describe('createCanvasEngine', () => { if (raw.status === 'ok' && baked.status === 'ok') { expect(baked.surface).not.toBe(raw.surface); expect((raw.surface as StubRasterSurface).callLog.some((entry) => entry.op === 'putImageData')).toBe(false); - expect((baked.surface as StubRasterSurface).callLog).toEqual( + const bakedLog = (baked.surface as StubRasterSurface).callLog; + // The baked surface only resamples; the adjusted pixels come from the layer-local copy it draws. + expect(bakedLog.some((entry) => entry.op === 'putImageData')).toBe(false); + const drawn = bakedLog.find((entry) => entry.op === 'drawImage')!.args[0]; + const copy = surfaces.find((surface) => surface.canvas === drawn)!; + expect(copy).not.toBe(raw.surface); + expect(copy.callLog).toEqual( expect.arrayContaining([ { args: [0, 0, 10, 10], op: 'getImageData' }, expect.objectContaining({ op: 'putImageData' }), diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.ts index 7d850b10de7..28d9e30f253 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.ts @@ -141,7 +141,6 @@ import { } from '@workbench/canvas-engine/engineStores'; import { exportRasterComposite as exportRasterCompositeWithDeps, - RasterCompositeOverBudgetError, type RasterCompositeExportRequest, type RasterCompositeExportSnapshot, } from '@workbench/canvas-engine/exportRasterComposite'; @@ -160,6 +159,7 @@ import { createMaskPatternTile } from '@workbench/canvas-engine/render/maskFill' import { colorLoupePixels, renderOverlay } from '@workbench/canvas-engine/render/overlayRenderer'; import { trimPaintCacheToAlpha } from '@workbench/canvas-engine/render/paintCacheTrim'; import { createDomRasterBackend, type RasterBackend, type RasterSurface } from '@workbench/canvas-engine/render/raster'; +import { RasterCompositeOverBudgetError } from '@workbench/canvas-engine/render/rasterComposite'; import { rasterizeSource, type ImageResolver, type RasterizeDeps } from '@workbench/canvas-engine/render/rasterizers'; import { fitThumbnailSize, getLayerThumbnailDisplayKey } from '@workbench/canvas-engine/render/thumbnail'; import { documentDeltaToLocal, liftSelectedPixels } from '@workbench/canvas-engine/selection/floatingSelection'; @@ -1254,6 +1254,8 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC isSupportedSource: isSupportedExportSource, layers: layerCache, invalidateLayerCache, + peekAdjustedCacheSurface: (layerId, surface, adjustments) => + rasterController.peekAdjustedCacheSurface(layerId, surface, adjustments), pin: (layerId) => rasterController.memory.pin(layerId), reserve: (bytes) => rasterController.memory.reserve(bytes, { generation: lifecycleGeneration, purpose: 'raster-export' }), @@ -2547,8 +2549,9 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC documentGeneration: rasterController.getDocumentGeneration(), lifecycleGeneration, }), + // Never builds the display's copy: a miss adjusts only the sampled region the export reserved. adjustedSurface: (layerId, surface, adjustments) => - rasterController.getAdjustedCacheSurface(layerId, surface, adjustments), + rasterController.peekAdjustedCacheSurface(layerId, surface, adjustments), getLayerSurface: requireLayerSurfaceForExport, isSnapshotCurrent: (snapshot) => !disposed && @@ -2660,6 +2663,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC isCacheReady: isLayerCacheReadyForOp, layers: layerCache, needsPixelPersistence: layerNeedsPixelPersistence, + peekAdjustedSurface: (layer, entry) => rasterController.peekAdjustedSurface(layer, entry), publishSelectedLayerIds: (primaryId, selectedIds) => opts.setSelectedLayerIds?.(primaryId, selectedIds), }, rasterize: { @@ -2677,6 +2681,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC getMaskPattern: getMaskPatternTile, isDisposed: () => disposed, isSupportedSource: isSupportedExportSource, + peekAdjustedSurface: (layer, entry) => rasterController.peekAdjustedSurface(layer, entry), pin: (layerId) => rasterController.memory.pin(layerId), projectId, rasterize: rasterizeLayerForThumbnail, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.test.ts index 835be340de9..4cd1ed145d3 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.test.ts @@ -1,6 +1,9 @@ import type { CanvasBlendMode } from '@workbench/canvas-engine/contracts'; +import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; +import type { StubRasterSurface } from '@workbench/canvas-engine/render/raster.testStub'; import type { Rect } from '@workbench/canvas-engine/types'; +import { applyAdjustments } from '@workbench/canvas-engine/render/adjustments'; import { createTestStubRasterBackend } from '@workbench/canvas-engine/render/raster.testStub'; import { readPsd, writePsd } from 'ag-psd'; import { describe, expect, it } from 'vitest'; @@ -368,3 +371,61 @@ describe('planPsdExport — folders', () => { expect(folder.children![0]!.opacity).toBeCloseTo(0.5, 2); }); }); + +describe('executePsdExport — adjustments', () => { + it("bakes the layer's own stack layer-local before the transform and group stacks at document pixels", async () => { + const grain = (id: string, seed: number) => + ({ + amount: 0.2, + distribution: 'gaussian', + id, + isEnabled: true, + grain: 'color', + response: 'flat', + size: 1, + seed, + type: 'noise', + }) as const; + const own = [grain('own', 1)]; + const enclosing = [grain('group', 2)]; + const contentRect: Rect = { height: 10, width: 20, x: -4, y: -3 }; + const plan = planPsdExport([ + layer({ + adjustments: own, + contentRect, + groupAdjustments: enclosing, + transform: { rotation: 0, scaleX: 0.5, scaleY: 0.5, x: 9, y: 6 }, + }), + ]); + const backend = createTestStubRasterBackend(); + const midGray = (width: number, height: number): ImageData => { + const data = new Uint8ClampedArray(width * height * 4).fill(128); + return { data, height, width } as ImageData; + }; + const writes: { surface: RasterSurface; pixels: ImageData }[] = []; + + await executePsdExport(plan, 'grain.psd', { + backend, + download: () => undefined, + getLayerSurface: () => Promise.resolve({ rect: contentRect, surface: backend.createSurface(20, 10) }), + readImageData: (_surface, rect) => midGray(rect.width, rect.height), + writeImageData: (surface, pixels) => { + writes.push({ pixels, surface }); + }, + writePsd: () => Promise.resolve(new ArrayBuffer(0)), + }); + + const [local, world] = writes; + const expectedLocal = midGray(20, 10); + applyAdjustments(expectedLocal, own, contentRect); + expect([...local!.pixels.data]).toEqual([...expectedLocal.data]); + // Scaled ½ and moved (9, 6), the content's world bounds round out to (7, 4) 10×6. + expect(findLayer(plan, 'a').worldRect).toEqual({ height: 6, width: 10, x: 7, y: 4 }); + const expectedWorld = midGray(10, 6); + applyAdjustments(expectedWorld, enclosing, { x: 7, y: 4 }); + expect([...world!.pixels.data]).toEqual([...expectedWorld.data]); + // The world surface resamples the adjusted layer-local copy, not the raw cache. + const drawn = (world!.surface as StubRasterSurface).callLog.find((call) => call.op === 'drawImage'); + expect(drawn?.args[0]).toBe(local!.surface.canvas); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.ts index 83ef76ca8de..232b818ff27 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/export/psdExport.ts @@ -71,8 +71,13 @@ export interface PsdExportLayerInput { isEnabled: boolean; /** Organizational color label; PSD carries it natively as `layerColor`. */ colorLabel?: CanvasColorLabel; - /** Non-destructive adjustments to bake into the layer's pixels, if any. */ + /** The layer's own non-destructive adjustments to bake into its pixels, if any (layer-local reference space). */ adjustments?: CanvasAdjustmentsContract; + /** + * Enclosing group stacks, innermost first, baked after the layer's own: PSD cannot represent them. They work in + * document pixels, as the groups do. + */ + groupAdjustments?: CanvasAdjustmentsContract; } /** A raster group: a folder in the PSD, children top-first. */ @@ -122,6 +127,7 @@ export interface PsdPlanLayer { contributes: boolean; colorLabel?: CanvasColorLabel; adjustments?: CanvasAdjustmentsContract; + groupAdjustments?: CanvasAdjustmentsContract; } /** A planned PSD folder; `children` are in ag-psd bottom-to-top order. */ @@ -233,6 +239,7 @@ export const planPsdExport = ( const top = worldRect.y - canvasRect.y; const layer: PsdPlanLayer = { adjustments: input.adjustments, + groupAdjustments: input.groupAdjustments, blendMode: mapped ?? 'normal', bottom: top + worldRect.height, colorLabel: input.colorLabel, @@ -361,7 +368,9 @@ const throwIfAborted = (signal?: AbortSignal): void => { /** * Bakes transformed pixels and adjustments into world-AABB bounds, returning surface and straight-alpha ImageData. - * Opacity/blend remain PSD properties. + * The layer's own stack adjusts layer-local pixels before the transform resamples them, as the display does; the + * enclosing group stacks then adjust the world pixels, which are document pixels at scale 1. Opacity/blend remain + * PSD properties. */ const bakeLayer = async ( planLayer: PsdPlanLayer, @@ -380,18 +389,27 @@ const bakeLayer = async ( const { rect, surface: cache } = await deps.getLayerSurface(planLayer.id); throwIfAborted(deps.signal); + let source = cache; + if (planLayer.adjustments && rect.width > 0 && rect.height > 0) { + source = deps.backend.createSurface(rect.width, rect.height); + setTransform(source.ctx, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }); + source.ctx.drawImage(cache.canvas, 0, 0); + const localPixels = read(source, { height: rect.height, width: rect.width, x: 0, y: 0 }); + applyAdjustments(localPixels, planLayer.adjustments, rect); + write(source, localPixels, 0, 0); + } // local→world then shift into AABB-local (translation only affects e/f). const local = layerMatrix(planLayer.transform); setTransform(ctx, { ...local, e: local.e - worldRect.x, f: local.f - worldRect.y }); // The cache holds pixels for `rect` in layer-local space; draw at that origin. if (rect.width > 0 && rect.height > 0) { - ctx.drawImage(cache.canvas, rect.x, rect.y); + ctx.drawImage(source.canvas, rect.x, rect.y); } const fullRect: Rect = { height, width, x: 0, y: 0 }; const imageData = read(surface, fullRect); - if (planLayer.adjustments) { - applyAdjustments(imageData, planLayer.adjustments); + if (planLayer.groupAdjustments) { + applyAdjustments(imageData, planLayer.groupAdjustments, worldRect); // Write the adjusted pixels back so the flattened composite (below) reuses // this surface directly. write(surface, imageData, 0, 0); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.test.ts index bbbbeef1972..13e308438ae 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.test.ts @@ -5,6 +5,7 @@ import type { Rect } from '@workbench/canvas-engine/types'; import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { createTestStubRasterBackend } from '@workbench/canvas-engine/render/raster.testStub'; +import { MEASURED_NOISY_TEXT } from '@workbench/canvas-engine/render/rasterComposite.testing'; import { describe, expect, it, vi } from 'vitest'; import type { ExportRasterCompositeDeps, RasterCompositeExportSnapshot } from './exportRasterComposite'; @@ -196,6 +197,73 @@ describe('exportRasterComposite', () => { expect(reserve).toHaveBeenCalledWith(24_576); }); + describe('a noisy text layer whose measured bounds exceed the estimate', () => { + const { bbox, layer, measured, plannedScratchPixels } = MEASURED_NOISY_TEXT; + /** Instruments allocation and reservation; `uncovered` lists allocations made beyond outstanding reservations. */ + const measuredDeps = () => { + const stub = createTestStubRasterBackend(); + const layerSurface = stub.createSurface(measured.width, measured.height); + const { deps } = makeDeps(makeDoc([layer])); + let allocatedBytes = 0; + let reservedBytes = 0; + const uncovered: number[] = []; + const allocate = (pixels: number) => { + allocatedBytes += pixels * 4; + if (allocatedBytes > reservedBytes) { + uncovered.push(pixels); + } + }; + deps.backend.createSurface = (width, height) => { + allocate(width * height); + return stub.createSurface(width, height); + }; + deps.readImageData = (surface, rect) => { + allocate(rect.width * rect.height); + return surface.ctx.getImageData(rect.x, rect.y, rect.width, rect.height); + }; + deps.reserve = (bytes) => { + reservedBytes += bytes; + return { lease: { release: () => undefined }, status: 'ok' }; + }; + const releaseLayer = vi.fn(); + deps.getLayerSurface = vi.fn(() => + Promise.resolve({ rect: measured, release: releaseLayer, surface: layerSurface }) + ); + return { allocatedBytes: () => allocatedBytes, deps, releaseLayer, uncovered }; + }; + + it('reserves the measured scratch before allocating it', async () => { + const { allocatedBytes, deps, uncovered } = measuredDeps(); + + await expect(exportRasterComposite({ bounds: 'rect', rect: bbox }, deps)).resolves.toMatchObject({ + status: 'ok', + }); + + // The measured 400×100 scratch and its ImageData were allocated, each within outstanding reservations. + expect(allocatedBytes()).toBeGreaterThanOrEqual( + (bbox.width * bbox.height + measured.width * measured.height * 2) * 4 + ); + expect(uncovered).toEqual([]); + }); + + it('refuses as over-budget instead of allocating past its reservation', async () => { + const { allocatedBytes, deps, releaseLayer } = measuredDeps(); + // Output, plus the planned scratch surface and its ImageData. + const upFrontBytes = (bbox.width * bbox.height + plannedScratchPixels * 2) * 4; + const memory = new RasterMemoryBudgetController({ budgetBytes: upFrontBytes }); + deps.reserve = (bytes) => memory.reserveOperation(bytes, { purpose: 'raster-export' }); + + await expect(exportRasterComposite({ bounds: 'rect', rect: bbox }, deps)).resolves.toEqual({ + status: 'over-budget', + }); + + // Only the output surface was allocated; the layer lease and every reservation are released. + expect(allocatedBytes()).toBe(bbox.width * bbox.height * 4); + expect(releaseLayer).toHaveBeenCalledOnce(); + expect(memory.snapshot().reservedBytes).toBe(0); + }); + }); + it('discards an encoded blob if the document changes during export', async () => { const { deps } = makeDeps(makeDoc([rasterLayer('raster')])); vi.mocked(deps.isSnapshotCurrent).mockReturnValueOnce(true).mockReturnValue(false); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.ts index eed1b08dc34..8d83553498e 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/exportRasterComposite.ts @@ -4,9 +4,10 @@ import type { Rect } from '@workbench/canvas-engine/types'; import { isEmpty } from '@workbench/canvas-engine/math/rect'; import { - type CompositeEntry, + compositeScratchPixels, getBaseRasterContentBounds, planBaseRasterComposite, + RasterCompositeOverBudgetError, renderRasterComposite, type RenderRasterCompositeDeps, } from '@workbench/canvas-engine/render/rasterComposite'; @@ -24,30 +25,15 @@ export interface RasterCompositeExportSnapshot { lifecycleGeneration: number; } -export class RasterCompositeOverBudgetError extends Error { - constructor() { - super('Raster composite allocation is over budget.'); - this.name = 'RasterCompositeOverBudgetError'; - } -} - export interface ExportRasterCompositeDeps extends RenderRasterCompositeDeps { backend: RenderRasterCompositeDeps['backend'] & { encodeSurface(surface: RasterSurface, type?: string): Promise; }; captureSnapshot(): RasterCompositeExportSnapshot; isSnapshotCurrent(snapshot: RasterCompositeExportSnapshot): boolean; - reserve?( - bytes: number - ): - | { status: 'ok'; lease: { release(): void } } - | { status: 'over-budget'; requestedBytes: number; availableBytes: number }; pin?(layerIds: readonly string[]): { release(): void }; } -const countGroupScopes = (scopes: CompositeEntry['groupScopes']): number => - (scopes ?? []).reduce((total, scope) => total + 1 + countGroupScopes(scope.children), 0); - export const exportRasterComposite = async ( request: RasterCompositeExportRequest, deps: ExportRasterCompositeDeps @@ -68,13 +54,8 @@ export const exportRasterComposite = async ( return { status: 'empty' }; } - // Each adjusted layer needs both a temporary surface and a same-sized - // ImageData buffer in addition to the final composite surface; each adjusted - // GROUP scope likewise isolates into a buffer plus an ImageData pass. - const surfaceCount = - 1 + - (entry.layers.filter((layer) => layer.adjustments !== undefined).length + countGroupScopes(entry.groupScopes)) * 2; - const reservation = deps.reserve?.(rect.width * rect.height * 4 * surfaceCount); + // The final composite surface, plus each adjusted layer's and group scope's scratch surface and its ImageData. + const reservation = deps.reserve?.((rect.width * rect.height + compositeScratchPixels(entry) * 2) * 4); if (reservation?.status === 'over-budget') { return { status: 'over-budget' }; } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/fontReferences.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/fontReferences.ts index 108c379be7d..7c3970b11d8 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/fontReferences.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/fontReferences.ts @@ -230,7 +230,9 @@ export const replaceCanvasFontReferences = ( changed = true; } } - const nextDocument = changed ? { ...document, stacks, version: 4 as const } : document; + // Custom-font typography needs schema 4; never lower a document already on a newer one. + const version = Math.max(document.version, 4) as CanvasDocumentContractV3['version']; + const nextDocument = changed ? { ...document, stacks, version } : document; return { clampedAxisTags: [...clampedAxisTags].sort(), document: nextDocument, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceCache.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceCache.ts index 3cdb5f51c9a..11e5b7b2890 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceCache.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceCache.ts @@ -22,6 +22,12 @@ export interface AdjustedSurfaceCache { entry: LayerCacheEntry, adjustments: CanvasAdjustmentsContract | undefined ): RasterSurface | null; + /** The memoized adjusted surface when it is current for `entry` and `adjustments`; never builds one. */ + peek( + layerId: string, + entry: LayerCacheEntry, + adjustments: CanvasAdjustmentsContract | undefined + ): RasterSurface | null; /** Drops a layer's memoized adjusted surface (e.g. on layer delete). */ delete(layerId: string): void; /** Number of memoized adjusted surfaces (for tests / accounting). */ @@ -92,7 +98,8 @@ export const createAdjustedSurfaceCache = ( region.height ); const imageData = ctx.getImageData(region.x, region.y, region.width, region.height); - applyAdjustments(imageData, adjustments); + // A layer's stack works in layer-local pixels; surface (0,0) sits at `entry.rect`'s origin. + applyAdjustments(imageData, adjustments, { x: entry.rect.x + region.x, y: entry.rect.y + region.y }); ctx.putImageData(imageData, region.x, region.y); return surface; }, @@ -113,6 +120,16 @@ export const createAdjustedSurfaceCache = ( cache.dispose(); }, get, + peek: (layerId, entry, adjustments) => + isIdentityAdjustments(adjustments) + ? null + : cache.peek({ + kind: 'adjustments', + layerId, + paramsKey: adjustmentsKey(adjustments), + source: entry.surface, + sourceVersion: entry.version, + }), size: cache.size, }; }; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceDamage.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceDamage.browser.test.ts index 0428e175874..357ed2f95fc 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceDamage.browser.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustedSurfaceDamage.browser.test.ts @@ -11,8 +11,20 @@ import { describe, expect, it } from 'vitest'; const WIDTH = 320; const HEIGHT = 240; +// Noise is position dependent: partial refreshes must place it by layer-local pixel, as a full rebuild does. const ADJUSTMENTS: CanvasAdjustmentsContract = [ { brightness: 0.25, contrast: 0.15, id: 'bc', isEnabled: true, type: 'brightness-contrast' }, + { + amount: 0.2, + distribution: 'gaussian', + id: 'grain', + isEnabled: true, + grain: 'color', + response: 'flat', + size: 1, + seed: 12345, + type: 'noise', + }, { id: 'sat', isEnabled: true, saturation: -0.2, type: 'hsl' }, ]; @@ -154,4 +166,13 @@ describe('partially-refreshed adjusted surfaces match a full rebuild', () => { expect(a.length).toBe(size); expect(worstDifference(a, b)).toBe(0); }); + + it('keeps noise anchored to layer-local pixels when the surface origin moves', () => { + const { adjusted, layers } = setup(true); + const before = pixelsOf(adjusted.get('L', layers.get('L')!, ADJUSTMENTS)!); + layers.growToRect('L', { height: HEIGHT + 80, width: WIDTH + 60, x: -30, y: -40 }); + const grown = adjusted.get('L', layers.get('L')!, ADJUSTMENTS)!; + // The original pixels now sit 30 right and 40 down in the surface, at unchanged layer-local coordinates. + expect(worstDifference(grown.ctx.getImageData(30, 40, WIDTH, HEIGHT).data, before)).toBe(0); + }); }); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.test.ts index 8bb2c573d6b..55ea14df38c 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.test.ts @@ -9,6 +9,9 @@ import { compileAdjustments, isIdentityAdjustmentEntry, isIdentityAdjustments, + isPositionDependentAdjustments, + NOISE_FULL_SCALE_SIGMA, + noiseGrainSpan, } from './adjustments'; /** Builds an ImageData-like object (node has no DOM ImageData; the shape is enough). */ @@ -49,6 +52,23 @@ const hue = (rotation: number, overrides: EntryOverrides = {}) => ({ id: 'hue', isEnabled: true, rotation, type: 'hue', ...overrides }) as const; const invert = (overrides: EntryOverrides = {}) => ({ id: 'inv', isEnabled: true, type: 'invert', ...overrides }) as const; +type NoiseEntry = Extract; +const noise = (amount: number, overrides: Partial = {}): NoiseEntry => ({ + amount, + distribution: 'gaussian', + grain: 'monochrome', + id: 'nz', + isEnabled: true, + response: 'flat', + seed: 0x1234abcd, + size: 1, + type: 'noise', + ...overrides, +}); +const GRAINS = ['monochrome', 'color', 'chroma'] as const; +const RESPONSES = ['flat', 'midtones'] as const; + +const ORIGIN = { x: 0, y: 0 }; describe('buildCurveLut', () => { it('is the identity for absent / empty / diagonal curves', () => { @@ -182,10 +202,10 @@ describe('compileAdjustments', () => { ], }); const sequential = imageData([10, 20, 30, 255, 200, 100, 50, 255]); - applyAdjustments(sequential, [curve]); - applyAdjustments(sequential, [bc(0.1, 0.4)]); + applyAdjustments(sequential, [curve], ORIGIN); + applyAdjustments(sequential, [bc(0.1, 0.4)], ORIGIN); const folded = imageData([10, 20, 30, 255, 200, 100, 50, 255]); - applyAdjustments(folded, [curve, bc(0.1, 0.4)]); + applyAdjustments(folded, [curve, bc(0.1, 0.4)], ORIGIN); expect(Array.from(folded.data)).toEqual(Array.from(sequential.data)); }); }); @@ -200,7 +220,7 @@ describe('exposure and channel-scoped levels', () => { it('applies exposure in linear light: +1 stop doubles linear energy', () => { // srgb 64 → linear ≈0.0513, doubled ≈0.1026 → srgb ≈90; srgb 188 → linear ≈0.503, doubled clamps → 255. const img = imageData([64, 188, 0, 255]); - applyAdjustments(img, [exposure(1)]); + applyAdjustments(img, [exposure(1)], ORIGIN); expect(Math.abs(img.data[0]! - 90)).toBeLessThanOrEqual(1); expect(img.data[1]).toBe(255); expect(img.data[2]).toBe(0); @@ -208,8 +228,8 @@ describe('exposure and channel-scoped levels', () => { it('darkening by a stop then brightening by a stop returns close to the original midtones', () => { const img = imageData([128, 96, 160, 255]); - applyAdjustments(img, [exposure(-1)]); - applyAdjustments(img, [exposure(1)]); + applyAdjustments(img, [exposure(-1)], ORIGIN); + applyAdjustments(img, [exposure(1)], ORIGIN); expect(Math.abs(img.data[0]! - 128)).toBeLessThanOrEqual(2); expect(Math.abs(img.data[1]! - 96)).toBeLessThanOrEqual(2); expect(Math.abs(img.data[2]! - 160)).toBeLessThanOrEqual(2); @@ -217,7 +237,7 @@ describe('exposure and channel-scoped levels', () => { it('a channel-scoped levels entry remaps only its channel', () => { const img = imageData([200, 200, 200, 255]); - applyAdjustments(img, [channelLevels('g', 128)]); + applyAdjustments(img, [channelLevels('g', 128)], ORIGIN); expect(img.data[0]).toBe(200); expect(img.data[1]).toBe(Math.round((200 / 255) * 128)); expect(img.data[2]).toBe(200); @@ -228,11 +248,11 @@ describe('exposure and channel-scoped levels', () => { expect(folded.map((segment) => segment.kind)).toEqual(['lut']); const sequential = imageData([10, 20, 30, 255, 200, 100, 50, 255]); - applyAdjustments(sequential, [exposure(0.5)]); - applyAdjustments(sequential, [channelLevels('b', 200)]); - applyAdjustments(sequential, [invert()]); + applyAdjustments(sequential, [exposure(0.5)], ORIGIN); + applyAdjustments(sequential, [channelLevels('b', 200)], ORIGIN); + applyAdjustments(sequential, [invert()], ORIGIN); const combined = imageData([10, 20, 30, 255, 200, 100, 50, 255]); - applyAdjustments(combined, [exposure(0.5), channelLevels('b', 200), invert()]); + applyAdjustments(combined, [exposure(0.5), channelLevels('b', 200), invert()], ORIGIN); expect([...combined.data]).toEqual([...sequential.data]); }); @@ -248,26 +268,26 @@ describe('exposure and channel-scoped levels', () => { describe('applyAdjustments', () => { it('is a no-op for an identity stack and never modifies alpha', () => { const img = imageData([10, 20, 30, 128]); - applyAdjustments(img, [bc(0, 0), hsl(0)]); + applyAdjustments(img, [bc(0, 0), hsl(0)], ORIGIN); expect(Array.from(img.data)).toEqual([10, 20, 30, 128]); - applyAdjustments(img, [bc(0.5, 0)]); + applyAdjustments(img, [bc(0.5, 0)], ORIGIN); expect(img.data[3]).toBe(128); }); it('brightens rgb additively and clamps', () => { const img = imageData([10, 20, 30, 255]); - applyAdjustments(img, [bc(0.5, 0)]); + applyAdjustments(img, [bc(0.5, 0)], ORIGIN); expect(img.data[0]).toBe(138); // 10 + 128 expect(img.data[1]).toBe(148); expect(img.data[2]).toBe(158); const clamped = imageData([200, 0, 0, 255]); - applyAdjustments(clamped, [bc(0.5, 0)]); + applyAdjustments(clamped, [bc(0.5, 0)], ORIGIN); expect(clamped.data[0]).toBe(255); }); it('applies contrast about mid-grey', () => { const img = imageData([64, 128, 192, 255]); - applyAdjustments(img, [bc(0, 1)]); + applyAdjustments(img, [bc(0, 1)], ORIGIN); expect(img.data[0]).toBe(0); // (64-128)*2+128 expect(img.data[1]).toBe(128); expect(img.data[2]).toBe(255); @@ -275,7 +295,7 @@ describe('applyAdjustments', () => { it('fully desaturates to luma at saturation -1', () => { const img = imageData([200, 100, 50, 255]); - applyAdjustments(img, [hsl(-1)]); + applyAdjustments(img, [hsl(-1)], ORIGIN); const luma = Math.round(0.299 * 200 + 0.587 * 100 + 0.114 * 50); expect(img.data[0]).toBe(luma); expect(img.data[1]).toBe(luma); @@ -284,24 +304,24 @@ describe('applyAdjustments', () => { it('inverts every channel exactly', () => { const img = imageData([0, 128, 255, 200]); - applyAdjustments(img, [invert()]); + applyAdjustments(img, [invert()], ORIGIN); expect(Array.from(img.data)).toEqual([255, 127, 0, 200]); }); it('remaps through levels: input range, gamma midtones, output range', () => { // Input 64..192 stretched to full range: 64 → 0, 192 → 255, midpoint → 128. const stretch = imageData([64, 128, 192, 255]); - applyAdjustments(stretch, [levels(64, 192, 1, 0, 255)]); + applyAdjustments(stretch, [levels(64, 192, 1, 0, 255)], ORIGIN); expect(Array.from(stretch.data.slice(0, 3))).toEqual([0, 128, 255]); // Gamma 2 lifts midtones: 128 → 255 * (0.5)^(1/2) ≈ 180. const midtones = imageData([128, 0, 255, 255]); - applyAdjustments(midtones, [levels(0, 255, 2, 0, 255)]); + applyAdjustments(midtones, [levels(0, 255, 2, 0, 255)], ORIGIN); expect(midtones.data[0]).toBe(Math.round(255 * Math.sqrt(128 / 255))); expect(midtones.data[1]).toBe(0); expect(midtones.data[2]).toBe(255); // Output compression maps black to 50 and white to 200. const compress = imageData([0, 255, 128, 255]); - applyAdjustments(compress, [levels(0, 255, 1, 50, 200)]); + applyAdjustments(compress, [levels(0, 255, 1, 50, 200)], ORIGIN); expect(compress.data[0]).toBe(50); expect(compress.data[1]).toBe(200); }); @@ -309,31 +329,460 @@ describe('applyAdjustments', () => { it('hue rotation cycles the primaries and preserves grays', () => { // 120° sends red toward green (the standard hueRotate matrix dims it; green must dominate). const red = imageData([255, 0, 0, 255]); - applyAdjustments(red, [hue(120)]); + applyAdjustments(red, [hue(120)], ORIGIN); expect(red.data[0]).toBe(0); expect(red.data[2]).toBe(0); expect(red.data[1]).toBeGreaterThan(90); // A full ±360° is identity; gray is on the rotation axis and never moves. const gray = imageData([128, 128, 128, 255]); - applyAdjustments(gray, [hue(90)]); + applyAdjustments(gray, [hue(90)], ORIGIN); expect(Array.from(gray.data)).toEqual([128, 128, 128, 255]); const wrapped = imageData([200, 100, 50, 255]); - applyAdjustments(wrapped, [hue(360)]); + applyAdjustments(wrapped, [hue(360)], ORIGIN); expect(Array.from(wrapped.data)).toEqual([200, 100, 50, 255]); }); it('entry order changes the pixels; a disabled entry renders exactly as a removed one', () => { const source = [180, 60, 40, 255]; const bcFirst = imageData([...source]); - applyAdjustments(bcFirst, [bc(0.3, 0.5), hsl(-0.8)]); + applyAdjustments(bcFirst, [bc(0.3, 0.5), hsl(-0.8)], ORIGIN); const hslFirst = imageData([...source]); - applyAdjustments(hslFirst, [hsl(-0.8), bc(0.3, 0.5)]); + applyAdjustments(hslFirst, [hsl(-0.8), bc(0.3, 0.5)], ORIGIN); expect(Array.from(bcFirst.data)).not.toEqual(Array.from(hslFirst.data)); const withDisabled = imageData([...source]); - applyAdjustments(withDisabled, [bc(0.3, 0.5), hsl(-0.8, { isEnabled: false })]); + applyAdjustments(withDisabled, [bc(0.3, 0.5), hsl(-0.8, { isEnabled: false })], ORIGIN); const without = imageData([...source]); - applyAdjustments(without, [bc(0.3, 0.5)]); + applyAdjustments(without, [bc(0.3, 0.5)], ORIGIN); expect(Array.from(withDisabled.data)).toEqual(Array.from(without.data)); }); }); + +/** A `width`×`height` ImageData whose pixels `pixel(x, y)` fills. */ +const imageOf = (width: number, height: number, pixel: (x: number, y: number) => readonly number[]): ImageData => { + const data = new Uint8ClampedArray(width * height * 4); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + data.set(pixel(x, y), (y * width + x) * 4); + } + } + return { data, height, width } as ImageData; +}; +const midGray = (width: number, height: number): ImageData => imageOf(width, height, () => [128, 128, 128, 255]); +/** A busy source: every channel and alpha vary, including fully transparent pixels. */ +const varied = (width: number, height: number): ImageData => + imageOf(width, height, (x, y) => [ + (x * 37 + y * 11) % 256, + (x * 5 + y * 53) % 256, + (x * y) % 256, + (x + y) % 5 === 0 ? 0 : 255 - x, + ]); +const crop = (image: ImageData, x: number, y: number, width: number, height: number): ImageData => + imageOf(width, height, (cx, cy) => { + const i = ((y + cy) * image.width + x + cx) * 4; + return Array.from(image.data.subarray(i, i + 4)); + }); +/** The per-channel offsets noise added to a mid-gray image. */ +const deltas = (image: ImageData): { r: number[]; g: number[]; b: number[] } => { + const r: number[] = []; + const g: number[] = []; + const b: number[] = []; + for (let i = 0; i < image.data.length; i += 4) { + r.push(image.data[i]! - 128); + g.push(image.data[i + 1]! - 128); + b.push(image.data[i + 2]! - 128); + } + return { b, g, r }; +}; +const mean = (values: readonly number[]): number => values.reduce((sum, v) => sum + v, 0) / values.length; +const stddev = (values: readonly number[]): number => { + const m = mean(values); + return Math.sqrt(values.reduce((sum, v) => sum + (v - m) ** 2, 0) / values.length); +}; +const correlation = (a: readonly number[], b: readonly number[]): number => { + const ma = mean(a); + const mb = mean(b); + let cov = 0; + for (let i = 0; i < a.length; i++) { + cov += (a[i]! - ma) * (b[i]! - mb); + } + return cov / a.length / (stddev(a) * stddev(b)); +}; + +describe('noise', () => { + it("is identity at amount 0 and keys every parameter without changing other types' keys", () => { + expect(isIdentityAdjustmentEntry(noise(0))).toBe(true); + expect(isIdentityAdjustmentEntry(noise(0.01))).toBe(false); + expect(isIdentityAdjustments([noise(0.5, { isEnabled: false })])).toBe(true); + + const base = adjustmentsKey([noise(0.1)]); + const changes = [ + noise(0.2), + noise(0.1, { distribution: 'uniform' }), + noise(0.1, { grain: 'color' }), + noise(0.1, { grain: 'chroma' }), + noise(0.1, { response: 'midtones' }), + noise(0.1, { size: 2.5 }), + noise(0.1, { seed: 7 }), + ]; + expect(new Set([base, ...changes.map((changed) => adjustmentsKey([changed]))]).size).toBe(changes.length + 1); + expect(adjustmentsKey([noise(0.1, { id: 'other' })])).toBe(base); + // Display caches key on these strings; the noise arm must leave the other types' keys as they were. + expect(adjustmentsKey([bc(0.2, 0.1), exposure(1), invert(), hue(30), hsl(0.5)])).toBe( + 'bc:0.2,0.1||ex:1||inv||h:30||s:0.5' + ); + }); + + it('is its own segment: LUT neighbours fold on each side but never across it, keeping stack order', () => { + const stack = [bc(0.1, 0.2), invert(), noise(0.2, { grain: 'color' }), levels(10, 240, 1.3, 0, 255), invert()]; + expect(compileAdjustments(stack).map((segment) => segment.kind)).toEqual(['lut', 'noise', 'lut']); + + const sequential = varied(16, 8); + applyAdjustments(sequential, [bc(0.1, 0.2), invert()], ORIGIN); + applyAdjustments(sequential, [noise(0.2, { grain: 'color' })], ORIGIN); + applyAdjustments(sequential, [levels(10, 240, 1.3, 0, 255), invert()], ORIGIN); + const combined = varied(16, 8); + applyAdjustments(combined, stack, ORIGIN); + expect(Array.from(combined.data)).toEqual(Array.from(sequential.data)); + }); + + it('is region independent: a sub-rectangle adjusted at its own origin matches the whole-image pass', () => { + const stack = [bc(0.1, 0.3), noise(0.15, { grain: 'color' }), hsl(-0.4), noise(0.05, { seed: 99 })]; + const origin = { x: -37, y: 12 }; + const whole = varied(40, 30); + applyAdjustments(whole, stack, origin); + + const region = crop(varied(40, 30), 5, 7, 23, 9); + applyAdjustments(region, stack, { x: origin.x + 5, y: origin.y + 7 }); + expect(Array.from(region.data)).toEqual(Array.from(crop(whole, 5, 7, 23, 9).data)); + }); + + it.each( + GRAINS.flatMap((grain) => RESPONSES.flatMap((response) => [1, 2.5, 8].map((size) => ({ grain, response, size })))) + )( + 'is region independent for $grain grain, $response response, size $size, including regions thinner than the grain', + (options) => { + const stack = [bc(0.05, 0.2), noise(0.2, options), hsl(-0.3)]; + const origin = { x: -21, y: 9 }; + const whole = varied(48, 40); + applyAdjustments(whole, stack, origin); + // Sub-rectangles away from, touching and crossing the edges, down to a single row and column. + for (const [x, y, width, height] of [ + [5, 7, 23, 9], + [0, 0, 48, 1], + [47, 0, 1, 40], + [30, 31, 18, 9], + ] as const) { + const region = crop(varied(48, 40), x, y, width, height); + applyAdjustments(region, stack, { x: origin.x + x, y: origin.y + y }); + expect(Array.from(region.data)).toEqual(Array.from(crop(whole, x, y, width, height).data)); + } + } + ); + + it('keeps size 1 even grain bit-identical to the per-pixel kernel it replaced', () => { + // Pinned from the kernel before grain size, tonal response and chroma existed. + const pinned = { + 'color-gaussian': [ + 96, 126, 156, 255, 104, 118, 156, 255, 120, 130, 122, 255, 134, 118, 174, 255, 84, 126, 150, 255, 114, 111, 151, + 255, 118, 135, 150, 255, 155, 121, 142, 255, + ], + 'color-uniform': [ + 95, 126, 158, 255, 102, 116, 158, 255, 120, 131, 129, 255, 135, 116, 171, 255, 82, 125, 152, 255, 116, 110, 154, + 255, 118, 137, 153, 255, 151, 119, 143, 255, + ], + 'monochrome-gaussian': [ + 96, 124, 146, 255, 104, 122, 144, 255, 120, 128, 150, 255, 134, 132, 154, 255, 84, 112, 124, 255, 114, 132, 144, + 255, 118, 126, 138, 255, 155, 153, 165, 255, + ], + 'monochrome-uniform': [ + 95, 123, 145, 255, 102, 120, 142, 255, 120, 128, 150, 255, 135, 133, 155, 255, 82, 110, 122, 255, 116, 134, 146, + 255, 118, 126, 138, 255, 151, 149, 161, 255, + ], + }; + for (const grain of ['monochrome', 'color'] as const) { + for (const distribution of ['gaussian', 'uniform'] as const) { + const image = imageOf(4, 2, (x, y) => [100 + x * 10, 128, 150 - y * 10, 255]); + applyAdjustments(image, [noise(0.1, { distribution, grain })], { x: -3, y: 5 }); + expect(Array.from(image.data)).toEqual(pinned[`${grain}-${distribution}`]); + } + } + }); + + it('is deterministic per seed and changes pattern with the seed', () => { + const first = midGray(32, 32); + const again = midGray(32, 32); + const reseeded = midGray(32, 32); + applyAdjustments(first, [noise(0.1)], ORIGIN); + applyAdjustments(again, [noise(0.1)], ORIGIN); + applyAdjustments(reseeded, [noise(0.1, { seed: 0x1234abce })], ORIGIN); + expect(Array.from(again.data)).toEqual(Array.from(first.data)); + const changed = deltas(first).r.filter((d, i) => d !== deltas(reseeded).r[i]).length; + expect(changed).toBeGreaterThan(32 * 32 * 0.8); + }); + + it('shares one sample across channels when monochrome and samples each channel independently in color', () => { + const mono = midGray(64, 64); + applyAdjustments(mono, [noise(0.1)], ORIGIN); + const monoDeltas = deltas(mono); + expect(monoDeltas.g).toEqual(monoDeltas.r); + expect(monoDeltas.b).toEqual(monoDeltas.r); + + const color = midGray(64, 64); + applyAdjustments(color, [noise(0.1, { grain: 'color' })], ORIGIN); + const { b, g, r } = deltas(color); + expect(Math.abs(correlation(r, g))).toBeLessThan(0.05); + expect(Math.abs(correlation(r, b))).toBeLessThan(0.05); + expect(Math.abs(correlation(g, b))).toBeLessThan(0.05); + }); + + it.each(['gaussian', 'uniform'] as const)( + "%s noise has zero mean, the amount's standard deviation, and no neighbour correlation", + (distribution) => { + const size = 256; + const image = midGray(size, size); + applyAdjustments(image, [noise(0.1, { distribution, grain: 'color' })], { x: -100, y: 3 }); + const sigma = 0.1 * NOISE_FULL_SCALE_SIGMA; + for (const channel of Object.values(deltas(image))) { + // 65,536 samples: the mean's standard error is ~0.05 and the deviation's ~0.4%. + expect(Math.abs(mean(channel))).toBeLessThan(0.25); + expect(stddev(channel) / sigma).toBeGreaterThan(0.98); + expect(stddev(channel) / sigma).toBeLessThan(1.02); + const right = channel.filter((_, i) => i % size !== size - 1); + const rightNeighbour = channel.filter((_, i) => i % size !== 0); + expect(Math.abs(correlation(right, rightNeighbour))).toBeLessThan(0.02); + expect(Math.abs(correlation(channel.slice(0, -size), channel.slice(size)))).toBeLessThan(0.02); + } + const red = deltas(image).r; + const beyondUniformBound = red.filter((d) => Math.abs(d) > Math.round(sigma * Math.sqrt(3))).length; + if (distribution === 'uniform') { + // Uniform with the same deviation spans ±σ√3. + expect(beyondUniformBound).toBe(0); + } else { + // A gaussian puts ~7% of samples beyond ±1.73σ. + expect(beyondUniformBound / red.length).toBeGreaterThan(0.05); + } + } + ); + + it.each(GRAINS.flatMap((grain) => [1, 4].map((size) => ({ grain, size }))))( + 'changes RGB only for $grain grain at size $size: alpha is preserved and fully transparent pixels are untouched', + (options) => { + const source = varied(32, 16); + const image = varied(32, 16); + applyAdjustments(image, [noise(1, { ...options, response: 'midtones' })], ORIGIN); + let changed = 0; + for (let i = 0; i < image.data.length; i += 4) { + expect(image.data[i + 3]).toBe(source.data[i + 3]); + if (source.data[i + 3] === 0) { + expect(Array.from(image.data.subarray(i, i + 3))).toEqual(Array.from(source.data.subarray(i, i + 3))); + } else if (image.data[i] !== source.data[i]) { + changed += 1; + } + } + expect(changed).toBeGreaterThan(0); + } + ); +}); + +/** Rec. 709 luma's midtone weight 4·Y′·(1 − Y′) of a neutral grey, computed independently of the kernel. */ +const greyWeight = (grey: number): number => 4 * (grey / 255) * (1 - grey / 255); + +describe('noise tonal response', () => { + it.each(GRAINS.flatMap((grain) => [1, 3].map((size) => ({ grain, size }))))( + 'midtones leaves pure black and white exactly and scales $grain grain by the tone at size $size', + (options) => { + const size = 128; + const greys = [0, 64, 128, 255]; + const flat = imageOf(size, size * greys.length, (_, y) => { + const grey = greys[Math.floor(y / size)]!; + return [grey, grey, grey, 255]; + }); + const midtones = imageOf(flat.width, flat.height, (x, y) => + Array.from(flat.data.subarray((y * size + x) * 4, (y * size + x) * 4 + 4)) + ); + // Light enough that the 64-grey band never clips at black. + applyAdjustments(flat, [noise(0.05, options)], ORIGIN); + applyAdjustments(midtones, [noise(0.05, { ...options, response: 'midtones' })], ORIGIN); + const band = (image: ImageData, index: number) => crop(image, 0, index * size, size, size); + /** Deviation from the band's grey across all channels. */ + const spread = (image: ImageData, grey: number) => { + const values: number[] = []; + for (let i = 0; i < image.data.length; i += 4) { + values.push(image.data[i]! - grey, image.data[i + 1]! - grey, image.data[i + 2]! - grey); + } + return Math.sqrt(values.reduce((sum, v) => sum + v * v, 0) / values.length); + }; + + expect(Array.from(band(midtones, 0).data)).toEqual( + Array.from( + band( + imageOf(size, size, () => [0, 0, 0, 255]), + 0 + ).data + ) + ); + expect(Array.from(band(midtones, 3).data)).toEqual( + Array.from(imageOf(size, size, () => [255, 255, 255, 255]).data) + ); + // Even response grains black and white too (it clips half of the samples there, but not all). + expect(spread(band(flat, 0), 0)).toBeGreaterThan(0); + expect(spread(band(flat, 3), 255)).toBeGreaterThan(0); + for (const index of [1, 2]) { + const grey = greys[index]!; + const ratio = spread(band(midtones, index), grey) / spread(band(flat, index), grey); + // Rounding to whole codes blurs the ratio slightly; mid-grey is full strength, 64 about three quarters. + expect(ratio).toBeGreaterThan(greyWeight(grey) - 0.05); + expect(ratio).toBeLessThan(greyWeight(grey) + 0.05); + } + } + ); +}); + +describe('noise grain size', () => { + const field = (size: number, grain: 'monochrome' | 'color' = 'monochrome') => { + const side = 384; + const image = midGray(side, side); + applyAdjustments(image, [noise(0.1, { grain, size })], { x: 1000, y: -77 }); + return { channels: deltas(image), side }; + }; + const rightNeighbourCorrelation = (channel: readonly number[], side: number) => + correlation( + channel.filter((_, i) => i % side !== side - 1), + channel.filter((_, i) => i % side !== 0) + ); + const belowNeighbourCorrelation = (channel: readonly number[], side: number) => + correlation(channel.slice(0, -side), channel.slice(side)); + + it.each([2, 4, 8])("keeps the amount's standard deviation and zero mean at size %d", (size) => { + const { channels } = field(size, 'color'); + const sigma = 0.1 * NOISE_FULL_SCALE_SIGMA; + for (const channel of Object.values(channels)) { + // Larger grain means fewer independent samples: at size 8 the deviation's standard error is about 2.5%. + expect(stddev(channel) / sigma).toBeGreaterThan(0.9); + expect(stddev(channel) / sigma).toBeLessThan(1.1); + expect(Math.abs(mean(channel))).toBeLessThan(1); + } + expect(Math.abs(correlation(channels.r, channels.g))).toBeLessThan(0.1); + }); + + it('correlates neighbours more as the grain grows, and not at all at size 1', () => { + const correlations = [1, 2, 4, 8].map((size) => { + const { channels, side } = field(size); + return [rightNeighbourCorrelation(channels.r, side), belowNeighbourCorrelation(channels.r, side)] as const; + }); + expect(Math.abs(correlations[0]![0])).toBeLessThan(0.02); + expect(Math.abs(correlations[0]![1])).toBeLessThan(0.02); + // Size 2 blurs by σ √3 / 2: taps 1, e^(−2/3), e^(−8/3), e⁻⁶ correlate adjacent pixels by about 0.71. + expect(correlations[1]![0]).toBeGreaterThan(0.6); + expect(correlations[1]![1]).toBeGreaterThan(0.6); + for (let i = 2; i < correlations.length; i++) { + expect(correlations[i]![0]).toBeGreaterThan(correlations[i - 1]![0]); + expect(correlations[i]![1]).toBeGreaterThan(correlations[i - 1]![1]); + } + }); + + it('runs sizes within one slider step of 1 as size 1, and the next step already correlates neighbours', () => { + const one = midGray(64, 64); + const barelyLarger = midGray(64, 64); + applyAdjustments(one, [noise(0.1)], ORIGIN); + applyAdjustments(barelyLarger, [noise(0.1, { size: 1.1 })], ORIGIN); + expect(Array.from(barelyLarger.data)).toEqual(Array.from(one.data)); + + // Size 1.2 blurs by σ ≈ 0.33: an outer tap of e^(−4.5) ≈ 0.011 correlates neighbours by about 0.02. + const { channels, side } = field(1.2); + expect(rightNeighbourCorrelation(channels.r, side)).toBeGreaterThan(0.01); + expect(belowNeighbourCorrelation(channels.r, side)).toBeGreaterThan(0.01); + }); + + it('grain span equals the standard deviation box-averaging leaves, per block side', () => { + const { channels, side } = field(8); + const block = 32; + const averages: number[] = []; + for (let by = 0; by < side; by += block) { + for (let bx = 0; bx < side; bx += block) { + let sum = 0; + for (let y = by; y < by + block; y++) { + for (let x = bx; x < bx + block; x++) { + sum += channels.r[y * side + x]!; + } + } + averages.push(sum / (block * block)); + } + } + // 144 blocks: the measured ratio's standard error is about 6%. + const ratio = + Math.sqrt(averages.reduce((sum, v) => sum + v * v, 0) / averages.length) / (0.1 * NOISE_FULL_SCALE_SIGMA); + expect(ratio / (noiseGrainSpan(8) / block)).toBeGreaterThan(0.75); + expect(ratio / (noiseGrainSpan(8) / block)).toBeLessThan(1.25); + expect(noiseGrainSpan(1)).toBe(1); + }); +}); + +/** OKLab [L, a, b] of an sRGB pixel, computed with the exact transfer curve and cube root. */ +const oklab = (r: number, g: number, b: number): readonly [number, number, number] => { + const linear = (v: number) => (v <= 0.04045 * 255 ? v / 255 / 12.92 : ((v / 255 + 0.055) / 1.055) ** 2.4); + const [lr, lg, lb] = [linear(r), linear(g), linear(b)]; + const l = Math.cbrt(0.4122214708 * lr + 0.5363325363 * lg + 0.0514459929 * lb); + const m = Math.cbrt(0.2119034982 * lr + 0.6806995451 * lg + 0.1073969566 * lb); + const s = Math.cbrt(0.0883024619 * lr + 0.2817188376 * lg + 0.6299787005 * lb); + return [ + 0.2104542553 * l + 0.793617785 * m - 0.0040720468 * s, + 1.9779984951 * l - 2.428592205 * m + 0.4505937099 * s, + 0.0259040371 * l + 0.7827717662 * m - 0.808675766 * s, + ]; +}; +/** Each pixel's OKLab coordinates. */ +const oklabPixels = (image: ImageData): (readonly [number, number, number])[] => { + const pixels: (readonly [number, number, number])[] = []; + for (let i = 0; i < image.data.length; i += 4) { + pixels.push(oklab(image.data[i]!, image.data[i + 1]!, image.data[i + 2]!)); + } + return pixels; +}; + +describe('chroma grain', () => { + it.each([1, 3])('varies hue and colorfulness but keeps OKLab lightness at size %d', (size) => { + // Muted midtones, so 5% grain stays inside the sRGB gamut and nothing clips. + const fill = (x: number, y: number) => [100 + (x >> 1), 110 + (y >> 1), 140 - (x >> 1), 255]; + const source = oklabPixels(imageOf(96, 96, fill)); + const image = imageOf(96, 96, fill); + applyAdjustments(image, [noise(0.05, { grain: 'chroma', size })], { x: 5, y: 5 }); + const adjusted = oklabPixels(image); + const lightnessError = adjusted.map(([lightness], i) => Math.abs(lightness - source[i]![0])); + const chromaShift = adjusted.map(([, a, b], i) => Math.hypot(a - source[i]![1], b - source[i]![2])); + // Rounding each channel to a whole code moves lightness by at most about 0.003; the grain moves chroma by + // σ√2 ≈ 0.023 on average. + expect(Math.max(...lightnessError)).toBeLessThan(0.004); + expect(mean(chromaShift)).toBeGreaterThan(0.01); + }); + + it('leaves neutral grey neutral on average, with colorfulness comparable to color grain', () => { + const side = 256; + const chroma = midGray(side, side); + const color = midGray(side, side); + applyAdjustments(chroma, [noise(0.1, { grain: 'chroma' })], ORIGIN); + applyAdjustments(color, [noise(0.1, { grain: 'color' })], ORIGIN); + const chromaLab = oklabPixels(chroma); + const colorLab = oklabPixels(color); + for (const axis of [1, 2] as const) { + const chromaAxis = chromaLab.map((pixel) => pixel[axis]); + // 65,536 samples of σ ≈ 0.032: the mean's standard error is about 0.0001; gamut clipping adds a little bias. + expect(Math.abs(mean(chromaAxis))).toBeLessThan(0.002); + // Chroma puts color grain's lightness spread onto a and b too, so it is somewhat more colorful. + const ratio = stddev(chromaAxis) / stddev(colorLab.map((pixel) => pixel[axis])); + expect(ratio).toBeGreaterThan(1); + expect(ratio).toBeLessThan(1.6); + } + }); +}); + +describe('isPositionDependentAdjustments', () => { + it('is true only while a noise entry contributes', () => { + expect(isPositionDependentAdjustments(undefined)).toBe(false); + expect(isPositionDependentAdjustments([bc(0.2, 0), invert(), hue(30)])).toBe(false); + expect(isPositionDependentAdjustments([bc(0.2, 0), noise(0.1)])).toBe(true); + expect(isPositionDependentAdjustments([noise(0)])).toBe(false); + expect(isPositionDependentAdjustments([noise(0.1, { isEnabled: false })])).toBe(false); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.ts index a5a9c9bb938..8466504145e 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/adjustments.ts @@ -136,6 +136,8 @@ export const isIdentityAdjustmentEntry = (entry: CanvasAdjustmentEntry): boolean return entry.rotation % 360 === 0; case 'invert': return false; + case 'noise': + return entry.amount === 0; case 'curves': return isIdentityCurve(entry.curves.r) && isIdentityCurve(entry.curves.g) && isIdentityCurve(entry.curves.b); } @@ -147,10 +149,18 @@ const contributes = (entry: CanvasAdjustmentEntry): boolean => entry.isEnabled & export const isIdentityAdjustments = (adjustments: CanvasAdjustmentsContract | undefined): boolean => !adjustments || adjustments.every((entry) => !contributes(entry)); +/** + * True when the stack's output depends on pixel position (contributing noise). Such a layer stack must be applied to + * layer-local pixels before resampling on every path, so resampling shapes its grain exactly as the display does. + */ +export const isPositionDependentAdjustments = (adjustments: CanvasAdjustmentsContract | undefined): boolean => + adjustments?.some((entry) => entry.type === 'noise' && contributes(entry)) ?? false; + /** * One step of the compiled pixel pass. Per-channel entries fold into LUT * segments (adjacent ones compose into ONE lut); saturation and hue are - * cross-channel and stand alone. + * cross-channel and stand alone. Noise depends on pixel position, so it is its + * own pass and LUTs never fold across it. */ export type CompiledAdjustmentSegment = | { @@ -160,7 +170,152 @@ export type CompiledAdjustmentSegment = readonly b: Uint8ClampedArray; } | { readonly kind: 'saturation'; readonly factor: number } - | { readonly kind: 'matrix'; readonly m: readonly number[] }; + | { readonly kind: 'matrix'; readonly m: readonly number[] } + | { + readonly kind: 'noise'; + readonly grain: NoiseEntry['grain']; + readonly midtones: boolean; + /** Each of the table's equiprobable quantiles scaled to the entry's σ: channel units, or OKLab a/b for chroma. */ + readonly samples: Float64Array; + /** The grain kernel's taps 0…R (see `grainKernel`); null at size 1, where every pixel is its own sample. */ + readonly kernel: Float64Array | null; + readonly seed: number; + }; + +type NoiseEntry = Extract; + +/** Standard deviation at `amount` 1, in 0–255 channel units (see the noise entry's contract). */ +export const NOISE_FULL_SCALE_SIGMA = 128; +/** Chroma grain's OKLab a and b standard deviation at `amount` 1 (see the noise entry's contract). */ +export const NOISE_CHROMA_FULL_SCALE_SIGMA = 0.32; + +const NOISE_TABLE_BITS = 10; +const NOISE_TABLE_SIZE = 1 << NOISE_TABLE_BITS; +const NOISE_TABLE_MASK = NOISE_TABLE_SIZE - 1; + +/** Acklam's approximation of the standard normal inverse CDF for `p` in (0, 0.5] (relative error < 1.2e-9). */ +const inverseNormalCdf = (p: number): number => { + const c0 = -7.784894002430293e-3; + const c1 = -3.223964580411365e-1; + const c2 = -2.400758277161838; + const c3 = -2.549732539343734; + const c4 = 4.374664141464968; + const c5 = 2.938163982698783; + const d0 = 7.784695709041462e-3; + const d1 = 3.224671290700398e-1; + const d2 = 2.445134137142996; + const d3 = 3.754408661907416; + if (p < 0.02425) { + const q = Math.sqrt(-2 * Math.log(p)); + return (((((c0 * q + c1) * q + c2) * q + c3) * q + c4) * q + c5) / ((((d0 * q + d1) * q + d2) * q + d3) * q + 1); + } + const q = p - 0.5; + const r = q * q; + const numerator = + (((((-3.969683028665376e1 * r + 2.209460984245205e2) * r - 2.759285104469687e2) * r + 1.38357751867269e2) * r - + 3.066479806614716e1) * + r + + 2.506628277459239) * + q; + const denominator = + ((((-5.447609879822406e1 * r + 1.615858368580409e2) * r - 1.556989798598866e2) * r + 6.680131188771972e1) * r - + 1.328068155288572e1) * + r + + 1; + return numerator / denominator; +}; + +/** + * `NOISE_TABLE_SIZE` equiprobable quantiles of a zero-mean, unit-variance distribution. A uniform table index then + * samples the distribution with no per-pixel `log`/`sqrt`. Mirrored so the mean is exactly zero, and rescaled so + * the discrete variance is exactly one; uniform spans ±√3 to match gaussian's standard deviation. + */ +const buildUnitNoiseTable = (quantile: (p: number) => number): Float64Array => { + const table = new Float64Array(NOISE_TABLE_SIZE); + let sumSquares = 0; + for (let i = 0; i < NOISE_TABLE_SIZE / 2; i++) { + const value = quantile((i + 0.5) / NOISE_TABLE_SIZE); + table[i] = value; + table[NOISE_TABLE_SIZE - 1 - i] = -value; + sumSquares += 2 * value * value; + } + const scale = 1 / Math.sqrt(sumSquares / NOISE_TABLE_SIZE); + for (let i = 0; i < NOISE_TABLE_SIZE; i++) { + table[i] *= scale; + } + return table; +}; + +// Built on first use: module scope stays free of evaluation work. +const unitNoiseTables: Partial> = {}; +const unitNoiseTable = (distribution: 'gaussian' | 'uniform'): Float64Array => + (unitNoiseTables[distribution] ??= buildUnitNoiseTable( + distribution === 'gaussian' ? inverseNormalCdf : (p) => Math.sqrt(3) * (2 * p - 1) + )); + +/** Below this, a kernel's outer taps change no 8-bit pixel visibly, so it runs as white noise at a third of the cost. */ +const GRAIN_MIN_OUTER_TAP = 1e-3; + +/** + * Taps 0…R of the symmetric, separable grain kernel for `size`: a Gaussian of σ `√(size² − 1) / 2` pixels truncated + * at 3σ, scaled so the full kernel's squared taps sum to one. Blurring unit-variance white noise along both axes then + * keeps unit variance exactly, so size never changes strength. The σ makes a grain about `size` pixels across: a + * pixel itself spans one, and blurring adds 4σ² to its squared width. Null at size 1 and wherever the first outer tap + * would stay under `GRAIN_MIN_OUTER_TAP` (sizes under about 1.14, within one 0.1 slider step of 1). + */ +const grainKernel = (size: number): Float64Array | null => { + if (size <= 1) { + return null; + } + const sigma = Math.sqrt(size * size - 1) / 2; + if (Math.exp(-1 / (2 * sigma * sigma)) < GRAIN_MIN_OUTER_TAP) { + return null; + } + const taps = new Float64Array(Math.ceil(3 * sigma) + 1); + let sumSquares = 0; + for (let j = 0; j < taps.length; j++) { + taps[j] = Math.exp((-j * j) / (2 * sigma * sigma)); + sumSquares += (j === 0 ? 1 : 2) * taps[j] * taps[j]; + } + const scale = 1 / Math.sqrt(sumSquares); + for (let j = 0; j < taps.length; j++) { + taps[j] *= scale; + } + return taps; +}; + +/** + * The grain's linear span in pixels, `(Σ taps)²` of its normalized kernel (1 at size 1). Box-averaging the grain over + * B×B pixels, with B much larger than the grain, leaves its standard deviation multiplied by `span / B`. + */ +export const noiseGrainSpan = (size: number): number => { + const kernel = grainKernel(size); + if (!kernel) { + return 1; + } + let sum = kernel[0]; + for (let j = 1; j < kernel.length; j++) { + sum += 2 * kernel[j]; + } + return sum * sum; +}; + +const noiseSegment = (entry: NoiseEntry): CompiledAdjustmentSegment => { + const unit = unitNoiseTable(entry.distribution); + const sigma = entry.amount * (entry.grain === 'chroma' ? NOISE_CHROMA_FULL_SCALE_SIGMA : NOISE_FULL_SCALE_SIGMA); + const samples = new Float64Array(NOISE_TABLE_SIZE); + for (let i = 0; i < NOISE_TABLE_SIZE; i++) { + samples[i] = sigma * unit[i]; + } + return { + grain: entry.grain, + kernel: grainKernel(entry.size), + kind: 'noise', + midtones: entry.response === 'midtones', + samples, + seed: entry.seed >>> 0, + }; +}; const brightnessContrastLut = (brightness: number, contrast: number): Uint8ClampedArray => { const offset = brightness * 255; @@ -285,6 +440,10 @@ export const compileAdjustments = (adjustments: CanvasAdjustmentsContract | unde segments.push({ kind: 'matrix', m: hueRotateMatrix(entry.rotation) }); continue; } + if (entry.type === 'noise') { + segments.push(noiseSegment(entry)); + continue; + } const luts = entryLuts(entry); const last = segments[segments.length - 1]; if (last?.kind === 'lut') { @@ -325,6 +484,8 @@ export const adjustmentsKey = (adjustments: CanvasAdjustmentsContract | undefine return `h:${entry.rotation}`; case 'invert': return 'inv'; + case 'noise': + return `nz:${entry.distribution},${entry.grain},${entry.response},${entry.size},${entry.amount},${entry.seed}`; case 'curves': return `cv:${curveKey(entry.curves.r)}|${curveKey(entry.curves.g)}|${curveKey(entry.curves.b)}`; } @@ -332,13 +493,14 @@ export const adjustmentsKey = (adjustments: CanvasAdjustmentsContract | undefine .join('||'); }; -/** Applies compiled LUT/saturation segments in one in-place pixel pass, preserving alpha. Identity is a no-op. */ -export const applyAdjustments = (imageData: ImageData, adjustments: CanvasAdjustmentsContract | undefined): void => { - const segments = compileAdjustments(adjustments); - if (segments.length === 0) { - return; - } - const { data } = imageData; +/** Integer pixel coordinate of an ImageData's top-left pixel in its adjustment stack's reference space. */ +export interface AdjustmentOrigin { + readonly x: number; + readonly y: number; +} + +/** One pixel-major pass over consecutive position-independent segments. */ +const applyColorSegments = (data: Uint8ClampedArray, segments: readonly CompiledAdjustmentSegment[]): void => { for (let i = 0; i + 3 < data.length; i += 4) { let r = data[i]; let g = data[i + 1]; @@ -353,7 +515,7 @@ export const applyAdjustments = (imageData: ImageData, adjustments: CanvasAdjust r = clamp255(Math.round(lum + (r - lum) * segment.factor)); g = clamp255(Math.round(lum + (g - lum) * segment.factor)); b = clamp255(Math.round(lum + (b - lum) * segment.factor)); - } else { + } else if (segment.kind === 'matrix') { const { m } = segment; const nr = m[0] * r + m[1] * g + m[2] * b; const ng = m[3] * r + m[4] * g + m[5] * b; @@ -368,3 +530,394 @@ export const applyAdjustments = (imageData: ImageData, adjustments: CanvasAdjust data[i + 2] = b; } }; + +/** Chris Wellons' `lowbias32`: a bijective, well-avalanched 32-bit integer finalizer. */ +const hash32 = (value: number): number => { + let h = value ^ (value >>> 16); + h = Math.imul(h, 0x7feb352d); + h ^= h >>> 15; + h = Math.imul(h, 0x846ca68b); + return h ^ (h >>> 16); +}; + +const NOISE_ROW_STRIDE = 0x9e3779b9; +const NOISE_STEP_SALT = 0x632be5ab; + +/** + * Row `y`'s Weyl sequence: a start hashed from its coordinate and an odd step hashed from the start. A per-row step + * keeps rows from being shifted copies of each other; pixel `x` hashes `start + x × step`. + */ +const noiseRowStart = (seedKey: number, y: number): number => hash32((seedKey + Math.imul(y, NOISE_ROW_STRIDE)) | 0); +const noiseRowStep = (rowStart: number): number => hash32(rowStart ^ NOISE_STEP_SALT) | 1; + +/** + * Fills `out` with white table samples keyed by `(seed, x, y)` for `count` pixels of row `y` from column `x0`. One + * hash feeds every channel from disjoint 10-bit fields. + */ +const fillWhiteRow = ( + out: readonly Float64Array[], + samples: Float64Array, + seedKey: number, + x0: number, + y: number, + count: number +): void => { + const rowStart = noiseRowStart(seedKey, y); + const step = noiseRowStep(rowStart); + let key = (rowStart + Math.imul(x0, step)) | 0; + const [c0, c1, c2] = out; + if (c2) { + for (let x = 0; x < count; x++, key = (key + step) | 0) { + const h = hash32(key); + c0[x] = samples[h & NOISE_TABLE_MASK]; + c1[x] = samples[(h >>> NOISE_TABLE_BITS) & NOISE_TABLE_MASK]; + c2[x] = samples[(h >>> (2 * NOISE_TABLE_BITS)) & NOISE_TABLE_MASK]; + } + } else if (c1) { + for (let x = 0; x < count; x++, key = (key + step) | 0) { + const h = hash32(key); + c0[x] = samples[h & NOISE_TABLE_MASK]; + c1[x] = samples[(h >>> NOISE_TABLE_BITS) & NOISE_TABLE_MASK]; + } + } else { + for (let x = 0; x < count; x++, key = (key + step) | 0) { + c0[x] = samples[hash32(key) & NOISE_TABLE_MASK]; + } + } +}; + +/** + * Streams the region's noise rows top to bottom, `emit(row, noise)` receiving each row's per-channel values. At size 1 + * they are the white samples; above it, the separable kernel blurs the white field, horizontally into a ring of 2R + 1 + * rows and then vertically. White samples past the region's edges come from the same hash, never from pixels, and + * every output sums the same samples in the same order wherever the region starts, so the field stays a pure + * function of the coordinate. Memory is O(R × width), never a padded copy of the region. + */ +const streamNoiseRows = ( + width: number, + height: number, + originX: number, + originY: number, + segment: Extract, + channels: number, + emit: (row: number, noise: readonly Float64Array[]) => void +): void => { + const { kernel, samples } = segment; + const seedKey = hash32(segment.seed); + const noise = Array.from({ length: channels }, () => new Float64Array(width)); + if (!kernel) { + for (let row = 0; row < height; row++) { + fillWhiteRow(noise, samples, seedKey, originX, originY + row, width); + emit(row, noise); + } + return; + } + const radius = kernel.length - 1; + const span = 2 * radius + 1; + const center = kernel[0]; + const white = Array.from({ length: channels }, () => new Float64Array(width + 2 * radius)); + const ring = Array.from({ length: channels }, () => new Float64Array(span * width)); + for (let row = 0; row < height + 2 * radius; row++) { + fillWhiteRow(white, samples, seedKey, originX - radius, originY - radius + row, width + 2 * radius); + const slot = (row % span) * width; + for (let c = 0; c < channels; c++) { + const source = white[c]; + const target = ring[c]; + for (let x = 0; x < width; x++) { + target[slot + x] = center * source[x + radius]; + } + for (let j = 1; j <= radius; j++) { + const tap = kernel[j]; + for (let x = 0, left = radius - j, right = radius + j; x < width; x++, left++, right++) { + target[slot + x] += tap * (source[left] + source[right]); + } + } + } + if (row < 2 * radius) { + continue; + } + // The ring now holds the 2R + 1 filtered rows around output row `row − 2R`. + const middle = ((row - radius) % span) * width; + for (let c = 0; c < channels; c++) { + const source = ring[c]; + const target = noise[c]; + for (let x = 0; x < width; x++) { + target[x] = center * source[middle + x]; + } + for (let k = 1; k <= radius; k++) { + const tap = kernel[k]; + const above = ((row - radius - k) % span) * width; + const below = ((row - radius + k) % span) * width; + for (let x = 0; x < width; x++) { + target[x] += tap * (source[above + x] + source[below + x]); + } + } + } + emit(row - 2 * radius, noise); + } +}; + +/** 255 × 10000: Rec. 709 luma weights in ten-thousandths summed over 0–255 channels, so white sums exactly to it. */ +const LUMA_SCALE = 2_550_000; +const MIDTONE_NORM = 4 / (LUMA_SCALE * LUMA_SCALE); + +/** 4·Y′·(1 − Y′) of a pixel's Rec. 709 luma Y′: 1 at mid-grey, exactly 0 at black and white (integer luma sum). */ +const midtoneWeight = (r: number, g: number, b: number): number => { + const luma = 2126 * r + 7152 * g + 722 * b; + return luma * (LUMA_SCALE - luma) * MIDTONE_NORM; +}; + +/** Linear-light encoding table resolution: dark codes sit about five entries apart, so rounding stays exact or ±1. */ +const LINEAR_TABLE_MAX = 16383; + +interface ChromaTables { + /** sRGB code → linear light. */ + readonly toLinear: Float64Array; + /** Linear light × `LINEAR_TABLE_MAX`, rounded → sRGB code. */ + readonly toSrgb: Uint8Array; + /** One float32 and its bits, the cube root's scratch. */ + readonly bits: Float32Array; + readonly word: Uint32Array; +} + +// Built on first use: module scope stays free of evaluation work. +let chromaTables: ChromaTables | null = null; +const getChromaTables = (): ChromaTables => { + if (!chromaTables) { + const toLinear = new Float64Array(LUT_SIZE); + for (let i = 0; i < LUT_SIZE; i++) { + toLinear[i] = srgbToLinear(i / 255); + } + const toSrgb = new Uint8Array(LINEAR_TABLE_MAX + 1); + for (let i = 0; i <= LINEAR_TABLE_MAX; i++) { + toSrgb[i] = Math.round(linearToSrgb(i / LINEAR_TABLE_MAX) * 255); + } + const bits = new Float32Array(1); + chromaTables = { bits, toLinear, toSrgb, word: new Uint32Array(bits.buffer) }; + } + return chromaTables; +}; + +const encodeSrgb = (toSrgb: Uint8Array, linear: number): number => + linear <= 0 ? 0 : linear >= 1 ? 255 : toSrgb[(linear * LINEAR_TABLE_MAX + 0.5) | 0]; + +/** + * Cube root of a non-negative value: the float32 exponent-divide guess (within ~4%) and one Halley step, relative + * error < 3·10⁻⁵ (far below one 8-bit code) at about a third of `Math.cbrt`'s cost. + */ +const fastCbrt = (value: number, bits: Float32Array, word: Uint32Array): number => { + bits[0] = value; + word[0] = ((word[0] / 3) | 0) + 709_921_077; + const guess = bits[0]; + const cube = guess * guess * guess; + return (guess * (cube + 2 * value)) / (2 * cube + value); +}; + +/** + * Adds OKLab a/b noise to one row, keeping OKLab L. Adding (0, δa, δb) in OKLab adds M₂⁻¹·(0, δa, δb) to the + * cube-rooted LMS, so L, a and b are never materialized. Out-of-gamut results clamp in linear light. + */ +const applyChromaRow = ( + data: Uint8ClampedArray, + start: number, + width: number, + noiseA: Float64Array, + noiseB: Float64Array, + midtones: boolean, + tables: ChromaTables +): void => { + const { bits, toLinear, toSrgb, word } = tables; + for (let x = 0, i = start; x < width; x++, i += 4) { + if (data[i + 3] === 0) { + continue; + } + const r = data[i]; + const g = data[i + 1]; + const b = data[i + 2]; + const weight = midtones ? midtoneWeight(r, g, b) : 1; + if (weight === 0) { + continue; + } + const da = noiseA[x] * weight; + const db = noiseB[x] * weight; + const lr = toLinear[r]; + const lg = toLinear[g]; + const lb = toLinear[b]; + const l = + fastCbrt(0.4122214708 * lr + 0.5363325363 * lg + 0.0514459929 * lb, bits, word) + + 0.3963377774 * da + + 0.2158037573 * db; + const m = + fastCbrt(0.2119034982 * lr + 0.6806995451 * lg + 0.1073969566 * lb, bits, word) - + 0.1055613458 * da - + 0.0638541728 * db; + const s = + fastCbrt(0.0883024619 * lr + 0.2817188376 * lg + 0.6299787005 * lb, bits, word) - + 0.0894841775 * da - + 1.291485548 * db; + const l3 = l * l * l; + const m3 = m * m * m; + const s3 = s * s * s; + data[i] = encodeSrgb(toSrgb, 4.0767416621 * l3 - 3.3077115913 * m3 + 0.2309699292 * s3); + data[i + 1] = encodeSrgb(toSrgb, -1.2684380046 * l3 + 2.6097574011 * m3 - 0.3413193965 * s3); + data[i + 2] = encodeSrgb(toSrgb, -0.0041960863 * l3 - 0.7034186147 * m3 + 1.707614701 * s3); + } +}; + +/** Adds monochrome or color noise to one row; fractional values round in the clamped store. */ +const applyRgbRow = ( + data: Uint8ClampedArray, + start: number, + width: number, + noise: readonly Float64Array[], + midtones: boolean +): void => { + const [nr, ng = nr, nb = nr] = noise; + for (let x = 0, i = start; x < width; x++, i += 4) { + if (data[i + 3] === 0) { + continue; + } + const r = data[i]; + const g = data[i + 1]; + const b = data[i + 2]; + // One luma weight for all channels keeps neutral pixels neutral on average. + const weight = midtones ? midtoneWeight(r, g, b) : 1; + data[i] = r + nr[x] * weight; + data[i + 1] = g + ng[x] * weight; + data[i + 2] = b + nb[x] * weight; + } +}; + +/** + * Size 1 monochrome or color grain straight from the hash, the most common case. Flat response adds integer offsets; + * midtones scales the samples by the pixel's weight and rounds in the clamped store. Monochrome reuses the first + * 10-bit field for every channel. + */ +const applyPixelRgbNoise = ( + data: Uint8ClampedArray, + width: number, + height: number, + originX: number, + originY: number, + segment: Extract +): void => { + const { midtones, samples } = segment; + const monochrome = segment.grain === 'monochrome'; + // Pixels are integers, so rounding a flat offset equals rounding the noisy value; midtones never reads these. + const offsets = midtones ? null : Int16Array.from(samples, Math.round); + const seedKey = hash32(segment.seed); + for (let row = 0, i = 0; row < height; row++) { + const rowStart = noiseRowStart(seedKey, originY + row); + const step = noiseRowStep(rowStart); + let key = (rowStart + Math.imul(originX, step)) | 0; + const rowEnd = i + width * 4; + if (midtones && monochrome) { + for (; i < rowEnd; i += 4, key = (key + step) | 0) { + if (data[i + 3] === 0) { + continue; + } + const r = data[i]; + const g = data[i + 1]; + const b = data[i + 2]; + const offset = samples[hash32(key) & NOISE_TABLE_MASK] * midtoneWeight(r, g, b); + data[i] = r + offset; + data[i + 1] = g + offset; + data[i + 2] = b + offset; + } + } else if (midtones) { + for (; i < rowEnd; i += 4, key = (key + step) | 0) { + if (data[i + 3] === 0) { + continue; + } + const r = data[i]; + const g = data[i + 1]; + const b = data[i + 2]; + // One luma weight for all channels keeps neutral pixels neutral on average. + const weight = midtoneWeight(r, g, b); + const h = hash32(key); + data[i] = r + samples[h & NOISE_TABLE_MASK] * weight; + data[i + 1] = g + samples[(h >>> NOISE_TABLE_BITS) & NOISE_TABLE_MASK] * weight; + data[i + 2] = b + samples[(h >>> (2 * NOISE_TABLE_BITS)) & NOISE_TABLE_MASK] * weight; + } + } else if (monochrome) { + for (; i < rowEnd; i += 4, key = (key + step) | 0) { + if (data[i + 3] === 0) { + continue; + } + const offset = offsets![hash32(key) & NOISE_TABLE_MASK]; + // Uint8ClampedArray stores clamp to 0–255. + data[i] += offset; + data[i + 1] += offset; + data[i + 2] += offset; + } + } else { + for (; i < rowEnd; i += 4, key = (key + step) | 0) { + if (data[i + 3] === 0) { + continue; + } + const h = hash32(key); + data[i] += offsets![h & NOISE_TABLE_MASK]; + data[i + 1] += offsets![(h >>> NOISE_TABLE_BITS) & NOISE_TABLE_MASK]; + data[i + 2] += offsets![(h >>> (2 * NOISE_TABLE_BITS)) & NOISE_TABLE_MASK]; + } + } + } +}; + +/** Adds the segment's grain keyed by `(seed, x, y)` in reference space; alpha and fully transparent pixels stay. */ +const applyNoise = ( + data: Uint8ClampedArray, + width: number, + height: number, + origin: AdjustmentOrigin, + segment: Extract +): void => { + const originX = Math.round(origin.x); + const originY = Math.round(origin.y); + const { grain, midtones } = segment; + if (grain !== 'chroma' && !segment.kernel) { + applyPixelRgbNoise(data, width, height, originX, originY, segment); + return; + } + if (grain === 'chroma') { + const tables = getChromaTables(); + streamNoiseRows(width, height, originX, originY, segment, 2, (row, noise) => + applyChromaRow(data, row * width * 4, width, noise[0], noise[1], midtones, tables) + ); + return; + } + streamNoiseRows(width, height, originX, originY, segment, grain === 'color' ? 3 : 1, (row, noise) => + applyRgbRow(data, row * width * 4, width, noise, midtones) + ); +}; + +/** + * Applies the compiled stack in place, preserving alpha; identity is a no-op. `origin` places the pixels in the + * stack's reference space. Every entry is region independent: adjusting a sub-rectangle with its matching origin + * yields exactly the pixels a whole-surface pass would there. + */ +export const applyAdjustments = ( + imageData: ImageData, + adjustments: CanvasAdjustmentsContract | undefined, + origin: AdjustmentOrigin +): void => { + const segments = compileAdjustments(adjustments); + if (segments.length === 0) { + return; + } + const { data, height, width } = imageData; + let runStart = 0; + for (let index = 0; index <= segments.length; index++) { + const segment = segments[index]; + if (segment && segment.kind !== 'noise') { + continue; + } + if (index > runStart) { + applyColorSegments(data, runStart === 0 && !segment ? segments : segments.slice(runStart, index)); + } + if (segment) { + applyNoise(data, width, height, origin, segment); + } + runStart = index + 1; + } +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/derivedSurfaceCache.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/derivedSurfaceCache.ts index 93fe95bd741..2ee856fd65b 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/derivedSurfaceCache.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/derivedSurfaceCache.ts @@ -19,6 +19,8 @@ export interface DerivedSurfaceRequest { export interface DerivedSurfaceCache { get(request: DerivedSurfaceRequest): RasterSurface; + /** The memoized surface `request` would reuse without work, or null; never builds and never touches LRU order. */ + peek(request: Omit): RasterSurface | null; delete(layerId: string, kind: DerivedSurfaceKind): void; deleteLayer(layerId: string): void; byteSize(): number; @@ -66,16 +68,17 @@ export const createDerivedSurfaceCache = ( adjustBytes(-entry.bytes); }; + const isCurrent = (existing: CacheEntry | undefined, request: Omit): boolean => + existing !== undefined && + existing.source === request.source && + existing.sourceVersion === request.sourceVersion && + existing.paramsKey === request.paramsKey; + const get = (request: DerivedSurfaceRequest): RasterSurface => { const key = entryKey(request.layerId, request.kind); const existing = entries.get(key); tick += 1; - if ( - existing && - existing.source === request.source && - existing.sourceVersion === request.sourceVersion && - existing.paramsKey === request.paramsKey - ) { + if (existing && isCurrent(existing, request)) { diagnostics?.increment('derivedCacheHits'); existing.lastUsed = tick; return existing.surface; @@ -145,6 +148,10 @@ export const createDerivedSurfaceCache = ( return evicted; }, get, + peek: (request) => { + const existing = entries.get(entryKey(request.layerId, request.kind)); + return existing && isCurrent(existing, request) ? existing.surface : null; + }, size: () => entries.size, tick: () => tick, }; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupAdjustmentComposite.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupAdjustmentComposite.browser.test.ts index 69e6f1a6539..c8554fb6145 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupAdjustmentComposite.browser.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupAdjustmentComposite.browser.test.ts @@ -5,13 +5,18 @@ import type { Rect } from '@workbench/canvas-engine/types'; import { createCanvasDiagnostics } from '@workbench/canvas-engine/diagnostics'; import { groupContract, stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { identity } from '@workbench/canvas-engine/math/mat2d'; +import { roundOut } from '@workbench/canvas-engine/math/rect'; import { createAdjustedSurfaceCache } from '@workbench/canvas-engine/render/adjustedSurfaceCache'; import { createColorSampler } from '@workbench/canvas-engine/render/colorSample'; import { compositeDocument } from '@workbench/canvas-engine/render/compositor'; import { createGroupSurfaceCache } from '@workbench/canvas-engine/render/groupSurfaceCache'; import { createLayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; import { createDomRasterBackend } from '@workbench/canvas-engine/render/raster'; -import { planBaseRasterComposite, renderRasterComposite } from '@workbench/canvas-engine/render/rasterComposite'; +import { + getCompositeLayerBounds, + planBaseRasterComposite, + renderRasterComposite, +} from '@workbench/canvas-engine/render/rasterComposite'; import { describe, expect, it } from 'vitest'; const WIDTH = 64; @@ -37,6 +42,27 @@ const gammaStack = (id: string): CanvasAdjustmentsContract => [ { gamma: 2, id, inBlack: 0, inWhite: 255, isEnabled: true, outBlack: 0, outWhite: 255, type: 'levels' }, ]; const invertStack = (id: string): CanvasAdjustmentsContract => [{ id, isEnabled: true, type: 'invert' }]; +/** Color grain: position dependent, so any misplaced origin shows as a different pattern. */ +const noiseStack = (id: string): CanvasAdjustmentsContract => [ + { + amount: 0.2, + distribution: 'gaussian', + id, + isEnabled: true, + grain: 'color', + response: 'flat', + size: 1, + seed: 4242, + type: 'noise', + }, +]; +const worstDifference = (a: Uint8ClampedArray, b: Uint8ClampedArray): number => { + let worst = 0; + for (let i = 0; i < a.length; i += 1) { + worst = Math.max(worst, Math.abs(a[i]! - b[i]!)); + } + return worst; +}; const sceneFor = (fills: Record) => { const backend = createDomRasterBackend(); @@ -145,6 +171,32 @@ describe('group adjustment composite', () => { expect(centerPixel(again)).toEqual(screenPx); }); + it('places group noise in document pixels identically on screen and in export', async () => { + // The member's translation moves the group surface off the document origin, so its slot rect matters. + const document = docWith([ + groupContract('g', [raster('gray', { transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 7, y: 5 } })], { + adjustments: noiseStack('gn'), + }), + ]); + const scene = sceneFor({ gray: '#808080' }); + const groupSurfaces = createGroupSurfaceCache({ + createSurface: (w, h) => scene.backend.createSurface(w, h), + damageSince: (id, version) => scene.caches.damageSince(id, version), + getAdjustedSurface: () => null, + getCacheEntry: (id) => scene.caches.get(id), + }); + const screen = scene.backend.createSurface(WIDTH, HEIGHT); + compositeDocument(screen, document, scene.caches, identity(), { + backend: scene.backend, + groupSurface: (scope, members, matrices, content) => groupSurfaces.get(scope, members, matrices, content), + }); + const exported = await renderRasterComposite(planBaseRasterComposite(document, BBOX), scene); + + const screenPixels = screen.ctx.getImageData(0, 0, WIDTH, HEIGHT).data; + expect(new Set(screenPixels.filter((_, i) => i % 4 === 0)).size).toBeGreaterThan(20); + expect(worstDifference(screenPixels, exported.ctx.getImageData(0, 0, WIDTH, HEIGHT).data)).toBeLessThanOrEqual(1); + }); + it('resolves sibling scopes nested inside a parent scope with correct member indexing', async () => { // Outer gamma-2 maps inverted `a` from 64 to about 221 and plain `b` to about 128; `d` does not cover center. const document = docWith([ @@ -474,6 +526,7 @@ describe('group-scoped previews and partial refresh', () => { ], ['an inverted group', { group: { adjustments: invertStack('ga'), opacity: 0.5 } }], ['a group with levels', { group: { adjustments: gammaStack('ga') } }], + ['a group with noise', { group: { adjustments: noiseStack('gn') } }], ['a member inside a nested group', { nested: true }], ])('refreshes a damaged region of %s to the same pixels as a full rebuild', (_, options) => { const refreshed = groupedScene(options); @@ -577,7 +630,7 @@ describe('layer adjustments in export', () => { ]); }); - it('adjusts a downscaled layer after resampling, so the work never exceeds the output', async () => { + it('adjusts a downscaled color-only layer after resampling, so the work never exceeds the output', async () => { const { scene } = adjustedScene(); const shrunk = docWith([ raster('graded', { @@ -604,6 +657,43 @@ describe('layer adjustments in export', () => { ]); }); + // Each output samples more layer pixels than it holds, where a color-only stack would adjust the output instead. + it.each<[string, CanvasRasterLayerContractV2['transform'], Rect | null]>([ + ['downscaled', { rotation: 0, scaleX: 0.5, scaleY: 0.5, x: 4, y: 3 }, { height: 10, width: 12, x: 10, y: 8 }], + // Framed on the whole layer: a rotated draw of a cropped region antialiases its cut edges. + ['rotated and downscaled', { rotation: Math.PI / 7, scaleX: 0.5, scaleY: 0.6, x: 20, y: 4 }, null], + ])( + 'exports a %s noisy layer as the display shows it: grain added layer-local, then resampled', + async (_, transform, frame) => { + const scene = sceneFor({}); + scene.caches.growToRect('grain', { height: 48, width: 48, x: 0, y: 0 }); + const ctx = scene.caches.get('grain')!.surface.ctx; + for (let x = 0; x < 48; x += 1) { + ctx.fillStyle = `rgb(${64 + x * 2}, 128, ${192 - x * 2})`; + ctx.fillRect(x, 0, 1, 48); + } + scene.caches.publishPixels('grain'); + const layer = raster('grain', { adjustments: noiseStack('gn'), transform }); + const document = docWith([layer]); + const adjusted = createAdjustedSurfaceCache(scene.backend); + const screen = scene.backend.createSurface(WIDTH, HEIGHT); + compositeDocument(screen, document, scene.caches, identity(), { + adjustedSurface: (_layer, entry) => adjusted.get(layer.id, entry, layer.adjustments), + backend: scene.backend, + }); + const bbox = frame ?? roundOut(getCompositeLayerBounds(planBaseRasterComposite(document, BBOX).layers)!); + + const exported = await renderRasterComposite(planBaseRasterComposite(document, bbox), scene); + + expect( + worstDifference( + exported.ctx.getImageData(0, 0, bbox.width, bbox.height).data, + screen.ctx.getImageData(bbox.x, bbox.y, bbox.width, bbox.height).data + ) + ).toBeLessThanOrEqual(1); + } + ); + it("draws the display's adjusted copy instead of adjusting again", async () => { const { document, layer, scene } = adjustedScene(); const adjusted = createAdjustedSurfaceCache(scene.backend); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupSurfaceCache.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupSurfaceCache.ts index 0ae54882b02..738039e8c0e 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupSurfaceCache.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/groupSurfaceCache.ts @@ -276,10 +276,11 @@ export const createGroupSurfaceCache = (deps: GroupSurfaceDeps): GroupSurfaceCac const view: Mat2d = { a: 1, b: 0, c: 0, d: 1, e: -slot.rect.x, f: -slot.rect.y }; drawRange(ctx, view, members, memberMatrices, content, baseIndex, scope.start, scope.end, scope.children); ctx.restore(); - // Every adjustment is per-pixel, so a region-limited pass equals a whole-surface one there. + // Adjustments are region independent and group stacks work in document pixels, so a region-limited pass at + // the region's document origin equals a whole-surface one there. if (!isIdentityAdjustments(scope.adjustments)) { const pixels = ctx.getImageData(region.x, region.y, region.width, region.height); - applyAdjustments(pixels, scope.adjustments); + applyAdjustments(pixels, scope.adjustments, { x: slot.rect.x + region.x, y: slot.rect.y + region.y }); ctx.putImageData(pixels, region.x, region.y); } }; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.browser.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.browser.test.ts new file mode 100644 index 00000000000..3448b987391 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.browser.test.ts @@ -0,0 +1,63 @@ +/** Real-canvas check that a floating selection's baked display copy matches the layer pixels it was lifted from. */ + +import type { CanvasRasterLayerContractV2 } from '@workbench/canvas-engine/contracts'; + +import { createAdjustedSurfaceCache } from '@workbench/canvas-engine/render/adjustedSurfaceCache'; +import { createLayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; +import { renderLayerDisplayEffect } from '@workbench/canvas-engine/render/layerDisplayEffect'; +import { createDomRasterBackend } from '@workbench/canvas-engine/render/raster'; +import { liftSelectedPixels } from '@workbench/canvas-engine/selection/floatingSelection'; +import { describe, expect, it } from 'vitest'; + +describe('floating selection display effect', () => { + it("bakes the layer's adjusted pixels at the lift rect, noise included", () => { + const backend = createDomRasterBackend(); + const layers = createLayerCacheStore(backend); + const entry = layers.growToRect('L', { height: 30, width: 40, x: -7, y: -5 }); + for (let x = 0; x < 40; x += 1) { + entry.surface.ctx.fillStyle = `rgb(${60 + x * 3}, 140, ${210 - x * 3})`; + entry.surface.ctx.fillRect(x, 0, 1, 30); + } + layers.publishPixels('L'); + const layer = { + adjustments: [ + { + amount: 0.25, + distribution: 'uniform', + id: 'n', + isEnabled: true, + grain: 'color', + response: 'flat', + size: 1, + seed: 77, + type: 'noise', + }, + ], + blendMode: 'normal', + id: 'L', + isEnabled: true, + isLocked: false, + name: 'L', + opacity: 1, + source: { bitmap: { height: 30, imageName: 'l.png', width: 40 }, offset: { x: -7, y: -5 }, type: 'paint' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 3, y: 2 }, + type: 'raster', + } satisfies CanvasRasterLayerContractV2; + const mask = backend.createSurface(12, 9); + mask.ctx.fillStyle = '#ffffff'; + mask.ctx.fillRect(0, 0, 12, 9); + + const lifted = liftSelectedPixels({ + backend, + cache: { rect: entry.rect, surface: entry.surface }, + layerMatrix: { a: 1, b: 0, c: 0, d: 1, e: 3, f: 2 }, + mask: { rect: { height: 9, width: 12, x: 5, y: 4 }, surface: mask }, + })!; + const display = renderLayerDisplayEffect(backend, layer, lifted.pixels.surface, lifted.pixels.rect)!; + + // The mask's document rect lands at layer-local (2, 2), surface (9, 7). + expect(lifted.pixels.rect).toEqual({ height: 9, width: 12, x: 2, y: 2 }); + const adjusted = createAdjustedSurfaceCache(backend).get('L', entry, layer.adjustments)!; + expect([...display.ctx.getImageData(0, 0, 12, 9).data]).toEqual([...adjusted.ctx.getImageData(9, 7, 12, 9).data]); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.test.ts index 677b53145a5..2a2668ccc8c 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.test.ts @@ -6,6 +6,7 @@ import { hasLayerDisplayEffect, renderLayerDisplayEffect } from './layerDisplayE import { createTestStubRasterBackend } from './raster.testStub'; const backend = createTestStubRasterBackend(); +const ORIGIN = { x: 0, y: 0 }; const controlLayer = (withTransparencyEffect: boolean): CanvasLayerContract => ({ adapter: { beginEndStepPct: [0, 1], controlMode: 'balanced', kind: 'controlnet', model: null, weight: 1 }, @@ -81,12 +82,12 @@ describe('hasLayerDisplayEffect', () => { describe('renderLayerDisplayEffect', () => { it('returns null when the layer has no display effect, allocating nothing', () => { const source = backend.createSurface(4, 4); - expect(renderLayerDisplayEffect(backend, controlLayer(false), source)).toBeNull(); + expect(renderLayerDisplayEffect(backend, controlLayer(false), source, ORIGIN)).toBeNull(); }); it('returns a same-sized copy for a control layer with the transparency effect', () => { const source = backend.createSurface(6, 5); - const out = renderLayerDisplayEffect(backend, controlLayer(true), source); + const out = renderLayerDisplayEffect(backend, controlLayer(true), source, ORIGIN); expect(out).not.toBeNull(); expect(out).not.toBe(source); @@ -95,7 +96,7 @@ describe('renderLayerDisplayEffect', () => { it('reads the source and writes the transformed pixels back', () => { const source = backend.createSurface(6, 5); - const out = renderLayerDisplayEffect(backend, controlLayer(true), source)!; + const out = renderLayerDisplayEffect(backend, controlLayer(true), source, ORIGIN)!; const log = (out as unknown as { callLog: { op: string; args: unknown[] }[] }).callLog; expect(log.some((entry) => entry.op === 'drawImage' && entry.args[0] === source.canvas)).toBe(true); @@ -104,14 +105,15 @@ describe('renderLayerDisplayEffect', () => { }); it('returns null for a zero-sized source', () => { - expect(renderLayerDisplayEffect(backend, controlLayer(true), backend.createSurface(0, 0))).toBeNull(); + expect(renderLayerDisplayEffect(backend, controlLayer(true), backend.createSurface(0, 0), ORIGIN)).toBeNull(); }); it('returns a copy for a raster layer with adjustments', () => { const out = renderLayerDisplayEffect( backend, rasterLayer([{ brightness: 0.4, contrast: 0, id: 'b', isEnabled: true, type: 'brightness-contrast' }]), - backend.createSurface(4, 4) + backend.createSurface(4, 4), + ORIGIN ); expect(out).not.toBeNull(); }); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.ts index df22342a7a6..89e879c70da 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/layerDisplayEffect.ts @@ -1,12 +1,16 @@ /** - * Floating selections need the same raster adjustments/control transparency as their layer. These per-pixel - * effects commute with cutting, so bake a display copy once at lift. Transform changes do not affect it; - * appearance changes cancel the float. + * Floating selections need the same raster adjustments/control transparency as their layer. These effects are + * region independent, so a display copy baked once at lift, at the cut's layer-local origin, matches the pixels it + * left (noise included). Transform changes do not affect it; appearance changes cancel the float. */ import type { CanvasLayerContract } from '@workbench/canvas-engine/contracts'; -import { applyAdjustments, isIdentityAdjustments } from '@workbench/canvas-engine/render/adjustments'; +import { + type AdjustmentOrigin, + applyAdjustments, + isIdentityAdjustments, +} from '@workbench/canvas-engine/render/adjustments'; import { applyLightnessToAlpha } from '@workbench/canvas-engine/render/controlTransparency'; import type { RasterBackend, RasterSurface } from './raster'; @@ -19,11 +23,15 @@ export const hasLayerDisplayEffect = (layer: CanvasLayerContract): boolean => { return layer.type === 'raster' && !isIdentityAdjustments(layer.adjustments); }; -/** Returns an effect-baked copy, or null when drawing the source directly needs no allocation. */ +/** + * Returns an effect-baked copy, or null when drawing the source directly needs no allocation. `origin` is the + * layer-local pixel of `source`'s top-left. + */ export const renderLayerDisplayEffect = ( backend: RasterBackend, layer: CanvasLayerContract, - source: RasterSurface + source: RasterSurface, + origin: AdjustmentOrigin ): RasterSurface | null => { if (!hasLayerDisplayEffect(layer) || source.width <= 0 || source.height <= 0) { return null; @@ -38,7 +46,7 @@ export const renderLayerDisplayEffect = ( const imageData = ctx.getImageData(0, 0, source.width, source.height); if (layer.type === 'raster') { - applyAdjustments(imageData, layer.adjustments); + applyAdjustments(imageData, layer.adjustments, origin); } else { applyLightnessToAlpha(imageData.data); } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/rasterComposite.testing.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/rasterComposite.testing.ts new file mode 100644 index 00000000000..b067515a629 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/rasterComposite.testing.ts @@ -0,0 +1,49 @@ +import type { CanvasRasterLayerContractV2 } from '@workbench/canvas-engine/contracts'; +import type { Rect } from '@workbench/canvas-engine/types'; + +/** + * A noisy text layer the browser measures wider than its estimate. Estimated 240×100 (0.6 em per glyph); measured + * 400×100. At scale 0.02 the 8×4 output samples the whole layer-local region, so noise adjusts 400×100 pixels where + * the composite plan expected 240×100. + */ +export const MEASURED_NOISY_TEXT = { + bbox: { height: 4, width: 8, x: 0, y: 0 } satisfies Rect, + /** Scratch pixels beyond the plan that rendering must reserve. */ + excessPixels: 400 * 100 - 240 * 100, + layer: { + adjustments: [ + { + amount: 0.2, + distribution: 'gaussian', + grain: 'monochrome', + id: 'grain', + isEnabled: true, + response: 'flat', + seed: 5, + size: 1, + type: 'noise', + }, + ], + blendMode: 'normal', + id: 'text', + isEnabled: true, + isLocked: false, + name: 'text', + opacity: 1, + source: { + align: 'left', + color: '#fff', + content: 'WWWW', + fontFamily: 'sans-serif', + fontSize: 100, + fontWeight: 400, + lineHeight: 1, + type: 'text', + }, + transform: { rotation: 0, scaleX: 0.02, scaleY: 0.02, x: 0, y: 0 }, + type: 'raster', + } satisfies CanvasRasterLayerContractV2, + measured: { height: 100, width: 400, x: 0, y: 0 } satisfies Rect, + /** The scratch the plan reserves from the estimate. */ + plannedScratchPixels: 240 * 100, +} as const; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/rasterComposite.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/rasterComposite.ts index 82a9098ab15..c1c812b57a9 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/render/rasterComposite.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/render/rasterComposite.ts @@ -15,7 +15,12 @@ import { compileDocumentLeaves } from '@workbench/canvas-engine/document-model/d import { getSourceContentRect } from '@workbench/canvas-engine/document/sources'; import { fromTRS, invert, multiply } from '@workbench/canvas-engine/math/mat2d'; import { expand, intersect, roundOut, transformBounds, union } from '@workbench/canvas-engine/math/rect'; -import { adjustmentsKey, applyAdjustments, isIdentityAdjustments } from '@workbench/canvas-engine/render/adjustments'; +import { + adjustmentsKey, + applyAdjustments, + isIdentityAdjustments, + isPositionDependentAdjustments, +} from '@workbench/canvas-engine/render/adjustments'; import { blendToComposite } from '@workbench/canvas-engine/render/compositor'; import { collectCompositedGroups, @@ -56,7 +61,10 @@ export interface RenderRasterCompositeDeps { }; /** Owned access to a layer's pixels, released once they are drawn. */ getLayerSurface(layerId: string): Promise<{ surface: RasterSurface; rect: Rect; release(): void }>; - /** The display's layer-local adjusted copy of `surface` (built and memoized on a miss), or null when `surface` is not a live cache. */ + /** + * The display's memoized layer-local adjusted copy of `surface`, or null on a miss or when `surface` is not a live + * cache. Never builds one: {@link compositeScratchPixels} covers only the sampled region a miss adjusts. + */ adjustedSurface?( layerId: string, surface: RasterSurface, @@ -64,6 +72,26 @@ export interface RenderRasterCompositeDeps { ): RasterSurface | null; readImageData?(surface: RasterSurface, rect: Rect): ImageData; writeImageData?(surface: RasterSurface, imageData: ImageData, x: number, y: number): void; + /** + * Reserves transient raster bytes. The caller reserves {@link compositeScratchPixels} up front; rendering reserves + * here only what a layer's actual pixels need beyond that estimate, and refuses when it does not fit. + */ + reserve?( + bytes: number + ): + | { status: 'ok'; lease: { release(): void } } + | { status: 'over-budget'; requestedBytes: number; availableBytes: number }; +} + +/** + * Thrown when a composite's pixels cannot be reserved; nothing beyond the reservation was allocated. Generation + * shows the message as its notice; export maps the class to `over-budget`. + */ +export class RasterCompositeOverBudgetError extends Error { + constructor() { + super('The canvas composite exceeds the available raster memory budget.'); + this.name = 'RasterCompositeOverBudgetError'; + } } const defaultReadImageData = (surface: RasterSurface, rect: Rect): ImageData => @@ -227,6 +255,49 @@ const layerMatrix = (ref: CompositeLayerRef): Mat2d => ref.transform.scaleY ); +/** The layer-local region of `content` an output over `bbox` samples, with one pixel of resampling margin. */ +const sampledLayerRegion = (ref: CompositeLayerRef, bbox: Rect, content: Rect): Rect | null => { + const inverse = invert(multiply(bboxView(bbox), layerMatrix(ref))); + const output: Rect = { height: Math.max(0, bbox.height), width: Math.max(0, bbox.width), x: 0, y: 0 }; + return inverse ? intersect(roundOut(expand(transformBounds(inverse, output), 1)), content) : content; +}; + +const countGroupScopes = (scopes: CompositeEntry['groupScopes']): number => + (scopes ?? []).reduce((total, scope) => total + 1 + countGroupScopes(scope.children), 0); + +const outputPixelsOf = (bbox: Rect): number => Math.max(0, bbox.width) * Math.max(0, bbox.height); + +/** + * Scratch pixels an adjusted layer whose pixels cover `content` needs: output-sized, except that a position-dependent + * stack adjusts the whole layer-local region the output samples, which exceeds the output when the layer is + * downscaled. + */ +const layerScratchPixels = (ref: CompositeLayerRef, bbox: Rect, content: Rect): number => { + const sampled = isPositionDependentAdjustments(ref.adjustments) ? sampledLayerRegion(ref, bbox, content) : null; + return Math.max(outputPixelsOf(bbox), sampled ? sampled.width * sampled.height : 0); +}; + +const plannedContent = (ref: CompositeLayerRef): Rect => ({ + ...ref.contentSize, + x: ref.contentOffset.x, + y: ref.contentOffset.y, +}); + +/** + * Pixels of the scratch buffers {@link renderRasterComposite} may allocate beyond its output, each also read back + * once as ImageData: one per group scope and adjusted layer. Estimated from planned content bounds; a layer whose + * rasterized pixels exceed them (measured text) reserves the difference through `deps.reserve` while it draws. + */ +export const compositeScratchPixels = (entry: CompositeEntry): number => { + let pixels = countGroupScopes(entry.groupScopes) * outputPixelsOf(entry.bbox); + for (const ref of entry.layers) { + if (ref.adjustments) { + pixels += layerScratchPixels(ref, entry.bbox, plannedContent(ref)); + } + } + return pixels; +}; + /** Composites an entry's layers, in z-order, onto a fresh bbox-sized surface. */ export const renderRasterComposite = async ( entry: CompositeEntry, @@ -249,7 +320,10 @@ export const renderRasterComposite = async ( } }; - /** Adjusts `[source at origin]` after resampling into an output-sized buffer; bounded by the output, not the layer. */ + /** + * Adjusts `[source at origin]` after resampling into an output-sized buffer; bounded by the output, not the layer. + * Only for position-independent stacks: they are per-pixel color maps, so the origin is irrelevant. + */ const drawAdjustedAtOutput = ( ctx: Ctx, ref: CompositeLayerRef, @@ -264,7 +338,7 @@ export const renderRasterComposite = async ( setTransform(temp.ctx, placed); temp.ctx.drawImage(source.canvas, origin.x, origin.y); const pixels = readImageData(temp, fullRect); - applyAdjustments(pixels, adjustments); + applyAdjustments(pixels, adjustments, bbox); writeImageData(temp, pixels, 0, 0); ctx.save(); setTransform(ctx, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }); @@ -275,43 +349,56 @@ export const renderRasterComposite = async ( }; const drawLayer = (ctx: Ctx, ref: CompositeLayerRef, layerSurface: { surface: RasterSurface; rect: Rect }): void => { - const placed = multiply(view, layerMatrix(ref)); - let source = layerSurface.surface; - let origin: Rect = layerSurface.rect; - if (ref.adjustments) { - // Adjust layer-local pixels before resampling, as the display does: reuse the display's adjusted copy, else - // adjust only the region the output samples. A downscaled layer samples more pixels than the output holds, - // so it adjusts after resampling instead, keeping the work bounded by the output. - const shared = deps.adjustedSurface?.(ref.id, source, ref.adjustments) ?? null; - if (shared) { - source = shared; - } else { - const inverse = invert(placed); - const sampled = inverse ? intersect(roundOut(expand(transformBounds(inverse, fullRect), 1)), origin) : origin; - if (!sampled) { - return; - } - if (sampled.width * sampled.height > width * height) { - drawAdjustedAtOutput(ctx, ref, source, origin, placed, ref.adjustments); - return; + let extra: { release(): void } | null = null; + try { + const placed = multiply(view, layerMatrix(ref)); + let source = layerSurface.surface; + let origin: Rect = layerSurface.rect; + if (ref.adjustments) { + // Adjust layer-local pixels before resampling, as the display does: reuse the display's adjusted copy, else + // adjust only the region the output samples. A downscaled layer samples more pixels than the output holds; + // a color-only stack then adjusts after resampling, keeping the work bounded by the output. Noise must stay + // layer-local so resampling averages it exactly as the display does, so its work is the sampled region. + const shared = deps.adjustedSurface?.(ref.id, source, ref.adjustments) ?? null; + if (shared) { + source = shared; + } else { + const sampled = sampledLayerRegion(ref, bbox, origin); + if (!sampled) { + return; + } + if (sampled.width * sampled.height > width * height && !isPositionDependentAdjustments(ref.adjustments)) { + drawAdjustedAtOutput(ctx, ref, source, origin, placed, ref.adjustments); + return; + } + // The caller reserved this scratch from planned content bounds; rasterized pixels can exceed them + // (measured text), so the excess surface and ImageData are reserved before they are allocated. + const excess = sampled.width * sampled.height - layerScratchPixels(ref, bbox, plannedContent(ref)); + const reservation = excess > 0 ? deps.reserve?.(excess * 2 * 4) : undefined; + if (reservation?.status === 'over-budget') { + throw new RasterCompositeOverBudgetError(); + } + extra = reservation?.lease ?? null; + const local = deps.backend.createSurface(sampled.width, sampled.height); + setTransform(local.ctx, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }); + local.ctx.drawImage(source.canvas, origin.x - sampled.x, origin.y - sampled.y); + const pixels = readImageData(local, { height: sampled.height, width: sampled.width, x: 0, y: 0 }); + applyAdjustments(pixels, ref.adjustments, sampled); + writeImageData(local, pixels, 0, 0); + source = local; + origin = sampled; } - const local = deps.backend.createSurface(sampled.width, sampled.height); - setTransform(local.ctx, { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }); - local.ctx.drawImage(source.canvas, origin.x - sampled.x, origin.y - sampled.y); - const pixels = readImageData(local, { height: sampled.height, width: sampled.width, x: 0, y: 0 }); - applyAdjustments(pixels, ref.adjustments); - writeImageData(local, pixels, 0, 0); - source = local; - origin = sampled; } + ctx.save(); + ctx.globalAlpha = ref.opacity; + ctx.globalCompositeOperation = blendToComposite(ref.blendMode); + setTransform(ctx, placed); + // Draw at the layer-local content origin (content-sized paint layers place their pixels off-zero). + ctx.drawImage(source.canvas, origin.x, origin.y); + ctx.restore(); + } finally { + extra?.release(); } - ctx.save(); - ctx.globalAlpha = ref.opacity; - ctx.globalCompositeOperation = blendToComposite(ref.blendMode); - setTransform(ctx, placed); - // Draw at the layer-local content origin (content-sized paint layers place their pixels off-zero). - ctx.drawImage(source.canvas, origin.x, origin.y); - ctx.restore(); }; /** Draws `[start, end)` bottom→top; a scope isolates into a buffer, applies its stack, lands source-over. */ @@ -331,7 +418,8 @@ export const renderRasterComposite = async ( await renderRange(buffer.ctx, scope.start, scope.end, scope.children); if (!isIdentityAdjustments(scope.adjustments)) { const pixels = readImageData(buffer, fullRect); - applyAdjustments(pixels, scope.adjustments); + // Group stacks work in document pixels; the buffer is the bbox at scale 1. + applyAdjustments(pixels, scope.adjustments, bbox); writeImageData(buffer, pixels, 0, 0); } ctx.save(); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/compositeForGeneration.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/compositeForGeneration.test.ts index 2387df857fa..dfa7978bfd6 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/compositeForGeneration.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/compositeForGeneration.test.ts @@ -10,8 +10,12 @@ import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { StubRasterSurface } from '@workbench/canvas-engine/render/raster.testStub'; import type { Rect } from '@workbench/canvas-operations/generationContracts'; +import { RasterMemoryBudgetController } from '@workbench/canvas-engine/controllers/rasterMemoryBudgetController'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { applyAdjustments } from '@workbench/canvas-engine/render/adjustments'; import { createTestStubRasterBackend } from '@workbench/canvas-engine/render/raster.testStub'; +import { RasterCompositeOverBudgetError } from '@workbench/canvas-engine/render/rasterComposite'; +import { MEASURED_NOISY_TEXT } from '@workbench/canvas-engine/render/rasterComposite.testing'; import { planComposites, planControlComposites } from '@workbench/canvas-operations/generationCompositePlan'; import { describe, expect, it, vi } from 'vitest'; @@ -250,9 +254,9 @@ describe('executeCompositePlan — upload + dedupe', () => { harness.deps.reserve = () => ({ availableBytes: 0, requestedBytes: 80_000, status: 'over-budget' }); const plan = planComposites(makeDoc([rasterLayer('a')]), BBOX); - await expect(executeCompositePlan(plan, harness.deps)).rejects.toThrow( - 'The canvas composite exceeds the available raster memory budget' - ); + const refused = executeCompositePlan(plan, harness.deps); + await expect(refused).rejects.toBeInstanceOf(RasterCompositeOverBudgetError); + await expect(refused).rejects.toThrow('The canvas composite exceeds the available raster memory budget.'); expect(harness.createdTargets).toHaveLength(0); }); @@ -457,4 +461,81 @@ describe('executeCompositePlan — raster adjustments baked into generation pixe await executeCompositePlan(planComposites(adjustedDoc, BBOX), harness.deps); expect(writeImageData).toHaveBeenCalledTimes(1); }); + + const grain = { + amount: 0.2, + distribution: 'gaussian', + id: 'grain', + isEnabled: true, + grain: 'color', + response: 'flat', + size: 1, + seed: 77, + type: 'noise', + } as const; + const midGray = (width: number, height: number): ImageData => { + const image = uniformImageData(width, height, 255); + for (let i = 0; i < image.data.length; i += 4) { + image.data.fill(128, i, i + 3); + } + return image; + }; + + // A noisy stack always adjusts the layer-local region the bbox samples (plus one pixel of resampling margin) at + // its layer-local origin, before resampling, even where a color-only stack would adjust the output instead. + it.each([ + // The sampled region (22×12) exceeds the 20×10 output. + { bbox: { height: 10, width: 20, x: 10, y: 5 }, origin: { x: 2, y: 1 }, scale: 1, size: [22, 12] }, + { bbox: { height: 100, width: 100, x: 20, y: 10 }, origin: { x: 12, y: 6 }, scale: 1, size: [52, 42] }, + // Downscaled 2×: output x 0–20 samples layer x 6–46, y 0–10 samples layer y 4–24. + { bbox: { height: 10, width: 20, x: 10, y: 5 }, origin: { x: 5, y: 3 }, scale: 0.5, size: [42, 22] }, + ])('adjusts noise layer-local at $origin.x,$origin.y for bbox $bbox.x,$bbox.y at scale $scale', async (testCase) => { + const written: ImageData[] = []; + const harness = makeHarness((_surface, rect) => midGray(rect.width, rect.height)); + harness.deps.writeImageData = (_surface, imageData) => { + written.push(imageData); + }; + const reserve = vi.fn((_bytes: number) => ({ lease: { release: () => undefined }, status: 'ok' as const })); + harness.deps.reserve = reserve; + const doc = makeDoc([ + rasterLayer('n', { + adjustments: [grain], + transform: { rotation: 0, scaleX: testCase.scale, scaleY: testCase.scale, x: 7, y: 3 }, + }), + ]); + + await executeCompositePlan(planComposites(doc, testCase.bbox), harness.deps); + + const [width, height] = testCase.size as [number, number]; + const expected = midGray(width, height); + applyAdjustments(expected, [grain], testCase.origin); + expect(written).toHaveLength(1); + expect(Array.from(written[0]!.data)).toEqual(Array.from(expected.data)); + // Output and scan buffers, plus the layer-local scratch surface and its ImageData, which can exceed the output. + const outputPixels = testCase.bbox.width * testCase.bbox.height; + expect(reserve).toHaveBeenCalledWith((outputPixels * 2 + Math.max(outputPixels, width * height) * 2) * 4); + }); + + it('refuses a measured text scratch past its reservation and releases everything', async () => { + const { bbox, layer, measured, plannedScratchPixels } = MEASURED_NOISY_TEXT; + const harness = makeHarness(); + const layerSurface = createTestStubRasterBackend().createSurface(measured.width, measured.height); + const releaseLayer = vi.fn(); + harness.deps.getLayerSurface = () => + Promise.resolve({ rect: measured, release: releaseLayer, surface: layerSurface }); + // Output and scan buffers, plus the planned scratch surface and its ImageData: no room for the measured excess. + const memory = new RasterMemoryBudgetController({ + budgetBytes: (bbox.width * bbox.height * 2 + plannedScratchPixels * 2) * 4, + }); + harness.deps.reserve = (bytes) => memory.reserveOperation(bytes, { purpose: 'invocation-composite' }); + + await expect(executeCompositePlan(planComposites(makeDoc([layer]), bbox), harness.deps)).rejects.toBeInstanceOf( + RasterCompositeOverBudgetError + ); + + expect(harness.createdTargets.map(({ height, width }) => [width, height])).toEqual([[bbox.width, bbox.height]]); + expect(releaseLayer).toHaveBeenCalledOnce(); + expect(memory.snapshot().reservedBytes).toBe(0); + expect(harness.uploadImage).not.toHaveBeenCalled(); + }); }); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/compositeForGeneration.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/compositeForGeneration.ts index 12728c34413..d9859d152e5 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/compositeForGeneration.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/compositeForGeneration.ts @@ -16,7 +16,11 @@ import type { import { sha256Hex } from '@platform/browser/sha256'; import { fromTRS, multiply } from '@workbench/canvas-engine/math/mat2d'; -import { renderRasterComposite } from '@workbench/canvas-engine/render/rasterComposite'; +import { + compositeScratchPixels, + RasterCompositeOverBudgetError, + renderRasterComposite, +} from '@workbench/canvas-engine/render/rasterComposite'; import { getCompositeLayerBounds } from '@workbench/canvas-operations/generationCompositePlan'; type Ctx = RasterSurface['ctx']; @@ -105,7 +109,10 @@ export interface ExecuteCompositePlanDeps { hashBlob?(blob: Blob): Promise; /** Reads a surface region's pixels for the coverage scan (default `getImageData`). */ readImageData?(surface: RasterSurface, rect: Rect): ImageData; - /** Reserves transient raster bytes for the complete composite operation. */ + /** + * Reserves transient raster bytes: once per composite for its planned buffers, and again while rendering for any + * layer whose rasterized pixels need more than planned. Over budget refuses with `RasterCompositeOverBudgetError`. + */ reserve?( bytes: number ): @@ -115,26 +122,19 @@ export interface ExecuteCompositePlanDeps { writeImageData?(surface: RasterSurface, imageData: ImageData, x: number, y: number): void; } -export class CompositeOverBudgetError extends Error { - constructor() { - super('The canvas composite exceeds the available raster memory budget.'); - this.name = 'CompositeOverBudgetError'; - } -} - const reserveComposite = ( entry: CompositeEntry, deps: ExecuteCompositePlanDeps, - layerCount: number + scratchPixels: number ): { release(): void } => { const pixels = Math.max(0, entry.bbox.width) * Math.max(0, entry.bbox.height); // Final/accumulator surface plus a final scan ImageData (except for control - // layers, which are made opaque by construction), plus one temporary surface - // and one ImageData buffer for every adjusted/mask layer. + // layers, which are made opaque by construction), plus a temporary surface + // and an ImageData buffer for every scratch pixel. const finalBuffers = entry.kind === 'control-layer' ? 1 : 2; - const reservation = deps.reserve?.(pixels * 4 * (finalBuffers + layerCount * 2)); + const reservation = deps.reserve?.((pixels * finalBuffers + scratchPixels * 2) * 4); if (reservation?.status === 'over-budget') { - throw new CompositeOverBudgetError(); + throw new RasterCompositeOverBudgetError(); } return reservation?.status === 'ok' ? reservation.lease : { release: () => undefined }; }; @@ -256,13 +256,7 @@ const executeRasterEntry = async ( }; } - const countGroupScopes = (scopes: typeof entry.groupScopes): number => - (scopes ?? []).reduce((total, scope) => total + 1 + countGroupScopes(scope.children), 0); - const reservation = reserveComposite( - entry, - deps, - entry.layers.filter((layer) => layer.adjustments !== undefined).length + countGroupScopes(entry.groupScopes) - ); + const reservation = reserveComposite(entry, deps, compositeScratchPixels(entry)); try { const surface = await renderRasterComposite(entry, deps); let bboxFullyCovered: boolean; @@ -484,7 +478,11 @@ export const executeMaskComposite = async ( }; } - const reservation = reserveComposite(entry, deps, entry.maskLayers?.length ?? 0); + const reservation = reserveComposite( + entry, + deps, + (entry.maskLayers?.length ?? 0) * Math.max(0, entry.bbox.width) * Math.max(0, entry.bbox.height) + ); try { const surface = await compositeMaskEntry(entry, deps, writeImageData, readImageData); const hasContent = hasNonWhitePixel( diff --git a/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts b/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts index c1ba50b64f1..a05b7cb14c1 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts @@ -15,6 +15,7 @@ import { normalizeCanvasDocumentContract, placementToTransform, } from './canvasMigration'; +import { MAX_SUPPORTED_CANVAS_SCHEMA_VERSION } from './canvasSchemaVersion'; import { createControlLayer, createEmptyPaintLayer, @@ -39,6 +40,8 @@ const refusal = (raw: unknown): Exclude, return result; }; +const FUTURE_CANVAS_SCHEMA_VERSION = MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1; + const withNodes = (nodes: unknown[], stack: 'raster' | 'control' | 'regional_guidance' | 'inpaint_mask' = 'raster') => { const state = createEmptyCanvasState(); return { ...state, document: { ...state.document, stacks: { ...state.document.stacks, [stack]: nodes } } }; @@ -326,6 +329,17 @@ describe('loadCanvasState', () => { outWhite: 128, type: 'levels', }, + { + amount: 0.1, + distribution: 'uniform', + grain: 'chroma', + id: 'a9', + isEnabled: true, + response: 'midtones', + seed: 3_000_000_000, + size: 2.5, + type: 'noise', + }, ]; const adjustmentsOf = (loaded: CanvasStateContractV3) => { const layer = loaded.document.stacks.raster[0]; @@ -380,6 +394,53 @@ describe('loadCanvasState', () => { expect(adjustmentsOf(invalidLevels)).toBeUndefined(); }); + it('accepts noise at its range limits and drops a stack whose noise entry is out of range or mistyped', () => { + const noise = { + amount: 0.1, + distribution: 'gaussian', + grain: 'monochrome', + id: 'n1', + isEnabled: true, + response: 'flat', + seed: 42, + size: 1, + type: 'noise', + }; + const loadStack = (adjustments: unknown[]) => { + const loaded = load(withNodes([{ ...createEmptyPaintLayer('Grain', 'grain'), adjustments }])); + const layer = loaded.document.stacks.raster[0]; + expect(layer).toBeDefined(); + return layer?.type === 'raster' ? layer.adjustments : null; + }; + + const limits = [ + { ...noise, amount: 0, seed: 0, size: 1 }, + { ...noise, amount: 1, grain: 'color', id: 'n2', response: 'midtones', seed: 0xffffffff, size: 8 }, + ]; + expect(loadStack(limits)).toEqual(limits); + + for (const invalid of [ + { amount: -0.01 }, + { amount: 1.01 }, + { amount: Number.NaN }, + { distribution: 'perlin' }, + { grain: 'luma' }, + { grain: undefined }, + { response: 'shadows' }, + { response: undefined }, + { size: 0.99 }, + { size: 8.01 }, + { size: Number.POSITIVE_INFINITY }, + { size: undefined }, + { seed: 1.5 }, + { seed: -1 }, + { seed: 0x1_0000_0000 }, + { seed: undefined }, + ]) { + expect(loadStack([{ ...noise, ...invalid }])).toBeUndefined(); + } + }); + it('normalizes control adapters in both the live document and saved snapshots', () => { const invalidLayer = { ...createControlLayer('Z Control', 'z-control'), @@ -545,7 +606,12 @@ describe('loadCanvasState', () => { }); it.each([ - ['a future outer version', { ...createEmptyCanvasState(), version: 5 }, 'state', 5], + [ + 'a future outer version', + { ...createEmptyCanvasState(), version: FUTURE_CANVAS_SCHEMA_VERSION }, + 'state', + FUTURE_CANVAS_SCHEMA_VERSION, + ], ['an older outer version', { ...createEmptyCanvasState(), version: 2 }, 'state', 2], ['a legacy version', { ...createEmptyCanvasState(), version: 1 }, 'state', 1], [ @@ -554,14 +620,30 @@ describe('loadCanvasState', () => { 'document', 2, ], + [ + 'a nested document of a future version', + { + ...createEmptyCanvasState(), + document: { ...createEmptyCanvasDocument(), version: FUTURE_CANVAS_SCHEMA_VERSION }, + }, + 'document', + FUTURE_CANVAS_SCHEMA_VERSION, + ], [ 'a snapshot of another version', { ...createEmptyCanvasState(), - snapshots: [{ createdAt: 'now', document: { ...createEmptyCanvasDocument(), version: 5 }, id: 'f', name: 'F' }], + snapshots: [ + { + createdAt: 'now', + document: { ...createEmptyCanvasDocument(), version: FUTURE_CANVAS_SCHEMA_VERSION }, + id: 'f', + name: 'F', + }, + ], }, 'snapshot', - 5, + FUTURE_CANVAS_SCHEMA_VERSION, ], ])('refuses %s before parsing anything', (_label, raw, scope, version) => { expect(refusal(raw)).toEqual({ raw, scope, status: 'unsupported-version', version }); @@ -646,6 +728,70 @@ describe('loadCanvasState', () => { expect(loaded.document.stacks.raster[0]).toMatchObject({ source: textLayer.source }); }); + describe('noise adjustments adopt schema 5', () => { + const noise = (isEnabled: boolean) => ({ + amount: 0.2, + distribution: 'gaussian', + grain: 'color', + id: `noise-${String(isEnabled)}`, + isEnabled, + response: 'flat', + seed: 7, + size: 1, + type: 'noise', + }); + const noisyLayer = (isEnabled = true) => ({ + ...createEmptyPaintLayer('Grain', 'grain'), + adjustments: [noise(isEnabled)], + }); + const declaring = (canvas: ReturnType, version: number) => ({ + ...canvas, + document: { ...canvas.document, version }, + version, + }); + + it.each([ + ['an enabled noise entry on a layer', [noisyLayer()]], + ['a disabled noise entry on a layer', [noisyLayer(false)]], + [ + 'a noise entry on a layer nested in groups', + [groupContract('outer', [groupContract('inner', [noisyLayer() as never])])], + ], + ['a noise entry on a group', [groupContract('graded', [], { adjustments: [noise(true) as never] })]], + ])('writes a v3 or v4 canvas holding %s as version 5', (_label, nodes) => { + for (const declared of [3, 4]) { + const loaded = load(declaring(withNodes(nodes), declared)); + + expect(loaded.version).toBe(5); + expect(loaded.document.version).toBe(5); + } + }); + + it('writes version 5 when only a snapshot holds noise, leaving the live document at its own version', () => { + const plain = createEmptyCanvasState(); + const noisy = withNodes([noisyLayer()]); + const loaded = load({ + ...plain, + snapshots: [{ createdAt: 'now', document: noisy.document, id: 's', name: 'S' }], + }); + + expect(loaded.version).toBe(5); + expect(loaded.document.version).toBe(3); + expect(loaded.snapshots[0]!.document.version).toBe(5); + }); + + it('keeps a declared version 5 after the noise is gone, and leaves a canvas without noise at its floor', () => { + expect(load(declaring(withNodes([createEmptyPaintLayer('Plain', 'plain')]), 5)).version).toBe(5); + expect(load(withNodes([createEmptyPaintLayer('Plain', 'plain')])).version).toBe(3); + }); + + it('re-validates an in-memory document holding noise as version 5', () => { + const { document } = withNodes([noisyLayer()]); + + expect(normalizeCanvasDocumentContract(document as never)?.version).toBe(5); + }); + }); + it('refuses a custom-font reference whose content hash is not a lowercase SHA-256', () => { const current = createEmptyCanvasState(); const textLayer = { diff --git a/invokeai/frontend/webv2/src/workbench/canvasMigration.ts b/invokeai/frontend/webv2/src/workbench/canvasMigration.ts index cc1805ea6a6..a28c48cd57b 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasMigration.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasMigration.ts @@ -123,6 +123,16 @@ const zAdjustmentEntry = z.discriminatedUnion('type', [ .refine((entry) => entry.inBlack < entry.inWhite && entry.gamma > 0), z.object({ ...zAdjustmentBase, rotation: zFiniteNumber, type: z.literal('hue') }), z.object({ ...zAdjustmentBase, type: z.literal('invert') }), + z.object({ + ...zAdjustmentBase, + amount: zFiniteNumber.min(0).max(1), + distribution: z.enum(['gaussian', 'uniform']), + grain: z.enum(['monochrome', 'color', 'chroma']), + response: z.enum(['flat', 'midtones']), + seed: z.number().int().min(0).max(0xffffffff), + size: zFiniteNumber.min(1).max(8), + type: z.literal('noise'), + }), ]); const zAdjustments = z.array(zAdjustmentEntry); const zControlAdapter = z @@ -359,22 +369,72 @@ const createDefaultStagingArea = (): CanvasStagingAreaContractV2 => ({ selectedImageIndex: 0, }); -/** v4 is adopted only when a document actually carries custom-font typography. */ +type CanvasSchemaVersion = CanvasDocumentContractV3['version']; + +const isSupportedCanvasSchemaVersion = (version: unknown): version is CanvasSchemaVersion => + typeof version === 'number' && + Number.isInteger(version) && + version >= MIN_SUPPORTED_CANVAS_SCHEMA_VERSION && + version <= MAX_SUPPORTED_CANVAS_SCHEMA_VERSION; + +/** The newest of the declared versions; a value outside the supported range counts as the oldest supported. */ +export const newestCanvasSchemaVersion = (versions: Iterable): CanvasSchemaVersion => { + let newest = MIN_SUPPORTED_CANVAS_SCHEMA_VERSION as CanvasSchemaVersion; + for (const version of versions) { + if (isSupportedCanvasSchemaVersion(version) && version > newest) { + newest = version; + } + } + return newest; +}; + const textSourceRequiresV4 = (source: unknown): boolean => isRecord(source) && source.type === 'text' && ('fontRef' in source || 'fontStyle' in source || 'fontVariations' in source); -const nodeRequiresV4 = (node: unknown): boolean => - isRecord(node) && - (textSourceRequiresV4(node.source) || (Array.isArray(node.children) && node.children.some(nodeRequiresV4))); +const hasNoiseAdjustment = (node: Record): boolean => + Array.isArray(node.adjustments) && node.adjustments.some((entry) => isRecord(entry) && entry.type === 'noise'); -/** Returns whether a document uses any v4-only custom-font source fields. */ -export const canvasDocumentRequiresFontSchemaV4 = (document: Pick): boolean => - Object.values(document.stacks).some((roots) => roots.some(nodeRequiresV4)); +/** v4: custom-font typography. v5: a noise adjustment, enabled or not, which a v4 reader would drop with its stack. */ +const nodeSchemaRequirement = (node: unknown): CanvasSchemaVersion => { + if (!isRecord(node)) { + return 3; + } + if (hasNoiseAdjustment(node)) { + return 5; + } + let required: CanvasSchemaVersion = textSourceRequiresV4(node.source) ? 4 : 3; + if (Array.isArray(node.children)) { + for (const child of node.children) { + required = Math.max(required, nodeSchemaRequirement(child)) as CanvasSchemaVersion; + if (required === MAX_SUPPORTED_CANVAS_SCHEMA_VERSION) { + break; + } + } + } + return required; +}; -const writableDocumentVersion = (declaredVersion: unknown, stacks: CanvasStackForests): 3 | 4 => - declaredVersion === 4 || canvasDocumentRequiresFontSchemaV4({ stacks }) ? 4 : 3; +/** + * The schema a document is written as: the newest it already declares (adoption is sticky) or the newest its content + * needs, so a build that cannot read that content refuses the document rather than dropping it on load. + */ +export const writableCanvasSchemaVersion = ( + declaredVersions: Iterable, + document: Pick +): CanvasSchemaVersion => { + let version = newestCanvasSchemaVersion(declaredVersions); + for (const roots of Object.values(document.stacks)) { + for (const root of roots) { + if (version === MAX_SUPPORTED_CANVAS_SCHEMA_VERSION) { + return version; + } + version = Math.max(version, nodeSchemaRequirement(root)) as CanvasSchemaVersion; + } + } + return version; +}; export const placementToTransform = ( placement: { x: number; y: number; width: number; height: number }, @@ -628,7 +688,7 @@ export const normalizeCanvasDocumentContract = ( ...document, ...normalizeCanvasDocumentGeometry(document.width, document.height, document.bbox), stacks: parsed.stacks, - version: writableDocumentVersion(document.version, parsed.stacks), + version: writableCanvasSchemaVersion([document.version], parsed), } : null; }; @@ -670,7 +730,7 @@ const loadCanvasDocument = ( ...normalizeCanvasDocumentGeometry(value.width, value.height, value.bbox), selectedLayerId, stacks: stacks.stacks, - version: writableDocumentVersion(value.version, stacks.stacks), + version: writableCanvasSchemaVersion([value.version], stacks), }, }; }; @@ -733,12 +793,11 @@ const loadCanvasStateStep = (canvas: unknown): LoadStep = documentRevision: asNumber(canvas.documentRevision, 0), snapshots, stagingArea: normalizeStagingArea(canvas), - version: - canvas.version === MAX_SUPPORTED_CANVAS_SCHEMA_VERSION || - document.value.version === MAX_SUPPORTED_CANVAS_SCHEMA_VERSION || - snapshots.some((snapshot) => snapshot.document.version === MAX_SUPPORTED_CANVAS_SCHEMA_VERSION) - ? MAX_SUPPORTED_CANVAS_SCHEMA_VERSION - : 3, + version: newestCanvasSchemaVersion([ + canvas.version, + document.value.version, + ...snapshots.map((snapshot) => snapshot.document.version), + ]), }, }; }; diff --git a/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.test.ts b/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.test.ts new file mode 100644 index 00000000000..1048582f214 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.test.ts @@ -0,0 +1,49 @@ +import type { CanvasAdjustmentEntry } from '@workbench/canvas-engine/contracts'; + +import { stackTopAnchor } from '@workbench/canvas-engine/document/insertionAnchors.testStub'; +import { describe, expect, it } from 'vitest'; + +import { applyCanvasProjectMutation } from './canvasProjectMutations'; +import { createEmptyPaintLayer } from './widgets/layers/layerOps'; +import { createDraftProject } from './workbenchState'; + +const noise: CanvasAdjustmentEntry = { + amount: 0.3, + distribution: 'uniform', + grain: 'color', + id: 'noise', + isEnabled: true, + response: 'midtones', + seed: 9, + size: 3, + type: 'noise', +}; + +describe('canvas schema adoption', () => { + it('moves a v4 canvas to schema 5 when a noise adjustment is added, and keeps 5 once it is removed', () => { + const project = createDraftProject([]); + const v4 = { + ...project, + canvas: { ...project.canvas, document: { ...project.canvas.document, version: 4 as const }, version: 4 as const }, + }; + + const added = applyCanvasProjectMutation(v4, { + anchor: stackTopAnchor(project.id), + layer: createEmptyPaintLayer('Grain', 'grain'), + type: 'addCanvasLayer', + }); + expect(added.canvas.version).toBe(4); + + const grained = applyCanvasProjectMutation(added, { + config: { adjustments: [noise], layerType: 'raster' }, + id: 'grain', + type: 'updateCanvasLayerConfig', + }); + expect(grained.canvas.version).toBe(5); + expect(grained.canvas.document.version).toBe(5); + + const cleared = applyCanvasProjectMutation(grained, { ids: ['grain'], type: 'removeCanvasLayers' }); + expect(cleared.canvas.version).toBe(5); + expect(cleared.canvas.document.version).toBe(5); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.ts b/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.ts index e1d38e5c4d4..5a02d456caf 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.ts @@ -44,7 +44,11 @@ import { isOverlayStack, layerStackOf, reorderSiblings } from '@workbench/canvas import { repairSelectedLayerId, selectionAfterAcceptedResult } from '@workbench/canvas-engine/document/selectionRepair'; import { prependProjectEvent } from '@workbench/projectEvents'; -import { canvasDocumentRequiresFontSchemaV4, normalizeCanvasDocumentContract } from './canvasMigration'; +import { + newestCanvasSchemaVersion, + normalizeCanvasDocumentContract, + writableCanvasSchemaVersion, +} from './canvasMigration'; import { getCanvasStagingCandidateFingerprint, getCanvasStagingSlotCount, @@ -117,8 +121,7 @@ const withRepairedSelection = (document: CanvasDocumentContractV3): CanvasDocume }; const setCanvasDocument = (project: Project, document: CanvasDocumentContractV3): Project => { - const schemaVersion: 3 | 4 = - project.canvas.version === 4 || document.version === 4 || canvasDocumentRequiresFontSchemaV4(document) ? 4 : 3; + const schemaVersion = writableCanvasSchemaVersion([project.canvas.version, document.version], document); const nextDocument = document.version === schemaVersion ? document : { ...document, version: schemaVersion }; return nextDocument === project.canvas.document && schemaVersion === project.canvas.version ? project @@ -131,8 +134,11 @@ const updateCanvasDocument = ( ): Project => setCanvasDocument(project, update(project.canvas.document)); const setCanvasState = (project: Project, canvas: CanvasStateContractV3): Project => { - const hasV4Snapshot = canvas.snapshots.some((snapshot) => snapshot.document.version === 4); - const schemaVersion: 3 | 4 = canvas.version === 4 || canvas.document.version === 4 || hasV4Snapshot ? 4 : 3; + const schemaVersion = newestCanvasSchemaVersion([ + canvas.version, + canvas.document.version, + ...canvas.snapshots.map((snapshot) => snapshot.document.version), + ]); const nextCanvas = canvas.version === schemaVersion ? canvas : { ...canvas, version: schemaVersion }; return nextCanvas === project.canvas ? project : { ...project, canvas: nextCanvas }; }; diff --git a/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts b/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts index d42b2a22f14..ef26990d861 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts @@ -1,8 +1,12 @@ /** Oldest canvas document schema this build reads; earlier documents are refused, not migrated. */ export const MIN_SUPPORTED_CANVAS_SCHEMA_VERSION = 3; -/** Newest canvas document schema this build can safely read, edit, and write. */ -export const MAX_SUPPORTED_CANVAS_SCHEMA_VERSION = 4; +/** + * Newest canvas document schema this build can safely read, edit, and write. A document adopts a newer schema only + * when it carries something older builds would drop on load and then lose on save: 4 for custom-font typography, 5 + * for noise adjustments. Older builds refuse the newer schema instead. + */ +export const MAX_SUPPORTED_CANVAS_SCHEMA_VERSION = 5; /** Whether a project whose documents need at least `minimumCanvasSchemaVersion` can be opened here. */ export const isCanvasSchemaVersionSupported = (minimumCanvasSchemaVersion: number): boolean => diff --git a/invokeai/frontend/webv2/src/workbench/projectCanvasGate.test.ts b/invokeai/frontend/webv2/src/workbench/projectCanvasGate.test.ts index 1804adca072..0b8075b0004 100644 --- a/invokeai/frontend/webv2/src/workbench/projectCanvasGate.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projectCanvasGate.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { createEmptyCanvasState } from './canvasMigration'; +import { MAX_SUPPORTED_CANVAS_SCHEMA_VERSION } from './canvasSchemaVersion'; import { gateProjectCanvases } from './projectCanvasGate'; const canvas = createEmptyCanvasState(); @@ -13,14 +14,14 @@ describe('gateProjectCanvases', () => { }); it('refuses a project whose live canvas is unsupported or invalid, keeping the raw document', () => { - const future = { canvas: { ...canvas, version: 5 }, id: 'p', name: 'P' }; + const future = { canvas: { ...canvas, version: MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1 }, id: 'p', name: 'P' }; const broken = { canvas: { ...canvas, version: '2' }, id: 'p', name: 'P' }; expect(gateProjectCanvases(future)).toMatchObject({ projectId: 'p', projectName: 'P', raw: future, - refusal: { scope: 'state', status: 'unsupported-version', version: 5 }, + refusal: { scope: 'state', status: 'unsupported-version', version: MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1 }, source: 'canvas', }); expect(gateProjectCanvases(broken)).toMatchObject({ diff --git a/invokeai/frontend/webv2/src/workbench/projects/README.md b/invokeai/frontend/webv2/src/workbench/projects/README.md index f40e9741497..5d201713584 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/README.md +++ b/invokeai/frontend/webv2/src/workbench/projects/README.md @@ -74,6 +74,8 @@ Cold-start offline editing not supported. When backend cannot load, unavailable New value in a closed canvas set, such as a control layer's adapter kind (`anima_lllite`), is added without raising canvas schema version or project's compatibility floor. Builds that know the value load and save it unchanged; builds that do not refuse the canvas instead of rewriting it, since coercing an unknown kind to a known one would replace the user's adapter on next autosave. In an older build the project is not opened and user sees that it "contains canvas data this version of Invoke cannot read"; saved copy is untouched and exportable, and a build that knows the value opens it as saved. Raise schema version instead when older builds could misread a document without refusing it. +Adjustment stacks are such a case: load drops a stack holding any entry it cannot parse, so an older build would load a document with a new adjustment type minus that layer's whole stack and autosave the loss. A new adjustment type therefore raises the canvas schema for documents that use it. A noise adjustment, enabled or not, on any layer or group at any depth of the live document or a saved snapshot, makes the canvas schema 5 (`canvasMigration.ts`, `writableCanvasSchemaVersion`); a document without one keeps its version. Adoption is sticky: removing the last noise entry leaves the document at 5, matching the server's monotonic floor. A build whose maximum is 4 refuses the project at the server and the canvas at load as an unsupported version. + Load does not reinterpret a known value either. Anima control layer saved as ControlNet before ControlNet-LLLite was supported keeps its kind and model; layer settings name the adapter type to switch to and offer a one-step switch that keeps model, weight and step range where the new kind can use them. ## Font dependencies diff --git a/invokeai/frontend/webv2/src/workbench/projects/api.test.ts b/invokeai/frontend/webv2/src/workbench/projects/api.test.ts index d6a14e0f3bc..acde0b97948 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/api.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projects/api.test.ts @@ -42,7 +42,7 @@ describe('createProjectSettled', () => { const post = () => ({ body: JSON.stringify({ ...request, - max_canvas_schema_version: 4, + max_canvas_schema_version: 5, max_document_schema_version: 4, minimum_canvas_schema_version: 3, }), @@ -93,7 +93,7 @@ describe('createProjectSettled', () => { expect(transport.apiFetchJson).toHaveBeenNthCalledWith(2, '/api/v1/projects/', post()); expect(transport.apiFetchJson).toHaveBeenNthCalledWith( 3, - '/api/v1/projects/project-1?max_canvas_schema_version=4&max_document_schema_version=4', + '/api/v1/projects/project-1?max_canvas_schema_version=5&max_document_schema_version=4', expect.anything() ); }); @@ -173,7 +173,7 @@ describe('createProjectSettled', () => { await expect(createProjectSettled(request, captureAccountScope())).resolves.toMatchObject(project); expect(transport.apiFetchJson).toHaveBeenNthCalledWith( 3, - '/api/v1/projects/project-1?max_canvas_schema_version=4&max_document_schema_version=4', + '/api/v1/projects/project-1?max_canvas_schema_version=5&max_document_schema_version=4', expect.anything() ); @@ -265,7 +265,7 @@ describe('canvas schema compatibility declarations', () => { await getProject('project/one'); expect(transport.apiFetchJson).toHaveBeenCalledWith( - '/api/v1/projects/project%2Fone?max_canvas_schema_version=4&max_document_schema_version=4', + '/api/v1/projects/project%2Fone?max_canvas_schema_version=5&max_document_schema_version=4', { signal: undefined } ); }); @@ -280,7 +280,7 @@ describe('canvas schema compatibility declarations', () => { data: {}, minimum_document_schema_version: 4, name: 'Project', - max_canvas_schema_version: 4, + max_canvas_schema_version: 5, max_document_schema_version: 4, minimum_canvas_schema_version: 3, }), @@ -305,7 +305,7 @@ describe('canvas schema compatibility declarations', () => { expected_revision: 4, minimum_canvas_schema_version: 3, name: 'Project', - max_canvas_schema_version: 4, + max_canvas_schema_version: 5, max_document_schema_version: 4, }), method: 'PUT', @@ -507,7 +507,7 @@ describe('project workspaces', () => { body: JSON.stringify({ data: { layout: {} }, expected_revision: 1, - max_canvas_schema_version: 4, + max_canvas_schema_version: 5, max_document_schema_version: 4, }), method: 'PUT', @@ -535,7 +535,7 @@ describe('project workspaces', () => { minimum_document_schema_version: 4, name: 'Project', workspace, - max_canvas_schema_version: 4, + max_canvas_schema_version: 5, max_document_schema_version: 4, }), method: 'PUT', @@ -581,7 +581,7 @@ describe('project workspaces', () => { minimum_document_schema_version: 4, name: 'Project', workspace: { layout: {} }, - max_canvas_schema_version: 4, + max_canvas_schema_version: 5, max_document_schema_version: 4, minimum_canvas_schema_version: 3, }), diff --git a/invokeai/frontend/webv2/src/workbench/projects/durableSyncedPersistence.test.ts b/invokeai/frontend/webv2/src/workbench/projects/durableSyncedPersistence.test.ts index 4cde7b365dd..becae26f4a3 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/durableSyncedPersistence.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projects/durableSyncedPersistence.test.ts @@ -3,7 +3,11 @@ import type { Project, WorkbenchState } from '@workbench/projectContracts'; import { accountLifecycle, captureAccountScope } from '@platform/state/accountLifecycle'; import { ApiError } from '@platform/transport/http'; +import { stackTopAnchor } from '@workbench/canvas-engine/document/insertionAnchors.testStub'; +import { applyCanvasProjectMutation } from '@workbench/canvasProjectMutations'; +import { MAX_SUPPORTED_CANVAS_SCHEMA_VERSION } from '@workbench/canvasSchemaVersion'; import { createAccountOwnedQueueRunJournal } from '@workbench/queue-integration/queueRunJournal'; +import { createEmptyPaintLayer } from '@workbench/widgets/layers/layerOps'; import { createDraftProject, createInitialWorkbenchState, @@ -109,7 +113,8 @@ const toSummary = (record: ProjectWithWorkspaceDTO): ProjectSummaryDTO => ({ workspace_updated_at: record.workspace_updated_at, }); -const createApi = () => { +/** A fake server; `clientMaxCanvasSchemaVersion` is the canvas schema its reader declares, refused below a floor. */ +const createApi = (clientMaxCanvasSchemaVersion = MAX_SUPPORTED_CANVAS_SCHEMA_VERSION) => { const records = new Map(); const api: DurableProjectPersistenceApi & { records: Map } = { records, @@ -145,6 +150,20 @@ const createApi = () => { if (!record) { return Promise.reject(Object.assign(new Error('not found'), { status: 404 })); } + if (record.minimum_canvas_schema_version > clientMaxCanvasSchemaVersion) { + return Promise.reject( + new ApiError( + JSON.stringify({ + detail: { + code: 'canvas_schema_unsupported', + max_canvas_schema_version: clientMaxCanvasSchemaVersion, + minimum_canvas_schema_version: record.minimum_canvas_schema_version, + }, + }), + 412 + ) + ); + } return Promise.resolve(structuredClone(record)); }), loadSession: vi.fn(() => Promise.resolve(null)), @@ -4280,6 +4299,42 @@ describe('saving the two halves', () => { settings: { ...project.settings, useCpuNoise: !project.settings.useCpuNoise }, }); + it('raises the canvas floor to 5 with the content that adds a noise adjustment', async () => { + const { api, project, service } = await open(toSplitRecord(createDraftProject([]))); + const grained = applyCanvasProjectMutation(project, { + anchor: stackTopAnchor(project.id), + layer: { + ...createEmptyPaintLayer('Grain', 'grain'), + adjustments: [ + { + amount: 0.1, + distribution: 'gaussian', + grain: 'monochrome', + id: 'noise', + isEnabled: false, + response: 'flat', + seed: 1, + size: 1, + type: 'noise', + }, + ], + }, + type: 'addCanvasLayer', + }); + + const saved = await service.saveWorkbench(stateWith([grained])); + + expect(saved.error).toBeNull(); + expect(vi.mocked(api.updateProject).mock.calls.map(([, request]) => request.minimum_canvas_schema_version)).toEqual( + [5] + ); + // A build that reads at most schema 4 is now refused the project by the server instead of loading it. + expect(api.records.get(project.id)).toMatchObject({ + data: { canvas: { document: { version: 5 }, version: 5 } }, + minimum_canvas_schema_version: 5, + }); + }); + it('sends only the workspace on the next save when the content already landed', async () => { const { api, project, service } = await open(toSplitRecord(createDraftProject([]))); vi.mocked(api.saveProjectWorkspace).mockRejectedValueOnce(new TypeError('Failed to fetch')); @@ -4723,6 +4778,52 @@ describe('a project from before the split whose workflow repeats an item id', () }); }); + it('raises the canvas floor to 5 when a whole save adds a noise adjustment, so a v4 reader is refused', async () => { + const project = withRepeatedNode(createDraftProject([])); + const { api, project: loaded, service } = await open(toRecord(project)); + const grained = applyCanvasProjectMutation(loaded, { + anchor: stackTopAnchor(loaded.id), + layer: { + ...createEmptyPaintLayer('Grain', 'grain'), + adjustments: [ + { + amount: 0.1, + distribution: 'uniform', + grain: 'monochrome', + id: 'noise', + isEnabled: true, + response: 'flat', + seed: 3, + size: 1, + type: 'noise', + }, + ], + }, + type: 'addCanvasLayer', + }); + + const saved = await service.saveWorkbench(stateWith([grained])); + + expect(saved.error).toBeNull(); + expect(api.saveProjectWorkspace).not.toHaveBeenCalled(); + expect(vi.mocked(api.updateProject).mock.calls.map(([, request]) => request.minimum_canvas_schema_version)).toEqual( + [5] + ); + const stored = api.records.get(project.id)!; + expect(stored).toMatchObject({ + data: { canvas: { version: 5 }, documentSchemaVersion: 3 }, + minimum_canvas_schema_version: 5, + }); + const v4Reader = createApi(4); + v4Reader.records.set(project.id, stored); + await expect( + createService(captureAccountScope(), v4Reader).hydrateProjectFromServer(project.id) + ).resolves.toMatchObject({ + refused: { refusal: { status: 'unsupported-version', version: 5 }, source: 'canvas' }, + status: 'refused', + }); + }); + it('is created whole, as an import or a recovered draft of one brings it back', async () => { const api = createApi(); const project = withRepeatedNode(createDraftProject([])); diff --git a/invokeai/frontend/webv2/src/workbench/projects/durableSyncedPersistence.ts b/invokeai/frontend/webv2/src/workbench/projects/durableSyncedPersistence.ts index d0bd517cce5..736e06df440 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/durableSyncedPersistence.ts +++ b/invokeai/frontend/webv2/src/workbench/projects/durableSyncedPersistence.ts @@ -1463,11 +1463,13 @@ export const createDurableSyncedWorkbenchPersistence = ( }); } if (keepsWhole && base) { + const raisedFloor = getRaisedCanvasSchemaFloor(document, base.minimumCanvasSchemaVersion); const updated = await api.updateProject( project.id, { data: document as unknown as Record, expected_revision: base.revision, + ...(raisedFloor === undefined ? {} : { minimum_canvas_schema_version: raisedFloor }), name: project.name, }, owner.signal diff --git a/invokeai/frontend/webv2/src/workbench/projects/library.test.ts b/invokeai/frontend/webv2/src/workbench/projects/library.test.ts index 89b8408a676..3bd51a75737 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/library.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projects/library.test.ts @@ -1,6 +1,7 @@ import type { BrowserLockResult } from '@platform/browser/webLocks'; import type * as accountLifecycleModule from '@platform/state/accountLifecycle'; +import { MAX_SUPPORTED_CANVAS_SCHEMA_VERSION } from '@workbench/canvasSchemaVersion'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import type * as libraryModule from './library'; @@ -147,7 +148,10 @@ describe('refreshProjectLibrary', () => { it('identifies summaries that require a newer canvas reader without fetching the document', async () => { api.listProjects.mockResolvedValue([ - { ...summaryDto('future', 'Future', '2026-06-10 10:00:00.000'), minimum_canvas_schema_version: 5 }, + { + ...summaryDto('future', 'Future', '2026-06-10 10:00:00.000'), + minimum_canvas_schema_version: MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1, + }, ]); await library.refreshProjectLibrary(); diff --git a/invokeai/frontend/webv2/src/workbench/projects/projectFile.test.ts b/invokeai/frontend/webv2/src/workbench/projects/projectFile.test.ts index ded7b9e26ee..3771878389f 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/projectFile.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projects/projectFile.test.ts @@ -1,4 +1,5 @@ import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { MAX_SUPPORTED_CANVAS_SCHEMA_VERSION } from '@workbench/canvasSchemaVersion'; import { createDraftProject } from '@workbench/workbenchState'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -658,7 +659,10 @@ describe('importing a project board', () => { const rewrittenEntries = new Map([...entries].map(([path, bytes]) => [path, binaryEntry(bytes)])); - rewrittenEntries.set('manifest.json', textEntry(JSON.stringify({ ...manifest, minimumCanvasSchemaVersion: 5 }))); + rewrittenEntries.set( + 'manifest.json', + textEntry(JSON.stringify({ ...manifest, minimumCanvasSchemaVersion: MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1 })) + ); const archive = new File([await writeArchive(rewrittenEntries)], original.name); await expect(projectFile.importProjectFile(archive)).rejects.toMatchObject({ reason: 'unsupported-version' }); @@ -728,6 +732,56 @@ describe('importProjectFile', () => { expect(createRequest.data.id).toBe(createRequest.project_id); }); + it('carries a noise adjustment through export and import at canvas schema 5', async () => { + const noise = { + amount: 0.2, + distribution: 'gaussian' as const, + grain: 'chroma' as const, + id: 'noise', + isEnabled: true, + response: 'midtones' as const, + seed: 12, + size: 2, + type: 'noise' as const, + }; + const draft = createDraftProject([]); + const project = { + ...draft, + canvas: { + ...draft.canvas, + document: { + ...draft.canvas.document, + stacks: stacksFrom([ + { + ...rasterImageLayer('grain', 'unused.png'), + adjustments: [noise], + source: { bitmap: null, type: 'paint' as const }, + }, + ]), + version: 5 as const, + }, + version: 5 as const, + }, + name: 'Grain', + }; + + acceptCreate(); + await projectFile.exportOpenProject(project); + const { readArchive, readEntryText } = await import('./invk/archive'); + const entries = await readArchive(new Uint8Array(await capturedArchive().arrayBuffer())); + expect(JSON.parse(readEntryText(entries.get('manifest.json')!))).toMatchObject({ minimumCanvasSchemaVersion: 5 }); + + await projectFile.importProjectFile(capturedArchive()); + + const createRequest = api.createProjectSettled.mock.calls[0]![0] as { + data: { canvas: { document: { stacks: { raster: { adjustments?: unknown }[] } }; version: number } }; + minimum_canvas_schema_version: number; + }; + expect(createRequest.minimum_canvas_schema_version).toBe(5); + expect(createRequest.data.canvas.version).toBe(5); + expect(createRequest.data.canvas.document.stacks.raster[0]?.adjustments).toEqual([noise]); + }); + describe('Auto-switch of an imported project', () => { /** A project as a build before Auto-switch saved it: the old progress toggle, no mode. */ const legacySettingsProject = () => { diff --git a/invokeai/frontend/webv2/src/workbench/projects/projectSync.test.ts b/invokeai/frontend/webv2/src/workbench/projects/projectSync.test.ts index 3c5c4051db6..ce9c6822bee 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/projectSync.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projects/projectSync.test.ts @@ -1,5 +1,6 @@ import type { Project } from '@workbench/projectContracts'; +import { MAX_SUPPORTED_CANVAS_SCHEMA_VERSION } from '@workbench/canvasSchemaVersion'; import { addProjectWorkflow, getActiveProjectGraph, recordProjectWorkflowRun } from '@workbench/projectWorkflows'; import { createDraftProject } from '@workbench/workbenchState'; import { createInitialWorkbenchState, workbenchReducer } from '@workbench/workbenchState.testing'; @@ -133,7 +134,10 @@ describe('project document serialization', () => { it('refuses a document whose canvas was written by a newer client, keeping the raw document', () => { const project = getProject(); const document = serializeProjectDocument(project); - const future = { ...document, canvas: { ...(document.canvas as object), version: 5 } }; + const future = { + ...document, + canvas: { ...(document.canvas as object), version: MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1 }, + }; const result = deserializeProjectDocument(future); @@ -141,7 +145,7 @@ describe('project document serialization', () => { refused: { projectId: project.id, raw: future, - refusal: { scope: 'state', status: 'unsupported-version', version: 5 }, + refusal: { scope: 'state', status: 'unsupported-version', version: MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1 }, source: 'canvas', }, status: 'refused', diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts index 20b64d84c98..165e844be82 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts @@ -964,6 +964,19 @@ describe('runCanvasInvocation', () => { expect(harness.releaseRasterSnapshot).toHaveBeenCalledTimes(1); }); + it('records the raster memory notice and dispatches nothing when the composite is over budget', async () => { + const harness = makeHarness(); + harness.executorDeps.reserve = (bytes) => ({ availableBytes: 0, requestedBytes: bytes, status: 'over-budget' }); + + await runCanvasInvocation(harness.deps); + + expect(harness.submittedGraphs()).toHaveLength(0); + expect(harness.notices().map((notice) => notice.message)).toEqual([ + 'The canvas composite exceeds the available raster memory budget.', + ]); + expect(harness.releaseRasterSnapshot).toHaveBeenCalledTimes(1); + }); + it('blocks invocation and hands a nonempty model-less control layer to the formatter', async () => { const harness = makeHarness({ document: docWithLayers([controlLayer('control')]) }); await runCanvasInvocation(harness.deps); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.browser.test.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.browser.test.tsx index 3faad0c7cb6..5e41487dea5 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.browser.test.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.browser.test.tsx @@ -39,6 +39,17 @@ const initialEntries = (): CanvasAdjustmentEntry[] => [ { curves: {}, id: 'cv1', isEnabled: true, type: 'curves' }, { id: 'hue1', isEnabled: true, rotation: 0, type: 'hue' }, { gamma: 1, id: 'lv1', inBlack: 0, inWhite: 255, isEnabled: true, outBlack: 0, outWhite: 255, type: 'levels' }, + { + amount: 0.1, + distribution: 'gaussian', + grain: 'monochrome', + id: 'nz1', + isEnabled: true, + response: 'midtones', + seed: 42, + size: 1, + type: 'noise', + }, ]; type Owner = 'raster' | 'group'; @@ -70,9 +81,15 @@ const landConcurrentEdit = (update: (adjustments: readonly CanvasAdjustmentEntry expect(outcome).toEqual({ status: 'committed' }); }); -const Harness = ({ entryId }: { entryId: string }) => { +const Harness = ({ entryId, withEngine }: { entryId: string; withEngine: boolean }) => { const layer = useSyncExternalStore(stub.subscribe, documentOwner); - return ; + return ( + + ); }; const settle = (action: () => void): Promise => @@ -83,7 +100,7 @@ const settle = (action: () => void): Promise => }); }); -const render = async (entryId = 'cv1', owner: Owner = 'raster', adjustments = initialEntries()) => { +const render = async (entryId = 'cv1', owner: Owner = 'raster', adjustments = initialEntries(), withEngine = true) => { applyThemeToRoot('classic'); host = document.createElement('div'); host.style.width = '260px'; @@ -102,7 +119,7 @@ const render = async (entryId = 'cv1', owner: Owner = 'raster', adjustments = in root?.render( - + ); @@ -335,6 +352,157 @@ describe('scalar and levels editors', () => { }); }); +type NoiseEntry = Extract; +const NOISE_INDEX = 4; + +/** The segmented-control item whose label reads `label`. */ +const segment = (label: string): HTMLElement => { + const item = [...host!.querySelectorAll('[data-scope="segment-group"][data-part="item"]')].find( + (candidate) => candidate.textContent?.trim() === label + ); + if (!item) { + throw new Error(`No segment labelled ${label}`); + } + return item; +}; +const newPatternButton = (): HTMLButtonElement => + host!.querySelector('button[aria-label="widgets.layers.adjustments.newNoisePattern"]')!; + +describe('noise editor', () => { + it("shows the entry's choices and commits a distribution change as one undoable patch", async () => { + await render('nz1'); + expect(host!.querySelector('[aria-label="widgets.layers.adjustments.noiseDistribution"]')).not.toBeNull(); + expect(segment('widgets.layers.adjustments.noiseDistributions.gaussian').dataset.state).toBe('checked'); + expect(segment('widgets.layers.adjustments.noiseGrains.monochrome').dataset.state).toBe('checked'); + expect(segment('widgets.layers.adjustments.noiseResponses.midtones').dataset.state).toBe('checked'); + expect(host!.querySelector('[aria-label="widgets.layers.adjustments.noiseResponse"]')).not.toBeNull(); + + await settle(() => segment('widgets.layers.adjustments.noiseDistributions.uniform').click()); + + expect(commits()).toHaveLength(1); + expect(forwardEntries(commits()[0]!)[NOISE_INDEX]).toEqual({ + ...initialEntries()[NOISE_INDEX], + distribution: 'uniform', + }); + expect(inverseEntries(commits()[0]!)).toEqual(initialEntries()); + await stub.engine.history.undo(); + expect(latestAdjustments()).toEqual(initialEntries()); + }); + + it('commits the grain, and nothing when the current choice is picked again', async () => { + await render('nz1'); + await settle(() => segment('widgets.layers.adjustments.noiseGrains.monochrome').click()); + expect(commits()).toHaveLength(0); + + await settle(() => segment('widgets.layers.adjustments.noiseGrains.chroma').click()); + expect(commits()).toHaveLength(1); + expect(forwardEntries(commits()[0]!)[NOISE_INDEX]).toEqual({ ...initialEntries()[NOISE_INDEX], grain: 'chroma' }); + expect(segment('widgets.layers.adjustments.noiseGrains.chroma').dataset.state).toBe('checked'); + }); + + it('commits the tonal response as one undoable patch', async () => { + await render('nz1'); + await settle(() => segment('widgets.layers.adjustments.noiseResponses.flat').click()); + + expect(commits()).toHaveLength(1); + expect(forwardEntries(commits()[0]!)[NOISE_INDEX]).toEqual({ ...initialEntries()[NOISE_INDEX], response: 'flat' }); + await stub.engine.history.undo(); + expect(latestAdjustments()).toEqual(initialEntries()); + }); + + it('edits the grain size in tenths of a pixel from the keyboard as one step, Shift stepping whole pixels', async () => { + await render('nz1'); + const slider = scrubber('widgets.layers.adjustments.noiseSize').querySelector('[role="slider"]')!; + expect(slider.getAttribute('aria-valuetext')).toBe('1.0 px'); + + await act(async () => { + slider.focus(); + await userEvent.keyboard('{ArrowRight>3/}'); + }); + expect(commits()).toHaveLength(1); + expect((forwardEntries(commits()[0]!)[NOISE_INDEX] as NoiseEntry).size).toBe(1.3); + + await act(async () => { + await userEvent.keyboard('{Shift>}{ArrowRight}{/Shift}'); + }); + expect(commits()).toHaveLength(2); + expect((latestAdjustments()[NOISE_INDEX] as NoiseEntry).size).toBe(2.3); + + await act(async () => { + await userEvent.keyboard('{End}'); + }); + expect((latestAdjustments()[NOISE_INDEX] as NoiseEntry).size).toBe(8); + await act(async () => { + await userEvent.keyboard('{ArrowRight}'); + }); + expect((latestAdjustments()[NOISE_INDEX] as NoiseEntry).size).toBe(8); + expect(latestAdjustments()[NOISE_INDEX]).toMatchObject({ amount: 0.1, grain: 'monochrome', seed: 42 }); + }); + + it('rounds a typed grain size to tenths of a pixel and clamps it to 8 px', async () => { + await render('nz1'); + const frame = scrubber('widgets.layers.adjustments.noiseSize'); + const type = async (text: string) => { + await act(async () => { + frame.querySelector('[role="slider"]')!.focus(); + await userEvent.keyboard(`${text}{Enter}`); + }); + }; + + await type('2.37'); + expect((latestAdjustments()[NOISE_INDEX] as NoiseEntry).size).toBe(2.4); + await type('20'); + expect((latestAdjustments()[NOISE_INDEX] as NoiseEntry).size).toBe(8); + expect(commits()).toHaveLength(2); + }); + + it('edits the amount in percent from the keyboard as one step', async () => { + await render('nz1'); + const slider = scrubber('widgets.layers.adjustments.noiseAmount').querySelector('[role="slider"]')!; + + await act(async () => { + slider.focus(); + await userEvent.keyboard('{ArrowRight>3/}'); + }); + + expect(commits()).toHaveLength(1); + const entry = forwardEntries(commits()[0]!)[NOISE_INDEX] as NoiseEntry; + expect(entry.amount).toBeCloseTo(0.13, 10); + expect(entry).toMatchObject({ distribution: 'gaussian', grain: 'monochrome', seed: 42, size: 1 }); + }); + + it('re-rolls the seed from the keyboard-reachable new-pattern button as one undoable step', async () => { + await render('nz1'); + const button = newPatternButton(); + expect(button).not.toBeNull(); + + await act(async () => { + button.focus(); + await userEvent.keyboard('{Enter}'); + }); + + expect(commits()).toHaveLength(1); + const entry = forwardEntries(commits()[0]!)[NOISE_INDEX] as NoiseEntry; + expect(entry).toMatchObject({ amount: 0.1, distribution: 'gaussian', grain: 'monochrome', id: 'nz1' }); + expect(Number.isInteger(entry.seed) && entry.seed >= 0 && entry.seed <= 0xffffffff).toBe(true); + expect(entry.seed).not.toBe(42); + expect(latestAdjustments()[NOISE_INDEX]).toMatchObject({ seed: entry.seed }); + + await stub.engine.history.undo(); + expect(latestAdjustments()).toEqual(initialEntries()); + }); + + it('disables every control without an engine', async () => { + await render('nz1', 'raster', initialEntries(), false); + expect(newPatternButton().disabled).toBe(true); + const radios = [...host!.querySelectorAll('[data-scope="segment-group"] input')]; + expect(radios).toHaveLength(7); + expect(radios.every((radio) => radio.disabled)).toBe(true); + expect(scrubber('widgets.layers.adjustments.noiseAmount').dataset.disabled).toBeDefined(); + expect(scrubber('widgets.layers.adjustments.noiseSize').dataset.disabled).toBeDefined(); + }); +}); + describe('curves entry editor', () => { it('moves a handle while dragging and commits the whole stack with the pre-gesture inverse', async () => { const svg = await render(); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.tsx index f59e41e1b19..e98435c0dff 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/AdjustmentSettings.tsx @@ -11,9 +11,13 @@ import type { PointerEvent as ReactPointerEvent } from 'react'; import { chakra, createListCollection, HStack, Stack, Text } from '@chakra-ui/react'; import { Field, Select, Slider } from '@platform/ui'; +import { IconButton } from '@platform/ui/Button'; import { ScrubberField } from '@platform/ui/ScrubberField'; +import { SegmentedControl } from '@platform/ui/SegmentedControl'; +import { Tooltip } from '@platform/ui/Tooltip'; import { buildCurveLut } from '@workbench/canvas-engine/api'; import { baselineConfig, useStructuralPreview } from '@workbench/widgets/canvas/useStructuralCommit'; +import { DicesIcon } from 'lucide-react'; import { useCallback, useMemo, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; @@ -25,6 +29,7 @@ import { finishCurveDragResult, getCurveGridCoordinates, } from './curveEditorMath'; +import { createNoiseSeed, DEFAULT_NOISE_AMOUNT, DEFAULT_NOISE_SIZE } from './layerOps'; /** * Edit adjustment subrows through whole-stack patch-config. Preview gestures live and commit once; tree rows own @@ -160,6 +165,10 @@ const AdjustmentEntryEditor = ({ (patch: Partial) => commitEntry(t('widgets.layers.adjustments.levels'), patch), [commitEntry, t] ); + const handleNoiseCommit = useCallback( + (patch: Partial) => commitEntry(t('widgets.layers.adjustments.noise'), patch), + [commitEntry, t] + ); const handleCurvesLive = useCallback((curves: CanvasAdjustmentCurves) => patchLive({ curves }), [patchLive]); const handleCurvesCommit = useCallback( (curves: CanvasAdjustmentCurves) => commitEntry(t('widgets.layers.adjustments.curves'), { curves }), @@ -214,6 +223,8 @@ const AdjustmentEntryEditor = ({ {t('widgets.layers.adjustments.invertHint')} ); + case 'noise': + return ; case 'curves': return ( ; +type NoiseEntry = Extract; const formatSigned = (value: number): string => `${value > 0 ? '+' : ''}${value}`; const formatDegrees = (value: number): string => `${Math.round(value)}°`; @@ -446,6 +458,150 @@ const LevelsEditor = ({ ); }; +const NOISE_DISTRIBUTIONS = ['gaussian', 'uniform'] as const; +const NOISE_GRAINS = ['monochrome', 'color', 'chroma'] as const; +const NOISE_RESPONSES = ['flat', 'midtones'] as const; +const NOISE_MIN_SIZE = 1; +const NOISE_MAX_SIZE = 8; + +const formatPercent = (value: number): string => `${value}%`; +const formatPixels = (value: number): string => `${value.toFixed(1)} px`; +/** Typed sizes keep their decimals (the field only clamps them); tenths of a pixel match the slider's grid. */ +const toNoiseSize = (value: number): number => Math.round(value * 10) / 10; + +/** One labelled segmented choice of the noise editor; picking the current option commits nothing. */ +const NoiseChoice = ({ + disabled, + label, + labelKey, + onCommit, + options, + value, +}: { + disabled: boolean; + label: string; + /** The i18n key prefix of each option's label. */ + labelKey: string; + onCommit: (value: Value) => void; + options: readonly Value[]; + value: Value; +}) => { + const { t } = useTranslation(); + const items = useMemo( + () => options.map((option) => ({ label: t(`${labelKey}.${option}`), value: option })), + [labelKey, options, t] + ); + const handleChange = useCallback( + (next: string) => { + if (next !== value) { + onCommit(next as Value); + } + }, + [onCommit, value] + ); + return ( + + + + ); +}; + +/** Distribution, grain, tonal response and size choices, an amount scrubber in percent, and a seed re-roll. */ +const NoiseEditor = ({ + disabled, + entry, + onCommit, + onLive, +}: { + disabled: boolean; + entry: NoiseEntry; + onLive: (patch: Partial) => void; + onCommit: (patch: Partial) => void; +}) => { + const { t } = useTranslation(); + const handleDistributionCommit = useCallback( + (distribution: NoiseEntry['distribution']) => onCommit({ distribution }), + [onCommit] + ); + const handleGrainCommit = useCallback((grain: NoiseEntry['grain']) => onCommit({ grain }), [onCommit]); + const handleResponseCommit = useCallback((response: NoiseEntry['response']) => onCommit({ response }), [onCommit]); + const handleSizeChange = useCallback((size: number) => onLive({ size: toNoiseSize(size) }), [onLive]); + const handleSizeCommit = useCallback((size: number) => onCommit({ size: toNoiseSize(size) }), [onCommit]); + const handleAmountChange = useCallback((percent: number) => onLive({ amount: percent / 100 }), [onLive]); + const handleAmountCommit = useCallback((percent: number) => onCommit({ amount: percent / 100 }), [onCommit]); + const handleNewPattern = useCallback(() => onCommit({ seed: createNoiseSeed() }), [onCommit]); + const newPatternLabel = t('widgets.layers.adjustments.newNoisePattern'); + + return ( + + + + + + + + + + + + + + + ); +}; + const sameChannel = (left: CanvasAdjustmentCurves, right: CanvasAdjustmentCurves, channel: CurveChannel): boolean => JSON.stringify(left[channel] ?? IDENTITY_CURVE) === JSON.stringify(right[channel] ?? IDENTITY_CURVE); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerChildRow.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerChildRow.tsx index 38fc8593cd8..df77b82103c 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerChildRow.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerChildRow.tsx @@ -7,6 +7,7 @@ import { WandSparklesIcon, ContrastIcon, DropletIcon, + FilmIcon, GaugeIcon, ImageIcon, RainbowIcon, @@ -40,6 +41,7 @@ const CHILD_ROW_GLYPHS: Record = { 'adjustment-hue': RainbowIcon, 'adjustment-invert': ContrastIcon, 'adjustment-levels': SlidersVerticalIcon, + 'adjustment-noise': FilmIcon, 'layer-region': WandSparklesIcon, 'mask-denoise': GaugeIcon, 'mask-noise': WavesIcon, @@ -123,6 +125,9 @@ const LayerChildRowComponent = ({ return rest; }, [dragDisabled, listeners]); const name = childRowName(child, t); + const detail = child.detailLabelKey + ? [child.detail, t(child.detailLabelKey)].filter((part) => part !== null).join(' · ') + : child.detail; const indentStyle = useMemo(() => ({ paddingLeft: `${child.depth * LAYER_TREE_INDENT_PX}px` }), [child.depth]); const handleSelect = useCallback(() => commands.selectChild(child), [child, commands]); @@ -296,9 +301,9 @@ const LayerChildRowComponent = ({ {name} )} - {child.detail !== null && !renaming ? ( + {detail !== null && !renaming ? ( - {child.detail} + {detail} ) : null} diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerContextMenu.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerContextMenu.tsx index 177f2033432..696ae935204 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerContextMenu.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerContextMenu.tsx @@ -109,7 +109,7 @@ import { copyRasterToRegionalGuidance, copyRegionalGuidanceToInpaintMask, createLayerId, - createIdentityAdjustment, + createAdjustmentEntry, createRegionalReferenceImage, fitLayerTransformToBbox, getControlTransparencyEffectPatch, @@ -827,7 +827,7 @@ const LayerMenu = ({ if (layer.type !== 'raster') { return; } - const entry = createIdentityAdjustment(type); + const entry = createAdjustmentEntry(type); const before = layer.adjustments ?? []; commitPrepared(t('widgets.layers.menu.addAdjustment'), (model) => model.prepare({ diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerGroupContextMenu.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerGroupContextMenu.tsx index db5fe639bba..e37a693c298 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerGroupContextMenu.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerGroupContextMenu.tsx @@ -40,9 +40,9 @@ import { useCallback, useMemo, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { COLOR_LABEL_ITEMS } from './colorLabels'; -import { ADJUSTMENT_ADD_ITEMS } from './layerContextActions'; +import { ADJUSTMENT_ADD_ITEMS, adjustmentAddActionId } from './layerContextActions'; import { canGroupSelection, groupLayers, ungroupLayers } from './layerGroupCommands'; -import { createIdentityAdjustment } from './layerOps'; +import { createAdjustmentEntry } from './layerOps'; const SUBMENU_POSITIONING = { placement: 'right-start' } as const; @@ -211,7 +211,7 @@ export const LayerGroupContextMenu = ({ const handleAddAdjustment = useCallback( (type: CanvasAdjustmentEntry['type']) => () => { - const entry = createIdentityAdjustment(type); + const entry = createAdjustmentEntry(type); const before = group.adjustments ?? []; commitPrepared(t('widgets.layers.menu.addAdjustment'), (model) => model.prepare({ @@ -302,7 +302,7 @@ export const LayerGroupContextMenu = ({ key={item.type} icon={item.icon} label={t(item.labelKey)} - value={`add-${item.type}`} + value={adjustmentAddActionId(item)} onSelect={handleAddAdjustment(item.type)} /> ))} diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayersTree.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayersTree.tsx index 9c582193b8a..2d97f886bfa 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayersTree.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayersTree.tsx @@ -47,7 +47,7 @@ import { } from './layerChildRows'; import { clearLayerChildSelection, selectLayerChild, useLayerChildSelection } from './layerChildSelection'; import { copyLayerToClipboard } from './layerExportActions'; -import { createAdjustmentId } from './layerOps'; +import { createAdjustmentId, createNoiseSeed } from './layerOps'; import { flattenPanelRows, isHeaderKey, @@ -627,6 +627,7 @@ export const LayersTree = ({ duplicateChild: (child) => runChildStructural(t('widgets.layers.modifiers.duplicateAdjustment'), child, { newId: createAdjustmentId(), + newNoiseSeed: createNoiseSeed(), type: 'duplicate', }), moveChild: (child, direction) => diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.test.ts index 6bf5b923711..2989544ec48 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.test.ts @@ -13,9 +13,11 @@ import { import { describe, expect, it } from 'vitest'; import { + childRowNameKey, getLayerChildItem, layerChildDropCommand, layerChildMoveTargets, + layerChildRemoveLabelKey, layerChildRowCommand, layerChildRowKey, projectLayerChildRows, @@ -310,6 +312,34 @@ describe('adjustment rows', () => { ]); }); + it('projects noise entries with their amount and a localized grain, apart from the mask noise row', () => { + const noise = (id: string, amount: number, grain: 'monochrome' | 'color' | 'chroma'): CanvasAdjustmentEntry => ({ + amount, + distribution: 'gaussian', + grain, + id, + isEnabled: true, + response: 'midtones', + seed: 1, + size: 1, + type: 'noise', + }); + const rows = projectLayerChildRows( + rasterRow( + documentFrom([ + rasterWith([noise('n1', 0.1, 'monochrome'), noise('n2', 0.257, 'color'), noise('n3', 0.05, 'chroma')]), + ]) + ).vm + ); + expect(rows.map((row) => [row.kind, row.itemId, row.detail, row.detailLabelKey])).toEqual([ + ['adjustment-noise', 'n1', '10%', 'widgets.layers.adjustments.noiseGrains.monochrome'], + ['adjustment-noise', 'n2', '26%', 'widgets.layers.adjustments.noiseGrains.color'], + ['adjustment-noise', 'n3', '5%', 'widgets.layers.adjustments.noiseGrains.chroma'], + ]); + expect(childRowNameKey('adjustment-noise')).not.toBe(childRowNameKey('mask-noise')); + expect(layerChildRemoveLabelKey('adjustment-noise')).toBe('widgets.layers.modifiers.removeAdjustment'); + }); + it('renames an entry, clears the name with null, and refuses no-op renames and non-adjustment kinds', () => { const document = documentFrom([rasterWith(), regionalWith([referenceImage('ref1')])]); const named = layerChildRowCommand(document, { itemId: 'a1', layerId: 'r1' }, { name: 'Warm up', type: 'rename' }); @@ -351,12 +381,35 @@ describe('adjustment rows', () => { const command = layerChildRowCommand( document, { itemId: 'a1', layerId: 'r1' }, - { newId: 'a1-copy', type: 'duplicate' } + { newId: 'a1-copy', newNoiseSeed: 5, type: 'duplicate' } ); const ids = (command!.config as unknown as { adjustments: { id: string }[] }).adjustments.map((entry) => entry.id); expect(ids).toEqual(['a1', 'a1-copy', 'a2', 'a3']); }); + it('reseeds a duplicated noise entry so the copies do not add identical grain', () => { + const noise: CanvasAdjustmentEntry = { + amount: 0.1, + distribution: 'uniform', + grain: 'color', + id: 'n1', + isEnabled: true, + response: 'flat', + seed: 7, + size: 2.5, + type: 'noise', + }; + const command = layerChildRowCommand( + documentFrom([rasterWith([noise])]), + { itemId: 'n1', layerId: 'r1' }, + { newId: 'n1-copy', newNoiseSeed: 99, type: 'duplicate' } + ); + expect((command!.config as unknown as { adjustments: CanvasAdjustmentEntry[] }).adjustments).toEqual([ + noise, + { ...noise, id: 'n1-copy', seed: 99 }, + ]); + }); + it('toggles and removes entries, and refuses move/duplicate for non-adjustment kinds', () => { const document = documentFrom([rasterWith(), regionalWith([referenceImage('ref1')])]); const toggled = layerChildRowCommand( @@ -375,7 +428,11 @@ describe('adjustment rows', () => { layerChildRowCommand(document, { itemId: 'ref1', layerId: 'rg1' }, { direction: 1, type: 'move' }) ).toBeNull(); expect( - layerChildRowCommand(document, { itemId: 'ref1', layerId: 'rg1' }, { newId: 'x', type: 'duplicate' }) + layerChildRowCommand( + document, + { itemId: 'ref1', layerId: 'rg1' }, + { newId: 'x', newNoiseSeed: 5, type: 'duplicate' } + ) ).toBeNull(); expect(getLayerChildItem(document, 'r1', 'a3')).toEqual({ isEnabled: true, kind: 'adjustment-curves' }); }); @@ -607,6 +664,6 @@ describe('layer regenerate region child row', () => { expect(removed).toMatchObject({ config: { inpaint: null, layerType: 'raster' } }); expect(layerChildRowCommand(document, target, { direction: 1, type: 'move' })).toBeNull(); - expect(layerChildRowCommand(document, target, { newId: 'x', type: 'duplicate' })).toBeNull(); + expect(layerChildRowCommand(document, target, { newId: 'x', newNoiseSeed: 5, type: 'duplicate' })).toBeNull(); }); }); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.ts index 3263eeaa4c4..b0eab663f3d 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerChildRows.ts @@ -29,7 +29,8 @@ export type LayerChildRowKind = | 'adjustment-curves' | 'adjustment-hsl' | 'adjustment-hue' - | 'adjustment-invert'; + | 'adjustment-invert' + | 'adjustment-noise'; type AdjustmentChildKind = Extract; @@ -45,6 +46,7 @@ const ADJUSTMENT_KIND_OF: Record, string> = { @@ -55,6 +57,7 @@ const CHILD_ROW_NAME_KEYS: Record, 'adjustment-hue': 'widgets.layers.adjustments.hue', 'adjustment-invert': 'widgets.layers.adjustments.invert', 'adjustment-levels': 'widgets.layers.adjustments.levels', + 'adjustment-noise': 'widgets.layers.adjustments.noise', 'layer-region': 'widgets.layers.modifiers.regenerateRegion', 'mask-denoise': 'widgets.layers.modifiers.denoise', 'mask-noise': 'widgets.layers.modifiers.noise', @@ -79,6 +82,8 @@ const adjustmentDetail = (entry: CanvasAdjustmentEntry): string | null => { } case 'levels': return entry.channel && entry.channel !== 'rgb' ? entry.channel.toUpperCase() : null; + case 'noise': + return `${Math.round(entry.amount * 100)}%`; case 'brightness-contrast': case 'curves': case 'invert': @@ -86,6 +91,10 @@ const adjustmentDetail = (entry: CanvasAdjustmentEntry): string | null => { } }; +/** The localized fact following an adjustment row's detail, as row fields to spread. */ +const adjustmentDetailLabel = (entry: CanvasAdjustmentEntry): { detailLabelKey?: string } => + entry.type === 'noise' ? { detailLabelKey: `widgets.layers.adjustments.noiseGrains.${entry.grain}` } : {}; + /** Kinds whose list order is document truth; their rows offer Move up/down and Duplicate. */ export const isOrderedChildKind = (kind: LayerChildRowKind): boolean => kind.startsWith('adjustment-'); @@ -115,6 +124,8 @@ export interface ProjectedChildRow { readonly image: { readonly imageName: string; readonly thumbnailUrl: string } | null; /** A compact preformatted fact shown beside the name (a percent, degrees), or `null`. */ readonly detail: string | null; + /** The i18n key of a localized fact shown after `detail` (noise's grain); absent when the kind has none. */ + readonly detailLabelKey?: string; /** The user-given name of a renameable item (adjustment entries), or `null`. */ readonly customName: string | null; } @@ -123,7 +134,8 @@ export type LayerChildRowAction = | { type: 'set-enabled'; isEnabled: boolean } | { type: 'remove' } | { type: 'move'; direction: -1 | 1 } - | { type: 'duplicate'; newId: string } + /** `newNoiseSeed` reseeds a duplicated noise entry: an identical copy would stack perfectly correlated grain. */ + | { type: 'duplicate'; newId: string; newNoiseSeed: number } | { type: 'rename'; name: string | null }; /** A child row's live item facts, resolved from the document; `null` when it is gone. */ @@ -257,6 +269,7 @@ export const projectLayerChildRows = (vm: SemanticNode): readonly ProjectedChild ...baseRow(vm), customName: adjustment.name ?? null, detail: adjustmentDetail(adjustment), + ...adjustmentDetailLabel(adjustment), image: null, isEnabled: adjustment.isEnabled, itemId: adjustment.id, @@ -335,6 +348,7 @@ export const layerChildRemoveLabelKey = (kind: LayerChildRowKind): string => { case 'adjustment-hsl': case 'adjustment-hue': case 'adjustment-invert': + case 'adjustment-noise': return 'widgets.layers.modifiers.removeAdjustment'; } }; @@ -614,7 +628,9 @@ export const layerChildRowCommand = ( const copy = source.type === 'curves' ? { ...source, curves: structuredClone(source.curves), id: action.newId } - : { ...source, id: action.newId }; + : source.type === 'noise' + ? { ...source, id: action.newId, seed: action.newNoiseSeed } + : { ...source, id: action.newId }; next = [...before.slice(0, index + 1), copy, ...before.slice(index + 1)]; break; } diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.test.ts index 0f11db2d47c..0782d6f96e0 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.test.ts @@ -319,6 +319,7 @@ describe('layer context action registry', () => { getLayerContextActionDefinition('move-to-front').handler(context); getLayerContextActionDefinition('add-curves').handler(context); + getLayerContextActionDefinition('add-noise-adjustment').handler(context); getLayerContextActionDefinition('filter').handler(context); getLayerContextActionDefinition('intersect').handler(context); getLayerContextActionDefinition('copy-to-control').handler(context); @@ -327,6 +328,7 @@ describe('layer context action registry', () => { expect(effects.reorder).toHaveBeenCalledWith('front', 'move-to-front'); expect(effects.addAdjustment).toHaveBeenCalledWith('curves'); + expect(effects.addAdjustment).toHaveBeenCalledWith('noise'); expect(effects.startFilter).toHaveBeenCalledWith(rasterLayer.id); expect(effects.booleanMerge).toHaveBeenCalledWith('intersect'); expect(effects.copyTo).toHaveBeenCalledWith('control'); @@ -480,6 +482,7 @@ describe('getLayerContextActions', () => { 'add-hsl', 'add-hue', 'add-invert', + 'add-noise-adjustment', 'add-regenerate-region', 'save-to-board', 'copy-to-clipboard', @@ -592,6 +595,8 @@ describe('getLayerContextActions', () => { it('offers add-noise and add-denoise-limit only while the mask lacks that modifier', () => { const bare = getLayerContextActions(makeState(makeLayer('inpaint_mask'))); expect(byId(bare, 'add-noise').isDisabled).toBe(false); + // The mask's noise modifier and the raster noise adjustment are separate actions. + expect(bare.some((action) => action.id === 'add-noise-adjustment')).toBe(false); expect(byId(bare, 'add-denoise-limit').isDisabled).toBe(false); const configured = getLayerContextActions( makeState({ @@ -627,6 +632,7 @@ describe('getLayerContextActions', () => { 'add-hsl', 'add-hue', 'add-invert', + 'add-noise-adjustment', 'add-regenerate-region', 'filter', 'select-object', @@ -707,6 +713,7 @@ describe('getLayerContextActions', () => { 'add-hsl', 'add-hue', 'add-invert', + 'add-noise-adjustment', 'add-regenerate-region', 'save-to-board', 'copy-to-clipboard', diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.ts index 0efddb1c633..6b399766ba6 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.ts @@ -32,6 +32,7 @@ import { CropIcon, DropletIcon, EyeIcon, + FilmIcon, EyeOffIcon, FolderPlusIcon, GaugeIcon, @@ -69,6 +70,7 @@ export type LayerContextActionId = | 'add-hsl' | 'add-hue' | 'add-invert' + | 'add-noise-adjustment' | 'merge-selected' | 'move-to-front' | 'move-forward' @@ -238,13 +240,15 @@ export interface LayerContextAction { export type LayerConfigPatchKind = 'control-transparency-effect' | 'regional-auto-negative'; const ALL_LAYER_TYPES = ['raster', 'control', 'inpaint_mask', 'regional_guidance'] as const; -/** The one add-adjustment catalog; the leaf action registry and the group menu both render from it. */ -export const ADJUSTMENT_ADD_ITEMS: readonly { +type AdjustmentAddItem = { readonly defaultLabel: string; readonly icon: LucideIcon; readonly labelKey: string; readonly type: CanvasAdjustmentEntry['type']; -}[] = [ +}; + +/** The one add-adjustment catalog; the leaf action registry and the group menu both render from it. */ +export const ADJUSTMENT_ADD_ITEMS: readonly AdjustmentAddItem[] = [ { defaultLabel: 'Brightness/Contrast', icon: SunMediumIcon, @@ -257,8 +261,13 @@ export const ADJUSTMENT_ADD_ITEMS: readonly { { defaultLabel: 'Saturation', icon: DropletIcon, labelKey: 'widgets.layers.adjustments.saturation', type: 'hsl' }, { defaultLabel: 'Hue', icon: RainbowIcon, labelKey: 'widgets.layers.adjustments.hue', type: 'hue' }, { defaultLabel: 'Invert', icon: ContrastIcon, labelKey: 'widgets.layers.adjustments.invert', type: 'invert' }, + { defaultLabel: 'Noise', icon: FilmIcon, labelKey: 'widgets.layers.adjustments.noise', type: 'noise' }, ]; +/** An add-adjustment item's action id; noise is suffixed because `add-noise` is the inpaint mask's modifier. */ +export const adjustmentAddActionId = (item: AdjustmentAddItem): LayerContextActionId => + item.type === 'noise' ? 'add-noise-adjustment' : (`add-${item.type}` as LayerContextActionId); + const RASTER_ONLY = ['raster'] as const; const CONTROL_ONLY = ['control'] as const; const RASTER_AND_CONTROL = ['raster', 'control'] as const; @@ -483,7 +492,7 @@ export const LAYER_CONTEXT_ACTION_DEFINITIONS: readonly LayerContextActionDefini defaultLabel: item.defaultLabel, handler: ({ effects }) => effects.addAdjustment(item.type), icon: item.icon, - id: `add-${item.type}` as LayerContextActionId, + id: adjustmentAddActionId(item), isEnabled: isLayerMutable, isVisible: alwaysVisible, labelKey: item.labelKey, diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextMenuLayout.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextMenuLayout.test.ts index 3092db9bb5b..c2c872deb22 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextMenuLayout.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextMenuLayout.test.ts @@ -88,7 +88,7 @@ describe('getLayerContextMenuLayout', () => { 'transform', 'rename', 'fit-to-bbox', - 'add-adjustment(add-brightness-contrast,add-exposure,add-levels,add-curves,add-hsl,add-hue,add-invert)', + 'add-adjustment(add-brightness-contrast,add-exposure,add-levels,add-curves,add-hsl,add-hue,add-invert,add-noise-adjustment)', 'add-regenerate-region', 'filter', 'select-object', diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.test.ts index 652cbc40d97..2b0c14e79ce 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.test.ts @@ -23,6 +23,7 @@ import { copyRasterToInpaintMask, copyRasterToRegionalGuidance, copyRegionalGuidanceToInpaintMask, + createAdjustmentEntry, createControlLayer, createEmptyPaintLayer, createInpaintMaskFromImage, @@ -105,6 +106,27 @@ describe('createEmptyPaintLayer', () => { }); }); +describe('createAdjustmentEntry', () => { + it('starts noise as visible film grain: monochrome, fading toward black and white, per-pixel, 10%, fresh seed', () => { + const first = createAdjustmentEntry('noise'); + const second = createAdjustmentEntry('noise'); + expect(first).toMatchObject({ + amount: 0.1, + distribution: 'gaussian', + grain: 'monochrome', + isEnabled: true, + response: 'midtones', + size: 1, + type: 'noise', + }); + for (const entry of [first, second]) { + const { seed } = entry as Extract; + expect(Number.isInteger(seed) && seed >= 0 && seed <= 0xffffffff).toBe(true); + } + expect(first.id).not.toBe(second.id); + }); +}); + describe('nextRegionalGuidanceFillColor', () => { it('derives the colour purely from the existing regional-guidance count (no session state)', () => { // Legacy pre-increment cycler: the first region (0 existing) gets index 1. diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.ts index 6ece605deef..5d40db00ca3 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.ts @@ -223,8 +223,23 @@ export const createReferenceImageId = (): string => createItemId('rgref'); /** Mints a fresh raster adjustment-entry id. */ export const createAdjustmentId = (): string => createItemId('adj'); -/** A fresh adjustment entry of `type` at its identity values (invert has none). */ -export const createIdentityAdjustment = (type: CanvasAdjustmentEntry['type']): CanvasAdjustmentEntry => { +/** + * A new noise entry's amount: σ ≈ 13 of 255, grain clearly visible at 100% zoom without swamping detail. Lower + * values suit subtle grain or dither; the slider covers both. + */ +export const DEFAULT_NOISE_AMOUNT = 0.1; + +/** A new noise entry's grain size: one independent sample per pixel, the finest grain. */ +export const DEFAULT_NOISE_SIZE = 1; + +/** A fresh uint32 pattern seed for a noise entry. */ +export const createNoiseSeed = (): number => crypto.getRandomValues(new Uint32Array(1))[0]!; + +/** + * A fresh adjustment entry of `type`. Parametric types start at identity so adding one changes nothing until edited; + * invert has no parameters and noise starts at a visible grain with a fresh seed, so both apply at once. + */ +export const createAdjustmentEntry = (type: CanvasAdjustmentEntry['type']): CanvasAdjustmentEntry => { const id = createAdjustmentId(); switch (type) { case 'brightness-contrast': @@ -241,6 +256,19 @@ export const createIdentityAdjustment = (type: CanvasAdjustmentEntry['type']): C return { id, isEnabled: true, rotation: 0, type }; case 'invert': return { id, isEnabled: true, type }; + case 'noise': + return { + amount: DEFAULT_NOISE_AMOUNT, + distribution: 'gaussian', + grain: 'monochrome', + id, + isEnabled: true, + // Grain that fades out in shadows and highlights reads as film rather than as a noise overlay. + response: 'midtones', + seed: createNoiseSeed(), + size: DEFAULT_NOISE_SIZE, + type, + }; } };