From 587f4cda7bc3d6ef513090a64c1e2a3e94000a96 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Fri, 23 Oct 2026 01:20:06 -0600 Subject: [PATCH 01/35] feat(canvas): keep a control layer's original beside its detected map Adds controlType, detection, original and originalTransform to the control layer contract with load-time repair rules and a pure derivation module; a document carrying them writes canvas schema 5. One control layer factory remains, creating untyped layers with no default model. --- .../webv2/src/workbench/canvas-engine/api.ts | 13 + .../src/workbench/canvas-engine/contracts.ts | 57 +++- .../document/controlDerivation.test.ts | 264 ++++++++++++++++++ .../document/controlDerivation.ts | 186 ++++++++++++ .../document/layerFactories.test.ts | 42 ++- .../canvas-engine/document/layerFactories.ts | 49 ++-- .../canvas-engine/fontReferences.test.ts | 13 + .../workbench/canvas-engine/fontReferences.ts | 3 +- .../transform/transformMath.test.ts | 90 ++++++ .../canvas-engine/transform/transformMath.ts | 28 +- .../importGalleryImages.test.ts | 21 +- .../canvas-operations/importGalleryImages.ts | 29 +- .../canvas-operations/useCanvasEngine.ts | 3 - .../src/workbench/canvasMigration.test.ts | 191 +++++++++++-- .../webv2/src/workbench/canvasMigration.ts | 138 +++++++-- .../src/workbench/canvasProjectMutations.ts | 16 +- .../src/workbench/canvasSchemaVersion.ts | 8 +- .../src/workbench/projectCanvasGate.test.ts | 5 +- .../webv2/src/workbench/projects/README.md | 6 + .../webv2/src/workbench/projects/api.test.ts | 18 +- .../src/workbench/projects/library.test.ts | 6 +- .../workbench/projects/projectAssets.test.ts | 17 ++ .../workbench/projects/projectFile.test.ts | 6 +- .../workbench/projects/projectSync.test.ts | 8 +- .../canvas/canvasHotkeyCommands.test.ts | 7 +- .../tool-options/SelectionOptionsRow.test.tsx | 7 +- .../ControlLayerSettings.browser.test.tsx | 26 +- .../widgets/layers/LayerContextMenu.tsx | 15 +- .../LayerFilterOperationButton.test.tsx | 7 +- .../LayerFilterOperationDelegation.test.tsx | 5 +- .../layers/controlModelOptions.test.ts | 63 +---- .../widgets/layers/controlModelOptions.ts | 24 +- .../layers/layerContextActions.test.ts | 12 +- .../layers/layerContextMenuLayout.test.ts | 10 +- .../workbench/widgets/layers/layerOps.test.ts | 75 +---- .../src/workbench/widgets/layers/layerOps.ts | 82 ++---- .../workbench/widgets/layers/useAddLayer.ts | 21 +- .../src/workbench/workbenchState.test.ts | 51 ++++ 38 files changed, 1224 insertions(+), 398 deletions(-) create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-engine/document/controlDerivation.test.ts create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-engine/document/controlDerivation.ts diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/api.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/api.ts index 7a6d5a77e5d..2e0cc310611 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/api.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/api.ts @@ -6,3 +6,16 @@ * the core dependency direction. */ export * from './capabilities'; +export { CONTROL_TYPES } from './contracts'; +export { + controlDetectionInputKey, + detachControlDerivation, + getControlDetectionInput, + isControlMapCurrent, + needsControlDetection, + rebaseControlOriginal, + restoreControlOriginal, + withDetectedMap, +} from './document/controlDerivation'; +export { createControlLayer, type CreateControlLayerInput, nextControlLayerName } from './document/layerFactories'; +export { composeLayerTransforms } from './transform/transformMath'; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts index fe73166ecaa..966f97d10a3 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts @@ -170,6 +170,9 @@ export type CanvasLayerSourceContract = span?: number; }; +/** The pixel-backed sources: an uploaded image or a paint bitmap. */ +export type CanvasPixelSourceContract = Extract; + /** Photoshop's layer-color palette; also the PSD `layerColor` vocabulary. */ export const CANVAS_COLOR_LABELS = ['red', 'orange', 'yellow', 'green', 'blue', 'violet', 'gray'] as const; export type CanvasColorLabel = (typeof CANVAS_COLOR_LABELS)[number]; @@ -282,17 +285,46 @@ export interface CanvasRasterLayerContractV2 extends CanvasLayerBaseContract { filter?: { type: string; settings: Record }; } +/** What a control layer's image should control; a closed set, so an unknown value refuses the document. */ +export const CONTROL_TYPES = ['pose', 'depth', 'edges', 'softedge', 'lineart', 'sketch', 'color', 'tile'] as const; +export type ControlType = (typeof CONTROL_TYPES)[number]; + +/** A detector (filter node type) and its settings; the type string stays open so newer detectors survive a load. */ +export interface CanvasControlDetectionContract { + type: string; + settings: Record; +} + +/** + * Control layers may derive their map from a kept original. Invariants (`document/controlDerivation.ts`): + * I1 no `controlType` ⇒ no `detection`, `original` or `originalTransform`; + * I2 `detection === null` ⇒ no `original`: `source` is used as-is; + * I3 `original` present ⇒ `detection` is an object and `source` is the map derived from it; + * I4 the map is current when `detection` is null or absent, or an `original` is kept and `filter` structurally equals + * `detection`; + * I5 detection runs on `original ?? (filter ? null : source)`, so map-only layers cannot re-detect. + */ export interface CanvasControlLayerContract extends CanvasLayerBaseContract { type: 'control'; + /** The pixels generation uses: the derived map once detection ran. */ source: CanvasLayerSourceContract; adapter: CanvasControlAdapterContract; withTransparencyEffect: boolean; - filter?: { type: string; settings: Record }; + /** The filter that produced `source`'s pixels; absent when they are not a filter result. */ + filter?: CanvasControlDetectionContract; /** * Display-only visibility for overlay layers; generation is unchanged. Absence means visible. Raster layers * instead use `isEnabled` for both visibility and participation. */ isHidden?: boolean; + /** What the image controls. Absent means no type chosen (legacy or new); never written as null. */ + controlType?: ControlType; + /** The detector for `controlType`. `null` means "None: use the image as-is"; absent means legacy (no choice). */ + detection?: CanvasControlDetectionContract | null; + /** The pre-detection pixels, kept so detection can re-run; a persisted image reference like `source`. */ + original?: CanvasPixelSourceContract; + /** Maps `original`'s local frame into the layer-local frame of `source`; absent means identity. */ + originalTransform?: CanvasLayerBaseContract['transform']; } export interface CanvasRegionalGuidanceLayerContract extends CanvasLayerBaseContract { @@ -384,8 +416,13 @@ export const CANVAS_MAX_NODE_DEPTH = 10; export const CANVAS_MAX_NODE_COUNT = 10_000; export interface CanvasDocumentContractV3 { - /** v3 is read for compatibility; v4 is the current writable schema. */ - version: 3 | 4; + /** + * Schema this document needs: 3 for plain content, 4 for custom-font typography, 5 for control-layer derivation + * fields. Each document mutation raises it to the newer of its content's need and the canvas state's version. A + * restored snapshot's document keeps its own, possibly lower, version until the next mutation; the state's version + * and the project's compatibility floor stay put meanwhile. + */ + version: 3 | 4 | 5; width: number; height: number; background: 'transparent' | { color: string }; @@ -395,9 +432,6 @@ export interface CanvasDocumentContractV3 { selectedLayerId: string | null; } -/** Current writable document contract. v3 remains accepted at the load boundary. */ -export type CanvasDocumentContractV4 = Omit & { version: 4 }; - export interface CanvasSnapshotContract { id: string; name: string; @@ -410,8 +444,12 @@ export interface CanvasStagingAreaContractV2 extends CanvasStagingAreaContract { } export interface CanvasStateContractV3 { - /** v3 is read for compatibility; v4 is the current writable schema. */ - version: 3 | 4; + /** + * Newest schema the state needs: the newest of its previous version, the live document's and every snapshot's + * (3 plain, 4 custom-font typography, 5 control-layer derivation fields). Sticky: removing that content never lowers + * it. + */ + version: 3 | 4 | 5; document: CanvasDocumentContractV3; /** * Monotonic wholesale-replacement counter. The mirror clears pixel history whenever it changes, including @@ -421,6 +459,3 @@ export interface CanvasStateContractV3 { snapshots: CanvasSnapshotContract[]; stagingArea: CanvasStagingAreaContractV2; } - -/** Current writable state contract. v3 remains accepted at the load boundary. */ -export type CanvasStateContractV4 = Omit & { version: 4 }; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/document/controlDerivation.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/controlDerivation.test.ts new file mode 100644 index 00000000000..0e95efd6e42 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/controlDerivation.test.ts @@ -0,0 +1,264 @@ +import type { CanvasControlLayerContract } from '@workbench/canvas-engine/contracts'; + +import { applyToPoint } from '@workbench/canvas-engine/math/mat2d'; +import { layerTransformMatrix } from '@workbench/canvas-engine/transform/transformMath'; +import { describe, expect, it } from 'vitest'; + +import { + controlDetectionInputKey, + detachControlDerivation, + getControlDetectionInput, + isControlMapCurrent, + needsControlDetection, + rebaseControlOriginal, + restoreControlOriginal, + withDetectedMap, +} from './controlDerivation'; +import { createControlLayer } from './layerFactories'; + +const image = (imageName: string) => ({ height: 64, imageName, width: 64 }); +const photo = { image: image('photo.png'), type: 'image' } as const; +const map = { bitmap: image('map.png'), offset: { x: 0, y: 0 }, type: 'paint' } as const; +const canny = { settings: { high_threshold: 200, low_threshold: 100 }, type: 'canny_edge_detection' }; +const IDENTITY = { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }; + +const layer = (patch: Partial = {}): CanvasControlLayerContract => ({ + ...createControlLayer({ id: 'c1', name: 'Control Layer 1' }), + ...patch, +}); +/** A typed layer whose photo was detected into `map`. */ +const detected = (patch: Partial = {}) => + layer({ controlType: 'edges', detection: canny, filter: canny, original: photo, source: map, ...patch }); + +describe('control derivation state', () => { + it.each([ + ['an empty legacy layer', layer(), { current: true, input: layer().source, needs: false }], + [ + 'a legacy filtered map (map-only)', + layer({ filter: canny, source: map }), + { current: true, input: null, needs: false }, + ], + [ + 'a typed layer using its image as-is', + layer({ controlType: 'depth', detection: null, source: photo }), + { current: true, input: photo, needs: false }, + ], + [ + 'a typed photo waiting for detection', + layer({ controlType: 'edges', detection: canny, source: photo }), + { current: false, input: photo, needs: true }, + ], + [ + 'a typed empty paint layer', + layer({ controlType: 'edges', detection: canny }), + { current: false, input: layer().source, needs: false }, + ], + ['a detected layer', detected(), { current: true, input: photo, needs: false }], + [ + 'a hand-painted detected map', + detected({ source: { ...map, bitmap: image('painted.png') } }), + { current: true, input: photo, needs: false }, + ], + [ + 'a detected layer whose detector settings changed', + detected({ detection: { ...canny, settings: { ...canny.settings, low_threshold: 50 } } }), + { current: false, input: photo, needs: true }, + ], + [ + 'a typed map-only layer (filter copy) choosing a detector', + layer({ controlType: 'edges', detection: canny, filter: canny, source: map }), + { current: false, input: null, needs: false }, + ], + ])('reads %s', (_label, subject, expected) => { + expect(getControlDetectionInput(subject)).toEqual(expected.input); + expect(isControlMapCurrent(subject)).toBe(expected.current); + expect(needsControlDetection(subject)).toBe(expected.needs); + }); + + it('compares the producing filter with the detection structurally', () => { + const reordered = { settings: { low_threshold: 100, high_threshold: 200 }, type: 'canny_edge_detection' }; + expect(isControlMapCurrent(detected({ filter: reordered }))).toBe(true); + }); +}); + +describe('controlDetectionInputKey', () => { + const key = controlDetectionInputKey(detected()); + + it('ignores weight, name and paint on the derived map', () => { + const subject = detected(); + expect(controlDetectionInputKey({ ...subject, adapter: { ...subject.adapter, weight: 1.4 } })).toBe(key); + expect(controlDetectionInputKey({ ...subject, name: 'Edges 1' })).toBe(key); + expect(controlDetectionInputKey({ ...subject, source: { ...map, bitmap: image('painted.png') } })).toBe(key); + }); + + it('is independent of detector settings key order', () => { + const reordered = { settings: { low_threshold: 100, high_threshold: 200 }, type: 'canny_edge_detection' }; + expect(controlDetectionInputKey(detected({ detection: reordered }))).toBe(key); + }); + + it.each([ + ['the input image', detected({ original: { image: image('other.png'), type: 'image' } })], + ['the original frame', detected({ originalTransform: { ...IDENTITY, x: 4 } })], + ['a detector setting', detected({ detection: { ...canny, settings: { ...canny.settings, low_threshold: 1 } } })], + ['the detector', detected({ detection: { settings: {}, type: 'mlsd_detection' } })], + ['the type', detected({ controlType: 'lineart' })], + ])('changes with %s', (_label, subject) => { + expect(controlDetectionInputKey(subject)).not.toBe(key); + }); + + it('follows the paint of a layer whose own pixels are the input', () => { + const drawn = layer({ controlType: 'edges', detection: canny, source: map }); + expect(controlDetectionInputKey({ ...drawn, source: { ...map, bitmap: image('more-strokes.png') } })).not.toBe( + controlDetectionInputKey(drawn) + ); + }); +}); + +describe('detachControlDerivation', () => { + it('keeps the map as the image and makes the detection "None"', () => { + const subject = detected({ originalTransform: { ...IDENTITY, x: 3 } }); + const detached = detachControlDerivation(subject); + expect(detached).toEqual({ ...layer({ controlType: 'edges', filter: canny, source: map }), detection: null }); + expect(detached).not.toHaveProperty('original'); + expect(detached).not.toHaveProperty('originalTransform'); + expect(isControlMapCurrent(detached)).toBe(true); + }); + + it('returns a layer without derivation unchanged', () => { + const legacy = layer({ filter: canny, source: map }); + expect(detachControlDerivation(legacy)).toBe(legacy); + const none = layer({ controlType: 'depth', detection: null, source: photo }); + expect(detachControlDerivation(none)).toBe(none); + }); +}); + +describe('rebaseControlOriginal', () => { + const point = { x: 7, y: -3 }; + const docPoint = (subject: CanvasControlLayerContract) => + applyToPoint( + layerTransformMatrix(subject.transform), + applyToPoint(layerTransformMatrix(subject.originalTransform ?? IDENTITY), point) + ); + + it('moves the original frame with a bake so the original lands where it was', () => { + const before = detected({ + originalTransform: { rotation: 0.3, scaleX: 0.5, scaleY: 0.5, x: 2, y: 1 }, + transform: { rotation: 0.2, scaleX: 2, scaleY: 2, x: 30, y: 40 }, + }); + const after = rebaseControlOriginal({ ...before, transform: IDENTITY }, before.transform); + expect(after.original).toBe(photo); + const expected = docPoint(before); + const actual = docPoint(after); + expect(actual.x).toBeCloseTo(expected.x, 9); + expect(actual.y).toBeCloseTo(expected.y, 9); + }); + + it('omits an identity frame and leaves layers without an original alone', () => { + const identity = rebaseControlOriginal(detected({ originalTransform: { ...IDENTITY, x: 5 } }), { + ...IDENTITY, + x: -5, + }); + expect(identity).not.toHaveProperty('originalTransform'); + const plain = layer({ controlType: 'edges', detection: canny, source: photo }); + expect(rebaseControlOriginal(plain, { ...IDENTITY, x: 9 })).toBe(plain); + }); + + it('detaches when the rebased frame would shear', () => { + const subject = detected({ originalTransform: { ...IDENTITY, rotation: Math.PI / 4 } }); + const after = rebaseControlOriginal(subject, { ...IDENTITY, scaleX: 2 }); + expect(after).toEqual(detachControlDerivation(subject)); + expect(after.detection).toBeNull(); + }); +}); + +describe('withDetectedMap', () => { + const result = { image: image('detected.png'), rect: { height: 64, width: 64, x: -2, y: 5 } }; + + it('keeps the input as the original and installs the map as a paint source', () => { + const pending = layer({ controlType: 'edges', detection: canny, source: photo }); + const next = withDetectedMap(pending, result, canny); + expect(next).toEqual({ + ...pending, + filter: canny, + original: photo, + source: { bitmap: image('detected.png'), offset: { x: -2, y: 5 }, type: 'paint' }, + }); + expect(next && isControlMapCurrent(next)).toBe(true); + expect(next && needsControlDetection(next)).toBe(false); + }); + + it('re-detects from the kept original, never from the previous map', () => { + const next = withDetectedMap(detected(), result, { settings: {}, type: 'mlsd_detection' }); + expect(next?.original).toBe(photo); + expect(next?.filter).toEqual({ settings: {}, type: 'mlsd_detection' }); + }); + + it('folds the original frame into the layer transform', () => { + const next = withDetectedMap( + detected({ originalTransform: { ...IDENTITY, scaleX: 2, scaleY: 2 }, transform: { ...IDENTITY, x: 10 } }), + result, + canny + ); + expect(next?.transform).toEqual({ rotation: 0, scaleX: 2, scaleY: 2, x: 10, y: 0 }); + expect(next).not.toHaveProperty('originalTransform'); + }); + + it.each([ + ['an untyped layer', layer({ source: photo })], + ['a layer without input pixels', layer({ controlType: 'edges', detection: canny })], + ['a map-only layer', layer({ controlType: 'edges', detection: canny, filter: canny, source: map })], + [ + 'a frame that would shear', + detected({ + originalTransform: { ...IDENTITY, rotation: Math.PI / 4 }, + transform: { ...IDENTITY, scaleX: 2 }, + }), + ], + ])('cannot land on %s', (_label, subject) => { + expect(withDetectedMap(subject, result, canny)).toBeNull(); + }); +}); + +describe('restoreControlOriginal', () => { + it('returns the original to the source in its own frame and undoes a detection exactly', () => { + const pending = layer({ + controlType: 'edges', + detection: canny, + source: photo, + transform: { rotation: 0.4, scaleX: 1.5, scaleY: 1.5, x: 8, y: 9 }, + }); + const landed = withDetectedMap( + pending, + { image: image('detected.png'), rect: { height: 64, width: 64, x: 0, y: 0 } }, + canny + )!; + expect(restoreControlOriginal(landed)).toEqual({ ...pending, detection: null }); + }); + + it('applies the original frame when it differs from the map', () => { + const restored = restoreControlOriginal( + detected({ originalTransform: { ...IDENTITY, x: 4 }, transform: { ...IDENTITY, scaleX: 2, scaleY: 2 } }) + ); + expect(restored).toMatchObject({ detection: null, source: photo, transform: { scaleX: 2, scaleY: 2, x: 8, y: 0 } }); + expect(restored).not.toHaveProperty('filter'); + expect(restored).not.toHaveProperty('original'); + }); + + it('keeps the map as the image when the original frame would shear', () => { + const subject = detected({ + originalTransform: { ...IDENTITY, rotation: Math.PI / 4 }, + transform: { ...IDENTITY, scaleX: 2 }, + }); + const restored = restoreControlOriginal(subject); + expect(restored).toMatchObject({ detection: null, source: map, transform: subject.transform }); + expect(restored).not.toHaveProperty('original'); + expect(restored).not.toHaveProperty('filter'); + }); + + it('only switches detection to "None" without an original, and leaves untyped layers alone', () => { + const pending = layer({ controlType: 'edges', detection: canny, source: photo }); + expect(restoreControlOriginal(pending)).toEqual({ ...pending, detection: null }); + const untyped = layer({ source: photo }); + expect(restoreControlOriginal(untyped)).toBe(untyped); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/document/controlDerivation.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/controlDerivation.ts new file mode 100644 index 00000000000..13f56b7fe8b --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/controlDerivation.ts @@ -0,0 +1,186 @@ +/** + * Pure rules for control layers that derive their map from a kept original; the invariants are documented on + * `CanvasControlLayerContract`. Every function returns its input unchanged when nothing changes. + */ + +import type { + CanvasControlDetectionContract, + CanvasControlLayerContract, + CanvasImageRef, + CanvasLayerSourceContract, + CanvasPixelSourceContract, +} from '@workbench/canvas-engine/contracts'; +import type { LayerTransform } from '@workbench/canvas-engine/transform/transformMath'; +import type { Rect } from '@workbench/canvas-engine/types'; + +import { areJsonValuesStructurallyEqual } from '@platform/core/json'; +import { composeLayerTransforms } from '@workbench/canvas-engine/transform/transformMath'; + +const asPixelSource = (source: CanvasLayerSourceContract): CanvasPixelSourceContract | null => + source.type === 'image' || source.type === 'paint' ? source : null; + +const hasPixels = (source: CanvasPixelSourceContract | null): source is CanvasPixelSourceContract => + source !== null && (source.type === 'image' || source.bitmap !== null); + +const isIdentityTransform = (transform: LayerTransform): boolean => + transform.x === 0 && + transform.y === 0 && + transform.rotation === 0 && + transform.scaleX === 1 && + transform.scaleY === 1; + +const withoutOriginal = (layer: CanvasControlLayerContract): CanvasControlLayerContract => { + const next = { ...layer }; + delete next.original; + delete next.originalTransform; + return next; +}; + +/** + * The pixels detection runs on: the kept original, else the layer's own source unless a filter produced it. Null for + * map-only layers (legacy filtered layers, filter copies) and for sources without pixels of their own. + */ +export const getControlDetectionInput = (layer: CanvasControlLayerContract): CanvasPixelSourceContract | null => + layer.original ?? (layer.filter ? null : asPixelSource(layer.source)); + +/** Whether `source` already reflects the chosen detection: no detection, or the kept original detected with it. */ +export const isControlMapCurrent = (layer: CanvasControlLayerContract): boolean => + layer.detection === null || + layer.detection === undefined || + (layer.original !== undefined && areJsonValuesStructurallyEqual(layer.filter, layer.detection)); + +/** Whether a typed layer's map is stale and there are pixels to detect it from. */ +export const needsControlDetection = (layer: CanvasControlLayerContract): boolean => + layer.controlType !== undefined && + layer.detection !== null && + layer.detection !== undefined && + !isControlMapCurrent(layer) && + hasPixels(getControlDetectionInput(layer)); + +const canonicalJson = (value: unknown): string => { + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(',')}]`; + } + if (value !== null && typeof value === 'object') { + const record = value as Record; + const entries = Object.keys(record) + .filter((key) => record[key] !== undefined) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`); + return `{${entries.join(',')}}`; + } + return JSON.stringify(value) ?? 'null'; +}; + +const inputRef = (input: CanvasPixelSourceContract | null): string => { + if (input === null) { + return 'none'; + } + if (input.type === 'image') { + return `image:${input.image.imageName}`; + } + return input.bitmap + ? `paint:${input.bitmap.imageName}@${String(input.offset?.x ?? 0)},${String(input.offset?.y ?? 0)}` + : 'paint:empty'; +}; + +/** + * Identifies what a detection run consumes: its input pixels and their frame, the detector with its settings, and + * the type. Weight, name and painting on the derived map leave it unchanged, so they never stale a running job. + */ +export const controlDetectionInputKey = (layer: CanvasControlLayerContract): string => + canonicalJson([ + inputRef(getControlDetectionInput(layer)), + layer.originalTransform ?? null, + layer.detection ?? null, + layer.controlType ?? null, + ]); + +/** Keeps the current pixels as the image itself: drops the original and makes a chosen detection "None". */ +export const detachControlDerivation = (layer: CanvasControlLayerContract): CanvasControlLayerContract => { + if (layer.original === undefined && layer.originalTransform === undefined && !layer.detection) { + return layer; + } + const next = withoutOriginal(layer); + if (next.detection !== undefined) { + next.detection = null; + } + return next; +}; + +/** + * Follows a bake of `bakedTransform` into the map's pixels (the layer transform then resets to identity) by moving + * the original's frame with it. A frame the bake would shear cannot be kept, so the layer detaches instead. + */ +export const rebaseControlOriginal = ( + layer: CanvasControlLayerContract, + bakedTransform: LayerTransform +): CanvasControlLayerContract => { + if (layer.original === undefined) { + return layer; + } + const originalTransform = layer.originalTransform + ? composeLayerTransforms(bakedTransform, layer.originalTransform) + : bakedTransform; + if (!originalTransform) { + return detachControlDerivation(layer); + } + if (isIdentityTransform(originalTransform)) { + if (layer.originalTransform === undefined) { + return layer; + } + const next = { ...layer }; + delete next.originalTransform; + return next; + } + return { ...layer, originalTransform: { ...originalTransform } }; +}; + +/** + * Lands a detected map. The map's `rect` is in the detection input's local frame, so a kept `originalTransform` folds + * into the layer transform. Null when the layer has no type (I1), no input pixels, or the fold would shear. + */ +export const withDetectedMap = ( + layer: CanvasControlLayerContract, + map: { image: CanvasImageRef; rect: Rect }, + detection: CanvasControlDetectionContract +): CanvasControlLayerContract | null => { + const original = getControlDetectionInput(layer); + if (layer.controlType === undefined || !hasPixels(original)) { + return null; + } + const transform = layer.originalTransform + ? composeLayerTransforms(layer.transform, layer.originalTransform) + : layer.transform; + if (!transform) { + return null; + } + const next: CanvasControlLayerContract = { + ...layer, + detection, + filter: detection, + original, + source: { bitmap: { ...map.image }, offset: { x: map.rect.x, y: map.rect.y }, type: 'paint' }, + transform, + }; + delete next.originalTransform; + return next; +}; + +/** + * Detection "None": the original becomes the source again in its own frame. When that frame would shear, the map + * stays as the as-is image and only the derivation is dropped. + */ +export const restoreControlOriginal = (layer: CanvasControlLayerContract): CanvasControlLayerContract => { + if (layer.original === undefined) { + return layer.controlType === undefined || layer.detection === null ? layer : { ...layer, detection: null }; + } + const transform = layer.originalTransform + ? composeLayerTransforms(layer.transform, layer.originalTransform) + : layer.transform; + const next = withoutOriginal( + transform ? { ...layer, detection: null, source: layer.original, transform } : { ...layer, detection: null } + ); + delete next.filter; + return next; +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerFactories.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerFactories.test.ts index 84405552977..63f282d5efd 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerFactories.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerFactories.test.ts @@ -1,17 +1,49 @@ import { CONTROL_ADAPTER_DEFAULTS } from '@workbench/controlAdapters'; import { describe, expect, it } from 'vitest'; -import { createControlLayer } from './layerFactories'; +import { createControlLayer, nextControlLayerName } from './layerFactories'; + +describe('createControlLayer', () => { + it('creates an empty, untyped, model-less layer on the legacy ControlNet defaults with transparency on', () => { + const layer = createControlLayer({ id: 'c1', name: 'Control Layer 1' }); + expect(layer).toEqual({ + adapter: { beginEndStepPct: [0, 0.75], controlMode: 'balanced', kind: 'controlnet', model: null, weight: 0.75 }, + blendMode: 'normal', + id: 'c1', + isEnabled: true, + isLocked: false, + name: 'Control Layer 1', + opacity: 1, + source: { bitmap: null, type: 'paint' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'control', + withTransparencyEffect: true, + }); + // The defaults' mutable step range is copied, never shared. + expect(layer.adapter.beginEndStepPct).not.toBe(CONTROL_ADAPTER_DEFAULTS.controlnet.beginEndStepPct); + }); -describe('createControlLayer (engine results and filters)', () => { it.each([ ['anima', 'anima_lllite'], ['z-image', 'z_image_control'], ['sdxl', 'controlnet'], - ] as const)('starts a %s layer on its kind with that kind’s defaults and the given model', (base, kind) => { - expect(createControlLayer('Control Layer 1', 'c1', base, 'model-key').adapter).toEqual({ + ] as const)('starts a %s layer on its kind’s defaults without a model', (base, kind) => { + expect(createControlLayer({ base, id: 'c1', name: 'Control Layer 1' }).adapter).toEqual({ ...CONTROL_ADAPTER_DEFAULTS[kind], - model: 'model-key', + model: null, }); }); + + it('uses the given pixel source', () => { + const source = { image: { height: 48, imageName: 'photo.png', width: 64 }, type: 'image' } as const; + expect(createControlLayer({ id: 'c1', name: 'C', source }).source).toBe(source); + }); +}); + +describe('nextControlLayerName', () => { + it('returns the first free "Control Layer N" gap', () => { + expect(nextControlLayerName([])).toBe('Control Layer 1'); + expect(nextControlLayerName(['Control Layer 1', 'Control Layer 3', 'Layer 2'])).toBe('Control Layer 2'); + expect(nextControlLayerName(['Control Layer 1', 'Control Layer 2'])).toBe('Control Layer 3'); + }); }); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerFactories.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerFactories.ts index 6e9fa753d04..b88e10931b5 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerFactories.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/layerFactories.ts @@ -3,6 +3,7 @@ import type { CanvasImageRef, CanvasInpaintMaskLayerContract, CanvasMaskFillContract, + CanvasPixelSourceContract, CanvasRegionalGuidanceLayerContract, } from '@workbench/canvas-engine/contracts'; import type { Rect } from '@workbench/canvas-engine/types'; @@ -88,23 +89,31 @@ export const createRegionalGuidanceFromImage = ( type: 'regional_guidance', }); -export const createControlLayer = ( - name: string, - id: string, - base?: string | null, - model?: string | null -): CanvasControlLayerContract => { - return { - adapter: createDefaultControlAdapter(base, model ?? null), - blendMode: 'normal', - id, - isEnabled: true, - isLocked: false, - name, - opacity: 1, - source: { bitmap: null, type: 'paint' }, - transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, - type: 'control', - withTransparencyEffect: true, - }; -}; +export interface CreateControlLayerInput { + name: string; + id: string; + /** Main model base whose adapter kind and defaults the layer starts on. */ + base?: string | null; + /** Defaults to an empty paint source. */ + source?: CanvasPixelSourceContract; +} + +/** The one control-layer factory: untyped, with no model, so generation resolves the model from the type. */ +export const createControlLayer = ({ + base, + id, + name, + source, +}: CreateControlLayerInput): CanvasControlLayerContract => ({ + adapter: createDefaultControlAdapter(base, null), + blendMode: 'normal', + id, + isEnabled: true, + isLocked: false, + name, + opacity: 1, + source: source ?? { bitmap: null, type: 'paint' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'control', + withTransparencyEffect: true, +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/fontReferences.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/fontReferences.test.ts index 788df6c41f7..98945bd6b86 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/fontReferences.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/fontReferences.test.ts @@ -123,6 +123,19 @@ describe('canvas font references', () => { expect(noOp).toMatchObject({ replacedCount: 0, document }); expect(sameCanvasTextFontRef(from, { ...from })).toBe(true); }); + it('raises a v3 document to schema 4 and keeps a v5 document at 5 when it remaps a font', () => { + const remap = (version: 3 | 4 | 5) => + replaceCanvasFontReferences({ ...documentFrom([textLayer('text', from)]), version }, from, { + fontRef: to, + style: 'normal', + weight: 400, + axes: [], + }).document.version; + + expect(remap(3)).toBe(4); + expect(remap(5)).toBe(5); + }); + it('uses target axis defaults when replacing a static face and synchronizes CSS typography', () => { const document = documentFrom([textLayer('text', from)]); const result = replaceCanvasFontReferences(document, from, { diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/fontReferences.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/fontReferences.ts index 108c379be7d..8fa670ab960 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/fontReferences.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/fontReferences.ts @@ -230,7 +230,8 @@ export const replaceCanvasFontReferences = ( changed = true; } } - const nextDocument = changed ? { ...document, stacks, version: 4 as const } : document; + const version = Math.max(document.version, 4) as CanvasDocumentContractV3['version']; + const nextDocument = changed ? { ...document, stacks, version } : document; return { clampedAxisTags: [...clampedAxisTags].sort(), document: nextDocument, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/transform/transformMath.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/transform/transformMath.test.ts index 5b51333dbc9..f5c416df688 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/transform/transformMath.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/transform/transformMath.test.ts @@ -8,6 +8,7 @@ import { applyRotate, applyScale, bakeMatrix, + composeLayerTransforms, IDENTITY_TRANSFORM, layerTransformMatrix, localHandlePoint, @@ -505,3 +506,92 @@ describe('IDENTITY_TRANSFORM', () => { expect(IDENTITY_TRANSFORM).toEqual({ rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }); }); }); + +describe('composeLayerTransforms', () => { + const points = [ + { x: 0, y: 0 }, + { x: 13, y: -7 }, + { x: -4.5, y: 21 }, + ]; + const expectSameMapping = (composed: LayerTransform | null, outer: LayerTransform, inner: LayerTransform) => { + expect(composed).not.toBeNull(); + for (const point of points) { + const expected = applyToPoint(layerTransformMatrix(outer), applyToPoint(layerTransformMatrix(inner), point)); + const actual = applyToPoint(layerTransformMatrix(composed!), point); + expect(actual.x).toBeCloseTo(expected.x, 9); + expect(actual.y).toBeCloseTo(expected.y, 9); + } + }; + + it('is neutral with the identity on either side', () => { + const t = { rotation: 0.4, scaleX: 2, scaleY: 0.5, x: 10, y: -3 }; + for (const composed of [ + composeLayerTransforms(IDENTITY_TRANSFORM, t), + composeLayerTransforms(t, IDENTITY_TRANSFORM), + ]) { + expect(composed!.rotation).toBeCloseTo(t.rotation, 12); + expect(composed!.scaleX).toBeCloseTo(t.scaleX, 12); + expect(composed!.scaleY).toBeCloseTo(t.scaleY, 12); + expect(composed!.x).toBeCloseTo(t.x, 12); + expect(composed!.y).toBeCloseTo(t.y, 12); + } + }); + + it('scales the inner translation by the outer transform and adds rotations', () => { + expect( + composeLayerTransforms( + { rotation: 0, scaleX: 2, scaleY: 3, x: 5, y: 7 }, + { rotation: 0, scaleX: 1, scaleY: 1, x: 10, y: 10 } + ) + ).toEqual({ rotation: 0, scaleX: 2, scaleY: 3, x: 25, y: 37 }); + const rotated = composeLayerTransforms( + { rotation: 0.3, scaleX: 1.5, scaleY: 1.5, x: 0, y: 0 }, + { rotation: 0.2, scaleX: 2, scaleY: 1, x: 0, y: 0 } + ); + expect(rotated!.rotation).toBeCloseTo(0.5, 12); + expect(rotated!.scaleX).toBeCloseTo(3, 12); + expect(rotated!.scaleY).toBeCloseTo(1.5, 12); + }); + + it('keeps a flip a flip instead of turning it into a half-turn', () => { + const flip = { rotation: 0, scaleX: -1, scaleY: 1, x: 0, y: 0 }; + const composed = composeLayerTransforms(IDENTITY_TRANSFORM, flip); + expect(composed!.scaleX).toBe(-1); + expect(composed!.scaleY).toBe(1); + expect(composed!.rotation).toBeCloseTo(0, 12); + }); + + it.each([ + [ + 'uniform scale around a rotation', + { rotation: 1.1, scaleX: 2, scaleY: 2, x: 4, y: -9 }, + { rotation: -0.7, scaleX: 0.5, scaleY: 3, x: -6, y: 2 }, + ], + [ + 'flips on both sides', + { rotation: 0.25, scaleX: -1.5, scaleY: 1.5, x: 1, y: 2 }, + { rotation: 2.5, scaleX: 1, scaleY: -2, x: 3, y: 4 }, + ], + [ + 'non-uniform scale over an unrotated inner', + { rotation: 0.9, scaleX: 3, scaleY: 0.25, x: 0, y: 8 }, + { rotation: 0, scaleX: -2, scaleY: 5, x: 7, y: 1 }, + ], + [ + 'a quarter turn swapping non-uniform axes', + { rotation: 0, scaleX: 2, scaleY: 3, x: 0, y: 0 }, + { rotation: Math.PI / 2, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + ], + ] as const)('maps points exactly like inner then outer: %s', (_label, outer, inner) => { + expectSameMapping(composeLayerTransforms(outer, inner), outer, inner); + }); + + it('refuses a composite that shears', () => { + expect( + composeLayerTransforms( + { rotation: 0, scaleX: 2, scaleY: 1, x: 0, y: 0 }, + { rotation: Math.PI / 4, scaleX: 1, scaleY: 1, x: 0, y: 0 } + ) + ).toBeNull(); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/transform/transformMath.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/transform/transformMath.ts index 80ea399960a..64cddea3291 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/transform/transformMath.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/transform/transformMath.ts @@ -7,7 +7,7 @@ import type { CanvasLayerBaseContract } from '@workbench/canvas-engine/contracts'; import type { Mat2d, Vec2 } from '@workbench/canvas-engine/types'; -import { applyToPoint, fromTRS } from '@workbench/canvas-engine/math/mat2d'; +import { applyToPoint, fromTRS, multiply } from '@workbench/canvas-engine/math/mat2d'; import { snapMovedPoint } from '@workbench/canvas-engine/math/snapping'; /** The editable layer transform (translate + per-axis scale + rotation in radians). */ @@ -390,3 +390,29 @@ export const bakeMatrix = (transform: LayerTransform): Mat2d => layerTransformMa /** The identity transform (a reset layer transform after a bake). */ export const IDENTITY_TRANSFORM: LayerTransform = { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }; + +/** Relative tolerance for treating a composite's axes as perpendicular. */ +const SHEAR_TOLERANCE = 1e-9; + +/** + * The transform that applies `inner`, then `outer`; null when the composite shears (a non-uniform scale outside a + * rotation) or collapses an axis, which translate/scale/rotate cannot express. A composite flip keeps the sign of + * `outer.scaleX × inner.scaleX` on `scaleX`, so composing a flip with the identity returns the flip rather than a + * half-turn. + */ +export const composeLayerTransforms = (outer: LayerTransform, inner: LayerTransform): LayerTransform | null => { + const { a, b, c, d, e, f } = multiply(layerTransformMatrix(outer), layerTransformMatrix(inner)); + const xAxis = Math.hypot(a, b); + const yAxis = Math.hypot(c, d); + if (xAxis === 0 || yAxis === 0 || Math.abs(a * c + b * d) > SHEAR_TOLERANCE * xAxis * yAxis) { + return null; + } + const scaleX = Math.sign(outer.scaleX * inner.scaleX) < 0 ? -xAxis : xAxis; + return { + rotation: Math.atan2(b / scaleX, a / scaleX), + scaleX, + scaleY: (a * d - b * c) / scaleX, + x: e, + y: f, + }; +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/importGalleryImages.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/importGalleryImages.test.ts index 0f721ed5178..a134b1e1d22 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/importGalleryImages.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/importGalleryImages.test.ts @@ -15,11 +15,10 @@ import { seedArchitectureCapabilities, } from '@features/generation/core/architectureCapabilities.testing'; import { accountLifecycle } from '@platform/state/accountLifecycle'; -import { getDocumentLeaves } from '@workbench/canvas-engine/api'; +import { createControlLayer, getDocumentLeaves } from '@workbench/canvas-engine/api'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { createTestInsertionAnchorCapture } from '@workbench/canvas-engine/document/insertionAnchors.testStub'; import { - createControlLayer, createEmptyPaintLayer, createInpaintMaskFromImage, createRegionalGuidanceFromImage, @@ -218,7 +217,7 @@ const expectedLayer = ( }; case 'control': return { - ...createControlLayer(`Control Layer ${index + 1}`, 'expected', base), + ...createControlLayer({ base, id: 'expected', name: `Control Layer ${index + 1}` }), id: expect.any(String) as string, source: { image: ref, type: 'image' }, transform: { rotation: 0, scaleX: 1, scaleY: 1, x: bbox.x, y: bbox.y }, @@ -305,13 +304,9 @@ describe('importGalleryImagesToCanvas', () => { }); }); - it('seeds new control layers with the default control model for the selected base', async () => { + it('imports control layers untyped and without a model, so generation resolves the model', async () => { const { project, state } = withProject((value) => setModel(value, 'sd-1')); const canvasEngine = engine(project.id); - const models = [ - { base: 'sd-1', key: 'sd1-canny', name: 'Canny', type: 'controlnet' }, - { base: 'sd-1', key: 'sd1-union', name: 'Union Pro', type: 'controlnet' }, - ] as unknown as Parameters[0]['models']; const result = await importGalleryImagesToCanvas({ destination: 'control', @@ -319,14 +314,16 @@ describe('importGalleryImagesToCanvas', () => { engine: canvasEngine, ...queriesFor(() => state), images: [image('a.png')], - models, project, }); expect(result.status).toBe('imported'); const [, forward] = vi.mocked(canvasEngine.layers.commitStructural).mock.calls[0]!; const layer = getForwardLayers(forward)[0]!; - expect(layer.type === 'control' ? layer.adapter.model : null).toBe('sd1-union'); + expect(layer).toMatchObject({ adapter: { kind: 'controlnet', model: null }, type: 'control' }); + expect(layer).not.toHaveProperty('controlType'); + expect(layer).not.toHaveProperty('detection'); + expect(layer).not.toHaveProperty('original'); }); it('fetches and uploads each resized control once with optimal model dimensions and durable hidden policy', async () => { @@ -382,13 +379,13 @@ describe('importGalleryImagesToCanvas', () => { const layers = getForwardLayers(forward); expect(layers).toEqual([ { - ...createControlLayer('Control Layer 1', 'expected', 'flux2'), + ...createControlLayer({ base: 'flux2', id: 'expected', name: 'Control Layer 1' }), id: expect.any(String), source: { image: { height: 768, imageName: 'resized-wide.png', width: 1360 }, type: 'image' }, transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 31, y: 47 }, }, { - ...createControlLayer('Control Layer 2', 'expected', 'flux2'), + ...createControlLayer({ base: 'flux2', id: 'expected', name: 'Control Layer 2' }), id: expect.any(String), source: { image: { height: 1024, imageName: 'resized-square.png', width: 1024 }, type: 'image' }, transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 31, y: 47 }, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/importGalleryImages.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/importGalleryImages.ts index 4ac9fa5bc6f..414b4ef30e7 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/importGalleryImages.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/importGalleryImages.ts @@ -1,5 +1,4 @@ import type { GalleryImage } from '@features/gallery'; -import type { ModelConfig } from '@features/models'; import type { CanvasDocumentCapability, CanvasLayerCapability, LayerStackKind } from '@workbench/canvas-engine/api'; import type { CanvasImageRef, CanvasLayerContract } from '@workbench/canvas-engine/contracts'; import type { CanvasProjectMutation } from '@workbench/canvasProjectMutations'; @@ -13,24 +12,25 @@ import { isArchitectureDescribed, normalizeGenerateWidgetValues, } from '@features/generation/settings'; -import { getModelsSnapshot } from '@features/models'; import { mapWithConcurrency } from '@platform/core/concurrency'; import { assertAccountScopeCurrent, captureAccountScope, registerAccountOwnedResource, } from '@platform/state/accountLifecycle'; -import { captureInsertionAnchor, getDocumentLeaves } from '@workbench/canvas-engine/api'; -import { resolveDefaultControlModelForBase } from '@workbench/widgets/layers/controlModelOptions'; import { + captureInsertionAnchor, createControlLayer, + getDocumentLeaves, + nextControlLayerName, +} from '@workbench/canvas-engine/api'; +import { createEmptyPaintLayer, createInpaintMaskFromImage, createLayerId, createRegionalGuidanceFromImage, createRegionalGuidanceLayerWithRefImage, DEFAULT_INPAINT_MASK_FILL, - nextControlLayerName, nextInpaintMaskName, nextRegionalGuidanceFillColor, nextRegionalGuidanceName, @@ -55,7 +55,6 @@ export type ImportGalleryImagesResult = interface BuildLayerContext { bbox: Project['canvas']['document']['bbox']; - defaultControlModel: string | null; existingLayers: readonly CanvasLayerContract[]; modelBase: string | null; } @@ -102,10 +101,12 @@ const buildLayer = ( case 'control': case 'control-resized': return withBboxOrigin( - { - ...createControlLayer(nextControlLayerName(names), id, context.modelBase, context.defaultControlModel), + createControlLayer({ + base: context.modelBase, + id, + name: nextControlLayerName(names), source: { image: ref, type: 'image' }, - }, + }), context.bbox ); case 'inpaint-mask': @@ -152,17 +153,14 @@ const buildLayer = ( const buildLayers = ( images: readonly LayerImage[], destination: GalleryCanvasImportDestination, - project: Project, - models: readonly ModelConfig[] + project: Project ): CanvasLayerContract[] => { const layers: CanvasLayerContract[] = []; const modelBase = getSelectedModelBase(project); - const defaultControlModel = resolveDefaultControlModelForBase(models, modelBase); for (const image of images) { layers.push( buildLayer(image, destination, { bbox: project.canvas.document.bbox, - defaultControlModel, existingLayers: [...getDocumentLeaves(project.canvas.document), ...layers], modelBase, }) @@ -244,8 +242,6 @@ export const importGalleryImagesToCanvas = async (options: { images: readonly GalleryImage[]; project: Project; fetchImage?: typeof fetch; - /** Model list used to pick a default control model; defaults to the loaded models snapshot. */ - models?: readonly ModelConfig[]; uploadImage?: typeof uploadCanvasImage; }): Promise => { const owner = captureAccountScope(); @@ -257,7 +253,6 @@ export const importGalleryImagesToCanvas = async (options: { getProject, images, isActiveProject, - models = getModelsSnapshot().models, project, uploadImage = uploadCanvasImage, } = options; @@ -298,7 +293,7 @@ export const importGalleryImagesToCanvas = async (options: { failedImageNames = resized.failedImageNames; } - const layers = buildLayers(layerImages, destination, project, models); + const layers = buildLayers(layerImages, destination, project); const previousSelectedLayerId = capturedDocument.selectedLayerId; const anchorFor = (stack: LayerStackKind) => matchingProjectEngine diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/useCanvasEngine.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/useCanvasEngine.ts index 1d82499ae70..198e5cf85ed 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/useCanvasEngine.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/useCanvasEngine.ts @@ -2,10 +2,8 @@ import type { CanvasEngine, ImageResolver } from '@workbench/canvas-engine/api'; import { useFontRuntime } from '@features/fonts/react'; import { galleryImageUrls } from '@features/gallery/utility'; -import { getModelsSnapshot } from '@features/models'; import { createCanvasProjectMutationPort } from '@workbench/canvasProjectMutationPort'; import { publishLayerPanelSelection, readLayerPanelState } from '@workbench/layerPanelState'; -import { resolveDefaultControlModelForBase } from '@workbench/widgets/layers/controlModelOptions'; import { getSelectedModelBase } from '@workbench/widgets/layers/selectedModel'; import { useActiveProjectId, @@ -86,7 +84,6 @@ export const useCanvasEngine = (): CanvasEngineHandle | null => { } }, liveEngines, - getDefaultControlModel: (base) => resolveDefaultControlModelForBase(getModelsSnapshot().models, base), getMainModelBase: () => { const project = store.getState().projects.find((candidate) => candidate.id === projectId); return project ? getSelectedModelBase(project) : null; diff --git a/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts b/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts index c1ba50b64f1..9e64f43657c 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts @@ -1,5 +1,6 @@ import type { CanvasStateContractV3 } from '@workbench/canvas-engine/api'; +import { createControlLayer } from '@workbench/canvas-engine/api'; import { groupContract, stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { describe, expect, it } from 'vitest'; @@ -14,9 +15,10 @@ import { loadCanvasState, normalizeCanvasDocumentContract, placementToTransform, + writableCanvasSchemaVersion, } from './canvasMigration'; +import { MAX_SUPPORTED_CANVAS_SCHEMA_VERSION } from './canvasSchemaVersion'; import { - createControlLayer, createEmptyPaintLayer, createInpaintMaskLayer, createRegionalGuidanceLayer, @@ -39,6 +41,8 @@ const refusal = (raw: unknown): Exclude, return result; }; +const FUTURE_CANVAS_SCHEMA_VERSION = MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1; + const withNodes = (nodes: unknown[], stack: 'raster' | 'control' | 'regional_guidance' | 'inpaint_mask' = 'raster') => { const state = createEmptyCanvasState(); return { ...state, document: { ...state.document, stacks: { ...state.document.stacks, [stack]: nodes } } }; @@ -78,7 +82,7 @@ describe('loadCanvasState', () => { { beginEndStepPct: [0.1, 0.9], controlMode: null, kind: 'flux2_control', model: 'flux2-union', weight: 0 }, ] as const; const layers = adapters.map((adapter, index) => ({ - ...createControlLayer(`Control ${index}`, `control-${index}`), + ...createControlLayer({ id: `control-${index}`, name: `Control ${index}` }), adapter, })); @@ -94,7 +98,7 @@ describe('loadCanvasState', () => { // unsupported. Load cannot tell an LLLite key from a ControlNet key without the catalog, so it keeps the kind; // the settings panel reports it and switching to LLLite keeps the model. const saved = { - ...createControlLayer('Anima Control', 'anima-control'), + ...createControlLayer({ id: 'anima-control', name: 'Anima Control' }), adapter: { beginEndStepPct: [0, 0.75], controlMode: 'balanced', @@ -111,11 +115,11 @@ describe('loadCanvasState', () => { it('refuses a control layer whose adapter kind this build does not know, naming the layer', () => { const future = { - ...createControlLayer('Future Control', 'future-control'), + ...createControlLayer({ id: 'future-control', name: 'Future Control' }), adapter: { beginEndStepPct: [0, 1], controlMode: null, kind: 'future_adapter', model: 'm', weight: 1 }, }; - const refused = refusal(withNodes([createControlLayer('Kept', 'kept'), future], 'control')); + const refused = refusal(withNodes([createControlLayer({ id: 'kept', name: 'Kept' }), future], 'control')); expect(refused.status).toBe('invalid'); expect(refused.status === 'invalid' ? refused.diagnostics : []).toEqual([ @@ -126,7 +130,12 @@ describe('loadCanvasState', () => { it('normalizes an incomplete persisted Z-Image control with backend defaults', () => { const loaded = load( withNodes( - [{ ...createControlLayer('Z Control', 'z-control'), adapter: { kind: 'z_image_control', model: 'z-control' } }], + [ + { + ...createControlLayer({ id: 'z-control', name: 'Z Control' }), + adapter: { kind: 'z_image_control', model: 'z-control' }, + }, + ], 'control' ) ); @@ -146,7 +155,7 @@ describe('loadCanvasState', () => { withNodes( [ { - ...createControlLayer('Q Control', 'q-control'), + ...createControlLayer({ id: 'q-control', name: 'Q Control' }), adapter: { kind: 'qwen_image_control', model: 'qwen-union', weight: -0.5 }, }, ], @@ -382,7 +391,7 @@ describe('loadCanvasState', () => { it('normalizes control adapters in both the live document and saved snapshots', () => { const invalidLayer = { - ...createControlLayer('Z Control', 'z-control'), + ...createControlLayer({ id: 'z-control', name: 'Z Control' }), adapter: { beginEndStepPct: [0.8, 0.2], controlMode: null, kind: 'z_image_control', model: null, weight: -1 }, }; const state = withNodes([invalidLayer], 'control'); @@ -403,7 +412,7 @@ describe('loadCanvasState', () => { const state = createEmptyCanvasState(); const validDocument = { ...state.document, - stacks: stacksFrom([createControlLayer('Snapshot Control', 'snapshot-control')]), + stacks: stacksFrom([createControlLayer({ id: 'snapshot-control', name: 'Snapshot Control' })]), }; const valid = { createdAt: 'now', document: validDocument, id: 'valid', name: 'Valid' }; const base = withNodes([createEmptyPaintLayer('Live', 'live')]); @@ -432,7 +441,7 @@ describe('loadCanvasState', () => { ['missing discriminant and base fields', { id: 'broken' }], ['unknown layer type', { ...createEmptyPaintLayer('Mystery', 'mystery'), type: 'mystery' }], ['raster with invalid source', { ...createEmptyPaintLayer('Raster', 'raster'), source: null }], - ['a leaf of another stack', createControlLayer('Control', 'control')], + ['a leaf of another stack', createControlLayer({ id: 'control', name: 'Control' })], ['a group without children', { ...groupContract('g'), children: null }], ['a group with a malformed child', groupContract('g', [{ ...createEmptyPaintLayer('Bad', 'bad'), opacity: 4 }])], ])('refuses a snapshot containing a malformed node: %s', (_label, malformedNode) => { @@ -460,7 +469,7 @@ describe('loadCanvasState', () => { it('round-trips valid snapshots containing every layer type and nested groups', () => { const state = createEmptyCanvasState(); const stacks = { - control: [groupContract('cg', [createControlLayer('Control', 'control')], { isHidden: true })], + control: [groupContract('cg', [createControlLayer({ id: 'control', name: 'Control' })], { isHidden: true })], inpaint_mask: [createInpaintMaskLayer('Mask', 'mask')], raster: [groupContract('rg', [createEmptyPaintLayer('Raster', 'raster'), groupContract('inner', [])])], regional_guidance: [createRegionalGuidanceLayer('Region', 0, 'region')], @@ -500,7 +509,7 @@ describe('loadCanvasState', () => { it.each([ [ 'a duplicated id across stacks', - { control: [createControlLayer('C', 'dup')], raster: [createEmptyPaintLayer('R', 'dup')] }, + { control: [createControlLayer({ id: 'dup', name: 'C' })], raster: [createEmptyPaintLayer('R', 'dup')] }, 'document.stacks.control[0].id', ], [ @@ -513,7 +522,11 @@ describe('loadCanvasState', () => { { raster: [groupContract('g', [], { isHidden: true })] }, 'document.stacks.raster[0].isHidden', ], - ['a leaf in the wrong stack', { raster: [createControlLayer('C', 'c')] }, 'document.stacks.raster[0]'], + [ + 'a leaf in the wrong stack', + { raster: [createControlLayer({ id: 'c', name: 'C' })] }, + 'document.stacks.raster[0]', + ], [ 'a group nested past the depth limit', { @@ -545,7 +558,12 @@ describe('loadCanvasState', () => { }); it.each([ - ['a future outer version', { ...createEmptyCanvasState(), version: 5 }, 'state', 5], + [ + 'a future outer version', + { ...createEmptyCanvasState(), version: FUTURE_CANVAS_SCHEMA_VERSION }, + 'state', + FUTURE_CANVAS_SCHEMA_VERSION, + ], ['an older outer version', { ...createEmptyCanvasState(), version: 2 }, 'state', 2], ['a legacy version', { ...createEmptyCanvasState(), version: 1 }, 'state', 1], [ @@ -554,14 +572,30 @@ describe('loadCanvasState', () => { 'document', 2, ], + [ + 'a nested document of a future version', + { + ...createEmptyCanvasState(), + document: { ...createEmptyCanvasDocument(), version: FUTURE_CANVAS_SCHEMA_VERSION }, + }, + 'document', + FUTURE_CANVAS_SCHEMA_VERSION, + ], [ 'a snapshot of another version', { ...createEmptyCanvasState(), - snapshots: [{ createdAt: 'now', document: { ...createEmptyCanvasDocument(), version: 5 }, id: 'f', name: 'F' }], + snapshots: [ + { + createdAt: 'now', + document: { ...createEmptyCanvasDocument(), version: FUTURE_CANVAS_SCHEMA_VERSION }, + id: 'f', + name: 'F', + }, + ], }, 'snapshot', - 5, + FUTURE_CANVAS_SCHEMA_VERSION, ], ])('refuses %s before parsing anything', (_label, raw, scope, version) => { expect(refusal(raw)).toEqual({ raw, scope, status: 'unsupported-version', version }); @@ -675,6 +709,131 @@ describe('loadCanvasState', () => { expect(refusal(legacy)).toMatchObject({ scope: 'document', status: 'invalid' }); }); + describe('control layer derivation fields', () => { + const image = (imageName: string) => ({ height: 64, imageName, width: 64 }); + const detection = { settings: { high_threshold: 200, low_threshold: 100 }, type: 'canny_edge_detection' }; + const typed = { + ...createControlLayer({ id: 'edges', name: 'Edges 1' }), + controlType: 'edges', + detection, + filter: detection, + original: { image: image('photo.png'), type: 'image' }, + originalTransform: { rotation: 0, scaleX: 2, scaleY: 2, x: 10, y: 20 }, + source: { bitmap: image('map.png'), offset: { x: 0, y: 0 }, type: 'paint' }, + }; + const legacy = { + ...createControlLayer({ id: 'legacy', name: 'Control Layer 1' }), + filter: detection, + source: { bitmap: image('map.png'), type: 'paint' }, + }; + const omit = (layer: Record, ...keys: string[]) => + Object.fromEntries(Object.entries(layer).filter(([key]) => !keys.includes(key))); + const declaring = (canvas: ReturnType, version: number) => ({ + ...canvas, + document: { ...canvas.document, version }, + version, + }); + /** Loads one control leaf, asserting no diagnostics and returning it with its exact key set. */ + const loadControl = (layer: unknown) => { + const result = loadCanvasState(withNodes([layer], 'control')); + expect(result).toMatchObject({ diagnostics: [], status: 'loaded' }); + return result.status === 'loaded' ? result.value.document.stacks.control[0] : undefined; + }; + const expectExactly = (actual: unknown, expected: Record) => { + expect(actual).toEqual(expected); + expect(Object.keys(actual as object).sort()).toEqual(Object.keys(expected).sort()); + }; + + it('round-trips a typed, detected layer and writes a v3 or v4 canvas as version 5', () => { + for (const declared of [3, 4]) { + const loaded = load(declaring(withNodes([typed], 'control'), declared)); + expectExactly(loaded.document.stacks.control[0], typed); + expect(loaded.version).toBe(5); + expect(loaded.document.version).toBe(5); + } + expect( + load(withNodes([groupContract('outer', [groupContract('inner', [typed as never])])], 'control')).version + ).toBe(5); + }); + + it('keeps a legacy control layer unchanged and at its declared version', () => { + const loaded = load(withNodes([legacy], 'control')); + expectExactly(loaded.document.stacks.control[0], legacy); + expect(loaded.version).toBe(3); + expect(loaded.document.version).toBe(3); + }); + + it('does not adopt version 5 for derivation keys present only as undefined', () => { + const undefinedKeys = { ...legacy, controlType: undefined, detection: undefined, original: undefined }; + expect(writableCanvasSchemaVersion([3], { stacks: withNodes([undefinedKeys], 'control').document.stacks })).toBe( + 3 + ); + expect(load(withNodes([undefinedKeys], 'control')).version).toBe(3); + }); + + it('never lowers a declared version 5 once the fields are gone', () => { + const loaded = load(declaring(withNodes([legacy], 'control'), 5)); + expect(loaded.version).toBe(5); + expect(loaded.document.version).toBe(5); + }); + + it('writes version 5 when only a snapshot holds a typed layer, leaving the live document at its own version', () => { + const loaded = load({ + ...createEmptyCanvasState(), + snapshots: [{ createdAt: 'now', document: withNodes([typed], 'control').document, id: 's', name: 'S' }], + }); + expect(loaded.version).toBe(5); + expect(loaded.document.version).toBe(3); + expect(loaded.snapshots[0]?.document.version).toBe(5); + }); + + it('keeps an unknown detector type and its settings', () => { + const future = { settings: { strength: 3 }, type: 'future_detector' }; + expectExactly(loadControl({ ...typed, detection: future, filter: future }), { + ...typed, + detection: future, + filter: future, + }); + }); + + it('refuses an unknown control type rather than rewriting it', () => { + expect(refusal(withNodes([{ ...typed, controlType: 'future_type' }], 'control'))).toMatchObject({ + scope: 'document', + status: 'invalid', + }); + }); + + const untyped = omit(typed, 'controlType', 'detection', 'original', 'originalTransform'); + const detached = { ...omit(typed, 'original', 'originalTransform'), detection: null }; + it.each([ + ['a null type (dropped, with everything it carried)', { ...typed, controlType: null }, untyped], + ['derivation fields without a type', omit(typed, 'controlType'), untyped], + [ + 'an original paint source without pixels (detached)', + { ...typed, original: { bitmap: null, type: 'paint' } }, + detached, + ], + [ + 'an original that is not a pixel source (detached)', + { ...typed, original: { content: 'x', type: 'text' } }, + detached, + ], + ['an original beside a "None" detection', { ...typed, detection: null }, detached], + [ + 'an original without a detection', + omit(typed, 'detection'), + omit(typed, 'detection', 'original', 'originalTransform'), + ], + [ + 'an original transform without an original', + omit(typed, 'original'), + omit(typed, 'original', 'originalTransform'), + ], + ])('silently repairs %s', (_label, input, expected) => { + expectExactly(loadControl(input), expected); + }); + }); + it('passes a valid state through normalized, keeping every persisted field', () => { const canvas = { ...withNodes([ diff --git a/invokeai/frontend/webv2/src/workbench/canvasMigration.ts b/invokeai/frontend/webv2/src/workbench/canvasMigration.ts index cc1805ea6a6..12c81ae2458 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasMigration.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasMigration.ts @@ -14,7 +14,12 @@ import type { import { getControlAdapterMinWeight } from '@features/generation/graph'; import { createItemId } from '@platform/browser/randomUuid'; -import { CANVAS_MAX_NODE_COUNT, CANVAS_MAX_NODE_DEPTH, LAYER_STACK_ORDER } from '@workbench/canvas-engine/api'; +import { + CANVAS_MAX_NODE_COUNT, + CANVAS_MAX_NODE_DEPTH, + CONTROL_TYPES, + LAYER_STACK_ORDER, +} from '@workbench/canvas-engine/api'; import { repairSelectedLayerId } from '@workbench/canvas-engine/document/selectionRepair'; import { z } from 'zod'; @@ -42,6 +47,8 @@ const zPaintSource = z.object({ offset: zCoordinate.optional(), type: z.literal('paint'), }); +const zImageSource = z.object({ image: zImageRef, type: z.literal('image') }); +const zPixelSource = z.discriminatedUnion('type', [zPaintSource, zImageSource]); const zTextFontRef = z.object({ contentHash: z.string().regex(/^[a-f0-9]{64}$/u), family: z.string(), @@ -50,7 +57,7 @@ const zTextFontRef = z.object({ }); const zLayerSource = z.discriminatedUnion('type', [ zPaintSource, - z.object({ image: zImageRef, type: z.literal('image') }), + zImageSource, z.object({ align: z.enum(['left', 'center', 'right']), color: z.string(), @@ -241,7 +248,12 @@ const zCanvasLayer = z.discriminatedUnion('type', [ }), zLayerBase.extend({ adapter: zControlAdapter, + // An unknown type refuses the document like an unknown adapter kind; detector strings stay open. + controlType: z.enum(CONTROL_TYPES).optional(), + detection: zFilter.nullable().optional(), filter: zFilter.optional(), + original: zPixelSource.optional(), + originalTransform: zTransform.optional(), // Missing display-only visibility means visible for older documents; only overlay types carry it. isHidden: z.boolean().optional(), source: zLayerSource, @@ -359,22 +371,75 @@ const createDefaultStagingArea = (): CanvasStagingAreaContractV2 => ({ selectedImageIndex: 0, }); -/** v4 is adopted only when a document actually carries custom-font typography. */ +type CanvasSchemaVersion = CanvasDocumentContractV3['version']; + +const isSupportedCanvasSchemaVersion = (version: unknown): version is CanvasSchemaVersion => + typeof version === 'number' && + Number.isInteger(version) && + version >= MIN_SUPPORTED_CANVAS_SCHEMA_VERSION && + version <= MAX_SUPPORTED_CANVAS_SCHEMA_VERSION; + +/** The newest of the declared versions; a value outside the supported range counts as the oldest supported. */ +export const newestCanvasSchemaVersion = (versions: Iterable): CanvasSchemaVersion => { + let newest = MIN_SUPPORTED_CANVAS_SCHEMA_VERSION as CanvasSchemaVersion; + for (const version of versions) { + if (isSupportedCanvasSchemaVersion(version) && version > newest) { + newest = version; + } + } + return newest; +}; + const textSourceRequiresV4 = (source: unknown): boolean => isRecord(source) && source.type === 'text' && ('fontRef' in source || 'fontStyle' in source || 'fontVariations' in source); -const nodeRequiresV4 = (node: unknown): boolean => - isRecord(node) && - (textSourceRequiresV4(node.source) || (Array.isArray(node.children) && node.children.some(nodeRequiresV4))); +/** The control derivation fields a v4 reader would strip on load, losing the kept original on its next save. */ +const CONTROL_DERIVATION_KEYS = ['controlType', 'detection', 'original', 'originalTransform'] as const; + +const controlLayerRequiresV5 = (node: Record): boolean => + node.type === 'control' && CONTROL_DERIVATION_KEYS.some((key) => node[key] !== undefined); -/** Returns whether a document uses any v4-only custom-font source fields. */ -export const canvasDocumentRequiresFontSchemaV4 = (document: Pick): boolean => - Object.values(document.stacks).some((roots) => roots.some(nodeRequiresV4)); +/** v4: custom-font typography. v5: a control layer carrying any derivation field. */ +const nodeSchemaRequirement = (node: unknown): CanvasSchemaVersion => { + if (!isRecord(node)) { + return 3; + } + if (controlLayerRequiresV5(node)) { + return 5; + } + let required: CanvasSchemaVersion = textSourceRequiresV4(node.source) ? 4 : 3; + if (Array.isArray(node.children)) { + for (const child of node.children) { + required = Math.max(required, nodeSchemaRequirement(child)) as CanvasSchemaVersion; + if (required === MAX_SUPPORTED_CANVAS_SCHEMA_VERSION) { + break; + } + } + } + return required; +}; -const writableDocumentVersion = (declaredVersion: unknown, stacks: CanvasStackForests): 3 | 4 => - declaredVersion === 4 || canvasDocumentRequiresFontSchemaV4({ stacks }) ? 4 : 3; +/** + * The schema a document is written as: the newest it already declares (adoption is sticky) or the newest its content + * needs, so a build that cannot read that content refuses the document rather than dropping it on load. + */ +export const writableCanvasSchemaVersion = ( + declaredVersions: Iterable, + document: Pick +): CanvasSchemaVersion => { + let version = newestCanvasSchemaVersion(declaredVersions); + for (const roots of Object.values(document.stacks)) { + for (const root of roots) { + if (version === MAX_SUPPORTED_CANVAS_SCHEMA_VERSION) { + return version; + } + version = Math.max(version, nodeSchemaRequirement(root)) as CanvasSchemaVersion; + } + } + return version; +}; export const placementToTransform = ( placement: { x: number; y: number; width: number; height: number }, @@ -504,15 +569,47 @@ interface ParseContext { count: number; } +const isPixelSourceWithPixels = (source: unknown): boolean => + isRecord(source) && (source.type === 'image' || (source.type === 'paint' && isRecord(source.bitmap))); + +/** + * Restores the control derivation invariants (see `CanvasControlLayerContract`) on a fresh leaf copy instead of + * refusing the document: each repair keeps the pixels generation uses and drops only bookkeeping that no longer holds. + */ +const repairControlDerivation = (layer: Record): Record => { + if (layer.controlType === null) { + delete layer.controlType; + } + if (layer.controlType === undefined) { + delete layer.detection; + delete layer.original; + delete layer.originalTransform; + return layer; + } + if (layer.original !== undefined && !isPixelSourceWithPixels(layer.original)) { + delete layer.original; + if (layer.detection !== undefined) { + layer.detection = null; + } + } + if (layer.original !== undefined && !isRecord(layer.detection)) { + delete layer.original; + } + if (layer.original === undefined) { + delete layer.originalTransform; + } + return layer; +}; + /** Fills the known optional defaults on a leaf, then validates it strictly. */ const parseLeaf = (value: Record, path: string, context: ParseContext): CanvasLayerContract | null => { let candidate: Record = value; if (value.type === 'control') { - candidate = { + candidate = repairControlDerivation({ ...value, adapter: normalizeControlAdapter(value.adapter), withTransparencyEffect: value.withTransparencyEffect === undefined ? true : value.withTransparencyEffect, - }; + }); } else if (value.type === 'regional_guidance') { candidate = { ...value, @@ -628,7 +725,7 @@ export const normalizeCanvasDocumentContract = ( ...document, ...normalizeCanvasDocumentGeometry(document.width, document.height, document.bbox), stacks: parsed.stacks, - version: writableDocumentVersion(document.version, parsed.stacks), + version: writableCanvasSchemaVersion([document.version], parsed), } : null; }; @@ -670,7 +767,7 @@ const loadCanvasDocument = ( ...normalizeCanvasDocumentGeometry(value.width, value.height, value.bbox), selectedLayerId, stacks: stacks.stacks, - version: writableDocumentVersion(value.version, stacks.stacks), + version: writableCanvasSchemaVersion([value.version], stacks), }, }; }; @@ -733,12 +830,11 @@ const loadCanvasStateStep = (canvas: unknown): LoadStep = documentRevision: asNumber(canvas.documentRevision, 0), snapshots, stagingArea: normalizeStagingArea(canvas), - version: - canvas.version === MAX_SUPPORTED_CANVAS_SCHEMA_VERSION || - document.value.version === MAX_SUPPORTED_CANVAS_SCHEMA_VERSION || - snapshots.some((snapshot) => snapshot.document.version === MAX_SUPPORTED_CANVAS_SCHEMA_VERSION) - ? MAX_SUPPORTED_CANVAS_SCHEMA_VERSION - : 3, + version: newestCanvasSchemaVersion([ + canvas.version, + document.value.version, + ...snapshots.map((snapshot) => snapshot.document.version), + ]), }, }; }; diff --git a/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.ts b/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.ts index e1d38e5c4d4..5a02d456caf 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasProjectMutations.ts @@ -44,7 +44,11 @@ import { isOverlayStack, layerStackOf, reorderSiblings } from '@workbench/canvas import { repairSelectedLayerId, selectionAfterAcceptedResult } from '@workbench/canvas-engine/document/selectionRepair'; import { prependProjectEvent } from '@workbench/projectEvents'; -import { canvasDocumentRequiresFontSchemaV4, normalizeCanvasDocumentContract } from './canvasMigration'; +import { + newestCanvasSchemaVersion, + normalizeCanvasDocumentContract, + writableCanvasSchemaVersion, +} from './canvasMigration'; import { getCanvasStagingCandidateFingerprint, getCanvasStagingSlotCount, @@ -117,8 +121,7 @@ const withRepairedSelection = (document: CanvasDocumentContractV3): CanvasDocume }; const setCanvasDocument = (project: Project, document: CanvasDocumentContractV3): Project => { - const schemaVersion: 3 | 4 = - project.canvas.version === 4 || document.version === 4 || canvasDocumentRequiresFontSchemaV4(document) ? 4 : 3; + const schemaVersion = writableCanvasSchemaVersion([project.canvas.version, document.version], document); const nextDocument = document.version === schemaVersion ? document : { ...document, version: schemaVersion }; return nextDocument === project.canvas.document && schemaVersion === project.canvas.version ? project @@ -131,8 +134,11 @@ const updateCanvasDocument = ( ): Project => setCanvasDocument(project, update(project.canvas.document)); const setCanvasState = (project: Project, canvas: CanvasStateContractV3): Project => { - const hasV4Snapshot = canvas.snapshots.some((snapshot) => snapshot.document.version === 4); - const schemaVersion: 3 | 4 = canvas.version === 4 || canvas.document.version === 4 || hasV4Snapshot ? 4 : 3; + const schemaVersion = newestCanvasSchemaVersion([ + canvas.version, + canvas.document.version, + ...canvas.snapshots.map((snapshot) => snapshot.document.version), + ]); const nextCanvas = canvas.version === schemaVersion ? canvas : { ...canvas, version: schemaVersion }; return nextCanvas === project.canvas ? project : { ...project, canvas: nextCanvas }; }; diff --git a/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts b/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts index d42b2a22f14..612d1ab8629 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts @@ -1,8 +1,12 @@ /** Oldest canvas document schema this build reads; earlier documents are refused, not migrated. */ export const MIN_SUPPORTED_CANVAS_SCHEMA_VERSION = 3; -/** Newest canvas document schema this build can safely read, edit, and write. */ -export const MAX_SUPPORTED_CANVAS_SCHEMA_VERSION = 4; +/** + * Newest canvas document schema this build can safely read, edit, and write. A document adopts a newer schema only + * when it carries something older builds would drop on load and then lose on save: 4 for custom-font typography, 5 + * for control-layer derivation fields. Older builds refuse the newer schema instead. + */ +export const MAX_SUPPORTED_CANVAS_SCHEMA_VERSION = 5; /** Whether a project whose documents need at least `minimumCanvasSchemaVersion` can be opened here. */ export const isCanvasSchemaVersionSupported = (minimumCanvasSchemaVersion: number): boolean => diff --git a/invokeai/frontend/webv2/src/workbench/projectCanvasGate.test.ts b/invokeai/frontend/webv2/src/workbench/projectCanvasGate.test.ts index 1804adca072..0b8075b0004 100644 --- a/invokeai/frontend/webv2/src/workbench/projectCanvasGate.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projectCanvasGate.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { createEmptyCanvasState } from './canvasMigration'; +import { MAX_SUPPORTED_CANVAS_SCHEMA_VERSION } from './canvasSchemaVersion'; import { gateProjectCanvases } from './projectCanvasGate'; const canvas = createEmptyCanvasState(); @@ -13,14 +14,14 @@ describe('gateProjectCanvases', () => { }); it('refuses a project whose live canvas is unsupported or invalid, keeping the raw document', () => { - const future = { canvas: { ...canvas, version: 5 }, id: 'p', name: 'P' }; + const future = { canvas: { ...canvas, version: MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1 }, id: 'p', name: 'P' }; const broken = { canvas: { ...canvas, version: '2' }, id: 'p', name: 'P' }; expect(gateProjectCanvases(future)).toMatchObject({ projectId: 'p', projectName: 'P', raw: future, - refusal: { scope: 'state', status: 'unsupported-version', version: 5 }, + refusal: { scope: 'state', status: 'unsupported-version', version: MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1 }, source: 'canvas', }); expect(gateProjectCanvases(broken)).toMatchObject({ diff --git a/invokeai/frontend/webv2/src/workbench/projects/README.md b/invokeai/frontend/webv2/src/workbench/projects/README.md index f40e9741497..380dd834d15 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/README.md +++ b/invokeai/frontend/webv2/src/workbench/projects/README.md @@ -84,6 +84,12 @@ Project archives record font dependencies by content hash. Export defaults to re Import verifies declared entries and checksums, then validates all embedded fonts before creating server resources. Fonts uploaded privately; canonical document's references remapped to destination IDs. Duplicate uploads reuse existing private records. Rollback removes only newly created fonts and only when project creation was not attempted or has confirmed absence; ambiguous create response keeps the resources. Font quota failures offer explicit references-only retry. Changing accounts cancels the operation and prevents cleanup under another user's credentials. +## Control layer derivation + +A control layer may keep the image it derives its map from, so detection can re-run. Four optional fields on the control leaf (`canvas-engine/contracts.ts`) carry it: `controlType` (what the image controls; a closed set, so an unknown value refuses the canvas), `detection` (the detector and its settings; `null` means "use the image as-is"; the detector name stays open), `original` (the pre-detection image or paint bitmap, a persisted image reference) and `originalTransform` (the original's frame inside the map's layer frame; absent means identity). `source` stays the pixels generation uses (the map once detected) and `filter` what produced them. Invariants, owned by `canvas-engine/document/controlDerivation.ts`: no type means none of the other three; a `null` detection keeps no original; an original implies a detection and a derived `source`. Load repairs violations silently by dropping the bookkeeping that no longer holds, never the pixels. + +A build that predates these fields would strip them on load and lose the original on its next save, so a control leaf carrying any of them makes the canvas schema 5 (`canvasMigration.ts`, `writableCanvasSchemaVersion`). Documents without them keep their version, and adoption is sticky like the server's floor. `original` names an image by `imageName` like every other reference, so asset holding, cleanup and `.invk` export cover it unchanged. + ## Verification Run `pnpm lint`, `pnpm test`, `pnpm test:browser`, and `pnpm run test:performance:build` from webv2. Queue receipt tests also cover backend admission, account isolation, project deletion, partial acceptance, idempotent retries. Browser tests exercise real IndexedDB transactions and Web Locks. diff --git a/invokeai/frontend/webv2/src/workbench/projects/api.test.ts b/invokeai/frontend/webv2/src/workbench/projects/api.test.ts index d6a14e0f3bc..acde0b97948 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/api.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projects/api.test.ts @@ -42,7 +42,7 @@ describe('createProjectSettled', () => { const post = () => ({ body: JSON.stringify({ ...request, - max_canvas_schema_version: 4, + max_canvas_schema_version: 5, max_document_schema_version: 4, minimum_canvas_schema_version: 3, }), @@ -93,7 +93,7 @@ describe('createProjectSettled', () => { expect(transport.apiFetchJson).toHaveBeenNthCalledWith(2, '/api/v1/projects/', post()); expect(transport.apiFetchJson).toHaveBeenNthCalledWith( 3, - '/api/v1/projects/project-1?max_canvas_schema_version=4&max_document_schema_version=4', + '/api/v1/projects/project-1?max_canvas_schema_version=5&max_document_schema_version=4', expect.anything() ); }); @@ -173,7 +173,7 @@ describe('createProjectSettled', () => { await expect(createProjectSettled(request, captureAccountScope())).resolves.toMatchObject(project); expect(transport.apiFetchJson).toHaveBeenNthCalledWith( 3, - '/api/v1/projects/project-1?max_canvas_schema_version=4&max_document_schema_version=4', + '/api/v1/projects/project-1?max_canvas_schema_version=5&max_document_schema_version=4', expect.anything() ); @@ -265,7 +265,7 @@ describe('canvas schema compatibility declarations', () => { await getProject('project/one'); expect(transport.apiFetchJson).toHaveBeenCalledWith( - '/api/v1/projects/project%2Fone?max_canvas_schema_version=4&max_document_schema_version=4', + '/api/v1/projects/project%2Fone?max_canvas_schema_version=5&max_document_schema_version=4', { signal: undefined } ); }); @@ -280,7 +280,7 @@ describe('canvas schema compatibility declarations', () => { data: {}, minimum_document_schema_version: 4, name: 'Project', - max_canvas_schema_version: 4, + max_canvas_schema_version: 5, max_document_schema_version: 4, minimum_canvas_schema_version: 3, }), @@ -305,7 +305,7 @@ describe('canvas schema compatibility declarations', () => { expected_revision: 4, minimum_canvas_schema_version: 3, name: 'Project', - max_canvas_schema_version: 4, + max_canvas_schema_version: 5, max_document_schema_version: 4, }), method: 'PUT', @@ -507,7 +507,7 @@ describe('project workspaces', () => { body: JSON.stringify({ data: { layout: {} }, expected_revision: 1, - max_canvas_schema_version: 4, + max_canvas_schema_version: 5, max_document_schema_version: 4, }), method: 'PUT', @@ -535,7 +535,7 @@ describe('project workspaces', () => { minimum_document_schema_version: 4, name: 'Project', workspace, - max_canvas_schema_version: 4, + max_canvas_schema_version: 5, max_document_schema_version: 4, }), method: 'PUT', @@ -581,7 +581,7 @@ describe('project workspaces', () => { minimum_document_schema_version: 4, name: 'Project', workspace: { layout: {} }, - max_canvas_schema_version: 4, + max_canvas_schema_version: 5, max_document_schema_version: 4, minimum_canvas_schema_version: 3, }), diff --git a/invokeai/frontend/webv2/src/workbench/projects/library.test.ts b/invokeai/frontend/webv2/src/workbench/projects/library.test.ts index 89b8408a676..3bd51a75737 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/library.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projects/library.test.ts @@ -1,6 +1,7 @@ import type { BrowserLockResult } from '@platform/browser/webLocks'; import type * as accountLifecycleModule from '@platform/state/accountLifecycle'; +import { MAX_SUPPORTED_CANVAS_SCHEMA_VERSION } from '@workbench/canvasSchemaVersion'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import type * as libraryModule from './library'; @@ -147,7 +148,10 @@ describe('refreshProjectLibrary', () => { it('identifies summaries that require a newer canvas reader without fetching the document', async () => { api.listProjects.mockResolvedValue([ - { ...summaryDto('future', 'Future', '2026-06-10 10:00:00.000'), minimum_canvas_schema_version: 5 }, + { + ...summaryDto('future', 'Future', '2026-06-10 10:00:00.000'), + minimum_canvas_schema_version: MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1, + }, ]); await library.refreshProjectLibrary(); diff --git a/invokeai/frontend/webv2/src/workbench/projects/projectAssets.test.ts b/invokeai/frontend/webv2/src/workbench/projects/projectAssets.test.ts index c35b6c2b412..b3a7e35ccc3 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/projectAssets.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projects/projectAssets.test.ts @@ -246,6 +246,23 @@ describe('collectLiveAssetRefs', () => { expect(images).toEqual(new Set(['mask.png', 'reference.png', 'graph-image.png', 'upscale-input.png'])); }); + it("holds a control layer's pre-detection original beside its derived map", () => { + const control = { + controlType: 'depth', + detection: { settings: {}, type: 'depth_anything_depth_estimation' }, + id: 'control-1', + original: { image: imageRef('photo.png'), type: 'image' }, + source: { bitmap: imageRef('depth-map.png'), type: 'paint' }, + type: 'control', + }; + const document = projectDocument({ canvas: { document: { stacks: { control: [control] } } } }); + + expect(collectLiveAssetRefs(document).images).toEqual( + new Set(['photo.png', 'depth-map.png', 'graph-image.png', 'upscale-input.png']) + ); + expect(collectHeldAssetRefs([document]).images.has('photo.png')).toBe(true); + }); + it('ignores empty names and non-string values', () => { const refs = collectLiveAssetRefs({ canvas: { imageName: '' }, settings: { image_name: 7, video_name: null } }); diff --git a/invokeai/frontend/webv2/src/workbench/projects/projectFile.test.ts b/invokeai/frontend/webv2/src/workbench/projects/projectFile.test.ts index ded7b9e26ee..40faafba9aa 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/projectFile.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projects/projectFile.test.ts @@ -1,4 +1,5 @@ import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { MAX_SUPPORTED_CANVAS_SCHEMA_VERSION } from '@workbench/canvasSchemaVersion'; import { createDraftProject } from '@workbench/workbenchState'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -658,7 +659,10 @@ describe('importing a project board', () => { const rewrittenEntries = new Map([...entries].map(([path, bytes]) => [path, binaryEntry(bytes)])); - rewrittenEntries.set('manifest.json', textEntry(JSON.stringify({ ...manifest, minimumCanvasSchemaVersion: 5 }))); + rewrittenEntries.set( + 'manifest.json', + textEntry(JSON.stringify({ ...manifest, minimumCanvasSchemaVersion: MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1 })) + ); const archive = new File([await writeArchive(rewrittenEntries)], original.name); await expect(projectFile.importProjectFile(archive)).rejects.toMatchObject({ reason: 'unsupported-version' }); diff --git a/invokeai/frontend/webv2/src/workbench/projects/projectSync.test.ts b/invokeai/frontend/webv2/src/workbench/projects/projectSync.test.ts index 3c5c4051db6..ce9c6822bee 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/projectSync.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projects/projectSync.test.ts @@ -1,5 +1,6 @@ import type { Project } from '@workbench/projectContracts'; +import { MAX_SUPPORTED_CANVAS_SCHEMA_VERSION } from '@workbench/canvasSchemaVersion'; import { addProjectWorkflow, getActiveProjectGraph, recordProjectWorkflowRun } from '@workbench/projectWorkflows'; import { createDraftProject } from '@workbench/workbenchState'; import { createInitialWorkbenchState, workbenchReducer } from '@workbench/workbenchState.testing'; @@ -133,7 +134,10 @@ describe('project document serialization', () => { it('refuses a document whose canvas was written by a newer client, keeping the raw document', () => { const project = getProject(); const document = serializeProjectDocument(project); - const future = { ...document, canvas: { ...(document.canvas as object), version: 5 } }; + const future = { + ...document, + canvas: { ...(document.canvas as object), version: MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1 }, + }; const result = deserializeProjectDocument(future); @@ -141,7 +145,7 @@ describe('project document serialization', () => { refused: { projectId: project.id, raw: future, - refusal: { scope: 'state', status: 'unsupported-version', version: 5 }, + refusal: { scope: 'state', status: 'unsupported-version', version: MAX_SUPPORTED_CANVAS_SCHEMA_VERSION + 1 }, source: 'canvas', }, status: 'refused', diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/canvasHotkeyCommands.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/canvasHotkeyCommands.test.ts index 777d98baf11..3e7c12221ba 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/canvasHotkeyCommands.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/canvasHotkeyCommands.test.ts @@ -9,10 +9,9 @@ import type { } from '@workbench/canvas-engine/api'; import type { CanvasProjectMutation } from '@workbench/canvasProjectMutations'; -import { createDocumentModel } from '@workbench/canvas-engine/api'; +import { createControlLayer, createDocumentModel } from '@workbench/canvas-engine/api'; import { groupContract, stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { stackTopAnchor } from '@workbench/canvas-engine/document/insertionAnchors.testStub'; -import { createControlLayer } from '@workbench/widgets/layers/layerOps'; import { beforeEach, describe, expect, it, vi, type Mock } from 'vitest'; import { executeCanvasHotkeyCommand, type CanvasHotkeyContext } from './canvasHotkeyCommands'; @@ -161,7 +160,7 @@ describe('staging precedence', () => { describe('layer navigation without staging', () => { const select = (id: string): CanvasProjectMutation => ({ id, type: 'setCanvasSelectedLayer' }); // Document order puts the control stack above the raster stack: b, a, c. - const layers = [rasterLayer('a'), createControlLayer('Control', 'b'), rasterLayer('c')]; + const layers = [rasterLayer('a'), createControlLayer({ id: 'b', name: 'Control' }), rasterLayer('c')]; it.each([ ['canvas.nextEntity', 'a', 'c'], @@ -659,7 +658,7 @@ describe('group selection', () => { ...documentOf([], selectedLayerId), stacks: stacksFrom(nodes), }); - const control = (id: string): CanvasLayerContract => createControlLayer(id, id); + const control = (id: string): CanvasLayerContract => createControlLayer({ id, name: id }); it('reorders, deletes and duplicates a selected group', () => { const document = nodeDocument([groupContract('g', [rasterLayer('a')]), rasterLayer('b')], 'g'); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/SelectionOptionsRow.test.tsx b/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/SelectionOptionsRow.test.tsx index b648af9c6aa..4adf1fe4896 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/SelectionOptionsRow.test.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/tool-options/SelectionOptionsRow.test.tsx @@ -5,8 +5,9 @@ import type { ReactNode } from 'react'; import { ChakraProvider } from '@chakra-ui/react'; import { system } from '@theme/system'; +import { createControlLayer } from '@workbench/canvas-engine/api'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; -import { createControlLayer, createEmptyPaintLayer } from '@workbench/widgets/layers/layerOps'; +import { createEmptyPaintLayer } from '@workbench/widgets/layers/layerOps'; import { createDraftProject } from '@workbench/workbenchState'; import { renderToStaticMarkup } from 'react-dom/server'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -81,14 +82,14 @@ beforeEach(() => { describe('SelectionActions select all', () => { it('stays available with no selection and no eligible layer', () => { hasSelection.current = false; - const captured = renderRow(createControlLayer('Control', 'control-1', 'sd-1', null)); + const captured = renderRow(createControlLayer({ base: 'sd-1', id: 'control-1', name: 'Control' })); expect(captured.get('widgets.canvas.toolOptions.selectAll')?.['aria-disabled']).toBe(false); }); }); describe('SelectionActions pixel target eligibility', () => { const rasterPaint = createEmptyPaintLayer('Raster', 'raster'); - const controlPaint = createControlLayer('Control', 'control'); + const controlPaint = createControlLayer({ id: 'control', name: 'Control' }); const rasterImage: CanvasLayerContract = { ...createEmptyPaintLayer('Raster image', 'raster-image'), source: { image: { height: 10, imageName: 'raster-image', width: 10 }, type: 'image' }, diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.browser.test.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.browser.test.tsx index fc5047aa9e9..f68d399db39 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.browser.test.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.browser.test.tsx @@ -12,7 +12,7 @@ import { accountLifecycle } from '@platform/state/accountLifecycle'; import { toaster } from '@platform/ui/toaster'; import { applyThemeToRoot } from '@theme/applyTheme'; import { system } from '@theme/system'; -import { createDocumentModel } from '@workbench/canvas-engine/api'; +import { createControlLayer, createDocumentModel } from '@workbench/canvas-engine/api'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { previewInverse } from '@workbench/canvas-engine/document-model/documentModel'; import { attachCanvasOperations } from '@workbench/canvas-operations/operationAccess'; @@ -25,7 +25,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { page, userEvent } from 'vitest/browser'; import { ControlLayerSettings } from './ControlLayerSettings'; -import { createControlLayer, createEmptyPaintLayer } from './layerOps'; +import { createEmptyPaintLayer } from './layerOps'; // Stable identities, as the real stores hand out, so only the table's arrival can re-read the policy. const catalog = vi.hoisted(() => ({ @@ -45,6 +45,12 @@ vi.mock('@features/models', async (importOriginal) => ({ })); vi.mock('./useSelectedMainModel', () => ({ useSelectedMainModel: () => catalog.mainModel })); +/** A control layer on `base`'s adapter defaults with `model` already chosen. */ +const withModel = (name: string, id: string, base: string, model: string | null): CanvasControlLayerContract => { + const created = createControlLayer({ base, id, name }); + return { ...created, adapter: { ...created.adapter, model } }; +}; + const getArchitectureCapabilities = vi.fn<() => Promise>(); vi.mock('@features/generation/data/architectureCapabilitiesApi', () => ({ getArchitectureCapabilities: () => getArchitectureCapabilities(), @@ -320,7 +326,7 @@ describe('ControlLayerSettings on an Anima main model', () => { id = 'control-1', values: Partial = {} ): CanvasControlLayerContract => { - const base = createControlLayer(id, id, 'anima', model); + const base = withModel(id, id, 'anima', model); return { ...base, adapter: { ...base.adapter, ...values, kind: kind as CanvasControlLayerContract['adapter']['kind'] }, @@ -493,7 +499,7 @@ describe('ControlLayerSettings on a Qwen Image main model', () => { { base: 'z-image', key: 'z-union', name: 'Z-Image ControlNet Union', type: 'controlnet' }, ]; /** A layer as the Layers panel creates it under a Qwen Image main model. */ - const qwenLayer = (id: string) => createControlLayer(id, id, 'qwen-image', 'qwen-union'); + const qwenLayer = (id: string) => withModel(id, id, 'qwen-image', 'qwen-union'); it('offers the Qwen Image ControlNet kind and its models, and accepts one without a warning', async () => { await renderWith(QWEN_MAIN, QWEN_MODELS, qwenLayer('control-1')); @@ -534,11 +540,7 @@ describe('ControlLayerSettings on a FLUX.2 main model', () => { ]; it('offers the FLUX.2 ControlNet kind and its models on [dev], and accepts one without a warning', async () => { - await renderWith( - FLUX2_DEV_MAIN, - FLUX2_MODELS, - createControlLayer('control-1', 'control-1', 'flux2', 'flux2-union') - ); + await renderWith(FLUX2_DEV_MAIN, FLUX2_MODELS, withModel('control-1', 'control-1', 'flux2', 'flux2-union')); expect(offersKind('flux2_control')).toBe(true); expect(offersKind('controlnet')).toBe(false); @@ -552,7 +554,7 @@ describe('ControlLayerSettings on a FLUX.2 main model', () => { catalog.models = FLUX2_MODELS; getArchitectureCapabilities.mockResolvedValueOnce(architectureCapabilitiesFixture); await settle(ensureArchitectureCapabilitiesLoaded); - const fresh = createControlLayer('control-1', 'control-1', 'flux2', null); + const fresh = withModel('control-1', 'control-1', 'flux2', null); const engine = engineWith([fresh]); engine.exports.hasExportableLayerContent = () => false; @@ -565,7 +567,7 @@ describe('ControlLayerSettings on a FLUX.2 main model', () => { describe('ControlLayerSettings weight', () => { // A full document layer (the panel reads its leaf), weighted 1 so a drag can move either way. const weighted = ((): CanvasControlLayerContract => { - const base = createControlLayer('Control 1', 'control-1', 'sd-1', 'sd1-controlnet'); + const base = withModel('Control 1', 'control-1', 'sd-1', 'sd1-controlnet'); return { ...base, adapter: { ...base.adapter, weight: 1 } }; })(); const weightSlider = () => page.getByRole('slider', { name: 'widgets.layers.control.weight' }); @@ -676,7 +678,7 @@ describe('ControlLayerSettings model choice', () => { name: 'SD 1.5 Canny', type: 'controlnet', }; - const modelless = createControlLayer('Control 1', 'control-1', 'sd-1', null); + const modelless = withModel('Control 1', 'control-1', 'sd-1', null); const chooseCanny = async (hasContent: boolean) => { catalog.models = [CANNY]; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerContextMenu.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerContextMenu.tsx index 177f2033432..77ec0dbecbc 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerContextMenu.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerContextMenu.tsx @@ -17,7 +17,6 @@ import type { ComponentProps, Dispatch, ReactNode } from 'react'; import { HStack, Icon, Menu, Portal, Text } from '@chakra-ui/react'; import { galleryTransfers } from '@features/gallery'; -import { useModelsSelector } from '@features/models'; import { canCopyImage } from '@platform/browser/clipboard'; import { IconButton, MenuActionItem, MenuContent, MenuIconItem, RenameDialog, Tooltip } from '@platform/ui'; import { useQueryClient } from '@tanstack/react-query'; @@ -73,7 +72,6 @@ import type { } from './layerContextMenuLayout'; import type { LayerMenuDialogKind, LayerMenuDialogState } from './layerMenuState'; -import { resolveDefaultControlModelForBase } from './controlModelOptions'; import { actionTargets, getLayerContextActions, @@ -238,8 +236,6 @@ const LayerMenu = ({ const notify = useNotify(); const queryClient = useQueryClient(); const base = useSelectedModelBase(); - const models = useModelsSelector((snapshot) => snapshot.models); - const defaultControlModel = useMemo(() => resolveDefaultControlModelForBase(models, base), [base, models]); const workflowAvailability = useLayerWorkflowAvailability(); const canvas = useActiveProjectSelector((project) => project.canvas); const queueItems = useActiveProjectSelector((project) => project.queue.items); @@ -463,7 +459,7 @@ const LayerMenu = ({ (targetType: CanvasLayerContract['type'], label: string) => { const converted = layer.type === 'raster' && targetType === 'control' - ? convertRasterToControl(layer, base, defaultControlModel) + ? convertRasterToControl(layer, base) : layer.type === 'raster' && targetType === 'inpaint_mask' ? convertRasterToInpaintMask(layer) : layer.type === 'raster' && targetType === 'regional_guidance' @@ -480,7 +476,7 @@ const LayerMenu = ({ throw makeStatusError('not-ready'); } }, - [base, defaultControlModel, engine, layer, makeStatusError] + [base, engine, layer, makeStatusError] ); const handleToggleVisibility = useCallback(() => { @@ -718,12 +714,9 @@ const LayerMenu = ({ const handleCopyToControl = useCallback(() => { if (layer.type === 'raster') { - addCopy( - copyRasterToControl(layer, createLayerId(), base, defaultControlModel), - getActionLabel('copy-to-control') - ); + addCopy(copyRasterToControl(layer, createLayerId(), base), getActionLabel('copy-to-control')); } - }, [addCopy, base, defaultControlModel, getActionLabel, layer]); + }, [addCopy, base, getActionLabel, layer]); const handleCopyToInpaintMask = useCallback(() => { const id = createLayerId(); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerFilterOperationButton.test.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerFilterOperationButton.test.tsx index d65e4efe5cd..6ba4456cc15 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerFilterOperationButton.test.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerFilterOperationButton.test.tsx @@ -4,7 +4,8 @@ import type { ReactNode } from 'react'; import { ChakraProvider } from '@chakra-ui/react'; import { system } from '@theme/system'; -import { createControlLayer, createEmptyPaintLayer } from '@workbench/widgets/layers/layerOps'; +import { createControlLayer } from '@workbench/canvas-engine/api'; +import { createEmptyPaintLayer } from '@workbench/widgets/layers/layerOps'; import { renderToStaticMarkup } from 'react-dom/server'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -138,7 +139,7 @@ describe('LayerFilterOperationButton eligibility', () => { it.each([ ['raster', createEmptyPaintLayer('Raster', 'raster')], - ['control', createControlLayer('Control', 'control')], + ['control', createControlLayer({ id: 'control', name: 'Control' })], ] as const)('enables a filterable %s layer', (_type, layer) => { const { engine } = createEngine(true); const { button, tooltip } = renderButton(engine, layer); @@ -177,7 +178,7 @@ describe('LayerFilterOperationButton launch interaction', () => { ['unsupported', 'widgets.layers.actions.unsupported'], ['not-ready', 'widgets.layers.actions.notReady'], ] as const)('keeps properties open and reports an engine %s rejection', (result, reasonKey) => { - const layer = createControlLayer('Control', 'control'); + const layer = createControlLayer({ id: 'control', name: 'Control' }); const { engine, startFilterOperation } = createEngine(true, () => result); const onOperationStarted = vi.fn(); const { button } = renderButton(engine, layer, onOperationStarted); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerFilterOperationDelegation.test.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerFilterOperationDelegation.test.tsx index f9b6753c433..a738a8114c2 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerFilterOperationDelegation.test.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/LayerFilterOperationDelegation.test.tsx @@ -6,7 +6,8 @@ import type { import { ChakraProvider } from '@chakra-ui/react'; import { system } from '@theme/system'; -import { createControlLayer, createEmptyPaintLayer } from '@workbench/widgets/layers/layerOps'; +import { createControlLayer } from '@workbench/canvas-engine/api'; +import { createEmptyPaintLayer } from '@workbench/widgets/layers/layerOps'; import { renderToStaticMarkup } from 'react-dom/server'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -72,7 +73,7 @@ describe('layer filter launch delegation', () => { }); it('delegates control-layer launch rendering to the shared control with the original props', () => { - const layer = createControlLayer('Control', 'control'); + const layer = createControlLayer({ id: 'control', name: 'Control' }); const onOperationStarted = vi.fn(); renderToStaticMarkup( diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/controlModelOptions.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/controlModelOptions.test.ts index 08ad765c51e..10139600fa5 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/controlModelOptions.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/controlModelOptions.test.ts @@ -7,8 +7,6 @@ import { describe, expect, it } from 'vitest'; import { getCompatibleControlModels, getReidentifyPendingControlModels, - resolveDefaultControlModel, - resolveDefaultControlModelForBase, switchControlAdapterKind, } from './controlModelOptions'; @@ -85,10 +83,6 @@ describe('getCompatibleControlModels — Anima ControlNet-LLLite', () => { expect(getCompatibleControlModels(models, null, 'anima_lllite')).toEqual([]); }); - it('defaults a new Anima control layer to an LLLite model', () => { - expect(resolveDefaultControlModelForBase(models, 'anima')).toBe('sketch'); - }); - it('lists only Anima adapters of unknown type as needing re-identify, for the LLLite kind', () => { const others = [ ...models, @@ -103,10 +97,10 @@ describe('getCompatibleControlModels — Anima ControlNet-LLLite', () => { expect(getReidentifyPendingControlModels(others, 'sdxl', 'anima_lllite')).toEqual([]); }); - it('never defaults a layer to an adapter that needs re-identifying', () => { + it('never offers an adapter that needs re-identifying', () => { expect( - resolveDefaultControlModelForBase([anima('unidentified', null), anima('inpainting', 4)], 'anima') - ).toBeNull(); + getCompatibleControlModels([anima('unidentified', null), anima('inpainting', 4)], 'anima', 'anima_lllite') + ).toEqual([]); }); }); @@ -156,54 +150,3 @@ describe('switchControlAdapterKind', () => { expect(switchControlAdapterKind(savedAsControlNet('sdxl-control'), 't2i_adapter', [sdxl], 'sdxl').model).toBeNull(); }); }); - -describe('resolveDefaultControlModel', () => { - it('prefers a union model, then tile, then the first compatible', () => { - const canny = model('canny', 'sdxl', 'controlnet'); - const tile = { ...model('tile-model', 'sdxl', 'controlnet'), name: 'ControlNet Tile' } as ModelConfig; - const union = { ...model('union-model', 'sdxl', 'controlnet'), name: 'Union Pro XL' } as ModelConfig; - - expect(resolveDefaultControlModel([canny, tile, union], 'sdxl', 'controlnet')).toBe('union-model'); - expect(resolveDefaultControlModel([canny, tile], 'sdxl', 'controlnet')).toBe('tile-model'); - expect(resolveDefaultControlModel([canny], 'sdxl', 'controlnet')).toBe('canny'); - }); - - it('only considers models compatible with the base and kind', () => { - const models = [ - { ...model('sd1-union', 'sd-1', 'controlnet'), name: 'Union' } as ModelConfig, - model('sdxl-canny', 'sdxl', 'controlnet'), - ]; - - expect(resolveDefaultControlModel(models, 'sdxl', 'controlnet')).toBe('sdxl-canny'); - expect(resolveDefaultControlModel(models, 'sdxl', 't2i_adapter')).toBeNull(); - expect(resolveDefaultControlModel([], 'sdxl', 'controlnet')).toBeNull(); - }); -}); - -describe('resolveDefaultControlModelForBase', () => { - it('resolves the z_image_control kind for the z-image base and controlnet otherwise', () => { - const models = [model('z-control', 'z-image', 'controlnet'), model('sd-control', 'sd-1', 'controlnet')]; - - expect(resolveDefaultControlModelForBase(models, 'z-image')).toBe('z-control'); - expect(resolveDefaultControlModelForBase(models, 'sd-1')).toBe('sd-control'); - expect(resolveDefaultControlModelForBase(models, 'sd-2')).toBeNull(); - }); - - it('starts a Qwen-Image control layer on the union ControlNet', () => { - const models = [ - model('qwen-canny', 'qwen-image', 'controlnet'), - { ...model('qwen-union', 'qwen-image', 'controlnet'), name: 'Qwen Image ControlNet Union' } as ModelConfig, - ]; - - expect(resolveDefaultControlModelForBase(models, 'qwen-image')).toBe('qwen-union'); - }); - - it('starts a FLUX.2 control layer on the union ControlNet of its own base', () => { - const models = [ - { ...model('flux-union', 'flux', 'controlnet'), name: 'FLUX.1 ControlNet Union' } as ModelConfig, - { ...model('flux2-union', 'flux2', 'controlnet'), name: 'FLUX.2 [dev] ControlNet Union' } as ModelConfig, - ]; - - expect(resolveDefaultControlModelForBase(models, 'flux2')).toBe('flux2-union'); - }); -}); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/controlModelOptions.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/controlModelOptions.ts index b2acddc05a3..4cddc2032bc 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/controlModelOptions.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/controlModelOptions.ts @@ -7,7 +7,7 @@ import { isControlModelUsableForKind, type ControlAdapterKind, } from '@features/generation/graph'; -import { CONTROL_ADAPTER_DEFAULTS, CONTROL_KIND_BASE, getDefaultControlAdapterKind } from '@workbench/controlAdapters'; +import { CONTROL_ADAPTER_DEFAULTS, CONTROL_KIND_BASE } from '@workbench/controlAdapters'; export const getCompatibleControlModels = ( models: readonly ModelConfig[], @@ -64,25 +64,3 @@ export const switchControlAdapterKind = ( weight: keepsWeight ? adapter.weight : defaults.weight, }; }; - -/** - * The control model a freshly created layer should start with: a union model - * (the generalist), then tile, then the first compatible model. Null when no - * compatible model is installed. - */ -export const resolveDefaultControlModel = ( - models: readonly ModelConfig[], - base: string | null, - kind: ControlAdapterKind -): string | null => { - const compatible = getCompatibleControlModels(models, base, kind); - const preferred = - compatible.find((model) => model.name.toLowerCase().includes('union')) ?? - compatible.find((model) => model.name.toLowerCase().includes('tile')) ?? - compatible[0]; - return preferred?.key ?? null; -}; - -/** Default model for the adapter kind the layer factories pick for `base`. */ -export const resolveDefaultControlModelForBase = (models: readonly ModelConfig[], base: string | null): string | null => - resolveDefaultControlModel(models, base, getDefaultControlAdapterKind(base)); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.test.ts index 0f11db2d47c..ac8127f8464 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextActions.test.ts @@ -5,6 +5,7 @@ import type { } from '@workbench/canvas-engine/contracts'; import { seedArchitectureCapabilities } from '@features/generation/core/architectureCapabilities.testing'; +import { createControlLayer } from '@workbench/canvas-engine/api'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { describe, expect, it, vi } from 'vitest'; @@ -18,12 +19,7 @@ import { type LayerContextActionRuntimeContext, type LayerContextActionState, } from './layerContextActions'; -import { - createControlLayer, - createEmptyPaintLayer, - createInpaintMaskLayer, - createRegionalGuidanceLayer, -} from './layerOps'; +import { createEmptyPaintLayer, createInpaintMaskLayer, createRegionalGuidanceLayer } from './layerOps'; seedArchitectureCapabilities(); @@ -41,7 +37,7 @@ const paintLayer = (id: string, patch: Partial = {} const rasterLayer = paintLayer('raster'); const nonEmptyControlLayer = { - ...createControlLayer('Control', 'control'), + ...createControlLayer({ id: 'control', name: 'Control' }), source: { image: { height: 10, imageName: 'control-image', width: 10 }, type: 'image' as const }, }; @@ -768,7 +764,7 @@ describe('getLayerContextActions', () => { }); it('exposes control-only transparency and raster conversion actions while filtering supported control content', () => { - const empty = createControlLayer('Control', 'control-empty'); + const empty = createControlLayer({ id: 'control-empty', name: 'Control' }); expect(idsFor(empty, [empty], { hasSupportedContent: false })).toEqual( expect.arrayContaining([ diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextMenuLayout.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextMenuLayout.test.ts index 3092db9bb5b..18fd0c2a708 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextMenuLayout.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerContextMenuLayout.test.ts @@ -4,6 +4,7 @@ import type { CanvasRasterLayerContractV2, } from '@workbench/canvas-engine/contracts'; +import { createControlLayer } from '@workbench/canvas-engine/api'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { describe, expect, it } from 'vitest'; @@ -15,12 +16,7 @@ import { getLayerContextMenuRenderEntries, type LayerContextMenuSection, } from './layerContextMenuLayout'; -import { - createControlLayer, - createEmptyPaintLayer, - createInpaintMaskLayer, - createRegionalGuidanceLayer, -} from './layerOps'; +import { createEmptyPaintLayer, createInpaintMaskLayer, createRegionalGuidanceLayer } from './layerOps'; const paintLayer = (id: string, patch: Partial = {}): CanvasLayerContract => ({ ...createEmptyPaintLayer(id, id), @@ -139,7 +135,7 @@ describe('getLayerContextMenuLayout', () => { it('places control filter and transparency actions before conversion operations', () => { const control = { - ...createControlLayer('Control', 'control'), + ...createControlLayer({ id: 'control', name: 'Control' }), source: { image: { height: 10, imageName: 'control-image', width: 10 }, type: 'image' as const }, }; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.test.ts index 652cbc40d97..10cda30e0f9 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.test.ts @@ -5,6 +5,7 @@ import type { } from '@workbench/canvas-engine/contracts'; import { seedArchitectureCapabilities } from '@features/generation/core/architectureCapabilities.testing'; +import { createControlLayer } from '@workbench/canvas-engine/api'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { describe, expect, it } from 'vitest'; @@ -23,7 +24,6 @@ import { copyRasterToInpaintMask, copyRasterToRegionalGuidance, copyRegionalGuidanceToInpaintMask, - createControlLayer, createEmptyPaintLayer, createInpaintMaskFromImage, createInpaintMaskLayer, @@ -39,7 +39,6 @@ import { getControlTransparencyEffectPatch, getRegionalGuidanceAutoNegativePatch, isMergeableRasterLayer, - nextControlLayerName, nextInpaintMaskName, nextRegionalGuidanceFillColor, REGIONAL_GUIDANCE_FILL_COLORS, @@ -273,7 +272,7 @@ describe('nextInpaintMaskName', () => { }); }); -describe('createControlLayer', () => { +describe('control adapter defaults', () => { it('centralizes legacy defaults for every adapter kind', () => { expect(CONTROL_ADAPTER_DEFAULTS).toEqual({ anima_lllite: { beginEndStepPct: [0, 1], controlMode: null, kind: 'anima_lllite', model: null, weight: 1 }, @@ -308,54 +307,12 @@ describe('createControlLayer', () => { expect(DEFAULT_CONTROL_ADAPTER.weight).toBe(0.75); expect(CONTROL_WEIGHT_BOUNDS).toEqual({ inputMax: 2, inputMin: -1, sliderMax: 2, sliderMin: 0, step: 0.05 }); }); - - it('builds an empty control layer with the legacy-default ControlNet adapter and transparency on', () => { - const layer = createControlLayer('Control Layer 1', 'c1'); - expect(layer).toMatchObject({ - id: 'c1', - isEnabled: true, - name: 'Control Layer 1', - type: 'control', - withTransparencyEffect: true, - }); - expect(layer.source).toEqual({ bitmap: null, type: 'paint' }); - expect(layer.adapter).toEqual(DEFAULT_CONTROL_ADAPTER); - expect(layer.adapter.weight).toBe(0.75); - // The default adapter's mutable array must be copied, not shared. - expect(layer.adapter.beginEndStepPct).not.toBe(DEFAULT_CONTROL_ADAPTER.beginEndStepPct); - expect(layer.adapter.beginEndStepPct).toEqual([0, 0.75]); - }); - - it('uses Z-Image control defaults when created for a Z-Image main model', () => { - expect(createControlLayer('Z Control', 'z1', 'z-image').adapter).toEqual(CONTROL_ADAPTER_DEFAULTS.z_image_control); - }); - - it('uses Anima ControlNet-LLLite defaults when created for an Anima main model', () => { - expect(createControlLayer('Anima Control', 'a1', 'anima', 'sketch').adapter).toEqual({ - ...CONTROL_ADAPTER_DEFAULTS.anima_lllite, - model: 'sketch', - }); - }); - - it('seeds the provided default control model and stays null without one', () => { - expect(createControlLayer('C', 'c1', 'sdxl', 'union-model').adapter.model).toBe('union-model'); - expect(createControlLayer('C', 'c1', 'sdxl').adapter.model).toBeNull(); - expect(createControlLayer('Z', 'z1', 'z-image', 'z-control').adapter.model).toBe('z-control'); - }); -}); - -describe('nextControlLayerName', () => { - it('returns the first free "Control Layer N" gap', () => { - expect(nextControlLayerName([])).toBe('Control Layer 1'); - expect(nextControlLayerName(['Control Layer 1', 'Control Layer 3'])).toBe('Control Layer 2'); - expect(nextControlLayerName(['Control Layer 1', 'Control Layer 2'])).toBe('Control Layer 3'); - }); }); describe('canConvertRasterControl', () => { it('is true for image/paint raster layers and control layers, false otherwise', () => { expect(canConvertRasterControl(createEmptyPaintLayer('p', 'p'))).toBe(true); - expect(canConvertRasterControl(createControlLayer('c', 'c'))).toBe(true); + expect(canConvertRasterControl(createControlLayer({ id: 'c', name: 'c' }))).toBe(true); expect( canConvertRasterControl( paintLayer('img', { source: { image: { height: 1, imageName: 'i', width: 1 }, type: 'image' } }) @@ -454,7 +411,7 @@ describe('fitLayerTransformToBbox', () => { describe('menu patch helpers', () => { it('builds control transparency effect patches', () => { - const layer = createControlLayer('control', 'c1'); + const layer = createControlLayer({ id: 'c1', name: 'control' }); expect(getControlTransparencyEffectPatch(layer)).toEqual({ layerType: 'control', withTransparencyEffect: false }); }); @@ -491,20 +448,18 @@ describe('convertRasterControlLayer', () => { expect(converted?.type === 'control' ? converted.adapter : null).toEqual(CONTROL_ADAPTER_DEFAULTS.z_image_control); }); - it('seeds the default control model through convert and copy paths', () => { - const converted = convertRasterControlLayer(paintLayer('r'), 'control', 'sdxl', 'union-model'); - expect(converted?.type === 'control' ? converted.adapter.model : null).toBe('union-model'); - - const viaConvert = convertRasterToControl(createEmptyPaintLayer('r2', 'r2'), 'sdxl', 'union-model'); - expect(viaConvert?.adapter.model).toBe('union-model'); - - const viaCopy = copyRasterToControl(createEmptyPaintLayer('r3', 'r3'), 'copy-id', 'sdxl', 'union-model'); - expect(viaCopy?.adapter.model).toBe('union-model'); - expect(copyRasterToControl(createEmptyPaintLayer('r4', 'r4'), 'copy-id-2', 'sdxl')?.adapter.model).toBeNull(); + it('converts and copies raster layers into untyped control layers without a model', () => { + const typed = { ...createEmptyPaintLayer('r', 'r'), filter: { settings: {}, type: 'canny_edge_detection' } }; + for (const control of [convertRasterToControl(typed, 'sdxl'), copyRasterToControl(typed, 'copy-id', 'sdxl')]) { + expect(control?.adapter).toEqual(DEFAULT_CONTROL_ADAPTER); + expect(control).not.toHaveProperty('controlType'); + expect(control).not.toHaveProperty('detection'); + expect(control).not.toHaveProperty('filter'); + } }); it('converts control→raster preserving the pixel source and round-trips', () => { - const control = createControlLayer('C', 'c1'); + const control = createControlLayer({ id: 'c1', name: 'C' }); const withSource = { ...control, source: { image: { height: 48, imageName: 'pic', width: 64 }, type: 'image' }, @@ -524,7 +479,7 @@ describe('convertRasterControlLayer', () => { }); it('returns null when the layer already has the target type or cannot convert', () => { - expect(convertRasterControlLayer(createControlLayer('c', 'c'), 'control')).toBeNull(); + expect(convertRasterControlLayer(createControlLayer({ id: 'c', name: 'c' }), 'control')).toBeNull(); expect(convertRasterControlLayer(createEmptyPaintLayer('p', 'p'), 'raster')).toBeNull(); expect(convertRasterControlLayer(createInpaintMaskLayer('m', 'm'), 'control')).toBeNull(); }); @@ -590,7 +545,7 @@ describe('layer copy and conversion parity', () => { }); it('copies control layers to pixel and mask-backed destinations', () => { - const control = { ...createControlLayer('Control', 'c1'), source: raster.source, transform }; + const control = { ...createControlLayer({ id: 'c1', name: 'Control' }), source: raster.source, transform }; expect(copyControlToRaster(control, 'r-copy')).toMatchObject({ id: 'r-copy', diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.ts index 6ece605deef..64bbe262f71 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/layerOps.ts @@ -7,9 +7,9 @@ import type { CanvasInpaintMaskLayerContract, CanvasLayerBaseContract, CanvasLayerContract, - CanvasLayerSourceContract, CanvasMaskContract, CanvasMaskFillContract, + CanvasPixelSourceContract, CanvasRasterLayerContractV2, CanvasRegionalGuidanceLayerContract, RegionalGuidanceReferenceImage, @@ -20,8 +20,13 @@ export type { CanvasStructuralEngine } from '@workbench/canvas-engine/api'; import { getRegionalGuidanceSupport } from '@features/generation/graph'; import { createItemId } from '@platform/browser/randomUuid'; -import { getSourceContentRect, isMergeableRasterLayer, mergeDownEligibility } from '@workbench/canvas-engine/api'; -import { CONTROL_ADAPTER_DEFAULTS, createDefaultControlAdapter } from '@workbench/controlAdapters'; +import { + createControlLayer, + getSourceContentRect, + isMergeableRasterLayer, + mergeDownEligibility, +} from '@workbench/canvas-engine/api'; +import { CONTROL_ADAPTER_DEFAULTS } from '@workbench/controlAdapters'; type LayerTransform = CanvasLayerBaseContract['transform']; @@ -306,46 +311,6 @@ export const DEFAULT_CONTROL_ADAPTER = CONTROL_ADAPTER_DEFAULTS.controlnet; /** A new mask modifier's magnitude (legacy defaults): noise level and denoise limit. */ export const MASK_MODIFIER_DEFAULTS = { denoise: 0.8, noise: 0.25 } as const; -/** The next free "Control Layer N" name given the existing layer names (N ≥ 1, first gap). */ -export const nextControlLayerName = (existingNames: readonly string[]): string => { - const used = new Set(); - for (const name of existingNames) { - const match = /^Control Layer (\d+)$/.exec(name.trim()); - if (match) { - const n = Number(match[1]); - if (Number.isInteger(n) && n > 0) { - used.add(n); - } - } - } - let n = 1; - while (used.has(n)) { - n += 1; - } - return `Control Layer ${n}`; -}; - -export const createControlLayer = ( - name: string, - id: string = createLayerId(), - base?: string | null, - model?: string | null -): CanvasControlLayerContract => { - return { - adapter: createDefaultControlAdapter(base, model ?? null), - blendMode: 'normal', - id, - isEnabled: true, - isLocked: false, - name, - opacity: 1, - source: { bitmap: null, type: 'paint' }, - transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, - type: 'control', - withTransparencyEffect: true, - }; -}; - /** Returns the transform that fits the layer's unrotated local content into the bbox. */ export const fitLayerTransformToBbox = ( layer: CanvasLayerContract, @@ -385,11 +350,10 @@ export const canConvertRasterControl = (layer: CanvasLayerContract): boolean => }; type PixelLayer = CanvasRasterLayerContractV2 | CanvasControlLayerContract; -type PixelSource = Extract; const cloneTransform = (layer: CanvasLayerContract): LayerTransform => ({ ...layer.transform }); -const clonePixelSource = (layer: PixelLayer): PixelSource | null => { +const clonePixelSource = (layer: PixelLayer): CanvasPixelSourceContract | null => { if (layer.source.type === 'image') { return { image: { ...layer.source.image }, type: 'image' }; } @@ -435,24 +399,19 @@ const destinationBase = (layer: CanvasLayerContract, id: string, isCopy: boolean transform: cloneTransform(layer), }); +/** An untyped, model-less control layer over the same pixels; generation resolves its model. */ const pixelLayerToControl = ( layer: PixelLayer, id: string, isCopy: boolean, - base?: string | null, - model?: string | null + base?: string | null ): CanvasControlLayerContract | null => { const source = clonePixelSource(layer); if (!source) { return null; } - return { - ...destinationBase(layer, id, isCopy), - adapter: createDefaultControlAdapter(base, model ?? null), - source, - type: 'control', - withTransparencyEffect: true, - }; + const destination = destinationBase(layer, id, isCopy); + return { ...createControlLayer({ base, id, name: destination.name, source }), ...destination }; }; const pixelLayerToInpaintMask = ( @@ -486,9 +445,8 @@ const pixelLayerToRegionalGuidance = ( export const copyRasterToControl = ( layer: CanvasRasterLayerContractV2, id: string, - base?: string | null, - model?: string | null -): CanvasControlLayerContract | null => pixelLayerToControl(layer, id, true, base, model); + base?: string | null +): CanvasControlLayerContract | null => pixelLayerToControl(layer, id, true, base); export const copyRasterToInpaintMask = ( layer: CanvasRasterLayerContractV2, @@ -502,9 +460,8 @@ export const copyRasterToRegionalGuidance = ( export const convertRasterToControl = ( layer: CanvasRasterLayerContractV2, - base?: string | null, - model?: string | null -): CanvasControlLayerContract | null => pixelLayerToControl(layer, layer.id, false, base, model); + base?: string | null +): CanvasControlLayerContract | null => pixelLayerToControl(layer, layer.id, false, base); export const convertRasterToInpaintMask = (layer: CanvasRasterLayerContractV2): CanvasInpaintMaskLayerContract | null => pixelLayerToInpaintMask(layer, layer.id, false); @@ -560,14 +517,13 @@ export const copyRegionalGuidanceToInpaintMask = ( export const convertRasterControlLayer = ( layer: CanvasLayerContract, targetType: 'raster' | 'control', - base?: string | null, - model?: string | null + base?: string | null ): CanvasLayerContract | null => { if (layer.type === targetType || !canConvertRasterControl(layer)) { return null; } if (layer.type === 'raster' && targetType === 'control') { - return convertRasterToControl(layer, base, model); + return convertRasterToControl(layer, base); } if (layer.type === 'control' && targetType === 'raster') { const source = clonePixelSource(layer); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/useAddLayer.ts b/invokeai/frontend/webv2/src/workbench/widgets/layers/useAddLayer.ts index 2210fb3d2c0..1bf3b22a37c 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/useAddLayer.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/useAddLayer.ts @@ -1,7 +1,11 @@ import type { CanvasNodeContract } from '@workbench/canvas-engine/api'; -import { useModelsSelector } from '@features/models'; -import { getDocumentIndex, getDocumentLeaves } from '@workbench/canvas-engine/api'; +import { + createControlLayer, + getDocumentIndex, + getDocumentLeaves, + nextControlLayerName, +} from '@workbench/canvas-engine/api'; import { useCanvasEngine } from '@workbench/canvas-operations/react'; import { setLayerGroupExpanded } from '@workbench/layerPanelState'; import { usePreparedCommit } from '@workbench/widgets/canvas/useStructuralCommit'; @@ -13,15 +17,12 @@ import { useTranslation } from 'react-i18next'; import type { AddLayerItemId } from './addLayerMenu'; import { isAddLayerItemAvailable } from './addLayerMenu'; -import { resolveDefaultControlModelForBase } from './controlModelOptions'; import { - createControlLayer, createEmptyPaintLayer, createInpaintMaskLayer, createLayerId, createRegionalGuidanceLayer, createRegionalGuidanceLayerWithRefImage, - nextControlLayerName, nextGroupName, nextInpaintMaskName, nextRegionalGuidanceName, @@ -38,7 +39,6 @@ export const useAddLayer = (): ((id: AddLayerItemId) => void) => { const projectId = useActiveProjectId(); const commitPrepared = usePreparedCommit(engine); const base = useSelectedModelBase(); - const models = useModelsSelector((snapshot) => snapshot.models); const layerNames = useActiveProjectSelector( (project) => getDocumentIndex(project.canvas.document).nodes.map((entry) => entry.node.name), (left, right) => left.length === right.length && left.every((name, index) => name === right[index]) @@ -91,12 +91,7 @@ export const useAddLayer = (): ((id: AddLayerItemId) => void) => { return; } case 'control': { - const layer = createControlLayer( - nextControlLayerName(layerNames), - undefined, - base, - resolveDefaultControlModelForBase(models, base) - ); + const layer = createControlLayer({ base, id: createLayerId(), name: nextControlLayerName(layerNames) }); add(t('widgets.layers.actions.addControlLayer'), layer); return; } @@ -121,6 +116,6 @@ export const useAddLayer = (): ((id: AddLayerItemId) => void) => { } } }, - [base, commitPrepared, layerNames, models, projectId, regionalGuidanceCount, t] + [base, commitPrepared, layerNames, projectId, regionalGuidanceCount, t] ); }; diff --git a/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts b/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts index 7014f1b0f36..8c36551c4c7 100644 --- a/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts +++ b/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts @@ -9356,3 +9356,54 @@ describe('auto invocation route switching', () => { expect(getRoute(state)).toMatchObject({ destination: 'canvas', sourceId: 'generate' }); }); }); + +describe('canvas schema adoption', () => { + const typedControl = (id: string): CanvasControlLayerContract => ({ + ...createControlLayer(id), + controlType: 'depth', + detection: { settings: {}, type: 'depth_anything_depth_estimation' }, + source: { image: { height: 64, imageName: 'photo.png', width: 64 }, type: 'image' }, + }); + const withV4Canvas = (layers: readonly CanvasLayerContract[]): WorkbenchState => + workbenchReducer(createInitialWorkbenchState(), { + document: { ...createEmptyCanvasDocument(), stacks: stacksFrom([...layers]), version: 4 }, + type: 'replaceCanvasDocument', + }); + const versions = (state: WorkbenchState) => [getCanvas(state).version, getCanvas(state).document.version]; + + it('moves a v4 canvas to schema 5 when a control layer gains a type, and keeps 5 after it loses it', () => { + let state = withV4Canvas([createControlLayer('ctrl')]); + expect(versions(state)).toEqual([4, 4]); + + state = workbenchReducer(state, { layer: typedControl('ctrl'), layerId: 'ctrl', type: 'replaceCanvasLayer' }); + expect(versions(state)).toEqual([5, 5]); + + state = workbenchReducer(state, { layer: createControlLayer('ctrl'), layerId: 'ctrl', type: 'replaceCanvasLayer' }); + expect(versions(state)).toEqual([5, 5]); + + state = workbenchReducer(state, { document: createEmptyCanvasDocument(), type: 'replaceCanvasDocument' }); + expect(getCanvas(state).version).toBe(5); + }); + + it('adopts 4, not 5, for custom-font typography alone', () => { + const text: CanvasLayerContract = { + ...createRasterLayer('text'), + source: { + align: 'left', + color: '#fff', + content: 'hello', + fontFamily: 'Example', + fontRef: { contentHash: 'a'.repeat(64), family: 'Example', id: 'font-1', label: 'Example Regular' }, + fontSize: 24, + fontWeight: 400, + lineHeight: 1.2, + type: 'text', + }, + }; + let state = withCanvasLayers(createInitialWorkbenchState(), [createRasterLayer('text')]); + expect(versions(state)).toEqual([3, 3]); + + state = workbenchReducer(state, { layer: text, layerId: 'text', type: 'replaceCanvasLayer' }); + expect(versions(state)).toEqual([4, 4]); + }); +}); From 28f2839a0ef81d44b7a5cc01f2545dcb8a86d579 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Fri, 23 Oct 2026 01:20:06 -0600 Subject: [PATCH 02/35] feat(canvas): resolve control models by type with remembered picks Classifies installed control models by purpose and resolves a typed layer's model at read time: explicit pick, remembered pick per base and type, the only dedicated model, a covering union, else a readiness reason. Adds the remembered-pick preference, the new validation reasons, and the FLUX InstantX control mode on union graphs. --- .../frontend/webv2/public/locales/en.json | 6 + .../workbenchOwnershipManifest.json | 2 + .../core/canvas/addControlLayers.test.ts | 24 ++ .../core/canvas/addControlLayers.ts | 10 +- .../core/canvas/compileCanvasGraph.test.ts | 21 ++ .../core/canvas/controlValidation.test.ts | 25 ++ .../core/canvas/controlValidation.ts | 12 +- .../src/workbench/controlAdapters.test.ts | 18 ++ .../webv2/src/workbench/controlAdapters.ts | 14 + .../src/workbench/controlLayerChecks.test.ts | 183 ++++++++++- .../webv2/src/workbench/controlLayerChecks.ts | 205 +++++++++--- .../workbench/controlModelResolution.test.ts | 279 +++++++++++++++++ .../src/workbench/controlModelResolution.ts | 190 +++++++++++ .../webv2/src/workbench/controlTypes.test.ts | 144 +++++++++ .../webv2/src/workbench/controlTypes.ts | 295 ++++++++++++++++++ .../webv2/src/workbench/invocation.test.ts | 52 ++- .../webv2/src/workbench/invocation.ts | 10 +- .../webv2/src/workbench/settings/contracts.ts | 5 + .../src/workbench/settings/store.test.ts | 68 ++++ .../webv2/src/workbench/settings/store.ts | 44 +++ .../useInvocationState.browser.test.tsx | 76 ++++- .../shell/topbar/useInvocationState.ts | 19 +- .../invoke/prepareCanvasInvocation.test.ts | 127 +++++++- .../canvas/invoke/prepareCanvasInvocation.ts | 53 ++-- .../widgets/layers/ControlLayerSettings.tsx | 3 +- 25 files changed, 1801 insertions(+), 84 deletions(-) create mode 100644 invokeai/frontend/webv2/src/workbench/controlModelResolution.test.ts create mode 100644 invokeai/frontend/webv2/src/workbench/controlModelResolution.ts create mode 100644 invokeai/frontend/webv2/src/workbench/controlTypes.test.ts create mode 100644 invokeai/frontend/webv2/src/workbench/controlTypes.ts diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index 6b51cf19e59..3945a66859f 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -4003,7 +4003,9 @@ "transparencyEffect": "Transparent preview", "validation": { "capabilities_unavailable": "waiting for model capabilities to load.", + "choose_model": "Several installed models fit this control type; choose one.", "control_lora_limit": "Only one Control LoRA can be used at a time.", + "detection_pending": "Detection is out of date; run it again from the layer's Properties.", "duplicate_lllite_model": "Another control layer already uses this ControlNet-LLLite model; each can be applied once per generation.", "flux2_control_limit": "Only one FLUX.2 [dev] ControlNet can be used at a time.", "flux2_klein_control": "FLUX.2 [dev] ControlNet is not compatible with FLUX.2 Klein models.", @@ -4014,6 +4016,10 @@ "lllite_channels_unknown": "This ControlNet-LLLite model's adapter type is unknown. Re-identify it in the Model Manager.", "lllite_inpaint_adapter": "This is an inpainting ControlNet-LLLite adapter; a control layer needs a control adapter.", "missing_model": "Select a control model before generating.", + "missing_type": "Choose what this layer should control.", + "models_unavailable": "Waiting for the model list to load.", + "no_control_models": "No control models are installed for the selected main model.", + "no_model_for_type": "No installed control model handles this type for the selected main model.", "qwen_image_control_limit": "Only one Qwen Image ControlNet can be used at a time.", "qwen_image_edit_control": "Qwen Image ControlNet is not compatible with Qwen Image Edit models.", "switch_adapter_kind": "Switch the adapter type to {{kind}} to use this layer with the selected model.", diff --git a/invokeai/frontend/webv2/src/architecture/workbenchOwnershipManifest.json b/invokeai/frontend/webv2/src/architecture/workbenchOwnershipManifest.json index f726a7f0831..738d4ee234a 100644 --- a/invokeai/frontend/webv2/src/architecture/workbenchOwnershipManifest.json +++ b/invokeai/frontend/webv2/src/architecture/workbenchOwnershipManifest.json @@ -97,6 +97,8 @@ "canvasStagingView.ts": "workbench", "controlAdapters.ts": "workbench", "controlLayerChecks.ts": "workbench", + "controlModelResolution.ts": "workbench", + "controlTypes.ts": "workbench", "deferredResource.ts": "workbench", "floatingWindows.ts": "workbench", "focusRegions.testing.ts": "workbench", diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/addControlLayers.test.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/addControlLayers.test.ts index 7312b1a8147..feba2b53bc8 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/addControlLayers.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/addControlLayers.test.ts @@ -198,6 +198,30 @@ describe('addControlLayers — controlnet on flux', () => { }); }); +describe('addControlLayers — FLUX InstantX Union mode', () => { + it('sets instantx_control_mode on a FLUX controlnet layer that carries one, mode 0 included', () => { + const graph = run({ + base: 'flux', + layers: [ + layer({ id: 'depth', instantxControlMode: 2, model: model('flux') }), + layer({ id: 'canny', instantxControlMode: 0, model: model('flux') }), + layer({ id: 'dedicated', instantxControlMode: null, model: model('flux') }), + ], + }); + + expect(graph.nodes['control_net_depth']?.instantx_control_mode).toBe(2); + expect(graph.nodes['control_net_canny']?.instantx_control_mode).toBe(0); + expect(graph.nodes['control_net_dedicated']).not.toHaveProperty('instantx_control_mode'); + }); + + it('never sets it outside FLUX, even when a layer carries one', () => { + const graph = run({ base: 'sdxl', layers: [layer({ instantxControlMode: 2, model: model('sdxl') })] }); + + expect(graph.nodes['control_net_layer1']?.type).toBe('controlnet'); + expect(graph.nodes['control_net_layer1']).not.toHaveProperty('instantx_control_mode'); + }); +}); + describe('addControlLayers — t2i_adapter on sdxl', () => { it('creates a t2i_adapter node with exact fields and wires collector to denoise.t2i_adapter', () => { const m = model('sdxl', 't2i_adapter'); diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/addControlLayers.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/addControlLayers.ts index fe080f9c8ac..a8f36198bf6 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/addControlLayers.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/addControlLayers.ts @@ -40,6 +40,8 @@ export interface ControlLayerGraphInput { controlMode: 'balanced' | 'more_prompt' | 'more_control' | 'unbalanced' | null; /** The resolved model's `cond_in_channels`; required to validate an Anima ControlNet-LLLite layer. */ modelCondInChannels?: number | null; + /** FLUX InstantX Union only: the mode for the layer's control type; other models and bases ignore it. */ + instantxControlMode?: number | null; } /** Kind support is separate from control-mode/model compatibility. */ @@ -195,8 +197,12 @@ export const addControlLayers = (graph: BackendGraphContract, options: AddContro image: { image_name: layer.imageName }, resize_mode: 'just_resize', type: controlNetNodeType(base), - // FLUX ControlNet has no control_mode; SD-family carries it. - ...(base === 'flux' ? {} : { control_mode: layer.controlMode ?? 'balanced' }), + // FLUX ControlNet has no control_mode; SD-family carries it. A FLUX union model takes its InstantX mode. + ...(base === 'flux' + ? typeof layer.instantxControlMode === 'number' + ? { instantx_control_mode: layer.instantxControlMode } + : {} + : { control_mode: layer.controlMode ?? 'balanced' }), }); addEdge(graph, node, 'control', ensureControlNetCollector(), 'item'); } else if (layer.kind === 't2i_adapter') { diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts index ea3a660c815..018babbf88e 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts @@ -874,6 +874,27 @@ describe('compileCanvasGraph — control layers (integration)', () => { weight: 0.65, }; + it('carries the InstantX mode of a FLUX union layer onto its flux_controlnet node, and none onto other bases', () => { + const unionLayer: ControlLayerGraphInput = { + ...controlNetLayer, + controlMode: null, + instantxControlMode: 4, + model: { base: 'flux', key: 'flux-union', name: 'FLUX ControlNet Union Pro', type: 'controlnet' }, + }; + const flux = compile(fluxModel, 'txt2img', { controlLayers: [unionLayer] }).backendGraph; + const sd1 = compile(sd1Model, 'img2img', { + controlLayers: [{ ...controlNetLayer, instantxControlMode: 4 }], + }).backendGraph; + + expect(flux.nodes['control_net_control-layer-1']).toMatchObject({ + instantx_control_mode: 4, + type: 'flux_controlnet', + }); + expect(flux.nodes['control_net_control-layer-1']).not.toHaveProperty('control_mode'); + expect(sd1.nodes['control_net_control-layer-1']?.type).toBe('controlnet'); + expect(sd1.nodes['control_net_control-layer-1']).not.toHaveProperty('instantx_control_mode'); + }); + it('grafts an enabled control layer through a real compiled canvas graph: control node args, collector → denoise.control wiring, base graph intact', () => { const { backendGraph, graph, mode } = compile(sd1Model, 'img2img', { controlLayers: [controlNetLayer] }); diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/controlValidation.test.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/controlValidation.test.ts index 3ca83ae2300..f116cb53eec 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/controlValidation.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/controlValidation.test.ts @@ -282,6 +282,31 @@ describe('Anima ControlNet-LLLite validation', () => { }); }); +describe('unresolved control models', () => { + it('reports why resolution found no model in place of missing_model, after kind and value checks', () => { + expect(getControlValidationReason({ ...valid, adapterModel: null, unresolvedModel: 'choose_model' })).toBe( + 'choose_model' + ); + expect(getControlValidationReason({ ...valid, adapterModel: null, unresolvedModel: null })).toBe('missing_model'); + expect( + getControlValidationReason({ ...valid, adapterModel: null, unresolvedModel: 'no_model_for_type', weight: 5 }) + ).toBe('invalid_adapter_values'); + expect(getControlValidationReason({ ...valid, unresolvedModel: 'no_control_models' })).toBeNull(); + }); + + it('carries the reason through the sequence without claiming a slot', () => { + const validate = createControlValidationSequence({ base: 'flux' }); + const controlLora = { + adapterModel: { base: 'flux', key: 'lora', type: 'control_lora' }, + beginEndStepPct: [0, 1] as [number, number], + kind: 'control_lora' as const, + weight: 1, + }; + expect(validate({ ...controlLora, adapterModel: null, unresolvedModel: 'missing_type' })).toBe('missing_type'); + expect(validate(controlLora)).toBeNull(); + }); +}); + describe('createControlValidationSequence', () => { const anima = (key: string, cond_in_channels: number | null = 3) => ({ adapterModel: { base: 'anima', cond_in_channels, key, type: 'controlnet' }, diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/controlValidation.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/controlValidation.ts index 677036fa7dc..40d84e8297e 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/controlValidation.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/controlValidation.ts @@ -19,6 +19,11 @@ export type ControlAdapterKind = (typeof CONTROL_ADAPTER_KINDS)[number]; export const CONTROL_VALIDATION_REASONS = [ 'capabilities_unavailable', 'missing_model', + 'missing_type', + 'choose_model', + 'no_model_for_type', + 'no_control_models', + 'models_unavailable', 'switch_adapter_kind', 'unsupported_adapter', 'incompatible_base', @@ -118,6 +123,8 @@ export const getControlValidationReason = (params: { flux2ControlIndex?: number; /** An earlier contributing Anima LLLite layer already applies this model; the denoiser applies each once. */ lliteModelInUse?: boolean; + /** Why model resolution found no model, reported in place of `missing_model`. */ + unresolvedModel?: ControlValidationReason | null; }): ControlValidationReason | null => { const { adapterModel, @@ -129,6 +136,7 @@ export const getControlValidationReason = (params: { mainBase, mainVariant, qwenImageControlIndex = 0, + unresolvedModel, weight, zImageControlIndex = 0, } = params; @@ -147,7 +155,7 @@ export const getControlValidationReason = (params: { return 'invalid_adapter_values'; } if (!adapterModel) { - return 'missing_model'; + return unresolvedModel ?? 'missing_model'; } // Unavailable capability data is distinct from unsupported adapters. if (!capabilitiesLoaded) { @@ -203,6 +211,7 @@ export const createControlValidationSequence = (main: { base: string; variant?: beginEndStepPct: [number, number]; kind: ControlAdapterKind; weight: number; + unresolvedModel?: ControlValidationReason | null; }): ControlValidationReason | null => { const { adapterModel, kind } = layer; const reason = getControlValidationReason({ @@ -215,6 +224,7 @@ export const createControlValidationSequence = (main: { base: string; variant?: mainBase: main.base, mainVariant: main.variant ?? undefined, qwenImageControlIndex: kind === 'qwen_image_control' ? qwenImageControlCount : 0, + unresolvedModel: layer.unresolvedModel, weight: layer.weight, zImageControlIndex: kind === 'z_image_control' ? zImageControlCount : 0, }); diff --git a/invokeai/frontend/webv2/src/workbench/controlAdapters.test.ts b/invokeai/frontend/webv2/src/workbench/controlAdapters.test.ts index 8a907d59662..64ebd114f97 100644 --- a/invokeai/frontend/webv2/src/workbench/controlAdapters.test.ts +++ b/invokeai/frontend/webv2/src/workbench/controlAdapters.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest'; import { CONTROL_ADAPTER_DEFAULTS, createDefaultControlAdapter, + deriveControlKind, getDefaultControlAdapterKind, normalizeControlAdapter, } from './controlAdapters'; @@ -108,3 +109,20 @@ describe('default control adapters', () => { expect(CONTROL_ADAPTER_DEFAULTS.anima_lllite.beginEndStepPct).toEqual([0, 1]); }); }); + +describe('deriveControlKind', () => { + it.each([ + [{ base: 'sdxl', type: 'controlnet' }, 'controlnet'], + [{ base: 'flux', type: 'controlnet' }, 'controlnet'], + [{ base: 'sdxl', type: 't2i_adapter' }, 't2i_adapter'], + [{ base: 'flux', type: 'control_lora' }, 'control_lora'], + [{ base: 'z-image', type: 'controlnet' }, 'z_image_control'], + [{ base: 'qwen-image', type: 'controlnet' }, 'qwen_image_control'], + [{ base: 'flux2', type: 'controlnet' }, 'flux2_control'], + [{ base: 'anima', type: 'controlnet' }, 'anima_lllite'], + [{ base: 'sdxl', type: 'main' }, null], + [{ base: 'sdxl', type: 'ip_adapter' }, null], + ] as const)('runs %j as %s', (model, kind) => { + expect(deriveControlKind(model)).toBe(kind); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/controlAdapters.ts b/invokeai/frontend/webv2/src/workbench/controlAdapters.ts index 8e146c3a127..f51eeabb219 100644 --- a/invokeai/frontend/webv2/src/workbench/controlAdapters.ts +++ b/invokeai/frontend/webv2/src/workbench/controlAdapters.ts @@ -75,6 +75,20 @@ export const getDefaultControlAdapterKind = (base: string | null | undefined): C (kind) => CONTROL_KIND_BASE[kind] === base ) ?? 'controlnet'; +/** + * The adapter kind a control model runs as on its own base: T2I-Adapter and Control LoRA by type, a `controlnet` + * model as its base's own kind (Z-Image, Qwen-Image, FLUX.2, Anima LLLite) or plain ControlNet. Null for other types. + */ +export const deriveControlKind = (model: { + type: string; + base: string; +}): CanvasControlAdapterContract['kind'] | null => { + if (model.type === 't2i_adapter' || model.type === 'control_lora') { + return model.type; + } + return model.type === 'controlnet' ? getDefaultControlAdapterKind(model.base) : null; +}; + /** A fresh copy of the default adapter for `base`, carrying `model`. */ export const createDefaultControlAdapter = ( base: string | null | undefined, diff --git a/invokeai/frontend/webv2/src/workbench/controlLayerChecks.test.ts b/invokeai/frontend/webv2/src/workbench/controlLayerChecks.test.ts index 3890a9c947b..7d71458b87b 100644 --- a/invokeai/frontend/webv2/src/workbench/controlLayerChecks.test.ts +++ b/invokeai/frontend/webv2/src/workbench/controlLayerChecks.test.ts @@ -1,12 +1,15 @@ import type { ModelConfig } from '@features/models'; import type { CanvasControlLayerContract } from '@workbench/canvas-engine/api'; +import { resetArchitectureCapabilities } from '@features/generation/core/architectureCapabilities'; import { seedArchitectureCapabilities } from '@features/generation/core/architectureCapabilities.testing'; import { CONTROL_ADAPTER_KINDS, CONTROL_VALIDATION_REASONS } from '@features/generation/graph'; import { createInstance } from 'i18next'; import { describe, expect, it } from 'vitest'; import { + createControlLayerCheckContext, + describeControlLayer, describeControlLayerIssue, describeControlLayerReason, getBlockingControlLayerIssues, @@ -25,8 +28,10 @@ const controlLayer = ( model?: string | null; source?: CanvasControlLayerContract['source']; weight?: number; + derivation?: Pick; } = {} ): CanvasControlLayerContract => ({ + ...overrides.derivation, adapter: { beginEndStepPct: [0, 1], controlMode: null, @@ -64,16 +69,33 @@ describe('hasControlLayerContent', () => { }); describe('getBlockingControlLayerIssues', () => { - it('reports a content-bearing model-less control layer with a human sentence', () => { + it('leaves out a content-bearing layer with neither a type nor a model: it warns but contributes nothing', () => { + const layer = controlLayer('Control Layer 1'); + + expect(getBlockingControlLayerIssues({ layers: [layer], mainModel: sdxl, models })).toEqual([]); + expect(getControlLayerAttentionReason(layer, sdxl, models)).toBe('missing_type'); + }); + + it('blocks an untyped layer whose model is no longer installed', () => { + expect( + getBlockingControlLayerIssues({ + layers: [controlLayer('legacy', { model: 'uninstalled' })], + mainModel: sdxl, + models, + }) + ).toMatchObject([{ code: 'missing_model', layerId: 'legacy' }]); + }); + + it('reports a typed content-bearing layer no model resolves, in structured form', () => { const issues = getBlockingControlLayerIssues({ - layers: [controlLayer('Control Layer 1')], + layers: [controlLayer('Control Layer 1', { derivation: { controlType: 'pose', detection: null } })], mainModel: sdxl, models, }); expect(issues).toEqual([ { - code: 'missing_model', + code: 'no_model_for_type', layerId: 'Control Layer 1', layerName: 'Control Layer 1', suggestedKind: null, @@ -191,8 +213,9 @@ describe('getBlockingControlLayerIssues', () => { }); it('reports issues in document order', () => { + const pose = { derivation: { controlType: 'pose', detection: null } } as const; const issues = getBlockingControlLayerIssues({ - layers: [controlLayer('top'), controlLayer('bottom')], + layers: [controlLayer('top', pose), controlLayer('bottom', pose)], mainModel: sdxl, models, }); @@ -202,18 +225,21 @@ describe('getBlockingControlLayerIssues', () => { }); describe('getControlLayerAttentionReason', () => { - it('warns about a missing model even without content', () => { + it('warns that a fresh layer needs a type, even without content', () => { const layer = controlLayer('fresh', { source: { bitmap: null, type: 'paint' } }); - expect(getControlLayerAttentionReason(layer, sdxl, models)).toBe('missing_model'); + expect(getControlLayerAttentionReason(layer, sdxl, models)).toBe('missing_type'); }); it('returns null for a healthy layer and ignores per-kind limits', () => { expect(getControlLayerAttentionReason(controlLayer('ok', { model: 'sdxl-control' }), sdxl, models)).toBeNull(); }); - it('warns that a Klein model cannot run the FLUX.2 control, even on a fresh layer without a model', () => { - const fresh = controlLayer('fresh', { kind: 'flux2_control', source: { bitmap: null, type: 'paint' } }); + it('warns that a Klein model cannot run the FLUX.2 control, even on a fresh typed layer without a model', () => { + const fresh = controlLayer('fresh', { + derivation: { controlType: 'depth', detection: null }, + source: { bitmap: null, type: 'paint' }, + }); const picked = controlLayer('picked', { kind: 'flux2_control', model: 'flux2-union', weight: 0.75 }); const flux2Models = [model('flux2-union', 'flux2', 'controlnet')]; @@ -248,6 +274,143 @@ describe('getControlLayerReasonInSequence', () => { }); }); +describe('typed control layers', () => { + const depthModels = [model('sdxl-depth-a', 'sdxl', 'controlnet'), model('sdxl-depth-b', 'sdxl', 'controlnet')]; + const typed = ( + id: string, + derivation: Partial> = {}, + overrides: Parameters[1] = {} + ) => controlLayer(id, { ...overrides, derivation: { controlType: 'depth', detection: null, ...derivation } }); + const depthDetection = { settings: {}, type: 'depth_anything_depth_estimation' }; + + it('asks for a model when several fit, and resolves with the remembered pick', () => { + const layers = [typed('depth')]; + + expect(getBlockingControlLayerIssues({ layers, mainModel: sdxl, models: depthModels })).toMatchObject([ + { code: 'choose_model' }, + ]); + expect( + getBlockingControlLayerIssues({ + context: createControlLayerCheckContext('sdxl', { 'sdxl:depth': 'sdxl-depth-b' }), + layers, + mainModel: sdxl, + models: depthModels, + }) + ).toEqual([]); + }); + + it('tells no control models apart from none for the type', () => { + expect(getBlockingControlLayerIssues({ layers: [typed('depth')], mainModel: sdxl, models: [] })).toMatchObject([ + { code: 'no_control_models' }, + ]); + }); + + it('never blocks on an empty layer, though it still warns', () => { + const empty = typed('empty', {}, { source: { bitmap: null, type: 'paint' } }); + + expect(getBlockingControlLayerIssues({ layers: [empty], mainModel: sdxl, models: depthModels })).toEqual([]); + expect(getControlLayerAttentionReason(empty, sdxl, depthModels)).toBe('choose_model'); + }); + + it('blocks on a map detection has not caught up with, after the model reasons', () => { + const pending = typed('pending', { detection: depthDetection }); + + expect( + getBlockingControlLayerIssues({ layers: [pending], mainModel: sdxl, models: [depthModels[0]!] }) + ).toMatchObject([{ code: 'detection_pending' }]); + expect(getBlockingControlLayerIssues({ layers: [pending], mainModel: sdxl, models: depthModels })).toMatchObject([ + { code: 'choose_model' }, + ]); + expect(getControlLayerAttentionReason(pending, sdxl, [depthModels[0]!])).toBe('detection_pending'); + }); + + it('counts per-generation limits by the kind the resolved model runs, not the stored kind', () => { + const fluxDepth = [model('flux-depth-lora', 'flux', 'control_lora')]; + const issues = getBlockingControlLayerIssues({ + layers: [typed('first'), typed('second')], + mainModel: { base: 'flux' }, + models: fluxDepth, + }); + + expect(issues.map(({ code, layerId }) => [layerId, code])).toEqual([['second', 'control_lora_limit']]); + }); + + it('still resolves models before capabilities load, and reports that they have not', () => { + const context = createControlLayerCheckContext('flux'); + const judge = () => + getBlockingControlLayerIssues({ + context, + layers: [typed('depth')], + mainModel: { base: 'flux' }, + models: [model('flux-depth-t2i', 'flux', 't2i_adapter')], + }); + + // Loaded, FLUX runs no T2I-Adapter, so no model fits. + expect(judge()).toMatchObject([{ code: 'no_control_models' }]); + // The context reads capabilities when it judges, not when it is made. + resetArchitectureCapabilities(); + expect(judge()).toMatchObject([{ code: 'capabilities_unavailable' }]); + }); +}); + +describe('describeControlLayer', () => { + const sdxlDepth = [model('sdxl-depth', 'sdxl', 'controlnet')]; + const ctx = { ...createControlLayerCheckContext('sdxl'), mainModel: sdxl, models: sdxlDepth }; + const detection = { settings: {}, type: 'depth_anything_depth_estimation' }; + const original = { image: { height: 64, imageName: 'photo.png', width: 64 }, type: 'image' } as const; + const depth = (derivation: Pick, empty = false) => + controlLayer('depth', { + derivation: { controlType: 'depth', ...derivation }, + ...(empty ? { source: { bitmap: null, type: 'paint' } } : {}), + }); + + it.each([ + ['empty', depth({ detection }, true)], + ['none', controlLayer('untyped', { model: 'sdxl-depth' })], + ['none', depth({ detection: null })], + ['stale', depth({ detection })], + ['current', depth({ detection, filter: detection, original })], + ['no-input', depth({ detection, filter: { settings: {}, type: 'canny_edge_detection' } })], + ] as const)('reports a %s map', (map, layer) => { + expect(describeControlLayer(layer, ctx).map).toBe(map); + }); + + it('resolves across every base without a main model, reporting only resolution and detection', () => { + const noMain = { + ...createControlLayerCheckContext(null), + mainModel: null, + models: [model('sd1-depth', 'sd-1', 'controlnet'), model('flux-t2i-depth', 'flux', 't2i_adapter')], + }; + + expect(describeControlLayer(depth({ detection: null }), noMain)).toMatchObject({ + reason: 'choose_model', + resolution: { candidates: noMain.models, how: 'ask' }, + }); + expect(describeControlLayer(depth({ detection }), { ...noMain, models: [noMain.models[0]!] })).toMatchObject({ + map: 'stale', + reason: 'detection_pending', + resolution: { how: 'only' }, + }); + }); + + it('carries the resolution, and with earlier layers judges per-generation limits as Invoke does', () => { + const flux = { + ...createControlLayerCheckContext('flux'), + mainModel: { base: 'flux' }, + models: [model('flux-depth-lora', 'flux', 'control_lora')], + }; + const layer = depth({ detection: null }); + + expect(describeControlLayer(layer, flux)).toMatchObject({ + reason: null, + resolution: { how: 'only', kind: 'control_lora' }, + }); + expect(describeControlLayer(layer, { ...flux, earlier: [depth({ detection: null })] }).reason).toBe( + 'control_lora_limit' + ); + }); +}); + const englishCatalogModules = import.meta.glob('../../public/locales/en.json', { eager: true, import: 'default' }); const english = createInstance(); await english.init({ @@ -260,7 +423,9 @@ const t = english.t.bind(english) as (key: string, options?: Record { it('has an English string for every reason code, adapter kind and hidden-model note', () => { const keys = [ - ...CONTROL_VALIDATION_REASONS.map((code) => `widgets.layers.control.validation.${code}`), + ...[...CONTROL_VALIDATION_REASONS, 'detection_pending'].map( + (code) => `widgets.layers.control.validation.${code}` + ), ...CONTROL_ADAPTER_KINDS.map((kind) => `widgets.layers.control.kinds.${kind}`), 'widgets.layers.control.hiddenModels.lllite_inpaint_adapter', 'widgets.layers.control.hiddenModels.lllite_channels_unknown', diff --git a/invokeai/frontend/webv2/src/workbench/controlLayerChecks.ts b/invokeai/frontend/webv2/src/workbench/controlLayerChecks.ts index 4958d679917..c0bf9a94ce0 100644 --- a/invokeai/frontend/webv2/src/workbench/controlLayerChecks.ts +++ b/invokeai/frontend/webv2/src/workbench/controlLayerChecks.ts @@ -5,63 +5,123 @@ import { createControlValidationSequence, getControlValidationReason, getSuggestedControlKind, + isControlKindSupportedForBase, type ControlAdapterKind, type ControlValidationReason, } from '@features/generation/graph'; -import { isCompositableControlLayer } from '@workbench/canvasLayerContent'; +import { hasArchitectureCapabilities } from '@features/generation/settings'; +import { getControlDetectionInput, isControlMapCurrent, needsControlDetection } from '@workbench/canvas-engine/api'; +import { hasControlLayerContent, isCompositableControlLayer } from '@workbench/canvasLayerContent'; + +import { resolveControlModel, type ControlModelMemory, type ControlModelResolution } from './controlModelResolution'; + +/** Everything that can hold a control layer back: a validation reason, or a map detection has not caught up with. */ +export type ControlLayerIssueCode = ControlValidationReason | 'detection_pending'; /** A control layer that blocks generation; the shell words it from the locale (see `controlLayerMessages`). */ export interface ControlLayerIssue { - code: ControlValidationReason; + code: ControlLayerIssueCode; layerId: string; layerName: string; /** For `switch_adapter_kind`: the kind the selected base runs. */ suggestedKind: ControlAdapterKind | null; } -export { hasControlLayerContent } from '@workbench/canvasLayerContent'; +/** What model resolution needs beyond the layer: remembered picks and the kinds the main model's base runs. */ +export interface ControlLayerCheckContext { + remembered: ControlModelMemory; + supportsKind: (kind: ControlAdapterKind) => boolean; +} + +/** + * Before capabilities load every kind counts as supported, so resolution still finds models; validation then + * reports `capabilities_unavailable` instead of a misleading missing model. Without a main model (`base` null) no + * base restricts kinds either. + */ +export const createControlLayerCheckContext = ( + base: string | null, + remembered: ControlModelMemory = {} +): ControlLayerCheckContext => ({ + remembered, + supportsKind: (kind) => base === null || !hasArchitectureCapabilities() || isControlKindSupportedForBase(base, kind), +}); -const resolveAdapterModel = (adapterModel: string | null, models: readonly ModelConfig[]): ModelConfig | null => - (adapterModel ? models.find((candidate) => candidate.key === adapterModel) : undefined) ?? null; +export { hasControlLayerContent } from '@workbench/canvasLayerContent'; type MainModel = { base: string; variant?: string | null }; -/** Validates control layers in generation order, as the invocation does. */ -const createLayerValidator = (mainModel: MainModel, models: readonly ModelConfig[]) => { +export interface ControlLayerJudgement { + resolution: ControlModelResolution; + reason: ControlLayerIssueCode | null; + /** A layer with no type and no model warns but contributes nothing, so it never blocks. */ + skip: boolean; +} + +const resolveLayer = ( + layer: CanvasControlLayerContract, + mainModel: MainModel, + models: readonly ModelConfig[], + context: ControlLayerCheckContext +): ControlModelResolution => + resolveControlModel({ + base: mainModel.base, + layer, + models, + remembered: context.remembered, + supportsKind: context.supportsKind, + }); + +/** + * Judges control layers in generation order, as the invocation does: per-generation limits count only layers that + * passed. Invoke readiness, the settings panel and the invocation collector share it. + */ +export const createControlLayerSequence = ( + mainModel: MainModel, + models: readonly ModelConfig[], + context: ControlLayerCheckContext = createControlLayerCheckContext(mainModel.base) +): ((layer: CanvasControlLayerContract) => ControlLayerJudgement) => { const validate = createControlValidationSequence(mainModel); - return ({ adapter }: CanvasControlLayerContract): ControlValidationReason | null => - validate({ - adapterModel: resolveAdapterModel(adapter.model, models), - beginEndStepPct: adapter.beginEndStepPct, - kind: adapter.kind, - weight: adapter.weight, - }); + return (layer) => { + const resolution = resolveLayer(layer, mainModel, models, context); + // A layer with no type and no model keeps its reason for the panel but is skipped; having no model, it never + // passes, so it never claims a per-generation slot. + const reason = + validate({ + adapterModel: resolution.model, + beginEndStepPct: layer.adapter.beginEndStepPct, + kind: resolution.kind, + unresolvedModel: resolution.unresolved, + weight: layer.adapter.weight, + }) ?? (needsControlDetection(layer) ? 'detection_pending' : null); + return { reason, resolution, skip: resolution.unresolved === 'missing_type' }; + }; }; /** * Validate the same enabled, content-bearing layers as createControlLayerCollector; invalid layers do not consume - * per-generation limits. + * per-generation limits, and layers without a type or model are left out. */ export const getBlockingControlLayerIssues = (params: { layers: readonly CanvasLayerContract[]; mainModel: MainModel; models: readonly ModelConfig[]; + context?: ControlLayerCheckContext; }): ControlLayerIssue[] => { const { layers, mainModel, models } = params; - const validate = createLayerValidator(mainModel, models); + const judge = createControlLayerSequence(mainModel, models, params.context); const issues: ControlLayerIssue[] = []; for (const layer of layers) { if (!isCompositableControlLayer(layer)) { continue; } - const code = validate(layer); - if (code) { + const { reason, skip } = judge(layer); + if (reason && !skip) { issues.push({ - code, + code: reason, layerId: layer.id, layerName: layer.name, - suggestedKind: code === 'switch_adapter_kind' ? getSuggestedControlKind(mainModel.base) : null, + suggestedKind: reason === 'switch_adapter_kind' ? getSuggestedControlKind(mainModel.base) : null, }); } } @@ -77,39 +137,106 @@ export const getControlLayerReasonInSequence = (params: { layer: CanvasControlLayerContract; mainModel: MainModel; models: readonly ModelConfig[]; -}): ControlValidationReason | null => { - const validate = createLayerValidator(params.mainModel, params.models); + context?: ControlLayerCheckContext; +}): ControlLayerIssueCode | null => { + const judge = createControlLayerSequence(params.mainModel, params.models, params.context); for (const earlier of params.earlier) { - validate(earlier); + judge(earlier); } - return validate(params.layer); + return judge(params.layer).reason; +}; + +const judgeAttention = ( + layer: CanvasControlLayerContract, + mainModel: MainModel, + models: readonly ModelConfig[], + context: ControlLayerCheckContext +): { resolution: ControlModelResolution; reason: ControlLayerIssueCode | null } => { + const resolution = resolveLayer(layer, mainModel, models, context); + const reason = + getControlValidationReason({ + adapterModel: resolution.model, + beginEndStepPct: layer.adapter.beginEndStepPct, + controlLoraIndex: 0, + flux2ControlIndex: 0, + kind: resolution.kind, + mainBase: mainModel.base, + mainVariant: mainModel.variant ?? undefined, + qwenImageControlIndex: 0, + unresolvedModel: resolution.unresolved, + weight: layer.adapter.weight, + zImageControlIndex: 0, + }) ?? (needsControlDetection(layer) ? 'detection_pending' : null); + return { reason, resolution }; }; /** Ambient warnings include empty layers but omit per-kind limits, which belong in the Invoke tooltip. */ export const getControlLayerAttentionReason = ( layer: CanvasControlLayerContract, mainModel: MainModel, - models: readonly ModelConfig[] -): ControlValidationReason | null => - getControlValidationReason({ - adapterModel: resolveAdapterModel(layer.adapter.model, models), - beginEndStepPct: layer.adapter.beginEndStepPct, - controlLoraIndex: 0, - flux2ControlIndex: 0, - kind: layer.adapter.kind, - mainBase: mainModel.base, - mainVariant: mainModel.variant ?? undefined, - qwenImageControlIndex: 0, - weight: layer.adapter.weight, - zImageControlIndex: 0, - }); + models: readonly ModelConfig[], + context: ControlLayerCheckContext = createControlLayerCheckContext(mainModel.base) +): ControlLayerIssueCode | null => judgeAttention(layer, mainModel, models, context).reason; + +/** + * Where a layer's control map stands. `empty`: no pixels yet. `none`: no type or no detection, so the pixels are the + * map. `current`: the map matches its detection. `stale`: detection must run again. `no-input`: the map is all the + * layer has, so it cannot be detected again until its image is replaced. + */ +export type ControlMapState = 'none' | 'current' | 'stale' | 'no-input' | 'empty'; + +const getControlMapState = (layer: CanvasControlLayerContract): ControlMapState => { + if (!hasControlLayerContent(layer) && !layer.original) { + return 'empty'; + } + if (!layer.controlType || !layer.detection) { + return 'none'; + } + if (isControlMapCurrent(layer)) { + return 'current'; + } + return getControlDetectionInput(layer) ? 'stale' : 'no-input'; +}; + +/** + * One layer as the settings panel, the layer row and Invoke present it. With `earlier` (the contributing layers + * above it) the reason includes per-generation limits, as Invoke judges it; without, it is the ambient warning. + * Without a main model the layer resolves across every base and only resolution and detection give a reason. + */ +export const describeControlLayer = ( + layer: CanvasControlLayerContract, + ctx: ControlLayerCheckContext & { + mainModel: MainModel | null; + models: readonly ModelConfig[]; + earlier?: readonly CanvasControlLayerContract[]; + } +): { resolution: ControlModelResolution; map: ControlMapState; reason: ControlLayerIssueCode | null } => { + const context = { remembered: ctx.remembered, supportsKind: ctx.supportsKind }; + let judgement: { resolution: ControlModelResolution; reason: ControlLayerIssueCode | null }; + if (!ctx.mainModel) { + const resolution = resolveControlModel({ ...context, base: null, layer, models: ctx.models }); + judgement = { + reason: resolution.unresolved ?? (needsControlDetection(layer) ? 'detection_pending' : null), + resolution, + }; + } else if (ctx.earlier) { + const judge = createControlLayerSequence(ctx.mainModel, ctx.models, context); + for (const earlier of ctx.earlier) { + judge(earlier); + } + judgement = judge(layer); + } else { + judgement = judgeAttention(layer, ctx.mainModel, ctx.models, context); + } + return { map: getControlMapState(layer), reason: judgement.reason, resolution: judgement.resolution }; +}; type Translate = (key: string, options?: Record) => string; /** The one user-facing wording of a reason, shared by the settings panel, the layer row and the Invoke tooltip. */ export const describeControlLayerReason = ( t: Translate, - code: ControlValidationReason, + code: ControlLayerIssueCode, suggestedKind: ControlAdapterKind | null ): string => t(`widgets.layers.control.validation.${code}`, { diff --git a/invokeai/frontend/webv2/src/workbench/controlModelResolution.test.ts b/invokeai/frontend/webv2/src/workbench/controlModelResolution.test.ts new file mode 100644 index 00000000000..f8dbc721c4a --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/controlModelResolution.test.ts @@ -0,0 +1,279 @@ +import type { ControlAdapterKind } from '@features/generation/graph'; +import type { ModelConfig } from '@features/models'; +import type { CanvasControlLayerContract, ControlType } from '@workbench/canvas-engine/api'; + +import { describe, expect, it } from 'vitest'; + +import { getTypeAvailability, resolveControlModel, type ControlModelMemory } from './controlModelResolution'; + +const model = (key: string, base: string, type = 'controlnet', extra: Partial = {}): ModelConfig => + ({ base, key, name: key, type, ...extra }) as ModelConfig; + +type Layer = Pick; + +const layer = ( + controlType: ControlType | undefined, + overrides: { detector?: string | null; model?: string | null; kind?: ControlAdapterKind } = {} +): Layer => ({ + adapter: { + beginEndStepPct: [0, 1], + controlMode: null, + kind: overrides.kind ?? 'controlnet', + model: overrides.model ?? null, + weight: 0.75, + }, + ...(controlType ? { controlType } : {}), + ...(overrides.detector === undefined + ? {} + : { detection: overrides.detector === null ? null : { settings: {}, type: overrides.detector } }), +}); + +const supportsAll = (): boolean => true; + +const resolve = ( + target: Layer, + base: string | null, + models: readonly ModelConfig[], + remembered: ControlModelMemory = {}, + supportsKind: (kind: ControlAdapterKind) => boolean = supportsAll +) => resolveControlModel({ base, layer: target, models, remembered, supportsKind }); + +const depthA = model('sdxl-depth-a', 'sdxl'); +const depthB = model('sdxl-depth-b', 'sdxl', 'controlnet', { + default_settings: { preprocessor: 'depth_anything_image_processor' }, +}); +const normal = model('sdxl-normal', 'sdxl'); +const pose = model('sdxl-openpose', 'sdxl'); +const sdxlUnion = model('sdxl-union-promax', 'sdxl'); +const sd1Depth = model('sd15-depth', 'sd-1'); + +describe('resolveControlModel without a type', () => { + it('applies the stored adapter as it is', () => { + const resolution = resolve(layer(undefined, { kind: 't2i_adapter', model: pose.key }), 'sdxl', [pose]); + + expect(resolution).toMatchObject({ how: 'none', kind: 't2i_adapter', model: pose, unresolved: null }); + expect(resolution.candidates).toEqual([]); + }); + + it('reports a missing type when the layer names no model', () => { + expect(resolve(layer(undefined), 'sdxl', [pose])).toMatchObject({ + how: 'none', + model: null, + unresolved: 'missing_type', + }); + }); + + it('reports a missing model, not a missing type, when an untyped layer names one that is not installed', () => { + expect(resolve(layer(undefined, { model: 'deleted' }), 'sdxl', [pose])).toMatchObject({ + how: 'none', + model: null, + unresolved: 'missing_model', + }); + }); +}); + +describe('resolveControlModel with a type', () => { + it('runs an explicit pick for this base as the kind that model runs', () => { + const t2i = model('t2i-depth', 'sdxl', 't2i_adapter'); + const resolution = resolve(layer('depth', { model: t2i.key }), 'sdxl', [depthA, t2i]); + + expect(resolution).toMatchObject({ how: 'explicit', kind: 't2i_adapter', model: t2i, unresolved: null }); + }); + + it('ignores a pick made for another base, and one that was deleted', () => { + expect(resolve(layer('depth', { model: sd1Depth.key }), 'sdxl', [sd1Depth, depthA])).toMatchObject({ + how: 'only', + model: depthA, + }); + expect(resolve(layer('depth', { model: 'deleted' }), 'sdxl', [depthA])).toMatchObject({ + how: 'only', + model: depthA, + }); + }); + + it('runs the only model made for the type', () => { + expect(resolve(layer('pose'), 'sdxl', [depthA, pose, sdxlUnion])).toMatchObject({ + how: 'only', + kind: 'controlnet', + model: pose, + }); + }); + + it('asks when several models fit, listing those made for the type first, then unions', () => { + const fluxDepth = model('flux-depth-lora', 'flux', 'control_lora'); + const fluxUnion = model('flux-union-pro', 'flux'); + const fluxUnion2 = model('flux-union-alpha', 'flux'); + + expect(resolve(layer('depth'), 'sdxl', [depthA, depthB])).toMatchObject({ + candidates: [depthA, depthB], + how: 'ask', + model: null, + unresolved: 'choose_model', + }); + expect(resolve(layer('depth'), 'flux', [fluxUnion, fluxDepth]).candidates).toEqual([fluxDepth, fluxUnion]); + expect(resolve(layer('edges'), 'flux', [fluxUnion, fluxUnion2])).toMatchObject({ + candidates: [fluxUnion, fluxUnion2], + how: 'ask', + unresolved: 'choose_model', + }); + }); + + it('settles a choice with the remembered pick for this base and type, only while it is a candidate', () => { + const models = [depthA, depthB, pose]; + + expect(resolve(layer('depth'), 'sdxl', models, { 'sdxl:depth': depthB.key })).toMatchObject({ + how: 'remembered', + model: depthB, + }); + expect(resolve(layer('depth'), 'sdxl', models, { 'sdxl:depth': pose.key }).how).toBe('ask'); + expect(resolve(layer('depth'), 'sdxl', models, { 'sdxl:depth': 'deleted' }).how).toBe('ask'); + expect(resolve(layer('depth'), 'sdxl', models, { 'sd-1:depth': depthB.key }).how).toBe('ask'); + }); + + it('matches the family the chosen detector produces', () => { + const canny = model('sdxl-canny', 'sdxl'); + const mlsd = model('sdxl-mlsd', 'sdxl'); + + expect( + resolve(layer('depth', { detector: 'depth_anything_depth_estimation' }), 'sdxl', [depthA, normal]).model + ).toBe(depthA); + expect(resolve(layer('depth', { detector: 'normal_map' }), 'sdxl', [depthA, normal]).model).toBe(normal); + expect(resolve(layer('edges', { detector: 'mlsd_detection' }), 'sdxl', [canny, mlsd]).model).toBe(mlsd); + expect(resolve(layer('edges', { detector: null }), 'sdxl', [canny, mlsd]).model).toBe(canny); + }); + + it('falls back to a model made for the type when none matches the family', () => { + expect(resolve(layer('depth'), 'sdxl', [normal])).toMatchObject({ how: 'only', model: normal }); + }); + + it('uses the one union model that covers the family', () => { + const zUnion = model('z-image-fun-controlnet', 'z-image'); + + expect(resolve(layer('depth'), 'z-image', [zUnion])).toMatchObject({ + how: 'union', + kind: 'z_image_control', + model: zUnion, + }); + expect(resolve(layer('lineart'), 'z-image', [zUnion])).toMatchObject({ + how: 'missing', + kind: 'z_image_control', + unresolved: 'no_model_for_type', + }); + }); + + it('tells no control models apart from none for this type', () => { + expect(resolve(layer('pose'), 'sdxl', [depthA, sd1Depth])).toMatchObject({ + how: 'missing', + unresolved: 'no_model_for_type', + }); + expect(resolve(layer('pose'), 'sdxl', [sd1Depth])).toMatchObject({ + how: 'missing', + kind: 'controlnet', + unresolved: 'no_control_models', + }); + // An SDXL union carries no coverage until the backend reports its control types. + expect(resolve(layer('pose'), 'sdxl', [sdxlUnion]).unresolved).toBe('no_model_for_type'); + }); + + it('only counts models the base runs and validation would accept', () => { + const t2i = model('t2i-depth', 'sdxl', 't2i_adapter'); + const noT2i = (kind: ControlAdapterKind) => kind !== 't2i_adapter'; + const lllite = model('anima-lllite-depth', 'anima', 'controlnet', { cond_in_channels: 3 }); + const inpaint = model('anima-lllite-depth-inpaint', 'anima', 'controlnet', { cond_in_channels: 4 }); + + expect(resolve(layer('depth'), 'sdxl', [t2i], {}, noT2i).unresolved).toBe('no_control_models'); + expect(resolve(layer('depth', { model: t2i.key }), 'sdxl', [t2i, depthA], {}, noT2i).model).toBe(depthA); + expect(resolve(layer('depth'), 'anima', [lllite, inpaint])).toMatchObject({ + how: 'only', + kind: 'anima_lllite', + model: lllite, + }); + }); + + it('considers every base when no main model is selected', () => { + expect(resolve(layer('depth'), null, [sd1Depth]).model).toBe(sd1Depth); + }); +}); + +describe('resolveControlModel model lists and family match', () => { + it('lists every model the layer could run on this base, beside the candidates for its type', () => { + const resolution = resolve(layer('depth'), 'sdxl', [ + depthA, + pose, + sdxlUnion, + sd1Depth, + model('sdxl-main', 'sdxl', 'main'), + ]); + + expect(resolution.candidates).toEqual([depthA]); + expect(resolution.usable).toEqual([depthA, pose, sdxlUnion]); + expect(resolve(layer(undefined, { model: pose.key }), 'sdxl', [pose]).usable).toEqual([]); + }); + + it('says whether the model matches the detector family or only the type', () => { + const fluxUnion = model('flux-union-pro', 'flux'); + + expect(resolve(layer('depth'), 'sdxl', [depthA]).familyMatch).toBe(true); + expect(resolve(layer('depth'), 'sdxl', [normal])).toMatchObject({ familyMatch: false, how: 'only', model: normal }); + expect(resolve(layer('depth'), 'sdxl', [normal], { 'sdxl:depth': normal.key }).familyMatch).toBe(false); + expect(resolve(layer('depth'), 'flux', [fluxUnion])).toMatchObject({ familyMatch: true, how: 'union' }); + expect(resolve(layer('depth', { model: pose.key }), 'sdxl', [pose])).toMatchObject({ + familyMatch: false, + how: 'explicit', + }); + expect(resolve(layer('depth'), 'sdxl', [depthA, depthB]).familyMatch).toBe(false); + }); +}); + +describe('resolveControlModel InstantX modes', () => { + const fluxUnion = model('FLUX.1-dev-Controlnet-Union-Pro', 'flux'); + + it.each([ + ['edges', 0], + ['tile', 1], + ['depth', 2], + ['pose', 4], + ] as const)('gives a FLUX union %s layer mode %s', (type, mode) => { + expect(resolve(layer(type), 'flux', [fluxUnion])).toMatchObject({ instantxControlMode: mode, model: fluxUnion }); + }); + + it('leaves the mode out for families InstantX lacks, dedicated FLUX models and other bases', () => { + expect( + resolve(layer('edges', { detector: 'mlsd_detection', model: fluxUnion.key }), 'flux', [fluxUnion]) + ).toMatchObject({ + how: 'explicit', + instantxControlMode: null, + }); + expect( + resolve(layer('depth'), 'flux', [model('flux-depth-lora', 'flux', 'control_lora')]).instantxControlMode + ).toBeNull(); + expect(resolve(layer('depth'), 'z-image', [model('z-image-fun', 'z-image')]).instantxControlMode).toBeNull(); + }); +}); + +describe('getTypeAvailability', () => { + const byType = (availability: ReturnType) => + Object.fromEntries(availability.map(({ count, state, type }) => [type, [state, count]])); + + it('hints how each type would resolve on a fresh layer', () => { + expect(byType(getTypeAvailability('sdxl', [depthA, depthB, pose, sdxlUnion, sd1Depth], supportsAll))).toEqual({ + color: ['missing', 0], + depth: ['ask', 2], + edges: ['missing', 0], + lineart: ['missing', 0], + pose: ['ready', 1], + sketch: ['missing', 0], + softedge: ['missing', 0], + tile: ['missing', 0], + }); + expect( + byType( + getTypeAvailability( + 'flux', + [model('flux-depth-lora', 'flux', 'control_lora'), model('flux-union', 'flux')], + supportsAll + ) + ) + ).toMatchObject({ depth: ['ready', 1], edges: ['union', 1], lineart: ['missing', 0], tile: ['union', 1] }); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/controlModelResolution.ts b/invokeai/frontend/webv2/src/workbench/controlModelResolution.ts new file mode 100644 index 00000000000..7e14a58dfd6 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/controlModelResolution.ts @@ -0,0 +1,190 @@ +import type { ControlAdapterKind } from '@features/generation/graph'; +import type { ModelConfig } from '@features/models'; +import type { CanvasControlLayerContract, ControlType } from '@workbench/canvas-engine/api'; + +import { getControlModelUnusableReason } from '@features/generation/graph'; +import { CONTROL_TYPES } from '@workbench/canvas-engine/api'; + +import { deriveControlKind, getDefaultControlAdapterKind } from './controlAdapters'; +import { CONTROL_TYPE_SPECS, classifyControlModel, getDetectorFamily, type ControlFamily } from './controlTypes'; + +/** The user's last model pick per main-model base and control type: `${base}:${controlType}` → model key. */ +export type ControlModelMemory = Readonly>; + +export const controlModelMemoryKey = (base: string, type: ControlType): string => `${base}:${type}`; + +/** + * Why no model drives a layer; validation reports it in place of `missing_model`. `missing_type`: no type and no + * model, so the layer contributes nothing. `missing_model`: an untyped layer names a model that is not installed. + */ +export type ControlModelUnresolvedReason = + | 'missing_type' + | 'missing_model' + | 'choose_model' + | 'no_model_for_type' + | 'no_control_models'; + +export interface ControlModelResolution { + /** + * `explicit`: the layer's own pick. `remembered`: the last pick for this base and type. `only`: the one model made + * for the type. `union`: the one union model covering it. `ask`: several fit. `missing`: none fit. `none`: the + * layer has no type, so its stored adapter applies as it is. + */ + how: 'explicit' | 'remembered' | 'only' | 'union' | 'ask' | 'missing' | 'none'; + model: ModelConfig | null; + /** The kind the layer runs as: the model's own, else the base's default (the stored kind for untyped layers). */ + kind: ControlAdapterKind; + /** Models that fit the type: those made for it first, then covering unions. */ + candidates: readonly ModelConfig[]; + /** Every model the layer could run on this base, whatever it was made for; empty for untyped layers. */ + usable: readonly ModelConfig[]; + /** + * Whether the model was made for (or, as a union, covers) the family the detector produces; false when it only + * shares the type's purpose (a normal-map model on a depth layer), and for untyped or unresolved layers. + */ + familyMatch: boolean; + /** FLUX InstantX Union mode for this type's family; null for every other model. */ + instantxControlMode: number | null; + unresolved: ControlModelUnresolvedReason | null; +} + +type SupportsKind = (kind: ControlAdapterKind) => boolean; + +interface CandidateSets { + usable: ModelConfig[]; + dedicated: ModelConfig[]; + unions: ModelConfig[]; +} + +/** A model a layer on `base` can run: same base, a control kind the base supports, and nothing about it refused. */ +const usableKind = (model: ModelConfig, base: string | null, supportsKind: SupportsKind): ControlAdapterKind | null => { + if (base !== null && model.base !== base) { + return null; + } + const kind = deriveControlKind(model); + return kind && supportsKind(kind) && getControlModelUnusableReason(model, kind) === null ? kind : null; +}; + +const collectCandidates = ( + type: ControlType, + family: ControlFamily, + base: string | null, + models: readonly ModelConfig[], + supportsKind: SupportsKind +): CandidateSets => { + const usable = models.filter((model) => usableKind(model, base, supportsKind) !== null); + const classes = new Map(usable.map((model) => [model, classifyControlModel(model)])); + const byFamily = usable.filter((model) => classes.get(model)!.family === family); + return { + dedicated: byFamily.length > 0 ? byFamily : usable.filter((model) => classes.get(model)!.purpose === type), + unions: usable.filter((model) => { + const modelClass = classes.get(model)!; + return modelClass.union && modelClass.covers.includes(family); + }), + usable, + }; +}; + +/** + * Which model drives a control layer: an explicit pick for this base, the remembered pick for (base, type), the + * only model made for the type, a choice between several, the one covering union model, or none. A pick made for + * another base is ignored, not cleared, so switching back restores it. + */ +export const resolveControlModel = (input: { + layer: Pick; + base: string | null; + models: readonly ModelConfig[]; + remembered: ControlModelMemory; + supportsKind: SupportsKind; +}): ControlModelResolution => { + const { base, layer, models, remembered, supportsKind } = input; + const picked = layer.adapter.model ? (models.find((model) => model.key === layer.adapter.model) ?? null) : null; + const type = layer.controlType; + + if (!type) { + return { + candidates: [], + familyMatch: false, + how: 'none', + instantxControlMode: null, + kind: layer.adapter.kind, + model: picked, + unresolved: layer.adapter.model ? (picked ? null : 'missing_model') : 'missing_type', + usable: [], + }; + } + + const family = getDetectorFamily(type, layer.detection?.type ?? null); + const { dedicated, unions, usable } = collectCandidates(type, family, base, models, supportsKind); + const candidates = [...dedicated, ...unions]; + const settle = ( + how: ControlModelResolution['how'], + model: ModelConfig | null, + unresolved: ControlModelUnresolvedReason | null = null + ): ControlModelResolution => { + const modelClass = model ? classifyControlModel(model) : null; + return { + candidates, + familyMatch: + modelClass !== null && + (modelClass.family === family || (modelClass.union && modelClass.covers.includes(family))), + how, + instantxControlMode: + base === 'flux' && modelClass?.union ? (CONTROL_TYPE_SPECS[type].instantxMode[family] ?? null) : null, + kind: (model ? deriveControlKind(model) : null) ?? getDefaultControlAdapterKind(base), + model, + unresolved, + usable, + }; + }; + + if (picked && usableKind(picked, base, supportsKind)) { + return settle('explicit', picked); + } + const rememberedKey = base === null ? undefined : remembered[controlModelMemoryKey(base, type)]; + const rememberedModel = rememberedKey ? candidates.find((model) => model.key === rememberedKey) : undefined; + if (rememberedModel) { + return settle('remembered', rememberedModel); + } + if (dedicated.length === 1) { + return settle('only', dedicated[0]!); + } + if (dedicated.length > 1) { + return settle('ask', null, 'choose_model'); + } + if (unions.length === 1) { + return settle('union', unions[0]!); + } + if (unions.length > 1) { + return settle('ask', null, 'choose_model'); + } + return settle('missing', null, usable.length === 0 ? 'no_control_models' : 'no_model_for_type'); +}; + +/** How a control type would resolve on a fresh layer, for the Type menu's hints. */ +export interface ControlTypeAvailability { + type: ControlType; + /** `ready`: one model made for it. `ask`: several. `union`: only union models cover it. `missing`: nothing. */ + state: 'ready' | 'ask' | 'union' | 'missing'; + /** Models behind the state: those made for the type, or the covering unions. */ + count: number; +} + +export const getTypeAvailability = ( + base: string | null, + models: readonly ModelConfig[], + supportsKind: SupportsKind +): readonly ControlTypeAvailability[] => + CONTROL_TYPES.map((type) => { + const { dedicated, unions } = collectCandidates( + type, + CONTROL_TYPE_SPECS[type].primaryFamily, + base, + models, + supportsKind + ); + if (dedicated.length > 0) { + return { count: dedicated.length, state: dedicated.length === 1 ? 'ready' : 'ask', type }; + } + return unions.length > 0 ? { count: unions.length, state: 'union', type } : { count: 0, state: 'missing', type }; + }); diff --git a/invokeai/frontend/webv2/src/workbench/controlTypes.test.ts b/invokeai/frontend/webv2/src/workbench/controlTypes.test.ts new file mode 100644 index 00000000000..db3c78575d2 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/controlTypes.test.ts @@ -0,0 +1,144 @@ +import type { ModelConfig } from '@features/models'; + +import { CONTROL_TYPES } from '@workbench/canvas-engine/api'; +import { CONTROL_FILTERS } from '@workbench/canvas-operations/filterGraphs'; +import { describe, expect, it } from 'vitest'; + +import { + CONTROL_TYPE_SPECS, + classifyControlModel, + controlTypeForDetector, + getDetectorFamily, + PREPROCESSOR_FAMILY, + type ControlModelClass, +} from './controlTypes'; + +describe('CONTROL_TYPE_SPECS', () => { + it('covers every control type with detectors the filter catalog runs', () => { + const filterTypes = new Set(CONTROL_FILTERS.map((filter) => filter.type)); + + expect(Object.keys(CONTROL_TYPE_SPECS).sort()).toEqual([...CONTROL_TYPES].sort()); + for (const [type, spec] of Object.entries(CONTROL_TYPE_SPECS)) { + expect(spec.id).toBe(type); + expect(spec.detectors.filter((detector) => !filterTypes.has(detector.type))).toEqual([]); + if (spec.defaultDetector !== null) { + expect(spec.detectors.map((detector) => detector.type)).toContain(spec.defaultDetector); + } + } + }); + + it('maps every generic detector to the family its spec names, so model and detector families agree', () => { + for (const spec of Object.values(CONTROL_TYPE_SPECS)) { + for (const detector of spec.detectors.filter((entry) => !entry.settings)) { + expect(PREPROCESSOR_FAMILY[detector.type], detector.type).toBe(detector.family); + } + } + }); +}); + +describe('getDetectorFamily', () => { + it.each([ + ['depth', 'normal_map', 'normal'], + ['depth', 'depth_anything_depth_estimation', 'depth'], + ['depth', null, 'depth'], + ['edges', 'mlsd_detection', 'mlsd'], + ['sketch', 'pidi_edge_detection', 'scribble'], + ['sketch', null, 'scribble'], + ['tile', null, 'tile'], + ['pose', 'canny_edge_detection', 'openpose'], + ] as const)('%s with %s produces %s', (type, detector, family) => { + expect(getDetectorFamily(type, detector)).toBe(family); + }); +}); + +describe('controlTypeForDetector', () => { + it.each([ + [{ settings: { scribble: true }, type: 'pidi_edge_detection' }, 'sketch'], + [{ settings: { scribble: false }, type: 'pidi_edge_detection' }, 'softedge'], + [{ type: 'pidi_edge_detection' }, 'softedge'], + [{ settings: {}, type: 'scribble_edge_detection' }, 'sketch'], + [{ settings: {}, type: 'normal_map' }, 'depth'], + [{ settings: {}, type: 'mediapipe_face_detection' }, 'pose'], + [{ settings: {}, type: 'content_shuffle' }, 'color'], + [{ settings: {}, type: 'img_blur' }, null], + ])('%j belongs to %s', (detector, type) => { + expect(controlTypeForDetector(detector)).toBe(type); + }); +}); + +const model = ( + name: string, + base: string, + type = 'controlnet', + preprocessor: string | null = null +): Pick => + ({ base, default_settings: preprocessor ? { preprocessor } : null, name, type }) as ModelConfig; + +const made = (purpose: ControlModelClass['purpose'], family: ControlModelClass['family']): ControlModelClass => ({ + covers: [], + family, + purpose, + union: false, +}); + +const FUN_COVERS = ['canny', 'mlsd', 'softedge', 'depth', 'openpose']; + +describe('classifyControlModel', () => { + it.each([ + [ + 'legacy preprocessor id', + model('My ControlNet', 'sdxl', 'controlnet', 'depth_anything_image_processor'), + made('depth', 'depth'), + ], + [ + 'legacy zoe id', + model('My ControlNet', 'sdxl', 'controlnet', 'zoe_depth_image_processor'), + made('depth', 'depth'), + ], + ['node id', model('My ControlNet', 'sdxl', 'controlnet', 'normal_map'), made('depth', 'normal')], + ['legacy mlsd id', model('Lines', 'sd-1', 'controlnet', 'mlsd_image_processor'), made('edges', 'mlsd')], + ['tile preprocessor', model('Custom', 'sdxl', 'controlnet', 'tile'), made('tile', 'tile')], + ['tile name', model('xinsir-controlnet-tile-sdxl', 'sdxl'), made('tile', 'tile')], + [ + 'scribble name over the backend lineart guess', + model('control_v11p_sd15_scribble', 'sd-1', 'controlnet', 'lineart_image_processor'), + made('sketch', 'scribble'), + ], + [ + 'anime line art over the preprocessor', + model('control_v11p_sd15s2_lineart_anime', 'sd-1', 'controlnet', 'lineart_image_processor'), + made('lineart', 'lineart_anime'), + ], + ['face name', model('controlnet mediapipe face', 'sd-1'), made('pose', 'face')], + ['name word', model('control_v11p_sd15_canny', 'sd-1'), made('edges', 'canny')], + ['name word prefix', model('control_v11p_sd15_normalbae', 'sd-1'), made('depth', 'normal')], + ['camelCase name', model('controlnetSoftEdge', 'sdxl'), made('softedge', 'softedge')], + ['openpose name', model('controlnet-openpose-sdxl-1.0', 'sdxl'), made('pose', 'openpose')], + ['no match inside unrelated words', model('cached-surface-model', 'sdxl'), made(null, null)], + ['unknown model', model('my-sdxl-control', 'sdxl'), made(null, null)], + ['T2I-Adapter', model('t2iadapter_canny_sd15v2', 'sd-1', 't2i_adapter'), made('edges', 'canny')], + ['Control LoRA', model('flux1-depth-dev-lora', 'flux', 'control_lora'), made('depth', 'depth')], + ['Anima LLLite', model('anima-lllite-sketch', 'anima', 'controlnet'), made('sketch', 'scribble')], + [ + 'saved preprocessor over a weak name word', + model('depth-something', 'sdxl', 'controlnet', 'canny_image_processor'), + made('edges', 'canny'), + ], + ] as const)('%s', (_label, input, expected) => { + expect(classifyControlModel(input)).toEqual(expected); + }); + + it.each([ + ['an SDXL union, offered only as an explicit pick', model('controlnet-union-sdxl-promax', 'sdxl'), []], + [ + 'a FLUX union, covering the InstantX modes', + model('FLUX.1-dev-Controlnet-Union-Pro', 'flux', 'controlnet', 'canny_image_processor'), + ['canny', 'tile', 'depth', 'openpose'], + ], + ['a Z-Image Fun union whatever its name', model('Z-Image-Turbo-Fun-Controlnet', 'z-image'), FUN_COVERS], + ['a Qwen-Image Fun union', model('Qwen-Image-2512-Fun-Controlnet-Union', 'qwen-image'), FUN_COVERS], + ['a FLUX.2 Fun union', model('FLUX.2-dev-Fun-Controlnet', 'flux2'), FUN_COVERS], + ] as const)('classifies %s', (_label, input, covers) => { + expect(classifyControlModel(input)).toEqual({ covers, family: null, purpose: null, union: true }); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/controlTypes.ts b/invokeai/frontend/webv2/src/workbench/controlTypes.ts new file mode 100644 index 00000000000..4693e3f4d11 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/controlTypes.ts @@ -0,0 +1,295 @@ +import type { ModelConfig } from '@features/models'; +import type { ControlType } from '@workbench/canvas-engine/api'; + +/** + * What a control map encodes. A control type groups the families its detectors produce; a control model is trained + * on one family (or, for union models, several). + */ +export type ControlFamily = + | 'openpose' + | 'face' + | 'depth' + | 'normal' + | 'canny' + | 'mlsd' + | 'softedge' + | 'lineart' + | 'lineart_anime' + | 'scribble' + | 'color' + | 'shuffle' + | 'tile'; + +/** A detector a control type offers: a filter type from `CONTROL_FILTERS`, with settings it fixes. */ +export interface ControlDetectorSpec { + type: string; + family: ControlFamily; + /** Settings this entry requires; a detector input matches it only when every one is equal. */ + settings?: Record; +} + +export interface ControlTypeSpec { + id: ControlType; + detectors: readonly ControlDetectorSpec[]; + /** The detector a new image starts with; null when the image is usually already the map. */ + defaultDetector: string | null; + /** The family of a map that skipped detection. */ + primaryFamily: ControlFamily; + /** FLUX InstantX Union `instantx_control_mode` per family; absent families run without a mode. */ + instantxMode: Partial>; +} + +export const CONTROL_TYPE_SPECS: Readonly> = { + color: { + defaultDetector: 'color_map', + detectors: [ + { family: 'color', type: 'color_map' }, + { family: 'shuffle', type: 'content_shuffle' }, + ], + id: 'color', + instantxMode: {}, + primaryFamily: 'color', + }, + depth: { + defaultDetector: 'depth_anything_depth_estimation', + detectors: [ + { family: 'depth', type: 'depth_anything_depth_estimation' }, + { family: 'normal', type: 'normal_map' }, + ], + id: 'depth', + instantxMode: { depth: 2 }, + primaryFamily: 'depth', + }, + edges: { + defaultDetector: 'canny_edge_detection', + detectors: [ + { family: 'canny', type: 'canny_edge_detection' }, + { family: 'mlsd', type: 'mlsd_detection' }, + ], + id: 'edges', + instantxMode: { canny: 0 }, + primaryFamily: 'canny', + }, + lineart: { + defaultDetector: 'lineart_edge_detection', + detectors: [ + { family: 'lineart', type: 'lineart_edge_detection' }, + { family: 'lineart_anime', type: 'lineart_anime_edge_detection' }, + ], + id: 'lineart', + instantxMode: {}, + primaryFamily: 'lineart', + }, + pose: { + defaultDetector: 'dw_openpose_detection', + detectors: [ + { family: 'openpose', type: 'dw_openpose_detection' }, + { family: 'face', type: 'mediapipe_face_detection' }, + ], + id: 'pose', + instantxMode: { openpose: 4 }, + primaryFamily: 'openpose', + }, + // Sketches are usually drawn as the map already, so a new sketch layer skips detection. + sketch: { + defaultDetector: null, + detectors: [ + { family: 'scribble', type: 'scribble_edge_detection' }, + { family: 'scribble', settings: { scribble: true }, type: 'pidi_edge_detection' }, + ], + id: 'sketch', + instantxMode: {}, + primaryFamily: 'scribble', + }, + softedge: { + defaultDetector: 'hed_edge_detection', + detectors: [ + { family: 'softedge', type: 'hed_edge_detection' }, + { family: 'softedge', type: 'pidi_edge_detection' }, + ], + id: 'softedge', + instantxMode: {}, + primaryFamily: 'softedge', + }, + tile: { + defaultDetector: null, + detectors: [], + id: 'tile', + instantxMode: { tile: 1 }, + primaryFamily: 'tile', + }, +}; + +const settingsMatch = (required: Record, actual: Record | undefined): boolean => + Object.entries(required).every(([key, value]) => actual?.[key] === value); + +/** The family a type's detector produces; no detector (or one the type does not list) means its primary family. */ +export const getDetectorFamily = (type: ControlType, detectorType: string | null): ControlFamily => { + const spec = CONTROL_TYPE_SPECS[type]; + return spec.detectors.find((detector) => detector.type === detectorType)?.family ?? spec.primaryFamily; +}; + +/** + * The control type a detector run belongs to, for maps made outside a control layer (a raster filter saved as + * control). An entry that fixes settings beats a generic one, so PiDi with `scribble` is a sketch. + */ +export const controlTypeForDetector = (detector: { + type: string; + settings?: Record; +}): ControlType | null => { + let generic: ControlType | null = null; + for (const spec of Object.values(CONTROL_TYPE_SPECS)) { + for (const entry of spec.detectors) { + if (entry.type !== detector.type) { + continue; + } + if (entry.settings) { + if (settingsMatch(entry.settings, detector.settings)) { + return spec.id; + } + } else { + generic ??= spec.id; + } + } + } + return generic; +}; + +/** + * Model `default_settings.preprocessor` values mapped to the family they produce: the legacy `*_image_processor` + * ids older installs carry, the filter node ids the model manager saves today, and `tile`. + */ +export const PREPROCESSOR_FAMILY: Readonly> = { + canny_edge_detection: 'canny', + canny_image_processor: 'canny', + color_map: 'color', + color_map_image_processor: 'color', + content_shuffle: 'shuffle', + content_shuffle_image_processor: 'shuffle', + depth_anything_depth_estimation: 'depth', + depth_anything_image_processor: 'depth', + dw_openpose_detection: 'openpose', + dw_openpose_image_processor: 'openpose', + hed_edge_detection: 'softedge', + hed_image_processor: 'softedge', + lineart_anime_edge_detection: 'lineart_anime', + lineart_anime_image_processor: 'lineart_anime', + lineart_edge_detection: 'lineart', + lineart_image_processor: 'lineart', + mediapipe_face_detection: 'face', + mediapipe_face_processor: 'face', + midas_depth_image_processor: 'depth', + mlsd_detection: 'mlsd', + mlsd_image_processor: 'mlsd', + normal_map: 'normal', + normalbae_image_processor: 'normal', + openpose_image_processor: 'openpose', + pidi_edge_detection: 'softedge', + pidi_image_processor: 'softedge', + scribble_edge_detection: 'scribble', + tile: 'tile', + zoe_depth_image_processor: 'depth', +}; + +const FAMILY_PURPOSE: Readonly> = { + canny: 'edges', + color: 'color', + depth: 'depth', + face: 'pose', + lineart: 'lineart', + lineart_anime: 'lineart', + mlsd: 'edges', + normal: 'depth', + openpose: 'pose', + scribble: 'sketch', + shuffle: 'color', + softedge: 'softedge', + tile: 'tile', +}; + +/** What a control model was trained for, as far as its name and saved preprocessor tell. */ +export interface ControlModelClass { + /** The control type it is made for; null for union models and models nothing identifies. */ + purpose: ControlType | null; + family: ControlFamily | null; + union: boolean; + /** Families a union model handles without an explicit pick. */ + covers: readonly ControlFamily[]; +} + +/** + * The Fun-ControlNet-Union models these bases install as `controlnet`. Both coverage lists follow the model cards' + * advertised conditions and still need confirming against them. + */ +const FUN_UNION_BASES: ReadonlySet = new Set(['z-image', 'qwen-image', 'flux2']); +const FUN_UNION_COVERS: readonly ControlFamily[] = ['canny', 'mlsd', 'softedge', 'depth', 'openpose']; +/** InstantX FLUX.1 Union modes the control types map onto. */ +const FLUX_UNION_COVERS: readonly ControlFamily[] = ['canny', 'tile', 'depth', 'openpose']; + +/** Lowercase name words, split at punctuation and camelCase boundaries. */ +const nameTokens = (name: string): string[] => + name + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter(Boolean); + +/** Prefix matching catches `normalbae` and `softedge` without matching inside unrelated words. */ +const hasToken = (tokens: readonly string[], ...keywords: string[]): boolean => + tokens.some((token) => keywords.some((keyword) => token.startsWith(keyword))); + +/** Names that outrank a saved preprocessor: the backend guesses `lineart` for scribble models, for one. */ +const strongNameFamily = (tokens: readonly string[]): ControlFamily | null => { + if (hasToken(tokens, 'tile')) { + return 'tile'; + } + if (hasToken(tokens, 'scribble', 'sketch')) { + return 'scribble'; + } + if (hasToken(tokens, 'lineart') && hasToken(tokens, 'anime')) { + return 'lineart_anime'; + } + if (hasToken(tokens, 'mediapipe', 'face')) { + return 'face'; + } + return null; +}; + +const WEAK_NAME_FAMILIES: readonly (readonly [ControlFamily, readonly string[]])[] = [ + ['canny', ['canny']], + ['depth', ['depth']], + ['openpose', ['openpose', 'pose']], + ['softedge', ['hed', 'softedge', 'soft']], + ['lineart', ['lineart']], + ['mlsd', ['mlsd']], + ['normal', ['normal']], + ['color', ['color']], + ['shuffle', ['shuffle']], +]; + +const weakNameFamily = (tokens: readonly string[]): ControlFamily | null => + WEAK_NAME_FAMILIES.find(([, keywords]) => hasToken(tokens, ...keywords))?.[0] ?? null; + +const preprocessorFamily = (model: Pick): ControlFamily | null => { + const preprocessor = model.default_settings?.preprocessor; + return preprocessor && Object.hasOwn(PREPROCESSOR_FAMILY, preprocessor) ? PREPROCESSOR_FAMILY[preprocessor]! : null; +}; + +/** + * Classify a control model: union models first (a base's Fun union, a FLUX union, any other `union` name), then + * strong name words, the saved preprocessor, and the remaining name words. + */ +export const classifyControlModel = ( + model: Pick +): ControlModelClass => { + const tokens = nameTokens(model.name); + if (model.type === 'controlnet' && FUN_UNION_BASES.has(model.base)) { + return { covers: FUN_UNION_COVERS, family: null, purpose: null, union: true }; + } + if (hasToken(tokens, 'union')) { + // Other bases' union models need the backend to expose their control types; until then they are explicit picks. + return { covers: model.base === 'flux' ? FLUX_UNION_COVERS : [], family: null, purpose: null, union: true }; + } + const family = strongNameFamily(tokens) ?? preprocessorFamily(model) ?? weakNameFamily(tokens); + return { covers: [], family, purpose: family ? FAMILY_PURPOSE[family] : null, union: false }; +}; diff --git a/invokeai/frontend/webv2/src/workbench/invocation.test.ts b/invokeai/frontend/webv2/src/workbench/invocation.test.ts index b3ecc0d7928..5c68cdb7ced 100644 --- a/invokeai/frontend/webv2/src/workbench/invocation.test.ts +++ b/invokeai/frontend/webv2/src/workbench/invocation.test.ts @@ -17,11 +17,27 @@ import { queryClient } from '@platform/query/client'; import { accountLifecycle, captureAccountScope } from '@platform/state/accountLifecycle'; import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type * as settingsStoreModule from './settings/store'; + // Dynamic prompt expansion is the only transport `submitResolvedInvocation` touches. const parseDynamicPromptsMock = vi.hoisted(() => vi.fn()); vi.mock('@features/generation/data/promptUtilities', () => ({ parseDynamicPrompts: parseDynamicPromptsMock })); +// Remembered control-model picks, which readiness reads imperatively from the account preferences. +const preferences = vi.hoisted(() => ({ controlModelMemory: {} as Record })); + +vi.mock('./settings/store', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getWorkbenchPreferences: () => ({ + ...actual.getWorkbenchPreferences(), + controlModelMemory: preferences.controlModelMemory, + }), + }; +}); + import { seedArchitectureCapabilities } from '@features/generation/core/architectureCapabilities.testing'; import { @@ -340,9 +356,9 @@ describe('resolveInvocationRoute — canvas source', () => { expect(route.validationReasons).toContain('The Canvas widget is not mounted in this project.'); }); - it('blocks a content-bearing control layer with no model once models are loaded', () => { + it('blocks a typed content-bearing control layer no model resolves once models are loaded', () => { const route = resolveInvocationRouteInput( - canvasInputFor({ layers: [modellessControlLayer()] }), + canvasInputFor({ layers: [{ ...modellessControlLayer(), controlType: 'depth', detection: null }] }), 'global', canvasRoute, loadedModels @@ -352,7 +368,7 @@ describe('resolveInvocationRoute — canvas source', () => { // Kept structured so the shell words it from the locale. expect(route.validationReasons).toContainEqual({ controlLayerIssue: { - code: 'missing_model', + code: 'no_control_models', layerId: expect.any(String), layerName: 'Control Layer 1', suggestedKind: null, @@ -360,6 +376,36 @@ describe('resolveInvocationRoute — canvas source', () => { }); }); + it('does not block on a content-bearing layer with neither a type nor a model', () => { + const route = resolveInvocationRouteInput( + canvasInputFor({ layers: [modellessControlLayer()] }), + 'global', + canvasRoute, + loadedModels + ); + + expect(route.validationReasons).toEqual([]); + }); + + it('re-evaluates a choice between control models against the remembered pick', () => { + const lllite = (key: string) => + ({ base: 'anima', cond_in_channels: 3, key, name: key, type: 'controlnet' }) as unknown as ModelConfig; + const models = [...loadedModels, lllite('anima-depth-a'), lllite('anima-depth-b')]; + const input = canvasInputFor({ layers: [{ ...modellessControlLayer(), controlType: 'depth', detection: null }] }); + const codes = () => + resolveInvocationRouteInput(input, 'global', canvasRoute, models).validationReasons.map((reason) => + typeof reason === 'object' && 'controlLayerIssue' in reason ? reason.controlLayerIssue.code : reason + ); + + try { + expect(codes()).toEqual(['choose_model']); + preferences.controlModelMemory = { 'anima:depth': 'anima-depth-b' }; + expect(codes()).toEqual([]); + } finally { + preferences.controlModelMemory = {}; + } + }); + it('does not block content-less control layers or check layers before models load', () => { const emptyLayerRoute = resolveInvocationRouteInput( canvasInputFor({ layers: [modellessControlLayer({ bitmap: null, type: 'paint' })] }), diff --git a/invokeai/frontend/webv2/src/workbench/invocation.ts b/invokeai/frontend/webv2/src/workbench/invocation.ts index cf562fb4756..db6d9ca0f25 100644 --- a/invokeai/frontend/webv2/src/workbench/invocation.ts +++ b/invokeai/frontend/webv2/src/workbench/invocation.ts @@ -27,8 +27,13 @@ import { localizeForLoopValidationReason } from '@features/workflow/utility'; import { areArraysEqual, createStableSelector } from '@platform/state/selectors'; import { compileContributingLayers } from '@workbench/canvas-engine/api'; -import { describeControlLayerIssue, getBlockingControlLayerIssues } from './controlLayerChecks'; +import { + createControlLayerCheckContext, + describeControlLayerIssue, + getBlockingControlLayerIssues, +} from './controlLayerChecks'; import { getActiveProjectGraph } from './projectWorkflows'; +import { getWorkbenchPreferences } from './settings/store'; import { getProjectWidgetValues } from './widgetState'; export interface InvocationSourceMeta { @@ -206,7 +211,7 @@ export const resolveInvocationRouteInput = ( const sourceId = route.sourceId; const destination = route.destination; const sourceWidgetId = sourceWidgetIds[sourceId]; - // Route-rendering surfaces subscribe to templates because readiness reads them imperatively. + // Route-rendering surfaces subscribe to templates and preferences because readiness reads them imperatively. const projectGraphReadiness = sourceId === 'workflow' ? getProjectGraphReadiness(input.projectGraph, getInvocationTemplatesSnapshot(), { @@ -255,6 +260,7 @@ export const resolveInvocationRouteInput = ( if (values && isSupportedGenerateModel(values.model)) { validationReasons.push( ...getBlockingControlLayerIssues({ + context: createControlLayerCheckContext(values.model.base, getWorkbenchPreferences().controlModelMemory), layers: input.canvasLayers, mainModel: values.model, models, diff --git a/invokeai/frontend/webv2/src/workbench/settings/contracts.ts b/invokeai/frontend/webv2/src/workbench/settings/contracts.ts index e1a04184969..4fef471f54d 100644 --- a/invokeai/frontend/webv2/src/workbench/settings/contracts.ts +++ b/invokeai/frontend/webv2/src/workbench/settings/contracts.ts @@ -127,6 +127,11 @@ export interface WorkbenchPreferences { krea2RebalancePresets: StoredRebalancePreset[]; /** User-saved Generate settings snapshots ("recipes"), in the order they were saved. */ generatePresets: StoredGeneratePreset[]; + /** + * The last control model picked per main-model base and control type (`${base}:${controlType}` → model key); + * control layers left on automatic use it. Not shown in Settings. + */ + controlModelMemory: Record; } export type SettingsSectionId = string; diff --git a/invokeai/frontend/webv2/src/workbench/settings/store.test.ts b/invokeai/frontend/webv2/src/workbench/settings/store.test.ts index b48ccfbbb1f..b0b4cbf0975 100644 --- a/invokeai/frontend/webv2/src/workbench/settings/store.test.ts +++ b/invokeai/frontend/webv2/src/workbench/settings/store.test.ts @@ -1,6 +1,7 @@ import type * as accountLifecycleModule from '@platform/state/accountLifecycle'; import { LOG_NAMESPACES } from '@platform/logging/contracts'; +import { CONTROL_TYPES } from '@workbench/canvas-engine/api'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { ProjectSettings } from './contracts'; @@ -556,3 +557,70 @@ describe('normalizeWorkbenchPreferences language', () => { expect(store.normalizeWorkbenchPreferences({ language: 'missing' as never }).language).toBe('en'); }); }); + +describe('controlModelMemory preference', () => { + it('starts empty and keeps only well-formed picks', () => { + expect(store.normalizeWorkbenchPreferences({}).controlModelMemory).toEqual({}); + expect( + store.normalizeWorkbenchPreferences({ controlModelMemory: ['sdxl:depth'] as never }).controlModelMemory + ).toEqual({}); + expect( + store.normalizeWorkbenchPreferences({ + controlModelMemory: { + ':depth': 'no-base', + 'sdxl:depth': 'depth-model', + 'sdxl:flux': 'unknown-type', + 'sdxl:pose': '', + 'sdxl:tile': 'x'.repeat(256), + 'sd-1:edges': 7 as never, + sdxl: 'no-type', + }, + }).controlModelMemory + ).toEqual({ 'sdxl:depth': 'depth-model' }); + }); + + it('accepts every control type, so the spelled-out key pattern stays in step with the canvas contract', () => { + const memory = Object.fromEntries(CONTROL_TYPES.map((type) => [`sdxl:${type}`, `${type}-model`])); + + expect(store.normalizeWorkbenchPreferences({ controlModelMemory: memory }).controlModelMemory).toEqual(memory); + }); + + it('keeps the newest 256 picks', () => { + const memory = Object.fromEntries(Array.from({ length: 300 }, (_, index) => [`base-${index}:depth`, `m${index}`])); + const kept = store.normalizeWorkbenchPreferences({ controlModelMemory: memory }).controlModelMemory; + + expect(Object.keys(kept)).toHaveLength(256); + expect(kept['base-299:depth']).toBe('m299'); + expect(kept['base-43:depth']).toBeUndefined(); + }); + + it('keeps the same record across unrelated preference patches, so readers that compare it do not re-run', async () => { + await store.loadWorkbenchSettings(); + await store.patchControlModelMemory('sdxl:depth', 'depth-a'); + const memory = store.getWorkbenchPreferences().controlModelMemory; + + await store.patchWorkbenchPreferences({ reduceMotion: true }); + + expect(store.getWorkbenchPreferences().controlModelMemory).toBe(memory); + expect(store.normalizeWorkbenchPreferences({ controlModelMemory: memory }).controlModelMemory).toBe(memory); + }); + + it('remembers, replaces and forgets a pick through the persisted preferences, and resets with the rest', async () => { + await store.loadWorkbenchSettings(); + + await store.patchControlModelMemory('sdxl:depth', 'depth-a'); + await store.patchControlModelMemory('flux:pose', 'flux-union'); + await store.patchControlModelMemory('sdxl:depth', 'depth-b'); + + expect(readBackendPreferences().controlModelMemory).toEqual({ 'flux:pose': 'flux-union', 'sdxl:depth': 'depth-b' }); + + await store.patchControlModelMemory('flux:pose', null); + + expect(store.getWorkbenchPreferences().controlModelMemory).toEqual({ 'sdxl:depth': 'depth-b' }); + expect(readBackendPreferences().controlModelMemory).toEqual({ 'sdxl:depth': 'depth-b' }); + + await store.clearWorkbenchSettings(); + + expect(store.getWorkbenchPreferences().controlModelMemory).toEqual({}); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/settings/store.ts b/invokeai/frontend/webv2/src/workbench/settings/store.ts index 1c8a7b205c0..e25526f3f53 100644 --- a/invokeai/frontend/webv2/src/workbench/settings/store.ts +++ b/invokeai/frontend/webv2/src/workbench/settings/store.ts @@ -50,6 +50,7 @@ export const DEFAULT_PREFERENCES: WorkbenchPreferences = { alphaNoticeAcknowledged: false, autoSwitchInvocationRoute: true, confirmImageDeletion: true, + controlModelMemory: {}, customHotkeys: {}, developerConsoleOutputEnabled: false, developerLogEnabled: true, @@ -265,6 +266,38 @@ const normalizeGeneratePresets = (values: unknown): StoredGeneratePreset[] => { return presets; }; +/** + * `${base}:${controlType}`. The control types are spelled out because importing them from the canvas engine would + * pull it into Launchpad's bundle; a store test keeps the list in step with `CONTROL_TYPES`. + */ +const CONTROL_MODEL_MEMORY_KEY = /^[^:]+:(?:pose|depth|edges|softedge|lineart|sketch|color|tile)$/; +const CONTROL_MODEL_MEMORY_MAX_ENTRIES = 256; +const CONTROL_MODEL_KEY_MAX_LENGTH = 255; + +/** + * Keeps well-formed entries, the newest {@link CONTROL_MODEL_MEMORY_MAX_ENTRIES} (a pick is re-inserted last), so the + * synced blob stays bounded. An already-valid record comes back as is, so patching another preference keeps its + * identity and does not re-run selectors that read it (Invoke readiness). + */ +const normalizeControlModelMemory = (values: unknown): Record => { + if (!values || typeof values !== 'object' || Array.isArray(values)) { + return {}; + } + + const entries = Object.entries(values); + const kept = entries + .filter( + (entry): entry is [string, string] => + CONTROL_MODEL_MEMORY_KEY.test(entry[0]) && + typeof entry[1] === 'string' && + entry[1].length > 0 && + entry[1].length <= CONTROL_MODEL_KEY_MAX_LENGTH + ) + .slice(-CONTROL_MODEL_MEMORY_MAX_ENTRIES); + + return kept.length === entries.length ? (values as Record) : Object.fromEntries(kept); +}; + export const isViewerAutoSwitchMode = (value: unknown): value is ViewerAutoSwitchMode => value === 'off' || value === 'latest' || value === 'progress'; @@ -308,6 +341,7 @@ export const normalizeWorkbenchPreferences = (preferences?: WorkbenchPreferences typeof preferences?.confirmImageDeletion === 'boolean' ? preferences.confirmImageDeletion : DEFAULT_PREFERENCES.confirmImageDeletion, + controlModelMemory: normalizeControlModelMemory(preferences?.controlModelMemory), customHotkeys: normalizeCustomHotkeys(preferences?.customHotkeys), developerLogEnabled: typeof preferences?.developerLogEnabled === 'boolean' @@ -700,6 +734,16 @@ export const patchWorkbenchPreferences = async (preferences: Partial => { + const controlModelMemory = { ...store.getSnapshot().preferences.controlModelMemory }; + delete controlModelMemory[key]; + if (modelKey !== null) { + controlModelMemory[key] = modelKey; + } + return patchWorkbenchPreferences({ controlModelMemory }); +}; + export const clearWorkbenchSettings = async (): Promise => { const owner = captureAccountScope(); const storageSuffix = owner.accountId === null ? getUserStorageScope() : owner.storageSuffix; diff --git a/invokeai/frontend/webv2/src/workbench/shell/topbar/useInvocationState.browser.test.tsx b/invokeai/frontend/webv2/src/workbench/shell/topbar/useInvocationState.browser.test.tsx index d0e5bdcaad1..4833f9a2008 100644 --- a/invokeai/frontend/webv2/src/workbench/shell/topbar/useInvocationState.browser.test.tsx +++ b/invokeai/frontend/webv2/src/workbench/shell/topbar/useInvocationState.browser.test.tsx @@ -1,5 +1,7 @@ import type { GenerateWidgetValues, MainModelConfig } from '@features/generation/contracts'; +import type { CanvasControlLayerContract } from '@workbench/canvas-engine/api'; import type { Project } from '@workbench/projectContracts'; +import type * as settingsStoreModule from '@workbench/settings/store'; import type { ReactNode } from 'react'; import { @@ -43,6 +45,29 @@ const templates = vi.hoisted(() => { }; }); +// A controllable stand-in for the account preferences, so a test can change the remembered control-model picks. +const preferences = vi.hoisted(() => { + const listeners = new Set<() => void>(); + let controlModelMemory: Record = {}; + + return { + get: () => controlModelMemory, + reset: () => { + controlModelMemory = {}; + }, + set: (next: Record) => { + controlModelMemory = next; + listeners.forEach((listener) => listener()); + }, + subscribe: (listener: () => void) => { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; + }, + }; +}); + vi.mock('@features/generation/react', () => ({ useDynamicPrompts: () => ({ count: 1, error: null, isDynamic: false, isError: false, isLoading: false, prompts: [] }), })); @@ -57,6 +82,14 @@ vi.mock('@features/workflow/react', async (importOriginal) => ({ getInvocationTemplatesSnapshot: () => templates.get(), subscribeInvocationTemplates: templates.subscribe, })); +vi.mock('@workbench/settings/store', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getWorkbenchPreferences: () => ({ ...actual.getWorkbenchPreferences(), controlModelMemory: preferences.get() }), + subscribeWorkbenchPreferences: (listener: () => void) => preferences.subscribe(listener), + }; +}); vi.mock('@workbench/activeInvocationSubmission', () => ({ submitActiveInvocation: harness.submit })); vi.mock('@workbench/invocationPreparation', () => ({ useIsInvocationPreparing: () => false })); vi.mock('@workbench/WorkbenchContext', () => ({ @@ -68,12 +101,17 @@ vi.mock('@workbench/WorkbenchContext', () => ({ })); vi.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string) => key }) })); +import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { createInitialWorkbenchState, workbenchReducer } from '@workbench/workbenchState.testing'; import { useInvocationState } from './useInvocationState'; const sdxlModel: MainModelConfig = { base: 'sdxl', key: 'sdxl-model', name: 'SDXL', type: 'main' }; -const modelsSnapshot = { models: [sdxlModel], status: 'loaded' }; +const depthModel = (key: string) => ({ base: 'sdxl', key, name: key, type: 'controlnet' }); +const modelsSnapshot = { + models: [sdxlModel, depthModel('sdxl-depth-a'), depthModel('sdxl-depth-b')], + status: 'loaded', +}; const activeProject = (state: ReturnType): Project => state.projects.find((candidate) => candidate.id === state.activeProjectId)!; @@ -135,6 +173,7 @@ afterEach(async () => { root = null; resetArchitectureCapabilities(); templates.reset(); + preferences.reset(); }); describe('useInvocationState and the architecture capability table', () => { @@ -196,3 +235,38 @@ describe('useInvocationState invoke', () => { expect(harness.submit.mock.calls[1]![0].prepend).toBeUndefined(); }); }); + +describe('useInvocationState and the remembered control-model picks', () => { + it('re-resolves a canvas route when a remembered pick settles a choice between control models', async () => { + const depthLayer: CanvasControlLayerContract = { + adapter: { beginEndStepPct: [0, 0.75], controlMode: 'balanced', kind: 'controlnet', model: null, weight: 0.75 }, + blendMode: 'normal', + controlType: 'depth', + detection: null, + id: 'depth-layer', + isEnabled: true, + isLocked: false, + name: 'Depth 1', + opacity: 1, + source: { image: { height: 64, imageName: 'depth.png', width: 64 }, type: 'image' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'control', + withTransparencyEffect: true, + }; + const project = buildGenerateProject(); + setArchitectureCapabilities(architectureCapabilitiesFixture); + await renderProbe({ + ...project, + canvas: { ...project.canvas, document: { ...project.canvas.document, stacks: stacksFrom([depthLayer]) } }, + invocation: { ...project.invocation, sourceId: 'canvas' }, + }); + + expect(blockingReasons()).toContain('widgets.layers.control.invalidLayer'); + + await act(() => { + preferences.set({ 'sdxl:depth': 'sdxl-depth-b' }); + }); + + expect(blockingReasons()).not.toContain('widgets.layers.control.invalidLayer'); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/shell/topbar/useInvocationState.ts b/invokeai/frontend/webv2/src/workbench/shell/topbar/useInvocationState.ts index 67bc74097ff..4d5e428ed6d 100644 --- a/invokeai/frontend/webv2/src/workbench/shell/topbar/useInvocationState.ts +++ b/invokeai/frontend/webv2/src/workbench/shell/topbar/useInvocationState.ts @@ -26,6 +26,7 @@ import { resolveInvocationRouteInput, } from '@workbench/invocation'; import { useIsInvocationPreparing } from '@workbench/invocationPreparation'; +import { getWorkbenchPreferences, subscribeWorkbenchPreferences } from '@workbench/settings/store'; import { useActiveProjectSelector, useWorkbenchCommands, @@ -38,31 +39,39 @@ import { useTranslation } from 'react-i18next'; const selectInvocationRouteInput = createInvocationRouteInputSelector(); /** - * Include capability and template snapshots in route selection so either load independently clears its readiness - * gate. + * Include capability, template and remembered control-model snapshots in route selection so each change + * independently re-evaluates its readiness gate. */ const subscribeRouteSources = (listener: () => void): (() => void) => { const unsubscribeCapabilities = subscribeArchitectureCapabilities(listener); const unsubscribeTemplates = subscribeInvocationTemplates(listener); + const unsubscribePreferences = subscribeWorkbenchPreferences(listener); return () => { unsubscribeCapabilities(); unsubscribeTemplates(); + unsubscribePreferences(); }; }; let routeSources: { capabilities: ReturnType; + controlModelMemory: ReturnType['controlModelMemory']; templates: ReturnType; } | null = null; -/** Referentially stable until one of the two stores publishes a new snapshot. */ +/** Referentially stable until one of the stores publishes a new snapshot. */ const getRouteSourcesSnapshot = () => { const capabilities = getArchitectureCapabilitiesSnapshot(); + const controlModelMemory = getWorkbenchPreferences().controlModelMemory; const templates = getInvocationTemplatesSnapshot(); - if (routeSources?.capabilities !== capabilities || routeSources.templates !== templates) { - routeSources = { capabilities, templates }; + if ( + routeSources?.capabilities !== capabilities || + routeSources.controlModelMemory !== controlModelMemory || + routeSources.templates !== templates + ) { + routeSources = { capabilities, controlModelMemory, templates }; } return routeSources; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts index a343e591c06..38c1fc3723d 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts @@ -151,6 +151,7 @@ interface HarnessOptions { outputOnlyMaskedRegions?: boolean; scaling?: RunCanvasInvocationDeps['scaling']; generateValues?: Record; + controlModelMemory?: RunCanvasInvocationDeps['controlModelMemory']; } const makeHarness = (options: HarnessOptions = {}): Harness => { @@ -258,6 +259,7 @@ const makeHarness = (options: HarnessOptions = {}): Harness => { const deps: RunCanvasInvocationDeps = { composeForGeneration: (composeOptions) => composeForGeneration(host, composeOptions), + controlModelMemory: options.controlModelMemory ?? {}, compositing: { ...DEFAULT_CANVAS_COMPOSITING, outputOnlyMaskedRegions: options.outputOnlyMaskedRegions ?? true, @@ -839,13 +841,136 @@ describe('runCanvasInvocation', () => { expect(harness.releaseRasterSnapshot).toHaveBeenCalledTimes(1); }); - it('blocks invocation and hands a nonempty model-less control layer to the formatter', async () => { + it('blocks a content-bearing control layer while the model list is still loading', async () => { const harness = makeHarness({ document: docWithLayers([controlLayer('control')]) }); await runCanvasInvocation(harness.deps); expect(harness.submittedGraphs()).toHaveLength(0); + expect(harness.notices().map((notice) => notice.message)).toEqual(['control: models_unavailable']); + }); + + it('blocks an untyped control layer whose model is no longer installed', async () => { + const harness = makeHarness({ + document: docWithLayers([controlLayer('control', { model: 'uninstalled' })]), + models: [], + }); + await runCanvasInvocation(harness.deps); + expect(harness.submittedGraphs()).toHaveLength(0); expect(harness.notices().map((notice) => notice.message)).toEqual(['control: missing_model']); }); + it('leaves out a nonempty control layer with neither a type nor a model, without blocking', async () => { + const harness = makeHarness({ document: docWithLayers([controlLayer('control')]), models: [] }); + await runCanvasInvocation(harness.deps); + expect(harness.notices()).toHaveLength(0); + expect(harness.submittedGraphs()).toHaveLength(1); + expect(harness.submittedGraphs()[0]?.graph.backendGraph?.nodes).not.toHaveProperty('control_net_control'); + }); + + describe('typed control layers', () => { + const controlModel = (key: string, base: string, type = 'controlnet', name = key) => ({ + base, + file_size: 1, + format: 'checkpoint', + hash: `${key}-hash`, + key, + name, + path: key, + source: key, + source_type: 'path' as const, + type, + }); + const typed = ( + id: string, + controlType: NonNullable, + detection: CanvasControlLayerContract['detection'] = null + ): CanvasControlLayerContract => ({ ...controlLayer(id), controlType, detection }); + const fluxMain = { ...sd1Model, base: 'flux' as const, key: 'flux', name: 'FLUX' }; + + it('runs the only model made for the type, as the kind that model runs', async () => { + const harness = makeHarness({ + document: docWithLayers([typed('depth-layer', 'depth')]), + models: [controlModel('sd1-depth', 'sd-1', 'controlnet', 'control_v11f1p_sd15_depth')], + }); + + await runCanvasInvocation(harness.deps); + + expect(harness.notices()).toHaveLength(0); + expect(harness.submittedGraphs()[0]?.graph.backendGraph?.nodes['control_net_depth-layer']).toMatchObject({ + control_model: { key: 'sd1-depth' }, + type: 'controlnet', + }); + }); + + it('runs the remembered pick when several models fit, and asks without one', async () => { + const models = [ + controlModel('depth-a', 'sd-1', 'controlnet', 'depth a'), + controlModel('depth-b', 'sd-1', 'controlnet', 'depth b'), + ]; + const asking = makeHarness({ document: docWithLayers([typed('depth-layer', 'depth')]), models }); + await runCanvasInvocation(asking.deps); + expect(asking.submittedGraphs()).toHaveLength(0); + expect(asking.notices()[0]?.message).toBe('depth-layer: choose_model'); + + const remembered = makeHarness({ + controlModelMemory: { 'sd-1:depth': 'depth-b' }, + document: docWithLayers([typed('depth-layer', 'depth')]), + models, + }); + await runCanvasInvocation(remembered.deps); + expect(remembered.notices()).toHaveLength(0); + expect( + remembered.submittedGraphs()[0]?.graph.backendGraph?.nodes['control_net_depth-layer']?.control_model + ).toMatchObject({ key: 'depth-b' }); + }); + + it('gives a FLUX union model the InstantX mode of the layer type', async () => { + const components = { + clipEmbedModel: { base: 'any', key: 'clip', name: 'CLIP Embed', type: 'clip_embed', variant: 'large' }, + t5EncoderModel: { base: 'any', key: 't5', name: 'T5 Encoder', type: 't5_encoder' }, + vae: { base: 'flux', key: 'flux-vae', name: 'FLUX VAE', type: 'vae' }, + }; + const harness = makeHarness({ + document: docWithLayers([typed('pose-layer', 'pose')]), + generateValues: components, + model: fluxMain, + models: [ + controlModel('flux-union', 'flux', 'controlnet', 'FLUX.1-dev-Controlnet-Union-Pro'), + ...(Object.values(components) as unknown as NonNullable), + ], + }); + + await runCanvasInvocation(harness.deps); + + expect(harness.notices()).toEqual([]); + expect(harness.submittedGraphs()[0]?.graph.backendGraph?.nodes['control_net_pose-layer']).toMatchObject({ + instantx_control_mode: 4, + type: 'flux_controlnet', + }); + }); + + it('blocks a layer whose type has no installed model', async () => { + const harness = makeHarness({ + document: docWithLayers([typed('pose-layer', 'pose')]), + models: [controlModel('sd1-depth', 'sd-1', 'controlnet', 'sd15 depth')], + }); + await runCanvasInvocation(harness.deps); + expect(harness.submittedGraphs()).toHaveLength(0); + expect(harness.notices()[0]?.message).toBe('pose-layer: no_model_for_type'); + }); + + it('blocks a layer whose map detection has not caught up', async () => { + const harness = makeHarness({ + document: docWithLayers([ + typed('depth-layer', 'depth', { settings: {}, type: 'depth_anything_depth_estimation' }), + ]), + models: [controlModel('sd1-depth', 'sd-1', 'controlnet', 'sd15 depth')], + }); + await runCanvasInvocation(harness.deps); + expect(harness.submittedGraphs()).toHaveLength(0); + expect(harness.notices()[0]?.message).toBe('depth-layer: detection_pending'); + }); + }); + it('blocks invocation when a nonempty control layer has malformed numeric settings', async () => { const model = { base: 'sd-1', diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts index ae525b4b8e3..48fd441ce1e 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts @@ -16,6 +16,8 @@ import type { RegionalGuidanceReferenceImage, } from '@workbench/canvas-engine/api'; import type { ComposeForGenerationOptions, ComposeForGenerationResult } from '@workbench/canvas-operations/api'; +import type { ControlLayerIssueCode } from '@workbench/controlLayerChecks'; +import type { ControlModelMemory } from '@workbench/controlModelResolution'; import type { ExpandedPositivePrompts, ResultDestination } from '@workbench/invocationContracts'; import type { WorkbenchNotificationKind } from '@workbench/projectContracts'; import type { WorkbenchCommands, WorkbenchNotificationCommands } from '@workbench/workbenchStore'; @@ -23,13 +25,11 @@ import type { WorkbenchCommands, WorkbenchNotificationCommands } from '@workbenc import { compileCanvasGraph, type CanvasCompileMode } from '@features/generation/canvasGraph'; import { type ControlAdapterKind, - createControlValidationSequence, detectCanvasMode, getSuggestedControlKind, getRegionalGuidanceRejectionReason, getRegionalGuidanceSupport, type ControlLayerGraphInput, - type ControlValidationReason, type RegionalGuidanceInput, type RegionalReferenceImageInput, resolveGenerateSeed, @@ -43,6 +43,8 @@ import { type AccountScope, } from '@platform/state/accountLifecycle'; import { getCanvasEngine, getCanvasOperations } from '@workbench/canvas-operations/api'; +import { createControlLayerCheckContext, createControlLayerSequence } from '@workbench/controlLayerChecks'; +import { getWorkbenchPreferences } from '@workbench/settings/store'; import { DEFAULT_CANVAS_COMPOSITING, type CanvasCompositingSettings, @@ -55,7 +57,7 @@ export const CANVAS_INVOKE_ERROR_TITLE = 'Canvas generation failed'; /** A control layer that blocks the invocation, as `formatControlLayerError` words it for the notice. */ export interface ControlLayerRejection { - code: ControlValidationReason; + code: ControlLayerIssueCode; layerName: string; suggestedKind: ControlAdapterKind | null; } @@ -63,7 +65,7 @@ export interface ControlLayerRejection { /** Structured control-layer rejection: callers word it from the locale; the message is for logs only. */ export class ControlLayerValidationError extends Error implements ControlLayerRejection { constructor( - readonly code: ControlValidationReason, + readonly code: ControlLayerIssueCode, readonly layerName: string, readonly suggestedKind: ControlAdapterKind | null ) { @@ -100,6 +102,8 @@ export interface RunCanvasInvocationDeps { generateValues: Record; /** Loaded models, for the same value/model sync the generate path performs. */ models?: readonly ModelConfig[]; + /** Remembered control-model picks, which resolve control layers left on automatic. */ + controlModelMemory: ControlModelMemory; /** The project settings the compiler consults. */ projectSettings: GenerationProjectSettings; /** Runtime-derived accelerator family recorded in generation metadata. */ @@ -124,41 +128,49 @@ const recordNotice = ( }; /** - * Validate enabled content-bearing controls in z-order and resolve adapters; invalid controls throw and block - * invocation. Join uploaded names with captured model metadata for graph inputs. + * Validate enabled content-bearing controls in z-order with the same judge Invoke readiness uses, resolving each + * layer's model; invalid controls throw and block invocation, and layers with no type or model are left out. Join + * uploaded names with captured model metadata for graph inputs. */ const createControlLayerCollector = ( model: GenerateModelConfig, - models: readonly ModelConfig[] | undefined + models: readonly ModelConfig[] | undefined, + remembered: ControlModelMemory ): { shouldComposite(layer: CanvasControlLayerContract): boolean; toGraphInputs(images: readonly { layerId: string; imageName: string }[]): ControlLayerGraphInput[]; } => { const metadata = new Map>(); - const validate = createControlValidationSequence(model); + const judge = models + ? createControlLayerSequence(model, models, createControlLayerCheckContext(model.base, remembered)) + : null; return { shouldComposite: (layer) => { const { adapter } = layer; - const resolved = adapter.model ? models?.find((candidate) => candidate.key === adapter.model) : undefined; - const rejection = validate({ - adapterModel: resolved ?? null, - beginEndStepPct: adapter.beginEndStepPct, - kind: adapter.kind, - weight: adapter.weight, - }); - if (rejection || !resolved) { + // Readiness skips control checks until models load; resolving against nothing would drop or misjudge layers. + if (!judge) { + throw new ControlLayerValidationError('models_unavailable', layer.name, null); + } + const { reason, resolution, skip } = judge(layer); + if (skip) { + return false; + } + const resolved = resolution.model; + if (reason || !resolved) { + const code = reason ?? 'missing_model'; throw new ControlLayerValidationError( - rejection ?? 'missing_model', + code, layer.name, - rejection === 'switch_adapter_kind' ? getSuggestedControlKind(model.base) : null + code === 'switch_adapter_kind' ? getSuggestedControlKind(model.base) : null ); } metadata.set(layer.id, { beginEndStepPct: adapter.beginEndStepPct, controlMode: adapter.controlMode, id: layer.id, - kind: adapter.kind, + instantxControlMode: resolution.instantxControlMode, + kind: resolution.kind, model: { base: resolved.base, key: resolved.key, @@ -315,7 +327,7 @@ export const runCanvasInvocation = async (deps: RunCanvasInvocationDeps): Promis // Flush persisted paint before capture so planning sees current bitmap references and bbox. await deps.flushPendingUploads(); - const controls = createControlLayerCollector(model, deps.models); + const controls = createControlLayerCollector(model, deps.models, deps.controlModelMemory); const regions = createRegionalGuidanceCollector(model); const composed = await deps.composeForGeneration({ // Mode-union drift guard: `detectCanvasMode`'s facts/mode types must stay @@ -459,6 +471,7 @@ export const prepareCanvasInvocation = async (args: PrepareCanvasInvocationArgs) commands: args.commands, expansion: args.expansion, composeForGeneration: (composeOptions) => operations.composeForGeneration(composeOptions), + controlModelMemory: getWorkbenchPreferences().controlModelMemory, flushPendingUploads: () => engine.lifecycle.flushPendingUploads(), formatControlLayerError: args.formatControlLayerError, generateValues: args.generateValues, diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.tsx index ce8e15db796..4c215e1ba41 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.tsx @@ -368,9 +368,10 @@ export const ControlLayerSettings = ({ engine, layer, onOperationStarted }: Cont !capabilitiesUnavailable && (hasContent || validationReason === 'missing_model' || + validationReason === 'missing_type' || validationReason === 'switch_adapter_kind' || validationReason === 'flux2_klein_control') && - !(validationReason === 'missing_model' && hiddenModelReasons.length > 0) + !((validationReason === 'missing_model' || validationReason === 'missing_type') && hiddenModelReasons.length > 0) ? validationReason : null; From 3472b7519c03c495ef11035613e7f8081ce47eb2 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Fri, 23 Oct 2026 01:20:06 -0600 Subject: [PATCH 03/35] feat(canvas): commit control-layer patches and detected maps in the engine ControlDerivationController publishes owner-less guarded patch and map commits that pair in history, so one undo reverts an intent and its map together. Materialize, transform, crop, generated and filter results rebase or detach the kept original; uploads can be flushed per layer. --- .../workbench/canvas-engine/ARCHITECTURE.md | 21 +- .../canvas-engine/applicationHost.ts | 3 + .../workbench/canvas-engine/capabilities.ts | 71 +++ .../controlDerivationController.test.ts | 406 +++++++++++++++++ .../controlDerivationController.ts | 414 ++++++++++++++++++ .../controllers/cropLayerController.test.ts | 40 +- .../controllers/cropLayerController.ts | 4 +- .../filterResultController.test.ts | 136 ++++++ .../controllers/filterResultController.ts | 27 +- .../generatedResultController.test.ts | 92 ++++ .../controllers/generatedResultController.ts | 19 +- .../layerOperationHarness.testing.ts | 29 +- .../controllers/persistenceController.test.ts | 105 ++++- .../controllers/persistenceController.ts | 29 ++ .../transformEditingController.test.ts | 78 +++- .../controllers/transformEditingController.ts | 61 ++- .../canvas-engine/document/bitmapStore.ts | 63 ++- .../editing/controlPixelEdit.test.ts | 41 ++ .../canvas-engine/editing/controlPixelEdit.ts | 8 + .../workbench/canvas-engine/engine.test.ts | 89 ++-- .../src/workbench/canvas-engine/engine.ts | 24 +- 21 files changed, 1654 insertions(+), 106 deletions(-) create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlDerivationController.test.ts create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlDerivationController.ts create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/filterResultController.test.ts create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/generatedResultController.test.ts diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/ARCHITECTURE.md b/invokeai/frontend/webv2/src/workbench/canvas-engine/ARCHITECTURE.md index 95ef03401b0..88f787ee178 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/ARCHITECTURE.md +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/ARCHITECTURE.md @@ -67,7 +67,7 @@ Current extracted ownership includes: - `RasterMemoryBudgetController`: base, derived, group, decoded, detached, reserved and overage byte accounting plus generation-scoped reservations and operation-owned reservations and pins. - `RenderController`: render targets, scheduler construction, previews, render lifecycle. - `HistoryController`: engine-owned history, guarded async replay with failure reporting, inactive byte trimming. -- `PersistenceController`: dirty-bitmap flush barriers and disposal. +- `PersistenceController`: dirty-bitmap flush barriers (whole document, or one layer through `applicationHost.flushLayerUploads`) and disposal. - `EditingController`: selection state and exclusive edit-lease lifecycle. - `InteractionController`: public tool commands and their disposal boundary. - `LayerController`: guarded layer mutations and thumbnail capability ports. @@ -76,6 +76,7 @@ Current extracted ownership includes: - `StagedResultController`: guarded staged-candidate acceptance and its history entry. - `PixelEditController`, `SelectionImageController`, `LayerMutationController`: atomic pixel/document publication at their respective boundaries. Pixel editor materializes control layers and raster-image layers in place before destructive edits, keeping conversion plus pixels in one undoable transaction. - `MaskResultController`, `FilterResultController`, `GeneratedResultController`: guarded application-result adoption through narrow host ports. +- `ControlDerivationController`: control-layer intent patches and detected-map commits (see "Control derivation"). Controller-local tests instantiate these boundaries with fakes. Integration tests in `engine.test.ts` cover composition, document routing, cross-controller invariants. @@ -182,6 +183,24 @@ Continue-staging variant uses same guarded transaction but inserts candidate as Undo removes accepted layer and restores prior selection. Redo restores identical layer and selection without recreating event or staging candidate. History entry replays failure-atomically; successful commit publishes through an admitted transaction, which clears any previous redo stack. Undo intentionally does not return accepted candidate to staging. +## Control derivation + +A typed control layer can keep the pixels it was detected from (`original`, framed by `originalTransform`) beside the derived map in `source`. The invariants are documented on `CanvasControlLayerContract`; `document/controlDerivation.ts` owns them as pure rules, and every engine path that changes a control layer's pixels or frame applies one of them so no edit leaves a map paired with an original it no longer derives from: + +| Operation | Rule | +| ----------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | +| Materialize before a pixel edit (`buildMaterializedPixelLayer`) | `rebaseControlOriginal` with the baked transform; a frame the bake would shear detaches | +| Transform Apply baking a paint map (`TransformEditingController`) | the same rebase in the bake's own step, so undo restores the whole before-contract | +| Crop to bbox | detach (the cropped pixels become the image) | +| Generated or Select Object result replacing a control layer | detach and drop `filter`: the result is the image, not a filter's map | +| Filter result replacing a control layer | detach, then `filter` names the filter | +| Filter result copied to a new control layer | no derivation carries over; `controlCopy.controlType` types the copy with `detection: null` | +| Paint on a map that needs no materialize | only `source` changes; the map stays current | + +`ControlDerivationController` (`engine.controlDerivation`) is the only writer of derivation fields besides those rules. The application's detection coordinator calls `commitControlLayerPatch` for an intent (type, detector, settings, image) and `commitControlMap` for a detection result, guarded by `captureControlDetectionGuard`: same `documentRevision` and same `controlDetectionInputKey`, which ignores weight, name and paint on the map, so those never discard a running detection. Neither commit passes the operation edit owner. A detection runs in the background while the user keeps working, so its commit must yield to whatever holds the document: an operation's lease (Filter, Select Object, Transform), a gesture, a replay or a session holding the layer's pixels all refuse it `busy`, and the coordinator retries once the lock releases. The guard capture and the map commit's freshness checks (before decoding, after it, and once admitted) read the reducer document, so a background job never ends the user's structural preview; only an admitted commit does, and the map is built from the layer as it stands after that. A map whose input is the layer's own unpersisted paint is also `busy` until `flushLayerUploads` lands those strokes in the `original` it records. + +History pairing keeps one undo per user intent. A patch with a `coalesceKey` replaces the newest entry while that is the controller's patch of the same layer and key, so a settings scrub is one step. A map replaces the newest entry while that is the controller's entry for the same layer that requested this input key (a patch that changed the detection input, or an earlier map), so one undo reverts "set type" and its map together; otherwise the map records its own "Detect " entry, and a weight or name patch in between is never undone together with a later map. Patches route as user edits; maps publish as `system` so a background result never changes the invocation route. Undo restores the exact before-state: a contract whose source is persisted re-rasterizes from it, and unpersisted paint the step replaced is captured and replayed with the entry. A source change discards the outgoing source's pending persistence. Undo and redo never schedule detection; a restored stale map shows as needing detection until the user asks again. + ## PSD export PSD export is a planner/executor transaction over immutable snapshots. `PsdExportController` captures one `CanvasDocumentSnapshot`, walks raster tree into planner inputs (groups as pass-through folders, exportable leaves with their captured content rects), then obtains a `CanvasRasterSnapshot` including disabled layers. `planPsdExport` reverses every level of top-first tree into ag-psd's bottom-to-top order, writes each node's own visibility flag (disabled group is a hidden folder), flattens merged preview from the leaves that contribute (enabled with every ancestor enabled), which is what Photoshop renders for the same flags. Folders whose subtree exports nothing are dropped. Execution reads only detached surfaces; cannot mix live layer caches from different document generations. diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/applicationHost.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/applicationHost.ts index 593ede1aa32..45a7130ce55 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/applicationHost.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/applicationHost.ts @@ -15,6 +15,7 @@ import type { CommitMaskImageResult, CommitMaskImageResultOptions, } from '@workbench/canvas-engine/controllers/maskResultController'; +import type { LayerFlushOptions, LayerFlushResult } from '@workbench/canvas-engine/controllers/persistenceController'; import type { CanvasCompositeExecutorDeps } from '@workbench/canvas-engine/rasterTransactions'; import type { RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { SamInteractionState, SamVisualInput } from '@workbench/canvas-engine/samInteraction'; @@ -42,6 +43,8 @@ export interface CanvasApplicationHost { decodeSelectObjectPreview(result: { image: CanvasImageRef; rect: Rect }, signal: AbortSignal): Promise; encodeSurface(surface: RasterSurface): Promise; exportBakedLayerBlob(layerId: string): Promise; + /** Persists one layer's unsaved paint, so work reading its persisted source sees every stroke. */ + flushLayerUploads(layerId: string, options: LayerFlushOptions): Promise; exportLayerPixels( layerId: string, options?: ExportLayerPixelsOptions diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/capabilities.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/capabilities.ts index 258771e6078..3df2085fb30 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/capabilities.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/capabilities.ts @@ -1,4 +1,6 @@ import type { + CanvasControlDetectionContract, + CanvasControlLayerContract, CanvasDocumentContractV3, CanvasImageRef, CanvasLayerContract, @@ -6,6 +8,7 @@ import type { CanvasStagingCandidateContract, CanvasStateContractV3, CanvasTextFontRef, + ControlType, } from '@workbench/canvas-engine/contracts'; import type { CanvasMutationOrigin } from '@workbench/canvas-engine/mutationContracts'; import type { StrokeCommittedEvent } from '@workbench/canvas-engine/tools/tool'; @@ -92,9 +95,77 @@ export interface CommitRasterFilterOptions { mode: 'replace' | 'copy'; filter?: RasterFilterSettings; target?: RasterFilterCommitTarget; + /** + * For `target: 'control'` copies: the control type the filter implies, or null for an untyped copy. A typed copy + * uses its pixels as-is (`detection: null`); either way the copy keeps no derivation of its source layer. + */ + controlCopy?: { controlType: ControlType | null }; requireExactImageDimensions?: boolean; signal?: AbortSignal; } + +/** What a control-map commit must still match: the document session and the detection input it ran on. */ +export interface ControlDetectionGuard { + readonly projectId: string; + readonly layerId: string; + readonly documentRevision: number; + /** `controlDetectionInputKey` of the layer when the run started. */ + readonly inputKey: string; +} + +export type ControlLayerPatchResult = + | { status: 'committed' } + | { + status: 'stale' | 'missing' | 'locked' | 'busy' | 'not-ready' | 'over-budget' | 'failed'; + message?: string; + }; + +export type ControlMapCommitResult = + | { status: 'committed' } + | { + status: + | 'stale' + | 'missing' + | 'locked' + | 'busy' + | 'not-ready' + | 'over-budget' + | 'aborted' + | 'unrepresentable' + | 'failed'; + message?: string; + }; + +/** + * Edits that keep a control layer's derived map in step with its type, detector and original. Neither commit takes + * the operation edit lease: both refuse `busy` while a modal operation, gesture, replay or layer session is running, + * and the caller retries later. + */ +export interface CanvasControlDerivationCapability { + /** + * Replaces one control layer's contract as one user-origin history entry; refused `stale` unless the live layer + * still equals `expected`. Consecutive patches of the same layer with the same `coalesceKey` replace the top entry. + */ + commitControlLayerPatch(options: { + label: string; + expected: CanvasControlLayerContract; + next: CanvasControlLayerContract; + coalesceKey?: string; + }): ControlLayerPatchResult; + /** What a detection run must still match to commit; null when the layer is missing or not a control layer. */ + captureControlDetectionGuard(layerId: string): ControlDetectionGuard | null; + /** + * Installs a detected map (a persisted image) as the layer's `source`, keeping the original, as a system-origin + * entry. Paired with the intent entry that requested it when that is still the newest step. + */ + commitControlMap(options: { + guard: ControlDetectionGuard; + image: CanvasImageRef; + rect: Rect; + detection: CanvasControlDetectionContract; + signal?: AbortSignal; + }): Promise; +} export type MaskImageResultTarget = 'inpaint_mask' | 'regional_guidance'; export interface CommitMaskImageResultOptions { guard: LayerExportGuard; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlDerivationController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlDerivationController.test.ts new file mode 100644 index 00000000000..c7445943d62 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlDerivationController.test.ts @@ -0,0 +1,406 @@ +import type { + CanvasControlLayerContract, + CanvasDocumentContractV3, + CanvasLayerContract, +} from '@workbench/canvas-engine/contracts'; +import type { Rect } from '@workbench/canvas-engine/types'; + +import { groupContract, stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { getDocumentLayer } from '@workbench/canvas-engine/document/documentIndex'; +import { describe, expect, it, vi } from 'vitest'; + +import { ControlDerivationController } from './controlDerivationController'; +import { createLayerOperationHarness } from './layerOperationHarness.testing'; + +const DEPTH = { settings: { model_size: 'small' }, type: 'depth_anything_depth_estimation' }; +const PHOTO = { height: 64, imageName: 'photo', width: 64 }; +const MAP = { height: 32, imageName: 'depth-map', width: 32 }; +const MAP_RECT: Rect = { height: 32, width: 32, x: 4, y: 8 }; +const CACHE_RECT: Rect = { height: 20, width: 20, x: 0, y: 0 }; + +const controlLayer = (overrides: Partial = {}): CanvasControlLayerContract => ({ + adapter: { beginEndStepPct: [0, 1], controlMode: 'balanced', kind: 'controlnet', model: null, weight: 1 }, + blendMode: 'normal', + id: 'control', + isEnabled: true, + isLocked: false, + name: 'Control Layer 1', + opacity: 1, + source: { image: PHOTO, type: 'image' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'control', + withTransparencyEffect: true, + ...overrides, +}); + +const rasterLayer: CanvasLayerContract = { + blendMode: 'normal', + id: 'raster', + isEnabled: true, + isLocked: false, + name: 'Raster', + opacity: 1, + source: { bitmap: null, type: 'paint' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'raster', +}; + +const createHarness = ( + layer: CanvasControlLayerContract = controlLayer(), + options: { historyBytes?: number; pendingPixels?: boolean; lockedParent?: boolean } = {} +) => { + const document: CanvasDocumentContractV3 = { + background: 'transparent', + bbox: { height: 64, width: 64, x: 0, y: 0 }, + height: 64, + selectedLayerId: layer.id, + stacks: stacksFrom([ + rasterLayer, + options.lockedParent ? groupContract('group', [layer], { isLocked: true }) : layer, + ]), + version: 3, + width: 64, + }; + const endStructuralPreview = vi.fn(); + const h = createLayerOperationHarness(document, { endStructuralPreview, historyBytes: options.historyBytes }); + h.layers.getOrCreateRect(layer.id, CACHE_RECT); + const fake = { documentRevision: 1, pendingPixels: options.pendingPixels ?? false, sessionHeld: false }; + const discardPersisted = vi.fn(); + const decodeImage = vi.fn((image: { width: number; height: number }) => + Promise.resolve({ + decodedHeight: image.height, + decodedWidth: image.width, + status: 'ok' as const, + surface: h.backend.createSurface(image.width, image.height), + }) + ); + const controller = new ControlDerivationController({ + captureCache: (target) => { + const entry = h.layers.get(target.id); + if (!entry) { + return null; + } + const pixels = h.backend.createSurface(entry.rect.width, entry.rect.height); + pixels.ctx.drawImage(entry.surface.canvas, 0, 0); + return { pixels, rect: { ...entry.rect } }; + }, + ctx: h.ctx, + decodeImage, + discardPersisted, + getDocumentRevision: () => fake.documentRevision, + hasPendingPixelWork: () => fake.pendingPixels, + isLayerSessionHeld: () => fake.sessionHeld, + }); + const live = (): CanvasControlLayerContract => getDocumentLayer(h.document(), layer.id) as CanvasControlLayerContract; + /** Commits `patch` against the live layer, as the coordinator's intents do. */ + const patch = (label: string, change: Partial, coalesceKey?: string) => + controller.commitControlLayerPatch({ coalesceKey, expected: live(), label, next: { ...live(), ...change } }); + const commitMap = (guard = controller.captureControlDetectionGuard(layer.id)!) => + controller.commitControlMap({ detection: DEPTH, guard, image: MAP, rect: MAP_RECT }); + return { ...h, commitMap, controller, decodeImage, discardPersisted, endStructuralPreview, fake, live, patch }; +}; + +describe('ControlDerivationController.commitControlLayerPatch', () => { + it('commits one user entry whose undo and redo restore the exact contracts', async () => { + const h = createHarness(); + const before = h.live(); + + expect(h.patch('Set control type', { controlType: 'depth', detection: DEPTH })).toEqual({ status: 'committed' }); + const after = h.live(); + expect(after).toMatchObject({ controlType: 'depth', detection: DEPTH }); + expect(h.history.entries().past).toEqual(['Set control type']); + expect(h.committedEdits).toHaveLength(1); + + await h.history.undo(); + expect(h.live()).toEqual(before); + await h.history.redo(); + expect(h.live()).toEqual(after); + }); + + it('coalesces consecutive patches with the same key into the top entry', async () => { + const h = createHarness(controlLayer({ controlType: 'depth', detection: DEPTH })); + const before = h.live(); + + h.patch('Detection settings', { detection: { ...DEPTH, settings: { model_size: 'base' } } }, 'settings'); + h.patch('Detection settings', { detection: { ...DEPTH, settings: { model_size: 'large' } } }, 'settings'); + expect(h.history.entries().past).toEqual(['Detection settings']); + + h.patch('Detection settings', { detection: DEPTH }, 'other'); + expect(h.history.entries().past).toHaveLength(2); + + await h.history.undo(); + await h.history.undo(); + expect(h.live()).toEqual(before); + }); + + it('refuses a patch prepared against a layer that has changed since', () => { + const h = createHarness(); + const expected = h.live(); + h.patch('Weight', { adapter: { ...expected.adapter, weight: 0.5 } }); + + expect( + h.controller.commitControlLayerPatch({ expected, label: 'Set type', next: { ...expected, controlType: 'pose' } }) + ).toEqual({ status: 'stale' }); + expect(h.history.entries().past).toEqual(['Weight']); + }); + + it('keeps the outgoing unpersisted pixels when it replaces the source', async () => { + const painted = controlLayer({ source: { bitmap: null, type: 'paint' } }); + const h = createHarness(painted, { pendingPixels: true }); + + h.patch('Replace image', { source: { image: PHOTO, type: 'image' } }); + expect(h.discardPersisted).toHaveBeenCalledWith('control'); + const installedBeforeUndo = h.installed.length; + + await h.history.undo(); + expect(h.live()).toEqual(painted); + expect(h.installed).toHaveLength(installedBeforeUndo + 1); + expect(h.layers.get('control')?.rect).toEqual(CACHE_RECT); + }); + + it('leaves pending persistence alone when the source does not change', () => { + const h = createHarness(controlLayer(), { pendingPixels: true }); + + h.patch('Set control type', { controlType: 'depth', detection: DEPTH }); + expect(h.discardPersisted).not.toHaveBeenCalled(); + }); + + it('is refused while an operation holds the edit lease or a session holds the layer', () => { + const h = createHarness(); + h.setEditingLocked(true); + expect(h.patch('Set control type', { controlType: 'depth' })).toEqual({ status: 'busy' }); + h.setEditingLocked(false); + h.fake.sessionHeld = true; + expect(h.patch('Set control type', { controlType: 'depth' })).toEqual({ status: 'busy' }); + expect(h.history.entries().past).toEqual([]); + expect(h.dispatched).toEqual([]); + }); + + it('is refused when its entry could never be kept', () => { + const h = createHarness(controlLayer(), { historyBytes: 16 }); + + expect(h.patch('Set control type', { controlType: 'depth' })).toEqual({ status: 'over-budget' }); + expect(h.dispatched).toEqual([]); + }); +}); + +describe('ControlDerivationController.commitControlMap', () => { + it('lands in the intent entry, so one undo restores the pre-intent layer', async () => { + const h = createHarness(); + const untyped = h.live(); + h.patch('Set control type', { controlType: 'depth', detection: DEPTH }); + + await expect(h.commitMap()).resolves.toEqual({ status: 'committed' }); + const mapped = h.live(); + expect(mapped).toMatchObject({ + detection: DEPTH, + filter: DEPTH, + original: { image: PHOTO, type: 'image' }, + source: { bitmap: MAP, offset: { x: 4, y: 8 }, type: 'paint' }, + }); + expect(h.history.entries().past).toEqual(['Set control type']); + expect(h.layers.get('control')?.rect).toEqual(MAP_RECT); + expect(h.discardPersisted).toHaveBeenCalledWith('control'); + // The patch routed as a user edit; the background map did not. + expect(h.committedEdits).toHaveLength(1); + + await h.history.undo(); + expect(h.live()).toEqual(untyped); + await h.history.redo(); + expect(h.live()).toEqual(mapped); + expect(h.layers.get('control')?.rect).toEqual(MAP_RECT); + }); + + it('restores the unpersisted map paint the intent and its map replaced', async () => { + const painted = controlLayer({ + controlType: 'depth', + detection: DEPTH, + filter: DEPTH, + original: { image: PHOTO, type: 'image' }, + source: { bitmap: { height: 20, imageName: 'old-map', width: 20 }, type: 'paint' }, + }); + const h = createHarness(painted, { pendingPixels: true }); + h.patch('Detection settings', { detection: { ...DEPTH, settings: { model_size: 'base' } } }, 'settings'); + + await expect(h.commitMap()).resolves.toEqual({ status: 'committed' }); + expect(h.layers.get('control')?.rect).toEqual(MAP_RECT); + + await h.history.undo(); + expect(h.live()).toEqual(painted); + expect(h.layers.get('control')?.rect).toEqual(CACHE_RECT); + }); + + it('records its own entry when the intent is no longer the newest step', async () => { + const h = createHarness(); + h.patch('Set control type', { controlType: 'depth', detection: DEPTH }); + const intent = h.live(); + const guard = h.controller.captureControlDetectionGuard('control')!; + h.patch('Weight', { adapter: { ...intent.adapter, weight: 0.4 } }); + + await expect(h.commitMap(guard)).resolves.toEqual({ status: 'committed' }); + expect(h.history.entries().past).toEqual(['Set control type', 'Weight', 'Detect depth']); + + await h.history.undo(); + expect(h.live()).toEqual({ ...intent, adapter: { ...intent.adapter, weight: 0.4 } }); + }); + + it('is not staled by weight, name or painting on the map', async () => { + const h = createHarness( + controlLayer({ + controlType: 'depth', + detection: DEPTH, + original: { image: PHOTO, type: 'image' }, + source: { bitmap: { height: 20, imageName: 'old-map', width: 20 }, type: 'paint' }, + }) + ); + const guard = h.controller.captureControlDetectionGuard('control')!; + h.patch('Rename', { adapter: { ...h.live().adapter, weight: 0.2 }, name: 'Depth 1' }); + h.ctx.dispatch({ + id: 'control', + source: { bitmap: { height: 20, imageName: 'painted-map', width: 20 }, type: 'paint' }, + type: 'updateCanvasLayerSource', + }); + + await expect(h.commitMap(guard)).resolves.toEqual({ status: 'committed' }); + expect(h.live()).toMatchObject({ adapter: { weight: 0.2 }, name: 'Depth 1', source: { bitmap: MAP } }); + }); + + it.each([ + ['the document was replaced', (h: ReturnType) => (h.fake.documentRevision += 1)], + [ + 'the detection input changed', + (h: ReturnType) => + h.patch('Detection settings', { detection: { ...DEPTH, settings: { model_size: 'large' } } }), + ], + ])('records nothing when %s', async (_case, change) => { + const h = createHarness(controlLayer({ controlType: 'depth', detection: DEPTH })); + const guard = h.controller.captureControlDetectionGuard('control')!; + change(h); + const document = h.document(); + const entries = h.history.entries(); + + await expect(h.commitMap(guard)).resolves.toEqual({ status: 'stale' }); + expect(h.document()).toBe(document); + expect(h.history.entries()).toEqual(entries); + }); + + it('is refused while an operation holds the edit lease or a session holds the layer', async () => { + const h = createHarness(controlLayer({ controlType: 'depth', detection: DEPTH })); + const guard = h.controller.captureControlDetectionGuard('control')!; + + h.setEditingLocked(true); + await expect(h.commitMap(guard)).resolves.toEqual({ status: 'busy' }); + h.setEditingLocked(false); + h.fake.sessionHeld = true; + await expect(h.commitMap(guard)).resolves.toEqual({ status: 'busy' }); + expect(h.dispatched).toEqual([]); + + h.fake.sessionHeld = false; + await expect(h.commitMap(guard)).resolves.toEqual({ status: 'committed' }); + }); + + it('waits for unpersisted paint on its own input to reach the original it records', async () => { + const drawn = controlLayer({ + controlType: 'depth', + detection: DEPTH, + source: { bitmap: { height: 20, imageName: 'drawn', width: 20 }, type: 'paint' }, + }); + const h = createHarness(drawn, { pendingPixels: true }); + + await expect(h.commitMap()).resolves.toEqual({ status: 'busy' }); + expect(h.dispatched).toEqual([]); + }); + + it('refuses maps it cannot represent and pixels it cannot prepare', async () => { + const h = createHarness(); + await expect(h.commitMap()).resolves.toEqual({ status: 'unrepresentable' }); + + const typed = createHarness(controlLayer({ controlType: 'depth', detection: DEPTH })); + typed.state.rasterBytes = 0; + await expect(typed.commitMap()).resolves.toEqual({ status: 'over-budget' }); + expect(typed.dispatched).toEqual([]); + expect(typed.history.entries().past).toEqual([]); + }); + + it('does not decode for a caller that already aborted', async () => { + const h = createHarness(controlLayer({ controlType: 'depth', detection: DEPTH })); + const guard = h.controller.captureControlDetectionGuard('control')!; + + await expect( + h.controller.commitControlMap({ + detection: DEPTH, + guard, + image: MAP, + rect: MAP_RECT, + signal: AbortSignal.abort(), + }) + ).resolves.toEqual({ status: 'aborted' }); + expect(h.decodeImage).not.toHaveBeenCalled(); + }); +}); + +describe('ControlDerivationController refusals', () => { + const typed = () => controlLayer({ controlType: 'depth', detection: DEPTH }); + + it('refuses both commits under a locked ancestor, before decoding', async () => { + const h = createHarness(typed(), { lockedParent: true }); + const guard = h.controller.captureControlDetectionGuard('control')!; + + expect(h.patch('Set control type', { controlType: 'pose' })).toEqual({ status: 'locked' }); + await expect(h.commitMap(guard)).resolves.toEqual({ status: 'locked' }); + expect(h.decodeImage).not.toHaveBeenCalled(); + expect(h.dispatched).toEqual([]); + }); + + it('reports a removed layer as missing', async () => { + const h = createHarness(typed()); + const expected = h.live(); + const guard = h.controller.captureControlDetectionGuard('control')!; + h.ctx.dispatch({ + enabledUpdates: [], + removeIds: ['control'], + selectedLayerId: null, + type: 'applyCanvasLayerStackMutation', + }); + + expect( + h.controller.commitControlLayerPatch({ expected, label: 'Set type', next: { ...expected, controlType: 'pose' } }) + ).toEqual({ status: 'missing' }); + await expect(h.commitMap(guard)).resolves.toEqual({ status: 'missing' }); + expect(h.history.entries().past).toEqual([]); + }); + + it('retries a decode the edit lock interrupted as busy', async () => { + const h = createHarness(typed()); + h.decodeImage.mockResolvedValueOnce({ status: 'stale' } as never); + + await expect(h.commitMap()).resolves.toEqual({ status: 'busy' }); + expect(h.dispatched).toEqual([]); + }); + + it('leaves an open structural preview alone until a commit is admitted', async () => { + const h = createHarness(typed()); + const guard = h.controller.captureControlDetectionGuard('control')!; + h.fake.sessionHeld = true; + await expect(h.commitMap(guard)).resolves.toEqual({ status: 'busy' }); + expect(h.endStructuralPreview).not.toHaveBeenCalled(); + + h.fake.sessionHeld = false; + await expect(h.commitMap(guard)).resolves.toEqual({ status: 'committed' }); + expect(h.endStructuralPreview).toHaveBeenCalled(); + }); +}); + +describe('ControlDerivationController.captureControlDetectionGuard', () => { + it('captures only control layers', () => { + const h = createHarness(); + + expect(h.controller.captureControlDetectionGuard('control')).toMatchObject({ + documentRevision: 1, + layerId: 'control', + projectId: h.ctx.projectId, + }); + expect(h.controller.captureControlDetectionGuard('raster')).toBeNull(); + expect(h.controller.captureControlDetectionGuard('absent')).toBeNull(); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlDerivationController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlDerivationController.ts new file mode 100644 index 00000000000..3373e0c07b7 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/controlDerivationController.ts @@ -0,0 +1,414 @@ +import type { + CanvasControlDerivationCapability, + ControlDetectionGuard, + ControlLayerPatchResult, + ControlMapCommitResult, +} from '@workbench/canvas-engine/capabilities'; +import type { + CanvasControlLayerContract, + CanvasDocumentContractV3, + CanvasImageRef, +} from '@workbench/canvas-engine/contracts'; +import type { LayerMutationControllerOptions } from '@workbench/canvas-engine/controllers/layerMutationController'; +import type { DecodeImageResult } from '@workbench/canvas-engine/controllers/rasterController'; +import type { HistoryEntryToken } from '@workbench/canvas-engine/history/history'; + +import { areJsonValuesStructurallyEqual } from '@platform/core/json'; +import { lookupDocumentLeaf } from '@workbench/canvas-engine/document-model/documentModel'; +import { + controlDetectionInputKey, + getControlDetectionInput, + withDetectedMap, +} from '@workbench/canvas-engine/document/controlDerivation'; +import { getDocumentLayer } from '@workbench/canvas-engine/document/documentIndex'; +import { collectHistoryMediaRefs, HISTORY_ENTRY_OVERHEAD_BYTES } from '@workbench/canvas-engine/history/history'; + +import type { CanvasMutationContext, EditStep, EditTransaction } from './mutationContext'; + +import { + guardedResultRefusal, + layerEditRefusal, + type LayerPixels, + replaceLayerStep, + replayWithPixels, + rgbaBytes, +} from './editSteps'; + +export interface ControlDerivationControllerOptions { + readonly ctx: Pick< + CanvasMutationContext, + | 'applyStep' + | 'begin' + | 'capturePermit' + | 'getDocument' + | 'getReducerDocument' + | 'historyTop' + | 'installPrepared' + | 'isPermitCurrent' + | 'preparePixels' + | 'projectId' + | 'reserveRaster' + >; + readonly captureCache: LayerMutationControllerOptions['captureCache']; + readonly decodeImage: ( + image: CanvasImageRef, + options: { signal?: AbortSignal; isCurrent?: () => boolean; scaleToImage?: boolean } + ) => Promise; + readonly discardPersisted: (layerId: string) => void; + /** The reducer state's wholesale-replacement counter; null without canvas state. */ + readonly getDocumentRevision: () => number | null; + /** Whether the layer's live pixels are not yet represented by its persisted source. */ + readonly hasPendingPixelWork: (layerId: string) => boolean; + /** Whether a session (transform, text, float, pixel edit, operation) owns the layer's live pixels. */ + readonly isLayerSessionHeld: (layerId: string) => boolean; +} + +/** The newest entry this controller recorded, so a later patch or map can amend it while it is still on top. */ +interface RecordedEntry { + readonly token: HistoryEntryToken; + readonly layerId: string; + readonly label: string; + readonly coalesceKey: string | undefined; + /** `controlDetectionInputKey` of the entry's after-state. */ + readonly inputKey: string; + /** + * Whether a map for `inputKey` belongs to this entry: it changed what detection runs on (an intent that requested + * the map), or it is a map itself. A weight or name patch is not, so undoing it never also drops a later map. + */ + readonly requestsMap: boolean; + /** The layer the entry's undo restores, with its unpersisted pixels when its source cannot reproduce them. */ + readonly before: CanvasControlLayerContract; + readonly beforePixels: LayerPixels | null; +} + +type BeforeState = + | { readonly before: CanvasControlLayerContract; readonly pixels: LayerPixels | null } + | { readonly refusal: 'not-ready' | 'over-budget' }; + +const asControlLayer = ( + document: CanvasDocumentContractV3 | null, + layerId: string +): CanvasControlLayerContract | null => { + const layer = getDocumentLayer(document, layerId); + return layer?.type === 'control' ? layer : null; +}; + +const sameSource = (a: CanvasControlLayerContract, b: CanvasControlLayerContract): boolean => + areJsonValuesStructurallyEqual(a.source, b.source); + +/** + * Commits control-layer derivation edits: intent patches (type, detector, settings, image) and background detection + * maps. Neither takes the operation edit owner, so both yield to a modal operation, gesture, replay or layer session + * and the caller retries. A map lands in the same undo step as the intent that requested it while that intent is + * still the newest step. + */ +export class ControlDerivationController implements CanvasControlDerivationCapability { + private last: RecordedEntry | null = null; + + constructor(private readonly options: ControlDerivationControllerOptions) {} + + commitControlLayerPatch({ + coalesceKey, + expected, + label, + next, + }: Parameters[0]): ControlLayerPatchResult { + const { ctx, ...o } = this.options; + if (next.id !== expected.id || next.type !== 'control') { + return { message: 'A control layer patch must keep the layer id and type.', status: 'failed' }; + } + const layerId = expected.id; + const document = ctx.getDocument(); + const live = asControlLayer(document, layerId); + if (!document || !live) { + return { status: 'missing' }; + } + if (live !== expected && !areJsonValuesStructurallyEqual(live, expected)) { + return { status: 'stale' }; + } + if (lookupDocumentLeaf(document, layerId)?.effectiveLocked !== false) { + return { status: 'locked' }; + } + if (o.isLayerSessionHeld(layerId)) { + return { status: 'busy' }; + } + const txn = ctx.begin({ historyBytes: HISTORY_ENTRY_OVERHEAD_BYTES }); + if (!('publish' in txn)) { + return { status: layerEditRefusal(txn.status) }; + } + try { + const amended = + coalesceKey !== undefined ? this.amendable(layerId, (entry) => entry.coalesceKey === coalesceKey) : null; + const resolved = this.resolveBefore(txn, document, live, next, amended); + if ('refusal' in resolved) { + return { status: resolved.refusal }; + } + const { before, pixels: beforePixels } = resolved; + const after = structuredClone(next); + const result = txn.publish( + amended?.label ?? label, + this.replaceStep(after, null, live), + { + bytes: (beforePixels ? rgbaBytes(beforePixels.rect) : 0) + HISTORY_ENTRY_OVERHEAD_BYTES, + heldAssetRefs: collectHistoryMediaRefs(before, after), + redo: () => ctx.applyStep(this.replaceStep(after, null, before)), + undo: () => this.restoreBefore(before, beforePixels, after), + }, + { origin: 'user', replacing: amended?.token } + ); + if (result.status !== 'committed') { + return result.status === 'dispatch-rejected' || result.status === 'postcondition-failed' + ? { status: 'failed' } + : { status: layerEditRefusal(result.status) }; + } + const inputKey = controlDetectionInputKey(after); + this.last = { + before, + beforePixels, + coalesceKey, + inputKey, + label: amended?.label ?? label, + layerId, + requestsMap: controlDetectionInputKey(before) !== inputKey, + token: result.token, + }; + return { status: 'committed' }; + } finally { + txn.end(); + } + } + + captureControlDetectionGuard(layerId: string): ControlDetectionGuard | null { + // A background read: the reducer document, so an open structural preview stays open. + const layer = asControlLayer(this.options.ctx.getReducerDocument(), layerId); + const documentRevision = this.options.getDocumentRevision(); + if (!layer || documentRevision === null) { + return null; + } + return { + documentRevision, + inputKey: controlDetectionInputKey(layer), + layerId, + projectId: this.options.ctx.projectId, + }; + } + + async commitControlMap({ + detection, + guard, + image, + rect, + signal, + }: Parameters[0]): Promise { + const { ctx, ...o } = this.options; + const permit = ctx.capturePermit(); + if (!permit) { + return { status: 'busy' }; + } + if (signal?.aborted) { + return { status: 'aborted' }; + } + /** + * Everything the map was computed against must still hold. It reads the reducer document, so a refusal leaves an + * open structural preview alone; only an admitted commit ends it. + */ + const check = (): + | { document: CanvasDocumentContractV3; live: CanvasControlLayerContract } + | ControlMapCommitResult => { + if (signal?.aborted) { + return { status: 'aborted' }; + } + if (!ctx.isPermitCurrent(permit)) { + return { status: 'busy' }; + } + const document = ctx.getReducerDocument(); + const live = asControlLayer(document, guard.layerId); + if (!document || !live) { + return { status: 'missing' }; + } + if (o.getDocumentRevision() !== guard.documentRevision || controlDetectionInputKey(live) !== guard.inputKey) { + return { status: 'stale' }; + } + if (lookupDocumentLeaf(document, live.id)?.effectiveLocked !== false) { + return { status: 'locked' }; + } + // Pixels painted on the input itself after the run started are not in the original it names yet. + const input = getControlDetectionInput(live); + const inputIsPendingSource = live.original === undefined && input !== null && o.hasPendingPixelWork(live.id); + if (o.isLayerSessionHeld(live.id) || inputIsPendingSource) { + return { status: 'busy' }; + } + return { document, live }; + }; + // Refusals the caller retries are cheap to find; find them before decoding. + const early = check(); + if ('status' in early) { + return early; + } + const mapRect = { ...rect }; + const map = { image: structuredClone(image), rect: mapRect }; + if (!withDetectedMap(early.live, map, detection)) { + return { status: 'unrepresentable' }; + } + try { + const decoded = await o.decodeImage(image, { + isCurrent: () => ctx.isPermitCurrent(permit), + scaleToImage: false, + signal, + }); + if (decoded.status !== 'ok') { + return { status: decoded.status === 'aborted' ? 'aborted' : 'busy' }; + } + const decodedCheck = check(); + if ('status' in decodedCheck) { + return decodedCheck; + } + const afterPixels: LayerPixels = { pixels: decoded.surface, rect: mapRect }; + const txn = ctx.begin({ historyBytes: rgbaBytes(mapRect) + HISTORY_ENTRY_OVERHEAD_BYTES }); + if (!('publish' in txn)) { + return { status: layerEditRefusal(txn.status) }; + } + try { + // Admission ended any structural preview; the map lands on the committed layer, never on previewed values. + const checked = check(); + if ('status' in checked) { + return checked; + } + const { document, live } = checked; + const after = withDetectedMap(live, map, structuredClone(detection)); + if (!after) { + return { status: 'unrepresentable' }; + } + if (!txn.reserveRaster(rgbaBytes(mapRect))) { + return { status: 'over-budget' }; + } + const paired = this.amendable(live.id, (entry) => entry.requestsMap && entry.inputKey === guard.inputKey); + const resolved = this.resolveBefore(txn, document, live, after, paired); + if ('refusal' in resolved) { + return { status: resolved.refusal }; + } + const { before, pixels: beforePixels } = resolved; + const prepared = ctx.preparePixels(live.id, mapRect, afterPixels.pixels); + const label = paired?.label ?? `Detect ${after.controlType ?? 'control'}`; + const result = txn.publish( + label, + this.replaceStep(after, prepared, live), + { + bytes: + (beforePixels ? rgbaBytes(beforePixels.rect) : 0) + rgbaBytes(mapRect) + HISTORY_ENTRY_OVERHEAD_BYTES, + heldAssetRefs: collectHistoryMediaRefs(before, after), + redo: () => + replayWithPixels(ctx, after.id, afterPixels, (replayed) => this.replaceStep(after, replayed, before)), + undo: () => this.restoreBefore(before, beforePixels, after), + }, + // A background result is not a user edit: it must never change the invocation route. + { origin: 'system', replacing: paired?.token } + ); + if (result.status !== 'committed') { + return { status: guardedResultRefusal(result) }; + } + this.last = { + before, + beforePixels, + coalesceKey: undefined, + inputKey: controlDetectionInputKey(after), + label, + layerId: after.id, + requestsMap: true, + token: result.token, + }; + return { status: 'committed' }; + } finally { + txn.end(); + } + } catch (error) { + if (signal?.aborted || (error instanceof Error && error.name === 'AbortError')) { + return { status: 'aborted' }; + } + return { message: error instanceof Error ? error.message : String(error), status: 'failed' }; + } + } + + dispose(): void { + this.last = null; + } + + /** + * The recorded entry a new commit may replace: still the newest step, for this layer, and matching. A record whose + * entry is not on top is dropped with the pixels it holds, trading pairing for memory: if a redo later puts that + * entry back on top, its map records its own entry instead of joining it. + */ + private amendable(layerId: string, matches: (entry: RecordedEntry) => boolean): RecordedEntry | null { + const entry = this.last; + if (!entry) { + return null; + } + if (entry.token !== this.options.ctx.historyTop()) { + this.last = null; + return null; + } + return entry.layerId === layerId && matches(entry) ? entry : null; + } + + /** + * The layer one undo returns to, and the pixels it needs: an amended entry's own before-state, else the live layer. + * Pixels are kept only when the step replaces a source whose pixels are not persisted yet; otherwise the restored + * contract's persisted source reproduces them exactly. + */ + private resolveBefore( + txn: EditTransaction, + document: CanvasDocumentContractV3, + live: CanvasControlLayerContract, + after: CanvasControlLayerContract, + amended: RecordedEntry | null + ): BeforeState { + const before = amended?.before ?? structuredClone(live); + if (amended?.beforePixels) { + return { before, pixels: amended.beforePixels }; + } + if (sameSource(before, after) || !sameSource(before, live) || !this.options.hasPendingPixelWork(live.id)) { + return { before, pixels: null }; + } + const pixels = this.options.captureCache( + live, + document, + (rect) => txn.growHistory(rgbaBytes(rect)) && txn.reserveRaster(rgbaBytes(rect)) + ); + if (pixels === 'not-ready' || pixels === 'over-budget') { + return { refusal: pixels }; + } + // Unpersisted pixels without a cache cannot be restored by undo. + return pixels ? { before, pixels } : { refusal: 'not-ready' }; + } + + /** + * Swaps `layer` in. A source change drops the outgoing source's pending persistence, which would otherwise land + * its old pixels over the new source. + */ + private replaceStep( + layer: CanvasControlLayerContract, + prepared: Parameters[2], + restore: CanvasControlLayerContract, + persist = false + ): EditStep { + return replaceLayerStep(this.options.ctx, layer, prepared, { + beforeInstall: sameSource(layer, restore) ? undefined : () => this.options.discardPersisted(layer.id), + persist, + restore, + }); + } + + private restoreBefore( + before: CanvasControlLayerContract, + pixels: LayerPixels | null, + after: CanvasControlLayerContract + ): void { + const { ctx } = this.options; + if (!pixels) { + ctx.applyStep(this.replaceStep(before, null, after)); + return; + } + // Restored pixels were never persisted, so they persist again. + replayWithPixels(ctx, before.id, pixels, (prepared) => this.replaceStep(before, prepared, after, true)); + } +} diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/cropLayerController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/cropLayerController.test.ts index 2ad09f6f55a..32bf5ebb81b 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/cropLayerController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/cropLayerController.test.ts @@ -23,13 +23,13 @@ const layer: CanvasLayerContract = { const CACHE_RECT: Rect = { height: 50, width: 50, x: -10, y: -10 }; const BBOX: Rect = { height: 20, width: 20, x: 0, y: 0 }; -const createHarness = (historyBytes?: number) => { +const createHarness = (historyBytes?: number, target: CanvasLayerContract = layer) => { const document: CanvasDocumentContractV3 = { background: 'transparent', bbox: BBOX, height: 100, - selectedLayerId: 'paint', - stacks: stacksFrom([layer]), + selectedLayerId: target.id, + stacks: stacksFrom([target]), version: 3, width: 100, }; @@ -60,7 +60,7 @@ const createHarness = (historyBytes?: number) => { isGuardCurrent: () => true, isSupportedSource: () => true, }); - return { ...h, controller, discardPersisted, original: getDocumentLayer(h.document(), 'paint')! }; + return { ...h, controller, discardPersisted, original: getDocumentLayer(h.document(), target.id)! }; }; describe('CropLayerController', () => { @@ -80,6 +80,38 @@ describe('CropLayerController', () => { expect(h.layers.get('paint')?.rect).toEqual(BBOX); }); + it("keeps a control layer's type but drops the original its cropped pixels no longer derive from", async () => { + const depth = { settings: {}, type: 'depth_anything_depth_estimation' }; + const h = createHarness(undefined, { + adapter: { beginEndStepPct: [0, 1], controlMode: 'balanced', kind: 'controlnet', model: null, weight: 1 }, + blendMode: 'normal', + controlType: 'depth', + detection: depth, + filter: depth, + id: 'paint', + isEnabled: true, + isLocked: false, + name: 'Depth 1', + opacity: 1, + original: { image: { height: 50, imageName: 'photo', width: 50 }, type: 'image' }, + originalTransform: { rotation: 0, scaleX: 1, scaleY: 1, x: 2, y: 2 }, + source: { bitmap: { height: 50, imageName: 'map', width: 50 }, offset: { x: -10, y: -10 }, type: 'paint' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'control', + withTransparencyEffect: true, + }); + + await expect(h.controller.crop('paint')).resolves.toEqual({ status: 'cropped' }); + const cropped = getDocumentLayer(h.document(), 'paint'); + expect(cropped).toMatchObject({ controlType: 'depth', detection: null }); + expect(cropped).not.toHaveProperty('original'); + expect(cropped).not.toHaveProperty('originalTransform'); + expect(cropped).not.toHaveProperty('filter'); + + await h.history.undo(); + expect(getDocumentLayer(h.document(), 'paint')).toEqual(h.original); + }); + it('refuses a crop it could never undo before anything changes', async () => { const h = createHarness(1024); const before = h.document(); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/cropLayerController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/cropLayerController.ts index e5b62f7fc75..df6af9411e8 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/cropLayerController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/cropLayerController.ts @@ -7,6 +7,7 @@ import type { import type { RasterBackend, RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Rect } from '@workbench/canvas-engine/types'; +import { detachControlDerivation } from '@workbench/canvas-engine/document/controlDerivation'; import { getDocumentLayer } from '@workbench/canvas-engine/document/documentIndex'; import { renderableSourceOf } from '@workbench/canvas-engine/document/sources'; import { collectHistoryMediaRefs, HISTORY_ENTRY_OVERHEAD_BYTES } from '@workbench/canvas-engine/history/history'; @@ -142,8 +143,9 @@ export class CropLayerController { const { adjustments: _adjustments, ...rest } = before; after = { ...rest, source: paint, transform: identity }; } else if (before.type === 'control') { + // The cropped pixels become the image itself: no original's frame survives the crop. const { filter: _filter, ...rest } = before; - after = { ...rest, source: paint, transform: identity }; + after = detachControlDerivation({ ...rest, source: paint, transform: identity }); } else { after = { ...before, mask: { ...before.mask, bitmap: null, offset: paint.offset }, transform: identity }; } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/filterResultController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/filterResultController.test.ts new file mode 100644 index 00000000000..551a2d358f7 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/filterResultController.test.ts @@ -0,0 +1,136 @@ +import type { + CanvasControlLayerContract, + CanvasDocumentContractV3, + CanvasLayerContract, +} from '@workbench/canvas-engine/contracts'; + +import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { getDocumentLayer } from '@workbench/canvas-engine/document/documentIndex'; +import { describe, expect, it } from 'vitest'; + +import type { CommitRasterFilterOptions } from './filterResultController'; + +import { FilterResultController } from './filterResultController'; +import { createLayerOperationHarness } from './layerOperationHarness.testing'; + +const DEPTH = { settings: {}, type: 'depth_anything_depth_estimation' }; +const CANNY = { settings: { low_threshold: 100 }, type: 'canny_edge_detection' }; +const RESULT = { height: 16, imageName: 'filtered', width: 16 }; + +const derivedControl: CanvasControlLayerContract = { + adapter: { beginEndStepPct: [0, 1], controlMode: 'balanced', kind: 'controlnet', model: 'depth-model', weight: 0.8 }, + blendMode: 'normal', + controlType: 'depth', + detection: DEPTH, + filter: DEPTH, + id: 'control', + isEnabled: true, + isLocked: false, + name: 'Depth 1', + opacity: 1, + original: { image: { height: 16, imageName: 'photo', width: 16 }, type: 'image' }, + originalTransform: { rotation: 0, scaleX: 1, scaleY: 1, x: 2, y: 2 }, + source: { bitmap: { height: 16, imageName: 'map', width: 16 }, type: 'paint' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'control', + withTransparencyEffect: true, +}; + +const rasterLayer: CanvasLayerContract = { + blendMode: 'normal', + id: 'raster', + isEnabled: true, + isLocked: false, + name: 'Photo', + opacity: 1, + source: { bitmap: null, type: 'paint' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'raster', +}; + +const createHarness = () => { + const document: CanvasDocumentContractV3 = { + background: 'transparent', + bbox: { height: 16, width: 16, x: 0, y: 0 }, + height: 16, + selectedLayerId: 'control', + stacks: stacksFrom([rasterLayer, derivedControl]), + version: 3, + width: 16, + }; + const h = createLayerOperationHarness(document); + h.layers.getOrCreateRect('control', { height: 16, width: 16, x: 0, y: 0 }); + const controller = new FilterResultController({ + captureCache: (target) => { + const entry = h.layers.get(target.id); + return entry ? { pixels: h.backend.createSurface(16, 16), rect: { ...entry.rect } } : null; + }, + ctx: h.ctx, + decodeImage: () => + Promise.resolve({ decodedHeight: 16, decodedWidth: 16, status: 'ok', surface: h.backend.createSurface(16, 16) }), + discardPersisted: () => undefined, + getMainModelBase: () => 'sdxl', + needsPixelPersistence: () => true, + }); + const commit = (layerId: string, options: Partial) => + controller.commit({ + filter: CANNY, + guard: { layerId } as CommitRasterFilterOptions['guard'], + image: RESULT, + mode: 'copy', + rect: { height: 16, width: 16, x: 0, y: 0 }, + ...options, + }); + return { ...h, commit }; +}; + +const committedLayer = (h: ReturnType, result: Awaited>) => { + expect(result.status).toBe('committed'); + return getDocumentLayer(h.document(), 'layerId' in result ? result.layerId : null); +}; + +describe('FilterResultController control derivation', () => { + it('replacing a derived control map keeps the filter result as the image itself', async () => { + const h = createHarness(); + + const replaced = committedLayer(h, await h.commit('control', { mode: 'replace' })); + + expect(replaced).toMatchObject({ controlType: 'depth', detection: null, filter: CANNY }); + expect(replaced).not.toHaveProperty('original'); + expect(replaced).not.toHaveProperty('originalTransform'); + await h.history.undo(); + expect(getDocumentLayer(h.document(), 'control')).toEqual(derivedControl); + }); + + it('copies a control result as a map of its own, typed by the filter when the caller says so', async () => { + const h = createHarness(); + + const typed = committedLayer( + h, + await h.commit('control', { controlCopy: { controlType: 'edges' }, target: 'control' }) + ); + expect(typed).toMatchObject({ controlType: 'edges', detection: null, filter: CANNY }); + expect(typed).not.toHaveProperty('original'); + expect(typed).not.toHaveProperty('originalTransform'); + + const untyped = committedLayer( + h, + await h.commit('control', { controlCopy: { controlType: null }, target: 'control' }) + ); + for (const key of ['controlType', 'detection', 'original', 'originalTransform']) { + expect(untyped).not.toHaveProperty(key); + } + expect(untyped).toMatchObject({ adapter: derivedControl.adapter, filter: CANNY }); + }); + + it('types a raster-to-control copy from the filter', async () => { + const h = createHarness(); + + const copy = committedLayer( + h, + await h.commit('raster', { controlCopy: { controlType: 'edges' }, target: 'control' }) + ); + + expect(copy).toMatchObject({ controlType: 'edges', detection: null, filter: CANNY, type: 'control' }); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/filterResultController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/filterResultController.ts index 0734757b66c..a9bdc78d0e4 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/filterResultController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/filterResultController.ts @@ -9,6 +9,7 @@ import type { LayerMutationControllerOptions } from '@workbench/canvas-engine/co import type { DecodeImageResult } from '@workbench/canvas-engine/controllers/rasterController'; import type { PreparedLayerCacheReplacement } from '@workbench/canvas-engine/render/layerCache'; +import { detachControlDerivation } from '@workbench/canvas-engine/document/controlDerivation'; import { getDocumentLayer } from '@workbench/canvas-engine/document/documentIndex'; import { createControlLayer } from '@workbench/canvas-engine/document/layerFactories'; import { LayerFilterOutputDimensionError } from '@workbench/canvas-engine/filterError'; @@ -60,7 +61,6 @@ export interface FilterResultControllerOptions { } ) => Promise; readonly discardPersisted: (layerId: string) => void; - readonly getDefaultControlModel: (base: string | null) => string | null; readonly getMainModelBase: () => string | null; readonly needsPixelPersistence: (layer: CanvasLayerContract) => boolean; } @@ -147,7 +147,7 @@ export class FilterResultController { const { adjustments: _adjustments, ...base } = liveLayer; return structuredClone({ ...base, filter: options.filter, source: paintSource }); })() - : structuredClone({ ...liveLayer, filter: options.filter, source: paintSource }); + : structuredClone({ ...detachControlDerivation(liveLayer), filter: options.filter, source: paintSource }); const discard = (): void => o.discardPersisted(liveLayer.id); const prepared = o.ctx.preparePixels(liveLayer.id, rect, pixels); const interrupted = this.interrupted(options); @@ -181,18 +181,21 @@ export class FilterResultController { const layerId = o.ctx.createLayerId(); let copy: CanvasLayerContract; if (options.target === 'control') { - const buildControlBase = (): CanvasControlLayerContract => { - const mainBase = o.getMainModelBase(); - return createControlLayer( - `${liveLayer.name} filtered`, - layerId, - mainBase, - o.getDefaultControlModel(mainBase) - ); - }; + const buildControlBase = (): CanvasControlLayerContract => + createControlLayer({ base: o.getMainModelBase(), id: layerId, name: `${liveLayer.name} filtered` }); const base = liveLayer.type === 'control' ? structuredClone(liveLayer) : buildControlBase(); + // The copy is a map of its own: none of the source layer's derivation carries over. + const { + controlType: _controlType, + detection: _detection, + original: _original, + originalTransform: _originalTransform, + ...untyped + } = base; + const controlType = options.controlCopy?.controlType ?? null; copy = { - ...base, + ...untyped, + ...(controlType === null ? {} : { controlType, detection: null }), filter: options.filter, id: layerId, name: `${liveLayer.name} filtered`, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/generatedResultController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/generatedResultController.test.ts new file mode 100644 index 00000000000..0f2d7015ea6 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/generatedResultController.test.ts @@ -0,0 +1,92 @@ +import type { CommitGeneratedImageOptions } from '@workbench/canvas-engine/capabilities'; +import type { CanvasControlLayerContract, CanvasDocumentContractV3 } from '@workbench/canvas-engine/contracts'; + +import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { getDocumentLayer } from '@workbench/canvas-engine/document/documentIndex'; +import { describe, expect, it } from 'vitest'; + +import { GeneratedResultController } from './generatedResultController'; +import { createLayerOperationHarness } from './layerOperationHarness.testing'; + +const DEPTH = { settings: {}, type: 'depth_anything_depth_estimation' }; + +const derivedControl: CanvasControlLayerContract = { + adapter: { beginEndStepPct: [0, 1], controlMode: 'balanced', kind: 'controlnet', model: null, weight: 1 }, + blendMode: 'normal', + controlType: 'depth', + detection: DEPTH, + filter: DEPTH, + id: 'control', + isEnabled: true, + isLocked: false, + name: 'Depth 1', + opacity: 1, + original: { image: { height: 16, imageName: 'photo', width: 16 }, type: 'image' }, + source: { bitmap: { height: 16, imageName: 'map', width: 16 }, type: 'paint' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 3, y: 4 }, + type: 'control', + withTransparencyEffect: true, +}; + +const createHarness = () => { + const document: CanvasDocumentContractV3 = { + background: 'transparent', + bbox: { height: 16, width: 16, x: 0, y: 0 }, + height: 16, + selectedLayerId: 'control', + stacks: stacksFrom([derivedControl]), + version: 3, + width: 16, + }; + const h = createLayerOperationHarness(document); + h.layers.getOrCreateRect('control', { height: 16, width: 16, x: 0, y: 0 }); + const controller = new GeneratedResultController({ + captureCache: (target) => { + const entry = h.layers.get(target.id); + return entry ? { pixels: h.backend.createSurface(16, 16), rect: { ...entry.rect } } : null; + }, + clearPreview: () => undefined, + ctx: h.ctx, + decodeImage: () => + Promise.resolve({ decodedHeight: 16, decodedWidth: 16, status: 'ok', surface: h.backend.createSurface(16, 16) }), + discardPersisted: () => undefined, + getMainModelBase: () => 'sdxl', + needsPixelPersistence: () => true, + }); + const commit = (target: CommitGeneratedImageOptions['target']) => + controller.commit({ + guard: { layerId: 'control' } as CommitGeneratedImageOptions['guard'], + image: { height: 16, imageName: 'workflow-result', width: 16 }, + origin: { x: 0, y: 0 }, + target, + }); + return { ...h, commit }; +}; + +describe('GeneratedResultController control derivation', () => { + it('replacing a derived control map keeps the result as the image itself', async () => { + const h = createHarness(); + + await expect(h.commit('replace')).resolves.toEqual({ layerId: 'control', status: 'committed' }); + + const replaced = getDocumentLayer(h.document(), 'control'); + expect(replaced).toMatchObject({ controlType: 'depth', detection: null }); + expect(replaced).not.toHaveProperty('original'); + // Not map-only: the result itself is what a later detection runs on. + expect(replaced).not.toHaveProperty('filter'); + await h.history.undo(); + expect(getDocumentLayer(h.document(), 'control')).toEqual(derivedControl); + }); + + it('copies a result to an untyped control layer', async () => { + const h = createHarness(); + + const result = await h.commit('copy-control'); + + const copy = getDocumentLayer(h.document(), 'layerId' in result ? result.layerId : null); + expect(copy?.type).toBe('control'); + for (const key of ['controlType', 'detection', 'original', 'originalTransform']) { + expect(copy).not.toHaveProperty(key); + } + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/generatedResultController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/generatedResultController.ts index 7a02709622f..d9e91a28374 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/generatedResultController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/generatedResultController.ts @@ -5,6 +5,7 @@ import type { DecodeImageResult } from '@workbench/canvas-engine/controllers/ras import type { PreparedLayerCacheReplacement } from '@workbench/canvas-engine/render/layerCache'; import type { LayerTransform } from '@workbench/canvas-engine/transform/transformMath'; +import { detachControlDerivation } from '@workbench/canvas-engine/document/controlDerivation'; import { getDocumentLayer, getDocumentLeaves } from '@workbench/canvas-engine/document/documentIndex'; import { createControlLayer, nextControlLayerName } from '@workbench/canvas-engine/document/layerFactories'; import { collectHistoryMediaRefs, HISTORY_ENTRY_OVERHEAD_BYTES } from '@workbench/canvas-engine/history/history'; @@ -44,7 +45,6 @@ export interface GeneratedResultControllerOptions { options: { signal?: AbortSignal; isCurrent?: () => boolean } ) => Promise; readonly discardPersisted: (layerId: string) => void; - readonly getDefaultControlModel: (base: string | null) => string | null; readonly getMainModelBase: () => string | null; readonly needsPixelPersistence: (layer: CanvasLayerContract) => boolean; } @@ -131,7 +131,9 @@ export class GeneratedResultController { const { adjustments: _adjustments, ...base } = structuredClone(liveLayer); after = { ...base, source, transform: identityTransform }; } else { - after = { ...structuredClone(liveLayer), source, transform: identityTransform }; + // A workflow result replaces the pixels outright: no filter produced them and no kept original derives them. + const { filter: _filter, ...base } = structuredClone(liveLayer); + after = detachControlDerivation({ ...base, source, transform: identityTransform }); } const clearPreview = (): void => o.clearPreview(liveLayer.id); const publishAfter = (): void => { @@ -170,14 +172,13 @@ export class GeneratedResultController { const layerId = o.ctx.createLayerId(); const selectedLayerId = document.selectedLayerId; const buildControlCopy = (): CanvasLayerContract => { - const mainBase = o.getMainModelBase(); return { - ...createControlLayer( - options.copyLayerName ?? nextControlLayerName(getDocumentLeaves(document).map((layer) => layer.name)), - layerId, - mainBase, - o.getDefaultControlModel(mainBase) - ), + ...createControlLayer({ + base: o.getMainModelBase(), + id: layerId, + name: + options.copyLayerName ?? nextControlLayerName(getDocumentLeaves(document).map((layer) => layer.name)), + }), source, transform: identityTransform, }; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerOperationHarness.testing.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerOperationHarness.testing.ts index 63168d97423..cc489800725 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerOperationHarness.testing.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/layerOperationHarness.testing.ts @@ -1,5 +1,5 @@ import type { CanvasDocumentContractV3 } from '@workbench/canvas-engine/contracts'; -import type { CanvasProjectMutation } from '@workbench/canvas-engine/mutationContracts'; +import type { CanvasEditIntent, CanvasProjectMutation } from '@workbench/canvas-engine/mutationContracts'; import type { PreparedLayerCacheReplacement } from '@workbench/canvas-engine/render/layerCache'; import { createHistory } from '@workbench/canvas-engine/history/history'; @@ -14,6 +14,8 @@ export interface LayerOperationHarnessOptions { /** History byte budget; an edit whose entry can never fit is refused. */ readonly historyBytes?: number; readonly gestureActive?: boolean; + /** Observes where the context ends an open structural preview (`getDocument`, `begin`). */ + readonly endStructuralPreview?: () => void; } /** @@ -34,6 +36,11 @@ export const createLayerOperationHarness = ( }); const listeners = new Set<() => void>(); const dispatched: CanvasProjectMutation[] = []; + /** User edits routed to the invocation controller; system-origin publications never appear here. */ + const committedEdits: CanvasEditIntent[] = []; + /** An operation's edit lease: while held, edits without the operation's owner are refused. */ + let editingLocked = false; + const lockListeners = new Set<() => void>(); const installed: PreparedLayerCacheReplacement[] = []; const state = { gestureActive: options.gestureActive ?? false, @@ -45,7 +52,9 @@ export const createLayerOperationHarness = ( let reserved = 0; let nextId = 0; const ctx = createCanvasMutationContext({ - commitEdit: () => undefined, + commitEdit: (intent) => { + committedEdits.push(intent); + }, createLayerId: () => `new-${(nextId += 1)}`, dispatch: (action) => { dispatched.push(action); @@ -62,7 +71,14 @@ export const createLayerOperationHarness = ( return true; }, editOwner: Symbol('harness'), - editingLocked: { get: () => false, subscribe: () => () => undefined }, + endStructuralPreview: options.endStructuralPreview, + editingLocked: { + get: () => editingLocked, + subscribe: (listener) => { + lockListeners.add(listener); + return () => lockListeners.delete(listener); + }, + }, getDocument: () => project.canvas.document, getReducerDocument: () => project.canvas.document, history, @@ -100,6 +116,7 @@ export const createLayerOperationHarness = ( }); return { backend, + committedEdits, ctx, dispatched, document: () => project.canvas.document, @@ -107,6 +124,12 @@ export const createLayerOperationHarness = ( installed, layers, reservedBytes: () => reserved, + setEditingLocked: (locked: boolean) => { + editingLocked = locked; + for (const listener of lockListeners) { + listener(); + } + }, state, }; }; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/persistenceController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/persistenceController.test.ts index 43b3f6f4b02..ad4879e2c71 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/persistenceController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/persistenceController.test.ts @@ -1,10 +1,56 @@ -import type { BitmapStore } from '@workbench/canvas-engine/document/bitmapStore'; +import type { CanvasLayerSourceContract } from '@workbench/canvas-engine/contracts'; +import type { BitmapStore, BitmapStoreTimers } from '@workbench/canvas-engine/document/bitmapStore'; +import type { CanvasImageUploadResult } from '@workbench/canvas-engine/document/imageUpload'; +import { createBitmapStore } from '@workbench/canvas-engine/document/bitmapStore'; +import { createTestStubRasterBackend } from '@workbench/canvas-engine/render/raster.testStub'; import { describe, expect, it, vi } from 'vitest'; import { PersistenceController } from './persistenceController'; -const createStore = (): BitmapStore => ({ +/** Debounce timers that never fire, so only a barrier flushes. */ +const idleTimers: BitmapStoreTimers = { clearTimeout: () => undefined, setTimeout: () => 0 }; + +/** A real bitmap store over two paint layers whose uploads the test resolves. */ +const createPersistence = () => { + const surface = createTestStubRasterBackend().createSurface(4, 4); + const sources = new Map([ + ['a', { bitmap: null, type: 'paint' }], + ['b', { bitmap: null, type: 'paint' }], + ]); + const uploads: { content: string; resolve: (result: CanvasImageUploadResult) => void }[] = []; + const store = createBitmapStore({ + dispatch: (action) => { + if (action.type === 'updateCanvasLayerSource') { + sources.set(action.id, action.source); + } + return true; + }, + encodeSurface: () => Promise.resolve(new Blob([`pixels-${String(uploads.length)}`])), + getLayerSource: (layerId) => sources.get(layerId) ?? null, + getLayerSurface: () => ({ offset: { x: 0, y: 0 }, surface }), + hashBlob: (blob) => blob.text(), + timers: idleTimers, + uploadImage: (blob) => + blob.text().then( + (text) => + new Promise((resolve) => { + uploads.push({ content: text, resolve }); + }) + ), + }); + const controller = new PersistenceController(store); + /** Settles every upload started so far. */ + const settleUploads = async (): Promise => { + await vi.waitFor(() => expect(uploads.length).toBeGreaterThan(0)); + for (const upload of uploads.splice(0)) { + upload.resolve({ height: 4, imageName: `${upload.content}.png`, width: 4 }); + } + }; + return { controller, settleUploads, sources, store, uploads }; +}; + +const createStoreDouble = (): BitmapStore => ({ discardLayer: vi.fn(), dispose: vi.fn(), flushPendingUploads: vi.fn(() => Promise.resolve()), @@ -18,7 +64,7 @@ const createStore = (): BitmapStore => ({ describe('PersistenceController', () => { it('provides a flush barrier and idempotent disposal', async () => { - const store = createStore(); + const store = createStoreDouble(); const controller = new PersistenceController(store); await controller.flush(); expect(store.flushPendingUploads).toHaveBeenCalledOnce(); @@ -26,4 +72,57 @@ describe('PersistenceController', () => { controller.dispose(); expect(store.dispose).toHaveBeenCalledOnce(); }); + + describe('flushLayer', () => { + it('reports a layer without unsaved pixels as clean', async () => { + const p = createPersistence(); + + await expect(p.controller.flushLayer('a')).resolves.toBe('clean'); + expect(p.uploads).toEqual([]); + }); + + it('persists only the named layer', async () => { + const p = createPersistence(); + p.store.markLayerDirty('b'); + p.store.markLayerDirty('a'); + + const flushed = p.controller.flushLayer('a'); + await p.settleUploads(); + + await expect(flushed).resolves.toBe('flushed'); + expect(p.sources.get('a')).toMatchObject({ bitmap: { imageName: 'pixels-0.png' } }); + expect(p.store.hasPendingWork('a')).toBe(false); + expect(p.store.hasPendingWork('b')).toBe(true); + expect(p.sources.get('b')).toEqual({ bitmap: null, type: 'paint' }); + }); + + it('stops waiting when aborted', async () => { + const p = createPersistence(); + p.store.markLayerDirty('a'); + const abort = new AbortController(); + + const flushed = p.controller.flushLayer('a', { signal: abort.signal }); + await vi.waitFor(() => expect(p.uploads).toHaveLength(1)); + abort.abort(); + + await expect(flushed).resolves.toBe('aborted'); + await expect(p.controller.flushLayer('a', { signal: AbortSignal.abort() })).resolves.toBe('aborted'); + }); + + it('stops waiting for a suspended layer when aborted, and flushes it once released', async () => { + const p = createPersistence(); + p.store.markLayerDirty('a'); + const release = p.store.suspendLayer('a'); + const abort = new AbortController(); + + const flushed = p.controller.flushLayer('a', { signal: abort.signal }); + abort.abort(); + await expect(flushed).resolves.toBe('aborted'); + + release(); + const next = p.controller.flushLayer('a'); + await p.settleUploads(); + await expect(next).resolves.toBe('flushed'); + }); + }); }); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/persistenceController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/persistenceController.ts index ede6df96c6d..15b0b92152c 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/persistenceController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/persistenceController.ts @@ -1,5 +1,15 @@ import type { BitmapStore, FlushPendingUploadsOptions } from '@workbench/canvas-engine/document/bitmapStore'; +/** + * How a one-layer flush ended: its pixels persisted, there was nothing to persist, they could not be, or the wait + * aborted. + */ +export type LayerFlushResult = 'flushed' | 'clean' | 'failed' | 'aborted'; + +export interface LayerFlushOptions { + readonly signal?: AbortSignal; +} + export class PersistenceController { readonly store: BitmapStore; private disposed = false; @@ -12,6 +22,25 @@ export class PersistenceController { return this.store.flushPendingUploads(options); } + /** + * The paint barrier scoped to one layer: waits only for that layer's dirty and in-flight uploads, including while an + * open edit holds its pixels. + */ + async flushLayer(layerId: string, { signal }: LayerFlushOptions = {}): Promise { + if (signal?.aborted) { + return 'aborted'; + } + if (!this.store.hasPendingWork(layerId)) { + return 'clean'; + } + try { + await this.store.flushPendingUploads({ layerIds: [layerId], signal }); + return 'flushed'; + } catch (error) { + return signal?.aborted || (error instanceof Error && error.name === 'AbortError') ? 'aborted' : 'failed'; + } + } + dispose(): void { if (this.disposed) { return; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/transformEditingController.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/transformEditingController.test.ts index 0f49ad5bab3..90b79e58183 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/transformEditingController.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/transformEditingController.test.ts @@ -1,13 +1,15 @@ -import type { CanvasLayerContract } from '@workbench/canvas-engine/contracts'; +import type { CanvasControlLayerContract, CanvasLayerContract } from '@workbench/canvas-engine/contracts'; import type { TransformSession } from '@workbench/canvas-engine/engineStores'; import type { CanvasProjectMutation } from '@workbench/canvas-engine/mutationContracts'; import { createCanvasMutationContext } from '@workbench/canvas-engine/controllers/mutationContext'; import { documentFrom, layerContract } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { restoreControlOriginal, withDetectedMap } from '@workbench/canvas-engine/document/controlDerivation'; import { getDocumentLayer } from '@workbench/canvas-engine/document/documentIndex'; import { createHistory } from '@workbench/canvas-engine/history/history'; import { createLayerCacheStore } from '@workbench/canvas-engine/render/layerCache'; import { createTestStubRasterBackend } from '@workbench/canvas-engine/render/raster.testStub'; +import { composeLayerTransforms } from '@workbench/canvas-engine/transform/transformMath'; import { applyCanvasProjectMutation } from '@workbench/canvasProjectMutations'; import { createInitialWorkbenchState } from '@workbench/workbenchState'; import { describe, expect, it, vi } from 'vitest'; @@ -184,4 +186,78 @@ describe('TransformEditingController', () => { expect(h.history.entries().past).toEqual(['Transform layer']); expect(h.calls.restoreCache).toHaveBeenCalledOnce(); }); + + it("moves a detected control layer's original with the baked map, in the same undo step", async () => { + const depth = { settings: {}, type: 'depth_anything_depth_estimation' }; + const originalTransform = { rotation: 0, scaleX: 1, scaleY: 1, x: 2, y: 3 }; + const detected: CanvasControlLayerContract = { + adapter: { beginEndStepPct: [0, 1], controlMode: 'balanced', kind: 'controlnet', model: null, weight: 1 }, + blendMode: 'normal', + controlType: 'depth', + detection: depth, + filter: depth, + id: 'control', + isEnabled: true, + isLocked: false, + name: 'Depth 1', + opacity: 1, + original: { image: { height: 4, imageName: 'photo', width: 4 }, type: 'image' }, + originalTransform, + source: { bitmap: { height: 4, imageName: 'map.png', width: 4 }, type: 'paint' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 10, y: 0 }, + type: 'control', + withTransparencyEffect: true, + }; + const applied = { rotation: 0, scaleX: 2, scaleY: 2, x: 12, y: 4 }; + const h = createHarness(detected); + h.controller.begin('control'); + h.controller.update(applied); + h.controller.apply(); + + const baked = h.layer() as CanvasControlLayerContract; + expect(baked.transform).toEqual({ rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }); + expect(baked).toMatchObject({ controlType: 'depth', detection: depth, original: detected.original }); + // Re-detection and "None" both land the original where the applied transform put it. + const placed = composeLayerTransforms(applied, originalTransform); + expect(restoreControlOriginal(baked).transform).toEqual(placed); + const map = { image: { height: 4, imageName: 'next-map', width: 4 }, rect: { height: 4, width: 4, x: 0, y: 0 } }; + expect(withDetectedMap(baked, map, depth)?.transform).toEqual(placed); + expect(h.history.entries().past).toEqual(['Transform layer']); + + await h.history.undo(); + expect(h.layer()).toEqual(detected); + }); + + it('holds the original a shearing bake detaches, so undo can restore it', async () => { + const depth = { settings: {}, type: 'depth_anything_depth_estimation' }; + const detected: CanvasControlLayerContract = { + adapter: { beginEndStepPct: [0, 1], controlMode: 'balanced', kind: 'controlnet', model: null, weight: 1 }, + blendMode: 'normal', + controlType: 'depth', + detection: depth, + filter: depth, + id: 'control', + isEnabled: true, + isLocked: false, + name: 'Depth 1', + opacity: 1, + original: { image: { height: 4, imageName: 'photo', width: 4 }, type: 'image' }, + originalTransform: { rotation: Math.PI / 4, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + source: { bitmap: { height: 4, imageName: 'map.png', width: 4 }, type: 'paint' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'control', + withTransparencyEffect: true, + }; + const h = createHarness(detected); + h.controller.begin('control'); + h.controller.update({ rotation: 0, scaleX: 2, scaleY: 1, x: 0, y: 0 }); + h.controller.apply(); + + expect(h.layer()).not.toHaveProperty('original'); + expect(h.layer()).toMatchObject({ detection: null }); + expect(h.history.heldAssetRefs().images).toContain('photo'); + + await h.history.undo(); + expect(h.layer()).toEqual(detected); + }); }); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/transformEditingController.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/transformEditingController.ts index c02cf110e6c..fb860a0b557 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/transformEditingController.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/controllers/transformEditingController.ts @@ -1,5 +1,5 @@ import type { CanvasEditRefusal, StructuralCommitResult } from '@workbench/canvas-engine/capabilities'; -import type { CanvasDocumentContractV3 } from '@workbench/canvas-engine/contracts'; +import type { CanvasDocumentContractV3, CanvasLayerContract } from '@workbench/canvas-engine/contracts'; import type { TransformSession } from '@workbench/canvas-engine/engineStores'; import type { CanvasProjectMutation } from '@workbench/canvas-engine/mutationContracts'; import type { LayerCacheEntry } from '@workbench/canvas-engine/render/layerCache'; @@ -8,22 +8,27 @@ import type { LayerTransform } from '@workbench/canvas-engine/transform/transfor import type { Rect } from '@workbench/canvas-engine/types'; import { lookupDocumentLeaf } from '@workbench/canvas-engine/document-model/documentModel'; +import { rebaseControlOriginal } from '@workbench/canvas-engine/document/controlDerivation'; import { getDocumentLayer } from '@workbench/canvas-engine/document/documentIndex'; import { isLeafEditable } from '@workbench/canvas-engine/document/layerEligibility'; import { isRenderableLayer } from '@workbench/canvas-engine/document/sources'; -import { HISTORY_ENTRY_OVERHEAD_BYTES, NO_HELD_ASSET_REFS } from '@workbench/canvas-engine/history/history'; +import { + collectHistoryMediaRefs, + HISTORY_ENTRY_OVERHEAD_BYTES, + NO_HELD_ASSET_REFS, +} from '@workbench/canvas-engine/history/history'; import { isEmpty, roundOut, transformBounds } from '@workbench/canvas-engine/math/rect'; import { hittableLayerSize } from '@workbench/canvas-engine/tools/moveHitTest'; import { bakeMatrix } from '@workbench/canvas-engine/transform/transformMath'; import type { CanvasMutationContext, EditStep } from './mutationContext'; -import { rgbaBytes, withReplayReservation } from './editSteps'; +import { replaceLayerStep, rgbaBytes, withReplayReservation } from './editSteps'; export interface TransformEditingControllerOptions { readonly session: { get(): TransformSession | null; set(value: TransformSession | null): void }; readonly backend: RasterBackend; - readonly ctx: Pick; + readonly ctx: Pick; readonly getDocument: () => CanvasDocumentContractV3 | null; readonly getCache: (layerId: string) => LayerCacheEntry | null; readonly setOverride: (layerId: string, transform: LayerTransform | null) => void; @@ -42,6 +47,19 @@ export interface TransformEditingControllerOptions { const IDENTITY: LayerTransform = { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }; +/** + * What a bake step changes on the layer: only its transform, or the whole contract when more than the transform moves + * with the pixels (a control layer's kept original). + */ +type BakeChange = + | { readonly kind: 'transform'; readonly from: LayerTransform; readonly to: LayerTransform } + | { readonly kind: 'contract'; readonly from: CanvasLayerContract; readonly to: CanvasLayerContract }; + +const reversed = (change: BakeChange): BakeChange => + change.kind === 'transform' + ? { from: change.to, kind: 'transform', to: change.from } + : { from: change.to, kind: 'contract', to: change.from }; + const isRefusal = (status: string): status is CanvasEditRefusal => status === 'busy' || status === 'gesture-active' || status === 'not-ready' || status === 'over-budget'; @@ -102,8 +120,12 @@ export class TransformEditingController { this.deps.invalidate({ layers: [session.layerId], overlay: true }); } - /** A step that sets the layer's transform and installs matching whole-layer pixels, or rolls the transform back. */ - private bakeStep(layerId: string, from: LayerTransform, to: LayerTransform, rect: Rect, pixels: ImageData): EditStep { + /** A step that makes `change` and installs matching whole-layer pixels, or rolls the change back. */ + private bakeStep(layerId: string, change: BakeChange, rect: Rect, pixels: ImageData): EditStep { + const install = (): void => this.deps.restoreCache(layerId, rect, pixels); + if (change.kind === 'contract') { + return { ...replaceLayerStep(this.deps.ctx, change.to, null, { restore: change.from }), install }; + } const hasTransform = (transform: LayerTransform) => (document: CanvasDocumentContractV3 | null): boolean => { @@ -111,12 +133,12 @@ export class TransformEditingController { return !!current && unchanged(current.transform, transform); }; return { - accepted: hasTransform(to), - install: () => this.deps.restoreCache(layerId, rect, pixels), - mutation: { id: layerId, patch: { transform: to }, type: 'updateCanvasLayer' }, + accepted: hasTransform(change.to), + install, + mutation: { id: layerId, patch: { transform: change.to }, type: 'updateCanvasLayer' }, rollback: { - mutation: { id: layerId, patch: { transform: from }, type: 'updateCanvasLayer' }, - restored: hasTransform(from), + mutation: { id: layerId, patch: { transform: change.from }, type: 'updateCanvasLayer' }, + restored: hasTransform(change.from), }, }; } @@ -190,12 +212,21 @@ export class TransformEditingController { context.drawImage(cache.surface.canvas, beforeRect.x, beforeRect.y); context.setTransform(1, 0, 0, 1, 0, 0); const after = context.getImageData(0, 0, afterRect.width, afterRect.height); - const oldTransform = { ...session.startTransform }; - const forward = this.bakeStep(layer.id, oldTransform, IDENTITY, afterRect, after); - const backward = this.bakeStep(layer.id, IDENTITY, oldTransform, beforeRect, before); + // A kept control original moves with the baked pixels (detaching when its frame would shear), in the same step. + const rebased = + layer.type === 'control' && layer.original !== undefined + ? rebaseControlOriginal({ ...layer, transform: { ...IDENTITY } }, session.transform) + : null; + const change: BakeChange = rebased + ? { from: layer, kind: 'contract', to: rebased } + : { from: { ...session.startTransform }, kind: 'transform', to: IDENTITY }; + const forward = this.bakeStep(layer.id, change, afterRect, after); + const backward = this.bakeStep(layer.id, reversed(change), beforeRect, before); const result = txn.publish('Transform layer', forward, { bytes: before.data.byteLength + after.data.byteLength + HISTORY_ENTRY_OVERHEAD_BYTES, - heldAssetRefs: NO_HELD_ASSET_REFS, + // A contract bake that detaches drops the original from the document; undo restores it, so the entry holds it. + heldAssetRefs: + change.kind === 'contract' ? collectHistoryMediaRefs(change.from, change.to) : NO_HELD_ASSET_REFS, redo: () => this.replay(forward, afterRect), undo: () => this.replay(backward, beforeRect), }); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/document/bitmapStore.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/bitmapStore.ts index 99341dd82af..dec3c0386c3 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/document/bitmapStore.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/document/bitmapStore.ts @@ -53,6 +53,10 @@ export interface FlushPendingUploadsOptions { * that the user cannot see through pass false and fail fast instead. */ readonly waitForHeldPixels?: boolean; + /** Limits the barrier to these layers' dirty and in-flight work; absent means every layer. */ + readonly layerIds?: readonly string[]; + /** Stops waiting with an `AbortError`; uploads already started still settle. */ + readonly signal?: AbortSignal; } /** Injectable timer seam (defaults to the global timers). */ @@ -140,7 +144,7 @@ export interface BitmapStore { suspendLayer(layerId: string): () => void; /** Cancels pending persistence and invalidates an in-flight result for one layer. */ discardLayer(layerId: string): void; - /** Flushes every dirty layer immediately and resolves once all in-flight uploads settle. */ + /** Flushes dirty layers (all, or `options.layerIds`) now and resolves once their in-flight uploads settle. */ flushPendingUploads(options?: FlushPendingUploadsOptions): Promise; /** * Detects the most recently applied paint ref so the engine skips self-echo rasterization. Different refs @@ -227,9 +231,20 @@ export const createBitmapStore = (deps: BitmapStoreDeps): BitmapStore => { resolve(); } }; - const waitForSuspensionChange = (): Promise => - new Promise((resolve) => { - suspensionWaiters.add(resolve); + const abortError = (): DOMException => new DOMException('Canvas pixel persistence wait aborted.', 'AbortError'); + /** Resolves on the next suspension change; an abort rejects and drops the waiter. */ + const waitForSuspensionChange = (signal?: AbortSignal): Promise => + new Promise((resolve, reject) => { + const onAbort = (): void => { + suspensionWaiters.delete(waiter); + reject(abortError()); + }; + const waiter = (): void => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }; + suspensionWaiters.add(waiter); + signal?.addEventListener('abort', onAbort, { once: true }); }); const clearTimer = (layerId: string): void => { @@ -665,24 +680,50 @@ export const createBitmapStore = (deps: BitmapStoreDeps): BitmapStore => { /** Safety net against a genuine infinite loop; real barrier calls settle in a handful of rounds. */ const MAX_BARRIER_ITERATIONS = 10_000; - const flushPendingUploads = async ({ waitForHeldPixels = true }: FlushPendingUploadsOptions = {}): Promise => { + /** Settles with `promise`, or rejects with an `AbortError` once `signal` aborts. */ + const untilAborted = (promise: Promise, signal: AbortSignal | undefined): Promise => { + if (!signal) { + return promise; + } + return new Promise((resolve, reject) => { + const onAbort = (): void => reject(abortError()); + if (signal.aborted) { + onAbort(); + return; + } + signal.addEventListener('abort', onAbort, { once: true }); + promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort)); + }); + }; + + const flushPendingUploads = async ({ + layerIds, + signal, + waitForHeldPixels = true, + }: FlushPendingUploadsOptions = {}): Promise => { // Flush dirty layers and await in-flight work until newer strokes also persist. Do not retry failures within // this barrier; poll deferred ownership until pixels are released, unless the caller cannot wait. + const scope = layerIds ? new Set(layerIds) : null; + const inScope = (layerId: string): boolean => scope === null || scope.has(layerId); const blockedThisBarrier = new Set(); for (let iteration = 0; iteration < MAX_BARRIER_ITERATIONS; iteration += 1) { - const toFlush = Array.from(dirty).filter((layerId) => !blockedThisBarrier.has(layerId) && !isSuspended(layerId)); + if (signal?.aborted) { + throw abortError(); + } + const scopedDirty = Array.from(dirty).filter(inScope); + const toFlush = scopedDirty.filter((layerId) => !blockedThisBarrier.has(layerId) && !isSuspended(layerId)); for (const layerId of toFlush) { clearTimer(layerId); void runFlush(layerId); } - const ops = [...inFlight.values()]; + const ops = [...inFlight].filter(([layerId]) => inScope(layerId)).map(([, op]) => op); if (ops.length === 0) { - const suspendedLayerIds = Array.from(dirty).filter((layerId) => isSuspended(layerId)); + const suspendedLayerIds = scopedDirty.filter((layerId) => isSuspended(layerId)); if (suspendedLayerIds.length > 0) { if (!waitForHeldPixels) { throw new BitmapPersistenceError(suspendedLayerIds, 'held'); } - await waitForSuspensionChange(); + await waitForSuspensionChange(signal); continue; } const unpersistedLayerIds = Array.from(blockedThisBarrier).filter( @@ -693,7 +734,7 @@ export const createBitmapStore = (deps: BitmapStoreDeps): BitmapStore => { } return; } - await Promise.all(ops); + await untilAborted(Promise.all(ops), signal); const deferredLayerIds: string[] = []; for (const layerId of toFlush) { if (dirty.has(layerId) && dirtyReason.get(layerId) === 'failure') { @@ -706,7 +747,7 @@ export const createBitmapStore = (deps: BitmapStoreDeps): BitmapStore => { if (!waitForHeldPixels) { throw new BitmapPersistenceError(deferredLayerIds, 'held'); } - await sleep(DEFAULT_DEFERRED_RETRY_MS); + await untilAborted(sleep(DEFAULT_DEFERRED_RETRY_MS), signal); } } throw new Error('Canvas pixel persistence barrier exceeded its iteration limit.'); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/editing/controlPixelEdit.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/editing/controlPixelEdit.test.ts index 46550bf1407..9e5c000716b 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/editing/controlPixelEdit.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/editing/controlPixelEdit.test.ts @@ -178,6 +178,47 @@ describe('buildMaterializedPixelLayer', () => { expect(before.source.type).toBe('image'); }); + it("moves a control layer's kept original with the baked transform", () => { + const depth = { settings: {}, type: 'depth_anything_depth_estimation' }; + const before = control({ + controlType: 'depth', + detection: depth, + filter: depth, + original: { image: { height: 10, imageName: 'photo', width: 10 }, type: 'image' }, + originalTransform: { rotation: 0, scaleX: 1, scaleY: 1, x: 5, y: 0 }, + transform: { rotation: 0, scaleX: 2, scaleY: 2, x: 10, y: 20 }, + }); + + const after = buildMaterializedPixelLayer(before, { height: 20, width: 20, x: 10, y: 20 }); + + expect(after).toMatchObject({ controlType: 'depth', detection: depth, original: before.original }); + expect(after.type === 'control' && after.originalTransform).toEqual({ + rotation: 0, + scaleX: 2, + scaleY: 2, + x: 20, + y: 20, + }); + }); + + it('detaches a control layer whose original the bake would shear', () => { + const depth = { settings: {}, type: 'depth_anything_depth_estimation' }; + const before = control({ + controlType: 'depth', + detection: depth, + filter: depth, + original: { image: { height: 10, imageName: 'photo', width: 10 }, type: 'image' }, + originalTransform: { rotation: Math.PI / 4, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + transform: { rotation: 0, scaleX: 2, scaleY: 1, x: 0, y: 0 }, + }); + + const after = buildMaterializedPixelLayer(before, { height: 10, width: 20, x: 0, y: 0 }); + + expect(after).toMatchObject({ controlType: 'depth', detection: null, filter: depth }); + expect(after).not.toHaveProperty('original'); + expect(after).not.toHaveProperty('originalTransform'); + }); + it('clears raster adjustments after baking them into a paint source', () => { const before: CanvasLayerContract = { adjustments: [ diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/editing/controlPixelEdit.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/editing/controlPixelEdit.ts index d826b436e80..67b9d59df04 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/editing/controlPixelEdit.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/editing/controlPixelEdit.ts @@ -5,6 +5,7 @@ import type { SemanticLeaf } from '@workbench/canvas-engine/document-model/seman import type { RasterBackend, RasterSurface } from '@workbench/canvas-engine/render/raster'; import type { Rect } from '@workbench/canvas-engine/types'; +import { rebaseControlOriginal } from '@workbench/canvas-engine/document/controlDerivation'; import { isLeafEditable } from '@workbench/canvas-engine/document/layerEligibility'; import { isEmptyPolygonShape } from '@workbench/canvas-engine/document/sources'; import { roundOut, transformBounds } from '@workbench/canvas-engine/math/rect'; @@ -70,12 +71,19 @@ export const decidePixelEdit = ({ return { status: 'materialize' }; }; +/** + * The layer after its transform is baked into paint pixels at `rect`. A control layer's kept original moves with the + * bake, or detaches when the bake would shear its frame. + */ export const buildMaterializedPixelLayer = (layer: PixelEditableLayer, rect: Rect): PixelEditableLayer => { const materialized: PixelEditableLayer = { ...structuredClone(layer), source: { bitmap: null, offset: { x: rect.x, y: rect.y }, type: 'paint' }, transform: { ...IDENTITY_TRANSFORM }, }; + if (materialized.type === 'control') { + return rebaseControlOriginal(materialized, layer.transform); + } if (materialized.type === 'raster') { // Raster adjustments are applied before the layer transform by the normal // compositor. Materialization bakes those displayed pixels so the eraser diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.test.ts index 327c87a54f8..daa3b7e618a 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.test.ts @@ -10762,46 +10762,12 @@ describe('commitGeneratedImageResult', () => { } const created = getDocumentLeaves(engine.document.getDocument()!).find((layer) => layer.id === result.layerId); expect(created?.type === 'control' ? created.adapter.kind : null).toBe(expectedKind); - // Without a getDefaultControlModel option the layer starts model-less. + // New control layers never carry a model; generation resolves it. expect(created?.type === 'control' ? created.adapter.model : 'wrong-type').toBeNull(); engine.lifecycle.dispose(); } ); - it('seeds workflow Copy To Control layers with the injected default control model', async () => { - const source = workflowRaster(); - const { projectId, store } = createReducerBackedStore(workflowDocument([source]), 'sd-1'); - const getDefaultControlModel = vi.fn((base: string | null) => (base === 'sd-1' ? 'sd1-union' : null)); - const engine = createCanvasEngine({ - backend: createTestStubRasterBackend(), - bitmapStore: createSpyBitmapStore(), - getDefaultControlModel, - imageResolver: () => Promise.resolve(new Blob()), - projectId, - store, - }); - const exported = await engine.exports.exportLayerPixels(source.id); - if (exported.status !== 'ok') { - throw new Error('expected an exportable workflow source'); - } - - const result = await engine.layers.commitGeneratedImageResult({ - guard: exported.guard, - image: generatedImage, - origin: generatedOrigin, - target: 'copy-control', - }); - - expect(result.status).toBe('committed'); - if (result.status !== 'committed') { - throw new Error('expected a committed control copy'); - } - expect(getDefaultControlModel).toHaveBeenCalledWith('sd-1'); - const created = getDocumentLeaves(engine.document.getDocument()!).find((layer) => layer.id === result.layerId); - expect(created?.type === 'control' ? created.adapter.model : null).toBe('sd1-union'); - engine.lifecycle.dispose(); - }); - it('replaces a raster at the generated origin and native size, clears baked adjustments, and replays exactly', async () => { const source = workflowRaster(); const before = structuredClone(source); @@ -10865,8 +10831,13 @@ describe('commitGeneratedImageResult', () => { engine.lifecycle.dispose(); }); - it('replaces a control while preserving adapter, filter, transparency effect, and layer presentation', async () => { + it('replaces a control while preserving adapter, transparency effect, and layer presentation', async () => { const source = workflowControl(); + if (source.type !== 'control') { + throw new Error('expected a control fixture'); + } + // The result is the image itself, not a filter's map, so it drops the filter that made the old pixels. + const { filter: _filter, ...unfiltered } = source; const { projectId, store } = createReducerBackedStore(workflowDocument([source])); const engine = createCanvasEngine({ backend: createTestStubRasterBackend(), @@ -10890,7 +10861,7 @@ describe('commitGeneratedImageResult', () => { ).resolves.toEqual({ layerId: source.id, status: 'committed' }); expect(getDocumentLeaves(engine.document.getDocument()!)[0]).toEqual({ - ...source, + ...unfiltered, source: { bitmap: generatedImage, offset: generatedOrigin, type: 'paint' }, transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, }); @@ -10898,8 +10869,7 @@ describe('commitGeneratedImageResult', () => { expect(getDocumentLeaves(engine.document.getDocument()!)[0]).toEqual(source); await engine.history.redo(); expect(getDocumentLeaves(engine.document.getDocument()!)[0]).toMatchObject({ - adapter: source.type === 'control' ? source.adapter : undefined, - filter: source.type === 'control' ? source.filter : undefined, + adapter: source.adapter, withTransparencyEffect: true, }); engine.lifecycle.dispose(); @@ -19270,3 +19240,44 @@ describe('color picker loupe', () => { engine.lifecycle.dispose(); }); }); + +describe('control derivation composition', () => { + it('commits control-layer patches without the operation lease, so a running operation refuses them', async () => { + const control: CanvasControlLayerContract = { + adapter: { beginEndStepPct: [0, 1], controlMode: 'balanced', kind: 'controlnet', model: null, weight: 1 }, + blendMode: 'normal', + id: 'control', + isEnabled: true, + isLocked: false, + name: 'Control Layer 1', + opacity: 1, + source: { image: { height: 10, imageName: 'photo', width: 10 }, type: 'image' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'control', + withTransparencyEffect: true, + }; + const { store } = createReactiveStore({ ...makeDoc(), stacks: stacksFrom([control]) }); + const { applicationHost, engine } = createCoreCanvasEngine({ + backend: createTestStubRasterBackend(), + imageResolver: () => Promise.resolve(new Blob()), + mutationPort: createTestMutationPort(store, 'p1'), + projectId: 'p1', + reportError: () => undefined, + uploadImage: () => Promise.resolve({ height: 1, imageName: 'durable', width: 1 }), + uploadIntermediateImage: () => Promise.resolve({ height: 1, imageName: 'transient', width: 1 }), + }); + const patch = { expected: control, label: 'Set control type', next: { ...control, controlType: 'depth' as const } }; + + engine.stores.documentEditingLocked.set(true); + expect(engine.controlDerivation.commitControlLayerPatch(patch)).toEqual({ status: 'busy' }); + engine.stores.documentEditingLocked.set(false); + expect(engine.controlDerivation.commitControlLayerPatch(patch)).toEqual({ status: 'committed' }); + expect(engine.controlDerivation.captureControlDetectionGuard('control')).toMatchObject({ + documentRevision: 0, + layerId: 'control', + projectId: 'p1', + }); + await expect(applicationHost.flushLayerUploads('control', {})).resolves.toBe('clean'); + engine.lifecycle.dispose(); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.ts index 7d850b10de7..f4a1ceb19e0 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/engine.ts @@ -1,4 +1,5 @@ import type { + CanvasControlDerivationCapability, CanvasEditRefusal, CanvasHistoryReplayStatus, CanvasHistoryCapability, @@ -93,6 +94,7 @@ import type { CanvasProjectMutationPort } from '@workbench/canvasProjectMutation import { createItemId } from '@platform/browser/randomUuid'; import { areJsonValuesStructurallyEqual } from '@platform/core/json'; +import { ControlDerivationController } from '@workbench/canvas-engine/controllers/controlDerivationController'; import { PixelEditController } from '@workbench/canvas-engine/controllers/controlPixelController'; import { EditingController } from '@workbench/canvas-engine/controllers/editingController'; import { FilterResultController } from '@workbench/canvas-engine/controllers/filterResultController'; @@ -267,8 +269,6 @@ export interface CanvasEngineOptions { uploadIntermediateImage(blob: Blob): Promise<{ height: number; imageName: string; width: number }>; /** Supplies the currently selected model base for core-created control layer contracts. */ getMainModelBase?: () => string | null; - /** Supplies the default control model key for core-created control layer contracts. */ - getDefaultControlModel?: (base: string | null) => string | null; /** Supplies the Layers panel's transient multi-selection for grouped moves. */ getSelectedLayerIds?: () => readonly string[]; /** Publishes engine history's transient multi-selection changes back to the Layers panel. */ @@ -311,6 +311,8 @@ export interface CanvasEngineInternalExportCapability extends CanvasEngineExport /** Private engine composition shape used only inside the Canvas implementation and its tests. */ export interface CanvasEngineImplementation extends CanvasEngine { + /** Control-layer intent and detection-map commits, for the application's detection coordinator. */ + readonly controlDerivation: CanvasControlDerivationCapability; readonly exports: CanvasEngineInternalExportCapability; readonly stores: EngineStores; } @@ -2078,9 +2080,6 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC const getMainModelBase = (): string | null => { return opts.getMainModelBase?.() ?? null; }; - const getDefaultControlModel = (base: string | null): string | null => { - return opts.getDefaultControlModel?.(base) ?? null; - }; /** Conversion reducers clone contracts, so their publication postcondition compares by value. */ const documentHasLayerContract = ( @@ -2233,19 +2232,27 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC ctx: mutationContext, decodeImage: (image, options) => rasterController.decodeImage(image, options), discardPersisted: (layerId) => bitmapStore.discardLayer(layerId), - getDefaultControlModel, getMainModelBase, needsPixelPersistence: layerNeedsPixelPersistence, }); const commitRasterFilterResult = filterResultController.commit.bind(filterResultController); + const controlDerivationController = new ControlDerivationController({ + captureCache: captureLayerCache, + ctx: mutationContext, + decodeImage: (image, options) => rasterController.decodeImage(image, options), + discardPersisted: (layerId) => bitmapStore.discardLayer(layerId), + getDocumentRevision: () => mutationPort.getCanvasState()?.documentRevision ?? null, + hasPendingPixelWork: (layerId) => bitmapStore.hasPendingWork(layerId), + isLayerSessionHeld: isLayerHeldBySession, + }); + const generatedResultController = new GeneratedResultController({ captureCache: captureLayerCache, clearPreview: clearFilterPreview, ctx: mutationContext, decodeImage: (image, options) => rasterController.decodeImage(image, options), discardPersisted: (layerId) => bitmapStore.discardLayer(layerId), - getDefaultControlModel, getMainModelBase, needsPixelPersistence: layerNeedsPixelPersistence, }); @@ -2419,6 +2426,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC cleanup.run(cancelOpenPixelEdit); cleanup.run(() => pixelEditController?.dispose()); cleanup.run(() => filterResultController.dispose()); + cleanup.run(() => controlDerivationController.dispose()); cleanup.run(() => generatedResultController.dispose()); cleanup.run(() => stagedResultController.dispose()); cleanup.run(() => editingController.dispose()); @@ -2869,6 +2877,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC encodeSurface: (surface) => backend.encodeSurface(surface, 'image/png'), exportBakedLayerBlob: (layerId) => exportBakedLayerBlob(layerId, { includeDisabled: true }), exportLayerPixels: rasterizeLayerPixelsForStructural, + flushLayerUploads: (layerId, options) => persistenceController.flushLayer(layerId, options), getCompositeExecutorDeps, getDocument: () => mirror.getDocument(), isGuardCurrent: isLayerExportGuardCurrent, @@ -2914,6 +2923,7 @@ export const createCanvasEngine = (opts: CanvasEngineOptions): CanvasEngineCoreC }; const engine: CanvasEngineImplementation = { + controlDerivation: controlDerivationController, diagnostics: diagnosticsCapability, document: documentCapability, edits: editingController.edits, From d428bb665544636c56dfb9def5d7c3b3f9cef2a0 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Fri, 23 Oct 2026 01:20:06 -0600 Subject: [PATCH 04/35] feat(canvas): detect control maps in the background An intent-driven coordinator debounces per layer, runs the detector graph on the kept original, retries commits refused by a modal operation, and exposes per-layer status. It never starts from document changes, so undo and redo never re-run detection. --- .../ControlDetectionCoordinator.test.ts | 548 ++++++++++++++++ .../ControlDetectionCoordinator.ts | 590 ++++++++++++++++++ .../src/workbench/canvas-operations/api.ts | 7 + .../workbench/canvas-operations/contracts.ts | 10 + .../controlDetectionRunner.test.ts | 90 +++ .../controlDetectionRunner.ts | 122 ++++ .../controlDetectionStatusStore.test.ts | 49 ++ .../controlDetectionStatusStore.ts | 82 +++ .../controlLayerIntents.test.ts | 235 +++++++ .../canvas-operations/controlLayerIntents.ts | 192 ++++++ ...reateCanvasEngine.controlDetection.test.ts | 184 ++++++ .../canvas-operations/createCanvasEngine.ts | 68 +- .../canvas-operations/operationTypes.ts | 49 ++ .../widgets/canvas/engineStoreHooks.ts | 16 + 14 files changed, 2237 insertions(+), 5 deletions(-) create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-operations/ControlDetectionCoordinator.test.ts create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-operations/ControlDetectionCoordinator.ts create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionRunner.test.ts create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionRunner.ts create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionStatusStore.test.ts create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionStatusStore.ts create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.test.ts create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.ts create mode 100644 invokeai/frontend/webv2/src/workbench/canvas-operations/createCanvasEngine.controlDetection.test.ts diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/ControlDetectionCoordinator.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/ControlDetectionCoordinator.test.ts new file mode 100644 index 00000000000..3f5513a7f1d --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/ControlDetectionCoordinator.test.ts @@ -0,0 +1,548 @@ +import type { ControlMapCommitResult } from '@workbench/canvas-engine/api'; +import type { CanvasControlLayerContract, CanvasDocumentContractV3 } from '@workbench/canvas-engine/contracts'; +import type { LayerFlushResult } from '@workbench/canvas-engine/controllers/persistenceController'; +import type { BackendGraphContract } from '@workbench/graphContracts'; + +import { createAccountLifecycle } from '@platform/state/accountLifecycle'; +import { + controlDetectionInputKey, + createControlLayer, + isControlMapCurrent, + withDetectedMap, +} from '@workbench/canvas-engine/api'; +import { documentFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { createCanvasEditGate, type CanvasEditLease } from '@workbench/canvas-engine/editGate'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { CONTROL_DETECTION_COMMIT_TIMEOUT_MS, createControlDetectionCoordinator } from './ControlDetectionCoordinator'; + +const DEBOUNCE_MS = 400; +const image = (imageName: string) => ({ height: 64, imageName, width: 64 }); +const photoLayer = (id: string, patch: Partial = {}): CanvasControlLayerContract => ({ + ...createControlLayer({ id, name: id }), + source: { image: image(`${id}.png`), type: 'image' }, + ...patch, +}); + +interface PendingGraph { + imageName: string; + signal: AbortSignal; + resolve(output: { height: number; imageName: string; width: number }): void; + reject(error: unknown): void; +} + +const createHarness = (initial: CanvasControlLayerContract[] = [photoLayer('a')]) => { + const layers = new Map(initial.map((layer) => [layer.id, layer])); + let document: CanvasDocumentContractV3 = documentFrom([...layers.values()]); + let revision = 0; + const publish = (): void => { + document = documentFrom([...layers.values()]); + }; + const setLayer = (layer: CanvasControlLayerContract): void => { + layers.set(layer.id, layer); + publish(); + }; + const removeLayer = (id: string): void => { + layers.delete(id); + publish(); + }; + + const gate = createCanvasEditGate(); + let lease: CanvasEditLease | null = null; + let mapCommitOverride: (() => ControlMapCommitResult) | null = null; + const derivation = { + captureControlDetectionGuard: vi.fn((layerId: string) => { + const layer = layers.get(layerId); + return layer + ? { documentRevision: revision, inputKey: controlDetectionInputKey(layer), layerId, projectId: 'p1' } + : null; + }), + commitControlLayerPatch: vi.fn( + (options: { next: CanvasControlLayerContract }): { status: 'committed' } | { status: 'busy' } => { + setLayer(options.next); + return { status: 'committed' }; + } + ), + commitControlMap: vi.fn( + (options: { + guard: { layerId: string; documentRevision: number; inputKey: string }; + image: { height: number; imageName: string; width: number }; + rect: { height: number; width: number; x: number; y: number }; + detection: { type: string; settings: Record }; + }): Promise => { + if (mapCommitOverride) { + return Promise.resolve(mapCommitOverride()); + } + // The engine refuses while a modal operation holds the edit lease. + if (lease?.isCurrent()) { + return Promise.resolve({ status: 'busy' }); + } + const layer = layers.get(options.guard.layerId); + if (!layer) { + return Promise.resolve({ status: 'missing' }); + } + if (options.guard.documentRevision !== revision || options.guard.inputKey !== controlDetectionInputKey(layer)) { + return Promise.resolve({ status: 'stale' }); + } + setLayer(withDetectedMap(layer, { image: options.image, rect: options.rect }, options.detection)!); + return Promise.resolve({ status: 'committed' }); + } + ), + }; + + const graphs: PendingGraph[] = []; + const runFilterGraph = vi.fn( + (options: { graph: BackendGraphContract; signal?: AbortSignal }) => + new Promise<{ height: number; imageName: string; width: number }>((resolve, reject) => { + const flatten = options.graph.nodes.control_filter_flatten as { image?: { image_name: string } } | undefined; + graphs.push({ + imageName: flatten?.image?.image_name ?? '', + reject, + resolve, + signal: options.signal!, + }); + }) + ); + const lockListeners = new Set<() => void>(); + const accounts = createAccountLifecycle(); + const flushLayerUploads = vi.fn((_layerId: string, _options: { signal?: AbortSignal }) => + Promise.resolve('clean') + ); + const makeDurable = vi.fn((_imageName: string) => Promise.resolve()); + const coordinator = createControlDetectionCoordinator({ + accounts: { capture: accounts.capture, register: accounts.register }, + derivation, + flushLayerUploads, + getDocument: () => document, + makeDurable, + projectId: 'p1', + runFilterGraph, + subscribeLockChanges: (listener) => { + lockListeners.add(listener); + return () => lockListeners.delete(listener); + }, + }); + + let mapCount = 0; + /** Completes the oldest pending graph with a 64×64 map. */ + const finishGraph = async (index = 0): Promise => { + const [pending] = graphs.splice(index, 1); + mapCount += 1; + pending!.resolve({ height: 64, imageName: `map-${String(mapCount)}.png`, width: 64 }); + await vi.advanceTimersByTimeAsync(0); + }; + + return { + accounts, + acquireLease: () => { + lease = gate.tryAcquire({ kind: 'filter', layerId: 'raster' }); + return lease; + }, + bumpRevision: () => { + revision += 1; + }, + coordinator, + derivation, + finishGraph, + flushLayerUploads, + graphs, + layer: (id = 'a') => layers.get(id)!, + lockListeners, + makeDurable, + emitLockChange: () => { + for (const listener of lockListeners) { + listener(); + } + }, + removeLayer, + runFilterGraph, + setLayer, + setMapCommit: (override: (() => ControlMapCommitResult) | null) => { + mapCommitOverride = override; + }, + status: (id = 'a') => coordinator.getStatus(id), + }; +}; + +const settle = (ms = DEBOUNCE_MS) => vi.advanceTimersByTimeAsync(ms); + +beforeEach(() => { + vi.useFakeTimers(); +}); +afterEach(() => { + vi.useRealTimers(); +}); + +describe('ControlDetectionCoordinator', () => { + it('debounces intents, runs one graph on the input and commits a durable map', async () => { + const h = createHarness(); + + expect(h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' })).toBe('updated'); + expect(h.status()).toEqual({ status: 'queued' }); + await settle(DEBOUNCE_MS - 1); + expect(h.coordinator.applyIntent('a', { kind: 'set-detector-settings', settings: { low_threshold: 50 } })).toBe( + 'updated' + ); + await settle(DEBOUNCE_MS - 1); + expect(h.runFilterGraph).not.toHaveBeenCalled(); + await settle(1); + + expect(h.runFilterGraph).toHaveBeenCalledOnce(); + expect(h.graphs[0]!.imageName).toBe('a.png'); + expect(h.status()).toEqual({ detector: 'canny_edge_detection', status: 'processing' }); + expect(h.derivation.commitControlLayerPatch).toHaveBeenLastCalledWith( + expect.objectContaining({ coalesceKey: 'a:canny_edge_detection:settings' }) + ); + + await h.finishGraph(); + + expect(h.makeDurable).toHaveBeenCalledExactlyOnceWith('map-1.png'); + expect(h.layer().source).toMatchObject({ bitmap: { imageName: 'map-1.png' }, type: 'paint' }); + expect(isControlMapCurrent(h.layer())).toBe(true); + expect(h.status()).toEqual({ status: 'idle' }); + }); + + it('maps refused patches without scheduling, and settles layers that need no detection', async () => { + const h = createHarness(); + h.derivation.commitControlLayerPatch.mockReturnValueOnce({ status: 'busy' }); + + expect(h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' })).toBe('blocked'); + expect(h.coordinator.applyIntent('missing', { kind: 'draw' })).toBe('missing'); + expect(h.coordinator.applyIntent('a', { controlType: 'tile', kind: 'set-type' })).toBe('updated'); + expect(h.coordinator.applyIntent('a', { controlType: 'tile', kind: 'set-type' })).toBe('unchanged'); + await settle(); + + expect(h.runFilterGraph).not.toHaveBeenCalled(); + expect(h.status()).toEqual({ status: 'idle' }); + }); + + it('aborts a running job when a newer intent changes its input, and keeps it otherwise', async () => { + const h = createHarness(); + h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' }); + await settle(); + const first = h.graphs.shift()!; + + expect(h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type', name: 'Edges 1' })).toBe('updated'); + await settle(); + expect(first.signal.aborted).toBe(false); + expect(h.runFilterGraph).toHaveBeenCalledOnce(); + + h.coordinator.applyIntent('a', { detector: 'mlsd_detection', kind: 'set-detector' }); + expect(first.signal.aborted).toBe(true); + await settle(); + expect(h.runFilterGraph).toHaveBeenCalledTimes(2); + + first.resolve({ height: 64, imageName: 'late.png', width: 64 }); + await settle(0); + expect(h.derivation.commitControlMap).not.toHaveBeenCalled(); + await h.finishGraph(); + expect(h.makeDurable).toHaveBeenCalledExactlyOnceWith('map-1.png'); + expect(h.layer().filter?.type).toBe('mlsd_detection'); + }); + + it.each([ + ['layer removal', (h: ReturnType) => h.removeLayer('a')], + ['a document swap', (h: ReturnType) => h.bumpRevision()], + [ + 'a different detector (undo)', + (h: ReturnType) => + h.setLayer({ ...h.layer(), detection: { settings: {}, type: 'mlsd_detection' } }), + ], + [ + 'an undo of a replaced image', + (h: ReturnType) => + h.setLayer({ ...h.layer(), source: { image: image('before-replace.png'), type: 'image' } }), + ], + ])('drops a result made stale by %s before preserving it', async (_label, staleIt) => { + const h = createHarness(); + h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' }); + await settle(); + + staleIt(h); + await h.finishGraph(); + await settle(); + + expect(h.makeDurable).not.toHaveBeenCalled(); + expect(h.derivation.commitControlMap).not.toHaveBeenCalled(); + expect(h.runFilterGraph).toHaveBeenCalledOnce(); + expect(h.status()).toEqual({ status: 'idle' }); + }); + + it('restarts after the layer’s own paint lands under a run, debounced and at most three times', async () => { + const paint = (name: string) => ({ bitmap: image(name), offset: { x: 0, y: 0 }, type: 'paint' }) as const; + const h = createHarness([photoLayer('a', { controlType: 'edges', detection: null, source: paint('strokes.png') })]); + h.coordinator.applyIntent('a', { detector: 'canny_edge_detection', kind: 'set-detector' }); + await settle(); + + for (let run = 1; run <= 4; run += 1) { + h.setLayer({ ...h.layer(), source: paint(`strokes-v${String(run)}.png`) }); + await h.finishGraph(); + if (run < 4) { + expect(h.status()).toEqual({ status: 'queued' }); + await settle(DEBOUNCE_MS - 1); + expect(h.runFilterGraph).toHaveBeenCalledTimes(run); + await settle(1); + expect(h.graphs[0]!.imageName).toBe(`strokes-v${String(run)}.png`); + } + } + await settle(); + + expect(h.runFilterGraph).toHaveBeenCalledTimes(4); + expect(h.derivation.commitControlMap).not.toHaveBeenCalled(); + expect(h.makeDurable).not.toHaveBeenCalled(); + expect(h.status()).toEqual({ status: 'idle' }); + }); + + it('lets a modal operation take the edit lease during a run, then commits after it releases', async () => { + const h = createHarness(); + h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' }); + await settle(); + + const lease = h.acquireLease(); + expect(lease).not.toBeNull(); + await h.finishGraph(); + expect(h.derivation.commitControlMap).toHaveBeenCalledOnce(); + expect(h.status()).toEqual({ status: 'queued' }); + + lease!.release(); + h.emitLockChange(); + await settle(0); + + expect(h.derivation.commitControlMap).toHaveBeenCalledTimes(2); + expect(h.makeDurable).toHaveBeenCalledOnce(); + expect(isControlMapCurrent(h.layer())).toBe(true); + expect(h.status()).toEqual({ status: 'idle' }); + }); + + it('retries a busy commit with backoff and reports commit-busy after the timeout', async () => { + const h = createHarness(); + h.setMapCommit(() => ({ status: 'busy' })); + h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' }); + await settle(); + await h.finishGraph(); + + const attempts = (): number => h.derivation.commitControlMap.mock.calls.length; + expect(attempts()).toBe(1); + await settle(249); + expect(attempts()).toBe(1); + await settle(1); + expect(attempts()).toBe(2); + await settle(500); + expect(attempts()).toBe(3); + await settle(1_000); + expect(attempts()).toBe(4); + await settle(2_000); + expect(attempts()).toBe(5); + expect(h.status()).toEqual({ status: 'queued' }); + + await settle(CONTROL_DETECTION_COMMIT_TIMEOUT_MS); + expect(h.status()).toEqual({ error: { code: 'commit-busy' }, status: 'error' }); + const final = attempts(); + await settle(10_000); + expect(attempts()).toBe(final); + }); + + it.each([ + [{ message: 'too big', status: 'over-budget' } as const, { code: 'over-budget', detail: 'too big' }], + [{ status: 'unrepresentable' } as const, { code: 'unrepresentable' }], + [{ message: 'decode failed', status: 'failed' } as const, { code: 'apply-failed', detail: 'decode failed' }], + ])('reports a %o commit as a typed error', async (result, error) => { + const h = createHarness(); + h.setMapCommit(() => result); + h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' }); + await settle(); + await h.finishGraph(); + + expect(h.status()).toEqual({ error, status: 'error' }); + }); + + it('reports graph failures and unknown detectors, and Retry runs again', async () => { + const h = createHarness(); + h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' }); + await settle(); + h.graphs.shift()!.reject(new Error('queue offline')); + await settle(0); + expect(h.status()).toEqual({ error: { code: 'process-failed', detail: 'queue offline' }, status: 'error' }); + + expect(h.coordinator.retry('a')).toBe('scheduled'); + await settle(0); + expect(h.runFilterGraph).toHaveBeenCalledTimes(2); + await h.finishGraph(); + expect(h.coordinator.retry('a')).toBe('not-needed'); + expect(h.coordinator.retry('missing')).toBe('missing'); + + h.coordinator.applyIntent('a', { detector: 'retired_detector', kind: 'set-detector' }); + await settle(); + expect(h.status()).toEqual({ error: { code: 'unknown-detector', detail: 'retired_detector' }, status: 'error' }); + expect(h.runFilterGraph).toHaveBeenCalledTimes(2); + }); + + it('flushes the layer’s own paint before detecting it', async () => { + const h = createHarness([ + photoLayer('a', { controlType: 'edges', detection: null, source: { bitmap: null, type: 'paint' } }), + ]); + h.flushLayerUploads.mockImplementation(() => { + h.setLayer({ ...h.layer(), source: { bitmap: image('strokes.png'), offset: { x: 3, y: 4 }, type: 'paint' } }); + return Promise.resolve('flushed'); + }); + + h.coordinator.applyIntent('a', { detector: 'canny_edge_detection', kind: 'set-detector' }); + await settle(); + + expect(h.flushLayerUploads).toHaveBeenCalledExactlyOnceWith('a', { signal: expect.any(AbortSignal) }); + expect(h.graphs[0]!.imageName).toBe('strokes.png'); + await h.finishGraph(); + expect(h.layer().original).toMatchObject({ bitmap: { imageName: 'strokes.png' } }); + }); + + it('never schedules detection from document changes', async () => { + const h = createHarness(); + // An undo back to a typed layer whose map is stale. + h.setLayer({ ...h.layer(), controlType: 'edges', detection: { settings: {}, type: 'canny_edge_detection' } }); + h.coordinator.reconcileWithDocument(); + await settle(10_000); + + expect(h.runFilterGraph).not.toHaveBeenCalled(); + expect(h.status()).toEqual({ status: 'idle' }); + }); + + it('runs at most two graphs at once', async () => { + const h = createHarness([photoLayer('a'), photoLayer('b'), photoLayer('c')]); + for (const id of ['a', 'b', 'c']) { + h.coordinator.applyIntent(id, { controlType: 'edges', kind: 'set-type' }); + } + await settle(); + + expect(h.runFilterGraph).toHaveBeenCalledTimes(2); + expect(h.status('c')).toEqual({ status: 'queued' }); + await h.finishGraph(); + expect(h.runFilterGraph).toHaveBeenCalledTimes(3); + expect(h.graphs.map((graph) => graph.imageName)).toEqual(['b.png', 'c.png']); + }); + + it('suspends on cooldown and resumes interrupted layers on activation', async () => { + const h = createHarness([photoLayer('a'), photoLayer('b')]); + h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' }); + await settle(); + const interrupted = h.graphs[0]!; + + h.coordinator.suspend(); + expect(interrupted.signal.aborted).toBe(true); + expect(h.status()).toEqual({ status: 'queued' }); + h.coordinator.applyIntent('b', { controlType: 'edges', kind: 'set-type' }); + await settle(10_000); + expect(h.runFilterGraph).toHaveBeenCalledOnce(); + + h.coordinator.resume(); + await settle(0); + expect(h.runFilterGraph).toHaveBeenCalledTimes(3); + }); + + it('drops every job and status when the account changes', async () => { + const h = createHarness(); + h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' }); + await settle(); + const running = h.graphs[0]!; + + h.accounts.invalidate(); + expect(running.signal.aborted).toBe(true); + await h.finishGraph(); + + expect(h.derivation.commitControlMap).not.toHaveBeenCalled(); + expect(h.status()).toEqual({ status: 'idle' }); + }); + + it('drops jobs and statuses of removed layers', async () => { + const h = createHarness(); + h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' }); + h.removeLayer('a'); + h.coordinator.reconcileWithDocument(); + await settle(); + + expect(h.status()).toEqual({ status: 'idle' }); + expect(h.runFilterGraph).not.toHaveBeenCalled(); + }); + + it('settles a leftover error once the document no longer needs detection, without scheduling', async () => { + const h = createHarness(); + const untyped = h.layer(); + h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' }); + await settle(); + h.graphs.shift()!.reject(new Error('queue offline')); + await settle(0); + expect(h.status()).toMatchObject({ status: 'error' }); + + // Unrelated document changes keep the error while the map is still stale. + h.coordinator.reconcileWithDocument(); + expect(h.status()).toMatchObject({ status: 'error' }); + + // Undo of the type change. + h.setLayer(untyped); + h.coordinator.reconcileWithDocument(); + await settle(10_000); + + expect(h.status()).toEqual({ status: 'idle' }); + expect(h.runFilterGraph).toHaveBeenCalledOnce(); + }); + + it('reports a failed paint flush as source-not-ready', async () => { + const h = createHarness([ + photoLayer('a', { controlType: 'edges', detection: null, source: { bitmap: null, type: 'paint' } }), + ]); + h.flushLayerUploads.mockResolvedValue('failed'); + + h.coordinator.applyIntent('a', { detector: 'canny_edge_detection', kind: 'set-detector' }); + await settle(); + + expect(h.status()).toEqual({ error: { code: 'source-not-ready' }, status: 'error' }); + expect(h.runFilterGraph).not.toHaveBeenCalled(); + }); + + it('reports a result that cannot be preserved as apply-failed, without committing it', async () => { + const h = createHarness(); + h.makeDurable.mockRejectedValue(new Error('PATCH 500')); + h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' }); + await settle(); + await h.finishGraph(); + + expect(h.status()).toEqual({ error: { code: 'apply-failed', detail: 'PATCH 500' }, status: 'error' }); + expect(h.derivation.commitControlMap).not.toHaveBeenCalled(); + }); + + it('gives up a queued slot when its job is cancelled', async () => { + const h = createHarness([photoLayer('a'), photoLayer('b'), photoLayer('c'), photoLayer('d')]); + for (const id of ['a', 'b', 'c', 'd']) { + h.coordinator.applyIntent(id, { controlType: 'edges', kind: 'set-type' }); + } + await settle(); + expect(h.runFilterGraph).toHaveBeenCalledTimes(2); + + // `c` waits for a slot; None cancels it. + h.coordinator.applyIntent('c', { detector: null, kind: 'set-detector' }); + expect(h.status('c')).toEqual({ status: 'idle' }); + await h.finishGraph(); + + expect(h.runFilterGraph).toHaveBeenCalledTimes(3); + expect(h.graphs.map((graph) => graph.imageName)).toEqual(['b.png', 'd.png']); + }); + + it('disposes idempotently, cancelling work and releasing subscriptions', async () => { + const h = createHarness(); + const listener = vi.fn(); + h.coordinator.subscribeStatus('a', listener); + h.coordinator.applyIntent('a', { controlType: 'edges', kind: 'set-type' }); + await settle(); + const running = h.graphs[0]!; + + h.coordinator.dispose(); + h.coordinator.dispose(); + + expect(running.signal.aborted).toBe(true); + expect(h.lockListeners.size).toBe(0); + expect(h.status()).toEqual({ status: 'idle' }); + expect(h.coordinator.applyIntent('a', { kind: 'draw' })).toBe('blocked'); + expect(h.coordinator.retry('a')).toBe('missing'); + await h.finishGraph(); + expect(h.derivation.commitControlMap).not.toHaveBeenCalled(); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/ControlDetectionCoordinator.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/ControlDetectionCoordinator.ts new file mode 100644 index 00000000000..0e50919d848 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/ControlDetectionCoordinator.ts @@ -0,0 +1,590 @@ +/** + * Background detection for control layers. Commands start it (a Properties intent, Retry, resuming after cooldown); + * document changes never do, so undo and redo cannot re-run detection. One job per layer, at most + * {@link CONTROL_DETECTION_MAX_CONCURRENT} graphs per engine; results land through the engine's guarded map commit, + * which refuses while a modal operation, gesture or layer session holds the document, so a finished result waits + * and retries instead of blocking them. + */ + +import type { AccountOwnedResource, AccountScope } from '@platform/state/accountLifecycle'; +import type { CanvasControlDerivationCapability, ControlDetectionGuard } from '@workbench/canvas-engine/api'; +import type { + CanvasControlDetectionContract, + CanvasControlLayerContract, + CanvasDocumentContractV3, +} from '@workbench/canvas-engine/contracts'; +import type { LayerFlushResult } from '@workbench/canvas-engine/controllers/persistenceController'; +import type { + ControlDetectionErrorCode, + ControlDetectionStatus, + ControlLayerIntent, + ControlLayerIntentResult, + RetryControlDetectionResult, +} from '@workbench/canvas-operations/operationTypes'; +import type { BackendGraphContract } from '@workbench/graphContracts'; + +import { areJsonValuesStructurallyEqual } from '@platform/core/json'; +import { captureAccountScope, registerAccountOwnedResource } from '@platform/state/accountLifecycle'; +import { + controlDetectionInputKey, + getControlDetectionInput, + isControlMapCurrent, + lookupDocumentLeaf, + needsControlDetection, +} from '@workbench/canvas-engine/api'; +import { + createControlDetectionStatusStore, + IDLE_CONTROL_DETECTION_STATUS, + type ControlDetectionStatusStore, +} from '@workbench/canvas-operations/controlDetectionStatusStore'; +import { applyControlLayerIntent } from '@workbench/canvas-operations/controlLayerIntents'; +import { buildFilterDefaults, getFilterDefinition } from '@workbench/canvas-operations/filterGraphs'; +import { FILTER_AUTO_PROCESS_DEBOUNCE_MS } from '@workbench/canvas-operations/filterOperationSession'; + +import { ControlDetectionFailure, runControlDetection, type ControlDetectionMap } from './controlDetectionRunner'; + +export const CONTROL_DETECTION_MAX_CONCURRENT = 2; +export const CONTROL_DETECTION_COMMIT_RETRY_MS = { initial: 250, max: 2_000 } as const; +/** How long a finished result keeps waiting for the document before it reports `commit-busy`. */ +export const CONTROL_DETECTION_COMMIT_TIMEOUT_MS = 30_000; +/** Restarts after the input pixels changed under a running job (painting, an upload landing). */ +export const CONTROL_DETECTION_MAX_RESTARTS = 3; + +/** The engine's control derivation capability. */ +export type ControlDerivationPort = Pick< + CanvasControlDerivationCapability, + 'captureControlDetectionGuard' | 'commitControlLayerPatch' | 'commitControlMap' +>; + +export interface ControlDetectionCoordinatorDeps { + readonly projectId: string; + readonly derivation: ControlDerivationPort; + getDocument(): CanvasDocumentContractV3 | null; + /** Persists the layer's pending paint, waiting for an edit that holds it. */ + flushLayerUploads(layerId: string, options: { signal?: AbortSignal }): Promise; + runFilterGraph(options: { + graph: BackendGraphContract; + outputNodeId: string; + signal?: AbortSignal; + }): Promise<{ height: number; imageName: string; width: number }>; + /** Adopts a result as canvas-owned pixels; called only for results about to commit. */ + makeDurable(imageName: string): Promise; + /** Notifies when the edit lease, interaction lock or an editing session may have been released. */ + subscribeLockChanges(listener: () => void): () => void; + readonly statuses?: ControlDetectionStatusStore; + /** Account lifetime; the platform lifecycle by default. */ + readonly accounts?: { + capture(): AccountScope; + register(resource: AccountOwnedResource): () => void; + }; +} + +export interface ControlDetectionCoordinator { + applyIntent(layerId: string, intent: ControlLayerIntent): ControlLayerIntentResult; + retry(layerId: string): RetryControlDetectionResult; + getStatus(layerId: string): ControlDetectionStatus; + subscribeStatus(layerId: string, listener: () => void): () => void; + /** Cancels every job and remembers its layer, for {@link resume}. */ + suspend(): void; + /** Restarts the layers {@link suspend} interrupted that still need a map. */ + resume(): void; + /** + * Follows a document change without scheduling work: drops jobs and statuses of removed layers, and settles a + * leftover error or queued status once the layer needs no detection (an undo past the failed edit). + */ + reconcileWithDocument(): void; + dispose(): void; +} + +const INTENT_LABELS: Readonly> = { + draw: 'Draw control image', + 'replace-image': 'Replace control image', + 'set-detector': 'Change detector', + 'set-detector-settings': 'Change detector settings', + 'set-type': 'Change control type', +}; + +const PATCH_REFUSALS: Readonly< + Record< + Exclude['status'], 'committed'>, + ControlLayerIntentResult + > +> = { + busy: 'blocked', + failed: 'failed', + locked: 'locked', + missing: 'missing', + 'not-ready': 'blocked', + 'over-budget': 'over-budget', + stale: 'blocked', +}; + +const QUEUED: ControlDetectionStatus = Object.freeze({ status: 'queued' }); + +type JobOutcome = + | { kind: 'committed' | 'drop' | 'restart' } + | { kind: 'error'; code: ControlDetectionErrorCode; detail?: string }; + +interface Job { + readonly layerId: string; + readonly controller: AbortController; + /** The input key it detects (updated once a flush settles the paint); a command for the same key keeps it. */ + inputKey: string; + readonly restarts: number; + timer: ReturnType | null; + running: boolean; + /** Ends a commit retry wait early (a lock was released). */ + wake: (() => void) | null; +} + +const detectorDefaults = (detector: string): Record => { + const definition = getFilterDefinition(detector); + return definition ? buildFilterDefaults(definition) : {}; +}; + +/** Whether the detection input is the layer's own paint, whose newest strokes may not be uploaded yet. */ +const isOwnPaintInput = (layer: CanvasControlLayerContract): boolean => + layer.original === undefined && !layer.filter && layer.source.type === 'paint'; + +/** + * Whether a run could produce a map: the document says the map is stale, or the input is the layer's own paint + * whose pixels may still be in flight (a flush settles which). + */ +const wantsDetection = (layer: CanvasControlLayerContract): boolean => + needsControlDetection(layer) || + (layer.controlType !== undefined && !!layer.detection && !isControlMapCurrent(layer) && isOwnPaintInput(layer)); + +/** + * Whether `after` differs from `before` only by its own paint landing (an upload, or strokes on the input): same type + * and detector, and both detect their own paint source. An image swap (undo or redo of a replaced image) does not + * qualify. + */ +const onlyOwnPaintChanged = (before: CanvasControlLayerContract, after: CanvasControlLayerContract): boolean => + isOwnPaintInput(before) && + isOwnPaintInput(after) && + before.controlType === after.controlType && + areJsonValuesStructurallyEqual(before.detection ?? null, after.detection ?? null); + +const failureDetail = (error: unknown): string | undefined => { + const detail = error instanceof Error ? error.message : typeof error === 'string' ? error : undefined; + return detail || undefined; +}; + +const errorOutcome = (code: ControlDetectionErrorCode, detail?: string): JobOutcome => + detail ? { code, detail, kind: 'error' } : { code, kind: 'error' }; + +let coordinatorCount = 0; + +export const createControlDetectionCoordinator = ( + deps: ControlDetectionCoordinatorDeps +): ControlDetectionCoordinator => { + const statuses = deps.statuses ?? createControlDetectionStatusStore(); + const accounts = deps.accounts ?? { capture: captureAccountScope, register: registerAccountOwnedResource }; + const jobs = new Map(); + const interrupted = new Set(); + let suspended = false; + let disposed = false; + + let runningGraphs = 0; + const slotWaiters: (() => void)[] = []; + const acquireSlot = (signal: AbortSignal): Promise => { + signal.throwIfAborted(); + if (runningGraphs < CONTROL_DETECTION_MAX_CONCURRENT) { + runningGraphs += 1; + return Promise.resolve(); + } + return new Promise((resolve, reject) => { + const grant = (): void => { + signal.removeEventListener('abort', onAbort); + resolve(); + }; + const onAbort = (): void => { + slotWaiters.splice(slotWaiters.indexOf(grant), 1); + reject(signal.reason); + }; + slotWaiters.push(grant); + signal.addEventListener('abort', onAbort, { once: true }); + }); + }; + /** Hands the slot straight to the next waiter, so the count never dips below a queued job. */ + const releaseSlot = (): void => { + const next = slotWaiters.shift(); + if (next) { + next(); + } else { + runningGraphs -= 1; + } + }; + + const readLayer = (layerId: string): CanvasControlLayerContract | null => { + const layer = lookupDocumentLeaf(deps.getDocument(), layerId)?.layer; + return layer?.type === 'control' ? layer : null; + }; + + const cancelJob = (layerId: string): void => { + const job = jobs.get(layerId); + if (!job) { + return; + } + jobs.delete(layerId); + if (job.timer !== null) { + clearTimeout(job.timer); + job.timer = null; + } + job.controller.abort(); + }; + + const waitForRetry = (job: Job, delayMs: number, signal: AbortSignal): Promise => + new Promise((resolve, reject) => { + const settle = (): void => { + clearTimeout(timer); + job.wake = null; + signal.removeEventListener('abort', onAbort); + }; + const done = (): void => { + settle(); + resolve(); + }; + const onAbort = (): void => { + settle(); + reject(signal.reason); + }; + const timer = setTimeout(done, delayMs); + job.wake = done; + signal.addEventListener('abort', onAbort, { once: true }); + }); + + /** + * Whether `guard` still describes the layer. The layer's own paint landing under the job (an upload, more strokes) + * runs again; anything else (removal, a document swap, an image swapped by undo or redo, another command) drops + * the result, so history never re-runs detection. + */ + const checkGuard = ( + guard: ControlDetectionGuard, + recipe: CanvasControlLayerContract + ): 'current' | 'drop' | 'restart' => { + const current = deps.derivation.captureControlDetectionGuard(guard.layerId); + if (!current || current.documentRevision !== guard.documentRevision) { + return 'drop'; + } + if (current.inputKey === guard.inputKey) { + return 'current'; + } + const layer = readLayer(guard.layerId); + return layer && needsControlDetection(layer) && onlyOwnPaintChanged(recipe, layer) ? 'restart' : 'drop'; + }; + + const commitWithRetry = async ( + job: Job, + guard: ControlDetectionGuard, + recipe: CanvasControlLayerContract, + detection: CanvasControlDetectionContract, + map: ControlDetectionMap, + signal: AbortSignal + ): Promise => { + const startedAt = Date.now(); + let delay: number = CONTROL_DETECTION_COMMIT_RETRY_MS.initial; + for (;;) { + const result = await deps.derivation.commitControlMap({ + detection, + guard, + image: map.image, + rect: map.rect, + signal, + }); + switch (result.status) { + case 'committed': + return { kind: 'committed' }; + case 'stale': { + const check = checkGuard(guard, recipe); + return { kind: check === 'restart' ? 'restart' : 'drop' }; + } + case 'missing': + case 'aborted': + signal.throwIfAborted(); + return { kind: 'drop' }; + case 'over-budget': + return errorOutcome('over-budget', result.message); + case 'unrepresentable': + return errorOutcome('unrepresentable', result.message); + case 'failed': + return errorOutcome('apply-failed', result.message); + case 'busy': + case 'locked': + case 'not-ready': { + const remaining = CONTROL_DETECTION_COMMIT_TIMEOUT_MS - (Date.now() - startedAt); + if (remaining <= 0) { + return errorOutcome('commit-busy', result.message); + } + statuses.set(job.layerId, QUEUED); + await waitForRetry(job, Math.min(delay, remaining), signal); + delay = Math.min(delay * 2, CONTROL_DETECTION_COMMIT_RETRY_MS.max); + break; + } + } + } + }; + + const detect = async (job: Job, signal: AbortSignal): Promise => { + let layer = readLayer(job.layerId); + if (!layer || !wantsDetection(layer)) { + return { kind: 'drop' }; + } + if (isOwnPaintInput(layer)) { + const flushed = await deps.flushLayerUploads(job.layerId, { signal }); + signal.throwIfAborted(); + if (flushed === 'aborted') { + return { kind: 'drop' }; + } + if (flushed === 'failed') { + return errorOutcome('source-not-ready'); + } + layer = readLayer(job.layerId); + } + if (!layer || !needsControlDetection(layer) || !layer.detection) { + return { kind: 'drop' }; + } + const input = getControlDetectionInput(layer); + const guard = deps.derivation.captureControlDetectionGuard(job.layerId); + if (!input || !guard) { + return { kind: 'drop' }; + } + job.inputKey = guard.inputKey; + const recipe = layer; + const { detection } = layer; + + await acquireSlot(signal); + let map: ControlDetectionMap; + try { + statuses.set(job.layerId, { detector: detection.type, status: 'processing' }); + map = await runControlDetection({ detection, input, runFilterGraph: deps.runFilterGraph, signal }); + } finally { + releaseSlot(); + } + + let check = checkGuard(guard, recipe); + if (check !== 'current') { + return { kind: check }; + } + try { + await deps.makeDurable(map.image.imageName); + } catch (error) { + signal.throwIfAborted(); + return errorOutcome('apply-failed', failureDetail(error)); + } + signal.throwIfAborted(); + check = checkGuard(guard, recipe); + if (check !== 'current') { + return { kind: check }; + } + return commitWithRetry(job, guard, recipe, detection, map, signal); + }; + + const schedule = (layerId: string, delayMs: number, layer: CanvasControlLayerContract, restarts = 0): void => { + if (disposed) { + return; + } + if (suspended) { + interrupted.add(layerId); + statuses.set(layerId, QUEUED); + return; + } + const inputKey = controlDetectionInputKey(layer); + const existing = jobs.get(layerId); + if (existing?.running && existing.inputKey === inputKey) { + return; + } + cancelJob(layerId); + const job: Job = { + controller: new AbortController(), + inputKey, + layerId, + restarts, + running: false, + timer: null, + wake: null, + }; + jobs.set(layerId, job); + statuses.set(layerId, QUEUED); + job.timer = setTimeout(() => { + job.timer = null; + void run(job); + }, delayMs); + }; + + const finish = (job: Job, outcome: JobOutcome): void => { + if (jobs.get(job.layerId) !== job) { + return; + } + jobs.delete(job.layerId); + if (outcome.kind === 'error') { + statuses.set( + job.layerId, + outcome.detail + ? { error: { code: outcome.code, detail: outcome.detail }, status: 'error' } + : { error: { code: outcome.code }, status: 'error' } + ); + return; + } + const layer = outcome.kind === 'restart' ? readLayer(job.layerId) : null; + if (layer && job.restarts < CONTROL_DETECTION_MAX_RESTARTS) { + schedule(job.layerId, FILTER_AUTO_PROCESS_DEBOUNCE_MS, layer, job.restarts + 1); + return; + } + statuses.set(job.layerId, IDLE_CONTROL_DETECTION_STATUS); + }; + + const run = async (job: Job): Promise => { + job.running = true; + const signal = AbortSignal.any([job.controller.signal, accounts.capture().signal]); + let outcome: JobOutcome; + try { + outcome = await detect(job, signal); + } catch (error) { + if (signal.aborted) { + // A cancel already settled the status; an account switch alone leaves nothing to show. + if (jobs.get(job.layerId) === job) { + jobs.delete(job.layerId); + statuses.set(job.layerId, IDLE_CONTROL_DETECTION_STATUS); + } + return; + } + outcome = + error instanceof ControlDetectionFailure + ? errorOutcome(error.code, error.detail) + : errorOutcome('process-failed', failureDetail(error)); + } + finish(job, outcome); + }; + + const cancelAll = (): void => { + for (const layerId of jobs.keys()) { + cancelJob(layerId); + } + }; + + const unsubscribeLocks = deps.subscribeLockChanges(() => { + for (const job of jobs.values()) { + job.wake?.(); + } + }); + const unregisterAccount = accounts.register({ + clear: () => { + cancelAll(); + interrupted.clear(); + statuses.clear(); + }, + name: `canvas-control-detection:${deps.projectId}:${String((coordinatorCount += 1))}`, + }); + + const settleWithoutDetection = (layerId: string): void => { + cancelJob(layerId); + interrupted.delete(layerId); + statuses.set(layerId, IDLE_CONTROL_DETECTION_STATUS); + }; + + return { + applyIntent: (layerId, intent) => { + if (disposed) { + return 'blocked'; + } + const layer = readLayer(layerId); + if (!layer) { + return 'missing'; + } + const next = applyControlLayerIntent(layer, intent, { detectorDefaults }); + if (next === 'no-detection-input') { + return next; + } + if (next === layer) { + return 'unchanged'; + } + const result = deps.derivation.commitControlLayerPatch({ + coalesceKey: + intent.kind === 'set-detector-settings' + ? `${layerId}:${layer.detection?.type ?? 'none'}:settings` + : undefined, + expected: layer, + label: INTENT_LABELS[intent.kind], + next, + }); + if (result.status !== 'committed') { + return PATCH_REFUSALS[result.status]; + } + if (wantsDetection(next)) { + schedule(layerId, FILTER_AUTO_PROCESS_DEBOUNCE_MS, next); + } else { + settleWithoutDetection(layerId); + } + return 'updated'; + }, + dispose: () => { + if (disposed) { + return; + } + disposed = true; + cancelAll(); + interrupted.clear(); + unsubscribeLocks(); + unregisterAccount(); + statuses.clear(); + }, + getStatus: statuses.get, + reconcileWithDocument: () => { + const tracked = new Set([...jobs.keys(), ...interrupted, ...statuses.layerIds()]); + for (const layerId of tracked) { + const layer = readLayer(layerId); + if (!layer) { + settleWithoutDetection(layerId); + continue; + } + // A job settles its own status; only one left behind by a failed or interrupted run is stale here. + const { status } = statuses.get(layerId); + if (!jobs.has(layerId) && (status === 'error' || status === 'queued') && !wantsDetection(layer)) { + interrupted.delete(layerId); + statuses.set(layerId, IDLE_CONTROL_DETECTION_STATUS); + } + } + }, + resume: () => { + if (disposed || !suspended) { + return; + } + suspended = false; + const layerIds = [...interrupted]; + interrupted.clear(); + for (const layerId of layerIds) { + const layer = readLayer(layerId); + if (layer && wantsDetection(layer)) { + schedule(layerId, 0, layer); + } else { + statuses.set(layerId, IDLE_CONTROL_DETECTION_STATUS); + } + } + }, + retry: (layerId) => { + const layer = disposed ? null : readLayer(layerId); + if (!layer) { + return 'missing'; + } + if (!wantsDetection(layer)) { + settleWithoutDetection(layerId); + return 'not-needed'; + } + schedule(layerId, 0, layer); + return 'scheduled'; + }, + subscribeStatus: statuses.subscribe, + suspend: () => { + if (disposed || suspended) { + return; + } + suspended = true; + for (const layerId of jobs.keys()) { + interrupted.add(layerId); + cancelJob(layerId); + statuses.set(layerId, QUEUED); + } + }, + }; +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/api.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/api.ts index e80771b31f5..db8252db1b0 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/api.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/api.ts @@ -25,8 +25,14 @@ export { } from './createFromBbox'; export { getCreateFromBboxNotice, type CreateFromBboxNotice } from './createFromBboxNotice'; export type { + ControlDetectionError, + ControlDetectionErrorCode, + ControlDetectionStatus, + ControlLayerIntent, + ControlLayerIntentResult, FilterOperationSessionState, FilterSessionErrorCode, + RetryControlDetectionResult, SamInput, SamModel, SamSessionError, @@ -34,6 +40,7 @@ export type { SamSessionSnapshot, } from './operationTypes'; export { getCanvasOperations } from './operationAccess'; +export { IDLE_CONTROL_DETECTION_STATUS } from './controlDetectionStatusStore'; export { getCanvasEngine } from './engineRegistry'; export { saveCanvasToGallery, type CanvasGallerySaveRegion } from './saveCanvasToGallery'; export { diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/contracts.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/contracts.ts index af4ae913e8d..d14cb17e978 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/contracts.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/contracts.ts @@ -18,6 +18,9 @@ import type { ComposeForGenerationResult, } from '@workbench/canvas-operations/generationComposite'; import type { + ControlDetectionStatus, + ControlLayerIntent, + ControlLayerIntentResult, FilterCommitTarget, FilterOperationSessionState, LayerFilterSettings, @@ -25,6 +28,7 @@ import type { SamModel, SamSessionError, SamSessionSnapshot, + RetryControlDetectionResult, } from '@workbench/canvas-operations/operationTypes'; import type { BackendGraphContract } from '@workbench/graphContracts'; import type { WorkbenchState } from '@workbench/projectContracts'; @@ -86,6 +90,12 @@ export interface CanvasOperationCapability { ): Promise; resetSelectObjectSession(): CanvasOperationMutationResult; cancelSelectObjectSession(): void; + /** Applies a Properties panel edit to a control layer, then detects its map in the background when needed. */ + applyControlLayerIntent(layerId: string, intent: ControlLayerIntent): ControlLayerIntentResult; + /** Runs detection again for a layer whose map is stale (after a failure, or undo back to a stale state). */ + retryControlDetection(layerId: string): RetryControlDetectionResult; + getControlDetectionStatus(layerId: string): ControlDetectionStatus; + subscribeControlDetectionStatus(layerId: string, listener: () => void): () => void; } /** Private composition shape; mutable stores and controller never cross the Canvas interface. */ diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionRunner.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionRunner.test.ts new file mode 100644 index 00000000000..ecd79389115 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionRunner.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { ControlDetectionFailure, runControlDetection } from './controlDetectionRunner'; +import { FILTER_NODE_ID } from './filterGraphs'; + +const canny = { settings: { high_threshold: 200, low_threshold: 100 }, type: 'canny_edge_detection' }; +const output = (width: number, height: number) => ({ height, imageName: 'map.png', width }); + +describe('runControlDetection', () => { + it('builds the graph on the persisted image at its content size and keeps the map in its frame', async () => { + const runFilterGraph = vi.fn(() => Promise.resolve(output(30, 20))); + const result = await runControlDetection({ + detection: canny, + input: { bitmap: { height: 20, imageName: 'paint.png', width: 30 }, offset: { x: -4, y: 6 }, type: 'paint' }, + runFilterGraph, + signal: new AbortController().signal, + }); + + const [{ graph, outputNodeId }] = runFilterGraph.mock.calls[0] as unknown as [ + { graph: { nodes: Record> }; outputNodeId: string }, + ]; + // Processed over a white input-sized background, with the input's alpha restored on the output. + expect(graph.nodes[FILTER_NODE_ID]).toMatchObject({ high_threshold: 200, type: 'canny_edge_detection' }); + expect(graph.nodes.control_filter_background).toMatchObject({ height: 20, width: 30 }); + expect(graph.nodes.control_filter_flatten).toMatchObject({ image: { image_name: 'paint.png' } }); + expect(graph.nodes.control_filter_source_alpha).toMatchObject({ image: { image_name: 'paint.png' } }); + expect(outputNodeId).toBe('control_filter_restore_alpha'); + expect(result).toEqual({ + image: { height: 20, imageName: 'map.png', width: 30 }, + rect: { height: 20, width: 30, x: -4, y: 6 }, + }); + }); + + it('places an image input at its origin', async () => { + const result = await runControlDetection({ + detection: canny, + input: { image: { height: 48, imageName: 'photo.png', width: 64 }, type: 'image' }, + runFilterGraph: () => Promise.resolve(output(64, 48)), + signal: new AbortController().signal, + }); + + expect(result.rect).toEqual({ height: 48, width: 64, x: 0, y: 0 }); + }); + + it.each([ + [ + 'unknown-detector', + { detection: { settings: {}, type: 'retired_detector' }, output: output(64, 64), image: true }, + ], + ['source-not-ready', { detection: canny, output: output(64, 64), image: false }], + ['output-dimensions', { detection: canny, output: output(32, 32), image: true }], + ] as const)('reports %s', async (code, scenario) => { + const runFilterGraph = vi.fn(() => Promise.resolve(scenario.output)); + const run = runControlDetection({ + detection: scenario.detection, + input: scenario.image + ? { image: { height: 64, imageName: 'photo.png', width: 64 }, type: 'image' } + : { bitmap: null, type: 'paint' }, + runFilterGraph, + signal: new AbortController().signal, + }); + + await expect(run).rejects.toBeInstanceOf(ControlDetectionFailure); + await expect(run).rejects.toMatchObject({ code }); + }); + + it('wraps graph failures with their message and lets aborts through', async () => { + const input = { image: { height: 64, imageName: 'photo.png', width: 64 }, type: 'image' } as const; + await expect( + runControlDetection({ + detection: canny, + input, + runFilterGraph: () => Promise.reject(new Error('queue offline')), + signal: new AbortController().signal, + }) + ).rejects.toMatchObject({ code: 'process-failed', detail: 'queue offline' }); + + const controller = new AbortController(); + const run = runControlDetection({ + detection: canny, + input, + runFilterGraph: () => { + controller.abort(); + return Promise.reject(new Error('cancelled')); + }, + signal: controller.signal, + }); + await expect(run).rejects.toMatchObject({ name: 'AbortError' }); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionRunner.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionRunner.ts new file mode 100644 index 00000000000..2cc3628667c --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionRunner.ts @@ -0,0 +1,122 @@ +/** Runs one detector graph on a control layer's detection input and places the map in the input's local frame. */ + +import type { + CanvasControlDetectionContract, + CanvasImageRef, + CanvasPixelSourceContract, +} from '@workbench/canvas-engine/contracts'; +import type { Rect } from '@workbench/canvas-engine/types'; +import type { ControlDetectionErrorCode } from '@workbench/canvas-operations/operationTypes'; +import type { BackendGraphContract } from '@workbench/graphContracts'; + +import { buildFilterGraph, getFilterDefinition } from '@workbench/canvas-operations/filterGraphs'; +import { + LayerFilterOutputDimensionError, + resolveFilterOutputRect, +} from '@workbench/canvas-operations/layerFilterRunner'; + +/** + * An expected detection failure with its typed code; `detail` is raw backend or engine text, never copy. Aborts + * propagate as `AbortError` instead. + */ +export class ControlDetectionFailure extends Error { + readonly code: ControlDetectionErrorCode; + readonly detail: string | undefined; + + constructor(code: ControlDetectionErrorCode, detail?: string) { + super(detail ?? code); + this.name = 'ControlDetectionFailure'; + this.code = code; + this.detail = detail; + } +} + +export interface ControlDetectionRunOptions { + input: CanvasPixelSourceContract; + detection: CanvasControlDetectionContract; + signal: AbortSignal; + runFilterGraph(options: { + graph: BackendGraphContract; + outputNodeId: string; + signal?: AbortSignal; + }): Promise<{ height: number; imageName: string; width: number }>; +} + +export interface ControlDetectionMap { + image: CanvasImageRef; + /** Where the map sits in the detection input's local frame. */ + rect: Rect; +} + +/** The persisted image and content rect of a detection input; null for a paint source without uploaded pixels. */ +export const getControlInputImage = (input: CanvasPixelSourceContract): { imageName: string; rect: Rect } | null => { + if (input.type === 'image') { + return { + imageName: input.image.imageName, + rect: { height: input.image.height, width: input.image.width, x: 0, y: 0 }, + }; + } + if (!input.bitmap) { + return null; + } + const offset = input.offset ?? { x: 0, y: 0 }; + return { + imageName: input.bitmap.imageName, + rect: { height: input.bitmap.height, width: input.bitmap.width, x: offset.x, y: offset.y }, + }; +}; + +const isAbortError = (error: unknown): boolean => error instanceof DOMException && error.name === 'AbortError'; + +/** Rejects as soon as `signal` aborts, so a cancelled run frees its concurrency slot without waiting on the queue. */ +const raceAbort = (promise: Promise, signal: AbortSignal): Promise => + new Promise((resolve, reject) => { + const onAbort = (): void => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort)); + }); + +export const runControlDetection = async (options: ControlDetectionRunOptions): Promise => { + options.signal.throwIfAborted(); + const { detection } = options; + if (!getFilterDefinition(detection.type)) { + throw new ControlDetectionFailure('unknown-detector', detection.type); + } + const source = getControlInputImage(options.input); + if (!source) { + throw new ControlDetectionFailure('source-not-ready'); + } + const built = buildFilterGraph(detection.type, source.imageName, detection.settings, { + height: source.rect.height, + preserveTransparency: true, + width: source.rect.width, + }); + let output: { height: number; imageName: string; width: number }; + try { + output = await raceAbort( + options.runFilterGraph({ graph: built.graph, outputNodeId: built.outputNodeId, signal: options.signal }), + options.signal + ); + } catch (error) { + options.signal.throwIfAborted(); + if (isAbortError(error)) { + throw error; + } + throw new ControlDetectionFailure('process-failed', error instanceof Error ? error.message : String(error)); + } + options.signal.throwIfAborted(); + try { + const rect = resolveFilterOutputRect({ + filterType: detection.type, + output, + settings: detection.settings, + source: source.rect, + }); + return { image: { height: output.height, imageName: output.imageName, width: output.width }, rect }; + } catch (error) { + if (error instanceof LayerFilterOutputDimensionError) { + throw new ControlDetectionFailure('output-dimensions', error.message); + } + throw error; + } +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionStatusStore.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionStatusStore.test.ts new file mode 100644 index 00000000000..0e7a90aa731 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionStatusStore.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { createControlDetectionStatusStore } from './controlDetectionStatusStore'; + +describe('controlDetectionStatusStore', () => { + it('notifies only the changed layer and keeps snapshots stable between changes', () => { + const store = createControlDetectionStatusStore(); + const first = vi.fn(); + const second = vi.fn(); + store.subscribe('a', first); + store.subscribe('b', second); + + store.set('a', { detector: 'canny_edge_detection', status: 'processing' }); + const snapshot = store.get('a'); + store.set('a', { detector: 'canny_edge_detection', status: 'processing' }); + + expect(first).toHaveBeenCalledOnce(); + expect(second).not.toHaveBeenCalled(); + expect(store.get('a')).toBe(snapshot); + expect(store.get('b')).toEqual({ status: 'idle' }); + }); + + it('forgets idle layers and clears every layer with a notification', () => { + const store = createControlDetectionStatusStore(); + const listener = vi.fn(); + store.set('a', { status: 'queued' }); + store.set('b', { error: { code: 'process-failed', detail: 'boom' }, status: 'error' }); + store.subscribe('b', listener); + + store.set('a', { status: 'idle' }); + expect(store.layerIds()).toEqual(['b']); + + store.clear(); + expect(listener).toHaveBeenCalledOnce(); + expect(store.layerIds()).toEqual([]); + expect(store.get('b')).toEqual({ status: 'idle' }); + }); + + it('stops notifying after unsubscribe', () => { + const store = createControlDetectionStatusStore(); + const listener = vi.fn(); + const unsubscribe = store.subscribe('a', listener); + + unsubscribe(); + store.set('a', { status: 'queued' }); + + expect(listener).not.toHaveBeenCalled(); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionStatusStore.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionStatusStore.ts new file mode 100644 index 00000000000..2dcedfad338 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlDetectionStatusStore.ts @@ -0,0 +1,82 @@ +/** Per-layer detection status for control layers; React-free and `useSyncExternalStore`-compatible. */ + +import type { ControlDetectionStatus } from '@workbench/canvas-operations/operationTypes'; + +export interface ControlDetectionStatusStore { + /** The layer's status; a layer without an entry is idle. Snapshots are stable until the next `set`. */ + get(layerId: string): ControlDetectionStatus; + /** Setting idle removes the entry, so the store only holds layers with something to show. */ + set(layerId: string, status: ControlDetectionStatus): void; + subscribe(layerId: string, listener: () => void): () => void; + /** Layer ids that currently hold a non-idle status. */ + layerIds(): string[]; + /** Resets every layer to idle, notifying their listeners. */ + clear(): void; +} + +export const IDLE_CONTROL_DETECTION_STATUS: ControlDetectionStatus = Object.freeze({ status: 'idle' }); + +const sameStatus = (a: ControlDetectionStatus, b: ControlDetectionStatus): boolean => { + if (a.status !== b.status) { + return false; + } + if (a.status === 'processing' && b.status === 'processing') { + return a.detector === b.detector; + } + if (a.status === 'error' && b.status === 'error') { + return a.error.code === b.error.code && a.error.detail === b.error.detail; + } + return true; +}; + +export const createControlDetectionStatusStore = (): ControlDetectionStatusStore => { + const statuses = new Map(); + const listeners = new Map void>>(); + + const notify = (layerId: string): void => { + for (const listener of listeners.get(layerId) ?? []) { + listener(); + } + }; + + const set = (layerId: string, status: ControlDetectionStatus): void => { + const previous = statuses.get(layerId) ?? IDLE_CONTROL_DETECTION_STATUS; + if (sameStatus(previous, status)) { + return; + } + if (status.status === 'idle') { + statuses.delete(layerId); + } else { + statuses.set(layerId, status); + } + notify(layerId); + }; + + return { + clear: () => { + const ids = [...statuses.keys()]; + statuses.clear(); + for (const layerId of ids) { + notify(layerId); + } + }, + get: (layerId) => statuses.get(layerId) ?? IDLE_CONTROL_DETECTION_STATUS, + layerIds: () => [...statuses.keys()], + set, + subscribe: (layerId, listener) => { + let layerListeners = listeners.get(layerId); + if (!layerListeners) { + layerListeners = new Set(); + listeners.set(layerId, layerListeners); + } + layerListeners.add(listener); + return () => { + const current = listeners.get(layerId); + current?.delete(listener); + if (current?.size === 0) { + listeners.delete(layerId); + } + }; + }, + }; +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.test.ts new file mode 100644 index 00000000000..d3946ed279f --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.test.ts @@ -0,0 +1,235 @@ +import type { CanvasControlLayerContract } from '@workbench/canvas-engine/contracts'; + +import { createControlLayer, needsControlDetection } from '@workbench/canvas-engine/api'; +import { describe, expect, it } from 'vitest'; + +import { applyControlLayerIntent } from './controlLayerIntents'; + +const image = (imageName: string, size = 64) => ({ height: size, imageName, width: size }); +const photo = { image: image('photo.png'), type: 'image' } as const; +const mapSource = { bitmap: image('map.png'), offset: { x: 0, y: 0 }, type: 'paint' } as const; +const depth = { settings: { model_size: 'small_v2' }, type: 'depth_anything_depth_estimation' }; +const options = { + detectorDefaults: (detector: string): Record => + ({ + canny_edge_detection: { high_threshold: 200, low_threshold: 100 }, + depth_anything_depth_estimation: { model_size: 'small_v2' }, + dw_openpose_detection: { draw_body: true }, + hed_edge_detection: { scribble: false }, + normal_map: {}, + pidi_edge_detection: { quantize_edges: false, scribble: false }, + })[detector] ?? {}, +}; +const SHIFTED = { rotation: 0, scaleX: 1, scaleY: 1, x: 40, y: 8 }; + +const layer = (patch: Partial = {}): CanvasControlLayerContract => ({ + ...createControlLayer({ id: 'c1', name: 'Control Layer 1' }), + adapter: { + beginEndStepPct: [0, 1], + controlMode: 'balanced', + kind: 'controlnet', + model: 'explicit-model', + weight: 1, + }, + ...patch, +}); +/** A depth layer whose photo was detected into a map. */ +const detected = (patch: Partial = {}) => + layer({ controlType: 'depth', detection: depth, filter: depth, original: photo, source: mapSource, ...patch }); + +const apply = (target: CanvasControlLayerContract, intent: Parameters[1]) => { + const next = applyControlLayerIntent(target, intent, options); + if (next === 'no-detection-input') { + throw new Error('unexpected refusal'); + } + return next; +}; + +describe('applyControlLayerIntent: set-type', () => { + it('types an image with the default detector and returns to automatic model choice', () => { + const next = apply(layer({ source: photo }), { controlType: 'depth', kind: 'set-type' }); + + expect(next.controlType).toBe('depth'); + expect(next.detection).toEqual(depth); + expect(next.adapter.model).toBeNull(); + expect(next.source).toEqual(photo); + expect(needsControlDetection(next)).toBe(true); + }); + + it('chooses None for an empty layer and for a map without an original', () => { + expect(apply(layer(), { controlType: 'depth', kind: 'set-type' }).detection).toBeNull(); + const legacyMap = layer({ filter: depth, source: mapSource }); + const typed = apply(legacyMap, { controlType: 'depth', kind: 'set-type' }); + + expect(typed.detection).toBeNull(); + expect(typed.source).toEqual(mapSource); + }); + + it('keeps a detector the new type also offers, settings included', () => { + const custom = { settings: { quantize_edges: true, scribble: true }, type: 'pidi_edge_detection' }; + const next = apply(layer({ controlType: 'softedge', detection: custom, source: photo }), { + controlType: 'sketch', + kind: 'set-type', + }); + + expect(next.detection).toBe(custom); + }); + + it("switches to the new type's default and keeps the original for re-detection", () => { + const next = apply(detected(), { controlType: 'pose', kind: 'set-type' }); + + expect(next.detection).toEqual({ settings: { draw_body: true }, type: 'dw_openpose_detection' }); + expect(next.original).toEqual(photo); + expect(needsControlDetection(next)).toBe(true); + }); + + it('restores the original for a type without a default detector', () => { + const next = apply(detected({ originalTransform: SHIFTED }), { controlType: 'tile', kind: 'set-type' }); + + expect(next).toMatchObject({ controlType: 'tile', detection: null, source: photo }); + expect(next.transform).toMatchObject({ x: 40, y: 8 }); + expect(next.original).toBeUndefined(); + expect(next.filter).toBeUndefined(); + }); + + it('removes the type and every derivation field, restoring the original first', () => { + const next = apply(detected(), { controlType: null, kind: 'set-type' }); + + expect(next.source).toEqual(photo); + for (const key of ['controlType', 'detection', 'original', 'originalTransform', 'filter'] as const) { + expect(next).not.toHaveProperty(key); + } + expect(next.adapter.model).toBe('explicit-model'); + }); + + it('renames in the same patch and reports an unchanged type as the same object', () => { + const typed = detected(); + + expect(apply(typed, { controlType: 'depth', kind: 'set-type' })).toBe(typed); + const renamed = apply(typed, { controlType: 'depth', kind: 'set-type', name: 'Depth 1' }); + expect(renamed.name).toBe('Depth 1'); + expect(renamed.adapter.model).toBe('explicit-model'); + expect(apply(layer({ source: photo }), { controlType: 'depth', kind: 'set-type', name: 'Depth 1' }).name).toBe( + 'Depth 1' + ); + }); +}); + +describe('applyControlLayerIntent: set-detector', () => { + it("starts from the detector's defaults merged with the type's fixed settings", () => { + const next = apply(layer({ controlType: 'sketch', detection: null, source: photo }), { + detector: 'pidi_edge_detection', + kind: 'set-detector', + }); + + expect(next.detection).toEqual({ + settings: { quantize_edges: false, scribble: true }, + type: 'pidi_edge_detection', + }); + }); + + it('keeps the original so the new detector runs on it', () => { + const next = apply(detected(), { detector: 'normal_map', kind: 'set-detector' }); + + expect(next.detection).toEqual({ settings: {}, type: 'normal_map' }); + expect(next.original).toEqual(photo); + expect(needsControlDetection(next)).toBe(true); + }); + + it('None restores the original', () => { + const next = apply(detected(), { detector: null, kind: 'set-detector' }); + + expect(next).toMatchObject({ detection: null, source: photo }); + expect(next.original).toBeUndefined(); + }); + + it('refuses a detector for a map without an original', () => { + expect( + applyControlLayerIntent( + layer({ controlType: 'depth', detection: null, filter: depth, source: mapSource }), + { detector: 'normal_map', kind: 'set-detector' }, + options + ) + ).toBe('no-detection-input'); + }); + + it('leaves the same detector and an untyped layer unchanged', () => { + const typed = detected(); + const untyped = layer({ source: photo }); + + expect(apply(typed, { detector: depth.type, kind: 'set-detector' })).toBe(typed); + expect(apply(untyped, { detector: depth.type, kind: 'set-detector' })).toBe(untyped); + }); +}); + +describe('applyControlLayerIntent: set-detector-settings', () => { + it('changes only the settings and keeps fixed ones', () => { + const sketch = layer({ + controlType: 'sketch', + detection: { settings: { quantize_edges: false, scribble: true }, type: 'pidi_edge_detection' }, + source: photo, + }); + const next = apply(sketch, { kind: 'set-detector-settings', settings: { quantize_edges: true, scribble: false } }); + + expect(next).toEqual({ + ...sketch, + detection: { settings: { quantize_edges: true, scribble: true }, type: 'pidi_edge_detection' }, + }); + }); + + it('leaves a layer without a detector, or with equal settings, unchanged', () => { + const none = layer({ controlType: 'depth', detection: null, source: photo }); + const typed = detected(); + + expect(apply(none, { kind: 'set-detector-settings', settings: { a: 1 } })).toBe(none); + expect(apply(typed, { kind: 'set-detector-settings', settings: { model_size: 'small_v2' } })).toBe(typed); + }); +}); + +describe('applyControlLayerIntent: replace-image and draw', () => { + const upload = image('upload.png', 128); + + it('replaces the source, drops derived pixels and stales the map', () => { + const next = apply(detected({ originalTransform: SHIFTED }), { + image: upload, + kind: 'replace-image', + transform: SHIFTED, + }); + + expect(next.source).toEqual({ image: upload, type: 'image' }); + expect(next.transform).toEqual(SHIFTED); + for (const key of ['original', 'originalTransform', 'filter'] as const) { + expect(next).not.toHaveProperty(key); + } + expect(next.detection).toEqual(depth); + expect(needsControlDetection(next)).toBe(true); + }); + + it("gives a first image the type's default detector when None was chosen only for lack of pixels", () => { + const next = apply(layer({ controlType: 'depth', detection: null }), { image: upload, kind: 'replace-image' }); + + expect(next.detection).toEqual(depth); + }); + + it('keeps None when the layer already had an image, or the type has no default', () => { + expect( + apply(layer({ controlType: 'depth', detection: null, source: photo }), { image: upload, kind: 'replace-image' }) + .detection + ).toBeNull(); + expect( + apply(layer({ controlType: 'sketch', detection: null }), { image: upload, kind: 'replace-image' }).detection + ).toBeNull(); + expect(apply(layer(), { image: upload, kind: 'replace-image' })).not.toHaveProperty('detection'); + }); + + it('draw empties the image and uses it as-is', () => { + const next = apply(detected(), { kind: 'draw' }); + + expect(next.source).toEqual({ bitmap: null, type: 'paint' }); + expect(next.detection).toBeNull(); + for (const key of ['original', 'originalTransform', 'filter'] as const) { + expect(next).not.toHaveProperty(key); + } + expect(apply(layer({ source: photo }), { kind: 'draw' })).not.toHaveProperty('detection'); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.ts new file mode 100644 index 00000000000..4d1d84f9a5f --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.ts @@ -0,0 +1,192 @@ +/** + * Pure rules for the control layer Properties panel: each intent maps the live layer to the next one. Detection + * itself never runs here; the coordinator schedules it when the result needs a fresh map. + */ + +import type { + CanvasControlDetectionContract, + CanvasControlLayerContract, + CanvasLayerSourceContract, + ControlType, +} from '@workbench/canvas-engine/contracts'; +import type { ControlLayerIntent } from '@workbench/canvas-operations/operationTypes'; + +import { areJsonValuesStructurallyEqual } from '@platform/core/json'; +import { getControlDetectionInput, restoreControlOriginal } from '@workbench/canvas-engine/api'; +import { CONTROL_TYPE_SPECS } from '@workbench/controlTypes'; + +export interface ControlLayerIntentOptions { + /** A detector's default settings; an unknown detector has none. */ + detectorDefaults(detector: string): Record; +} + +const hasPixels = (source: CanvasLayerSourceContract | null): boolean => + source !== null && (source.type === 'image' || (source.type === 'paint' && source.bitmap !== null)); + +/** Whether there are pixels detection could run on (I5): map-only and empty layers have none. */ +const hasDetectionPixels = (layer: CanvasControlLayerContract): boolean => hasPixels(getControlDetectionInput(layer)); + +const settingsMatch = (required: Record, actual: Record): boolean => + Object.entries(required).every(([key, value]) => areJsonValuesStructurallyEqual(actual[key], value)); + +/** The type's entry for `detector`; a type may fix some of its settings (PiDi as a sketch detector). */ +const findDetectorSpec = (type: ControlType, detector: string) => + CONTROL_TYPE_SPECS[type].detectors.find((entry) => entry.type === detector); + +const belongsToType = (type: ControlType, detection: CanvasControlDetectionContract): boolean => + CONTROL_TYPE_SPECS[type].detectors.some( + (entry) => entry.type === detection.type && settingsMatch(entry.settings ?? {}, detection.settings) + ); + +const detectionFor = ( + type: ControlType, + detector: string, + options: ControlLayerIntentOptions +): CanvasControlDetectionContract => ({ + settings: { ...options.detectorDefaults(detector), ...findDetectorSpec(type, detector)?.settings }, + type: detector, +}); + +/** Drops every pixel the layer derived from or was filtered from; the caller sets the new source. */ +const withoutDerivedPixels = (layer: CanvasControlLayerContract): CanvasControlLayerContract => { + const next = { ...layer }; + delete next.original; + delete next.originalTransform; + delete next.filter; + return next; +}; + +const setType = ( + layer: CanvasControlLayerContract, + controlType: ControlType | null, + options: ControlLayerIntentOptions +): CanvasControlLayerContract => { + if (controlType === null) { + if (layer.controlType === undefined) { + return layer; + } + // Restore first, so the image the user brought is what an untyped layer keeps. + const next = { ...restoreControlOriginal(layer) }; + delete next.controlType; + delete next.detection; + delete next.original; + delete next.originalTransform; + return next; + } + if (layer.controlType === controlType) { + return layer; + } + const typed: CanvasControlLayerContract = { + ...layer, + adapter: { ...layer.adapter, model: null }, + controlType, + }; + const spec = CONTROL_TYPE_SPECS[controlType]; + let detection: CanvasControlDetectionContract | null; + if (!hasDetectionPixels(layer)) { + detection = null; + } else if (layer.detection && belongsToType(controlType, layer.detection)) { + detection = layer.detection; + } else { + detection = spec.defaultDetector ? detectionFor(controlType, spec.defaultDetector, options) : null; + } + return detection === null ? restoreControlOriginal(typed) : { ...typed, detection }; +}; + +const setDetector = ( + layer: CanvasControlLayerContract, + detector: string | null, + options: ControlLayerIntentOptions +): CanvasControlLayerContract | 'no-detection-input' => { + if (layer.controlType === undefined) { + return layer; + } + if (detector === null) { + return restoreControlOriginal(layer); + } + if (getControlDetectionInput(layer) === null) { + return 'no-detection-input'; + } + if (layer.detection?.type === detector) { + return layer; + } + return { ...layer, detection: detectionFor(layer.controlType, detector, options) }; +}; + +const setDetectorSettings = ( + layer: CanvasControlLayerContract, + settings: Record +): CanvasControlLayerContract => { + if (layer.controlType === undefined || !layer.detection) { + return layer; + } + const fixed = findDetectorSpec(layer.controlType, layer.detection.type)?.settings; + return { + ...layer, + detection: { settings: { ...layer.detection.settings, ...settings, ...fixed }, type: layer.detection.type }, + }; +}; + +const replaceImage = ( + layer: CanvasControlLayerContract, + intent: Extract, + options: ControlLayerIntentOptions +): CanvasControlLayerContract => { + const next: CanvasControlLayerContract = { + ...withoutDerivedPixels(layer), + source: { image: { ...intent.image }, type: 'image' }, + }; + if (intent.transform) { + next.transform = { ...intent.transform }; + } + // None was chosen only because there was nothing to detect; a first image gets the type's detector. + const wasEmpty = layer.original === undefined && !hasPixels(layer.source); + const defaultDetector = layer.controlType ? CONTROL_TYPE_SPECS[layer.controlType].defaultDetector : null; + if (layer.controlType && layer.detection === null && wasEmpty && defaultDetector) { + next.detection = detectionFor(layer.controlType, defaultDetector, options); + } + return next; +}; + +const draw = (layer: CanvasControlLayerContract): CanvasControlLayerContract => { + const next: CanvasControlLayerContract = { ...withoutDerivedPixels(layer), source: { bitmap: null, type: 'paint' } }; + if (next.controlType !== undefined) { + next.detection = null; + } + return next; +}; + +/** + * The layer after `intent`, or the same object when nothing changes. `'no-detection-input'` refuses a detector for a + * layer whose pixels are a map without an original (a legacy filter result): replace the image first. + */ +export const applyControlLayerIntent = ( + layer: CanvasControlLayerContract, + intent: ControlLayerIntent, + options: ControlLayerIntentOptions +): CanvasControlLayerContract | 'no-detection-input' => { + let next: CanvasControlLayerContract | 'no-detection-input'; + switch (intent.kind) { + case 'set-type': { + const typed = setType(layer, intent.controlType, options); + next = intent.name === undefined || intent.name === typed.name ? typed : { ...typed, name: intent.name }; + break; + } + case 'set-detector': + next = setDetector(layer, intent.detector, options); + break; + case 'set-detector-settings': + next = setDetectorSettings(layer, intent.settings); + break; + case 'replace-image': + next = replaceImage(layer, intent, options); + break; + case 'draw': + next = draw(layer); + break; + } + if (next === 'no-detection-input') { + return next; + } + return areJsonValuesStructurallyEqual(next, layer) ? layer : next; +}; diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/createCanvasEngine.controlDetection.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/createCanvasEngine.controlDetection.test.ts new file mode 100644 index 00000000000..c998ebf3beb --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/createCanvasEngine.controlDetection.test.ts @@ -0,0 +1,184 @@ +import type { + CanvasControlLayerContract, + CanvasLayerContract, + CanvasRasterLayerContractV2, +} from '@workbench/canvas-engine/contracts'; +import type { StubRasterBackend } from '@workbench/canvas-engine/render/raster.testStub'; +import type { CanvasUtilityGraphResult } from '@workbench/canvas-operations/contracts'; + +import { createControlLayer, getDocumentLeaves } from '@workbench/canvas-engine/api'; +import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { createTestStubRasterBackend } from '@workbench/canvas-engine/render/raster.testStub'; +import { createEmptyCanvasDocument } from '@workbench/canvasMigration'; +import { createCanvasProjectMutationPort } from '@workbench/canvasProjectMutationPort'; +import { createWorkbenchStore } from '@workbench/workbenchStore'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { createCanvasEngine } from './createCanvasEngine'; +import { getCanvasOperations } from './operationAccess'; + +const DEBOUNCE_MS = 400; +const SIZE = 64; + +const photoControl = (id: string): CanvasControlLayerContract => ({ + ...createControlLayer({ id, name: id }), + source: { image: { height: SIZE, imageName: `${id}.png`, width: SIZE }, type: 'image' }, +}); + +const shapeRaster = (id: string): CanvasRasterLayerContractV2 => ({ + blendMode: 'normal', + id, + isEnabled: true, + isLocked: false, + name: id, + opacity: 1, + source: { fill: '#fff', height: SIZE, kind: 'rect', stroke: null, strokeWidth: 0, type: 'shape', width: SIZE }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 0, y: 0 }, + type: 'raster', +}); + +/** Decodes every image at the test size, so a detected map lands on its input. */ +const sizedBackend = (): StubRasterBackend => ({ + ...createTestStubRasterBackend(), + createImageBitmap: () => + Promise.resolve({ close: () => undefined, height: SIZE, width: SIZE } as unknown as ImageBitmap), +}); + +/** A real engine; graphs stay pending unless `runGraph` answers them. */ +const setup = ( + layers: CanvasLayerContract[] = [photoControl('c1')], + runGraph: (options: { signal?: AbortSignal }) => Promise = () => new Promise(() => {}) +) => { + const store = createWorkbenchStore(); + const projectId = store.getState().activeProjectId; + store.commands.canvas.apply(projectId, { + document: { ...createEmptyCanvasDocument(), selectedLayerId: layers[0]!.id, stacks: stacksFrom(layers) }, + type: 'replaceCanvasDocument', + }); + const graphSignals: AbortSignal[] = []; + const graph = vi.fn((options: { signal?: AbortSignal }) => { + graphSignals.push(options.signal!); + return runGraph(options); + }); + const makeDurable = vi.fn((_imageName: string) => Promise.resolve()); + const engine = createCanvasEngine({ + backend: sizedBackend(), + controlDetectionDeps: { makeDurable }, + ensureProjectOnServer: () => Promise.resolve(), + filterDeps: { runGraph: graph, uploadIntermediate: () => Promise.resolve({ imageName: 'filter-input.png' }) }, + imageResolver: () => Promise.resolve(new Blob()), + mutationPort: createCanvasProjectMutationPort(store, projectId), + projectId, + reportError: () => undefined, + }); + const operations = getCanvasOperations(engine); + const leaves = () => getDocumentLeaves(store.queries.getProject(projectId)!.canvas.document); + const layer = (id = 'c1') => leaves().find((leaf) => leaf.id === id) as CanvasControlLayerContract | undefined; + return { engine, graph, graphSignals, layer, leaves, makeDurable, operations }; +}; + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('createCanvasEngine control detection', () => { + it('detects after the debounce, lands the map and undoes the intent with its map in one step', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const { engine, graph, layer, makeDurable, operations } = setup(undefined, () => + Promise.resolve({ height: SIZE, imageName: 'edges-map.png', width: SIZE }) + ); + const before = layer(); + + expect(operations.applyControlLayerIntent('c1', { controlType: 'edges', kind: 'set-type' })).toBe('updated'); + expect(operations.getControlDetectionStatus('c1')).toEqual({ status: 'queued' }); + await vi.advanceTimersByTimeAsync(DEBOUNCE_MS - 1); + expect(graph).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + await vi.waitFor(() => expect(operations.getControlDetectionStatus('c1')).toEqual({ status: 'idle' })); + + expect(graph).toHaveBeenCalledOnce(); + expect(makeDurable).toHaveBeenCalledExactlyOnceWith('edges-map.png'); + expect(layer()).toMatchObject({ + controlType: 'edges', + detection: { type: 'canny_edge_detection' }, + filter: { type: 'canny_edge_detection' }, + original: before!.source, + source: { bitmap: { imageName: 'edges-map.png' }, type: 'paint' }, + }); + + expect(await engine.history.undo()).toBe('applied'); + expect(layer()).toEqual(before); + expect(engine.stores.canUndo.get()).toBe(false); + expect(graph).toHaveBeenCalledOnce(); + engine.lifecycle.dispose(); + }); + + it('suspends detection on cooldown and resumes it on activation', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const { engine, graph, graphSignals, operations } = setup(); + operations.applyControlLayerIntent('c1', { controlType: 'edges', kind: 'set-type' }); + await vi.advanceTimersByTimeAsync(DEBOUNCE_MS); + + void engine.lifecycle.beginCooldown(); + expect(graphSignals[0]!.aborted).toBe(true); + expect(operations.getControlDetectionStatus('c1')).toEqual({ status: 'queued' }); + + engine.lifecycle.activate(); + await vi.advanceTimersByTimeAsync(0); + expect(graph).toHaveBeenCalledTimes(2); + engine.lifecycle.dispose(); + }); + + it('drops the status of a deleted layer', () => { + const { engine, operations } = setup(); + operations.applyControlLayerIntent('c1', { controlType: 'edges', kind: 'set-type' }); + + const remove = engine.document.model()!.prepare({ ids: ['c1'], type: 'remove' }); + if (remove.status !== 'prepared') { + throw new Error('expected a prepared removal'); + } + expect(engine.layers.commitPrepared('Delete', remove.edit).status).toBe('committed'); + expect(operations.getControlDetectionStatus('c1')).toEqual({ status: 'idle' }); + engine.lifecycle.dispose(); + }); + + it('aborts running detection on dispose, idempotently', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const { engine, graphSignals, operations } = setup(); + const listener = vi.fn(); + operations.subscribeControlDetectionStatus('c1', listener); + operations.applyControlLayerIntent('c1', { controlType: 'edges', kind: 'set-type' }); + await vi.advanceTimersByTimeAsync(DEBOUNCE_MS); + expect(graphSignals).toHaveLength(1); + listener.mockClear(); + + engine.lifecycle.dispose(); + engine.lifecycle.dispose(); + + expect(graphSignals[0]!.aborted).toBe(true); + expect(operations.getControlDetectionStatus('c1')).toEqual({ status: 'idle' }); + expect(listener).toHaveBeenCalledOnce(); + expect(operations.applyControlLayerIntent('c1', { kind: 'draw' })).toBe('blocked'); + }); + + it('types a canny filter copy saved as control as an edges layer used as-is', async () => { + const { engine, leaves, operations } = setup([shapeRaster('r1')], () => + Promise.resolve({ height: SIZE, imageName: 'canny.png', width: SIZE }) + ); + expect((await engine.exports.exportLayerPixels('r1')).status).toBe('ok'); + expect(operations.startFilterOperation('r1')).toBe('started'); + operations.updateFilterOperation({ + settings: { high_threshold: 200, low_threshold: 100 }, + type: 'canny_edge_detection', + }); + await expect(operations.processFilterOperation()).resolves.toBe('completed'); + + await expect(operations.commitFilterOperation('control', () => Promise.resolve())).resolves.toBe('committed'); + + const copies = leaves().filter((leaf) => leaf.type === 'control'); + expect(copies).toHaveLength(1); + expect(copies[0]).toMatchObject({ controlType: 'edges', detection: null }); + expect(copies[0]).not.toHaveProperty('original'); + engine.lifecycle.dispose(); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/createCanvasEngine.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/createCanvasEngine.ts index c92f73125a2..db293bed9c7 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/createCanvasEngine.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/createCanvasEngine.ts @@ -6,12 +6,15 @@ import type { CanvasUtilityGraphResult } from '@workbench/canvas-operations/cont import type { GenerationCompositeHost } from '@workbench/canvas-operations/generationComposite'; import type { BackendGraphContract } from '@workbench/graphContracts'; +import { galleryDurability } from '@features/gallery'; import { assertAccountScopeCurrent, captureAccountScope } from '@platform/state/accountLifecycle'; import { createCanvasEngine as createCanvasEngineCore } from '@workbench/canvas-engine/engine'; import { canvasApplicationPort } from '@workbench/canvas-operations/applicationPort'; import { isUploadProjectNotFound } from '@workbench/canvas-operations/backend/canvasImages'; import { createBoundedCompositeDedupeCache } from '@workbench/canvas-operations/compositeForGeneration'; +import { createControlDetectionCoordinator } from '@workbench/canvas-operations/ControlDetectionCoordinator'; import { composeForGeneration } from '@workbench/canvas-operations/generationComposite'; +import { controlTypeForDetector } from '@workbench/controlTypes'; import type { CanvasOperationCapability, CanvasOperationImplementation } from './contracts'; @@ -35,6 +38,10 @@ export interface CanvasEngineOptions extends Omit< signal?: AbortSignal; }): Promise; }; + controlDetectionDeps?: { + /** Adopts a detected map as canvas-owned pixels; the gallery's canvas-asset transition by default. */ + makeDurable(imageName: string): Promise; + }; filterDeps?: { uploadIntermediate(blob: Blob, signal?: AbortSignal): Promise<{ imageName: string }>; runGraph(options: { @@ -63,7 +70,7 @@ export type CanvasEngine = CoreCanvasEngineImplementation; /** Application composition root: owns SAM/filter sessions, queues, uploads, and their core capability adapters. */ export const createCanvasEngine = (options: CanvasEngineOptions): CanvasEngine => { - const { ensureProjectOnServer, filterDeps, selectObjectDeps, ...coreOptions } = options; + const { controlDetectionDeps, ensureProjectOnServer, filterDeps, selectObjectDeps, ...coreOptions } = options; const owner = captureAccountScope(); const uploadLifetime = new AbortController(); let projectMissingAt = -Infinity; @@ -120,6 +127,14 @@ export const createCanvasEngine = (options: CanvasEngineOptions): CanvasEngine = edits: core.edits, isGuardCurrent: host.isGuardCurrent, }); + const runFilterGraph = async (runOptions: { + graph: BackendGraphContract; + outputNodeId?: string; + signal?: AbortSignal; + }): Promise => { + const output = await (filterDeps?.runGraph(runOptions) ?? canvasApplicationPort.runGraph(runOptions)); + return { height: output.height, imageName: output.imageName, width: output.width }; + }; let filterCoordinator!: ReturnType; const selectObjectCoordinator = canvasApplicationPort.createSelectObjectCoordinator({ @@ -203,6 +218,7 @@ export const createCanvasEngine = (options: CanvasEngineOptions): CanvasEngine = return Promise.resolve({ status: 'locked' }); } return host.commitFilter({ + ...(target === 'control' ? { controlCopy: { controlType: controlTypeForDetector(draft) } } : {}), filter: draft, guard, image, @@ -219,10 +235,7 @@ export const createCanvasEngine = (options: CanvasEngineOptions): CanvasEngine = host.publishFilterPreview(guard.layerId, imageName, rect, guard, filterType), }), isInteractionLocked: host.isInteractionLocked, - runFilterGraph: async (runOptions) => { - const output = await (filterDeps?.runGraph(runOptions) ?? canvasApplicationPort.runGraph(runOptions)); - return { height: output.height, imageName: output.imageName, width: output.width }; - }, + runFilterGraph, selectLayer: host.selectLayer, setViewTool: host.setViewTool, stores, @@ -233,6 +246,35 @@ export const createCanvasEngine = (options: CanvasEngineOptions): CanvasEngine = }, }); + const interactionLockListeners = new Set<() => void>(); + // Any of these may be what refused a detection commit; each release lets a waiting result retry. + const lockStores = [ + core.stores.documentEditingLocked, + core.stores.transformSession, + core.stores.textEditSession, + core.stores.documentEpoch, + ]; + const controlDetection = createControlDetectionCoordinator({ + derivation: core.controlDerivation, + flushLayerUploads: host.flushLayerUploads, + getDocument: host.getDocument, + makeDurable: controlDetectionDeps?.makeDurable ?? galleryDurability.makeCanvasAsset, + projectId: options.projectId, + runFilterGraph, + subscribeLockChanges: (listener) => { + interactionLockListeners.add(listener); + const unsubscribers = lockStores.map((store) => store.subscribe(listener)); + return () => { + interactionLockListeners.delete(listener); + for (const unsubscribe of unsubscribers) { + unsubscribe(); + } + }; + }, + }); + // Statuses follow the document (deleted layers, undo past a failure); it never schedules detection. + const unsubscribeControlReconcile = core.stores.documentEpoch.subscribe(controlDetection.reconcileWithDocument); + const syncEditingLock = (): void => { const operation = controller.getSnapshot(); core.stores.documentEditingLocked.set(operation.status === 'active'); @@ -281,12 +323,14 @@ export const createCanvasEngine = (options: CanvasEngineOptions): CanvasEngine = }; const operations: CanvasOperationImplementation = { + applyControlLayerIntent: controlDetection.applyIntent, applySelectObjectSession: selectObjectCoordinator.apply, cancelFilterOperation: filterCoordinator.cancel, cancelSelectObjectSession: selectObjectCoordinator.cancel, commitFilterOperation: filterCoordinator.commit, composeForGeneration: (composeOptions) => composeForGeneration(generationCompositeHost, composeOptions), controller, + getControlDetectionStatus: controlDetection.getStatus, getFilterSessionState: stores.filterSession.get, getOperationState: controller.getSnapshot, getSamSessionState: stores.samSession.get, @@ -294,11 +338,13 @@ export const createCanvasEngine = (options: CanvasEngineOptions): CanvasEngine = processSelectObjectSession: selectObjectCoordinator.process, resetFilterOperation: filterCoordinator.reset, resetSelectObjectSession: selectObjectCoordinator.reset, + retryControlDetection: controlDetection.retry, saveSelectObjectSession: selectObjectCoordinator.save, setFilterOperationAutoProcess: filterCoordinator.setAutoProcess, startFilterOperation: filterCoordinator.start, startSelectObject: selectObjectCoordinator.start, stores, + subscribeControlDetectionStatus: controlDetection.subscribeStatus, subscribeFilterSession: stores.filterSession.subscribe, subscribeOperation: controller.subscribe, subscribeSamSession: stores.samSession.subscribe, @@ -331,6 +377,9 @@ export const createCanvasEngine = (options: CanvasEngineOptions): CanvasEngine = if (!locked && selectObjectCoordinator.isActive()) { host.setSamTool(); } + for (const listener of interactionLockListeners) { + listener(); + } }, }; @@ -346,6 +395,8 @@ export const createCanvasEngine = (options: CanvasEngineOptions): CanvasEngine = unsubscribeToolChanges(); unsubscribeSamInteraction(); unsubscribeEditingLock(); + unsubscribeControlReconcile(); + controlDetection.dispose(); filterCoordinator.dispose(); selectObjectCoordinator.dispose(); controller.dispose(); @@ -355,9 +406,16 @@ export const createCanvasEngine = (options: CanvasEngineOptions): CanvasEngine = }; const lifecycle = { ...core.lifecycle, + activate: () => { + core.lifecycle.activate(); + if (!disposed) { + controlDetection.resume(); + } + }, beginCooldown: () => { filterCoordinator.cancel(); selectObjectCoordinator.cancel(); + controlDetection.suspend(); controller.invalidateDocument(options.projectId); return core.lifecycle.beginCooldown(); }, diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/operationTypes.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/operationTypes.ts index fec8922ea85..29389dbb2de 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/operationTypes.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/operationTypes.ts @@ -1,10 +1,12 @@ import type { LayerExportGuard } from '@workbench/canvas-engine/api'; import type { GuardedMutationRefusal } from '@workbench/canvas-engine/capabilities'; +import type { CanvasImageRef, ControlType } from '@workbench/canvas-engine/contracts'; import type { RasterFilterCommitTarget, RasterFilterSettings, } from '@workbench/canvas-engine/controllers/filterResultController'; import type { SamVisualInput } from '@workbench/canvas-engine/samInteraction'; +import type { LayerTransform } from '@workbench/canvas-engine/transform/transformMath'; import type { Rect, Vec2 } from '@workbench/canvas-engine/types'; export type { SamVisualInput } from '@workbench/canvas-engine/samInteraction'; @@ -91,3 +93,50 @@ export interface FilterOperationSessionState { status: 'ready' | 'processing' | 'committing' | 'error'; } export type FilterCommitTarget = RasterFilterCommitTarget; + +/** + * A control layer edit from the Properties panel. `set-type` may rename in the same history entry; `set-detector` + * starts from the detector's defaults with the type's fixed settings; `draw` empties the image for painting. + */ +export type ControlLayerIntent = + | { kind: 'set-type'; controlType: ControlType | null; name?: string } + | { kind: 'set-detector'; detector: string | null } + | { kind: 'set-detector-settings'; settings: Record } + | { kind: 'replace-image'; image: CanvasImageRef; transform?: LayerTransform } + | { kind: 'draw' }; +/** + * How an intent ended. `unchanged`: the layer already matched. `no-detection-input`: the layer's pixels are a map + * without an original, so a detector cannot run until the image is replaced. `missing`: no such control layer. + * `locked`: the layer or a group above it is locked. `blocked`: an operation, gesture or session holds the + * document; try again once it ends. `over-budget` and `failed`: the engine could not apply the edit. + */ +export type ControlLayerIntentResult = + | 'updated' + | 'unchanged' + | 'no-detection-input' + | 'missing' + | 'locked' + | 'blocked' + | 'over-budget' + | 'failed'; +/** `not-needed`: the map is current or there is nothing to detect from. */ +export type RetryControlDetectionResult = 'scheduled' | 'not-needed' | 'missing'; +/** Typed detection failures; the view translates the code and shows `detail` as technical detail. */ +export type ControlDetectionErrorCode = + | 'process-failed' + | 'apply-failed' + | 'unknown-detector' + | 'output-dimensions' + | 'source-not-ready' + | 'over-budget' + | 'commit-busy' + | 'unrepresentable'; +export interface ControlDetectionError { + code: ControlDetectionErrorCode; + detail?: string; +} +export type ControlDetectionStatus = + | { status: 'idle' } + | { status: 'queued' } + | { status: 'processing'; detector: string } + | { status: 'error'; error: ControlDetectionError }; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/engineStoreHooks.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/engineStoreHooks.ts index 0629fcae787..e01e0c22c68 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/engineStoreHooks.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/engineStoreHooks.ts @@ -20,7 +20,9 @@ import type { import { getCanvasOperations, + IDLE_CONTROL_DETECTION_STATUS, type CanvasOperationState, + type ControlDetectionStatus, type FilterOperationSessionState, type SamSessionSnapshot, } from '@workbench/canvas-operations/api'; @@ -142,6 +144,20 @@ export const useFilterSession = (engine: object): FilterOperationSessionState | ); }; +/** One control layer's background detection status; a null engine reads as idle without subscribing. */ +export const useControlDetectionStatus = (engine: object | null, layerId: string): ControlDetectionStatus => { + const subscribe = useCallback( + (listener: () => void) => + engine ? getCanvasOperations(engine).subscribeControlDetectionStatus(layerId, listener) : () => undefined, + [engine, layerId] + ); + const getSnapshot = useCallback( + () => (engine ? getCanvasOperations(engine).getControlDetectionStatus(layerId) : IDLE_CONTROL_DETECTION_STATUS), + [engine, layerId] + ); + return useSyncExternalStore(subscribe, getSnapshot, getSnapshot); +}; + /** Whether the engine-owned canvas history has an entry to undo (enables the header undo button). */ export const useCanvasCanUndo = (engine: CanvasCoreStoreCapability): boolean => useCanvasInteractionState(engine, 'canUndo'); From 82ffe4a6eeaec1c4d7c3eab32de262f6b735e14c Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Fri, 23 Oct 2026 01:20:06 -0600 Subject: [PATCH 05/35] perf(canvas): record the eager control-layer modules and bytes Owner sets gain the derivation, taxonomy, controller and coordinator modules; editor route bytes are re-recorded by hand (+1.3 %), timings and paths unchanged. --- .../performance/architecture-baseline.json | 4 + .../webv2/performance/browser-baseline.json | 74 ++++++++++++------- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/invokeai/frontend/webv2/performance/architecture-baseline.json b/invokeai/frontend/webv2/performance/architecture-baseline.json index edcdf690391..67ab9b84e0b 100644 --- a/invokeai/frontend/webv2/performance/architecture-baseline.json +++ b/invokeai/frontend/webv2/performance/architecture-baseline.json @@ -835,10 +835,12 @@ "source:src/workbench/canvas-engine/document-model/screenComposition.ts", "source:src/workbench/canvas-engine/document-model/semanticLeaf.ts", "source:src/workbench/canvas-engine/document-model/semanticNode.ts", + "source:src/workbench/canvas-engine/document/controlDerivation.ts", "source:src/workbench/canvas-engine/document/documentIndex.ts", "source:src/workbench/canvas-engine/document/documentTree.ts", "source:src/workbench/canvas-engine/document/insertionAnchors.ts", "source:src/workbench/canvas-engine/document/layerEligibility.ts", + "source:src/workbench/canvas-engine/document/layerFactories.ts", "source:src/workbench/canvas-engine/document/layerStacks.ts", "source:src/workbench/canvas-engine/document/mergeVisible.ts", "source:src/workbench/canvas-engine/document/selectionRepair.ts", @@ -869,6 +871,8 @@ "source:src/workbench/components/WorkbenchUnavailableScreen.tsx", "source:src/workbench/controlAdapters.ts", "source:src/workbench/controlLayerChecks.ts", + "source:src/workbench/controlModelResolution.ts", + "source:src/workbench/controlTypes.ts", "source:src/workbench/deferredResource.ts", "source:src/workbench/diagnostics/loggingPreferences.ts", "source:src/workbench/extensions/contributionSource.ts", diff --git a/invokeai/frontend/webv2/performance/browser-baseline.json b/invokeai/frontend/webv2/performance/browser-baseline.json index f83b5c8d8f2..5e5f827d170 100644 --- a/invokeai/frontend/webv2/performance/browser-baseline.json +++ b/invokeai/frontend/webv2/performance/browser-baseline.json @@ -106,11 +106,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752413, - "otherRawBytes": 226185, + "otherRawBytes": 227936, "requestCount": 87, - "scriptRawBytes": 4199867, + "scriptRawBytes": 4256250, "scriptRequestCount": 82, - "totalRawBytes": 4496273 + "totalRawBytes": 4554407 }, "routeReadyMedianMs": 360.90000009536743, "scriptSourceOwnerSet": "editor-minimal-static", @@ -144,11 +144,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752413, - "otherRawBytes": 226185, + "otherRawBytes": 227936, "requestCount": 87, - "scriptRawBytes": 4199867, + "scriptRawBytes": 4256250, "scriptRequestCount": 82, - "totalRawBytes": 4496273 + "totalRawBytes": 4554407 }, "routeReadyMedianMs": 367.80000019073486, "scriptSourceOwnerSet": "editor-minimal-static", @@ -182,11 +182,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752413, - "otherRawBytes": 226185, + "otherRawBytes": 227936, "requestCount": 96, - "scriptRawBytes": 4718489, + "scriptRawBytes": 4779082, "scriptRequestCount": 91, - "totalRawBytes": 5014895 + "totalRawBytes": 5077239 }, "routeReadyMedianMs": 1095.3000001907349, "scriptSourceOwnerSet": "editor-canvas-static", @@ -220,11 +220,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752413, - "otherRawBytes": 226185, + "otherRawBytes": 227936, "requestCount": 96, - "scriptRawBytes": 4718489, + "scriptRawBytes": 4779082, "scriptRequestCount": 91, - "totalRawBytes": 5014895 + "totalRawBytes": 5077239 }, "routeReadyMedianMs": 1042.5, "scriptSourceOwnerSet": "editor-canvas-static", @@ -258,11 +258,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752413, - "otherRawBytes": 226185, + "otherRawBytes": 227936, "requestCount": 94, - "scriptRawBytes": 4644510, + "scriptRawBytes": 4708392, "scriptRequestCount": 88, - "totalRawBytes": 4956329 + "totalRawBytes": 5021962 }, "routeReadyMedianMs": 1093.0999999046326, "scriptSourceOwnerSet": "editor-workflow-static", @@ -296,11 +296,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752413, - "otherRawBytes": 226185, + "otherRawBytes": 227936, "requestCount": 94, - "scriptRawBytes": 4644510, + "scriptRawBytes": 4708392, "scriptRequestCount": 88, - "totalRawBytes": 4956329 + "totalRawBytes": 5021962 }, "routeReadyMedianMs": 1105.0999999046326, "scriptSourceOwnerSet": "editor-workflow-static", @@ -334,11 +334,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752413, - "otherRawBytes": 226185, + "otherRawBytes": 227936, "requestCount": 87, - "scriptRawBytes": 4199867, + "scriptRawBytes": 4256250, "scriptRequestCount": 82, - "totalRawBytes": 4496273 + "totalRawBytes": 4554407 }, "routeReadyMedianMs": 1324.5, "scriptSourceOwnerSet": "editor-minimal-static", @@ -372,11 +372,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752413, - "otherRawBytes": 226185, + "otherRawBytes": 227936, "requestCount": 87, - "scriptRawBytes": 4199867, + "scriptRawBytes": 4256250, "scriptRequestCount": 82, - "totalRawBytes": 4496273 + "totalRawBytes": 4554407 }, "routeReadyMedianMs": 1301.5, "scriptSourceOwnerSet": "editor-minimal-static", @@ -1376,6 +1376,7 @@ "source:src/workbench/canvas-engine/color.ts", "source:src/workbench/canvas-engine/contracts.ts", "source:src/workbench/canvas-engine/controllers/booleanMergeController.ts", + "source:src/workbench/canvas-engine/controllers/controlDerivationController.ts", "source:src/workbench/canvas-engine/controllers/controlPixelController.ts", "source:src/workbench/canvas-engine/controllers/copyLayerController.ts", "source:src/workbench/canvas-engine/controllers/cropLayerController.ts", @@ -1416,6 +1417,7 @@ "source:src/workbench/canvas-engine/document-model/semanticLeaf.ts", "source:src/workbench/canvas-engine/document-model/semanticNode.ts", "source:src/workbench/canvas-engine/document/bitmapStore.ts", + "source:src/workbench/canvas-engine/document/controlDerivation.ts", "source:src/workbench/canvas-engine/document/documentIndex.ts", "source:src/workbench/canvas-engine/document/documentMirror.ts", "source:src/workbench/canvas-engine/document/documentTree.ts", @@ -1518,6 +1520,7 @@ "source:src/workbench/canvas-engine/transform/transformMath.ts", "source:src/workbench/canvas-engine/types.ts", "source:src/workbench/canvas-engine/viewport.ts", + "source:src/workbench/canvas-operations/ControlDetectionCoordinator.ts", "source:src/workbench/canvas-operations/FilterOperationCoordinator.ts", "source:src/workbench/canvas-operations/SelectObjectCoordinator.ts", "source:src/workbench/canvas-operations/api.ts", @@ -1525,6 +1528,9 @@ "source:src/workbench/canvas-operations/backend/canvasImages.ts", "source:src/workbench/canvas-operations/canvasImportNotice.ts", "source:src/workbench/canvas-operations/compositeForGeneration.ts", + "source:src/workbench/canvas-operations/controlDetectionRunner.ts", + "source:src/workbench/canvas-operations/controlDetectionStatusStore.ts", + "source:src/workbench/canvas-operations/controlLayerIntents.ts", "source:src/workbench/canvas-operations/controlRecommendations.ts", "source:src/workbench/canvas-operations/createCanvasEngine.ts", "source:src/workbench/canvas-operations/createCanvasFromImages.ts", @@ -1557,6 +1563,8 @@ "source:src/workbench/components/WorkbenchUnavailableScreen.tsx", "source:src/workbench/controlAdapters.ts", "source:src/workbench/controlLayerChecks.ts", + "source:src/workbench/controlModelResolution.ts", + "source:src/workbench/controlTypes.ts", "source:src/workbench/deferredResource.ts", "source:src/workbench/diagnostics/loggingPreferences.ts", "source:src/workbench/extensions/contributionSource.ts", @@ -1790,7 +1798,6 @@ "source:src/workbench/widgets/image-map/ImageMapDataRuntime.tsx", "source:src/workbench/widgets/image-map/manifest.ts", "source:src/workbench/widgets/image-map/settingsContribution.ts", - "source:src/workbench/widgets/layers/controlModelOptions.ts", "source:src/workbench/widgets/layers/layerOps.ts", "source:src/workbench/widgets/layers/manifest.ts", "source:src/workbench/widgets/layers/panes/editorPaneLayout.ts", @@ -2482,6 +2489,7 @@ "source:src/workbench/canvas-engine/color.ts", "source:src/workbench/canvas-engine/contracts.ts", "source:src/workbench/canvas-engine/controllers/booleanMergeController.ts", + "source:src/workbench/canvas-engine/controllers/controlDerivationController.ts", "source:src/workbench/canvas-engine/controllers/controlPixelController.ts", "source:src/workbench/canvas-engine/controllers/copyLayerController.ts", "source:src/workbench/canvas-engine/controllers/cropLayerController.ts", @@ -2522,6 +2530,7 @@ "source:src/workbench/canvas-engine/document-model/semanticLeaf.ts", "source:src/workbench/canvas-engine/document-model/semanticNode.ts", "source:src/workbench/canvas-engine/document/bitmapStore.ts", + "source:src/workbench/canvas-engine/document/controlDerivation.ts", "source:src/workbench/canvas-engine/document/documentIndex.ts", "source:src/workbench/canvas-engine/document/documentMirror.ts", "source:src/workbench/canvas-engine/document/documentTree.ts", @@ -2624,6 +2633,7 @@ "source:src/workbench/canvas-engine/transform/transformMath.ts", "source:src/workbench/canvas-engine/types.ts", "source:src/workbench/canvas-engine/viewport.ts", + "source:src/workbench/canvas-operations/ControlDetectionCoordinator.ts", "source:src/workbench/canvas-operations/FilterOperationCoordinator.ts", "source:src/workbench/canvas-operations/SelectObjectCoordinator.ts", "source:src/workbench/canvas-operations/api.ts", @@ -2631,6 +2641,9 @@ "source:src/workbench/canvas-operations/backend/canvasImages.ts", "source:src/workbench/canvas-operations/canvasImportNotice.ts", "source:src/workbench/canvas-operations/compositeForGeneration.ts", + "source:src/workbench/canvas-operations/controlDetectionRunner.ts", + "source:src/workbench/canvas-operations/controlDetectionStatusStore.ts", + "source:src/workbench/canvas-operations/controlLayerIntents.ts", "source:src/workbench/canvas-operations/controlRecommendations.ts", "source:src/workbench/canvas-operations/createCanvasEngine.ts", "source:src/workbench/canvas-operations/createCanvasFromImages.ts", @@ -2667,6 +2680,8 @@ "source:src/workbench/components/WorkbenchUnavailableScreen.tsx", "source:src/workbench/controlAdapters.ts", "source:src/workbench/controlLayerChecks.ts", + "source:src/workbench/controlModelResolution.ts", + "source:src/workbench/controlTypes.ts", "source:src/workbench/deferredResource.ts", "source:src/workbench/diagnostics/loggingPreferences.ts", "source:src/workbench/extensions/contributionSource.ts", @@ -3811,6 +3826,7 @@ "source:src/workbench/canvas-engine/color.ts", "source:src/workbench/canvas-engine/contracts.ts", "source:src/workbench/canvas-engine/controllers/booleanMergeController.ts", + "source:src/workbench/canvas-engine/controllers/controlDerivationController.ts", "source:src/workbench/canvas-engine/controllers/controlPixelController.ts", "source:src/workbench/canvas-engine/controllers/copyLayerController.ts", "source:src/workbench/canvas-engine/controllers/cropLayerController.ts", @@ -3851,6 +3867,7 @@ "source:src/workbench/canvas-engine/document-model/semanticLeaf.ts", "source:src/workbench/canvas-engine/document-model/semanticNode.ts", "source:src/workbench/canvas-engine/document/bitmapStore.ts", + "source:src/workbench/canvas-engine/document/controlDerivation.ts", "source:src/workbench/canvas-engine/document/documentIndex.ts", "source:src/workbench/canvas-engine/document/documentMirror.ts", "source:src/workbench/canvas-engine/document/documentTree.ts", @@ -3953,6 +3970,7 @@ "source:src/workbench/canvas-engine/transform/transformMath.ts", "source:src/workbench/canvas-engine/types.ts", "source:src/workbench/canvas-engine/viewport.ts", + "source:src/workbench/canvas-operations/ControlDetectionCoordinator.ts", "source:src/workbench/canvas-operations/FilterOperationCoordinator.ts", "source:src/workbench/canvas-operations/SelectObjectCoordinator.ts", "source:src/workbench/canvas-operations/api.ts", @@ -3960,6 +3978,9 @@ "source:src/workbench/canvas-operations/backend/canvasImages.ts", "source:src/workbench/canvas-operations/canvasImportNotice.ts", "source:src/workbench/canvas-operations/compositeForGeneration.ts", + "source:src/workbench/canvas-operations/controlDetectionRunner.ts", + "source:src/workbench/canvas-operations/controlDetectionStatusStore.ts", + "source:src/workbench/canvas-operations/controlLayerIntents.ts", "source:src/workbench/canvas-operations/controlRecommendations.ts", "source:src/workbench/canvas-operations/createCanvasEngine.ts", "source:src/workbench/canvas-operations/createCanvasFromImages.ts", @@ -3992,6 +4013,8 @@ "source:src/workbench/components/WorkbenchUnavailableScreen.tsx", "source:src/workbench/controlAdapters.ts", "source:src/workbench/controlLayerChecks.ts", + "source:src/workbench/controlModelResolution.ts", + "source:src/workbench/controlTypes.ts", "source:src/workbench/deferredResource.ts", "source:src/workbench/diagnostics/loggingPreferences.ts", "source:src/workbench/extensions/contributionSource.ts", @@ -4225,7 +4248,6 @@ "source:src/workbench/widgets/image-map/ImageMapDataRuntime.tsx", "source:src/workbench/widgets/image-map/manifest.ts", "source:src/workbench/widgets/image-map/settingsContribution.ts", - "source:src/workbench/widgets/layers/controlModelOptions.ts", "source:src/workbench/widgets/layers/layerOps.ts", "source:src/workbench/widgets/layers/manifest.ts", "source:src/workbench/widgets/layers/panes/editorPaneLayout.ts", From a99e6409999219c594940e73fb08bcd6394d3d6d Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Fri, 23 Oct 2026 02:49:25 -0600 Subject: [PATCH 06/35] feat(ui): label-less select groups and segmented enum filter params Select renders a group whose label is null under a separator, PropertyControlRow accepts a label id for aria-labelledby, and LayerFilterControls can render short enums as a segmented control. --- .../src/platform/ui/Select.browser.test.tsx | 45 ++++++++++++++- .../frontend/webv2/src/platform/ui/Select.tsx | 44 +++++++++------ .../tool-presentation/PropertyPrimitives.tsx | 15 ++++- .../LayerFilterControls.browser.test.tsx | 55 ++++++++++++++++++- .../widgets/layers/LayerFilterControls.tsx | 40 ++++++++++++-- 5 files changed, 171 insertions(+), 28 deletions(-) diff --git a/invokeai/frontend/webv2/src/platform/ui/Select.browser.test.tsx b/invokeai/frontend/webv2/src/platform/ui/Select.browser.test.tsx index 4ba67da02f7..0f44670d8c4 100644 --- a/invokeai/frontend/webv2/src/platform/ui/Select.browser.test.tsx +++ b/invokeai/frontend/webv2/src/platform/ui/Select.browser.test.tsx @@ -2,7 +2,8 @@ import { ChakraProvider, createListCollection } from '@chakra-ui/react'; import { system } from '@theme/system'; import { act } from 'react'; import { createRoot, type Root } from 'react-dom/client'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { userEvent } from 'vitest/browser'; import { Select } from './Select'; @@ -11,6 +12,14 @@ const RATIO_ITEMS = [ { label: '1:1', value: '1:1' }, ]; const RATIO_VALUE = ['16:9']; +const DETECTOR_ITEMS = [ + { group: 'detectors', label: 'Canny', value: 'canny' }, + { group: 'detectors', label: 'MLSD', value: 'mlsd' }, + { group: 'none', label: 'None: image is already an edge map', value: 'none' }, +]; +const DETECTOR_VALUE = ['canny']; +const getDetectorGroup = (item: { group: string }) => item.group; +const renderDetectorGroupLabel = (group: string) => (group === 'detectors' ? 'Detectors' : null); (globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; @@ -80,4 +89,38 @@ describe('Select', () => { await act(() => trigger().click()); await expect.poll(() => document.querySelectorAll('[role="option"]').length).toBe(1); }); + + it('puts a header-less group under a divider and moves the keyboard across it', async () => { + const onValueChange = vi.fn(); + await act(() => { + root.render( + + + + {actionLabel ? ( + + {model.action === 'retry' ? : } + + ) : null} + + + + + + + {announcement} + + {open && model.settings ? ( + + + + ) : null} + + ); +}; + +const renderDetectionItem = (item: DetectionItem) => ( + + + {item.label} + {item.hint ? : null} + + {item.hint ? ( + + {item.hint} + + ) : null} + +); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.browser.test.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.browser.test.tsx index f68d399db39..60a68fa2c9a 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.browser.test.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.browser.test.tsx @@ -4,89 +4,122 @@ import type { CanvasLayerContract, CanvasLayerPreviewMutation, } from '@workbench/canvas-engine/api'; +import type { ControlLayerIntent, ControlLayerIntentResult } from '@workbench/canvas-operations/api'; import { ChakraProvider } from '@chakra-ui/react'; import { architectureCapabilitiesFixture } from '@features/generation/core/architectureCapabilities.testing'; import { ensureArchitectureCapabilitiesLoaded } from '@features/generation/data/architectureCapabilitiesStore'; +import { auditAccessibility } from '@platform/browser/auditAccessibility.testing'; import { accountLifecycle } from '@platform/state/accountLifecycle'; import { toaster } from '@platform/ui/toaster'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { applyThemeToRoot } from '@theme/applyTheme'; -import { system } from '@theme/system'; +import { DEFAULT_THEME_ID, system } from '@theme/system'; import { createControlLayer, createDocumentModel } from '@workbench/canvas-engine/api'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { previewInverse } from '@workbench/canvas-engine/document-model/documentModel'; +import { createControlDetectionStatusStore } from '@workbench/canvas-operations/controlDetectionStatusStore'; import { attachCanvasOperations } from '@workbench/canvas-operations/operationAccess'; import { createEmptyCanvasDocument } from '@workbench/canvasMigration'; +import { resetPropertyGroupCollapse } from '@workbench/widgets/canvas/tool-presentation/propertyGroupStore'; import { createInstance } from 'i18next'; import { act } from 'react'; import { createRoot, type Root } from 'react-dom/client'; -import { I18nextProvider, initReactI18next } from 'react-i18next'; -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { I18nextProvider } from 'react-i18next'; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { page, userEvent } from 'vitest/browser'; import { ControlLayerSettings } from './ControlLayerSettings'; -import { createEmptyPaintLayer } from './layerOps'; - -// Stable identities, as the real stores hand out, so only the table's arrival can re-read the policy. -const catalog = vi.hoisted(() => ({ - mainModel: { base: 'sd-1', key: 'sd1-main', name: 'SD 1.5', type: 'main' } as Record, - models: [{ base: 'sd-1', key: 'sd1-controlnet', name: 'SD 1.5 ControlNet', type: 'controlnet' }] as Record< - string, - unknown - >[], + +const harness = vi.hoisted(() => ({ + canManageModels: true, + mainModel: { base: 'sdxl', key: 'sdxl-main', name: 'SDXL', type: 'main' } as Record | null, + memory: [] as [string, string | null][], + startersRequested: 0, + models: [] as Record[], + navigate: [] as unknown[], })); -// Tests that swap the catalog restore these identities afterwards. -const SD1_CATALOG = { ...catalog }; vi.mock('@features/models', async (importOriginal) => ({ ...(await importOriginal>()), - useModelsSelector: (selector: (snapshot: { models: typeof catalog.models }) => unknown) => - selector({ models: catalog.models }), + // The real one fetches the starter catalog. + ensureStartersLoaded: () => { + harness.startersRequested += 1; + }, + useModelsSelector: (selector: (snapshot: { models: typeof harness.models }) => unknown) => + selector({ models: harness.models }), +})); +vi.mock('./useSelectedMainModel', () => ({ useSelectedMainModel: () => harness.mainModel })); +vi.mock('@features/identity', async (importOriginal) => ({ + ...(await importOriginal>()), + useCapabilities: () => ({ canManageModels: harness.canManageModels, canManageNodes: true, canManageUsers: true }), +})); +vi.mock('@tanstack/react-router', async (importOriginal) => ({ + ...(await importOriginal>()), + useNavigate: () => (options: unknown) => { + harness.navigate.push(options); + return Promise.resolve(); + }, +})); +vi.mock('@workbench/WorkbenchContext', () => ({ + useActiveProjectId: () => 'test-project', + useOptionalWorkbenchCommands: () => null, +})); +vi.mock('@workbench/settings/store', async (importOriginal) => ({ + ...(await importOriginal>()), + // The real one pushes preferences to the server. + patchControlModelMemory: (key: string, model: string | null) => { + harness.memory.push([key, model]); + return Promise.resolve(); + }, })); -vi.mock('./useSelectedMainModel', () => ({ useSelectedMainModel: () => catalog.mainModel })); - -/** A control layer on `base`'s adapter defaults with `model` already chosen. */ -const withModel = (name: string, id: string, base: string, model: string | null): CanvasControlLayerContract => { - const created = createControlLayer({ base, id, name }); - return { ...created, adapter: { ...created.adapter, model } }; -}; - const getArchitectureCapabilities = vi.fn<() => Promise>(); vi.mock('@features/generation/data/architectureCapabilitiesApi', () => ({ getArchitectureCapabilities: () => getArchitectureCapabilities(), })); const i18n = createInstance(); -void i18n.use(initReactI18next).init({ fallbackLng: 'en', initAsync: false, lng: 'en', resources: {} }); +beforeAll(async () => { + const translation = (await (await fetch('/locales/en.json')).json()) as Record; + await i18n.init({ initAsync: false, lng: 'en', resources: { en: { translation } } }); +}); (globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; +const model = (key: string, type = 'controlnet', base = 'sdxl') => ({ base, key, name: key, source: key, type }); +const DEPTH_A = model('controlnet-depth-sdxl-1.0'); +const DEPTH_B = model('controlnet-zoe-depth-sdxl'); +const POSE = model('OpenPose XL'); +const T2I_CANNY = model('T2I Canny XL', 't2i_adapter'); + +const PHOTO = { image: { height: 64, imageName: 'photo.png', width: 96 }, type: 'image' } as const; + +const controlLayer = (overrides: Partial = {}): CanvasControlLayerContract => { + const created = createControlLayer({ base: 'sdxl', id: 'control-1', name: 'Control Layer 1' }); + return { ...created, ...overrides, adapter: { ...created.adapter, ...overrides.adapter } }; +}; +/** A typed layer detected from a photo, its map current. */ +const detectedLayer = ( + controlType: CanvasControlLayerContract['controlType'], + detector: string, + overrides: Partial = {} +) => + controlLayer({ + controlType, + detection: { settings: {}, type: detector }, + filter: { settings: {}, type: detector }, + original: PHOTO, + source: PHOTO, + ...overrides, + }); + let host: HTMLDivElement | null = null; let root: Root | null = null; +const queryClient = new QueryClient(); -const layer = { - adapter: { - beginEndStepPct: [0, 1], - controlMode: 'balanced', - kind: 'controlnet', - model: 'sd1-controlnet', - weight: 1, - }, - filter: null, - id: 'control-1', - isEnabled: true, - isLocked: false, - name: 'Control 1', - opacity: 1, - type: 'control', - withTransparencyEffect: false, -} as unknown as CanvasControlLayerContract; - -const ignoreOperationStarted = (): void => undefined; - -const settle = async (run: () => void = () => undefined) => { +const settle = async (run: () => unknown = () => undefined) => { await act(async () => { - run(); + await run(); await new Promise((resolve) => { globalThis.setTimeout(resolve, 0); }); @@ -95,47 +128,30 @@ const settle = async (run: () => void = () => undefined) => { type Engine = NonNullable[0]['engine']>; -/** Committed document edits, in order; each carries the adapter patch the panel prepared. */ const commits: { label: string; edit: unknown }[] = []; - -/** Live previews the panel sent, and the baselines a cancelled gesture restored. */ const previews: CanvasLayerPreviewMutation[] = []; const restores: CanvasLayerPreviewMutation[] = []; -let editingLocked = false; -let previewsRefused = false; -/** Cleared when the engine ends the open session, as an undo landing mid-gesture does. */ -let sessionActive = true; - -const modelOf = (layers: readonly CanvasLayerContract[], editRevision = 0) => - createDocumentModel( - { ...createEmptyCanvasDocument(), stacks: stacksFrom(layers) }, - { editRevision, projectId: 'test-project' } - ); - -/** - * Just enough engine for the panel's reads and commits: every layer has content and contributes, in order. - * `replaceLayers` swaps the document the panel reads, as an undo landing mid-gesture would. - */ -const engineWith = ( - layers: CanvasControlLayerContract[], - { hasContent = true, operations = {} }: { hasContent?: boolean; operations?: object } = {} -): Engine & { replaceLayers(next: CanvasLayerContract[]): void } => { - let revision = 0; - let model = modelOf(layers); +const intents: ControlLayerIntent[] = []; +const retries: string[] = []; +let intentResult: ControlLayerIntentResult = 'updated'; +const statuses = createControlDetectionStatusStore(); + +const engineWith = (layers: CanvasLayerContract[], { hasContent = true } = {}): Engine => { + const document = { ...createEmptyCanvasDocument(), stacks: stacksFrom(layers) }; + const documentModel = createDocumentModel(document, { editRevision: 0, projectId: 'test-project' }); const engine = { - document: { model: () => model }, - exports: { hasExportableLayerContent: () => hasContent }, - interaction: { get: () => editingLocked, subscribe: () => () => undefined }, + document: { getDocument: () => document, model: () => documentModel }, + exports: { + exportRasterComposite: () => ({ status: 'empty' }), + hasExportableLayerContent: () => hasContent, + }, + interaction: { get: () => false, subscribe: () => () => undefined }, layers: { beginStructuralPreview: () => { - if (previewsRefused) { - return null; - } - // As the engine's session: the first preview captures what it replaces, and cancel restores it. let baseline: CanvasLayerPreviewMutation | null = null; return { apply: (action: CanvasLayerPreviewMutation) => { - baseline ??= previewInverse(model.getNode(action.id)!, action); + baseline ??= previewInverse(documentModel.getNode(action.id)!, action); previews.push(action); return true; }, @@ -149,7 +165,7 @@ const engineWith = ( commits.push({ edit, label }); return { status: 'committed' as const }; }, - isActive: () => sessionActive, + isActive: () => true, }; }, commitPrepared: (label: string, edit: unknown) => { @@ -158,551 +174,509 @@ const engineWith = ( }, endStructuralPreview: () => undefined, }, - replaceLayers: (next: CanvasLayerContract[]) => { - model = modelOf(next, (revision += 1)); + lifecycle: { flushPendingUploads: () => Promise.resolve() }, + projectId: 'test-project', + tools: { setTool: vi.fn() }, + } as unknown as Engine; + attachCanvasOperations(engine, { + applyControlLayerIntent: (_layerId: string, intent: ControlLayerIntent) => { + intents.push(intent); + return intentResult; + }, + getControlDetectionStatus: statuses.get, + retryControlDetection: (layerId: string) => { + retries.push(layerId); + return 'scheduled'; }, - } as unknown as Engine & { replaceLayers(next: CanvasLayerContract[]): void }; - attachCanvasOperations(engine, operations as never); + subscribeControlDetectionStatus: statuses.subscribe, + } as never); return engine; }; -const render = async (controlLayer: CanvasControlLayerContract = layer, engine: Engine | null = null) => { - applyThemeToRoot('classic'); +const render = async ( + layer: CanvasControlLayerContract, + { engine = engineWith([layer]), width = 320 }: { engine?: Engine | null; width?: number } = {} +) => { + applyThemeToRoot(DEFAULT_THEME_ID); host = document.createElement('div'); - host.style.width = '260px'; + host.style.width = `${width}px`; document.body.append(host); root = createRoot(host); await settle(() => root?.render( - + + + ) ); }; +const unmount = async () => { + await act(() => root?.unmount()); + host?.remove(); + host = null; + root = null; +}; -/** The kind Select renders a hidden native select, so its offered kinds are readable without opening it. */ -const offersKind = (kind: string) => host!.querySelector(`option[value="${kind}"]`) !== null; -const textOf = (role: 'alert' | 'status') => - [...host!.querySelectorAll(`[role="${role}"]`)].map((element) => element.textContent).join(' '); -const retryButton = () => [...host!.querySelectorAll('button')].find((button) => button.textContent === 'common.retry'); -/** The model Select's hidden native options, as the kind ones are read. */ -const modelOptions = () => - [...host!.querySelectorAll('select')] - .flatMap((select) => [...select.options]) - .filter((option) => catalog.models.some((model) => model.key === option.value)); -const offeredModels = () => - modelOptions() - .filter((option) => !option.disabled) - .map((option) => option.value); -const reidentifyPendingModels = () => - modelOptions() - .filter((option) => option.disabled) - .map((option) => option.value); - -/** Renders `layers.at(-1)` as the selected layer of a document holding all of them, each with content. */ -const renderWith = async ( - mainModel: Record | null, - models: Record[], - ...layers: CanvasControlLayerContract[] -) => { - catalog.mainModel = mainModel as Record; - catalog.models = models; - commits.length = 0; +const loadCapabilities = async () => { getArchitectureCapabilities.mockResolvedValueOnce(architectureCapabilitiesFixture); await settle(ensureArchitectureCapabilitiesLoaded); - await render(layers.at(-1), engineWith(layers)); }; +beforeEach(async () => { + resetPropertyGroupCollapse(); + harness.models = [DEPTH_A, POSE]; + await loadCapabilities(); +}); + afterEach(async () => { - await act(() => root?.unmount()); - host?.remove(); - host = null; - root = null; + await unmount(); getArchitectureCapabilities.mockReset(); - Object.assign(catalog, SD1_CATALOG); + harness.canManageModels = true; + harness.mainModel = { base: 'sdxl', key: 'sdxl-main', name: 'SDXL', type: 'main' }; + harness.memory.length = 0; + harness.startersRequested = 0; + harness.navigate.length = 0; commits.length = 0; previews.length = 0; restores.length = 0; - editingLocked = false; - previewsRefused = false; - sessionActive = true; - // Returns the capability store and the core registry to their unloaded state between tests. + intents.length = 0; + retries.length = 0; + intentResult = 'updated'; + statuses.clear(); + resetPropertyGroupCollapse(); accountLifecycle.invalidate(); }); -describe('ControlLayerSettings and the capability table', () => { - it('says the adapter is waiting for model capabilities rather than that it is unsupported', async () => { - await render(); - - expect(textOf('status')).toContain('widgets.layers.control.capabilitiesLoading'); +const ROW_LABELS = ['Image', 'Type', 'Detection', 'Model', 'Weight', 'Steps', 'Priority']; +/** Where each row's label sits, relative to the panel. */ +const rowTops = () => { + const top = host!.getBoundingClientRect().top; + return ROW_LABELS.map((label) => { + const element = [...host!.querySelectorAll('p')].find((candidate) => candidate.textContent === label); + if (!element) { + throw new Error(`No row labelled ${label}`); + } + return Math.round(element.getBoundingClientRect().top - top); }); - - it('keeps the retry and its focus through a retry that fails again, then offers the adapter kinds', async () => { - // Capability arrival must update kinds even when the model base is unchanged. - getArchitectureCapabilities.mockRejectedValueOnce(new Error('Fixture capability outage.')); - await settle(ensureArchitectureCapabilitiesLoaded); - await render(); - - expect(textOf('alert')).toContain('widgets.layers.control.capabilitiesLoadFailed'); - expect(offersKind('controlnet')).toBe(false); - - // A retry that fails again: the button never unmounts, so a keyboard user's focus stays on it. - let failLoad: (error: Error) => void = () => undefined; - getArchitectureCapabilities.mockReturnValueOnce( - new Promise((_, reject) => { - failLoad = reject; - }) - ); - await settle(() => { - retryButton()?.focus(); - retryButton()?.click(); - }); - expect(document.activeElement).toBe(retryButton()); - expect(retryButton()?.getAttribute('aria-busy')).toBe('true'); - - await settle(() => failLoad(new Error('Still out.'))); - expect(document.activeElement).toBe(retryButton()); - - getArchitectureCapabilities.mockResolvedValueOnce(architectureCapabilitiesFixture); - await settle(() => retryButton()?.click()); - - expect(offersKind('controlnet')).toBe(true); - expect(retryButton()).toBeUndefined(); - // The button that held focus is gone; focus moved into the panel instead of falling to . - expect(document.activeElement).not.toBe(document.body); - expect(host!.contains(document.activeElement)).toBe(true); +}; +const combobox = (name: string) => page.getByRole('combobox', { exact: true, name }); +const choose = async (name: string, option: string | RegExp) => { + await act(() => combobox(name).click()); + await act(() => page.getByRole('option', { name: option }).click()); + await settle(); +}; +const adapterOf = (index: number) => + (commits[index]!.edit as { forward: { config: { adapter: Record } } }).forward.config.adapter; + +describe('ControlLayerSettings rows', () => { + it('keeps every row where it is across the empty, ready, warning, choosing and detecting states', async () => { + const states: [CanvasControlLayerContract, () => void][] = [ + [controlLayer({}), () => undefined], + [detectedLayer('depth', 'depth_anything_depth_estimation'), () => undefined], + [controlLayer({ source: PHOTO }), () => undefined], + [ + detectedLayer('depth', 'depth_anything_depth_estimation'), + () => { + // Two depth models: the Model row asks, and its caption runs to two lines. + harness.models = [DEPTH_A, DEPTH_B, POSE]; + }, + ], + [ + detectedLayer('depth', 'depth_anything_depth_estimation'), + () => statuses.set('control-1', { detector: 'depth_anything_depth_estimation', status: 'processing' }), + ], + ]; + // Captions wrap differently at each width; the rows must not care. + for (const width of [320, 450]) { + const tops: number[][] = []; + harness.models = [DEPTH_A, POSE]; + statuses.clear(); + for (const [layer, prepare] of states) { + prepare(); + await render(layer, { width }); + tops.push(rowTops()); + expect(host!.scrollWidth, 'no horizontal overflow').toBeLessThanOrEqual(host!.clientWidth); + await unmount(); + } + expect( + tops.map((entry) => entry.join(',')), + `at ${width} px` + ).toEqual(tops.map(() => tops[0]!.join(','))); + // The fixed rows fit the Properties pane budget. + expect(tops[0]!.at(-1)!).toBeLessThan(281); + } }); - it('leaves focus where the user moved it while the retry was in flight', async () => { - getArchitectureCapabilities.mockRejectedValueOnce(new Error('Fixture capability outage.')); - await settle(ensureArchitectureCapabilitiesLoaded); - await render(); - - let finishLoad: (rows: ArchitectureCapabilitiesRow[]) => void = () => undefined; - getArchitectureCapabilities.mockReturnValueOnce( - new Promise((resolve) => { - finishLoad = resolve; - }) - ); - await settle(() => { - retryButton()?.focus(); - retryButton()?.click(); - }); - - // Someone starts typing elsewhere; the load landing must not pull them into this panel. - const elsewhere = document.createElement('input'); - document.body.append(elsewhere); - elsewhere.focus(); - try { - await settle(() => finishLoad(architectureCapabilitiesFixture)); - - expect(retryButton()).toBeUndefined(); - expect(document.activeElement).toBe(elsewhere); - } finally { - elsewhere.remove(); + it('passes an accessibility audit empty, with a warning and ready', async () => { + for (const layer of [ + controlLayer({}), + controlLayer({ source: PHOTO }), + detectedLayer('depth', 'depth_anything_depth_estimation'), + ]) { + await render(layer); + expect(await auditAccessibility(host!)).toEqual([]); + await unmount(); } }); -}); -describe('ControlLayerSettings on an Anima main model', () => { - const lllite = (key: string, name: string, cond_in_channels: number | null) => ({ - base: 'anima', - cond_in_channels, - key, - name, - type: 'controlnet', - }); - const ANIMA_MODELS = [ - lllite('sketch', 'Anima LLLite Sketch', 3), - lllite('inpainting', 'Anima LLLite Inpainting', 4), - lllite('unidentified', 'Anima LLLite (old install)', null), - ]; - const animaLayer = ( - kind: string, - model: string | null, - id = 'control-1', - values: Partial = {} - ): CanvasControlLayerContract => { - const base = withModel(id, id, 'anima', model); - return { - ...base, - adapter: { ...base.adapter, ...values, kind: kind as CanvasControlLayerContract['adapter']['kind'] }, - }; - }; - const button = (name: string) => [...host!.querySelectorAll('button')].find((b) => b.textContent === name); - /** The adapter each commit wrote, read from the prepared edit's forward patch. */ - const committedAdapters = () => - commits - .map(({ edit }) => JSON.stringify(edit)) - .map((json) => { - const match = /"adapter":(\{[^{}]*"beginEndStepPct":\[[^\]]*\][^{}]*\})/u.exec(json); - return match ? (JSON.parse(match[1]!) as Record) : null; - }); - - const ANIMA_MAIN = { base: 'anima', key: 'anima-main', name: 'Anima', type: 'main' }; - const renderAnima = (...layers: CanvasControlLayerContract[]) => renderWith(ANIMA_MAIN, ANIMA_MODELS, ...layers); - - it('offers ControlNet-LLLite and its control adapters, and accepts one without a warning', async () => { - await renderAnima(animaLayer('anima_lllite', 'sketch')); - - expect(offersKind('anima_lllite')).toBe(true); - expect(offersKind('controlnet')).toBe(false); - expect(offeredModels()).toEqual(['sketch']); - expect(host!.textContent).toContain('Anima LLLite Sketch'); - expect(textOf('alert')).toBe(''); + it('flags an image without a type and explains that the model follows the type', async () => { + await render(controlLayer({ source: PHOTO })); + + const type = combobox('Type'); + await expect.element(type).toHaveAttribute('aria-invalid', 'true'); + await expect.element(type).toHaveAccessibleDescription('Choose a type and the model follows.'); + await expect.element(combobox('Detection')).toBeDisabled(); + + // A missing type warns; it is not an error, so neither the border nor the focus ring turns red. + const trigger = type.element() as HTMLElement; + // The placeholder is written in the warning colour; the border matches it. + const warning = getComputedStyle(trigger.querySelector('[data-part="value-text"]')!).color; + expect(getComputedStyle(trigger).borderTopColor).toBe(warning); + // Keyboard focus, so the focus-visible ring applies. + trigger.focus(); + await act(() => userEvent.keyboard('{Shift>}{Tab}{/Shift}{Tab}')); + expect(document.activeElement).toBe(trigger); + expect(getComputedStyle(trigger).borderTopColor).toBe(warning); }); +}); - it('turns a layer saved as ControlNet into LLLite in one step, keeping its model, weight and steps', async () => { - await renderAnima(animaLayer('controlnet', 'sketch', 'old', { beginEndStepPct: [0.1, 0.7], weight: 0.6 })); - - // The alert names the fix, and the model list does not present any model as usable under the wrong kind. - expect(textOf('alert')).toContain('widgets.layers.control.validation.switch_adapter_kind'); - expect(offeredModels()).toEqual([]); - expect(host!.textContent).toContain('Anima LLLite Sketch'); +describe('ControlLayerSettings type and detection', () => { + it('hints each type with what is installed, and renames an auto-named layer after the type', async () => { + harness.models = [DEPTH_A, DEPTH_B, POSE]; + await render(controlLayer({ source: PHOTO })); - await settle(() => button('widgets.layers.control.switchKind')?.click()); + await act(() => combobox('Type').click()); + for (const name of ['Depth, 2 models', 'Pose, 1 model', 'Line art, Not installed']) { + await expect.element(page.getByRole('option', { exact: true, name })).toBeVisible(); + } + await act(() => page.getByRole('option', { name: /^Depth/ }).click()); + await settle(); - expect(committedAdapters()).toEqual([ - { beginEndStepPct: [0.1, 0.7], controlMode: null, kind: 'anima_lllite', model: 'sketch', weight: 0.6 }, - ]); + expect(intents).toEqual([{ controlType: 'depth', kind: 'set-type', name: 'Depth 1' }]); }); - it('asks for the switch, not a model, on a model-less layer saved as ControlNet', async () => { - await renderAnima(animaLayer('controlnet', null)); + it('keeps a name the user gave the layer', async () => { + await render(controlLayer({ name: 'Background pose', source: PHOTO })); + await choose('Type', /^Pose/); - expect(textOf('alert')).toContain('widgets.layers.control.validation.switch_adapter_kind'); - expect(textOf('alert')).not.toContain('missing_model'); + expect(intents).toEqual([{ controlType: 'pose', kind: 'set-type', name: undefined }]); }); - it('switches through the Adapter type select the same way', async () => { - await renderAnima(animaLayer('controlnet', 'sketch', 'old', { beginEndStepPct: [0.1, 0.7], weight: 0.6 })); + it("offers the type's detectors, then None after a divider", async () => { + await render(detectedLayer('depth', 'depth_anything_depth_estimation')); - await act(() => page.getByRole('combobox', { name: 'widgets.layers.control.kind' }).click()); - await act(() => page.getByRole('option', { name: 'widgets.layers.control.kinds.anima_lllite' }).click()); + await act(() => combobox('Detection').click()); + const options = [...document.querySelectorAll('[role="option"]')].map((item) => item.textContent); + expect(options).toEqual(['Depth Anything', 'Normal map', 'None: image is already a depth map']); + await act(() => page.getByRole('option', { name: 'Normal map' }).click()); await settle(); + await choose('Detection', 'None: image is already a depth map'); - expect(committedAdapters()).toEqual([ - { beginEndStepPct: [0.1, 0.7], controlMode: null, kind: 'anima_lllite', model: 'sketch', weight: 0.6 }, + expect(intents).toEqual([ + { detector: 'normal_map', kind: 'set-detector' }, + { detector: null, kind: 'set-detector' }, ]); }); - it('explains an empty LLLite list instead of only asking for a model', async () => { - await renderWith(ANIMA_MAIN, ANIMA_MODELS.slice(1), animaLayer('anima_lllite', null)); + it('shows a spinner and says what runs, then offers Run again after a failure', async () => { + await render(detectedLayer('depth', 'depth_anything_depth_estimation')); - expect(offeredModels()).toEqual([]); - expect(host!.textContent).toContain('widgets.layers.control.hiddenModels.lllite_inpaint_adapter'); - expect(host!.textContent).toContain('widgets.layers.control.hiddenModels.lllite_channels_unknown'); - expect(textOf('alert')).toBe(''); + await settle(() => + statuses.set('control-1', { detector: 'depth_anything_depth_estimation', status: 'processing' }) + ); + expect(combobox('Detection').element().querySelector('.chakra-spinner')).not.toBeNull(); + expect( + page + .getByRole('status') + .elements() + .map((element) => element.textContent) + ).toContain('Running Depth Anything…'); + + await settle(() => statuses.set('control-1', { error: { code: 'process-failed' }, status: 'error' })); + expect(combobox('Detection').element().querySelector('.chakra-spinner')).toBeNull(); + expect( + page + .getByRole('status') + .elements() + .map((element) => element.textContent) + ).toContain('Depth Anything failed: the detector could not run'); + await act(() => page.getByRole('button', { name: 'Run again' }).click()); + + expect(retries).toEqual(['control-1']); }); - it('lists adapters that need re-identifying as disabled entries after the usable ones', async () => { - const models = [...ANIMA_MODELS, lllite('unidentified-2', 'Anima LLLite (older install)', null)]; - await renderWith(ANIMA_MAIN, models, animaLayer('anima_lllite', 'sketch')); - - expect(offeredModels()).toEqual(['sketch']); - expect(reidentifyPendingModels()).toEqual(['unidentified', 'unidentified-2']); - - await act(() => page.getByRole('combobox', { name: 'widgets.layers.control.model' }).click()); - const options = [...document.querySelectorAll('[role="option"]')]; - // The suffix is text in the option's name, not only a muted color. - expect(options.map((option) => option.textContent)).toEqual([ - 'Anima LLLite Sketch', - 'Anima LLLite (old install) (widgets.layers.control.needsReidentify)', - 'Anima LLLite (older install) (widgets.layers.control.needsReidentify)', - ]); - expect(options.map((option) => option.getAttribute('aria-disabled'))).toEqual([null, 'true', 'true']); + it('offers Detect again for an out-of-date map once nothing is running', async () => { + // The map was detected with other settings, as after an undo of a settings change. + await render( + detectedLayer('depth', 'depth_anything_depth_estimation', { + filter: { settings: { model_size: 'large' }, type: 'depth_anything_depth_estimation' }, + }) + ); + await act(() => page.getByRole('button', { name: 'Detect again' }).click()); - // Neither a click nor Enter on a disabled entry chooses it. - await act(() => options[1]!.click()); - await act(() => userEvent.keyboard('{ArrowDown}{ArrowDown}{Enter}')); - await settle(); - expect(commits).toEqual([]); + expect(retries).toEqual(['control-1']); }); - it('turns an entry usable once the store holds its re-identified record', async () => { - const layerWithoutModel = animaLayer('anima_lllite', null); - await renderWith(ANIMA_MAIN, [ANIMA_MODELS[2]!], layerWithoutModel); - expect(offeredModels()).toEqual([]); - expect(reidentifyPendingModels()).toEqual(['unidentified']); - - // The models store replaced the record; the mocked selector reads the catalog on the next render. - catalog.models = [{ ...ANIMA_MODELS[2]!, cond_in_channels: 3 }]; - await settle(() => - root?.render( - - - - - - ) + it('opens the detector settings inline and sends one intent per scrub gesture', async () => { + await render( + detectedLayer('edges', 'canny_edge_detection', { detection: { settings: {}, type: 'canny_edge_detection' } }) ); + const toggle = page.getByRole('button', { name: 'Detector settings' }); + await expect.element(toggle).toHaveAttribute('aria-expanded', 'false'); - expect(reidentifyPendingModels()).toEqual([]); - expect(offeredModels()).toEqual(['unidentified']); - }); + await act(() => toggle.click()); + await expect.element(toggle).toHaveAttribute('aria-expanded', 'true'); + const region = document.getElementById(toggle.element().getAttribute('aria-controls')!)!; + expect(region.getAttribute('role')).toBe('group'); - it('flags a second layer applying the same LLLite model, but not the first', async () => { - const first = animaLayer('anima_lllite', 'sketch', 'first'); - const second = animaLayer('anima_lllite', 'sketch', 'second'); - await renderAnima(first, second); - expect(textOf('alert')).toBe('widgets.layers.control.validation.duplicate_lllite_model'); + const frame = region.querySelector('[data-scope="scrubber"]')!; + const rect = frame.getBoundingClientRect(); + const pointer = (target: EventTarget, type: string, x: number) => + settle(() => + target.dispatchEvent(new PointerEvent(type, { bubbles: true, button: 0, clientX: x, pointerId: 1 })) + ); + await pointer(frame, 'pointerdown', rect.left + 10); + await pointer(window, 'pointermove', rect.left + 40); + await pointer(window, 'pointermove', rect.left + 80); + await pointer(window, 'pointerup', rect.left + 80); - await act(() => root?.unmount()); - host?.remove(); - await render(first, engineWith([first, second])); - expect(textOf('alert')).toBe(''); + expect(intents.map((intent) => intent.kind)).toEqual(['set-detector-settings']); }); - it('names an unusable adapter by its catalog name beside the alert about it', async () => { - await renderAnima(animaLayer('anima_lllite', 'inpainting')); + it('says why None has no settings', async () => { + await render( + detectedLayer('tile', 'canny_edge_detection', { detection: null, filter: undefined, original: undefined }) + ); - expect(textOf('alert')).toBe('widgets.layers.control.validation.lllite_inpaint_adapter'); - expect(host!.textContent).toContain('Anima LLLite Inpainting'); - expect(host!.textContent).not.toContain('widgets.layers.control.selectModel'); + const toggle = page.getByRole('button', { name: 'Detector settings' }); + await expect.element(toggle).toHaveAttribute('aria-disabled', 'true'); + await expect.element(toggle).toHaveAccessibleDescription('No settings for None: Tile uses the image directly'); }); +}); - it('offers ControlNet for an LLLite layer once the main model is not Anima, dropping the LLLite model', async () => { - await renderWith( - { base: 'sdxl', key: 'sdxl-main', name: 'SDXL', type: 'main' }, - [...ANIMA_MODELS, { base: 'sdxl', key: 'sdxl-control', name: 'SDXL ControlNet', type: 'controlnet' }], - animaLayer('anima_lllite', 'sketch', 'lllite', { beginEndStepPct: [0.2, 0.8], weight: 0.9 }) - ); - expect(textOf('alert')).toContain('widgets.layers.control.validation.switch_adapter_kind'); +describe('ControlLayerSettings model', () => { + it('asks for a choice among several models, and records a pick for the layer and for later layers', async () => { + harness.models = [DEPTH_A, DEPTH_B]; + await render(detectedLayer('depth', 'depth_anything_depth_estimation')); - await settle(() => button('widgets.layers.control.switchKind')?.click()); + expect(combobox('Model').element().textContent).toContain('Choose a depth model'); + expect(host!.textContent).toContain('2 models fit: controlnet-depth-sdxl-1.0 or controlnet-zoe-depth-sdxl'); + await act(() => combobox('Model').click()); + expect(page.getByRole('group', { name: 'Made for depth' }).elements()).toHaveLength(1); + await act(() => page.getByRole('option', { name: /^controlnet-zoe-depth-sdxl/ }).click()); + await settle(); - expect(committedAdapters()).toEqual([ - { beginEndStepPct: [0.2, 0.8], controlMode: 'balanced', kind: 'controlnet', model: null, weight: 0.9 }, - ]); + expect(commits.map(({ label }) => label)).toEqual(['Model']); + expect(adapterOf(0)).toEqual({ kind: 'controlnet', model: 'controlnet-zoe-depth-sdxl' }); + expect(harness.memory).toEqual([['sdxl:depth', 'controlnet-zoe-depth-sdxl']]); }); - it('hides the base-bound LLLite kind while no main model is selected', async () => { - await renderWith(null, ANIMA_MODELS, animaLayer('controlnet', null)); + it('asks for the starter catalog when the list opens, and leaves origins out until it arrives', async () => { + harness.models = [DEPTH_A, DEPTH_B]; + await render(detectedLayer('depth', 'depth_anything_depth_estimation')); + expect(harness.startersRequested).toBe(0); - expect(offersKind('controlnet')).toBe(true); - expect(offersKind('anima_lllite')).toBe(false); - }); -}); + await act(() => combobox('Model').click()); -describe('ControlLayerSettings on a Qwen Image main model', () => { - const QWEN_MAIN = { base: 'qwen-image', key: 'qwen-main', name: 'Qwen Image 2512', type: 'main' }; - const QWEN_EDIT_MAIN = { ...QWEN_MAIN, key: 'qwen-edit', name: 'Qwen Image Edit 2511', variant: 'edit' }; - const QWEN_MODELS = [ - { base: 'qwen-image', key: 'qwen-union', name: 'Qwen Image ControlNet Union', type: 'controlnet' }, - { base: 'z-image', key: 'z-union', name: 'Z-Image ControlNet Union', type: 'controlnet' }, - ]; - /** A layer as the Layers panel creates it under a Qwen Image main model. */ - const qwenLayer = (id: string) => withModel(id, id, 'qwen-image', 'qwen-union'); - - it('offers the Qwen Image ControlNet kind and its models, and accepts one without a warning', async () => { - await renderWith(QWEN_MAIN, QWEN_MODELS, qwenLayer('control-1')); - - expect(offersKind('qwen_image_control')).toBe(true); - expect(offersKind('controlnet')).toBe(false); - expect(offersKind('z_image_control')).toBe(false); - expect(offeredModels()).toEqual(['qwen-union']); - expect(host!.textContent).toContain('Qwen Image ControlNet Union'); - expect(textOf('alert')).toBe(''); + expect(harness.startersRequested).toBe(1); + await expect + .element(page.getByRole('option', { exact: true, name: 'controlnet-zoe-depth-sdxl, ControlNet' })) + .toBeVisible(); }); - it('says the control cannot run on a Qwen Image Edit model', async () => { - await renderWith(QWEN_EDIT_MAIN, QWEN_MODELS, qwenLayer('control-1')); + it('describes the Model choice with its caption', async () => { + await render(detectedLayer('depth', 'depth_anything_depth_estimation')); - expect(textOf('alert')).toBe('widgets.layers.control.validation.qwen_image_edit_control'); + await expect.element(combobox('Model')).toHaveAccessibleDescription('ControlNet · your only depth model'); }); - it('flags the second of two controls, but not the first', async () => { - const first = qwenLayer('first'); - const second = qwenLayer('second'); - await renderWith(QWEN_MAIN, QWEN_MODELS, first, second); - expect(textOf('alert')).toBe('widgets.layers.control.validation.qwen_image_control_limit'); + it('resets a pick back to automatic and returns focus to the model choice', async () => { + harness.models = [DEPTH_A, DEPTH_B]; + const picked = detectedLayer('depth', 'depth_anything_depth_estimation', { + adapter: { ...controlLayer().adapter, model: 'controlnet-zoe-depth-sdxl' }, + }); + await render(picked); + expect(host!.textContent).toContain('ControlNet · your pick for depth'); - await act(() => root?.unmount()); - host?.remove(); - await render(first, engineWith([first, second])); - expect(textOf('alert')).toBe(''); - }); -}); + await act(() => page.getByRole('button', { name: 'Reset' }).click()); + await settle(); -describe('ControlLayerSettings on a FLUX.2 main model', () => { - const FLUX2_DEV_MAIN = { base: 'flux2', key: 'flux2-dev', name: 'FLUX.2 [dev]', type: 'main', variant: 'dev' }; - const FLUX2_KLEIN_MAIN = { ...FLUX2_DEV_MAIN, key: 'flux2-klein', name: 'FLUX.2 Klein 9B', variant: 'klein_9b' }; - const FLUX2_MODELS = [ - { base: 'flux2', key: 'flux2-union', name: 'FLUX.2 [dev] ControlNet Union', type: 'controlnet' }, - { base: 'flux', key: 'flux-union', name: 'FLUX.1 ControlNet Union', type: 'controlnet' }, - ]; - - it('offers the FLUX.2 ControlNet kind and its models on [dev], and accepts one without a warning', async () => { - await renderWith(FLUX2_DEV_MAIN, FLUX2_MODELS, withModel('control-1', 'control-1', 'flux2', 'flux2-union')); - - expect(offersKind('flux2_control')).toBe(true); - expect(offersKind('controlnet')).toBe(false); - expect(offeredModels()).toEqual(['flux2-union']); - expect(host!.textContent).toContain('FLUX.2 [dev] ControlNet Union'); - expect(textOf('alert')).toBe(''); + expect(adapterOf(0)).toEqual({ model: null }); + expect(harness.memory).toEqual([['sdxl:depth', null]]); + expect(document.activeElement).toBe(combobox('Model').element()); }); - it('tells a Klein user on a fresh, empty layer that the ControlNet cannot run, before asking for a model', async () => { - catalog.mainModel = FLUX2_KLEIN_MAIN; - catalog.models = FLUX2_MODELS; - getArchitectureCapabilities.mockResolvedValueOnce(architectureCapabilitiesFixture); - await settle(ensureArchitectureCapabilitiesLoaded); - const fresh = withModel('control-1', 'control-1', 'flux2', null); - const engine = engineWith([fresh]); - engine.exports.hasExportableLayerContent = () => false; - - await render(fresh, engine); - - expect(textOf('alert')).toBe('widgets.layers.control.validation.flux2_klein_control'); + it('names the only model, and points to the Model Manager when none fits', async () => { + await render(detectedLayer('depth', 'depth_anything_depth_estimation')); + expect(host!.textContent).toContain('ControlNet · your only depth model'); + await unmount(); + + await render(detectedLayer('lineart', 'lineart_edge_detection')); + expect(combobox('Model').element().textContent).toContain('None for SDXL'); + await act(() => page.getByRole('button', { name: 'Find one' }).click()); + expect(harness.navigate).toEqual([{ search: { project: 'test-project' }, to: '/models' }]); + await unmount(); + + // Without the right to manage models, the caption says it without the link. + harness.canManageModels = false; + harness.models = []; + await render(detectedLayer('depth', 'depth_anything_depth_estimation')); + expect(host!.textContent).toContain('No control models are installed for SDXL.'); + expect(page.getByRole('button', { name: 'Install starter models' }).elements()).toHaveLength(0); }); }); -describe('ControlLayerSettings weight', () => { - // A full document layer (the panel reads its leaf), weighted 1 so a drag can move either way. - const weighted = ((): CanvasControlLayerContract => { - const base = withModel('Control 1', 'control-1', 'sd-1', 'sd1-controlnet'); - return { ...base, adapter: { ...base.adapter, weight: 1 } }; - })(); - const weightSlider = () => page.getByRole('slider', { name: 'widgets.layers.control.weight' }); - const weightOf = (edit: unknown, side: 'forward' | 'inverse') => - (edit as Record)[side].config.adapter.weight; - /** - * One drag on the weight scrubber through each offset (fractions of its 0–2 track), released at the last unless - * `release` is false. - */ - const dragWeight = async (offsets: number[], release = true) => { - const frame = weightSlider().element().closest('[data-scope="scrubber"]')!; - const rect = frame.getBoundingClientRect(); - const startX = rect.left + rect.width / 2; - const xAt = (offset: number) => startX + offset * (rect.width - 20); +describe('ControlLayerSettings adapter rows', () => { + const weightSlider = () => page.getByRole('slider', { name: 'Weight' }); + + it('records one step for a weight drag, from the weight it started at', async () => { + const layer = detectedLayer('depth', 'depth_anything_depth_estimation', { + adapter: { ...controlLayer().adapter, weight: 1 }, + }); + await render(layer); + const control = weightSlider() + .element() + .closest('[data-part="root"]')! + .querySelector('[data-part="control"]')!; + const rect = control.getBoundingClientRect(); + const xAt = (weight: number) => rect.left + (rect.width * weight) / 2; const pointer = (target: EventTarget, type: string, x: number) => settle(() => - target.dispatchEvent(new PointerEvent(type, { bubbles: true, button: 0, clientX: x, pointerId: 1 })) + target.dispatchEvent( + new PointerEvent(type, { + bubbles: true, + button: 0, + clientX: x, + clientY: rect.top + rect.height / 2, + pointerId: 1, + }) + ) ); - await pointer(frame, 'pointerdown', startX); - for (const offset of offsets) { - await pointer(window, 'pointermove', xAt(offset)); - } - if (release) { - await pointer(window, 'pointerup', xAt(offsets.at(-1) ?? 0)); - } - }; - const typeWeight = async (text: string) => { - await act(async () => { - (weightSlider().element() as HTMLElement).focus(); - await userEvent.keyboard(`{Enter}${text}{Enter}`); - }); - }; - - it('previews a drag and records it as one step from the weight it started at', async () => { - await render(weighted, engineWith([weighted])); - await dragWeight([0.1, 0.25]); + await pointer(control, 'pointerdown', xAt(1)); + await pointer(document, 'pointermove', xAt(1.25)); + await pointer(document, 'pointermove', xAt(1.5)); + await pointer(document, 'pointerup', xAt(1.5)); expect(previews.length).toBeGreaterThanOrEqual(2); expect(commits).toHaveLength(1); - expect(weightOf(commits[0]!.edit, 'forward')).toBe(1.5); - expect(weightOf(commits[0]!.edit, 'inverse')).toBe(1); + const { forward, inverse } = commits[0]!.edit as Record< + 'forward' | 'inverse', + { config: { adapter: { weight: number } } } + >; + // The release lands within a step of where the drag ended, recorded from where it began. + expect(forward.config.adapter.weight).toBeCloseTo(1.5, 0); + expect(forward.config.adapter.weight).toBe( + (previews.at(-1) as { config: { adapter: { weight: number } } }).config.adapter.weight + ); + expect(inverse.config.adapter.weight).toBe(1); }); - it('says nothing when an undo mid-drag turns the layer back into raster and the panel unmounts', async () => { - const engine = engineWith([weighted]); - await render(weighted, engine); - await dragWeight([0.25], false); - expect(previews.length).toBeGreaterThan(0); - - // The undo ends the engine's session and the document no longer holds a control layer, so the layer list - // unmounts this panel; the unmount settles the drag against a document that refuses it. - sessionActive = false; - engine.replaceLayers([createEmptyPaintLayer(weighted.name, weighted.id)]); - const created = vi.spyOn(toaster, 'create'); - try { - await act(() => root?.unmount()); - root = null; + it('clamps typed weights to the bounds the model allows', async () => { + await render(detectedLayer('depth', 'depth_anything_depth_estimation')); + const input = page.getByRole('spinbutton', { name: 'Weight' }); + await act(async () => { + await userEvent.clear(input); + await userEvent.type(input, '-3{Enter}'); + }); + await act(async () => { + await userEvent.clear(input); + await userEvent.type(input, '9{Enter}'); + }); - expect(commits).toHaveLength(0); - expect(created).not.toHaveBeenCalled(); - } finally { - created.mockRestore(); - } + expect(commits.map((_, index) => adapterOf(index).weight)).toEqual([-1, 2]); }); - it('records nothing and restores the weight when a drag returns to where it started', async () => { - await render(weighted, engineWith([weighted])); - await dragWeight([0.25, 0]); + it('reads out the step range', async () => { + await render( + detectedLayer('depth', 'depth_anything_depth_estimation', { + adapter: { ...controlLayer().adapter, beginEndStepPct: [0.1, 0.75] }, + }) + ); - expect(commits).toHaveLength(0); - expect(restores).toEqual([ - { config: { adapter: { weight: 1 }, layerType: 'control' }, id: 'control-1', type: 'updateCanvasLayerConfig' }, - ]); + expect(host!.textContent).toContain('10–75%'); }); - it('clamps typed weights to the typed bounds, which reach past the track', async () => { - await render(weighted, engineWith([weighted])); - await typeWeight('-3'); - - expect(commits.map(({ edit }) => weightOf(edit, 'forward'))).toEqual([-1]); - - await typeWeight('9'); - - expect(commits.map(({ edit }) => weightOf(edit, 'forward'))).toEqual([-1, 2]); + it('sets the priority for a ControlNet and dims it, with the reason, for a T2I Adapter', async () => { + await render(detectedLayer('depth', 'depth_anything_depth_estimation')); + await act(() => page.getByRole('radiogroup', { name: 'Priority' }).getByText('Control', { exact: true }).click()); + await settle(); + expect(adapterOf(0)).toEqual({ controlMode: 'more_control' }); + await unmount(); + + harness.models = [T2I_CANNY]; + await render(detectedLayer('edges', 'canny_edge_detection')); + const group = page.getByRole('radiogroup', { name: 'Priority' }); + await expect.element(group).toHaveAccessibleDescription('Priority applies to ControlNet models only.'); + await expect.element(page.getByRole('radio', { name: 'Control' })).toBeDisabled(); }); +}); - it('still attempts a typed weight when previews are refused, so the refusal is reported', async () => { - previewsRefused = true; - await render(weighted, engineWith([weighted])); - await typeWeight('0.5'); - - expect(commits.map(({ edit }) => weightOf(edit, 'forward'))).toEqual([0.5]); +describe('ControlLayerSettings and the capability table', () => { + const retryButton = () => [...host!.querySelectorAll('button')].find((button) => button.textContent === 'Retry'); + + it('says the layer is checked once model capabilities load', async () => { + accountLifecycle.invalidate(); + await render(detectedLayer('depth', 'depth_anything_depth_estimation')); + + expect( + page + .getByRole('status') + .elements() + .map((element) => element.textContent) + ).toContain('This layer is checked once model capabilities have loaded.'); }); - it('disables the weight while document editing is locked', async () => { - editingLocked = true; - await render(weighted, engineWith([weighted])); + it('keeps the retry and its focus through a retry that fails again, then hands focus to the panel', async () => { + accountLifecycle.invalidate(); + getArchitectureCapabilities.mockRejectedValueOnce(new Error('Fixture capability outage.')); + await settle(ensureArchitectureCapabilitiesLoaded); + await render(detectedLayer('depth', 'depth_anything_depth_estimation')); - await expect.element(weightSlider()).toHaveAttribute('aria-disabled', 'true'); - }); -}); + let failLoad: (error: Error) => void = () => undefined; + getArchitectureCapabilities.mockReturnValueOnce( + new Promise((_, reject) => { + failLoad = reject; + }) + ); + await settle(() => { + retryButton()?.focus(); + retryButton()?.click(); + }); + expect(document.activeElement).toBe(retryButton()); + expect(retryButton()?.getAttribute('aria-busy')).toBe('true'); -describe('ControlLayerSettings model choice', () => { - const CANNY = { - base: 'sd-1', - default_settings: { preprocessor: 'canny_image_processor' }, - key: 'sd1-canny', - name: 'SD 1.5 Canny', - type: 'controlnet', - }; - const modelless = withModel('Control 1', 'control-1', 'sd-1', null); - - const chooseCanny = async (hasContent: boolean) => { - catalog.models = [CANNY]; - getArchitectureCapabilities.mockResolvedValueOnce(architectureCapabilitiesFixture); - await settle(ensureArchitectureCapabilitiesLoaded); - const startFilterOperation = vi.fn(() => 'started' as const); - await render(modelless, engineWith([modelless], { hasContent, operations: { startFilterOperation } })); - await act(() => page.getByRole('combobox', { name: 'widgets.layers.control.model' }).click()); - await act(() => page.getByRole('option', { name: 'SD 1.5 Canny' }).click()); - await settle(); - return startFilterOperation; - }; + await settle(() => failLoad(new Error('Still out.'))); + expect(document.activeElement).toBe(retryButton()); - it("starts the model's recommended filter on a layer with pixels", async () => { - const startFilterOperation = await chooseCanny(true); + getArchitectureCapabilities.mockResolvedValueOnce(architectureCapabilitiesFixture); + await settle(() => retryButton()?.click()); - expect(commits.map(({ label }) => label)).toEqual(['widgets.layers.control.model']); - expect(startFilterOperation).toHaveBeenCalledExactlyOnceWith('control-1', 'canny_edge_detection'); + expect(retryButton()).toBeUndefined(); + expect(host!.contains(document.activeElement)).toBe(true); }); +}); - it('only sets the model on a layer without pixels yet, leaving nothing to filter', async () => { - const startFilterOperation = await chooseCanny(false); +describe('ControlLayerSettings intent refusals', () => { + it.each([ + ['blocked', 'The canvas is busy. Try again when the current action finishes.'], + ['locked', 'Unlock the layer before using this action.'], + ['no-detection-input', 'Replace the image to detect again'], + ['over-budget', 'Layer action failed'], + ] as const)('says why a %s edit did not land', async (result, title) => { + const created = vi.spyOn(toaster, 'create'); + try { + intentResult = result; + await render(detectedLayer('depth', 'depth_anything_depth_estimation')); + await choose('Detection', 'Normal map'); - expect(commits.map(({ label }) => label)).toEqual(['widgets.layers.control.model']); - expect(startFilterOperation).not.toHaveBeenCalled(); + expect(intents).toEqual([{ detector: 'normal_map', kind: 'set-detector' }]); + expect(created).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ title })); + } finally { + created.mockRestore(); + } }); }); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.tsx index 4c215e1ba41..047940f818e 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlLayerSettings.tsx @@ -1,488 +1,391 @@ -import type { SelectValueChangeDetails, SliderValueChangeDetails } from '@chakra-ui/react'; import type { ArchitectureCapabilitiesSnapshot } from '@features/generation/runtime'; +import type { ModelConfig } from '@features/models'; import type { CanvasControlAdapterContract, CanvasControlLayerContract, - CanvasDocumentCapability, + CanvasStructuralEngine, + ControlType, } from '@workbench/canvas-engine/api'; -import type { LayerFilterOperationEngine } from '@workbench/widgets/layers/LayerFilterOperationButton'; -import type { CanvasStructuralEngine } from '@workbench/widgets/layers/layerOps'; +import type { ControlLayerIntent, ControlLayerIntentResult } from '@workbench/canvas-operations/api'; +import type { CanvasEngineHandle } from '@workbench/canvas-operations/react'; -import { createListCollection, HStack, Stack, Switch, Text } from '@chakra-ui/react'; -import { - CONTROL_ADAPTER_KINDS, - getControlAdapterMinWeight, - getControlModelUnusableReason, - getSuggestedControlKind, - isControlKindSupportedForBase, - type ControlAdapterKind, -} from '@features/generation/graph'; +import { HStack, Separator, Stack, Text } from '@chakra-ui/react'; +import { getControlModelUnusableReason } from '@features/generation/graph'; import { ensureArchitectureCapabilitiesLoaded, getArchitectureCapabilitiesSnapshot, subscribeArchitectureCapabilities, } from '@features/generation/runtime'; -import { useModelsSelector } from '@features/models'; +import { useCapabilities } from '@features/identity'; +import { getModelBaseLabel, useModelsSelector, useStartersSelector } from '@features/models'; import { focusFirstOperable } from '@platform/react/focusIfUnclaimed'; import { useExternalStoreSelector } from '@platform/state/selectors'; -import { Button, Field, Select, Slider } from '@platform/ui'; -import { ScrubberField } from '@platform/ui/ScrubberField'; -import { getCanvasOperations, resolveDefaultFilterForModel } from '@workbench/canvas-operations/api'; -import { CONTROL_ADAPTER_DEFAULTS, CONTROL_KIND_BASE } from '@workbench/controlAdapters'; -import { describeControlLayerReason, getControlLayerReasonInSequence } from '@workbench/controlLayerChecks'; -import { useCanvasDocumentEditingLocked, useCanvasEngineRead } from '@workbench/widgets/canvas/engineStoreHooks'; +import { Button } from '@platform/ui/Button'; +import { useNavigate } from '@tanstack/react-router'; +import { CONTROL_TYPES, getDocumentLeaves } from '@workbench/canvas-engine/api'; +import { getCanvasOperations } from '@workbench/canvas-operations/api'; +import { deriveControlKind } from '@workbench/controlAdapters'; +import { createControlLayerCheckContext, describeControlLayer } from '@workbench/controlLayerChecks'; +import { controlModelMemoryKey, getTypeAvailability } from '@workbench/controlModelResolution'; +import { patchControlModelMemory, useWorkbenchPreferenceSelector } from '@workbench/settings/store'; +import { useNotify } from '@workbench/useNotify'; +import { + useCanvasDocumentEditingLocked, + useCanvasEngineRead, + useControlDetectionStatus, +} from '@workbench/widgets/canvas/engineStoreHooks'; import { baselineConfig, useStructuralPreview } from '@workbench/widgets/canvas/useStructuralCommit'; -import { useCallback, useMemo, useRef, useState } from 'react'; +import { useActiveProjectId } from '@workbench/WorkbenchContext'; +import { useCallback, useId, useMemo, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; -import { - getCompatibleControlModels, - getReidentifyPendingControlModels, - switchControlAdapterKind, -} from './controlModelOptions'; -import { LayerFilterOperationButton } from './LayerFilterOperationButton'; -import { CONTROL_WEIGHT_BOUNDS } from './layerOps'; -import { runLayerFilterOperation } from './layerPropertiesOperation'; +import { ControlAdapterRows, type ControlAdapterEditing } from './ControlAdapterRows'; +import { ControlDetectionRow } from './ControlDetectionRow'; +import { getReidentifyPendingControlModels } from './controlModelOptions'; +import { ControlModelRow } from './ControlModelRow'; +import { deriveControlPanel, getControlImageState } from './controlPanelModel'; +import { ControlSourceRow } from './ControlSourceRow'; +import { ControlTypeRow } from './ControlTypeRow'; +import { isAutoControlLayerName, nextControlTypeLayerName } from './layerOps'; +import { useControlSourceActions } from './useControlSourceActions'; import { useSelectedMainModel } from './useSelectedMainModel'; -const SELECT_POSITIONING = { placement: 'bottom-end', sameWidth: true } as const; - const selectCapabilitiesStatus = (snapshot: ArchitectureCapabilitiesSnapshot) => snapshot.status; const selectCapabilitiesRevision = (snapshot: ArchitectureCapabilitiesSnapshot) => snapshot.revision; -const CONTROL_MODES: readonly NonNullable[] = [ - 'balanced', - 'more_prompt', - 'more_control', - 'unbalanced', -]; +const selectControlModelMemory = (preferences: { controlModelMemory: Record }) => + preferences.controlModelMemory; +const selectStarterModels = (snapshot: { + status: string; + response: { starter_models: { source: string }[] } | null; +}) => (snapshot.status === 'loaded' ? (snapshot.response?.starter_models ?? null) : null); -const formatUnitPercent = (value: number): string => `${Math.round(value * 100)}%`; -const formatWeight = (value: number): string => value.toFixed(2); +/** Read from the engine in place of a reason when the layer does not contribute: its ambient warning applies. */ +const AMBIENT = 'ambient'; -/** Quiet warning tint for the model select while no control model is chosen. */ -const MISSING_MODEL_VALUE_TEXT_PROPS = { color: 'fg.warning' } as const; +/** Held back from an empty model list, so the caption can say why it is empty. */ +const HIDDEN_MODEL_REASONS: ReadonlySet = new Set(['lllite_inpaint_adapter', 'lllite_channels_unknown']); -interface ControlModelItem { - /** Listed but not selectable: the model needs re-identifying before a control layer can use it. */ - disabled?: boolean; - label: string; - value: string; -} +export type ControlLayerSettingsEngine = CanvasStructuralEngine & + Pick; interface ControlLayerSettingsProps { - engine: - | (CanvasStructuralEngine & LayerFilterOperationEngine & { readonly document: CanvasDocumentCapability }) - | null; + engine: ControlLayerSettingsEngine | null; layer: CanvasControlLayerContract; - onOperationStarted(): void; } -/** Edit adapters through canvas undo; utility-queue filter previews leave the document untouched until Apply. */ -export const ControlLayerSettings = ({ engine, layer, onOperationStarted }: ControlLayerSettingsProps) => { +/** + * The control layer Properties panel: Image · Type · Detection · Model, then Weight · Steps · Priority. The rows stay + * put; only their values change. Type, detector and image edits go through the canvas operations, which detect the + * map in the background; adapter values commit through canvas undo like any layer property. + */ +export const ControlLayerSettings = ({ engine, layer }: ControlLayerSettingsProps) => { const { t } = useTranslation(); - const { commit: commitPrepared, preview: previewStructural } = useStructuralPreview(engine); + const notify = useNotify(); + const navigate = useNavigate(); + const projectId = useActiveProjectId(); + const { canManageModels } = useCapabilities(); + const { cancel: cancelPreview, commit: commitPrepared, preview: previewStructural } = useStructuralPreview(engine); const editingLocked = useCanvasDocumentEditingLocked(engine); const models = useModelsSelector((snapshot) => snapshot.models); const mainModel = useSelectedMainModel(); - const base = mainModel?.base ?? null; - const { adapter } = layer; - const weightInputMin = getControlAdapterMinWeight(adapter.kind); - - // A previewed gesture records from where its preview started; any other change from the live adapter. - const commitAdapter = useCallback( - (next: Partial, label: string) => { - const config = { adapter: next, layerType: 'control' } as const; - commitPrepared(label, (model, baseline) => - model.prepare({ before: baselineConfig(baseline, config), config, id: layer.id, type: 'patch-config' }) - ); - }, - [commitPrepared, layer.id] + const baseId = mainModel?.base ?? null; + const remembered = useWorkbenchPreferenceSelector(selectControlModelMemory); + const starterModels = useStartersSelector(selectStarterModels); + const starterSources = useMemo( + () => (starterModels ? new Set(starterModels.map((model) => model.source)) : null), + [starterModels] ); + const captionId = useId(); const capabilitiesStatus = useExternalStoreSelector( subscribeArchitectureCapabilities, getArchitectureCapabilitiesSnapshot, selectCapabilitiesStatus ); - const [hasRequestedCapabilitiesRetry, setHasRequestedCapabilitiesRetry] = useState(false); - const rootRef = useRef(null); - const retryCapabilities = useCallback(() => { - if (hasRequestedCapabilitiesRetry && capabilitiesStatus === 'loading') { - return; - } - setHasRequestedCapabilitiesRetry(true); - // The request belongs to this retry; a later reload started elsewhere is not this panel's retry. - void ensureArchitectureCapabilitiesLoaded().then(() => setHasRequestedCapabilitiesRetry(false)); - }, [capabilitiesStatus, hasRequestedCapabilitiesRetry]); - // Transfer retry-button focus to the loaded panel during ref cleanup before removal can send it to body. - const handOverFocusOnLoad = useCallback((surface: HTMLDivElement | null) => { - if (!surface) { - return undefined; - } - return () => { - if (surface.contains(document.activeElement) && getArchitectureCapabilitiesSnapshot().status === 'loaded') { - focusFirstOperable(rootRef.current); - } - }; - }, []); - // Read supported adapter kinds inside the capability selector so successful retries replace the pre-load empty - // list. - const kindOptions = useExternalStoreSelector( + // Kind support is read from the capability table when a check runs, so its arrival must re-run the checks. + const capabilitiesRevision = useExternalStoreSelector( subscribeArchitectureCapabilities, getArchitectureCapabilitiesSnapshot, - useCallback( - () => - CONTROL_ADAPTER_KINDS.filter((kind) => - // Without a main model there is no base to offer a base's own kind for. - base ? isControlKindSupportedForBase(base, kind) : CONTROL_KIND_BASE[kind] === undefined - ), - [base] - ) + selectCapabilitiesRevision ); - // The kind to offer when this layer's kind is one the main model cannot run. - const suggestedKind = useExternalStoreSelector( - subscribeArchitectureCapabilities, - getArchitectureCapabilitiesSnapshot, - useCallback( - () => (base && !isControlKindSupportedForBase(base, adapter.kind) ? getSuggestedControlKind(base) : null), - [adapter.kind, base] - ) + const checkContext = useMemo(() => { + void capabilitiesRevision; + return createControlLayerCheckContext(baseId, remembered); + }, [baseId, capabilitiesRevision, remembered]); + const described = useMemo( + () => describeControlLayer(layer, { ...checkContext, mainModel, models }), + [checkContext, layer, mainModel, models] ); - const kindCollection = useMemo( - () => - createListCollection({ - items: kindOptions.map((kind) => ({ label: t(`widgets.layers.control.kinds.${kind}`), value: kind })), - }), - [kindOptions, t] + // Judged as Invoke judges it: contributing, content-bearing control layers above claim limited slots first. + const sequenceReason = useCanvasEngineRead(engine, () => { + if (!engine || !mainModel || !engine.exports.hasExportableLayerContent(layer.id)) { + return AMBIENT; + } + const leaves = engine.document.model()?.compileLeaves() ?? []; + const index = leaves.findIndex((leaf) => leaf.layer.id === layer.id); + if (!(leaves[index]?.contributionEnabled ?? false)) { + return AMBIENT; + } + const earlier = leaves + .slice(0, Math.max(0, index)) + .filter((leaf) => leaf.contributionEnabled && engine.exports.hasExportableLayerContent(leaf.id)) + .map((leaf) => leaf.layer) + .filter((candidate): candidate is CanvasControlLayerContract => candidate.type === 'control'); + return describeControlLayer(layer, { ...checkContext, earlier, mainModel, models }).reason; + }); + const reason = sequenceReason === AMBIENT ? described.reason : (sequenceReason as typeof described.reason); + const availability = useMemo( + () => getTypeAvailability(baseId, models, checkContext.supportsKind), + [baseId, checkContext, models] ); - - // Adapter models matching the current kind + base (mirrors the generate model list). A kind the main model cannot - // run offers none: the fix is switching kind, and listing its models would present them as usable. - const modelOptions = useMemo( - () => (suggestedKind ? [] : getCompatibleControlModels(models, base, adapter.kind)), - [adapter.kind, base, models, suggestedKind] + const status = useControlDetectionStatus(engine, layer.id); + const sourceHasPixels = useCanvasEngineRead( + engine, + () => engine?.exports.hasExportableLayerContent(layer.id) ?? false ); - // Why an empty list is empty, when installed models of the right family were held back. + const imageState = getControlImageState(layer, sourceHasPixels); + const { resolution } = described; + // Installed models of the base the list holds back: said in the caption when nothing else is left to pick. const hiddenModelReasons = useMemo(() => { - const kindBase = CONTROL_KIND_BASE[adapter.kind]; - if (modelOptions.length > 0 || !kindBase || base !== kindBase) { + if (baseId === null || resolution.model) { return []; } const reasons = models - .filter((model) => model.base === base) - .map((model) => getControlModelUnusableReason(model, adapter.kind)) - .filter((reason) => reason === 'lllite_inpaint_adapter' || reason === 'lllite_channels_unknown'); + .filter((model) => model.base === baseId) + .map((model) => { + const kind = deriveControlKind(model); + return kind ? getControlModelUnusableReason(model, kind) : null; + }) + .filter((code): code is NonNullable => code !== null && HIDDEN_MODEL_REASONS.has(code)); return [...new Set(reasons)].sort(); - }, [adapter.kind, base, modelOptions.length, models]); - // Listed after the usable models, disabled, whatever the hints setting: the entry is the only place users see them. - const reidentifyPendingModels = useMemo( - () => (suggestedKind ? [] : getReidentifyPendingControlModels(models, base, adapter.kind)), - [adapter.kind, base, models, suggestedKind] - ); - const modelCollection = useMemo( - () => - createListCollection({ - items: [ - ...modelOptions.map((model) => ({ label: model.name, value: model.key })), - ...reidentifyPendingModels.map((model) => ({ disabled: true, label: model.name, value: model.key })), - ], - }), - [modelOptions, reidentifyPendingModels] - ); - const renderModelItem = useCallback( - (item: ControlModelItem) => - item.disabled ? ( - <> - {item.label}{' '} - - ({t('widgets.layers.control.needsReidentify')}) - - - ) : ( - item.label - ), - [t] + }, [baseId, models, resolution.model]); + const reidentifyPending = useMemo( + () => (layer.controlType ? getReidentifyPendingControlModels(models, baseId) : []), + [baseId, layer.controlType, models] ); + const base = baseId ? { id: baseId, label: getModelBaseLabel(baseId) } : null; + const panel = deriveControlPanel({ + availability, + base, + hiddenModelReasons: reidentifyPending.length > 0 ? [] : hiddenModelReasons, + imageState, + layer, + map: described.map, + reason, + resolution, + status, + }); - const switchKind = useCallback( - (kind: ControlAdapterKind) => { - if (kind === adapter.kind) { - return; + const reportIntent = useCallback( + (result: ControlLayerIntentResult) => { + switch (result) { + case 'blocked': + notify.info(t('widgets.layers.actions.busy')); + return; + case 'locked': + notify.info(t('widgets.layers.actions.locked')); + return; + case 'no-detection-input': + notify.info(t('widgets.layers.control.detectionNoInput')); + return; + case 'over-budget': + notify.error(t('widgets.layers.actions.actionFailed'), t('widgets.layers.actions.overBudget')); + return; + case 'failed': + notify.error(t('widgets.layers.actions.actionFailed'), t('widgets.layers.actions.operationFailed')); + return; + default: + // `updated`, `unchanged`, and `missing` (the layer went away, and this panel with it) say nothing. } - commitAdapter(switchControlAdapterKind(adapter, kind, models, base), t('widgets.layers.control.kind')); }, - [adapter, base, commitAdapter, models, t] + [notify, t] ); - const handleKindChange = useCallback( - ({ value }: SelectValueChangeDetails) => { - const kind = value[0] as ControlAdapterKind | undefined; - if (kind) { - switchKind(kind); + const applyIntent = useCallback( + (intent: ControlLayerIntent): ControlLayerIntentResult | null => { + if (!engine) { + return null; } + const result = getCanvasOperations(engine).applyControlLayerIntent(layer.id, intent); + reportIntent(result); + return result; }, - [switchKind] + [engine, layer.id, reportIntent] ); - const handleSwitchToSuggestedKind = useCallback(() => { - if (suggestedKind) { - switchKind(suggestedKind); + const retryDetection = useCallback(() => { + if (engine) { + getCanvasOperations(engine).retryControlDetection(layer.id); } - }, [suggestedKind, switchKind]); - - const handleModelChange = useCallback( - ({ value }: SelectValueChangeDetails) => { - const model = value[0] ?? null; - if (model !== adapter.model) { - commitAdapter({ model }, t('widgets.layers.control.model')); - const selected = models.find((candidate) => candidate.key === model); - const recommendation = resolveDefaultFilterForModel(selected); - // As the Filter button: a layer without pixels has nothing to filter, and choosing the model before adding - // the control image is the usual order. - if (recommendation && !layer.filter && engine?.exports.hasExportableLayerContent(layer.id)) { - runLayerFilterOperation( - () => getCanvasOperations(engine).startFilterOperation(layer.id, recommendation), - onOperationStarted - ); - } - } - }, - [adapter.model, commitAdapter, engine, layer.filter, layer.id, models, onOperationStarted, t] - ); + }, [engine, layer.id]); - const handleControlModeChange = useCallback( - ({ value }: SelectValueChangeDetails) => { - const mode = value[0] as CanvasControlAdapterContract['controlMode'] | undefined; - if (mode && mode !== adapter.controlMode) { - commitAdapter({ controlMode: mode }, t('widgets.layers.control.mode')); - } - }, - [adapter.controlMode, commitAdapter, t] - ); + const source = useControlSourceActions({ applyIntent, engine, layerId: layer.id, reportIntent }); + const sourceLabel = (() => { + if (imageState !== 'image') { + return t('widgets.layers.control.source.drawn'); + } + const pixels = layer.original ?? layer.source; + if (pixels.type !== 'image') { + return ''; + } + const origin = source.origins[pixels.image.imageName]; + return origin?.kind === 'file' + ? origin.name + : origin?.kind === 'frame' + ? t('widgets.layers.control.source.fromFrame') + : t('widgets.layers.control.source.imageSize', { height: pixels.image.height, width: pixels.image.width }); + })(); - const handleWeightChange = useCallback( - (next: number) => { - previewStructural({ - config: { adapter: { weight: next }, layerType: 'control' }, - id: layer.id, - type: 'updateCanvasLayerConfig', + const handleTypeChange = useCallback( + (controlType: ControlType, typeLabel: string) => { + const typeLabels = CONTROL_TYPES.map((type) => t(`widgets.layers.control.types.${type}`)); + const document = engine?.document.getDocument(); + const names = document + ? getDocumentLeaves(document) + .filter((leaf) => leaf.id !== layer.id) + .map((leaf) => leaf.name) + : []; + applyIntent({ + controlType, + kind: 'set-type', + name: isAutoControlLayerName(layer.name, typeLabels) ? nextControlTypeLayerName(typeLabel, names) : undefined, }); }, - [previewStructural, layer.id] + [applyIntent, engine, layer.id, layer.name, t] + ); + const handleDetectorChange = useCallback( + (detector: string | null) => applyIntent({ detector, kind: 'set-detector' }), + [applyIntent] ); - // A gesture ending where it started prepares nothing and drops its previews; one whose previews were refused - // (editing locked) still commits, so the refusal is reported instead of dropping the value. - const handleWeightChangeEnd = useCallback( - (next: number) => commitAdapter({ weight: next }, t('widgets.layers.control.weight')), - [commitAdapter, t] + const handleSettingsChange = useCallback( + (settings: Record) => applyIntent({ kind: 'set-detector-settings', settings }), + [applyIntent] ); - const handleRangeChange = useCallback( - ({ value }: SliderValueChangeDetails) => { - if (value.length === 2) { + // A previewed gesture records from where its preview started; any other change from the live adapter. + const commitAdapter = useCallback( + (next: Partial, label: string) => { + const config = { adapter: next, layerType: 'control' } as const; + commitPrepared(label, (model, baseline) => + model.prepare({ before: baselineConfig(baseline, config), config, id: layer.id, type: 'patch-config' }) + ); + }, + [commitPrepared, layer.id] + ); + const adapterEditing = useMemo( + () => ({ + cancel: cancelPreview, + commit: commitAdapter, + preview: (patch) => { previewStructural({ - config: { adapter: { beginEndStepPct: [value[0]!, value[1]!] }, layerType: 'control' }, + config: { adapter: patch, layerType: 'control' }, id: layer.id, type: 'updateCanvasLayerConfig', }); - } - }, - [previewStructural, layer.id] - ); - const handleRangeChangeEnd = useCallback( - ({ value }: SliderValueChangeDetails) => { - if (value.length === 2) { - commitAdapter({ beginEndStepPct: [value[0]!, value[1]!] }, t('widgets.layers.control.stepRange')); - } - }, - [commitAdapter, t] + }, + }), + [cancelPreview, commitAdapter, layer.id, previewStructural] ); - const handleTransparencyToggle = useCallback( - ({ checked }: { checked: boolean }) => { - commitPrepared(t('widgets.layers.control.transparencyEffect'), (model) => - model.prepare({ - before: { layerType: 'control', withTransparencyEffect: layer.withTransparencyEffect }, - config: { layerType: 'control', withTransparencyEffect: checked }, - id: layer.id, - type: 'patch-config', - }) - ); + const memoryKey = baseId && layer.controlType ? controlModelMemoryKey(baseId, layer.controlType) : null; + const handleModelPick = useCallback( + (model: ModelConfig) => { + const kind = deriveControlKind(model); + commitAdapter(kind ? { kind, model: model.key } : { model: model.key }, t('widgets.layers.control.model')); + if (memoryKey) { + void patchControlModelMemory(memoryKey, model.key); + } }, - [commitPrepared, layer.id, layer.withTransparencyEffect, t] - ); - - const controlModeCollection = useMemo( - () => - createListCollection({ - items: CONTROL_MODES.map((mode) => ({ label: t(`widgets.layers.control.modes.${mode}`), value: mode })), - }), - [t] + [commitAdapter, memoryKey, t] ); + const handleModelReset = useCallback(() => { + commitAdapter({ model: null }, t('widgets.layers.control.model')); + if (memoryKey) { + void patchControlModelMemory(memoryKey, null); + } + }, [commitAdapter, memoryKey, t]); + const findModels = useCallback(() => { + void navigate({ search: { project: projectId }, to: '/models' }); + }, [navigate, projectId]); - const kindValue = useMemo(() => [adapter.kind], [adapter.kind]); - const modelValue = useMemo(() => (adapter.model ? [adapter.model] : []), [adapter.model]); - const controlModeValue = useMemo(() => [adapter.controlMode ?? 'balanced'], [adapter.controlMode]); - const rangeValue = useMemo(() => [...adapter.beginEndStepPct], [adapter.beginEndStepPct]); - const rangeAria = useMemo(() => [t('widgets.layers.control.beginStep'), t('widgets.layers.control.endStep')], [t]); - - // The catalog name, not the filtered options: an unusable model still needs a name for the alert to refer to. - const adapterModel = models.find((model) => model.key === adapter.model) ?? null; - const hasContent = useCanvasEngineRead(engine, () => engine?.exports.hasExportableLayerContent(layer.id) ?? false); - // Re-read the reason when the capability table changes; the engine read below consults it. - const capabilitiesRevision = useExternalStoreSelector( - subscribeArchitectureCapabilities, - getArchitectureCapabilitiesSnapshot, - selectCapabilitiesRevision - ); - // Judged as the invocation judges it: earlier contributing control layers with content claim limited slots first, - // and only layers that pass claim one. - const validationReason = useCanvasEngineRead(engine, () => { - void capabilitiesRevision; - if (!mainModel) { - return null; + const [hasRequestedCapabilitiesRetry, setHasRequestedCapabilitiesRetry] = useState(false); + const rootRef = useRef(null); + const retryCapabilities = useCallback(() => { + if (hasRequestedCapabilitiesRetry && capabilitiesStatus === 'loading') { + return; } - const leaves = engine?.document.model()?.compileLeaves() ?? []; - const index = leaves.findIndex((leaf) => leaf.layer.id === layer.id); - const contributing = engine ? (leaves[index]?.contributionEnabled ?? false) : layer.isEnabled; - if (!contributing) { - return null; + setHasRequestedCapabilitiesRetry(true); + // The request belongs to this retry; a later reload started elsewhere is not this panel's retry. + void ensureArchitectureCapabilitiesLoaded().then(() => setHasRequestedCapabilitiesRetry(false)); + }, [capabilitiesStatus, hasRequestedCapabilitiesRetry]); + // Transfer retry-button focus to the loaded panel during ref cleanup before removal can send it to body. + const handOverFocusOnLoad = useCallback((surface: HTMLDivElement | null) => { + if (!surface) { + return undefined; } - const earlier = leaves - .slice(0, Math.max(0, index)) - .filter((leaf) => leaf.contributionEnabled && engine?.exports.hasExportableLayerContent(leaf.id)) - .map((leaf) => leaf.layer) - .filter((candidate): candidate is CanvasControlLayerContract => candidate.type === 'control'); - return getControlLayerReasonInSequence({ earlier, layer, mainModel, models }); - }); - // Content-dependent problems only matter once the layer has pixels, but a - // missing model deserves the warning even on a fresh empty layer. A missing - // capability table is not a problem with this layer and is shown on its own. - const capabilitiesUnavailable = validationReason === 'capabilities_unavailable'; + return () => { + if (surface.contains(document.activeElement) && getArchitectureCapabilitiesSnapshot().status === 'loaded') { + focusFirstOperable(rootRef.current); + } + }; + }, []); + const capabilitiesUnavailable = reason === 'capabilities_unavailable'; // The click flips the status to `loading` synchronously; keeping the failure surface mounted for the // retry it started keeps the button, and the user's focus, in place. const isRetryingCapabilities = hasRequestedCapabilitiesRetry && capabilitiesStatus === 'loading'; const showCapabilitiesFailure = capabilitiesUnavailable && (capabilitiesStatus === 'error' || isRetryingCapabilities); - // A missing model, a kind to switch or a main model that cannot run the kind at all is worth saying on a fresh empty - // layer; an empty model list explains itself. - const visibleValidationReason = - validationReason && - !capabilitiesUnavailable && - (hasContent || - validationReason === 'missing_model' || - validationReason === 'missing_type' || - validationReason === 'switch_adapter_kind' || - validationReason === 'flux2_klein_control') && - !((validationReason === 'missing_model' || validationReason === 'missing_type') && hiddenModelReasons.length > 0) - ? validationReason - : null; return ( - - - - - - + + + + - - - - {adapter.kind === 'controlnet' ? ( - - + + + + + {/* Always two lines tall, so the rows below never move as the explanation changes. */} + + {model.caption ? format(model.caption) : null}{' '} + {actionButton && rememberedHint ? {actionButton} : actionButton} + + + + + ); +}; + +const renderModelItem = (item: ModelItem) => ( + + + {item.label} + {/* Keeps the option's accessible name readable: "name, caption". */} + + + + {item.caption} + + +); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlSourceRow.browser.test.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlSourceRow.browser.test.tsx new file mode 100644 index 00000000000..e56e39e6076 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlSourceRow.browser.test.tsx @@ -0,0 +1,267 @@ +import type * as gallery from '@features/gallery'; +import type { CanvasControlLayerContract, CanvasLayerContract } from '@workbench/canvas-engine/api'; +import type { ControlLayerIntent } from '@workbench/canvas-operations/api'; + +import { ChakraProvider } from '@chakra-ui/react'; +import { accountLifecycle } from '@platform/state/accountLifecycle'; +import { toaster } from '@platform/ui/toaster'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { applyThemeToRoot } from '@theme/applyTheme'; +import { system } from '@theme/system'; +import { createControlLayer, createDocumentModel } from '@workbench/canvas-engine/api'; +import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; +import { createControlDetectionStatusStore } from '@workbench/canvas-operations/controlDetectionStatusStore'; +import { attachCanvasOperations } from '@workbench/canvas-operations/operationAccess'; +import { createEmptyCanvasDocument } from '@workbench/canvasMigration'; +import { createInstance } from 'i18next'; +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { I18nextProvider } from 'react-i18next'; +import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; +import { page, userEvent } from 'vitest/browser'; + +import { ControlLayerSettings } from './ControlLayerSettings'; + +const harness = vi.hoisted(() => ({ + canvasUploads: [] as { category: unknown; project: unknown }[], + galleryUploads: [] as File[], + /** Holds the gallery upload until a test releases it. */ + release: null as (() => void) | null, +})); + +vi.mock('@features/gallery', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + galleryTransfers: { + ...actual.galleryTransfers, + upload: async (file: File) => { + harness.galleryUploads.push(file); + await new Promise((resolve) => { + harness.release = resolve; + }); + return { height: 48, imageName: 'uploaded.png', width: 80 }; + }, + }, + }; +}); +vi.mock('@features/gallery/queries', async (importOriginal) => ({ + ...(await importOriginal>()), + invalidateGallery: () => Promise.resolve(), +})); +vi.mock('@workbench/canvas-operations/backend/canvasImages', async (importOriginal) => ({ + ...(await importOriginal>()), + uploadCanvasImage: (_blob: Blob, options: { imageCategory?: string; projectId?: string }) => { + harness.canvasUploads.push({ category: options.imageCategory, project: options.projectId }); + return Promise.resolve({ height: 32, imageName: 'bbox.png', width: 32 }); + }, +})); +vi.mock('./useSelectedMainModel', () => ({ useSelectedMainModel: () => null })); +vi.mock('@workbench/WorkbenchContext', () => ({ + useActiveProjectId: () => 'test-project', + useOptionalWorkbenchCommands: () => null, +})); + +const i18n = createInstance(); +beforeAll(async () => { + const translation = (await (await fetch('/locales/en.json')).json()) as Record; + await i18n.init({ initAsync: false, lng: 'en', resources: { en: { translation } } }); +}); + +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +const PHOTO = { image: { height: 64, imageName: 'photo.png', width: 96 }, type: 'image' } as const; +const BBOX = { height: 512, width: 512, x: 40, y: 24 }; +const empty = createControlLayer({ id: 'control-1', name: 'Control Layer 1' }); +const filled: CanvasControlLayerContract = { ...empty, source: PHOTO }; + +type Engine = NonNullable[0]['engine']>; + +const intents: ControlLayerIntent[] = []; +let composite: { status: 'ok'; blob: Blob; rect: typeof BBOX } | { status: 'empty' } = { status: 'empty' }; +let host: HTMLDivElement | null = null; +let root: Root | null = null; +let engine: Engine | null = null; +const setTool = vi.fn(); + +const engineWith = (layers: CanvasLayerContract[]): Engine => { + const document = { ...createEmptyCanvasDocument(), bbox: BBOX, stacks: stacksFrom(layers) }; + const model = createDocumentModel(document, { editRevision: 0, projectId: 'test-project' }); + const statuses = createControlDetectionStatusStore(); + const created = { + document: { getDocument: () => document, model: () => model }, + exports: { exportRasterComposite: () => composite, hasExportableLayerContent: () => false }, + interaction: { get: () => false, subscribe: () => () => undefined }, + layers: { beginStructuralPreview: () => null, endStructuralPreview: () => undefined }, + lifecycle: { flushPendingUploads: () => Promise.resolve() }, + projectId: 'test-project', + tools: { setTool }, + } as unknown as Engine; + attachCanvasOperations(created, { + applyControlLayerIntent: (_layerId: string, intent: ControlLayerIntent) => { + intents.push(intent); + return 'updated'; + }, + getControlDetectionStatus: statuses.get, + subscribeControlDetectionStatus: statuses.subscribe, + } as never); + return created; +}; + +const queryClient = new QueryClient(); +const view = (layer: CanvasControlLayerContract) => ( + + + + + + + +); +const settle = async (run: () => unknown = () => undefined) => { + await act(async () => { + await run(); + await new Promise((resolve) => { + globalThis.setTimeout(resolve, 0); + }); + }); +}; +const render = async (layer: CanvasControlLayerContract) => { + applyThemeToRoot('classic'); + host = document.createElement('div'); + host.style.width = '320px'; + document.body.append(host); + root = createRoot(host); + engine = engineWith([layer]); + await settle(() => root?.render(view(layer))); +}; +/** The document now holds `layer`, as after the intent landed. */ +const rerender = (layer: CanvasControlLayerContract) => settle(() => root?.render(view(layer))); + +afterEach(async () => { + await act(() => root?.unmount()); + host?.remove(); + host = null; + root = null; + intents.length = 0; + harness.canvasUploads.length = 0; + harness.galleryUploads.length = 0; + harness.release = null; + composite = { status: 'empty' }; + setTool.mockClear(); + accountLifecycle.invalidate(); +}); + +const button = (name: string) => page.getByRole('button', { exact: true, name }); +const imageRow = () => page.getByRole('group', { name: 'Image' }); + +describe('Control layer Image row', () => { + it('uploads a file, shows the upload in progress, and fills the layer at the bbox origin', async () => { + await render(empty); + const input = host!.querySelector('input[type="file"]')!; + await act(async () => { + await userEvent.upload(input, new File(['png'], 'portrait.png', { type: 'image/png' })); + }); + await settle(); + + expect(harness.galleryUploads.map((file) => file.name)).toEqual(['portrait.png']); + // The pressed action shows its progress; the others wait for it. + await expect.element(button('Upload')).toHaveAttribute('data-loading'); + await expect.element(button('Frame')).toBeDisabled(); + await expect.element(button('Draw')).toBeDisabled(); + + await settle(() => harness.release?.()); + expect(intents).toEqual([ + { + image: { height: 48, imageName: 'uploaded.png', width: 80 }, + kind: 'replace-image', + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 40, y: 24 }, + }, + ]); + + await rerender({ + ...empty, + source: { image: { height: 48, imageName: 'uploaded.png', width: 80 }, type: 'image' }, + }); + expect(imageRow().element().textContent).toContain('portrait.png'); + }); + + it('fills the layer from what is visible inside the frame, uploaded as a control image', async () => { + composite = { blob: new Blob(['png']), rect: BBOX, status: 'ok' }; + await render(empty); + await act(() => button('Frame').click()); + await settle(); + + expect(harness.canvasUploads).toEqual([{ category: 'control', project: 'test-project' }]); + expect(intents).toEqual([ + { + image: { height: 32, imageName: 'bbox.png', width: 32 }, + kind: 'replace-image', + transform: { rotation: 0, scaleX: 1, scaleY: 1, x: 40, y: 24 }, + }, + ]); + await rerender({ ...empty, source: { image: { height: 32, imageName: 'bbox.png', width: 32 }, type: 'image' } }); + expect(imageRow().element().textContent).toContain('From frame'); + }); + + it('says so when nothing is visible inside the frame', async () => { + const created = vi.spyOn(toaster, 'create'); + try { + await render(empty); + await act(() => button('Frame').click()); + await settle(); + + expect(intents).toEqual([]); + expect(created).toHaveBeenCalledWith(expect.objectContaining({ title: 'Nothing is visible inside the frame' })); + } finally { + created.mockRestore(); + } + }); + + it('drops an upload that finishes after the account changed, without a notice', async () => { + accountLifecycle.activate('control-upload-a'); + const created = vi.spyOn(toaster, 'create'); + try { + await render(empty); + const input = host!.querySelector('input[type="file"]')!; + await act(async () => { + await userEvent.upload(input, new File(['png'], 'portrait.png', { type: 'image/png' })); + }); + await settle(); + accountLifecycle.activate('control-upload-b'); + await settle(() => harness.release?.()); + + expect(intents).toEqual([]); + expect(created).not.toHaveBeenCalled(); + // The row is ready for the next action. + await expect.element(button('Upload')).not.toHaveAttribute('data-loading'); + } finally { + created.mockRestore(); + } + }); + + it('switches the layer to drawing and picks the brush', async () => { + await render(empty); + await act(() => button('Draw').click()); + + expect(intents).toEqual([{ kind: 'draw' }]); + expect(setTool).toHaveBeenCalledExactlyOnceWith('brush'); + }); + + it('replaces a filled image from the keyboard and hands focus to the row that replaces the menu', async () => { + await render(filled); + expect(imageRow().element().textContent).toContain('Image · 96×64'); + const replace = button('Replace'); + await act(() => (replace.element() as HTMLElement).focus()); + await act(() => userEvent.keyboard('{Enter}')); + await expect.element(page.getByRole('menuitem', { name: 'Draw' })).toBeVisible(); + await act(() => userEvent.keyboard('{ArrowDown}{ArrowDown}{Enter}')); + await settle(); + + expect(intents).toEqual([{ kind: 'draw' }]); + // Drawing empties the image: the empty actions take the place of the Replace menu, and focus with them. + await rerender(empty); + expect(imageRow().element().contains(document.activeElement)).toBe(true); + expect(document.activeElement?.textContent).toBe('Upload'); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlSourceRow.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlSourceRow.tsx new file mode 100644 index 00000000000..359fe68865b --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlSourceRow.tsx @@ -0,0 +1,216 @@ +import type { ChangeEvent } from 'react'; + +import { Box, HStack, Icon, Image, Input, Menu, Portal, Text } from '@chakra-ui/react'; +import { galleryImageUrls } from '@features/gallery/utility'; +import { focusIfUnclaimed } from '@platform/react/focusIfUnclaimed'; +import { Button } from '@platform/ui/Button'; +import { MenuContent, MenuItem } from '@platform/ui/Menu'; +import { Tooltip, useTooltipTriggerIds } from '@platform/ui/Tooltip'; +import { PropertyControlRow } from '@workbench/widgets/canvas/tool-presentation/PropertyPrimitives'; +import { BrushIcon, ChevronDownIcon, ScanIcon, UploadIcon } from 'lucide-react'; +import { useCallback, useId, useRef } from 'react'; +import { useTranslation } from 'react-i18next'; + +import type { ControlImageState } from './controlPanelModel'; +import type { ControlSourceAction } from './useControlSourceActions'; + +const REPLACE_MENU_POSITIONING = { placement: 'bottom-end' } as const; +/** In a narrow pane the three empty-state actions keep their words and drop their icons. */ +const NARROW_HIDDEN = { '@container (max-width: 15rem)': { display: 'none' } } as const; + +const ACTIONS: readonly { action: ControlSourceAction; icon: typeof UploadIcon }[] = [ + { action: 'upload', icon: UploadIcon }, + { action: 'frame', icon: ScanIcon }, + { action: 'draw', icon: BrushIcon }, +]; + +export interface ControlSourceRowProps { + state: ControlImageState; + /** The image the layer derives from, for its thumbnail; null for drawn pixels. */ + imageName: string | null; + /** What the filled row calls its image. */ + label: string; + busy: ControlSourceAction | null; + disabled: boolean; + onUpload(file: File): void; + onFrame(): void; + onDraw(): void; +} + +/** + * The Image row: Upload / Frame / Draw while the layer is empty, else a thumbnail, its name and a Replace menu with + * the same three actions. Focus that sat on the part that went away moves to the part that replaced it. + */ +export const ControlSourceRow = ({ + busy, + disabled, + imageName, + label, + onFrame, + onDraw, + onUpload, + state, +}: ControlSourceRowProps) => { + const { t } = useTranslation(); + const labelId = useId(); + const fileInputRef = useRef(null); + const replaceIds = useTooltipTriggerIds(); + // Which part this row showed last, so focus is handed over only when one replaces the other, never on mount. + const shownRef = useRef<'empty' | 'filled' | null>(null); + const handOver = useCallback((element: HTMLElement | null, part: 'empty' | 'filled') => { + if (!element) { + return; + } + if (shownRef.current !== null && shownRef.current !== part) { + focusIfUnclaimed(element); + } + shownRef.current = part; + }, []); + const emptyRef = useCallback((element: HTMLElement | null) => handOver(element, 'empty'), [handOver]); + const filledRef = useCallback((element: HTMLElement | null) => handOver(element, 'filled'), [handOver]); + + const openUpload = useCallback(() => fileInputRef.current?.click(), []); + const handleFileChange = useCallback( + (event: ChangeEvent) => { + const file = event.currentTarget.files?.[0]; + event.currentTarget.value = ''; + if (file) { + onUpload(file); + } + }, + [onUpload] + ); + const run = useCallback( + (action: ControlSourceAction) => { + switch (action) { + case 'upload': + openUpload(); + return; + case 'frame': + onFrame(); + return; + case 'draw': + onDraw(); + } + }, + [onDraw, onFrame, openUpload] + ); + const handleMenuSelect = useCallback(({ value }: { value: string }) => run(value as ControlSourceAction), [run]); + + return ( + + {state === 'empty' ? ( + + {ACTIONS.map(({ action, icon }) => ( + + ))} + + ) : ( + + {imageName ? ( + + ) : ( + + + + )} + + {label} + + + + + + + + + + + {ACTIONS.map(({ action, icon }) => ( + + + {t(`widgets.layers.control.source.${action}`)} + + ))} + + + + + + )} + + + ); +}; + +const SourceActionButton = ({ + action, + busy, + disabled, + icon, + onRun, +}: { + action: ControlSourceAction; + busy: ControlSourceAction | null; + disabled: boolean; + icon: typeof UploadIcon; + onRun(action: ControlSourceAction): void; +}) => { + const { t } = useTranslation(); + const handleClick = useCallback(() => onRun(action), [action, onRun]); + const label = t(`widgets.layers.control.source.${action}`); + return ( + + {/* Named explicitly: while loading, the spinner replaces the visible text. */} + + + ); +}; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlTypeRow.tsx b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlTypeRow.tsx new file mode 100644 index 00000000000..86d3b5f8a97 --- /dev/null +++ b/invokeai/frontend/webv2/src/workbench/widgets/layers/ControlTypeRow.tsx @@ -0,0 +1,116 @@ +import type { SelectValueChangeDetails } from '@chakra-ui/react'; +import type { ControlType } from '@workbench/canvas-engine/api'; + +import { createListCollection, HStack, Text } from '@chakra-ui/react'; +import { Select } from '@platform/ui/Select'; +import { CONTROL_TYPES } from '@workbench/canvas-engine/api'; +import { PropertyControlRow } from '@workbench/widgets/canvas/tool-presentation/PropertyPrimitives'; +import { useCallback, useMemo } from 'react'; +import { useTranslation } from 'react-i18next'; + +import { formatPanelText, type ControlPanelModel, type PanelTone } from './controlPanelModel'; +import { OptionNameSeparator } from './OptionNameSeparator'; + +const SELECT_POSITIONING = { placement: 'bottom-end', sameWidth: true } as const; +const TONE_COLORS: Record = { + default: 'fg', + error: 'fg.error', + muted: 'fg.muted', + warning: 'fg.warning', +}; +const PLACEHOLDER_PROPS = { color: 'fg.muted' } as const; +const INVALID_PLACEHOLDER_PROPS = { color: 'fg.warning' } as const; +// The recipe paints `aria-invalid` triggers as errors; a missing type only warns, so its rule is outranked here. +// The theme has no `border.warning`; warning borders use the warning foreground, as the warning input shell does. +const WARNING_TRIGGER_CSS = { + '&[aria-invalid=true]': { borderColor: 'fg.warning' }, + '&[aria-invalid=true]:hover': { bg: 'bg.warning', borderColor: 'fg.warning' }, + '&[aria-invalid=true]:focus-visible': { + '--focus-ring-color': 'var(--chakra-colors-fg-warning) !important', + borderColor: 'fg.warning', + }, +} as const; + +interface TypeItem { + hint: string; + hintColor: string; + label: string; + value: ControlType; +} + +/** The Type row: what the image controls, each choice hinted with what is installed for it. */ +export const ControlTypeRow = ({ + describedBy, + disabled, + model, + onChange, +}: { + /** What explains a missing type, read while the row warns. */ + describedBy: string; + disabled: boolean; + model: ControlPanelModel['type']; + onChange(type: ControlType, label: string): void; +}) => { + const { i18n, t } = useTranslation(); + const label = t('widgets.layers.control.rows.type'); + const collection = useMemo( + () => + createListCollection({ + items: CONTROL_TYPES.map((type) => ({ + hint: formatPanelText(t, model.hints[type].text, i18n.language), + hintColor: TONE_COLORS[model.hints[type].tone], + label: t(`widgets.layers.control.types.${type}`), + value: type, + })), + }), + [i18n.language, model.hints, t] + ); + const value = useMemo(() => (model.value ? [model.value] : []), [model.value]); + const triggerProps = useMemo( + () => + model.invalid ? { 'aria-describedby': describedBy, 'aria-invalid': true, css: WARNING_TRIGGER_CSS } : undefined, + [describedBy, model.invalid] + ); + const handleChange = useCallback( + ({ value: next }: SelectValueChangeDetails) => { + const type = next[0] as ControlType | undefined; + if (type && type !== model.value) { + onChange(type, t(`widgets.layers.control.types.${type}`)); + } + }, + [model.value, onChange, t] + ); + return ( + + ', { key: 'ArrowRight' })).toBe(true); + expect(press('', { key: 'Home' })).toBe(true); + expect(press('', { key: 'ArrowUp' })).toBe(true); + expect(press('
x
', { key: 'End' })).toBe(true); + expect(press('
', { key: 'ArrowLeft' })).toBe(true); + }); + + it('leaves other keys and chords to shortcuts', () => { + expect(press('', { key: 'a' })).toBe(false); + expect(press('', { ctrlKey: true, key: 'ArrowRight' })).toBe(false); + expect(press('', { altKey: true, key: 'ArrowRight' })).toBe(false); + }); + + it('is not about text fields, which isEditableHotkeyTarget covers', () => { + expect(press('', { key: 'ArrowRight' })).toBe(false); + expect(press('', { key: 'ArrowRight' })).toBe(false); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/hotkeys/keys.ts b/invokeai/frontend/webv2/src/workbench/hotkeys/keys.ts index 339b6f5f23e..8dc6b016478 100644 --- a/invokeai/frontend/webv2/src/workbench/hotkeys/keys.ts +++ b/invokeai/frontend/webv2/src/workbench/hotkeys/keys.ts @@ -257,6 +257,23 @@ const NON_TEXT_INPUT_TYPES = new Set([ 'submit', ]); +/** What a focused radio group or slider does with a key itself: move its selection or value. */ +const NATIVE_CONTROL_KEYS = new Set(['ArrowDown', 'ArrowLeft', 'ArrowRight', 'ArrowUp', 'End', 'Home']); + +/** + * Whether a press is the focused radio group's or slider's own navigation (arrows, Home, End without Ctrl, Meta or + * Alt). These controls take no typed text, so {@link isEditableHotkeyTarget} lets chords through from them, but a + * plain shortcut on the same key (Gallery's arrow navigation) would prevent the default and freeze the control. + */ +export const isNativeControlKeyPress = (event: KeyboardEvent): boolean => + !event.ctrlKey && + !event.metaKey && + !event.altKey && + NATIVE_CONTROL_KEYS.has(event.key) && + typeof Element !== 'undefined' && + event.target instanceof Element && + event.target.closest('input[type="radio"], input[type="range"], [role="radiogroup"], [role="slider"]') !== null; + export const isEditableHotkeyTarget = (target: EventTarget | null): boolean => { if (typeof HTMLElement === 'undefined') { return false; diff --git a/invokeai/frontend/webv2/src/workbench/hotkeys/resolve.browser.test.ts b/invokeai/frontend/webv2/src/workbench/hotkeys/resolve.browser.test.ts index 8a52b280b28..5f5adeeaae3 100644 --- a/invokeai/frontend/webv2/src/workbench/hotkeys/resolve.browser.test.ts +++ b/invokeai/frontend/webv2/src/workbench/hotkeys/resolve.browser.test.ts @@ -131,3 +131,61 @@ describe('resolveHotkey inside an open popup', () => { ); }); }); + +describe('resolveHotkey on a focused radio group or slider', () => { + const host = document.createElement('div'); + + document.body.append(host); + + afterEach(() => { + host.replaceChildren(); + }); + + const context: HotkeyContext = { + activeInstanceId: 'gallery', + activeWidgetTypeId: 'gallery', + focusedRegion: 'right', + isModalPresent: false, + projectId: 'project-1', + }; + // Gallery's arrow navigation, as its grid registers it. + const galleryRight: RegisteredHotkey = { + category: 'gallery', + commandId: 'gallery.galleryNavRight', + defaultKeys: ['arrowright'], + id: 'gallery.galleryNavRight', + implemented: true, + keys: ['arrowright'], + scope: { kind: 'widget', typeIds: ['gallery'] }, + title: 'Navigate right', + }; + const editableRight: RegisteredHotkey = { + allowInEditable: true, + category: 'app', + commandId: 'editable', + defaultKeys: ['arrowright'], + id: 'editable', + implemented: true, + keys: ['arrowright'], + scope: { kind: 'global' }, + title: 'Editable', + }; + + const pressRightIn = (html: string, hotkey: RegisteredHotkey) => { + host.innerHTML = html; + const event = new KeyboardEvent('keydown', { bubbles: true, key: 'ArrowRight' }); + host.querySelector('[data-target]')!.dispatchEvent(event); + + return resolveHotkey({ context, event, hotkeys: [hotkey], matchedKey: 'arrowright' })?.commandId ?? null; + }; + + it('leaves the arrow to the radio group instead of moving the gallery selection', () => { + expect(pressRightIn('', galleryRight)).toBeNull(); + expect(pressRightIn('', galleryRight)).toBeNull(); + expect(pressRightIn('', galleryRight)).toBe('gallery.galleryNavRight'); + }); + + it('still resolves a binding that opts into editable targets', () => { + expect(pressRightIn('', editableRight)).toBe('editable'); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/hotkeys/resolve.ts b/invokeai/frontend/webv2/src/workbench/hotkeys/resolve.ts index 5419a5d4c63..bc3d5f5ac74 100644 --- a/invokeai/frontend/webv2/src/workbench/hotkeys/resolve.ts +++ b/invokeai/frontend/webv2/src/workbench/hotkeys/resolve.ts @@ -1,6 +1,6 @@ import type { HotkeyContext, RegisteredHotkey } from './types'; -import { IS_MAC_OS, isEditableHotkeyTarget, normalizeHotkeyString } from './keys'; +import { IS_MAC_OS, isEditableHotkeyTarget, isNativeControlKeyPress, normalizeHotkeyString } from './keys'; /** * The keys a binding presses on this platform: `mod` is Cmd on macOS and Ctrl elsewhere. The runtime fires the first @@ -90,12 +90,13 @@ export const resolveHotkey = ({ } const pressed = toPlatformHotkey(matchedKey); - const isEditable = isEditableHotkeyTarget(event.target); + // A focused text field, radio group or slider keeps the press unless the binding opts into editable targets. + const isOwnedByTarget = isEditableHotkeyTarget(event.target) || isNativeControlKeyPress(event); return ( hotkeys .filter((hotkey) => hotkey.implemented !== false && hotkey.keys.some((key) => toPlatformHotkey(key) === pressed)) - .filter((hotkey) => hotkey.allowInEditable || !isEditable) + .filter((hotkey) => hotkey.allowInEditable || !isOwnedByTarget) .filter((hotkey) => isClaimedTarget(hotkey, event.target)) .filter((hotkey) => hotkey.allowInModal || !context.isModalPresent) .map((hotkey) => ({ hotkey, priority: getScopePriority(hotkey, context) })) From 1611bee859fe2cd9c3517b70a484ca9e26fa79bf Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Fri, 23 Oct 2026 09:50:00 -0600 Subject: [PATCH 20/35] perf(canvas): record the control view controller The browser baseline lists the new engine controller in the editor owner sets. --- invokeai/frontend/webv2/performance/browser-baseline.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/invokeai/frontend/webv2/performance/browser-baseline.json b/invokeai/frontend/webv2/performance/browser-baseline.json index e6e4eccb4a0..03ab2ffaa0f 100644 --- a/invokeai/frontend/webv2/performance/browser-baseline.json +++ b/invokeai/frontend/webv2/performance/browser-baseline.json @@ -1378,6 +1378,7 @@ "source:src/workbench/canvas-engine/controllers/booleanMergeController.ts", "source:src/workbench/canvas-engine/controllers/controlDerivationController.ts", "source:src/workbench/canvas-engine/controllers/controlPixelController.ts", + "source:src/workbench/canvas-engine/controllers/controlViewController.ts", "source:src/workbench/canvas-engine/controllers/copyLayerController.ts", "source:src/workbench/canvas-engine/controllers/cropLayerController.ts", "source:src/workbench/canvas-engine/controllers/editSteps.ts", @@ -2493,6 +2494,7 @@ "source:src/workbench/canvas-engine/controllers/booleanMergeController.ts", "source:src/workbench/canvas-engine/controllers/controlDerivationController.ts", "source:src/workbench/canvas-engine/controllers/controlPixelController.ts", + "source:src/workbench/canvas-engine/controllers/controlViewController.ts", "source:src/workbench/canvas-engine/controllers/copyLayerController.ts", "source:src/workbench/canvas-engine/controllers/cropLayerController.ts", "source:src/workbench/canvas-engine/controllers/editSteps.ts", @@ -3840,6 +3842,7 @@ "source:src/workbench/canvas-engine/controllers/booleanMergeController.ts", "source:src/workbench/canvas-engine/controllers/controlDerivationController.ts", "source:src/workbench/canvas-engine/controllers/controlPixelController.ts", + "source:src/workbench/canvas-engine/controllers/controlViewController.ts", "source:src/workbench/canvas-engine/controllers/copyLayerController.ts", "source:src/workbench/canvas-engine/controllers/cropLayerController.ts", "source:src/workbench/canvas-engine/controllers/editSteps.ts", From 3eea88beed2342fba1f40c0ddbbf3dde4e8b7e21 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Fri, 23 Oct 2026 14:44:19 -0600 Subject: [PATCH 21/35] perf(canvas): record byte budgets after merging main Locale and editor chunks grew by the merged control layer and noise adjustment code; the workflow activation chunk shrank. --- .../webv2/performance/browser-baseline.json | 86 +++++++++---------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/invokeai/frontend/webv2/performance/browser-baseline.json b/invokeai/frontend/webv2/performance/browser-baseline.json index dcac5d45f17..74345de80eb 100644 --- a/invokeai/frontend/webv2/performance/browser-baseline.json +++ b/invokeai/frontend/webv2/performance/browser-baseline.json @@ -1,6 +1,6 @@ { "browserExecutable": "/home/lstein/.cache/ms-playwright/chromium-1243/chrome-linux64/chrome", - "capturedAt": "2026-10-18", + "capturedAt": "2026-10-23", "routes": [ { "activatedResourceBaseline": { @@ -30,11 +30,11 @@ "fontRawBytes": 23664, "imageRawBytes": 0, "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "otherRawBytes": 236538, "requestCount": 29, - "scriptRawBytes": 1882471, + "scriptRawBytes": 1883609, "scriptRequestCount": 25, - "totalRawBytes": 2139551 + "totalRawBytes": 2145970 }, "routeReadyMedianMs": 215.30000019073486, "scriptSourceOwnerSet": "launchpad-static", @@ -68,11 +68,11 @@ "fontRawBytes": 23664, "imageRawBytes": 0, "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "otherRawBytes": 236538, "requestCount": 29, - "scriptRawBytes": 1882471, + "scriptRawBytes": 1883609, "scriptRequestCount": 25, - "totalRawBytes": 2139551 + "totalRawBytes": 2145970 }, "routeReadyMedianMs": 225.39999961853027, "scriptSourceOwnerSet": "launchpad-static", @@ -106,11 +106,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "otherRawBytes": 236538, "requestCount": 87, - "scriptRawBytes": 4250746, + "scriptRawBytes": 4299336, "scriptRequestCount": 82, - "totalRawBytes": 4552224 + "totalRawBytes": 4606095 }, "routeReadyMedianMs": 360.90000009536743, "scriptSourceOwnerSet": "editor-minimal-static", @@ -144,11 +144,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "otherRawBytes": 236538, "requestCount": 87, - "scriptRawBytes": 4250746, + "scriptRawBytes": 4299336, "scriptRequestCount": 82, - "totalRawBytes": 4552224 + "totalRawBytes": 4606095 }, "routeReadyMedianMs": 367.80000019073486, "scriptSourceOwnerSet": "editor-minimal-static", @@ -159,12 +159,12 @@ "cssRawBytes": 0, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 311154, + "largestAssetRawBytes": 335789, "otherRawBytes": 0, "requestCount": 9, - "scriptRawBytes": 529607, + "scriptRawBytes": 559656, "scriptRequestCount": 9, - "totalRawBytes": 529607 + "totalRawBytes": 559656 }, "domContentLoadedMedianMs": 100.40000009536743, "id": "editor-canvas", @@ -182,11 +182,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "otherRawBytes": 236538, "requestCount": 96, - "scriptRawBytes": 4823800, + "scriptRawBytes": 4858992, "scriptRequestCount": 91, - "totalRawBytes": 5126937 + "totalRawBytes": 5165751 }, "routeReadyMedianMs": 1095.3000001907349, "scriptSourceOwnerSet": "editor-canvas-static", @@ -197,12 +197,12 @@ "cssRawBytes": 0, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 311154, + "largestAssetRawBytes": 335789, "otherRawBytes": 0, "requestCount": 9, - "scriptRawBytes": 529607, + "scriptRawBytes": 559656, "scriptRequestCount": 9, - "totalRawBytes": 529607 + "totalRawBytes": 559656 }, "domContentLoadedMedianMs": 97.7000002861023, "id": "editor-canvas", @@ -220,11 +220,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "otherRawBytes": 236538, "requestCount": 96, - "scriptRawBytes": 4823800, + "scriptRawBytes": 4858992, "scriptRequestCount": 91, - "totalRawBytes": 5126937 + "totalRawBytes": 5165751 }, "routeReadyMedianMs": 1042.5, "scriptSourceOwnerSet": "editor-canvas-static", @@ -235,12 +235,12 @@ "cssRawBytes": 15413, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 208822, + "largestAssetRawBytes": 208701, "otherRawBytes": 0, "requestCount": 7, - "scriptRawBytes": 451936, + "scriptRawBytes": 454172, "scriptRequestCount": 6, - "totalRawBytes": 467349 + "totalRawBytes": 469585 }, "domContentLoadedMedianMs": 101.40000009536743, "id": "editor-workflow", @@ -258,11 +258,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "otherRawBytes": 236538, "requestCount": 94, - "scriptRawBytes": 4750033, + "scriptRawBytes": 4753508, "scriptRequestCount": 88, - "totalRawBytes": 5019779 + "totalRawBytes": 5075680 }, "routeReadyMedianMs": 1093.0999999046326, "scriptSourceOwnerSet": "editor-workflow-static", @@ -273,12 +273,12 @@ "cssRawBytes": 15413, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 208822, + "largestAssetRawBytes": 208701, "otherRawBytes": 0, "requestCount": 7, - "scriptRawBytes": 451936, + "scriptRawBytes": 454172, "scriptRequestCount": 6, - "totalRawBytes": 467349 + "totalRawBytes": 469585 }, "domContentLoadedMedianMs": 99.89999961853027, "id": "editor-workflow", @@ -296,11 +296,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "otherRawBytes": 236538, "requestCount": 94, - "scriptRawBytes": 4750033, + "scriptRawBytes": 4753508, "scriptRequestCount": 88, - "totalRawBytes": 5019779 + "totalRawBytes": 5075680 }, "routeReadyMedianMs": 1105.0999999046326, "scriptSourceOwnerSet": "editor-workflow-static", @@ -334,11 +334,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "otherRawBytes": 236538, "requestCount": 87, - "scriptRawBytes": 4250746, + "scriptRawBytes": 4299336, "scriptRequestCount": 82, - "totalRawBytes": 4552224 + "totalRawBytes": 4606095 }, "routeReadyMedianMs": 1324.5, "scriptSourceOwnerSet": "editor-minimal-static", @@ -372,11 +372,11 @@ "fontRawBytes": 23664, "imageRawBytes": 44398, "largestAssetRawBytes": 752422, - "otherRawBytes": 231257, + "otherRawBytes": 236538, "requestCount": 87, - "scriptRawBytes": 4250746, + "scriptRawBytes": 4299336, "scriptRequestCount": 82, - "totalRawBytes": 4552224 + "totalRawBytes": 4606095 }, "routeReadyMedianMs": 1301.5, "scriptSourceOwnerSet": "editor-minimal-static", From 2d41b0a118a7c8cc39500d7c56812ed7fb47cadf Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sat, 24 Oct 2026 01:41:38 -0600 Subject: [PATCH 22/35] perf(canvas): record byte budgets after merging main The editor-canvas activation chunk grew with the merged adjustment and control layer code; the launchpad script bytes shrank. --- .../webv2/performance/browser-baseline.json | 106 +++++++++--------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/invokeai/frontend/webv2/performance/browser-baseline.json b/invokeai/frontend/webv2/performance/browser-baseline.json index 68ad00c91f7..22b35d7d11e 100644 --- a/invokeai/frontend/webv2/performance/browser-baseline.json +++ b/invokeai/frontend/webv2/performance/browser-baseline.json @@ -1,6 +1,6 @@ { "browserExecutable": "/home/lstein/.cache/ms-playwright/chromium-1243/chrome-linux64/chrome", - "capturedAt": "2026-10-18", + "capturedAt": "2026-10-24", "routes": [ { "activatedResourceBaseline": { @@ -29,12 +29,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 0, - "largestAssetRawBytes": 752431, - "otherRawBytes": 235365, + "largestAssetRawBytes": 752422, + "otherRawBytes": 239041, "requestCount": 29, - "scriptRawBytes": 1897694, + "scriptRawBytes": 1884943, "scriptRequestCount": 25, - "totalRawBytes": 2158882 + "totalRawBytes": 2149807 }, "routeReadyMedianMs": 215.30000019073486, "scriptSourceOwnerSet": "launchpad-static", @@ -67,12 +67,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 0, - "largestAssetRawBytes": 752431, - "otherRawBytes": 235365, + "largestAssetRawBytes": 752422, + "otherRawBytes": 239041, "requestCount": 29, - "scriptRawBytes": 1897694, + "scriptRawBytes": 1884943, "scriptRequestCount": 25, - "totalRawBytes": 2158882 + "totalRawBytes": 2149807 }, "routeReadyMedianMs": 225.39999961853027, "scriptSourceOwnerSet": "launchpad-static", @@ -105,12 +105,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752431, - "otherRawBytes": 235365, + "largestAssetRawBytes": 752422, + "otherRawBytes": 239041, "requestCount": 87, - "scriptRawBytes": 4312524, + "scriptRawBytes": 4326894, "scriptRequestCount": 82, - "totalRawBytes": 4618110 + "totalRawBytes": 4636156 }, "routeReadyMedianMs": 360.90000009536743, "scriptSourceOwnerSet": "editor-minimal-static", @@ -143,12 +143,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752431, - "otherRawBytes": 235365, + "largestAssetRawBytes": 752422, + "otherRawBytes": 239041, "requestCount": 87, - "scriptRawBytes": 4312524, + "scriptRawBytes": 4326894, "scriptRequestCount": 82, - "totalRawBytes": 4618110 + "totalRawBytes": 4636156 }, "routeReadyMedianMs": 367.80000019073486, "scriptSourceOwnerSet": "editor-minimal-static", @@ -159,12 +159,12 @@ "cssRawBytes": 0, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 317101, + "largestAssetRawBytes": 341949, "otherRawBytes": 0, "requestCount": 9, - "scriptRawBytes": 540817, + "scriptRawBytes": 567866, "scriptRequestCount": 9, - "totalRawBytes": 540817 + "totalRawBytes": 567866 }, "domContentLoadedMedianMs": 100.40000009536743, "id": "editor-canvas", @@ -181,12 +181,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752431, - "otherRawBytes": 235365, + "largestAssetRawBytes": 752422, + "otherRawBytes": 239041, "requestCount": 96, - "scriptRawBytes": 4853620, + "scriptRawBytes": 4894760, "scriptRequestCount": 91, - "totalRawBytes": 5159206 + "totalRawBytes": 5204022 }, "routeReadyMedianMs": 1095.3000001907349, "scriptSourceOwnerSet": "editor-canvas-static", @@ -197,12 +197,12 @@ "cssRawBytes": 0, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 317101, + "largestAssetRawBytes": 341949, "otherRawBytes": 0, "requestCount": 9, - "scriptRawBytes": 540817, + "scriptRawBytes": 567866, "scriptRequestCount": 9, - "totalRawBytes": 540817 + "totalRawBytes": 567866 }, "domContentLoadedMedianMs": 97.7000002861023, "id": "editor-canvas", @@ -219,12 +219,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752431, - "otherRawBytes": 235365, + "largestAssetRawBytes": 752422, + "otherRawBytes": 239041, "requestCount": 96, - "scriptRawBytes": 4853620, + "scriptRawBytes": 4894760, "scriptRequestCount": 91, - "totalRawBytes": 5159206 + "totalRawBytes": 5204022 }, "routeReadyMedianMs": 1042.5, "scriptSourceOwnerSet": "editor-canvas-static", @@ -235,12 +235,12 @@ "cssRawBytes": 15413, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 208808, + "largestAssetRawBytes": 208717, "otherRawBytes": 0, "requestCount": 7, - "scriptRawBytes": 452154, + "scriptRawBytes": 454188, "scriptRequestCount": 6, - "totalRawBytes": 467567 + "totalRawBytes": 469601 }, "domContentLoadedMedianMs": 101.40000009536743, "id": "editor-workflow", @@ -257,12 +257,12 @@ "cssRawBytes": 17572, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752431, - "otherRawBytes": 235365, + "largestAssetRawBytes": 752422, + "otherRawBytes": 239041, "requestCount": 94, - "scriptRawBytes": 4772177, + "scriptRawBytes": 4781082, "scriptRequestCount": 88, - "totalRawBytes": 5093176 + "totalRawBytes": 5105757 }, "routeReadyMedianMs": 1093.0999999046326, "scriptSourceOwnerSet": "editor-workflow-static", @@ -273,12 +273,12 @@ "cssRawBytes": 15413, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 208808, + "largestAssetRawBytes": 208717, "otherRawBytes": 0, "requestCount": 7, - "scriptRawBytes": 452154, + "scriptRawBytes": 454188, "scriptRequestCount": 6, - "totalRawBytes": 467567 + "totalRawBytes": 469601 }, "domContentLoadedMedianMs": 99.89999961853027, "id": "editor-workflow", @@ -295,12 +295,12 @@ "cssRawBytes": 17572, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752431, - "otherRawBytes": 235365, + "largestAssetRawBytes": 752422, + "otherRawBytes": 239041, "requestCount": 94, - "scriptRawBytes": 4772177, + "scriptRawBytes": 4781082, "scriptRequestCount": 88, - "totalRawBytes": 5093176 + "totalRawBytes": 5105757 }, "routeReadyMedianMs": 1105.0999999046326, "scriptSourceOwnerSet": "editor-workflow-static", @@ -333,12 +333,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752431, - "otherRawBytes": 235365, + "largestAssetRawBytes": 752422, + "otherRawBytes": 239041, "requestCount": 87, - "scriptRawBytes": 4312524, + "scriptRawBytes": 4326894, "scriptRequestCount": 82, - "totalRawBytes": 4618110 + "totalRawBytes": 4636156 }, "routeReadyMedianMs": 1324.5, "scriptSourceOwnerSet": "editor-minimal-static", @@ -371,12 +371,12 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 752431, - "otherRawBytes": 235365, + "largestAssetRawBytes": 752422, + "otherRawBytes": 239041, "requestCount": 87, - "scriptRawBytes": 4312524, + "scriptRawBytes": 4326894, "scriptRequestCount": 82, - "totalRawBytes": 4618110 + "totalRawBytes": 4636156 }, "routeReadyMedianMs": 1301.5, "scriptSourceOwnerSet": "editor-minimal-static", From 11c5680304c2d1f10591272d97912b9935cc0cef Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sun, 25 Oct 2026 00:46:30 -0600 Subject: [PATCH 23/35] feat(canvas): adopt schema 6 for control layer derivation fields Schema 5 stays for the noise, clarity, texture and dehaze adjustments that main already ships; a control leaf with a derivation field now requires 6, so older builds refuse the document instead of stripping the fields. The schema walk takes the maximum over a node and its children. --- .../development/Front End/project-files.mdx | 8 ++-- .../src/workbench/canvas-engine/contracts.ts | 14 +++--- .../src/workbench/canvasMigration.test.ts | 46 +++++++++++++------ .../webv2/src/workbench/canvasMigration.ts | 35 +++++++------- .../src/workbench/canvasSchemaVersion.test.ts | 8 +++- .../src/workbench/canvasSchemaVersion.ts | 6 +-- .../webv2/src/workbench/projects/README.md | 4 +- .../webv2/src/workbench/projects/api.test.ts | 18 ++++---- .../workbench/projects/projectFile.test.ts | 38 +++++++++++++++ .../src/workbench/workbenchState.test.ts | 8 ++-- 10 files changed, 125 insertions(+), 60 deletions(-) diff --git a/docs/src/content/docs/development/Front End/project-files.mdx b/docs/src/content/docs/development/Front End/project-files.mdx index 5dcf56162fb..21a501637a1 100644 --- a/docs/src/content/docs/development/Front End/project-files.mdx +++ b/docs/src/content/docs/development/Front End/project-files.mdx @@ -44,10 +44,12 @@ would drop part of it on load and lose it on the next autosave. Schema 4 marks c typography (a text source with `fontRef`, `fontStyle` or `fontVariations`). Schema 5 marks a noise, clarity, texture or dehaze adjustment, enabled or not, on any layer or group at any depth; an older reader would drop the -whole adjustment stack holding it. The live document and each saved canvas snapshot are versioned +whole adjustment stack holding it. Schema 6 marks a control layer carrying any derivation field +(`controlType`, `detection`, `original` or `originalTransform`), which an older reader would strip, +losing the kept original. The live document and each saved canvas snapshot are versioned this way, and the canvas state takes the newest of them. Adoption is sticky: removing the feature -does not lower the version, as the server floor cannot be lowered either. A canvas with neither -stays at 3, so older builds keep opening it. A build refuses a canvas that declares a schema above +does not lower the version, as the server floor cannot be lowered either. A canvas with none of +these stays at 3, so older builds keep opening it. A build refuses a canvas that declares a schema above its maximum as an unsupported version before parsing it, and once the floor is raised the server refuses the project to a client declaring a lower maximum. diff --git a/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts b/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts index 6576203376f..9061599e913 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-engine/contracts.ts @@ -479,12 +479,12 @@ export const CANVAS_MAX_NODE_COUNT = 10_000; export interface CanvasDocumentContractV3 { /** * Schema this document needs: 3 for plain content, 4 for custom-font typography, 5 for noise or spatial - * (clarity, texture, dehaze) adjustments or control-layer derivation fields. Each document mutation raises it to the - * newer of its content's need and the canvas state's version. A restored snapshot's document keeps its own, possibly - * lower, version until the next mutation; the state's version and the project's compatibility floor stay put - * meanwhile. + * (clarity, texture, dehaze) adjustments, 6 for control-layer derivation fields. Each document mutation raises it to + * the newer of its content's need and the canvas state's version. A restored snapshot's document keeps its own, + * possibly lower, version until the next mutation; the state's version and the project's compatibility floor stay + * put meanwhile. */ - version: 3 | 4 | 5; + version: 3 | 4 | 5 | 6; width: number; height: number; background: 'transparent' | { color: string }; @@ -508,10 +508,10 @@ export interface CanvasStagingAreaContractV2 extends CanvasStagingAreaContract { export interface CanvasStateContractV3 { /** * Newest schema the state needs: the newest of its previous version, the live document's and every snapshot's - * (3 plain, 4 custom-font typography, 5 noise or spatial adjustments or control-layer derivation fields). Sticky: + * (3 plain, 4 custom-font typography, 5 noise or spatial adjustments, 6 control-layer derivation fields). Sticky: * removing that content never lowers it. */ - version: 3 | 4 | 5; + version: 3 | 4 | 5 | 6; document: CanvasDocumentContractV3; /** * Monotonic wholesale-replacement counter. The mirror clears pixel history whenever it changes, including diff --git a/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts b/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts index 24efd8744c0..e5ccb806059 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts @@ -916,16 +916,36 @@ describe('loadCanvasState', () => { expect(Object.keys(actual as object).sort()).toEqual(Object.keys(expected).sort()); }; - it('round-trips a typed, detected layer and writes a v3 or v4 canvas as version 5', () => { - for (const declared of [3, 4]) { + it('round-trips a typed, detected layer and writes a v3, v4 or v5 canvas as version 6', () => { + for (const declared of [3, 4, 5]) { const loaded = load(declaring(withNodes([typed], 'control'), declared)); expectExactly(loaded.document.stacks.control[0], typed); - expect(loaded.version).toBe(5); - expect(loaded.document.version).toBe(5); + expect(loaded.version).toBe(6); + expect(loaded.document.version).toBe(6); } expect( load(withNodes([groupContract('outer', [groupContract('inner', [typed as never])])], 'control')).version - ).toBe(5); + ).toBe(6); + }); + + it.each([ + ['controlType', { controlType: 'edges' }], + ['detection', { detection: null }], + ['original', { original: { image: image('photo.png'), type: 'image' } }], + ['originalTransform', { originalTransform: { rotation: 0, scaleX: 2, scaleY: 2, x: 0, y: 0 } }], + ])('requires version 6 for a control leaf carrying only %s', (_key, fields) => { + const stacks = withNodes([{ ...legacy, ...fields }], 'control').document.stacks; + expect(writableCanvasSchemaVersion([3], { stacks })).toBe(6); + }); + + it('requires version 6, not 5, when a group adjustment and a typed child share a document', () => { + const spatial = { amount: 40, id: 'clarity', isEnabled: true, size: 40, type: 'clarity' }; + const graded = groupContract('graded', [typed as never], { adjustments: [spatial as never] }); + expect(load(withNodes([graded], 'control')).version).toBe(6); + + const adjusted = withNodes([{ ...createEmptyPaintLayer('Detail', 'detail'), adjustments: [spatial] }]); + const stacks = { ...adjusted.document.stacks, control: [typed] }; + expect(load({ ...adjusted, document: { ...adjusted.document, stacks } }).version).toBe(6); }); it('keeps a legacy control layer unchanged and at its declared version', () => { @@ -935,7 +955,7 @@ describe('loadCanvasState', () => { expect(loaded.document.version).toBe(3); }); - it('does not adopt version 5 for derivation keys present only as undefined', () => { + it('does not adopt version 6 for derivation keys present only as undefined', () => { const undefinedKeys = { ...legacy, controlType: undefined, detection: undefined, original: undefined }; expect(writableCanvasSchemaVersion([3], { stacks: withNodes([undefinedKeys], 'control').document.stacks })).toBe( 3 @@ -943,20 +963,20 @@ describe('loadCanvasState', () => { expect(load(withNodes([undefinedKeys], 'control')).version).toBe(3); }); - it('never lowers a declared version 5 once the fields are gone', () => { - const loaded = load(declaring(withNodes([legacy], 'control'), 5)); - expect(loaded.version).toBe(5); - expect(loaded.document.version).toBe(5); + it('never lowers a declared version 6 once the fields are gone', () => { + const loaded = load(declaring(withNodes([legacy], 'control'), 6)); + expect(loaded.version).toBe(6); + expect(loaded.document.version).toBe(6); }); - it('writes version 5 when only a snapshot holds a typed layer, leaving the live document at its own version', () => { + it('writes version 6 when only a snapshot holds a typed layer, leaving the live document at its own version', () => { const loaded = load({ ...createEmptyCanvasState(), snapshots: [{ createdAt: 'now', document: withNodes([typed], 'control').document, id: 's', name: 'S' }], }); - expect(loaded.version).toBe(5); + expect(loaded.version).toBe(6); expect(loaded.document.version).toBe(3); - expect(loaded.snapshots[0]?.document.version).toBe(5); + expect(loaded.snapshots[0]?.document.version).toBe(6); }); it('keeps an unknown detector type and its settings', () => { diff --git a/invokeai/frontend/webv2/src/workbench/canvasMigration.ts b/invokeai/frontend/webv2/src/workbench/canvasMigration.ts index e17915f0339..e541406f7c3 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasMigration.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasMigration.ts @@ -416,30 +416,37 @@ const hasV5Adjustment = (node: Record): boolean => Array.isArray(node.adjustments) && node.adjustments.some((entry) => isRecord(entry) && V5_ADJUSTMENT_TYPES.has(entry.type)); -/** The control derivation fields a v4 reader would strip on load, losing the kept original on its next save. */ +/** The control derivation fields a v5 reader would strip on load, losing the kept original on its next save. */ const CONTROL_DERIVATION_KEYS = ['controlType', 'detection', 'original', 'originalTransform'] as const; -const controlLayerRequiresV5 = (node: Record): boolean => +const controlLayerRequiresV6 = (node: Record): boolean => node.type === 'control' && CONTROL_DERIVATION_KEYS.some((key) => node[key] !== undefined); +const ownSchemaRequirement = (node: Record): CanvasSchemaVersion => { + if (controlLayerRequiresV6(node)) { + return 6; + } + if (hasV5Adjustment(node)) { + return 5; + } + return textSourceRequiresV4(node.source) ? 4 : 3; +}; + /** - * v4: custom-font typography. v5: a noise or spatial adjustment, enabled or not, which a v4 reader would drop, or a - * control layer carrying any derivation field. + * The newest schema a node or any descendant needs. v4: custom-font typography. v5: a noise or spatial adjustment, + * enabled or not, which a v4 reader would drop. v6: a control layer carrying any derivation field. */ const nodeSchemaRequirement = (node: unknown): CanvasSchemaVersion => { if (!isRecord(node)) { return 3; } - if (hasV5Adjustment(node) || controlLayerRequiresV5(node)) { - return 5; - } - let required: CanvasSchemaVersion = textSourceRequiresV4(node.source) ? 4 : 3; + let required = ownSchemaRequirement(node); if (Array.isArray(node.children)) { for (const child of node.children) { - required = Math.max(required, nodeSchemaRequirement(child)) as CanvasSchemaVersion; if (required === MAX_SUPPORTED_CANVAS_SCHEMA_VERSION) { break; } + required = Math.max(required, nodeSchemaRequirement(child)) as CanvasSchemaVersion; } } return required; @@ -519,15 +526,7 @@ type LoadStep = Extract, { status: 'loaded' }> | Refusal; type DeclaredVersion = { kind: 'current' } | { kind: 'other'; version: number } | { kind: 'malformed' }; const classifyVersion = (value: unknown): DeclaredVersion => { - if (value === MAX_SUPPORTED_CANVAS_SCHEMA_VERSION) { - return { kind: 'current' }; - } - if ( - typeof value === 'number' && - Number.isInteger(value) && - value >= MIN_SUPPORTED_CANVAS_SCHEMA_VERSION && - value < MAX_SUPPORTED_CANVAS_SCHEMA_VERSION - ) { + if (isSupportedCanvasSchemaVersion(value)) { return { kind: 'current' }; } if (typeof value === 'number' && Number.isInteger(value) && value >= 1) { diff --git a/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.test.ts b/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.test.ts index 029f3d2bf2d..b6f7c234d1d 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { getProjectCanvasSchemaRequirement } from './canvasSchemaVersion'; +import { getProjectCanvasSchemaRequirement, isCanvasSchemaVersionSupported } from './canvasSchemaVersion'; describe('getProjectCanvasSchemaRequirement', () => { it('uses only the live canvas schema version', () => { @@ -19,3 +19,9 @@ describe('getProjectCanvasSchemaRequirement', () => { expect(() => getProjectCanvasSchemaRequirement(document)).toThrow(); }); }); + +describe('isCanvasSchemaVersionSupported', () => { + it('opens projects needing schema 3 through 6, including control-layer derivation, and refuses the rest', () => { + expect([2, 3, 4, 5, 6, 7].map(isCanvasSchemaVersionSupported)).toEqual([false, true, true, true, true, false]); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts b/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts index e125abcb983..c9017c6fdb9 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasSchemaVersion.ts @@ -4,10 +4,10 @@ export const MIN_SUPPORTED_CANVAS_SCHEMA_VERSION = 3; /** * Newest canvas document schema this build can safely read, edit, and write. A document adopts a newer schema only * when it carries something older builds would drop on load and then lose on save: 4 for custom-font typography, 5 - * for noise, clarity, texture or dehaze adjustments or control-layer derivation fields (`controlType`, `detection`, - * `original`, `originalTransform`). Older builds refuse the newer schema instead. + * for noise, clarity, texture or dehaze adjustments, 6 for control-layer derivation fields (`controlType`, + * `detection`, `original`, `originalTransform`). Older builds refuse the newer schema instead. */ -export const MAX_SUPPORTED_CANVAS_SCHEMA_VERSION = 5; +export const MAX_SUPPORTED_CANVAS_SCHEMA_VERSION = 6; /** Whether a project whose documents need at least `minimumCanvasSchemaVersion` can be opened here. */ export const isCanvasSchemaVersionSupported = (minimumCanvasSchemaVersion: number): boolean => diff --git a/invokeai/frontend/webv2/src/workbench/projects/README.md b/invokeai/frontend/webv2/src/workbench/projects/README.md index 12f07ed5e97..9b957447684 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/README.md +++ b/invokeai/frontend/webv2/src/workbench/projects/README.md @@ -74,7 +74,7 @@ Cold-start offline editing not supported. When backend cannot load, unavailable New value in a closed canvas set, such as a control layer's adapter kind (`anima_lllite`), is added without raising canvas schema version or project's compatibility floor. Builds that know the value load and save it unchanged; builds that do not refuse the canvas instead of rewriting it, since coercing an unknown kind to a known one would replace the user's adapter on next autosave. In an older build the project is not opened and user sees that it "contains canvas data this version of Invoke cannot read"; saved copy is untouched and exportable, and a build that knows the value opens it as saved. Raise schema version instead when older builds could misread a document without refusing it. -Adjustment stacks are such a case: load drops a stack holding any entry it cannot parse, so an older build would load a document with a new adjustment type minus that layer's whole stack and autosave the loss. A new adjustment type therefore raises the canvas schema for documents that use it. A noise, clarity, texture or dehaze adjustment, enabled or not, on any layer or group at any depth of the live document or a saved snapshot, makes the canvas schema 5 (`canvasMigration.ts`, `writableCanvasSchemaVersion`), as control-layer derivation fields do (see [Control layer derivation](#control-layer-derivation)); a document with neither keeps its version. Adoption is sticky: removing the last such entry leaves the document at 5, matching the server's monotonic floor. A build whose maximum is 4 refuses the project at the server and the canvas at load as an unsupported version. +Adjustment stacks are such a case: load drops a stack holding any entry it cannot parse, so an older build would load a document with a new adjustment type minus that layer's whole stack and autosave the loss. A new adjustment type therefore raises the canvas schema for documents that use it. A noise, clarity, texture or dehaze adjustment, enabled or not, on any layer or group at any depth of the live document or a saved snapshot, makes the canvas schema at least 5 (`canvasMigration.ts`, `writableCanvasSchemaVersion`); control-layer derivation fields raise it to 6 (see [Control layer derivation](#control-layer-derivation)). A document with neither keeps its version. Adoption is sticky: removing the last such entry leaves the document at its adopted schema, matching the server's monotonic floor. A build whose maximum is lower refuses the project at the server and the canvas at load as an unsupported version. Load does not reinterpret a known value either. A control layer saved with a kind the selected main model cannot run (an Anima layer saved as ControlNet before ControlNet-LLLite was supported) keeps its kind and model. Choosing a control type for it re-resolves the model for the selected main model, and the kind follows the resolved model. @@ -90,7 +90,7 @@ Import verifies declared entries and checksums, then validates all embedded font A control layer may keep the image it derives its map from, so detection can re-run. Four optional fields on the control leaf (`canvas-engine/contracts.ts`) carry it: `controlType` (what the image controls; a closed set, so an unknown value refuses the canvas), `detection` (the detector and its settings; `null` means "use the image as-is"; the detector name stays open), `original` (the pre-detection image or paint bitmap, a persisted image reference) and `originalTransform` (the original's frame inside the map's layer frame; absent means identity). `source` stays the pixels generation uses (the map once detected) and `filter` what produced them. Invariants, owned by `canvas-engine/document/controlDerivation.ts`: no type means none of the other three; a `null` detection keeps no original; an original implies a detection and a derived `source`. Load repairs violations silently by dropping the bookkeeping that no longer holds, never the pixels. -A build that predates these fields would strip them on load and lose the original on its next save, so a control leaf carrying any of them makes the canvas schema 5 (`canvasMigration.ts`, `writableCanvasSchemaVersion`), the same schema a noise, clarity, texture or dehaze adjustment raises. Documents with neither keep their version, and adoption is sticky like the server's floor. `original` names an image by `imageName` like every other reference, so asset holding, cleanup and `.invk` export cover it unchanged. +A build that predates these fields would strip them on load and lose the original on its next save, so a control leaf carrying any of them makes the canvas schema 6 (`canvasMigration.ts`, `writableCanvasSchemaVersion`), one above the schema 5 that noise, clarity, texture and dehaze adjustments raise: a schema 5 build cannot read the fields and refuses the canvas instead. Documents without them keep their version, and adoption is sticky like the server's floor. `original` names an image by `imageName` like every other reference, so asset holding, cleanup and `.invk` export cover it unchanged. ## Verification diff --git a/invokeai/frontend/webv2/src/workbench/projects/api.test.ts b/invokeai/frontend/webv2/src/workbench/projects/api.test.ts index acde0b97948..c921eacf3ec 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/api.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projects/api.test.ts @@ -42,7 +42,7 @@ describe('createProjectSettled', () => { const post = () => ({ body: JSON.stringify({ ...request, - max_canvas_schema_version: 5, + max_canvas_schema_version: 6, max_document_schema_version: 4, minimum_canvas_schema_version: 3, }), @@ -93,7 +93,7 @@ describe('createProjectSettled', () => { expect(transport.apiFetchJson).toHaveBeenNthCalledWith(2, '/api/v1/projects/', post()); expect(transport.apiFetchJson).toHaveBeenNthCalledWith( 3, - '/api/v1/projects/project-1?max_canvas_schema_version=5&max_document_schema_version=4', + '/api/v1/projects/project-1?max_canvas_schema_version=6&max_document_schema_version=4', expect.anything() ); }); @@ -173,7 +173,7 @@ describe('createProjectSettled', () => { await expect(createProjectSettled(request, captureAccountScope())).resolves.toMatchObject(project); expect(transport.apiFetchJson).toHaveBeenNthCalledWith( 3, - '/api/v1/projects/project-1?max_canvas_schema_version=5&max_document_schema_version=4', + '/api/v1/projects/project-1?max_canvas_schema_version=6&max_document_schema_version=4', expect.anything() ); @@ -265,7 +265,7 @@ describe('canvas schema compatibility declarations', () => { await getProject('project/one'); expect(transport.apiFetchJson).toHaveBeenCalledWith( - '/api/v1/projects/project%2Fone?max_canvas_schema_version=5&max_document_schema_version=4', + '/api/v1/projects/project%2Fone?max_canvas_schema_version=6&max_document_schema_version=4', { signal: undefined } ); }); @@ -280,7 +280,7 @@ describe('canvas schema compatibility declarations', () => { data: {}, minimum_document_schema_version: 4, name: 'Project', - max_canvas_schema_version: 5, + max_canvas_schema_version: 6, max_document_schema_version: 4, minimum_canvas_schema_version: 3, }), @@ -305,7 +305,7 @@ describe('canvas schema compatibility declarations', () => { expected_revision: 4, minimum_canvas_schema_version: 3, name: 'Project', - max_canvas_schema_version: 5, + max_canvas_schema_version: 6, max_document_schema_version: 4, }), method: 'PUT', @@ -507,7 +507,7 @@ describe('project workspaces', () => { body: JSON.stringify({ data: { layout: {} }, expected_revision: 1, - max_canvas_schema_version: 5, + max_canvas_schema_version: 6, max_document_schema_version: 4, }), method: 'PUT', @@ -535,7 +535,7 @@ describe('project workspaces', () => { minimum_document_schema_version: 4, name: 'Project', workspace, - max_canvas_schema_version: 5, + max_canvas_schema_version: 6, max_document_schema_version: 4, }), method: 'PUT', @@ -581,7 +581,7 @@ describe('project workspaces', () => { minimum_document_schema_version: 4, name: 'Project', workspace: { layout: {} }, - max_canvas_schema_version: 5, + max_canvas_schema_version: 6, max_document_schema_version: 4, minimum_canvas_schema_version: 3, }), diff --git a/invokeai/frontend/webv2/src/workbench/projects/projectFile.test.ts b/invokeai/frontend/webv2/src/workbench/projects/projectFile.test.ts index 3771878389f..80abc4a87ad 100644 --- a/invokeai/frontend/webv2/src/workbench/projects/projectFile.test.ts +++ b/invokeai/frontend/webv2/src/workbench/projects/projectFile.test.ts @@ -1,3 +1,4 @@ +import { createControlLayer } from '@workbench/canvas-engine/api'; import { stacksFrom } from '@workbench/canvas-engine/document-model/documentFixtures.testStub'; import { MAX_SUPPORTED_CANVAS_SCHEMA_VERSION } from '@workbench/canvasSchemaVersion'; import { createDraftProject } from '@workbench/workbenchState'; @@ -782,6 +783,43 @@ describe('importProjectFile', () => { expect(createRequest.data.canvas.document.stacks.raster[0]?.adjustments).toEqual([noise]); }); + it('carries a typed control layer through export and import at canvas schema 6', async () => { + const typed = { + ...createControlLayer({ base: null, id: 'pose', name: 'Pose' }), + controlType: 'pose' as const, + detection: null, + }; + const draft = createDraftProject([]); + const project = { + ...draft, + canvas: { + ...draft.canvas, + document: { ...draft.canvas.document, stacks: stacksFrom([typed]), version: 6 as const }, + version: 6 as const, + }, + name: 'Pose', + }; + + acceptCreate(); + await projectFile.exportOpenProject(project); + const { readArchive, readEntryText } = await import('./invk/archive'); + const entries = await readArchive(new Uint8Array(await capturedArchive().arrayBuffer())); + expect(JSON.parse(readEntryText(entries.get('manifest.json')!))).toMatchObject({ minimumCanvasSchemaVersion: 6 }); + + await projectFile.importProjectFile(capturedArchive()); + + const createRequest = api.createProjectSettled.mock.calls[0]![0] as { + data: { canvas: { document: { stacks: { control: Record[] } }; version: number } }; + minimum_canvas_schema_version: number; + }; + expect(createRequest.minimum_canvas_schema_version).toBe(6); + expect(createRequest.data.canvas.version).toBe(6); + expect(createRequest.data.canvas.document.stacks.control[0]).toMatchObject({ + controlType: 'pose', + detection: null, + }); + }); + describe('Auto-switch of an imported project', () => { /** A project as a build before Auto-switch saved it: the old progress toggle, no mode. */ const legacySettingsProject = () => { diff --git a/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts b/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts index c8578b8eea2..c9909152477 100644 --- a/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts +++ b/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts @@ -9379,18 +9379,18 @@ describe('canvas schema adoption', () => { }); const versions = (state: WorkbenchState) => [getCanvas(state).version, getCanvas(state).document.version]; - it('moves a v4 canvas to schema 5 when a control layer gains a type, and keeps 5 after it loses it', () => { + it('moves a v4 canvas to schema 6 when a control layer gains a type, and keeps 6 after it loses it', () => { let state = withV4Canvas([createControlLayer('ctrl')]); expect(versions(state)).toEqual([4, 4]); state = workbenchReducer(state, { layer: typedControl('ctrl'), layerId: 'ctrl', type: 'replaceCanvasLayer' }); - expect(versions(state)).toEqual([5, 5]); + expect(versions(state)).toEqual([6, 6]); state = workbenchReducer(state, { layer: createControlLayer('ctrl'), layerId: 'ctrl', type: 'replaceCanvasLayer' }); - expect(versions(state)).toEqual([5, 5]); + expect(versions(state)).toEqual([6, 6]); state = workbenchReducer(state, { document: createEmptyCanvasDocument(), type: 'replaceCanvasDocument' }); - expect(getCanvas(state).version).toBe(5); + expect(getCanvas(state).version).toBe(6); }); it('adopts 4, not 5, for custom-font typography alone', () => { From 2d3f5f8669651336b4a5ac0be465ffc4901e1f16 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sun, 25 Oct 2026 00:46:30 -0600 Subject: [PATCH 24/35] fix(backend): ignore the InstantX control mode on non-union FLUX ControlNets A checkpoint without a mode embedder asserted when a mode arrived, although the field is documented as ignored for such models. --- .../controlnet/instantx_controlnet_flux.py | 4 +- .../test_instantx_controlnet_flux.py | 79 +++++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 tests/backend/flux/controlnet/test_instantx_controlnet_flux.py diff --git a/invokeai/backend/flux/controlnet/instantx_controlnet_flux.py b/invokeai/backend/flux/controlnet/instantx_controlnet_flux.py index 1af5fbdfc09..e981ba074d9 100644 --- a/invokeai/backend/flux/controlnet/instantx_controlnet_flux.py +++ b/invokeai/backend/flux/controlnet/instantx_controlnet_flux.py @@ -143,8 +143,8 @@ def forward( controlnet_mode_emb = self.controlnet_mode_embedder(controlnet_mode) txt = torch.cat([controlnet_mode_emb, txt], dim=1) txt_ids = torch.cat([txt_ids[:, :1, :], txt_ids], dim=1) - else: - assert controlnet_mode is None + # A non-union model has no mode embedder, so a supplied controlnet_mode is ignored. Callers pick the mode from + # the model name, and some checkpoints keep "union" in the name after their mode embedding was removed. ids = torch.cat((txt_ids, img_ids), dim=1) pe = self.pe_embedder(ids) diff --git a/tests/backend/flux/controlnet/test_instantx_controlnet_flux.py b/tests/backend/flux/controlnet/test_instantx_controlnet_flux.py new file mode 100644 index 00000000000..378f856de11 --- /dev/null +++ b/tests/backend/flux/controlnet/test_instantx_controlnet_flux.py @@ -0,0 +1,79 @@ +import torch + +from invokeai.backend.flux.controlnet.instantx_controlnet_flux import ( + InstantXControlNetFlux, + InstantXControlNetFluxOutput, +) +from invokeai.backend.flux.model import FluxParams + +TINY_PARAMS = FluxParams( + in_channels=4, + vec_in_dim=4, + context_in_dim=4, + hidden_size=16, + mlp_ratio=2.0, + num_heads=2, + depth=1, + depth_single_blocks=1, + axes_dim=[2, 2, 4], + theta=10_000, + qkv_bias=True, + guidance_embed=False, +) + + +def _tiny_controlnet(num_control_modes: int | None) -> InstantXControlNetFlux: + torch.manual_seed(0) + model = InstantXControlNetFlux(TINY_PARAMS, num_control_modes=num_control_modes) + # The ControlNet output projections start at zero, which would make every output identical. Randomize all weights + # so that a change in conditioning shows up in the outputs. + with torch.no_grad(): + for param in model.parameters(): + param.normal_(std=0.5) + return model.eval() + + +def _forward(model: InstantXControlNetFlux, controlnet_mode: torch.Tensor | None) -> InstantXControlNetFluxOutput: + generator = torch.Generator().manual_seed(1) + img_tokens, txt_tokens = 6, 3 + img = torch.randn(1, img_tokens, TINY_PARAMS.in_channels, generator=generator) + with torch.no_grad(): + return model( + controlnet_cond=torch.randn(1, img_tokens, TINY_PARAMS.in_channels, generator=generator), + controlnet_mode=controlnet_mode, + img=img, + img_ids=torch.randn(1, img_tokens, 3, generator=generator), + txt=torch.randn(1, txt_tokens, TINY_PARAMS.context_in_dim, generator=generator), + txt_ids=torch.zeros(1, txt_tokens, 3), + timesteps=torch.tensor([0.5]), + y=torch.randn(1, TINY_PARAMS.vec_in_dim, generator=generator), + ) + + +def _samples(output: InstantXControlNetFluxOutput) -> list[torch.Tensor]: + assert output.controlnet_block_samples is not None + assert output.controlnet_single_block_samples is not None + return output.controlnet_block_samples + output.controlnet_single_block_samples + + +def test_non_union_controlnet_ignores_a_supplied_control_mode(): + model = _tiny_controlnet(num_control_modes=None) + + without_mode = _samples(_forward(model, controlnet_mode=None)) + with_mode = _samples(_forward(model, controlnet_mode=torch.tensor([[2]]))) + + assert len(with_mode) == len(without_mode) == 2 + for actual, expected in zip(with_mode, without_mode, strict=True): + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +def test_union_controlnet_conditions_on_the_control_mode(): + model = _tiny_controlnet(num_control_modes=5) + + without_mode = _samples(_forward(model, controlnet_mode=None)) + mode_0 = _samples(_forward(model, controlnet_mode=torch.tensor([[0]]))) + mode_2 = _samples(_forward(model, controlnet_mode=torch.tensor([[2]]))) + + for with_mode in (mode_0, mode_2): + assert any(not torch.allclose(a, b) for a, b in zip(with_mode, without_mode, strict=True)) + assert any(not torch.allclose(a, b) for a, b in zip(mode_0, mode_2, strict=True)) From 6954174ffd4dd853e9719f19726e9d5bcf95a85d Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sun, 25 Oct 2026 00:46:30 -0600 Subject: [PATCH 25/35] fix(canvas): treat the stored control adapter kind as advisory Load repair and the schema check keep weight, mode and step range within the loosest bounds instead of the stored kind's, because the panel, readiness validation and the graph work from the resolved kind. A layer edited under another main model no longer loses its values or fails to load. --- .../src/workbench/canvasMigration.test.ts | 33 +++++++++- .../webv2/src/workbench/canvasMigration.ts | 12 ++-- .../src/workbench/controlAdapters.test.ts | 62 +++++++++++-------- .../webv2/src/workbench/controlAdapters.ts | 54 ++++++++++------ 4 files changed, 107 insertions(+), 54 deletions(-) diff --git a/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts b/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts index e5ccb806059..a31bdbf1491 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasMigration.test.ts @@ -150,13 +150,40 @@ describe('loadCanvasState', () => { }); }); - it('repairs a persisted Qwen-Image control weight below the node floor with the kind default', () => { + it('keeps a control layer mode and negative weight whatever kind it stores, live, in snapshots and in session', () => { + // Created under Z-Image, then run on SDXL with a negative weight: the stored kind is advisory, the resolved one + // governs, so the document must load and save as it is. + const adapter = { + beginEndStepPct: [0.1, 0.9], + controlMode: 'more_control', + kind: 'z_image_control', + model: null, + weight: -0.3, + }; + const state = withNodes([{ ...createControlLayer({ id: 'z-control', name: 'Z Control' }), adapter }], 'control'); + const loaded = load({ + ...state, + snapshots: [{ createdAt: 'now', document: state.document, id: 'snapshot', name: 'Snapshot' }], + }); + const adapterOf = (document: (typeof loaded)['document']) => { + const layer = document.stacks.control[0]; + return layer?.type === 'control' ? layer.adapter : null; + }; + + expect(adapterOf(loaded.document)).toEqual(adapter); + expect(adapterOf(loaded.snapshots[0]!.document)).toEqual(adapter); + // Snapshot save and restore and document replace normalize in session; a refusal there would silently no-op. + const normalized = normalizeCanvasDocumentContract(loaded.document); + expect(normalized && adapterOf(normalized)).toEqual(adapter); + }); + + it('repairs a persisted control weight outside what any kind runs with the kind default', () => { const loaded = load( withNodes( [ { ...createControlLayer({ id: 'q-control', name: 'Q Control' }), - adapter: { kind: 'qwen_image_control', model: 'qwen-union', weight: -0.5 }, + adapter: { kind: 'qwen_image_control', model: 'qwen-union', weight: -1.5 }, }, ], 'control' @@ -450,7 +477,7 @@ describe('loadCanvasState', () => { it('normalizes control adapters in both the live document and saved snapshots', () => { const invalidLayer = { ...createControlLayer({ id: 'z-control', name: 'Z Control' }), - adapter: { beginEndStepPct: [0.8, 0.2], controlMode: null, kind: 'z_image_control', model: null, weight: -1 }, + adapter: { beginEndStepPct: [0.8, 0.2], controlMode: null, kind: 'z_image_control', model: null, weight: -1.5 }, }; const state = withNodes([invalidLayer], 'control'); const loaded = load({ diff --git a/invokeai/frontend/webv2/src/workbench/canvasMigration.ts b/invokeai/frontend/webv2/src/workbench/canvasMigration.ts index e541406f7c3..96a0ee5b8ab 100644 --- a/invokeai/frontend/webv2/src/workbench/canvasMigration.ts +++ b/invokeai/frontend/webv2/src/workbench/canvasMigration.ts @@ -12,7 +12,6 @@ import type { LayerStackKind, } from '@workbench/canvas-engine/api'; -import { getControlAdapterMinWeight } from '@features/generation/graph'; import { createItemId } from '@platform/browser/randomUuid'; import { CANVAS_MAX_NODE_COUNT, @@ -26,7 +25,7 @@ import { z } from 'zod'; import type { CanvasLoadDiagnostic, CanvasLoadResult, CanvasVersionScope } from './canvasLoadContracts'; import { MAX_SUPPORTED_CANVAS_SCHEMA_VERSION, MIN_SUPPORTED_CANVAS_SCHEMA_VERSION } from './canvasSchemaVersion'; -import { normalizeControlAdapter } from './controlAdapters'; +import { isStorableControlStepRange, isStorableControlWeight, normalizeControlAdapter } from './controlAdapters'; export const DEFAULT_CANVAS_DOCUMENT_WIDTH = 1024; export const DEFAULT_CANVAS_DOCUMENT_HEIGHT = 1024; @@ -163,11 +162,10 @@ const zControlAdapter = z model: z.string().nullable(), weight: zFiniteNumber, }) - .refine(({ beginEndStepPct }) => { - const [begin, end] = beginEndStepPct; - return begin >= 0 && end <= 1 && begin < end; - }) - .refine(({ kind, weight }) => weight >= getControlAdapterMinWeight(kind) && weight <= 2); + // The same bounds for every kind: the stored kind is advisory (the resolved kind governs in the panel, readiness + // validation and the graph), and a refusal here would reject the whole document. + .refine(({ beginEndStepPct }) => isStorableControlStepRange(beginEndStepPct)) + .refine(({ weight }) => isStorableControlWeight(weight)); const zMaskFill = z.object({ color: z.string(), style: z.enum(['solid', 'grid', 'crosshatch', 'diagonal', 'horizontal', 'vertical']), diff --git a/invokeai/frontend/webv2/src/workbench/controlAdapters.test.ts b/invokeai/frontend/webv2/src/workbench/controlAdapters.test.ts index 64ebd114f97..8d369babc98 100644 --- a/invokeai/frontend/webv2/src/workbench/controlAdapters.test.ts +++ b/invokeai/frontend/webv2/src/workbench/controlAdapters.test.ts @@ -22,43 +22,53 @@ describe('normalizeControlAdapter', () => { expect(normalizeControlAdapter(adapter)).toEqual(adapter); }); + it.each([null, 'z-image-union'])( + 'keeps the mode and a negative weight whatever kind is stored (model %s): the resolved kind governs', + (model) => { + // Created under Z-Image, now resolved to an SDXL ControlNet; a stored pick may belong to the other base. + const adapter = { + beginEndStepPct: [0.1, 0.8], + controlMode: 'more_control', + kind: 'z_image_control', + model, + weight: -0.5, + }; + expect(normalizeControlAdapter(adapter)).toEqual(adapter); + } + ); + it.each([ ['controlnet', Number.NaN], ['controlnet', Number.POSITIVE_INFINITY], ['controlnet', -1.01], ['controlnet', 2.01], - ['z_image_control', -0.01], + ['z_image_control', -1.5], ['z_image_control', 2.01], ['anima_lllite', 2.01], - ['qwen_image_control', -0.01], - ['qwen_image_control', 2.01], - ['flux2_control', -0.01], + ['qwen_image_control', -1.01], ['flux2_control', 2.01], - ] as const)('repairs invalid %s weight %s with its default', (kind, weight) => { - const adapter = { ...CONTROL_ADAPTER_DEFAULTS[kind], weight }; + ] as const)('repairs a %s weight of %s, outside what any kind runs, with its default', (kind, weight) => { + const adapter = { ...CONTROL_ADAPTER_DEFAULTS[kind], model: 'm', weight }; expect(normalizeControlAdapter(adapter)).toMatchObject({ kind, weight: CONTROL_ADAPTER_DEFAULTS[kind].weight }); }); - it.each(['z_image_control', 'qwen_image_control', 'flux2_control'] as const)( - 'rebuilds a %s adapter from the contract fields', - (kind) => { - const persisted = { - beginEndStepPct: [0.1, 0.9], - controlMode: 'balanced', - extra: true, - kind, - model: 'm', - weight: 1.5, - }; - expect(normalizeControlAdapter(persisted)).toEqual({ - beginEndStepPct: [0.1, 0.9], - controlMode: null, - kind, - model: 'm', - weight: 1.5, - }); - } - ); + it('maps the contract fields only, and repairs an unknown mode with the kind default', () => { + const persisted = { + beginEndStepPct: [0.1, 0.9], + controlMode: 'sideways', + extra: true, + kind: 'controlnet', + model: 7, + weight: 1.5, + }; + expect(normalizeControlAdapter(persisted)).toEqual({ + beginEndStepPct: [0.1, 0.9], + controlMode: 'balanced', + kind: 'controlnet', + model: null, + weight: 1.5, + }); + }); it.each([ [Number.NaN, 1], diff --git a/invokeai/frontend/webv2/src/workbench/controlAdapters.ts b/invokeai/frontend/webv2/src/workbench/controlAdapters.ts index f51eeabb219..f3d168251dc 100644 --- a/invokeai/frontend/webv2/src/workbench/controlAdapters.ts +++ b/invokeai/frontend/webv2/src/workbench/controlAdapters.ts @@ -1,7 +1,5 @@ import type { CanvasControlAdapterContract } from '@workbench/canvas-engine/api'; -import { areControlAdapterValuesValid, getControlAdapterMinWeight } from '@features/generation/graph'; - export const CONTROL_ADAPTER_DEFAULTS: Readonly< Record > = { @@ -103,29 +101,49 @@ const isRecord = (value: unknown): value is Record => typeof va const isControlAdapterKind = (value: unknown): value is CanvasControlAdapterContract['kind'] => typeof value === 'string' && Object.hasOwn(CONTROL_ADAPTER_DEFAULTS, value); -/** Repairs persisted numeric fields while preserving valid values and adapter kinds. */ +const CONTROL_MODES: readonly unknown[] = ['balanced', 'more_prompt', 'more_control', 'unbalanced']; + +/** + * Whether a stored weight can be kept: finite and within -1..2, the widest range any kind accepts. A stored adapter's + * kind is advisory (the layer runs as the kind its model resolves to, which the panel, readiness validation and the + * graph all use), so persistence must not judge the weight by it: a Fun kind's 0 floor would refuse or reset a + * negative weight a layer created under that base may now validly run with. + */ +export const isStorableControlWeight = (weight: unknown): weight is number => + typeof weight === 'number' && Number.isFinite(weight) && weight >= -1 && weight <= 2; + +/** Whether a stored begin/end step range can be kept: finite, within 0..1 and ordered; the same for every kind. */ +export const isStorableControlStepRange = (range: unknown): range is [number, number] => + Array.isArray(range) && + range.length === 2 && + typeof range[0] === 'number' && + typeof range[1] === 'number' && + Number.isFinite(range[0]) && + Number.isFinite(range[1]) && + range[0] >= 0 && + range[1] <= 1 && + range[0] < range[1]; + +/** + * Maps a persisted adapter to the contract, keeping every stored value persistence can hold and repairing the rest + * with the stored kind's defaults. No kind-specific reset applies (see {@link isStorableControlWeight}): the stored + * kind and model are kept as they are, and the resolved kind governs what runs. + */ export const normalizeControlAdapter = (value: unknown): unknown => { if (!isRecord(value) || !isControlAdapterKind(value.kind)) { return value; } const defaults = CONTROL_ADAPTER_DEFAULTS[value.kind]; - const minWeight = getControlAdapterMinWeight(value.kind); - const weight = - typeof value.weight === 'number' && Number.isFinite(value.weight) && value.weight >= minWeight && value.weight <= 2 - ? value.weight - : defaults.weight; - const beginEndStepPct: [number, number] = areControlAdapterValuesValid(value.kind, weight, value.beginEndStepPct) - ? [value.beginEndStepPct[0], value.beginEndStepPct[1]] - : [defaults.beginEndStepPct[0], defaults.beginEndStepPct[1]]; - - if (value.kind !== 'z_image_control' && value.kind !== 'qwen_image_control' && value.kind !== 'flux2_control') { - return { ...defaults, ...value, beginEndStepPct, weight } satisfies CanvasControlAdapterContract; - } return { - beginEndStepPct, - controlMode: null, + beginEndStepPct: isStorableControlStepRange(value.beginEndStepPct) + ? [value.beginEndStepPct[0], value.beginEndStepPct[1]] + : [defaults.beginEndStepPct[0], defaults.beginEndStepPct[1]], + controlMode: + value.controlMode === null || CONTROL_MODES.includes(value.controlMode) + ? (value.controlMode as CanvasControlAdapterContract['controlMode']) + : defaults.controlMode, kind: value.kind, model: typeof value.model === 'string' ? value.model : null, - weight, + weight: isStorableControlWeight(value.weight) ? value.weight : defaults.weight, } satisfies CanvasControlAdapterContract; }; From 144bc97b10ff47dcd7279266b530a9edcb0aa545 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sun, 25 Oct 2026 00:46:30 -0600 Subject: [PATCH 26/35] fix(canvas): union coverage, remembered picks and untyped model choice SDXL union models cover their documented types, only candidate picks are remembered, untyped layers list every usable model, classification is cached per config, and the set-type intent takes a type. --- .../controlLayerIntents.test.ts | 10 --- .../canvas-operations/controlLayerIntents.ts | 14 +--- .../canvas-operations/operationTypes.ts | 2 +- .../webv2/src/workbench/controlLayerChecks.ts | 55 +++----------- .../workbench/controlModelResolution.test.ts | 75 ++++++++++++++++--- .../src/workbench/controlModelResolution.ts | 27 +++++-- .../webv2/src/workbench/controlTypes.test.ts | 16 +++- .../webv2/src/workbench/controlTypes.ts | 49 ++++++++++-- 8 files changed, 152 insertions(+), 96 deletions(-) diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.test.ts index d3946ed279f..ed24616c1c1 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.test.ts @@ -92,16 +92,6 @@ describe('applyControlLayerIntent: set-type', () => { expect(next.filter).toBeUndefined(); }); - it('removes the type and every derivation field, restoring the original first', () => { - const next = apply(detected(), { controlType: null, kind: 'set-type' }); - - expect(next.source).toEqual(photo); - for (const key of ['controlType', 'detection', 'original', 'originalTransform', 'filter'] as const) { - expect(next).not.toHaveProperty(key); - } - expect(next.adapter.model).toBe('explicit-model'); - }); - it('renames in the same patch and reports an unchanged type as the same object', () => { const typed = detected(); diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.ts index 4d1d84f9a5f..510bcf2d05e 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/controlLayerIntents.ts @@ -58,21 +58,9 @@ const withoutDerivedPixels = (layer: CanvasControlLayerContract): CanvasControlL const setType = ( layer: CanvasControlLayerContract, - controlType: ControlType | null, + controlType: ControlType, options: ControlLayerIntentOptions ): CanvasControlLayerContract => { - if (controlType === null) { - if (layer.controlType === undefined) { - return layer; - } - // Restore first, so the image the user brought is what an untyped layer keeps. - const next = { ...restoreControlOriginal(layer) }; - delete next.controlType; - delete next.detection; - delete next.original; - delete next.originalTransform; - return next; - } if (layer.controlType === controlType) { return layer; } diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/operationTypes.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/operationTypes.ts index 055dc14fb12..0ae42e57455 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/operationTypes.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/operationTypes.ts @@ -107,7 +107,7 @@ export type FilterCommitTarget = RasterFilterCommitTarget; * starts from the detector's defaults with the type's fixed settings; `draw` empties the image for painting. */ export type ControlLayerIntent = - | { kind: 'set-type'; controlType: ControlType | null; name?: string } + | { kind: 'set-type'; controlType: ControlType; name?: string } | { kind: 'set-detector'; detector: string | null } | { kind: 'set-detector-settings'; settings: Record } | { kind: 'replace-image'; image: CanvasImageRef; transform?: LayerTransform } diff --git a/invokeai/frontend/webv2/src/workbench/controlLayerChecks.ts b/invokeai/frontend/webv2/src/workbench/controlLayerChecks.ts index f1105fbf9f3..f42d4d833b0 100644 --- a/invokeai/frontend/webv2/src/workbench/controlLayerChecks.ts +++ b/invokeai/frontend/webv2/src/workbench/controlLayerChecks.ts @@ -3,7 +3,6 @@ import type { CanvasControlLayerContract, CanvasLayerContract } from '@workbench import { createControlValidationSequence, - getControlValidationReason, isControlKindSupportedForBase, type ControlAdapterKind, type ControlValidationReason, @@ -54,20 +53,6 @@ export interface ControlLayerJudgement { skip: boolean; } -const resolveLayer = ( - layer: CanvasControlLayerContract, - mainModel: MainModel, - models: readonly ModelConfig[], - context: ControlLayerCheckContext -): ControlModelResolution => - resolveControlModel({ - base: mainModel.base, - layer, - models, - remembered: context.remembered, - supportsKind: context.supportsKind, - }); - /** * Judges control layers in generation order, as the invocation does: per-generation limits count only layers that * passed. Invoke readiness, the settings panel and the invocation collector share it. @@ -79,7 +64,13 @@ export const createControlLayerSequence = ( ): ((layer: CanvasControlLayerContract) => ControlLayerJudgement) => { const validate = createControlValidationSequence(mainModel); return (layer) => { - const resolution = resolveLayer(layer, mainModel, models, context); + const resolution = resolveControlModel({ + base: mainModel.base, + layer, + models, + remembered: context.remembered, + supportsKind: context.supportsKind, + }); // A layer with no type and no model keeps its reason for the panel but is skipped; having no model, it never // passes, so it never claims a per-generation slot. const reason = @@ -120,31 +111,6 @@ export const getBlockingControlLayerIssues = (params: { return issues; }; -/** Ambient warnings include empty layers but omit per-kind limits, which belong in the Invoke tooltip. */ -const judgeAttention = ( - layer: CanvasControlLayerContract, - mainModel: MainModel, - models: readonly ModelConfig[], - context: ControlLayerCheckContext -): { resolution: ControlModelResolution; reason: ControlLayerIssueCode | null } => { - const resolution = resolveLayer(layer, mainModel, models, context); - const reason = - getControlValidationReason({ - adapterModel: resolution.model, - beginEndStepPct: layer.adapter.beginEndStepPct, - controlLoraIndex: 0, - flux2ControlIndex: 0, - kind: resolution.kind, - mainBase: mainModel.base, - mainVariant: mainModel.variant ?? undefined, - qwenImageControlIndex: 0, - unresolvedModel: resolution.unresolved, - weight: layer.adapter.weight, - zImageControlIndex: 0, - }) ?? (needsControlDetection(layer) ? 'detection_pending' : null); - return { reason, resolution }; -}; - /** * Where a layer's control map stands. `empty`: no pixels yet. `none`: no type or no detection, so the pixels are the * map. `current`: the map matches its detection. `stale`: detection must run again. `no-input`: the map is all the @@ -186,14 +152,13 @@ export const describeControlLayer = ( reason: resolution.unresolved ?? (needsControlDetection(layer) ? 'detection_pending' : null), resolution, }; - } else if (ctx.earlier) { + } else { + // Without `earlier` the layer comes first in a fresh sequence, so no per-generation limit can apply. const judge = createControlLayerSequence(ctx.mainModel, ctx.models, context); - for (const earlier of ctx.earlier) { + for (const earlier of ctx.earlier ?? []) { judge(earlier); } judgement = judge(layer); - } else { - judgement = judgeAttention(layer, ctx.mainModel, ctx.models, context); } return { map: getControlMapState(layer), reason: judgement.reason, resolution: judgement.resolution }; }; diff --git a/invokeai/frontend/webv2/src/workbench/controlModelResolution.test.ts b/invokeai/frontend/webv2/src/workbench/controlModelResolution.test.ts index f8dbc721c4a..2f21ac401c6 100644 --- a/invokeai/frontend/webv2/src/workbench/controlModelResolution.test.ts +++ b/invokeai/frontend/webv2/src/workbench/controlModelResolution.test.ts @@ -4,7 +4,12 @@ import type { CanvasControlLayerContract, ControlType } from '@workbench/canvas- import { describe, expect, it } from 'vitest'; -import { getTypeAvailability, resolveControlModel, type ControlModelMemory } from './controlModelResolution'; +import { + getTypeAvailability, + isRememberableControlPick, + resolveControlModel, + type ControlModelMemory, +} from './controlModelResolution'; const model = (key: string, base: string, type = 'controlnet', extra: Partial = {}): ModelConfig => ({ base, key, name: key, type, ...extra }) as ModelConfig; @@ -44,7 +49,7 @@ const depthB = model('sdxl-depth-b', 'sdxl', 'controlnet', { }); const normal = model('sdxl-normal', 'sdxl'); const pose = model('sdxl-openpose', 'sdxl'); -const sdxlUnion = model('sdxl-union-promax', 'sdxl'); +const sdxlUnion = model('xinsir-controlnet-union-sdxl-promax', 'sdxl'); const sd1Depth = model('sd15-depth', 'sd-1'); describe('resolveControlModel without a type', () => { @@ -55,6 +60,25 @@ describe('resolveControlModel without a type', () => { expect(resolution.candidates).toEqual([]); }); + it('lists every model it could run on the base, for a direct pick, and no candidates', () => { + const resolution = resolve(layer(undefined), 'sdxl', [ + depthA, + pose, + sdxlUnion, + sd1Depth, + model('sdxl-main', 'sdxl', 'main'), + model('my-sdxl-control', 'sdxl'), + ]); + + expect(resolution.usable.map((entry) => entry.key)).toEqual([ + depthA.key, + pose.key, + sdxlUnion.key, + 'my-sdxl-control', + ]); + expect(resolution.candidates).toEqual([]); + }); + it('reports a missing type when the layer names no model', () => { expect(resolve(layer(undefined), 'sdxl', [pose])).toMatchObject({ how: 'none', @@ -130,6 +154,16 @@ describe('resolveControlModel with a type', () => { expect(resolve(layer('depth'), 'sdxl', models, { 'sd-1:depth': depthB.key }).how).toBe('ask'); }); + it('prefers the remembered covering union over the only model made for the type', () => { + const models = [depthA, sdxlUnion]; + + expect(resolve(layer('depth'), 'sdxl', models)).toMatchObject({ how: 'only', model: depthA }); + expect(resolve(layer('depth'), 'sdxl', models, { 'sdxl:depth': sdxlUnion.key })).toMatchObject({ + how: 'remembered', + model: sdxlUnion, + }); + }); + it('matches the family the chosen detector produces', () => { const canny = model('sdxl-canny', 'sdxl'); const mlsd = model('sdxl-mlsd', 'sdxl'); @@ -149,6 +183,9 @@ describe('resolveControlModel with a type', () => { it('uses the one union model that covers the family', () => { const zUnion = model('z-image-fun-controlnet', 'z-image'); + expect(resolve(layer('pose'), 'sdxl', [sdxlUnion])).toMatchObject({ how: 'union', model: sdxlUnion }); + expect(resolve(layer('color'), 'sdxl', [sdxlUnion]).unresolved).toBe('no_model_for_type'); + expect(resolve(layer('depth'), 'z-image', [zUnion])).toMatchObject({ how: 'union', kind: 'z_image_control', @@ -171,8 +208,10 @@ describe('resolveControlModel with a type', () => { kind: 'controlnet', unresolved: 'no_control_models', }); - // An SDXL union carries no coverage until the backend reports its control types. - expect(resolve(layer('pose'), 'sdxl', [sdxlUnion]).unresolved).toBe('no_model_for_type'); + // An SD1.5 union carries no coverage until the backend reports its control types. + expect(resolve(layer('pose'), 'sd-1', [model('controlnet-union-sd15', 'sd-1')]).unresolved).toBe( + 'no_model_for_type' + ); }); it('only counts models the base runs and validation would accept', () => { @@ -205,9 +244,8 @@ describe('resolveControlModel model lists and family match', () => { model('sdxl-main', 'sdxl', 'main'), ]); - expect(resolution.candidates).toEqual([depthA]); + expect(resolution.candidates).toEqual([depthA, sdxlUnion]); expect(resolution.usable).toEqual([depthA, pose, sdxlUnion]); - expect(resolve(layer(undefined, { model: pose.key }), 'sdxl', [pose]).usable).toEqual([]); }); it('says whether the model matches the detector family or only the type', () => { @@ -225,6 +263,21 @@ describe('resolveControlModel model lists and family match', () => { }); }); +describe('isRememberableControlPick', () => { + it('remembers only a candidate for the type and detector family', () => { + const unpurposed = model('my-sdxl-control', 'sdxl'); + const models = [depthA, normal, pose, sdxlUnion, unpurposed]; + const resolution = resolve(layer('depth', { detector: 'depth_anything_depth_estimation' }), 'sdxl', models); + + expect(isRememberableControlPick(resolution, depthA)).toBe(true); + expect(isRememberableControlPick(resolution, sdxlUnion)).toBe(true); + expect(isRememberableControlPick(resolution, normal)).toBe(false); + expect(isRememberableControlPick(resolution, pose)).toBe(false); + expect(isRememberableControlPick(resolution, unpurposed)).toBe(false); + expect(isRememberableControlPick(resolve(layer(undefined), 'sdxl', models), depthA)).toBe(false); + }); +}); + describe('resolveControlModel InstantX modes', () => { const fluxUnion = model('FLUX.1-dev-Controlnet-Union-Pro', 'flux'); @@ -259,12 +312,12 @@ describe('getTypeAvailability', () => { expect(byType(getTypeAvailability('sdxl', [depthA, depthB, pose, sdxlUnion, sd1Depth], supportsAll))).toEqual({ color: ['missing', 0], depth: ['ask', 2], - edges: ['missing', 0], - lineart: ['missing', 0], + edges: ['union', 1], + lineart: ['union', 1], pose: ['ready', 1], - sketch: ['missing', 0], - softedge: ['missing', 0], - tile: ['missing', 0], + sketch: ['union', 1], + softedge: ['union', 1], + tile: ['union', 1], }); expect( byType( diff --git a/invokeai/frontend/webv2/src/workbench/controlModelResolution.ts b/invokeai/frontend/webv2/src/workbench/controlModelResolution.ts index 7e14a58dfd6..5f2e317e47a 100644 --- a/invokeai/frontend/webv2/src/workbench/controlModelResolution.ts +++ b/invokeai/frontend/webv2/src/workbench/controlModelResolution.ts @@ -26,9 +26,10 @@ export type ControlModelUnresolvedReason = export interface ControlModelResolution { /** - * `explicit`: the layer's own pick. `remembered`: the last pick for this base and type. `only`: the one model made - * for the type. `union`: the one union model covering it. `ask`: several fit. `missing`: none fit. `none`: the - * layer has no type, so its stored adapter applies as it is. + * `explicit`: the layer's own pick. `remembered`: the last pick for this base and type, still among `candidates`; + * it applies to every layer of that base and type without an explicit pick. `only`: the one model made for the + * type. `union`: the one union model covering it. `ask`: several fit. `missing`: none fit. `none`: the layer has + * no type, so its stored adapter applies as it is. */ how: 'explicit' | 'remembered' | 'only' | 'union' | 'ask' | 'missing' | 'none'; model: ModelConfig | null; @@ -36,7 +37,7 @@ export interface ControlModelResolution { kind: ControlAdapterKind; /** Models that fit the type: those made for it first, then covering unions. */ candidates: readonly ModelConfig[]; - /** Every model the layer could run on this base, whatever it was made for; empty for untyped layers. */ + /** Every model the layer could run on this base, whatever it was made for, typed or not. */ usable: readonly ModelConfig[]; /** * Whether the model was made for (or, as a union, covers) the family the detector produces; false when it only @@ -65,6 +66,9 @@ const usableKind = (model: ModelConfig, base: string | null, supportsKind: Suppo return kind && supportsKind(kind) && getControlModelUnusableReason(model, kind) === null ? kind : null; }; +const usableModels = (base: string | null, models: readonly ModelConfig[], supportsKind: SupportsKind) => + models.filter((model) => usableKind(model, base, supportsKind) !== null); + const collectCandidates = ( type: ControlType, family: ControlFamily, @@ -72,7 +76,7 @@ const collectCandidates = ( models: readonly ModelConfig[], supportsKind: SupportsKind ): CandidateSets => { - const usable = models.filter((model) => usableKind(model, base, supportsKind) !== null); + const usable = usableModels(base, models, supportsKind); const classes = new Map(usable.map((model) => [model, classifyControlModel(model)])); const byFamily = usable.filter((model) => classes.get(model)!.family === family); return { @@ -88,7 +92,8 @@ const collectCandidates = ( /** * Which model drives a control layer: an explicit pick for this base, the remembered pick for (base, type), the * only model made for the type, a choice between several, the one covering union model, or none. A pick made for - * another base is ignored, not cleared, so switching back restores it. + * another base is ignored, not cleared, so switching back restores it. An untyped layer runs its stored adapter as + * it is; its `usable` still lists every model it could run on the base, for a direct pick. */ export const resolveControlModel = (input: { layer: Pick; @@ -110,7 +115,7 @@ export const resolveControlModel = (input: { kind: layer.adapter.kind, model: picked, unresolved: layer.adapter.model ? (picked ? null : 'missing_model') : 'missing_type', - usable: [], + usable: usableModels(base, models, supportsKind), }; } @@ -161,6 +166,14 @@ export const resolveControlModel = (input: { return settle('missing', null, usable.length === 0 ? 'no_control_models' : 'no_model_for_type'); }; +/** + * Whether picking `model` should become the remembered pick for the layer's base and type. Resolution honours a + * remembered pick only among `candidates`, and it then drives every unpicked layer of that base and type, so a model + * made for another family, or one nothing identifies, stays this layer's own pick. + */ +export const isRememberableControlPick = (resolution: ControlModelResolution, model: ModelConfig): boolean => + resolution.candidates.some((candidate) => candidate.key === model.key); + /** How a control type would resolve on a fresh layer, for the Type menu's hints. */ export interface ControlTypeAvailability { type: ControlType; diff --git a/invokeai/frontend/webv2/src/workbench/controlTypes.test.ts b/invokeai/frontend/webv2/src/workbench/controlTypes.test.ts index db3c78575d2..6e05904c272 100644 --- a/invokeai/frontend/webv2/src/workbench/controlTypes.test.ts +++ b/invokeai/frontend/webv2/src/workbench/controlTypes.test.ts @@ -129,7 +129,12 @@ describe('classifyControlModel', () => { }); it.each([ - ['an SDXL union, offered only as an explicit pick', model('controlnet-union-sdxl-promax', 'sdxl'), []], + [ + 'an SDXL union, covering the xinsir ProMax conditions', + model('xinsir-controlnet-union-sdxl-promax', 'sdxl'), + ['openpose', 'depth', 'canny', 'mlsd', 'lineart', 'lineart_anime', 'scribble', 'softedge', 'normal', 'tile'], + ], + ['an SD1.5 union, offered only as an explicit pick', model('controlnet-union-sd15', 'sd-1'), []], [ 'a FLUX union, covering the InstantX modes', model('FLUX.1-dev-Controlnet-Union-Pro', 'flux', 'controlnet', 'canny_image_processor'), @@ -141,4 +146,13 @@ describe('classifyControlModel', () => { ] as const)('classifies %s', (_label, input, covers) => { expect(classifyControlModel(input)).toEqual({ covers, family: null, purpose: null, union: true }); }); + + it('classifies a replaced model config afresh', () => { + const config = model('control_v11p_sd15_canny', 'sd-1'); + expect(classifyControlModel(config).family).toBe('canny'); + + // A renamed model arrives as a new config object; the class must follow the new name. + expect(classifyControlModel({ ...config, name: 'control_v11f1p_sd15_depth' }).family).toBe('depth'); + expect(classifyControlModel(config).family).toBe('canny'); + }); }); diff --git a/invokeai/frontend/webv2/src/workbench/controlTypes.ts b/invokeai/frontend/webv2/src/workbench/controlTypes.ts index 4693e3f4d11..a81e54756bd 100644 --- a/invokeai/frontend/webv2/src/workbench/controlTypes.ts +++ b/invokeai/frontend/webv2/src/workbench/controlTypes.ts @@ -225,6 +225,23 @@ const FUN_UNION_BASES: ReadonlySet = new Set(['z-image', 'qwen-image', ' const FUN_UNION_COVERS: readonly ControlFamily[] = ['canny', 'mlsd', 'softedge', 'depth', 'openpose']; /** InstantX FLUX.1 Union modes the control types map onto. */ const FLUX_UNION_COVERS: readonly ControlFamily[] = ['canny', 'tile', 'depth', 'openpose']; +/** The xinsir SDXL Union (ProMax) conditions; upscale also takes an SDXL union as its tile ControlNet. */ +const SDXL_UNION_COVERS: readonly ControlFamily[] = [ + 'openpose', + 'depth', + 'canny', + 'mlsd', + 'lineart', + 'lineart_anime', + 'scribble', + 'softedge', + 'normal', + 'tile', +]; +const UNION_COVERS_BY_BASE: Readonly> = { + flux: FLUX_UNION_COVERS, + sdxl: SDXL_UNION_COVERS, +}; /** Lowercase name words, split at punctuation and camelCase boundaries. */ const nameTokens = (name: string): string[] => @@ -275,21 +292,37 @@ const preprocessorFamily = (model: Pick): Contr return preprocessor && Object.hasOwn(PREPROCESSOR_FAMILY, preprocessor) ? PREPROCESSOR_FAMILY[preprocessor]! : null; }; -/** - * Classify a control model: union models first (a base's Fun union, a FLUX union, any other `union` name), then - * strong name words, the saved preprocessor, and the remaining name words. - */ -export const classifyControlModel = ( - model: Pick -): ControlModelClass => { +type ClassifiableModel = Pick; + +const classify = (model: ClassifiableModel): ControlModelClass => { const tokens = nameTokens(model.name); if (model.type === 'controlnet' && FUN_UNION_BASES.has(model.base)) { return { covers: FUN_UNION_COVERS, family: null, purpose: null, union: true }; } if (hasToken(tokens, 'union')) { // Other bases' union models need the backend to expose their control types; until then they are explicit picks. - return { covers: model.base === 'flux' ? FLUX_UNION_COVERS : [], family: null, purpose: null, union: true }; + return { covers: UNION_COVERS_BY_BASE[model.base] ?? [], family: null, purpose: null, union: true }; } const family = strongNameFamily(tokens) ?? preprocessorFamily(model) ?? weakNameFamily(tokens); return { covers: [], family, purpose: family ? FAMILY_PURPOSE[family] : null, union: false }; }; + +/** + * Resolution, row renders and readiness ticks classify the same model configs over and over. Configs are replaced, + * never mutated, so a config object keys its class for as long as anything holds it. + */ +const classCache = new WeakMap(); + +/** + * Classify a control model: union models first (a base's Fun union; a FLUX or SDXL union, covering its known + * conditions; any other `union` name, covering nothing), then strong name words, the saved preprocessor, and the + * remaining name words. + */ +export const classifyControlModel = (model: ClassifiableModel): ControlModelClass => { + let modelClass = classCache.get(model); + if (!modelClass) { + modelClass = classify(model); + classCache.set(model, modelClass); + } + return modelClass; +}; From c2bfdbba6ee412f3a0e3266c0e2e6be209e5c590 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sun, 25 Oct 2026 00:46:30 -0600 Subject: [PATCH 27/35] feat(canvas): control panel states and image file drops An image file dropped on the canvas fills the selected empty control layer through the shared upload path with busy and refusal states; legacy Unbalanced stays visible; Priority says why it is unavailable, including for FLUX ControlNets; queued detection is worded as waiting; an empty typed layer's Detection waits for an image; remembered picks read differently from explicit ones. --- .../frontend/webv2/public/locales/en.json | 26 ++- .../scripts/run-accessibility-journeys.mjs | 21 +- .../src/platform/ui/Select.browser.test.tsx | 24 ++ .../src/workbench/canvas-operations/api.ts | 11 + .../controlSourceActions.test.ts | 14 +- .../canvas-operations/controlSourceActions.ts | 220 +++++++++++++++++- .../canvas/CanvasWidgetView.browser.test.tsx | 150 +++++++++++- .../widgets/canvas/CanvasWidgetView.test.tsx | 15 +- .../widgets/canvas/CanvasWidgetView.tsx | 65 +++++- .../canvas/ControlLayerCanvasOverlay.tsx | 47 +++- .../tool-presentation/PropertyPrimitives.tsx | 34 ++- .../widgets/layers/ControlAdapterRows.tsx | 67 ++++-- .../ControlLayerSettings.browser.test.tsx | 214 ++++++++++++++++- .../widgets/layers/ControlLayerSettings.tsx | 53 ++--- .../ControlLayerStatusIcon.browser.test.tsx | 21 ++ .../widgets/layers/ControlLayerStatusIcon.tsx | 23 +- .../widgets/layers/ControlModelRow.tsx | 55 +++-- .../widgets/layers/controlPanelModel.test.ts | 55 ++++- .../widgets/layers/controlPanelModel.ts | 43 ++-- .../widgets/layers/useControlSourceActions.ts | 111 ++++----- 20 files changed, 1064 insertions(+), 205 deletions(-) diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index a1a2f575782..90314723d15 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -3642,7 +3642,13 @@ }, "controlHint": { "title": "Drop an image here", - "body": "or pick one in Properties" + "body": "or pick one in Properties", + "uploading": "Uploading…" + }, + "fileDrop": { + "oneFile": "Drop one image at a time.", + "notImage": "Only image files can be dropped on the canvas.", + "noTarget": "Select an empty control layer to drop an image on it." }, "controlView": { "label": "Control layer view", @@ -4082,7 +4088,7 @@ "capabilities_unavailable": "waiting for model capabilities to load.", "choose_model": "Several installed models fit this control type; choose one.", "control_lora_limit": "Only one Control LoRA can be used at a time.", - "detection_pending": "Detection is out of date; run it again from the layer's Properties.", + "detection_pending": "Detection is running or out of date; wait for it, or run it again from the layer's Properties.", "duplicate_lllite_model": "Another control layer already uses this ControlNet-LLLite model; each can be applied once per generation.", "flux2_control_limit": "Only one FLUX.2 [dev] ControlNet can be used at a time.", "flux2_klein_control": "FLUX.2 [dev] ControlNet is not compatible with FLUX.2 Klein models.", @@ -4105,6 +4111,7 @@ }, "weight": "Weight", "detecting": "Running {{detector}}…", + "detectionQueued": "Waiting to run {{detector}}…", "detectionErrors": { "process-failed": "the detector could not run", "apply-failed": "the map could not be applied", @@ -4117,6 +4124,8 @@ }, "detectionFailed": "{{detector}} failed: {{reason}}", "detectionNeedsImage": "Add an image to detect from", + "detectionChosenOnImage": "Chosen when an image is added", + "detectionAddImageFirst": "Add an image first", "detectionNoInput": "Replace the image to detect again", "detectionNoSettings": "No settings for {{detector}}", "detectionNone": { @@ -4158,6 +4167,7 @@ "only": "{{kind}} · your only {{type}} model", "union": "{{kind}} · via Union, which covers {{type}}", "pick": "{{kind}} · your pick for {{type}}", + "remembered": "{{kind}} · remembered for {{type}}", "ambiguous_other": "{{count}} models fit: {{models}}", "noneForBase": "No installed model handles {{type}} for {{base}}.", "noModels": "No control models are installed for {{base}}.", @@ -4170,14 +4180,15 @@ "modelGroups": { "purpose": "Made for {{type}}", "union": "Union", - "unknown": "Purpose not set" + "unknown": "Purpose not set", + "installed": "Installed models" }, "modelItemCaption": "{{kind}} · {{origin}}", "modelOrigin": { "starter": "starter", "downloaded": "downloaded" }, - "modelRememberedHint": "Your pick is used for every {{type}} layer on {{base}} that has no model of its own.", + "modelRememberedHint": "Reset also forgets the {{type}} model remembered for {{base}}, in every project.", "modelValue": { "choose": "Choose a {{type}} model", "noneForBase": "None for {{base}}", @@ -4187,9 +4198,11 @@ "priorityModes": { "more_prompt": "Prompt", "balanced": "Balanced", - "more_control": "Control" + "more_control": "Control", + "unbalanced": "Unbalanced" }, "priorityUnavailable": "Priority applies to ControlNet models only.", + "priorityUnavailableFlux": "FLUX ControlNets have no priority setting.", "rows": { "image": "Image", "type": "Type", @@ -4211,7 +4224,8 @@ "frameEmpty": "Nothing is visible inside the frame", "frameNotReady": "The canvas is still preparing image data. Try again shortly.", "frameFailed": "Couldn't use the frame", - "uploadFailed": "Couldn't use that image" + "uploadFailed": "Couldn't use that image", + "uploadPending": "An image is still uploading to this layer." }, "stepsReadout": "{{begin}}–{{end}}%", "summary": { diff --git a/invokeai/frontend/webv2/scripts/run-accessibility-journeys.mjs b/invokeai/frontend/webv2/scripts/run-accessibility-journeys.mjs index 6aea04b15f6..55f8f54beb6 100644 --- a/invokeai/frontend/webv2/scripts/run-accessibility-journeys.mjs +++ b/invokeai/frontend/webv2/scripts/run-accessibility-journeys.mjs @@ -245,11 +245,30 @@ const surfaces = [ await page.reload({ waitUntil: 'domcontentloaded' }); await waitForWorkbench(page); await selectLayoutPreset(page, 'Edit', 'Canvas'); + // The representative main model is SD 1.x; the seeded control models are SDXL, so switch to an SDXL main + // (Fixture Model 006) through the Generate panel's model picker. + await page.getByText('Fixture Model 001', { exact: true }).click(); + const search = page.getByRole('combobox', { name: /^Search / }); + await search.fill('Fixture Model 006'); + await page.getByRole('option', { name: /Fixture Model 006(?!\d)/ }).click(); + // Once the picker has closed, the trigger is the only place naming the new model. + await search.waitFor({ state: 'hidden' }); + await page.getByText('Fixture Model 006', { exact: true }).waitFor(); await page.getByRole('button', { exact: true, name: 'Add layer' }).click(); await page.getByRole('menuitem', { exact: true, name: 'Control layer' }).click(); // The new layer's Properties panel and the canvas hint over its empty source are both scanned. - await page.getByRole('combobox', { exact: true, name: 'Type' }).waitFor(); + const type = page.getByRole('combobox', { exact: true, name: 'Type' }); + await type.waitFor(); await page.getByRole('note').filter({ hasText: 'Drop an image here' }).waitFor(); + await waitForSettledDocument(page); + await assertNoAxeViolations(page, 'workbench-canvas-control-layer-untyped'); + // Typed as Depth: the seed holds two depth models, so the panel asks for one; the runner scans this state. + await type.click(); + await page.getByRole('option', { name: /^Depth/ }).click(); + await page + .getByRole('combobox', { exact: true, name: 'Model' }) + .filter({ hasText: 'Choose a depth model' }) + .waitFor(); }, }, { diff --git a/invokeai/frontend/webv2/src/platform/ui/Select.browser.test.tsx b/invokeai/frontend/webv2/src/platform/ui/Select.browser.test.tsx index 0f44670d8c4..0d895204cea 100644 --- a/invokeai/frontend/webv2/src/platform/ui/Select.browser.test.tsx +++ b/invokeai/frontend/webv2/src/platform/ui/Select.browser.test.tsx @@ -123,4 +123,28 @@ describe('Select', () => { await act(() => userEvent.keyboard('{ArrowDown}{ArrowDown}{Enter}')); expect(onValueChange).toHaveBeenLastCalledWith(expect.objectContaining({ value: ['none'] })); }); + + it('opens with a header-less group without a divider above it', async () => { + // The header-less group comes first: a divider belongs between groups, never above the first option. + const items = [DETECTOR_ITEMS[2]!, DETECTOR_ITEMS[0]!, DETECTOR_ITEMS[1]!]; + await act(() => { + root.render( + +