diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index e1fd84a66..d13b2c758 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -8,6 +8,9 @@ import { Attribute } from 'vue-media-annotator/use/AttributeTypes'; import { CustomStyle } from 'vue-media-annotator/StyleManager'; import { AttributeTrackFilter } from 'vue-media-annotator/AttributeTrackFilterControls'; import { ImageEnhancements } from 'vue-media-annotator/use/useImageEnhancements'; +import { + CameraHomographies, CameraCorrespondences, CameraTransformTypes, RegistrationSource, +} from 'vue-media-annotator/CameraRegistrationStore'; import type { PercentileStretch } from 'vue-media-annotator/use/useImageEnhancements'; type DatasetType = 'image-sequence' | 'video' | 'multi' | 'large-image'; @@ -136,6 +139,12 @@ export interface MultiCamImportFolderArgs { sourceList: Record; // path/track file per camera @@ -186,9 +195,14 @@ interface DatasetMetaMutable { attributes?: Readonly>; attributeTrackFilters?: Readonly>; datasetInfo?: Record; + cameraHomographies?: CameraHomographies; + cameraCorrespondences?: CameraCorrespondences; + cameraTransformTypes?: CameraTransformTypes; + /** Producer provenance of the camera registration (see RegistrationSource). */ + cameraRegistrationSource?: RegistrationSource | null; error?: string; } -const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo']; +const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences', 'cameraTransformTypes', 'cameraRegistrationSource']; interface DatasetMeta extends DatasetMetaMutable { id: Readonly; @@ -278,7 +292,7 @@ interface Api { saveAttributeTrackFilters(datasetId: string, args: SaveAttributeTrackFilterArgs): Promise; // Non-Endpoint shared functions - openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'zip', directory?: boolean): + openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'zip' | 'transform', directory?: boolean): Promise<{canceled?: boolean; filePaths: string[]; fileList?: File[]; root?: string}>; /** Desktop: immediate child directory names under a parent folder (multicam subfolder import). */ listImmediateSubfolders?(parentPath: string): Promise; @@ -294,6 +308,12 @@ interface Api { ): Promise; /** Desktop: stereoscopic calibration file in a parent folder root. */ findParentFolderCalibrationFile?(parentPath: string): Promise; + /** + * Desktop: every DIVE camera-calibration .json (alignment transforms) in a + * parent folder root: per-camera *_registration.json files first, then + * other self-identified candidates. + */ + findParentFolderTransformFiles?(parentPath: string): Promise; /** True when the dataset folder has an attached stereoscopic calibration file. */ hasCalibrationFile?(datasetId: string): Promise; /** Web: stash a calibration File for multicam upload lookup. */ @@ -315,6 +335,16 @@ interface Api { saveCalibration?(path: string): Promise<{ savedPath: string; updatedDatasetIds: string[] }>; /** Desktop: set the stereo camera/calibration file for a single dataset. */ importCalibrationFile?(datasetId: string, path: string): Promise<{ calibration: string }>; + /** + * Merge a DIVE registration .json into an existing multicam dataset's + * saved camera registration. Web reads the provided File; desktop reads + * the path. options.camera keeps only the file's pairs naming that + * camera, replacing that camera's current pairs while other cameras' + * pairs are kept. + */ + importCameraRegistration?(datasetId: string, path: string, file?: File, + options?: { camera?: string }): + Promise<{ cameras: string[]; pairCount: number }>; /** Desktop: copy the dataset's current camera/calibration file out to destPath. */ exportCalibrationFile?(datasetId: string, destPath: string): Promise<{ exportedPath: string }>; /** Download/export the dataset's current calibration file (platform-specific). */ diff --git a/client/dive-common/components/ImportAnnotations.vue b/client/dive-common/components/ImportAnnotations.vue index ec67c4f9c..2694ef839 100644 --- a/client/dive-common/components/ImportAnnotations.vue +++ b/client/dive-common/components/ImportAnnotations.vue @@ -6,11 +6,14 @@ import { useApi } from 'dive-common/apispec'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import { clientSettings } from 'dive-common/store/settings'; import clearLengthAttributes from 'dive-common/utils/clearLengthAttributes'; +import warpAnnotationsAcrossCameras from 'dive-common/utils/warpAnnotationsAcrossCameras'; import { cloneDeep } from 'lodash'; import { useAnnotationSets, useAnnotationSet, useHandler, useCameraStore, useSelectedCamera, + useAlignedView, useCameraRegistration, } from 'vue-media-annotator/provides'; import { getResponseError } from 'vue-media-annotator/utils'; +import { parentDatasetId } from 'dive-common/compositeDatasetId'; export default defineComponent({ name: 'ImportAnnotations', @@ -52,7 +55,20 @@ export default defineComponent({ const { reloadAnnotations, save } = useHandler(); const cameraStore = useCameraStore(); const selectedCamera = useSelectedCamera(); + const alignedView = useAlignedView(); const isMulticamDataset = computed(() => cameraStore.camMap.value.size > 1); + // Warping detections onto other cameras requires the whole rig to be + // registered (a native->reference transform for every camera). + const canWarpToAllCameras = computed( + () => isMulticamDataset.value && alignedView.available.value, + ); + const warpToAllCameras = ref(false); + const warpToAllCamerasHint = computed(() => { + const progress = alignedView.registrationProgress.value; + return progress + ? `${progress.registered}/${progress.total} cameras registered` + : ''; + }); const activeCameraName = computed(() => { if (!isMulticamDataset.value) { return null; @@ -68,6 +84,35 @@ export default defineComponent({ const cameraFileSupported = computed( () => !!api.importCalibrationFile && props.subType === 'stereo', ); + // Camera registration import (per-camera transform files) is meaningful + // for any multicam dataset. + const cameraRegistration = useCameraRegistration(); + const registrationSupported = computed( + () => !!api.importCameraRegistration && isMulticamDataset.value, + ); + // One import button per non-reference camera pair, labeled in the + // direction of the mapping -- "Import ir → eo" registers ir onto the + // reference camera (the import dialog's Reference Camera choice, + // published by the viewer), matching the + // _to__registration.json file names -- and colored by + // whether that camera already has a registration (importing onto an + // existing one replaces it, after confirmation). + const registrationImportTargets = computed(() => { + if (!registrationSupported.value) { + return []; + } + const pairKeys = [ + ...Object.keys(cameraRegistration.homographies.value), + ...Object.keys(cameraRegistration.correspondences.value), + ]; + const cams = [...cameraStore.camMap.value.keys()]; + const reference = alignedView.reference.value ?? cams[0]; + return cams.filter((camera) => camera !== reference).map((camera) => ({ + camera, + label: `Import ${camera} → ${reference}`, + registered: pairKeys.some((key) => key.split('::').includes(camera)), + })); + }); const currentCalibrationName = computed(() => { if (!props.calibrationFile) return ''; return props.calibrationFile.replace(/^.*[\\/]/, ''); @@ -141,9 +186,24 @@ export default defineComponent({ } if (importFile) { - processing.value = false; await reloadAnnotations(); + if ( + warpToAllCameras.value + && canWarpToAllCameras.value + && activeCameraName.value + && alignedView.toReference.value + ) { + const warped = warpAnnotationsAcrossCameras( + cameraStore, + alignedView.toReference.value, + activeCameraName.value, + ); + if (warped.tracks > 0) { + await save(); + } + } } + processing.value = false; } } catch (error) { const text = [getResponseError(error)]; @@ -182,6 +242,61 @@ export default defineComponent({ }); } }; + const openRegistrationUpload = async (camera: string) => { + if (!api.importCameraRegistration) return; + const target = registrationImportTargets.value.find((entry) => entry.camera === camera); + if (target?.registered) { + const confirmed = await prompt({ + title: 'Replace Registration?', + text: `Camera "${camera}" already has a registration. Importing will replace it.`, + positiveButton: 'Replace', + negativeButton: 'Cancel', + confirm: true, + }); + if (!confirmed) return; + } + try { + const ret = await openFromDisk('transform'); + if (ret.canceled || !ret.filePaths.length) return; + menuOpen.value = false; + processing.value = true; + const result = await api.importCameraRegistration( + props.datasetId, + ret.filePaths[0], + ret.fileList?.[0], + { camera }, + ); + // Rehydrate the store from the freshly persisted meta so the Align + // View and mirroring pick up the new transforms immediately. + const meta = await api.loadMetadata(parentDatasetId(props.datasetId)); + cameraRegistration.hydrate( + meta.cameraHomographies, + meta.cameraCorrespondences, + meta.cameraTransformTypes, + meta.cameraRegistrationSource, + ); + processing.value = false; + const unknown = result.cameras.filter((name) => !cameraStore.camMap.value.has(name)); + if (unknown.length) { + await prompt({ + title: 'Registration Imported', + text: [ + `Imported ${result.pairCount} pair(s), but the file names camera(s) not in this dataset:`, + unknown.join(', '), + 'Pair bodies name their own cameras, so these pairs will not resolve until matching cameras exist.', + ], + positiveButton: 'OK', + }); + } + } catch (error) { + processing.value = false; + prompt({ + title: 'Registration Import Failed', + text: [getResponseError(error)], + positiveButton: 'OK', + }); + } + }; const applyLastCalibration = async () => { if (!api.importCalibrationFile || !lastCalibrationPath.value) return; try { @@ -211,10 +326,13 @@ export default defineComponent({ return { openUpload, openCalibrationUpload, + openRegistrationUpload, applyLastCalibration, showLastCalibrationSuggestion, lastCalibrationFileName, cameraFileSupported, + registrationSupported, + registrationImportTargets, currentCalibrationName, processing, menuOpen, @@ -225,6 +343,9 @@ export default defineComponent({ currentSet, isMulticamDataset, activeCameraName, + canWarpToAllCameras, + warpToAllCameras, + warpToAllCamerasHint, }; }, }); @@ -260,7 +381,7 @@ export default defineComponent({ - Import Annotation Data + Import Supplementary Data - Export Annotation Data + Export Supplementary Data + + + Zip all cameras: media, annotations, calibration, and dataset metadata diff --git a/client/src/AlignedViewStore.ts b/client/src/AlignedViewStore.ts new file mode 100644 index 000000000..0053e0ab2 --- /dev/null +++ b/client/src/AlignedViewStore.ts @@ -0,0 +1,130 @@ +import { + computed, ref, ComputedRef, Ref, +} from 'vue'; +import type { Matrix3, Point } from './homography'; +import { + cameraPairTransform, isIdentityMatrix3, mapPoint, +} from './alignedView'; + +/** + * Shared reactive state for the multicam "aligned view" toggle: when every + * non-reference camera has a stored/fitted transform into the reference + * camera's space, the user may warp each camera's display into that shared + * space during normal annotation review and link pan/zoom across all cameras + * (SEAL-TK features 2 + 3). + * + * Stored annotation geometry ALWAYS remains in native image space (decision + * D3); the transforms exposed here are applied at draw time only. + */ +export default class AlignedViewStore { + /** User toggle: whether the aligned view is requested. */ + enabled: Ref; + + /** + * Reference camera name that registration maps onto (the import dialog's + * Reference Camera choice, falling back to first in display order). Set + * whenever the dataset is multicam -- even before the rig resolves -- so + * UI can name the reference; null for single-camera datasets. + */ + reference: Ref; + + /** + * Per-camera native->reference matrices covering EVERY loaded camera, or + * null when at least one camera lacks a usable transform (see + * {@link resolveToReferenceTransforms}). + */ + toReference: Ref | null>; + + /** + * Externally suspended (e.g. while registration point picking is active, + * which records raw native-space clicks and manages its own aligned + * preview). Suspension un-warps the display without losing the toggle. + */ + suspended: Ref; + + /** + * How much of the rig resolves: cameras with a usable transform into the + * reference space, out of all loaded cameras. Maintained by the viewer + * alongside {@link setTransforms}; null for single-camera datasets. Lets + * UI outside the viewer core (e.g. the import menu) show the same + * "N/M cameras registered" status as the Align View toggle without + * re-deriving the pair graph. + */ + registrationProgress: Ref<{ registered: number; total: number } | null>; + + /** A usable transform exists for every camera, so the toggle may be shown. */ + available: ComputedRef; + + /** The aligned view is currently applied to rendering and navigation. */ + active: ComputedRef; + + constructor() { + this.enabled = ref(false); + this.reference = ref(null); + this.toReference = ref(null); + this.suspended = ref(false); + this.registrationProgress = ref(null); + this.available = computed(() => this.reference.value !== null + && this.toReference.value !== null + && Object.keys(this.toReference.value).length > 1); + this.active = computed(() => this.enabled.value + && !this.suspended.value + && this.available.value); + } + + /** Replace the resolved transform set (null when unavailable). */ + setTransforms(reference: string | null, toReference: Record | null) { + this.reference.value = reference; + this.toReference.value = toReference; + } + + setRegistrationProgress(progress: { registered: number; total: number } | null) { + this.registrationProgress.value = progress; + } + + setEnabled(enabled: boolean) { + this.enabled.value = enabled; + } + + setSuspended(suspended: boolean) { + this.suspended.value = suspended; + } + + /** + * Display transform for `camera` (native -> aligned/reference space), or + * null when the aligned view is inactive or the camera renders unwarped + * (the reference camera / identity). Callers treat null as "draw exactly + * as today", keeping the off state byte-identical to current behavior. + */ + cameraTransform(camera: string): Matrix3 | null { + if (!this.active.value || !this.toReference.value) { + return null; + } + const matrix = this.toReference.value[camera]; + if (!matrix || isIdentityMatrix3(matrix)) { + return null; + } + return matrix; + } + + /** + * Matrix mapping `from`-camera native pixels onto `to`-camera native + * pixels, composed through the reference space. Null when inactive or + * either camera is unresolved. + */ + cameraToCamera(from: string, to: string): Matrix3 | null { + if (!this.active.value || !this.toReference.value) { + return null; + } + return cameraPairTransform(this.toReference.value, from, to); + } + + /** Map a native-space point of `from` onto `to`'s native space (null when unavailable). */ + mapCameraPoint(from: string, to: string, point: Point): Point | null { + const matrix = this.cameraToCamera(from, to); + if (!matrix) { + return null; + } + return mapPoint(matrix, point); + } +} diff --git a/client/src/CameraRegistrationStore.spec.ts b/client/src/CameraRegistrationStore.spec.ts new file mode 100644 index 000000000..8b3e52834 --- /dev/null +++ b/client/src/CameraRegistrationStore.spec.ts @@ -0,0 +1,339 @@ +/// +import CameraRegistrationStore from './CameraRegistrationStore'; + +describe('CameraRegistrationStore', () => { + // Pure translation by (+5, -3): trivially invertible. + const translate = [[1, 0, 5], [0, 1, -3], [0, 0, 1]]; + // Four correspondence rows consistent with `translate`: right = left + (5, -3). + const translationPointRows = [ + [0, 0, 5, -3], + [10, 0, 15, -3], + [10, 10, 15, 7], + [0, 10, 5, 7], + ]; + + it('produces a directional pairKey that preserves left/right order', () => { + const store = new CameraRegistrationStore(); + expect(store.pairKey('rgb', 'ir')).toEqual('rgb::ir'); + expect(store.pairKey('rgb', 'ir')).not.toEqual(store.pairKey('ir', 'rgb')); + }); + + it('hydrates homographies and resets prior calibration state', () => { + const store = new CameraRegistrationStore(); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: [[1, 1, 6, -2]], transformType: 'translation', + }], + })); + const saved = { 'a::b': { AtoB: [[1, 0, 0], [0, 1, 0], [0, 0, 1]], BtoA: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] } }; + store.hydrate(saved); + expect(store.homographies.value).toEqual(saved); + expect(store.correspondences.value).toEqual({}); + expect(store.transformTypes.value).toEqual({}); + }); + + it('hydrates transform types alongside homographies', () => { + const store = new CameraRegistrationStore(); + store.hydrate({}, {}, { 'a::b': 'rigid' }); + expect(store.transformTypeForPair('a::b')).toBe('rigid'); + expect(store.transformTypeForPair('unset::pair')).toBe('similarity'); + }); + + it('hydrates correspondences and resumes id allocation', () => { + const store = new CameraRegistrationStore(); + const correspondences = { + 'rgb::ir': [ + { id: 1, a: [1, 2], b: [3, 4] }, + { id: 2, a: [5, 6], b: [7, 8] }, + ], + }; + store.hydrate({}, correspondences); + expect(store.correspondences.value).toEqual(correspondences); + // Points loaded afterwards pick up ids after the highest restored id. + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ left: 'rgb', right: 'ir', points: [[9, 9, 10, 10]] }], + })); + expect(store.correspondences.value['rgb::ir'][0].id).toBe(3); + }); + + describe('dirty / markSaved', () => { + it('tracks unsaved changes and resets on markSaved and hydrate', () => { + const store = new CameraRegistrationStore(); + expect(store.dirty.value).toBe(false); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: translate, + }], + })); + expect(store.dirty.value).toBe(true); + store.markSaved(); + expect(store.dirty.value).toBe(false); + store.hydrate({ 'a::b': { AtoB: translate, BtoA: translate } }); + // Freshly hydrated state is the saved baseline. + expect(store.dirty.value).toBe(false); + }); + }); + + describe('sourceIsMixed', () => { + it('flags only the mixed composite stamp the file merger produces', () => { + const store = new CameraRegistrationStore(); + expect(store.sourceIsMixed()).toBe(false); + store.hydrate(undefined, undefined, undefined, { producer: 'kamera', run: 'fl07' }); + expect(store.sourceIsMixed()).toBe(false); + store.hydrate(undefined, undefined, undefined, { + mixed: true, + files: { 'calibration_ir.json': { run: 'fl07' }, 'calibration_uv.json': { run: 'fl09' } }, + }); + expect(store.sourceIsMixed()).toBe(true); + }); + }); + + describe('loaded (file-sourced) homographies', () => { + /** Load a matrix-only (point-less) pair from calibration JSON, marking it 'loaded'. */ + function loadMatrixOnlyPair(store: CameraRegistrationStore, left: string, right: string, rightToLeft: number[][]) { + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left, right, points: [], leftToRight: null, rightToLeft, + }], + })); + } + + it('loads a matrix-only pair as B->A with its inverse as A->B and no points', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + loadMatrixOnlyPair(store, 'left', 'right', translate); + expect(store.correspondences.value[key]).toHaveLength(0); + expect(store.isLoadedHomography(key)).toBe(true); + const homog = store.homographies.value[key]; + expect(homog.BtoA).toEqual(translate); + expect(homog.AtoB[0][2]).toBeCloseTo(-5); + expect(homog.AtoB[1][2]).toBeCloseTo(3); + }); + + it('rejects a singular loaded matrix', () => { + const store = new CameraRegistrationStore(); + expect(() => loadMatrixOnlyPair(store, 'left', 'right', [[0, 0, 0], [0, 0, 0], [0, 0, 0]])) + .toThrow(/singular/); + }); + + it('hydrate marks an under-pointed homography as loaded', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + store.hydrate({ [key]: { AtoB: translate, BtoA: translate } }, {}, {}); + expect(store.isLoadedHomography(key)).toBe(true); + }); + }); + + describe('calibration JSON file round trip', () => { + it('serializes and reloads all pairs', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', + right: 'right', + points: translationPointRows, + leftToRight: translate, + rightToLeft: null, + transformType: 'translation', + }], + })); + const json = store.toRegistrationJson(); + + const restored = new CameraRegistrationStore(); + const result = restored.loadRegistrationText(json); + expect(result.pairCount).toBe(1); + expect(result.cameras.sort()).toEqual(['left', 'right']); + expect(restored.correspondences.value[key]).toHaveLength(4); + expect(restored.transformTypeForPair(key)).toBe('translation'); + expect(restored.homographies.value[key].AtoB[0][2]).toBeCloseTo(5); + // The missing direction was derived by inversion and round-tripped. + expect(restored.homographies.value[key].BtoA[0][2]).toBeCloseTo(-5); + // Enough points back the homography, so it is treated as fitted. + expect(restored.isLoadedHomography(key)).toBe(false); + }); + + it('includes pairs that only have a loaded homography (no points)', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('uv', 'ir'); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'uv', right: 'ir', points: [], leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], rightToLeft: null, + }], + })); + const restored = new CameraRegistrationStore(); + restored.loadRegistrationText(store.toRegistrationJson()); + expect(restored.homographies.value[key]).toBeDefined(); + expect(restored.isLoadedHomography(key)).toBe(true); + }); + + it('loads a desktop-persisted calibration.json (no "type" field, one direction only)', () => { + const store = new CameraRegistrationStore(); + const result = store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'eo', + right: 'ir', + points: [[0, 0, 5, -3]], + leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + rightToLeft: null, + transformType: 'translation', + }], + })); + expect(result.pairCount).toBe(1); + const key = store.pairKey('eo', 'ir'); + expect(store.correspondences.value[key]).toHaveLength(1); + // The missing direction is derived by inversion. + expect(store.homographies.value[key].BtoA[0][2]).toBeCloseTo(-5); + expect(store.transformTypeForPair(key)).toBe('translation'); + }); + + it('preserves the producer source stamp across a load/save round trip', () => { + const store = new CameraRegistrationStore(); + const source = { model: 'colmap-2026-07-01', swathe: 'fl07_C' }; + store.loadRegistrationText(JSON.stringify({ + version: 1, + source, + pairs: [{ + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: translate, + }], + })); + expect(store.source.value).toStrictEqual(source); + const saved = JSON.parse(store.toRegistrationJson()); + expect(saved.source).toStrictEqual(source); + }); + + it('omits the source key when no stamp was loaded', () => { + const store = new CameraRegistrationStore(); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, + }], + })); + expect('source' in JSON.parse(store.toRegistrationJson())).toBe(false); + }); + + it('clears a previous stamp when loading a file without one', () => { + const store = new CameraRegistrationStore(); + store.loadRegistrationText(JSON.stringify({ + version: 1, + source: { model: 'old' }, + pairs: [], + })); + store.loadRegistrationText(JSON.stringify({ version: 1, pairs: [] })); + expect(store.source.value).toBeNull(); + }); + + it('rejects a non-object source', () => { + const store = new CameraRegistrationStore(); + expect(() => store.loadRegistrationText(JSON.stringify({ + version: 1, source: 'colmap', pairs: [], + }))).toThrow(/"source" must be an object/); + }); + + it('hydrate restores the source stamp', () => { + const store = new CameraRegistrationStore(); + store.hydrate({}, {}, {}, { model: 'colmap-x' }); + expect(store.source.value).toStrictEqual({ model: 'colmap-x' }); + store.hydrate({}, {}, {}); + expect(store.source.value).toBeNull(); + }); + + it('flags a point-backed homography as refined when a source stamp is loaded', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + // Fresh from the producer (matrix-only): loaded, not refined. + store.loadRegistrationText(JSON.stringify({ + version: 1, + source: { model: 'colmap-x' }, + pairs: [{ + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: translate, + }], + })); + expect(store.isRefinedFromSource(key)).toBe(false); + // Point-backed under a stamp: the pair has diverged from the producer. + store.loadRegistrationText(JSON.stringify({ + version: 1, + source: { model: 'colmap-x' }, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, transformType: 'translation', + }], + })); + expect(store.isRefinedFromSource(key)).toBe(true); + }); + + it('does not flag point-backed homographies as refined when no source stamp is loaded', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, transformType: 'translation', + }], + })); + expect(store.isRefinedFromSource(key)).toBe(false); + }); + + it('keeps the refined flag across a save/load round trip', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + store.loadRegistrationText(JSON.stringify({ + version: 1, + source: { model: 'colmap-x' }, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, transformType: 'translation', + }], + })); + expect(store.isRefinedFromSource(key)).toBe(true); + + const restored = new CameraRegistrationStore(); + restored.loadRegistrationText(store.toRegistrationJson()); + // The refined pair saved with its backing points, so it re-marks as + // fitted (refined) rather than loaded. + expect(restored.isRefinedFromSource(key)).toBe(true); + }); + + it('rejects non-JSON, missing pairs, malformed pairs, and bad matrices without clobbering state', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, + }], + })); + expect(() => store.loadRegistrationText('not json')).toThrow(/valid JSON/); + expect(() => store.loadRegistrationText('{"type": "other"}')).toThrow(/pairs/); + expect(() => store.loadRegistrationText(JSON.stringify({ + version: 1, pairs: [{ left: 'a', right: 'a' }], + }))).toThrow(/distinct/); + expect(() => store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'a', + right: 'b', + points: [], + leftToRight: [[1, 0], [0, 1]], + rightToLeft: null, + }], + }))).toThrow(/3x3/); + expect(() => store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ left: 'a', right: 'b', points: [[1, 2, 3]] }], + }))).toThrow(/points row/); + expect(() => store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ left: 'a', right: 'b', transformType: 'bogus' }], + }))).toThrow(/transformType/); + // Failed loads left the existing calibration alone. + expect(store.correspondences.value[key]).toHaveLength(4); + }); + }); +}); diff --git a/client/src/CameraRegistrationStore.ts b/client/src/CameraRegistrationStore.ts new file mode 100644 index 000000000..e59aafea9 --- /dev/null +++ b/client/src/CameraRegistrationStore.ts @@ -0,0 +1,378 @@ +import { + ref, computed, Ref, ComputedRef, +} from 'vue'; +import { + invert3, Matrix3, Point, +} from './homography'; +import { + TransformType, TRANSFORM_TYPES, DEFAULT_TRANSFORM_TYPE, minPointsForTransform, +} from './transform'; + +/** + * A single picked point pair. `a` is the point in the left camera (camA), `b` + * the point in the right camera (camB). Left/right is the order the user chose, + * which is preserved (not alphabetized) so it survives round trips through the + * calibration JSON's ordered `[leftX, leftY, rightX, rightY]` rows. + */ +export interface Correspondence { + id: number; + a: Point; + b: Point; +} + +/** Both directions of the fitted alignment transform for one camera pair. */ +export interface PairHomography { + /** Maps left (camA) image coordinates onto right (camB). */ + AtoB: Matrix3; + /** Maps right (camB) image coordinates onto left (camA). */ + BtoA: Matrix3; +} + +/** Fitted transforms keyed by {@link CameraRegistrationStore.pairKey}. */ +export type CameraHomographies = Record; + +/** + * Where a pair's homography came from: fitted in-app from picked points, or + * loaded from a calibration file (which may carry no points at all). Loaded + * homographies persist through refit checks that would otherwise clear an + * under-pointed pair, until enough points are picked to fit a replacement. + */ +type HomographySource = 'fit' | 'loaded'; + +/** + * Free-form provenance stamped into the calibration file by whatever produced + * the transforms (e.g. an external COLMAP/KAMERA model step: model version, + * swathe/flight id, generation time). DIVE never interprets it -- it is + * preserved verbatim through load/refine/save round trips so an external + * re-solver can tell which model version a returning file was refined against. + */ +export type RegistrationSource = Record; + +/** + * One camera pair in the portable registration JSON file. This is the same + * self-describing shape the desktop platform persists as the project's + * standalone per-camera _to__registration.json files + * (see desktop backend/native/common.ts), so a panel-saved file, the + * on-disk artifacts, and an import-time seed are all interchangeable: + * correspondences flattened as [leftX, leftY, rightX, rightY] rows, plus + * both fitted directions (null when unfitted). + */ +export interface RegistrationFilePair { + left: string; + right: string; + points?: number[][]; + leftToRight?: Matrix3 | null; + rightToLeft?: Matrix3 | null; + transformType?: TransformType; +} + +/** Portable calibration file: everything needed to restore all pairs. */ +export interface RegistrationFile { + /** Written by panel saves for self-identification; optional on load. */ + type?: string; + version: number; + /** Producer provenance, preserved verbatim across round trips. */ + source?: RegistrationSource | null; + pairs: RegistrationFilePair[]; +} + +/** Identifying `type` value of the registration JSON format. */ +export const REGISTRATION_FILE_TYPE = 'dive-camera-registration'; + +/** Picked correspondences keyed by {@link CameraRegistrationStore.pairKey}. */ +export type CameraCorrespondences = Record; + +/** Chosen fit model per pair, keyed by {@link CameraRegistrationStore.pairKey}. Missing entries default to 'similarity'. */ +export type CameraTransformTypes = Record; + +/** + * Shared, reactive store for camera-calibration data (correspondences, + * fitted/loaded homographies, transform-type choices, and producer + * provenance). Lives in vue-media-annotator so both the annotation layers + * (client/src/layers) and the dive-common side can consume it via the + * provide/inject system. Handles persistence: hydrating saved state and + * loading/saving the portable calibration JSON format. + */ +export default class CameraRegistrationStore { + correspondences: Ref; + + homographies: Ref; + + transformTypes: Ref; + + /** + * Provenance of the loaded calibration (see {@link RegistrationSource}). + * Deliberately NOT cleared by in-app edits or refits -- refinements are + * exactly what should travel back to the producer stamped with the model + * lineage they were made against. Replaced (or cleared) only when a + * calibration file is loaded or the store is re-hydrated. + */ + source: Ref; + + /** True when the calibration has unsaved changes since the last save or load. */ + dirty: ComputedRef; + + private nextId: number; + + /** Provenance per homography key; missing entries behave like 'fit'. */ + private homographySources: Record; + + /** Serialized calibration at the last save/load, the baseline for {@link dirty}. */ + private savedSnapshot: Ref; + + constructor() { + this.correspondences = ref({}); + this.homographies = ref({}); + this.transformTypes = ref({}); + this.source = ref(null); + this.nextId = 1; + this.homographySources = {}; + this.savedSnapshot = ref(this.registrationSnapshot()); + this.dirty = computed(() => this.registrationSnapshot() !== this.savedSnapshot.value); + } + + /** Serialize the saved-to-dataset calibration state (points, transforms, provenance). */ + private registrationSnapshot(): string { + return JSON.stringify({ + homographies: this.homographies.value, + correspondences: this.correspondences.value, + transformTypes: this.transformTypes.value, + source: this.source.value, + }); + } + + /** Capture the current calibration as the saved baseline, so {@link dirty} reads false. */ + markSaved() { + this.savedSnapshot.value = this.registrationSnapshot(); + } + + /** + * True when the loaded calibration was assembled from per-camera files + * whose producer stamps disagree (the loader records that as a + * `{ mixed: true, files: {...} }` composite source) -- i.e. the rig may + * mix calibration generations and deserves a visible warning rather than + * silent composition. + */ + sourceIsMixed(): boolean { + return Boolean(this.source.value + && (this.source.value as Record).mixed === true); + } + + /** + * Directional key for a camera pair: `left::right`. Order is significant and + * preserved so left/right (e.g. RGB vs IR) survives for ordered exports. + */ + // eslint-disable-next-line class-methods-use-this + pairKey(camA: string, camB: string): string { + return `${camA}::${camB}`; + } + + /** + * True when `key`'s homography came from a calibration file rather than an + * in-app fit. Not independently reactive -- always read alongside + * {@link homographies} (provenance only changes when that map does). + */ + isLoadedHomography(key: string): boolean { + return this.homographySources[key] === 'loaded'; + } + + /** + * True when `key`'s transform was fitted from in-app picked points while a + * producer-stamped calibration is loaded -- i.e. the pair has diverged from + * what the stamped {@link source} shipped (producer files carry matrix-only + * pairs, so any point-backed fit is a human refinement). Derived rather + * than stored: it survives save/reload naturally, because point-backed + * pairs re-mark as fitted on hydrate. Read alongside {@link homographies}, + * same reactivity caveat as {@link isLoadedHomography}. + */ + isRefinedFromSource(key: string): boolean { + return this.source.value !== null + && this.homographies.value[key] !== undefined + && this.homographySources[key] === 'fit'; + } + + /** The chosen fit model for `key`, defaulting to {@link DEFAULT_TRANSFORM_TYPE} when unset. */ + transformTypeForPair(key: string): TransformType { + return this.transformTypes.value[key] || DEFAULT_TRANSFORM_TYPE; + } + + /** + * Serialize every pair with content (points and/or a homography) as the + * portable calibration JSON file (see {@link RegistrationFile}). Pairs whose + * only state is a transform-type choice are omitted. + */ + toRegistrationJson(): string { + const keys = new Set([ + ...Object.keys(this.correspondences.value).filter( + (key) => this.correspondences.value[key].length > 0, + ), + ...Object.keys(this.homographies.value), + ]); + const pairs: RegistrationFilePair[] = [...keys].sort().map((key) => { + const [left, right] = key.split('::'); + const homography = this.homographies.value[key] || null; + return { + left, + right, + points: (this.correspondences.value[key] || []).map((c) => [c.a[0], c.a[1], c.b[0], c.b[1]]), + leftToRight: homography ? homography.AtoB : null, + rightToLeft: homography ? homography.BtoA : null, + transformType: this.transformTypeForPair(key), + }; + }); + const file: RegistrationFile = { + type: REGISTRATION_FILE_TYPE, + version: 1, + ...(this.source.value ? { source: this.source.value } : {}), + pairs, + }; + return JSON.stringify(file, null, 2); + } + + /** + * Parse and load a calibration JSON file (the format written by + * {@link toRegistrationJson}), REPLACING all pairs' correspondences, + * homographies, and transform types. Throws a descriptive Error on + * malformed input without touching current state. Returns the camera names + * referenced by the file so callers can warn about ones missing from the + * loaded dataset. + */ + loadRegistrationText(text: string): { cameras: string[]; pairCount: number } { + let data: unknown; + try { + data = JSON.parse(text); + } catch { + throw new Error('File is not valid JSON'); + } + const file = data as Partial; + if (!Array.isArray(file?.pairs)) { + throw new Error('Not a DIVE camera registration file (expected a "pairs" list)'); + } + const source = CameraRegistrationStore.readSource(file.source); + const correspondences: CameraCorrespondences = {}; + const homographies: CameraHomographies = {}; + const transformTypes: CameraTransformTypes = {}; + const cameras = new Set(); + file.pairs.forEach((pair, i) => { + const context = `Pair ${i + 1}`; + if (typeof pair?.left !== 'string' || typeof pair?.right !== 'string' + || !pair.left || !pair.right || pair.left === pair.right) { + throw new Error(`${context}: "left" and "right" must be two distinct camera names`); + } + const key = this.pairKey(pair.left, pair.right); + cameras.add(pair.left); + cameras.add(pair.right); + if (pair.transformType !== undefined) { + if (!TRANSFORM_TYPES.some((t) => t.value === pair.transformType)) { + throw new Error( + `${context}: unknown transformType "${pair.transformType}" (expected one of ${TRANSFORM_TYPES.map((t) => t.value).join(', ')})`, + ); + } + transformTypes[key] = pair.transformType; + } + correspondences[key] = (pair.points || []).map((row, j) => { + const [ax, ay, bx, by] = CameraRegistrationStore.readPointsRow(row, `${context}, points row ${j + 1}`); + // eslint-disable-next-line no-plusplus + return { id: this.nextId++, a: [ax, ay] as Point, b: [bx, by] as Point }; + }); + const leftToRight = (pair.leftToRight === null || pair.leftToRight === undefined) + ? null + : CameraRegistrationStore.readMatrix(pair.leftToRight, `${context}, leftToRight`); + const rightToLeft = (pair.rightToLeft === null || pair.rightToLeft === undefined) + ? null + : CameraRegistrationStore.readMatrix(pair.rightToLeft, `${context}, rightToLeft`); + if (leftToRight || rightToLeft) { + // If only one direction is present, derive the other by inversion + // (readMatrix guarantees invertibility). + homographies[key] = { + AtoB: leftToRight ?? invert3(rightToLeft as Matrix3), + BtoA: rightToLeft ?? invert3(leftToRight as Matrix3), + }; + } + }); + this.correspondences.value = correspondences; + this.homographies.value = homographies; + this.transformTypes.value = transformTypes; + this.source.value = source; + this.markHomographySources(); + return { cameras: [...cameras], pairCount: file.pairs.length }; + } + + /** Validate an untrusted `source` value: a plain object, or absent (-> null). */ + private static readSource(raw: unknown): RegistrationSource | null { + if (raw === undefined || raw === null) { + return null; + } + if (typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error('"source" must be an object when present'); + } + return raw as RegistrationSource; + } + + /** Validate an untrusted value as a 4-element finite [leftX, leftY, rightX, rightY] row. */ + private static readPointsRow(raw: unknown, context: string): [number, number, number, number] { + if (!Array.isArray(raw) || raw.length !== 4) { + throw new Error(`${context}: expected [leftX, leftY, rightX, rightY]`); + } + const nums = raw.map(Number); + if (nums.some((n) => !Number.isFinite(n))) { + throw new Error(`${context}: expected [leftX, leftY, rightX, rightY] with finite numbers`); + } + return [nums[0], nums[1], nums[2], nums[3]]; + } + + /** Validate an untrusted value as an invertible row-major 3x3 matrix. */ + private static readMatrix(raw: unknown, context: string): Matrix3 { + if (!Array.isArray(raw) || raw.length !== 3 + || raw.some((row) => !Array.isArray(row) || row.length !== 3)) { + throw new Error(`${context}: expected a 3x3 matrix`); + } + const m = (raw as unknown[][]).map((row) => row.map(Number)); + if (m.some((row) => row.some((n) => !Number.isFinite(n)))) { + throw new Error(`${context}: matrix entries must be finite numbers`); + } + try { + invert3(m); + } catch { + throw new Error(`${context}: matrix is singular (not invertible)`); + } + return m; + } + + /** + * Reset homography provenance after bulk-loading state: a homography whose + * pair lacks enough points for its transform type can only have come from a + * file ('loaded', so refit checks preserve it); one with enough points is + * treated as fitted from them. + */ + private markHomographySources() { + this.homographySources = {}; + Object.keys(this.homographies.value).forEach((key) => { + const count = (this.correspondences.value[key] || []).length; + const required = minPointsForTransform(this.transformTypeForPair(key)); + this.homographySources[key] = count >= required ? 'fit' : 'loaded'; + }); + } + + /** Reset state and load saved homographies, correspondences, transform type choices, and provenance. */ + hydrate( + homographies?: CameraHomographies, + correspondences?: CameraCorrespondences, + transformTypes?: CameraTransformTypes, + source?: RegistrationSource | null, + ) { + this.homographies.value = homographies ? { ...homographies } : {}; + this.correspondences.value = correspondences ? { ...correspondences } : {}; + this.transformTypes.value = transformTypes ? { ...transformTypes } : {}; + this.source.value = source ?? null; + this.markHomographySources(); + // Resume id allocation past any restored correspondences. + let maxId = 0; + Object.values(this.correspondences.value).forEach((list) => { + list.forEach((c) => { maxId = Math.max(maxId, c.id); }); + }); + this.nextId = maxId + 1; + // The freshly loaded state is the saved baseline. + this.markSaved(); + } +} diff --git a/client/src/alignedView.spec.ts b/client/src/alignedView.spec.ts new file mode 100644 index 000000000..278b52f28 --- /dev/null +++ b/client/src/alignedView.spec.ts @@ -0,0 +1,301 @@ +/// +import { + IDENTITY3, + isIdentityMatrix3, + readTransformMatrix, + composeThroughPairs, + resolveToReferenceTransforms, + unresolvedCameras, + cameraPairTransform, + mapPoint, + mapBounds, + mapRotatedBounds, + mapGeoJSONFeatures, +} from './alignedView'; +import { applyHomography, Matrix3, Point } from './homography'; +import type { CameraHomographies } from './CameraRegistrationStore'; +import AlignedViewStore from './AlignedViewStore'; + +/** Simple affine helpers for readable fixtures. */ +function translation(tx: number, ty: number): Matrix3 { + return [[1, 0, tx], [0, 1, ty], [0, 0, 1]]; +} +function scale(s: number): Matrix3 { + return [[s, 0, 0], [0, s, 0], [0, 0, 1]]; +} + +function expectPointClose(actual: Point, expected: Point) { + expect(actual[0]).toBeCloseTo(expected[0], 6); + expect(actual[1]).toBeCloseTo(expected[1], 6); +} + +describe('isIdentityMatrix3', () => { + it('accepts the identity and rejects non-identity', () => { + expect(isIdentityMatrix3(IDENTITY3)).toBe(true); + expect(isIdentityMatrix3(translation(0, 0))).toBe(true); + expect(isIdentityMatrix3(translation(1, 0))).toBe(false); + expect(isIdentityMatrix3(scale(2))).toBe(false); + }); +}); + +describe('readTransformMatrix', () => { + it('accepts a valid row-major 3x3', () => { + const m = readTransformMatrix([[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + expect(m).toEqual([[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); + it('rejects malformed shapes', () => { + expect(readTransformMatrix(undefined)).toBeNull(); + expect(readTransformMatrix(null)).toBeNull(); + expect(readTransformMatrix('matrix')).toBeNull(); + expect(readTransformMatrix([[1, 0], [0, 1]])).toBeNull(); + expect(readTransformMatrix([[1, 0, 0], [0, 1, 0]])).toBeNull(); + expect(readTransformMatrix([[1, 0, 0], [0, 1, 0], [0, 0]])).toBeNull(); + }); + it('rejects non-finite values', () => { + expect(readTransformMatrix([[1, 0, NaN], [0, 1, 0], [0, 0, 1]])).toBeNull(); + expect(readTransformMatrix([[1, 0, Infinity], [0, 1, 0], [0, 0, 1]])).toBeNull(); + expect(readTransformMatrix([[1, 0, '5'], [0, 1, 'abc'], [0, 0, 1]])).toBeNull(); + }); + it('rejects singular matrices', () => { + expect(readTransformMatrix([[1, 1, 0], [1, 1, 0], [0, 0, 1]])).toBeNull(); + expect(readTransformMatrix([[0, 0, 0], [0, 0, 0], [0, 0, 0]])).toBeNull(); + }); +}); + +describe('composeThroughPairs', () => { + const irToEo = translation(100, 50); + const uvToIr = scale(2); + const homographies: CameraHomographies = { + // eo::ir stored with AtoB = eo->ir, so BtoA = ir->eo. + 'eo::ir': { AtoB: [[1, 0, -100], [0, 1, -50], [0, 0, 1]], BtoA: irToEo }, + // uv::ir stored with AtoB = uv->ir. + 'uv::ir': { AtoB: uvToIr, BtoA: [[0.5, 0, 0], [0, 0.5, 0], [0, 0, 1]] }, + }; + it('returns identity for the reference itself', () => { + expect(composeThroughPairs('eo', 'eo', homographies)).toEqual(IDENTITY3); + }); + it('uses a direct pair edge in either direction', () => { + const m = composeThroughPairs('ir', 'eo', homographies); + expect(m).not.toBeNull(); + expectPointClose(applyHomography(m as Matrix3, [10, 20]), [110, 70]); + }); + it('composes multi-hop paths (uv -> ir -> eo)', () => { + const m = composeThroughPairs('uv', 'eo', homographies); + expect(m).not.toBeNull(); + // uv (x, y) -> ir (2x, 2y) -> eo (2x + 100, 2y + 50). + expectPointClose(applyHomography(m as Matrix3, [10, 20]), [120, 90]); + }); + it('returns null when no path exists', () => { + expect(composeThroughPairs('flir', 'eo', homographies)).toBeNull(); + expect(composeThroughPairs('ir', 'eo', {})).toBeNull(); + }); +}); + +describe('resolveToReferenceTransforms', () => { + const cameras = ['eo', 'ir', 'uv']; + const homographies: CameraHomographies = { + 'eo::ir': { AtoB: translation(-100, 0), BtoA: translation(100, 0) }, + 'uv::ir': { AtoB: scale(2), BtoA: scale(0.5) }, + }; + it('resolves every camera through calibration pairs', () => { + const result = resolveToReferenceTransforms(cameras, 'eo', homographies); + expect(result).not.toBeNull(); + const transforms = result as Record; + expect(transforms.eo).toEqual(IDENTITY3); + expectPointClose(applyHomography(transforms.ir, [5, 5]), [105, 5]); + expectPointClose(applyHomography(transforms.uv, [5, 5]), [110, 10]); + }); + it('is all-or-none: any unresolved camera fails the whole set', () => { + expect(resolveToReferenceTransforms(['eo', 'ir', 'flir'], 'eo', homographies)).toBeNull(); + expect(resolveToReferenceTransforms(cameras, 'eo', {})).toBeNull(); + }); + it('fails when the reference is not among the cameras or the list is empty', () => { + expect(resolveToReferenceTransforms([], 'eo', homographies)).toBeNull(); + expect(resolveToReferenceTransforms(['ir', 'uv'], 'eo', homographies)).toBeNull(); + }); +}); + +describe('unresolvedCameras', () => { + const homographies: CameraHomographies = { + 'eo::ir': { AtoB: translation(-100, 0), BtoA: translation(100, 0) }, + }; + it('names cameras with no path to the reference', () => { + expect(unresolvedCameras(['eo', 'ir', 'uv'], 'eo', homographies)).toEqual(['uv']); + expect(unresolvedCameras(['eo', 'ir', 'uv'], 'eo', {})).toEqual(['ir', 'uv']); + }); + it('is empty when every camera resolves', () => { + expect(unresolvedCameras(['eo', 'ir'], 'eo', homographies)).toEqual([]); + }); +}); + +describe('cameraPairTransform', () => { + it('composes from -> reference -> to', () => { + const toReference = { + eo: IDENTITY3, + ir: translation(100, 0), + uv: scale(2), + }; + // ir (x, y) -> ref (x + 100, y) -> uv ((x + 100) / 2, y / 2). + const m = cameraPairTransform(toReference, 'ir', 'uv'); + expect(m).not.toBeNull(); + expectPointClose(applyHomography(m as Matrix3, [10, 20]), [55, 10]); + }); + it('returns null for unresolved cameras', () => { + expect(cameraPairTransform({ eo: IDENTITY3 }, 'eo', 'ir')).toBeNull(); + expect(cameraPairTransform({ eo: IDENTITY3 }, 'ir', 'eo')).toBeNull(); + }); +}); + +describe('mapPoint', () => { + it('is identity for a null matrix', () => { + expect(mapPoint(null, [3, 4])).toEqual([3, 4]); + }); + it('applies the matrix otherwise', () => { + expectPointClose(mapPoint(translation(1, 2), [3, 4]), [4, 6]); + }); +}); + +describe('mapBounds', () => { + it('translates and scales an axis-aligned box', () => { + expect(mapBounds(translation(10, -5), [0, 0, 4, 6])).toEqual([10, -5, 14, 1]); + expect(mapBounds(scale(2), [1, 2, 3, 4])).toEqual([2, 4, 6, 8]); + }); + it('returns the AABB of the mapped corners under rotation', () => { + // 90-degree rotation about the origin: (x, y) -> (-y, x). + const rot90: Matrix3 = [[0, -1, 0], [1, 0, 0], [0, 0, 1]]; + const [x1, y1, x2, y2] = mapBounds(rot90, [0, 0, 4, 2]); + expect(x1).toBeCloseTo(-2, 6); + expect(y1).toBeCloseTo(0, 6); + expect(x2).toBeCloseTo(0, 6); + expect(y2).toBeCloseTo(4, 6); + }); +}); + +describe('mapRotatedBounds', () => { + const rot = (angle: number): Matrix3 => [ + [Math.cos(angle), -Math.sin(angle), 0], + [Math.sin(angle), Math.cos(angle), 0], + [0, 0, 1], + ]; + it('adds the transform rotation and keeps the rect size under similarity', () => { + const theta = Math.PI / 6; + const result = mapRotatedBounds(rot(theta), [10, 20, 50, 40], Math.PI / 8); + expect(result.rotation).toBeCloseTo(Math.PI / 8 + theta, 6); + // The rect's own dimensions (40 x 20) are preserved by a pure rotation. + expect(result.bounds[2] - result.bounds[0]).toBeCloseTo(40, 6); + expect(result.bounds[3] - result.bounds[1]).toBeCloseTo(20, 6); + }); + it('is a passthrough for the identity', () => { + const result = mapRotatedBounds(IDENTITY3, [10, 20, 50, 40], 0.3); + expect(result.rotation).toBeCloseTo(0.3, 6); + expect(result.bounds[0]).toBeCloseTo(10, 6); + expect(result.bounds[1]).toBeCloseTo(20, 6); + expect(result.bounds[2]).toBeCloseTo(50, 6); + expect(result.bounds[3]).toBeCloseTo(40, 6); + }); + it('scales the rect and centers under translation + scale', () => { + const matrix = [[2, 0, 100], [0, 2, -10], [0, 0, 1]]; + const result = mapRotatedBounds(matrix, [0, 0, 10, 20], 0.5); + expect(result.rotation).toBeCloseTo(0.5, 6); + expect(result.bounds[2] - result.bounds[0]).toBeCloseTo(20, 6); + expect(result.bounds[3] - result.bounds[1]).toBeCloseTo(40, 6); + // The center (5, 10) maps to (110, 10). + expect((result.bounds[0] + result.bounds[2]) / 2).toBeCloseTo(110, 6); + expect((result.bounds[1] + result.bounds[3]) / 2).toBeCloseTo(10, 6); + }); +}); + +describe('mapGeoJSONFeatures', () => { + it('maps points, lines, and polygon rings without mutating the source', () => { + const source: GeoJSON.Feature[] = [ + { + type: 'Feature', + geometry: { type: 'Point', coordinates: [1, 2] }, + properties: { key: 'head' }, + }, + { + type: 'Feature', + geometry: { type: 'LineString', coordinates: [[0, 0], [3, 4]] }, + properties: { key: 'HeadTails' }, + }, + { + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [[[0, 0], [4, 0], [4, 4], [0, 0]], [[1, 1], [2, 1], [2, 2], [1, 1]]], + }, + properties: { key: '' }, + }, + ]; + const mapped = mapGeoJSONFeatures(translation(10, 20), source); + expect(mapped[0].geometry).toEqual({ type: 'Point', coordinates: [11, 22] }); + expect(mapped[1].geometry).toEqual({ type: 'LineString', coordinates: [[10, 20], [13, 24]] }); + expect(mapped[2].geometry.type).toBe('Polygon'); + expect((mapped[2].geometry as GeoJSON.Polygon).coordinates[1][2]).toEqual([12, 22]); + expect(mapped[0].properties).toEqual({ key: 'head' }); + // Deep copy: the source geometry/properties are untouched and unshared. + expect(source[0].geometry).toEqual({ type: 'Point', coordinates: [1, 2] }); + expect(mapped[0].properties).not.toBe(source[0].properties); + expect(mapped[2].geometry).not.toBe(source[2].geometry); + }); +}); + +describe('AlignedViewStore', () => { + function makeResolvedStore() { + const store = new AlignedViewStore(); + store.setTransforms('eo', { + eo: IDENTITY3, + ir: translation(100, 0), + }); + return store; + } + + it('is unavailable and inactive by default', () => { + const store = new AlignedViewStore(); + expect(store.available.value).toBe(false); + store.setEnabled(true); + expect(store.active.value).toBe(false); + expect(store.cameraTransform('ir')).toBeNull(); + expect(store.cameraToCamera('eo', 'ir')).toBeNull(); + }); + + it('activates only when enabled, available, and not suspended', () => { + const store = makeResolvedStore(); + expect(store.available.value).toBe(true); + expect(store.active.value).toBe(false); + store.setEnabled(true); + expect(store.active.value).toBe(true); + store.setSuspended(true); + expect(store.active.value).toBe(false); + expect(store.cameraTransform('ir')).toBeNull(); + store.setSuspended(false); + expect(store.active.value).toBe(true); + }); + + it('exposes a display transform for warped cameras and null for the reference', () => { + const store = makeResolvedStore(); + store.setEnabled(true); + expect(store.cameraTransform('eo')).toBeNull(); + const m = store.cameraTransform('ir'); + expect(m).not.toBeNull(); + expectPointClose(applyHomography(m as Matrix3, [1, 1]), [101, 1]); + // Unknown camera also renders unwarped. + expect(store.cameraTransform('uv')).toBeNull(); + }); + + it('maps cross-camera points through the reference', () => { + const store = makeResolvedStore(); + store.setEnabled(true); + expectPointClose(store.mapCameraPoint('ir', 'eo', [0, 0]) as Point, [100, 0]); + expectPointClose(store.mapCameraPoint('eo', 'ir', [100, 0]) as Point, [0, 0]); + expect(store.mapCameraPoint('eo', 'flir', [0, 0])).toBeNull(); + }); + + it('becomes unavailable when a camera set collapses to one entry', () => { + const store = new AlignedViewStore(); + store.setTransforms('eo', { eo: IDENTITY3 }); + store.setEnabled(true); + expect(store.available.value).toBe(false); + expect(store.active.value).toBe(false); + }); +}); diff --git a/client/src/alignedView.ts b/client/src/alignedView.ts new file mode 100644 index 000000000..a176bf242 --- /dev/null +++ b/client/src/alignedView.ts @@ -0,0 +1,305 @@ +/** + * Pure helpers for the multicam "aligned view" (SEAL-TK features 2 + 3): + * resolving a per-camera native->reference-space transform for every loaded + * camera from the available sources, and composing camera-to-camera mappings + * through the reference camera. + * + * Conventions (documented assumptions, see migration plan Q3): + * - The REFERENCE camera is the first camera in display order + * (`meta.multiCamMedia.cameraOrder[0]`, i.e. `multiCamList[0]`). Its + * transform is the identity. + * - Calibration-tool homographies are stored per directional pair key + * `camA::camB` with `AtoB` mapping camA pixels onto camB (and `BtoA` the + * inverse); a camera's path to the reference is found by composing pair + * edges (breadth-first, so up-to-3-camera rigs may chain e.g. UV->IR->EO). + * - The calibration store is the SINGLE source the viewer resolves from: + * whatever the calibration panel shows/saves is what the Align button + * applies. External calibration .json files enter that same store, either + * through the panel's Load calibration button or by seeding the saved + * calibration at multicam import time. + */ +import { + Matrix3, matMul3, invert3, applyHomography, Point, +} from './homography'; +import type { CameraHomographies } from './CameraRegistrationStore'; +import type { RectBounds } from './utils'; + +export const IDENTITY3: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + +/** True when `m` is (numerically) the identity transform. */ +export function isIdentityMatrix3(m: Matrix3, eps = 1e-9): boolean { + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + if (Math.abs(m[i][j] - IDENTITY3[i][j]) > eps) { + return false; + } + } + } + return true; +} + +/** + * Defensively validate an untrusted value (e.g. read from dataset meta JSON) + * as a row-major 3x3 matrix usable as a display transform. Returns null for + * anything malformed, non-finite, or singular (non-invertible) rather than + * throwing, so callers can treat "bad matrix" as "no matrix". + */ +export function readTransformMatrix(raw: unknown): Matrix3 | null { + if (!Array.isArray(raw) || raw.length !== 3) { + return null; + } + const m: number[][] = []; + for (let i = 0; i < 3; i += 1) { + const row = raw[i]; + if (!Array.isArray(row) || row.length !== 3) { + return null; + } + const nums = row.map(Number); + if (nums.some((v) => !Number.isFinite(v))) { + return null; + } + m.push(nums); + } + try { + invert3(m); + } catch { + return null; + } + return m; +} + +/** One directed edge of the calibration-pair graph: `matrix` maps `from` pixels onto `to`. */ +interface PairEdge { + to: string; + matrix: Matrix3; +} + +/** Build the bidirectional camera adjacency from calibration pair homographies. */ +function buildPairGraph(homographies: CameraHomographies): Record { + const graph: Record = {}; + const addEdge = (from: string, to: string, matrix: Matrix3) => { + if (!graph[from]) { + graph[from] = []; + } + graph[from].push({ to, matrix }); + }; + Object.entries(homographies).forEach(([key, pair]) => { + const [camA, camB] = key.split('::'); + if (!camA || !camB || camA === camB || !pair) { + return; + } + addEdge(camA, camB, pair.AtoB); + addEdge(camB, camA, pair.BtoA); + }); + return graph; +} + +/** + * Compose a `camera` -> `reference` matrix by walking the calibration pair + * graph breadth-first (shortest hop count). Returns null when no path exists. + */ +export function composeThroughPairs( + camera: string, + reference: string, + homographies: CameraHomographies, +): Matrix3 | null { + if (camera === reference) { + return IDENTITY3; + } + const graph = buildPairGraph(homographies); + // BFS queue of (camera, accumulated camera->node matrix). + const queue: { node: string; matrix: Matrix3 }[] = [{ node: camera, matrix: IDENTITY3 }]; + const visited = new Set([camera]); + while (queue.length) { + const { node, matrix } = queue.shift() as { node: string; matrix: Matrix3 }; + const edges = graph[node] || []; + for (let i = 0; i < edges.length; i += 1) { + const edge = edges[i]; + if (!visited.has(edge.to)) { + // p_to = edge.matrix * p_node and p_node = matrix * p_camera. + const composed = matMul3(edge.matrix, matrix); + if (edge.to === reference) { + return composed; + } + visited.add(edge.to); + queue.push({ node: edge.to, matrix: composed }); + } + } + } + return null; +} + +/** + * Resolve a native->reference matrix for EVERY camera in `cameras` from the + * calibration store's pair homographies, or null when any camera lacks a + * usable transform (the aligned view is all-or-none: a partially-aligned + * display would be misleading). The reference camera always maps by the + * identity. + */ +export function resolveToReferenceTransforms( + cameras: string[], + reference: string, + homographies: CameraHomographies, +): Record | null { + if (!cameras.length || !cameras.includes(reference)) { + return null; + } + const result: Record = {}; + for (let i = 0; i < cameras.length; i += 1) { + const camera = cameras[i]; + if (camera === reference) { + result[camera] = IDENTITY3; + } else { + const matrix = composeThroughPairs(camera, reference, homographies); + if (!matrix) { + return null; + } + result[camera] = matrix; + } + } + return result; +} + +/** + * The cameras in `cameras` with no composed transform path to `reference` -- + * the reason {@link resolveToReferenceTransforms} returned null, named so the + * UI can say exactly which pair(s) still need calibrating instead of silently + * hiding the aligned view. + */ +export function unresolvedCameras( + cameras: string[], + reference: string, + homographies: CameraHomographies, +): string[] { + return cameras.filter( + (camera) => camera !== reference + && composeThroughPairs(camera, reference, homographies) === null, + ); +} + +/** + * Matrix mapping `from`-camera native pixels onto `to`-camera native pixels, + * composed through the shared reference space: + * p_to = inv(T_to->ref) * T_from->ref * p_from. + * Returns null when either camera has no resolved transform. + */ +export function cameraPairTransform( + toReference: Record, + from: string, + to: string, +): Matrix3 | null { + const fromRef = toReference[from]; + const toRef = toReference[to]; + if (!fromRef || !toRef) { + return null; + } + try { + return matMul3(invert3(toRef), fromRef); + } catch { + return null; + } +} + +/** Map a point through a nullable transform (identity when null). */ +export function mapPoint(matrix: Matrix3 | null, point: Point): Point { + if (!matrix) { + return point; + } + return applyHomography(matrix, point); +} + +/** Axis-aligned bounding box of a set of points. */ +function pointsToBounds(points: Point[]): RectBounds { + const xs = points.map((p) => p[0]); + const ys = points.map((p) => p[1]); + return [Math.min(...xs), Math.min(...ys), Math.max(...xs), Math.max(...ys)]; +} + +/** The four corners of a RectBounds, clockwise from (x1, y1). */ +function boundsCorners(bounds: RectBounds): Point[] { + return [ + [bounds[0], bounds[1]], + [bounds[2], bounds[1]], + [bounds[2], bounds[3]], + [bounds[0], bounds[3]], + ]; +} + +/** + * Map an axis-aligned RectBounds through a homography. A non-affine transform + * turns a rectangle into a general quad, so the result is the axis-aligned + * bounding box of the four mapped corners. + */ +export function mapBounds(matrix: Matrix3, bounds: RectBounds): RectBounds { + return pointsToBounds(boundsCorners(bounds).map((c) => applyHomography(matrix, c))); +} + +/** + * Map a rotated rectangle (RectBounds + rotation in radians about the bounds + * center, the convention of utils.rotateGeoJSONCoordinates) through a + * homography, returning a best-fit rotated rectangle in the target space: + * the mapped direction of the rect's top edge gives the new rotation, and the + * mapped corners un-rotated about their centroid give the new bounds. Exact + * for similarity transforms; a least-surprise approximation for the rest. + */ +export function mapRotatedBounds( + matrix: Matrix3, + bounds: RectBounds, + rotation: number, +): { bounds: RectBounds; rotation: number } { + const cx = (bounds[0] + bounds[2]) / 2; + const cy = (bounds[1] + bounds[3]) / 2; + const rotate = (p: Point, angle: number, ox: number, oy: number): Point => { + const cos = Math.cos(angle); + const sin = Math.sin(angle); + const dx = p[0] - ox; + const dy = p[1] - oy; + return [ox + dx * cos - dy * sin, oy + dx * sin + dy * cos]; + }; + const mapped = boundsCorners(bounds) + .map((c) => applyHomography(matrix, rotate(c, rotation, cx, cy))); + // The top edge (corner 0 -> corner 1) points along +x at zero rotation, so + // its mapped direction is the combined rotation in the target space. + const newRotation = Math.atan2( + mapped[1][1] - mapped[0][1], + mapped[1][0] - mapped[0][0], + ); + const mcx = mapped.reduce((sum, p) => sum + p[0], 0) / mapped.length; + const mcy = mapped.reduce((sum, p) => sum + p[1], 0) / mapped.length; + const unrotated = mapped.map((p) => rotate(p, -newRotation, mcx, mcy)); + return { bounds: pointsToBounds(unrotated), rotation: newRotation }; +} + +type MappableGeometry = GeoJSON.Point | GeoJSON.LineString | GeoJSON.Polygon; + +/** + * Deep-copy GeoJSON features (the shapes tracks store per-frame: Point, + * LineString, Polygon) with every coordinate mapped through a homography. + * Unsupported geometry types are copied through unmapped. + */ +export function mapGeoJSONFeatures( + matrix: Matrix3, + features: GeoJSON.Feature[], +): GeoJSON.Feature[] { + const mapPosition = (p: GeoJSON.Position): GeoJSON.Position => ( + applyHomography(matrix, [p[0], p[1]]) + ); + return features.map((feature) => { + let geometry: MappableGeometry; + if (feature.geometry.type === 'Point') { + geometry = { type: 'Point', coordinates: mapPosition(feature.geometry.coordinates) }; + } else if (feature.geometry.type === 'LineString') { + geometry = { type: 'LineString', coordinates: feature.geometry.coordinates.map(mapPosition) }; + } else { + geometry = { + type: 'Polygon', + coordinates: feature.geometry.coordinates.map((ring) => ring.map(mapPosition)), + }; + } + return { + ...feature, + geometry: geometry as T, + properties: feature.properties ? { ...feature.properties } : feature.properties, + }; + }); +} diff --git a/client/src/cameraRegistrationFiles.spec.ts b/client/src/cameraRegistrationFiles.spec.ts new file mode 100644 index 000000000..f832cea5f --- /dev/null +++ b/client/src/cameraRegistrationFiles.spec.ts @@ -0,0 +1,165 @@ +// eslint-disable-next-line import/no-extraneous-dependencies -- Vitest is only used in tests +import { describe, expect, it } from 'vitest'; + +import { + buildPerCameraRegistrationFiles, + registrationValuesSummary, + filterRegistrationValues, + mergeRegistrationValues, + CameraRegistrationValues, +} from './cameraRegistrationFiles'; + +const IDENTITY = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; +const SHIFT = [[1, 0, 5], [0, 1, -3], [0, 0, 1]]; +const UNSHIFT = [[1, 0, -5], [0, 1, 3], [0, 0, 1]]; + +function values(partial: Partial): CameraRegistrationValues { + return { + homographies: {}, + correspondences: {}, + transformTypes: {}, + source: null, + ...partial, + }; +} + +describe('buildPerCameraRegistrationFiles', () => { + it('groups each pair under its non-reference camera, sorted', () => { + const files = buildPerCameraRegistrationFiles(values({ + homographies: { + 'rgb::uv': { AtoB: IDENTITY, BtoA: IDENTITY }, + 'ir::rgb': { AtoB: SHIFT, BtoA: UNSHIFT }, + }, + }), 'rgb'); + expect(files.map((f) => f.camera)).toStrictEqual(['ir', 'uv']); + expect(files.map((f) => f.name)).toStrictEqual(['ir_to_rgb_registration.json', 'uv_to_rgb_registration.json']); + // The pair whose RIGHT camera is the reference files under its left. + expect(files[0].body.pairs).toStrictEqual([{ + left: 'ir', + right: 'rgb', + points: [], + leftToRight: SHIFT, + rightToLeft: UNSHIFT, + transformType: 'similarity', + }]); + }); + + it('falls back to right-camera grouping without a reference', () => { + const files = buildPerCameraRegistrationFiles(values({ + homographies: { 'rgb::ir': { AtoB: SHIFT, BtoA: UNSHIFT } }, + }), null); + expect(files.map((f) => f.camera)).toStrictEqual(['ir']); + }); + + it('self-identifies files and carries a plain source stamp', () => { + const source = { producer: 'kamera', run: 'fl07' }; + const [file] = buildPerCameraRegistrationFiles(values({ + homographies: { 'rgb::ir': { AtoB: SHIFT, BtoA: UNSHIFT } }, + source, + }), 'rgb'); + expect(file.body.type).toBe('dive-camera-registration'); + expect(file.body.source).toStrictEqual(source); + }); + + it('never stamps files with a mixed composite source', () => { + const [file] = buildPerCameraRegistrationFiles(values({ + homographies: { 'rgb::ir': { AtoB: SHIFT, BtoA: UNSHIFT } }, + source: { mixed: true, files: {} }, + }), 'rgb'); + expect('source' in file.body).toBe(false); + }); + + it('serializes correspondences as leftX leftY rightX rightY rows', () => { + const [file] = buildPerCameraRegistrationFiles(values({ + correspondences: { + 'rgb::ir': [{ id: 1, a: [10, 20], b: [12, 22] }], + }, + }), 'rgb'); + expect(file.body.pairs[0].points).toStrictEqual([[10, 20, 12, 22]]); + expect(file.body.pairs[0].leftToRight).toBeNull(); + }); +}); + +describe('filterRegistrationValues', () => { + const multi = values({ + homographies: { + 'rgb::ir': { AtoB: SHIFT, BtoA: UNSHIFT }, + 'rgb::uv': { AtoB: IDENTITY, BtoA: IDENTITY }, + }, + transformTypes: { 'rgb::ir': 'rigid', 'rgb::uv': 'affine' }, + source: { producer: 'kamera' }, + }); + + it('keeps only pairs naming the camera, on either side', () => { + const filtered = filterRegistrationValues(multi, 'ir'); + expect(Object.keys(filtered.homographies)).toStrictEqual(['rgb::ir']); + expect(Object.keys(filtered.transformTypes)).toStrictEqual(['rgb::ir']); + expect(filtered.source).toStrictEqual({ producer: 'kamera' }); + }); + + it('yields an empty calibration for an unknown camera', () => { + expect(registrationValuesSummary(filterRegistrationValues(multi, 'zz')).pairCount).toBe(0); + }); +}); + +describe('registrationValuesSummary', () => { + it('counts distinct pairs and names their cameras', () => { + const summary = registrationValuesSummary(values({ + homographies: { 'rgb::ir': { AtoB: SHIFT, BtoA: UNSHIFT } }, + correspondences: { 'rgb::uv': [{ id: 1, a: [1, 2], b: [3, 4] }] }, + })); + expect(summary.pairCount).toBe(2); + expect(summary.cameras.sort()).toStrictEqual(['ir', 'rgb', 'uv']); + }); +}); + +describe('mergeRegistrationValues', () => { + const existing = values({ + homographies: { 'rgb::ir': { AtoB: SHIFT, BtoA: UNSHIFT } }, + correspondences: { 'rgb::ir': [{ id: 1, a: [1, 2], b: [3, 4] }] }, + transformTypes: { 'rgb::ir': 'rigid' }, + source: { producer: 'kamera', run: 'fl07' }, + }); + + it('keeps pairs the import does not name', () => { + const merged = mergeRegistrationValues(existing, values({ + homographies: { 'rgb::uv': { AtoB: IDENTITY, BtoA: IDENTITY } }, + }), 'calibration_uv.json'); + expect(Object.keys(merged.homographies).sort()).toStrictEqual(['rgb::ir', 'rgb::uv']); + expect(merged.transformTypes['rgb::ir']).toBe('rigid'); + }); + + it('replaces a named pair wholly, dropping stale points and model choice', () => { + const merged = mergeRegistrationValues(existing, values({ + homographies: { 'rgb::ir': { AtoB: IDENTITY, BtoA: IDENTITY } }, + }), 'calibration_ir.json'); + expect(merged.homographies['rgb::ir'].AtoB).toStrictEqual(IDENTITY); + expect(merged.correspondences['rgb::ir']).toBeUndefined(); + expect(merged.transformTypes['rgb::ir']).toBeUndefined(); + }); + + it('keeps the existing stamp when the import carries none', () => { + const merged = mergeRegistrationValues(existing, values({ + homographies: { 'rgb::uv': { AtoB: IDENTITY, BtoA: IDENTITY } }, + }), 'calibration_uv.json'); + expect(merged.source).toStrictEqual({ producer: 'kamera', run: 'fl07' }); + }); + + it('keeps a single stamp when both agree, mixes when they disagree', () => { + const agreeing = mergeRegistrationValues(existing, values({ + source: { producer: 'kamera', run: 'fl07' }, + }), 'calibration_uv.json'); + expect(agreeing.source).toStrictEqual({ producer: 'kamera', run: 'fl07' }); + + const disagreeing = mergeRegistrationValues(existing, values({ + source: { producer: 'kamera', run: 'fl09' }, + }), 'calibration_uv.json'); + expect(disagreeing.source).toStrictEqual({ + mixed: true, + files: { + previous: { producer: 'kamera', run: 'fl07' }, + 'calibration_uv.json': { producer: 'kamera', run: 'fl09' }, + }, + }); + }); +}); diff --git a/client/src/cameraRegistrationFiles.ts b/client/src/cameraRegistrationFiles.ts new file mode 100644 index 000000000..c727cf8fc --- /dev/null +++ b/client/src/cameraRegistrationFiles.ts @@ -0,0 +1,202 @@ +/** + * The portable per-camera image-registration file format, shared by every + * producer and consumer of _to__registration.json files: + * the desktop backend (persistence + export), the web client (export + * downloads + import uploads), and the multicam import seed. One file per + * non-reference camera, named for the mapping it carries (camera registers + * onto the reference) and self-identified with + * `type: 'dive-camera-registration'`; pair bodies name their own cameras, + * so file names are discovery/provenance only. + */ +import { + RegistrationFile, RegistrationFilePair, RegistrationSource, + CameraCorrespondences, CameraHomographies, CameraTransformTypes, + REGISTRATION_FILE_TYPE, +} from './CameraRegistrationStore'; +import { DEFAULT_TRANSFORM_TYPE } from './transform'; + +/** The complete in-app calibration state for one dataset. */ +export interface CameraRegistrationValues { + homographies: CameraHomographies; + correspondences: CameraCorrespondences; + transformTypes: CameraTransformTypes; + source: RegistrationSource | null; +} + +/** + * File name for one camera's registration. The destination (the camera it + * registers onto -- normally the rig reference) is part of the name so the + * file states its own direction: ir_to_eo_registration.json registers ir + * onto eo. Omitted when the camera's pairs have no single partner. + */ +export function registrationFileName(camera: string, destination: string | null): string { + return destination + ? `${camera}_to_${destination}_registration.json` + : `${camera}_registration.json`; +} + +/** + * Restrict a calibration to the pairs naming `camera` (either side). Used by + * the per-camera import buttons so a multi-pair file only contributes the + * chosen camera's pair(s). + */ +export function filterRegistrationValues( + values: CameraRegistrationValues, + camera: string, +): CameraRegistrationValues { + const keep = (key: string) => key.split('::').includes(camera); + const filterRecord = (record: Record): Record => Object.fromEntries( + Object.entries(record).filter(([key]) => keep(key)), + ); + return { + homographies: filterRecord(values.homographies), + correspondences: filterRecord(values.correspondences), + transformTypes: filterRecord(values.transformTypes), + source: values.source, + }; +} + +/** The distinct camera names and pair count a calibration holds. */ +export function registrationValuesSummary( + values: CameraRegistrationValues, +): { cameras: string[]; pairCount: number } { + const keys = new Set([ + ...Object.keys(values.homographies), + ...Object.keys(values.correspondences), + ...Object.keys(values.transformTypes), + ]); + const cameras = new Set(); + keys.forEach((key) => key.split('::').forEach((name) => cameras.add(name))); + return { cameras: [...cameras], pairCount: keys.size }; +} + +/** + * Convert the in-app calibration state (keyed by directional "left::right") + * into the self-describing list of file pairs. + */ +function toRegistrationFilePairs(values: CameraRegistrationValues): RegistrationFilePair[] { + const keys = new Set([ + ...Object.keys(values.homographies), + ...Object.keys(values.correspondences), + ...Object.keys(values.transformTypes), + ]); + return [...keys].map((key) => { + const [left, right] = key.split('::'); + const homography = values.homographies[key]; + return { + left, + right, + points: (values.correspondences[key] || []).map((c) => [c.a[0], c.a[1], c.b[0], c.b[1]]), + leftToRight: homography ? homography.AtoB : null, + rightToLeft: homography ? homography.BtoA : null, + transformType: values.transformTypes[key] || DEFAULT_TRANSFORM_TYPE, + }; + }); +} + +/** + * Group a calibration into its per-camera file bodies: one self-identified + * _to__registration.json per non-reference camera + * (reference = first camera in display order). A pair not touching the + * reference files under its right camera, named for that pair's other side; + * grouping is cosmetic either way since pair bodies are authoritative on + * load. + */ +export function buildPerCameraRegistrationFiles( + values: CameraRegistrationValues, + referenceCamera: string | null, +): { camera: string; destination: string | null; name: string; body: RegistrationFile }[] { + const pairsByCamera = new Map(); + toRegistrationFilePairs(values).forEach((pair) => { + let camera = pair.right; + if (referenceCamera !== null && pair.right === referenceCamera + && pair.left !== referenceCamera) { + camera = pair.left; + } + pairsByCamera.set(camera, [...(pairsByCamera.get(camera) ?? []), pair]); + }); + // A mixed composite stamp describes the assembled SET, not any single file; + // stamping every per-camera file with it would present a unanimous + // (therefore "consistent") rig on the next load, hiding the very mismatch + // it records. Per-file stamps resume with the next externally produced file. + const fileSource = values.source + && (values.source as Record).mixed === true + ? null + : values.source; + return [...pairsByCamera.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([camera, pairs]) => { + // The camera this file's content warps onto: the single partner named + // across its pairs (normally the rig reference). Multiple partners + // leave the name destination-free rather than picking one arbitrarily. + const partners = new Set(pairs.flatMap( + (pair) => [pair.left, pair.right].filter((name) => name !== camera), + )); + const destination = partners.size === 1 ? [...partners][0] : null; + return { + camera, + destination, + name: registrationFileName(camera, destination), + body: { + type: REGISTRATION_FILE_TYPE, + version: 1, + ...(fileSource ? { source: fileSource } : {}), + pairs, + }, + }; + }); +} + +/** + * Merge a newly imported calibration into a dataset's existing one. Every + * pair the import names replaces that pair wholly (points, transforms, and + * model choice together -- a pair is one artifact); pairs it doesn't name + * are kept, so per-camera files can be imported one at a time to assemble a + * rig. Producer stamps follow the same policy as multi-file loading: + * agreement keeps the stamp, disagreement is recorded as a + * `{ mixed: true, files: {...} }` composite so the client can warn about a + * rig assembled from different calibration generations. + */ +export function mergeRegistrationValues( + existing: CameraRegistrationValues, + incoming: CameraRegistrationValues, + incomingLabel: string, +): CameraRegistrationValues { + const homographies = { ...existing.homographies }; + const correspondences = { ...existing.correspondences }; + const transformTypes = { ...existing.transformTypes }; + const incomingKeys = new Set([ + ...Object.keys(incoming.homographies), + ...Object.keys(incoming.correspondences), + ...Object.keys(incoming.transformTypes), + ]); + incomingKeys.forEach((key) => { + delete homographies[key]; + delete correspondences[key]; + delete transformTypes[key]; + if (incoming.homographies[key]) { + homographies[key] = incoming.homographies[key]; + } + if (incoming.correspondences[key]?.length) { + correspondences[key] = incoming.correspondences[key]; + } + if (incoming.transformTypes[key]) { + transformTypes[key] = incoming.transformTypes[key]; + } + }); + let source: RegistrationSource | null; + if (incoming.source === null) { + source = existing.source; + } else if (existing.source === null + || JSON.stringify(existing.source) === JSON.stringify(incoming.source)) { + source = incoming.source; + } else { + source = { + mixed: true, + files: { previous: existing.source, [incomingLabel]: incoming.source }, + }; + } + return { + homographies, correspondences, transformTypes, source, + }; +} diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 376ef45cc..82ffdccb3 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -15,13 +15,18 @@ import OverlapLayer from '../layers/AnnotationLayers/OverlapLayer'; import EditAnnotationLayer, { EditAnnotationTypes } from '../layers/EditAnnotationLayer'; import LassoSelectionLayer from '../layers/LassoSelectionLayer'; +import AlignedImageLayer from '../layers/AlignedImageLayer'; import { FrameDataTrack } from '../layers/LayerTypes'; +import { applyHomography, invert3, Matrix3 } from '../homography'; +import { mapBounds, mapRotatedBounds, mapGeoJSONFeatures } from '../alignedView'; +import type { Feature } from '../track'; import TextLayer, { FormatTextRow } from '../layers/AnnotationLayers/TextLayer'; import AttributeLayer from '../layers/AnnotationLayers/AttributeLayer'; import AttributeBoxLayer from '../layers/AnnotationLayers/AttributeBoxLayer'; import type { AnnotationId } from '../BaseAnnotation'; import { geojsonToBound, isRotationValue, ROTATION_ATTRIBUTE_NAME, featureHasSegmentationPolygon, + getRotationFromAttributes, } from '../utils'; import { VisibleAnnotationTypes } from '../layers'; import UILayer from '../layers/UILayers/UILayer'; @@ -39,6 +44,7 @@ import { useAnnotatorPreferences, useGroupStyleManager, useCameraStore, + useAlignedView, useSelectedCamera, useAttributes, useComparisonSets, @@ -77,6 +83,12 @@ export default defineComponent({ // Viewer may not provide lasso context in tests or minimal embeds. } const cameraStore = useCameraStore(); + let alignedView: ReturnType | undefined; + try { + alignedView = useAlignedView(); + } catch { + // aligned view store may not be provided in tests or minimal embeds. + } const selectedCamera = useSelectedCamera(); const comparison = useComparisonSets(); const trackStore = cameraStore.camMap.value.get(props.camera)?.trackStore; @@ -107,6 +119,144 @@ export default defineComponent({ const flickNumberRef = annotator.flick; const hasFrameRef = annotator.hasFrame; + /** + * Resolve another camera's currently displayed frame image (for the ghost + * overlay and the aligned-view warp). Matches the `quad.image` data used + * by ImageAnnotator and the `quad.video` data used by VideoAnnotator -- + * geojs' canvas quad renderer supports both as texture sources. + * LargeImageAnnotator (tiled/geospatial imagery) has no single resolvable + * image element, so it returns null and the ghost overlay / display warp + * is simply unavailable for those datasets; picking itself + * (native-coordinate inverse mapping) is unaffected either way. + */ + const getCameraImage = (camera: string) => { + let viewer; + try { + // getController throws for an unknown/cleared camera; the ghost and + // aligned-warp rAF loops call this after a dataset reload has cleared + // the controllers, so swallow it here rather than let it escape into + // the animation-frame callback uncaught. + viewer = aggregateController.value.getController(camera)?.geoViewerRef?.value; + } catch { + return null; + } + if (!viewer || typeof viewer.layers !== 'function') { + return null; + } + const layerList = viewer.layers(); + for (let i = 0; i < layerList.length; i += 1) { + const layer = layerList[i]; + if (typeof layer.features === 'function') { + const features = layer.features(); + for (let j = 0; j < features.length; j += 1) { + const data = typeof features[j].data === 'function' ? features[j].data() : undefined; + const datum = Array.isArray(data) ? data[0] : undefined; + if (datum && datum.image) { + const image = datum.image as HTMLImageElement; + return { + source: image, kind: 'image' as const, width: image.naturalWidth, height: image.naturalHeight, + }; + } + if (datum && datum.video) { + const video = datum.video as HTMLVideoElement; + return { + source: video, kind: 'video' as const, width: video.videoWidth, height: video.videoHeight, + }; + } + } + } + } + return null; + }; + + /** + * Aligned view (SEAL-TK features 2 + 3): while active, this camera's + * display transform (native -> reference space, null when unwarped). + * Stored geometry stays native (decision D3); the transform is applied + * at draw time only. + */ + const alignedDisplayTransform = computed( + () => (alignedView ? alignedView.cameraTransform(props.camera) : null), + ); + /** + * Inverse of the display transform (reference/display space -> this + * camera's native space). The edit layer operates in geojs map + * coordinates -- display space -- so draws and edits made while the + * aligned view warps this camera must be mapped back through this before + * being committed to (native) track storage. + */ + const alignedDisplayInverse = computed(() => { + const matrix = alignedDisplayTransform.value; + if (!matrix) { + return null; + } + try { + return invert3(matrix); + } catch { + return null; + } + }); + /** Map a native-space location into display space for view centering. */ + const mapDisplayPoint = (x: number, y: number) => { + const matrix = alignedDisplayTransform.value; + if (!matrix) { + return { x, y }; + } + const [mx, my] = applyHomography(matrix, [x, y]); + return { x: mx, y: my }; + }; + /** + * Copy a native-space track feature into display space for the edit + * layer (identity passthrough when this camera renders unwarped), so + * edit handles land on the warped imagery. The stored feature is never + * mutated (decision D3: storage stays native). + */ + function featureToDisplay(feature: Feature | null): Feature | null { + const matrix = alignedDisplayTransform.value; + if (!matrix || !feature) { + return feature; + } + const mapped: Feature = { ...feature }; + const rotation = getRotationFromAttributes(feature.attributes); + if (feature.bounds) { + if (rotation !== undefined) { + const rotated = mapRotatedBounds(matrix, feature.bounds, rotation); + mapped.bounds = rotated.bounds; + mapped.attributes = { + ...feature.attributes, + [ROTATION_ATTRIBUTE_NAME]: rotated.rotation, + }; + } else { + mapped.bounds = mapBounds(matrix, feature.bounds); + } + } + if (feature.geometry) { + mapped.geometry = { + ...feature.geometry, + features: mapGeoJSONFeatures(matrix, feature.geometry.features), + }; + } + return mapped; + } + + // Created before the annotation layers below so its geojs layer z-orders + // beneath boxes/polygons/text (geojs stacks layers by creation order). + const alignedImageLayer = new AlignedImageLayer({ + annotator, + getImage: () => { + try { + return getCameraImage(props.camera); + } catch { + // Controllers may be cleared mid-poll during a dataset reload. + return null; + } + }, + getTransform: () => alignedDisplayTransform.value, + // Right-click means "remove last point" while creating/editing + // geometry; recenter everywhere else. + getRecenterEnabled: () => !editingModeRef.value, + }); + const rectAnnotationLayer = new RectangleLayer({ annotator, stateStyling: trackStyleManager.stateStyles, @@ -187,7 +337,13 @@ export default defineComponent({ watch([segmentationPointsRef, frameNumberRef, selectedCamera], ([newPoints, currentFrame, currentCamera]) => { if (newPoints.points.length > 0 && newPoints.frameNum === currentFrame && props.camera === currentCamera) { - segmentationPointsLayer.updatePoints(newPoints.points, newPoints.labels); + // Prompt points are stored in native image space; render them where + // the warped imagery actually is (identity when unwarped). + const displayPoints = newPoints.points.map((p): [number, number] => { + const { x, y } = mapDisplayPoint(p[0], p[1]); + return [x, y]; + }); + segmentationPointsLayer.updatePoints(displayPoints, newPoints.labels); } else { segmentationPointsLayer.clear(); } @@ -262,6 +418,11 @@ export default defineComponent({ selectedKey: string, colorBy: string, ) { + // Drawing and editing work on every camera while the aligned view is + // on: the edit layer operates in display (warped) space -- it is fed + // display-space copies of the geometry (featureToDisplay below) and its + // draws/edits are mapped back to native through alignedDisplayInverse + // in the update:geojson handler before committing to track storage. const currentFrameIds: AnnotationId[] | undefined = trackStore?.intervalTree .search([frame, frame]) .map((str) => parseInt(str, 10)); @@ -320,9 +481,13 @@ export default defineComponent({ } if (clientSettings.annotatorPreferences.lockedCamera.enabled) { if (trackFrame.features?.bounds) { + // Under the aligned view the display is warped, so center + // on the displayed (warped) location, not the native one. const coords = { - x: (trackFrame.features.bounds[0] + trackFrame.features.bounds[2]) / 2.0, - y: (trackFrame.features.bounds[1] + trackFrame.features.bounds[3]) / 2.0, + ...mapDisplayPoint( + (trackFrame.features.bounds[0] + trackFrame.features.bounds[2]) / 2.0, + (trackFrame.features.bounds[1] + trackFrame.features.bounds[3]) / 2.0, + ), z: 0, }; const [x0, y0, x1, y1] = trackFrame.features.bounds; @@ -341,10 +506,14 @@ export default defineComponent({ const halfWidth = (width * multiplyBoundsVal) / 2.0; const halfHeight = (height * multiplyBoundsVal) / 2.0; - const left = centerX - halfWidth; - const right = centerX + halfWidth; - const top = centerY - halfHeight; - const bottom = centerY + halfHeight; + // Map the zoom-target corners into display space too + // (identity when the aligned view is off). + const ulMapped = mapDisplayPoint(centerX - halfWidth, centerY - halfHeight); + const lrMapped = mapDisplayPoint(centerX + halfWidth, centerY + halfHeight); + const left = Math.min(ulMapped.x, lrMapped.x); + const right = Math.max(ulMapped.x, lrMapped.x); + const top = Math.min(ulMapped.y, lrMapped.y); + const bottom = Math.max(ulMapped.y, lrMapped.y); const zoomAndCenter = annotator.geoViewerRef.value.zoomAndCenterFromBounds({ left, top, right, bottom, @@ -384,7 +553,11 @@ export default defineComponent({ } else { lineLayer.disable(); } - if (visibleModes.includes('TrackTail')) { + // Track tails read multi-frame geometry straight from the trackStore + // (not FrameDataTrack) and are not routed through the display + // transform, so they are hidden for warped cameras while the aligned + // view is on rather than rendered in the wrong (native) space. + if (visibleModes.includes('TrackTail') && !alignedDisplayTransform.value) { tailLayer.updateSettings( frame, annotatorPrefs.value.trackTails.before, @@ -432,7 +605,13 @@ export default defineComponent({ if (editingTrack) { editAnnotationLayer.setType(editingTrack); editAnnotationLayer.setKey(selectedKey); - editAnnotationLayer.changeData(editingTracks); + // The edit layer works in display space: hand it display-space + // copies of the feature so its handles land on warped imagery + // (identity when this camera renders unwarped). + editAnnotationLayer.changeData(editingTracks.map((trackFrame) => ({ + ...trackFrame, + features: featureToDisplay(trackFrame.features), + }))); } } else if (editingTrack && props.camera !== selectedCamera.value && (isCreatingNewDetection(frame, selectedTrackId) @@ -506,6 +685,49 @@ export default defineComponent({ refreshLayers(); }); + /** Layers whose stored-geometry rendering follows the aligned-view warp. */ + const displayTransformedLayers = [ + rectAnnotationLayer, + overlapLayer, + polyAnnotationLayer, + lineLayer, + pointLayer, + textLayer, + attributeBoxLayer, + attributeLayer, + ]; + + /** + * Apply (or clear) the aligned-view display transform: warp the imagery + * quad and point every geometry layer's draw-time mapping at the same + * matrix, then re-render. Immediate so a LayerManager created while the + * aligned view is already on (e.g. a view-mode switch) starts warped. + */ + watch(alignedDisplayTransform, (matrix) => { + displayTransformedLayers.forEach((layer) => layer.setDisplayTransform(matrix)); + alignedImageLayer.update(); + updateLayers( + frameNumberRef.value, + editingModeRef.value, + selectedTrackIdRef.value, + multiSeletListRef.value, + enabledTracksRef.value, + visibleModesRef.value, + selectedKeyRef.value, + props.colorBy, + ); + }, { immediate: true }); + + // The warped imagery must follow frame changes: the annotator swaps its + // element asynchronously after each seek, and AlignedImageLayer + // polls briefly after every trigger to catch that swap. Guarded so this + // is a strict no-op whenever the camera renders unwarped. + watch(frameNumberRef, () => { + if (alignedDisplayTransform.value) { + alignedImageLayer.update(); + } + }); + /** Shallow watch */ watch( [ @@ -745,16 +967,31 @@ export default defineComponent({ return; } } + // Under the aligned view this camera renders warped, so the draw/edit + // just made lives in display (reference) space: map it back to this + // camera's native space before committing to track storage (decision + // D3 -- storage stays native). Identity when the camera is unwarped. + const inverse = alignedDisplayInverse.value; if (type === 'rectangle') { - const bounds = geojsonToBound(data as GeoJSON.Feature); + let bounds = geojsonToBound(data as GeoJSON.Feature); // Extract rotation from properties if it exists - const rotation = data.properties && isRotationValue(data.properties?.[ROTATION_ATTRIBUTE_NAME]) + let rotation = data.properties && isRotationValue(data.properties?.[ROTATION_ATTRIBUTE_NAME]) ? data.properties[ROTATION_ATTRIBUTE_NAME] as number : undefined; + if (inverse) { + if (rotation !== undefined) { + const mapped = mapRotatedBounds(inverse, bounds, rotation); + bounds = mapped.bounds; + rotation = mapped.rotation; + } else { + bounds = mapBounds(inverse, bounds); + } + } cb(); handler.updateRectBounds(frameNumberRef.value, flickNumberRef.value, bounds, rotation); } else { - handler.updateGeoJSON(mode, frameNumberRef.value, flickNumberRef.value, data, key, cb); + const nativeData = inverse ? mapGeoJSONFeatures(inverse, [data])[0] : data; + handler.updateGeoJSON(mode, frameNumberRef.value, flickNumberRef.value, nativeData, key, cb); } // Jump into edit mode if we completed a new shape if (geometryCompleteEvent) { @@ -779,8 +1016,13 @@ export default defineComponent({ ); // Handle clicks outside the edit polygon to allow selecting other polygons editAnnotationLayer.bus.$on('click-outside-edit', (geo: { x: number; y: number }) => { - // Check which polygon was clicked by iterating through formatted data - const point: [number, number] = [geo.x, geo.y]; + // Check which polygon was clicked by iterating through formatted data. + // The click arrives in display space while formattedData is native, so + // map it back through the aligned-view inverse (identity when unwarped). + const inverse = alignedDisplayInverse.value; + const point: [number, number] = inverse + ? applyHomography(inverse, [geo.x, geo.y]) + : [geo.x, geo.y]; const polygonData = polyAnnotationLayer.formattedData; // Find the polygon that contains the click point diff --git a/client/src/components/annotators/mediaControllerType.ts b/client/src/components/annotators/mediaControllerType.ts index ece6d1d12..ff771830d 100644 --- a/client/src/components/annotators/mediaControllerType.ts +++ b/client/src/components/annotators/mediaControllerType.ts @@ -45,6 +45,16 @@ export interface AggregateMediaController { cameraSync: Readonly>; /** Incremented when the viewer is resized, used to trigger layer redraws */ resizeTrigger: Readonly>; + /** + * True only while onResize is applying its programmatic size()/resetZoom() + * to the panes. resetZoom emits GeoJS pan/zoom events synchronously; the + * linked-viewer navigation (useAlignedNavigation / useCalibrationNavigation) + * must ignore those so one pane's native-space reset isn't broadcast to the + * others as if it were a shared-space move (which parks warped panes on an + * empty corner). The resizeTrigger bump that follows re-snaps every pane from + * the reference once the reset has settled. + */ + resizing: Readonly>; /** * Global aligned-timeline slot indices with at least one camera missing a * frame (see AlignedFrameResolver's gapSlots); empty whenever alignment diff --git a/client/src/components/annotators/useAlignedNavigation.spec.ts b/client/src/components/annotators/useAlignedNavigation.spec.ts new file mode 100644 index 000000000..a28108171 --- /dev/null +++ b/client/src/components/annotators/useAlignedNavigation.spec.ts @@ -0,0 +1,195 @@ +/// +import { + ref, shallowRef, nextTick, Ref, +} from 'vue'; +import useAlignedNavigation from './useAlignedNavigation'; +import AlignedViewStore from '../../AlignedViewStore'; +import type { AggregateMediaController } from './mediaControllerType'; +import type { Matrix3 } from '../../homography'; + +vi.mock('geojs', () => ({ default: { event: { pan: 'geo_pan', zoom: 'geo_zoom' } } })); + +const IDENTITY: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + +/** Minimal stand-in for a geojs viewer: center/zoom state + geoOn events. */ +function fakeViewer(baseUnitsPerPixel: number) { + const state = { center: { x: 0, y: 0 }, zoom: 0 }; + const handlers: Record void>> = {}; + return { + geoOn(evt: string, handler: () => void) { + handlers[evt] = handlers[evt] || []; + handlers[evt].push(handler); + }, + geoOff(evt: string, handler: () => void) { + handlers[evt] = (handlers[evt] || []).filter((h) => h !== handler); + }, + center(c?: { x: number; y: number }) { + if (c) { + state.center = { ...c }; + } + return state.center; + }, + zoom(z?: number) { + if (z !== undefined) { + state.zoom = z; + } + return state.zoom; + }, + unitsPerPixel(z: number) { + return baseUnitsPerPixel / 2 ** z; + }, + trigger(evt: string) { + (handlers[evt] || []).forEach((h) => h()); + }, + }; +} + +function makeHarness() { + // EO: high-res pane (fine zoom-0 baseline); IR: low-res pane. + const eo = fakeViewer(1); + const ir = fakeViewer(8); + const resetZoom = vi.fn(); + const controllers: Record; resetZoom: () => void }> = { + eo: { geoViewerRef: ref(eo), resetZoom }, + ir: { geoViewerRef: ref(ir), resetZoom }, + }; + const cameraSync = ref(false); + const resizing = ref(false); + const resizeTrigger = ref(0); + // shallowRef: a plain ref would deep-unwrap the nested cameraSync / + // resizeTrigger refs, unlike the real aggregate controller object. + const aggregate = shallowRef({ + cameraSync, + resizeTrigger, + resizing, + getController: (name: string) => controllers[name], + }) as unknown as Ref; + const cameras = ref(['eo', 'ir']); + const alignedView = new AlignedViewStore(); + useAlignedNavigation(aggregate, alignedView, cameras); + return { + eo, ir, cameraSync, resizing, resizeTrigger, alignedView, resetZoom, + }; +} + +describe('useAlignedNavigation', () => { + it('snaps panes to the reference view immediately on activation', async () => { + const { eo, ir, alignedView } = makeHarness(); + eo.center({ x: 250, y: 150 }); + eo.zoom(1); // units-per-pixel = 0.5 + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[1, 0, 100], [0, 1, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + // No pan/zoom event fired: activation itself aligned the panes. + expect(ir.center()).toEqual({ x: 250, y: 150 }); + expect(ir.zoom()).toBeCloseTo(Math.log2(8 / 0.5), 6); + }); + + it('links panes by IDENTITY center in the shared reference space', async () => { + const { eo, ir, alignedView } = makeHarness(); + // ir's imagery renders warped through ir->eo (translation +100), so its + // pane coordinates ARE reference coordinates: the link must NOT map the + // center through the camera-to-camera transform again. + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[1, 0, 100], [0, 1, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + eo.center({ x: 500, y: 300 }); + eo.zoom(2); // source units-per-pixel = 1 / 2^2 = 0.25 reference units + eo.trigger('geo_pan'); + + expect(ir.center()).toEqual({ x: 500, y: 300 }); + // Same reference extent through ir's own zoom-0 baseline: + // log2(8 / 0.25) = 5. + expect(ir.zoom()).toBeCloseTo(5, 6); + }); + + it('propagates from the warped pane back to the reference pane identically', async () => { + const { eo, ir, alignedView } = makeHarness(); + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[2, 0, 0], [0, 2, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + ir.center({ x: 40, y: 60 }); + ir.trigger('geo_pan'); + + expect(eo.center()).toEqual({ x: 40, y: 60 }); + }); + + it('resets every pane to its native view when the aligned view deactivates', async () => { + const { alignedView, resetZoom } = makeHarness(); + alignedView.setTransforms('eo', { eo: IDENTITY, ir: IDENTITY }); + alignedView.setEnabled(true); + await nextTick(); + expect(resetZoom).not.toHaveBeenCalled(); + + alignedView.setEnabled(false); + await nextTick(); + // Once per pane (eo + ir). + expect(resetZoom).toHaveBeenCalledTimes(2); + }); + + it('ignores the native-space pan/zoom events onResize emits while resizing', async () => { + const { + eo, ir, resizing, resizeTrigger, alignedView, + } = makeHarness(); + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[1, 0, 100], [0, 1, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + // A good aligned view: both panes centered in the shared reference space. + eo.center({ x: 500, y: 300 }); + eo.zoom(2); + eo.trigger('geo_pan'); + expect(ir.center()).toEqual({ x: 500, y: 300 }); + + // onResize resets each pane to its OWN native bounds and fires pan events. + // ir's reset drops it to its native center; while resizing this must NOT be + // broadcast into the reference space (that is what parked panes in a black + // corner). eo -- the reference -- must stay put. + resizing.value = true; + ir.center({ x: 40, y: 60 }); + ir.trigger('geo_pan'); + expect(eo.center()).toEqual({ x: 500, y: 300 }); + resizing.value = false; + + // The resizeTrigger bump that follows re-snaps every pane from the + // reference, so ir lands back on the reference-space center. + resizeTrigger.value += 1; + await nextTick(); + expect(ir.center()).toEqual({ x: 500, y: 300 }); + }); + + it('stands down while inactive, suspended, or raw camera sync is on', async () => { + const { + eo, ir, cameraSync, alignedView, + } = makeHarness(); + alignedView.setTransforms('eo', { eo: IDENTITY, ir: IDENTITY }); + alignedView.setEnabled(true); + await nextTick(); + + cameraSync.value = true; + eo.center({ x: 9, y: 9 }); + eo.trigger('geo_pan'); + expect(ir.center()).toEqual({ x: 0, y: 0 }); + cameraSync.value = false; + + alignedView.setSuspended(true); + await nextTick(); + eo.trigger('geo_pan'); + expect(ir.center()).toEqual({ x: 0, y: 0 }); + }); +}); diff --git a/client/src/components/annotators/useAlignedNavigation.ts b/client/src/components/annotators/useAlignedNavigation.ts new file mode 100644 index 000000000..133e35749 --- /dev/null +++ b/client/src/components/annotators/useAlignedNavigation.ts @@ -0,0 +1,105 @@ +import { Ref, watch } from 'vue'; +import type { AggregateMediaController } from './mediaControllerType'; +import type AlignedViewStore from '../../AlignedViewStore'; +import useLinkedViewers from './useLinkedViewers'; + +/** + * Links pan/zoom recentering across ALL loaded cameras while the aligned view + * is active (SEAL-TK feature 3). + * + * While active, every pane RENDERS in the shared reference space, so the link + * between panes is the IDENTITY on coordinates: same center, same + * reference-units-per-screen-pixel. (Mapping centers through the + * camera-to-camera transforms here would re-apply a transform the rendering has + * already applied.) Distinct from the raw "sync cameras" toggle (Controls.vue), + * which forwards raw screen deltas for UNWARPED panes and is hidden whenever the + * aligned view is available, and from the calibration pair link + * ({@link useCalibrationNavigation}), which maps through the homography and + * stands down while the aligned view is active. + */ +export default function useAlignedNavigation( + aggregateController: Ref, + alignedView: AlignedViewStore, + cameras: Ref, +) { + const { + viewer, teardown, attach, guarded, applyView, + } = useLinkedViewers(aggregateController); + + function link(camera: string) { + return () => guarded(() => { + if (!alignedView.active.value) { + return; + } + // onResize resets each pane to its own native bounds and emits pan/zoom + // events; ignore them so a non-reference pane's native center doesn't get + // copied into the shared reference space. setup() re-snaps from the + // reference once the resize settles (via the resizeTrigger watch). + if (aggregateController.value.resizing.value) { + return; + } + // Never fight the raw screen-delta sync (unreachable while the aligned + // view is available, but be defensive about two handlers on one event). + if (aggregateController.value.cameraSync.value) { + return; + } + const source = viewer(camera); + if (!source) { + return; + } + // Shared reference space: copy the center verbatim and match the extent. + const center = source.center(); + const view = { + center: { x: center.x, y: center.y }, + unitsPerPixel: source.unitsPerPixel(source.zoom()), + }; + cameras.value.forEach((other) => { + if (other !== camera) { + const target = viewer(other); + if (target) { + applyView(target, view); + } + } + }); + }); + } + + function setup() { + teardown(); + if (!alignedView.active.value) { + return; + } + cameras.value.forEach((camera) => attach(camera, link(camera))); + // Snap immediately from the reference pane so hitting Align lines every pane + // up right away instead of waiting for the first pan/zoom event. + const reference = alignedView.reference.value; + if (reference && cameras.value.includes(reference)) { + link(reference)(); + } + } + + watch( + [ + alignedView.active, + alignedView.toReference, + cameras, + aggregateController.value.resizeTrigger, + ], + setup, + ); + + // Leaving the aligned view strands every pane at reference-space centers/zooms + // while content reverts to native coordinates -- reset each to its full native + // view so the imagery is back on-screen. + watch(alignedView.active, (active, wasActive) => { + if (!active && wasActive) { + cameras.value.forEach((camera) => { + try { + aggregateController.value.getController(camera).resetZoom(); + } catch { + // A pane may already be torn down during dataset unload. + } + }); + } + }); +} diff --git a/client/src/components/annotators/useLinkedViewers.ts b/client/src/components/annotators/useLinkedViewers.ts new file mode 100644 index 000000000..fd757f434 --- /dev/null +++ b/client/src/components/annotators/useLinkedViewers.ts @@ -0,0 +1,94 @@ +import geo from 'geojs'; +import { onBeforeUnmount, Ref } from 'vue'; +import type { AggregateMediaController } from './mediaControllerType'; + +/** + * Where to move a linked pane: the world-space center to show, and how many + * world units one screen pixel should span there (null leaves the pane's + * current zoom untouched, for when a local scale can't be resolved). + */ +export interface LinkedView { + center: { x: number; y: number }; + unitsPerPixel: number | null; +} + +/** + * Shared plumbing for the two multicam pan/zoom links -- the calibration pair + * link ({@link useCalibrationNavigation}) and the aligned-view link + * ({@link useAlignedNavigation}). Both attach geojs pan/zoom listeners to a set + * of panes and, when one moves, drive the others to a matching view; they + * differ only in how a source pane's view maps onto a target pane. This owns + * the parts that don't: a re-entrancy guard so a driven update doesn't echo + * back, listener bookkeeping, and the zoom-baseline conversion (geojs zoom is + * log2-based, and panes sized to different-resolution images keep their own + * zoom-0 baselines, so matching an extent converts through the target's + * baseline rather than copying the zoom level). + */ +export default function useLinkedViewers( + aggregateController: Ref, +) { + // Setting a pane's center/zoom from a handler must not re-trigger its own listener. + let guard = false; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let attached: { viewer: any; handler: () => void }[] = []; + + /** This camera's geojs viewer, or null when it isn't initialized/known yet. */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function viewer(camera: string): any { + try { + return aggregateController.value.getController(camera).geoViewerRef.value || null; + } catch { + return null; + } + } + + /** Detach every attached pan/zoom listener. */ + function teardown() { + attached.forEach(({ viewer: v, handler }) => { + v.geoOff(geo.event.pan, handler); + v.geoOff(geo.event.zoom, handler); + }); + attached = []; + } + + /** Attach a pan+zoom handler to `camera`'s pane; no-op when it has no viewer yet. */ + function attach(camera: string, handler: () => void) { + const v = viewer(camera); + if (!v) { + return; + } + v.geoOn(geo.event.pan, handler); + v.geoOn(geo.event.zoom, handler); + attached.push({ viewer: v, handler }); + } + + /** Run `fn` with the re-entrancy guard raised (a no-op while already guarded). */ + function guarded(fn: () => void) { + if (guard) { + return; + } + guard = true; + try { + fn(); + } finally { + guard = false; + } + } + + /** Drive `target` to `view`: center, plus a matching extent via the target's own zoom-0 baseline. */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function applyView(target: any, view: LinkedView) { + if (view.unitsPerPixel !== null) { + const targetZoom = Math.log2(target.unitsPerPixel(0) / view.unitsPerPixel); + if (Number.isFinite(targetZoom)) { + target.zoom(targetZoom); + } + } + target.center(view.center); + } + + onBeforeUnmount(teardown); + return { + viewer, teardown, attach, guarded, applyView, + }; +} diff --git a/client/src/components/annotators/useMediaController.ts b/client/src/components/annotators/useMediaController.ts index 5b3317277..e81f884c1 100644 --- a/client/src/components/annotators/useMediaController.ts +++ b/client/src/components/annotators/useMediaController.ts @@ -93,6 +93,10 @@ export function useMediaController() { let cameraControllerSymbols: Record = {}; const synchronizeCameras: Ref = ref(false); const resizeTrigger: Ref = ref(0); + // Raised only while onResize applies its programmatic resetZoom, so the + // linked-viewer navigation ignores the resulting pan/zoom events (see + // AggregateMediaController.resizing). + const resizing: Ref = ref(false); // shallowRef: an AlignedFrameResolver carries nested Refs (slotCount, frameRate) // that must NOT be deep-reactive-converted/auto-unwrapped by a plain ref(). const alignedFrameResolver: Ref = shallowRef(null); @@ -127,6 +131,7 @@ export function useMediaController() { toggleSynchronizeCameras, cameraSync: synchronizeCameras, resizeTrigger, + resizing, alignedGapSlots, seekCameraFrame: aggregateSeekCameraFrame, }; @@ -226,7 +231,18 @@ export function useMediaController() { // Resize maps first, then redraw annotation layers once GeoJS has settled. if (resized) { window.requestAnimationFrame(() => { - pendingResizes.forEach((applyResize) => applyResize()); + // resetZoom recenters each pane on its OWN native bounds and emits + // pan/zoom events synchronously. Suppress the linked-viewer navigation + // for the duration so a non-reference pane's native center isn't + // broadcast into the shared/reference space (which would strand warped + // panes on an empty corner). The resizeTrigger bump below then re-snaps + // every pane from a clean reference view. + resizing.value = true; + try { + pendingResizes.forEach((applyResize) => applyResize()); + } finally { + resizing.value = false; + } window.requestAnimationFrame(() => { resizeTrigger.value += 1; }); @@ -686,6 +702,7 @@ export function useMediaController() { toggleSynchronizeCameras, cameraSync: synchronizeCameras, resizeTrigger, + resizing, alignedGapSlots, seekCameraFrame: aggregateSeekCameraFrame, }; diff --git a/client/src/components/controls/Controls.vue b/client/src/components/controls/Controls.vue index c580c5bfc..9223b4459 100644 --- a/client/src/components/controls/Controls.vue +++ b/client/src/components/controls/Controls.vue @@ -10,7 +10,9 @@ import { DatasetType, useApi } from 'dive-common/apispec'; import { computeGapGradient } from 'dive-common/alignedTimeline'; import { frameToTimestamp } from 'vue-media-annotator/utils'; import { injectAggregateController } from '../annotators/useMediaController'; -import { useTime, useTrackFilters, useDatasetId } from '../../provides'; +import { + useTime, useTrackFilters, useDatasetId, useAlignedView, +} from '../../provides'; export default defineComponent({ name: 'Controls', @@ -38,6 +40,35 @@ export default defineComponent({ dragging: false, }); const mediaController = injectAggregateController().value; + let alignedView: ReturnType | undefined; + try { + alignedView = useAlignedView(); + } catch { + // aligned view store may not be provided in tests or minimal embeds. + } + /** + * The raw screen-delta camera sync is only offered while the transform + * -aware aligned view (the Align button) is unavailable: once every + * camera has a calibration transform, Align is the single place to link + * pan/zoom, and this cruder link (which assumes identical pixel scale + * between panes) would just be a second, worse toggle for the same thing. + */ + const rawSyncAvailable = computed(() => mediaController.cameras.value.length > 1 + && !alignedView?.available.value); + const toggleRawSync = () => { + if (rawSyncAvailable.value) { + mediaController.toggleSynchronizeCameras(!mediaController.cameraSync.value); + } + }; + // If transforms become available while the raw sync is on, switch it off: + // its toggle is hidden from that point, and the aligned-view link stands + // down while raw sync is enabled, so a stuck-on raw sync would silently + // block the Align button's linking with no visible control to clear it. + watch(rawSyncAvailable, (available) => { + if (!available && mediaController.cameraSync.value) { + mediaController.toggleSynchronizeCameras(false); + } + }, { immediate: true }); const isVideo = computed(() => props.datasetType === 'video'); const { frameRate } = useTime(); const { visible } = usePrompt(); @@ -229,6 +260,8 @@ export default defineComponent({ activeTimeFilter, data, mediaController, + rawSyncAvailable, + toggleRawSync, dragHandler, input, alignedGapGradient, @@ -274,7 +307,7 @@ export default defineComponent({ { bind: 'd', handler: mediaController.prevFrame, disabled: visible() }, { bind: 'l', - handler: () => mediaController.toggleSynchronizeCameras(!mediaController.cameraSync.value), + handler: toggleRawSync, disabled: visible(), }, ]" @@ -626,12 +659,12 @@ export default defineComponent({ {{ mediaController.cameraSync.value ? 'mdi-link' : 'mdi-link-off' }} @@ -946,12 +979,12 @@ export default defineComponent({ {{ mediaController.cameraSync.value ? 'mdi-link' : 'mdi-link-off' }} @@ -1236,13 +1269,13 @@ export default defineComponent({ {{ mediaController.cameraSync.value ? 'mdi-link' : 'mdi-link-off' }} diff --git a/client/src/components/index.ts b/client/src/components/index.ts index ec4d30a9a..21164ef6b 100644 --- a/client/src/components/index.ts +++ b/client/src/components/index.ts @@ -28,6 +28,7 @@ import TypePicker from './TypePicker.vue'; export * from './annotators/useMediaController'; export { default as useAnnotatorImageCursor } from './annotators/useAnnotatorImageCursor'; +export { default as useAlignedNavigation } from './annotators/useAlignedNavigation'; export { /* Annotators */ AnnotatorImageCursor, diff --git a/client/src/homography.spec.ts b/client/src/homography.spec.ts new file mode 100644 index 000000000..9facf0512 --- /dev/null +++ b/client/src/homography.spec.ts @@ -0,0 +1,258 @@ +/// +import { + solveHomography, + applyHomography, + invert3, + matMul3, + subdivideWarpQuads, + warpGridSize, + localLinkedScale, + Point, + Matrix3, +} from './homography'; + +function expectMatrixClose(actual: Matrix3, expected: Matrix3, tol = 1e-6) { + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + expect(actual[i][j]).toBeCloseTo(expected[i][j], 5); + // tol referenced to keep signature explicit + expect(Math.abs(actual[i][j] - expected[i][j])).toBeLessThan(tol * 10); + } + } +} + +const unitSquare: Point[] = [[0, 0], [1, 0], [1, 1], [0, 1]]; + +describe('homography', () => { + it('recovers the identity from identical correspondences', () => { + const H = solveHomography(unitSquare, unitSquare); + expectMatrixClose(H, [[1, 0, 0], [0, 1, 0], [0, 0, 1]]); + }); + + it('recovers a pure translation', () => { + const dst = unitSquare.map(([x, y]): Point => [x + 5, y - 3]); + const H = solveHomography(unitSquare, dst); + expectMatrixClose(H, [[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); + + it('recovers a scale + translation', () => { + const H = solveHomography(unitSquare, [[10, 10], [30, 10], [30, 30], [10, 30]]); + expectMatrixClose(H, [[20, 0, 10], [0, 20, 10], [0, 0, 1]]); + }); + + it('maps source points onto destination points (least-squares, >4 pts)', () => { + // A known projective transform applied to 5 points; solver should recover it. + const truth: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + const src: Point[] = [[0, 0], [100, 0], [100, 80], [0, 80], [50, 40]]; + const dst = src.map((p) => applyHomography(truth, p)); + const H = solveHomography(src, dst); + src.forEach((p) => { + const [u, v] = applyHomography(H, p); + const [eu, ev] = applyHomography(truth, p); + expect(u).toBeCloseTo(eu, 3); + expect(v).toBeCloseTo(ev, 3); + }); + }); + + it('round-trips through its inverse (H * H^-1 ~= I)', () => { + const H = solveHomography(unitSquare, [[10, 5], [40, 8], [38, 35], [9, 33]]); + const product = matMul3(H, invert3(H)); + const scale = 1 / product[2][2]; + const normalized = product.map((r) => r.map((c) => c * scale)) as Matrix3; + expectMatrixClose(normalized, [[1, 0, 0], [0, 1, 0], [0, 0, 1]]); + }); + + it('throws with fewer than 4 correspondences', () => { + expect(() => solveHomography(unitSquare.slice(0, 3), unitSquare.slice(0, 3))).toThrow(); + }); + + it('throws on a degenerate (collinear) point configuration despite having 4 points', () => { + const collinear: Point[] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + expect(() => solveHomography(collinear, collinear)).toThrow(/degenerate/i); + }); +}); + +describe('warpGridSize', () => { + const affine: Matrix3 = [[1.5, 0.2, 30], [-0.1, 0.9, -12], [0, 0, 1]]; + const projective: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + + it('returns 1 for a pure affine transform', () => { + expect(warpGridSize(affine, 640, 480)).toBe(1); + }); + + it('returns maxN when perspective terms are non-negligible', () => { + expect(warpGridSize(projective, 640, 480)).toBe(8); + expect(warpGridSize(projective, 640, 480, 12)).toBe(12); + }); + + it('returns 1 for negligible perspective terms', () => { + // Perspective terms exist but vary w by only ~0.006% over the extent. + const nearAffine: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [1e-7, 0, 1]]; + expect(warpGridSize(nearAffine, 640, 480)).toBe(1); + }); + + it('returns maxN when the horizon crosses the image (w changes sign)', () => { + const extreme: Matrix3 = [[1, 0, 0], [0, 1, 0], [-0.01, 0, 1]]; + // w at x=0 is 1, at x=640 is -5.4. + expect(warpGridSize(extreme, 640, 480)).toBe(8); + }); +}); + +describe('subdivideWarpQuads', () => { + const projective: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + + it('with n=1 produces a single quad matching the full image corners', () => { + const [quad, ...rest] = subdivideWarpQuads(projective, 640, 480, 1); + expect(rest).toHaveLength(0); + expect(quad.crop).toEqual({ + left: 0, top: 0, right: 640, bottom: 480, + }); + expect(quad.ul).toEqual(applyHomography(projective, [0, 0])); + expect(quad.ur).toEqual(applyHomography(projective, [640, 0])); + expect(quad.lr).toEqual(applyHomography(projective, [640, 480])); + expect(quad.ll).toEqual(applyHomography(projective, [0, 480])); + }); + + it('maps every sub-quad corner through the exact homography', () => { + const n = 8; + const quads = subdivideWarpQuads(projective, 640, 480, n); + expect(quads).toHaveLength(n * n); + quads.forEach((q) => { + expect(q.ul).toEqual(applyHomography(projective, [q.crop.left, q.crop.top])); + expect(q.ur).toEqual(applyHomography(projective, [q.crop.right, q.crop.top])); + expect(q.lr).toEqual(applyHomography(projective, [q.crop.right, q.crop.bottom])); + expect(q.ll).toEqual(applyHomography(projective, [q.crop.left, q.crop.bottom])); + }); + }); + + it('expands cells by the overlap (clamped to the image), corners still exact', () => { + const n = 4; + const overlap = 2; + const plain = subdivideWarpQuads(projective, 640, 480, n); + const padded = subdivideWarpQuads(projective, 640, 480, n, overlap); + expect(padded).toHaveLength(plain.length); + padded.forEach((q, i) => { + const base = plain[i].crop; + expect(q.crop.left).toBe(Math.max(0, base.left - overlap)); + expect(q.crop.right).toBe(Math.min(640, base.right + overlap)); + expect(q.crop.top).toBe(Math.max(0, base.top - overlap)); + expect(q.crop.bottom).toBe(Math.min(480, base.bottom + overlap)); + // Corners remain the exact projective mapping of the (expanded) crop, + // so overlapping regions of adjacent cells render identical content. + expect(q.ul).toEqual(applyHomography(projective, [q.crop.left, q.crop.top])); + expect(q.lr).toEqual(applyHomography(projective, [q.crop.right, q.crop.bottom])); + }); + // Interior edges overlap: cell 0's right crop passes cell 1's left crop. + expect(padded[0].crop.right).toBeGreaterThan(padded[1].crop.left); + }); + + it('tiles the source image exactly, with integer grid lines and no gaps', () => { + const n = 8; + const width = 641; // not divisible by n + const height = 479; + const quads = subdivideWarpQuads(projective, width, height, n); + const lefts = new Set(quads.map((q) => q.crop.left)); + const tops = new Set(quads.map((q) => q.crop.top)); + expect(lefts.size).toBe(n); + expect(tops.size).toBe(n); + quads.forEach((q) => { + [q.crop.left, q.crop.top, q.crop.right, q.crop.bottom].forEach((v) => { + expect(Number.isInteger(v)).toBe(true); + }); + expect(q.crop.right).toBeGreaterThan(q.crop.left); + expect(q.crop.bottom).toBeGreaterThan(q.crop.top); + // Each cell's right/bottom edge is another cell's left/top edge or the border. + expect(q.crop.right === width || lefts.has(q.crop.right)).toBe(true); + expect(q.crop.bottom === height || tops.has(q.crop.bottom)).toBe(true); + }); + // Crop areas sum to the full image area (exact tiling, no overlap/gap). + const area = quads.reduce( + (sum, q) => sum + (q.crop.right - q.crop.left) * (q.crop.bottom - q.crop.top), + 0, + ); + expect(area).toBe(width * height); + }); + + it('skips degenerate zero-area cells for images smaller than the grid', () => { + const quads = subdivideWarpQuads(projective, 3, 3, 8); + expect(quads.length).toBeGreaterThan(0); + quads.forEach((q) => { + expect(q.crop.right).toBeGreaterThan(q.crop.left); + expect(q.crop.bottom).toBeGreaterThan(q.crop.top); + }); + const area = quads.reduce( + (sum, q) => sum + (q.crop.right - q.crop.left) * (q.crop.bottom - q.crop.top), + 0, + ); + expect(area).toBe(9); + }); + + it('sub-quad centers stay close to the true projective warp (approximation quality)', () => { + // The deviation between the piecewise-affine render and the true warp is + // largest in cell interiors; measure it at the center of every cell as the + // distance between the true projective warp of the cell's source center + // and the average of the four warped corners (what an affine-ish renderer + // produces there). + const width = 640; + const height = 480; + const centerError = (quads: ReturnType) => Math.max( + ...quads.map((q) => { + const midSrc: Point = [ + (q.crop.left + q.crop.right) / 2, + (q.crop.top + q.crop.bottom) / 2, + ]; + const truth = applyHomography(projective, midSrc); + const approx: Point = [ + (q.ul[0] + q.ur[0] + q.lr[0] + q.ll[0]) / 4, + (q.ul[1] + q.ur[1] + q.lr[1] + q.ll[1]) / 4, + ]; + return Math.hypot(approx[0] - truth[0], approx[1] - truth[1]); + }), + ); + const singleQuadError = centerError(subdivideWarpQuads(projective, width, height, 1)); + const gridError = centerError(subdivideWarpQuads(projective, width, height, 8)); + // This matrix has strong perspective: a single (parallelogram-rendered) + // quad is tens of pixels off at the image center. + expect(singleQuadError).toBeGreaterThan(10); + // Error shrinks roughly with 1/n^2; at n=8 even this extreme perspective + // (w varies ~2x across the image) is down to a couple of pixels. + expect(gridError).toBeLessThan(3); + expect(gridError).toBeLessThan(singleQuadError / 20); + }); +}); + +describe('localLinkedScale', () => { + const mapper = (matrix: Matrix3) => (p: Point) => applyHomography(matrix, p); + + it('returns 1 for the identity mapping', () => { + const identity: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + expect(localLinkedScale(mapper(identity), [100, 200])).toBeCloseTo(1); + }); + + it('recovers a uniform similarity scale regardless of rotation', () => { + const s = 2.5; + const cos = Math.cos(Math.PI / 6) * s; + const sin = Math.sin(Math.PI / 6) * s; + const similarity: Matrix3 = [[cos, -sin, 10], [sin, cos, -4], [0, 0, 1]]; + expect(localLinkedScale(mapper(similarity), [50, 75])).toBeCloseTo(s); + }); + + it('samples the local scale of a projective transform at the given point', () => { + const homography: Matrix3 = [[1, 0, 0], [0, 1, 0], [0.001, 0, 1]]; + const nearOrigin = localLinkedScale(mapper(homography), [0, 0], 1); + const farRight = localLinkedScale(mapper(homography), [500, 0], 1); + expect(nearOrigin).toBeCloseTo(1, 1); + // At x=500 the perspective divide (w = 1.5) has shrunk the local scale + // well below 1; exact value differs per axis, so just assert the shrink. + expect(farRight).not.toBeNull(); + expect(farRight as number).toBeLessThan(0.7); + }); + + it('returns null when the mapping is unavailable', () => { + expect(localLinkedScale(() => null, [10, 10])).toBeNull(); + }); + + it('returns null for a degenerate (collapsing) mapping', () => { + expect(localLinkedScale(() => [3, 3], [10, 10])).toBeNull(); + }); +}); diff --git a/client/src/homography.ts b/client/src/homography.ts new file mode 100644 index 000000000..ff09188a7 --- /dev/null +++ b/client/src/homography.ts @@ -0,0 +1,309 @@ +/** + * Self-contained homography estimation via the normalized Direct Linear Transform. + * + * Given >= 4 point correspondences src[i] -> dst[i] (both [x, y] in image + * coordinates), {@link solveHomography} returns the 3x3 matrix H such that, in + * homogeneous coordinates, dst ~= H * src. Exact for 4 points, least-squares for + * more. This is the client-side analogue of OpenCV's cv2.findHomography used by + * the keypointgui reference app; the warp itself is done by geojs (quadFeature). + */ + +export type Point = [number, number]; +export type Matrix3 = number[][]; + +/** Multiply two 3x3 matrices. */ +export function matMul3(a: Matrix3, b: Matrix3): Matrix3 { + const out: Matrix3 = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]; + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + let sum = 0; + for (let k = 0; k < 3; k += 1) { + sum += a[i][k] * b[k][j]; + } + out[i][j] = sum; + } + } + return out; +} + +/** Invert a 3x3 matrix. Throws if the matrix is singular. */ +export function invert3(m: Matrix3): Matrix3 { + const [a, b, c] = m[0]; + const [d, e, f] = m[1]; + const [g, h, i] = m[2]; + const A = e * i - f * h; + const B = -(d * i - f * g); + const C = d * h - e * g; + const det = a * A + b * B + c * C; + if (Math.abs(det) < 1e-12) { + throw new Error('Cannot invert singular matrix'); + } + const invDet = 1 / det; + return [ + [A * invDet, -(b * i - c * h) * invDet, (b * f - c * e) * invDet], + [B * invDet, (a * i - c * g) * invDet, -(a * f - c * d) * invDet], + [C * invDet, -(a * h - b * g) * invDet, (a * e - b * d) * invDet], + ]; +} + +/** Apply a 3x3 homography to a single point (perspective divide). */ +export function applyHomography(h: Matrix3, p: Point): Point { + const x = h[0][0] * p[0] + h[0][1] * p[1] + h[0][2]; + const y = h[1][0] * p[0] + h[1][1] * p[1] + h[1][2]; + const w = h[2][0] * p[0] + h[2][1] * p[1] + h[2][2]; + return [x / w, y / w]; +} + +/** + * Local scale factor of a point mapping around `center`: how many target-image + * pixels one source-image pixel spans there, estimated by probing `delta` + * pixels along each axis and averaging. For similarity/affine transforms this + * is constant; for homographies it varies with position, which is why it's + * sampled at a specific point (e.g. the current view center for linked + * pan/zoom). Returns null when the mapping is unavailable or degenerate at + * that point. + */ +export function localLinkedScale( + mapPoint: (p: Point) => Point | null, + center: Point, + delta = 10, +): number | null { + const mapped = mapPoint(center); + const mappedX = mapPoint([center[0] + delta, center[1]]); + const mappedY = mapPoint([center[0], center[1] + delta]); + if (!mapped || !mappedX || !mappedY) { + return null; + } + const scaleX = Math.hypot(mappedX[0] - mapped[0], mappedX[1] - mapped[1]) / delta; + const scaleY = Math.hypot(mappedY[0] - mapped[0], mappedY[1] - mapped[1]) / delta; + const scale = (scaleX + scaleY) / 2; + if (!Number.isFinite(scale) || scale <= 0) { + return null; + } + return scale; +} + +/** + * One cell of a subdivided image warp: the axis-aligned source-image rectangle + * `crop` (in source pixels) and the four destination corners it maps to under + * the exact projective transform. See {@link subdivideWarpQuads}. + */ +export interface WarpQuad { + ul: Point; + ur: Point; + lr: Point; + ll: Point; + crop: { left: number; top: number; right: number; bottom: number }; +} + +/** + * Choose a subdivision grid size for rendering the warp of a `width` x `height` + * image through `h` with an affine-only quad renderer (e.g. geojs' canvas + * renderer, which draws each quad from only three of its corners). A pure + * affine matrix (zero perspective row terms) warps exactly as a single + * parallelogram, so 1 is returned; when the perspective terms are + * non-negligible over the image extent, `maxN` is returned so each sub-quad is + * approximately affine. + */ +export function warpGridSize(h: Matrix3, width: number, height: number, maxN = 8): number { + // Homogeneous w at each image corner: constant w <=> affine transform. + const wAt = (x: number, y: number) => h[2][0] * x + h[2][1] * y + h[2][2]; + const ws = [wAt(0, 0), wAt(width, 0), wAt(width, height), wAt(0, height)]; + if (ws.some((w) => !Number.isFinite(w) || w === 0)) { + return maxN; + } + const absW = ws.map((w) => Math.abs(w)); + const maxAbs = Math.max(...absW); + const minAbs = Math.min(...absW); + // Sign change means the horizon line crosses the image: definitely projective. + if (ws.some((w) => w * ws[0] < 0)) { + return maxN; + } + // Relative variation of w across the quad; below ~0.1% the affine + // approximation is visually indistinguishable (sub-pixel for typical sizes). + return (maxAbs - minAbs) / maxAbs < 1e-3 ? 1 : maxN; +} + +/** + * Subdivide the warp of a `width` x `height` image through `h` into an n x n + * grid of {@link WarpQuad}s. Every sub-quad corner is mapped through the exact + * projective transform, so rendering each cell as an (approximately affine) + * textured quad converges to the true projective warp as n grows -- unlike + * rendering the whole image as a single quad, which an affine canvas renderer + * collapses to a parallelogram. Grid lines land on integer source pixels; + * degenerate (zero-area) cells from tiny images are skipped. + * + * `overlap` expands each cell by that many source pixels (clamped to the + * image). The canvas renderer antialiases every quad's border against the + * transparent background, so abutting cells meet as two half-transparent + * edges and show as dark seam lines along the grid; overlapping cells paint + * over each other's seams with pixel-identical content (corners still map + * through the same exact homography). Only for opaque drawing -- translucent + * quads would double-blend in the overlap, so semi-transparent consumers + * must apply opacity at the layer level and draw quads opaque. + */ +export function subdivideWarpQuads( + h: Matrix3, + width: number, + height: number, + n: number, + overlap = 0, +): WarpQuad[] { + const cells = Math.max(1, Math.floor(n)); + const xs: number[] = []; + const ys: number[] = []; + for (let i = 0; i <= cells; i += 1) { + xs.push(Math.round((i * width) / cells)); + ys.push(Math.round((i * height) / cells)); + } + const quads: WarpQuad[] = []; + for (let row = 0; row < cells; row += 1) { + for (let col = 0; col < cells; col += 1) { + const left = Math.max(0, xs[col] - overlap); + const right = Math.min(width, xs[col + 1] + overlap); + const top = Math.max(0, ys[row] - overlap); + const bottom = Math.min(height, ys[row + 1] + overlap); + if (right <= left || bottom <= top) { + // eslint-disable-next-line no-continue + continue; + } + quads.push({ + ul: applyHomography(h, [left, top]), + ur: applyHomography(h, [right, top]), + lr: applyHomography(h, [right, bottom]), + ll: applyHomography(h, [left, bottom]), + crop: { + left, top, right, bottom, + }, + }); + } + } + return quads; +} + +/** + * Hartley normalization: translate points to the centroid and scale so the + * mean distance from the origin is sqrt(2). Returns the normalized points and + * the 3x3 transform T such that normalized = T * original. + */ +function normalizePoints(pts: Point[]): { normalized: Point[]; transform: Matrix3 } { + const n = pts.length; + let cx = 0; + let cy = 0; + pts.forEach(([x, y]) => { cx += x; cy += y; }); + cx /= n; + cy /= n; + let meanDist = 0; + pts.forEach(([x, y]) => { meanDist += Math.hypot(x - cx, y - cy); }); + meanDist /= n; + const scale = meanDist > 1e-12 ? Math.SQRT2 / meanDist : 1; + const transform: Matrix3 = [ + [scale, 0, -scale * cx], + [0, scale, -scale * cy], + [0, 0, 1], + ]; + const normalized = pts.map(([x, y]): Point => [(x - cx) * scale, (y - cy) * scale]); + return { normalized, transform }; +} + +/** + * Solve the linear system A x = b for x using Gaussian elimination with partial + * pivoting. A is square (n x n), modified in place. + */ +/* eslint-disable no-param-reassign */ +export function solveLinearSystem(A: number[][], b: number[]): number[] { + const n = b.length; + for (let col = 0; col < n; col += 1) { + // Partial pivot: find the row with the largest magnitude in this column. + let pivot = col; + for (let row = col + 1; row < n; row += 1) { + if (Math.abs(A[row][col]) > Math.abs(A[pivot][col])) { + pivot = row; + } + } + if (Math.abs(A[pivot][col]) < 1e-12) { + throw new Error('Degenerate point configuration; cannot solve linear system'); + } + if (pivot !== col) { + [A[col], A[pivot]] = [A[pivot], A[col]]; + [b[col], b[pivot]] = [b[pivot], b[col]]; + } + // Eliminate below. + for (let row = col + 1; row < n; row += 1) { + const factor = A[row][col] / A[col][col]; + for (let k = col; k < n; k += 1) { + A[row][k] -= factor * A[col][k]; + } + b[row] -= factor * b[col]; + } + } + // Back-substitution. + const x = new Array(n).fill(0); + for (let row = n - 1; row >= 0; row -= 1) { + let sum = b[row]; + for (let k = row + 1; k < n; k += 1) { + sum -= A[row][k] * x[k]; + } + x[row] = sum / A[row][row]; + } + return x; +} +/* eslint-enable no-param-reassign */ + +/** + * Estimate the homography H (dst ~= H * src) from >= 4 correspondences. + * + * Uses the h33 = 1 formulation in Hartley-normalized coordinates, solved via the + * normal equations (least-squares for > 4 points, exact for 4). Normalization + * keeps the system well-conditioned for typical image-pixel magnitudes. + */ +export function solveHomography(src: Point[], dst: Point[]): Matrix3 { + if (src.length !== dst.length) { + throw new Error('src and dst must have the same number of points'); + } + if (src.length < 4) { + throw new Error('At least 4 point correspondences are required'); + } + + const { normalized: srcN, transform: T1 } = normalizePoints(src); + const { normalized: dstN, transform: T2 } = normalizePoints(dst); + + // Build the 2N x 8 design matrix for the unknowns [h11..h32] (h33 fixed to 1). + const rows: number[][] = []; + const rhs: number[] = []; + for (let i = 0; i < srcN.length; i += 1) { + const [x, y] = srcN[i]; + const [u, v] = dstN[i]; + rows.push([x, y, 1, 0, 0, 0, -x * u, -y * u]); + rhs.push(u); + rows.push([0, 0, 0, x, y, 1, -x * v, -y * v]); + rhs.push(v); + } + + // Normal equations: (A^T A) h = A^T b -> an 8x8 system. + const ata: number[][] = Array.from({ length: 8 }, () => new Array(8).fill(0)); + const atb: number[] = new Array(8).fill(0); + for (let r = 0; r < rows.length; r += 1) { + const row = rows[r]; + for (let i = 0; i < 8; i += 1) { + atb[i] += row[i] * rhs[r]; + for (let j = 0; j < 8; j += 1) { + ata[i][j] += row[i] * row[j]; + } + } + } + + const h = solveLinearSystem(ata, atb); + const hNorm: Matrix3 = [ + [h[0], h[1], h[2]], + [h[3], h[4], h[5]], + [h[6], h[7], 1], + ]; + + // Denormalize: H = inv(T2) * Hnorm * T1. + const denorm = matMul3(matMul3(invert3(T2), hNorm), T1); + + // Scale so H[2][2] == 1 for a canonical form. + const scale = Math.abs(denorm[2][2]) > 1e-12 ? 1 / denorm[2][2] : 1; + return denorm.map((row) => row.map((value) => value * scale)); +} diff --git a/client/src/index.ts b/client/src/index.ts index 33708c1c5..b67cdf27b 100644 --- a/client/src/index.ts +++ b/client/src/index.ts @@ -2,9 +2,11 @@ import * as layers from './layers'; import * as components from './components'; +import AlignedViewStore from './AlignedViewStore'; import BaseAnnotation from './BaseAnnotation'; import BaseAnnotationStore from './BaseAnnotationStore'; import CameraStore from './CameraStore'; +import CameraRegistrationStore from './CameraRegistrationStore'; import Group from './Group'; import GroupFilterControls from './GroupFilterControls'; import GroupStore from './GroupStore'; @@ -25,9 +27,11 @@ export { layers, components, /* other */ + AlignedViewStore, BaseAnnotation, BaseAnnotationStore, CameraStore, + CameraRegistrationStore, Group, GroupFilterControls, GroupStore, diff --git a/client/src/layers/AlignedImageLayer.ts b/client/src/layers/AlignedImageLayer.ts new file mode 100644 index 000000000..157819181 --- /dev/null +++ b/client/src/layers/AlignedImageLayer.ts @@ -0,0 +1,260 @@ +import geo from 'geojs'; +import type { MediaController } from '../components/annotators/mediaControllerType'; +import { Matrix3, subdivideWarpQuads, warpGridSize } from '../homography'; + +export interface CameraImage { + /** The texture source for the geojs quad feature: an `` for image sequences, a `