From 1b599cbb630dee17dc8bcc352fa53a140e7c18c8 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Thu, 9 Jul 2026 16:16:04 -0400 Subject: [PATCH 1/7] Add camera-calibration math and persistence foundation homography.ts: 3x3 matrix primitives (multiply, invert, apply, linear and DLT solvers) plus warp-grid subdivision helpers. transform.ts: keypointgui-style transform models (translation / rigid / similarity / affine / homography) that all estimate to a plain Matrix3, with Similarity as the single shared default. CameraCalibrationStore: the persistence core for camera-rig calibration -- per-pair homographies, correspondences, transform types, and producer provenance, with hydrate() from dataset meta, dirty tracking, and the portable calibration.json round trip (toCalibrationJson / loadCalibrationText). Interactive creation (point picking, fitting, overlay preview) is not part of this branch; it layers on top in the Manual Alignment GUI. Co-Authored-By: Claude Fable 5 --- client/src/CameraCalibrationStore.spec.ts | 325 +++++++++++++++++++ client/src/CameraCalibrationStore.ts | 365 ++++++++++++++++++++++ client/src/homography.spec.ts | 258 +++++++++++++++ client/src/homography.ts | 309 ++++++++++++++++++ client/src/transform.spec.ts | 163 ++++++++++ client/src/transform.ts | 180 +++++++++++ 6 files changed, 1600 insertions(+) create mode 100644 client/src/CameraCalibrationStore.spec.ts create mode 100644 client/src/CameraCalibrationStore.ts create mode 100644 client/src/homography.spec.ts create mode 100644 client/src/homography.ts create mode 100644 client/src/transform.spec.ts create mode 100644 client/src/transform.ts diff --git a/client/src/CameraCalibrationStore.spec.ts b/client/src/CameraCalibrationStore.spec.ts new file mode 100644 index 000000000..b498607ab --- /dev/null +++ b/client/src/CameraCalibrationStore.spec.ts @@ -0,0 +1,325 @@ +/// +import CameraCalibrationStore from './CameraCalibrationStore'; + +describe('CameraCalibrationStore', () => { + // Pure translation by (+5, -3): trivially invertible. + const translate = [[1, 0, 5], [0, 1, -3], [0, 0, 1]]; + // Four correspondence rows consistent with `translate`: right = left + (5, -3). + const translationPointRows = [ + [0, 0, 5, -3], + [10, 0, 15, -3], + [10, 10, 15, 7], + [0, 10, 5, 7], + ]; + + it('produces a directional pairKey that preserves left/right order', () => { + const store = new CameraCalibrationStore(); + expect(store.pairKey('rgb', 'ir')).toEqual('rgb::ir'); + expect(store.pairKey('rgb', 'ir')).not.toEqual(store.pairKey('ir', 'rgb')); + }); + + it('hydrates homographies and resets prior calibration state', () => { + const store = new CameraCalibrationStore(); + store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: [[1, 1, 6, -2]], transformType: 'translation', + }], + })); + const saved = { 'a::b': { AtoB: [[1, 0, 0], [0, 1, 0], [0, 0, 1]], BtoA: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] } }; + store.hydrate(saved); + expect(store.homographies.value).toEqual(saved); + expect(store.correspondences.value).toEqual({}); + expect(store.transformTypes.value).toEqual({}); + }); + + it('hydrates transform types alongside homographies', () => { + const store = new CameraCalibrationStore(); + store.hydrate({}, {}, { 'a::b': 'rigid' }); + expect(store.transformTypeForPair('a::b')).toBe('rigid'); + expect(store.transformTypeForPair('unset::pair')).toBe('similarity'); + }); + + it('hydrates correspondences and resumes id allocation', () => { + const store = new CameraCalibrationStore(); + const correspondences = { + 'rgb::ir': [ + { id: 1, a: [1, 2], b: [3, 4] }, + { id: 2, a: [5, 6], b: [7, 8] }, + ], + }; + store.hydrate({}, correspondences); + expect(store.correspondences.value).toEqual(correspondences); + // Points loaded afterwards pick up ids after the highest restored id. + store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ left: 'rgb', right: 'ir', points: [[9, 9, 10, 10]] }], + })); + expect(store.correspondences.value['rgb::ir'][0].id).toBe(3); + }); + + describe('dirty / markSaved', () => { + it('tracks unsaved changes and resets on markSaved and hydrate', () => { + const store = new CameraCalibrationStore(); + expect(store.dirty.value).toBe(false); + store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: translate, + }], + })); + expect(store.dirty.value).toBe(true); + store.markSaved(); + expect(store.dirty.value).toBe(false); + store.hydrate({ 'a::b': { AtoB: translate, BtoA: translate } }); + // Freshly hydrated state is the saved baseline. + expect(store.dirty.value).toBe(false); + }); + }); + + describe('loaded (file-sourced) homographies', () => { + /** Load a matrix-only (point-less) pair from calibration JSON, marking it 'loaded'. */ + function loadMatrixOnlyPair(store: CameraCalibrationStore, left: string, right: string, rightToLeft: number[][]) { + store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left, right, points: [], leftToRight: null, rightToLeft, + }], + })); + } + + it('loads a matrix-only pair as B->A with its inverse as A->B and no points', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('left', 'right'); + loadMatrixOnlyPair(store, 'left', 'right', translate); + expect(store.correspondences.value[key]).toHaveLength(0); + expect(store.isLoadedHomography(key)).toBe(true); + const homog = store.homographies.value[key]; + expect(homog.BtoA).toEqual(translate); + expect(homog.AtoB[0][2]).toBeCloseTo(-5); + expect(homog.AtoB[1][2]).toBeCloseTo(3); + }); + + it('rejects a singular loaded matrix', () => { + const store = new CameraCalibrationStore(); + expect(() => loadMatrixOnlyPair(store, 'left', 'right', [[0, 0, 0], [0, 0, 0], [0, 0, 0]])) + .toThrow(/singular/); + }); + + it('hydrate marks an under-pointed homography as loaded', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('left', 'right'); + store.hydrate({ [key]: { AtoB: translate, BtoA: translate } }, {}, {}); + expect(store.isLoadedHomography(key)).toBe(true); + }); + }); + + describe('calibration JSON file round trip', () => { + it('serializes and reloads all pairs', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('left', 'right'); + store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', + right: 'right', + points: translationPointRows, + leftToRight: translate, + rightToLeft: null, + transformType: 'translation', + }], + })); + const json = store.toCalibrationJson(); + + const restored = new CameraCalibrationStore(); + const result = restored.loadCalibrationText(json); + expect(result.pairCount).toBe(1); + expect(result.cameras.sort()).toEqual(['left', 'right']); + expect(restored.correspondences.value[key]).toHaveLength(4); + expect(restored.transformTypeForPair(key)).toBe('translation'); + expect(restored.homographies.value[key].AtoB[0][2]).toBeCloseTo(5); + // The missing direction was derived by inversion and round-tripped. + expect(restored.homographies.value[key].BtoA[0][2]).toBeCloseTo(-5); + // Enough points back the homography, so it is treated as fitted. + expect(restored.isLoadedHomography(key)).toBe(false); + }); + + it('includes pairs that only have a loaded homography (no points)', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('uv', 'ir'); + store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'uv', right: 'ir', points: [], leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], rightToLeft: null, + }], + })); + const restored = new CameraCalibrationStore(); + restored.loadCalibrationText(store.toCalibrationJson()); + expect(restored.homographies.value[key]).toBeDefined(); + expect(restored.isLoadedHomography(key)).toBe(true); + }); + + it('loads a desktop-persisted calibration.json (no "type" field, one direction only)', () => { + const store = new CameraCalibrationStore(); + const result = store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'eo', + right: 'ir', + points: [[0, 0, 5, -3]], + leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + rightToLeft: null, + transformType: 'translation', + }], + })); + expect(result.pairCount).toBe(1); + const key = store.pairKey('eo', 'ir'); + expect(store.correspondences.value[key]).toHaveLength(1); + // The missing direction is derived by inversion. + expect(store.homographies.value[key].BtoA[0][2]).toBeCloseTo(-5); + expect(store.transformTypeForPair(key)).toBe('translation'); + }); + + it('preserves the producer source stamp across a load/save round trip', () => { + const store = new CameraCalibrationStore(); + const source = { model: 'colmap-2026-07-01', swathe: 'fl07_C' }; + store.loadCalibrationText(JSON.stringify({ + version: 1, + source, + pairs: [{ + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: translate, + }], + })); + expect(store.source.value).toStrictEqual(source); + const saved = JSON.parse(store.toCalibrationJson()); + expect(saved.source).toStrictEqual(source); + }); + + it('omits the source key when no stamp was loaded', () => { + const store = new CameraCalibrationStore(); + store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, + }], + })); + expect('source' in JSON.parse(store.toCalibrationJson())).toBe(false); + }); + + it('clears a previous stamp when loading a file without one', () => { + const store = new CameraCalibrationStore(); + store.loadCalibrationText(JSON.stringify({ + version: 1, + source: { model: 'old' }, + pairs: [], + })); + store.loadCalibrationText(JSON.stringify({ version: 1, pairs: [] })); + expect(store.source.value).toBeNull(); + }); + + it('rejects a non-object source', () => { + const store = new CameraCalibrationStore(); + expect(() => store.loadCalibrationText(JSON.stringify({ + version: 1, source: 'colmap', pairs: [], + }))).toThrow(/"source" must be an object/); + }); + + it('hydrate restores the source stamp', () => { + const store = new CameraCalibrationStore(); + store.hydrate({}, {}, {}, { model: 'colmap-x' }); + expect(store.source.value).toStrictEqual({ model: 'colmap-x' }); + store.hydrate({}, {}, {}); + expect(store.source.value).toBeNull(); + }); + + it('flags a point-backed homography as refined when a source stamp is loaded', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('left', 'right'); + // Fresh from the producer (matrix-only): loaded, not refined. + store.loadCalibrationText(JSON.stringify({ + version: 1, + source: { model: 'colmap-x' }, + pairs: [{ + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: translate, + }], + })); + expect(store.isRefinedFromSource(key)).toBe(false); + // Point-backed under a stamp: the pair has diverged from the producer. + store.loadCalibrationText(JSON.stringify({ + version: 1, + source: { model: 'colmap-x' }, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, transformType: 'translation', + }], + })); + expect(store.isRefinedFromSource(key)).toBe(true); + }); + + it('does not flag point-backed homographies as refined when no source stamp is loaded', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('left', 'right'); + store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, transformType: 'translation', + }], + })); + expect(store.isRefinedFromSource(key)).toBe(false); + }); + + it('keeps the refined flag across a save/load round trip', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('left', 'right'); + store.loadCalibrationText(JSON.stringify({ + version: 1, + source: { model: 'colmap-x' }, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, transformType: 'translation', + }], + })); + expect(store.isRefinedFromSource(key)).toBe(true); + + const restored = new CameraCalibrationStore(); + restored.loadCalibrationText(store.toCalibrationJson()); + // The refined pair saved with its backing points, so it re-marks as + // fitted (refined) rather than loaded. + expect(restored.isRefinedFromSource(key)).toBe(true); + }); + + it('rejects non-JSON, missing pairs, malformed pairs, and bad matrices without clobbering state', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('left', 'right'); + store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, + }], + })); + expect(() => store.loadCalibrationText('not json')).toThrow(/valid JSON/); + expect(() => store.loadCalibrationText('{"type": "other"}')).toThrow(/pairs/); + expect(() => store.loadCalibrationText(JSON.stringify({ + version: 1, pairs: [{ left: 'a', right: 'a' }], + }))).toThrow(/distinct/); + expect(() => store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'a', + right: 'b', + points: [], + leftToRight: [[1, 0], [0, 1]], + rightToLeft: null, + }], + }))).toThrow(/3x3/); + expect(() => store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ left: 'a', right: 'b', points: [[1, 2, 3]] }], + }))).toThrow(/points row/); + expect(() => store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ left: 'a', right: 'b', transformType: 'bogus' }], + }))).toThrow(/transformType/); + // Failed loads left the existing calibration alone. + expect(store.correspondences.value[key]).toHaveLength(4); + }); + }); +}); diff --git a/client/src/CameraCalibrationStore.ts b/client/src/CameraCalibrationStore.ts new file mode 100644 index 000000000..90c02fe44 --- /dev/null +++ b/client/src/CameraCalibrationStore.ts @@ -0,0 +1,365 @@ +import { + ref, computed, Ref, ComputedRef, +} from 'vue'; +import { + invert3, Matrix3, Point, +} from './homography'; +import { + TransformType, TRANSFORM_TYPES, DEFAULT_TRANSFORM_TYPE, minPointsForTransform, +} from './transform'; + +/** + * A single picked point pair. `a` is the point in the left camera (camA), `b` + * the point in the right camera (camB). Left/right is the order the user chose, + * which is preserved (not alphabetized) so it survives round trips through the + * calibration JSON's ordered `[leftX, leftY, rightX, rightY]` rows. + */ +export interface Correspondence { + id: number; + a: Point; + b: Point; +} + +/** Both directions of the fitted alignment transform for one camera pair. */ +export interface PairHomography { + /** Maps left (camA) image coordinates onto right (camB). */ + AtoB: Matrix3; + /** Maps right (camB) image coordinates onto left (camA). */ + BtoA: Matrix3; +} + +/** Fitted transforms keyed by {@link CameraCalibrationStore.pairKey}. */ +export type CameraHomographies = Record; + +/** + * Where a pair's homography came from: fitted in-app from picked points, or + * loaded from a calibration file (which may carry no points at all). Loaded + * homographies persist through refit checks that would otherwise clear an + * under-pointed pair, until enough points are picked to fit a replacement. + */ +type HomographySource = 'fit' | 'loaded'; + +/** + * Free-form provenance stamped into the calibration file by whatever produced + * the transforms (e.g. an external COLMAP/KAMERA model step: model version, + * swathe/flight id, generation time). DIVE never interprets it -- it is + * preserved verbatim through load/refine/save round trips so an external + * re-solver can tell which model version a returning file was refined against. + */ +export type CalibrationSource = Record; + +/** + * One camera pair in the portable calibration JSON file. This is the same + * self-describing shape the desktop platform persists as the project's + * standalone calibration.json (see desktop backend/native/common.ts), so a + * panel-saved file, the on-disk artifact, and an import-time seed are all + * interchangeable: correspondences flattened as [leftX, leftY, rightX, + * rightY] rows, plus both fitted directions (null when unfitted). + */ +export interface CalibrationFilePair { + left: string; + right: string; + points?: number[][]; + leftToRight?: Matrix3 | null; + rightToLeft?: Matrix3 | null; + transformType?: TransformType; +} + +/** Portable calibration file: everything needed to restore all pairs. */ +export interface CalibrationFile { + /** Written by panel saves for self-identification; optional on load. */ + type?: string; + version: number; + /** Producer provenance, preserved verbatim across round trips. */ + source?: CalibrationSource | null; + pairs: CalibrationFilePair[]; +} + +/** Identifying `type` value of the calibration JSON format. */ +export const CALIBRATION_FILE_TYPE = 'dive-camera-calibration'; + +/** Picked correspondences keyed by {@link CameraCalibrationStore.pairKey}. */ +export type CameraCorrespondences = Record; + +/** Chosen fit model per pair, keyed by {@link CameraCalibrationStore.pairKey}. Missing entries default to 'similarity'. */ +export type CameraTransformTypes = Record; + +/** + * Shared, reactive store for camera-calibration data (correspondences, + * fitted/loaded homographies, transform-type choices, and producer + * provenance). Lives in vue-media-annotator so both the annotation layers + * (client/src/layers) and the dive-common side can consume it via the + * provide/inject system. Handles persistence: hydrating saved state and + * loading/saving the portable calibration JSON format. + */ +export default class CameraCalibrationStore { + correspondences: Ref; + + homographies: Ref; + + transformTypes: Ref; + + /** + * Provenance of the loaded calibration (see {@link CalibrationSource}). + * Deliberately NOT cleared by in-app edits or refits -- refinements are + * exactly what should travel back to the producer stamped with the model + * lineage they were made against. Replaced (or cleared) only when a + * calibration file is loaded or the store is re-hydrated. + */ + source: Ref; + + /** True when the calibration has unsaved changes since the last save or load. */ + dirty: ComputedRef; + + private nextId: number; + + /** Provenance per homography key; missing entries behave like 'fit'. */ + private homographySources: Record; + + /** Serialized calibration at the last save/load, the baseline for {@link dirty}. */ + private savedSnapshot: Ref; + + constructor() { + this.correspondences = ref({}); + this.homographies = ref({}); + this.transformTypes = ref({}); + this.source = ref(null); + this.nextId = 1; + this.homographySources = {}; + this.savedSnapshot = ref(this.calibrationSnapshot()); + this.dirty = computed(() => this.calibrationSnapshot() !== this.savedSnapshot.value); + } + + /** Serialize the saved-to-dataset calibration state (points, transforms, provenance). */ + private calibrationSnapshot(): string { + return JSON.stringify({ + homographies: this.homographies.value, + correspondences: this.correspondences.value, + transformTypes: this.transformTypes.value, + source: this.source.value, + }); + } + + /** Capture the current calibration as the saved baseline, so {@link dirty} reads false. */ + markSaved() { + this.savedSnapshot.value = this.calibrationSnapshot(); + } + + /** + * Directional key for a camera pair: `left::right`. Order is significant and + * preserved so left/right (e.g. RGB vs IR) survives for ordered exports. + */ + // eslint-disable-next-line class-methods-use-this + pairKey(camA: string, camB: string): string { + return `${camA}::${camB}`; + } + + /** + * True when `key`'s homography came from a calibration file rather than an + * in-app fit. Not independently reactive -- always read alongside + * {@link homographies} (provenance only changes when that map does). + */ + isLoadedHomography(key: string): boolean { + return this.homographySources[key] === 'loaded'; + } + + /** + * True when `key`'s transform was fitted from in-app picked points while a + * producer-stamped calibration is loaded -- i.e. the pair has diverged from + * what the stamped {@link source} shipped (producer files carry matrix-only + * pairs, so any point-backed fit is a human refinement). Derived rather + * than stored: it survives save/reload naturally, because point-backed + * pairs re-mark as fitted on hydrate. Read alongside {@link homographies}, + * same reactivity caveat as {@link isLoadedHomography}. + */ + isRefinedFromSource(key: string): boolean { + return this.source.value !== null + && this.homographies.value[key] !== undefined + && this.homographySources[key] === 'fit'; + } + + /** The chosen fit model for `key`, defaulting to {@link DEFAULT_TRANSFORM_TYPE} when unset. */ + transformTypeForPair(key: string): TransformType { + return this.transformTypes.value[key] || DEFAULT_TRANSFORM_TYPE; + } + + /** + * Serialize every pair with content (points and/or a homography) as the + * portable calibration JSON file (see {@link CalibrationFile}). Pairs whose + * only state is a transform-type choice are omitted. + */ + toCalibrationJson(): string { + const keys = new Set([ + ...Object.keys(this.correspondences.value).filter( + (key) => this.correspondences.value[key].length > 0, + ), + ...Object.keys(this.homographies.value), + ]); + const pairs: CalibrationFilePair[] = [...keys].sort().map((key) => { + const [left, right] = key.split('::'); + const homography = this.homographies.value[key] || null; + return { + left, + right, + points: (this.correspondences.value[key] || []).map((c) => [c.a[0], c.a[1], c.b[0], c.b[1]]), + leftToRight: homography ? homography.AtoB : null, + rightToLeft: homography ? homography.BtoA : null, + transformType: this.transformTypeForPair(key), + }; + }); + const file: CalibrationFile = { + type: CALIBRATION_FILE_TYPE, + version: 1, + ...(this.source.value ? { source: this.source.value } : {}), + pairs, + }; + return JSON.stringify(file, null, 2); + } + + /** + * Parse and load a calibration JSON file (the format written by + * {@link toCalibrationJson}), REPLACING all pairs' correspondences, + * homographies, and transform types. Throws a descriptive Error on + * malformed input without touching current state. Returns the camera names + * referenced by the file so callers can warn about ones missing from the + * loaded dataset. + */ + loadCalibrationText(text: string): { cameras: string[]; pairCount: number } { + let data: unknown; + try { + data = JSON.parse(text); + } catch { + throw new Error('File is not valid JSON'); + } + const file = data as Partial; + if (!Array.isArray(file?.pairs)) { + throw new Error('Not a DIVE camera calibration file (expected a "pairs" list)'); + } + const source = CameraCalibrationStore.readSource(file.source); + const correspondences: CameraCorrespondences = {}; + const homographies: CameraHomographies = {}; + const transformTypes: CameraTransformTypes = {}; + const cameras = new Set(); + file.pairs.forEach((pair, i) => { + const context = `Pair ${i + 1}`; + if (typeof pair?.left !== 'string' || typeof pair?.right !== 'string' + || !pair.left || !pair.right || pair.left === pair.right) { + throw new Error(`${context}: "left" and "right" must be two distinct camera names`); + } + const key = this.pairKey(pair.left, pair.right); + cameras.add(pair.left); + cameras.add(pair.right); + if (pair.transformType !== undefined) { + if (!TRANSFORM_TYPES.some((t) => t.value === pair.transformType)) { + throw new Error( + `${context}: unknown transformType "${pair.transformType}" (expected one of ${TRANSFORM_TYPES.map((t) => t.value).join(', ')})`, + ); + } + transformTypes[key] = pair.transformType; + } + correspondences[key] = (pair.points || []).map((row, j) => { + const [ax, ay, bx, by] = CameraCalibrationStore.readPointsRow(row, `${context}, points row ${j + 1}`); + // eslint-disable-next-line no-plusplus + return { id: this.nextId++, a: [ax, ay] as Point, b: [bx, by] as Point }; + }); + const leftToRight = (pair.leftToRight === null || pair.leftToRight === undefined) + ? null + : CameraCalibrationStore.readMatrix(pair.leftToRight, `${context}, leftToRight`); + const rightToLeft = (pair.rightToLeft === null || pair.rightToLeft === undefined) + ? null + : CameraCalibrationStore.readMatrix(pair.rightToLeft, `${context}, rightToLeft`); + if (leftToRight || rightToLeft) { + // If only one direction is present, derive the other by inversion + // (readMatrix guarantees invertibility). + homographies[key] = { + AtoB: leftToRight ?? invert3(rightToLeft as Matrix3), + BtoA: rightToLeft ?? invert3(leftToRight as Matrix3), + }; + } + }); + this.correspondences.value = correspondences; + this.homographies.value = homographies; + this.transformTypes.value = transformTypes; + this.source.value = source; + this.markHomographySources(); + return { cameras: [...cameras], pairCount: file.pairs.length }; + } + + /** Validate an untrusted `source` value: a plain object, or absent (-> null). */ + private static readSource(raw: unknown): CalibrationSource | null { + if (raw === undefined || raw === null) { + return null; + } + if (typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error('"source" must be an object when present'); + } + return raw as CalibrationSource; + } + + /** Validate an untrusted value as a 4-element finite [leftX, leftY, rightX, rightY] row. */ + private static readPointsRow(raw: unknown, context: string): [number, number, number, number] { + if (!Array.isArray(raw) || raw.length !== 4) { + throw new Error(`${context}: expected [leftX, leftY, rightX, rightY]`); + } + const nums = raw.map(Number); + if (nums.some((n) => !Number.isFinite(n))) { + throw new Error(`${context}: expected [leftX, leftY, rightX, rightY] with finite numbers`); + } + return [nums[0], nums[1], nums[2], nums[3]]; + } + + /** Validate an untrusted value as an invertible row-major 3x3 matrix. */ + private static readMatrix(raw: unknown, context: string): Matrix3 { + if (!Array.isArray(raw) || raw.length !== 3 + || raw.some((row) => !Array.isArray(row) || row.length !== 3)) { + throw new Error(`${context}: expected a 3x3 matrix`); + } + const m = (raw as unknown[][]).map((row) => row.map(Number)); + if (m.some((row) => row.some((n) => !Number.isFinite(n)))) { + throw new Error(`${context}: matrix entries must be finite numbers`); + } + try { + invert3(m); + } catch { + throw new Error(`${context}: matrix is singular (not invertible)`); + } + return m; + } + + /** + * Reset homography provenance after bulk-loading state: a homography whose + * pair lacks enough points for its transform type can only have come from a + * file ('loaded', so refit checks preserve it); one with enough points is + * treated as fitted from them. + */ + private markHomographySources() { + this.homographySources = {}; + Object.keys(this.homographies.value).forEach((key) => { + const count = (this.correspondences.value[key] || []).length; + const required = minPointsForTransform(this.transformTypeForPair(key)); + this.homographySources[key] = count >= required ? 'fit' : 'loaded'; + }); + } + + /** Reset state and load saved homographies, correspondences, transform type choices, and provenance. */ + hydrate( + homographies?: CameraHomographies, + correspondences?: CameraCorrespondences, + transformTypes?: CameraTransformTypes, + source?: CalibrationSource | null, + ) { + this.homographies.value = homographies ? { ...homographies } : {}; + this.correspondences.value = correspondences ? { ...correspondences } : {}; + this.transformTypes.value = transformTypes ? { ...transformTypes } : {}; + this.source.value = source ?? null; + this.markHomographySources(); + // Resume id allocation past any restored correspondences. + let maxId = 0; + Object.values(this.correspondences.value).forEach((list) => { + list.forEach((c) => { maxId = Math.max(maxId, c.id); }); + }); + this.nextId = maxId + 1; + // The freshly loaded state is the saved baseline. + this.markSaved(); + } +} diff --git a/client/src/homography.spec.ts b/client/src/homography.spec.ts new file mode 100644 index 000000000..9facf0512 --- /dev/null +++ b/client/src/homography.spec.ts @@ -0,0 +1,258 @@ +/// +import { + solveHomography, + applyHomography, + invert3, + matMul3, + subdivideWarpQuads, + warpGridSize, + localLinkedScale, + Point, + Matrix3, +} from './homography'; + +function expectMatrixClose(actual: Matrix3, expected: Matrix3, tol = 1e-6) { + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + expect(actual[i][j]).toBeCloseTo(expected[i][j], 5); + // tol referenced to keep signature explicit + expect(Math.abs(actual[i][j] - expected[i][j])).toBeLessThan(tol * 10); + } + } +} + +const unitSquare: Point[] = [[0, 0], [1, 0], [1, 1], [0, 1]]; + +describe('homography', () => { + it('recovers the identity from identical correspondences', () => { + const H = solveHomography(unitSquare, unitSquare); + expectMatrixClose(H, [[1, 0, 0], [0, 1, 0], [0, 0, 1]]); + }); + + it('recovers a pure translation', () => { + const dst = unitSquare.map(([x, y]): Point => [x + 5, y - 3]); + const H = solveHomography(unitSquare, dst); + expectMatrixClose(H, [[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); + + it('recovers a scale + translation', () => { + const H = solveHomography(unitSquare, [[10, 10], [30, 10], [30, 30], [10, 30]]); + expectMatrixClose(H, [[20, 0, 10], [0, 20, 10], [0, 0, 1]]); + }); + + it('maps source points onto destination points (least-squares, >4 pts)', () => { + // A known projective transform applied to 5 points; solver should recover it. + const truth: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + const src: Point[] = [[0, 0], [100, 0], [100, 80], [0, 80], [50, 40]]; + const dst = src.map((p) => applyHomography(truth, p)); + const H = solveHomography(src, dst); + src.forEach((p) => { + const [u, v] = applyHomography(H, p); + const [eu, ev] = applyHomography(truth, p); + expect(u).toBeCloseTo(eu, 3); + expect(v).toBeCloseTo(ev, 3); + }); + }); + + it('round-trips through its inverse (H * H^-1 ~= I)', () => { + const H = solveHomography(unitSquare, [[10, 5], [40, 8], [38, 35], [9, 33]]); + const product = matMul3(H, invert3(H)); + const scale = 1 / product[2][2]; + const normalized = product.map((r) => r.map((c) => c * scale)) as Matrix3; + expectMatrixClose(normalized, [[1, 0, 0], [0, 1, 0], [0, 0, 1]]); + }); + + it('throws with fewer than 4 correspondences', () => { + expect(() => solveHomography(unitSquare.slice(0, 3), unitSquare.slice(0, 3))).toThrow(); + }); + + it('throws on a degenerate (collinear) point configuration despite having 4 points', () => { + const collinear: Point[] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + expect(() => solveHomography(collinear, collinear)).toThrow(/degenerate/i); + }); +}); + +describe('warpGridSize', () => { + const affine: Matrix3 = [[1.5, 0.2, 30], [-0.1, 0.9, -12], [0, 0, 1]]; + const projective: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + + it('returns 1 for a pure affine transform', () => { + expect(warpGridSize(affine, 640, 480)).toBe(1); + }); + + it('returns maxN when perspective terms are non-negligible', () => { + expect(warpGridSize(projective, 640, 480)).toBe(8); + expect(warpGridSize(projective, 640, 480, 12)).toBe(12); + }); + + it('returns 1 for negligible perspective terms', () => { + // Perspective terms exist but vary w by only ~0.006% over the extent. + const nearAffine: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [1e-7, 0, 1]]; + expect(warpGridSize(nearAffine, 640, 480)).toBe(1); + }); + + it('returns maxN when the horizon crosses the image (w changes sign)', () => { + const extreme: Matrix3 = [[1, 0, 0], [0, 1, 0], [-0.01, 0, 1]]; + // w at x=0 is 1, at x=640 is -5.4. + expect(warpGridSize(extreme, 640, 480)).toBe(8); + }); +}); + +describe('subdivideWarpQuads', () => { + const projective: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + + it('with n=1 produces a single quad matching the full image corners', () => { + const [quad, ...rest] = subdivideWarpQuads(projective, 640, 480, 1); + expect(rest).toHaveLength(0); + expect(quad.crop).toEqual({ + left: 0, top: 0, right: 640, bottom: 480, + }); + expect(quad.ul).toEqual(applyHomography(projective, [0, 0])); + expect(quad.ur).toEqual(applyHomography(projective, [640, 0])); + expect(quad.lr).toEqual(applyHomography(projective, [640, 480])); + expect(quad.ll).toEqual(applyHomography(projective, [0, 480])); + }); + + it('maps every sub-quad corner through the exact homography', () => { + const n = 8; + const quads = subdivideWarpQuads(projective, 640, 480, n); + expect(quads).toHaveLength(n * n); + quads.forEach((q) => { + expect(q.ul).toEqual(applyHomography(projective, [q.crop.left, q.crop.top])); + expect(q.ur).toEqual(applyHomography(projective, [q.crop.right, q.crop.top])); + expect(q.lr).toEqual(applyHomography(projective, [q.crop.right, q.crop.bottom])); + expect(q.ll).toEqual(applyHomography(projective, [q.crop.left, q.crop.bottom])); + }); + }); + + it('expands cells by the overlap (clamped to the image), corners still exact', () => { + const n = 4; + const overlap = 2; + const plain = subdivideWarpQuads(projective, 640, 480, n); + const padded = subdivideWarpQuads(projective, 640, 480, n, overlap); + expect(padded).toHaveLength(plain.length); + padded.forEach((q, i) => { + const base = plain[i].crop; + expect(q.crop.left).toBe(Math.max(0, base.left - overlap)); + expect(q.crop.right).toBe(Math.min(640, base.right + overlap)); + expect(q.crop.top).toBe(Math.max(0, base.top - overlap)); + expect(q.crop.bottom).toBe(Math.min(480, base.bottom + overlap)); + // Corners remain the exact projective mapping of the (expanded) crop, + // so overlapping regions of adjacent cells render identical content. + expect(q.ul).toEqual(applyHomography(projective, [q.crop.left, q.crop.top])); + expect(q.lr).toEqual(applyHomography(projective, [q.crop.right, q.crop.bottom])); + }); + // Interior edges overlap: cell 0's right crop passes cell 1's left crop. + expect(padded[0].crop.right).toBeGreaterThan(padded[1].crop.left); + }); + + it('tiles the source image exactly, with integer grid lines and no gaps', () => { + const n = 8; + const width = 641; // not divisible by n + const height = 479; + const quads = subdivideWarpQuads(projective, width, height, n); + const lefts = new Set(quads.map((q) => q.crop.left)); + const tops = new Set(quads.map((q) => q.crop.top)); + expect(lefts.size).toBe(n); + expect(tops.size).toBe(n); + quads.forEach((q) => { + [q.crop.left, q.crop.top, q.crop.right, q.crop.bottom].forEach((v) => { + expect(Number.isInteger(v)).toBe(true); + }); + expect(q.crop.right).toBeGreaterThan(q.crop.left); + expect(q.crop.bottom).toBeGreaterThan(q.crop.top); + // Each cell's right/bottom edge is another cell's left/top edge or the border. + expect(q.crop.right === width || lefts.has(q.crop.right)).toBe(true); + expect(q.crop.bottom === height || tops.has(q.crop.bottom)).toBe(true); + }); + // Crop areas sum to the full image area (exact tiling, no overlap/gap). + const area = quads.reduce( + (sum, q) => sum + (q.crop.right - q.crop.left) * (q.crop.bottom - q.crop.top), + 0, + ); + expect(area).toBe(width * height); + }); + + it('skips degenerate zero-area cells for images smaller than the grid', () => { + const quads = subdivideWarpQuads(projective, 3, 3, 8); + expect(quads.length).toBeGreaterThan(0); + quads.forEach((q) => { + expect(q.crop.right).toBeGreaterThan(q.crop.left); + expect(q.crop.bottom).toBeGreaterThan(q.crop.top); + }); + const area = quads.reduce( + (sum, q) => sum + (q.crop.right - q.crop.left) * (q.crop.bottom - q.crop.top), + 0, + ); + expect(area).toBe(9); + }); + + it('sub-quad centers stay close to the true projective warp (approximation quality)', () => { + // The deviation between the piecewise-affine render and the true warp is + // largest in cell interiors; measure it at the center of every cell as the + // distance between the true projective warp of the cell's source center + // and the average of the four warped corners (what an affine-ish renderer + // produces there). + const width = 640; + const height = 480; + const centerError = (quads: ReturnType) => Math.max( + ...quads.map((q) => { + const midSrc: Point = [ + (q.crop.left + q.crop.right) / 2, + (q.crop.top + q.crop.bottom) / 2, + ]; + const truth = applyHomography(projective, midSrc); + const approx: Point = [ + (q.ul[0] + q.ur[0] + q.lr[0] + q.ll[0]) / 4, + (q.ul[1] + q.ur[1] + q.lr[1] + q.ll[1]) / 4, + ]; + return Math.hypot(approx[0] - truth[0], approx[1] - truth[1]); + }), + ); + const singleQuadError = centerError(subdivideWarpQuads(projective, width, height, 1)); + const gridError = centerError(subdivideWarpQuads(projective, width, height, 8)); + // This matrix has strong perspective: a single (parallelogram-rendered) + // quad is tens of pixels off at the image center. + expect(singleQuadError).toBeGreaterThan(10); + // Error shrinks roughly with 1/n^2; at n=8 even this extreme perspective + // (w varies ~2x across the image) is down to a couple of pixels. + expect(gridError).toBeLessThan(3); + expect(gridError).toBeLessThan(singleQuadError / 20); + }); +}); + +describe('localLinkedScale', () => { + const mapper = (matrix: Matrix3) => (p: Point) => applyHomography(matrix, p); + + it('returns 1 for the identity mapping', () => { + const identity: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + expect(localLinkedScale(mapper(identity), [100, 200])).toBeCloseTo(1); + }); + + it('recovers a uniform similarity scale regardless of rotation', () => { + const s = 2.5; + const cos = Math.cos(Math.PI / 6) * s; + const sin = Math.sin(Math.PI / 6) * s; + const similarity: Matrix3 = [[cos, -sin, 10], [sin, cos, -4], [0, 0, 1]]; + expect(localLinkedScale(mapper(similarity), [50, 75])).toBeCloseTo(s); + }); + + it('samples the local scale of a projective transform at the given point', () => { + const homography: Matrix3 = [[1, 0, 0], [0, 1, 0], [0.001, 0, 1]]; + const nearOrigin = localLinkedScale(mapper(homography), [0, 0], 1); + const farRight = localLinkedScale(mapper(homography), [500, 0], 1); + expect(nearOrigin).toBeCloseTo(1, 1); + // At x=500 the perspective divide (w = 1.5) has shrunk the local scale + // well below 1; exact value differs per axis, so just assert the shrink. + expect(farRight).not.toBeNull(); + expect(farRight as number).toBeLessThan(0.7); + }); + + it('returns null when the mapping is unavailable', () => { + expect(localLinkedScale(() => null, [10, 10])).toBeNull(); + }); + + it('returns null for a degenerate (collapsing) mapping', () => { + expect(localLinkedScale(() => [3, 3], [10, 10])).toBeNull(); + }); +}); diff --git a/client/src/homography.ts b/client/src/homography.ts new file mode 100644 index 000000000..ff09188a7 --- /dev/null +++ b/client/src/homography.ts @@ -0,0 +1,309 @@ +/** + * Self-contained homography estimation via the normalized Direct Linear Transform. + * + * Given >= 4 point correspondences src[i] -> dst[i] (both [x, y] in image + * coordinates), {@link solveHomography} returns the 3x3 matrix H such that, in + * homogeneous coordinates, dst ~= H * src. Exact for 4 points, least-squares for + * more. This is the client-side analogue of OpenCV's cv2.findHomography used by + * the keypointgui reference app; the warp itself is done by geojs (quadFeature). + */ + +export type Point = [number, number]; +export type Matrix3 = number[][]; + +/** Multiply two 3x3 matrices. */ +export function matMul3(a: Matrix3, b: Matrix3): Matrix3 { + const out: Matrix3 = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]; + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + let sum = 0; + for (let k = 0; k < 3; k += 1) { + sum += a[i][k] * b[k][j]; + } + out[i][j] = sum; + } + } + return out; +} + +/** Invert a 3x3 matrix. Throws if the matrix is singular. */ +export function invert3(m: Matrix3): Matrix3 { + const [a, b, c] = m[0]; + const [d, e, f] = m[1]; + const [g, h, i] = m[2]; + const A = e * i - f * h; + const B = -(d * i - f * g); + const C = d * h - e * g; + const det = a * A + b * B + c * C; + if (Math.abs(det) < 1e-12) { + throw new Error('Cannot invert singular matrix'); + } + const invDet = 1 / det; + return [ + [A * invDet, -(b * i - c * h) * invDet, (b * f - c * e) * invDet], + [B * invDet, (a * i - c * g) * invDet, -(a * f - c * d) * invDet], + [C * invDet, -(a * h - b * g) * invDet, (a * e - b * d) * invDet], + ]; +} + +/** Apply a 3x3 homography to a single point (perspective divide). */ +export function applyHomography(h: Matrix3, p: Point): Point { + const x = h[0][0] * p[0] + h[0][1] * p[1] + h[0][2]; + const y = h[1][0] * p[0] + h[1][1] * p[1] + h[1][2]; + const w = h[2][0] * p[0] + h[2][1] * p[1] + h[2][2]; + return [x / w, y / w]; +} + +/** + * Local scale factor of a point mapping around `center`: how many target-image + * pixels one source-image pixel spans there, estimated by probing `delta` + * pixels along each axis and averaging. For similarity/affine transforms this + * is constant; for homographies it varies with position, which is why it's + * sampled at a specific point (e.g. the current view center for linked + * pan/zoom). Returns null when the mapping is unavailable or degenerate at + * that point. + */ +export function localLinkedScale( + mapPoint: (p: Point) => Point | null, + center: Point, + delta = 10, +): number | null { + const mapped = mapPoint(center); + const mappedX = mapPoint([center[0] + delta, center[1]]); + const mappedY = mapPoint([center[0], center[1] + delta]); + if (!mapped || !mappedX || !mappedY) { + return null; + } + const scaleX = Math.hypot(mappedX[0] - mapped[0], mappedX[1] - mapped[1]) / delta; + const scaleY = Math.hypot(mappedY[0] - mapped[0], mappedY[1] - mapped[1]) / delta; + const scale = (scaleX + scaleY) / 2; + if (!Number.isFinite(scale) || scale <= 0) { + return null; + } + return scale; +} + +/** + * One cell of a subdivided image warp: the axis-aligned source-image rectangle + * `crop` (in source pixels) and the four destination corners it maps to under + * the exact projective transform. See {@link subdivideWarpQuads}. + */ +export interface WarpQuad { + ul: Point; + ur: Point; + lr: Point; + ll: Point; + crop: { left: number; top: number; right: number; bottom: number }; +} + +/** + * Choose a subdivision grid size for rendering the warp of a `width` x `height` + * image through `h` with an affine-only quad renderer (e.g. geojs' canvas + * renderer, which draws each quad from only three of its corners). A pure + * affine matrix (zero perspective row terms) warps exactly as a single + * parallelogram, so 1 is returned; when the perspective terms are + * non-negligible over the image extent, `maxN` is returned so each sub-quad is + * approximately affine. + */ +export function warpGridSize(h: Matrix3, width: number, height: number, maxN = 8): number { + // Homogeneous w at each image corner: constant w <=> affine transform. + const wAt = (x: number, y: number) => h[2][0] * x + h[2][1] * y + h[2][2]; + const ws = [wAt(0, 0), wAt(width, 0), wAt(width, height), wAt(0, height)]; + if (ws.some((w) => !Number.isFinite(w) || w === 0)) { + return maxN; + } + const absW = ws.map((w) => Math.abs(w)); + const maxAbs = Math.max(...absW); + const minAbs = Math.min(...absW); + // Sign change means the horizon line crosses the image: definitely projective. + if (ws.some((w) => w * ws[0] < 0)) { + return maxN; + } + // Relative variation of w across the quad; below ~0.1% the affine + // approximation is visually indistinguishable (sub-pixel for typical sizes). + return (maxAbs - minAbs) / maxAbs < 1e-3 ? 1 : maxN; +} + +/** + * Subdivide the warp of a `width` x `height` image through `h` into an n x n + * grid of {@link WarpQuad}s. Every sub-quad corner is mapped through the exact + * projective transform, so rendering each cell as an (approximately affine) + * textured quad converges to the true projective warp as n grows -- unlike + * rendering the whole image as a single quad, which an affine canvas renderer + * collapses to a parallelogram. Grid lines land on integer source pixels; + * degenerate (zero-area) cells from tiny images are skipped. + * + * `overlap` expands each cell by that many source pixels (clamped to the + * image). The canvas renderer antialiases every quad's border against the + * transparent background, so abutting cells meet as two half-transparent + * edges and show as dark seam lines along the grid; overlapping cells paint + * over each other's seams with pixel-identical content (corners still map + * through the same exact homography). Only for opaque drawing -- translucent + * quads would double-blend in the overlap, so semi-transparent consumers + * must apply opacity at the layer level and draw quads opaque. + */ +export function subdivideWarpQuads( + h: Matrix3, + width: number, + height: number, + n: number, + overlap = 0, +): WarpQuad[] { + const cells = Math.max(1, Math.floor(n)); + const xs: number[] = []; + const ys: number[] = []; + for (let i = 0; i <= cells; i += 1) { + xs.push(Math.round((i * width) / cells)); + ys.push(Math.round((i * height) / cells)); + } + const quads: WarpQuad[] = []; + for (let row = 0; row < cells; row += 1) { + for (let col = 0; col < cells; col += 1) { + const left = Math.max(0, xs[col] - overlap); + const right = Math.min(width, xs[col + 1] + overlap); + const top = Math.max(0, ys[row] - overlap); + const bottom = Math.min(height, ys[row + 1] + overlap); + if (right <= left || bottom <= top) { + // eslint-disable-next-line no-continue + continue; + } + quads.push({ + ul: applyHomography(h, [left, top]), + ur: applyHomography(h, [right, top]), + lr: applyHomography(h, [right, bottom]), + ll: applyHomography(h, [left, bottom]), + crop: { + left, top, right, bottom, + }, + }); + } + } + return quads; +} + +/** + * Hartley normalization: translate points to the centroid and scale so the + * mean distance from the origin is sqrt(2). Returns the normalized points and + * the 3x3 transform T such that normalized = T * original. + */ +function normalizePoints(pts: Point[]): { normalized: Point[]; transform: Matrix3 } { + const n = pts.length; + let cx = 0; + let cy = 0; + pts.forEach(([x, y]) => { cx += x; cy += y; }); + cx /= n; + cy /= n; + let meanDist = 0; + pts.forEach(([x, y]) => { meanDist += Math.hypot(x - cx, y - cy); }); + meanDist /= n; + const scale = meanDist > 1e-12 ? Math.SQRT2 / meanDist : 1; + const transform: Matrix3 = [ + [scale, 0, -scale * cx], + [0, scale, -scale * cy], + [0, 0, 1], + ]; + const normalized = pts.map(([x, y]): Point => [(x - cx) * scale, (y - cy) * scale]); + return { normalized, transform }; +} + +/** + * Solve the linear system A x = b for x using Gaussian elimination with partial + * pivoting. A is square (n x n), modified in place. + */ +/* eslint-disable no-param-reassign */ +export function solveLinearSystem(A: number[][], b: number[]): number[] { + const n = b.length; + for (let col = 0; col < n; col += 1) { + // Partial pivot: find the row with the largest magnitude in this column. + let pivot = col; + for (let row = col + 1; row < n; row += 1) { + if (Math.abs(A[row][col]) > Math.abs(A[pivot][col])) { + pivot = row; + } + } + if (Math.abs(A[pivot][col]) < 1e-12) { + throw new Error('Degenerate point configuration; cannot solve linear system'); + } + if (pivot !== col) { + [A[col], A[pivot]] = [A[pivot], A[col]]; + [b[col], b[pivot]] = [b[pivot], b[col]]; + } + // Eliminate below. + for (let row = col + 1; row < n; row += 1) { + const factor = A[row][col] / A[col][col]; + for (let k = col; k < n; k += 1) { + A[row][k] -= factor * A[col][k]; + } + b[row] -= factor * b[col]; + } + } + // Back-substitution. + const x = new Array(n).fill(0); + for (let row = n - 1; row >= 0; row -= 1) { + let sum = b[row]; + for (let k = row + 1; k < n; k += 1) { + sum -= A[row][k] * x[k]; + } + x[row] = sum / A[row][row]; + } + return x; +} +/* eslint-enable no-param-reassign */ + +/** + * Estimate the homography H (dst ~= H * src) from >= 4 correspondences. + * + * Uses the h33 = 1 formulation in Hartley-normalized coordinates, solved via the + * normal equations (least-squares for > 4 points, exact for 4). Normalization + * keeps the system well-conditioned for typical image-pixel magnitudes. + */ +export function solveHomography(src: Point[], dst: Point[]): Matrix3 { + if (src.length !== dst.length) { + throw new Error('src and dst must have the same number of points'); + } + if (src.length < 4) { + throw new Error('At least 4 point correspondences are required'); + } + + const { normalized: srcN, transform: T1 } = normalizePoints(src); + const { normalized: dstN, transform: T2 } = normalizePoints(dst); + + // Build the 2N x 8 design matrix for the unknowns [h11..h32] (h33 fixed to 1). + const rows: number[][] = []; + const rhs: number[] = []; + for (let i = 0; i < srcN.length; i += 1) { + const [x, y] = srcN[i]; + const [u, v] = dstN[i]; + rows.push([x, y, 1, 0, 0, 0, -x * u, -y * u]); + rhs.push(u); + rows.push([0, 0, 0, x, y, 1, -x * v, -y * v]); + rhs.push(v); + } + + // Normal equations: (A^T A) h = A^T b -> an 8x8 system. + const ata: number[][] = Array.from({ length: 8 }, () => new Array(8).fill(0)); + const atb: number[] = new Array(8).fill(0); + for (let r = 0; r < rows.length; r += 1) { + const row = rows[r]; + for (let i = 0; i < 8; i += 1) { + atb[i] += row[i] * rhs[r]; + for (let j = 0; j < 8; j += 1) { + ata[i][j] += row[i] * row[j]; + } + } + } + + const h = solveLinearSystem(ata, atb); + const hNorm: Matrix3 = [ + [h[0], h[1], h[2]], + [h[3], h[4], h[5]], + [h[6], h[7], 1], + ]; + + // Denormalize: H = inv(T2) * Hnorm * T1. + const denorm = matMul3(matMul3(invert3(T2), hNorm), T1); + + // Scale so H[2][2] == 1 for a canonical form. + const scale = Math.abs(denorm[2][2]) > 1e-12 ? 1 / denorm[2][2] : 1; + return denorm.map((row) => row.map((value) => value * scale)); +} diff --git a/client/src/transform.spec.ts b/client/src/transform.spec.ts new file mode 100644 index 000000000..ba8d0ab2c --- /dev/null +++ b/client/src/transform.spec.ts @@ -0,0 +1,163 @@ +/// +import { applyHomography, solveHomography, Matrix3 } from './homography'; +import { + TransformType, + minPointsForTransform, + estimateTranslation, + estimateRigid, + estimateSimilarity, + estimateAffine, + estimateTransform, + Point, +} from './transform'; + +function expectMatrixClose(actual: Matrix3, expected: Matrix3, precision = 4) { + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + expect(actual[i][j]).toBeCloseTo(expected[i][j], precision); + } + } +} + +function expectRoundTrip(h: Matrix3, src: Point[], dst: Point[], precision = 4) { + src.forEach((p, i) => { + const [x, y] = applyHomography(h, p); + expect(x).toBeCloseTo(dst[i][0], precision); + expect(y).toBeCloseTo(dst[i][1], precision); + }); +} + +const fixtureSrc: Point[] = [[0, 0], [10, 0], [10, 10], [0, 10], [5, 3]]; + +describe('minPointsForTransform', () => { + it('returns the keypointgui-matching minimum for each type', () => { + const expected: Record = { + translation: 1, rigid: 2, similarity: 2, affine: 3, homography: 4, + }; + (Object.keys(expected) as TransformType[]).forEach((type) => { + expect(minPointsForTransform(type)).toBe(expected[type]); + }); + }); +}); + +describe('estimateTranslation', () => { + it('recovers an exact translation from a single point', () => { + const H = estimateTranslation([[3, 4]], [[8, 1]]); + expectMatrixClose(H, [[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); + + it('recovers the mean translation from multiple points', () => { + const dst = fixtureSrc.map(([x, y]): Point => [x + 5, y - 3]); + const H = estimateTranslation(fixtureSrc, dst); + expectMatrixClose(H, [[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); +}); + +describe('estimateRigid', () => { + it('recovers a known rotation + translation from 2 points', () => { + const theta = Math.PI / 6; + const cos = Math.cos(theta); + const sin = Math.sin(theta); + const truth: Matrix3 = [[cos, -sin, 5], [sin, cos, -3], [0, 0, 1]]; + const src = fixtureSrc.slice(0, 2); + const dst = src.map((p) => applyHomography(truth, p)); + const H = estimateRigid(src, dst); + expectMatrixClose(H, truth); + // 2x2 block is a proper rotation (unit determinant, no scale/reflection). + const det = H[0][0] * H[1][1] - H[0][1] * H[1][0]; + expect(det).toBeCloseTo(1, 5); + }); + + it('least-squares fits a rigid transform from more than 2 noisy points', () => { + const theta = Math.PI / 4; + const cos = Math.cos(theta); + const sin = Math.sin(theta); + const truth: Matrix3 = [[cos, -sin, 2], [sin, cos, 7], [0, 0, 1]]; + const dst = fixtureSrc.map((p) => applyHomography(truth, p)); + const H = estimateRigid(fixtureSrc, dst); + expectRoundTrip(H, fixtureSrc, dst); + }); +}); + +describe('estimateSimilarity', () => { + it('recovers a known rotation + uniform scale + translation from 2 points', () => { + const theta = Math.PI / 3; + const scale = 2.5; + const cos = Math.cos(theta) * scale; + const sin = Math.sin(theta) * scale; + const truth: Matrix3 = [[cos, -sin, 4], [sin, cos, -6], [0, 0, 1]]; + const src = fixtureSrc.slice(0, 2); + const dst = src.map((p) => applyHomography(truth, p)); + const H = estimateSimilarity(src, dst); + expectMatrixClose(H, truth); + }); + + it('least-squares fits a similarity transform from more points', () => { + const theta = -Math.PI / 5; + const scale = 0.75; + const cos = Math.cos(theta) * scale; + const sin = Math.sin(theta) * scale; + const truth: Matrix3 = [[cos, -sin, -1], [sin, cos, 3], [0, 0, 1]]; + const dst = fixtureSrc.map((p) => applyHomography(truth, p)); + const H = estimateSimilarity(fixtureSrc, dst); + expectRoundTrip(H, fixtureSrc, dst); + }); +}); + +describe('estimateAffine', () => { + const truth: Matrix3 = [[1.2, 0.3, 5], [0.1, 0.9, -4], [0, 0, 1]]; + + it('recovers a known affine transform exactly from 3 non-collinear points', () => { + const src = fixtureSrc.slice(0, 3); + const dst = src.map((p) => applyHomography(truth, p)); + const H = estimateAffine(src, dst); + expectMatrixClose(H, truth); + }); + + it('recovers the same affine transform from more than 3 points', () => { + const dst = fixtureSrc.map((p) => applyHomography(truth, p)); + const H = estimateAffine(fixtureSrc, dst); + expectMatrixClose(H, truth); + }); +}); + +describe('estimateTransform', () => { + it('delegates to solveHomography for the homography type', () => { + const truth: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + const src: Point[] = [[0, 0], [100, 0], [100, 80], [0, 80], [50, 40]]; + const dst = src.map((p) => applyHomography(truth, p)); + expectMatrixClose(estimateTransform('homography', src, dst), solveHomography(src, dst)); + }); + + it('throws below the minimum point count for each type', () => { + const one: Point[] = [[0, 0]]; + const two: Point[] = [[0, 0], [1, 0]]; + expect(() => estimateTransform('translation', [], [])).toThrow(); + expect(() => estimateTransform('rigid', one, one)).toThrow(); + expect(() => estimateTransform('similarity', one, one)).toThrow(); + expect(() => estimateTransform('affine', two, two)).toThrow(); + expect(() => estimateTransform('homography', fixtureSrc.slice(0, 3), fixtureSrc.slice(0, 3))).toThrow(); + }); + + it('throws when src and dst lengths differ', () => { + expect(() => estimateTransform('translation', [[0, 0]], [])).toThrow(); + }); + + it('round-trips src -> dst for every transform type', () => { + const cases: { type: TransformType; truth: Matrix3 }[] = [ + { type: 'translation', truth: [[1, 0, 5], [0, 1, -3], [0, 0, 1]] }, + { type: 'rigid', truth: [[Math.cos(0.4), -Math.sin(0.4), 1], [Math.sin(0.4), Math.cos(0.4), 2], [0, 0, 1]] }, + { + type: 'similarity', + truth: [[1.5 * Math.cos(0.2), -1.5 * Math.sin(0.2), 3], [1.5 * Math.sin(0.2), 1.5 * Math.cos(0.2), 4], [0, 0, 1]], + }, + { type: 'affine', truth: [[1.1, 0.2, 2], [-0.1, 0.95, 6], [0, 0, 1]] }, + { type: 'homography', truth: [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]] }, + ]; + cases.forEach(({ type, truth }) => { + const dst = fixtureSrc.map((p) => applyHomography(truth, p)); + const H = estimateTransform(type, fixtureSrc, dst); + expectRoundTrip(H, fixtureSrc, dst, 3); + }); + }); +}); diff --git a/client/src/transform.ts b/client/src/transform.ts new file mode 100644 index 000000000..34461b272 --- /dev/null +++ b/client/src/transform.ts @@ -0,0 +1,180 @@ +/** + * Alignment transform models for camera calibration, layered on top of the matrix + * primitives in {@link "./homography"} (`Matrix3`, `applyHomography`, `invert3`, + * `solveLinearSystem`). Mirrors keypointgui's `fit_homography` transform types + * (translation / rigid / similarity / affine / homography) so near-rigid EO/IR + * rigs can be fit with fewer, more stable point pairs than a full homography + * requires. Every estimator returns a plain `Matrix3`, so callers (warping, + * inverse-mapping) never need to special-case the transform type. + */ + +import { + Point, Matrix3, solveLinearSystem, solveHomography, +} from './homography'; + +export type TransformType = 'translation' | 'rigid' | 'similarity' | 'affine' | 'homography'; + +/** UI-friendly ordered list of transform types, for dropdowns. */ +export const TRANSFORM_TYPES: { value: TransformType; text: string }[] = [ + { value: 'translation', text: 'Translation' }, + { value: 'rigid', text: 'Rigid' }, + { value: 'similarity', text: 'Similarity' }, + { value: 'affine', text: 'Affine' }, + { value: 'homography', text: 'Homography' }, +]; + +/** + * The transform type assumed for a pair that has no explicit choice yet. Near- + * rigid EO/IR rigs fit stably from few points with a similarity model, so it is + * the sensible default the UI shows. MUST be the single source of this default + * everywhere a pair's type is resolved (the store, the panel, and both platform + * persistence layers), so a pair fitted at the default resolves to the same + * model after a save/reload on every platform. + */ +export const DEFAULT_TRANSFORM_TYPE: TransformType = 'similarity'; + +const MIN_POINTS: Record = { + translation: 1, + rigid: 2, + similarity: 2, + affine: 3, + homography: 4, +}; + +/** Minimum correspondence count to fit `type`, matching keypointgui's fit_homography. */ +export function minPointsForTransform(type: TransformType): number { + return MIN_POINTS[type]; +} + +/** Pure translation: H = identity with translation = mean(dst - src). Exact for 1+ points. */ +export function estimateTranslation(src: Point[], dst: Point[]): Matrix3 { + const n = src.length; + let dx = 0; + let dy = 0; + for (let i = 0; i < n; i += 1) { + dx += dst[i][0] - src[i][0]; + dy += dst[i][1] - src[i][1]; + } + dx /= n; + dy /= n; + return [[1, 0, dx], [0, 1, dy], [0, 0, 1]]; +} + +/** + * Closed-form 2D Procrustes/Umeyama fit: rotation (plus optional uniform scale) + * and translation minimizing sum |R*src + t - dst|^2. SVD-free because a 2D + * rotation is a single angle: `theta = atan2(B, A)` is the angle that maximizes + * the projected alignment sum, and (for similarity) the optimal uniform scale is + * the ratio of that alignment's magnitude to the source points' spread. + */ +function estimateRotationScaleTranslation( + src: Point[], + dst: Point[], + allowScale: boolean, +): Matrix3 { + const n = src.length; + let csx = 0; + let csy = 0; + let cdx = 0; + let cdy = 0; + for (let i = 0; i < n; i += 1) { + csx += src[i][0]; + csy += src[i][1]; + cdx += dst[i][0]; + cdy += dst[i][1]; + } + csx /= n; + csy /= n; + cdx /= n; + cdy /= n; + + let a = 0; + let b = 0; + let srcVar = 0; + for (let i = 0; i < n; i += 1) { + const sx = src[i][0] - csx; + const sy = src[i][1] - csy; + const dx = dst[i][0] - cdx; + const dy = dst[i][1] - cdy; + a += sx * dx + sy * dy; + b += sx * dy - sy * dx; + srcVar += sx * sx + sy * sy; + } + + const theta = Math.atan2(b, a); + const scale = allowScale && srcVar > 1e-12 ? Math.hypot(a, b) / srcVar : 1; + const r00 = Math.cos(theta) * scale; + const r01 = -Math.sin(theta) * scale; + const r10 = Math.sin(theta) * scale; + const r11 = Math.cos(theta) * scale; + const tx = cdx - (r00 * csx + r01 * csy); + const ty = cdy - (r10 * csx + r11 * csy); + return [[r00, r01, tx], [r10, r11, ty], [0, 0, 1]]; +} + +/** Rotation + translation only (no scale, no shear). Matches keypointgui's rigid fit. */ +export function estimateRigid(src: Point[], dst: Point[]): Matrix3 { + return estimateRotationScaleTranslation(src, dst, false); +} + +/** Rotation + uniform scale + translation. Matches keypointgui's similarity fit. */ +export function estimateSimilarity(src: Point[], dst: Point[]): Matrix3 { + return estimateRotationScaleTranslation(src, dst, true); +} + +/** + * Full 6-DOF affine fit via two independent 3-unknown least-squares solves (one + * output row at a time): dst_x = a*src_x + b*src_y + tx, dst_y = c*src_x + d*src_y + * + ty. Exact for 3 non-collinear points, least-squares for more. + */ +export function estimateAffine(src: Point[], dst: Point[]): Matrix3 { + const n = src.length; + const ata: number[][] = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]; + const atbX: number[] = [0, 0, 0]; + const atbY: number[] = [0, 0, 0]; + for (let i = 0; i < n; i += 1) { + const row = [src[i][0], src[i][1], 1]; + const dx = dst[i][0]; + const dy = dst[i][1]; + for (let r = 0; r < 3; r += 1) { + atbX[r] += row[r] * dx; + atbY[r] += row[r] * dy; + for (let c = 0; c < 3; c += 1) { + ata[r][c] += row[r] * row[c]; + } + } + } + // solveLinearSystem mutates its arguments in place, so each solve gets its own copy. + const rowX = solveLinearSystem(ata.map((row) => [...row]), [...atbX]); + const rowY = solveLinearSystem(ata.map((row) => [...row]), [...atbY]); + return [ + [rowX[0], rowX[1], rowX[2]], + [rowY[0], rowY[1], rowY[2]], + [0, 0, 1], + ]; +} + +/** Estimate a `Matrix3` mapping src -> dst using `type`, enforcing its minimum point count. */ +export function estimateTransform(type: TransformType, src: Point[], dst: Point[]): Matrix3 { + if (src.length !== dst.length) { + throw new Error('src and dst must have the same number of points'); + } + const required = minPointsForTransform(type); + if (src.length < required) { + throw new Error(`At least ${required} point pair(s) are required for a ${type} transform`); + } + switch (type) { + case 'translation': + return estimateTranslation(src, dst); + case 'rigid': + return estimateRigid(src, dst); + case 'similarity': + return estimateSimilarity(src, dst); + case 'affine': + return estimateAffine(src, dst); + case 'homography': + return solveHomography(src, dst); + default: + throw new Error(`Unknown transform type: ${type}`); + } +} From d368f0f683e20b2a820e54c3d341a565780c893f Mon Sep 17 00:00:00 2001 From: romleiaj Date: Thu, 9 Jul 2026 16:16:16 -0400 Subject: [PATCH 2/7] Load external camera calibrations into datasets Persist cameraHomographies / cameraCorrespondences / cameraTransformTypes / cameraCalibrationSource in dataset meta on both platforms (server allowlist + desktop meta merge), and load calibrations produced externally (e.g. by kamera) at import time: the multicam import dialog gains a per-camera transform-file picker, the desktop backend parses DIVE calibration .json pairs (fromCalibrationPairs / toCalibrationPairs) and auto-discovers a calibration.json in the import parent folder, and saveMetadata writes the portable calibration.json alongside the project so the calibration travels with the dataset. Co-Authored-By: Claude Fable 5 --- client/dive-common/apispec.ts | 20 +- .../ImportMultiCamChooseTransform.vue | 46 ++++ .../ImportMultiCamDialog.vue | 5 + .../ImportMultiCamMultiFolder.vue | 13 ++ .../ImportMultiCamSubfolders.vue | 14 ++ .../useImportMultiCamDialog.ts | 81 ++++++- client/dive-common/constants.ts | 9 + client/platform/desktop/backend/ipcService.ts | 5 + .../desktop/backend/native/common.spec.ts | 176 ++++++++++++++ .../platform/desktop/backend/native/common.ts | 220 ++++++++++++++++++ .../desktop/backend/native/multiCamImport.ts | 63 ++++- .../native/multiCamTransformImport.spec.ts | 126 ++++++++++ client/platform/desktop/frontend/api.ts | 15 +- .../desktop/frontend/components/Recent.vue | 1 + client/platform/web-girder/utils.ts | 8 +- server/dive_server/crud_dataset.py | 10 +- server/dive_utils/models.py | 15 ++ server/tests/test_update_metadata.py | 21 ++ 18 files changed, 834 insertions(+), 14 deletions(-) create mode 100644 client/dive-common/components/ImportMultiCamDialog/ImportMultiCamChooseTransform.vue create mode 100644 client/platform/desktop/backend/native/multiCamTransformImport.spec.ts diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index e1fd84a66..0bc77b2c0 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -8,6 +8,9 @@ import { Attribute } from 'vue-media-annotator/use/AttributeTypes'; import { CustomStyle } from 'vue-media-annotator/StyleManager'; import { AttributeTrackFilter } from 'vue-media-annotator/AttributeTrackFilterControls'; import { ImageEnhancements } from 'vue-media-annotator/use/useImageEnhancements'; +import { + CameraHomographies, CameraCorrespondences, CameraTransformTypes, CalibrationSource, +} from 'vue-media-annotator/CameraCalibrationStore'; import type { PercentileStretch } from 'vue-media-annotator/use/useImageEnhancements'; type DatasetType = 'image-sequence' | 'video' | 'multi' | 'large-image'; @@ -136,6 +139,12 @@ export interface MultiCamImportFolderArgs { sourceList: Record; // path/track file per camera @@ -186,9 +195,14 @@ interface DatasetMetaMutable { attributes?: Readonly>; attributeTrackFilters?: Readonly>; datasetInfo?: Record; + cameraHomographies?: CameraHomographies; + cameraCorrespondences?: CameraCorrespondences; + cameraTransformTypes?: CameraTransformTypes; + /** Producer provenance of the camera calibration (see CalibrationSource). */ + cameraCalibrationSource?: CalibrationSource | null; error?: string; } -const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo']; +const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences', 'cameraTransformTypes', 'cameraCalibrationSource']; interface DatasetMeta extends DatasetMetaMutable { id: Readonly; @@ -278,7 +292,7 @@ interface Api { saveAttributeTrackFilters(datasetId: string, args: SaveAttributeTrackFilterArgs): Promise; // Non-Endpoint shared functions - openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'zip', directory?: boolean): + openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'zip' | 'transform', directory?: boolean): Promise<{canceled?: boolean; filePaths: string[]; fileList?: File[]; root?: string}>; /** Desktop: immediate child directory names under a parent folder (multicam subfolder import). */ listImmediateSubfolders?(parentPath: string): Promise; @@ -294,6 +308,8 @@ interface Api { ): Promise; /** Desktop: stereoscopic calibration file in a parent folder root. */ findParentFolderCalibrationFile?(parentPath: string): Promise; + /** Desktop: DIVE camera-calibration .json (alignment transforms) in a parent folder root. */ + findParentFolderTransformFile?(parentPath: string): Promise; /** True when the dataset folder has an attached stereoscopic calibration file. */ hasCalibrationFile?(datasetId: string): Promise; /** Web: stash a calibration File for multicam upload lookup. */ diff --git a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamChooseTransform.vue b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamChooseTransform.vue new file mode 100644 index 000000000..9e1c3bd1a --- /dev/null +++ b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamChooseTransform.vue @@ -0,0 +1,46 @@ + + + + diff --git a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue index b6471e36d..9ada86978 100644 --- a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue +++ b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue @@ -42,6 +42,11 @@ export default defineComponent({ type: Boolean, default: false, }, + /** Offer per-camera calibration .json transform pickers (desktop multicam only). */ + enableTransformImport: { + type: Boolean, + default: false, + }, registerSubfolderCameras: { type: Function as PropType<(assignments: { cameraName: string; diff --git a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamMultiFolder.vue b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamMultiFolder.vue index b661f653b..d0182cb41 100644 --- a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamMultiFolder.vue +++ b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamMultiFolder.vue @@ -8,6 +8,7 @@ import { DatasetType } from 'dive-common/apispec'; import ImportMultiCamCameraGroup from './ImportMultiCamCameraGroup.vue'; import ImportMultiCamChooseSource from './ImportMultiCamChooseSource.vue'; import ImportMultiCamChooseAnnotation from './ImportMultiCamChooseAnnotation.vue'; +import ImportMultiCamChooseTransform from './ImportMultiCamChooseTransform.vue'; import ImportMultiCamAddType from './ImportMultiCamAddType.vue'; import ImportMultiCamCameraOrderControls from './ImportMultiCamCameraOrderControls.vue'; import { importMultiCamContextProp } from './importMultiCamContext'; @@ -18,6 +19,7 @@ export default defineComponent({ ImportMultiCamCameraGroup, ImportMultiCamChooseSource, ImportMultiCamChooseAnnotation, + ImportMultiCamChooseTransform, ImportMultiCamAddType, ImportMultiCamCameraOrderControls, }, @@ -43,6 +45,9 @@ export default defineComponent({ addNewSet: props.ctx.addNewSet, open: props.ctx.open, openAnnotationFile: props.ctx.openAnnotationFile, + showTransformFileField: props.ctx.showTransformFileField, + openTransformFile: props.ctx.openTransformFile, + clearTransformFile: props.ctx.clearTransformFile, }; }, }); @@ -81,6 +86,14 @@ export default defineComponent({ @clear="folderList[key].trackFile = ''" @open="openAnnotationFile(key)" /> + {{ pendingImportPayloads[key].jsonMeta.originalImageFiles.length }} files + Promise; + /** + * Offer a per-camera calibration .json transform file picker for cameras + * after the first (desktop only; the file is parsed by the desktop backend + * at import time). Ignored for stereo imports, which use calibration files. + */ + enableTransformImport?: boolean; enableSubfolderImport?: boolean; registerSubfolderCameras?: (assignments: { cameraName: string; @@ -59,11 +65,13 @@ export function useImportMultiCamDialog( listParentFolderCameras, resolveMulticamCameraSourcePath, findParentFolderCalibrationFile, + findParentFolderTransformFile, } = useApi(); const importType: Ref = ref(''); const folderList: Ref> = ref({}); const parentFolderName = ref(''); @@ -107,6 +115,12 @@ export function useImportMultiCamDialog( && !!lastCalibrationPath.value, ); + // Per-camera transform pickers: multicam only (stereo uses calibration), + // folder-based modes only, and never for the first (reference) camera. + const transformImportEnabled = computed( + () => !!props.enableTransformImport && !props.stereo, + ); + const orderedCameraKeys = computed(() => { const keys = Object.keys(folderList.value); const ordered = cameraOrder.value.filter((key) => keys.includes(key)); @@ -138,8 +152,8 @@ export function useImportMultiCamDialog( defaultDisplay.value = props.stereo ? 'left' : 'center'; if (props.stereo && importType.value === 'multi') { folderList.value = { - left: { sourcePath: '', trackFile: '' }, - right: { sourcePath: '', trackFile: '' }, + left: { sourcePath: '', trackFile: '', transformFile: '' }, + right: { sourcePath: '', trackFile: '', transformFile: '' }, }; } else { folderList.value = {}; @@ -401,6 +415,7 @@ export function useImportMultiCamDialog( Vue.set(folderList.value, cameraName, { sourcePath, trackFile: '', + transformFile: '', type: props.registerSubfolderCameras && props.dataType !== VideoType ? inferSubfolderImportType(files, { largeImageForTiff: true }) : props.dataType, @@ -414,6 +429,7 @@ export function useImportMultiCamDialog( { preferLeftForStereo: props.stereo }, ); syncDefaultDisplay(); + await discoverParentFolderTransform(parentPath); }); } @@ -440,6 +456,7 @@ export function useImportMultiCamDialog( Vue.set(folderList.value, newKey, { sourcePath: (importType.value === 'subfolders' && !listParentFolderCameras) ? newKey : sourcePath, trackFile: entry.trackFile, + transformFile: entry.transformFile, }); Vue.delete(folderList.value, oldKey); @@ -479,6 +496,7 @@ export function useImportMultiCamDialog( Vue.set(subfolderOriginalNames.value, cameraKey, displayName); folderList.value[cameraKey].sourcePath = resolvedPath; folderList.value[cameraKey].trackFile = ''; + folderList.value[cameraKey].transformFile = ''; if (props.registerSubfolderCameras && files.length) { props.registerSubfolderCameras([{ cameraName: cameraKey, @@ -503,6 +521,23 @@ export function useImportMultiCamDialog( } } + function showTransformFileField(folder: string) { + return transformImportEnabled.value + && (importType.value === 'multi' || importType.value === 'subfolders') + && orderedCameraKeys.value.indexOf(folder) > 0; + } + + async function openTransformFile(folder: string) { + const ret = await openFromDisk('transform'); + if (!ret.canceled && ret.filePaths?.length) { + [folderList.value[folder].transformFile] = ret.filePaths; + } + } + + function clearTransformFile(folder: string) { + folderList.value[folder].transformFile = ''; + } + async function open( dstype: DatasetType | 'calibration' | 'text', folder: string | 'calibration', @@ -525,6 +560,7 @@ export function useImportMultiCamDialog( folderList.value[folder].sourcePath = path; } folderList.value[folder].trackFile = ''; + folderList.value[folder].transformFile = ''; const { sourcePath } = folderList.value[folder]; if (props.registerSubfolderCameras && ret.fileList?.length) { props.registerSubfolderCameras([{ @@ -583,7 +619,7 @@ export function useImportMultiCamDialog( const addNewSet = (name: string) => { if (importType.value === 'multi') { - Vue.set(folderList.value, name, { sourcePath: '', trackFile: '' }); + Vue.set(folderList.value, name, { sourcePath: '', trackFile: '', transformFile: '' }); Vue.set(pendingImportPayloads.value, name, null); cameraOrder.value = [...cameraOrder.value, name]; } else if (importType.value === 'keyword') { @@ -601,12 +637,22 @@ export function useImportMultiCamDialog( const sourceList: MultiCamImportFolderArgs['sourceList'] = {}; orderedCameraKeys.value.forEach((key) => { if (folderList.value[key]) { - const { sourcePath, trackFile, type } = folderList.value[key]; + const { + sourcePath, trackFile, transformFile, type, + } = folderList.value[key]; + if (type === 'multi') { + // Sub Cameras shouldn't be multi types + return; + } sourceList[key] = { sourcePath, trackFile, ...(type ? { type } : {}), }; + // Transforms only apply to cameras after the first (reference) one. + if (transformFile && showTransformFileField(key)) { + sourceList[key].transformFile = transformFile; + } } }); const args: MultiCamImportFolderArgs = { @@ -670,6 +716,29 @@ export function useImportMultiCamDialog( return folderName; } + /** + * Auto-attach a DIVE camera-calibration .json found in the parent folder + * root as the import's transform file (desktop multicam subfolder imports + * only). It is attached to the first camera after the reference -- the + * file's pairs name their own cameras, so which slot carries it doesn't + * matter -- and shows in that camera's (clearable) transform field. + */ + async function discoverParentFolderTransform(parentPath: string) { + if (!transformImportEnabled.value || !findParentFolderTransformFile) { + return; + } + const discovered = await findParentFolderTransformFile(parentPath); + if (!discovered) { + return; + } + const target = orderedCameraKeys.value.find( + (key, index) => index > 0 && folderList.value[key] && !folderList.value[key].transformFile, + ); + if (target) { + folderList.value[target].transformFile = discovered; + } + } + async function discoverParentFolderCalibration( parentPath: string, fileList?: File[], @@ -737,6 +806,10 @@ export function useImportMultiCamDialog( deleteSet, onRenameCamera, openAnnotationFile, + transformImportEnabled, + showTransformFileField, + openTransformFile, + clearTransformFile, clearCalibration, }; } diff --git a/client/dive-common/constants.ts b/client/dive-common/constants.ts index 620750528..77d2600a1 100644 --- a/client/dive-common/constants.ts +++ b/client/dive-common/constants.ts @@ -165,6 +165,14 @@ const listFileTypes = [ 'txt', ]; +/** + * Per-camera alignment transform files: DIVE calibration .json (the + * calibration panel's save format). + */ +const transformFileTypes = [ + 'json', +]; + const zipFileTypes = [ 'zip', ]; @@ -212,6 +220,7 @@ export { getLargeImageAllowedExtensions, inputAnnotationFileTypes, listFileTypes, + transformFileTypes, zipFileTypes, stereoPipelineMarker, calibrationFileMarker, diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index 070d4bf44..94f3bbf24 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -203,6 +203,11 @@ export default function register() { { path }: { path: string }, ) => common.findParentFolderCalibrationFile(path)); + ipcMain.handle('find-parent-folder-transform-file', async ( + event, + { path }: { path: string }, + ) => common.findParentFolderTransformFile(path)); + ipcMain.handle('dataset-has-calibration-file', async ( event, { datasetId }: { datasetId: string }, diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 828acab62..da3c46d10 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -174,6 +174,21 @@ beforeEach(() => { }, '/home/user/testPairs': { ...fileSystemData }, '/home/user/output': {}, + '/home/user/transformDiscovery': { + exactName: { + 'aaa-stamped.json': JSON.stringify({ type: 'dive-camera-calibration', version: 1, pairs: [] }), + 'calibration.json': JSON.stringify({ type: 'dive-camera-calibration', version: 1, pairs: [] }), + }, + otherName: { + 'a-rig-calibration.json': JSON.stringify({ calibrations: {} }), + 'broken.json': '{not json', + 'z-transforms.json': JSON.stringify({ type: 'dive-camera-calibration', version: 1, pairs: [] }), + }, + none: { + 'rig.json': JSON.stringify({ some: 'thing' }), + 'notes.txt': 'not json', + }, + }, '/home/user/data': { annotationImport: { 'viame.csv': emptyCsvString, @@ -641,6 +656,167 @@ describe('native.common', () => { }); }); + it('saveMetadata writes calibration.json (pairs + points) and reloads it', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + const final = res.meta; + // Directional key: rgb is left, ir is right. + const cameraHomographies = { + 'rgb::ir': { + AtoB: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + BtoA: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }, + }; + const cameraCorrespondences = { + 'rgb::ir': [ + { id: 1, a: [10, 20], b: [12, 22] }, + { id: 2, a: [30, 40], b: [33, 44] }, + ], + }; + + await common.saveMetadata(settings, final.id, { cameraHomographies, cameraCorrespondences }); + + // Persisted as a standalone calibration.json: pairs labeled left/right, with + // points laid out as leftX leftY rightX rightY. + const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); + const calibrationPath = npath.join(projectDir, 'calibration.json'); + expect(await fs.pathExists(calibrationPath)).toBe(true); + const calibration = await fs.readJSON(calibrationPath); + expect(calibration.pairs).toStrictEqual([ + { + left: 'rgb', + right: 'ir', + points: [[10, 20, 12, 22], [30, 40, 33, 44]], + leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + rightToLeft: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + // No explicit choice was saved, so persistence fills the default model. + transformType: 'similarity', + }, + ]); + + // Not embedded in meta.json. + const meta = await fs.readJSON(npath.join(projectDir, 'meta.json')); + expect(meta.cameraHomographies).toBeUndefined(); + expect(meta.cameraCorrespondences).toBeUndefined(); + expect(meta.cameraTransformTypes).toBeUndefined(); + + // Rehydrated on load back into the in-app shapes. + const reloaded = await common.loadMetadata(settings, final.id, urlMapper); + expect(reloaded.cameraHomographies).toStrictEqual(cameraHomographies); + expect(reloaded.cameraCorrespondences).toStrictEqual(cameraCorrespondences); + expect(reloaded.cameraTransformTypes).toStrictEqual({ 'rgb::ir': 'similarity' }); + }); + + it('saveMetadata persists a non-default transformType per pair and reloads it', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + const final = res.meta; + const cameraHomographies = { + 'rgb::ir': { + AtoB: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + BtoA: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }, + }; + const cameraTransformTypes = { 'rgb::ir': 'rigid' as const }; + + await common.saveMetadata(settings, final.id, { cameraHomographies, cameraTransformTypes }); + + const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); + const calibration = await fs.readJSON(npath.join(projectDir, 'calibration.json')); + expect(calibration.pairs[0].transformType).toBe('rigid'); + + const reloaded = await common.loadMetadata(settings, final.id, urlMapper); + expect(reloaded.cameraTransformTypes).toStrictEqual(cameraTransformTypes); + }); + + describe('findParentFolderTransformFile', () => { + it('prefers a file named calibration.json among marked candidates', async () => { + const found = await common.findParentFolderTransformFile('/home/user/transformDiscovery/exactName'); + expect(found).toBe(npath.join('/home/user/transformDiscovery/exactName', 'calibration.json')); + }); + + it('finds a marked file under any name, skipping unmarked and broken JSON', async () => { + const found = await common.findParentFolderTransformFile('/home/user/transformDiscovery/otherName'); + expect(found).toBe(npath.join('/home/user/transformDiscovery/otherName', 'z-transforms.json')); + }); + + it('returns null when no self-identified calibration json exists', async () => { + expect(await common.findParentFolderTransformFile('/home/user/transformDiscovery/none')).toBeNull(); + }); + + it('returns null for a missing directory', async () => { + expect(await common.findParentFolderTransformFile('/home/user/doesNotExist')).toBeNull(); + }); + }); + + it('fromCalibrationPairs derives a missing matrix direction by inversion', () => { + const { homographies } = common.fromCalibrationPairs([{ + left: 'eo', + right: 'ir', + points: [], + leftToRight: null, + rightToLeft: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }]); + expect(homographies['eo::ir'].BtoA).toEqual([[1, 0, -5], [0, 1, 3], [0, 0, 1]]); + expect(homographies['eo::ir'].AtoB[0][2]).toBeCloseTo(5); + expect(homographies['eo::ir'].AtoB[1][2]).toBeCloseTo(-3); + }); + + it('fromCalibrationPairs keeps points but skips the matrix for singular input', () => { + const { homographies, correspondences } = common.fromCalibrationPairs([{ + left: 'eo', + right: 'ir', + points: [[1, 2, 3, 4]], + leftToRight: [[0, 0, 0], [0, 0, 0], [0, 0, 0]], + rightToLeft: null, + }]); + expect(homographies['eo::ir']).toBeUndefined(); + expect(correspondences['eo::ir']).toHaveLength(1); + }); + + it('saveMetadata persists the calibration source stamp and reloads it', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + const final = res.meta; + const cameraHomographies = { + 'rgb::ir': { + AtoB: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + BtoA: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }, + }; + const source = { model: 'colmap-2026-07-01', swathe: 'fl07_C' }; + + await common.saveMetadata(settings, final.id, { + cameraHomographies, + cameraCalibrationSource: source, + }); + + const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); + const calibrationPath = npath.join(projectDir, 'calibration.json'); + expect((await fs.readJSON(calibrationPath)).source).toStrictEqual(source); + const reloaded = await common.loadMetadata(settings, final.id, urlMapper); + expect(reloaded.cameraCalibrationSource).toStrictEqual(source); + + // A save that doesn't mention the stamp leaves it alone. + await common.saveMetadata(settings, final.id, { + cameraTransformTypes: { 'rgb::ir': 'rigid' }, + }); + expect((await fs.readJSON(calibrationPath)).source).toStrictEqual(source); + + // An explicit null clears it. + await common.saveMetadata(settings, final.id, { + cameraHomographies, + cameraCalibrationSource: null, + }); + expect('source' in (await fs.readJSON(calibrationPath))).toBe(false); + }); + it('import with CSV annotations without specifying track file', async () => { const payload = await common.beginMediaImport('/home/user/data/imageSuccessWithAnnotations'); payload.trackFileAbsPath = ''; //It returns null be default but users change it. diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 0c890ba03..53229b3ca 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -19,6 +19,9 @@ import { import { DefaultConfidence } from 'vue-media-annotator/BaseFilterControls'; import { TrackData } from 'vue-media-annotator/track'; import { GroupData } from 'vue-media-annotator/Group'; +import { TransformType, DEFAULT_TRANSFORM_TYPE } from 'vue-media-annotator/transform'; +import { CALIBRATION_FILE_TYPE } from 'vue-media-annotator/CameraCalibrationStore'; +import { invert3, Matrix3 } from 'vue-media-annotator/homography'; import { DatasetType, Pipelines, SaveDetectionsArgs, FrameImage, DatasetMetaMutable, TrainingConfig, TrainingConfigs, SaveAttributeArgs, @@ -71,6 +74,10 @@ const AuxFolderName = 'auxiliary'; const JsonTrackFileName = /^result(_.*)?\.json$/i; const JsonFileName = /^.*\.json$/i; const JsonMetaFileName = 'meta.json'; +// Standalone camera-to-camera (modality-to-modality) alignment transforms, +// stored separately from meta.json in the dataset directory. +const CalibrationFileName = 'calibration.json'; +const CalibrationFileVersion = 1; const CsvFileName = /^.*\.csv$/i; const YAMLFileName = /^.*\.ya?ml$/i; /** @@ -368,6 +375,28 @@ async function loadMetadata( const projectDirData = await getValidatedProjectDir(settings, datasetId); const projectMetaData = await loadJsonMetadata(projectDirData.metaFileAbsPath); + // Load standalone camera calibration (transforms + correspondences), if present. + const calibrationFileAbsPath = npath.join(projectDirData.basePath, CalibrationFileName); + let { + cameraHomographies, cameraCorrespondences, cameraTransformTypes, cameraCalibrationSource, + } = projectMetaData; + if (await fs.pathExists(calibrationFileAbsPath)) { + try { + const calibration = await _loadAsJson(calibrationFileAbsPath); + if (calibration && Array.isArray(calibration.pairs)) { + ({ + homographies: cameraHomographies, + correspondences: cameraCorrespondences, + transformTypes: cameraTransformTypes, + } = fromCalibrationPairs(calibration.pairs)); + cameraCalibrationSource = readCalibrationSource(calibration.source); + } + } catch (err) { + // A malformed calibration.json should not block loading the dataset. + console.warn(`Unable to read ${calibrationFileAbsPath}: ${err}`); + } + } + let videoUrl = ''; let imageData = [] as FrameImage[]; let multiCamMedia: MultiCamMedia | null = null; @@ -435,6 +464,10 @@ async function loadMetadata( imageData, multiCamMedia, subType, + cameraHomographies, + cameraCorrespondences, + cameraTransformTypes, + cameraCalibrationSource, }; } @@ -690,6 +723,104 @@ async function _saveAsJson(absPath: string, data: unknown) { await fs.writeFile(absPath, serialized); } +type CameraHomographies = NonNullable; +type CameraCorrespondences = NonNullable; +type CameraTransformTypes = NonNullable; +type CalibrationSource = NonNullable; + +/** + * Best-effort read of the calibration file's producer provenance stamp: a + * plain object, or null for anything else. Preserved verbatim across + * load/refine/save round trips; never interpreted by DIVE. + */ +function readCalibrationSource(raw: unknown): CalibrationSource | null { + if (raw && typeof raw === 'object' && !Array.isArray(raw)) { + return raw as CalibrationSource; + } + return null; +} + +/** + * One camera pair in calibration.json. `left`/`right` are camera (folder) names; + * `points` are the picked correspondences as rows of `leftX leftY rightX rightY`; + * `leftToRight`/`rightToLeft` are the fitted + * 3x3 homographies, when a fit has been performed; `transformType` is the fit + * model used to compute them (defaults to {@link DEFAULT_TRANSFORM_TYPE} when + * absent, matching the in-app default so a pair fitted at the default resolves + * to the same model after a save/reload). + */ +interface CalibrationPair { + left: string; + right: string; + points: number[][]; + leftToRight: number[][] | null; + rightToLeft: number[][] | null; + transformType?: TransformType; +} + +/** + * Convert the in-app calibration state (keyed by directional "left::right") into + * the self-describing list of pairs persisted in calibration.json. + */ +function toCalibrationPairs( + homographies: CameraHomographies, + correspondences: CameraCorrespondences, + transformTypes: CameraTransformTypes, +): CalibrationPair[] { + const keys = new Set([ + ...Object.keys(homographies), ...Object.keys(correspondences), ...Object.keys(transformTypes), + ]); + return [...keys].map((key) => { + const [left, right] = key.split('::'); + const homography = homographies[key]; + return { + left, + right, + points: (correspondences[key] || []).map((c) => [c.a[0], c.a[1], c.b[0], c.b[1]]), + leftToRight: homography ? homography.AtoB : null, + rightToLeft: homography ? homography.BtoA : null, + transformType: transformTypes[key] || DEFAULT_TRANSFORM_TYPE, + }; + }); +} + +/** Rebuild the in-app homographies/correspondences/transform types from calibration.json pairs. */ +function fromCalibrationPairs( + pairs: CalibrationPair[], +): { + homographies: CameraHomographies; + correspondences: CameraCorrespondences; + transformTypes: CameraTransformTypes; + } { + const homographies: CameraHomographies = {}; + const correspondences: CameraCorrespondences = {}; + const transformTypes: CameraTransformTypes = {}; + pairs.forEach((pair) => { + const key = `${pair.left}::${pair.right}`; + // Mirror the panel loader (CameraCalibrationStore.loadCalibrationText): + // producer files may carry only one fitted direction, so derive the + // missing one by inversion. A singular matrix can't participate in the + // warp either way, so such pairs contribute points only. + if (pair.leftToRight || pair.rightToLeft) { + try { + homographies[key] = { + AtoB: pair.leftToRight ?? invert3(pair.rightToLeft as Matrix3), + BtoA: pair.rightToLeft ?? invert3(pair.leftToRight as Matrix3), + }; + } catch { + // Singular / non-invertible: skip the matrix, keep the points. + } + } + if (pair.points && pair.points.length) { + correspondences[key] = pair.points.map((p, i) => ({ + id: i + 1, a: [p[0], p[1]], b: [p[2], p[3]], + })); + } + transformTypes[key] = pair.transformType || DEFAULT_TRANSFORM_TYPE; + }); + return { homographies, correspondences, transformTypes }; +} + async function saveMetadata(settings: Settings, datasetId: string, args: DatasetMetaMutable) { const projectDirInfo = await getValidatedProjectDir(settings, datasetId); const release = await _acquireLock(projectDirInfo.basePath, projectDirInfo.metaFileAbsPath, 'meta'); @@ -719,6 +850,51 @@ async function saveMetadata(settings: Settings, datasetId: string, args: Dataset existing.datasetInfo = args.datasetInfo; } + // Camera calibration (transforms + the points behind them) is persisted as a + // standalone calibration.json in the dataset directory rather than embedded in + // meta.json, so it is easy to find, hand-edit, and consume as a self-contained + // artifact. + if (args.cameraHomographies || args.cameraCorrespondences || args.cameraTransformTypes + || args.cameraCalibrationSource) { + const calibrationFileAbsPath = npath.join(projectDirInfo.basePath, CalibrationFileName); + // Start from whatever is on disk so a partial update doesn't clobber the rest. + let homographies: CameraHomographies = {}; + let correspondences: CameraCorrespondences = {}; + let transformTypes: CameraTransformTypes = {}; + let source: CalibrationSource | null = null; + if (await fs.pathExists(calibrationFileAbsPath)) { + try { + const existingCalibration = await _loadAsJson(calibrationFileAbsPath); + if (existingCalibration && Array.isArray(existingCalibration.pairs)) { + ({ homographies, correspondences, transformTypes } = fromCalibrationPairs( + existingCalibration.pairs, + )); + source = readCalibrationSource(existingCalibration.source); + } + } catch (err) { + console.warn(`Unable to read existing ${calibrationFileAbsPath}: ${err}`); + } + } + if (args.cameraHomographies) { + homographies = args.cameraHomographies; + } + if (args.cameraCorrespondences) { + correspondences = args.cameraCorrespondences; + } + if (args.cameraTransformTypes) { + transformTypes = args.cameraTransformTypes; + } + // undefined leaves the on-disk stamp alone; null/object replaces it. + if (args.cameraCalibrationSource !== undefined) { + source = args.cameraCalibrationSource; + } + await _saveAsJson(calibrationFileAbsPath, { + version: CalibrationFileVersion, + ...(source ? { source } : {}), + pairs: toCalibrationPairs(homographies, correspondences, transformTypes), + }); + } + await _saveAsJson(projectDirInfo.metaFileAbsPath, existing); await release(); } @@ -1156,6 +1332,48 @@ async function findParentFolderCalibrationFile(parentPath: string): Promise { + if (!await fs.pathExists(parentPath)) { + return null; + } + const stat = await fs.stat(parentPath); + if (!stat.isDirectory()) { + return null; + } + const children = await fs.readdir(parentPath, { withFileTypes: true }); + const candidates = children + .filter((entry) => entry.isFile() && /\.json$/i.test(entry.name)) + .map((entry) => entry.name) + .sort((a, b) => { + const aExact = a.toLowerCase() === CalibrationFileName ? 0 : 1; + const bExact = b.toLowerCase() === CalibrationFileName ? 0 : 1; + return aExact - bExact || a.localeCompare(b); + }); + // eslint-disable-next-line no-restricted-syntax + for (const name of candidates) { + const absPath = npath.join(parentPath, name); + try { + // eslint-disable-next-line no-await-in-loop -- candidates checked in priority order + const data = await fs.readJson(absPath); + if (data && data.type === CALIBRATION_FILE_TYPE && Array.isArray(data.pairs)) { + return absPath; + } + } catch { + // Unreadable/non-JSON candidates are simply not matches. + } + } + return null; +} + /** * Resolve the import path for one camera subfolder (directory or first video file). */ @@ -2135,6 +2353,8 @@ export { listImmediateSubfolders, listParentFolderCameras, findParentFolderCalibrationFile, + findParentFolderTransformFile, + fromCalibrationPairs, resolveMulticamCameraSourcePath, findTrackandMetaFileinFolder, getLastCalibrationPath, diff --git a/client/platform/desktop/backend/native/multiCamImport.ts b/client/platform/desktop/backend/native/multiCamImport.ts index 8b3b1cf44..40af5c47b 100644 --- a/client/platform/desktop/backend/native/multiCamImport.ts +++ b/client/platform/desktop/backend/native/multiCamImport.ts @@ -3,6 +3,7 @@ import fs from 'fs-extra'; import mime from 'mime-types'; import { DatasetType, + DatasetMetaMutable, MultiCamImportFolderArgs, MultiCamImportKeywordArgs, MultiCamImportArgs, @@ -16,7 +17,13 @@ import { Camera, } from 'platform/desktop/constants'; import { checkMedia } from 'platform/desktop/backend/native/mediaJobs'; -import { findImagesInFolder } from './common'; +import { readTransformMatrix } from 'vue-media-annotator/alignedView'; +import { findImagesInFolder, fromCalibrationPairs } from './common'; + +type CameraHomographies = NonNullable; +type CameraCorrespondences = NonNullable; +type CameraTransformTypes = NonNullable; +type CalibrationSource = NonNullable; function isFolderArgs(s: MultiCamImportArgs): s is MultiCamImportFolderArgs { if ('sourceList' in s && 'defaultDisplay' in s) { @@ -89,6 +96,52 @@ async function beginMultiCamImport(args: MultiCamImportArgs): Promise { + if (!item.transformFile) { + return; + } + try { + // A DIVE calibration .json (the panel's save format / the project + // calibration.json shape): pairs name their own cameras, so merge + // them all in. + const data = await fs.readJson(item.transformFile); + if (!data || !Array.isArray(data.pairs)) { + throw new Error('not a DIVE calibration file (expected a "pairs" list)'); + } + const parsed = fromCalibrationPairs(data.pairs); + Object.entries(parsed.homographies).forEach(([key, homography]) => { + if (!readTransformMatrix(homography.AtoB) || !readTransformMatrix(homography.BtoA)) { + throw new Error(`pair "${key.split('::').join(' / ')}" has an invalid 3x3 transform matrix`); + } + seedHomographies[key] = homography; + }); + Object.assign(seedCorrespondences, parsed.correspondences); + Object.assign(seedTransformTypes, parsed.transformTypes); + // Producer provenance travels with the seed. With one transform file + // per dataset (the expected case) this is that file's stamp; with + // several, the last stamped file wins, matching the merge order of the + // pairs above. + if (data.source && typeof data.source === 'object' && !Array.isArray(data.source)) { + seedCalibrationSource = data.source as CalibrationSource; + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new Error(`Camera "${cameraName}": invalid transform file: ${message}`); + } + }); + } + const jsonMeta: JsonMeta = { version: JsonMetaCurrentVersion, type: datasetType, @@ -111,6 +164,14 @@ async function beginMultiCamImport(args: MultiCamImportArgs): Promise ({ + checkMedia: vi.fn(() => Promise.resolve({ + websafe: true, + originalFpsString: '30/1', + originalFps: 30, + videoDimensions: { width: 1920, height: 1080 }, + })), +})); + +let tmpDir: string; +let calibrationJsonPath: string; +let badCalibrationJsonPath: string; + +function sourceListWith(transformFile?: string) { + return { + eo: { sourcePath: npath.join(tmpDir, 'eo'), trackFile: '' }, + ir: { + sourcePath: npath.join(tmpDir, 'ir'), + trackFile: '', + ...(transformFile ? { transformFile } : {}), + }, + }; +} + +beforeAll(() => { + tmpDir = fs.mkdtempSync(npath.join(os.tmpdir(), 'multicam-transform-import-')); + ['eo', 'ir'].forEach((camera) => { + const dir = npath.join(tmpDir, camera); + fs.mkdirSync(dir); + ['frame0.png', 'frame1.png'].forEach((name) => { + fs.writeFileSync(npath.join(dir, name), ''); + }); + }); + calibrationJsonPath = npath.join(tmpDir, 'calibration.json'); + fs.writeJsonSync(calibrationJsonPath, { + type: 'dive-camera-calibration', + version: 1, + source: { model: 'colmap-2026-07-01', swathe: 'fl07_C' }, + pairs: [{ + left: 'eo', + right: 'ir', + points: [[0, 0, 5, -3], [10, 0, 15, -3]], + leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + rightToLeft: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + transformType: 'translation', + }], + }); + badCalibrationJsonPath = npath.join(tmpDir, 'not-a-calibration.json'); + fs.writeJsonSync(badCalibrationJsonPath, { some: 'other json' }); +}); + +afterAll(() => { + fs.removeSync(tmpDir); +}); + +describe('multiCamImport transform wire-through', () => { + it('seeds the saved calibration (points and all) from a DIVE calibration .json', async () => { + const output = await beginMultiCamImport({ + datasetName: 'calibration_json_test', + defaultDisplay: 'eo', + cameraOrder: ['eo', 'ir'], + sourceList: sourceListWith(calibrationJsonPath), + type: 'image-sequence', + }); + expect(output.jsonMeta.cameraHomographies?.['eo::ir'].AtoB).toEqual( + [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + ); + expect(output.jsonMeta.cameraCorrespondences?.['eo::ir']).toHaveLength(2); + expect(output.jsonMeta.cameraTransformTypes?.['eo::ir']).toBe('translation'); + // The producer provenance stamp travels with the seed. + expect(output.jsonMeta.cameraCalibrationSource).toEqual( + { model: 'colmap-2026-07-01', swathe: 'fl07_C' }, + ); + }); + + it('leaves the calibration unset when no transform file is given', async () => { + const output = await beginMultiCamImport({ + datasetName: 'no_transform', + defaultDisplay: 'eo', + sourceList: sourceListWith(), + type: 'image-sequence', + }); + expect(output.jsonMeta.cameraHomographies).toBeUndefined(); + expect(output.jsonMeta.cameraCorrespondences).toBeUndefined(); + }); + + it('fails the import for a .json without a pairs list', async () => { + await expect(beginMultiCamImport({ + datasetName: 'bad_calibration_json', + defaultDisplay: 'eo', + sourceList: sourceListWith(badCalibrationJsonPath), + type: 'image-sequence', + })).rejects.toThrow(/Camera "ir": invalid transform file: not a DIVE calibration file/); + }); + + it('fails the import for a file that is not JSON at all', async () => { + const notJsonPath = npath.join(tmpDir, 'not-json.json'); + fs.writeFileSync(notJsonPath, 'not json content'); + await expect(beginMultiCamImport({ + datasetName: 'not_json_transform', + defaultDisplay: 'eo', + sourceList: sourceListWith(notJsonPath), + type: 'image-sequence', + })).rejects.toThrow(/Camera "ir": invalid transform file/); + }); + + it('fails the import when the transform file does not exist', async () => { + await expect(beginMultiCamImport({ + datasetName: 'missing_transform', + defaultDisplay: 'eo', + sourceList: sourceListWith(npath.join(tmpDir, 'does-not-exist.json')), + type: 'image-sequence', + })).rejects.toThrow(/Camera "ir": invalid transform file/); + }); +}); diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index e773257ee..807a512d8 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -13,7 +13,7 @@ import type { import { fileVideoTypes, calibrationFileTypes, inputAnnotationFileTypes, listFileTypes, - largeImageDesktopTypes, + largeImageDesktopTypes, transformFileTypes, } from 'dive-common/constants'; import { DesktopMetadata, NvidiaSmiReply, @@ -47,7 +47,7 @@ function joinPath(dir: string, filename: string) { * Native functions that run entirely in the renderer */ -async function openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text', directory = false) { +async function openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'transform', directory = false) { let filters: FileFilter[] = []; const allFiles = { name: 'All Files', extensions: ['*'] }; if (datasetType === 'video') { @@ -73,6 +73,12 @@ async function openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | allFiles, ]; } + if (datasetType === 'transform') { + filters = [ + { name: 'Transform / calibration', extensions: transformFileTypes }, + allFiles, + ]; + } if (datasetType === 'text') { filters = [ { name: 'text', extensions: listFileTypes }, @@ -200,6 +206,10 @@ function findParentFolderCalibrationFile(parentPath: string): Promise { + return window.diveDesktop.invoke('find-parent-folder-transform-file', { path: parentPath }); +} + function hasCalibrationFile(datasetId: string): Promise { return window.diveDesktop.invoke('dataset-has-calibration-file', { datasetId }); } @@ -698,6 +708,7 @@ export { listParentFolderCameras, resolveMulticamCameraSourcePath, findParentFolderCalibrationFile, + findParentFolderTransformFile, hasCalibrationFile, bulkImportMedia, deleteDataset, diff --git a/client/platform/desktop/frontend/components/Recent.vue b/client/platform/desktop/frontend/components/Recent.vue index d0a076616..b701156cf 100644 --- a/client/platform/desktop/frontend/components/Recent.vue +++ b/client/platform/desktop/frontend/components/Recent.vue @@ -309,6 +309,7 @@ export default defineComponent({ :stereo="stereo" :data-type="multiCamOpenType" :enable-subfolder-import="true" + :enable-transform-import="true" :import-media="importMedia" @begin-multicam-import="multiCamImport($event)" @abort="importMultiCamDialog = false" diff --git a/client/platform/web-girder/utils.ts b/client/platform/web-girder/utils.ts index f30b21ebd..91870cbed 100644 --- a/client/platform/web-girder/utils.ts +++ b/client/platform/web-girder/utils.ts @@ -2,7 +2,8 @@ import { UploadManager, Location } from '@girder/components/src'; import { calibrationFileTypes, inputAnnotationFileTypes, inputAnnotationTypes, getLargeImageAllowedExtensions, getLargeImageFileAccept, - otherImageTypes, otherVideoTypes, websafeImageTypes, websafeVideoTypes, zipFileTypes, + otherImageTypes, otherVideoTypes, transformFileTypes, + websafeImageTypes, websafeVideoTypes, zipFileTypes, } from 'dive-common/constants'; import { DatasetType } from 'dive-common/apispec'; import type { LocationType, RootlessLocationType } from 'platform/web-girder/store/types'; @@ -38,7 +39,7 @@ function getRouteFromLocation(location: LocationType): string { } async function openFromDisk( - datasetType: DatasetType | 'calibration' | 'annotation' | 'text' | 'zip', + datasetType: DatasetType | 'calibration' | 'annotation' | 'text' | 'zip' | 'transform', directory = false, ): Promise<{ canceled: boolean; filePaths: string[]; fileList?: File[]; root?: string }> { const input: HTMLInputElement = document.createElement('input'); @@ -66,6 +67,9 @@ async function openFromDisk( .concat(inputAnnotationFileTypes.map((item) => `.${item}`)).join(','); } else if (datasetType === 'zip') { input.accept = zipFileTypes.map((item) => `.${item}`).join(','); + } else if (datasetType === 'transform') { + input.accept = transformFileTypes.map((item) => `.${item}`).join(','); + input.multiple = false; } else if (datasetType === 'text') { input.accept = '.txt,.text'; input.multiple = false; diff --git a/server/dive_server/crud_dataset.py b/server/dive_server/crud_dataset.py index 6e22c3d96..7247ad4b3 100644 --- a/server/dive_server/crud_dataset.py +++ b/server/dive_server/crud_dataset.py @@ -402,9 +402,13 @@ def update_metadata(dsFolder: types.GirderModel, data: dict, verify=True): ) for name, value in validated.dict(exclude_none=True).items(): dsFolder['meta'][name] = value - # exclude_none drops explicit null; client sends timeFilters: null to disable. - if 'timeFilters' in data and data['timeFilters'] is None: - dsFolder['meta'].pop('timeFilters', None) + # exclude_none drops explicit null, so a field the client nulls to clear it + # must be popped by hand. timeFilters: null disables the filter; + # cameraCalibrationSource: null drops a stale producer-provenance stamp when + # the calibration is cleared or hand-refined. + for nullable in ('timeFilters', 'cameraCalibrationSource'): + if nullable in data and data[nullable] is None: + dsFolder['meta'].pop(nullable, None) Folder().save(dsFolder) return dsFolder['meta'] diff --git a/server/dive_utils/models.py b/server/dive_utils/models.py index 75b33383a..7af312147 100644 --- a/server/dive_utils/models.py +++ b/server/dive_utils/models.py @@ -240,6 +240,21 @@ class MetadataMutable(BaseModel): attributes: Optional[Dict[str, Attribute]] attributeTrackFilters: Optional[Dict[str, AttributeTrackFilter]] datasetInfo: Optional[types.DatasetInfo] + # Per-camera-pair alignment homographies, keyed by directional "left::right". + # Each value holds the 3x3 AtoB / BtoA matrices. + cameraHomographies: Optional[Dict[str, Dict[str, List[List[float]]]]] + # The picked point correspondences behind those homographies, keyed the same + # way. Each entry is a list of {id, a: [x, y], b: [x, y]} pairs. + cameraCorrespondences: Optional[Dict[str, List[Dict[str, Any]]]] + # The fit model used to compute each pair's homography (translation / rigid / + # similarity / affine / homography), keyed the same way. Missing entries + # default to 'similarity' client-side. + cameraTransformTypes: Optional[Dict[str, str]] + # Free-form producer provenance for the camera calibration (e.g. an external + # model step's version / swathe / generation time). Never interpreted by + # DIVE; preserved verbatim so refined calibrations can be traced back to the + # model version they were made against. + cameraCalibrationSource: Optional[Dict[str, Any]] fps: Optional[float] @staticmethod diff --git a/server/tests/test_update_metadata.py b/server/tests/test_update_metadata.py index ba08140a7..58863e8da 100644 --- a/server/tests/test_update_metadata.py +++ b/server/tests/test_update_metadata.py @@ -26,6 +26,27 @@ def test_update_metadata_clears_time_filters_when_null(_verify, folder_cls): folder_cls.return_value.save.assert_called_once() +@patch('dive_server.crud_dataset.Folder') +@patch('dive_server.crud_dataset.crud.verify_dataset') +def test_update_metadata_clears_calibration_source_when_null(_verify, folder_cls): + # A cleared / hand-refined calibration sends cameraCalibrationSource: null to + # drop a stale producer stamp; exclude_none would otherwise leave it behind. + folder = { + '_id': 'dataset-id', + 'meta': { + 'annotate': True, + 'type': 'image-sequence', + 'cameraCalibrationSource': {'model': 'colmap-v3', 'swathe': '17'}, + }, + } + folder_cls.return_value.save = MagicMock(side_effect=lambda f: f) + + crud_dataset.update_metadata(folder, {'cameraCalibrationSource': None}) + + assert 'cameraCalibrationSource' not in folder['meta'] + folder_cls.return_value.save.assert_called_once() + + @patch('dive_server.crud_dataset.Folder') @patch('dive_server.crud_dataset.crud.verify_dataset') def test_update_metadata_sets_time_filters(_verify, folder_cls): From 1139ff67a425b804a3689555e910842ca96a6d6b Mon Sep 17 00:00:00 2001 From: romleiaj Date: Thu, 9 Jul 2026 16:16:30 -0400 Subject: [PATCH 3/7] Add the Align View: warped display, linked navigation, and mirroring When every camera resolves a transform into the reference camera's space (first camera in display order, composed through the calibration pair graph), an Align button warps each pane's imagery and annotations into that shared space at draw time -- stored geometry always stays native per camera. Includes: AlignedViewStore + resolution wiring in the Viewer; AlignedImageLayer (sub-quad warped imagery with seam overlap); display-transform routing through every annotation layer; transform-aware linked pan/zoom (useAlignedNavigation/useLinkedViewers, with the raw screen-delta camera sync retired once transforms exist); drawing/editing on any camera with edits mapped back through the inverse; and a continuous cross-camera mirror that re-projects every geometry edit onto all other calibrated cameras. Co-Authored-By: Claude Fable 5 --- client/dive-common/components/Viewer.vue | 110 ++++++- client/dive-common/use/useModeManager.spec.ts | 161 +++++++++ client/dive-common/use/useModeManager.ts | 145 +++++++++ client/src/AlignedViewStore.ts | 110 +++++++ client/src/alignedView.spec.ts | 301 +++++++++++++++++ client/src/alignedView.ts | 305 ++++++++++++++++++ client/src/components/LayerManager.vue | 270 +++++++++++++++- .../annotators/mediaControllerType.ts | 10 + .../annotators/useAlignedNavigation.spec.ts | 195 +++++++++++ .../annotators/useAlignedNavigation.ts | 105 ++++++ .../components/annotators/useLinkedViewers.ts | 94 ++++++ .../annotators/useMediaController.ts | 19 +- client/src/components/controls/Controls.vue | 49 ++- client/src/components/index.ts | 1 + client/src/index.ts | 4 + client/src/layers/AlignedImageLayer.ts | 260 +++++++++++++++ .../AnnotationLayers/AttributeBoxLayer.ts | 2 +- .../layers/AnnotationLayers/AttributeLayer.ts | 2 +- .../src/layers/AnnotationLayers/LineLayer.ts | 2 +- .../layers/AnnotationLayers/OverlapLayer.ts | 2 +- .../src/layers/AnnotationLayers/PointLayer.ts | 1 + .../layers/AnnotationLayers/PolygonLayer.ts | 2 +- .../layers/AnnotationLayers/RectangleLayer.ts | 4 +- .../src/layers/AnnotationLayers/TextLayer.ts | 2 +- client/src/layers/BaseLayer.ts | 37 +++ client/src/provides.ts | 18 ++ 26 files changed, 2179 insertions(+), 32 deletions(-) create mode 100644 client/dive-common/use/useModeManager.spec.ts create mode 100644 client/src/AlignedViewStore.ts create mode 100644 client/src/alignedView.spec.ts create mode 100644 client/src/alignedView.ts create mode 100644 client/src/components/annotators/useAlignedNavigation.spec.ts create mode 100644 client/src/components/annotators/useAlignedNavigation.ts create mode 100644 client/src/components/annotators/useLinkedViewers.ts create mode 100644 client/src/layers/AlignedImageLayer.ts diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 24f459080..de03505fa 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -18,8 +18,11 @@ import { import { Track, Group, CameraStore, + CameraCalibrationStore, + AlignedViewStore, StyleManager, TrackFilterControls, GroupFilterControls, } from 'vue-media-annotator/index'; +import { resolveToReferenceTransforms, unresolvedCameras } from 'vue-media-annotator/alignedView'; import { provideAnnotator, LassoModeSymbol } from 'vue-media-annotator/provides'; import { @@ -28,6 +31,7 @@ import { LargeImageAnnotator, LayerManager, useMediaController, + useAlignedNavigation, TrackList, FilterList, } from 'vue-media-annotator/components'; @@ -395,6 +399,55 @@ export default defineComponent({ const groupStyleManager = new StyleManager({ markChangesPending, vuetify }); const cameraStore = new CameraStore({ markChangesPending }); + const cameraCalibration = new CameraCalibrationStore(); + + /** + * Aligned view (SEAL-TK features 2 + 3): when every non-reference camera + * has a usable transform into the reference camera's space, the user may + * warp displays and link pan/zoom across all cameras during normal + * review. Reference camera = first camera in display order + * (multiCamList[0]). Transforms come from the calibration store's pair + * homographies (loaded from a calibration file or the dataset's saved + * meta), composed through the pair graph. + */ + const alignedView = new AlignedViewStore(); + const alignedResolution = computed(() => { + if (multiCamList.value.length < 2) { + return null; + } + const reference = multiCamList.value[0]; + const toReference = resolveToReferenceTransforms( + multiCamList.value, + reference, + cameraCalibration.homographies.value, + ); + return toReference ? { reference, toReference } : null; + }); + watch(alignedResolution, (resolution) => { + alignedView.setTransforms( + resolution?.reference ?? null, + resolution?.toReference ?? null, + ); + }, { immediate: true }); + useAlignedNavigation(aggregateController, alignedView, multiCamList); + const alignedViewAvailable = computed(() => alignedView.available.value); + const alignedViewEnabled = computed(() => alignedView.enabled.value); + const toggleAlignedView = () => { + alignedView.setEnabled(!alignedView.enabled.value); + }; + const alignedViewTooltip = computed(() => { + if (alignedViewEnabled.value) { + return 'Align View on (draw/edit on any camera)'; + } + const cams = multiCamList.value; + const unresolved = cams.length >= 2 + ? unresolvedCameras(cams, cams[0], cameraCalibration.homographies.value) + : []; + if (unresolved.length) { + return `Align View — ${cams.length - unresolved.length}/${cams.length} cameras calibrated`; + } + return 'Align View'; + }); // This context for removal const removeGroups = (id: AnnotationId) => { cameraStore.removeGroups(id); @@ -462,6 +515,7 @@ export default defineComponent({ cameraStore, aggregateController, readonlyState, + alignedView, isStereoscopicDataset: computed(() => subType.value === 'stereo'), onStereoAnnotationComplete: (params: StereoAnnotationCompleteParams) => { emit('stereo-annotation-complete', params); @@ -1377,6 +1431,17 @@ export default defineComponent({ if (meta.attributeTrackFilters) { trackFilters.loadTrackAttributesFilter(Object.values(meta.attributeTrackFilters)); } + // Rehydrate any saved camera-to-camera calibration homographies, points, + // transform types, and producer provenance + cameraCalibration.hydrate( + meta.cameraHomographies, + meta.cameraCorrespondences, + meta.cameraTransformTypes, + meta.cameraCalibrationSource, + ); + // Reset the aligned-view toggle for the newly loaded dataset (no + // persistence this phase). + alignedView.setEnabled(false); setImageEnhancements( imageEnhancementsByCamera.value[selectedCamera.value] ?? { ...defaultImageEnhancements }, ); @@ -1457,7 +1522,13 @@ export default defineComponent({ if (previous) observer.unobserve(previous.$el); if (controlsRef.value) observer.observe(controlsRef.value.$el); }); - watch([controlsCollapsed, sidebarMode], async () => { + // Opening/closing the context sidebar shrinks or widens the camera panes, + // but nothing else notices: the only ResizeObserver watches the controls + // bar, which is position:absolute in side layout and so keeps its content + // width when the panes resize. Trigger a resize explicitly so the panes' + // GeoJS size() stays in sync (an unnoticed shrink leaves content anchored + // in a corner). + watch([controlsCollapsed, sidebarMode, () => context.state.active], async () => { await nextTick(); handleResize(); }); @@ -1502,6 +1573,8 @@ export default defineComponent({ annotatorPreferences: toRef(clientSettings, 'annotatorPreferences'), attributes, cameraStore, + cameraCalibration, + alignedView, datasetId, editingMode, groupFilters, @@ -1800,6 +1873,10 @@ export default defineComponent({ deleteAttributeHandler, saveTooltipText, showMultiCamToolbar, + alignedViewAvailable, + alignedViewEnabled, + alignedViewTooltip, + toggleAlignedView, seekToFrame, resetAggregateZoom, /* large image methods */ @@ -1983,6 +2060,37 @@ export default defineComponent({ {{ item }} {{ item === defaultCamera ? '(Default)' : '' }} + + + + {{ alignedViewTooltip }} + diff --git a/client/dive-common/use/useModeManager.spec.ts b/client/dive-common/use/useModeManager.spec.ts new file mode 100644 index 000000000..40eae73af --- /dev/null +++ b/client/dive-common/use/useModeManager.spec.ts @@ -0,0 +1,161 @@ +/// +/** + * Functional tests for the Align View cross-camera mirror: drawing/editing a + * track on one camera while the aligned view is active re-projects the + * geometry onto every other calibrated camera under the same track id. + */ +import { ref } from 'vue'; +import CameraStore from 'vue-media-annotator/CameraStore'; +import AlignedViewStore from 'vue-media-annotator/AlignedViewStore'; +import TrackFilterControls from 'vue-media-annotator/TrackFilterControls'; +import GroupFilterControls from 'vue-media-annotator/GroupFilterControls'; +import { IDENTITY3 } from 'vue-media-annotator/alignedView'; +import type { Matrix3 } from 'vue-media-annotator/homography'; +import type { AggregateMediaController } from 'vue-media-annotator/components/annotators/mediaControllerType'; +import type { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; +import useModeManager from './useModeManager'; + +function translation(tx: number, ty: number): Matrix3 { + return [[1, 0, tx], [0, 1, ty], [0, 0, 1]]; +} + +function makeHarness() { + const cameraStore = new CameraStore({ markChangesPending: () => undefined }); + cameraStore.removeCamera('singleCam'); + cameraStore.addCamera('left'); + cameraStore.addCamera('right'); + + // right -> reference(left) shifts x by -100, so left -> right adds +100. + const alignedView = new AlignedViewStore(); + alignedView.setTransforms('left', { + left: IDENTITY3, + right: translation(-100, 0), + }); + alignedView.setEnabled(true); + + const perCamera: Record; hasFrame: ReturnType }> = { + left: { frame: ref(0), hasFrame: ref(true) }, + right: { frame: ref(0), hasFrame: ref(true) }, + }; + const aggregateController = ref({ + frame: ref(0), + nextFrame: () => undefined, + seekCameraFrame: () => undefined, + getController: (name: string) => perCamera[name], + } as unknown as AggregateMediaController); + + const groupFilterControls = new GroupFilterControls({ + sorted: cameraStore.sortedGroups, + remove: () => undefined, + markChangesPending: () => undefined, + setType: () => undefined, + removeTypes: () => [], + }); + const trackFilterControls = new TrackFilterControls({ + sorted: cameraStore.sortedTracks, + remove: () => undefined, + markChangesPending: () => undefined, + lookupGroups: cameraStore.lookupGroups.bind(cameraStore), + getTrack: (id: AnnotationId, camera = 'singleCam') => cameraStore.getTrack(id, camera), + groupFilterControls, + setType: () => undefined, + removeTypes: () => [], + }); + + const modeManager = useModeManager({ + cameraStore, + trackFilterControls, + groupFilterControls, + aggregateController, + readonlyState: ref(false), + recipes: [], + alignedView, + }); + modeManager.selectedCamera.value = 'left'; + return { + cameraStore, alignedView, modeManager, perCamera, + }; +} + +describe('useModeManager aligned-view track mirroring', () => { + it('mirrors a drawn rectangle onto the other camera, creating the same-id track', () => { + const { cameraStore, modeManager } = makeHarness(); + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.updateRectBounds(0, 0, [10, 20, 30, 40]); + + const mirrored = cameraStore.getPossibleTrack(trackId, 'right'); + expect(mirrored).toBeDefined(); + expect(mirrored?.features[0]?.bounds).toEqual([110, 20, 130, 40]); + expect(mirrored?.confidencePairs[0][0]) + .toEqual(cameraStore.getTrack(trackId, 'left').confidencePairs[0][0]); + }); + + it('continuously re-mirrors subsequent edits (continuous mirror)', () => { + const { cameraStore, modeManager } = makeHarness(); + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.updateRectBounds(0, 0, [10, 20, 30, 40]); + modeManager.handler.trackSelect(trackId, true); + modeManager.handler.updateRectBounds(0, 0, [50, 60, 70, 80]); + + const mirrored = cameraStore.getTrack(trackId, 'right'); + expect(mirrored.features[0]?.bounds).toEqual([150, 60, 170, 80]); + }); + + it('mirrors polygon geometry coordinates into the target camera space', () => { + const { cameraStore, modeManager } = makeHarness(); + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.setTrackFeature(0, [0, 0, 4, 4], [{ + type: 'Feature', + geometry: { type: 'Polygon', coordinates: [[[0, 0], [4, 0], [4, 4], [0, 0]]] }, + properties: { key: '' }, + }]); + + const mirrored = cameraStore.getTrack(trackId, 'right'); + expect(mirrored.features[0]?.bounds).toEqual([100, 0, 104, 4]); + const polygon = mirrored.features[0]?.geometry?.features[0]; + expect(polygon?.geometry).toEqual({ + type: 'Polygon', + coordinates: [[[100, 0], [104, 0], [104, 4], [100, 0]]], + }); + }); + + it('maps through per-camera local frames when they diverge (aligned timeline)', () => { + const { cameraStore, modeManager, perCamera } = makeHarness(); + perCamera.left.frame.value = 5; + perCamera.right.frame.value = 8; + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.updateRectBounds(5, 0, [10, 20, 30, 40]); + + const mirrored = cameraStore.getTrack(trackId, 'right'); + expect(mirrored.features[8]?.bounds).toEqual([110, 20, 130, 40]); + expect(mirrored.features[5]).toBeUndefined(); + }); + + it('skips cameras with no frame at the current aligned slot', () => { + const { cameraStore, modeManager, perCamera } = makeHarness(); + perCamera.right.hasFrame.value = false; + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.updateRectBounds(0, 0, [10, 20, 30, 40]); + + expect(cameraStore.getPossibleTrack(trackId, 'right')).toBeUndefined(); + }); + + it('does not mirror when the aligned view is disabled', () => { + const { cameraStore, alignedView, modeManager } = makeHarness(); + alignedView.setEnabled(false); + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.updateRectBounds(0, 0, [10, 20, 30, 40]); + + expect(cameraStore.getPossibleTrack(trackId, 'right')).toBeUndefined(); + expect(cameraStore.getTrack(trackId, 'left').features[0]?.bounds).toEqual([10, 20, 30, 40]); + }); + + it('does not mirror while the aligned view is suspended (calibration picking)', () => { + const { cameraStore, alignedView, modeManager } = makeHarness(); + alignedView.setSuspended(true); + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.updateRectBounds(0, 0, [10, 20, 30, 40]); + + expect(cameraStore.getPossibleTrack(trackId, 'right')).toBeUndefined(); + }); +}); diff --git a/client/dive-common/use/useModeManager.ts b/client/dive-common/use/useModeManager.ts index 45e8813fa..78a8e17ab 100644 --- a/client/dive-common/use/useModeManager.ts +++ b/client/dive-common/use/useModeManager.ts @@ -7,8 +7,13 @@ import { RectBounds, updateBounds, validateRotation, + getRotationFromAttributes, ROTATION_ATTRIBUTE_NAME, } from 'vue-media-annotator/utils'; +import type AlignedViewStore from 'vue-media-annotator/AlignedViewStore'; +import { + mapBounds, mapRotatedBounds, mapGeoJSONFeatures, +} from 'vue-media-annotator/alignedView'; import { EditAnnotationTypes, VisibleAnnotationTypes } from 'vue-media-annotator/layers'; import { AggregateMediaController } from 'vue-media-annotator/components/annotators/mediaControllerType'; @@ -97,6 +102,7 @@ export default function useModeManager({ aggregateController, readonlyState, recipes, + alignedView, isStereoscopicDataset, onStereoAnnotationComplete, onStereoAnnotationReset, @@ -108,6 +114,12 @@ export default function useModeManager({ aggregateController: Ref; readonlyState: Readonly>; recipes: Recipe[]; + /** + * When provided, geometry drawn/edited while the Align View is active is + * mirrored onto every other calibrated camera (see + * {@link mirrorFeatureToAlignedCameras}). + */ + alignedView?: AlignedViewStore; /** When set, interactive stereo only runs on stereoscopic datasets. */ isStereoscopicDataset?: Ref; onStereoAnnotationComplete?: (params: StereoAnnotationCompleteParams) => void; @@ -553,6 +565,123 @@ export default function useModeManager({ creating = newCreatingValue; } + /** + * Continuous cross-camera mirror: while the Align View is active (the rig + * is fully calibrated and the user toggled the aligned display), re-project + * the source camera's keyframe at `sourceFrame` onto every other camera + * through the calibrated camera-to-camera homographies, creating the + * same-id track on cameras where it doesn't exist yet. Stored geometry + * stays native per camera (decision D3): each camera receives coordinates + * already mapped into its own image space. + * + * Called after every geometry write/removal below; a no-op when the + * aligned view is off, so single-camera and unaligned multicam behavior is + * unchanged. The camera that received the edit is always the source, so + * later edits on any camera re-sync the others (continuous mirror -- + * per-camera fine-tuning is intentionally overwritten by the next edit). + */ + function mirrorFeatureToAlignedCameras(trackId: AnnotationId, sourceFrame: number) { + if (!alignedView?.active.value) { + return; + } + const sourceCamera = selectedCamera.value; + const sourceTrack = cameraStore.getPossibleTrack(trackId, sourceCamera); + if (!sourceTrack) { + return; + } + const sourceFeature = sourceTrack.features[sourceFrame]; + const hasKeyframe = Boolean(sourceFeature && sourceFeature.keyframe); + // Under an aligned timeline, cameras sit on different local frames for + // the same instant; the per-camera controller frames give the mapping + // for the CURRENT slot. Edits at any other frame (e.g. multi-frame + // segmentation) fall back to the same local frame number. + const sourceIsCurrentFrame = sourceFrame === selectedCameraFrame(); + cameraStore.camMap.value.forEach(({ trackStore }, cameraName) => { + if (cameraName === sourceCamera) { + return; + } + const matrix = alignedView.cameraToCamera(sourceCamera, cameraName); + if (!matrix) { + return; + } + let targetFrame = sourceFrame; + if (sourceIsCurrentFrame) { + try { + const controller = aggregateController.value.getController(cameraName); + if (!controller.hasFrame.value) { + // This camera has no frame at the current aligned slot. + return; + } + targetFrame = controller.frame.value; + } catch { + // No controller mounted for the camera; assume matching local frames. + } + } + let targetTrack = trackStore.getPossible(trackId); + if (!hasKeyframe) { + // The source keyframe was removed: drop the mirrored keyframe too, + // and the mirrored track itself when that leaves it empty. + if (targetTrack && targetTrack.features[targetFrame]?.keyframe) { + targetTrack.deleteFeature(targetFrame); + if (targetTrack.begin === targetTrack.end + && !targetTrack.getFeature(targetTrack.begin).some((item) => item !== null)) { + trackStore.remove(trackId); + } + } + return; + } + const mappedGeometry = sourceFeature.geometry + ? mapGeoJSONFeatures(matrix, sourceFeature.geometry.features) + : []; + let mappedBounds: RectBounds | undefined; + let mappedRotation: number | undefined; + const sourceRotation = getRotationFromAttributes(sourceFeature.attributes); + if (sourceFeature.bounds) { + if (sourceRotation !== undefined) { + const mapped = mapRotatedBounds(matrix, sourceFeature.bounds, sourceRotation); + mappedBounds = mapped.bounds; + mappedRotation = validateRotation(mapped.rotation); + } else { + mappedBounds = mapBounds(matrix, sourceFeature.bounds); + } + } + if (!targetTrack) { + targetTrack = trackStore.add( + targetFrame, + sourceTrack.confidencePairs[0][0], + undefined, + trackId, + ); + } + // setFeature only upserts geometry by (key, type): drop mirrored + // geometry the source no longer has so deletions propagate too. + const targetFeature = targetTrack.features[targetFrame]; + if (targetFeature?.geometry) { + const mappedKeys = new Set(mappedGeometry.map( + (geo) => `${geo.properties?.key ?? ''}|${geo.geometry.type}`, + )); + targetFeature.geometry.features = targetFeature.geometry.features.filter( + (geo) => mappedKeys.has(`${geo.properties?.key ?? ''}|${geo.geometry.type}`), + ); + } + targetTrack.setFeature({ + frame: targetFrame, + flick: sourceFeature.flick, + bounds: mappedBounds, + keyframe: true, + interpolate: sourceFeature.interpolate, + }, mappedGeometry); + if (mappedRotation !== undefined) { + targetTrack.setFeatureAttribute(targetFrame, ROTATION_ATTRIBUTE_NAME, mappedRotation); + } else { + const written = targetTrack.features[targetFrame]; + if (written?.attributes && ROTATION_ATTRIBUTE_NAME in written.attributes) { + targetTrack.setFeatureAttribute(targetFrame, ROTATION_ATTRIBUTE_NAME, undefined); + } + } + }); + } + function handleUpdateRectBounds(frameNum: number, flickNum: number, bounds: RectBounds, rotation?: number) { if (selectedTrackId.value !== null) { const track = cameraStore.getPossibleTrack(selectedTrackId.value, selectedCamera.value); @@ -588,6 +717,9 @@ export default function useModeManager({ if (isEditingExisting && track.attributes?.userCreated !== true) { track.setFeatureAttribute(frameNum, 'userModified', true); } + + mirrorFeatureToAlignedCameras(track.id, frameNum); + // Capture track ID before newTrackSettingsAfterLogic, which may // create a new track in continuous detection mode and change // selectedTrackId @@ -633,6 +765,8 @@ export default function useModeManager({ interpolate: _shouldInterpolate(interpolate), }, geometry); + mirrorFeatureToAlignedCameras(track.id, frameNum); + if (runAfterLogic) { newTrackSettingsAfterLogic(track); } @@ -759,6 +893,8 @@ export default function useModeManager({ track.setFeatureAttribute(frameNum, 'userModified', true); } + mirrorFeatureToAlignedCameras(track.id, frameNum); + // Only perform "initialization" after the first shape. // Treat this as a completed annotation if eventType is editing // Or none of the recieps reported that they were unfinished. @@ -833,6 +969,7 @@ export default function useModeManager({ ); } }); + mirrorFeatureToAlignedCameras(track.id, selectedCameraFrame()); } } handleSelectFeatureHandle(-1); @@ -869,6 +1006,8 @@ export default function useModeManager({ } } + mirrorFeatureToAlignedCameras(track.id, frameNum); + _nudgeEditingCanary(); } } @@ -1323,6 +1462,8 @@ export default function useModeManager({ interpolate, }, polygonGeometry); + mirrorFeatureToAlignedCameras(track.id, targetFrame); + _nudgeEditingCanary(); // Interactive stereo: as soon as the left polygon is predicted, generate @@ -1424,6 +1565,8 @@ export default function useModeManager({ interpolate, }, polygonGeometry); + mirrorFeatureToAlignedCameras(track.id, frameNum); + // Note: the other-camera (stereo) annotation is generated earlier, on // each fresh prediction (handleSegmentationPredictionReady), so there is // no need to regenerate it on confirm. @@ -1501,6 +1644,8 @@ export default function useModeManager({ : []); } + mirrorFeatureToAlignedCameras(track.id, data.frameNum); + if (onStereoAnnotationReset && stereoInteractiveActive()) { onStereoAnnotationReset({ trackId: selectedTrackId.value as number, diff --git a/client/src/AlignedViewStore.ts b/client/src/AlignedViewStore.ts new file mode 100644 index 000000000..0a603aabc --- /dev/null +++ b/client/src/AlignedViewStore.ts @@ -0,0 +1,110 @@ +import { + computed, ref, ComputedRef, Ref, +} from 'vue'; +import type { Matrix3, Point } from './homography'; +import { + cameraPairTransform, isIdentityMatrix3, mapPoint, +} from './alignedView'; + +/** + * Shared reactive state for the multicam "aligned view" toggle: when every + * non-reference camera has a stored/fitted transform into the reference + * camera's space, the user may warp each camera's display into that shared + * space during normal annotation review and link pan/zoom across all cameras + * (SEAL-TK features 2 + 3). + * + * Stored annotation geometry ALWAYS remains in native image space (decision + * D3); the transforms exposed here are applied at draw time only. + */ +export default class AlignedViewStore { + /** User toggle: whether the aligned view is requested. */ + enabled: Ref; + + /** Reference camera name (first camera in display order), null when unresolved. */ + reference: Ref; + + /** + * Per-camera native->reference matrices covering EVERY loaded camera, or + * null when at least one camera lacks a usable transform (see + * {@link resolveToReferenceTransforms}). + */ + toReference: Ref | null>; + + /** + * Externally suspended (e.g. while calibration point picking is active, + * which records raw native-space clicks and manages its own aligned + * preview). Suspension un-warps the display without losing the toggle. + */ + suspended: Ref; + + /** A usable transform exists for every camera, so the toggle may be shown. */ + available: ComputedRef; + + /** The aligned view is currently applied to rendering and navigation. */ + active: ComputedRef; + + constructor() { + this.enabled = ref(false); + this.reference = ref(null); + this.toReference = ref(null); + this.suspended = ref(false); + this.available = computed(() => this.reference.value !== null + && this.toReference.value !== null + && Object.keys(this.toReference.value).length > 1); + this.active = computed(() => this.enabled.value + && !this.suspended.value + && this.available.value); + } + + /** Replace the resolved transform set (null when unavailable). */ + setTransforms(reference: string | null, toReference: Record | null) { + this.reference.value = reference; + this.toReference.value = toReference; + } + + setEnabled(enabled: boolean) { + this.enabled.value = enabled; + } + + setSuspended(suspended: boolean) { + this.suspended.value = suspended; + } + + /** + * Display transform for `camera` (native -> aligned/reference space), or + * null when the aligned view is inactive or the camera renders unwarped + * (the reference camera / identity). Callers treat null as "draw exactly + * as today", keeping the off state byte-identical to current behavior. + */ + cameraTransform(camera: string): Matrix3 | null { + if (!this.active.value || !this.toReference.value) { + return null; + } + const matrix = this.toReference.value[camera]; + if (!matrix || isIdentityMatrix3(matrix)) { + return null; + } + return matrix; + } + + /** + * Matrix mapping `from`-camera native pixels onto `to`-camera native + * pixels, composed through the reference space. Null when inactive or + * either camera is unresolved. + */ + cameraToCamera(from: string, to: string): Matrix3 | null { + if (!this.active.value || !this.toReference.value) { + return null; + } + return cameraPairTransform(this.toReference.value, from, to); + } + + /** Map a native-space point of `from` onto `to`'s native space (null when unavailable). */ + mapCameraPoint(from: string, to: string, point: Point): Point | null { + const matrix = this.cameraToCamera(from, to); + if (!matrix) { + return null; + } + return mapPoint(matrix, point); + } +} diff --git a/client/src/alignedView.spec.ts b/client/src/alignedView.spec.ts new file mode 100644 index 000000000..bb5c6acf3 --- /dev/null +++ b/client/src/alignedView.spec.ts @@ -0,0 +1,301 @@ +/// +import { + IDENTITY3, + isIdentityMatrix3, + readTransformMatrix, + composeThroughPairs, + resolveToReferenceTransforms, + unresolvedCameras, + cameraPairTransform, + mapPoint, + mapBounds, + mapRotatedBounds, + mapGeoJSONFeatures, +} from './alignedView'; +import { applyHomography, Matrix3, Point } from './homography'; +import type { CameraHomographies } from './CameraCalibrationStore'; +import AlignedViewStore from './AlignedViewStore'; + +/** Simple affine helpers for readable fixtures. */ +function translation(tx: number, ty: number): Matrix3 { + return [[1, 0, tx], [0, 1, ty], [0, 0, 1]]; +} +function scale(s: number): Matrix3 { + return [[s, 0, 0], [0, s, 0], [0, 0, 1]]; +} + +function expectPointClose(actual: Point, expected: Point) { + expect(actual[0]).toBeCloseTo(expected[0], 6); + expect(actual[1]).toBeCloseTo(expected[1], 6); +} + +describe('isIdentityMatrix3', () => { + it('accepts the identity and rejects non-identity', () => { + expect(isIdentityMatrix3(IDENTITY3)).toBe(true); + expect(isIdentityMatrix3(translation(0, 0))).toBe(true); + expect(isIdentityMatrix3(translation(1, 0))).toBe(false); + expect(isIdentityMatrix3(scale(2))).toBe(false); + }); +}); + +describe('readTransformMatrix', () => { + it('accepts a valid row-major 3x3', () => { + const m = readTransformMatrix([[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + expect(m).toEqual([[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); + it('rejects malformed shapes', () => { + expect(readTransformMatrix(undefined)).toBeNull(); + expect(readTransformMatrix(null)).toBeNull(); + expect(readTransformMatrix('matrix')).toBeNull(); + expect(readTransformMatrix([[1, 0], [0, 1]])).toBeNull(); + expect(readTransformMatrix([[1, 0, 0], [0, 1, 0]])).toBeNull(); + expect(readTransformMatrix([[1, 0, 0], [0, 1, 0], [0, 0]])).toBeNull(); + }); + it('rejects non-finite values', () => { + expect(readTransformMatrix([[1, 0, NaN], [0, 1, 0], [0, 0, 1]])).toBeNull(); + expect(readTransformMatrix([[1, 0, Infinity], [0, 1, 0], [0, 0, 1]])).toBeNull(); + expect(readTransformMatrix([[1, 0, '5'], [0, 1, 'abc'], [0, 0, 1]])).toBeNull(); + }); + it('rejects singular matrices', () => { + expect(readTransformMatrix([[1, 1, 0], [1, 1, 0], [0, 0, 1]])).toBeNull(); + expect(readTransformMatrix([[0, 0, 0], [0, 0, 0], [0, 0, 0]])).toBeNull(); + }); +}); + +describe('composeThroughPairs', () => { + const irToEo = translation(100, 50); + const uvToIr = scale(2); + const homographies: CameraHomographies = { + // eo::ir stored with AtoB = eo->ir, so BtoA = ir->eo. + 'eo::ir': { AtoB: [[1, 0, -100], [0, 1, -50], [0, 0, 1]], BtoA: irToEo }, + // uv::ir stored with AtoB = uv->ir. + 'uv::ir': { AtoB: uvToIr, BtoA: [[0.5, 0, 0], [0, 0.5, 0], [0, 0, 1]] }, + }; + it('returns identity for the reference itself', () => { + expect(composeThroughPairs('eo', 'eo', homographies)).toEqual(IDENTITY3); + }); + it('uses a direct pair edge in either direction', () => { + const m = composeThroughPairs('ir', 'eo', homographies); + expect(m).not.toBeNull(); + expectPointClose(applyHomography(m as Matrix3, [10, 20]), [110, 70]); + }); + it('composes multi-hop paths (uv -> ir -> eo)', () => { + const m = composeThroughPairs('uv', 'eo', homographies); + expect(m).not.toBeNull(); + // uv (x, y) -> ir (2x, 2y) -> eo (2x + 100, 2y + 50). + expectPointClose(applyHomography(m as Matrix3, [10, 20]), [120, 90]); + }); + it('returns null when no path exists', () => { + expect(composeThroughPairs('flir', 'eo', homographies)).toBeNull(); + expect(composeThroughPairs('ir', 'eo', {})).toBeNull(); + }); +}); + +describe('resolveToReferenceTransforms', () => { + const cameras = ['eo', 'ir', 'uv']; + const homographies: CameraHomographies = { + 'eo::ir': { AtoB: translation(-100, 0), BtoA: translation(100, 0) }, + 'uv::ir': { AtoB: scale(2), BtoA: scale(0.5) }, + }; + it('resolves every camera through calibration pairs', () => { + const result = resolveToReferenceTransforms(cameras, 'eo', homographies); + expect(result).not.toBeNull(); + const transforms = result as Record; + expect(transforms.eo).toEqual(IDENTITY3); + expectPointClose(applyHomography(transforms.ir, [5, 5]), [105, 5]); + expectPointClose(applyHomography(transforms.uv, [5, 5]), [110, 10]); + }); + it('is all-or-none: any unresolved camera fails the whole set', () => { + expect(resolveToReferenceTransforms(['eo', 'ir', 'flir'], 'eo', homographies)).toBeNull(); + expect(resolveToReferenceTransforms(cameras, 'eo', {})).toBeNull(); + }); + it('fails when the reference is not among the cameras or the list is empty', () => { + expect(resolveToReferenceTransforms([], 'eo', homographies)).toBeNull(); + expect(resolveToReferenceTransforms(['ir', 'uv'], 'eo', homographies)).toBeNull(); + }); +}); + +describe('unresolvedCameras', () => { + const homographies: CameraHomographies = { + 'eo::ir': { AtoB: translation(-100, 0), BtoA: translation(100, 0) }, + }; + it('names cameras with no path to the reference', () => { + expect(unresolvedCameras(['eo', 'ir', 'uv'], 'eo', homographies)).toEqual(['uv']); + expect(unresolvedCameras(['eo', 'ir', 'uv'], 'eo', {})).toEqual(['ir', 'uv']); + }); + it('is empty when every camera resolves', () => { + expect(unresolvedCameras(['eo', 'ir'], 'eo', homographies)).toEqual([]); + }); +}); + +describe('cameraPairTransform', () => { + it('composes from -> reference -> to', () => { + const toReference = { + eo: IDENTITY3, + ir: translation(100, 0), + uv: scale(2), + }; + // ir (x, y) -> ref (x + 100, y) -> uv ((x + 100) / 2, y / 2). + const m = cameraPairTransform(toReference, 'ir', 'uv'); + expect(m).not.toBeNull(); + expectPointClose(applyHomography(m as Matrix3, [10, 20]), [55, 10]); + }); + it('returns null for unresolved cameras', () => { + expect(cameraPairTransform({ eo: IDENTITY3 }, 'eo', 'ir')).toBeNull(); + expect(cameraPairTransform({ eo: IDENTITY3 }, 'ir', 'eo')).toBeNull(); + }); +}); + +describe('mapPoint', () => { + it('is identity for a null matrix', () => { + expect(mapPoint(null, [3, 4])).toEqual([3, 4]); + }); + it('applies the matrix otherwise', () => { + expectPointClose(mapPoint(translation(1, 2), [3, 4]), [4, 6]); + }); +}); + +describe('mapBounds', () => { + it('translates and scales an axis-aligned box', () => { + expect(mapBounds(translation(10, -5), [0, 0, 4, 6])).toEqual([10, -5, 14, 1]); + expect(mapBounds(scale(2), [1, 2, 3, 4])).toEqual([2, 4, 6, 8]); + }); + it('returns the AABB of the mapped corners under rotation', () => { + // 90-degree rotation about the origin: (x, y) -> (-y, x). + const rot90: Matrix3 = [[0, -1, 0], [1, 0, 0], [0, 0, 1]]; + const [x1, y1, x2, y2] = mapBounds(rot90, [0, 0, 4, 2]); + expect(x1).toBeCloseTo(-2, 6); + expect(y1).toBeCloseTo(0, 6); + expect(x2).toBeCloseTo(0, 6); + expect(y2).toBeCloseTo(4, 6); + }); +}); + +describe('mapRotatedBounds', () => { + const rot = (angle: number): Matrix3 => [ + [Math.cos(angle), -Math.sin(angle), 0], + [Math.sin(angle), Math.cos(angle), 0], + [0, 0, 1], + ]; + it('adds the transform rotation and keeps the rect size under similarity', () => { + const theta = Math.PI / 6; + const result = mapRotatedBounds(rot(theta), [10, 20, 50, 40], Math.PI / 8); + expect(result.rotation).toBeCloseTo(Math.PI / 8 + theta, 6); + // The rect's own dimensions (40 x 20) are preserved by a pure rotation. + expect(result.bounds[2] - result.bounds[0]).toBeCloseTo(40, 6); + expect(result.bounds[3] - result.bounds[1]).toBeCloseTo(20, 6); + }); + it('is a passthrough for the identity', () => { + const result = mapRotatedBounds(IDENTITY3, [10, 20, 50, 40], 0.3); + expect(result.rotation).toBeCloseTo(0.3, 6); + expect(result.bounds[0]).toBeCloseTo(10, 6); + expect(result.bounds[1]).toBeCloseTo(20, 6); + expect(result.bounds[2]).toBeCloseTo(50, 6); + expect(result.bounds[3]).toBeCloseTo(40, 6); + }); + it('scales the rect and centers under translation + scale', () => { + const matrix = [[2, 0, 100], [0, 2, -10], [0, 0, 1]]; + const result = mapRotatedBounds(matrix, [0, 0, 10, 20], 0.5); + expect(result.rotation).toBeCloseTo(0.5, 6); + expect(result.bounds[2] - result.bounds[0]).toBeCloseTo(20, 6); + expect(result.bounds[3] - result.bounds[1]).toBeCloseTo(40, 6); + // The center (5, 10) maps to (110, 10). + expect((result.bounds[0] + result.bounds[2]) / 2).toBeCloseTo(110, 6); + expect((result.bounds[1] + result.bounds[3]) / 2).toBeCloseTo(10, 6); + }); +}); + +describe('mapGeoJSONFeatures', () => { + it('maps points, lines, and polygon rings without mutating the source', () => { + const source: GeoJSON.Feature[] = [ + { + type: 'Feature', + geometry: { type: 'Point', coordinates: [1, 2] }, + properties: { key: 'head' }, + }, + { + type: 'Feature', + geometry: { type: 'LineString', coordinates: [[0, 0], [3, 4]] }, + properties: { key: 'HeadTails' }, + }, + { + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [[[0, 0], [4, 0], [4, 4], [0, 0]], [[1, 1], [2, 1], [2, 2], [1, 1]]], + }, + properties: { key: '' }, + }, + ]; + const mapped = mapGeoJSONFeatures(translation(10, 20), source); + expect(mapped[0].geometry).toEqual({ type: 'Point', coordinates: [11, 22] }); + expect(mapped[1].geometry).toEqual({ type: 'LineString', coordinates: [[10, 20], [13, 24]] }); + expect(mapped[2].geometry.type).toBe('Polygon'); + expect((mapped[2].geometry as GeoJSON.Polygon).coordinates[1][2]).toEqual([12, 22]); + expect(mapped[0].properties).toEqual({ key: 'head' }); + // Deep copy: the source geometry/properties are untouched and unshared. + expect(source[0].geometry).toEqual({ type: 'Point', coordinates: [1, 2] }); + expect(mapped[0].properties).not.toBe(source[0].properties); + expect(mapped[2].geometry).not.toBe(source[2].geometry); + }); +}); + +describe('AlignedViewStore', () => { + function makeResolvedStore() { + const store = new AlignedViewStore(); + store.setTransforms('eo', { + eo: IDENTITY3, + ir: translation(100, 0), + }); + return store; + } + + it('is unavailable and inactive by default', () => { + const store = new AlignedViewStore(); + expect(store.available.value).toBe(false); + store.setEnabled(true); + expect(store.active.value).toBe(false); + expect(store.cameraTransform('ir')).toBeNull(); + expect(store.cameraToCamera('eo', 'ir')).toBeNull(); + }); + + it('activates only when enabled, available, and not suspended', () => { + const store = makeResolvedStore(); + expect(store.available.value).toBe(true); + expect(store.active.value).toBe(false); + store.setEnabled(true); + expect(store.active.value).toBe(true); + store.setSuspended(true); + expect(store.active.value).toBe(false); + expect(store.cameraTransform('ir')).toBeNull(); + store.setSuspended(false); + expect(store.active.value).toBe(true); + }); + + it('exposes a display transform for warped cameras and null for the reference', () => { + const store = makeResolvedStore(); + store.setEnabled(true); + expect(store.cameraTransform('eo')).toBeNull(); + const m = store.cameraTransform('ir'); + expect(m).not.toBeNull(); + expectPointClose(applyHomography(m as Matrix3, [1, 1]), [101, 1]); + // Unknown camera also renders unwarped. + expect(store.cameraTransform('uv')).toBeNull(); + }); + + it('maps cross-camera points through the reference', () => { + const store = makeResolvedStore(); + store.setEnabled(true); + expectPointClose(store.mapCameraPoint('ir', 'eo', [0, 0]) as Point, [100, 0]); + expectPointClose(store.mapCameraPoint('eo', 'ir', [100, 0]) as Point, [0, 0]); + expect(store.mapCameraPoint('eo', 'flir', [0, 0])).toBeNull(); + }); + + it('becomes unavailable when a camera set collapses to one entry', () => { + const store = new AlignedViewStore(); + store.setTransforms('eo', { eo: IDENTITY3 }); + store.setEnabled(true); + expect(store.available.value).toBe(false); + expect(store.active.value).toBe(false); + }); +}); diff --git a/client/src/alignedView.ts b/client/src/alignedView.ts new file mode 100644 index 000000000..eed028c0e --- /dev/null +++ b/client/src/alignedView.ts @@ -0,0 +1,305 @@ +/** + * Pure helpers for the multicam "aligned view" (SEAL-TK features 2 + 3): + * resolving a per-camera native->reference-space transform for every loaded + * camera from the available sources, and composing camera-to-camera mappings + * through the reference camera. + * + * Conventions (documented assumptions, see migration plan Q3): + * - The REFERENCE camera is the first camera in display order + * (`meta.multiCamMedia.cameraOrder[0]`, i.e. `multiCamList[0]`). Its + * transform is the identity. + * - Calibration-tool homographies are stored per directional pair key + * `camA::camB` with `AtoB` mapping camA pixels onto camB (and `BtoA` the + * inverse); a camera's path to the reference is found by composing pair + * edges (breadth-first, so up-to-3-camera rigs may chain e.g. UV->IR->EO). + * - The calibration store is the SINGLE source the viewer resolves from: + * whatever the calibration panel shows/saves is what the Align button + * applies. External calibration .json files enter that same store, either + * through the panel's Load calibration button or by seeding the saved + * calibration at multicam import time. + */ +import { + Matrix3, matMul3, invert3, applyHomography, Point, +} from './homography'; +import type { CameraHomographies } from './CameraCalibrationStore'; +import type { RectBounds } from './utils'; + +export const IDENTITY3: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + +/** True when `m` is (numerically) the identity transform. */ +export function isIdentityMatrix3(m: Matrix3, eps = 1e-9): boolean { + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + if (Math.abs(m[i][j] - IDENTITY3[i][j]) > eps) { + return false; + } + } + } + return true; +} + +/** + * Defensively validate an untrusted value (e.g. read from dataset meta JSON) + * as a row-major 3x3 matrix usable as a display transform. Returns null for + * anything malformed, non-finite, or singular (non-invertible) rather than + * throwing, so callers can treat "bad matrix" as "no matrix". + */ +export function readTransformMatrix(raw: unknown): Matrix3 | null { + if (!Array.isArray(raw) || raw.length !== 3) { + return null; + } + const m: number[][] = []; + for (let i = 0; i < 3; i += 1) { + const row = raw[i]; + if (!Array.isArray(row) || row.length !== 3) { + return null; + } + const nums = row.map(Number); + if (nums.some((v) => !Number.isFinite(v))) { + return null; + } + m.push(nums); + } + try { + invert3(m); + } catch { + return null; + } + return m; +} + +/** One directed edge of the calibration-pair graph: `matrix` maps `from` pixels onto `to`. */ +interface PairEdge { + to: string; + matrix: Matrix3; +} + +/** Build the bidirectional camera adjacency from calibration pair homographies. */ +function buildPairGraph(homographies: CameraHomographies): Record { + const graph: Record = {}; + const addEdge = (from: string, to: string, matrix: Matrix3) => { + if (!graph[from]) { + graph[from] = []; + } + graph[from].push({ to, matrix }); + }; + Object.entries(homographies).forEach(([key, pair]) => { + const [camA, camB] = key.split('::'); + if (!camA || !camB || camA === camB || !pair) { + return; + } + addEdge(camA, camB, pair.AtoB); + addEdge(camB, camA, pair.BtoA); + }); + return graph; +} + +/** + * Compose a `camera` -> `reference` matrix by walking the calibration pair + * graph breadth-first (shortest hop count). Returns null when no path exists. + */ +export function composeThroughPairs( + camera: string, + reference: string, + homographies: CameraHomographies, +): Matrix3 | null { + if (camera === reference) { + return IDENTITY3; + } + const graph = buildPairGraph(homographies); + // BFS queue of (camera, accumulated camera->node matrix). + const queue: { node: string; matrix: Matrix3 }[] = [{ node: camera, matrix: IDENTITY3 }]; + const visited = new Set([camera]); + while (queue.length) { + const { node, matrix } = queue.shift() as { node: string; matrix: Matrix3 }; + const edges = graph[node] || []; + for (let i = 0; i < edges.length; i += 1) { + const edge = edges[i]; + if (!visited.has(edge.to)) { + // p_to = edge.matrix * p_node and p_node = matrix * p_camera. + const composed = matMul3(edge.matrix, matrix); + if (edge.to === reference) { + return composed; + } + visited.add(edge.to); + queue.push({ node: edge.to, matrix: composed }); + } + } + } + return null; +} + +/** + * Resolve a native->reference matrix for EVERY camera in `cameras` from the + * calibration store's pair homographies, or null when any camera lacks a + * usable transform (the aligned view is all-or-none: a partially-aligned + * display would be misleading). The reference camera always maps by the + * identity. + */ +export function resolveToReferenceTransforms( + cameras: string[], + reference: string, + homographies: CameraHomographies, +): Record | null { + if (!cameras.length || !cameras.includes(reference)) { + return null; + } + const result: Record = {}; + for (let i = 0; i < cameras.length; i += 1) { + const camera = cameras[i]; + if (camera === reference) { + result[camera] = IDENTITY3; + } else { + const matrix = composeThroughPairs(camera, reference, homographies); + if (!matrix) { + return null; + } + result[camera] = matrix; + } + } + return result; +} + +/** + * The cameras in `cameras` with no composed transform path to `reference` -- + * the reason {@link resolveToReferenceTransforms} returned null, named so the + * UI can say exactly which pair(s) still need calibrating instead of silently + * hiding the aligned view. + */ +export function unresolvedCameras( + cameras: string[], + reference: string, + homographies: CameraHomographies, +): string[] { + return cameras.filter( + (camera) => camera !== reference + && composeThroughPairs(camera, reference, homographies) === null, + ); +} + +/** + * Matrix mapping `from`-camera native pixels onto `to`-camera native pixels, + * composed through the shared reference space: + * p_to = inv(T_to->ref) * T_from->ref * p_from. + * Returns null when either camera has no resolved transform. + */ +export function cameraPairTransform( + toReference: Record, + from: string, + to: string, +): Matrix3 | null { + const fromRef = toReference[from]; + const toRef = toReference[to]; + if (!fromRef || !toRef) { + return null; + } + try { + return matMul3(invert3(toRef), fromRef); + } catch { + return null; + } +} + +/** Map a point through a nullable transform (identity when null). */ +export function mapPoint(matrix: Matrix3 | null, point: Point): Point { + if (!matrix) { + return point; + } + return applyHomography(matrix, point); +} + +/** Axis-aligned bounding box of a set of points. */ +function pointsToBounds(points: Point[]): RectBounds { + const xs = points.map((p) => p[0]); + const ys = points.map((p) => p[1]); + return [Math.min(...xs), Math.min(...ys), Math.max(...xs), Math.max(...ys)]; +} + +/** The four corners of a RectBounds, clockwise from (x1, y1). */ +function boundsCorners(bounds: RectBounds): Point[] { + return [ + [bounds[0], bounds[1]], + [bounds[2], bounds[1]], + [bounds[2], bounds[3]], + [bounds[0], bounds[3]], + ]; +} + +/** + * Map an axis-aligned RectBounds through a homography. A non-affine transform + * turns a rectangle into a general quad, so the result is the axis-aligned + * bounding box of the four mapped corners. + */ +export function mapBounds(matrix: Matrix3, bounds: RectBounds): RectBounds { + return pointsToBounds(boundsCorners(bounds).map((c) => applyHomography(matrix, c))); +} + +/** + * Map a rotated rectangle (RectBounds + rotation in radians about the bounds + * center, the convention of utils.rotateGeoJSONCoordinates) through a + * homography, returning a best-fit rotated rectangle in the target space: + * the mapped direction of the rect's top edge gives the new rotation, and the + * mapped corners un-rotated about their centroid give the new bounds. Exact + * for similarity transforms; a least-surprise approximation for the rest. + */ +export function mapRotatedBounds( + matrix: Matrix3, + bounds: RectBounds, + rotation: number, +): { bounds: RectBounds; rotation: number } { + const cx = (bounds[0] + bounds[2]) / 2; + const cy = (bounds[1] + bounds[3]) / 2; + const rotate = (p: Point, angle: number, ox: number, oy: number): Point => { + const cos = Math.cos(angle); + const sin = Math.sin(angle); + const dx = p[0] - ox; + const dy = p[1] - oy; + return [ox + dx * cos - dy * sin, oy + dx * sin + dy * cos]; + }; + const mapped = boundsCorners(bounds) + .map((c) => applyHomography(matrix, rotate(c, rotation, cx, cy))); + // The top edge (corner 0 -> corner 1) points along +x at zero rotation, so + // its mapped direction is the combined rotation in the target space. + const newRotation = Math.atan2( + mapped[1][1] - mapped[0][1], + mapped[1][0] - mapped[0][0], + ); + const mcx = mapped.reduce((sum, p) => sum + p[0], 0) / mapped.length; + const mcy = mapped.reduce((sum, p) => sum + p[1], 0) / mapped.length; + const unrotated = mapped.map((p) => rotate(p, -newRotation, mcx, mcy)); + return { bounds: pointsToBounds(unrotated), rotation: newRotation }; +} + +type MappableGeometry = GeoJSON.Point | GeoJSON.LineString | GeoJSON.Polygon; + +/** + * Deep-copy GeoJSON features (the shapes tracks store per-frame: Point, + * LineString, Polygon) with every coordinate mapped through a homography. + * Unsupported geometry types are copied through unmapped. + */ +export function mapGeoJSONFeatures( + matrix: Matrix3, + features: GeoJSON.Feature[], +): GeoJSON.Feature[] { + const mapPosition = (p: GeoJSON.Position): GeoJSON.Position => ( + applyHomography(matrix, [p[0], p[1]]) + ); + return features.map((feature) => { + let geometry: MappableGeometry; + if (feature.geometry.type === 'Point') { + geometry = { type: 'Point', coordinates: mapPosition(feature.geometry.coordinates) }; + } else if (feature.geometry.type === 'LineString') { + geometry = { type: 'LineString', coordinates: feature.geometry.coordinates.map(mapPosition) }; + } else { + geometry = { + type: 'Polygon', + coordinates: feature.geometry.coordinates.map((ring) => ring.map(mapPosition)), + }; + } + return { + ...feature, + geometry: geometry as T, + properties: feature.properties ? { ...feature.properties } : feature.properties, + }; + }); +} diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 376ef45cc..82ffdccb3 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -15,13 +15,18 @@ import OverlapLayer from '../layers/AnnotationLayers/OverlapLayer'; import EditAnnotationLayer, { EditAnnotationTypes } from '../layers/EditAnnotationLayer'; import LassoSelectionLayer from '../layers/LassoSelectionLayer'; +import AlignedImageLayer from '../layers/AlignedImageLayer'; import { FrameDataTrack } from '../layers/LayerTypes'; +import { applyHomography, invert3, Matrix3 } from '../homography'; +import { mapBounds, mapRotatedBounds, mapGeoJSONFeatures } from '../alignedView'; +import type { Feature } from '../track'; import TextLayer, { FormatTextRow } from '../layers/AnnotationLayers/TextLayer'; import AttributeLayer from '../layers/AnnotationLayers/AttributeLayer'; import AttributeBoxLayer from '../layers/AnnotationLayers/AttributeBoxLayer'; import type { AnnotationId } from '../BaseAnnotation'; import { geojsonToBound, isRotationValue, ROTATION_ATTRIBUTE_NAME, featureHasSegmentationPolygon, + getRotationFromAttributes, } from '../utils'; import { VisibleAnnotationTypes } from '../layers'; import UILayer from '../layers/UILayers/UILayer'; @@ -39,6 +44,7 @@ import { useAnnotatorPreferences, useGroupStyleManager, useCameraStore, + useAlignedView, useSelectedCamera, useAttributes, useComparisonSets, @@ -77,6 +83,12 @@ export default defineComponent({ // Viewer may not provide lasso context in tests or minimal embeds. } const cameraStore = useCameraStore(); + let alignedView: ReturnType | undefined; + try { + alignedView = useAlignedView(); + } catch { + // aligned view store may not be provided in tests or minimal embeds. + } const selectedCamera = useSelectedCamera(); const comparison = useComparisonSets(); const trackStore = cameraStore.camMap.value.get(props.camera)?.trackStore; @@ -107,6 +119,144 @@ export default defineComponent({ const flickNumberRef = annotator.flick; const hasFrameRef = annotator.hasFrame; + /** + * Resolve another camera's currently displayed frame image (for the ghost + * overlay and the aligned-view warp). Matches the `quad.image` data used + * by ImageAnnotator and the `quad.video` data used by VideoAnnotator -- + * geojs' canvas quad renderer supports both as texture sources. + * LargeImageAnnotator (tiled/geospatial imagery) has no single resolvable + * image element, so it returns null and the ghost overlay / display warp + * is simply unavailable for those datasets; picking itself + * (native-coordinate inverse mapping) is unaffected either way. + */ + const getCameraImage = (camera: string) => { + let viewer; + try { + // getController throws for an unknown/cleared camera; the ghost and + // aligned-warp rAF loops call this after a dataset reload has cleared + // the controllers, so swallow it here rather than let it escape into + // the animation-frame callback uncaught. + viewer = aggregateController.value.getController(camera)?.geoViewerRef?.value; + } catch { + return null; + } + if (!viewer || typeof viewer.layers !== 'function') { + return null; + } + const layerList = viewer.layers(); + for (let i = 0; i < layerList.length; i += 1) { + const layer = layerList[i]; + if (typeof layer.features === 'function') { + const features = layer.features(); + for (let j = 0; j < features.length; j += 1) { + const data = typeof features[j].data === 'function' ? features[j].data() : undefined; + const datum = Array.isArray(data) ? data[0] : undefined; + if (datum && datum.image) { + const image = datum.image as HTMLImageElement; + return { + source: image, kind: 'image' as const, width: image.naturalWidth, height: image.naturalHeight, + }; + } + if (datum && datum.video) { + const video = datum.video as HTMLVideoElement; + return { + source: video, kind: 'video' as const, width: video.videoWidth, height: video.videoHeight, + }; + } + } + } + } + return null; + }; + + /** + * Aligned view (SEAL-TK features 2 + 3): while active, this camera's + * display transform (native -> reference space, null when unwarped). + * Stored geometry stays native (decision D3); the transform is applied + * at draw time only. + */ + const alignedDisplayTransform = computed( + () => (alignedView ? alignedView.cameraTransform(props.camera) : null), + ); + /** + * Inverse of the display transform (reference/display space -> this + * camera's native space). The edit layer operates in geojs map + * coordinates -- display space -- so draws and edits made while the + * aligned view warps this camera must be mapped back through this before + * being committed to (native) track storage. + */ + const alignedDisplayInverse = computed(() => { + const matrix = alignedDisplayTransform.value; + if (!matrix) { + return null; + } + try { + return invert3(matrix); + } catch { + return null; + } + }); + /** Map a native-space location into display space for view centering. */ + const mapDisplayPoint = (x: number, y: number) => { + const matrix = alignedDisplayTransform.value; + if (!matrix) { + return { x, y }; + } + const [mx, my] = applyHomography(matrix, [x, y]); + return { x: mx, y: my }; + }; + /** + * Copy a native-space track feature into display space for the edit + * layer (identity passthrough when this camera renders unwarped), so + * edit handles land on the warped imagery. The stored feature is never + * mutated (decision D3: storage stays native). + */ + function featureToDisplay(feature: Feature | null): Feature | null { + const matrix = alignedDisplayTransform.value; + if (!matrix || !feature) { + return feature; + } + const mapped: Feature = { ...feature }; + const rotation = getRotationFromAttributes(feature.attributes); + if (feature.bounds) { + if (rotation !== undefined) { + const rotated = mapRotatedBounds(matrix, feature.bounds, rotation); + mapped.bounds = rotated.bounds; + mapped.attributes = { + ...feature.attributes, + [ROTATION_ATTRIBUTE_NAME]: rotated.rotation, + }; + } else { + mapped.bounds = mapBounds(matrix, feature.bounds); + } + } + if (feature.geometry) { + mapped.geometry = { + ...feature.geometry, + features: mapGeoJSONFeatures(matrix, feature.geometry.features), + }; + } + return mapped; + } + + // Created before the annotation layers below so its geojs layer z-orders + // beneath boxes/polygons/text (geojs stacks layers by creation order). + const alignedImageLayer = new AlignedImageLayer({ + annotator, + getImage: () => { + try { + return getCameraImage(props.camera); + } catch { + // Controllers may be cleared mid-poll during a dataset reload. + return null; + } + }, + getTransform: () => alignedDisplayTransform.value, + // Right-click means "remove last point" while creating/editing + // geometry; recenter everywhere else. + getRecenterEnabled: () => !editingModeRef.value, + }); + const rectAnnotationLayer = new RectangleLayer({ annotator, stateStyling: trackStyleManager.stateStyles, @@ -187,7 +337,13 @@ export default defineComponent({ watch([segmentationPointsRef, frameNumberRef, selectedCamera], ([newPoints, currentFrame, currentCamera]) => { if (newPoints.points.length > 0 && newPoints.frameNum === currentFrame && props.camera === currentCamera) { - segmentationPointsLayer.updatePoints(newPoints.points, newPoints.labels); + // Prompt points are stored in native image space; render them where + // the warped imagery actually is (identity when unwarped). + const displayPoints = newPoints.points.map((p): [number, number] => { + const { x, y } = mapDisplayPoint(p[0], p[1]); + return [x, y]; + }); + segmentationPointsLayer.updatePoints(displayPoints, newPoints.labels); } else { segmentationPointsLayer.clear(); } @@ -262,6 +418,11 @@ export default defineComponent({ selectedKey: string, colorBy: string, ) { + // Drawing and editing work on every camera while the aligned view is + // on: the edit layer operates in display (warped) space -- it is fed + // display-space copies of the geometry (featureToDisplay below) and its + // draws/edits are mapped back to native through alignedDisplayInverse + // in the update:geojson handler before committing to track storage. const currentFrameIds: AnnotationId[] | undefined = trackStore?.intervalTree .search([frame, frame]) .map((str) => parseInt(str, 10)); @@ -320,9 +481,13 @@ export default defineComponent({ } if (clientSettings.annotatorPreferences.lockedCamera.enabled) { if (trackFrame.features?.bounds) { + // Under the aligned view the display is warped, so center + // on the displayed (warped) location, not the native one. const coords = { - x: (trackFrame.features.bounds[0] + trackFrame.features.bounds[2]) / 2.0, - y: (trackFrame.features.bounds[1] + trackFrame.features.bounds[3]) / 2.0, + ...mapDisplayPoint( + (trackFrame.features.bounds[0] + trackFrame.features.bounds[2]) / 2.0, + (trackFrame.features.bounds[1] + trackFrame.features.bounds[3]) / 2.0, + ), z: 0, }; const [x0, y0, x1, y1] = trackFrame.features.bounds; @@ -341,10 +506,14 @@ export default defineComponent({ const halfWidth = (width * multiplyBoundsVal) / 2.0; const halfHeight = (height * multiplyBoundsVal) / 2.0; - const left = centerX - halfWidth; - const right = centerX + halfWidth; - const top = centerY - halfHeight; - const bottom = centerY + halfHeight; + // Map the zoom-target corners into display space too + // (identity when the aligned view is off). + const ulMapped = mapDisplayPoint(centerX - halfWidth, centerY - halfHeight); + const lrMapped = mapDisplayPoint(centerX + halfWidth, centerY + halfHeight); + const left = Math.min(ulMapped.x, lrMapped.x); + const right = Math.max(ulMapped.x, lrMapped.x); + const top = Math.min(ulMapped.y, lrMapped.y); + const bottom = Math.max(ulMapped.y, lrMapped.y); const zoomAndCenter = annotator.geoViewerRef.value.zoomAndCenterFromBounds({ left, top, right, bottom, @@ -384,7 +553,11 @@ export default defineComponent({ } else { lineLayer.disable(); } - if (visibleModes.includes('TrackTail')) { + // Track tails read multi-frame geometry straight from the trackStore + // (not FrameDataTrack) and are not routed through the display + // transform, so they are hidden for warped cameras while the aligned + // view is on rather than rendered in the wrong (native) space. + if (visibleModes.includes('TrackTail') && !alignedDisplayTransform.value) { tailLayer.updateSettings( frame, annotatorPrefs.value.trackTails.before, @@ -432,7 +605,13 @@ export default defineComponent({ if (editingTrack) { editAnnotationLayer.setType(editingTrack); editAnnotationLayer.setKey(selectedKey); - editAnnotationLayer.changeData(editingTracks); + // The edit layer works in display space: hand it display-space + // copies of the feature so its handles land on warped imagery + // (identity when this camera renders unwarped). + editAnnotationLayer.changeData(editingTracks.map((trackFrame) => ({ + ...trackFrame, + features: featureToDisplay(trackFrame.features), + }))); } } else if (editingTrack && props.camera !== selectedCamera.value && (isCreatingNewDetection(frame, selectedTrackId) @@ -506,6 +685,49 @@ export default defineComponent({ refreshLayers(); }); + /** Layers whose stored-geometry rendering follows the aligned-view warp. */ + const displayTransformedLayers = [ + rectAnnotationLayer, + overlapLayer, + polyAnnotationLayer, + lineLayer, + pointLayer, + textLayer, + attributeBoxLayer, + attributeLayer, + ]; + + /** + * Apply (or clear) the aligned-view display transform: warp the imagery + * quad and point every geometry layer's draw-time mapping at the same + * matrix, then re-render. Immediate so a LayerManager created while the + * aligned view is already on (e.g. a view-mode switch) starts warped. + */ + watch(alignedDisplayTransform, (matrix) => { + displayTransformedLayers.forEach((layer) => layer.setDisplayTransform(matrix)); + alignedImageLayer.update(); + updateLayers( + frameNumberRef.value, + editingModeRef.value, + selectedTrackIdRef.value, + multiSeletListRef.value, + enabledTracksRef.value, + visibleModesRef.value, + selectedKeyRef.value, + props.colorBy, + ); + }, { immediate: true }); + + // The warped imagery must follow frame changes: the annotator swaps its + // element asynchronously after each seek, and AlignedImageLayer + // polls briefly after every trigger to catch that swap. Guarded so this + // is a strict no-op whenever the camera renders unwarped. + watch(frameNumberRef, () => { + if (alignedDisplayTransform.value) { + alignedImageLayer.update(); + } + }); + /** Shallow watch */ watch( [ @@ -745,16 +967,31 @@ export default defineComponent({ return; } } + // Under the aligned view this camera renders warped, so the draw/edit + // just made lives in display (reference) space: map it back to this + // camera's native space before committing to track storage (decision + // D3 -- storage stays native). Identity when the camera is unwarped. + const inverse = alignedDisplayInverse.value; if (type === 'rectangle') { - const bounds = geojsonToBound(data as GeoJSON.Feature); + let bounds = geojsonToBound(data as GeoJSON.Feature); // Extract rotation from properties if it exists - const rotation = data.properties && isRotationValue(data.properties?.[ROTATION_ATTRIBUTE_NAME]) + let rotation = data.properties && isRotationValue(data.properties?.[ROTATION_ATTRIBUTE_NAME]) ? data.properties[ROTATION_ATTRIBUTE_NAME] as number : undefined; + if (inverse) { + if (rotation !== undefined) { + const mapped = mapRotatedBounds(inverse, bounds, rotation); + bounds = mapped.bounds; + rotation = mapped.rotation; + } else { + bounds = mapBounds(inverse, bounds); + } + } cb(); handler.updateRectBounds(frameNumberRef.value, flickNumberRef.value, bounds, rotation); } else { - handler.updateGeoJSON(mode, frameNumberRef.value, flickNumberRef.value, data, key, cb); + const nativeData = inverse ? mapGeoJSONFeatures(inverse, [data])[0] : data; + handler.updateGeoJSON(mode, frameNumberRef.value, flickNumberRef.value, nativeData, key, cb); } // Jump into edit mode if we completed a new shape if (geometryCompleteEvent) { @@ -779,8 +1016,13 @@ export default defineComponent({ ); // Handle clicks outside the edit polygon to allow selecting other polygons editAnnotationLayer.bus.$on('click-outside-edit', (geo: { x: number; y: number }) => { - // Check which polygon was clicked by iterating through formatted data - const point: [number, number] = [geo.x, geo.y]; + // Check which polygon was clicked by iterating through formatted data. + // The click arrives in display space while formattedData is native, so + // map it back through the aligned-view inverse (identity when unwarped). + const inverse = alignedDisplayInverse.value; + const point: [number, number] = inverse + ? applyHomography(inverse, [geo.x, geo.y]) + : [geo.x, geo.y]; const polygonData = polyAnnotationLayer.formattedData; // Find the polygon that contains the click point diff --git a/client/src/components/annotators/mediaControllerType.ts b/client/src/components/annotators/mediaControllerType.ts index ece6d1d12..ff771830d 100644 --- a/client/src/components/annotators/mediaControllerType.ts +++ b/client/src/components/annotators/mediaControllerType.ts @@ -45,6 +45,16 @@ export interface AggregateMediaController { cameraSync: Readonly>; /** Incremented when the viewer is resized, used to trigger layer redraws */ resizeTrigger: Readonly>; + /** + * True only while onResize is applying its programmatic size()/resetZoom() + * to the panes. resetZoom emits GeoJS pan/zoom events synchronously; the + * linked-viewer navigation (useAlignedNavigation / useCalibrationNavigation) + * must ignore those so one pane's native-space reset isn't broadcast to the + * others as if it were a shared-space move (which parks warped panes on an + * empty corner). The resizeTrigger bump that follows re-snaps every pane from + * the reference once the reset has settled. + */ + resizing: Readonly>; /** * Global aligned-timeline slot indices with at least one camera missing a * frame (see AlignedFrameResolver's gapSlots); empty whenever alignment diff --git a/client/src/components/annotators/useAlignedNavigation.spec.ts b/client/src/components/annotators/useAlignedNavigation.spec.ts new file mode 100644 index 000000000..a28108171 --- /dev/null +++ b/client/src/components/annotators/useAlignedNavigation.spec.ts @@ -0,0 +1,195 @@ +/// +import { + ref, shallowRef, nextTick, Ref, +} from 'vue'; +import useAlignedNavigation from './useAlignedNavigation'; +import AlignedViewStore from '../../AlignedViewStore'; +import type { AggregateMediaController } from './mediaControllerType'; +import type { Matrix3 } from '../../homography'; + +vi.mock('geojs', () => ({ default: { event: { pan: 'geo_pan', zoom: 'geo_zoom' } } })); + +const IDENTITY: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + +/** Minimal stand-in for a geojs viewer: center/zoom state + geoOn events. */ +function fakeViewer(baseUnitsPerPixel: number) { + const state = { center: { x: 0, y: 0 }, zoom: 0 }; + const handlers: Record void>> = {}; + return { + geoOn(evt: string, handler: () => void) { + handlers[evt] = handlers[evt] || []; + handlers[evt].push(handler); + }, + geoOff(evt: string, handler: () => void) { + handlers[evt] = (handlers[evt] || []).filter((h) => h !== handler); + }, + center(c?: { x: number; y: number }) { + if (c) { + state.center = { ...c }; + } + return state.center; + }, + zoom(z?: number) { + if (z !== undefined) { + state.zoom = z; + } + return state.zoom; + }, + unitsPerPixel(z: number) { + return baseUnitsPerPixel / 2 ** z; + }, + trigger(evt: string) { + (handlers[evt] || []).forEach((h) => h()); + }, + }; +} + +function makeHarness() { + // EO: high-res pane (fine zoom-0 baseline); IR: low-res pane. + const eo = fakeViewer(1); + const ir = fakeViewer(8); + const resetZoom = vi.fn(); + const controllers: Record; resetZoom: () => void }> = { + eo: { geoViewerRef: ref(eo), resetZoom }, + ir: { geoViewerRef: ref(ir), resetZoom }, + }; + const cameraSync = ref(false); + const resizing = ref(false); + const resizeTrigger = ref(0); + // shallowRef: a plain ref would deep-unwrap the nested cameraSync / + // resizeTrigger refs, unlike the real aggregate controller object. + const aggregate = shallowRef({ + cameraSync, + resizeTrigger, + resizing, + getController: (name: string) => controllers[name], + }) as unknown as Ref; + const cameras = ref(['eo', 'ir']); + const alignedView = new AlignedViewStore(); + useAlignedNavigation(aggregate, alignedView, cameras); + return { + eo, ir, cameraSync, resizing, resizeTrigger, alignedView, resetZoom, + }; +} + +describe('useAlignedNavigation', () => { + it('snaps panes to the reference view immediately on activation', async () => { + const { eo, ir, alignedView } = makeHarness(); + eo.center({ x: 250, y: 150 }); + eo.zoom(1); // units-per-pixel = 0.5 + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[1, 0, 100], [0, 1, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + // No pan/zoom event fired: activation itself aligned the panes. + expect(ir.center()).toEqual({ x: 250, y: 150 }); + expect(ir.zoom()).toBeCloseTo(Math.log2(8 / 0.5), 6); + }); + + it('links panes by IDENTITY center in the shared reference space', async () => { + const { eo, ir, alignedView } = makeHarness(); + // ir's imagery renders warped through ir->eo (translation +100), so its + // pane coordinates ARE reference coordinates: the link must NOT map the + // center through the camera-to-camera transform again. + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[1, 0, 100], [0, 1, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + eo.center({ x: 500, y: 300 }); + eo.zoom(2); // source units-per-pixel = 1 / 2^2 = 0.25 reference units + eo.trigger('geo_pan'); + + expect(ir.center()).toEqual({ x: 500, y: 300 }); + // Same reference extent through ir's own zoom-0 baseline: + // log2(8 / 0.25) = 5. + expect(ir.zoom()).toBeCloseTo(5, 6); + }); + + it('propagates from the warped pane back to the reference pane identically', async () => { + const { eo, ir, alignedView } = makeHarness(); + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[2, 0, 0], [0, 2, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + ir.center({ x: 40, y: 60 }); + ir.trigger('geo_pan'); + + expect(eo.center()).toEqual({ x: 40, y: 60 }); + }); + + it('resets every pane to its native view when the aligned view deactivates', async () => { + const { alignedView, resetZoom } = makeHarness(); + alignedView.setTransforms('eo', { eo: IDENTITY, ir: IDENTITY }); + alignedView.setEnabled(true); + await nextTick(); + expect(resetZoom).not.toHaveBeenCalled(); + + alignedView.setEnabled(false); + await nextTick(); + // Once per pane (eo + ir). + expect(resetZoom).toHaveBeenCalledTimes(2); + }); + + it('ignores the native-space pan/zoom events onResize emits while resizing', async () => { + const { + eo, ir, resizing, resizeTrigger, alignedView, + } = makeHarness(); + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[1, 0, 100], [0, 1, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + // A good aligned view: both panes centered in the shared reference space. + eo.center({ x: 500, y: 300 }); + eo.zoom(2); + eo.trigger('geo_pan'); + expect(ir.center()).toEqual({ x: 500, y: 300 }); + + // onResize resets each pane to its OWN native bounds and fires pan events. + // ir's reset drops it to its native center; while resizing this must NOT be + // broadcast into the reference space (that is what parked panes in a black + // corner). eo -- the reference -- must stay put. + resizing.value = true; + ir.center({ x: 40, y: 60 }); + ir.trigger('geo_pan'); + expect(eo.center()).toEqual({ x: 500, y: 300 }); + resizing.value = false; + + // The resizeTrigger bump that follows re-snaps every pane from the + // reference, so ir lands back on the reference-space center. + resizeTrigger.value += 1; + await nextTick(); + expect(ir.center()).toEqual({ x: 500, y: 300 }); + }); + + it('stands down while inactive, suspended, or raw camera sync is on', async () => { + const { + eo, ir, cameraSync, alignedView, + } = makeHarness(); + alignedView.setTransforms('eo', { eo: IDENTITY, ir: IDENTITY }); + alignedView.setEnabled(true); + await nextTick(); + + cameraSync.value = true; + eo.center({ x: 9, y: 9 }); + eo.trigger('geo_pan'); + expect(ir.center()).toEqual({ x: 0, y: 0 }); + cameraSync.value = false; + + alignedView.setSuspended(true); + await nextTick(); + eo.trigger('geo_pan'); + expect(ir.center()).toEqual({ x: 0, y: 0 }); + }); +}); diff --git a/client/src/components/annotators/useAlignedNavigation.ts b/client/src/components/annotators/useAlignedNavigation.ts new file mode 100644 index 000000000..133e35749 --- /dev/null +++ b/client/src/components/annotators/useAlignedNavigation.ts @@ -0,0 +1,105 @@ +import { Ref, watch } from 'vue'; +import type { AggregateMediaController } from './mediaControllerType'; +import type AlignedViewStore from '../../AlignedViewStore'; +import useLinkedViewers from './useLinkedViewers'; + +/** + * Links pan/zoom recentering across ALL loaded cameras while the aligned view + * is active (SEAL-TK feature 3). + * + * While active, every pane RENDERS in the shared reference space, so the link + * between panes is the IDENTITY on coordinates: same center, same + * reference-units-per-screen-pixel. (Mapping centers through the + * camera-to-camera transforms here would re-apply a transform the rendering has + * already applied.) Distinct from the raw "sync cameras" toggle (Controls.vue), + * which forwards raw screen deltas for UNWARPED panes and is hidden whenever the + * aligned view is available, and from the calibration pair link + * ({@link useCalibrationNavigation}), which maps through the homography and + * stands down while the aligned view is active. + */ +export default function useAlignedNavigation( + aggregateController: Ref, + alignedView: AlignedViewStore, + cameras: Ref, +) { + const { + viewer, teardown, attach, guarded, applyView, + } = useLinkedViewers(aggregateController); + + function link(camera: string) { + return () => guarded(() => { + if (!alignedView.active.value) { + return; + } + // onResize resets each pane to its own native bounds and emits pan/zoom + // events; ignore them so a non-reference pane's native center doesn't get + // copied into the shared reference space. setup() re-snaps from the + // reference once the resize settles (via the resizeTrigger watch). + if (aggregateController.value.resizing.value) { + return; + } + // Never fight the raw screen-delta sync (unreachable while the aligned + // view is available, but be defensive about two handlers on one event). + if (aggregateController.value.cameraSync.value) { + return; + } + const source = viewer(camera); + if (!source) { + return; + } + // Shared reference space: copy the center verbatim and match the extent. + const center = source.center(); + const view = { + center: { x: center.x, y: center.y }, + unitsPerPixel: source.unitsPerPixel(source.zoom()), + }; + cameras.value.forEach((other) => { + if (other !== camera) { + const target = viewer(other); + if (target) { + applyView(target, view); + } + } + }); + }); + } + + function setup() { + teardown(); + if (!alignedView.active.value) { + return; + } + cameras.value.forEach((camera) => attach(camera, link(camera))); + // Snap immediately from the reference pane so hitting Align lines every pane + // up right away instead of waiting for the first pan/zoom event. + const reference = alignedView.reference.value; + if (reference && cameras.value.includes(reference)) { + link(reference)(); + } + } + + watch( + [ + alignedView.active, + alignedView.toReference, + cameras, + aggregateController.value.resizeTrigger, + ], + setup, + ); + + // Leaving the aligned view strands every pane at reference-space centers/zooms + // while content reverts to native coordinates -- reset each to its full native + // view so the imagery is back on-screen. + watch(alignedView.active, (active, wasActive) => { + if (!active && wasActive) { + cameras.value.forEach((camera) => { + try { + aggregateController.value.getController(camera).resetZoom(); + } catch { + // A pane may already be torn down during dataset unload. + } + }); + } + }); +} diff --git a/client/src/components/annotators/useLinkedViewers.ts b/client/src/components/annotators/useLinkedViewers.ts new file mode 100644 index 000000000..fd757f434 --- /dev/null +++ b/client/src/components/annotators/useLinkedViewers.ts @@ -0,0 +1,94 @@ +import geo from 'geojs'; +import { onBeforeUnmount, Ref } from 'vue'; +import type { AggregateMediaController } from './mediaControllerType'; + +/** + * Where to move a linked pane: the world-space center to show, and how many + * world units one screen pixel should span there (null leaves the pane's + * current zoom untouched, for when a local scale can't be resolved). + */ +export interface LinkedView { + center: { x: number; y: number }; + unitsPerPixel: number | null; +} + +/** + * Shared plumbing for the two multicam pan/zoom links -- the calibration pair + * link ({@link useCalibrationNavigation}) and the aligned-view link + * ({@link useAlignedNavigation}). Both attach geojs pan/zoom listeners to a set + * of panes and, when one moves, drive the others to a matching view; they + * differ only in how a source pane's view maps onto a target pane. This owns + * the parts that don't: a re-entrancy guard so a driven update doesn't echo + * back, listener bookkeeping, and the zoom-baseline conversion (geojs zoom is + * log2-based, and panes sized to different-resolution images keep their own + * zoom-0 baselines, so matching an extent converts through the target's + * baseline rather than copying the zoom level). + */ +export default function useLinkedViewers( + aggregateController: Ref, +) { + // Setting a pane's center/zoom from a handler must not re-trigger its own listener. + let guard = false; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let attached: { viewer: any; handler: () => void }[] = []; + + /** This camera's geojs viewer, or null when it isn't initialized/known yet. */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function viewer(camera: string): any { + try { + return aggregateController.value.getController(camera).geoViewerRef.value || null; + } catch { + return null; + } + } + + /** Detach every attached pan/zoom listener. */ + function teardown() { + attached.forEach(({ viewer: v, handler }) => { + v.geoOff(geo.event.pan, handler); + v.geoOff(geo.event.zoom, handler); + }); + attached = []; + } + + /** Attach a pan+zoom handler to `camera`'s pane; no-op when it has no viewer yet. */ + function attach(camera: string, handler: () => void) { + const v = viewer(camera); + if (!v) { + return; + } + v.geoOn(geo.event.pan, handler); + v.geoOn(geo.event.zoom, handler); + attached.push({ viewer: v, handler }); + } + + /** Run `fn` with the re-entrancy guard raised (a no-op while already guarded). */ + function guarded(fn: () => void) { + if (guard) { + return; + } + guard = true; + try { + fn(); + } finally { + guard = false; + } + } + + /** Drive `target` to `view`: center, plus a matching extent via the target's own zoom-0 baseline. */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function applyView(target: any, view: LinkedView) { + if (view.unitsPerPixel !== null) { + const targetZoom = Math.log2(target.unitsPerPixel(0) / view.unitsPerPixel); + if (Number.isFinite(targetZoom)) { + target.zoom(targetZoom); + } + } + target.center(view.center); + } + + onBeforeUnmount(teardown); + return { + viewer, teardown, attach, guarded, applyView, + }; +} diff --git a/client/src/components/annotators/useMediaController.ts b/client/src/components/annotators/useMediaController.ts index 5b3317277..e81f884c1 100644 --- a/client/src/components/annotators/useMediaController.ts +++ b/client/src/components/annotators/useMediaController.ts @@ -93,6 +93,10 @@ export function useMediaController() { let cameraControllerSymbols: Record = {}; const synchronizeCameras: Ref = ref(false); const resizeTrigger: Ref = ref(0); + // Raised only while onResize applies its programmatic resetZoom, so the + // linked-viewer navigation ignores the resulting pan/zoom events (see + // AggregateMediaController.resizing). + const resizing: Ref = ref(false); // shallowRef: an AlignedFrameResolver carries nested Refs (slotCount, frameRate) // that must NOT be deep-reactive-converted/auto-unwrapped by a plain ref(). const alignedFrameResolver: Ref = shallowRef(null); @@ -127,6 +131,7 @@ export function useMediaController() { toggleSynchronizeCameras, cameraSync: synchronizeCameras, resizeTrigger, + resizing, alignedGapSlots, seekCameraFrame: aggregateSeekCameraFrame, }; @@ -226,7 +231,18 @@ export function useMediaController() { // Resize maps first, then redraw annotation layers once GeoJS has settled. if (resized) { window.requestAnimationFrame(() => { - pendingResizes.forEach((applyResize) => applyResize()); + // resetZoom recenters each pane on its OWN native bounds and emits + // pan/zoom events synchronously. Suppress the linked-viewer navigation + // for the duration so a non-reference pane's native center isn't + // broadcast into the shared/reference space (which would strand warped + // panes on an empty corner). The resizeTrigger bump below then re-snaps + // every pane from a clean reference view. + resizing.value = true; + try { + pendingResizes.forEach((applyResize) => applyResize()); + } finally { + resizing.value = false; + } window.requestAnimationFrame(() => { resizeTrigger.value += 1; }); @@ -686,6 +702,7 @@ export function useMediaController() { toggleSynchronizeCameras, cameraSync: synchronizeCameras, resizeTrigger, + resizing, alignedGapSlots, seekCameraFrame: aggregateSeekCameraFrame, }; diff --git a/client/src/components/controls/Controls.vue b/client/src/components/controls/Controls.vue index c580c5bfc..9223b4459 100644 --- a/client/src/components/controls/Controls.vue +++ b/client/src/components/controls/Controls.vue @@ -10,7 +10,9 @@ import { DatasetType, useApi } from 'dive-common/apispec'; import { computeGapGradient } from 'dive-common/alignedTimeline'; import { frameToTimestamp } from 'vue-media-annotator/utils'; import { injectAggregateController } from '../annotators/useMediaController'; -import { useTime, useTrackFilters, useDatasetId } from '../../provides'; +import { + useTime, useTrackFilters, useDatasetId, useAlignedView, +} from '../../provides'; export default defineComponent({ name: 'Controls', @@ -38,6 +40,35 @@ export default defineComponent({ dragging: false, }); const mediaController = injectAggregateController().value; + let alignedView: ReturnType | undefined; + try { + alignedView = useAlignedView(); + } catch { + // aligned view store may not be provided in tests or minimal embeds. + } + /** + * The raw screen-delta camera sync is only offered while the transform + * -aware aligned view (the Align button) is unavailable: once every + * camera has a calibration transform, Align is the single place to link + * pan/zoom, and this cruder link (which assumes identical pixel scale + * between panes) would just be a second, worse toggle for the same thing. + */ + const rawSyncAvailable = computed(() => mediaController.cameras.value.length > 1 + && !alignedView?.available.value); + const toggleRawSync = () => { + if (rawSyncAvailable.value) { + mediaController.toggleSynchronizeCameras(!mediaController.cameraSync.value); + } + }; + // If transforms become available while the raw sync is on, switch it off: + // its toggle is hidden from that point, and the aligned-view link stands + // down while raw sync is enabled, so a stuck-on raw sync would silently + // block the Align button's linking with no visible control to clear it. + watch(rawSyncAvailable, (available) => { + if (!available && mediaController.cameraSync.value) { + mediaController.toggleSynchronizeCameras(false); + } + }, { immediate: true }); const isVideo = computed(() => props.datasetType === 'video'); const { frameRate } = useTime(); const { visible } = usePrompt(); @@ -229,6 +260,8 @@ export default defineComponent({ activeTimeFilter, data, mediaController, + rawSyncAvailable, + toggleRawSync, dragHandler, input, alignedGapGradient, @@ -274,7 +307,7 @@ export default defineComponent({ { bind: 'd', handler: mediaController.prevFrame, disabled: visible() }, { bind: 'l', - handler: () => mediaController.toggleSynchronizeCameras(!mediaController.cameraSync.value), + handler: toggleRawSync, disabled: visible(), }, ]" @@ -626,12 +659,12 @@ export default defineComponent({ {{ mediaController.cameraSync.value ? 'mdi-link' : 'mdi-link-off' }} @@ -946,12 +979,12 @@ export default defineComponent({ {{ mediaController.cameraSync.value ? 'mdi-link' : 'mdi-link-off' }} @@ -1236,13 +1269,13 @@ export default defineComponent({ {{ mediaController.cameraSync.value ? 'mdi-link' : 'mdi-link-off' }} diff --git a/client/src/components/index.ts b/client/src/components/index.ts index ec4d30a9a..21164ef6b 100644 --- a/client/src/components/index.ts +++ b/client/src/components/index.ts @@ -28,6 +28,7 @@ import TypePicker from './TypePicker.vue'; export * from './annotators/useMediaController'; export { default as useAnnotatorImageCursor } from './annotators/useAnnotatorImageCursor'; +export { default as useAlignedNavigation } from './annotators/useAlignedNavigation'; export { /* Annotators */ AnnotatorImageCursor, diff --git a/client/src/index.ts b/client/src/index.ts index 33708c1c5..45f010cbd 100644 --- a/client/src/index.ts +++ b/client/src/index.ts @@ -2,9 +2,11 @@ import * as layers from './layers'; import * as components from './components'; +import AlignedViewStore from './AlignedViewStore'; import BaseAnnotation from './BaseAnnotation'; import BaseAnnotationStore from './BaseAnnotationStore'; import CameraStore from './CameraStore'; +import CameraCalibrationStore from './CameraCalibrationStore'; import Group from './Group'; import GroupFilterControls from './GroupFilterControls'; import GroupStore from './GroupStore'; @@ -25,9 +27,11 @@ export { layers, components, /* other */ + AlignedViewStore, BaseAnnotation, BaseAnnotationStore, CameraStore, + CameraCalibrationStore, Group, GroupFilterControls, GroupStore, diff --git a/client/src/layers/AlignedImageLayer.ts b/client/src/layers/AlignedImageLayer.ts new file mode 100644 index 000000000..157819181 --- /dev/null +++ b/client/src/layers/AlignedImageLayer.ts @@ -0,0 +1,260 @@ +import geo from 'geojs'; +import type { MediaController } from '../components/annotators/mediaControllerType'; +import { Matrix3, subdivideWarpQuads, warpGridSize } from '../homography'; + +export interface CameraImage { + /** The texture source for the geojs quad feature: an `` for image sequences, a ` {{ activeCameraName }} + diff --git a/client/dive-common/utils/warpAnnotationsAcrossCameras.spec.ts b/client/dive-common/utils/warpAnnotationsAcrossCameras.spec.ts new file mode 100644 index 000000000..f47f9b41f --- /dev/null +++ b/client/dive-common/utils/warpAnnotationsAcrossCameras.spec.ts @@ -0,0 +1,108 @@ +/// +import CameraStore from 'vue-media-annotator/CameraStore'; +import type { Matrix3 } from 'vue-media-annotator/homography'; +import { IDENTITY3 } from 'vue-media-annotator/alignedView'; +import { ROTATION_ATTRIBUTE_NAME } from 'vue-media-annotator/utils'; +import warpAnnotationsAcrossCameras from './warpAnnotationsAcrossCameras'; + +function translation(tx: number, ty: number): Matrix3 { + return [[1, 0, tx], [0, 1, ty], [0, 0, 1]]; +} + +function makeCameraStore(cameras: string[]) { + const store = new CameraStore({ markChangesPending: () => null }); + cameras.forEach((camera) => store.addCamera(camera)); + store.removeCamera('singleCam'); + return store; +} + +function trackStoreFor(store: CameraStore, camera: string) { + const mapped = store.camMap.value.get(camera); + if (!mapped) { + throw new Error(`no camera ${camera}`); + } + return mapped.trackStore; +} + +const headPoint: GeoJSON.Feature = { + type: 'Feature', + properties: { key: 'head' }, + geometry: { type: 'Point', coordinates: [5, 5] }, +}; + +/** left is the reference; right's native->reference is translation(10, 20), + * so left->right maps points by (-10, -20). */ +const TO_REFERENCE: Record = { + left: IDENTITY3, + right: translation(10, 20), +}; + +describe('warpAnnotationsAcrossCameras', () => { + it('copies tracks onto the other camera with warped geometry', () => { + const store = makeCameraStore(['left', 'right']); + const leftStore = trackStoreFor(store, 'left'); + const track = leftStore.add(3, 'fish', undefined, leftStore.getNewId()); + track.confidencePairs = [['fish', 0.9], ['shark', 0.1]]; + track.setFeature({ + frame: 3, bounds: [0, 0, 10, 10], keyframe: true, interpolate: true, + }, [headPoint]); + track.setFeatureAttribute(3, 'note', 'hello'); + track.setFeatureAttribute(3, ROTATION_ATTRIBUTE_NAME, 0.5); + + const result = warpAnnotationsAcrossCameras(store, TO_REFERENCE, 'left'); + expect(result).toEqual({ tracks: 1, cameras: 1 }); + + const target = trackStoreFor(store, 'right').getPossible(track.id); + expect(target).toBeDefined(); + const feature = target?.features[3]; + expect(feature?.keyframe).toBe(true); + expect(feature?.interpolate).toBe(true); + expect(feature?.bounds).toEqual([-10, -20, 0, -10]); + expect(feature?.geometry?.features[0].geometry.coordinates).toEqual([-5, -15]); + expect(feature?.attributes?.note).toBe('hello'); + // pure translation leaves the rotation angle unchanged + expect(feature?.attributes?.[ROTATION_ATTRIBUTE_NAME]).toBeCloseTo(0.5, 6); + expect(target?.confidencePairs).toEqual([['fish', 0.9], ['shark', 0.1]]); + }); + + it('copies every keyframe and skips cameras without a transform', () => { + const store = makeCameraStore(['left', 'right', 'ir']); + const leftStore = trackStoreFor(store, 'left'); + const track = leftStore.add(0, 'fish', undefined, leftStore.getNewId()); + track.setFeature({ frame: 0, bounds: [0, 0, 10, 10], keyframe: true }); + track.setFeature({ frame: 5, bounds: [30, 30, 40, 40], keyframe: true }); + + // 'ir' has no entry in TO_REFERENCE, so it cannot receive copies + const result = warpAnnotationsAcrossCameras(store, TO_REFERENCE, 'left'); + expect(result).toEqual({ tracks: 1, cameras: 1 }); + expect(trackStoreFor(store, 'ir').getPossible(track.id)).toBeUndefined(); + + const target = trackStoreFor(store, 'right').getPossible(track.id); + expect(target?.features[0]?.bounds).toEqual([-10, -20, 0, -10]); + expect(target?.features[5]?.bounds).toEqual([20, 10, 30, 20]); + }); + + it('overwrites keyframes of an existing same-id target track', () => { + const store = makeCameraStore(['left', 'right']); + const leftStore = trackStoreFor(store, 'left'); + const rightStore = trackStoreFor(store, 'right'); + const track = leftStore.add(3, 'fish', undefined, leftStore.getNewId()); + track.setFeature({ frame: 3, bounds: [0, 0, 10, 10], keyframe: true }); + const existing = rightStore.add(3, 'other', undefined, track.id); + existing.setFeature({ frame: 3, bounds: [100, 100, 200, 200], keyframe: true }); + existing.setFeature({ frame: 7, bounds: [1, 1, 2, 2], keyframe: true }); + + warpAnnotationsAcrossCameras(store, TO_REFERENCE, 'left'); + expect(existing.features[3]?.bounds).toEqual([-10, -20, 0, -10]); + // untouched keyframes on the target survive + expect(existing.features[7]?.bounds).toEqual([1, 1, 2, 2]); + }); + + it('returns zeros when the source camera or its tracks are missing', () => { + const store = makeCameraStore(['left', 'right']); + expect(warpAnnotationsAcrossCameras(store, TO_REFERENCE, 'missing')) + .toEqual({ tracks: 0, cameras: 0 }); + expect(warpAnnotationsAcrossCameras(store, TO_REFERENCE, 'left')) + .toEqual({ tracks: 0, cameras: 0 }); + }); +}); diff --git a/client/dive-common/utils/warpAnnotationsAcrossCameras.ts b/client/dive-common/utils/warpAnnotationsAcrossCameras.ts new file mode 100644 index 000000000..46fbe7828 --- /dev/null +++ b/client/dive-common/utils/warpAnnotationsAcrossCameras.ts @@ -0,0 +1,124 @@ +import { cloneDeep } from 'lodash'; +import type { CameraStore, Track } from 'vue-media-annotator/index'; +import type { Matrix3 } from 'vue-media-annotator/homography'; +import { + cameraPairTransform, mapBounds, mapGeoJSONFeatures, mapRotatedBounds, +} from 'vue-media-annotator/alignedView'; +import { + RectBounds, + ROTATION_ATTRIBUTE_NAME, + getRotationFromAttributes, + validateRotation, +} from 'vue-media-annotator/utils'; + +export interface WarpAnnotationsResult { + /** Distinct source tracks copied onto at least one other camera. */ + tracks: number; + /** Cameras (excluding the source) that received warped annotations. */ + cameras: number; +} + +/** + * Copy every track on `sourceCamera` onto every other calibrated camera, + * warping geometry through the camera-to-camera homographies (composed via + * the reference space, see {@link cameraPairTransform}). Used after a + * single-camera annotation import into a calibrated multicam dataset so the + * detections appear on all cameras. + * + * Copies land on the same local frame number: a batch import carries no + * temporal-alignment information, and calibrated rigs (e.g. EO/IR) capture + * in lockstep. Tracks keep their id across cameras — the same convention the + * Align View continuous mirror uses — so a target camera's existing same-id + * track has its keyframes overwritten at the copied frames. Mutations go + * through the Track/TrackStore APIs so change-tracking fires; the caller is + * responsible for persisting the result (e.g. handler.save()). + * + * @param cameraStore the multicam store holding per-camera tracks + * @param toReference per-camera native->reference matrices + * (AlignedViewStore.toReference — present whenever the rig is calibrated, + * independent of the Align View display toggle) + * @param sourceCamera camera whose tracks are copied out + */ +export default function warpAnnotationsAcrossCameras( + cameraStore: CameraStore, + toReference: Record, + sourceCamera: string, +): WarpAnnotationsResult { + const result: WarpAnnotationsResult = { tracks: 0, cameras: 0 }; + const sourceStore = cameraStore.camMap.value.get(sourceCamera)?.trackStore; + if (!sourceStore) { + return result; + } + const copiedTracks = new Set(); + cameraStore.camMap.value.forEach(({ trackStore }, cameraName) => { + if (cameraName === sourceCamera) { + return; + } + const matrix = cameraPairTransform(toReference, sourceCamera, cameraName); + if (!matrix) { + return; + } + let cameraReceived = false; + sourceStore.annotationMap.forEach((annotation) => { + const sourceTrack = annotation as Track; + let targetTrack = trackStore.getPossible(sourceTrack.id) as Track | undefined; + sourceTrack.featureIndex.forEach((frame) => { + const feature = sourceTrack.features[frame]; + if (!feature || !feature.keyframe) { + return; + } + const mappedGeometry = feature.geometry + ? mapGeoJSONFeatures(matrix, feature.geometry.features) + : []; + let mappedBounds: RectBounds | undefined; + let mappedRotation: number | undefined; + const sourceRotation = getRotationFromAttributes(feature.attributes); + if (feature.bounds) { + if (sourceRotation !== undefined) { + const mapped = mapRotatedBounds(matrix, feature.bounds, sourceRotation); + mappedBounds = mapped.bounds; + mappedRotation = validateRotation(mapped.rotation); + } else { + mappedBounds = mapBounds(matrix, feature.bounds); + } + } + if (!targetTrack) { + targetTrack = trackStore.add( + frame, + sourceTrack.getType()[0], + undefined, + sourceTrack.id, + ); + targetTrack.confidencePairs = cloneDeep(sourceTrack.confidencePairs); + Object.entries(sourceTrack.attributes).forEach(([key, value]) => { + targetTrack?.setAttribute(key, cloneDeep(value)); + }); + } + targetTrack.setFeature({ + frame, + flick: feature.flick, + bounds: mappedBounds, + keyframe: true, + interpolate: feature.interpolate, + }, mappedGeometry); + if (feature.attributes) { + Object.entries(feature.attributes).forEach(([key, value]) => { + if (key !== ROTATION_ATTRIBUTE_NAME) { + targetTrack?.setFeatureAttribute(frame, key, cloneDeep(value)); + } + }); + } + if (mappedRotation !== undefined) { + targetTrack.setFeatureAttribute(frame, ROTATION_ATTRIBUTE_NAME, mappedRotation); + } + copiedTracks.add(sourceTrack.id); + cameraReceived = true; + }); + }); + if (cameraReceived) { + result.cameras += 1; + } + }); + result.tracks = copiedTracks.size; + return result; +} From 64177bf3b724a66d98d60699730aaa03a9cd0d13 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Thu, 9 Jul 2026 16:57:45 -0400 Subject: [PATCH 5/7] Store the calibration as one file per camera The dataset's saved calibration moves from a single all-pairs calibration.json to one calibration_.json per non-reference camera (a pair not touching the reference files under its right camera), matching the per-camera files users already handle from kamera. File names are discovery/provenance only: the pair's left/right names inside each file body stay authoritative, so a misnamed or copied file can never rebind a transform to the wrong camera. The legacy single file is still read and is migrated (rewritten per-camera, then removed) on the next save; stale per-camera files are cleaned up so the on-disk set always mirrors the saved calibration. Per-file producer stamps are compared when the set is merged (project load and import seeding): agreement keeps the stamp, disagreement becomes a { mixed: true, files: {...} } composite that the Align View tooltip surfaces as a mixed-generation warning instead of composing silently. A save never writes the composite into per-camera files -- that would read back as a unanimous rig. Parent-folder transform discovery now returns every self-identified calibration file (calibration.json first, then per-camera files), and the multicam import dialog auto-attaches calibration_.json to its matching camera slot. Files written by DIVE now also self-identify with type: dive-camera-calibration so discovery recognizes them. Co-Authored-By: Claude Fable 5 --- client/dive-common/apispec.ts | 8 +- .../useImportMultiCamDialog.ts | 42 +-- client/dive-common/components/Viewer.vue | 12 +- client/platform/desktop/backend/ipcService.ts | 4 +- .../desktop/backend/native/common.spec.ts | 148 +++++++++-- .../platform/desktop/backend/native/common.ts | 242 +++++++++++++----- .../desktop/backend/native/multiCamImport.ts | 21 +- client/platform/desktop/frontend/api.ts | 6 +- client/src/CameraCalibrationStore.spec.ts | 14 + client/src/CameraCalibrationStore.ts | 21 +- 10 files changed, 402 insertions(+), 116 deletions(-) diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 0bc77b2c0..756faf2e4 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -308,8 +308,12 @@ interface Api { ): Promise; /** Desktop: stereoscopic calibration file in a parent folder root. */ findParentFolderCalibrationFile?(parentPath: string): Promise; - /** Desktop: DIVE camera-calibration .json (alignment transforms) in a parent folder root. */ - findParentFolderTransformFile?(parentPath: string): Promise; + /** + * Desktop: every DIVE camera-calibration .json (alignment transforms) in a + * parent folder root: calibration.json first, then per-camera + * calibration_.json files, then other self-identified candidates. + */ + findParentFolderTransformFiles?(parentPath: string): Promise; /** True when the dataset folder has an attached stereoscopic calibration file. */ hasCalibrationFile?(datasetId: string): Promise; /** Web: stash a calibration File for multicam upload lookup. */ diff --git a/client/dive-common/components/ImportMultiCamDialog/useImportMultiCamDialog.ts b/client/dive-common/components/ImportMultiCamDialog/useImportMultiCamDialog.ts index 05dd13308..069d8eca3 100644 --- a/client/dive-common/components/ImportMultiCamDialog/useImportMultiCamDialog.ts +++ b/client/dive-common/components/ImportMultiCamDialog/useImportMultiCamDialog.ts @@ -65,7 +65,7 @@ export function useImportMultiCamDialog( listParentFolderCameras, resolveMulticamCameraSourcePath, findParentFolderCalibrationFile, - findParentFolderTransformFile, + findParentFolderTransformFiles, } = useApi(); const importType: Ref = ref(''); const folderList: Ref.json is attached to its + * matching camera slot; anything else (e.g. an all-pairs calibration.json) + * goes to the first free camera after the reference -- the file's pairs + * name their own cameras, so which slot carries it doesn't matter. Each + * shows in that camera's (clearable) transform field. */ async function discoverParentFolderTransform(parentPath: string) { - if (!transformImportEnabled.value || !findParentFolderTransformFile) { + if (!transformImportEnabled.value || !findParentFolderTransformFiles) { return; } - const discovered = await findParentFolderTransformFile(parentPath); - if (!discovered) { - return; - } - const target = orderedCameraKeys.value.find( - (key, index) => index > 0 && folderList.value[key] && !folderList.value[key].transformFile, - ); - if (target) { - folderList.value[target].transformFile = discovered; - } + const discovered = await findParentFolderTransformFiles(parentPath); + discovered.forEach((filePath) => { + const fileName = filePath.replace(/^.*[\\/]/, ''); + const cameraMatch = /^calibration_(.+)\.json$/i.exec(fileName); + let target: string | undefined; + if (cameraMatch && folderList.value[cameraMatch[1]]) { + target = !folderList.value[cameraMatch[1]].transformFile ? cameraMatch[1] : undefined; + } else { + target = orderedCameraKeys.value.find( + (key, index) => index > 0 && folderList.value[key] && !folderList.value[key].transformFile, + ); + } + if (target) { + folderList.value[target].transformFile = filePath; + } + }); } async function discoverParentFolderCalibration( diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index de03505fa..9e03e3539 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -436,17 +436,23 @@ export default defineComponent({ alignedView.setEnabled(!alignedView.enabled.value); }; const alignedViewTooltip = computed(() => { + // Per-camera calibration files with disagreeing producer stamps mean + // the rig may mix calibration generations -- say so instead of + // composing silently. + const mixedNote = cameraCalibration.sourceIsMixed() + ? ' — warning: mixed calibration file generations' + : ''; if (alignedViewEnabled.value) { - return 'Align View on (draw/edit on any camera)'; + return `Align View on (draw/edit on any camera)${mixedNote}`; } const cams = multiCamList.value; const unresolved = cams.length >= 2 ? unresolvedCameras(cams, cams[0], cameraCalibration.homographies.value) : []; if (unresolved.length) { - return `Align View — ${cams.length - unresolved.length}/${cams.length} cameras calibrated`; + return `Align View — ${cams.length - unresolved.length}/${cams.length} cameras calibrated${mixedNote}`; } - return 'Align View'; + return `Align View${mixedNote}`; }); // This context for removal const removeGroups = (id: AnnotationId) => { diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index 94f3bbf24..16e90e046 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -203,10 +203,10 @@ export default function register() { { path }: { path: string }, ) => common.findParentFolderCalibrationFile(path)); - ipcMain.handle('find-parent-folder-transform-file', async ( + ipcMain.handle('find-parent-folder-transform-files', async ( event, { path }: { path: string }, - ) => common.findParentFolderTransformFile(path)); + ) => common.findParentFolderTransformFiles(path)); ipcMain.handle('dataset-has-calibration-file', async ( event, diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index da3c46d10..4df182193 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -184,6 +184,11 @@ beforeEach(() => { 'broken.json': '{not json', 'z-transforms.json': JSON.stringify({ type: 'dive-camera-calibration', version: 1, pairs: [] }), }, + perCamera: { + 'calibration_uv.json': JSON.stringify({ type: 'dive-camera-calibration', version: 1, pairs: [] }), + 'calibration_ir.json': JSON.stringify({ type: 'dive-camera-calibration', version: 1, pairs: [] }), + 'stray.json': JSON.stringify({ some: 'thing' }), + }, none: { 'rig.json': JSON.stringify({ some: 'thing' }), 'notes.txt': 'not json', @@ -656,7 +661,7 @@ describe('native.common', () => { }); }); - it('saveMetadata writes calibration.json (pairs + points) and reloads it', async () => { + it('saveMetadata writes per-camera calibration files (pairs + points) and reloads them', async () => { const payload = await common.beginMediaImport( '/home/user/data/imageLists/success/image_list.txt', ); @@ -678,12 +683,16 @@ describe('native.common', () => { await common.saveMetadata(settings, final.id, { cameraHomographies, cameraCorrespondences }); - // Persisted as a standalone calibration.json: pairs labeled left/right, with - // points laid out as leftX leftY rightX rightY. + // Persisted as a standalone per-camera file (named for the pair's + // non-reference camera): pairs labeled left/right, with points laid out + // as leftX leftY rightX rightY. No legacy all-pairs calibration.json. const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); - const calibrationPath = npath.join(projectDir, 'calibration.json'); + const calibrationPath = npath.join(projectDir, 'calibration_ir.json'); expect(await fs.pathExists(calibrationPath)).toBe(true); + expect(await fs.pathExists(npath.join(projectDir, 'calibration.json'))).toBe(false); const calibration = await fs.readJSON(calibrationPath); + // Self-identifies so parent-folder discovery recognizes it. + expect(calibration.type).toBe('dive-camera-calibration'); expect(calibration.pairs).toStrictEqual([ { left: 'rgb', @@ -726,30 +735,43 @@ describe('native.common', () => { await common.saveMetadata(settings, final.id, { cameraHomographies, cameraTransformTypes }); const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); - const calibration = await fs.readJSON(npath.join(projectDir, 'calibration.json')); + const calibration = await fs.readJSON(npath.join(projectDir, 'calibration_ir.json')); expect(calibration.pairs[0].transformType).toBe('rigid'); const reloaded = await common.loadMetadata(settings, final.id, urlMapper); expect(reloaded.cameraTransformTypes).toStrictEqual(cameraTransformTypes); }); - describe('findParentFolderTransformFile', () => { - it('prefers a file named calibration.json among marked candidates', async () => { - const found = await common.findParentFolderTransformFile('/home/user/transformDiscovery/exactName'); - expect(found).toBe(npath.join('/home/user/transformDiscovery/exactName', 'calibration.json')); + describe('findParentFolderTransformFiles', () => { + it('orders a file named calibration.json before other marked candidates', async () => { + const found = await common.findParentFolderTransformFiles('/home/user/transformDiscovery/exactName'); + expect(found).toStrictEqual([ + npath.join('/home/user/transformDiscovery/exactName', 'calibration.json'), + npath.join('/home/user/transformDiscovery/exactName', 'aaa-stamped.json'), + ]); + }); + + it('finds marked files under any name, skipping unmarked and broken JSON', async () => { + const found = await common.findParentFolderTransformFiles('/home/user/transformDiscovery/otherName'); + expect(found).toStrictEqual([ + npath.join('/home/user/transformDiscovery/otherName', 'z-transforms.json'), + ]); }); - it('finds a marked file under any name, skipping unmarked and broken JSON', async () => { - const found = await common.findParentFolderTransformFile('/home/user/transformDiscovery/otherName'); - expect(found).toBe(npath.join('/home/user/transformDiscovery/otherName', 'z-transforms.json')); + it('finds per-camera calibration_.json files, alphabetically', async () => { + const found = await common.findParentFolderTransformFiles('/home/user/transformDiscovery/perCamera'); + expect(found).toStrictEqual([ + npath.join('/home/user/transformDiscovery/perCamera', 'calibration_ir.json'), + npath.join('/home/user/transformDiscovery/perCamera', 'calibration_uv.json'), + ]); }); - it('returns null when no self-identified calibration json exists', async () => { - expect(await common.findParentFolderTransformFile('/home/user/transformDiscovery/none')).toBeNull(); + it('returns empty when no self-identified calibration json exists', async () => { + expect(await common.findParentFolderTransformFiles('/home/user/transformDiscovery/none')).toStrictEqual([]); }); - it('returns null for a missing directory', async () => { - expect(await common.findParentFolderTransformFile('/home/user/doesNotExist')).toBeNull(); + it('returns empty for a missing directory', async () => { + expect(await common.findParentFolderTransformFiles('/home/user/doesNotExist')).toStrictEqual([]); }); }); @@ -798,7 +820,7 @@ describe('native.common', () => { }); const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); - const calibrationPath = npath.join(projectDir, 'calibration.json'); + const calibrationPath = npath.join(projectDir, 'calibration_ir.json'); expect((await fs.readJSON(calibrationPath)).source).toStrictEqual(source); const reloaded = await common.loadMetadata(settings, final.id, urlMapper); expect(reloaded.cameraCalibrationSource).toStrictEqual(source); @@ -817,6 +839,98 @@ describe('native.common', () => { expect('source' in (await fs.readJSON(calibrationPath))).toBe(false); }); + it('migrates a legacy all-pairs calibration.json to per-camera files on save', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + const final = res.meta; + const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); + + // A legacy single-file calibration holding two pairs. + await fs.writeJSON(npath.join(projectDir, 'calibration.json'), { + version: 1, + pairs: [ + { + left: 'rgb', right: 'ir', points: [], leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], rightToLeft: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }, + { + left: 'rgb', right: 'uv', points: [], leftToRight: [[1, 0, 8], [0, 1, 2], [0, 0, 1]], rightToLeft: [[1, 0, -8], [0, 1, -2], [0, 0, 1]], + }, + ], + }); + + // Loads fine from the legacy file. + const loaded = await common.loadMetadata(settings, final.id, urlMapper); + expect(Object.keys(loaded.cameraHomographies ?? {}).sort()).toStrictEqual(['rgb::ir', 'rgb::uv']); + + // A save re-writes the set as per-camera files and removes the legacy one. + await common.saveMetadata(settings, final.id, { + cameraHomographies: loaded.cameraHomographies, + cameraCorrespondences: loaded.cameraCorrespondences, + cameraTransformTypes: loaded.cameraTransformTypes, + }); + expect(await fs.pathExists(npath.join(projectDir, 'calibration.json'))).toBe(false); + expect(await fs.pathExists(npath.join(projectDir, 'calibration_ir.json'))).toBe(true); + expect(await fs.pathExists(npath.join(projectDir, 'calibration_uv.json'))).toBe(true); + + const reloaded = await common.loadMetadata(settings, final.id, urlMapper); + expect(reloaded.cameraHomographies).toStrictEqual(loaded.cameraHomographies); + }); + + it('merges per-camera calibration files and flags disagreeing source stamps', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + const final = res.meta; + const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); + + const irPair = { + left: 'rgb', right: 'ir', points: [], leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], rightToLeft: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }; + const uvPair = { + left: 'rgb', right: 'uv', points: [], leftToRight: [[1, 0, 8], [0, 1, 2], [0, 0, 1]], rightToLeft: [[1, 0, -8], [0, 1, -2], [0, 0, 1]], + }; + await fs.writeJSON(npath.join(projectDir, 'calibration_ir.json'), { + version: 1, source: { producer: 'kamera', run: 'fl07' }, pairs: [irPair], + }); + await fs.writeJSON(npath.join(projectDir, 'calibration_uv.json'), { + version: 1, source: { producer: 'kamera', run: 'fl09' }, pairs: [uvPair], + }); + + // Both pairs merge; the disagreeing stamps become a mixed composite so + // the client can warn about a rig assembled from different generations. + const mixed = await common.loadMetadata(settings, final.id, urlMapper); + expect(Object.keys(mixed.cameraHomographies ?? {}).sort()).toStrictEqual(['rgb::ir', 'rgb::uv']); + expect(mixed.cameraCalibrationSource).toStrictEqual({ + mixed: true, + files: { + 'calibration_ir.json': { producer: 'kamera', run: 'fl07' }, + 'calibration_uv.json': { producer: 'kamera', run: 'fl09' }, + }, + }); + + // Agreeing stamps stay a single plain stamp. + await fs.writeJSON(npath.join(projectDir, 'calibration_uv.json'), { + version: 1, source: { producer: 'kamera', run: 'fl07' }, pairs: [uvPair], + }); + const agreeing = await common.loadMetadata(settings, final.id, urlMapper); + expect(agreeing.cameraCalibrationSource).toStrictEqual({ producer: 'kamera', run: 'fl07' }); + + // A save of the mixed set never stamps the per-camera files with the + // composite (that would read as a unanimous rig on the next load). + await fs.writeJSON(npath.join(projectDir, 'calibration_uv.json'), { + version: 1, source: { producer: 'kamera', run: 'fl09' }, pairs: [uvPair], + }); + const beforeSave = await common.loadMetadata(settings, final.id, urlMapper); + await common.saveMetadata(settings, final.id, { + cameraHomographies: beforeSave.cameraHomographies, + cameraCalibrationSource: beforeSave.cameraCalibrationSource, + }); + expect('source' in (await fs.readJSON(npath.join(projectDir, 'calibration_ir.json')))).toBe(false); + }); + it('import with CSV annotations without specifying track file', async () => { const payload = await common.beginMediaImport('/home/user/data/imageSuccessWithAnnotations'); payload.trackFileAbsPath = ''; //It returns null be default but users change it. diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 53229b3ca..288c3935a 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -75,9 +75,20 @@ const JsonTrackFileName = /^result(_.*)?\.json$/i; const JsonFileName = /^.*\.json$/i; const JsonMetaFileName = 'meta.json'; // Standalone camera-to-camera (modality-to-modality) alignment transforms, -// stored separately from meta.json in the dataset directory. +// stored separately from meta.json in the dataset directory. The current +// convention is one file per non-reference camera (calibration_.json, +// each holding that camera's pair(s)); a single legacy calibration.json +// holding every pair is still read. File NAMES are discovery/provenance only: +// the pair's left/right names inside the file body are always authoritative, +// so a misnamed or copied file can never rebind a transform to the wrong +// camera. const CalibrationFileName = 'calibration.json'; +const PerCameraCalibrationFileName = /^calibration_(.+)\.json$/i; const CalibrationFileVersion = 1; + +function perCameraCalibrationFileName(camera: string): string { + return `calibration_${camera}.json`; +} const CsvFileName = /^.*\.csv$/i; const YAMLFileName = /^.*\.ya?ml$/i; /** @@ -375,26 +386,20 @@ async function loadMetadata( const projectDirData = await getValidatedProjectDir(settings, datasetId); const projectMetaData = await loadJsonMetadata(projectDirData.metaFileAbsPath); - // Load standalone camera calibration (transforms + correspondences), if present. - const calibrationFileAbsPath = npath.join(projectDirData.basePath, CalibrationFileName); + // Load standalone camera calibration (transforms + correspondences), if + // present: the per-camera calibration_.json files and/or the legacy + // all-pairs calibration.json, merged. let { cameraHomographies, cameraCorrespondences, cameraTransformTypes, cameraCalibrationSource, } = projectMetaData; - if (await fs.pathExists(calibrationFileAbsPath)) { - try { - const calibration = await _loadAsJson(calibrationFileAbsPath); - if (calibration && Array.isArray(calibration.pairs)) { - ({ - homographies: cameraHomographies, - correspondences: cameraCorrespondences, - transformTypes: cameraTransformTypes, - } = fromCalibrationPairs(calibration.pairs)); - cameraCalibrationSource = readCalibrationSource(calibration.source); - } - } catch (err) { - // A malformed calibration.json should not block loading the dataset. - console.warn(`Unable to read ${calibrationFileAbsPath}: ${err}`); - } + const loadedCalibration = await loadCalibrationFiles(projectDirData.basePath); + if (loadedCalibration.found) { + ({ + homographies: cameraHomographies, + correspondences: cameraCorrespondences, + transformTypes: cameraTransformTypes, + source: cameraCalibrationSource, + } = loadedCalibration); } let videoUrl = ''; @@ -821,6 +826,92 @@ function fromCalibrationPairs( return { homographies, correspondences, transformTypes }; } +/** + * Merge the per-file producer stamps of a calibration file set. All stamped + * files agreeing (deep-equal) yields that stamp; disagreement yields a + * composite `{ mixed: true, files: {...} }` so the client can surface a + * mixed-generation warning instead of composing silently -- the failure mode + * per-camera files invite is a rig assembled from files regenerated at + * different times. + */ +function mergeCalibrationSources( + stamps: { file: string; source: CalibrationSource | null }[], +): CalibrationSource | null { + const stamped = stamps.filter((entry) => entry.source !== null); + if (stamped.length === 0) { + return null; + } + const first = JSON.stringify(stamped[0].source); + if (stamped.every((entry) => JSON.stringify(entry.source) === first)) { + return stamped[0].source; + } + return { + mixed: true, + files: Object.fromEntries(stamps.map((entry) => [entry.file, entry.source])), + }; +} + +/** + * Read and merge every calibration file in a dataset directory: the legacy + * all-pairs calibration.json (if present) plus each per-camera + * calibration_.json, in that order, sorted by name for determinism. + * Pair bodies are authoritative (file names are ignored for binding); a pair + * key appearing in more than one file keeps the last occurrence with a + * warning. `found` is true when at least one file parsed as a calibration, + * so callers can distinguish "no calibration files" from an empty one. + */ +async function loadCalibrationFiles(basePath: string): Promise<{ + found: boolean; + homographies: CameraHomographies; + correspondences: CameraCorrespondences; + transformTypes: CameraTransformTypes; + source: CalibrationSource | null; +}> { + const names: string[] = []; + if (await fs.pathExists(npath.join(basePath, CalibrationFileName))) { + names.push(CalibrationFileName); + } + try { + const entries = await fs.readdir(basePath, { withFileTypes: true }); + names.push(...entries + .filter((entry) => entry.isFile() && PerCameraCalibrationFileName.test(entry.name)) + .map((entry) => entry.name) + .sort((a, b) => a.localeCompare(b))); + } catch { + // Unreadable directory: treated the same as no calibration files. + } + let found = false; + const mergedPairs = new Map(); + const stamps: { file: string; source: CalibrationSource | null }[] = []; + // eslint-disable-next-line no-restricted-syntax + for (const name of names) { + const absPath = npath.join(basePath, name); + try { + // eslint-disable-next-line no-await-in-loop -- files merged in deterministic order + const calibration = await _loadAsJson(absPath); + if (calibration && Array.isArray(calibration.pairs)) { + found = true; + (calibration.pairs as CalibrationPair[]).forEach((pair) => { + const key = `${pair.left}::${pair.right}`; + if (mergedPairs.has(key)) { + console.warn(`Calibration pair ${key} appears in multiple files; keeping ${name}`); + } + mergedPairs.set(key, pair); + }); + stamps.push({ file: name, source: readCalibrationSource(calibration.source) }); + } + } catch (err) { + // A malformed calibration file should not block loading the dataset. + console.warn(`Unable to read ${absPath}: ${err}`); + } + } + return { + found, + ...fromCalibrationPairs([...mergedPairs.values()]), + source: mergeCalibrationSources(stamps), + }; +} + async function saveMetadata(settings: Settings, datasetId: string, args: DatasetMetaMutable) { const projectDirInfo = await getValidatedProjectDir(settings, datasetId); const release = await _acquireLock(projectDirInfo.basePath, projectDirInfo.metaFileAbsPath, 'meta'); @@ -850,31 +941,19 @@ async function saveMetadata(settings: Settings, datasetId: string, args: Dataset existing.datasetInfo = args.datasetInfo; } - // Camera calibration (transforms + the points behind them) is persisted as a - // standalone calibration.json in the dataset directory rather than embedded in - // meta.json, so it is easy to find, hand-edit, and consume as a self-contained - // artifact. + // Camera calibration (transforms + the points behind them) is persisted as + // standalone calibration_.json files in the dataset directory -- + // one per non-reference camera, matching the per-camera files users handle + // from kamera -- rather than embedded in meta.json, so each camera's + // calibration is easy to find, hand-edit, and consume as a self-contained + // artifact. Any legacy all-pairs calibration.json is migrated (read above, + // removed below) on the first save. if (args.cameraHomographies || args.cameraCorrespondences || args.cameraTransformTypes || args.cameraCalibrationSource) { - const calibrationFileAbsPath = npath.join(projectDirInfo.basePath, CalibrationFileName); // Start from whatever is on disk so a partial update doesn't clobber the rest. - let homographies: CameraHomographies = {}; - let correspondences: CameraCorrespondences = {}; - let transformTypes: CameraTransformTypes = {}; - let source: CalibrationSource | null = null; - if (await fs.pathExists(calibrationFileAbsPath)) { - try { - const existingCalibration = await _loadAsJson(calibrationFileAbsPath); - if (existingCalibration && Array.isArray(existingCalibration.pairs)) { - ({ homographies, correspondences, transformTypes } = fromCalibrationPairs( - existingCalibration.pairs, - )); - source = readCalibrationSource(existingCalibration.source); - } - } catch (err) { - console.warn(`Unable to read existing ${calibrationFileAbsPath}: ${err}`); - } - } + const onDisk = await loadCalibrationFiles(projectDirInfo.basePath); + let { homographies, correspondences, transformTypes } = onDisk; + let { source } = onDisk; if (args.cameraHomographies) { homographies = args.cameraHomographies; } @@ -888,11 +967,52 @@ async function saveMetadata(settings: Settings, datasetId: string, args: Dataset if (args.cameraCalibrationSource !== undefined) { source = args.cameraCalibrationSource; } - await _saveAsJson(calibrationFileAbsPath, { - version: CalibrationFileVersion, - ...(source ? { source } : {}), - pairs: toCalibrationPairs(homographies, correspondences, transformTypes), + // Group pairs by their non-reference camera (reference = first camera in + // display order). A pair not touching the reference files under its right + // camera; grouping is cosmetic either way since pair bodies are + // authoritative on load. + const reference = existing.multiCam + ? orderedMultiCamCameraNames(existing.multiCam)[0] ?? null + : null; + const pairsByCamera = new Map(); + toCalibrationPairs(homographies, correspondences, transformTypes).forEach((pair) => { + let camera = pair.right; + if (reference !== null && pair.right === reference && pair.left !== reference) { + camera = pair.left; + } + pairsByCamera.set(camera, [...(pairsByCamera.get(camera) ?? []), pair]); }); + // A mixed composite stamp describes the merged SET, not any single file; + // stamping every per-camera file with it would make the next load read a + // unanimous (therefore "consistent") rig, hiding the very mismatch it + // records. Per-file stamps resume with the next externally produced file. + const fileSource = source && (source as Record).mixed === true + ? null + : source; + const expected = new Set(); + await Promise.all([...pairsByCamera.entries()].map(([camera, pairs]) => { + const name = perCameraCalibrationFileName(camera); + expected.add(name); + return _saveAsJson(npath.join(projectDirInfo.basePath, name), { + type: CALIBRATION_FILE_TYPE, + version: CalibrationFileVersion, + ...(fileSource ? { source: fileSource } : {}), + pairs, + }); + })); + // Remove the legacy all-pairs file and any per-camera file whose pairs no + // longer exist (e.g. a cleared pair), so the on-disk set always mirrors + // the saved calibration exactly. + try { + const entries = await fs.readdir(projectDirInfo.basePath, { withFileTypes: true }); + await Promise.all(entries + .filter((entry) => entry.isFile() + && !expected.has(entry.name) + && (entry.name === CalibrationFileName || PerCameraCalibrationFileName.test(entry.name))) + .map((entry) => fs.remove(npath.join(projectDirInfo.basePath, entry.name)))); + } catch (err) { + console.warn(`Unable to clean up stale calibration files: ${err}`); + } } await _saveAsJson(projectDirInfo.metaFileAbsPath, existing); @@ -1333,31 +1453,34 @@ async function findParentFolderCalibrationFile(parentPath: string): Promise.json files, then any other + * self-identified candidates, alphabetically within each group. */ -async function findParentFolderTransformFile(parentPath: string): Promise { +async function findParentFolderTransformFiles(parentPath: string): Promise { if (!await fs.pathExists(parentPath)) { - return null; + return []; } const stat = await fs.stat(parentPath); if (!stat.isDirectory()) { - return null; + return []; } const children = await fs.readdir(parentPath, { withFileTypes: true }); + const rank = (name: string) => { + if (name.toLowerCase() === CalibrationFileName) return 0; + if (PerCameraCalibrationFileName.test(name)) return 1; + return 2; + }; const candidates = children .filter((entry) => entry.isFile() && /\.json$/i.test(entry.name)) .map((entry) => entry.name) - .sort((a, b) => { - const aExact = a.toLowerCase() === CalibrationFileName ? 0 : 1; - const bExact = b.toLowerCase() === CalibrationFileName ? 0 : 1; - return aExact - bExact || a.localeCompare(b); - }); + .sort((a, b) => rank(a) - rank(b) || a.localeCompare(b)); + const found: string[] = []; // eslint-disable-next-line no-restricted-syntax for (const name of candidates) { const absPath = npath.join(parentPath, name); @@ -1365,13 +1488,13 @@ async function findParentFolderTransformFile(parentPath: string): Promise; type CameraCorrespondences = NonNullable; @@ -103,7 +103,7 @@ async function beginMultiCamImport(args: MultiCamImportArgs): Promise { - return window.diveDesktop.invoke('find-parent-folder-transform-file', { path: parentPath }); +function findParentFolderTransformFiles(parentPath: string): Promise { + return window.diveDesktop.invoke('find-parent-folder-transform-files', { path: parentPath }); } function hasCalibrationFile(datasetId: string): Promise { @@ -708,7 +708,7 @@ export { listParentFolderCameras, resolveMulticamCameraSourcePath, findParentFolderCalibrationFile, - findParentFolderTransformFile, + findParentFolderTransformFiles, hasCalibrationFile, bulkImportMedia, deleteDataset, diff --git a/client/src/CameraCalibrationStore.spec.ts b/client/src/CameraCalibrationStore.spec.ts index b498607ab..7810d9ff6 100644 --- a/client/src/CameraCalibrationStore.spec.ts +++ b/client/src/CameraCalibrationStore.spec.ts @@ -77,6 +77,20 @@ describe('CameraCalibrationStore', () => { }); }); + describe('sourceIsMixed', () => { + it('flags only the mixed composite stamp the file merger produces', () => { + const store = new CameraCalibrationStore(); + expect(store.sourceIsMixed()).toBe(false); + store.hydrate(undefined, undefined, undefined, { producer: 'kamera', run: 'fl07' }); + expect(store.sourceIsMixed()).toBe(false); + store.hydrate(undefined, undefined, undefined, { + mixed: true, + files: { 'calibration_ir.json': { run: 'fl07' }, 'calibration_uv.json': { run: 'fl09' } }, + }); + expect(store.sourceIsMixed()).toBe(true); + }); + }); + describe('loaded (file-sourced) homographies', () => { /** Load a matrix-only (point-less) pair from calibration JSON, marking it 'loaded'. */ function loadMatrixOnlyPair(store: CameraCalibrationStore, left: string, right: string, rightToLeft: number[][]) { diff --git a/client/src/CameraCalibrationStore.ts b/client/src/CameraCalibrationStore.ts index 90c02fe44..6842d7608 100644 --- a/client/src/CameraCalibrationStore.ts +++ b/client/src/CameraCalibrationStore.ts @@ -51,10 +51,11 @@ export type CalibrationSource = Record; /** * One camera pair in the portable calibration JSON file. This is the same * self-describing shape the desktop platform persists as the project's - * standalone calibration.json (see desktop backend/native/common.ts), so a - * panel-saved file, the on-disk artifact, and an import-time seed are all - * interchangeable: correspondences flattened as [leftX, leftY, rightX, - * rightY] rows, plus both fitted directions (null when unfitted). + * standalone per-camera calibration_.json files (see desktop + * backend/native/common.ts), so a panel-saved file, the on-disk artifacts, + * and an import-time seed are all interchangeable: correspondences flattened + * as [leftX, leftY, rightX, rightY] rows, plus both fitted directions (null + * when unfitted). */ export interface CalibrationFilePair { left: string; @@ -145,6 +146,18 @@ export default class CameraCalibrationStore { this.savedSnapshot.value = this.calibrationSnapshot(); } + /** + * True when the loaded calibration was assembled from per-camera files + * whose producer stamps disagree (the loader records that as a + * `{ mixed: true, files: {...} }` composite source) -- i.e. the rig may + * mix calibration generations and deserves a visible warning rather than + * silent composition. + */ + sourceIsMixed(): boolean { + return Boolean(this.source.value + && (this.source.value as Record).mixed === true); + } + /** * Directional key for a camera pair: `left::right`. Order is significant and * preserved so left/right (e.g. RGB vs IR) survives for ordered exports. From 11a3303234deff421ea27103388e9d8565dcc6ba Mon Sep 17 00:00:00 2001 From: romleiaj Date: Thu, 9 Jul 2026 21:24:00 -0400 Subject: [PATCH 6/7] Export the camera calibration and drop the all-pairs calibration.json The per-camera calibration_.json files are now the only on-disk form: the single all-pairs calibration.json is no longer read, migrated, or given priority during import discovery. The Export menu gains explicit calibration export -- one file per calibrated camera, or all of them zipped -- built from the same per-camera grouping the save path uses, and the multicam Everything zip now carries the files too. Exports resolve the calibration the same way loading does (standalone files first, the import-time meta seed otherwise) and never stamp a file with a mixed composite source. Co-Authored-By: Claude Fable 5 --- .../useImportMultiCamDialog.ts | 8 +- client/platform/desktop/backend/ipcService.ts | 5 + .../desktop/backend/native/common.spec.ts | 144 +++++++---- .../platform/desktop/backend/native/common.ts | 231 ++++++++++++------ .../desktop/backend/native/multiCamImport.ts | 2 +- client/platform/desktop/frontend/api.ts | 13 + .../desktop/frontend/components/Export.vue | 86 ++++++- 7 files changed, 370 insertions(+), 119 deletions(-) diff --git a/client/dive-common/components/ImportMultiCamDialog/useImportMultiCamDialog.ts b/client/dive-common/components/ImportMultiCamDialog/useImportMultiCamDialog.ts index 069d8eca3..17103930b 100644 --- a/client/dive-common/components/ImportMultiCamDialog/useImportMultiCamDialog.ts +++ b/client/dive-common/components/ImportMultiCamDialog/useImportMultiCamDialog.ts @@ -720,10 +720,10 @@ export function useImportMultiCamDialog( * Auto-attach the DIVE camera-calibration .json files found in the parent * folder root as the import's transform files (desktop multicam subfolder * imports only). A per-camera calibration_.json is attached to its - * matching camera slot; anything else (e.g. an all-pairs calibration.json) - * goes to the first free camera after the reference -- the file's pairs - * name their own cameras, so which slot carries it doesn't matter. Each - * shows in that camera's (clearable) transform field. + * matching camera slot; any other self-identified calibration file goes to + * the first free camera after the reference -- the file's pairs name their + * own cameras, so which slot carries it doesn't matter. Each shows in that + * camera's (clearable) transform field. */ async function discoverParentFolderTransform(parentPath: string) { if (!transformImportEnabled.value || !findParentFolderTransformFiles) { diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index 16e90e046..37d3b213c 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -269,6 +269,11 @@ export default function register() { return { exportedPath }; }); + ipcMain.handle('export-camera-calibration', async (_, { id, destPath, camera }: { id: string; destPath: string; camera?: string }) => { + const exportedPath = await common.exportCameraCalibration(settings.get(), id, destPath, camera); + return { exportedPath }; + }); + ipcMain.handle('get-dataset-calibration', async (_, { datasetId }: { datasetId: string }) => common.getDatasetCalibration(settings.get(), datasetId)); ipcMain.handle('delete-calibration', async (_, { datasetId }: { datasetId: string }) => { diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 4df182193..3d3dd2fd5 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -1,5 +1,6 @@ import mockfs from 'mock-fs'; import npath from 'path'; +import os from 'os'; import fs from 'fs-extra'; import { Console } from 'console'; @@ -174,6 +175,8 @@ beforeEach(() => { }, '/home/user/testPairs': { ...fileSystemData }, '/home/user/output': {}, + // Zip exports stage their files in a real temp dir before archiving. + [os.tmpdir()]: {}, '/home/user/transformDiscovery': { exactName: { 'aaa-stamped.json': JSON.stringify({ type: 'dive-camera-calibration', version: 1, pairs: [] }), @@ -685,7 +688,7 @@ describe('native.common', () => { // Persisted as a standalone per-camera file (named for the pair's // non-reference camera): pairs labeled left/right, with points laid out - // as leftX leftY rightX rightY. No legacy all-pairs calibration.json. + // as leftX leftY rightX rightY. Never a single all-pairs calibration.json. const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); const calibrationPath = npath.join(projectDir, 'calibration_ir.json'); expect(await fs.pathExists(calibrationPath)).toBe(true); @@ -743,11 +746,11 @@ describe('native.common', () => { }); describe('findParentFolderTransformFiles', () => { - it('orders a file named calibration.json before other marked candidates', async () => { + it('gives a file named calibration.json no special priority', async () => { const found = await common.findParentFolderTransformFiles('/home/user/transformDiscovery/exactName'); expect(found).toStrictEqual([ - npath.join('/home/user/transformDiscovery/exactName', 'calibration.json'), npath.join('/home/user/transformDiscovery/exactName', 'aaa-stamped.json'), + npath.join('/home/user/transformDiscovery/exactName', 'calibration.json'), ]); }); @@ -839,45 +842,6 @@ describe('native.common', () => { expect('source' in (await fs.readJSON(calibrationPath))).toBe(false); }); - it('migrates a legacy all-pairs calibration.json to per-camera files on save', async () => { - const payload = await common.beginMediaImport( - '/home/user/data/imageLists/success/image_list.txt', - ); - const res = await common.finalizeMediaImport(settings, payload); - const final = res.meta; - const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); - - // A legacy single-file calibration holding two pairs. - await fs.writeJSON(npath.join(projectDir, 'calibration.json'), { - version: 1, - pairs: [ - { - left: 'rgb', right: 'ir', points: [], leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], rightToLeft: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], - }, - { - left: 'rgb', right: 'uv', points: [], leftToRight: [[1, 0, 8], [0, 1, 2], [0, 0, 1]], rightToLeft: [[1, 0, -8], [0, 1, -2], [0, 0, 1]], - }, - ], - }); - - // Loads fine from the legacy file. - const loaded = await common.loadMetadata(settings, final.id, urlMapper); - expect(Object.keys(loaded.cameraHomographies ?? {}).sort()).toStrictEqual(['rgb::ir', 'rgb::uv']); - - // A save re-writes the set as per-camera files and removes the legacy one. - await common.saveMetadata(settings, final.id, { - cameraHomographies: loaded.cameraHomographies, - cameraCorrespondences: loaded.cameraCorrespondences, - cameraTransformTypes: loaded.cameraTransformTypes, - }); - expect(await fs.pathExists(npath.join(projectDir, 'calibration.json'))).toBe(false); - expect(await fs.pathExists(npath.join(projectDir, 'calibration_ir.json'))).toBe(true); - expect(await fs.pathExists(npath.join(projectDir, 'calibration_uv.json'))).toBe(true); - - const reloaded = await common.loadMetadata(settings, final.id, urlMapper); - expect(reloaded.cameraHomographies).toStrictEqual(loaded.cameraHomographies); - }); - it('merges per-camera calibration files and flags disagreeing source stamps', async () => { const payload = await common.beginMediaImport( '/home/user/data/imageLists/success/image_list.txt', @@ -931,6 +895,102 @@ describe('native.common', () => { expect('source' in (await fs.readJSON(npath.join(projectDir, 'calibration_ir.json')))).toBe(false); }); + it('exportCameraCalibration writes a single per-camera file from the saved calibration', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + const final = res.meta; + const cameraHomographies = { + 'rgb::ir': { + AtoB: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + BtoA: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }, + 'rgb::uv': { + AtoB: [[1, 0, 8], [0, 1, 2], [0, 0, 1]], + BtoA: [[1, 0, -8], [0, 1, -2], [0, 0, 1]], + }, + }; + const source = { producer: 'kamera', run: 'fl07' }; + await common.saveMetadata(settings, final.id, { + cameraHomographies, + cameraCalibrationSource: source, + }); + + const destPath = '/home/user/output/calibration_ir.json'; + await common.exportCameraCalibration(settings, final.id, destPath, 'ir'); + const exported = await fs.readJSON(destPath); + // Self-identifies so parent-folder discovery recognizes it on re-import, + // and carries only its own camera's pair plus the producer stamp. + expect(exported.type).toBe('dive-camera-calibration'); + expect(exported.source).toStrictEqual(source); + expect(exported.pairs).toHaveLength(1); + expect(exported.pairs[0].left).toBe('rgb'); + expect(exported.pairs[0].right).toBe('ir'); + expect(exported.pairs[0].leftToRight).toStrictEqual([[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + + // A camera with no calibration refuses. + await expect(common.exportCameraCalibration(settings, final.id, '/home/user/output/nope.json', 'zz')).rejects.toThrow('no calibration for camera'); + }); + + it('exportCameraCalibration zips every per-camera file when no camera is given', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + const final = res.meta; + await common.saveMetadata(settings, final.id, { + cameraHomographies: { + 'rgb::ir': { + AtoB: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + BtoA: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }, + 'rgb::uv': { + AtoB: [[1, 0, 8], [0, 1, 2], [0, 0, 1]], + BtoA: [[1, 0, -8], [0, 1, -2], [0, 0, 1]], + }, + }, + }); + + const destPath = '/home/user/output/calibrations.zip'; + await common.exportCameraCalibration(settings, final.id, destPath); + expect((await fs.stat(destPath)).size).toBeGreaterThan(0); + }); + + it('exportCameraCalibration never stamps exported files with a mixed composite', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + const final = res.meta; + // A mixed composite stamp describes the assembled set, not any single + // file; stamping an exported file with it would present a unanimous rig. + await common.saveMetadata(settings, final.id, { + cameraHomographies: { + 'rgb::ir': { + AtoB: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + BtoA: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }, + }, + cameraCalibrationSource: { + mixed: true, + files: { 'calibration_ir.json': { producer: 'kamera', run: 'fl07' } }, + }, + }); + + const destPath = '/home/user/output/calibration_ir.json'; + await common.exportCameraCalibration(settings, final.id, destPath, 'ir'); + expect('source' in (await fs.readJSON(destPath))).toBe(false); + }); + + it('exportCameraCalibration refuses when the dataset has no calibration', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + await expect(common.exportCameraCalibration(settings, res.meta.id, '/home/user/output/none.json', 'ir')).rejects.toThrow('no camera calibration to export'); + }); + it('import with CSV annotations without specifying track file', async () => { const payload = await common.beginMediaImport('/home/user/data/imageSuccessWithAnnotations'); payload.trackFileAbsPath = ''; //It returns null be default but users change it. diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 288c3935a..045df1ba2 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -75,14 +75,13 @@ const JsonTrackFileName = /^result(_.*)?\.json$/i; const JsonFileName = /^.*\.json$/i; const JsonMetaFileName = 'meta.json'; // Standalone camera-to-camera (modality-to-modality) alignment transforms, -// stored separately from meta.json in the dataset directory. The current -// convention is one file per non-reference camera (calibration_.json, -// each holding that camera's pair(s)); a single legacy calibration.json -// holding every pair is still read. File NAMES are discovery/provenance only: -// the pair's left/right names inside the file body are always authoritative, -// so a misnamed or copied file can never rebind a transform to the wrong -// camera. -const CalibrationFileName = 'calibration.json'; +// stored separately from meta.json in the dataset directory: one file per +// non-reference camera (calibration_.json, matching the per-camera +// files kamera users handle), each holding that camera's pair(s). There is +// deliberately no single all-pairs file. File NAMES are discovery/provenance +// only: the pair's left/right names inside the file body are always +// authoritative, so a misnamed or copied file can never rebind a transform +// to the wrong camera. const PerCameraCalibrationFileName = /^calibration_(.+)\.json$/i; const CalibrationFileVersion = 1; @@ -386,9 +385,9 @@ async function loadMetadata( const projectDirData = await getValidatedProjectDir(settings, datasetId); const projectMetaData = await loadJsonMetadata(projectDirData.metaFileAbsPath); - // Load standalone camera calibration (transforms + correspondences), if - // present: the per-camera calibration_.json files and/or the legacy - // all-pairs calibration.json, merged. + // Load standalone camera calibration (transforms + correspondences) from + // the per-camera calibration_.json files, if present; the dataset + // meta fields serve as the import-time seed until a save writes the files. let { cameraHomographies, cameraCorrespondences, cameraTransformTypes, cameraCalibrationSource, } = projectMetaData; @@ -852,13 +851,12 @@ function mergeCalibrationSources( } /** - * Read and merge every calibration file in a dataset directory: the legacy - * all-pairs calibration.json (if present) plus each per-camera - * calibration_.json, in that order, sorted by name for determinism. - * Pair bodies are authoritative (file names are ignored for binding); a pair - * key appearing in more than one file keeps the last occurrence with a - * warning. `found` is true when at least one file parsed as a calibration, - * so callers can distinguish "no calibration files" from an empty one. + * Read and merge every per-camera calibration_.json in a dataset + * directory, sorted by name for determinism. Pair bodies are authoritative + * (file names are ignored for binding); a pair key appearing in more than + * one file keeps the last occurrence with a warning. `found` is true when at + * least one file parsed as a calibration, so callers can distinguish "no + * calibration files" from an empty one. */ async function loadCalibrationFiles(basePath: string): Promise<{ found: boolean; @@ -867,16 +865,13 @@ async function loadCalibrationFiles(basePath: string): Promise<{ transformTypes: CameraTransformTypes; source: CalibrationSource | null; }> { - const names: string[] = []; - if (await fs.pathExists(npath.join(basePath, CalibrationFileName))) { - names.push(CalibrationFileName); - } + let names: string[] = []; try { const entries = await fs.readdir(basePath, { withFileTypes: true }); - names.push(...entries + names = entries .filter((entry) => entry.isFile() && PerCameraCalibrationFileName.test(entry.name)) .map((entry) => entry.name) - .sort((a, b) => a.localeCompare(b))); + .sort((a, b) => a.localeCompare(b)); } catch { // Unreadable directory: treated the same as no calibration files. } @@ -946,8 +941,7 @@ async function saveMetadata(settings: Settings, datasetId: string, args: Dataset // one per non-reference camera, matching the per-camera files users handle // from kamera -- rather than embedded in meta.json, so each camera's // calibration is easy to find, hand-edit, and consume as a self-contained - // artifact. Any legacy all-pairs calibration.json is migrated (read above, - // removed below) on the first save. + // artifact. There is deliberately never a single all-pairs file. if (args.cameraHomographies || args.cameraCorrespondences || args.cameraTransformTypes || args.cameraCalibrationSource) { // Start from whatever is on disk so a partial update doesn't clobber the rest. @@ -967,48 +961,22 @@ async function saveMetadata(settings: Settings, datasetId: string, args: Dataset if (args.cameraCalibrationSource !== undefined) { source = args.cameraCalibrationSource; } - // Group pairs by their non-reference camera (reference = first camera in - // display order). A pair not touching the reference files under its right - // camera; grouping is cosmetic either way since pair bodies are - // authoritative on load. - const reference = existing.multiCam - ? orderedMultiCamCameraNames(existing.multiCam)[0] ?? null - : null; - const pairsByCamera = new Map(); - toCalibrationPairs(homographies, correspondences, transformTypes).forEach((pair) => { - let camera = pair.right; - if (reference !== null && pair.right === reference && pair.left !== reference) { - camera = pair.left; - } - pairsByCamera.set(camera, [...(pairsByCamera.get(camera) ?? []), pair]); - }); - // A mixed composite stamp describes the merged SET, not any single file; - // stamping every per-camera file with it would make the next load read a - // unanimous (therefore "consistent") rig, hiding the very mismatch it - // records. Per-file stamps resume with the next externally produced file. - const fileSource = source && (source as Record).mixed === true - ? null - : source; - const expected = new Set(); - await Promise.all([...pairsByCamera.entries()].map(([camera, pairs]) => { - const name = perCameraCalibrationFileName(camera); - expected.add(name); - return _saveAsJson(npath.join(projectDirInfo.basePath, name), { - type: CALIBRATION_FILE_TYPE, - version: CalibrationFileVersion, - ...(fileSource ? { source: fileSource } : {}), - pairs, - }); - })); - // Remove the legacy all-pairs file and any per-camera file whose pairs no - // longer exist (e.g. a cleared pair), so the on-disk set always mirrors - // the saved calibration exactly. + const files = buildPerCameraCalibrationFiles( + { + homographies, correspondences, transformTypes, source, + }, + existing.multiCam, + ); + const expected = new Set(files.map((file) => file.name)); + await Promise.all(files.map((file) => _saveAsJson(npath.join(projectDirInfo.basePath, file.name), file.body))); + // Remove any per-camera file whose pairs no longer exist (e.g. a cleared + // pair), so the on-disk set always mirrors the saved calibration exactly. try { const entries = await fs.readdir(projectDirInfo.basePath, { withFileTypes: true }); await Promise.all(entries .filter((entry) => entry.isFile() && !expected.has(entry.name) - && (entry.name === CalibrationFileName || PerCameraCalibrationFileName.test(entry.name))) + && PerCameraCalibrationFileName.test(entry.name)) .map((entry) => fs.remove(npath.join(projectDirInfo.basePath, entry.name)))); } catch (err) { console.warn(`Unable to clean up stale calibration files: ${err}`); @@ -1458,9 +1426,9 @@ async function findParentFolderCalibrationFile(parentPath: string): Promise.json files, then any other - * self-identified candidates, alphabetically within each group. + * Ordered for deterministic attachment: per-camera calibration_.json + * files first, then any other self-identified candidates, alphabetically + * within each group. */ async function findParentFolderTransformFiles(parentPath: string): Promise { if (!await fs.pathExists(parentPath)) { @@ -1471,11 +1439,7 @@ async function findParentFolderTransformFiles(parentPath: string): Promise { - if (name.toLowerCase() === CalibrationFileName) return 0; - if (PerCameraCalibrationFileName.test(name)) return 1; - return 2; - }; + const rank = (name: string) => (PerCameraCalibrationFileName.test(name) ? 0 : 1); const candidates = children .filter((entry) => entry.isFile() && /\.json$/i.test(entry.name)) .map((entry) => entry.name) @@ -2078,6 +2042,121 @@ async function zipDirectory(sourceDir: string, destZipPath: string): Promise.json per non-reference camera (reference = first + * camera in display order), the same per-camera format kamera produces and + * the multicam import consumes. A pair not touching the reference files + * under its right camera; grouping is cosmetic either way since pair bodies + * are authoritative on load. + */ +function buildPerCameraCalibrationFiles( + calibration: CalibrationValues, + multiCam: JsonMeta['multiCam'], +): { name: string; body: object }[] { + const reference = multiCam + ? orderedMultiCamCameraNames(multiCam)[0] ?? null + : null; + const pairsByCamera = new Map(); + toCalibrationPairs( + calibration.homographies, + calibration.correspondences, + calibration.transformTypes, + ).forEach((pair) => { + let camera = pair.right; + if (reference !== null && pair.right === reference && pair.left !== reference) { + camera = pair.left; + } + pairsByCamera.set(camera, [...(pairsByCamera.get(camera) ?? []), pair]); + }); + // A mixed composite stamp describes the assembled SET, not any single file; + // stamping every per-camera file with it would present a unanimous + // (therefore "consistent") rig on the next load, hiding the very mismatch + // it records. Per-file stamps resume with the next externally produced file. + const fileSource = calibration.source + && (calibration.source as Record).mixed === true + ? null + : calibration.source; + return [...pairsByCamera.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([camera, pairs]) => ({ + name: perCameraCalibrationFileName(camera), + body: { + type: CALIBRATION_FILE_TYPE, + version: CalibrationFileVersion, + ...(fileSource ? { source: fileSource } : {}), + pairs, + }, + })); +} + +/** + * The calibration a dataset currently resolves to, from the same sources + * loadMetadata uses: the standalone per-camera files, or the import-time + * seed in the dataset meta when no files have been written yet. + */ +async function loadEffectiveCalibration( + basePath: string, + meta: JsonMeta, +): Promise { + const onDisk = await loadCalibrationFiles(basePath); + if (onDisk.found) { + return onDisk; + } + return { + homographies: meta.cameraHomographies ?? {}, + correspondences: meta.cameraCorrespondences ?? {}, + transformTypes: meta.cameraTransformTypes ?? {}, + source: meta.cameraCalibrationSource ?? null, + }; +} + +/** + * Export a dataset's camera calibration. With a camera name, write that + * camera's single calibration_.json to destPath; without one, write + * every per-camera file into a zip at destPath. + * @returns the destination path written + */ +async function exportCameraCalibration( + settings: Settings, + datasetId: string, + destPath: string, + camera?: string, +): Promise { + const projectDirInfo = await getValidatedProjectDir(settings, datasetId.split('/')[0]); + const meta = await loadJsonMetadata(projectDirInfo.metaFileAbsPath); + const calibration = await loadEffectiveCalibration(projectDirInfo.basePath, meta); + const files = buildPerCameraCalibrationFiles(calibration, meta.multiCam); + if (!files.length) { + throw new Error(`Dataset ${datasetId} has no camera calibration to export.`); + } + if (camera !== undefined) { + const match = files.find((file) => file.name === perCameraCalibrationFileName(camera)); + if (!match) { + throw new Error(`Dataset ${datasetId} has no calibration for camera "${camera}".`); + } + await _saveAsJson(destPath, match.body); + return destPath; + } + const tempDir = await fs.mkdtemp(npath.join(os.tmpdir(), 'dive-calibration-')); + try { + await Promise.all(files.map( + (file) => _saveAsJson(npath.join(tempDir, file.name), file.body), + )); + await zipDirectory(tempDir, destPath); + } finally { + await fs.remove(tempDir); + } + return destPath; +} + async function exportMulticamEverything( settings: Settings, args: ExportMulticamEverythingArgs, @@ -2122,6 +2201,15 @@ async function exportMulticamEverything( await fs.copy(calibrationPath, npath.join(datasetDir, calibrationName)); } + // Regenerate the camera-alignment calibration as its per-camera files so + // the zip carries it even when only the import-time seed exists. + const alignmentCalibration = await loadEffectiveCalibration(parentDirInfo.basePath, parentMeta); + await Promise.all( + buildPerCameraCalibrationFiles(alignmentCalibration, parentMeta.multiCam).map( + (file) => _saveAsJson(npath.join(datasetDir, file.name), file.body), + ), + ); + for (let i = 0; i < cameraNames.length; i += 1) { const cameraName = cameraNames[i]; // eslint-disable-next-line no-await-in-loop @@ -2489,6 +2577,7 @@ export { getDatasetCalibrationExportPath, setDatasetCalibration, exportDatasetCalibration, + exportCameraCalibration, getDatasetCalibration, deleteDatasetCalibration, }; diff --git a/client/platform/desktop/backend/native/multiCamImport.ts b/client/platform/desktop/backend/native/multiCamImport.ts index 260c35a12..4810bcec7 100644 --- a/client/platform/desktop/backend/native/multiCamImport.ts +++ b/client/platform/desktop/backend/native/multiCamImport.ts @@ -99,7 +99,7 @@ async function beginMultiCamImport(args: MultiCamImportArgs): Promise.json + * when a camera is given, otherwise every per-camera file zipped to destPath. + */ +function exportCameraCalibration( + datasetId: string, + destPath: string, + camera?: string, +): Promise<{ exportedPath: string }> { + return window.diveDesktop.invoke('export-camera-calibration', { id: datasetId, destPath, camera }); +} + function getDatasetCalibration(datasetId: string): Promise { return window.diveDesktop.invoke('get-dataset-calibration', { datasetId }); } @@ -723,6 +735,7 @@ export { saveCalibration, importCalibrationFile, exportCalibrationFile, + exportCameraCalibration, getDatasetCalibration, downloadCalibration, deleteCalibration, diff --git a/client/platform/desktop/frontend/components/Export.vue b/client/platform/desktop/frontend/components/Export.vue index eb0728f26..695797fef 100644 --- a/client/platform/desktop/frontend/components/Export.vue +++ b/client/platform/desktop/frontend/components/Export.vue @@ -8,8 +8,10 @@ import { } from 'vue-media-annotator/provides'; import AutosavePrompt from 'dive-common/components/AutosavePrompt.vue'; import { MultiType } from 'dive-common/constants'; +import { orderedMultiCamCameraNames } from 'dive-common/multicamDisplay'; import { - loadMetadata, exportDataset, exportConfiguration, exportCalibrationFile, exportMulticamEverything, + loadMetadata, exportDataset, exportConfiguration, exportCalibrationFile, + exportCameraCalibration, exportMulticamEverything, } from 'platform/desktop/frontend/api'; import type { JsonMeta } from 'platform/desktop/constants'; @@ -90,6 +92,55 @@ export default defineComponent({ () => data.meta?.subType === 'stereo' && !!calibrationExportName.value, ); + // Cameras with an exportable alignment calibration: each pair files under + // its non-reference camera (reference = first camera in display order), + // mirroring how the backend groups pairs into calibration_.json. + const calibrationCameras = computed(() => { + const { meta } = data; + if (!meta || meta.type !== MultiType || !meta.multiCam) { + return []; + } + const pairKeys = new Set([ + ...Object.keys(meta.cameraHomographies ?? {}), + ...Object.keys(meta.cameraCorrespondences ?? {}), + ...Object.keys(meta.cameraTransformTypes ?? {}), + ]); + const reference = orderedMultiCamCameraNames(meta.multiCam)[0] ?? null; + const cameras = new Set(); + pairKeys.forEach((key) => { + const [left, right] = key.split('::'); + let camera = right; + if (reference !== null && right === reference && left !== reference) { + camera = left; + } + cameras.add(camera); + }); + return [...cameras].sort(); + }); + + async function exportCalibration(camera?: string) { + const defaultName = camera === undefined + ? `${data.meta?.name ?? 'dataset'}_calibration.zip` + : `calibration_${camera}.json`; + const location = await window.diveDesktop.showSaveDialog({ + title: 'Export Camera Calibration', + defaultPath: defaultName, + }); + if (location.canceled || !location.filePath) return; + try { + data.err = null; + const { exportedPath } = await exportCameraCalibration( + parentId.value, + location.filePath, + camera, + ); + data.outPath = exportedPath; + } catch (err) { + data.err = err; + throw err; + } + } + async function exportCameraFile() { if (!calibrationExportName.value) return; const location = await window.diveDesktop.showSaveDialog({ @@ -147,6 +198,8 @@ export default defineComponent({ data, doExport, exportCameraFile, + exportCalibration, + calibrationCameras, cameraFileSupported, calibrationExportName, savePrompt, @@ -352,6 +405,37 @@ export default defineComponent({ +