diff --git a/client/dive-common/alignedTimeline.spec.ts b/client/dive-common/alignedTimeline.spec.ts new file mode 100644 index 000000000..d0d0520fb --- /dev/null +++ b/client/dive-common/alignedTimeline.spec.ts @@ -0,0 +1,211 @@ +import type { FrameImage } from './apispec'; +import { + buildAlignedTimeline, buildInverseAlignedIndex, canAlign, computeGapGradient, computeGapSlots, +} from './alignedTimeline'; + +function frame(timestamp?: number): FrameImage { + return { url: `url-${timestamp}`, filename: `frame-${timestamp}.png`, timestamp }; +} + +describe('alignedTimeline', () => { + it('aligns when every camera has a timestamp on every frame', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100), frame(200), frame(300)], + }; + expect(canAlign(camerasFrames)).toBe(true); + const result = buildAlignedTimeline(camerasFrames); + expect(result).toEqual({ + aligned: true, + slots: [ + { A: 0, B: 0 }, + { A: 1, B: 1 }, + { A: 2, B: 2 }, + ], + }); + }); + + it('falls back when any single frame anywhere is missing a timestamp', () => { + const camerasFrames = { + A: [frame(100), frame(undefined)], + B: [frame(100), frame(200)], + }; + expect(canAlign(camerasFrames)).toBe(false); + expect(buildAlignedTimeline(camerasFrames)).toEqual({ aligned: false }); + }); + + it('resolves a genuinely missing frame on one camera to undefined at that slot', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100), frame(300)], + }; + const result = buildAlignedTimeline(camerasFrames); + expect(result).toEqual({ + aligned: true, + slots: [ + { A: 0, B: 0 }, + { A: 1, B: undefined }, + { A: 2, B: 1 }, + ], + }); + }); + + it('merges cameras with clock drift within tolerance into one slot', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100.45), frame(200.4), frame(300.3)], + }; + const result = buildAlignedTimeline(camerasFrames, 0.5); + expect(result).toEqual({ + aligned: true, + slots: [ + { A: 0, B: 0 }, + { A: 1, B: 1 }, + { A: 2, B: 2 }, + ], + }); + }); + + it('splits two same-camera frames within one tolerance window into separate slots', () => { + const camerasFrames = { + A: [frame(100), frame(100.2)], + B: [frame(100.1)], + }; + const result = buildAlignedTimeline(camerasFrames, 0.5); + expect(result).toEqual({ + aligned: true, + slots: [ + { A: 0, B: 0 }, + { A: 1, B: undefined }, + ], + }); + }); + + it('pairs frames positionally when every frame shares one identical timestamp', () => { + // Calibration-style datasets (e.g. C/L/R shots stamped with a single + // collection timestamp) must reduce to positional pairing, not spill each + // camera's frames into separate mostly-blank slots. + const t = 1712495277.206341; + const camerasFrames = { + EO: [frame(t), frame(t), frame(t)], + IR: [frame(t), frame(t), frame(t)], + }; + const result = buildAlignedTimeline(camerasFrames, 0.5); + expect(result).toEqual({ + aligned: true, + slots: [ + { EO: 0, IR: 0 }, + { EO: 1, IR: 1 }, + { EO: 2, IR: 2 }, + ], + }); + }); + + it('disqualifies a dataset with an empty-array (e.g. video-backed) camera', () => { + const camerasFrames = { + A: [] as FrameImage[], + B: [frame(100)], + }; + expect(canAlign(camerasFrames)).toBe(false); + expect(buildAlignedTimeline(camerasFrames)).toEqual({ aligned: false }); + }); + + it('disqualifies datasets with fewer than two cameras', () => { + const camerasFrames = { + A: [frame(100), frame(200)], + }; + expect(canAlign(camerasFrames)).toBe(false); + expect(buildAlignedTimeline(camerasFrames)).toEqual({ aligned: false }); + }); + + describe('computeGapSlots', () => { + it('returns no gaps when every camera has a frame at every slot', () => { + const camerasFrames = { + A: [frame(100), frame(200)], + B: [frame(100), frame(200)], + }; + const result = buildAlignedTimeline(camerasFrames); + expect(result.aligned).toBe(true); + expect(result.aligned && computeGapSlots(result.slots)).toEqual([]); + }); + + it('flags slots where any camera is missing a frame', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100), frame(300)], + }; + const result = buildAlignedTimeline(camerasFrames); + expect(result.aligned).toBe(true); + expect(result.aligned && computeGapSlots(result.slots)).toEqual([1]); + }); + }); + + describe('computeGapGradient', () => { + it('returns none with no gaps or a non-positive maxFrame', () => { + expect(computeGapGradient([], 10)).toBe('none'); + expect(computeGapGradient([1], 0)).toBe('none'); + expect(computeGapGradient([1], -5)).toBe('none'); + }); + + it('centers a gap band on the slider thumb position for its slot', () => { + // Thumb for value 4 of maxFrame 8 sits at 50%; band is [43.75%, 56.25%]. + expect(computeGapGradient([4], 8)).toBe( + 'linear-gradient(to right, transparent 43.75%, #ff5252 43.75%, #ff5252 56.25%, transparent 56.25%)', + ); + }); + + it('keeps a trailing gap at slot === maxFrame visible (clamped to 100%)', () => { + expect(computeGapGradient([8], 8)).toBe( + 'linear-gradient(to right, transparent 93.75%, #ff5252 93.75%, #ff5252 100%, transparent 100%)', + ); + }); + + it('keeps a leading gap at slot 0 visible (clamped to 0%)', () => { + expect(computeGapGradient([0], 8)).toBe( + 'linear-gradient(to right, transparent 0%, #ff5252 0%, #ff5252 6.25%, transparent 6.25%)', + ); + }); + + it('merges consecutive gap slots into a single band', () => { + expect(computeGapGradient([2, 3], 8)).toBe( + 'linear-gradient(to right, transparent 18.75%, #ff5252 18.75%, #ff5252 43.75%, transparent 43.75%)', + ); + }); + + it('emits one band per distinct gap run', () => { + expect(computeGapGradient([1, 3], 8)).toBe( + 'linear-gradient(to right, ' + + 'transparent 6.25%, #ff5252 6.25%, #ff5252 18.75%, transparent 18.75%, ' + + 'transparent 31.25%, #ff5252 31.25%, #ff5252 43.75%, transparent 43.75%)', + ); + }); + + it('widens sub-minimum-width bands around their center so they stay visible', () => { + // One slot of 1024 is ~0.098% wide -- below the 0.25% minimum, so the + // band is widened to exactly 0.25% centered on the thumb at 50%. + expect(computeGapGradient([512], 1024)).toBe( + 'linear-gradient(to right, transparent 49.875%, #ff5252 49.875%, #ff5252 50.125%, transparent 50.125%)', + ); + }); + }); + + describe('buildInverseAlignedIndex', () => { + it('maps each camera\'s local frame back to its global slot', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100), frame(300)], + }; + const result = buildAlignedTimeline(camerasFrames); + expect(result.aligned).toBe(true); + if (!result.aligned) return; + const inverse = buildInverseAlignedIndex(result.slots); + expect(inverse.A.get(0)).toBe(0); + expect(inverse.A.get(1)).toBe(1); + expect(inverse.A.get(2)).toBe(2); + expect(inverse.B.get(0)).toBe(0); + expect(inverse.B.get(1)).toBe(2); + // B has no local frame 2 -- it only ever appears in slots 0 and 2. + expect(inverse.B.get(2)).toBeUndefined(); + }); + }); +}); diff --git a/client/dive-common/alignedTimeline.ts b/client/dive-common/alignedTimeline.ts new file mode 100644 index 000000000..88307e34a --- /dev/null +++ b/client/dive-common/alignedTimeline.ts @@ -0,0 +1,212 @@ +import type { FrameImage } from './apispec'; + +/** + * Max time delta, in seconds, between two cameras' frame timestamps for them to + * be considered "the same instant" (SEAL feature 5). In KAMERA sample data the + * C/L/R cameras share an identical capture timestamp per shot, so for now this + * requires an exact match (0). Widen it only if real captures turn out to have + * sub-second jitter that needs absorbing. + */ +export const FRAME_ALIGNMENT_TOLERANCE_SECONDS = 0; + +/** + * camera name -> local index into that camera's own FrameImage[], or undefined + * if no frame from that camera falls within tolerance of this timeline slot. + */ +export type AlignedSlot = Record; + +export type TimelineResult = + | { aligned: true; slots: AlignedSlot[] } + | { aligned: false }; + +interface FrameTriple { + camera: string; + localIndex: number; + timestamp: number; +} + +/** + * A dataset qualifies for aligned playback only when there are at least two + * cameras, every camera has at least one frame, and every frame on every + * camera has a defined timestamp. A single untimestamped frame anywhere means + * the whole dataset falls back to today's exact positional alignment -- the + * safe, conservative default for datasets whose filenames don't follow a + * recognized timestamp convention (see dive_utils/timestamp_parser.py). + */ +export function canAlign(camerasFrames: Record): boolean { + const cameraNames = Object.keys(camerasFrames); + if (cameraNames.length < 2) { + return false; + } + return cameraNames.every((camera) => { + const frames = camerasFrames[camera]; + return frames.length > 0 && frames.every((frame) => frame.timestamp !== undefined); + }); +} + +/** + * Builds a global aligned timeline from each camera's own frame list, when + * every frame across every camera carries a timestamp (see canAlign). + * + * Algorithm: flatten every camera's frames into (camera, localIndex, + * timestamp) triples, sort ascending by timestamp (tied by camera name then + * local index, for determinism), then sweep once: open a slot at the first + * unassigned triple's timestamp, and absorb subsequent triples within + * `toleranceSeconds` of that slot's start time as long as that camera isn't + * already filled in the open slot. Otherwise close the slot and open a new + * one at that triple. This is deliberately simple (O(n log n), single pass, + * no bipartite matching) -- appropriate given there's no real data yet to + * justify anything smarter. + * + * One explicit simplification: if a camera has two frames within tolerance of + * the same open slot's start (e.g. a faster capture rate than other + * cameras), the second spills into a new slot rather than overwriting the + * first -- first-come-first-served per camera per slot, not silent data loss. + */ +export function buildAlignedTimeline( + camerasFrames: Record, + toleranceSeconds: number = FRAME_ALIGNMENT_TOLERANCE_SECONDS, +): TimelineResult { + if (!canAlign(camerasFrames)) { + return { aligned: false }; + } + + const cameraNames = Object.keys(camerasFrames); + const triples: FrameTriple[] = []; + cameraNames.forEach((camera) => { + camerasFrames[camera].forEach((frame, localIndex) => { + // frame.timestamp is guaranteed defined here by canAlign() above + triples.push({ camera, localIndex, timestamp: frame.timestamp as number }); + }); + }); + + // Tie-break equal timestamps by local index BEFORE camera name: when several + // cameras share identical capture timestamps (e.g. a calibration set where + // every C/L/R shot carries one collection timestamp), interleaving by index + // pairs frame i of every camera into the same slot. Camera-name-first + // ordering would instead sweep all of one camera's same-time frames into + // consecutive single-camera slots (spill), producing a timeline of mostly + // blank panes for data that is really positionally aligned. + triples.sort((a, b) => ( + a.timestamp - b.timestamp + || a.localIndex - b.localIndex + || a.camera.localeCompare(b.camera) + )); + + const slots: AlignedSlot[] = []; + let openSlot: AlignedSlot | null = null; + let openSlotStartTime = 0; + + triples.forEach((triple) => { + const withinTolerance = openSlot !== null + && triple.timestamp - openSlotStartTime <= toleranceSeconds; + const cameraAlreadyFilled = openSlot !== null && openSlot[triple.camera] !== undefined; + + if (!withinTolerance || cameraAlreadyFilled) { + if (openSlot !== null) { + slots.push(openSlot); + } + openSlot = Object.fromEntries(cameraNames.map((camera) => [camera, undefined])); + openSlotStartTime = triple.timestamp; + } + + (openSlot as AlignedSlot)[triple.camera] = triple.localIndex; + }); + if (openSlot !== null) { + slots.push(openSlot); + } + + return { aligned: true, slots }; +} + +/** + * Inverse of a slot lookup: for each camera, maps its own local frame index + * to the global aligned-timeline slot it appears in. Used to translate a + * "seek this camera to its own local frame X" request (e.g. jumping to a + * track's start/end, which is stored in local-frame units) into the correct + * global slot, so every camera stays aligned instead of just the one camera + * jumping independently. Each camera appears in at most one slot per local + * frame (buildAlignedTimeline's first-come-first-served rule), so this is + * always a 1:1 mapping. + */ +export type InverseAlignedIndex = Record>; + +export function buildInverseAlignedIndex(slots: AlignedSlot[]): InverseAlignedIndex { + const inverse: InverseAlignedIndex = {}; + slots.forEach((slot, slotIndex) => { + Object.entries(slot).forEach(([camera, localIndex]) => { + if (localIndex === undefined) { + return; + } + if (!inverse[camera]) { + inverse[camera] = new Map(); + } + inverse[camera].set(localIndex, slotIndex); + }); + }); + return inverse; +} + +/** + * Builds the CSS linear-gradient that Controls.vue paints under the frame + * scrubber to mark aligned-timeline gap slots (slots where at least one + * camera has no frame). Pure so it is unit-testable. + * + * Geometry: a v-slider thumb for value v sits at v/maxFrame of the track + * width, so each gap band is centered on the thumb position for its slot -- + * slot s paints [(s - 0.5) / maxFrame, (s + 0.5) / maxFrame], clamped to + * [0, 100]. Consecutive gap slots are merged into a single band so the CSS + * stop count tracks the number of distinct gaps, not the number of gap + * frames. Bands narrower than `minWidthPct` are widened around their center + * so single-frame gaps stay visible on long timelines. + */ +export function computeGapGradient( + gapSlots: number[], + maxFrame: number, + minWidthPct = 0.25, +): string { + if (!gapSlots.length || maxFrame <= 0) { + return 'none'; + } + const ranges: [number, number][] = []; + let rangeStart = gapSlots[0]; + let rangeEnd = gapSlots[0]; + for (let i = 1; i < gapSlots.length; i += 1) { + if (gapSlots[i] === rangeEnd + 1) { + rangeEnd = gapSlots[i]; + } else { + ranges.push([rangeStart, rangeEnd]); + rangeStart = gapSlots[i]; + rangeEnd = gapSlots[i]; + } + } + ranges.push([rangeStart, rangeEnd]); + const toPct = (frame: number) => (frame / maxFrame) * 100; + const stops: string[] = []; + ranges.forEach(([start, end]) => { + let startPct = Math.max(0, toPct(start - 0.5)); + let endPct = Math.min(100, toPct(end + 0.5)); + if (endPct - startPct < minWidthPct) { + const mid = (startPct + endPct) / 2; + startPct = Math.max(0, mid - minWidthPct / 2); + endPct = Math.min(100, mid + minWidthPct / 2); + } + stops.push(`transparent ${startPct}%`, `#ff5252 ${startPct}%`, `#ff5252 ${endPct}%`, `transparent ${endPct}%`); + }); + return `linear-gradient(to right, ${stops.join(', ')})`; +} + +/** + * Global aligned-timeline slot indices where at least one loaded camera has + * no frame -- i.e. scrubbing to that slot will blank at least one camera's + * pane. Used to render a gap indicator on the timeline scrubber. + */ +export function computeGapSlots(slots: AlignedSlot[]): number[] { + const gaps: number[] = []; + slots.forEach((slot, index) => { + if (Object.values(slot).some((localIndex) => localIndex === undefined)) { + gaps.push(index); + } + }); + return gaps; +} diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index e8cba0276..e1fd84a66 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -124,6 +124,8 @@ interface FrameImage { filename: string; /** Required for large-image (tiled) datasets; used as itemId for getTiles/getTileURL */ id?: string; + /** Best-effort capture timestamp (epoch seconds) parsed from the filename, when available */ + timestamp?: number; } export interface MultiCamImportFolderArgs { diff --git a/client/dive-common/components/ControlsContainer.vue b/client/dive-common/components/ControlsContainer.vue index 1e547658b..75fc1f512 100644 --- a/client/dive-common/components/ControlsContainer.vue +++ b/client/dive-common/components/ControlsContainer.vue @@ -19,6 +19,7 @@ import { useAttributesFilters, useCameraStore, useSelectedCamera, + useTime, } from '../../src/provides'; export default defineComponent({ @@ -126,16 +127,28 @@ export default defineComponent({ handler.trackSelect(trackId, false, modifiers); } + const aggregateController = injectAggregateController(); const { - maxFrame, frame, seek, volume, setVolume, setSpeed, speed, - } = injectAggregateController().value; + maxFrame, volume, setVolume, setSpeed, speed, + } = aggregateController.value; + // The timeline charts (line/event charts) are built from trackStores in + // the selected camera's own local frame space. Under an aligned timeline + // (SEAL feature 5) the aggregate controller's frame/seek operate in + // global slot space, which diverges from local frames -- so the playhead + // uses time.frame (which tracks the selected camera's local frame) and + // chart click-seeks are translated through seekCameraFrame. Both are + // passthroughs when alignment isn't active. + const { frame: localFrame } = useTime(); + function seekToFrame(frame: number) { + aggregateController.value.seekCameraFrame(selectedCamera.value, frame); + } return { currentView, toggleView, maxFrame, multiCam, - frame, - seek, + frame: localFrame, + seek: seekToFrame, volume, setVolume, speed, diff --git a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue index cf2d03738..b6471e36d 100644 --- a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue +++ b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue @@ -138,6 +138,16 @@ export default defineComponent({ /> + + = ref([]); const defaultDisplay = ref('left'); const importAnnotationFilesCheck = ref(false); + // When set, cameras may have differing frame counts; frames are aligned downstream + // by their filename-encoded timestamps instead of by exact positional index. Enables + // importing datasets with dropped frames (e.g. KAMERA). See validateMulticamImageSets. + const inferFrameIndexFromFilename = ref(false); const { error: importError, request: importRequest } = useRequest(); onMounted(async () => { @@ -261,6 +265,7 @@ export function useImportMultiCamDialog( filteredImages.value, Object.keys(globList.value).length, props.dataType, + inferFrameIndexFromFilename.value, ); }); @@ -715,6 +720,7 @@ export function useImportMultiCamDialog( canMoveCamera, moveCamera, importAnnotationFilesCheck, + inferFrameIndexFromFilename, parentFolderName, subfolderLayoutLabel, subfolderOriginalNames, diff --git a/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.spec.ts b/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.spec.ts index a7439b080..22de7649a 100644 --- a/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.spec.ts +++ b/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.spec.ts @@ -26,6 +26,26 @@ describe('validateMulticamImageSets', () => { )).toBe('All cameras should have the same length of 1'); }); + it('allows unequal image counts when inferring frame index from filename', () => { + expect(validateMulticamImageSets( + 'multi', + { left: ['a.png'], right: ['a.png', 'b.png'] }, + 0, + 'image-sequence', + true, + )).toBeNull(); + }); + + it('still requires non-empty cameras when inferring frame index from filename', () => { + expect(validateMulticamImageSets( + 'multi', + { left: [], right: ['a.png'] }, + 0, + 'image-sequence', + true, + )).toBe('Requires filtered Images for left '); + }); + it('rejects overlapping filenames in keyword mode', () => { expect(validateMulticamImageSets( 'keyword', diff --git a/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.ts b/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.ts index e17c9ccd7..e505dbeca 100644 --- a/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.ts +++ b/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.ts @@ -1,6 +1,11 @@ /** * Pure validation for multicam image lists: non-empty filters, equal frame counts, * and mutually exclusive filenames in keyword/glob mode. Video imports skip image checks. + * + * When `inferFrameIndexFromFilename` is set, the equal-frame-count check is skipped: + * datasets with dropped frames (e.g. KAMERA, which encodes a capture timestamp in each + * filename) legitimately have differing per-camera counts and are aligned downstream by + * their filename timestamps rather than by exact positional index. */ export type MulticamImportType = 'multi' | 'keyword' | 'subfolders' | ''; @@ -9,6 +14,7 @@ export function validateMulticamImageSets( filteredImages: Record, globListKeyCount: number, dataType: string, + inferFrameIndexFromFilename = false, ): string | null { if (importType === 'keyword' && globListKeyCount === 0) { return 'Add at least 1 filter pattern'; @@ -30,7 +36,7 @@ export function validateMulticamImageSets( if (length === -1) { length = images.length; } - if (length !== images.length) { + if (!inferFrameIndexFromFilename && length !== images.length) { return `All cameras should have the same length of ${length}`; } if (importType === 'keyword' diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 879f0ad9f..24f459080 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -32,6 +32,7 @@ import { FilterList, } from 'vue-media-annotator/components'; import type { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; +import type { SetTimeFunc } from 'vue-media-annotator/use/useTimeObserver'; import { getResponseError, featureHasSegmentationPolygon } from 'vue-media-annotator/utils'; /* DIVE COMMON */ @@ -62,6 +63,9 @@ import type { import clientSettingsSetup, { clientSettings, isStereoInteractiveModeEnabled } from 'dive-common/store/settings'; import { useApi, FrameImage, DatasetType } from 'dive-common/apispec'; import { orderedMultiCamCameraNames } from 'dive-common/multicamDisplay'; +import { + buildAlignedTimeline, buildInverseAlignedIndex, computeGapSlots, TimelineResult, +} from 'dive-common/alignedTimeline'; import { computeOutputs, computeIsDefault, @@ -166,6 +170,7 @@ export default defineComponent({ aggregateController, onResize, clear: mediaControllerClear, + setAlignedFrameResolver, } = useMediaController(); const { time, updateTime, initialize: initTime } = useTimeObserver(); const imageData = ref({ singleCam: [] } as Record); @@ -189,6 +194,53 @@ export default defineComponent({ // Total tracks total: 0, }); + /** + * Global aligned-timeline resolution (SEAL feature 5, Phase II): only + * engages when every camera in a multicam dataset has a timestamp on + * every frame (see alignedTimeline.ts's canAlign). Otherwise -- including + * always for singleCam datasets -- playback falls back to today's exact + * positional (broadcast-same-index) behavior via useMediaController.ts. + */ + const alignedTimeline = computed(() => { + if (!progress.loaded || multiCamList.value.length < 2) { + return { aligned: false }; + } + // Only consider cameras that are actually part of this dataset: + // imageData could retain keys from before the load (e.g. the initial + // 'singleCam' entry), and a single leftover empty camera would make + // canAlign() disqualify the whole dataset. + const camerasFrames: Record = {}; + multiCamList.value.forEach((camera) => { + camerasFrames[camera] = imageData.value[camera] ?? []; + }); + return buildAlignedTimeline(camerasFrames); + }); + // Serialized shape of the currently installed timeline. The computed + // re-evaluates whenever any camera's imageData array identity changes -- + // including pure display-URL swaps (e.g. the percentile-stretch remap) + // that don't alter timestamps at all. Installing a new resolver re-seeks + // every camera (a visible reload/blank flash), so skip the reinstall + // when the slot structure is unchanged. + let installedTimelineKey: string | null = null; + watch(alignedTimeline, (result) => { + const timelineKey = result.aligned ? JSON.stringify(result.slots) : null; + if (timelineKey === installedTimelineKey) { + return; + } + installedTimelineKey = timelineKey; + if (result.aligned) { + const inverseIndex = buildInverseAlignedIndex(result.slots); + setAlignedFrameResolver({ + slotCount: computed(() => result.slots.length), + frameRate: time.frameRate, + resolveSlot: (f) => result.slots[f] ?? {}, + resolveGlobalSlot: (camera, localFrame) => inverseIndex[camera]?.get(localFrame), + gapSlots: computed(() => computeGapSlots(result.slots)), + }); + } else { + setAlignedFrameResolver(null); + } + }, { immediate: true }); const controlsRef = ref(); const controlsHeight = ref(0); const controlsCollapsed = ref(false); @@ -422,6 +474,24 @@ export default defineComponent({ }, }); + /** + * Every camera pane calls updateTime() from its own seek/play/pause, but + * useTime()'s frame/flick is a single shared value consumed app-wide as + * "the current frame" (track split, keyframe toggling, attribute editing, + * etc.). Under an aligned timeline (SEAL feature 5) cameras can sit on + * different local frames for the same instant, so only the selected + * camera's updates may reach it -- otherwise whichever camera's annotator + * happened to seek last would silently win, regardless of which camera + * the user is actually looking at/editing. + */ + function selectedCameraUpdateTime(camera: string): SetTimeFunc { + return (data) => { + if (selectedCamera.value === camera) { + updateTime(data); + } + }; + } + const { attributesList: attributes, loadAttributes, @@ -982,6 +1052,19 @@ export default defineComponent({ } } selectedCamera.value = camera; + // Immediately resync the shared time observer to the newly selected + // camera's own local frame (see selectedCameraUpdateTime) -- otherwise + // it would keep reporting the previously selected camera's local frame + // until the next seek/play/pause happens to land on this camera. + // During load (loadData calls changeCamera before progress.loaded, so + // no annotator has mounted yet) there is no controller for the camera: + // skip the resync gracefully -- the annotator syncs time on mount. + try { + const newCameraController = aggregateController.value.getController(camera); + updateTime({ frame: newCameraController.frame.value, flick: newCameraController.flick.value }); + } catch { + // No controller registered for this camera (yet); nothing to resync. + } /** * Enters edit mode if no track exists for the camera and forcing edit mode * or if a track exists and are alrady in edit mode we don't set it again @@ -1009,7 +1092,20 @@ export default defineComponent({ if (!track) { return false; } - return track.getFeature(aggregateController.value.frame.value)[0] == null; + // Must use selectedCamera's own local frame, not aggregateController's + // frame: under an aligned timeline (SEAL feature 5) the aggregate frame + // is the global slot index, which diverges from any camera's local + // frame -- and getFeature() is keyed by local frame, same as tracks are + // stored. See LayerManager.vue's identically-named helper. + let cameraFrame: number; + try { + cameraFrame = aggregateController.value.getController(selectedCamera.value).frame.value; + } catch { + // This camera's annotator never mounted (e.g. mid load/reload); fall + // back to the aggregate frame rather than throwing. + cameraFrame = aggregateController.value.frame.value; + } + return track.getFeature(cameraFrame)[0] == null; }; // While editing, the creation cursor is live on any camera still missing // the selected track's geometry at this frame (see LayerManager's @@ -1139,6 +1235,16 @@ export default defineComponent({ frameRate: meta.fps, originalFps: meta.originalFps || null, }); + // Rebuild imageData with exactly this dataset's cameras, dropping the + // initial 'singleCam' placeholder (on multicam datasets) and any + // previous dataset's leftovers. A stale empty entry would make + // alignedTimeline's canAlign() disqualify the dataset. Replacing the + // whole object (rather than adding keys with bracket assignment, + // which is non-reactive for new keys under Vue 2.7) also keeps the + // alignedTimeline computed and the template reactive to these keys. + imageData.value = Object.fromEntries( + multiCamList.value.map((camera) => [camera, [] as FrameImage[]]), + ); for (let i = 0; i < multiCamList.value.length; i += 1) { const camera = multiCamList.value[i]; let cameraId = baseMulticamDatasetId.value; @@ -1452,8 +1558,13 @@ export default defineComponent({ )); function seekToFrame(frame: number) { + // `frame` arrives in the selected camera's own local frame space (track + // begin/end from TrackItem/TrackList/TrackDetailsPanel, keyframe + // navigation, BottomPanel...). Under an aligned timeline (SEAL feature + // 5) the aggregate seek() expects a global slot index, so translate via + // seekCameraFrame -- a passthrough when alignment isn't active. try { - aggregateController.value.seek(frame); + aggregateController.value.seekCameraFrame(selectedCamera.value, frame); } catch { // Ignore seek requests while controllers are initializing. } @@ -1699,7 +1810,7 @@ export default defineComponent({ save, saveThreshold, saveTimeFilter, - updateTime, + selectedCameraUpdateTime, // multicam multiCamList, defaultCamera, @@ -2018,7 +2129,7 @@ export default defineComponent({ v-bind="{ imageData: imageData[camera], videoUrl: videoUrl[camera], - updateTime, + updateTime: selectedCameraUpdateTime(camera), frameRate, originalFps, camera, @@ -2115,7 +2226,7 @@ export default defineComponent({ v-bind="{ imageData: imageData[camera], videoUrl: videoUrl[camera], - updateTime, + updateTime: selectedCameraUpdateTime(camera), frameRate, originalFps, camera, diff --git a/client/dive-common/frameTimestamp.spec.ts b/client/dive-common/frameTimestamp.spec.ts new file mode 100644 index 000000000..dd9457e6e --- /dev/null +++ b/client/dive-common/frameTimestamp.spec.ts @@ -0,0 +1,55 @@ +import type { FrameImage } from 'dive-common/apispec'; +import { attachFrameTimestamps, parseFrameTimestamp } from 'dive-common/frameTimestamp'; + +describe('parseFrameTimestamp', () => { + it('parses a YYYYMMDD_HHMMSS datestamp', () => { + expect(parseFrameTimestamp('left_20230615_143022.png')).toBe(1686839422); + }); + + it('parses a real KAMERA filename with microsecond precision', () => { + // Confirmed convention from real sample data (data/test_data). + expect(parseFrameTimestamp('kamera_calibration_fl02_C_20240407_130757.206341_ir.tif')) + .toBeCloseTo(1712495277.206341, 6); + }); + + it('parses a datestamp with a fractional-second suffix', () => { + expect(parseFrameTimestamp('left_20230615_143022.500.png')).toBe(1686839422.5); + }); + + it('parses a bare epoch-milliseconds filename', () => { + expect(parseFrameTimestamp('img_1719843225123.tif')).toBeCloseTo(1719843225.123, 6); + }); + + it('parses a bare epoch-seconds filename', () => { + expect(parseFrameTimestamp('img_1719843225.tif')).toBe(1719843225); + }); + + it('returns undefined for a plain sequential filename', () => { + expect(parseFrameTimestamp('img_00001.png')).toBeUndefined(); + }); + + it('returns undefined for a short frame-counter filename', () => { + expect(parseFrameTimestamp('frame042.tif')).toBeUndefined(); + }); + + it('returns undefined for an implausible-range digit run', () => { + expect(parseFrameTimestamp('img_0000000001.png')).toBeUndefined(); + }); + + it('is extension-agnostic (same stem, different extension)', () => { + expect(parseFrameTimestamp('left_20230615_143022.tif')) + .toBe(parseFrameTimestamp('left_20230615_143022.png')); + }); +}); + +describe('attachFrameTimestamps', () => { + it('populates timestamp in place from each frame filename', () => { + const frames: FrameImage[] = [ + { url: 'a', filename: 'left_20230615_143022.png' }, + { url: 'b', filename: 'img_00001.png' }, + ]; + attachFrameTimestamps(frames); + expect(frames[0].timestamp).toBe(1686839422); + expect(frames[1].timestamp).toBeUndefined(); + }); +}); diff --git a/client/dive-common/frameTimestamp.ts b/client/dive-common/frameTimestamp.ts new file mode 100644 index 000000000..b5a3bbda0 --- /dev/null +++ b/client/dive-common/frameTimestamp.ts @@ -0,0 +1,104 @@ +import type { FrameImage } from 'dive-common/apispec'; + +/** + * Single source of truth for parsing a frame's capture timestamp out of its + * filename (SEAL feature 5). Used by both platforms -- the desktop backend + * (platform/desktop/backend/native) when enumerating local media, and the + * web-girder API layer (platform/web-girder/api) after fetching media from + * girder -- so there is exactly one implementation to maintain. + */ + +interface FrameTimestampPattern { + name: string; + regex: RegExp; + toSeconds: (match: RegExpMatchArray) => number | undefined; +} + +/* Roughly year 2000 to year 2100, used to reject implausible epoch candidates */ +function isPlausibleEpochSeconds(seconds: number): boolean { + return seconds >= 946684800 && seconds <= 4102444800; +} + +function dateStampToSeconds(match: RegExpMatchArray): number | undefined { + const [, y, mo, d, h, mi, s, frac] = match; + const year = Number(y); + const month = Number(mo); + const day = Number(d); + const hour = Number(h); + const minute = Number(mi); + const second = Number(s); + if (month < 1 || month > 12) return undefined; + if (day < 1 || day > 31) return undefined; + if (hour > 23 || minute > 59 || second > 59) return undefined; + const millis = Date.UTC(year, month - 1, day, hour, minute, second); + const fracSeconds = frac ? Number(`0.${frac}`) : 0; + return millis / 1000 + fracSeconds; +} + +/* + * Parses a frame timestamp from a filename using a small ordered list of + * conventions. The primary convention is KAMERA's, confirmed against sample + * data (see the datestamp entry); the epoch-based patterns are fallbacks for + * other capture systems. Each entry is tried in order against the + * extension-stripped filename stem; the first regex that matches AND passes its + * own plausibility check wins. + */ +const FRAME_TIMESTAMP_PATTERNS: FrameTimestampPattern[] = [ + { + // KAMERA convention: YYYYMMDD[_-]HHMMSS with an optional fractional-second + // suffix, e.g. kamera_calibration_fl02_C_20240407_130757.206341_ir.tif + name: 'datestamp', + regex: /(? { + const seconds = Number(match[1]) / 1000; + return isPlausibleEpochSeconds(seconds) ? seconds : undefined; + }, + }, + { + // bare epoch seconds, e.g. img_1719843225.tif + name: 'epoch-seconds', + regex: /(? { + const seconds = Number(match[1]); + return isPlausibleEpochSeconds(seconds) ? seconds : undefined; + }, + }, +]; + +/** + * Frame capture timestamp, in epoch seconds, parsed from a filename. Returns + * undefined (never throws) when no recognized convention matches; callers must + * treat that as "no timestamp available". + */ +export function parseFrameTimestamp(filename: string): number | undefined { + const stem = filename.replace(/\.[^./\\]+$/, ''); + for (let i = 0; i < FRAME_TIMESTAMP_PATTERNS.length; i += 1) { + const { regex, toSeconds } = FRAME_TIMESTAMP_PATTERNS[i]; + const match = stem.match(regex); + if (match) { + const seconds = toSeconds(match); + if (seconds !== undefined) { + return seconds; + } + } + } + return undefined; +} + +/** + * Populates `timestamp` on each frame in place from its filename. Convenience + * for the web-girder API layer, which receives frames without timestamps and + * parses them client-side (the girder server no longer does this). + */ +export function attachFrameTimestamps(frames: FrameImage[]): void { + frames.forEach((frame) => { + // eslint-disable-next-line no-param-reassign + frame.timestamp = parseFrameTimestamp(frame.filename); + }); +} diff --git a/client/dive-common/use/useModeManager.ts b/client/dive-common/use/useModeManager.ts index ad322c869..45e8813fa 100644 --- a/client/dive-common/use/useModeManager.ts +++ b/client/dive-common/use/useModeManager.ts @@ -129,6 +129,26 @@ export default function useModeManager({ const groupSettings = toRef(clientSettings, 'groupSettings'); const selectedCamera = ref('singleCam'); + /** + * The currently selected camera's own local frame number. Under an aligned + * timeline (SEAL feature 5), aggregateController.value.frame is the global + * slot index, which diverges from any camera's local frame -- but tracks + * are stored/keyed by local frame. Every read/write below that keys track + * storage for selectedCamera must go through this, not aggregateController + * directly. See LayerManager.vue and Viewer.vue's isCreatingNewDetection + * for the identical pattern elsewhere. + */ + function selectedCameraFrame(): number { + try { + return aggregateController.value.getController(selectedCamera.value).frame.value; + } catch { + // No controller registered for the selected camera (e.g. its annotator + // hasn't mounted yet during load/reload); fall back to the aggregate + // frame rather than throwing. + return aggregateController.value.frame.value; + } + } + const linkingState = ref(false); const linkingTrack: Ref = ref(null); const linkingCamera = ref(''); @@ -235,11 +255,11 @@ export default function useModeManager({ const editingDetails = computed(() => { _depend(); if (editingMode.value && selectedTrackId.value !== null) { - const { frame } = aggregateController.value; + const frame = selectedCameraFrame(); try { const track = cameraStore.getPossibleTrack(selectedTrackId.value, selectedCamera.value); if (track) { - const [feature] = track.getFeature(frame.value); + const [feature] = track.getFeature(frame); if (feature) { if (!feature?.bounds?.length) { return 'Creating'; @@ -285,17 +305,28 @@ export default function useModeManager({ } function seekNearest(track: Track) { - // Seek to the nearest point in the track. - const { frame } = aggregateController.value; - if (frame.value < track.begin) { - aggregateController.value.seek(track.begin); - } else if (frame.value > track.end) { - aggregateController.value.seek(track.end); + // Seek to the nearest point in the track. Compares/seeks using + // selectedCamera's own local frame (see selectedCameraFrame) rather than + // aggregateController.frame directly -- under an aligned timeline (SEAL + // feature 5) that's the global slot index, a different number space than + // track.begin/end. seekCameraFrame translates back through the timeline + // so every camera stays aligned; it's a no-op passthrough when alignment + // isn't active. + const frame = selectedCameraFrame(); + if (frame < track.begin) { + aggregateController.value.seekCameraFrame(selectedCamera.value, track.begin); + } else if (frame > track.end) { + aggregateController.value.seekCameraFrame(selectedCamera.value, track.end); } } + // frame is selectedCamera's own local frame (e.g. FilterList.vue's + // "jump to peak frame" derives it from selectedCamera's own trackStore) -- + // route through seekCameraFrame so this still lands correctly under an + // aligned timeline (SEAL feature 5), where local and global frame numbers + // can diverge. function seekFrame(frame: number) { - aggregateController.value.seek(frame); + aggregateController.value.seekCameraFrame(selectedCamera.value, frame); } async function _setLinkingTrack(trackId: TrackId) { @@ -472,7 +503,7 @@ export default function useModeManager({ } // Handles adding a new track with the NewTrack Settings handleEscapeMode(); - const { frame } = aggregateController.value; + const frame = selectedCameraFrame(); let trackType = trackSettings.value.newTrackSettings.type; if (overrideTrackId !== undefined) { const track = cameraStore.getAnyPossibleTrack(overrideTrackId); @@ -487,7 +518,7 @@ export default function useModeManager({ const trackStore = cameraStore.camMap.value.get(selectedCamera.value)?.trackStore; if (trackStore) { const newTrackId = trackStore.add( - frame.value, + frame, trackType, selectedTrackId.value || undefined, overrideTrackId, @@ -793,9 +824,8 @@ export default function useModeManager({ if (track) { recipes.forEach((r) => { if (r.active.value) { - const { frame } = aggregateController.value; r.deletePoint( - frame.value, + selectedCameraFrame(), track, selectedFeatureHandle.value, selectedKey.value, @@ -813,8 +843,7 @@ export default function useModeManager({ if (selectedTrackId.value !== null) { const track = cameraStore.getPossibleTrack(selectedTrackId.value, selectedCamera.value); if (track) { - const { frame } = aggregateController.value; - const frameNum = frame.value; + const frameNum = selectedCameraFrame(); recipes.forEach((r) => { if (r.active.value) { r.delete(frameNum, track, selectedKey.value, annotationModes.editing); @@ -927,8 +956,7 @@ export default function useModeManager({ // When in LineString mode, set the selected key so EditAnnotationLayer // can find the geometry (lines are stored with a recipe key like 'HeadTails'). if (editing) { - const { frame } = aggregateController.value; - const [feature] = track.getFeature(frame.value); + const [feature] = track.getFeature(selectedCameraFrame()); if (feature?.geometry?.features?.length) { if (annotationModes.editing === 'LineString') { const lineFeature = feature.geometry.features.find( @@ -1218,7 +1246,6 @@ export default function useModeManager({ return; } - const { frame } = aggregateController.value; const track = cameraStore.getPossibleTrack(selectedTrackId.value, selectedCamera.value); if (!track) { return; @@ -1252,7 +1279,7 @@ export default function useModeManager({ // Update the track's feature with the preview polygon // Use frame number from the result if provided, otherwise current frame - const targetFrame = result.frameNum ?? frame.value; + const targetFrame = result.frameNum ?? selectedCameraFrame(); const { interpolate } = track.canInterpolate(targetFrame); // Save original feature state before first prediction modifies the track @@ -1509,8 +1536,7 @@ export default function useModeManager({ if (polygonRecipe && 'setAddingPolygon' in polygonRecipe) { const track = cameraStore.getPossibleTrack(selectedTrackId.value, selectedCamera.value); if (track) { - const { frame } = aggregateController.value; - const newKey = track.getNextPolygonKey(frame.value); + const newKey = track.getNextPolygonKey(selectedCameraFrame()); (polygonRecipe as { setAddingPolygon: (key: string) => void }).setAddingPolygon(newKey); } } diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 4598d323c..828acab62 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -294,7 +294,7 @@ beforeEach(() => { fps: 5, originalBasePath: '/home/user/media/projectid1data', originalImageFiles: [ - 'foo.png', + 'foo_20230615_143022.png', 'bar.png', ], } as JsonMeta), @@ -342,14 +342,17 @@ beforeEach(() => { 'meta.json': JSON.stringify({ type: 'multi', multiCam: { + defaultDisplay: 'left', cameras: { left: { type: 'image-sequence', originalBasePath: '/home/user/viamedata/DIVE_Projects/stereoDataset/left', + originalImageFiles: ['left_20230615_143022.png', 'left_00002.png'], }, right: { type: 'image-sequence', originalBasePath: '/home/user/viamedata/DIVE_Projects/stereoDataset/right', + originalImageFiles: ['right_00001.png', 'right_00002.png'], }, }, }, @@ -453,7 +456,21 @@ describe('native.common', () => { const data = await common.loadMetadata(settings, 'projectid1', urlMapper); expect(data.id).toBe('projectid1'); expect(data.imageData.map(({ filename }) => filename)).toEqual([ - 'foo.png', 'bar.png', + 'foo_20230615_143022.png', 'bar.png', + ]); + expect(data.imageData[0].timestamp).toBe(1686839422); + expect(data.imageData[1].timestamp).toBeUndefined(); + }); + + it('loadJsonMetadata parses per-camera frame timestamps for multicam datasets', async () => { + const data = await common.loadMetadata(settings, 'stereoDataset', urlMapper); + expect(data.multiCamMedia).not.toBeNull(); + const { cameras } = data.multiCamMedia!; + expect(cameras.left.imageData.map(({ timestamp }) => timestamp)).toEqual([ + 1686839422, undefined, + ]); + expect(cameras.right.imageData.map(({ timestamp }) => timestamp)).toEqual([ + undefined, undefined, ]); }); diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 5973c13cd..0c890ba03 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -56,6 +56,7 @@ import { import { cleanString, filterByGlob, makeid, strNumericCompare, } from 'platform/desktop/sharedUtils'; +import { parseFrameTimestamp } from 'dive-common/frameTimestamp'; import processTrackAttributes from './attributeProcessor'; import { upgrade } from './migrations'; @@ -404,13 +405,16 @@ async function loadMetadata( imageData = projectMetaData.transcodedImageFiles.map((filename: string) => ({ url: makeMediaUrl(npath.join(projectDirData.basePath, filename)), filename, + timestamp: parseFrameTimestamp(filename), })); } else { imageData = projectMetaData.originalImageFiles.map((pathOrFilename: string) => { const absPath = npath.join(projectMetaData.originalBasePath, pathOrFilename); + const filename = npath.basename(absPath); return { url: makeMediaUrl(absPath), - filename: npath.basename(absPath), + filename, + timestamp: parseFrameTimestamp(filename), }; }); } diff --git a/client/platform/desktop/backend/native/multiCamUtils.ts b/client/platform/desktop/backend/native/multiCamUtils.ts index 00dc796ce..f6ccf7b5d 100644 --- a/client/platform/desktop/backend/native/multiCamUtils.ts +++ b/client/platform/desktop/backend/native/multiCamUtils.ts @@ -10,6 +10,7 @@ import { JsonMeta, Settings } from 'platform/desktop/constants'; // eslint-disable-next-line import/no-cycle import { loadAnnotationFile, loadJsonMetadata, getValidatedProjectDir } from 'platform/desktop/backend/native/common'; import { serialize } from 'platform/desktop/backend/serializers/viame'; +import { parseFrameTimestamp } from 'dive-common/frameTimestamp'; /** * Figure out the destination location @@ -171,6 +172,7 @@ function getMultiCamUrls( imageData = displayFilenames.map((filename: string) => ({ url: makeMediaUrl(npath.join(originalBasePath, filename)), filename, + timestamp: parseFrameTimestamp(filename), })); } else if (value.type === 'video') { let displayFilename = value.originalVideoFile; diff --git a/client/platform/web-girder/api/dataset.service.ts b/client/platform/web-girder/api/dataset.service.ts index 55a44f3a4..3c632850a 100644 --- a/client/platform/web-girder/api/dataset.service.ts +++ b/client/platform/web-girder/api/dataset.service.ts @@ -4,6 +4,7 @@ import { DatasetMetaMutable, FrameImage, SaveAttributeArgs, SaveAttributeTrackFilterArgs, } from 'dive-common/apispec'; import { calibrationFileMarker, jsonCalibrationFileMarker } from 'dive-common/constants'; +import { attachFrameTimestamps } from 'dive-common/frameTimestamp'; import { parentDatasetId } from 'dive-common/compositeDatasetId'; import { isStereoCalibrationFileName } from 'dive-common/stereoParentFolder'; import { GirderMetadataStatic } from 'platform/web-girder/constants'; @@ -21,6 +22,12 @@ async function getDataset(datasetId: string) { if (compositeId) { response.data.id = compositeId; } + // Parse per-frame capture timestamps client-side (single shared implementation + // with desktop; see dive-common/frameTimestamp.ts). The girder server no longer + // does this, so multicam per-camera frames arrive without timestamps. + Object.values(response.data.multiCamMedia?.cameras ?? {}).forEach( + (camera) => attachFrameTimestamps(camera.imageData), + ); return response; } @@ -61,7 +68,10 @@ export interface DatasetSourceMedia { async function getDatasetMedia(datasetId: string) { const { folderId } = await resolveDatasetFolderId(datasetId); - return girderRest.get(`dive_dataset/${folderId}/media`); + const response = await girderRest.get(`dive_dataset/${folderId}/media`); + // Parse per-frame capture timestamps client-side (see getDataset above). + attachFrameTimestamps(response.data.imageData ?? []); + return response; } function clone({ diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 2e6456844..376ef45cc 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -105,6 +105,7 @@ export default defineComponent({ const annotator = aggregateController.value.getController(props.camera); const frameNumberRef = annotator.frame; const flickNumberRef = annotator.flick; + const hasFrameRef = annotator.hasFrame; const rectAnnotationLayer = new RectangleLayer({ annotator, @@ -454,11 +455,34 @@ export default defineComponent({ } /** - * TODO: for some reason, GeoJS requires us to initialize - * by calling the render function twice. This is a bug. - * https://github.com/Kitware/dive/issues/365 + * Disables every annotation/edit layer without touching stored track + * data. Used when this camera has no frame at the current aligned- + * timeline slot (hasFrameRef false) -- frameNumberRef still holds the + * last real local frame, so leaving the layers as-is would draw stale + * boxes over a blank pane. */ - [1, 2].forEach(() => { + function disableAllLayers() { + rectAnnotationLayer.disable(); + overlapLayer.disable(); + polyAnnotationLayer.disable(); + lineLayer.disable(); + pointLayer.disable(); + tailLayer.disable(); + textLayer.disable(); + attributeLayer.disable(); + attributeBoxLayer.disable(); + editAnnotationLayer.disable(); + segmentationPointsLayer.clear(); + hoverOvered.value = []; + uiLayer.setToolTipWidget('customToolTip', false); + } + + /** Re-runs updateLayers with current ref values, or blanks the layers when this camera has no frame at the current aligned-timeline slot. */ + function refreshLayers() { + if (!hasFrameRef.value) { + disableAllLayers(); + return; + } updateLayers( frameNumberRef.value, editingModeRef.value, @@ -469,6 +493,17 @@ export default defineComponent({ selectedKeyRef.value, props.colorBy, ); + } + + watch(hasFrameRef, () => refreshLayers()); + + /** + * TODO: for some reason, GeoJS requires us to initialize + * by calling the render function twice. This is a bug. + * https://github.com/Kitware/dive/issues/365 + */ + [1, 2].forEach(() => { + refreshLayers(); }); /** Shallow watch */ @@ -486,16 +521,7 @@ export default defineComponent({ selectedKeyRef, ], () => { - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }, ); @@ -503,32 +529,14 @@ export default defineComponent({ watch( annotatorPrefs, () => { - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }, { deep: true }, ); watch(attributes, () => { updateAttributes(); - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }); /** Watch for resize events to redraw layers after view mode changes */ @@ -655,16 +663,7 @@ export default defineComponent({ // This is especially important when already editing the same track // since annotation-right-clicked won't be emitted in that case window.setTimeout(() => { - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }, 0); }); polyAnnotationLayer.bus.$on('annotation-ctrl-clicked', Clicked); @@ -681,16 +680,7 @@ export default defineComponent({ // Force layer update to load the newly selected polygon // Use nextTick to ensure the selectedKey ref has been updated window.setTimeout(() => { - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }, 0); }); // Handle right-click outside polygons to finalize/cancel creation @@ -700,16 +690,7 @@ export default defineComponent({ handler.cancelCreation(); handler.selectFeatureHandle(-1, ''); window.setTimeout(() => { - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }, 0); } }); @@ -780,16 +761,7 @@ export default defineComponent({ // Suppress the select that rides along with this same finalizing click. justFinalizedCreation = true; window.setTimeout(() => { justFinalizedCreation = false; }, 0); - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); } }); editAnnotationLayer.bus.$on( @@ -834,16 +806,7 @@ export default defineComponent({ handler.selectFeatureHandle(-1, polygonKey); // Force layer update to load the newly selected polygon window.setTimeout(() => { - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }, 0); } }); diff --git a/client/src/components/annotators/ImageAnnotator.vue b/client/src/components/annotators/ImageAnnotator.vue index 08b57f09b..2d8f414a9 100644 --- a/client/src/components/annotators/ImageAnnotator.vue +++ b/client/src/components/annotators/ImageAnnotator.vue @@ -76,6 +76,7 @@ export default defineComponent({ container, initializeViewer, mediaController, + externallyDriven, } = cameraInitializer(props.camera, { // allow hoisting for these functions to pass a reference before defining them. // eslint-disable-next-line @typescript-eslint/no-use-before-define @@ -236,10 +237,27 @@ export default defineComponent({ cacheNewRange(min, max); } } - async function seek(f: number) { + async function seek(f: number | undefined) { if (!data.ready) { return; } + if (f === undefined) { + // No frame for this camera at the current aligned-timeline slot: blank + // the pane. Deliberately leaves data.frame/data.filename untouched -- + // those are read elsewhere (e.g. annotation-overlay lookups) and this + // phase doesn't touch annotation storage. + data.hasFrame = false; + if (local.quadFeature !== undefined) { + local.quadFeature.layer().node().css('visibility', 'hidden'); + } + return; + } + if (!data.hasFrame) { + data.hasFrame = true; + if (local.quadFeature !== undefined) { + local.quadFeature.layer().node().css('visibility', ''); + } + } let newFrame = f; if (f < 0) newFrame = 0; if (f > data.maxFrame) newFrame = data.maxFrame; @@ -317,7 +335,12 @@ export default defineComponent({ async function play() { try { data.playing = true; - syncWithVideo(data.frame + 1); + // When a global aligned timeline is driving playback, the aggregate + // controller's own centralized tick calls seek() directly -- this + // camera must not also free-run its own loop. + if (!externallyDriven.value) { + syncWithVideo(data.frame + 1); + } props.updateTime(data); } catch (ex) { console.error(ex); @@ -538,6 +561,12 @@ export default defineComponent({ Loading +
+ No frame at this instant +
diff --git a/client/src/components/annotators/VideoAnnotator.vue b/client/src/components/annotators/VideoAnnotator.vue index 0517f6ce4..aea42d6e3 100644 --- a/client/src/components/annotators/VideoAnnotator.vue +++ b/client/src/components/annotators/VideoAnnotator.vue @@ -121,6 +121,7 @@ export default defineComponent({ container, initializeViewer, mediaController, + externallyDriven, } = cameraInitializer(props.camera, { // allow hoisting for these functions. // eslint-disable-next-line @typescript-eslint/no-use-before-define @@ -148,7 +149,26 @@ export default defineComponent({ video.pause(); } }); - async function seek(frame: number) { + async function seek(frame: number | undefined) { + if (frame === undefined) { + // No frame for this camera at the current aligned-timeline slot: blank + // the pane. Leaves data.frame untouched -- it's read elsewhere (e.g. + // annotation-overlay lookups) and this phase doesn't touch annotation + // storage. In practice unreachable today since a video-backed camera + // (empty imageData) always disqualifies the whole dataset from aligned + // mode (see alignedTimeline.ts's canAlign) -- kept for symmetry/safety. + data.hasFrame = false; + if (quadFeatureLayer !== undefined) { + quadFeatureLayer.node().css('visibility', 'hidden'); + } + return; + } + if (!data.hasFrame) { + data.hasFrame = true; + if (quadFeatureLayer !== undefined) { + quadFeatureLayer.node().css('visibility', ''); + } + } /** Only perform seek for whole frame numbers */ const requestedFrame = Math.round(frame); /** Different seek approaches based on known information */ @@ -191,7 +211,12 @@ export default defineComponent({ await video.play(); data.playing = true; props.updateTime(data); - syncWithVideo(); + // When a global aligned timeline is driving playback, the aggregate + // controller's own centralized tick calls seek() directly -- this + // camera must not also free-run its own loop. + if (!externallyDriven.value) { + syncWithVideo(); + } } catch (ex) { console.error(ex); } @@ -367,6 +392,12 @@ export default defineComponent({ @mouseleave="cursorHandler.handleMouseLeave" @mouseover="cursorHandler.handleMouseEnter" /> +
+ No frame at this instant +
diff --git a/client/src/components/annotators/annotator.scss b/client/src/components/annotators/annotator.scss index 1abfa8229..363717153 100644 --- a/client/src/components/annotators/annotator.scss +++ b/client/src/components/annotators/annotator.scss @@ -37,6 +37,19 @@ .geojs-map.annotation-input { cursor: inherit; } + .no-frame-overlay { + z-index: 20; + margin: 0; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + padding: 8px 16px; + background: rgba(0, 0, 0, 0.6); + color: white; + border-radius: 4px; + pointer-events: none; + } } .selected-camera{ diff --git a/client/src/components/annotators/mediaControllerType.ts b/client/src/components/annotators/mediaControllerType.ts index 27a955ef1..ece6d1d12 100644 --- a/client/src/components/annotators/mediaControllerType.ts +++ b/client/src/components/annotators/mediaControllerType.ts @@ -1,5 +1,33 @@ import type { Ref } from 'vue'; +/** + * Supplied by Viewer.vue when every camera in a multicam dataset has a + * timestamp on every frame (see dive-common/alignedTimeline.ts). Translates + * a global aligned-timeline slot into each camera's own local frame index, + * or undefined for a camera with no frame at that slot. Absent (null) + * whenever alignment isn't possible/applicable -- including always for + * single-camera datasets -- in which case playback falls back to today's + * exact positional (broadcast-same-index) behavior. + */ +export interface AlignedFrameResolver { + slotCount: Readonly>; + frameRate: Readonly>; + resolveSlot: (globalFrame: number) => Record; + /** + * Inverse of resolveSlot (see dive-common/alignedTimeline.ts's + * buildInverseAlignedIndex): given a camera and its own local frame, + * returns the global aligned-timeline slot it appears in, or undefined if + * that local frame isn't part of any slot. + */ + resolveGlobalSlot: (camera: string, localFrame: number) => number | undefined; + /** + * Global slot indices where at least one camera has no frame (see + * dive-common/alignedTimeline.ts's computeGapSlots) -- used to render a + * gap indicator on the timeline scrubber. + */ + gapSlots: Readonly>; +} + /** * AggregateMediaController provides an interface for time and a few * other properties of all cameras in the annotator window. @@ -17,6 +45,12 @@ export interface AggregateMediaController { cameraSync: Readonly>; /** Incremented when the viewer is resized, used to trigger layer redraws */ resizeTrigger: Readonly>; + /** + * Global aligned-timeline slot indices with at least one camera missing a + * frame (see AlignedFrameResolver's gapSlots); empty whenever alignment + * isn't active. + */ + alignedGapSlots: Readonly>; pause: () => void; play: () => void; @@ -28,6 +62,15 @@ export interface AggregateMediaController { setSpeed: (speed: number) => void; getController: (cameraName: string) => MediaController; toggleSynchronizeCameras: (sync: boolean) => void; + /** + * Seeks so that `camera` lands on its own local frame `localFrame` (e.g. + * jumping to a track's stored begin/end, which is in local-frame units). + * Under an aligned timeline (see AlignedFrameResolver) this translates + * through the global slot so every camera stays aligned; otherwise it's + * equivalent to seek(localFrame), since local and global frame numbers are + * identical under today's positional broadcast. + */ + seekCameraFrame: (camera: string, localFrame: number) => void; } /** @@ -43,6 +86,18 @@ export interface MediaController extends AggregateMediaController { geoViewerRef: Readonly>; /** @deprecated may be removed in a future release */ syncedFrame: Readonly>; + /** + * False when an aligned-timeline slot has no frame for this camera (see + * AlignedFrameResolver) -- the pane is blank and LayerManager should not + * draw annotations for the stale `frame` value left over from before. + */ + hasFrame: Readonly>; + /** + * Per-camera seek accepts undefined when this camera has no frame at the + * current aligned-timeline slot (see AlignedFrameResolver) -- the camera + * should blank its pane rather than draw anything for that slot. + */ + seek: (frame: number | undefined) => void; centerOn(coords: { x: number; y: number; z: number }): void; transition(coords: { x: number; y:number}, duration: number, zoom?: number): void; diff --git a/client/src/components/annotators/useMediaController.spec.ts b/client/src/components/annotators/useMediaController.spec.ts new file mode 100644 index 000000000..dd6d7f3a4 --- /dev/null +++ b/client/src/components/annotators/useMediaController.spec.ts @@ -0,0 +1,302 @@ +// @vitest-environment jsdom +import { defineComponent, ref } from 'vue'; +// eslint-disable-next-line import/no-extraneous-dependencies -- @vue/test-utils is only used in tests +import { mount } from '@vue/test-utils'; +import { useMediaController } from './useMediaController'; +import type { AlignedFrameResolver } from './mediaControllerType'; + +function noop() { /* unused setVolume/setSpeed stub */ } + +/** + * useMediaController() calls Vue's provide(), so it needs a component setup() + * context. initialize() itself (unlike initializeViewer()) never touches + * GeoJS/DOM, so camera controllers can be registered directly here with + * mocked seek/play/pause -- no need to mount real annotator components. + */ +function mountMediaController() { + const seekA = vi.fn(); + const playA = vi.fn(); + const pauseA = vi.fn(); + const seekB = vi.fn(); + const playB = vi.fn(); + const pauseB = vi.fn(); + + let composable!: ReturnType; + + const Host = defineComponent({ + setup() { + composable = useMediaController(); + composable.initialize('A', { + seek: seekA, play: playA, pause: pauseA, setVolume: noop, setSpeed: noop, + }); + composable.initialize('B', { + seek: seekB, play: playB, pause: pauseB, setVolume: noop, setSpeed: noop, + }); + return {}; + }, + template: '
', + }); + + const wrapper = mount(Host); + return { + wrapper, + composable, + mocks: { + seekA, playA, pauseA, seekB, playB, pauseB, + }, + }; +} + +/** + * Camera A is missing a frame at slot 1; camera B has one at every slot. A + * never reports local frame 1 at all in this mock (slot 1 has no A entry, + * and slot 2's A entry is local frame 2, not a shifted-down 1) -- so A's + * local frame space has a hole at 1, identical to the global slot space. + */ +function makeGappedResolver(slotCount = 3, frameRate = 2): AlignedFrameResolver { + return { + slotCount: ref(slotCount), + frameRate: ref(frameRate), + resolveSlot: (f: number) => ({ + A: f === 1 ? undefined : f, + B: f, + }), + resolveGlobalSlot: (camera: string, localFrame: number) => { + if (camera === 'B') return localFrame; + if (camera === 'A') return (localFrame === 0 || localFrame === 2) ? localFrame : undefined; + return undefined; + }, + gapSlots: ref([1]), + }; +} + +/** + * Camera A drops the frame at global slot 1, so unlike makeGappedResolver's + * A, its OWN frame array is contiguous: local frame 0 is slot 0, and local + * frame 1 (its very next captured frame) is slot 2 -- a realistic shift + * between local and global numbering, exercising the non-trivial direction + * of resolveGlobalSlot. + */ +function makeShiftedResolver(): AlignedFrameResolver { + // A's own frame array only has two entries: slot 0's frame, then slot 2's. + const aSlotForGlobalFrame: Record = { 0: 0, 1: undefined, 2: 1 }; + const aGlobalSlotForLocalFrame: Record = { 0: 0, 1: 2 }; + return { + slotCount: ref(3), + frameRate: ref(2), + resolveSlot: (f: number) => ({ + A: aSlotForGlobalFrame[f], + B: f, + }), + resolveGlobalSlot: (camera: string, localFrame: number) => { + if (camera === 'B') return localFrame; + if (camera === 'A') return aGlobalSlotForLocalFrame[localFrame]; + return undefined; + }, + gapSlots: ref([1]), + }; +} + +describe('useMediaController', () => { + it('without a resolver, seek broadcasts the identical frame to every camera', () => { + const { composable, mocks } = mountMediaController(); + composable.aggregateController.value.seek(5); + expect(mocks.seekA).toHaveBeenCalledWith(5); + expect(mocks.seekB).toHaveBeenCalledWith(5); + }); + + it('without a resolver, maxFrame/frame alias the first-initialized camera (today\'s behavior)', () => { + const { composable } = mountMediaController(); + const keyA = Object.keys(composable.state) + .find((k) => composable.state[k].cameraName === 'A'); + expect(keyA).toBeDefined(); + composable.state[keyA as string].maxFrame = 10; + composable.state[keyA as string].frame = 4; + expect(composable.aggregateController.value.maxFrame.value).toBe(10); + expect(composable.aggregateController.value.frame.value).toBe(4); + }); + + it('with a resolver, seek(n) calls each camera with the resolved per-camera frame, including undefined', () => { + const { composable, mocks } = mountMediaController(); + composable.setAlignedFrameResolver(makeGappedResolver()); + composable.aggregateController.value.seek(1); + expect(mocks.seekA).toHaveBeenCalledWith(undefined); + expect(mocks.seekB).toHaveBeenCalledWith(1); + expect(composable.aggregateController.value.frame.value).toBe(1); + expect(composable.aggregateController.value.maxFrame.value).toBe(2); + }); + + it('with a resolver, nextFrame/prevFrame advance the global slot pointer, not each camera\'s own local frame', () => { + const { composable, mocks } = mountMediaController(); + composable.setAlignedFrameResolver(makeGappedResolver()); + composable.aggregateController.value.seek(0); + mocks.seekA.mockClear(); + mocks.seekB.mockClear(); + + composable.aggregateController.value.nextFrame(); + expect(composable.aggregateController.value.frame.value).toBe(1); + expect(mocks.seekA).toHaveBeenLastCalledWith(undefined); + expect(mocks.seekB).toHaveBeenLastCalledWith(1); + + composable.aggregateController.value.nextFrame(); + expect(composable.aggregateController.value.frame.value).toBe(2); + expect(mocks.seekA).toHaveBeenLastCalledWith(2); + expect(mocks.seekB).toHaveBeenLastCalledWith(2); + + composable.aggregateController.value.prevFrame(); + expect(composable.aggregateController.value.frame.value).toBe(1); + expect(mocks.seekA).toHaveBeenLastCalledWith(undefined); + expect(mocks.seekB).toHaveBeenLastCalledWith(1); + }); + + it('play() drives one centralized tick at frameRate and stops at slotCount - 1', () => { + vi.useFakeTimers(); + try { + const { composable, mocks } = mountMediaController(); + composable.setAlignedFrameResolver(makeGappedResolver(3, 2)); // 2fps => 500ms/tick + composable.aggregateController.value.seek(0); + mocks.playA.mockClear(); + mocks.playB.mockClear(); + + composable.aggregateController.value.play(); + expect(mocks.playA).toHaveBeenCalledTimes(1); + expect(mocks.playB).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(500); + expect(composable.aggregateController.value.frame.value).toBe(1); + + vi.advanceTimersByTime(500); + expect(composable.aggregateController.value.frame.value).toBe(2); + + mocks.pauseA.mockClear(); + mocks.pauseB.mockClear(); + vi.advanceTimersByTime(500); + // reached slotCount - 1: auto-pauses and stops advancing + expect(mocks.pauseA).toHaveBeenCalledTimes(1); + expect(mocks.pauseB).toHaveBeenCalledTimes(1); + expect(composable.aggregateController.value.frame.value).toBe(2); + + vi.advanceTimersByTime(1000); + expect(composable.aggregateController.value.frame.value).toBe(2); + } finally { + vi.useRealTimers(); + } + }); + + it('pause() stops the centralized tick', () => { + vi.useFakeTimers(); + try { + const { composable, mocks } = mountMediaController(); + composable.setAlignedFrameResolver(makeGappedResolver(10, 2)); + composable.aggregateController.value.seek(0); + composable.aggregateController.value.play(); + vi.advanceTimersByTime(500); + expect(composable.aggregateController.value.frame.value).toBe(1); + + composable.aggregateController.value.pause(); + expect(mocks.pauseA).toHaveBeenCalled(); + expect(mocks.pauseB).toHaveBeenCalled(); + + vi.advanceTimersByTime(2000); + expect(composable.aggregateController.value.frame.value).toBe(1); + } finally { + vi.useRealTimers(); + } + }); + + it('without a resolver, seekCameraFrame(camera, frame) is equivalent to seek(frame)', () => { + const { composable, mocks } = mountMediaController(); + composable.aggregateController.value.seekCameraFrame('A', 5); + expect(mocks.seekA).toHaveBeenCalledWith(5); + expect(mocks.seekB).toHaveBeenCalledWith(5); + }); + + it('with a resolver, seekCameraFrame(camera, localFrame) translates a shifted local frame to the correct global slot', () => { + const { composable, mocks } = mountMediaController(); + composable.setAlignedFrameResolver(makeShiftedResolver()); + + // A's own local frame 1 is really global slot 2 (A dropped the frame at slot 1). + composable.aggregateController.value.seekCameraFrame('A', 1); + expect(composable.aggregateController.value.frame.value).toBe(2); + expect(mocks.seekA).toHaveBeenLastCalledWith(1); + expect(mocks.seekB).toHaveBeenLastCalledWith(2); + }); + + it('with a resolver, seekCameraFrame falls back to seeking only that camera when its local frame maps to no slot', () => { + const { composable, mocks } = mountMediaController(); + composable.setAlignedFrameResolver(makeGappedResolver()); + mocks.seekB.mockClear(); + + // A never reports local frame 1 in this resolver -- no slot maps to it. + composable.aggregateController.value.seekCameraFrame('A', 1); + expect(mocks.seekA).toHaveBeenLastCalledWith(1); + expect(mocks.seekB).not.toHaveBeenCalled(); + }); + + it('exposes alignedGapSlots from the resolver, empty when unaligned', () => { + const { composable } = mountMediaController(); + expect(composable.aggregateController.value.alignedGapSlots.value).toEqual([]); + + composable.setAlignedFrameResolver(makeGappedResolver()); + expect(composable.aggregateController.value.alignedGapSlots.value).toEqual([1]); + + composable.setAlignedFrameResolver(null); + expect(composable.aggregateController.value.alignedGapSlots.value).toEqual([]); + }); + + it('installing a resolver immediately performs an aligned seek to slot 0', () => { + const { composable, mocks } = mountMediaController(); + // Gap at slot 0: A has no frame there and must blank right away rather + // than continuing to show its local frame 0 until the first user seek. + const resolver: AlignedFrameResolver = { + slotCount: ref(3), + frameRate: ref(2), + resolveSlot: (f: number) => ({ + A: f === 0 ? undefined : f - 1, + B: f, + }), + resolveGlobalSlot: (camera: string, localFrame: number) => ( + camera === 'B' ? localFrame : localFrame + 1), + gapSlots: ref([0]), + }; + composable.setAlignedFrameResolver(resolver); + expect(mocks.seekA).toHaveBeenCalledWith(undefined); + expect(mocks.seekB).toHaveBeenCalledWith(0); + expect(composable.aggregateController.value.frame.value).toBe(0); + }); + + it('re-applies the current aligned slot when a camera registers after the resolver is installed', async () => { + const { composable, wrapper } = mountMediaController(); + const resolver: AlignedFrameResolver = { + slotCount: ref(3), + frameRate: ref(2), + resolveSlot: (f: number) => ({ A: f, B: f, C: f + 10 }), + resolveGlobalSlot: (_camera: string, localFrame: number) => localFrame, + gapSlots: ref([]), + }; + composable.setAlignedFrameResolver(resolver); + composable.aggregateController.value.seek(1); + + // A late-mounting annotator (e.g. it registered after Viewer installed + // the resolver) self-seeks to its own local frame 0 during init; the + // roster watcher must re-seek it onto the current slot afterwards. + const seekC = vi.fn(); + composable.initialize('C', { + seek: seekC, play: noop, pause: noop, setVolume: noop, setSpeed: noop, + }); + await wrapper.vm.$nextTick(); + expect(seekC).toHaveBeenLastCalledWith(11); + expect(composable.aggregateController.value.frame.value).toBe(1); + }); + + it('setAlignedFrameResolver(null) resets the aligned frame and hands control back to camera aliasing', () => { + const { composable } = mountMediaController(); + composable.setAlignedFrameResolver(makeGappedResolver()); + composable.aggregateController.value.seek(2); + expect(composable.aggregateController.value.frame.value).toBe(2); + + composable.setAlignedFrameResolver(null); + // Falls back to the first-initialized camera's own (untouched, still 0) local frame ref. + expect(composable.aggregateController.value.frame.value).toBe(0); + }); +}); diff --git a/client/src/components/annotators/useMediaController.ts b/client/src/components/annotators/useMediaController.ts index 3349dba8c..5b3317277 100644 --- a/client/src/components/annotators/useMediaController.ts +++ b/client/src/components/annotators/useMediaController.ts @@ -4,12 +4,12 @@ import geo, { GeoEvent } from 'geojs'; import * as d3 from 'd3'; import Vue, { - ref, reactive, provide, toRef, Ref, UnwrapRef, computed, + ref, shallowRef, reactive, provide, toRef, Ref, UnwrapRef, computed, watch, } from 'vue'; import { map, over } from 'lodash'; import { use } from '../../provides'; -import type { AggregateMediaController, MediaController } from './mediaControllerType'; +import type { AggregateMediaController, AlignedFrameResolver, MediaController } from './mediaControllerType'; const AggregateControllerSymbol = Symbol('aggregate-controller'); const CameraInitializerSymbol = Symbol('camera-initializer'); @@ -29,6 +29,8 @@ interface MediaControllerReactiveData { speed: number; maxFrame: number; syncedFrame: number; + /** False when an aligned-timeline slot has no frame for this camera; pane should blank. */ + hasFrame: boolean; cursor: string; imageCursor: string; imageCursorEditing: boolean; @@ -54,12 +56,18 @@ interface CameraInitializerReturn { initializeViewer: (width: number, height: number, tileWidth?: number, tileHeight?: number, isMap?: boolean, geoSpatial?: boolean) => void; mediaController: MediaController; + /** + * True whenever a global aligned timeline is driving playback (see + * AlignedFrameResolver) -- callers should skip starting their own internal + * frame-advance loop and rely on the aggregate controller's seek instead. + */ + externallyDriven: Readonly>; } type CameraInitializerFunc = (cameraName: string, { seek, play, pause, setVolume, setSpeed, }: { - seek(frame: number): void; + seek(frame: number | undefined): void; play(): void; pause(): void; setVolume(level: number): void; @@ -85,6 +93,13 @@ export function useMediaController() { let cameraControllerSymbols: Record = {}; const synchronizeCameras: Ref = ref(false); const resizeTrigger: Ref = ref(0); + // shallowRef: an AlignedFrameResolver carries nested Refs (slotCount, frameRate) + // that must NOT be deep-reactive-converted/auto-unwrapped by a plain ref(). + const alignedFrameResolver: Ref = shallowRef(null); + const alignedCurrentFrame: Ref = ref(0); + const externallyDriven = computed(() => alignedFrameResolver.value !== null); + const alignedGapSlots = computed(() => alignedFrameResolver.value?.gapSlots.value ?? []); + let alignedPlaybackTimer: ReturnType | undefined; const emptyControllerFrame = ref(0); const emptyControllerMaxFrame = ref(0); const emptyControllerPlaying = ref(false); @@ -112,7 +127,51 @@ export function useMediaController() { toggleSynchronizeCameras, cameraSync: synchronizeCameras, resizeTrigger, + alignedGapSlots, + seekCameraFrame: aggregateSeekCameraFrame, }; + function stopAlignedPlaybackTimer() { + if (alignedPlaybackTimer !== undefined) { + clearTimeout(alignedPlaybackTimer); + alignedPlaybackTimer = undefined; + } + } + + /** + * Set (or clear, with null) the resolver that translates a global + * aligned-timeline slot into each camera's own local frame index. Called + * by Viewer.vue whenever its computed aligned timeline changes. + */ + function setAlignedFrameResolver(resolver: AlignedFrameResolver | null) { + stopAlignedPlaybackTimer(); + alignedFrameResolver.value = resolver; + if (resolver) { + // Immediately perform an aligned seek to slot 0 so that any camera with + // no frame at that slot blanks right away, rather than continuing to + // show its own local frame 0 until the first user-driven seek. + alignedSeek(resolver, 0); + } else { + alignedCurrentFrame.value = 0; + } + } + + /** + * Re-apply the current aligned slot whenever the camera roster changes + * while a resolver is active. Annotators self-seek to their own local + * frame 0 during init (see ImageAnnotator's init()), so a camera that + * mounts after the resolver was installed would wrongly display its local + * frame 0 even when the current slot has no frame for it. flush: 'post' + * ensures this runs after the mounting annotator's own init-time seek. + * (Watch the length via a getter: under Vue 2.7 a shallow watch on the ref + * itself does not fire for in-place array mutation like push/splice.) + */ + watch(() => cameras.value.length, () => { + const resolver = alignedFrameResolver.value; + if (resolver && cameras.value.length > 0) { + alignedSeek(resolver, alignedCurrentFrame.value); + } + }, { flush: 'post' }); + function clear() { geoViewers = {}; containers = {}; @@ -121,6 +180,7 @@ export function useMediaController() { subControllers = []; state = {}; cameraControllerSymbols = {}; + setAlignedFrameResolver(null); } function getController(camera?: string) { @@ -211,7 +271,7 @@ export function useMediaController() { function initialize(cameraName: string, { seek: _seek, play: _play, pause: _pause, setVolume: _setVolume, setSpeed: _setSpeed, }: { - seek(frame: number): void; + seek(frame: number | undefined): void; play(): void; pause(): void; setVolume(level: number): void; @@ -255,6 +315,7 @@ export function useMediaController() { speed: 1.0, maxFrame: 0, syncedFrame: 0, + hasFrame: true, cursor: 'default', imageCursor: '', imageCursorEditing: false, @@ -472,6 +533,7 @@ export function useMediaController() { maxFrame: toRef(state[camera], 'maxFrame'), speed: toRef(state[camera], 'speed'), syncedFrame: toRef(state[camera], 'syncedFrame'), + hasFrame: toRef(state[camera], 'hasFrame'), prevFrame, nextFrame, play: _play, @@ -502,7 +564,96 @@ export function useMediaController() { cursorHandler, initializeViewer, mediaController, + externallyDriven, + }; + } + + /** Seeks every camera to its own local frame for the given global aligned-timeline slot. */ + function alignedSeek(resolver: AlignedFrameResolver, rawFrame: number) { + const maxFrame = Math.max(0, resolver.slotCount.value - 1); + const clamped = Math.min(Math.max(rawFrame, 0), maxFrame); + alignedCurrentFrame.value = clamped; + const perCamera = resolver.resolveSlot(clamped); + subControllers.forEach((mc) => { + mc.seek(perCamera[mc.cameraName.value]); + }); + } + + function aggregateSeek(frame: number) { + const resolver = alignedFrameResolver.value; + if (resolver) { + alignedSeek(resolver, frame); + } else { + subControllers.forEach((mc) => mc.seek(frame)); + } + } + + /** + * Seeks so that `camera` lands on its own local frame `localFrame`. Without + * alignment, local and global frame numbers are identical (today's + * positional broadcast), so this is just aggregateSeek. Under an aligned + * timeline, translates through the global slot via resolveGlobalSlot so + * every camera stays aligned; falls back to seeking only `camera` itself + * if that local frame isn't part of any slot (shouldn't normally happen). + */ + function aggregateSeekCameraFrame(camera: string, localFrame: number) { + const resolver = alignedFrameResolver.value; + if (!resolver) { + aggregateSeek(localFrame); + return; + } + const slot = resolver.resolveGlobalSlot(camera, localFrame); + if (slot !== undefined) { + alignedSeek(resolver, slot); + } else { + getController(camera).seek(localFrame); + } + } + + function aggregateNextFrame() { + const resolver = alignedFrameResolver.value; + if (resolver) { + alignedSeek(resolver, alignedCurrentFrame.value + 1); + } else { + subControllers.forEach((mc) => mc.nextFrame()); + } + } + + function aggregatePrevFrame() { + const resolver = alignedFrameResolver.value; + if (resolver) { + alignedSeek(resolver, alignedCurrentFrame.value - 1); + } else { + subControllers.forEach((mc) => mc.prevFrame()); + } + } + + function aggregatePause() { + subControllers.forEach((mc) => mc.pause()); + stopAlignedPlaybackTimer(); + } + + function aggregatePlay() { + // Each camera still flips its own `playing` UI state; when a resolver is + // set, ImageAnnotator/VideoAnnotator skip starting their own internal + // frame-advance loop (see externallyDriven) and this centralized tick + // drives seeks instead. + subControllers.forEach((mc) => mc.play()); + const resolver = alignedFrameResolver.value; + if (!resolver) { + return; + } + stopAlignedPlaybackTimer(); + const tick = () => { + const maxFrame = Math.max(0, resolver.slotCount.value - 1); + if (alignedCurrentFrame.value >= maxFrame) { + aggregatePause(); + return; + } + alignedSeek(resolver, alignedCurrentFrame.value + 1); + alignedPlaybackTimer = setTimeout(tick, 1000 / Math.max(1, resolver.frameRate.value)); }; + alignedPlaybackTimer = setTimeout(tick, 1000 / Math.max(1, resolver.frameRate.value)); } const aggregateController: Ref = computed(() => { @@ -512,19 +663,22 @@ export function useMediaController() { return emptyAggregateController; } const defaultController = getController(); + const resolver = alignedFrameResolver.value; return { cameras: computed(() => cameras.value.map((v) => String(v))), - maxFrame: defaultController.maxFrame, - frame: defaultController.frame, - seek: over(map(subControllers, 'seek')), - nextFrame: over(map(subControllers, 'nextFrame')), - prevFrame: over(map(subControllers, 'prevFrame')), + maxFrame: resolver + ? computed(() => Math.max(0, resolver.slotCount.value - 1)) + : defaultController.maxFrame, + frame: resolver ? alignedCurrentFrame : defaultController.frame, + seek: aggregateSeek, + nextFrame: aggregateNextFrame, + prevFrame: aggregatePrevFrame, volume: defaultController.volume, setVolume: over(map(subControllers, 'setVolume')), speed: defaultController.speed, setSpeed: over(map(subControllers, 'setSpeed')), - pause: over(map(subControllers, 'pause')), - play: over(map(subControllers, 'play')), + pause: aggregatePause, + play: aggregatePlay, playing: defaultController.playing, resetZoom: over(map(subControllers, 'resetZoom')), currentTime: defaultController.currentTime, @@ -532,6 +686,8 @@ export function useMediaController() { toggleSynchronizeCameras, cameraSync: synchronizeCameras, resizeTrigger, + alignedGapSlots, + seekCameraFrame: aggregateSeekCameraFrame, }; }); @@ -547,5 +703,6 @@ export function useMediaController() { initialize, onResize, clear, + setAlignedFrameResolver, }; } diff --git a/client/src/components/controls/Controls.vue b/client/src/components/controls/Controls.vue index 3bf2239a2..c580c5bfc 100644 --- a/client/src/components/controls/Controls.vue +++ b/client/src/components/controls/Controls.vue @@ -7,6 +7,7 @@ import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import context from 'dive-common/store/context'; import { clientSettings } from 'dive-common/store/settings'; import { DatasetType, useApi } from 'dive-common/apispec'; +import { computeGapGradient } from 'dive-common/alignedTimeline'; import { frameToTimestamp } from 'vue-media-annotator/utils'; import { injectAggregateController } from '../annotators/useMediaController'; import { useTime, useTrackFilters, useDatasetId } from '../../provides'; @@ -60,6 +61,21 @@ export default defineComponent({ } data.frame = value; } + + /** + * A CSS gradient overlay marking timeline slots where at least one camera + * has no frame (SEAL feature 5's aligned timeline, see alignedTimeline.ts). + * Uses a gradient rather than one element per gap so this stays cheap + * regardless of how many gaps there are. Each band is centered on the + * v-slider thumb position for its slot (see computeGapGradient); this is + * a lightweight visual approximation drawn under the v-slider, not + * pixel-exact with its clickable thumb track. + */ + const alignedGapGradient = computed(() => computeGapGradient( + mediaController.alignedGapSlots.value, + mediaController.maxFrame.value, + )); + const alignedGapCount = computed(() => mediaController.alignedGapSlots.value.length); function togglePlay(_: HTMLElement, keyEvent: KeyboardEvent) { // Prevent scroll from spacebar and other default effects. keyEvent.preventDefault(); @@ -215,6 +231,8 @@ export default defineComponent({ mediaController, dragHandler, input, + alignedGapGradient, + alignedGapCount, togglePlay, toggleEnhancements, visible, @@ -276,6 +294,12 @@ export default defineComponent({ @end="dragHandler.end" @input="input" /> +