diff --git a/client/dive-common/alignedTimeline.spec.ts b/client/dive-common/alignedTimeline.spec.ts new file mode 100644 index 000000000..d0d0520fb --- /dev/null +++ b/client/dive-common/alignedTimeline.spec.ts @@ -0,0 +1,211 @@ +import type { FrameImage } from './apispec'; +import { + buildAlignedTimeline, buildInverseAlignedIndex, canAlign, computeGapGradient, computeGapSlots, +} from './alignedTimeline'; + +function frame(timestamp?: number): FrameImage { + return { url: `url-${timestamp}`, filename: `frame-${timestamp}.png`, timestamp }; +} + +describe('alignedTimeline', () => { + it('aligns when every camera has a timestamp on every frame', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100), frame(200), frame(300)], + }; + expect(canAlign(camerasFrames)).toBe(true); + const result = buildAlignedTimeline(camerasFrames); + expect(result).toEqual({ + aligned: true, + slots: [ + { A: 0, B: 0 }, + { A: 1, B: 1 }, + { A: 2, B: 2 }, + ], + }); + }); + + it('falls back when any single frame anywhere is missing a timestamp', () => { + const camerasFrames = { + A: [frame(100), frame(undefined)], + B: [frame(100), frame(200)], + }; + expect(canAlign(camerasFrames)).toBe(false); + expect(buildAlignedTimeline(camerasFrames)).toEqual({ aligned: false }); + }); + + it('resolves a genuinely missing frame on one camera to undefined at that slot', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100), frame(300)], + }; + const result = buildAlignedTimeline(camerasFrames); + expect(result).toEqual({ + aligned: true, + slots: [ + { A: 0, B: 0 }, + { A: 1, B: undefined }, + { A: 2, B: 1 }, + ], + }); + }); + + it('merges cameras with clock drift within tolerance into one slot', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100.45), frame(200.4), frame(300.3)], + }; + const result = buildAlignedTimeline(camerasFrames, 0.5); + expect(result).toEqual({ + aligned: true, + slots: [ + { A: 0, B: 0 }, + { A: 1, B: 1 }, + { A: 2, B: 2 }, + ], + }); + }); + + it('splits two same-camera frames within one tolerance window into separate slots', () => { + const camerasFrames = { + A: [frame(100), frame(100.2)], + B: [frame(100.1)], + }; + const result = buildAlignedTimeline(camerasFrames, 0.5); + expect(result).toEqual({ + aligned: true, + slots: [ + { A: 0, B: 0 }, + { A: 1, B: undefined }, + ], + }); + }); + + it('pairs frames positionally when every frame shares one identical timestamp', () => { + // Calibration-style datasets (e.g. C/L/R shots stamped with a single + // collection timestamp) must reduce to positional pairing, not spill each + // camera's frames into separate mostly-blank slots. + const t = 1712495277.206341; + const camerasFrames = { + EO: [frame(t), frame(t), frame(t)], + IR: [frame(t), frame(t), frame(t)], + }; + const result = buildAlignedTimeline(camerasFrames, 0.5); + expect(result).toEqual({ + aligned: true, + slots: [ + { EO: 0, IR: 0 }, + { EO: 1, IR: 1 }, + { EO: 2, IR: 2 }, + ], + }); + }); + + it('disqualifies a dataset with an empty-array (e.g. video-backed) camera', () => { + const camerasFrames = { + A: [] as FrameImage[], + B: [frame(100)], + }; + expect(canAlign(camerasFrames)).toBe(false); + expect(buildAlignedTimeline(camerasFrames)).toEqual({ aligned: false }); + }); + + it('disqualifies datasets with fewer than two cameras', () => { + const camerasFrames = { + A: [frame(100), frame(200)], + }; + expect(canAlign(camerasFrames)).toBe(false); + expect(buildAlignedTimeline(camerasFrames)).toEqual({ aligned: false }); + }); + + describe('computeGapSlots', () => { + it('returns no gaps when every camera has a frame at every slot', () => { + const camerasFrames = { + A: [frame(100), frame(200)], + B: [frame(100), frame(200)], + }; + const result = buildAlignedTimeline(camerasFrames); + expect(result.aligned).toBe(true); + expect(result.aligned && computeGapSlots(result.slots)).toEqual([]); + }); + + it('flags slots where any camera is missing a frame', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100), frame(300)], + }; + const result = buildAlignedTimeline(camerasFrames); + expect(result.aligned).toBe(true); + expect(result.aligned && computeGapSlots(result.slots)).toEqual([1]); + }); + }); + + describe('computeGapGradient', () => { + it('returns none with no gaps or a non-positive maxFrame', () => { + expect(computeGapGradient([], 10)).toBe('none'); + expect(computeGapGradient([1], 0)).toBe('none'); + expect(computeGapGradient([1], -5)).toBe('none'); + }); + + it('centers a gap band on the slider thumb position for its slot', () => { + // Thumb for value 4 of maxFrame 8 sits at 50%; band is [43.75%, 56.25%]. + expect(computeGapGradient([4], 8)).toBe( + 'linear-gradient(to right, transparent 43.75%, #ff5252 43.75%, #ff5252 56.25%, transparent 56.25%)', + ); + }); + + it('keeps a trailing gap at slot === maxFrame visible (clamped to 100%)', () => { + expect(computeGapGradient([8], 8)).toBe( + 'linear-gradient(to right, transparent 93.75%, #ff5252 93.75%, #ff5252 100%, transparent 100%)', + ); + }); + + it('keeps a leading gap at slot 0 visible (clamped to 0%)', () => { + expect(computeGapGradient([0], 8)).toBe( + 'linear-gradient(to right, transparent 0%, #ff5252 0%, #ff5252 6.25%, transparent 6.25%)', + ); + }); + + it('merges consecutive gap slots into a single band', () => { + expect(computeGapGradient([2, 3], 8)).toBe( + 'linear-gradient(to right, transparent 18.75%, #ff5252 18.75%, #ff5252 43.75%, transparent 43.75%)', + ); + }); + + it('emits one band per distinct gap run', () => { + expect(computeGapGradient([1, 3], 8)).toBe( + 'linear-gradient(to right, ' + + 'transparent 6.25%, #ff5252 6.25%, #ff5252 18.75%, transparent 18.75%, ' + + 'transparent 31.25%, #ff5252 31.25%, #ff5252 43.75%, transparent 43.75%)', + ); + }); + + it('widens sub-minimum-width bands around their center so they stay visible', () => { + // One slot of 1024 is ~0.098% wide -- below the 0.25% minimum, so the + // band is widened to exactly 0.25% centered on the thumb at 50%. + expect(computeGapGradient([512], 1024)).toBe( + 'linear-gradient(to right, transparent 49.875%, #ff5252 49.875%, #ff5252 50.125%, transparent 50.125%)', + ); + }); + }); + + describe('buildInverseAlignedIndex', () => { + it('maps each camera\'s local frame back to its global slot', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100), frame(300)], + }; + const result = buildAlignedTimeline(camerasFrames); + expect(result.aligned).toBe(true); + if (!result.aligned) return; + const inverse = buildInverseAlignedIndex(result.slots); + expect(inverse.A.get(0)).toBe(0); + expect(inverse.A.get(1)).toBe(1); + expect(inverse.A.get(2)).toBe(2); + expect(inverse.B.get(0)).toBe(0); + expect(inverse.B.get(1)).toBe(2); + // B has no local frame 2 -- it only ever appears in slots 0 and 2. + expect(inverse.B.get(2)).toBeUndefined(); + }); + }); +}); diff --git a/client/dive-common/alignedTimeline.ts b/client/dive-common/alignedTimeline.ts new file mode 100644 index 000000000..88307e34a --- /dev/null +++ b/client/dive-common/alignedTimeline.ts @@ -0,0 +1,212 @@ +import type { FrameImage } from './apispec'; + +/** + * Max time delta, in seconds, between two cameras' frame timestamps for them to + * be considered "the same instant" (SEAL feature 5). In KAMERA sample data the + * C/L/R cameras share an identical capture timestamp per shot, so for now this + * requires an exact match (0). Widen it only if real captures turn out to have + * sub-second jitter that needs absorbing. + */ +export const FRAME_ALIGNMENT_TOLERANCE_SECONDS = 0; + +/** + * camera name -> local index into that camera's own FrameImage[], or undefined + * if no frame from that camera falls within tolerance of this timeline slot. + */ +export type AlignedSlot = Record; + +export type TimelineResult = + | { aligned: true; slots: AlignedSlot[] } + | { aligned: false }; + +interface FrameTriple { + camera: string; + localIndex: number; + timestamp: number; +} + +/** + * A dataset qualifies for aligned playback only when there are at least two + * cameras, every camera has at least one frame, and every frame on every + * camera has a defined timestamp. A single untimestamped frame anywhere means + * the whole dataset falls back to today's exact positional alignment -- the + * safe, conservative default for datasets whose filenames don't follow a + * recognized timestamp convention (see dive_utils/timestamp_parser.py). + */ +export function canAlign(camerasFrames: Record): boolean { + const cameraNames = Object.keys(camerasFrames); + if (cameraNames.length < 2) { + return false; + } + return cameraNames.every((camera) => { + const frames = camerasFrames[camera]; + return frames.length > 0 && frames.every((frame) => frame.timestamp !== undefined); + }); +} + +/** + * Builds a global aligned timeline from each camera's own frame list, when + * every frame across every camera carries a timestamp (see canAlign). + * + * Algorithm: flatten every camera's frames into (camera, localIndex, + * timestamp) triples, sort ascending by timestamp (tied by camera name then + * local index, for determinism), then sweep once: open a slot at the first + * unassigned triple's timestamp, and absorb subsequent triples within + * `toleranceSeconds` of that slot's start time as long as that camera isn't + * already filled in the open slot. Otherwise close the slot and open a new + * one at that triple. This is deliberately simple (O(n log n), single pass, + * no bipartite matching) -- appropriate given there's no real data yet to + * justify anything smarter. + * + * One explicit simplification: if a camera has two frames within tolerance of + * the same open slot's start (e.g. a faster capture rate than other + * cameras), the second spills into a new slot rather than overwriting the + * first -- first-come-first-served per camera per slot, not silent data loss. + */ +export function buildAlignedTimeline( + camerasFrames: Record, + toleranceSeconds: number = FRAME_ALIGNMENT_TOLERANCE_SECONDS, +): TimelineResult { + if (!canAlign(camerasFrames)) { + return { aligned: false }; + } + + const cameraNames = Object.keys(camerasFrames); + const triples: FrameTriple[] = []; + cameraNames.forEach((camera) => { + camerasFrames[camera].forEach((frame, localIndex) => { + // frame.timestamp is guaranteed defined here by canAlign() above + triples.push({ camera, localIndex, timestamp: frame.timestamp as number }); + }); + }); + + // Tie-break equal timestamps by local index BEFORE camera name: when several + // cameras share identical capture timestamps (e.g. a calibration set where + // every C/L/R shot carries one collection timestamp), interleaving by index + // pairs frame i of every camera into the same slot. Camera-name-first + // ordering would instead sweep all of one camera's same-time frames into + // consecutive single-camera slots (spill), producing a timeline of mostly + // blank panes for data that is really positionally aligned. + triples.sort((a, b) => ( + a.timestamp - b.timestamp + || a.localIndex - b.localIndex + || a.camera.localeCompare(b.camera) + )); + + const slots: AlignedSlot[] = []; + let openSlot: AlignedSlot | null = null; + let openSlotStartTime = 0; + + triples.forEach((triple) => { + const withinTolerance = openSlot !== null + && triple.timestamp - openSlotStartTime <= toleranceSeconds; + const cameraAlreadyFilled = openSlot !== null && openSlot[triple.camera] !== undefined; + + if (!withinTolerance || cameraAlreadyFilled) { + if (openSlot !== null) { + slots.push(openSlot); + } + openSlot = Object.fromEntries(cameraNames.map((camera) => [camera, undefined])); + openSlotStartTime = triple.timestamp; + } + + (openSlot as AlignedSlot)[triple.camera] = triple.localIndex; + }); + if (openSlot !== null) { + slots.push(openSlot); + } + + return { aligned: true, slots }; +} + +/** + * Inverse of a slot lookup: for each camera, maps its own local frame index + * to the global aligned-timeline slot it appears in. Used to translate a + * "seek this camera to its own local frame X" request (e.g. jumping to a + * track's start/end, which is stored in local-frame units) into the correct + * global slot, so every camera stays aligned instead of just the one camera + * jumping independently. Each camera appears in at most one slot per local + * frame (buildAlignedTimeline's first-come-first-served rule), so this is + * always a 1:1 mapping. + */ +export type InverseAlignedIndex = Record>; + +export function buildInverseAlignedIndex(slots: AlignedSlot[]): InverseAlignedIndex { + const inverse: InverseAlignedIndex = {}; + slots.forEach((slot, slotIndex) => { + Object.entries(slot).forEach(([camera, localIndex]) => { + if (localIndex === undefined) { + return; + } + if (!inverse[camera]) { + inverse[camera] = new Map(); + } + inverse[camera].set(localIndex, slotIndex); + }); + }); + return inverse; +} + +/** + * Builds the CSS linear-gradient that Controls.vue paints under the frame + * scrubber to mark aligned-timeline gap slots (slots where at least one + * camera has no frame). Pure so it is unit-testable. + * + * Geometry: a v-slider thumb for value v sits at v/maxFrame of the track + * width, so each gap band is centered on the thumb position for its slot -- + * slot s paints [(s - 0.5) / maxFrame, (s + 0.5) / maxFrame], clamped to + * [0, 100]. Consecutive gap slots are merged into a single band so the CSS + * stop count tracks the number of distinct gaps, not the number of gap + * frames. Bands narrower than `minWidthPct` are widened around their center + * so single-frame gaps stay visible on long timelines. + */ +export function computeGapGradient( + gapSlots: number[], + maxFrame: number, + minWidthPct = 0.25, +): string { + if (!gapSlots.length || maxFrame <= 0) { + return 'none'; + } + const ranges: [number, number][] = []; + let rangeStart = gapSlots[0]; + let rangeEnd = gapSlots[0]; + for (let i = 1; i < gapSlots.length; i += 1) { + if (gapSlots[i] === rangeEnd + 1) { + rangeEnd = gapSlots[i]; + } else { + ranges.push([rangeStart, rangeEnd]); + rangeStart = gapSlots[i]; + rangeEnd = gapSlots[i]; + } + } + ranges.push([rangeStart, rangeEnd]); + const toPct = (frame: number) => (frame / maxFrame) * 100; + const stops: string[] = []; + ranges.forEach(([start, end]) => { + let startPct = Math.max(0, toPct(start - 0.5)); + let endPct = Math.min(100, toPct(end + 0.5)); + if (endPct - startPct < minWidthPct) { + const mid = (startPct + endPct) / 2; + startPct = Math.max(0, mid - minWidthPct / 2); + endPct = Math.min(100, mid + minWidthPct / 2); + } + stops.push(`transparent ${startPct}%`, `#ff5252 ${startPct}%`, `#ff5252 ${endPct}%`, `transparent ${endPct}%`); + }); + return `linear-gradient(to right, ${stops.join(', ')})`; +} + +/** + * Global aligned-timeline slot indices where at least one loaded camera has + * no frame -- i.e. scrubbing to that slot will blank at least one camera's + * pane. Used to render a gap indicator on the timeline scrubber. + */ +export function computeGapSlots(slots: AlignedSlot[]): number[] { + const gaps: number[] = []; + slots.forEach((slot, index) => { + if (Object.values(slot).some((localIndex) => localIndex === undefined)) { + gaps.push(index); + } + }); + return gaps; +} diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 78c40b8ec..e1fd84a66 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -8,6 +8,7 @@ import { Attribute } from 'vue-media-annotator/use/AttributeTypes'; import { CustomStyle } from 'vue-media-annotator/StyleManager'; import { AttributeTrackFilter } from 'vue-media-annotator/AttributeTrackFilterControls'; import { ImageEnhancements } from 'vue-media-annotator/use/useImageEnhancements'; +import type { PercentileStretch } from 'vue-media-annotator/use/useImageEnhancements'; type DatasetType = 'image-sequence' | 'video' | 'multi' | 'large-image'; type MultiTrackRecord = Record; @@ -123,6 +124,8 @@ interface FrameImage { filename: string; /** Required for large-image (tiled) datasets; used as itemId for getTiles/getTileURL */ id?: string; + /** Best-effort capture timestamp (epoch seconds) parsed from the filename, when available */ + timestamp?: number; } export interface MultiCamImportFolderArgs { @@ -133,9 +136,11 @@ export interface MultiCamImportFolderArgs { sourceList: Record; // path/track file per camera calibrationFile?: string; // NPZ calibation matrix file - type: 'image-sequence' | 'video'; + type: 'image-sequence' | 'video' | 'large-image'; } export interface MultiCamImportKeywordArgs { @@ -297,6 +302,12 @@ interface Api { // eslint-disable-next-line @typescript-eslint/no-explicit-any getTileURL?(itemId: string, x: number, y: number, level: number, query: Record): string; + getTileHistogram?(itemId: string, options?: { + bins?: number; + frame?: number; + width?: number; + height?: number; + }): Promise; importAnnotationFile(id: string, path: string, file?: File, additive?: boolean, additivePrepend?: string, set?: string): Promise; // Desktop-only calibration persistence functions @@ -527,3 +538,5 @@ export { MultiCamMedia, MediaImportResponse, }; + +export type { PercentileStretch }; diff --git a/client/dive-common/components/ControlsContainer.vue b/client/dive-common/components/ControlsContainer.vue index c0f135df7..ef7d380ae 100644 --- a/client/dive-common/components/ControlsContainer.vue +++ b/client/dive-common/components/ControlsContainer.vue @@ -19,6 +19,7 @@ import { useAttributesFilters, useCameraStore, useSelectedCamera, + useTime, } from '../../src/provides'; export default defineComponent({ @@ -126,16 +127,39 @@ export default defineComponent({ handler.trackSelect(trackId, false, modifiers); } + const aggregateController = injectAggregateController(); const { - maxFrame, frame, seek, volume, setVolume, setSpeed, speed, - } = injectAggregateController().value; + volume, setVolume, setSpeed, speed, + } = aggregateController.value; + // The timeline charts (line/event charts) are built from trackStores in + // the selected camera's own local frame space. Under an aligned timeline + // (SEAL feature 5) the aggregate controller's frame/maxFrame/seek operate + // in global slot space, which diverges from local frames -- so the + // playhead, axis extent, and chart click-seeks all stay in local space: + // time.frame + the selected camera's maxFrame, with seeks translated + // through seekCameraFrame. All three are passthroughs when alignment + // isn't active. (Controls.vue's main scrubber correctly stays in global + // space; mixing that maxFrame here with a local playhead caused drift.) + const { frame: localFrame } = useTime(); + const timelineMaxFrame = computed(() => { + try { + return aggregateController.value.getController(selectedCamera.value).maxFrame.value; + } catch { + // Selected camera's annotator hasn't mounted yet (e.g. mid load); + // fall back to the aggregate max rather than throwing. + return aggregateController.value.maxFrame.value; + } + }); + function seekToFrame(frame: number) { + aggregateController.value.seekCameraFrame(selectedCamera.value, frame); + } return { currentView, toggleView, - maxFrame, + maxFrame: timelineMaxFrame, multiCam, - frame, - seek, + frame: localFrame, + seek: seekToFrame, volume, setVolume, speed, @@ -322,7 +346,7 @@ export default defineComponent({