From d5e9208b7fb349565d9dee7f9d5c3e8ceb423bf2 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Wed, 1 Jul 2026 15:01:13 -0400 Subject: [PATCH 01/16] Parse per-frame capture timestamps from filenames Multicam datasets assume cameras are frame-aligned by position, but cameras in a rig can independently drop frames, so index i isn't necessarily the same real-world moment across cameras. Phase I of SEAL feature 5: derive a best-effort timestamp per frame from its filename and expose it on FrameImage/MediaResource, so later phases can build a real aligned timeline instead of assuming index parity. Parsing is provisional (no real MML sample filenames yet) and fully optional -- undefined/None is the expected common case and must not affect existing datasets. No meta.json schema change: the timestamp is computed at read time from the filename, mirroring how url/ filename are already derived, not persisted. --- client/dive-common/apispec.ts | 2 + .../desktop/backend/native/common.spec.ts | 21 +++- .../platform/desktop/backend/native/common.ts | 7 +- .../desktop/backend/native/multiCamUtils.ts | 2 + client/platform/desktop/sharedUtils.spec.ts | 37 +++++- client/platform/desktop/sharedUtils.ts | 82 ++++++++++++++ server/dive_server/crud_dataset.py | 12 +- server/dive_utils/models.py | 3 + server/dive_utils/timestamp_parser.py | 107 ++++++++++++++++++ server/tests/test_multicam_dataset.py | 27 ++++- server/tests/test_timestamp_parser.py | 41 +++++++ 11 files changed, 333 insertions(+), 8 deletions(-) create mode 100644 server/dive_utils/timestamp_parser.py create mode 100644 server/tests/test_timestamp_parser.py diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 78c40b8ec..05507b3bd 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -123,6 +123,8 @@ interface FrameImage { filename: string; /** Required for large-image (tiled) datasets; used as itemId for getTiles/getTileURL */ id?: string; + /** Best-effort capture timestamp (epoch seconds) parsed from the filename, when available */ + timestamp?: number; } export interface MultiCamImportFolderArgs { diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 4598d323c..828acab62 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -294,7 +294,7 @@ beforeEach(() => { fps: 5, originalBasePath: '/home/user/media/projectid1data', originalImageFiles: [ - 'foo.png', + 'foo_20230615_143022.png', 'bar.png', ], } as JsonMeta), @@ -342,14 +342,17 @@ beforeEach(() => { 'meta.json': JSON.stringify({ type: 'multi', multiCam: { + defaultDisplay: 'left', cameras: { left: { type: 'image-sequence', originalBasePath: '/home/user/viamedata/DIVE_Projects/stereoDataset/left', + originalImageFiles: ['left_20230615_143022.png', 'left_00002.png'], }, right: { type: 'image-sequence', originalBasePath: '/home/user/viamedata/DIVE_Projects/stereoDataset/right', + originalImageFiles: ['right_00001.png', 'right_00002.png'], }, }, }, @@ -453,7 +456,21 @@ describe('native.common', () => { const data = await common.loadMetadata(settings, 'projectid1', urlMapper); expect(data.id).toBe('projectid1'); expect(data.imageData.map(({ filename }) => filename)).toEqual([ - 'foo.png', 'bar.png', + 'foo_20230615_143022.png', 'bar.png', + ]); + expect(data.imageData[0].timestamp).toBe(1686839422); + expect(data.imageData[1].timestamp).toBeUndefined(); + }); + + it('loadJsonMetadata parses per-camera frame timestamps for multicam datasets', async () => { + const data = await common.loadMetadata(settings, 'stereoDataset', urlMapper); + expect(data.multiCamMedia).not.toBeNull(); + const { cameras } = data.multiCamMedia!; + expect(cameras.left.imageData.map(({ timestamp }) => timestamp)).toEqual([ + 1686839422, undefined, + ]); + expect(cameras.right.imageData.map(({ timestamp }) => timestamp)).toEqual([ + undefined, undefined, ]); }); diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 6939811e0..0d0aa3d18 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -54,7 +54,7 @@ import { JobType, LastCalibrationBaseName, } from 'platform/desktop/constants'; import { - cleanString, filterByGlob, makeid, strNumericCompare, + cleanString, filterByGlob, makeid, parseFrameTimestamp, strNumericCompare, } from 'platform/desktop/sharedUtils'; import processTrackAttributes from './attributeProcessor'; @@ -404,13 +404,16 @@ async function loadMetadata( imageData = projectMetaData.transcodedImageFiles.map((filename: string) => ({ url: makeMediaUrl(npath.join(projectDirData.basePath, filename)), filename, + timestamp: parseFrameTimestamp(filename), })); } else { imageData = projectMetaData.originalImageFiles.map((pathOrFilename: string) => { const absPath = npath.join(projectMetaData.originalBasePath, pathOrFilename); + const filename = npath.basename(absPath); return { url: makeMediaUrl(absPath), - filename: npath.basename(absPath), + filename, + timestamp: parseFrameTimestamp(filename), }; }); } diff --git a/client/platform/desktop/backend/native/multiCamUtils.ts b/client/platform/desktop/backend/native/multiCamUtils.ts index 00dc796ce..5668deb77 100644 --- a/client/platform/desktop/backend/native/multiCamUtils.ts +++ b/client/platform/desktop/backend/native/multiCamUtils.ts @@ -10,6 +10,7 @@ import { JsonMeta, Settings } from 'platform/desktop/constants'; // eslint-disable-next-line import/no-cycle import { loadAnnotationFile, loadJsonMetadata, getValidatedProjectDir } from 'platform/desktop/backend/native/common'; import { serialize } from 'platform/desktop/backend/serializers/viame'; +import { parseFrameTimestamp } from 'platform/desktop/sharedUtils'; /** * Figure out the destination location @@ -171,6 +172,7 @@ function getMultiCamUrls( imageData = displayFilenames.map((filename: string) => ({ url: makeMediaUrl(npath.join(originalBasePath, filename)), filename, + timestamp: parseFrameTimestamp(filename), })); } else if (value.type === 'video') { let displayFilename = value.originalVideoFile; diff --git a/client/platform/desktop/sharedUtils.spec.ts b/client/platform/desktop/sharedUtils.spec.ts index a35a52781..ceaf9cd5d 100644 --- a/client/platform/desktop/sharedUtils.spec.ts +++ b/client/platform/desktop/sharedUtils.spec.ts @@ -1,7 +1,7 @@ /// import fs from 'fs-extra'; import { cloneDeep } from 'lodash'; -import { strNumericCompare } from './sharedUtils'; +import { parseFrameTimestamp, strNumericCompare } from './sharedUtils'; /** Matches tests in python utilities */ const testTuple: string[][][] = fs.readJSONSync('../testutils/imagesort.spec.json'); @@ -14,4 +14,39 @@ describe('sharedUtils', () => { expect(input.sort(strNumericCompare)).not.toEqual(copy); }); }); + + describe('parseFrameTimestamp', () => { + it('parses a YYYYMMDD_HHMMSS datestamp', () => { + expect(parseFrameTimestamp('left_20230615_143022.png')).toBe(1686839422); + }); + + it('parses a datestamp with a fractional-second suffix', () => { + expect(parseFrameTimestamp('left_20230615_143022.500.png')).toBe(1686839422.5); + }); + + it('parses a bare epoch-milliseconds filename', () => { + expect(parseFrameTimestamp('img_1719843225123.tif')).toBeCloseTo(1719843225.123, 6); + }); + + it('parses a bare epoch-seconds filename', () => { + expect(parseFrameTimestamp('img_1719843225.tif')).toBe(1719843225); + }); + + it('returns undefined for a plain sequential filename', () => { + expect(parseFrameTimestamp('img_00001.png')).toBeUndefined(); + }); + + it('returns undefined for a short frame-counter filename', () => { + expect(parseFrameTimestamp('frame042.tif')).toBeUndefined(); + }); + + it('returns undefined for an implausible-range digit run', () => { + expect(parseFrameTimestamp('img_0000000001.png')).toBeUndefined(); + }); + + it('is extension-agnostic (same stem, different extension)', () => { + expect(parseFrameTimestamp('left_20230615_143022.tif')) + .toBe(parseFrameTimestamp('left_20230615_143022.png')); + }); + }); }); diff --git a/client/platform/desktop/sharedUtils.ts b/client/platform/desktop/sharedUtils.ts index 7073d945f..23aa3467b 100644 --- a/client/platform/desktop/sharedUtils.ts +++ b/client/platform/desktop/sharedUtils.ts @@ -64,10 +64,92 @@ function strNumericCompare(input1: string, input2: string) { throw new Error('Unreachable'); } +interface FrameTimestampPattern { + name: string; + regex: RegExp; + toSeconds: (match: RegExpMatchArray) => number | undefined; +} + +/* Roughly year 2000 to year 2100, used to reject implausible epoch guesses */ +function isPlausibleEpochSeconds(seconds: number): boolean { + return seconds >= 946684800 && seconds <= 4102444800; +} + +function dateStampToSeconds(match: RegExpMatchArray): number | undefined { + const [, y, mo, d, h, mi, s, frac] = match; + const year = Number(y); + const month = Number(mo); + const day = Number(d); + const hour = Number(h); + const minute = Number(mi); + const second = Number(s); + if (month < 1 || month > 12) return undefined; + if (day < 1 || day > 31) return undefined; + if (hour > 23 || minute > 59 || second > 59) return undefined; + const millis = Date.UTC(year, month - 1, day, hour, minute, second); + const fracSeconds = frac ? Number(`0.${frac}`) : 0; + return millis / 1000 + fracSeconds; +} + +/* + * PROVISIONAL (SEAL feature 5, Phase I): parses a frame timestamp from a filename + * using a small ordered list of guessed conventions. No real MML sample filenames + * exist yet; replace/extend this list once real naming conventions are known. + * Each entry is tried in order against the extension-stripped filename stem; the + * first regex that matches AND passes its own plausibility check wins. + */ +const FRAME_TIMESTAMP_PATTERNS: FrameTimestampPattern[] = [ + { + // YYYYMMDD[_-]HHMMSS, optionally with a fractional-second suffix + name: 'datestamp', + regex: /(? { + const seconds = Number(match[1]) / 1000; + return isPlausibleEpochSeconds(seconds) ? seconds : undefined; + }, + }, + { + // bare epoch seconds, e.g. img_1719843225.tif + name: 'epoch-seconds', + regex: /(? { + const seconds = Number(match[1]); + return isPlausibleEpochSeconds(seconds) ? seconds : undefined; + }, + }, +]; + +/** + * Best-effort frame capture timestamp, in epoch seconds, parsed from a filename. + * Returns undefined (never throws) when no known convention matches -- this is + * the expected common case and callers must treat it as "no timestamp available". + */ +function parseFrameTimestamp(filename: string): number | undefined { + const stem = filename.replace(/\.[^./\\]+$/, ''); + for (let i = 0; i < FRAME_TIMESTAMP_PATTERNS.length; i += 1) { + const { regex, toSeconds } = FRAME_TIMESTAMP_PATTERNS[i]; + const match = stem.match(regex); + if (match) { + const seconds = toSeconds(match); + if (seconds !== undefined) { + return seconds; + } + } + } + return undefined; +} + export { cleanString, filterByGlob, makeid, + parseFrameTimestamp, strChunks, strNumericCompare, }; diff --git a/server/dive_server/crud_dataset.py b/server/dive_server/crud_dataset.py index c0f7dc24f..3fe765681 100644 --- a/server/dive_server/crud_dataset.py +++ b/server/dive_server/crud_dataset.py @@ -16,7 +16,16 @@ from dive_server import crud, crud_annotation from dive_tasks import tasks -from dive_utils import TRUTHY_META_VALUES, asbool, calibration_format, constants, fromMeta, models, types +from dive_utils import ( + TRUTHY_META_VALUES, + asbool, + calibration_format, + constants, + fromMeta, + models, + timestamp_parser, + types, +) from dive_utils.serializers import kwcoco @@ -357,6 +366,7 @@ def get_media( id=str(image["_id"]), url=get_url(dsFolder, image), filename=image['name'], + timestamp=timestamp_parser.parse_frame_timestamp(image['name']), ) for image in crud.valid_images(dsFolder, user) ] diff --git a/server/dive_utils/models.py b/server/dive_utils/models.py index 75b33383a..227d572c7 100644 --- a/server/dive_utils/models.py +++ b/server/dive_utils/models.py @@ -260,6 +260,9 @@ class MediaResource(BaseModel): url: str id: str filename: str + # Best-effort capture timestamp (epoch seconds) parsed from the filename. + # See dive_utils.timestamp_parser -- absent when no known convention matches. + timestamp: Optional[float] = None class MultiCamCameraMeta(BaseModel): diff --git a/server/dive_utils/timestamp_parser.py b/server/dive_utils/timestamp_parser.py new file mode 100644 index 000000000..c59ec6fff --- /dev/null +++ b/server/dive_utils/timestamp_parser.py @@ -0,0 +1,107 @@ +""" +PROVISIONAL (SEAL feature 5, Phase I): best-effort frame capture timestamp +parsing from a filename, using a small ordered list of guessed conventions. + +No real MML sample filenames exist yet; replace/extend the pattern list once +real naming conventions are known. Mirrors +client/platform/desktop/sharedUtils.ts's parseFrameTimestamp -- keep the two +implementations in sync. +""" + +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable, List, Match, NamedTuple, Optional + +# Roughly year 2000 to year 2100, used to reject implausible epoch guesses +_MIN_PLAUSIBLE_EPOCH_SECONDS = 946684800 +_MAX_PLAUSIBLE_EPOCH_SECONDS = 4102444800 + + +def _is_plausible_epoch_seconds(seconds: float) -> bool: + return _MIN_PLAUSIBLE_EPOCH_SECONDS <= seconds <= _MAX_PLAUSIBLE_EPOCH_SECONDS + + +def _datestamp_to_seconds(match: Match) -> Optional[float]: + year, month, day, hour, minute, second, frac = match.groups() + month_i, day_i, hour_i, minute_i, second_i = ( + int(month), + int(day), + int(hour), + int(minute), + int(second), + ) + if month_i < 1 or month_i > 12: + return None + if day_i < 1 or day_i > 31: + return None + if hour_i > 23 or minute_i > 59 or second_i > 59: + return None + try: + dt = datetime( + int(year), month_i, day_i, hour_i, minute_i, second_i, tzinfo=timezone.utc + ) + except ValueError: + # e.g. Feb 30 -- a valid-looking but nonexistent calendar date + return None + frac_seconds = float(f'0.{frac}') if frac else 0.0 + return dt.timestamp() + frac_seconds + + +def _epoch_millis_to_seconds(match: Match) -> Optional[float]: + seconds = int(match.group(1)) / 1000 + return seconds if _is_plausible_epoch_seconds(seconds) else None + + +def _epoch_seconds_to_seconds(match: Match) -> Optional[float]: + seconds = float(match.group(1)) + return seconds if _is_plausible_epoch_seconds(seconds) else None + + +class _FrameTimestampPattern(NamedTuple): + name: str + regex: re.Pattern + to_seconds: Callable[[Match], Optional[float]] + + +# Tried in order against the extension-stripped filename stem; the first regex +# that matches AND passes its own plausibility check wins. +_FRAME_TIMESTAMP_PATTERNS: List[_FrameTimestampPattern] = [ + _FrameTimestampPattern( + # YYYYMMDD[_-]HHMMSS, optionally with a fractional-second suffix + name='datestamp', + regex=re.compile( + r'(? Optional[float]: + """ + Best-effort frame capture timestamp, in epoch seconds, parsed from a + filename. Returns None (never raises) when no known convention matches -- + this is the expected common case and callers must treat it as + "no timestamp available". + """ + stem = Path(filename).stem + for pattern in _FRAME_TIMESTAMP_PATTERNS: + match = pattern.regex.search(stem) + if match: + seconds = pattern.to_seconds(match) + if seconds is not None: + return seconds + return None diff --git a/server/tests/test_multicam_dataset.py b/server/tests/test_multicam_dataset.py index 2f7ed13f2..50932098a 100644 --- a/server/tests/test_multicam_dataset.py +++ b/server/tests/test_multicam_dataset.py @@ -105,7 +105,12 @@ def load_folder(folder_id, level=None, user=None): folder_cls.return_value.load.side_effect = load_folder - left_image = MediaResource(id='img-left', url='/api/v1/.../left.png', filename='left.png') + left_image = MediaResource( + id='img-left', + url='/api/v1/.../left.png', + filename='left_20230615_143022.png', + timestamp=1686839422.0, + ) right_image = MediaResource(id='img-right', url='/api/v1/.../right.png', filename='right.png') def media_for_child(child_folder, child_user): @@ -124,7 +129,9 @@ def media_for_child(child_folder, child_user): assert result.multiCamMedia.defaultDisplay == 'left' assert result.multiCamMedia.cameraOrder == ['left', 'right'] assert set(result.multiCamMedia.cameras.keys()) == {'left', 'right'} - assert result.multiCamMedia.cameras['left'].imageData[0].filename == 'left.png' + assert result.multiCamMedia.cameras['left'].imageData[0].filename == 'left_20230615_143022.png' + assert result.multiCamMedia.cameras['left'].imageData[0].timestamp == 1686839422.0 + assert result.multiCamMedia.cameras['right'].imageData[0].timestamp is None assert 'multiCam' not in result.dict() @@ -140,6 +147,22 @@ def test_get_media_multi_parent_returns_empty(_verify): assert result.sourceVideo is None +@patch('dive_server.crud_dataset.crud.valid_images') +@patch('dive_server.crud_dataset.crud.verify_dataset') +def test_get_media_image_sequence_parses_frame_timestamps(_verify, valid_images_mock): + folder = _child_folder('left-id', 'left') + user = {'login': 'tester'} + valid_images_mock.return_value = [ + {'_id': 'img-parseable', 'name': 'left_20230615_143022.png'}, + {'_id': 'img-unparseable', 'name': 'left_00001.png'}, + ] + + result = crud_dataset.get_media(folder, user) + + assert result.imageData[0].timestamp == 1686839422.0 + assert result.imageData[1].timestamp is None + + @patch('dive_server.crud_dataset.get_media') @patch('dive_server.crud_dataset.Folder') def test_get_multi_cam_media_missing_child_raises(folder_cls, get_media_mock): diff --git a/server/tests/test_timestamp_parser.py b/server/tests/test_timestamp_parser.py new file mode 100644 index 000000000..fe514029d --- /dev/null +++ b/server/tests/test_timestamp_parser.py @@ -0,0 +1,41 @@ +import pytest + +from dive_utils import timestamp_parser + + +def test_parses_datestamp(): + assert timestamp_parser.parse_frame_timestamp('left_20230615_143022.png') == 1686839422.0 + + +def test_parses_datestamp_with_fractional_seconds(): + assert timestamp_parser.parse_frame_timestamp('left_20230615_143022.500.png') == pytest.approx( + 1686839422.5 + ) + + +def test_parses_bare_epoch_millis(): + assert timestamp_parser.parse_frame_timestamp('img_1719843225123.tif') == pytest.approx( + 1719843225.123 + ) + + +def test_parses_bare_epoch_seconds(): + assert timestamp_parser.parse_frame_timestamp('img_1719843225.tif') == 1719843225.0 + + +def test_returns_none_for_plain_sequential_filename(): + assert timestamp_parser.parse_frame_timestamp('img_00001.png') is None + + +def test_returns_none_for_short_frame_counter_filename(): + assert timestamp_parser.parse_frame_timestamp('frame042.tif') is None + + +def test_returns_none_for_implausible_range_digit_run(): + assert timestamp_parser.parse_frame_timestamp('img_0000000001.png') is None + + +def test_is_extension_agnostic(): + tif = timestamp_parser.parse_frame_timestamp('left_20230615_143022.tif') + png = timestamp_parser.parse_frame_timestamp('left_20230615_143022.png') + assert tif == png From 4d490875578bfcee5d50ce2989cb92f10619a839 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Wed, 1 Jul 2026 15:48:08 -0400 Subject: [PATCH 02/16] Align multicam playback to a global timestamp-derived timeline Multicam playback assumed cameras were frame-aligned by position, but cameras in a rig can independently drop frames, so index i wasn't necessarily the same real-world moment across cameras. Phase II of SEAL feature 5: when every camera has a timestamp on every frame, build a global aligned timeline and resolve each camera's own local frame per slot, blanking a camera's pane when it has none at that instant. Falls back byte-identically to today's positional broadcast whenever any camera lacks full timestamp coverage -- zero risk to existing datasets. Also fixes a latent gap this surfaced: continuous playback bypassed the aggregate controller entirely (each camera free-ran its own frame-advance loop), so aligned mode would have applied to scrubbing but silently reverted to raw per-camera advancement during playback. Centralizes a single tick in the aggregate controller when aligned. --- client/dive-common/alignedTimeline.spec.ts | 98 ++++++++++ client/dive-common/alignedTimeline.ts | 112 +++++++++++ client/dive-common/components/Viewer.vue | 26 +++ .../components/annotators/ImageAnnotator.vue | 27 ++- .../components/annotators/VideoAnnotator.vue | 29 ++- .../annotators/mediaControllerType.ts | 21 ++ .../annotators/useMediaController.spec.ts | 179 ++++++++++++++++++ .../annotators/useMediaController.ts | 127 +++++++++++-- 8 files changed, 604 insertions(+), 15 deletions(-) create mode 100644 client/dive-common/alignedTimeline.spec.ts create mode 100644 client/dive-common/alignedTimeline.ts create mode 100644 client/src/components/annotators/useMediaController.spec.ts diff --git a/client/dive-common/alignedTimeline.spec.ts b/client/dive-common/alignedTimeline.spec.ts new file mode 100644 index 000000000..08e4b2c8d --- /dev/null +++ b/client/dive-common/alignedTimeline.spec.ts @@ -0,0 +1,98 @@ +import type { FrameImage } from './apispec'; +import { buildAlignedTimeline, canAlign } from './alignedTimeline'; + +function frame(timestamp?: number): FrameImage { + return { url: `url-${timestamp}`, filename: `frame-${timestamp}.png`, timestamp }; +} + +describe('alignedTimeline', () => { + it('aligns when every camera has a timestamp on every frame', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100), frame(200), frame(300)], + }; + expect(canAlign(camerasFrames)).toBe(true); + const result = buildAlignedTimeline(camerasFrames); + expect(result).toEqual({ + aligned: true, + slots: [ + { A: 0, B: 0 }, + { A: 1, B: 1 }, + { A: 2, B: 2 }, + ], + }); + }); + + it('falls back when any single frame anywhere is missing a timestamp', () => { + const camerasFrames = { + A: [frame(100), frame(undefined)], + B: [frame(100), frame(200)], + }; + expect(canAlign(camerasFrames)).toBe(false); + expect(buildAlignedTimeline(camerasFrames)).toEqual({ aligned: false }); + }); + + it('resolves a genuine out-of-tolerance gap on one camera to undefined at that slot', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100.1), frame(300.2)], + }; + const result = buildAlignedTimeline(camerasFrames); + expect(result).toEqual({ + aligned: true, + slots: [ + { A: 0, B: 0 }, + { A: 1, B: undefined }, + { A: 2, B: 1 }, + ], + }); + }); + + it('merges cameras with clock drift within tolerance into one slot', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100.45), frame(200.4), frame(300.3)], + }; + const result = buildAlignedTimeline(camerasFrames, 0.5); + expect(result).toEqual({ + aligned: true, + slots: [ + { A: 0, B: 0 }, + { A: 1, B: 1 }, + { A: 2, B: 2 }, + ], + }); + }); + + it('splits two same-camera frames within one tolerance window into separate slots', () => { + const camerasFrames = { + A: [frame(100), frame(100.2)], + B: [frame(100.1)], + }; + const result = buildAlignedTimeline(camerasFrames, 0.5); + expect(result).toEqual({ + aligned: true, + slots: [ + { A: 0, B: 0 }, + { A: 1, B: undefined }, + ], + }); + }); + + it('disqualifies a dataset with an empty-array (e.g. video-backed) camera', () => { + const camerasFrames = { + A: [] as FrameImage[], + B: [frame(100)], + }; + expect(canAlign(camerasFrames)).toBe(false); + expect(buildAlignedTimeline(camerasFrames)).toEqual({ aligned: false }); + }); + + it('disqualifies datasets with fewer than two cameras', () => { + const camerasFrames = { + A: [frame(100), frame(200)], + }; + expect(canAlign(camerasFrames)).toBe(false); + expect(buildAlignedTimeline(camerasFrames)).toEqual({ aligned: false }); + }); +}); diff --git a/client/dive-common/alignedTimeline.ts b/client/dive-common/alignedTimeline.ts new file mode 100644 index 000000000..2cf3501aa --- /dev/null +++ b/client/dive-common/alignedTimeline.ts @@ -0,0 +1,112 @@ +import type { FrameImage } from './apispec'; + +/** + * PROVISIONAL (SEAL feature 5, Phase II): max time delta, in seconds, between + * two cameras' frame timestamps for them to be considered "the same instant." + * No real timestamped multicam sample data exists yet (SEALTK_MIGRATION_PLAN.md + * Q2) -- trivial to retune once real data is available. + */ +export const FRAME_ALIGNMENT_TOLERANCE_SECONDS = 0.5; + +/** + * camera name -> local index into that camera's own FrameImage[], or undefined + * if no frame from that camera falls within tolerance of this timeline slot. + */ +export type AlignedSlot = Record; + +export type TimelineResult = + | { aligned: true; slots: AlignedSlot[] } + | { aligned: false }; + +interface FrameTriple { + camera: string; + localIndex: number; + timestamp: number; +} + +/** + * A dataset qualifies for aligned playback only when there are at least two + * cameras, every camera has at least one frame, and every frame on every + * camera has a defined timestamp. A single untimestamped frame anywhere means + * the whole dataset falls back to today's exact positional alignment -- this + * is the safe, conservative default until real MML naming conventions (and + * thus better timestamp coverage) are confirmed. + */ +export function canAlign(camerasFrames: Record): boolean { + const cameraNames = Object.keys(camerasFrames); + if (cameraNames.length < 2) { + return false; + } + return cameraNames.every((camera) => { + const frames = camerasFrames[camera]; + return frames.length > 0 && frames.every((frame) => frame.timestamp !== undefined); + }); +} + +/** + * Builds a global aligned timeline from each camera's own frame list, when + * every frame across every camera carries a timestamp (see canAlign). + * + * Algorithm: flatten every camera's frames into (camera, localIndex, + * timestamp) triples, sort ascending by timestamp (tied by camera name then + * local index, for determinism), then sweep once: open a slot at the first + * unassigned triple's timestamp, and absorb subsequent triples within + * `toleranceSeconds` of that slot's start time as long as that camera isn't + * already filled in the open slot. Otherwise close the slot and open a new + * one at that triple. This is deliberately simple (O(n log n), single pass, + * no bipartite matching) -- appropriate given there's no real data yet to + * justify anything smarter. + * + * One explicit simplification: if a camera has two frames within tolerance of + * the same open slot's start (e.g. a faster capture rate than other + * cameras), the second spills into a new slot rather than overwriting the + * first -- first-come-first-served per camera per slot, not silent data loss. + */ +export function buildAlignedTimeline( + camerasFrames: Record, + toleranceSeconds: number = FRAME_ALIGNMENT_TOLERANCE_SECONDS, +): TimelineResult { + if (!canAlign(camerasFrames)) { + return { aligned: false }; + } + + const cameraNames = Object.keys(camerasFrames); + const triples: FrameTriple[] = []; + cameraNames.forEach((camera) => { + camerasFrames[camera].forEach((frame, localIndex) => { + // frame.timestamp is guaranteed defined here by canAlign() above + triples.push({ camera, localIndex, timestamp: frame.timestamp as number }); + }); + }); + + triples.sort((a, b) => ( + a.timestamp - b.timestamp + || a.camera.localeCompare(b.camera) + || a.localIndex - b.localIndex + )); + + const slots: AlignedSlot[] = []; + let openSlot: AlignedSlot | null = null; + let openSlotStartTime = 0; + + triples.forEach((triple) => { + const withinTolerance = openSlot !== null + && triple.timestamp - openSlotStartTime <= toleranceSeconds; + const cameraAlreadyFilled = openSlot !== null && openSlot[triple.camera] !== undefined; + + if (!withinTolerance || cameraAlreadyFilled) { + if (openSlot !== null) { + slots.push(openSlot); + } + openSlot = Object.fromEntries(cameraNames.map((camera) => [camera, undefined])); + openSlotStartTime = triple.timestamp; + } + + (openSlot as AlignedSlot)[triple.camera] = triple.localIndex; + }); + if (openSlot !== null) { + slots.push(openSlot); + } + + return { aligned: true, slots }; +} diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 83a157ed2..c3f5eaadd 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -62,6 +62,7 @@ import type { import clientSettingsSetup, { clientSettings, isStereoInteractiveModeEnabled } from 'dive-common/store/settings'; import { useApi, FrameImage, DatasetType } from 'dive-common/apispec'; import { orderedMultiCamCameraNames } from 'dive-common/multicamDisplay'; +import { buildAlignedTimeline, TimelineResult } from 'dive-common/alignedTimeline'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import context from 'dive-common/store/context'; import { MarkChangesPendingFilter } from 'vue-media-annotator/BaseFilterControls'; @@ -154,6 +155,7 @@ export default defineComponent({ aggregateController, onResize, clear: mediaControllerClear, + setAlignedFrameResolver, } = useMediaController(); const { time, updateTime, initialize: initTime } = useTimeObserver(); const imageData = ref({ singleCam: [] } as Record); @@ -175,6 +177,30 @@ export default defineComponent({ // Total tracks total: 0, }); + /** + * Global aligned-timeline resolution (SEAL feature 5, Phase II): only + * engages when every camera in a multicam dataset has a timestamp on + * every frame (see alignedTimeline.ts's canAlign). Otherwise -- including + * always for singleCam datasets -- playback falls back to today's exact + * positional (broadcast-same-index) behavior via useMediaController.ts. + */ + const alignedTimeline = computed(() => { + if (!progress.loaded || multiCamList.value.length < 2) { + return { aligned: false }; + } + return buildAlignedTimeline(imageData.value); + }); + watch(alignedTimeline, (result) => { + if (result.aligned) { + setAlignedFrameResolver({ + slotCount: computed(() => result.slots.length), + frameRate: time.frameRate, + resolveSlot: (f) => result.slots[f] ?? {}, + }); + } else { + setAlignedFrameResolver(null); + } + }, { immediate: true }); const controlsRef = ref(); const controlsHeight = ref(0); const controlsCollapsed = ref(false); diff --git a/client/src/components/annotators/ImageAnnotator.vue b/client/src/components/annotators/ImageAnnotator.vue index bb9e50630..ef4dbe431 100644 --- a/client/src/components/annotators/ImageAnnotator.vue +++ b/client/src/components/annotators/ImageAnnotator.vue @@ -72,6 +72,7 @@ export default defineComponent({ container, initializeViewer, mediaController, + externallyDriven, } = cameraInitializer(props.camera, { // allow hoisting for these functions to pass a reference before defining them. // eslint-disable-next-line @typescript-eslint/no-use-before-define @@ -232,10 +233,27 @@ export default defineComponent({ cacheNewRange(min, max); } } - async function seek(f: number) { + async function seek(f: number | undefined) { if (!data.ready) { return; } + if (f === undefined) { + // No frame for this camera at the current aligned-timeline slot: blank + // the pane. Deliberately leaves data.frame/data.filename untouched -- + // those are read elsewhere (e.g. annotation-overlay lookups) and this + // phase doesn't touch annotation storage. + data.hasFrame = false; + if (local.quadFeature !== undefined) { + local.quadFeature.layer().node().css('visibility', 'hidden'); + } + return; + } + if (!data.hasFrame) { + data.hasFrame = true; + if (local.quadFeature !== undefined) { + local.quadFeature.layer().node().css('visibility', ''); + } + } let newFrame = f; if (f < 0) newFrame = 0; if (f > data.maxFrame) newFrame = data.maxFrame; @@ -313,7 +331,12 @@ export default defineComponent({ async function play() { try { data.playing = true; - syncWithVideo(data.frame + 1); + // When a global aligned timeline is driving playback, the aggregate + // controller's own centralized tick calls seek() directly -- this + // camera must not also free-run its own loop. + if (!externallyDriven.value) { + syncWithVideo(data.frame + 1); + } props.updateTime(data); } catch (ex) { console.error(ex); diff --git a/client/src/components/annotators/VideoAnnotator.vue b/client/src/components/annotators/VideoAnnotator.vue index 02125937e..946a6ae37 100644 --- a/client/src/components/annotators/VideoAnnotator.vue +++ b/client/src/components/annotators/VideoAnnotator.vue @@ -117,6 +117,7 @@ export default defineComponent({ container, initializeViewer, mediaController, + externallyDriven, } = cameraInitializer(props.camera, { // allow hoisting for these functions. // eslint-disable-next-line @typescript-eslint/no-use-before-define @@ -144,7 +145,26 @@ export default defineComponent({ video.pause(); } }); - async function seek(frame: number) { + async function seek(frame: number | undefined) { + if (frame === undefined) { + // No frame for this camera at the current aligned-timeline slot: blank + // the pane. Leaves data.frame untouched -- it's read elsewhere (e.g. + // annotation-overlay lookups) and this phase doesn't touch annotation + // storage. In practice unreachable today since a video-backed camera + // (empty imageData) always disqualifies the whole dataset from aligned + // mode (see alignedTimeline.ts's canAlign) -- kept for symmetry/safety. + data.hasFrame = false; + if (quadFeatureLayer !== undefined) { + quadFeatureLayer.node().css('visibility', 'hidden'); + } + return; + } + if (!data.hasFrame) { + data.hasFrame = true; + if (quadFeatureLayer !== undefined) { + quadFeatureLayer.node().css('visibility', ''); + } + } /** Only perform seek for whole frame numbers */ const requestedFrame = Math.round(frame); /** Different seek approaches based on known information */ @@ -187,7 +207,12 @@ export default defineComponent({ await video.play(); data.playing = true; props.updateTime(data); - syncWithVideo(); + // When a global aligned timeline is driving playback, the aggregate + // controller's own centralized tick calls seek() directly -- this + // camera must not also free-run its own loop. + if (!externallyDriven.value) { + syncWithVideo(); + } } catch (ex) { console.error(ex); } diff --git a/client/src/components/annotators/mediaControllerType.ts b/client/src/components/annotators/mediaControllerType.ts index 27a955ef1..834ce47fa 100644 --- a/client/src/components/annotators/mediaControllerType.ts +++ b/client/src/components/annotators/mediaControllerType.ts @@ -1,5 +1,20 @@ import type { Ref } from 'vue'; +/** + * Supplied by Viewer.vue when every camera in a multicam dataset has a + * timestamp on every frame (see dive-common/alignedTimeline.ts). Translates + * a global aligned-timeline slot into each camera's own local frame index, + * or undefined for a camera with no frame at that slot. Absent (null) + * whenever alignment isn't possible/applicable -- including always for + * single-camera datasets -- in which case playback falls back to today's + * exact positional (broadcast-same-index) behavior. + */ +export interface AlignedFrameResolver { + slotCount: Readonly>; + frameRate: Readonly>; + resolveSlot: (globalFrame: number) => Record; +} + /** * AggregateMediaController provides an interface for time and a few * other properties of all cameras in the annotator window. @@ -43,6 +58,12 @@ export interface MediaController extends AggregateMediaController { geoViewerRef: Readonly>; /** @deprecated may be removed in a future release */ syncedFrame: Readonly>; + /** + * Per-camera seek accepts undefined when this camera has no frame at the + * current aligned-timeline slot (see AlignedFrameResolver) -- the camera + * should blank its pane rather than draw anything for that slot. + */ + seek: (frame: number | undefined) => void; centerOn(coords: { x: number; y: number; z: number }): void; transition(coords: { x: number; y:number}, duration: number, zoom?: number): void; diff --git a/client/src/components/annotators/useMediaController.spec.ts b/client/src/components/annotators/useMediaController.spec.ts new file mode 100644 index 000000000..b8ac00651 --- /dev/null +++ b/client/src/components/annotators/useMediaController.spec.ts @@ -0,0 +1,179 @@ +// @vitest-environment jsdom +import { defineComponent, ref } from 'vue'; +// eslint-disable-next-line import/no-extraneous-dependencies -- @vue/test-utils is only used in tests +import { mount } from '@vue/test-utils'; +import { useMediaController } from './useMediaController'; +import type { AlignedFrameResolver } from './mediaControllerType'; + +function noop() { /* unused setVolume/setSpeed stub */ } + +/** + * useMediaController() calls Vue's provide(), so it needs a component setup() + * context. initialize() itself (unlike initializeViewer()) never touches + * GeoJS/DOM, so camera controllers can be registered directly here with + * mocked seek/play/pause -- no need to mount real annotator components. + */ +function mountMediaController() { + const seekA = vi.fn(); + const playA = vi.fn(); + const pauseA = vi.fn(); + const seekB = vi.fn(); + const playB = vi.fn(); + const pauseB = vi.fn(); + + let composable!: ReturnType; + + const Host = defineComponent({ + setup() { + composable = useMediaController(); + composable.initialize('A', { + seek: seekA, play: playA, pause: pauseA, setVolume: noop, setSpeed: noop, + }); + composable.initialize('B', { + seek: seekB, play: playB, pause: pauseB, setVolume: noop, setSpeed: noop, + }); + return {}; + }, + template: '
', + }); + + const wrapper = mount(Host); + return { + wrapper, + composable, + mocks: { + seekA, playA, pauseA, seekB, playB, pauseB, + }, + }; +} + +/** Camera A is missing a frame at slot 1; camera B has one at every slot. */ +function makeGappedResolver(slotCount = 3, frameRate = 2): AlignedFrameResolver { + return { + slotCount: ref(slotCount), + frameRate: ref(frameRate), + resolveSlot: (f: number) => ({ + A: f === 1 ? undefined : f, + B: f, + }), + }; +} + +describe('useMediaController', () => { + it('without a resolver, seek broadcasts the identical frame to every camera', () => { + const { composable, mocks } = mountMediaController(); + composable.aggregateController.value.seek(5); + expect(mocks.seekA).toHaveBeenCalledWith(5); + expect(mocks.seekB).toHaveBeenCalledWith(5); + }); + + it('without a resolver, maxFrame/frame alias the first-initialized camera (today\'s behavior)', () => { + const { composable } = mountMediaController(); + const keyA = Object.keys(composable.state) + .find((k) => composable.state[k].cameraName === 'A'); + expect(keyA).toBeDefined(); + composable.state[keyA as string].maxFrame = 10; + composable.state[keyA as string].frame = 4; + expect(composable.aggregateController.value.maxFrame.value).toBe(10); + expect(composable.aggregateController.value.frame.value).toBe(4); + }); + + it('with a resolver, seek(n) calls each camera with the resolved per-camera frame, including undefined', () => { + const { composable, mocks } = mountMediaController(); + composable.setAlignedFrameResolver(makeGappedResolver()); + composable.aggregateController.value.seek(1); + expect(mocks.seekA).toHaveBeenCalledWith(undefined); + expect(mocks.seekB).toHaveBeenCalledWith(1); + expect(composable.aggregateController.value.frame.value).toBe(1); + expect(composable.aggregateController.value.maxFrame.value).toBe(2); + }); + + it('with a resolver, nextFrame/prevFrame advance the global slot pointer, not each camera\'s own local frame', () => { + const { composable, mocks } = mountMediaController(); + composable.setAlignedFrameResolver(makeGappedResolver()); + composable.aggregateController.value.seek(0); + mocks.seekA.mockClear(); + mocks.seekB.mockClear(); + + composable.aggregateController.value.nextFrame(); + expect(composable.aggregateController.value.frame.value).toBe(1); + expect(mocks.seekA).toHaveBeenLastCalledWith(undefined); + expect(mocks.seekB).toHaveBeenLastCalledWith(1); + + composable.aggregateController.value.nextFrame(); + expect(composable.aggregateController.value.frame.value).toBe(2); + expect(mocks.seekA).toHaveBeenLastCalledWith(2); + expect(mocks.seekB).toHaveBeenLastCalledWith(2); + + composable.aggregateController.value.prevFrame(); + expect(composable.aggregateController.value.frame.value).toBe(1); + expect(mocks.seekA).toHaveBeenLastCalledWith(undefined); + expect(mocks.seekB).toHaveBeenLastCalledWith(1); + }); + + it('play() drives one centralized tick at frameRate and stops at slotCount - 1', () => { + vi.useFakeTimers(); + try { + const { composable, mocks } = mountMediaController(); + composable.setAlignedFrameResolver(makeGappedResolver(3, 2)); // 2fps => 500ms/tick + composable.aggregateController.value.seek(0); + mocks.playA.mockClear(); + mocks.playB.mockClear(); + + composable.aggregateController.value.play(); + expect(mocks.playA).toHaveBeenCalledTimes(1); + expect(mocks.playB).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(500); + expect(composable.aggregateController.value.frame.value).toBe(1); + + vi.advanceTimersByTime(500); + expect(composable.aggregateController.value.frame.value).toBe(2); + + mocks.pauseA.mockClear(); + mocks.pauseB.mockClear(); + vi.advanceTimersByTime(500); + // reached slotCount - 1: auto-pauses and stops advancing + expect(mocks.pauseA).toHaveBeenCalledTimes(1); + expect(mocks.pauseB).toHaveBeenCalledTimes(1); + expect(composable.aggregateController.value.frame.value).toBe(2); + + vi.advanceTimersByTime(1000); + expect(composable.aggregateController.value.frame.value).toBe(2); + } finally { + vi.useRealTimers(); + } + }); + + it('pause() stops the centralized tick', () => { + vi.useFakeTimers(); + try { + const { composable, mocks } = mountMediaController(); + composable.setAlignedFrameResolver(makeGappedResolver(10, 2)); + composable.aggregateController.value.seek(0); + composable.aggregateController.value.play(); + vi.advanceTimersByTime(500); + expect(composable.aggregateController.value.frame.value).toBe(1); + + composable.aggregateController.value.pause(); + expect(mocks.pauseA).toHaveBeenCalled(); + expect(mocks.pauseB).toHaveBeenCalled(); + + vi.advanceTimersByTime(2000); + expect(composable.aggregateController.value.frame.value).toBe(1); + } finally { + vi.useRealTimers(); + } + }); + + it('setAlignedFrameResolver(null) resets the aligned frame and hands control back to camera aliasing', () => { + const { composable } = mountMediaController(); + composable.setAlignedFrameResolver(makeGappedResolver()); + composable.aggregateController.value.seek(2); + expect(composable.aggregateController.value.frame.value).toBe(2); + + composable.setAlignedFrameResolver(null); + // Falls back to the first-initialized camera's own (untouched, still 0) local frame ref. + expect(composable.aggregateController.value.frame.value).toBe(0); + }); +}); diff --git a/client/src/components/annotators/useMediaController.ts b/client/src/components/annotators/useMediaController.ts index 2908ed45b..678730044 100644 --- a/client/src/components/annotators/useMediaController.ts +++ b/client/src/components/annotators/useMediaController.ts @@ -4,12 +4,12 @@ import geo, { GeoEvent } from 'geojs'; import * as d3 from 'd3'; import Vue, { - ref, reactive, provide, toRef, Ref, UnwrapRef, computed, + ref, shallowRef, reactive, provide, toRef, Ref, UnwrapRef, computed, } from 'vue'; import { map, over } from 'lodash'; import { use } from '../../provides'; -import type { AggregateMediaController, MediaController } from './mediaControllerType'; +import type { AggregateMediaController, AlignedFrameResolver, MediaController } from './mediaControllerType'; const AggregateControllerSymbol = Symbol('aggregate-controller'); const CameraInitializerSymbol = Symbol('camera-initializer'); @@ -29,6 +29,8 @@ interface MediaControllerReactiveData { speed: number; maxFrame: number; syncedFrame: number; + /** False when an aligned-timeline slot has no frame for this camera; pane should blank. */ + hasFrame: boolean; cursor: string; imageCursor: string; imageCursorEditing: boolean; @@ -54,12 +56,18 @@ interface CameraInitializerReturn { initializeViewer: (width: number, height: number, tileWidth?: number, tileHeight?: number, isMap?: boolean, geoSpatial?: boolean) => void; mediaController: MediaController; + /** + * True whenever a global aligned timeline is driving playback (see + * AlignedFrameResolver) -- callers should skip starting their own internal + * frame-advance loop and rely on the aggregate controller's seek instead. + */ + externallyDriven: Readonly>; } type CameraInitializerFunc = (cameraName: string, { seek, play, pause, setVolume, setSpeed, }: { - seek(frame: number): void; + seek(frame: number | undefined): void; play(): void; pause(): void; setVolume(level: number): void; @@ -85,6 +93,12 @@ export function useMediaController() { let cameraControllerSymbols: Record = {}; const synchronizeCameras: Ref = ref(false); const resizeTrigger: Ref = ref(0); + // shallowRef: an AlignedFrameResolver carries nested Refs (slotCount, frameRate) + // that must NOT be deep-reactive-converted/auto-unwrapped by a plain ref(). + const alignedFrameResolver: Ref = shallowRef(null); + const alignedCurrentFrame: Ref = ref(0); + const externallyDriven = computed(() => alignedFrameResolver.value !== null); + let alignedPlaybackTimer: ReturnType | undefined; const emptyControllerFrame = ref(0); const emptyControllerMaxFrame = ref(0); const emptyControllerPlaying = ref(false); @@ -113,6 +127,24 @@ export function useMediaController() { cameraSync: synchronizeCameras, resizeTrigger, }; + function stopAlignedPlaybackTimer() { + if (alignedPlaybackTimer !== undefined) { + clearTimeout(alignedPlaybackTimer); + alignedPlaybackTimer = undefined; + } + } + + /** + * Set (or clear, with null) the resolver that translates a global + * aligned-timeline slot into each camera's own local frame index. Called + * by Viewer.vue whenever its computed aligned timeline changes. + */ + function setAlignedFrameResolver(resolver: AlignedFrameResolver | null) { + stopAlignedPlaybackTimer(); + alignedFrameResolver.value = resolver; + alignedCurrentFrame.value = 0; + } + function clear() { geoViewers = {}; containers = {}; @@ -121,6 +153,7 @@ export function useMediaController() { subControllers = []; state = {}; cameraControllerSymbols = {}; + setAlignedFrameResolver(null); } function getController(camera?: string) { @@ -204,7 +237,7 @@ export function useMediaController() { function initialize(cameraName: string, { seek: _seek, play: _play, pause: _pause, setVolume: _setVolume, setSpeed: _setSpeed, }: { - seek(frame: number): void; + seek(frame: number | undefined): void; play(): void; pause(): void; setVolume(level: number): void; @@ -248,6 +281,7 @@ export function useMediaController() { speed: 1.0, maxFrame: 0, syncedFrame: 0, + hasFrame: true, cursor: 'default', imageCursor: '', imageCursorEditing: false, @@ -495,7 +529,74 @@ export function useMediaController() { cursorHandler, initializeViewer, mediaController, + externallyDriven, + }; + } + + /** Seeks every camera to its own local frame for the given global aligned-timeline slot. */ + function alignedSeek(resolver: AlignedFrameResolver, rawFrame: number) { + const maxFrame = Math.max(0, resolver.slotCount.value - 1); + const clamped = Math.min(Math.max(rawFrame, 0), maxFrame); + alignedCurrentFrame.value = clamped; + const perCamera = resolver.resolveSlot(clamped); + subControllers.forEach((mc) => { + mc.seek(perCamera[mc.cameraName.value]); + }); + } + + function aggregateSeek(frame: number) { + const resolver = alignedFrameResolver.value; + if (resolver) { + alignedSeek(resolver, frame); + } else { + subControllers.forEach((mc) => mc.seek(frame)); + } + } + + function aggregateNextFrame() { + const resolver = alignedFrameResolver.value; + if (resolver) { + alignedSeek(resolver, alignedCurrentFrame.value + 1); + } else { + subControllers.forEach((mc) => mc.nextFrame()); + } + } + + function aggregatePrevFrame() { + const resolver = alignedFrameResolver.value; + if (resolver) { + alignedSeek(resolver, alignedCurrentFrame.value - 1); + } else { + subControllers.forEach((mc) => mc.prevFrame()); + } + } + + function aggregatePause() { + subControllers.forEach((mc) => mc.pause()); + stopAlignedPlaybackTimer(); + } + + function aggregatePlay() { + // Each camera still flips its own `playing` UI state; when a resolver is + // set, ImageAnnotator/VideoAnnotator skip starting their own internal + // frame-advance loop (see externallyDriven) and this centralized tick + // drives seeks instead. + subControllers.forEach((mc) => mc.play()); + const resolver = alignedFrameResolver.value; + if (!resolver) { + return; + } + stopAlignedPlaybackTimer(); + const tick = () => { + const maxFrame = Math.max(0, resolver.slotCount.value - 1); + if (alignedCurrentFrame.value >= maxFrame) { + aggregatePause(); + return; + } + alignedSeek(resolver, alignedCurrentFrame.value + 1); + alignedPlaybackTimer = setTimeout(tick, 1000 / Math.max(1, resolver.frameRate.value)); }; + alignedPlaybackTimer = setTimeout(tick, 1000 / Math.max(1, resolver.frameRate.value)); } const aggregateController: Ref = computed(() => { @@ -505,19 +606,22 @@ export function useMediaController() { return emptyAggregateController; } const defaultController = getController(); + const resolver = alignedFrameResolver.value; return { cameras: computed(() => cameras.value.map((v) => String(v))), - maxFrame: defaultController.maxFrame, - frame: defaultController.frame, - seek: over(map(subControllers, 'seek')), - nextFrame: over(map(subControllers, 'nextFrame')), - prevFrame: over(map(subControllers, 'prevFrame')), + maxFrame: resolver + ? computed(() => Math.max(0, resolver.slotCount.value - 1)) + : defaultController.maxFrame, + frame: resolver ? alignedCurrentFrame : defaultController.frame, + seek: aggregateSeek, + nextFrame: aggregateNextFrame, + prevFrame: aggregatePrevFrame, volume: defaultController.volume, setVolume: over(map(subControllers, 'setVolume')), speed: defaultController.speed, setSpeed: over(map(subControllers, 'setSpeed')), - pause: over(map(subControllers, 'pause')), - play: over(map(subControllers, 'play')), + pause: aggregatePause, + play: aggregatePlay, playing: defaultController.playing, resetZoom: over(map(subControllers, 'resetZoom')), currentTime: defaultController.currentTime, @@ -540,5 +644,6 @@ export function useMediaController() { initialize, onResize, clear, + setAlignedFrameResolver, }; } From 4998df4b7e028e338ca074fb2b128a1d25003168 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Wed, 1 Jul 2026 16:30:47 -0400 Subject: [PATCH 03/16] Fix stale annotation overlay and add UI for blanked aligned-timeline panes Phase III of SEAL feature 5. Phase II blanked a camera's image when the global aligned timeline has no frame for it at a slot, but left two gaps: LayerManager kept drawing annotations for whatever local frame the camera was on before blanking, since seek(undefined) deliberately leaves data.frame untouched and LayerManager's watchers are keyed on frame number -- so stale boxes floated over an otherwise-blank pane. The pane itself also gave no visual indication it was blanked rather than just failing to load. Expose the existing per-camera hasFrame flag on MediaController, and have LayerManager watch it: every prior updateLayers() call site now goes through a new refreshLayers(), which disables all annotation/edit layers instead whenever hasFrame is false. ImageAnnotator/VideoAnnotator now render a 'eNo frame at this instant' overlay over a blanked pane. --- client/src/components/LayerManager.vue | 142 ++++++------------ .../components/annotators/ImageAnnotator.vue | 6 + .../components/annotators/VideoAnnotator.vue | 6 + .../src/components/annotators/annotator.scss | 13 ++ .../annotators/mediaControllerType.ts | 6 + .../annotators/useMediaController.ts | 1 + 6 files changed, 80 insertions(+), 94 deletions(-) diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index ac0824c7d..e7764eb5e 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -105,6 +105,7 @@ export default defineComponent({ const annotator = aggregateController.value.getController(props.camera); const frameNumberRef = annotator.frame; const flickNumberRef = annotator.flick; + const hasFrameRef = annotator.hasFrame; const rectAnnotationLayer = new RectangleLayer({ annotator, @@ -454,11 +455,34 @@ export default defineComponent({ } /** - * TODO: for some reason, GeoJS requires us to initialize - * by calling the render function twice. This is a bug. - * https://github.com/Kitware/dive/issues/365 + * Disables every annotation/edit layer without touching stored track + * data. Used when this camera has no frame at the current aligned- + * timeline slot (hasFrameRef false) -- frameNumberRef still holds the + * last real local frame, so leaving the layers as-is would draw stale + * boxes over a blank pane. */ - [1, 2].forEach(() => { + function disableAllLayers() { + rectAnnotationLayer.disable(); + overlapLayer.disable(); + polyAnnotationLayer.disable(); + lineLayer.disable(); + pointLayer.disable(); + tailLayer.disable(); + textLayer.disable(); + attributeLayer.disable(); + attributeBoxLayer.disable(); + editAnnotationLayer.disable(); + segmentationPointsLayer.clear(); + hoverOvered.value = []; + uiLayer.setToolTipWidget('customToolTip', false); + } + + /** Re-runs updateLayers with current ref values, or blanks the layers when this camera has no frame at the current aligned-timeline slot. */ + function refreshLayers() { + if (!hasFrameRef.value) { + disableAllLayers(); + return; + } updateLayers( frameNumberRef.value, editingModeRef.value, @@ -469,6 +493,17 @@ export default defineComponent({ selectedKeyRef.value, props.colorBy, ); + } + + watch(hasFrameRef, () => refreshLayers()); + + /** + * TODO: for some reason, GeoJS requires us to initialize + * by calling the render function twice. This is a bug. + * https://github.com/Kitware/dive/issues/365 + */ + [1, 2].forEach(() => { + refreshLayers(); }); /** Shallow watch */ @@ -486,16 +521,7 @@ export default defineComponent({ selectedKeyRef, ], () => { - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }, ); @@ -503,48 +529,21 @@ export default defineComponent({ watch( annotatorPrefs, () => { - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }, { deep: true }, ); watch(attributes, () => { updateAttributes(); - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }); /** Watch for resize events to redraw layers after view mode changes */ watch( () => aggregateController.value.resizeTrigger.value, () => { - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }, ); @@ -650,16 +649,7 @@ export default defineComponent({ // This is especially important when already editing the same track // since annotation-right-clicked won't be emitted in that case window.setTimeout(() => { - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }, 0); }); polyAnnotationLayer.bus.$on('annotation-ctrl-clicked', Clicked); @@ -676,16 +666,7 @@ export default defineComponent({ // Force layer update to load the newly selected polygon // Use nextTick to ensure the selectedKey ref has been updated window.setTimeout(() => { - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }, 0); }); // Handle right-click outside polygons to finalize/cancel creation @@ -695,16 +676,7 @@ export default defineComponent({ handler.cancelCreation(); handler.selectFeatureHandle(-1, ''); window.setTimeout(() => { - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }, 0); } }); @@ -775,16 +747,7 @@ export default defineComponent({ // Suppress the select that rides along with this same finalizing click. justFinalizedCreation = true; window.setTimeout(() => { justFinalizedCreation = false; }, 0); - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); } }); editAnnotationLayer.bus.$on( @@ -829,16 +792,7 @@ export default defineComponent({ handler.selectFeatureHandle(-1, polygonKey); // Force layer update to load the newly selected polygon window.setTimeout(() => { - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); + refreshLayers(); }, 0); } }); diff --git a/client/src/components/annotators/ImageAnnotator.vue b/client/src/components/annotators/ImageAnnotator.vue index ef4dbe431..5d5e9cb5d 100644 --- a/client/src/components/annotators/ImageAnnotator.vue +++ b/client/src/components/annotators/ImageAnnotator.vue @@ -524,6 +524,12 @@ export default defineComponent({ Loading
+
+ No frame at this instant +
diff --git a/client/src/components/annotators/VideoAnnotator.vue b/client/src/components/annotators/VideoAnnotator.vue index 946a6ae37..c421591da 100644 --- a/client/src/components/annotators/VideoAnnotator.vue +++ b/client/src/components/annotators/VideoAnnotator.vue @@ -388,6 +388,12 @@ export default defineComponent({ @mouseleave="cursorHandler.handleMouseLeave" @mouseover="cursorHandler.handleMouseEnter" /> +
+ No frame at this instant +
diff --git a/client/src/components/annotators/annotator.scss b/client/src/components/annotators/annotator.scss index 1abfa8229..363717153 100644 --- a/client/src/components/annotators/annotator.scss +++ b/client/src/components/annotators/annotator.scss @@ -37,6 +37,19 @@ .geojs-map.annotation-input { cursor: inherit; } + .no-frame-overlay { + z-index: 20; + margin: 0; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + padding: 8px 16px; + background: rgba(0, 0, 0, 0.6); + color: white; + border-radius: 4px; + pointer-events: none; + } } .selected-camera{ diff --git a/client/src/components/annotators/mediaControllerType.ts b/client/src/components/annotators/mediaControllerType.ts index 834ce47fa..1c66037f4 100644 --- a/client/src/components/annotators/mediaControllerType.ts +++ b/client/src/components/annotators/mediaControllerType.ts @@ -58,6 +58,12 @@ export interface MediaController extends AggregateMediaController { geoViewerRef: Readonly>; /** @deprecated may be removed in a future release */ syncedFrame: Readonly>; + /** + * False when an aligned-timeline slot has no frame for this camera (see + * AlignedFrameResolver) -- the pane is blank and LayerManager should not + * draw annotations for the stale `frame` value left over from before. + */ + hasFrame: Readonly>; /** * Per-camera seek accepts undefined when this camera has no frame at the * current aligned-timeline slot (see AlignedFrameResolver) -- the camera diff --git a/client/src/components/annotators/useMediaController.ts b/client/src/components/annotators/useMediaController.ts index 678730044..51eedc617 100644 --- a/client/src/components/annotators/useMediaController.ts +++ b/client/src/components/annotators/useMediaController.ts @@ -499,6 +499,7 @@ export function useMediaController() { maxFrame: toRef(state[camera], 'maxFrame'), speed: toRef(state[camera], 'speed'), syncedFrame: toRef(state[camera], 'syncedFrame'), + hasFrame: toRef(state[camera], 'hasFrame'), prevFrame, nextFrame, play: _play, From 5f34f2a74e72a3e62d028839a5a69a64fc6f51fd Mon Sep 17 00:00:00 2001 From: romleiaj Date: Fri, 3 Jul 2026 22:09:12 -0400 Subject: [PATCH 04/16] Fix local-vs-global frame bugs in editing/seek and add timeline gap indicator Track create/edit/delete and seek operations were keying off the global aligned slot index instead of the selected camera's local frame, writing detections to wrong frame numbers when alignment is active. Adds selectedCameraFrame() in useModeManager, seekCameraFrame() with an inverse aligned index, per-selected-camera time updates in Viewer, and an aligned-gap indicator bar under the frame slider in Controls. Co-Authored-By: Claude Fable 5 --- client/dive-common/alignedTimeline.spec.ts | 46 ++++++++++- client/dive-common/alignedTimeline.ts | 43 ++++++++++ client/dive-common/components/Viewer.vue | 46 +++++++++-- client/dive-common/use/useModeManager.ts | 61 +++++++++----- .../annotators/mediaControllerType.ts | 28 +++++++ .../annotators/useMediaController.spec.ts | 80 ++++++++++++++++++- .../annotators/useMediaController.ts | 27 +++++++ client/src/components/controls/Controls.vue | 60 ++++++++++++++ 8 files changed, 363 insertions(+), 28 deletions(-) diff --git a/client/dive-common/alignedTimeline.spec.ts b/client/dive-common/alignedTimeline.spec.ts index 08e4b2c8d..9271fca8a 100644 --- a/client/dive-common/alignedTimeline.spec.ts +++ b/client/dive-common/alignedTimeline.spec.ts @@ -1,5 +1,7 @@ import type { FrameImage } from './apispec'; -import { buildAlignedTimeline, canAlign } from './alignedTimeline'; +import { + buildAlignedTimeline, buildInverseAlignedIndex, canAlign, computeGapSlots, +} from './alignedTimeline'; function frame(timestamp?: number): FrameImage { return { url: `url-${timestamp}`, filename: `frame-${timestamp}.png`, timestamp }; @@ -95,4 +97,46 @@ describe('alignedTimeline', () => { expect(canAlign(camerasFrames)).toBe(false); expect(buildAlignedTimeline(camerasFrames)).toEqual({ aligned: false }); }); + + describe('computeGapSlots', () => { + it('returns no gaps when every camera has a frame at every slot', () => { + const camerasFrames = { + A: [frame(100), frame(200)], + B: [frame(100), frame(200)], + }; + const result = buildAlignedTimeline(camerasFrames); + expect(result.aligned).toBe(true); + expect(result.aligned && computeGapSlots(result.slots)).toEqual([]); + }); + + it('flags slots where any camera is missing a frame', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100.1), frame(300.2)], + }; + const result = buildAlignedTimeline(camerasFrames); + expect(result.aligned).toBe(true); + expect(result.aligned && computeGapSlots(result.slots)).toEqual([1]); + }); + }); + + describe('buildInverseAlignedIndex', () => { + it('maps each camera\'s local frame back to its global slot', () => { + const camerasFrames = { + A: [frame(100), frame(200), frame(300)], + B: [frame(100.1), frame(300.2)], + }; + const result = buildAlignedTimeline(camerasFrames); + expect(result.aligned).toBe(true); + if (!result.aligned) return; + const inverse = buildInverseAlignedIndex(result.slots); + expect(inverse.A.get(0)).toBe(0); + expect(inverse.A.get(1)).toBe(1); + expect(inverse.A.get(2)).toBe(2); + expect(inverse.B.get(0)).toBe(0); + expect(inverse.B.get(1)).toBe(2); + // B has no local frame 2 -- it only ever appears in slots 0 and 2. + expect(inverse.B.get(2)).toBeUndefined(); + }); + }); }); diff --git a/client/dive-common/alignedTimeline.ts b/client/dive-common/alignedTimeline.ts index 2cf3501aa..1613d4843 100644 --- a/client/dive-common/alignedTimeline.ts +++ b/client/dive-common/alignedTimeline.ts @@ -110,3 +110,46 @@ export function buildAlignedTimeline( return { aligned: true, slots }; } + +/** + * Inverse of a slot lookup: for each camera, maps its own local frame index + * to the global aligned-timeline slot it appears in. Used to translate a + * "seek this camera to its own local frame X" request (e.g. jumping to a + * track's start/end, which is stored in local-frame units) into the correct + * global slot, so every camera stays aligned instead of just the one camera + * jumping independently. Each camera appears in at most one slot per local + * frame (buildAlignedTimeline's first-come-first-served rule), so this is + * always a 1:1 mapping. + */ +export type InverseAlignedIndex = Record>; + +export function buildInverseAlignedIndex(slots: AlignedSlot[]): InverseAlignedIndex { + const inverse: InverseAlignedIndex = {}; + slots.forEach((slot, slotIndex) => { + Object.entries(slot).forEach(([camera, localIndex]) => { + if (localIndex === undefined) { + return; + } + if (!inverse[camera]) { + inverse[camera] = new Map(); + } + inverse[camera].set(localIndex, slotIndex); + }); + }); + return inverse; +} + +/** + * Global aligned-timeline slot indices where at least one loaded camera has + * no frame -- i.e. scrubbing to that slot will blank at least one camera's + * pane. Used to render a gap indicator on the timeline scrubber. + */ +export function computeGapSlots(slots: AlignedSlot[]): number[] { + const gaps: number[] = []; + slots.forEach((slot, index) => { + if (Object.values(slot).some((localIndex) => localIndex === undefined)) { + gaps.push(index); + } + }); + return gaps; +} diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index c3f5eaadd..48e549980 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -32,6 +32,7 @@ import { FilterList, } from 'vue-media-annotator/components'; import type { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; +import type { SetTimeFunc } from 'vue-media-annotator/use/useTimeObserver'; import { getResponseError, featureHasSegmentationPolygon } from 'vue-media-annotator/utils'; /* DIVE COMMON */ @@ -62,7 +63,9 @@ import type { import clientSettingsSetup, { clientSettings, isStereoInteractiveModeEnabled } from 'dive-common/store/settings'; import { useApi, FrameImage, DatasetType } from 'dive-common/apispec'; import { orderedMultiCamCameraNames } from 'dive-common/multicamDisplay'; -import { buildAlignedTimeline, TimelineResult } from 'dive-common/alignedTimeline'; +import { + buildAlignedTimeline, buildInverseAlignedIndex, computeGapSlots, TimelineResult, +} from 'dive-common/alignedTimeline'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import context from 'dive-common/store/context'; import { MarkChangesPendingFilter } from 'vue-media-annotator/BaseFilterControls'; @@ -192,10 +195,13 @@ export default defineComponent({ }); watch(alignedTimeline, (result) => { if (result.aligned) { + const inverseIndex = buildInverseAlignedIndex(result.slots); setAlignedFrameResolver({ slotCount: computed(() => result.slots.length), frameRate: time.frameRate, resolveSlot: (f) => result.slots[f] ?? {}, + resolveGlobalSlot: (camera, localFrame) => inverseIndex[camera]?.get(localFrame), + gapSlots: computed(() => computeGapSlots(result.slots)), }); } else { setAlignedFrameResolver(null); @@ -416,6 +422,24 @@ export default defineComponent({ }, }); + /** + * Every camera pane calls updateTime() from its own seek/play/pause, but + * useTime()'s frame/flick is a single shared value consumed app-wide as + * "the current frame" (track split, keyframe toggling, attribute editing, + * etc.). Under an aligned timeline (SEAL feature 5) cameras can sit on + * different local frames for the same instant, so only the selected + * camera's updates may reach it -- otherwise whichever camera's annotator + * happened to seek last would silently win, regardless of which camera + * the user is actually looking at/editing. + */ + function selectedCameraUpdateTime(camera: string): SetTimeFunc { + return (data) => { + if (selectedCamera.value === camera) { + updateTime(data); + } + }; + } + const { attributesList: attributes, loadAttributes, @@ -794,6 +818,12 @@ export default defineComponent({ } } selectedCamera.value = camera; + // Immediately resync the shared time observer to the newly selected + // camera's own local frame (see selectedCameraUpdateTime) -- otherwise + // it would keep reporting the previously selected camera's local frame + // until the next seek/play/pause happens to land on this camera. + const newCameraController = aggregateController.value.getController(camera); + updateTime({ frame: newCameraController.frame.value, flick: newCameraController.flick.value }); /** * Enters edit mode if no track exists for the camera and forcing edit mode * or if a track exists and are alrady in edit mode we don't set it again @@ -821,7 +851,13 @@ export default defineComponent({ if (!track) { return false; } - return track.getFeature(aggregateController.value.frame.value)[0] == null; + // Must use selectedCamera's own local frame, not aggregateController's + // frame: under an aligned timeline (SEAL feature 5) the aggregate frame + // is the global slot index, which diverges from any camera's local + // frame -- and getFeature() is keyed by local frame, same as tracks are + // stored. See LayerManager.vue's identically-named helper. + const cameraFrame = aggregateController.value.getController(selectedCamera.value).frame.value; + return track.getFeature(cameraFrame)[0] == null; }; // While editing, the creation cursor is live on any camera still missing // the selected track's geometry at this frame (see LayerManager's @@ -1438,7 +1474,7 @@ export default defineComponent({ save, saveThreshold, saveTimeFilter, - updateTime, + selectedCameraUpdateTime, // multicam multiCamList, defaultCamera, @@ -1760,7 +1796,7 @@ export default defineComponent({ v-bind="{ imageData: imageData[camera], videoUrl: videoUrl[camera], - updateTime, + updateTime: selectedCameraUpdateTime(camera), frameRate, originalFps, camera, @@ -1856,7 +1892,7 @@ export default defineComponent({ v-bind="{ imageData: imageData[camera], videoUrl: videoUrl[camera], - updateTime, + updateTime: selectedCameraUpdateTime(camera), frameRate, originalFps, camera, diff --git a/client/dive-common/use/useModeManager.ts b/client/dive-common/use/useModeManager.ts index 38f17ea68..dc9504590 100644 --- a/client/dive-common/use/useModeManager.ts +++ b/client/dive-common/use/useModeManager.ts @@ -121,6 +121,19 @@ export default function useModeManager({ const groupSettings = toRef(clientSettings, 'groupSettings'); const selectedCamera = ref('singleCam'); + /** + * The currently selected camera's own local frame number. Under an aligned + * timeline (SEAL feature 5), aggregateController.value.frame is the global + * slot index, which diverges from any camera's local frame -- but tracks + * are stored/keyed by local frame. Every read/write below that keys track + * storage for selectedCamera must go through this, not aggregateController + * directly. See LayerManager.vue and Viewer.vue's isCreatingNewDetection + * for the identical pattern elsewhere. + */ + function selectedCameraFrame(): number { + return aggregateController.value.getController(selectedCamera.value).frame.value; + } + const linkingState = ref(false); const linkingTrack: Ref = ref(null); const linkingCamera = ref(''); @@ -227,11 +240,11 @@ export default function useModeManager({ const editingDetails = computed(() => { _depend(); if (editingMode.value && selectedTrackId.value !== null) { - const { frame } = aggregateController.value; + const frame = selectedCameraFrame(); try { const track = cameraStore.getPossibleTrack(selectedTrackId.value, selectedCamera.value); if (track) { - const [feature] = track.getFeature(frame.value); + const [feature] = track.getFeature(frame); if (feature) { if (!feature?.bounds?.length) { return 'Creating'; @@ -277,17 +290,28 @@ export default function useModeManager({ } function seekNearest(track: Track) { - // Seek to the nearest point in the track. - const { frame } = aggregateController.value; - if (frame.value < track.begin) { - aggregateController.value.seek(track.begin); - } else if (frame.value > track.end) { - aggregateController.value.seek(track.end); + // Seek to the nearest point in the track. Compares/seeks using + // selectedCamera's own local frame (see selectedCameraFrame) rather than + // aggregateController.frame directly -- under an aligned timeline (SEAL + // feature 5) that's the global slot index, a different number space than + // track.begin/end. seekCameraFrame translates back through the timeline + // so every camera stays aligned; it's a no-op passthrough when alignment + // isn't active. + const frame = selectedCameraFrame(); + if (frame < track.begin) { + aggregateController.value.seekCameraFrame(selectedCamera.value, track.begin); + } else if (frame > track.end) { + aggregateController.value.seekCameraFrame(selectedCamera.value, track.end); } } + // frame is selectedCamera's own local frame (e.g. FilterList.vue's + // "jump to peak frame" derives it from selectedCamera's own trackStore) -- + // route through seekCameraFrame so this still lands correctly under an + // aligned timeline (SEAL feature 5), where local and global frame numbers + // can diverge. function seekFrame(frame: number) { - aggregateController.value.seek(frame); + aggregateController.value.seekCameraFrame(selectedCamera.value, frame); } async function _setLinkingTrack(trackId: TrackId) { @@ -464,7 +488,7 @@ export default function useModeManager({ } // Handles adding a new track with the NewTrack Settings handleEscapeMode(); - const { frame } = aggregateController.value; + const frame = selectedCameraFrame(); let trackType = trackSettings.value.newTrackSettings.type; if (overrideTrackId !== undefined) { const track = cameraStore.getAnyPossibleTrack(overrideTrackId); @@ -479,7 +503,7 @@ export default function useModeManager({ const trackStore = cameraStore.camMap.value.get(selectedCamera.value)?.trackStore; if (trackStore) { const newTrackId = trackStore.add( - frame.value, + frame, trackType, selectedTrackId.value || undefined, overrideTrackId, @@ -785,9 +809,8 @@ export default function useModeManager({ if (track) { recipes.forEach((r) => { if (r.active.value) { - const { frame } = aggregateController.value; r.deletePoint( - frame.value, + selectedCameraFrame(), track, selectedFeatureHandle.value, selectedKey.value, @@ -805,8 +828,7 @@ export default function useModeManager({ if (selectedTrackId.value !== null) { const track = cameraStore.getPossibleTrack(selectedTrackId.value, selectedCamera.value); if (track) { - const { frame } = aggregateController.value; - const frameNum = frame.value; + const frameNum = selectedCameraFrame(); recipes.forEach((r) => { if (r.active.value) { r.delete(frameNum, track, selectedKey.value, annotationModes.editing); @@ -919,8 +941,7 @@ export default function useModeManager({ // When in LineString mode, set the selected key so EditAnnotationLayer // can find the geometry (lines are stored with a recipe key like 'HeadTails'). if (editing) { - const { frame } = aggregateController.value; - const [feature] = track.getFeature(frame.value); + const [feature] = track.getFeature(selectedCameraFrame()); if (feature?.geometry?.features?.length) { if (annotationModes.editing === 'LineString') { const lineFeature = feature.geometry.features.find( @@ -1210,7 +1231,6 @@ export default function useModeManager({ return; } - const { frame } = aggregateController.value; const track = cameraStore.getPossibleTrack(selectedTrackId.value, selectedCamera.value); if (!track) { return; @@ -1244,7 +1264,7 @@ export default function useModeManager({ // Update the track's feature with the preview polygon // Use frame number from the result if provided, otherwise current frame - const targetFrame = result.frameNum ?? frame.value; + const targetFrame = result.frameNum ?? selectedCameraFrame(); const { interpolate } = track.canInterpolate(targetFrame); // Save original feature state before first prediction modifies the track @@ -1501,8 +1521,7 @@ export default function useModeManager({ if (polygonRecipe && 'setAddingPolygon' in polygonRecipe) { const track = cameraStore.getPossibleTrack(selectedTrackId.value, selectedCamera.value); if (track) { - const { frame } = aggregateController.value; - const newKey = track.getNextPolygonKey(frame.value); + const newKey = track.getNextPolygonKey(selectedCameraFrame()); (polygonRecipe as { setAddingPolygon: (key: string) => void }).setAddingPolygon(newKey); } } diff --git a/client/src/components/annotators/mediaControllerType.ts b/client/src/components/annotators/mediaControllerType.ts index 1c66037f4..ece6d1d12 100644 --- a/client/src/components/annotators/mediaControllerType.ts +++ b/client/src/components/annotators/mediaControllerType.ts @@ -13,6 +13,19 @@ export interface AlignedFrameResolver { slotCount: Readonly>; frameRate: Readonly>; resolveSlot: (globalFrame: number) => Record; + /** + * Inverse of resolveSlot (see dive-common/alignedTimeline.ts's + * buildInverseAlignedIndex): given a camera and its own local frame, + * returns the global aligned-timeline slot it appears in, or undefined if + * that local frame isn't part of any slot. + */ + resolveGlobalSlot: (camera: string, localFrame: number) => number | undefined; + /** + * Global slot indices where at least one camera has no frame (see + * dive-common/alignedTimeline.ts's computeGapSlots) -- used to render a + * gap indicator on the timeline scrubber. + */ + gapSlots: Readonly>; } /** @@ -32,6 +45,12 @@ export interface AggregateMediaController { cameraSync: Readonly>; /** Incremented when the viewer is resized, used to trigger layer redraws */ resizeTrigger: Readonly>; + /** + * Global aligned-timeline slot indices with at least one camera missing a + * frame (see AlignedFrameResolver's gapSlots); empty whenever alignment + * isn't active. + */ + alignedGapSlots: Readonly>; pause: () => void; play: () => void; @@ -43,6 +62,15 @@ export interface AggregateMediaController { setSpeed: (speed: number) => void; getController: (cameraName: string) => MediaController; toggleSynchronizeCameras: (sync: boolean) => void; + /** + * Seeks so that `camera` lands on its own local frame `localFrame` (e.g. + * jumping to a track's stored begin/end, which is in local-frame units). + * Under an aligned timeline (see AlignedFrameResolver) this translates + * through the global slot so every camera stays aligned; otherwise it's + * equivalent to seek(localFrame), since local and global frame numbers are + * identical under today's positional broadcast. + */ + seekCameraFrame: (camera: string, localFrame: number) => void; } /** diff --git a/client/src/components/annotators/useMediaController.spec.ts b/client/src/components/annotators/useMediaController.spec.ts index b8ac00651..81c4fb87c 100644 --- a/client/src/components/annotators/useMediaController.spec.ts +++ b/client/src/components/annotators/useMediaController.spec.ts @@ -47,7 +47,12 @@ function mountMediaController() { }; } -/** Camera A is missing a frame at slot 1; camera B has one at every slot. */ +/** + * Camera A is missing a frame at slot 1; camera B has one at every slot. A + * never reports local frame 1 at all in this mock (slot 1 has no A entry, + * and slot 2's A entry is local frame 2, not a shifted-down 1) -- so A's + * local frame space has a hole at 1, identical to the global slot space. + */ function makeGappedResolver(slotCount = 3, frameRate = 2): AlignedFrameResolver { return { slotCount: ref(slotCount), @@ -56,6 +61,39 @@ function makeGappedResolver(slotCount = 3, frameRate = 2): AlignedFrameResolver A: f === 1 ? undefined : f, B: f, }), + resolveGlobalSlot: (camera: string, localFrame: number) => { + if (camera === 'B') return localFrame; + if (camera === 'A') return (localFrame === 0 || localFrame === 2) ? localFrame : undefined; + return undefined; + }, + gapSlots: ref([1]), + }; +} + +/** + * Camera A drops the frame at global slot 1, so unlike makeGappedResolver's + * A, its OWN frame array is contiguous: local frame 0 is slot 0, and local + * frame 1 (its very next captured frame) is slot 2 -- a realistic shift + * between local and global numbering, exercising the non-trivial direction + * of resolveGlobalSlot. + */ +function makeShiftedResolver(): AlignedFrameResolver { + // A's own frame array only has two entries: slot 0's frame, then slot 2's. + const aSlotForGlobalFrame: Record = { 0: 0, 1: undefined, 2: 1 }; + const aGlobalSlotForLocalFrame: Record = { 0: 0, 1: 2 }; + return { + slotCount: ref(3), + frameRate: ref(2), + resolveSlot: (f: number) => ({ + A: aSlotForGlobalFrame[f], + B: f, + }), + resolveGlobalSlot: (camera: string, localFrame: number) => { + if (camera === 'B') return localFrame; + if (camera === 'A') return aGlobalSlotForLocalFrame[localFrame]; + return undefined; + }, + gapSlots: ref([1]), }; } @@ -166,6 +204,46 @@ describe('useMediaController', () => { } }); + it('without a resolver, seekCameraFrame(camera, frame) is equivalent to seek(frame)', () => { + const { composable, mocks } = mountMediaController(); + composable.aggregateController.value.seekCameraFrame('A', 5); + expect(mocks.seekA).toHaveBeenCalledWith(5); + expect(mocks.seekB).toHaveBeenCalledWith(5); + }); + + it('with a resolver, seekCameraFrame(camera, localFrame) translates a shifted local frame to the correct global slot', () => { + const { composable, mocks } = mountMediaController(); + composable.setAlignedFrameResolver(makeShiftedResolver()); + + // A's own local frame 1 is really global slot 2 (A dropped the frame at slot 1). + composable.aggregateController.value.seekCameraFrame('A', 1); + expect(composable.aggregateController.value.frame.value).toBe(2); + expect(mocks.seekA).toHaveBeenLastCalledWith(1); + expect(mocks.seekB).toHaveBeenLastCalledWith(2); + }); + + it('with a resolver, seekCameraFrame falls back to seeking only that camera when its local frame maps to no slot', () => { + const { composable, mocks } = mountMediaController(); + composable.setAlignedFrameResolver(makeGappedResolver()); + mocks.seekB.mockClear(); + + // A never reports local frame 1 in this resolver -- no slot maps to it. + composable.aggregateController.value.seekCameraFrame('A', 1); + expect(mocks.seekA).toHaveBeenLastCalledWith(1); + expect(mocks.seekB).not.toHaveBeenCalled(); + }); + + it('exposes alignedGapSlots from the resolver, empty when unaligned', () => { + const { composable } = mountMediaController(); + expect(composable.aggregateController.value.alignedGapSlots.value).toEqual([]); + + composable.setAlignedFrameResolver(makeGappedResolver()); + expect(composable.aggregateController.value.alignedGapSlots.value).toEqual([1]); + + composable.setAlignedFrameResolver(null); + expect(composable.aggregateController.value.alignedGapSlots.value).toEqual([]); + }); + it('setAlignedFrameResolver(null) resets the aligned frame and hands control back to camera aliasing', () => { const { composable } = mountMediaController(); composable.setAlignedFrameResolver(makeGappedResolver()); diff --git a/client/src/components/annotators/useMediaController.ts b/client/src/components/annotators/useMediaController.ts index 51eedc617..7b9865493 100644 --- a/client/src/components/annotators/useMediaController.ts +++ b/client/src/components/annotators/useMediaController.ts @@ -98,6 +98,7 @@ export function useMediaController() { const alignedFrameResolver: Ref = shallowRef(null); const alignedCurrentFrame: Ref = ref(0); const externallyDriven = computed(() => alignedFrameResolver.value !== null); + const alignedGapSlots = computed(() => alignedFrameResolver.value?.gapSlots.value ?? []); let alignedPlaybackTimer: ReturnType | undefined; const emptyControllerFrame = ref(0); const emptyControllerMaxFrame = ref(0); @@ -126,6 +127,8 @@ export function useMediaController() { toggleSynchronizeCameras, cameraSync: synchronizeCameras, resizeTrigger, + alignedGapSlots, + seekCameraFrame: aggregateSeekCameraFrame, }; function stopAlignedPlaybackTimer() { if (alignedPlaybackTimer !== undefined) { @@ -554,6 +557,28 @@ export function useMediaController() { } } + /** + * Seeks so that `camera` lands on its own local frame `localFrame`. Without + * alignment, local and global frame numbers are identical (today's + * positional broadcast), so this is just aggregateSeek. Under an aligned + * timeline, translates through the global slot via resolveGlobalSlot so + * every camera stays aligned; falls back to seeking only `camera` itself + * if that local frame isn't part of any slot (shouldn't normally happen). + */ + function aggregateSeekCameraFrame(camera: string, localFrame: number) { + const resolver = alignedFrameResolver.value; + if (!resolver) { + aggregateSeek(localFrame); + return; + } + const slot = resolver.resolveGlobalSlot(camera, localFrame); + if (slot !== undefined) { + alignedSeek(resolver, slot); + } else { + getController(camera).seek(localFrame); + } + } + function aggregateNextFrame() { const resolver = alignedFrameResolver.value; if (resolver) { @@ -630,6 +655,8 @@ export function useMediaController() { toggleSynchronizeCameras, cameraSync: synchronizeCameras, resizeTrigger, + alignedGapSlots, + seekCameraFrame: aggregateSeekCameraFrame, }; }); diff --git a/client/src/components/controls/Controls.vue b/client/src/components/controls/Controls.vue index 3bf2239a2..1344abad9 100644 --- a/client/src/components/controls/Controls.vue +++ b/client/src/components/controls/Controls.vue @@ -60,6 +60,52 @@ export default defineComponent({ } data.frame = value; } + + /** + * A CSS gradient overlay marking timeline slots where at least one camera + * has no frame (SEAL feature 5's aligned timeline, see alignedTimeline.ts). + * Uses a gradient rather than one element per gap so this stays cheap + * regardless of how many gaps there are; consecutive gap frames are + * merged into a single stop so the stop count tracks the number of + * distinct gaps, not the number of gap frames. This is a lightweight + * visual approximation drawn under the v-slider, not pixel-exact with its + * clickable thumb track. + */ + const alignedGapGradient = computed(() => { + const gaps = mediaController.alignedGapSlots.value; + const maxFrame = mediaController.maxFrame.value; + if (!gaps.length || maxFrame <= 0) { + return 'none'; + } + const ranges: [number, number][] = []; + let rangeStart = gaps[0]; + let rangeEnd = gaps[0]; + for (let i = 1; i < gaps.length; i += 1) { + if (gaps[i] === rangeEnd + 1) { + rangeEnd = gaps[i]; + } else { + ranges.push([rangeStart, rangeEnd]); + rangeStart = gaps[i]; + rangeEnd = gaps[i]; + } + } + ranges.push([rangeStart, rangeEnd]); + const toPct = (frame: number) => (frame / maxFrame) * 100; + const minWidthPct = 0.25; // keep single-frame gaps visible at any zoom + const stops: string[] = []; + ranges.forEach(([start, end]) => { + let startPct = toPct(start); + let endPct = toPct(end + 1); + if (endPct - startPct < minWidthPct) { + const mid = (startPct + endPct) / 2; + startPct = Math.max(0, mid - minWidthPct / 2); + endPct = Math.min(100, mid + minWidthPct / 2); + } + stops.push(`transparent ${startPct}%`, `#ff5252 ${startPct}%`, `#ff5252 ${endPct}%`, `transparent ${endPct}%`); + }); + return `linear-gradient(to right, ${stops.join(', ')})`; + }); + const alignedGapCount = computed(() => mediaController.alignedGapSlots.value.length); function togglePlay(_: HTMLElement, keyEvent: KeyboardEvent) { // Prevent scroll from spacebar and other default effects. keyEvent.preventDefault(); @@ -215,6 +261,8 @@ export default defineComponent({ mediaController, dragHandler, input, + alignedGapGradient, + alignedGapCount, togglePlay, toggleEnhancements, visible, @@ -276,6 +324,12 @@ export default defineComponent({ @end="dragHandler.end" @input="input" /> +
--- .../ImportMultiCamDialog.vue | 10 ++++++++++ .../useImportMultiCamDialog.ts | 6 ++++++ .../validateMulticamImageSets.spec.ts | 20 +++++++++++++++++++ .../validateMulticamImageSets.ts | 8 +++++++- server/dive_server/crud_dataset.py | 14 ++++++------- 5 files changed, 49 insertions(+), 9 deletions(-) diff --git a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue index 03250ba8b..11afd3228 100644 --- a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue +++ b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue @@ -138,6 +138,16 @@ export default defineComponent({ />
+ + = ref([]); const defaultDisplay = ref('left'); const importAnnotationFilesCheck = ref(false); + // When set, cameras may have differing frame counts; frames are aligned downstream + // by their filename-encoded timestamps instead of by exact positional index. Enables + // importing datasets with dropped frames (e.g. KAMERA). See validateMulticamImageSets. + const inferFrameIndexFromFilename = ref(false); const { error: importError, request: importRequest } = useRequest(); onMounted(async () => { @@ -256,6 +260,7 @@ export function useImportMultiCamDialog( filteredImages.value, Object.keys(globList.value).length, props.dataType, + inferFrameIndexFromFilename.value, ); }); @@ -678,6 +683,7 @@ export function useImportMultiCamDialog( canMoveCamera, moveCamera, importAnnotationFilesCheck, + inferFrameIndexFromFilename, parentFolderName, subfolderLayoutLabel, subfolderOriginalNames, diff --git a/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.spec.ts b/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.spec.ts index a7439b080..22de7649a 100644 --- a/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.spec.ts +++ b/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.spec.ts @@ -26,6 +26,26 @@ describe('validateMulticamImageSets', () => { )).toBe('All cameras should have the same length of 1'); }); + it('allows unequal image counts when inferring frame index from filename', () => { + expect(validateMulticamImageSets( + 'multi', + { left: ['a.png'], right: ['a.png', 'b.png'] }, + 0, + 'image-sequence', + true, + )).toBeNull(); + }); + + it('still requires non-empty cameras when inferring frame index from filename', () => { + expect(validateMulticamImageSets( + 'multi', + { left: [], right: ['a.png'] }, + 0, + 'image-sequence', + true, + )).toBe('Requires filtered Images for left '); + }); + it('rejects overlapping filenames in keyword mode', () => { expect(validateMulticamImageSets( 'keyword', diff --git a/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.ts b/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.ts index e17c9ccd7..e505dbeca 100644 --- a/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.ts +++ b/client/dive-common/components/ImportMultiCamDialog/validateMulticamImageSets.ts @@ -1,6 +1,11 @@ /** * Pure validation for multicam image lists: non-empty filters, equal frame counts, * and mutually exclusive filenames in keyword/glob mode. Video imports skip image checks. + * + * When `inferFrameIndexFromFilename` is set, the equal-frame-count check is skipped: + * datasets with dropped frames (e.g. KAMERA, which encodes a capture timestamp in each + * filename) legitimately have differing per-camera counts and are aligned downstream by + * their filename timestamps rather than by exact positional index. */ export type MulticamImportType = 'multi' | 'keyword' | 'subfolders' | ''; @@ -9,6 +14,7 @@ export function validateMulticamImageSets( filteredImages: Record, globListKeyCount: number, dataType: string, + inferFrameIndexFromFilename = false, ): string | null { if (importType === 'keyword' && globListKeyCount === 0) { return 'Add at least 1 filter pattern'; @@ -30,7 +36,7 @@ export function validateMulticamImageSets( if (length === -1) { length = images.length; } - if (length !== images.length) { + if (!inferFrameIndexFromFilename && length !== images.length) { return `All cameras should have the same length of ${length}`; } if (importType === 'keyword' diff --git a/server/dive_server/crud_dataset.py b/server/dive_server/crud_dataset.py index 3fe765681..c514b85cf 100644 --- a/server/dive_server/crud_dataset.py +++ b/server/dive_server/crud_dataset.py @@ -1233,7 +1233,6 @@ def create_multicam( camera_order = list(cameras.keys()) loaded_children: Dict[str, types.GirderModel] = {} - frame_counts: List[int] = [] child_fps_by_name: Dict[str, float] = {} for name in camera_order: cam = cameras[name] @@ -1257,7 +1256,9 @@ def create_multicam( ) child_fps = fromMeta(child, constants.FPSMarker) child_fps_by_name[name] = child_fps - frame_counts.append(_child_media_frame_count(child, user, validated.type)) + # Called for its validation side effect (e.g. a video camera missing its + # processed video raises here); differing counts across cameras are allowed. + _child_media_frame_count(child, user, validated.type) loaded_children[name] = child use_video_fps = validated.type == constants.VideoType and validated.fps == -1 @@ -1276,12 +1277,9 @@ def create_multicam( code=400, ) - if len(set(frame_counts)) > 1: - expected = frame_counts[0] - raise RestException( - f'All cameras must have the same number of frames (expected {expected})', - code=400, - ) + # NOTE: cameras are intentionally allowed to have differing frame counts. + # Frame alignment pairs frames across cameras downstream, so a per-camera + # frame-count equality check would reject the primary use case for this feature. default_child = loaded_children[validated.defaultDisplay] parent_folder_doc = parent_folder From d5c956c2fee718aa551dfb31e89dd035cf710e58 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Tue, 7 Jul 2026 12:06:07 -0400 Subject: [PATCH 12/16] Remove provisional language --- client/dive-common/alignedTimeline.ts | 15 +++++------ client/platform/desktop/sharedUtils.spec.ts | 6 +++++ client/platform/desktop/sharedUtils.ts | 22 ++++++++-------- server/dive_utils/timestamp_parser.py | 28 ++++++++++++--------- server/tests/test_timestamp_parser.py | 8 ++++++ 5 files changed, 50 insertions(+), 29 deletions(-) diff --git a/client/dive-common/alignedTimeline.ts b/client/dive-common/alignedTimeline.ts index 1309566dd..cc7ba957f 100644 --- a/client/dive-common/alignedTimeline.ts +++ b/client/dive-common/alignedTimeline.ts @@ -1,10 +1,11 @@ import type { FrameImage } from './apispec'; /** - * PROVISIONAL (SEAL feature 5, Phase II): max time delta, in seconds, between - * two cameras' frame timestamps for them to be considered "the same instant." - * No real timestamped multicam sample data exists yet (SEALTK_MIGRATION_PLAN.md - * Q2) -- trivial to retune once real data is available. + * Max time delta, in seconds, between two cameras' frame timestamps for them to + * be considered "the same instant" (SEAL feature 5). In KAMERA sample data the + * C/L/R cameras share an identical capture timestamp per shot, so this mainly + * has to absorb sub-second jitter/rounding; 0.5s leaves margin for less tightly + * synchronized full-flight captures. Retune if real cadence differs. */ export const FRAME_ALIGNMENT_TOLERANCE_SECONDS = 0.5; @@ -28,9 +29,9 @@ interface FrameTriple { * A dataset qualifies for aligned playback only when there are at least two * cameras, every camera has at least one frame, and every frame on every * camera has a defined timestamp. A single untimestamped frame anywhere means - * the whole dataset falls back to today's exact positional alignment -- this - * is the safe, conservative default until real MML naming conventions (and - * thus better timestamp coverage) are confirmed. + * the whole dataset falls back to today's exact positional alignment -- the + * safe, conservative default for datasets whose filenames don't follow a + * recognized timestamp convention (see dive_utils/timestamp_parser.py). */ export function canAlign(camerasFrames: Record): boolean { const cameraNames = Object.keys(camerasFrames); diff --git a/client/platform/desktop/sharedUtils.spec.ts b/client/platform/desktop/sharedUtils.spec.ts index ceaf9cd5d..a891b7a5f 100644 --- a/client/platform/desktop/sharedUtils.spec.ts +++ b/client/platform/desktop/sharedUtils.spec.ts @@ -20,6 +20,12 @@ describe('sharedUtils', () => { expect(parseFrameTimestamp('left_20230615_143022.png')).toBe(1686839422); }); + it('parses a real KAMERA filename with microsecond precision', () => { + // Confirmed convention from real sample data (data/test_data). + expect(parseFrameTimestamp('kamera_calibration_fl02_C_20240407_130757.206341_ir.tif')) + .toBeCloseTo(1712495277.206341, 6); + }); + it('parses a datestamp with a fractional-second suffix', () => { expect(parseFrameTimestamp('left_20230615_143022.500.png')).toBe(1686839422.5); }); diff --git a/client/platform/desktop/sharedUtils.ts b/client/platform/desktop/sharedUtils.ts index 23aa3467b..288522c2b 100644 --- a/client/platform/desktop/sharedUtils.ts +++ b/client/platform/desktop/sharedUtils.ts @@ -70,7 +70,7 @@ interface FrameTimestampPattern { toSeconds: (match: RegExpMatchArray) => number | undefined; } -/* Roughly year 2000 to year 2100, used to reject implausible epoch guesses */ +/* Roughly year 2000 to year 2100, used to reject implausible epoch candidates */ function isPlausibleEpochSeconds(seconds: number): boolean { return seconds >= 946684800 && seconds <= 4102444800; } @@ -92,15 +92,17 @@ function dateStampToSeconds(match: RegExpMatchArray): number | undefined { } /* - * PROVISIONAL (SEAL feature 5, Phase I): parses a frame timestamp from a filename - * using a small ordered list of guessed conventions. No real MML sample filenames - * exist yet; replace/extend this list once real naming conventions are known. - * Each entry is tried in order against the extension-stripped filename stem; the - * first regex that matches AND passes its own plausibility check wins. + * Parses a frame timestamp from a filename using a small ordered list of + * conventions (SEAL feature 5). The primary convention is KAMERA's, confirmed + * against sample data (see the datestamp entry); the epoch-based patterns are + * fallbacks for other capture systems. Each entry is tried in order against the + * extension-stripped filename stem; the first regex that matches AND passes its + * own plausibility check wins. */ const FRAME_TIMESTAMP_PATTERNS: FrameTimestampPattern[] = [ { - // YYYYMMDD[_-]HHMMSS, optionally with a fractional-second suffix + // KAMERA convention: YYYYMMDD[_-]HHMMSS with an optional fractional-second + // suffix, e.g. kamera_calibration_fl02_C_20240407_130757.206341_ir.tif name: 'datestamp', regex: /(? Optional[float]: """ - Best-effort frame capture timestamp, in epoch seconds, parsed from a - filename. Returns None (never raises) when no known convention matches -- - this is the expected common case and callers must treat it as - "no timestamp available". + Frame capture timestamp, in epoch seconds, parsed from a filename. Returns + None (never raises) when no recognized convention matches; callers must + treat that as "no timestamp available". """ stem = Path(filename).stem for pattern in _FRAME_TIMESTAMP_PATTERNS: diff --git a/server/tests/test_timestamp_parser.py b/server/tests/test_timestamp_parser.py index fe514029d..c220ac449 100644 --- a/server/tests/test_timestamp_parser.py +++ b/server/tests/test_timestamp_parser.py @@ -7,6 +7,14 @@ def test_parses_datestamp(): assert timestamp_parser.parse_frame_timestamp('left_20230615_143022.png') == 1686839422.0 +def test_parses_kamera_filename_with_microseconds(): + # Confirmed KAMERA convention from real sample data (data/test_data): a + # YYYYMMDD_HHMMSS.ffffff datestamp with microsecond precision. + assert timestamp_parser.parse_frame_timestamp( + 'kamera_calibration_fl02_C_20240407_130757.206341_ir.tif' + ) == pytest.approx(1712495277.206341) + + def test_parses_datestamp_with_fractional_seconds(): assert timestamp_parser.parse_frame_timestamp('left_20230615_143022.500.png') == pytest.approx( 1686839422.5 From 196f45bc80161a98581af2ad5175f0de72595c91 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Tue, 7 Jul 2026 12:24:47 -0400 Subject: [PATCH 13/16] Unify frame-timestamp parsing on a single client implementation Replace the hand-synced Python + TypeScript filename timestamp parsers with one shared TS module (dive-common/frameTimestamp.ts) used by both platforms. The girder server previously parsed each frame's timestamp in get_media and embedded it in the /media response, but nothing server-side ever read it -- it existed only to reach the client. Move parsing to the client: the web-girder API layer now attaches timestamps from filenames after fetching media (both the single-cam /media response and multicam per-camera imageData from /dive_dataset), mirroring how the desktop backend already parses on media enumeration. - Add dive-common/frameTimestamp.ts (parseFrameTimestamp + attachFrameTimestamps) and move its tests out of sharedUtils.spec.ts. - Point the desktop backend importers at the new shared module. - Delete server dive_utils/timestamp_parser.py and its test; drop the now-dead MediaResource.timestamp field and the get_media parsing call; update the multicam dataset tests that asserted server-side timestamps. Co-Authored-By: Claude Opus 4.8 (1M context) --- client/dive-common/frameTimestamp.spec.ts | 55 +++++++++ client/dive-common/frameTimestamp.ts | 104 ++++++++++++++++ .../platform/desktop/backend/native/common.ts | 3 +- .../desktop/backend/native/multiCamUtils.ts | 2 +- client/platform/desktop/sharedUtils.spec.ts | 43 +------ client/platform/desktop/sharedUtils.ts | 84 ------------- .../web-girder/api/dataset.service.ts | 12 +- server/dive_server/crud_dataset.py | 2 - server/dive_utils/models.py | 3 - server/dive_utils/timestamp_parser.py | 111 ------------------ server/tests/test_multicam_dataset.py | 19 --- server/tests/test_timestamp_parser.py | 49 -------- 12 files changed, 174 insertions(+), 313 deletions(-) create mode 100644 client/dive-common/frameTimestamp.spec.ts create mode 100644 client/dive-common/frameTimestamp.ts delete mode 100644 server/dive_utils/timestamp_parser.py delete mode 100644 server/tests/test_timestamp_parser.py diff --git a/client/dive-common/frameTimestamp.spec.ts b/client/dive-common/frameTimestamp.spec.ts new file mode 100644 index 000000000..dd9457e6e --- /dev/null +++ b/client/dive-common/frameTimestamp.spec.ts @@ -0,0 +1,55 @@ +import type { FrameImage } from 'dive-common/apispec'; +import { attachFrameTimestamps, parseFrameTimestamp } from 'dive-common/frameTimestamp'; + +describe('parseFrameTimestamp', () => { + it('parses a YYYYMMDD_HHMMSS datestamp', () => { + expect(parseFrameTimestamp('left_20230615_143022.png')).toBe(1686839422); + }); + + it('parses a real KAMERA filename with microsecond precision', () => { + // Confirmed convention from real sample data (data/test_data). + expect(parseFrameTimestamp('kamera_calibration_fl02_C_20240407_130757.206341_ir.tif')) + .toBeCloseTo(1712495277.206341, 6); + }); + + it('parses a datestamp with a fractional-second suffix', () => { + expect(parseFrameTimestamp('left_20230615_143022.500.png')).toBe(1686839422.5); + }); + + it('parses a bare epoch-milliseconds filename', () => { + expect(parseFrameTimestamp('img_1719843225123.tif')).toBeCloseTo(1719843225.123, 6); + }); + + it('parses a bare epoch-seconds filename', () => { + expect(parseFrameTimestamp('img_1719843225.tif')).toBe(1719843225); + }); + + it('returns undefined for a plain sequential filename', () => { + expect(parseFrameTimestamp('img_00001.png')).toBeUndefined(); + }); + + it('returns undefined for a short frame-counter filename', () => { + expect(parseFrameTimestamp('frame042.tif')).toBeUndefined(); + }); + + it('returns undefined for an implausible-range digit run', () => { + expect(parseFrameTimestamp('img_0000000001.png')).toBeUndefined(); + }); + + it('is extension-agnostic (same stem, different extension)', () => { + expect(parseFrameTimestamp('left_20230615_143022.tif')) + .toBe(parseFrameTimestamp('left_20230615_143022.png')); + }); +}); + +describe('attachFrameTimestamps', () => { + it('populates timestamp in place from each frame filename', () => { + const frames: FrameImage[] = [ + { url: 'a', filename: 'left_20230615_143022.png' }, + { url: 'b', filename: 'img_00001.png' }, + ]; + attachFrameTimestamps(frames); + expect(frames[0].timestamp).toBe(1686839422); + expect(frames[1].timestamp).toBeUndefined(); + }); +}); diff --git a/client/dive-common/frameTimestamp.ts b/client/dive-common/frameTimestamp.ts new file mode 100644 index 000000000..b5a3bbda0 --- /dev/null +++ b/client/dive-common/frameTimestamp.ts @@ -0,0 +1,104 @@ +import type { FrameImage } from 'dive-common/apispec'; + +/** + * Single source of truth for parsing a frame's capture timestamp out of its + * filename (SEAL feature 5). Used by both platforms -- the desktop backend + * (platform/desktop/backend/native) when enumerating local media, and the + * web-girder API layer (platform/web-girder/api) after fetching media from + * girder -- so there is exactly one implementation to maintain. + */ + +interface FrameTimestampPattern { + name: string; + regex: RegExp; + toSeconds: (match: RegExpMatchArray) => number | undefined; +} + +/* Roughly year 2000 to year 2100, used to reject implausible epoch candidates */ +function isPlausibleEpochSeconds(seconds: number): boolean { + return seconds >= 946684800 && seconds <= 4102444800; +} + +function dateStampToSeconds(match: RegExpMatchArray): number | undefined { + const [, y, mo, d, h, mi, s, frac] = match; + const year = Number(y); + const month = Number(mo); + const day = Number(d); + const hour = Number(h); + const minute = Number(mi); + const second = Number(s); + if (month < 1 || month > 12) return undefined; + if (day < 1 || day > 31) return undefined; + if (hour > 23 || minute > 59 || second > 59) return undefined; + const millis = Date.UTC(year, month - 1, day, hour, minute, second); + const fracSeconds = frac ? Number(`0.${frac}`) : 0; + return millis / 1000 + fracSeconds; +} + +/* + * Parses a frame timestamp from a filename using a small ordered list of + * conventions. The primary convention is KAMERA's, confirmed against sample + * data (see the datestamp entry); the epoch-based patterns are fallbacks for + * other capture systems. Each entry is tried in order against the + * extension-stripped filename stem; the first regex that matches AND passes its + * own plausibility check wins. + */ +const FRAME_TIMESTAMP_PATTERNS: FrameTimestampPattern[] = [ + { + // KAMERA convention: YYYYMMDD[_-]HHMMSS with an optional fractional-second + // suffix, e.g. kamera_calibration_fl02_C_20240407_130757.206341_ir.tif + name: 'datestamp', + regex: /(? { + const seconds = Number(match[1]) / 1000; + return isPlausibleEpochSeconds(seconds) ? seconds : undefined; + }, + }, + { + // bare epoch seconds, e.g. img_1719843225.tif + name: 'epoch-seconds', + regex: /(? { + const seconds = Number(match[1]); + return isPlausibleEpochSeconds(seconds) ? seconds : undefined; + }, + }, +]; + +/** + * Frame capture timestamp, in epoch seconds, parsed from a filename. Returns + * undefined (never throws) when no recognized convention matches; callers must + * treat that as "no timestamp available". + */ +export function parseFrameTimestamp(filename: string): number | undefined { + const stem = filename.replace(/\.[^./\\]+$/, ''); + for (let i = 0; i < FRAME_TIMESTAMP_PATTERNS.length; i += 1) { + const { regex, toSeconds } = FRAME_TIMESTAMP_PATTERNS[i]; + const match = stem.match(regex); + if (match) { + const seconds = toSeconds(match); + if (seconds !== undefined) { + return seconds; + } + } + } + return undefined; +} + +/** + * Populates `timestamp` on each frame in place from its filename. Convenience + * for the web-girder API layer, which receives frames without timestamps and + * parses them client-side (the girder server no longer does this). + */ +export function attachFrameTimestamps(frames: FrameImage[]): void { + frames.forEach((frame) => { + // eslint-disable-next-line no-param-reassign + frame.timestamp = parseFrameTimestamp(frame.filename); + }); +} diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 0d0aa3d18..ede273fd5 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -54,8 +54,9 @@ import { JobType, LastCalibrationBaseName, } from 'platform/desktop/constants'; import { - cleanString, filterByGlob, makeid, parseFrameTimestamp, strNumericCompare, + cleanString, filterByGlob, makeid, strNumericCompare, } from 'platform/desktop/sharedUtils'; +import { parseFrameTimestamp } from 'dive-common/frameTimestamp'; import processTrackAttributes from './attributeProcessor'; import { upgrade } from './migrations'; diff --git a/client/platform/desktop/backend/native/multiCamUtils.ts b/client/platform/desktop/backend/native/multiCamUtils.ts index 5668deb77..f6ccf7b5d 100644 --- a/client/platform/desktop/backend/native/multiCamUtils.ts +++ b/client/platform/desktop/backend/native/multiCamUtils.ts @@ -10,7 +10,7 @@ import { JsonMeta, Settings } from 'platform/desktop/constants'; // eslint-disable-next-line import/no-cycle import { loadAnnotationFile, loadJsonMetadata, getValidatedProjectDir } from 'platform/desktop/backend/native/common'; import { serialize } from 'platform/desktop/backend/serializers/viame'; -import { parseFrameTimestamp } from 'platform/desktop/sharedUtils'; +import { parseFrameTimestamp } from 'dive-common/frameTimestamp'; /** * Figure out the destination location diff --git a/client/platform/desktop/sharedUtils.spec.ts b/client/platform/desktop/sharedUtils.spec.ts index a891b7a5f..a35a52781 100644 --- a/client/platform/desktop/sharedUtils.spec.ts +++ b/client/platform/desktop/sharedUtils.spec.ts @@ -1,7 +1,7 @@ /// import fs from 'fs-extra'; import { cloneDeep } from 'lodash'; -import { parseFrameTimestamp, strNumericCompare } from './sharedUtils'; +import { strNumericCompare } from './sharedUtils'; /** Matches tests in python utilities */ const testTuple: string[][][] = fs.readJSONSync('../testutils/imagesort.spec.json'); @@ -14,45 +14,4 @@ describe('sharedUtils', () => { expect(input.sort(strNumericCompare)).not.toEqual(copy); }); }); - - describe('parseFrameTimestamp', () => { - it('parses a YYYYMMDD_HHMMSS datestamp', () => { - expect(parseFrameTimestamp('left_20230615_143022.png')).toBe(1686839422); - }); - - it('parses a real KAMERA filename with microsecond precision', () => { - // Confirmed convention from real sample data (data/test_data). - expect(parseFrameTimestamp('kamera_calibration_fl02_C_20240407_130757.206341_ir.tif')) - .toBeCloseTo(1712495277.206341, 6); - }); - - it('parses a datestamp with a fractional-second suffix', () => { - expect(parseFrameTimestamp('left_20230615_143022.500.png')).toBe(1686839422.5); - }); - - it('parses a bare epoch-milliseconds filename', () => { - expect(parseFrameTimestamp('img_1719843225123.tif')).toBeCloseTo(1719843225.123, 6); - }); - - it('parses a bare epoch-seconds filename', () => { - expect(parseFrameTimestamp('img_1719843225.tif')).toBe(1719843225); - }); - - it('returns undefined for a plain sequential filename', () => { - expect(parseFrameTimestamp('img_00001.png')).toBeUndefined(); - }); - - it('returns undefined for a short frame-counter filename', () => { - expect(parseFrameTimestamp('frame042.tif')).toBeUndefined(); - }); - - it('returns undefined for an implausible-range digit run', () => { - expect(parseFrameTimestamp('img_0000000001.png')).toBeUndefined(); - }); - - it('is extension-agnostic (same stem, different extension)', () => { - expect(parseFrameTimestamp('left_20230615_143022.tif')) - .toBe(parseFrameTimestamp('left_20230615_143022.png')); - }); - }); }); diff --git a/client/platform/desktop/sharedUtils.ts b/client/platform/desktop/sharedUtils.ts index 288522c2b..7073d945f 100644 --- a/client/platform/desktop/sharedUtils.ts +++ b/client/platform/desktop/sharedUtils.ts @@ -64,94 +64,10 @@ function strNumericCompare(input1: string, input2: string) { throw new Error('Unreachable'); } -interface FrameTimestampPattern { - name: string; - regex: RegExp; - toSeconds: (match: RegExpMatchArray) => number | undefined; -} - -/* Roughly year 2000 to year 2100, used to reject implausible epoch candidates */ -function isPlausibleEpochSeconds(seconds: number): boolean { - return seconds >= 946684800 && seconds <= 4102444800; -} - -function dateStampToSeconds(match: RegExpMatchArray): number | undefined { - const [, y, mo, d, h, mi, s, frac] = match; - const year = Number(y); - const month = Number(mo); - const day = Number(d); - const hour = Number(h); - const minute = Number(mi); - const second = Number(s); - if (month < 1 || month > 12) return undefined; - if (day < 1 || day > 31) return undefined; - if (hour > 23 || minute > 59 || second > 59) return undefined; - const millis = Date.UTC(year, month - 1, day, hour, minute, second); - const fracSeconds = frac ? Number(`0.${frac}`) : 0; - return millis / 1000 + fracSeconds; -} - -/* - * Parses a frame timestamp from a filename using a small ordered list of - * conventions (SEAL feature 5). The primary convention is KAMERA's, confirmed - * against sample data (see the datestamp entry); the epoch-based patterns are - * fallbacks for other capture systems. Each entry is tried in order against the - * extension-stripped filename stem; the first regex that matches AND passes its - * own plausibility check wins. - */ -const FRAME_TIMESTAMP_PATTERNS: FrameTimestampPattern[] = [ - { - // KAMERA convention: YYYYMMDD[_-]HHMMSS with an optional fractional-second - // suffix, e.g. kamera_calibration_fl02_C_20240407_130757.206341_ir.tif - name: 'datestamp', - regex: /(? { - const seconds = Number(match[1]) / 1000; - return isPlausibleEpochSeconds(seconds) ? seconds : undefined; - }, - }, - { - // bare epoch seconds, e.g. img_1719843225.tif - name: 'epoch-seconds', - regex: /(? { - const seconds = Number(match[1]); - return isPlausibleEpochSeconds(seconds) ? seconds : undefined; - }, - }, -]; - -/** - * Frame capture timestamp, in epoch seconds, parsed from a filename. Returns - * undefined (never throws) when no recognized convention matches; callers must - * treat that as "no timestamp available". - */ -function parseFrameTimestamp(filename: string): number | undefined { - const stem = filename.replace(/\.[^./\\]+$/, ''); - for (let i = 0; i < FRAME_TIMESTAMP_PATTERNS.length; i += 1) { - const { regex, toSeconds } = FRAME_TIMESTAMP_PATTERNS[i]; - const match = stem.match(regex); - if (match) { - const seconds = toSeconds(match); - if (seconds !== undefined) { - return seconds; - } - } - } - return undefined; -} - export { cleanString, filterByGlob, makeid, - parseFrameTimestamp, strChunks, strNumericCompare, }; diff --git a/client/platform/web-girder/api/dataset.service.ts b/client/platform/web-girder/api/dataset.service.ts index c2503d493..5180808a1 100644 --- a/client/platform/web-girder/api/dataset.service.ts +++ b/client/platform/web-girder/api/dataset.service.ts @@ -4,6 +4,7 @@ import { DatasetMetaMutable, FrameImage, SaveAttributeArgs, SaveAttributeTrackFilterArgs, } from 'dive-common/apispec'; import { calibrationFileMarker, jsonCalibrationFileMarker } from 'dive-common/constants'; +import { attachFrameTimestamps } from 'dive-common/frameTimestamp'; import { parentDatasetId } from 'dive-common/compositeDatasetId'; import { isStereoCalibrationFileName } from 'dive-common/stereoParentFolder'; import { GirderMetadataStatic } from 'platform/web-girder/constants'; @@ -21,6 +22,12 @@ async function getDataset(datasetId: string) { if (compositeId) { response.data.id = compositeId; } + // Parse per-frame capture timestamps client-side (single shared implementation + // with desktop; see dive-common/frameTimestamp.ts). The girder server no longer + // does this, so multicam per-camera frames arrive without timestamps. + Object.values(response.data.multiCamMedia?.cameras ?? {}).forEach( + (camera) => attachFrameTimestamps(camera.imageData), + ); return response; } @@ -61,7 +68,10 @@ export interface DatasetSourceMedia { async function getDatasetMedia(datasetId: string) { const { folderId } = await resolveDatasetFolderId(datasetId); - return girderRest.get(`dive_dataset/${folderId}/media`); + const response = await girderRest.get(`dive_dataset/${folderId}/media`); + // Parse per-frame capture timestamps client-side (see getDataset above). + attachFrameTimestamps(response.data.imageData ?? []); + return response; } function clone({ diff --git a/server/dive_server/crud_dataset.py b/server/dive_server/crud_dataset.py index c514b85cf..e3912c805 100644 --- a/server/dive_server/crud_dataset.py +++ b/server/dive_server/crud_dataset.py @@ -23,7 +23,6 @@ constants, fromMeta, models, - timestamp_parser, types, ) from dive_utils.serializers import kwcoco @@ -366,7 +365,6 @@ def get_media( id=str(image["_id"]), url=get_url(dsFolder, image), filename=image['name'], - timestamp=timestamp_parser.parse_frame_timestamp(image['name']), ) for image in crud.valid_images(dsFolder, user) ] diff --git a/server/dive_utils/models.py b/server/dive_utils/models.py index 227d572c7..75b33383a 100644 --- a/server/dive_utils/models.py +++ b/server/dive_utils/models.py @@ -260,9 +260,6 @@ class MediaResource(BaseModel): url: str id: str filename: str - # Best-effort capture timestamp (epoch seconds) parsed from the filename. - # See dive_utils.timestamp_parser -- absent when no known convention matches. - timestamp: Optional[float] = None class MultiCamCameraMeta(BaseModel): diff --git a/server/dive_utils/timestamp_parser.py b/server/dive_utils/timestamp_parser.py deleted file mode 100644 index 4b58af0e5..000000000 --- a/server/dive_utils/timestamp_parser.py +++ /dev/null @@ -1,111 +0,0 @@ -""" -Frame capture timestamp parsing from a filename, using a small ordered list of -conventions (SEAL feature 5). - -The primary convention is KAMERA's, confirmed against sample data: a -YYYYMMDD_HHMMSS.ffffff datestamp embedded in names like -kamera_calibration_fl02_C_20240407_130757.206341_ir.tif. The remaining -epoch-based patterns are fallbacks for other capture systems; extend the list -as additional conventions surface. - -Mirrors client/platform/desktop/sharedUtils.ts's parseFrameTimestamp -- keep the -two implementations in sync. -""" - -import re -from datetime import datetime, timezone -from pathlib import Path -from typing import Callable, List, Match, NamedTuple, Optional - -# Roughly year 2000 to year 2100, used to reject implausible epoch candidates -_MIN_PLAUSIBLE_EPOCH_SECONDS = 946684800 -_MAX_PLAUSIBLE_EPOCH_SECONDS = 4102444800 - - -def _is_plausible_epoch_seconds(seconds: float) -> bool: - return _MIN_PLAUSIBLE_EPOCH_SECONDS <= seconds <= _MAX_PLAUSIBLE_EPOCH_SECONDS - - -def _datestamp_to_seconds(match: Match) -> Optional[float]: - year, month, day, hour, minute, second, frac = match.groups() - month_i, day_i, hour_i, minute_i, second_i = ( - int(month), - int(day), - int(hour), - int(minute), - int(second), - ) - if month_i < 1 or month_i > 12: - return None - if day_i < 1 or day_i > 31: - return None - if hour_i > 23 or minute_i > 59 or second_i > 59: - return None - try: - dt = datetime( - int(year), month_i, day_i, hour_i, minute_i, second_i, tzinfo=timezone.utc - ) - except ValueError: - # e.g. Feb 30 -- a valid-looking but nonexistent calendar date - return None - frac_seconds = float(f'0.{frac}') if frac else 0.0 - return dt.timestamp() + frac_seconds - - -def _epoch_millis_to_seconds(match: Match) -> Optional[float]: - seconds = int(match.group(1)) / 1000 - return seconds if _is_plausible_epoch_seconds(seconds) else None - - -def _epoch_seconds_to_seconds(match: Match) -> Optional[float]: - seconds = float(match.group(1)) - return seconds if _is_plausible_epoch_seconds(seconds) else None - - -class _FrameTimestampPattern(NamedTuple): - name: str - regex: re.Pattern - to_seconds: Callable[[Match], Optional[float]] - - -# Tried in order against the extension-stripped filename stem; the first regex -# that matches AND passes its own plausibility check wins. -_FRAME_TIMESTAMP_PATTERNS: List[_FrameTimestampPattern] = [ - _FrameTimestampPattern( - # KAMERA convention: YYYYMMDD[_-]HHMMSS with an optional fractional-second - # suffix, e.g. kamera_calibration_fl02_C_20240407_130757.206341_ir.tif - name='datestamp', - regex=re.compile( - r'(? Optional[float]: - """ - Frame capture timestamp, in epoch seconds, parsed from a filename. Returns - None (never raises) when no recognized convention matches; callers must - treat that as "no timestamp available". - """ - stem = Path(filename).stem - for pattern in _FRAME_TIMESTAMP_PATTERNS: - match = pattern.regex.search(stem) - if match: - seconds = pattern.to_seconds(match) - if seconds is not None: - return seconds - return None diff --git a/server/tests/test_multicam_dataset.py b/server/tests/test_multicam_dataset.py index 50932098a..0f31b6966 100644 --- a/server/tests/test_multicam_dataset.py +++ b/server/tests/test_multicam_dataset.py @@ -109,7 +109,6 @@ def load_folder(folder_id, level=None, user=None): id='img-left', url='/api/v1/.../left.png', filename='left_20230615_143022.png', - timestamp=1686839422.0, ) right_image = MediaResource(id='img-right', url='/api/v1/.../right.png', filename='right.png') @@ -130,8 +129,6 @@ def media_for_child(child_folder, child_user): assert result.multiCamMedia.cameraOrder == ['left', 'right'] assert set(result.multiCamMedia.cameras.keys()) == {'left', 'right'} assert result.multiCamMedia.cameras['left'].imageData[0].filename == 'left_20230615_143022.png' - assert result.multiCamMedia.cameras['left'].imageData[0].timestamp == 1686839422.0 - assert result.multiCamMedia.cameras['right'].imageData[0].timestamp is None assert 'multiCam' not in result.dict() @@ -147,22 +144,6 @@ def test_get_media_multi_parent_returns_empty(_verify): assert result.sourceVideo is None -@patch('dive_server.crud_dataset.crud.valid_images') -@patch('dive_server.crud_dataset.crud.verify_dataset') -def test_get_media_image_sequence_parses_frame_timestamps(_verify, valid_images_mock): - folder = _child_folder('left-id', 'left') - user = {'login': 'tester'} - valid_images_mock.return_value = [ - {'_id': 'img-parseable', 'name': 'left_20230615_143022.png'}, - {'_id': 'img-unparseable', 'name': 'left_00001.png'}, - ] - - result = crud_dataset.get_media(folder, user) - - assert result.imageData[0].timestamp == 1686839422.0 - assert result.imageData[1].timestamp is None - - @patch('dive_server.crud_dataset.get_media') @patch('dive_server.crud_dataset.Folder') def test_get_multi_cam_media_missing_child_raises(folder_cls, get_media_mock): diff --git a/server/tests/test_timestamp_parser.py b/server/tests/test_timestamp_parser.py deleted file mode 100644 index c220ac449..000000000 --- a/server/tests/test_timestamp_parser.py +++ /dev/null @@ -1,49 +0,0 @@ -import pytest - -from dive_utils import timestamp_parser - - -def test_parses_datestamp(): - assert timestamp_parser.parse_frame_timestamp('left_20230615_143022.png') == 1686839422.0 - - -def test_parses_kamera_filename_with_microseconds(): - # Confirmed KAMERA convention from real sample data (data/test_data): a - # YYYYMMDD_HHMMSS.ffffff datestamp with microsecond precision. - assert timestamp_parser.parse_frame_timestamp( - 'kamera_calibration_fl02_C_20240407_130757.206341_ir.tif' - ) == pytest.approx(1712495277.206341) - - -def test_parses_datestamp_with_fractional_seconds(): - assert timestamp_parser.parse_frame_timestamp('left_20230615_143022.500.png') == pytest.approx( - 1686839422.5 - ) - - -def test_parses_bare_epoch_millis(): - assert timestamp_parser.parse_frame_timestamp('img_1719843225123.tif') == pytest.approx( - 1719843225.123 - ) - - -def test_parses_bare_epoch_seconds(): - assert timestamp_parser.parse_frame_timestamp('img_1719843225.tif') == 1719843225.0 - - -def test_returns_none_for_plain_sequential_filename(): - assert timestamp_parser.parse_frame_timestamp('img_00001.png') is None - - -def test_returns_none_for_short_frame_counter_filename(): - assert timestamp_parser.parse_frame_timestamp('frame042.tif') is None - - -def test_returns_none_for_implausible_range_digit_run(): - assert timestamp_parser.parse_frame_timestamp('img_0000000001.png') is None - - -def test_is_extension_agnostic(): - tif = timestamp_parser.parse_frame_timestamp('left_20230615_143022.tif') - png = timestamp_parser.parse_frame_timestamp('left_20230615_143022.png') - assert tif == png From 7443347922e9c9f3ae75533a5db61360b5555a02 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Tue, 7 Jul 2026 12:36:52 -0400 Subject: [PATCH 14/16] Shrink alignment to 0s, from 0.5s tolerance --- client/dive-common/alignedTimeline.spec.ts | 8 ++++---- client/dive-common/alignedTimeline.ts | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/client/dive-common/alignedTimeline.spec.ts b/client/dive-common/alignedTimeline.spec.ts index 25a2f39c6..d0d0520fb 100644 --- a/client/dive-common/alignedTimeline.spec.ts +++ b/client/dive-common/alignedTimeline.spec.ts @@ -34,10 +34,10 @@ describe('alignedTimeline', () => { expect(buildAlignedTimeline(camerasFrames)).toEqual({ aligned: false }); }); - it('resolves a genuine out-of-tolerance gap on one camera to undefined at that slot', () => { + it('resolves a genuinely missing frame on one camera to undefined at that slot', () => { const camerasFrames = { A: [frame(100), frame(200), frame(300)], - B: [frame(100.1), frame(300.2)], + B: [frame(100), frame(300)], }; const result = buildAlignedTimeline(camerasFrames); expect(result).toEqual({ @@ -132,7 +132,7 @@ describe('alignedTimeline', () => { it('flags slots where any camera is missing a frame', () => { const camerasFrames = { A: [frame(100), frame(200), frame(300)], - B: [frame(100.1), frame(300.2)], + B: [frame(100), frame(300)], }; const result = buildAlignedTimeline(camerasFrames); expect(result.aligned).toBe(true); @@ -193,7 +193,7 @@ describe('alignedTimeline', () => { it('maps each camera\'s local frame back to its global slot', () => { const camerasFrames = { A: [frame(100), frame(200), frame(300)], - B: [frame(100.1), frame(300.2)], + B: [frame(100), frame(300)], }; const result = buildAlignedTimeline(camerasFrames); expect(result.aligned).toBe(true); diff --git a/client/dive-common/alignedTimeline.ts b/client/dive-common/alignedTimeline.ts index cc7ba957f..88307e34a 100644 --- a/client/dive-common/alignedTimeline.ts +++ b/client/dive-common/alignedTimeline.ts @@ -3,11 +3,11 @@ import type { FrameImage } from './apispec'; /** * Max time delta, in seconds, between two cameras' frame timestamps for them to * be considered "the same instant" (SEAL feature 5). In KAMERA sample data the - * C/L/R cameras share an identical capture timestamp per shot, so this mainly - * has to absorb sub-second jitter/rounding; 0.5s leaves margin for less tightly - * synchronized full-flight captures. Retune if real cadence differs. + * C/L/R cameras share an identical capture timestamp per shot, so for now this + * requires an exact match (0). Widen it only if real captures turn out to have + * sub-second jitter that needs absorbing. */ -export const FRAME_ALIGNMENT_TOLERANCE_SECONDS = 0.5; +export const FRAME_ALIGNMENT_TOLERANCE_SECONDS = 0; /** * camera name -> local index into that camera's own FrameImage[], or undefined From 918fac42c05ae682659e60bb783aa2befbd6d88c Mon Sep 17 00:00:00 2001 From: romleiaj Date: Tue, 7 Jul 2026 13:03:41 -0400 Subject: [PATCH 15/16] Update tests - matching intended metadata behavior --- server/tests/test_create_multicam.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/server/tests/test_create_multicam.py b/server/tests/test_create_multicam.py index a1bc72e8c..896fd8762 100644 --- a/server/tests/test_create_multicam.py +++ b/server/tests/test_create_multicam.py @@ -69,13 +69,20 @@ def load_folder(folder_id, level=None, user=None): assert set(saved_meta[constants.MultiCamMarker]['cameras'].keys()) == {'left', 'right'} +@patch('dive_server.crud_dataset.Item') +@patch('dive_server.crud_dataset.crud.get_or_create_auxiliary_folder') @patch('dive_server.crud_dataset.crud.valid_images') @patch('dive_server.crud_dataset.Folder') @patch('dive_server.crud_dataset.crud.verify_dataset') -def test_create_multicam_rejects_mismatched_frame_counts(_verify, folder_cls, valid_images_mock): +def test_create_multicam_accepts_mismatched_frame_counts( + _verify, folder_cls, valid_images_mock, _aux, item_cls +): + """Cameras with differing frame counts are paired downstream by frame alignment, + so create_multicam must not reject them.""" user = {'login': 'tester'} - left = _child_folder('left-id', 'cam-left') - right = _child_folder('right-id', 'cam-right') + dataset_parent = _dataset_parent() + left = _child_folder('left-id', 'left') + right = _child_folder('right-id', 'right') folder_cls.return_value.load.side_effect = lambda fid, **kwargs: { 'left-id': left, @@ -89,14 +96,18 @@ def test_create_multicam_rejects_mismatched_frame_counts(_verify, folder_cls, va 'type': 'image-sequence', 'subType': 'stereo', 'defaultDisplay': 'left', + 'cameraOrder': ['left', 'right'], 'cameras': { 'left': {'folderId': 'left-id'}, 'right': {'folderId': 'right-id'}, }, } - with pytest.raises(RestException, match='same number of frames'): - crud_dataset.create_multicam(user, _dataset_parent(), data) + result = crud_dataset.create_multicam(user, dataset_parent, data) + + assert result == dataset_parent + saved_meta = folder_cls.return_value.save.call_args_list[-1][0][0]['meta'] + assert saved_meta[constants.TypeMarker] == constants.MultiType @patch('dive_server.crud_dataset.Item') From e2405c830b5ff7e367581f1559fbbfac831027fd Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Wed, 8 Jul 2026 14:30:46 -0400 Subject: [PATCH 16/16] documentation update --- docs/DataFormats.md | 16 +++++++++++++ docs/Large-Image-Support.md | 3 +++ docs/Multicamera-data.md | 46 ++++++++++++++++++++++++++++++++++++- docs/UI-Timeline.md | 8 ++++++- docs/Web-Version.md | 5 ++-- docs/index.md | 1 + 6 files changed, 75 insertions(+), 4 deletions(-) diff --git a/docs/DataFormats.md b/docs/DataFormats.md index d825c6fea..0503ed752 100644 --- a/docs/DataFormats.md +++ b/docs/DataFormats.md @@ -190,6 +190,22 @@ interface DatasetMetaMutable { sharpen, and optional percentile stretch bounds). See [Image Enhancements](UI-Image-Enhancements.md) for platform support of high bit-depth stretch. +### Media frame metadata + +Each frame in an image-sequence or multicam dataset may carry a `timestamp` field (epoch +seconds) parsed from the filename at load time. When every frame on every camera in a +multicam dataset has a timestamp, DIVE builds a global aligned timeline for playback. See +[Aligned playback and timestamps](Multicamera-data.md#aligned-playback-and-timestamps). + +```typescript +interface FrameImage { + url: string; + filename: string; + id?: string; // large-image item id (web tiled TIFF) + timestamp?: number; // capture time in epoch seconds, when parseable from filename +} +``` + ## VIAME CSV Read the [VIAME CSV Specification](https://viame.readthedocs.io/en/latest/sections/detection_file_conversions.html). diff --git a/docs/Large-Image-Support.md b/docs/Large-Image-Support.md index 90941904c..e33bb81ce 100644 --- a/docs/Large-Image-Support.md +++ b/docs/Large-Image-Support.md @@ -15,6 +15,9 @@ file first. Pre-scaling to 8-bit COG (below) is still useful when you want a fixed display range at import time or when preparing files for tools that do not support dynamic tile styling. +On **Web**, an upload batch that contains only `.tif` / `.tiff` files is classified as a +`large-image` dataset (not an image sequence). JPEG or PNG uploads remain image sequences. + ## Requirement: Internal Overviews Large images must include internal overview levels (reduced-resolution pyramid levels) for tile rendering. Files without internal overviews can fail to load in the large-image viewer. diff --git a/docs/Multicamera-data.md b/docs/Multicamera-data.md index 2c97197d7..c6534bb46 100644 --- a/docs/Multicamera-data.md +++ b/docs/Multicamera-data.md @@ -7,6 +7,7 @@ DIVE supports **multicamera** and **stereo** datasets on both the [web version]( | Import stereo (2 cameras + calibration) | ✔️ | ✔️ | | Import multicam (2 or 3 cameras) | ✔️ | ✔️ | | Batch multicam import (collect folders) | ✔️ | ✔️ | +| Timestamp-aligned multicam playback | ✔️ | ✔️ | | View and annotate across cameras | ✔️ | ✔️ | | MultiCamera Tools (link/unlink tracks) | ✔️ | ✔️ | | Run stereo / multicam VIAME pipelines | ✔️ | ✔️ | @@ -32,7 +33,7 @@ Multicam import is available from the standard upload dialog on [viame.kitware.c * ==:material-binoculars: Stereoscopic== — exactly 2 cameras and a calibration `.npz` file. * ==:material-camera-burst: MultiCam== — 2 or 3 cameras; no calibration file required. * ==:material-folder-multiple-image: MultiCam Batch== — import many multicam datasets at once from a folder of **collect** subfolders (image sequences only). See [Batch multicam import](#batch-multicam-import). -6. In the import dialog, assign a source folder or video file to each camera. All cameras must use the same media type (all image sequences or all videos) and must have the same number of frames (or matching video duration). +6. In the import dialog, assign a source folder or video file to each camera. All cameras must use the same media type (all image sequences or all videos). By default, every camera must have the same number of frames (or matching video duration). For **image-sequence** imports with capture timestamps in filenames, enable **Infer frame index from filename** to allow unequal per-camera counts — see [Infer frame index from filename](#infer-frame-index-from-filename). 7. Optionally attach a per-camera annotation file during import. 8. Enter a dataset name, choose the default display camera, and click ==Begin Import==. 9. When upload finishes, DIVE opens the new multicam dataset in the annotator. @@ -100,6 +101,49 @@ On **Web**, the folder picker uploads all files under the chosen root; scanning For a single multicam dataset from one parent folder (one collect, camera subfolders only), use ==MultiCam== with the **parent-folder** import mode instead of MultiCam Batch. +### Infer frame index from filename + +At the bottom of the multicam import dialog (image sequences only), **Infer frame index from filename** relaxes the equal-frame-count requirement when filenames encode capture time. Enable it for datasets where cameras may have different numbers of frames — for example when one camera dropped occasional shots but each surviving frame still carries a parseable timestamp. + +When enabled, DIVE skips the import-time check that every camera has the same image count. During playback, frames are aligned by those filename timestamps rather than by positional index (see [Aligned playback and timestamps](#aligned-playback-and-timestamps)). + +Example filename (KAMERA-style datestamp): + +``` +test_seattle_2020_fl09_C_20200830_020814.141365_rgb.jpg +``` + +This option is available on both Web and Desktop for ==MultiCam== imports. It does not apply to video or stereo imports. + +### Aligned playback and timestamps + +On multicam **image-sequence** datasets, DIVE can synchronize playback across cameras using capture timestamps parsed from each frame's filename. When alignment is active, scrubbing and playback use a single **global timeline** whose slots group frames that share the same capture instant across cameras. + +**When alignment activates:** + +* The dataset has **two or three** cameras. +* **Every** frame on **every** camera has a parseable timestamp in its filename. + +If any frame on any camera lacks a timestamp, DIVE falls back to **positional alignment** — each camera advances by its own local frame index, as in earlier DIVE versions. + +**Supported filename conventions** (first match wins): + +| Pattern | Example | +|---------|---------| +| Datestamp `YYYYMMDD[_-]HHMMSS` with optional fractional seconds | `kamera_fl02_C_20240407_130757.206341_ir.tif` | +| Epoch milliseconds (13 digits) | `img_1719843225123.tif` | +| Epoch seconds (10 digits) | `img_1719843225.tif` | + +Timestamps are parsed automatically when media loads; no import setting is required beyond having recognizable filenames (and, if counts differ, enabling **Infer frame index from filename** at import time). + +**What you see in the viewer:** + +* The timeline **frame counter** reflects the global aligned slot index, not an individual camera's local index. +* When the current slot has no frame for a camera, that camera's pane **blanks** and its annotation overlay clears until you scrub to a slot where that camera has data. +* Red bands on the [timeline scrubber](UI-Timeline.md#multicam-gap-indicators) mark slots where at least one camera is missing a frame. + +Identical capture timestamps on every camera (common in synchronized rigs) are grouped into the same slot. If one camera has extra frames at the same timestamp, the surplus frames spill into later slots rather than overwriting the first match. + ## Data/Track Organization Data is loaded amongst multiple folders to create a multicamera dataset. In these cases trackIds will be linked if they are the same across the cameras. Selection of a trackId that exists across multiple cameras will be linked together in the [Track List](UI-Track-List.md). diff --git a/docs/UI-Timeline.md b/docs/UI-Timeline.md index 693d38e1c..62414e3d5 100644 --- a/docs/UI-Timeline.md +++ b/docs/UI-Timeline.md @@ -18,7 +18,13 @@ The timeline provides a control bar and a few different temporal visualizations. * ==:material-lock-open:== will enable camera lock, which causes the annotation view to auto-zoom and pan to whatever annotation is currently selected. This is useful when reviewing the output of a pipeline. * Hovering over the camera lock will open additional settings for forcing transition and locking to a zoomed in multiple of the bbox size * ==:material-image-filter-center-focus:== or the ++r++ key will reset zoom/pan in the annotation view. -* ==:material-contrast-box:== will open the image contrast adjustment panel. +* ==:material-contrast-box:== will open the [Image Enhancements](UI-Image-Enhancements.md) panel. + +## Multicam gap indicators + +On multicam image-sequence datasets where [timestamp-aligned playback](Multicamera-data.md#aligned-playback-and-timestamps) is active, the frame scrubber may show **red bands** beneath the slider track. Each band marks a global timeline slot where at least one camera has no frame at that instant — scrubbing there blanks that camera's pane. + +Hover the scrubber to see a tooltip with the count of such gap slots. Consecutive gap slots are merged into a single visible band on long sequences. ## Detection/Track Count diff --git a/docs/Web-Version.md b/docs/Web-Version.md index 7760d3575..86bd508e3 100644 --- a/docs/Web-Version.md +++ b/docs/Web-Version.md @@ -27,6 +27,7 @@ A user account is required to store data and run pipelines on viame.kitware.com. * Select a video or multi-select a group of image frames. * Use ++ctrl++ or ++shift++ to click every file you want to upload. * If you already have `annotations.csv` or an annotation or configuration JSON select that too. + * Uploads that contain only `.tif` / `.tiff` files are classified as [large-image](Large-Image-Support.md) datasets rather than image sequences. * Choose a name for the dataset and enter the optional playback frame rate or select other optional files. * Press ==Start Upload== * In the data browser, a new ==Launch Annotator== button will appear next to your data @@ -59,9 +60,9 @@ DIVE Web supports importing **stereo** (2 cameras + calibration) and **multicam* * ==:material-folder-multiple-image: MultiCam Batch== — import **many** multicam image-sequence datasets from a root folder of collect subfolders (see [Batch multicam import](Multicamera-data.md#batch-multicam-import)). 5. For Stereoscopic or MultiCam, assign media to each camera, set a dataset name and default display camera, then start the import. For MultiCam Batch, choose the root folder, review the scan table, edit dataset names, select collects, and start the batch. -All cameras in one import must share the same media type and frame count. Stereoscopic imports require a calibration `.npz` file. MultiCam Batch is **image-sequence only** (not video or stereo). +All cameras in one import must share the same media type. By default, every camera must have the same frame count (or matching video duration). For image-sequence ==MultiCam== imports with capture timestamps in filenames, enable **Infer frame index from filename** in the import dialog to allow unequal per-camera counts — see [Multicamera and Stereo Data](Multicamera-data.md#infer-frame-index-from-filename). Stereoscopic imports require a calibration `.npz` file. MultiCam Batch is **image-sequence only** (not video or stereo). -For camera selection, linked tracks, MultiCamera Tools, and pipeline details, see [Multicamera and Stereo Data](Multicamera-data.md). +For camera selection, linked tracks, MultiCamera Tools, timestamp-aligned playback, and pipeline details, see [Multicamera and Stereo Data](Multicamera-data.md). !!! note diff --git a/docs/index.md b/docs/index.md index 1714c3291..d4946331b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -22,6 +22,7 @@ Load your own images and videos | ✔️ | ✔️ |     Import using image lists | ❌ | ✔️ Multicamera and stereo datasets | ✔️ | ✔️ |     Batch multicam import (collect folders) | ✔️ | ✔️ +|     Timestamp-aligned multicam playback | ✔️ | ✔️ Load annotations from [supported formats](DataFormats.md) | ✔️ | ✔️ Create new object and track annotation | ✔️ | ✔️ Annotation export | ✔️ | ✔️