diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 78c40b8ec..354fc3656 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -125,6 +125,27 @@ interface FrameImage { id?: string; } +/** Compact resolved frame-metadata payload. Rows are aligned to `columns[camera]`. */ +interface ResolvedFrameMetadata { + /** camera key -> frame number -> cell values aligned to `columns[camera]`. */ + cameras: Record>; + /** camera key -> matched sidecar filenames in precedence order. */ + sources: Record; + /** camera key -> payload column names in source/merge order. */ + columns: Record; +} + +/** One declared frame-metadata sidecar source loaded by a platform implementation. */ +interface FrameMetadataSourceText { + name: string; + text: string; +} + +/** Declared frame-metadata sidecar text per camera, in precedence order. */ +interface FrameMetadataSourcesResponse { + cameras: Record; +} + export interface MultiCamImportFolderArgs { datasetName?: string; // Girder parent folder name (required on web) defaultDisplay: string; // In multicam the default camera to display @@ -169,6 +190,10 @@ interface MediaImportResponse { globPattern: string; mediaConvertList: string[]; } + +/** User-editable datasetInfo stored on the dataset's backing metadata object. */ +type DatasetInfoFields = Record; + /** * The parts of metadata a user should be able to modify. */ @@ -180,7 +205,7 @@ interface DatasetMetaMutable { imageEnhancements?: ImageEnhancements; attributes?: Readonly>; attributeTrackFilters?: Readonly>; - datasetInfo?: Record; + datasetInfo?: DatasetInfoFields; error?: string; } const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo']; @@ -266,6 +291,7 @@ interface Api { loadMetadata(datasetId: string): Promise; loadDetections(datasetId: string, revision?: number, set?: string): Promise; + loadFrameMetadata?(datasetId: string): Promise; saveDetections(datasetId: string, args: SaveDetectionsArgs): Promise; saveMetadata(datasetId: string, metadata: DatasetMetaMutable): Promise; @@ -502,6 +528,7 @@ export { AnnotationSchema, Api, DatasetMeta, + DatasetInfoFields, DatasetMetaMutable, DatasetMetaMutableKeys, DatasetType, @@ -512,6 +539,9 @@ export { SubType, PipelineParamType, FrameImage, + ResolvedFrameMetadata, + FrameMetadataSourceText, + FrameMetadataSourcesResponse, MultiTrackRecord, MultiGroupRecord, Pipe, diff --git a/client/dive-common/components/DatasetInfo.vue b/client/dive-common/components/DatasetInfo.vue deleted file mode 100644 index aa3f4e23a..000000000 --- a/client/dive-common/components/DatasetInfo.vue +++ /dev/null @@ -1,324 +0,0 @@ - - - - - diff --git a/client/dive-common/components/DatasetInfo/CustomDatasetInfoPanel.vue b/client/dive-common/components/DatasetInfo/CustomDatasetInfoPanel.vue new file mode 100644 index 000000000..0f94f85dc --- /dev/null +++ b/client/dive-common/components/DatasetInfo/CustomDatasetInfoPanel.vue @@ -0,0 +1,227 @@ + + + + + diff --git a/client/dive-common/components/DatasetInfo/DatasetInfo.spec.ts b/client/dive-common/components/DatasetInfo/DatasetInfo.spec.ts new file mode 100644 index 000000000..a40b2702e --- /dev/null +++ b/client/dive-common/components/DatasetInfo/DatasetInfo.spec.ts @@ -0,0 +1,330 @@ +// @vitest-environment jsdom +/* eslint-disable import/no-extraneous-dependencies */ +import { mount } from '@vue/test-utils'; +import Vue, { + CreateElement, defineComponent, nextTick, ref, +} from 'vue'; + +import { + beforeEach, describe, expect, it, vi, +} from 'vitest'; + +import { + DatasetMeta, + provideApi, +} from 'dive-common/apispec'; +import type { FrameMetadataSourcesResponse } from 'dive-common/apispec'; +import { + __resetFrameMetadataSessionCache, +} from 'dive-common/use/useFrameMetadata'; +import { + dummyHandler, + dummyState, + provideAnnotator, +} from 'vue-media-annotator/provides'; +import { useMediaController } from 'vue-media-annotator/components'; +import DatasetInfo from './DatasetInfo.vue'; + +Vue.config.ignoredElements = [/^v-/]; + +const VTooltipStub = { + render(this: Vue, h: CreateElement) { + return h('div', [this.$scopedSlots.activator?.({ on: {}, attrs: {} })]); + }, +}; + +async function settleOnce() { + await nextTick(); + await new Promise((resolve) => { window.setTimeout(resolve, 0); }); +} + +async function settle() { + await settleOnce(); + await settleOnce(); + await settleOnce(); + await settleOnce(); +} + +const defaultMetadata: DatasetMeta = { + id: 'dataset-id', + imageData: [], + videoUrl: undefined, + type: 'image-sequence', + fps: 5, + name: 'Mouss Set', + createdAt: '2024-01-02T03:04:05.000Z', + originalFps: 10, + subType: null, + multiCamMedia: null, + datasetInfo: { + cruise: '2403', + station: 'TXN-012', + }, +}; + +function apiWithMetadata({ + metadata, + frameMetadata, + frameMetadataSupported, + saveMetadata, +}: { + metadata: DatasetMeta; + frameMetadata: FrameMetadataSourcesResponse; + frameMetadataSupported: boolean; + saveMetadata: Parameters[0]['saveMetadata']; +}): Parameters[0] { + return { + getPipelineList: async () => ({}), + runPipeline: async () => undefined, + deleteTrainedPipeline: async () => undefined, + exportTrainedPipeline: async () => undefined, + getDatasetCalibration: async () => null, + getTrainingConfigurations: async () => ({ training: { configs: [], default: '' }, models: {} }), + runTraining: async () => undefined, + loadMetadata: vi.fn(async () => metadata), + loadDetections: async () => ({ + version: 2, + tracks: [], + groups: [], + sets: [], + }), + ...(frameMetadataSupported ? { + loadFrameMetadata: vi.fn(async () => frameMetadata), + } : {}), + saveDetections: async () => undefined, + saveMetadata, + saveAttributes: async () => undefined, + saveAttributeTrackFilters: async () => undefined, + openFromDisk: async () => ({ canceled: true, filePaths: [] }), + importAnnotationFile: async () => false, + }; +} + +function mountDatasetInfo({ + frameMetadata = { cameras: {} }, + mediaNames = { port: ['port001.png'], starboard: ['starboard001.png'] }, + selectedCamera = 'port', + frame = 0, + readOnlyMode = true, + metadata = defaultMetadata, + frameMetadataSupported = true, +}: { + frameMetadata?: FrameMetadataSourcesResponse; + mediaNames?: Record; + selectedCamera?: string; + frame?: number; + readOnlyMode?: boolean; + metadata?: DatasetMeta; + frameMetadataSupported?: boolean; +} = {}) { + const state = dummyState(); + state.datasetId = ref('dataset-id'); + state.selectedCamera = ref(selectedCamera); + state.time = { + ...state.time, + frame: ref(frame), + }; + state.readOnlyMode = ref(readOnlyMode); + + const saveMetadata = vi.fn(async () => undefined); + const api = apiWithMetadata({ + metadata, + frameMetadata, + frameMetadataSupported, + saveMetadata, + }); + + const Root = defineComponent({ + components: { DatasetInfo }, + setup() { + provideApi(api); + const { initialize } = useMediaController(); + Object.entries(mediaNames).forEach(([camera, names]) => { + if (names === undefined) { + return; + } + const { state: cameraState } = initialize(camera, { + seek: () => undefined, + play: () => undefined, + pause: () => undefined, + setVolume: () => undefined, + setSpeed: () => undefined, + }); + cameraState.filenames = names; + }); + provideAnnotator( + state, + dummyHandler(() => undefined), + {} as Parameters[2], + ); + return {}; + }, + template: '', + }); + + return { + wrapper: mount(Root, { + stubs: { + DatasetInfoFieldDialog: true, + 'v-tooltip': VTooltipStub, + }, + }), + state, + loadFrameMetadata: api.loadFrameMetadata, + }; +} + +function frameMetadataRows(wrapper: ReturnType) { + return wrapper.findAll('.frame-metadata-row').wrappers.map((row) => ({ + key: row.find('.frame-metadata-key').text(), + value: row.find('.frame-metadata-value').element.textContent, + })); +} + +describe('DatasetInfo', () => { + beforeEach(() => { + __resetFrameMetadataSessionCache(); + }); + + it('renders the useful frame, dataset, and custom dataset info in order', async () => { + const { wrapper } = mountDatasetInfo({ + frameMetadata: { + cameras: { + port: [{ + name: 'frame_metadata.csv', + text: 'filename,latitude,depth_m,note\nport001.png,58.10,100,raw text\n', + }], + }, + }, + }); + + await settle(); + + const headers = wrapper.findAll('.dataset-info-panel-header').wrappers; + expect(headers.map((header) => header.text())).toEqual([ + expect.stringContaining('Frame Metadata'), + expect.stringContaining('Dataset Info'), + expect.stringContaining('Custom Dataset Info'), + ]); + expect(frameMetadataRows(wrapper)).toEqual([ + { key: 'filename', value: 'port001.png' }, + { key: 'latitude', value: '58.10' }, + { key: 'depth_m', value: '100' }, + { key: 'note', value: 'raw text' }, + ]); + + const text = wrapper.text(); + expect(text.indexOf('Frame Metadata')).toBeLessThan(text.indexOf('Dataset Info')); + expect(text.indexOf('Dataset Info')).toBeLessThan(text.indexOf('Custom Dataset Info')); + expect(wrapper.find('.frame-metadata-source-icon').attributes('aria-label')) + .toBe('Source: frame_metadata.csv'); + expect(wrapper.find('.dataset-info-section').text()).toContain('Mouss Set'); + expect(wrapper.find('.dataset-info-section').text()).toContain('image-sequence'); + expect(wrapper.find('.custom-dataset-info-section').text()).toContain('cruise'); + expect(wrapper.find('.custom-dataset-info-section').text()).toContain('2403'); + }); + + it('keeps frame metadata read-only when custom dataset info is editable', async () => { + const { wrapper } = mountDatasetInfo({ + readOnlyMode: false, + frameMetadata: { + cameras: { + port: [{ + name: 'frame_metadata.csv', + text: 'filename,latitude\nport001.png,58.10\n', + }], + }, + }, + }); + + await settle(); + + expect(wrapper.find('.frame-metadata-section').find('v-text-field').exists()).toBe(false); + expect(wrapper.find('.frame-metadata-section').find('v-btn').exists()).toBe(false); + expect(wrapper.find('.custom-dataset-info-section').find('v-text-field').exists()).toBe(true); + expect(wrapper.find('.custom-dataset-info-section').find('v-btn').exists()).toBe(true); + }); + + it.each([ + { + name: 'no sidecar', + options: {}, + expected: 'No frame metadata source found.', + }, + { + name: 'unsupported dataset type', + options: { metadata: { ...defaultMetadata, type: 'video' } }, + expected: 'Frame metadata is available for image-sequence datasets only.', + }, + { + name: 'unsupported platform', + options: { frameMetadataSupported: false }, + expected: 'Frame metadata is available for image-sequence datasets only.', + }, + { + name: 'unmatched sidecar', + options: { + frameMetadata: { + cameras: { + port: [{ + name: 'frame_metadata.csv', + text: 'filename,depth\nother001.png,10\n', + }], + }, + }, + }, + expected: 'A frame metadata file (frame_metadata.csv) is present but none of its rows matched', + }, + { + name: 'no row for current frame', + options: { + frame: 1, + frameMetadata: { + cameras: { + port: [{ + name: 'frame_metadata.csv', + text: 'filename,depth\nport001.png,10\n', + }], + }, + }, + }, + expected: 'No frame metadata for the current frame.', + }, + ])('shows the $name empty state', async ({ options, expected }) => { + const { wrapper } = mountDatasetInfo(options); + + await settle(); + + expect(wrapper.find('.frame-metadata-section').text()).toContain(expected); + }); + + it('follows the selected camera', async () => { + const { wrapper, state, loadFrameMetadata } = mountDatasetInfo({ + frameMetadata: { + cameras: { + port: [{ + name: 'frame_metadata.csv', + text: 'filename,latitude\nport001.png,58.10\n', + }], + starboard: [{ + name: 'frame-metadata.txt', + text: 'filename,latitude\nstarboard001.png,59.10\n', + }], + }, + }, + }); + + await settle(); + expect(wrapper.find('.frame-metadata-section').text()).toContain('58.10'); + + state.selectedCamera.value = 'starboard'; + await settle(); + + const section = wrapper.find('.frame-metadata-section'); + expect(section.text()).toContain('59.10'); + expect(section.text()).not.toContain('58.10'); + expect(wrapper.find('.frame-metadata-source-icon').attributes('aria-label')) + .toBe('Source: frame-metadata.txt'); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + }); +}); diff --git a/client/dive-common/components/DatasetInfo/DatasetInfo.vue b/client/dive-common/components/DatasetInfo/DatasetInfo.vue new file mode 100644 index 000000000..2efd95768 --- /dev/null +++ b/client/dive-common/components/DatasetInfo/DatasetInfo.vue @@ -0,0 +1,212 @@ + + + + + diff --git a/client/dive-common/components/DatasetMetaEditorDialog.vue b/client/dive-common/components/DatasetInfo/DatasetInfoFieldDialog.vue similarity index 98% rename from client/dive-common/components/DatasetMetaEditorDialog.vue rename to client/dive-common/components/DatasetInfo/DatasetInfoFieldDialog.vue index 3ccbaa75a..a2e506fb1 100644 --- a/client/dive-common/components/DatasetMetaEditorDialog.vue +++ b/client/dive-common/components/DatasetInfo/DatasetInfoFieldDialog.vue @@ -4,7 +4,7 @@ import { } from 'vue'; export default defineComponent({ - name: 'DatasetMetaEditorDialog', + name: 'DatasetInfoFieldDialog', props: { value: { diff --git a/client/dive-common/components/DatasetInfo/DatasetInfoPanel.vue b/client/dive-common/components/DatasetInfo/DatasetInfoPanel.vue new file mode 100644 index 000000000..2d84aa98f --- /dev/null +++ b/client/dive-common/components/DatasetInfo/DatasetInfoPanel.vue @@ -0,0 +1,70 @@ + + + + + diff --git a/client/dive-common/components/DatasetInfo/FrameMetadataPanel.vue b/client/dive-common/components/DatasetInfo/FrameMetadataPanel.vue new file mode 100644 index 000000000..ffe7d080d --- /dev/null +++ b/client/dive-common/components/DatasetInfo/FrameMetadataPanel.vue @@ -0,0 +1,94 @@ + + + + + diff --git a/client/dive-common/constants.ts b/client/dive-common/constants.ts index 620750528..d2e7f5bf6 100644 --- a/client/dive-common/constants.ts +++ b/client/dive-common/constants.ts @@ -82,6 +82,16 @@ const largeImageTypes = [ 'image/ntf', ]; +/** Extension-only formats for basic image sequences (aligned with server validImageFormats). */ +const basicImageFileExtensions = [ + 'png', + 'jpg', + 'jpeg', + 'sgi', + 'bmp', + 'pgm', +]; + /** Extension-only formats for large-image uploads (aligned with server validLargeImageFormats). */ const largeImageFileExtensions = [ 'nitf', @@ -204,6 +214,7 @@ export { websafeVideoTypes, inputAnnotationTypes, largeImageTypes, + basicImageFileExtensions, largeImageFileExtensions, largeImageDesktopTypes, largeImageWebAccept, diff --git a/client/dive-common/frameMetadata/csvTokenizer.ts b/client/dive-common/frameMetadata/csvTokenizer.ts new file mode 100644 index 000000000..fd0f3021d --- /dev/null +++ b/client/dive-common/frameMetadata/csvTokenizer.ts @@ -0,0 +1,91 @@ +type DelimitedTableDelimiter = ',' | '\t'; + +// Match Python's csv.field_size_limit() default so browser and desktop reject pathological cells +// consistently. +const FIELD_SIZE_LIMIT = 131072; + +// Keep tokenization node-free because frame metadata parsing runs in both Electron and the browser +// renderer. The parser is lenient with bare quotes because field logs commonly contain units such +// as `5"`. +function scanDelimitedRows(text: string, delimiter: DelimitedTableDelimiter): string[][] { + const rows: string[][] = []; + let row: string[] = []; + let field = ''; + let inQuotedField = false; + let atFieldStart = true; + let rowHasContent = false; + const endField = () => { + row.push(field); + field = ''; + atFieldStart = true; + }; + const endRow = () => { + endField(); + rows.push(row); + row = []; + rowHasContent = false; + }; + const { length } = text; + let i = 0; + while (i < length) { + const ch = text[i]; + if (inQuotedField) { + if (ch === '"') { + if (text[i + 1] === '"') { + field += '"'; + i += 2; + } else { + inQuotedField = false; + i += 1; + } + } else { + field += ch; + i += 1; + } + } else if (ch === '"' && atFieldStart) { + inQuotedField = true; + atFieldStart = false; + rowHasContent = true; + i += 1; + } else if (ch === delimiter) { + rowHasContent = true; + endField(); + i += 1; + } else if (ch === '\n' || ch === '\r') { + if (ch === '\r' && text[i + 1] === '\n') { + i += 1; + } + endRow(); + i += 1; + } else { + field += ch; + atFieldStart = false; + rowHasContent = true; + i += 1; + } + } + if (rowHasContent || field.length > 0 || row.length > 0) { + endField(); + rows.push(row); + } + return rows; +} + +function parseDelimitedRows(text: string, delimiter: DelimitedTableDelimiter): string[][] { + if (text.includes('\0')) { + throw new Error('line contains NUL'); + } + const rows = scanDelimitedRows(text, delimiter); + if (rows.some((row) => row.some((cell) => cell.length > FIELD_SIZE_LIMIT))) { + throw new Error('field larger than the size limit'); + } + return rows.map((row) => row.map((cell) => cell.trim())); +} + +export { + parseDelimitedRows, +}; + +export type { + DelimitedTableDelimiter, +}; diff --git a/client/dive-common/frameMetadata/naming.ts b/client/dive-common/frameMetadata/naming.ts new file mode 100644 index 000000000..a81de1b89 --- /dev/null +++ b/client/dive-common/frameMetadata/naming.ts @@ -0,0 +1,14 @@ +// Keep this basename-only predicate mirrored with the server helper because matching names are +// auto-read as frame metadata instead of imported as annotations. +const FRAME_METADATA_SOURCE_NAMES = new Set([ + 'frame-metadata.csv', + 'frame-metadata.txt', + 'frame_metadata.csv', + 'frame_metadata.txt', +]); + +export default function isFrameMetadataSourceName(name: string): boolean { + const separator = Math.max(name.lastIndexOf('/'), name.lastIndexOf('\\')); + const basename = separator === -1 ? name : name.slice(separator + 1); + return FRAME_METADATA_SOURCE_NAMES.has(basename.toLowerCase()); +} diff --git a/client/dive-common/frameMetadata/parser.spec.ts b/client/dive-common/frameMetadata/parser.spec.ts new file mode 100644 index 000000000..1c3bb1f19 --- /dev/null +++ b/client/dive-common/frameMetadata/parser.spec.ts @@ -0,0 +1,702 @@ +/// + +import fs from 'fs'; +import path from 'path'; + +import { + ParsedFrameMetadata, + normalizeAlignmentKey, + parseFrameMetadataSource, +} from './parser'; +import isFrameMetadataSourceName from './naming'; +import { buildFrameAlignmentIndex } from './resolve'; + +type FixtureRecord = Record; +type ExpectedSource = { + header: string[]; + recordsByFrame: Record; + cameras: Record; +}; +type ExpectedFixture = { + sources: Record; +}; + +const syntheticHeader = [ + 'port_image', + 'depth_m', + 'heading', + 'starboard_image', +]; + +const syntheticSources: Record = { + 'synthetic_auv_nav_rect.txt': [ + ['rect_port_0001.tif', '192.80', '174.5', 'rect_starboard_0001.tif'], + ['rect_port_0002.tif', '193.05', '175.1', 'rect_starboard_0002.tif'], + ], + 'synthetic_auv_nav_jpg.txt': [ + ['jpg_port_0001.jpg', '88.40', '92.5', 'jpg_starboard_0001.jpg'], + ['jpg_port_0002.jpg', '88.72', '93.1', 'jpg_starboard_0002.jpg'], + ], +}; + +function sourceText(sourceName: string): string { + const rows = syntheticSources[sourceName]; + return [syntheticHeader.join(' '), ...rows.map((row) => row.join(' ')), ''].join('\n'); +} + +function expectedSource(rows: string[][]): ExpectedSource { + return { + header: syntheticHeader, + recordsByFrame: Object.fromEntries(rows.map((row, frame) => [ + String(frame), + Object.fromEntries(syntheticHeader.map((field, index) => [field, row[index]])), + ])), + cameras: { + port: expectedCamera('port_image', rows), + starboard: expectedCamera('starboard_image', rows), + }, + }; +} + +function expectedCamera(joinColumn: string, rows: string[][]) { + return { + joinColumn, + payloadColumns: syntheticHeader.filter((column) => column !== joinColumn), + frames: rows.map((_, frame) => String(frame)), + }; +} + +function loadExpectedFixture(): ExpectedFixture { + return { + sources: Object.fromEntries( + Object.entries(syntheticSources).map(([sourceName, rows]) => [ + sourceName, + expectedSource(rows), + ]), + ), + }; +} + +function fixtureText(sourceName: string): string { + return sourceText(sourceName); +} + +function frameAlignmentEntries( + cameraRecords: Record, + joinColumn: string, +): Map { + return new Map(Object.entries(cameraRecords).map(([frame, record]) => ( + [normalizeAlignmentKey(record[joinColumn]), Number(frame)] + ))); +} + +function recordsByFrame( + source: ParsedFrameMetadata, + keys: Map, +): Record { + const records: Record = {}; + Array.from(keys.entries()) + .sort(([, frameA], [, frameB]) => frameA - frameB) + .forEach(([key, frame]) => { + if (source.records[key] !== undefined) { + records[String(frame)] = source.records[key]; + } + }); + return records; +} + +describe('shared frame metadata parser', () => { + it('normalizes alignment keys the same way as image name maps', () => { + expect(normalizeAlignmentKey('nested/20191009.154056.00082_rect_color.tif')).toBe( + '20191009.154056.00082_rect_color', + ); + }); + + it('joins only the leftmost image column when several columns match', () => { + // Two image columns are both join candidates, but only the leftmost is the join + // column; the second camera's column is neither joined nor shown as payload. + const keys = new Map([ + ['20191009.154056.00082_rect_color', 0], + ['20191009.154056.00081_rect_color', 0], + ]); + const text = [ + 'port_image date time latitude longitude water_depth altitude starboard_image', + '20191009.154056.00082_rect_color.tif 2019/10/09 15:40:56.1122 46.575870 -124.603094 192.80 2.78 20191009.154056.00081_rect_color.tif', + '', + ].join('\n'); + + const source = parseFrameMetadataSource(text, keys, 'nav.txt'); + + expect(source).not.toBeNull(); + expect(source?.sourceName).toBe('nav.txt'); + expect(Object.keys(source?.records || {})).toEqual([ + '20191009.154056.00082_rect_color', + ]); + const portRecord = source?.records['20191009.154056.00082_rect_color']; + const expectedColumns = [ + 'port_image', + 'date', + 'time', + 'latitude', + 'longitude', + 'water_depth', + 'altitude', + ]; + expect(Object.keys(portRecord || {})).toEqual(expectedColumns); + // Columns are reported in file order, excluding the second (non-join) image candidate. + expect(source?.columns).toEqual(expectedColumns); + expect(portRecord?.latitude).toBe('46.575870'); + expect(Object.values(portRecord || {}).every((value) => typeof value === 'string')).toBe(true); + }); + + it('parses comma, tab, and whitespace delimited sources', () => { + const keys = new Map([['image_0001', 0]]); + + [ + 'filename,depth,latitude\nimage_0001.jpg,192.80,46.575870\n', + 'filename\tdepth\tlatitude\nimage_0001.jpg\t192.80\t46.575870\n', + 'filename depth latitude\nimage_0001.jpg 192.80 46.575870\n', + ].forEach((text) => { + const source = parseFrameMetadataSource(text, keys); + + expect(source).not.toBeNull(); + expect(source?.records.image_0001).toEqual({ + filename: 'image_0001.jpg', + depth: '192.80', + latitude: '46.575870', + }); + }); + }); + + it('picks the leftmost join column on a score tie', () => { + // Both image columns match one alignment key each (single-row floor of 1), so both are + // join candidates; the leftmost wins and claims the only record. + const keys = new Map([ + ['20191009.154056.00082_rect_color', 0], + ['20191009.154056.00081_rect_color', 0], + ]); + const text = [ + 'port_image,latitude,starboard_image', + '20191009.154056.00082_rect_color.tif,46.575870,20191009.154056.00081_rect_color.tif', + '', + ].join('\n'); + + const source = parseFrameMetadataSource(text, keys); + + expect(source).not.toBeNull(); + expect(Object.keys(source?.records || {})).toEqual([ + '20191009.154056.00082_rect_color', + ]); + // The second image column is a join candidate, so it is neither joined nor payload. + expect(Object.keys(source?.records['20191009.154056.00082_rect_color'] || {})).toEqual([ + 'port_image', + 'latitude', + ]); + }); + + it('picks the argmax join column over a lower-scoring match', () => { + // `left` matches all 3 rows; `right` matches only the c.png cell (score 1 < floor 2), + // so `right` stays payload and every record is keyed by `left`. + const keys = new Map([['a.png', 0], ['b.png', 1], ['c.png', 2]]); + const text = 'left,right,depth\na.png,z.png,10\nb.png,y.png,11\nc.png,a.png,12\n'; + + const source = parseFrameMetadataSource(text, keys); + + expect(source).not.toBeNull(); + expect(Object.keys(source?.records || {}).sort()).toEqual(['a', 'b', 'c']); + expect(source?.records.a).toEqual({ left: 'a.png', right: 'z.png', depth: '10' }); + }); + + it('parses a sidecar containing a bare double-quote character', () => { + const keys = new Map([['image_0001', 0]]); + const text = [ + 'filename,depth', + 'image_0001.jpg,5"', + '', + ].join('\n'); + + const source = parseFrameMetadataSource(text, keys); + + expect(source).not.toBeNull(); + expect(source?.records.image_0001.depth).toBe('5"'); + }); + + it('accepts VIAME-shaped frame metadata without the VIAME header', () => { + const keys = new Map([['image_0001', 0]]); + const text = [ + 'index,image,frame,x,y,depth,altitude,heading,temperature', + '1,image_0001.jpg,100,46.5,-124.6,192.8,2.7,180.5,4.2', + '', + ].join('\n'); + + const source = parseFrameMetadataSource(text, keys); + + expect(source).not.toBeNull(); + expect(source?.records.image_0001.depth).toBe('192.8'); + }); + + it('rejects bare image lists and unrelated text', () => { + const keys = new Map([['image_0001', 0]]); + + expect(parseFrameMetadataSource('image\nimage_0001.jpg\n', keys)).toBeNull(); + expect(parseFrameMetadataSource('note,value\nhello,world\n', keys)).toBeNull(); + }); + + it('matches the shared synthetic AUV fixture', () => { + const expectedFixture = loadExpectedFixture(); + + Object.entries(expectedFixture.sources).forEach(([sourceName, expected]) => { + const text = fixtureText(sourceName); + Object.entries(expected.cameras).forEach(([camera, cameraExpected]) => { + const expectedRecords = Object.fromEntries( + cameraExpected.frames.map((frame) => [frame, expected.recordsByFrame[frame]]), + ); + const { joinColumn } = cameraExpected; + const keys = frameAlignmentEntries(expectedRecords, joinColumn); + const source = parseFrameMetadataSource(text, keys, sourceName); + + expect(source).not.toBeNull(); + if (source === null) { + throw new Error(`Expected ${sourceName} to parse for ${camera}`); + } + expect(source.sourceName).toBe(sourceName); + expect(recordsByFrame(source, keys)).toEqual(expectedRecords); + expect(Object.values(source.records).every((record) => ( + Object.values(record).every((value) => typeof value === 'string') + ))).toBe(true); + }); + }); + }); + + it('does not promote a hash-prefixed row to a header', () => { + expect(parseFrameMetadataSource( + '# filename,depth,heading\nimg001.png,10,180\n', + { 'img001.png': 0 }, + )).toBeNull(); + }); + + it('requires a real whitespace header row after comments', () => { + expect(parseFrameMetadataSource( + '# filename depth heading\nimg001.png 10 180\n', + { 'img001.png': 0 }, + )).toBeNull(); + }); + + it('skips a standalone comment block above the real header', () => { + const source = parseFrameMetadataSource( + '# vehicle: AUV, dive 42\nfilename,depth\nimg001.png,10\n', + { 'img001.png': 0 }, + ); + + expect(source).not.toBeNull(); + expect(source?.records.img001).toEqual({ filename: 'img001.png', depth: '10' }); + }); + + it('strips a leading UTF-8 BOM from the header cell', () => { + const source = parseFrameMetadataSource( + 'filename,depth\nimg001.png,10\n', + { 'img001.png': 0 }, + ); + + expect(source).not.toBeNull(); + expect(source?.records.img001).toEqual({ filename: 'img001.png', depth: '10' }); + }); + + it('rejects NUL-poisoned and oversized fields as no source', () => { + expect(parseFrameMetadataSource( + 'filename,alt\x00\nimg001.png,42\x00\n', + { 'img001.png': 0 }, + )).toBeNull(); + + const hugeField = `filename,notes\nimg001.png,${'x'.repeat(140000)}\n`; + expect(parseFrameMetadataSource(hugeField, { 'img001.png': 0 })).toBeNull(); + }); + + it('normalizes a double-extension alignment key exactly once', () => { + const source = parseFrameMetadataSource( + 'filename,depth\nIMG_001.jpg.png,10\n', + { 'IMG_001.jpg.png': 0 }, + ); + + expect(source).not.toBeNull(); + expect(source?.records['IMG_001.jpg']).toEqual({ + filename: 'IMG_001.jpg.png', + depth: '10', + }); + }); + + it('keeps the non-join column as payload in a two-column source', () => { + const source = parseFrameMetadataSource( + 'image,altitude\nimg001.png,10\nimg002.png,12\n', + { 'img001.png': 0, 'img002.png': 1 }, + ); + + expect(source).not.toBeNull(); + expect(source?.records.img001).toEqual({ image: 'img001.png', altitude: '10' }); + expect(source?.records.img002).toEqual({ image: 'img002.png', altitude: '12' }); + }); + + it('does not misattribute a numeric payload colliding with an image stem', () => { + const keys: Record = {}; + for (let index = 1; index <= 50; index += 1) { + keys[`${index}.png`] = index - 1; + } + const source = parseFrameMetadataSource( + 'image,altitude,depth\n1.png,42,100\n2.png,999,200\n', + keys, + ); + + expect(source).not.toBeNull(); + expect(Object.keys(source?.records || {}).sort()).toEqual(['1', '2']); + expect(source?.records['1']).toEqual({ image: '1.png', altitude: '42', depth: '100' }); + }); + + it('keeps the first row for duplicate alignment keys', () => { + const source = parseFrameMetadataSource( + 'filename,depth\nimg001.png,10\nimg001.png,99\n', + { 'img001.png': 0 }, + ); + + expect(source).not.toBeNull(); + expect(source?.records.img001.depth).toBe('10'); + }); + + it('sniffs the delimiter from the data line, not a prose comment', () => { + // A delimiter-free `#` comment over comma data sniffs the comma from the first + // non-comment line, so a comment-headed sidecar still joins. + const source = parseFrameMetadataSource( + '# AUV nav log\nfilename,depth\nimg001.png,10\n', + { 'img001.png': 0 }, + ); + + expect(source).not.toBeNull(); + expect(source?.records.img001).toEqual({ filename: 'img001.png', depth: '10' }); + }); + + it('prefers a data-line tab over a comma in the comment prose', () => { + const source = parseFrameMetadataSource( + '# Position (lat, lon) log\nfilename\tdepth\nimg001.png\t10\n', + { 'img001.png': 0 }, + ); + + expect(source).not.toBeNull(); + expect(source?.records.img001).toEqual({ filename: 'img001.png', depth: '10' }); + }); + + it('returns null when every non-empty line is a comment', () => { + expect(parseFrameMetadataSource('# one\n# two\n# three\n', { 'img001.png': 0 })).toBeNull(); + }); + + it('drops a leading all-empty row before header selection', () => { + // The tokenizer keeps `,,,` as an all-empty row; without the filter it becomes the header. + const source = parseFrameMetadataSource( + ',,,\nfilename,depth\nimg001.png,100\n', + { 'img001.png': 0 }, + ); + + expect(source).not.toBeNull(); + expect(source?.records.img001).toEqual({ filename: 'img001.png', depth: '100' }); + }); + + it('accepts a wide row whose cells are each within the field limit', () => { + // Total row length exceeds 131072 but no single cell does, so it stays frame metadata. + const cell = 'a'.repeat(7000); + const columns = Array.from({ length: 20 }, (_, index) => `c${index + 1}`); + const header = ['filename', ...columns].join(','); + const row = ['img001.png', ...columns.map(() => cell)].join(','); + const source = parseFrameMetadataSource(`${header}\n${row}\n`, { 'img001.png': 0 }); + + expect(source).not.toBeNull(); + expect(source?.records.img001.filename).toBe('img001.png'); + expect(source?.records.img001.c1).toBe(cell); + expect(source?.records.img001.c20).toBe(cell); + }); + + it('rejects a source with a single cell over the field limit', () => { + const oversized = `filename,notes\nimg001.png,${'x'.repeat(131073)}\n`; + expect(parseFrameMetadataSource(oversized, { 'img001.png': 0 })).toBeNull(); + }); + + it('accepts a prebuilt index identically to raw alignment keys', () => { + const text = 'filename,depth\nimg001.png,10\n'; + const fromKeys = parseFrameMetadataSource(text, { 'img001.png': 0 }); + const fromIndex = parseFrameMetadataSource(text, buildFrameAlignmentIndex(['img001.png'])); + + expect(fromKeys).not.toBeNull(); + expect(fromIndex).not.toBeNull(); + expect(fromIndex?.records).toEqual(fromKeys?.records); + expect(fromIndex?.records.img001).toEqual({ filename: 'img001.png', depth: '10' }); + }); +}); + +type ParserCase = { + name: string; + text: string; + frameAlignmentEntries: Record; + records: Record> | null; + assertParsed?: (parsed: ParsedFrameMetadata | null) => void; +}; + +const defaultParserFrameAlignmentEntries = { 'img001.png': 0, 'img002.png': 1 }; +const wideCell = 'a'.repeat(7000); +const wideColumns = Array.from({ length: 20 }, (_, index) => `c${index + 1}`); + +function numericCollisionFrameAlignmentEntries(): Record { + const keys: Record = {}; + for (let index = 1; index <= 50; index += 1) { + keys[`${index}.png`] = index - 1; + } + return keys; +} + +const parserCases: ParserCase[] = [ + { + name: 'all-comments', + text: '# comment one\n# comment two\n# comment three\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: null, + }, + { + name: 'basic', + text: 'filename,depth\nimg001.png,10\nimg002.png,12\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: { + img001: { filename: 'img001.png', depth: '10' }, + img002: { filename: 'img002.png', depth: '12' }, + }, + }, + { + name: 'bom-frame-metadata', + text: '\ufefffilename,depth\nimg001.png,10\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: { + img001: { filename: 'img001.png', depth: '10' }, + }, + }, + { + name: 'comment-block', + text: '# vehicle: AUV, dive 42\nfilename,depth\nimg001.png,10\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: { + img001: { filename: 'img001.png', depth: '10' }, + }, + }, + { + name: 'comment-comma-over-tsv', + text: '# Position (lat, lon) log\nfilename\tdepth\nimg001.png\t10\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: { + img001: { filename: 'img001.png', depth: '10' }, + }, + }, + { + name: 'comment-prose-header', + text: '# AUV nav log\nfilename,depth\nimg001.png,10\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: { + img001: { filename: 'img001.png', depth: '10' }, + }, + }, + { + name: 'double-extension', + text: 'filename,depth\nIMG_001.jpg.png,10\n', + frameAlignmentEntries: { 'IMG_001.jpg.png': 0 }, + records: { + 'IMG_001.jpg': { filename: 'IMG_001.jpg.png', depth: '10' }, + }, + }, + { + name: 'duplicate-rows', + text: 'filename,depth\nimg001.png,10\nimg001.png,99\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: { + img001: { filename: 'img001.png', depth: '10' }, + }, + }, + { + name: 'hash-header', + text: '# filename,depth,heading\nimg001.png,10,180\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: null, + }, + { + name: 'huge-field', + text: `filename,notes\nimg001.png,${'x'.repeat(131073)}\n`, + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: null, + }, + { + name: 'leading-empty-row', + text: ',,,\nfilename,depth\nimg001.png,100\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: { + img001: { filename: 'img001.png', depth: '100' }, + }, + }, + { + name: 'nav-whitespace', + text: '# filename depth heading\nimg001.png 10 180\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: null, + }, + { + name: 'no-join', + text: 'station,depth\nA,10\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: null, + }, + { + name: 'nul-bytes', + text: 'filename,alt\0\nimg001.png,42\0\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: null, + }, + { + name: 'numeric-collision', + text: 'image,altitude,depth\n1.png,42,100\n2.png,999,200\n', + frameAlignmentEntries: numericCollisionFrameAlignmentEntries(), + records: { + 1: { image: '1.png', altitude: '42', depth: '100' }, + 2: { image: '2.png', altitude: '999', depth: '200' }, + }, + }, + { + name: 'pandas-index', + text: ',filename,depth\n0,img001.png,10\n1,img002.png,12\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: { + img001: { filename: 'img001.png', depth: '10' }, + img002: { filename: 'img002.png', depth: '12' }, + }, + }, + { + name: 'proto', + text: 'filename,depth\n__proto__.png,10\n', + frameAlignmentEntries: { '__proto__.png': 0 }, + records: { + ['__proto__']: { filename: '__proto__.png', depth: '10' }, + }, + assertParsed: (parsed) => { + expect(parsed).not.toBeNull(); + expect(Object.prototype.hasOwnProperty.call(parsed?.records ?? {}, '__proto__')).toBe(true); + }, + }, + { + name: 'tab-comma-units', + text: 'filename\tPosition (lat, lon)\tDepth (m)\nimg001.png\t42.35, -70.90\t12.4\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: { + img001: { + filename: 'img001.png', + 'Position (lat, lon)': '42.35, -70.90', + 'Depth (m)': '12.4', + }, + }, + }, + { + name: 'trailing-comma', + text: 'filename,depth,\nimg001.png,10,\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: { + img001: { filename: 'img001.png', depth: '10' }, + }, + }, + { + name: 'two-column', + text: 'image,altitude\nimg001.png,10\nimg002.png,12\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: { + img001: { image: 'img001.png', altitude: '10' }, + img002: { image: 'img002.png', altitude: '12' }, + }, + }, + { + name: 'viame-annotations', + text: [ + '# 1: Detection or Track-id,2: Video or Image Identifier,3: Unique Frame Identifier,4-7: Img-bbox(TL_x,TL_y,BR_x,BR_y),8: Detection or Length Confidence,9: Target Length (0 or -1 if invalid),10-11+: Repeated Species,Confidence Pairs or Attributes', + '1,img001.png,0,10,20,30,40,1.0,-1,fish,0.9', + '', + ].join('\n'), + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: null, + }, + { + name: 'wide-row', + text: `${['filename', ...wideColumns].join(',')}\n${[ + 'img001.png', + ...wideColumns.map(() => wideCell), + ].join(',')}\n`, + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: { + img001: { + filename: 'img001.png', + ...Object.fromEntries(wideColumns.map((column) => [column, wideCell])), + }, + }, + }, + { + name: 'windows-paths', + text: 'filename,depth\nimages\\img001.png,10\n', + frameAlignmentEntries: defaultParserFrameAlignmentEntries, + records: { + img001: { filename: 'images\\img001.png', depth: '10' }, + }, + }, +]; + +// Resolve the shared naming truth table at the repo root. The spec lives at +// client/dive-common/frameMetadata, three levels below the root; when the runner does not +// provide __dirname, npm test runs with CWD=client/ (one level below root). +function sourceNamesTruthTablePath(): string { + const fromDirname = typeof __dirname === 'string' + ? path.resolve( + __dirname, + '../../../testdata/frame-metadata-conformance/source_names.expected.json', + ) + : null; + if (fromDirname && fs.existsSync(fromDirname)) { + return fromDirname; + } + return path.resolve('../testdata/frame-metadata-conformance/source_names.expected.json'); +} + +// Compare records structurally through sorted [key, value] pairs so equality is +// unaffected by prototype and so an own "__proto__" key participates in the comparison +// instead of being read off the chain. +function toComparable(records: Record> | null) { + if (records === null) { + return null; + } + return Object.keys(records).sort().map((frame) => { + const record = records[frame]; + return [frame, Object.keys(record).sort().map((field) => [field, record[field]])]; + }); +} + +describe('inline frame-metadata parser cases', () => { + parserCases.forEach((parserCase) => { + it(`matches ${parserCase.name}`, () => { + const parsed = parseFrameMetadataSource(parserCase.text, parserCase.frameAlignmentEntries); + const records = parsed === null ? null : parsed.records; + + expect(toComparable(records)).toEqual(toComparable(parserCase.records)); + parserCase.assertParsed?.(parsed); + }); + }); +}); + +describe('shared frame-metadata naming', () => { + it('matches the shared source-name predicate truth table', () => { + // The truth table is shared with the server harness so the mirrored predicate cannot drift. + const truthTablePath = sourceNamesTruthTablePath(); + expect(fs.existsSync(truthTablePath)).toBe(true); + const truthTable = JSON.parse(fs.readFileSync(truthTablePath, 'utf-8')) as Record; + Object.entries(truthTable).forEach(([name, expected]) => { + expect(isFrameMetadataSourceName(name)).toBe(expected); + }); + }); +}); diff --git a/client/dive-common/frameMetadata/parser.ts b/client/dive-common/frameMetadata/parser.ts new file mode 100644 index 000000000..ea91d6303 --- /dev/null +++ b/client/dive-common/frameMetadata/parser.ts @@ -0,0 +1,278 @@ +import { basicImageFileExtensions, largeImageFileExtensions } from 'dive-common/constants'; +import { parseDelimitedRows } from './csvTokenizer'; +import type { DelimitedTableDelimiter } from './csvTokenizer'; + +// Shared by the desktop backend and the web client. Keep this node-free so the same parser runs +// in Electron and in the browser renderer. + +type FrameMetadataRow = Record; +type FrameAlignmentEntries = Map | Record; + +interface ParsedFrameMetadata { + sourceName?: string; + // Payload column names in file order; the resolver relies on this to avoid JS object-key + // reordering for numeric-named headers. + columns: string[]; + records: Record; +} + +// Parser call sites share this shape so join scoring works from one normalized frame-alignment +// vocabulary instead of mixing raw filenames with normalized values. +interface FrameAlignmentIndex { + alignmentKeys: Set; + frameByAlignmentKey: Map; +} + +// Mirror of the server's allValidLargeImageFormats (validImageFormats + validLargeImageFormats): +// the extensions normalizeAlignmentKey strips so a filename cell matches its alignment key. +const imageExtensions = new Set([ + ...basicImageFileExtensions, + ...largeImageFileExtensions, +]); + +// Split a basename into (stem, lowercased extension). Mirrors node path.extname semantics: a +// leading dot is not an extension (so '.gitignore' has no extension), matching how the server +// strips one image extension from an alignment key. +function splitExtension(basename: string): { stem: string; extension: string } { + const dot = basename.lastIndexOf('.'); + if (dot <= 0) { + return { stem: basename, extension: '' }; + } + return { stem: basename.slice(0, dot), extension: basename.slice(dot + 1).toLowerCase() }; +} + +function normalizeAlignmentKey(value: string): string { + // Split on both separators so Windows-style path cells (images\IMG.png) normalize the same way + // they would on a posix build. Strip exactly one image extension so a filename cell matches its + // alignment key. + const basename = String(value).trim().split(/[\\/]/).pop() ?? ''; + const { stem, extension } = splitExtension(basename); + return imageExtensions.has(extension) ? stem : basename; +} + +// The shared normalization/collision rule for resolver-built indexes and raw alignment maps. +export function frameAlignmentIndexFromEntries( + entries: Iterable<[string, number]>, +): FrameAlignmentIndex { + const alignmentKeys = new Set(); + const frameByAlignmentKey = new Map(); + Array.from(entries).forEach(([name, frame]) => { + const key = normalizeAlignmentKey(name); + alignmentKeys.add(key); + frameByAlignmentKey.set(key, frame); + }); + return { alignmentKeys, frameByAlignmentKey }; +} + +function normalizeFrameAlignmentEntries( + frameAlignmentEntries: FrameAlignmentEntries, +): FrameAlignmentIndex { + return frameAlignmentIndexFromEntries( + frameAlignmentEntries instanceof Map + ? frameAlignmentEntries + : Object.entries(frameAlignmentEntries), + ); +} + +function isFrameAlignmentIndex( + value: FrameAlignmentIndex | FrameAlignmentEntries, +): value is FrameAlignmentIndex { + return !(value instanceof Map) && (value as FrameAlignmentIndex).alignmentKeys instanceof Set; +} + +function dropLeadingCommentRows(rawRows: string[][]): string[][] { + let index = 0; + while (index < rawRows.length && rawRows[index].length > 0 && rawRows[index][0].startsWith('#')) { + index += 1; + } + return rawRows.slice(index); +} + +function nullPrototypeRecord(): Record { + return Object.create(null); +} + +function buildTable( + headerCells: string[], + dataRows: string[][], +): { header: string[]; rows: FrameMetadataRow[] } { + const rawHeader = headerCells.map((cell) => cell.trim()); + // Empty header cells usually come from pandas indexes or trailing commas. + const keptIndices = rawHeader.reduce((indices, cell, index) => { + if (cell.length > 0) { + indices.push(index); + } + return indices; + }, []); + if (keptIndices.length === 0) { + return { header: [], rows: [] }; + } + const header = keptIndices.map((index) => rawHeader[index]); + + const rows: FrameMetadataRow[] = []; + dataRows.forEach((rawRow) => { + const values = rawRow.map((cell) => cell.trim()); + if (!values.some((cell) => cell.length > 0)) { + return; + } + const row = nullPrototypeRecord(); + keptIndices.forEach((index, position) => { + row[header[position]] = values[index] || ''; + }); + rows.push(row); + }); + return { header, rows }; +} + +// Sidecars may include multiple image columns; score by alignment-key matches so each camera can use +// its own column while ties remain stable in file order. +function selectJoinColumn( + header: string[], + rows: FrameMetadataRow[], + alignmentKeys: Set, +): { joinColumn: string | null; candidates: string[] } { + const threshold = Math.min(2, rows.length); + const scores = header.map((column) => ({ + column, + score: rows.reduce((total, row) => ( + row[column] && alignmentKeys.has(normalizeAlignmentKey(row[column])) ? total + 1 : total + ), 0), + })); + const candidates = scores.filter(({ score }) => score >= threshold && score > 0); + if (candidates.length === 0) { + return { joinColumn: null, candidates: [] }; + } + const joinColumn = candidates.reduce((best, candidate) => ( + candidate.score > best.score ? candidate : best + )).column; + return { joinColumn, candidates: candidates.map(({ column }) => column) }; +} + +function selectHeaderAndRows(rawRows: string[][]): { header: string[]; rows: FrameMetadataRow[] } { + const body = dropLeadingCommentRows(rawRows); + if (body.length === 0) { + return { header: [], rows: [] }; + } + return buildTable(body[0], body.slice(1)); +} + +function projectRecord(row: FrameMetadataRow, fields: string[]): FrameMetadataRow { + const record = nullPrototypeRecord(); + fields.forEach((field) => { + record[field] = row[field] || ''; + }); + return record; +} + +function parseFrameMetadataSource( + text: string, + alignmentIndex: FrameAlignmentIndex | FrameAlignmentEntries, + sourceName?: string, +): ParsedFrameMetadata | null { + const index = isFrameAlignmentIndex(alignmentIndex) + ? alignmentIndex + : normalizeFrameAlignmentEntries(alignmentIndex); + const { alignmentKeys } = index; + + if (text.includes('\0')) { + return null; + } + const content = text.charCodeAt(0) === 0xFEFF ? text.slice(1) : text; + + const rawRows = readRows(content); + if (rawRows.length === 0) { + return null; + } + + const { header, rows } = selectHeaderAndRows(rawRows); + if (header.length === 0 || rows.length === 0) { + return null; + } + + const { joinColumn, candidates: joinCandidates } = selectJoinColumn( + header, + rows, + alignmentKeys, + ); + if (joinColumn === null) { + return null; + } + + const payloadColumns = header.filter((column) => !joinCandidates.includes(column)); + if (payloadColumns.length === 0) { + return null; + } + + const otherJoinCandidates = new Set(joinCandidates.filter((column) => column !== joinColumn)); + const recordFields = header.filter((column) => !otherJoinCandidates.has(column)); + + const records = nullPrototypeRecord(); + rows.forEach((row) => { + const key = normalizeAlignmentKey(row[joinColumn] || ''); + if (alignmentKeys.has(key) && !(key in records)) { + records[key] = projectRecord(row, recordFields); + } + }); + + if (Object.keys(records).length === 0) { + return null; + } + + return { sourceName, columns: recordFields, records }; +} + +function readRows(text: string): string[][] { + const sniff = sniffLine(text); + if (sniff === null) { + return []; + } + + const delimiter = sniffDelimiter(sniff); + if (delimiter === null) { + return text + .split(/\r?\n/) + .filter((line) => line.trim().length > 0) + .map((line) => line.trim().split(/\s+/)); + } + + try { + // Drop delimiter-only rows before header selection; otherwise `,,,` becomes the header. + return parseDelimitedRows(text, delimiter).filter((row) => row.some((cell) => cell.length > 0)); + } catch { + // NUL bytes, oversized fields, etc. are treated as "not a frame metadata source" instead of + // surfacing a parse error. + return []; + } +} + +// Ignore leading prose comments while sniffing so commas inside prose do not override TSV data. +function sniffLine(text: string): string | null { + const lines = text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0); + return lines.find((line) => !line.startsWith('#')) ?? lines[0] ?? null; +} + +function sniffDelimiter(line: string): DelimitedTableDelimiter | null { + // Count-based so a tab-delimited header whose field names contain commas + // (e.g. a parenthesized unit like "Position (lat, lon)") still sniffs as TSV. + const commas = line.split(',').length - 1; + const tabs = line.split('\t').length - 1; + if (tabs > commas) { + return '\t'; + } + if (commas > 0) { + return ','; + } + return null; +} + +export { + FrameMetadataRow, + FrameAlignmentIndex, + FrameAlignmentEntries, + ParsedFrameMetadata, + normalizeAlignmentKey, + parseFrameMetadataSource, +}; diff --git a/client/dive-common/frameMetadata/resolve.spec.ts b/client/dive-common/frameMetadata/resolve.spec.ts new file mode 100644 index 000000000..44ef5cb45 --- /dev/null +++ b/client/dive-common/frameMetadata/resolve.spec.ts @@ -0,0 +1,137 @@ +/// + +import { buildFrameAlignmentIndex, resolveCameras } from './resolve'; + +describe('buildFrameAlignmentIndex', () => { + it('normalizes an ordered media list to keys and frame numbers', () => { + const index = buildFrameAlignmentIndex(['img001.png', 'nested/img002.png']); + + expect(index.alignmentKeys).toEqual(new Set(['img001', 'img002'])); + expect(index.frameByAlignmentKey).toEqual(new Map([['img001', 0], ['img002', 1]])); + }); + + it('is tolerant of duplicate basenames (last-wins, never throws)', () => { + // The import-path validator throws on duplicate basenames; the read-time index must not + // because that would blank the camera's metadata. + const index = buildFrameAlignmentIndex(['a/img001.png', 'b/img001.png', 'img002.png']); + + expect(index.alignmentKeys).toEqual(new Set(['img001', 'img002'])); + expect(index.frameByAlignmentKey.get('img001')).toBe(1); + expect(index.frameByAlignmentKey.get('img002')).toBe(2); + }); +}); + +describe('resolveCameras', () => { + it('resolves a single camera single source into the compact payload', () => { + const alignmentIndex = buildFrameAlignmentIndex(['img001.png', 'img002.png']); + const resolved = resolveCameras( + { singleCam: [['frame_metadata.csv', 'filename,depth\nimg001.png,10\nimg002.png,12\n']] }, + { singleCam: alignmentIndex }, + ); + + expect(resolved.columns.singleCam).toEqual(['filename', 'depth']); + expect(resolved.cameras.singleCam[0]).toEqual(['img001.png', '10']); + expect(resolved.cameras.singleCam[1]).toEqual(['img002.png', '12']); + expect(resolved.sources.singleCam).toEqual(['frame_metadata.csv']); + }); + + it('first-wins merges frames across sources in precedence order', () => { + const alignmentIndex = buildFrameAlignmentIndex(['img001.png', 'img002.png']); + const resolved = resolveCameras( + { + singleCam: [ + // The earlier (higher-precedence) source claims frame 0. + ['frame-metadata.txt', 'filename,depth\nimg001.png,10\n'], + // The later source re-lists frame 0 (ignored) and adds frame 1. + ['frame_metadata.csv', 'filename,depth\nimg001.png,99\nimg002.png,20\n'], + ], + }, + { singleCam: alignmentIndex }, + ); + + expect(resolved.cameras.singleCam[0]).toEqual(['img001.png', '10']); + expect(resolved.cameras.singleCam[1]).toEqual(['img002.png', '20']); + expect(resolved.sources.singleCam).toEqual(['frame-metadata.txt', 'frame_metadata.csv']); + }); + + it('unions columns across sources in precedence and file order', () => { + const alignmentIndex = buildFrameAlignmentIndex(['img001.png', 'img002.png']); + const resolved = resolveCameras( + { + singleCam: [ + ['frame-metadata.txt', 'filename,depth\nimg001.png,10\n'], + ['frame_metadata.csv', 'filename,heading\nimg002.png,180\n'], + ], + }, + { singleCam: alignmentIndex }, + ); + + expect(resolved.columns.singleCam).toEqual(['filename', 'depth', 'heading']); + // A frame's row is aligned to the union columns; cells its source lacked are empty. + expect(resolved.cameras.singleCam[0]).toEqual(['img001.png', '10', '']); + expect(resolved.cameras.singleCam[1]).toEqual(['img002.png', '', '180']); + }); + + it('selects a per-camera join column from a shared multicam source', () => { + // One wide sidecar binds both cameras: left joins on port_image, right on starboard_image, + // and each camera keys rows by its own media order. + const text = [ + 'port_image,depth,starboard_image', + 'port001.tif,10,star001.tif', + 'port002.tif,12,star002.tif', + '', + ].join('\n'); + const resolved = resolveCameras( + { left: [['frame_metadata.csv', text]], right: [['frame_metadata.csv', text]] }, + { + left: buildFrameAlignmentIndex(['port001.tif', 'port002.tif']), + // Reversed order so the join, not the row order, drives right's frame keys. + right: buildFrameAlignmentIndex(['star002.tif', 'star001.tif']), + }, + ); + + expect(resolved.cameras.left[0]).toEqual(['port001.tif', '10', 'star001.tif']); + expect(resolved.cameras.left[1]).toEqual(['port002.tif', '12', 'star002.tif']); + // right frame 0 is star002 (its media index 0), which is the second data row. + expect(resolved.cameras.right[0]).toEqual(['port002.tif', '12', 'star002.tif']); + expect(resolved.cameras.right[1]).toEqual(['port001.tif', '10', 'star001.tif']); + }); + + it('tolerates duplicate media basenames when resolving (last-wins frame)', () => { + const alignmentIndex = buildFrameAlignmentIndex(['a/img001.png', 'b/img001.png', 'img002.png']); + const resolved = resolveCameras( + { singleCam: [['frame_metadata.csv', 'filename,depth\nimg001.png,10\nimg002.png,12\n']] }, + { singleCam: alignmentIndex }, + ); + + // img001 resolves to the last (frame 1) of its duplicate basenames. + expect(resolved.cameras.singleCam[1]).toEqual(['img001.png', '10']); + expect(resolved.cameras.singleCam[2]).toEqual(['img002.png', '12']); + }); + + it('keeps numeric-named header columns in file order, not JS object-key order', () => { + // Object keys would iterate numeric-like names as ['1','2','3']; the explicit columns array + // preserves the file order ['3','1','2'] (readtime deferred finding #8). + const alignmentIndex = buildFrameAlignmentIndex(['img001.png', 'img002.png']); + const text = 'filename,3,1,2\nimg001.png,c,a,b\nimg002.png,cc,aa,bb\n'; + const resolved = resolveCameras( + { singleCam: [['frame_metadata.csv', text]] }, + { singleCam: alignmentIndex }, + ); + + expect(resolved.columns.singleCam).toEqual(['filename', '3', '1', '2']); + expect(resolved.cameras.singleCam[0]).toEqual(['img001.png', 'c', 'a', 'b']); + expect(resolved.cameras.singleCam[1]).toEqual(['img002.png', 'cc', 'aa', 'bb']); + }); + + it('omits a camera with no matching sidecar', () => { + const resolved = resolveCameras( + { singleCam: [['frame_metadata.csv', 'station,depth\nA,10\n']] }, + { singleCam: buildFrameAlignmentIndex(['img001.png', 'img002.png']) }, + ); + + expect(resolved.cameras.singleCam).toBeUndefined(); + expect(resolved.sources.singleCam).toBeUndefined(); + expect(resolved.columns.singleCam).toBeUndefined(); + }); +}); diff --git a/client/dive-common/frameMetadata/resolve.ts b/client/dive-common/frameMetadata/resolve.ts new file mode 100644 index 000000000..f1de83806 --- /dev/null +++ b/client/dive-common/frameMetadata/resolve.ts @@ -0,0 +1,83 @@ +import type { ResolvedFrameMetadata } from 'dive-common/apispec'; +import { frameAlignmentIndexFromEntries, parseFrameMetadataSource } from './parser'; +import type { FrameAlignmentIndex, ParsedFrameMetadata } from './parser'; + +type CameraCandidateTexts = Record; +type CameraFrameAlignmentIndexes = Record; + +// The read path must tolerate duplicate basenames because rejecting here would hide all metadata +// for the camera. Later media entries win for consistency with the ordered media list. +function buildFrameAlignmentIndex(mediaNames: string[]): FrameAlignmentIndex { + return frameAlignmentIndexFromEntries(mediaNames.map((name, frame) => [name, frame])); +} + +function unionColumns(sources: ParsedFrameMetadata[]): string[] { + const seen = new Set(); + const columns: string[] = []; + sources.forEach((source) => { + source.columns.forEach((column) => { + if (!seen.has(column)) { + seen.add(column); + columns.push(column); + } + }); + }); + return columns; +} + +function resolveCameras( + cameraTexts: CameraCandidateTexts, + alignmentIndexesByCamera: CameraFrameAlignmentIndexes, +): ResolvedFrameMetadata { + const cameras: ResolvedFrameMetadata['cameras'] = {}; + const sources: ResolvedFrameMetadata['sources'] = {}; + const columns: ResolvedFrameMetadata['columns'] = {}; + + Object.entries(cameraTexts).forEach(([camera, candidates]) => { + const index = alignmentIndexesByCamera[camera]; + if (index === undefined) { + return; + } + + const parsed = candidates + .map(([sourceName, text]) => parseFrameMetadataSource(text, index, sourceName)) + .filter((source): source is ParsedFrameMetadata => source !== null); + if (parsed.length === 0) { + return; + } + + const cameraColumns = unionColumns(parsed); + + // Higher-precedence sidecars should not be overwritten by fallback locations. + const records: Record = {}; + parsed.forEach((source) => { + Object.entries(source.records).forEach(([alignmentKey, values]) => { + const frame = index.frameByAlignmentKey.get(alignmentKey); + if (frame === undefined || records[frame] !== undefined) { + return; + } + records[frame] = cameraColumns.map((column) => values[column] ?? ''); + }); + }); + + cameras[camera] = records; + columns[camera] = cameraColumns; + sources[camera] = parsed + .map((source) => source.sourceName) + .filter((name): name is string => name !== undefined); + }); + + return { cameras, sources, columns }; +} + +export { + buildFrameAlignmentIndex, + resolveCameras, +}; + +export type { + CameraCandidateTexts, + CameraFrameAlignmentIndexes, + FrameAlignmentIndex, + ResolvedFrameMetadata, +}; diff --git a/client/dive-common/store/context.ts b/client/dive-common/store/context.ts index c6ee24ff6..2a12e096e 100644 --- a/client/dive-common/store/context.ts +++ b/client/dive-common/store/context.ts @@ -6,7 +6,7 @@ import GroupSidebar from 'dive-common/components/GroupSidebar.vue'; import AttributesSideBar from 'dive-common/components/Attributes/AttributesSideBar.vue'; import MultiCamTools from 'dive-common/components/MultiCamTools.vue'; import AttributeTrackFilters from 'vue-media-annotator/components/AttributeTrackFilters.vue'; -import DatasetInfo from 'dive-common/components/DatasetInfo.vue'; +import DatasetInfo from 'dive-common/components/DatasetInfo/DatasetInfo.vue'; interface ContextState { last: string; @@ -20,7 +20,7 @@ interface ComponentMapItem { } const state: ContextState = reactive({ - last: 'TypeThreshold', + last: 'DatasetInfo', active: null, subCategory: null, }); @@ -67,7 +67,7 @@ function unregister(item: ComponentMapItem) { } function resetActive() { - state.last = 'TypeThreshold'; + state.last = 'DatasetInfo'; state.active = null; } diff --git a/client/dive-common/use/index.ts b/client/dive-common/use/index.ts index e35d1597e..3c6fa0349 100644 --- a/client/dive-common/use/index.ts +++ b/client/dive-common/use/index.ts @@ -2,10 +2,12 @@ import useModeManager from './useModeManager'; import useSave from './useSave'; import useRequest from './useRequest'; import { useLassoMode } from './useLassoMode'; +import { useFrameMetadata } from './useFrameMetadata'; export { useModeManager, useRequest, useSave, useLassoMode, + useFrameMetadata, }; diff --git a/client/dive-common/use/useFrameMetadata.spec.ts b/client/dive-common/use/useFrameMetadata.spec.ts new file mode 100644 index 000000000..8effa510a --- /dev/null +++ b/client/dive-common/use/useFrameMetadata.spec.ts @@ -0,0 +1,350 @@ +import { nextTick, ref } from 'vue'; + +// eslint-disable-next-line import/no-extraneous-dependencies -- Vitest is only used in tests +import { + beforeEach, describe, expect, it, vi, +} from 'vitest'; + +import type { FrameMetadataSourcesResponse } from '../apispec'; +import { __resetFrameMetadataSessionCache, useFrameMetadata } from './useFrameMetadata'; + +// Drain Vue's watcher scheduler and the promise/microtask + macrotask queues so a dataset switch, +// an async source load/resolve, and any lazy per-camera pass all settle before we assert. +async function settle() { + for (let i = 0; i < 4; i += 1) { + // eslint-disable-next-line no-await-in-loop + await nextTick(); + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setTimeout(resolve, 0); }); + } +} + +describe('useFrameMetadata', () => { + // The module-level session cache (FIX 6) is intentionally global; reset it so no dataset id or + // resolved payload from one test's mocks leaks into the next. + beforeEach(() => { + __resetFrameMetadataSessionCache(); + }); + + it('discards a stale response after the dataset switches (stale-response token)', async () => { + let resolveFirst: (payload: FrameMetadataSourcesResponse) => void = () => {}; + const first = new Promise((resolve) => { resolveFirst = resolve; }); + const second: FrameMetadataSourcesResponse = { + cameras: { + singleCam: [{ + name: 'frame_metadata.csv', + text: 'filename,label\nimg001.png,second\n', + }], + }, + }; + const loadFrameMetadata = vi.fn() + .mockReturnValueOnce(first) + .mockResolvedValueOnce(second); + const getCameraMediaNames = vi.fn((camera: string) => ( + camera === 'singleCam' ? ['img001.png'] : undefined + )); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('singleCam'); + const metadata = useFrameMetadata({ + datasetId, frame, selectedCamera, loadFrameMetadata, getCameraMediaNames, + }); + + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(metadata.loading.value).toBe(true); + + datasetId.value = 'dataset-b'; + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(2); + expect(metadata.loading.value).toBe(false); + expect(metadata.currentEntries.value).toEqual([['filename', 'img001.png'], ['label', 'second']]); + + // The stale dataset-a response arrives late; its token no longer matches, so it is ignored. + resolveFirst({ + cameras: { + singleCam: [{ + name: 'frame-metadata.txt', + text: 'filename,label\nimg001.png,stale\n', + }], + }, + }); + await settle(); + expect(metadata.currentEntries.value).toEqual([['filename', 'img001.png'], ['label', 'second']]); + expect(metadata.currentSources.value).toEqual(['frame_metadata.csv']); + expect(metadata.error.value).toBeNull(); + }); + + it('negative-caches an empty source listing and never refetches until a dataset switch', async () => { + const loadFrameMetadata = vi.fn(async () => ({ cameras: {} })); + const getCameraMediaNames = vi.fn(() => [] as string[]); + + const datasetId = ref('dataset-a'); + const frame = ref(10); + const selectedCamera = ref('singleCam'); + const metadata = useFrameMetadata({ + datasetId, + frame, + selectedCamera, + loadFrameMetadata, + getCameraMediaNames, + }); + + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(metadata.hasMetadataSource.value).toBe(false); + expect(metadata.currentEntries.value).toEqual([]); + + // Scrubbing frames and switching cameras on the same dataset must not refetch. + frame.value = 500; + await settle(); + frame.value = 5000; + await settle(); + selectedCamera.value = 'starboard'; + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + + // A dataset switch re-runs source loading. + datasetId.value = 'dataset-b'; + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(2); + expect(loadFrameMetadata).toHaveBeenLastCalledWith('dataset-b'); + }); + + it('resolves sidecars against the media list and exposes the current frame row', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + cameras: { + singleCam: [{ + name: 'frame_metadata.csv', + text: 'filename,depth\nimg001.png,10\nimg002.png,12\n', + }], + }, + })); + const getCameraMediaNames = vi.fn((camera: string) => ( + camera === 'singleCam' ? ['img001.png', 'img002.png'] : undefined + )); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('singleCam'); + const metadata = useFrameMetadata({ + datasetId, + frame, + selectedCamera, + loadFrameMetadata, + getCameraMediaNames, + }); + + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(metadata.hasMetadataSource.value).toBe(true); + expect(metadata.currentSources.value).toEqual(['frame_metadata.csv']); + expect(metadata.currentEntries.value).toEqual([['filename', 'img001.png'], ['depth', '10']]); + + // Scrubbing re-materializes the row lazily from held data, with no refetch. + frame.value = 1; + await nextTick(); + expect(metadata.currentEntries.value).toEqual([['filename', 'img002.png'], ['depth', '12']]); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + + // A frame with no matching row shows nothing (empty-state), not blank columns. + frame.value = 2; + await nextTick(); + expect(metadata.currentEntries.value).toEqual([]); + }); + + it('resolves a multicam camera lazily when its media list loads after selection', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + cameras: { + port: [{ + name: 'frame_metadata.csv', + text: 'filename,depth\nport001.png,10\n', + }], + starboard: [{ + name: 'frame-metadata.txt', + text: 'filename,depth\nstar001.png,20\n', + }], + }, + })); + // starboard's ordered media list is not available until it is selected. + const media: Record = { + port: ['port001.png'], + starboard: undefined, + }; + const getCameraMediaNames = vi.fn((camera: string) => media[camera]); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('port'); + const metadata = useFrameMetadata({ + datasetId, + frame, + selectedCamera, + loadFrameMetadata, + getCameraMediaNames, + }); + + await settle(); + // port resolved eagerly; starboard deferred because its media list was not yet available. + expect(metadata.currentEntries.value).toEqual([['filename', 'port001.png'], ['depth', '10']]); + + // The media list arrives and the user selects starboard: it resolves on selection. + media.starboard = ['star001.png']; + selectedCamera.value = 'starboard'; + await settle(); + expect(metadata.hasMetadataSource.value).toBe(true); + expect(metadata.currentSources.value).toEqual(['frame-metadata.txt']); + expect(metadata.currentEntries.value).toEqual([['filename', 'star001.png'], ['depth', '20']]); + // No source refetch across the whole interaction. + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + }); + + it('follows the active camera for currentSources and hasMetadataSource', async () => { + const loadFrameMetadata = vi.fn(async (): Promise => ({ + cameras: { + port: [ + { name: 'frame_metadata.csv', text: 'filename,latitude\nport001.png,58.10\n' }, + { name: 'frame-metadata.txt', text: 'filename,latitude\nport001.png,99.00\n' }, + ], + starboard: [ + { name: 'frame_metadata.csv', text: 'filename,latitude\nstar001.png,59.10\n' }, + ], + }, + })); + const getCameraMediaNames = vi.fn((camera: string) => ({ + port: ['port001.png'], + starboard: ['star001.png'], + }[camera])); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('port'); + const metadata = useFrameMetadata({ + datasetId, frame, selectedCamera, loadFrameMetadata, getCameraMediaNames, + }); + + await settle(); + expect(metadata.currentSources.value).toEqual(['frame_metadata.csv', 'frame-metadata.txt']); + expect(metadata.hasMetadataSource.value).toBe(true); + expect(metadata.currentEntries.value).toEqual([['filename', 'port001.png'], ['latitude', '58.10']]); + + selectedCamera.value = 'starboard'; + await nextTick(); + expect(metadata.currentSources.value).toEqual(['frame_metadata.csv']); + expect(metadata.hasMetadataSource.value).toBe(true); + expect(metadata.currentEntries.value).toEqual([['filename', 'star001.png'], ['latitude', '59.10']]); + + selectedCamera.value = 'stern'; + await nextTick(); + expect(metadata.currentSources.value).toEqual([]); + expect(metadata.hasMetadataSource.value).toBe(false); + expect(metadata.currentEntries.value).toEqual([]); + }); + + it('resolves the single camera once its initially-empty media list populates (reactive retry)', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + cameras: { + singleCam: [{ + name: 'frame_metadata.csv', + text: 'filename,depth\nimg001.png,10\n', + }], + }, + })); + // Mirrors Viewer.vue: imageData (and so getCameraMediaNames) starts at `[]`, not `undefined`. + const media = ref([]); + const getCameraMediaNames = vi.fn((camera: string) => ( + camera === 'singleCam' ? media.value : undefined + )); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('singleCam'); + const metadata = useFrameMetadata({ + datasetId, + frame, + selectedCamera, + loadFrameMetadata, + getCameraMediaNames, + }); + + await settle(); + // The sidecar source is loaded, but an empty media list must defer, not claim-and-drop it. + expect(metadata.hasMetadataSource.value).toBe(false); + expect(metadata.hasSidecarItems.value).toBe(true); + + // The media list populates later, with no dataset/camera change; the reactive retry resolves. + media.value = ['img001.png']; + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(metadata.hasMetadataSource.value).toBe(true); + expect(metadata.currentEntries.value).toEqual([['filename', 'img001.png'], ['depth', '10']]); + }); + + it('exposes hasSidecarItems for a present sidecar whose rows match no media filename', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + cameras: { + singleCam: [{ + name: 'frame_metadata.csv', + text: 'filename,depth\nother001.png,10\n', + }], + }, + })); + const getCameraMediaNames = vi.fn((camera: string) => ( + camera === 'singleCam' ? ['img001.png'] : undefined + )); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('singleCam'); + const metadata = useFrameMetadata({ + datasetId, + frame, + selectedCamera, + loadFrameMetadata, + getCameraMediaNames, + }); + + await settle(); + expect(metadata.hasSidecarItems.value).toBe(true); + expect(metadata.sidecarSourceNames.value).toEqual(['frame_metadata.csv']); + expect(metadata.hasMetadataSource.value).toBe(false); + expect(metadata.currentEntries.value).toEqual([]); + }); + + it('reuses the module-level session cache across composable instances (panel remount)', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + cameras: { + singleCam: [{ + name: 'frame_metadata.csv', + text: 'filename,depth\nimg001.png,10\n', + }], + }, + })); + const getCameraMediaNames = vi.fn((camera: string) => ( + camera === 'singleCam' ? ['img001.png'] : undefined + )); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('singleCam'); + const options = { + datasetId, frame, selectedCamera, loadFrameMetadata, getCameraMediaNames, + }; + + const first = useFrameMetadata(options); + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(first.hasMetadataSource.value).toBe(true); + + // SidebarContext.vue mounts DatasetInfo with v-if: closing/reopening the panel destroys and + // re-creates this composable. A second instance for the same dataset must hydrate from the + // module cache rather than reloading the sidecar. + const second = useFrameMetadata(options); + await settle(); + + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(second.hasMetadataSource.value).toBe(true); + expect(second.currentEntries.value).toEqual([['filename', 'img001.png'], ['depth', '10']]); + }); +}); diff --git a/client/dive-common/use/useFrameMetadata.ts b/client/dive-common/use/useFrameMetadata.ts new file mode 100644 index 000000000..687d94cac --- /dev/null +++ b/client/dive-common/use/useFrameMetadata.ts @@ -0,0 +1,283 @@ +import { + computed, readonly, ref, watch, +} from 'vue'; +import type { Ref } from 'vue'; +import { getResponseError } from 'vue-media-annotator/utils'; + +import type { + FrameMetadataSourceText, + FrameMetadataSourcesResponse, + ResolvedFrameMetadata, +} from 'dive-common/apispec'; +import { buildFrameAlignmentIndex, resolveCameras } from 'dive-common/frameMetadata/resolve'; + +export interface UseFrameMetadataOptions { + /** Current dataset id (parent-root id for multicam). */ + datasetId: Readonly>; + /** Current playhead frame number. Drives `currentEntries`; never triggers a fetch. */ + frame: Readonly>; + /** Active camera key (`singleCam` for single-camera datasets). */ + selectedCamera: Readonly>; + /** Ordered image filenames for a camera, where frame number is the array index. */ + getCameraMediaNames?: (camera: string) => string[] | undefined; + loadFrameMetadata?: (datasetId: string) => Promise; +} + +/** The compact, session-cacheable slice of a resolved dataset's state (no raw sidecar text). */ +interface FrameMetadataCacheEntry { + cameras: ResolvedFrameMetadata['cameras']; + sources: ResolvedFrameMetadata['sources']; + columns: ResolvedFrameMetadata['columns']; + sourceNamesByCamera: Record; + resolvedCameras: Set; + sourcesLoaded: boolean; +} + +// Cache the last resolved dataset so a panel remount can render without refetching sidecars. +let sessionCache: { datasetId: string; entry: FrameMetadataCacheEntry } | null = null; + +export function __resetFrameMetadataSessionCache() { + sessionCache = null; +} + +export function useFrameMetadata({ + datasetId, + frame, + selectedCamera, + getCameraMediaNames, + loadFrameMetadata, +}: UseFrameMetadataOptions) { + const cameras = ref({}); + const sources = ref({}); + const columns = ref({}); + const loading = ref(false); + const error = ref(null); + + let token = 0; + let loadedDatasetId: string | null = null; + + const sourceNamesByCamera = ref>({}); + let sourceTextsByCamera: Record = {}; + let sourcesLoaded = false; + const resolvedCameras = new Set(); + + let inFlight = 0; + function beginWork() { + inFlight += 1; + loading.value = true; + } + function endWork(reqToken: number) { + inFlight = Math.max(0, inFlight - 1); + if (inFlight === 0 && reqToken === token) { + loading.value = false; + } + } + + function reset() { + cameras.value = {}; + sources.value = {}; + columns.value = {}; + error.value = null; + loading.value = false; + inFlight = 0; + sourceNamesByCamera.value = {}; + sourceTextsByCamera = {}; + sourcesLoaded = false; + resolvedCameras.clear(); + } + + function mergeResolved(result: ResolvedFrameMetadata) { + cameras.value = { ...cameras.value, ...result.cameras }; + sources.value = { ...sources.value, ...result.sources }; + columns.value = { ...columns.value, ...result.columns }; + } + + function syncSessionCache() { + if (loadedDatasetId === null) { + return; + } + sessionCache = { + datasetId: loadedDatasetId, + entry: { + cameras: { ...cameras.value }, + sources: { ...sources.value }, + columns: { ...columns.value }, + sourceNamesByCamera: { ...sourceNamesByCamera.value }, + resolvedCameras: new Set(resolvedCameras), + sourcesLoaded, + }, + }; + } + + function hydrateFromCache(entry: FrameMetadataCacheEntry) { + cameras.value = { ...entry.cameras }; + sources.value = { ...entry.sources }; + columns.value = { ...entry.columns }; + sourceNamesByCamera.value = { ...entry.sourceNamesByCamera }; + resolvedCameras.clear(); + entry.resolvedCameras.forEach((camera) => resolvedCameras.add(camera)); + sourcesLoaded = entry.sourcesLoaded; + } + + function activeCameraNeedsSourceText() { + return ( + (sourceNamesByCamera.value[selectedCamera.value]?.length ?? 0) > 0 + && !resolvedCameras.has(selectedCamera.value) + && sourceTextsByCamera[selectedCamera.value] === undefined + ); + } + + function resolveCamera( + camera: string, + requestToken: number, + ) { + if (resolvedCameras.has(camera)) { + return; + } + const texts = sourceTextsByCamera[camera]; + if (texts === undefined || texts.length === 0) { + return; + } + const mediaNames = getCameraMediaNames?.(camera); + if (mediaNames === undefined || mediaNames.length === 0) { + return; + } + if (requestToken !== token) { + return; + } + resolvedCameras.add(camera); + try { + const candidates = texts.map((item): [string, string] => [item.name, item.text]); + const index = buildFrameAlignmentIndex(mediaNames); + mergeResolved(resolveCameras({ [camera]: candidates }, { [camera]: index })); + syncSessionCache(); + } catch (err) { + resolvedCameras.delete(camera); + if (requestToken === token) { + error.value = getResponseError(err); + } + } + } + + async function runLoad(id: string, requestToken: number) { + if (loadFrameMetadata === undefined) { + return; + } + beginWork(); + try { + const response = await loadFrameMetadata(id); + if (requestToken !== token) { + return; + } + sourceTextsByCamera = response.cameras ?? {}; + sourceNamesByCamera.value = Object.fromEntries( + Object.entries(sourceTextsByCamera).map(([camera, items]) => ( + [camera, items.map((item) => item.name)] + )), + ); + sourcesLoaded = true; + Object.keys(sourceTextsByCamera).forEach((camera) => resolveCamera(camera, requestToken)); + if (requestToken === token) { + syncSessionCache(); + } + } catch (err) { + if (requestToken === token) { + error.value = getResponseError(err); + } + } finally { + endWork(requestToken); + } + } + + async function resolveActiveCamera(id: string, requestToken: number) { + if (activeCameraNeedsSourceText() && loadFrameMetadata !== undefined) { + await runLoad(id, requestToken); + } else if (sourcesLoaded) { + resolveCamera(selectedCamera.value, requestToken); + } + } + + async function ensure() { + const id = datasetId.value; + if (!id) { + if (loadedDatasetId !== null) { + token += 1; + loadedDatasetId = null; + reset(); + } + return; + } + if (id !== loadedDatasetId) { + token += 1; + loadedDatasetId = id; + reset(); + const requestToken = token; + const cached = sessionCache !== null && sessionCache.datasetId === id + ? sessionCache.entry + : null; + if (cached) { + hydrateFromCache(cached); + await resolveActiveCamera(id, requestToken); + } else if (loadFrameMetadata !== undefined) { + await runLoad(id, requestToken); + } + return; + } + await resolveActiveCamera(id, token); + } + + watch( + [datasetId, selectedCamera], + () => { ensure(); }, + { immediate: true }, + ); + + // The viewer can expose an empty media list before filenames finish loading. + const activeCameraMediaCount = computed(() => ( + getCameraMediaNames ? (getCameraMediaNames(selectedCamera.value)?.length ?? 0) : 0 + )); + watch(activeCameraMediaCount, (count, prev) => { + if ( + count > 0 + && (prev ?? 0) === 0 + && sourcesLoaded + && datasetId.value === loadedDatasetId + && !resolvedCameras.has(selectedCamera.value) + ) { + ensure(); + } + }); + + const currentEntries = computed<[string, string][]>(() => { + const camera = selectedCamera.value; + const cols = columns.value[camera]; + const row = cameras.value[camera]?.[frame.value]; + if (cols === undefined || row === undefined) { + return []; + } + return cols.map((column, i): [string, string] => [column, row[i] ?? '']); + }); + + const currentSources = computed(() => sources.value[selectedCamera.value] ?? []); + const hasMetadataSource = computed(() => selectedCamera.value in sources.value); + + const hasSidecarItems = computed(() => ( + (sourceNamesByCamera.value[selectedCamera.value]?.length ?? 0) > 0 + )); + const sidecarSourceNames = computed(() => ( + sourceNamesByCamera.value[selectedCamera.value] ?? [] + )); + + return { + currentEntries, + currentSources, + hasMetadataSource, + hasSidecarItems, + sidecarSourceNames, + loading: readonly(loading), + error: readonly(error), + ensure, + }; +} + +export type UseFrameMetadata = ReturnType; diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index c8b01ab7b..efbf38095 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -222,6 +222,11 @@ export default function register() { return ret; }); + ipcMain.handle('load-frame-metadata', async ( + event, + { datasetId }: { datasetId: string }, + ) => common.loadFrameMetadata(settings.get(), datasetId)); + ipcMain.handle('import-multicam-media', async (event, { args }: { args: MultiCamImportArgs }) => { const ret = await beginMultiCamImport(args); diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 4598d323c..ab6beaf49 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -259,6 +259,113 @@ beforeEach(() => { 'file1.csv': '', 'file2.csv': '', }, + frameMetadataSource: { + 'image_0001.jpg': '', + 'image_0002.jpg': '', + 'image_0003.jpg': '', + 'frame-metadata.txt': [ + 'filename,depth,temperature', + 'image_0001.jpg,192.80,4.0', + 'image_0002.jpg,193.10,4.1', + 'image_0003.jpg,193.40,4.2', + '', + ].join('\n'), + }, + // Two sidecars in one folder: name-sorted TXT wins per-frame over CSV; CSV + // introduces a new column ('temperature') and a new frame (image_0003). + frameMetadataPrecedence: { + 'image_0001.jpg': '', + 'image_0002.jpg': '', + 'image_0003.jpg': '', + 'frame-metadata.txt': [ + 'filename,depth', + 'image_0001.jpg,10', + 'image_0002.jpg,20', + '', + ].join('\n'), + 'frame_metadata.csv': [ + 'filename,depth,temperature', + 'image_0001.jpg,99,5.0', + 'image_0003.jpg,40,7.0', + '', + ].join('\n'), + }, + frameMetadataDoubleExt: { + 'photo.jpg.png': '', + 'frame_metadata.csv': [ + 'filename,depth', + 'photo.jpg.png,10', + '', + ].join('\n'), + }, + frameMetadataNoSource: { + 'image_0001.jpg': '', + 'notes.txt': 'note,value\nhello,world\n', + }, + // Duplicate image basename across subfolders: the read path keys last-wins instead of + // throwing like the import-path validator. + frameMetadataDupSource: { + 'frame_metadata.csv': [ + 'filename,depth', + 'image_0001.jpg,10', + 'image_0002.jpg,20', + '', + ].join('\n'), + }, + // Multicam whose root (originalBasePath) is also the 'left' camera's media dir; the shared + // sidecar must be listed once for 'left' and 'right' names other images so it joins nothing. + multicamRootDedup: { + 'frame_metadata.csv': [ + 'filename,depth', + 'img_l1.jpg,100', + 'img_l2.jpg,200', + '', + ].join('\n'), + right: { + 'img_r1.jpg': '', + }, + }, + // Multicam whose parent root holds one wide sidecar naming both cameras' images; each + // camera's media lives in its own subfolder, so the shared root file binds both cameras. + multicamSharedRoot: { + 'frame_metadata.csv': [ + 'filename,depth', + 'img_l1.jpg,100', + 'img_l2.jpg,110', + 'img_r1.jpg,300', + 'img_r2.jpg,310', + '', + ].join('\n'), + left: { + 'img_l1.jpg': '', + 'img_l2.jpg': '', + }, + right: { + 'img_r1.jpg': '', + 'img_r2.jpg': '', + }, + }, + // Import-gate fixtures. + fmGateMixed: { + // Sorts first but is declared frame metadata: skipped as a track-file candidate, left on disk. + 'frame_metadata.csv': 'filename,depth\nimage_0001.jpg,10\n', + 'zzz_annotations.csv': '# comment line\n# metadata,fps: 32,"whatever"\n#comment line', + }, + fmGateFrameMetadataOnly: { + 'frame_metadata.csv': 'filename,depth\nimage_0001.jpg,10\n', + }, + fmGateExplicit: { + 'frame_metadata.csv': 'filename,depth\nimage_0001.jpg,10\n', + }, + // A plain CSV (not declared) whose contents fail to parse as VIAME annotations: the error + // gains the rename hint. The unterminated quote makes csv-parse reject. + fmGateViameFail: { + 'nav.csv': 'filename,depth\n"unterminated,10\n', + }, + // No size gate: a declared sidecar over the old 10 MB bound is discovered without being read. + frameMetadataSized: { + 'frame_metadata.csv': 'filename,depth\nimage_0001.jpg,10\n', + }, }, '/home/user/viamedata': { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -313,6 +420,135 @@ beforeEach(() => { 'result_whatever.json': JSON.stringify({}), auxiliary: {}, }, + projectidFrameMetadata: { + 'meta.json': JSON.stringify({ + version: 1, + id: 'projectidFrameMetadata', + type: 'image-sequence', + fps: 5, + originalBasePath: '/home/user/data/frameMetadataSource', + originalImageFiles: [ + 'image_0001.jpg', + 'image_0002.jpg', + 'image_0003.jpg', + ], + }), + 'result_whatever.json': JSON.stringify({}), + auxiliary: {}, + }, + projectidFrameMetadataPrecedence: { + 'meta.json': JSON.stringify({ + version: 1, + id: 'projectidFrameMetadataPrecedence', + type: 'image-sequence', + fps: 5, + originalBasePath: '/home/user/data/frameMetadataPrecedence', + originalImageFiles: [ + 'image_0001.jpg', + 'image_0002.jpg', + 'image_0003.jpg', + ], + }), + 'result_whatever.json': JSON.stringify({}), + auxiliary: {}, + }, + projectidFrameMetadataNoSource: { + 'meta.json': JSON.stringify({ + version: 1, + id: 'projectidFrameMetadataNoSource', + type: 'image-sequence', + fps: 5, + originalBasePath: '/home/user/data/frameMetadataNoSource', + originalImageFiles: [ + 'image_0001.jpg', + ], + }), + 'result_whatever.json': JSON.stringify({}), + auxiliary: {}, + }, + projectidFrameMetadataDoubleExt: { + 'meta.json': JSON.stringify({ + version: 1, + id: 'projectidFrameMetadataDoubleExt', + type: 'image-sequence', + fps: 5, + originalBasePath: '/home/user/data/frameMetadataDoubleExt', + originalImageFiles: [ + 'photo.jpg.png', + ], + }), + 'result_whatever.json': JSON.stringify({}), + auxiliary: {}, + }, + projectidFrameMetadataDup: { + 'meta.json': JSON.stringify({ + version: 1, + id: 'projectidFrameMetadataDup', + type: 'image-sequence', + fps: 5, + originalBasePath: '/home/user/data/frameMetadataDupSource', + // Two files share the basename image_0001.jpg (different subfolders). + originalImageFiles: [ + 'a/image_0001.jpg', + 'b/image_0001.jpg', + 'a/image_0002.jpg', + ], + }), + 'result_whatever.json': JSON.stringify({}), + auxiliary: {}, + }, + projectidMulticamRootDedup: { + 'meta.json': JSON.stringify({ + version: 1, + id: 'projectidMulticamRootDedup', + type: 'multi', + fps: 5, + originalBasePath: '/home/user/data/multicamRootDedup', + multiCam: { + defaultDisplay: 'left', + cameras: { + left: { + type: 'image-sequence', + originalBasePath: '/home/user/data/multicamRootDedup', + originalImageFiles: ['img_l1.jpg', 'img_l2.jpg'], + }, + right: { + type: 'image-sequence', + originalBasePath: '/home/user/data/multicamRootDedup/right', + originalImageFiles: ['img_r1.jpg'], + }, + }, + }, + }), + 'result_whatever.json': JSON.stringify({}), + auxiliary: {}, + }, + projectidMulticamSharedRoot: { + 'meta.json': JSON.stringify({ + version: 1, + id: 'projectidMulticamSharedRoot', + type: 'multi', + fps: 5, + originalBasePath: '/home/user/data/multicamSharedRoot', + multiCam: { + defaultDisplay: 'left', + cameras: { + left: { + type: 'image-sequence', + originalBasePath: '/home/user/data/multicamSharedRoot/left', + originalImageFiles: ['img_l1.jpg', 'img_l2.jpg'], + }, + right: { + type: 'image-sequence', + originalBasePath: '/home/user/data/multicamSharedRoot/right', + originalImageFiles: ['img_r1.jpg', 'img_r2.jpg'], + }, + }, + }, + }), + 'result_whatever.json': JSON.stringify({}), + auxiliary: {}, + }, projectid2Bad: { 'meta.json': '{}', // Won't match @@ -855,6 +1091,147 @@ describe('native.common', () => { }); }); +describe('frame metadata read path (source text loading)', () => { + it('loads a single-camera sidecar as source text', async () => { + const data = await common.loadFrameMetadata(settings, 'projectidFrameMetadata'); + expect(data.cameras.singleCam.map((source) => source.name)).toEqual(['frame-metadata.txt']); + expect(data.cameras.singleCam[0].text).toContain('image_0001.jpg'); + expect(data.cameras.singleCam[0].text).toContain('depth'); + }); + + it('loads two sidecars in precedence order', async () => { + const data = await common.loadFrameMetadata(settings, 'projectidFrameMetadataPrecedence'); + expect(data.cameras.singleCam.map((source) => source.name)) + .toEqual(['frame-metadata.txt', 'frame_metadata.csv']); + expect(data.cameras.singleCam[0].text).toContain('image_0001.jpg'); + expect(data.cameras.singleCam[1].text).toContain('temperature'); + }); + + it('omits a camera whose only text file is not a declared sidecar', async () => { + await expect(common.loadFrameMetadata(settings, 'projectidFrameMetadataNoSource')) + .resolves.toEqual({ cameras: {} }); + }); + + it('loads a sidecar that names double-extension media by full name', async () => { + const data = await common.loadFrameMetadata(settings, 'projectidFrameMetadataDoubleExt'); + expect(data.cameras.singleCam.map((source) => source.name)).toEqual(['frame_metadata.csv']); + expect(data.cameras.singleCam[0].text).toContain('photo.jpg.png'); + }); + + it('loads a sidecar for duplicate image basenames without validating media keys', async () => { + const data = await common.loadFrameMetadata(settings, 'projectidFrameMetadataDup'); + expect(data.cameras.singleCam.map((source) => source.name)).toEqual(['frame_metadata.csv']); + expect(data.cameras.singleCam[0].text).toContain('image_0001.jpg'); + }); + + it('loads a shared multicam root sidecar once per camera candidate list', async () => { + const data = await common.loadFrameMetadata(settings, 'projectidMulticamRootDedup'); + expect(data.cameras.left.map((source) => source.name)).toEqual(['frame_metadata.csv']); + expect(data.cameras.right.map((source) => source.name)).toEqual(['frame_metadata.csv']); + expect(data.cameras.left[0].text).toContain('img_l1.jpg'); + }); + + it('loads both cameras from one shared parent-root sidecar', async () => { + const data = await common.loadFrameMetadata(settings, 'projectidMulticamSharedRoot'); + expect(data.cameras.left.map((source) => source.name)).toEqual(['frame_metadata.csv']); + expect(data.cameras.right.map((source) => source.name)).toEqual(['frame_metadata.csv']); + expect(data.cameras.left[0].text).toContain('img_l1.jpg'); + expect(data.cameras.right[0].text).toContain('img_r1.jpg'); + }); + + it('returns empty maps for a non-image-sequence (video) dataset', async () => { + await expect(common.loadFrameMetadata(settings, 'projectid1VideoGood')) + .resolves.toEqual({ cameras: {} }); + }); +}); + +describe('frame metadata discovery', () => { + it('lists a declared sidecar larger than the old 10 MB bound without reading it', async () => { + // 10 MB + 1 byte: over the old (deleted) size cap. A declared sidecar is never size-gated and + // is only stat'd during discovery, not read. + const dir = '/home/user/data/frameMetadataSized'; + await fs.writeFile(npath.join(dir, 'frame_metadata.csv'), Buffer.alloc(10 * 1024 * 1024 + 1, 0x61)); + + const candidates = await common.frameMetadataCandidateDescriptors(dir); + const names = candidates.map((candidate) => candidate.name); + expect(names).toContain('frame_metadata.csv'); + const huge = candidates.find((candidate) => candidate.name === 'frame_metadata.csv'); + expect(huge?.absolutePath).toBe(npath.join(dir, 'frame_metadata.csv')); + }); + + it('frameMetadataSourceDirectories returns the base path for a directory import', () => { + expect(common.frameMetadataSourceDirectories({ + originalBasePath: '/data/set', + originalImageFiles: ['img001.png'], + })).toEqual([npath.resolve('/data/set')]); + }); + + it('frameMetadataSourceDirectories lists the image-list directory then the absolute image directory', () => { + expect(common.frameMetadataSourceDirectories({ + originalBasePath: '', + originalImageFiles: ['/imgs/a.png'], + imageListPath: '/lists/list.txt', + })).toEqual([npath.resolve('/lists'), npath.resolve('/imgs')]); + }); + + it('frameMetadataSourceDirectories lists every absolute image directory from image-list entries', () => { + expect(common.frameMetadataSourceDirectories({ + originalBasePath: '', + originalImageFiles: ['/imgs/a.png', '/extra/b.png', '/imgs/c.png'], + imageListPath: '/lists/list.txt', + })).toEqual([npath.resolve('/lists'), npath.resolve('/imgs'), npath.resolve('/extra')]); + }); + + it('frameMetadataSourceDirectories dedupes directories that resolve to the same path', () => { + expect(common.frameMetadataSourceDirectories({ + originalBasePath: '/data', + originalImageFiles: ['/data/a.png'], + imageListPath: '/data/list.txt', + })).toEqual([npath.resolve('/data')]); + }); + + it('frameMetadataSourceDirectories ignores a relative first image (covered by the base path)', () => { + expect(common.frameMetadataSourceDirectories({ + originalBasePath: '/base', + originalImageFiles: ['sub/a.png'], + })).toEqual([npath.resolve('/base')]); + }); +}); + +describe('frame metadata import gates', () => { + it('picks the annotation CSV and leaves a declared frame metadata sidecar in place', async () => { + // 'frame_metadata.csv' sorts first but is declared frame metadata: it must be skipped as a + // track-file candidate and stay on disk for read-time discovery. + const dir = '/home/user/data/fmGateMixed'; + const { trackFileAbsPath } = await common.findTrackandMetaFileinFolder(dir); + expect(trackFileAbsPath).toBe(npath.join(dir, 'zzz_annotations.csv')); + expect(fs.existsSync(npath.join(dir, 'frame_metadata.csv'))).toBe(true); + }); + + it('leaves no track file when only a declared frame metadata sidecar is present', async () => { + const { trackFileAbsPath } = await common.findTrackandMetaFileinFolder( + '/home/user/data/fmGateFrameMetadataOnly', + ); + expect(trackFileAbsPath).toBeFalsy(); + }); + + it('rejects an explicit import of a frame metadata file', async () => { + await expect(common.ingestDataFiles( + settings, + 'projectid1', + ['/home/user/data/fmGateExplicit/frame_metadata.csv'], + )).rejects.toThrow(/frame metadata file/); + }); + + it('adds a rename hint when a plain frame-metadata-shaped CSV fails VIAME import', async () => { + await expect(common.ingestDataFiles( + settings, + 'projectid1', + ['/home/user/data/fmGateViameFail/nav.csv'], + )).rejects.toThrow(/rename it to frame-metadata\.csv/); + }); +}); + afterEach(() => { mockfs.restore(); }); diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 6939811e0..c5507df89 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -30,7 +30,10 @@ import { PipeMetadata, PipelineParamType, DatasetCalibrationResult, + FrameMetadataSourcesResponse, } from 'dive-common/apispec'; +import isFrameMetadataSourceName from 'dive-common/frameMetadata/naming'; +import { parentDatasetId } from 'dive-common/compositeDatasetId'; import * as viameSerializers from 'platform/desktop/backend/serializers/viame'; import * as nistSerializers from 'platform/desktop/backend/serializers/nist'; import * as dive from 'platform/desktop/backend/serializers/dive'; @@ -439,6 +442,215 @@ async function loadDetections(settings: Settings, datasetId: string) { return loadAnnotationFile(projectDirData.trackFileAbsPath); } +/** + * Frame-metadata read path. Declared frame metadata sidecars are discovered on disk and read here. + * Parsing and resolving against media filenames happens in the renderer via the shared TypeScript + * resolver (dive-common/frameMetadata). Nothing derived is persisted; the sidecar the user dropped + * is the only stored form. + */ +interface ImageSequenceFrameMetadataSource { + originalBasePath: string; + originalImageFiles: string[]; + imageListPath?: string; +} + +// A declared sidecar resolved to its on-disk identity (no read yet). There is no parse/read cache +// in this design: the resolver is called once per request and reads texts lazily. +interface FrameMetadataCandidate { + name: string; + absolutePath: string; +} + +// Directories that may hold a camera's sidecars: the media base path, the image-list directory, +// and any absolute image-list entry directories. Deduped, order-preserving. +function frameMetadataSourceDirectories(source: ImageSequenceFrameMetadataSource): string[] { + const directories = [ + source.originalBasePath, + source.imageListPath ? npath.dirname(source.imageListPath) : '', + // Only absolute image-list entries reveal extra directories; relative names are + // resolved against originalBasePath, already covered above. + ...source.originalImageFiles + .filter((image) => npath.isAbsolute(image)) + .map((image) => npath.dirname(image)), + ]; + // uniq keeps first-occurrence order, so folder precedence (base path first) is preserved. + return uniq(directories.filter(Boolean).map((directory) => npath.resolve(directory))); +} + +// Discover declared sidecars in one directory: name-filtered (isFrameMetadataSourceName), +// case-insensitively sorted so first-wins precedence within a folder is deterministic and matches +// the server's sorted order, and never size-gated (no-cap product decision). Non-file entries and +// unreadable entries are skipped so one bad sidecar never sinks the scan. +async function frameMetadataCandidateDescriptors( + directory: string | null, +): Promise { + if (!directory || !(await fs.pathExists(directory))) { + return []; + } + const names = (await fs.readdir(directory)) + .filter(isFrameMetadataSourceName) + .sort((a, b) => { + const al = a.toLowerCase(); + const bl = b.toLowerCase(); + if (al < bl) { + return -1; + } + if (al > bl) { + return 1; + } + return 0; + }); + const candidates = await Promise.all(names + .map(async (name): Promise => { + try { + const absolutePath = npath.join(directory, name); + const stat = await fs.stat(absolutePath); + if (!stat.isFile()) { + return null; + } + return { name, absolutePath }; + } catch { + // A broken symlink or unreadable sidecar must not sink the whole scan. + return null; + } + })); + return candidates.filter((candidate): candidate is FrameMetadataCandidate => candidate !== null); +} + +async function gatherFrameMetadataCandidates( + directories: string[], +): Promise { + const perDirectory = await Promise.all( + directories.map((directory) => frameMetadataCandidateDescriptors(directory)), + ); + return ([] as FrameMetadataCandidate[]).concat(...perDirectory); +} + +// Read each candidate's text (memoized by path so a sidecar shared across cameras is read once), +// returning [sourceName, text] pairs in candidate (precedence) order. An unreadable candidate is +// skipped rather than sinking the others. +async function readCandidateTexts( + candidates: FrameMetadataCandidate[], + textMemo: Map, +): Promise> { + const texts = await Promise.all( + candidates.map(async (candidate): Promise<[string, string] | null> => { + try { + const memoized = textMemo.get(candidate.absolutePath); + const text = memoized ?? await fs.readFile(candidate.absolutePath, 'utf-8'); + if (memoized === undefined) { + textMemo.set(candidate.absolutePath, text); + } + return [candidate.name, text]; + } catch { + return null; + } + }), + ); + return texts.filter((entry): entry is [string, string] => entry !== null); +} + +function commonParentDirectory(paths: string[]): string | null { + const resolved = paths.filter((item) => item).map((item) => npath.resolve(item)); + if (!resolved.length) { + return null; + } + const [first, ...rest] = resolved; + const firstParts = first.split(npath.sep); + let { length } = firstParts; + rest.forEach((candidate) => { + const parts = candidate.split(npath.sep); + length = Math.min(length, parts.length); + for (let i = 0; i < length; i += 1) { + if (firstParts[i] !== parts[i]) { + length = i; + break; + } + } + }); + const prefix = firstParts.slice(0, length).join(npath.sep); + // Return null rather than the filesystem root when cameras share no common prefix, + // so an unrelated-mount multicam never scans every sidecar at '/'. + return prefix || null; +} + +async function loadMulticamFrameMetadata( + projectMetaData: JsonMeta, +): Promise { + const { multiCam } = projectMetaData; + if (!multiCam) { + return { cameras: {} }; + } + + const cameraEntries = orderedMultiCamCameraNames({ + cameras: multiCam.cameras, + defaultDisplay: multiCam.defaultDisplay, + }).map((cameraName) => [cameraName, multiCam.cameras[cameraName]] as const); + + const rootDirectory = projectMetaData.originalBasePath + || commonParentDirectory(cameraEntries.flatMap(([, camera]) => ( + frameMetadataSourceDirectories(camera) + ))); + const resolvedRoot = rootDirectory ? npath.resolve(rootDirectory) : null; + const rootCandidates = await frameMetadataCandidateDescriptors(rootDirectory); + + // Read each candidate file at most once per request, even when cameras share the parent/root. + const textMemo = new Map(); + + const cameras: FrameMetadataSourcesResponse['cameras'] = {}; + + // Cameras load independently; resolve their candidate texts in parallel. + await Promise.all(cameraEntries.map(async ([cameraName, cameraMeta]) => { + if (cameraMeta.type !== 'image-sequence') { + return; + } + // Camera-local directories precede the multicam parent/root folder (precedence). + const cameraDirectories = frameMetadataSourceDirectories(cameraMeta); + const cameraCandidates = await gatherFrameMetadataCandidates(cameraDirectories); + // Root dedup (mirror of the server's folder-id dedup): a camera whose own media dir is the + // root already gathered the root's sidecars, so do not concatenate them a second time. + const candidates = resolvedRoot !== null && !cameraDirectories.includes(resolvedRoot) + ? cameraCandidates.concat(rootCandidates) + : cameraCandidates; + const texts = await readCandidateTexts(candidates, textMemo); + if (texts.length) { + cameras[cameraName] = texts.map(([name, text]) => ({ name, text })); + } + })); + + return { cameras }; +} + +async function loadFrameMetadata( + settings: Settings, + datasetId: string, +): Promise { + const parentId = parentDatasetId(datasetId); + const projectDirData = await getValidatedProjectDir(settings, parentId); + const projectMetaData = await loadJsonMetadata(projectDirData.metaFileAbsPath); + + if (projectMetaData.type === MultiType) { + return loadMulticamFrameMetadata(projectMetaData); + } + if (projectMetaData.type !== 'image-sequence') { + return { cameras: {} }; + } + + const candidates = await gatherFrameMetadataCandidates( + frameMetadataSourceDirectories(projectMetaData), + ); + const textMemo = new Map(); + const texts = await readCandidateTexts(candidates, textMemo); + if (!texts.length) { + return { cameras: {} }; + } + return { + cameras: { + singleCam: texts.map(([name, text]) => ({ name, text })), + }, + }; +} + /** * Look through DIVE project path, find subfolders that * look like datasets, and return them. @@ -775,6 +987,11 @@ async function _ingestFilePath( if (fs.statSync(path).size === 0) { return null; } + // Frame-metadata sidecars are read from the media folder, so importing one as annotations would + // copy it into the wrong storage path. + if (isFrameMetadataSourceName(path)) { + throw new Error(`${npath.basename(path)} is a frame metadata file. Place it in the dataset's media folder; it is read automatically and cannot be imported as annotations.`); + } let warnings: string[] = []; // Make a copy of the file in aux const projectInfo = getProjectDir(settings, datasetId); @@ -807,7 +1024,13 @@ async function _ingestFilePath( } } else if (CsvFileName.test(path)) { // VIAME CSV File - const data = await viameSerializers.parseFile(path, imageMap); + let data: Awaited>; + try { + data = await viameSerializers.parseFile(path, imageMap); + } catch (e) { + // A plain CSV that fails to parse as annotations is often frame metadata: hint at the rename. + throw new Error(`${(e as Error).message} If this file is frame metadata rather than annotations, rename it to frame-metadata.csv.`); + } annotations.tracks = data[0].tracks; annotations.groups = data[0].groups; meta.fps = data[0].fps; @@ -1013,7 +1236,10 @@ async function findTrackandMetaFileinFolder(path: string) { let { trackFileAbsPath } = results; const { metaFileAbsPath } = results; if (!trackFileAbsPath) { - const csvFileCandidates = await _findCSVTrackFiles(path); + // Declared frame metadata sidecars stay in place for read-time discovery; the first + // remaining CSV is unconditionally the annotation track file. + const csvFileCandidates = (await _findCSVTrackFiles(path)) + .filter((candidate) => !isFrameMetadataSourceName(candidate)); if (csvFileCandidates.length) { [trackFileAbsPath] = csvFileCandidates; } @@ -2040,6 +2266,9 @@ export { loadJsonMetadata, loadAnnotationFile, loadDetections, + loadFrameMetadata, + frameMetadataSourceDirectories, + frameMetadataCandidateDescriptors, openLink, openPathInFileManager, ingestDataFiles, diff --git a/client/platform/desktop/backend/native/segmentation.ts b/client/platform/desktop/backend/native/segmentation.ts index e71ad0d1e..28e7af924 100644 --- a/client/platform/desktop/backend/native/segmentation.ts +++ b/client/platform/desktop/backend/native/segmentation.ts @@ -2,8 +2,8 @@ * Interactive Segmentation types. * * The runtime service manager was merged into the unified InteractiveServiceManager - * (see ./interactive). This module now only holds the segmentation request/response - * contracts shared between that manager and the IPC layer. + * (see ./interactive). This module now only holds the segmentation types shared + * between that manager and the IPC layer. */ import { StereoMeasurement } from './stereo'; diff --git a/client/platform/desktop/backend/native/stereo.ts b/client/platform/desktop/backend/native/stereo.ts index 4e07a1fec..88945ecdf 100644 --- a/client/platform/desktop/backend/native/stereo.ts +++ b/client/platform/desktop/backend/native/stereo.ts @@ -2,8 +2,8 @@ * Interactive Stereo types. * * The runtime service manager was merged into the unified InteractiveServiceManager - * (see ./interactive). This module now only holds the stereo request/response - * contracts shared between that manager and the IPC layer. + * (see ./interactive). This module now only holds the stereo types shared + * between that manager and the IPC layer. */ /** Error message shown to users when stereo service fails to load */ diff --git a/client/platform/desktop/backend/serializers/viame.ts b/client/platform/desktop/backend/serializers/viame.ts index 25333581f..d04913109 100644 --- a/client/platform/desktop/backend/serializers/viame.ts +++ b/client/platform/desktop/backend/serializers/viame.ts @@ -11,7 +11,9 @@ import moment from 'moment'; import { cloneDeep, flattenDeep, isEmpty } from 'lodash'; import { pipeline, Readable, Writable } from 'stream'; -import { AnnotationSchema, MultiGroupRecord, MultiTrackRecord } from 'dive-common/apispec'; +import { + AnnotationSchema, DatasetInfoFields, MultiGroupRecord, MultiTrackRecord, +} from 'dive-common/apispec'; import { JsonMeta } from 'platform/desktop/constants'; import { splitExt } from 'platform/desktop/backend/native/utils'; // Imports that involve actual code require relative imports because ts-node barely works @@ -44,7 +46,7 @@ export interface AnnotationFileData { groups: MultiGroupRecord; fps?: number; execTime?: number; - datasetInfo?: Record; + datasetInfo?: DatasetInfoFields; } // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/matchAll @@ -126,11 +128,11 @@ function syncDetectionLengthFields(feature: Feature): Feature { } /** - * Read dataset metadata from a `dataset_info: ` comment field. Returns the parsed + * Read dataset info from a `dataset_info: ` comment field. Returns the parsed * object, or a `warning` if the field is present but unusable so the import can continue. */ function parseDatasetInfo(row: string[]): { - datasetInfo?: Record; + datasetInfo?: DatasetInfoFields; warning?: string; } { const field = row.find((f) => f.trim().startsWith('dataset_info:')); @@ -141,7 +143,7 @@ function parseDatasetInfo(row: string[]): { try { const parsed = JSON.parse(json); if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { - return { datasetInfo: parsed as Record }; + return { datasetInfo: parsed as DatasetInfoFields }; } // eslint-disable-next-line no-nested-ternary const kind = parsed === null ? 'null' : Array.isArray(parsed) ? 'array' : typeof parsed; @@ -410,7 +412,7 @@ async function parse(input: Readable, imageMap?: Map): Promise<[ }); let fps: number | undefined; let execTime: number | undefined; - let datasetInfo: Record | undefined; + let datasetInfo: DatasetInfoFields | undefined; const dataMap = new Map(); const missingImages: string[] = []; const foundImages: {image: string; frame: number; csvFrame: number}[] = []; diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 942f2d7d9..742c2fb54 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -4,7 +4,7 @@ import type { DatasetMetaMutable, DatasetType, MultiCamImportArgs, Pipe, Pipelines, PipelineParams, SaveAttributeArgs, SaveAttributeTrackFilterArgs, SaveDetectionsArgs, TrainingConfigs, - DatasetCalibrationResult, + DatasetCalibrationResult, FrameMetadataSourcesResponse, SegmentationPredictRequest, SegmentationPredictResponse, SegmentationStatusResponse, SegmentationStereoSegmentRequest, SegmentationStereoSegmentResponse, TextQueryRequest, TextQueryResponse, RefineDetectionsRequest, RefineDetectionsResponse, @@ -609,6 +609,10 @@ async function loadDetections(datasetId: string) { }; } +function loadFrameMetadata(datasetId: string): Promise { + return window.diveDesktop.invoke('load-frame-metadata', { datasetId }); +} + async function saveMetadata(id: string, args: DatasetMetaMutable) { const client = await getClient(); return client.post(`dataset/${id}/meta`, args); @@ -669,6 +673,7 @@ export { /* Standard Specification APIs */ loadMetadata, loadDetections, + loadFrameMetadata, getPipelineList, deleteTrainedPipeline, runPipeline, diff --git a/client/platform/web-girder/App.vue b/client/platform/web-girder/App.vue index 8786b2321..cbd586c6e 100644 --- a/client/platform/web-girder/App.vue +++ b/client/platform/web-girder/App.vue @@ -17,6 +17,7 @@ import { runPipeline, exportTrainedPipeline, getDatasetCalibration, + loadFrameMetadata, getTrainingConfigurations, runTraining, saveMetadata, @@ -67,6 +68,7 @@ export default defineComponent({ getTrainingConfigurations: unwrap(getTrainingConfigurations), runTraining: unwrap(runTraining), loadDetections, + loadFrameMetadata, saveDetections: unwrap(saveDetections), saveMetadata: unwrap(saveMetadata), saveAttributes: unwrap(saveAttributes), diff --git a/client/platform/web-girder/api/dataset.service.ts b/client/platform/web-girder/api/dataset.service.ts index c2503d493..b13838c42 100644 --- a/client/platform/web-girder/api/dataset.service.ts +++ b/client/platform/web-girder/api/dataset.service.ts @@ -1,7 +1,8 @@ import type { GirderModel } from '@girder/components/src'; import { - DatasetMetaMutable, FrameImage, SaveAttributeArgs, SaveAttributeTrackFilterArgs, + DatasetMetaMutable, DatasetType, FrameImage, + SaveAttributeArgs, SaveAttributeTrackFilterArgs, } from 'dive-common/apispec'; import { calibrationFileMarker, jsonCalibrationFileMarker } from 'dive-common/constants'; import { parentDatasetId } from 'dive-common/compositeDatasetId'; @@ -173,12 +174,26 @@ async function saveMetadata(datasetId: string, metadata: DatasetMetaMutable) { return girderRest.patch(`/dive_dataset/${folderId}`, metadata); } -interface ValidationResponse { - ok: boolean; - type: 'video' | 'image-sequence'; +export interface ValidatedUploadRoleMap { media: string[]; annotations: string[]; + datasetConfig: string[]; + frameMetadata: string[]; +} + +export interface IgnoredUploadFile { + name: string; + reason: string; +} + +export interface ValidationResponse { + ok: boolean; + // Empty string when validation fails (no single media type could be determined). + type: DatasetType | ''; message: string; + roles: ValidatedUploadRoleMap; + upload: string[]; + ignored: IgnoredUploadFile[]; } function validateUploadGroup(names: string[]) { diff --git a/client/platform/web-girder/api/frameMetadata.service.ts b/client/platform/web-girder/api/frameMetadata.service.ts new file mode 100644 index 000000000..6d3597251 --- /dev/null +++ b/client/platform/web-girder/api/frameMetadata.service.ts @@ -0,0 +1,56 @@ +import type { FrameMetadataSourcesResponse } from 'dive-common/apispec'; +import { parentDatasetId } from 'dive-common/compositeDatasetId'; +import girderRest from 'platform/web-girder/plugins/girder'; + +interface FrameMetadataSourceItem { + itemId: string; + name: string; +} + +interface FrameMetadataSourceItemsResponse { + cameras: Record; +} + +async function getFrameMetadataSourceItems(datasetId: string): Promise { + const folderId = parentDatasetId(datasetId); + const { data } = await girderRest.get( + `dive_dataset/${folderId}/frame_metadata_sources`, + ); + return data; +} + +/** + * Download a Girder item's raw bytes as text over the existing item-download route. Used by the + * web frame-metadata read path to hand sidecar text to the shared parser. The axios JSON + * transform is disabled so a numeric-heavy CSV/TXT is returned verbatim, never coerced. + */ +async function downloadItemText(itemId: string): Promise { + const { data } = await girderRest.get(`item/${itemId}/download`, { + responseType: 'text', + transformResponse: [(value: string) => value], + }); + return typeof data === 'string' ? data : String(data); +} + +async function loadFrameMetadata(datasetId: string): Promise { + const response = await getFrameMetadataSourceItems(datasetId); + const textByItemId = new Map>(); + const cameraEntries = await Promise.all( + Object.entries(response.cameras ?? {}).map(async ([camera, items]) => { + const sources = await Promise.all( + items.map(async (item) => { + let pending = textByItemId.get(item.itemId); + if (pending === undefined) { + pending = downloadItemText(item.itemId); + textByItemId.set(item.itemId, pending); + } + return { name: item.name, text: await pending }; + }), + ); + return [camera, sources] as const; + }), + ); + return { cameras: Object.fromEntries(cameraEntries) }; +} + +export default loadFrameMetadata; diff --git a/client/platform/web-girder/api/index.ts b/client/platform/web-girder/api/index.ts index 63e1ac4f1..3249aa0a1 100644 --- a/client/platform/web-girder/api/index.ts +++ b/client/platform/web-girder/api/index.ts @@ -8,6 +8,7 @@ import { clearCalibrationFolderMetadata, getDatasetCalibration } from './dataset export * from './annotation.service'; export * from './configuration.service'; export * from './dataset.service'; +export { default as loadFrameMetadata } from './frameMetadata.service'; export * from './girder.service'; export * from './multicamResolve'; export * from './rpc.service'; diff --git a/client/platform/web-girder/multicamFileRegistry.spec.ts b/client/platform/web-girder/multicamFileRegistry.spec.ts index a535b4a3d..7516f1298 100644 --- a/client/platform/web-girder/multicamFileRegistry.spec.ts +++ b/client/platform/web-girder/multicamFileRegistry.spec.ts @@ -6,10 +6,39 @@ import { import { clearMulticamFileRegistry, getCalibrationFile, + getCameraPackageFiles, getLastCalibration, + mediaFileNamesForImport, saveCalibration, + stashAnnotationFile, stashCalibrationFile, } from './multicamFileRegistry'; +import { buildValidatedUploadPackage } from './uploadPackage'; +import type { ValidationResponse } from './api/dataset.service'; + +function file(name: string): File { + return new File([name], name, { type: 'application/octet-stream' }); +} + +function fileWithPath(name: string, relPath: string): File { + const f = file(name); + Object.defineProperty(f, 'webkitRelativePath', { value: relPath, configurable: true }); + return f; +} + +function validation(overrides: Partial): ValidationResponse { + return { + ok: true, + type: 'image-sequence', + message: '', + roles: { + media: [], annotations: [], datasetConfig: [], frameMetadata: [], + }, + upload: [], + ignored: [], + ...overrides, + }; +} describe('multicamFileRegistry calibration', () => { const storage = new Map(); @@ -30,10 +59,10 @@ describe('multicamFileRegistry calibration', () => { }); it('resolves calibration by basename when stashed with a path-like key', () => { - const file = new File(['{}'], 'stereo-cal.json', { type: 'application/json' }); - stashCalibrationFile('folder/stereo-cal.json', file); - expect(getCalibrationFile('stereo-cal.json')).toBe(file); - expect(getCalibrationFile('folder/stereo-cal.json')).toBe(file); + const cal = new File(['{}'], 'stereo-cal.json', { type: 'application/json' }); + stashCalibrationFile('folder/stereo-cal.json', cal); + expect(getCalibrationFile('stereo-cal.json')).toBe(cal); + expect(getCalibrationFile('folder/stereo-cal.json')).toBe(cal); }); it('does not restore last calibration from localStorage without a session File', async () => { @@ -43,9 +72,110 @@ describe('multicamFileRegistry calibration', () => { }); it('restores last calibration when the File is still in the registry', async () => { - const file = new File(['{}'], 'cal.json', { type: 'application/json' }); - stashCalibrationFile('cal.json', file); + const cal = new File(['{}'], 'cal.json', { type: 'application/json' }); + stashCalibrationFile('cal.json', cal); await saveCalibration('cal.json'); await expect(getLastCalibration()).resolves.toBe('cal.json'); }); }); + +describe('multicam camera package construction', () => { + beforeEach(() => { + clearMulticamFileRegistry(); + }); + + it('flattens folder files before validation so names match uploaded names', () => { + const folderFiles = [ + fileWithPath('img001.png', 'cam1/img001.png'), + fileWithPath('img002.png', 'cam1/img002.png'), + ]; + const cameraFiles = getCameraPackageFiles(folderFiles); + // The server validates and Girder uploads flat basenames, not folder paths. + expect(cameraFiles.map((f) => f.name)).toEqual(['img001.png', 'img002.png']); + expect( + cameraFiles.every((f) => !f.webkitRelativePath || f.webkitRelativePath === f.name), + ).toBe(true); + }); + + it('reports only media filenames for multicam pre-import validation', () => { + const files = [ + fileWithPath('img001.png', 'cam1/img001.png'), + fileWithPath('frame_metadata.csv', 'cam1/frame_metadata.csv'), + fileWithPath('tracks.csv', 'cam1/tracks.csv'), + ]; + expect(mediaFileNamesForImport(files, 'image-sequence')).toEqual(['img001.png']); + }); + + it('appends the explicit track file to the validation input', () => { + const folderFiles = [file('img001.png')]; + const track = file('tracks.csv'); + stashAnnotationFile('cam1/tracks.csv', track); + const cameraFiles = getCameraPackageFiles(folderFiles, 'cam1/tracks.csv'); + expect(cameraFiles.map((f) => f.name)).toEqual(['img001.png', 'tracks.csv']); + expect(cameraFiles).toContain(track); + }); + + it('deduplicates the explicit track file by name', () => { + const track = file('tracks.csv'); + stashAnnotationFile('cam1/tracks.csv', track); + // The same File object is also part of the camera folder selection. + const folderFiles = [file('img001.png'), track]; + const cameraFiles = getCameraPackageFiles(folderFiles, 'cam1/tracks.csv'); + expect(cameraFiles.map((f) => f.name)).toEqual(['img001.png', 'tracks.csv']); + expect(cameraFiles.filter((f) => f.name === 'tracks.csv')).toHaveLength(1); + }); + + it('dedupes an explicit track file that shares a name with a folder file (distinct object)', () => { + const explicitTrack = file('tracks.csv'); + stashAnnotationFile('cam1/tracks.csv', explicitTrack); + const folderTrack = file('tracks.csv'); + const folderFiles = [file('img001.png'), folderTrack]; + const cameraFiles = getCameraPackageFiles(folderFiles, 'cam1/tracks.csv'); + expect(cameraFiles.map((f) => f.name)).toEqual(['img001.png', 'tracks.csv']); + expect(cameraFiles).toContain(folderTrack); + expect(cameraFiles).not.toContain(explicitTrack); + }); + + it('uploads validated camera-folder annotations and sidecars, not just media', () => { + // tracks.csv and frame_metadata.csv are auto-detected in the camera folder; both + // must ride along with the camera, which the old media-only path dropped. + const folderFiles = [file('img001.png'), file('tracks.csv'), file('frame_metadata.csv')]; + const cameraFiles = getCameraPackageFiles(folderFiles); + const resp = validation({ + roles: { + media: ['img001.png'], + annotations: ['tracks.csv'], + datasetConfig: [], + frameMetadata: ['frame_metadata.csv'], + }, + upload: ['img001.png', 'tracks.csv', 'frame_metadata.csv'], + }); + const { uploadFiles } = buildValidatedUploadPackage(cameraFiles, resp); + expect(uploadFiles.map((f) => f.name)).toEqual(['img001.png', 'tracks.csv', 'frame_metadata.csv']); + }); + + it('uploads the explicit track file only when validation accepts it', () => { + const folderFiles = [file('img001.png')]; + const track = file('tracks.csv'); + stashAnnotationFile('cam1/tracks.csv', track); + const cameraFiles = getCameraPackageFiles(folderFiles, 'cam1/tracks.csv'); + + // Accepted: the server lists the track file in upload. + const accepted = validation({ + roles: { + media: ['img001.png'], annotations: ['tracks.csv'], datasetConfig: [], frameMetadata: [], + }, + upload: ['img001.png', 'tracks.csv'], + }); + expect(buildValidatedUploadPackage(cameraFiles, accepted).uploadFiles).toContain(track); + + // Omitted by validation: the track file is present in the input but not uploaded. + const omitted = validation({ + roles: { + media: ['img001.png'], annotations: [], datasetConfig: [], frameMetadata: [], + }, + upload: ['img001.png'], + }); + expect(buildValidatedUploadPackage(cameraFiles, omitted).uploadFiles).not.toContain(track); + }); +}); diff --git a/client/platform/web-girder/multicamFileRegistry.ts b/client/platform/web-girder/multicamFileRegistry.ts index 3fb568501..24db521d1 100644 --- a/client/platform/web-girder/multicamFileRegistry.ts +++ b/client/platform/web-girder/multicamFileRegistry.ts @@ -2,6 +2,12 @@ import { Location } from '@girder/components/src'; import { parentDatasetId } from 'dive-common/compositeDatasetId'; +import { + ImageSequenceType, + VideoType, + basicImageFileExtensions, + fileVideoTypes, +} from 'dive-common/constants'; import { openFromDisk, GirderUploadManager } from './utils'; const LAST_CALIBRATION_STORAGE_KEY = 'dive_web_last_calibration'; @@ -87,8 +93,18 @@ export function flattenUploadFiles(files: File[]): File[] { }); } -export function mediaFileNamesForImport(files: File[]): string[] { - return flattenUploadFiles(files).map((file) => file.name); +function fileExtension(fileName: string): string { + return fileName.split('.').pop()?.toLowerCase() ?? ''; +} + +export function mediaFileNamesForImport( + files: File[], + mediaType: typeof ImageSequenceType | typeof VideoType = ImageSequenceType, +): string[] { + const allowedExtensions = mediaType === VideoType ? fileVideoTypes : basicImageFileExtensions; + return flattenUploadFiles(files) + .map((file) => file.name) + .filter((name) => allowedExtensions.includes(fileExtension(name))); } export function stashAnnotationFile(key: string, file: File): void { @@ -99,6 +115,26 @@ export function getAnnotationFile(key: string): File | undefined { return annotationFilesByKey.get(key); } +/** + * Assemble the complete file package for one multicam camera: the camera folder's + * files plus any explicit annotation file chosen for that camera. The explicit + * annotation file is deduplicated by name, so a file that is both present in the + * folder selection and picked explicitly (a distinct File object from a separate + * picker) is only sent once — otherwise the server would validate a duplicate name. + * + * `flattenUploadFiles` is applied here, before validation, so the names sent to + * the server for validation match the names uploaded to Girder. + */ +export function getCameraPackageFiles(folderFiles: File[], annotationKey?: string): File[] { + const flattened = flattenUploadFiles(folderFiles); + const flattenedNames = new Set(flattened.map((file) => file.name)); + const annotationFile = annotationKey ? getAnnotationFile(annotationKey) : undefined; + const explicit = annotationFile && !flattenedNames.has(annotationFile.name) + ? [annotationFile] + : []; + return [...flattened, ...explicit]; +} + function calibrationLookupKeys(key: string): string[] { const keys = new Set(); if (key) { diff --git a/client/platform/web-girder/uploadPackage.spec.ts b/client/platform/web-girder/uploadPackage.spec.ts new file mode 100644 index 000000000..be5cbb4f8 --- /dev/null +++ b/client/platform/web-girder/uploadPackage.spec.ts @@ -0,0 +1,141 @@ +// eslint-disable-next-line import/no-extraneous-dependencies -- Vitest is only used in tests +import { describe, expect, it } from 'vitest'; + +import { buildValidatedUploadPackage } from './uploadPackage'; +import type { ValidationResponse } from './api/dataset.service'; + +function file(name: string): File { + return new File([name], name, { type: 'application/octet-stream' }); +} + +function validation(overrides: Partial): ValidationResponse { + return { + ok: true, + type: 'image-sequence', + message: '', + roles: { + media: [], + annotations: [], + datasetConfig: [], + frameMetadata: [], + }, + upload: [], + ignored: [], + ...overrides, + }; +} + +describe('buildValidatedUploadPackage', () => { + it('keeps both an annotation CSV and a frame-metadata sidecar in uploadFiles', () => { + const selected = [file('img001.png'), file('annotations.csv'), file('frame_metadata.csv')]; + const resp = validation({ + roles: { + media: ['img001.png'], + annotations: ['annotations.csv'], + datasetConfig: [], + frameMetadata: ['frame_metadata.csv'], + }, + upload: ['img001.png', 'annotations.csv', 'frame_metadata.csv'], + }); + + const { uploadFiles, roles, ignored } = buildValidatedUploadPackage(selected, resp); + + expect(uploadFiles.map((f) => f.name)).toEqual([ + 'img001.png', + 'annotations.csv', + 'frame_metadata.csv', + ]); + // The original File objects are preserved. + expect(uploadFiles).toEqual([selected[0], selected[1], selected[2]]); + expect(roles).toBe(resp.roles); + expect(ignored).toBe(resp.ignored); + }); + + it('excludes ignored unsupported files from uploadFiles', () => { + const selected = [file('img001.png'), file('notes.txt')]; + const resp = validation({ + roles: { + media: ['img001.png'], + annotations: [], + datasetConfig: [], + frameMetadata: [], + }, + upload: ['img001.png'], + ignored: [{ name: 'notes.txt', reason: 'Unsupported side file' }], + }); + + const { uploadFiles, ignored } = buildValidatedUploadPackage(selected, resp); + + expect(uploadFiles.map((f) => f.name)).toEqual(['img001.png']); + expect(uploadFiles.some((f) => f.name === 'notes.txt')).toBe(false); + expect(ignored).toEqual([{ name: 'notes.txt', reason: 'Unsupported side file' }]); + }); + + it('does not upload files that validation omitted from the upload list', () => { + const selected = [file('img001.png'), file('img002.png'), file('extra.csv')]; + const resp = validation({ + roles: { + media: ['img001.png', 'img002.png'], + annotations: [], + datasetConfig: [], + frameMetadata: [], + }, + // extra.csv is neither uploaded nor ignored: it simply is not consumed. + upload: ['img001.png', 'img002.png'], + }); + + const { uploadFiles } = buildValidatedUploadPackage(selected, resp); + + expect(uploadFiles.map((f) => f.name)).toEqual(['img001.png', 'img002.png']); + expect(uploadFiles.some((f) => f.name === 'extra.csv')).toBe(false); + }); + + it('preserves duplicate names only as many times as validation lists them', () => { + const first = file('dup.png'); + const second = file('dup.png'); + const selected = [first, second]; + const resp = validation({ + roles: { + media: ['dup.png', 'dup.png'], + annotations: [], + datasetConfig: [], + frameMetadata: [], + }, + upload: ['dup.png', 'dup.png'], + }); + + const { uploadFiles } = buildValidatedUploadPackage(selected, resp); + + expect(uploadFiles).toEqual([first, second]); + }); + + it('throws when validation references a file that was not selected', () => { + const selected = [file('img001.png')]; + const resp = validation({ + roles: { + media: ['img001.png'], + annotations: ['ghost.csv'], + datasetConfig: [], + frameMetadata: [], + }, + upload: ['img001.png', 'ghost.csv'], + }); + + expect(() => buildValidatedUploadPackage(selected, resp)).toThrow(/ghost\.csv/); + }); + + it('throws when validation lists a name more times than it was selected', () => { + const selected = [file('img001.png')]; + const resp = validation({ + roles: { + media: ['img001.png', 'img001.png'], + annotations: [], + datasetConfig: [], + frameMetadata: [], + }, + upload: ['img001.png', 'img001.png'], + }); + + expect(() => buildValidatedUploadPackage(selected, resp)).toThrow(/img001\.png/); + }); +}); diff --git a/client/platform/web-girder/uploadPackage.ts b/client/platform/web-girder/uploadPackage.ts new file mode 100644 index 000000000..696b78587 --- /dev/null +++ b/client/platform/web-girder/uploadPackage.ts @@ -0,0 +1,44 @@ +import type { + IgnoredUploadFile, + ValidatedUploadRoleMap, + ValidationResponse, +} from './api/dataset.service'; + +export interface ValidatedUploadPackage { + uploadFiles: File[]; + roles: ValidatedUploadRoleMap; + ignored: IgnoredUploadFile[]; +} + +export function buildValidatedUploadPackage( + selectedFiles: File[], + validation: ValidationResponse, +): ValidatedUploadPackage { + const filesByName = new Map(); + selectedFiles.forEach((file) => { + const existing = filesByName.get(file.name); + if (existing) { + existing.push(file); + } else { + filesByName.set(file.name, [file]); + } + }); + + const uploadFiles = validation.upload.map((name) => { + // Consume selected files in order so duplicate names are preserved only as + // many times as validation lists them. + const file = filesByName.get(name)?.shift(); + if (!file) { + throw new Error( + `Upload validation referenced "${name}", which is not among the selected files.`, + ); + } + return file; + }); + + return { + uploadFiles, + roles: validation.roles, + ignored: validation.ignored, + }; +} diff --git a/client/platform/web-girder/views/Upload.vue b/client/platform/web-girder/views/Upload.vue index 98b734234..7d41d330c 100644 --- a/client/platform/web-girder/views/Upload.vue +++ b/client/platform/web-girder/views/Upload.vue @@ -5,9 +5,7 @@ import { import { useRouter } from 'vue-router/composables'; import { - ImageSequenceType, VideoType, DefaultVideoFPS, FPSOptions, - inputAnnotationFileTypes, websafeVideoTypes, otherVideoTypes, - websafeImageTypes, otherImageTypes, JsonMetaRegEx, getLargeImageFileAccept, LargeImageType, + ImageSequenceType, VideoType, DefaultVideoFPS, FPSOptions, LargeImageType, } from 'dive-common/constants'; import { @@ -27,12 +25,17 @@ import { validateUploadGroup, waitForFolderDatasetReady, } from 'platform/web-girder/api'; +import type { + IgnoredUploadFile, + ValidatedUploadRoleMap, +} from 'platform/web-girder/api'; +import { buildValidatedUploadPackage } from 'platform/web-girder/uploadPackage'; import { clearMulticamFileRegistry, - getAnnotationFile, getCalibrationFile, + getCameraPackageFiles, getFilesForSourceKey, - flattenUploadFiles, + mediaFileNamesForImport, removeCameraFolderFiles, renameCameraFolderFiles, stashCameraFolderFiles, @@ -55,23 +58,27 @@ export interface InteralFiles { current: number; size: number; }; - upload: null; //Mixin function - result: null; //Mixin stuff + upload: null; + result: null; } export interface PendingUpload { createSubFolders: boolean; name: string; files: InteralFiles[]; - meta: null | File; - annotationFile: null | File; - mediaList: File[]; + uploadFiles: File[]; + roles: ValidatedUploadRoleMap; + ignored: IgnoredUploadFile[]; type: DatasetType | 'zip'; fps: number; uploading: boolean; skipTranscoding?: boolean; } +const emptyRoleMap = (): ValidatedUploadRoleMap => ({ + media: [], annotations: [], datasetConfig: [], frameMetadata: [], +}); + interface GirderUpload { formatSize: (a: number) => string; totalProgress: number; @@ -81,10 +88,9 @@ interface GirderUpload { name: string; fps: number; type: DatasetType; - mediaList: File[]; - meta?: File | null; - annotationFile?: File | null; + uploadFiles: File[]; skipTranscoding?: boolean; + parentFolderId?: string; }) => Promise<{ folder: { _id: string }; jobIds: string[] }>; } @@ -173,10 +179,10 @@ export default defineComponent({ pendingUploads.value.push({ createSubFolders: false, name: defaultFilename, - files: [], //Will be set in the GirderUpload Component - meta: null, - annotationFile: null, - mediaList: allFiles, + files: [], + uploadFiles: allFiles, + roles: emptyRoleMap(), + ignored: [], type: 'zip', fps, uploading: false, @@ -184,135 +190,62 @@ export default defineComponent({ }; const addPendingUpload = async ( - name: string, allFiles: File[], - meta: File | null, - annotationFile: File | null, - mediaList: File[], suggestedFps?: number, // suggested FPS for large/images ) => { - const resp = (await validateUploadGroup(allFiles.map((f) => f.name))).data; - if (!resp.ok) { - if (resp.message) { - preUploadErrorMessage.value = resp.message; + const validation = (await validateUploadGroup(allFiles.map((f) => f.name))).data; + if (!validation.ok) { + // Block: surface the reason and do not create an uploadable pending row. + if (validation.message) { + preUploadErrorMessage.value = validation.message; } - throw new Error(resp.message); + throw new Error(validation.message || 'Upload validation failed'); } + // Server validation is the single authority for what uploads; the browser + // never re-classifies. uploadFiles is the original File objects, in the + // server's upload order. + const { uploadFiles, roles, ignored } = buildValidatedUploadPackage(allFiles, validation); const fps = suggestedFps || clientSettings.annotationFPS || DefaultVideoFPS; - const defaultFilename = resp.media[0]; - const validFiles = resp.media.concat(resp.annotations); - // mapping needs to be done for the mixin upload functions - const internalFiles = allFiles - .filter((f) => validFiles.includes(f.name)); - let createSubFolders = false; - if (resp.type === 'video') { - if (resp.media.length > 1) { - createSubFolders = true; - } - } + const defaultFilename = roles.media[0] ?? uploadFiles[0]?.name ?? ''; + // Only a multi-video upload fans out into per-video subfolders. + const createSubFolders = validation.type === VideoType && roles.media.length > 1; pendingUploads.value.push({ createSubFolders, name: - internalFiles.length > 1 + uploadFiles.length > 1 ? defaultFilename.replace(fileSuffixRegex, '') : defaultFilename, - files: [], //Will be set in the GirderUpload Component - meta, - annotationFile, - mediaList, - type: resp.type, + files: [], + uploadFiles, + roles, + ignored, + type: validation.type, fps, uploading: false, skipTranscoding: true, }); }; /** - * Processes the imported media files to distinguish between - * Media Files - Default files that aren't the Annotation or Meta - * Annotation File - CSV or JSON file, or more in the future - * Meta File - Right now a json file which has 'meta' or 'config in the name - */ - const processImport = (files: { - canceled: boolean; filePaths: string[]; fileList?: File[]; - }) => { - //Check for auto files for meta and annotations - const output: { - annotationFile: null | File; - metaFile: null | File; - mediaList: File[]; - fullList: File[]; - } = { - annotationFile: null, - metaFile: null, - mediaList: [], - fullList: [], - }; - const jsonFiles: [string, number][] = []; - const csvFiles: [string, number][] = []; - if (files.fileList) { - files.filePaths.forEach((item, index) => { - if (item.indexOf('.json') !== -1) { - jsonFiles.push([item, index]); - } else if (item.indexOf('.csv') !== -1) { - csvFiles.push([item, index]); - } - }); - output.mediaList = files.fileList.filter((item) => ( - item.name.indexOf('.json') === -1 && item.name.indexOf('.csv') === -1)); - const metaIndex = jsonFiles.findIndex((item) => (JsonMetaRegEx.test(item[0]))); - if (metaIndex !== -1) { - output.metaFile = files.fileList[jsonFiles[metaIndex][1]]; - jsonFiles.splice(metaIndex, 1); //remove chosen meta from list - } - if (jsonFiles.length === 1 && csvFiles.length === 0) { // only remaining json file - output.annotationFile = files.fileList[jsonFiles[0][1]]; - } else if (csvFiles.length) { // Prefer First CSV if both found - output.annotationFile = files.fileList[csvFiles[0][1]]; - } else if (jsonFiles.length > 1) { //multiple jsons, filter out additional meta/configs - const filtered = jsonFiles.filter((item) => (!JsonMetaRegEx.test(item[0]) && (item[0].indexOf('.json') !== -1))); - if (filtered.length) { // take first filtered JSON file - output.annotationFile = files.fileList[filtered[0][1]]; - } - } - output.fullList = [...output.mediaList]; - if (output.annotationFile) { - output.fullList.push(output.annotationFile); - } - if (output.metaFile) { - output.fullList.push(output.metaFile); - } - } - return output; - }; - /** - * Initial opening of file dialog + * Initial opening of file dialog. The complete selection is sent to server + * validation; the browser no longer pre-classifies files into buckets. */ const openImport = async (dstype: DatasetType | 'zip') => { const ret = await openFromDisk(dstype); - if (!ret.canceled && ret.fileList) { - const processed = processImport(ret); - if (processed?.fullList?.length === 0) return; - if (processed && processed.fullList) { - const name = processed.fullList.length === 1 ? processed.fullList[0].name : ''; - preUploadErrorMessage.value = null; - try { - if (dstype !== 'zip') { - const suggestedFps = dstype === 'image-sequence' || dstype === 'large-image' ? 1 : undefined; - await addPendingUpload( - name, - processed.fullList, - processed.metaFile, - processed.annotationFile, - processed.mediaList, - suggestedFps, - ); - } else { - addPendingZipUpload(name, processed.fullList); - } - } catch (err) { - preUploadErrorMessage.value = err.response?.data?.message || err; - } + if (ret.canceled || !ret.fileList || ret.fileList.length === 0) { + return; + } + const allFiles = ret.fileList; + preUploadErrorMessage.value = null; + try { + if (dstype === 'zip') { + const name = allFiles.length === 1 ? allFiles[0].name : ''; + addPendingZipUpload(name, allFiles); + } else { + const suggestedFps = dstype === 'image-sequence' || dstype === 'large-image' ? 1 : undefined; + await addPendingUpload(allFiles, suggestedFps); } + } catch (err) { + preUploadErrorMessage.value = err.response?.data?.message || err; } }; const openMultiCamDialog = (args: { stereo: boolean; openType: 'image-sequence' | 'video' }) => { @@ -320,24 +253,12 @@ export default defineComponent({ multiCamOpenType.value = args.openType; importMultiCamDialog.value = true; }; - const filterFileUpload = (type: DatasetType | 'meta' | 'annotation') => { - if (type === 'meta') { - return '.json'; - } if (type === 'annotation') { - return inputAnnotationFileTypes.map((item) => `.${item}`).join(','); - } if (type === 'video') { - return websafeVideoTypes.concat(otherVideoTypes); - } if (type === 'large-image') { - return getLargeImageFileAccept(); - } - return websafeImageTypes.concat(otherImageTypes); - }; - const multiCamImportCheck = (sourcePath: string): MediaImportResponse => { const files = getFilesForSourceKey(sourcePath) ?? []; + const mediaType = multiCamOpenType.value === VideoType ? VideoType : ImageSequenceType; return { jsonMeta: { - originalImageFiles: files.map((file) => file.name), + originalImageFiles: mediaFileNamesForImport(files, mediaType), }, globPattern: '', mediaConvertList: [], @@ -404,33 +325,41 @@ export default defineComponent({ .filter((name) => args.sourceList[name]) .map((name) => [name, args.sourceList[name]] as const); const totalCameras = cameraEntries.length; + // Collect files the server accepted-but-ignored per camera so they can be + // surfaced before navigation — no selected file is silently dropped. + const ignoredAcrossCameras: { camera: string; name: string; reason: string }[] = []; for (let i = 0; i < cameraEntries.length; i += 1) { const [cameraName, source] = cameraEntries[i]; - const files = flattenUploadFiles(getFilesForSourceKey(source.sourcePath) ?? []); - if (!files?.length) { + const folderFiles = getFilesForSourceKey(source.sourcePath) ?? []; + if (!folderFiles.length) { throw new Error(`No media files found for camera "${cameraName}"`); } + // Flatten before validation so the names the server validates match the + // names uploaded to Girder. + const cameraFiles = getCameraPackageFiles(folderFiles, source.trackFile); // eslint-disable-next-line no-await-in-loop -- validate then upload each camera sequentially - const validation = (await validateUploadGroup(files.map((f) => f.name))).data; + const validation = (await validateUploadGroup(cameraFiles.map((f) => f.name))).data; if (!validation.ok) { throw new Error(validation.message || `Invalid files for camera "${cameraName}"`); } if (validation.type !== args.type) { throw new Error(`Camera "${cameraName}" must use ${args.type} media`); } - const mediaList = files.filter((file) => validation.media.includes(file.name)); - const annotationFile = source.trackFile - ? getAnnotationFile(source.trackFile) - : undefined; + // Server validation is the authority: upload exactly what it accepted, + // which includes validated camera-folder sidecars and annotation/config + // files, not just validation.roles.media. + const { uploadFiles } = buildValidatedUploadPackage(cameraFiles, validation); + validation.ignored.forEach((entry) => ignoredAcrossCameras.push({ + camera: cameraName, name: entry.name, reason: entry.reason, + })); trackMulticamCameraUploadProgress(i, totalCameras, cameraName); // eslint-disable-next-line no-await-in-loop const { folder, jobIds } = await uploadComponent.uploadCameraDataset({ name: cameraName, fps, type: args.type, - mediaList, - annotationFile: annotationFile ?? null, + uploadFiles, skipTranscoding: true, parentFolderId: datasetFolder._id, }); @@ -495,6 +424,19 @@ export default defineComponent({ }); multicamLinked = true; + if (ignoredAcrossCameras.length) { + await prompt({ + title: 'Some files were not uploaded', + text: [ + 'These selected files were not needed for the dataset and were left out:', + ...ignoredAcrossCameras.map( + (entry) => `${entry.camera}: ${entry.name} — ${entry.reason}`, + ), + ], + positiveButton: 'OK', + }); + } + setMulticamImportProgress(100, 'Opening viewer…'); clearMulticamFileRegistry(); await router.push({ name: 'viewer', params: { id: parentFolder._id } }); @@ -519,11 +461,8 @@ export default defineComponent({ const filesNotUploaded = (item: PendingUpload) => item.files.filter( (file) => file.status !== 'done' && file.status !== 'error', ).length; - // Processes the pending upload from the GirderUpload system to determine the progress const computeUploadProgress = (pendingUpload: PendingUpload) => { - // use methods and properties from mixins if (girderUpload.value) { - //Need to use the girderUpload ref to get these values out of the mixin const { formatSize, totalProgress, totalSize } = girderUpload.value; if (pendingUpload.files.length === 1 && !pendingUpload.uploading) { return formatSize(pendingUpload.files[0].progress.size); @@ -554,6 +493,20 @@ export default defineComponent({ const getFilenameInputValue = (pendingUpload: PendingUpload) => ( pendingUpload.createSubFolders && pendingUpload.type !== 'zip' ? 'default' : pendingUpload.name ); + const roleSummaryLines = (pendingUpload: PendingUpload): string[] => { + const { roles } = pendingUpload; + const lines: string[] = []; + const pushLine = (count: number, noun: string) => { + if (count > 0) { + lines.push(`${count} ${noun}${count === 1 ? '' : 's'}`); + } + }; + pushLine(roles.media.length, 'media file'); + pushLine(roles.annotations.length, 'annotation file'); + pushLine(roles.datasetConfig.length, 'configuration file'); + pushLine(roles.frameMetadata.length, 'frame metadata file'); + return lines; + }; const remove = (pendingUpload: PendingUpload) => { const index = pendingUploads.value.indexOf(pendingUpload); pendingUploads.value.splice(index, 1); @@ -607,9 +560,7 @@ export default defineComponent({ //methods close, openImport, - processImport, openMultiCamDialog, - filterFileUpload, multiCamImportCheck, multiCamImport, registerSubfolderCameras, @@ -620,6 +571,7 @@ export default defineComponent({ getFilenameInputValue, getFilenameInputStateDisabled, getFilenameInputStateHint, + roleSummaryLines, addPendingUpload, remove, abort, @@ -666,7 +618,7 @@ export default defineComponent({ :unregister-subfolder-camera="unregisterSubfolderCamera" :rename-subfolder-camera="renameSubfolderCamera" :import-media="multiCamImportCheck" - @begin-multicam-import="multiCamImport($event)" + @begin-multicam-import="multiCamImport" @abort="importMultiCamDialog = false; preUploadErrorMessage = null" /> @@ -736,9 +688,9 @@ export default defineComponent({ type="number" required label="FPS" - :append-icon="pendingUpload.annotationFile + :append-icon="pendingUpload.roles.annotations.length ? 'mdi-alert' : 'mdi-chevron-down'" - :hint="pendingUpload.annotationFile + :hint="pendingUpload.roles.annotations.length ? 'should match annotation fps' : 'annotation fps'" persistent-hint @change="clientSettings.annotationFPS = $event" @@ -758,54 +710,30 @@ export default defineComponent({ - + - - - - - - - - - +
+ {{ line }} +
+
+
+ Ignored (not uploaded): +
+
+ {{ ignoredFile.name }} ({{ ignoredFile.reason }}) +
+
diff --git a/client/platform/web-girder/views/UploadGirder.vue b/client/platform/web-girder/views/UploadGirder.vue index 9a48d0ea3..badac1996 100644 --- a/client/platform/web-girder/views/UploadGirder.vue +++ b/client/platform/web-girder/views/UploadGirder.vue @@ -89,14 +89,11 @@ export default Vue.extend({ }; }, async uploadPending(pendingUpload, uploaded) { - const { - name, createSubFolders, meta, annotationFile, mediaList, - } = pendingUpload; - //Combine the files for uploading - let files = mediaList.map((item) => this.convertFileToInternal(item)); - files.push(this.convertFileToInternal(meta)); - files.push(this.convertFileToInternal(annotationFile)); - files = files.filter((item) => item !== null); + const { name, createSubFolders, uploadFiles } = pendingUpload; + // The validated package is the only source of files to upload. + const files = uploadFiles + .map(this.convertFileToInternal) + .filter((item) => item !== null); // eslint-disable-next-line no-param-reassign pendingUpload.files = files; const fps = parseInt(pendingUpload.fps, 10); @@ -145,7 +142,6 @@ export default Vue.extend({ }, async uploadFiles(name, folder, files, uploaded, skipTranscoding = false) { let jobIds = []; - // function called after mixins upload finishes const postUpload = async (data) => { uploaded.push({ folder, @@ -159,9 +155,7 @@ export default Vue.extend({ throw err; } }; - // Sets the files used by the fileUploader mixin this.setFiles(files); - // Upload Mixin function to start uploading await this.start({ dest: folder, postUpload, @@ -173,13 +167,12 @@ export default Vue.extend({ * Upload a single camera dataset folder (used by multicam import). */ async uploadCameraDataset({ - name, fps, type, mediaList, meta = null, annotationFile = null, skipTranscoding = true, - parentFolderId = null, + name, fps, type, uploadFiles, skipTranscoding = true, parentFolderId = null, }) { - let files = mediaList.map((item) => this.convertFileToInternal(item)); - files.push(this.convertFileToInternal(meta)); - files.push(this.convertFileToInternal(annotationFile)); - files = files.filter((item) => item !== null); + // The validated package is the only source of files to upload for the camera. + const files = uploadFiles + .map(this.convertFileToInternal) + .filter((item) => item !== null); const folder = await this.createUploadFolder(name, parseInt(fps, 10), type, parentFolderId); if (!folder) { throw new Error(`Failed to create folder for camera ${name}`); diff --git a/client/src/components/annotators/ImageAnnotator.vue b/client/src/components/annotators/ImageAnnotator.vue index bb9e50630..2cd1c405d 100644 --- a/client/src/components/annotators/ImageAnnotator.vue +++ b/client/src/components/annotators/ImageAnnotator.vue @@ -2,6 +2,7 @@ import { defineComponent, ref, onUnmounted, PropType, toRef, watch, } from 'vue'; +import { map } from 'lodash'; import { ImageEnhancementOutputs } from 'vue-media-annotator/use/useImageEnhancements'; import { SetTimeFunc } from '../../use/useTimeObserver'; import AnnotatorImageCursor from './AnnotatorImageCursor.vue'; @@ -83,6 +84,7 @@ export default defineComponent({ toRef(data, 'imageCursorEditing'), ); data.maxFrame = props.imageData.length - 1; + data.filenames = map(props.imageData, 'filename'); // Below are configuration settings we can set until we decide on good numbers to utilize. let local = { playCache: 1, // seconds required to be fully cached before playback @@ -360,6 +362,7 @@ export default defineComponent({ } function init() { data.maxFrame = props.imageData.length - 1; + data.filenames = map(props.imageData, 'filename'); // Below are configuration settings we can set until we decide on good numbers to utilize. local = { playCache: 1, // seconds required to be fully cached before playback diff --git a/client/src/components/annotators/LargeImageAnnotator.vue b/client/src/components/annotators/LargeImageAnnotator.vue index 1f2d10cad..f2d621198 100644 --- a/client/src/components/annotators/LargeImageAnnotator.vue +++ b/client/src/components/annotators/LargeImageAnnotator.vue @@ -3,6 +3,7 @@ import { defineComponent, ref, onUnmounted, PropType, toRef, watch, computed, } from 'vue'; import geo from 'geojs'; +import { map } from 'lodash'; import { ImageEnhancementOutputs } from 'vue-media-annotator/use/useImageEnhancements'; import { SetTimeFunc } from '../../use/useTimeObserver'; import AnnotatorImageCursor from './AnnotatorImageCursor.vue'; @@ -364,6 +365,7 @@ export default defineComponent({ tileLoadError.value = ''; copiedConversionCommand.value = false; data.maxFrame = props.imageData.length - 1; + data.filenames = map(props.imageData, 'filename'); // Below are configuration settings we can set until we decide on good numbers to utilize. local = { playCache: 1, // seconds required to be fully cached before playback diff --git a/client/src/components/annotators/mediaControllerType.ts b/client/src/components/annotators/mediaControllerType.ts index 27a955ef1..f8cbf921f 100644 --- a/client/src/components/annotators/mediaControllerType.ts +++ b/client/src/components/annotators/mediaControllerType.ts @@ -38,6 +38,7 @@ export interface MediaController extends AggregateMediaController { cameraName: Readonly>; duration: Readonly>; filename: Readonly>; + filenames: Readonly>; flick: Readonly>; // eslint-disable-next-line @typescript-eslint/no-explicit-any geoViewerRef: Readonly>; diff --git a/client/src/components/annotators/useMediaController.ts b/client/src/components/annotators/useMediaController.ts index 2908ed45b..7ef81b62c 100644 --- a/client/src/components/annotators/useMediaController.ts +++ b/client/src/components/annotators/useMediaController.ts @@ -28,6 +28,7 @@ interface MediaControllerReactiveData { volume: number; speed: number; maxFrame: number; + filenames: string[]; syncedFrame: number; cursor: string; imageCursor: string; @@ -247,6 +248,7 @@ export function useMediaController() { volume: 0, speed: 1.0, maxFrame: 0, + filenames: [], syncedFrame: 0, cursor: 'default', imageCursor: '', @@ -460,6 +462,7 @@ export function useMediaController() { frame: toRef(state[camera], 'frame'), flick: toRef(state[camera], 'flick'), filename: toRef(state[camera], 'filename'), + filenames: toRef(state[camera], 'filenames'), duration: toRef(state[camera], 'duration'), volume: toRef(state[camera], 'volume'), maxFrame: toRef(state[camera], 'maxFrame'), diff --git a/docs/DataFormats.md b/docs/DataFormats.md index e9491e9fa..9b52d239e 100644 --- a/docs/DataFormats.md +++ b/docs/DataFormats.md @@ -5,7 +5,9 @@ hide: # Data Formats -DIVE Desktop and Web support a number of annotation and configuration formats. The following formats can be uploaded or imported alongside your media and will be automatically parsed. +DIVE Desktop and Web support a number of annotation, configuration, and +media-side metadata formats. The annotation and configuration formats below can +be uploaded or imported alongside your media and will be automatically parsed. * DIVE Annotation JSON (default annotation format) * DIVE Configuration JSON @@ -13,6 +15,10 @@ DIVE Desktop and Web support a number of annotation and configuration formats. * KPF (KWIVER Packet Format) * COCO and KWCOCO +Frame metadata sidecars are media files rather than annotation imports. See +[Frame Metadata Sidecars](Frame-Metadata.md) for their naming, placement, and +text-file format. + ## DIVE Annotation JSON !!! info diff --git a/docs/Frame-Metadata.md b/docs/Frame-Metadata.md new file mode 100644 index 000000000..5211eda45 --- /dev/null +++ b/docs/Frame-Metadata.md @@ -0,0 +1,64 @@ +# Frame Metadata Sidecars + +DIVE can display read-only frame metadata for image-sequence datasets in the +[Dataset Info panel](UI-DatasetInfo.md#frame-metadata). Common examples are +timestamp, latitude, longitude, depth, altitude, or vehicle state. + +Frame metadata is not annotation data. DIVE reads it from a sidecar file next to +the imagery when the dataset opens. The values are not edited in DIVE, imported +as annotations, or included in annotation exports. + +## Filename + +Name the file `frame-metadata.csv` or `frame-metadata.txt`. The snake_case names +`frame_metadata.csv` and `frame_metadata.txt` are also accepted. + +The filename identifies the file as frame metadata. Other CSV or text files are +handled by the normal annotation import flow. + +## File Format + +Use a delimited text file with: + +* a header row, +* comma, tab, or whitespace-separated columns, +* at least one column containing image filenames, +* at least one metadata column beyond the filename column. + +Rows are matched to frames by filename value, not by row order. Filename +extensions are ignored during matching, so `img_0001.tif` can match an image key +of `img_0001`. + +Example: + +```text +image_file timestamp latitude longitude water_depth +img_0001.tif 15:40:56 46.575870 -124.603094 192.80 +img_0002.tif 15:41:04 46.575912 -124.603080 193.10 +``` + +Values are displayed as strings in the order they appear in the file. DIVE does +not infer units or data types from the column names. + +## Placement + +For a single-camera image sequence, place the sidecar in the dataset folder +beside the images. + +For a multicamera image sequence, use either: + +* one shared sidecar in the multicamera parent folder, or +* one sidecar in each camera folder. + +A shared multicamera file can contain one filename column per camera, such as +`port_image` and `starboard_image`. Each active camera displays the rows that +match that camera's images. + +When both shared and camera-specific sidecars are present, the camera-specific +sidecar is used first. + +## Limits + +Frame metadata sidecars are supported for image-sequence datasets. Frame +metadata for videos, embedded KLV, embedded EXIF, and choosing a sidecar from +another location are not currently supported. diff --git a/docs/UI-DatasetInfo.md b/docs/UI-DatasetInfo.md index 38a9dfd85..4ff144823 100644 --- a/docs/UI-DatasetInfo.md +++ b/docs/UI-DatasetInfo.md @@ -1,7 +1,7 @@ # Dataset Info -The **Dataset Info** panel shows properties of the whole dataset and lets you attach -custom metadata to it. It is one pane of the +The **Dataset Info** panel shows read-only frame metadata, properties of the +whole dataset, and custom metadata attached to it. It is one pane of the [context sidebar](UI-Navigation-Editing-Bar.md#context-sidebar-web). Metadata you add travels with the dataset: it is shown while annotating and written into @@ -13,6 +13,10 @@ re-link annotations to their source records. ![Dataset Info panel](images/General/DatasetInfo.png){ width=220px align=right } +**Frame Metadata** (read-only): per-frame metadata for the active image, such +as timestamp, latitude, longitude, depth, or altitude. The panel shows only the +source fields for the active frame, in the order they appear in the source file. + **Standard information** (read-only): Name, Type, FPS, Original FPS and Subtype (when set), Created date, and ID (the Girder folder id). @@ -21,6 +25,27 @@ example a station id, cruise number, or dive number.
+## Frame Metadata + +The Frame Metadata section shows values for the active image in an +image-sequence dataset. The values update as the playhead moves. + +The panel shows the source fields in file order and includes a **`Source:`** line +naming the sidecar file that supplied them. For multicamera datasets, the panel +follows the active camera. + +The section may show an empty state when: + +* the dataset type does not support frame metadata, +* no frame metadata sidecar is present, +* the current frame has no matching row. + +Frame metadata is read-only. There is no edit, save, import, or export flow for +these values. + +See [Frame Metadata Sidecars](Frame-Metadata.md) for supported filenames, file +format, and placement. + ## Where the data is stored Custom metadata lives on the dataset's folder metadata under the `datasetInfo` key — the diff --git a/mkdocs.yml b/mkdocs.yml index 3944a0ad8..c4436c5f4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -19,6 +19,9 @@ theme: extra_css: [extra.css] +not_in_nav: | + Frame-Metadata.md + markdown_extensions: - admonition - attr_list diff --git a/server/dive_server/crud.py b/server/dive_server/crud.py index a584cb9dd..7ec0dfabe 100644 --- a/server/dive_server/crud.py +++ b/server/dive_server/crud.py @@ -24,6 +24,7 @@ class FileType(Enum): COCO_JSON = 3 DIVE_CONF = 4 MEVA_KPF = 5 + FRAME_METADATA = 6 def get_validated_model(model: BaseModel, **kwargs): diff --git a/server/dive_server/crud_dataset.py b/server/dive_server/crud_dataset.py index c0f7dc24f..b250b192e 100644 --- a/server/dive_server/crud_dataset.py +++ b/server/dive_server/crud_dataset.py @@ -7,16 +7,25 @@ import cherrypy from girder.constants import AccessType from girder.exceptions import RestException +from girder.models.file import File from girder.models.folder import Folder from girder.models.item import Item -from girder.models.file import File from girder.models.token import Token from girder.utility import ziputil from pydantic.main import BaseModel from dive_server import crud, crud_annotation from dive_tasks import tasks -from dive_utils import TRUTHY_META_VALUES, asbool, calibration_format, constants, fromMeta, models, types +from dive_utils import ( + TRUTHY_META_VALUES, + asbool, + calibration_format, + constants, + frame_metadata, + fromMeta, + models, + types, +) from dive_utils.serializers import kwcoco @@ -115,6 +124,8 @@ def _create_multicam_soft_clone( creator=owner, ) cloned_folder['meta'] = copy.deepcopy(source_folder['meta']) + media_source_folder = crud.getCloneRoot(owner, source_folder) + cloned_folder[constants.ForeignMediaIdMarker] = str(media_source_folder['_id']) cloned_folder['meta'][constants.PublishedMarker] = False if constants.ConfidenceFiltersMarker not in cloned_folder['meta']: cloned_folder['meta'][constants.ConfidenceFiltersMarker] = {'default': 0.1} @@ -244,6 +255,30 @@ def _multicam_camera_order(multi_cam: dict) -> List[str]: return list(cameras_meta.keys()) +def _iter_multicam_camera_folders( + multi_cam: dict, + user: types.GirderUserModel, +) -> Iterable[Tuple[str, types.GirderModel]]: + """Yield (camera_name, camera_folder) in display order, loading each child folder. + + Raises a 400 when a camera entry is missing its folderId, and a 404 when a camera's + folder cannot be loaded, matching the media and frame-metadata read paths. + """ + cameras_meta = multi_cam.get('cameras') or {} + for camera_name in _multicam_camera_order(multi_cam): + cam_info = cameras_meta[camera_name] + folder_id = cam_info.get('folderId') + if not folder_id: + raise RestException(f'Camera "{camera_name}" missing folderId', code=400) + child = Folder().load(folder_id, level=AccessType.READ, user=user) + if child is None: + raise RestException( + f'Camera folder for "{camera_name}" was not found', + code=404, + ) + yield camera_name, child + + def get_multi_cam_media( dsFolder: types.GirderModel, user: types.GirderUserModel ) -> models.MultiCamMedia: @@ -378,6 +413,95 @@ def get_media( ) +# Keep this in sync with the client's SingleCameraFrameMetadataKey so single-camera metadata uses +# the same key on both sides. +SINGLE_CAMERA_FRAME_METADATA_KEY = 'singleCam' + + +def _frame_metadata_source_items( + folder: types.GirderModel, +) -> List[Dict[str, str]]: + """List a folder's declared sidecar items as {itemId, name}, name-sorted (no download). + + Only identity is resolved here: the server classifies sidecars by name and never reads, + parses, or joins them. The client downloads and parses the bytes at read time. + """ + items = [ + item + for item in Folder().childItems(folder) + if frame_metadata.is_frame_metadata_source_name(item['name']) + ] + return [ + {'itemId': str(item['_id']), 'name': item['name']} + for item in sorted(items, key=lambda entry: str(entry['name']).lower()) + ] + + +def _collect_frame_metadata_sources( + folders: Iterable[Optional[types.GirderModel]], +) -> List[Dict[str, str]]: + """Assemble ordered {itemId, name} sidecar descriptors across folders in precedence order. + + Folders are deduped by id so a folder shared across cameras (the dataset folder, the + parent clone root) contributes its sidecars once, and a co-located dataset (whose clone + root is the folder itself) is listed at most once. + """ + seen_folder_ids: set[str] = set() + sources: List[Dict[str, str]] = [] + for folder in folders: + if folder is None: + continue + folder_id = str(folder['_id']) + if folder_id in seen_folder_ids: + continue + seen_folder_ids.add(folder_id) + sources.extend(_frame_metadata_source_items(folder)) + return sources + + +def load_frame_metadata_sources( + dsFolder: types.GirderModel, + user: types.GirderUserModel, +) -> dict: + """List declared frame-metadata sidecar items per camera, in precedence order (no parsing). + + Returns ``{'cameras': {: [{'itemId', 'name'}]}}``. Cameras are keyed by the + single-camera key or the multicam camera names; items precede in precedence order + (camera folder -> its clone root -> dataset folder -> parent root), deduped by folder id. + Non-image-sequence media types return no cameras. + """ + crud.verify_dataset(dsFolder) + source_type = fromMeta(dsFolder, constants.TypeMarker) + if source_type == constants.MultiType: + return _load_multicam_frame_metadata_sources(dsFolder, user) + if source_type != constants.ImageSequenceType: + return {'cameras': {}} + # Prefer sidecars stored directly on the dataset before falling back to the clone media root. + sources = _collect_frame_metadata_sources([dsFolder, crud.getCloneRoot(user, dsFolder)]) + if not sources: + return {'cameras': {}} + return {'cameras': {SINGLE_CAMERA_FRAME_METADATA_KEY: sources}} + + +def _load_multicam_frame_metadata_sources( + dsFolder: types.GirderModel, + user: types.GirderUserModel, +) -> dict: + multi_cam = fromMeta(dsFolder, constants.MultiCamMarker) or {} + parent_root = crud.getCloneRoot(user, dsFolder) + cameras: Dict[str, List[Dict[str, str]]] = {} + for camera_name, child in _iter_multicam_camera_folders(multi_cam, user): + if fromMeta(child, constants.TypeMarker) != constants.ImageSequenceType: + continue + # Prefer camera-local sidecars before falling back to shared multicam parent folders. + sources = _collect_frame_metadata_sources( + [child, crud.getCloneRoot(user, child), dsFolder, parent_root] + ) + if sources: + cameras[camera_name] = sources + return {'cameras': cameras} + + class MetadataMutableUpdateArgs(models.MetadataMutable): """Update schema for mutable metadata fields""" @@ -963,9 +1087,9 @@ def _source_calibration_items_in_folder_root(folder_id: str): Camera media lives in child folders; only direct items on folder_id are considered. """ for cal_item in Item().find({'folderId': _mongo_id(folder_id)}, sort=[('created', -1)]): - if _item_has_source_calibration_marker(cal_item) and constants.stereoCalibrationRegex.search( - cal_item['name'] - ): + if _item_has_source_calibration_marker( + cal_item + ) and constants.stereoCalibrationRegex.search(cal_item['name']): yield cal_item @@ -1348,55 +1472,113 @@ def create_multicam( def validate_files(files: List[str]): """ - Given a collection of filenames, guess based on regular expressions - if the collection represents a valid dataset, and if so, which files - represent which type of data + Given a collection of filenames, classify each into a semantic upload role. + + The response is authoritative for what the client uploads: accepted files are + listed under ``roles`` and concatenated into ``upload`` in a stable order, while + any file that is not needed is returned in ``ignored`` with a visible reason. No + selected file is silently discarded. """ + # Partition frame-metadata sidecars first so they never trip the + # generic csv/txt classification below. + frame_meta = [f for f in files if frame_metadata.is_frame_metadata_source_name(f)] + frame_meta_set = set(frame_meta) + + videos = [f for f in files if constants.videoRegex.search(f) and f not in frame_meta_set] + images = [f for f in files if constants.imageRegex.search(f) and f not in frame_meta_set] + large_images = [ + f for f in files if constants.largeImageRegEx.search(f) and f not in frame_meta_set + ] + media = images + videos + large_images + + # Keep dataset config filename detection aligned with the client's JsonMetaRegEx. + dataset_config = [ + f for f in files if constants.jsonRegex.search(f) and constants.metaRegex.search(f) + ] + dataset_config_set = set(dataset_config) + + annotation_csvs = [f for f in files if constants.csvRegex.search(f) and f not in frame_meta_set] + annotation_ymls = [f for f in files if constants.ymlRegex.search(f)] + annotation_jsons = [ + f for f in files if constants.jsonRegex.search(f) and f not in dataset_config_set + ] + annotations = annotation_csvs + annotation_ymls + annotation_jsons + + if len(videos): + mediatype = constants.VideoType + elif len(images): + mediatype = constants.ImageSequenceType + elif len(large_images): + mediatype = constants.LargeImageType + else: + mediatype = "" + ok = True message = "" - mediatype = "" - videos = [f for f in files if constants.videoRegex.search(f)] - csvs = [f for f in files if constants.csvRegex.search(f)] - images = [f for f in files if constants.imageRegex.search(f)] - large_images = [f for f in files if constants.largeImageRegEx.search(f)] - ymls = [f for f in files if constants.ymlRegex.search(f)] - jsons = [f for f in files if constants.jsonRegex.search(f)] if len(videos) and (len(images) or len(large_images)): ok = False message = "Do not upload images and videos in the same batch." elif len(large_images) and len(images): ok = False message = "Do not upload images and tile images in the same batch." - elif len(csvs) > 1: + elif len(annotation_csvs) > 1: ok = False message = "Can only upload a single CSV Annotation per import" - elif len(jsons) > 2: + elif len(dataset_config) > 1: ok = False - message = ( - "Can only upload a single JSON Annotation and single configuration JSON per import" - ) - elif len(csvs) == 1 and len(ymls): + message = "Can only upload a single configuration JSON per import" + elif len(annotation_jsons) > 1: + ok = False + message = "Can only upload a single annotation JSON per import" + elif len(annotation_csvs) and len(annotation_ymls): ok = False message = "Cannot mix annotation import types" - elif len(videos) > 1 and (len(csvs) or len(ymls) or len(jsons)): + elif len(annotation_ymls) > 1: + ok = False + message = "Can only upload a single YAML Annotation per import" + elif len(annotation_csvs) + len(annotation_ymls) + len(annotation_jsons) > 1: + # Multiple annotation sources across formats (e.g. CSV + JSON) would silently + # overwrite each other at import, so only one annotation source is allowed. + ok = False + message = "Cannot mix annotation import types" + elif len(videos) > 1 and (len(annotations) or len(dataset_config)): ok = False message = "Annotation upload is not supported when multiple videos are uploaded" - elif (not len(videos)) and (not len(images)) and (not len(large_images)): + elif not (len(videos) or len(images) or len(large_images)): ok = False message = "No supported media-type files found" - elif len(videos): - mediatype = constants.VideoType - elif len(images): - mediatype = constants.ImageSequenceType - elif len(large_images): - mediatype = constants.LargeImageType + + # Frame metadata is only read for image sequences (Decision D4). On any other media + # type the sidecar is preserved as ignored-with-reason rather than uploaded. + frame_metadata_role = frame_meta if mediatype == constants.ImageSequenceType else [] + + roles = { + "media": media, + "annotations": annotations, + "datasetConfig": dataset_config, + "frameMetadata": frame_metadata_role, + } + upload = media + annotations + dataset_config + frame_metadata_role + + accepted = set(upload) + ignored = [] + for f in files: + if f in accepted: + continue + if f in frame_meta_set: + ignored.append( + {"name": f, "reason": "Frame metadata is not supported for this media type"} + ) + else: + ignored.append({"name": f, "reason": "Unsupported side file"}) return { "ok": ok, "message": message, - "type": mediatype, - "media": images + videos + large_images, - "annotations": csvs + ymls + jsons, + "type": mediatype if ok else "", + "roles": roles, + "upload": upload, + "ignored": ignored, } diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py index d91292e4f..a80aea7d1 100644 --- a/server/dive_server/crud_rpc.py +++ b/server/dive_server/crud_rpc.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta import json -from typing import Dict, List, Optional, Tuple, TypedDict from pathlib import Path +from typing import Dict, List, Optional, Tuple, TypedDict from girder.constants import AccessType from girder.exceptions import RestException @@ -19,7 +19,15 @@ from dive_server import crud, crud_annotation, crud_dataset from dive_tasks import tasks from dive_tasks.multicam_pipeline import is_stereo_or_multicam_pipeline, pipeline_requires_input -from dive_utils import TRUTHY_META_VALUES, asbool, constants, fromMeta, models, types +from dive_utils import ( + TRUTHY_META_VALUES, + asbool, + constants, + frame_metadata, + fromMeta, + models, + types, +) from dive_utils.constants import TrainingModelExtensions from dive_utils.serializers import dive, kpf, kwcoco, viame @@ -530,8 +538,20 @@ def _get_data_by_type( """ if file is None: return None, None + # Frame metadata sidecars are declared by basename: classify without + # reading, parsing, or joining. Their content is read only at read time, by the + # client's shared TypeScript parser, never here. + if frame_metadata.is_frame_metadata_source_name(file['name']): + return { + 'annotations': None, + 'meta': None, + 'attributes': None, + 'type': crud.FileType.FRAME_METADATA, + }, None file_generator = File().download(file, headers=False)() - file_string = b"".join(list(file_generator)).decode() + # Annotation content is decoded strictly (utf-8-sig tolerates a BOM but not other + # non-UTF-8 bytes): a decode failure surfaces as the loud "Failed to import" path. + file_string = b"".join(list(file_generator)).decode('utf-8-sig') data_dict = None warnings = None @@ -642,22 +662,54 @@ def process_items( """ Discover unprocessed items in a dataset and process them by type in order of creation """ - unprocessed_items = Folder().childItems( - folder, - filters={ - "$or": [ - {"lowerName": {"$regex": constants.csvRegex}}, - {"lowerName": {"$regex": constants.jsonRegex}}, - {"lowerName": {"$regex": constants.ymlRegex}}, - ] - }, - # Processing order: oldest to newest - sort=[("created", pymongo.ASCENDING)], + unprocessed_items = list( + Folder().childItems( + folder, + filters={ + "$and": [ + { + "$or": [ + {"lowerName": {"$regex": constants.csvRegex}}, + {"lowerName": {"$regex": constants.jsonRegex}}, + {"lowerName": {"$regex": constants.ymlRegex}}, + ] + }, + # Frame-metadata sidecars are marked processed but stay in the dataset + # folder; excluding them here keeps a left-in-place sidecar from being + # re-swept on a later postprocess. + {f'meta.{constants.ProcessedMarker}': {'$ne': True}}, + ] + }, + # Processing order: oldest to newest + sort=[("created", pymongo.ASCENDING)], + ) ) - auxiliary = crud.get_or_create_auxiliary_folder( - folder, - user, + + # Side door: two plain annotation CSVs reaching the folder together can + # only happen outside the pre-upload validation path (validate_files blocks it). Reject + # before touching any item so both survive for the user to retry. + annotation_csv_names = sorted( + item['name'] + for item in unprocessed_items + if constants.csvRegex.search(item['name']) + and not frame_metadata.is_frame_metadata_source_name(item['name']) ) + if len(annotation_csv_names) >= 2: + raise RestException( + 'Multiple annotation CSVs present; delete all but one of: ' + f'{", ".join(annotation_csv_names)}' + ) + + # image_map (extension-stripped media stems) feeds the VIAME parser and only exists for + # image-sequence folders. Skip the media walk entirely when only sidecars are present: + # a declared sidecar is classified by name and never joined here. + image_map = None + if fromMeta(folder, constants.TypeMarker) == constants.ImageSequenceType and any( + not frame_metadata.is_frame_metadata_source_name(item['name']) for item in unprocessed_items + ): + image_map = crud.valid_image_names_dict(crud.valid_images(folder, user)) + + auxiliary = None aggregate_warnings = [] for item in unprocessed_items: file: Optional[types.GirderModel] = next(Item().childFiles(item), None) @@ -665,22 +717,39 @@ def process_items( raise RestException('Item had no associated files') try: - image_map = None - if fromMeta(folder, constants.TypeMarker) == 'image-sequence': - image_map = crud.valid_image_names_dict(crud.valid_images(folder, user)) results, warnings = _get_data_by_type(file, image_map=image_map) if warnings: aggregate_warnings += warnings except Exception as e: Item().remove(item) if isinstance(e, ValueError): - raise RestException(f'Failed to import {file["name"]}: {e}') from e + hint = ( + ' If this file is frame metadata rather than annotations, rename it to ' + 'frame-metadata.csv and re-upload.' + if constants.csvRegex.search(file['name']) + else '' + ) + raise RestException(f'Failed to import {file["name"]}: {e}{hint}') from e raise RestException(f'{file["name"]} was not a supported file type: {e}') from e if results is None: Item().remove(item) raise RestException(f'Unknown file type for {file["name"]}') + if results['type'] == crud.FileType.FRAME_METADATA: + # Declared frame metadata sidecar: keep it in the dataset folder for read-time + # discovery. Mark it processed so it is not re-swept, but never import it as + # annotations, move it, or remove it. + item['meta'][constants.ProcessedMarker] = True + Item().save(item) + aggregate_warnings.append( + f'{file["name"]} was stored as frame metadata, not annotations; ' + 'it stays in the dataset folder.' + ) + continue + + if auxiliary is None: + auxiliary = crud.get_or_create_auxiliary_folder(folder, user) item['meta'][constants.ProcessedMarker] = True Item().move(item, auxiliary) if results['annotations']: diff --git a/server/dive_server/event.py b/server/dive_server/event.py index 1ca52bc6e..3ec1c06e2 100644 --- a/server/dive_server/event.py +++ b/server/dive_server/event.py @@ -16,7 +16,7 @@ from girder_plugin_worker.utils import getWorkerApiUrl from dive_tasks.dive_batch_postprocess import DIVEBatchPostprocessTaskParams -from dive_utils import asbool, fromMeta +from dive_utils import asbool, frame_metadata, fromMeta from dive_utils.constants import ( AnnotationFileFutureProcessMarker, AssetstoreSourceMarker, @@ -106,6 +106,10 @@ def process_assetstore_import(event, meta: dict): # Set the dataset to Video Type dataset_type = VideoType elif possibleAnnotationRegex.search(importPath): + if frame_metadata.is_frame_metadata_source_name(item['name']): + # Declared frame metadata sidecars are plain files: leave them in place for + # read-time discovery, no marker and no relocation. + return # Look for parent folder with same name parentFolder = Folder().findOne({"_id": item["folderId"]}) userId = parentFolder['creatorId'] or parentFolder['baseParentId'] @@ -161,6 +165,9 @@ def process_dangling_annotation_files(folder, user): {'folderId': folder['_id'], f'meta.{AnnotationFileFutureProcessMarker}': True} ) for item in annotation_items: + if frame_metadata.is_frame_metadata_source_name(item['name']): + # Declared frame metadata sidecars are plain files: never marked or relocated. + continue # check if the parent folder of the annotation item is of type image or large image parent_folder_id = item['folderId'] parent_folder = Folder().findOne({'_id': parent_folder_id}) diff --git a/server/dive_server/views_dataset.py b/server/dive_server/views_dataset.py index 7978590f0..e478df332 100644 --- a/server/dive_server/views_dataset.py +++ b/server/dive_server/views_dataset.py @@ -40,6 +40,7 @@ def __init__(self, resourceName): self.route("GET", ("calibration",), self.get_dataset_calibration) self.route("POST", (":id", "calibration"), self.set_dataset_calibration) self.route("GET", (":id", "media"), self.get_media) + self.route("GET", (":id", "frame_metadata_sources"), self.get_frame_metadata_sources) self.route("GET", ("export",), self.export) self.route("GET", (":id", "configuration"), self.get_configuration) self.route("GET", (":id", "media", ":mediaId", "download"), self.download_media) @@ -202,11 +203,10 @@ def list_datasets( ) def get_meta(self, folder): return crud_dataset.get_dataset(folder, self.getCurrentUser()).dict(exclude_none=True) - + @access.user @autoDescribeRoute( - Description("Get calibration information of dataset") - .modelParam( + Description("Get calibration information of dataset").modelParam( "folderId", description="Folder id of a video clip", model=Folder, @@ -219,9 +219,7 @@ def get_dataset_calibration( self, folder, ): - return crud_dataset.get_calibration( - self.getCurrentUser(), folder - ) + return crud_dataset.get_calibration(self.getCurrentUser(), folder) @access.user @autoDescribeRoute( @@ -262,6 +260,17 @@ def get_configuration(self, folder): def get_media(self, folder): return crud_dataset.get_media(folder, self.getCurrentUser()).dict(exclude_none=True) + @access.user + @autoDescribeRoute( + Description( + "List declared frame-metadata sidecar items per camera, in precedence order. " + "The server classifies sidecars by name only and never parses them; the client " + "downloads and parses the bytes." + ).modelParam("id", level=AccessType.READ, **DatasetModelParam) + ) + def get_frame_metadata_sources(self, folder): + return crud_dataset.load_frame_metadata_sources(folder, self.getCurrentUser()) + @access.public(scope=TokenScope.DATA_READ, cookie=True) @autoDescribeRoute( Description("Export all selected datasets") diff --git a/server/dive_tasks/pipeline_discovery.py b/server/dive_tasks/pipeline_discovery.py index 447eb998d..e8f41e395 100644 --- a/server/dive_tasks/pipeline_discovery.py +++ b/server/dive_tasks/pipeline_discovery.py @@ -155,7 +155,9 @@ def extract_pipe_metadata(file_path: Path) -> PipeMetadata: r'^#\s*Requires\s+Calibration:\s*(.*)', line_raw, re.IGNORECASE ) if calibration_match: - metadata["requiresCalibration"] = calibration_match.group(1).strip().lower() in ( + metadata["requiresCalibration"] = calibration_match.group( + 1 + ).strip().lower() in ( 'true', 'yes', '1', diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py index 5b089dac5..025008d29 100644 --- a/server/dive_tasks/tasks.py +++ b/server/dive_tasks/tasks.py @@ -837,7 +837,9 @@ def convert_calibration(self: Task, itemId: str): folder = gc.getFolder(folder_id) multi_cam = (folder.get('meta') or {}).get(constants.MultiCamMarker) or {} if str(multi_cam.get(constants.CalibrationItemIdMarker)) != str(itemId): - manager.write('Calibration source was replaced before linking JSON; discarding output.\n') + manager.write( + 'Calibration source was replaced before linking JSON; discarding output.\n' + ) gc.delete(f'item/{json_item_id}') return diff --git a/server/dive_tasks/utils.py b/server/dive_tasks/utils.py index a19053a51..94671bd33 100644 --- a/server/dive_tasks/utils.py +++ b/server/dive_tasks/utils.py @@ -246,8 +246,8 @@ def upload_zipped_flat_media_files( root_folderId = folderId default_fps = gc.getFolder(root_folderId).get(f"meta.{constants.FPSMarker}", -1) if validation.get('ok', False): - manager.write(f"Annotations: {validation['annotations']}\n") - manager.write(f"Media: {validation['media']}\n") + manager.write(f"Annotations: {validation['roles']['annotations']}\n") + manager.write(f"Media: {validation['roles']['media']}\n") dataset_type = validation['type'] manager.write(f"Type: {dataset_type}\n") if create_subfolder != '': diff --git a/server/dive_utils/calibration_format.py b/server/dive_utils/calibration_format.py index b1d7ab37f..c03640352 100644 --- a/server/dive_utils/calibration_format.py +++ b/server/dive_utils/calibration_format.py @@ -28,7 +28,7 @@ def is_valid_json_bytes(data: bytes) -> bool: def looks_like_zip_bytes(data: bytes) -> bool: """True when data starts with ZIP local-file-header magic (e.g. mislabeled .npz).""" - return len(data) >= 2 and data[0] == 0x50 and data[1] == 0x4b + return len(data) >= 2 and data[0] == 0x50 and data[1] == 0x4B def calibration_upload_is_final_json(file_name: str, file_bytes: bytes) -> bool: @@ -112,5 +112,7 @@ def parse_stereo_calibration_json(data: dict) -> types.DatasetStereoCalibration: for json_key, field in optional_fields: value = optional_calibration_number(data, json_key) if value is not None: - result[field] = int(value) if field.startswith('grid') or field.startswith('image') else value + result[field] = ( + int(value) if field.startswith('grid') or field.startswith('image') else value + ) return result diff --git a/server/dive_utils/frame_metadata.py b/server/dive_utils/frame_metadata.py new file mode 100644 index 000000000..b57318e4d --- /dev/null +++ b/server/dive_utils/frame_metadata.py @@ -0,0 +1,24 @@ +"""Frame-metadata sidecar name predicate. + +DIVE reserves ``frame-metadata.csv`` and ``frame-metadata.txt`` as the preferred +declared frame-metadata sidecar files; ``frame_metadata.csv`` and +``frame_metadata.txt`` are also accepted (case-insensitive basenames). This module +is the Python mirror of the shared TypeScript predicate; it classifies by name +only and never parses frame metadata. +""" + +import re + +FRAME_METADATA_SOURCE_NAMES = { + 'frame-metadata.csv', + 'frame-metadata.txt', + 'frame_metadata.csv', + 'frame_metadata.txt', +} +PATH_SPLIT_RE = re.compile(r'[/\\]') + + +def is_frame_metadata_source_name(name: str) -> bool: + """A frame metadata sidecar is declared by basename.""" + basename = PATH_SPLIT_RE.split(name)[-1] + return basename.lower() in FRAME_METADATA_SOURCE_NAMES diff --git a/server/dive_utils/serializers/viame.py b/server/dive_utils/serializers/viame.py index ff8e34540..df701a704 100644 --- a/server/dive_utils/serializers/viame.py +++ b/server/dive_utils/serializers/viame.py @@ -108,13 +108,14 @@ def get_next_polygon_key(features: Dict[str, Any]) -> str: return '' # Count existing polygons to determine the next key polygon_count = sum( - 1 for f in features["geometry"]["features"] - if f["geometry"]["type"] == "Polygon" + 1 for f in features["geometry"]["features"] if f["geometry"]["type"] == "Polygon" ) return str(polygon_count) if polygon_count > 0 else '' -def create_geoJSONFeature(features: Dict[str, Any], type: str, coords: List[Any], key='', auto_key=False): +def create_geoJSONFeature( + features: Dict[str, Any], type: str, coords: List[Any], key='', auto_key=False +): feature = {} if "geometry" not in features: features["geometry"] = {"type": "FeatureCollection", "features": []} @@ -153,10 +154,7 @@ def add_hole_to_polygon(features: Dict[str, Any], coords: List[Any], key=''): if "geometry" not in features or not features["geometry"]["features"]: return for subfeature in features["geometry"]["features"]: - if ( - subfeature["geometry"]["type"] == 'Polygon' - and subfeature["properties"]["key"] == key - ): + if subfeature["geometry"]["type"] == 'Polygon' and subfeature["properties"]["key"] == key: # Add hole as additional ring to the polygon coordinates subfeature["geometry"]["coordinates"].append(coords) break @@ -206,10 +204,7 @@ def _parse_row(row: List[str]) -> Tuple[Dict, Dict, Dict, List, List]: # (poly) x1 y1 x2 y2 ... - polygon (multiple allowed, auto-keyed internally) # (hole) x1 y1 x2 y2 ... - hole in the most recent polygon - poly_regex = re.match( - r"^\(poly\)\s*((?:-?[0-9]+\.*-?[0-9]*\s*)+)", - row[j] - ) + poly_regex = re.match(r"^\(poly\)\s*((?:-?[0-9]+\.*-?[0-9]*\s*)+)", row[j]) if poly_regex: temp = [float(x) for x in poly_regex.group(1).split()] coords = [[temp[i], temp[i + 1]] for i in range(0, len(temp), 2)] @@ -217,16 +212,17 @@ def _parse_row(row: List[str]) -> Tuple[Dict, Dict, Dict, List, List]: create_geoJSONFeature(features, 'Polygon', coords, auto_key=True) # (hole) x1 y1 x2 y2 ... - hole in the most recent polygon - hole_regex = re.match( - r"^\(hole\)\s*((?:-?[0-9]+\.*-?[0-9]*\s*)+)", - row[j] - ) + hole_regex = re.match(r"^\(hole\)\s*((?:-?[0-9]+\.*-?[0-9]*\s*)+)", row[j]) if hole_regex: temp = [float(x) for x in hole_regex.group(1).split()] coords = [[temp[i], temp[i + 1]] for i in range(0, len(temp), 2)] # Add hole to the most recent polygon (last one added) if "geometry" in features and features["geometry"]["features"]: - polygons = [f for f in features["geometry"]["features"] if f["geometry"]["type"] == "Polygon"] + polygons = [ + f + for f in features["geometry"]["features"] + if f["geometry"]["type"] == "Polygon" + ] if polygons: last_poly_key = polygons[-1]["properties"]["key"] add_hole_to_polygon(features, coords, last_poly_key) @@ -385,7 +381,9 @@ def parse_metadata_row(row: List[str]) -> Dict[str, Any]: def load_csv_as_tracks_and_attributes( rows: List[str], imageMap: Optional[Dict[str, int]] = None, -) -> Tuple[types.DIVEAnnotationSchema, types.Attributes, types.Warnings, Optional[str], types.DatasetInfo]: +) -> Tuple[ + types.DIVEAnnotationSchema, types.Attributes, types.Warnings, Optional[str], types.DatasetInfo +]: """ Convert VIAME CSV to json tracks @@ -650,11 +648,7 @@ def export_tracks_as_csv( attr_length = candidate except (TypeError, ValueError): attr_length = None - resolved_length = ( - attr_length - if attr_length is not None - else feature.fishLength - ) + resolved_length = attr_length if attr_length is not None else feature.fishLength export_length = ( resolved_length if resolved_length is not None and resolved_length == resolved_length diff --git a/server/tests/test_calibration_format.py b/server/tests/test_calibration_format.py index dccf1e933..ffceb4c4f 100644 --- a/server/tests/test_calibration_format.py +++ b/server/tests/test_calibration_format.py @@ -18,15 +18,17 @@ def test_is_valid_json_bytes(): def test_parse_stereo_calibration_json_int_casts_grid_and_image_sizes(): - parsed = calibration_format.parse_stereo_calibration_json({ - 'R': [1, 0, 0, 0, 1, 0, 0, 0, 1], - 'T': [0.1, 0.2, 0.3], - 'grid_height': 8.0, - 'grid_width': 11.0, - 'image_height': 1080.9, - 'image_width': 1920.1, - 'cx_left': 1.5, - }) + parsed = calibration_format.parse_stereo_calibration_json( + { + 'R': [1, 0, 0, 0, 1, 0, 0, 0, 1], + 'T': [0.1, 0.2, 0.3], + 'grid_height': 8.0, + 'grid_width': 11.0, + 'image_height': 1080.9, + 'image_width': 1920.1, + 'cx_left': 1.5, + } + ) assert parsed['gridHeight'] == 8 assert parsed['gridWidth'] == 11 assert parsed['imageHeight'] == 1080 diff --git a/server/tests/test_event_frame_metadata.py b/server/tests/test_event_frame_metadata.py new file mode 100644 index 000000000..dcf40c427 --- /dev/null +++ b/server/tests/test_event_frame_metadata.py @@ -0,0 +1,85 @@ +import types as pytypes +from unittest.mock import patch + +from dive_server import event +from dive_utils.constants import AnnotationFileFutureProcessMarker + + +def _event(info): + return pytypes.SimpleNamespace(info=info) + + +# A real 24-hex ObjectId is required because process_assetstore_import wraps creatorId in ObjectId. +_OWNER_ID = '000000000000000000000000' + + +@patch('dive_server.event.User') +@patch('dive_server.event.Folder') +@patch('dive_server.event.Item') +def test_assetstore_import_leaves_meta_sidecar_unmarked(item_cls, folder_cls, user_cls): + item = {'_id': 'i1', 'name': 'frame_metadata.csv', 'meta': {}, 'folderId': 'f1'} + item_model = item_cls.return_value + item_model.findOne.return_value = item + + event.process_assetstore_import( + _event({'type': 'item', 'importPath': '/data/frame_metadata.csv', 'id': 'i1'}), + {}, + ) + + # Declared sidecar: no future-process marker, no save, no relocation, folder untouched. + assert AnnotationFileFutureProcessMarker not in item['meta'] + item_model.save.assert_not_called() + item_model.move.assert_not_called() + folder_cls.return_value.findOne.assert_not_called() + + +@patch('dive_server.event.User') +@patch('dive_server.event.Folder') +@patch('dive_server.event.Item') +def test_assetstore_import_marks_plain_annotation_csv(item_cls, folder_cls, user_cls): + item = {'_id': 'i1', 'name': 'dets.csv', 'meta': {}, 'folderId': 'f1'} + item_model = item_cls.return_value + item_model.findOne.return_value = item + parent_folder = {'_id': 'f1', 'creatorId': _OWNER_ID, 'baseParentId': None, 'meta': {}} + folder_model = folder_cls.return_value + + def find_one(query): + if query == {'_id': 'f1'}: + return parent_folder + return None # no co-named video folder exists + + folder_model.findOne.side_effect = find_one + user_cls.return_value.findOne.return_value = {'_id': _OWNER_ID} + + event.process_assetstore_import( + _event({'type': 'item', 'importPath': '/data/dets.csv', 'id': 'i1'}), + {}, + ) + + # A plain annotation CSV is still marked for future processing, as on main. + assert item['meta'][AnnotationFileFutureProcessMarker] is True + item_model.save.assert_called_once() + item_model.move.assert_not_called() + + +@patch('dive_server.event.Folder') +@patch('dive_server.event.Item') +def test_process_dangling_skips_meta_sidecar(item_cls, folder_cls): + meta_item = { + '_id': 'm1', + 'name': 'frame_metadata.csv', + 'meta': {AnnotationFileFutureProcessMarker: True}, + 'folderId': 'f1', + } + item_model = item_cls.return_value + item_model.find.return_value = [meta_item] + folder_model = folder_cls.return_value + folder_model.childFolders.return_value = [] + + event.process_dangling_annotation_files({'_id': 'f1'}, {'_id': 'u1'}) + + # The declared sidecar is skipped: marker unchanged, never saved or relocated. + assert meta_item['meta'][AnnotationFileFutureProcessMarker] is True + item_model.save.assert_not_called() + item_model.move.assert_not_called() + folder_model.findOne.assert_not_called() diff --git a/server/tests/test_frame_metadata_naming.py b/server/tests/test_frame_metadata_naming.py new file mode 100644 index 000000000..43d21b3c2 --- /dev/null +++ b/server/tests/test_frame_metadata_naming.py @@ -0,0 +1,26 @@ +"""Python parity check for frame-metadata source names. + +``frame_metadata`` / ``frame-metadata`` with a ``.csv`` or ``.txt`` extension is the +single declared-by-name flag that travels every ingestion path. The predicate is mirrored +in Python (``dive_utils.frame_metadata``) and in the shared TypeScript module; both +harnesses assert the same accepted/rejected truth table so the two mirrors stay aligned. +""" + +import json +from pathlib import Path + +from dive_utils.frame_metadata import is_frame_metadata_source_name + +SOURCE_NAMES_FIXTURE = ( + Path(__file__).parents[2] + / 'testdata' + / 'frame-metadata-conformance' + / 'source_names.expected.json' +) + + +def test_source_names_predicate_matches_shared_truth_table(): + truth_table = json.loads(SOURCE_NAMES_FIXTURE.read_text(encoding='utf-8')) + assert truth_table, 'source_names truth table fixture is empty' + for name, expected in truth_table.items(): + assert is_frame_metadata_source_name(name) is expected, name diff --git a/server/tests/test_frame_metadata_sources.py b/server/tests/test_frame_metadata_sources.py new file mode 100644 index 000000000..d20a20e48 --- /dev/null +++ b/server/tests/test_frame_metadata_sources.py @@ -0,0 +1,299 @@ +from unittest.mock import patch + +from girder.exceptions import RestException +import pytest + +from dive_server import crud_dataset +from dive_server.views_dataset import DatasetResource +from dive_utils import constants + + +def _dataset_folder(): + return { + '_id': 'dataset-id', + 'name': 'single-camera', + 'meta': {'annotate': True, 'type': constants.ImageSequenceType}, + } + + +def _clone_dataset_folder(folder_id: str): + return { + '_id': folder_id, + 'name': folder_id, + 'meta': {'annotate': True, 'type': constants.ImageSequenceType}, + } + + +def _root_folder(folder_id: str): + return {'_id': folder_id, 'name': folder_id, 'meta': {}} + + +def _camera_folder(folder_id: str, name: str): + return { + '_id': folder_id, + 'name': name, + 'meta': {'annotate': True, 'type': constants.ImageSequenceType, 'fps': 5}, + } + + +def _multicam_parent_folder(): + return { + '_id': 'parent-id', + 'name': 'stereo-camera', + 'meta': { + 'annotate': True, + 'type': constants.MultiType, + 'fps': 5, + 'multiCam': { + 'defaultDisplay': 'port', + 'cameraOrder': ['port', 'starboard'], + 'cameras': { + 'port': {'folderId': 'port-id', 'type': constants.ImageSequenceType}, + 'starboard': { + 'folderId': 'starboard-id', + 'type': constants.ImageSequenceType, + }, + }, + }, + }, + } + + +def _source_item(name: str): + return {'_id': f'{name}-id', 'name': name} + + +def _descriptor(name: str): + return {'itemId': f'{name}-id', 'name': name} + + +def _child_items_by_folder(folder_model, items_by_folder_id): + def child_items(folder): + return items_by_folder_id.get(folder['_id'], []) + + folder_model.childItems.side_effect = child_items + + +def _wire_multicam_folders(folder_model, children): + def load_folder(folder_id, level=None, user=None): + return children.get(folder_id) + + folder_model.load.side_effect = load_folder + + +def _wire_clone_roots(get_clone_root, roots_by_folder_id): + def clone_root(user, folder): + return roots_by_folder_id[folder['_id']] + + get_clone_root.side_effect = clone_root + + +@patch('dive_server.crud_dataset.Folder') +@patch('dive_server.crud_dataset.crud.getCloneRoot') +def test_sources_single_camera_co_located_name_sorted_and_deduped(get_clone_root, folder_cls): + dataset = _dataset_folder() + user = {'_id': 'user-id'} + # Co-located (non-clone): the media root is the dataset folder itself, so it is listed once. + get_clone_root.return_value = dataset + folder_model = folder_cls.return_value + folder_model.childItems.return_value = [ + _source_item('image_0001.jpg'), + _source_item('frame_metadata.csv'), + _source_item('frame_metadata.txt'), + _source_item('frame-metadata.csv'), + _source_item('frame-metadata.txt'), + _source_item('nav.meta.csv'), + _source_item('frame_metadata.json'), + ] + + result = crud_dataset.load_frame_metadata_sources(dataset, user) + + # Only declared sidecars, name-sorted case-insensitively, keyed by the single-camera key. + assert result == { + 'cameras': { + 'singleCam': [ + _descriptor('frame-metadata.csv'), + _descriptor('frame-metadata.txt'), + _descriptor('frame_metadata.csv'), + _descriptor('frame_metadata.txt'), + ], + }, + } + folder_model.childItems.assert_called_once_with(dataset) + + +@patch('dive_server.crud_dataset.Folder') +@patch('dive_server.crud_dataset.crud.getCloneRoot') +def test_sources_single_camera_clone_local_precedes_root(get_clone_root, folder_cls): + clone = _clone_dataset_folder('clone-id') + source_root = _root_folder('source-root-id') + user = {'_id': 'user-id'} + get_clone_root.return_value = source_root + folder_model = folder_cls.return_value + _child_items_by_folder( + folder_model, + { + 'clone-id': [_source_item('frame_metadata.csv')], + 'source-root-id': [_source_item('frame-metadata.txt'), _source_item('image_0001.jpg')], + }, + ) + + result = crud_dataset.load_frame_metadata_sources(clone, user) + + # Precedence: the clone-local sidecar is listed before the media-root sidecar. + assert result == { + 'cameras': { + 'singleCam': [_descriptor('frame_metadata.csv'), _descriptor('frame-metadata.txt')], + }, + } + + +@patch('dive_server.crud_dataset.Folder') +@patch('dive_server.crud_dataset.crud.getCloneRoot') +def test_sources_single_camera_no_sidecars_returns_empty(get_clone_root, folder_cls): + dataset = _dataset_folder() + user = {'_id': 'user-id'} + get_clone_root.return_value = dataset + folder_cls.return_value.childItems.return_value = [ + _source_item('image_0001.jpg'), + _source_item('frame_metadata.json'), + ] + + result = crud_dataset.load_frame_metadata_sources(dataset, user) + + assert result == {'cameras': {}} + + +@pytest.mark.parametrize('dataset_type', [constants.VideoType, constants.LargeImageType]) +@patch('dive_server.crud_dataset.Folder') +@patch('dive_server.crud_dataset.crud.getCloneRoot') +def test_sources_unsupported_media_type_returns_empty(get_clone_root, folder_cls, dataset_type): + dataset = { + '_id': 'ds', + 'name': 'x', + 'meta': {'annotate': True, 'type': dataset_type, 'fps': 5}, + } + + result = crud_dataset.load_frame_metadata_sources(dataset, {'_id': 'user-id'}) + + # Non-image-sequence media types never expose sidecars; no folder is even walked. + assert result == {'cameras': {}} + folder_cls.return_value.childItems.assert_not_called() + get_clone_root.assert_not_called() + + +@patch('dive_server.crud_dataset.Folder') +@patch('dive_server.crud_dataset.crud.getCloneRoot') +def test_sources_multicam_routes_per_camera_with_shared_parent_root(get_clone_root, folder_cls): + parent = _multicam_parent_folder() + port = _camera_folder('port-id', 'port') + starboard = _camera_folder('starboard-id', 'starboard') + parent_root = _root_folder('parent-root-id') + port_root = _root_folder('port-root-id') + starboard_root = _root_folder('starboard-root-id') + user = {'_id': 'user-id'} + + folder_model = folder_cls.return_value + _wire_multicam_folders(folder_model, {'port-id': port, 'starboard-id': starboard}) + _child_items_by_folder( + folder_model, + { + 'port-id': [_source_item('frame_metadata.csv')], + 'port-root-id': [], + 'starboard-id': [], + 'starboard-root-id': [], + 'parent-id': [], + 'parent-root-id': [_source_item('frame-metadata.txt')], + }, + ) + _wire_clone_roots( + get_clone_root, + { + 'parent-id': parent_root, + 'port-id': port_root, + 'starboard-id': starboard_root, + }, + ) + + result = crud_dataset.load_frame_metadata_sources(parent, user) + + # Camera-local precedes the shared parent root; the shared sidecar binds both cameras. + assert result == { + 'cameras': { + 'port': [_descriptor('frame_metadata.csv'), _descriptor('frame-metadata.txt')], + 'starboard': [_descriptor('frame-metadata.txt')], + }, + } + + +@patch('dive_server.crud_dataset.Folder') +@patch('dive_server.crud_dataset.crud.getCloneRoot') +def test_sources_multicam_skips_non_image_sequence_camera(get_clone_root, folder_cls): + parent = _multicam_parent_folder() + port = _camera_folder('port-id', 'port') + starboard = {'_id': 'starboard-id', 'name': 'starboard', 'meta': {'type': constants.VideoType}} + parent_root = _root_folder('parent-root-id') + user = {'_id': 'user-id'} + + folder_model = folder_cls.return_value + _wire_multicam_folders(folder_model, {'port-id': port, 'starboard-id': starboard}) + _child_items_by_folder( + folder_model, + { + 'port-id': [_source_item('frame_metadata.csv')], + 'port-root-id': [], + 'parent-id': [], + 'parent-root-id': [], + }, + ) + _wire_clone_roots( + get_clone_root, + {'parent-id': parent_root, 'port-id': _root_folder('port-root-id')}, + ) + + result = crud_dataset.load_frame_metadata_sources(parent, user) + + # The video camera is skipped entirely; only the image-sequence camera is keyed. + assert result == {'cameras': {'port': [_descriptor('frame_metadata.csv')]}} + + +@patch('dive_server.crud_dataset.Folder') +@patch('dive_server.crud_dataset.crud.getCloneRoot') +def test_sources_multicam_missing_folder_id_raises_400(get_clone_root, folder_cls): + parent = _multicam_parent_folder() + # Drop folderId from the first camera in order so the guard fires before any folder load. + parent['meta']['multiCam']['cameras']['port'].pop('folderId') + user = {'_id': 'user-id'} + get_clone_root.return_value = _root_folder('parent-root-id') + + with pytest.raises(RestException, match='Camera "port" missing folderId') as exc_info: + crud_dataset._load_multicam_frame_metadata_sources(parent, user) + + assert exc_info.value.code == 400 + + +@patch('girder.api.rest.Resource.route') +def test_dataset_resource_registers_frame_metadata_sources_route(route): + with patch('dive_server.views_dataset.Folder'): + resource = DatasetResource('dive_dataset') + + assert any( + call.args == ("GET", (":id", "frame_metadata_sources"), resource.get_frame_metadata_sources) + for call in route.call_args_list + ) + + +@patch('dive_server.views_dataset.crud_dataset.load_frame_metadata_sources') +def test_get_frame_metadata_sources_route_delegates(load_sources): + dataset = _dataset_folder() + user = {'_id': 'user-id'} + load_sources.return_value = {'cameras': {'singleCam': [_descriptor('frame_metadata.csv')]}} + + with patch('dive_server.views_dataset.Folder'): + resource = DatasetResource('dive_dataset') + resource.getCurrentUser = lambda: user + method = DatasetResource.get_frame_metadata_sources.__wrapped__.__wrapped__ + result = method(resource, dataset) + + assert result == {'cameras': {'singleCam': [_descriptor('frame_metadata.csv')]}} + load_sources.assert_called_once_with(dataset, user) diff --git a/server/tests/test_multicam_export_clone.py b/server/tests/test_multicam_export_clone.py index 21af2c7a7..247ec9f19 100644 --- a/server/tests/test_multicam_export_clone.py +++ b/server/tests/test_multicam_export_clone.py @@ -48,6 +48,14 @@ def _child_folder(folder_id: str, name: str): } +def _source_item(name: str): + return {'_id': f'{name}-id', 'name': name} + + +def _descriptor(name: str): + return {'itemId': f'{name}-id', 'name': name} + + @patch('dive_server.crud_dataset.find_json_calibration_item_id', return_value=None) @patch('dive_server.crud_dataset.find_calibration_item_id', return_value=None) @patch('dive_server.crud_dataset.crud_annotation.clone_annotations') @@ -129,7 +137,78 @@ def test_create_multicam_soft_clone_copies_calibration( clone_cals_mock.assert_called_once() saved_meta = folder_cls.return_value.save.call_args_list[-1][0][0]['meta'] assert saved_meta[constants.MultiCamMarker][constants.CalibrationItemIdMarker] == 'new-cal-src' - assert saved_meta[constants.MultiCamMarker][constants.JsonCalibrationItemIdMarker] == 'new-cal-json' + assert ( + saved_meta[constants.MultiCamMarker][constants.JsonCalibrationItemIdMarker] + == 'new-cal-json' + ) + + +@patch('dive_server.crud_dataset.find_json_calibration_item_id', return_value=None) +@patch('dive_server.crud_dataset.find_calibration_item_id', return_value=None) +@patch('dive_server.crud_dataset.crud_annotation.clone_annotations') +@patch('dive_server.crud_dataset.crud.get_or_create_auxiliary_folder') +@patch('dive_server.crud_dataset._create_single_camera_soft_clone') +@patch('dive_server.crud_dataset.Folder') +@patch('dive_server.crud.Folder') +def test_multicam_soft_clone_preserves_shared_parent_frame_metadata_source( + crud_folder_cls, + folder_cls, + create_soft_clone_mock, + _aux, + _clone_ann, + _find_cal, + _find_json_cal, +): + owner = {'login': 'tester'} + source = _multi_parent_folder() + parent = {'_id': 'dest-parent'} + left = _child_folder('left-id', 'left') + right = _child_folder('right-id', 'right') + cloned_left = { + **copy.deepcopy(left), + '_id': 'clone-left-id', + constants.ForeignMediaIdMarker: 'left-id', + } + cloned_right = { + **copy.deepcopy(right), + '_id': 'clone-right-id', + constants.ForeignMediaIdMarker: 'right-id', + } + cloned_parent = copy.deepcopy(source) + cloned_parent['_id'] = 'clone-parent-id' + + folder_model = folder_cls.return_value + folder_model.createFolder.return_value = cloned_parent + folders_by_id = { + 'parent-id': source, + 'left-id': left, + 'right-id': right, + 'clone-left-id': cloned_left, + 'clone-right-id': cloned_right, + } + folder_model.load.side_effect = lambda folder_id, **kwargs: folders_by_id.get(folder_id) + crud_folder_cls.return_value.load.side_effect = lambda folder_id, **kwargs: folders_by_id.get( + folder_id + ) + folder_model.childItems.side_effect = lambda folder: { + 'parent-id': [_source_item('frame-metadata.txt')], + 'clone-parent-id': [], + 'left-id': [], + 'right-id': [], + 'clone-left-id': [], + 'clone-right-id': [], + }.get(folder['_id'], []) + create_soft_clone_mock.side_effect = [cloned_left, cloned_right] + + result = crud_dataset.createSoftClone(owner, source, parent, 'Clone stereo', None) + sources = crud_dataset.load_frame_metadata_sources(result, owner) + + assert sources == { + 'cameras': { + 'left': [_descriptor('frame-metadata.txt')], + 'right': [_descriptor('frame-metadata.txt')], + }, + } @patch('dive_server.crud_dataset._yield_single_dataset_export') diff --git a/server/tests/test_process_items.py b/server/tests/test_process_items.py new file mode 100644 index 000000000..53da7913f --- /dev/null +++ b/server/tests/test_process_items.py @@ -0,0 +1,197 @@ +from unittest.mock import patch + +from girder.exceptions import RestException +import pytest + +from dive_server.crud_rpc import process_items +from dive_utils import constants + +VIAME_HEADER = ( + '# 1: Detection or Track-id, 2: Video or Image Identifier, 3: Unique Frame Identifier, ' + '4-7: Img-bbox(TL_x,TL_y,BR_x,BR_y), 8: Detection or Length Confidence, ' + '9: Fish Length, 10-11+: Repeated Species' +) + + +def _viame_csv(filename='image_0001.jpg'): + """A minimal, well-formed VIAME annotation CSV with the DIVE comment header.""" + return f'{VIAME_HEADER}\n0,{filename},0,10,10,50,50,1.0,-1,fish,0.9\n' + + +def _download_side_effect(bytes_by_file_id): + def download(file, headers=False): + return lambda: [bytes_by_file_id[file['_id']]] + + return download + + +def _childfiles_side_effect(file_by_item_id): + def child_files(item): + return iter([file_by_item_id[item['_id']]]) + + return child_files + + +@pytest.mark.parametrize( + 'dataset_type', + [constants.VideoType, constants.LargeImageType, constants.ImageSequenceType], +) +@patch('dive_server.crud_rpc.crud_annotation.save_annotations') +@patch('dive_server.crud_rpc.crud.get_or_create_auxiliary_folder') +@patch('dive_server.crud_rpc.File') +@patch('dive_server.crud_rpc.Item') +@patch('dive_server.crud_rpc.Folder') +def test_frame_metadata_csv_is_kept_in_place_for_every_media_type( + folder_cls, item_cls, file_cls, get_auxiliary_folder, save_annotations, dataset_type +): + folder = {'_id': 'ds', 'meta': {'type': dataset_type, 'fps': 5}} + item = {'_id': 'item-id', 'name': 'frame_metadata.csv', 'meta': {}} + file = {'_id': 'file-id', 'name': 'frame_metadata.csv', 'exts': ['csv']} + + folder_cls.return_value.childItems.return_value = [item] + item_cls.return_value.childFiles.side_effect = _childfiles_side_effect({'item-id': file}) + + warnings = process_items(folder, {'_id': 'user-id'}) + + # Declared by name: marked processed and left in the dataset folder, never imported as + # annotations, moved, or removed. The bytes are never even downloaded. + assert len(warnings) == 1 + assert 'frame_metadata.csv' in warnings[0] + assert 'stays in the dataset folder' in warnings[0] + assert item['meta'][constants.ProcessedMarker] is True + item_cls.return_value.save.assert_called_once_with(item) + item_cls.return_value.move.assert_not_called() + item_cls.return_value.remove.assert_not_called() + save_annotations.assert_not_called() + file_cls.return_value.download.assert_not_called() + + +@patch('dive_server.crud_rpc.crud.get_or_create_auxiliary_folder') +@patch('dive_server.crud_rpc.File') +@patch('dive_server.crud_rpc.Item') +@patch('dive_server.crud_rpc.Folder') +def test_frame_metadata_csv_marked_processed_is_not_reswept( + folder_cls, item_cls, file_cls, get_auxiliary_folder +): + folder = {'_id': 'ds', 'meta': {'type': constants.ImageSequenceType, 'fps': 5}} + item = {'_id': 'item-id', 'name': 'frame_metadata.csv', 'meta': {}} + file = {'_id': 'file-id', 'name': 'frame_metadata.csv', 'exts': ['csv']} + + # Emulate the ProcessedMarker "$ne: True" query filter: a marked sidecar is no longer + # listed, so a later process_items call never re-adjudicates it. + def child_items(_folder, filters=None, sort=None): + if item['meta'].get(constants.ProcessedMarker) is True: + return [] + return [item] + + folder_cls.return_value.childItems.side_effect = child_items + item_cls.return_value.childFiles.side_effect = _childfiles_side_effect({'item-id': file}) + + warnings = process_items(folder, {'_id': 'user-id'}) + assert len(warnings) == 1 + assert item['meta'][constants.ProcessedMarker] is True + item_cls.return_value.save.assert_called_once_with(item) + + # Second pass: the marked sidecar is excluded, so it is not re-saved or re-warned. + second_warnings = process_items(folder, {'_id': 'user-id'}) + assert second_warnings == [] + item_cls.return_value.save.assert_called_once() + + +@patch('dive_server.crud_rpc.crud_annotation.save_annotations') +@patch('dive_server.crud_rpc.crud.valid_images') +@patch('dive_server.crud_rpc.crud.get_or_create_auxiliary_folder') +@patch('dive_server.crud_rpc.File') +@patch('dive_server.crud_rpc.Item') +@patch('dive_server.crud_rpc.Folder') +def test_plain_annotation_csv_still_imports( + folder_cls, item_cls, file_cls, get_auxiliary_folder, valid_images, save_annotations +): + # The keep-in-place guard must not intercept an ordinary annotation CSV: it is still + # moved to the auxiliary folder and its tracks saved. + folder = {'_id': 'ds', 'meta': {'type': constants.ImageSequenceType, 'fps': 5}} + item = {'_id': 'item-id', 'name': 'annotations.csv', 'meta': {}} + file = {'_id': 'file-id', 'name': 'annotations.csv', 'exts': ['csv']} + + folder_cls.return_value.childItems.return_value = [item] + item_cls.return_value.childFiles.side_effect = _childfiles_side_effect({'item-id': file}) + file_cls.return_value.download.side_effect = _download_side_effect( + {'file-id': _viame_csv().encode()} + ) + get_auxiliary_folder.return_value = {'_id': 'aux-id'} + valid_images.return_value = [{'name': 'image_0001.jpg'}, {'name': 'image_0002.jpg'}] + + warnings = process_items(folder, {'_id': 'user-id'}) + + assert warnings == [] + item_cls.return_value.move.assert_called_once_with(item, {'_id': 'aux-id'}) + save_annotations.assert_called_once() + # An imported annotation is not tagged as a kept-in-place sidecar. + item_cls.return_value.save.assert_not_called() + + +@patch('dive_server.crud_rpc.crud_annotation.save_annotations') +@patch('dive_server.crud_rpc.crud.valid_images') +@patch('dive_server.crud_rpc.crud.get_or_create_auxiliary_folder') +@patch('dive_server.crud_rpc.File') +@patch('dive_server.crud_rpc.Item') +@patch('dive_server.crud_rpc.Folder') +def test_two_plain_csvs_in_folder_side_door_guard( + folder_cls, item_cls, file_cls, get_auxiliary_folder, valid_images, save_annotations +): + # Two plain (non-frame-metadata) annotation CSVs can only reach the folder together outside the + # pre-upload validation path. The guard fires before anything is removed, moved, or saved. + folder = {'_id': 'ds', 'meta': {'type': constants.ImageSequenceType, 'fps': 5}} + item_a = {'_id': 'a', 'name': 'a.csv', 'meta': {}} + item_b = {'_id': 'b', 'name': 'b.csv', 'meta': {}} + file_a = {'_id': 'fa', 'name': 'a.csv', 'exts': ['csv']} + file_b = {'_id': 'fb', 'name': 'b.csv', 'exts': ['csv']} + + folder_cls.return_value.childItems.return_value = [item_a, item_b] + item_cls.return_value.childFiles.side_effect = _childfiles_side_effect( + {'a': file_a, 'b': file_b} + ) + file_cls.return_value.download.side_effect = _download_side_effect( + {'fa': _viame_csv('image_0001.jpg').encode(), 'fb': _viame_csv('image_0002.jpg').encode()} + ) + + with pytest.raises(RestException, match='Multiple annotation CSVs present') as excinfo: + process_items(folder, {'_id': 'user-id'}) + + assert 'a.csv' in str(excinfo.value) + assert 'b.csv' in str(excinfo.value) + item_cls.return_value.remove.assert_not_called() + item_cls.return_value.move.assert_not_called() + item_cls.return_value.save.assert_not_called() + save_annotations.assert_not_called() + get_auxiliary_folder.assert_not_called() + + +@patch('dive_server.crud_rpc.crud_annotation.save_annotations') +@patch('dive_server.crud_rpc.crud.valid_images') +@patch('dive_server.crud_rpc.crud.get_or_create_auxiliary_folder') +@patch('dive_server.crud_rpc.File') +@patch('dive_server.crud_rpc.Item') +@patch('dive_server.crud_rpc.Folder') +def test_undecodable_plain_csv_fails_loudly_with_rename_hint( + folder_cls, item_cls, file_cls, get_auxiliary_folder, valid_images, save_annotations +): + # A plain .csv whose bytes are not valid UTF-8 fails the strict annotation decode; the + # loud failure carries the rename hint pointing frame metadata users at frame-metadata.csv. + folder = {'_id': 'ds', 'meta': {'type': constants.ImageSequenceType, 'fps': 5}} + item = {'_id': 'item-id', 'name': 'annotations.csv', 'meta': {}} + file = {'_id': 'file-id', 'name': 'annotations.csv', 'exts': ['csv']} + # Latin-1 accented bytes are invalid UTF-8, so the utf-8-sig decode raises (ValueError). + raw = 'filename,species\nimage_0001.jpg,poisson-\xe9p\xe9e\n'.encode('latin-1') + + folder_cls.return_value.childItems.return_value = [item] + item_cls.return_value.childFiles.side_effect = _childfiles_side_effect({'item-id': file}) + file_cls.return_value.download.side_effect = _download_side_effect({'file-id': raw}) + valid_images.return_value = [{'name': 'image_0001.jpg'}, {'name': 'image_0002.jpg'}] + + with pytest.raises(RestException, match='Failed to import annotations.csv') as excinfo: + process_items(folder, {'_id': 'user-id'}) + + assert 'frame-metadata.csv' in str(excinfo.value) + item_cls.return_value.remove.assert_called_once_with(item) + save_annotations.assert_not_called() diff --git a/server/tests/test_serialize_viame_csv.py b/server/tests/test_serialize_viame_csv.py index ea647f0ec..6255c5270 100644 --- a/server/tests/test_serialize_viame_csv.py +++ b/server/tests/test_serialize_viame_csv.py @@ -214,7 +214,9 @@ "properties": {"key": ""}, "geometry": { "type": "Polygon", - "coordinates": [[[100, 100], [200, 100], [200, 200], [100, 200]]], + "coordinates": [ + [[100, 100], [200, 100], [200, 200], [100, 200]] + ], }, }, { @@ -222,7 +224,9 @@ "properties": {"key": "1"}, "geometry": { "type": "Polygon", - "coordinates": [[[300, 300], [400, 300], [400, 400], [300, 400]]], + "coordinates": [ + [[300, 300], [400, 300], [400, 400], [300, 400]] + ], }, }, ], @@ -560,7 +564,5 @@ def test_fps_parsed_case_insensitively_from_metadata(): csv_text = ''.join(viame.export_tracks_as_csv([], header=True, fps=23.976)) assert '# metadata' in csv_text and 'fps: 23.976' in csv_text # exported lowercase rows = csv_text.splitlines() - _annotations, _attributes, _warnings, fps, _info = viame.load_csv_as_tracks_and_attributes( - rows - ) + _annotations, _attributes, _warnings, fps, _info = viame.load_csv_as_tracks_and_attributes(rows) assert float(fps) == 23.976 diff --git a/server/tests/test_validate_files.py b/server/tests/test_validate_files.py new file mode 100644 index 000000000..fefa85f23 --- /dev/null +++ b/server/tests/test_validate_files.py @@ -0,0 +1,214 @@ +from dive_server.crud_dataset import validate_files +from dive_utils import constants + +FRAME_METADATA_UNSUPPORTED_REASON = "Frame metadata is not supported for this media type" +UNSUPPORTED_SIDE_FILE_REASON = "Unsupported side file" + + +def _ignored_names(result): + return [entry["name"] for entry in result["ignored"]] + + +def _ignored_reason(result, name): + return next(entry["reason"] for entry in result["ignored"] if entry["name"] == name) + + +def test_image_sequence_with_annotation_csv_and_frame_metadata_csv(): + result = validate_files(['image_0001.jpg', 'tracks.csv', 'frame_metadata.csv']) + + assert result['ok'] is True + assert result['type'] == constants.ImageSequenceType + assert 'tracks.csv' in result['roles']['annotations'] + assert 'frame_metadata.csv' in result['roles']['frameMetadata'] + # The annotation CSV is not misclassified as a sidecar, and the sidecar is not an annotation. + assert 'frame_metadata.csv' not in result['roles']['annotations'] + assert 'tracks.csv' not in result['roles']['frameMetadata'] + assert 'tracks.csv' in result['upload'] + assert 'frame_metadata.csv' in result['upload'] + assert result['ignored'] == [] + + +def test_image_sequence_with_frame_metadata_csv_and_txt_names(): + result = validate_files( + [ + 'image_0001.jpg', + 'frame_metadata.csv', + 'frame_metadata.txt', + 'frame-metadata.csv', + 'frame-metadata.txt', + ] + ) + + assert result['ok'] is True + assert result['type'] == constants.ImageSequenceType + assert result['roles']['frameMetadata'] == [ + 'frame_metadata.csv', + 'frame_metadata.txt', + 'frame-metadata.csv', + 'frame-metadata.txt', + ] + assert 'frame_metadata.csv' in result['upload'] + assert 'frame_metadata.txt' in result['upload'] + assert 'frame-metadata.csv' in result['upload'] + assert 'frame-metadata.txt' in result['upload'] + assert result['roles']['annotations'] == [] + assert result['ignored'] == [] + + +def test_image_sequence_with_frame_metadata_paths_for_multicamera_import(): + result = validate_files( + [ + 'left/image_0001.jpg', + 'left/frame-metadata.txt', + 'right/image_0001.jpg', + 'right/frame_metadata.csv', + ] + ) + + assert result['ok'] is True + assert result['type'] == constants.ImageSequenceType + assert result['roles']['frameMetadata'] == [ + 'left/frame-metadata.txt', + 'right/frame_metadata.csv', + ] + assert 'left/frame-metadata.txt' in result['upload'] + assert 'right/frame_metadata.csv' in result['upload'] + + +def test_image_sequence_with_yaml_annotation(): + result = validate_files(['image_0001.jpg', 'annotations.yml']) + + assert result['ok'] is True + assert result['type'] == constants.ImageSequenceType + assert 'annotations.yml' in result['roles']['annotations'] + assert 'annotations.yml' in result['upload'] + + +def test_image_sequence_with_plain_txt_is_ignored(): + result = validate_files(['image_0001.jpg', 'notes.txt']) + + assert result['ok'] is True + assert result['type'] == constants.ImageSequenceType + assert 'notes.txt' in _ignored_names(result) + assert _ignored_reason(result, 'notes.txt') == UNSUPPORTED_SIDE_FILE_REASON + assert 'notes.txt' not in result['upload'] + # The rest of the package is unaffected. + assert result['roles']['media'] == ['image_0001.jpg'] + + +def test_image_sequence_with_unsupported_extension_is_ignored(): + result = validate_files(['image_0001.jpg', 'weird.xyz']) + + assert result['ok'] is True + assert 'weird.xyz' in _ignored_names(result) + assert _ignored_reason(result, 'weird.xyz') == UNSUPPORTED_SIDE_FILE_REASON + assert 'weird.xyz' not in result['upload'] + + +def test_two_plain_annotation_csvs_are_rejected(): + result = validate_files(['image_0001.jpg', 'a.csv', 'b.csv']) + + assert result['ok'] is False + assert result['message'] == "Can only upload a single CSV Annotation per import" + + +def test_image_sequence_with_config_json_is_dataset_config(): + result = validate_files(['image_0001.jpg', 'meta.json']) + + assert result['ok'] is True + assert 'meta.json' in result['roles']['datasetConfig'] + assert 'meta.json' not in result['roles']['annotations'] + assert 'meta.json' in result['upload'] + + +def test_annotation_json_and_config_json_are_distinguished(): + result = validate_files(['image_0001.jpg', 'tracks.json', 'config.json']) + + assert result['ok'] is True + assert 'config.json' in result['roles']['datasetConfig'] + assert 'tracks.json' in result['roles']['annotations'] + assert 'config.json' not in result['roles']['annotations'] + + +def test_video_with_frame_metadata_csv_ignores_sidecar(): + result = validate_files(['movie.mp4', 'frame_metadata.csv']) + + assert result['ok'] is True + assert result['type'] == constants.VideoType + assert result['roles']['frameMetadata'] == [] + assert 'frame_metadata.csv' in _ignored_names(result) + assert _ignored_reason(result, 'frame_metadata.csv') == FRAME_METADATA_UNSUPPORTED_REASON + assert 'frame_metadata.csv' not in result['upload'] + + +def test_large_image_with_frame_metadata_csv_ignores_sidecar(): + result = validate_files(['mosaic.tif', 'frame_metadata.csv']) + + assert result['ok'] is True + assert result['type'] == constants.LargeImageType + assert result['roles']['frameMetadata'] == [] + assert 'frame_metadata.csv' in _ignored_names(result) + assert _ignored_reason(result, 'frame_metadata.csv') == FRAME_METADATA_UNSUPPORTED_REASON + assert 'frame_metadata.csv' not in result['upload'] + + +def test_upload_order_is_media_then_annotations_then_config_then_frame_metadata(): + result = validate_files(['image_0001.jpg', 'tracks.csv', 'meta.json', 'frame_metadata.csv']) + + assert result['ok'] is True + assert result['upload'] == ['image_0001.jpg', 'tracks.csv', 'meta.json', 'frame_metadata.csv'] + + +def test_images_and_videos_mixed_is_rejected(): + result = validate_files(['image_0001.jpg', 'movie.mp4']) + + assert result['ok'] is False + assert result['type'] == '' + assert result['message'] == "Do not upload images and videos in the same batch." + + +def test_csv_and_yaml_mixed_is_rejected(): + result = validate_files(['image_0001.jpg', 'tracks.csv', 'config.yml']) + + assert result['ok'] is False + assert result['message'] == "Cannot mix annotation import types" + + +def test_csv_and_annotation_json_mixed_is_rejected(): + # Two annotation sources of different formats would silently overwrite at import. + result = validate_files(['image_0001.jpg', 'tracks.csv', 'tracks.json']) + + assert result['ok'] is False + assert result['message'] == "Cannot mix annotation import types" + + +def test_yaml_and_annotation_json_mixed_is_rejected(): + result = validate_files(['image_0001.jpg', 'tracks.yml', 'tracks.json']) + + assert result['ok'] is False + assert result['message'] == "Cannot mix annotation import types" + + +def test_multiple_videos_with_config_json_is_rejected(): + # A single dataset-config JSON cannot apply to a multi-video (subfolder) upload. + result = validate_files(['a.mp4', 'b.mp4', 'config.json']) + + assert result['ok'] is False + assert ( + result['message'] == "Annotation upload is not supported when multiple videos are uploaded" + ) + + +def test_multiple_videos_without_annotations_is_allowed(): + result = validate_files(['a.mp4', 'b.mp4']) + + assert result['ok'] is True + assert result['type'] == constants.VideoType + assert result['roles']['media'] == ['a.mp4', 'b.mp4'] + + +def test_no_media_is_rejected(): + result = validate_files(['tracks.csv']) + + assert result['ok'] is False + assert result['message'] == "No supported media-type files found" diff --git a/testdata/frame-metadata-conformance/source_names.expected.json b/testdata/frame-metadata-conformance/source_names.expected.json new file mode 100644 index 000000000..065de0a94 --- /dev/null +++ b/testdata/frame-metadata-conformance/source_names.expected.json @@ -0,0 +1,19 @@ +{ + "frame-metadata.txt": true, + "frame-metadata.csv": true, + "FRAME-METADATA.TXT": true, + "frame_metadata.csv": true, + "frame_metadata.txt": true, + "FRAME_METADATA.CSV": true, + "left/frame-metadata.txt": true, + "left/frame-metadata.csv": true, + "right\\frame_metadata.csv": true, + "right\\frame_metadata.txt": true, + "nav.meta.csv": false, + "a.b.meta.txt": false, + "nav.csv": false, + "nav.txt": false, + "prefix-frame-metadata.txt": false, + "frame_metadata.csv.bak": false, + ".meta.csv": false +}