diff --git a/client/dive-common/components/TrackSettingsPanel.vue b/client/dive-common/components/TrackSettingsPanel.vue index 8144c3aea..a1ac15d86 100644 --- a/client/dive-common/components/TrackSettingsPanel.vue +++ b/client/dive-common/components/TrackSettingsPanel.vue @@ -7,6 +7,7 @@ import { computed, } from 'vue'; import { clientSettings } from 'dive-common/store/settings'; +import { STEREO_MATCH_METHODS } from 'dive-common/use/stereo/stereoMatcher'; import isDesktopRuntime from 'dive-common/isDesktopRuntime'; export default defineComponent({ @@ -40,6 +41,7 @@ export default defineComponent({ showMultiCamToolbar: 'Show multi-camera tools in the top toolbar when a track is selected', stereoUpdateLengths: 'When a line annotation is modified on a detection that is linked across both cameras, recompute its stereo measurement (length, midpoint, range, RMS) automatically.', stereoAutoCompute: 'When an annotation is drawn on one camera and the other camera has no detection for it yet, automatically warp it to the other camera using stereo disparity.', + stereoMatchMethod: 'How a point is matched on the other camera. Template matching correlates the source patch along the epipolar line. Foundation stereo runs a dense disparity network over the pair once, which is steadier where the patch is hard to match but needs its model to be configured.', }); const modes = ref(['Track', 'Detection']); // Add unknown as the default type to the typeList @@ -52,6 +54,7 @@ export default defineComponent({ help, modes, typeList, + stereoMatchMethods: STEREO_MATCH_METHODS, }; }, }); @@ -445,6 +448,42 @@ export default defineComponent({ + + + + + + + + {{ help.stereoMatchMethod }} + + + diff --git a/client/dive-common/store/settings.ts b/client/dive-common/store/settings.ts index c5d145b42..fdffc69a8 100644 --- a/client/dive-common/store/settings.ts +++ b/client/dive-common/store/settings.ts @@ -2,6 +2,8 @@ import { Ref, watch, reactive } from 'vue'; import { cloneDeep, merge } from 'lodash'; import { AnnotatorPreferences } from 'vue-media-annotator/types'; import isDesktopRuntime from 'dive-common/isDesktopRuntime'; +import { DEFAULT_STEREO_MATCH_METHOD } from 'dive-common/use/stereo/stereoMatcher'; +import type { StereoMatchMethod } from 'dive-common/use/stereo/stereoMatcher'; interface ColumnVisibilitySettings { type: boolean; @@ -94,6 +96,9 @@ interface AnnotationSettings { // Warp an annotation drawn on one camera to the other camera when that // camera has no detection for it yet. autoComputeOtherCamera: boolean; + // Which correspondence method the warp uses: 'ncc' template matching or + // 'foundation' dense disparity. + matchMethod: StereoMatchMethod; loading: boolean; loadingMessage: string; }; @@ -191,6 +196,7 @@ const defaultSettings: AnnotationSettings = { clearLengthOnCameraFileLoad: true, updateLengthsOnModify: true, autoComputeOtherCamera: false, + matchMethod: DEFAULT_STEREO_MATCH_METHOD, loading: false, loadingMessage: '', }, diff --git a/client/dive-common/use/stereo/README.md b/client/dive-common/use/stereo/README.md index 84a866588..387a8489a 100644 --- a/client/dive-common/use/stereo/README.md +++ b/client/dive-common/use/stereo/README.md @@ -1,9 +1,20 @@ # Client-side stereo transfer and measurement (ONNX) Warp a detection annotated on one camera onto the other camera and measure its -length, entirely in the browser / Electron renderer — no backend — using VIAME's -epipolar template-matching model (stereo measurement "method 1") exported to -ONNX and run with `onnxruntime-web`. +length, entirely in the browser / Electron renderer — no backend — running the +correspondence model with `onnxruntime-web`. + +Two correspondence methods are available, chosen from **Track Settings → Stereo +Settings → Correspondence method**: + +| Method | Model | How it matches | +| --- | --- | --- | +| **Template matching (NCC)** — default | VIAME's epipolar template-matching model (stereo measurement "method 1"), bundled | Per point: generate epipolar candidates, NCC the source patch along that curve | +| **Foundation stereo (disparity)** | A Fast-FoundationStereo ONNX export, **not bundled** | Once per frame: rectify the pair, run a dense disparity network, read each point's shift out of the map | + +They are interchangeable behind the `StereoMatcher` interface, so everything +downstream — box/line/polygon warping, measurement, bulk transfer — is identical +either way. This is the client counterpart to the desktop backend stereo service: the desktop `ViewerLoader` warps and measures via native IPC (`stereoTransferLine` / @@ -15,6 +26,9 @@ work client-side so it also works on the web. | File | Role | | --- | --- | | `StereoOnnxMatcher.ts` | Loads the `match` ONNX model and warps source points → target points via NCC along the epipolar curve. | +| `StereoFoundationMatcher.ts` | Loads a Fast-FoundationStereo ONNX export, rectifies the pair into the network's input resolution, and reads each point's correspondence from the dense disparity map. | +| `stereoMatcher.ts` | The `StereoMatcher` contract both matchers satisfy, the `StereoMatchMethod` union, and the dropdown's labels. | +| `rectify.ts` | Stereo rectification ported from OpenCV `cvStereoRectify` (Rodrigues, rectifying rotations, point rectify/unrectify, and the inverse map used to sample a rectified image). Only the foundation method needs it. | | `calibration.ts` | `StereoRig` + loaders (`rigFromNpz`, `rigFromJson`) mirroring VIAME's `read_stereo_rig`; `invertRig` to swap the source/target camera. | | `npz.ts` | Minimal `.npz`/`.npy` reader (calibration files are NumPy archives). | | `image.ts` | RGBA → BT.601 grayscale (matches OpenCV `BGR2GRAY` used by the C++ NCC). | @@ -107,3 +121,71 @@ The disparity range is scene-dependent — VIAME's batch measurement pipes ship binding, calibration download, and the GeoJS frame-pixel read in `frameSource.geoViewerToImageElement`) is type-checked and lint-clean but has not been exercised in a running web viewer with a real stereo dataset. + + +## Foundation stereo method + +### Why a second method + +The NCC matcher needs the source patch to be photometrically matchable in the +other view. Where that fails — obstructed viewpoints, repetitive substrate, low +contrast — it either mismatches or declines. A dense disparity network does not +depend on patch correlation, and it costs one network pass per frame no matter +how many points are warped, so bulk-warping a whole camera amortises well. + +Its trade is setup: the model is large and must be supplied. + +### Supplying the model + +Unlike the NCC graph (small, committed at `client/public/models/stereo_match.onnx`), +Fast-FoundationStereo exports run ~100 MB and are **not** committed. Obtain an +export from the Fast-FoundationStereo release, serve it, and point the web glue +at it: + +```ts +useStereoOnnxWeb({ + ..., + foundationModelUrl: '/models/stereo_foundation.onnx', + foundationModelSpec: { height: 576, width: 960 }, // the export's sidecar image_size +}); +``` + +The default URL is `/models/stereo_foundation.onnx` and the default spec is +576×960. `foundationModelSpec` **must** match the export: the graph fixes its +input resolution, and the sidecar `.yaml` shipped beside each export gives it as +`image_size: [H, W]`. With no model served, selecting the method reports that it +could not load and the warp no-ops — the same way a missing calibration does. + +### How it works + +1. Solve the rectifying rotations for the rig once per calibration + (`computeRectification`), sized to the network's input resolution. +2. Build the rectified pair by inverse-mapping each output pixel back to its + source pixel and bilinear-sampling. Rectify and resize are fused, so the cost + is the network's resolution rather than the frame's. +3. Run the network to get dense disparity in rectified pixels. +4. Per point: rectify it, pool the disparities in a small window by median, + shift `x` by that disparity, and unrectify into the target image. + +Step 4 pools rather than sampling the single pixel deliberately. A head or tail +tip is a couple of pixels wide at the network's working resolution, so the +disparity exactly at the tip is frequently the background's; the median over a +small window rejects that without dragging the estimate off the animal. + +The network emits no confidence channel, so the reported `score` is the fraction +of the pooled window carrying a finite positive disparity, and a match is +accepted when that clears `DEFAULT_MIN_VALID_FRACTION` **and** the implied +disparity falls inside the configured search range — the same range that bounds +the NCC search. + +### Testing status + +- **Tested** (`tests/rectify.spec.ts`): Rodrigues round-trip, orthonormality of + the rectifying rotations, the defining rectification property (a 3D point + lands on the same row in both rectified views), disparity positive and + decreasing with range, and pixel round-trip through rectify/unrectify with and + without distortion. +- **Not tested**: `StereoFoundationMatcher` end-to-end, which needs a ~100 MB + model the repo does not carry. The geometry it depends on is covered above; + the network call, disparity pooling and the settings dropdown have not been + exercised against a real export in a running viewer. diff --git a/client/dive-common/use/stereo/StereoFoundationMatcher.ts b/client/dive-common/use/stereo/StereoFoundationMatcher.ts new file mode 100644 index 000000000..a0250d697 --- /dev/null +++ b/client/dive-common/use/stereo/StereoFoundationMatcher.ts @@ -0,0 +1,201 @@ +/** + * Client-side wrapper around a Fast-FoundationStereo ONNX export (NVIDIA), as + * the second stereo correspondence method alongside {@link StereoOnnxMatcher}. + * + * Where the NCC matcher searches the epipolar curve per point, this one runs a + * dense disparity network over the whole pair once and reads each point's + * correspondence out of the disparity map. That costs one network pass per + * frame regardless of how many points are warped, and it does not depend on the + * source patch being photometrically matchable — which is what makes it hold up + * on footage where template correlation struggles (obstructed views, repetitive + * substrate, low contrast). + * + * The model is NOT bundled: the exports are ~100 MB, far past what belongs in + * the repo. Point {@link StereoFoundationMatcher.create} at a served or + * user-supplied model. Exports are published with the Fast-FoundationStereo + * release as `_iters__res_x.onnx` plus a sidecar `.yaml` giving + * `image_size`; the graph takes `left_image`/`right_image` as [1,3,H,W] RGB in + * [0,1] and returns `disparity` as [1,1,H,W] in rectified pixels. + */ + +import * as ort from 'onnxruntime-web'; + +import { GrayImage } from './image'; +import { StereoRig } from './calibration'; +import type { WarpOptions, WarpResult } from './StereoOnnxMatcher'; +import { + Rectification, computeRectification, rectifyPoint, rectifyMapper, unrectifyPoint, +} from './rectify'; + +/** + * Half-width of the window whose disparities are pooled for one point. + * + * A head or tail tip is a couple of pixels wide at the network's working + * resolution, so the disparity sampled exactly at the tip is often the + * background's. Pooling a small neighbourhood by median rejects that without + * dragging the estimate off the animal. + */ +export const DEFAULT_SAMPLE_RADIUS = 3; + +/** + * Fraction of the pooled window that must carry a finite positive disparity for + * the match to be accepted. The network emits a dense map with no confidence + * channel, so validity density is the available proxy. + */ +export const DEFAULT_MIN_VALID_FRACTION = 0.34; + +export interface FoundationModelSpec { + /** Network input size, from the export's sidecar yaml `image_size: [H, W]`. */ + height: number; + width: number; +} + +/** Bilinear sample of a single-channel image, NaN outside. */ +function sampleBilinear(data: Float32Array, width: number, height: number, x: number, y: number): number { + if (!(x >= 0 && y >= 0 && x <= width - 1 && y <= height - 1)) return NaN; + const x0 = Math.floor(x); + const y0 = Math.floor(y); + const x1 = Math.min(x0 + 1, width - 1); + const y1 = Math.min(y0 + 1, height - 1); + const fx = x - x0; + const fy = y - y0; + const a = data[y0 * width + x0]; + const b = data[y0 * width + x1]; + const c = data[y1 * width + x0]; + const d = data[y1 * width + x1]; + return a * (1 - fx) * (1 - fy) + b * fx * (1 - fy) + c * (1 - fx) * fy + d * fx * fy; +} + +/** Remap a grayscale frame through an inverse map into an RGB [1,3,H,W] tensor. */ +function remapToRgbTensor(src: GrayImage, mapX: Float32Array, mapY: Float32Array, width: number, height: number): ort.Tensor { + const plane = width * height; + const out = new Float32Array(plane * 3); + for (let i = 0; i < plane; i += 1) { + const v = sampleBilinear(src.data, src.width, src.height, mapX[i], mapY[i]); + const g = Number.isNaN(v) ? 0 : v; + out[i] = g; + out[plane + i] = g; + out[2 * plane + i] = g; + } + return new ort.Tensor('float32', out, [1, 3, height, width]); +} + +export class StereoFoundationMatcher { + private session: ort.InferenceSession; + + private spec: FoundationModelSpec; + + /** Rectification + inverse maps, rebuilt only when the rig or size changes. */ + private cache: { + key: string; rect: Rectification; + src: { mapX: Float32Array; mapY: Float32Array }; + tgt: { mapX: Float32Array; mapY: Float32Array }; + } | null = null; + + private constructor(session: ort.InferenceSession, spec: FoundationModelSpec) { + this.session = session; + this.spec = spec; + } + + /** + * Create a matcher from a model URL or model bytes. `spec` is the export's + * input resolution (its sidecar yaml `image_size`), which the graph fixes. + */ + static async create( + model: string | ArrayBuffer | Uint8Array, + spec: FoundationModelSpec, + opts: { threads?: number } = {}, + ): Promise { + ort.env.wasm.numThreads = opts.threads ?? 1; + ort.env.wasm.proxy = false; + const session = await ort.InferenceSession.create(model as string, { + executionProviders: ['wasm'], + graphOptimizationLevel: 'all', + }); + return new StereoFoundationMatcher(session, spec); + } + + /** Rectification and inverse maps for this rig at the model's resolution. */ + private geometry(rig: StereoRig) { + const key = `${rig.Kl.join(',')}|${rig.R.join(',')}|${rig.T.join(',')}`; + if (this.cache && this.cache.key === key) return this.cache; + const rect = computeRectification(rig, this.spec.width, this.spec.height); + this.cache = { + key, + rect, + src: rectifyMapper(rig, rect, false), + tgt: rectifyMapper(rig, rect, true), + }; + return this.cache; + } + + /** + * Warp source-image points onto the target image, matching + * {@link StereoOnnxMatcher.warpPoints} so the two are interchangeable. + * + * `opts.range` bounds the accepted disparity exactly as it bounds the NCC + * search: a correspondence outside it is rejected rather than trusted. + */ + async warpPoints( + points: [number, number][], + source: GrayImage, + target: GrayImage, + rig: StereoRig, + opts: WarpOptions, + ): Promise { + const { rect, src, tgt } = this.geometry(rig); + const { width, height } = this.spec; + + const feeds: Record = { + left_image: remapToRgbTensor(source, src.mapX, src.mapY, width, height), + right_image: remapToRgbTensor(target, tgt.mapX, tgt.mapY, width, height), + }; + const out = await this.session.run(feeds); + const disparity = out.disparity.data as Float32Array; + + const radius = DEFAULT_SAMPLE_RADIUS; + const minValid = DEFAULT_MIN_VALID_FRACTION; + const [minDisp, maxDisp] = 'minDisparity' in opts.range + ? [opts.range.minDisparity, opts.range.maxDisparity] + : [0, Number.POSITIVE_INFINITY]; + // The search range is expressed in source-image pixels; the network works + // at its own resolution, so carry the bound across in the same ratio. + const dispScale = width / source.width; + + return points.map(([px, py]) => { + const [rx, ry] = rectifyPoint(px, py, rig, rect, false); + const fail: WarpResult = { + x: NaN, y: NaN, score: 0, secondScore: 0, accepted: false, + }; + if (!Number.isFinite(rx) || !Number.isFinite(ry)) return fail; + + const samples: number[] = []; + let considered = 0; + for (let dy = -radius; dy <= radius; dy += 1) { + for (let dx = -radius; dx <= radius; dx += 1) { + considered += 1; + const v = sampleBilinear(disparity, width, height, rx + dx, ry + dy); + if (Number.isFinite(v) && v > 0) samples.push(v); + } + } + if (!samples.length) return fail; + + samples.sort((a, b) => a - b); + const d = samples[Math.floor(samples.length / 2)]; + const validFraction = samples.length / considered; + + const dSource = d / dispScale; + const inRange = dSource >= minDisp && dSource <= maxDisp; + const [ox, oy] = unrectifyPoint(rx - d, ry, rig, rect, true); + if (!Number.isFinite(ox) || !Number.isFinite(oy)) return fail; + + return { + x: ox, + y: oy, + score: validFraction, + secondScore: 0, + accepted: validFraction >= minValid && inRange, + }; + }); + } +} diff --git a/client/dive-common/use/stereo/index.ts b/client/dive-common/use/stereo/index.ts index cc5ad56bd..01db38c26 100644 --- a/client/dive-common/use/stereo/index.ts +++ b/client/dive-common/use/stereo/index.ts @@ -1,4 +1,14 @@ export { StereoOnnxMatcher } from './StereoOnnxMatcher'; +export { StereoFoundationMatcher } from './StereoFoundationMatcher'; +export type { FoundationModelSpec } from './StereoFoundationMatcher'; +export { + DEFAULT_STEREO_MATCH_METHOD, STEREO_MATCH_METHODS, +} from './stereoMatcher'; +export type { StereoMatcher, StereoMatchMethod } from './stereoMatcher'; +export { + computeRectification, rectifyPoint, unrectifyPoint, rectifyMapper, +} from './rectify'; +export type { Rectification } from './rectify'; export type { WarpOptions, WarpResult, SearchRange } from './StereoOnnxMatcher'; export { rigFromNpz, rigFromNpzArrays, rigFromJson, baseline, diff --git a/client/dive-common/use/stereo/rectify.ts b/client/dive-common/use/stereo/rectify.ts new file mode 100644 index 000000000..79cad859d --- /dev/null +++ b/client/dive-common/use/stereo/rectify.ts @@ -0,0 +1,166 @@ +/** + * Stereo rectification, ported from OpenCV's `cvStereoRectify` (Bouguet). + * + * The NCC matcher searches the epipolar curve directly and needs none of this. + * A disparity network does: it consumes a rectified pair, where corresponding + * points share a row and the correspondence is a pure horizontal shift. + * + * Nothing here builds a full-resolution rectified image. `rectifyMapper` + * returns the inverse map (rectified pixel -> source pixel) so a caller can + * sample straight into the network's input resolution, fusing rectify+resize + * into one bilinear read per output pixel. + */ + +import { StereoRig } from './calibration'; +import { mapPoint, unmap } from './triangulate'; + +export type Mat3 = Float32Array; +type Vec3 = [number, number, number]; + +export interface Rectification { + /** Rectifying rotations for the source and target cameras. */ + R1: Mat3; + R2: Mat3; + /** Shared focal length and principal point of the rectified pair. */ + f: number; + cx: number; + cy: number; + /** Rectified image size these were solved for. */ + width: number; + height: number; +} + +function matMul(a: ArrayLike, b: ArrayLike): Mat3 { + const m = new Float32Array(9); + for (let r = 0; r < 3; r += 1) { + for (let c = 0; c < 3; c += 1) { + m[r * 3 + c] = a[r * 3] * b[c] + a[r * 3 + 1] * b[3 + c] + a[r * 3 + 2] * b[6 + c]; + } + } + return m; +} + +function transpose(a: ArrayLike): Mat3 { + return Float32Array.from([a[0], a[3], a[6], a[1], a[4], a[7], a[2], a[5], a[8]]); +} + +function matVec(a: ArrayLike, v: ArrayLike): Vec3 { + return [ + a[0] * v[0] + a[1] * v[1] + a[2] * v[2], + a[3] * v[0] + a[4] * v[1] + a[5] * v[2], + a[6] * v[0] + a[7] * v[1] + a[8] * v[2], + ]; +} + +/** Rotation matrix -> rotation vector (axis * angle). */ +export function rodriguesInv(R: ArrayLike): Vec3 { + const trace = R[0] + R[4] + R[8]; + const cos = Math.min(1, Math.max(-1, (trace - 1) / 2)); + const angle = Math.acos(cos); + if (angle < 1e-9) return [0, 0, 0]; + const s = angle / (2 * Math.sin(angle)); + return [s * (R[7] - R[5]), s * (R[2] - R[6]), s * (R[3] - R[1])]; +} + +/** Rotation vector -> rotation matrix. */ +export function rodrigues(v: ArrayLike): Mat3 { + const theta = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]); + if (theta < 1e-9) return Float32Array.from([1, 0, 0, 0, 1, 0, 0, 0, 1]); + const [x, y, z] = [v[0] / theta, v[1] / theta, v[2] / theta]; + const c = Math.cos(theta); + const s = Math.sin(theta); + const t = 1 - c; + return Float32Array.from([ + t * x * x + c, t * x * y - s * z, t * x * z + s * y, + t * x * y + s * z, t * y * y + c, t * y * z - s * x, + t * x * z - s * y, t * y * z + s * x, t * z * z + c, + ]); +} + +/** + * Solve the rectifying rotations for a rig. + * + * The focal length is the smaller of the two cameras' so the rectified frustum + * stays inside both, and the principal point is centred on the output. This is + * OpenCV's `alpha = 0`-free behaviour: no zoom-to-valid-region crop, which on a + * rig whose baseline sits far from horizontal demands an extreme zoom and can + * push the whole scene off canvas. + */ +export function computeRectification(rig: StereoRig, width: number, height: number): Rectification { + // Half-rotate both cameras toward each other: r = R^(-1/2). + const om = rodriguesInv(rig.R); + const r = rodrigues([-om[0] / 2, -om[1] / 2, -om[2] / 2]); + const t = matVec(r, rig.T); + + // New x axis along the (half-rotated) baseline. + const nt = Math.hypot(t[0], t[1], t[2]) || 1; + const horizontal = Math.abs(t[0]) > Math.abs(t[1]); + const idx = horizontal ? 0 : 1; + const uu: Vec3 = [0, 0, 0]; + uu[idx] = t[idx] > 0 ? 1 : -1; + + // Rotate about the axis that carries the baseline onto uu. + const ww: Vec3 = [ + t[1] * uu[2] - t[2] * uu[1], + t[2] * uu[0] - t[0] * uu[2], + t[0] * uu[1] - t[1] * uu[0], + ]; + const nw = Math.hypot(ww[0], ww[1], ww[2]); + let wR: Mat3; + if (nw < 1e-12) { + wR = Float32Array.from([1, 0, 0, 0, 1, 0, 0, 0, 1]); + } else { + const scale = Math.acos(Math.abs(t[idx]) / nt) / nw; + wR = rodrigues([ww[0] * scale, ww[1] * scale, ww[2] * scale]); + } + + return { + R1: matMul(wR, transpose(r)), + R2: matMul(wR, r), + f: Math.min(rig.Kl[0], rig.Kr[0]), + cx: (width - 1) / 2, + cy: (height - 1) / 2, + width, + height, + }; +} + +/** Source pixel -> rectified pixel, for the source (R1) or target (R2) camera. */ +export function rectifyPoint(px: number, py: number, rig: StereoRig, rect: Rectification, target: boolean): [number, number] { + const K = target ? rig.Kr : rig.Kl; + const d = target ? rig.distr : rig.distl; + const R = target ? rect.R2 : rect.R1; + const [nx, ny] = unmap(px, py, K, d); + const p = matVec(R, [nx, ny, 1]); + if (p[2] === 0) return [NaN, NaN]; + return [rect.f * (p[0] / p[2]) + rect.cx, rect.f * (p[1] / p[2]) + rect.cy]; +} + +/** Rectified pixel -> source pixel (the inverse of {@link rectifyPoint}). */ +export function unrectifyPoint(rx: number, ry: number, rig: StereoRig, rect: Rectification, target: boolean): [number, number] { + const K = target ? rig.Kr : rig.Kl; + const d = target ? rig.distr : rig.distl; + const R = target ? rect.R2 : rect.R1; + const p = matVec(transpose(R), [(rx - rect.cx) / rect.f, (ry - rect.cy) / rect.f, 1]); + if (p[2] === 0) return [NaN, NaN]; + return mapPoint(p[0] / p[2], p[1] / p[2], K, d); +} + +/** + * Inverse map for building a rectified image: for each rectified pixel, the + * source pixel to sample. Returned as flat x/y arrays of length width*height so + * the caller can bilinear-sample without recomputing the projection per frame. + */ +export function rectifyMapper(rig: StereoRig, rect: Rectification, target: boolean): { mapX: Float32Array; mapY: Float32Array } { + const { width, height } = rect; + const mapX = new Float32Array(width * height); + const mapY = new Float32Array(width * height); + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const [sx, sy] = unrectifyPoint(x, y, rig, rect, target); + mapX[y * width + x] = sx; + mapY[y * width + x] = sy; + } + } + return { mapX, mapY }; +} diff --git a/client/dive-common/use/stereo/stereoMatcher.ts b/client/dive-common/use/stereo/stereoMatcher.ts new file mode 100644 index 000000000..be22ac7d7 --- /dev/null +++ b/client/dive-common/use/stereo/stereoMatcher.ts @@ -0,0 +1,32 @@ +/** + * The contract both correspondence methods satisfy, so the transfer composable + * and its callers never branch on which one is selected. + */ + +import { GrayImage } from './image'; +import { StereoRig } from './calibration'; +import type { WarpOptions, WarpResult } from './StereoOnnxMatcher'; + +/** + * `ncc` — epipolar candidates + NCC template matching (VIAME method 1). + * `foundation` — dense Fast-FoundationStereo disparity, read per point. + */ +export type StereoMatchMethod = 'ncc' | 'foundation'; + +export const DEFAULT_STEREO_MATCH_METHOD: StereoMatchMethod = 'ncc'; + +export interface StereoMatcher { + warpPoints( + points: [number, number][], + source: GrayImage, + target: GrayImage, + rig: StereoRig, + opts: WarpOptions, + ): Promise; +} + +/** Labels for the method selector. */ +export const STEREO_MATCH_METHODS: { value: StereoMatchMethod; text: string }[] = [ + { value: 'ncc', text: 'Template matching (NCC)' }, + { value: 'foundation', text: 'Foundation stereo (disparity)' }, +]; diff --git a/client/dive-common/use/stereo/tests/rectify.spec.ts b/client/dive-common/use/stereo/tests/rectify.spec.ts new file mode 100644 index 000000000..3daded55c --- /dev/null +++ b/client/dive-common/use/stereo/tests/rectify.spec.ts @@ -0,0 +1,122 @@ +/** + * Rectification is the only new geometry the foundation matcher adds, and it is + * the part that silently produces plausible-but-wrong warps if it is off. These + * check it against properties that must hold for any correct rectification, + * rather than against a golden matrix. + */ + +import { describe, it, expect } from 'vitest'; + +import { + computeRectification, rectifyPoint, unrectifyPoint, rodrigues, rodriguesInv, +} from '../rectify'; +import { StereoRig } from '../calibration'; +import { project } from '../triangulate'; + +const I3 = Float32Array.from([1, 0, 0, 0, 1, 0, 0, 0, 1]); +const Z3 = Float32Array.from([0, 0, 0]); + +/** Project a world point (left-camera frame) into one of the rig's cameras. */ +function projectInto(p: [number, number, number], rig: StereoRig, target: boolean): [number, number] { + return target + ? project(p, rig.Kr, rig.distr, rig.R, rig.T) + : project(p, rig.Kl, rig.distl, I3, Z3); +} + +const W = 960; +const H = 576; + +/** A rig with a mostly-horizontal baseline and a small relative rotation. */ +function makeRig(rotation: [number, number, number] = [0.01, -0.02, 0.004]): StereoRig { + const K = Float32Array.from([1000, 0, 640, 0, 1000, 400, 0, 0, 1]); + return { + Kl: K, + Kr: Float32Array.from(K), + distl: new Float32Array(8), + distr: new Float32Array(8), + R: rodrigues(rotation), + T: Float32Array.from([-200, -30, -5]), + }; +} + +describe('rodrigues', () => { + it('round-trips a rotation vector through the matrix form', () => { + const v: [number, number, number] = [0.11, -0.24, 0.07]; + const back = rodriguesInv(rodrigues(v)); + back.forEach((c, i) => expect(c).toBeCloseTo(v[i], 6)); + }); + + it('returns identity for a zero rotation', () => { + const R = rodrigues([0, 0, 0]); + expect(Array.from(R)).toEqual([1, 0, 0, 0, 1, 0, 0, 0, 1]); + }); +}); + +describe('computeRectification', () => { + it('produces orthonormal rectifying rotations', () => { + const { R1, R2 } = computeRectification(makeRig(), W, H); + [R1, R2].forEach((R) => { + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + const dot = R[i * 3] * R[j * 3] + R[i * 3 + 1] * R[j * 3 + 1] + R[i * 3 + 2] * R[j * 3 + 2]; + expect(dot).toBeCloseTo(i === j ? 1 : 0, 5); + } + } + }); + }); + + it('puts corresponding points on the same rectified row', () => { + // The defining property of rectification: a 3D point seen by both cameras + // must land on one row, so disparity is a pure horizontal shift. + const rig = makeRig(); + const rect = computeRectification(rig, W, H); + const worldPoints: [number, number, number][] = [ + [0, 0, 3000], [400, -200, 2500], [-350, 250, 4000], [120, 90, 1800], + ]; + worldPoints.forEach((p) => { + const left = projectInto(p, rig, false); + const right = projectInto(p, rig, true); + const [, ly] = rectifyPoint(left[0], left[1], rig, rect, false); + const [, ry] = rectifyPoint(right[0], right[1], rig, rect, true); + expect(ry).toBeCloseTo(ly, 2); + }); + }); + + it('gives a positive disparity that shrinks with range', () => { + const rig = makeRig(); + const rect = computeRectification(rig, W, H); + const disparityAt = (z: number) => { + const p: [number, number, number] = [0, 0, z]; + const [lx] = rectifyPoint(...projectInto(p, rig, false), rig, rect, false); + const [rx] = rectifyPoint(...projectInto(p, rig, true), rig, rect, true); + return lx - rx; + }; + const near = disparityAt(1500); + const far = disparityAt(6000); + expect(near).toBeGreaterThan(0); + expect(far).toBeGreaterThan(0); + expect(near).toBeGreaterThan(far); + }); +}); + +describe('rectifyPoint / unrectifyPoint', () => { + it('round-trips a pixel on both cameras, with and without distortion', () => { + const plain = makeRig(); + const distorted: StereoRig = { + ...plain, + distl: Float32Array.from([-0.16, 0.10, -0.001, 0.002, 0, 0, 0, 0]), + distr: Float32Array.from([-0.15, 0.09, -0.001, 0.002, 0, 0, 0, 0]), + }; + [plain, distorted].forEach((rig) => { + const rect = computeRectification(rig, W, H); + [false, true].forEach((target) => { + [[640, 400], [300, 180], [900, 550]].forEach(([px, py]) => { + const [rx, ry] = rectifyPoint(px, py, rig, rect, target); + const [bx, by] = unrectifyPoint(rx, ry, rig, rect, target); + expect(bx).toBeCloseTo(px, 2); + expect(by).toBeCloseTo(py, 2); + }); + }); + }); + }); +}); diff --git a/client/dive-common/use/stereo/useStereoOnnxTransfer.ts b/client/dive-common/use/stereo/useStereoOnnxTransfer.ts index 0b53ab744..57f062c51 100644 --- a/client/dive-common/use/stereo/useStereoOnnxTransfer.ts +++ b/client/dive-common/use/stereo/useStereoOnnxTransfer.ts @@ -19,7 +19,8 @@ import Track from 'vue-media-annotator/track'; import { RectBounds } from 'vue-media-annotator/utils'; import { HeadPointKey, TailPointKey, HeadTailLineKey } from 'dive-common/recipes/headtail'; import type { StereoAnnotationCompleteParams } from '../useModeManager'; -import { StereoOnnxMatcher, SearchRange } from './StereoOnnxMatcher'; +import type { SearchRange } from './StereoOnnxMatcher'; +import type { StereoMatcher } from './stereoMatcher'; import { StereoRig, invertRig } from './calibration'; import { rgbaToGray, RgbaImage } from './image'; import { measureLine, aggregateLengths, StereoMeasurement } from './triangulate'; @@ -32,8 +33,12 @@ export interface StereoOnnxTransferConfig { getLeftCameraName: () => string; /** Stereo calibration, or null if unavailable (transfer is then skipped). */ getRig: () => Promise; - /** The (lazily created / cached) ONNX matcher, or null if unavailable. */ - getMatcher: () => Promise; + /** + * The (lazily created / cached) matcher for the selected method, or null if + * unavailable. Either correspondence method satisfies {@link StereoMatcher}, + * so nothing downstream branches on which one is in use. + */ + getMatcher: () => Promise; /** Full-resolution RGBA pixels for a camera at a frame, or null. */ getFrame: (cameraName: string, frameNum: number) => Promise; /** Disparity- or depth-based search range for the correspondence search. */ diff --git a/client/platform/web-girder/useStereoOnnxWeb.ts b/client/platform/web-girder/useStereoOnnxWeb.ts index a27ed8fc2..64305ac48 100644 --- a/client/platform/web-girder/useStereoOnnxWeb.ts +++ b/client/platform/web-girder/useStereoOnnxWeb.ts @@ -17,6 +17,10 @@ import { clientSettings } from 'dive-common/store/settings'; import useStereoOnnxTransfer from 'dive-common/use/stereo/useStereoOnnxTransfer'; import { StereoOnnxMatcher } from 'dive-common/use/stereo/StereoOnnxMatcher'; +import { StereoFoundationMatcher } from 'dive-common/use/stereo/StereoFoundationMatcher'; +import type { FoundationModelSpec } from 'dive-common/use/stereo/StereoFoundationMatcher'; +import { DEFAULT_STEREO_MATCH_METHOD } from 'dive-common/use/stereo/stereoMatcher'; +import type { StereoMatcher, StereoMatchMethod } from 'dive-common/use/stereo/stereoMatcher'; import type { SearchRange } from 'dive-common/use/stereo/StereoOnnxMatcher'; import { rigFromNpz, rigFromJson, StereoRig, @@ -27,6 +31,16 @@ import type { StereoMeasurement } from 'dive-common/use/stereo/triangulate'; import { getCalibrationFile, getLastCalibration } from './multicamFileRegistry'; const DEFAULT_MODEL_URL = '/models/stereo_match.onnx'; +/** + * Fast-FoundationStereo is opt-in and unbundled: the exports run ~100 MB, so + * unlike the NCC graph this one is not committed. Serve an export here (or pass + * `foundationModelUrl`) and give its sidecar `image_size` as + * `foundationModelSpec`; with no model served the dropdown's foundation option + * reports that it could not load and the warp no-ops, exactly as a missing + * calibration does. + */ +const DEFAULT_FOUNDATION_MODEL_URL = '/models/stereo_foundation.onnx'; +const DEFAULT_FOUNDATION_SPEC: FoundationModelSpec = { height: 576, width: 960 }; // Mirrors epipolar_min_disparity / epipolar_max_disparity in VIAME's // configs/pipelines/interactive_stereo_template.conf, which is what the desktop // interactive stereo service loads. Scene-dependent, and hidden config there @@ -41,6 +55,11 @@ export interface StereoOnnxWebOptions { /** Dataset (folder) id used to look up the stored calibration. */ getDatasetId: () => string; modelUrl?: string; + foundationModelUrl?: string; + /** Input resolution of the foundation export (its sidecar yaml `image_size`). */ + foundationModelSpec?: FoundationModelSpec; + /** Overrides the user's dropdown choice; mainly for tests. */ + getMatchMethod?: () => StereoMatchMethod; range?: SearchRange; onStatus?: (message: string | null) => void; onError?: (message: string) => void; @@ -80,22 +99,32 @@ async function urlToRgba(url: string): Promise { export default function useStereoOnnxWeb(opts: StereoOnnxWebOptions) { const modelUrl = opts.modelUrl ?? DEFAULT_MODEL_URL; - let matcher: StereoOnnxMatcher | null = null; - let matcherTried = false; + const foundationModelUrl = opts.foundationModelUrl ?? DEFAULT_FOUNDATION_MODEL_URL; + const foundationSpec = opts.foundationModelSpec ?? DEFAULT_FOUNDATION_SPEC; + // Cached per method: switching the dropdown must not reload the other model, + // and a method that failed to load must not be retried on every warp. + const matchers: Partial> = {}; let rig: StereoRig | null = null; let rigKey: string | null = null; - async function getMatcher(): Promise { - if (!matcher && !matcherTried) { - matcherTried = true; - try { - matcher = await StereoOnnxMatcher.create(modelUrl); - } catch (err) { - console.warn('[StereoOnnx] failed to load model', modelUrl, err); - matcher = null; - } + function currentMethod(): StereoMatchMethod { + if (opts.getMatchMethod) return opts.getMatchMethod(); + return clientSettings.stereoSettings.matchMethod ?? DEFAULT_STEREO_MATCH_METHOD; + } + + async function getMatcher(): Promise { + const method = currentMethod(); + if (method in matchers) return matchers[method] ?? null; + const url = method === 'foundation' ? foundationModelUrl : modelUrl; + try { + matchers[method] = method === 'foundation' + ? await StereoFoundationMatcher.create(url, foundationSpec) + : await StereoOnnxMatcher.create(url); + } catch (err) { + console.warn('[StereoOnnx] failed to load model', method, url, err); + matchers[method] = null; } - return matcher; + return matchers[method] ?? null; } function parseRig(name: string, buffer: ArrayBuffer): Promise {