diff --git a/packages/openclinxr/arena/model-vetting/src/index.ts b/packages/openclinxr/arena/model-vetting/src/index.ts index 354c9f26c..1b8b4c203 100644 --- a/packages/openclinxr/arena/model-vetting/src/index.ts +++ b/packages/openclinxr/arena/model-vetting/src/index.ts @@ -1,5 +1,10 @@ export * from "./types.js"; export * from "./logic.js"; +export { + visemeTimelineFromRhubarbCues, + visemeForRhubarbValue, +} from "./viseme-timeline.js"; +export type { RhubarbMouthCue, VisemeCueTiming } from "./viseme-timeline.js"; export type { CandidatePromotionStatus, CandidateRiggingDelta, diff --git a/packages/openclinxr/arena/model-vetting/src/the-viseme-timeline-consumes-rhubarb-cues.test.ts b/packages/openclinxr/arena/model-vetting/src/the-viseme-timeline-consumes-rhubarb-cues.test.ts new file mode 100644 index 000000000..914318679 --- /dev/null +++ b/packages/openclinxr/arena/model-vetting/src/the-viseme-timeline-consumes-rhubarb-cues.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; +import { + PEDS_ASTHMA_PATIENT_VISeme_DIALOGUE_UTTERANCE, + buildVisemeTimelineFromDialogue, + visemeForRhubarbValue, + visemeOpenness, + visemeTimelineFromRhubarbCues, + type RhubarbMouthCue, +} from "./viseme-timeline.js"; + +/** + * Tracked factory-shaped Rhubarb JSON (seconds, not fitted ms). + * Two mouth shapes over 400ms. Failed treatment: live gitignored evidence JSON. + */ +const MAYA_RHUBARB_DB_400MS_JSON = `{ + "metadata": { "duration": 0.4, "soundFile": "fixture-maya.wav" }, + "mouthCues": [ + { "start": 0.0, "end": 0.2, "value": "D" }, + { "start": 0.2, "end": 0.4, "value": "B" } + ] +}`; + +const MAYA_RHUBARB_FX_400MS_JSON = `{ + "metadata": { "duration": 0.4, "soundFile": "fixture-maya.wav" }, + "mouthCues": [ + { "start": 0.0, "end": 0.2, "value": "F" }, + { "start": 0.2, "end": 0.4, "value": "X" } + ] +}`; + +const ROUND_TRIP_MS = 10; + +function cuesFromTrackedJson(raw: string): RhubarbMouthCue[] { + const parsed = JSON.parse(raw) as { mouthCues: RhubarbMouthCue[] }; + return parsed.mouthCues; +} + +describe("the viseme timeline consumes rhubarb cues", () => { + it("builds Maya's timeline from Rhubarb D/B cues, not letter-class heuristics", () => { + const cues = cuesFromTrackedJson(MAYA_RHUBARB_DB_400MS_JSON); + const fromCues = visemeTimelineFromRhubarbCues(cues, { + sourceWavPath: "fixture-maya.wav", + dialogueText: PEDS_ASTHMA_PATIENT_VISeme_DIALOGUE_UTTERANCE, + }); + const fromLetters = buildVisemeTimelineFromDialogue(PEDS_ASTHMA_PATIENT_VISeme_DIALOGUE_UTTERANCE); + + expect(fromCues.mappingMode).toBe("rhubarb_cue_json"); + expect(fromCues.mappingMode).not.toBe("deterministic_text_phoneme_viseme_runtime_cue"); + expect(fromCues.actorId).toBe("patient_maya_johnson_v1"); + expect(fromCues.phonemeSequence).toEqual(["D", "B"]); + expect(fromCues.visemeSequence).toEqual(["OH", "E"]); + expect(fromCues.visemeSequence).not.toEqual(fromLetters.visemeSequence); + expect(fromCues.sourceWavPath).toBe("fixture-maya.wav"); + }); + + it("round-trips Rhubarb start/end within 10ms over a 400ms two-shape fixture", () => { + const cues = cuesFromTrackedJson(MAYA_RHUBARB_DB_400MS_JSON); + const timeline = visemeTimelineFromRhubarbCues(cues); + expect(timeline.cueTimings).toHaveLength(2); + expect(Math.abs((timeline.durationMs ?? 0) - 400)).toBeLessThanOrEqual(ROUND_TRIP_MS); + for (const [index, cue] of cues.entries()) { + const timing = timeline.cueTimings?.[index]; + expect(timing).toBeDefined(); + expect(Math.abs((timing?.startMs ?? NaN) - cue.start * 1000)).toBeLessThanOrEqual(ROUND_TRIP_MS); + expect(Math.abs((timing?.endMs ?? NaN) - cue.end * 1000)).toBeLessThanOrEqual(ROUND_TRIP_MS); + expect(timing?.value).toBe(cue.value); + } + }); + + it("maps every Rhubarb A-H/X value to the landed runtime token table", () => { + const table: ReadonlyArray = [ + ["A", "AA"], + ["B", "E"], + ["C", "IH"], + ["D", "OH"], + ["E", "OU"], + ["F", "FV"], + ["G", "L"], + ["H", "OU"], + ["X", "sil"], + ]; + expect(table.map(([letter]) => letter).join("")).toBe("ABCDEFGHX"); + for (const [letter, token] of table) { + expect(visemeForRhubarbValue(letter)).toBe(token); + expect(visemeForRhubarbValue(letter.toLowerCase())).toBe(token); + } + const cues: RhubarbMouthCue[] = table.map(([value], index) => ({ + start: index * 0.05, + end: (index + 1) * 0.05, + value, + })); + expect(visemeTimelineFromRhubarbCues(cues).visemeSequence).toEqual(table.map(([, token]) => token)); + }); + + it("yields different openness sequences for two different cue fixtures", () => { + const db = visemeTimelineFromRhubarbCues(cuesFromTrackedJson(MAYA_RHUBARB_DB_400MS_JSON)); + const fx = visemeTimelineFromRhubarbCues(cuesFromTrackedJson(MAYA_RHUBARB_FX_400MS_JSON)); + expect(fx.mappingMode).toBe("rhubarb_cue_json"); + expect(fx.phonemeSequence).toEqual(["F", "X"]); + expect(fx.visemeSequence).toEqual(["FV", "sil"]); + expect(db.visemeSequence).not.toEqual(fx.visemeSequence); + const dbOpenness = db.visemeSequence.map(visemeOpenness); + const fxOpenness = fx.visemeSequence.map(visemeOpenness); + expect(dbOpenness).not.toEqual(fxOpenness); + }); +}); diff --git a/packages/openclinxr/arena/model-vetting/src/viseme-timeline.ts b/packages/openclinxr/arena/model-vetting/src/viseme-timeline.ts index 73e5240f4..1ceb03458 100644 --- a/packages/openclinxr/arena/model-vetting/src/viseme-timeline.ts +++ b/packages/openclinxr/arena/model-vetting/src/viseme-timeline.ts @@ -1,7 +1,22 @@ export const PEDS_ASTHMA_PATIENT_VISeme_DIALOGUE_UTTERANCE = "Maya Johnson: It is hard to breathe and my chest feels tight."; -export type VisemeTimelineMappingMode = "deterministic_text_phoneme_viseme_runtime_cue"; +export type VisemeTimelineMappingMode = + | "deterministic_text_phoneme_viseme_runtime_cue" + | "rhubarb_cue_json"; + +/** One mouth cue from factory lip_sync Rhubarb `--exportFormat json` (start/end in seconds). */ +export type RhubarbMouthCue = { + start: number; + end: number; + value: string; +}; + +export type VisemeCueTiming = { + startMs: number; + endMs: number; + value: string; +}; export type VisemeTimeline = { dialogueText: string; @@ -11,16 +26,45 @@ export type VisemeTimeline = { mappingMode: VisemeTimelineMappingMode; traceTag: "work_of_breathing_assessment"; actorId: "patient_maya_johnson_v1"; + sourceWavPath?: string; + cueTimings?: VisemeCueTiming[]; +}; + +/** + * Rhubarb mouth-shape letters → same runtime tokens as ui-xr viseme-baked-cues + * (A=AA, B=E, C=IH, D=OH, E=OU, F=FV, G=L, H=OU, X=sil). Copied here so this + * package does not import ui-xr. + */ +const RHUBARB_VALUE_TO_VISEME: Record = { + A: "AA", + B: "E", + C: "IH", + D: "OH", + E: "OU", + F: "FV", + G: "L", + H: "OU", + X: "sil", }; +const RHUBARB_START_END_ROUND_TRIP_MS = 10; + const VISEME_OPENNESS: Record = { rest: 0, + sil: 0, closed: 0.08, teeth: 0.2, + FV: 0.2, rounded: 0.34, + OU: 0.34, wide: 0.46, + E: 0.46, + IH: 0.46, mid: 0.52, + L: 0.52, open: 0.78, + AA: 0.78, + OH: 0.78, }; export function phonemeSequenceForDialogue(text: string): string[] { @@ -48,6 +92,54 @@ export function visemeForPhoneme(phoneme: string): string { return "mid"; } +export function visemeForRhubarbValue(value: string): string { + return RHUBARB_VALUE_TO_VISEME[value.toUpperCase()] ?? "sil"; +} + +/** + * Factory Q5 path: ingest lip_sync Rhubarb cue JSON (`start`/`end` seconds, `value` A–H/X). + * Does not spawn `say` or `rhubarb`. + */ +export function visemeTimelineFromRhubarbCues( + cues: readonly RhubarbMouthCue[], + input?: { sourceWavPath?: string; dialogueText?: string }, +): VisemeTimeline { + if (cues.length === 0) { + throw new Error("visemeTimelineFromRhubarbCues requires at least one Rhubarb mouth cue"); + } + const cueTimings: VisemeCueTiming[] = cues.map((cue) => ({ + startMs: cue.start * 1000, + endMs: cue.end * 1000, + value: cue.value, + })); + for (const [index, cue] of cues.entries()) { + const timing = cueTimings[index]; + if ( + timing === undefined || + Math.abs(timing.startMs - cue.start * 1000) > RHUBARB_START_END_ROUND_TRIP_MS || + Math.abs(timing.endMs - cue.end * 1000) > RHUBARB_START_END_ROUND_TRIP_MS + ) { + throw new Error(`Rhubarb cue ${index} start/end did not round-trip within ${RHUBARB_START_END_ROUND_TRIP_MS}ms`); + } + } + const last = cueTimings[cueTimings.length - 1]; + const visemeSequence = cues.map((cue) => visemeForRhubarbValue(cue.value)); + const timeline: VisemeTimeline = { + dialogueText: input?.dialogueText ?? PEDS_ASTHMA_PATIENT_VISeme_DIALOGUE_UTTERANCE, + phonemeSequence: cues.map((cue) => cue.value), + visemeSequence, + durationMs: last?.endMs ?? 0, + mappingMode: "rhubarb_cue_json", + traceTag: "work_of_breathing_assessment", + actorId: "patient_maya_johnson_v1", + cueTimings, + }; + if (input?.sourceWavPath !== undefined) { + timeline.sourceWavPath = input.sourceWavPath; + } + return timeline; +} + export function visemeOpenness(viseme: string): number { return VISEME_OPENNESS[viseme] ?? 0.35; } @@ -57,6 +149,7 @@ export function humanoidDialogueDurationMs(phonemeCount: number, extendedCapture return extendedCapture ? Math.max(baseDurationMs, 4500) : baseDurationMs; } +/** Fixture-only letter heuristic. Prefer visemeTimelineFromRhubarbCues when cues exist. */ export function buildVisemeTimelineFromDialogue( dialogueText: string, input?: { extendedCapture?: boolean }, @@ -81,6 +174,23 @@ export function visemeAtTimelineProgress(timeline: VisemeTimeline, progress: num openness: number; } { const clampedProgress = Math.min(1, Math.max(0, progress)); + if (timeline.cueTimings && timeline.cueTimings.length > 0 && timeline.durationMs > 0) { + const atMs = clampedProgress * timeline.durationMs; + let index = timeline.cueTimings.length - 1; + for (const [i, cue] of timeline.cueTimings.entries()) { + if (atMs >= cue.startMs && atMs < cue.endMs) { + index = i; + break; + } + } + const viseme = timeline.visemeSequence[index] ?? "rest"; + return { + index, + phoneme: timeline.phonemeSequence[index] ?? "sil", + viseme, + openness: visemeOpenness(viseme), + }; + } const index = Math.min( timeline.visemeSequence.length - 1, Math.max(0, Math.floor(clampedProgress * timeline.visemeSequence.length)), @@ -128,7 +238,10 @@ export function applyMorphTargetVisemeCue( appliedTargetCount += 1; } if (typeof browConcernIndex === "number") { - mesh.morphTargetInfluences[browConcernIndex] = Math.min(0.95, Math.max(0, viseme === "rest" ? 0.12 : 0.28)); + mesh.morphTargetInfluences[browConcernIndex] = Math.min( + 0.95, + Math.max(0, viseme === "rest" || viseme === "sil" ? 0.12 : 0.28), + ); appliedTargetCount += 1; } if (typeof cheekTensionIndex === "number") {