feat(model-vetting): ingest factory Rhubarb viseme cues

packages/openclinxr/arena/model-vetting/src/index.ts

@@ -1,5 +1,10 @@
 export * from "./types.js";
 export * from "./logic.js";
+export {
+  visemeTimelineFromRhubarbCues,
+  visemeForRhubarbValue,
+} from "./viseme-timeline.js";
+export type { RhubarbMouthCue, VisemeCueTiming } from "./viseme-timeline.js";
 export type {
   CandidatePromotionStatus,
   CandidateRiggingDelta,

packages/openclinxr/arena/model-vetting/src/the-viseme-timeline-consumes-rhubarb-cues.test.ts

@@ -0,0 +1,106 @@
+import { describe, expect, it } from "vitest";
+import {
+  PEDS_ASTHMA_PATIENT_VISeme_DIALOGUE_UTTERANCE,
+  buildVisemeTimelineFromDialogue,
+  visemeForRhubarbValue,
+  visemeOpenness,
+  visemeTimelineFromRhubarbCues,
+  type RhubarbMouthCue,
+} from "./viseme-timeline.js";
+
+/**
+ * Tracked factory-shaped Rhubarb JSON (seconds, not fitted ms).
+ * Two mouth shapes over 400ms. Failed treatment: live gitignored evidence JSON.
+ */
+const MAYA_RHUBARB_DB_400MS_JSON = `{
+  "metadata": { "duration": 0.4, "soundFile": "fixture-maya.wav" },
+  "mouthCues": [
+    { "start": 0.0, "end": 0.2, "value": "D" },
+    { "start": 0.2, "end": 0.4, "value": "B" }
+  ]
+}`;
+
+const MAYA_RHUBARB_FX_400MS_JSON = `{
+  "metadata": { "duration": 0.4, "soundFile": "fixture-maya.wav" },
+  "mouthCues": [
+    { "start": 0.0, "end": 0.2, "value": "F" },
+    { "start": 0.2, "end": 0.4, "value": "X" }
+  ]
+}`;
+
+const ROUND_TRIP_MS = 10;
+
+function cuesFromTrackedJson(raw: string): RhubarbMouthCue[] {
+  const parsed = JSON.parse(raw) as { mouthCues: RhubarbMouthCue[] };
+  return parsed.mouthCues;
+}
+
+describe("the viseme timeline consumes rhubarb cues", () => {
+  it("builds Maya's timeline from Rhubarb D/B cues, not letter-class heuristics", () => {
+    const cues = cuesFromTrackedJson(MAYA_RHUBARB_DB_400MS_JSON);
+    const fromCues = visemeTimelineFromRhubarbCues(cues, {
+      sourceWavPath: "fixture-maya.wav",
+      dialogueText: PEDS_ASTHMA_PATIENT_VISeme_DIALOGUE_UTTERANCE,
+    });
+    const fromLetters = buildVisemeTimelineFromDialogue(PEDS_ASTHMA_PATIENT_VISeme_DIALOGUE_UTTERANCE);
+
+    expect(fromCues.mappingMode).toBe("rhubarb_cue_json");
+    expect(fromCues.mappingMode).not.toBe("deterministic_text_phoneme_viseme_runtime_cue");
+    expect(fromCues.actorId).toBe("patient_maya_johnson_v1");
+    expect(fromCues.phonemeSequence).toEqual(["D", "B"]);
+    expect(fromCues.visemeSequence).toEqual(["OH", "E"]);
+    expect(fromCues.visemeSequence).not.toEqual(fromLetters.visemeSequence);
+    expect(fromCues.sourceWavPath).toBe("fixture-maya.wav");
+  });
+
+  it("round-trips Rhubarb start/end within 10ms over a 400ms two-shape fixture", () => {
+    const cues = cuesFromTrackedJson(MAYA_RHUBARB_DB_400MS_JSON);
+    const timeline = visemeTimelineFromRhubarbCues(cues);
+    expect(timeline.cueTimings).toHaveLength(2);
+    expect(Math.abs((timeline.durationMs ?? 0) - 400)).toBeLessThanOrEqual(ROUND_TRIP_MS);
+    for (const [index, cue] of cues.entries()) {
+      const timing = timeline.cueTimings?.[index];
+      expect(timing).toBeDefined();
+      expect(Math.abs((timing?.startMs ?? NaN) - cue.start * 1000)).toBeLessThanOrEqual(ROUND_TRIP_MS);
+      expect(Math.abs((timing?.endMs ?? NaN) - cue.end * 1000)).toBeLessThanOrEqual(ROUND_TRIP_MS);
+      expect(timing?.value).toBe(cue.value);
+    }
+  });
+
+  it("maps every Rhubarb A-H/X value to the landed runtime token table", () => {
+    const table: ReadonlyArray<readonly [string, string]> = [
+      ["A", "AA"],
+      ["B", "E"],
+      ["C", "IH"],
+      ["D", "OH"],
+      ["E", "OU"],
+      ["F", "FV"],
+      ["G", "L"],
+      ["H", "OU"],
+      ["X", "sil"],
+    ];
+    expect(table.map(([letter]) => letter).join("")).toBe("ABCDEFGHX");
+    for (const [letter, token] of table) {
+      expect(visemeForRhubarbValue(letter)).toBe(token);
+      expect(visemeForRhubarbValue(letter.toLowerCase())).toBe(token);
+    }
+    const cues: RhubarbMouthCue[] = table.map(([value], index) => ({
+      start: index * 0.05,
+      end: (index + 1) * 0.05,
+      value,
+    }));
+    expect(visemeTimelineFromRhubarbCues(cues).visemeSequence).toEqual(table.map(([, token]) => token));
+  });
+
+  it("yields different openness sequences for two different cue fixtures", () => {
+    const db = visemeTimelineFromRhubarbCues(cuesFromTrackedJson(MAYA_RHUBARB_DB_400MS_JSON));
+    const fx = visemeTimelineFromRhubarbCues(cuesFromTrackedJson(MAYA_RHUBARB_FX_400MS_JSON));
+    expect(fx.mappingMode).toBe("rhubarb_cue_json");
+    expect(fx.phonemeSequence).toEqual(["F", "X"]);
+    expect(fx.visemeSequence).toEqual(["FV", "sil"]);
+    expect(db.visemeSequence).not.toEqual(fx.visemeSequence);
+    const dbOpenness = db.visemeSequence.map(visemeOpenness);
+    const fxOpenness = fx.visemeSequence.map(visemeOpenness);
+    expect(dbOpenness).not.toEqual(fxOpenness);
+  });
+});

packages/openclinxr/arena/model-vetting/src/viseme-timeline.ts

@@ -1,7 +1,22 @@
 export const PEDS_ASTHMA_PATIENT_VISeme_DIALOGUE_UTTERANCE =
   "Maya Johnson: It is hard to breathe and my chest feels tight.";
 
-export type VisemeTimelineMappingMode = "deterministic_text_phoneme_viseme_runtime_cue";
+export type VisemeTimelineMappingMode =
+  | "deterministic_text_phoneme_viseme_runtime_cue"
+  | "rhubarb_cue_json";
+
+/** One mouth cue from factory lip_sync Rhubarb `--exportFormat json` (start/end in seconds). */
+export type RhubarbMouthCue = {
+  start: number;
+  end: number;
+  value: string;
+};
+
+export type VisemeCueTiming = {
+  startMs: number;
+  endMs: number;
+  value: string;
+};
 
 export type VisemeTimeline = {
   dialogueText: string;
@@ -11,16 +26,45 @@ export type VisemeTimeline = {
   mappingMode: VisemeTimelineMappingMode;
   traceTag: "work_of_breathing_assessment";
   actorId: "patient_maya_johnson_v1";
+  sourceWavPath?: string;
+  cueTimings?: VisemeCueTiming[];
+};
+
+/**
+ * Rhubarb mouth-shape letters → same runtime tokens as ui-xr viseme-baked-cues
+ * (A=AA, B=E, C=IH, D=OH, E=OU, F=FV, G=L, H=OU, X=sil). Copied here so this
+ * package does not import ui-xr.
+ */
+const RHUBARB_VALUE_TO_VISEME: Record<string, string> = {
+  A: "AA",
+  B: "E",
+  C: "IH",
+  D: "OH",
+  E: "OU",
+  F: "FV",
+  G: "L",
+  H: "OU",
+  X: "sil",
 };
 
+const RHUBARB_START_END_ROUND_TRIP_MS = 10;
+
 const VISEME_OPENNESS: Record<string, number> = {
   rest: 0,
+  sil: 0,
   closed: 0.08,
   teeth: 0.2,
+  FV: 0.2,
   rounded: 0.34,
+  OU: 0.34,
   wide: 0.46,
+  E: 0.46,
+  IH: 0.46,
   mid: 0.52,
+  L: 0.52,
   open: 0.78,
+  AA: 0.78,
+  OH: 0.78,
 };
 
 export function phonemeSequenceForDialogue(text: string): string[] {
@@ -48,6 +92,54 @@ export function visemeForPhoneme(phoneme: string): string {
   return "mid";
 }
 
+export function visemeForRhubarbValue(value: string): string {
+  return RHUBARB_VALUE_TO_VISEME[value.toUpperCase()] ?? "sil";
+}
+
+/**
+ * Factory Q5 path: ingest lip_sync Rhubarb cue JSON (`start`/`end` seconds, `value` A–H/X).
+ * Does not spawn `say` or `rhubarb`.
+ */
+export function visemeTimelineFromRhubarbCues(
+  cues: readonly RhubarbMouthCue[],
+  input?: { sourceWavPath?: string; dialogueText?: string },
+): VisemeTimeline {
+  if (cues.length === 0) {
+    throw new Error("visemeTimelineFromRhubarbCues requires at least one Rhubarb mouth cue");
+  }
+  const cueTimings: VisemeCueTiming[] = cues.map((cue) => ({
+    startMs: cue.start * 1000,
+    endMs: cue.end * 1000,
+    value: cue.value,
+  }));
+  for (const [index, cue] of cues.entries()) {
+    const timing = cueTimings[index];
+    if (
+      timing === undefined ||
+      Math.abs(timing.startMs - cue.start * 1000) > RHUBARB_START_END_ROUND_TRIP_MS ||
+      Math.abs(timing.endMs - cue.end * 1000) > RHUBARB_START_END_ROUND_TRIP_MS
+    ) {
+      throw new Error(`Rhubarb cue ${index} start/end did not round-trip within ${RHUBARB_START_END_ROUND_TRIP_MS}ms`);
+    }
+  }
+  const last = cueTimings[cueTimings.length - 1];
+  const visemeSequence = cues.map((cue) => visemeForRhubarbValue(cue.value));
+  const timeline: VisemeTimeline = {
+    dialogueText: input?.dialogueText ?? PEDS_ASTHMA_PATIENT_VISeme_DIALOGUE_UTTERANCE,
+    phonemeSequence: cues.map((cue) => cue.value),
+    visemeSequence,
+    durationMs: last?.endMs ?? 0,
+    mappingMode: "rhubarb_cue_json",
+    traceTag: "work_of_breathing_assessment",
+    actorId: "patient_maya_johnson_v1",
+    cueTimings,
+  };
+  if (input?.sourceWavPath !== undefined) {
+    timeline.sourceWavPath = input.sourceWavPath;
+  }
+  return timeline;
+}
+
 export function visemeOpenness(viseme: string): number {
   return VISEME_OPENNESS[viseme] ?? 0.35;
 }
@@ -57,6 +149,7 @@ export function humanoidDialogueDurationMs(phonemeCount: number, extendedCapture
   return extendedCapture ? Math.max(baseDurationMs, 4500) : baseDurationMs;
 }
 
+/** Fixture-only letter heuristic. Prefer visemeTimelineFromRhubarbCues when cues exist. */
 export function buildVisemeTimelineFromDialogue(
   dialogueText: string,
   input?: { extendedCapture?: boolean },
@@ -81,6 +174,23 @@ export function visemeAtTimelineProgress(timeline: VisemeTimeline, progress: num
   openness: number;
 } {
   const clampedProgress = Math.min(1, Math.max(0, progress));
+  if (timeline.cueTimings && timeline.cueTimings.length > 0 && timeline.durationMs > 0) {
+    const atMs = clampedProgress * timeline.durationMs;
+    let index = timeline.cueTimings.length - 1;
+    for (const [i, cue] of timeline.cueTimings.entries()) {
+      if (atMs >= cue.startMs && atMs < cue.endMs) {
+        index = i;
+        break;
+      }
+    }
+    const viseme = timeline.visemeSequence[index] ?? "rest";
+    return {
+      index,
+      phoneme: timeline.phonemeSequence[index] ?? "sil",
+      viseme,
+      openness: visemeOpenness(viseme),
+    };
+  }
   const index = Math.min(
     timeline.visemeSequence.length - 1,
     Math.max(0, Math.floor(clampedProgress * timeline.visemeSequence.length)),
@@ -128,7 +238,10 @@ export function applyMorphTargetVisemeCue(
       appliedTargetCount += 1;
     }
     if (typeof browConcernIndex === "number") {
-      mesh.morphTargetInfluences[browConcernIndex] = Math.min(0.95, Math.max(0, viseme === "rest" ? 0.12 : 0.28));
+      mesh.morphTargetInfluences[browConcernIndex] = Math.min(
+        0.95,
+        Math.max(0, viseme === "rest" || viseme === "sil" ? 0.12 : 0.28),
+      );
       appliedTargetCount += 1;
     }
     if (typeof cheekTensionIndex === "number") {