Skip to content

feat(iwsdk): add MPFB radial pulse interaction evidence #798

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gidich merged 2 commits into from
Sep 4, 2026
Merged

feat(iwsdk): add MPFB radial pulse interaction evidence #798

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
971cfdc
feat(iwsdk): add MPFB radial pulse interaction evidence
gidich Sep 4, 2026
9e69291
fix(iwsdk): stay within spike entry budget
gidich Sep 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 43 additions & 44 deletions apps/arena/ui-xr-iwsdk-spike/src/main.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import {
BoxGeometry,
BufferGeometry,
CanvasTexture,
CapsuleGeometry,
Color,
Expand All @@ -9,8 +8,6 @@ import {
DoubleSide,
Group,
HemisphereLight,
Line,
LineBasicMaterial,
Mesh,
MeshBasicMaterial,
MeshStandardMaterial,
Expand All @@ -21,14 +18,15 @@ import {
Vector3,
WebGLRenderer,
} from "three";
import { XRControllerModelFactory } from "three/addons/webxr/XRControllerModelFactory.js";
import { XRHandModelFactory } from "three/addons/webxr/XRHandModelFactory.js";
import {
buildMixedRealitySupportState,
hasApprovedMixedRealityOperatorGate,
type MixedRealitySupportState,
mixedRealityOptionalFeatures,
} from "./mixed-reality-state.js";
import { loadMpfbRadialPulsePatient } from "./mpfb-radial-pulse-patient.js";
import { createRadialPulseSceneInteraction } from "./radial-pulse-scene.js";
import { hasRadialPulseDemo, shouldHydrateOptionalIwsdkPackages, shouldInstallUikitmlSpatialTextPanel } from "./radial-pulse-state.js";
import {
buildIwsdkSidecarFrameStats,
buildIwsdkSidecarRuntimeEvidence,
Expand All @@ -42,7 +40,6 @@ import {
type IwsdkSidecarRuntimeState,
type IwsdkSidecarXrEntryEvidence,
type IwsdkSidecarXrEntryStatus,
iwsdkSidecarPrimitiveHandModelProfile,
iwsdkSidecarSceneObjectNames,
recordIwsdkSidecarXrEntryEvidence,
summarizeIwsdkSidecarReadiness,
Expand All @@ -53,6 +50,7 @@ import {
type UikitmlSpatialTextEvidence,
type UikitmlSpatialTextRuntimePanel,
} from "./uikitml-spatial-text.js";
import { addControllerAffordances, addHandModels } from "./xr-input-models.js";
import "./styles.css";

type NavigatorWithXr = Navigator & {
Expand Down Expand Up @@ -233,6 +231,8 @@ function requireElement<TElement extends Element>(selector: string): TElement {
const bootStartedAtMs = performance.now();
const iwerAutoEnterVrQueryFlag = "iwerAutoEnterVr=true";
const iwerEvidenceViewWideQueryFlag = "iwerEvidenceView=wide";
const radialPulseDemoEnabled = hasRadialPulseDemo(window.location.search);
document.body.classList.toggle("radial-pulse-demo", radialPulseDemoEnabled);

function recordBootPhase(phase: string, error?: unknown): void {
const current: OpenClinXrBootEvidence = window.__openClinXrBootEvidence ?? { app: "ui-xr-iwsdk-spike", events: [] };
Expand Down Expand Up @@ -283,6 +283,10 @@ app.innerHTML = `
<main class="spike-shell">
<section class="spike-stage" aria-label="IWSDK emergency department station scene">
<canvas id="iwsdk-sidecar-canvas" aria-label="IWSDK ED chest pain bay preview"></canvas>
<div class="pulse-guidance" aria-label="Radial pulse interaction safety cue">
<strong>Consent / assent checked</strong>
<span>Stop immediately if discomfort is reported.</span>
</div>
<div class="status-strip">
<div class="status-lane full-vr-lane" aria-label="Full VR status">
<span id="xr-status">Full VR checking</span>
Expand All @@ -294,6 +298,7 @@ app.innerHTML = `
</div>
<span id="iwsdk-status">IWSDK evidence pending</span>
<span id="trace-summary">Trace 0/${state.requiredTraceTags.length}</span>
<span id="pulse-status" class="pulse-status">Radial pulse ready</span>
</div>
</section>
<aside class="spike-panel" aria-label="IWSDK sidecar controls and evidence">
Expand Down Expand Up @@ -338,6 +343,7 @@ const inputExportCount = requireElement<HTMLElement>("#input-export-count");
const dialogueLine = requireElement<HTMLElement>("#dialogue-line");
const enterXrButton = requireElement<HTMLButtonElement>("#enter-xr-button");
const enterMrButton = requireElement<HTMLButtonElement>("#enter-mr-button");
const pulseStatus = requireElement<HTMLElement>("#pulse-status");

window.__openClinXrIwsdkSidecarEvidence = buildIwsdkSidecarRuntimeEvidence({
iwsdkCoreExportCount,
Expand All @@ -359,7 +365,7 @@ window.__openClinXrUikitmlSpatialTextEvidence = buildUikitmlSpatialTextEvidence(
status: "source_registered",
renderMode: "compile_only",
});
scheduleIwsdkEvidenceHydration();
if (shouldHydrateOptionalIwsdkPackages(radialPulseDemoEnabled)) scheduleIwsdkEvidenceHydration();

function buildIwerEvidenceViewEvidence(input: {
mode: IwerEvidenceViewMode;
Expand Down Expand Up @@ -539,6 +545,7 @@ function createStationScene(): StationSceneRuntime {
let activeXrSession: XrSession | undefined;
let lastLocomotionAtMs: number | null = null;
let lastAnimateAtMs = performance.now();
const radialPulseDemoStartedAtMs = performance.now();
let lastRenderLoopAtMs = 0;
const flatPreviewFallbackFrameMs = 1000 / 30;
let uikitmlSpatialTextPanel: UikitmlSpatialTextRuntimePanel | null = null;
Expand All @@ -557,6 +564,10 @@ function createStationScene(): StationSceneRuntime {
const camera = new PerspectiveCamera(52, 1, 0.1, 100);
camera.position.set(0, 1.7, 5.2);
camera.lookAt(0, 1.1, 0);
if (radialPulseDemoEnabled) {
camera.position.set(0.05, 1.48, 2.65);
camera.lookAt(-0.24, 1.3, 0.34);
}
locomotionRig.add(camera);

const ambient = new HemisphereLight(0xf0fff5, 0x17322c, 2.1);
Expand Down Expand Up @@ -589,6 +600,12 @@ function createStationScene(): StationSceneRuntime {
patient.scale.set(1.1, 1.1, 1.1);
scene.add(patient);

const radialPulse = createRadialPulseSceneInteraction({
deterministicDemo: radialPulseDemoEnabled,
demoStartedAtMs: radialPulseDemoStartedAtMs,
statusElement: pulseStatus,
});
scene.add(radialPulse.group);
const nurse = actorMesh(0x4f93c7);
nurse.name = "openclinxr.ed-chest-pain.nurse-maria-alvarez";
nurse.position.set(1.45, 0.95, 0.55);
Expand Down Expand Up @@ -633,7 +650,19 @@ function createStationScene(): StationSceneRuntime {
});
inputPanel.mesh.position.set(0.15, 0.78, -1.2);
scene.add(inputPanel.mesh);
void installUikitmlSpatialTextPanel();
if (radialPulseDemoEnabled) {
clinicalPanel.mesh.visible = false;
dialoguePanel.mesh.visible = false;
inputPanel.mesh.visible = false;
nurse.visible = false;
spouse.visible = false;
monitor.visible = false;
clockMesh.visible = false;
patient.visible = false;
void loadMpfbRadialPulsePatient({ scene, camera, interaction: radialPulse })
.catch((error: unknown) => recordBootPhase("radial_pulse_mpfb_patient_failed", error));
}
if (shouldInstallUikitmlSpatialTextPanel(radialPulseDemoEnabled)) void installUikitmlSpatialTextPanel();
let lastPanelSignature = "";
applyEvidenceCaptureLayout({
enabled: iwerEvidenceViewMode === "wide_iwer_capture",
Expand All @@ -643,7 +672,10 @@ function createStationScene(): StationSceneRuntime {
});
const controllerAffordancesRendered = iwerEvidenceViewMode !== "wide_iwer_capture";
if (controllerAffordancesRendered) {
addControllerAffordances(renderer, scene);
addControllerAffordances(renderer, scene, {
onContactStart: (mode) => radialPulse.beginContact(performance.now(), mode),
onContactEnd: () => radialPulse.endContact(performance.now()),
});
}
window.__openClinXrIwerEvidenceViewEvidence = buildIwerEvidenceViewEvidence({
mode: iwerEvidenceViewMode,
Expand Down Expand Up @@ -702,6 +734,7 @@ function createStationScene(): StationSceneRuntime {
rigPosition: inputEvidence.rigPosition,
});
updateVrPanels(inputEvidence);
radialPulse.update(now);
uikitmlSpatialTextPanel?.update(deltaMs);
resize();
patient.rotation.y = Math.sin(now / 1200) * 0.08;
Expand Down Expand Up @@ -864,6 +897,7 @@ function createStationScene(): StationSceneRuntime {
try {
const panel = await createUikitmlSpatialTextPanel();
uikitmlSpatialTextPanel = panel;
panel.group.visible = !radialPulseDemoEnabled;
scene.add(panel.group);
window.__openClinXrUikitmlSpatialTextEvidence = panel.evidence;
recordBootPhase("uikitml_spatial_text_loaded");
Expand Down Expand Up @@ -1088,41 +1122,6 @@ function drawWrappedText(
return currentY + lineHeight;
}

function addControllerAffordances(renderer: WebGLRenderer, scene: Scene): void {
const controllerModelFactory = new XRControllerModelFactory();
const gripNames = [
"openclinxr.ed-chest-pain.controller-grip-left",
"openclinxr.ed-chest-pain.controller-grip-right",
];
for (let index = 0; index < 2; index += 1) {
const controller = renderer.xr.getController(index);
controller.name = `openclinxr.ed-chest-pain.controller-${index + 1}`;
const ray = new Line(
new BufferGeometry().setFromPoints([new Vector3(0, 0, 0), new Vector3(0, 0, -3)]),
new LineBasicMaterial({ color: 0x8bd8bf }),
);
ray.name = `openclinxr.ed-chest-pain.controller-ray-${index + 1}`;
controller.add(ray);
scene.add(controller);
const controllerGrip = renderer.xr.getControllerGrip(index);
controllerGrip.name = gripNames[index] ?? `openclinxr.ed-chest-pain.controller-grip-${index + 1}`;
controllerGrip.add(controllerModelFactory.createControllerModel(controllerGrip));
scene.add(controllerGrip);
}
}

function addHandModels(renderer: WebGLRenderer, scene: Scene): void {
const handModelFactory = new XRHandModelFactory();
for (let index = 0; index < 2; index += 1) {
const hand = renderer.xr.getHand(index);
hand.name = `openclinxr.ed-chest-pain.hand-${index + 1}`;
const handModel = handModelFactory.createHandModel(hand, iwsdkSidecarPrimitiveHandModelProfile);
handModel.name = `openclinxr.ed-chest-pain.hand-model-${index + 1}`;
hand.add(handModel);
scene.add(hand);
}
}

type KeyboardLocomotionState = {
forward: number;
strafe: number;
Expand Down
76 changes: 76 additions & 0 deletions apps/arena/ui-xr-iwsdk-spike/src/mpfb-radial-pulse-patient.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import { Box3, type Object3D, type PerspectiveCamera, type Scene, Vector3 } from "three";
import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
import type { RadialPulseSceneInteraction } from "./radial-pulse-scene.js";

export const radialPulsePatientAssetName = "mpfb-street-adult-male.glb" as const;
export const radialPulsePatientAssetUrl = new URL(
"../../../ui-xr/public/generated-humanoids/mpfb-street-adult-male.glb",
import.meta.url,
).href;
export const radialPulseWristBoneName = "wrist.R" as const;
export const radialPulseWristRuntimeNodeName = "wristR" as const;

export async function loadMpfbRadialPulsePatient(input: {
scene: Scene;
camera: PerspectiveCamera;
interaction: RadialPulseSceneInteraction;
}): Promise<Object3D> {
try {
const gltf = await new GLTFLoader().loadAsync(radialPulsePatientAssetUrl);
const patient = gltf.scene;
patient.name = "openclinxr.radial-pulse.mpfb-street-adult-male";
normalizePatient(patient);
input.scene.add(patient);
poseRightArmForWristPresentation(patient);
patient.updateWorldMatrix(true, true);
const wrist = patient.getObjectByName(radialPulseWristRuntimeNodeName);
if (!wrist) {
throw new Error(`MPFB patient is missing ${radialPulseWristBoneName}`);
}
input.interaction.attachToWrist(wrist, radialPulsePatientAssetUrl);
framePatientAndWrist(input.camera, patient, wrist);
return patient;
} catch (error) {
input.interaction.recordPatientLoadFailure(radialPulsePatientAssetUrl);
throw error;
}
}

function poseRightArmForWristPresentation(patient: Object3D): void {
const upperArm = patient.getObjectByName("upperarm01R");
const lowerArm = patient.getObjectByName("lowerarm01R");
const wrist = patient.getObjectByName(radialPulseWristRuntimeNodeName);
if (!upperArm || !lowerArm || !wrist) {
return;
}
upperArm.rotateZ(0.5);
upperArm.rotateX(-0.12);
lowerArm.rotateZ(-1.15);
lowerArm.rotateX(0.18);
wrist.rotateY(-0.28);
}

function normalizePatient(patient: Object3D): void {
const initialBounds = new Box3().setFromObject(patient);
const initialSize = initialBounds.getSize(new Vector3());
const scale = 1.72 / Math.max(initialSize.y, 0.001);
patient.scale.setScalar(scale);
patient.updateWorldMatrix(true, true);
const bounds = new Box3().setFromObject(patient);
const center = bounds.getCenter(new Vector3());
patient.position.set(-center.x - 0.25, 0.55 - bounds.min.y, -center.z);
}

function framePatientAndWrist(camera: PerspectiveCamera, patient: Object3D, wrist: Object3D): void {
patient.updateWorldMatrix(true, true);
const bounds = new Box3().setFromObject(patient);
const torso = bounds.getCenter(new Vector3());
torso.y += 0.12;
const wristPosition = wrist.getWorldPosition(new Vector3());
const focus = torso.lerp(wristPosition, 0.46);
focus.y += 0.16;
camera.fov = 44;
camera.position.set(focus.x, focus.y + 0.03, bounds.max.z + 1.24);
camera.lookAt(focus);
camera.updateProjectionMatrix();
}
Loading
Loading