From 41e2befd8cad7cfbb8df721831606ef31ea85523 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 27 May 2026 17:21:04 -0700 Subject: [PATCH] Horizontal Autoplacement API Adds a `xb.world.autoplaceOnHorizontalSurface` to place objects on horizontal planes. Updates the modelviewer sample to leverage this API to place each object. Replaces the cylinder in the model viewer sample with an animated torus to be more interesting. - Support optional semantic `label` on simulator planes in `SimulatorPlane`, `DetectedPlane`, and `SimulatorWorld` to allow custom floor/table categorization in emulator scenes. - Implement `getObjectBoundingBox` in `HorizontalAutoplacement` to exclude `ModelViewer` interaction handles (platform, rotation mesh) from collision boxes and bounding offsets. - Reuse pre-calculated `bbox` in `getObjectBoundingBox` to cleanly support Gaussian Splatting models without disappearing offsets. - Increase sampling `gridSteps` to 15 in `HorizontalAutoplacement` for tighter, high-precision object packing on tables. - Register simulated plane meshes inside the `PlaneDetector` map to allow dynamic targeting and visualization. - Swap cylinder mesh with animated, electric-teal `TorusKnotGeometry` in `ModelViewerScene` sample. --- samples/modelviewer/ModelViewerScene.js | 70 +++- samples/modelviewer/main.js | 1 + src/constants.ts | 2 +- src/simulator/SimulatorWorld.ts | 17 +- src/utils/TemporalPolyfill.ts | 29 ++ src/world/HorizontalPlacement.ts | 408 ++++++++++++++++++++++++ src/world/World.ts | 57 +++- src/world/planes/DetectedPlane.ts | 2 +- src/world/planes/PlaneDetector.ts | 5 +- src/world/planes/SimulatorPlane.ts | 3 + tsconfig.base.json | 5 +- 11 files changed, 573 insertions(+), 26 deletions(-) create mode 100644 src/utils/TemporalPolyfill.ts create mode 100644 src/world/HorizontalPlacement.ts diff --git a/samples/modelviewer/ModelViewerScene.js b/samples/modelviewer/ModelViewerScene.js index 071ebc55..9212ca2f 100644 --- a/samples/modelviewer/ModelViewerScene.js +++ b/samples/modelviewer/ModelViewerScene.js @@ -10,15 +10,16 @@ const PROPRIETARY_ASSETS_BASE_URL = 'https://cdn.jsdelivr.net/gh/xrblocks/proprietary-assets@main/'; export class ModelViewerScene extends xb.Script { - constructor() { - super(); - } + loadedObjects = []; + placedObjects = new Set(); + sessionStarted = false; + torusMesh = null; async init() { xb.core.input.addReticles(); this.addLights(); this.createModelFromObject(); - await Promise.all([ + return Promise.all([ this.createModelFromGLTF(), this.createModelFromAnimatedGLTF(), this.createModelFromSplat(), @@ -33,19 +34,55 @@ export class ModelViewerScene extends xb.Script { this.add(light); } + update() { + if (this.torusMesh) { + this.torusMesh.rotation.x += 0.015; + this.torusMesh.rotation.y += 0.015; + } + } + + onSimulatorStarted() { + this.onXRSessionStarted(); + } + + onXRSessionStarted() { + this.sessionStarted = true; + this.placeLoadedObjects(); + } + + placeLoadedObjects() { + for (const model of this.loadedObjects) { + this.placeObject(model); + } + } + + placeObject(model) { + if (!this.sessionStarted || this.placedObjects.has(model)) return; + this.placedObjects.add(model); + return xb.world.placeOnHorizontalSurface(model, { + seconds: 30, + }); + } + createModelFromObject() { const model = new xb.ModelViewer({}); - model.add( - new THREE.Mesh( - new THREE.CylinderGeometry(0.15, 0.15, 0.4), - new THREE.MeshPhongMaterial({color: 0xdb5461}) - ) + const torusMesh = new THREE.Mesh( + new THREE.TorusKnotGeometry(0.1, 0.03, 100, 16), + new THREE.MeshPhongMaterial({ + color: 0x00f5d4, + shininess: 100, + specular: 0xffffff, + }) ); + this.torusMesh = torusMesh; + model.add(torusMesh); model.setupBoundingBox(); model.setupRaycastCylinder(); model.setupPlatform(); - model.position.set(-0.15, 0.75, -1.65); + model.position.set(-0.6, 0.5, -1.5); this.add(model); + this.loadedObjects.push(model); + this.placeObject(model); } async createModelFromGLTF() { @@ -59,7 +96,9 @@ export class ModelViewerScene extends xb.Script { }, renderer: xb.core.renderer, }); - model.position.set(0, 0.78, -1.1); + model.position.set(-0.2, 0.5, -1.5); + this.loadedObjects.push(model); + this.placeObject(model); } async createModelFromAnimatedGLTF() { @@ -73,7 +112,9 @@ export class ModelViewerScene extends xb.Script { }, renderer: xb.core.renderer, }); - model.position.set(0.9, 0.68, -0.95); + model.position.set(0.2, 0.5, -1.5); + this.loadedObjects.push(model); + this.placeObject(model); } async createModelFromSplat() { @@ -86,8 +127,9 @@ export class ModelViewerScene extends xb.Script { rotation: {x: 0, y: 180, z: 0}, }, }); - model.position.set(0.4, 0.78, -1.1); - model.rotation.set(0, -Math.PI / 6, 0); + model.position.set(0.6, 0.5, -1.5); + this.loadedObjects.push(model); + this.placeObject(model); } async createModelInPanel() { diff --git a/samples/modelviewer/main.js b/samples/modelviewer/main.js index f7cdba48..4b4ad2b1 100644 --- a/samples/modelviewer/main.js +++ b/samples/modelviewer/main.js @@ -18,5 +18,6 @@ document.addEventListener('DOMContentLoaded', async () => { }, ]; options.setAppTitle('Model Viewer'); + options.world.enablePlaneDetection(); await xb.init(options); }); diff --git a/src/constants.ts b/src/constants.ts index 9a0a7667..d9af22a7 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -103,4 +103,4 @@ export const DEFAULT_DEVICE_CAMERA_WIDTH = 1280; export const DEFAULT_DEVICE_CAMERA_HEIGHT = 720; export const XR_BLOCKS_ASSETS_PATH = - 'https://cdn.jsdelivr.net/gh/xrblocks/assets@a500427f2dfc12312df1a75860460244bab3a146/'; + 'https://cdn.jsdelivr.net/gh/xrblocks/assets@02bbbf2093d20bcefdac18c65d3ff0f2b94b7535/'; diff --git a/src/simulator/SimulatorWorld.ts b/src/simulator/SimulatorWorld.ts index 732016be..a8e8ea3d 100644 --- a/src/simulator/SimulatorWorld.ts +++ b/src/simulator/SimulatorWorld.ts @@ -1,6 +1,9 @@ import * as THREE from 'three'; import {Options} from '../core/Options'; -import {SimulatorPlane} from '../world/planes/SimulatorPlane'; +import { + SimulatorPlane, + SimulatorPlaneType, +} from '../world/planes/SimulatorPlane'; import {World} from '../world/World'; // World sensing for the simulator. @@ -12,6 +15,8 @@ export class SimulatorWorld { async init(options: Options, world: World) { this.options = options; this.world = world; + // Wait for World script initialization to complete first + await world.initializedPromise; const activeEnv = options.simulator.environments[options.simulator.activeEnvironmentIndex]; if (options.world.planes.enabled && activeEnv?.scenePlanesPath) { @@ -28,7 +33,8 @@ export class SimulatorWorld { response.json() )) as { planes: { - type: string; + type: SimulatorPlaneType; + label?: string; area: number; position: { x: number; @@ -42,7 +48,7 @@ export class SimulatorWorld { }[]; }[]; }; - const planes = planesData.planes.map((plane) => { + const planes: SimulatorPlane[] = planesData.planes.map((plane) => { return { type: plane.type, area: plane.area, @@ -57,8 +63,9 @@ export class SimulatorWorld { plane.quaternion[2], plane.quaternion[3] ), - polygon: plane.polygon, - } as unknown as SimulatorPlane; + polygon: plane.polygon.map((p) => new THREE.Vector2(p.x, p.y)), + label: plane.label, + }; }); this.world.planes!.setSimulatorPlanes(planes); } catch (error) { diff --git a/src/utils/TemporalPolyfill.ts b/src/utils/TemporalPolyfill.ts new file mode 100644 index 00000000..017f0584 --- /dev/null +++ b/src/utils/TemporalPolyfill.ts @@ -0,0 +1,29 @@ +/** + * Temporary polyfill until Chrome 148 is rolled out more widely. + * Converts a Temporal.Duration or Temporal.DurationLike to milliseconds. + */ +export function durationToMs( + duration: Temporal.Duration | Temporal.DurationLike +): number { + if (typeof Temporal !== 'undefined') { + return Temporal.Duration.from(duration).total({unit: 'millisecond'}); + } + + // If duration is a string (ISO 8601) and native Temporal failed/is absent, fallback to 0 + if (typeof duration === 'string') { + return 0; + } + + let ms = 0; + if (duration.milliseconds) ms += duration.milliseconds; + if (duration.seconds) ms += duration.seconds * 1000; + if (duration.minutes) ms += duration.minutes * 60 * 1000; + if (duration.hours) ms += duration.hours * 60 * 60 * 1000; + if (duration.days) ms += duration.days * 24 * 60 * 60 * 1000; + if (duration.weeks) ms += duration.weeks * 7 * 24 * 60 * 60 * 1000; + if (duration.months) ms += duration.months * 30 * 24 * 60 * 60 * 1000; + if (duration.years) ms += duration.years * 365 * 24 * 60 * 60 * 1000; + if (duration.microseconds) ms += duration.microseconds / 1000; + if (duration.nanoseconds) ms += duration.nanoseconds / 1000000; + return ms; +} diff --git a/src/world/HorizontalPlacement.ts b/src/world/HorizontalPlacement.ts new file mode 100644 index 00000000..1a0b9653 --- /dev/null +++ b/src/world/HorizontalPlacement.ts @@ -0,0 +1,408 @@ +import * as THREE from 'three'; +import {WaitFrame} from '../core/components/WaitFrame'; +import {PlaneDetector} from './planes/PlaneDetector'; +import {MeshDetector} from './mesh/MeshDetector'; +import {DetectedPlane} from './planes/DetectedPlane'; +import {durationToMs} from '../utils/TemporalPolyfill'; + +/** + * Places an object onto a suitable horizontal plane in the environment. + * It prioritizes planes in front of the user, prefers tables/elevated surfaces over floors, + * and ensures the object does not intersect other existing objects or other planes in the scene. + * If placement fails in the current frame, it continues retrying frame-by-frame until the timeout is reached. + * + * ### Algorithm Details: + * 1. **Filter Horizontal Surfaces**: Fetches all planes and filters for horizontal surfaces, completely skipping + * any upside-down planes (whose normal y-component points downwards). + * 2. **Obstacle Gathering**: Traverses the active Three.js scene to identify visible collidable meshes, including + * all other detected plane meshes (excluding the placement plane itself) and excluding user rigs/helpers. + * 3. **User-Centric Grid Sampling**: For each horizontal plane, projects the user's position onto the plane, restricts + * the sampling range to a 3.0-meter radius bounding box centered around this projection, and grid-samples coordinates + * strictly within the plane's polygon boundaries. + * 4. **Priority Scoring**: Assigns scores to sampled candidates, prioritizing elevated surfaces (tables over floors) + * located comfortably in front of the user (0.4m to 3.0m range, pointing in camera look direction). + * 5. **Validation & Collision Check**: Evaluates candidates in descending score order, temporarily positioning and orienting + * the object upright on the plane normal facing the user. Validates placement against the bounds of collidable obstacles. + * 6. **Frame Yielding & Retry**: If no candidates succeed in the current frame, yields to the next frame via `waitFrame` + * and repeats the process until a clean spot is found or the timeout is reached. + * + * @param objectToPlace - The Three.js Object3D to place. + * @param camera - The current active camera (to evaluate user position and look direction). + * @param scene - The active scene containing all collidable obstacles. + * @param planes - The PlaneDetector instance providing detected real-world planes. + * @param meshes - The MeshDetector instance providing environmental mesh obstacles. + * @param waitFrame - The WaitFrame component to yield execution between frames. + * @param timeout - Optional timeout duration as a Temporal.Duration or Temporal.DurationLike object (defaults to 500ms). + * @returns A promise resolving to true if successfully placed, false otherwise. + */ +export async function placeOnHorizontalSurface( + objectToPlace: THREE.Object3D, + camera: THREE.Camera, + scene: THREE.Scene, + planes: PlaneDetector | undefined, + meshes: MeshDetector | undefined, + waitFrame: WaitFrame, + timer: THREE.Timer, + timeout: Temporal.Duration | Temporal.DurationLike = {milliseconds: 500} +): Promise { + const timeoutSeconds = durationToMs(timeout) / 1000; + const startElapsed = timer.getElapsed(); + while (true) { + // Check timeout at the start of each frame loop + const elapsed = timer.getElapsed() - startElapsed; + if (elapsed >= timeoutSeconds) { + return false; + } + + if (!planes) { + await waitFrame.waitFrame(); + continue; + } + + const allPlanes = planes.get(); + + const horizontalPlanes = allPlanes.filter((plane) => { + const orientation = (plane.orientation || '').toLowerCase(); + const label = (plane.label || '').toLowerCase(); + const planeNormal = new THREE.Vector3(0, 1, 0) + .applyQuaternion(plane.quaternion) + .normalize(); + + // Skip upside-down planes (e.g., ceilings or undersides of tables) + if (planeNormal.y < 0) { + return false; + } + + return ( + orientation === 'horizontal' || + label === 'floor' || + label === 'table' || + label === 'desk' || + label === 'counter' || + label === 'horizontal' + ); + }); + + if (horizontalPlanes.length === 0) { + await waitFrame.waitFrame(); + continue; + } + + // Gather all visible collidable obstacles from the scene graph + const collidableObjects: THREE.Object3D[] = []; + scene.traverse((child) => { + if (!child.visible) return; + if (child === objectToPlace || isDescendantOf(child, objectToPlace)) + return; + if (child === planes) return; + if (meshes && isDescendantOf(child, meshes)) return; + if (child === camera || child === scene) return; + if ( + child.name && + (child.name.includes('controller') || + child.name.includes('reticle') || + child.name.includes('helper')) + ) { + return; + } + if ((child as THREE.Mesh).isMesh) { + collidableObjects.push(child); + } + }); + + // Generate and score candidate positions on all horizontal planes + const candidates: { + plane: DetectedPlane; + point: THREE.Vector3; + score: number; + }[] = []; + + const cameraPos = camera.getWorldPosition(new THREE.Vector3()); + const cameraForward = new THREE.Vector3(0, 0, -1) + .applyQuaternion(camera.quaternion) + .normalize(); + + for (const plane of horizontalPlanes) { + const polygon = getLocalPolygon(plane); + if (polygon.length === 0) continue; + + // Convert camera pos to local plane coordinates to center our grid around the user + const localCameraPos = plane.worldToLocal(cameraPos.clone()); + const localProjected = new THREE.Vector2( + localCameraPos.x, + localCameraPos.z + ); + + const polygonMinX = Math.min(...polygon.map((p) => p.x)); + const polygonMaxX = Math.max(...polygon.map((p) => p.x)); + const polygonMinY = Math.min(...polygon.map((p) => p.y)); + const polygonMaxY = Math.max(...polygon.map((p) => p.y)); + + // Restrict search bounds to 3.0 meters radius around user's local projection + const searchRadius = 3.0; + const minX = Math.max(polygonMinX, localProjected.x - searchRadius); + const maxX = Math.min(polygonMaxX, localProjected.x + searchRadius); + const minY = Math.max(polygonMinY, localProjected.y - searchRadius); + const maxY = Math.min(polygonMaxY, localProjected.y + searchRadius); + + // If restricted bounds are invalid, the plane is completely out of range + if (minX >= maxX || minY >= maxY) { + continue; + } + + // Grid sampling within restricted bounding box + const gridSteps = 15; + const localPoints: THREE.Vector2[] = []; + + // Try user's local projection point first if it is inside the polygon + if (isPointInPolygon(localProjected, polygon)) { + localPoints.push(localProjected); + } + + // Try plane center if it lies within our restricted search bounds + const center = new THREE.Vector2( + (polygonMinX + polygonMaxX) / 2, + (polygonMinY + polygonMaxY) / 2 + ); + if ( + center.x >= minX && + center.x <= maxX && + center.y >= minY && + center.y <= maxY + ) { + if (isPointInPolygon(center, polygon)) { + localPoints.push(center); + } + } + + for (let i = 0; i < gridSteps; i++) { + const x = minX + (i / (gridSteps - 1)) * (maxX - minX); + for (let j = 0; j < gridSteps; j++) { + const z = minY + (j / (gridSteps - 1)) * (maxY - minY); + const candidatePt = new THREE.Vector2(x, z); + if (isPointInPolygon(candidatePt, polygon)) { + localPoints.push(candidatePt); + } + } + } + + for (const localPt of localPoints) { + const localVec = new THREE.Vector3(localPt.x, 0, localPt.y); + const worldPt = plane.localToWorld(localVec); + + // 1. Table vs Floor preference + const label = (plane.label || '').toLowerCase(); + let semanticScore = 50; + if (label === 'table' || label === 'desk' || label === 'counter') { + semanticScore = 100; + } else if (label === 'floor') { + semanticScore = 0; + } + + // 2. Height preferences (tables are elevated horizontal planes) + const heightDiff = worldPt.y - cameraPos.y; + let heightScore = worldPt.y * 20; + if (heightDiff > 0) { + heightScore -= heightDiff * 100; // penalize points above user camera height + } + + // 3. User's look direction alignment (prioritize spots in front of user) + const toPoint = worldPt.clone().sub(cameraPos); + const distance = toPoint.length(); + + // Hard Distance Cutoff: Skip candidates too close or too far + if (distance < 0.4 || distance > 3.0) { + continue; + } + + const alignment = toPoint.normalize().dot(cameraForward); + + let alignmentScore = -1000; // heavy penalty for spots behind user + if (alignment >= 0) { + alignmentScore = alignment * 50; + } + + // 4. Comfortable interaction distance penalty + const distancePenalty = -Math.abs(distance - 1.5) * 10; + + const score = + semanticScore + heightScore + alignmentScore + distancePenalty; + + candidates.push({ + plane, + point: worldPt, + score, + }); + } + } + + // Sort all candidates in descending order of their score + candidates.sort((a, b) => b.score - a.score); + + let placed = false; + const origPosition = objectToPlace.position.clone(); + const origQuaternion = objectToPlace.quaternion.clone(); + + for (const cand of candidates) { + // Verify timeout inside the validation loop to abort quickly if running slow + if (timer.getElapsed() - startElapsed >= timeoutSeconds) { + break; + } + + // Temporarily place at origin to calculate bounding box offsets with rotation applied + objectToPlace.position.set(0, 0, 0); + + // Orient the object upright on the plane normal and face the camera + const planeNormal = new THREE.Vector3(0, 1, 0) + .applyQuaternion(cand.plane.quaternion) + .normalize(); + const forwardVector = cameraPos.clone().sub(cand.point); + forwardVector.projectOnPlane(planeNormal).normalize(); + const rightVector = new THREE.Vector3() + .crossVectors(planeNormal, forwardVector) + .normalize(); + + const rotationMatrix = new THREE.Matrix4().makeBasis( + rightVector, + planeNormal, + forwardVector + ); + objectToPlace.quaternion.setFromRotationMatrix(rotationMatrix); + objectToPlace.updateMatrixWorld(true); + + // Calculate bounding box at the origin to find bottom offset along world Y axis + const tempBox = getObjectBoundingBox(objectToPlace); + const bottomOffset = -tempBox.min.y; + + // Set final position, offsetting vertically so bottom of bbox aligns with horizontal plane + objectToPlace.position.copy(cand.point); + objectToPlace.position.y += bottomOffset; + objectToPlace.updateMatrixWorld(true); + + // Calculate bounding box and verify intersections with scene obstacles + const objectBox = getObjectBoundingBox(objectToPlace); + + // Shrink and shift collision box slightly to avoid grounding collisions with the table mesh + const collisionBox = objectBox.clone(); + + let collision = false; + const obstacleBox = new THREE.Box3(); + for (const obstacle of collidableObjects) { + if (obstacle === cand.plane) { + continue; + } + obstacle.updateMatrixWorld(true); + obstacleBox.setFromObject(obstacle); + if (collisionBox.intersectsBox(obstacleBox)) { + collision = true; + break; + } + } + + if (!collision) { + placed = true; + break; // Successful placement! + } + } + + if (placed) { + return true; + } + + // Restore initial state if placement failed in the current frame + objectToPlace.position.copy(origPosition); + objectToPlace.quaternion.copy(origQuaternion); + + // Yield execution until the next frame starts + await waitFrame.waitFrame(); + } +} + +// --- Helper Functions --- + +function getLocalPolygon(plane: DetectedPlane): THREE.Vector2[] { + if (plane.simulatorPlane) { + return plane.simulatorPlane.polygon; + } else if (plane.xrPlane) { + return plane.xrPlane.polygon.map((p) => new THREE.Vector2(p.x, p.z)); + } + return []; +} + +function isPointInPolygon( + point: THREE.Vector2, + polygon: THREE.Vector2[] +): boolean { + let inside = false; + for (let i = 0, j = polygon.length - 1; i < polygon.length; j = i++) { + const xi = polygon[i].x, + yi = polygon[i].y; + const xj = polygon[j].x, + yj = polygon[j].y; + + const intersect = + yi > point.y !== yj > point.y && + point.x < ((xj - xi) * (point.y - yi)) / (yj - yi) + xi; + if (intersect) inside = !inside; + } + return inside; +} + +function isDescendantOf( + child: THREE.Object3D, + parent: THREE.Object3D +): boolean { + let current = child.parent; + while (current) { + if (current === parent) return true; + current = current.parent; + } + return false; +} + +function getObjectBoundingBox(object: THREE.Object3D): THREE.Box3 { + // If the object has a pre-calculated bounding box (e.g. ModelViewer), use it directly + if ('bbox' in object) { + const customBbox = (object as {bbox?: THREE.Box3}).bbox; + if (customBbox && !customBbox.isEmpty()) { + object.updateMatrixWorld(true); + return customBbox.clone().applyMatrix4(object.matrixWorld); + } + } + + const box = new THREE.Box3(); + + function traverse(node: THREE.Object3D) { + if (!node.visible) return; + + // Ignore the model viewer's platform, rotation cylinder, and control bar meshes + const name = node.constructor.name; + if ( + name === 'ModelViewerPlatform' || + name === 'RotationRaycastMesh' || + node.name === 'Platform' + ) { + return; + } + + const mesh = node as THREE.Mesh; + if (mesh.isMesh) { + if (mesh.geometry) { + if (!mesh.geometry.boundingBox) { + mesh.geometry.computeBoundingBox(); + } + const tempBox = mesh.geometry.boundingBox!.clone(); + tempBox.applyMatrix4(mesh.matrixWorld); + box.union(tempBox); + } + } + + for (const child of node.children) { + traverse(child); + } + } + + object.updateMatrixWorld(true); + traverse(object); + return box; +} diff --git a/src/world/World.ts b/src/world/World.ts index d12f85ab..1f8b4474 100644 --- a/src/world/World.ts +++ b/src/world/World.ts @@ -1,6 +1,7 @@ import * as THREE from 'three'; import {Script} from '../core/Script'; +import {WaitFrame} from '../core/components/WaitFrame'; import {placeObjectAtIntersectionFacingTarget} from '../utils/ObjectPlacement'; import {ObjectDetector} from './objects/ObjectDetector'; @@ -8,8 +9,9 @@ import {PlaneDetector} from './planes/PlaneDetector'; import {WorldOptions} from './WorldOptions'; import {MeshDetector} from './mesh/MeshDetector'; import {SoundDetector} from './sounds/SoundDetector'; +import {placeOnHorizontalSurface} from './HorizontalPlacement'; + // Import other modules as they are implemented in future. -// import { SceneMesh } from '/depth/SceneMesh.js'; // import { LightEstimation } from '/lighting/LightEstimation.js'; // import { HumanRecognizer } from '/human/HumanRecognizer.js'; @@ -23,6 +25,8 @@ export class World extends Script { static dependencies = { options: WorldOptions, camera: THREE.Camera, + waitFrame: WaitFrame, + timer: THREE.Timer, }; editorIcon = 'sensors'; @@ -69,10 +73,17 @@ export class World extends Script { private raycaster = new THREE.Raycaster(); private camera!: THREE.Camera; + private waitFrame!: WaitFrame; + private timer!: THREE.Timer; // Whether we need to initiate a room capture. private needsRoomCapture = false; + private resolveInitialized!: () => void; + readonly initializedPromise = new Promise((resolve) => { + this.resolveInitialized = resolve; + }); + /** * Initializes the world-sensing modules based on the provided configuration. * This method is called automatically by the XRCore. @@ -80,14 +91,21 @@ export class World extends Script { override async init({ options, camera, + waitFrame, + timer, }: { options: WorldOptions; camera: THREE.Camera; + waitFrame: WaitFrame; + timer: THREE.Timer; }) { this.options = options; this.camera = camera; + this.waitFrame = waitFrame; + this.timer = timer; if (!this.options || !this.options.enabled) { + this.resolveInitialized(); return; } @@ -125,6 +143,7 @@ export class World extends Script { this.humans = new HumanRecognizer(); } */ + this.resolveInitialized(); } /** @@ -198,6 +217,42 @@ export class World extends Script { return false; } + /** + * Places an object onto a suitable horizontal plane in the environment. + * It prioritizes planes in front of the user, prefers tables/elevated surfaces over floors, + * and ensures the object does not intersect other existing objects or other planes in the scene. + * If placement fails in the current frame, it continues retrying frame-by-frame until the timeout is reached. + * + * @param objectToPlace - The Three.js Object3D to place. + * @param timeout - Optional timeout duration as a Temporal.Duration or Temporal.DurationLike object (defaults to 500ms). + * @returns A promise resolving to true if successfully placed, false otherwise. + */ + async placeOnHorizontalSurface( + objectToPlace: THREE.Object3D, + timeout: Temporal.Duration | Temporal.DurationLike = {milliseconds: 500} + ): Promise { + // Wait for World script initialization to complete first + await this.initializedPromise; + + // Walk up parent hierarchy to find the root THREE.Scene + let sceneObj: THREE.Object3D | null = this.parent; + while (sceneObj && !(sceneObj instanceof THREE.Scene)) { + sceneObj = sceneObj.parent; + } + const rootScene = (sceneObj as THREE.Scene) || this; + + return placeOnHorizontalSurface( + objectToPlace, + this.camera, + rootScene, + this.planes, + this.meshes, + this.waitFrame, + this.timer, + timeout + ); + } + /** * Toggles the visibility of all debug visualizations for world features. * @param visible - Whether the visualizations should be visible. diff --git a/src/world/planes/DetectedPlane.ts b/src/world/planes/DetectedPlane.ts index 98daad0d..3099e8be 100644 --- a/src/world/planes/DetectedPlane.ts +++ b/src/world/planes/DetectedPlane.ts @@ -56,7 +56,7 @@ export class DetectedPlane extends THREE.Mesh { this.label = xrPlane.semanticLabel; this.orientation = xrPlane.orientation; } else if (simulatorPlane) { - this.label = simulatorPlane.type; + this.label = simulatorPlane.label || simulatorPlane.type; this.orientation = simulatorPlane.type; this.position.copy(simulatorPlane.position); this.quaternion.copy(simulatorPlane.quaternion); diff --git a/src/world/planes/PlaneDetector.ts b/src/world/planes/PlaneDetector.ts index 4fb407ac..e9337b7c 100644 --- a/src/world/planes/PlaneDetector.ts +++ b/src/world/planes/PlaneDetector.ts @@ -16,7 +16,7 @@ export class PlaneDetector extends Script { /** * A map from the WebXR `XRPlane` object to our custom `DetectedPlane` mesh. */ - private _detectedPlanes = new Map(); + private _detectedPlanes = new Map(); /** * The material used for visualizing planes when debugging. @@ -134,7 +134,7 @@ export class PlaneDetector extends Script { * Removes a `Plane` mesh from the scene and disposes of its resources. * @param xrPlane - The WebXR plane object to remove. */ - private _removePlaneMesh(xrPlane: XRPlane) { + private _removePlaneMesh(xrPlane: XRPlane | SimulatorPlane) { const planeMesh = this._detectedPlanes.get(xrPlane); if (planeMesh) { planeMesh.geometry.dispose(); @@ -195,6 +195,7 @@ export class PlaneDetector extends Script { const material = this._debugMaterial || new THREE.MeshBasicMaterial({visible: false}); const planeMesh = new DetectedPlane(null, material, plane); + this._detectedPlanes.set(plane, planeMesh); this.add(planeMesh); return planeMesh; } diff --git a/src/world/planes/SimulatorPlane.ts b/src/world/planes/SimulatorPlane.ts index b114daed..c79c9521 100644 --- a/src/world/planes/SimulatorPlane.ts +++ b/src/world/planes/SimulatorPlane.ts @@ -6,6 +6,9 @@ export interface SimulatorPlane { /** 'horizontal' or 'vertical' */ type: SimulatorPlaneType; + /** Optional semantic label for the plane (e.g., 'table', 'floor') */ + label?: string; + /** Total surface area in square meters */ area: number; diff --git a/tsconfig.base.json b/tsconfig.base.json index 4bf28b7d..327c1f63 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "es2022", + "target": "es2023", "module": "es2022", "moduleResolution": "bundler", "declaration": true, @@ -11,6 +11,7 @@ "forceConsistentCasingInFileNames": true, "experimentalDecorators": true, "useDefineForClassFields": false, - "types": ["dom-speech-recognition"] + "types": ["dom-speech-recognition"], + "lib": ["dom", "es2023", "esnext.temporal"] } }