diff --git a/docs/runtime/polkavm-app-abi-v1.md b/docs/runtime/polkavm-app-abi-v1.md index da0c4eb..dc63642 100644 --- a/docs/runtime/polkavm-app-abi-v1.md +++ b/docs/runtime/polkavm-app-abi-v1.md @@ -238,7 +238,7 @@ and returns the number of bytes written. It never writes a partial record. Zero means that no event was available or that the capacity was smaller than one record. -An input record is: +The legacy fixed record layout is: ```text offset type field @@ -252,18 +252,54 @@ offset type field ABI v1 event types are: ```text -1 key down -2 key up -3 pointer button down -4 pointer button up -5 pointer position -6 pointer delta -7 surface metrics +1 key down +2 key up +3 pointer button down +4 pointer button up +5 pointer position +6 pointer delta +7 surface metrics +8 committed UTF-8 text chunk +9 IME preedit UTF-8 chunk +10 IME commit UTF-8 chunk +11 IME enabled +12 IME disabled or cancelled +13 focus (`code` is 0 or 1) +14 wheel delta (`x` and `y` are signed i16) ``` -The device-input contract defines code values, coordinate interpretation, and -surface-metric scaling. ABI v1 does not define touch, wheel, UTF-8 text, IME, -or focus events. +Text and IME records use `code` bits 0–2 as a payload length from zero through +six, bit 6 for the first chunk, and bit 7 for the last chunk. Bytes 2–7 contain +the chunk and zero padding. A complete text event is at most 4 KiB. The Host +MUST queue all chunks of one event atomically; the guest MUST reject malformed +flag sequences or invalid UTF-8. + +### UI semantics + +```text +host_ui_semantics_submit(pointer: u32, length: u32) -> u32 +``` + +The guest may submit one complete UTF-8 JSON semantic tree per `init` or +`update` call. The tree is presentation output, not an instruction to invoke +guest functions. Hosts use its roles, labels, values, actions, focus, and +surface-relative bounds for accessibility and UI automation, then deliver +actual pointer, keyboard, text, or IME records for every interaction. + +The version 1 object contains `version`, monotonic `generation`, and `nodes`. +Each node contains a nonzero numeric `id`, nullable `parent`, `role`, `name`, +`value`, `[x0,y0,x1,y1]` bounds, `actions`, `disabled`, and `focused`. Version 1 +allows at most 1,024 nodes, 1 KiB per name or value, and 256 KiB for the whole +tree. It requires exactly one root, unique IDs, existing parents, finite ordered +bounds, and no unknown object fields. + +Return values: + +```text +0 accepted +1 malformed, out-of-bounds, or over-limit tree +2 a tree was already submitted during this call +``` ### Motion diff --git a/js/packages/pvm-browser-runtime/src/pvm-runtime-core.js b/js/packages/pvm-browser-runtime/src/pvm-runtime-core.js index 6e62c5e..c65d0d4 100644 --- a/js/packages/pvm-browser-runtime/src/pvm-runtime-core.js +++ b/js/packages/pvm-browser-runtime/src/pvm-runtime-core.js @@ -132,6 +132,19 @@ globalThis.createPvmRuntime = (endpoint) => { postMessage({ type: "tri2d", bytes }, [bytes.buffer]); } + function drainUiSemantics() { + if (!pvm.pvm_browser_take_ui_semantics?.()) { + return; + } + const length = pvm.pvm_browser_ui_semantics_length(); + const bytes = new Uint8Array( + pvm.memory.buffer, + pvm.pvm_browser_ui_semantics_pointer(), + length, + ).slice(); + postMessage({ type: "ui-semantics", bytes }, [bytes.buffer]); + } + function drainGpuBatches() { while (pvm.pvm_browser_take_gpu_batch?.()) { const length = pvm.pvm_browser_gpu_batch_length(); @@ -214,6 +227,7 @@ globalThis.createPvmRuntime = (endpoint) => { ); drainFrame(); drainTri2d(); + drainUiSemantics(); drainGpuBatches(); drainTruapiRequests(); drainAudio(); @@ -547,15 +561,23 @@ globalThis.createPvmRuntime = (endpoint) => { translated.sendInput(bytes); return; } - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if (bytes[0] <= 7) { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + check( + pvm.pvm_browser_send_input( + bytes[0], + bytes[1], + view.getUint16(2, true), + view.getUint16(4, true), + ), + "send PolkaVM browser input", + ); + return; + } + stage(bytes); check( - pvm.pvm_browser_send_input( - bytes[0], - bytes[1], - view.getUint16(2, true), - view.getUint16(4, true), - ), - "send PolkaVM browser input", + pvm.pvm_browser_send_input_record(), + "send PolkaVM browser extended input", ); } diff --git a/js/packages/pvm-browser-runtime/src/pvm-wasm-translated.js b/js/packages/pvm-browser-runtime/src/pvm-wasm-translated.js index 006d692..2b71cdb 100644 --- a/js/packages/pvm-browser-runtime/src/pvm-wasm-translated.js +++ b/js/packages/pvm-browser-runtime/src/pvm-wasm-translated.js @@ -27,6 +27,9 @@ const MAX_AUDIO_SAMPLES = 48000 * 2; const MAX_FRAME_BYTES = 16 * 1024 * 1024; const MAX_TRI2D_BYTES = 8 * 1024 * 1024; + const MAX_UI_SEMANTICS_BYTES = 256 * 1024; + const MAX_UI_SEMANTIC_NODES = 1024; + const MAX_UI_SEMANTIC_STRING_BYTES = 1024; const MAX_GPU_BATCH_BYTES = 4 * 1024 * 1024; const MAX_GPU_EVENT_BYTES = 64 * 1024; const MAX_GPU_EVENTS = 256; @@ -56,6 +59,55 @@ const decoder = new TextDecoder(); const encoder = new TextEncoder(); + function validUiSemantics(bytes) { + let snapshot; + try { + snapshot = JSON.parse(decoder.decode(bytes)); + } catch { + return false; + } + if ( + snapshot?.version !== 1 || + !Number.isSafeInteger(snapshot.generation) || + snapshot.generation < 0 || + !Array.isArray(snapshot.nodes) || + !snapshot.nodes.length || + snapshot.nodes.length > MAX_UI_SEMANTIC_NODES + ) { + return false; + } + const ids = new Set(); + let roots = 0; + for (const node of snapshot.nodes) { + if ( + typeof node?.id !== "string" || + !/^[0-9a-f]{1,16}$/.test(node.id) || + ids.has(node.id) || + !Array.isArray(node.bounds) || + node.bounds.length !== 4 || + !node.bounds.every(Number.isFinite) || + node.bounds[2] < node.bounds[0] || + node.bounds[3] < node.bounds[1] || + typeof node.name !== "string" || + encoder.encode(node.name).byteLength > MAX_UI_SEMANTIC_STRING_BYTES || + typeof node.value !== "string" || + encoder.encode(node.value).byteLength > MAX_UI_SEMANTIC_STRING_BYTES + ) { + return false; + } + ids.add(node.id); + if (node.parent === null) { + roots++; + } + } + return ( + roots === 1 && + snapshot.nodes.every( + (node) => node.parent === null || ids.has(node.parent), + ) + ); + } + function readMetadata(module) { const sections = WebAssembly.Module.customSections( module, @@ -300,6 +352,7 @@ this.truapiResponses = []; this.truapiResponseBytes = 0; this.tri2dSubmitted = false; + this.uiSemanticsSubmitted = false; this.maxGas = BigInt(maxGas); this.input = []; this.coreInput = []; @@ -354,6 +407,7 @@ this.timeMs = timeMs; this.gpuSubmits = 0; this.truapiRequests = 0; + this.uiSemanticsSubmitted = false; this.truapiRequestBytes = 0; this.#resetBudget( this.coreVm && !this.coreVmStarted @@ -748,6 +802,26 @@ this.#setReg(7, 0n); return false; } + case "host_ui_semantics_submit": { + const length = this.#u32(a1); + if (!length || length > MAX_UI_SEMANTICS_BYTES) { + this.#setReg(7, 1n); + return false; + } + if (this.uiSemanticsSubmitted) { + this.#setReg(7, 2n); + return false; + } + const bytes = this.#read(this.#u32(a0), length); + if (!validUiSemantics(bytes)) { + this.#setReg(7, 1n); + return false; + } + this.emit({ type: "ui-semantics", bytes }, [bytes.buffer]); + this.uiSemanticsSubmitted = true; + this.#setReg(7, 0n); + return false; + } case "host_gpu_capabilities": { if (this.graphicsProfile !== "webgpu-raster") { this.#setReg(7, BigInt(GPU_ERROR_INVALID_STATE)); diff --git a/rust/crates/pvm-runtime-assets/assets/SHA256SUMS b/rust/crates/pvm-runtime-assets/assets/SHA256SUMS index cefe514..5145b41 100644 --- a/rust/crates/pvm-runtime-assets/assets/SHA256SUMS +++ b/rust/crates/pvm-runtime-assets/assets/SHA256SUMS @@ -1,6 +1,6 @@ -3b448399a39ee571765f41266032c621ce3668176d7f0d6372c85d6068e7d4a5 pvm-browser-runtime.wasm -55ce96805e22f60ec9a4da072a316c34c1f650d99bdf1421805c1ba35c2d8161 pvm-worker.js +e5272df5ed0ef9c682ef39198161b44c4b46f1fa17c2df6f3d7c216c5934c32b pvm-browser-runtime.wasm +48df143a60691765029e39555a9b4a3e51176000806fb3abb8a04f22e97c89ba pvm-worker.js 33418e2c81c117539569eb4cf91af4d058cdfae7dd4556b13f3687f6d6b3bae4 pvm-gpu-worker.js -a4c2a9ed4274c79b0c11ddbac163f1de200d41005ca1ce54909a5b8116d82c46 pvm-wasm-translated.js -79dce3ab925e78e9ef8e5df6e34fb2823e6f362cbc97d265a9ffdcc22cc24ed4 pvm-runtime-core.js +3bdb850c57ec1fc32f0d08bacca74313a2d77a6d4d4fedffcad87ddfd3028a7b pvm-wasm-translated.js +7e746495b2e3048fcba922f714f439e84942175c67c8e73de36f5b892af1f965 pvm-runtime-core.js 9c929f5d5c64a1b75e7e48485d7c3944ed6838112177ea778827a2c407c2d820 pvm-wasm-worker-entry.js diff --git a/rust/crates/pvm-runtime-assets/assets/pvm-browser-runtime.wasm b/rust/crates/pvm-runtime-assets/assets/pvm-browser-runtime.wasm index 95639a2..59ec337 100755 Binary files a/rust/crates/pvm-runtime-assets/assets/pvm-browser-runtime.wasm and b/rust/crates/pvm-runtime-assets/assets/pvm-browser-runtime.wasm differ diff --git a/rust/crates/pvm-runtime-assets/assets/pvm-runtime-core.js b/rust/crates/pvm-runtime-assets/assets/pvm-runtime-core.js index 6e62c5e..c65d0d4 100644 --- a/rust/crates/pvm-runtime-assets/assets/pvm-runtime-core.js +++ b/rust/crates/pvm-runtime-assets/assets/pvm-runtime-core.js @@ -132,6 +132,19 @@ globalThis.createPvmRuntime = (endpoint) => { postMessage({ type: "tri2d", bytes }, [bytes.buffer]); } + function drainUiSemantics() { + if (!pvm.pvm_browser_take_ui_semantics?.()) { + return; + } + const length = pvm.pvm_browser_ui_semantics_length(); + const bytes = new Uint8Array( + pvm.memory.buffer, + pvm.pvm_browser_ui_semantics_pointer(), + length, + ).slice(); + postMessage({ type: "ui-semantics", bytes }, [bytes.buffer]); + } + function drainGpuBatches() { while (pvm.pvm_browser_take_gpu_batch?.()) { const length = pvm.pvm_browser_gpu_batch_length(); @@ -214,6 +227,7 @@ globalThis.createPvmRuntime = (endpoint) => { ); drainFrame(); drainTri2d(); + drainUiSemantics(); drainGpuBatches(); drainTruapiRequests(); drainAudio(); @@ -547,15 +561,23 @@ globalThis.createPvmRuntime = (endpoint) => { translated.sendInput(bytes); return; } - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if (bytes[0] <= 7) { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + check( + pvm.pvm_browser_send_input( + bytes[0], + bytes[1], + view.getUint16(2, true), + view.getUint16(4, true), + ), + "send PolkaVM browser input", + ); + return; + } + stage(bytes); check( - pvm.pvm_browser_send_input( - bytes[0], - bytes[1], - view.getUint16(2, true), - view.getUint16(4, true), - ), - "send PolkaVM browser input", + pvm.pvm_browser_send_input_record(), + "send PolkaVM browser extended input", ); } diff --git a/rust/crates/pvm-runtime-assets/assets/pvm-wasm-translated.js b/rust/crates/pvm-runtime-assets/assets/pvm-wasm-translated.js index 006d692..2b71cdb 100644 --- a/rust/crates/pvm-runtime-assets/assets/pvm-wasm-translated.js +++ b/rust/crates/pvm-runtime-assets/assets/pvm-wasm-translated.js @@ -27,6 +27,9 @@ const MAX_AUDIO_SAMPLES = 48000 * 2; const MAX_FRAME_BYTES = 16 * 1024 * 1024; const MAX_TRI2D_BYTES = 8 * 1024 * 1024; + const MAX_UI_SEMANTICS_BYTES = 256 * 1024; + const MAX_UI_SEMANTIC_NODES = 1024; + const MAX_UI_SEMANTIC_STRING_BYTES = 1024; const MAX_GPU_BATCH_BYTES = 4 * 1024 * 1024; const MAX_GPU_EVENT_BYTES = 64 * 1024; const MAX_GPU_EVENTS = 256; @@ -56,6 +59,55 @@ const decoder = new TextDecoder(); const encoder = new TextEncoder(); + function validUiSemantics(bytes) { + let snapshot; + try { + snapshot = JSON.parse(decoder.decode(bytes)); + } catch { + return false; + } + if ( + snapshot?.version !== 1 || + !Number.isSafeInteger(snapshot.generation) || + snapshot.generation < 0 || + !Array.isArray(snapshot.nodes) || + !snapshot.nodes.length || + snapshot.nodes.length > MAX_UI_SEMANTIC_NODES + ) { + return false; + } + const ids = new Set(); + let roots = 0; + for (const node of snapshot.nodes) { + if ( + typeof node?.id !== "string" || + !/^[0-9a-f]{1,16}$/.test(node.id) || + ids.has(node.id) || + !Array.isArray(node.bounds) || + node.bounds.length !== 4 || + !node.bounds.every(Number.isFinite) || + node.bounds[2] < node.bounds[0] || + node.bounds[3] < node.bounds[1] || + typeof node.name !== "string" || + encoder.encode(node.name).byteLength > MAX_UI_SEMANTIC_STRING_BYTES || + typeof node.value !== "string" || + encoder.encode(node.value).byteLength > MAX_UI_SEMANTIC_STRING_BYTES + ) { + return false; + } + ids.add(node.id); + if (node.parent === null) { + roots++; + } + } + return ( + roots === 1 && + snapshot.nodes.every( + (node) => node.parent === null || ids.has(node.parent), + ) + ); + } + function readMetadata(module) { const sections = WebAssembly.Module.customSections( module, @@ -300,6 +352,7 @@ this.truapiResponses = []; this.truapiResponseBytes = 0; this.tri2dSubmitted = false; + this.uiSemanticsSubmitted = false; this.maxGas = BigInt(maxGas); this.input = []; this.coreInput = []; @@ -354,6 +407,7 @@ this.timeMs = timeMs; this.gpuSubmits = 0; this.truapiRequests = 0; + this.uiSemanticsSubmitted = false; this.truapiRequestBytes = 0; this.#resetBudget( this.coreVm && !this.coreVmStarted @@ -748,6 +802,26 @@ this.#setReg(7, 0n); return false; } + case "host_ui_semantics_submit": { + const length = this.#u32(a1); + if (!length || length > MAX_UI_SEMANTICS_BYTES) { + this.#setReg(7, 1n); + return false; + } + if (this.uiSemanticsSubmitted) { + this.#setReg(7, 2n); + return false; + } + const bytes = this.#read(this.#u32(a0), length); + if (!validUiSemantics(bytes)) { + this.#setReg(7, 1n); + return false; + } + this.emit({ type: "ui-semantics", bytes }, [bytes.buffer]); + this.uiSemanticsSubmitted = true; + this.#setReg(7, 0n); + return false; + } case "host_gpu_capabilities": { if (this.graphicsProfile !== "webgpu-raster") { this.#setReg(7, BigInt(GPU_ERROR_INVALID_STATE)); diff --git a/rust/crates/pvm-runtime-assets/assets/pvm-worker.js b/rust/crates/pvm-runtime-assets/assets/pvm-worker.js index 3bebb86..de6efb7 100644 --- a/rust/crates/pvm-runtime-assets/assets/pvm-worker.js +++ b/rust/crates/pvm-runtime-assets/assets/pvm-worker.js @@ -27,6 +27,9 @@ const MAX_AUDIO_SAMPLES = 48000 * 2; const MAX_FRAME_BYTES = 16 * 1024 * 1024; const MAX_TRI2D_BYTES = 8 * 1024 * 1024; + const MAX_UI_SEMANTICS_BYTES = 256 * 1024; + const MAX_UI_SEMANTIC_NODES = 1024; + const MAX_UI_SEMANTIC_STRING_BYTES = 1024; const MAX_GPU_BATCH_BYTES = 4 * 1024 * 1024; const MAX_GPU_EVENT_BYTES = 64 * 1024; const MAX_GPU_EVENTS = 256; @@ -56,6 +59,55 @@ const decoder = new TextDecoder(); const encoder = new TextEncoder(); + function validUiSemantics(bytes) { + let snapshot; + try { + snapshot = JSON.parse(decoder.decode(bytes)); + } catch { + return false; + } + if ( + snapshot?.version !== 1 || + !Number.isSafeInteger(snapshot.generation) || + snapshot.generation < 0 || + !Array.isArray(snapshot.nodes) || + !snapshot.nodes.length || + snapshot.nodes.length > MAX_UI_SEMANTIC_NODES + ) { + return false; + } + const ids = new Set(); + let roots = 0; + for (const node of snapshot.nodes) { + if ( + typeof node?.id !== "string" || + !/^[0-9a-f]{1,16}$/.test(node.id) || + ids.has(node.id) || + !Array.isArray(node.bounds) || + node.bounds.length !== 4 || + !node.bounds.every(Number.isFinite) || + node.bounds[2] < node.bounds[0] || + node.bounds[3] < node.bounds[1] || + typeof node.name !== "string" || + encoder.encode(node.name).byteLength > MAX_UI_SEMANTIC_STRING_BYTES || + typeof node.value !== "string" || + encoder.encode(node.value).byteLength > MAX_UI_SEMANTIC_STRING_BYTES + ) { + return false; + } + ids.add(node.id); + if (node.parent === null) { + roots++; + } + } + return ( + roots === 1 && + snapshot.nodes.every( + (node) => node.parent === null || ids.has(node.parent), + ) + ); + } + function readMetadata(module) { const sections = WebAssembly.Module.customSections( module, @@ -300,6 +352,7 @@ this.truapiResponses = []; this.truapiResponseBytes = 0; this.tri2dSubmitted = false; + this.uiSemanticsSubmitted = false; this.maxGas = BigInt(maxGas); this.input = []; this.coreInput = []; @@ -354,6 +407,7 @@ this.timeMs = timeMs; this.gpuSubmits = 0; this.truapiRequests = 0; + this.uiSemanticsSubmitted = false; this.truapiRequestBytes = 0; this.#resetBudget( this.coreVm && !this.coreVmStarted @@ -748,6 +802,26 @@ this.#setReg(7, 0n); return false; } + case "host_ui_semantics_submit": { + const length = this.#u32(a1); + if (!length || length > MAX_UI_SEMANTICS_BYTES) { + this.#setReg(7, 1n); + return false; + } + if (this.uiSemanticsSubmitted) { + this.#setReg(7, 2n); + return false; + } + const bytes = this.#read(this.#u32(a0), length); + if (!validUiSemantics(bytes)) { + this.#setReg(7, 1n); + return false; + } + this.emit({ type: "ui-semantics", bytes }, [bytes.buffer]); + this.uiSemanticsSubmitted = true; + this.#setReg(7, 0n); + return false; + } case "host_gpu_capabilities": { if (this.graphicsProfile !== "webgpu-raster") { this.#setReg(7, BigInt(GPU_ERROR_INVALID_STATE)); @@ -1569,6 +1643,19 @@ globalThis.createPvmRuntime = (endpoint) => { postMessage({ type: "tri2d", bytes }, [bytes.buffer]); } + function drainUiSemantics() { + if (!pvm.pvm_browser_take_ui_semantics?.()) { + return; + } + const length = pvm.pvm_browser_ui_semantics_length(); + const bytes = new Uint8Array( + pvm.memory.buffer, + pvm.pvm_browser_ui_semantics_pointer(), + length, + ).slice(); + postMessage({ type: "ui-semantics", bytes }, [bytes.buffer]); + } + function drainGpuBatches() { while (pvm.pvm_browser_take_gpu_batch?.()) { const length = pvm.pvm_browser_gpu_batch_length(); @@ -1651,6 +1738,7 @@ globalThis.createPvmRuntime = (endpoint) => { ); drainFrame(); drainTri2d(); + drainUiSemantics(); drainGpuBatches(); drainTruapiRequests(); drainAudio(); @@ -1984,15 +2072,23 @@ globalThis.createPvmRuntime = (endpoint) => { translated.sendInput(bytes); return; } - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if (bytes[0] <= 7) { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + check( + pvm.pvm_browser_send_input( + bytes[0], + bytes[1], + view.getUint16(2, true), + view.getUint16(4, true), + ), + "send PolkaVM browser input", + ); + return; + } + stage(bytes); check( - pvm.pvm_browser_send_input( - bytes[0], - bytes[1], - view.getUint16(2, true), - view.getUint16(4, true), - ), - "send PolkaVM browser input", + pvm.pvm_browser_send_input_record(), + "send PolkaVM browser extended input", ); } diff --git a/rust/crates/pvm-runtime-assets/src/lib.rs b/rust/crates/pvm-runtime-assets/src/lib.rs index de68743..9cbbca0 100644 --- a/rust/crates/pvm-runtime-assets/src/lib.rs +++ b/rust/crates/pvm-runtime-assets/src/lib.rs @@ -26,13 +26,13 @@ const ASSETS: [BrowserAsset; 7] = [ path: "pvm-browser-runtime.wasm", content_type: "application/wasm", bytes: include_bytes!("../assets/pvm-browser-runtime.wasm"), - sha256: "3b448399a39ee571765f41266032c621ce3668176d7f0d6372c85d6068e7d4a5", + sha256: "e5272df5ed0ef9c682ef39198161b44c4b46f1fa17c2df6f3d7c216c5934c32b", }, BrowserAsset { path: "pvm-worker.js", content_type: "text/javascript", bytes: include_bytes!("../assets/pvm-worker.js"), - sha256: "55ce96805e22f60ec9a4da072a316c34c1f650d99bdf1421805c1ba35c2d8161", + sha256: "48df143a60691765029e39555a9b4a3e51176000806fb3abb8a04f22e97c89ba", }, BrowserAsset { path: "pvm-gpu-worker.js", @@ -44,13 +44,13 @@ const ASSETS: [BrowserAsset; 7] = [ path: "pvm-wasm-translated.js", content_type: "text/javascript", bytes: include_bytes!("../assets/pvm-wasm-translated.js"), - sha256: "a4c2a9ed4274c79b0c11ddbac163f1de200d41005ca1ce54909a5b8116d82c46", + sha256: "3bdb850c57ec1fc32f0d08bacca74313a2d77a6d4d4fedffcad87ddfd3028a7b", }, BrowserAsset { path: "pvm-runtime-core.js", content_type: "text/javascript", bytes: include_bytes!("../assets/pvm-runtime-core.js"), - sha256: "79dce3ab925e78e9ef8e5df6e34fb2823e6f362cbc97d265a9ffdcc22cc24ed4", + sha256: "7e746495b2e3048fcba922f714f439e84942175c67c8e73de36f5b892af1f965", }, BrowserAsset { path: "pvm-wasm-worker-entry.js", @@ -62,7 +62,7 @@ const ASSETS: [BrowserAsset; 7] = [ path: "SHA256SUMS", content_type: "text/plain", bytes: include_bytes!("../assets/SHA256SUMS"), - sha256: "8e582ed7a2aca06918f2866e7977af85142333793c15c063d9a2c9b9432f7410", + sha256: "a9c87b9444779af81ea7cf01f004016a01034bd78665502b2f24d8e86962c719", }, ]; diff --git a/rust/crates/pvm-runtime/src/application.rs b/rust/crates/pvm-runtime/src/application.rs index 4dadfff..dc04d15 100644 --- a/rust/crates/pvm-runtime/src/application.rs +++ b/rust/crates/pvm-runtime/src/application.rs @@ -5,7 +5,7 @@ use crate::corevm::{Interruption, Vm}; use crate::{ AudioChunk, Frame, GpuBatch, InputEvent, InputEventType, PresentationProfile, Runtime, - Tri2dFrame, MAX_FRAME_BYTES, + TextInputKind, Tri2dFrame, UiSemanticsFrame, INPUT_EVENT_BYTES, MAX_FRAME_BYTES, }; use anyhow::{anyhow, Context, Result}; use polkavm::ProgramBlob; @@ -137,6 +137,20 @@ impl ApplicationRuntime { } } + pub fn send_input_record(&mut self, record: [u8; INPUT_EVENT_BYTES]) -> Result<()> { + match self { + Self::Cooperative(runtime) => runtime.send_input_record(record), + Self::CoreVm(_) => Err(anyhow!("CoreVM does not support extended input records")), + } + } + + pub fn send_text_input(&mut self, kind: TextInputKind, text: &str) -> Result<()> { + match self { + Self::Cooperative(runtime) => runtime.send_text_input(kind, text), + Self::CoreVm(_) => Err(anyhow!("CoreVM does not support text input")), + } + } + pub fn set_motion_availability( &mut self, availability: crate::motion_wire::MotionAvailability, @@ -223,6 +237,13 @@ impl ApplicationRuntime { } } + pub fn take_ui_semantics(&mut self) -> Option { + match self { + Self::Cooperative(runtime) => runtime.take_ui_semantics(), + Self::CoreVm(_) => None, + } + } + pub fn take_audio(&mut self) -> Option { match self { Self::Cooperative(runtime) => runtime.take_audio(), diff --git a/rust/crates/pvm-runtime/src/lib.rs b/rust/crates/pvm-runtime/src/lib.rs index be597a4..f16ed6b 100644 --- a/rust/crates/pvm-runtime/src/lib.rs +++ b/rust/crates/pvm-runtime/src/lib.rs @@ -14,6 +14,7 @@ mod manifest; mod native_ffi; mod quake_keys; mod tri2d; +mod ui; #[cfg(target_arch = "wasm32")] mod wasm; #[cfg(any(target_arch = "wasm32", test))] @@ -40,6 +41,13 @@ pub use tri2d::{ MAX_TRI2D_SURFACE_SIZE, MAX_TRI2D_TEXTURES, MAX_TRI2D_TEXTURE_BYTES, MAX_TRI2D_TEXTURE_SIZE, MAX_TRI2D_VERTICES, TRI2D_HEADER_BYTES, TRI2D_MAGIC, TRI2D_VERSION, }; +pub use ui::{ + encode_text_input, focus_record, ime_state_record, wheel_record, TextInputKind, + UiSemanticAction, UiSemanticNode, UiSemanticRole, UiSemanticSnapshot, UiSemanticsFrame, + INPUT_FOCUS, INPUT_IME_COMMIT, INPUT_IME_DISABLED, INPUT_IME_ENABLED, INPUT_IME_PREEDIT, + INPUT_TEXT_COMMIT, INPUT_WHEEL, MAX_UI_SEMANTICS_BYTES, MAX_UI_SEMANTIC_NODES, + MAX_UI_SEMANTIC_STRING_BYTES, MAX_UI_TEXT_BYTES, +}; pub const ABI_VERSION: u32 = 1; @@ -347,9 +355,11 @@ struct HostState { audio_enabled: bool, presentation: PresentationProfile, tri2d_submitted: bool, + ui_semantics: Option, + ui_semantics_submitted: bool, audio: VecDeque, audio_samples: usize, - input: VecDeque, + input: VecDeque<[u8; INPUT_EVENT_BYTES]>, assets: HashMap>, clock: HostClock, logs: VecDeque, @@ -383,6 +393,8 @@ impl HostState { audio_enabled, tri2d_state: tri2d::Tri2dState::default(), tri2d_submitted: false, + ui_semantics: None, + ui_semantics_submitted: false, audio: VecDeque::new(), audio_samples: 0, input: VecDeque::new(), @@ -412,6 +424,7 @@ impl HostState { self.hostcalls_remaining = max_hostcalls; self.sleep_ms_remaining = max_sleep_ms; self.tri2d_submitted = false; + self.ui_semantics_submitted = false; self.gpu_submits_remaining = MAX_GPU_SUBMITS_PER_TICK; self.gpu_upload_bytes_remaining = MAX_GPU_UPLOAD_BYTES_PER_TICK; } @@ -433,19 +446,24 @@ impl HostState { } fn queue_input(&mut self, event: InputEvent) { - if event.event_type == InputEventType::SurfaceMetrics { + let _ = self.queue_input_record(event.encode()); + } + + fn queue_input_record(&mut self, record: [u8; INPUT_EVENT_BYTES]) -> Result<()> { + ui::validate_input_record(&record)?; + if record[0] == InputEventType::SurfaceMetrics as u8 { if let Some(position) = self .input .iter() - .rposition(|queued| queued.event_type == InputEventType::SurfaceMetrics) + .rposition(|queued| queued[0] == InputEventType::SurfaceMetrics as u8) { self.input.remove(position); } - } else if event.event_type == InputEventType::PointerMove + } else if record[0] == InputEventType::PointerMove as u8 && self .input .back() - .is_some_and(|queued| queued.event_type == InputEventType::PointerMove) + .is_some_and(|queued| queued[0] == InputEventType::PointerMove as u8) { self.input.pop_back(); } @@ -453,19 +471,45 @@ impl HostState { let discardable = self .input .iter() - .position(|queued| queued.event_type == InputEventType::PointerMove) + .position(|queued| queued[0] == InputEventType::PointerMove as u8) .or_else(|| { - self.input - .iter() - .position(|queued| queued.event_type != InputEventType::SurfaceMetrics) + self.input.iter().position(|queued| { + matches!( + queued[0], + value if value == InputEventType::PointerDelta as u8 + || value == ui::INPUT_WHEEL + ) + }) }); if let Some(position) = discardable { self.input.remove(position); } else { - self.input.pop_front(); + bail!("input queue is full"); } } - self.input.push_back(event); + self.input.push_back(record); + Ok(()) + } + + fn queue_input_records(&mut self, records: Vec<[u8; INPUT_EVENT_BYTES]>) -> Result<()> { + if self.input.len().saturating_add(records.len()) > MAX_QUEUED_INPUT_EVENTS { + bail!("input queue cannot accept a complete text event"); + } + for record in &records { + ui::validate_input_record(record)?; + } + self.input.extend(records); + Ok(()) + } + + fn queue_ui_semantics(&mut self, bytes: Vec) -> Result<()> { + if self.ui_semantics_submitted { + bail!("UI semantics were already submitted during this call"); + } + ui::validate_ui_semantics(&bytes)?; + self.ui_semantics = Some(UiSemanticsFrame { bytes }); + self.ui_semantics_submitted = true; + Ok(()) } fn take_truapi_request(&mut self) -> Option> { @@ -644,6 +688,30 @@ impl Runtime { ) .context("define host_tri2d_submit")?; + linker + .define_typed( + "host_ui_semantics_submit", + |caller: polkavm::Caller<'_, HostState>, + pointer: u32, + length: u32| + -> Result { + let length = length as usize; + if length == 0 || length > MAX_UI_SEMANTICS_BYTES { + return Ok(1); + } + caller.user_data.charge_hostcall(length)?; + if caller.user_data.ui_semantics_submitted { + return Ok(2); + } + let bytes = read_guest_memory(caller.instance, pointer, length)?; + if caller.user_data.queue_ui_semantics(bytes).is_err() { + return Ok(1); + } + Ok(0) + }, + ) + .context("define host_ui_semantics_submit")?; + linker .define_typed( "host_gpu_capabilities", @@ -835,7 +903,7 @@ impl Runtime { .ok_or_else(|| anyhow!("guest input destination overflow"))?; caller .instance - .write_memory(destination, &event.encode()) + .write_memory(destination, &event) .map_err(|error| anyhow!("write guest input: {error:?}"))?; written += INPUT_EVENT_BYTES as u32; } @@ -1090,6 +1158,15 @@ impl Runtime { self.state.queue_input(event); } + pub fn send_input_record(&mut self, record: [u8; INPUT_EVENT_BYTES]) -> Result<()> { + self.state.queue_input_record(record) + } + + pub fn send_text_input(&mut self, kind: TextInputKind, text: &str) -> Result<()> { + self.state + .queue_input_records(ui::encode_text_input(kind, text)?) + } + pub fn set_motion_availability(&mut self, availability: motion_wire::MotionAvailability) { self.state.motion.set_availability(availability); } @@ -1149,6 +1226,10 @@ impl Runtime { self.state.tri2d.take() } + pub fn take_ui_semantics(&mut self) -> Option { + self.state.ui_semantics.take() + } + pub fn take_audio(&mut self) -> Option { let chunk = self.state.audio.pop_front()?; self.state.audio_samples -= chunk.samples.len(); @@ -1504,7 +1585,10 @@ mod tests { }); } assert_eq!(state.input.len(), 1); - assert_eq!(state.input.back().unwrap().x, 9_999); + assert_eq!( + u16::from_le_bytes(state.input.back().unwrap()[2..4].try_into().unwrap()), + 9_999 + ); state.queue_input(InputEvent { event_type: InputEventType::SurfaceMetrics, @@ -1522,11 +1606,14 @@ mod tests { state .input .iter() - .filter(|event| event.event_type == InputEventType::SurfaceMetrics) + .filter(|event| event[0] == InputEventType::SurfaceMetrics as u8) .count(), 1 ); - assert_eq!(state.input.back().unwrap().x, 2_560); + assert_eq!( + u16::from_le_bytes(state.input.back().unwrap()[2..4].try_into().unwrap()), + 2_560 + ); for index in 0..(MAX_QUEUED_INPUT_EVENTS + 100) { state.queue_input(InputEvent { @@ -1541,7 +1628,7 @@ mod tests { state .input .iter() - .filter(|event| event.event_type == InputEventType::SurfaceMetrics) + .filter(|event| event[0] == InputEventType::SurfaceMetrics as u8) .count(), 1 ); @@ -1553,8 +1640,8 @@ mod tests { }); assert_eq!(state.input.len(), MAX_QUEUED_INPUT_EVENTS); assert_eq!( - state.input.back().unwrap().event_type, - InputEventType::SurfaceMetrics + state.input.back().unwrap()[0], + InputEventType::SurfaceMetrics as u8 ); } diff --git a/rust/crates/pvm-runtime/src/ui.rs b/rust/crates/pvm-runtime/src/ui.rs new file mode 100644 index 0000000..7360281 --- /dev/null +++ b/rust/crates/pvm-runtime/src/ui.rs @@ -0,0 +1,281 @@ +use anyhow::{anyhow, bail, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; + +use crate::INPUT_EVENT_BYTES; + +pub const MAX_UI_TEXT_BYTES: usize = 4 * 1024; +pub const MAX_UI_SEMANTICS_BYTES: usize = 256 * 1024; +pub const MAX_UI_SEMANTIC_NODES: usize = 1_024; +pub const MAX_UI_SEMANTIC_STRING_BYTES: usize = 1_024; + +pub const INPUT_TEXT_COMMIT: u8 = 8; +pub const INPUT_IME_PREEDIT: u8 = 9; +pub const INPUT_IME_COMMIT: u8 = 10; +pub const INPUT_IME_ENABLED: u8 = 11; +pub const INPUT_IME_DISABLED: u8 = 12; +pub const INPUT_FOCUS: u8 = 13; +pub const INPUT_WHEEL: u8 = 14; + +const CHUNK_LENGTH_MASK: u8 = 0x07; +const CHUNK_FIRST: u8 = 0x40; +const CHUNK_LAST: u8 = 0x80; +const CHUNK_ALLOWED: u8 = CHUNK_LENGTH_MASK | CHUNK_FIRST | CHUNK_LAST; +const CHUNK_BYTES: usize = 6; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TextInputKind { + Text, + ImePreedit, + ImeCommit, +} + +impl TextInputKind { + fn event_type(self) -> u8 { + match self { + Self::Text => INPUT_TEXT_COMMIT, + Self::ImePreedit => INPUT_IME_PREEDIT, + Self::ImeCommit => INPUT_IME_COMMIT, + } + } +} + +pub fn encode_text_input(kind: TextInputKind, text: &str) -> Result> { + let bytes = text.as_bytes(); + if bytes.len() > MAX_UI_TEXT_BYTES { + bail!("UI text exceeds {MAX_UI_TEXT_BYTES} bytes"); + } + let chunks = bytes.len().max(1).div_ceil(CHUNK_BYTES); + let mut records = Vec::with_capacity(chunks); + for index in 0..chunks { + let start = index * CHUNK_BYTES; + let end = bytes.len().min(start + CHUNK_BYTES); + let chunk = &bytes[start..end]; + let mut record = [0u8; INPUT_EVENT_BYTES]; + record[0] = kind.event_type(); + record[1] = u8::try_from(chunk.len()).unwrap(); + if index == 0 { + record[1] |= CHUNK_FIRST; + } + if index + 1 == chunks { + record[1] |= CHUNK_LAST; + } + record[2..2 + chunk.len()].copy_from_slice(chunk); + records.push(record); + } + Ok(records) +} + +pub fn ime_state_record(enabled: bool) -> [u8; INPUT_EVENT_BYTES] { + let mut record = [0u8; INPUT_EVENT_BYTES]; + record[0] = if enabled { + INPUT_IME_ENABLED + } else { + INPUT_IME_DISABLED + }; + record +} + +pub fn focus_record(focused: bool) -> [u8; INPUT_EVENT_BYTES] { + let mut record = [0u8; INPUT_EVENT_BYTES]; + record[0] = INPUT_FOCUS; + record[1] = u8::from(focused); + record +} + +pub fn wheel_record(delta_x: i16, delta_y: i16) -> [u8; INPUT_EVENT_BYTES] { + let mut record = [0u8; INPUT_EVENT_BYTES]; + record[0] = INPUT_WHEEL; + record[2..4].copy_from_slice(&delta_x.to_le_bytes()); + record[4..6].copy_from_slice(&delta_y.to_le_bytes()); + record +} + +pub(crate) fn validate_input_record(record: &[u8; INPUT_EVENT_BYTES]) -> Result<()> { + match record[0] { + 1..=7 => { + if record[6..] != [0, 0] { + bail!("fixed input record has nonzero reserved bytes"); + } + } + INPUT_TEXT_COMMIT | INPUT_IME_PREEDIT | INPUT_IME_COMMIT => { + if record[1] & !CHUNK_ALLOWED != 0 { + bail!("text input record has invalid flags"); + } + let length = usize::from(record[1] & CHUNK_LENGTH_MASK); + if length > CHUNK_BYTES || record[2 + length..].iter().any(|byte| *byte != 0) { + bail!("text input record has invalid padding"); + } + } + INPUT_IME_ENABLED | INPUT_IME_DISABLED => { + if record[1..].iter().any(|byte| *byte != 0) { + bail!("IME state record has nonzero payload"); + } + } + INPUT_FOCUS => { + if record[1] > 1 || record[2..].iter().any(|byte| *byte != 0) { + bail!("focus record is malformed"); + } + } + INPUT_WHEEL => { + if record[1] != 0 || record[6..] != [0, 0] { + bail!("wheel record is malformed"); + } + } + _ => bail!("unsupported input record type {}", record[0]), + } + Ok(()) +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum UiSemanticRole { + Window, + Group, + Label, + Button, + Link, + CheckBox, + Slider, + TextInput, + MultilineTextInput, + PasswordInput, + Image, + Unknown, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum UiSemanticAction { + Click, + Focus, + SetValue, + Increment, + Decrement, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct UiSemanticNode { + pub id: String, + pub parent: Option, + pub role: UiSemanticRole, + #[serde(default)] + pub name: String, + #[serde(default)] + pub value: String, + pub bounds: [f32; 4], + #[serde(default)] + pub actions: Vec, + #[serde(default)] + pub disabled: bool, + #[serde(default)] + pub focused: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct UiSemanticSnapshot { + pub version: u32, + pub generation: u64, + pub nodes: Vec, +} + +#[derive(Clone, Debug)] +pub struct UiSemanticsFrame { + pub bytes: Vec, +} + +pub(crate) fn validate_ui_semantics(bytes: &[u8]) -> Result<()> { + if bytes.is_empty() || bytes.len() > MAX_UI_SEMANTICS_BYTES { + bail!("UI semantics must contain 1..={MAX_UI_SEMANTICS_BYTES} bytes"); + } + let snapshot: UiSemanticSnapshot = serde_json::from_slice(bytes) + .map_err(|error| anyhow!("invalid UI semantics JSON: {error}"))?; + if snapshot.version != 1 || snapshot.nodes.is_empty() { + bail!("UI semantics must contain a version 1 node tree"); + } + if snapshot.nodes.len() > MAX_UI_SEMANTIC_NODES { + bail!("UI semantics exceed {MAX_UI_SEMANTIC_NODES} nodes"); + } + let mut ids = HashSet::with_capacity(snapshot.nodes.len()); + let mut roots = 0usize; + for node in &snapshot.nodes { + if !valid_semantic_id(&node.id) || !ids.insert(node.id.clone()) { + bail!("UI semantics contain an invalid or duplicate node id"); + } + if node.parent.is_none() { + roots += 1; + } + if node.name.len() > MAX_UI_SEMANTIC_STRING_BYTES + || node.value.len() > MAX_UI_SEMANTIC_STRING_BYTES + { + bail!("UI semantic node string exceeds {MAX_UI_SEMANTIC_STRING_BYTES} bytes"); + } + let [x0, y0, x1, y1] = node.bounds; + if !node.bounds.iter().all(|value| value.is_finite()) || x1 < x0 || y1 < y0 { + bail!("UI semantic node has invalid bounds"); + } + } + if roots != 1 { + bail!("UI semantics must contain exactly one root"); + } + for node in &snapshot.nodes { + if node + .parent + .as_ref() + .is_some_and(|parent| !ids.contains(parent)) + { + bail!("UI semantic node references an unknown parent"); + } + } + Ok(()) +} + +fn valid_semantic_id(value: &str) -> bool { + !value.is_empty() + && value.len() <= 16 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn text_records_round_trip_chunk_boundaries() { + let records = encode_text_input(TextInputKind::Text, "hello π").unwrap(); + assert_eq!(records.len(), 2); + assert_ne!(records[0][1] & CHUNK_FIRST, 0); + assert_ne!(records[1][1] & CHUNK_LAST, 0); + for record in &records { + validate_input_record(record).unwrap(); + } + } + + #[test] + fn semantics_require_one_bounded_tree() { + let valid = serde_json::to_vec(&UiSemanticSnapshot { + version: 1, + generation: 7, + nodes: vec![UiSemanticNode { + id: "1".into(), + parent: None, + role: UiSemanticRole::Window, + name: "Playground".into(), + value: String::new(), + bounds: [0.0, 0.0, 640.0, 480.0], + actions: Vec::new(), + disabled: false, + focused: false, + }], + }) + .unwrap(); + validate_ui_semantics(&valid).unwrap(); + + let malformed = br#"{"version":1,"generation":1,"nodes":[]}"#; + assert!(validate_ui_semantics(malformed).is_err()); + } +} diff --git a/rust/crates/pvm-runtime/src/wasm.rs b/rust/crates/pvm-runtime/src/wasm.rs index 2221017..f6748de 100644 --- a/rust/crates/pvm-runtime/src/wasm.rs +++ b/rust/crates/pvm-runtime/src/wasm.rs @@ -4,8 +4,8 @@ use crate::{ ApplicationRuntime, AudioChunk, Frame, GpuBatch, InputEvent, InputEventType, - PresentationProfile, Tri2dFrame, MAX_ASSET_BYTES, MAX_ASSET_FILES, MAX_ASSET_FILE_BYTES, - MAX_PROGRAM_BYTES, + PresentationProfile, Tri2dFrame, UiSemanticsFrame, INPUT_EVENT_BYTES, MAX_ASSET_BYTES, + MAX_ASSET_FILES, MAX_ASSET_FILE_BYTES, MAX_PROGRAM_BYTES, }; use anyhow::{anyhow, Result}; use polkavm::BackendKind; @@ -35,6 +35,7 @@ struct BrowserHost { staging: Vec, frame: Option, tri2d: Option, + ui_semantics: Option, gpu_batch: Option, audio: Option, truapi_request: Option>, @@ -51,6 +52,7 @@ impl BrowserHost { staging: Vec::new(), frame: None, tri2d: None, + ui_semantics: None, gpu_batch: None, audio: None, truapi_request: None, @@ -71,6 +73,7 @@ impl BrowserHost { fn clear_outputs(&mut self) { self.frame = None; self.tri2d = None; + self.ui_semantics = None; self.gpu_batch = None; self.truapi_request = None; self.audio = None; @@ -335,6 +338,17 @@ pub extern "C" fn pvm_browser_send_input(event_type: u32, code: u32, x: u32, y: }) } +#[no_mangle] +pub extern "C" fn pvm_browser_send_input_record() -> u32 { + status(|host| { + let bytes = std::mem::take(&mut host.staging); + let record: [u8; INPUT_EVENT_BYTES] = bytes + .try_into() + .map_err(|_| anyhow!("extended input record must contain {INPUT_EVENT_BYTES} bytes"))?; + host.running()?.send_input_record(record) + }) +} + #[no_mangle] pub extern "C" fn pvm_browser_take_frame() -> u32 { HOST.with(|host| { @@ -409,6 +423,38 @@ pub extern "C" fn pvm_browser_tri2d_length() -> u32 { }) } +#[no_mangle] +pub extern "C" fn pvm_browser_take_ui_semantics() -> u32 { + HOST.with(|host| { + let mut host = host.borrow_mut(); + host.ui_semantics = match &mut host.phase { + Phase::Running(runtime) => runtime.take_ui_semantics(), + _ => None, + }; + u32::from(host.ui_semantics.is_some()) + }) +} + +#[no_mangle] +pub extern "C" fn pvm_browser_ui_semantics_pointer() -> u32 { + HOST.with(|host| { + host.borrow() + .ui_semantics + .as_ref() + .map_or(0, |frame| frame.bytes.as_ptr() as usize as u32) + }) +} + +#[no_mangle] +pub extern "C" fn pvm_browser_ui_semantics_length() -> u32 { + HOST.with(|host| { + host.borrow() + .ui_semantics + .as_ref() + .map_or(0, |frame| frame.bytes.len() as u32) + }) +} + #[no_mangle] pub extern "C" fn pvm_browser_take_gpu_batch() -> u32 { HOST.with(|host| {