feat(ui): add semantic and text input ABI

docs/runtime/polkavm-app-abi-v1.md

@@ -238,7 +238,7 @@ and returns the number of bytes written. It never writes a partial record.
 Zero means that no event was available or that the capacity was smaller than
 one record.
 
-An input record is:
+The legacy fixed record layout is:
 
 ```text
 offset  type  field
@@ -252,18 +252,54 @@ offset  type  field
 ABI v1 event types are:
 
 ```text
-1  key down
-2  key up
-3  pointer button down
-4  pointer button up
-5  pointer position
-6  pointer delta
-7  surface metrics
+1   key down
+2   key up
+3   pointer button down
+4   pointer button up
+5   pointer position
+6   pointer delta
+7   surface metrics
+8   committed UTF-8 text chunk
+9   IME preedit UTF-8 chunk
+10  IME commit UTF-8 chunk
+11  IME enabled
+12  IME disabled or cancelled
+13  focus (`code` is 0 or 1)
+14  wheel delta (`x` and `y` are signed i16)
 ```
 
-The device-input contract defines code values, coordinate interpretation, and
-surface-metric scaling. ABI v1 does not define touch, wheel, UTF-8 text, IME,
-or focus events.
+Text and IME records use `code` bits 0–2 as a payload length from zero through
+six, bit 6 for the first chunk, and bit 7 for the last chunk. Bytes 2–7 contain
+the chunk and zero padding. A complete text event is at most 4 KiB. The Host
+MUST queue all chunks of one event atomically; the guest MUST reject malformed
+flag sequences or invalid UTF-8.
+
+### UI semantics
+
+```text
+host_ui_semantics_submit(pointer: u32, length: u32) -> u32
+```
+
+The guest may submit one complete UTF-8 JSON semantic tree per `init` or
+`update` call. The tree is presentation output, not an instruction to invoke
+guest functions. Hosts use its roles, labels, values, actions, focus, and
+surface-relative bounds for accessibility and UI automation, then deliver
+actual pointer, keyboard, text, or IME records for every interaction.
+
+The version 1 object contains `version`, monotonic `generation`, and `nodes`.
+Each node contains a nonzero numeric `id`, nullable `parent`, `role`, `name`,
+`value`, `[x0,y0,x1,y1]` bounds, `actions`, `disabled`, and `focused`. Version 1
+allows at most 1,024 nodes, 1 KiB per name or value, and 256 KiB for the whole
+tree. It requires exactly one root, unique IDs, existing parents, finite ordered
+bounds, and no unknown object fields.
+
+Return values:
+
+```text
+0  accepted
+1  malformed, out-of-bounds, or over-limit tree
+2  a tree was already submitted during this call
+```
 
 ### Motion
 

js/packages/pvm-browser-runtime/src/pvm-runtime-core.js

@@ -132,6 +132,19 @@ globalThis.createPvmRuntime = (endpoint) => {
     postMessage({ type: "tri2d", bytes }, [bytes.buffer]);
   }
 
+  function drainUiSemantics() {
+    if (!pvm.pvm_browser_take_ui_semantics?.()) {
+      return;
+    }
+    const length = pvm.pvm_browser_ui_semantics_length();
+    const bytes = new Uint8Array(
+      pvm.memory.buffer,
+      pvm.pvm_browser_ui_semantics_pointer(),
+      length,
+    ).slice();
+    postMessage({ type: "ui-semantics", bytes }, [bytes.buffer]);
+  }
+
   function drainGpuBatches() {
     while (pvm.pvm_browser_take_gpu_batch?.()) {
       const length = pvm.pvm_browser_gpu_batch_length();
@@ -214,6 +227,7 @@ globalThis.createPvmRuntime = (endpoint) => {
         );
         drainFrame();
         drainTri2d();
+        drainUiSemantics();
         drainGpuBatches();
         drainTruapiRequests();
         drainAudio();
@@ -547,15 +561,23 @@ globalThis.createPvmRuntime = (endpoint) => {
       translated.sendInput(bytes);
       return;
     }
-    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    if (bytes[0] <= 7) {
+      const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+      check(
+        pvm.pvm_browser_send_input(
+          bytes[0],
+          bytes[1],
+          view.getUint16(2, true),
+          view.getUint16(4, true),
+        ),
+        "send PolkaVM browser input",
+      );
+      return;
+    }
+    stage(bytes);
     check(
-      pvm.pvm_browser_send_input(
-        bytes[0],
-        bytes[1],
-        view.getUint16(2, true),
-        view.getUint16(4, true),
-      ),
-      "send PolkaVM browser input",
+      pvm.pvm_browser_send_input_record(),
+      "send PolkaVM browser extended input",
     );
   }
 

js/packages/pvm-browser-runtime/src/pvm-wasm-translated.js

@@ -27,6 +27,9 @@
   const MAX_AUDIO_SAMPLES = 48000 * 2;
   const MAX_FRAME_BYTES = 16 * 1024 * 1024;
   const MAX_TRI2D_BYTES = 8 * 1024 * 1024;
+  const MAX_UI_SEMANTICS_BYTES = 256 * 1024;
+  const MAX_UI_SEMANTIC_NODES = 1024;
+  const MAX_UI_SEMANTIC_STRING_BYTES = 1024;
   const MAX_GPU_BATCH_BYTES = 4 * 1024 * 1024;
   const MAX_GPU_EVENT_BYTES = 64 * 1024;
   const MAX_GPU_EVENTS = 256;
@@ -56,6 +59,55 @@
   const decoder = new TextDecoder();
   const encoder = new TextEncoder();
 
+  function validUiSemantics(bytes) {
+    let snapshot;
+    try {
+      snapshot = JSON.parse(decoder.decode(bytes));
+    } catch {
+      return false;
+    }
+    if (
+      snapshot?.version !== 1 ||
+      !Number.isSafeInteger(snapshot.generation) ||
+      snapshot.generation < 0 ||
+      !Array.isArray(snapshot.nodes) ||
+      !snapshot.nodes.length ||
+      snapshot.nodes.length > MAX_UI_SEMANTIC_NODES
+    ) {
+      return false;
+    }
+    const ids = new Set();
+    let roots = 0;
+    for (const node of snapshot.nodes) {
+      if (
+        typeof node?.id !== "string" ||
+        !/^[0-9a-f]{1,16}$/.test(node.id) ||
+        ids.has(node.id) ||
+        !Array.isArray(node.bounds) ||
+        node.bounds.length !== 4 ||
+        !node.bounds.every(Number.isFinite) ||
+        node.bounds[2] < node.bounds[0] ||
+        node.bounds[3] < node.bounds[1] ||
+        typeof node.name !== "string" ||
+        encoder.encode(node.name).byteLength > MAX_UI_SEMANTIC_STRING_BYTES ||
+        typeof node.value !== "string" ||
+        encoder.encode(node.value).byteLength > MAX_UI_SEMANTIC_STRING_BYTES
+      ) {
+        return false;
+      }
+      ids.add(node.id);
+      if (node.parent === null) {
+        roots++;
+      }
+    }
+    return (
+      roots === 1 &&
+      snapshot.nodes.every(
+        (node) => node.parent === null || ids.has(node.parent),
+      )
+    );
+  }
+
   function readMetadata(module) {
     const sections = WebAssembly.Module.customSections(
       module,
@@ -300,6 +352,7 @@
       this.truapiResponses = [];
       this.truapiResponseBytes = 0;
       this.tri2dSubmitted = false;
+      this.uiSemanticsSubmitted = false;
       this.maxGas = BigInt(maxGas);
       this.input = [];
       this.coreInput = [];
@@ -354,6 +407,7 @@
       this.timeMs = timeMs;
       this.gpuSubmits = 0;
       this.truapiRequests = 0;
+      this.uiSemanticsSubmitted = false;
       this.truapiRequestBytes = 0;
       this.#resetBudget(
         this.coreVm && !this.coreVmStarted
@@ -748,6 +802,26 @@
           this.#setReg(7, 0n);
           return false;
         }
+        case "host_ui_semantics_submit": {
+          const length = this.#u32(a1);
+          if (!length || length > MAX_UI_SEMANTICS_BYTES) {
+            this.#setReg(7, 1n);
+            return false;
+          }
+          if (this.uiSemanticsSubmitted) {
+            this.#setReg(7, 2n);
+            return false;
+          }
+          const bytes = this.#read(this.#u32(a0), length);
+          if (!validUiSemantics(bytes)) {
+            this.#setReg(7, 1n);
+            return false;
+          }
+          this.emit({ type: "ui-semantics", bytes }, [bytes.buffer]);
+          this.uiSemanticsSubmitted = true;
+          this.#setReg(7, 0n);
+          return false;
+        }
         case "host_gpu_capabilities": {
           if (this.graphicsProfile !== "webgpu-raster") {
             this.#setReg(7, BigInt(GPU_ERROR_INVALID_STATE));

rust/crates/pvm-runtime-assets/assets/SHA256SUMS

@@ -1,6 +1,6 @@
-3b448399a39ee571765f41266032c621ce3668176d7f0d6372c85d6068e7d4a5  pvm-browser-runtime.wasm
-55ce96805e22f60ec9a4da072a316c34c1f650d99bdf1421805c1ba35c2d8161  pvm-worker.js
+e5272df5ed0ef9c682ef39198161b44c4b46f1fa17c2df6f3d7c216c5934c32b  pvm-browser-runtime.wasm
+48df143a60691765029e39555a9b4a3e51176000806fb3abb8a04f22e97c89ba  pvm-worker.js
 33418e2c81c117539569eb4cf91af4d058cdfae7dd4556b13f3687f6d6b3bae4  pvm-gpu-worker.js
-a4c2a9ed4274c79b0c11ddbac163f1de200d41005ca1ce54909a5b8116d82c46  pvm-wasm-translated.js
-79dce3ab925e78e9ef8e5df6e34fb2823e6f362cbc97d265a9ffdcc22cc24ed4  pvm-runtime-core.js
+3bdb850c57ec1fc32f0d08bacca74313a2d77a6d4d4fedffcad87ddfd3028a7b  pvm-wasm-translated.js
+7e746495b2e3048fcba922f714f439e84942175c67c8e73de36f5b892af1f965  pvm-runtime-core.js
 9c929f5d5c64a1b75e7e48485d7c3944ed6838112177ea778827a2c407c2d820  pvm-wasm-worker-entry.js

rust/crates/pvm-runtime-assets/assets/pvm-runtime-core.js

@@ -132,6 +132,19 @@ globalThis.createPvmRuntime = (endpoint) => {
     postMessage({ type: "tri2d", bytes }, [bytes.buffer]);
   }
 
+  function drainUiSemantics() {
+    if (!pvm.pvm_browser_take_ui_semantics?.()) {
+      return;
+    }
+    const length = pvm.pvm_browser_ui_semantics_length();
+    const bytes = new Uint8Array(
+      pvm.memory.buffer,
+      pvm.pvm_browser_ui_semantics_pointer(),
+      length,
+    ).slice();
+    postMessage({ type: "ui-semantics", bytes }, [bytes.buffer]);
+  }
+
   function drainGpuBatches() {
     while (pvm.pvm_browser_take_gpu_batch?.()) {
       const length = pvm.pvm_browser_gpu_batch_length();
@@ -214,6 +227,7 @@ globalThis.createPvmRuntime = (endpoint) => {
         );
         drainFrame();
         drainTri2d();
+        drainUiSemantics();
         drainGpuBatches();
         drainTruapiRequests();
         drainAudio();
@@ -547,15 +561,23 @@ globalThis.createPvmRuntime = (endpoint) => {
       translated.sendInput(bytes);
       return;
     }
-    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    if (bytes[0] <= 7) {
+      const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+      check(
+        pvm.pvm_browser_send_input(
+          bytes[0],
+          bytes[1],
+          view.getUint16(2, true),
+          view.getUint16(4, true),
+        ),
+        "send PolkaVM browser input",
+      );
+      return;
+    }
+    stage(bytes);
     check(
-      pvm.pvm_browser_send_input(
-        bytes[0],
-        bytes[1],
-        view.getUint16(2, true),
-        view.getUint16(4, true),
-      ),
-      "send PolkaVM browser input",
+      pvm.pvm_browser_send_input_record(),
+      "send PolkaVM browser extended input",
     );
   }