Improved canvas keydown events extractor

OUTPUT-EXAMPLE.md

@@ -80,6 +80,20 @@ JSON example containing all possible step types produced by the sequence recorde
     "value": "search query",
     "frame": ""
   },
+  {
+    "timestamp": 1718724007500,
+    "css": "#flutterCanvas",
+    "xpath": "/html/body/div/form/canvas",
+    "type": "bfill_value",
+    "value": "admin@example.com",
+    "frame": "",
+    "coords": {
+      "x": 190,
+      "y": 200,
+      "width": 380,
+      "height": 520
+    }
+  },
   {
     "timestamp": 1718724008000,
     "css": "#username",
@@ -146,7 +160,7 @@ JSON example containing all possible step types produced by the sequence recorde
 | `dblclick` | Double click on an element |
 | `mouseover` | Hover over an element (debounced at 500ms) |
 | `fill_value` | Value entered into an `<input>` or `<textarea>` (captured on blur or Enter) |
-| `bfill_value` | Buffered fill — keystroke-reconstructed value when the DOM `.value` is empty (e.g. virtual keyboards, frameworks that clear the field) |
+| `bfill_value` | Buffered fill — keystroke-reconstructed value when the DOM `.value` is empty (e.g. virtual keyboards, frameworks that clear the field, or canvas-rendered forms). For canvas inputs, includes `coords` from the preceding `bclick` that activated the virtual field |
 | `press_key` | Key press, currently only Enter (`value: 13`), emitted after a `fill_value`/`bfill_value` on Enter |
 | `change` (`check`) | Checkbox or radio button toggled; includes `checked` boolean |
 | `change` (`select`) | Single `<select>` dropdown changed; includes `selected` index and `value` |

package.json

@@ -1,6 +1,6 @@
 {
   "name": "snyk-api-and-web-record-sequence",
-  "version": "1.2.2",
+  "version": "1.2.3",
   "description": "Snyk API & Web Record login/sequence",
   "license": "MIT",
   "scripts": {

src/manifest-firefox.json

@@ -1,6 +1,6 @@
 {
   "name": "Snyk API & Web Sequence Recorder",
-  "version": "1.2.2",
+  "version": "1.2.3",
   "browser_specific_settings": {
     "gecko": {
       "id": "sequence-recorder@probely.com",

src/pages/Content/modules/collectEvents.js

@@ -22,6 +22,8 @@ let stoMouseover = false;
 let pendingInput = null;
 let flushedInput = null;
 const keystrokeBuffers = new WeakMap();
+let pendingCanvasInput = null;
+let canvasKeystrokeBuffer = '';
 
 function getEffectiveValue(element) {
   const domValue = element.value;
@@ -140,7 +142,9 @@ export function interceptEvents(event, doc, ifrSelector, callback) {
     }
   }
   if (selector && selector.toLowerCase() === 'html') {
-    return;
+    if (!(type === 'keydown' && pendingCanvasInput)) {
+      return;
+    }
   }
   let xpath = null;
   try {
@@ -183,6 +187,21 @@ export function interceptEvents(event, doc, ifrSelector, callback) {
       pendingInput = null;
     }
 
+    if (pendingCanvasInput && canvasKeystrokeBuffer.length > 0 && callback) {
+      const canvasFillEvent = {
+        ...pendingCanvasInput.oEventBase,
+        type: 'bfill_value',
+        value: canvasKeystrokeBuffer,
+        frame: pendingCanvasInput.frame,
+      };
+      if (pendingCanvasInput.shadow_host_css) {
+        canvasFillEvent.shadow_host_css = pendingCanvasInput.shadow_host_css;
+      }
+      callback({ messageType: 'events', event: { ...canvasFillEvent } });
+    }
+    canvasKeystrokeBuffer = '';
+    pendingCanvasInput = null;
+
     if (
       lastNodes.return === lastNodes.change &&
       tgt !== lastNodes.return &&
@@ -280,6 +299,14 @@ export function interceptEvents(event, doc, ifrSelector, callback) {
         }
       }
     }
+    if (typeStr === 'bclick') {
+      pendingCanvasInput = {
+        element: tgt,
+        oEventBase: { ...oEventBase },
+        frame: ifrSelector,
+        shadow_host_css: oEventToSend.shadow_host_css || null,
+      };
+    }
   } else if (type === 'dblclick') {
     lastNodes.dblclick = tgt;
     oEventToSend = {
@@ -360,6 +387,14 @@ export function interceptEvents(event, doc, ifrSelector, callback) {
           keystrokeBuffers.set(tgt, buf.slice(0, -1));
         }
       }
+    } else if (pendingCanvasInput) {
+      if (event.key && event.key.length === 1) {
+        canvasKeystrokeBuffer += event.key;
+      } else if (event.key === 'Backspace') {
+        if (canvasKeystrokeBuffer.length > 0) {
+          canvasKeystrokeBuffer = canvasKeystrokeBuffer.slice(0, -1);
+        }
+      }
     }
     if (
       nodeName === 'input' &&
@@ -384,6 +419,31 @@ export function interceptEvents(event, doc, ifrSelector, callback) {
         }
       }
       keystrokeBuffers.delete(tgt);
+    } else if (pendingCanvasInput && event.keyCode === 13) {
+      if (canvasKeystrokeBuffer.length > 0 && callback) {
+        const canvasFillEvent = {
+          ...pendingCanvasInput.oEventBase,
+          type: 'bfill_value',
+          value: canvasKeystrokeBuffer,
+          frame: pendingCanvasInput.frame,
+        };
+        if (pendingCanvasInput.shadow_host_css) {
+          canvasFillEvent.shadow_host_css = pendingCanvasInput.shadow_host_css;
+        }
+        callback({ messageType: 'events', event: { ...canvasFillEvent } });
+        canvasKeystrokeBuffer = '';
+      }
+      if (callback) {
+        const canvasReturnEvent = {
+          ...pendingCanvasInput.oEventBase,
+          timestamp: new Date().getTime(),
+          type: 'press_key',
+          value: 13,
+          frame: pendingCanvasInput.frame,
+        };
+        callback({ messageType: 'events', event: { ...canvasReturnEvent } });
+      }
+      pendingCanvasInput = null;
     }
   } else if (type === 'blur') {
     lastNodes.blur = tgt;

test-build-firefox/manifest.json

@@ -1,6 +1,6 @@
 {
   "name": "Snyk API & Web Sequence Recorder",
-  "version": "1.2.2",
+  "version": "1.2.3",
   "browser_specific_settings": {
     "gecko": {
       "id": "sequence-recorder@probely.com",

test-build/manifest.json

@@ -1 +1 @@
-{"version":"1.2.2","manifest_version":3,"name":"Snyk API & Web Sequence Recorder","action":{"default_popup":"popup.html","default_icon":{"16":"icon-34.png","48":"icon-48.png"}},"icons":{"128":"icon-128.png"},"background":{"service_worker":"background.bundle.js"},"content_scripts":[{"matches":["http://*/*","https://*/*"],"js":["contentScript.bundle.js"],"css":["content.styles.css"],"run_at":"document_start","all_frames":true,"match_about_blank":true}],"web_accessible_resources":[{"resources":["content.styles.css","icon-128.png","icon-34.png"],"matches":["<all_urls>"]}],"permissions":["storage","activeTab"],"host_permissions":["http://*/*","https://*/*"]}
+{"version":"1.2.3","manifest_version":3,"name":"Snyk API & Web Sequence Recorder","action":{"default_popup":"popup.html","default_icon":{"16":"icon-34.png","48":"icon-48.png"}},"icons":{"128":"icon-128.png"},"background":{"service_worker":"background.bundle.js"},"content_scripts":[{"matches":["http://*/*","https://*/*"],"js":["contentScript.bundle.js"],"css":["content.styles.css"],"run_at":"document_start","all_frames":true,"match_about_blank":true}],"web_accessible_resources":[{"resources":["content.styles.css","icon-128.png","icon-34.png"],"matches":["<all_urls>"]}],"permissions":["storage","activeTab"],"host_permissions":["http://*/*","https://*/*"]}