Merge branch 'staging-v4' of github.com:simstudioai/sim into staging-v4

Author: icecrasher321

apps/desktop/src/main/browser-agent/cdp.ts

@@ -1,9 +1,12 @@
 /**
  * CDP instrumentation for agent tabs via `webContents.debugger`: auto-handles
  * the page states that would otherwise wedge automation (JS dialogs, file
- * choosers) and captures screenshots that work even while the view is hidden.
- * The user sees and drives the real embedded page, so there is no screencast
- * and no synthetic input here.
+ * choosers), captures screenshots that work even while the view is hidden,
+ * and dispatches TRUSTED input (key events, text insertion). Trusted input
+ * goes through Blink's real input pipeline — unlike synthetic DOM
+ * `KeyboardEvent`s, it triggers default actions (select-all, deletion, caret
+ * movement, character insertion) and is honored by code editors. The user
+ * sees and drives the real embedded page, so there is no screencast.
  */
 import { createLogger } from '@sim/logger'
 import type { WebContents } from 'electron'
@@ -89,3 +92,29 @@ export async function captureScreenshot(contents: WebContents): Promise<string>
   })
   return `data:image/jpeg;base64,${result.data}`
 }
+
+/** One half of a trusted key press (`Input.dispatchKeyEvent` params). */
+export interface CdpKeyEvent {
+  type: 'keyDown' | 'rawKeyDown' | 'keyUp'
+  modifiers: number
+  key: string
+  code: string
+  windowsVirtualKeyCode: number
+  nativeVirtualKeyCode: number
+  text?: string
+  /** Blink editing commands to run with the event (macOS shortcut parity). */
+  commands?: string[]
+}
+
+/** Dispatches one trusted key event through Blink's input pipeline. */
+export async function dispatchKeyEvent(contents: WebContents, event: CdpKeyEvent): Promise<void> {
+  await send(contents, 'Input.dispatchKeyEvent', event as unknown as Record<string, unknown>)
+}
+
+/**
+ * Inserts text at the focused element's selection (replacing it) through the
+ * native IME path — works in plain fields and code editors alike.
+ */
+export async function insertText(contents: WebContents, text: string): Promise<void> {
+  await send(contents, 'Input.insertText', { text })
+}

apps/desktop/src/main/browser-agent/driver.test.ts

@@ -36,6 +36,72 @@ describe('parseKeyCombo', () => {
   })
 })
 
+describe('buildKeyDispatchPlan', () => {
+  let driver: DriverModule
+
+  beforeEach(async () => {
+    driver = await freshDriver()
+  })
+
+  it('carries text for printable keys so Blink inserts the character', () => {
+    const [down, up] = driver.buildKeyDispatchPlan(driver.parseKeyCombo('a'), 'linux')
+    expect(down).toMatchObject({ type: 'keyDown', text: 'a', key: 'a', modifiers: 0 })
+    expect(up).toMatchObject({ type: 'keyUp', key: 'a' })
+  })
+
+  it('sends Enter with a carriage return so defaults fire (form submit)', () => {
+    const [down] = driver.buildKeyDispatchPlan(driver.parseKeyCombo('Enter'), 'linux')
+    expect(down).toMatchObject({ type: 'keyDown', text: '\r', windowsVirtualKeyCode: 13 })
+  })
+
+  it('sends editing keys as rawKeyDown without text', () => {
+    const [down] = driver.buildKeyDispatchPlan(driver.parseKeyCombo('Backspace'), 'linux')
+    expect(down.type).toBe('rawKeyDown')
+    expect(down.text).toBeUndefined()
+  })
+
+  it('maps Cmd shortcuts to Blink editing commands on macOS only', () => {
+    const combo = driver.parseKeyCombo('Cmd+A')
+    const [macDown] = driver.buildKeyDispatchPlan(combo, 'darwin')
+    expect(macDown.commands).toEqual(['selectAll'])
+    expect(macDown.modifiers).toBe(4)
+    const [linuxDown] = driver.buildKeyDispatchPlan(combo, 'linux')
+    expect(linuxDown.commands).toBeUndefined()
+  })
+
+  it('treats Control shortcuts as Cmd on macOS (the model does not know the host OS)', () => {
+    const combo = driver.parseKeyCombo('Control+A')
+    const [macDown] = driver.buildKeyDispatchPlan(combo, 'darwin')
+    expect(macDown.commands).toEqual(['selectAll'])
+    expect(macDown.modifiers).toBe(4) // ctrl normalized away, meta set
+    // On Linux/Windows Ctrl+A is Blink-native; no rewrite.
+    const [linuxDown] = driver.buildKeyDispatchPlan(combo, 'linux')
+    expect(linuxDown.modifiers).toBe(2)
+    expect(linuxDown.commands).toBeUndefined()
+  })
+
+  it('does not rewrite non-editing Control combos on macOS', () => {
+    const [down] = driver.buildKeyDispatchPlan(driver.parseKeyCombo('Control+K'), 'darwin')
+    expect(down.modifiers).toBe(2)
+    expect(down.commands).toBeUndefined()
+  })
+
+  it('maps Cmd+Shift+Z to redo and Cmd+Z to undo on macOS', () => {
+    const [redo] = driver.buildKeyDispatchPlan(driver.parseKeyCombo('Cmd+Shift+Z'), 'darwin')
+    expect(redo.commands).toEqual(['redo'])
+    const [undo] = driver.buildKeyDispatchPlan(driver.parseKeyCombo('Cmd+Z'), 'darwin')
+    expect(undo.commands).toEqual(['undo'])
+  })
+
+  it('encodes the CDP modifier bitmask (Alt=1 Ctrl=2 Meta=4 Shift=8)', () => {
+    const [down] = driver.buildKeyDispatchPlan(driver.parseKeyCombo('Control+Shift+K'), 'linux')
+    expect(down.modifiers).toBe(2 | 8)
+    // Modified letters must not carry text — they are shortcuts, not typing.
+    expect(down.type).toBe('rawKeyDown')
+    expect(down.text).toBeUndefined()
+  })
+})
+
 describe('executeTool', () => {
   let driver: DriverModule
 

apps/desktop/src/main/browser-agent/driver.ts

@@ -4,12 +4,16 @@
  * live page state.
  *
  * Perception drives through injected page functions (element registry with a
- * structural outline); actuation is synthetic DOM events from those same
- * functions. The user needs no input translation at all — the real page is
- * embedded in the Sim window, so their clicks and typing are native. Tool
- * calls serialize through a queue — one real browser can only do one thing at
- * a time — and every call is bounded by a watchdog so the Sim side always
- * gets a response instead of waiting out its own timeout against silence.
+ * structural outline). Keyboard actuation (press_key, type) goes through
+ * TRUSTED CDP input events — synthetic DOM KeyboardEvents never trigger
+ * default editing actions (select-all, deletion, character insertion) and are
+ * ignored by code editors, so they exist only as a fallback. Clicks still use
+ * injected functions (element-targeted, no coordinate math). The user needs
+ * no input translation at all — the real page is embedded in the Sim window,
+ * so their clicks and typing are native. Tool calls serialize through a
+ * queue — one real browser can only do one thing at a time — and every call
+ * is bounded by a watchdog so the Sim side always gets a response instead of
+ * waiting out its own timeout against silence.
  */
 import type {
   BrowserPageState,
@@ -23,10 +27,12 @@ import * as cdp from '@/main/browser-agent/cdp'
 import {
   clickElement,
   collectSnapshot,
+  focusElementForTyping,
   getViewportInfo,
   hoverElement,
   pageContainsText,
   pressKeyOnPage,
+  readActiveElementState,
   readPageText,
   scrollPage,
   selectOptionInElement,
@@ -343,6 +349,101 @@ export function parseKeyCombo(combo: string): ParsedCombo {
   throw new ToolError(`Unrecognized key: "${keyPart}"`)
 }
 
+// ---------------------------------------------------------------------------
+// Trusted key dispatch (CDP Input.dispatchKeyEvent)
+// ---------------------------------------------------------------------------
+
+/** CDP `Input` modifier bitmask: Alt=1, Ctrl=2, Meta=4, Shift=8. */
+function cdpModifiers(combo: ParsedCombo): number {
+  return (combo.alt ? 1 : 0) | (combo.ctrl ? 2 : 0) | (combo.meta ? 4 : 0) | (combo.shift ? 8 : 0)
+}
+
+function editingCommandFor(combo: ParsedCombo): string | null {
+  switch (combo.key.toLowerCase()) {
+    case 'a':
+      return 'selectAll'
+    case 'c':
+      return 'copy'
+    case 'x':
+      return 'cut'
+    case 'v':
+      return 'paste'
+    case 'z':
+      return combo.shift ? 'redo' : 'undo'
+    default:
+      return null
+  }
+}
+
+/**
+ * On macOS the editing shortcuts are bound in the system menu layer, which
+ * CDP key events never traverse — so Blink must be told the editing command
+ * explicitly (same technique as Puppeteer/Playwright). The model doesn't know
+ * the host OS and often says "Control+A", so on macOS Ctrl is treated as Cmd
+ * for these shortcuts: both must select all, not silently no-op. On other
+ * platforms Ctrl+key is handled inside Blink and needs no help.
+ */
+function normalizeComboForPlatform(combo: ParsedCombo, platform: NodeJS.Platform): ParsedCombo {
+  if (platform !== 'darwin' || !combo.ctrl || combo.meta || editingCommandFor(combo) === null) {
+    return combo
+  }
+  return { ...combo, ctrl: false, meta: true }
+}
+
+function macEditingCommands(combo: ParsedCombo, platform: NodeJS.Platform): string[] {
+  if (platform !== 'darwin' || !combo.meta) return []
+  const command = editingCommandFor(combo)
+  return command ? [command] : []
+}
+
+/**
+ * Builds the trusted keyDown/keyUp pair for a combo. Printable keys without
+ * ctrl/meta carry `text` so Blink inserts the character; Enter carries "\r"
+ * so it activates defaults (form submission, newline). Everything else is a
+ * rawKeyDown, which still drives Blink's default editing actions (Backspace
+ * deletes, arrows move the caret, Ctrl/Cmd+A selects all).
+ */
+export function buildKeyDispatchPlan(
+  rawCombo: ParsedCombo,
+  platform: NodeJS.Platform = process.platform
+): [cdp.CdpKeyEvent, cdp.CdpKeyEvent] {
+  const combo = normalizeComboForPlatform(rawCombo, platform)
+  const modifiers = cdpModifiers(combo)
+  const base = {
+    modifiers,
+    key: combo.key,
+    code: combo.code,
+    windowsVirtualKeyCode: combo.keyCode,
+    nativeVirtualKeyCode: combo.keyCode,
+  }
+  const printable = combo.key.length === 1 && !combo.ctrl && !combo.meta
+  const text = combo.key === 'Enter' ? '\r' : printable ? combo.key : undefined
+  const commands = macEditingCommands(combo, platform)
+  const down: cdp.CdpKeyEvent = {
+    ...base,
+    type: text !== undefined ? 'keyDown' : 'rawKeyDown',
+    ...(text !== undefined ? { text } : {}),
+    ...(commands.length > 0 ? { commands } : {}),
+  }
+  return [down, { ...base, type: 'keyUp' }]
+}
+
+/** Presses a combo through the trusted pipeline. Throws on CDP failure. */
+async function dispatchKeyCombo(contents: WebContents, combo: ParsedCombo): Promise<void> {
+  const [down, up] = buildKeyDispatchPlan(combo)
+  await cdp.dispatchKeyEvent(contents, down)
+  await cdp.dispatchKeyEvent(contents, up)
+}
+
+/**
+ * Post-action readback so the model sees the real effect (selection size,
+ * value length) instead of assuming the key "worked".
+ */
+async function activeElementState(contents: WebContents): Promise<Record<string, unknown>> {
+  const state = await execInPage(contents, readActiveElementState, []).catch(() => null)
+  return typeof state === 'object' && state !== null ? (state as Record<string, unknown>) : {}
+}
+
 // ---------------------------------------------------------------------------
 // Takeover
 // ---------------------------------------------------------------------------
@@ -513,28 +614,62 @@ async function executeToolInner(
     }
 
     case 'browser_type': {
+      const elementId = requireNum(params, 'elementId')
+      const text = requireStr(params, 'text')
+      const submit = params.submit === true
       const contents = session.requireTab().view.webContents
-      return unwrapPageResult(
-        await execInPage(contents, typeIntoElement, [
-          requireNum(params, 'elementId'),
-          requireStr(params, 'text'),
-          params.submit === true,
-        ])
-      )
+
+      // Native path: focus + select current content, then insert through the
+      // IME pipeline so the text REPLACES what's there — the only write path
+      // code editors (CodeMirror/Monaco) honor. The DOM selection set by the
+      // page function covers plain fields; the real select-all keystroke
+      // right after covers editors that track selection in their own model
+      // (their keymaps handle it synchronously, where DOM-selection sync is
+      // async and can lose a race with the insert). Falls back to the
+      // synthetic value-setter when CDP is unavailable.
+      unwrapPageResult(await execInPage(contents, focusElementForTyping, [elementId]))
+      try {
+        await dispatchKeyCombo(
+          contents,
+          parseKeyCombo(process.platform === 'darwin' ? 'Cmd+A' : 'Control+A')
+        )
+        await cdp.insertText(contents, text)
+      } catch {
+        return unwrapPageResult(
+          await execInPage(contents, typeIntoElement, [elementId, text, submit])
+        )
+      }
+      if (submit) {
+        await dispatchKeyCombo(contents, parseKeyCombo('Enter')).catch(() => {})
+      }
+      const state = await activeElementState(contents)
+      return { typed: true, replacedExisting: true, submitted: submit, ...state }
     }
 
     case 'browser_press_key': {
       const combo = parseKeyCombo(requireStr(params, 'key'))
       const contents = session.requireTab().view.webContents
-      return await execInPage(contents, pressKeyOnPage, [
-        combo.key,
-        combo.code,
-        combo.keyCode,
-        combo.ctrl,
-        combo.meta,
-        combo.shift,
-        combo.alt,
-      ])
+      try {
+        await dispatchKeyCombo(contents, combo)
+      } catch {
+        // CDP unavailable (debugger detached): synthetic DOM fallback. It
+        // cannot trigger default editing actions, so say so in the result.
+        const fallback = await execInPage(contents, pressKeyOnPage, [
+          combo.key,
+          combo.code,
+          combo.keyCode,
+          combo.ctrl,
+          combo.meta,
+          combo.shift,
+          combo.alt,
+        ])
+        return {
+          ...(typeof fallback === 'object' && fallback !== null ? fallback : {}),
+          note: 'Delivered as a synthetic page event; editing shortcuts may not take effect.',
+        }
+      }
+      const state = await activeElementState(contents)
+      return { pressed: requireStr(params, 'key'), ...state }
     }
 
     case 'browser_scroll': {