feat(browser): batch planned page actions and keep snapshot ids after refused clicks (#8214)

* fix(browser): keep snapshot ids valid after an observed action is refused

An observed click refused before dispatch (for example, covered by an overlay)
never starts its observation, yet it invalidated every id in the snapshot, so
the agent could not click the overlay's own close button without re-observing.
A failed observed action with no dispatched outcome now only cancels an
in-flight capture, which clears the refs itself when it starts.

* feat(browser): run planned page actions in order with browser_batch

The desktop runs 2-8 single-page interactions from one authorized call, in
order, stopping at the first failed action or after an action navigates,
switches tabs, or invalidates the snapshot's element ids. A batch stopped by a
failure reports an error status; one stopped by a page change does not. Once
an action has run, a cancelled or timed-out batch reports an unknown outcome
so the agent does not repeat it.

* fix(browser): mark batched actions pending and stop batches on same-document URL changes

A batch now reports its outcome as pending before every action, since an
action can dispatch input before it returns; a cancelled or timed-out batch
therefore never reads as not started. It also stops after an action changes
the tab URL within the document, not only on cross-document navigation.

Author: waleedlatif1

apps/desktop/src/main/browser-agent/driver.test.ts

@@ -1,4 +1,5 @@
 import { BROWSER_TOOL_QUEUE_WAIT_TIMEOUT_MS } from '@sim/browser-protocol'
+import { toRecord } from '@sim/utils/object'
 import type { MenuItemConstructorOptions, WebContents } from 'electron'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
@@ -2097,13 +2098,20 @@ describe('credential protection', () => {
       for (const [fnName, value] of Object.entries(replies)) {
         if (isPageCall(expression, fnName)) return Promise.resolve(value)
       }
-      if (isPageCall(expression, 'clickElement')) {
-        return Promise.resolve({ dispatched: false, x: 24, y: 48, element: 'Test' })
-      }
+      if (isPageCall(expression, 'clickElement')) return Promise.resolve(CLICK_TARGET)
       return Promise.resolve(undefined)
     })
   }
 
+  function mousePresses(contents: Awaited<ReturnType<typeof openPage>>): number {
+    return cdpCalls(contents, 'Input.dispatchMouseEvent').filter(
+      ([, params]) => toRecord(params).type === 'mousePressed'
+    ).length
+  }
+
+  /** What the page reports for an ordinary click target before native dispatch. */
+  const CLICK_TARGET = { dispatched: false, x: 24, y: 48, element: 'Test' }
+
   function cdpCalls(contents: Awaited<ReturnType<typeof openPage>>, method: string): unknown[][] {
     return vi
       .mocked(contents.debugger.sendCommand)
@@ -3425,6 +3433,217 @@ describe('credential protection', () => {
     expect(second).not.toMatchObject({ result: { notices: expect.anything() } })
   })
 
+  it('runs batched actions in order and returns each result', async () => {
+    const contents = await openPage()
+    respondWith(contents, {})
+
+    const result = await driver.executeTool('chat-test', 'browser_batch', {
+      actions: [
+        { tool: 'browser_click', args: { elementId: 0 } },
+        { tool: 'browser_click', args: { elementId: 0 } },
+      ],
+    })
+
+    expect(result).toMatchObject({
+      ok: true,
+      result: {
+        completed: true,
+        completedCount: 2,
+        results: [
+          { index: 0, tool: 'browser_click', result: { dispatched: true } },
+          { index: 1, tool: 'browser_click', result: { dispatched: true } },
+        ],
+      },
+    })
+  })
+
+  it('stops a batch at the first failed action and keeps earlier results', async () => {
+    const contents = await openPage()
+    vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => {
+      if (!isPageCall(expression, 'clickElement')) return Promise.resolve(undefined)
+      return Promise.resolve(
+        mousePresses(contents) > 0 ? { error: 'obstructed', blocker: 'IMG' } : CLICK_TARGET
+      )
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_batch', {
+      actions: [
+        { tool: 'browser_click', args: { elementId: 0 } },
+        { tool: 'browser_click', args: { elementId: 0 } },
+        { tool: 'browser_click', args: { elementId: 0 } },
+      ],
+    })
+
+    expect(mousePresses(contents)).toBe(1)
+    expect(result).toMatchObject({
+      ok: true,
+      result: {
+        completed: false,
+        completedCount: 1,
+        stoppedIndex: 1,
+        stoppedBy: 'failure',
+        error: expect.stringContaining('covered by IMG'),
+      },
+    })
+  })
+
+  it('stops a batch after an action navigates the page', async () => {
+    const contents = await openPage()
+    respondWith(contents, {})
+    const sendCommand = vi.mocked(contents.debugger.sendCommand)
+    const dispatch = sendCommand.getMockImplementation()
+    sendCommand.mockImplementation((method, params) => {
+      if (method === 'Input.dispatchMouseEvent' && toRecord(params).type === 'mouseReleased') {
+        emitContentsEvent(contents, 'did-navigate')
+      }
+      return dispatch?.(method, params) ?? Promise.resolve(undefined)
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_batch', {
+      actions: [
+        { tool: 'browser_click', args: { elementId: 0 } },
+        { tool: 'browser_click', args: { elementId: 0 } },
+      ],
+    })
+
+    expect(mousePresses(contents)).toBe(1)
+    expect(result).toMatchObject({
+      ok: true,
+      result: { completed: false, completedCount: 1, stoppedIndex: 1, stoppedBy: 'page-change' },
+    })
+  })
+
+  it('stops a batch after an action changes the URL within the document', async () => {
+    const contents = await openPage()
+    respondWith(contents, {})
+    const url = contents.getURL()
+    const sendCommand = vi.mocked(contents.debugger.sendCommand)
+    const dispatch = sendCommand.getMockImplementation()
+    sendCommand.mockImplementation((method, params) => {
+      if (method === 'Input.dispatchMouseEvent' && toRecord(params).type === 'mouseReleased') {
+        vi.mocked(contents.getURL).mockReturnValue(`${url}#next`)
+        emitContentsEvent(contents, 'did-navigate-in-page')
+      }
+      return dispatch?.(method, params) ?? Promise.resolve(undefined)
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_batch', {
+      actions: [
+        { tool: 'browser_click', args: { elementId: 0 } },
+        { tool: 'browser_click', args: { elementId: 0 } },
+      ],
+    })
+
+    expect(mousePresses(contents)).toBe(1)
+    expect(result).toMatchObject({
+      ok: true,
+      result: { completed: false, completedCount: 1, stoppedIndex: 1, stoppedBy: 'page-change' },
+    })
+  })
+
+  it('reports a batch cancelled during its first action as an unknown outcome', async () => {
+    const contents = await openPage()
+    respondWith(contents, {})
+    const sendCommand = vi.mocked(contents.debugger.sendCommand)
+    const dispatch = sendCommand.getMockImplementation()
+    sendCommand.mockImplementation((method, params) =>
+      method === 'Input.dispatchKeyEvent'
+        ? new Promise(() => {})
+        : (dispatch?.(method, params) ?? Promise.resolve(undefined))
+    )
+
+    const pending = driver.executeTool(
+      'chat-test',
+      'browser_batch',
+      {
+        actions: [
+          { tool: 'browser_press_key', args: { key: 'Enter' } },
+          { tool: 'browser_click', args: { elementId: 0 } },
+        ],
+      },
+      'batch-first-call'
+    )
+    await vi.waitFor(() => expect(cdpCalls(contents, 'Input.dispatchKeyEvent')).toHaveLength(1))
+    driver.cancelTool('chat-test', 'batch-first-call')
+
+    await expect(pending).resolves.toMatchObject({
+      ok: true,
+      result: { outcomeUnknown: true, doNotRetry: true },
+    })
+  })
+
+  it('reports a batch cancelled after an action ran as an unknown outcome', async () => {
+    const contents = await openPage()
+    respondWith(contents, {})
+    const sendCommand = vi.mocked(contents.debugger.sendCommand)
+    const dispatch = sendCommand.getMockImplementation()
+    sendCommand.mockImplementation((method, params) =>
+      method === 'Input.dispatchKeyEvent'
+        ? new Promise(() => {})
+        : (dispatch?.(method, params) ?? Promise.resolve(undefined))
+    )
+
+    const pending = driver.executeTool(
+      'chat-test',
+      'browser_batch',
+      {
+        actions: [
+          { tool: 'browser_click', args: { elementId: 0 } },
+          { tool: 'browser_press_key', args: { key: 'Enter' } },
+        ],
+      },
+      'batch-call'
+    )
+    await vi.waitFor(() => expect(cdpCalls(contents, 'Input.dispatchKeyEvent')).toHaveLength(1))
+    driver.cancelTool('chat-test', 'batch-call')
+
+    await expect(pending).resolves.toMatchObject({
+      ok: true,
+      result: { outcomeUnknown: true, doNotRetry: true },
+    })
+  })
+
+  it('rejects batches that name non-action tools or observe per action', async () => {
+    await openPage()
+
+    const navigation = await driver.executeTool('chat-test', 'browser_batch', {
+      actions: [
+        { tool: 'browser_navigate', args: { url: 'https://example.com' } },
+        { tool: 'browser_click', args: { elementId: 0 } },
+      ],
+    })
+    const observed = await driver.executeTool('chat-test', 'browser_batch', {
+      actions: [
+        { tool: 'browser_click', args: { elementId: 0, observe: {} } },
+        { tool: 'browser_click', args: { elementId: 0 } },
+      ],
+    })
+
+    expect(navigation).toMatchObject({
+      ok: false,
+      error: expect.stringContaining('Batch action 0'),
+    })
+    expect(observed).toMatchObject({ ok: false, error: expect.stringContaining('cannot observe') })
+  })
+
+  it('keeps element ids valid when an observed action is refused before dispatch', async () => {
+    const contents = await openPage()
+    respondWith(contents, { clickElement: { error: 'obstructed', blocker: 'IMG' } })
+
+    const refused = await driver.executeTool('chat-test', 'browser_click', {
+      elementId: 0,
+      observe: {},
+    })
+    respondWith(contents, {})
+    const retried = await driver.executeTool('chat-test', 'browser_click', { elementId: 0 })
+
+    expect(refused).toEqual({
+      ok: false,
+      error: expect.stringContaining('That element is covered by IMG'),
+    })
+    expect(retried).toMatchObject({ ok: true, result: { dispatched: true } })
+  })
+
   it('invalidates element ids when the active tab changes', async () => {
     await openPage()
     await driver.executeTool('chat-test', 'browser_open_tab', {})