Skip to content

Commit d9295b2

Browse files
waleedlatif1waleedlatif1
committed
fix(browser): mark batched actions pending and stop batches on same-document URL changes
A batch now reports its outcome as pending before every action, since an action can dispatch input before it returns; a cancelled or timed-out batch therefore never reads as not started. It also stops after an action changes the tab URL within the document, not only on cross-document navigation.
1 parent 4548dfa commit d9295b2

4 files changed

Lines changed: 95 additions & 12 deletions

File tree

‎apps/desktop/src/main/browser-agent/driver.test.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3513,6 +3513,65 @@ describe('credential protection', () => {
35133513
})
35143514
})
35153515

3516+
it('stops a batch after an action changes the URL within the document', async () => {
3517+
const contents = await openPage()
3518+
respondWith(contents, {})
3519+
const url = contents.getURL()
3520+
const sendCommand = vi.mocked(contents.debugger.sendCommand)
3521+
const dispatch = sendCommand.getMockImplementation()
3522+
sendCommand.mockImplementation((method, params) => {
3523+
if (method === 'Input.dispatchMouseEvent' && toRecord(params).type === 'mouseReleased') {
3524+
vi.mocked(contents.getURL).mockReturnValue(`${url}#next`)
3525+
emitContentsEvent(contents, 'did-navigate-in-page')
3526+
}
3527+
return dispatch?.(method, params) ?? Promise.resolve(undefined)
3528+
})
3529+
3530+
const result = await driver.executeTool('chat-test', 'browser_batch', {
3531+
actions: [
3532+
{ tool: 'browser_click', args: { elementId: 0 } },
3533+
{ tool: 'browser_click', args: { elementId: 0 } },
3534+
],
3535+
})
3536+
3537+
expect(mousePresses(contents)).toBe(1)
3538+
expect(result).toMatchObject({
3539+
ok: true,
3540+
result: { completed: false, completedCount: 1, stoppedIndex: 1, stoppedBy: 'page-change' },
3541+
})
3542+
})
3543+
3544+
it('reports a batch cancelled during its first action as an unknown outcome', async () => {
3545+
const contents = await openPage()
3546+
respondWith(contents, {})
3547+
const sendCommand = vi.mocked(contents.debugger.sendCommand)
3548+
const dispatch = sendCommand.getMockImplementation()
3549+
sendCommand.mockImplementation((method, params) =>
3550+
method === 'Input.dispatchKeyEvent'
3551+
? new Promise(() => {})
3552+
: (dispatch?.(method, params) ?? Promise.resolve(undefined))
3553+
)
3554+
3555+
const pending = driver.executeTool(
3556+
'chat-test',
3557+
'browser_batch',
3558+
{
3559+
actions: [
3560+
{ tool: 'browser_press_key', args: { key: 'Enter' } },
3561+
{ tool: 'browser_click', args: { elementId: 0 } },
3562+
],
3563+
},
3564+
'batch-first-call'
3565+
)
3566+
await vi.waitFor(() => expect(cdpCalls(contents, 'Input.dispatchKeyEvent')).toHaveLength(1))
3567+
driver.cancelTool('chat-test', 'batch-first-call')
3568+
3569+
await expect(pending).resolves.toMatchObject({
3570+
ok: true,
3571+
result: { outcomeUnknown: true, doNotRetry: true },
3572+
})
3573+
})
3574+
35163575
it('reports a batch cancelled after an action ran as an unknown outcome', async () => {
35173576
const contents = await openPage()
35183577
respondWith(contents, {})

‎apps/desktop/src/main/browser-agent/driver.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3455,13 +3455,16 @@ async function executeToolInner(
34553455
for (const [index, action] of actions.entries()) {
34563456
let tab: ReturnType<typeof session.requireAutomationTab>
34573457
let epoch: number
3458+
let url: string
34583459
let snapshotValid: boolean
34593460
let result: unknown
3460-
// Once an action has run, a cancelled or timed-out batch must not read as never started.
3461-
if (index > 0) onActionOutcome?.({ status: 'pending' })
3461+
// An action may dispatch input before it returns, so a cancelled or timed-out batch must
3462+
// never read as not started once any action has begun.
3463+
onActionOutcome?.({ status: 'pending' })
34623464
try {
34633465
tab = session.requireAutomationTab()
34643466
epoch = navigationEpoch(tab.view.webContents)
3467+
url = tab.view.webContents.getURL()
34653468
snapshotValid = driverScopeState().snapshotTabId === tab.id
34663469
result = await executeToolInner(
34673470
action.tool,
@@ -3483,6 +3486,7 @@ async function executeToolInner(
34833486
if (
34843487
session.automationTab()?.id !== tab.id ||
34853488
navigationEpoch(tab.view.webContents) !== epoch ||
3489+
tab.view.webContents.getURL() !== url ||
34863490
(snapshotValid && driverScopeState().snapshotTabId !== tab.id)
34873491
) {
34883492
return stopped(

‎apps/sim/lib/mothership/generated/tool-catalog-v1.ts‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -401,22 +401,20 @@ export const BrowserBatch: ToolCatalogEntry = {
401401
route: 'client',
402402
mode: 'async',
403403
parameters: {
404-
type: 'object',
404+
additionalProperties: false,
405405
properties: {
406406
actions: {
407-
type: 'array',
408407
description:
409408
"Ordered list of 2–8 actions. Each names one action tool and gives exactly that tool's parameters in args, without observe.",
410409
items: {
411-
type: 'object',
410+
additionalProperties: false,
412411
properties: {
413412
args: {
414-
type: 'object',
415413
description:
416414
'That tool\'s own parameters, for example {"elementId": 12} for browser_click or {"key": "Enter"} for browser_press_key.',
415+
type: 'object',
417416
},
418417
tool: {
419-
type: 'string',
420418
description: 'The action tool to run.',
421419
enum: [
422420
'browser_click',
@@ -429,10 +427,15 @@ export const BrowserBatch: ToolCatalogEntry = {
429427
'browser_set_checked',
430428
'browser_hover',
431429
],
430+
type: 'string',
432431
},
433432
},
434433
required: ['tool', 'args'],
434+
type: 'object',
435435
},
436+
maxItems: 8,
437+
minItems: 2,
438+
type: 'array',
436439
},
437440
observe: {
438441
type: 'object',
@@ -448,6 +451,7 @@ export const BrowserBatch: ToolCatalogEntry = {
448451
},
449452
},
450453
required: ['actions'],
454+
type: 'object',
451455
},
452456
resultSchema: {
453457
type: 'object',
@@ -475,6 +479,12 @@ export const BrowserBatch: ToolCatalogEntry = {
475479
required: ['index', 'tool', 'result'],
476480
},
477481
},
482+
stoppedBy: {
483+
type: 'string',
484+
description:
485+
'failure when an action failed; page-change when an action changed the page and the rest were skipped.',
486+
enum: ['failure', 'page-change'],
487+
},
478488
stoppedIndex: {
479489
type: 'number',
480490
description:

‎apps/sim/lib/mothership/generated/tool-schemas-v1.ts‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -83,22 +83,20 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
8383
},
8484
browser_batch: {
8585
parameters: {
86-
type: 'object',
86+
additionalProperties: false,
8787
properties: {
8888
actions: {
89-
type: 'array',
9089
description:
9190
"Ordered list of 2–8 actions. Each names one action tool and gives exactly that tool's parameters in args, without observe.",
9291
items: {
93-
type: 'object',
92+
additionalProperties: false,
9493
properties: {
9594
args: {
96-
type: 'object',
9795
description:
9896
'That tool\'s own parameters, for example {"elementId": 12} for browser_click or {"key": "Enter"} for browser_press_key.',
97+
type: 'object',
9998
},
10099
tool: {
101-
type: 'string',
102100
description: 'The action tool to run.',
103101
enum: [
104102
'browser_click',
@@ -111,10 +109,15 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
111109
'browser_set_checked',
112110
'browser_hover',
113111
],
112+
type: 'string',
114113
},
115114
},
116115
required: ['tool', 'args'],
116+
type: 'object',
117117
},
118+
maxItems: 8,
119+
minItems: 2,
120+
type: 'array',
118121
},
119122
observe: {
120123
type: 'object',
@@ -130,6 +133,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
130133
},
131134
},
132135
required: ['actions'],
136+
type: 'object',
133137
},
134138
resultSchema: {
135139
type: 'object',
@@ -167,6 +171,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
167171
required: ['index', 'tool', 'result'],
168172
},
169173
},
174+
stoppedBy: {
175+
type: 'string',
176+
description:
177+
'failure when an action failed; page-change when an action changed the page and the rest were skipped.',
178+
enum: ['failure', 'page-change'],
179+
},
170180
stoppedIndex: {
171181
type: 'number',
172182
description:

0 commit comments

Comments
 (0)