fix(desktop): give the page back to the user mid-action, and stay awake until recovered results land

- A browser action now rechecks, right before each input it sends, whether the
  user is working in the page: between a batch's steps or during a click's
  target probes, the user wins and the action stops before its next input.
- The wait for the user is bounded inside the action's own deadline. A user
  who keeps working gets the model a clear "not run" instead of a timeout with
  an unknown outcome.
- The user's activity belongs to the tab they acted in. Once the agent drives
  another tab (the old one closed or replaced, the session reopened), it no
  longer holds anything up.
- Results a previous app run left behind keep the machine awake until Sim has
  them, like a running call's.

Author: waleedlatif1

apps/desktop/e2e/background-executor.spec.ts

@@ -769,6 +769,44 @@ test.describe('background executor', () => {
     })
   })
 
+  test('I: an action the user never stops working long enough for does not run', async () => {
+    app = (await launch(mkdtempSync(join(tmpdir(), 'sim-executor-i2-')))).app
+    const deviceId = await registeredDevice()
+    const opened = sim.issue(deviceId, CHAT_A, 'browser_open_url', {
+      url: `${sim.origin}/counter?chat=I2`,
+    })
+    const outline = ((await settled(opened)).data?.snapshot as { outline: string }).outline
+    const button = refFor(outline, 'Count visit')
+    const typeInAgentPage = () =>
+      app?.evaluate(({ webContents }) => {
+        const page = webContents
+          .getAllWebContents()
+          .find((contents) => contents.getURL().includes('/counter?chat=I2'))
+        page?.sendInputEvent({ type: 'keyDown', keyCode: 'Tab' })
+        page?.sendInputEvent({ type: 'keyUp', keyCode: 'Tab' })
+      })
+
+    await typeInAgentPage()
+    const click = sim.issue(deviceId, CHAT_A, 'browser_click', { elementId: button })
+    let typing = true
+    const keepTyping = (async () => {
+      while (typing) {
+        await typeInAgentPage()
+        await sleep(1_000)
+      }
+    })()
+
+    await check('I: the click reports it never ran, instead of timing out', async () => {
+      const completion = await settled(click, 60_000)
+      typing = false
+      await keepTyping
+      expect(completion.status).toBe('error')
+      expect(completion.message).toContain('Not run: the user kept working in this page')
+      expect(completion.data).not.toMatchObject({ outcomeUnknown: true })
+      expect(sim.hits.get('I2') ?? 0).toBe(0)
+    })
+  })
+
   test('J: the machine stays awake only while a chat has work running', async () => {
     app = (await launch(mkdtempSync(join(tmpdir(), 'sim-executor-j-')))).app
     await app.evaluate(({ powerSaveBlocker }) => {

apps/desktop/src/main/browser-agent/driver.ts

@@ -5096,6 +5096,12 @@ function withNotices(result: unknown): unknown {
 
 /** Quiet time after the user's last click or keystroke before the agent takes the page back. */
 const USER_TAKEOVER_IDLE_MS = 4_000
+/** The longest an action waits for the user to finish; past it, the action does not run. */
+const USER_TAKEOVER_MAX_WAIT_MS = 30_000
+const USER_KEPT_WORKING =
+  'Not run: the user kept working in this page, so this browser action never started and nothing was sent to the page. Ask the user whether they are done before acting in this page again.'
+const USER_TOOK_OVER_MID_ACTION =
+  'Stopped: the user started working in this page, so this browser action stopped before sending its next input. Earlier steps may have taken effect; inspect the page before continuing.'
 
 function isUserWorkingInPage(): boolean {
   const since = session.msSinceUserIntervention()
@@ -5107,14 +5113,25 @@ function isUserWorkingInPage(): boolean {
  * or scrolling in the tab the agent drives, the agent's next action waits, marked as needing
  * attention, and resumes once the user has left the page alone for a few seconds.
  */
-async function yieldToUser(toolCallId: string | undefined, signal: AbortSignal): Promise<void> {
+async function yieldToUser(
+  toolCallId: string | undefined,
+  signal: AbortSignal,
+  maxWaitMs: number
+): Promise<void> {
   session.setAutomationNeedsAttention(true)
   logger.info('Browser automation yielding to the user', { toolCallId })
-  while (isUserWorkingInPage()) {
-    await interruptibleSleep(250, signal)
-    if (signal.aborted) throw new ToolError('This browser action was cancelled.')
+  // Bounded well inside the action's own deadline, so a user who keeps working gets the model a
+  // clear "not run" rather than a timeout with an unknown outcome.
+  const deadline = Date.now() + maxWaitMs
+  try {
+    while (isUserWorkingInPage()) {
+      if (Date.now() >= deadline) throw new ToolError(USER_KEPT_WORKING)
+      await interruptibleSleep(250, signal)
+      if (signal.aborted) throw new ToolError('This browser action was cancelled.')
+    }
+  } finally {
+    session.setAutomationNeedsAttention(false)
   }
-  session.setAutomationNeedsAttention(false)
   logger.info('Browser automation resumed after the user stopped', { toolCallId })
 }
 
@@ -5214,20 +5231,32 @@ export async function executeTool(
           session.setAutomationActive(true)
         }
         try {
-          if (tool !== 'browser_request_takeover' && isUserWorkingInPage()) {
-            await yieldToUser(toolCallId, executionController.signal)
+          const yieldsToUser = tool !== 'browser_request_takeover'
+          const watchdogMs = browserToolWatchdogMs(tool, params)
+          if (yieldsToUser && isUserWorkingInPage()) {
+            await yieldToUser(
+              toolCallId,
+              executionController.signal,
+              watchdogMs === null
+                ? USER_TAKEOVER_MAX_WAIT_MS
+                : Math.min(USER_TAKEOVER_MAX_WAIT_MS, Math.floor(watchdogMs / 2))
+            )
           }
           const response = dialogResponse(tool, params)
           state.dialogResponse = response
             ? { contents: session.requireAutomationTab().view.webContents, response }
             : null
           const executionEpoch = ++state.toolExecutionEpoch
-          const watchdogMs = browserToolWatchdogMs(tool, params)
           const executionDeadline = watchdogMs === null ? undefined : Date.now() + watchdogMs
+          // Checked right before each input the action sends: a user who starts working in the
+          // page mid-action (between a batch's steps, during a click's target probes) wins it.
           const assertCurrentExecution = () => {
             if (state.toolExecutionEpoch !== executionEpoch) {
               throw new ToolError('This browser action expired before it could dispatch input.')
             }
+            if (yieldsToUser && isUserWorkingInPage()) {
+              throw new ToolError(USER_TOOK_OVER_MID_ACTION)
+            }
           }
           let actionOutcome: BrowserActionOutcome | undefined
           const execution = withPostActionObservation(

apps/desktop/src/main/browser-agent/session.ts

@@ -298,8 +298,11 @@ interface BrowserScopeState {
    */
   findingTabId: string | null
   findingRequestId: number | null
-  /** When the user last clicked, scrolled or typed in the tab the agent drives. */
-  userInterventionAt: number | null
+  /**
+   * When the user last clicked, scrolled or typed in the tab the agent drives, and in which tab:
+   * activity in a tab the agent no longer drives (closed, replaced) holds nothing up.
+   */
+  userIntervention: { tabId: string; at: number } | null
 }
 
 function createBrowserScopeState(): BrowserScopeState {
@@ -321,7 +324,7 @@ function createBrowserScopeState(): BrowserScopeState {
     automationNeedsAttention: false,
     findingTabId: null,
     findingRequestId: null,
-    userInterventionAt: null,
+    userIntervention: null,
   }
 }
 
@@ -2371,7 +2374,7 @@ function initializeTabView(
         return
       }
       const tab = tabs.find((entry) => entry.view.webContents === contents)
-      if (tab?.id === currentScope.automationTabId) noteUserIntervention()
+      if (tab && tab.id === currentScope.automationTabId) noteUserIntervention(tab.id)
       if (tab?.id === currentScope.activeTabId) {
         currentScope.visibleTabUserSelected = true
         if (mouse.type === 'mouseDown') tab.lastRealUserGestureAt = Date.now()
@@ -2491,9 +2494,10 @@ function initializeTabView(
       if (
         !isDispatchingAgentInput(contents) &&
         input.type === 'keyDown' &&
-        tab?.id === currentScope.automationTabId
+        tab &&
+        tab.id === currentScope.automationTabId
       ) {
-        noteUserIntervention()
+        noteUserIntervention(tab.id)
       }
       if (!isDispatchingAgentInput(contents) && tab?.id === currentScope.activeTabId) {
         currentScope.visibleTabUserSelected = true
@@ -3129,19 +3133,20 @@ export function claimActiveTabForUser(): AgentTab | null {
 }
 
 /** The user just acted in the page the agent drives; the agent yields until they stop. */
-function noteUserIntervention(): void {
-  currentScope.userInterventionAt = Date.now()
+function noteUserIntervention(tabId: string): void {
+  currentScope.userIntervention = { tabId, at: Date.now() }
 }
 
-/** How long ago the user last acted in the current scope's agent tab, or null if never. */
+/** How long ago the user last acted in the tab the agent drives now, or null if never. */
 export function msSinceUserIntervention(): number | null {
-  const at = currentScope.userInterventionAt
-  return at === null ? null : Math.max(0, Date.now() - at)
+  const intervention = currentScope.userIntervention
+  if (!intervention || intervention.tabId !== currentScope.automationTabId) return null
+  return Math.max(0, Date.now() - intervention.at)
 }
 
 /** Explicit hand-back after takeover lets automation resume in the same page. */
 export function returnAutomationTabToAgent(): void {
-  currentScope.userInterventionAt = null
+  currentScope.userIntervention = null
   if (currentScope.activeTabId === currentScope.automationTabId) {
     currentScope.visibleTabUserSelected = false
   }

apps/desktop/src/main/desktop-executor/executor.test.ts

@@ -152,16 +152,18 @@ function setup(options: { leaseRenewMs?: number; maxHeldCalls?: number } = {}) {
   const journal = new MemoryJournal()
   const runner = new FakeRunner()
   const onUnregistered = vi.fn()
+  const busy: boolean[] = []
   const executor = new DesktopExecutor({
     client: sim.client,
     journal,
     runner,
     leaseRenewMs: options.leaseRenewMs ?? 60_000,
     retryBaseMs: 5,
     onUnregistered,
+    onBusyChange: (value) => busy.push(value),
     ...(options.maxHeldCalls ? { maxHeldCalls: options.maxHeldCalls } : {}),
   })
-  return { sim, journal, runner, executor, onUnregistered }
+  return { sim, journal, runner, executor, onUnregistered, busy }
 }
 
 describe('claiming', () => {
@@ -431,6 +433,38 @@ describe('restarting', () => {
   })
 })
 
+describe('keeping the machine awake', () => {
+  it('is busy from the claim until Sim has the result', async () => {
+    const { sim, runner, executor, busy } = setup()
+    sim.inbox = [callItem('call-1', 'chat-a')]
+    await executor.reconcile()
+    await vi.waitFor(() => expect(runner.started).toEqual(['call-1']))
+    expect(busy).toEqual([true])
+
+    runner.finish('call-1')
+
+    await vi.waitFor(() => expect(busy).toEqual([true, false]))
+    expect(sim.completions).toHaveLength(1)
+  })
+
+  it('stays busy while a result a previous run left is still on its way to Sim', async () => {
+    const { sim, journal, executor, busy } = setup()
+    sim.completeErrors = [new DeviceRequestError(503, 'deploying')]
+    await journal.put({
+      toolCallId: 'result-1',
+      state: 'result',
+      executionToken: 't-result',
+      completion: DONE,
+    })
+
+    await executor.recover()
+    expect(busy).toEqual([true])
+
+    await vi.waitFor(() => expect(sim.completions).toHaveLength(1))
+    await vi.waitFor(() => expect(busy).toEqual([true, false]))
+  })
+})
+
 describe('registration', () => {
   it('asks to register again when Sim no longer recognizes the device', async () => {
     const { sim, executor, onUnregistered } = setup()

apps/desktop/src/main/desktop-executor/executor.ts

@@ -86,6 +86,8 @@ export class DesktopExecutor {
   private paused = false
   private disposed = false
   private busy = false
+  /** Results a previous app run left, still on their way to Sim. */
+  private recoveredInFlight = 0
 
   constructor(private readonly options: DesktopExecutorOptions) {}
 
@@ -131,7 +133,13 @@ export class DesktopExecutor {
         toolCallId: entry.toolCallId,
         state: entry.state,
       })
-      void this.deliver(entry.toolCallId, entry.executionToken, completion)
+      // Held awake like a running call: the result exists only on this machine until Sim has it.
+      this.recoveredInFlight += 1
+      this.updateBusy()
+      void this.deliver(entry.toolCallId, entry.executionToken, completion).finally(() => {
+        this.recoveredInFlight -= 1
+        this.updateBusy()
+      })
     }
   }
 
@@ -413,7 +421,7 @@ export class DesktopExecutor {
   }
 
   private updateBusy(): void {
-    const busy = this.held.size > 0
+    const busy = this.held.size > 0 || this.recoveredInFlight > 0
     if (busy === this.busy) return
     this.busy = busy
     this.options.onBusyChange?.(busy)

apps/desktop/src/main/desktop-executor/sleep-blocker.ts

@@ -16,6 +16,11 @@ interface SleepBlockerDeps {
   powerSaveBlocker: PowerSaveBlocker
 }
 
+/**
+ * Holds one `prevent-app-suspension` power blocker while the executor is busy and the switch is
+ * on, and releases it as soon as either stops being true. `setBusy` follows the executor;
+ * `refresh` follows the switch.
+ */
 export function createSleepBlocker(deps: SleepBlockerDeps) {
   let busy = false
   let blockerId: number | null = null