Merge remote-tracking branch 'origin/feat/plan-discovery-hillclimb' into feat/plan-discovery-hillclimb

Author: Sg312

apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx

@@ -90,6 +90,7 @@ function ComputerUseSettingsControls() {
               <span className='text-[var(--text-muted)] text-sm'>Allowed</span>
             ) : (
               <Chip
+                aria-label={`Open System Settings for ${permission === 'accessibility' ? 'Accessibility' : 'Screen Recording'}`}
                 disabled={pending || !status?.enabled}
                 onClick={() =>
                   void update(async () => setStatus(await bridge.requestPermission(permission)))

apps/sim/lib/mothership/tools/client/browser-tool-execution.test.ts

@@ -1062,38 +1062,47 @@ describe('executeBrowserToolOnClient', () => {
     await flush()
   })
 
-  it('compacts a large known result before unload-safe delivery', async () => {
-    mockExecuteBrowserTool.mockResolvedValue({
-      dataUrl: `data:image/jpeg;base64,${'A'.repeat(64 * 1024)}`,
-      url: 'https://example.com',
-    })
-    let finishReport: () => void = () => {}
-    mockReportCompletion.mockImplementation(
-      () =>
-        new Promise<void>((resolve) => {
-          finishReport = resolve
-        })
-    )
-    const sendBeacon = vi.mocked(navigator.sendBeacon)
-    const toolCallId = nextToolCallId()
+  it.each([false, true])(
+    'compacts a large result without overstating outcome certainty (unknown=%s)',
+    async (outcomeUnknown) => {
+      mockExecuteBrowserTool.mockResolvedValue({
+        dataUrl: `data:image/jpeg;base64,${'A'.repeat(64 * 1024)}`,
+        url: 'https://example.com',
+        outcomeUnknown,
+        doNotRetry: outcomeUnknown,
+      })
+      let finishReport: () => void = () => {}
+      mockReportCompletion.mockImplementation(
+        () =>
+          new Promise<void>((resolve) => {
+            finishReport = resolve
+          })
+      )
+      const sendBeacon = vi.mocked(navigator.sendBeacon)
+      const toolCallId = nextToolCallId()
 
-    executeBrowserToolOnClient(toolCallId, 'browser_screenshot', {})
-    await flush()
-    window.dispatchEvent(new Event('pagehide'))
-    await flush()
+      executeBrowserToolOnClient(toolCallId, 'browser_screenshot', {})
+      await flush()
+      window.dispatchEvent(new Event('pagehide'))
+      await flush()
 
-    const beaconPayload = sendBeacon.mock.calls[0]?.[1] as NodeBlob
-    const payload = JSON.parse(await beaconPayload.text())
-    expect(beaconPayload.size).toBeLessThanOrEqual(48 * 1024)
-    expect(payload).toMatchObject({
-      toolCallId,
-      status: 'success',
-      data: { resultOmittedDuringPageExit: true },
-    })
-    expect(payload.data.attachment).toBeUndefined()
-    finishReport()
-    await flush()
-  })
+      const beaconPayload = sendBeacon.mock.calls[0]?.[1] as NodeBlob
+      const payload = JSON.parse(await beaconPayload.text())
+      expect(beaconPayload.size).toBeLessThanOrEqual(48 * 1024)
+      expect(payload).toMatchObject({
+        toolCallId,
+        status: outcomeUnknown ? 'error' : 'success',
+        data: {
+          resultOmittedDuringPageExit: true,
+          ...(outcomeUnknown ? { outcomeUnknown: true, doNotRetry: true } : {}),
+        },
+      })
+      expect(payload.data.attachment).toBeUndefined()
+      if (outcomeUnknown) expect(payload.data.note).not.toContain('known terminal state')
+      finishReport()
+      await flush()
+    }
+  )
 
   it.each([['not accepted', () => false]])(
     'uses keepalive fallback for known success when the page-exit beacon %s',

apps/sim/lib/mothership/tools/client/computer-tool-unload.test.ts

@@ -164,64 +164,80 @@ describe('native computer completion during page unload', () => {
     }
   })
 
-  it('retains confirmed zero-dispatch failure after all delivery paths fail and releases it after replay acknowledgment', async () => {
-    native.execute.mockRejectedValueOnce(
-      new ComputerUseError({
-        code: 'activation_required',
-        message: 'Activate first.',
-        dispatchState: 'not_started',
+  it.each([false, true])(
+    'retains native input uncertainty through delivery failure and replay (unknown=%s)',
+    async (outcomeUnknown) => {
+      native.execute.mockRejectedValueOnce(
+        new ComputerUseError({
+          code: outcomeUnknown ? 'focus_changed' : 'activation_required',
+          message: outcomeUnknown
+            ? 'The exact editor is no longer focused; remaining input was not dispatched. Observe again.'
+            : 'Activate first.',
+          ...(outcomeUnknown ? {} : { dispatchState: 'not_started' as const }),
+        })
+      )
+      const fetch = vi
+        .fn<typeof globalThis.fetch>()
+        .mockResolvedValue(new Response(null, { status: 503 }))
+      vi.stubGlobal('fetch', fetch)
+      const bodies: NodeBlob[] = []
+      Object.defineProperty(navigator, 'sendBeacon', {
+        configurable: true,
+        value: (_url: string, body: NodeBlob) => {
+          bodies.push(body)
+          return false
+        },
       })
-    )
-    const fetch = vi
-      .fn<typeof globalThis.fetch>()
-      .mockResolvedValue(new Response(null, { status: 503 }))
-    vi.stubGlobal('fetch', fetch)
-    const bodies: NodeBlob[] = []
-    Object.defineProperty(navigator, 'sendBeacon', {
-      configurable: true,
-      value: (_url: string, body: NodeBlob) => {
-        bodies.push(body)
-        return false
-      },
-    })
-    const id = nextId()
-    const timestamp = new Date().toISOString()
-    const execution = executeComputerToolOnClient(id, { action: 'list_apps' }, timestamp)
-    await vi.runAllTimersAsync()
-    await execution
-    try {
+      const id = nextId()
+      const timestamp = new Date().toISOString()
+      const input = {
+        action: 'type_text' as const,
+        bundleId: 'com.example.Fixture',
+        snapshotId: 'before',
+        elementId: 'editor',
+        text: 'prefix then interruption',
+      }
+      const execution = executeComputerToolOnClient(id, input, timestamp)
+      await vi.runAllTimersAsync()
+      await execution
+      try {
+        window.dispatchEvent(new Event('pagehide'))
+        await flushMicrotasks()
+        expect(bodies).toHaveLength(1)
+        expect(copilotConfirmBodySchema.parse(JSON.parse(await bodies[0].text()))).toMatchObject({
+          toolCallId: id,
+          status: 'error',
+          data: {
+            code: outcomeUnknown ? 'focus_changed' : 'activation_required',
+            ...(outcomeUnknown ? {} : { dispatchState: 'not_started' }),
+            doNotRetry: outcomeUnknown,
+            outcomeUnknown,
+          },
+        })
+        const fallback = fetch.mock.calls.find(([, init]) => init?.keepalive)
+        expect(fallback?.[1]?.body).toBe(await bodies[0].text())
+        expect(native.cancel).not.toHaveBeenCalled()
+      } finally {
+        fetch.mockResolvedValue(new Response(null, { status: 200 }))
+        await executeComputerToolOnClient(id, input, timestamp)
+      }
+      const count = bodies.length
       window.dispatchEvent(new Event('pagehide'))
-      await flushMicrotasks()
-      expect(bodies).toHaveLength(1)
-      expect(copilotConfirmBodySchema.parse(JSON.parse(await bodies[0].text()))).toMatchObject({
+      expect(bodies).toHaveLength(count)
+      expect(
+        copilotConfirmBodySchema.parse(JSON.parse(String(fetch.mock.lastCall?.[1]?.body)))
+      ).toMatchObject({
         toolCallId: id,
         status: 'error',
         data: {
-          code: 'activation_required',
-          dispatchState: 'not_started',
-          doNotRetry: false,
-          outcomeUnknown: false,
+          ...(outcomeUnknown ? {} : { dispatchState: 'not_started' }),
+          outcomeUnknown,
+          doNotRetry: outcomeUnknown,
         },
       })
-      const fallback = fetch.mock.calls.find(([, init]) => init?.keepalive)
-      expect(fallback?.[1]?.body).toBe(await bodies[0].text())
-      expect(native.cancel).not.toHaveBeenCalled()
-    } finally {
-      fetch.mockResolvedValue(new Response(null, { status: 200 }))
-      await executeComputerToolOnClient(id, { action: 'list_apps' }, timestamp)
+      expect(native.execute).toHaveBeenCalledOnce()
     }
-    const count = bodies.length
-    window.dispatchEvent(new Event('pagehide'))
-    expect(bodies).toHaveLength(count)
-    expect(
-      copilotConfirmBodySchema.parse(JSON.parse(String(fetch.mock.lastCall?.[1]?.body)))
-    ).toMatchObject({
-      toolCallId: id,
-      status: 'error',
-      data: { dispatchState: 'not_started', outcomeUnknown: false },
-    })
-    expect(native.execute).toHaveBeenCalledOnce()
-  })
+  )
 
   it('reports an uncertain action and cancels only while native work is still pending', async () => {
     const nativeResult = createDeferred<ComputerUseResult>()

apps/sim/lib/mothership/tools/client/page-exit-completion.ts

@@ -35,7 +35,7 @@ export function compactCompletionForPageExit(
       ...(data.doNotRetry === true ? { doNotRetry: true } : {}),
       ...(data.sessionClosed === true ? { sessionClosed: true } : {}),
       resultOmittedDuringPageExit: true,
-      note: 'The action reached a known terminal state, but its full result was too large for unload-safe delivery. Do not repeat a side-effecting action. Take a fresh observation to recover current state.',
+      note: 'The full result was too large for unload-safe delivery. Do not repeat a side-effecting action. Take a fresh observation to recover current state.',
     },
   }
 }