Skip to content

Commit 657c8de

Browse files
committed
fix(desktop): bound native requests and completion images
1 parent 85751b0 commit 657c8de

11 files changed

Lines changed: 3889 additions & 12018 deletions

File tree

‎apps/desktop/src/main/computer-use/native-client.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,21 @@ describe('native computer use transport', () => {
7474
expect(reset).not.toHaveBeenCalled()
7575
})
7676

77+
it.each(['\u0000', '漢'])(
78+
'rejects requests above the native byte limit without disrupting the helper (%j)',
79+
async (character) => {
80+
const { client, reset } = helper(`let count = 0; ${reader}
81+
count++;
82+
process.stdout.write(JSON.stringify({id:request.id,result:{...${status},accessibility:count===2}})+'\\n');
83+
});`)
84+
await client.request('status', {})
85+
const params = { text: character.repeat(character === '\u0000' ? 32_000 : 48_000) }
86+
await expect(client.request('type_text', params)).rejects.toThrow('exceeds 128 KiB')
87+
await expect(client.request('status', {})).resolves.toMatchObject({ accessibility: true })
88+
expect(reset).not.toHaveBeenCalled()
89+
}
90+
)
91+
7792
it('rejects malformed output and can start a fresh helper afterward', async () => {
7893
const { client, reset } = helper(`${reader}
7994
process.stdout.write(request.method==='bad'?'not json\\n':JSON.stringify({id:request.id,result:${status}})+'\\n');

‎apps/desktop/src/main/computer-use/native-client.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ import {
66
} from '@sim/desktop-bridge/computer-use'
77
import { generateId } from '@sim/utils/id'
88

9+
// ComputerUse.swift bounds each request line independently of the decoded argument schema.
10+
const MAX_REQUEST_BYTES = 128 * 1024
911
const MAX_FRAME_BYTES = 16 * 1024 * 1024
1012
const REQUEST_TIMEOUT_MS = 30_000
1113

@@ -38,14 +40,21 @@ export class NativeComputerUseClient implements ComputerUseNativeClient {
3840
await this.stopping
3941
if (generation !== this.generation) throw new Error('Computer Use stopped.')
4042
if (this.pending.size >= 16) return Promise.reject(new Error('Computer Use is busy.'))
41-
const child = this.start()
4243
const id = generateId()
44+
const request = JSON.stringify({ id, method, params })
45+
if (Buffer.byteLength(request, 'utf8') > MAX_REQUEST_BYTES)
46+
throw new ComputerUseError({
47+
code: 'invalid_request',
48+
message: 'Computer Use request exceeds 128 KiB.',
49+
dispatchState: 'not_started',
50+
})
51+
const child = this.start()
4352
return new Promise((resolve, reject) => {
4453
const timer = setTimeout(() => {
4554
this.stopWithError(new Error('Computer Use timed out; inspect the app before retrying.'))
4655
}, REQUEST_TIMEOUT_MS)
4756
this.pending.set(id, { resolve, reject, timer })
48-
child.stdin.write(`${JSON.stringify({ id, method, params })}\n`, (error) => {
57+
child.stdin.write(`${request}\n`, (error) => {
4958
if (error && this.child === child)
5059
this.stopWithError(new Error('Computer Use helper disconnected.'))
5160
})

‎apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,40 @@ describe('native computer model projection', () => {
134134
}
135135
)
136136

137+
it.each([false, true])(
138+
'omits an oversized screenshot while retaining usable state and dispatch certainty (nested=%s)',
139+
(nested) => {
140+
const state = snapshot()
141+
state.screenshot = {
142+
base64: Buffer.alloc(8 * 1024 * 1024).toString('base64'),
143+
mimeType: 'image/png',
144+
width: 3840,
145+
height: 2160,
146+
}
147+
const result = computerToolResultForModel(nested ? sequence(state) : state)
148+
const body = JSON.stringify({
149+
toolCallId: 'large-screenshot',
150+
status: 'success',
151+
message: 'Observed',
152+
data: result,
153+
})
154+
expect(Buffer.byteLength(body)).toBeLessThan(10 * 1024 * 1024)
155+
expect(result).not.toHaveProperty('observations')
156+
const projected = 'observation' in result ? result.observation : result
157+
expect(projected).toMatchObject({
158+
screenshotError: expect.stringContaining('omitted'),
159+
accessibilityTree: expect.stringContaining('editor AXTextArea'),
160+
})
161+
expect(projected).not.toHaveProperty('screenshotSize')
162+
if (nested)
163+
expect(result).toMatchObject({
164+
dispatched: true,
165+
verified: false,
166+
sequence: { completedSteps: 1 },
167+
})
168+
}
169+
)
170+
137171
it('does not split an emoji when bounding text for the model', () => {
138172
const state = snapshot()
139173
state.nodes[0] = {

‎apps/sim/lib/mothership/tools/client/computer-tool-result.ts‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import type { ComputerUseResult, ComputerUseSnapshot } from '@sim/desktop-bridge/computer-use'
22
import { truncateAtCodePoint } from '@sim/utils/string'
33

4+
// Keep encoded pixels below the 10 MiB completion body cap, with room for state and its envelope.
5+
const MAX_SCREENSHOT_BASE64_BYTES = 9 * 1024 * 1024
46
const MAX_STATE_BYTES = 44 * 1024
57
const encoder = new TextEncoder()
68

@@ -44,23 +46,32 @@ function nodeRow(node: ComputerUseSnapshot['nodes'][number]): string {
4446

4547
function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) {
4648
const { screenshot, nodes, windows: allWindows, ...state } = result
49+
const image =
50+
screenshot && screenshot.base64.length <= MAX_SCREENSHOT_BASE64_BYTES ? screenshot : undefined
51+
const screenshotOmitted = Boolean(screenshot && !image)
4752
const selectedWindow = allWindows.find((candidate) => candidate.windowId === state.windowId)
4853
const windows = [
4954
...(selectedWindow ? [selectedWindow] : []),
5055
...allWindows.filter((candidate) => candidate.windowId !== state.windowId).slice(0, 19),
5156
].map((window) => ({ ...window, title: truncateAtCodePoint(window.title, 256, '') }))
5257
const model = {
5358
...state,
59+
...(screenshotOmitted
60+
? {
61+
screenshotError:
62+
'Screenshot omitted because its encoded size exceeds completion limits. Accessibility state is still available.',
63+
}
64+
: {}),
5465
windows,
5566
omittedWindowCount: allWindows.length - windows.length,
5667
accessibilityTree: '',
5768
omittedNodeCount: nodes.length,
5869
nodeTextLimit: 512,
59-
...(screenshot
70+
...(image
6071
? {
61-
screenshotSize: { width: screenshot.width, height: screenshot.height },
72+
screenshotSize: { width: image.width, height: image.height },
6273
content: selectedWindow
63-
? `Screenshot of window ${state.windowId}. Coordinate actions use window-local macOS points. Convert encoded image coordinates: x = imageX * ${selectedWindow.width} / ${screenshot.width}; y = imageY * ${selectedWindow.height} / ${screenshot.height}. Accessibility screenRect coordinates are global points; subtract window origin (${selectedWindow.x}, ${selectedWindow.y}). Prefer element IDs.`
74+
? `Screenshot of window ${state.windowId}. Coordinate actions use window-local macOS points. Convert encoded image coordinates: x = imageX * ${selectedWindow.width} / ${image.width}; y = imageY * ${selectedWindow.height} / ${image.height}. Accessibility screenRect coordinates are global points; subtract window origin (${selectedWindow.x}, ${selectedWindow.y}). Prefer element IDs.`
6475
: 'Screenshot coordinate mapping is unavailable. Use accessibility element IDs or take a fresh state before acting.',
6576
}
6677
: {}),
@@ -145,9 +156,14 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) {
145156
state.truncated ||
146157
selected.size < nodes.length ||
147158
windows.length < allWindows.length ||
148-
clipped,
149-
observations: screenshot
150-
? [{ name: 'Computer screenshot', mediaType: screenshot.mimeType, data: screenshot.base64 }]
151-
: undefined,
159+
clipped ||
160+
screenshotOmitted,
161+
...(image
162+
? {
163+
observations: [
164+
{ name: 'Computer screenshot', mediaType: image.mimeType, data: image.base64 },
165+
],
166+
}
167+
: {}),
152168
}
153169
}

0 commit comments

Comments
 (0)