|
1 | 1 | import type { ComputerUseResult, ComputerUseSnapshot } from '@sim/desktop-bridge/computer-use' |
2 | 2 | import { truncateAtCodePoint } from '@sim/utils/string' |
3 | 3 |
|
| 4 | +// Keep encoded pixels below the 10 MiB completion body cap, with room for state and its envelope. |
| 5 | +const MAX_SCREENSHOT_BASE64_BYTES = 9 * 1024 * 1024 |
4 | 6 | const MAX_STATE_BYTES = 44 * 1024 |
5 | 7 | const encoder = new TextEncoder() |
6 | 8 |
|
@@ -44,23 +46,32 @@ function nodeRow(node: ComputerUseSnapshot['nodes'][number]): string { |
44 | 46 |
|
45 | 47 | function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { |
46 | 48 | const { screenshot, nodes, windows: allWindows, ...state } = result |
| 49 | + const image = |
| 50 | + screenshot && screenshot.base64.length <= MAX_SCREENSHOT_BASE64_BYTES ? screenshot : undefined |
| 51 | + const screenshotOmitted = Boolean(screenshot && !image) |
47 | 52 | const selectedWindow = allWindows.find((candidate) => candidate.windowId === state.windowId) |
48 | 53 | const windows = [ |
49 | 54 | ...(selectedWindow ? [selectedWindow] : []), |
50 | 55 | ...allWindows.filter((candidate) => candidate.windowId !== state.windowId).slice(0, 19), |
51 | 56 | ].map((window) => ({ ...window, title: truncateAtCodePoint(window.title, 256, '') })) |
52 | 57 | const model = { |
53 | 58 | ...state, |
| 59 | + ...(screenshotOmitted |
| 60 | + ? { |
| 61 | + screenshotError: |
| 62 | + 'Screenshot omitted because its encoded size exceeds completion limits. Accessibility state is still available.', |
| 63 | + } |
| 64 | + : {}), |
54 | 65 | windows, |
55 | 66 | omittedWindowCount: allWindows.length - windows.length, |
56 | 67 | accessibilityTree: '', |
57 | 68 | omittedNodeCount: nodes.length, |
58 | 69 | nodeTextLimit: 512, |
59 | | - ...(screenshot |
| 70 | + ...(image |
60 | 71 | ? { |
61 | | - screenshotSize: { width: screenshot.width, height: screenshot.height }, |
| 72 | + screenshotSize: { width: image.width, height: image.height }, |
62 | 73 | content: selectedWindow |
63 | | - ? `Screenshot of window ${state.windowId}. Coordinate actions use window-local macOS points. Convert encoded image coordinates: x = imageX * ${selectedWindow.width} / ${screenshot.width}; y = imageY * ${selectedWindow.height} / ${screenshot.height}. Accessibility screenRect coordinates are global points; subtract window origin (${selectedWindow.x}, ${selectedWindow.y}). Prefer element IDs.` |
| 74 | + ? `Screenshot of window ${state.windowId}. Coordinate actions use window-local macOS points. Convert encoded image coordinates: x = imageX * ${selectedWindow.width} / ${image.width}; y = imageY * ${selectedWindow.height} / ${image.height}. Accessibility screenRect coordinates are global points; subtract window origin (${selectedWindow.x}, ${selectedWindow.y}). Prefer element IDs.` |
64 | 75 | : 'Screenshot coordinate mapping is unavailable. Use accessibility element IDs or take a fresh state before acting.', |
65 | 76 | } |
66 | 77 | : {}), |
@@ -145,9 +156,14 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { |
145 | 156 | state.truncated || |
146 | 157 | selected.size < nodes.length || |
147 | 158 | windows.length < allWindows.length || |
148 | | - clipped, |
149 | | - observations: screenshot |
150 | | - ? [{ name: 'Computer screenshot', mediaType: screenshot.mimeType, data: screenshot.base64 }] |
151 | | - : undefined, |
| 159 | + clipped || |
| 160 | + screenshotOmitted, |
| 161 | + ...(image |
| 162 | + ? { |
| 163 | + observations: [ |
| 164 | + { name: 'Computer screenshot', mediaType: image.mimeType, data: image.base64 }, |
| 165 | + ], |
| 166 | + } |
| 167 | + : {}), |
152 | 168 | } |
153 | 169 | } |
0 commit comments