Skip to content

Commit 39c9fe6

Browse files
Sg312Sg312
committed
Browser shortcuts
1 parent 46a62bd commit 39c9fe6

9 files changed

Lines changed: 240 additions & 1 deletion

File tree

‎apps/desktop/src/main/browser-agent/session.test.ts‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@ interface MockView {
1313
setPermissionRequestHandler: ReturnType<typeof vi.fn>
1414
setPermissionCheckHandler: ReturnType<typeof vi.fn>
1515
}
16+
on: ReturnType<typeof vi.fn>
1617
setWindowOpenHandler: ReturnType<typeof vi.fn>
1718
loadURL: ReturnType<typeof vi.fn>
19+
focus: ReturnType<typeof vi.fn>
1820
isDestroyed: ReturnType<typeof vi.fn>
1921
setBackgroundThrottling: ReturnType<typeof vi.fn>
2022
capturePage: ReturnType<typeof vi.fn>
@@ -75,6 +77,78 @@ describe('browser-agent session', () => {
7577
expect(session.listTabs()[0]).toMatchObject({ tabId: first.id, active: true })
7678
})
7779

80+
it('normalizes browser shortcuts to Command on macOS and Control elsewhere', () => {
81+
const input = {
82+
type: 'keyDown',
83+
key: 'l',
84+
isAutoRepeat: false,
85+
isComposing: false,
86+
shift: false,
87+
control: false,
88+
alt: false,
89+
meta: true,
90+
}
91+
92+
expect(session.browserShortcutForInput(input, 'darwin')).toBe('focus-omnibox')
93+
expect(session.browserShortcutForInput(input, 'win32')).toBeNull()
94+
expect(session.browserShortcutForInput({ ...input, meta: false, control: true }, 'win32')).toBe(
95+
'focus-omnibox'
96+
)
97+
expect(session.browserShortcutForInput({ ...input, key: 't' }, 'darwin')).toBe('new-tab')
98+
expect(session.browserShortcutForInput({ ...input, key: 'w' }, 'darwin')).toBe('close-tab')
99+
expect(
100+
session.browserShortcutForInput({ ...input, key: 't', shift: true }, 'darwin')
101+
).toBeNull()
102+
})
103+
104+
it('handles browser shortcuts from a focused native tab', () => {
105+
session.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 })
106+
const first = session.requireTab()
107+
const firstContents = (first.view as unknown as MockView).webContents
108+
const beforeInput = firstContents.on.mock.calls.find(
109+
([eventName]) => eventName === 'before-input-event'
110+
)?.[1] as
111+
| ((event: { preventDefault: () => void }, input: Record<string, unknown>) => void)
112+
| undefined
113+
const event = { preventDefault: vi.fn() }
114+
const input = {
115+
type: 'keyDown',
116+
key: 'l',
117+
isAutoRepeat: false,
118+
isComposing: false,
119+
shift: false,
120+
control: process.platform !== 'darwin',
121+
alt: false,
122+
meta: process.platform === 'darwin',
123+
}
124+
125+
beforeInput?.(event, input)
126+
expect(event.preventDefault).toHaveBeenCalled()
127+
expect(win.webContents.focus).toHaveBeenCalled()
128+
expect(win.webContents.send).toHaveBeenLastCalledWith('browser-agent:focus-omnibox', 'select')
129+
130+
beforeInput?.(event, { ...input, key: 't' })
131+
expect(session.listTabs()).toHaveLength(2)
132+
expect(win.webContents.send).toHaveBeenLastCalledWith('browser-agent:focus-omnibox', 'clear')
133+
134+
const second = session.activeTab()
135+
expect(second).not.toBeNull()
136+
const secondContents = (second?.view as unknown as MockView).webContents
137+
const secondBeforeInput = secondContents.on.mock.calls.find(
138+
([eventName]) => eventName === 'before-input-event'
139+
)?.[1] as
140+
| ((event: { preventDefault: () => void }, input: Record<string, unknown>) => void)
141+
| undefined
142+
secondBeforeInput?.(event, { ...input, key: 'w' })
143+
expect(session.listTabs()).toHaveLength(1)
144+
expect(firstContents.focus).toHaveBeenCalled()
145+
146+
beforeInput?.(event, { ...input, key: 'w' })
147+
expect(session.listTabs()).toHaveLength(1)
148+
expect(session.listTabs()[0].tabId).not.toBe(first.id)
149+
expect(win.webContents.send).toHaveBeenLastCalledWith('browser-agent:focus-omnibox', 'clear')
150+
})
151+
78152
it('only disables hidden-page throttling while browser automation is active', () => {
79153
const tab = session.ensureTab()
80154
const contents = (tab.view as unknown as MockView).webContents
@@ -167,6 +241,20 @@ describe('browser-agent session', () => {
167241
expect(removeChildView).toHaveBeenCalledWith(tab.view)
168242
})
169243

244+
it('creates one real default tab when the browser panel becomes visible', () => {
245+
expect(session.listTabs()).toHaveLength(0)
246+
247+
session.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 })
248+
249+
expect(session.listTabs()).toHaveLength(1)
250+
expect(session.getTabsState().activeTabId).toBe(session.listTabs()[0].tabId)
251+
252+
const firstTabId = session.listTabs()[0].tabId
253+
session.closeTab(firstTabId)
254+
expect(session.listTabs()).toHaveLength(1)
255+
expect(session.listTabs()[0].tabId).not.toBe(firstTabId)
256+
})
257+
170258
it('clears a stale attachment without touching a destroyed host window', () => {
171259
const tab = session.ensureTab()
172260
session.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 })

‎apps/desktop/src/main/browser-agent/session.ts‎

Lines changed: 88 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import {
2+
type BrowserOmniboxFocusMode,
23
type BrowserPanelBounds,
34
type BrowserPanelSnapshot,
45
type BrowserTabState,
@@ -8,7 +9,7 @@ import {
89
} from '@sim/browser-protocol'
910
import { createLogger } from '@sim/logger'
1011
import { getErrorMessage } from '@sim/utils/errors'
11-
import type { BrowserWindow, Session, WebContents } from 'electron'
12+
import type { BrowserWindow, Input, Session, WebContents } from 'electron'
1213
import { nativeTheme, WebContentsView } from 'electron'
1314
import { registerAgentWebContents } from '@/main/browser-agent/registry'
1415

@@ -49,6 +50,45 @@ export interface AgentSessionEvents {
4950
const PANEL_LEASE_TTL_MS = 2_500
5051
const PANEL_LEASE_CHECK_MS = 1_000
5152

53+
export type BrowserShortcut = 'focus-omnibox' | 'new-tab' | 'close-tab'
54+
55+
type BrowserShortcutInput = Pick<
56+
Input,
57+
'type' | 'key' | 'isAutoRepeat' | 'isComposing' | 'shift' | 'control' | 'alt' | 'meta'
58+
>
59+
60+
/**
61+
* Resolves browser-level shortcuts using Command on macOS and Control
62+
* elsewhere. Modified/composing/repeated keystrokes stay with the page.
63+
*/
64+
export function browserShortcutForInput(
65+
input: BrowserShortcutInput,
66+
platform: NodeJS.Platform = process.platform
67+
): BrowserShortcut | null {
68+
if (
69+
input.type !== 'keyDown' ||
70+
input.isAutoRepeat ||
71+
input.isComposing ||
72+
input.shift ||
73+
input.alt
74+
) {
75+
return null
76+
}
77+
const primaryModifier = platform === 'darwin' ? input.meta : input.control
78+
if (!primaryModifier) return null
79+
80+
switch (input.key.toLowerCase()) {
81+
case 'l':
82+
return 'focus-omnibox'
83+
case 't':
84+
return 'new-tab'
85+
case 'w':
86+
return 'close-tab'
87+
default:
88+
return null
89+
}
90+
}
91+
5292
const tabs: AgentTab[] = []
5393
let activeTabId: string | null = null
5494
let nextTabId = 1
@@ -96,6 +136,13 @@ function configureAgentPartition(ses: Session): void {
96136
})
97137
}
98138

139+
function focusRendererOmnibox(mode: BrowserOmniboxFocusMode): void {
140+
const win = getMainWindow()
141+
if (!win || win.isDestroyed()) return
142+
win.webContents.focus()
143+
win.webContents.send('browser-agent:focus-omnibox', mode)
144+
}
145+
99146
function createTabView(): WebContentsView {
100147
const view = new WebContentsView({
101148
webPreferences: {
@@ -137,6 +184,34 @@ function createTabView(): WebContentsView {
137184
contents.on('will-prevent-unload', (event) => {
138185
event.preventDefault()
139186
})
187+
contents.on('before-input-event', (event, input) => {
188+
const shortcut = browserShortcutForInput(input)
189+
if (!shortcut) return
190+
191+
event.preventDefault()
192+
if (shortcut === 'focus-omnibox') {
193+
focusRendererOmnibox('select')
194+
return
195+
}
196+
if (shortcut === 'new-tab') {
197+
if (listTabs().length < MAX_BROWSER_TABS) {
198+
addTab()
199+
focusRendererOmnibox('clear')
200+
}
201+
return
202+
}
203+
204+
const tab = tabs.find((entry) => entry.view === view)
205+
if (!tab) return
206+
const closingLastTab = listTabs().length === 1
207+
closeTab(tab.id)
208+
const active = activeTab()
209+
if (closingLastTab || !active || !active.view.webContents.getURL()) {
210+
focusRendererOmnibox('clear')
211+
} else {
212+
active.view.webContents.focus()
213+
}
214+
})
140215

141216
events?.onTabCreated(contents)
142217
return view
@@ -307,6 +382,12 @@ function layout(): void {
307382
/** Renderer-reported panel rect (null = panel hidden/unmounted). */
308383
export function setPanelBounds(bounds: BrowserPanelBounds | null): void {
309384
panelBounds = bounds
385+
// A visible browser resource always represents one open browser window.
386+
// Materialize its initial about:blank tab before layout so the tab strip,
387+
// omnibox, and native session never disagree about an empty state.
388+
if (bounds !== null && !hasSession()) {
389+
ensureTab()
390+
}
310391
if (bounds === null) {
311392
panelOccluded = false
312393
panelSnapshotGeneration++
@@ -401,6 +482,12 @@ export function closeTab(tabId: string): void {
401482
events?.onActiveTabChanged(active.view.webContents)
402483
}
403484
}
485+
// Closing the last tab must not leave a visible browser resource with an
486+
// empty strip. Replace it with a fresh New tab, matching normal browser UI.
487+
if (!hasSession() && panelBounds !== null) {
488+
addTab()
489+
return
490+
}
404491
events?.onTabsChanged()
405492
if (!hasSession()) {
406493
events?.onSessionClosed()

‎apps/desktop/src/preload/index.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type {
2+
BrowserOmniboxFocusMode,
23
BrowserPageState,
34
BrowserPanelAction,
45
BrowserPanelBounds,
@@ -94,6 +95,13 @@ const api: SimDesktopApi = {
9495
setTheme: (theme: BrowserTheme): void => {
9596
ipcRenderer.send('browser-agent:set-theme', theme)
9697
},
98+
onFocusOmnibox: (callback: (mode: BrowserOmniboxFocusMode) => void): (() => void) => {
99+
const listener = (_event: unknown, mode: BrowserOmniboxFocusMode) => callback(mode)
100+
ipcRenderer.on('browser-agent:focus-omnibox', listener)
101+
return () => {
102+
ipcRenderer.removeListener('browser-agent:focus-omnibox', listener)
103+
}
104+
},
97105
onPanelSnapshot: (callback: (snapshot: BrowserPanelSnapshot) => void): (() => void) => {
98106
const listener = (_event: unknown, snapshot: BrowserPanelSnapshot) => callback(snapshot)
99107
ipcRenderer.on('browser-agent:panel-snapshot', listener)

‎apps/desktop/src/test/electron-mock.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ function createWebContentsMock() {
148148
getTitle: vi.fn(() => 'Example'),
149149
loadURL: vi.fn(() => Promise.resolve()),
150150
reload: vi.fn(),
151+
focus: vi.fn(),
151152
close: vi.fn(),
152153
isDestroyed: vi.fn(() => false),
153154
isLoading: vi.fn(() => false),
@@ -205,6 +206,7 @@ export class BrowserWindow {
205206
setZoomLevel: vi.fn(),
206207
getZoomLevel: vi.fn(() => 0),
207208
executeJavaScript: vi.fn(() => Promise.resolve(true)),
209+
focus: vi.fn(),
208210
send: vi.fn(),
209211
setWindowOpenHandler: vi.fn(),
210212
isDevToolsOpened: vi.fn(() => false),

‎apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.tsx‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { Button, ChipInput, Tooltip } from '@sim/emcn'
66
import { ArrowLeft, ArrowRight, Cursor, RefreshCw, Search } from '@sim/emcn/icons'
77
import { useTheme } from 'next-themes'
88
import {
9+
onBrowserOmniboxFocus,
910
reportBrowserPanelBounds,
1011
reportBrowserTheme,
1112
sendBrowserPanelAction,
@@ -55,6 +56,8 @@ export function BrowserSession() {
5556
const hostRef = useRef<HTMLDivElement>(null)
5657
const urlInputRef = useRef<HTMLInputElement>(null)
5758
const panelOccluded = useBrowserPanelOcclusion(hostRef)
59+
const pageUrlRef = useRef(pageState?.url ?? '')
60+
pageUrlRef.current = pageState?.url ?? ''
5861
/** Non-null while the user is editing the URL bar; otherwise it mirrors the page. */
5962
const [urlDraft, setUrlDraft] = useState<string | null>(null)
6063

@@ -64,6 +67,27 @@ export function BrowserSession() {
6467
}
6568
}, [theme])
6669

70+
useEffect(() => {
71+
let focusRaf: number | null = null
72+
const unsubscribe = onBrowserOmniboxFocus((mode) => {
73+
setUrlDraft(mode === 'clear' ? '' : pageUrlRef.current)
74+
if (focusRaf !== null) {
75+
cancelAnimationFrame(focusRaf)
76+
}
77+
focusRaf = requestAnimationFrame(() => {
78+
focusRaf = null
79+
urlInputRef.current?.focus()
80+
urlInputRef.current?.select()
81+
})
82+
})
83+
return () => {
84+
unsubscribe()
85+
if (focusRaf !== null) {
86+
cancelAnimationFrame(focusRaf)
87+
}
88+
}
89+
}, [])
90+
6791
// Keep the embedded view glued to the host rect without forcing layout on
6892
// every animation frame. ResizeObserver covers panel transitions/resizes;
6993
// viewport resize and captured scroll cover position changes. A one-second

‎apps/sim/lib/browser-agent/transport.test.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { beforeEach, describe, expect, it, vi } from 'vitest'
22

33
const {
4+
onFocusOmnibox,
45
onPanelSnapshot,
56
setPageState,
67
setPanelBounds,
@@ -11,6 +12,7 @@ const {
1112
setTabsState,
1213
setTabsSupported,
1314
} = vi.hoisted(() => ({
15+
onFocusOmnibox: vi.fn(),
1416
onPanelSnapshot: vi.fn(),
1517
setPageState: vi.fn(),
1618
setPanelBounds: vi.fn(),
@@ -27,6 +29,7 @@ vi.mock('@/lib/desktop', () => ({
2729
browserAgent: {
2830
executeTool: vi.fn(),
2931
getTabsState: vi.fn(async () => ({ tabs: [], activeTabId: null })),
32+
onFocusOmnibox,
3033
onPageState: vi.fn(),
3134
onPanelSnapshot,
3235
onSessionStatus: vi.fn(),
@@ -53,6 +56,7 @@ vi.mock('@/stores/browser-session/store', () => ({
5356

5457
import {
5558
initBrowserAgentTransport,
59+
onBrowserOmniboxFocus,
5660
reportBrowserPanelBounds,
5761
reportBrowserPanelOcclusion,
5862
reportBrowserTheme,
@@ -100,4 +104,13 @@ describe('browser panel transport', () => {
100104

101105
expect(setTheme.mock.calls).toEqual([['dark'], ['light'], ['system']])
102106
})
107+
108+
it('subscribes to native omnibox focus requests', () => {
109+
const unsubscribe = vi.fn()
110+
const callback = vi.fn()
111+
onFocusOmnibox.mockReturnValue(unsubscribe)
112+
113+
expect(onBrowserOmniboxFocus(callback)).toBe(unsubscribe)
114+
expect(onFocusOmnibox).toHaveBeenCalledWith(callback)
115+
})
103116
})

‎apps/sim/lib/browser-agent/transport.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
* subagent is never offered.
1414
*/
1515
import type {
16+
BrowserOmniboxFocusMode,
1617
BrowserPageState,
1718
BrowserPanelAction,
1819
BrowserPanelBounds,
@@ -115,6 +116,13 @@ export function reportBrowserTheme(theme: BrowserTheme): void {
115116
bridge()?.setTheme?.(theme)
116117
}
117118

119+
/** Subscribes to native browser shortcuts that target the renderer omnibox. */
120+
export function onBrowserOmniboxFocus(
121+
callback: (mode: BrowserOmniboxFocusMode) => void
122+
): () => void {
123+
return bridge()?.onFocusOmnibox?.(callback) ?? (() => {})
124+
}
125+
118126
/**
119127
* Reports the panel's current rect (viewport CSS pixels), or null when the
120128
* panel is hidden/unmounted. The embedded view tracks this rect.

0 commit comments

Comments
 (0)