From 0a788a756ba8e63c27b8466f049a29c9eaf7b4da Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Mon, 3 Aug 2026 17:05:40 +0300 Subject: [PATCH 1/2] Support semicolon page shortcuts --- src/chrome/src/agent/agent.js | 3 +- src/chrome/src/agent/permission-gate.js | 3 +- src/chrome/src/agent/planner.js | 4 +- src/chrome/src/agent/tools.js | 8 ++-- src/chrome/src/content/content.js | 3 +- src/firefox/src/agent/permission-gate.js | 3 +- src/firefox/src/agent/planner.js | 4 +- src/firefox/src/agent/tools.js | 8 ++-- src/firefox/src/content/content.js | 3 +- test/fixtures/run.mjs | 29 ++++++++++++ test/run.js | 57 +++++++++++++++++++++--- 11 files changed, 102 insertions(+), 23 deletions(-) diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index 5ceefe2d2..4bc5d6222 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -18161,7 +18161,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const key = args.key; const repeatRaw = Number(args.repeat ?? 1); const repeat = Math.max(1, Math.min(3, Number.isFinite(repeatRaw) ? Math.floor(repeatRaw) : 1)); - const SUPPORTED_KEYS = ['Escape', 'Tab', 'Enter', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight']; + const SUPPORTED_KEYS = ['Escape', 'Tab', 'Enter', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight', ';']; if (!SUPPORTED_KEYS.includes(key)) { return { success: false, @@ -18185,6 +18185,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ArrowUp: { code: 'ArrowUp', windowsVirtualKeyCode: 38 }, ArrowRight: { code: 'ArrowRight', windowsVirtualKeyCode: 39 }, ArrowDown: { code: 'ArrowDown', windowsVirtualKeyCode: 40 }, + ';': { code: 'Semicolon', windowsVirtualKeyCode: 186 }, }[key]; for (let i = 0; i < repeat; i++) { diff --git a/src/chrome/src/agent/permission-gate.js b/src/chrome/src/agent/permission-gate.js index d6bd483df..706f7b713 100644 --- a/src/chrome/src/agent/permission-gate.js +++ b/src/chrome/src/agent/permission-gate.js @@ -180,7 +180,8 @@ const TOOL_CAPABILITY = { * via chrome.downloads → DOWNLOAD. These are not model-exposed tools. * - set_field: TYPE normally, but CLICK when submit:true (pressing Enter * submits the form — a TYPE grant must not authorize a submit). - * - press_keys: Enter can submit/activate → CLICK; Tab/Escape are benign. + * - press_keys: Enter and page shortcuts such as ; can mutate → CLICK; + * Tab/Escape are benign. */ export function capabilityFor(name, args) { args = args || {}; diff --git a/src/chrome/src/agent/planner.js b/src/chrome/src/agent/planner.js index b217e45b6..fd58f0e8b 100644 --- a/src/chrome/src/agent/planner.js +++ b/src/chrome/src/agent/planner.js @@ -69,7 +69,7 @@ Rules: memory: scratchpad_write, progress_update, progress_read schedule: schedule_task (future/recurring work the user explicitly asked for), schedule_resume (pause CURRENT run blocked on external event) finish: done -- press_keys supports only unmodified Escape, Tab, Enter, and arrow keys. Never plan Ctrl/Cmd/Alt/Shift combinations or browser UI shortcuts. To select one literal page-text match, plan find_text instead of Ctrl/Cmd+F. Each find_text call replaces the previous selection and does not open browser Find UI; never plan sequential calls as simultaneous highlights. +- press_keys supports only unmodified Escape, Tab, Enter, arrow keys, and ; (semicolon, for page shortcuts such as Gmail Expand all). Never plan Ctrl/Cmd/Alt/Shift combinations or browser UI shortcuts. To select one literal page-text match, plan find_text instead of Ctrl/Cmd+F. Each find_text call replaces the previous selection and does not open browser Find UI; never plan sequential calls as simultaneous highlights. - For repeated same-kind UI mutations (for example following many users), plan visible UI first with bounded batches, verification, progress_update, and wait_for_stable pacing; do not plan one huge same-shape click/tool batch. - Do not invent a prerequisite to discover a raw identifier (email address, account ID, username, or similar) when the target UI provides a name-based contact/entity picker and the user already supplied a human-readable name. Plan to use the picker first. Inspect surrounding pages or messages for the raw identifier only if the picker fails, returns multiple ambiguous matches, or the user explicitly asked for the identifier itself. - Set confidence from 0.0 to 1.0 for how clear and safe this plan is. Use 0.90+ only when the task, page state, and next steps are straightforward; use lower scores for ambiguity, destructive changes, payments, credentials, bulk mutations, or uncertain page state. @@ -128,7 +128,7 @@ Rules: - schedule_task supports one-shot times and fixed-minute intervals only. Calendar/cron recurrence such as monthly is unsupported: classify it as clarify, explain the limitation in localized.summary, and ask for a one-shot time or fixed interval. Never convert calendar recurrence into an approximate interval. - Canonical summary, steps, and risks must be English. localized fields must use the requested wbLocale. - For execute, keep the compact plan to 1–4 steps. For plan_only, provide 2–8 useful steps. For respond and clarify, steps may be empty. -- press_keys supports only unmodified Escape, Tab, Enter, and arrow keys. Never plan modifier combinations or browser UI shortcuts; use find_text to select one page-text match instead of Ctrl/Cmd+F. Each call replaces the previous selection and cannot create simultaneous highlights or browser Find UI. +- press_keys supports only unmodified Escape, Tab, Enter, arrow keys, and ; (semicolon, for page shortcuts such as Gmail Expand all). Never plan modifier combinations or browser UI shortcuts; use find_text to select one page-text match instead of Ctrl/Cmd+F. Each call replaces the previous selection and cannot create simultaneous highlights or browser Find UI. - Do not invent URLs, credentials, tool names, or facts.`; export function normalizePlannerLocale(value) { diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.js index 94fb26a5b..689c7b020 100644 --- a/src/chrome/src/agent/tools.js +++ b/src/chrome/src/agent/tools.js @@ -306,11 +306,11 @@ export const AGENT_TOOLS = [ type: 'function', function: { name: 'press_keys', - description: 'Press one unmodified keyboard key. Supports only Escape, Tab, Enter, ArrowUp, ArrowDown, ArrowLeft, and ArrowRight. Ctrl/Cmd/Alt/Shift combinations and browser shortcuts such as Ctrl+F are not supported. Use find_text to select one page-text match.', + description: 'Press one unmodified keyboard key. Supports Escape, Tab, Enter, ArrowUp, ArrowDown, ArrowLeft, ArrowRight, and ; (semicolon, for page shortcuts such as Gmail Expand all). Ctrl/Cmd/Alt/Shift combinations and browser shortcuts such as Ctrl+F are not supported. Use find_text to select one page-text match.', parameters: { type: 'object', properties: { - key: { type: 'string', enum: ['Escape', 'Tab', 'Enter', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight'], description: 'Key to press.' }, + key: { type: 'string', enum: ['Escape', 'Tab', 'Enter', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight', ';'], description: 'Key to press.' }, repeat: { type: 'number', description: 'How many times to press the key (default: 1, max: 3).' }, }, required: ['key'], @@ -1469,7 +1469,7 @@ Available tools: - schedule_task: Create a one-shot or fixed-minute-interval task only when the user explicitly asks for future scheduled work. It does not support calendar/cron recurrence; never approximate monthly recurrence. Prefer URL targets for repeatable automations; current_tab is strict and fails if the tab changes. - get_selection: Get highlighted text - find_text: Select one literal page-text match instead of Ctrl/Cmd+F. Each call replaces the previous selection; it does not open browser Find UI or keep multiple terms highlighted. -- press_keys: Press only unmodified Escape/Tab/Enter/arrows. Modifier combinations and browser shortcuts are unsupported. +- press_keys: Press only unmodified Escape/Tab/Enter/arrows or ; (semicolon). Modifier combinations and browser shortcuts are unsupported. - new_tab: Open a background reference tab; the current run stays on its original tab - clarify: Pause and ask the user a question. Use ONLY for material ambiguity that you cannot resolve by reading the page (e.g. "my API key" on a site with multiple plugins that each have one). Unanswered clarifies auto-select options[0] after the timeout (default 60s) with source=timeout (not high-risk approval); Settings Instant yields source=auto (intentional auto-approve — continue). Put the safe/default first. Do NOT use to confirm correct actions; do NOT call before every step. Budget 1-2 per run, max. - done: Signal task completion @@ -1809,7 +1809,7 @@ TOOLS — use only these: - get_accessibility_tree: PREFERRED read. Flat-text tree with roles, names, and stable ref_ids. Use filter:"visible" by default. - click_ax({ref_id}) / set_checked({ref_id, checked}) / type_ax({ref_id, text}) / set_field({ref_id, text, submit}): act on nodes by ref_id. set_field is preferred for text fields; set_checked is required for native checkboxes. - read_page: prose fallback for long articles. get_window_info: inspect browser window/viewport size. scroll, navigate({url}), go_back()/go_forward(): walk the run tab's history. new_tab({url}) only opens a background reference tab and never retargets the run. -- get_interactive_elements: legacy indexed element list (use when the tree misses elements). click({text}) / type_text({text}) / press_keys({key}): legacy fallbacks. press_keys supports only unmodified Escape/Tab/Enter/arrows, never Ctrl/Cmd/Alt/Shift combinations or browser shortcuts. +- get_interactive_elements: legacy indexed element list (use when the tree misses elements). click({text}) / type_text({text}) / press_keys({key}): legacy fallbacks. press_keys supports only unmodified Escape/Tab/Enter/arrows or ; (semicolon), never Ctrl/Cmd/Alt/Shift combinations or browser shortcuts. - extract_data: tables/headings/images/links. get_selection: read highlighted text. find_text({text}): select one literal page-text match; each call replaces the previous selection and never creates simultaneous highlights or browser Find UI. read_pdf: read a PDF. - wait_for_element({selector}) / wait_for_stable({quietMs}): wait for an element / for the page to go quiet after an action. - schedule_resume({after_seconds|run_at, reason, resume_instruction}): terminal durable pause for this current task. diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js index d36a1ed2d..b9ceb3057 100644 --- a/src/chrome/src/content/content.js +++ b/src/chrome/src/content/content.js @@ -1935,7 +1935,7 @@ const key = params?.key; const repeatRaw = Number(params?.repeat ?? 1); const repeat = Math.max(1, Math.min(3, Number.isFinite(repeatRaw) ? Math.floor(repeatRaw) : 1)); - const SUPPORTED_KEYS = ['Escape', 'Tab', 'Enter', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight']; + const SUPPORTED_KEYS = ['Escape', 'Tab', 'Enter', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight', ';']; if (!SUPPORTED_KEYS.includes(key)) { return { success: false, @@ -1959,6 +1959,7 @@ ArrowUp: { code: 'ArrowUp', keyCode: 38 }, ArrowRight: { code: 'ArrowRight', keyCode: 39 }, ArrowDown: { code: 'ArrowDown', keyCode: 40 }, + ';': { code: 'Semicolon', keyCode: 186 }, }[key]; const target = (document.activeElement && document.activeElement !== document.body) ? document.activeElement diff --git a/src/firefox/src/agent/permission-gate.js b/src/firefox/src/agent/permission-gate.js index 1f45795cf..a957fb4bc 100644 --- a/src/firefox/src/agent/permission-gate.js +++ b/src/firefox/src/agent/permission-gate.js @@ -167,7 +167,8 @@ const TOOL_CAPABILITY = { * via downloads → DOWNLOAD. These are not model-exposed tools. * - set_field: TYPE normally, but CLICK when submit:true (pressing Enter * submits the form — a TYPE grant must not authorize a submit). - * - press_keys: Enter can submit/activate → CLICK; Tab/Escape are benign. + * - press_keys: Enter and page shortcuts such as ; can mutate → CLICK; + * Tab/Escape are benign. */ export function capabilityFor(name, args) { args = args || {}; diff --git a/src/firefox/src/agent/planner.js b/src/firefox/src/agent/planner.js index b217e45b6..fd58f0e8b 100644 --- a/src/firefox/src/agent/planner.js +++ b/src/firefox/src/agent/planner.js @@ -69,7 +69,7 @@ Rules: memory: scratchpad_write, progress_update, progress_read schedule: schedule_task (future/recurring work the user explicitly asked for), schedule_resume (pause CURRENT run blocked on external event) finish: done -- press_keys supports only unmodified Escape, Tab, Enter, and arrow keys. Never plan Ctrl/Cmd/Alt/Shift combinations or browser UI shortcuts. To select one literal page-text match, plan find_text instead of Ctrl/Cmd+F. Each find_text call replaces the previous selection and does not open browser Find UI; never plan sequential calls as simultaneous highlights. +- press_keys supports only unmodified Escape, Tab, Enter, arrow keys, and ; (semicolon, for page shortcuts such as Gmail Expand all). Never plan Ctrl/Cmd/Alt/Shift combinations or browser UI shortcuts. To select one literal page-text match, plan find_text instead of Ctrl/Cmd+F. Each find_text call replaces the previous selection and does not open browser Find UI; never plan sequential calls as simultaneous highlights. - For repeated same-kind UI mutations (for example following many users), plan visible UI first with bounded batches, verification, progress_update, and wait_for_stable pacing; do not plan one huge same-shape click/tool batch. - Do not invent a prerequisite to discover a raw identifier (email address, account ID, username, or similar) when the target UI provides a name-based contact/entity picker and the user already supplied a human-readable name. Plan to use the picker first. Inspect surrounding pages or messages for the raw identifier only if the picker fails, returns multiple ambiguous matches, or the user explicitly asked for the identifier itself. - Set confidence from 0.0 to 1.0 for how clear and safe this plan is. Use 0.90+ only when the task, page state, and next steps are straightforward; use lower scores for ambiguity, destructive changes, payments, credentials, bulk mutations, or uncertain page state. @@ -128,7 +128,7 @@ Rules: - schedule_task supports one-shot times and fixed-minute intervals only. Calendar/cron recurrence such as monthly is unsupported: classify it as clarify, explain the limitation in localized.summary, and ask for a one-shot time or fixed interval. Never convert calendar recurrence into an approximate interval. - Canonical summary, steps, and risks must be English. localized fields must use the requested wbLocale. - For execute, keep the compact plan to 1–4 steps. For plan_only, provide 2–8 useful steps. For respond and clarify, steps may be empty. -- press_keys supports only unmodified Escape, Tab, Enter, and arrow keys. Never plan modifier combinations or browser UI shortcuts; use find_text to select one page-text match instead of Ctrl/Cmd+F. Each call replaces the previous selection and cannot create simultaneous highlights or browser Find UI. +- press_keys supports only unmodified Escape, Tab, Enter, arrow keys, and ; (semicolon, for page shortcuts such as Gmail Expand all). Never plan modifier combinations or browser UI shortcuts; use find_text to select one page-text match instead of Ctrl/Cmd+F. Each call replaces the previous selection and cannot create simultaneous highlights or browser Find UI. - Do not invent URLs, credentials, tool names, or facts.`; export function normalizePlannerLocale(value) { diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.js index e28b95003..7e26f1753 100644 --- a/src/firefox/src/agent/tools.js +++ b/src/firefox/src/agent/tools.js @@ -306,11 +306,11 @@ export const AGENT_TOOLS = [ type: 'function', function: { name: 'press_keys', - description: 'Press one unmodified keyboard key. Supports only Escape, Tab, Enter, ArrowUp, ArrowDown, ArrowLeft, and ArrowRight. Ctrl/Cmd/Alt/Shift combinations and browser shortcuts such as Ctrl+F are not supported. Use find_text to select one page-text match. Firefox dispatches synthetic events, so native controls may not react on every site.', + description: 'Press one unmodified keyboard key. Supports Escape, Tab, Enter, ArrowUp, ArrowDown, ArrowLeft, ArrowRight, and ; (semicolon, for page shortcuts such as Gmail Expand all). Ctrl/Cmd/Alt/Shift combinations and browser shortcuts such as Ctrl+F are not supported. Use find_text to select one page-text match. Firefox dispatches synthetic events, so native controls may not react on every site.', parameters: { type: 'object', properties: { - key: { type: 'string', enum: ['Escape', 'Tab', 'Enter', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight'], description: 'Key to press.' }, + key: { type: 'string', enum: ['Escape', 'Tab', 'Enter', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight', ';'], description: 'Key to press.' }, repeat: { type: 'number', description: 'How many times to press the key (default: 1, max: 3).' }, }, required: ['key'], @@ -1340,7 +1340,7 @@ Available tools: - schedule_task: Create a one-shot or fixed-minute-interval task only when the user explicitly asks for future scheduled work. It does not support calendar/cron recurrence; never approximate monthly recurrence. Prefer URL targets for repeatable automations; current_tab is strict and fails if the tab changes. - get_selection: Get highlighted text - find_text: Select one literal page-text match instead of Ctrl/Cmd+F. Each call replaces the previous selection; it does not open browser Find UI or keep multiple terms highlighted. -- press_keys: Press only unmodified Escape/Tab/Enter/arrows. Modifier combinations and browser shortcuts are unsupported. +- press_keys: Press only unmodified Escape/Tab/Enter/arrows or ; (semicolon). Modifier combinations and browser shortcuts are unsupported. - new_tab: Open a background reference tab; the current run stays on its original tab - clarify: Pause and ask the user a question. Use ONLY for material ambiguity that you cannot resolve by reading the page (e.g. "my API key" on a site with multiple plugins that each have one). Unanswered clarifies auto-select options[0] after the timeout (default 60s) with source=timeout (not high-risk approval); Settings Instant yields source=auto (intentional auto-approve — continue). Put the safe/default first. Do NOT use to confirm correct actions; do NOT call before every step. Budget 1-2 per run, max. - done: Signal task completion @@ -1564,7 +1564,7 @@ TOOLS — use only these: - get_accessibility_tree: PREFERRED read. Flat-text tree with roles, names, and stable ref_ids. Use filter:"visible" by default. - click_ax({ref_id}) / set_checked({ref_id, checked}) / type_ax({ref_id, text}) / set_field({ref_id, text, submit}): act on nodes by ref_id. set_field is preferred for text fields; set_checked is required for native checkboxes. - read_page: prose fallback for long articles. get_window_info: inspect browser window/viewport size. scroll, navigate({url}), go_back()/go_forward(): walk the run tab's history. new_tab({url}) only opens a background reference tab and never retargets the run. -- get_interactive_elements: legacy indexed element list (use when the tree misses elements). click({text}) / type_text({text}) / press_keys({key}): legacy fallbacks. press_keys supports only unmodified Escape/Tab/Enter/arrows, never Ctrl/Cmd/Alt/Shift combinations or browser shortcuts. +- get_interactive_elements: legacy indexed element list (use when the tree misses elements). click({text}) / type_text({text}) / press_keys({key}): legacy fallbacks. press_keys supports only unmodified Escape/Tab/Enter/arrows or ; (semicolon), never Ctrl/Cmd/Alt/Shift combinations or browser shortcuts. - extract_data: tables/headings/images/links. get_selection: read highlighted text. find_text({text}): select one literal page-text match; each call replaces the previous selection and never creates simultaneous highlights or browser Find UI. read_pdf: read a PDF. - wait_for_element({selector}) / wait_for_stable({quietMs}): wait for an element / for the page to go quiet after an action. - schedule_resume({after_seconds|run_at, reason, resume_instruction}): terminal durable pause for this current task. diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js index f148e654f..45c064a78 100644 --- a/src/firefox/src/content/content.js +++ b/src/firefox/src/content/content.js @@ -2129,7 +2129,7 @@ const key = params?.key; const repeatRaw = Number(params?.repeat ?? 1); const repeat = Math.max(1, Math.min(3, Number.isFinite(repeatRaw) ? Math.floor(repeatRaw) : 1)); - const SUPPORTED_KEYS = ['Escape', 'Tab', 'Enter', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight']; + const SUPPORTED_KEYS = ['Escape', 'Tab', 'Enter', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight', ';']; if (!SUPPORTED_KEYS.includes(key)) { return { success: false, @@ -2154,6 +2154,7 @@ ArrowUp: { code: 'ArrowUp', keyCode: 38 }, ArrowRight: { code: 'ArrowRight', keyCode: 39 }, ArrowDown: { code: 'ArrowDown', keyCode: 40 }, + ';': { code: 'Semicolon', keyCode: 186 }, }[key]; const target = (document.activeElement && document.activeElement !== document.body) ? document.activeElement diff --git a/test/fixtures/run.mjs b/test/fixtures/run.mjs index 69ad31d8a..63a79948e 100644 --- a/test/fixtures/run.mjs +++ b/test/fixtures/run.mjs @@ -1590,6 +1590,35 @@ for (const browserKind of ['chrome', 'firefox']) { } }); + test(`press_keys (${browserKind}): semicolon dispatches a page shortcut event`, async (page) => { + await setupContentFixture(page, 'trusted-click-fallback.html', browserKind); + await page.evaluate(() => { + window.__semicolonShortcutEvents = []; + document.addEventListener('keydown', (event) => { + if (event.key !== ';') return; + window.__semicolonShortcutEvents.push({ + key: event.key, + code: event.code, + keyCode: event.keyCode, + }); + }); + }); + + const result = await call(page, 'press_keys', { key: ';' }); + if (result?.success !== true || result?.dispatched !== true || result?.key !== ';') { + throw new Error(`semicolon should dispatch successfully, got: ${JSON.stringify(result)}`); + } + const events = await page.evaluate(() => window.__semicolonShortcutEvents); + if ( + events?.length !== 1 + || events[0]?.key !== ';' + || events[0]?.code !== 'Semicolon' + || events[0]?.keyCode !== 186 + ) { + throw new Error(`semicolon shortcut metadata mismatch: ${JSON.stringify(events)}`); + } + }); + test(`checkbox tools (${browserKind}): AX state and set_checked are explicit and idempotent`, async (page) => { await setupContentFixture(page, 'trusted-click-fallback.html', browserKind); const tree = await call(page, 'get_accessibility_tree', { filter: 'all', maxDepth: 10, maxChars: 30000 }); diff --git a/test/run.js b/test/run.js index 2df2aff08..0272e87af 100644 --- a/test/run.js +++ b/test/run.js @@ -1916,6 +1916,43 @@ test('navigation-prone detection includes only submit-capable key and field call } }); +test('Chrome press_keys dispatches semicolon as a trusted CDP shortcut', async () => { + const originalAttach = cdpClientCh.attach; + const originalSendCommand = cdpClientCh.sendCommand; + const calls = []; + try { + cdpClientCh.attach = async () => ({ attached: true }); + cdpClientCh.sendCommand = async (tabId, method, params) => { + calls.push({ tabId, method, params }); + return {}; + }; + + const result = await new AgentCh({}).executeTool(42, 'press_keys', { key: ';' }); + assert.deepEqual(result, { + success: true, + dispatched: true, + method: 'cdp-key', + key: ';', + repeat: 1, + }); + assert.deepEqual(calls, [ + { + tabId: 42, + method: 'Input.dispatchKeyEvent', + params: { type: 'keyDown', key: ';', code: 'Semicolon', windowsVirtualKeyCode: 186 }, + }, + { + tabId: 42, + method: 'Input.dispatchKeyEvent', + params: { type: 'keyUp', key: ';', code: 'Semicolon', windowsVirtualKeyCode: 186 }, + }, + ]); + } finally { + cdpClientCh.attach = originalAttach; + cdpClientCh.sendCommand = originalSendCommand; + } +}); + test('agent URL normalization preserves query and hash for nav change detection', () => { for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) { const agent = new AgentClass({}); @@ -12627,8 +12664,10 @@ test('getToolsForMode: find_text replaces unsupported modifier shortcuts', () => const pressKeys = fullTools.find(t => t.function.name === 'press_keys'); assert.deepEqual(pressKeys.function.parameters.properties.key.enum, [ - 'Escape', 'Tab', 'Enter', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight', + 'Escape', 'Tab', 'Enter', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight', ';', ]); + assert.equal(askNames.includes('press_keys'), false, `${label}: Ask mode must remain read-only`); + assert.match(pressKeys.function.description, /semicolon.*Gmail Expand all/i); assert.match(pressKeys.function.description, /Ctrl\/Cmd\/Alt\/Shift combinations.*not supported/i); assert.doesNotMatch(pressKeys.function.parameters.properties.key.enum.join(' '), /Control|Meta|Alt|Shift|KeyF/); @@ -40337,11 +40376,17 @@ test('capabilitiesFor: set_field({submit}) requires BOTH type and click', () => assert.deepEqual(capabilitiesFor('read_page', {}), []); }); -test('press_keys: Enter is a submit (CLICK); Tab/Escape are benign (null)', () => { - assert.equal(capabilityFor('press_keys', { key: 'Enter' }), Capability.CLICK); - assert.equal(capabilityFor('press_keys', { key: 'Escape' }), null); - assert.equal(capabilityFor('press_keys', { key: 'Tab' }), null); - assert.equal(capabilityFor('press_keys', {}), Capability.CLICK); // unknown → gate, fail safe +test('press_keys: Enter and semicolon shortcuts require CLICK; Tab/Escape are benign', () => { + for (const [label, capabilityFn, Capabilities] of [ + ['chrome', capabilityForCh, CapabilityCh], + ['firefox', capabilityFor, Capability], + ]) { + assert.equal(capabilityFn('press_keys', { key: 'Enter' }), Capabilities.CLICK, `${label}: Enter`); + assert.equal(capabilityFn('press_keys', { key: ';' }), Capabilities.CLICK, `${label}: semicolon`); + assert.equal(capabilityFn('press_keys', { key: 'Escape' }), null, `${label}: Escape`); + assert.equal(capabilityFn('press_keys', { key: 'Tab' }), null, `${label}: Tab`); + assert.equal(capabilityFn('press_keys', {}), Capabilities.CLICK, `${label}: unknown key`); // fail safe + } }); test('submit controls bypass native select guards in click paths', () => { From eae6b5020dd483f5224b7802710a4df1d65ffbc9 Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Mon, 3 Aug 2026 17:16:13 +0300 Subject: [PATCH 2/2] Use Gecko semicolon metadata --- src/firefox/src/content/content.js | 2 +- test/README.md | 2 +- test/fixtures/run.mjs | 117 +++++++++++++++++------------ 3 files changed, 69 insertions(+), 52 deletions(-) diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js index 45c064a78..00c3ff30d 100644 --- a/src/firefox/src/content/content.js +++ b/src/firefox/src/content/content.js @@ -2154,7 +2154,7 @@ ArrowUp: { code: 'ArrowUp', keyCode: 38 }, ArrowRight: { code: 'ArrowRight', keyCode: 39 }, ArrowDown: { code: 'ArrowDown', keyCode: 40 }, - ';': { code: 'Semicolon', keyCode: 186 }, + ';': { code: 'Semicolon', keyCode: 59 }, }[key]; const target = (document.activeElement && document.activeElement !== document.body) ? document.activeElement diff --git a/test/README.md b/test/README.md index 0d72f65f1..d021ef21f 100644 --- a/test/README.md +++ b/test/README.md @@ -6,7 +6,7 @@ Three layers, each with a different scope, speed, and cost profile. ```bash npm install -npx playwright install chromium +npx playwright install chromium firefox ``` ## 1. Unit — `npm test` diff --git a/test/fixtures/run.mjs b/test/fixtures/run.mjs index 63a79948e..3eb8a1977 100644 --- a/test/fixtures/run.mjs +++ b/test/fixtures/run.mjs @@ -1,10 +1,10 @@ #!/usr/bin/env node // Fixtures runner for v4.0.1 overlay defenses. // -// Loads each fixture HTML in Chromium, injects the Chrome build's content.js -// with a stubbed `chrome.runtime`, and drives `click({text})` through the -// message handler. Asserts on response shape + which DOM element actually -// got the click. +// Loads fixture HTML in Chromium, plus targeted Firefox-engine regressions, +// injects the matching build's content.js with a stubbed extension runtime, +// and drives tools through the message handler. Asserts on response shape + +// which DOM element actually received the interaction. // // No LLM, no API keys, no real sites — just deterministic regression checks // for _findTopmostModal scoping, the occlusion hit-test, and the rich @@ -12,7 +12,7 @@ // // Run: npm run test:fixtures -import { chromium } from 'playwright'; +import { chromium, firefox as playwrightFirefox } from 'playwright'; import { fileURLToPath } from 'node:url'; import { readFile } from 'node:fs/promises'; import path from 'node:path'; @@ -323,6 +323,8 @@ async function selectFixtureText(page, selector = '#copy') { const tests = []; function test(name, fn) { tests.push({ name, fn }); } +const firefoxTests = []; +function firefoxTest(name, fn) { firefoxTests.push({ name, fn }); } for (const [label, browserKind] of [['Chrome', 'chrome'], ['Firefox', 'firefox']]) { test(`${label}: blocking NYTimes registration dialog suppresses article DOM`, async (page) => { @@ -1590,35 +1592,6 @@ for (const browserKind of ['chrome', 'firefox']) { } }); - test(`press_keys (${browserKind}): semicolon dispatches a page shortcut event`, async (page) => { - await setupContentFixture(page, 'trusted-click-fallback.html', browserKind); - await page.evaluate(() => { - window.__semicolonShortcutEvents = []; - document.addEventListener('keydown', (event) => { - if (event.key !== ';') return; - window.__semicolonShortcutEvents.push({ - key: event.key, - code: event.code, - keyCode: event.keyCode, - }); - }); - }); - - const result = await call(page, 'press_keys', { key: ';' }); - if (result?.success !== true || result?.dispatched !== true || result?.key !== ';') { - throw new Error(`semicolon should dispatch successfully, got: ${JSON.stringify(result)}`); - } - const events = await page.evaluate(() => window.__semicolonShortcutEvents); - if ( - events?.length !== 1 - || events[0]?.key !== ';' - || events[0]?.code !== 'Semicolon' - || events[0]?.keyCode !== 186 - ) { - throw new Error(`semicolon shortcut metadata mismatch: ${JSON.stringify(events)}`); - } - }); - test(`checkbox tools (${browserKind}): AX state and set_checked are explicit and idempotent`, async (page) => { await setupContentFixture(page, 'trusted-click-fallback.html', browserKind); const tree = await call(page, 'get_accessibility_tree', { filter: 'all', maxDepth: 10, maxChars: 30000 }); @@ -1772,6 +1745,45 @@ for (const browserKind of ['chrome', 'firefox']) { }); } +async function assertSemicolonShortcutEvent(page, browserKind, expectedKeyCode) { + await setupContentFixture(page, 'trusted-click-fallback.html', browserKind); + await page.evaluate(() => { + window.__semicolonShortcutEvents = []; + document.addEventListener('keydown', (event) => { + if (event.key !== ';') return; + window.__semicolonShortcutEvents.push({ + key: event.key, + code: event.code, + keyCode: event.keyCode, + which: event.which, + }); + }); + }); + + const result = await call(page, 'press_keys', { key: ';' }); + if (result?.success !== true || result?.dispatched !== true || result?.key !== ';') { + throw new Error(`semicolon should dispatch successfully, got: ${JSON.stringify(result)}`); + } + const events = await page.evaluate(() => window.__semicolonShortcutEvents); + if ( + events?.length !== 1 + || events[0]?.key !== ';' + || events[0]?.code !== 'Semicolon' + || events[0]?.keyCode !== expectedKeyCode + || events[0]?.which !== expectedKeyCode + ) { + throw new Error(`semicolon shortcut metadata mismatch: ${JSON.stringify(events)}`); + } +} + +test('press_keys (chrome): semicolon dispatches Chromium-compatible metadata', async (page) => { + await assertSemicolonShortcutEvent(page, 'chrome', 186); +}); + +firefoxTest('press_keys (firefox engine): semicolon dispatches Gecko-compatible metadata', async (page) => { + await assertSemicolonShortcutEvent(page, 'firefox', 59); +}); + test('set_checked (chrome): post-click verification survives same-document route changes', async (page) => { await setupContentFixture(page, 'trusted-click-fallback.html', 'chrome'); const tree = await call(page, 'get_accessibility_tree', { filter: 'all', maxDepth: 10, maxChars: 30000 }); @@ -2958,23 +2970,28 @@ test('SMD: X photo modal wins over background timeline media', async (page) => { }); (async () => { - const browser = await chromium.launch(); - const context = await browser.newContext(); let passed = 0, failed = 0; - for (const t of tests) { - const page = await context.newPage(); - try { - await t.fn(page); - console.log(` ✓ ${t.name}`); - passed++; - } catch (e) { - console.log(` ✗ ${t.name}\n ${e.message}`); - failed++; - } finally { - await page.close(); + const runTests = async (browserType, entries) => { + const browser = await browserType.launch(); + const context = await browser.newContext(); + for (const t of entries) { + const page = await context.newPage(); + try { + await t.fn(page); + console.log(` ✓ ${t.name}`); + passed++; + } catch (e) { + console.log(` ✗ ${t.name}\n ${e.message}`); + failed++; + } finally { + await page.close(); + } } - } - await browser.close(); - console.log(`\n ${passed} passed, ${failed} failed (${tests.length} total)`); + await browser.close(); + }; + + await runTests(chromium, tests); + await runTests(playwrightFirefox, firefoxTests); + console.log(`\n ${passed} passed, ${failed} failed (${tests.length + firefoxTests.length} total)`); process.exit(failed > 0 ? 1 : 0); })();