diff --git a/README.md b/README.md index 7955dc4..4b89d34 100644 --- a/README.md +++ b/README.md @@ -369,65 +369,65 @@ type ToolContext = { }; ``` -The daemon routes `POST /tool/:name` requests through the registry, applies Zod validation on inputs, executes the tool through the request queue, and captures observations (extension state, test IDs, a11y snapshot) after each execution. +The daemon routes `POST /tool/:name` requests through the registry, applies Zod validation on inputs, executes the tool through the request queue, and captures observations (extension state, test IDs, a11y snapshot) after mutating tools and `run_steps`. **Registered tools:** -| Tool | Description | -| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Lifecycle** | | -| `build` | Triggers an extension build using the configured `BuildCapability`. Accepts build type and force options. | -| `launch` | Launches a new browser session with the configured extension. Supports state modes (`default`, `onboarding`, `custom`), fixture presets, tag metadata, and optional contract seeding on start. | -| `cleanup` | Tears down the active browser session and cleans up all resources (browser, services, fixtures). | -| **Interaction** | | -| `click` | Clicks an element identified by a11y ref, test ID, or CSS selector. Waits for the element to be visible before clicking. Supports `within` to scope the target inside a parent element. | -| `type` | Types text into an input element identified by a11y ref, test ID, or CSS selector. Clears the field first, then sets the new value (uses Playwright's `fill()`). Supports `within` scoping. | -| `wait_for` | Waits for an element to become visible on the page within a configurable timeout. Supports `within` to scope the target inside a parent element. | -| `get_text` | Reads the text content of an element identified by a11y ref, test ID, or CSS selector. Returns the text, target descriptor, and character length. Supports `within` scoping. Categorized as read-only (no observations in response). | -| `clipboard` | Reads from or writes to the system clipboard via Chrome DevTools Protocol. Useful for pasting seed phrases or copying addresses. | -| **Navigation** | | -| `navigate` | Navigates the browser to a named screen (`home`, `settings`, `notification`) or an arbitrary URL. | -| `switch_to_tab` | Switches the active page to a tab matching a given role (e.g., `extension`, `dapp`) or URL prefix. | -| `close_tab` | Closes a browser tab matching a given role or URL. Falls back to the extension tab if the active tab is closed. | -| `wait_for_notification` | Waits for the extension notification popup to appear within a timeout. Returns the notification page URL. | -| **Discovery** | | -| `describe_screen` | Captures a comprehensive screen snapshot: extension state, visible test IDs, trimmed a11y tree with refs, and optional screenshot. | -| `accessibility_snapshot` | Captures a trimmed accessibility tree of the current page with deterministic refs (`e1`, `e2`, ...). Supports scoping to a root CSS selector. | -| `list_testids` | Collects all visible `data-testid` attributes from the current page with text previews and visibility status. | -| **State** | | -| `get_state` | Retrieves the current extension state (URL, screen, network, balance, account) and tracked tab information. | -| `get_context` | Returns the current environment context (`e2e` or `prod`), session status, available capabilities, and whether context switching is allowed. | -| `set_context` | Switches the session environment between `e2e` and `prod` modes. Blocked while a session is active. | -| **Screenshots** | | -| `screenshot` | Captures a screenshot of the current page. Supports naming, full-page capture, scoping to a CSS selector, and optional base64 output. | -| **Contracts** | | -| `seed_contract` | Deploys a single smart contract to the local Anvil chain by name. Requires `ContractSeedingCapability`. | -| `seed_contracts` | Deploys multiple smart contracts in sequence. Returns both successful deployments and individual failures. | -| `get_contract_address` | Looks up the deployed address of a contract by name from the session's deployment registry. | -| `list_contracts` | Lists all contracts deployed in the current session with addresses and deployment timestamps. | -| **Batching** | | -| `run_steps` | Executes a batch of tool invocations sequentially. Supports `stopOnError` to halt on first failure, `includeObservations` (`'all'`, `'none'`, `'failures'`) to control observations, and `batchTimeoutMs` to set an overall deadline (remaining steps are skipped on timeout). Accepts tool aliases like `navigate_home` / `navigate-home`. Returns per-step results with timing. | -| **Advanced** | | -| `mock_network` | Adds, clears, lists, and inspects targeted Playwright network mocks on the active browser context. Unmatched same-origin requests are continued unchanged. | -| `cdp` | Sends a raw Chrome DevTools Protocol command, dispatched through the active platform driver. **Browser:** targets the page's Chrome CDP session (full Runtime/DOM/Network/Page surface); destructive methods (`Browser.close`, `Target.closeTarget`, etc.) are blocked. **Mobile (iOS/Android):** targets the app's React Native Hermes JS runtime via Metro's inspector proxy (delegates to `@metamask/device-mcp`, needs a DEBUG build with Metro running); only the JS-engine subset exists (`Runtime`, `Debugger`, `Log`, `HeapProfiler` — no DOM/Page/Network) and `Runtime.terminateExecution` / `Inspector.detached` are blocked. Optional `metroPort` / `appId` are mobile-only (ignored on browser). Categorized as mutating — run `describe_screen` afterward to re-sync. See [Browser vs Mobile CDP](#browser-vs-mobile-hermes-cdp). | -| **Hermes (mobile only)** | | -| `hermes_targets` | Lists and diagnoses the debuggable Hermes targets Metro exposes (iOS and Android), reporting which target would be chosen or why selection is ambiguous. Use to confirm Metro is running and the app is registered. Pass `all` to bypass the appId filter and discover the real appId. Mobile only — there is no browser equivalent. | -| **Mobile (iOS/Android only)** | | -| `scroll_to_element` | Scrolls the screen until an element (a11y ref, test ID, or selector) becomes visible. Supports `direction` (`up`/`down`) and `maxAttempts`. Mobile only. | -| `device_swipe` | Swipes the screen in a `direction` (`up`/`down`/`left`/`right`), with optional `startX`/`startY`/`distance`. Mobile only. | -| `long_press` | Long-presses an element (a11y ref, test ID, or selector) for an optional `durationMs`. Mobile only. | -| `tap_coordinates` | Taps raw `x`/`y` screen coordinates. Mobile only. | -| `dismiss_keyboard` | Dismisses the on-screen keyboard. Mobile only. | -| `dismiss_alert` | Dismisses a native alert, accepting it when `accept` is true. Mobile only. | -| `get_alert_text` | Returns the text of a visible native alert. Read-only. Mobile only. | -| `open_app` | Launches or foregrounds an app by `bundleId`. Mobile only. | -| `close_app` | Terminates an app by `bundleId`. Mobile only. | -| `press_button` | Presses a hardware/system button. One of `home`, `back`, `enter`, `lock`. Mobile only. | -| `device_context` | Lists native/webview contexts (`list`) or switches to one (`switch` + `name`). Mobile only. | -| `device_clipboard` | Reads (`read`) or writes (`write` + `text`) the device clipboard. Distinct from the browser `clipboard` tool (which uses CDP). Mobile only. | -| `screen_recording` | Starts (`start`, optional `outputPath`) or stops (`stop`) a screen recording. When provided, `outputPath` is sandboxed to the configured `artifactsDir` (paths escaping it are rejected). Read-only. Mobile only. | -| `device_logs` | Fetches device logs with optional `durationSeconds` and `filter`. Read-only. Mobile only. | -| `generate_locators` | Generates ranked locator suggestions (identifier > label > text > type, with confidence) for every interactive element on the current screen, by walking the device snapshot. Read-only. Mobile only. | +| Tool | Description | +| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Lifecycle** | | +| `build` | Triggers an extension build using the configured `BuildCapability`. Accepts build type and force options. | +| `launch` | Launches a new browser session with the configured extension. Supports state modes (`default`, `onboarding`, `custom`), fixture presets, tag metadata, and optional contract seeding on start. | +| `cleanup` | Tears down the active browser session and cleans up all resources (browser, services, fixtures). | +| **Interaction** | | +| `click` | Clicks an element identified by a11y ref, test ID, or CSS selector. Waits for the element to be visible before clicking. Supports `within` to scope the target inside a parent element. | +| `type` | Types text into an input element identified by a11y ref, test ID, or CSS selector. Clears the field first, then sets the new value (uses Playwright's `fill()`). Supports `within` scoping. | +| `wait_for` | Waits for an element to become visible on the page within a configurable timeout. Supports `within` to scope the target inside a parent element. | +| `get_text` | Reads the text content of an element identified by a11y ref, test ID, or CSS selector. Returns the text, target descriptor, and character length. Supports `within` scoping. Categorized as read-only (no observations in response). | +| `clipboard` | Reads from or writes to the system clipboard via Chrome DevTools Protocol. Useful for pasting seed phrases or copying addresses. | +| **Navigation** | | +| `navigate` | Navigates the browser to a named screen (`home`, `settings`, `notification`) or an arbitrary URL. | +| `switch_to_tab` | Switches the active page to a tab matching a given role (e.g., `extension`, `dapp`) or URL prefix. | +| `close_tab` | Closes a browser tab matching a given role or URL. Falls back to the extension tab if the active tab is closed. | +| `wait_for_notification` | Waits for the extension notification popup to appear within a timeout. Returns the notification page URL. | +| **Discovery** | | +| `describe_screen` | Captures a comprehensive screen snapshot: extension state, visible test IDs, trimmed a11y tree with refs, and optional screenshot. | +| `accessibility_snapshot` | Captures a trimmed accessibility tree of the current page with deterministic refs (`e1`, `e2`, ...). Supports scoping to a root CSS selector. | +| `list_testids` | Collects all visible `data-testid` attributes from the current page with text previews and visibility status. | +| **State** | | +| `get_state` | Retrieves the current extension state (URL, screen, network, balance, account) and tracked tab information. | +| `get_context` | Returns the current environment context (`e2e` or `prod`), session status, available capabilities, and whether context switching is allowed. | +| `set_context` | Switches the session environment between `e2e` and `prod` modes. Blocked while a session is active. | +| **Screenshots** | | +| `screenshot` | Captures a screenshot of the current page. Supports naming, full-page capture, scoping to a CSS selector, and optional base64 output. | +| **Contracts** | | +| `seed_contract` | Deploys a single smart contract to the local Anvil chain by name. Requires `ContractSeedingCapability`. | +| `seed_contracts` | Deploys multiple smart contracts in sequence. Returns both successful deployments and individual failures. | +| `get_contract_address` | Looks up the deployed address of a contract by name from the session's deployment registry. | +| `list_contracts` | Lists all contracts deployed in the current session with addresses and deployment timestamps. | +| **Batching** | | +| `run_steps` | Executes a batch of tool invocations sequentially. Supports `stopOnError` to halt on first failure, `includeObservations` (`'all'`, `'none'`, `'failures'`) to control observations, and `batchTimeoutMs` to set an overall deadline (remaining steps are skipped on timeout). Accepts tool aliases like `navigate_home` / `navigate-home`. Returns per-step results with timing. | +| **Advanced** | | +| `mock_network` | Adds, clears, lists, and inspects targeted Playwright network mocks on the active browser context. Unmatched same-origin requests are continued unchanged. | +| `cdp` | Sends a raw Chrome DevTools Protocol command, dispatched through the active platform driver. **Browser:** targets the page's Chrome CDP session (full Runtime/DOM/Network/Page surface); destructive methods (`Browser.close`, `Target.closeTarget`, etc.) are blocked. **Mobile (iOS/Android):** targets the app's React Native Hermes JS runtime via Metro's inspector proxy (delegates to `@metamask/device-mcp`, needs a DEBUG build with Metro running); only the JS-engine subset exists (`Runtime`, `Debugger`, `Log`, `HeapProfiler` — no DOM/Page/Network) and `Runtime.terminateExecution` / `Inspector.detached` are blocked. Optional `metroPort` / `appId` are mobile-only (ignored on browser). Categorized as `cdp`: no observation is collected and refs are not refreshed, so run `describe_screen` after a call that changes state. See [Browser vs Mobile CDP](#browser-vs-mobile-hermes-cdp). | +| **Hermes (mobile only)** | | +| `hermes_targets` | Lists and diagnoses the debuggable Hermes targets Metro exposes (iOS and Android), reporting which target would be chosen or why selection is ambiguous. Use to confirm Metro is running and the app is registered. Pass `all` to bypass the appId filter and discover the real appId. Mobile only — there is no browser equivalent. | +| **Mobile (iOS/Android only)** | | +| `scroll_to_element` | Scrolls the screen until an element (a11y ref, test ID, or selector) becomes visible. Supports `direction` (`up`/`down`) and `maxAttempts`. Mobile only. | +| `device_swipe` | Swipes the screen in a `direction` (`up`/`down`/`left`/`right`), with optional `startX`/`startY`/`distance`. Mobile only. | +| `long_press` | Long-presses an element (a11y ref, test ID, or selector) for an optional `durationMs`. Mobile only. | +| `tap_coordinates` | Taps raw `x`/`y` screen coordinates. Mobile only. | +| `dismiss_keyboard` | Dismisses the on-screen keyboard. Mobile only. | +| `dismiss_alert` | Dismisses a native alert, accepting it when `accept` is true. Mobile only. | +| `get_alert_text` | Returns the text of a visible native alert. Read-only. Mobile only. | +| `open_app` | Launches or foregrounds an app by `bundleId`. Mobile only. | +| `close_app` | Terminates an app by `bundleId`. Mobile only. | +| `press_button` | Presses a hardware/system button. One of `home`, `back`, `enter`, `lock`. Mobile only. | +| `device_context` | Lists native/webview contexts (`list`) or switches to one (`switch` + `name`). Mobile only. | +| `device_clipboard` | Reads (`read`) or writes (`write` + `text`) the device clipboard. Distinct from the browser `clipboard` tool (which uses CDP). Mobile only. | +| `screen_recording` | Starts (`start`, optional `outputPath`) or stops (`stop`) a screen recording. When provided, `outputPath` is sandboxed to the configured `artifactsDir` (paths escaping it are rejected). Read-only. Mobile only. | +| `device_logs` | Fetches device logs with optional `durationSeconds` and `filter`. Read-only. Mobile only. | +| `generate_locators` | Generates ranked locator suggestions (identifier > label > text > type, with confidence) for every interactive element on the current screen, by walking the device snapshot. Read-only. Mobile only. | ### Accessibility References @@ -525,7 +525,7 @@ All responses follow a consistent shape: { ok: false, error: { code: string, message: string } } ``` -The `observations` field is included for **mutating** tools (click, type, navigate, launch, cleanup, build, etc.) and for `run_steps` when its `includeObservations` parameter is `'all'` (default) or `'failures'`. **Read-only** and **discovery** tools omit observations from the response. +The `observations` field is included for **mutating** tools (click, type, navigate, launch, cleanup, build, etc.) and for `run_steps` when its `includeObservations` parameter is `'all'` (default) or `'failures'`. **Read-only**, **discovery** and `cdp` calls skip observation collection entirely (`meta.timing.observationMs` is 0). ## CLI Reference @@ -625,7 +625,7 @@ mm cdp Network.enable mm cdp DOM.getDocument '{"depth":2}' --timeout 60000 ``` -The `params-json` argument must be a valid JSON object. The `--timeout` flag sets a per-command timeout (default: 30 000 ms, max: 30 000 ms). The tool is categorized as **mutating** — run `mm describe-screen` afterward to re-sync session state. +The `params-json` argument must be a valid JSON object. The `--timeout` flag sets a per-command timeout (default: 30 000 ms, max: 30 000 ms). The tool is categorized as `cdp`: it returns no observation and does not refresh refs, so run `mm describe-screen` after a call that changes state. #### Browser vs Mobile (Hermes) CDP diff --git a/SKILL.md b/SKILL.md index b70f170..2f7f222 100644 --- a/SKILL.md +++ b/SKILL.md @@ -25,12 +25,13 @@ mm cleanup --shutdown # 5. Clean up when done Tool responses include different data based on the tool's category: -| Category | Examples | Observations in response? | -| ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------- | -| **Mutating** | click, type, navigate, launch, cleanup, build, clipboard, cdp, mock_network, scroll_to_element, device_swipe, long_press, tap_coordinates, dismiss_keyboard, dismiss_alert, open_app, close_app, press_button, device_context, device_clipboard | Yes — `state` + `a11y` (compacted) + `testIds` | -| **Read-only** | get_state, get_text, get_context, set_context, get_alert_text, screen_recording, device_logs, generate_locators, hermes_targets | No — faster response | -| **Discovery** | describe_screen, list_testids, accessibility_snapshot, screenshot | Data is already in `result` | -| **Batch** | run_steps | Controlled by `includeObservations` param | +| Category | Examples | Observations in response? | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------- | +| **Mutating** | click, type, navigate, launch, cleanup, build, clipboard, mock_network, scroll_to_element, device_swipe, long_press, tap_coordinates, dismiss_keyboard, dismiss_alert, open_app, close_app, press_button, device_context, device_clipboard | Yes — `state` + `a11y` (compacted) + `testIds` | +| **Read-only** | get_state, get_text, get_context, set_context, get_alert_text, screen_recording, device_logs, generate_locators, hermes_targets | No — faster response | +| **Discovery** | describe_screen, list_testids, accessibility_snapshot, screenshot | Data is already in `result` | +| **CDP** | cdp | No — raw result only | +| **Batch** | run_steps | Controlled by `includeObservations` param | **Observation Compaction:** Mutating tool observations are **compacted** before returning: option runs of 3 or more under a combobox or listbox are replaced with a single summary node (e.g., `"55 options (refs e2–e56)"`). The `describe-screen` tool always returns the **full, unfiltered** a11y tree — use it when you need the complete option list. @@ -44,7 +45,7 @@ Tool responses include different data based on the tool's category: } ``` -The `observations.a11y.nodes` field contains **only the changed and new nodes** (not all nodes). The baseline resets after `describe-screen`, `launch`, or `cleanup` — the next mutation returns a full compact observation (no `diff` field). When the diff would be larger than the full observation, the full option-filtered observation is returned instead (no `diff` field). +The `observations.a11y.nodes` field contains **only the changed and new nodes** (not all nodes). The baseline resets after `describe-screen`, `accessibility-snapshot`, `launch`, or `cleanup` — the next mutation returns a full compact observation (no `diff` field). When the diff would be larger than the full observation, the full option-filtered observation is returned instead (no `diff` field). ### Using inline observations (mutating tools) diff --git a/src/server/create-server.test.ts b/src/server/create-server.test.ts index d3ba21a..087d665 100644 --- a/src/server/create-server.test.ts +++ b/src/server/create-server.test.ts @@ -167,12 +167,16 @@ describe('shouldCollectObservations', () => { expect(shouldCollectObservations('mutating')).toBe(true); }); - it('returns true for readonly', () => { - expect(shouldCollectObservations('readonly')).toBe(true); + it('returns false for readonly', () => { + expect(shouldCollectObservations('readonly')).toBe(false); + }); + + it('returns false for discovery', () => { + expect(shouldCollectObservations('discovery')).toBe(false); }); - it('returns true for discovery', () => { - expect(shouldCollectObservations('discovery')).toBe(true); + it('returns false for cdp', () => { + expect(shouldCollectObservations('cdp')).toBe(false); }); it('returns true for batch with default policy', () => { @@ -227,6 +231,10 @@ describe('shouldIncludeObservationsInResponse', () => { ); }); + it('returns false for cdp', () => { + expect(shouldIncludeObservationsInResponse('cdp', okResult)).toBe(false); + }); + it("returns true for batch with 'all' (default)", () => { expect(shouldIncludeObservationsInResponse('batch', okResult, {})).toBe( true, @@ -881,7 +889,7 @@ describe('createServer with active session', () => { } }); - it('observation Playwright helpers called for read-only tools', async () => { + it('observation Playwright helpers not called for read-only tools', async () => { const { collectTestIds, collectTrimmedA11ySnapshot } = await import('../tools/utils/discovery.js'); const collectTestIdsSpy = vi.mocked(collectTestIds); @@ -896,8 +904,8 @@ describe('createServer with active session', () => { body: JSON.stringify({}), }); - expect(collectTestIdsSpy).toHaveBeenCalled(); - expect(collectA11ySpy).toHaveBeenCalled(); + expect(collectTestIdsSpy).not.toHaveBeenCalled(); + expect(collectA11ySpy).not.toHaveBeenCalled(); }); it('observation Playwright helpers called for mutating tools', async () => { @@ -1004,7 +1012,7 @@ describe('createServer with active session', () => { expect(body.observations?.state.currentScreen).toBe('unknown'); }); - it('does not recheck for readonly tool category', async () => { + it('collects no observation for readonly tool category', async () => { mockSM.getExtensionState.mockReset(); mockSM.getExtensionState.mockResolvedValue({ isLoaded: true, @@ -1020,12 +1028,14 @@ describe('createServer with active session', () => { body: JSON.stringify({}), }, ); + const body = (await res.json()) as { observations?: unknown }; expect(res.status).toBe(200); - expect(mockSM.getExtensionState).toHaveBeenCalledTimes(1); + expect(body.observations).toBeUndefined(); + expect(mockSM.getExtensionState).not.toHaveBeenCalled(); }); - it('does not recheck for discovery tool category', async () => { + it('collects no observation for discovery tool category', async () => { mockSM.getExtensionState.mockReset(); mockSM.getExtensionState.mockResolvedValue({ isLoaded: true, @@ -1041,9 +1051,11 @@ describe('createServer with active session', () => { body: JSON.stringify({}), }, ); + const body = (await res.json()) as { observations?: unknown }; expect(res.status).toBe(200); - expect(mockSM.getExtensionState).toHaveBeenCalledTimes(1); + expect(body.observations).toBeUndefined(); + expect(mockSM.getExtensionState).not.toHaveBeenCalled(); }); it('does not recheck for batch tool category', async () => { @@ -1141,6 +1153,48 @@ describe('createServer with active session', () => { expect(body.meta?.timing.observationMs).toBeGreaterThanOrEqual(50); }); + it.each([ + ['readonly', '/tool/get_state'], + ['discovery', '/tool/list_testids'], + ['cdp', '/tool/cdp', JSON.stringify({ method: 'Runtime.evaluate' })], + ])( + 'reports observationMs 0 for %s tools with an active session', + async (category, urlPath, requestBody?: string) => { + if (category === 'cdp') { + mockSM.getPlatformDriver.mockReturnValue({ + getPlatform: () => 'browser', + cdp: vi.fn().mockResolvedValue({ ok: true, result: { value: 2 } }), + }); + } + + const { status, body } = await post(urlPath, requestBody); + + expect(status).toBe(200); + expect(body.ok).toBe(true); + expect( + (body as TimedBody & { observations?: unknown }).observations, + ).toBeUndefined(); + expectValidTiming(body); + expect(body.meta?.timing.observationMs).toBe(0); + }, + ); + + it('reports observationMs above 0 for mutating tools', async () => { + mockSM.getExtensionState.mockImplementationOnce( + async () => + new Promise((resolve) => + setTimeout(() => resolve({ isLoaded: true, currentUrl: '' }), 20), + ), + ); + + const { body } = await post('/cleanup'); + + expect( + (body as TimedBody & { observations?: unknown }).observations, + ).toBeDefined(); + expect(body.meta?.timing.observationMs).toBeGreaterThan(0); + }); + it('reports observationMs 0 when no session is active', async () => { mockSM.hasActiveSession.mockReturnValue(false); @@ -1547,6 +1601,105 @@ describe('observation compaction in HTTP responses', () => { expect(body.observations?.a11y.nodes.length).toBeGreaterThan(1); }); + it('accessibility_snapshot resets the diff baseline', async () => { + const { collectTrimmedA11ySnapshot } = + await import('../tools/utils/discovery.js'); + vi.mocked(collectTrimmedA11ySnapshot) + .mockResolvedValueOnce({ + nodes: initialButtons as never, + refMap: new Map(), + }) + .mockResolvedValueOnce({ + nodes: initialButtons as never, + refMap: new Map(), + }) + .mockResolvedValueOnce({ + nodes: changedButtons as never, + refMap: new Map(), + }); + + await httpRequest(`http://127.0.0.1:${state.port}/tool/click`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ a11yRef: 'e1' }), + }); + + await httpRequest( + `http://127.0.0.1:${state.port}/tool/accessibility_snapshot`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({}), + }, + ); + + const res = await httpRequest(`http://127.0.0.1:${state.port}/tool/click`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ a11yRef: 'e1' }), + }); + const body = (await res.json()) as { + ok: boolean; + observations?: { + a11y: { + nodes: unknown[]; + diff?: unknown; + }; + }; + }; + + expect(res.status).toBe(200); + expect(body.observations).toBeDefined(); + expect(body.observations?.a11y.diff).toBeUndefined(); + expect(body.observations?.a11y.nodes.length).toBeGreaterThan(1); + }); + + it('read tools do not move the diff baseline', async () => { + const { collectTrimmedA11ySnapshot } = + await import('../tools/utils/discovery.js'); + vi.mocked(collectTrimmedA11ySnapshot) + .mockResolvedValueOnce({ + nodes: initialButtons as never, + refMap: new Map(), + }) + .mockResolvedValueOnce({ + nodes: changedButtons as never, + refMap: new Map(), + }); + + await httpRequest(`http://127.0.0.1:${state.port}/tool/click`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ a11yRef: 'e1' }), + }); + + await httpRequest(`http://127.0.0.1:${state.port}/tool/get_state`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({}), + }); + + const res = await httpRequest(`http://127.0.0.1:${state.port}/tool/click`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ a11yRef: 'e1' }), + }); + const body = (await res.json()) as { + ok: boolean; + observations?: { + a11y: { + diff?: { added: string[]; removed: string[]; unchanged: number }; + }; + }; + }; + + expect(res.status).toBe(200); + expect(body.observations?.a11y.diff).toStrictEqual({ + added: ['e3'], + removed: ['e2'], + unchanged: 1, + }); + }); it('falls back to the full observation when the diff is not smaller', async () => { const { collectTrimmedA11ySnapshot } = await import('../tools/utils/discovery.js'); diff --git a/src/server/create-server.ts b/src/server/create-server.ts index 062f472..4430025 100644 --- a/src/server/create-server.ts +++ b/src/server/create-server.ts @@ -22,6 +22,7 @@ import { checkPlatformGate, } from '../tools/registry.js'; import type { ToolCategory } from '../tools/registry.js'; +import type { RunStepsInput } from '../tools/types'; import { createDefaultObservation } from '../tools/types/observation.js'; import type { StepRecordObservation } from '../tools/types/observation.js'; import { @@ -74,11 +75,12 @@ export function buildResponseBody( } /** - * Whether to run Playwright observation collection for this tool invocation. + * Whether to collect an observation (state, test IDs, a11y tree) after a tool + * runs. Only tools that change the UI pay for it: * - * Observations are collected for every tool, whether or not they appear in - * the HTTP response (they also feed the next response's diff). The only - * exception is batch with `'none'` policy, which skips collection entirely. + * - `mutating`: yes. + * - `batch` (`run_steps`): yes, unless `includeObservations` is `'none'`. + * - `readonly`, `discovery`, `cdp`: no. * * @param category - The tool category to check. * @param validatedInput - The validated input payload (checked for batch policy). @@ -88,17 +90,19 @@ export function shouldCollectObservations( category: ToolCategory, validatedInput?: Record, ): boolean { + if (category === 'mutating') { + return true; + } if (category === 'batch') { - const policy = - (validatedInput as { includeObservations?: string }) - ?.includeObservations ?? 'all'; - return policy !== 'none'; + return batchObservationPolicy(validatedInput) !== 'none'; } - return true; + return false; } /** - * Whether to include observations in the HTTP response. + * Whether to include the collected observation in the HTTP response. Never + * true when {@link shouldCollectObservations} is false. For `run_steps` with + * `includeObservations: 'failures'`, only failed batches include it. * * @param category - The tool category. * @param toolResult - The result returned by the tool. @@ -110,27 +114,35 @@ export function shouldIncludeObservationsInResponse( toolResult: ToolResponse, validatedInput?: Record, ): boolean { - if (category === 'mutating') { - return true; + if (!shouldCollectObservations(category, validatedInput)) { + return false; } - if (category === 'batch') { - const policy = - (validatedInput as { includeObservations?: string }) - ?.includeObservations ?? 'all'; - if (policy === 'none') { - return false; + if ( + category === 'batch' && + batchObservationPolicy(validatedInput) === 'failures' + ) { + if (!toolResult.ok) { + return true; } - if (policy === 'failures') { - if (!toolResult.ok) { - return true; - } - const result = toolResult.result as Record; - const summary = result?.summary as Record | undefined; - return summary !== undefined && !summary.ok; - } - return true; // 'all' + const result = toolResult.result as Record; + const summary = result?.summary as Record | undefined; + return summary !== undefined && !summary.ok; } - return false; // readonly, discovery + return true; +} + +/** + * Reads the `run_steps` observation policy. + * + * @param validatedInput - The validated `run_steps` input. + * @returns The policy, `'all'` by default. + */ +function batchObservationPolicy( + validatedInput: Record | undefined, +): NonNullable { + const policy = + validatedInput?.includeObservations as RunStepsInput['includeObservations']; + return policy ?? 'all'; } /** @@ -455,6 +467,7 @@ export function createServer(config: ServerConfig): ServerInstance { if ( toolName === 'describe_screen' || + toolName === 'accessibility_snapshot' || toolName === 'launch' || toolName === 'cleanup' ) { diff --git a/src/tools/registry.test.ts b/src/tools/registry.test.ts index 3693ffa..8055ec4 100644 --- a/src/tools/registry.test.ts +++ b/src/tools/registry.test.ts @@ -85,6 +85,10 @@ describe('TOOL_CATEGORIES and getToolCategory', () => { expect(getToolCategory('describe_screen')).toBe('discovery'); }); + it('getToolCategory returns cdp for cdp', () => { + expect(getToolCategory('cdp')).toBe('cdp'); + }); + it('getToolCategory returns batch for run_steps', () => { expect(getToolCategory('run_steps')).toBe('batch'); }); diff --git a/src/tools/registry.ts b/src/tools/registry.ts index ff85eaf..9656d98 100644 --- a/src/tools/registry.ts +++ b/src/tools/registry.ts @@ -104,10 +104,26 @@ export const toolRegistry = new Map>([ ['generate_locators', generateLocatorsTool], ]); -export type ToolCategory = 'mutating' | 'readonly' | 'discovery' | 'batch'; +/** + * Tool categories decide whether the daemon collects an observation after the + * tool runs (see `shouldCollectObservations` in `server/create-server.ts`): + * + * - `mutating`: changes the UI; an observation is collected and returned. + * - `readonly` / `discovery`: reads only; no observation. + * - `cdp`: raw CDP calls; mostly reads (`Runtime.evaluate`), so no observation + * (until a per-call `observe` level lets the caller ask for one). + * - `batch`: `run_steps`; one observation after the batch unless + * `includeObservations` is `'none'`. + */ +export type ToolCategory = + | 'mutating' + | 'readonly' + | 'discovery' + | 'cdp' + | 'batch'; export const TOOL_CATEGORIES: Record = { - // MUTATING (26) + // MUTATING (25) click: 'mutating', type: 'mutating', navigate: 'mutating', @@ -121,7 +137,6 @@ export const TOOL_CATEGORIES: Record = { wait_for_notification: 'mutating', seed_contract: 'mutating', seed_contracts: 'mutating', - cdp: 'mutating', mock_network: 'mutating', scroll_to_element: 'mutating', device_swipe: 'mutating', @@ -155,6 +170,8 @@ export const TOOL_CATEGORIES: Record = { list_testids: 'discovery', accessibility_snapshot: 'discovery', screenshot: 'discovery', + // CDP (1) + cdp: 'cdp', // BATCH (1) run_steps: 'batch', };