From b7e47020dfdac18475b09cb6b2e1b941ce809042 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sat, 24 Oct 2026 10:45:16 +0400 Subject: [PATCH] feat(devices): show devices in an interactive card Replace mobile_choose_devices with mobile_show_devices, the one user-facing device tool. Without selectionMode it lists every simulator, emulator and physical device in an inline card with Boot/Stop and a Use button that sends the selection to chat. With selectionMode it uses the native Codex form when available, the card elsewhere, and asks in chat on hosts with neither. Candidate devices are optional and discovered when omitted. Card results return a short summary so the model does not repeat the list. Adds the devices telemetry surface and a mode attribute on the existing device_picker metrics. Closes #32 Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 33 ++++--- docs/mcp-tools.md | 2 +- docs/telemetry.md | 18 +++- scripts/smoke-claude-package.mjs | 5 +- skills/mobile-dev/SKILL.md | 25 +++-- src/server/device-choice-tools.ts | 126 ++++++++++++++++-------- src/server/plugin.ts | 20 ++-- src/server/telemetry.ts | 2 +- src/shared/device-choice.ts | 63 +++++++++++- src/shared/telemetry.ts | 2 +- src/ui/app.tsx | 2 + src/ui/components/devices-card.tsx | 103 ++++++++++++++++++++ src/ui/devices-app.tsx | 41 ++++++++ src/ui/devices-controller.ts | 151 +++++++++++++++++++++++++++++ src/ui/style.css | 2 +- src/ui/telemetry.ts | 4 +- tests/device-choice-tools.test.ts | 73 +++++++++++++- tests/plugin.test.ts | 15 ++- tests/telemetry.test.ts | 10 +- 19 files changed, 593 insertions(+), 104 deletions(-) create mode 100644 src/ui/components/devices-card.tsx create mode 100644 src/ui/devices-app.tsx create mode 100644 src/ui/devices-controller.ts diff --git a/docs/architecture.md b/docs/architecture.md index 3ab56a70..9db7d9c2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -74,14 +74,26 @@ The MCP server returns a flat element list with `nodeId`, `parentId` and `depth` Annotations send text only. Each note includes the user request, element label, test ID when present, React component and owner names when available, and nearby text. React debug creation stacks resolve through one bounded request to the matching Metro server's `/symbolicate` endpoint. Source maps supply an app file, line, column and function so the agent can open the element's JSX directly. This is the element creation site, not necessarily the component definition. Library frames and unresolved bundle locations never become edit locations. Missing source maps leave selection working and mark the source location unavailable. Coordinates remain a fallback. The paused image stays local for editing; use Screenshot separately to attach an image. -## Native device requests - -When a mobile task has ambiguous targets, `mobile_choose_devices` asks the user -through Codex's native inline request form. The server lists this tool only for -clients that advertise the `openai/elicitation` form extension; other hosts ask in -chat. The agent discovers suitable devices -first, then supplies their IDs, platform and kind, a task-specific question, and -optional operation details. The form shows canonical device names, platform/type, +## Devices card and device requests + +`mobile_show_devices` is the one user-facing device tool. Without `selectionMode` +it lists every discovered simulator, emulator and physical device in an inline +card, grouped as running, physical, then stopped by runtime. The user can boot or +stop simulators and emulators from the card and send selected devices to chat. A +backend that cannot list devices (no Android SDK, older Xcode) leaves the others +listed with a note. The tool's text result is a short summary so the model does +not repeat the list; hosts without app cards receive the list as text instead. + +When a mobile task has ambiguous targets, the agent sets `selectionMode` with a +task-specific `message`. Hosts that advertise the `openai/elicitation` form +extension (Codex) get the native inline request form described below, and the card +then shows only the outcome. Hosts that render MCP app cards get the same card with +only selectable candidates and a Use button that sends the choice to chat; this +call returns immediately and tells the model to wait for that reply. Hosts with +neither return an error so the agent asks in chat. Candidate `devices` are +optional: omitted, the tool discovers them itself, and the form offers at most 20, +running devices first. The agent may instead supply discovered IDs, platform and +kind, plus optional operation details. The form shows canonical device names, platform/type, runtime, optional verified app labels, and platform phone illustrations. Use `selectionMode: "multiple"` to let the user choose several devices. No device is preselected. @@ -93,9 +105,8 @@ apps, open streams or start recordings. Stopped simulators/AVDs can be offered f build tasks; profiling candidates should already be running. Physical iOS uses the hardware UDID and requires a connected paired device. -This local MCP connection uses the OpenAI Extensions SDK's native form elicitation. -The host must advertise `extensions["openai/elicitation"].form`; unsupported hosts -return an explicit error. Codex owns layout and button labels, so the native form +The form uses the OpenAI Extensions SDK's native form elicitation and requires +`extensions["openai/elicitation"].form`. Codex owns layout and button labels, so the native form does not reproduce custom footer buttons from a design mockup. Thumbnails are illustrations, not captured app screens. diff --git a/docs/mcp-tools.md b/docs/mcp-tools.md index 23098dd5..01e30459 100644 --- a/docs/mcp-tools.md +++ b/docs/mcp-tools.md @@ -7,7 +7,7 @@ | `mobile_open_simulator` | Open the native panel, optionally targeting a platform, device kind, or discovered device ID | | `mobile_open_workspace` | Open fullscreen with the same optional device targeting | | `mobile_list_simulators` | Start the bundled backend if needed and list devices | -| `mobile_choose_devices` | Ask through the native inline form for one or several verified task targets | +| `mobile_show_devices` | Show devices in an inline card with Boot/Stop, or ask the user to choose task targets through the native form or the card | | `mobile_list_ios_devices` | Discover physical iPhones and iPads with USB/Wi-Fi, pairing state, UDID, and CoreDevice ID | | `mobile_start_baguette` | Retry or reconnect the bundled backend | | `mobile_boot_simulator` | Boot one listed device | diff --git a/docs/telemetry.md b/docs/telemetry.md index 49144463..ffe0d327 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -309,6 +309,14 @@ Counters record range selection, run toggles and Ask actions. No recording IDs, titles, selections or device measurements enter this telemetry. The comparison MCP operation retains the existing sampled server trace and handled-error path. +Devices cards use the `devices` surface and view with shared readiness, interaction, +frame pacing and teardown coverage. `ui.devices.process` measures result validation +in milliseconds, and `ui.devices.message_ack` ends when the host acknowledges the +Use message. Gauges report the listed and selected device counts. Counters record +`ui.devices.boot`, `ui.devices.stop`, their `_failed` variants and +`ui.devices.send`; boot and stop calls also keep the shared `ui.action` / +`ui.action.result` counters and sampled MCP client spans. Device names, IDs, runtimes, questions and chat text are never sent. + `ui.annotations.tree_processing` measures local element processing in milliseconds, including React Native nodes when available. Since 0.1.66, normal inspection validates flat records here; server-side tree flattening falls within `ui.annotations.inspection`, which measures the MCP inspection round trip, including native accessibility and optional Metro work. `ui.annotations.runtime_available` counts snapshots with runtime elements. `ui.annotations.inspection_fallback` counts native-tool retries. `ui.annotations.inspection_truncated` counts snapshots that reach the collector's work or measurement limits. Inspection timing includes failed calls and retries. It uses the current simulator surface and the same bounded timing windows as other UI measurements. Tree contents and selected regions are not sent to Sentry. Since 0.1.69, inspection timing also includes the bounded Metro source-map lookup. `ui.annotations.source_available` counts snapshots with at least one resolved source location; `ui.annotations.message_build` measures text construction for Send to chat. Source-map transport failures use the existing server error handler with a fixed message. Source paths, component names, creation stacks, note text and images are never sent to Sentry. @@ -397,11 +405,15 @@ mechanisms; they do not by themselves identify the cause of an ADB stall. Unhandled JavaScript errors and rejected promises, React render errors, and handled MCP tool failures produce issues. Expected stopped-device errors and cancelled operations are excluded. Sentry traces 10% of ordinary tool actions, continuing the UI trace through the MCP bridge. Frame reads, polling, discovery and pointer input are excluded from trace sampling. The SDK does not record MCP arguments or results. -Native device requests retain the ordinary sampled MCP trace. `device_picker.prepare` +Device requests retain the ordinary sampled MCP trace. `device_picker.prepare` measures candidate discovery/validation in milliseconds, excluding time spent waiting for the user. `device_picker.result` counts accept, cancel, decline, unsupported and -failed outcomes; `device_picker.selected` records only the number of selected devices. -Attributes contain only the selection mode and outcome. Unexpected handled failures +failed outcomes, plus `rendered` when the card is returned; `device_picker.selected` +records only the number of selected devices. Attributes contain only the selection +mode (`none` when browsing), outcome and a fixed `mode` (`form`, `card`, `browse`). +Since the picker became `mobile_show_devices`, `prepare` also covers discovery of +every device when candidates are omitted, so browse durations are not comparable +with earlier validation-only samples; filter by `mode`. Unexpected handled failures use fixed messages. Device IDs/names, app labels, questions, operation details and thumbnails are never sent to Sentry. The form is rendered by the host, so plugin UI readiness/render timing cannot measure that surface. diff --git a/scripts/smoke-claude-package.mjs b/scripts/smoke-claude-package.mjs index aafdceff..7926b060 100644 --- a/scripts/smoke-claude-package.mjs +++ b/scripts/smoke-claude-package.mjs @@ -41,10 +41,9 @@ try { transport.stderr?.on("data", chunk => { diagnostics = (diagnostics + chunk).slice(-5000); }); try { await client.connect(transport); - assert.match(client.getInstructions(), /otherwise in chat/); + assert.match(client.getInstructions(), /in chat when it is unsupported/); const { tools } = await client.listTools(); - assert.equal(tools.some(tool => tool.name === "mobile_choose_devices"), forms); - for (const name of ["mobile_open_workspace", "mobile_open_simulator", "mobile_list_simulators", "mobile_list_android_devices"]) { + for (const name of ["mobile_show_devices", "mobile_open_workspace", "mobile_open_simulator", "mobile_list_simulators", "mobile_list_android_devices"]) { assert.ok(tools.some(tool => tool.name === name), name); } const workspace = tools.find(tool => tool.name === "mobile_open_workspace"); diff --git a/skills/mobile-dev/SKILL.md b/skills/mobile-dev/SKILL.md index 2eebb234..52a2b3e7 100644 --- a/skills/mobile-dev/SKILL.md +++ b/skills/mobile-dev/SKILL.md @@ -22,7 +22,7 @@ device tools. 1. Open the panel beside the chat with `mobile_open_simulator` by default. For an existing app, open it when starting device work; for a new app, open it before the first device launch. Reuse an open Mobile Dev panel. Use `mobile_open_workspace` when the user asks for fullscreen. Use device-list tools without opening the panel when the user requests a tool-only workflow or the host cannot show panels. A request limited to planning, docs, code review, or compilation does not need a panel or a device launch. Both opening tools accept optional `platform` (`ios` or `android`), `kind` (`simulator`, `emulator`, or `physical`), and `deviceId`. Pass only intent established by the user's request or app project: for example, `{ platform: "android", kind: "emulator" }` for Android emulator work, or `{ platform: "ios", kind: "physical" }` for an explicitly requested physical iPhone. Omit undecided fields for open-ended work. Pass `deviceId` only after discovering the exact device; discovery can precede opening when needed to identify a named target. A missing matching device stays unselected instead of connecting to another target. Opening selects a stopped simulator or AVD without booting it. Follow later manual selections. -2. Use the task and app project to choose the platform. Follow the user's device choice; otherwise reuse a compatible device shared by the panel, then a suitable running simulator or emulator. If no suitable device runs, choose a compatible installed simulator or AVD. Pick among equivalent devices yourself for routine app development. When the intended target is ambiguous (for example, the app is open on several devices for a performance run), use `mobile_choose_devices` to ask through the native request form. Read device IDs from tool results or panel context; never invent them. +2. Use the task and app project to choose the platform. Follow the user's device choice; otherwise reuse a compatible device shared by the panel, then a suitable running simulator or emulator. If no suitable device runs, choose a compatible installed simulator or AVD. Pick among equivalent devices yourself for routine app development. When the intended target is ambiguous (for example, the app is open on several devices for a performance run), use `mobile_show_devices` with a `selectionMode` to ask the user. Read device IDs from tool results or panel context; never invent them. 3. Boot a chosen stopped iOS simulator with `mobile_boot_simulator`, or an Android AVD with `mobile_boot_android_emulator`. A mobile app development request permits choosing and booting a suitable installed simulator without a separate device-choice question. Reuse running devices without rebooting them. Follow the Android section below for device discovery and serials. Opening the panel alone does not boot a device. 4. Build the app with the app project's own tools and the chosen device ID; the project's run command may also install and launch it. To restart an already installed app, use `mobile_launch_app` instead of rebuilding. Follow the project's framework skills where available. Before starting an app dev server, check for an existing server from that project and reuse it. Keep the same device for the panel, build target, and agent control; read the panel's shared IDs before acting and follow later user selections. Leave the panel and app available while continuing app work. Do not launch the app when the user asks only for compilation. 5. Agent Device is temporarily disabled in this plugin while iterating on inline performance charts. Use the available Mobile Dev tools; do not search for Agent Device tools or start its CLI separately. @@ -43,20 +43,27 @@ These tools work on iOS simulators and Android devices with `{ platform, deviceI The panel's settings button opens the same controls under its App tab. Its Device tab also switches to a clean status bar (9:41, full battery and signal) for screenshots and simulates LTE, 3G, Edge, or offline networks. Those device settings are panel-only, so ask the user to set them. Android throttling applies right away to standard app networking such as fetch and OkHttp, leaves local dev servers like Metro at full speed, and is unavailable while the device uses its own proxy. iOS throttling applies to URLSession and fetch requests in apps launched after it was turned on, so relaunch the app after the user enables it. -### Choosing a task's devices +### Showing and choosing devices -When the user's task leaves the target unclear, discover suitable candidates with -`mobile_list_simulators`, `mobile_list_ios_devices`, or `mobile_list_android_devices`, -then call `mobile_choose_devices`. Pass a task-specific `message`, optional `context` -for operation details, and `devices` containing each candidate's `platform`, `kind` +When the user asks to list or see their devices, call `mobile_show_devices` with no +arguments. It shows an interactive card; reply with at most a one-line summary and +do not repeat the list. Keep the list tools for your own lookups. + +When the user's task leaves the target unclear, call `mobile_show_devices` with a +task-specific `message` and `selectionMode`. Omit `devices` to offer every selectable +device, or discover suitable candidates with `mobile_list_simulators`, +`mobile_list_ios_devices`, or `mobile_list_android_devices` and pass them. Pass +optional `context` for operation details, and `devices` containing each candidate's `platform`, `kind` (`simulator`, `emulator`, or `physical`) and `deviceId` from its discovered `udid`. Physical iOS uses the hardware UDID, never the CoreDevice ID. Set `appName` only after verifying that app on the candidate; for profiling, use `mobile_performance_sources` and offer only suitable running targets. -The default `selectionMode: "single"` asks for one device. Use `"multiple"` when -the task supports several targets or the user wants a comparison. The tool waits -for the user's answer and returns `action` and verified `devices`. After +Use `selectionMode: "single"` for one device and `"multiple"` when the task supports +several targets or the user wants a comparison. With a native request form the tool +waits for the user's answer and returns `action` and verified `devices`. Otherwise +it shows the choice in the card and returns immediately; wait for the user's chat +reply naming the devices, and do not start the pending task before it. After `action: "accept"`, keep each returned device ID with its platform and kind while performing the task. Selection does not boot a device, open a panel, launch an app, or start a recording. For Android CPU targets omit `kind`; for physical iOS CPU diff --git a/src/server/device-choice-tools.ts b/src/server/device-choice-tools.ts index b9466197..223a151d 100644 --- a/src/server/device-choice-tools.ts +++ b/src/server/device-choice-tools.ts @@ -1,11 +1,12 @@ import * as Sentry from "@sentry/node"; import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import type { CallToolResult, ClientCapabilities } from "@modelcontextprotocol/sdk/types.js"; +import { getUiCapability, registerAppTool, RESOURCE_MIME_TYPE } from "@modelcontextprotocol/ext-apps/server"; import type { OpenAIExtensions, OpenAIForm, OpenAIFormOption } from "@openai/mcp-extensions/server"; -import { deviceChoiceInputSchema } from "../shared/device-choice.ts"; +import { androidChoices, deviceChoiceInputSchema, DEVICES_URI, isRunningDevice, isSelectableDevice, physicalIosChoices, simulatorChoices } from "../shared/device-choice.ts"; import type { DeviceChoice, DeviceChoiceInput, DeviceChoiceReference } from "../shared/device-choice.ts"; import type { PhysicalIosDevice } from "../shared/ios-devices.ts"; -import type { SimulatorDevice, Status } from "../shared/protocol.ts"; +import type { Status } from "../shared/protocol.ts"; import { errorMessage } from "../shared/protocol.ts"; import { captureServerError } from "./telemetry.ts"; import { SimulatorUnavailableError } from "./simulator-unavailable.ts"; @@ -16,6 +17,9 @@ export type DeviceChoiceSources = { android: () => Promise; physicalIos: () => Promise; }; +type DeviceChoiceMode = "form" | "card" | "browse"; + +const FORM_OPTION_LIMIT = 20; export function deviceChoiceForm(input: DeviceChoiceInput, devices: DeviceChoice[]): OpenAIForm { const options: OpenAIFormOption[] = devices.map((device, index) => { @@ -40,6 +44,12 @@ export function deviceChoiceForm(input: DeviceChoiceInput, devices: DeviceChoice }; } +async function statusDevices(read: () => Promise) { + const status = await read(); + if (status.connected === false) throw new SimulatorUnavailableError(status.error ?? "Device discovery is unavailable."); + return status; +} + export async function resolveDeviceChoices(references: DeviceChoiceReference[], sources: DeviceChoiceSources): Promise { const keys = new Set(); for (const reference of references) { @@ -47,75 +57,107 @@ export async function resolveDeviceChoices(references: DeviceChoiceReference[], if (keys.has(key)) throw new SimulatorUnavailableError("Each candidate device must appear only once."); keys.add(key); } - const reads: Promise[] = []; - async function statusDevices(read: () => Promise) { - const status = await read(); - if (status.connected === false) throw new SimulatorUnavailableError(status.error ?? "Device discovery is unavailable."); - return status.devices; - } - const hasSimulators = references.some(device => device.platform === "ios" && device.kind === "simulator"); - const hasAndroid = references.some(device => device.platform === "android"); - const hasPhysicalIos = references.some(device => device.platform === "ios" && device.kind === "physical"); - if (hasSimulators) { - const read = statusDevices(sources.simulators); - reads.push(read); - } - if (hasAndroid) { - const read = statusDevices(sources.android); - reads.push(read); - } - if (hasPhysicalIos) { - const read = sources.physicalIos(); - reads.push(read); - } + const reads: Promise[] = []; + if (references.some(device => device.platform === "ios" && device.kind === "simulator")) reads.push(statusDevices(sources.simulators).then(simulatorChoices)); + if (references.some(device => device.platform === "android")) reads.push(statusDevices(sources.android).then(androidChoices)); + if (references.some(device => device.platform === "ios" && device.kind === "physical")) reads.push(sources.physicalIos().then(physicalIosChoices)); const lists = await Promise.all(reads); const discovered = lists.flat(); return references.map(reference => { - const device = discovered.find(candidate => { - const platform = candidate.platform ?? "ios"; - const kind = candidate.kind ?? "simulator"; - return platform === reference.platform && kind === reference.kind && candidate.udid === reference.deviceId; - }); + const device = discovered.find(candidate => candidate.platform === reference.platform && candidate.kind === reference.kind && candidate.deviceId === reference.deviceId); if (device === undefined) throw new SimulatorUnavailableError("A candidate device is no longer available. Refresh discovery before requesting a selection."); - if (device.kind === "physical" && device.platform === "ios") { - if (device.state !== "connected" || device.pairingState !== "paired") throw new SimulatorUnavailableError("Connect and pair the candidate iPhone or iPad before requesting a selection."); - } else if (device.state !== "Booted" && device.state !== "Shutdown") { + if (isSelectableDevice(device) === false) { + if (device.kind === "physical" && device.platform === "ios") throw new SimulatorUnavailableError("Connect and pair the candidate iPhone or iPad before requesting a selection."); throw new SimulatorUnavailableError("A candidate device is offline or unauthorized. Reconnect it before requesting a selection."); } - return { ...reference, name: device.name, runtime: device.runtime, state: device.state }; + return { ...reference, name: device.name, runtime: device.runtime, state: device.state, ...(device.transportType ? { transportType: device.transportType } : {}), ...(device.pairingState ? { pairingState: device.pairingState } : {}) }; }); } +// Lists every device; one unavailable backend (no Android SDK, older Xcode) leaves the others listed. +export async function discoverDeviceChoices(sources: DeviceChoiceSources): Promise<{ devices: DeviceChoice[]; errors: string[] }> { + const reads = await Promise.allSettled([ + statusDevices(sources.simulators).then(simulatorChoices), + statusDevices(sources.android).then(androidChoices), + sources.physicalIos().then(physicalIosChoices), + ]); + const devices: DeviceChoice[] = []; + const errors: string[] = []; + for (const read of reads) { + if (read.status === "fulfilled") devices.push(...read.value); + else errors.push(errorMessage(read.reason)); + } + return { devices, errors }; +} + export function supportsDeviceChoiceForms(capabilities: ClientCapabilities | undefined): boolean { const formCapability = capabilities?.extensions?.["openai/elicitation"]; return typeof formCapability === "object" && formCapability !== null && "form" in formCapability; } +// Codex renders app cards without advertising the standard extension; its form capability identifies it. +export function supportsDeviceCards(capabilities: ClientCapabilities | undefined): boolean { + return getUiCapability(capabilities)?.mimeTypes?.includes(RESOURCE_MIME_TYPE) === true || supportsDeviceChoiceForms(capabilities); +} + +function summary(devices: DeviceChoice[]) { + const running = devices.filter(isRunningDevice).length; + const physical = devices.filter(device => device.kind === "physical" && !isRunningDevice(device)).length; + const parts = [`${running} running`]; + if (physical) parts.push(`${physical} disconnected physical`); + parts.push(`${devices.length - running - physical} stopped`); + return parts.join(", "); +} + export function registerDeviceChoiceTools(server: McpServer, extensions: Pick, sources: DeviceChoiceSources) { - server.registerTool("mobile_choose_devices", { - title: "Choose a device", - description: "Ask the user which device(s) to use through a native inline request form when a mobile task has ambiguous targets. Discover candidates first with mobile_list_simulators, mobile_list_ios_devices or mobile_list_android_devices; pass their IDs, platform and kind. Add appName only when you have verified the app is on that device. message explains the task/question; context supplies operation details such as duration. selectionMode multiple permits selecting several. Wait for action accept, then use only returned devices. cancel/decline means do not perform the pending task. Does not boot devices, launch apps or start recording. Requires a host supporting OpenAI form elicitation.", + registerAppTool(server, "mobile_show_devices", { + title: "Devices", + description: "Show the user's iOS simulators, Android emulators and physical devices in an interactive card where they can boot, stop and pick devices. Use it whenever the user asks to list, show or see devices, or when a task has ambiguous targets. Omit devices to list everything discovered, or pass candidate IDs, platform and kind from mobile_list_simulators, mobile_list_ios_devices or mobile_list_android_devices. To ask the user to choose, set selectionMode (single or multiple) with a message explaining the task; context supplies operation details such as duration. Add appName only when you have verified the app is on that device. With a native request form the call waits: on action accept use only the returned devices; cancel/decline means do not perform the pending task. Otherwise the card shows the choice and the user replies in chat; wait for that reply before starting the task. Do not repeat the device list the card shows. This tool never boots devices, launches apps or starts recording; only the user can boot or stop a device from the card. For internal lookups use the list tools instead.", inputSchema: deviceChoiceInputSchema, annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false }, + _meta: { ui: { resourceUri: DEVICES_URI } }, }, async (input: DeviceChoiceInput, extra): Promise => { - const attributes = { selection_mode: input.selectionMode }; + const capabilities = server.server.getClientCapabilities(); + const cards = supportsDeviceCards(capabilities); + let mode: DeviceChoiceMode = "browse"; + if (input.selectionMode !== undefined) mode = supportsDeviceChoiceForms(capabilities) ? "form" : "card"; + const attributes = { selection_mode: input.selectionMode ?? "none", mode }; let stage: "prepare" | "request" | "validate" = "prepare"; try { - if (!supportsDeviceChoiceForms(server.server.getClientCapabilities())) { + if (mode === "card" && cards === false) { Sentry.metrics.count("device_picker.result", 1, { attributes: { ...attributes, outcome: "unsupported" } }); - return { isError: true, content: [{ type: "text", text: "This host does not support OpenAI native request forms. No device was selected and the pending task must not start." }] }; + return { isError: true, content: [{ type: "text", text: "This host can show neither a native request form nor an app card. No device was selected; ask the user in chat and do not start the pending task until they choose." }] }; } const startedAt = performance.now(); let devices: DeviceChoice[]; - try { devices = await resolveDeviceChoices(input.devices, sources); } - finally { + let errors: string[] = []; + try { + if (input.devices) devices = await resolveDeviceChoices(input.devices, sources); + else { + ({ devices, errors } = await discoverDeviceChoices(sources)); + if (mode !== "browse") devices = devices.filter(isSelectableDevice); + // The native form offers a bounded list; running devices are the likeliest targets. + if (mode === "form") devices = devices.sort((a, b) => Number(isRunningDevice(b)) - Number(isRunningDevice(a))).slice(0, FORM_OPTION_LIMIT); + if (devices.length === 0 && errors.length > 0) throw new SimulatorUnavailableError(errors.join(" ")); + } + } finally { const duration = performance.now() - startedAt; Sentry.metrics.distribution("device_picker.prepare", duration, { unit: "millisecond", attributes }); } extra.signal.throwIfAborted(); + if (mode !== "form") { + Sentry.metrics.count("device_picker.result", 1, { attributes: { ...attributes, outcome: "rendered" } }); + const data = { mode: mode === "card" ? "choose" : "browse", ...(input.selectionMode ? { selectionMode: input.selectionMode } : {}), ...(input.message ? { message: input.message } : {}), ...(input.context ? { context: input.context } : {}), devices, errors }; + let text = cards ? `Showing ${devices.length} devices in the Mobile Dev card (${summary(devices)}). Do not repeat the list.` : JSON.stringify(devices); + if (errors.length) text += ` Some devices could not be listed: ${errors.join(" ")}`; + if (mode === "card") text += " Wait for the user to choose in the card and reply in chat before starting the pending task."; + return { content: [{ type: "text", text }], structuredContent: data }; + } + if (devices.length === 0) throw new SimulatorUnavailableError("No device can be selected. Boot, connect or pair a device first."); + const references: DeviceChoiceReference[] = devices.map(({ platform, kind, deviceId, appName }) => ({ platform, kind, deviceId, ...(appName ? { appName } : {}) }) as DeviceChoiceReference); const requestedSchema = deviceChoiceForm(input, devices); stage = "request"; - const answer = await extensions.elicitInput({ mode: "form", message: input.message, requestedSchema }, { signal: extra.signal, timeout: 600_000 }); + const answer = await extensions.elicitInput({ mode: "form", message: input.message!, requestedSchema }, { signal: extra.signal, timeout: 600_000 }); if (answer.action !== "accept") { Sentry.metrics.count("device_picker.result", 1, { attributes: { ...attributes, outcome: answer.action } }); return { content: [{ type: "text", text: "Device selection cancelled. Do not start the pending task." }], structuredContent: { action: answer.action, devices: [] } }; @@ -130,7 +172,7 @@ export function registerDeviceChoiceTools(server: McpServer, extensions: Pick key === `device-${candidateIndex + 1}`); if (index < 0) throw new SimulatorUnavailableError("The selected device was not offered. Request a new selection."); - selected.push(input.devices[index]); + selected.push(references[index]); } const current = await resolveDeviceChoices(selected, sources); extra.signal.throwIfAborted(); diff --git a/src/server/plugin.ts b/src/server/plugin.ts index b9283758..b792c536 100644 --- a/src/server/plugin.ts +++ b/src/server/plugin.ts @@ -23,7 +23,8 @@ import { registerAndroidTools } from "./android-tools.ts"; import { Baguette } from "./baguette.ts"; import { registerIosMirrorTools } from "./ios-mirror-tools.ts"; import { listIosDevices, registerIosDeviceTools } from "./ios-devices.ts"; -import { registerDeviceChoiceTools, supportsDeviceChoiceForms } from "./device-choice-tools.ts"; +import { registerDeviceChoiceTools } from "./device-choice-tools.ts"; +import { DEVICES_URI } from "../shared/device-choice.ts"; import { StreamSessions } from "./stream-sessions.ts"; import { LogSessions } from "./log-sessions.ts"; import { registerLogTools } from "./log-tools.ts"; @@ -96,22 +97,16 @@ export async function createPlugin(html: string | (() => Promise), b const cpu = selectedCpu; const fps = new DisplayFpsSessions(); const server = new McpServer({ name: "mobile-dev", version: PLUGIN_VERSION }, { - instructions: "For mobile app development, open mobile_open_simulator before the first device launch, or reuse the open view. Use selected device IDs. If a task has ambiguous device targets, discover candidates and ask the user with mobile_choose_devices when it is available, otherwise in chat; proceed only after they choose. For a local CPU or React profile path, use mobile_analyze_profile to show TraceSift findings, then wait for the user to choose Investigate. See the Mobile Dev skill, when installed, for device control, logs, profiles, and performance workflows.", + instructions: "For mobile app development, open mobile_open_simulator before the first device launch, or reuse the open view. Use selected device IDs. When the user asks to list or see devices, show them with mobile_show_devices instead of writing a list. If a task has ambiguous device targets, ask the user with mobile_show_devices and a selectionMode, or in chat when it is unsupported; proceed only after they choose. For a local CPU or React profile path, use mobile_analyze_profile to show TraceSift findings, then wait for the user to choose Investigate. See the Mobile Dev skill, when installed, for device control, logs, profiles, and performance workflows.", }); instrumentMcpServer(server); const updates = new PluginUpdates(); - addToolIcons(server, { mobile_open_workspace: PHONE_ICONS, mobile_open_simulator: PHONE_ICONS, mobile_choose_devices: PHONE_ICONS }); + addToolIcons(server, { mobile_open_workspace: PHONE_ICONS, mobile_open_simulator: PHONE_ICONS, mobile_show_devices: PHONE_ICONS }); registerPluginUpdateTools(server, updates); const extensions = new OpenAIExtensions(server); - // The native form is a client extension; list the tool only for hosts that render it. - let deviceChoiceRegistered = false; - server.server.oninitialized = () => { - if (deviceChoiceRegistered || !supportsDeviceChoiceForms(server.server.getClientCapabilities())) return; - deviceChoiceRegistered = true; - registerDeviceChoiceTools(server, extensions, { - simulators: () => baguette.start(), android: () => android.list(), physicalIos: listIosDevices, - }); - }; + registerDeviceChoiceTools(server, extensions, { + simulators: () => baguette.start(), android: () => android.list(), physicalIos: listIosDevices, + }); registerIosDeviceTools(server); const closeIosMirror = registerIosMirrorTools(server, APP_URI); registerLogTools(server, logs, baguette); @@ -206,6 +201,7 @@ export async function createPlugin(html: string | (() => Promise), b ["mobile-dev-recording", RECORDING_URI, "recording"], ["mobile-dev-comparison", COMPARISON_URI, "comparison"], ["mobile-dev-tracesift", TRACESIFT_URI, "tracesift"], + ["mobile-dev-devices", DEVICES_URI, "devices"], ]) { const readCard = async (uri: URL) => { const resource = typeof html === "string" ? { html } : await html(); diff --git a/src/server/telemetry.ts b/src/server/telemetry.ts index 9737d6fa..3154fd30 100644 --- a/src/server/telemetry.ts +++ b/src/server/telemetry.ts @@ -250,7 +250,7 @@ const resourceKinds: Record = { "android-stream:": "android_video", "ios-video:": "ios_video", "ui:": "ui", }; const allowedContext: Record = { - surface: ["logs", "performance", "storage", "simulator", "recording", "comparison", "tracesift"], view: ["panel", "workspace", "recording", "comparison", "tracesift"], layout: ["ios", "android", "both", "none"], + surface: ["logs", "performance", "storage", "simulator", "recording", "comparison", "tracesift", "devices"], view: ["panel", "workspace", "recording", "comparison", "tracesift", "devices"], layout: ["ios", "android", "both", "none"], device_platform: ["ios", "android", "mixed"], device_kind: ["physical", "simulator", "emulator", "none"], }; diff --git a/src/shared/device-choice.ts b/src/shared/device-choice.ts index 93f0b2b0..18c446ff 100644 --- a/src/shared/device-choice.ts +++ b/src/shared/device-choice.ts @@ -1,4 +1,9 @@ import { z } from "zod"; +import { PLUGIN_VERSION } from "./version.ts"; +import type { PhysicalIosDevice } from "./ios-devices.ts"; +import type { Status } from "./protocol.ts"; + +export const DEVICES_URI = `ui://mobile-dev/${PLUGIN_VERSION}/devices.html`; const deviceId = z.string().min(1).max(256).regex(/^[a-zA-Z0-9_.:-]+$/); const appName = z.string().trim().min(1).max(128).optional(); @@ -20,12 +25,60 @@ export const deviceChoiceReferenceSchema = z.discriminatedUnion("platform", [ } }); -const message = z.string().trim().min(1).max(1024); +const message = z.string().trim().min(1).max(1024).optional(); const context = z.string().trim().min(1).max(512).optional(); -const selectionMode = z.enum(["single", "multiple"]).default("single"); -const devices = z.array(deviceChoiceReferenceSchema).min(1).max(20); +const selectionMode = z.enum(["single", "multiple"]).optional(); +const devices = z.array(deviceChoiceReferenceSchema).min(1).max(20).optional(); -export const deviceChoiceInputSchema = z.object({ message, context, selectionMode, devices }).strict(); +export const deviceChoiceInputSchema = z.object({ message, context, selectionMode, devices }).strict().superRefine((input, issues) => { + if (input.selectionMode !== undefined && input.message === undefined) issues.addIssue({ code: "custom", path: ["message"], message: "Explain the task when asking the user to choose." }); +}); export type DeviceChoiceInput = z.infer; export type DeviceChoiceReference = z.infer; -export type DeviceChoice = DeviceChoiceReference & { name: string; runtime: string; state: string }; +export type DeviceChoice = DeviceChoiceReference & { name: string; runtime: string; state: string; transportType?: string; pairingState?: string }; + +const deviceChoiceSchema = z.object({ + platform: z.enum(["ios", "android"]), kind: z.enum(["simulator", "emulator", "physical"]), deviceId: z.string(), appName: z.string().optional(), + name: z.string(), runtime: z.string(), state: z.string(), transportType: z.string().optional(), pairingState: z.string().optional(), +}); +// Browse and card results list devices for the user; a form result carries its outcome instead. +export const devicesViewSchema = z.union([ + z.object({ action: z.enum(["accept", "cancel", "decline"]), devices: z.array(deviceChoiceSchema) }), + z.object({ + mode: z.enum(["browse", "choose"]), selectionMode: z.enum(["single", "multiple"]).optional(), + message: z.string().optional(), context: z.string().optional(), + devices: z.array(deviceChoiceSchema), errors: z.array(z.string()), + }), +]); +export type DevicesView = z.infer; + +export function simulatorChoices(status: Status): DeviceChoice[] { + return status.devices.flatMap(device => device.platform === "android" || device.kind === "physical" ? [] + : [{ platform: "ios" as const, kind: "simulator" as const, deviceId: device.udid, name: device.name, runtime: device.runtime, state: device.state }]); +} + +export function androidChoices(status: Status): DeviceChoice[] { + return status.devices.flatMap((device): DeviceChoice[] => { + if (device.platform !== "android") return []; + const choice = { platform: "android" as const, deviceId: device.udid, name: device.name, runtime: device.runtime, state: device.state }; + if (device.kind === "physical") return [{ ...choice, kind: "physical" as const, transportType: device.transportType }]; + return [{ ...choice, kind: "emulator" as const }]; + }); +} + +export function physicalIosChoices(devices: PhysicalIosDevice[]): DeviceChoice[] { + return devices.map(device => ({ + platform: "ios", kind: "physical", deviceId: device.udid, name: device.name, runtime: device.runtime, state: device.state, + transportType: device.transportType, pairingState: device.pairingState, + })); +} + +export function isRunningDevice(device: DeviceChoice): boolean { + return device.platform === "ios" && device.kind === "physical" ? device.state === "connected" : device.state === "Booted"; +} + +// The same rule the native form applies before offering a device. +export function isSelectableDevice(device: DeviceChoice): boolean { + if (device.platform === "ios" && device.kind === "physical") return device.state === "connected" && device.pairingState === "paired"; + return device.state === "Booted" || device.state === "Shutdown"; +} diff --git a/src/shared/telemetry.ts b/src/shared/telemetry.ts index a45070d3..f90e92f6 100644 --- a/src/shared/telemetry.ts +++ b/src/shared/telemetry.ts @@ -19,7 +19,7 @@ export function validateTelemetryEnvironment(value: unknown): TelemetryEnvironme throw new Error("Sentry environment must be development or release."); } -export type Surface = "logs" | "performance" | "storage" | "simulator" | "recording" | "comparison" | "tracesift"; +export type Surface = "logs" | "performance" | "storage" | "simulator" | "recording" | "comparison" | "tracesift" | "devices"; export type TelemetryAttributes = Record; type Options = NonNullable[0]>; type ErrorEvent = Parameters>[0]; diff --git a/src/ui/app.tsx b/src/ui/app.tsx index 607f8da9..4944c6e3 100644 --- a/src/ui/app.tsx +++ b/src/ui/app.tsx @@ -4,8 +4,10 @@ import "./style.css"; import { startRecordingApp } from "./recording-app.tsx"; import { startComparisonApp } from "./comparison-app.tsx"; import { startTraceSiftApp } from "./tracesift-app.tsx"; +import { startDevicesApp } from "./devices-app.tsx"; if (document.documentElement.dataset.view === "recording") startRecordingApp(); else if (document.documentElement.dataset.view === "comparison") startComparisonApp(); else if (document.documentElement.dataset.view === "tracesift") startTraceSiftApp(); +else if (document.documentElement.dataset.view === "devices") startDevicesApp(); else void import("./workspace-app.tsx"); diff --git a/src/ui/components/devices-card.tsx b/src/ui/components/devices-card.tsx new file mode 100644 index 00000000..b690ede7 --- /dev/null +++ b/src/ui/components/devices-card.tsx @@ -0,0 +1,103 @@ +import { useState, useSyncExternalStore } from "react"; +import { ChevronRightIcon, GlassesIcon, LoaderCircleIcon, MessageCircleIcon, PlayIcon, SmartphoneIcon, SquareIcon, TabletIcon } from "lucide-react"; +import { isRunningDevice } from "../../shared/device-choice.ts"; +import type { DeviceChoice } from "../../shared/device-choice.ts"; +import { physicalConnectionLabel } from "../../shared/ios-devices.ts"; +import { canBootDevice, canStopDevice, deviceKey, deviceKindLabel } from "../devices-controller.ts"; +import type { DevicesController } from "../devices-controller.ts"; +import { Button } from "./ui/button"; +import { Input } from "./ui/input"; + +const FILTER_THRESHOLD = 8; + +function stateLabel(device: DeviceChoice) { + if (device.state === "Booted") return device.kind === "physical" ? "Connected" : "Running"; + if (device.state === "Shutdown") return "Stopped"; + return device.state.charAt(0).toUpperCase() + device.state.slice(1); +} + +function DeviceIcon({ device }: { device: DeviceChoice }) { + const Icon = /vision/i.test(device.runtime) ? GlassesIcon : /ipad|tablet|pixel.*tab/i.test(device.name) ? TabletIcon : SmartphoneIcon; + return {isRunningDevice(device) && }; +} + +function DeviceRow({ controller, device, showRuntime }: { controller: DevicesController; device: DeviceChoice; showRuntime: boolean }) { + const state = useSyncExternalStore(controller.subscribe, controller.getSnapshot); + const key = deviceKey(device); + const view = state.view; + const single = view !== undefined && "mode" in view && view.selectionMode === "single"; + const selectable = state.canMessage && controller.canSelect(device); + const details = [deviceKindLabel(device)]; + if (showRuntime && device.runtime && device.runtime !== "Android") details.push(device.runtime); + if (device.kind === "physical") details.push(physicalConnectionLabel(device.transportType)); + if (device.appName) details.push(device.appName); + details.push(stateLabel(device)); + const pending = state.pendingKey === key; + const busy = state.pendingKey !== "" || state.sending; + const control = selectable && controller.toggle(device)} aria-label={`Select ${device.name}`} />; + return
  • + + {canBootDevice(device) && } + {canStopDevice(device) && } +
    • ; +} + +function Section({ title, children, collapsible = false, defaultOpen = true }: { title: string; children: React.ReactNode; collapsible?: boolean; defaultOpen?: boolean }) { + const [open, setOpen] = useState(defaultOpen); + if (collapsible === false) return

    {title}

      {children}
    ; + return
    + + {open &&
      {children}
    } +
    ; +} + +export function DevicesCard({ controller }: { controller: DevicesController }) { + const state = useSyncExternalStore(controller.subscribe, controller.getSnapshot); + const [filter, setFilter] = useState(""); + const view = state.view; + if (view === undefined) return
    {state.error || "Loading devices…"}
    ; + if ("action" in view) { + const text = view.action === "accept" ? `Selected ${view.devices.map(device => device.name).join(", ")}` : "No device selected"; + return
    {text}
    ; + } + const query = filter.trim().toLowerCase(); + const visible = query ? state.devices.filter(device => `${device.name} ${device.runtime}`.toLowerCase().includes(query)) : state.devices; + const running = visible.filter(isRunningDevice); + const offline = visible.filter(device => device.kind === "physical" && !isRunningDevice(device)); + const stopped = visible.filter(device => device.kind !== "physical" && !isRunningDevice(device)); + const runtimes = new Map(); + for (const device of stopped) runtimes.set(device.runtime || "Other", [...runtimes.get(device.runtime || "Other") ?? [], device]); + // iOS first, then other Apple runtimes, then Android; newest version first within each. + const rank = (runtime: string) => runtime.startsWith("iOS") ? 0 : runtime === "Android" ? 2 : 1; + const groups = [...runtimes].sort(([a], [b]) => rank(a) - rank(b) || b.localeCompare(a, undefined, { numeric: true })); + const expanded = query !== "" || stopped.length <= FILTER_THRESHOLD; + const choosing = view.mode === "choose"; + const count = state.selected.size; + return
    +
    +

    {view.message ?? "Devices"}

    + {view.context &&

    {view.context}

    } +
    + {state.devices.length > FILTER_THRESHOLD && setFilter(event.target.value)} aria-label="Filter devices" className="h-8" />} + {visible.length === 0 &&

    {query ? "No devices match this filter." : "No devices found."}

    } + {running.length > 0 &&
    {running.map(device => )}
    } + {offline.length > 0 &&
    {offline.map(device => )}
    } + {groups.map(([runtime, devices]) =>
    {devices.map(device => )}
    )} + {view.errors.map(error =>

    {error}

    )} + {state.error &&

    {state.error}

    } + {state.sent &&

    Choice sent to chat.

    } + {state.canMessage === false && choosing &&

    Reply in chat with the device to use.

    } + {state.canMessage && state.sent === false && (choosing || count > 0) &&
    + +
    } +
    ; +} diff --git a/src/ui/devices-app.tsx b/src/ui/devices-app.tsx new file mode 100644 index 00000000..1c6d4cc7 --- /dev/null +++ b/src/ui/devices-app.tsx @@ -0,0 +1,41 @@ +import { App, applyDocumentTheme, applyHostFonts, applyHostStyleVariables } from "@modelcontextprotocol/ext-apps"; +import { createRoot } from "react-dom/client"; +import { DevicesController } from "./devices-controller.ts"; +import { DevicesCard } from "./components/devices-card.tsx"; +import { ErrorBoundary, captureUiError, setUiSurface, startUiTelemetry, stopUiTelemetry } from "./telemetry.ts"; +import { PLUGIN_VERSION } from "../shared/version.ts"; + +export function startDevicesApp() { + const app = new App({ name: "mobile-dev-devices", version: PLUGIN_VERSION }); + startUiTelemetry(app); + setUiSurface("devices"); + const controller = new DevicesController(app); + const root = createRoot(document.getElementById("root")!); + root.render(These devices could not render. Ask again to list them.

    }>    ); + function hostChanged() { + const host = app.getHostContext(); + if (host?.theme) applyDocumentTheme(host.theme); + if (host?.styles?.variables) applyHostStyleVariables(host.styles.variables); + if (host?.styles?.css?.fonts) applyHostFonts(host.styles.css.fonts); + controller.hostChanged(); + } + app.addEventListener("hostcontextchanged", hostChanged); + app.ontoolresult = result => { + try { controller.accept(result); } + catch (error) { captureUiError(error, "devices.result"); } + }; + let closing: Promise | undefined; + function close() { + if (closing) return closing; + controller.dispose(); + root.unmount(); + window.removeEventListener("pagehide", pageHide); + app.removeEventListener("hostcontextchanged", hostChanged); + closing = stopUiTelemetry(); + return closing; + } + function pageHide(event: PageTransitionEvent) { if (event.persisted === false) void close(); } + window.addEventListener("pagehide", pageHide); + app.onteardown = async () => { await close(); return {}; }; + void app.connect().then(hostChanged).catch(error => { captureUiError(error, "devices.connect"); }); +} diff --git a/src/ui/devices-controller.ts b/src/ui/devices-controller.ts new file mode 100644 index 00000000..9f7d33f5 --- /dev/null +++ b/src/ui/devices-controller.ts @@ -0,0 +1,151 @@ +import { chatMessage } from "./chat-message.ts"; +import type { App } from "@modelcontextprotocol/ext-apps"; +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { androidChoices, devicesViewSchema, isSelectableDevice, simulatorChoices } from "../shared/device-choice.ts"; +import type { DeviceChoice, DevicesView } from "../shared/device-choice.ts"; +import type { Status } from "../shared/protocol.ts"; +import { captureUiError, countUiEvent, markUiSurfaceReady, recordUiTiming, setUiGauge, setUiSurface, setUiTelemetryContext } from "./telemetry.ts"; + +type DevicesState = { + view?: DevicesView; + devices: DeviceChoice[]; + selected: ReadonlySet; + pendingKey: string; + canMessage: boolean; + sending: boolean; + sent: boolean; + error: string; +}; + +export function deviceKey(device: Pick) { return `${device.platform}:${device.deviceId}`; } +export function canBootDevice(device: DeviceChoice) { return device.kind !== "physical" && device.state === "Shutdown"; } +export function canStopDevice(device: DeviceChoice) { + if (device.state !== "Booted") return false; + return device.platform === "ios" ? device.kind === "simulator" : device.kind === "emulator" && /^emulator-\d+$/.test(device.deviceId); +} +export function deviceKindLabel(device: DeviceChoice) { + const platform = device.platform === "ios" ? "iOS" : "Android"; + return `${platform} ${device.kind === "physical" ? "device" : device.kind}`; +} + +export class DevicesController { + private readonly app: App; + private state: DevicesState = { devices: [], selected: new Set(), pendingKey: "", canMessage: false, sending: false, sent: false, error: "" }; + private listeners = new Set<() => void>(); + private disposed = false; + private readyAt = performance.now(); + + constructor(app: App) { this.app = app; } + subscribe = (listener: () => void) => { + this.listeners.add(listener); + return () => { this.listeners.delete(listener); }; + }; + getSnapshot = () => this.state; + private update(next: Partial) { + if (this.disposed) return; + this.state = { ...this.state, ...next }; + for (const listener of this.listeners) listener(); + } + get choosing() { return this.state.view !== undefined && "mode" in this.state.view && this.state.view.mode === "choose"; } + canSelect(device: DeviceChoice) { + const view = this.state.view; + if (view === undefined || "action" in view || this.state.sent) return false; + return view.mode === "browse" || isSelectableDevice(device); + } + hostChanged() { + const capabilities = this.app.getHostCapabilities(); + const canMessage = Boolean(capabilities?.message?.text); + this.update({ canMessage }); + } + accept(result: CallToolResult) { + if (this.disposed) return; + if (result.isError) { + const texts = result.content.filter(item => item.type === "text"); + const error = texts.map(item => item.text).join("\n"); + this.update({ error }); + return; + } + if (result.structuredContent?.devices === undefined) return; + const startedAt = performance.now(); + const view = devicesViewSchema.parse(result.structuredContent); + setUiSurface("devices"); + const platforms = new Set(view.devices.map(device => device.platform)); + const [platform] = platforms; + setUiTelemetryContext({ device_platform: platforms.size === 1 ? platform : "mixed", device_kind: "none" }); + setUiGauge("ui.devices.count", view.devices.length); + this.update({ view, devices: view.devices as DeviceChoice[], selected: new Set(), sent: false, error: "" }); + recordUiTiming("ui.devices.process", performance.now() - startedAt); + markUiSurfaceReady(this.readyAt); + } + toggle = (device: DeviceChoice) => { + if (this.canSelect(device) === false) return; + const key = deviceKey(device); + const view = this.state.view; + const single = view !== undefined && "mode" in view && view.selectionMode === "single"; + const selected = new Set(single ? [] : this.state.selected); + if (this.state.selected.has(key)) selected.delete(key); + else selected.add(key); + this.update({ selected }); + }; + boot = (device: DeviceChoice) => this.changeState(device, "boot"); + stop = (device: DeviceChoice) => this.changeState(device, "shutdown"); + private async changeState(device: DeviceChoice, action: "boot" | "shutdown") { + const allowed = action === "boot" ? canBootDevice(device) : canStopDevice(device); + if (allowed === false || this.state.pendingKey || this.disposed) return; + const key = deviceKey(device); + this.update({ pendingKey: key, error: "" }); + const event = action === "boot" ? "ui.devices.boot" : "ui.devices.stop"; + try { + const ios = device.platform === "ios"; + const name = ios ? `mobile_${action}_simulator` : `mobile_${action}_android_emulator`; + const result = await this.app.callServerTool({ name, arguments: ios ? { udid: device.deviceId } : { deviceId: device.deviceId } }, { timeout: 150_000 }); + if (result.isError) { + const texts = result.content.filter(item => item.type === "text"); + throw new Error(texts.map(item => item.text).join("\n") || `Could not ${action === "boot" ? "boot" : "stop"} ${device.name}.`); + } + const status = result.structuredContent as Status; + this.refresh(ios ? simulatorChoices(status) : androidChoices(status), device.platform); + countUiEvent(event); + } catch (error) { + countUiEvent(`${event}_failed`); + this.update({ error: error instanceof Error ? error.message : `Could not change ${device.name}.` }); + } finally { this.update({ pendingKey: "" }); } + } + // An Android AVD gets a new serial when it boots, so emulators also match by AVD name. + private refresh(fresh: DeviceChoice[], platform: DeviceChoice["platform"]) { + const selected = new Set(this.state.selected); + const devices = this.state.devices.map(device => { + if (device.platform !== platform || device.kind === "physical" && platform === "ios") return device; + const match = fresh.find(candidate => candidate.kind === device.kind && candidate.deviceId === device.deviceId) + ?? (device.kind === "emulator" ? fresh.find(candidate => candidate.kind === "emulator" && candidate.name === device.name) : undefined); + if (match === undefined) return device; + const next = { ...device, deviceId: match.deviceId, state: match.state, runtime: match.runtime } as DeviceChoice; + if (selected.delete(deviceKey(device))) selected.add(deviceKey(next)); + return next; + }); + this.update({ devices, selected }); + } + async send() { + const { devices, selected, canMessage, sending } = this.state; + const chosen = devices.filter(device => selected.has(deviceKey(device))); + if (chosen.length === 0 || canMessage === false || sending || this.disposed) return; + this.update({ sending: true, error: "" }); + const items = chosen.map(device => `${device.name} (${deviceKindLabel(device)}, ID ${device.deviceId})`); + const prompt = `Use ${items.length === 1 ? items[0] : `these devices: ${items.join(", ")}`}.`; + const message = chatMessage(this.app, prompt); + const startedAt = performance.now(); + try { + const result = await this.app.sendMessage(message); + if (result.isError) throw new Error("The host could not send this choice to chat."); + countUiEvent("ui.devices.send"); + recordUiTiming("ui.devices.message_ack", performance.now() - startedAt); + setUiGauge("ui.devices.selected", chosen.length); + this.update({ sent: this.choosing, selected: new Set() }); + } catch (error) { + captureUiError(error, "devices.send"); + const text = error instanceof Error ? error.message : "Could not send this choice to chat."; + this.update({ error: text }); + } finally { this.update({ sending: false }); } + } + dispose() { this.disposed = true; this.listeners.clear(); } +} diff --git a/src/ui/style.css b/src/ui/style.css index 856aae41..4397b5ca 100644 --- a/src/ui/style.css +++ b/src/ui/style.css @@ -2,7 +2,7 @@ html[data-theme="dark"] { color-scheme: dark; } html, body, #root { height: 100%; overflow: hidden; } body { font-size: 12px; } -html:is([data-view="recording"], [data-view="comparison"], [data-view="tracesift"]), html:is([data-view="recording"], [data-view="comparison"], [data-view="tracesift"]) body, html:is([data-view="recording"], [data-view="comparison"], [data-view="tracesift"]) #root { height: auto; overflow: visible; } +html:is([data-view="recording"], [data-view="comparison"], [data-view="tracesift"], [data-view="devices"]), html:is([data-view="recording"], [data-view="comparison"], [data-view="tracesift"], [data-view="devices"]) body, html:is([data-view="recording"], [data-view="comparison"], [data-view="tracesift"], [data-view="devices"]) #root { height: auto; overflow: visible; } html[data-view="recording"], html[data-view="recording"] body, html[data-view="recording"] #root { margin: 0; padding: 0; } .recording-card { container-type: inline-size; margin: 12px; border: 1px solid var(--border); border-radius: 12px; background: var(--background); color: var(--foreground); font-size: 13px; overflow: hidden; } .recording-header { display: flex; align-items: center; gap: 16px; padding: 18px 22px; border-bottom: 1px solid var(--border); } diff --git a/src/ui/telemetry.ts b/src/ui/telemetry.ts index 44f4442e..4b6d3182 100644 --- a/src/ui/telemetry.ts +++ b/src/ui/telemetry.ts @@ -243,8 +243,8 @@ export function startUiTelemetry(app: App) { running = enabled; if (running === false) return; const requestedView = document.documentElement.dataset.view; - const view = requestedView === "recording" || requestedView === "workspace" || requestedView === "comparison" || requestedView === "tracesift" ? requestedView : "panel"; - if (view === "recording" || view === "comparison" || view === "tracesift") surface = view; + const view = requestedView === "recording" || requestedView === "workspace" || requestedView === "comparison" || requestedView === "tracesift" || requestedView === "devices" ? requestedView : "panel"; + if (view === "recording" || view === "comparison" || view === "tracesift" || view === "devices") surface = view; attributes = { ...attributes, surface, view }; Sentry.setTags({ component: "ui", surface, view }); Sentry.setAttributes(attributes); diff --git a/tests/device-choice-tools.test.ts b/tests/device-choice-tools.test.ts index 2c8199bf..01c28396 100644 --- a/tests/device-choice-tools.test.ts +++ b/tests/device-choice-tools.test.ts @@ -75,7 +75,7 @@ test("the native request waits for a single answer and returns verified device I await server.connect(serverTransport); await client.connect(clientTransport); let completed = false; - const pending = client.callTool({ name: "mobile_choose_devices", arguments: input }).then(result => { completed = true; return result; }); + const pending = client.callTool({ name: "mobile_show_devices", arguments: { ...input, selectionMode: "single" } }).then(result => { completed = true; return result; }); await displayed; assert.equal(completed, false); answer({ action: "accept", content: { device: "device-1" } }); @@ -110,7 +110,7 @@ for (const action of ["accept", "cancel", "decline", "invalid", "duplicate", "em }); await server.connect(serverTransport); await client.connect(clientTransport); - const result = await client.callTool({ name: "mobile_choose_devices", arguments: { ...input, selectionMode: "multiple" } }); + const result = await client.callTool({ name: "mobile_show_devices", arguments: { ...input, selectionMode: "multiple" } }); if (action === "accept") { assert.equal(result.isError, undefined); const selected = result.structuredContent?.devices; @@ -126,7 +126,7 @@ for (const action of ["accept", "cancel", "decline", "invalid", "duplicate", "em }); } -test("unsupported hosts fail explicitly without discovering devices or taking action", async t => { +test("hosts without forms or cards fail a choice explicitly without discovering devices", async t => { const server = new McpServer({ name: "device-choices", version: "1" }); const extensions = new OpenAIExtensions(server); const fail = async (): Promise => { throw new Error("Should not discover"); }; @@ -136,11 +136,11 @@ test("unsupported hosts fail explicitly without discovering devices or taking ac t.after(async () => { await client.close(); await server.close(); }); await server.connect(serverTransport); await client.connect(clientTransport); - const result = await client.callTool({ name: "mobile_choose_devices", arguments: input }); + const result = await client.callTool({ name: "mobile_show_devices", arguments: { ...input, selectionMode: "single" } }); assert.equal(result.isError, true); const content = result.content; assert.ok(Array.isArray(content) && content[0].type === "text"); - assert.match(content[0].text, /does not support OpenAI native request forms/); + assert.match(content[0].text, /neither a native request form nor an app card/); }); test("discovery rejects duplicate, missing, wrong-kind and unauthorized candidates", async () => { @@ -178,3 +178,66 @@ test("physical devices use hardware IDs; stopped simulators remain selectable fo assert.match(field.oneOf[0].description ?? "", /Stopped/); assert.equal(field.default, undefined, "No target is silently preselected."); }); + +async function connectedClient(t: test.TestContext, discovery: DeviceChoiceSources, capabilities = {}) { + const server = new McpServer({ name: "device-choices", version: "1" }); + registerDeviceChoiceTools(server, new OpenAIExtensions(server), discovery); + const client = new Client({ name: "card-host", version: "1" }, { capabilities }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + t.after(async () => { await client.close(); await server.close(); }); + await server.connect(serverTransport); + await client.connect(clientTransport); + return client; +} +const cardHost = { extensions: { "io.modelcontextprotocol/ui": { mimeTypes: ["text/html;profile=mcp-app"] } } }; +function text(result: Awaited>) { + const content = result.content; + assert.ok(Array.isArray(content) && content[0].type === "text"); + return content[0].text; +} + +test("browsing lists every source in the card and tolerates an unavailable backend", async t => { + const discovery = sources(); + discovery.physicalIos = async () => { throw new Error("devicectl needs Xcode 27."); }; + const client = await connectedClient(t, discovery, cardHost); + const result = await client.callTool({ name: "mobile_show_devices", arguments: {} }); + assert.equal(result.isError, undefined); + assert.equal(result.structuredContent?.mode, "browse"); + const devices = result.structuredContent?.devices; + assert.ok(Array.isArray(devices)); + assert.deepEqual(devices.map(device => device.deviceId), [simulatorId, "emulator-5554"]); + assert.deepEqual(result.structuredContent?.errors, ["devicectl needs Xcode 27."]); + assert.match(text(result), /Showing 2 devices in the Mobile Dev card \(2 running, 0 stopped\)\. Do not repeat the list\./); + assert.match(text(result), /devicectl needs Xcode 27/); +}); + +test("browsing without app cards returns the device list as text", async t => { + const discovery = sources(); + discovery.physicalIos = async () => []; + const client = await connectedClient(t, discovery); + const result = await client.callTool({ name: "mobile_show_devices", arguments: {} }); + const listed = JSON.parse(text(result)); + assert.equal(listed.length, 2); +}); + +test("choosing in a card host renders selectable candidates and asks the model to wait", async t => { + const discovery = sources(); + discovery.physicalIos = async () => [{ + udid: "00008140-0011223344556677", coreDeviceId: simulatorId, name: "Away iPhone", model: "iPhone 17", productType: "iPhone18,1", + state: "disconnected", runtime: "iOS 27", platform: "ios", kind: "physical", transportType: "localNetwork", pairingState: "paired", + }]; + const client = await connectedClient(t, discovery, cardHost); + const result = await client.callTool({ name: "mobile_show_devices", arguments: { message: "Which device?", selectionMode: "multiple" } }); + assert.equal(result.structuredContent?.mode, "choose"); + assert.equal(result.structuredContent?.selectionMode, "multiple"); + const devices = result.structuredContent?.devices; + assert.ok(Array.isArray(devices)); + assert.equal(devices.some(device => device.name === "Away iPhone"), false, "Disconnected devices cannot be chosen."); + assert.match(text(result), /Wait for the user to choose in the card/); +}); + +test("a choice requires a message", async t => { + const client = await connectedClient(t, sources(), cardHost); + const result = await client.callTool({ name: "mobile_show_devices", arguments: { selectionMode: "single" } }); + assert.equal(result.isError, true); +}); diff --git a/tests/plugin.test.ts b/tests/plugin.test.ts index 143d984e..9db6e0e6 100644 --- a/tests/plugin.test.ts +++ b/tests/plugin.test.ts @@ -12,6 +12,7 @@ import { APP_URI, WORKSPACE_URI } from "../src/server/plugin.ts"; import { RECORDING_URI } from "../src/shared/recordings.ts"; import { COMPARISON_URI } from "../src/shared/performance-comparison.ts"; import { TRACESIFT_URI } from "../src/shared/tracesift.ts"; +import { DEVICES_URI } from "../src/shared/device-choice.ts"; import { Baguette } from "../src/server/baguette.ts"; import { parseBaseUrl } from "../src/shared/protocol.ts"; import { createTestPlugin, fakeBaguette, fakeSimulatorInput, UDID, OTHER_UDID, SCREEN, PNG } from "./fixtures.ts"; @@ -39,7 +40,7 @@ test("both opening tools accept optional device intent and preserve it in their assert.equal(boots.length, 0); }); -test("the device choice tool is listed only for clients that render native forms", async t => { +test("the devices tool is listed for every client with its card", async t => { const fake = await fakeBaguette(); t.after(() => fake.close()); for (const forms of [false, true]) { @@ -50,10 +51,11 @@ test("the device choice tool is listed only for clients that render native forms await plugin.server.connect(serverTransport); await client.connect(clientTransport); const { tools } = await client.listTools(); - const choice = tools.find(tool => tool.name === "mobile_choose_devices"); - assert.equal(!!choice, forms); - if (choice) assert.ok(choice.icons?.length); - assert.match(client.getInstructions() ?? "", /mobile_choose_devices when it is available, otherwise in chat/); + const choice = tools.find(tool => tool.name === "mobile_show_devices"); + assert.ok(choice?.icons?.length); + const resourceUri = (choice._meta?.ui as { resourceUri?: string } | undefined)?.resourceUri; + assert.equal(resourceUri, DEVICES_URI); + assert.match(client.getInstructions() ?? "", /mobile_show_devices instead of writing a list/); await client.close(); await plugin.close(); } }); @@ -247,6 +249,9 @@ test("MCP tools expose native entrypoints and complete the simulator workflow", const traceSiftResource = await client.readResource({ uri: TRACESIFT_URI }); assert.match(traceSiftResource.contents[0].text as string, /data-view="tracesift"/); assert.deepEqual(traceSiftResource.contents[0]._meta, recordingResource.contents[0]._meta); + const devicesResource = await client.readResource({ uri: DEVICES_URI }); + assert.match(devicesResource.contents[0].text as string, /data-view="devices"/); + assert.deepEqual(devicesResource.contents[0]._meta, recordingResource.contents[0]._meta); const analyzeProfile = tools.tools.find(tool => tool.name === "mobile_analyze_profile"); assert.deepEqual(analyzeProfile?._meta?.ui, { resourceUri: TRACESIFT_URI, visibility: ["app", "model"] }); assert.deepEqual(tools.tools.find(tool => tool.name === "mobile_remember_tracesift_analysis")?._meta?.ui, { visibility: ["app"] }); diff --git a/tests/telemetry.test.ts b/tests/telemetry.test.ts index ec5651b6..7a62ad62 100644 --- a/tests/telemetry.test.ts +++ b/tests/telemetry.test.ts @@ -526,16 +526,18 @@ test("native device picker measures preparation and outcomes without private for t.after(async () => { await client.close(); await server.close(); await Sentry.close(); }); await server.connect(serverTransport); await client.connect(clientTransport); - const arguments_ = { message: "PRIVATE_QUESTION", context: "PRIVATE_OPERATION", devices: [ + const arguments_ = { selectionMode: "single", message: "PRIVATE_QUESTION", context: "PRIVATE_OPERATION", devices: [ { platform: "android", kind: "physical", deviceId: "PRIVATE_SERIAL", appName: "PRIVATE_APP" }, ] }; for (const action of ["accept", "cancel", "decline"] as const) { answer = action === "accept" ? { action, content: { device: "device-1" } } : { action }; - const result = await client.callTool({ name: "mobile_choose_devices", arguments: arguments_ }); + const result = await client.callTool({ name: "mobile_show_devices", arguments: arguments_ }); assert.equal(result.isError, undefined); } + const browse = await client.callTool({ name: "mobile_show_devices", arguments: {} }); + assert.equal(browse.isError, undefined); answer = { action: "accept", content: { device: "not-offered" } }; - const invalid = await client.callTool({ name: "mobile_choose_devices", arguments: arguments_ }); + const invalid = await client.callTool({ name: "mobile_show_devices", arguments: arguments_ }); assert.equal(invalid.isError, true); await Sentry.close(); const encoded = JSON.stringify(envelopes); @@ -545,6 +547,8 @@ test("native device picker measures preparation and outcomes without private for contains(encoded, "PRIVATE", false); contains(encoded, "device-1", false); contains(encoded, "selection_mode"); + contains(encoded, "rendered"); + contains(encoded, "browse"); contains(encoded, "millisecond"); const items = envelopes.flatMap(envelope => envelope[1]); const errors = items.filter(item => item[0].type === "event");