From fad77ac956929be26f92779125bc1a12197f318f Mon Sep 17 00:00:00 2001 From: cryptotavares Date: Sun, 25 Oct 2026 08:38:17 +0000 Subject: [PATCH 1/2] feat: filter mobile status/nav bars and dev overlay; viewport from snapshot Ticket P1-13. - filterAppHierarchy (new src/platform/mobile-hierarchy.ts) runs once per snapshot, before the a11y tree, testIds and generated locators are built, so all of them agree: - Android: drop top-level windows shaped like a bar (full width or height, at most 15 % thick) whose ids are all `com.android.systemui:*` (status bar: clock, signal, battery change on every snapshot; nav bar). Other system windows (biometric prompt, consent dialogs, permission dialogs, keyboard) stay. Flat hierarchies: drop only status/nav bar ids. - iOS / Android: drop the Expo dev menu button (`gearshape.fill` plus its "Tools" label). `MobilePlatformDriver` option `excludeDevOverlay` (default true, new exported `MobilePlatformDriverOptions`). - The testId viewport comes from the snapshot (iOS root frame, Android extent of the top-level windows); getWindowSize() is called (and cached) only when the snapshot has none. Before, every testId read made one extra device call. - Mobile settle: the time budget includes the first capture, and an unchanged screen is not re-captured when the first capture already took >= 1 s (MOBILE_SETTLE_SLOW_CAPTURE_MS). Busy screens still retry. - Observations, describe_screen, list_testids and accessibility_snapshot take one snapshot each (P1-08 captureScreen); tests lock that in. --- README.md | 4 +- SKILL.md | 2 + src/index.ts | 1 + .../__fixtures__/android-login-snapshot.json | 992 ++++++++++++++++++ src/platform/index.ts | 1 + src/platform/mobile-hierarchy.test.ts | 590 +++++++++++ src/platform/mobile-hierarchy.ts | 256 +++++ src/platform/mobile-platform-driver.test.ts | 316 ++++++ src/platform/mobile-platform-driver.ts | 108 +- src/server/create-server.test.ts | 201 +++- src/server/create-server.ts | 20 +- src/tools/utils/constants.ts | 8 + 12 files changed, 2470 insertions(+), 29 deletions(-) create mode 100644 src/platform/__fixtures__/android-login-snapshot.json create mode 100644 src/platform/mobile-hierarchy.test.ts create mode 100644 src/platform/mobile-hierarchy.ts diff --git a/README.md b/README.md index 5b51bf8..3a6a040 100644 --- a/README.md +++ b/README.md @@ -449,7 +449,9 @@ This accessibility-first approach provides reliable element targeting that survi Refs are numbered in tree order each time the daemon reads the screen (`describe_screen`, `accessibility_snapshot`, or an observation). The same screen gives the same refs, so an unchanged screen diffs as unchanged. When elements appear or disappear above an element, its ref can change: use the refs from the latest response. When a call returns both a tree and an observation (for example `describe_screen` with `observe`), the observation's refs are the ones that resolve. `list_testids` does not change refs. -On mobile, after a mutating or batch call, the daemon may take the snapshot again after 300 ms: while it shows a busy element (a label ending in `, busy`, an Android `ProgressBar`, or an iOS `ActivityIndicator` / `ProgressIndicator` / `UIActivityIndicatorView`), and at most once when it still equals the previous screen. In total at most 3 extra snapshots and about 2 s. The test IDs come from the same snapshot as the tree. `describe_screen` on mobile also takes one snapshot for both. +On mobile, after a mutating or batch call, the daemon may take the snapshot again after 300 ms: while it shows a busy element (a label ending in `, busy`, an Android `ProgressBar`, or an iOS `ActivityIndicator` / `ProgressIndicator` / `UIActivityIndicatorView`), and at most once when it still equals the previous screen (not when the first snapshot took 1 s or more: the screen already had that time to change). In total at most 3 extra snapshots, and no wait that would end more than about 2 s after the first snapshot started. The test IDs come from the same snapshot as the tree. `describe_screen` on mobile also takes one snapshot for both. + +On mobile, each observation, `describe_screen`, `list_testids` and `accessibility_snapshot` takes one device snapshot (plus the settle retries above). The viewport for `visible` comes from the snapshot's root frames; the device window size is read at most once per session (and not refreshed after a rotation), only when a snapshot has no root frame. Elements that are not part of the app are removed from the tree, the refs, the test IDs and `generate_locators`: on Android, the system status bar and navigation bar windows (a full-width or full-height bar window whose ids are all `com.android.systemui:id/*`; other System UI windows such as the biometric prompt stay); on both platforms, the Expo DevLauncher "Tools" button of development builds (iOS `gearshape.fill` and its "Tools" text). Pass `{ excludeDevOverlay: false }` as the third `MobilePlatformDriver` argument to keep the DevLauncher button. Taps and waits inside `@metamask/device-mcp` still search the whole snapshot. ### Session Metadata diff --git a/SKILL.md b/SKILL.md index fb2078b..8ad09cf 100644 --- a/SKILL.md +++ b/SKILL.md @@ -51,6 +51,8 @@ mm get-state --observe full # a read that also returns an observation Observations contain `testIds` and `a11y` only — no app `state`. Call `mm get-state` for state. +On mobile, each observation and `describe-screen` comes from one device snapshot. The Android system status bar and navigation bar and the Expo DevLauncher "Tools" button (iOS `gearshape.fill`) are not part of the app and are removed from `testIds` and `a11y`. Other system windows, such as the biometric prompt (`com.android.systemui:id/biometric_*`), are kept. + **Observation Compaction:** Observations are **compacted** before returning: option runs of 3 or more under a combobox or listbox are replaced with a single summary node (e.g., `"55 options (refs e2–e56)"`). The `describe-screen` tool always returns the **full, unfiltered** a11y tree — use it when you need the complete option list. **Diff-Based Observations:** After the first observation sets a baseline, later calls at level `diff` return **diff-based** observations. The `observations.a11y.diff` field (when present) shows what changed: diff --git a/src/index.ts b/src/index.ts index 50eb544..e3cded3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,6 +11,7 @@ export type { GetTextActionResult, PlatformScreenshotOptions, ScreenCapture, + MobilePlatformDriverOptions, } from './platform'; export { PlaywrightPlatformDriver, MobilePlatformDriver } from './platform'; diff --git a/src/platform/__fixtures__/android-login-snapshot.json b/src/platform/__fixtures__/android-login-snapshot.json new file mode 100644 index 0000000..7c7c206 --- /dev/null +++ b/src/platform/__fixtures__/android-login-snapshot.json @@ -0,0 +1,992 @@ +[ + { + "type": "android.widget.FrameLayout", + "frame": { "x": 0, "y": 0, "width": 1080, "height": 63 }, + "enabled": true, + "children": [ + { + "type": "android.widget.FrameLayout", + "identifier": "com.android.systemui:id/status_bar_launch_animation_container", + "frame": { "x": 0, "y": 0, "width": 1080, "height": 63 }, + "enabled": true + }, + { + "type": "android.widget.FrameLayout", + "identifier": "com.android.systemui:id/status_bar_container", + "frame": { "x": 0, "y": 0, "width": 1080, "height": 63 }, + "enabled": true, + "children": [ + { + "type": "androidx.compose.ui.platform.ComposeView", + "frame": { "x": 0, "y": 0, "width": 1080, "height": 63 }, + "enabled": true, + "children": [ + { + "type": "android.view.View", + "frame": { "x": 0, "y": 0, "width": 1080, "height": 63 }, + "enabled": true, + "children": [ + { + "type": "androidx.compose.ui.viewinterop.ViewFactoryHolder", + "frame": { "x": 0, "y": 0, "width": 1080, "height": 63 }, + "enabled": true, + "children": [ + { + "type": "android.widget.FrameLayout", + "identifier": "com.android.systemui:id/status_bar", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 63 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.LinearLayout", + "identifier": "com.android.systemui:id/status_bar_contents", + "frame": { + "x": 0, + "y": 0, + "width": 1027, + "height": 63 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.FrameLayout", + "identifier": "com.android.systemui:id/status_bar_start_side_container", + "frame": { + "x": 11, + "y": 0, + "width": 502, + "height": 63 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.FrameLayout", + "identifier": "com.android.systemui:id/status_bar_start_side_content", + "frame": { + "x": 11, + "y": 0, + "width": 115, + "height": 63 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.LinearLayout", + "identifier": "com.android.systemui:id/status_bar_start_side_except_heads_up", + "frame": { + "x": 11, + "y": 0, + "width": 115, + "height": 63 + }, + "enabled": true, + "children": [ + { + "type": "androidx.compose.ui.platform.ComposeView", + "frame": { + "x": 0, + "y": 0, + "width": 0, + "height": 0 + }, + "enabled": true, + "children": [ + { + "type": "android.view.View", + "frame": { + "x": 0, + "y": 0, + "width": 0, + "height": 0 + }, + "enabled": true + } + ] + }, + { + "type": "android.widget.TextView", + "label": "8:28 AM", + "value": "8:28", + "identifier": "com.android.systemui:id/clock", + "frame": { + "x": 11, + "y": 2, + "width": 115, + "height": 58 + }, + "enabled": true + }, + { + "type": "android.widget.FrameLayout", + "identifier": "com.android.systemui:id/notification_icon_area", + "frame": { + "x": 126, + "y": 0, + "width": 0, + "height": 63 + }, + "enabled": true, + "children": [ + { + "type": "android.view.ViewGroup", + "identifier": "com.android.systemui:id/notificationIcons", + "frame": { + "x": 126, + "y": 0, + "width": 0, + "height": 63 + }, + "enabled": true + } + ] + } + ] + } + ] + } + ] + }, + { + "type": "android.widget.FrameLayout", + "identifier": "com.android.systemui:id/status_bar_end_side_container", + "frame": { + "x": 513, + "y": 0, + "width": 503, + "height": 63 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.LinearLayout", + "identifier": "com.android.systemui:id/status_bar_end_side_content", + "frame": { + "x": 799, + "y": 0, + "width": 217, + "height": 63 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.LinearLayout", + "identifier": "com.android.systemui:id/system_icons", + "frame": { + "x": 799, + "y": 2, + "width": 217, + "height": 58 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.LinearLayout", + "identifier": "com.android.systemui:id/statusIcons", + "frame": { + "x": 807, + "y": 2, + "width": 128, + "height": 58 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.FrameLayout", + "label": "T-Mobile, two bars.", + "identifier": "com.android.systemui:id/mobile_combo", + "frame": { + "x": 807, + "y": 2, + "width": 61, + "height": 58 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.LinearLayout", + "identifier": "com.android.systemui:id/mobile_group", + "frame": { + "x": 815, + "y": 2, + "width": 45, + "height": 58 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.FrameLayout", + "frame": { + "x": 815, + "y": 15, + "width": 45, + "height": 32 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.ImageView", + "identifier": "com.android.systemui:id/mobile_signal", + "frame": { + "x": 815, + "y": 15, + "width": 45, + "height": 32 + }, + "enabled": true + } + ] + } + ] + } + ] + }, + { + "type": "android.widget.FrameLayout", + "frame": { + "x": 0, + "y": 0, + "width": 0, + "height": 0 + }, + "enabled": true + }, + { + "type": "android.widget.FrameLayout", + "frame": { + "x": 0, + "y": 0, + "width": 0, + "height": 0 + }, + "enabled": true + }, + { + "type": "android.widget.FrameLayout", + "identifier": "com.android.systemui:id/wifi_combo", + "frame": { + "x": 868, + "y": 2, + "width": 59, + "height": 58 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.LinearLayout", + "identifier": "com.android.systemui:id/wifi_group", + "frame": { + "x": 876, + "y": 2, + "width": 43, + "height": 58 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.FrameLayout", + "identifier": "com.android.systemui:id/wifi_combo", + "frame": { + "x": 876, + "y": 15, + "width": 43, + "height": 32 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.ImageView", + "label": "Wifi signal full.", + "identifier": "com.android.systemui:id/wifi_signal", + "frame": { + "x": 876, + "y": 15, + "width": 43, + "height": 32 + }, + "enabled": true + } + ] + } + ] + } + ] + } + ] + }, + { + "type": "androidx.compose.ui.platform.ComposeView", + "frame": { + "x": 935, + "y": 14, + "width": 70, + "height": 34 + }, + "enabled": true, + "children": [ + { + "type": "android.view.View", + "frame": { + "x": 935, + "y": 14, + "width": 70, + "height": 34 + }, + "enabled": true, + "children": [ + { + "type": "android.view.View", + "identifier": "com.android.systemui:id/battery", + "frame": { + "x": 935, + "y": 14, + "width": 70, + "height": 34 + }, + "enabled": true, + "children": [ + { + "type": "android.view.View", + "label": "Battery 100 percent.", + "frame": { + "x": 935, + "y": 14, + "width": 63, + "height": 34 + }, + "enabled": true + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "type": "android.widget.FrameLayout", + "frame": { "x": 0, "y": 0, "width": 1080, "height": 2400 }, + "enabled": true, + "children": [ + { + "type": "android.widget.LinearLayout", + "frame": { "x": 0, "y": 0, "width": 1080, "height": 2400 }, + "enabled": true, + "children": [ + { + "type": "android.widget.FrameLayout", + "frame": { "x": 0, "y": 0, "width": 1080, "height": 2400 }, + "enabled": true, + "children": [ + { + "type": "android.widget.LinearLayout", + "identifier": "io.metamask:id/action_bar_root", + "frame": { "x": 0, "y": 0, "width": 1080, "height": 2400 }, + "enabled": true, + "children": [ + { + "type": "android.widget.FrameLayout", + "identifier": "android:id/content", + "frame": { "x": 0, "y": 0, "width": 1080, "height": 2400 }, + "enabled": true, + "children": [ + { + "type": "android.widget.FrameLayout", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.LinearLayout", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "androidx.compose.ui.platform.ComposeView", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.view.View", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.view.View", + "frame": { + "x": 0, + "y": 63, + "width": 221, + "height": 221 + }, + "enabled": true, + "children": [ + { + "type": "android.view.View", + "frame": { + "x": 902, + "y": 105, + "width": 137, + "height": 137 + }, + "enabled": true, + "children": [ + { + "type": "android.view.View", + "frame": { + "x": 908, + "y": 111, + "width": 126, + "height": 126 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.ImageView", + "label": "Tools", + "frame": { + "x": 937, + "y": 140, + "width": 68, + "height": 68 + }, + "enabled": true + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "type": "android.widget.FrameLayout", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.view.View", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true + } + ] + }, + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 0, + "height": 0 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.FrameLayout", + "identifier": "npm:@metamask/bitcoin-wallet-snap", + "frame": { + "x": 0, + "y": 0, + "width": 0, + "height": 0 + }, + "enabled": true, + "children": [ + { + "type": "android.webkit.WebView", + "frame": { + "x": 0, + "y": 0, + "width": 0, + "height": 0 + }, + "enabled": true + } + ] + } + ] + }, + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.ScrollView", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.LinearLayout", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 0 + }, + "enabled": true + }, + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 0 + }, + "enabled": true + }, + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.ScrollView", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.LinearLayout", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 0 + }, + "enabled": true + }, + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 0 + }, + "enabled": true + }, + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true + }, + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.ScrollView", + "frame": { + "x": 0, + "y": 63, + "width": 1080, + "height": 2337 + }, + "enabled": true, + "children": [ + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 63, + "width": 1080, + "height": 2337 + }, + "enabled": true, + "children": [ + { + "type": "android.view.ViewGroup", + "identifier": "login", + "frame": { + "x": 0, + "y": 105, + "width": 1080, + "height": 2253 + }, + "enabled": true + }, + { + "type": "android.view.ViewGroup", + "identifier": "login-download-logs-button", + "frame": { + "x": 289, + "y": 602, + "width": 504, + "height": 273 + }, + "enabled": true, + "children": [ + { + "type": "com.horcrux.svg.SvgView", + "identifier": "metamask-wordmark", + "frame": { + "x": 289, + "y": 602, + "width": 504, + "height": 273 + }, + "enabled": true, + "children": [ + { + "type": "com.horcrux.svg.GroupView", + "frame": { + "x": 289, + "y": 602, + "width": 504, + "height": 273 + }, + "enabled": true, + "children": [ + { + "type": "com.horcrux.svg.PathView", + "frame": { + "x": 289, + "y": 602, + "width": 504, + "height": 273 + }, + "enabled": true + } + ] + } + ] + } + ] + }, + { + "type": "android.view.ViewGroup", + "frame": { + "x": 42, + "y": 1232, + "width": 996, + "height": 126 + }, + "enabled": true + }, + { + "type": "android.widget.EditText", + "label": "login-password-input", + "value": "Enter password", + "identifier": "login-password-input", + "frame": { + "x": 86, + "y": 1234, + "width": 908, + "height": 121 + }, + "enabled": true + }, + { + "type": "android.widget.Button", + "label": "Unlock", + "identifier": "log-in-button", + "frame": { + "x": 42, + "y": 1400, + "width": 996, + "height": 126 + }, + "enabled": false, + "children": [ + { + "type": "android.widget.TextView", + "label": "Unlock", + "frame": { + "x": 471, + "y": 1432, + "width": 138, + "height": 63 + }, + "enabled": true + } + ] + }, + { + "type": "android.widget.Button", + "label": "Forgot password?", + "identifier": "reset-wallet-button", + "frame": { + "x": 42, + "y": 1568, + "width": 996, + "height": 126 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.TextView", + "label": "Forgot password?", + "frame": { + "x": 363, + "y": 1600, + "width": 355, + "height": 63 + }, + "enabled": true + } + ] + } + ] + } + ] + } + ] + }, + { + "type": "android.view.ViewGroup", + "frame": { + "x": 0, + "y": 1518, + "width": 1080, + "height": 882 + }, + "enabled": true, + "children": [ + { + "type": "android.widget.FrameLayout", + "identifier": "fox-animation", + "frame": { + "x": 0, + "y": 1518, + "width": 1080, + "height": 882 + }, + "enabled": true, + "children": [ + { + "type": "android.view.TextureView", + "frame": { + "x": 0, + "y": 1518, + "width": 1080, + "height": 882 + }, + "enabled": true + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "type": "android.view.ViewGroup", + "frame": { "x": 0, "y": 2400, "width": 1080, "height": 0 }, + "enabled": true + } + ] + } + ] + } + ] + } + ] + } +] diff --git a/src/platform/index.ts b/src/platform/index.ts index dd96aed..142561f 100644 --- a/src/platform/index.ts +++ b/src/platform/index.ts @@ -12,3 +12,4 @@ export type { export { PlaywrightPlatformDriver } from './playwright-driver.js'; export { MobilePlatformDriver } from './mobile-platform-driver.js'; +export type { MobilePlatformDriverOptions } from './mobile-platform-driver.js'; diff --git a/src/platform/mobile-hierarchy.test.ts b/src/platform/mobile-hierarchy.test.ts new file mode 100644 index 0000000..57ffdc9 --- /dev/null +++ b/src/platform/mobile-hierarchy.test.ts @@ -0,0 +1,590 @@ +import type { UIElement } from '@metamask/device-mcp'; +import { describe, it, expect } from 'vitest'; + +import androidLoginSnapshot from './__fixtures__/android-login-snapshot.json'; +import { + filterAppHierarchy, + resourceIdPackage, + viewportFromHierarchy, +} from './mobile-hierarchy.js'; + +function el( + type: string, + overrides: Partial = {}, + children?: UIElement[], +): UIElement { + return { + type, + frame: { x: 0, y: 0, width: 10, height: 10 }, + enabled: true, + ...overrides, + ...(children ? { children } : {}), + }; +} + +function identifiers(elements: UIElement[]): string[] { + const ids: string[] = []; + const walk = (nodes: UIElement[]): void => { + for (const node of nodes) { + if (node.identifier) { + ids.push(node.identifier); + } + walk(node.children ?? []); + } + }; + walk(elements); + return ids; +} + +function labels(elements: UIElement[]): string[] { + const out: string[] = []; + const walk = (nodes: UIElement[]): void => { + for (const node of nodes) { + if (node.label) { + out.push(node.label); + } + walk(node.children ?? []); + } + }; + walk(elements); + return out; +} + +/** + * Shaped like a real Android (adb, uiautomator) snapshot of MetaMask: a + * status bar window from System UI, then the app window, which holds the + * Expo DevLauncher "Tools" button in a Compose view and the React Native + * screen (proof/p1-08/android/describe-screen.txt). + * + * @returns The hierarchy. + */ +function androidSnapshot(): UIElement[] { + const frame = (x: number, y: number, width: number, height: number) => ({ + frame: { x, y, width, height }, + }); + return [ + el('android.widget.FrameLayout', frame(0, 0, 1080, 63), [ + el( + 'android.widget.FrameLayout', + { + identifier: 'com.android.systemui:id/status_bar_container', + ...frame(0, 0, 1080, 63), + }, + [ + el( + 'android.widget.FrameLayout', + { + identifier: 'com.android.systemui:id/status_bar', + ...frame(0, 0, 1080, 63), + }, + [ + el('android.widget.TextView', { + identifier: 'com.android.systemui:id/clock', + value: '12:46 AM', + }), + el('android.widget.FrameLayout', { + identifier: 'com.android.systemui:id/wifi_combo', + }), + ], + ), + ], + ), + el('android.view.View', { label: 'Battery 100 percent.' }), + ]), + el('android.widget.FrameLayout', frame(0, 0, 1080, 2400), [ + el( + 'android.widget.LinearLayout', + { + identifier: 'io.metamask:id/action_bar_root', + ...frame(0, 0, 1080, 2400), + }, + [ + el( + 'android.widget.FrameLayout', + { identifier: 'android:id/content', ...frame(0, 0, 1080, 2400) }, + [ + el('androidx.compose.ui.platform.ComposeView', {}, [ + el('android.view.View', {}, [ + el('android.widget.ImageView', { + label: 'Tools', + ...frame(937, 140, 68, 68), + }), + ]), + ]), + el('android.view.ViewGroup', {}, [ + el('android.view.ViewGroup', { identifier: 'wallet-screen' }), + el('android.widget.Button', { + identifier: 'tab-bar-item-Wallet', + label: 'Wallet', + }), + el('android.widget.TextView', { value: 'Tools' }), + ]), + ], + ), + ], + ), + ]), + ]; +} + +/** + * Shaped like a real iOS (idb describe-all) snapshot of a development build: + * the app element, then the DevLauncher window with the gear image and its + * "Tools" text (proof/p1-26/ios/launch.txt). + * + * @returns The hierarchy. + */ +function iosSnapshot(): UIElement[] { + const screen = { frame: { x: 0, y: 0, width: 402, height: 874 } }; + return [ + el('Other', { label: 'MetaMask', ...screen }, [ + el('Application', screen, [ + el('Button', { identifier: 'log-in-button', label: 'Unlock' }), + el('StaticText', { label: 'Tools' }), + ]), + el('Application', screen, [ + el('UITransitionView', screen, [ + el('UIDropShadowView', screen, [ + el('_UIHostingView', screen, [ + el('SwiftUI.AccessibilityNode', screen, [ + el('Image', { + identifier: 'gearshape.fill', + label: 'Gear Shape, Filled', + }), + el('StaticText', { label: 'Tools' }), + ]), + ]), + ]), + ]), + ]), + ]), + ]; +} + +describe('resourceIdPackage', () => { + it.each([ + ['com.android.systemui:id/clock', 'com.android.systemui'], + ['io.metamask:id/action_bar_root', 'io.metamask'], + ['android:id/content', 'android'], + ['wallet-screen', undefined], + ['npm:@metamask/solana-wallet-snap', undefined], + ['asset-ETH', undefined], + ])('reads the package of %s', (identifier, expected) => { + expect(resourceIdPackage(identifier)).toBe(expected); + }); +}); + +describe('filterAppHierarchy', () => { + describe('android', () => { + it('drops the status bar window with its subtree (AC2)', () => { + const filtered = filterAppHierarchy(androidSnapshot(), { + platform: 'android', + }); + + expect(filtered).toHaveLength(1); + expect(filtered[0]?.frame.height).toBe(2400); + expect(identifiers(filtered)).toStrictEqual([ + 'io.metamask:id/action_bar_root', + 'android:id/content', + 'wallet-screen', + 'tab-bar-item-Wallet', + ]); + // The unidentified battery node goes with its window. + expect(labels(filtered)).not.toContain('Battery 100 percent.'); + }); + + describe('real snapshot: login screen (proof/p1-13/before/android)', () => { + const snapshot = androidLoginSnapshot as UIElement[]; + const count = (elements: UIElement[]): number => + elements.reduce( + (total, node) => total + 1 + count(node.children ?? []), + 0, + ); + + it('reduces to the app window, with no System UI test IDs', () => { + const filtered = filterAppHierarchy(snapshot, { platform: 'android' }); + + // Input: status bar window (34 nodes, 22 systemui ids, some under + // ComposeViews without ids) + app window (54 nodes, trimmed). + expect(snapshot).toHaveLength(2); + expect(count(snapshot)).toBe(88); + expect( + identifiers(snapshot).filter((id) => id.includes('systemui')), + ).toHaveLength(22); + + expect(filtered).toStrictEqual([ + // The app window minus the DevLauncher "Tools" ImageView and the + // 6 unnamed containers that held only it (LinearLayout > + // ComposeView > 4 Views). + expect.objectContaining({ frame: snapshot[1]?.frame }), + ]); + expect(count(filtered)).toBe(54 - 7); + expect( + identifiers(filtered).filter((id) => id.includes('systemui')), + ).toStrictEqual([]); + expect(identifiers(filtered)).toStrictEqual([ + 'io.metamask:id/action_bar_root', + 'android:id/content', + 'npm:@metamask/bitcoin-wallet-snap', + 'login', + 'login-download-logs-button', + 'metamask-wordmark', + 'login-password-input', + 'log-in-button', + 'reset-wallet-button', + 'fox-animation', + ]); + }); + + it('keeps only the bar filter with excludeDevOverlay: false', () => { + const filtered = filterAppHierarchy(snapshot, { + platform: 'android', + excludeDevOverlay: false, + }); + + expect(filtered).toStrictEqual([snapshot[1]]); + }); + }); + + it('keeps a System UI window that is not a bar (biometric prompt)', () => { + const [statusBar, app] = androidSnapshot() as [UIElement, UIElement]; + const biometric = el( + 'android.widget.FrameLayout', + { frame: { x: 0, y: 1500, width: 1080, height: 900 } }, + [ + el('android.widget.TextView', { + identifier: 'com.android.systemui:id/biometric_title', + value: 'Unlock MetaMask', + }), + el('android.widget.Button', { + identifier: 'com.android.systemui:id/button_use_credential', + label: 'Use PIN', + }), + ], + ); + + const filtered = filterAppHierarchy([statusBar, app, biometric], { + platform: 'android', + }); + + expect(filtered).toHaveLength(2); + expect(filtered[1]).toBe(biometric); + expect( + identifiers(filtered).filter((id) => id.includes('systemui')), + ).toStrictEqual([ + 'com.android.systemui:id/biometric_title', + 'com.android.systemui:id/button_use_credential', + ]); + }); + + it('keeps a bar-shaped window that holds an app identifier', () => { + const banner = el( + 'android.widget.FrameLayout', + { frame: { x: 0, y: 0, width: 1080, height: 63 } }, + [ + el('android.view.View', { + identifier: 'com.android.systemui:id/status_bar', + }), + el('android.widget.TextView', { identifier: 'toast-message' }), + ], + ); + const app = el('android.widget.FrameLayout', { + frame: { x: 0, y: 0, width: 1080, height: 2400 }, + identifier: 'io.metamask:id/action_bar_root', + }); + + expect( + filterAppHierarchy([banner, app], { platform: 'android' }), + ).toStrictEqual([banner, app]); + }); + + it('keeps a bar-shaped window with no identifiers', () => { + const bar = el( + 'android.widget.FrameLayout', + { frame: { x: 0, y: 0, width: 1080, height: 63 } }, + [el('android.view.View', { label: 'Battery 100 percent.' })], + ); + const app = el('android.widget.FrameLayout', { + frame: { x: 0, y: 0, width: 1080, height: 2400 }, + }); + + expect( + filterAppHierarchy([bar, app], { platform: 'android' }), + ).toStrictEqual([bar, app]); + }); + + it('drops the navigation bar and a side navigation bar (landscape)', () => { + const navBar = (frame: UIElement['frame']): UIElement => + el('android.widget.FrameLayout', { frame }, [ + el('android.widget.FrameLayout', { + identifier: 'com.android.systemui:id/navigation_bar_frame', + }), + el('android.widget.ImageView', { + identifier: 'com.android.systemui:id/home_handle', + }), + ]); + const portraitApp = el('android.widget.FrameLayout', { + frame: { x: 0, y: 0, width: 1080, height: 2400 }, + identifier: 'wallet-screen', + }); + const landscapeApp = el('android.widget.FrameLayout', { + frame: { x: 0, y: 0, width: 2270, height: 1080 }, + identifier: 'wallet-screen', + }); + + expect( + filterAppHierarchy( + [portraitApp, navBar({ x: 0, y: 2270, width: 1080, height: 130 })], + { platform: 'android' }, + ), + ).toStrictEqual([portraitApp]); + expect( + filterAppHierarchy( + [landscapeApp, navBar({ x: 2270, y: 0, width: 130, height: 1080 })], + { platform: 'android' }, + ), + ).toStrictEqual([landscapeApp]); + }); + + it('drops a status-bar-only snapshot', () => { + const [statusBar] = androidSnapshot() as [UIElement]; + + expect( + filterAppHierarchy([statusBar], { platform: 'android' }), + ).toStrictEqual([]); + }); + + it('keeps windows when the snapshot gives no screen size or a window has no size', () => { + const statusBarIds = [ + el('android.widget.FrameLayout', { + identifier: 'com.android.systemui:id/status_bar', + }), + ]; + const offOrigin = el( + 'android.widget.FrameLayout', + { frame: { x: 0, y: 2270, width: 1080, height: 130 } }, + statusBarIds, + ); + const sizeless = el( + 'android.widget.FrameLayout', + { frame: { x: 0, y: 0, width: 0, height: 0 } }, + statusBarIds, + ); + const app = el('android.widget.FrameLayout', { + frame: { x: 0, y: 0, width: 1080, height: 2400 }, + }); + + expect( + filterAppHierarchy([offOrigin], { platform: 'android' }), + ).toStrictEqual([offOrigin]); + expect( + filterAppHierarchy([sizeless, app], { platform: 'android' }), + ).toStrictEqual([sizeless, app]); + }); + + it('drops only status and navigation bar ids from a flat hierarchy', () => { + const flat = [ + el('android.widget.FrameLayout', { + identifier: 'com.android.systemui:id/status_bar', + }), + el('android.widget.TextView', { + identifier: 'com.android.systemui:id/clock', + value: '8:28 AM', + }), + el('android.widget.FrameLayout', { + identifier: 'com.android.systemui:id/navigation_bar_frame', + }), + el('android.widget.FrameLayout', { + identifier: 'com.android.systemui:id/nav_bar_container', + }), + el('android.widget.Button', { + identifier: 'com.android.systemui:id/button_use_credential', + }), + el('android.widget.Button', { identifier: 'log-in-button' }), + ]; + + expect( + identifiers(filterAppHierarchy(flat, { platform: 'android' })), + ).toStrictEqual([ + 'com.android.systemui:id/clock', + 'com.android.systemui:id/button_use_credential', + 'log-in-button', + ]); + }); + + it('keeps elements of other packages that are not System UI', () => { + const dialog = el('android.widget.FrameLayout', {}, [ + el('android.widget.Button', { + identifier: + 'com.android.permissioncontroller:id/permission_allow_button', + label: 'Allow', + }), + ]); + + const filtered = filterAppHierarchy([dialog], { platform: 'android' }); + + expect(identifiers(filtered)).toStrictEqual([ + 'com.android.permissioncontroller:id/permission_allow_button', + ]); + }); + + it('drops the DevLauncher Tools button in a Compose view only', () => { + const filtered = filterAppHierarchy(androidSnapshot(), { + platform: 'android', + }); + + // The React Native "Tools" text (a value, not a Compose ImageView) + // stays; the overlay ImageView is gone. + expect(labels(filtered)).toStrictEqual(['Wallet']); + }); + + it('keeps the DevLauncher Tools button when excludeDevOverlay is false', () => { + const filtered = filterAppHierarchy(androidSnapshot(), { + platform: 'android', + excludeDevOverlay: false, + }); + + expect(labels(filtered)).toContain('Tools'); + }); + + it('does not filter System UI ids on iOS', () => { + const node = el('Button', { identifier: 'com.android.systemui:id/x' }); + + expect( + filterAppHierarchy([node], { + platform: 'ios', + excludeDevOverlay: false, + }), + ).toStrictEqual([node]); + }); + }); + + describe('ios', () => { + it('drops the DevLauncher gear and its Tools text', () => { + const filtered = filterAppHierarchy(iosSnapshot(), { platform: 'ios' }); + + expect(identifiers(filtered)).toStrictEqual(['log-in-button']); + // The app's own "Tools" text (not after the gear) is kept; the empty + // DevLauncher window is removed. + expect(labels(filtered)).toStrictEqual(['MetaMask', 'Unlock', 'Tools']); + expect(filtered[0]?.children).toHaveLength(1); + }); + + it('keeps the overlay when excludeDevOverlay is false', () => { + const snapshot = iosSnapshot(); + + const filtered = filterAppHierarchy(snapshot, { + platform: 'ios', + excludeDevOverlay: false, + }); + + expect(filtered).toStrictEqual(snapshot); + }); + + it('drops the overlay from a flat idb list', () => { + const flat = [ + el('Other', { label: 'MetaMask' }), + el('Button', { identifier: 'log-in-button' }), + el('Image', { identifier: 'gearshape.fill' }), + el('StaticText', { label: 'Tools' }), + ]; + + const filtered = filterAppHierarchy(flat, { platform: 'ios' }); + + expect( + filtered.map((node) => node.identifier ?? node.label), + ).toStrictEqual(['MetaMask', 'log-in-button']); + }); + }); + + it('returns unchanged subtrees by reference and does not modify the input', () => { + const snapshot = androidSnapshot(); + const before = JSON.stringify(snapshot); + + const filtered = filterAppHierarchy(snapshot, { platform: 'android' }); + + expect(JSON.stringify(snapshot)).toBe(before); + const appContent = filtered[0]?.children?.[0]?.children?.[0]; + const reactRoot = appContent?.children?.[0]; + expect(reactRoot).toBe( + snapshot[1]?.children?.[0]?.children?.[0]?.children?.[1], + ); + }); +}); + +describe('viewportFromHierarchy', () => { + it('reads the iOS viewport from the first element', () => { + expect(viewportFromHierarchy(iosSnapshot(), 'ios')).toStrictEqual({ + width: 402, + height: 874, + }); + }); + + it('ignores later top-level iOS elements (flat idb list)', () => { + const flat = [ + el('Other', { frame: { x: 0, y: 0, width: 402, height: 874 } }), + el('UIView', { frame: { x: 0, y: 0, width: 402, height: 2938 } }), + ]; + + expect(viewportFromHierarchy(flat, 'ios')).toStrictEqual({ + width: 402, + height: 874, + }); + }); + + it('reads the Android viewport from the largest top-level window', () => { + expect(viewportFromHierarchy(androidSnapshot(), 'android')).toStrictEqual({ + width: 1080, + height: 2400, + }); + }); + + it('reads the Android viewport as the extent of all top-level windows', () => { + const windows = [ + el('Window', { frame: { x: 0, y: 0, width: 1080, height: 63 } }), + el('Window', { frame: { x: 0, y: 63, width: 1080, height: 2207 } }), + el('Window', { frame: { x: 0, y: 2270, width: 1080, height: 130 } }), + ]; + + expect(viewportFromHierarchy(windows, 'android')).toStrictEqual({ + width: 1080, + height: 2400, + }); + }); + + it('includes an off-origin Android window in the extent', () => { + const windows = [ + el('Window', { frame: { x: 0, y: 0, width: 1080, height: 63 } }), + el('Dialog', { frame: { x: 40, y: 1500, width: 1000, height: 900 } }), + ]; + + expect(viewportFromHierarchy(windows, 'android')).toStrictEqual({ + width: 1080, + height: 2400, + }); + }); + + it('reads a status-bar-only Android snapshot as the bar', () => { + const [statusBar] = androidSnapshot() as [UIElement]; + + expect(viewportFromHierarchy([statusBar], 'android')).toStrictEqual({ + width: 1080, + height: 63, + }); + }); + + it.each([ + ['empty', []], + [ + 'zero-size', + [el('Other', { frame: { x: 0, y: 0, width: 0, height: 0 } })], + ], + [ + 'off-origin', + [el('Dialog', { frame: { x: 40, y: 600, width: 1000, height: 800 } })], + ], + ])('returns undefined for a %s root', (_name, hierarchy) => { + expect(viewportFromHierarchy(hierarchy, 'android')).toBeUndefined(); + expect(viewportFromHierarchy(hierarchy, 'ios')).toBeUndefined(); + }); +}); diff --git a/src/platform/mobile-hierarchy.ts b/src/platform/mobile-hierarchy.ts new file mode 100644 index 0000000..7567f5f --- /dev/null +++ b/src/platform/mobile-hierarchy.ts @@ -0,0 +1,256 @@ +import type { UIElement } from '@metamask/device-mcp'; + +/** Options of `filterAppHierarchy`. */ +export type AppHierarchyFilterOptions = { + /** Device platform of the snapshot. */ + platform: 'ios' | 'android'; + /** + * Drop the Expo DevLauncher overlay of development builds (the floating + * "Tools" button). Default true. + */ + excludeDevOverlay?: boolean; +}; + +/** Device viewport, in the units of the snapshot frames. */ +export type Viewport = { width: number; height: number }; + +/** + * Android resource-id with a package: `:id/`. React Native + * testIDs have no package (`wallet-screen`). + */ +const RESOURCE_ID_PACKAGE = /^([A-Za-z][\w.]*):id\//u; + +/** Android package of System UI (status bar, navigation bar, and more). */ +const ANDROID_SYSTEM_UI_PACKAGE = 'com.android.systemui'; + +/** + * Status bar and navigation bar containers, for a flat hierarchy (no window + * structure, e.g. Appium) where the bar-window rule cannot apply. + */ +const ANDROID_BAR_IDENTIFIER = + /^com\.android\.systemui:id\/(?:status_bar|navigation_bar|nav_bar)/u; + +/** + * A bar window is at most this fraction of its length thick (on a 1080 px + * wide screen: status bar 63 px, navigation bar up to about 130 px, limit + * 162 px; a biometric prompt is several hundred px high). + */ +const ANDROID_BAR_MAX_THICKNESS = 0.15; + +/** + * iOS: Expo DevLauncher floating button. idb reports an `Image` with + * identifier `gearshape.fill` (label "Gear Shape, Filled") followed by a + * `StaticText` "Tools", in a window of their own (`Application` > + * `UITransitionView` > `UIDropShadowView` > `_UIHostingView` > + * `SwiftUI.AccessibilityNode`). + */ +const IOS_DEV_OVERLAY_IDENTIFIER = 'gearshape.fill'; +const IOS_STATIC_TEXT = 'StaticText'; +const DEV_OVERLAY_LABEL = 'Tools'; + +/** + * Android: Expo DevLauncher floating button, an `ImageView` labelled "Tools" + * in a Jetpack Compose view (React Native does not render Compose views). + */ +const ANDROID_COMPOSE_VIEW = 'androidx.compose.ui.platform.ComposeView'; +const ANDROID_IMAGE_VIEW = 'android.widget.ImageView'; + +/** + * @param identifier - An element identifier. + * @returns The Android package of a `:id/` resource-id, or + * undefined when the identifier has no package. + */ +export function resourceIdPackage(identifier: string): string | undefined { + return RESOURCE_ID_PACKAGE.exec(identifier)?.[1]; +} + +/** + * Removes elements that are not part of the app from a device snapshot. Use + * the result for the accessibility tree, the refs and the test IDs, so they + * agree: + * + * - Android: the status bar and navigation bar windows. A top-level element + * (a window) is removed when every identifier in it is + * `com.android.systemui:id/*` and its frame is a bar: the full screen width + * (or height, for a side navigation bar in landscape) and at most 15% of + * that length thick. Other System + * UI windows stay: the biometric prompt, screen capture consent, the + * notification shade. Flat hierarchy (no top-level element has children): + * only elements whose identifier starts with + * `com.android.systemui:id/status_bar`, `navigation_bar` or `nav_bar`. + * - `excludeDevOverlay` (default true): the Expo DevLauncher "Tools" button + * of development builds. iOS: the `gearshape.fill` image and the "Tools" + * text right after it. Android: a "Tools" `ImageView` inside a Compose view. + * + * A container that held only removed elements, and has no identifier, label + * or value, is removed too. The input is not modified. + * + * @param hierarchy - The snapshot hierarchy. + * @param options - Platform and filters. + * @returns The filtered hierarchy. + */ +export function filterAppHierarchy( + hierarchy: UIElement[], + options: AppHierarchyFilterOptions, +): UIElement[] { + const { platform } = options; + const excludeDevOverlay = options.excludeDevOverlay ?? true; + const roots = + platform === 'android' ? removeAndroidBars(hierarchy) : hierarchy; + if (!excludeDevOverlay) { + return roots; + } + + const isDevOverlay = ( + el: UIElement, + previous: UIElement | undefined, + inComposeView: boolean, + ): boolean => { + if (platform === 'ios') { + return ( + el.identifier === IOS_DEV_OVERLAY_IDENTIFIER || + (el.type === IOS_STATIC_TEXT && + el.label === DEV_OVERLAY_LABEL && + previous?.identifier === IOS_DEV_OVERLAY_IDENTIFIER) + ); + } + return ( + inComposeView && + el.type === ANDROID_IMAGE_VIEW && + el.label === DEV_OVERLAY_LABEL + ); + }; + + const walk = (elements: UIElement[], inComposeView: boolean): UIElement[] => { + const kept: UIElement[] = []; + elements.forEach((el, index) => { + if (isDevOverlay(el, elements[index - 1], inComposeView)) { + return; + } + if (!el.children?.length) { + kept.push(el); + return; + } + const children = walk( + el.children, + inComposeView || el.type === ANDROID_COMPOSE_VIEW, + ); + if (children.length === 0 && !el.identifier && !el.label && !el.value) { + // It held only removed elements. + return; + } + const unchanged = + children.length === el.children.length && + children.every((child, at) => child === el.children?.[at]); + kept.push(unchanged ? el : { ...el, children }); + }); + return kept; + }; + + return walk(roots, false); +} + +/** + * Android: removes the status bar and navigation bar (see + * `filterAppHierarchy`). + * + * @param hierarchy - The snapshot hierarchy. + * @returns The hierarchy without the bars. + */ +function removeAndroidBars(hierarchy: UIElement[]): UIElement[] { + const flat = hierarchy.every((el) => !el.children?.length); + if (flat) { + return hierarchy.filter( + (el) => !el.identifier || !ANDROID_BAR_IDENTIFIER.test(el.identifier), + ); + } + const screen = viewportFromHierarchy(hierarchy, 'android'); + if (!screen) { + return hierarchy; + } + return hierarchy.filter( + (el) => !(isBarFrame(el.frame, screen) && isSystemUiOnly(el)), + ); +} + +/** + * @param frame - A window frame. + * @param screen - The screen size. + * @returns True when the frame spans the full screen width and is at most + * `ANDROID_BAR_MAX_THICKNESS` of that width high (status bar, navigation + * bar), or spans the full screen height and is at most that fraction of it + * wide (side navigation bar in landscape). The thickness is measured against + * the bar's own length, so a snapshot that holds only the status bar (its + * extent is the bar) still finds it. + */ +function isBarFrame(frame: UIElement['frame'], screen: Viewport): boolean { + if (frame.width <= 0 || frame.height <= 0) { + return false; + } + const horizontal = + frame.width >= screen.width && + frame.height <= frame.width * ANDROID_BAR_MAX_THICKNESS; + const vertical = + frame.height >= screen.height && + frame.width <= frame.height * ANDROID_BAR_MAX_THICKNESS; + return horizontal || vertical; +} + +/** + * @param root - A top-level element. + * @returns True when the element tree has at least one identifier and every + * identifier is a `com.android.systemui` resource-id. + */ +function isSystemUiOnly(root: UIElement): boolean { + let found = false; + const allSystemUi = (el: UIElement): boolean => { + if (el.identifier) { + if (resourceIdPackage(el.identifier) !== ANDROID_SYSTEM_UI_PACKAGE) { + return false; + } + found = true; + } + return (el.children ?? []).every(allSystemUi); + }; + return allSystemUi(root) && found; +} + +/** + * Reads the device viewport from a snapshot, with no device call, in the + * units of its frames (iOS points, Android pixels). + * + * - iOS (`idb describe-all`): the frame of the first element, the app, when + * it is at the origin (later top-level elements of a flat list can be scroll + * content). + * - Android: the top-level elements are windows (status bar, app, dialogs). + * The extent of their frames, `max(x + width)` by `max(y + height)`, when + * the leftmost and topmost of them are at 0. A snapshot that holds only a + * dialog window does not show the screen origin, so it gives no viewport. + * + * Pass the unfiltered hierarchy. + * + * @param hierarchy - The snapshot hierarchy. + * @param platform - Device platform of the snapshot. + * @returns The viewport, or undefined when the top-level frames do not give + * one. + */ +export function viewportFromHierarchy( + hierarchy: UIElement[], + platform: 'ios' | 'android', +): Viewport | undefined { + const frames = (platform === 'ios' ? hierarchy.slice(0, 1) : hierarchy) + .map((el) => el.frame) + .filter((frame) => frame.width > 0 && frame.height > 0); + if (frames.length === 0) { + return undefined; + } + const minX = Math.min(...frames.map((frame) => frame.x)); + const minY = Math.min(...frames.map((frame) => frame.y)); + if (minX !== 0 || minY !== 0) { + return undefined; + } + return { + width: Math.max(...frames.map((frame) => frame.x + frame.width)), + height: Math.max(...frames.map((frame) => frame.y + frame.height)), + }; +} diff --git a/src/platform/mobile-platform-driver.test.ts b/src/platform/mobile-platform-driver.test.ts index 0e6af62..34061a7 100644 --- a/src/platform/mobile-platform-driver.test.ts +++ b/src/platform/mobile-platform-driver.test.ts @@ -1,6 +1,7 @@ import type { DeviceBackend, UIElement } from '@metamask/device-mcp'; import { describe, it, expect, vi, afterEach } from 'vitest'; +import androidLoginSnapshot from './__fixtures__/android-login-snapshot.json'; import { MobilePlatformDriver } from './mobile-platform-driver.js'; function makeElement(overrides: Partial = {}): UIElement { @@ -117,6 +118,299 @@ describe('MobilePlatformDriver', () => { }); }); + describe('P1-13: one snapshot, app-only hierarchy, viewport', () => { + const snapshotOf = async ( + platform: 'ios' | 'android', + hierarchy: UIElement[], + ): ReturnType => ({ + platform, + hierarchy, + raw: '', + timestamp: 0, + }); + + // Shaped like a real adb snapshot: System UI status bar window, then + // the app window with the DevLauncher "Tools" button in a Compose view. + const androidHierarchy = (): UIElement[] => [ + makeElement({ + type: 'android.widget.FrameLayout', + frame: { x: 0, y: 0, width: 1080, height: 63 }, + children: [ + makeElement({ + type: 'android.widget.FrameLayout', + identifier: 'com.android.systemui:id/status_bar', + children: [ + makeElement({ + type: 'android.widget.TextView', + identifier: 'com.android.systemui:id/clock', + value: '12:46 AM', + }), + ], + }), + ], + }), + makeElement({ + type: 'android.widget.FrameLayout', + frame: { x: 0, y: 0, width: 1080, height: 2400 }, + children: [ + makeElement({ + type: 'android.widget.LinearLayout', + identifier: 'io.metamask:id/action_bar_root', + frame: { x: 0, y: 0, width: 1080, height: 2400 }, + children: [ + makeElement({ + type: 'androidx.compose.ui.platform.ComposeView', + children: [ + makeElement({ + type: 'android.widget.ImageView', + label: 'Tools', + }), + ], + }), + makeElement({ + type: 'android.widget.Button', + identifier: 'log-in-button', + label: 'Unlock', + frame: { x: 40, y: 1800, width: 1000, height: 120 }, + }), + makeElement({ + type: 'android.widget.TextView', + identifier: 'below-fold', + frame: { x: 0, y: 2600, width: 1080, height: 60 }, + }), + ], + }), + ], + }), + ]; + + it('lists the app test IDs of a real Android login snapshot (AC2)', async () => { + const backend = createMockBackend({ + platform: 'android', + snapshot: vi.fn(async () => + snapshotOf('android', androidLoginSnapshot as UIElement[]), + ), + getWindowSize: vi.fn().mockResolvedValue({ width: 1080, height: 2400 }), + }); + const driver = new MobilePlatformDriver(backend); + + const screen = await driver.captureScreen(); + const testIds = (await screen.getTestIds()).map((item) => item.testId); + + // Before P1-13: 35 test IDs, 22 of them com.android.systemui. + expect(testIds.filter((id) => id.includes('systemui'))).toStrictEqual([]); + expect(testIds).toContain('log-in-button'); + expect(screen.nodes[0]?.bounds).toStrictEqual({ + x: 0, + y: 0, + width: 1080, + height: 2400, + }); + expect(backend.getWindowSize).not.toHaveBeenCalled(); + }); + + it('takes one snapshot and no window size for a capture (AC1)', async () => { + const backend = createMockBackend({ + platform: 'android', + snapshot: vi.fn(async () => snapshotOf('android', androidHierarchy())), + getWindowSize: vi.fn().mockResolvedValue({ width: 1080, height: 2400 }), + }); + const driver = new MobilePlatformDriver(backend); + + const screen = await driver.captureScreen(); + await screen.getTestIds(); + await screen.getTestIds(1); + + expect(backend.snapshot).toHaveBeenCalledTimes(1); + // The viewport comes from the snapshot's top-level window frames. + expect(backend.getWindowSize).not.toHaveBeenCalled(); + }); + + it('drops com.android.systemui test IDs, nodes and refs (AC2)', async () => { + const backend = createMockBackend({ + platform: 'android', + snapshot: vi.fn(async () => snapshotOf('android', androidHierarchy())), + }); + const driver = new MobilePlatformDriver(backend); + + const screen = await driver.captureScreen(); + const testIds = await screen.getTestIds(); + const listed = await driver.getTestIds(); + const tree = await driver.getAccessibilityTree(); + + for (const ids of [ + testIds.map((item) => item.testId), + listed.map((item) => item.testId), + screen.nodes.flatMap((node) => (node.testId ? [node.testId] : [])), + [...screen.refMap.values()], + tree.nodes.map((node) => node.name), + ]) { + expect(ids.join(' ')).not.toMatch(/systemui|12:46|Tools/u); + } + expect(testIds).toStrictEqual([ + { + testId: 'io.metamask:id/action_bar_root', + tag: 'android.widget.LinearLayout', + text: undefined, + visible: true, + }, + { + testId: 'log-in-button', + tag: 'android.widget.Button', + text: 'Unlock', + visible: true, + }, + { + testId: 'below-fold', + tag: 'android.widget.TextView', + text: undefined, + visible: false, + }, + ]); + // Tree and test IDs come from the same filtered hierarchy. + expect( + screen.nodes.flatMap((node) => (node.testId ? [node.testId] : [])), + ).toStrictEqual(testIds.map((item) => item.testId)); + expect(screen.nodes[0]?.ref).toBe('e1'); + expect(screen.nodes[0]?.role).toBe('android.widget.FrameLayout'); + }); + + it('drops the iOS DevLauncher overlay by default', async () => { + const hierarchy = [ + makeElement({ + type: 'Other', + label: 'MetaMask', + frame: { x: 0, y: 0, width: 402, height: 874 }, + children: [ + makeElement({ identifier: 'log-in-button', label: 'Unlock' }), + makeElement({ + type: 'Image', + identifier: 'gearshape.fill', + label: 'Gear Shape, Filled', + }), + makeElement({ type: 'StaticText', label: 'Tools' }), + ], + }), + ]; + const backend = createMockBackend({ + snapshot: vi.fn(async () => snapshotOf('ios', hierarchy)), + }); + const driver = new MobilePlatformDriver(backend); + + const screen = await driver.captureScreen(); + + expect(screen.nodes.map((node) => node.name)).toStrictEqual([ + 'MetaMask', + 'Unlock', + ]); + expect( + (await screen.getTestIds()).map((item) => item.testId), + ).toStrictEqual(['log-in-button']); + }); + + it('keeps the DevLauncher overlay with excludeDevOverlay: false', async () => { + const hierarchy = [ + makeElement({ type: 'Image', identifier: 'gearshape.fill' }), + makeElement({ type: 'StaticText', label: 'Tools' }), + ]; + const backend = createMockBackend({ + snapshot: vi.fn(async () => snapshotOf('ios', hierarchy)), + }); + const driver = new MobilePlatformDriver(backend, 'io.metamask', { + excludeDevOverlay: false, + }); + + const screen = await driver.captureScreen(); + + expect(screen.nodes.map((node) => node.name)).toStrictEqual([ + '', + 'Tools', + ]); + expect( + (await screen.getTestIds()).map((item) => item.testId), + ).toStrictEqual(['gearshape.fill']); + }); + + it('reads the window size at most once across observations without a root frame', async () => { + const backend = createMockBackend({ + snapshot: vi.fn(async () => + snapshotOf('ios', [ + makeElement({ + identifier: 'below', + frame: { x: 0, y: 2000, width: 10, height: 10 }, + }), + ]), + ), + getWindowSize: vi.fn().mockResolvedValue({ width: 402, height: 874 }), + }); + const driver = new MobilePlatformDriver(backend); + + for (let i = 0; i < 5; i++) { + const screen = await driver.captureScreen(); + expect((await screen.getTestIds())[0]?.visible).toBe(false); + } + await driver.getTestIds(); + + expect(backend.snapshot).toHaveBeenCalledTimes(6); + expect(backend.getWindowSize).toHaveBeenCalledTimes(1); + }); + + it('does not cache a failed window size read', async () => { + const backend = createMockBackend({ + snapshot: vi.fn(async () => + snapshotOf('ios', [ + makeElement({ + identifier: 'below', + frame: { x: 0, y: 2000, width: 10, height: 10 }, + }), + ]), + ), + getWindowSize: vi + .fn() + .mockRejectedValueOnce(new Error('idb describe failed')) + .mockResolvedValue({ width: 402, height: 874 }), + }); + const driver = new MobilePlatformDriver(backend); + + const first = await driver.getTestIds(); + const second = await driver.getTestIds(); + await driver.getTestIds(); + + expect(first[0]?.visible).toBe(true); + expect(second[0]?.visible).toBe(false); + expect(backend.getWindowSize).toHaveBeenCalledTimes(2); + }); + + it('prefers the snapshot viewport over the window size', async () => { + // Landscape: the root frame is wider than `wm size` (natural + // orientation); the snapshot is what the screen shows now. + const backend = createMockBackend({ + platform: 'android', + snapshot: vi.fn(async () => + snapshotOf('android', [ + makeElement({ + type: 'android.widget.FrameLayout', + frame: { x: 0, y: 0, width: 2400, height: 1080 }, + children: [ + makeElement({ + identifier: 'right-side', + frame: { x: 2000, y: 100, width: 200, height: 100 }, + }), + ], + }), + ]), + ), + getWindowSize: vi.fn().mockResolvedValue({ width: 1080, height: 2400 }), + }); + const driver = new MobilePlatformDriver(backend); + + const items = await driver.getTestIds(); + + expect(items[0]?.visible).toBe(true); + expect(backend.getWindowSize).not.toHaveBeenCalled(); + }); + }); + describe('getPlatform', () => { it('returns ios for ios backend', () => { const driver = new MobilePlatformDriver(createMockBackend()); @@ -1267,6 +1561,28 @@ describe('MobilePlatformDriver', () => { }); }); + it('leaves out the status bar and the DevLauncher overlay', async () => { + const backend = createMockBackend({ + platform: 'android', + snapshot: vi.fn().mockResolvedValue({ + platform: 'android', + hierarchy: androidLoginSnapshot, + raw: '', + timestamp: 0, + }), + }); + const driver = new MobilePlatformDriver(backend); + + const locators = await driver.generateLocators(); + + expect(backend.snapshot).toHaveBeenCalledTimes(1); + const values = locators.flatMap((locator) => + locator.suggestions.map((suggestion) => suggestion.value), + ); + expect(values).toContain('log-in-button'); + expect(values.join(' ')).not.toMatch(/systemui|Tools/u); + }); + it('returns an empty list when the snapshot has no interactive elements', async () => { const driver = new MobilePlatformDriver(createMockBackend()); diff --git a/src/platform/mobile-platform-driver.ts b/src/platform/mobile-platform-driver.ts index 4eee7aa..1933ccc 100644 --- a/src/platform/mobile-platform-driver.ts +++ b/src/platform/mobile-platform-driver.ts @@ -14,6 +14,11 @@ import type { UIElement, } from '@metamask/device-mcp'; +import { + filterAppHierarchy, + viewportFromHierarchy, +} from './mobile-hierarchy.js'; +import type { Viewport } from './mobile-hierarchy.js'; import type { IPlatformDriver, TargetType, @@ -57,6 +62,15 @@ const HERMES_BLOCKED_METHOD_CODE = 'HERMES_BLOCKED_METHOD'; // agent-facing contract matches the browser and Hermes drivers. const WEBVIEW_BLOCKED_METHOD_CODE = 'WEBVIEW_BLOCKED_METHOD'; +/** Options of `MobilePlatformDriver`. */ +export type MobilePlatformDriverOptions = { + /** + * Remove the Expo DevLauncher "Tools" button of development builds from + * the accessibility tree and the test IDs. Default true. + */ + excludeDevOverlay?: boolean; +}; + /** * Platform driver for mobile devices backed by @metamask/device-mcp. * Wraps DeviceBackend behind the IPlatformDriver interface so the @@ -73,14 +87,28 @@ export class MobilePlatformDriver implements IPlatformDriver { // across mobile sessions (e.g. a relaunch against a different simulator). readonly #hermesSession: HermesSession; + readonly #excludeDevOverlay: boolean; + + #windowSize: Viewport | undefined; + /** * @param backend - The device-mcp backend to delegate to. * @param bundleId - The app bundle ID for getAppState calls. + * @param options - Driver options. + * @param options.excludeDevOverlay - Remove the Expo DevLauncher "Tools" + * button of development builds from the accessibility tree and the test + * IDs (default true). Android System UI (status bar, navigation bar) is + * always removed. */ - constructor(backend: DeviceBackend, bundleId: string = DEFAULT_BUNDLE_ID) { + constructor( + backend: DeviceBackend, + bundleId: string = DEFAULT_BUNDLE_ID, + options: MobilePlatformDriverOptions = {}, + ) { this.#backend = backend; this.#bundleId = bundleId; this.#hermesSession = new HermesSession({ platform: backend.platform }); + this.#excludeDevOverlay = options.excludeDevOverlay ?? true; } /** @@ -197,8 +225,8 @@ export class MobilePlatformDriver implements IPlatformDriver { async getAccessibilityTree( _rootSelector?: string, ): Promise<{ nodes: A11yNodeTrimmed[]; refMap: Map }> { - const snapshot = await this.#backend.snapshot(); - return normalizeSnapshot(snapshot.hierarchy); + const { hierarchy } = await this.#snapshotApp(); + return normalizeSnapshot(hierarchy); } /** @@ -208,8 +236,8 @@ export class MobilePlatformDriver implements IPlatformDriver { * viewport; elements with unknown geometry are assumed visible. */ async getTestIds(limit?: number): Promise { - const snapshot = await this.#backend.snapshot(); - return this.#testIdsOf(snapshot.hierarchy, limit); + const { hierarchy, viewport } = await this.#snapshotApp(); + return this.#testIdsOf(hierarchy, viewport, limit); } /** @@ -219,36 +247,76 @@ export class MobilePlatformDriver implements IPlatformDriver { * @returns The nodes, refMap and a test ID reader for the same snapshot. */ async captureScreen(): Promise { - const snapshot = await this.#backend.snapshot(); + const { hierarchy, viewport } = await this.#snapshotApp(); return { - ...normalizeSnapshot(snapshot.hierarchy), + ...normalizeSnapshot(hierarchy), getTestIds: async (limit?: number) => - this.#testIdsOf(snapshot.hierarchy, limit), + this.#testIdsOf(hierarchy, viewport, limit), }; } /** - * @param hierarchy - The UIElement tree of a snapshot. + * Takes one device snapshot, reads the viewport from it, and removes the + * elements that are not part of the app (see `filterAppHierarchy`), so the + * tree, the refs and the test IDs all use the same filtered hierarchy. + * + * @returns The filtered hierarchy and the viewport of the snapshot. + */ + async #snapshotApp(): Promise<{ + hierarchy: UIElement[]; + viewport: Viewport | undefined; + }> { + const snapshot = await this.#backend.snapshot(); + const { platform } = this.#backend; + return { + hierarchy: filterAppHierarchy(snapshot.hierarchy, { + platform, + excludeDevOverlay: this.#excludeDevOverlay, + }), + viewport: viewportFromHierarchy(snapshot.hierarchy, platform), + }; + } + + /** + * @param hierarchy - The filtered UIElement tree of a snapshot. + * @param snapshotViewport - The viewport read from that snapshot, if any. * @param limit - Maximum number of test IDs to return. * @returns The test IDs, with visibility against the device viewport. */ async #testIdsOf( hierarchy: UIElement[], + snapshotViewport: Viewport | undefined, limit?: number, ): Promise { const items: TestIdItem[] = []; const max = limit ?? OBSERVATION_TESTID_LIMIT; - let viewport: { width: number; height: number } | undefined; - try { - viewport = await this.#backend.getWindowSize(); - } catch { - // Without a viewport we cannot judge visibility; report everything. - viewport = undefined; - } + const viewport = snapshotViewport ?? (await this.#cachedWindowSize()); collectTestIds(hierarchy, items, max, viewport); return items; } + /** + * The device window size, read from the backend once per driver (a device + * round trip: `idb describe` / `adb shell wm size`). Only used when a + * snapshot has no top-level frame to read the viewport from. A failed read + * is not cached. The cached size is never invalidated, so after a rotation + * it keeps the old orientation; snapshots with root frames (all idb and + * adb snapshots seen so far) do not use it. + * + * @returns The window size, or undefined when the backend cannot read it. + */ + async #cachedWindowSize(): Promise { + if (!this.#windowSize) { + try { + this.#windowSize = await this.#backend.getWindowSize(); + } catch { + // Without a viewport we cannot judge visibility; report everything. + return undefined; + } + } + return this.#windowSize; + } + /** * @param options - Screenshot options. * @returns Screenshot result with the path and optional base64 data. @@ -649,13 +717,15 @@ export class MobilePlatformDriver implements IPlatformDriver { /** * Generates ranked locator suggestions for every interactive element on the - * current screen by walking the device snapshot hierarchy. Mobile only. + * current screen by walking the device snapshot hierarchy, without the + * elements that are not part of the app (see `filterAppHierarchy`). Mobile + * only. * * @returns The ranked element locators. */ async generateLocators(): Promise { - const snapshot = await this.#backend.snapshot(); - return generateLocators(snapshot.hierarchy); + const { hierarchy } = await this.#snapshotApp(); + return generateLocators(hierarchy); } } diff --git a/src/server/create-server.test.ts b/src/server/create-server.test.ts index 877a17b..7db8f0a 100644 --- a/src/server/create-server.test.ts +++ b/src/server/create-server.test.ts @@ -2679,6 +2679,142 @@ describe('createServer abort, ref generations and observations', () => { }); }); + describe('P1-13: device snapshots per observation (MobilePlatformDriver)', () => { + type Element = { + type: string; + label?: string; + value?: string; + identifier?: string; + frame: { x: number; y: number; width: number; height: number }; + enabled: boolean; + children?: Element[]; + }; + + const frame = { x: 0, y: 0, width: 1080, height: 2400 }; + // Android screen: the status bar (System UI, clock changes every + // snapshot) and the app window. + const screen = (clock: string, names: string[]): Element[] => [ + { + type: 'android.widget.FrameLayout', + frame: { x: 0, y: 0, width: 1080, height: 63 }, + enabled: true, + children: [ + { + type: 'android.widget.TextView', + identifier: 'com.android.systemui:id/clock', + value: clock, + frame: { x: 11, y: 2, width: 136, height: 58 }, + enabled: true, + }, + ], + }, + { + type: 'android.widget.FrameLayout', + frame, + enabled: true, + children: names.map((name) => ({ + type: 'android.widget.Button', + identifier: `btn-${name}`, + label: name, + frame: { x: 0, y: 100, width: 200, height: 100 }, + enabled: true, + })), + }, + ]; + + let minute = 0; + let names: string[] = []; + let backend: { + platform: 'android'; + snapshot: ReturnType; + getWindowSize: ReturnType; + getAppState: ReturnType; + waitForElement: ReturnType; + tapElement: ReturnType; + }; + + const startWithRealDriver = async (): Promise => { + minute = 0; + names = ['Wallet']; + backend = { + platform: 'android', + // A new clock value on every snapshot, like a real status bar. + snapshot: vi.fn(async () => { + minute += 1; + return { + platform: 'android', + hierarchy: screen(`12:${String(minute).padStart(2, '0')}`, names), + raw: '', + timestamp: 0, + }; + }), + getWindowSize: vi.fn().mockResolvedValue({ width: 1080, height: 2400 }), + getAppState: vi + .fn() + .mockResolvedValue({ bundleId: 'io.metamask', state: 'Running' }), + waitForElement: vi.fn().mockResolvedValue(undefined), + tapElement: vi.fn().mockResolvedValue({ success: true }), + }; + const { MobilePlatformDriver } = + await import('../platform/mobile-platform-driver.js'); + const realDriver = new MobilePlatformDriver( + backend as unknown as ConstructorParameters< + typeof MobilePlatformDriver + >[0], + ); + await startServer({ mobile: false }); + mockSM.getPlatformDriver.mockReturnValue(realDriver); + }; + + it('takes one backend.snapshot() per observation when the screen changed (AC1)', async () => { + await startWithRealDriver(); + + await post('/tool/click', { testId: 'btn-Wallet' }); + expect(backend.snapshot).toHaveBeenCalledTimes(1); + + names = ['Send']; + const { body } = await post('/tool/click', { testId: 'btn-Wallet' }); + + expect(backend.snapshot).toHaveBeenCalledTimes(2); + expect(backend.getWindowSize).not.toHaveBeenCalled(); + const testIds = ( + body as unknown as { observations: { testIds: { testId: string }[] } } + ).observations.testIds.map((item) => item.testId); + expect(testIds).toStrictEqual(['btn-Send']); + }); + + it('retries an unchanged screen once even though the Android clock changed', async () => { + await startWithRealDriver(); + + await post('/tool/click', { testId: 'btn-Wallet' }); + const { body } = await post('/tool/click', { testId: 'btn-Wallet' }); + + // Second click: the screen equals the last one once System UI is + // removed, so the P1-09 settle takes its one unchanged retry. + expect(backend.snapshot).toHaveBeenCalledTimes(3); + expect(body.observations?.a11y.nodes).toStrictEqual([]); + expect(body.observations?.a11y.diff).toStrictEqual({ + added: [], + removed: [], + unchanged: 2, + }); + }); + + it.each([ + ['describe_screen', {}], + ['list_testids', {}], + ['accessibility_snapshot', {}], + ])('takes one backend.snapshot() for %s', async (tool, input) => { + await startWithRealDriver(); + + const { body } = await post(`/tool/${tool}`, input); + + expect(body.ok).toBe(true); + expect(backend.snapshot).toHaveBeenCalledTimes(1); + expect(JSON.stringify(body)).not.toContain('systemui'); + }); + }); + describe('P1-09: mobile settle after a mutating tool', () => { it('re-snapshots once after 300 ms when the screen looks unchanged', async () => { await startServer(); @@ -2820,15 +2956,74 @@ describe('createServer abort, ref generations and observations', () => { it('stops settling at MOBILE_SETTLE_MAX_MS', async () => { await startServer(); driver.getAccessibilityTree.mockImplementation(async () => { - await sleep(700); + await sleep(800); + return buildTree(['Unlock, busy']); + }); + + await post('/tool/click', { testId: 'x' }); + + // 1st snapshot 800 ms, then 300 ms + 800 ms = 1900 ms; a second wait + // would end at 2200 ms, after the 2000 ms cap (the first snapshot + // counts). Kept well away from the cap so timer jitter cannot flip it. + expect(driver.getAccessibilityTree).toHaveBeenCalledTimes(2); + }); + + it('counts the first snapshot against MOBILE_SETTLE_MAX_MS', async () => { + await startServer(); + let calls = 0; + driver.getAccessibilityTree.mockImplementation(async () => { + calls += 1; + // The click's first capture is slow (a slow Android snapshot). + await sleep(calls === 2 ? 1_600 : 0); return buildTree(['Unlock, busy']); }); + await post('/tool/click', { testId: 'x', observe: 'none' }); + await post('/tool/click', { testId: 'x' }); + + // 1600 ms capture + 300 ms wait + snapshot = 1900 ms; a second wait + // would end at 2200 ms, after 2000 ms. Counting from after the first + // capture would allow 3 extra snapshots. + expect(driver.getAccessibilityTree).toHaveBeenCalledTimes(2); + }); + + it('does not retry an unchanged screen after a slow first capture', async () => { + await startServer(); + let calls = 0; + driver.getAccessibilityTree.mockImplementation(async () => { + calls += 1; + // Well above MOBILE_SETTLE_SLOW_CAPTURE_MS: timers can fire a ms + // early against Date.now(). + await sleep(calls === 2 ? 1_200 : 0); + return buildTree(['Same']); + }); + + await post('/tool/click', { testId: 'x' }); + await post('/tool/click', { testId: 'x' }); + + // 1 (first click) + 1 slow capture, no unchanged retry. + expect(driver.getAccessibilityTree).toHaveBeenCalledTimes(2); + }); + + it('still retries a busy screen after a slow first capture', async () => { + await startServer(); + let calls = 0; + driver.getAccessibilityTree.mockImplementation(async () => { + calls += 1; + if (calls === 2) { + await sleep(1_200); + return buildTree(['Unlock, busy']); + } + return buildTree(calls === 1 ? ['Unlock'] : ['Wallet']); + }); + await post('/tool/click', { testId: 'x' }); + const { body } = await post('/tool/click', { testId: 'x' }); - // 1st snapshot, then (300 ms + 700 ms) twice = 2000 ms; a third wait - // would end after the 2000 ms cap. expect(driver.getAccessibilityTree).toHaveBeenCalledTimes(3); + expect( + body.observations?.a11y.nodes.map((node) => node.name), + ).toStrictEqual(['Wallet']); }); it('does not settle when the tool set refs', async () => { diff --git a/src/server/create-server.ts b/src/server/create-server.ts index 5a49a58..d9dd8cd 100644 --- a/src/server/create-server.ts +++ b/src/server/create-server.ts @@ -41,6 +41,7 @@ import { MOBILE_SETTLE_MAX_MS, MOBILE_SETTLE_MAX_SNAPSHOTS, MOBILE_SETTLE_MS, + MOBILE_SETTLE_SLOW_CAPTURE_MS, MOBILE_SETTLE_UNCHANGED_RETRIES, OBSERVATION_TESTID_LIMIT, OBSERVATION_TIMEOUT_MS, @@ -363,7 +364,8 @@ export function createServer(config: ServerConfig): ServerInstance { /** * Takes one mobile snapshot: one device snapshot when the driver has - * `captureScreen`, else the tree and the test IDs separately. + * `captureScreen` (`MobilePlatformDriver`), else the tree and the test IDs + * separately (two device snapshots). * * @param driver - The mobile platform driver. * @returns The tree and a test ID reader. @@ -390,12 +392,15 @@ export function createServer(config: ServerConfig): ServerInstance { * - Unchanged (equals the last screen the agent received, ignoring refs) * and not busy: retry at most `MOBILE_SETTLE_UNCHANGED_RETRIES` time(s). * Many taps leave the screen as it is (for example tapping the tab that is - * already open), so more retries would only add device snapshots. On - * Android the status bar (clock, signal) usually differs between - * snapshots, so this rule rarely fires there. + * already open), so more retries would only add device snapshots. This + * applies to Android too: the mobile driver removes the status bar + * (clock, signal), so it does not make every snapshot differ. No unchanged + * retry when the first snapshot took `MOBILE_SETTLE_SLOW_CAPTURE_MS` or + * more: the UI already had that time to change. * - In total: at most `MOBILE_SETTLE_MAX_SNAPSHOTS` extra snapshots, and no * wait that would end later than `MOBILE_SETTLE_MAX_MS` after the first - * snapshot. A spinner that never stops costs at most these limits. + * snapshot started (the snapshots count, so a slow Android capture leaves + * less time). A spinner that never stops costs at most these limits. * * The test IDs come from the final snapshot, so they describe the same * screen as the tree. @@ -410,9 +415,11 @@ export function createServer(config: ServerConfig): ServerInstance { settle: boolean, signal: AbortSignal, ): Promise { + const settleStartedAt = Date.now(); let screen = await captureMobile(driver); if (settle) { - const settleStartedAt = Date.now(); + const slowCapture = + Date.now() - settleStartedAt >= MOBILE_SETTLE_SLOW_CAPTURE_MS; const previous = lastScreen ? screenSignature(lastScreen) : null; let unchangedRetries = 0; const shouldRetry = (nodes: A11yNodeTrimmed[]): boolean => { @@ -421,6 +428,7 @@ export function createServer(config: ServerConfig): ServerInstance { } if ( previous === null || + slowCapture || unchangedRetries >= MOBILE_SETTLE_UNCHANGED_RETRIES || screenSignature(nodes) !== previous ) { diff --git a/src/tools/utils/constants.ts b/src/tools/utils/constants.ts index 6b8d333..825b207 100644 --- a/src/tools/utils/constants.ts +++ b/src/tools/utils/constants.ts @@ -64,6 +64,14 @@ export const MOBILE_SETTLE_MAX_SNAPSHOTS = 3; */ export const MOBILE_SETTLE_UNCHANGED_RETRIES = 1; +/** + * Mobile observation settle: when the first snapshot took at least this long + * (an Android `am instrument` capture can take seconds), the UI already had + * that time to change, so an unchanged screen is not retried. A busy screen + * still is. + */ +export const MOBILE_SETTLE_SLOW_CAPTURE_MS = 1_000; + /** Timeout for graceful HTTP server close during daemon shutdown */ export const GRACEFUL_SHUTDOWN_TIMEOUT_MS = 10_000; From 262b778f9bb2bf10b3bb7a6dc80d3bcb6cf0c771 Mon Sep 17 00:00:00 2001 From: cryptotavares Date: Mon, 26 Oct 2026 18:11:54 +0000 Subject: [PATCH 2/2] refactor: clarify mobile hierarchy child reuse --- src/platform/mobile-hierarchy.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/platform/mobile-hierarchy.ts b/src/platform/mobile-hierarchy.ts index 7567f5f..d73d9b2 100644 --- a/src/platform/mobile-hierarchy.ts +++ b/src/platform/mobile-hierarchy.ts @@ -127,12 +127,13 @@ export function filterAppHierarchy( if (isDevOverlay(el, elements[index - 1], inComposeView)) { return; } - if (!el.children?.length) { + const originalChildren = el.children; + if (!originalChildren?.length) { kept.push(el); return; } const children = walk( - el.children, + originalChildren, inComposeView || el.type === ANDROID_COMPOSE_VIEW, ); if (children.length === 0 && !el.identifier && !el.label && !el.value) { @@ -140,8 +141,8 @@ export function filterAppHierarchy( return; } const unchanged = - children.length === el.children.length && - children.every((child, at) => child === el.children?.[at]); + children.length === originalChildren.length && + children.every((child, at) => child === originalChildren[at]); kept.push(unchanged ? el : { ...el, children }); }); return kept;