From e3e15fa1fc066592e3c30c67dc09f3c6d78ddbf6 Mon Sep 17 00:00:00 2001 From: Patrick Bertsch Date: Mon, 31 Aug 2026 16:07:06 -0600 Subject: [PATCH] fix: tap no longer bypasses occlusion via the direct-invoke fast path (FP-10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _tryDirectTap invokes a Semantics-wrapped button's onTap by walking the Element tree structurally, with no relationship to paint order — unlike a real hit-tested pointer tap, which Flutter's own pointer dispatch resolves correctly. That let `tap` succeed against a widget hidden behind a modal barrier, loading overlay, or Stack sibling a real user's tap would hit instead. tap now runs a read-only hit test (the same hitTestInView call Flutter's own dispatch makes internally) before taking the direct-invoke path, and only uses it when the target is genuinely the topmost thing at its own screen position — otherwise it falls through to the existing real hit-tested pointer tap, which already lands on whatever's actually on top. No change needed for _doubleTap/_longPress — they never had the direct-invoke branch to begin with. Version bump to 0.14.0 (agent + CLI, kept in lockstep) to ship this alongside the already-unreleased FP-6 GPS route simulation feature. --- CHANGELOG.md | 13 ++++ VERSION | 2 +- docs/wiki/Home.md | 2 +- probe_agent/CHANGELOG.md | 10 +++ probe_agent/lib/src/executor.dart | 74 +++++++++++++++---- probe_agent/pubspec.yaml | 2 +- probe_agent/test/tap_occlusion_test.dart | 90 ++++++++++++++++++++++++ vscode/package.json | 2 +- 8 files changed, 177 insertions(+), 18 deletions(-) create mode 100644 probe_agent/test/tap_occlusion_test.dart diff --git a/CHANGELOG.md b/CHANGELOG.md index 2909598..e974319 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +## [0.14.0] - 2026-08-31 + ### Added - **GPS route simulation (`travel to ... over N seconds`, FP-6).** A new block-style ProbeScript construct — matching Maestro's `travel` command — that walks the device's GPS location through @@ -34,6 +36,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Added `studio/app_test.go` — the studio module previously had zero test coverage — covering the pure/testable surface (`isDeviceReady`, `extractLineCol`, file-path guards, `Lint`, `ListDir`, `Connect`/`ConnectWiFi` input validation). +- **`tap` could invoke a Semantics-wrapped button's `onTap` even when something else covered it + on screen (FP-10, #265).** `_tryDirectTap` walks the Element tree structurally to find a + `GestureDetector`/`InkResponse` descendant and calls its `onTap` directly (added for PT-04/PT-05, + since a real hit-tested pointer tap doesn't reliably reach focus/`onTap` through a `Semantics` + wrapper) — but that walk has no relationship to paint order, so it could fire on a button hidden + behind a modal barrier, loading overlay, or an unrelated `Stack` sibling that a real user's tap + would hit instead. `tap` now runs a read-only hit test (the same `hitTestInView` call Flutter's + own pointer dispatch makes internally) before taking the direct-invoke path, and only uses it + when the target is genuinely the topmost thing at its own screen position — otherwise it falls + through to the existing real hit-tested pointer tap, which already lands on whatever's actually + on top. `probe_agent/lib/src/executor.dart`; `probe_agent/test/tap_occlusion_test.dart`. ## [0.13.0] - 2026-08-15 diff --git a/VERSION b/VERSION index 54d1a4f..a803cc2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.13.0 +0.14.0 diff --git a/docs/wiki/Home.md b/docs/wiki/Home.md index ebd2999..fb7f4f3 100644 --- a/docs/wiki/Home.md +++ b/docs/wiki/Home.md @@ -18,7 +18,7 @@ Welcome to the FlutterProbe wiki. This documentation covers architecture details ## Project Status -FlutterProbe is in active development. Current version: **0.13.0**. +FlutterProbe is in active development. Current version: **0.14.0**. ### Repository Structure diff --git a/probe_agent/CHANGELOG.md b/probe_agent/CHANGELOG.md index 4b7d0c1..2291d83 100644 --- a/probe_agent/CHANGELOG.md +++ b/probe_agent/CHANGELOG.md @@ -2,6 +2,16 @@ ## [Unreleased] +## 0.14.0 - 2026-08-31 + +- Fixed: `tap #id` could invoke a Semantics-wrapped button's `onTap` directly even when something + else (a modal barrier, a loading overlay, an unrelated `Stack` sibling) covered it on screen — + `_tryDirectTap`'s Element-tree walk has no relationship to paint order. Now gated behind a + read-only hit test (the same `hitTestInView` call Flutter's own pointer dispatch uses + internally); only takes the direct-invoke fast path when the target is genuinely the topmost + thing at its own screen position, otherwise falls through to the existing real hit-tested + pointer tap (FP-10). + ## 0.13.0 - 2026-08-15 - No agent-side changes — version kept in lockstep with the CLI's 0.13.0 release. diff --git a/probe_agent/lib/src/executor.dart b/probe_agent/lib/src/executor.dart index 409d40e..1944d1b 100644 --- a/probe_agent/lib/src/executor.dart +++ b/probe_agent/lib/src/executor.dart @@ -328,20 +328,33 @@ class ProbeExecutor { // Check if the matched element is a Semantics wrapper — if so, the // synthetic gesture may not reach the GestureDetector child. In that - // case, invoke onTap directly instead of using pointer events. + // case, invoke onTap directly instead of using pointer events — but + // only when the target is actually the topmost thing at its own + // screen position (FP-10). Direct invocation has no relationship to + // paint order (unlike a real pointer tap, which Flutter's own + // hit-testing already resolves correctly — see _createGesture below), + // so without this guard it could fire onTap on a button that's + // actually hidden behind a modal barrier, loading overlay, or Stack + // sibling a real user's tap would hit instead. if (element.widget is Semantics) { - final tapped = _tryDirectTap(element); - if (tapped) return; + final target = _findDirectTapTarget(element); + final targetBox = target?.renderObject; + if (target != null && targetBox is RenderBox && _isTopmostAt(targetBox, center)) { + _invokeOnTap(target); + return; + } + // Not found, or occluded — fall through to a real hit-tested + // pointer tap, which lands on whatever is actually on top instead. } final gesture = await _createGesture(center); await gesture.up(); } - /// Walks down from [element] to find a GestureDetector or InkResponse - /// child and invokes its onTap directly. Only used when the matched - /// element is a Semantics wrapper where synthetic pointer events are - /// unreliable. Returns true if onTap was invoked. + /// Walks down from [element] to find the nearest GestureDetector or + /// InkResponse descendant with a non-null onTap — the same widget a + /// direct tap would invoke. Returns the Element without invoking + /// anything, so the caller can check occlusion (FP-10) before firing. /// /// PT-05: checks `InkResponse` rather than only `InkWell` — `InkWell` is /// just a subclass of `InkResponse` with a fixed splash shape, and modern @@ -356,20 +369,18 @@ class ProbeExecutor { /// Semantics-wrapped button with no onTap SemanticsAction, or shadowed by /// an overlapping Semantics node) since Semantics doesn't participate in /// hit-testing at all. - bool _tryDirectTap(Element element) { - bool found = false; + Element? _findDirectTapTarget(Element element) { + Element? found; void visit(Element e) { - if (found) return; + if (found != null) return; try { final widget = e.widget; if (widget is GestureDetector && widget.onTap != null) { - widget.onTap!(); - found = true; + found = e; return; } if (widget is InkResponse && widget.onTap != null) { - widget.onTap!(); - found = true; + found = e; return; } e.visitChildren(visit); @@ -381,6 +392,41 @@ class ProbeExecutor { return found; } + /// Invokes the onTap callback on an Element located by + /// [_findDirectTapTarget]. + void _invokeOnTap(Element element) { + final widget = element.widget; + if (widget is GestureDetector) { + widget.onTap!(); + } else if (widget is InkResponse) { + widget.onTap!(); + } + } + + /// FP-10: returns true if [target] is the render object a real pointer + /// tap at [position] would actually reach — i.e. nothing else (a + /// ModalBarrier, a loading overlay, an unrelated Stack sibling) is + /// painted on top of it at that exact point. + /// + /// Uses the same `hitTestInView` call Flutter's own pointer dispatch + /// makes internally (see `GestureBinding.handlePointerEvent`), but as a + /// read-only query — no event is actually dispatched, so this has no + /// side effects on the widget tree. + bool _isTopmostAt(RenderObject target, Offset position) { + final view = WidgetsBinding.instance.platformDispatcher.implicitView; + if (view == null) return true; // no view to hit-test against — don't block + final result = HitTestResult(); + GestureBinding.instance.hitTestInView(result, position, view.viewId); + for (final entry in result.path) { + RenderObject? candidate = entry.target is RenderObject ? entry.target as RenderObject : null; + while (candidate != null) { + if (identical(candidate, target)) return true; + candidate = candidate.parent; + } + } + return false; + } + Future _doubleTap(Map sel) async { final element = _requireElement(sel); final box = element.renderObject as RenderBox; diff --git a/probe_agent/pubspec.yaml b/probe_agent/pubspec.yaml index 3a338ce..2ebd7b5 100644 --- a/probe_agent/pubspec.yaml +++ b/probe_agent/pubspec.yaml @@ -3,7 +3,7 @@ description: >- On-device E2E test agent for FlutterProbe. Embeds in your Flutter app and executes test commands via direct widget-tree access with sub-50ms latency. -version: 0.13.0 +version: 0.14.0 homepage: https://flutterprobe.dev repository: https://github.com/AlphaWaveSystems/flutter-probe issue_tracker: https://github.com/AlphaWaveSystems/flutter-probe/issues diff --git a/probe_agent/test/tap_occlusion_test.dart b/probe_agent/test/tap_occlusion_test.dart new file mode 100644 index 0000000..0b1a417 --- /dev/null +++ b/probe_agent/test/tap_occlusion_test.dart @@ -0,0 +1,90 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter_probe_agent/src/executor.dart'; +import 'package:flutter_probe_agent/src/protocol.dart'; + +void main() { + group('tap #id respects occlusion (FP-10)', () { + testWidgets( + 'a button covered by an opaque overlay is not tapped via the direct-invoke fast path', + (tester) async { + var tapped = false; + await tester.pumpWidget(MaterialApp( + home: Scaffold( + body: Stack( + children: [ + Semantics( + identifier: 'covered_button', + button: true, + child: InkResponse( + onTap: () => tapped = true, + child: const SizedBox(width: 48, height: 48, child: Icon(Icons.add)), + ), + ), + // Painted after the button, so it's on top in the same Stack — + // a real user's finger would hit this, not the button beneath it. + Positioned.fill( + child: ModalBarrier(color: Colors.black.withValues(alpha: 0.5)), + ), + ], + ), + ), + )); + + final executor = ProbeExecutor((_) {}); + await executor.dispatch(ProbeRequest( + jsonrpc: '2.0', + id: 1, + method: ProbeMethods.tap, + params: { + 'selector': {'kind': 'id', 'text': '#covered_button'}, + }, + )); + await tester.pump(); + + expect(tapped, isFalse, + reason: '_tryDirectTap invokes onTap by walking the Element tree ' + 'structurally, with no relationship to paint order — without ' + 'an occlusion guard it would fire even though the ModalBarrier ' + 'covers the button on screen. A real tap should land on the ' + 'barrier instead, exactly like a real user\'s finger would.'); + }); + + testWidgets('an unobstructed button in the same shape is still tapped directly', + (tester) async { + var tapped = false; + await tester.pumpWidget(MaterialApp( + home: Scaffold( + body: Stack( + children: [ + Semantics( + identifier: 'uncovered_button', + button: true, + child: InkResponse( + onTap: () => tapped = true, + child: const SizedBox(width: 48, height: 48, child: Icon(Icons.add)), + ), + ), + ], + ), + ), + )); + + final executor = ProbeExecutor((_) {}); + await executor.dispatch(ProbeRequest( + jsonrpc: '2.0', + id: 1, + method: ProbeMethods.tap, + params: { + 'selector': {'kind': 'id', 'text': '#uncovered_button'}, + }, + )); + await tester.pump(); + + expect(tapped, isTrue, + reason: 'PT-05 regression check: the occlusion guard must not ' + 'block a tap on a button that genuinely is the topmost thing ' + 'at its own screen position.'); + }); + }); +} diff --git a/vscode/package.json b/vscode/package.json index fcaf5c8..12d726f 100644 --- a/vscode/package.json +++ b/vscode/package.json @@ -2,7 +2,7 @@ "name": "flutterprobe", "displayName": "FlutterProbe", "description": "High-performance E2E testing for Flutter apps — ProbeScript language support, local & cloud device testing (BrowserStack, Sauce Labs, AWS Device Farm, Firebase Test Lab, LambdaTest), visual regression, test recording, and Studio integration", - "version": "0.13.0", + "version": "0.14.0", "publisher": "flutterprobe", "icon": "resources/probe-icon.png", "engines": { "vscode": "^1.85.0" },