From 552bb1b54cdf37e70c59cc17c38164b36c62f041 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 18 Aug 2026 18:54:42 +0200 Subject: [PATCH 01/10] feat(snapshot): move scope into presentation Use one preorder label/identifier/value policy across Swift and TypeScript, keep scoped iOS acquisition conservative, and remove the daemon's second scope pass. Non-vacuity: label-only matching failed identifier/value parity fixtures; Android pass-through failed its boundary test; disconnecting Swift applyScope produced eight scope/depth/projection failures. --- .github/workflows/ios.yml | 4 +- .../RunnerSnapshotScopePolicy.swift | 99 ++++ .../RunnerTests+AXSnapshotFallback.swift | 4 +- .../RunnerTests+FlatSnapshotFiltering.swift | 100 +--- .../RunnerTests+PrivateAXPresentation.swift | 12 +- .../RunnerTests+Snapshot.swift | 17 +- .../RunnerTests+SnapshotCapturePlan.swift | 26 +- .../RunnerTests+SnapshotPresentation.swift | 79 ++- ...unnerTests+SnapshotPresentationTests.swift | 89 +++ contracts/fixtures/snapshot-scope-policy.json | 18 + .../adr/0004-ios-snapshot-backend-strategy.md | 9 + packages/contracts/src/facades/snapshot.ts | 1 + packages/contracts/src/snapshot-scope.test.ts | 13 +- packages/contracts/src/snapshot-scope.ts | 17 +- .../__tests__/snapshot-quality-latch.test.ts | 10 +- src/daemon/handlers/__tests__/find.test.ts | 14 - .../__tests__/snapshot-capture.test.ts | 6 +- src/daemon/handlers/snapshot-capture.ts | 19 +- .../ios/transitions.test.ts | 7 +- .../__tests__/ui-hierarchy-scope.test.ts | 4 +- src/platforms/android/ui-hierarchy-scope.ts | 6 +- src/platforms/android/ui-hierarchy.ts | 9 +- src/snapshot/snapshot-desktop-surface.ts | 10 +- .../provider-scenarios/ios-lifecycle.test.ts | 537 +++++++++--------- 24 files changed, 663 insertions(+), 447 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSnapshotScopePolicy.swift diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index bd555a44c8..c6990b2b4c 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -149,6 +149,8 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationPreservesCurrentWireShape \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationOwnsBackendNeutralEligibility \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationOwnsScopeAndRelativeDepth \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotScopePolicyMatchesGoldenParityTable \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testFlatSnapshotProjectionMatchesElementReverseScrollCapture \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXDepthLimitedRequiresEveryFrontierResolved \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDeepExtensionCountsMissedFrontiers \ @@ -156,7 +158,7 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXPresentationKeepsOffscreenSubtreeExcludedWhenChildFramesAreClamped \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXGeometrylessSemanticsAreNeverActionableOrScrollContexts \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXScopeSelectsSubtreeNotMatchingLabels \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXAcquisitionDoesNotInterpretScope \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXInteractiveFiltersLoginLikeHiddenDrawer \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDecodedPreferredBackendReachesOptionsAndApplicablePlan \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSparsePayloadReasonMatrix \ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSnapshotScopePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSnapshotScopePolicy.swift new file mode 100644 index 0000000000..5f9d104ed2 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSnapshotScopePolicy.swift @@ -0,0 +1,99 @@ +import Foundation + +enum SnapshotScopeSelection: Equatable { + case unscoped + case matched(Int) + case missing +} + +/// Cross-runtime snapshot scope specification. +/// +/// A non-empty scope selects the first node in presentation preorder whose label, identifier, or +/// value contains the trimmed query case-insensitively. Missing matches publish an empty projection. +enum SnapshotScopePolicy { + static func select( + fromPreorder nodes: [Node], + scope: String?, + semanticValues: (Node) -> [String?] + ) -> SnapshotScopeSelection { + guard let query = normalized(scope) else { return .unscoped } + for (index, node) in nodes.enumerated() { + if semanticValues(node).contains(where: { value in + value?.lowercased().contains(query) == true + }) { + return .matched(index) + } + } + return .missing + } + + static func isActive(_ scope: String?) -> Bool { + normalized(scope) != nil + } + + private static func normalized(_ scope: String?) -> String? { + guard let query = scope?.trimmingCharacters(in: .whitespacesAndNewlines), !query.isEmpty else { + return nil + } + return query.lowercased() + } +} + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct SnapshotScopeFixture: Decodable { + struct Node: Decodable { + let depth: Int + let label: String? + let identifier: String? + let value: String? + } + + let name: String + let scope: String + let nodes: [Node] + let expectedSubtreeIndexes: [Int] +} + +extension RunnerTests { + func testSnapshotScopePolicyMatchesGoldenParityTable() throws { + // Non-vacuity: label-only semantic values fail the identifier-only and value-only fixtures. + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("snapshot-scope-policy.json") + let cases = try JSONDecoder().decode( + [SnapshotScopeFixture].self, + from: Data(contentsOf: fixtureURL) + ) + XCTAssertFalse(cases.isEmpty, "parity table must not be empty") + + for fixture in cases { + let selected = SnapshotScopePolicy.select( + fromPreorder: fixture.nodes, + scope: fixture.scope, + semanticValues: { [$0.label, $0.identifier, $0.value] } + ) + let actual: [Int] + switch selected { + case .unscoped: + actual = Array(fixture.nodes.indices) + case .missing: + actual = [] + case .matched(let start): + let rootDepth = fixture.nodes[start].depth + var end = start + 1 + while end < fixture.nodes.count, fixture.nodes[end].depth > rootDepth { + end += 1 + } + actual = Array(start.. Bool { - let haystack = [label, identifier, valueText ?? ""].joined(separator: "\n") - return haystack.localizedCaseInsensitiveContains(scope) - } } struct FlatSnapshotFilterDecision { let include: Bool - let insideMatchedScope: Bool } enum FlatSnapshotVisibilityPolicy { @@ -120,24 +111,11 @@ extension RunnerTests { func flatSnapshotFilterDecision( _ node: FlatSnapshotFilterNode, options: PresentationOptions, - visibilityPolicy: FlatSnapshotVisibilityPolicy, - insideMatchedScope: Bool + visibilityPolicy: FlatSnapshotVisibilityPolicy ) -> FlatSnapshotFilterDecision { - let scope = options.scope?.trimmingCharacters(in: .whitespacesAndNewlines) - let scopeActive = scope?.isEmpty == false - let matchesScope: Bool - if scopeActive, let scope { - matchesScope = node.matchesScope(scope) - } else { - matchesScope = false - } - let nowInsideScope = insideMatchedScope || matchesScope - let include: Bool if node.isRoot { include = true - } else if scopeActive && !nowInsideScope { - include = false } else if !node.visible && (options.interactiveOnly || visibilityPolicy == .viewportProjected) { @@ -146,7 +124,7 @@ extension RunnerTests { include = true } - return FlatSnapshotFilterDecision(include: include, insideMatchedScope: nowInsideScope) + return FlatSnapshotFilterDecision(include: include) } func privateAXInteractiveCandidate(rawElementType: Int) -> Bool { @@ -309,30 +287,18 @@ extension RunnerTests { func testFlatSnapshotFilterDecisionMatrixCoversOptions() { let visibleContent = FlatSnapshotFilterNode( isRoot: false, - label: "Welcome back", - identifier: "", - valueText: nil, visible: true ) let hiddenInteractive = FlatSnapshotFilterNode( isRoot: false, - label: "Hidden menu", - identifier: "", - valueText: nil, visible: false ) let decorative = FlatSnapshotFilterNode( isRoot: false, - label: "", - identifier: "", - valueText: nil, visible: true ) let hiddenRoot = FlatSnapshotFilterNode( isRoot: true, - label: "App", - identifier: "", - valueText: nil, visible: false ) @@ -340,92 +306,42 @@ extension RunnerTests { flatSnapshotFilterDecision( visibleContent, options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), - visibilityPolicy: .interactiveOnly, - insideMatchedScope: false + visibilityPolicy: .interactiveOnly ).include ) XCTAssertFalse( flatSnapshotFilterDecision( hiddenInteractive, options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false), - visibilityPolicy: .interactiveOnly, - insideMatchedScope: false + visibilityPolicy: .interactiveOnly ).include ) XCTAssertFalse( flatSnapshotFilterDecision( hiddenInteractive, options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), - visibilityPolicy: .viewportProjected, - insideMatchedScope: false + visibilityPolicy: .viewportProjected ).include ) XCTAssertTrue( flatSnapshotFilterDecision( hiddenInteractive, options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), - visibilityPolicy: .interactiveOnly, - insideMatchedScope: false + visibilityPolicy: .interactiveOnly ).include ) XCTAssertTrue( flatSnapshotFilterDecision( hiddenRoot, options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), - visibilityPolicy: .viewportProjected, - insideMatchedScope: false + visibilityPolicy: .viewportProjected ).include ) XCTAssertTrue( flatSnapshotFilterDecision( decorative, options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), - visibilityPolicy: .interactiveOnly, - insideMatchedScope: false - ).include - ) - } - - func testFlatSnapshotFilterDecisionCarriesSubtreeScopeState() { - let scopeRoot = FlatSnapshotFilterNode( - isRoot: false, - label: "", - identifier: "homeScreen", - valueText: nil, - visible: true - ) - let unmatchedDescendant = FlatSnapshotFilterNode( - isRoot: false, - label: "Post body without the scope text", - identifier: "", - valueText: nil, - visible: true - ) - let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: "homeScreen", raw: false) - - let rootDecision = flatSnapshotFilterDecision( - scopeRoot, - options: options, - visibilityPolicy: .interactiveOnly, - insideMatchedScope: false - ) - XCTAssertTrue(rootDecision.include) - XCTAssertTrue(rootDecision.insideMatchedScope) - - XCTAssertTrue( - flatSnapshotFilterDecision( - unmatchedDescendant, - options: options, - visibilityPolicy: .interactiveOnly, - insideMatchedScope: rootDecision.insideMatchedScope - ).include - ) - XCTAssertFalse( - flatSnapshotFilterDecision( - unmatchedDescendant, - options: options, - visibilityPolicy: .interactiveOnly, - insideMatchedScope: false + visibilityPolicy: .interactiveOnly ).include ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift index 10521f0ed0..8182aa13da 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift @@ -9,14 +9,14 @@ extension RunnerTests { var nodes: [RawAXNode] = [] var hints: [Int: (above: Bool, below: Bool)] = [:] appendPrivateAXNode(rawRoot, to: &nodes, hints: &hints, options: options, viewport: viewport, - depth: 0, parentIndex: nil, insideMatchedScope: false, scrollContext: nil, + depth: 0, parentIndex: nil, scrollContext: nil, projectionCursor: .root) return applyHiddenContentHints(hints, to: nodes) } private func appendPrivateAXNode(_ raw: [String: Any], to nodes: inout [RawAXNode], hints: inout [Int: (above: Bool, below: Bool)], options: PresentationOptions, viewport: CGRect, - depth: Int, parentIndex: Int?, insideMatchedScope: Bool, + depth: Int, parentIndex: Int?, scrollContext: (index: Int, rect: CGRect)?, projectionCursor: FlatSnapshotProjectionCursor) { if let limit = options.depth, depth > limit { return } @@ -51,10 +51,8 @@ extension RunnerTests { let projection = projectionTransition.decision let presentationVisible = projection.presentationVisible && !negligibleDecoration let decision = flatSnapshotFilterDecision( - FlatSnapshotFilterNode(isRoot: parentIndex == nil, label: label, identifier: identifier, - valueText: value.isEmpty ? nil : value, visible: presentationVisible), - options: options, visibilityPolicy: .viewportProjected, - insideMatchedScope: insideMatchedScope) + FlatSnapshotFilterNode(isRoot: parentIndex == nil, visible: presentationVisible), + options: options, visibilityPolicy: .viewportProjected) let include = decision.include if let hiddenFrame = projectionTransition.hiddenContentFrame, let scrollContext { @@ -88,7 +86,7 @@ extension RunnerTests { for child in children { appendPrivateAXNode(child, to: &nodes, hints: &hints, options: options, viewport: viewport, depth: depth + 1, parentIndex: currentIndex, - insideMatchedScope: decision.insideMatchedScope, scrollContext: nextScrollContext, + scrollContext: nextScrollContext, projectionCursor: projection.descendants) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index a2a36c8356..0eea6ceb15 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -942,26 +942,23 @@ extension RunnerTests { captureDeadline: Date = .distantFuture, treeCaptureSliceBudgetOverride: TimeInterval? = nil ) throws -> SnapshotTraversalContext? { - let (viewport, queryRoot) = try runMainThreadWork( + let viewport = try runMainThreadWork( command: nil, timeout: min(1.0, max(0.1, captureDeadline.timeIntervalSinceNow)), timeoutError: snapshotMainThreadTimeoutError("preparing tree snapshot") ) { - ( - self.safeSnapshotViewport(app: app), - options.scope.flatMap { self.findScopeElement(app: app, scope: $0) } ?? app - ) + self.safeSnapshotViewport(app: app) } let treeSliceBudget = treeCaptureSliceBudgetOverride ?? treeCaptureSliceBudget let slice = min(treeSliceBudget, max(0.5, captureDeadline.timeIntervalSinceNow)) - guard let rootSnapshot = try captureSnapshotRootBounded(queryRoot, sliceSeconds: slice) else { + guard let rootSnapshot = try captureSnapshotRootBounded(app, sliceSeconds: slice) else { return nil } let (flatSnapshots, snapshotRanges) = flattenedSnapshots(rootSnapshot) return SnapshotTraversalContext( - queryRoot: queryRoot, + queryRoot: app, rootSnapshot: rootSnapshot, viewport: viewport, flatSnapshots: flatSnapshots, @@ -1599,16 +1596,12 @@ extension RunnerTests { let hittable = visible && enabled && element.isHittable let filterNode = FlatSnapshotFilterNode( isRoot: false, - label: label, - identifier: identifier, - valueText: valueText, visible: visible ) if !flatSnapshotFilterDecision( filterNode, options: options, - visibilityPolicy: .interactiveOnly, - insideMatchedScope: false + visibilityPolicy: .interactiveOnly ).include { return } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index a28db1dfe5..3dc3eab3c5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -97,6 +97,9 @@ struct SnapshotBackendCapture { let effectiveDepth: Int? /// Set by the private AX backend when the capture asked for custom actions. var customActions: SnapshotCustomActionCoverage? = nil + /// Broad presentation used only by the quality classifier when a scope narrows publication. + /// A legitimate missing scope is an empty healthy projection, not backend failure evidence. + var qualityPayload: DataPayload? = nil } extension RunnerTests { @@ -365,7 +368,7 @@ extension RunnerTests { continue } - if let sparseReason = Self.sparsePayloadReason(capture.payload) { + if let sparseReason = Self.sparsePayloadReason(capture.qualityPayload ?? capture.payload) { if firstFailure == nil { firstFailure = sparseReason } if Self.payloadNodeCount(capture.payload) > Self.payloadNodeCount(best?.capture.payload) { best = (kind, capture) @@ -456,13 +459,14 @@ extension RunnerTests { deadline: Date, treeCaptureSliceBudgetOverride: TimeInterval? ) throws -> SnapshotBackendCapture? { + let acquisitionOptions = SnapshotPresentation.conservativeAcquisitionOptions(for: options) let acquisition: SnapshotAcquisition? switch kind { case .recursiveTree: guard let context = try makeSnapshotTraversalContext( app: app, - options: options, + options: acquisitionOptions, captureDeadline: deadline, treeCaptureSliceBudgetOverride: treeCaptureSliceBudgetOverride ) @@ -474,9 +478,9 @@ extension RunnerTests { timeout: min(treeCaptureSliceBudget, max(0.5, deadline.timeIntervalSinceNow)), timeoutError: snapshotMainThreadTimeoutError("processing tree snapshot") ) { - options.raw - ? try self.rawTreeSnapshotAcquisition(context: context, options: options) - : self.recursiveTreeSnapshotAcquisition(context: context, options: options) + acquisitionOptions.raw + ? try self.rawTreeSnapshotAcquisition(context: context, options: acquisitionOptions) + : self.recursiveTreeSnapshotAcquisition(context: context, options: acquisitionOptions) } case .querySweep: acquisition = try runMainThreadWork( @@ -484,10 +488,18 @@ extension RunnerTests { timeout: min(Self.flatInteractiveFallbackBudget, max(0.1, deadline.timeIntervalSinceNow)), timeoutError: snapshotMainThreadTimeoutError("running query-sweep snapshot") ) { - self.querySweepSnapshotAcquisition(app: app, options: options, planDeadline: deadline) + self.querySweepSnapshotAcquisition( + app: app, + options: acquisitionOptions, + planDeadline: deadline + ) } case .privateAX: - acquisition = privateAXSnapshotAcquisition(app: app, options: options, deadline: deadline) + acquisition = privateAXSnapshotAcquisition( + app: app, + options: acquisitionOptions, + deadline: deadline + ) } guard let acquisition else { return nil } return SnapshotPresentation.present(acquisition, options: options) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentation.swift index 4b8f9907c6..359fce72a6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentation.swift @@ -115,13 +115,36 @@ enum SnapshotPresentation { _ acquisition: SnapshotAcquisition, options: PresentationOptions ) -> SnapshotBackendCapture { - SnapshotBackendCapture( - payload: DataPayload( + let scopedRawNodes = applyScope(to: acquisition.nodes, options: options) + let nodes = presentedNodes(from: scopedRawNodes, options: options) + let qualityPayload: DataPayload? = SnapshotScopePolicy.isActive(options.scope) + ? DataPayload( nodes: presentedNodes(from: acquisition.nodes, options: options), truncated: acquisition.truncated + ) + : nil + return SnapshotBackendCapture( + payload: DataPayload( + nodes: nodes, + truncated: acquisition.truncated ), effectiveDepth: acquisition.effectiveDepth, - customActions: acquisition.customActions + customActions: acquisition.customActions, + qualityPayload: qualityPayload + ) + } + + /// Scope and depth cannot safely narrow acquisition until a backend proves its hint complete. + /// Acquire the broad tree, then apply both relative to the selected presentation subtree. + static func conservativeAcquisitionOptions(for options: PresentationOptions) -> PresentationOptions { + guard SnapshotScopePolicy.isActive(options.scope) else { return options } + return PresentationOptions( + interactiveOnly: options.interactiveOnly, + depth: nil, + scope: nil, + raw: options.raw, + preferredBackend: options.preferredBackend, + customActions: options.customActions ) } @@ -167,6 +190,56 @@ enum SnapshotPresentation { return nodes } + private static func applyScope( + to rawNodes: [RawAXNode], + options: PresentationOptions + ) -> [RawAXNode] { + switch SnapshotScopePolicy.select( + fromPreorder: rawNodes, + scope: options.scope, + semanticValues: { [$0.label, $0.identifier, $0.value] } + ) { + case .unscoped: + return rawNodes + case .missing: + return [] + case .matched(let startIndex): + let startDepth = rawNodes[startIndex].depth + var endIndex = startIndex + 1 + while endIndex < rawNodes.count, rawNodes[endIndex].depth > startDepth { + endIndex += 1 + } + let maxDepth = options.depth ?? Int.max + return reindex( + Array(rawNodes[startIndex.. [RawAXNode] { + let indexMap = Dictionary(uniqueKeysWithValues: rawNodes.enumerated().map { ($0.element.index, $0.offset) }) + return rawNodes.enumerated().map { offset, raw in + RawAXNode( + index: offset, + type: raw.type, + label: raw.label, + identifier: raw.identifier, + value: raw.value, + rect: raw.rect, + enabled: raw.enabled, + focused: raw.focused, + selected: raw.selected, + hittable: raw.hittable, + depth: max(0, raw.depth - depthOffset), + parentIndex: raw.parentIndex.flatMap { indexMap[$0] }, + hiddenContentAbove: raw.hiddenContentAbove, + hiddenContentBelow: raw.hiddenContentBelow, + actions: raw.actions + ) + } + } + private static func isEligibleForRegularPresentation(_ node: RawAXNode) -> Bool { // The top-level carrier owns viewport geometry and must survive even for query-sweep's // deliberately unlabeled synthetic Application node. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationTests.swift index 6cf3dee4c1..15136d79f4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationTests.swift @@ -136,5 +136,94 @@ extension RunnerTests { XCTAssertEqual(raw.last?.depth, 2) XCTAssertEqual(raw.last?.parentIndex, 9) } + + func testSnapshotPresentationOwnsScopeAndRelativeDepth() throws { + // Non-vacuity: disconnecting applyScope produces eight scope, depth, and raw-projection failures. + func node( + _ index: Int, + type: String, + label: String? = nil, + identifier: String? = nil, + depth: Int, + parentIndex: Int? + ) -> RawAXNode { + RawAXNode( + index: index, + type: type, + label: label, + identifier: identifier, + value: nil, + rect: SnapshotRect(x: 0, y: Double(index * 20), width: 100, height: 20), + enabled: true, + focused: nil, + selected: nil, + hittable: type == "Button", + depth: depth, + parentIndex: parentIndex, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + } + + let acquisition = SnapshotAcquisition( + nodes: [ + node(0, type: "Application", label: "App", depth: 0, parentIndex: nil), + node(1, type: "Button", label: "Earlier sibling", depth: 1, parentIndex: 0), + node(2, type: "Other", identifier: "scope-root", depth: 1, parentIndex: 0), + node(3, type: "StaticText", label: "Child", depth: 2, parentIndex: 2), + node(4, type: "Image", depth: 2, parentIndex: 2), + node(5, type: "Button", label: "Grandchild", depth: 3, parentIndex: 3), + node(6, type: "Button", label: "Outside sibling", depth: 1, parentIndex: 0), + ], + truncated: false, + effectiveDepth: nil + ) + let options = PresentationOptions( + interactiveOnly: true, + depth: 1, + scope: " SCOPE-ROOT ", + raw: false + ) + let capture = SnapshotPresentation.present(acquisition, options: options) + let nodes = try XCTUnwrap(capture.payload.nodes) + + XCTAssertEqual(nodes.map(\.label), [nil, "Child"]) + XCTAssertEqual(nodes.map(\.identifier), ["scope-root", nil]) + XCTAssertEqual(nodes.map(\.index), [0, 1]) + XCTAssertEqual(nodes.map(\.depth), [0, 1]) + XCTAssertEqual(nodes.map(\.parentIndex), [nil, 0]) + XCTAssertEqual(capture.qualityPayload?.nodes?.count, 6) + + let raw = try XCTUnwrap( + SnapshotPresentation.present( + acquisition, + options: PresentationOptions( + interactiveOnly: true, + depth: 1, + scope: "scope-root", + raw: true + ) + ).payload.nodes + ) + XCTAssertEqual(raw.map(\.type), ["Other", "StaticText", "Image"]) + XCTAssertEqual(raw.map(\.depth), [0, 1, 1]) + + let hint = SnapshotPresentation.conservativeAcquisitionOptions(for: options) + XCTAssertNil(hint.scope) + XCTAssertNil(hint.depth) + XCTAssertTrue(hint.interactiveOnly) + + let missing = SnapshotPresentation.present( + acquisition, + options: PresentationOptions( + interactiveOnly: true, + depth: 1, + scope: "missing", + raw: false + ) + ) + XCTAssertEqual(missing.payload.nodes?.count, 0) + XCTAssertNil(RunnerTests.sparsePayloadReason(try XCTUnwrap(missing.qualityPayload))) + } } #endif diff --git a/contracts/fixtures/snapshot-scope-policy.json b/contracts/fixtures/snapshot-scope-policy.json index 0722813349..c1df0bef09 100644 --- a/contracts/fixtures/snapshot-scope-policy.json +++ b/contracts/fixtures/snapshot-scope-policy.json @@ -1,4 +1,22 @@ [ + { + "name": "blank scope keeps the full projection", + "scope": " ", + "nodes": [ + { + "depth": 0, + "type": "FrameLayout", + "label": "Root" + }, + { + "depth": 1, + "type": "Button", + "label": "Continue" + } + ], + "expectedRootIndex": null, + "expectedSubtreeIndexes": [0, 1] + }, { "name": "first document-order match wins over a shallower later sibling (breadth-first would pick index 3)", "scope": "Settings", diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index c5a46c22aa..d1a2fabff4 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -114,6 +114,15 @@ not daemon publication membership; backend-blind daemon compaction retains owner noise suppressions. When eligibility removes a structural wrapper, presentation reparents its surviving descendants to the nearest surviving ancestor and normalizes their indexes and depths. +The second semantic layer makes scope a presentation specification rather than an acquisition or +daemon-compaction policy. A trimmed non-empty scope selects the first presentation-preorder node +whose label, identifier, or value contains it case-insensitively; the selected subtree is re-rooted, +depth is applied relative to that root, and no match publishes an empty healthy projection. Swift +and TypeScript implementations are pinned by `contracts/fixtures/snapshot-scope-policy.json`. +Scoped iOS acquisition stays broad (including when depth is requested) until an adapter can prove a +narrowing hint complete. The daemon never reapplies scope after the wire; Android selects its root +inside its TypeScript presentation and desktop surface runtimes retain their platform projection. + When adding new iOS snapshot behavior, maintainers should first decide which strategy owns it. If a change tries to make regular snapshots fast by dropping visible controls behind a node budget, or tries to make raw snapshots safe by silently truncating, it is probably crossing strategy diff --git a/packages/contracts/src/facades/snapshot.ts b/packages/contracts/src/facades/snapshot.ts index 98f91e04a8..56cb1dc0fe 100644 --- a/packages/contracts/src/facades/snapshot.ts +++ b/packages/contracts/src/facades/snapshot.ts @@ -2,6 +2,7 @@ export { isScrollableNodeLike, isScrollableType } from '../snapshot-scroll.ts'; export { findSnapshotScopeRange, matchesSnapshotScope, + normalizeSnapshotScope, reindexSnapshotNodes, type SnapshotScopeCandidate, } from '../snapshot-scope.ts'; diff --git a/packages/contracts/src/snapshot-scope.test.ts b/packages/contracts/src/snapshot-scope.test.ts index f41bcebe30..5cc8b53ff9 100644 --- a/packages/contracts/src/snapshot-scope.test.ts +++ b/packages/contracts/src/snapshot-scope.test.ts @@ -2,12 +2,11 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; -import { matchesSnapshotScope } from './facades/snapshot.ts'; +import { matchesSnapshotScope, normalizeSnapshotScope } from './facades/snapshot.ts'; // Golden scope-policy table (#1797 / #1832 C2): the SAME JSON is asserted against every runtime // that resolves `--scope` — this predicate, the Android platform projection -// (src/platforms/android/__tests__/ui-hierarchy-scope.test.ts), the daemon's post-wire pass -// (src/snapshot/snapshot-desktop-surface.test.ts) and the Swift runner twin once #1797 lands it. +// (src/platforms/android/__tests__/ui-hierarchy-scope.test.ts), and the Swift runner twin. // This file proves the PREDICATE + first-document-order-match rule; subtree slicing is proved by // the runtime legs. @@ -35,6 +34,14 @@ test('the shared scope predicate picks the golden table root as the first docume assert.ok(cases.length > 0, 'scope policy table must not be empty'); assert.equal(new Set(cases.map((fixture) => fixture.name)).size, cases.length); for (const fixture of cases) { + if (!normalizeSnapshotScope(fixture.scope)) { + assert.equal(fixture.expectedRootIndex, null, fixture.name); + assert.deepEqual( + fixture.expectedSubtreeIndexes, + fixture.nodes.map((_, index) => index), + ); + continue; + } const rootIndex = fixture.nodes.findIndex((node) => matchesSnapshotScope(node, fixture.scope)); assert.equal(rootIndex === -1 ? null : rootIndex, fixture.expectedRootIndex, fixture.name); assert.deepEqual( diff --git a/packages/contracts/src/snapshot-scope.ts b/packages/contracts/src/snapshot-scope.ts index c575540dae..93f8f376f6 100644 --- a/packages/contracts/src/snapshot-scope.ts +++ b/packages/contracts/src/snapshot-scope.ts @@ -10,11 +10,9 @@ * - the scoped snapshot is that root's presented subtree, re-rooted at depth 0 and reindexed; * - no match yields an EMPTY snapshot — never the full tree. * - * The golden table `contracts/fixtures/snapshot-scope-policy.json` pins the rule; every runtime - * that resolves scope (Android projection, the daemon's post-wire pass, the Swift runner twin) is - * asserted against the same table so drift turns CI red on whichever side changed. The - * contributes-content clause is vacuous for a runtime that only ever sees presented nodes (the - * post-wire pass): there, a match always contributes itself. + * The golden table `contracts/fixtures/snapshot-scope-policy.json` pins the rule; the Android + * projection and Swift runner are asserted against the same table so drift turns CI red on + * whichever side changed. */ export type SnapshotScopeCandidate = { label?: string | null; @@ -22,8 +20,14 @@ export type SnapshotScopeCandidate = { identifier?: string | null; }; +export function normalizeSnapshotScope(scope: string | undefined): string | null { + const query = scope?.trim().toLowerCase(); + return query ? query : null; +} + export function matchesSnapshotScope(node: SnapshotScopeCandidate, scope: string): boolean { - const query = scope.toLowerCase(); + const query = normalizeSnapshotScope(scope); + if (!query) return false; return [node.label, node.value, node.identifier].some((field) => (field ?? '').toLowerCase().includes(query), ); @@ -37,6 +41,7 @@ export function findSnapshotScopeRange( nodes: readonly (SnapshotScopeCandidate & { depth?: number })[], scope: string, ): { start: number; end: number } | null { + if (!normalizeSnapshotScope(scope)) return null; const start = nodes.findIndex((node) => matchesSnapshotScope(node, scope)); if (start === -1) return null; const rootDepth = nodes[start]?.depth ?? 0; diff --git a/src/daemon/__tests__/snapshot-quality-latch.test.ts b/src/daemon/__tests__/snapshot-quality-latch.test.ts index 0eb0f8e105..7eed7f06a0 100644 --- a/src/daemon/__tests__/snapshot-quality-latch.test.ts +++ b/src/daemon/__tests__/snapshot-quality-latch.test.ts @@ -275,9 +275,13 @@ test('an empty ref-scoped diff latches on the captured verdict, not the retained }, ], }; - // The deferred capture holds no node labeled 'Continue', so the '@e1' scope - // resolves to zero nodes and the retention path runs. - seedCapture(deferredVerdict(), 'Something else'); + // The runner owns scope publication and returns the healthy empty projection for a miss. + dispatchCommandMock.mockResolvedValue({ + backend: 'xctest', + truncated: false, + quality: deferredVerdict(), + nodes: [], + }); const diff = await dispatchSnapshotDiffViaRuntime({ req: { diff --git a/src/daemon/handlers/__tests__/find.test.ts b/src/daemon/handlers/__tests__/find.test.ts index 6076b287b8..146c0cc6ab 100644 --- a/src/daemon/handlers/__tests__/find.test.ts +++ b/src/daemon/handlers/__tests__/find.test.ts @@ -317,17 +317,10 @@ test('handleFindCommands click uses query-scoped full retry when sparse verdict nodes: [ { index: 0, - type: 'Application', - hittable: false, - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, type: 'Button', label: 'Search', hittable: true, rect: { x: 80, y: 792, width: 78, height: 48 }, - parentIndex: 0, }, ], }, @@ -423,17 +416,10 @@ test('handleFindCommands click scopes full retry for legacy sparse shape when un nodes: [ { index: 0, - type: 'Application', - hittable: false, - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, type: 'Button', label: 'Search', hittable: true, rect: { x: 80, y: 792, width: 78, height: 48 }, - parentIndex: 0, }, ], }, diff --git a/src/daemon/handlers/__tests__/snapshot-capture.test.ts b/src/daemon/handlers/__tests__/snapshot-capture.test.ts index 840239f59f..57c7aa5cc2 100644 --- a/src/daemon/handlers/__tests__/snapshot-capture.test.ts +++ b/src/daemon/handlers/__tests__/snapshot-capture.test.ts @@ -10,7 +10,7 @@ import { const captureSnapshotWithInteractor = vi.hoisted(() => vi.fn()); vi.mock('../snapshot-interactor-capture.ts', () => ({ captureSnapshotWithInteractor })); -test('iOS interactive capture does not send local presentation scope to XCTest', async () => { +test('iOS interactive capture sends scope to runner presentation', async () => { captureSnapshotWithInteractor.mockClear(); captureSnapshotWithInteractor.mockResolvedValueOnce({ nodes: [], backend: 'xctest' }); @@ -25,12 +25,12 @@ test('iOS interactive capture does not send local presentation scope to XCTest', expect(captureSnapshotWithInteractor).toHaveBeenCalledOnce(); expect(captureSnapshotWithInteractor).toHaveBeenCalledWith( expect.objectContaining({ - options: expect.objectContaining({ interactiveOnly: true, scope: undefined }), + options: expect.objectContaining({ interactiveOnly: true, scope: 'action file' }), }), ); }); -test('snapshot capture preserves backend scope outside iOS interactive presentation', async () => { +test('snapshot capture preserves scope for every other platform projection', async () => { captureSnapshotWithInteractor.mockClear(); for (const [device, flags] of [ [ANDROID_EMULATOR, { snapshotInteractiveOnly: true, snapshotScope: 'action file' }], diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index 21561d2a95..fa1d324d93 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -4,7 +4,7 @@ import { snapshotCaptureAnnotationsFrom, type SnapshotCaptureAnnotations, } from '@agent-device/contracts/capture'; -import { isIosFamily, publicPlatformString } from '@agent-device/kernel/device'; +import { publicPlatformString } from '@agent-device/kernel/device'; import { findNodeByRef, normalizeRef, @@ -85,7 +85,7 @@ export async function captureSnapshotData(params: CaptureSnapshotParams): Promis const { device, session, logPath } = params; const context = contextFromFlags( logPath, - snapshotCaptureFlagsForBackend(device, resolveSnapshotStateFlags(params)), + resolveSnapshotStateFlags(params), session?.appBundleId, session?.trace?.outPath, ); @@ -157,21 +157,6 @@ function resolveSnapshotStateFlags( }; } -function snapshotCaptureFlagsForBackend( - device: SessionState['device'], - flags: CommandFlags | undefined, -): CommandFlags | undefined { - if ( - !isIosFamily(device) || - flags?.snapshotInteractiveOnly !== true || - flags.snapshotRaw === true || - flags.snapshotScope === undefined - ) { - return flags; - } - return { ...flags, snapshotScope: undefined }; -} - export function resolveSnapshotScope( snapshotScope: string | undefined, session: SessionState | undefined, diff --git a/src/daemon/snapshot-presentation/ios/transitions.test.ts b/src/daemon/snapshot-presentation/ios/transitions.test.ts index 6824ea402b..4c0eac475a 100644 --- a/src/daemon/snapshot-presentation/ios/transitions.test.ts +++ b/src/daemon/snapshot-presentation/ios/transitions.test.ts @@ -5,20 +5,21 @@ import { buildSnapshotState } from '../../snapshot-state.ts'; import { presentIosInteractiveSnapshot } from './index.ts'; import { navigationTitleWithAppProvidedDetailsAffordanceNodes } from './transitions.fixtures.ts'; -test('iOS presentation applies transition semantics before scoping the snapshot', () => { +test('iOS daemon presentation applies transitions without reapplying runner-owned scope', () => { const snapshot = buildSnapshotState( { nodes: navigationTitleWithAppProvidedDetailsAffordanceNodes, backend: 'xctest' }, { snapshotInteractiveOnly: true, snapshotScope: 'DisplayNameTextField' }, ); - expect(snapshot.nodes).toHaveLength(1); - expect(snapshot.nodes[0]).toEqual( + expect(snapshot.nodes).toHaveLength(8); + expect(snapshot.nodes).toContainEqual( expect.objectContaining({ type: 'Button', label: 'Team Standup', identifier: 'DisplayNameTextField', }), ); + expect(snapshot.nodes).toContainEqual(expect.objectContaining({ label: 'Video Call' })); }); test('iOS presentation promotes an app-provided navigation title affordance without stealing a content action', () => { diff --git a/src/platforms/android/__tests__/ui-hierarchy-scope.test.ts b/src/platforms/android/__tests__/ui-hierarchy-scope.test.ts index c3d90c92ac..1df72da821 100644 --- a/src/platforms/android/__tests__/ui-hierarchy-scope.test.ts +++ b/src/platforms/android/__tests__/ui-hierarchy-scope.test.ts @@ -6,8 +6,8 @@ import { buildSnapshotState } from '../../../daemon/snapshot-state.ts'; import { parseUiHierarchy } from './ui-hierarchy-fixtures.ts'; // Android's scope leg of the golden table (#1832 C2). Android resolves `--scope` exactly once, -// over the PRESENTED nodes of the requested projection, inside its projection; the daemon's -// post-wire pass skips the android backend. Pinned here: the projection agrees with +// over the PRESENTED nodes of the requested projection, inside its projection; the daemon never +// reapplies scope after the wire. Pinned here: the projection agrees with // contracts/fixtures/snapshot-scope-policy.json, scope resolves after membership, ancestor // context above the scope root still shapes membership inside it, --depth is scope-relative, and // a scoped Android snapshot survives buildSnapshotState untouched. diff --git a/src/platforms/android/ui-hierarchy-scope.ts b/src/platforms/android/ui-hierarchy-scope.ts index 641fd2c7fe..788746f56b 100644 --- a/src/platforms/android/ui-hierarchy-scope.ts +++ b/src/platforms/android/ui-hierarchy-scope.ts @@ -33,9 +33,9 @@ export type AndroidPresentedNodes = { * can differ from the compacted depth printed beside each node — pre-existing, tracked on #1832.) * `reindexSnapshotNodes` drops parent links that pointed outside the slice. * - * This is the ONLY scope pass an Android snapshot goes through — the daemon's post-wire - * `scopeSnapshotNodes` skips the android backend — and it runs after the walk, so ancestor context - * above the scope root (hittable / collection / chrome) still shapes membership inside it. + * This is the ONLY scope pass an Android snapshot goes through — the daemon never reapplies scope + * after the wire — and it runs after the walk, so ancestor context above the scope root (hittable / + * collection / chrome) still shapes membership inside it. * `sourceNodes` stay parallel to `nodes` for hint bridging. */ export function scopePresentedAndroidSnapshot< diff --git a/src/platforms/android/ui-hierarchy.ts b/src/platforms/android/ui-hierarchy.ts index d300a49809..020b106f50 100644 --- a/src/platforms/android/ui-hierarchy.ts +++ b/src/platforms/android/ui-hierarchy.ts @@ -1,7 +1,7 @@ import type { RawSnapshotNode, Rect, SnapshotOptions } from '@agent-device/kernel/snapshot'; import { parseBounds } from '@agent-device/kernel/bounds'; import { decodeXmlCharacterReferences } from '@agent-device/xml'; -import { isScrollableType } from '@agent-device/contracts/snapshot'; +import { isScrollableType, normalizeSnapshotScope } from '@agent-device/contracts/snapshot'; import { isAgentTarget, isGenericAndroidId, @@ -126,13 +126,14 @@ export function buildUiHierarchySnapshot( options: SnapshotOptions, ): AndroidBuiltSnapshot { const requestedDepth = options.depth ?? Number.POSITIVE_INFINITY; + const scope = normalizeSnapshotScope(options.scope); const state: AndroidSnapshotBuildState = { nodes: [], sourceNodes: [], ...(maxNodes !== undefined ? { maxNodes } : {}), // Under --scope, depth is relative to the scope root, which is only known once the tree is // presented: walk unbounded and cut after scoping. - maxDepth: options.scope ? Number.POSITIVE_INFINITY : requestedDepth, + maxDepth: scope ? Number.POSITIVE_INFINITY : requestedDepth, options, analysis: analyzeAndroidTree(tree), interactiveDescendantMemo: new Map(), @@ -147,8 +148,8 @@ export function buildUiHierarchySnapshot( if (state.truncated) break; } - const { nodes, sourceNodes } = options.scope - ? scopePresentedAndroidSnapshot(state, tree.children, options.scope, requestedDepth) + const { nodes, sourceNodes } = scope + ? scopePresentedAndroidSnapshot(state, tree.children, scope, requestedDepth) : state; const snapshot = { nodes, sourceNodes, analysis: state.analysis }; return state.truncated ? { ...snapshot, truncated: true } : snapshot; diff --git a/src/snapshot/snapshot-desktop-surface.ts b/src/snapshot/snapshot-desktop-surface.ts index 875a749101..521d80b5f2 100644 --- a/src/snapshot/snapshot-desktop-surface.ts +++ b/src/snapshot/snapshot-desktop-surface.ts @@ -2,7 +2,11 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import type { SnapshotOptions, SnapshotResult } from '@agent-device/contracts/interaction'; import type { CaptureSnapshotInput, SnapshotRuntimeHost } from '@agent-device/contracts/platform'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { findSnapshotScopeRange, reindexSnapshotNodes } from '@agent-device/contracts/snapshot'; +import { + findSnapshotScopeRange, + normalizeSnapshotScope, + reindexSnapshotNodes, +} from '@agent-device/contracts/snapshot'; type SnapshotSurfaceOptions = NonNullable; @@ -76,7 +80,9 @@ function shapeDesktopSurfaceSnapshot( /** The shared scope specification applied post-wire (`@agent-device/contracts/snapshot`). */ export function scopeSnapshotNodes(nodes: RawSnapshotNode[], scope: string): RawSnapshotNode[] { - const range = findSnapshotScopeRange(nodes, scope); + const normalizedScope = normalizeSnapshotScope(scope); + if (!normalizedScope) return reindexSnapshotNodes(nodes); + const range = findSnapshotScopeRange(nodes, normalizedScope); if (!range) return []; const slice = nodes.slice(range.start, range.end); return reindexSnapshotNodes(slice, slice[0]?.depth ?? 0); diff --git a/test/integration/provider-scenarios/ios-lifecycle.test.ts b/test/integration/provider-scenarios/ios-lifecycle.test.ts index 2becd3f0c5..7e1f8030b3 100644 --- a/test/integration/provider-scenarios/ios-lifecycle.test.ts +++ b/test/integration/provider-scenarios/ios-lifecycle.test.ts @@ -15,289 +15,300 @@ import { import { withProviderScenarioResource } from './harness.ts'; import { PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS } from './test-timeouts.ts'; -test('Provider-backed integration iOS Settings flow uses scripted simctl and runner providers', async () => { - await withProviderScenarioResource( - createIosSettingsWorld, - async ({ appPath, appleTool, daemon, inventoryRequests, runnerTranscript }) => { - const scopedDevices = await daemon.client().devices.list({ - platform: 'ios', - iosSimulatorDeviceSet: '/tmp/provider-scenario-simulators', - }); - assert.equal(scopedDevices.length, 1); - assert.equal(scopedDevices[0]?.id, PROVIDER_SCENARIO_IOS_SIMULATOR.id); +function testSlowProviderScenario(name: string, run: () => Promise): void { + test(name, run, PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS); +} + +testSlowProviderScenario( + 'Provider-backed integration iOS Settings flow uses scripted simctl and runner providers', + async () => { + await withProviderScenarioResource( + createIosSettingsWorld, + async ({ appPath, appleTool, daemon, inventoryRequests, runnerTranscript }) => { + const scopedDevices = await daemon.client().devices.list({ + platform: 'ios', + iosSimulatorDeviceSet: '/tmp/provider-scenario-simulators', + }); + assert.equal(scopedDevices.length, 1); + assert.equal(scopedDevices[0]?.id, PROVIDER_SCENARIO_IOS_SIMULATOR.id); - await runProviderScenario(daemon, [ - { - name: 'open settings app', - command: 'open', - positionals: ['com.apple.Preferences'], - flags: { platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, - expectData: { - appBundleId: 'com.apple.Preferences', - device_udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + await runProviderScenario(daemon, [ + { + name: 'open settings app', + command: 'open', + positionals: ['com.apple.Preferences'], + flags: { platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, + expectData: { + appBundleId: 'com.apple.Preferences', + device_udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + }, }, - }, - { - name: 'read app session state', - command: 'appstate', - flags: { platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, - expectData: { - platform: 'ios', - appBundleId: 'com.apple.Preferences', - source: 'session', - device_udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, - ios_simulator_device_set: null, + { + name: 'read app session state', + command: 'appstate', + flags: { platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, + expectData: { + platform: 'ios', + appBundleId: 'com.apple.Preferences', + source: 'session', + device_udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + ios_simulator_device_set: null, + }, }, - }, - { - name: 'prepare iOS runner', - command: 'prepare', - positionals: ['ios-runner'], - flags: { platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, - expectData: { - action: 'ios-runner', - platform: 'ios', - deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, - runner: { uptimeMs: 42 }, + { + name: 'prepare iOS runner', + command: 'prepare', + positionals: ['ios-runner'], + flags: { platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, + expectData: { + action: 'ios-runner', + platform: 'ios', + deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + runner: { uptimeMs: 42 }, + }, }, - }, - { - name: 'capture settings snapshot', - command: 'snapshot', - flags: { snapshotInteractiveOnly: true }, - assert: (firstSnapshot) => { - assert.equal(firstSnapshot.json?.result?.data?.nodes?.[0]?.label, 'General'); - assert.equal(firstSnapshot.json?.result?.data?.nodes?.[0]?.ref, 'e1'); + { + name: 'capture settings snapshot', + command: 'snapshot', + flags: { snapshotInteractiveOnly: true }, + assert: (firstSnapshot) => { + assert.equal(firstSnapshot.json?.result?.data?.nodes?.[0]?.label, 'General'); + assert.equal(firstSnapshot.json?.result?.data?.nodes?.[0]?.ref, 'e1'); + }, }, - }, - { - name: 'reopen existing session app', - command: 'open', - positionals: ['com.apple.Preferences'], - expectData: { appBundleId: 'com.apple.Preferences' }, - }, - { - name: 'capture iOS launch console while reopening app', - command: 'open', - positionals: ['com.apple.Preferences'], - flags: { - launchConsole: path.join(path.dirname(appPath), 'launch-console.log'), + { + name: 'reopen existing session app', + command: 'open', + positionals: ['com.apple.Preferences'], + expectData: { appBundleId: 'com.apple.Preferences' }, }, - expectData: { appBundleId: 'com.apple.Preferences' }, - }, - { - name: 'reinstall demo app', - command: 'reinstall', - positionals: ['com.example.demo', appPath], - expectData: { platform: 'ios', bundleId: 'com.example.demo', appPath }, - }, - { - name: 'install demo app', - command: 'install', - positionals: ['com.example.demo', appPath], - expectData: { platform: 'ios', bundleId: 'com.example.demo', appPath }, - }, - { - name: 'list user apps by default', - command: 'apps', - assert: (apps) => { - assert.deepEqual(apps.json?.result?.data?.apps, ['Demo (com.example.demo)']); + { + name: 'capture iOS launch console while reopening app', + command: 'open', + positionals: ['com.apple.Preferences'], + flags: { + launchConsole: path.join(path.dirname(appPath), 'launch-console.log'), + }, + expectData: { appBundleId: 'com.apple.Preferences' }, }, - }, - { - name: 'list all apps with flag', - command: 'apps', - flags: { appsFilter: 'all' }, - assert: (apps) => { - assert.deepEqual(apps.json?.result?.data?.apps, [ - 'Maps (com.apple.Maps)', - 'Demo (com.example.demo)', - ]); + { + name: 'reinstall demo app', + command: 'reinstall', + positionals: ['com.example.demo', appPath], + expectData: { platform: 'ios', bundleId: 'com.example.demo', appPath }, }, - }, - { - name: 'refresh snapshot after install', - command: 'snapshot', - flags: { snapshotInteractiveOnly: true }, - }, - { - name: 'press snapshot ref', - command: 'press', - positionals: ['@e1'], - expectData: { x: 196, y: 122 }, - }, - { - name: 'pinch current app', - command: 'gesture', - input: { kind: 'pinch', scale: 0.8, origin: { x: 196, y: 122 } }, - expectData: { - kind: 'pinch', - durationMs: 300, - pointerCount: 2, - from: { x: 196, y: 122 }, - to: { x: 196, y: 122 }, + { + name: 'install demo app', + command: 'install', + positionals: ['com.example.demo', appPath], + expectData: { platform: 'ios', bundleId: 'com.example.demo', appPath }, }, - }, - { - name: 'pan current app', - command: 'gesture', - input: { - kind: 'pan', - origin: { x: 196, y: 122 }, - delta: { x: 80, y: 0 }, - durationMs: 500, + { + name: 'list user apps by default', + command: 'apps', + assert: (apps) => { + assert.deepEqual(apps.json?.result?.data?.apps, ['Demo (com.example.demo)']); + }, }, - expectData: { - kind: 'pan', - durationMs: 500, - pointerCount: 1, - from: { x: 196, y: 122 }, - to: { x: 276, y: 122 }, + { + name: 'list all apps with flag', + command: 'apps', + flags: { appsFilter: 'all' }, + assert: (apps) => { + assert.deepEqual(apps.json?.result?.data?.apps, [ + 'Maps (com.apple.Maps)', + 'Demo (com.example.demo)', + ]); + }, }, - }, - { - name: 'fling current app', - command: 'gesture', - input: { - kind: 'fling', - direction: 'right', - origin: { x: 196, y: 122 }, - distance: 180, + { + name: 'refresh snapshot after install', + command: 'snapshot', + flags: { snapshotInteractiveOnly: true }, }, - expectData: { - kind: 'fling', - durationMs: 100, - pointerCount: 1, - from: { x: 196, y: 122 }, - to: { x: 376, y: 122 }, + { + name: 'press snapshot ref', + command: 'press', + positionals: ['@e1'], + expectData: { x: 196, y: 122 }, }, - }, - { - name: 'rotate current app content', - command: 'gesture', - input: { kind: 'rotate', degrees: 35, origin: { x: 196, y: 122 } }, - expectData: { - kind: 'rotate', - durationMs: 300, - pointerCount: 2, - from: { x: 196, y: 122 }, - to: { x: 196, y: 122 }, + { + name: 'pinch current app', + command: 'gesture', + input: { kind: 'pinch', scale: 0.8, origin: { x: 196, y: 122 } }, + expectData: { + kind: 'pinch', + durationMs: 300, + pointerCount: 2, + from: { x: 196, y: 122 }, + to: { x: 196, y: 122 }, + }, }, - }, - { - name: 'transform current app content', - command: 'gesture', - input: { - kind: 'transform', - origin: { x: 196, y: 122 }, - delta: { x: 40, y: -20 }, - scale: 1.5, - degrees: 35, - durationMs: 700, + { + name: 'pan current app', + command: 'gesture', + input: { + kind: 'pan', + origin: { x: 196, y: 122 }, + delta: { x: 80, y: 0 }, + durationMs: 500, + }, + expectData: { + kind: 'pan', + durationMs: 500, + pointerCount: 1, + from: { x: 196, y: 122 }, + to: { x: 276, y: 122 }, + }, }, - expectData: { - kind: 'transform', - durationMs: 700, - pointerCount: 2, - from: { x: 196, y: 122 }, - to: { x: 236, y: 102 }, + { + name: 'fling current app', + command: 'gesture', + input: { + kind: 'fling', + direction: 'right', + origin: { x: 196, y: 122 }, + distance: 180, + }, + expectData: { + kind: 'fling', + durationMs: 100, + pointerCount: 1, + from: { x: 196, y: 122 }, + to: { x: 376, y: 122 }, + }, }, - }, - { - name: 'get ref attrs', - command: 'get', - positionals: ['attrs', '@e1'], - assert: (getAttrs) => { - assert.equal(getAttrs.json?.result?.data?.node?.label, 'General'); + { + name: 'rotate current app content', + command: 'gesture', + input: { kind: 'rotate', degrees: 35, origin: { x: 196, y: 122 } }, + expectData: { + kind: 'rotate', + durationMs: 300, + pointerCount: 2, + from: { x: 196, y: 122 }, + to: { x: 196, y: 122 }, + }, }, - }, - { - name: 'assert visible selector', - command: 'is', - positionals: ['visible', 'label=General'], - expectData: { pass: true }, - }, - { - name: 'find attrs by label', - command: 'find', - positionals: ['label', 'General', 'get', 'attrs'], - expectData: { ref: '@e1' }, - }, - { - name: 'wait for text', - command: 'wait', - positionals: ['text', 'General', '100'], - expectData: { text: 'General' }, - }, - { - name: 'navigate with explicit system back mode', - command: 'back', - flags: { backMode: 'system' }, - expectData: { mode: 'system' }, - }, - { - name: 'write clipboard', - command: 'clipboard', - positionals: ['write', 'runner otp 246810'], - expectData: { textLength: 17 }, - }, - { - name: 'read clipboard', - command: 'clipboard', - positionals: ['read'], - expectData: { text: 'runner otp 246810' }, - }, - { - name: 'dismiss keyboard', - command: 'keyboard', - positionals: ['dismiss'], - expectData: { platform: 'ios', action: 'dismiss', dismissed: true }, - }, - { - name: 'list active iOS session', - command: 'session_list', - assert: (list) => { - const sessions = list.json?.result?.data?.sessions; - assert.equal(sessions?.length, 1); - assert.equal(sessions?.[0]?.name, 'default'); - assert.equal(sessions?.[0]?.platform, 'ios'); - assert.equal(sessions?.[0]?.device_udid, PROVIDER_SCENARIO_IOS_SIMULATOR.id); - assert.equal(sessions?.[0]?.ios_simulator_device_set, null); + { + name: 'transform current app content', + command: 'gesture', + input: { + kind: 'transform', + origin: { x: 196, y: 122 }, + delta: { x: 40, y: -20 }, + scale: 1.5, + degrees: 35, + durationMs: 700, + }, + expectData: { + kind: 'transform', + durationMs: 700, + pointerCount: 2, + from: { x: 196, y: 122 }, + to: { x: 236, y: 102 }, + }, }, - }, - { name: 'close settings session', command: 'close' }, - { - name: 'list sessions after close', - command: 'session_list', - assert: (list) => { - assert.deepEqual(list.json?.result?.data?.sessions, []); + { + name: 'get ref attrs', + command: 'get', + positionals: ['attrs', '@e1'], + assert: (getAttrs) => { + assert.equal(getAttrs.json?.result?.data?.node?.label, 'General'); + }, }, - }, - ]); + { + name: 'assert visible selector', + command: 'is', + positionals: ['visible', 'label=General'], + expectData: { pass: true }, + }, + { + name: 'find attrs by label', + command: 'find', + positionals: ['label', 'General', 'get', 'attrs'], + expectData: { ref: '@e1' }, + }, + { + name: 'wait for text', + command: 'wait', + positionals: ['text', 'General', '100'], + expectData: { text: 'General' }, + }, + { + name: 'navigate with explicit system back mode', + command: 'back', + flags: { backMode: 'system' }, + expectData: { mode: 'system' }, + }, + { + name: 'write clipboard', + command: 'clipboard', + positionals: ['write', 'runner otp 246810'], + expectData: { textLength: 17 }, + }, + { + name: 'read clipboard', + command: 'clipboard', + positionals: ['read'], + expectData: { text: 'runner otp 246810' }, + }, + { + name: 'dismiss keyboard', + command: 'keyboard', + positionals: ['dismiss'], + expectData: { platform: 'ios', action: 'dismiss', dismissed: true }, + }, + { + name: 'list active iOS session', + command: 'session_list', + assert: (list) => { + const sessions = list.json?.result?.data?.sessions; + assert.equal(sessions?.length, 1); + assert.equal(sessions?.[0]?.name, 'default'); + assert.equal(sessions?.[0]?.platform, 'ios'); + assert.equal(sessions?.[0]?.device_udid, PROVIDER_SCENARIO_IOS_SIMULATOR.id); + assert.equal(sessions?.[0]?.ios_simulator_device_set, null); + }, + }, + { name: 'close settings session', command: 'close' }, + { + name: 'list sessions after close', + command: 'session_list', + assert: (list) => { + assert.deepEqual(list.json?.result?.data?.sessions, []); + }, + }, + ]); - runnerTranscript.assertComplete(); - assertFlatToolCall(appleTool.calls, ['simctl', 'launch', 'sim-1', 'com.apple.Preferences']); - assertFlatToolCall(appleTool.calls, [ - 'simctl', - 'launch', - '--console-pty', - 'sim-1', - 'com.apple.Preferences', - ]); - assertFlatToolCall(appleTool.calls, ['simctl', 'uninstall', 'sim-1', 'com.example.demo']); - assertFlatToolCall(appleTool.calls, ['plist', 'readJson', path.join(appPath, 'Info.plist')]); - assertFlatToolCall(appleTool.calls, ['simctl', 'install', 'sim-1', appPath]); - assertFlatToolCall(appleTool.calls, ['simctl', 'pbcopy', 'sim-1']); - assertFlatToolCall(appleTool.calls, ['simctl', 'pbpaste', 'sim-1']); - assert.ok( - inventoryRequests.some( - (request) => request.iosSimulatorSetPath === '/tmp/provider-scenario-simulators', - ), - JSON.stringify(inventoryRequests), - ); - }, - ); -}); + runnerTranscript.assertComplete(); + assertFlatToolCall(appleTool.calls, ['simctl', 'launch', 'sim-1', 'com.apple.Preferences']); + assertFlatToolCall(appleTool.calls, [ + 'simctl', + 'launch', + '--console-pty', + 'sim-1', + 'com.apple.Preferences', + ]); + assertFlatToolCall(appleTool.calls, ['simctl', 'uninstall', 'sim-1', 'com.example.demo']); + assertFlatToolCall(appleTool.calls, [ + 'plist', + 'readJson', + path.join(appPath, 'Info.plist'), + ]); + assertFlatToolCall(appleTool.calls, ['simctl', 'install', 'sim-1', appPath]); + assertFlatToolCall(appleTool.calls, ['simctl', 'pbcopy', 'sim-1']); + assertFlatToolCall(appleTool.calls, ['simctl', 'pbpaste', 'sim-1']); + assert.ok( + inventoryRequests.some( + (request) => request.iosSimulatorSetPath === '/tmp/provider-scenario-simulators', + ), + JSON.stringify(inventoryRequests), + ); + }, + ); + }, +); test( 'Provider-backed integration iOS regular snapshot preserves fixed bottom tabs after scroll content', From 0599920bcde1eaa99c49e9163cda8682ce1209c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 20 Aug 2026 08:15:02 +0200 Subject: [PATCH 02/10] fix(snapshot): select scopes with presented content --- .../RunnerSnapshotScopePolicy.swift | 46 +++++++++++++---- .../RunnerTests+SnapshotPresentation.swift | 17 ++++--- contracts/fixtures/snapshot-scope-policy.json | 51 +++++++++++++++++++ packages/contracts/src/snapshot-scope.test.ts | 18 +++++-- .../__tests__/ui-hierarchy-scope.test.ts | 30 +++++++---- 5 files changed, 132 insertions(+), 30 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSnapshotScopePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSnapshotScopePolicy.swift index 5f9d104ed2..ba99ea5ba9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSnapshotScopePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSnapshotScopePolicy.swift @@ -9,18 +9,23 @@ enum SnapshotScopeSelection: Equatable { /// Cross-runtime snapshot scope specification. /// /// A non-empty scope selects the first node in presentation preorder whose label, identifier, or -/// value contains the trimmed query case-insensitively. Missing matches publish an empty projection. +/// value contains the trimmed query case-insensitively and whose subtree contributes to the +/// requested projection. Missing matches publish an empty projection. enum SnapshotScopePolicy { static func select( fromPreorder nodes: [Node], scope: String?, - semanticValues: (Node) -> [String?] + depth: (Node) -> Int, + semanticValues: (Node) -> [String?], + subtreeContributes: (Range) -> Bool ) -> SnapshotScopeSelection { guard let query = normalized(scope) else { return .unscoped } for (index, node) in nodes.enumerated() { - if semanticValues(node).contains(where: { value in + guard semanticValues(node).contains(where: { value in value?.lowercased().contains(query) == true - }) { + }) else { continue } + let range = subtreeRange(from: index, in: nodes, depth: depth) + if subtreeContributes(range) { return .matched(index) } } @@ -37,6 +42,19 @@ enum SnapshotScopePolicy { } return query.lowercased() } + + static func subtreeRange( + from start: Int, + in nodes: [Node], + depth: (Node) -> Int + ) -> Range { + let rootDepth = depth(nodes[start]) + var end = start + 1 + while end < nodes.count, depth(nodes[end]) > rootDepth { + end += 1 + } + return start.. rootDepth { - end += 1 - } - actual = Array(start.. startDepth { - endIndex += 1 - } + let range = SnapshotScopePolicy.subtreeRange( + from: startIndex, + in: rawNodes, + depth: \.depth + ) let maxDepth = options.depth ?? Int.max return reindex( - Array(rawNodes[startIndex.. { +test('the shared scope policy picks the first matching subtree with presented content', () => { const cases = JSON.parse(fs.readFileSync(TABLE_PATH, 'utf8')) as ScopePolicyCase[]; assert.ok(cases.length > 0, 'scope policy table must not be empty'); assert.equal(new Set(cases.map((fixture) => fixture.name)).size, cases.length); @@ -42,7 +48,13 @@ test('the shared scope predicate picks the golden table root as the first docume ); continue; } - const rootIndex = fixture.nodes.findIndex((node) => matchesSnapshotScope(node, fixture.scope)); + const rootIndex = fixture.nodes.findIndex( + (node, index) => + matchesSnapshotScope(node, fixture.scope) && + subtreeIndexes(fixture.nodes, index).some( + (subtreeIndex) => fixture.nodes[subtreeIndex]?.presented !== false, + ), + ); assert.equal(rootIndex === -1 ? null : rootIndex, fixture.expectedRootIndex, fixture.name); assert.deepEqual( subtreeIndexes(fixture.nodes, fixture.expectedRootIndex), diff --git a/src/platforms/android/__tests__/ui-hierarchy-scope.test.ts b/src/platforms/android/__tests__/ui-hierarchy-scope.test.ts index 1df72da821..a9d8c7660d 100644 --- a/src/platforms/android/__tests__/ui-hierarchy-scope.test.ts +++ b/src/platforms/android/__tests__/ui-hierarchy-scope.test.ts @@ -15,7 +15,14 @@ import { parseUiHierarchy } from './ui-hierarchy-fixtures.ts'; type ScopePolicyCase = { name: string; scope: string; - nodes: Array<{ depth: number; label?: string; value?: string; identifier?: string }>; + nodes: Array<{ + depth: number; + type?: string; + label?: string; + value?: string; + identifier?: string; + presented?: boolean; + }>; expectedSubtreeIndexes: number[]; }; @@ -28,8 +35,9 @@ const TABLE_PATH = path.resolve( * Renders the flat golden list as helper XML. Fixture position rides in the bounds' x origin so * the assertion never depends on which text field carried the match. Android reads * `label = text || content-desc` and `value = text`, so a fixture value goes to `text` and a - * label with no value to `content-desc`. Nodes render as `TextView` (non-structural, so regular - * membership keeps every one of them): the leg measures the scope rule, not membership. + * label with no value to `content-desc`. Most cases render as `TextView` so regular membership + * keeps every node. Contribution cases declare their Android type and expected membership so the + * same table also exercises scope selection after membership. */ function scopePolicyXml(nodes: ScopePolicyCase['nodes']): string { const lines: string[] = ['']; @@ -40,9 +48,10 @@ function scopePolicyXml(nodes: ScopePolicyCase['nodes']): string { lines.push(''); } const attrs = [ - `class="android.widget.TextView"`, + `class="${node.type === 'ViewGroup' ? 'android.view.ViewGroup' : `android.widget.${node.type ?? 'TextView'}`}"`, `bounds="[${index},0][${index + 1},1]"`, `visible-to-user="true"`, + node.type === 'Button' ? `clickable="true"` : '', node.value !== undefined ? `text="${node.value}"` : '', node.label !== undefined && node.value === undefined ? `content-desc="${node.label}"` : '', node.identifier !== undefined ? `resource-id="${node.identifier}"` : '', @@ -62,18 +71,19 @@ test('the Android projection agrees with every golden scope-policy table case', const cases = JSON.parse(fs.readFileSync(TABLE_PATH, 'utf8')) as ScopePolicyCase[]; assert.ok(cases.length > 0); for (const fixture of cases) { - // Raw and regular both keep every rendered node, so each leg measures the scope RULE. There is - // no -i leg here on purpose: bare TextViews carry no action, so -i would drop the whole fixture - // and the leg would measure membership instead. The scope/membership interplay is pinned by the - // two projection tests below, on shapes where -i keeps real content. - for (const projection of [{ raw: true }, {}]) { + const hasMembershipCase = fixture.nodes.some((node) => node.presented !== undefined); + const projections = hasMembershipCase ? [{ interactiveOnly: true }] : [{ raw: true }, {}]; + for (const projection of projections) { const result = parseUiHierarchy(scopePolicyXml(fixture.nodes), undefined, { ...projection, scope: fixture.scope, }); + const expected = fixture.expectedSubtreeIndexes.filter( + (index) => fixture.nodes[index]?.presented !== false, + ); assert.deepEqual( result.nodes.map((node) => node.rect?.x), - fixture.expectedSubtreeIndexes, + expected, `${fixture.name} (${JSON.stringify(projection)})`, ); if (result.nodes.length > 0) assert.equal(result.nodes[0]?.depth, 0, fixture.name); From 65ebde5a88e9ecc38396f3c59361d1da58c2c5d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 20 Aug 2026 08:16:47 +0200 Subject: [PATCH 03/10] docs(snapshot): describe presentation-owned scope --- website/docs/docs/commands.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 4a0cbe5a9f..66f6a932da 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -348,8 +348,8 @@ agent-device get attrs @e1 full tree. Under `-i` that means scoping to a layout container returns the actionable elements inside it, even when the container itself is filtered out. `--depth` then counts from the scope root. `@ref` scopes by that element's label from the last snapshot. Android resolves scope inside - its projection; on iOS the runner first narrows capture by label/identifier and the daemon applies - the rule to the presented tree. + its TypeScript presentation; iOS keeps acquisition broad and resolves scope once inside the + runner's Swift presentation. The daemon does not reapply scope after either platform returns. - `--actions` names the custom accessibility affordances an element merged away (iOS `UIAccessibilityCustomAction`, React Native `accessibilityActions`), so a card whose reply/options controls are not separate elements still lists them. It is iOS-simulator-only and exists for From 4ff7fe07cfd2f69f47983d68c0546858fc2b83c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 20 Aug 2026 08:17:50 +0200 Subject: [PATCH 04/10] docs(snapshot): record contribution-aware scope --- docs/adr/0004-ios-snapshot-backend-strategy.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index d1a2fabff4..58457d69ac 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -115,10 +115,11 @@ noise suppressions. When eligibility removes a structural wrapper, presentation surviving descendants to the nearest surviving ancestor and normalizes their indexes and depths. The second semantic layer makes scope a presentation specification rather than an acquisition or -daemon-compaction policy. A trimmed non-empty scope selects the first presentation-preorder node -whose label, identifier, or value contains it case-insensitively; the selected subtree is re-rooted, -depth is applied relative to that root, and no match publishes an empty healthy projection. Swift -and TypeScript implementations are pinned by `contracts/fixtures/snapshot-scope-policy.json`. +daemon-compaction policy. A trimmed non-empty scope selects the first presentation-preorder match +whose subtree contributes to the requested projection; matching inspects label, identifier, and +value case-insensitively. The selected subtree is re-rooted, depth is applied relative to that root, +and no match publishes an empty healthy projection. Swift and TypeScript implementations are pinned +by `contracts/fixtures/snapshot-scope-policy.json`. Scoped iOS acquisition stays broad (including when depth is requested) until an adapter can prove a narrowing hint complete. The daemon never reapplies scope after the wire; Android selects its root inside its TypeScript presentation and desktop surface runtimes retain their platform projection. From f7f11e9594f4ef3f0c213dc90b97df1d4330cb2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 20 Aug 2026 08:21:05 +0200 Subject: [PATCH 05/10] refactor(snapshot): drop unrelated provider churn --- .../provider-scenarios/ios-lifecycle.test.ts | 537 +++++++++--------- 1 file changed, 263 insertions(+), 274 deletions(-) diff --git a/test/integration/provider-scenarios/ios-lifecycle.test.ts b/test/integration/provider-scenarios/ios-lifecycle.test.ts index 7e1f8030b3..2becd3f0c5 100644 --- a/test/integration/provider-scenarios/ios-lifecycle.test.ts +++ b/test/integration/provider-scenarios/ios-lifecycle.test.ts @@ -15,300 +15,289 @@ import { import { withProviderScenarioResource } from './harness.ts'; import { PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS } from './test-timeouts.ts'; -function testSlowProviderScenario(name: string, run: () => Promise): void { - test(name, run, PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS); -} - -testSlowProviderScenario( - 'Provider-backed integration iOS Settings flow uses scripted simctl and runner providers', - async () => { - await withProviderScenarioResource( - createIosSettingsWorld, - async ({ appPath, appleTool, daemon, inventoryRequests, runnerTranscript }) => { - const scopedDevices = await daemon.client().devices.list({ - platform: 'ios', - iosSimulatorDeviceSet: '/tmp/provider-scenario-simulators', - }); - assert.equal(scopedDevices.length, 1); - assert.equal(scopedDevices[0]?.id, PROVIDER_SCENARIO_IOS_SIMULATOR.id); +test('Provider-backed integration iOS Settings flow uses scripted simctl and runner providers', async () => { + await withProviderScenarioResource( + createIosSettingsWorld, + async ({ appPath, appleTool, daemon, inventoryRequests, runnerTranscript }) => { + const scopedDevices = await daemon.client().devices.list({ + platform: 'ios', + iosSimulatorDeviceSet: '/tmp/provider-scenario-simulators', + }); + assert.equal(scopedDevices.length, 1); + assert.equal(scopedDevices[0]?.id, PROVIDER_SCENARIO_IOS_SIMULATOR.id); - await runProviderScenario(daemon, [ - { - name: 'open settings app', - command: 'open', - positionals: ['com.apple.Preferences'], - flags: { platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, - expectData: { - appBundleId: 'com.apple.Preferences', - device_udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, - }, - }, - { - name: 'read app session state', - command: 'appstate', - flags: { platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, - expectData: { - platform: 'ios', - appBundleId: 'com.apple.Preferences', - source: 'session', - device_udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, - ios_simulator_device_set: null, - }, - }, - { - name: 'prepare iOS runner', - command: 'prepare', - positionals: ['ios-runner'], - flags: { platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, - expectData: { - action: 'ios-runner', - platform: 'ios', - deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, - runner: { uptimeMs: 42 }, - }, - }, - { - name: 'capture settings snapshot', - command: 'snapshot', - flags: { snapshotInteractiveOnly: true }, - assert: (firstSnapshot) => { - assert.equal(firstSnapshot.json?.result?.data?.nodes?.[0]?.label, 'General'); - assert.equal(firstSnapshot.json?.result?.data?.nodes?.[0]?.ref, 'e1'); - }, - }, - { - name: 'reopen existing session app', - command: 'open', - positionals: ['com.apple.Preferences'], - expectData: { appBundleId: 'com.apple.Preferences' }, - }, - { - name: 'capture iOS launch console while reopening app', - command: 'open', - positionals: ['com.apple.Preferences'], - flags: { - launchConsole: path.join(path.dirname(appPath), 'launch-console.log'), - }, - expectData: { appBundleId: 'com.apple.Preferences' }, - }, - { - name: 'reinstall demo app', - command: 'reinstall', - positionals: ['com.example.demo', appPath], - expectData: { platform: 'ios', bundleId: 'com.example.demo', appPath }, - }, - { - name: 'install demo app', - command: 'install', - positionals: ['com.example.demo', appPath], - expectData: { platform: 'ios', bundleId: 'com.example.demo', appPath }, + await runProviderScenario(daemon, [ + { + name: 'open settings app', + command: 'open', + positionals: ['com.apple.Preferences'], + flags: { platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, + expectData: { + appBundleId: 'com.apple.Preferences', + device_udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, }, - { - name: 'list user apps by default', - command: 'apps', - assert: (apps) => { - assert.deepEqual(apps.json?.result?.data?.apps, ['Demo (com.example.demo)']); - }, + }, + { + name: 'read app session state', + command: 'appstate', + flags: { platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, + expectData: { + platform: 'ios', + appBundleId: 'com.apple.Preferences', + source: 'session', + device_udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + ios_simulator_device_set: null, }, - { - name: 'list all apps with flag', - command: 'apps', - flags: { appsFilter: 'all' }, - assert: (apps) => { - assert.deepEqual(apps.json?.result?.data?.apps, [ - 'Maps (com.apple.Maps)', - 'Demo (com.example.demo)', - ]); - }, + }, + { + name: 'prepare iOS runner', + command: 'prepare', + positionals: ['ios-runner'], + flags: { platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, + expectData: { + action: 'ios-runner', + platform: 'ios', + deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + runner: { uptimeMs: 42 }, }, - { - name: 'refresh snapshot after install', - command: 'snapshot', - flags: { snapshotInteractiveOnly: true }, + }, + { + name: 'capture settings snapshot', + command: 'snapshot', + flags: { snapshotInteractiveOnly: true }, + assert: (firstSnapshot) => { + assert.equal(firstSnapshot.json?.result?.data?.nodes?.[0]?.label, 'General'); + assert.equal(firstSnapshot.json?.result?.data?.nodes?.[0]?.ref, 'e1'); }, - { - name: 'press snapshot ref', - command: 'press', - positionals: ['@e1'], - expectData: { x: 196, y: 122 }, + }, + { + name: 'reopen existing session app', + command: 'open', + positionals: ['com.apple.Preferences'], + expectData: { appBundleId: 'com.apple.Preferences' }, + }, + { + name: 'capture iOS launch console while reopening app', + command: 'open', + positionals: ['com.apple.Preferences'], + flags: { + launchConsole: path.join(path.dirname(appPath), 'launch-console.log'), }, - { - name: 'pinch current app', - command: 'gesture', - input: { kind: 'pinch', scale: 0.8, origin: { x: 196, y: 122 } }, - expectData: { - kind: 'pinch', - durationMs: 300, - pointerCount: 2, - from: { x: 196, y: 122 }, - to: { x: 196, y: 122 }, - }, + expectData: { appBundleId: 'com.apple.Preferences' }, + }, + { + name: 'reinstall demo app', + command: 'reinstall', + positionals: ['com.example.demo', appPath], + expectData: { platform: 'ios', bundleId: 'com.example.demo', appPath }, + }, + { + name: 'install demo app', + command: 'install', + positionals: ['com.example.demo', appPath], + expectData: { platform: 'ios', bundleId: 'com.example.demo', appPath }, + }, + { + name: 'list user apps by default', + command: 'apps', + assert: (apps) => { + assert.deepEqual(apps.json?.result?.data?.apps, ['Demo (com.example.demo)']); }, - { - name: 'pan current app', - command: 'gesture', - input: { - kind: 'pan', - origin: { x: 196, y: 122 }, - delta: { x: 80, y: 0 }, - durationMs: 500, - }, - expectData: { - kind: 'pan', - durationMs: 500, - pointerCount: 1, - from: { x: 196, y: 122 }, - to: { x: 276, y: 122 }, - }, + }, + { + name: 'list all apps with flag', + command: 'apps', + flags: { appsFilter: 'all' }, + assert: (apps) => { + assert.deepEqual(apps.json?.result?.data?.apps, [ + 'Maps (com.apple.Maps)', + 'Demo (com.example.demo)', + ]); }, - { - name: 'fling current app', - command: 'gesture', - input: { - kind: 'fling', - direction: 'right', - origin: { x: 196, y: 122 }, - distance: 180, - }, - expectData: { - kind: 'fling', - durationMs: 100, - pointerCount: 1, - from: { x: 196, y: 122 }, - to: { x: 376, y: 122 }, - }, + }, + { + name: 'refresh snapshot after install', + command: 'snapshot', + flags: { snapshotInteractiveOnly: true }, + }, + { + name: 'press snapshot ref', + command: 'press', + positionals: ['@e1'], + expectData: { x: 196, y: 122 }, + }, + { + name: 'pinch current app', + command: 'gesture', + input: { kind: 'pinch', scale: 0.8, origin: { x: 196, y: 122 } }, + expectData: { + kind: 'pinch', + durationMs: 300, + pointerCount: 2, + from: { x: 196, y: 122 }, + to: { x: 196, y: 122 }, }, - { - name: 'rotate current app content', - command: 'gesture', - input: { kind: 'rotate', degrees: 35, origin: { x: 196, y: 122 } }, - expectData: { - kind: 'rotate', - durationMs: 300, - pointerCount: 2, - from: { x: 196, y: 122 }, - to: { x: 196, y: 122 }, - }, + }, + { + name: 'pan current app', + command: 'gesture', + input: { + kind: 'pan', + origin: { x: 196, y: 122 }, + delta: { x: 80, y: 0 }, + durationMs: 500, }, - { - name: 'transform current app content', - command: 'gesture', - input: { - kind: 'transform', - origin: { x: 196, y: 122 }, - delta: { x: 40, y: -20 }, - scale: 1.5, - degrees: 35, - durationMs: 700, - }, - expectData: { - kind: 'transform', - durationMs: 700, - pointerCount: 2, - from: { x: 196, y: 122 }, - to: { x: 236, y: 102 }, - }, + expectData: { + kind: 'pan', + durationMs: 500, + pointerCount: 1, + from: { x: 196, y: 122 }, + to: { x: 276, y: 122 }, }, - { - name: 'get ref attrs', - command: 'get', - positionals: ['attrs', '@e1'], - assert: (getAttrs) => { - assert.equal(getAttrs.json?.result?.data?.node?.label, 'General'); - }, + }, + { + name: 'fling current app', + command: 'gesture', + input: { + kind: 'fling', + direction: 'right', + origin: { x: 196, y: 122 }, + distance: 180, }, - { - name: 'assert visible selector', - command: 'is', - positionals: ['visible', 'label=General'], - expectData: { pass: true }, + expectData: { + kind: 'fling', + durationMs: 100, + pointerCount: 1, + from: { x: 196, y: 122 }, + to: { x: 376, y: 122 }, }, - { - name: 'find attrs by label', - command: 'find', - positionals: ['label', 'General', 'get', 'attrs'], - expectData: { ref: '@e1' }, + }, + { + name: 'rotate current app content', + command: 'gesture', + input: { kind: 'rotate', degrees: 35, origin: { x: 196, y: 122 } }, + expectData: { + kind: 'rotate', + durationMs: 300, + pointerCount: 2, + from: { x: 196, y: 122 }, + to: { x: 196, y: 122 }, }, - { - name: 'wait for text', - command: 'wait', - positionals: ['text', 'General', '100'], - expectData: { text: 'General' }, + }, + { + name: 'transform current app content', + command: 'gesture', + input: { + kind: 'transform', + origin: { x: 196, y: 122 }, + delta: { x: 40, y: -20 }, + scale: 1.5, + degrees: 35, + durationMs: 700, }, - { - name: 'navigate with explicit system back mode', - command: 'back', - flags: { backMode: 'system' }, - expectData: { mode: 'system' }, + expectData: { + kind: 'transform', + durationMs: 700, + pointerCount: 2, + from: { x: 196, y: 122 }, + to: { x: 236, y: 102 }, }, - { - name: 'write clipboard', - command: 'clipboard', - positionals: ['write', 'runner otp 246810'], - expectData: { textLength: 17 }, + }, + { + name: 'get ref attrs', + command: 'get', + positionals: ['attrs', '@e1'], + assert: (getAttrs) => { + assert.equal(getAttrs.json?.result?.data?.node?.label, 'General'); }, - { - name: 'read clipboard', - command: 'clipboard', - positionals: ['read'], - expectData: { text: 'runner otp 246810' }, + }, + { + name: 'assert visible selector', + command: 'is', + positionals: ['visible', 'label=General'], + expectData: { pass: true }, + }, + { + name: 'find attrs by label', + command: 'find', + positionals: ['label', 'General', 'get', 'attrs'], + expectData: { ref: '@e1' }, + }, + { + name: 'wait for text', + command: 'wait', + positionals: ['text', 'General', '100'], + expectData: { text: 'General' }, + }, + { + name: 'navigate with explicit system back mode', + command: 'back', + flags: { backMode: 'system' }, + expectData: { mode: 'system' }, + }, + { + name: 'write clipboard', + command: 'clipboard', + positionals: ['write', 'runner otp 246810'], + expectData: { textLength: 17 }, + }, + { + name: 'read clipboard', + command: 'clipboard', + positionals: ['read'], + expectData: { text: 'runner otp 246810' }, + }, + { + name: 'dismiss keyboard', + command: 'keyboard', + positionals: ['dismiss'], + expectData: { platform: 'ios', action: 'dismiss', dismissed: true }, + }, + { + name: 'list active iOS session', + command: 'session_list', + assert: (list) => { + const sessions = list.json?.result?.data?.sessions; + assert.equal(sessions?.length, 1); + assert.equal(sessions?.[0]?.name, 'default'); + assert.equal(sessions?.[0]?.platform, 'ios'); + assert.equal(sessions?.[0]?.device_udid, PROVIDER_SCENARIO_IOS_SIMULATOR.id); + assert.equal(sessions?.[0]?.ios_simulator_device_set, null); }, - { - name: 'dismiss keyboard', - command: 'keyboard', - positionals: ['dismiss'], - expectData: { platform: 'ios', action: 'dismiss', dismissed: true }, + }, + { name: 'close settings session', command: 'close' }, + { + name: 'list sessions after close', + command: 'session_list', + assert: (list) => { + assert.deepEqual(list.json?.result?.data?.sessions, []); }, - { - name: 'list active iOS session', - command: 'session_list', - assert: (list) => { - const sessions = list.json?.result?.data?.sessions; - assert.equal(sessions?.length, 1); - assert.equal(sessions?.[0]?.name, 'default'); - assert.equal(sessions?.[0]?.platform, 'ios'); - assert.equal(sessions?.[0]?.device_udid, PROVIDER_SCENARIO_IOS_SIMULATOR.id); - assert.equal(sessions?.[0]?.ios_simulator_device_set, null); - }, - }, - { name: 'close settings session', command: 'close' }, - { - name: 'list sessions after close', - command: 'session_list', - assert: (list) => { - assert.deepEqual(list.json?.result?.data?.sessions, []); - }, - }, - ]); + }, + ]); - runnerTranscript.assertComplete(); - assertFlatToolCall(appleTool.calls, ['simctl', 'launch', 'sim-1', 'com.apple.Preferences']); - assertFlatToolCall(appleTool.calls, [ - 'simctl', - 'launch', - '--console-pty', - 'sim-1', - 'com.apple.Preferences', - ]); - assertFlatToolCall(appleTool.calls, ['simctl', 'uninstall', 'sim-1', 'com.example.demo']); - assertFlatToolCall(appleTool.calls, [ - 'plist', - 'readJson', - path.join(appPath, 'Info.plist'), - ]); - assertFlatToolCall(appleTool.calls, ['simctl', 'install', 'sim-1', appPath]); - assertFlatToolCall(appleTool.calls, ['simctl', 'pbcopy', 'sim-1']); - assertFlatToolCall(appleTool.calls, ['simctl', 'pbpaste', 'sim-1']); - assert.ok( - inventoryRequests.some( - (request) => request.iosSimulatorSetPath === '/tmp/provider-scenario-simulators', - ), - JSON.stringify(inventoryRequests), - ); - }, - ); - }, -); + runnerTranscript.assertComplete(); + assertFlatToolCall(appleTool.calls, ['simctl', 'launch', 'sim-1', 'com.apple.Preferences']); + assertFlatToolCall(appleTool.calls, [ + 'simctl', + 'launch', + '--console-pty', + 'sim-1', + 'com.apple.Preferences', + ]); + assertFlatToolCall(appleTool.calls, ['simctl', 'uninstall', 'sim-1', 'com.example.demo']); + assertFlatToolCall(appleTool.calls, ['plist', 'readJson', path.join(appPath, 'Info.plist')]); + assertFlatToolCall(appleTool.calls, ['simctl', 'install', 'sim-1', appPath]); + assertFlatToolCall(appleTool.calls, ['simctl', 'pbcopy', 'sim-1']); + assertFlatToolCall(appleTool.calls, ['simctl', 'pbpaste', 'sim-1']); + assert.ok( + inventoryRequests.some( + (request) => request.iosSimulatorSetPath === '/tmp/provider-scenario-simulators', + ), + JSON.stringify(inventoryRequests), + ); + }, + ); +}); test( 'Provider-backed integration iOS regular snapshot preserves fixed bottom tabs after scroll content', From 04e7f9f014f00b4780473df654075e9482892d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 20 Aug 2026 08:27:13 +0200 Subject: [PATCH 06/10] fix(snapshot): accept healthy empty scoped capture --- .../interactor-runner-provider.test.ts | 33 +++++++++++++++++++ src/platforms/apple/interactor.ts | 7 +++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/platforms/apple/__tests__/interactor-runner-provider.test.ts b/src/platforms/apple/__tests__/interactor-runner-provider.test.ts index d20ff5d832..0c9d07a752 100644 --- a/src/platforms/apple/__tests__/interactor-runner-provider.test.ts +++ b/src/platforms/apple/__tests__/interactor-runner-provider.test.ts @@ -148,6 +148,39 @@ test('snapshot over the injected transport keeps the shared xctest result shape' assert.equal(result.nodes?.length, 2); }); +test('snapshot accepts only structured healthy empty scope results', async () => { + const healthyEmptyProvider: AppleRunnerProvider = { + runCommand: async () => ({ + nodes: [], + snapshotQuality: { state: 'healthy', backend: 'tree' }, + }), + }; + const interactor = createAppleInteractor(IOS_SIMULATOR, {}, healthyEmptyProvider); + + const scoped = await interactor.snapshot({ scope: 'missing' }); + assert.deepEqual(scoped.nodes, []); + assert.equal(scoped.backend, 'xctest'); + assert.equal(scoped.quality?.state, 'healthy'); + await assert.rejects( + interactor.snapshot(), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + error.message === 'XCTest snapshot returned 0 nodes on iOS simulator.', + ); + + const legacyEmpty = createAppleInteractor( + IOS_SIMULATOR, + {}, + { + runCommand: async () => ({ nodes: [] }), + }, + ); + await assert.rejects(legacyEmpty.snapshot({ scope: 'missing' }), { + code: 'COMMAND_FAILED', + }); +}); + // #1634 P2: the backend pin must actually reach the wire — the daemon test // stops at the dispatch context and the Swift test starts at the parsed // command, so this is the assertion that fails if the interactor stops diff --git a/src/platforms/apple/interactor.ts b/src/platforms/apple/interactor.ts index 6f586e9657..e53d104f6b 100644 --- a/src/platforms/apple/interactor.ts +++ b/src/platforms/apple/interactor.ts @@ -19,6 +19,7 @@ import { } from './core/runner/runner-provider.ts'; import { toAppleTvRemoteButton } from '@agent-device/contracts/interaction'; import { DEVICE_ROTATIONS, type DeviceRotation } from '@agent-device/contracts/device'; +import { normalizeSnapshotScope } from '@agent-device/contracts/snapshot'; import { withDiagnosticTimer } from '../../utils/diagnostics.ts'; import { isMacOs, isTvOsDevice, type DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; @@ -183,7 +184,11 @@ async function captureAppleRunnerSnapshot( ), ); const nodes = result.nodes ?? []; - if (nodes.length === 0 && device.kind === 'simulator') { + const isValidEmptyScope = + normalizeSnapshotScope(options?.scope) !== null && + result.quality !== undefined && + result.quality.state !== 'sparse'; + if (nodes.length === 0 && device.kind === 'simulator' && !isValidEmptyScope) { throw new AppError('COMMAND_FAILED', 'XCTest snapshot returned 0 nodes on iOS simulator.'); } return { From d9e4f06e06d6cba87606a0980d737ef59ea409ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 20 Aug 2026 08:31:46 +0200 Subject: [PATCH 07/10] refactor(snapshot): isolate empty-scope admission --- src/platforms/apple/interactor.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/platforms/apple/interactor.ts b/src/platforms/apple/interactor.ts index e53d104f6b..457af6e16a 100644 --- a/src/platforms/apple/interactor.ts +++ b/src/platforms/apple/interactor.ts @@ -184,10 +184,7 @@ async function captureAppleRunnerSnapshot( ), ); const nodes = result.nodes ?? []; - const isValidEmptyScope = - normalizeSnapshotScope(options?.scope) !== null && - result.quality !== undefined && - result.quality.state !== 'sparse'; + const isValidEmptyScope = acceptsEmptyScopedSnapshot(options, result.quality); if (nodes.length === 0 && device.kind === 'simulator' && !isValidEmptyScope) { throw new AppError('COMMAND_FAILED', 'XCTest snapshot returned 0 nodes on iOS simulator.'); } @@ -201,6 +198,15 @@ async function captureAppleRunnerSnapshot( }; } +function acceptsEmptyScopedSnapshot( + options: SnapshotOptions | undefined, + quality: SnapshotQualityVerdict | undefined, +): boolean { + return ( + normalizeSnapshotScope(options?.scope) !== null && quality?.state !== 'sparse' && !!quality + ); +} + function mergeRunnerCallSignal( options: RunnerCallOptions, signal: AbortSignal | undefined, From 9055b4c17474bf59dc27cddc058cb3acb9595523 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 20 Aug 2026 08:38:40 +0200 Subject: [PATCH 08/10] fix(snapshot): align scope ownership across runtimes --- packages/contracts/src/snapshot-scope.ts | 16 ++++++++++------ src/daemon/snapshot-state.ts | 10 +++++----- src/platforms/apple/interactor.ts | 4 +++- src/snapshot/snapshot-desktop-surface.test.ts | 12 ++++++++++-- src/snapshot/snapshot-desktop-surface.ts | 16 +++++++++++++--- 5 files changed, 41 insertions(+), 17 deletions(-) diff --git a/packages/contracts/src/snapshot-scope.ts b/packages/contracts/src/snapshot-scope.ts index 93f8f376f6..42017edee1 100644 --- a/packages/contracts/src/snapshot-scope.ts +++ b/packages/contracts/src/snapshot-scope.ts @@ -40,14 +40,18 @@ export function matchesSnapshotScope(node: SnapshotScopeCandidate, scope: string export function findSnapshotScopeRange( nodes: readonly (SnapshotScopeCandidate & { depth?: number })[], scope: string, + subtreeContributes: (range: { start: number; end: number }) => boolean = () => true, ): { start: number; end: number } | null { if (!normalizeSnapshotScope(scope)) return null; - const start = nodes.findIndex((node) => matchesSnapshotScope(node, scope)); - if (start === -1) return null; - const rootDepth = nodes[start]?.depth ?? 0; - let end = start + 1; - while (end < nodes.length && (nodes[end]?.depth ?? 0) > rootDepth) end += 1; - return { start, end }; + for (const [start, node] of nodes.entries()) { + if (!matchesSnapshotScope(node, scope)) continue; + const rootDepth = node.depth ?? 0; + let end = start + 1; + while (end < nodes.length && (nodes[end]?.depth ?? 0) > rootDepth) end += 1; + const range = { start, end }; + if (subtreeContributes(range)) return range; + } + return null; } /** Re-roots a document-order slice: fresh indexes, remapped parents, depth rebased by `depthOffset`. */ diff --git a/src/daemon/snapshot-state.ts b/src/daemon/snapshot-state.ts index d9526ca8bc..b0eef02863 100644 --- a/src/daemon/snapshot-state.ts +++ b/src/daemon/snapshot-state.ts @@ -69,13 +69,13 @@ export function buildSnapshotState( } /** - * Scope resolves once per snapshot. Android resolves it inside its projection (the platform - * matcher implements the shared scope specification, `@agent-device/contracts/snapshot`), and the - * macOS helper scopes at capture; a second pass here would re-match inside an already-scoped tree - * and hand the two layers different no-match semantics (#1832 C2). + * Scope resolves once per snapshot. Android and XCTest resolve it inside their projection (the + * platform matchers implement the shared scope specification, `@agent-device/contracts/snapshot`), + * and the macOS helper scopes at capture; a second pass here would re-match inside an already-scoped + * tree and hand the two layers different no-match semantics (#1832 C2). */ function backendScopesAfterWire(backend: SnapshotBackend | undefined): boolean { - return backend !== 'macos-helper' && backend !== 'android'; + return backend !== 'macos-helper' && backend !== 'android' && backend !== 'xctest'; } function shouldPresentIosInteractiveSnapshot( diff --git a/src/platforms/apple/interactor.ts b/src/platforms/apple/interactor.ts index 457af6e16a..060e67a026 100644 --- a/src/platforms/apple/interactor.ts +++ b/src/platforms/apple/interactor.ts @@ -203,7 +203,9 @@ function acceptsEmptyScopedSnapshot( quality: SnapshotQualityVerdict | undefined, ): boolean { return ( - normalizeSnapshotScope(options?.scope) !== null && quality?.state !== 'sparse' && !!quality + normalizeSnapshotScope(options?.scope) !== null && + quality !== undefined && + quality.state !== 'sparse' ); } diff --git a/src/snapshot/snapshot-desktop-surface.test.ts b/src/snapshot/snapshot-desktop-surface.test.ts index fb4050b90f..31377fc9d8 100644 --- a/src/snapshot/snapshot-desktop-surface.test.ts +++ b/src/snapshot/snapshot-desktop-surface.test.ts @@ -106,7 +106,13 @@ test('scopeSnapshotNodes agrees with every golden scope-policy table case', () = ) as Array<{ name: string; scope: string; - nodes: Array<{ depth: number; label?: string; value?: string; identifier?: string }>; + nodes: Array<{ + depth: number; + label?: string; + value?: string; + identifier?: string; + presented?: boolean; + }>; expectedSubtreeIndexes: number[]; }>; expect(cases.length).toBeGreaterThan(0); @@ -118,7 +124,9 @@ test('scopeSnapshotNodes agrees with every golden scope-policy table case', () = parents[node.depth] = index; return { ...node, index, parentIndex, rect: { x: index, y: 0, width: 1, height: 1 } }; }); - const scoped = scopeSnapshotNodes(nodes, fixture.scope); + const scoped = scopeSnapshotNodes(nodes, fixture.scope, (range) => + nodes.slice(range.start, range.end).some((node) => node.presented !== false), + ); expect( scoped.map((node) => node.rect?.x), fixture.name, diff --git a/src/snapshot/snapshot-desktop-surface.ts b/src/snapshot/snapshot-desktop-surface.ts index 521d80b5f2..756f544e3d 100644 --- a/src/snapshot/snapshot-desktop-surface.ts +++ b/src/snapshot/snapshot-desktop-surface.ts @@ -72,17 +72,27 @@ function shapeDesktopSurfaceSnapshot( options: Pick, ): SnapshotResult { let nodes = data.nodes ?? []; - if (options.scope) nodes = scopeSnapshotNodes(nodes, options.scope); + if (options.scope) { + nodes = scopeSnapshotNodes(nodes, options.scope, (range) => + options.interactiveOnly + ? nodes.slice(range.start, range.end).some(isInteractiveSnapshotNode) + : true, + ); + } if (options.interactiveOnly) nodes = filterInteractiveSnapshotNodes(nodes); if (typeof options.depth === 'number') nodes = filterSnapshotNodesByDepth(nodes, options.depth); return { ...data, nodes }; } /** The shared scope specification applied post-wire (`@agent-device/contracts/snapshot`). */ -export function scopeSnapshotNodes(nodes: RawSnapshotNode[], scope: string): RawSnapshotNode[] { +export function scopeSnapshotNodes( + nodes: RawSnapshotNode[], + scope: string, + subtreeContributes?: (range: { start: number; end: number }) => boolean, +): RawSnapshotNode[] { const normalizedScope = normalizeSnapshotScope(scope); if (!normalizedScope) return reindexSnapshotNodes(nodes); - const range = findSnapshotScopeRange(nodes, normalizedScope); + const range = findSnapshotScopeRange(nodes, normalizedScope, subtreeContributes); if (!range) return []; const slice = nodes.slice(range.start, range.end); return reindexSnapshotNodes(slice, slice[0]?.depth ?? 0); From 70ef0b6d53a202fee90bbb6b6b58a9df4ad69b91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 20 Aug 2026 08:43:52 +0200 Subject: [PATCH 09/10] test(snapshot): pin post-wire scope owner --- src/daemon/__tests__/snapshot-state.test.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/daemon/__tests__/snapshot-state.test.ts b/src/daemon/__tests__/snapshot-state.test.ts index ee62cdf64d..b4ca5fb969 100644 --- a/src/daemon/__tests__/snapshot-state.test.ts +++ b/src/daemon/__tests__/snapshot-state.test.ts @@ -378,16 +378,13 @@ test('buildSnapshotState leaves raw snapshot hittability untouched', () => { ).toBeUndefined(); }); -test('buildSnapshotState returns empty nodes when scoped snapshot has no label match', () => { +test('buildSnapshotState returns empty nodes when a post-wire scoped snapshot has no match', () => { const nodes = [ { index: 0, depth: 0, type: 'Window', label: 'Root' }, { index: 1, depth: 1, type: 'Button', label: 'Search' }, ]; - const state = buildSnapshotState( - { nodes, backend: 'xctest' }, - { snapshotScope: 'zzzz-no-match-token' }, - ); + const state = buildSnapshotState({ nodes }, { snapshotScope: 'zzzz-no-match-token' }); expect(state.nodes).toEqual([]); }); From 85ac56681b51c0f1a71a7a090e5e879c440f03ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 20 Aug 2026 08:56:06 +0200 Subject: [PATCH 10/10] test(snapshot): retain find test shrink --- src/__tests__/test-file-size-ratchet.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__tests__/test-file-size-ratchet.test.ts b/src/__tests__/test-file-size-ratchet.test.ts index 44ec4dfd68..1f3a440e61 100644 --- a/src/__tests__/test-file-size-ratchet.test.ts +++ b/src/__tests__/test-file-size-ratchet.test.ts @@ -48,7 +48,7 @@ const PINNED_TEST_FILE_LINES: Readonly> = Object.freeze({ 'src/platforms/apple/core/__tests__/runner-command-retry.test.ts': 1327, 'src/__tests__/cli-client-commands.test.ts': 1317, 'src/__tests__/cli-config.test.ts': 1282, - 'src/daemon/handlers/__tests__/find.test.ts': 1237, + 'src/daemon/handlers/__tests__/find.test.ts': 1223, 'src/platforms/apple/core/__tests__/perf.test.ts': 1222, 'src/mcp/__tests__/command-tools.test.ts': 1218, 'src/daemon/handlers/__tests__/session-replay-divergence.test.ts': 1215,