From 142496275c016874442dfa65280f63d9e4ccca04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 11:24:56 +0100 Subject: [PATCH 01/22] refactor(ios): bind surface coordinates to their measured geometry --- .../RunnerRequestPanel.swift | 5 +- .../RunnerScrollViewportPolicy.swift | 31 ++-- .../RunnerTests+Alert.swift | 2 +- .../RunnerTests+CommandExecution.swift | 2 +- .../RunnerTests+Interaction.swift | 10 +- .../RunnerTests+Navigation.swift | 2 +- .../RunnerTests+ScrollDragExecution.swift | 4 +- .../RunnerTests+Snapshot.swift | 2 +- ...RunnerTests+SynthesizedGesturePolicy.swift | 22 +-- .../RunnerTests+SynthesizedInteraction.swift | 134 ++++++-------- .../RunnerTests+TypeExecution.swift | 2 +- .../UnitTests/RunnerRequestPanelTests.swift | 163 +++++++++++++----- ...unnerTests+ScrollViewportPolicyTests.swift | 3 +- ...rTests+SynthesizedGesturePolicyTests.swift | 3 +- .../RunnerTests+XCTestEventBridgeTests.swift | 4 +- .../SnapshotCoordinateSpace.swift | 60 +++++++ 16 files changed, 265 insertions(+), 184 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index fa17a6bd0b..b712353a15 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -1,4 +1,5 @@ import XCTest +import AgentDeviceSnapshotPresentation // The lit panel of the command in flight (ADR 0025), main-owned and written behind main work a // previous command left queued, so that work still reads its own command's panel. @@ -11,7 +12,7 @@ struct LitPanelSurface { let key: LitPanelBookingKey let displayID: UInt let window: XCUIElement - let frame: CGRect + let geometry: SurfaceGeometry } /// The command's lit panel and what it booked on it. Read and written only on the main thread. @@ -79,7 +80,7 @@ extension RunnerTests { guard failure == .none, let element = window as? XCUIElement else { return RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure) } - litPanelState.surface = LitPanelSurface(key: key, displayID: litPanel.displayID, window: element, frame: frame) + litPanelState.surface = LitPanelSurface(key: key, displayID: litPanel.displayID, window: element, geometry: surfaceGeometry(windowFrame: frame, app: app)) return nil } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift index 955f9389fd..5e6379afeb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift @@ -32,14 +32,7 @@ enum RunnerScrollKeyboardClip: Equatable { /** Where one directional scroll may place its swipe, once the keyboard has taken its share. */ enum RunnerScrollViewport { - /** - * The band to plan the swipe inside, the frame to rotate its coordinates against, and the keyboard - * top when the band was clipped for one. The two frames are separate on purpose: a clip shortens - * only the band, while `CoordinateSpaceRotation.native(point:)` derives a `landscapeRight` native x from the - * frame's HEIGHT, so rotating inside the band moves the dispatched path sideways off the planned - * one. - */ - case swipe(planFrame: CGRect, coordinateFrame: CGRect, keyboardMinY: Double?) + case swipe(planFrame: CGRect, keyboardMinY: Double?) /** Nothing to swipe. The caller answers `occlusionRunnerCode` and performs no gesture. */ case occluded(keyboardMinY: Double, visibleHeight: Double) } @@ -49,7 +42,6 @@ enum RunnerScrollViewport { struct ScrollGestureDispatch { let plan: RunnerScrollGesturePlan let planFrame: CGRect - let coordinateFrame: CGRect let keyboardMinY: Double? } @@ -91,7 +83,7 @@ extension RunnerScrollViewport { switch self { case .occluded(let keyboardMinY, let visibleHeight): return .occluded(keyboardMinY: keyboardMinY, visibleHeight: visibleHeight) - case .swipe(let planFrame, let coordinateFrame, let keyboardMinY): + case .swipe(let planFrame, let keyboardMinY): guard planFrame.width > 0, planFrame.height > 0 else { return .unusableFrame } @@ -108,7 +100,6 @@ extension RunnerScrollViewport { ScrollGestureDispatch( plan: plan, planFrame: planFrame, - coordinateFrame: coordinateFrame, keyboardMinY: keyboardMinY ) ) @@ -168,9 +159,9 @@ enum ScrollViewportPolicy { static func frames(referenceFrame: CGRect, clip: RunnerScrollKeyboardClip) -> RunnerScrollViewport { switch clip { case .unobstructed: - return .swipe(planFrame: referenceFrame, coordinateFrame: referenceFrame, keyboardMinY: nil) + return .swipe(planFrame: referenceFrame, keyboardMinY: nil) case .avoided(let frame, let keyboardMinY): - return .swipe(planFrame: frame, coordinateFrame: referenceFrame, keyboardMinY: keyboardMinY) + return .swipe(planFrame: frame, keyboardMinY: keyboardMinY) case .occluded(let keyboardMinY, let visibleHeight): return .occluded(keyboardMinY: keyboardMinY, visibleHeight: visibleHeight) } @@ -195,17 +186,17 @@ extension RunnerTests { // Every scroll reports its decision, including the two ways it avoids reading the keyboard at // all: unavailable accessibility health, and a probe that finds no keyboard. guard context.allowsKeyboardProbe else { - logScrollViewport(decision: "probeSkipped", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context) - return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: .unobstructed) + logScrollViewport(decision: "probeSkipped", keyboardMinY: nil, swipeHeight: context.geometry.windowOnScreen.rect.height, context: context) + return ScrollViewportPolicy.frames(referenceFrame: context.geometry.windowOnScreen.rect, clip: .unobstructed) } guard let keyboardFrame = visibleKeyboardFrame(app: app) else { - logScrollViewport(decision: "noKeyboard", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context) - return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: .unobstructed) + logScrollViewport(decision: "noKeyboard", keyboardMinY: nil, swipeHeight: context.geometry.windowOnScreen.rect.height, context: context) + return ScrollViewportPolicy.frames(referenceFrame: context.geometry.windowOnScreen.rect, clip: .unobstructed) } - let clip = ScrollViewportPolicy.clip(viewport: context.referenceFrame, keyboard: keyboardFrame) + let clip = ScrollViewportPolicy.clip(viewport: context.geometry.windowOnScreen.rect, keyboard: keyboardFrame) switch clip { case .unobstructed: - logScrollViewport(decision: "unobstructed", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context) + logScrollViewport(decision: "unobstructed", keyboardMinY: nil, swipeHeight: context.geometry.windowOnScreen.rect.height, context: context) case .avoided(let frame, let keyboardMinY): logScrollViewport( decision: "avoided", @@ -221,7 +212,7 @@ extension RunnerTests { context: context ) } - return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: clip) + return ScrollViewportPolicy.frames(referenceFrame: context.geometry.windowOnScreen.rect, clip: clip) #else let fallbackFrame = try resolvedTouchReferenceFrame(app: app, appFrame: app.frame) return ScrollViewportPolicy.frames(referenceFrame: fallbackFrame, clip: .unobstructed) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index decf0f6b05..c144245138 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -211,7 +211,7 @@ extension RunnerTests { if let window = alert.window { // Never fall back to the owner app's first window: on a foldable it can be another panel. guard let context = SynthesizedCoordinateContext( - validating: safely("ALERT_WINDOW", CGRect.null) { window.frame }, + validating: surfaceGeometry(windowFrame: safely("ALERT_WINDOW", CGRect.null) { window.frame }, app: alert.ownerApp), resolvedWindow: window, accessibilityHealth: mainOwned.accessibilityHealth ) else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 6ef1f29369..80befb1a17 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -338,7 +338,7 @@ extension RunnerTests { y2: gesture.planFrame.minY + gesture.plan.y2, durationMs: defaults.durationMs, message: "scrolled", - context: scrollContext.withReferenceFrame(gesture.coordinateFrame), + context: scrollContext, releaseBehavior: command.scrollReleaseBehavior ) ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index dd469d9c6a..c37542deba 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -55,9 +55,6 @@ extension RunnerTests { let points: DragPoints let context: SynthesizedCoordinateContext - var referenceFrame: CGRect { - context.referenceFrame - } } struct SelectorElementMatch { @@ -240,7 +237,7 @@ extension RunnerTests { // lit-panel request uses its booked window, because app.frame names one // panel (ADR 0025). private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) throws -> Bool { - let windowFrame = try requestLitPanel == nil ? app.frame : requireRequestLitPanelSurface(app: app).frame + let windowFrame = try requestLitPanel == nil ? app.frame : requireRequestLitPanelSurface(app: app).geometry.windowOnScreen.rect let frame = element.frame if frame.isEmpty { return false @@ -282,7 +279,7 @@ extension RunnerTests { func resolveRunnerWindowElement(app: XCUIApplication) -> (window: XCUIElement, frame: CGRect)? { if requestLitPanel != nil { guard bookRequestLitPanelSurface(app: app) == nil, let booked = litPanelState.surface else { return nil } - return (booked.window, booked.frame) + return (booked.window, booked.geometry.windowOnScreen.rect) } for window in app.windows.allElementsBoundByIndex where window.exists { let frame = window.frame @@ -648,7 +645,8 @@ extension RunnerTests { } let booked = try requireRequestLitPanelSurface(app: app) let origin = booked.window.coordinate(withNormalizedOffset: .zero) - return origin.withOffset(CGVector(dx: x - booked.frame.minX, dy: y - booked.frame.minY)) + let point = booked.geometry.contentPoint(from: SurfacePoint(CGPoint(x: x, y: y))).point + return origin.withOffset(CGVector(dx: point.x, dy: point.y)) #else let resolved = try resolveRunnerWindow(app: app) let root = resolved.window ?? app diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index f35f0d16f6..fd85b8b781 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -228,7 +228,7 @@ extension RunnerTests { app: app, windowQueryRejected: windowQueryRejected ), let point = Self.topLeadingNavigationFallbackPoint( - in: context.referenceFrame, + in: context.geometry.windowOnScreen.rect, topSafeAreaInset: screenTopSafeAreaInset() ) else { return .unavailable diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift index 4b1ae239cc..87c8911dcf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift @@ -188,8 +188,8 @@ extension RunnerTests { y: plan.points.y, x2: plan.points.x2, y2: plan.points.y2, - referenceWidth: Double(plan.referenceFrame.width), - referenceHeight: Double(plan.referenceFrame.height) + referenceWidth: Double(plan.context.geometry.windowOnScreen.rect.width), + referenceHeight: Double(plan.context.geometry.windowOnScreen.rect.height) ) switch try performSynthesizedGesture(activeApp, kind: policyKind, context: plan.context, synthesize: { try synthesizedDragAt( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 82f6d386cc..503d6916df 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -299,7 +299,7 @@ extension RunnerTests { /// A lit-panel snapshot's root, raw or regular, is the booked box (ADR 0025). static func publishLitPanelRoot(_ root: inout RawAXNode, surface: LitPanelSurface?) { if let surface { - root.rect = SnapshotRect(surface.frame) + root.rect = SnapshotRect(surface.geometry.windowOnScreen.rect) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index df29a9509d..65917943d1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -38,22 +38,10 @@ enum SynthesizedGesturePolicyKind: String, Equatable, Hashable { } struct SynthesizedCoordinateContext { - let referenceFrame: CGRect - /// The window `referenceFrame` was measured on. Synthesized records route their display ID - /// through this same window so geometry and routing can never name different windows. Nil is the - /// sole-screen route: XCTest could not resolve the window, `referenceFrame` is the device's only - /// screen, and the record is routed to that screen. + let geometry: SurfaceGeometry let resolvedWindow: XCUIElement? let accessibilityHealth: RunnerAccessibilityHealth - func withReferenceFrame(_ frame: CGRect) -> SynthesizedCoordinateContext { - SynthesizedCoordinateContext( - referenceFrame: frame, - resolvedWindow: resolvedWindow, - accessibilityHealth: accessibilityHealth - ) - } - /// The keyboard probe is not free — `visibleKeyboardFrame` resolves `app.keyboards.firstMatch` /// with a live AX fetch — so `.unavailable` skips it: there the fetch is known not to answer, /// and a gesture that needs the frame fails open on a missing one. `.unknown` still probes; @@ -205,15 +193,15 @@ extension RunnerTests { } extension SynthesizedCoordinateContext { - /// Nil when `referenceFrame` is not a finite, non-empty box a point can be placed in. + /// Nil when the window has no finite, non-empty box. init?( - validating referenceFrame: CGRect, + validating geometry: SurfaceGeometry, resolvedWindow: XCUIElement?, accessibilityHealth: RunnerAccessibilityHealth ) { - guard SnapshotGeometry.isPositiveFinite(referenceFrame) else { return nil } + guard SnapshotGeometry.isPositiveFinite(geometry.windowOnScreen.rect) else { return nil } self.init( - referenceFrame: referenceFrame, + geometry: geometry, resolvedWindow: resolvedWindow, accessibilityHealth: accessibilityHealth ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 4b11d38245..329b025de5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -20,35 +20,24 @@ extension RunnerTests { hint: "Retry with finite x, y, x2, and y2 values." ) } - let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) guard let context = try context ?? synthesizedCoordinateContext(app: app) else { return .unsupported( message: "synthesized coordinate drag could not resolve an app window with a finite screen frame", hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." ) } - let frame = context.referenceFrame - guard - let rotationFrame = synthesizedRotationFrame(app: app, fallback: frame, interfaceOrientation: orientation) - else { + guard let geometry = digitizerGeometry(app: app, surface: context.geometry) else { return Self.unmeasuredSoleDisplayOutcome(gesture: "drag") } - let start = CoordinateSpaceRotation.native( - point: CGPoint(x: x, y: y), - in: rotationFrame, - interfaceOrientation: orientation - ) - let end = CoordinateSpaceRotation.native( - point: CGPoint(x: x2, y: y2), - in: rotationFrame, - interfaceOrientation: orientation - ) + let frame = context.geometry.windowOnScreen.rect + let start = geometry.digitizerPoint(from: SurfacePoint(CGPoint(x: x, y: y))).point + let end = geometry.digitizerPoint(from: SurfacePoint(CGPoint(x: x2, y: y2))).point logSynthesizedDispatch( kind: "drag", start: start, end: end, referenceFrame: frame, - orientation: orientation + orientation: context.geometry.orientation ) let message = switch profile { case .controlledScroll: @@ -302,11 +291,12 @@ extension RunnerTests { y: Double, context: SynthesizedCoordinateContext ) -> TouchVisualizationFrame { - TouchVisualizationFrame( - x: Double(context.referenceFrame.minX) + x, - y: Double(context.referenceFrame.minY) + y, - referenceWidth: Double(context.referenceFrame.width), - referenceHeight: Double(context.referenceFrame.height) + let screen = context.geometry.screenPoint(from: SurfacePoint(CGPoint(x: x, y: y))).point + return TouchVisualizationFrame( + x: Double(screen.x), + y: Double(screen.y), + referenceWidth: Double(context.geometry.windowOnScreen.rect.width), + referenceHeight: Double(context.geometry.windowOnScreen.rect.height) ) } @@ -351,8 +341,8 @@ extension RunnerTests { } // The sole-screen route means the windows query just failed, so a keyboard probe would too. return context.resolvedWindow == nil - ? context.referenceFrame - : frameAvoidingKeyboard(app: app, frame: context.referenceFrame) + ? context.geometry.windowOnScreen.rect + : frameAvoidingKeyboard(app: app, frame: context.geometry.windowOnScreen.rect) #else return try resolvedTouchReferenceFrame(app: app, appFrame: app.frame) #endif @@ -451,6 +441,11 @@ extension RunnerTests { windowQueryRejected: Bool = false ) throws -> SynthesizedCoordinateContext? { #if os(iOS) + if requestLitPanel != nil { + guard bookRequestLitPanelSurface(app: app) == nil, let surface = litPanelState.surface else { return nil } + return SynthesizedCoordinateContext(geometry: surface.geometry, resolvedWindow: surface.window, + accessibilityHealth: mainOwned.accessibilityHealth) + } var health = mainOwned.accessibilityHealth let window: XCUIElement? let referenceFrame: CGRect @@ -477,7 +472,7 @@ extension RunnerTests { return nil } return SynthesizedCoordinateContext( - validating: referenceFrame, + validating: surfaceGeometry(windowFrame: referenceFrame, app: app), resolvedWindow: window, accessibilityHealth: health ) @@ -518,28 +513,16 @@ extension RunnerTests { return RunnerResolveSoleScreen(app, &resolvedScreen, &resolvedDisplayID) == nil } - /// The box a synthesized turn swaps side lengths about. The digitizer turns about its display, so a - /// placed iPad window's smaller box is not the basis (#3899). A named lit panel supplies its full - /// screen box independently of the booked window translation (ADR 0025). When the device's only screen - /// will not report its size, `fallback` is the basis only while the app provably fills that - /// screen; otherwise this is nil and the gesture refuses before it dispatches. Portrait and an - /// unknown orientation leave the point as it is, so they keep `fallback` without a read; landscape - /// swaps the box's sides and upside-down mirrors the point across them, so both read it. @MainActor - func synthesizedRotationFrame(app: XCUIApplication, fallback: CGRect, interfaceOrientation: Int) -> CGRect? { - if let panel = requestLitPanel { - return Self.soleScreenReferenceFrame( - portraitSize: CGSize(width: panel.widthPt, height: panel.heightPt), - interfaceOrientation: interfaceOrientation - ) ?? fallback - } - guard XCUIScreen.screens.count == 1 else { return fallback } - switch interfaceOrientation { + func digitizerGeometry(app: XCUIApplication, surface: SurfaceGeometry) -> SurfaceGeometry? { + if surface.panelNativeSize != nil { return surface } + guard XCUIScreen.screens.count == 1 else { return surface } + switch surface.orientation { case RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight, RunnerInterfaceOrientation.portraitUpsideDown: break default: - return fallback + return surface } guard let portraitSize = soleDisplayPortraitSize() else { let routeAdmits = soleScreenRouteAdmits(app) @@ -547,10 +530,16 @@ extension RunnerTests { "AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=rotation-basis basis=%@", routeAdmits ? "window" : "refused" ) - return routeAdmits ? fallback : nil + return routeAdmits ? surface : nil } - return Self.soleScreenReferenceFrame(portraitSize: portraitSize, interfaceOrientation: interfaceOrientation) - ?? fallback + return SurfaceGeometry(windowOnScreen: surface.windowOnScreen, + panelNativeSize: portraitSize, orientation: surface.orientation) + } + + func surfaceGeometry(windowFrame: CGRect, app: XCUIApplication) -> SurfaceGeometry { + SurfaceGeometry(windowOnScreen: SurfaceRect(windowFrame), + panelNativeSize: requestLitPanel.map { CGSize(width: $0.widthPt, height: $0.heightPt) }, + orientation: Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))) } private enum SynthesizedPointGesture: String { @@ -585,21 +574,11 @@ extension RunnerTests { gesture: SynthesizedPointGesture ) -> PointGestureResolution { if let context { - guard - let rotationFrame = synthesizedRotationFrame( - app: app, - fallback: context.referenceFrame, - interfaceOrientation: orientation - ) - else { + guard let geometry = digitizerGeometry(app: app, surface: context.geometry) else { return .refused(Self.unmeasuredSoleDisplayOutcome(gesture: gesture.rawValue)) } - let point = CoordinateSpaceRotation.native( - point: CGPoint(x: x, y: y), - in: rotationFrame, - interfaceOrientation: orientation - ) - return .target(point: point, resolvedWindow: context.resolvedWindow, referenceFrame: context.referenceFrame) + let point = geometry.digitizerPoint(from: SurfacePoint(CGPoint(x: x, y: y))).point + return .target(point: point, resolvedWindow: context.resolvedWindow, referenceFrame: context.geometry.windowOnScreen.rect) } if let unrotated = unrotatedSoleScreenPoint(app: app, x: x, y: y, interfaceOrientation: orientation) { return .target(point: unrotated, resolvedWindow: nil, referenceFrame: .null) @@ -659,17 +638,15 @@ extension RunnerTests { portraitSize size: CGSize, interfaceOrientation: Int ) -> CGRect? { - guard size.width > 0, size.height > 0 else { return nil } - switch interfaceOrientation { - case RunnerInterfaceOrientation.portrait, RunnerInterfaceOrientation.portraitUpsideDown: - return CGRect(origin: .zero, size: size) - case RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight: - return CGRect(x: 0, y: 0, width: size.height, height: size.width) - default: - return nil - } + SurfaceGeometry.panelFrame(nativeSize: size, orientation: interfaceOrientation) } +#if !os(iOS) + func surfaceGeometry(windowFrame: CGRect, app: XCUIApplication) -> SurfaceGeometry { + SurfaceGeometry(windowOnScreen: SurfaceRect(windowFrame), panelNativeSize: nil, orientation: 0) + } +#endif + func swipe(app: XCUIApplication, direction: String) throws -> DragVisualizationFrame? { if performTvRemoteSwipeIfAvailable(direction: direction) { let frame = try resolvedTouchReferenceFrame(app: app, appFrame: app.frame) @@ -814,27 +791,15 @@ extension RunnerTests { plan: RunnerGesturePlan ) -> RunnerInteractionOutcome { #if os(iOS) - let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - // The portable planner and validation use this exact viewport. Using app.frame here can - // diverge when XCTest unions transformed/off-screen descendants into the application frame. - let frame = CGRect( - x: plan.viewport.x, - y: plan.viewport.y, - width: plan.viewport.width, - height: plan.viewport.height - ) - guard - let rotationFrame = synthesizedRotationFrame(app: app, fallback: frame, interfaceOrientation: orientation) - else { + guard let context = synthesizedCoordinateContext(app: app) else { + return .unsupported(message: "gesture could not resolve a surface", hint: nil) + } + guard let geometry = digitizerGeometry(app: app, surface: context.geometry) else { return Self.unmeasuredSoleDisplayOutcome(gesture: "gesture") } let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in pointer.samples.map { sample in - let point = CoordinateSpaceRotation.native( - point: CGPoint(x: sample.point.x, y: sample.point.y), - in: rotationFrame, - interfaceOrientation: orientation - ) + let point = geometry.digitizerPoint(from: SurfacePoint(CGPoint(x: sample.point.x, y: sample.point.y))).point return [ "x": NSNumber(value: Double(point.x)), "y": NSNumber(value: Double(point.y)), @@ -842,10 +807,9 @@ extension RunnerTests { ] } } - let resolvedWindow = resolveRunnerWindowElement(app: app)?.window if let message = RunnerSynthesizedGesture.synthesizeGesture( withApplication: app, - resolvedWindow: resolvedWindow, + resolvedWindow: context.resolvedWindow, pointerSamples: pointerSamples ) { return .unsupported( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index 7299948cef..5f4d794b32 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -123,7 +123,7 @@ extension RunnerTests { let point = target.refreshPoint let frame: CGRect if let resolvedCoordinateContext { - frame = resolvedCoordinateContext.referenceFrame + frame = resolvedCoordinateContext.geometry.windowOnScreen.rect } else if point != nil { frame = activeApp.frame } else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 9582c348e9..f56c3d9441 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -10,6 +10,17 @@ private final class LitPanelBookingScreen: NSObject { init(_ displayID: UInt = 3) { self.displayID = displayID } } +@MainActor +private final class SurfaceSelectorQuery: NSObject { + let elements: [XCUIElement] + init(_ elements: [XCUIElement]) { self.elements = elements } + @objc(matchingPredicate:) + func matching(_ predicate: NSPredicate) -> XCUIElementQuery { + unsafeBitCast(SurfaceSelectorQuery(elements.filter { predicate.evaluate(with: $0) }), to: XCUIElementQuery.self) + } + @objc var allElementsBoundByIndex: [XCUIElement] { elements } +} + @MainActor private final class LitPanelBookingProbe: NSObject { static var originalValueForKey: IMP? @@ -18,6 +29,12 @@ private final class LitPanelBookingProbe: NSObject { static var maxWindowListReads: Int? static var window: XCUIElement? static var darkWindow: XCUIElement? + static var target: XCUIElement? + static var darkTarget: XCUIElement? + static var composedDarkWindow: DisplayWindowFixture? + static var synthesizedPoint: CGPoint? + static var synthesizedDisplay: UInt? + static var coordinate: XCUICoordinate? static var windowFrame = CGRect.zero static var orientation = RunnerInterfaceOrientation.landscapeLeft @@ -34,6 +51,7 @@ private final class LitPanelBookingProbe: NSObject { case "allElementsBoundByIndex": Self.windowListReads += 1 if let max = Self.maxWindowListReads, Self.windowListReads > max { return [] } + if let dark = Self.composedDarkWindow { return [dark, Self.window!] as [NSObject] } return [Self.window!, Self.darkWindow].compactMap { $0 } case "screen": return LitPanelBookingScreen(self === Self.darkWindow ? 1 : 3) case "exists": return true @@ -48,6 +66,30 @@ private final class LitPanelBookingProbe: NSObject { } } + @objc var exists: Bool { true } + @objc var isHittable: Bool { true } + @objc var elementType: UInt { XCUIElement.ElementType.button.rawValue } + @objc var label: String { "same-selector" } + @objc var frame: CGRect { + (self === Self.target || self === Self.darkTarget) + ? CGRect(x: 190, y: 140, width: 20, height: 20) : Self.windowFrame + } + @objc(descendantsMatchingType:) + func descendants(matching type: UInt) -> XCUIElementQuery { + let elements = self === Self.window ? [Self.target!] : [Self.darkTarget!, Self.target!] + return unsafeBitCast(SurfaceSelectorQuery(elements), to: XCUIElementQuery.self) + } + + @objc(synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:) + class func synthesizeTap(_ app: AnyObject, resolvedWindow: AnyObject?, x: Double, y: Double, + deadline: NSDate?, errorMessage: AutoreleasingUnsafeMutablePointer?) -> Int { + var display: UInt = 0 + if let resolvedWindow { _ = RunnerResolveWindowDisplayID(resolvedWindow, &display) } + synthesizedDisplay = display + synthesizedPoint = CGPoint(x: x, y: y) + return 0 + } + @objc(coordinateWithNormalizedOffset:) func coordinateWithNormalizedOffset(_ offset: CGVector) -> XCUICoordinate { Self.anchor = self @@ -71,48 +113,65 @@ private final class LitPanelBookingProbe: NSObject { extension RunnerTests { @MainActor func testLitPanelBookingAndCoordinateFallbackShareScreenPointsForPartialWindows() throws { - try withLitPanelBookingProbe { window in - for (frame, orientation, expected) in [ - ( - CGRect(x: 80, y: 0, width: 871, height: 669), RunnerInterfaceOrientation.landscapeLeft, - CGRect(x: 0, y: 0, width: 951, height: 669) - ), - ( - CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portrait, - CGRect(x: 0, y: 0, width: 669, height: 951) - ), - ( - CGRect(x: 80, y: 0, width: 180, height: 669), RunnerInterfaceOrientation.landscapeRight, - CGRect(x: 0, y: 0, width: 951, height: 669) - ), - ( - CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portraitUpsideDown, - CGRect(x: 0, y: 0, width: 669, height: 951) - ), - ] { - LitPanelBookingProbe.windowFrame = frame - LitPanelBookingProbe.orientation = orientation - LitPanelBookingProbe.tappedPoint = nil - setRequestLitPanel(from: try runnerCommandFixture( - #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# - )) - XCTAssertNil(bookRequestLitPanelSurface(app: app)) - let booked = try XCTUnwrap(litPanelState.surface) - XCTAssertTrue(booked.window === window) - XCTAssertEqual(booked.frame, frame) - XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: booked.frame, interfaceOrientation: orientation), expected) - if case .unsupported(let message, _) = try tapAt(app: app, x: 200, y: 150) { XCTFail(message) } - XCTAssertTrue(LitPanelBookingProbe.anchor === window) - XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) - } + try withLitPanelBookingProbe(composed: true) { window in + for (frame, orientation, expected, digitizer) in [ + ( + CGRect(x: 80, y: 0, width: 871, height: 669), RunnerInterfaceOrientation.landscapeLeft, + CGRect(x: 0, y: 0, width: 951, height: 669), CGPoint(x: 150, y: 751) + ), + ( + CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portrait, + CGRect(x: 0, y: 0, width: 669, height: 951), CGPoint(x: 200, y: 150) + ), + ( + CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.landscapeRight, + CGRect(x: 0, y: 0, width: 951, height: 669), CGPoint(x: 519, y: 200) + ), + ( + CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portraitUpsideDown, + CGRect(x: 0, y: 0, width: 669, height: 951), CGPoint(x: 469, y: 801) + ), + ] { + LitPanelBookingProbe.windowFrame = frame + LitPanelBookingProbe.orientation = orientation LitPanelBookingProbe.tappedPoint = nil setRequestLitPanel(from: try runnerCommandFixture( - #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# )) - XCTAssertThrowsError(try tapAt(app: app, x: 200, y: 150)) { - XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .offLitPanel) + XCTAssertNil(bookRequestLitPanelSurface(app: app)) + let booked = try XCTUnwrap(litPanelState.surface) + XCTAssertTrue(booked.window === window) + XCTAssertEqual(booked.geometry.windowOnScreen.rect, frame) + XCTAssertEqual(booked.geometry.digitizerBasis.rect, expected) + for synthesized in [true, false] { + LitPanelBookingProbe.synthesizedPoint = nil + LitPanelBookingProbe.synthesizedDisplay = nil + let command = try runnerCommandFixture( + #"{"command":"tap","selectorKey":"label","selectorValue":"same-selector","x":200,"y":150,"allowNonHittableCoordinateFallback":true,"synthesized":\#(synthesized),"litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + ) + let response = try executeOnMainPrepared(command: command, activeApp: app) + XCTAssertTrue(response.ok, response.error?.message ?? "") + if synthesized { + XCTAssertEqual(LitPanelBookingProbe.synthesizedDisplay, 3) + XCTAssertEqual(LitPanelBookingProbe.synthesizedPoint, digitizer) + } else { + var display: UInt = 0 + XCTAssertNil(RunnerResolveWindowDisplayID(try XCTUnwrap(LitPanelBookingProbe.anchor), &display)) + XCTAssertEqual(display, 3) + XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) + } } - XCTAssertNil(LitPanelBookingProbe.tappedPoint) + XCTAssertTrue(LitPanelBookingProbe.anchor === window) + XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) + } + LitPanelBookingProbe.tappedPoint = nil + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertThrowsError(try tapAt(app: app, x: 200, y: 150)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .offLitPanel) + } + XCTAssertNil(LitPanelBookingProbe.tappedPoint) } } @@ -283,12 +342,17 @@ extension RunnerTests { } @MainActor - private func withLitPanelBookingProbe(_ body: (XCUIElement) throws -> Void) throws { + private func withLitPanelBookingProbe(composed: Bool = false, _ body: (XCUIElement) throws -> Void) throws { app.launch() defer { app.terminate() } let query = app.windows let window = query.firstMatch LitPanelBookingProbe.window = window + if composed { + LitPanelBookingProbe.target = app.buttons.element(boundBy: 0) + LitPanelBookingProbe.darkTarget = app.buttons.element(boundBy: 1) + LitPanelBookingProbe.composedDarkWindow = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 466, height: 678), displayID: 1) + } LitPanelBookingProbe.coordinate = window.coordinate(withNormalizedOffset: .zero) let windowClass = try XCTUnwrap(object_getClass(window)) let coordinateClass = try XCTUnwrap(object_getClass(LitPanelBookingProbe.coordinate!)) @@ -306,19 +370,34 @@ extension RunnerTests { (class_getInstanceMethod(coordinateClass, #selector(XCUICoordinate.tap)), class_getInstanceMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.tap))), ] + let attributeSwaps: [(Method?, Method?)] = [ + (#selector(getter: XCUIElement.exists), #selector(getter: LitPanelBookingProbe.exists)), + (#selector(getter: XCUIElement.isHittable), #selector(getter: LitPanelBookingProbe.isHittable)), + (#selector(getter: XCUIElement.elementType), #selector(getter: LitPanelBookingProbe.elementType)), + (#selector(getter: XCUIElement.label), #selector(getter: LitPanelBookingProbe.label)), + (#selector(getter: XCUIElement.frame), #selector(getter: LitPanelBookingProbe.frame)), + (#selector(XCUIElement.descendants(matching:)), #selector(LitPanelBookingProbe.descendants(matching:))), + ].map { (class_getInstanceMethod(windowClass, $0.0), class_getInstanceMethod(LitPanelBookingProbe.self, $0.1)) } + let synthesisSwap = ( + class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:")), + class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.synthesizeTap(_:resolvedWindow:x:y:deadline:errorMessage:))) + ) var originals: [(Method, IMP)] = [] defer { for (method, original) in originals.reversed() { method_setImplementation(method, original) } LitPanelBookingProbe.darkWindow = nil LitPanelBookingProbe.window = nil LitPanelBookingProbe.coordinate = nil + LitPanelBookingProbe.target = nil + LitPanelBookingProbe.darkTarget = nil + LitPanelBookingProbe.composedDarkWindow = nil LitPanelBookingProbe.anchor = nil LitPanelBookingProbe.originalValueForKey = nil LitPanelBookingProbe.maxWindowListReads = nil litPanelState.panel = nil litPanelState.surface = nil } - for (method, stub) in swaps { + for (method, stub) in swaps + (composed ? attributeSwaps + [synthesisSwap] : []) { let method = try XCTUnwrap(method) let stub = try XCTUnwrap(stub) originals.append((method, method_setImplementation(method, method_getImplementation(stub)))) @@ -426,9 +505,9 @@ extension RunnerTests { )) XCTAssertNil(bookRequestLitPanelSurface(app: app)) let context = try XCTUnwrap(try synthesizedCoordinateContext(app: app)) - XCTAssertEqual(context.referenceFrame, frame, "the booked reference is the window's own frame") + XCTAssertEqual(context.geometry.windowOnScreen.rect, frame, "the booked reference is the window's own frame") XCTAssertEqual( - synthesizedRotationFrame(app: app, fallback: context.referenceFrame, interfaceOrientation: RunnerInterfaceOrientation.portrait), + SurfaceGeometry.panelFrame(nativeSize: context.geometry.panelNativeSize!, orientation: RunnerInterfaceOrientation.portrait), CGRect(x: 0, y: 0, width: frame.width, height: frame.height + 80), "the turn is about the whole lit panel, not the window" ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift index 208397cc95..6f6b363f03 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift @@ -103,7 +103,6 @@ extension RunnerTests { } XCTAssertEqual(gesture.planFrame, band) XCTAssertEqual(gesture.keyboardMinY, keyboardMinY) - XCTAssertEqual(gesture.coordinateFrame, viewport, "the rotation basis must survive the clip") XCTAssertLessThanOrEqual( max(gesture.plan.y1, gesture.plan.y2), keyboard.minY - ScrollViewportPolicy.accessoryAllowance, @@ -129,7 +128,7 @@ extension RunnerTests { let orientedStartY = gesture.planFrame.minY + gesture.plan.y1 let dispatchedFromViewport = CoordinateSpaceRotation.native( point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), - in: gesture.coordinateFrame, + in: viewport, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight ) let dispatchedFromBand = CoordinateSpaceRotation.native( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift index 63880cb5da..312b075aba 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift @@ -1,3 +1,4 @@ +import AgentDeviceSnapshotPresentation import XCTest #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) @@ -160,7 +161,7 @@ extension RunnerTests { accessibilityHealth: RunnerAccessibilityHealth ) -> SynthesizedCoordinateContext { SynthesizedCoordinateContext( - referenceFrame: CGRect(x: 0, y: 0, width: 390, height: 844), + geometry: SurfaceGeometry(windowOnScreen: SurfaceRect(CGRect(x: 0, y: 0, width: 390, height: 844)), panelNativeSize: nil, orientation: 1), resolvedWindow: app.windows.firstMatch, accessibilityHealth: accessibilityHealth ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index c689a677ce..4dbba175b8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -1,7 +1,7 @@ import XCTest #if AGENT_DEVICE_RUNNER_UNIT_TESTS -private final class DisplayScreenFixture: NSObject { +final class DisplayScreenFixture: NSObject { @objc let displayID: UInt init(_ displayID: UInt) { self.displayID = displayID } } @@ -9,7 +9,7 @@ private final class DisplayScreenFixture: NSObject { /// A window the runtime answers only once its query has resolved. `frame` is the read that /// resolves it: while the query is unresolved `screen` names main, which is what a fresh runner /// process that has not queried this app yet reports (#2728). -private final class DisplayWindowFixture: NSObject { +final class DisplayWindowFixture: NSObject { private let windowFrame: CGRect private let resolvedDisplayID: UInt private let present: Bool diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 637c242546..5b4bc96152 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -13,6 +13,66 @@ public enum RunnerInterfaceOrientation { public static let landscapeLeft = 4 } +public enum WindowContentSpace {} +public enum InterfaceScreenSpace {} +public enum NativeDigitizerSpace {} + +public struct SurfacePoint: Equatable { + public let point: CGPoint + public init(_ point: CGPoint) { self.point = point } +} + +public struct SurfaceRect: Equatable { + public let rect: CGRect + public init(_ rect: CGRect) { self.rect = rect } +} + +/// Geometry measured once for a window. Panel size is absent when the runtime cannot prove it. +public struct SurfaceGeometry { + public let windowOnScreen: SurfaceRect + public let panelNativeSize: CGSize? + public let orientation: Int + + public init(windowOnScreen: SurfaceRect, panelNativeSize: CGSize?, orientation: Int) { + self.windowOnScreen = windowOnScreen + self.panelNativeSize = panelNativeSize + self.orientation = orientation + } + + public var digitizerBasis: SurfaceRect { + SurfaceRect(panelNativeSize.flatMap { Self.panelFrame(nativeSize: $0, orientation: orientation) } + ?? windowOnScreen.rect) + } + + public func screenPoint(from content: SurfacePoint) -> SurfacePoint { + let frame = windowOnScreen.rect + let space = SnapshotGeometrySpace.windowContent(offset: CGSize(width: frame.minX, height: frame.minY)) + return SurfacePoint(space.orientedFrame(of: CGRect(origin: content.point, size: .zero)).origin) + } + + public func contentPoint(from screen: SurfacePoint) -> SurfacePoint { + let origin = screenPoint(from: SurfacePoint(.zero)).point + return SurfacePoint(CGPoint(x: screen.point.x - origin.x, y: screen.point.y - origin.y)) + } + + public func digitizerPoint(from screen: SurfacePoint) -> SurfacePoint { + SurfacePoint(CoordinateSpaceRotation.native( + point: screen.point, in: digitizerBasis.rect, interfaceOrientation: orientation)) + } + + public static func panelFrame(nativeSize size: CGSize, orientation: Int) -> CGRect? { + guard size.width > 0, size.height > 0 else { return nil } + switch orientation { + case RunnerInterfaceOrientation.portrait, RunnerInterfaceOrientation.portraitUpsideDown: + return CGRect(origin: .zero, size: size) + case RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight: + return CGRect(x: 0, y: 0, width: size.height, height: size.width) + default: + return nil + } + } +} + public enum CoordinateSpaceRotation { // Dispatch turns a point's axes and never moves its origin: the digitizer's origin is the // display's own, where a placed window's screen points are already measured from (#3899). From c5f8291364f30d13fc57c1d27f0e7c269d555df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 11:26:05 +0100 Subject: [PATCH 02/22] test(ios): preserve dark-window facts in the composed surface harness --- .../UnitTests/RunnerRequestPanelTests.swift | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index f56c3d9441..762220da22 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -46,6 +46,13 @@ private final class LitPanelBookingProbe: NSObject { nonisolated override func value(forKey key: String) -> Any? { MainActor.assumeIsolated { + let original = unsafeBitCast( + Self.originalValueForKey!, + to: (@convention(c) (AnyObject, Selector, NSString) -> AnyObject?).self + ) + if self is DisplayWindowFixture { + return original(self, #selector(NSObject.value(forKey:)), key as NSString) + } switch key { case "windows": return Self.query case "allElementsBoundByIndex": @@ -58,10 +65,6 @@ private final class LitPanelBookingProbe: NSObject { case "frame": return NSValue(cgRect: Self.windowFrame) default: break } - let original = unsafeBitCast( - Self.originalValueForKey!, - to: (@convention(c) (AnyObject, Selector, NSString) -> AnyObject?).self - ) return original(self, #selector(NSObject.value(forKey:)), key as NSString) } } From b0a5db7d6a23a197face31ef657885db20e25047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 11:39:48 +0100 Subject: [PATCH 03/22] test(ios): keep bridge window facts independent of XCTest swizzles --- .../UnitTests/RunnerRequestPanelTests.swift | 3 --- .../UnitTests/RunnerTests+XCTestEventBridgeTests.swift | 9 +++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 762220da22..6a49b6d49b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -50,9 +50,6 @@ private final class LitPanelBookingProbe: NSObject { Self.originalValueForKey!, to: (@convention(c) (AnyObject, Selector, NSString) -> AnyObject?).self ) - if self is DisplayWindowFixture { - return original(self, #selector(NSObject.value(forKey:)), key as NSString) - } switch key { case "windows": return Self.query case "allElementsBoundByIndex": diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index 4dbba175b8..0c859ee3b6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -28,6 +28,15 @@ final class DisplayWindowFixture: NSObject { // Handed back as one object so a test can prove the resolver captures the screen the window // reported rather than a fresh value for the same key. Unresolved, it names main. @objc var screen: NSObject { resolved ? windowScreen : DisplayScreenFixture(1) } + + override func value(forKey key: String) -> Any? { + switch key { + case "exists": return exists + case "frame": return NSValue(cgRect: frame) + case "screen": return screen + default: return super.value(forKey: key) + } + } } private final class DisplayWindowsFixture: NSObject { From 152bb6251f7bac3d2806da6351f1678a352a094a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 11:59:28 +0100 Subject: [PATCH 04/22] style(ios): align the composed surface test after rebase --- .../UnitTests/RunnerRequestPanelTests.swift | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 6a49b6d49b..0b37ea8e38 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -114,64 +114,64 @@ extension RunnerTests { @MainActor func testLitPanelBookingAndCoordinateFallbackShareScreenPointsForPartialWindows() throws { try withLitPanelBookingProbe(composed: true) { window in - for (frame, orientation, expected, digitizer) in [ - ( - CGRect(x: 80, y: 0, width: 871, height: 669), RunnerInterfaceOrientation.landscapeLeft, - CGRect(x: 0, y: 0, width: 951, height: 669), CGPoint(x: 150, y: 751) - ), - ( - CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portrait, - CGRect(x: 0, y: 0, width: 669, height: 951), CGPoint(x: 200, y: 150) - ), - ( - CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.landscapeRight, - CGRect(x: 0, y: 0, width: 951, height: 669), CGPoint(x: 519, y: 200) - ), - ( - CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portraitUpsideDown, - CGRect(x: 0, y: 0, width: 669, height: 951), CGPoint(x: 469, y: 801) - ), - ] { - LitPanelBookingProbe.windowFrame = frame - LitPanelBookingProbe.orientation = orientation + for (frame, orientation, expected, digitizer) in [ + ( + CGRect(x: 80, y: 0, width: 871, height: 669), RunnerInterfaceOrientation.landscapeLeft, + CGRect(x: 0, y: 0, width: 951, height: 669), CGPoint(x: 150, y: 751) + ), + ( + CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portrait, + CGRect(x: 0, y: 0, width: 669, height: 951), CGPoint(x: 200, y: 150) + ), + ( + CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.landscapeRight, + CGRect(x: 0, y: 0, width: 951, height: 669), CGPoint(x: 519, y: 200) + ), + ( + CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portraitUpsideDown, + CGRect(x: 0, y: 0, width: 669, height: 951), CGPoint(x: 469, y: 801) + ), + ] { + LitPanelBookingProbe.windowFrame = frame + LitPanelBookingProbe.orientation = orientation + LitPanelBookingProbe.tappedPoint = nil + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertNil(bookRequestLitPanelSurface(app: app)) + let booked = try XCTUnwrap(litPanelState.surface) + XCTAssertTrue(booked.window === window) + XCTAssertEqual(booked.geometry.windowOnScreen.rect, frame) + XCTAssertEqual(booked.geometry.digitizerBasis.rect, expected) + for synthesized in [true, false] { + LitPanelBookingProbe.synthesizedPoint = nil + LitPanelBookingProbe.synthesizedDisplay = nil + let command = try runnerCommandFixture( + #"{"command":"tap","selectorKey":"label","selectorValue":"same-selector","x":200,"y":150,"allowNonHittableCoordinateFallback":true,"synthesized":\#(synthesized),"litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + ) + let response = try executeOnMainPrepared(command: command, activeApp: app) + XCTAssertTrue(response.ok, response.error?.message ?? "") + if synthesized { + XCTAssertEqual(LitPanelBookingProbe.synthesizedDisplay, 3) + XCTAssertEqual(LitPanelBookingProbe.synthesizedPoint, digitizer) + } else { + var display: UInt = 0 + XCTAssertNil(RunnerResolveWindowDisplayID(try XCTUnwrap(LitPanelBookingProbe.anchor), &display)) + XCTAssertEqual(display, 3) + XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) + } + } + XCTAssertTrue(LitPanelBookingProbe.anchor === window) + XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) + } LitPanelBookingProbe.tappedPoint = nil setRequestLitPanel(from: try runnerCommandFixture( - #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# )) - XCTAssertNil(bookRequestLitPanelSurface(app: app)) - let booked = try XCTUnwrap(litPanelState.surface) - XCTAssertTrue(booked.window === window) - XCTAssertEqual(booked.geometry.windowOnScreen.rect, frame) - XCTAssertEqual(booked.geometry.digitizerBasis.rect, expected) - for synthesized in [true, false] { - LitPanelBookingProbe.synthesizedPoint = nil - LitPanelBookingProbe.synthesizedDisplay = nil - let command = try runnerCommandFixture( - #"{"command":"tap","selectorKey":"label","selectorValue":"same-selector","x":200,"y":150,"allowNonHittableCoordinateFallback":true,"synthesized":\#(synthesized),"litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# - ) - let response = try executeOnMainPrepared(command: command, activeApp: app) - XCTAssertTrue(response.ok, response.error?.message ?? "") - if synthesized { - XCTAssertEqual(LitPanelBookingProbe.synthesizedDisplay, 3) - XCTAssertEqual(LitPanelBookingProbe.synthesizedPoint, digitizer) - } else { - var display: UInt = 0 - XCTAssertNil(RunnerResolveWindowDisplayID(try XCTUnwrap(LitPanelBookingProbe.anchor), &display)) - XCTAssertEqual(display, 3) - XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) - } + XCTAssertThrowsError(try tapAt(app: app, x: 200, y: 150)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .offLitPanel) } - XCTAssertTrue(LitPanelBookingProbe.anchor === window) - XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) - } - LitPanelBookingProbe.tappedPoint = nil - setRequestLitPanel(from: try runnerCommandFixture( - #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# - )) - XCTAssertThrowsError(try tapAt(app: app, x: 200, y: 150)) { - XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .offLitPanel) - } - XCTAssertNil(LitPanelBookingProbe.tappedPoint) + XCTAssertNil(LitPanelBookingProbe.tappedPoint) } } From afcb2d2ee7956dd1651e21043c06296aa221dffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 12:18:33 +0100 Subject: [PATCH 05/22] test(apple): box fixture rectangles on the host XCTest lane --- .../UnitTests/RunnerTests+XCTestEventBridgeTests.swift | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index 0c859ee3b6..6dd48b02ad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -32,7 +32,12 @@ final class DisplayWindowFixture: NSObject { override func value(forKey key: String) -> Any? { switch key { case "exists": return exists - case "frame": return NSValue(cgRect: frame) + case "frame": +#if os(macOS) + return NSValue(rect: frame) +#else + return NSValue(cgRect: frame) +#endif case "screen": return screen default: return super.value(forKey: key) } From bafe84506b6a3a202a54927ea2b06eda1a63b156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 12:41:07 +0100 Subject: [PATCH 06/22] refactor(apple): make immutable surface geometry sendable --- .../SnapshotCoordinateSpace.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 5b4bc96152..2d62dd87dc 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -17,18 +17,18 @@ public enum WindowContentSpace {} public enum InterfaceScreenSpace {} public enum NativeDigitizerSpace {} -public struct SurfacePoint: Equatable { +public struct SurfacePoint: Equatable, Sendable { public let point: CGPoint public init(_ point: CGPoint) { self.point = point } } -public struct SurfaceRect: Equatable { +public struct SurfaceRect: Equatable, Sendable { public let rect: CGRect public init(_ rect: CGRect) { self.rect = rect } } /// Geometry measured once for a window. Panel size is absent when the runtime cannot prove it. -public struct SurfaceGeometry { +public struct SurfaceGeometry: Sendable { public let windowOnScreen: SurfaceRect public let panelNativeSize: CGSize? public let orientation: Int From b0fdffdfe32fa6ba88b2c9ce065c670e661f8c26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 12:55:20 +0100 Subject: [PATCH 07/22] refactor(ios): preserve typed booking refusals after the parent update --- .../AgentDeviceRunnerUITests/RunnerTests+Interaction.swift | 2 +- .../RunnerTests+SynthesizedInteraction.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index c37542deba..ab24ad6f58 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -267,7 +267,7 @@ extension RunnerTests { func resolveRunnerWindow(app: XCUIApplication) throws -> (window: XCUIElement?, frame: CGRect) { if requestLitPanel != nil { let booked = try requireRequestLitPanelSurface(app: app) - return (booked.window, booked.frame) + return (booked.window, booked.geometry.windowOnScreen.rect) } guard let resolved = resolveRunnerWindowElement(app: app) else { return (nil, app.frame) } return (resolved.window, resolved.frame) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 329b025de5..8f9b916944 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -452,7 +452,7 @@ extension RunnerTests { if requestLitPanel != nil { let booked = try requireRequestLitPanelSurface(app: app) guard let context = SynthesizedCoordinateContext( - validating: booked.frame, resolvedWindow: booked.window, accessibilityHealth: health + validating: booked.geometry, resolvedWindow: booked.window, accessibilityHealth: health ) else { throw RunnerAppScreenCaptureFailure.unresolvedScreen } return context } From b9d69bfb8699497aaeaba20c0cad18717ae0aad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 12:57:42 +0100 Subject: [PATCH 08/22] refactor(ios): propagate prepared geometry refusals to sampled gestures --- .../RunnerTests+CommandExecution.swift | 2 +- .../RunnerTests+SynthesizedInteraction.swift | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 80befb1a17..feebfd72cb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -680,7 +680,7 @@ extension RunnerTests { ) case .sampled: let (timing, outcome) = try performGesture(activeApp, idleTimeout: false) { - sampledPlannedGesture(app: activeApp, plan: plan) + try sampledPlannedGesture(app: activeApp, plan: plan) } return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 8f9b916944..5ec96c4fe5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -441,11 +441,6 @@ extension RunnerTests { windowQueryRejected: Bool = false ) throws -> SynthesizedCoordinateContext? { #if os(iOS) - if requestLitPanel != nil { - guard bookRequestLitPanelSurface(app: app) == nil, let surface = litPanelState.surface else { return nil } - return SynthesizedCoordinateContext(geometry: surface.geometry, resolvedWindow: surface.window, - accessibilityHealth: mainOwned.accessibilityHealth) - } var health = mainOwned.accessibilityHealth let window: XCUIElement? let referenceFrame: CGRect @@ -789,9 +784,9 @@ extension RunnerTests { func sampledPlannedGesture( app: XCUIApplication, plan: RunnerGesturePlan - ) -> RunnerInteractionOutcome { + ) throws -> RunnerInteractionOutcome { #if os(iOS) - guard let context = synthesizedCoordinateContext(app: app) else { + guard let context = try synthesizedCoordinateContext(app: app) else { return .unsupported(message: "gesture could not resolve a surface", hint: nil) } guard let geometry = digitizerGeometry(app: app, surface: context.geometry) else { From 0cc584debbdee417acfa453a7a7716f14f35f1bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 13:30:29 +0100 Subject: [PATCH 09/22] test(ios): require preparation before composed surface lookup --- .../UnitTests/RunnerRequestPanelTests.swift | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 0b37ea8e38..1e7fdb3d48 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -34,6 +34,7 @@ private final class LitPanelBookingProbe: NSObject { static var composedDarkWindow: DisplayWindowFixture? static var synthesizedPoint: CGPoint? static var synthesizedDisplay: UInt? + static var beforeElementLookup: (() -> Void)? static var coordinate: XCUICoordinate? static var windowFrame = CGRect.zero @@ -76,6 +77,7 @@ private final class LitPanelBookingProbe: NSObject { } @objc(descendantsMatchingType:) func descendants(matching type: UInt) -> XCUIElementQuery { + Self.beforeElementLookup?() let elements = self === Self.window ? [Self.target!] : [Self.darkTarget!, Self.target!] return unsafeBitCast(SurfaceSelectorQuery(elements), to: XCUIElementQuery.self) } @@ -114,6 +116,9 @@ extension RunnerTests { @MainActor func testLitPanelBookingAndCoordinateFallbackShareScreenPointsForPartialWindows() throws { try withLitPanelBookingProbe(composed: true) { window in + LitPanelBookingProbe.beforeElementLookup = { + XCTAssertNotNil(self.litPanelState.surface, "Preparation must book before element lookup") + } for (frame, orientation, expected, digitizer) in [ ( CGRect(x: 80, y: 0, width: 871, height: 669), RunnerInterfaceOrientation.landscapeLeft, @@ -134,23 +139,22 @@ extension RunnerTests { ] { LitPanelBookingProbe.windowFrame = frame LitPanelBookingProbe.orientation = orientation - LitPanelBookingProbe.tappedPoint = nil - setRequestLitPanel(from: try runnerCommandFixture( - #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# - )) - XCTAssertNil(bookRequestLitPanelSurface(app: app)) - let booked = try XCTUnwrap(litPanelState.surface) - XCTAssertTrue(booked.window === window) - XCTAssertEqual(booked.geometry.windowOnScreen.rect, frame) - XCTAssertEqual(booked.geometry.digitizerBasis.rect, expected) for synthesized in [true, false] { + LitPanelBookingProbe.anchor = nil + LitPanelBookingProbe.tappedPoint = nil LitPanelBookingProbe.synthesizedPoint = nil LitPanelBookingProbe.synthesizedDisplay = nil let command = try runnerCommandFixture( #"{"command":"tap","selectorKey":"label","selectorValue":"same-selector","x":200,"y":150,"allowNonHittableCoordinateFallback":true,"synthesized":\#(synthesized),"litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# ) + let requestID = setRequestLitPanel(from: command) + defer { endRequestLitPanel(requestID) } let response = try executeOnMainPrepared(command: command, activeApp: app) XCTAssertTrue(response.ok, response.error?.message ?? "") + let booked = try XCTUnwrap(litPanelState.surface) + XCTAssertTrue(booked.window === window) + XCTAssertEqual(booked.geometry.windowOnScreen.rect, frame) + XCTAssertEqual(booked.geometry.digitizerBasis.rect, expected) if synthesized { XCTAssertEqual(LitPanelBookingProbe.synthesizedDisplay, 3) XCTAssertEqual(LitPanelBookingProbe.synthesizedPoint, digitizer) @@ -391,6 +395,7 @@ extension RunnerTests { LitPanelBookingProbe.target = nil LitPanelBookingProbe.darkTarget = nil LitPanelBookingProbe.composedDarkWindow = nil + LitPanelBookingProbe.beforeElementLookup = nil LitPanelBookingProbe.anchor = nil LitPanelBookingProbe.originalValueForKey = nil LitPanelBookingProbe.maxWindowListReads = nil From 9809174618d5730f9b9fb6e69b4f6527579fc585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 13:35:27 +0100 Subject: [PATCH 10/22] refactor(ios): preserve sole-screen admission in typed digitizer geometry --- .../RunnerTests+SynthesizedInteraction.swift | 7 +++-- ...nerTests+SynthesizedInteractionTests.swift | 31 ++++++++++++------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 5ec96c4fe5..61875734e0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -96,13 +96,14 @@ extension RunnerTests { hint: "Retry with finite x and y values." ) } - let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + let context = try context ?? synthesizedCoordinateContext(app: app) + let orientation = context?.geometry.orientation ?? Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) let resolution = pointGestureTarget( app: app, x: x, y: y, orientation: orientation, - context: try context ?? synthesizedCoordinateContext(app: app), + context: context, gesture: .tap ) guard case .target(let point, let resolvedWindow, let referenceFrame) = resolution else { @@ -170,7 +171,7 @@ extension RunnerTests { hint: "Retry with finite x and y values and a positive durationMs." ) } - let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + let orientation = context?.geometry.orientation ?? Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) let resolution = pointGestureTarget( app: app, x: x, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 7df11aa361..08656f270b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -118,7 +118,7 @@ extension RunnerTests { let resolved = try XCTUnwrap(resolveRunnerWindowElement(app: app)) context = try XCTUnwrap( SynthesizedCoordinateContext( - validating: windowFrame ?? resolved.frame, + validating: surfaceGeometry(windowFrame: windowFrame ?? resolved.frame, app: app), resolvedWindow: resolved.window, accessibilityHealth: .healthy ) @@ -309,6 +309,15 @@ extension RunnerTests { XCTAssertEqual(displayID, 0) } + @MainActor + private func unscopedDigitizerBasis( + app: XCUIApplication, window: CGRect, interfaceOrientation: Int + ) -> CGRect? { + digitizerGeometry(app: app, surface: SurfaceGeometry( + windowOnScreen: SurfaceRect(window), panelNativeSize: nil, orientation: interfaceOrientation + ))?.digitizerBasis.rect + } + /// Physical iOS can report no system application's AX frame while its display is measurable. @MainActor func testSoleDisplaySizeDoesNotRequireASystemApplicationAccessibilityFrame() throws { @@ -326,8 +335,8 @@ extension RunnerTests { defer { restoreAXRuntime() } mainOwned.soleDisplayPortraitSize = nil let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) - let measured = try XCTUnwrap(synthesizedRotationFrame( - app: app, fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + let measured = try XCTUnwrap(unscopedDigitizerBasis( + app: app, window: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight )) XCTAssertEqual(measured.origin, .zero) XCTAssertEqual(measured.width, max(frame.width, frame.height), accuracy: 0.5) @@ -361,7 +370,7 @@ extension RunnerTests { (RunnerInterfaceOrientation.landscapeRight, CGSize(width: screen.height, height: screen.width)), ] for (orientation, expected) in expectations { - let frame = try XCTUnwrap(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: orientation)) + let frame = try XCTUnwrap(unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: orientation)) XCTAssertEqual(frame.origin, .zero) XCTAssertEqual(frame.width, expected.width, accuracy: 0.5) XCTAssertEqual(frame.height, expected.height, accuracy: 0.5) @@ -377,21 +386,21 @@ extension RunnerTests { let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) let landscape = RunnerInterfaceOrientation.landscapeRight mainOwned.soleDisplayPortraitSize = nil - let measured = try XCTUnwrap(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape)) + let measured = try XCTUnwrap(unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: landscape)) XCTAssertNotEqual(measured, placedWindow) let nextCommand = try runnerCommandFixture(#"{"command":"uptime","commandId":"next-command"}"#) // Nothing between the install and the restore throws, so the restore always runs. let restoreDisplay = try simulateUnreadableDisplay() - XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape), measured) + XCTAssertEqual(unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: landscape), measured) _ = try? executeOnMainPrepared(command: nextCommand, activeApp: app) XCTAssertNotEqual( - synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape), + unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: landscape), measured, "the next command must read the size again rather than reuse the kept one" ) restoreDisplay() - XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape), measured) + XCTAssertEqual(unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: landscape), measured) } /// #4091: a portrait turn leaves the point as it is, so an unreadable display box neither refuses @@ -468,7 +477,7 @@ extension RunnerTests { let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) let windowFrame = try XCTUnwrap(context).referenceFrame XCTAssertEqual( - synthesizedRotationFrame(app: app, fallback: windowFrame, interfaceOrientation: orientation), + unscopedDigitizerBasis(app: app, window: windowFrame, interfaceOrientation: orientation), windowFrame ) let outcome = try synthesizedTapAt(app: app, x: 40, y: 40, context: context) @@ -492,10 +501,10 @@ extension RunnerTests { ) XCTAssertNil( - synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight) + unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight) ) XCTAssertEqual( - synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.portrait), + unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.portrait), placedWindow, "portrait skips the box read (only landscape and upside-down read it), so it must not refuse" ) From a7c547b064d718d4729025cee17862553f3b8794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 13:36:48 +0100 Subject: [PATCH 11/22] test(ios): carry typed geometry in the parent regression fixtures --- .../UnitTests/RunnerTests+SynthesizedInteractionTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 08656f270b..06d02c3a7e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -475,7 +475,7 @@ extension RunnerTests { ) let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - let windowFrame = try XCTUnwrap(context).referenceFrame + let windowFrame = try XCTUnwrap(context).geometry.windowOnScreen.rect XCTAssertEqual( unscopedDigitizerBasis(app: app, window: windowFrame, interfaceOrientation: orientation), windowFrame From d88b858b377e8cd6a2528c54e4855631b486468a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 14:09:12 +0100 Subject: [PATCH 12/22] fix(ios): stop stale sequences and share synthesis orientation --- .../RunnerSynthesizedGesture.h | 5 + .../RunnerSynthesizedGesture.m | 27 +- .../RunnerTests+SequenceExecution.swift | 15 +- .../RunnerTests+SynthesizedInteraction.swift | 5 + .../UnitTests/RunnerRequestPanelTests.swift | 232 ++++++++++++++++-- ...RunnerTests+ScrollDragExecutionTests.swift | 5 +- ...unnerTests+ScrollViewportPolicyTests.swift | 62 ----- .../RunnerTests+SynthesizedGestureTests.swift | 2 + src/commands/schema/cli-help-gestures.ts | 1 + website/docs/docs/commands.md | 2 + 10 files changed, 259 insertions(+), 97 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h index 640e9c0bcf..b9be2924f1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h @@ -16,8 +16,10 @@ typedef NS_ENUM(NSInteger, RunnerTapSynthesisStatus) { // `resolvedWindow` is the app window the caller already resolved for the gesture's reference // frame, so the record's display ID is read from that same window. Pass nil for the sole-screen // route: the app's tree could not name its window and the screen is proven instead. +// `interfaceOrientation` is the orientation used to convert the supplied points. + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation x:(double)x y:(double)y x2:(double)x2 @@ -26,6 +28,7 @@ typedef NS_ENUM(NSInteger, RunnerTapSynthesisStatus) { + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation x:(double)x y:(double)y x2:(double)x2 @@ -34,6 +37,7 @@ typedef NS_ENUM(NSInteger, RunnerTapSynthesisStatus) { + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation x:(double)x y:(double)y deadline:(NSDate * _Nullable)deadline @@ -43,6 +47,7 @@ typedef NS_ENUM(NSInteger, RunnerTapSynthesisStatus) { // starts contact; subsequent samples move it; all pointers lift at their final offset. + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation pointerSamples:(NSArray *> *> *)pointerSamples; // UIInterfaceOrientation of the app (1 portrait, 2 upsideDown, 3 landscapeRight, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m index fa61a820cc..e4a7e18ee2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m @@ -17,7 +17,6 @@ typedef struct { RunnerXCTestEventBridge core; SEL initRecordSelector; - SEL interfaceOrientationSelector; SEL initPathSelector; SEL moveSelector; SEL liftSelector; @@ -37,6 +36,7 @@ typedef id (*RunnerDragPointerPathFactory)( static NSString * _Nullable RunnerCreateEventRecord( id application, id _Nullable resolvedWindow, + NSInteger interfaceOrientation, NSString *recordName, RunnerGestureEventBridge *bridge, id *record @@ -60,6 +60,7 @@ static id RunnerControlledScrollPointerPath( static NSString * _Nullable RunnerTrySynthesizeDrag( id application, id _Nullable resolvedWindow, + NSInteger interfaceOrientation, CGPoint start, CGPoint end, double durationMs, @@ -69,6 +70,7 @@ static id RunnerControlledScrollPointerPath( static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( id application, id _Nullable resolvedWindow, + NSInteger interfaceOrientation, CGPoint point, NSDate * _Nullable deadline, NSString * _Nullable * _Nullable errorMessage @@ -87,6 +89,7 @@ @implementation RunnerSynthesizedGesture + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation x:(double)x y:(double)y x2:(double)x2 @@ -96,6 +99,7 @@ + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application return RunnerTrySynthesizeDrag( application, resolvedWindow, + interfaceOrientation, CGPointMake(x, y), CGPointMake(x2, y2), durationMs, @@ -109,6 +113,7 @@ + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation x:(double)x y:(double)y x2:(double)x2 @@ -118,6 +123,7 @@ + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)applicatio return RunnerTrySynthesizeDrag( application, resolvedWindow, + interfaceOrientation, CGPointMake(x, y), CGPointMake(x2, y2), durationMs, @@ -131,13 +137,14 @@ + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)applicatio + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation x:(double)x y:(double)y deadline:(NSDate * _Nullable)deadline errorMessage:(NSString * _Nullable * _Nullable)errorMessage { if (errorMessage != NULL) *errorMessage = nil; @try { - return RunnerTrySynthesizeTap(application, resolvedWindow, CGPointMake(x, y), deadline, errorMessage); + return RunnerTrySynthesizeTap(application, resolvedWindow, interfaceOrientation, CGPointMake(x, y), deadline, errorMessage); } @catch (NSException *exception) { if (errorMessage != NULL) { *errorMessage = RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed"); @@ -148,6 +155,7 @@ + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation pointerSamples:(NSArray *> *> *)pointerSamples { @try { RunnerGestureEventBridge bridge; @@ -155,6 +163,7 @@ + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application NSString *error = RunnerCreateEventRecord( application, resolvedWindow, + interfaceOrientation, @"agent-device-gesture-plan", &bridge, &record @@ -210,6 +219,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { static NSString * _Nullable RunnerTrySynthesizeDrag( id application, id _Nullable resolvedWindow, + NSInteger interfaceOrientation, CGPoint start, CGPoint end, double durationMs, @@ -218,7 +228,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { ) { RunnerGestureEventBridge bridge; id record = nil; - NSString *error = RunnerCreateEventRecord(application, resolvedWindow, recordName, &bridge, &record); + NSString *error = RunnerCreateEventRecord(application, resolvedWindow, interfaceOrientation, recordName, &bridge, &record); if (error != nil) return error; id path = pathFactory(&bridge, start, end, durationMs); @@ -233,6 +243,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( id application, id _Nullable resolvedWindow, + NSInteger interfaceOrientation, CGPoint point, NSDate * _Nullable deadline, NSString * _Nullable * _Nullable errorMessage @@ -242,6 +253,7 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( NSString *error = RunnerCreateEventRecord( application, resolvedWindow, + interfaceOrientation, @"agent-device-tap", &bridge, &record @@ -274,7 +286,6 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( if (missing != nil) return missing; SEL initRecordSelector = NSSelectorFromString(@"initWithName:displayID:interfaceOrientation:"); - SEL interfaceOrientationSelector = NSSelectorFromString(@"interfaceOrientation"); SEL initPathSelector = NSSelectorFromString(@"initForTouchAtPoint:offset:"); SEL moveSelector = NSSelectorFromString(@"moveToPoint:atOffset:"); SEL liftSelector = NSSelectorFromString(@"liftUpAtOffset:"); @@ -295,15 +306,10 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( core.pathClass, liftSelector, @"liftUpAtOffset:", RunnerGestureSynthesisSurface ); if (missing != nil) return missing; - missing = RunnerRequireApplicationSelector( - application, interfaceOrientationSelector, @"interfaceOrientation", RunnerGestureSynthesisSurface - ); - if (missing != nil) return missing; *bridge = (RunnerGestureEventBridge){ .core = core, .initRecordSelector = initRecordSelector, - .interfaceOrientationSelector = interfaceOrientationSelector, .initPathSelector = initPathSelector, .moveSelector = moveSelector, .liftSelector = liftSelector, @@ -314,6 +320,7 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( static NSString * _Nullable RunnerCreateEventRecord( id application, id _Nullable resolvedWindow, + NSInteger interfaceOrientation, NSString *recordName, RunnerGestureEventBridge *bridge, id *record @@ -329,8 +336,6 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( ? RunnerResolveSoleScreen(application, &soleScreen, &displayID) : RunnerResolveWindowDisplayID(resolvedWindow, &displayID); if (displayError != nil) return displayError; - NSInteger interfaceOrientation = - ((RunnerMsgSendInteger)objc_msgSend)(application, bridge->interfaceOrientationSelector); NSInteger targetProcessID = ((RunnerMsgSendInteger)objc_msgSend)(application, bridge->core.processIDSelector); if (targetProcessID <= 0) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 11b714522d..036aa58e2e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -1,4 +1,5 @@ import XCTest +import AgentDeviceSnapshotPresentation extension RunnerTests { // MARK: - Sequence command @@ -45,8 +46,20 @@ extension RunnerTests { ?? resolvedTouchVisualizationFrame(app: activeApp, x: firstStep.x!, y: firstStep.y!) : nil +#if os(iOS) + let orientation = synthesizedContext?.geometry.orientation ?? litPanelState.surface?.geometry.orientation + ?? Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: activeApp)) +#endif let execution = try assembleSequenceExecution(steps: steps) { _, step in - try performSequenceStep(step, activeApp: activeApp, synthesizedContext: synthesizedContext) +#if os(iOS) + if orientation != Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: activeApp)) { + return finishedSequenceStep(step, timing: measureGesture {}, outcome: .unsupported( + message: "sequence stopped because the screen orientation changed", + hint: "Inspect the screen before continuing with the remaining actions." + )) + } +#endif + return try performSequenceStep(step, activeApp: activeApp, synthesizedContext: synthesizedContext) } return sequenceResponse(execution: execution, touchFrame: firstFrame) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 61875734e0..29179f0bdf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -44,6 +44,7 @@ extension RunnerTests { RunnerSynthesizedGesture.synthesizeControlledScroll( withApplication: app, resolvedWindow: context.resolvedWindow, + interfaceOrientation: geometry.orientation, x: Double(start.x), y: Double(start.y), x2: Double(end.x), @@ -54,6 +55,7 @@ extension RunnerTests { RunnerSynthesizedGesture.synthesizeSwipe( withApplication: app, resolvedWindow: context.resolvedWindow, + interfaceOrientation: geometry.orientation, x: Double(start.x), y: Double(start.y), x2: Double(end.x), @@ -120,6 +122,7 @@ extension RunnerTests { let status = RunnerSynthesizedGesture.synthesizeTap( withApplication: app, resolvedWindow: resolvedWindow, + interfaceOrientation: orientation, x: Double(point.x), y: Double(point.y), deadline: deadline, @@ -200,6 +203,7 @@ extension RunnerTests { if let message = RunnerSynthesizedGesture.synthesizeGesture( withApplication: app, resolvedWindow: resolvedWindow, + interfaceOrientation: orientation, pointerSamples: [samples] ) { return .unsupported( @@ -806,6 +810,7 @@ extension RunnerTests { if let message = RunnerSynthesizedGesture.synthesizeGesture( withApplication: app, resolvedWindow: context.resolvedWindow, + interfaceOrientation: geometry.orientation, pointerSamples: pointerSamples ) { return .unsupported( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 1e7fdb3d48..91a30718f0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -19,6 +19,14 @@ private final class SurfaceSelectorQuery: NSObject { unsafeBitCast(SurfaceSelectorQuery(elements.filter { predicate.evaluate(with: $0) }), to: XCUIElementQuery.self) } @objc var allElementsBoundByIndex: [XCUIElement] { elements } + @objc var firstMatch: XCUIElement { elements[0] } +} + +private struct RecordedSurfaceDispatch { + let displayID: UInt + let start: CGPoint + let end: CGPoint? + let orientation: Int } @MainActor @@ -32,8 +40,17 @@ private final class LitPanelBookingProbe: NSObject { static var target: XCUIElement? static var darkTarget: XCUIElement? static var composedDarkWindow: DisplayWindowFixture? - static var synthesizedPoint: CGPoint? - static var synthesizedDisplay: UInt? + static var dispatches: [RecordedSurfaceDispatch] = [] + static var synthesizedPoint: CGPoint? { dispatches.last?.start } + static var synthesizedDisplay: UInt? { dispatches.last?.displayID } + static var application: XCUIApplication? + static var applicationFrame: CGRect? + static var keyboard: XCUIElement? + static var keyboardFrame: CGRect? + static var targetFrame = CGRect(x: 190, y: 140, width: 20, height: 20) + static var windowDisplayID: UInt = 3 + static var includeDarkSelector = true + static var onDispatch: (() -> Void)? static var beforeElementLookup: (() -> Void)? static var coordinate: XCUICoordinate? @@ -58,40 +75,70 @@ private final class LitPanelBookingProbe: NSObject { if let max = Self.maxWindowListReads, Self.windowListReads > max { return [] } if let dark = Self.composedDarkWindow { return [dark, Self.window!] as [NSObject] } return [Self.window!, Self.darkWindow].compactMap { $0 } - case "screen": return LitPanelBookingScreen(self === Self.darkWindow ? 1 : 3) + case "screen": return LitPanelBookingScreen(self === Self.darkWindow ? 1 : Self.windowDisplayID) case "exists": return true - case "frame": return NSValue(cgRect: Self.windowFrame) + case "frame": return NSValue(cgRect: Self.elementFrame(self)) default: break } return original(self, #selector(NSObject.value(forKey:)), key as NSString) } } - @objc var exists: Bool { true } + @objc var exists: Bool { self !== Self.keyboard || Self.keyboardFrame != nil } @objc var isHittable: Bool { true } @objc var elementType: UInt { XCUIElement.ElementType.button.rawValue } @objc var label: String { "same-selector" } - @objc var frame: CGRect { - (self === Self.target || self === Self.darkTarget) - ? CGRect(x: 190, y: 140, width: 20, height: 20) : Self.windowFrame + @objc var frame: CGRect { Self.elementFrame(self) } + private static func elementFrame(_ element: AnyObject) -> CGRect { + if element === keyboard { return keyboardFrame ?? .zero } + if element === application, let applicationFrame { return applicationFrame } + return element === target || element === darkTarget ? targetFrame : windowFrame + } + @objc var keyboards: XCUIElementQuery { + unsafeBitCast(SurfaceSelectorQuery([Self.keyboard!]), to: XCUIElementQuery.self) } + @objc var allElementsBoundByIndex: [XCUIElement] { [Self.window!] } @objc(descendantsMatchingType:) func descendants(matching type: UInt) -> XCUIElementQuery { Self.beforeElementLookup?() - let elements = self === Self.window ? [Self.target!] : [Self.darkTarget!, Self.target!] + let elements = self === Self.window || !Self.includeDarkSelector ? [Self.target!] : [Self.darkTarget!, Self.target!] return unsafeBitCast(SurfaceSelectorQuery(elements), to: XCUIElementQuery.self) } - @objc(synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:) - class func synthesizeTap(_ app: AnyObject, resolvedWindow: AnyObject?, x: Double, y: Double, + @objc(synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:x:y:deadline:errorMessage:) + class func synthesizeTap(_ app: AnyObject, resolvedWindow: AnyObject?, interfaceOrientation: Int, x: Double, y: Double, deadline: NSDate?, errorMessage: AutoreleasingUnsafeMutablePointer?) -> Int { - var display: UInt = 0 - if let resolvedWindow { _ = RunnerResolveWindowDisplayID(resolvedWindow, &display) } - synthesizedDisplay = display - synthesizedPoint = CGPoint(x: x, y: y) + record(window: resolvedWindow, orientation: interfaceOrientation, start: CGPoint(x: x, y: y)) return 0 } + @objc(synthesizeControlledScrollWithApplication:resolvedWindow:interfaceOrientation:x:y:x2:y2:durationMs:) + class func synthesizeDrag(_ app: AnyObject, resolvedWindow: AnyObject?, interfaceOrientation: Int, + x: Double, y: Double, x2: Double, y2: Double, durationMs: Double) -> String? { + record(window: resolvedWindow, orientation: interfaceOrientation, + start: CGPoint(x: x, y: y), end: CGPoint(x: x2, y: y2)) + return nil + } + + @objc(synthesizeGestureWithApplication:resolvedWindow:interfaceOrientation:pointerSamples:) + class func synthesizeGesture(_ app: AnyObject, resolvedWindow: AnyObject?, interfaceOrientation: Int, + pointerSamples: [[[String: NSNumber]]]) -> String? { + guard let first = pointerSamples.first?.first, let last = pointerSamples.first?.last else { + return "fixture pointer path is empty" + } + record(window: resolvedWindow, orientation: interfaceOrientation, + start: CGPoint(x: first["x"]!.doubleValue, y: first["y"]!.doubleValue), + end: CGPoint(x: last["x"]!.doubleValue, y: last["y"]!.doubleValue)) + return nil + } + + private static func record(window: AnyObject?, orientation: Int, start: CGPoint, end: CGPoint? = nil) { + var display: UInt = 0 + if let window { _ = RunnerResolveWindowDisplayID(window, &display) } + dispatches.append(RecordedSurfaceDispatch(displayID: display, start: start, end: end, orientation: orientation)) + onDispatch?() + } + @objc(coordinateWithNormalizedOffset:) func coordinateWithNormalizedOffset(_ offset: CGVector) -> XCUICoordinate { Self.anchor = self @@ -109,7 +156,10 @@ private final class LitPanelBookingProbe: NSObject { return Self.coordinate! } - @objc func tap() { Self.tappedPoint = Self.point } + @objc func tap() { + Self.tappedPoint = Self.point + Self.onDispatch?() + } } extension RunnerTests { @@ -142,8 +192,7 @@ extension RunnerTests { for synthesized in [true, false] { LitPanelBookingProbe.anchor = nil LitPanelBookingProbe.tappedPoint = nil - LitPanelBookingProbe.synthesizedPoint = nil - LitPanelBookingProbe.synthesizedDisplay = nil + LitPanelBookingProbe.dispatches.removeAll() let command = try runnerCommandFixture( #"{"command":"tap","selectorKey":"label","selectorValue":"same-selector","x":200,"y":150,"allowNonHittableCoordinateFallback":true,"synthesized":\#(synthesized),"litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# ) @@ -180,6 +229,125 @@ extension RunnerTests { } } + @MainActor + func testRepeatedPressStopsWhenTheFirstActionChangesOrientation() throws { + try withLitPanelBookingProbe(composed: true) { _ in + for synthesized in [false, true] { + for nextKind in ["tap", "longPress"] { + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 80, width: 669, height: 871) + LitPanelBookingProbe.orientation = RunnerInterfaceOrientation.portrait + LitPanelBookingProbe.dispatches.removeAll() + LitPanelBookingProbe.onDispatch = { + LitPanelBookingProbe.orientation = RunnerInterfaceOrientation.landscapeLeft + LitPanelBookingProbe.windowFrame = CGRect(x: 80, y: 0, width: 871, height: 669) + } + let command = try runnerCommandFixture( + #"{"command":"sequence","litPanel":{"displayId":3,"widthPt":669,"heightPt":951},"steps":[{"kind":"tap","x":200,"y":150,"synthesized":\#(synthesized)},{"kind":"\#(nextKind)","x":200,"y":150,"synthesized":true,"durationMs":100}]}"# + ) + let requestID = setRequestLitPanel(from: command) + defer { endRequestLitPanel(requestID) } + let response = try executeOnMainPrepared(command: command, activeApp: app) + XCTAssertTrue(response.ok, response.error?.message ?? "") + XCTAssertEqual(response.data?.completedSteps, 1) + XCTAssertEqual(response.data?.failedStepIndex, 1) + XCTAssertEqual(response.data?.sequenceResults?.last?.errorCode, "UNSUPPORTED_OPERATION") + XCTAssertEqual(LitPanelBookingProbe.dispatches.count, synthesized ? 1 : 0) + if synthesized { + let dispatch = try XCTUnwrap(LitPanelBookingProbe.dispatches.first) + XCTAssertEqual(dispatch.displayID, 3) + XCTAssertEqual(dispatch.start, CGPoint(x: 200, y: 150)) + XCTAssertEqual(dispatch.orientation, RunnerInterfaceOrientation.portrait) + } + } + } + } + } + + @MainActor + func testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis() throws { + let health = mainOwned.accessibilityHealth + defer { mainOwned.accessibilityHealth = health } + try withLitPanelBookingProbe(composed: true) { _ in + mainOwned.accessibilityHealth = .healthy + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 80, width: 669, height: 871) + LitPanelBookingProbe.orientation = RunnerInterfaceOrientation.landscapeRight + LitPanelBookingProbe.keyboardFrame = CGRect(x: 0, y: 600, width: 951, height: 300) + let command = try runnerCommandFixture( + #"{"command":"scroll","direction":"down","pixels":200,"durationMs":400,"litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + ) + let requestID = setRequestLitPanel(from: command) + defer { endRequestLitPanel(requestID) } + let response = try executeOnMainPrepared(command: command, activeApp: app) + XCTAssertTrue(response.ok, response.error?.message ?? "") + XCTAssertEqual(response.data?.referenceWidth, 669) + XCTAssertEqual(response.data?.referenceHeight, 508) + XCTAssertEqual(response.data?.keyboardAvoided, true) + XCTAssertEqual(response.data?.keyboardMinY, 600) + XCTAssertEqual(response.data?.x, 335) + XCTAssertEqual(response.data?.y, 434) + XCTAssertEqual(response.data?.x2, 335) + XCTAssertEqual(response.data?.y2, 234) + XCTAssertEqual(LitPanelBookingProbe.dispatches.count, 1) + let dispatch = try XCTUnwrap(LitPanelBookingProbe.dispatches.first) + XCTAssertEqual(dispatch.displayID, 3) + XCTAssertEqual(dispatch.orientation, RunnerInterfaceOrientation.landscapeRight) + XCTAssertEqual(dispatch.start, CGPoint(x: 235, y: 335)) + XCTAssertEqual(dispatch.end, CGPoint(x: 435, y: 335)) + } + } + + @MainActor + func testPlacedIPadWindowDispatchUsesTheMeasuredSoleDisplay() throws { + try XCTSkipUnless(XCUIScreen.screens.count == 1, "the fixture requires a single-display host") + let cached = mainOwned.soleDisplayPortraitSize + defer { mainOwned.soleDisplayPortraitSize = cached } + mainOwned.soleDisplayPortraitSize = nil + let selector = #selector(NSObject.value(forKey:)) + let inherited = try XCTUnwrap(class_getInstanceMethod(XCUIScreen.self, selector)) + class_addMethod(XCUIScreen.self, selector, method_getImplementation(inherited), method_getTypeEncoding(inherited)) + let target = try XCTUnwrap(class_getInstanceMethod(XCUIScreen.self, selector)) + let original = method_getImplementation(target) + typealias ReadValue = @convention(c) (AnyObject, Selector, NSString) -> AnyObject? + let readValue = unsafeBitCast(original, to: ReadValue.self) + var screenReads = 0 + let read: @convention(block) (AnyObject, NSString) -> AnyObject? = { screen, key in + if key == "bounds" { + screenReads += 1 + return NSValue(cgRect: CGRect(x: 0, y: 0, width: 2048, height: 2732)) + } + if key == "scale" { return NSNumber(value: 2) } + return readValue(screen, selector, key) + } + let replacement = imp_implementationWithBlock(read) + method_setImplementation(target, replacement) + defer { + method_setImplementation(target, original) + imp_removeBlock(replacement) + } + try withLitPanelBookingProbe(composed: true) { _ in + LitPanelBookingProbe.windowDisplayID = 1 + LitPanelBookingProbe.includeDarkSelector = false + LitPanelBookingProbe.windowFrame = CGRect(x: 224, y: 207, width: 386, height: 682) + LitPanelBookingProbe.applicationFrame = CGRect(x: 0, y: 0, width: 1366, height: 1024) + LitPanelBookingProbe.targetFrame = CGRect(x: 290, y: 390, width: 20, height: 20) + LitPanelBookingProbe.orientation = RunnerInterfaceOrientation.landscapeRight + let command = try runnerCommandFixture( + #"{"command":"tap","selectorKey":"label","selectorValue":"same-selector","synthesized":true}"# + ) + let requestID = setRequestLitPanel(from: command) + defer { endRequestLitPanel(requestID) } + let response = try executeOnMainPrepared(command: command, activeApp: app) + XCTAssertTrue(response.ok, response.error?.message ?? "") + XCTAssertGreaterThan(screenReads, 0) + XCTAssertEqual(mainOwned.soleDisplayPortraitSize, CGSize(width: 1024, height: 1366)) + XCTAssertEqual(LitPanelBookingProbe.dispatches.count, 1) + let dispatch = try XCTUnwrap(LitPanelBookingProbe.dispatches.first) + XCTAssertEqual(dispatch.displayID, 1) + XCTAssertEqual(dispatch.orientation, RunnerInterfaceOrientation.landscapeRight) + XCTAssertEqual(dispatch.start, CGPoint(x: 624, y: 300)) + } + } + @MainActor func testCollapsedTabCandidatesExcludeOverlappingDarkWindowsAndKeepTheCaptureScope() throws { try withLitPanelBookingProbe { window in @@ -353,6 +521,8 @@ extension RunnerTests { let window = query.firstMatch LitPanelBookingProbe.window = window if composed { + LitPanelBookingProbe.application = app + LitPanelBookingProbe.keyboard = app.keyboards.firstMatch LitPanelBookingProbe.target = app.buttons.element(boundBy: 0) LitPanelBookingProbe.darkTarget = app.buttons.element(boundBy: 1) LitPanelBookingProbe.composedDarkWindow = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 466, height: 678), displayID: 1) @@ -383,9 +553,20 @@ extension RunnerTests { (#selector(XCUIElement.descendants(matching:)), #selector(LitPanelBookingProbe.descendants(matching:))), ].map { (class_getInstanceMethod(windowClass, $0.0), class_getInstanceMethod(LitPanelBookingProbe.self, $0.1)) } let synthesisSwap = ( - class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:")), - class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.synthesizeTap(_:resolvedWindow:x:y:deadline:errorMessage:))) + class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:x:y:deadline:errorMessage:")), + class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.synthesizeTap(_:resolvedWindow:interfaceOrientation:x:y:deadline:errorMessage:))) ) + let dragStub = class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.synthesizeDrag(_:resolvedWindow:interfaceOrientation:x:y:x2:y2:durationMs:))) + let composedSwaps: [(Method?, Method?)] = [ + (class_getInstanceMethod(try XCTUnwrap(object_getClass(app)), NSSelectorFromString("keyboards")), + class_getInstanceMethod(LitPanelBookingProbe.self, #selector(getter: LitPanelBookingProbe.keyboards))), + (class_getInstanceMethod(try XCTUnwrap(object_getClass(query)), #selector(getter: XCUIElementQuery.allElementsBoundByIndex)), + class_getInstanceMethod(LitPanelBookingProbe.self, #selector(getter: LitPanelBookingProbe.allElementsBoundByIndex))), + (class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeControlledScrollWithApplication:resolvedWindow:interfaceOrientation:x:y:x2:y2:durationMs:")), dragStub), + (class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeSwipeWithApplication:resolvedWindow:interfaceOrientation:x:y:x2:y2:durationMs:")), dragStub), + (class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeGestureWithApplication:resolvedWindow:interfaceOrientation:pointerSamples:")), + class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.synthesizeGesture(_:resolvedWindow:interfaceOrientation:pointerSamples:)))) + ] var originals: [(Method, IMP)] = [] defer { for (method, original) in originals.reversed() { method_setImplementation(method, original) } @@ -396,13 +577,22 @@ extension RunnerTests { LitPanelBookingProbe.darkTarget = nil LitPanelBookingProbe.composedDarkWindow = nil LitPanelBookingProbe.beforeElementLookup = nil + LitPanelBookingProbe.application = nil + LitPanelBookingProbe.applicationFrame = nil + LitPanelBookingProbe.keyboard = nil + LitPanelBookingProbe.keyboardFrame = nil + LitPanelBookingProbe.targetFrame = CGRect(x: 190, y: 140, width: 20, height: 20) + LitPanelBookingProbe.windowDisplayID = 3 + LitPanelBookingProbe.includeDarkSelector = true + LitPanelBookingProbe.dispatches.removeAll() + LitPanelBookingProbe.onDispatch = nil LitPanelBookingProbe.anchor = nil LitPanelBookingProbe.originalValueForKey = nil LitPanelBookingProbe.maxWindowListReads = nil litPanelState.panel = nil litPanelState.surface = nil } - for (method, stub) in swaps + (composed ? attributeSwaps + [synthesisSwap] : []) { + for (method, stub) in swaps + (composed ? attributeSwaps + [synthesisSwap] + composedSwaps : []) { let method = try XCTUnwrap(method) let stub = try XCTUnwrap(stub) originals.append((method, method_setImplementation(method, method_getImplementation(stub)))) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift index 3fb8879a82..4a4085bd01 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift @@ -5,10 +5,11 @@ import AgentDeviceSnapshotPresentation import ObjectiveC.runtime private final class RunnerSynthesizedSwipeFailureStub: NSObject { - @objc(synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:) + @objc(synthesizeSwipeWithApplication:resolvedWindow:interfaceOrientation:x:y:x2:y2:durationMs:) class func synthesizeSwipe( application: XCUIApplication, resolvedWindow: Any?, + interfaceOrientation: Int, x: Double, y: Double, x2: Double, @@ -56,7 +57,7 @@ extension RunnerTests { @MainActor func testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails() throws { let selector = NSSelectorFromString( - "synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:" + "synthesizeSwipeWithApplication:resolvedWindow:interfaceOrientation:x:y:x2:y2:durationMs:" ) guard let synthesizedSwipeMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift index 6f6b363f03..b08b97d52b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift @@ -82,68 +82,6 @@ extension RunnerTests { XCTAssertEqual(constants.accessoryAllowance, ScrollViewportPolicy.accessoryAllowance) } - /// A clipped landscape band shortens the frame, and `CoordinateSpaceRotation.native(point:)` derives a - /// `landscapeRight` native x from the frame's HEIGHT. Rotating inside the band therefore moves the - /// dispatched path sideways by exactly what the keyboard took, off the lane the plan was built for, - /// so the plan band and the coordinate basis stay separate values through dispatch (#2500). - func testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis() throws { - let viewport = CGRect(x: 0, y: 0, width: 1210, height: 834) - let keyboard = CGRect(x: 0, y: 588, width: 1210, height: 246) - let clip = ScrollViewportPolicy.clip(viewport: viewport, keyboard: keyboard) - guard case .avoided(let band, let keyboardMinY) = clip else { - return XCTFail("expected a landscape keyboard to be avoided, got \(clip)") - } - XCTAssertEqual(band.height, 576) - - guard case .gesture(let gesture) = ScrollViewportPolicy.frames( - referenceFrame: viewport, - clip: clip - ).gestureDispatch(direction: .up, amount: nil, pixels: nil) else { - return XCTFail("expected a gesture inside the clipped band") - } - XCTAssertEqual(gesture.planFrame, band) - XCTAssertEqual(gesture.keyboardMinY, keyboardMinY) - XCTAssertLessThanOrEqual( - max(gesture.plan.y1, gesture.plan.y2), - keyboard.minY - ScrollViewportPolicy.accessoryAllowance, - "a landscape swipe must stay clear of the keys" - ) - - let reported = gesture.attachingEvidence( - to: Response( - ok: true, - data: DataPayload(referenceWidth: viewport.width, referenceHeight: viewport.height), - error: nil - ) - ) - XCTAssertEqual( - reported.data?.referenceHeight, - band.height, - "the payload names the band the plan ran inside, not the synthesis frame" - ) - XCTAssertEqual(reported.data?.referenceWidth, viewport.width) - XCTAssertEqual(reported.data?.keyboardMinY, keyboardMinY) - XCTAssertEqual(reported.data?.keyboardAvoided, true) - - let orientedStartY = gesture.planFrame.minY + gesture.plan.y1 - let dispatchedFromViewport = CoordinateSpaceRotation.native( - point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), - in: viewport, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ) - let dispatchedFromBand = CoordinateSpaceRotation.native( - point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), - in: gesture.planFrame, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ) - XCTAssertEqual( - dispatchedFromViewport.x - dispatchedFromBand.x, - viewport.height - band.height, - accuracy: 0.001, - "rotating inside the clipped band would shift native x by what the keyboard took" - ) - } - private func loadScrollViewportPolicyFixture() throws -> ScrollViewportPolicyFixture { let fixtureURL = URL(fileURLWithPath: #filePath) .deletingLastPathComponent() // UnitTests diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift index a50f261504..448d63dbd9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift @@ -54,6 +54,7 @@ extension RunnerTests { let expired = RunnerSynthesizedGesture.synthesizeTap( withApplication: TapApplicationFixture(), resolvedWindow: window, + interfaceOrientation: 1, x: 50, y: 50, deadline: deadline, @@ -69,6 +70,7 @@ extension RunnerTests { let control = RunnerSynthesizedGesture.synthesizeTap( withApplication: TapApplicationFixture(), resolvedWindow: TapWindowFixture {}, + interfaceOrientation: 1, x: 50, y: 50, deadline: .distantFuture, diff --git a/src/commands/schema/cli-help-gestures.ts b/src/commands/schema/cli-help-gestures.ts index acc7ee477c..d6965721a8 100644 --- a/src/commands/schema/cli-help-gestures.ts +++ b/src/commands/schema/cli-help-gestures.ts @@ -26,6 +26,7 @@ Shapes: For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. gesture tap touches 2 to 5 distinct points and holds them for --hold-ms (default 50, min 16); --count repeats the same contacts (cap 200) with --pause-ms between, the series must fit 60000ms counting about 500ms per repetition, and a repetition costs about 450ms on a physical iPhone. Use press for one finger. Platform quirks: + If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. Cancelled gesture and swipe requests return COMMAND_FAILED with details.reason=request_canceled and record no action. Repetition pauses are interruptible. iOS transform/pinch/rotate use private XCTest synthesis for a continuous two-finger pan/scale/rotation path; verify app metrics instead of assuming requested values map exactly to recognizer output. A gesture tap of three or more fingers can trigger iOS system gestures (undo/redo, the edit menu) and fast repeats can trigger double-tap recognizers, so verify app state; iOS touches the contacts down and lifts them together; Android touches them down up to 8ms apart and lifts them 8ms apart in reverse order. Android transform injects a geometric two-finger path; app recognizers may report non-exact pan/scale/rotation -- verify semantic app state or coarse per-component effects instead of exact numeric deltas unless the app exposes stable metrics. If Android needs exact values, prefer isolated gesture pan --pointer-count 2, gesture pinch, or gesture rotate over one combined transform: diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index ecce60609b..cc8b2a21bc 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -485,6 +485,8 @@ agent-device gesture tap 80 300 200 450 310 600 # three fingers at once agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two-finger tap ``` +If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. + A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. From 2bdc046fc6922c2858e48975557569575415972b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 14:15:36 +0100 Subject: [PATCH 13/22] fix(ios): keep gesture evidence in interface screen coordinates --- .../RunnerTests+CommandExecution.swift | 2 +- .../RunnerTests+SequenceExecution.swift | 2 +- .../RunnerTests+SynthesizedInteraction.swift | 10 +- .../UnitTests/RunnerRequestPanelTests.swift | 2 + .../RunnerTests+SynthesizedGestureTests.swift | 106 ++++++++++++------ 5 files changed, 78 insertions(+), 44 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index feebfd72cb..1fc524fbb1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -247,7 +247,7 @@ extension RunnerTests { try synthesizedLongPressAt(app: activeApp, x: x, y: y, durationMs: durationMs, context: context) }) { case .performed(let timing): - let touchFrame = context.map { touchVisualizationFrame(x: x, y: y, context: $0) } + let touchFrame = context.map { touchVisualizationFrame(point: SurfacePoint(CGPoint(x: x, y: y)), context: $0) } return gestureResponse(message: "long pressed", timing: timing, frame: .touch(touchFrame)) case .xctestFallback(let message, let hint): fallback = GestureFallback(strategy: "xctest-coordinate-long-press", message: message, hint: hint) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 036aa58e2e..f88aaa4dd8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -42,7 +42,7 @@ extension RunnerTests { // synthesized context already names the frame, so the app tree is not read for it again. let firstStep = steps[0] let firstFrame = try (firstStep.x != nil && firstStep.y != nil) - ? synthesizedContext.map { touchVisualizationFrame(x: firstStep.x!, y: firstStep.y!, context: $0) } + ? synthesizedContext.map { touchVisualizationFrame(point: SurfacePoint(CGPoint(x: firstStep.x!, y: firstStep.y!)), context: $0) } ?? resolvedTouchVisualizationFrame(app: activeApp, x: firstStep.x!, y: firstStep.y!) : nil diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 29179f0bdf..9d752c9d0b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -292,14 +292,12 @@ extension RunnerTests { /// The touch visualization frame of a synthesized gesture, read from its context instead of the /// app tree. func touchVisualizationFrame( - x: Double, - y: Double, + point: SurfacePoint, context: SynthesizedCoordinateContext ) -> TouchVisualizationFrame { - let screen = context.geometry.screenPoint(from: SurfacePoint(CGPoint(x: x, y: y))).point - return TouchVisualizationFrame( - x: Double(screen.x), - y: Double(screen.y), + TouchVisualizationFrame( + x: Double(point.point.x), + y: Double(point.point.y), referenceWidth: Double(context.geometry.windowOnScreen.rect.width), referenceHeight: Double(context.geometry.windowOnScreen.rect.height) ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 91a30718f0..f602839093 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -248,6 +248,8 @@ extension RunnerTests { defer { endRequestLitPanel(requestID) } let response = try executeOnMainPrepared(command: command, activeApp: app) XCTAssertTrue(response.ok, response.error?.message ?? "") + XCTAssertEqual(response.data?.x, 200) + XCTAssertEqual(response.data?.y, 150) XCTAssertEqual(response.data?.completedSteps, 1) XCTAssertEqual(response.data?.failedStepIndex, 1) XCTAssertEqual(response.data?.sequenceResults?.last?.errorCode, "UNSUPPORTED_OPERATION") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift index 448d63dbd9..61b62c157e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift @@ -24,15 +24,81 @@ private final class TapApplicationFixture: NSObject { private final class TapRecordSpy: NSObject { static var syntheses = 0 + static var recordOrientations: [Int] = [] + static var recordReadError: String? @objc(synthesizeWithError:) func synthesize(error: AutoreleasingUnsafeMutablePointer?) -> Bool { Self.syntheses += 1 + Self.recordReadError = RunnerObjCExceptionCatcher.catchException { + if let orientation = self.value(forKey: "interfaceOrientation") as? NSNumber { + Self.recordOrientations.append(orientation.intValue) + } + } return true } } extension RunnerTests { func testTapDeadlineExpiringDuringEventPreparationDoesNotSynthesize() throws { + try withTapRecordSpy { + let deadline = Date().addingTimeInterval(1) + var prepared = false + var preparationBeganBeforeDeadline = false + let window = TapWindowFixture { + preparationBeganBeforeDeadline = Date() < deadline + while Date() < deadline { Thread.sleep(forTimeInterval: 0.001) } + prepared = true + } + var message: NSString? + let expired = RunnerSynthesizedGesture.synthesizeTap( + withApplication: TapApplicationFixture(), + resolvedWindow: window, + interfaceOrientation: 1, + x: 50, + y: 50, + deadline: deadline, + errorMessage: &message + ) + + XCTAssertTrue(preparationBeganBeforeDeadline) + XCTAssertTrue(prepared) + XCTAssertEqual(expired, .deadlineExceeded) + XCTAssertNil(message) + XCTAssertEqual(TapRecordSpy.syntheses, 0) + + let control = RunnerSynthesizedGesture.synthesizeTap( + withApplication: TapApplicationFixture(), + resolvedWindow: TapWindowFixture {}, + interfaceOrientation: 1, + x: 50, + y: 50, + deadline: .distantFuture, + errorMessage: &message + ) + XCTAssertEqual(control, .succeeded, message as String? ?? "") + XCTAssertEqual(TapRecordSpy.syntheses, 1) + } + } + + func testSynthesizedRecordKeepsTheConvertedPointsOrientation() throws { + try withTapRecordSpy { + var message: NSString? + let result = RunnerSynthesizedGesture.synthesizeTap( + withApplication: TapApplicationFixture(), + resolvedWindow: TapWindowFixture {}, + interfaceOrientation: 4, + x: 50, + y: 50, + deadline: .distantFuture, + errorMessage: &message + ) + XCTAssertEqual(result, .succeeded, message as String? ?? "") + XCTAssertNil(TapRecordSpy.recordReadError) + XCTAssertEqual(TapRecordSpy.recordOrientations, [4]) + } + } + + private func withTapRecordSpy(_ body: () throws -> Void) throws { let recordClass = try XCTUnwrap(NSClassFromString("XCSynthesizedEventRecord")) let selector = NSSelectorFromString("synthesizeWithError:") let method = try XCTUnwrap(class_getInstanceMethod(recordClass, selector)) @@ -42,42 +108,10 @@ extension RunnerTests { defer { method_setImplementation(method, original) } TapRecordSpy.syntheses = 0 - let deadline = Date().addingTimeInterval(1) - var prepared = false - var preparationBeganBeforeDeadline = false - let window = TapWindowFixture { - preparationBeganBeforeDeadline = Date() < deadline - while Date() < deadline { Thread.sleep(forTimeInterval: 0.001) } - prepared = true - } - var message: NSString? - let expired = RunnerSynthesizedGesture.synthesizeTap( - withApplication: TapApplicationFixture(), - resolvedWindow: window, - interfaceOrientation: 1, - x: 50, - y: 50, - deadline: deadline, - errorMessage: &message - ) - - XCTAssertTrue(preparationBeganBeforeDeadline) - XCTAssertTrue(prepared) - XCTAssertEqual(expired, .deadlineExceeded) - XCTAssertNil(message) - XCTAssertEqual(TapRecordSpy.syntheses, 0) - - let control = RunnerSynthesizedGesture.synthesizeTap( - withApplication: TapApplicationFixture(), - resolvedWindow: TapWindowFixture {}, - interfaceOrientation: 1, - x: 50, - y: 50, - deadline: .distantFuture, - errorMessage: &message - ) - XCTAssertEqual(control, .succeeded, message as String? ?? "") - XCTAssertEqual(TapRecordSpy.syntheses, 1) + TapRecordSpy.recordOrientations = [] + TapRecordSpy.recordReadError = nil + try body() } + } #endif From f8eb1699e88280ffca50843be907591c107de7d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 14:18:14 +0100 Subject: [PATCH 14/22] style(ios): align the synthesis orientation interface --- .../AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h | 6 +++--- .../AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m | 6 +++--- .../UnitTests/RunnerTests+SynthesizedGestureTests.swift | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h index b9be2924f1..d19690b3e6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h @@ -28,7 +28,7 @@ typedef NS_ENUM(NSInteger, RunnerTapSynthesisStatus) { + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow - interfaceOrientation:(NSInteger)interfaceOrientation + interfaceOrientation:(NSInteger)interfaceOrientation x:(double)x y:(double)y x2:(double)x2 @@ -37,7 +37,7 @@ typedef NS_ENUM(NSInteger, RunnerTapSynthesisStatus) { + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow - interfaceOrientation:(NSInteger)interfaceOrientation + interfaceOrientation:(NSInteger)interfaceOrientation x:(double)x y:(double)y deadline:(NSDate * _Nullable)deadline @@ -47,7 +47,7 @@ typedef NS_ENUM(NSInteger, RunnerTapSynthesisStatus) { // starts contact; subsequent samples move it; all pointers lift at their final offset. + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow - interfaceOrientation:(NSInteger)interfaceOrientation + interfaceOrientation:(NSInteger)interfaceOrientation pointerSamples:(NSArray *> *> *)pointerSamples; // UIInterfaceOrientation of the app (1 portrait, 2 upsideDown, 3 landscapeRight, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m index e4a7e18ee2..858fd139a7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m @@ -113,7 +113,7 @@ + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow - interfaceOrientation:(NSInteger)interfaceOrientation + interfaceOrientation:(NSInteger)interfaceOrientation x:(double)x y:(double)y x2:(double)x2 @@ -137,7 +137,7 @@ + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)applicatio + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow - interfaceOrientation:(NSInteger)interfaceOrientation + interfaceOrientation:(NSInteger)interfaceOrientation x:(double)x y:(double)y deadline:(NSDate * _Nullable)deadline @@ -155,7 +155,7 @@ + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow - interfaceOrientation:(NSInteger)interfaceOrientation + interfaceOrientation:(NSInteger)interfaceOrientation pointerSamples:(NSArray *> *> *)pointerSamples { @try { RunnerGestureEventBridge bridge; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift index 61b62c157e..4ff5dbbf3d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift @@ -59,13 +59,13 @@ extension RunnerTests { deadline: deadline, errorMessage: &message ) - + XCTAssertTrue(preparationBeganBeforeDeadline) XCTAssertTrue(prepared) XCTAssertEqual(expired, .deadlineExceeded) XCTAssertNil(message) XCTAssertEqual(TapRecordSpy.syntheses, 0) - + let control = RunnerSynthesizedGesture.synthesizeTap( withApplication: TapApplicationFixture(), resolvedWindow: TapWindowFixture {}, From bf2566b84dffef3d96c9b25d10db365d5546c47b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 14:54:50 +0100 Subject: [PATCH 15/22] test(ios): keep placed-window query facts distinct from selector results --- .../UnitTests/RunnerRequestPanelTests.swift | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index f602839093..b38f807b1f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -23,6 +23,7 @@ private final class SurfaceSelectorQuery: NSObject { } private struct RecordedSurfaceDispatch { + let window: AnyObject? let displayID: UInt let start: CGPoint let end: CGPoint? @@ -100,6 +101,9 @@ private final class LitPanelBookingProbe: NSObject { @objc var allElementsBoundByIndex: [XCUIElement] { [Self.window!] } @objc(descendantsMatchingType:) func descendants(matching type: UInt) -> XCUIElementQuery { + if type == XCUIElement.ElementType.window.rawValue { + return unsafeBitCast(SurfaceSelectorQuery([Self.window!]), to: XCUIElementQuery.self) + } Self.beforeElementLookup?() let elements = self === Self.window || !Self.includeDarkSelector ? [Self.target!] : [Self.darkTarget!, Self.target!] return unsafeBitCast(SurfaceSelectorQuery(elements), to: XCUIElementQuery.self) @@ -135,7 +139,7 @@ private final class LitPanelBookingProbe: NSObject { private static func record(window: AnyObject?, orientation: Int, start: CGPoint, end: CGPoint? = nil) { var display: UInt = 0 if let window { _ = RunnerResolveWindowDisplayID(window, &display) } - dispatches.append(RecordedSurfaceDispatch(displayID: display, start: start, end: end, orientation: orientation)) + dispatches.append(RecordedSurfaceDispatch(window: window, displayID: display, start: start, end: end, orientation: orientation)) onDispatch?() } @@ -342,11 +346,15 @@ extension RunnerTests { XCTAssertTrue(response.ok, response.error?.message ?? "") XCTAssertGreaterThan(screenReads, 0) XCTAssertEqual(mainOwned.soleDisplayPortraitSize, CGSize(width: 1024, height: 1366)) + XCTAssertEqual(response.data?.referenceWidth, 386) + XCTAssertEqual(response.data?.referenceHeight, 682) XCTAssertEqual(LitPanelBookingProbe.dispatches.count, 1) let dispatch = try XCTUnwrap(LitPanelBookingProbe.dispatches.first) XCTAssertEqual(dispatch.displayID, 1) XCTAssertEqual(dispatch.orientation, RunnerInterfaceOrientation.landscapeRight) XCTAssertEqual(dispatch.start, CGPoint(x: 624, y: 300)) + XCTAssertTrue(dispatch.window === LitPanelBookingProbe.window) + XCTAssertEqual((dispatch.window as? XCUIElement)?.frame, CGRect(x: 224, y: 207, width: 386, height: 682)) } } From 6fadb47e81ccaf4ad9b87352e7763831fbb5be69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 15:25:29 +0100 Subject: [PATCH 16/22] docs: record geometry review evidence and live limits --- .../surface-geometry-4189-2026-11-03.md | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 docs/evidence/surface-geometry-4189-2026-11-03.md diff --git a/docs/evidence/surface-geometry-4189-2026-11-03.md b/docs/evidence/surface-geometry-4189-2026-11-03.md new file mode 100644 index 0000000000..afc5109db4 --- /dev/null +++ b/docs/evidence/surface-geometry-4189-2026-11-03.md @@ -0,0 +1,65 @@ +# Surface geometry review evidence, 2026-11-03 + +Source under test: `451ba0e7cdedd6773d85ac7955d18304de6f1cdf`. +Base: `ec80eecb1e0251d51d93aa68071c88577f1ea7ba` (`fix/foldable-springboard-surface`, #4122). +This packet addresses [the maintainer review](https://github.com/callstack/agent-device/pull/4189#issuecomment-6604084919). +The PR remains a draft because the requested live matrix is incomplete. + +## Native checks + +The unit-enabled iOS runner build and mandatory isolation scan passed. +Eleven method-selected native tests passed with `AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS=1` and private derived data. +The selection covers composed partial-window lookup and both dispatch routes, orientation-changing sequences, keyboard-clipped scroll dispatch, measured placed-iPad dispatch, actual native record orientation, tap deadlines, sole-display caching, windowless portrait/landscape admission, and both parent dark-window/exception regressions. +`swift test --package-path apple/snapshot-presentation` passed all 50 tests at the source SHA. + +The production-boundary assertions include: + +- Window `(0,80,669,871)`, panel `669x951`, orientation `3`: clipped scroll evidence is `669x508`; screen endpoints `(335,434)` and `(335,234)` dispatch as `(235,335)` and `(435,335)` on display `3`. +- Placed iPad window `(224,207,386,682)`: the production screen reader measures `1024x1366`; screen point `(300,400)` dispatches as `(624,300)` on display `1` in orientation `3`. +- A first sequence action changes orientation: later tap/hold steps refuse, completed steps remain `1`, and the native event count does not increase. +- Native record construction receives orientation `4` while the fake application's current getter reports `1`; the actual record stores `4`. + +## Mutation proof + +An isolated checkout ran the same five-test selection at `fe226cd7becf0b63ebde0009b2a0d4792b9df1e3`. +Each mutation built successfully and failed the relevant assertions with xcodebuild exit `65`. +The baseline and restored runs each passed all five tests. + +| Deliberate production defect | Failed assertions | +| --- | ---: | +| Rotate using the window box | 14 | +| Remove sequence orientation refusal | 16 | +| Rotate drag using the keyboard-clipped box | 2 | +| Skip sole-display measurement | 3 | +| Read orientation again during native record creation | 1 | +| Add window origin again to touch evidence | 4 | + +After correcting the placed-window query fake, its one-test baseline, missing-measurement mutation, and restored run were repeated at `451ba0e7`: pass, three assertion failures, pass. +The reviewed checkout remained clean during the experiments. +One initial failing run stalled while collecting diagnostics after its suite finished; the entire experiment was rerun with supported `-collect-test-diagnostics never` and normal failing exits. + +## Live observations and limits + +Only Duo `4F879835-4AB3-4046-B033-5AB769209DD4` and iPhone `DBEA214F-555F-4DDA-B62B-E34285CE63DB` were used. +Every UI command used the device-bound command returned by `device_open`. +That installed `0.21.31` client connected to this worktree's `0.21.37-dev` daemon through a private authenticated HTTP config and `--state-dir /private/tmp/agent-device-surface-4189`. +The daemon restarted with `AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH=$PWD/.tmp/surface-live-4189-derived` and rebuilt the runner from this checkout. +Its cache records source fingerprint `f4aeca12f9da03f67581231f3feb28c76d3030602255fc119ae10fdd5b376721`. + +The Tester ran from this worktree's Metro on port `8199`. +`fold open` reported hinge `180` and native lit panel `669x951`. +The command sequence was `orientation `, `press 'label="Form"'`, `fill 'id="field-name"' ''`, and `get text 'id="field-name"'`. +The portrait request read back `Geometry portrait 451ba0e7`; the right request read back `Geometry right 451ba0e7`. +A left request first refused a keyboard-covered Form target and reported `TEXT_ENTRY_MISMATCH` on fill; the advised retry with `--delay-ms 80` read back `Geometry left 451ba0e7`. +A fresh snapshot and lit capture succeeded; the capture was `951x669` logical points. +The requested rotations did not consistently match application orientation, so these runs do not prove the three-orientation matrix. +The portrait-request tap logged native point `(617,515)`, reference `(0,0,871,669)`, display `3`, orientation `4`. +The right-request tap logged native point `(334,52)`, reference `(0,80,669,871)`, display `3`, orientation `2`. + +On iPhone, `open com.apple.Preferences`, `snapshot`, and `press 'label="General"'` succeeded. +The visible screen changed to General; the native record logged display `1`, orientation `1`, point `(220,319)`, reference `(0,0,440,956)`. + +Missing live evidence: verified Duo orientations, half-open interaction, scoped-set coverage, explicit XCTest fallback, hold, clipped landscape scroll, sampled gesture, rotation-changing repeat, and AX-unavailable iPhone routes. +The two-device assignment excludes extra scoped clones and live iPad hardware. +The CLI offers no forced single-tap XCTest route; the composed native harness covers that route. +Cleanup reported Duo hinge `0`; both simulators shut down; the owned daemon released both claims cleanly; Metro stopped. From ec70e9e9e114346d82dc1e326f01e1db21c2514f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 15:32:47 +0100 Subject: [PATCH 17/22] refactor(ios): consolidate native drag dispatch and surface fixtures --- .../RunnerSynthesizedGesture.h | 31 ++- .../RunnerSynthesizedGesture.m | 52 +----- .../RunnerTests+SynthesizedInteraction.swift | 50 ++--- .../UnitTests/RunnerRequestPanelTests.swift | 176 ++++++++---------- .../RunnerTests+AlertObservationTests.swift | 23 +-- .../RunnerTests+CommandExecutionTests.swift | 27 +-- ...RunnerTests+ScrollDragExecutionTests.swift | 10 +- .../RunnerTests+SynthesizedGestureTests.swift | 9 +- ...nerTests+SynthesizedInteractionTests.swift | 9 +- .../surface-geometry-4189-2026-11-03.md | 7 + 10 files changed, 157 insertions(+), 237 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h index d19690b3e6..4ae66d4deb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h @@ -11,35 +11,28 @@ typedef NS_ENUM(NSInteger, RunnerTapSynthesisStatus) { RunnerTapSynthesisStatusFailed, }; +typedef struct { + CGPoint start; + CGPoint end; + double durationMs; + BOOL controlledScroll; +} RunnerSynthesizedDrag; + @interface RunnerSynthesizedGesture : NSObject // `resolvedWindow` is the app window the caller already resolved for the gesture's reference // frame, so the record's display ID is read from that same window. Pass nil for the sole-screen // route: the app's tree could not name its window and the screen is proven instead. // `interfaceOrientation` is the orientation used to convert the supplied points. -+ (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application - resolvedWindow:(id _Nullable)resolvedWindow - interfaceOrientation:(NSInteger)interfaceOrientation - x:(double)x - y:(double)y - x2:(double)x2 - y2:(double)y2 - durationMs:(double)durationMs; - -+ (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application - resolvedWindow:(id _Nullable)resolvedWindow - interfaceOrientation:(NSInteger)interfaceOrientation - x:(double)x - y:(double)y - x2:(double)x2 - y2:(double)y2 - durationMs:(double)durationMs; ++ (NSString * _Nullable)synthesizeDragWithApplication:(id)application + resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation + drag:(RunnerSynthesizedDrag)drag; + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow interfaceOrientation:(NSInteger)interfaceOrientation - x:(double)x - y:(double)y + point:(CGPoint)point deadline:(NSDate * _Nullable)deadline errorMessage:(NSString * _Nullable * _Nullable)errorMessage; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m index 858fd139a7..e19fc7b607 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m @@ -87,48 +87,15 @@ static id RunnerTapPointerPath( @implementation RunnerSynthesizedGesture -+ (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application - resolvedWindow:(id _Nullable)resolvedWindow - interfaceOrientation:(NSInteger)interfaceOrientation - x:(double)x - y:(double)y - x2:(double)x2 - y2:(double)y2 - durationMs:(double)durationMs { - @try { - return RunnerTrySynthesizeDrag( - application, - resolvedWindow, - interfaceOrientation, - CGPointMake(x, y), - CGPointMake(x2, y2), - durationMs, - @"agent-device-swipe", - RunnerSwipePointerPath - ); - } @catch (NSException *exception) { - return RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed"); - } -} - -+ (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application - resolvedWindow:(id _Nullable)resolvedWindow - interfaceOrientation:(NSInteger)interfaceOrientation - x:(double)x - y:(double)y - x2:(double)x2 - y2:(double)y2 - durationMs:(double)durationMs { ++ (NSString * _Nullable)synthesizeDragWithApplication:(id)application + resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation + drag:(RunnerSynthesizedDrag)drag { @try { return RunnerTrySynthesizeDrag( - application, - resolvedWindow, - interfaceOrientation, - CGPointMake(x, y), - CGPointMake(x2, y2), - durationMs, - @"agent-device-controlled-scroll", - RunnerControlledScrollPointerPath + application, resolvedWindow, interfaceOrientation, drag.start, drag.end, drag.durationMs, + drag.controlledScroll ? @"agent-device-controlled-scroll" : @"agent-device-swipe", + drag.controlledScroll ? RunnerControlledScrollPointerPath : RunnerSwipePointerPath ); } @catch (NSException *exception) { return RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed"); @@ -138,13 +105,12 @@ + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)applicatio + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow interfaceOrientation:(NSInteger)interfaceOrientation - x:(double)x - y:(double)y + point:(CGPoint)point deadline:(NSDate * _Nullable)deadline errorMessage:(NSString * _Nullable * _Nullable)errorMessage { if (errorMessage != NULL) *errorMessage = nil; @try { - return RunnerTrySynthesizeTap(application, resolvedWindow, interfaceOrientation, CGPointMake(x, y), deadline, errorMessage); + return RunnerTrySynthesizeTap(application, resolvedWindow, interfaceOrientation, point, deadline, errorMessage); } @catch (NSException *exception) { if (errorMessage != NULL) { *errorMessage = RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed"); diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 9d752c9d0b..f644c06599 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -39,30 +39,13 @@ extension RunnerTests { referenceFrame: frame, orientation: context.geometry.orientation ) - let message = switch profile { - case .controlledScroll: - RunnerSynthesizedGesture.synthesizeControlledScroll( - withApplication: app, - resolvedWindow: context.resolvedWindow, - interfaceOrientation: geometry.orientation, - x: Double(start.x), - y: Double(start.y), - x2: Double(end.x), - y2: Double(end.y), - durationMs: durationMs - ) - case .fastSwipe: - RunnerSynthesizedGesture.synthesizeSwipe( - withApplication: app, - resolvedWindow: context.resolvedWindow, - interfaceOrientation: geometry.orientation, - x: Double(start.x), - y: Double(start.y), - x2: Double(end.x), - y2: Double(end.y), - durationMs: durationMs - ) - } + let message = RunnerSynthesizedGesture.synthesizeDrag( + withApplication: app, + resolvedWindow: context.resolvedWindow, + interfaceOrientation: geometry.orientation, + drag: RunnerSynthesizedDrag(start: start, end: end, durationMs: durationMs, + controlledScroll: ObjCBool(profile == .controlledScroll)) + ) if let message { return .unsupported( message: message, @@ -99,16 +82,14 @@ extension RunnerTests { ) } let context = try context ?? synthesizedCoordinateContext(app: app) - let orientation = context?.geometry.orientation ?? Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) let resolution = pointGestureTarget( app: app, x: x, y: y, - orientation: orientation, context: context, gesture: .tap ) - guard case .target(let point, let resolvedWindow, let referenceFrame) = resolution else { + guard case .target(let point, let resolvedWindow, let referenceFrame, let orientation) = resolution else { return resolution.refusal } logSynthesizedDispatch( @@ -123,8 +104,7 @@ extension RunnerTests { withApplication: app, resolvedWindow: resolvedWindow, interfaceOrientation: orientation, - x: Double(point.x), - y: Double(point.y), + point: point, deadline: deadline, errorMessage: &message ) @@ -174,16 +154,14 @@ extension RunnerTests { hint: "Retry with finite x and y values and a positive durationMs." ) } - let orientation = context?.geometry.orientation ?? Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) let resolution = pointGestureTarget( app: app, x: x, y: y, - orientation: orientation, context: context, gesture: .longpress ) - guard case .target(let point, let resolvedWindow, let referenceFrame) = resolution else { + guard case .target(let point, let resolvedWindow, let referenceFrame, let orientation) = resolution else { return resolution.refusal } logSynthesizedDispatch( @@ -550,7 +528,7 @@ extension RunnerTests { private enum PointGestureResolution { /// A nil `resolvedWindow` routes the record to the sole screen. - case target(point: CGPoint, resolvedWindow: XCUIElement?, referenceFrame: CGRect) + case target(point: CGPoint, resolvedWindow: XCUIElement?, referenceFrame: CGRect, orientation: Int) case refused(RunnerInteractionOutcome) /// Read only on the branch that is not a target, so its `.performed` answer is never returned. @@ -567,7 +545,6 @@ extension RunnerTests { app: XCUIApplication, x: Double, y: Double, - orientation: Int, context: SynthesizedCoordinateContext?, gesture: SynthesizedPointGesture ) -> PointGestureResolution { @@ -576,10 +553,11 @@ extension RunnerTests { return .refused(Self.unmeasuredSoleDisplayOutcome(gesture: gesture.rawValue)) } let point = geometry.digitizerPoint(from: SurfacePoint(CGPoint(x: x, y: y))).point - return .target(point: point, resolvedWindow: context.resolvedWindow, referenceFrame: context.geometry.windowOnScreen.rect) + return .target(point: point, resolvedWindow: context.resolvedWindow, referenceFrame: context.geometry.windowOnScreen.rect, orientation: geometry.orientation) } + let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) if let unrotated = unrotatedSoleScreenPoint(app: app, x: x, y: y, interfaceOrientation: orientation) { - return .target(point: unrotated, resolvedWindow: nil, referenceFrame: .null) + return .target(point: unrotated, resolvedWindow: nil, referenceFrame: .null, orientation: orientation) } return .refused( .unsupported( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index b38f807b1f..895cd1821c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -109,18 +109,18 @@ private final class LitPanelBookingProbe: NSObject { return unsafeBitCast(SurfaceSelectorQuery(elements), to: XCUIElementQuery.self) } - @objc(synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:x:y:deadline:errorMessage:) - class func synthesizeTap(_ app: AnyObject, resolvedWindow: AnyObject?, interfaceOrientation: Int, x: Double, y: Double, + @objc(synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:point:deadline:errorMessage:) + class func synthesizeTap(_ app: AnyObject, resolvedWindow: AnyObject?, interfaceOrientation: Int, point: CGPoint, deadline: NSDate?, errorMessage: AutoreleasingUnsafeMutablePointer?) -> Int { - record(window: resolvedWindow, orientation: interfaceOrientation, start: CGPoint(x: x, y: y)) + record(window: resolvedWindow, orientation: interfaceOrientation, start: point) return 0 } - @objc(synthesizeControlledScrollWithApplication:resolvedWindow:interfaceOrientation:x:y:x2:y2:durationMs:) + @objc(synthesizeDragWithApplication:resolvedWindow:interfaceOrientation:drag:) class func synthesizeDrag(_ app: AnyObject, resolvedWindow: AnyObject?, interfaceOrientation: Int, - x: Double, y: Double, x2: Double, y2: Double, durationMs: Double) -> String? { + drag: RunnerSynthesizedDrag) -> String? { record(window: resolvedWindow, orientation: interfaceOrientation, - start: CGPoint(x: x, y: y), end: CGPoint(x: x2, y: y2)) + start: drag.start, end: drag.end) return nil } @@ -200,22 +200,20 @@ extension RunnerTests { let command = try runnerCommandFixture( #"{"command":"tap","selectorKey":"label","selectorValue":"same-selector","x":200,"y":150,"allowNonHittableCoordinateFallback":true,"synthesized":\#(synthesized),"litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# ) - let requestID = setRequestLitPanel(from: command) - defer { endRequestLitPanel(requestID) } - let response = try executeOnMainPrepared(command: command, activeApp: app) - XCTAssertTrue(response.ok, response.error?.message ?? "") - let booked = try XCTUnwrap(litPanelState.surface) - XCTAssertTrue(booked.window === window) - XCTAssertEqual(booked.geometry.windowOnScreen.rect, frame) - XCTAssertEqual(booked.geometry.digitizerBasis.rect, expected) - if synthesized { - XCTAssertEqual(LitPanelBookingProbe.synthesizedDisplay, 3) - XCTAssertEqual(LitPanelBookingProbe.synthesizedPoint, digitizer) - } else { - var display: UInt = 0 - XCTAssertNil(RunnerResolveWindowDisplayID(try XCTUnwrap(LitPanelBookingProbe.anchor), &display)) - XCTAssertEqual(display, 3) - XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) + try withPreparedSurfaceCommand(command) { _ in + let booked = try XCTUnwrap(litPanelState.surface) + XCTAssertTrue(booked.window === window) + XCTAssertEqual(booked.geometry.windowOnScreen.rect, frame) + XCTAssertEqual(booked.geometry.digitizerBasis.rect, expected) + if synthesized { + XCTAssertEqual(LitPanelBookingProbe.synthesizedDisplay, 3) + XCTAssertEqual(LitPanelBookingProbe.synthesizedPoint, digitizer) + } else { + var display: UInt = 0 + XCTAssertNil(RunnerResolveWindowDisplayID(try XCTUnwrap(LitPanelBookingProbe.anchor), &display)) + XCTAssertEqual(display, 3) + XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) + } } } XCTAssertTrue(LitPanelBookingProbe.anchor === window) @@ -248,21 +246,19 @@ extension RunnerTests { let command = try runnerCommandFixture( #"{"command":"sequence","litPanel":{"displayId":3,"widthPt":669,"heightPt":951},"steps":[{"kind":"tap","x":200,"y":150,"synthesized":\#(synthesized)},{"kind":"\#(nextKind)","x":200,"y":150,"synthesized":true,"durationMs":100}]}"# ) - let requestID = setRequestLitPanel(from: command) - defer { endRequestLitPanel(requestID) } - let response = try executeOnMainPrepared(command: command, activeApp: app) - XCTAssertTrue(response.ok, response.error?.message ?? "") - XCTAssertEqual(response.data?.x, 200) - XCTAssertEqual(response.data?.y, 150) - XCTAssertEqual(response.data?.completedSteps, 1) - XCTAssertEqual(response.data?.failedStepIndex, 1) - XCTAssertEqual(response.data?.sequenceResults?.last?.errorCode, "UNSUPPORTED_OPERATION") - XCTAssertEqual(LitPanelBookingProbe.dispatches.count, synthesized ? 1 : 0) - if synthesized { - let dispatch = try XCTUnwrap(LitPanelBookingProbe.dispatches.first) - XCTAssertEqual(dispatch.displayID, 3) - XCTAssertEqual(dispatch.start, CGPoint(x: 200, y: 150)) - XCTAssertEqual(dispatch.orientation, RunnerInterfaceOrientation.portrait) + try withPreparedSurfaceCommand(command) { response in + XCTAssertEqual(response.data?.x, 200) + XCTAssertEqual(response.data?.y, 150) + XCTAssertEqual(response.data?.completedSteps, 1) + XCTAssertEqual(response.data?.failedStepIndex, 1) + XCTAssertEqual(response.data?.sequenceResults?.last?.errorCode, "UNSUPPORTED_OPERATION") + XCTAssertEqual(LitPanelBookingProbe.dispatches.count, synthesized ? 1 : 0) + if synthesized { + let dispatch = try XCTUnwrap(LitPanelBookingProbe.dispatches.first) + XCTAssertEqual(dispatch.displayID, 3) + XCTAssertEqual(dispatch.start, CGPoint(x: 200, y: 150)) + XCTAssertEqual(dispatch.orientation, RunnerInterfaceOrientation.portrait) + } } } } @@ -281,24 +277,22 @@ extension RunnerTests { let command = try runnerCommandFixture( #"{"command":"scroll","direction":"down","pixels":200,"durationMs":400,"litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# ) - let requestID = setRequestLitPanel(from: command) - defer { endRequestLitPanel(requestID) } - let response = try executeOnMainPrepared(command: command, activeApp: app) - XCTAssertTrue(response.ok, response.error?.message ?? "") - XCTAssertEqual(response.data?.referenceWidth, 669) - XCTAssertEqual(response.data?.referenceHeight, 508) - XCTAssertEqual(response.data?.keyboardAvoided, true) - XCTAssertEqual(response.data?.keyboardMinY, 600) - XCTAssertEqual(response.data?.x, 335) - XCTAssertEqual(response.data?.y, 434) - XCTAssertEqual(response.data?.x2, 335) - XCTAssertEqual(response.data?.y2, 234) - XCTAssertEqual(LitPanelBookingProbe.dispatches.count, 1) - let dispatch = try XCTUnwrap(LitPanelBookingProbe.dispatches.first) - XCTAssertEqual(dispatch.displayID, 3) - XCTAssertEqual(dispatch.orientation, RunnerInterfaceOrientation.landscapeRight) - XCTAssertEqual(dispatch.start, CGPoint(x: 235, y: 335)) - XCTAssertEqual(dispatch.end, CGPoint(x: 435, y: 335)) + try withPreparedSurfaceCommand(command) { response in + XCTAssertEqual(response.data?.referenceWidth, 669) + XCTAssertEqual(response.data?.referenceHeight, 508) + XCTAssertEqual(response.data?.keyboardAvoided, true) + XCTAssertEqual(response.data?.keyboardMinY, 600) + XCTAssertEqual(response.data?.x, 335) + XCTAssertEqual(response.data?.y, 434) + XCTAssertEqual(response.data?.x2, 335) + XCTAssertEqual(response.data?.y2, 234) + XCTAssertEqual(LitPanelBookingProbe.dispatches.count, 1) + let dispatch = try XCTUnwrap(LitPanelBookingProbe.dispatches.first) + XCTAssertEqual(dispatch.displayID, 3) + XCTAssertEqual(dispatch.orientation, RunnerInterfaceOrientation.landscapeRight) + XCTAssertEqual(dispatch.start, CGPoint(x: 235, y: 335)) + XCTAssertEqual(dispatch.end, CGPoint(x: 435, y: 335)) + } } } @@ -308,28 +302,12 @@ extension RunnerTests { let cached = mainOwned.soleDisplayPortraitSize defer { mainOwned.soleDisplayPortraitSize = cached } mainOwned.soleDisplayPortraitSize = nil - let selector = #selector(NSObject.value(forKey:)) - let inherited = try XCTUnwrap(class_getInstanceMethod(XCUIScreen.self, selector)) - class_addMethod(XCUIScreen.self, selector, method_getImplementation(inherited), method_getTypeEncoding(inherited)) - let target = try XCTUnwrap(class_getInstanceMethod(XCUIScreen.self, selector)) - let original = method_getImplementation(target) - typealias ReadValue = @convention(c) (AnyObject, Selector, NSString) -> AnyObject? - let readValue = unsafeBitCast(original, to: ReadValue.self) var screenReads = 0 - let read: @convention(block) (AnyObject, NSString) -> AnyObject? = { screen, key in - if key == "bounds" { - screenReads += 1 - return NSValue(cgRect: CGRect(x: 0, y: 0, width: 2048, height: 2732)) - } - if key == "scale" { return NSNumber(value: 2) } - return readValue(screen, selector, key) - } - let replacement = imp_implementationWithBlock(read) - method_setImplementation(target, replacement) - defer { - method_setImplementation(target, original) - imp_removeBlock(replacement) - } + let restore = try overrideSoleScreenValues([ + "bounds": NSValue(cgRect: CGRect(x: 0, y: 0, width: 2048, height: 2732)), + "scale": NSNumber(value: 2) + ], didRead: { if $0 == "bounds" { screenReads += 1 } }) + defer { restore() } try withLitPanelBookingProbe(composed: true) { _ in LitPanelBookingProbe.windowDisplayID = 1 LitPanelBookingProbe.includeDarkSelector = false @@ -340,21 +318,19 @@ extension RunnerTests { let command = try runnerCommandFixture( #"{"command":"tap","selectorKey":"label","selectorValue":"same-selector","synthesized":true}"# ) - let requestID = setRequestLitPanel(from: command) - defer { endRequestLitPanel(requestID) } - let response = try executeOnMainPrepared(command: command, activeApp: app) - XCTAssertTrue(response.ok, response.error?.message ?? "") - XCTAssertGreaterThan(screenReads, 0) - XCTAssertEqual(mainOwned.soleDisplayPortraitSize, CGSize(width: 1024, height: 1366)) - XCTAssertEqual(response.data?.referenceWidth, 386) - XCTAssertEqual(response.data?.referenceHeight, 682) - XCTAssertEqual(LitPanelBookingProbe.dispatches.count, 1) - let dispatch = try XCTUnwrap(LitPanelBookingProbe.dispatches.first) - XCTAssertEqual(dispatch.displayID, 1) - XCTAssertEqual(dispatch.orientation, RunnerInterfaceOrientation.landscapeRight) - XCTAssertEqual(dispatch.start, CGPoint(x: 624, y: 300)) - XCTAssertTrue(dispatch.window === LitPanelBookingProbe.window) - XCTAssertEqual((dispatch.window as? XCUIElement)?.frame, CGRect(x: 224, y: 207, width: 386, height: 682)) + try withPreparedSurfaceCommand(command) { response in + XCTAssertGreaterThan(screenReads, 0) + XCTAssertEqual(mainOwned.soleDisplayPortraitSize, CGSize(width: 1024, height: 1366)) + XCTAssertEqual(response.data?.referenceWidth, 386) + XCTAssertEqual(response.data?.referenceHeight, 682) + XCTAssertEqual(LitPanelBookingProbe.dispatches.count, 1) + let dispatch = try XCTUnwrap(LitPanelBookingProbe.dispatches.first) + XCTAssertEqual(dispatch.displayID, 1) + XCTAssertEqual(dispatch.orientation, RunnerInterfaceOrientation.landscapeRight) + XCTAssertEqual(dispatch.start, CGPoint(x: 624, y: 300)) + XCTAssertTrue(dispatch.window === LitPanelBookingProbe.window) + XCTAssertEqual((dispatch.window as? XCUIElement)?.frame, CGRect(x: 224, y: 207, width: 386, height: 682)) + } } } @@ -523,6 +499,15 @@ extension RunnerTests { } } + @MainActor + private func withPreparedSurfaceCommand(_ command: Command, _ assertions: (Response) throws -> Void) throws { + let requestID = setRequestLitPanel(from: command) + defer { endRequestLitPanel(requestID) } + let response = try executeOnMainPrepared(command: command, activeApp: app) + XCTAssertTrue(response.ok, response.error?.message ?? "") + try assertions(response) + } + @MainActor private func withLitPanelBookingProbe(composed: Bool = false, _ body: (XCUIElement) throws -> Void) throws { app.launch() @@ -563,17 +548,16 @@ extension RunnerTests { (#selector(XCUIElement.descendants(matching:)), #selector(LitPanelBookingProbe.descendants(matching:))), ].map { (class_getInstanceMethod(windowClass, $0.0), class_getInstanceMethod(LitPanelBookingProbe.self, $0.1)) } let synthesisSwap = ( - class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:x:y:deadline:errorMessage:")), - class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.synthesizeTap(_:resolvedWindow:interfaceOrientation:x:y:deadline:errorMessage:))) + class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:point:deadline:errorMessage:")), + class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.synthesizeTap(_:resolvedWindow:interfaceOrientation:point:deadline:errorMessage:))) ) - let dragStub = class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.synthesizeDrag(_:resolvedWindow:interfaceOrientation:x:y:x2:y2:durationMs:))) + let dragStub = class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.synthesizeDrag(_:resolvedWindow:interfaceOrientation:drag:))) let composedSwaps: [(Method?, Method?)] = [ (class_getInstanceMethod(try XCTUnwrap(object_getClass(app)), NSSelectorFromString("keyboards")), class_getInstanceMethod(LitPanelBookingProbe.self, #selector(getter: LitPanelBookingProbe.keyboards))), (class_getInstanceMethod(try XCTUnwrap(object_getClass(query)), #selector(getter: XCUIElementQuery.allElementsBoundByIndex)), class_getInstanceMethod(LitPanelBookingProbe.self, #selector(getter: LitPanelBookingProbe.allElementsBoundByIndex))), - (class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeControlledScrollWithApplication:resolvedWindow:interfaceOrientation:x:y:x2:y2:durationMs:")), dragStub), - (class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeSwipeWithApplication:resolvedWindow:interfaceOrientation:x:y:x2:y2:durationMs:")), dragStub), + (class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeDragWithApplication:resolvedWindow:interfaceOrientation:drag:")), dragStub), (class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeGestureWithApplication:resolvedWindow:interfaceOrientation:pointerSamples:")), class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.synthesizeGesture(_:resolvedWindow:interfaceOrientation:pointerSamples:)))) ] diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift index 1c43da4740..ac6cbb1f12 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift @@ -3,20 +3,21 @@ import XCTest #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) import ObjectiveC.runtime -private final class AlertTapDeadlineStub: NSObject { +final class RunnerTapSynthesisStub: NSObject { static var status: RunnerTapSynthesisStatus? static var interruptionHandlingDisabled: Bool? - @objc(synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:) + @objc(synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:point:deadline:errorMessage:) class func synthesizeTap( application: XCUIApplication, resolvedWindow: Any?, - x: Double, - y: Double, + interfaceOrientation: Int, + point: CGPoint, deadline: NSDate?, errorMessage: AutoreleasingUnsafeMutablePointer? ) -> RunnerTapSynthesisStatus { interruptionHandlingDisabled = (application.value(forKey: "doesNotHandleUIInterruptions") as? NSNumber)?.boolValue + if status == .failed { errorMessage?.pointee = "forced private synthesis failure" } return status ?? (deadline == nil ? .succeeded : .deadlineExceeded) } } @@ -132,7 +133,7 @@ extension RunnerTests { XCTAssertTrue(app.alerts.firstMatch.exists) XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 0; replacement actions: 0") - AlertTapDeadlineStub.status = RunnerTapSynthesisStatus(rawValue: 999) + RunnerTapSynthesisStub.status = RunnerTapSynthesisStatus(rawValue: 999) let unknown = try activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) guard case .unsupported? = unknown else { return XCTFail("an unknown native status must report failure rather than deadline expiry") @@ -193,12 +194,12 @@ extension RunnerTests { let button = try XCTUnwrap(alert.actions.first { $0.label == "OK" }) let frame = button.frame try withAlertTapStub { - AlertTapDeadlineStub.status = .deadlineExceeded + RunnerTapSynthesisStub.status = .deadlineExceeded let outcome = try activateAlertButton( alert, button: button, action: "accept", frame: frame, deadline: .distantFuture ) XCTAssertNil(outcome) - XCTAssertEqual(AlertTapDeadlineStub.interruptionHandlingDisabled, true) + XCTAssertEqual(RunnerTapSynthesisStub.interruptionHandlingDisabled, true) } } @@ -218,13 +219,13 @@ extension RunnerTests { @MainActor private func withAlertTapStub(_ body: () throws -> Void) throws { - let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:") + let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:point:deadline:errorMessage:") let method = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedGesture.self, selector)) - let stub = try XCTUnwrap(class_getClassMethod(AlertTapDeadlineStub.self, selector)) + let stub = try XCTUnwrap(class_getClassMethod(RunnerTapSynthesisStub.self, selector)) let original = method_setImplementation(method, method_getImplementation(stub)) defer { - AlertTapDeadlineStub.status = nil - AlertTapDeadlineStub.interruptionHandlingDisabled = nil + RunnerTapSynthesisStub.status = nil + RunnerTapSynthesisStub.interruptionHandlingDisabled = nil method_setImplementation(method, original) } try body() diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift index 7786296fa5..27d36a8f79 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift @@ -8,21 +8,6 @@ private final class RunnerCoordinateTapObserver { var count = 0 } -private final class RunnerSynthesizedTapFailureStub: NSObject { - @objc(synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:) - class func synthesizeTap( - application: XCUIApplication, - resolvedWindow: Any?, - x: Double, - y: Double, - deadline: NSDate?, - errorMessage: AutoreleasingUnsafeMutablePointer? - ) -> RunnerTapSynthesisStatus { - errorMessage?.pointee = "forced private synthesis failure" - return .failed - } -} - extension RunnerTests { private func observeCoordinateTaps(droppingFirst: Bool = false) throws -> ( observer: RunnerCoordinateTapObserver, restore: () -> Void @@ -47,12 +32,18 @@ extension RunnerTests { /// Makes private tap synthesis fail until the returned closure restores it. func forceSynthesizedTapFailure() throws -> () -> Void { - let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:") + let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:point:deadline:errorMessage:") let synthesizedTapMethod = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedGesture.self, selector)) - let failureStubMethod = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedTapFailureStub.self, selector)) + let failureStubMethod = try XCTUnwrap(class_getClassMethod(RunnerTapSynthesisStub.self, selector)) let originalImplementation = method_getImplementation(synthesizedTapMethod) method_setImplementation(synthesizedTapMethod, method_getImplementation(failureStubMethod)) - return { method_setImplementation(synthesizedTapMethod, originalImplementation) } + let status = RunnerTapSynthesisStub.status + RunnerTapSynthesisStub.status = .failed + return { + RunnerTapSynthesisStub.status = status + RunnerTapSynthesisStub.interruptionHandlingDisabled = nil + method_setImplementation(synthesizedTapMethod, originalImplementation) + } } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift index 4a4085bd01..36d29a55fe 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift @@ -5,16 +5,12 @@ import AgentDeviceSnapshotPresentation import ObjectiveC.runtime private final class RunnerSynthesizedSwipeFailureStub: NSObject { - @objc(synthesizeSwipeWithApplication:resolvedWindow:interfaceOrientation:x:y:x2:y2:durationMs:) + @objc(synthesizeDragWithApplication:resolvedWindow:interfaceOrientation:drag:) class func synthesizeSwipe( application: XCUIApplication, resolvedWindow: Any?, interfaceOrientation: Int, - x: Double, - y: Double, - x2: Double, - y2: Double, - durationMs: Double + drag: RunnerSynthesizedDrag ) -> String? { "forced private synthesis failure" } @@ -57,7 +53,7 @@ extension RunnerTests { @MainActor func testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails() throws { let selector = NSSelectorFromString( - "synthesizeSwipeWithApplication:resolvedWindow:interfaceOrientation:x:y:x2:y2:durationMs:" + "synthesizeDragWithApplication:resolvedWindow:interfaceOrientation:drag:" ) guard let synthesizedSwipeMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift index 4ff5dbbf3d..49c6c14f24 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift @@ -54,8 +54,7 @@ extension RunnerTests { withApplication: TapApplicationFixture(), resolvedWindow: window, interfaceOrientation: 1, - x: 50, - y: 50, + point: CGPoint(x: 50, y: 50), deadline: deadline, errorMessage: &message ) @@ -70,8 +69,7 @@ extension RunnerTests { withApplication: TapApplicationFixture(), resolvedWindow: TapWindowFixture {}, interfaceOrientation: 1, - x: 50, - y: 50, + point: CGPoint(x: 50, y: 50), deadline: .distantFuture, errorMessage: &message ) @@ -87,8 +85,7 @@ extension RunnerTests { withApplication: TapApplicationFixture(), resolvedWindow: TapWindowFixture {}, interfaceOrientation: 4, - x: 50, - y: 50, + point: CGPoint(x: 50, y: 50), deadline: .distantFuture, errorMessage: &message ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 06d02c3a7e..e5fe6918ef 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -54,6 +54,12 @@ extension RunnerTests { } private func simulateUnreadableDisplay() throws -> () -> Void { + try overrideSoleScreenValues(["scale": NSNumber(value: 0)]) + } + + func overrideSoleScreenValues( + _ values: [String: AnyObject], didRead: @escaping (String) -> Void = { _ in } + ) throws -> () -> Void { let selector = #selector(NSObject.value(forKey:)) let inherited = try XCTUnwrap(class_getInstanceMethod(XCUIScreen.self, selector)) class_addMethod(XCUIScreen.self, selector, method_getImplementation(inherited), method_getTypeEncoding(inherited)) @@ -62,7 +68,8 @@ extension RunnerTests { typealias ReadValue = @convention(c) (AnyObject, Selector, NSString) -> AnyObject? let readValue = unsafeBitCast(original, to: ReadValue.self) let read: @convention(block) (AnyObject, NSString) -> AnyObject? = { screen, key in - if key == "scale" { return NSNumber(value: 0) } + didRead(key as String) + if let value = values[key as String] { return value } return readValue(screen, selector, key) } let replacement = imp_implementationWithBlock(read) diff --git a/docs/evidence/surface-geometry-4189-2026-11-03.md b/docs/evidence/surface-geometry-4189-2026-11-03.md index afc5109db4..6b5858fbc2 100644 --- a/docs/evidence/surface-geometry-4189-2026-11-03.md +++ b/docs/evidence/surface-geometry-4189-2026-11-03.md @@ -63,3 +63,10 @@ Missing live evidence: verified Duo orientations, half-open interaction, scoped- The two-device assignment excludes extra scoped clones and live iPad hardware. The CLI offers no forced single-tap XCTest route; the composed native harness covers that route. Cleanup reported Duo hinge `0`; both simulators shut down; the owned daemon released both claims cleanly; Metro stopped. + +## Final API consolidation + +The first full gate at `3090b336` rejected ten new native duplicate blocks. +The follow-up consolidates drag dispatch, passes tap points as `CGPoint`, resolves point orientation with its target, and shares screen-query and preparation fixtures. +Two existing fallback/alert stubs were also updated to the current native selector. +The live observations above precede this API consolidation; final-head native and gate results are recorded in the PR validation section. From 5ded1a6ead71e1f9cc3493d610d090d874cccee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 16:38:39 +0100 Subject: [PATCH 18/22] fix(apple): retain press series orientation across chunks --- .../RunnerTests+Models.swift | 2 + .../RunnerTests+SequenceExecution.swift | 10 +- .../UnitTests/RunnerRequestPanelTests.swift | 37 ++++++ .../UnitTests/RunnerTests+ModelsTests.swift | 1 + contracts/fixtures/runner-requests.json | 1 + .../index-dispatch-disclosure.test.ts | 5 +- .../src/__tests__/runner-sequence.test.ts | 119 +++++++++++++++++- packages/apple-runner/src/runner-contract.ts | 14 ++- packages/apple-runner/src/runner-sequence.ts | 25 +++- .../src/foldable/runner-panel.test.ts | 1 + 10 files changed, 207 insertions(+), 8 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 4b89a2c814..d2920e728e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -277,6 +277,7 @@ struct Command: Codable { let synthesized: Bool? let coordinateTapTextInputEvidence: CoordinateTapTextInputEvidence? let litPanel: RunnerLitPanelFact? + let sequenceOrientation: Int? let steps: [SequenceStep]? } @@ -530,6 +531,7 @@ struct DataPayload: Codable { /// daemon read `ok` as proof the runner drained (#2552). var runnerMainThreadBusy: Bool? var completedSteps: Int? + var sequenceOrientation: Int? var failedStepIndex: Int? var sequenceResults: [SequenceStepResult]? var targetActivation: TargetActivationFactPayload? diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index f88aaa4dd8..f8a32a5f94 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -47,8 +47,10 @@ extension RunnerTests { : nil #if os(iOS) - let orientation = synthesizedContext?.geometry.orientation ?? litPanelState.surface?.geometry.orientation + let orientation = command.sequenceOrientation ?? synthesizedContext?.geometry.orientation ?? litPanelState.surface?.geometry.orientation ?? Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: activeApp)) +#else + let orientation: Int? = nil #endif let execution = try assembleSequenceExecution(steps: steps) { _, step in #if os(iOS) @@ -61,7 +63,7 @@ extension RunnerTests { #endif return try performSequenceStep(step, activeApp: activeApp, synthesizedContext: synthesizedContext) } - return sequenceResponse(execution: execution, touchFrame: firstFrame) + return sequenceResponse(execution: execution, touchFrame: firstFrame, sequenceOrientation: orientation) } /// Pure, device-free assembler: runs each step in order via `perform`, stops at the first @@ -243,7 +245,8 @@ extension RunnerTests { private func sequenceResponse( execution: SequenceExecutionResult, - touchFrame: TouchVisualizationFrame? + touchFrame: TouchVisualizationFrame?, + sequenceOrientation: Int? ) -> Response { return Response( ok: true, @@ -256,6 +259,7 @@ extension RunnerTests { referenceWidth: touchFrame?.referenceWidth, referenceHeight: touchFrame?.referenceHeight, completedSteps: execution.completedSteps, + sequenceOrientation: sequenceOrientation, failedStepIndex: execution.failedStepIndex, sequenceResults: execution.results ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 895cd1821c..4bd70c86c7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -265,6 +265,43 @@ extension RunnerTests { } } + @MainActor + func testRepeatedPressCarriesOrientationAcrossStepAndBudgetChunkBoundaries() throws { + try withLitPanelBookingProbe(composed: true) { _ in + for chunkSize in [1, 20] { + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 80, width: 669, height: 871) + LitPanelBookingProbe.orientation = RunnerInterfaceOrientation.portrait + LitPanelBookingProbe.dispatches.removeAll() + LitPanelBookingProbe.onDispatch = { + if LitPanelBookingProbe.dispatches.count == chunkSize { + LitPanelBookingProbe.orientation = RunnerInterfaceOrientation.landscapeLeft + LitPanelBookingProbe.windowFrame = CGRect(x: 80, y: 0, width: 871, height: 669) + } + } + let step = #"{"kind":"tap","x":200,"y":150,"synthesized":true}"# + let steps = Array(repeating: step, count: chunkSize).joined(separator: ",") + let first = try runnerCommandFixture( + #"{"command":"sequence","litPanel":{"displayId":3,"widthPt":669,"heightPt":951},"steps":[\#(steps)]}"# + ) + try withPreparedSurfaceCommand(first) { response in + XCTAssertEqual(response.data?.completedSteps, chunkSize) + let wire = try XCTUnwrap(try JSONSerialization.jsonObject(with: JSONEncoder().encode(response)) as? [String: Any]) + let data = try XCTUnwrap(wire["data"] as? [String: Any]) + XCTAssertEqual(data["sequenceOrientation"] as? Int, RunnerInterfaceOrientation.portrait) + } + let continuation = try runnerCommandFixture( + #"{"command":"sequence","sequenceOrientation":1,"litPanel":{"displayId":3,"widthPt":669,"heightPt":951},"steps":[\#(step)]}"# + ) + try withPreparedSurfaceCommand(continuation) { response in + XCTAssertEqual(response.data?.completedSteps, 0) + XCTAssertEqual(response.data?.failedStepIndex, 0) + XCTAssertEqual(response.data?.sequenceResults?.first?.errorCode, "UNSUPPORTED_OPERATION") + } + XCTAssertEqual(LitPanelBookingProbe.dispatches.count, chunkSize) + } + } + } + @MainActor func testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis() throws { let health = mainOwned.accessibilityHealth diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift index d2a3644c56..c8eabb962d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift @@ -78,6 +78,7 @@ extension RunnerTests { let samples = pointers.flatMap { $0["samples"] as? [[String: Any]] ?? [] } let litPanels = requests.compactMap { $0["litPanel"] as? [String: Any] } let commands = try entries.map { try decodeProductionRunnerRequest($0.request, $0.name) } + XCTAssertEqual(commands.compactMap(\.sequenceOrientation).first, 1) let command = try XCTUnwrap(commands.first, "no production request") let plan = try XCTUnwrap( commands.compactMap(\.gesturePlan).first, diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index f86080eabd..71c1fc1fe0 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -64,6 +64,7 @@ {"name": "ios-simulator.runner-disposal.shutdown", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "shutdown", "commandId": ""}}, {"name": "ios-simulator.runner-lifecycle-prepare.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": ""}}, {"name": "ios-simulator.runner-selector-query.label", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "querySelector", "selectorKey": "label", "selectorValue": "Go", "appBundleId": "com.example.app", "commandId": ""}}, + {"name": "ios-simulator.runner-sequence.bound-orientation", "producer": "packages/apple-runner/src/__tests__/runner-sequence.test.ts", "request": {"command": "sequence", "steps": [{"kind": "tap", "x": 10, "y": 20, "synthesized": true}], "appBundleId": "com.example.app", "sequenceOrientation": 1, "commandId": ""}}, {"name": "ios-simulator.runner-session-readiness.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": ""}}, {"name": "macos.desktop-scroll.amount", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "desktopScroll", "direction": "down", "amount": 0.5, "durationMs": 300, "appBundleId": "com.example.app", "commandId": ""}}, {"name": "macos.desktop-scroll.pixels", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "desktopScroll", "direction": "up", "pixels": 200, "durationMs": 300, "appBundleId": "com.example.app", "commandId": ""}}, diff --git a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts index b4599c0c92..a861324e16 100644 --- a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts +++ b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts @@ -114,7 +114,10 @@ async function pressSeriesRefusedOnSecondChunk(): Promise { const firstChunk = Array.from({ length: 20 }, () => ({ ok: true, kind: 'tap' })); server = await startFakeRunnerServer({ sequence: [ - { kind: 'ok', data: { completedSteps: 20, sequenceResults: firstChunk } }, + { + kind: 'ok', + data: { sequenceOrientation: 1, completedSteps: 20, sequenceResults: firstChunk }, + }, { kind: 'runnerError', code: 'RUNNER_BUSY', message: 'runner busy' }, ], }); diff --git a/packages/apple-runner/src/__tests__/runner-sequence.test.ts b/packages/apple-runner/src/__tests__/runner-sequence.test.ts index abf89e7161..88cbc4c1b1 100644 --- a/packages/apple-runner/src/__tests__/runner-sequence.test.ts +++ b/packages/apple-runner/src/__tests__/runner-sequence.test.ts @@ -3,11 +3,128 @@ import assert from 'node:assert/strict'; import { AppError } from '@agent-device/kernel/errors'; import { runApplePressSeries } from '../runner-sequence.ts'; import { IOS_SIMULATOR } from './device-fixtures.ts'; +import type { RunnerCommand } from '../runner-contract.ts'; +import { runAppleRunnerCommand } from '../index.ts'; +import { withAppleRunnerProvider } from '../runner-provider.ts'; +import { assertProducedRunnerRequests } from '@agent-device/test-kit'; + +test('press-series continuations carry the original orientation on the runner wire', async () => { + const requests: RunnerCommand[] = []; + await withAppleRunnerProvider( + async (_device, command) => { + requests.push(command); + return { + sequenceOrientation: 1, + completedSteps: command.steps?.length ?? 0, + sequenceResults: command.steps?.map(({ kind }) => ({ ok: true, kind })) ?? [], + }; + }, + { deviceId: IOS_SIMULATOR.id }, + async () => { + await runApplePressSeries( + IOS_SIMULATOR, + { x: 10, y: 20 }, + { button: 'primary', count: 21, intervalMs: 0, holdMs: 0, jitterPx: 0, doubleTap: false }, + 'com.example.app', + async (command) => await runAppleRunnerCommand(IOS_SIMULATOR, command, {}), + ); + }, + ); + assert.equal(requests.length, 2); + assertProducedRunnerRequests(import.meta.filename, [ + ['ios-simulator.runner-sequence.bound-orientation', requests[1]], + ]); +}); + +test.each([ + { boundary: 'step cap', count: 21, intervalMs: 0, completed: 20 }, + { boundary: 'time budget', count: 3, intervalMs: 10_000, completed: 1 }, +])( + 'an orientation change at the $boundary stops the next chunk', + async ({ count, intervalMs, completed }) => { + let orientation = 1; + let dispatched = 0; + const requests: RunnerCommand[] = []; + await assert.rejects( + runApplePressSeries( + IOS_SIMULATOR, + { x: 10, y: 20 }, + { button: 'primary', count, intervalMs, holdMs: 0, jitterPx: 0, doubleTap: false }, + undefined, + async (command) => { + requests.push(command); + const bound = command.sequenceOrientation ?? orientation; + const results = []; + let completedSteps = 0; + for (const step of command.steps ?? []) { + if (bound !== orientation) { + results.push({ ok: false, kind: step.kind, errorCode: 'UNSUPPORTED_OPERATION' }); + return { + sequenceOrientation: bound, + completedSteps, + failedStepIndex: completedSteps, + sequenceResults: results, + }; + } + dispatched += 1; + completedSteps += 1; + results.push({ ok: true, kind: step.kind }); + if (dispatched === completed) orientation = 3; + } + return { sequenceOrientation: bound, completedSteps, sequenceResults: results }; + }, + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'UNSUPPORTED_OPERATION'); + assert.equal(error.details?.failedStepIndex, completed); + assert.equal(error.details?.completedSteps, completed); + assert.equal(error.details?.dispatched, 'unknown'); + assert.equal(error.details?.dispatchedSteps, completed); + return true; + }, + ); + assert.equal(dispatched, completed); + assert.equal(requests.length, 2); + assert.equal(requests[0]?.steps?.length, completed); + assert.equal(requests[1]?.sequenceOrientation, 1); + }, +); + +test.each([undefined, 0, 5, 1.5, '1'])( + 'a missing or invalid first-chunk orientation (%s) cannot authorize another chunk', + async (sequenceOrientation) => { + let requests = 0; + await assert.rejects( + runApplePressSeries( + IOS_SIMULATOR, + { x: 10, y: 20 }, + { button: 'primary', count: 21, intervalMs: 0, holdMs: 0, jitterPx: 0, doubleTap: false }, + undefined, + async (command) => { + requests += 1; + return { + sequenceOrientation, + completedSteps: command.steps!.length, + sequenceResults: command.steps!.map(({ kind }) => ({ ok: true, kind })), + }; + }, + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'runner_sequence_orientation_unavailable'); + assert.equal(error.details?.dispatchedSteps, 20); + return true; + }, + ); + assert.equal(requests, 1); + }, +); test('a step failure in a later chunk counts the taps every chunk completed', async () => { const ok = (count: number) => Array.from({ length: count }, () => ({ ok: true, kind: 'tap' })); const answers = [ - { completedSteps: 20, sequenceResults: ok(20) }, + { sequenceOrientation: 1, completedSteps: 20, sequenceResults: ok(20) }, { completedSteps: 2, sequenceResults: [...ok(2), { ok: false, kind: 'tap' }] }, ]; await assert.rejects( diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index 8219bf8015..a64f02adac 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -168,6 +168,7 @@ export type RunnerCommand = { * command that resolves a screen (ADR 0025). */ litPanel?: { displayId: number; widthPt: number; heightPt: number }; + sequenceOrientation?: RunnerSequenceOrientation; steps?: RunnerSequenceStep[]; }; @@ -573,8 +574,11 @@ export type ParsedRunnerSequenceResult = { results: RunnerSequenceStepResult[]; completedSteps: number; failedStepIndex?: number; + sequenceOrientation?: RunnerSequenceOrientation; }; +export type RunnerSequenceOrientation = 1 | 2 | 3 | 4; + function isSequenceableKind(kind: unknown): kind is SequenceableRunnerStepKind { return ( typeof kind === 'string' && (SEQUENCEABLE_RUNNER_STEP_KINDS as readonly string[]).includes(kind) @@ -703,7 +707,15 @@ export function parseRunnerSequenceResult( throw buildSequenceStepError(results, completedSteps, failedStepIndex); } - return { results, completedSteps, failedStepIndex }; + const orientation = data.sequenceOrientation; + return { + results, + completedSteps, + failedStepIndex, + ...(orientation === 1 || orientation === 2 || orientation === 3 || orientation === 4 + ? { sequenceOrientation: orientation } + : {}), + }; } function buildSequenceStepError( diff --git a/packages/apple-runner/src/runner-sequence.ts b/packages/apple-runner/src/runner-sequence.ts index 1496eba29c..12a06efd5c 100644 --- a/packages/apple-runner/src/runner-sequence.ts +++ b/packages/apple-runner/src/runner-sequence.ts @@ -1,6 +1,10 @@ import { pressJitter } from '@agent-device/contracts/touch-runtime'; import type { PressPointOptions } from '@agent-device/contracts/interactor-types'; -import { runnerSynthesizesTap, type DeviceInfo } from '@agent-device/kernel/device'; +import { + isHandheldAppleDevice, + runnerSynthesizesTap, + type DeviceInfo, +} from '@agent-device/kernel/device'; import { AppError, discloseDispatchAfterSteps } from '@agent-device/kernel/errors'; import { MAX_RUNNER_SEQUENCE_STEPS, @@ -35,10 +39,14 @@ export async function runApplePressSeries( let completedSteps = 0; const sequenceResults: unknown[] = []; let stepOffset = 0; + let sequenceOrientation: RunnerCommand['sequenceOrientation']; + const bindsOrientation = isHandheldAppleDevice(device) && commands.length > 1; for (const command of commands) { let result: Record; try { - result = await runCommand(command); + result = await runCommand( + sequenceOrientation === undefined ? command : { ...command, sequenceOrientation }, + ); } catch (error) { throw discloseDispatchAfterSteps(error, completedSteps); } @@ -57,6 +65,19 @@ export async function runApplePressSeries( completedSteps += parsed.completedSteps; sequenceResults.push(...parsed.results); stepOffset += command.steps?.length ?? 0; + if (bindsOrientation && sequenceOrientation === undefined) { + if (parsed.sequenceOrientation === undefined) { + throw discloseDispatchAfterSteps( + new AppError('COMMAND_FAILED', 'The runner did not return the press series orientation', { + reason: 'runner_sequence_orientation_unavailable', + completedSteps, + hint: 'Run snapshot -i to inspect the current UI before continuing with the remaining actions.', + }), + completedSteps, + ); + } + sequenceOrientation = parsed.sequenceOrientation; + } } // Preserve the first response's acquisition frame and the last response's gesture completion. return { diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts index f1461b60bb..b1c76fc015 100644 --- a/packages/platform-apple/src/foldable/runner-panel.test.ts +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -265,6 +265,7 @@ test('every press-series chunk uses the lit-panel naming path', async () => { runCommand: async (target, command, options) => { await base.runCommand(target, command, options); return { + sequenceOrientation: 1, completedSteps: command.steps?.length ?? 0, sequenceResults: command.steps?.map(({ kind }) => ({ ok: true, kind })) ?? [], }; From ef5326cc53e33f7034cb7747d66c9c0a7bc3d78f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 16:40:31 +0100 Subject: [PATCH 19/22] refactor(apple): isolate required series orientation check --- packages/apple-runner/src/runner-sequence.ts | 33 +++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/packages/apple-runner/src/runner-sequence.ts b/packages/apple-runner/src/runner-sequence.ts index 12a06efd5c..f9cf022723 100644 --- a/packages/apple-runner/src/runner-sequence.ts +++ b/packages/apple-runner/src/runner-sequence.ts @@ -11,6 +11,7 @@ import { buildRunnerSequenceCommand, parseRunnerSequenceResult, type RunnerCommand, + type RunnerSequenceOrientation, type RunnerSequenceStep, } from './runner-contract.ts'; @@ -65,18 +66,11 @@ export async function runApplePressSeries( completedSteps += parsed.completedSteps; sequenceResults.push(...parsed.results); stepOffset += command.steps?.length ?? 0; - if (bindsOrientation && sequenceOrientation === undefined) { - if (parsed.sequenceOrientation === undefined) { - throw discloseDispatchAfterSteps( - new AppError('COMMAND_FAILED', 'The runner did not return the press series orientation', { - reason: 'runner_sequence_orientation_unavailable', - completedSteps, - hint: 'Run snapshot -i to inspect the current UI before continuing with the remaining actions.', - }), - completedSteps, - ); - } - sequenceOrientation = parsed.sequenceOrientation; + if (bindsOrientation) { + sequenceOrientation ??= requirePressSeriesOrientation( + parsed.sequenceOrientation, + completedSteps, + ); } } // Preserve the first response's acquisition frame and the last response's gesture completion. @@ -91,6 +85,21 @@ export async function runApplePressSeries( }; } +function requirePressSeriesOrientation( + orientation: RunnerSequenceOrientation | undefined, + completedSteps: number, +): RunnerSequenceOrientation { + if (orientation !== undefined) return orientation; + throw discloseDispatchAfterSteps( + new AppError('COMMAND_FAILED', 'The runner did not return the press series orientation', { + reason: 'runner_sequence_orientation_unavailable', + completedSteps, + hint: 'Run snapshot -i to inspect the current UI before continuing with the remaining actions.', + }), + completedSteps, + ); +} + /** * Chunks by both the runner step cap and estimated wall time so no request approaches the 30s * main-thread watchdog. A single over-budget step remains a one-step chunk. From eec9ac56ab599a395d83bd65f6da84cdd392f769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 16:54:38 +0100 Subject: [PATCH 20/22] test(apple): drive bound sequences through the interactor --- contracts/fixtures/runner-requests.json | 2 +- .../src/__tests__/runner-sequence.test.ts | 31 --------------- .../press-series-runner-requests.test.ts | 39 +++++++++++++++++++ .../platform-apple/src/interactions.test.ts | 1 + 4 files changed, 41 insertions(+), 32 deletions(-) create mode 100644 packages/platform-apple/src/__tests__/press-series-runner-requests.test.ts diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index 71c1fc1fe0..79d72ea12a 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -64,7 +64,7 @@ {"name": "ios-simulator.runner-disposal.shutdown", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "shutdown", "commandId": ""}}, {"name": "ios-simulator.runner-lifecycle-prepare.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": ""}}, {"name": "ios-simulator.runner-selector-query.label", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "querySelector", "selectorKey": "label", "selectorValue": "Go", "appBundleId": "com.example.app", "commandId": ""}}, - {"name": "ios-simulator.runner-sequence.bound-orientation", "producer": "packages/apple-runner/src/__tests__/runner-sequence.test.ts", "request": {"command": "sequence", "steps": [{"kind": "tap", "x": 10, "y": 20, "synthesized": true}], "appBundleId": "com.example.app", "sequenceOrientation": 1, "commandId": ""}}, + {"name": "ios-simulator.runner-sequence.bound-orientation", "producer": "packages/platform-apple/src/__tests__/press-series-runner-requests.test.ts", "request": {"command": "sequence", "steps": [{"kind": "tap", "x": 10, "y": 20, "synthesized": true}], "appBundleId": "com.example.app", "sequenceOrientation": 1, "commandId": ""}}, {"name": "ios-simulator.runner-session-readiness.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": ""}}, {"name": "macos.desktop-scroll.amount", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "desktopScroll", "direction": "down", "amount": 0.5, "durationMs": 300, "appBundleId": "com.example.app", "commandId": ""}}, {"name": "macos.desktop-scroll.pixels", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "desktopScroll", "direction": "up", "pixels": 200, "durationMs": 300, "appBundleId": "com.example.app", "commandId": ""}}, diff --git a/packages/apple-runner/src/__tests__/runner-sequence.test.ts b/packages/apple-runner/src/__tests__/runner-sequence.test.ts index 88cbc4c1b1..2e31650546 100644 --- a/packages/apple-runner/src/__tests__/runner-sequence.test.ts +++ b/packages/apple-runner/src/__tests__/runner-sequence.test.ts @@ -4,37 +4,6 @@ import { AppError } from '@agent-device/kernel/errors'; import { runApplePressSeries } from '../runner-sequence.ts'; import { IOS_SIMULATOR } from './device-fixtures.ts'; import type { RunnerCommand } from '../runner-contract.ts'; -import { runAppleRunnerCommand } from '../index.ts'; -import { withAppleRunnerProvider } from '../runner-provider.ts'; -import { assertProducedRunnerRequests } from '@agent-device/test-kit'; - -test('press-series continuations carry the original orientation on the runner wire', async () => { - const requests: RunnerCommand[] = []; - await withAppleRunnerProvider( - async (_device, command) => { - requests.push(command); - return { - sequenceOrientation: 1, - completedSteps: command.steps?.length ?? 0, - sequenceResults: command.steps?.map(({ kind }) => ({ ok: true, kind })) ?? [], - }; - }, - { deviceId: IOS_SIMULATOR.id }, - async () => { - await runApplePressSeries( - IOS_SIMULATOR, - { x: 10, y: 20 }, - { button: 'primary', count: 21, intervalMs: 0, holdMs: 0, jitterPx: 0, doubleTap: false }, - 'com.example.app', - async (command) => await runAppleRunnerCommand(IOS_SIMULATOR, command, {}), - ); - }, - ); - assert.equal(requests.length, 2); - assertProducedRunnerRequests(import.meta.filename, [ - ['ios-simulator.runner-sequence.bound-orientation', requests[1]], - ]); -}); test.each([ { boundary: 'step cap', count: 21, intervalMs: 0, completed: 20 }, diff --git a/packages/platform-apple/src/__tests__/press-series-runner-requests.test.ts b/packages/platform-apple/src/__tests__/press-series-runner-requests.test.ts new file mode 100644 index 0000000000..8de418ca1b --- /dev/null +++ b/packages/platform-apple/src/__tests__/press-series-runner-requests.test.ts @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + assertProducedRunnerRequests, + bindTestCommandHost, + IOS_SIMULATOR, +} from '@agent-device/test-kit'; +import type { RunnerCommand } from '@agent-device/apple-runner'; +import { createAppleInteractor } from '../interactor.ts'; + +bindTestCommandHost(); + +test('press-series continuations carry the original orientation on the runner wire', async () => { + const requests: RunnerCommand[] = []; + const interactor = createAppleInteractor( + IOS_SIMULATOR, + { appBundleId: 'com.example.app' }, + { env: process.env, cwd: process.cwd() }, + { + hasLiveSession: () => true, + runCommand: async (_device, request) => { + requests.push(request); + return { + sequenceOrientation: 1, + completedSteps: request.steps?.length ?? 0, + sequenceResults: request.steps?.map(({ kind }) => ({ ok: true, kind })) ?? [], + }; + }, + }, + ); + await interactor.pressPoint!( + { x: 10, y: 20 }, + { button: 'primary', count: 21, intervalMs: 0, holdMs: 0, jitterPx: 0, doubleTap: false }, + ); + assert.equal(requests.length, 2); + assertProducedRunnerRequests(import.meta.filename, [ + ['ios-simulator.runner-sequence.bound-orientation', requests[1]], + ]); +}); diff --git a/packages/platform-apple/src/interactions.test.ts b/packages/platform-apple/src/interactions.test.ts index 2d792780e4..30fb29a0fd 100644 --- a/packages/platform-apple/src/interactions.test.ts +++ b/packages/platform-apple/src/interactions.test.ts @@ -299,6 +299,7 @@ test('iosRunnerOverrides owns alternate-button presses', async () => { test('iosRunnerOverrides remaps a later chunk failure to global press indices', async () => { mockRunAppleRunnerCommand .mockResolvedValueOnce({ + sequenceOrientation: 1, completedSteps: 20, sequenceResults: Array.from({ length: 20 }, () => ({ ok: true, kind: 'tap' })), }) From 84f1e6adccd54e4131d763c5c65f4880b6ac91f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 17:04:09 +0100 Subject: [PATCH 21/22] test(apple): name sequence wire drive after interactions --- contracts/fixtures/runner-requests.json | 2 +- ...est.ts => interactions-press-series-runner-requests.test.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename packages/platform-apple/src/__tests__/{press-series-runner-requests.test.ts => interactions-press-series-runner-requests.test.ts} (100%) diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index 79d72ea12a..a4be401fd4 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -64,7 +64,7 @@ {"name": "ios-simulator.runner-disposal.shutdown", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "shutdown", "commandId": ""}}, {"name": "ios-simulator.runner-lifecycle-prepare.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": ""}}, {"name": "ios-simulator.runner-selector-query.label", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "querySelector", "selectorKey": "label", "selectorValue": "Go", "appBundleId": "com.example.app", "commandId": ""}}, - {"name": "ios-simulator.runner-sequence.bound-orientation", "producer": "packages/platform-apple/src/__tests__/press-series-runner-requests.test.ts", "request": {"command": "sequence", "steps": [{"kind": "tap", "x": 10, "y": 20, "synthesized": true}], "appBundleId": "com.example.app", "sequenceOrientation": 1, "commandId": ""}}, + {"name": "ios-simulator.runner-sequence.bound-orientation", "producer": "packages/platform-apple/src/__tests__/interactions-press-series-runner-requests.test.ts", "request": {"command": "sequence", "steps": [{"kind": "tap", "x": 10, "y": 20, "synthesized": true}], "appBundleId": "com.example.app", "sequenceOrientation": 1, "commandId": ""}}, {"name": "ios-simulator.runner-session-readiness.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": ""}}, {"name": "macos.desktop-scroll.amount", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "desktopScroll", "direction": "down", "amount": 0.5, "durationMs": 300, "appBundleId": "com.example.app", "commandId": ""}}, {"name": "macos.desktop-scroll.pixels", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "desktopScroll", "direction": "up", "pixels": 200, "durationMs": 300, "appBundleId": "com.example.app", "commandId": ""}}, diff --git a/packages/platform-apple/src/__tests__/press-series-runner-requests.test.ts b/packages/platform-apple/src/__tests__/interactions-press-series-runner-requests.test.ts similarity index 100% rename from packages/platform-apple/src/__tests__/press-series-runner-requests.test.ts rename to packages/platform-apple/src/__tests__/interactions-press-series-runner-requests.test.ts From 2b9940af2a9e3e91c44e14535c026f96a2c2ac41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 4 Nov 2026 15:32:15 +0100 Subject: [PATCH 22/22] docs: record the natural rejected-query route on a sole-display iPhone --- .../surface-geometry-4189-2026-11-03.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/evidence/surface-geometry-4189-2026-11-03.md b/docs/evidence/surface-geometry-4189-2026-11-03.md index 6b5858fbc2..86bdff8ce1 100644 --- a/docs/evidence/surface-geometry-4189-2026-11-03.md +++ b/docs/evidence/surface-geometry-4189-2026-11-03.md @@ -70,3 +70,32 @@ The first full gate at `3090b336` rejected ten new native duplicate blocks. The follow-up consolidates drag dispatch, passes tap points as `CGPoint`, resolves point orientation with its target, and shares screen-query and preparation fixtures. Two existing fallback/alert stubs were also updated to the current native selector. The live observations above precede this API consolidation; final-head native and gate results are recorded in the PR validation section. + +## Natural rejected-query route, 2026-11-04 + +Source under test: `84f1e6adccd54e4131d763c5c65f4880b6ac91f9`, the branch rebased onto `main` at `7d42395ff7e1c03b643c2cf56683808bf5427cf3`. +This section closes the `AX-unavailable iPhone routes` limit above and answers [the readiness review](https://github.com/callstack/agent-device/pull/4189#issuecomment-6627588706). +No production source, and no fault patch, was involved: the app's Heavy feed fixture exceeds the depth XCTest can serialize, so the accessibility query refuses on its own. + +Setup: sole-display `iPhone 17 Pro` simulator `5FEADD02-98E4-4F01-861C-07003C1A3291`, iOS 26.2, driven through the command returned by `device_open` in session `surface4189`. +A daemon from this checkout (`0.21.37-dev`, `codeOrigin=checkout`, signature `graph:123:89e270aa80446b4444ddc2a562ce3e7ee7b4b469`) owned a private state directory `/private/tmp/agent-device-4189-ax` and built the exact-head runner into the pinned derived path `.tmp/4189-ax-derived`. +`Agent Device Tester` was that simulator's existing install with its JavaScript repacked from this checkout; bundle hashes: original `ca4c0c3d5eedf000792a143c10f3c036309d9f16d5d4c2899fcf3e053f26ffab`, repacked `ac93ac57877b43f52dbce40abac65d4711af339b24f3629fb1cd9af01223448d`. The original bundle was reinstalled afterward. + +`back --in-app` on the heavy feed took the early rejected-query route. One `kAXErrorIllegalArgument` rejection was recorded, then `IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION`, then `SYNTHESIZED_SOLE_SCREEN_ROUTE resolved={{0, 0}, {402, 874}}`. +No further rejection was recorded before delivery, so the route opened without another windows query. +Delivery logged `BACK_FALLBACK_TOP_INSET read=62.0 windows=1`, a tap at `(32.2,84.0)` against reference `(0,0,402,874)` in orientation `1`, and a record naming display `1`. +`IN_APP_BACK_VISUAL_VERIFICATION … changed=yes` reported the visible navigation outcome, and a follow-up read of `heavy-feed-title` refused because the screen had left the feed. + +In the same state, three gestures reached the sole-screen route through their own refused windows query, each preceded by its own recorded rejection: + +| Command | Native evidence | Fresh outcome | +| --- | --- | --- | +| `press 273.33 458.5` | record `agent-device-tap`, display `1`, orientation `1`, point `(273.3,458.5)` | `Liked posts: 0` became `Liked posts: 1` | +| `longpress 273.33 458.5 900` | record `agent-device-gesture-plan`, display `1`, orientation `1` | `heavy-feed-last-action` became `Last action: unliked post 0`: the hold's contact toggled the same like control the tap had pressed | +| `gesture pan 201 400 0 -220 800` | record `agent-device-gesture-plan`, display `1`, orientation `1` | `heavy-post-2-like` moved from y `1001.3` to y `798.3` | + +Limits of this run. Only `back --in-app` passes `windowQueryRejected` into the coordinate context, so the gesture rows prove the sole-screen route after a refused windows query, not the early argument. +The pan ran the sampled plan; `SYNTHESIZED_DISPATCH kind=longPress` is shared by both hold routes, so the record name separates the hold from the tap. +This covers one sole-display iPhone on the default fixture shape: not a foldable, a physical device, a scoped simulator set, or a global accessibility outage. Snapshots still answered through the private route. + +Cleanup: the session and the owned daemon were closed, Metro stopped, the original app bundle was reinstalled, and the simulator was left booted with its claim released.