diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index fa17a6bd0b..b712353a15 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -1,4 +1,5 @@ import XCTest +import AgentDeviceSnapshotPresentation // The lit panel of the command in flight (ADR 0025), main-owned and written behind main work a // previous command left queued, so that work still reads its own command's panel. @@ -11,7 +12,7 @@ struct LitPanelSurface { let key: LitPanelBookingKey let displayID: UInt let window: XCUIElement - let frame: CGRect + let geometry: SurfaceGeometry } /// The command's lit panel and what it booked on it. Read and written only on the main thread. @@ -79,7 +80,7 @@ extension RunnerTests { guard failure == .none, let element = window as? XCUIElement else { return RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure) } - litPanelState.surface = LitPanelSurface(key: key, displayID: litPanel.displayID, window: element, frame: frame) + litPanelState.surface = LitPanelSurface(key: key, displayID: litPanel.displayID, window: element, geometry: surfaceGeometry(windowFrame: frame, app: app)) return nil } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift index 955f9389fd..5e6379afeb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift @@ -32,14 +32,7 @@ enum RunnerScrollKeyboardClip: Equatable { /** Where one directional scroll may place its swipe, once the keyboard has taken its share. */ enum RunnerScrollViewport { - /** - * The band to plan the swipe inside, the frame to rotate its coordinates against, and the keyboard - * top when the band was clipped for one. The two frames are separate on purpose: a clip shortens - * only the band, while `CoordinateSpaceRotation.native(point:)` derives a `landscapeRight` native x from the - * frame's HEIGHT, so rotating inside the band moves the dispatched path sideways off the planned - * one. - */ - case swipe(planFrame: CGRect, coordinateFrame: CGRect, keyboardMinY: Double?) + case swipe(planFrame: CGRect, keyboardMinY: Double?) /** Nothing to swipe. The caller answers `occlusionRunnerCode` and performs no gesture. */ case occluded(keyboardMinY: Double, visibleHeight: Double) } @@ -49,7 +42,6 @@ enum RunnerScrollViewport { struct ScrollGestureDispatch { let plan: RunnerScrollGesturePlan let planFrame: CGRect - let coordinateFrame: CGRect let keyboardMinY: Double? } @@ -91,7 +83,7 @@ extension RunnerScrollViewport { switch self { case .occluded(let keyboardMinY, let visibleHeight): return .occluded(keyboardMinY: keyboardMinY, visibleHeight: visibleHeight) - case .swipe(let planFrame, let coordinateFrame, let keyboardMinY): + case .swipe(let planFrame, let keyboardMinY): guard planFrame.width > 0, planFrame.height > 0 else { return .unusableFrame } @@ -108,7 +100,6 @@ extension RunnerScrollViewport { ScrollGestureDispatch( plan: plan, planFrame: planFrame, - coordinateFrame: coordinateFrame, keyboardMinY: keyboardMinY ) ) @@ -168,9 +159,9 @@ enum ScrollViewportPolicy { static func frames(referenceFrame: CGRect, clip: RunnerScrollKeyboardClip) -> RunnerScrollViewport { switch clip { case .unobstructed: - return .swipe(planFrame: referenceFrame, coordinateFrame: referenceFrame, keyboardMinY: nil) + return .swipe(planFrame: referenceFrame, keyboardMinY: nil) case .avoided(let frame, let keyboardMinY): - return .swipe(planFrame: frame, coordinateFrame: referenceFrame, keyboardMinY: keyboardMinY) + return .swipe(planFrame: frame, keyboardMinY: keyboardMinY) case .occluded(let keyboardMinY, let visibleHeight): return .occluded(keyboardMinY: keyboardMinY, visibleHeight: visibleHeight) } @@ -195,17 +186,17 @@ extension RunnerTests { // Every scroll reports its decision, including the two ways it avoids reading the keyboard at // all: unavailable accessibility health, and a probe that finds no keyboard. guard context.allowsKeyboardProbe else { - logScrollViewport(decision: "probeSkipped", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context) - return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: .unobstructed) + logScrollViewport(decision: "probeSkipped", keyboardMinY: nil, swipeHeight: context.geometry.windowOnScreen.rect.height, context: context) + return ScrollViewportPolicy.frames(referenceFrame: context.geometry.windowOnScreen.rect, clip: .unobstructed) } guard let keyboardFrame = visibleKeyboardFrame(app: app) else { - logScrollViewport(decision: "noKeyboard", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context) - return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: .unobstructed) + logScrollViewport(decision: "noKeyboard", keyboardMinY: nil, swipeHeight: context.geometry.windowOnScreen.rect.height, context: context) + return ScrollViewportPolicy.frames(referenceFrame: context.geometry.windowOnScreen.rect, clip: .unobstructed) } - let clip = ScrollViewportPolicy.clip(viewport: context.referenceFrame, keyboard: keyboardFrame) + let clip = ScrollViewportPolicy.clip(viewport: context.geometry.windowOnScreen.rect, keyboard: keyboardFrame) switch clip { case .unobstructed: - logScrollViewport(decision: "unobstructed", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context) + logScrollViewport(decision: "unobstructed", keyboardMinY: nil, swipeHeight: context.geometry.windowOnScreen.rect.height, context: context) case .avoided(let frame, let keyboardMinY): logScrollViewport( decision: "avoided", @@ -221,7 +212,7 @@ extension RunnerTests { context: context ) } - return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: clip) + return ScrollViewportPolicy.frames(referenceFrame: context.geometry.windowOnScreen.rect, clip: clip) #else let fallbackFrame = try resolvedTouchReferenceFrame(app: app, appFrame: app.frame) return ScrollViewportPolicy.frames(referenceFrame: fallbackFrame, clip: .unobstructed) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h index 640e9c0bcf..4ae66d4deb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h @@ -11,31 +11,28 @@ typedef NS_ENUM(NSInteger, RunnerTapSynthesisStatus) { RunnerTapSynthesisStatusFailed, }; +typedef struct { + CGPoint start; + CGPoint end; + double durationMs; + BOOL controlledScroll; +} RunnerSynthesizedDrag; + @interface RunnerSynthesizedGesture : NSObject // `resolvedWindow` is the app window the caller already resolved for the gesture's reference // frame, so the record's display ID is read from that same window. Pass nil for the sole-screen // route: the app's tree could not name its window and the screen is proven instead. -+ (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application - resolvedWindow:(id _Nullable)resolvedWindow - x:(double)x - y:(double)y - x2:(double)x2 - y2:(double)y2 - durationMs:(double)durationMs; - -+ (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application - resolvedWindow:(id _Nullable)resolvedWindow - x:(double)x - y:(double)y - x2:(double)x2 - y2:(double)y2 - durationMs:(double)durationMs; +// `interfaceOrientation` is the orientation used to convert the supplied points. ++ (NSString * _Nullable)synthesizeDragWithApplication:(id)application + resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation + drag:(RunnerSynthesizedDrag)drag; + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow - x:(double)x - y:(double)y + interfaceOrientation:(NSInteger)interfaceOrientation + point:(CGPoint)point deadline:(NSDate * _Nullable)deadline errorMessage:(NSString * _Nullable * _Nullable)errorMessage; @@ -43,6 +40,7 @@ typedef NS_ENUM(NSInteger, RunnerTapSynthesisStatus) { // starts contact; subsequent samples move it; all pointers lift at their final offset. + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation pointerSamples:(NSArray *> *> *)pointerSamples; // UIInterfaceOrientation of the app (1 portrait, 2 upsideDown, 3 landscapeRight, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m index fa61a820cc..e19fc7b607 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m @@ -17,7 +17,6 @@ typedef struct { RunnerXCTestEventBridge core; SEL initRecordSelector; - SEL interfaceOrientationSelector; SEL initPathSelector; SEL moveSelector; SEL liftSelector; @@ -37,6 +36,7 @@ typedef id (*RunnerDragPointerPathFactory)( static NSString * _Nullable RunnerCreateEventRecord( id application, id _Nullable resolvedWindow, + NSInteger interfaceOrientation, NSString *recordName, RunnerGestureEventBridge *bridge, id *record @@ -60,6 +60,7 @@ static id RunnerControlledScrollPointerPath( static NSString * _Nullable RunnerTrySynthesizeDrag( id application, id _Nullable resolvedWindow, + NSInteger interfaceOrientation, CGPoint start, CGPoint end, double durationMs, @@ -69,6 +70,7 @@ static id RunnerControlledScrollPointerPath( static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( id application, id _Nullable resolvedWindow, + NSInteger interfaceOrientation, CGPoint point, NSDate * _Nullable deadline, NSString * _Nullable * _Nullable errorMessage @@ -85,44 +87,15 @@ static id RunnerTapPointerPath( @implementation RunnerSynthesizedGesture -+ (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application - resolvedWindow:(id _Nullable)resolvedWindow - x:(double)x - y:(double)y - x2:(double)x2 - y2:(double)y2 - durationMs:(double)durationMs { - @try { - return RunnerTrySynthesizeDrag( - application, - resolvedWindow, - CGPointMake(x, y), - CGPointMake(x2, y2), - durationMs, - @"agent-device-swipe", - RunnerSwipePointerPath - ); - } @catch (NSException *exception) { - return RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed"); - } -} - -+ (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application - resolvedWindow:(id _Nullable)resolvedWindow - x:(double)x - y:(double)y - x2:(double)x2 - y2:(double)y2 - durationMs:(double)durationMs { ++ (NSString * _Nullable)synthesizeDragWithApplication:(id)application + resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation + drag:(RunnerSynthesizedDrag)drag { @try { return RunnerTrySynthesizeDrag( - application, - resolvedWindow, - CGPointMake(x, y), - CGPointMake(x2, y2), - durationMs, - @"agent-device-controlled-scroll", - RunnerControlledScrollPointerPath + application, resolvedWindow, interfaceOrientation, drag.start, drag.end, drag.durationMs, + drag.controlledScroll ? @"agent-device-controlled-scroll" : @"agent-device-swipe", + drag.controlledScroll ? RunnerControlledScrollPointerPath : RunnerSwipePointerPath ); } @catch (NSException *exception) { return RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed"); @@ -131,13 +104,13 @@ + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)applicatio + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow - x:(double)x - y:(double)y + interfaceOrientation:(NSInteger)interfaceOrientation + point:(CGPoint)point deadline:(NSDate * _Nullable)deadline errorMessage:(NSString * _Nullable * _Nullable)errorMessage { if (errorMessage != NULL) *errorMessage = nil; @try { - return RunnerTrySynthesizeTap(application, resolvedWindow, CGPointMake(x, y), deadline, errorMessage); + return RunnerTrySynthesizeTap(application, resolvedWindow, interfaceOrientation, point, deadline, errorMessage); } @catch (NSException *exception) { if (errorMessage != NULL) { *errorMessage = RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed"); @@ -148,6 +121,7 @@ + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application resolvedWindow:(id _Nullable)resolvedWindow + interfaceOrientation:(NSInteger)interfaceOrientation pointerSamples:(NSArray *> *> *)pointerSamples { @try { RunnerGestureEventBridge bridge; @@ -155,6 +129,7 @@ + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application NSString *error = RunnerCreateEventRecord( application, resolvedWindow, + interfaceOrientation, @"agent-device-gesture-plan", &bridge, &record @@ -210,6 +185,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { static NSString * _Nullable RunnerTrySynthesizeDrag( id application, id _Nullable resolvedWindow, + NSInteger interfaceOrientation, CGPoint start, CGPoint end, double durationMs, @@ -218,7 +194,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { ) { RunnerGestureEventBridge bridge; id record = nil; - NSString *error = RunnerCreateEventRecord(application, resolvedWindow, recordName, &bridge, &record); + NSString *error = RunnerCreateEventRecord(application, resolvedWindow, interfaceOrientation, recordName, &bridge, &record); if (error != nil) return error; id path = pathFactory(&bridge, start, end, durationMs); @@ -233,6 +209,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( id application, id _Nullable resolvedWindow, + NSInteger interfaceOrientation, CGPoint point, NSDate * _Nullable deadline, NSString * _Nullable * _Nullable errorMessage @@ -242,6 +219,7 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( NSString *error = RunnerCreateEventRecord( application, resolvedWindow, + interfaceOrientation, @"agent-device-tap", &bridge, &record @@ -274,7 +252,6 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( if (missing != nil) return missing; SEL initRecordSelector = NSSelectorFromString(@"initWithName:displayID:interfaceOrientation:"); - SEL interfaceOrientationSelector = NSSelectorFromString(@"interfaceOrientation"); SEL initPathSelector = NSSelectorFromString(@"initForTouchAtPoint:offset:"); SEL moveSelector = NSSelectorFromString(@"moveToPoint:atOffset:"); SEL liftSelector = NSSelectorFromString(@"liftUpAtOffset:"); @@ -295,15 +272,10 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( core.pathClass, liftSelector, @"liftUpAtOffset:", RunnerGestureSynthesisSurface ); if (missing != nil) return missing; - missing = RunnerRequireApplicationSelector( - application, interfaceOrientationSelector, @"interfaceOrientation", RunnerGestureSynthesisSurface - ); - if (missing != nil) return missing; *bridge = (RunnerGestureEventBridge){ .core = core, .initRecordSelector = initRecordSelector, - .interfaceOrientationSelector = interfaceOrientationSelector, .initPathSelector = initPathSelector, .moveSelector = moveSelector, .liftSelector = liftSelector, @@ -314,6 +286,7 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( static NSString * _Nullable RunnerCreateEventRecord( id application, id _Nullable resolvedWindow, + NSInteger interfaceOrientation, NSString *recordName, RunnerGestureEventBridge *bridge, id *record @@ -329,8 +302,6 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap( ? RunnerResolveSoleScreen(application, &soleScreen, &displayID) : RunnerResolveWindowDisplayID(resolvedWindow, &displayID); if (displayError != nil) return displayError; - NSInteger interfaceOrientation = - ((RunnerMsgSendInteger)objc_msgSend)(application, bridge->interfaceOrientationSelector); NSInteger targetProcessID = ((RunnerMsgSendInteger)objc_msgSend)(application, bridge->core.processIDSelector); if (targetProcessID <= 0) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index decf0f6b05..c144245138 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -211,7 +211,7 @@ extension RunnerTests { if let window = alert.window { // Never fall back to the owner app's first window: on a foldable it can be another panel. guard let context = SynthesizedCoordinateContext( - validating: safely("ALERT_WINDOW", CGRect.null) { window.frame }, + validating: surfaceGeometry(windowFrame: safely("ALERT_WINDOW", CGRect.null) { window.frame }, app: alert.ownerApp), resolvedWindow: window, accessibilityHealth: mainOwned.accessibilityHealth ) else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 6ef1f29369..1fc524fbb1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -247,7 +247,7 @@ extension RunnerTests { try synthesizedLongPressAt(app: activeApp, x: x, y: y, durationMs: durationMs, context: context) }) { case .performed(let timing): - let touchFrame = context.map { touchVisualizationFrame(x: x, y: y, context: $0) } + let touchFrame = context.map { touchVisualizationFrame(point: SurfacePoint(CGPoint(x: x, y: y)), context: $0) } return gestureResponse(message: "long pressed", timing: timing, frame: .touch(touchFrame)) case .xctestFallback(let message, let hint): fallback = GestureFallback(strategy: "xctest-coordinate-long-press", message: message, hint: hint) @@ -338,7 +338,7 @@ extension RunnerTests { y2: gesture.planFrame.minY + gesture.plan.y2, durationMs: defaults.durationMs, message: "scrolled", - context: scrollContext.withReferenceFrame(gesture.coordinateFrame), + context: scrollContext, releaseBehavior: command.scrollReleaseBehavior ) ) @@ -680,7 +680,7 @@ extension RunnerTests { ) case .sampled: let (timing, outcome) = try performGesture(activeApp, idleTimeout: false) { - sampledPlannedGesture(app: activeApp, plan: plan) + try sampledPlannedGesture(app: activeApp, plan: plan) } return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index dd469d9c6a..ab24ad6f58 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -55,9 +55,6 @@ extension RunnerTests { let points: DragPoints let context: SynthesizedCoordinateContext - var referenceFrame: CGRect { - context.referenceFrame - } } struct SelectorElementMatch { @@ -240,7 +237,7 @@ extension RunnerTests { // lit-panel request uses its booked window, because app.frame names one // panel (ADR 0025). private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) throws -> Bool { - let windowFrame = try requestLitPanel == nil ? app.frame : requireRequestLitPanelSurface(app: app).frame + let windowFrame = try requestLitPanel == nil ? app.frame : requireRequestLitPanelSurface(app: app).geometry.windowOnScreen.rect let frame = element.frame if frame.isEmpty { return false @@ -270,7 +267,7 @@ extension RunnerTests { func resolveRunnerWindow(app: XCUIApplication) throws -> (window: XCUIElement?, frame: CGRect) { if requestLitPanel != nil { let booked = try requireRequestLitPanelSurface(app: app) - return (booked.window, booked.frame) + return (booked.window, booked.geometry.windowOnScreen.rect) } guard let resolved = resolveRunnerWindowElement(app: app) else { return (nil, app.frame) } return (resolved.window, resolved.frame) @@ -282,7 +279,7 @@ extension RunnerTests { func resolveRunnerWindowElement(app: XCUIApplication) -> (window: XCUIElement, frame: CGRect)? { if requestLitPanel != nil { guard bookRequestLitPanelSurface(app: app) == nil, let booked = litPanelState.surface else { return nil } - return (booked.window, booked.frame) + return (booked.window, booked.geometry.windowOnScreen.rect) } for window in app.windows.allElementsBoundByIndex where window.exists { let frame = window.frame @@ -648,7 +645,8 @@ extension RunnerTests { } let booked = try requireRequestLitPanelSurface(app: app) let origin = booked.window.coordinate(withNormalizedOffset: .zero) - return origin.withOffset(CGVector(dx: x - booked.frame.minX, dy: y - booked.frame.minY)) + let point = booked.geometry.contentPoint(from: SurfacePoint(CGPoint(x: x, y: y))).point + return origin.withOffset(CGVector(dx: point.x, dy: point.y)) #else let resolved = try resolveRunnerWindow(app: app) let root = resolved.window ?? app diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 4b89a2c814..d2920e728e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -277,6 +277,7 @@ struct Command: Codable { let synthesized: Bool? let coordinateTapTextInputEvidence: CoordinateTapTextInputEvidence? let litPanel: RunnerLitPanelFact? + let sequenceOrientation: Int? let steps: [SequenceStep]? } @@ -530,6 +531,7 @@ struct DataPayload: Codable { /// daemon read `ok` as proof the runner drained (#2552). var runnerMainThreadBusy: Bool? var completedSteps: Int? + var sequenceOrientation: Int? var failedStepIndex: Int? var sequenceResults: [SequenceStepResult]? var targetActivation: TargetActivationFactPayload? diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index f35f0d16f6..fd85b8b781 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -228,7 +228,7 @@ extension RunnerTests { app: app, windowQueryRejected: windowQueryRejected ), let point = Self.topLeadingNavigationFallbackPoint( - in: context.referenceFrame, + in: context.geometry.windowOnScreen.rect, topSafeAreaInset: screenTopSafeAreaInset() ) else { return .unavailable diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift index 4b1ae239cc..87c8911dcf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift @@ -188,8 +188,8 @@ extension RunnerTests { y: plan.points.y, x2: plan.points.x2, y2: plan.points.y2, - referenceWidth: Double(plan.referenceFrame.width), - referenceHeight: Double(plan.referenceFrame.height) + referenceWidth: Double(plan.context.geometry.windowOnScreen.rect.width), + referenceHeight: Double(plan.context.geometry.windowOnScreen.rect.height) ) switch try performSynthesizedGesture(activeApp, kind: policyKind, context: plan.context, synthesize: { try synthesizedDragAt( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 11b714522d..f8a32a5f94 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -1,4 +1,5 @@ import XCTest +import AgentDeviceSnapshotPresentation extension RunnerTests { // MARK: - Sequence command @@ -41,14 +42,28 @@ extension RunnerTests { // synthesized context already names the frame, so the app tree is not read for it again. let firstStep = steps[0] let firstFrame = try (firstStep.x != nil && firstStep.y != nil) - ? synthesizedContext.map { touchVisualizationFrame(x: firstStep.x!, y: firstStep.y!, context: $0) } + ? synthesizedContext.map { touchVisualizationFrame(point: SurfacePoint(CGPoint(x: firstStep.x!, y: firstStep.y!)), context: $0) } ?? resolvedTouchVisualizationFrame(app: activeApp, x: firstStep.x!, y: firstStep.y!) : nil +#if os(iOS) + let orientation = command.sequenceOrientation ?? synthesizedContext?.geometry.orientation ?? litPanelState.surface?.geometry.orientation + ?? Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: activeApp)) +#else + let orientation: Int? = nil +#endif let execution = try assembleSequenceExecution(steps: steps) { _, step in - try performSequenceStep(step, activeApp: activeApp, synthesizedContext: synthesizedContext) +#if os(iOS) + if orientation != Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: activeApp)) { + return finishedSequenceStep(step, timing: measureGesture {}, outcome: .unsupported( + message: "sequence stopped because the screen orientation changed", + hint: "Inspect the screen before continuing with the remaining actions." + )) + } +#endif + return try performSequenceStep(step, activeApp: activeApp, synthesizedContext: synthesizedContext) } - return sequenceResponse(execution: execution, touchFrame: firstFrame) + return sequenceResponse(execution: execution, touchFrame: firstFrame, sequenceOrientation: orientation) } /// Pure, device-free assembler: runs each step in order via `perform`, stops at the first @@ -230,7 +245,8 @@ extension RunnerTests { private func sequenceResponse( execution: SequenceExecutionResult, - touchFrame: TouchVisualizationFrame? + touchFrame: TouchVisualizationFrame?, + sequenceOrientation: Int? ) -> Response { return Response( ok: true, @@ -243,6 +259,7 @@ extension RunnerTests { referenceWidth: touchFrame?.referenceWidth, referenceHeight: touchFrame?.referenceHeight, completedSteps: execution.completedSteps, + sequenceOrientation: sequenceOrientation, failedStepIndex: execution.failedStepIndex, sequenceResults: execution.results ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 82f6d386cc..503d6916df 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -299,7 +299,7 @@ extension RunnerTests { /// A lit-panel snapshot's root, raw or regular, is the booked box (ADR 0025). static func publishLitPanelRoot(_ root: inout RawAXNode, surface: LitPanelSurface?) { if let surface { - root.rect = SnapshotRect(surface.frame) + root.rect = SnapshotRect(surface.geometry.windowOnScreen.rect) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index df29a9509d..65917943d1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -38,22 +38,10 @@ enum SynthesizedGesturePolicyKind: String, Equatable, Hashable { } struct SynthesizedCoordinateContext { - let referenceFrame: CGRect - /// The window `referenceFrame` was measured on. Synthesized records route their display ID - /// through this same window so geometry and routing can never name different windows. Nil is the - /// sole-screen route: XCTest could not resolve the window, `referenceFrame` is the device's only - /// screen, and the record is routed to that screen. + let geometry: SurfaceGeometry let resolvedWindow: XCUIElement? let accessibilityHealth: RunnerAccessibilityHealth - func withReferenceFrame(_ frame: CGRect) -> SynthesizedCoordinateContext { - SynthesizedCoordinateContext( - referenceFrame: frame, - resolvedWindow: resolvedWindow, - accessibilityHealth: accessibilityHealth - ) - } - /// The keyboard probe is not free — `visibleKeyboardFrame` resolves `app.keyboards.firstMatch` /// with a live AX fetch — so `.unavailable` skips it: there the fetch is known not to answer, /// and a gesture that needs the frame fails open on a missing one. `.unknown` still probes; @@ -205,15 +193,15 @@ extension RunnerTests { } extension SynthesizedCoordinateContext { - /// Nil when `referenceFrame` is not a finite, non-empty box a point can be placed in. + /// Nil when the window has no finite, non-empty box. init?( - validating referenceFrame: CGRect, + validating geometry: SurfaceGeometry, resolvedWindow: XCUIElement?, accessibilityHealth: RunnerAccessibilityHealth ) { - guard SnapshotGeometry.isPositiveFinite(referenceFrame) else { return nil } + guard SnapshotGeometry.isPositiveFinite(geometry.windowOnScreen.rect) else { return nil } self.init( - referenceFrame: referenceFrame, + geometry: geometry, resolvedWindow: resolvedWindow, accessibilityHealth: accessibilityHealth ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 4b11d38245..f644c06599 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -20,58 +20,32 @@ extension RunnerTests { hint: "Retry with finite x, y, x2, and y2 values." ) } - let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) guard let context = try context ?? synthesizedCoordinateContext(app: app) else { return .unsupported( message: "synthesized coordinate drag could not resolve an app window with a finite screen frame", hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." ) } - let frame = context.referenceFrame - guard - let rotationFrame = synthesizedRotationFrame(app: app, fallback: frame, interfaceOrientation: orientation) - else { + guard let geometry = digitizerGeometry(app: app, surface: context.geometry) else { return Self.unmeasuredSoleDisplayOutcome(gesture: "drag") } - let start = CoordinateSpaceRotation.native( - point: CGPoint(x: x, y: y), - in: rotationFrame, - interfaceOrientation: orientation - ) - let end = CoordinateSpaceRotation.native( - point: CGPoint(x: x2, y: y2), - in: rotationFrame, - interfaceOrientation: orientation - ) + let frame = context.geometry.windowOnScreen.rect + let start = geometry.digitizerPoint(from: SurfacePoint(CGPoint(x: x, y: y))).point + let end = geometry.digitizerPoint(from: SurfacePoint(CGPoint(x: x2, y: y2))).point logSynthesizedDispatch( kind: "drag", start: start, end: end, referenceFrame: frame, - orientation: orientation + orientation: context.geometry.orientation + ) + let message = RunnerSynthesizedGesture.synthesizeDrag( + withApplication: app, + resolvedWindow: context.resolvedWindow, + interfaceOrientation: geometry.orientation, + drag: RunnerSynthesizedDrag(start: start, end: end, durationMs: durationMs, + controlledScroll: ObjCBool(profile == .controlledScroll)) ) - let message = switch profile { - case .controlledScroll: - RunnerSynthesizedGesture.synthesizeControlledScroll( - withApplication: app, - resolvedWindow: context.resolvedWindow, - x: Double(start.x), - y: Double(start.y), - x2: Double(end.x), - y2: Double(end.y), - durationMs: durationMs - ) - case .fastSwipe: - RunnerSynthesizedGesture.synthesizeSwipe( - withApplication: app, - resolvedWindow: context.resolvedWindow, - x: Double(start.x), - y: Double(start.y), - x2: Double(end.x), - y2: Double(end.y), - durationMs: durationMs - ) - } if let message { return .unsupported( message: message, @@ -107,16 +81,15 @@ extension RunnerTests { hint: "Retry with finite x and y values." ) } - let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + let context = try context ?? synthesizedCoordinateContext(app: app) let resolution = pointGestureTarget( app: app, x: x, y: y, - orientation: orientation, - context: try context ?? synthesizedCoordinateContext(app: app), + context: context, gesture: .tap ) - guard case .target(let point, let resolvedWindow, let referenceFrame) = resolution else { + guard case .target(let point, let resolvedWindow, let referenceFrame, let orientation) = resolution else { return resolution.refusal } logSynthesizedDispatch( @@ -130,8 +103,8 @@ extension RunnerTests { let status = RunnerSynthesizedGesture.synthesizeTap( withApplication: app, resolvedWindow: resolvedWindow, - x: Double(point.x), - y: Double(point.y), + interfaceOrientation: orientation, + point: point, deadline: deadline, errorMessage: &message ) @@ -181,16 +154,14 @@ extension RunnerTests { hint: "Retry with finite x and y values and a positive durationMs." ) } - let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) let resolution = pointGestureTarget( app: app, x: x, y: y, - orientation: orientation, context: context, gesture: .longpress ) - guard case .target(let point, let resolvedWindow, let referenceFrame) = resolution else { + guard case .target(let point, let resolvedWindow, let referenceFrame, let orientation) = resolution else { return resolution.refusal } logSynthesizedDispatch( @@ -210,6 +181,7 @@ extension RunnerTests { if let message = RunnerSynthesizedGesture.synthesizeGesture( withApplication: app, resolvedWindow: resolvedWindow, + interfaceOrientation: orientation, pointerSamples: [samples] ) { return .unsupported( @@ -298,15 +270,14 @@ extension RunnerTests { /// The touch visualization frame of a synthesized gesture, read from its context instead of the /// app tree. func touchVisualizationFrame( - x: Double, - y: Double, + point: SurfacePoint, context: SynthesizedCoordinateContext ) -> TouchVisualizationFrame { TouchVisualizationFrame( - x: Double(context.referenceFrame.minX) + x, - y: Double(context.referenceFrame.minY) + y, - referenceWidth: Double(context.referenceFrame.width), - referenceHeight: Double(context.referenceFrame.height) + x: Double(point.point.x), + y: Double(point.point.y), + referenceWidth: Double(context.geometry.windowOnScreen.rect.width), + referenceHeight: Double(context.geometry.windowOnScreen.rect.height) ) } @@ -351,8 +322,8 @@ extension RunnerTests { } // The sole-screen route means the windows query just failed, so a keyboard probe would too. return context.resolvedWindow == nil - ? context.referenceFrame - : frameAvoidingKeyboard(app: app, frame: context.referenceFrame) + ? context.geometry.windowOnScreen.rect + : frameAvoidingKeyboard(app: app, frame: context.geometry.windowOnScreen.rect) #else return try resolvedTouchReferenceFrame(app: app, appFrame: app.frame) #endif @@ -457,7 +428,7 @@ extension RunnerTests { if requestLitPanel != nil { let booked = try requireRequestLitPanelSurface(app: app) guard let context = SynthesizedCoordinateContext( - validating: booked.frame, resolvedWindow: booked.window, accessibilityHealth: health + validating: booked.geometry, resolvedWindow: booked.window, accessibilityHealth: health ) else { throw RunnerAppScreenCaptureFailure.unresolvedScreen } return context } @@ -477,7 +448,7 @@ extension RunnerTests { return nil } return SynthesizedCoordinateContext( - validating: referenceFrame, + validating: surfaceGeometry(windowFrame: referenceFrame, app: app), resolvedWindow: window, accessibilityHealth: health ) @@ -518,28 +489,16 @@ extension RunnerTests { return RunnerResolveSoleScreen(app, &resolvedScreen, &resolvedDisplayID) == nil } - /// The box a synthesized turn swaps side lengths about. The digitizer turns about its display, so a - /// placed iPad window's smaller box is not the basis (#3899). A named lit panel supplies its full - /// screen box independently of the booked window translation (ADR 0025). When the device's only screen - /// will not report its size, `fallback` is the basis only while the app provably fills that - /// screen; otherwise this is nil and the gesture refuses before it dispatches. Portrait and an - /// unknown orientation leave the point as it is, so they keep `fallback` without a read; landscape - /// swaps the box's sides and upside-down mirrors the point across them, so both read it. @MainActor - func synthesizedRotationFrame(app: XCUIApplication, fallback: CGRect, interfaceOrientation: Int) -> CGRect? { - if let panel = requestLitPanel { - return Self.soleScreenReferenceFrame( - portraitSize: CGSize(width: panel.widthPt, height: panel.heightPt), - interfaceOrientation: interfaceOrientation - ) ?? fallback - } - guard XCUIScreen.screens.count == 1 else { return fallback } - switch interfaceOrientation { + func digitizerGeometry(app: XCUIApplication, surface: SurfaceGeometry) -> SurfaceGeometry? { + if surface.panelNativeSize != nil { return surface } + guard XCUIScreen.screens.count == 1 else { return surface } + switch surface.orientation { case RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight, RunnerInterfaceOrientation.portraitUpsideDown: break default: - return fallback + return surface } guard let portraitSize = soleDisplayPortraitSize() else { let routeAdmits = soleScreenRouteAdmits(app) @@ -547,10 +506,16 @@ extension RunnerTests { "AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=rotation-basis basis=%@", routeAdmits ? "window" : "refused" ) - return routeAdmits ? fallback : nil + return routeAdmits ? surface : nil } - return Self.soleScreenReferenceFrame(portraitSize: portraitSize, interfaceOrientation: interfaceOrientation) - ?? fallback + return SurfaceGeometry(windowOnScreen: surface.windowOnScreen, + panelNativeSize: portraitSize, orientation: surface.orientation) + } + + func surfaceGeometry(windowFrame: CGRect, app: XCUIApplication) -> SurfaceGeometry { + SurfaceGeometry(windowOnScreen: SurfaceRect(windowFrame), + panelNativeSize: requestLitPanel.map { CGSize(width: $0.widthPt, height: $0.heightPt) }, + orientation: Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))) } private enum SynthesizedPointGesture: String { @@ -563,7 +528,7 @@ extension RunnerTests { private enum PointGestureResolution { /// A nil `resolvedWindow` routes the record to the sole screen. - case target(point: CGPoint, resolvedWindow: XCUIElement?, referenceFrame: CGRect) + case target(point: CGPoint, resolvedWindow: XCUIElement?, referenceFrame: CGRect, orientation: Int) case refused(RunnerInteractionOutcome) /// Read only on the branch that is not a target, so its `.performed` answer is never returned. @@ -580,29 +545,19 @@ extension RunnerTests { app: XCUIApplication, x: Double, y: Double, - orientation: Int, context: SynthesizedCoordinateContext?, gesture: SynthesizedPointGesture ) -> PointGestureResolution { if let context { - guard - let rotationFrame = synthesizedRotationFrame( - app: app, - fallback: context.referenceFrame, - interfaceOrientation: orientation - ) - else { + guard let geometry = digitizerGeometry(app: app, surface: context.geometry) else { return .refused(Self.unmeasuredSoleDisplayOutcome(gesture: gesture.rawValue)) } - let point = CoordinateSpaceRotation.native( - point: CGPoint(x: x, y: y), - in: rotationFrame, - interfaceOrientation: orientation - ) - return .target(point: point, resolvedWindow: context.resolvedWindow, referenceFrame: context.referenceFrame) + let point = geometry.digitizerPoint(from: SurfacePoint(CGPoint(x: x, y: y))).point + return .target(point: point, resolvedWindow: context.resolvedWindow, referenceFrame: context.geometry.windowOnScreen.rect, orientation: geometry.orientation) } + let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) if let unrotated = unrotatedSoleScreenPoint(app: app, x: x, y: y, interfaceOrientation: orientation) { - return .target(point: unrotated, resolvedWindow: nil, referenceFrame: .null) + return .target(point: unrotated, resolvedWindow: nil, referenceFrame: .null, orientation: orientation) } return .refused( .unsupported( @@ -659,17 +614,15 @@ extension RunnerTests { portraitSize size: CGSize, interfaceOrientation: Int ) -> CGRect? { - guard size.width > 0, size.height > 0 else { return nil } - switch interfaceOrientation { - case RunnerInterfaceOrientation.portrait, RunnerInterfaceOrientation.portraitUpsideDown: - return CGRect(origin: .zero, size: size) - case RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight: - return CGRect(x: 0, y: 0, width: size.height, height: size.width) - default: - return nil - } + SurfaceGeometry.panelFrame(nativeSize: size, orientation: interfaceOrientation) } +#if !os(iOS) + func surfaceGeometry(windowFrame: CGRect, app: XCUIApplication) -> SurfaceGeometry { + SurfaceGeometry(windowOnScreen: SurfaceRect(windowFrame), panelNativeSize: nil, orientation: 0) + } +#endif + func swipe(app: XCUIApplication, direction: String) throws -> DragVisualizationFrame? { if performTvRemoteSwipeIfAvailable(direction: direction) { let frame = try resolvedTouchReferenceFrame(app: app, appFrame: app.frame) @@ -812,29 +765,17 @@ extension RunnerTests { func sampledPlannedGesture( app: XCUIApplication, plan: RunnerGesturePlan - ) -> RunnerInteractionOutcome { + ) throws -> RunnerInteractionOutcome { #if os(iOS) - let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - // The portable planner and validation use this exact viewport. Using app.frame here can - // diverge when XCTest unions transformed/off-screen descendants into the application frame. - let frame = CGRect( - x: plan.viewport.x, - y: plan.viewport.y, - width: plan.viewport.width, - height: plan.viewport.height - ) - guard - let rotationFrame = synthesizedRotationFrame(app: app, fallback: frame, interfaceOrientation: orientation) - else { + guard let context = try synthesizedCoordinateContext(app: app) else { + return .unsupported(message: "gesture could not resolve a surface", hint: nil) + } + guard let geometry = digitizerGeometry(app: app, surface: context.geometry) else { return Self.unmeasuredSoleDisplayOutcome(gesture: "gesture") } let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in pointer.samples.map { sample in - let point = CoordinateSpaceRotation.native( - point: CGPoint(x: sample.point.x, y: sample.point.y), - in: rotationFrame, - interfaceOrientation: orientation - ) + let point = geometry.digitizerPoint(from: SurfacePoint(CGPoint(x: sample.point.x, y: sample.point.y))).point return [ "x": NSNumber(value: Double(point.x)), "y": NSNumber(value: Double(point.y)), @@ -842,10 +783,10 @@ extension RunnerTests { ] } } - let resolvedWindow = resolveRunnerWindowElement(app: app)?.window if let message = RunnerSynthesizedGesture.synthesizeGesture( withApplication: app, - resolvedWindow: resolvedWindow, + resolvedWindow: context.resolvedWindow, + interfaceOrientation: geometry.orientation, pointerSamples: pointerSamples ) { return .unsupported( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index 7299948cef..5f4d794b32 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -123,7 +123,7 @@ extension RunnerTests { let point = target.refreshPoint let frame: CGRect if let resolvedCoordinateContext { - frame = resolvedCoordinateContext.referenceFrame + frame = resolvedCoordinateContext.geometry.windowOnScreen.rect } else if point != nil { frame = activeApp.frame } else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 9582c348e9..4bd70c86c7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -10,6 +10,26 @@ private final class LitPanelBookingScreen: NSObject { init(_ displayID: UInt = 3) { self.displayID = displayID } } +@MainActor +private final class SurfaceSelectorQuery: NSObject { + let elements: [XCUIElement] + init(_ elements: [XCUIElement]) { self.elements = elements } + @objc(matchingPredicate:) + func matching(_ predicate: NSPredicate) -> XCUIElementQuery { + unsafeBitCast(SurfaceSelectorQuery(elements.filter { predicate.evaluate(with: $0) }), to: XCUIElementQuery.self) + } + @objc var allElementsBoundByIndex: [XCUIElement] { elements } + @objc var firstMatch: XCUIElement { elements[0] } +} + +private struct RecordedSurfaceDispatch { + let window: AnyObject? + let displayID: UInt + let start: CGPoint + let end: CGPoint? + let orientation: Int +} + @MainActor private final class LitPanelBookingProbe: NSObject { static var originalValueForKey: IMP? @@ -18,6 +38,22 @@ private final class LitPanelBookingProbe: NSObject { static var maxWindowListReads: Int? static var window: XCUIElement? static var darkWindow: XCUIElement? + static var target: XCUIElement? + static var darkTarget: XCUIElement? + static var composedDarkWindow: DisplayWindowFixture? + static var dispatches: [RecordedSurfaceDispatch] = [] + static var synthesizedPoint: CGPoint? { dispatches.last?.start } + static var synthesizedDisplay: UInt? { dispatches.last?.displayID } + static var application: XCUIApplication? + static var applicationFrame: CGRect? + static var keyboard: XCUIElement? + static var keyboardFrame: CGRect? + static var targetFrame = CGRect(x: 190, y: 140, width: 20, height: 20) + static var windowDisplayID: UInt = 3 + static var includeDarkSelector = true + static var onDispatch: (() -> Void)? + static var beforeElementLookup: (() -> Void)? + static var coordinate: XCUICoordinate? static var windowFrame = CGRect.zero static var orientation = RunnerInterfaceOrientation.landscapeLeft @@ -29,25 +65,84 @@ private final class LitPanelBookingProbe: NSObject { nonisolated override func value(forKey key: String) -> Any? { MainActor.assumeIsolated { + let original = unsafeBitCast( + Self.originalValueForKey!, + to: (@convention(c) (AnyObject, Selector, NSString) -> AnyObject?).self + ) switch key { case "windows": return Self.query case "allElementsBoundByIndex": Self.windowListReads += 1 if let max = Self.maxWindowListReads, Self.windowListReads > max { return [] } + if let dark = Self.composedDarkWindow { return [dark, Self.window!] as [NSObject] } return [Self.window!, Self.darkWindow].compactMap { $0 } - case "screen": return LitPanelBookingScreen(self === Self.darkWindow ? 1 : 3) + case "screen": return LitPanelBookingScreen(self === Self.darkWindow ? 1 : Self.windowDisplayID) case "exists": return true - case "frame": return NSValue(cgRect: Self.windowFrame) + case "frame": return NSValue(cgRect: Self.elementFrame(self)) default: break } - let original = unsafeBitCast( - Self.originalValueForKey!, - to: (@convention(c) (AnyObject, Selector, NSString) -> AnyObject?).self - ) return original(self, #selector(NSObject.value(forKey:)), key as NSString) } } + @objc var exists: Bool { self !== Self.keyboard || Self.keyboardFrame != nil } + @objc var isHittable: Bool { true } + @objc var elementType: UInt { XCUIElement.ElementType.button.rawValue } + @objc var label: String { "same-selector" } + @objc var frame: CGRect { Self.elementFrame(self) } + private static func elementFrame(_ element: AnyObject) -> CGRect { + if element === keyboard { return keyboardFrame ?? .zero } + if element === application, let applicationFrame { return applicationFrame } + return element === target || element === darkTarget ? targetFrame : windowFrame + } + @objc var keyboards: XCUIElementQuery { + unsafeBitCast(SurfaceSelectorQuery([Self.keyboard!]), to: XCUIElementQuery.self) + } + @objc var allElementsBoundByIndex: [XCUIElement] { [Self.window!] } + @objc(descendantsMatchingType:) + func descendants(matching type: UInt) -> XCUIElementQuery { + if type == XCUIElement.ElementType.window.rawValue { + return unsafeBitCast(SurfaceSelectorQuery([Self.window!]), to: XCUIElementQuery.self) + } + Self.beforeElementLookup?() + let elements = self === Self.window || !Self.includeDarkSelector ? [Self.target!] : [Self.darkTarget!, Self.target!] + return unsafeBitCast(SurfaceSelectorQuery(elements), to: XCUIElementQuery.self) + } + + @objc(synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:point:deadline:errorMessage:) + class func synthesizeTap(_ app: AnyObject, resolvedWindow: AnyObject?, interfaceOrientation: Int, point: CGPoint, + deadline: NSDate?, errorMessage: AutoreleasingUnsafeMutablePointer?) -> Int { + record(window: resolvedWindow, orientation: interfaceOrientation, start: point) + return 0 + } + + @objc(synthesizeDragWithApplication:resolvedWindow:interfaceOrientation:drag:) + class func synthesizeDrag(_ app: AnyObject, resolvedWindow: AnyObject?, interfaceOrientation: Int, + drag: RunnerSynthesizedDrag) -> String? { + record(window: resolvedWindow, orientation: interfaceOrientation, + start: drag.start, end: drag.end) + return nil + } + + @objc(synthesizeGestureWithApplication:resolvedWindow:interfaceOrientation:pointerSamples:) + class func synthesizeGesture(_ app: AnyObject, resolvedWindow: AnyObject?, interfaceOrientation: Int, + pointerSamples: [[[String: NSNumber]]]) -> String? { + guard let first = pointerSamples.first?.first, let last = pointerSamples.first?.last else { + return "fixture pointer path is empty" + } + record(window: resolvedWindow, orientation: interfaceOrientation, + start: CGPoint(x: first["x"]!.doubleValue, y: first["y"]!.doubleValue), + end: CGPoint(x: last["x"]!.doubleValue, y: last["y"]!.doubleValue)) + return nil + } + + private static func record(window: AnyObject?, orientation: Int, start: CGPoint, end: CGPoint? = nil) { + var display: UInt = 0 + if let window { _ = RunnerResolveWindowDisplayID(window, &display) } + dispatches.append(RecordedSurfaceDispatch(window: window, displayID: display, start: start, end: end, orientation: orientation)) + onDispatch?() + } + @objc(coordinateWithNormalizedOffset:) func coordinateWithNormalizedOffset(_ offset: CGVector) -> XCUICoordinate { Self.anchor = self @@ -65,43 +160,62 @@ private final class LitPanelBookingProbe: NSObject { return Self.coordinate! } - @objc func tap() { Self.tappedPoint = Self.point } + @objc func tap() { + Self.tappedPoint = Self.point + Self.onDispatch?() + } } extension RunnerTests { @MainActor func testLitPanelBookingAndCoordinateFallbackShareScreenPointsForPartialWindows() throws { - try withLitPanelBookingProbe { window in - for (frame, orientation, expected) in [ + try withLitPanelBookingProbe(composed: true) { window in + LitPanelBookingProbe.beforeElementLookup = { + XCTAssertNotNil(self.litPanelState.surface, "Preparation must book before element lookup") + } + for (frame, orientation, expected, digitizer) in [ ( CGRect(x: 80, y: 0, width: 871, height: 669), RunnerInterfaceOrientation.landscapeLeft, - CGRect(x: 0, y: 0, width: 951, height: 669) + CGRect(x: 0, y: 0, width: 951, height: 669), CGPoint(x: 150, y: 751) ), ( CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portrait, - CGRect(x: 0, y: 0, width: 669, height: 951) + CGRect(x: 0, y: 0, width: 669, height: 951), CGPoint(x: 200, y: 150) ), ( - CGRect(x: 80, y: 0, width: 180, height: 669), RunnerInterfaceOrientation.landscapeRight, - CGRect(x: 0, y: 0, width: 951, height: 669) + CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.landscapeRight, + CGRect(x: 0, y: 0, width: 951, height: 669), CGPoint(x: 519, y: 200) ), ( CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portraitUpsideDown, - CGRect(x: 0, y: 0, width: 669, height: 951) + CGRect(x: 0, y: 0, width: 669, height: 951), CGPoint(x: 469, y: 801) ), ] { LitPanelBookingProbe.windowFrame = frame LitPanelBookingProbe.orientation = orientation - LitPanelBookingProbe.tappedPoint = nil - setRequestLitPanel(from: try runnerCommandFixture( - #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# - )) - XCTAssertNil(bookRequestLitPanelSurface(app: app)) - let booked = try XCTUnwrap(litPanelState.surface) - XCTAssertTrue(booked.window === window) - XCTAssertEqual(booked.frame, frame) - XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: booked.frame, interfaceOrientation: orientation), expected) - if case .unsupported(let message, _) = try tapAt(app: app, x: 200, y: 150) { XCTFail(message) } + for synthesized in [true, false] { + LitPanelBookingProbe.anchor = nil + LitPanelBookingProbe.tappedPoint = nil + LitPanelBookingProbe.dispatches.removeAll() + let command = try runnerCommandFixture( + #"{"command":"tap","selectorKey":"label","selectorValue":"same-selector","x":200,"y":150,"allowNonHittableCoordinateFallback":true,"synthesized":\#(synthesized),"litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + ) + try withPreparedSurfaceCommand(command) { _ in + let booked = try XCTUnwrap(litPanelState.surface) + XCTAssertTrue(booked.window === window) + XCTAssertEqual(booked.geometry.windowOnScreen.rect, frame) + XCTAssertEqual(booked.geometry.digitizerBasis.rect, expected) + if synthesized { + XCTAssertEqual(LitPanelBookingProbe.synthesizedDisplay, 3) + XCTAssertEqual(LitPanelBookingProbe.synthesizedPoint, digitizer) + } else { + var display: UInt = 0 + XCTAssertNil(RunnerResolveWindowDisplayID(try XCTUnwrap(LitPanelBookingProbe.anchor), &display)) + XCTAssertEqual(display, 3) + XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) + } + } + } XCTAssertTrue(LitPanelBookingProbe.anchor === window) XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) } @@ -117,6 +231,146 @@ extension RunnerTests { } } + @MainActor + func testRepeatedPressStopsWhenTheFirstActionChangesOrientation() throws { + try withLitPanelBookingProbe(composed: true) { _ in + for synthesized in [false, true] { + for nextKind in ["tap", "longPress"] { + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 80, width: 669, height: 871) + LitPanelBookingProbe.orientation = RunnerInterfaceOrientation.portrait + LitPanelBookingProbe.dispatches.removeAll() + LitPanelBookingProbe.onDispatch = { + LitPanelBookingProbe.orientation = RunnerInterfaceOrientation.landscapeLeft + LitPanelBookingProbe.windowFrame = CGRect(x: 80, y: 0, width: 871, height: 669) + } + let command = try runnerCommandFixture( + #"{"command":"sequence","litPanel":{"displayId":3,"widthPt":669,"heightPt":951},"steps":[{"kind":"tap","x":200,"y":150,"synthesized":\#(synthesized)},{"kind":"\#(nextKind)","x":200,"y":150,"synthesized":true,"durationMs":100}]}"# + ) + try withPreparedSurfaceCommand(command) { response in + XCTAssertEqual(response.data?.x, 200) + XCTAssertEqual(response.data?.y, 150) + XCTAssertEqual(response.data?.completedSteps, 1) + XCTAssertEqual(response.data?.failedStepIndex, 1) + XCTAssertEqual(response.data?.sequenceResults?.last?.errorCode, "UNSUPPORTED_OPERATION") + XCTAssertEqual(LitPanelBookingProbe.dispatches.count, synthesized ? 1 : 0) + if synthesized { + let dispatch = try XCTUnwrap(LitPanelBookingProbe.dispatches.first) + XCTAssertEqual(dispatch.displayID, 3) + XCTAssertEqual(dispatch.start, CGPoint(x: 200, y: 150)) + XCTAssertEqual(dispatch.orientation, RunnerInterfaceOrientation.portrait) + } + } + } + } + } + } + + @MainActor + func testRepeatedPressCarriesOrientationAcrossStepAndBudgetChunkBoundaries() throws { + try withLitPanelBookingProbe(composed: true) { _ in + for chunkSize in [1, 20] { + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 80, width: 669, height: 871) + LitPanelBookingProbe.orientation = RunnerInterfaceOrientation.portrait + LitPanelBookingProbe.dispatches.removeAll() + LitPanelBookingProbe.onDispatch = { + if LitPanelBookingProbe.dispatches.count == chunkSize { + LitPanelBookingProbe.orientation = RunnerInterfaceOrientation.landscapeLeft + LitPanelBookingProbe.windowFrame = CGRect(x: 80, y: 0, width: 871, height: 669) + } + } + let step = #"{"kind":"tap","x":200,"y":150,"synthesized":true}"# + let steps = Array(repeating: step, count: chunkSize).joined(separator: ",") + let first = try runnerCommandFixture( + #"{"command":"sequence","litPanel":{"displayId":3,"widthPt":669,"heightPt":951},"steps":[\#(steps)]}"# + ) + try withPreparedSurfaceCommand(first) { response in + XCTAssertEqual(response.data?.completedSteps, chunkSize) + let wire = try XCTUnwrap(try JSONSerialization.jsonObject(with: JSONEncoder().encode(response)) as? [String: Any]) + let data = try XCTUnwrap(wire["data"] as? [String: Any]) + XCTAssertEqual(data["sequenceOrientation"] as? Int, RunnerInterfaceOrientation.portrait) + } + let continuation = try runnerCommandFixture( + #"{"command":"sequence","sequenceOrientation":1,"litPanel":{"displayId":3,"widthPt":669,"heightPt":951},"steps":[\#(step)]}"# + ) + try withPreparedSurfaceCommand(continuation) { response in + XCTAssertEqual(response.data?.completedSteps, 0) + XCTAssertEqual(response.data?.failedStepIndex, 0) + XCTAssertEqual(response.data?.sequenceResults?.first?.errorCode, "UNSUPPORTED_OPERATION") + } + XCTAssertEqual(LitPanelBookingProbe.dispatches.count, chunkSize) + } + } + } + + @MainActor + func testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis() throws { + let health = mainOwned.accessibilityHealth + defer { mainOwned.accessibilityHealth = health } + try withLitPanelBookingProbe(composed: true) { _ in + mainOwned.accessibilityHealth = .healthy + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 80, width: 669, height: 871) + LitPanelBookingProbe.orientation = RunnerInterfaceOrientation.landscapeRight + LitPanelBookingProbe.keyboardFrame = CGRect(x: 0, y: 600, width: 951, height: 300) + let command = try runnerCommandFixture( + #"{"command":"scroll","direction":"down","pixels":200,"durationMs":400,"litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + ) + try withPreparedSurfaceCommand(command) { response in + XCTAssertEqual(response.data?.referenceWidth, 669) + XCTAssertEqual(response.data?.referenceHeight, 508) + XCTAssertEqual(response.data?.keyboardAvoided, true) + XCTAssertEqual(response.data?.keyboardMinY, 600) + XCTAssertEqual(response.data?.x, 335) + XCTAssertEqual(response.data?.y, 434) + XCTAssertEqual(response.data?.x2, 335) + XCTAssertEqual(response.data?.y2, 234) + XCTAssertEqual(LitPanelBookingProbe.dispatches.count, 1) + let dispatch = try XCTUnwrap(LitPanelBookingProbe.dispatches.first) + XCTAssertEqual(dispatch.displayID, 3) + XCTAssertEqual(dispatch.orientation, RunnerInterfaceOrientation.landscapeRight) + XCTAssertEqual(dispatch.start, CGPoint(x: 235, y: 335)) + XCTAssertEqual(dispatch.end, CGPoint(x: 435, y: 335)) + } + } + } + + @MainActor + func testPlacedIPadWindowDispatchUsesTheMeasuredSoleDisplay() throws { + try XCTSkipUnless(XCUIScreen.screens.count == 1, "the fixture requires a single-display host") + let cached = mainOwned.soleDisplayPortraitSize + defer { mainOwned.soleDisplayPortraitSize = cached } + mainOwned.soleDisplayPortraitSize = nil + var screenReads = 0 + let restore = try overrideSoleScreenValues([ + "bounds": NSValue(cgRect: CGRect(x: 0, y: 0, width: 2048, height: 2732)), + "scale": NSNumber(value: 2) + ], didRead: { if $0 == "bounds" { screenReads += 1 } }) + defer { restore() } + try withLitPanelBookingProbe(composed: true) { _ in + LitPanelBookingProbe.windowDisplayID = 1 + LitPanelBookingProbe.includeDarkSelector = false + LitPanelBookingProbe.windowFrame = CGRect(x: 224, y: 207, width: 386, height: 682) + LitPanelBookingProbe.applicationFrame = CGRect(x: 0, y: 0, width: 1366, height: 1024) + LitPanelBookingProbe.targetFrame = CGRect(x: 290, y: 390, width: 20, height: 20) + LitPanelBookingProbe.orientation = RunnerInterfaceOrientation.landscapeRight + let command = try runnerCommandFixture( + #"{"command":"tap","selectorKey":"label","selectorValue":"same-selector","synthesized":true}"# + ) + try withPreparedSurfaceCommand(command) { response in + XCTAssertGreaterThan(screenReads, 0) + XCTAssertEqual(mainOwned.soleDisplayPortraitSize, CGSize(width: 1024, height: 1366)) + XCTAssertEqual(response.data?.referenceWidth, 386) + XCTAssertEqual(response.data?.referenceHeight, 682) + XCTAssertEqual(LitPanelBookingProbe.dispatches.count, 1) + let dispatch = try XCTUnwrap(LitPanelBookingProbe.dispatches.first) + XCTAssertEqual(dispatch.displayID, 1) + XCTAssertEqual(dispatch.orientation, RunnerInterfaceOrientation.landscapeRight) + XCTAssertEqual(dispatch.start, CGPoint(x: 624, y: 300)) + XCTAssertTrue(dispatch.window === LitPanelBookingProbe.window) + XCTAssertEqual((dispatch.window as? XCUIElement)?.frame, CGRect(x: 224, y: 207, width: 386, height: 682)) + } + } + } + @MainActor func testCollapsedTabCandidatesExcludeOverlappingDarkWindowsAndKeepTheCaptureScope() throws { try withLitPanelBookingProbe { window in @@ -283,12 +537,28 @@ extension RunnerTests { } @MainActor - private func withLitPanelBookingProbe(_ body: (XCUIElement) throws -> Void) throws { + private func withPreparedSurfaceCommand(_ command: Command, _ assertions: (Response) throws -> Void) throws { + let requestID = setRequestLitPanel(from: command) + defer { endRequestLitPanel(requestID) } + let response = try executeOnMainPrepared(command: command, activeApp: app) + XCTAssertTrue(response.ok, response.error?.message ?? "") + try assertions(response) + } + + @MainActor + private func withLitPanelBookingProbe(composed: Bool = false, _ body: (XCUIElement) throws -> Void) throws { app.launch() defer { app.terminate() } let query = app.windows let window = query.firstMatch LitPanelBookingProbe.window = window + if composed { + LitPanelBookingProbe.application = app + LitPanelBookingProbe.keyboard = app.keyboards.firstMatch + LitPanelBookingProbe.target = app.buttons.element(boundBy: 0) + LitPanelBookingProbe.darkTarget = app.buttons.element(boundBy: 1) + LitPanelBookingProbe.composedDarkWindow = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 466, height: 678), displayID: 1) + } LitPanelBookingProbe.coordinate = window.coordinate(withNormalizedOffset: .zero) let windowClass = try XCTUnwrap(object_getClass(window)) let coordinateClass = try XCTUnwrap(object_getClass(LitPanelBookingProbe.coordinate!)) @@ -306,19 +576,54 @@ extension RunnerTests { (class_getInstanceMethod(coordinateClass, #selector(XCUICoordinate.tap)), class_getInstanceMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.tap))), ] + let attributeSwaps: [(Method?, Method?)] = [ + (#selector(getter: XCUIElement.exists), #selector(getter: LitPanelBookingProbe.exists)), + (#selector(getter: XCUIElement.isHittable), #selector(getter: LitPanelBookingProbe.isHittable)), + (#selector(getter: XCUIElement.elementType), #selector(getter: LitPanelBookingProbe.elementType)), + (#selector(getter: XCUIElement.label), #selector(getter: LitPanelBookingProbe.label)), + (#selector(getter: XCUIElement.frame), #selector(getter: LitPanelBookingProbe.frame)), + (#selector(XCUIElement.descendants(matching:)), #selector(LitPanelBookingProbe.descendants(matching:))), + ].map { (class_getInstanceMethod(windowClass, $0.0), class_getInstanceMethod(LitPanelBookingProbe.self, $0.1)) } + let synthesisSwap = ( + class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:point:deadline:errorMessage:")), + class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.synthesizeTap(_:resolvedWindow:interfaceOrientation:point:deadline:errorMessage:))) + ) + let dragStub = class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.synthesizeDrag(_:resolvedWindow:interfaceOrientation:drag:))) + let composedSwaps: [(Method?, Method?)] = [ + (class_getInstanceMethod(try XCTUnwrap(object_getClass(app)), NSSelectorFromString("keyboards")), + class_getInstanceMethod(LitPanelBookingProbe.self, #selector(getter: LitPanelBookingProbe.keyboards))), + (class_getInstanceMethod(try XCTUnwrap(object_getClass(query)), #selector(getter: XCUIElementQuery.allElementsBoundByIndex)), + class_getInstanceMethod(LitPanelBookingProbe.self, #selector(getter: LitPanelBookingProbe.allElementsBoundByIndex))), + (class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeDragWithApplication:resolvedWindow:interfaceOrientation:drag:")), dragStub), + (class_getClassMethod(RunnerSynthesizedGesture.self, NSSelectorFromString("synthesizeGestureWithApplication:resolvedWindow:interfaceOrientation:pointerSamples:")), + class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.synthesizeGesture(_:resolvedWindow:interfaceOrientation:pointerSamples:)))) + ] var originals: [(Method, IMP)] = [] defer { for (method, original) in originals.reversed() { method_setImplementation(method, original) } LitPanelBookingProbe.darkWindow = nil LitPanelBookingProbe.window = nil LitPanelBookingProbe.coordinate = nil + LitPanelBookingProbe.target = nil + LitPanelBookingProbe.darkTarget = nil + LitPanelBookingProbe.composedDarkWindow = nil + LitPanelBookingProbe.beforeElementLookup = nil + LitPanelBookingProbe.application = nil + LitPanelBookingProbe.applicationFrame = nil + LitPanelBookingProbe.keyboard = nil + LitPanelBookingProbe.keyboardFrame = nil + LitPanelBookingProbe.targetFrame = CGRect(x: 190, y: 140, width: 20, height: 20) + LitPanelBookingProbe.windowDisplayID = 3 + LitPanelBookingProbe.includeDarkSelector = true + LitPanelBookingProbe.dispatches.removeAll() + LitPanelBookingProbe.onDispatch = nil LitPanelBookingProbe.anchor = nil LitPanelBookingProbe.originalValueForKey = nil LitPanelBookingProbe.maxWindowListReads = nil litPanelState.panel = nil litPanelState.surface = nil } - for (method, stub) in swaps { + for (method, stub) in swaps + (composed ? attributeSwaps + [synthesisSwap] + composedSwaps : []) { let method = try XCTUnwrap(method) let stub = try XCTUnwrap(stub) originals.append((method, method_setImplementation(method, method_getImplementation(stub)))) @@ -426,9 +731,9 @@ extension RunnerTests { )) XCTAssertNil(bookRequestLitPanelSurface(app: app)) let context = try XCTUnwrap(try synthesizedCoordinateContext(app: app)) - XCTAssertEqual(context.referenceFrame, frame, "the booked reference is the window's own frame") + XCTAssertEqual(context.geometry.windowOnScreen.rect, frame, "the booked reference is the window's own frame") XCTAssertEqual( - synthesizedRotationFrame(app: app, fallback: context.referenceFrame, interfaceOrientation: RunnerInterfaceOrientation.portrait), + SurfaceGeometry.panelFrame(nativeSize: context.geometry.panelNativeSize!, orientation: RunnerInterfaceOrientation.portrait), CGRect(x: 0, y: 0, width: frame.width, height: frame.height + 80), "the turn is about the whole lit panel, not the window" ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift index 1c43da4740..ac6cbb1f12 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift @@ -3,20 +3,21 @@ import XCTest #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) import ObjectiveC.runtime -private final class AlertTapDeadlineStub: NSObject { +final class RunnerTapSynthesisStub: NSObject { static var status: RunnerTapSynthesisStatus? static var interruptionHandlingDisabled: Bool? - @objc(synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:) + @objc(synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:point:deadline:errorMessage:) class func synthesizeTap( application: XCUIApplication, resolvedWindow: Any?, - x: Double, - y: Double, + interfaceOrientation: Int, + point: CGPoint, deadline: NSDate?, errorMessage: AutoreleasingUnsafeMutablePointer? ) -> RunnerTapSynthesisStatus { interruptionHandlingDisabled = (application.value(forKey: "doesNotHandleUIInterruptions") as? NSNumber)?.boolValue + if status == .failed { errorMessage?.pointee = "forced private synthesis failure" } return status ?? (deadline == nil ? .succeeded : .deadlineExceeded) } } @@ -132,7 +133,7 @@ extension RunnerTests { XCTAssertTrue(app.alerts.firstMatch.exists) XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 0; replacement actions: 0") - AlertTapDeadlineStub.status = RunnerTapSynthesisStatus(rawValue: 999) + RunnerTapSynthesisStub.status = RunnerTapSynthesisStatus(rawValue: 999) let unknown = try activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) guard case .unsupported? = unknown else { return XCTFail("an unknown native status must report failure rather than deadline expiry") @@ -193,12 +194,12 @@ extension RunnerTests { let button = try XCTUnwrap(alert.actions.first { $0.label == "OK" }) let frame = button.frame try withAlertTapStub { - AlertTapDeadlineStub.status = .deadlineExceeded + RunnerTapSynthesisStub.status = .deadlineExceeded let outcome = try activateAlertButton( alert, button: button, action: "accept", frame: frame, deadline: .distantFuture ) XCTAssertNil(outcome) - XCTAssertEqual(AlertTapDeadlineStub.interruptionHandlingDisabled, true) + XCTAssertEqual(RunnerTapSynthesisStub.interruptionHandlingDisabled, true) } } @@ -218,13 +219,13 @@ extension RunnerTests { @MainActor private func withAlertTapStub(_ body: () throws -> Void) throws { - let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:") + let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:point:deadline:errorMessage:") let method = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedGesture.self, selector)) - let stub = try XCTUnwrap(class_getClassMethod(AlertTapDeadlineStub.self, selector)) + let stub = try XCTUnwrap(class_getClassMethod(RunnerTapSynthesisStub.self, selector)) let original = method_setImplementation(method, method_getImplementation(stub)) defer { - AlertTapDeadlineStub.status = nil - AlertTapDeadlineStub.interruptionHandlingDisabled = nil + RunnerTapSynthesisStub.status = nil + RunnerTapSynthesisStub.interruptionHandlingDisabled = nil method_setImplementation(method, original) } try body() diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift index 7786296fa5..27d36a8f79 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift @@ -8,21 +8,6 @@ private final class RunnerCoordinateTapObserver { var count = 0 } -private final class RunnerSynthesizedTapFailureStub: NSObject { - @objc(synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:) - class func synthesizeTap( - application: XCUIApplication, - resolvedWindow: Any?, - x: Double, - y: Double, - deadline: NSDate?, - errorMessage: AutoreleasingUnsafeMutablePointer? - ) -> RunnerTapSynthesisStatus { - errorMessage?.pointee = "forced private synthesis failure" - return .failed - } -} - extension RunnerTests { private func observeCoordinateTaps(droppingFirst: Bool = false) throws -> ( observer: RunnerCoordinateTapObserver, restore: () -> Void @@ -47,12 +32,18 @@ extension RunnerTests { /// Makes private tap synthesis fail until the returned closure restores it. func forceSynthesizedTapFailure() throws -> () -> Void { - let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:") + let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:interfaceOrientation:point:deadline:errorMessage:") let synthesizedTapMethod = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedGesture.self, selector)) - let failureStubMethod = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedTapFailureStub.self, selector)) + let failureStubMethod = try XCTUnwrap(class_getClassMethod(RunnerTapSynthesisStub.self, selector)) let originalImplementation = method_getImplementation(synthesizedTapMethod) method_setImplementation(synthesizedTapMethod, method_getImplementation(failureStubMethod)) - return { method_setImplementation(synthesizedTapMethod, originalImplementation) } + let status = RunnerTapSynthesisStub.status + RunnerTapSynthesisStub.status = .failed + return { + RunnerTapSynthesisStub.status = status + RunnerTapSynthesisStub.interruptionHandlingDisabled = nil + method_setImplementation(synthesizedTapMethod, originalImplementation) + } } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift index d2a3644c56..c8eabb962d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift @@ -78,6 +78,7 @@ extension RunnerTests { let samples = pointers.flatMap { $0["samples"] as? [[String: Any]] ?? [] } let litPanels = requests.compactMap { $0["litPanel"] as? [String: Any] } let commands = try entries.map { try decodeProductionRunnerRequest($0.request, $0.name) } + XCTAssertEqual(commands.compactMap(\.sequenceOrientation).first, 1) let command = try XCTUnwrap(commands.first, "no production request") let plan = try XCTUnwrap( commands.compactMap(\.gesturePlan).first, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift index 3fb8879a82..36d29a55fe 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift @@ -5,15 +5,12 @@ import AgentDeviceSnapshotPresentation import ObjectiveC.runtime private final class RunnerSynthesizedSwipeFailureStub: NSObject { - @objc(synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:) + @objc(synthesizeDragWithApplication:resolvedWindow:interfaceOrientation:drag:) class func synthesizeSwipe( application: XCUIApplication, resolvedWindow: Any?, - x: Double, - y: Double, - x2: Double, - y2: Double, - durationMs: Double + interfaceOrientation: Int, + drag: RunnerSynthesizedDrag ) -> String? { "forced private synthesis failure" } @@ -56,7 +53,7 @@ extension RunnerTests { @MainActor func testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails() throws { let selector = NSSelectorFromString( - "synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:" + "synthesizeDragWithApplication:resolvedWindow:interfaceOrientation:drag:" ) guard let synthesizedSwipeMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift index 208397cc95..b08b97d52b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift @@ -82,69 +82,6 @@ extension RunnerTests { XCTAssertEqual(constants.accessoryAllowance, ScrollViewportPolicy.accessoryAllowance) } - /// A clipped landscape band shortens the frame, and `CoordinateSpaceRotation.native(point:)` derives a - /// `landscapeRight` native x from the frame's HEIGHT. Rotating inside the band therefore moves the - /// dispatched path sideways by exactly what the keyboard took, off the lane the plan was built for, - /// so the plan band and the coordinate basis stay separate values through dispatch (#2500). - func testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis() throws { - let viewport = CGRect(x: 0, y: 0, width: 1210, height: 834) - let keyboard = CGRect(x: 0, y: 588, width: 1210, height: 246) - let clip = ScrollViewportPolicy.clip(viewport: viewport, keyboard: keyboard) - guard case .avoided(let band, let keyboardMinY) = clip else { - return XCTFail("expected a landscape keyboard to be avoided, got \(clip)") - } - XCTAssertEqual(band.height, 576) - - guard case .gesture(let gesture) = ScrollViewportPolicy.frames( - referenceFrame: viewport, - clip: clip - ).gestureDispatch(direction: .up, amount: nil, pixels: nil) else { - return XCTFail("expected a gesture inside the clipped band") - } - XCTAssertEqual(gesture.planFrame, band) - XCTAssertEqual(gesture.keyboardMinY, keyboardMinY) - XCTAssertEqual(gesture.coordinateFrame, viewport, "the rotation basis must survive the clip") - XCTAssertLessThanOrEqual( - max(gesture.plan.y1, gesture.plan.y2), - keyboard.minY - ScrollViewportPolicy.accessoryAllowance, - "a landscape swipe must stay clear of the keys" - ) - - let reported = gesture.attachingEvidence( - to: Response( - ok: true, - data: DataPayload(referenceWidth: viewport.width, referenceHeight: viewport.height), - error: nil - ) - ) - XCTAssertEqual( - reported.data?.referenceHeight, - band.height, - "the payload names the band the plan ran inside, not the synthesis frame" - ) - XCTAssertEqual(reported.data?.referenceWidth, viewport.width) - XCTAssertEqual(reported.data?.keyboardMinY, keyboardMinY) - XCTAssertEqual(reported.data?.keyboardAvoided, true) - - let orientedStartY = gesture.planFrame.minY + gesture.plan.y1 - let dispatchedFromViewport = CoordinateSpaceRotation.native( - point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), - in: gesture.coordinateFrame, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ) - let dispatchedFromBand = CoordinateSpaceRotation.native( - point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), - in: gesture.planFrame, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ) - XCTAssertEqual( - dispatchedFromViewport.x - dispatchedFromBand.x, - viewport.height - band.height, - accuracy: 0.001, - "rotating inside the clipped band would shift native x by what the keyboard took" - ) - } - private func loadScrollViewportPolicyFixture() throws -> ScrollViewportPolicyFixture { let fixtureURL = URL(fileURLWithPath: #filePath) .deletingLastPathComponent() // UnitTests diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift index 63880cb5da..312b075aba 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift @@ -1,3 +1,4 @@ +import AgentDeviceSnapshotPresentation import XCTest #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) @@ -160,7 +161,7 @@ extension RunnerTests { accessibilityHealth: RunnerAccessibilityHealth ) -> SynthesizedCoordinateContext { SynthesizedCoordinateContext( - referenceFrame: CGRect(x: 0, y: 0, width: 390, height: 844), + geometry: SurfaceGeometry(windowOnScreen: SurfaceRect(CGRect(x: 0, y: 0, width: 390, height: 844)), panelNativeSize: nil, orientation: 1), resolvedWindow: app.windows.firstMatch, accessibilityHealth: accessibilityHealth ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift index a50f261504..49c6c14f24 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGestureTests.swift @@ -24,15 +24,78 @@ private final class TapApplicationFixture: NSObject { private final class TapRecordSpy: NSObject { static var syntheses = 0 + static var recordOrientations: [Int] = [] + static var recordReadError: String? @objc(synthesizeWithError:) func synthesize(error: AutoreleasingUnsafeMutablePointer?) -> Bool { Self.syntheses += 1 + Self.recordReadError = RunnerObjCExceptionCatcher.catchException { + if let orientation = self.value(forKey: "interfaceOrientation") as? NSNumber { + Self.recordOrientations.append(orientation.intValue) + } + } return true } } extension RunnerTests { func testTapDeadlineExpiringDuringEventPreparationDoesNotSynthesize() throws { + try withTapRecordSpy { + let deadline = Date().addingTimeInterval(1) + var prepared = false + var preparationBeganBeforeDeadline = false + let window = TapWindowFixture { + preparationBeganBeforeDeadline = Date() < deadline + while Date() < deadline { Thread.sleep(forTimeInterval: 0.001) } + prepared = true + } + var message: NSString? + let expired = RunnerSynthesizedGesture.synthesizeTap( + withApplication: TapApplicationFixture(), + resolvedWindow: window, + interfaceOrientation: 1, + point: CGPoint(x: 50, y: 50), + deadline: deadline, + errorMessage: &message + ) + + XCTAssertTrue(preparationBeganBeforeDeadline) + XCTAssertTrue(prepared) + XCTAssertEqual(expired, .deadlineExceeded) + XCTAssertNil(message) + XCTAssertEqual(TapRecordSpy.syntheses, 0) + + let control = RunnerSynthesizedGesture.synthesizeTap( + withApplication: TapApplicationFixture(), + resolvedWindow: TapWindowFixture {}, + interfaceOrientation: 1, + point: CGPoint(x: 50, y: 50), + deadline: .distantFuture, + errorMessage: &message + ) + XCTAssertEqual(control, .succeeded, message as String? ?? "") + XCTAssertEqual(TapRecordSpy.syntheses, 1) + } + } + + func testSynthesizedRecordKeepsTheConvertedPointsOrientation() throws { + try withTapRecordSpy { + var message: NSString? + let result = RunnerSynthesizedGesture.synthesizeTap( + withApplication: TapApplicationFixture(), + resolvedWindow: TapWindowFixture {}, + interfaceOrientation: 4, + point: CGPoint(x: 50, y: 50), + deadline: .distantFuture, + errorMessage: &message + ) + XCTAssertEqual(result, .succeeded, message as String? ?? "") + XCTAssertNil(TapRecordSpy.recordReadError) + XCTAssertEqual(TapRecordSpy.recordOrientations, [4]) + } + } + + private func withTapRecordSpy(_ body: () throws -> Void) throws { let recordClass = try XCTUnwrap(NSClassFromString("XCSynthesizedEventRecord")) let selector = NSSelectorFromString("synthesizeWithError:") let method = try XCTUnwrap(class_getInstanceMethod(recordClass, selector)) @@ -42,40 +105,10 @@ extension RunnerTests { defer { method_setImplementation(method, original) } TapRecordSpy.syntheses = 0 - let deadline = Date().addingTimeInterval(1) - var prepared = false - var preparationBeganBeforeDeadline = false - let window = TapWindowFixture { - preparationBeganBeforeDeadline = Date() < deadline - while Date() < deadline { Thread.sleep(forTimeInterval: 0.001) } - prepared = true - } - var message: NSString? - let expired = RunnerSynthesizedGesture.synthesizeTap( - withApplication: TapApplicationFixture(), - resolvedWindow: window, - x: 50, - y: 50, - deadline: deadline, - errorMessage: &message - ) - - XCTAssertTrue(preparationBeganBeforeDeadline) - XCTAssertTrue(prepared) - XCTAssertEqual(expired, .deadlineExceeded) - XCTAssertNil(message) - XCTAssertEqual(TapRecordSpy.syntheses, 0) - - let control = RunnerSynthesizedGesture.synthesizeTap( - withApplication: TapApplicationFixture(), - resolvedWindow: TapWindowFixture {}, - x: 50, - y: 50, - deadline: .distantFuture, - errorMessage: &message - ) - XCTAssertEqual(control, .succeeded, message as String? ?? "") - XCTAssertEqual(TapRecordSpy.syntheses, 1) + TapRecordSpy.recordOrientations = [] + TapRecordSpy.recordReadError = nil + try body() } + } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 7df11aa361..e5fe6918ef 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -54,6 +54,12 @@ extension RunnerTests { } private func simulateUnreadableDisplay() throws -> () -> Void { + try overrideSoleScreenValues(["scale": NSNumber(value: 0)]) + } + + func overrideSoleScreenValues( + _ values: [String: AnyObject], didRead: @escaping (String) -> Void = { _ in } + ) throws -> () -> Void { let selector = #selector(NSObject.value(forKey:)) let inherited = try XCTUnwrap(class_getInstanceMethod(XCUIScreen.self, selector)) class_addMethod(XCUIScreen.self, selector, method_getImplementation(inherited), method_getTypeEncoding(inherited)) @@ -62,7 +68,8 @@ extension RunnerTests { typealias ReadValue = @convention(c) (AnyObject, Selector, NSString) -> AnyObject? let readValue = unsafeBitCast(original, to: ReadValue.self) let read: @convention(block) (AnyObject, NSString) -> AnyObject? = { screen, key in - if key == "scale" { return NSNumber(value: 0) } + didRead(key as String) + if let value = values[key as String] { return value } return readValue(screen, selector, key) } let replacement = imp_implementationWithBlock(read) @@ -118,7 +125,7 @@ extension RunnerTests { let resolved = try XCTUnwrap(resolveRunnerWindowElement(app: app)) context = try XCTUnwrap( SynthesizedCoordinateContext( - validating: windowFrame ?? resolved.frame, + validating: surfaceGeometry(windowFrame: windowFrame ?? resolved.frame, app: app), resolvedWindow: resolved.window, accessibilityHealth: .healthy ) @@ -309,6 +316,15 @@ extension RunnerTests { XCTAssertEqual(displayID, 0) } + @MainActor + private func unscopedDigitizerBasis( + app: XCUIApplication, window: CGRect, interfaceOrientation: Int + ) -> CGRect? { + digitizerGeometry(app: app, surface: SurfaceGeometry( + windowOnScreen: SurfaceRect(window), panelNativeSize: nil, orientation: interfaceOrientation + ))?.digitizerBasis.rect + } + /// Physical iOS can report no system application's AX frame while its display is measurable. @MainActor func testSoleDisplaySizeDoesNotRequireASystemApplicationAccessibilityFrame() throws { @@ -326,8 +342,8 @@ extension RunnerTests { defer { restoreAXRuntime() } mainOwned.soleDisplayPortraitSize = nil let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) - let measured = try XCTUnwrap(synthesizedRotationFrame( - app: app, fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + let measured = try XCTUnwrap(unscopedDigitizerBasis( + app: app, window: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight )) XCTAssertEqual(measured.origin, .zero) XCTAssertEqual(measured.width, max(frame.width, frame.height), accuracy: 0.5) @@ -361,7 +377,7 @@ extension RunnerTests { (RunnerInterfaceOrientation.landscapeRight, CGSize(width: screen.height, height: screen.width)), ] for (orientation, expected) in expectations { - let frame = try XCTUnwrap(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: orientation)) + let frame = try XCTUnwrap(unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: orientation)) XCTAssertEqual(frame.origin, .zero) XCTAssertEqual(frame.width, expected.width, accuracy: 0.5) XCTAssertEqual(frame.height, expected.height, accuracy: 0.5) @@ -377,21 +393,21 @@ extension RunnerTests { let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) let landscape = RunnerInterfaceOrientation.landscapeRight mainOwned.soleDisplayPortraitSize = nil - let measured = try XCTUnwrap(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape)) + let measured = try XCTUnwrap(unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: landscape)) XCTAssertNotEqual(measured, placedWindow) let nextCommand = try runnerCommandFixture(#"{"command":"uptime","commandId":"next-command"}"#) // Nothing between the install and the restore throws, so the restore always runs. let restoreDisplay = try simulateUnreadableDisplay() - XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape), measured) + XCTAssertEqual(unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: landscape), measured) _ = try? executeOnMainPrepared(command: nextCommand, activeApp: app) XCTAssertNotEqual( - synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape), + unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: landscape), measured, "the next command must read the size again rather than reuse the kept one" ) restoreDisplay() - XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape), measured) + XCTAssertEqual(unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: landscape), measured) } /// #4091: a portrait turn leaves the point as it is, so an unreadable display box neither refuses @@ -466,9 +482,9 @@ extension RunnerTests { ) let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - let windowFrame = try XCTUnwrap(context).referenceFrame + let windowFrame = try XCTUnwrap(context).geometry.windowOnScreen.rect XCTAssertEqual( - synthesizedRotationFrame(app: app, fallback: windowFrame, interfaceOrientation: orientation), + unscopedDigitizerBasis(app: app, window: windowFrame, interfaceOrientation: orientation), windowFrame ) let outcome = try synthesizedTapAt(app: app, x: 40, y: 40, context: context) @@ -492,10 +508,10 @@ extension RunnerTests { ) XCTAssertNil( - synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight) + unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight) ) XCTAssertEqual( - synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.portrait), + unscopedDigitizerBasis(app: app, window: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.portrait), placedWindow, "portrait skips the box read (only landscape and upside-down read it), so it must not refuse" ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index c689a677ce..6dd48b02ad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -1,7 +1,7 @@ import XCTest #if AGENT_DEVICE_RUNNER_UNIT_TESTS -private final class DisplayScreenFixture: NSObject { +final class DisplayScreenFixture: NSObject { @objc let displayID: UInt init(_ displayID: UInt) { self.displayID = displayID } } @@ -9,7 +9,7 @@ private final class DisplayScreenFixture: NSObject { /// A window the runtime answers only once its query has resolved. `frame` is the read that /// resolves it: while the query is unresolved `screen` names main, which is what a fresh runner /// process that has not queried this app yet reports (#2728). -private final class DisplayWindowFixture: NSObject { +final class DisplayWindowFixture: NSObject { private let windowFrame: CGRect private let resolvedDisplayID: UInt private let present: Bool @@ -28,6 +28,20 @@ private final class DisplayWindowFixture: NSObject { // Handed back as one object so a test can prove the resolver captures the screen the window // reported rather than a fresh value for the same key. Unresolved, it names main. @objc var screen: NSObject { resolved ? windowScreen : DisplayScreenFixture(1) } + + override func value(forKey key: String) -> Any? { + switch key { + case "exists": return exists + case "frame": +#if os(macOS) + return NSValue(rect: frame) +#else + return NSValue(cgRect: frame) +#endif + case "screen": return screen + default: return super.value(forKey: key) + } + } } private final class DisplayWindowsFixture: NSObject { diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 637c242546..2d62dd87dc 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -13,6 +13,66 @@ public enum RunnerInterfaceOrientation { public static let landscapeLeft = 4 } +public enum WindowContentSpace {} +public enum InterfaceScreenSpace {} +public enum NativeDigitizerSpace {} + +public struct SurfacePoint: Equatable, Sendable { + public let point: CGPoint + public init(_ point: CGPoint) { self.point = point } +} + +public struct SurfaceRect: Equatable, Sendable { + public let rect: CGRect + public init(_ rect: CGRect) { self.rect = rect } +} + +/// Geometry measured once for a window. Panel size is absent when the runtime cannot prove it. +public struct SurfaceGeometry: Sendable { + public let windowOnScreen: SurfaceRect + public let panelNativeSize: CGSize? + public let orientation: Int + + public init(windowOnScreen: SurfaceRect, panelNativeSize: CGSize?, orientation: Int) { + self.windowOnScreen = windowOnScreen + self.panelNativeSize = panelNativeSize + self.orientation = orientation + } + + public var digitizerBasis: SurfaceRect { + SurfaceRect(panelNativeSize.flatMap { Self.panelFrame(nativeSize: $0, orientation: orientation) } + ?? windowOnScreen.rect) + } + + public func screenPoint(from content: SurfacePoint) -> SurfacePoint { + let frame = windowOnScreen.rect + let space = SnapshotGeometrySpace.windowContent(offset: CGSize(width: frame.minX, height: frame.minY)) + return SurfacePoint(space.orientedFrame(of: CGRect(origin: content.point, size: .zero)).origin) + } + + public func contentPoint(from screen: SurfacePoint) -> SurfacePoint { + let origin = screenPoint(from: SurfacePoint(.zero)).point + return SurfacePoint(CGPoint(x: screen.point.x - origin.x, y: screen.point.y - origin.y)) + } + + public func digitizerPoint(from screen: SurfacePoint) -> SurfacePoint { + SurfacePoint(CoordinateSpaceRotation.native( + point: screen.point, in: digitizerBasis.rect, interfaceOrientation: orientation)) + } + + public static func panelFrame(nativeSize size: CGSize, orientation: Int) -> CGRect? { + guard size.width > 0, size.height > 0 else { return nil } + switch orientation { + case RunnerInterfaceOrientation.portrait, RunnerInterfaceOrientation.portraitUpsideDown: + return CGRect(origin: .zero, size: size) + case RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight: + return CGRect(x: 0, y: 0, width: size.height, height: size.width) + default: + return nil + } + } +} + public enum CoordinateSpaceRotation { // Dispatch turns a point's axes and never moves its origin: the digitizer's origin is the // display's own, where a placed window's screen points are already measured from (#3899). diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index f86080eabd..a4be401fd4 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -64,6 +64,7 @@ {"name": "ios-simulator.runner-disposal.shutdown", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "shutdown", "commandId": ""}}, {"name": "ios-simulator.runner-lifecycle-prepare.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": ""}}, {"name": "ios-simulator.runner-selector-query.label", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "querySelector", "selectorKey": "label", "selectorValue": "Go", "appBundleId": "com.example.app", "commandId": ""}}, + {"name": "ios-simulator.runner-sequence.bound-orientation", "producer": "packages/platform-apple/src/__tests__/interactions-press-series-runner-requests.test.ts", "request": {"command": "sequence", "steps": [{"kind": "tap", "x": 10, "y": 20, "synthesized": true}], "appBundleId": "com.example.app", "sequenceOrientation": 1, "commandId": ""}}, {"name": "ios-simulator.runner-session-readiness.uptime", "producer": "packages/apple-runner/src/__tests__/index-runner-requests.test.ts", "request": {"command": "uptime", "commandId": ""}}, {"name": "macos.desktop-scroll.amount", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "desktopScroll", "direction": "down", "amount": 0.5, "durationMs": 300, "appBundleId": "com.example.app", "commandId": ""}}, {"name": "macos.desktop-scroll.pixels", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "desktopScroll", "direction": "up", "pixels": 200, "durationMs": 300, "appBundleId": "com.example.app", "commandId": ""}}, diff --git a/docs/evidence/surface-geometry-4189-2026-11-03.md b/docs/evidence/surface-geometry-4189-2026-11-03.md new file mode 100644 index 0000000000..86bdff8ce1 --- /dev/null +++ b/docs/evidence/surface-geometry-4189-2026-11-03.md @@ -0,0 +1,101 @@ +# Surface geometry review evidence, 2026-11-03 + +Source under test: `451ba0e7cdedd6773d85ac7955d18304de6f1cdf`. +Base: `ec80eecb1e0251d51d93aa68071c88577f1ea7ba` (`fix/foldable-springboard-surface`, #4122). +This packet addresses [the maintainer review](https://github.com/callstack/agent-device/pull/4189#issuecomment-6604084919). +The PR remains a draft because the requested live matrix is incomplete. + +## Native checks + +The unit-enabled iOS runner build and mandatory isolation scan passed. +Eleven method-selected native tests passed with `AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS=1` and private derived data. +The selection covers composed partial-window lookup and both dispatch routes, orientation-changing sequences, keyboard-clipped scroll dispatch, measured placed-iPad dispatch, actual native record orientation, tap deadlines, sole-display caching, windowless portrait/landscape admission, and both parent dark-window/exception regressions. +`swift test --package-path apple/snapshot-presentation` passed all 50 tests at the source SHA. + +The production-boundary assertions include: + +- Window `(0,80,669,871)`, panel `669x951`, orientation `3`: clipped scroll evidence is `669x508`; screen endpoints `(335,434)` and `(335,234)` dispatch as `(235,335)` and `(435,335)` on display `3`. +- Placed iPad window `(224,207,386,682)`: the production screen reader measures `1024x1366`; screen point `(300,400)` dispatches as `(624,300)` on display `1` in orientation `3`. +- A first sequence action changes orientation: later tap/hold steps refuse, completed steps remain `1`, and the native event count does not increase. +- Native record construction receives orientation `4` while the fake application's current getter reports `1`; the actual record stores `4`. + +## Mutation proof + +An isolated checkout ran the same five-test selection at `fe226cd7becf0b63ebde0009b2a0d4792b9df1e3`. +Each mutation built successfully and failed the relevant assertions with xcodebuild exit `65`. +The baseline and restored runs each passed all five tests. + +| Deliberate production defect | Failed assertions | +| --- | ---: | +| Rotate using the window box | 14 | +| Remove sequence orientation refusal | 16 | +| Rotate drag using the keyboard-clipped box | 2 | +| Skip sole-display measurement | 3 | +| Read orientation again during native record creation | 1 | +| Add window origin again to touch evidence | 4 | + +After correcting the placed-window query fake, its one-test baseline, missing-measurement mutation, and restored run were repeated at `451ba0e7`: pass, three assertion failures, pass. +The reviewed checkout remained clean during the experiments. +One initial failing run stalled while collecting diagnostics after its suite finished; the entire experiment was rerun with supported `-collect-test-diagnostics never` and normal failing exits. + +## Live observations and limits + +Only Duo `4F879835-4AB3-4046-B033-5AB769209DD4` and iPhone `DBEA214F-555F-4DDA-B62B-E34285CE63DB` were used. +Every UI command used the device-bound command returned by `device_open`. +That installed `0.21.31` client connected to this worktree's `0.21.37-dev` daemon through a private authenticated HTTP config and `--state-dir /private/tmp/agent-device-surface-4189`. +The daemon restarted with `AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH=$PWD/.tmp/surface-live-4189-derived` and rebuilt the runner from this checkout. +Its cache records source fingerprint `f4aeca12f9da03f67581231f3feb28c76d3030602255fc119ae10fdd5b376721`. + +The Tester ran from this worktree's Metro on port `8199`. +`fold open` reported hinge `180` and native lit panel `669x951`. +The command sequence was `orientation `, `press 'label="Form"'`, `fill 'id="field-name"' ''`, and `get text 'id="field-name"'`. +The portrait request read back `Geometry portrait 451ba0e7`; the right request read back `Geometry right 451ba0e7`. +A left request first refused a keyboard-covered Form target and reported `TEXT_ENTRY_MISMATCH` on fill; the advised retry with `--delay-ms 80` read back `Geometry left 451ba0e7`. +A fresh snapshot and lit capture succeeded; the capture was `951x669` logical points. +The requested rotations did not consistently match application orientation, so these runs do not prove the three-orientation matrix. +The portrait-request tap logged native point `(617,515)`, reference `(0,0,871,669)`, display `3`, orientation `4`. +The right-request tap logged native point `(334,52)`, reference `(0,80,669,871)`, display `3`, orientation `2`. + +On iPhone, `open com.apple.Preferences`, `snapshot`, and `press 'label="General"'` succeeded. +The visible screen changed to General; the native record logged display `1`, orientation `1`, point `(220,319)`, reference `(0,0,440,956)`. + +Missing live evidence: verified Duo orientations, half-open interaction, scoped-set coverage, explicit XCTest fallback, hold, clipped landscape scroll, sampled gesture, rotation-changing repeat, and AX-unavailable iPhone routes. +The two-device assignment excludes extra scoped clones and live iPad hardware. +The CLI offers no forced single-tap XCTest route; the composed native harness covers that route. +Cleanup reported Duo hinge `0`; both simulators shut down; the owned daemon released both claims cleanly; Metro stopped. + +## Final API consolidation + +The first full gate at `3090b336` rejected ten new native duplicate blocks. +The follow-up consolidates drag dispatch, passes tap points as `CGPoint`, resolves point orientation with its target, and shares screen-query and preparation fixtures. +Two existing fallback/alert stubs were also updated to the current native selector. +The live observations above precede this API consolidation; final-head native and gate results are recorded in the PR validation section. + +## Natural rejected-query route, 2026-11-04 + +Source under test: `84f1e6adccd54e4131d763c5c65f4880b6ac91f9`, the branch rebased onto `main` at `7d42395ff7e1c03b643c2cf56683808bf5427cf3`. +This section closes the `AX-unavailable iPhone routes` limit above and answers [the readiness review](https://github.com/callstack/agent-device/pull/4189#issuecomment-6627588706). +No production source, and no fault patch, was involved: the app's Heavy feed fixture exceeds the depth XCTest can serialize, so the accessibility query refuses on its own. + +Setup: sole-display `iPhone 17 Pro` simulator `5FEADD02-98E4-4F01-861C-07003C1A3291`, iOS 26.2, driven through the command returned by `device_open` in session `surface4189`. +A daemon from this checkout (`0.21.37-dev`, `codeOrigin=checkout`, signature `graph:123:89e270aa80446b4444ddc2a562ce3e7ee7b4b469`) owned a private state directory `/private/tmp/agent-device-4189-ax` and built the exact-head runner into the pinned derived path `.tmp/4189-ax-derived`. +`Agent Device Tester` was that simulator's existing install with its JavaScript repacked from this checkout; bundle hashes: original `ca4c0c3d5eedf000792a143c10f3c036309d9f16d5d4c2899fcf3e053f26ffab`, repacked `ac93ac57877b43f52dbce40abac65d4711af339b24f3629fb1cd9af01223448d`. The original bundle was reinstalled afterward. + +`back --in-app` on the heavy feed took the early rejected-query route. One `kAXErrorIllegalArgument` rejection was recorded, then `IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION`, then `SYNTHESIZED_SOLE_SCREEN_ROUTE resolved={{0, 0}, {402, 874}}`. +No further rejection was recorded before delivery, so the route opened without another windows query. +Delivery logged `BACK_FALLBACK_TOP_INSET read=62.0 windows=1`, a tap at `(32.2,84.0)` against reference `(0,0,402,874)` in orientation `1`, and a record naming display `1`. +`IN_APP_BACK_VISUAL_VERIFICATION … changed=yes` reported the visible navigation outcome, and a follow-up read of `heavy-feed-title` refused because the screen had left the feed. + +In the same state, three gestures reached the sole-screen route through their own refused windows query, each preceded by its own recorded rejection: + +| Command | Native evidence | Fresh outcome | +| --- | --- | --- | +| `press 273.33 458.5` | record `agent-device-tap`, display `1`, orientation `1`, point `(273.3,458.5)` | `Liked posts: 0` became `Liked posts: 1` | +| `longpress 273.33 458.5 900` | record `agent-device-gesture-plan`, display `1`, orientation `1` | `heavy-feed-last-action` became `Last action: unliked post 0`: the hold's contact toggled the same like control the tap had pressed | +| `gesture pan 201 400 0 -220 800` | record `agent-device-gesture-plan`, display `1`, orientation `1` | `heavy-post-2-like` moved from y `1001.3` to y `798.3` | + +Limits of this run. Only `back --in-app` passes `windowQueryRejected` into the coordinate context, so the gesture rows prove the sole-screen route after a refused windows query, not the early argument. +The pan ran the sampled plan; `SYNTHESIZED_DISPATCH kind=longPress` is shared by both hold routes, so the record name separates the hold from the tap. +This covers one sole-display iPhone on the default fixture shape: not a foldable, a physical device, a scoped simulator set, or a global accessibility outage. Snapshots still answered through the private route. + +Cleanup: the session and the owned daemon were closed, Metro stopped, the original app bundle was reinstalled, and the simulator was left booted with its claim released. diff --git a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts index b4599c0c92..a861324e16 100644 --- a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts +++ b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts @@ -114,7 +114,10 @@ async function pressSeriesRefusedOnSecondChunk(): Promise { const firstChunk = Array.from({ length: 20 }, () => ({ ok: true, kind: 'tap' })); server = await startFakeRunnerServer({ sequence: [ - { kind: 'ok', data: { completedSteps: 20, sequenceResults: firstChunk } }, + { + kind: 'ok', + data: { sequenceOrientation: 1, completedSteps: 20, sequenceResults: firstChunk }, + }, { kind: 'runnerError', code: 'RUNNER_BUSY', message: 'runner busy' }, ], }); diff --git a/packages/apple-runner/src/__tests__/runner-sequence.test.ts b/packages/apple-runner/src/__tests__/runner-sequence.test.ts index abf89e7161..2e31650546 100644 --- a/packages/apple-runner/src/__tests__/runner-sequence.test.ts +++ b/packages/apple-runner/src/__tests__/runner-sequence.test.ts @@ -3,11 +3,97 @@ import assert from 'node:assert/strict'; import { AppError } from '@agent-device/kernel/errors'; import { runApplePressSeries } from '../runner-sequence.ts'; import { IOS_SIMULATOR } from './device-fixtures.ts'; +import type { RunnerCommand } from '../runner-contract.ts'; + +test.each([ + { boundary: 'step cap', count: 21, intervalMs: 0, completed: 20 }, + { boundary: 'time budget', count: 3, intervalMs: 10_000, completed: 1 }, +])( + 'an orientation change at the $boundary stops the next chunk', + async ({ count, intervalMs, completed }) => { + let orientation = 1; + let dispatched = 0; + const requests: RunnerCommand[] = []; + await assert.rejects( + runApplePressSeries( + IOS_SIMULATOR, + { x: 10, y: 20 }, + { button: 'primary', count, intervalMs, holdMs: 0, jitterPx: 0, doubleTap: false }, + undefined, + async (command) => { + requests.push(command); + const bound = command.sequenceOrientation ?? orientation; + const results = []; + let completedSteps = 0; + for (const step of command.steps ?? []) { + if (bound !== orientation) { + results.push({ ok: false, kind: step.kind, errorCode: 'UNSUPPORTED_OPERATION' }); + return { + sequenceOrientation: bound, + completedSteps, + failedStepIndex: completedSteps, + sequenceResults: results, + }; + } + dispatched += 1; + completedSteps += 1; + results.push({ ok: true, kind: step.kind }); + if (dispatched === completed) orientation = 3; + } + return { sequenceOrientation: bound, completedSteps, sequenceResults: results }; + }, + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'UNSUPPORTED_OPERATION'); + assert.equal(error.details?.failedStepIndex, completed); + assert.equal(error.details?.completedSteps, completed); + assert.equal(error.details?.dispatched, 'unknown'); + assert.equal(error.details?.dispatchedSteps, completed); + return true; + }, + ); + assert.equal(dispatched, completed); + assert.equal(requests.length, 2); + assert.equal(requests[0]?.steps?.length, completed); + assert.equal(requests[1]?.sequenceOrientation, 1); + }, +); + +test.each([undefined, 0, 5, 1.5, '1'])( + 'a missing or invalid first-chunk orientation (%s) cannot authorize another chunk', + async (sequenceOrientation) => { + let requests = 0; + await assert.rejects( + runApplePressSeries( + IOS_SIMULATOR, + { x: 10, y: 20 }, + { button: 'primary', count: 21, intervalMs: 0, holdMs: 0, jitterPx: 0, doubleTap: false }, + undefined, + async (command) => { + requests += 1; + return { + sequenceOrientation, + completedSteps: command.steps!.length, + sequenceResults: command.steps!.map(({ kind }) => ({ ok: true, kind })), + }; + }, + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'runner_sequence_orientation_unavailable'); + assert.equal(error.details?.dispatchedSteps, 20); + return true; + }, + ); + assert.equal(requests, 1); + }, +); test('a step failure in a later chunk counts the taps every chunk completed', async () => { const ok = (count: number) => Array.from({ length: count }, () => ({ ok: true, kind: 'tap' })); const answers = [ - { completedSteps: 20, sequenceResults: ok(20) }, + { sequenceOrientation: 1, completedSteps: 20, sequenceResults: ok(20) }, { completedSteps: 2, sequenceResults: [...ok(2), { ok: false, kind: 'tap' }] }, ]; await assert.rejects( diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index 8219bf8015..a64f02adac 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -168,6 +168,7 @@ export type RunnerCommand = { * command that resolves a screen (ADR 0025). */ litPanel?: { displayId: number; widthPt: number; heightPt: number }; + sequenceOrientation?: RunnerSequenceOrientation; steps?: RunnerSequenceStep[]; }; @@ -573,8 +574,11 @@ export type ParsedRunnerSequenceResult = { results: RunnerSequenceStepResult[]; completedSteps: number; failedStepIndex?: number; + sequenceOrientation?: RunnerSequenceOrientation; }; +export type RunnerSequenceOrientation = 1 | 2 | 3 | 4; + function isSequenceableKind(kind: unknown): kind is SequenceableRunnerStepKind { return ( typeof kind === 'string' && (SEQUENCEABLE_RUNNER_STEP_KINDS as readonly string[]).includes(kind) @@ -703,7 +707,15 @@ export function parseRunnerSequenceResult( throw buildSequenceStepError(results, completedSteps, failedStepIndex); } - return { results, completedSteps, failedStepIndex }; + const orientation = data.sequenceOrientation; + return { + results, + completedSteps, + failedStepIndex, + ...(orientation === 1 || orientation === 2 || orientation === 3 || orientation === 4 + ? { sequenceOrientation: orientation } + : {}), + }; } function buildSequenceStepError( diff --git a/packages/apple-runner/src/runner-sequence.ts b/packages/apple-runner/src/runner-sequence.ts index 1496eba29c..f9cf022723 100644 --- a/packages/apple-runner/src/runner-sequence.ts +++ b/packages/apple-runner/src/runner-sequence.ts @@ -1,12 +1,17 @@ import { pressJitter } from '@agent-device/contracts/touch-runtime'; import type { PressPointOptions } from '@agent-device/contracts/interactor-types'; -import { runnerSynthesizesTap, type DeviceInfo } from '@agent-device/kernel/device'; +import { + isHandheldAppleDevice, + runnerSynthesizesTap, + type DeviceInfo, +} from '@agent-device/kernel/device'; import { AppError, discloseDispatchAfterSteps } from '@agent-device/kernel/errors'; import { MAX_RUNNER_SEQUENCE_STEPS, buildRunnerSequenceCommand, parseRunnerSequenceResult, type RunnerCommand, + type RunnerSequenceOrientation, type RunnerSequenceStep, } from './runner-contract.ts'; @@ -35,10 +40,14 @@ export async function runApplePressSeries( let completedSteps = 0; const sequenceResults: unknown[] = []; let stepOffset = 0; + let sequenceOrientation: RunnerCommand['sequenceOrientation']; + const bindsOrientation = isHandheldAppleDevice(device) && commands.length > 1; for (const command of commands) { let result: Record; try { - result = await runCommand(command); + result = await runCommand( + sequenceOrientation === undefined ? command : { ...command, sequenceOrientation }, + ); } catch (error) { throw discloseDispatchAfterSteps(error, completedSteps); } @@ -57,6 +66,12 @@ export async function runApplePressSeries( completedSteps += parsed.completedSteps; sequenceResults.push(...parsed.results); stepOffset += command.steps?.length ?? 0; + if (bindsOrientation) { + sequenceOrientation ??= requirePressSeriesOrientation( + parsed.sequenceOrientation, + completedSteps, + ); + } } // Preserve the first response's acquisition frame and the last response's gesture completion. return { @@ -70,6 +85,21 @@ export async function runApplePressSeries( }; } +function requirePressSeriesOrientation( + orientation: RunnerSequenceOrientation | undefined, + completedSteps: number, +): RunnerSequenceOrientation { + if (orientation !== undefined) return orientation; + throw discloseDispatchAfterSteps( + new AppError('COMMAND_FAILED', 'The runner did not return the press series orientation', { + reason: 'runner_sequence_orientation_unavailable', + completedSteps, + hint: 'Run snapshot -i to inspect the current UI before continuing with the remaining actions.', + }), + completedSteps, + ); +} + /** * Chunks by both the runner step cap and estimated wall time so no request approaches the 30s * main-thread watchdog. A single over-budget step remains a one-step chunk. diff --git a/packages/platform-apple/src/__tests__/interactions-press-series-runner-requests.test.ts b/packages/platform-apple/src/__tests__/interactions-press-series-runner-requests.test.ts new file mode 100644 index 0000000000..8de418ca1b --- /dev/null +++ b/packages/platform-apple/src/__tests__/interactions-press-series-runner-requests.test.ts @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + assertProducedRunnerRequests, + bindTestCommandHost, + IOS_SIMULATOR, +} from '@agent-device/test-kit'; +import type { RunnerCommand } from '@agent-device/apple-runner'; +import { createAppleInteractor } from '../interactor.ts'; + +bindTestCommandHost(); + +test('press-series continuations carry the original orientation on the runner wire', async () => { + const requests: RunnerCommand[] = []; + const interactor = createAppleInteractor( + IOS_SIMULATOR, + { appBundleId: 'com.example.app' }, + { env: process.env, cwd: process.cwd() }, + { + hasLiveSession: () => true, + runCommand: async (_device, request) => { + requests.push(request); + return { + sequenceOrientation: 1, + completedSteps: request.steps?.length ?? 0, + sequenceResults: request.steps?.map(({ kind }) => ({ ok: true, kind })) ?? [], + }; + }, + }, + ); + await interactor.pressPoint!( + { x: 10, y: 20 }, + { button: 'primary', count: 21, intervalMs: 0, holdMs: 0, jitterPx: 0, doubleTap: false }, + ); + assert.equal(requests.length, 2); + assertProducedRunnerRequests(import.meta.filename, [ + ['ios-simulator.runner-sequence.bound-orientation', requests[1]], + ]); +}); diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts index f1461b60bb..b1c76fc015 100644 --- a/packages/platform-apple/src/foldable/runner-panel.test.ts +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -265,6 +265,7 @@ test('every press-series chunk uses the lit-panel naming path', async () => { runCommand: async (target, command, options) => { await base.runCommand(target, command, options); return { + sequenceOrientation: 1, completedSteps: command.steps?.length ?? 0, sequenceResults: command.steps?.map(({ kind }) => ({ ok: true, kind })) ?? [], }; diff --git a/packages/platform-apple/src/interactions.test.ts b/packages/platform-apple/src/interactions.test.ts index 2d792780e4..30fb29a0fd 100644 --- a/packages/platform-apple/src/interactions.test.ts +++ b/packages/platform-apple/src/interactions.test.ts @@ -299,6 +299,7 @@ test('iosRunnerOverrides owns alternate-button presses', async () => { test('iosRunnerOverrides remaps a later chunk failure to global press indices', async () => { mockRunAppleRunnerCommand .mockResolvedValueOnce({ + sequenceOrientation: 1, completedSteps: 20, sequenceResults: Array.from({ length: 20 }, () => ({ ok: true, kind: 'tap' })), }) diff --git a/src/commands/schema/cli-help-gestures.ts b/src/commands/schema/cli-help-gestures.ts index acc7ee477c..d6965721a8 100644 --- a/src/commands/schema/cli-help-gestures.ts +++ b/src/commands/schema/cli-help-gestures.ts @@ -26,6 +26,7 @@ Shapes: For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. gesture tap touches 2 to 5 distinct points and holds them for --hold-ms (default 50, min 16); --count repeats the same contacts (cap 200) with --pause-ms between, the series must fit 60000ms counting about 500ms per repetition, and a repetition costs about 450ms on a physical iPhone. Use press for one finger. Platform quirks: + If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. Cancelled gesture and swipe requests return COMMAND_FAILED with details.reason=request_canceled and record no action. Repetition pauses are interruptible. iOS transform/pinch/rotate use private XCTest synthesis for a continuous two-finger pan/scale/rotation path; verify app metrics instead of assuming requested values map exactly to recognizer output. A gesture tap of three or more fingers can trigger iOS system gestures (undo/redo, the edit menu) and fast repeats can trigger double-tap recognizers, so verify app state; iOS touches the contacts down and lifts them together; Android touches them down up to 8ms apart and lifts them 8ms apart in reverse order. Android transform injects a geometric two-finger path; app recognizers may report non-exact pan/scale/rotation -- verify semantic app state or coarse per-component effects instead of exact numeric deltas unless the app exposes stable metrics. If Android needs exact values, prefer isolated gesture pan --pointer-count 2, gesture pinch, or gesture rotate over one combined transform: diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index ecce60609b..cc8b2a21bc 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -485,6 +485,8 @@ agent-device gesture tap 80 300 200 450 310 600 # three fingers at once agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two-finger tap ``` +If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. + A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point.