refactor(ios): bind surface geometry and synthesis orientation

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift

@@ -1,4 +1,5 @@
 import XCTest
+import AgentDeviceSnapshotPresentation
 
 // The lit panel of the command in flight (ADR 0025), main-owned and written behind main work a
 // previous command left queued, so that work still reads its own command's panel.
@@ -11,7 +12,7 @@ struct LitPanelSurface {
   let key: LitPanelBookingKey
   let displayID: UInt
   let window: XCUIElement
-  let frame: CGRect
+  let geometry: SurfaceGeometry
 }
 
 /// The command's lit panel and what it booked on it. Read and written only on the main thread.
@@ -79,7 +80,7 @@ extension RunnerTests {
     guard failure == .none, let element = window as? XCUIElement else {
       return RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure)
     }
-    litPanelState.surface = LitPanelSurface(key: key, displayID: litPanel.displayID, window: element, frame: frame)
+    litPanelState.surface = LitPanelSurface(key: key, displayID: litPanel.displayID, window: element, geometry: surfaceGeometry(windowFrame: frame, app: app))
     return nil
   }
 

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift

@@ -32,14 +32,7 @@ enum RunnerScrollKeyboardClip: Equatable {
 
 /** Where one directional scroll may place its swipe, once the keyboard has taken its share. */
 enum RunnerScrollViewport {
-  /**
-   * The band to plan the swipe inside, the frame to rotate its coordinates against, and the keyboard
-   * top when the band was clipped for one. The two frames are separate on purpose: a clip shortens
-   * only the band, while `CoordinateSpaceRotation.native(point:)` derives a `landscapeRight` native x from the
-   * frame's HEIGHT, so rotating inside the band moves the dispatched path sideways off the planned
-   * one.
-   */
-  case swipe(planFrame: CGRect, coordinateFrame: CGRect, keyboardMinY: Double?)
+  case swipe(planFrame: CGRect, keyboardMinY: Double?)
   /** Nothing to swipe. The caller answers `occlusionRunnerCode` and performs no gesture. */
   case occluded(keyboardMinY: Double, visibleHeight: Double)
 }
@@ -49,7 +42,6 @@ enum RunnerScrollViewport {
 struct ScrollGestureDispatch {
   let plan: RunnerScrollGesturePlan
   let planFrame: CGRect
-  let coordinateFrame: CGRect
   let keyboardMinY: Double?
 }
 
@@ -91,7 +83,7 @@ extension RunnerScrollViewport {
     switch self {
     case .occluded(let keyboardMinY, let visibleHeight):
       return .occluded(keyboardMinY: keyboardMinY, visibleHeight: visibleHeight)
-    case .swipe(let planFrame, let coordinateFrame, let keyboardMinY):
+    case .swipe(let planFrame, let keyboardMinY):
       guard planFrame.width > 0, planFrame.height > 0 else {
         return .unusableFrame
       }
@@ -108,7 +100,6 @@ extension RunnerScrollViewport {
         ScrollGestureDispatch(
           plan: plan,
           planFrame: planFrame,
-          coordinateFrame: coordinateFrame,
           keyboardMinY: keyboardMinY
         )
       )
@@ -168,9 +159,9 @@ enum ScrollViewportPolicy {
   static func frames(referenceFrame: CGRect, clip: RunnerScrollKeyboardClip) -> RunnerScrollViewport {
     switch clip {
     case .unobstructed:
-      return .swipe(planFrame: referenceFrame, coordinateFrame: referenceFrame, keyboardMinY: nil)
+      return .swipe(planFrame: referenceFrame, keyboardMinY: nil)
     case .avoided(let frame, let keyboardMinY):
-      return .swipe(planFrame: frame, coordinateFrame: referenceFrame, keyboardMinY: keyboardMinY)
+      return .swipe(planFrame: frame, keyboardMinY: keyboardMinY)
     case .occluded(let keyboardMinY, let visibleHeight):
       return .occluded(keyboardMinY: keyboardMinY, visibleHeight: visibleHeight)
     }
@@ -195,17 +186,17 @@ extension RunnerTests {
     // Every scroll reports its decision, including the two ways it avoids reading the keyboard at
     // all: unavailable accessibility health, and a probe that finds no keyboard.
     guard context.allowsKeyboardProbe else {
-      logScrollViewport(decision: "probeSkipped", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context)
-      return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: .unobstructed)
+      logScrollViewport(decision: "probeSkipped", keyboardMinY: nil, swipeHeight: context.geometry.windowOnScreen.rect.height, context: context)
+      return ScrollViewportPolicy.frames(referenceFrame: context.geometry.windowOnScreen.rect, clip: .unobstructed)
     }
     guard let keyboardFrame = visibleKeyboardFrame(app: app) else {
-      logScrollViewport(decision: "noKeyboard", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context)
-      return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: .unobstructed)
+      logScrollViewport(decision: "noKeyboard", keyboardMinY: nil, swipeHeight: context.geometry.windowOnScreen.rect.height, context: context)
+      return ScrollViewportPolicy.frames(referenceFrame: context.geometry.windowOnScreen.rect, clip: .unobstructed)
     }
-    let clip = ScrollViewportPolicy.clip(viewport: context.referenceFrame, keyboard: keyboardFrame)
+    let clip = ScrollViewportPolicy.clip(viewport: context.geometry.windowOnScreen.rect, keyboard: keyboardFrame)
     switch clip {
     case .unobstructed:
-      logScrollViewport(decision: "unobstructed", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context)
+      logScrollViewport(decision: "unobstructed", keyboardMinY: nil, swipeHeight: context.geometry.windowOnScreen.rect.height, context: context)
     case .avoided(let frame, let keyboardMinY):
       logScrollViewport(
         decision: "avoided",
@@ -221,7 +212,7 @@ extension RunnerTests {
         context: context
       )
     }
-    return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: clip)
+    return ScrollViewportPolicy.frames(referenceFrame: context.geometry.windowOnScreen.rect, clip: clip)
 #else
     let fallbackFrame = try resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
     return ScrollViewportPolicy.frames(referenceFrame: fallbackFrame, clip: .unobstructed)

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h

@@ -11,38 +11,36 @@ typedef NS_ENUM(NSInteger, RunnerTapSynthesisStatus) {
   RunnerTapSynthesisStatusFailed,
 };
 
+typedef struct {
+  CGPoint start;
+  CGPoint end;
+  double durationMs;
+  BOOL controlledScroll;
+} RunnerSynthesizedDrag;
+
 @interface RunnerSynthesizedGesture : NSObject
 
 // `resolvedWindow` is the app window the caller already resolved for the gesture's reference
 // frame, so the record's display ID is read from that same window. Pass nil for the sole-screen
 // route: the app's tree could not name its window and the screen is proven instead.
-+ (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application
-                                        resolvedWindow:(id _Nullable)resolvedWindow
-                                                    x:(double)x
-                                                    y:(double)y
-                                                   x2:(double)x2
-                                                   y2:(double)y2
-                                            durationMs:(double)durationMs;
-
-+ (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application
-                                                 resolvedWindow:(id _Nullable)resolvedWindow
-                                                                x:(double)x
-                                                                y:(double)y
-                                                               x2:(double)x2
-                                                               y2:(double)y2
-                                                        durationMs:(double)durationMs;
+// `interfaceOrientation` is the orientation used to convert the supplied points.
++ (NSString * _Nullable)synthesizeDragWithApplication:(id)application
+                                      resolvedWindow:(id _Nullable)resolvedWindow
+                                interfaceOrientation:(NSInteger)interfaceOrientation
+                                                drag:(RunnerSynthesizedDrag)drag;
 
 + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application
                                       resolvedWindow:(id _Nullable)resolvedWindow
-                                                   x:(double)x
-                                                   y:(double)y
+                                interfaceOrientation:(NSInteger)interfaceOrientation
+                                               point:(CGPoint)point
                                             deadline:(NSDate * _Nullable)deadline
                                         errorMessage:(NSString * _Nullable * _Nullable)errorMessage;
 
 // Each pointer is an ordered array of { x, y, offsetMs } samples. The first sample
 // starts contact; subsequent samples move it; all pointers lift at their final offset.
 + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application
                                           resolvedWindow:(id _Nullable)resolvedWindow
+                                    interfaceOrientation:(NSInteger)interfaceOrientation
                                           pointerSamples:(NSArray<NSArray<NSDictionary<NSString *, NSNumber *> *> *> *)pointerSamples;
 
 // UIInterfaceOrientation of the app (1 portrait, 2 upsideDown, 3 landscapeRight,

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m

@@ -17,7 +17,6 @@
 typedef struct {
   RunnerXCTestEventBridge core;
   SEL initRecordSelector;
-  SEL interfaceOrientationSelector;
   SEL initPathSelector;
   SEL moveSelector;
   SEL liftSelector;
@@ -37,6 +36,7 @@ typedef id (*RunnerDragPointerPathFactory)(
 static NSString * _Nullable RunnerCreateEventRecord(
   id application,
   id _Nullable resolvedWindow,
+  NSInteger interfaceOrientation,
   NSString *recordName,
   RunnerGestureEventBridge *bridge,
   id *record
@@ -60,6 +60,7 @@ static id RunnerControlledScrollPointerPath(
 static NSString * _Nullable RunnerTrySynthesizeDrag(
   id application,
   id _Nullable resolvedWindow,
+  NSInteger interfaceOrientation,
   CGPoint start,
   CGPoint end,
   double durationMs,
@@ -69,6 +70,7 @@ static id RunnerControlledScrollPointerPath(
 static RunnerTapSynthesisStatus RunnerTrySynthesizeTap(
   id application,
   id _Nullable resolvedWindow,
+  NSInteger interfaceOrientation,
   CGPoint point,
   NSDate * _Nullable deadline,
   NSString * _Nullable * _Nullable errorMessage
@@ -85,44 +87,15 @@ static id RunnerTapPointerPath(
 
 @implementation RunnerSynthesizedGesture
 
-+ (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application
-                                        resolvedWindow:(id _Nullable)resolvedWindow
-                                                     x:(double)x
-                                                     y:(double)y
-                                                    x2:(double)x2
-                                                    y2:(double)y2
-                                            durationMs:(double)durationMs {
-  @try {
-    return RunnerTrySynthesizeDrag(
-      application,
-      resolvedWindow,
-      CGPointMake(x, y),
-      CGPointMake(x2, y2),
-      durationMs,
-      @"agent-device-swipe",
-      RunnerSwipePointerPath
-    );
-  } @catch (NSException *exception) {
-    return RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed");
-  }
-}
-
-+ (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application
-                                                 resolvedWindow:(id _Nullable)resolvedWindow
-                                                                x:(double)x
-                                                                y:(double)y
-                                                               x2:(double)x2
-                                                               y2:(double)y2
-                                                       durationMs:(double)durationMs {
++ (NSString * _Nullable)synthesizeDragWithApplication:(id)application
+                                      resolvedWindow:(id _Nullable)resolvedWindow
+                                interfaceOrientation:(NSInteger)interfaceOrientation
+                                                drag:(RunnerSynthesizedDrag)drag {
   @try {
     return RunnerTrySynthesizeDrag(
-      application,
-      resolvedWindow,
-      CGPointMake(x, y),
-      CGPointMake(x2, y2),
-      durationMs,
-      @"agent-device-controlled-scroll",
-      RunnerControlledScrollPointerPath
+      application, resolvedWindow, interfaceOrientation, drag.start, drag.end, drag.durationMs,
+      drag.controlledScroll ? @"agent-device-controlled-scroll" : @"agent-device-swipe",
+      drag.controlledScroll ? RunnerControlledScrollPointerPath : RunnerSwipePointerPath
     );
   } @catch (NSException *exception) {
     return RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed");
@@ -131,13 +104,13 @@ + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)applicatio
 
 + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application
                                       resolvedWindow:(id _Nullable)resolvedWindow
-                                                   x:(double)x
-                                                   y:(double)y
+                                interfaceOrientation:(NSInteger)interfaceOrientation
+                                               point:(CGPoint)point
                                             deadline:(NSDate * _Nullable)deadline
                                         errorMessage:(NSString * _Nullable * _Nullable)errorMessage {
   if (errorMessage != NULL) *errorMessage = nil;
   @try {
-    return RunnerTrySynthesizeTap(application, resolvedWindow, CGPointMake(x, y), deadline, errorMessage);
+    return RunnerTrySynthesizeTap(application, resolvedWindow, interfaceOrientation, point, deadline, errorMessage);
   } @catch (NSException *exception) {
     if (errorMessage != NULL) {
       *errorMessage = RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed");
@@ -148,13 +121,15 @@ + (RunnerTapSynthesisStatus)synthesizeTapWithApplication:(id)application
 
 + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application
                                           resolvedWindow:(id _Nullable)resolvedWindow
+                                    interfaceOrientation:(NSInteger)interfaceOrientation
                                           pointerSamples:(NSArray<NSArray<NSDictionary<NSString *, NSNumber *> *> *> *)pointerSamples {
   @try {
     RunnerGestureEventBridge bridge;
     id record = nil;
     NSString *error = RunnerCreateEventRecord(
       application,
       resolvedWindow,
+      interfaceOrientation,
       @"agent-device-gesture-plan",
       &bridge,
       &record
@@ -210,6 +185,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application {
 static NSString * _Nullable RunnerTrySynthesizeDrag(
   id application,
   id _Nullable resolvedWindow,
+  NSInteger interfaceOrientation,
   CGPoint start,
   CGPoint end,
   double durationMs,
@@ -218,7 +194,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application {
 ) {
   RunnerGestureEventBridge bridge;
   id record = nil;
-  NSString *error = RunnerCreateEventRecord(application, resolvedWindow, recordName, &bridge, &record);
+  NSString *error = RunnerCreateEventRecord(application, resolvedWindow, interfaceOrientation, recordName, &bridge, &record);
   if (error != nil) return error;
 
   id path = pathFactory(&bridge, start, end, durationMs);
@@ -233,6 +209,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application {
 static RunnerTapSynthesisStatus RunnerTrySynthesizeTap(
   id application,
   id _Nullable resolvedWindow,
+  NSInteger interfaceOrientation,
   CGPoint point,
   NSDate * _Nullable deadline,
   NSString * _Nullable * _Nullable errorMessage
@@ -242,6 +219,7 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap(
   NSString *error = RunnerCreateEventRecord(
     application,
     resolvedWindow,
+    interfaceOrientation,
     @"agent-device-tap",
     &bridge,
     &record
@@ -274,7 +252,6 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap(
   if (missing != nil) return missing;
 
   SEL initRecordSelector = NSSelectorFromString(@"initWithName:displayID:interfaceOrientation:");
-  SEL interfaceOrientationSelector = NSSelectorFromString(@"interfaceOrientation");
   SEL initPathSelector = NSSelectorFromString(@"initForTouchAtPoint:offset:");
   SEL moveSelector = NSSelectorFromString(@"moveToPoint:atOffset:");
   SEL liftSelector = NSSelectorFromString(@"liftUpAtOffset:");
@@ -295,15 +272,10 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap(
     core.pathClass, liftSelector, @"liftUpAtOffset:", RunnerGestureSynthesisSurface
   );
   if (missing != nil) return missing;
-  missing = RunnerRequireApplicationSelector(
-    application, interfaceOrientationSelector, @"interfaceOrientation", RunnerGestureSynthesisSurface
-  );
-  if (missing != nil) return missing;
 
   *bridge = (RunnerGestureEventBridge){
     .core = core,
     .initRecordSelector = initRecordSelector,
-    .interfaceOrientationSelector = interfaceOrientationSelector,
     .initPathSelector = initPathSelector,
     .moveSelector = moveSelector,
     .liftSelector = liftSelector,
@@ -314,6 +286,7 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap(
 static NSString * _Nullable RunnerCreateEventRecord(
   id application,
   id _Nullable resolvedWindow,
+  NSInteger interfaceOrientation,
   NSString *recordName,
   RunnerGestureEventBridge *bridge,
   id *record
@@ -329,8 +302,6 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap(
     ? RunnerResolveSoleScreen(application, &soleScreen, &displayID)
     : RunnerResolveWindowDisplayID(resolvedWindow, &displayID);
   if (displayError != nil) return displayError;
-  NSInteger interfaceOrientation =
-    ((RunnerMsgSendInteger)objc_msgSend)(application, bridge->interfaceOrientationSelector);
   NSInteger targetProcessID =
     ((RunnerMsgSendInteger)objc_msgSend)(application, bridge->core.processIDSelector);
   if (targetProcessID <= 0) {

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift

@@ -211,7 +211,7 @@ extension RunnerTests {
         if let window = alert.window {
           // Never fall back to the owner app's first window: on a foldable it can be another panel.
           guard let context = SynthesizedCoordinateContext(
-            validating: safely("ALERT_WINDOW", CGRect.null) { window.frame },
+            validating: surfaceGeometry(windowFrame: safely("ALERT_WINDOW", CGRect.null) { window.frame }, app: alert.ownerApp),
             resolvedWindow: window,
             accessibilityHealth: mainOwned.accessibilityHealth
           ) else {

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -247,7 +247,7 @@ extension RunnerTests {
         try synthesizedLongPressAt(app: activeApp, x: x, y: y, durationMs: durationMs, context: context)
       }) {
       case .performed(let timing):
-        let touchFrame = context.map { touchVisualizationFrame(x: x, y: y, context: $0) }
+        let touchFrame = context.map { touchVisualizationFrame(point: SurfacePoint(CGPoint(x: x, y: y)), context: $0) }
         return gestureResponse(message: "long pressed", timing: timing, frame: .touch(touchFrame))
       case .xctestFallback(let message, let hint):
         fallback = GestureFallback(strategy: "xctest-coordinate-long-press", message: message, hint: hint)
@@ -338,7 +338,7 @@ extension RunnerTests {
             y2: gesture.planFrame.minY + gesture.plan.y2,
             durationMs: defaults.durationMs,
             message: "scrolled",
-            context: scrollContext.withReferenceFrame(gesture.coordinateFrame),
+            context: scrollContext,
             releaseBehavior: command.scrollReleaseBehavior
           )
         )
@@ -680,7 +680,7 @@ extension RunnerTests {
         )
       case .sampled:
         let (timing, outcome) = try performGesture(activeApp, idleTimeout: false) {
-          sampledPlannedGesture(app: activeApp, plan: plan)
+          try sampledPlannedGesture(app: activeApp, plan: plan)
         }
         return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome)
       }