From 360110193a4bcbb5fe707285fda39b39135294d3 Mon Sep 17 00:00:00 2001 From: gronxb Date: Wed, 4 Nov 2026 12:48:15 +0900 Subject: [PATCH 01/12] fix(ios): deliver simulator edge swipes as screen-edge touches XCTest synthesized touches carry no digitizer edge mask, so on an iOS simulator gesture swipe left-edge and right-edge never started a UIScreenEdgePanGestureRecognizer or the navigation edge pop gesture. Edge swipe presets now plan a screen-edge contact. On an iOS simulator the runner dispatches it as one HID touch with the edge mask, through the touchscreen service of the app's display. Other gestures and physical devices keep their current paths. Fixes #4208 --- ...AgentDeviceRunnerUITests-Bridging-Header.h | 1 + .../RunnerSimulatorEdgeTouch.h | 28 ++ .../RunnerSimulatorEdgeTouch.m | 281 ++++++++++++++++++ .../RunnerTests+CommandExecution.swift | 7 + .../RunnerTests+Interaction.swift | 4 + .../RunnerTests+Models.swift | 2 + .../RunnerTests+ScreenEdgeGesture.swift | 47 +++ .../RunnerTests+SynthesizedInteraction.swift | 66 +++- ...nerTests+SynthesizedInteractionTests.swift | 40 +++ contracts/fixtures/runner-requests.json | 1 + .../src/gesture-plan-pointers.test.ts | 13 + packages/contracts/src/gesture-plan-types.ts | 7 + packages/contracts/src/gesture-plan.ts | 4 +- packages/contracts/src/scroll-gesture.ts | 5 + .../__tests__/index-runner-requests.test.ts | 11 + 15 files changed, 505 insertions(+), 12 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.h create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenEdgeGesture.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h index 33fd2fb488..90b894ea09 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h @@ -1,5 +1,6 @@ #import "RunnerObjCExceptionCatcher.h" #import "RunnerAXSnapshotBridge.h" +#import "RunnerSimulatorEdgeTouch.h" #import "RunnerSynthesizedGesture.h" #import "RunnerSynthesizedTextEntry.h" #import "RunnerXCTestEventBridge.h" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.h new file mode 100644 index 0000000000..0def8170ea --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.h @@ -0,0 +1,28 @@ +#import +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +#if TARGET_OS_IOS && TARGET_OS_SIMULATOR + +// Dispatches one touch that UIKit marks as a screen-edge touch, so +// UIScreenEdgePanGestureRecognizer and the navigation back gesture can begin. +// XCTest synthesized touches never carry the digitizer edge mask that UIKit reads. +@interface RunnerSimulatorEdgeTouch : NSObject + +// The touch goes to the touchscreen of the display that hosts the app: `resolvedWindow`'s display, +// or the sole screen when it is nil, as for XCTest synthesis. `samples` is the ordered +// { x, y, offsetMs } path of one contact in portrait screen points, and `screenSize` is that +// screen's portrait size in points. The call returns after the contact lifts. +// Returns nil on success, or the reason the touch could not be dispatched. ++ (NSString * _Nullable)dispatchWithApplication:(id)application + resolvedWindow:(id _Nullable)resolvedWindow + samples:(NSArray *> *)samples + screenSize:(CGSize)screenSize; + +@end + +#endif + +NS_ASSUME_NONNULL_END diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m new file mode 100644 index 0000000000..8b691b538e --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m @@ -0,0 +1,281 @@ +#import "RunnerSimulatorEdgeTouch.h" + +#if TARGET_OS_IOS && TARGET_OS_SIMULATOR + +#import "RunnerXCTestEventBridge.h" +#import +#import +#import + +typedef CFTypeRef RunnerHIDEventRef; +typedef RunnerHIDEventRef (*RunnerCreateDigitizerEvent)( + CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, double, double, + double, double, double, Boolean, Boolean, uint32_t); +typedef RunnerHIDEventRef (*RunnerCreateDigitizerFingerEvent)( + CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, double, double, double, double, double, + Boolean, Boolean, uint32_t); +typedef void (*RunnerAppendEvent)(RunnerHIDEventRef, RunnerHIDEventRef, uint32_t); +typedef void (*RunnerSetIntegerValue)(RunnerHIDEventRef, uint32_t, long); +typedef void (*RunnerSetFloatValue)(RunnerHIDEventRef, uint32_t, double); +typedef void (*RunnerSetSenderID)(RunnerHIDEventRef, uint64_t); +typedef CFTypeRef (*RunnerCreateEventSystemClient)(CFAllocatorRef); +typedef CFArrayRef (*RunnerCopyServices)(CFTypeRef); +typedef CFTypeRef (*RunnerCopyServiceProperty)(CFTypeRef, CFStringRef); +typedef CFTypeRef (*RunnerGetServiceRegistryID)(CFTypeRef); +typedef void (*RunnerDispatchEvent)(CFTypeRef, RunnerHIDEventRef); + +typedef struct { + RunnerCreateDigitizerEvent createDigitizerEvent; + RunnerCreateDigitizerFingerEvent createFingerEvent; + RunnerAppendEvent appendEvent; + RunnerSetIntegerValue setIntegerValue; + RunnerSetFloatValue setFloatValue; + RunnerSetSenderID setSenderID; + RunnerCreateEventSystemClient createClient; + RunnerCopyServices copyServices; + RunnerCopyServiceProperty copyServiceProperty; + RunnerGetServiceRegistryID getServiceRegistryID; + RunnerDispatchEvent dispatchEvent; +} RunnerHIDFunctions; + +// IOHIDDigitizerEventMask bits. +static const uint32_t RunnerDigitizerRange = 1u << 0; +static const uint32_t RunnerDigitizerTouch = 1u << 1; +static const uint32_t RunnerDigitizerPosition = 1u << 2; +static const uint32_t RunnerDigitizerIdentity = 1u << 5; +static const uint32_t RunnerDigitizerFromEdgeFlat = 1u << 10; +static const uint32_t RunnerDigitizerFromEdgeTip = 1u << 11; +static const uint32_t RunnerDigitizerEdgeMask = RunnerDigitizerFromEdgeFlat | RunnerDigitizerFromEdgeTip; + +// IOHIDEventField values of the digitizer event type (11). +static const uint32_t RunnerDigitizerFieldMajorRadius = (11u << 16) | 20u; +static const uint32_t RunnerDigitizerFieldMinorRadius = (11u << 16) | 21u; +static const uint32_t RunnerDigitizerFieldIsDisplayIntegrated = (11u << 16) | 25u; + +static const uint32_t RunnerTransducerHand = 3; +static const uint32_t RunnerFingerIdentity = 2; +static const double RunnerFingerRadius = 0.04; +static const double RunnerFingerPressure = 0.5; +static const double RunnerFrameIntervalMs = 16.0; + +// The simulator touchscreen HID service: digitizer usage page, touch screen usage. +static const long RunnerDigitizerUsagePage = 13; +static const long RunnerTouchScreenUsage = 4; + +static NSString * _Nullable RunnerResolveHIDFunctions(RunnerHIDFunctions *functions) { + void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW); + if (iokit == NULL) { + return @"simulator HID dispatch is unavailable: IOKit could not be loaded"; + } + struct { void **slot; const char *name; } symbols[] = { + { (void **)&functions->createDigitizerEvent, "IOHIDEventCreateDigitizerEvent" }, + { (void **)&functions->createFingerEvent, "IOHIDEventCreateDigitizerFingerEvent" }, + { (void **)&functions->appendEvent, "IOHIDEventAppendEvent" }, + { (void **)&functions->setIntegerValue, "IOHIDEventSetIntegerValue" }, + { (void **)&functions->setFloatValue, "IOHIDEventSetFloatValue" }, + { (void **)&functions->setSenderID, "IOHIDEventSetSenderID" }, + { (void **)&functions->createClient, "IOHIDEventSystemClientCreate" }, + { (void **)&functions->copyServices, "IOHIDEventSystemClientCopyServices" }, + { (void **)&functions->copyServiceProperty, "IOHIDServiceClientCopyProperty" }, + { (void **)&functions->getServiceRegistryID, "IOHIDServiceClientGetRegistryID" }, + { (void **)&functions->dispatchEvent, "IOHIDEventSystemClientDispatchEvent" }, + }; + for (size_t index = 0; index < sizeof(symbols) / sizeof(symbols[0]); index++) { + *symbols[index].slot = dlsym(iokit, symbols[index].name); + if (*symbols[index].slot == NULL) { + return [NSString stringWithFormat:@"simulator HID dispatch is unavailable: %s is missing", + symbols[index].name]; + } + } + return nil; +} + +static long RunnerServiceIntegerProperty( + const RunnerHIDFunctions *functions, + CFTypeRef service, + CFStringRef key +) { + CFTypeRef value = functions->copyServiceProperty(service, key); + long result = -1; + if (value != NULL && CFGetTypeID(value) == CFNumberGetTypeID()) { + CFNumberGetValue((CFNumberRef)value, kCFNumberLongType, &result); + } + if (value != NULL) CFRelease(value); + return result; +} + +static NSString * _Nullable RunnerServiceStringProperty( + const RunnerHIDFunctions *functions, + CFTypeRef service, + CFStringRef key +) { + CFTypeRef value = functions->copyServiceProperty(service, key); + NSString *result = nil; + if (value != NULL && CFGetTypeID(value) == CFStringGetTypeID()) { + result = [(__bridge NSString *)value copy]; + } + if (value != NULL) CFRelease(value); + return result; +} + +// The display UUID that a touchscreen HID service names for `displayID`, the XCTest display ID. +// A simulator runtime that gives its displays no UUID answers nil, and so do its touchscreens. +static NSString * _Nullable RunnerDisplayUUID(NSUInteger displayID) { + Class displayClass = NSClassFromString(@"CADisplay"); + SEL displaysSelector = NSSelectorFromString(@"displays"); + SEL displayIdSelector = NSSelectorFromString(@"displayId"); + SEL uniqueIdSelector = NSSelectorFromString(@"uniqueId"); + if (displayClass == Nil || ![displayClass respondsToSelector:displaysSelector]) return nil; + NSArray *displays = ((id (*)(id, SEL))objc_msgSend)(displayClass, displaysSelector); + for (id display in displays) { + if (![display respondsToSelector:displayIdSelector] || ![display respondsToSelector:uniqueIdSelector]) continue; + if (((unsigned int (*)(id, SEL))objc_msgSend)(display, displayIdSelector) != displayID) continue; + id uniqueId = ((id (*)(id, SEL))objc_msgSend)(display, uniqueIdSelector); + return [uniqueId isKindOfClass:[NSString class]] ? uniqueId : nil; + } + return nil; +} + +// Touches reach the app only with its display's touchscreen service as their sender. +static NSString * _Nullable RunnerResolveTouchscreenSender( + const RunnerHIDFunctions *functions, + CFTypeRef client, + NSUInteger displayID, + uint64_t *sender +) { + NSString *displayUUID = RunnerDisplayUUID(displayID); + CFArrayRef services = functions->copyServices(client); + NSUInteger matches = 0; + for (CFIndex index = 0; services != NULL && index < CFArrayGetCount(services); index++) { + CFTypeRef service = CFArrayGetValueAtIndex(services, index); + if (RunnerServiceIntegerProperty(functions, service, CFSTR("PrimaryUsagePage")) != RunnerDigitizerUsagePage || + RunnerServiceIntegerProperty(functions, service, CFSTR("PrimaryUsage")) != RunnerTouchScreenUsage || + ![RunnerServiceStringProperty(functions, service, CFSTR("Transport")) isEqualToString:@"CoreDevice"]) { + continue; + } + NSString *serviceDisplayUUID = RunnerServiceStringProperty(functions, service, CFSTR("displayUUID")); + if (serviceDisplayUUID != displayUUID && ![serviceDisplayUUID isEqualToString:displayUUID]) continue; + CFTypeRef registryID = functions->getServiceRegistryID(service); + if (registryID != NULL && CFGetTypeID(registryID) == CFNumberGetTypeID()) { + CFNumberGetValue((CFNumberRef)registryID, kCFNumberSInt64Type, sender); + matches += 1; + } + } + if (services != NULL) CFRelease(services); + if (matches == 1) return nil; + return [NSString stringWithFormat:@"display %lu has %lu simulator touchscreen HID services; an edge touch needs exactly one", + (unsigned long)displayID, (unsigned long)matches]; +} + +static void RunnerDispatchFrame( + const RunnerHIDFunctions *functions, + CFTypeRef client, + uint64_t sender, + CGPoint normalized, + uint32_t mask, + Boolean touching +) { + uint64_t timestamp = mach_absolute_time(); + RunnerHIDEventRef hand = functions->createDigitizerEvent( + kCFAllocatorDefault, timestamp, RunnerTransducerHand, 0, 0, mask, 0, + normalized.x, normalized.y, 0, 0, 0, touching, touching, 0); + RunnerHIDEventRef finger = functions->createFingerEvent( + kCFAllocatorDefault, timestamp, 1, RunnerFingerIdentity, mask, normalized.x, normalized.y, 0, + touching ? RunnerFingerPressure : 0, 0, touching, touching, 0); + if (hand == NULL || finger == NULL) { + if (hand != NULL) CFRelease(hand); + if (finger != NULL) CFRelease(finger); + return; + } + functions->setIntegerValue(hand, RunnerDigitizerFieldIsDisplayIntegrated, 1); + functions->setFloatValue(finger, RunnerDigitizerFieldMajorRadius, RunnerFingerRadius); + functions->setFloatValue(finger, RunnerDigitizerFieldMinorRadius, RunnerFingerRadius); + functions->appendEvent(hand, finger, 0); + functions->setSenderID(hand, sender); + functions->dispatchEvent(client, hand); + CFRelease(finger); + CFRelease(hand); +} + +static CGPoint RunnerSamplePoint(NSDictionary *sample, CGSize screenSize) { + return CGPointMake(sample[@"x"].doubleValue / screenSize.width, sample[@"y"].doubleValue / screenSize.height); +} + +// The contact position at `offsetMs`, interpolated between the planned samples. +static CGPoint RunnerPointAtOffset( + NSArray *> *samples, + double offsetMs, + CGSize screenSize +) { + for (NSUInteger index = 1; index < samples.count; index++) { + NSDictionary *end = samples[index]; + if (offsetMs > end[@"offsetMs"].doubleValue) continue; + NSDictionary *start = samples[index - 1]; + double startOffset = start[@"offsetMs"].doubleValue; + double span = end[@"offsetMs"].doubleValue - startOffset; + double progress = span > 0 ? (offsetMs - startOffset) / span : 1; + CGPoint from = RunnerSamplePoint(start, screenSize); + CGPoint to = RunnerSamplePoint(end, screenSize); + return CGPointMake(from.x + (to.x - from.x) * progress, from.y + (to.y - from.y) * progress); + } + return RunnerSamplePoint(samples.lastObject, screenSize); +} + +static void RunnerWaitUntil(uint64_t startTicks, double offsetMs) { + static mach_timebase_info_data_t timebase; + if (timebase.denom == 0) mach_timebase_info(&timebase); + uint64_t nanoseconds = (uint64_t)(offsetMs * NSEC_PER_MSEC); + mach_wait_until(startTicks + nanoseconds * timebase.denom / timebase.numer); +} + +@implementation RunnerSimulatorEdgeTouch + ++ (NSString * _Nullable)dispatchWithApplication:(id)application + resolvedWindow:(id _Nullable)resolvedWindow + samples:(NSArray *> *)samples + screenSize:(CGSize)screenSize { + if (samples.count < 2 || screenSize.width <= 0 || screenSize.height <= 0) { + return @"an edge touch needs at least two samples and a measured screen"; + } + NSUInteger displayID = 0; + id soleScreen = nil; + NSString *displayError = resolvedWindow == nil + ? RunnerResolveSoleScreen(application, &soleScreen, &displayID) + : RunnerResolveWindowDisplayID(resolvedWindow, &displayID); + if (displayError != nil) return displayError; + RunnerHIDFunctions functions = {0}; + NSString *failure = RunnerResolveHIDFunctions(&functions); + if (failure != nil) return failure; + CFTypeRef client = functions.createClient(kCFAllocatorDefault); + if (client == NULL) return @"simulator HID dispatch is unavailable: no event system client"; + uint64_t sender = 0; + failure = RunnerResolveTouchscreenSender(&functions, client, displayID, &sender); + if (failure != nil) { + CFRelease(client); + return failure; + } + + double durationMs = samples.lastObject[@"offsetMs"].doubleValue; + uint64_t startTicks = mach_absolute_time(); + RunnerDispatchFrame( + &functions, client, sender, RunnerPointAtOffset(samples, 0, screenSize), + RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerIdentity | RunnerDigitizerEdgeMask, true); + for (double offsetMs = RunnerFrameIntervalMs; offsetMs < durationMs + RunnerFrameIntervalMs; + offsetMs += RunnerFrameIntervalMs) { + double frameOffset = MIN(offsetMs, durationMs); + RunnerWaitUntil(startTicks, frameOffset); + RunnerDispatchFrame( + &functions, client, sender, RunnerPointAtOffset(samples, frameOffset, screenSize), + RunnerDigitizerPosition | RunnerDigitizerEdgeMask, true); + } + RunnerWaitUntil(startTicks, durationMs + RunnerFrameIntervalMs); + RunnerDispatchFrame( + &functions, client, sender, RunnerPointAtOffset(samples, durationMs, screenSize), + RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerIdentity, false); + CFRelease(client); + return nil; +} + +@end + +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 1fc524fbb1..8786a127e5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -683,6 +683,13 @@ extension RunnerTests { try sampledPlannedGesture(app: activeApp, plan: plan) } return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) +#if os(iOS) && targetEnvironment(simulator) + case .screenEdgeTouch: + let (timing, outcome) = try performGesture(activeApp, idleTimeout: false) { + try screenEdgePlannedGesture(app: activeApp, plan: plan) + } + return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) +#endif } case .gestureViewport: let frame = try gestureViewportFrame(app: activeApp) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index ab24ad6f58..28cdc54ebf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -15,6 +15,10 @@ extension RunnerTests { enum PlannedGestureExecution: Equatable { case fastSwipe case sampled +#if os(iOS) && targetEnvironment(simulator) + /// One HID touch with the digitizer edge mask, which only a simulator can dispatch. + case screenEdgeTouch +#endif } enum SynthesizedDragProfile: Equatable { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index d2920e728e..d04bddb86c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -343,6 +343,8 @@ struct RunnerGesturePlan: Codable { let topology: String let intent: String let executionProfile: String? + /// `runnerScreenEdgeContactOrigin` when the contact starts as a screen-edge touch. + let contactOrigin: String? let durationMs: Double let viewport: RunnerGestureViewport let pointers: [RunnerGesturePointer] diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenEdgeGesture.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenEdgeGesture.swift new file mode 100644 index 0000000000..40f93e7988 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenEdgeGesture.swift @@ -0,0 +1,47 @@ +import XCTest + +/// The plan value for a contact that starts as a screen-edge touch. Mirrors +/// `GestureContactOrigin` in packages/contracts/src/gesture-plan-types.ts. +let runnerScreenEdgeContactOrigin = "screen-edge" + +extension RunnerTests { +#if os(iOS) && targetEnvironment(simulator) + /// Dispatches a screen-edge plan as one HID touch with the digitizer edge mask. XCTest + /// synthesized touches never carry that mask, so UIKit edge recognizers ignore them. + @MainActor + func screenEdgePlannedGesture( + app: XCUIApplication, + plan: RunnerGesturePlan + ) throws -> RunnerInteractionOutcome { + // The coordinate context books a foldable's lit-panel window and keeps its typed refusal. + let planned: NativePlannedGesture + switch try nativePlannedGesture(app: app, plan: plan, gesture: "edge swipe") { + case .resolved(let resolved): planned = resolved + case .refused(let outcome): return outcome + } + guard let portraitSize = edgeTouchDisplayPortraitSize(), let samples = planned.pointerSamples.first else { + return Self.unmeasuredSoleDisplayOutcome(gesture: "edge swipe") + } + if let message = RunnerSimulatorEdgeTouch.dispatch( + withApplication: app, + resolvedWindow: planned.context.resolvedWindow, + samples: samples, + screenSize: portraitSize + ) { + return .unsupported( + message: message, + hint: "Retry after the app is foregrounded on a simulator display with a touchscreen." + ) + } + return .performed + } + + /// The portrait size of the display that hosts the app: the request's lit panel on a foldable, + /// whose sides swap with the window's turn, else the sole screen. + @MainActor + private func edgeTouchDisplayPortraitSize() -> CGSize? { + guard let panel = requestLitPanel else { return soleDisplayPortraitSize() } + return CGSize(width: min(panel.widthPt, panel.heightPt), height: max(panel.widthPt, panel.heightPt)) + } +#endif +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index f644c06599..ea7ab31495 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -589,7 +589,7 @@ extension RunnerTests { /// Not `UIScreen.main`: the runner process can read that in compatibility mode (#4091). A failed /// read is not kept, so the next gesture reads again. @MainActor - private func soleDisplayPortraitSize() -> CGSize? { + func soleDisplayPortraitSize() -> CGSize? { if let cached = mainOwned.soleDisplayPortraitSize { return cached } let screen = RunnerSoleScreenSize() guard screen.width > 0, screen.height > 0 else { return nil } @@ -672,6 +672,14 @@ extension RunnerTests { } else if plan.executionProfile != nil { return "multi-touch gesture cannot define a single-pointer execution profile" } + if let contactOrigin = plan.contactOrigin { + guard contactOrigin == runnerScreenEdgeContactOrigin else { + return "planned gesture has an unsupported contact origin" + } + guard plan.topology == "single" else { + return "only a single-pointer gesture can start as a screen-edge touch" + } + } guard plan.durationMs.isFinite, plan.durationMs >= 16, plan.durationMs <= 10_000 else { return "planned gesture durationMs must be between 16 and 10000" } @@ -756,22 +764,41 @@ extension RunnerTests { } func plannedGestureExecution(for plan: RunnerGesturePlan) -> PlannedGestureExecution { - plan.topology == "single" && plan.executionProfile == "endpoint-hold" +#if os(iOS) && targetEnvironment(simulator) + if plan.contactOrigin == runnerScreenEdgeContactOrigin { + return .screenEdgeTouch + } +#endif + return plan.topology == "single" && plan.executionProfile == "endpoint-hold" ? .fastSwipe : .sampled } +#if os(iOS) + /// A planned gesture's surface and each pointer's `{ x, y, offsetMs }` samples in native + /// (portrait-up) digitizer points. + struct NativePlannedGesture { + let context: SynthesizedCoordinateContext + let orientation: Int + let pointerSamples: [[[String: NSNumber]]] + } + + enum NativePlannedGestureResolution { + case resolved(NativePlannedGesture) + case refused(RunnerInteractionOutcome) + } + @MainActor - func sampledPlannedGesture( + func nativePlannedGesture( app: XCUIApplication, - plan: RunnerGesturePlan - ) throws -> RunnerInteractionOutcome { -#if os(iOS) + plan: RunnerGesturePlan, + gesture: String + ) throws -> NativePlannedGestureResolution { guard let context = try synthesizedCoordinateContext(app: app) else { - return .unsupported(message: "gesture could not resolve a surface", hint: nil) + return .refused(.unsupported(message: "\(gesture) could not resolve a surface", hint: nil)) } guard let geometry = digitizerGeometry(app: app, surface: context.geometry) else { - return Self.unmeasuredSoleDisplayOutcome(gesture: "gesture") + return .refused(Self.unmeasuredSoleDisplayOutcome(gesture: gesture)) } let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in pointer.samples.map { sample in @@ -783,11 +810,28 @@ extension RunnerTests { ] } } + return .resolved(NativePlannedGesture( + context: context, orientation: geometry.orientation, pointerSamples: pointerSamples + )) + } +#endif + + @MainActor + func sampledPlannedGesture( + app: XCUIApplication, + plan: RunnerGesturePlan + ) throws -> RunnerInteractionOutcome { +#if os(iOS) + let planned: NativePlannedGesture + switch try nativePlannedGesture(app: app, plan: plan, gesture: "gesture") { + case .resolved(let resolved): planned = resolved + case .refused(let outcome): return outcome + } if let message = RunnerSynthesizedGesture.synthesizeGesture( withApplication: app, - resolvedWindow: context.resolvedWindow, - interfaceOrientation: geometry.orientation, - pointerSamples: pointerSamples + resolvedWindow: planned.context.resolvedWindow, + interfaceOrientation: planned.orientation, + pointerSamples: planned.pointerSamples ) { return .unsupported( message: message, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index e5fe6918ef..de2eef5983 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -675,5 +675,45 @@ extension RunnerTests { ) } + func testScreenEdgeFlingUsesTheEdgeTouchOnlyOnASimulator() throws { + let edge = try decodedFlingPlan(contactOrigin: #""screen-edge""#) + let inPage = try decodedFlingPlan(contactOrigin: nil) + + XCTAssertNil(plannedGestureValidationError(edge)) +#if os(iOS) && targetEnvironment(simulator) + XCTAssertEqual(plannedGestureExecution(for: edge), .screenEdgeTouch) +#else + XCTAssertEqual(plannedGestureExecution(for: edge), .fastSwipe) +#endif + XCTAssertEqual(plannedGestureExecution(for: inPage), .fastSwipe) + } + + func testPlannedGestureRejectsAnUnknownContactOrigin() throws { + let plan = try decodedFlingPlan(contactOrigin: #""screen-corner""#) + + XCTAssertEqual(plannedGestureValidationError(plan), "planned gesture has an unsupported contact origin") + } + + func testMultiTouchGestureCannotStartAsAScreenEdgeTouch() throws { + let plan = try JSONDecoder().decode( + RunnerGesturePlan.self, + from: Data( + #"{"topology":"multi","intent":"pan","contactOrigin":"screen-edge","durationMs":32,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":80,"y":80}},{"offsetMs":32,"point":{"x":100,"y":90}}]},{"pointerId":1,"samples":[{"offsetMs":0,"point":{"x":80,"y":120}},{"offsetMs":32,"point":{"x":100,"y":130}}]}]}"#.utf8 + ) + ) + + XCTAssertEqual( + plannedGestureValidationError(plan), + "only a single-pointer gesture can start as a screen-edge touch" + ) + } + + /// A right-edge fling; `contactOrigin` is the raw JSON value, or nil to omit the field. + private func decodedFlingPlan(contactOrigin: String?) throws -> RunnerGesturePlan { + let origin = contactOrigin.map { #""contactOrigin":\#($0),"# } ?? "" + let json = #"{"topology":"single","intent":"fling","executionProfile":"endpoint-hold",\#(origin)"durationMs":100,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":8,"y":150}},{"offsetMs":100,"point":{"x":170,"y":150}}]}]}"# + return try JSONDecoder().decode(RunnerGesturePlan.self, from: Data(json.utf8)) + } + } #endif diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index a4be401fd4..484b08fd5e 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -22,6 +22,7 @@ {"name": "ios-simulator.interactions-focus.synthesized", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "tap", "x": 10, "y": 20, "synthesized": true, "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactions-gesture-viewport.read", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "gestureViewport", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactions-gesture.multi", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "gesture", "gesturePlan": {"topology": "multi", "intent": "pinch", "durationMs": 250, "viewport": {"x": 0, "y": 0, "width": 390, "height": 844}, "pointers": [{"pointerId": 0, "samples": [{"offsetMs": 0, "point": {"x": 150, "y": 422}}, {"offsetMs": 250, "point": {"x": 100, "y": 422}}]}, {"pointerId": 1, "samples": [{"offsetMs": 0, "point": {"x": 240, "y": 422}}, {"offsetMs": 250, "point": {"x": 290, "y": 422}}]}]}, "appBundleId": "com.example.app", "commandId": ""}}, + {"name": "ios-simulator.interactions-gesture.screen-edge", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "gesture", "gesturePlan": {"topology": "single", "intent": "fling", "executionProfile": "endpoint-hold", "durationMs": 100, "viewport": {"x": 0, "y": 0, "width": 390, "height": 844}, "pointers": [{"pointerId": 0, "samples": [{"offsetMs": 0, "point": {"x": 8, "y": 422}}, {"offsetMs": 100, "point": {"x": 331, "y": 422}}]}], "contactOrigin": "screen-edge"}, "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactions-gesture.single", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "gesture", "gesturePlan": {"topology": "single", "intent": "pan", "executionProfile": "timed-pan", "durationMs": 120, "viewport": {"x": 0, "y": 0, "width": 390, "height": 844}, "pointers": [{"pointerId": 0, "samples": [{"offsetMs": 0, "point": {"x": 100, "y": 400}}, {"offsetMs": 120, "point": {"x": 100, "y": 200}}]}]}, "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactions-gesture.tap", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "gesture", "gesturePlan": {"topology": "multi", "intent": "tap", "durationMs": 50, "viewport": {"x": 0, "y": 0, "width": 390, "height": 844}, "pointers": [{"pointerId": 0, "samples": [{"offsetMs": 0, "point": {"x": 100, "y": 300}}, {"offsetMs": 50, "point": {"x": 100, "y": 300}}]}, {"pointerId": 1, "samples": [{"offsetMs": 0, "point": {"x": 200, "y": 450}}, {"offsetMs": 50, "point": {"x": 200, "y": 450}}]}, {"pointerId": 2, "samples": [{"offsetMs": 0, "point": {"x": 300, "y": 600}}, {"offsetMs": 50, "point": {"x": 300, "y": 600}}]}]}, "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactions-long-press.duration", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "longPress", "x": 10, "y": 20, "durationMs": 600, "appBundleId": "com.example.app", "commandId": ""}}, diff --git a/packages/contracts/src/gesture-plan-pointers.test.ts b/packages/contracts/src/gesture-plan-pointers.test.ts index 4c4ae77cd7..a98029c077 100644 --- a/packages/contracts/src/gesture-plan-pointers.test.ts +++ b/packages/contracts/src/gesture-plan-pointers.test.ts @@ -64,6 +64,19 @@ describe('single-pointer plans', () => { assert.deepEqual(rightFling.pointers[0].samples.at(-1)?.point, { x: 360, y: 430 }); }); + test('edge presets start as screen-edge contacts and in-page presets do not', () => { + const contactOrigin = (preset: 'left' | 'right' | 'left-edge' | 'right-edge') => { + const plan = buildGesturePlan({ intent: 'fling', preset }, PORTRAIT); + assert.ok(plan.topology === 'single'); + return plan.contactOrigin; + }; + + assert.equal(contactOrigin('left-edge'), 'screen-edge'); + assert.equal(contactOrigin('right-edge'), 'screen-edge'); + assert.equal(contactOrigin('left'), undefined); + assert.equal(contactOrigin('right'), undefined); + }); + test('single-pointer pan preserves its execution profile', () => { const plan = buildGesturePlan( { diff --git a/packages/contracts/src/gesture-plan-types.ts b/packages/contracts/src/gesture-plan-types.ts index ee3d5183b2..5d94d0335b 100644 --- a/packages/contracts/src/gesture-plan-types.ts +++ b/packages/contracts/src/gesture-plan-types.ts @@ -87,10 +87,17 @@ export type SinglePointerTrajectory = { ]; }; +/** + * Where a contact starts. `screen-edge` asks the platform to deliver the contact as a screen-edge + * touch, which edge gesture recognizers require. + */ +export type GestureContactOrigin = 'screen-edge'; + export type SinglePointerGesturePlan = { topology: 'single'; intent: 'fling' | 'pan'; executionProfile: GestureExecutionProfile; + contactOrigin?: GestureContactOrigin; durationMs: number; viewport: Rect; pointers: readonly [SinglePointerTrajectory]; diff --git a/packages/contracts/src/gesture-plan.ts b/packages/contracts/src/gesture-plan.ts index bd4e3f2ac9..8a6db00b0e 100644 --- a/packages/contracts/src/gesture-plan.ts +++ b/packages/contracts/src/gesture-plan.ts @@ -4,6 +4,7 @@ import type { Point, Rect } from '@agent-device/kernel/snapshot'; import { buildSwipePresetGesturePlan, gestureDirectionDelta, + isScreenEdgeSwipePreset, type SwipePreset, } from './scroll-gesture.ts'; import { @@ -212,7 +213,7 @@ function buildFlingPlan( ): SinglePointerGesturePlan { if ('preset' in input) { const { from, to } = presetGestureEndpoints(input.preset, viewport); - return buildSinglePointerPlan( + const plan = buildSinglePointerPlan( 'fling', from, to, @@ -220,6 +221,7 @@ function buildFlingPlan( viewport, 'endpoint-hold', ); + return isScreenEdgeSwipePreset(input.preset) ? { ...plan, contactOrigin: 'screen-edge' } : plan; } if ('from' in input) { return buildSinglePointerPlan( diff --git a/packages/contracts/src/scroll-gesture.ts b/packages/contracts/src/scroll-gesture.ts index 9149863ccc..04fc049b04 100644 --- a/packages/contracts/src/scroll-gesture.ts +++ b/packages/contracts/src/scroll-gesture.ts @@ -14,6 +14,11 @@ export const SCROLL_DIRECTIONS = ['up', 'down', 'left', 'right'] as const; export type ScrollDirection = (typeof SCROLL_DIRECTIONS)[number]; export const SWIPE_PRESETS = ['left', 'right', 'left-edge', 'right-edge'] as const; export type SwipePreset = (typeof SWIPE_PRESETS)[number]; +const SCREEN_EDGE_SWIPE_PRESETS: ReadonlySet = new Set(['left-edge', 'right-edge']); +/** Edge presets drive screen-edge gestures, such as a back swipe. */ +export function isScreenEdgeSwipePreset(preset: SwipePreset): boolean { + return SCREEN_EDGE_SWIPE_PRESETS.has(preset); +} export const SWIPE_PATTERNS = ['one-way', 'ping-pong'] as const; export type SwipePattern = (typeof SWIPE_PATTERNS)[number]; export const SWIPE_REPETITION_MAX = 200; diff --git a/packages/platform-apple/src/__tests__/index-runner-requests.test.ts b/packages/platform-apple/src/__tests__/index-runner-requests.test.ts index 9792fa629a..bc3bedd74d 100644 --- a/packages/platform-apple/src/__tests__/index-runner-requests.test.ts +++ b/packages/platform-apple/src/__tests__/index-runner-requests.test.ts @@ -9,6 +9,7 @@ import { import fs from 'node:fs'; import path from 'node:path'; import { test } from 'vitest'; +import { buildGesturePlan } from '@agent-device/contracts/gesture-plan'; import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types'; import type { Interactor, PressPointOptions } from '@agent-device/contracts/interactor-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -131,6 +132,16 @@ const INTERACTOR_SITES: Record = { (i) => i.performGesture!(pinchPlan()), ], 'ios-simulator.interactions-gesture.tap': [IOS_SIMULATOR, (i) => i.performGesture!(tapPlan())], + 'ios-simulator.interactions-gesture.screen-edge': [ + IOS_SIMULATOR, + (i) => + i.performGesture!( + buildGesturePlan( + { intent: 'fling', preset: 'right-edge' }, + { x: 0, y: 0, width: 390, height: 844 }, + ), + ), + ], 'tvos.interactions-scroll.remote-press': [ TVOS_SIMULATOR, (i) => i.scroll('down', { durationMs: 300 }), From c163454ba3adb7b6534a7d73b5e9b001c69dfb17 Mon Sep 17 00:00:00 2001 From: gronxb Date: Wed, 4 Nov 2026 13:46:46 +0900 Subject: [PATCH 02/12] fix(ios): report an edge touch whose HID event cannot be created A frame that failed to build was dropped while the gesture still reported success. Fail before touch-down, lift early after a failed move, and return the reason instead. --- .../RunnerSimulatorEdgeTouch.m | 39 ++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m index 8b691b538e..9db7a3f6bb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m @@ -167,7 +167,8 @@ static long RunnerServiceIntegerProperty( (unsigned long)displayID, (unsigned long)matches]; } -static void RunnerDispatchFrame( +// Returns NO when the event could not be created, so nothing was dispatched. +static BOOL RunnerDispatchFrame( const RunnerHIDFunctions *functions, CFTypeRef client, uint64_t sender, @@ -185,7 +186,7 @@ static void RunnerDispatchFrame( if (hand == NULL || finger == NULL) { if (hand != NULL) CFRelease(hand); if (finger != NULL) CFRelease(finger); - return; + return NO; } functions->setIntegerValue(hand, RunnerDigitizerFieldIsDisplayIntegrated, 1); functions->setFloatValue(finger, RunnerDigitizerFieldMajorRadius, RunnerFingerRadius); @@ -195,6 +196,7 @@ static void RunnerDispatchFrame( functions->dispatchEvent(client, hand); CFRelease(finger); CFRelease(hand); + return YES; } static CGPoint RunnerSamplePoint(NSDictionary *sample, CGSize screenSize) { @@ -257,22 +259,33 @@ + (NSString * _Nullable)dispatchWithApplication:(id)application double durationMs = samples.lastObject[@"offsetMs"].doubleValue; uint64_t startTicks = mach_absolute_time(); - RunnerDispatchFrame( - &functions, client, sender, RunnerPointAtOffset(samples, 0, screenSize), - RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerIdentity | RunnerDigitizerEdgeMask, true); - for (double offsetMs = RunnerFrameIntervalMs; offsetMs < durationMs + RunnerFrameIntervalMs; - offsetMs += RunnerFrameIntervalMs) { + if (!RunnerDispatchFrame( + &functions, client, sender, RunnerPointAtOffset(samples, 0, screenSize), + RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerIdentity | RunnerDigitizerEdgeMask, true)) { + CFRelease(client); + return @"simulator HID dispatch could not create the touch-down event"; + } + double liftOffsetMs = durationMs + RunnerFrameIntervalMs; + BOOL moved = YES; + for (double offsetMs = RunnerFrameIntervalMs; offsetMs < liftOffsetMs; offsetMs += RunnerFrameIntervalMs) { double frameOffset = MIN(offsetMs, durationMs); RunnerWaitUntil(startTicks, frameOffset); - RunnerDispatchFrame( - &functions, client, sender, RunnerPointAtOffset(samples, frameOffset, screenSize), - RunnerDigitizerPosition | RunnerDigitizerEdgeMask, true); + if (!RunnerDispatchFrame( + &functions, client, sender, RunnerPointAtOffset(samples, frameOffset, screenSize), + RunnerDigitizerPosition | RunnerDigitizerEdgeMask, true)) { + moved = NO; + liftOffsetMs = frameOffset; + break; + } } - RunnerWaitUntil(startTicks, durationMs + RunnerFrameIntervalMs); - RunnerDispatchFrame( - &functions, client, sender, RunnerPointAtOffset(samples, durationMs, screenSize), + // The contact is down, so it lifts even after a failed move. + RunnerWaitUntil(startTicks, liftOffsetMs); + BOOL lifted = RunnerDispatchFrame( + &functions, client, sender, RunnerPointAtOffset(samples, MIN(liftOffsetMs, durationMs), screenSize), RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerIdentity, false); CFRelease(client); + if (!lifted) return @"simulator HID dispatch could not create the touch-up event"; + if (!moved) return @"simulator HID dispatch could not create a touch-move event; the contact lifted early"; return nil; } From 7c40712000e09a13c0829c98ecb10888b6d611af Mon Sep 17 00:00:00 2001 From: gronxb Date: Wed, 4 Nov 2026 14:31:56 +0900 Subject: [PATCH 03/12] fix(ios): send simulator edge touches as the system's locked edge swipe The edge recognizer on a rotated panel, such as the iPhone Duo inner panel, never began: FromEdgeFlat/Tip touches carry no swipe direction, and a fast first move left the edge region in one frame. A hand swipe on the simulator carries SwipeLocked and the native swipe direction on every frame and starts slowly. Send the same mask, ease the contact in along the planned path, and use the display's oldest touchscreen service when a window adds another. --- .../RunnerSimulatorEdgeTouch.m | 64 ++++++++++++++----- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m index 9db7a3f6bb..1975af7c1f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m @@ -38,14 +38,16 @@ typedef RunnerHIDEventRef (*RunnerCreateDigitizerFingerEvent)( RunnerDispatchEvent dispatchEvent; } RunnerHIDFunctions; -// IOHIDDigitizerEventMask bits. +// IOHIDDigitizerEventMask bits (IOHIDFamily IOHIDEventTypes.h). A real edge swipe on a simulator +// carries Range/Touch/Position plus SwipeLocked and the swipe direction on every frame. static const uint32_t RunnerDigitizerRange = 1u << 0; static const uint32_t RunnerDigitizerTouch = 1u << 1; static const uint32_t RunnerDigitizerPosition = 1u << 2; -static const uint32_t RunnerDigitizerIdentity = 1u << 5; -static const uint32_t RunnerDigitizerFromEdgeFlat = 1u << 10; -static const uint32_t RunnerDigitizerFromEdgeTip = 1u << 11; -static const uint32_t RunnerDigitizerEdgeMask = RunnerDigitizerFromEdgeFlat | RunnerDigitizerFromEdgeTip; +static const uint32_t RunnerDigitizerSwipeLocked = 1u << 18; +static const uint32_t RunnerDigitizerSwipeUp = 1u << 24; +static const uint32_t RunnerDigitizerSwipeDown = 1u << 25; +static const uint32_t RunnerDigitizerSwipeLeft = 1u << 26; +static const uint32_t RunnerDigitizerSwipeRight = 1u << 27; // IOHIDEventField values of the digitizer event type (11). static const uint32_t RunnerDigitizerFieldMajorRadius = (11u << 16) | 20u; @@ -136,7 +138,8 @@ static long RunnerServiceIntegerProperty( return nil; } -// Touches reach the app only with its display's touchscreen service as their sender. +// Touches reach the app only with its display's touchscreen service as their sender. A display has +// one service per attached simulator client; the oldest, created at boot, is always present. static NSString * _Nullable RunnerResolveTouchscreenSender( const RunnerHIDFunctions *functions, CFTypeRef client, @@ -156,15 +159,18 @@ static long RunnerServiceIntegerProperty( NSString *serviceDisplayUUID = RunnerServiceStringProperty(functions, service, CFSTR("displayUUID")); if (serviceDisplayUUID != displayUUID && ![serviceDisplayUUID isEqualToString:displayUUID]) continue; CFTypeRef registryID = functions->getServiceRegistryID(service); - if (registryID != NULL && CFGetTypeID(registryID) == CFNumberGetTypeID()) { - CFNumberGetValue((CFNumberRef)registryID, kCFNumberSInt64Type, sender); - matches += 1; + uint64_t candidate = 0; + if (registryID != NULL && CFGetTypeID(registryID) == CFNumberGetTypeID() && + CFNumberGetValue((CFNumberRef)registryID, kCFNumberSInt64Type, &candidate) && + (matches == 0 || candidate < *sender)) { + *sender = candidate; } + if (registryID != NULL) matches += 1; } if (services != NULL) CFRelease(services); - if (matches == 1) return nil; - return [NSString stringWithFormat:@"display %lu has %lu simulator touchscreen HID services; an edge touch needs exactly one", - (unsigned long)displayID, (unsigned long)matches]; + if (matches > 0) return nil; + return [NSString stringWithFormat:@"display %lu has no simulator touchscreen HID service", + (unsigned long)displayID]; } // Returns NO when the event could not be created, so nothing was dispatched. @@ -203,6 +209,15 @@ static CGPoint RunnerSamplePoint(NSDictionary *sample, C return CGPointMake(sample[@"x"].doubleValue / screenSize.width, sample[@"y"].doubleValue / screenSize.height); } +// The swipe direction the system reports for a contact moving from `start` to `end`, in the +// display's native axes, where y grows downward. +static uint32_t RunnerSwipeDirection(CGPoint start, CGPoint end) { + double dx = end.x - start.x; + double dy = end.y - start.y; + if (fabs(dx) >= fabs(dy)) return dx >= 0 ? RunnerDigitizerSwipeRight : RunnerDigitizerSwipeLeft; + return dy >= 0 ? RunnerDigitizerSwipeDown : RunnerDigitizerSwipeUp; +} + // The contact position at `offsetMs`, interpolated between the planned samples. static CGPoint RunnerPointAtOffset( NSArray *> *samples, @@ -223,6 +238,17 @@ static CGPoint RunnerPointAtOffset( return RunnerSamplePoint(samples.lastObject, screenSize); } +// The contact position at `offsetMs` when it eases in quadratically along the planned path. +static CGPoint RunnerEasedPoint( + NSArray *> *samples, + double offsetMs, + double durationMs, + CGSize screenSize +) { + double progress = durationMs > 0 ? offsetMs / durationMs : 1; + return RunnerPointAtOffset(samples, durationMs * progress * progress, screenSize); +} + static void RunnerWaitUntil(uint64_t startTicks, double offsetMs) { static mach_timebase_info_data_t timebase; if (timebase.denom == 0) mach_timebase_info(&timebase); @@ -258,10 +284,12 @@ + (NSString * _Nullable)dispatchWithApplication:(id)application } double durationMs = samples.lastObject[@"offsetMs"].doubleValue; + uint32_t swipe = RunnerDigitizerSwipeLocked | + RunnerSwipeDirection(RunnerPointAtOffset(samples, 0, screenSize), RunnerPointAtOffset(samples, durationMs, screenSize)); uint64_t startTicks = mach_absolute_time(); if (!RunnerDispatchFrame( &functions, client, sender, RunnerPointAtOffset(samples, 0, screenSize), - RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerIdentity | RunnerDigitizerEdgeMask, true)) { + RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerPosition | swipe, true)) { CFRelease(client); return @"simulator HID dispatch could not create the touch-down event"; } @@ -270,9 +298,11 @@ + (NSString * _Nullable)dispatchWithApplication:(id)application for (double offsetMs = RunnerFrameIntervalMs; offsetMs < liftOffsetMs; offsetMs += RunnerFrameIntervalMs) { double frameOffset = MIN(offsetMs, durationMs); RunnerWaitUntil(startTicks, frameOffset); + // The edge recognizer fails when the first move leaves the edge region in one frame, as a fast + // finger does on a wide panel, so the contact eases in like a real swipe. if (!RunnerDispatchFrame( - &functions, client, sender, RunnerPointAtOffset(samples, frameOffset, screenSize), - RunnerDigitizerPosition | RunnerDigitizerEdgeMask, true)) { + &functions, client, sender, RunnerEasedPoint(samples, frameOffset, durationMs, screenSize), + RunnerDigitizerPosition | swipe, true)) { moved = NO; liftOffsetMs = frameOffset; break; @@ -281,8 +311,8 @@ + (NSString * _Nullable)dispatchWithApplication:(id)application // The contact is down, so it lifts even after a failed move. RunnerWaitUntil(startTicks, liftOffsetMs); BOOL lifted = RunnerDispatchFrame( - &functions, client, sender, RunnerPointAtOffset(samples, MIN(liftOffsetMs, durationMs), screenSize), - RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerIdentity, false); + &functions, client, sender, RunnerEasedPoint(samples, MIN(liftOffsetMs, durationMs), durationMs, screenSize), + RunnerDigitizerRange | RunnerDigitizerTouch | swipe, false); CFRelease(client); if (!lifted) return @"simulator HID dispatch could not create the touch-up event"; if (!moved) return @"simulator HID dispatch could not create a touch-move event; the contact lifted early"; From 2f19cf362ef7e05c782d7c55461435870dfb6f62 Mon Sep 17 00:00:00 2001 From: gronxb Date: Wed, 4 Nov 2026 14:42:13 +0900 Subject: [PATCH 04/12] fix(ios): pick the touchscreen sender only from readable registry IDs A service whose registry ID could not be read still counted as a match, so a later valid service could leave the sender at 0. --- .../RunnerSimulatorEdgeTouch.m | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m index 1975af7c1f..2fd132a13a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m @@ -160,12 +160,12 @@ static long RunnerServiceIntegerProperty( if (serviceDisplayUUID != displayUUID && ![serviceDisplayUUID isEqualToString:displayUUID]) continue; CFTypeRef registryID = functions->getServiceRegistryID(service); uint64_t candidate = 0; - if (registryID != NULL && CFGetTypeID(registryID) == CFNumberGetTypeID() && - CFNumberGetValue((CFNumberRef)registryID, kCFNumberSInt64Type, &candidate) && - (matches == 0 || candidate < *sender)) { - *sender = candidate; + if (registryID == NULL || CFGetTypeID(registryID) != CFNumberGetTypeID() || + !CFNumberGetValue((CFNumberRef)registryID, kCFNumberSInt64Type, &candidate)) { + continue; } - if (registryID != NULL) matches += 1; + if (matches == 0 || candidate < *sender) *sender = candidate; + matches += 1; } if (services != NULL) CFRelease(services); if (matches > 0) return nil; From 4ce52e64f3b264a2ac5f3a9e070954f5d591e79d Mon Sep 17 00:00:00 2001 From: gronxb Date: Wed, 4 Nov 2026 15:51:15 +0900 Subject: [PATCH 05/12] fix(ios): end every simulator edge contact with a reserved terminal event A touch-up that could not be built after touch-down left UIKit with an active finger. The edge contact owner now builds a cancel event before touch-down and, after touch-down, sends the touch-up or that cancel on every exit. Unit tests fail each HID constructor in turn and check what reaches the event system. --- ...AgentDeviceRunnerUITests-Bridging-Header.h | 3 + .../RunnerSimulatorEdgeTouch+Internal.h | 56 +++++++ .../RunnerSimulatorEdgeTouch.m | 142 +++++++++--------- .../UnitTests/RunnerEdgeContactTesting.h | 19 +++ .../UnitTests/RunnerEdgeContactTesting.m | 70 +++++++++ .../RunnerTests+ScreenEdgeGestureTests.swift | 41 +++++ 6 files changed, 260 insertions(+), 71 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch+Internal.h create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.h create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h index 90b894ea09..dc74dcf5ad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h @@ -8,3 +8,6 @@ #if __has_include("UnitTests/RunnerAXModalScopeCapture.h") #import "UnitTests/RunnerAXModalScopeCapture.h" #endif +#if __has_include("UnitTests/RunnerEdgeContactTesting.h") +#import "UnitTests/RunnerEdgeContactTesting.h" +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch+Internal.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch+Internal.h new file mode 100644 index 0000000000..26e30ea6e8 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch+Internal.h @@ -0,0 +1,56 @@ +#import +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +#if TARGET_OS_IOS && TARGET_OS_SIMULATOR + +typedef CFTypeRef RunnerHIDEventRef; +typedef RunnerHIDEventRef _Nullable (*RunnerCreateDigitizerEvent)( + CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, double, double, + double, double, double, Boolean, Boolean, uint32_t); +typedef RunnerHIDEventRef _Nullable (*RunnerCreateDigitizerFingerEvent)( + CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, double, double, double, double, double, + Boolean, Boolean, uint32_t); +typedef void (*RunnerAppendEvent)(RunnerHIDEventRef, RunnerHIDEventRef, uint32_t); +typedef void (*RunnerSetIntegerValue)(RunnerHIDEventRef, uint32_t, long); +typedef void (*RunnerSetFloatValue)(RunnerHIDEventRef, uint32_t, double); +typedef void (*RunnerSetSenderID)(RunnerHIDEventRef, uint64_t); +typedef void (*RunnerSetTimeStamp)(RunnerHIDEventRef, uint64_t); +typedef CFTypeRef _Nullable (*RunnerCreateEventSystemClient)(CFAllocatorRef); +typedef CFArrayRef _Nullable (*RunnerCopyServices)(CFTypeRef); +typedef CFTypeRef _Nullable (*RunnerCopyServiceProperty)(CFTypeRef, CFStringRef); +typedef CFTypeRef _Nullable (*RunnerGetServiceRegistryID)(CFTypeRef); +typedef void (*RunnerDispatchEvent)(CFTypeRef, RunnerHIDEventRef); + +// The IOKit HID entry points the edge touch uses, resolved at run time. +typedef struct { + RunnerCreateDigitizerEvent createDigitizerEvent; + RunnerCreateDigitizerFingerEvent createFingerEvent; + RunnerAppendEvent appendEvent; + RunnerSetIntegerValue setIntegerValue; + RunnerSetFloatValue setFloatValue; + RunnerSetSenderID setSenderID; + RunnerSetTimeStamp setTimeStamp; + RunnerCreateEventSystemClient createClient; + RunnerCopyServices copyServices; + RunnerCopyServiceProperty copyServiceProperty; + RunnerGetServiceRegistryID getServiceRegistryID; + RunnerDispatchEvent dispatchEvent; +} RunnerHIDFunctions; + +// Owns one edge contact from touch-down to its terminal event. The cancel event is built before +// touch-down; after touch-down, every exit dispatches either the touch-up or that cancel event. +// Returns nil on success, or the reason the contact did not complete. +FOUNDATION_EXPORT NSString * _Nullable RunnerDispatchEdgeContact( + const RunnerHIDFunctions *functions, + CFTypeRef client, + uint64_t sender, + NSArray *> *samples, + CGSize screenSize +); + +#endif + +NS_ASSUME_NONNULL_END diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m index 2fd132a13a..5886285c3d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m @@ -1,4 +1,5 @@ #import "RunnerSimulatorEdgeTouch.h" +#import "RunnerSimulatorEdgeTouch+Internal.h" #if TARGET_OS_IOS && TARGET_OS_SIMULATOR @@ -7,42 +8,13 @@ #import #import -typedef CFTypeRef RunnerHIDEventRef; -typedef RunnerHIDEventRef (*RunnerCreateDigitizerEvent)( - CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, double, double, - double, double, double, Boolean, Boolean, uint32_t); -typedef RunnerHIDEventRef (*RunnerCreateDigitizerFingerEvent)( - CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, double, double, double, double, double, - Boolean, Boolean, uint32_t); -typedef void (*RunnerAppendEvent)(RunnerHIDEventRef, RunnerHIDEventRef, uint32_t); -typedef void (*RunnerSetIntegerValue)(RunnerHIDEventRef, uint32_t, long); -typedef void (*RunnerSetFloatValue)(RunnerHIDEventRef, uint32_t, double); -typedef void (*RunnerSetSenderID)(RunnerHIDEventRef, uint64_t); -typedef CFTypeRef (*RunnerCreateEventSystemClient)(CFAllocatorRef); -typedef CFArrayRef (*RunnerCopyServices)(CFTypeRef); -typedef CFTypeRef (*RunnerCopyServiceProperty)(CFTypeRef, CFStringRef); -typedef CFTypeRef (*RunnerGetServiceRegistryID)(CFTypeRef); -typedef void (*RunnerDispatchEvent)(CFTypeRef, RunnerHIDEventRef); - -typedef struct { - RunnerCreateDigitizerEvent createDigitizerEvent; - RunnerCreateDigitizerFingerEvent createFingerEvent; - RunnerAppendEvent appendEvent; - RunnerSetIntegerValue setIntegerValue; - RunnerSetFloatValue setFloatValue; - RunnerSetSenderID setSenderID; - RunnerCreateEventSystemClient createClient; - RunnerCopyServices copyServices; - RunnerCopyServiceProperty copyServiceProperty; - RunnerGetServiceRegistryID getServiceRegistryID; - RunnerDispatchEvent dispatchEvent; -} RunnerHIDFunctions; // IOHIDDigitizerEventMask bits (IOHIDFamily IOHIDEventTypes.h). A real edge swipe on a simulator // carries Range/Touch/Position plus SwipeLocked and the swipe direction on every frame. static const uint32_t RunnerDigitizerRange = 1u << 0; static const uint32_t RunnerDigitizerTouch = 1u << 1; static const uint32_t RunnerDigitizerPosition = 1u << 2; +static const uint32_t RunnerDigitizerCancel = 1u << 7; static const uint32_t RunnerDigitizerSwipeLocked = 1u << 18; static const uint32_t RunnerDigitizerSwipeUp = 1u << 24; static const uint32_t RunnerDigitizerSwipeDown = 1u << 25; @@ -76,6 +48,7 @@ typedef RunnerHIDEventRef (*RunnerCreateDigitizerFingerEvent)( { (void **)&functions->setIntegerValue, "IOHIDEventSetIntegerValue" }, { (void **)&functions->setFloatValue, "IOHIDEventSetFloatValue" }, { (void **)&functions->setSenderID, "IOHIDEventSetSenderID" }, + { (void **)&functions->setTimeStamp, "IOHIDEventSetTimeStamp" }, { (void **)&functions->createClient, "IOHIDEventSystemClientCreate" }, { (void **)&functions->copyServices, "IOHIDEventSystemClientCopyServices" }, { (void **)&functions->copyServiceProperty, "IOHIDServiceClientCopyProperty" }, @@ -173,10 +146,9 @@ static long RunnerServiceIntegerProperty( (unsigned long)displayID]; } -// Returns NO when the event could not be created, so nothing was dispatched. -static BOOL RunnerDispatchFrame( +// One retained hand event with its finger, or NULL when either could not be created. +static RunnerHIDEventRef _Nullable RunnerCreateFrame( const RunnerHIDFunctions *functions, - CFTypeRef client, uint64_t sender, CGPoint normalized, uint32_t mask, @@ -192,17 +164,20 @@ static BOOL RunnerDispatchFrame( if (hand == NULL || finger == NULL) { if (hand != NULL) CFRelease(hand); if (finger != NULL) CFRelease(finger); - return NO; + return NULL; } functions->setIntegerValue(hand, RunnerDigitizerFieldIsDisplayIntegrated, 1); functions->setFloatValue(finger, RunnerDigitizerFieldMajorRadius, RunnerFingerRadius); functions->setFloatValue(finger, RunnerDigitizerFieldMinorRadius, RunnerFingerRadius); functions->appendEvent(hand, finger, 0); - functions->setSenderID(hand, sender); - functions->dispatchEvent(client, hand); CFRelease(finger); - CFRelease(hand); - return YES; + functions->setSenderID(hand, sender); + return hand; +} + +static void RunnerDispatchAndRelease(const RunnerHIDFunctions *functions, CFTypeRef client, RunnerHIDEventRef event) { + functions->dispatchEvent(client, event); + CFRelease(event); } static CGPoint RunnerSamplePoint(NSDictionary *sample, CGSize screenSize) { @@ -256,6 +231,62 @@ static void RunnerWaitUntil(uint64_t startTicks, double offsetMs) { mach_wait_until(startTicks + nanoseconds * timebase.denom / timebase.numer); } +NSString * _Nullable RunnerDispatchEdgeContact( + const RunnerHIDFunctions *functions, + CFTypeRef client, + uint64_t sender, + NSArray *> *samples, + CGSize screenSize +) { + double durationMs = samples.lastObject[@"offsetMs"].doubleValue; + CGPoint start = RunnerPointAtOffset(samples, 0, screenSize); + uint32_t swipe = RunnerDigitizerSwipeLocked | + RunnerSwipeDirection(start, RunnerPointAtOffset(samples, durationMs, screenSize)); + // The terminal event exists before touch-down, so a contact that goes down always ends. + RunnerHIDEventRef cancel = RunnerCreateFrame( + functions, sender, start, RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerCancel | swipe, false); + if (cancel == NULL) return @"simulator HID dispatch could not create the touch-cancel event"; + RunnerHIDEventRef down = RunnerCreateFrame( + functions, sender, start, RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerPosition | swipe, true); + if (down == NULL) { + CFRelease(cancel); + return @"simulator HID dispatch could not create the touch-down event"; + } + uint64_t startTicks = mach_absolute_time(); + RunnerDispatchAndRelease(functions, client, down); + NSString *failure = nil; + for (double offsetMs = RunnerFrameIntervalMs; offsetMs < durationMs + RunnerFrameIntervalMs; + offsetMs += RunnerFrameIntervalMs) { + double frameOffset = MIN(offsetMs, durationMs); + RunnerWaitUntil(startTicks, frameOffset); + // The edge recognizer fails when the first move leaves the edge region in one frame, as a fast + // finger does on a wide panel, so the contact eases in like a real swipe. + RunnerHIDEventRef move = RunnerCreateFrame( + functions, sender, RunnerEasedPoint(samples, frameOffset, durationMs, screenSize), + RunnerDigitizerPosition | swipe, true); + if (move == NULL) { + failure = @"simulator HID dispatch could not create a touch-move event; the contact was cancelled"; + break; + } + RunnerDispatchAndRelease(functions, client, move); + } + if (failure == nil) { + RunnerWaitUntil(startTicks, durationMs + RunnerFrameIntervalMs); + RunnerHIDEventRef lift = RunnerCreateFrame( + functions, sender, RunnerPointAtOffset(samples, durationMs, screenSize), + RunnerDigitizerRange | RunnerDigitizerTouch | swipe, false); + if (lift != NULL) { + RunnerDispatchAndRelease(functions, client, lift); + CFRelease(cancel); + return nil; + } + failure = @"simulator HID dispatch could not create the touch-up event; the contact was cancelled"; + } + functions->setTimeStamp(cancel, mach_absolute_time()); + RunnerDispatchAndRelease(functions, client, cancel); + return failure; +} + @implementation RunnerSimulatorEdgeTouch + (NSString * _Nullable)dispatchWithApplication:(id)application @@ -283,40 +314,9 @@ + (NSString * _Nullable)dispatchWithApplication:(id)application return failure; } - double durationMs = samples.lastObject[@"offsetMs"].doubleValue; - uint32_t swipe = RunnerDigitizerSwipeLocked | - RunnerSwipeDirection(RunnerPointAtOffset(samples, 0, screenSize), RunnerPointAtOffset(samples, durationMs, screenSize)); - uint64_t startTicks = mach_absolute_time(); - if (!RunnerDispatchFrame( - &functions, client, sender, RunnerPointAtOffset(samples, 0, screenSize), - RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerPosition | swipe, true)) { - CFRelease(client); - return @"simulator HID dispatch could not create the touch-down event"; - } - double liftOffsetMs = durationMs + RunnerFrameIntervalMs; - BOOL moved = YES; - for (double offsetMs = RunnerFrameIntervalMs; offsetMs < liftOffsetMs; offsetMs += RunnerFrameIntervalMs) { - double frameOffset = MIN(offsetMs, durationMs); - RunnerWaitUntil(startTicks, frameOffset); - // The edge recognizer fails when the first move leaves the edge region in one frame, as a fast - // finger does on a wide panel, so the contact eases in like a real swipe. - if (!RunnerDispatchFrame( - &functions, client, sender, RunnerEasedPoint(samples, frameOffset, durationMs, screenSize), - RunnerDigitizerPosition | swipe, true)) { - moved = NO; - liftOffsetMs = frameOffset; - break; - } - } - // The contact is down, so it lifts even after a failed move. - RunnerWaitUntil(startTicks, liftOffsetMs); - BOOL lifted = RunnerDispatchFrame( - &functions, client, sender, RunnerEasedPoint(samples, MIN(liftOffsetMs, durationMs), durationMs, screenSize), - RunnerDigitizerRange | RunnerDigitizerTouch | swipe, false); + failure = RunnerDispatchEdgeContact(&functions, client, sender, samples, screenSize); CFRelease(client); - if (!lifted) return @"simulator HID dispatch could not create the touch-up event"; - if (!moved) return @"simulator HID dispatch could not create a touch-move event; the contact lifted early"; - return nil; + return failure; } @end diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.h new file mode 100644 index 0000000000..a92caf8d07 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.h @@ -0,0 +1,19 @@ +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +#if TARGET_OS_IOS && TARGET_OS_SIMULATOR + +/// Runs `RunnerDispatchEdgeContact` on a 48 ms path whose HID event constructors fail from the +/// `failingFrame`th hand event on: 0 is the reserved cancel, 1 touch-down, 2 to 4 the moves, 5 the +/// touch-up, and -1 never. Returns each dispatched event in order as `mask`, `touching`, and +/// `restamped` (whether its timestamp was set after it was built). +FOUNDATION_EXPORT NSArray *> *RunnerEdgeContactDispatches( + NSInteger failingFrame, + NSString * _Nullable * _Nullable failure +); + +#endif + +NS_ASSUME_NONNULL_END diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m new file mode 100644 index 0000000000..9fe03b0f47 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m @@ -0,0 +1,70 @@ +#if defined(AGENT_DEVICE_RUNNER_UNIT_TESTS) +#import "RunnerEdgeContactTesting.h" + +#if TARGET_OS_IOS && TARGET_OS_SIMULATOR + +#import "RunnerSimulatorEdgeTouch+Internal.h" + +static NSInteger RunnerTestingFailingFrame; +static NSInteger RunnerTestingCreatedFrames; +static NSMutableArray *> *RunnerTestingDispatched; + +static RunnerHIDEventRef RunnerTestingCreateHand( + CFAllocatorRef allocator, uint64_t timestamp, uint32_t transducer, uint32_t index, uint32_t identity, + uint32_t mask, uint32_t buttons, double x, double y, double z, double tip, double barrel, + Boolean range, Boolean touch, uint32_t options +) { + NSInteger frame = RunnerTestingCreatedFrames++; + if (RunnerTestingFailingFrame >= 0 && frame >= RunnerTestingFailingFrame) return NULL; + NSMutableDictionary *event = [@{ @"mask": @(mask), @"touching": @(touch), @"restamped": @NO } mutableCopy]; + return CFBridgingRetain(event); +} + +static RunnerHIDEventRef RunnerTestingCreateFinger( + CFAllocatorRef allocator, uint64_t timestamp, uint32_t index, uint32_t identity, uint32_t mask, + double x, double y, double z, double tip, double twist, Boolean range, Boolean touch, uint32_t options +) { + return CFBridgingRetain([NSMutableDictionary dictionary]); +} + +static void RunnerTestingAppend(RunnerHIDEventRef parent, RunnerHIDEventRef child, uint32_t options) {} +static void RunnerTestingSetInteger(RunnerHIDEventRef event, uint32_t field, long value) {} +static void RunnerTestingSetFloat(RunnerHIDEventRef event, uint32_t field, double value) {} +static void RunnerTestingSetSender(RunnerHIDEventRef event, uint64_t sender) {} + +static void RunnerTestingSetTimeStamp(RunnerHIDEventRef event, uint64_t timestamp) { + ((__bridge NSMutableDictionary *)event)[@"restamped"] = @YES; +} + +static void RunnerTestingDispatch(CFTypeRef client, RunnerHIDEventRef event) { + [RunnerTestingDispatched addObject:[(__bridge NSDictionary *)event copy]]; +} + +NSArray *> *RunnerEdgeContactDispatches( + NSInteger failingFrame, + NSString **failure +) { + RunnerTestingFailingFrame = failingFrame; + RunnerTestingCreatedFrames = 0; + RunnerTestingDispatched = [NSMutableArray array]; + RunnerHIDFunctions functions = { + .createDigitizerEvent = RunnerTestingCreateHand, + .createFingerEvent = RunnerTestingCreateFinger, + .appendEvent = RunnerTestingAppend, + .setIntegerValue = RunnerTestingSetInteger, + .setFloatValue = RunnerTestingSetFloat, + .setSenderID = RunnerTestingSetSender, + .setTimeStamp = RunnerTestingSetTimeStamp, + .dispatchEvent = RunnerTestingDispatch, + }; + NSArray *samples = @[ + @{ @"x": @0, @"y": @50, @"offsetMs": @0 }, + @{ @"x": @100, @"y": @50, @"offsetMs": @48 }, + ]; + NSString *result = RunnerDispatchEdgeContact(&functions, (__bridge CFTypeRef)samples, 1, samples, CGSizeMake(100, 100)); + if (failure != NULL) *failure = result; + return [RunnerTestingDispatched copy]; +} + +#endif +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift new file mode 100644 index 0000000000..13d90324a0 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift @@ -0,0 +1,41 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) && targetEnvironment(simulator) +private let cancelBit: UInt32 = 1 << 7 + +extension RunnerTests { + func testEdgeContactLiftsWithoutUsingTheReservedCancel() { + var failure: NSString? + let events = RunnerEdgeContactDispatches(-1, &failure) + + XCTAssertNil(failure) + XCTAssertEqual(events.count, 5, "touch-down, three moves, touch-up") + XCTAssertEqual(events.map { $0["touching"]?.boolValue }, [true, true, true, true, false]) + XCTAssertFalse(events.contains { ($0["mask"]?.uint32Value ?? 0) & cancelBit != 0 }) + } + + func testEdgeContactDispatchesNothingWhenItCannotReserveItsEnd() { + for failingFrame in [0, 1] { + var failure: NSString? + let events = RunnerEdgeContactDispatches(failingFrame, &failure) + + XCTAssertNotNil(failure, "frame \(failingFrame)") + XCTAssertTrue(events.isEmpty, "frame \(failingFrame): nothing goes down without its terminal event") + } + } + + func testEdgeContactCancelsWhenAMoveOrTheLiftCannotBeBuilt() { + for (failingFrame, movesBeforeCancel) in [(2, 0), (3, 1), (5, 3)] { + var failure: NSString? + let events = RunnerEdgeContactDispatches(failingFrame, &failure) + let terminal = events.last + + XCTAssertNotNil(failure, "frame \(failingFrame)") + XCTAssertEqual(events.count, 2 + movesBeforeCancel, "frame \(failingFrame): down, moves, cancel") + XCTAssertEqual(terminal?["touching"]?.boolValue, false, "frame \(failingFrame)") + XCTAssertNotEqual((terminal?["mask"]?.uint32Value ?? 0) & cancelBit, 0, "frame \(failingFrame)") + XCTAssertEqual(terminal?["restamped"]?.boolValue, true, "frame \(failingFrame): stamped at dispatch") + } + } +} +#endif From 41ec47d5f674de0f5aaaf438216b7f47ea1b84a9 Mon Sep 17 00:00:00 2001 From: gronxb Date: Wed, 4 Nov 2026 15:51:15 +0900 Subject: [PATCH 06/12] fix(ios): report the edge swipe schedule the runner ran Edge flings ran an eased HID schedule but reported endpoint-hold. The runner execution policy now resolves the contact and its schedule together and returns the schedule as executionProfile. The daemon keeps a reported schedule and infers the planned profile only when a backend reports none. ADR 0013 records edge-ease-in and the reporting rule. --- .../RunnerTests+CommandExecution.swift | 11 ++++--- .../RunnerTests+GestureExecution.swift | 7 +++++ .../RunnerTests+Interaction.swift | 21 ++++++++++++-- .../RunnerTests+Models.swift | 2 ++ .../RunnerTests+SynthesizedInteraction.swift | 5 ++-- ...nerTests+SynthesizedInteractionTests.swift | 17 +++++++++-- docs/adr/0013-unified-gesture-plans.md | 14 ++++++++- packages/contracts/src/gesture-plan-types.ts | 11 +++++++ .../interaction-gesture-response.test.ts | 29 +++++++++++++++++++ .../internal/interaction-gesture-response.ts | 22 ++++++++++---- 10 files changed, 120 insertions(+), 19 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 8786a127e5..28b53a0fb7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -661,12 +661,14 @@ extension RunnerTests { error: ErrorPayload(code: "INVALID_ARGS", message: validationError) ) } - switch plannedGestureExecution(for: plan) { + let execution = plannedGestureExecution(for: plan) + let response: Response + switch execution { case .fastSwipe: // Validation above guarantees a non-empty, single-pointer path for this execution kind. let first = plan.pointers[0].samples.first!.point let last = plan.pointers[0].samples.last!.point - return canonicalPlannedGestureResponse( + response = canonicalPlannedGestureResponse( try executeDragGesture( activeApp: activeApp, x: first.x, @@ -682,15 +684,16 @@ extension RunnerTests { let (timing, outcome) = try performGesture(activeApp, idleTimeout: false) { try sampledPlannedGesture(app: activeApp, plan: plan) } - return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) + response = plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) #if os(iOS) && targetEnvironment(simulator) case .screenEdgeTouch: let (timing, outcome) = try performGesture(activeApp, idleTimeout: false) { try screenEdgePlannedGesture(app: activeApp, plan: plan) } - return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) + response = plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) #endif } + return Self.reportingSchedule(execution.schedule, in: response) case .gestureViewport: let frame = try gestureViewportFrame(app: activeApp) guard !frame.isNull, !frame.isInfinite, !frame.isEmpty else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift index b148a21c88..e5c49d0158 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift @@ -152,6 +152,13 @@ extension RunnerTests { ) } + /// A successful planned gesture reports the motion schedule its route ran. + static func reportingSchedule(_ schedule: String?, in response: Response) -> Response { + guard response.ok, var data = response.data, let schedule else { return response } + data.executionProfile = schedule + return Response(ok: true, data: data) + } + func plannedGestureResponse( plan: RunnerGesturePlan, timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 28cdc54ebf..9e097dc1eb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -12,13 +12,30 @@ private struct RunnerUnsupportedOperationError: LocalizedError { } extension RunnerTests { + /// How a planned gesture reaches the device, with the one-pointer motion schedule that route runs. enum PlannedGestureExecution: Equatable { case fastSwipe - case sampled + /// `schedule` is the plan's profile for one pointer, and nil for several contacts. + case sampled(schedule: String?) #if os(iOS) && targetEnvironment(simulator) - /// One HID touch with the digitizer edge mask, which only a simulator can dispatch. + /// One HID touch marked as a screen-edge swipe, which only a simulator can dispatch. case screenEdgeTouch #endif + + /// The schedule the response reports. Mirrors `GestureExecutedProfile` in + /// packages/contracts/src/gesture-plan-types.ts. + var schedule: String? { + switch self { + case .fastSwipe: + return "endpoint-hold" + case .sampled(let schedule): + return schedule +#if os(iOS) && targetEnvironment(simulator) + case .screenEdgeTouch: + return "edge-ease-in" +#endif + } + } } enum SynthesizedDragProfile: Equatable { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index d04bddb86c..755879ac0a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -517,6 +517,8 @@ struct DataPayload: Codable { var keyboardAvoided: Bool? var keyboardMinY: Double? var maestroNonHittableCoordinateFallbackUsed: Bool? + /// The one-pointer motion schedule a planned gesture ran. + var executionProfile: String? var textEntryRoute: String? /// A replacement whose field changed but whose value cannot echo the typed text: "unconfirmed", /// with the TS `FillUnconfirmedVerification` evidence in `requested`, `before`, `after`, `target`. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index ea7ab31495..3e3fd7bbaa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -769,9 +769,8 @@ extension RunnerTests { return .screenEdgeTouch } #endif - return plan.topology == "single" && plan.executionProfile == "endpoint-hold" - ? .fastSwipe - : .sampled + guard plan.topology == "single" else { return .sampled(schedule: nil) } + return plan.executionProfile == "endpoint-hold" ? .fastSwipe : .sampled(schedule: plan.executionProfile) } #if os(iOS) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index de2eef5983..a0d8f5b2b5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -550,7 +550,7 @@ extension RunnerTests { ) XCTAssertNil(plannedGestureValidationError(plan)) - XCTAssertEqual(plannedGestureExecution(for: plan), .sampled) + XCTAssertEqual(plannedGestureExecution(for: plan), .sampled(schedule: nil)) } func testPlannedMultiTouchGestureRejectsMismatchedOffsets() throws { @@ -571,7 +571,7 @@ extension RunnerTests { for count in [2, 5] { let plan = try decodedMultiContactPlan(intent: "tap", starts: tapStarts(count)) XCTAssertNil(plannedGestureValidationError(plan), "\(count) contacts") - XCTAssertEqual(plannedGestureExecution(for: plan), .sampled) + XCTAssertEqual(plannedGestureExecution(for: plan), .sampled(schedule: nil)) } } @@ -646,7 +646,7 @@ extension RunnerTests { ) ) - XCTAssertEqual(plannedGestureExecution(for: plan), .sampled) + XCTAssertEqual(plannedGestureExecution(for: plan), .sampled(schedule: "timed-pan")) } func testSinglePointerEndpointHoldUsesFastSwipeExecution() throws { @@ -682,10 +682,21 @@ extension RunnerTests { XCTAssertNil(plannedGestureValidationError(edge)) #if os(iOS) && targetEnvironment(simulator) XCTAssertEqual(plannedGestureExecution(for: edge), .screenEdgeTouch) + XCTAssertEqual(plannedGestureExecution(for: edge).schedule, "edge-ease-in") #else XCTAssertEqual(plannedGestureExecution(for: edge), .fastSwipe) #endif XCTAssertEqual(plannedGestureExecution(for: inPage), .fastSwipe) + XCTAssertEqual(plannedGestureExecution(for: inPage).schedule, "endpoint-hold") + } + + func testOnlyASuccessfulPlannedGestureReportsItsSchedule() { + let performed = Response(ok: true, data: DataPayload(message: "fling")) + let refused = Response(ok: false, error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: "refused")) + + XCTAssertEqual(Self.reportingSchedule("edge-ease-in", in: performed).data?.executionProfile, "edge-ease-in") + XCTAssertNil(Self.reportingSchedule(nil, in: performed).data?.executionProfile) + XCTAssertNil(Self.reportingSchedule("edge-ease-in", in: refused).data) } func testPlannedGestureRejectsAnUnknownContactOrigin() throws { diff --git a/docs/adr/0013-unified-gesture-plans.md b/docs/adr/0013-unified-gesture-plans.md index ef336fe436..f147a1bdd2 100644 --- a/docs/adr/0013-unified-gesture-plans.md +++ b/docs/adr/0013-unified-gesture-plans.md @@ -24,6 +24,8 @@ Completed process history is in git history. - For target-authored drag, resolve both endpoints before device injection. On replay, verify both identities before pointer-down. - Admit target-authored drag only on Android touch devices and iOS/iPadOS. - iOS timing changes require live simulator evidence from an independent app-observed postcondition. +- A backend reports the one-pointer schedule it ran as `executionProfile`. The daemon reports the planned profile only for a backend that reports none. + `edge-ease-in` is a reported schedule that no plan can request (amended 2026-11, issue #4208). ## Context @@ -155,7 +157,7 @@ Platform adapters consume the canonical plan: Core admission and the Apple adapter both consume the same shared multi-touch support policy. Multi-touch is capability-gated to iOS and iPadOS, on simulators and physical devices, through the XCTest runner. - iOS deliberately retains three one-contact motion schedules behind the shared private-XCTest event bridge. + iOS deliberately retains three one-contact motion schedules behind the shared private-XCTest event bridge, and a fourth outside it. Apple's public drag API models the phases independently as an [initial hold, movement velocity, and destination hold](https://developer.apple.com/documentation/xcuiautomation/xcuicoordinate/press%28forduration%3Athendragto%3Awithvelocity%3Athenholdforduration%3A%29). UIKit also reports the finger-up velocity together with the scroll view's predicted [resting content offset](https://developer.apple.com/documentation/uikit/uiscrollviewdelegate/scrollviewwillenddragging%28_%3Awithvelocity%3Atargetcontentoffset%3A%29). UIKit then applies the scroll view's configured [post-lift deceleration rate](https://developer.apple.com/documentation/uikit/uiscrollview/decelerationrate-swift.property). @@ -169,6 +171,16 @@ Platform adapters consume the canonical plan: - The runner's fused `scroll` reuses `endpoint-hold` for inertial release. For controlled release, the fused `scroll` instead expands the movement to evenly spaced samples, one per approximately 16 ms, capped at 30. Controlled release uses cubic ease-out `s(t) = 1 - (1 - t)³`. + - `edge-ease-in` runs only on an iOS simulator, for a plan whose `contactOrigin` is `screen-edge` (the `left-edge` and `right-edge` presets). + XCTest touches carry no edge information, even at x = 0, so UIKit edge recognizers never begin. + The runner therefore sends one HID digitizer contact through the touchscreen service of the app's display, as a hand swipe arrives: + every frame carries `SwipeLocked` and the native swipe direction. + The contact eases in quadratically along the planned samples in 16 ms frames and lifts one frame after the last. + A first move that leaves the edge region in one frame fails the recognizer on a wide panel, such as the iPhone Duo inner panel. + The cancel event is built before touch-down; after touch-down every exit sends the touch-up or that cancel. + Physical devices keep `endpoint-hold` for the same plan. + Rejected: an XCTest swipe from x = 0 carries no edge either; the `FromEdgeFlat`/`FromEdgeTip` mask names no direction and fails on a rotated panel; + a longer preset duration in the shared planner would move simulator policy into planning and change every platform. A linear segment has constant movement velocity through its endpoint, unless a destination hold follows it. Cubic ease-out decelerates from the first movement and has zero slope at lift. diff --git a/packages/contracts/src/gesture-plan-types.ts b/packages/contracts/src/gesture-plan-types.ts index 5d94d0335b..4d1b33fff9 100644 --- a/packages/contracts/src/gesture-plan-types.ts +++ b/packages/contracts/src/gesture-plan-types.ts @@ -34,6 +34,17 @@ export type DragGestureInput = { /** Selects one-pointer release timing without changing semantic gesture intent. */ export type GestureExecutionProfile = 'endpoint-hold' | 'timed-pan'; +/** + * The one-pointer motion schedule a backend ran, as gesture results report it. Beyond the plan + * profiles, `edge-ease-in` is the iOS simulator screen-edge contact, which a plan cannot request. + */ +const GESTURE_EXECUTED_PROFILES = ['endpoint-hold', 'timed-pan', 'edge-ease-in'] as const; +export type GestureExecutedProfile = (typeof GESTURE_EXECUTED_PROFILES)[number]; + +export function isGestureExecutedProfile(value: unknown): value is GestureExecutedProfile { + return (GESTURE_EXECUTED_PROFILES as readonly unknown[]).includes(value); +} + export type GestureSemanticInput = | { intent: 'fling'; from: Point; to: Point } | { intent: 'fling'; preset: SwipePreset } diff --git a/packages/daemon/src/execution/interaction/internal/__tests__/interaction-gesture-response.test.ts b/packages/daemon/src/execution/interaction/internal/__tests__/interaction-gesture-response.test.ts index eb725de89d..303366c517 100644 --- a/packages/daemon/src/execution/interaction/internal/__tests__/interaction-gesture-response.test.ts +++ b/packages/daemon/src/execution/interaction/internal/__tests__/interaction-gesture-response.test.ts @@ -30,3 +30,32 @@ test('gesture response builder preserves the dual-target drag disclosure and tim timing: { executionProfile: 'timed-pan', gestureDurationMs: 1_300 }, }); }); + +test('the schedule a backend reports it ran replaces the planned one, and an unknown one does not', () => { + const fling = (backendResult: Record) => + gestureResponseData( + { + kind: 'fling', + durationMs: 100, + pointerCount: 1, + from: { x: 8, y: 426 }, + to: { x: 334, y: 426 }, + backendResult, + message: 'Flung', + }, + { executionProfile: 'endpoint-hold' }, + ); + + expect(fling({ executionProfile: 'edge-ease-in' })).toMatchObject({ + executionProfile: 'edge-ease-in', + timing: { executionProfile: 'edge-ease-in', gestureDurationMs: 100 }, + }); + expect(fling({})).toMatchObject({ + executionProfile: 'endpoint-hold', + timing: { executionProfile: 'endpoint-hold' }, + }); + expect(fling({ executionProfile: 'warp' })).toMatchObject({ + executionProfile: 'endpoint-hold', + timing: { executionProfile: 'endpoint-hold' }, + }); +}); diff --git a/packages/daemon/src/execution/interaction/internal/interaction-gesture-response.ts b/packages/daemon/src/execution/interaction/internal/interaction-gesture-response.ts index 2dde27cfc6..556c214565 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-gesture-response.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-gesture-response.ts @@ -1,6 +1,7 @@ -import type { - GestureContactCount, - GestureIntent, +import { + isGestureExecutedProfile, + type GestureContactCount, + type GestureIntent, } from '@agent-device/contracts/gesture-plan-types'; import type { Point } from '@agent-device/kernel/snapshot'; @@ -15,13 +16,21 @@ type GestureResponseResult = { message?: string; }; -/** The sole response construction site shared by coordinate and target-authored gestures. */ +/** + * The sole response construction site shared by coordinate and target-authored gestures. The + * schedule the backend reports it ran wins; the caller's `executionProfile` names the planned + * schedule only for a backend that reports none. + */ export function gestureResponseData( result: GestureResponseResult, extra: Record = {}, ): Record { - const executionProfile = - typeof extra.executionProfile === 'string' ? extra.executionProfile : undefined; + const reported = result.backendResult?.executionProfile; + const executionProfile = isGestureExecutedProfile(reported) + ? reported + : typeof extra.executionProfile === 'string' + ? extra.executionProfile + : undefined; return { kind: result.kind, durationMs: result.durationMs, @@ -33,6 +42,7 @@ export function gestureResponseData( ...extra, ...(executionProfile ? { + executionProfile, timing: { executionProfile, gestureDurationMs: result.durationMs, From 125a02ad4757d4c91645c9dcaa6acb0d297fe2b1 Mon Sep 17 00:00:00 2001 From: gronxb Date: Wed, 4 Nov 2026 16:05:38 +0900 Subject: [PATCH 07/12] fix(ios): keep the gesture schedule the backend reports Address review: a backend-reported executionProfile is what ran, so the daemon keeps any reported string and reports the planned profile only when a backend reports none. Drop the TS schedule list that existed only to filter it. The cancel test also checks that touch-down and each move before the cancel touched. --- .../RunnerTests+Interaction.swift | 3 +-- .../RunnerTests+ScreenEdgeGestureTests.swift | 5 +++++ packages/contracts/src/gesture-plan-types.ts | 11 ----------- .../interaction-gesture-response.test.ts | 8 ++++++-- .../internal/interaction-gesture-response.ts | 18 +++++++++--------- 5 files changed, 21 insertions(+), 24 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 9e097dc1eb..fec8cc8636 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -22,8 +22,7 @@ extension RunnerTests { case screenEdgeTouch #endif - /// The schedule the response reports. Mirrors `GestureExecutedProfile` in - /// packages/contracts/src/gesture-plan-types.ts. + /// The schedule the response reports as `executionProfile`; ADR 0013 names each one. var schedule: String? { switch self { case .fastSwipe: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift index 13d90324a0..f6d7e1327d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift @@ -32,6 +32,11 @@ extension RunnerTests { XCTAssertNotNil(failure, "frame \(failingFrame)") XCTAssertEqual(events.count, 2 + movesBeforeCancel, "frame \(failingFrame): down, moves, cancel") + XCTAssertEqual( + events.dropLast().map { $0["touching"]?.boolValue }, + Array(repeating: true, count: 1 + movesBeforeCancel), + "frame \(failingFrame): touch-down and every move before the cancel touched" + ) XCTAssertEqual(terminal?["touching"]?.boolValue, false, "frame \(failingFrame)") XCTAssertNotEqual((terminal?["mask"]?.uint32Value ?? 0) & cancelBit, 0, "frame \(failingFrame)") XCTAssertEqual(terminal?["restamped"]?.boolValue, true, "frame \(failingFrame): stamped at dispatch") diff --git a/packages/contracts/src/gesture-plan-types.ts b/packages/contracts/src/gesture-plan-types.ts index 4d1b33fff9..5d94d0335b 100644 --- a/packages/contracts/src/gesture-plan-types.ts +++ b/packages/contracts/src/gesture-plan-types.ts @@ -34,17 +34,6 @@ export type DragGestureInput = { /** Selects one-pointer release timing without changing semantic gesture intent. */ export type GestureExecutionProfile = 'endpoint-hold' | 'timed-pan'; -/** - * The one-pointer motion schedule a backend ran, as gesture results report it. Beyond the plan - * profiles, `edge-ease-in` is the iOS simulator screen-edge contact, which a plan cannot request. - */ -const GESTURE_EXECUTED_PROFILES = ['endpoint-hold', 'timed-pan', 'edge-ease-in'] as const; -export type GestureExecutedProfile = (typeof GESTURE_EXECUTED_PROFILES)[number]; - -export function isGestureExecutedProfile(value: unknown): value is GestureExecutedProfile { - return (GESTURE_EXECUTED_PROFILES as readonly unknown[]).includes(value); -} - export type GestureSemanticInput = | { intent: 'fling'; from: Point; to: Point } | { intent: 'fling'; preset: SwipePreset } diff --git a/packages/daemon/src/execution/interaction/internal/__tests__/interaction-gesture-response.test.ts b/packages/daemon/src/execution/interaction/internal/__tests__/interaction-gesture-response.test.ts index 303366c517..7ea0915348 100644 --- a/packages/daemon/src/execution/interaction/internal/__tests__/interaction-gesture-response.test.ts +++ b/packages/daemon/src/execution/interaction/internal/__tests__/interaction-gesture-response.test.ts @@ -31,7 +31,7 @@ test('gesture response builder preserves the dual-target drag disclosure and tim }); }); -test('the schedule a backend reports it ran replaces the planned one, and an unknown one does not', () => { +test('the schedule a backend reports it ran replaces the planned one', () => { const fling = (backendResult: Record) => gestureResponseData( { @@ -54,7 +54,11 @@ test('the schedule a backend reports it ran replaces the planned one, and an unk executionProfile: 'endpoint-hold', timing: { executionProfile: 'endpoint-hold' }, }); - expect(fling({ executionProfile: 'warp' })).toMatchObject({ + expect(fling({ executionProfile: 'future-schedule' })).toMatchObject({ + executionProfile: 'future-schedule', + timing: { executionProfile: 'future-schedule' }, + }); + expect(fling({ executionProfile: 42 })).toMatchObject({ executionProfile: 'endpoint-hold', timing: { executionProfile: 'endpoint-hold' }, }); diff --git a/packages/daemon/src/execution/interaction/internal/interaction-gesture-response.ts b/packages/daemon/src/execution/interaction/internal/interaction-gesture-response.ts index 556c214565..f935e0ce77 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-gesture-response.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-gesture-response.ts @@ -1,7 +1,6 @@ -import { - isGestureExecutedProfile, - type GestureContactCount, - type GestureIntent, +import type { + GestureContactCount, + GestureIntent, } from '@agent-device/contracts/gesture-plan-types'; import type { Point } from '@agent-device/kernel/snapshot'; @@ -26,11 +25,12 @@ export function gestureResponseData( extra: Record = {}, ): Record { const reported = result.backendResult?.executionProfile; - const executionProfile = isGestureExecutedProfile(reported) - ? reported - : typeof extra.executionProfile === 'string' - ? extra.executionProfile - : undefined; + const executionProfile = + typeof reported === 'string' && reported !== '' + ? reported + : typeof extra.executionProfile === 'string' + ? extra.executionProfile + : undefined; return { kind: result.kind, durationMs: result.durationMs, From 76b14754e130f7f20fc5f87c1dd9c2e68976ce3e Mon Sep 17 00:00:00 2001 From: gronxb Date: Wed, 4 Nov 2026 16:46:24 +0900 Subject: [PATCH 08/12] refactor(ios): keep the edge contact owner private to its file Address review: the contact function and HID table were exported only so unit tests could inject faults. Make them static and compile the test facade in the same file under AGENT_DEVICE_RUNNER_UNIT_TESTS, so the packaged runner ships no fault-injection interface. --- .../RunnerSimulatorEdgeTouch+Internal.h | 56 --------- .../RunnerSimulatorEdgeTouch.m | 106 +++++++++++++++++- .../UnitTests/RunnerEdgeContactTesting.m | 70 ------------ 3 files changed, 104 insertions(+), 128 deletions(-) delete mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch+Internal.h delete mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch+Internal.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch+Internal.h deleted file mode 100644 index 26e30ea6e8..0000000000 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch+Internal.h +++ /dev/null @@ -1,56 +0,0 @@ -#import -#import -#import - -NS_ASSUME_NONNULL_BEGIN - -#if TARGET_OS_IOS && TARGET_OS_SIMULATOR - -typedef CFTypeRef RunnerHIDEventRef; -typedef RunnerHIDEventRef _Nullable (*RunnerCreateDigitizerEvent)( - CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, double, double, - double, double, double, Boolean, Boolean, uint32_t); -typedef RunnerHIDEventRef _Nullable (*RunnerCreateDigitizerFingerEvent)( - CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, double, double, double, double, double, - Boolean, Boolean, uint32_t); -typedef void (*RunnerAppendEvent)(RunnerHIDEventRef, RunnerHIDEventRef, uint32_t); -typedef void (*RunnerSetIntegerValue)(RunnerHIDEventRef, uint32_t, long); -typedef void (*RunnerSetFloatValue)(RunnerHIDEventRef, uint32_t, double); -typedef void (*RunnerSetSenderID)(RunnerHIDEventRef, uint64_t); -typedef void (*RunnerSetTimeStamp)(RunnerHIDEventRef, uint64_t); -typedef CFTypeRef _Nullable (*RunnerCreateEventSystemClient)(CFAllocatorRef); -typedef CFArrayRef _Nullable (*RunnerCopyServices)(CFTypeRef); -typedef CFTypeRef _Nullable (*RunnerCopyServiceProperty)(CFTypeRef, CFStringRef); -typedef CFTypeRef _Nullable (*RunnerGetServiceRegistryID)(CFTypeRef); -typedef void (*RunnerDispatchEvent)(CFTypeRef, RunnerHIDEventRef); - -// The IOKit HID entry points the edge touch uses, resolved at run time. -typedef struct { - RunnerCreateDigitizerEvent createDigitizerEvent; - RunnerCreateDigitizerFingerEvent createFingerEvent; - RunnerAppendEvent appendEvent; - RunnerSetIntegerValue setIntegerValue; - RunnerSetFloatValue setFloatValue; - RunnerSetSenderID setSenderID; - RunnerSetTimeStamp setTimeStamp; - RunnerCreateEventSystemClient createClient; - RunnerCopyServices copyServices; - RunnerCopyServiceProperty copyServiceProperty; - RunnerGetServiceRegistryID getServiceRegistryID; - RunnerDispatchEvent dispatchEvent; -} RunnerHIDFunctions; - -// Owns one edge contact from touch-down to its terminal event. The cancel event is built before -// touch-down; after touch-down, every exit dispatches either the touch-up or that cancel event. -// Returns nil on success, or the reason the contact did not complete. -FOUNDATION_EXPORT NSString * _Nullable RunnerDispatchEdgeContact( - const RunnerHIDFunctions *functions, - CFTypeRef client, - uint64_t sender, - NSArray *> *samples, - CGSize screenSize -); - -#endif - -NS_ASSUME_NONNULL_END diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m index 5886285c3d..00056ae892 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m @@ -1,5 +1,4 @@ #import "RunnerSimulatorEdgeTouch.h" -#import "RunnerSimulatorEdgeTouch+Internal.h" #if TARGET_OS_IOS && TARGET_OS_SIMULATOR @@ -8,6 +7,40 @@ #import #import +typedef CFTypeRef RunnerHIDEventRef; +typedef RunnerHIDEventRef _Nullable (*RunnerCreateDigitizerEvent)( + CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, double, double, + double, double, double, Boolean, Boolean, uint32_t); +typedef RunnerHIDEventRef _Nullable (*RunnerCreateDigitizerFingerEvent)( + CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, double, double, double, double, double, + Boolean, Boolean, uint32_t); +typedef void (*RunnerAppendEvent)(RunnerHIDEventRef, RunnerHIDEventRef, uint32_t); +typedef void (*RunnerSetIntegerValue)(RunnerHIDEventRef, uint32_t, long); +typedef void (*RunnerSetFloatValue)(RunnerHIDEventRef, uint32_t, double); +typedef void (*RunnerSetSenderID)(RunnerHIDEventRef, uint64_t); +typedef void (*RunnerSetTimeStamp)(RunnerHIDEventRef, uint64_t); +typedef CFTypeRef _Nullable (*RunnerCreateEventSystemClient)(CFAllocatorRef); +typedef CFArrayRef _Nullable (*RunnerCopyServices)(CFTypeRef); +typedef CFTypeRef _Nullable (*RunnerCopyServiceProperty)(CFTypeRef, CFStringRef); +typedef CFTypeRef _Nullable (*RunnerGetServiceRegistryID)(CFTypeRef); +typedef void (*RunnerDispatchEvent)(CFTypeRef, RunnerHIDEventRef); + +// The IOKit HID entry points the edge touch uses, resolved at run time. +typedef struct { + RunnerCreateDigitizerEvent createDigitizerEvent; + RunnerCreateDigitizerFingerEvent createFingerEvent; + RunnerAppendEvent appendEvent; + RunnerSetIntegerValue setIntegerValue; + RunnerSetFloatValue setFloatValue; + RunnerSetSenderID setSenderID; + RunnerSetTimeStamp setTimeStamp; + RunnerCreateEventSystemClient createClient; + RunnerCopyServices copyServices; + RunnerCopyServiceProperty copyServiceProperty; + RunnerGetServiceRegistryID getServiceRegistryID; + RunnerDispatchEvent dispatchEvent; +} RunnerHIDFunctions; + // IOHIDDigitizerEventMask bits (IOHIDFamily IOHIDEventTypes.h). A real edge swipe on a simulator // carries Range/Touch/Position plus SwipeLocked and the swipe direction on every frame. @@ -231,7 +264,10 @@ static void RunnerWaitUntil(uint64_t startTicks, double offsetMs) { mach_wait_until(startTicks + nanoseconds * timebase.denom / timebase.numer); } -NSString * _Nullable RunnerDispatchEdgeContact( +// Owns one edge contact from touch-down to its terminal event. The cancel event is built before +// touch-down; after touch-down, every exit dispatches either the touch-up or that cancel event. +// Returns nil on success, or the reason the contact did not complete. +static NSString * _Nullable RunnerDispatchEdgeContact( const RunnerHIDFunctions *functions, CFTypeRef client, uint64_t sender, @@ -321,4 +357,70 @@ + (NSString * _Nullable)dispatchWithApplication:(id)application @end +#if defined(AGENT_DEVICE_RUNNER_UNIT_TESTS) +#import "UnitTests/RunnerEdgeContactTesting.h" + +// Test facade: the contact owner with HID constructors that fail on demand. Unit-test builds only. +static NSInteger RunnerTestingFailingFrame; +static NSInteger RunnerTestingCreatedFrames; +static NSMutableArray *> *RunnerTestingDispatched; + +static RunnerHIDEventRef RunnerTestingCreateHand( + CFAllocatorRef allocator, uint64_t timestamp, uint32_t transducer, uint32_t index, uint32_t identity, + uint32_t mask, uint32_t buttons, double x, double y, double z, double tip, double barrel, + Boolean range, Boolean touch, uint32_t options +) { + NSInteger frame = RunnerTestingCreatedFrames++; + if (RunnerTestingFailingFrame >= 0 && frame >= RunnerTestingFailingFrame) return NULL; + NSMutableDictionary *event = [@{ @"mask": @(mask), @"touching": @(touch), @"restamped": @NO } mutableCopy]; + return CFBridgingRetain(event); +} + +static RunnerHIDEventRef RunnerTestingCreateFinger( + CFAllocatorRef allocator, uint64_t timestamp, uint32_t index, uint32_t identity, uint32_t mask, + double x, double y, double z, double tip, double twist, Boolean range, Boolean touch, uint32_t options +) { + return CFBridgingRetain([NSMutableDictionary dictionary]); +} + +static void RunnerTestingAppend(RunnerHIDEventRef parent, RunnerHIDEventRef child, uint32_t options) {} +static void RunnerTestingSetInteger(RunnerHIDEventRef event, uint32_t field, long value) {} +static void RunnerTestingSetFloat(RunnerHIDEventRef event, uint32_t field, double value) {} +static void RunnerTestingSetSender(RunnerHIDEventRef event, uint64_t sender) {} + +static void RunnerTestingSetTimeStamp(RunnerHIDEventRef event, uint64_t timestamp) { + ((__bridge NSMutableDictionary *)event)[@"restamped"] = @YES; +} + +static void RunnerTestingDispatch(CFTypeRef client, RunnerHIDEventRef event) { + [RunnerTestingDispatched addObject:[(__bridge NSDictionary *)event copy]]; +} + +NSArray *> *RunnerEdgeContactDispatches( + NSInteger failingFrame, + NSString **failure +) { + RunnerTestingFailingFrame = failingFrame; + RunnerTestingCreatedFrames = 0; + RunnerTestingDispatched = [NSMutableArray array]; + RunnerHIDFunctions functions = { + .createDigitizerEvent = RunnerTestingCreateHand, + .createFingerEvent = RunnerTestingCreateFinger, + .appendEvent = RunnerTestingAppend, + .setIntegerValue = RunnerTestingSetInteger, + .setFloatValue = RunnerTestingSetFloat, + .setSenderID = RunnerTestingSetSender, + .setTimeStamp = RunnerTestingSetTimeStamp, + .dispatchEvent = RunnerTestingDispatch, + }; + NSArray *samples = @[ + @{ @"x": @0, @"y": @50, @"offsetMs": @0 }, + @{ @"x": @100, @"y": @50, @"offsetMs": @48 }, + ]; + NSString *result = RunnerDispatchEdgeContact(&functions, (__bridge CFTypeRef)samples, 1, samples, CGSizeMake(100, 100)); + if (failure != NULL) *failure = result; + return [RunnerTestingDispatched copy]; +} +#endif + #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m deleted file mode 100644 index 9fe03b0f47..0000000000 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m +++ /dev/null @@ -1,70 +0,0 @@ -#if defined(AGENT_DEVICE_RUNNER_UNIT_TESTS) -#import "RunnerEdgeContactTesting.h" - -#if TARGET_OS_IOS && TARGET_OS_SIMULATOR - -#import "RunnerSimulatorEdgeTouch+Internal.h" - -static NSInteger RunnerTestingFailingFrame; -static NSInteger RunnerTestingCreatedFrames; -static NSMutableArray *> *RunnerTestingDispatched; - -static RunnerHIDEventRef RunnerTestingCreateHand( - CFAllocatorRef allocator, uint64_t timestamp, uint32_t transducer, uint32_t index, uint32_t identity, - uint32_t mask, uint32_t buttons, double x, double y, double z, double tip, double barrel, - Boolean range, Boolean touch, uint32_t options -) { - NSInteger frame = RunnerTestingCreatedFrames++; - if (RunnerTestingFailingFrame >= 0 && frame >= RunnerTestingFailingFrame) return NULL; - NSMutableDictionary *event = [@{ @"mask": @(mask), @"touching": @(touch), @"restamped": @NO } mutableCopy]; - return CFBridgingRetain(event); -} - -static RunnerHIDEventRef RunnerTestingCreateFinger( - CFAllocatorRef allocator, uint64_t timestamp, uint32_t index, uint32_t identity, uint32_t mask, - double x, double y, double z, double tip, double twist, Boolean range, Boolean touch, uint32_t options -) { - return CFBridgingRetain([NSMutableDictionary dictionary]); -} - -static void RunnerTestingAppend(RunnerHIDEventRef parent, RunnerHIDEventRef child, uint32_t options) {} -static void RunnerTestingSetInteger(RunnerHIDEventRef event, uint32_t field, long value) {} -static void RunnerTestingSetFloat(RunnerHIDEventRef event, uint32_t field, double value) {} -static void RunnerTestingSetSender(RunnerHIDEventRef event, uint64_t sender) {} - -static void RunnerTestingSetTimeStamp(RunnerHIDEventRef event, uint64_t timestamp) { - ((__bridge NSMutableDictionary *)event)[@"restamped"] = @YES; -} - -static void RunnerTestingDispatch(CFTypeRef client, RunnerHIDEventRef event) { - [RunnerTestingDispatched addObject:[(__bridge NSDictionary *)event copy]]; -} - -NSArray *> *RunnerEdgeContactDispatches( - NSInteger failingFrame, - NSString **failure -) { - RunnerTestingFailingFrame = failingFrame; - RunnerTestingCreatedFrames = 0; - RunnerTestingDispatched = [NSMutableArray array]; - RunnerHIDFunctions functions = { - .createDigitizerEvent = RunnerTestingCreateHand, - .createFingerEvent = RunnerTestingCreateFinger, - .appendEvent = RunnerTestingAppend, - .setIntegerValue = RunnerTestingSetInteger, - .setFloatValue = RunnerTestingSetFloat, - .setSenderID = RunnerTestingSetSender, - .setTimeStamp = RunnerTestingSetTimeStamp, - .dispatchEvent = RunnerTestingDispatch, - }; - NSArray *samples = @[ - @{ @"x": @0, @"y": @50, @"offsetMs": @0 }, - @{ @"x": @100, @"y": @50, @"offsetMs": @48 }, - ]; - NSString *result = RunnerDispatchEdgeContact(&functions, (__bridge CFTypeRef)samples, 1, samples, CGSizeMake(100, 100)); - if (failure != NULL) *failure = result; - return [RunnerTestingDispatched copy]; -} - -#endif -#endif From 8bbcfeca321ae2813919eeb01f27b251f802a2d5 Mon Sep 17 00:00:00 2001 From: gronxb Date: Wed, 4 Nov 2026 16:57:02 +0900 Subject: [PATCH 09/12] refactor(ios): keep the edge contact test facade out of the packaged runner Address review: the packager copies Objective-C sources verbatim, so the AGENT_DEVICE_RUNNER_UNIT_TESTS block in RunnerSimulatorEdgeTouch.m shipped and the package leak guard rejected it. Move the contact owner into a header whose declarations all have internal linkage, include it from the runner and from the test facade in UnitTests/, and keep the shipped source free of the unit-test macro. --- .../RunnerSimulatorEdgeContact.h | 217 ++++++++++++++ .../RunnerSimulatorEdgeTouch.m | 270 +----------------- .../UnitTests/RunnerEdgeContactTesting.m | 70 +++++ 3 files changed, 288 insertions(+), 269 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeContact.h create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeContact.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeContact.h new file mode 100644 index 0000000000..52bbbc01a1 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeContact.h @@ -0,0 +1,217 @@ +#import +#import +#import +#import + +// One simulator edge contact, from its reserved terminal event to touch-up. Every declaration here +// has internal linkage, so the runner exports none of it; RunnerSimulatorEdgeTouch.m and its unit +// tests each compile their own copy. + +#if TARGET_OS_IOS && TARGET_OS_SIMULATOR + +NS_ASSUME_NONNULL_BEGIN + +typedef CFTypeRef RunnerHIDEventRef; +typedef RunnerHIDEventRef _Nullable (*RunnerCreateDigitizerEvent)( + CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, double, double, + double, double, double, Boolean, Boolean, uint32_t); +typedef RunnerHIDEventRef _Nullable (*RunnerCreateDigitizerFingerEvent)( + CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, double, double, double, double, double, + Boolean, Boolean, uint32_t); +typedef void (*RunnerAppendEvent)(RunnerHIDEventRef, RunnerHIDEventRef, uint32_t); +typedef void (*RunnerSetIntegerValue)(RunnerHIDEventRef, uint32_t, long); +typedef void (*RunnerSetFloatValue)(RunnerHIDEventRef, uint32_t, double); +typedef void (*RunnerSetSenderID)(RunnerHIDEventRef, uint64_t); +typedef void (*RunnerSetTimeStamp)(RunnerHIDEventRef, uint64_t); +typedef CFTypeRef _Nullable (*RunnerCreateEventSystemClient)(CFAllocatorRef); +typedef CFArrayRef _Nullable (*RunnerCopyServices)(CFTypeRef); +typedef CFTypeRef _Nullable (*RunnerCopyServiceProperty)(CFTypeRef, CFStringRef); +typedef CFTypeRef _Nullable (*RunnerGetServiceRegistryID)(CFTypeRef); +typedef void (*RunnerDispatchEvent)(CFTypeRef, RunnerHIDEventRef); + +// The IOKit HID entry points the edge touch uses, resolved at run time. +typedef struct { + RunnerCreateDigitizerEvent createDigitizerEvent; + RunnerCreateDigitizerFingerEvent createFingerEvent; + RunnerAppendEvent appendEvent; + RunnerSetIntegerValue setIntegerValue; + RunnerSetFloatValue setFloatValue; + RunnerSetSenderID setSenderID; + RunnerSetTimeStamp setTimeStamp; + RunnerCreateEventSystemClient createClient; + RunnerCopyServices copyServices; + RunnerCopyServiceProperty copyServiceProperty; + RunnerGetServiceRegistryID getServiceRegistryID; + RunnerDispatchEvent dispatchEvent; +} RunnerHIDFunctions; + +// IOHIDDigitizerEventMask bits (IOHIDFamily IOHIDEventTypes.h). A real edge swipe on a simulator +// carries Range/Touch/Position plus SwipeLocked and the swipe direction on every frame. +static const uint32_t RunnerDigitizerRange = 1u << 0; +static const uint32_t RunnerDigitizerTouch = 1u << 1; +static const uint32_t RunnerDigitizerPosition = 1u << 2; +static const uint32_t RunnerDigitizerCancel = 1u << 7; +static const uint32_t RunnerDigitizerSwipeLocked = 1u << 18; +static const uint32_t RunnerDigitizerSwipeUp = 1u << 24; +static const uint32_t RunnerDigitizerSwipeDown = 1u << 25; +static const uint32_t RunnerDigitizerSwipeLeft = 1u << 26; +static const uint32_t RunnerDigitizerSwipeRight = 1u << 27; + +// IOHIDEventField values of the digitizer event type (11). +static const uint32_t RunnerDigitizerFieldMajorRadius = (11u << 16) | 20u; +static const uint32_t RunnerDigitizerFieldMinorRadius = (11u << 16) | 21u; +static const uint32_t RunnerDigitizerFieldIsDisplayIntegrated = (11u << 16) | 25u; + +static const uint32_t RunnerTransducerHand = 3; +static const uint32_t RunnerFingerIdentity = 2; +static const double RunnerFingerRadius = 0.04; +static const double RunnerFingerPressure = 0.5; +static const double RunnerFrameIntervalMs = 16.0; + +// One retained hand event with its finger, or NULL when either could not be created. +static inline RunnerHIDEventRef _Nullable RunnerCreateFrame( + const RunnerHIDFunctions *functions, + uint64_t sender, + CGPoint normalized, + uint32_t mask, + Boolean touching +) { + uint64_t timestamp = mach_absolute_time(); + RunnerHIDEventRef hand = functions->createDigitizerEvent( + kCFAllocatorDefault, timestamp, RunnerTransducerHand, 0, 0, mask, 0, + normalized.x, normalized.y, 0, 0, 0, touching, touching, 0); + RunnerHIDEventRef finger = functions->createFingerEvent( + kCFAllocatorDefault, timestamp, 1, RunnerFingerIdentity, mask, normalized.x, normalized.y, 0, + touching ? RunnerFingerPressure : 0, 0, touching, touching, 0); + if (hand == NULL || finger == NULL) { + if (hand != NULL) CFRelease(hand); + if (finger != NULL) CFRelease(finger); + return NULL; + } + functions->setIntegerValue(hand, RunnerDigitizerFieldIsDisplayIntegrated, 1); + functions->setFloatValue(finger, RunnerDigitizerFieldMajorRadius, RunnerFingerRadius); + functions->setFloatValue(finger, RunnerDigitizerFieldMinorRadius, RunnerFingerRadius); + functions->appendEvent(hand, finger, 0); + CFRelease(finger); + functions->setSenderID(hand, sender); + return hand; +} + +static inline void RunnerDispatchAndRelease(const RunnerHIDFunctions *functions, CFTypeRef client, RunnerHIDEventRef event) { + functions->dispatchEvent(client, event); + CFRelease(event); +} + +static inline CGPoint RunnerSamplePoint(NSDictionary *sample, CGSize screenSize) { + return CGPointMake(sample[@"x"].doubleValue / screenSize.width, sample[@"y"].doubleValue / screenSize.height); +} + +// The swipe direction the system reports for a contact moving from `start` to `end`, in the +// display's native axes, where y grows downward. +static inline uint32_t RunnerSwipeDirection(CGPoint start, CGPoint end) { + double dx = end.x - start.x; + double dy = end.y - start.y; + if (fabs(dx) >= fabs(dy)) return dx >= 0 ? RunnerDigitizerSwipeRight : RunnerDigitizerSwipeLeft; + return dy >= 0 ? RunnerDigitizerSwipeDown : RunnerDigitizerSwipeUp; +} + +// The contact position at `offsetMs`, interpolated between the planned samples. +static inline CGPoint RunnerPointAtOffset( + NSArray *> *samples, + double offsetMs, + CGSize screenSize +) { + for (NSUInteger index = 1; index < samples.count; index++) { + NSDictionary *end = samples[index]; + if (offsetMs > end[@"offsetMs"].doubleValue) continue; + NSDictionary *start = samples[index - 1]; + double startOffset = start[@"offsetMs"].doubleValue; + double span = end[@"offsetMs"].doubleValue - startOffset; + double progress = span > 0 ? (offsetMs - startOffset) / span : 1; + CGPoint from = RunnerSamplePoint(start, screenSize); + CGPoint to = RunnerSamplePoint(end, screenSize); + return CGPointMake(from.x + (to.x - from.x) * progress, from.y + (to.y - from.y) * progress); + } + return RunnerSamplePoint(samples.lastObject, screenSize); +} + +// The contact position at `offsetMs` when it eases in quadratically along the planned path. +static inline CGPoint RunnerEasedPoint( + NSArray *> *samples, + double offsetMs, + double durationMs, + CGSize screenSize +) { + double progress = durationMs > 0 ? offsetMs / durationMs : 1; + return RunnerPointAtOffset(samples, durationMs * progress * progress, screenSize); +} + +static inline void RunnerWaitUntil(uint64_t startTicks, double offsetMs) { + static mach_timebase_info_data_t timebase; + if (timebase.denom == 0) mach_timebase_info(&timebase); + uint64_t nanoseconds = (uint64_t)(offsetMs * NSEC_PER_MSEC); + mach_wait_until(startTicks + nanoseconds * timebase.denom / timebase.numer); +} + +// Owns one edge contact from touch-down to its terminal event. The cancel event is built before +// touch-down; after touch-down, every exit dispatches either the touch-up or that cancel event. +// Returns nil on success, or the reason the contact did not complete. +static inline NSString * _Nullable RunnerDispatchEdgeContact( + const RunnerHIDFunctions *functions, + CFTypeRef client, + uint64_t sender, + NSArray *> *samples, + CGSize screenSize +) { + double durationMs = samples.lastObject[@"offsetMs"].doubleValue; + CGPoint start = RunnerPointAtOffset(samples, 0, screenSize); + uint32_t swipe = RunnerDigitizerSwipeLocked | + RunnerSwipeDirection(start, RunnerPointAtOffset(samples, durationMs, screenSize)); + // The terminal event exists before touch-down, so a contact that goes down always ends. + RunnerHIDEventRef cancel = RunnerCreateFrame( + functions, sender, start, RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerCancel | swipe, false); + if (cancel == NULL) return @"simulator HID dispatch could not create the touch-cancel event"; + RunnerHIDEventRef down = RunnerCreateFrame( + functions, sender, start, RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerPosition | swipe, true); + if (down == NULL) { + CFRelease(cancel); + return @"simulator HID dispatch could not create the touch-down event"; + } + uint64_t startTicks = mach_absolute_time(); + RunnerDispatchAndRelease(functions, client, down); + NSString *failure = nil; + for (double offsetMs = RunnerFrameIntervalMs; offsetMs < durationMs + RunnerFrameIntervalMs; + offsetMs += RunnerFrameIntervalMs) { + double frameOffset = MIN(offsetMs, durationMs); + RunnerWaitUntil(startTicks, frameOffset); + // The edge recognizer fails when the first move leaves the edge region in one frame, as a fast + // finger does on a wide panel, so the contact eases in like a real swipe. + RunnerHIDEventRef move = RunnerCreateFrame( + functions, sender, RunnerEasedPoint(samples, frameOffset, durationMs, screenSize), + RunnerDigitizerPosition | swipe, true); + if (move == NULL) { + failure = @"simulator HID dispatch could not create a touch-move event; the contact was cancelled"; + break; + } + RunnerDispatchAndRelease(functions, client, move); + } + if (failure == nil) { + RunnerWaitUntil(startTicks, durationMs + RunnerFrameIntervalMs); + RunnerHIDEventRef lift = RunnerCreateFrame( + functions, sender, RunnerPointAtOffset(samples, durationMs, screenSize), + RunnerDigitizerRange | RunnerDigitizerTouch | swipe, false); + if (lift != NULL) { + RunnerDispatchAndRelease(functions, client, lift); + CFRelease(cancel); + return nil; + } + failure = @"simulator HID dispatch could not create the touch-up event; the contact was cancelled"; + } + functions->setTimeStamp(cancel, mach_absolute_time()); + RunnerDispatchAndRelease(functions, client, cancel); + return failure; +} + +NS_ASSUME_NONNULL_END + +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m index 00056ae892..2192e8af51 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m @@ -2,69 +2,11 @@ #if TARGET_OS_IOS && TARGET_OS_SIMULATOR +#import "RunnerSimulatorEdgeContact.h" #import "RunnerXCTestEventBridge.h" #import -#import #import -typedef CFTypeRef RunnerHIDEventRef; -typedef RunnerHIDEventRef _Nullable (*RunnerCreateDigitizerEvent)( - CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, double, double, - double, double, double, Boolean, Boolean, uint32_t); -typedef RunnerHIDEventRef _Nullable (*RunnerCreateDigitizerFingerEvent)( - CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, double, double, double, double, double, - Boolean, Boolean, uint32_t); -typedef void (*RunnerAppendEvent)(RunnerHIDEventRef, RunnerHIDEventRef, uint32_t); -typedef void (*RunnerSetIntegerValue)(RunnerHIDEventRef, uint32_t, long); -typedef void (*RunnerSetFloatValue)(RunnerHIDEventRef, uint32_t, double); -typedef void (*RunnerSetSenderID)(RunnerHIDEventRef, uint64_t); -typedef void (*RunnerSetTimeStamp)(RunnerHIDEventRef, uint64_t); -typedef CFTypeRef _Nullable (*RunnerCreateEventSystemClient)(CFAllocatorRef); -typedef CFArrayRef _Nullable (*RunnerCopyServices)(CFTypeRef); -typedef CFTypeRef _Nullable (*RunnerCopyServiceProperty)(CFTypeRef, CFStringRef); -typedef CFTypeRef _Nullable (*RunnerGetServiceRegistryID)(CFTypeRef); -typedef void (*RunnerDispatchEvent)(CFTypeRef, RunnerHIDEventRef); - -// The IOKit HID entry points the edge touch uses, resolved at run time. -typedef struct { - RunnerCreateDigitizerEvent createDigitizerEvent; - RunnerCreateDigitizerFingerEvent createFingerEvent; - RunnerAppendEvent appendEvent; - RunnerSetIntegerValue setIntegerValue; - RunnerSetFloatValue setFloatValue; - RunnerSetSenderID setSenderID; - RunnerSetTimeStamp setTimeStamp; - RunnerCreateEventSystemClient createClient; - RunnerCopyServices copyServices; - RunnerCopyServiceProperty copyServiceProperty; - RunnerGetServiceRegistryID getServiceRegistryID; - RunnerDispatchEvent dispatchEvent; -} RunnerHIDFunctions; - - -// IOHIDDigitizerEventMask bits (IOHIDFamily IOHIDEventTypes.h). A real edge swipe on a simulator -// carries Range/Touch/Position plus SwipeLocked and the swipe direction on every frame. -static const uint32_t RunnerDigitizerRange = 1u << 0; -static const uint32_t RunnerDigitizerTouch = 1u << 1; -static const uint32_t RunnerDigitizerPosition = 1u << 2; -static const uint32_t RunnerDigitizerCancel = 1u << 7; -static const uint32_t RunnerDigitizerSwipeLocked = 1u << 18; -static const uint32_t RunnerDigitizerSwipeUp = 1u << 24; -static const uint32_t RunnerDigitizerSwipeDown = 1u << 25; -static const uint32_t RunnerDigitizerSwipeLeft = 1u << 26; -static const uint32_t RunnerDigitizerSwipeRight = 1u << 27; - -// IOHIDEventField values of the digitizer event type (11). -static const uint32_t RunnerDigitizerFieldMajorRadius = (11u << 16) | 20u; -static const uint32_t RunnerDigitizerFieldMinorRadius = (11u << 16) | 21u; -static const uint32_t RunnerDigitizerFieldIsDisplayIntegrated = (11u << 16) | 25u; - -static const uint32_t RunnerTransducerHand = 3; -static const uint32_t RunnerFingerIdentity = 2; -static const double RunnerFingerRadius = 0.04; -static const double RunnerFingerPressure = 0.5; -static const double RunnerFrameIntervalMs = 16.0; - // The simulator touchscreen HID service: digitizer usage page, touch screen usage. static const long RunnerDigitizerUsagePage = 13; static const long RunnerTouchScreenUsage = 4; @@ -179,150 +121,6 @@ static long RunnerServiceIntegerProperty( (unsigned long)displayID]; } -// One retained hand event with its finger, or NULL when either could not be created. -static RunnerHIDEventRef _Nullable RunnerCreateFrame( - const RunnerHIDFunctions *functions, - uint64_t sender, - CGPoint normalized, - uint32_t mask, - Boolean touching -) { - uint64_t timestamp = mach_absolute_time(); - RunnerHIDEventRef hand = functions->createDigitizerEvent( - kCFAllocatorDefault, timestamp, RunnerTransducerHand, 0, 0, mask, 0, - normalized.x, normalized.y, 0, 0, 0, touching, touching, 0); - RunnerHIDEventRef finger = functions->createFingerEvent( - kCFAllocatorDefault, timestamp, 1, RunnerFingerIdentity, mask, normalized.x, normalized.y, 0, - touching ? RunnerFingerPressure : 0, 0, touching, touching, 0); - if (hand == NULL || finger == NULL) { - if (hand != NULL) CFRelease(hand); - if (finger != NULL) CFRelease(finger); - return NULL; - } - functions->setIntegerValue(hand, RunnerDigitizerFieldIsDisplayIntegrated, 1); - functions->setFloatValue(finger, RunnerDigitizerFieldMajorRadius, RunnerFingerRadius); - functions->setFloatValue(finger, RunnerDigitizerFieldMinorRadius, RunnerFingerRadius); - functions->appendEvent(hand, finger, 0); - CFRelease(finger); - functions->setSenderID(hand, sender); - return hand; -} - -static void RunnerDispatchAndRelease(const RunnerHIDFunctions *functions, CFTypeRef client, RunnerHIDEventRef event) { - functions->dispatchEvent(client, event); - CFRelease(event); -} - -static CGPoint RunnerSamplePoint(NSDictionary *sample, CGSize screenSize) { - return CGPointMake(sample[@"x"].doubleValue / screenSize.width, sample[@"y"].doubleValue / screenSize.height); -} - -// The swipe direction the system reports for a contact moving from `start` to `end`, in the -// display's native axes, where y grows downward. -static uint32_t RunnerSwipeDirection(CGPoint start, CGPoint end) { - double dx = end.x - start.x; - double dy = end.y - start.y; - if (fabs(dx) >= fabs(dy)) return dx >= 0 ? RunnerDigitizerSwipeRight : RunnerDigitizerSwipeLeft; - return dy >= 0 ? RunnerDigitizerSwipeDown : RunnerDigitizerSwipeUp; -} - -// The contact position at `offsetMs`, interpolated between the planned samples. -static CGPoint RunnerPointAtOffset( - NSArray *> *samples, - double offsetMs, - CGSize screenSize -) { - for (NSUInteger index = 1; index < samples.count; index++) { - NSDictionary *end = samples[index]; - if (offsetMs > end[@"offsetMs"].doubleValue) continue; - NSDictionary *start = samples[index - 1]; - double startOffset = start[@"offsetMs"].doubleValue; - double span = end[@"offsetMs"].doubleValue - startOffset; - double progress = span > 0 ? (offsetMs - startOffset) / span : 1; - CGPoint from = RunnerSamplePoint(start, screenSize); - CGPoint to = RunnerSamplePoint(end, screenSize); - return CGPointMake(from.x + (to.x - from.x) * progress, from.y + (to.y - from.y) * progress); - } - return RunnerSamplePoint(samples.lastObject, screenSize); -} - -// The contact position at `offsetMs` when it eases in quadratically along the planned path. -static CGPoint RunnerEasedPoint( - NSArray *> *samples, - double offsetMs, - double durationMs, - CGSize screenSize -) { - double progress = durationMs > 0 ? offsetMs / durationMs : 1; - return RunnerPointAtOffset(samples, durationMs * progress * progress, screenSize); -} - -static void RunnerWaitUntil(uint64_t startTicks, double offsetMs) { - static mach_timebase_info_data_t timebase; - if (timebase.denom == 0) mach_timebase_info(&timebase); - uint64_t nanoseconds = (uint64_t)(offsetMs * NSEC_PER_MSEC); - mach_wait_until(startTicks + nanoseconds * timebase.denom / timebase.numer); -} - -// Owns one edge contact from touch-down to its terminal event. The cancel event is built before -// touch-down; after touch-down, every exit dispatches either the touch-up or that cancel event. -// Returns nil on success, or the reason the contact did not complete. -static NSString * _Nullable RunnerDispatchEdgeContact( - const RunnerHIDFunctions *functions, - CFTypeRef client, - uint64_t sender, - NSArray *> *samples, - CGSize screenSize -) { - double durationMs = samples.lastObject[@"offsetMs"].doubleValue; - CGPoint start = RunnerPointAtOffset(samples, 0, screenSize); - uint32_t swipe = RunnerDigitizerSwipeLocked | - RunnerSwipeDirection(start, RunnerPointAtOffset(samples, durationMs, screenSize)); - // The terminal event exists before touch-down, so a contact that goes down always ends. - RunnerHIDEventRef cancel = RunnerCreateFrame( - functions, sender, start, RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerCancel | swipe, false); - if (cancel == NULL) return @"simulator HID dispatch could not create the touch-cancel event"; - RunnerHIDEventRef down = RunnerCreateFrame( - functions, sender, start, RunnerDigitizerRange | RunnerDigitizerTouch | RunnerDigitizerPosition | swipe, true); - if (down == NULL) { - CFRelease(cancel); - return @"simulator HID dispatch could not create the touch-down event"; - } - uint64_t startTicks = mach_absolute_time(); - RunnerDispatchAndRelease(functions, client, down); - NSString *failure = nil; - for (double offsetMs = RunnerFrameIntervalMs; offsetMs < durationMs + RunnerFrameIntervalMs; - offsetMs += RunnerFrameIntervalMs) { - double frameOffset = MIN(offsetMs, durationMs); - RunnerWaitUntil(startTicks, frameOffset); - // The edge recognizer fails when the first move leaves the edge region in one frame, as a fast - // finger does on a wide panel, so the contact eases in like a real swipe. - RunnerHIDEventRef move = RunnerCreateFrame( - functions, sender, RunnerEasedPoint(samples, frameOffset, durationMs, screenSize), - RunnerDigitizerPosition | swipe, true); - if (move == NULL) { - failure = @"simulator HID dispatch could not create a touch-move event; the contact was cancelled"; - break; - } - RunnerDispatchAndRelease(functions, client, move); - } - if (failure == nil) { - RunnerWaitUntil(startTicks, durationMs + RunnerFrameIntervalMs); - RunnerHIDEventRef lift = RunnerCreateFrame( - functions, sender, RunnerPointAtOffset(samples, durationMs, screenSize), - RunnerDigitizerRange | RunnerDigitizerTouch | swipe, false); - if (lift != NULL) { - RunnerDispatchAndRelease(functions, client, lift); - CFRelease(cancel); - return nil; - } - failure = @"simulator HID dispatch could not create the touch-up event; the contact was cancelled"; - } - functions->setTimeStamp(cancel, mach_absolute_time()); - RunnerDispatchAndRelease(functions, client, cancel); - return failure; -} - @implementation RunnerSimulatorEdgeTouch + (NSString * _Nullable)dispatchWithApplication:(id)application @@ -357,70 +155,4 @@ + (NSString * _Nullable)dispatchWithApplication:(id)application @end -#if defined(AGENT_DEVICE_RUNNER_UNIT_TESTS) -#import "UnitTests/RunnerEdgeContactTesting.h" - -// Test facade: the contact owner with HID constructors that fail on demand. Unit-test builds only. -static NSInteger RunnerTestingFailingFrame; -static NSInteger RunnerTestingCreatedFrames; -static NSMutableArray *> *RunnerTestingDispatched; - -static RunnerHIDEventRef RunnerTestingCreateHand( - CFAllocatorRef allocator, uint64_t timestamp, uint32_t transducer, uint32_t index, uint32_t identity, - uint32_t mask, uint32_t buttons, double x, double y, double z, double tip, double barrel, - Boolean range, Boolean touch, uint32_t options -) { - NSInteger frame = RunnerTestingCreatedFrames++; - if (RunnerTestingFailingFrame >= 0 && frame >= RunnerTestingFailingFrame) return NULL; - NSMutableDictionary *event = [@{ @"mask": @(mask), @"touching": @(touch), @"restamped": @NO } mutableCopy]; - return CFBridgingRetain(event); -} - -static RunnerHIDEventRef RunnerTestingCreateFinger( - CFAllocatorRef allocator, uint64_t timestamp, uint32_t index, uint32_t identity, uint32_t mask, - double x, double y, double z, double tip, double twist, Boolean range, Boolean touch, uint32_t options -) { - return CFBridgingRetain([NSMutableDictionary dictionary]); -} - -static void RunnerTestingAppend(RunnerHIDEventRef parent, RunnerHIDEventRef child, uint32_t options) {} -static void RunnerTestingSetInteger(RunnerHIDEventRef event, uint32_t field, long value) {} -static void RunnerTestingSetFloat(RunnerHIDEventRef event, uint32_t field, double value) {} -static void RunnerTestingSetSender(RunnerHIDEventRef event, uint64_t sender) {} - -static void RunnerTestingSetTimeStamp(RunnerHIDEventRef event, uint64_t timestamp) { - ((__bridge NSMutableDictionary *)event)[@"restamped"] = @YES; -} - -static void RunnerTestingDispatch(CFTypeRef client, RunnerHIDEventRef event) { - [RunnerTestingDispatched addObject:[(__bridge NSDictionary *)event copy]]; -} - -NSArray *> *RunnerEdgeContactDispatches( - NSInteger failingFrame, - NSString **failure -) { - RunnerTestingFailingFrame = failingFrame; - RunnerTestingCreatedFrames = 0; - RunnerTestingDispatched = [NSMutableArray array]; - RunnerHIDFunctions functions = { - .createDigitizerEvent = RunnerTestingCreateHand, - .createFingerEvent = RunnerTestingCreateFinger, - .appendEvent = RunnerTestingAppend, - .setIntegerValue = RunnerTestingSetInteger, - .setFloatValue = RunnerTestingSetFloat, - .setSenderID = RunnerTestingSetSender, - .setTimeStamp = RunnerTestingSetTimeStamp, - .dispatchEvent = RunnerTestingDispatch, - }; - NSArray *samples = @[ - @{ @"x": @0, @"y": @50, @"offsetMs": @0 }, - @{ @"x": @100, @"y": @50, @"offsetMs": @48 }, - ]; - NSString *result = RunnerDispatchEdgeContact(&functions, (__bridge CFTypeRef)samples, 1, samples, CGSizeMake(100, 100)); - if (failure != NULL) *failure = result; - return [RunnerTestingDispatched copy]; -} -#endif - #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m new file mode 100644 index 0000000000..66503678fb --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m @@ -0,0 +1,70 @@ +#if defined(AGENT_DEVICE_RUNNER_UNIT_TESTS) +#import "RunnerEdgeContactTesting.h" + +#if TARGET_OS_IOS && TARGET_OS_SIMULATOR + +#import "RunnerSimulatorEdgeContact.h" + +static NSInteger RunnerTestingFailingFrame; +static NSInteger RunnerTestingCreatedFrames; +static NSMutableArray *> *RunnerTestingDispatched; + +static RunnerHIDEventRef RunnerTestingCreateHand( + CFAllocatorRef allocator, uint64_t timestamp, uint32_t transducer, uint32_t index, uint32_t identity, + uint32_t mask, uint32_t buttons, double x, double y, double z, double tip, double barrel, + Boolean range, Boolean touch, uint32_t options +) { + NSInteger frame = RunnerTestingCreatedFrames++; + if (RunnerTestingFailingFrame >= 0 && frame >= RunnerTestingFailingFrame) return NULL; + NSMutableDictionary *event = [@{ @"mask": @(mask), @"touching": @(touch), @"restamped": @NO } mutableCopy]; + return CFBridgingRetain(event); +} + +static RunnerHIDEventRef RunnerTestingCreateFinger( + CFAllocatorRef allocator, uint64_t timestamp, uint32_t index, uint32_t identity, uint32_t mask, + double x, double y, double z, double tip, double twist, Boolean range, Boolean touch, uint32_t options +) { + return CFBridgingRetain([NSMutableDictionary dictionary]); +} + +static void RunnerTestingAppend(RunnerHIDEventRef parent, RunnerHIDEventRef child, uint32_t options) {} +static void RunnerTestingSetInteger(RunnerHIDEventRef event, uint32_t field, long value) {} +static void RunnerTestingSetFloat(RunnerHIDEventRef event, uint32_t field, double value) {} +static void RunnerTestingSetSender(RunnerHIDEventRef event, uint64_t sender) {} + +static void RunnerTestingSetTimeStamp(RunnerHIDEventRef event, uint64_t timestamp) { + ((__bridge NSMutableDictionary *)event)[@"restamped"] = @YES; +} + +static void RunnerTestingDispatch(CFTypeRef client, RunnerHIDEventRef event) { + [RunnerTestingDispatched addObject:[(__bridge NSDictionary *)event copy]]; +} + +NSArray *> *RunnerEdgeContactDispatches( + NSInteger failingFrame, + NSString **failure +) { + RunnerTestingFailingFrame = failingFrame; + RunnerTestingCreatedFrames = 0; + RunnerTestingDispatched = [NSMutableArray array]; + RunnerHIDFunctions functions = { + .createDigitizerEvent = RunnerTestingCreateHand, + .createFingerEvent = RunnerTestingCreateFinger, + .appendEvent = RunnerTestingAppend, + .setIntegerValue = RunnerTestingSetInteger, + .setFloatValue = RunnerTestingSetFloat, + .setSenderID = RunnerTestingSetSender, + .setTimeStamp = RunnerTestingSetTimeStamp, + .dispatchEvent = RunnerTestingDispatch, + }; + NSArray *samples = @[ + @{ @"x": @0, @"y": @50, @"offsetMs": @0 }, + @{ @"x": @100, @"y": @50, @"offsetMs": @48 }, + ]; + NSString *result = RunnerDispatchEdgeContact(&functions, (__bridge CFTypeRef)samples, 1, samples, CGSizeMake(100, 100)); + if (failure != NULL) *failure = result; + return [RunnerTestingDispatched copy]; +} + +#endif +#endif From b18a49d351408131f1342a2938bb620bc7ff6361 Mon Sep 17 00:00:00 2001 From: gronxb Date: Wed, 4 Nov 2026 21:04:48 +0900 Subject: [PATCH 10/12] fix(ios): select a simulator touchscreen that reports no Transport Reported on iOS 18.1 and 26.5 simulators: the display's touchscreen service passed the usage, display and registry checks but read no Transport, so edge swipes failed with "no simulator touchscreen HID service". The oldest CoreDevice touchscreen still wins. Without one, the display's only remaining candidate is used, and several are refused as ambiguous with how each Transport read (missing, other type, other value). The no-candidate error now names the stage that rejected the touchscreens. Unit tests cover the reported service, CoreDevice precedence, duplicates, ambiguity and the error. --- .../RunnerSimulatorEdgeContact.h | 138 ++++++++++++++++++ .../RunnerSimulatorEdgeTouch.m | 60 +------- .../UnitTests/RunnerEdgeContactTesting.h | 9 ++ .../UnitTests/RunnerEdgeContactTesting.m | 26 ++++ .../RunnerTests+ScreenEdgeGestureTests.swift | 66 +++++++++ 5 files changed, 240 insertions(+), 59 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeContact.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeContact.h index 52bbbc01a1..522440aded 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeContact.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeContact.h @@ -212,6 +212,144 @@ static inline NSString * _Nullable RunnerDispatchEdgeContact( return failure; } +// The simulator touchscreen HID service: digitizer usage page, touch screen usage. +static const long RunnerDigitizerUsagePage = 13; +static const long RunnerTouchScreenUsage = 4; + +static inline long RunnerServiceIntegerProperty( + const RunnerHIDFunctions *functions, + CFTypeRef service, + CFStringRef key +) { + CFTypeRef value = functions->copyServiceProperty(service, key); + long result = -1; + if (value != NULL && CFGetTypeID(value) == CFNumberGetTypeID()) { + CFNumberGetValue((CFNumberRef)value, kCFNumberLongType, &result); + } + if (value != NULL) CFRelease(value); + return result; +} + +static inline NSString * _Nullable RunnerServiceStringProperty( + const RunnerHIDFunctions *functions, + CFTypeRef service, + CFStringRef key +) { + CFTypeRef value = functions->copyServiceProperty(service, key); + NSString *result = nil; + if (value != NULL && CFGetTypeID(value) == CFStringGetTypeID()) { + result = [(__bridge NSString *)value copy]; + } + if (value != NULL) CFRelease(value); + return result; +} + +// How a service property reads: absent, present with a type other than the one asked for, or present. +typedef NS_ENUM(NSInteger, RunnerServicePropertyRead) { + RunnerServicePropertyMissing, + RunnerServicePropertyUnexpectedType, + RunnerServicePropertyPresent, +}; + +static inline RunnerServicePropertyRead RunnerReadServiceStringProperty( + const RunnerHIDFunctions *functions, + CFTypeRef service, + CFStringRef key, + NSString * _Nullable __autoreleasing * _Nonnull value +) { + *value = nil; + CFTypeRef property = functions->copyServiceProperty(service, key); + if (property == NULL) return RunnerServicePropertyMissing; + RunnerServicePropertyRead read = RunnerServicePropertyUnexpectedType; + if (CFGetTypeID(property) == CFStringGetTypeID()) { + *value = [(__bridge NSString *)property copy]; + read = RunnerServicePropertyPresent; + } + CFRelease(property); + return read; +} + +// Touches reach the app only with its display's touchscreen service as their sender. A candidate is +// a touchscreen (digitizer usage page, touch screen usage) on the app's display with a readable +// registry ID. A simulator client names its service's Transport `CoreDevice`, one service per +// attached client; the oldest, created at boot, is always present and wins. Some simulator hosts +// name no Transport at all; their display's only candidate is its touchscreen, and several are +// ambiguous. +static inline NSString * _Nullable RunnerSelectTouchscreenSender( + const RunnerHIDFunctions *functions, + CFTypeRef client, + NSString * _Nullable displayUUID, + NSUInteger displayID, + uint64_t *sender +) { + NSUInteger touchscreens = 0; + NSUInteger onDisplay = 0; + NSUInteger coreDevice = 0; + NSUInteger transportMissing = 0; + NSUInteger transportOtherType = 0; + NSUInteger transportOtherValue = 0; + uint64_t oldestCoreDevice = 0; + uint64_t otherCandidate = 0; + CFArrayRef services = functions->copyServices(client); + for (CFIndex index = 0; services != NULL && index < CFArrayGetCount(services); index++) { + CFTypeRef service = CFArrayGetValueAtIndex(services, index); + if (RunnerServiceIntegerProperty(functions, service, CFSTR("PrimaryUsagePage")) != RunnerDigitizerUsagePage || + RunnerServiceIntegerProperty(functions, service, CFSTR("PrimaryUsage")) != RunnerTouchScreenUsage) { + continue; + } + touchscreens += 1; + NSString *serviceDisplayUUID = RunnerServiceStringProperty(functions, service, CFSTR("displayUUID")); + if (serviceDisplayUUID != displayUUID && ![serviceDisplayUUID isEqualToString:displayUUID]) continue; + onDisplay += 1; + CFTypeRef registryID = functions->getServiceRegistryID(service); + uint64_t candidate = 0; + if (registryID == NULL || CFGetTypeID(registryID) != CFNumberGetTypeID() || + !CFNumberGetValue((CFNumberRef)registryID, kCFNumberSInt64Type, &candidate)) { + continue; + } + NSString *transport = nil; + switch (RunnerReadServiceStringProperty(functions, service, CFSTR("Transport"), &transport)) { + case RunnerServicePropertyPresent: + if ([transport isEqualToString:@"CoreDevice"]) { + if (coreDevice == 0 || candidate < oldestCoreDevice) oldestCoreDevice = candidate; + coreDevice += 1; + continue; + } + transportOtherValue += 1; + break; + case RunnerServicePropertyMissing: + transportMissing += 1; + break; + case RunnerServicePropertyUnexpectedType: + transportOtherType += 1; + break; + } + otherCandidate = candidate; + } + if (services != NULL) CFRelease(services); + NSUInteger others = transportMissing + transportOtherType + transportOtherValue; + if (coreDevice > 0) { + *sender = oldestCoreDevice; + return nil; + } + if (others == 1) { + *sender = otherCandidate; + return nil; + } + if (others > 1) { + return [NSString stringWithFormat: + @"display %lu has %lu touchscreen HID services without a CoreDevice transport " + @"(missing: %lu, other type: %lu, other value: %lu), so an edge touch cannot choose one", + (unsigned long)displayID, (unsigned long)others, (unsigned long)transportMissing, + (unsigned long)transportOtherType, (unsigned long)transportOtherValue]; + } + return [NSString stringWithFormat: + @"display %lu has no simulator touchscreen HID service: %lu touchscreen service%@, %lu on this " + @"display, 0 with a readable registry ID", + (unsigned long)displayID, (unsigned long)touchscreens, touchscreens == 1 ? @"" : @"s", + (unsigned long)onDisplay]; +} + NS_ASSUME_NONNULL_END #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m index 2192e8af51..e3677a3a6e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m @@ -7,10 +7,6 @@ #import #import -// The simulator touchscreen HID service: digitizer usage page, touch screen usage. -static const long RunnerDigitizerUsagePage = 13; -static const long RunnerTouchScreenUsage = 4; - static NSString * _Nullable RunnerResolveHIDFunctions(RunnerHIDFunctions *functions) { void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW); if (iokit == NULL) { @@ -40,34 +36,6 @@ return nil; } -static long RunnerServiceIntegerProperty( - const RunnerHIDFunctions *functions, - CFTypeRef service, - CFStringRef key -) { - CFTypeRef value = functions->copyServiceProperty(service, key); - long result = -1; - if (value != NULL && CFGetTypeID(value) == CFNumberGetTypeID()) { - CFNumberGetValue((CFNumberRef)value, kCFNumberLongType, &result); - } - if (value != NULL) CFRelease(value); - return result; -} - -static NSString * _Nullable RunnerServiceStringProperty( - const RunnerHIDFunctions *functions, - CFTypeRef service, - CFStringRef key -) { - CFTypeRef value = functions->copyServiceProperty(service, key); - NSString *result = nil; - if (value != NULL && CFGetTypeID(value) == CFStringGetTypeID()) { - result = [(__bridge NSString *)value copy]; - } - if (value != NULL) CFRelease(value); - return result; -} - // The display UUID that a touchscreen HID service names for `displayID`, the XCTest display ID. // A simulator runtime that gives its displays no UUID answers nil, and so do its touchscreens. static NSString * _Nullable RunnerDisplayUUID(NSUInteger displayID) { @@ -86,39 +54,13 @@ static long RunnerServiceIntegerProperty( return nil; } -// Touches reach the app only with its display's touchscreen service as their sender. A display has -// one service per attached simulator client; the oldest, created at boot, is always present. static NSString * _Nullable RunnerResolveTouchscreenSender( const RunnerHIDFunctions *functions, CFTypeRef client, NSUInteger displayID, uint64_t *sender ) { - NSString *displayUUID = RunnerDisplayUUID(displayID); - CFArrayRef services = functions->copyServices(client); - NSUInteger matches = 0; - for (CFIndex index = 0; services != NULL && index < CFArrayGetCount(services); index++) { - CFTypeRef service = CFArrayGetValueAtIndex(services, index); - if (RunnerServiceIntegerProperty(functions, service, CFSTR("PrimaryUsagePage")) != RunnerDigitizerUsagePage || - RunnerServiceIntegerProperty(functions, service, CFSTR("PrimaryUsage")) != RunnerTouchScreenUsage || - ![RunnerServiceStringProperty(functions, service, CFSTR("Transport")) isEqualToString:@"CoreDevice"]) { - continue; - } - NSString *serviceDisplayUUID = RunnerServiceStringProperty(functions, service, CFSTR("displayUUID")); - if (serviceDisplayUUID != displayUUID && ![serviceDisplayUUID isEqualToString:displayUUID]) continue; - CFTypeRef registryID = functions->getServiceRegistryID(service); - uint64_t candidate = 0; - if (registryID == NULL || CFGetTypeID(registryID) != CFNumberGetTypeID() || - !CFNumberGetValue((CFNumberRef)registryID, kCFNumberSInt64Type, &candidate)) { - continue; - } - if (matches == 0 || candidate < *sender) *sender = candidate; - matches += 1; - } - if (services != NULL) CFRelease(services); - if (matches > 0) return nil; - return [NSString stringWithFormat:@"display %lu has no simulator touchscreen HID service", - (unsigned long)displayID]; + return RunnerSelectTouchscreenSender(functions, client, RunnerDisplayUUID(displayID), displayID, sender); } @implementation RunnerSimulatorEdgeTouch diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.h index a92caf8d07..ea7d2219a7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.h @@ -14,6 +14,15 @@ FOUNDATION_EXPORT NSArray *> *RunnerEdgeCon NSString * _Nullable * _Nullable failure ); +/// Runs the touchscreen sender selection over fake HID services. Each service is a dictionary of +/// its properties (`PrimaryUsagePage`, `PrimaryUsage`, `Transport`, `displayUUID`) plus +/// `registryID`; an absent key is a missing property. Returns the failure, or nil with `sender` set. +FOUNDATION_EXPORT NSString * _Nullable RunnerTouchscreenSelection( + NSArray *> *services, + NSString * _Nullable displayUUID, + uint64_t *sender +); + #endif NS_ASSUME_NONNULL_END diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m index 66503678fb..f83d3ea779 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m @@ -66,5 +66,31 @@ static void RunnerTestingDispatch(CFTypeRef client, RunnerHIDEventRef event) { return [RunnerTestingDispatched copy]; } +static CFArrayRef RunnerTestingCopyServices(CFTypeRef client) { + return CFBridgingRetain([(__bridge NSArray *)client copy]); +} + +static CFTypeRef RunnerTestingCopyServiceProperty(CFTypeRef service, CFStringRef key) { + id value = ((__bridge NSDictionary *)service)[(__bridge NSString *)key]; + return value == nil ? NULL : CFBridgingRetain(value); +} + +static CFTypeRef RunnerTestingGetServiceRegistryID(CFTypeRef service) { + return (__bridge CFTypeRef)((__bridge NSDictionary *)service)[@"registryID"]; +} + +NSString *RunnerTouchscreenSelection( + NSArray *> *services, + NSString *displayUUID, + uint64_t *sender +) { + RunnerHIDFunctions functions = { + .copyServices = RunnerTestingCopyServices, + .copyServiceProperty = RunnerTestingCopyServiceProperty, + .getServiceRegistryID = RunnerTestingGetServiceRegistryID, + }; + return RunnerSelectTouchscreenSender(&functions, (__bridge CFTypeRef)services, displayUUID, 1, sender); +} + #endif #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift index f6d7e1327d..8354dabed6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift @@ -42,5 +42,71 @@ extension RunnerTests { XCTAssertEqual(terminal?["restamped"]?.boolValue, true, "frame \(failingFrame): stamped at dispatch") } } + + func testTouchscreenWithoutATransportIsTheDisplaysOnlyCandidate() { + // Reported on iOS 18.1 and 26.5 simulators: the touchscreen service reads no Transport. + let selection = touchscreenSelection([touchscreen(registryID: 7, transport: nil)]) + + XCTAssertNil(selection.failure) + XCTAssertEqual(selection.sender, 7) + } + + func testCoreDeviceTouchscreenWinsOverAServiceWithoutTransport() { + let selection = touchscreenSelection([ + touchscreen(registryID: 3, transport: nil), + touchscreen(registryID: 9, transport: "CoreDevice"), + ]) + + XCTAssertNil(selection.failure) + XCTAssertEqual(selection.sender, 9) + } + + func testOldestCoreDeviceTouchscreenWinsWhenAWindowAddsAnother() { + let selection = touchscreenSelection([ + touchscreen(registryID: 20, transport: "CoreDevice"), + touchscreen(registryID: 10, transport: "CoreDevice"), + ]) + + XCTAssertNil(selection.failure) + XCTAssertEqual(selection.sender, 10) + } + + func testTouchscreensWithoutCoreDeviceAreAmbiguousWhenThereAreSeveral() { + let selection = touchscreenSelection([ + touchscreen(registryID: 3, transport: nil), + touchscreen(registryID: 4, transport: 42 as NSNumber), + ]) + + XCTAssertEqual(selection.sender, 0) + let failure = selection.failure ?? "" + XCTAssertTrue(failure.contains("2 touchscreen HID services without a CoreDevice transport"), failure) + XCTAssertTrue(failure.contains("missing: 1"), failure) + XCTAssertTrue(failure.contains("other type: 1"), failure) + } + + func testNoCandidateNamesTheConditionThatRejectedTheTouchscreens() { + let otherDisplay = touchscreenSelection([touchscreen(registryID: 5, transport: "CoreDevice", displayUUID: "D2")]) + let unreadableID = touchscreenSelection([touchscreen(registryID: "5" as NSString, transport: "CoreDevice")]) + + XCTAssertTrue(otherDisplay.failure?.contains("1 touchscreen service, 0 on this display") == true, otherDisplay.failure ?? "nil") + XCTAssertTrue(unreadableID.failure?.contains("1 on this display, 0 with a readable registry ID") == true, unreadableID.failure ?? "nil") + } + + private func touchscreen(registryID: Any, transport: Any?, displayUUID: String = "D1") -> [String: Any] { + var service: [String: Any] = [ + "PrimaryUsagePage": 13, + "PrimaryUsage": 4, + "displayUUID": displayUUID, + "registryID": registryID, + ] + if let transport { service["Transport"] = transport } + return service + } + + private func touchscreenSelection(_ services: [[String: Any]]) -> (failure: String?, sender: UInt64) { + var sender: UInt64 = 0 + let failure = RunnerTouchscreenSelection(services, "D1", &sender) + return (failure, sender) + } } #endif From 3b94f2eaeeeaffcae4f14c2dd06a4eac65b7c49c Mon Sep 17 00:00:00 2001 From: gronxb Date: Wed, 4 Nov 2026 22:04:23 +0900 Subject: [PATCH 11/12] fix(ios): use a touchscreen without Transport only as the display's sole one Address review: the fallback accepted a sole candidate whatever its Transport read. Only a missing Transport has been reported, so a touchscreen with a Transport of another value or type is no longer a candidate, and the display refuses to pick a sender while one is present without a CoreDevice touchscreen. The runner logs the selected service and its properties as AGENT_DEVICE_RUNNER_EDGE_TOUCHSCREEN so a host run shows which sender an edge touch used. --- .../RunnerSimulatorEdgeContact.h | 77 ++++++++++++++----- .../RunnerSimulatorEdgeTouch.m | 8 +- .../UnitTests/RunnerEdgeContactTesting.h | 6 +- .../UnitTests/RunnerEdgeContactTesting.m | 8 +- .../RunnerTests+ScreenEdgeGestureTests.swift | 57 +++++++++++--- 5 files changed, 124 insertions(+), 32 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeContact.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeContact.h index 522440aded..5857abda9c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeContact.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeContact.h @@ -269,19 +269,52 @@ static inline RunnerServicePropertyRead RunnerReadServiceStringProperty( return read; } +// One service property as the selection log shows it: its value, ``, or ``. +static inline NSString *RunnerDescribeServiceProperty( + const RunnerHIDFunctions *functions, + CFTypeRef service, + CFStringRef key +) { + CFTypeRef property = functions->copyServiceProperty(service, key); + if (property == NULL) return @""; + CFTypeID type = CFGetTypeID(property); + NSString *description = (type == CFStringGetTypeID() || type == CFNumberGetTypeID() || type == CFBooleanGetTypeID()) + ? [(__bridge id)property description] + : @""; + CFRelease(property); + return description; +} + +static inline NSString *RunnerDescribeTouchscreen( + const RunnerHIDFunctions *functions, + CFTypeRef service, + uint64_t registryID +) { + return [NSString stringWithFormat:@"registryID=%llu Transport=%@ Product=%@ Built-In=%@ displayUUID=%@", + registryID, + RunnerDescribeServiceProperty(functions, service, CFSTR("Transport")), + RunnerDescribeServiceProperty(functions, service, CFSTR("Product")), + RunnerDescribeServiceProperty(functions, service, CFSTR("Built-In")), + RunnerDescribeServiceProperty(functions, service, CFSTR("displayUUID"))]; +} + // Touches reach the app only with its display's touchscreen service as their sender. A candidate is // a touchscreen (digitizer usage page, touch screen usage) on the app's display with a readable // registry ID. A simulator client names its service's Transport `CoreDevice`, one service per // attached client; the oldest, created at boot, is always present and wins. Some simulator hosts -// name no Transport at all; their display's only candidate is its touchscreen, and several are -// ambiguous. +// name no Transport at all; without a CoreDevice candidate, the display's sole touchscreen without a +// Transport is used. Several such touchscreens are ambiguous, and a Transport of another value or +// type is not a known simulator touchscreen, so both refuse rather than pick a sender. +// `selection` describes the chosen service for the runner log. static inline NSString * _Nullable RunnerSelectTouchscreenSender( const RunnerHIDFunctions *functions, CFTypeRef client, NSString * _Nullable displayUUID, NSUInteger displayID, - uint64_t *sender + uint64_t *sender, + NSString * _Nullable __autoreleasing * _Nonnull selection ) { + *selection = nil; NSUInteger touchscreens = 0; NSUInteger onDisplay = 0; NSUInteger coreDevice = 0; @@ -289,7 +322,9 @@ static inline NSString * _Nullable RunnerSelectTouchscreenSender( NSUInteger transportOtherType = 0; NSUInteger transportOtherValue = 0; uint64_t oldestCoreDevice = 0; - uint64_t otherCandidate = 0; + uint64_t missingCandidate = 0; + NSString *oldestCoreDeviceDescription = nil; + NSString *missingDescription = nil; CFArrayRef services = functions->copyServices(client); for (CFIndex index = 0; services != NULL && index < CFArrayGetCount(services); index++) { CFTypeRef service = CFArrayGetValueAtIndex(services, index); @@ -310,38 +345,44 @@ static inline NSString * _Nullable RunnerSelectTouchscreenSender( NSString *transport = nil; switch (RunnerReadServiceStringProperty(functions, service, CFSTR("Transport"), &transport)) { case RunnerServicePropertyPresent: - if ([transport isEqualToString:@"CoreDevice"]) { - if (coreDevice == 0 || candidate < oldestCoreDevice) oldestCoreDevice = candidate; - coreDevice += 1; - continue; + if (![transport isEqualToString:@"CoreDevice"]) { + transportOtherValue += 1; + } else if (coreDevice++ == 0 || candidate < oldestCoreDevice) { + oldestCoreDevice = candidate; + oldestCoreDeviceDescription = RunnerDescribeTouchscreen(functions, service, candidate); } - transportOtherValue += 1; break; case RunnerServicePropertyMissing: transportMissing += 1; + missingCandidate = candidate; + missingDescription = RunnerDescribeTouchscreen(functions, service, candidate); break; case RunnerServicePropertyUnexpectedType: transportOtherType += 1; break; } - otherCandidate = candidate; } if (services != NULL) CFRelease(services); - NSUInteger others = transportMissing + transportOtherType + transportOtherValue; if (coreDevice > 0) { *sender = oldestCoreDevice; + *selection = oldestCoreDeviceDescription; return nil; } - if (others == 1) { - *sender = otherCandidate; + if (transportOtherType + transportOtherValue > 0) { + return [NSString stringWithFormat: + @"display %lu has touchscreen HID services with an unrecognized Transport (other type: %lu, " + @"other value: %lu) and none with a CoreDevice transport, so an edge touch cannot trust a sender", + (unsigned long)displayID, (unsigned long)transportOtherType, (unsigned long)transportOtherValue]; + } + if (transportMissing == 1) { + *sender = missingCandidate; + *selection = missingDescription; return nil; } - if (others > 1) { + if (transportMissing > 1) { return [NSString stringWithFormat: - @"display %lu has %lu touchscreen HID services without a CoreDevice transport " - @"(missing: %lu, other type: %lu, other value: %lu), so an edge touch cannot choose one", - (unsigned long)displayID, (unsigned long)others, (unsigned long)transportMissing, - (unsigned long)transportOtherType, (unsigned long)transportOtherValue]; + @"display %lu has %lu touchscreen HID services without a Transport, so an edge touch cannot choose one", + (unsigned long)displayID, (unsigned long)transportMissing]; } return [NSString stringWithFormat: @"display %lu has no simulator touchscreen HID service: %lu touchscreen service%@, %lu on this " diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m index e3677a3a6e..c7765479fe 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSimulatorEdgeTouch.m @@ -60,7 +60,13 @@ NSUInteger displayID, uint64_t *sender ) { - return RunnerSelectTouchscreenSender(functions, client, RunnerDisplayUUID(displayID), displayID, sender); + NSString *selection = nil; + NSString *failure = RunnerSelectTouchscreenSender( + functions, client, RunnerDisplayUUID(displayID), displayID, sender, &selection); + if (failure == nil) { + NSLog(@"AGENT_DEVICE_RUNNER_EDGE_TOUCHSCREEN display=%lu %@", (unsigned long)displayID, selection); + } + return failure; } @implementation RunnerSimulatorEdgeTouch diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.h index ea7d2219a7..65f9acdc82 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.h @@ -16,11 +16,13 @@ FOUNDATION_EXPORT NSArray *> *RunnerEdgeCon /// Runs the touchscreen sender selection over fake HID services. Each service is a dictionary of /// its properties (`PrimaryUsagePage`, `PrimaryUsage`, `Transport`, `displayUUID`) plus -/// `registryID`; an absent key is a missing property. Returns the failure, or nil with `sender` set. +/// `registryID`; an absent key is a missing property. Returns the failure, or nil with `sender` set +/// and `selection` describing the chosen service. FOUNDATION_EXPORT NSString * _Nullable RunnerTouchscreenSelection( NSArray *> *services, NSString * _Nullable displayUUID, - uint64_t *sender + uint64_t *sender, + NSString * _Nullable * _Nullable selection ); #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m index f83d3ea779..192654a0ef 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerEdgeContactTesting.m @@ -82,14 +82,18 @@ static CFTypeRef RunnerTestingGetServiceRegistryID(CFTypeRef service) { NSString *RunnerTouchscreenSelection( NSArray *> *services, NSString *displayUUID, - uint64_t *sender + uint64_t *sender, + NSString **selection ) { RunnerHIDFunctions functions = { .copyServices = RunnerTestingCopyServices, .copyServiceProperty = RunnerTestingCopyServiceProperty, .getServiceRegistryID = RunnerTestingGetServiceRegistryID, }; - return RunnerSelectTouchscreenSender(&functions, (__bridge CFTypeRef)services, displayUUID, 1, sender); + NSString *described = nil; + NSString *failure = RunnerSelectTouchscreenSender(&functions, (__bridge CFTypeRef)services, displayUUID, 1, sender, &described); + if (selection != NULL) *selection = described; + return failure; } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift index 8354dabed6..cd31d508e2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScreenEdgeGestureTests.swift @@ -49,6 +49,10 @@ extension RunnerTests { XCTAssertNil(selection.failure) XCTAssertEqual(selection.sender, 7) + XCTAssertEqual( + selection.description, + "registryID=7 Transport= Product= Built-In= displayUUID=D1" + ) } func testCoreDeviceTouchscreenWinsOverAServiceWithoutTransport() { @@ -71,17 +75,49 @@ extension RunnerTests { XCTAssertEqual(selection.sender, 10) } - func testTouchscreensWithoutCoreDeviceAreAmbiguousWhenThereAreSeveral() { + func testSeveralTouchscreensWithoutATransportAreAmbiguous() { let selection = touchscreenSelection([ touchscreen(registryID: 3, transport: nil), - touchscreen(registryID: 4, transport: 42 as NSNumber), + touchscreen(registryID: 4, transport: nil), ]) XCTAssertEqual(selection.sender, 0) - let failure = selection.failure ?? "" - XCTAssertTrue(failure.contains("2 touchscreen HID services without a CoreDevice transport"), failure) - XCTAssertTrue(failure.contains("missing: 1"), failure) - XCTAssertTrue(failure.contains("other type: 1"), failure) + XCTAssertEqual( + selection.failure, + "display 1 has 2 touchscreen HID services without a Transport, so an edge touch cannot choose one" + ) + } + + func testAnUnrecognizedTransportIsNeverChosenWithoutACoreDeviceTouchscreen() { + let otherType = touchscreenSelection([touchscreen(registryID: 4, transport: 42 as NSNumber)]) + let otherValue = touchscreenSelection([touchscreen(registryID: 4, transport: "USB")]) + let besideAMissingOne = touchscreenSelection([ + touchscreen(registryID: 3, transport: nil), + touchscreen(registryID: 4, transport: "USB"), + ]) + + for (selection, counts) in [ + (otherType, "other type: 1, other value: 0"), + (otherValue, "other type: 0, other value: 1"), + (besideAMissingOne, "other type: 0, other value: 1"), + ] { + XCTAssertEqual(selection.sender, 0, counts) + XCTAssertTrue( + selection.failure?.contains("unrecognized Transport (\(counts))") == true, + selection.failure ?? "nil" + ) + } + } + + func testCoreDeviceTouchscreenWinsBesideAnUnrecognizedTransport() { + let selection = touchscreenSelection([ + touchscreen(registryID: 4, transport: "USB"), + touchscreen(registryID: 9, transport: "CoreDevice"), + ]) + + XCTAssertNil(selection.failure) + XCTAssertEqual(selection.sender, 9) + XCTAssertTrue(selection.description?.hasPrefix("registryID=9 Transport=CoreDevice") == true) } func testNoCandidateNamesTheConditionThatRejectedTheTouchscreens() { @@ -103,10 +139,13 @@ extension RunnerTests { return service } - private func touchscreenSelection(_ services: [[String: Any]]) -> (failure: String?, sender: UInt64) { + private func touchscreenSelection( + _ services: [[String: Any]] + ) -> (failure: String?, sender: UInt64, description: String?) { var sender: UInt64 = 0 - let failure = RunnerTouchscreenSelection(services, "D1", &sender) - return (failure, sender) + var description: NSString? + let failure = RunnerTouchscreenSelection(services, "D1", &sender, &description) + return (failure, sender, description as String?) } } #endif From 7483e8d42a0e2bfa5a33105b51f6165610ba5850 Mon Sep 17 00:00:00 2001 From: gronxb Date: Thu, 5 Nov 2026 08:59:14 +0900 Subject: [PATCH 12/12] chore: retrigger CI