diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index 2cdc8ed63d..f9db080912 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -12,6 +12,29 @@ #import #import +// Mirrors the App Tracking Transparency prompt's real button order: the affirmative button is +// last (#3627). The answer label records which button the app actually took, so a test asserts +// the app's answer instead of the command's own report. `--agent-device-alert-affirmative-only` +// shows the same alert with no decline button at all. +static UIAlertController *RunnerAlertDeclineChoiceFixture(BOOL affirmativeOnly, UILabel *answer) { + UIAlertController *alert = [UIAlertController + alertControllerWithTitle:@"“Runner” Would Like to Track Your Activity" + message:@"Track your activity across other companies' apps and websites." + preferredStyle:UIAlertControllerStyleAlert]; + NSArray *buttons = affirmativeOnly + ? @[ @[@"Allow", @"Authorized"], @[@"OK", @"Authorized"] ] + : @[ @[@"Ask App Not to Track", @"Declined"], @[@"Allow", @"Authorized"] ]; + for (NSArray *button in buttons) { + [alert addAction:[UIAlertAction actionWithTitle:button[0] + style:UIAlertActionStyleDefault + handler:^(UIAlertAction *action) { + (void)action; + answer.text = button[1]; + }]]; + } + return alert; +} + // A plain text field whose accessibility value is the masked rendering of its text. This is how // a React Native `secureTextEntry` input reaches XCTest: `.textField`, not `.secureTextField`, // with a value of bullets (tester-army/e2e#872). @@ -354,7 +377,7 @@ - (void)presentAlertFixtureReplacement:(BOOL)replacement { for (NSString *buttonTitle in @[@"Cancel", @"OK"]) { UIAlertActionStyle style = [buttonTitle isEqualToString:@"Cancel"] ? UIAlertActionStyleCancel : UIAlertActionStyleDefault; - [alert addAction:[UIAlertAction actionWithTitle:buttonTitle style:style handler:^(UIAlertAction *action) { + UIAlertAction *button = [UIAlertAction actionWithTitle:buttonTitle style:style handler:^(UIAlertAction *action) { (void)action; if (!replacement) { self.alertActivationBusyAnswer.text = self.alertActivationBusyBackstop != nil @@ -373,33 +396,17 @@ - (void)presentAlertFixtureReplacement:(BOOL)replacement { [self presentAlertFixtureReplacement:YES]; }]; } - }]]; + }]; + button.enabled = ![arguments containsObject:@"--agent-device-alert-replacement-disabled-buttons"]; + [alert addAction:button]; } [self presentViewController:alert animated:NO completion:nil]; } -// Mirrors the App Tracking Transparency prompt's real button order: the affirmative button is -// last (#3627). The answer label records which button the app actually took, so a test asserts -// the app's answer instead of the command's own report. `--agent-device-alert-affirmative-only` -// shows the same alert with no decline button at all. - (void)presentAlertDeclineChoiceFixture { BOOL affirmativeOnly = [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-affirmative-only"]; - UIAlertController *alert = [UIAlertController - alertControllerWithTitle:@"“Runner” Would Like to Track Your Activity" - message:@"Track your activity across other companies' apps and websites." - preferredStyle:UIAlertControllerStyleAlert]; - NSArray *buttons = affirmativeOnly - ? @[ @[@"Allow", @"Authorized"], @[@"OK", @"Authorized"] ] - : @[ @[@"Ask App Not to Track", @"Declined"], @[@"Allow", @"Authorized"] ]; - for (NSArray *button in buttons) { - [alert addAction:[UIAlertAction actionWithTitle:button[0] - style:UIAlertActionStyleDefault - handler:^(UIAlertAction *action) { - (void)action; - self.alertDeclineAnswer.text = button[1]; - }]]; - } + UIAlertController *alert = RunnerAlertDeclineChoiceFixture(affirmativeOnly, self.alertDeclineAnswer); [self presentViewController:alert animated:NO completion:nil]; } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertVerification.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertVerification.swift index bdff3215c5..f9567599b2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertVerification.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertVerification.swift @@ -10,17 +10,16 @@ enum RunnerAlertObservation { enum RunnerAlertVerification: Equatable { case disappeared case presentationChanged - case stillVisible case unconfirmed case timedOut static func verify( original: AlertSurfacePresentation, observation: RunnerAlertObservation - ) -> RunnerAlertVerification { + ) -> RunnerAlertVerification? { switch observation { case .visible(let current): - return original == current ? .stillVisible : .presentationChanged + return original == current ? nil : .presentationChanged case .absent: return .disappeared case .unavailable: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index 16d6ca5d77..decf0f6b05 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -5,7 +5,7 @@ extension RunnerTests { struct RunnerAlert { let root: XCUIElement let ownerApp: XCUIApplication - let buttons: [XCUIElement] + let actions: [XCUIElement] let source: AlertSurfaceSource /// The window hosting a SpringBoard modal; nil routes through `ownerApp`. let window: XCUIElement? @@ -13,8 +13,9 @@ extension RunnerTests { enum AlertResolution { case absent - /// A blocking system modal is on screen but its remote host could not be read. + /// Required alert acquisition did not establish presence or absence. case unresolved + case notActionable case resolved(RunnerAlert) } @@ -26,29 +27,13 @@ extension RunnerTests { "No alert button was activated. Inspect the current alert before deciding on an action." static let alertUnresolvedHint = - "A system prompt is visible but alert could not read it. Inspect it with screenshot, then snapshot -i, and press @ref on the visible button." + "Alert could not finish reading the current prompt state. Inspect it with screenshot, then snapshot -i, and press @ref on the visible button." static func alertCommandTimeout(timeoutMs: Double?) -> TimeInterval { guard let timeoutMs, timeoutMs.isFinite else { return defaultAlertCommandTimeout } return max(0.001, timeoutMs / 1000) } - /// The alert answer for a blocking system modal. Nil means no modal is on screen, so the active - /// app's own alert surfaces are read next. - static func alertResolution( - forSystemModal resolution: BlockingSystemModalResolution, - resolved: (ResolvedBlockingSystemModal) -> AlertResolution - ) -> AlertResolution? { - switch resolution { - case .resolved(let modal): - return resolved(modal) - case .unresolved: - return .unresolved - case .absent: - return nil - } - } - @MainActor func resolveAlert(app activeApp: XCUIApplication, deadline: Date) -> AlertResolution { #if AGENT_DEVICE_RUNNER_UNIT_TESTS @@ -56,48 +41,56 @@ extension RunnerTests { return override(deadline) } #endif + do { #if !os(macOS) - if let resolution = Self.alertResolution( - forSystemModal: resolveBlockingSystemModal(deadline: deadline), - resolved: { modal in - runnerAlert( - root: modal.root, - ownerApp: modal.ownerApp, - actions: modal.actions, - source: .blockingSystemModal, - window: modal.window - ) + let systemResolution = resolveBlockingSystemModal(deadline: deadline) + switch systemResolution { + case .absent: break + case .resolved, .unresolved, .notActionable: return systemResolution } - ) { - return resolution - } #endif - let state = activeApp.state - guard state != .notRunning && state != .unknown else { + let state = try readModalValue(deadline: deadline) { activeApp.state } + guard state != .unknown else { return .unresolved } + guard state != .notRunning else { return .absent } + if let alert = try firstExistingElement(in: readModalElements(deadline: deadline) { + activeApp.alerts.allElementsBoundByIndex + }, deadline: deadline) { + return try resolveAppAlert(root: alert, ownerApp: activeApp, source: .appAlert, deadline: deadline) + } + if let dialog = try firstExistingElement(in: readModalElements(deadline: deadline) { + activeApp.otherElements.matching(identifier: AlertSurface.safariDialogIdentifier).allElementsBoundByIndex + }, deadline: deadline) { + return try resolveAppAlert(root: dialog, ownerApp: activeApp, source: .safariDialog, deadline: deadline) + } + if let popup = try firstDismissPopupWindow(in: activeApp, deadline: deadline) { + return try resolveAppAlert(root: popup, ownerApp: activeApp, source: .dismissPopup, deadline: deadline) + } return .absent + } catch { + return .unresolved } - // Guard the query: when a remote-hosted modal (e.g. the AccessorySetupKit - // picker) was just dismissed, the dismissal re-check re-enters here with the - // now-gone host as `activeApp`, and its `alerts` query raises - // kAXErrorServerNotFound. safeElementsQuery absorbs it and reports no alert. - if let alert = firstExistingElement(in: safeElementsQuery { activeApp.alerts.allElementsBoundByIndex }) { - return runnerAlert(root: alert, ownerApp: activeApp, actions: actionableElements(in: alert), source: .appAlert) - } - if let dialog = firstExistingElement(in: safeElementsQuery { - activeApp.otherElements.matching(identifier: AlertSurface.safariDialogIdentifier).allElementsBoundByIndex - }) { - return runnerAlert(root: dialog, ownerApp: activeApp, actions: actionableElements(in: dialog), source: .safariDialog) - } - if let popup = firstDismissPopupWindow(in: activeApp) { - return runnerAlert(root: popup, ownerApp: activeApp, actions: actionableElements(in: popup), source: .dismissPopup) - } - return .absent + } + + private func resolveAppAlert( + root: XCUIElement, ownerApp: XCUIApplication, source: AlertSurfaceSource, deadline: Date + ) throws -> AlertResolution { + let actions = try readModalActions(in: root, deadline: deadline) + guard !actions.isEmpty else { return .notActionable } + return .resolved(RunnerAlert(root: root, ownerApp: ownerApp, actions: actions, source: source, window: nil)) } @MainActor func handleAlert(_ alert: RunnerAlert, action: String, deadline: Date) throws -> Response { if action == "accept" || action == "dismiss" { - guard let button = chooseAlertButton(alert, action: action) else { + let selectedButton: XCUIElement? + do { + selectedButton = try chooseAlertButton(alert, action: action, deadline: deadline) + } catch ModalReadFailure.deadlineExceeded { + return alertVerificationResponse(.timedOut, action: action, activated: false) + } catch { + return alertUnresolvedResponse() + } + guard let button = selectedButton else { // The choice refused before any activation, so this discloses dispatched: no like the // runner's other pre-run refusals. return Response(ok: false, error: ErrorPayload( @@ -149,7 +142,7 @@ extension RunnerTests { } while true { sleepFor(min(0.2, max(0, deadline.timeIntervalSinceNow))) - let verification = RunnerAlertVerification.verify( + guard let verification = RunnerAlertVerification.verify( original: original, observation: observeAlert( in: alert.ownerApp, @@ -157,14 +150,20 @@ extension RunnerTests { observesApplicationRoot: observesApplicationRoot, deadline: deadline ) - ) - if verification == .stillVisible { continue } + ) else { continue } return alertVerificationResponse(verification, action: action, activated: true) } } - let title = preferredAlertTitle(alert.root, buttons: alert.buttons) - let items = alert.buttons.map { $0.label.trimmingCharacters(in: .whitespacesAndNewlines) } + let items: [String] + do { + items = try readModalValue(deadline: deadline) { + alert.actions.map { $0.label.trimmingCharacters(in: .whitespacesAndNewlines) } + } + } catch { + return alertUnresolvedResponse() + } + let title = safely("ALERT_TITLE", "Alert") { preferredAlertTitle(alert.root, buttons: alert.actions) } return Response( ok: true, data: DataPayload( @@ -283,32 +282,28 @@ extension RunnerTests { operation() } - private func runnerAlert( - root: XCUIElement, - ownerApp: XCUIApplication, - actions: [XCUIElement], - source: AlertSurfaceSource, - window: XCUIElement? = nil - ) -> AlertResolution { - let buttons = actions.filter { isEnabledElement($0) } - guard !buttons.isEmpty else { - return .absent - } - return .resolved(RunnerAlert(root: root, ownerApp: ownerApp, buttons: buttons, source: source, window: window)) + func alertUnresolvedResponse() -> Response { + Response(ok: false, error: ErrorPayload( + code: RunnerWireErrorCode.alertUnresolved, + message: "alert could not finish reading the current prompt state", + hint: Self.alertUnresolvedHint + )) } - private func firstExistingElement(in elements: [XCUIElement]) -> XCUIElement? { - elements.first { isVisibleElement($0) } + private func firstExistingElement(in elements: [XCUIElement], deadline: Date) throws -> XCUIElement? { + try elements.first { element in + try readModalValue(deadline: deadline) { isVisibleElement(element) } + } } /// The marker is matched inside XCTest's query, so the screen is read once per query. Reading each /// descendant instead costs one round trip per element, and on a screen whose tree changes while /// it is read (a loading web view) each vanished element adds XCTest's retry cycle. `containing` /// also matches a window that is itself the marker. - private func firstDismissPopupWindow(in app: XCUIApplication) -> XCUIElement? { - firstExistingElement(in: safeElementsQuery { + private func firstDismissPopupWindow(in app: XCUIApplication, deadline: Date) throws -> XCUIElement? { + try firstExistingElement(in: readModalElements(deadline: deadline) { app.windows.containing(Self.dismissPopupMarker).allElementsBoundByIndex - }) + }, deadline: deadline) } private static let dismissPopupMarker = NSPredicate( @@ -319,16 +314,20 @@ extension RunnerTests { // Internal, not private: the dismiss-popup wiring is pinned by // RunnerTests+AlertTests, and the label rule itself by RunnerAlertButtonChoice. - func chooseAlertButton(_ alert: RunnerAlert, action: String) -> XCUIElement? { + func chooseAlertButton(_ alert: RunnerAlert, action: String, deadline: Date) throws -> XCUIElement? { + let buttons = try readModalValue(deadline: deadline) { + alert.actions.filter { $0.exists && $0.isEnabled } + } + let labels = try readModalValue(deadline: deadline) { buttons.map(\.label) } guard let index = RunnerAlertButtonChoice.buttonIndex( - for: alert.buttons.map(\.label), + for: labels, action: action, popupMarkerSurface: alert.source == .dismissPopup, vocabulary: .system ) else { return nil } - return alert.buttons[index] + return buttons[index] } /// The alert's own title: the first non-empty static text inside the alert that is neither a @@ -384,10 +383,6 @@ extension RunnerTests { element.exists && !element.frame.isNull && !element.frame.isEmpty } - private func isEnabledElement(_ element: XCUIElement) -> Bool { - safely("ALERT_BUTTON_ENABLED", false) { element.exists && element.isEnabled } - } - // A snapshot can expose an alert's button a beat before the owning app has made it // hittable, and a starved host delays the app's layout and hit-testing further. This // activation is not repeated, so a tap issued into that window is dropped, the diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift index af1faa89cd..b29fb00e10 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift @@ -82,9 +82,6 @@ extension RunnerTests { case .timedOut: code = "ALERT_DEADLINE_EXCEEDED" message = "alert \(action) exhausted its deadline" - case .stillVisible: - code = "INTERACTION_FAILED" - message = "alert \(action) still observes an unchanged alert presentation" case .unconfirmed: code = "ALERT_CONFIRMATION_UNAVAILABLE" message = "alert \(action) could not read the alert presentation" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift index c3a8bdb701..542ad00733 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift @@ -1,52 +1,28 @@ import XCTest +import AgentDeviceSnapshotPresentation extension RunnerTests { - struct ResolvedBlockingSystemModal { - let root: XCUIElement - let ownerApp: XCUIApplication - let actions: [XCUIElement] - /// The SpringBoard window hosting `root`; nil for a remote host queried as a whole app. - let window: XCUIElement? - } - - enum BlockingSystemModalResolution { - case absent - case unresolved - case resolved(ResolvedBlockingSystemModal) - } - - func resolveBlockingSystemModal( - deadline: Date - ) -> BlockingSystemModalResolution { - guard Self.hasSpringBoardSystemModalHost else { - return .absent - } - - guard let springboardModal = firstBlockingSystemModal( - in: springboard, - deadline: deadline - ) else { - return .absent - } - - let springboardActions = actionableElements(in: springboardModal.root) - if !RemoteHostedSystemModalPolicy.shouldProbeRemoteHost( - springboardActionCount: springboardActions.count - ) { - return .resolved( - ResolvedBlockingSystemModal( - root: springboardModal.root, - ownerApp: springboard, - actions: springboardActions, - window: springboardModal.window - ) - ) - } - - guard Date() < deadline, let remoteModal = remoteHostedSystemModal(deadline: deadline) else { + func resolveBlockingSystemModal(deadline: Date) -> AlertResolution { + guard Self.hasSpringBoardSystemModalHost else { return .absent } + do { + guard let modal = try firstBlockingSystemModal(in: springboard, deadline: deadline) else { + return .absent + } + let actions = try readModalActions(in: modal.root, deadline: deadline) + if RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: actions.count) { + guard let remote = try remoteHostedSystemModal(deadline: deadline) else { return .unresolved } + return .resolved(remote) + } + return .resolved(RunnerAlert( + root: modal.root, ownerApp: springboard, actions: actions, + source: .blockingSystemModal, window: modal.window + )) + } catch { + if case ModalReadFailure.deadlineExceeded = error { + NSLog("AGENT_DEVICE_RUNNER_SYSTEM_MODAL_PROBE_DEADLINE stage=required-read") + } return .unresolved } - return .resolved(remoteModal) } // AccessorySetupUI mirrors into SpringBoard, but its mirrored elements are not @@ -55,19 +31,18 @@ extension RunnerTests { "com.apple.AccessorySetupUI" ] - private func remoteHostedSystemModal(deadline: Date) -> ResolvedBlockingSystemModal? { + private func remoteHostedSystemModal(deadline: Date) throws -> RunnerAlert? { for bundleId in Self.remoteSystemModalHostBundleIds { - guard Date() < deadline else { return nil } let host = XCUIApplication(bundleIdentifier: bundleId) - let state = safely("REMOTE_MODAL_STATE", XCUIApplication.State.unknown) { host.state } + let state = try readModalValue(deadline: deadline) { host.state } guard RemoteHostedSystemModalPolicy.isEligibleHostState(state) else { continue } - guard Date() < deadline else { return nil } - // A dismissed remote host can raise kAXErrorServerNotFound. The safe queries - // inside actionableElements turn that disappearance into an empty result. - let actions = actionableElements(in: host) + let actions = try readModalActions(in: host, deadline: deadline) guard !actions.isEmpty else { continue } - return ResolvedBlockingSystemModal(root: host, ownerApp: host, actions: actions, window: nil) + return RunnerAlert( + root: host, ownerApp: host, actions: actions, + source: .blockingSystemModal, window: nil + ) } return nil } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 4bc64ae5f1..6ef1f29369 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -645,15 +645,8 @@ extension RunnerTests { ok: false, error: ErrorPayload(code: "ALERT_NOT_FOUND", message: "alert not found") ) - case .unresolved: - return Response( - ok: false, - error: ErrorPayload( - code: RunnerWireErrorCode.alertUnresolved, - message: "a system prompt is visible but its buttons could not be read", - hint: RunnerTests.alertUnresolvedHint - ) - ) + case .unresolved, .notActionable: + return alertUnresolvedResponse() } case .gesture: guard let plan = command.gesturePlan else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift index daba0f64f9..7cd0205668 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift @@ -16,7 +16,7 @@ extension RunnerTests { } func snapshotForBlockingSystemModal( - _ resolvedModal: ResolvedBlockingSystemModal, + _ resolvedModal: RunnerAlert, options: PresentationOptions = PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), deadline: Date = .distantFuture ) throws -> DataPayload? { @@ -113,36 +113,28 @@ extension RunnerTests { func firstBlockingSystemModal( in springboard: XCUIApplication, deadline: Date = .distantFuture - ) -> SpringBoardModal? { - if let alert = firstBlockingSystemModal(in: springboard, matching: .alert, deadline: deadline) { + ) throws -> SpringBoardModal? { + if let alert = try firstBlockingSystemModal(in: springboard, matching: .alert, deadline: deadline) { return alert } - // Don't start the second (sheet) enumeration once the shared probe deadline is spent (#1244). - if Date() >= deadline { - NSLog("AGENT_DEVICE_RUNNER_SYSTEM_MODAL_PROBE_DEADLINE stage=sheets") - return nil - } - - return firstBlockingSystemModal(in: springboard, matching: .sheet, deadline: deadline) + return try firstBlockingSystemModal(in: springboard, matching: .sheet, deadline: deadline) } private func firstBlockingSystemModal( in springboard: XCUIApplication, matching type: XCUIElement.ElementType, deadline: Date - ) -> SpringBoardModal? { - let windows = safeElementsQuery { + ) throws -> SpringBoardModal? { + let windows = try readModalElements(deadline: deadline) { springboard.windows.containing(type, identifier: nil).allElementsBoundByIndex } for window in windows { - guard Date() < deadline else { return nil } - let modals = safeElementsQuery { + let modals = try readModalElements(deadline: deadline) { window.descendants(matching: type).allElementsBoundByIndex } for modal in modals { - guard Date() < deadline else { return nil } - if safeIsBlockingSystemModal(modal, in: window) { + if try readModalValue(deadline: deadline, { isBlockingSystemModal(modal, in: window) }) { return SpringBoardModal(root: modal, window: window) } } @@ -150,41 +142,57 @@ extension RunnerTests { return nil } - func safeElementsQuery(_ fetch: () -> [XCUIElement]) -> [XCUIElement] { - safely("MODAL_QUERY", [], fetch) + enum ModalReadFailure: Error { + case queryFailed + case deadlineExceeded } - private func safeIsBlockingSystemModal(_ element: XCUIElement, in window: XCUIElement) -> Bool { - safely("MODAL_CHECK", false) { isBlockingSystemModal(element, in: window) } + func readModalValue(deadline: Date, _ fetch: () -> T) throws -> T { + guard Date() < deadline else { throw ModalReadFailure.deadlineExceeded } + let (result, exception) = catchingObjCException(fallback: nil as T?) { fetch() } + guard exception == nil, let result else { throw ModalReadFailure.queryFailed } + guard Date() < deadline else { throw ModalReadFailure.deadlineExceeded } + return result + } + + func readModalElements(deadline: Date, _ fetch: () -> [XCUIElement]) throws -> [XCUIElement] { + try readModalValue(deadline: deadline) { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + if let override = modalElementsQueryOverrideForTesting { return override() } +#endif + return fetch() + } + } + + func safeElementsQuery(_ fetch: () -> [XCUIElement]) -> [XCUIElement] { + safely("MODAL_QUERY", [], fetch) } private func isBlockingSystemModal(_ element: XCUIElement, in window: XCUIElement) -> Bool { element.exists && AlertSurface.isPlaced(element.frame, in: window.frame) } - func actionableElements(in element: XCUIElement) -> [XCUIElement] { + func readModalActions(in element: XCUIElement, deadline: Date) throws -> [XCUIElement] { var seen = Set() var actions: [XCUIElement] = [] - let descendants = actionableTypes.flatMap { modalDescendants(in: element, matching: $0) } - for candidate in descendants { - if !safeIsActionableCandidate(candidate, seen: &seen) { continue } - actions.append(candidate) + for type in actionableTypes { + let descendants = try readModalElements(deadline: deadline) { + element.descendants(matching: type).allElementsBoundByIndex + } + for candidate in descendants { + let actionable = try readModalValue(deadline: deadline) { + if !candidate.exists || !candidate.isHittable { return false } + let frame = candidate.frame + if frame.isNull || frame.isEmpty { return false } + let key = "\(candidate.elementType.rawValue)-\(frame.origin.x)-\(frame.origin.y)-\(frame.size.width)-\(frame.size.height)-\(candidate.label)" + return seen.insert(key).inserted + } + if actionable { actions.append(candidate) } + } } return actions } - private func safeIsActionableCandidate(_ candidate: XCUIElement, seen: inout Set) -> Bool { - safely("MODAL_ACTION", false) { - if !candidate.exists || !candidate.isHittable { return false } - let frame = candidate.frame - if frame.isNull || frame.isEmpty { return false } - let key = "\(candidate.elementType.rawValue)-\(frame.origin.x)-\(frame.origin.y)-\(frame.size.width)-\(frame.size.height)-\(candidate.label)" - if seen.contains(key) { return false } - seen.insert(key) - return true - } - } - private func informativeElements(in element: XCUIElement, deadline: Date) -> [XCUIElement] { var seen = Set() var contents: [XCUIElement] = [] diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 2666ccccc5..edfafba624 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -38,8 +38,8 @@ final class RunnerTests: XCTestCase { /// channel, so the runner cannot tell whether an input is there. /// `ALERT_BUTTON_NOT_FOUND` is the alert button choice refusing before any activation, so it /// discloses dispatched: no. - /// `ALERT_UNRESOLVED` is a visible blocking system modal whose host could not be read: a prompt - /// is on screen, so it must never read as absence. It activates nothing, so dispatched: no. + /// `ALERT_UNRESOLVED` means required alert acquisition failed or exhausted its deadline. + /// It establishes no absence and activates nothing, so dispatched: no. enum RunnerWireErrorCode { static let mainThreadTimeout = "MAIN_THREAD_TIMEOUT" static let appNotRunning = "APP_NOT_RUNNING" @@ -245,6 +245,7 @@ final class RunnerTests: XCTestCase { /// out-of-process sheet the tier cannot read. Production builds compile none of this. var privateAXAcquisitionOverrideForTesting: (() -> SnapshotAcquisition?)? var blockingSystemModalPresenceOverrideForTesting: Bool? + var modalElementsQueryOverrideForTesting: (() -> [XCUIElement])? var alertResolutionOverrideForTesting: (@MainActor (Date) -> AlertResolution)? var alertButtonHittabilityProbeOverrideForTesting: (@MainActor (Date) -> Bool)? // Unit-test-only seam for the in-place surface probe (`presentedSystemSurfaceHost`): a registered diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 4e832de878..9582c348e9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -272,7 +272,7 @@ extension RunnerTests { let held = holdingRecordedIssues { do { _ = try activateAlertButton( - RunnerAlert(root: app, ownerApp: app, buttons: [app], source: .appAlert, window: nil), + RunnerAlert(root: app, ownerApp: app, actions: [app], source: .appAlert, window: nil), button: app, action: "accept", frame: CGRect(x: 10, y: 10, width: 40, height: 40), deadline: .distantFuture ) } catch { alertRefusal = error } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift index 380c8c9199..84be886374 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift @@ -168,7 +168,7 @@ extension RunnerTests { // Resolution is a pure read; a contended hosted simulator can spend far more than the 10 s // command timeout on it, so it runs under the sibling fixtures' resolution allowance. let alert = try requireResolvedAlert(within: RunnerTests.alertResolutionAllowance) - XCTAssertEqual(alert.buttons.map(\.label).last, "Allow", "the fixture keeps the affirmative button last") + XCTAssertEqual(alert.actions.map(\.label).last, "Allow", "the fixture keeps the affirmative button last") let response = try handleAlert( alert, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift index de2db7af2c..1c43da4740 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift @@ -82,7 +82,7 @@ extension RunnerTests { func testAlertActivationAfterDeadlineDoesNotTapTheOriginal() throws { defer { tearDownReplacementAlert() } let alert = try launchReplacementAlert() - let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" }) + let button = try XCTUnwrap(alert.actions.first { $0.label == "OK" }) let frame = button.frame let outcome = try activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantPast) @@ -99,11 +99,11 @@ extension RunnerTests { let alert = RunnerAlert( root: resolved.root, ownerApp: resolved.ownerApp, - buttons: resolved.buttons, + actions: resolved.actions, source: resolved.source, window: app.windows.matching(identifier: "agent-device-missing-window").firstMatch ) - let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" }) + let button = try XCTUnwrap(alert.actions.first { $0.label == "OK" }) let frame = button.frame let options = XCTExpectedFailure.Options() @@ -123,7 +123,7 @@ extension RunnerTests { func testAlertDeadlineDuringSynthesisPreparationLeavesTheOriginalUntouched() throws { defer { tearDownReplacementAlert() } let alert = try launchReplacementAlert() - let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" }) + let button = try XCTUnwrap(alert.actions.first { $0.label == "OK" }) let frame = button.frame try withAlertTapStub { let outcome = try activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) @@ -190,7 +190,7 @@ extension RunnerTests { @MainActor private func assertAlertActivationDisablesInterruptionHandling(_ alert: RunnerAlert) throws { - let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" }) + let button = try XCTUnwrap(alert.actions.first { $0.label == "OK" }) let frame = button.frame try withAlertTapStub { AlertTapDeadlineStub.status = .deadlineExceeded diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift index cbdefc8023..aa5f918152 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift @@ -54,7 +54,7 @@ extension RunnerTests { let alert = try requireResolvedAlert(within: RunnerTests.defaultAlertCommandTimeout) XCTAssertEqual(alert.source, .dismissPopup) XCTAssertEqual(alert.root.elementType, .window) - XCTAssertTrue(alert.buttons.contains { $0.identifier == " Dismiss Popup " }) + XCTAssertTrue(alert.actions.contains { $0.identifier == " Dismiss Popup " }) } @MainActor @@ -65,7 +65,7 @@ extension RunnerTests { let alert = try requireResolvedAlert(within: RunnerTests.defaultAlertCommandTimeout) XCTAssertEqual(alert.source, .dismissPopup) XCTAssertEqual(alert.root.identifier, "Dismiss popup") - XCTAssertTrue(alert.buttons.contains { $0.identifier == "agent-device-close-popover" }) + XCTAssertTrue(alert.actions.contains { $0.identifier == "agent-device-close-popover" }) } /// `alert dismiss` on the marker window must still name its close button: the marker is the @@ -77,7 +77,9 @@ extension RunnerTests { defer { terminateCrowdedScreen() } let alert = try requireResolvedAlert(within: RunnerTests.defaultAlertCommandTimeout) - let button = try XCTUnwrap(chooseAlertButton(alert, action: "dismiss")) + let button = try XCTUnwrap(try chooseAlertButton( + alert, action: "dismiss", deadline: Date().addingTimeInterval(Self.defaultAlertCommandTimeout) + )) XCTAssertEqual(button.identifier, "agent-device-close-popover") } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertVerificationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertVerificationTests.swift index 10cb4f37f0..5af1988099 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertVerificationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertVerificationTests.swift @@ -29,7 +29,7 @@ extension RunnerTests { let original = AlertSurfacePresentation(title: title, content: [], buttons: ["OK"]) XCTAssertEqual( RunnerAlertVerification.verify(original: original, observation: .visible(original)), - .stillVisible + nil ) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift index 435e3213ca..2f2e101bde 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift @@ -1,4 +1,5 @@ import XCTest +import AgentDeviceSnapshotPresentation #if AGENT_DEVICE_RUNNER_UNIT_TESTS extension RunnerTests { @@ -15,26 +16,185 @@ extension RunnerTests { XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.unknown)) } - /// An unreadable modal is still a prompt on screen: answering it as absent makes the host retry - /// and report `ALERT_NOT_FOUND` while the prompt blocks the app. - func testSystemModalAlertResolutionKeepsAnUnreadableModalUnresolved() { - var resolvedModalCount = 0 - let resolveModal: (ResolvedBlockingSystemModal) -> AlertResolution = { _ in - resolvedModalCount += 1 - return .absent + func testRequiredModalReadPreservesSuccessfulEmptyAndOptionalValues() throws { + let elements: [XCUIElement] = try readModalValue(deadline: .distantFuture) { [] } + XCTAssertTrue(elements.isEmpty) + let element: XCUIElement? = try readModalValue(deadline: .distantFuture) { nil as XCUIElement? } + XCTAssertNil(element) + let visible = try readModalValue(deadline: .distantFuture) { false } + XCTAssertFalse(visible) + XCTAssertNoThrow(try readModalValue(deadline: .distantFuture) { () }) + } + +#if os(iOS) + @MainActor + func testAlertRequiredQueryFailureNeverReportsAbsence() throws { + modalElementsQueryOverrideForTesting = { + NSException(name: .internalInconsistencyException, reason: "injected required query failure", userInfo: nil).raise() + return [] + } + defer { modalElementsQueryOverrideForTesting = nil } + XCTAssertThrowsError(try firstBlockingSystemModal(in: springboard)) { error in + guard case ModalReadFailure.queryFailed = error else { return XCTFail("unexpected error: \(error)") } + } + for action in ["get", "accept", "wait"] { + let command = try runnerCommandFixture( + #"{"command":"alert","commandId":"required-query-\#(action)","action":"\#(action)"}"# + ) + let response = try executeOnMainPrepared(command: command, activeApp: app) + XCTAssertEqual(response.error?.code, RunnerWireErrorCode.alertUnresolved, action) } + } - guard case .unresolved = Self.alertResolution(forSystemModal: .unresolved, resolved: resolveModal) else { - return XCTFail("an unreadable system modal must resolve as unresolved") + @MainActor + func testAlertSpentSearchDeadlineNeverReportsAbsence() throws { + let deadline = Date.distantPast + XCTAssertThrowsError(try firstBlockingSystemModal(in: springboard, deadline: deadline)) { error in + guard case ModalReadFailure.deadlineExceeded = error else { return XCTFail("unexpected error: \(error)") } } - XCTAssertNil(Self.alertResolution(forSystemModal: .absent, resolved: resolveModal)) - let modal = ResolvedBlockingSystemModal(root: app, ownerApp: app, actions: [], window: nil) - guard case .absent = Self.alertResolution(forSystemModal: .resolved(modal), resolved: resolveModal) else { - return XCTFail("a readable system modal must answer with its own resolution") + for action in ["get", "accept", "wait"] { + let command = try runnerCommandFixture( + #"{"command":"alert","commandId":"spent-search-\#(action)","action":"\#(action)"}"# + ) + let response = try executeOnMainPrepared(command: command, activeApp: app, alertDeadline: deadline) + XCTAssertEqual(response.error?.code, RunnerWireErrorCode.alertUnresolved, action) } - XCTAssertEqual(resolvedModalCount, 1) } + @MainActor + func testAlertSearchFinishingAfterDeadlineNeverReportsAbsence() { + modalElementsQueryOverrideForTesting = { + Thread.sleep(forTimeInterval: 0.02) + return [] + } + defer { modalElementsQueryOverrideForTesting = nil } + guard case .unresolved = resolveBlockingSystemModal(deadline: Date().addingTimeInterval(0.01)) else { + return XCTFail("an empty query that finished too late cannot establish absence") + } + } + + @MainActor + func testAlertSuccessfulEmptySearchIsAbsent() { + modalElementsQueryOverrideForTesting = { [] } + defer { modalElementsQueryOverrideForTesting = nil } + guard case .absent = resolveBlockingSystemModal(deadline: .distantFuture) else { + return XCTFail("a completed empty search establishes absence") + } + } + + @MainActor + func testAlertRequiredActionsQueryFailureThrows() { + modalElementsQueryOverrideForTesting = { + NSException(name: .internalInconsistencyException, reason: "injected actions query failure", userInfo: nil).raise() + return [] + } + defer { modalElementsQueryOverrideForTesting = nil } + XCTAssertThrowsError(try readModalActions(in: app, deadline: .distantFuture)) { error in + guard case ModalReadFailure.queryFailed = error else { return XCTFail("unexpected error: \(error)") } + } + } + + @MainActor + func testUnreadableRemoteModalMirrorNeverResolvesAsAnEmptyAlert() throws { + app.launchArguments = ["--agent-device-alert-replacement-regression"] + app.launch() + defer { + modalElementsQueryOverrideForTesting = nil + app.terminate() + } + let modal = app.alerts.firstMatch + XCTAssertTrue(modal.waitForFixtureExistence(timeout: appExistenceTimeout)) + let window = app.windows.firstMatch + var queryIndex = 0 + modalElementsQueryOverrideForTesting = { + queryIndex += 1 + switch queryIndex { + case 1: return [window] + case 2: return [modal] + default: return [] + } + } + for action in ["get", "accept", "dismiss", "wait"] { + queryIndex = 0 + let command = try runnerCommandFixture( + #"{"command":"alert","commandId":"unreadable-mirror-\#(action)","action":"\#(action)"}"# + ) + let response = try executeOnMainPrepared(command: command, activeApp: app) + XCTAssertFalse(response.ok, action) + XCTAssertEqual(response.error?.code, RunnerWireErrorCode.alertUnresolved, action) + } + XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 0; replacement actions: 0") + } + + @MainActor + func testAlertWithNoEnabledButtonsStaysResolved() throws { + app.launchArguments = ["--agent-device-alert-replacement-regression", "--agent-device-alert-replacement-disabled-buttons"] + app.launch() + defer { app.terminate() } + let root = app.alerts.firstMatch + XCTAssertTrue(root.waitForFixtureExistence(timeout: appExistenceTimeout)) + let buttons = root.buttons.allElementsBoundByIndex + XCTAssertEqual(buttons.count, 2) + XCTAssertTrue(buttons.allSatisfy { !$0.isEnabled }) + guard case .resolved(let alert) = resolveAlert( + app: app, deadline: Date().addingTimeInterval(Self.defaultAlertCommandTimeout) + ) else { return XCTFail("a recognized surface remains resolved without enabled buttons") } + XCTAssertEqual(alert.source, .appAlert) + XCTAssertEqual(alert.actions.count, 2) + let inspection = try handleAlert(alert, action: "get", deadline: .distantFuture) + XCTAssertTrue(inspection.ok) + XCTAssertEqual(inspection.data?.message, "First confirmation") + XCTAssertEqual(inspection.data?.items, ["Cancel", "OK"]) + for action in ["accept", "dismiss"] { + let response = try handleAlert(alert, action: action, deadline: .distantFuture) + XCTAssertEqual(response.error?.code, RunnerWireErrorCode.alertButtonNotFound, action) + } + } + + @MainActor + func testAppAlertWithoutHittableControlsRemainsRetriableUntilControlsAppear() throws { + for action in ["get", "wait", "accept", "dismiss"] { + app.launchArguments = ["--agent-device-alert-replacement-regression"] + app.launch() + defer { + modalElementsQueryOverrideForTesting = nil + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let root = app.alerts.firstMatch + XCTAssertTrue(root.waitForFixtureExistence(timeout: appExistenceTimeout)) + let covered = app.staticTexts["agent-device-alert-actions"] + XCTAssertTrue(covered.exists) + XCTAssertFalse(covered.isHittable) + var queryIndex = 0 + modalElementsQueryOverrideForTesting = { + queryIndex += 1 + if queryIndex <= 2 { return [] } + if queryIndex == 3 { return [root] } + return [covered] + } + let command = try runnerCommandFixture( + #"{"command":"alert","commandId":"app-controls-\#(action)","action":"\#(action)"}"# + ) + for _ in 0..<2 { + queryIndex = 0 + let response = try executeOnMainPrepared(command: command, activeApp: app) + XCTAssertFalse(response.ok, action) + XCTAssertEqual(response.error?.code, RunnerWireErrorCode.alertUnresolved, action) + } + XCTAssertEqual(covered.label, "First actions: 0; replacement actions: 0") + modalElementsQueryOverrideForTesting = nil + let response = try executeOnMainPrepared(command: command, activeApp: app) + XCTAssertTrue(response.ok, "\(action): \(String(describing: response.error))") + if action == "get" || action == "wait" { + XCTAssertEqual(response.data?.items, ["Cancel", "OK"]) + } else { + XCTAssertEqual(covered.label, "First actions: 1; replacement actions: 0") + } + } + } +#endif + // No SpringBoard host (`hasSpringBoardSystemModalHost`) means modal resolution must return // `.absent` without probing com.apple.springboard (#1351). Written for tvOS, where no lane // ever executed it; `resolveBlockingSystemModal` takes that decision at RUNTIME off the same diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift index 8043005827..4a9b1033f9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift @@ -178,7 +178,7 @@ extension RunnerTests { let items = ["Cancel", "Open"] let foreign = XCUIApplication(bundleIdentifier: "com.example.app") func alert(owner: XCUIApplication, source: AlertSurfaceSource) -> RunnerAlert { - RunnerAlert(root: owner, ownerApp: owner, buttons: [], source: source, window: nil) + RunnerAlert(root: owner, ownerApp: owner, actions: [], source: source, window: nil) } XCTAssertTrue(marksLaunchConfirmation(alert(owner: springboard, source: .blockingSystemModal), title: title, items: items)) XCTAssertFalse(marksLaunchConfirmation(alert(owner: foreign, source: .appAlert), title: title, items: items)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift index d848007beb..2e841b112f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift @@ -9,10 +9,12 @@ extension RunnerTests { app.launch() defer { app.terminate() } XCTAssertTrue(app.switches["native-off"].waitForFixtureExistence(timeout: appExistenceTimeout)) - let resolved = ResolvedBlockingSystemModal( + let actions = try readModalActions(in: app, deadline: .distantFuture) + let resolved = RunnerAlert( root: app, ownerApp: app, - actions: actionableElements(in: app), + actions: actions, + source: .blockingSystemModal, window: nil ) let payload = try XCTUnwrap(try snapshotForBlockingSystemModal(resolved)) @@ -21,5 +23,21 @@ extension RunnerTests { XCTAssertEqual(off.value, "0") XCTAssertEqual(on.value, "1") } + + @MainActor + func testBlockingModalSnapshotIncludesDisabledActions() throws { + app.launchArguments = ["--agent-device-alert-replacement-regression", "--agent-device-alert-replacement-disabled-buttons"] + app.launch() + defer { app.terminate() } + let alert = try requireResolvedAlert(within: Self.defaultAlertCommandTimeout) + XCTAssertEqual(alert.actions.count, 2) + for raw in [false, true] { + let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: raw) + let payload = try XCTUnwrap(try snapshotForBlockingSystemModal(alert, options: options)) + let buttons = try XCTUnwrap(payload.nodes).filter { $0.type == "Button" } + XCTAssertEqual(buttons.compactMap(\.label).sorted(), ["Cancel", "OK"], "raw=\(raw)") + XCTAssertTrue(buttons.allSatisfy { $0.enabled == false }, "raw=\(raw)") + } + } #endif } diff --git a/contracts/fixtures/dispatch-disclosure.json b/contracts/fixtures/dispatch-disclosure.json index 446bd0283a..e3974c7426 100644 --- a/contracts/fixtures/dispatch-disclosure.json +++ b/contracts/fixtures/dispatch-disclosure.json @@ -226,7 +226,7 @@ { "id": "ios-runner.reply.ALERT_UNRESOLVED", "producer": "ios-runner", - "trigger": "structured runner reply with code ALERT_UNRESOLVED: a visible blocking system modal whose host could not be read, refused before any activation", + "trigger": "structured runner reply with code ALERT_UNRESOLVED: required alert acquisition failed, exhausted its deadline, or found an app surface without actionable controls, refused before any activation", "dispatched": "no" }, { diff --git a/packages/contracts/src/alert-contract.ts b/packages/contracts/src/alert-contract.ts index 312522dee4..ecf4f6a764 100644 --- a/packages/contracts/src/alert-contract.ts +++ b/packages/contracts/src/alert-contract.ts @@ -32,9 +32,9 @@ export const ALERT_NOT_FOUND_REASON = 'alert-not-found'; export const ALERT_BUTTON_NOT_FOUND_RUNNER_CODE = 'ALERT_BUTTON_NOT_FOUND'; /** - * The Apple XCUITest runner saw a blocking system modal but could not read its remote host, so - * it has no buttons to report or press. A prompt is on screen: this is never absence, and it - * travels as its own wire code with `dispatched: no`. + * The Apple XCUITest runner could not complete required alert acquisition: a query failed, + * the search deadline expired, a system modal's host could not be read, or an app surface has + * no actionable controls yet. It travels as its own wire code with `dispatched: no`. */ export const ALERT_UNRESOLVED_RUNNER_CODE = 'ALERT_UNRESOLVED'; diff --git a/packages/platform-apple/src/__tests__/alert.test.ts b/packages/platform-apple/src/__tests__/alert.test.ts index 21211b7165..e951d83b43 100644 --- a/packages/platform-apple/src/__tests__/alert.test.ts +++ b/packages/platform-apple/src/__tests__/alert.test.ts @@ -347,29 +347,151 @@ test('a wait that polls past an unreadable prompt into a readable one answers it assert.deepEqual(result, { title: 'Pair accessory' }); }); -test('an exhausted accept on an unreadable prompt keeps the runner hint, not the absence fallback', async () => { +test.each(['accept', 'dismiss'] as const)( + 'an exhausted %s on an unreadable prompt keeps the runner hint, not the absence fallback', + async (action) => { + vi.useFakeTimers(); + let calls = 0; + mockRunner.mockImplementation(async () => { + calls += 1; + throw runnerUnresolved(); + }); + + const outcome = actOnAppleAlert( + IOS_SIMULATOR, + runnerOptions, + action, + undefined, + process.env, + ).then( + () => undefined, + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(3_500); + const error = (await outcome) as AppError; + + assert.ok(calls > 1, 'an unreadable prompt is retried in the action window'); + assert.equal(error.code, ALERT_UNRESOLVED_RUNNER_CODE); + assert.match(String(error.details?.hint), /screenshot/); + assert.doesNotMatch(String(error.details?.hint), /scoped snapshot/i); + }, +); + +test.each([ + ['unresolved', 4_630], + ['absent', 4_630], + ['unresolved', 4_400], + ['absent', 4_400], +] as const)( + 'a wait retains a completed %s observation after a %d ms read', + async (state, readMs) => { + vi.useFakeTimers(); + const observed = state === 'unresolved' ? runnerUnresolved() : runnerAbsence(); + mockRunner.mockImplementation(async () => { + if (mockRunner.mock.calls.length > 1) { + throw new AppError('COMMAND_FAILED', 'runner timed out', { + runnerErrorCode: 'MAIN_THREAD_TIMEOUT', + dispatched: 'no', + }); + } + await new Promise((resolve) => setTimeout(resolve, readMs)); + throw observed; + }); + let settled = false; + const outcome = awaitAppleAlert(IOS_SIMULATOR, runnerOptions, { timeoutMs: 5_000 }, process.env) + .then( + () => undefined, + (error: unknown) => error, + ) + .then((result) => { + settled = true; + return result; + }); + await vi.advanceTimersByTimeAsync(4_999); + assert.equal(mockRunner.mock.calls.length, 1); + assert.equal(settled, false); + await vi.advanceTimersByTimeAsync(1); + const error = (await outcome) as AppError; + assert.equal(mockRunner.mock.calls.length, 1); + assert.equal( + (mockRunner.mock.calls[0]?.[1] as { timeoutMs: number } | undefined)?.timeoutMs, + 5_000, + ); + assert.equal((mockRunner.mock.calls[0]?.[1] as { action: string } | undefined)?.action, 'get'); + if (state === 'unresolved') { + assert.equal(error, observed); + assert.equal(error.details?.dispatched, 'no'); + } else { + assert.equal(error.message, 'alert wait timed out'); + assert.equal(error.details?.runnerErrorCode, ALERT_NOT_FOUND_RUNNER_CODE); + } + }, +); + +test('a dispatched wait read propagates its main-thread watchdog after an unresolved observation', async () => { vi.useFakeTimers(); - let calls = 0; - mockRunner.mockImplementation(async () => { - calls += 1; - throw runnerUnresolved(); + const watchdog = new AppError('COMMAND_FAILED', 'runner main thread timed out', { + runnerErrorCode: 'MAIN_THREAD_TIMEOUT', + dispatched: 'no', }); - - const outcome = actOnAppleAlert( + mockRunner.mockImplementation(async (_device, command) => { + if (mockRunner.mock.calls.length === 1) throw runnerUnresolved(); + await new Promise((resolve) => + setTimeout(resolve, (command as { timeoutMs: number }).timeoutMs), + ); + throw watchdog; + }); + const outcome = awaitAppleAlert( IOS_SIMULATOR, runnerOptions, - 'accept', - undefined, + { timeoutMs: 5_000 }, process.env, ).then( () => undefined, (error: unknown) => error, ); - await vi.advanceTimersByTimeAsync(3_500); - const error = (await outcome) as AppError; + await vi.advanceTimersByTimeAsync(5_000); + assert.equal(await outcome, watchdog); + assert.equal(mockRunner.mock.calls.length, 2); + assert.equal( + (mockRunner.mock.calls[1]?.[1] as { timeoutMs: number } | undefined)?.timeoutMs, + 4_700, + ); + assert.ok( + mockRunner.mock.calls.every((call) => (call[1] as { action: string }).action === 'get'), + ); + assert.equal(watchdog.details?.dispatched, 'no'); +}); - assert.ok(calls > 1, 'an unreadable prompt is retried in the action window'); - assert.equal(error.code, ALERT_UNRESOLVED_RUNNER_CODE); - assert.match(String(error.details?.hint), /screenshot/); - assert.doesNotMatch(String(error.details?.hint), /scoped snapshot/i); +test('a wait admits another read when more than one poll interval remains', async () => { + vi.useFakeTimers(); + mockRunner.mockImplementation(async () => { + if (mockRunner.mock.calls.length === 1) { + await new Promise((resolve) => setTimeout(resolve, 4_399)); + throw runnerUnresolved(); + } + return { title: 'Ready' }; + }); + const outcome = awaitAppleAlert(IOS_SIMULATOR, runnerOptions, { timeoutMs: 5_000 }, process.env); + await vi.advanceTimersByTimeAsync(4_699); + assert.deepEqual(await outcome, { title: 'Ready' }); + assert.equal( + (mockRunner.mock.calls[1]?.[1] as { timeoutMs: number } | undefined)?.timeoutMs, + 301, + ); }); + +test.each(['accept', 'dismiss'] as const)( + 'an %s retries an unresolved alert and acts when controls become readable', + async (action) => { + vi.useFakeTimers(); + mockRunner.mockRejectedValueOnce(runnerUnresolved()).mockResolvedValueOnce({ message: 'done' }); + const outcome = actOnAppleAlert(IOS_SIMULATOR, runnerOptions, action, undefined, process.env); + await vi.advanceTimersByTimeAsync(300); + assert.deepEqual(await outcome, { message: 'done' }); + assert.equal(mockRunner.mock.calls.length, 2); + assert.ok( + mockRunner.mock.calls.every((call) => (call[1] as { action: string }).action === action), + ); + }, +); diff --git a/packages/platform-apple/src/alert.ts b/packages/platform-apple/src/alert.ts index 3fcce5ba3f..df425bf618 100644 --- a/packages/platform-apple/src/alert.ts +++ b/packages/platform-apple/src/alert.ts @@ -85,6 +85,8 @@ export async function readAppleAlert( /** * Poll `get` until one answers or the caller's window runs out. The first attempt gets the whole * window and later ones only what is left, so a runner that blocks cannot outlive the budget. + * After a completed observation, another read needs more than one polling interval of budget. + * A dispatched read still owns its result, including a watchdog timeout. */ export async function awaitAppleAlert( device: DeviceInfo, @@ -95,12 +97,15 @@ export async function awaitAppleAlert( const runAlert = runAppleAlert(device, runnerOptions, options, env); const timeout = options?.timeoutMs ?? DEFAULT_ALERT_TIMEOUT_MS; const start = Date.now(); - let firstAttempt = true; let lastRetriable: AppError | undefined; while (Date.now() - start < timeout) { + const remainingMs = remainingBudgetMs(start, timeout); + if (lastRetriable && remainingMs <= ALERT_POLL_INTERVAL_MS) { + await sleep(remainingMs); + break; + } try { - const budgetMs = firstAttempt ? timeout : remainingBudgetMs(start, timeout); - firstAttempt = false; + const budgetMs = lastRetriable ? remainingMs : timeout; return await runAlert('get', budgetMs); } catch (error) { // Only a typed absence or an unreadable prompt is worth waiting out. Anything else — a dead @@ -109,9 +114,12 @@ export async function awaitAppleAlert( if (!isAlertRetriableError(error)) throw error; lastRetriable = error; } - await sleep(ALERT_POLL_INTERVAL_MS); + const remainingMsAfterRead = timeout - (Date.now() - start); + if (remainingMsAfterRead > 0) { + await sleep(Math.min(ALERT_POLL_INTERVAL_MS, remainingMsAfterRead)); + } } - if (lastRetriable && isAlertUnresolvedError(lastRetriable)) throw lastRetriable; + if (isAlertUnresolvedError(lastRetriable)) throw lastRetriable; throw new AppError('COMMAND_FAILED', 'alert wait timed out', lastRetriable?.details); } diff --git a/src/commands/schema/cli-help-workflows.ts b/src/commands/schema/cli-help-workflows.ts index cbf67c7cb7..3b85574470 100644 --- a/src/commands/schema/cli-help-workflows.ts +++ b/src/commands/schema/cli-help-workflows.ts @@ -128,7 +128,7 @@ Alerts: If alert reports no alert but a sheet is visible, alert did not recognize it. Press the button by the ref snapshot -i prints (@e3 is an example): agent-device snapshot -i agent-device press @e3 - ALERT_UNRESOLVED means a system prompt is visible but unreadable. Inspect it with screenshot, then snapshot -i. + ALERT_UNRESOLVED means alert could not finish reading the prompt state or found an app alert, Safari dialog, or dismiss-popup window without actionable controls. Inspect it with screenshot, then snapshot -i. Do not use settings permission to answer a dialog already on screen. Reserve settings permission for setup/resetting permission state before a flow. Diagnostics and traces: diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 87306b5289..ecce60609b 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -445,9 +445,10 @@ agent-device alert dismiss - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. - The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators the local XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If a notification, privacy, or location table cannot be read, the runner drops the permission translations and keeps the English labels; the `Open in …?` table is read apart, so its labels survive that, and the permission translations survive it being unreadable. Physical iOS devices, tvOS, and visionOS always use the English labels. Limrun iOS recognizes English labels only, because its tree capture cannot read the device's string tables. On local iOS simulators, `alert get` marks the `Open in …?` launch confirmation with `launchConfirmationPrompt: true`; `open` reads that mark to answer the prompt. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Limrun still needs at least two actionable buttons to recognize an alert. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. - On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or localized decline copy outside the preferred languages' prompt strings fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). -- An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. +- An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline during action or confirmation after the alert is resolved fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. - iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. -- On the Apple XCUITest runner, a blocking system prompt whose buttons the runner cannot read fails with `error.code: ALERT_UNRESOLVED` and `details.dispatched: no`. A prompt is on screen, so this is never "no alert". `alert get` reports it at once. `accept`, `dismiss`, and `alert wait` retry it within their windows, like absence, and report it, not a timeout, when the last check still saw it. Inspect the prompt with `screenshot`, then `snapshot -i`, and `press @ref` on the visible button. +- On the Apple XCUITest runner, an alert search that fails, runs out of time, or finds an app alert, Safari dialog, or dismiss-popup window with no actionable controls fails with `error.code: ALERT_UNRESOLVED` and `details.dispatched: no`. `alert get` reports it at once. `accept`, `dismiss`, and `alert wait` retry it within their windows, like absence, and report it, not a timeout, when the last check remained unresolved. Inspect the prompt with `screenshot`, then `snapshot -i`, and `press @ref` on the visible button. +- On the Apple XCUITest runner, `alert get` describes a readable alert even when all its buttons are disabled, and lists those disabled buttons. `alert accept` and `alert dismiss` fail with `ALERT_BUTTON_NOT_FOUND`. - If a sheet or permission prompt is visible but `alert` reports none, `alert` did not recognize it. Run `snapshot -i -s ""`, then `press @ref` on the visible button. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. ## Interactions