diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSurfacePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSurfacePolicy.swift new file mode 100644 index 0000000000..cf5cf752f0 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSurfacePolicy.swift @@ -0,0 +1,39 @@ +import XCTest + +enum RunnerSurfacePolicy: String { + case none + case appWindows = "app-windows" + case observedScreen = "observed-screen" + case resolvedModal = "resolved-modal" +} + +extension CommandType { + var surfacePolicy: RunnerSurfacePolicy { + switch self { + case .mouseClick, .remotePress, .desktopScroll, .snapshotBridge, .home, .rotate, + .actionButton, .volumeUp, .volumeDown, .keyboardReturn, .recordStop, .status, .uptime, + .appState, .pasteboardWrite, .activate, .terminate, .targetReset, .shutdown: + return .none + case .tap, .longPress, .drag, .type, .swipe, .scroll, .findText, .querySelector, .readText, + .snapshot, .backInApp, .backSystem, .gesture, .gestureViewport, .appSwitcher, + .keyboardDismiss, .sequence: + return .appWindows + case .screenshot, .recordStart: + return .observedScreen + case .alert: + return .resolvedModal + } + } +} + +extension RunnerTests { + @MainActor + func prepareCommandSurface(command: Command, app: XCUIApplication) -> RunnerAppScreenCaptureFailure? { + switch command.command.surfacePolicy { + case .appWindows: + return bookRequestLitPanelSurface(app: app) + case .none, .observedScreen, .resolvedModal: + return nil + } + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index c5d3f55785..4614d347bd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -39,8 +39,7 @@ extension RunnerTests { if command.invalidatesRememberedTextEntryTap { clearRememberedTextEntryTap() } - // A screenshot resolves its own surface, the target app and then the system surface. - if command.command != .screenshot, let refusal = bookRequestLitPanelSurface(app: activeApp) { + if let refusal = prepareCommandSurface(command: command, app: activeApp) { return refusal.response } switch command.command { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift index 1f2e9866d3..10496e5e94 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift @@ -18,7 +18,7 @@ extension RunnerTests { case .response(let response): return .response(response) case .context(let context): - if let refusal = self.bookRequestLitPanelSurface(app: context.app) { + if let refusal = self.prepareCommandSurface(command: command, app: context.app) { return .response(refusal.response) } if self.requestLitPanel != nil, let refusal = Self.litPanelBackendPinRefusal(command) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift index 8bba410ff5..293685b255 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift @@ -48,6 +48,11 @@ extension RunnerTests { XCTAssertEqual(orphaned, [], "CommandType cases with no production request") } + func testRunnerSurfacePreparationMatchesTheSharedPolicyFixture() throws { + let policies = try XCTUnwrap(runnerContractFixture("runner-surface-policy.json") as? [String: String]) + XCTAssertEqual(policies, Dictionary(uniqueKeysWithValues: CommandType.allCases.map { ($0.rawValue, $0.surfacePolicy.rawValue) })) + } + func testQueuedReadOnlyTraitsMatchTheDaemonParityFixture() throws { let rows = try XCTUnwrap( runnerContractFixture("runner-command-read-only.json") as? [[String: Any]] diff --git a/contracts/fixtures/runner-surface-policy.json b/contracts/fixtures/runner-surface-policy.json new file mode 100644 index 0000000000..933b248c88 --- /dev/null +++ b/contracts/fixtures/runner-surface-policy.json @@ -0,0 +1,41 @@ +{ + "tap": "app-windows", + "mouseClick": "none", + "longPress": "app-windows", + "drag": "app-windows", + "remotePress": "none", + "type": "app-windows", + "swipe": "app-windows", + "scroll": "app-windows", + "desktopScroll": "none", + "findText": "app-windows", + "querySelector": "app-windows", + "readText": "app-windows", + "snapshot": "app-windows", + "snapshotBridge": "none", + "screenshot": "observed-screen", + "backInApp": "app-windows", + "backSystem": "app-windows", + "home": "none", + "rotate": "none", + "gesture": "app-windows", + "gestureViewport": "app-windows", + "appSwitcher": "app-windows", + "actionButton": "none", + "volumeUp": "none", + "volumeDown": "none", + "keyboardDismiss": "app-windows", + "keyboardReturn": "none", + "alert": "resolved-modal", + "sequence": "app-windows", + "recordStart": "observed-screen", + "recordStop": "none", + "status": "none", + "uptime": "none", + "appState": "none", + "pasteboardWrite": "none", + "activate": "none", + "terminate": "none", + "targetReset": "none", + "shutdown": "none" +} diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 297c18dc5e..2a07c7b059 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -720,13 +720,21 @@ Android reports the app window on the next snapshot instead of a panel. SpringBoard keeps windows on both panels. Its first window is the outer panel's status bar, whichever panel is lit. The runner booked gestures, the synthesized reference frame, the snapshot viewport, and the capture display against that window. A home-screen gesture could therefore land on a dark panel. -Decision: the host names the lit panel in each runner request that resolves a screen. `RUNNER_COMMAND_RESOLVES_SURFACE` in `packages/apple-runner/src/runner-command-traits.ts` is the one list of these commands: `tap`, `longPress`, `drag`, `type`, `swipe`, `scroll`, `findText`, `querySelector`, `readText`, `snapshot`, `screenshot`, `backInApp`, `backSystem`, `gesture`, `gestureViewport`, `appSwitcher`, `keyboardDismiss`, `sequence`, and `recordStart`. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The read never outlasts the request's `timeoutMs`. A multi-panel table read that names the lit panel but spends the request budget refuses with `COMMAND_FAILED`, `reason: panel-probe-budget-exhausted` and `dispatched: no`, without a runner error code. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. The runner gets what the read left of `timeoutMs`. A device known to be multi-panel whose table does not answer is refused with `COMMAND_FAILED`, `reason: panel-table-unreadable` and `runnerErrorCode: APP_SCREEN_UNRESOLVED`, before anything is sent. Only a device whose panel count was never read keeps the historic first-window route unchanged. +Decision: requests that search app windows or observe a screen name the lit panel before dispatch. +The [TypeScript policy](../../packages/apple-runner/src/runner-command-traits.ts), [Swift policy](../../apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSurfacePolicy.swift), and [shared fixture](../../contracts/fixtures/runner-surface-policy.json) own command classification. +Modal commands resolve their prompt through the modal owner instead of booking an app window in advance. -With a named panel, the runner walks at most eight windows to book one; element search reads every window that reports the lit display. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A window that the walk cannot place refuses the request with `APP_SCREEN_UNRESOLVED`, even if a full-panel window follows it. A list past the cap with no full-panel window refuses the same way. An app whose windows are all on another display is `APP_SCREEN_OFF_LIT_PANEL`. An app with no window at all is `APP_SCREEN_WINDOW_UNRESOLVED`. A window list that cannot be read is `APP_SCREEN_UNRESOLVED`, so it never reaches the system-surface fallback. +Omitting `litPanel` preserves the historic route. +This includes single-panel devices, unsupported devices, unknown or backed-off inventory, and multi-panel devices whose power is ambiguous or whose inventory is incomplete. +A known multi-panel device whose table does not answer refuses before dispatch with `COMMAND_FAILED`, `reason: panel-table-unreadable` and `runnerErrorCode: APP_SCREEN_UNRESOLVED`; a read that spends the request budget refuses with `reason: panel-probe-budget-exhausted` ([owner](../../packages/platform-apple/src/foldable/runner-panel.ts), [tests](../../packages/platform-apple/src/foldable/runner-panel.test.ts)). -The request books the window by bundle identity and request identity before it acts, and releases the booking when the request ends. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window of the same app, so they never fall back to the application's own frame. Every element read of such a request (`findText`, `querySelector`, `readText`, a selector `tap`, and the text-input lookup of `tap` and `type`) searches only under the application's windows that report the lit display. The element search rereads the complete window list for each read, including windows beyond the booking cap, because mutations can open another window. If the list cannot be read or contains no proven lit windows, the search throws `APP_SCREEN_UNRESOLVED` instead of answering no match. A lit-panel refusal during text-entry confirmation propagates with its typed code. The panel is written onto the main queue behind work already queued there, so work that a previous command abandoned still reads that command's panel. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. +A request that names a lit panel books its window before dispatch and refuses before any gesture or read: `APP_SCREEN_OFF_LIT_PANEL` when every app window is on another display, `APP_SCREEN_WINDOW_UNRESOLVED` when the app has no window, and `APP_SCREEN_UNRESOLVED` when the window list cannot be read or placed ([resolver](../../apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m), [resolver tests](../../apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift), [booking tests](../../apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift)). +The booking walk is bounded, while element search reads every window on the lit display and refuses with `APP_SCREEN_UNRESOLVED` instead of answering no match when it proves none ([owner](../../apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift), [booking tests](../../apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift)). +Booking is request-scoped, so delayed work stays on its own command's panel ([owner](../../apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift)). -A snapshot runs only the tree tier, because only that tier can prove which display a window is on. A pin to another backend is refused with `UNSUPPORTED_OPERATION`. The snapshot traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the booked window's frame. `--raw` applies the same window filter and the same root frame, so raw and regular coordinates share one space. The snapshot reads the booked window only from its capture target. Collapsed-tab fallback candidates use the shared proven-lit element-query owner with that capture target's pinned display ID. A failed root walk propagates `APP_SCREEN_UNRESOLVED`. A `customActions` request with no explicit backend pin implies the private AX pin, so its refusal names `customActions`. +A lit-panel snapshot keeps only windows whose snapshot reports the lit display, drops a window that reports none, and refuses with `APP_SCREEN_UNRESOLVED` when no window is proven ([owner](../../apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift), [booking tests](../../apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift)). +A pin to any backend other than the tree backend refuses with `UNSUPPORTED_OPERATION`, because only that backend proves display identity ([owner](../../apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift), [booking tests](../../apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift)). +A capture asks the system surface only after `APP_SCREEN_WINDOW_UNRESOLVED`; an app whose windows are all on the dark panel refuses with `APP_SCREEN_OFF_LIT_PANEL` and is never sent to the system surface ([owner](../../apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift), [capture tests](../../apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift)). Rejected alternatives: diff --git a/packages/apple-runner/src/__tests__/runner-command-traits.test.ts b/packages/apple-runner/src/__tests__/runner-command-traits.test.ts index d0cb3e779a..fd0896691a 100644 --- a/packages/apple-runner/src/__tests__/runner-command-traits.test.ts +++ b/packages/apple-runner/src/__tests__/runner-command-traits.test.ts @@ -8,12 +8,26 @@ import { isReadOnlyRunnerCommand, readRunnerCommandTraits, RUNNER_COMMAND_TRAITS, + runnerCommandSurfacePolicy, type RunnerCommandTraits, } from '../runner-command-traits.ts'; import { readSwiftOffQueueCommands } from './runner-swift-settlement-fixtures.ts'; const RUNNER_COMMANDS = Object.keys(RUNNER_COMMAND_TRAITS) as Array; +test('runner surface policies match the cross-language fixture', () => { + const fixture = new URL( + '../../../../contracts/fixtures/runner-surface-policy.json', + import.meta.url, + ); + assert.deepEqual( + Object.fromEntries( + RUNNER_COMMANDS.map((command) => [command, runnerCommandSurfacePolicy({ command })]), + ), + JSON.parse(fs.readFileSync(fixture, 'utf8')), + ); +}); + test('runner command trait table pins lifecycle-sensitive command groups', () => { const groups = { preflightSkippableTouchMutation: commandsWithTraits(hotMutation()), diff --git a/packages/apple-runner/src/index.ts b/packages/apple-runner/src/index.ts index bdd464e2a9..477f057334 100644 --- a/packages/apple-runner/src/index.ts +++ b/packages/apple-runner/src/index.ts @@ -41,7 +41,7 @@ export { type RunnerCommand, type RunnerSnapshotBridgeRequest, } from './runner-contract.ts'; -export { runnerCommandResolvesSurface } from './runner-command-traits.ts'; +export { runnerCommandSurfacePolicy } from './runner-command-traits.ts'; export { isRunnerMainThreadTimeoutError } from './runner-error-classification.ts'; export { withAppleRunnerProvider, diff --git a/packages/apple-runner/src/runner-command-traits.ts b/packages/apple-runner/src/runner-command-traits.ts index 327f9c1bb9..a6dea8dcce 100644 --- a/packages/apple-runner/src/runner-command-traits.ts +++ b/packages/apple-runner/src/runner-command-traits.ts @@ -109,55 +109,52 @@ export const RUNNER_COMMAND_TRAITS = { shutdown: DEFAULT_TRAITS, } as const satisfies Record; -/** - * Whether the runner resolves a screen for the command: a window to book a gesture on, a viewport, - * a display to sample, or windows to search for an element. Such a request for a multi-panel device - * names the lit panel (ADR 0025). This table is the one list of lit-panel-scoped commands. - */ -const RUNNER_COMMAND_RESOLVES_SURFACE = { - tap: true, - mouseClick: false, - longPress: true, - drag: true, - remotePress: false, - type: true, - swipe: true, - scroll: true, - desktopScroll: false, - findText: true, - querySelector: true, - readText: true, - snapshot: true, - screenshot: true, - backInApp: true, - backSystem: true, - home: false, - rotate: false, - gesture: true, - gestureViewport: true, - appSwitcher: true, - actionButton: false, - volumeUp: false, - volumeDown: false, - keyboardDismiss: true, - keyboardReturn: false, - alert: false, - sequence: true, - recordStart: true, - recordStop: false, - status: false, - uptime: false, - appState: false, - snapshotBridge: false, - pasteboardWrite: false, - activate: false, - terminate: false, - targetReset: false, - shutdown: false, -} as const satisfies Record; - -export function runnerCommandResolvesSurface(command: RunnerCommand): boolean { - return RUNNER_COMMAND_RESOLVES_SURFACE[command.command]; +export type RunnerSurfacePolicy = 'none' | 'app-windows' | 'observed-screen' | 'resolved-modal'; + +const RUNNER_COMMAND_SURFACE_POLICY = { + tap: 'app-windows', + mouseClick: 'none', + longPress: 'app-windows', + drag: 'app-windows', + remotePress: 'none', + type: 'app-windows', + swipe: 'app-windows', + scroll: 'app-windows', + desktopScroll: 'none', + findText: 'app-windows', + querySelector: 'app-windows', + readText: 'app-windows', + snapshot: 'app-windows', + screenshot: 'observed-screen', + backInApp: 'app-windows', + backSystem: 'app-windows', + home: 'none', + rotate: 'none', + gesture: 'app-windows', + gestureViewport: 'app-windows', + appSwitcher: 'app-windows', + actionButton: 'none', + volumeUp: 'none', + volumeDown: 'none', + keyboardDismiss: 'app-windows', + keyboardReturn: 'none', + alert: 'resolved-modal', + sequence: 'app-windows', + recordStart: 'observed-screen', + recordStop: 'none', + status: 'none', + uptime: 'none', + appState: 'none', + snapshotBridge: 'none', + pasteboardWrite: 'none', + activate: 'none', + terminate: 'none', + targetReset: 'none', + shutdown: 'none', +} as const satisfies Record; + +export function runnerCommandSurfacePolicy(command: RunnerCommand): RunnerSurfacePolicy { + return RUNNER_COMMAND_SURFACE_POLICY[command.command]; } export function isReadOnlyRunnerCommand(command: RunnerCommand): boolean { diff --git a/packages/platform-apple/src/core/runner-client.ts b/packages/platform-apple/src/core/runner-client.ts index 9dd55eefb1..09c81d280e 100644 --- a/packages/platform-apple/src/core/runner-client.ts +++ b/packages/platform-apple/src/core/runner-client.ts @@ -1,7 +1,7 @@ import { bindAppleRunnerHost, resolveRunnerRequestSignal, - runnerCommandResolvesSurface, + runnerCommandSurfacePolicy, runAppleRunnerCommand as runHostBoundAppleRunnerCommand, type AppleRunnerCommandOptions, type RunnerCommand, @@ -29,8 +29,13 @@ export async function runAppleRunnerCommand( command: RunnerCommand, options: AppleRunnerCommandOptions = {}, ): Promise> { - if (!runnerCommandResolvesSurface(command)) { - return await runHostBoundAppleRunnerCommand(device, command, options); + switch (runnerCommandSurfacePolicy(command)) { + case 'none': + case 'resolved-modal': + return await runHostBoundAppleRunnerCommand(device, command, options); + case 'app-windows': + case 'observed-screen': + break; } const { nameRunnerRequestLitPanel } = await import('../foldable/runner-panel.ts'); return await runHostBoundAppleRunnerCommand( diff --git a/packages/platform-apple/src/foldable/runner-panel.ts b/packages/platform-apple/src/foldable/runner-panel.ts index 0e8010172d..4fc8bc5a58 100644 --- a/packages/platform-apple/src/foldable/runner-panel.ts +++ b/packages/platform-apple/src/foldable/runner-panel.ts @@ -23,7 +23,8 @@ function panelTableRefusal(device: DeviceInfo, message: string): AppError { /** * The lit panel a runner request names, read fresh per request because a fold changes it (ADR * 0025). The read never outlasts the request's `timeoutMs`. Undefined keeps the historic - * first-window route, for a device not known to have more than one panel. + * first-window route whenever the read cannot name one lit panel. An unreadable table on a + * known multi-panel device refuses instead. */ export async function resolveRunnerRequestLitPanel( device: DeviceInfo,