From 81195db3b9422ecb54a834ff1e825bd3ad4c5591 Mon Sep 17 00:00:00 2001 From: Vitalii Yehorov Date: Tue, 3 Nov 2026 10:03:45 +0100 Subject: [PATCH 1/8] feat(screenshot): select the XCTest runner capture with --capture-backend runner On iOS simulators with a Dynamic Island, simctl io screenshot draws the island into every capture (no --mask policy removes it), while XCUIScreen, which Maestro and other XCTest tools capture with, leaves it out. A visual baseline made by such a tool therefore cannot be matched by agent-device's default capture. The runner capture already existed for the Maestro port behind an internal flag. screenshot --capture-backend runner (SDK and MCP: captureBackend: 'runner') selects it on Apple targets. It is refused on other platforms and with --pixel-density, since the runner encodes the device density. The Maestro port now uses the public flag; flags.maestro.screenshotCaptureBackend is removed. Co-Authored-By: Claude Opus 5.5 --- .../command-surface/src/capture/index.test.ts | 9 +++++ .../src/capture/screenshot-options.test.ts | 6 ++++ .../command-surface/src/capture/screenshot.ts | 2 ++ packages/contracts/src/client-capture.ts | 3 ++ packages/contracts/src/command-flags.ts | 1 - packages/contracts/src/facades/capture.ts | 2 ++ packages/contracts/src/screenshot.ts | 25 ++++++++++++++ .../daemon/src/core/__tests__/context.test.ts | 4 +-- packages/daemon/src/core/context.ts | 1 - .../screenshot-runtime-execution.test.ts | 34 +++++++++++++++++++ .../src/execution/screenshot-runtime.ts | 33 ++++++++++++++++-- .../__tests__/daemon-runtime-port.test.ts | 8 ++--- .../daemon-runtime-public-operation.test.ts | 2 +- .../daemon-runtime-public-operation.ts | 2 +- scripts/integration-progress-model.ts | 7 +++- .../__tests__/args-parse-interaction.test.ts | 3 ++ website/docs/docs/commands.md | 2 ++ 17 files changed, 129 insertions(+), 15 deletions(-) diff --git a/packages/command-surface/src/capture/index.test.ts b/packages/command-surface/src/capture/index.test.ts index 28a6aeac72..308a763174 100644 --- a/packages/command-surface/src/capture/index.test.ts +++ b/packages/command-surface/src/capture/index.test.ts @@ -92,6 +92,15 @@ describe('capture command interface', () => { }); }); + test('reads and writes the screenshot capture backend', () => { + const input = screenshotCliReader(['page.png'], flags({ screenshotCaptureBackend: 'runner' })); + expect(input).toMatchObject({ path: 'page.png', captureBackend: 'runner' }); + expect(screenshotDaemonWriter(input)).toMatchObject({ + command: 'screenshot', + options: { screenshotCaptureBackend: 'runner' }, + }); + }); + test('reads diff snapshot input only', () => { expect( diffCliReader(['snapshot'], flags({ snapshotDepth: 4, out: './diff.json' })), diff --git a/packages/command-surface/src/capture/screenshot-options.test.ts b/packages/command-surface/src/capture/screenshot-options.test.ts index 4140eea6a3..3fcf2dfa07 100644 --- a/packages/command-surface/src/capture/screenshot-options.test.ts +++ b/packages/command-surface/src/capture/screenshot-options.test.ts @@ -25,6 +25,7 @@ test('screenshot flag projection maps CLI flags to runtime options', () => { screenshotScale: 0.3, screenshotNoStabilize: true, screenshotNormalizeStatusBar: true, + screenshotCaptureBackend: 'runner', }), { overlayRefs: true, @@ -35,6 +36,7 @@ test('screenshot flag projection maps CLI flags to runtime options', () => { scale: 0.3, stabilize: false, normalizeStatusBar: true, + captureBackend: 'runner', }, ); }); @@ -49,6 +51,7 @@ test('screenshot flag projection maps public options to request flags', () => { fullscreen: true, stabilize: false, normalizeStatusBar: true, + captureBackend: 'runner', }), { overlayRefs: true, @@ -58,6 +61,7 @@ test('screenshot flag projection maps public options to request flags', () => { screenshotFullscreen: true, screenshotNoStabilize: true, screenshotNormalizeStatusBar: true, + screenshotCaptureBackend: 'runner', }, ); }); @@ -152,6 +156,7 @@ test('screenshot script flags use the shared recorded flag contract', () => { 'screenshotScale', 'screenshotNoStabilize', 'screenshotNormalizeStatusBar', + 'screenshotCaptureBackend', ]); assert.deepEqual( SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS.map((definition) => definition.key), @@ -167,6 +172,7 @@ test('screenshot script flags use the shared recorded flag contract', () => { 'screenshotScale', 'screenshotNoStabilize', 'screenshotNormalizeStatusBar', + 'screenshotCaptureBackend', ]); }); diff --git a/packages/command-surface/src/capture/screenshot.ts b/packages/command-surface/src/capture/screenshot.ts index df08b13bc4..17e1b81b5c 100644 --- a/packages/command-surface/src/capture/screenshot.ts +++ b/packages/command-surface/src/capture/screenshot.ts @@ -3,6 +3,7 @@ import type { CaptureScreenshotOptions } from '@agent-device/contracts/client'; import { SESSION_SURFACES } from '@agent-device/contracts/session'; import { RETIRED_SCREENSHOT_MAX_SIZE, + SCREENSHOT_CAPTURE_BACKENDS, SCREENSHOT_COMMAND_FLAG_KEYS, SCREENSHOT_SCALE_LIMITS, SCREENSHOT_SCREEN_MASKS, @@ -50,6 +51,7 @@ const screenshotCommandMetadata = defineFieldCommandMetadata( maxSize: retiredField(RETIRED_SCREENSHOT_MAX_SIZE.migration.screenshot), stabilize: booleanField(), normalizeStatusBar: booleanField(), + captureBackend: enumField(SCREENSHOT_CAPTURE_BACKENDS), surface: enumField(SESSION_SURFACES), }, ); diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index 0df298488c..ecf4cde0d7 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -1,6 +1,7 @@ import type { ScreenshotScreenMask } from './screenshot.ts'; // The public API vocabulary for snapshot, screenshot and diff capture. +import type { ScreenshotCaptureBackend } from './screenshot.ts'; import type { SessionSurface } from './session-surface.ts'; import type { PublicSnapshotCaptureAnnotations } from './snapshot-capture-annotations.ts'; import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts'; @@ -118,6 +119,8 @@ export type CaptureScreenshotOptions = AgentDeviceRequestOverrides & { scale?: number; stabilize?: boolean; normalizeStatusBar?: boolean; + /** `runner`: capture an Apple target through the XCTest runner instead of `simctl`. */ + captureBackend?: ScreenshotCaptureBackend; surface?: SessionSurface; }; diff --git a/packages/contracts/src/command-flags.ts b/packages/contracts/src/command-flags.ts index 7b56cc0e6d..1986a47c17 100644 --- a/packages/contracts/src/command-flags.ts +++ b/packages/contracts/src/command-flags.ts @@ -22,7 +22,6 @@ export type MaestroRuntimeFlags = { allowNonHittableCoordinateFallback?: boolean; expectedTapPoint?: Point; prewarmRunnerBeforeOpen?: boolean; - screenshotCaptureBackend?: 'runner'; }; export type CommandFlags = Omit & { diff --git a/packages/contracts/src/facades/capture.ts b/packages/contracts/src/facades/capture.ts index ca05162baf..5245d1da06 100644 --- a/packages/contracts/src/facades/capture.ts +++ b/packages/contracts/src/facades/capture.ts @@ -2,6 +2,7 @@ export type { DiffSnapshotCommandResult, SnapshotDiffLine, SnapshotDiffSummary } export { RETIRED_SCREENSHOT_MAX_SIZE, SCREENSHOT_ACTION_FLAG_KEYS, + SCREENSHOT_CAPTURE_BACKENDS, SCREENSHOT_COMMAND_FLAG_KEYS, SCREENSHOT_CROP_REASONS, SCREENSHOT_FULLSCREEN_REASONS, @@ -19,6 +20,7 @@ export { validateScreenshotScale, } from '../screenshot.ts'; export type { + ScreenshotCaptureBackend, ScreenshotCropReason, ScreenshotScreenMask, ScreenshotDispatchFlags, diff --git a/packages/contracts/src/screenshot.ts b/packages/contracts/src/screenshot.ts index be62912d25..d4e661a1e5 100644 --- a/packages/contracts/src/screenshot.ts +++ b/packages/contracts/src/screenshot.ts @@ -88,6 +88,11 @@ export const SCREENSHOT_FULLSCREEN_REASONS = { export type ScreenshotFullscreenReason = (typeof SCREENSHOT_FULLSCREEN_REASONS)[keyof typeof SCREENSHOT_FULLSCREEN_REASONS]; +/** The capture paths a screenshot can name instead of the platform default. */ +export const SCREENSHOT_CAPTURE_BACKENDS = ['runner'] as const; + +export type ScreenshotCaptureBackend = (typeof SCREENSHOT_CAPTURE_BACKENDS)[number]; + export const SCREENSHOT_COMMAND_FLAG_KEYS = [ 'out', 'overlayRefs', @@ -98,6 +103,7 @@ export const SCREENSHOT_COMMAND_FLAG_KEYS = [ 'screenshotScale', 'screenshotNoStabilize', 'screenshotNormalizeStatusBar', + 'screenshotCaptureBackend', ] as const; export const SCREENSHOT_ACTION_FLAG_KEYS = [ @@ -108,6 +114,7 @@ export const SCREENSHOT_ACTION_FLAG_KEYS = [ 'screenshotScale', 'screenshotNoStabilize', 'screenshotNormalizeStatusBar', + 'screenshotCaptureBackend', ] as const; type ScreenshotSpecificFlagKey = (typeof SCREENSHOT_ACTION_FLAG_KEYS)[number]; @@ -200,6 +207,17 @@ export const SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS: readonly ScreenshotSpecificFl projectConfig: true, recorded: true, }, + { + key: 'screenshotCaptureBackend', + names: ['--capture-backend'], + type: 'enum', + enumValues: SCREENSHOT_CAPTURE_BACKENDS, + usageLabel: '--capture-backend runner', + usageDescription: + 'Screenshot: on Apple targets capture through the XCTest runner (XCUIScreen), which leaves out the simulator-drawn Dynamic Island, instead of simctl; output is at the device density, so --pixel-density is refused with it', + projectConfig: true, + recorded: false, + }, ]; const SCREENSHOT_SCRIPT_BOOLEAN_FLAGS = [ @@ -244,6 +262,7 @@ export type ScreenshotRequestFlags = { screenshotScale?: number; screenshotNoStabilize?: boolean; screenshotNormalizeStatusBar?: boolean; + screenshotCaptureBackend?: ScreenshotCaptureBackend; }; export type ScreenshotDispatchFlags = Pick< @@ -253,6 +272,7 @@ export type ScreenshotDispatchFlags = Pick< | 'screenshotFullscreen' | 'screenshotNoStabilize' | 'screenshotNormalizeStatusBar' + | 'screenshotCaptureBackend' >; export type ScreenshotRuntimeFlags = Pick< @@ -264,6 +284,7 @@ export type ScreenshotRuntimeFlags = Pick< | 'screenshotScale' | 'screenshotNoStabilize' | 'screenshotNormalizeStatusBar' + | 'screenshotCaptureBackend' >; export type ScreenshotPublicOptions = { @@ -275,6 +296,7 @@ export type ScreenshotPublicOptions = { scale?: number; stabilize?: boolean; normalizeStatusBar?: boolean; + captureBackend?: ScreenshotCaptureBackend; }; export type ScreenshotRuntimeOptions = { @@ -286,6 +308,7 @@ export type ScreenshotRuntimeOptions = { scale?: number; stabilize?: boolean; normalizeStatusBar?: boolean; + captureBackend?: ScreenshotCaptureBackend; }; export function screenshotOptionsFromFlags( @@ -301,6 +324,7 @@ export function screenshotOptionsFromFlags( stabilize: flags?.screenshotNoStabilize === undefined ? undefined : !flags.screenshotNoStabilize, normalizeStatusBar: flags?.screenshotNormalizeStatusBar, + captureBackend: flags?.screenshotCaptureBackend, }); } @@ -321,6 +345,7 @@ export function screenshotFlagsFromOptions( (options.stabilize === undefined ? undefined : !options.stabilize), screenshotNormalizeStatusBar: options.screenshotNormalizeStatusBar ?? options.normalizeStatusBar, + screenshotCaptureBackend: options.screenshotCaptureBackend ?? options.captureBackend, }); } diff --git a/packages/daemon/src/core/__tests__/context.test.ts b/packages/daemon/src/core/__tests__/context.test.ts index 92ead9909a..5718b9cc26 100644 --- a/packages/daemon/src/core/__tests__/context.test.ts +++ b/packages/daemon/src/core/__tests__/context.test.ts @@ -8,9 +8,9 @@ test('contextFromFlags propagates back mode into the dispatch context', () => { assert.equal(context.backMode, 'system'); }); -test('contextFromFlags forwards the Maestro runner screenshot backend', () => { +test('contextFromFlags forwards the runner screenshot capture backend', () => { const context = contextFromFlags('/tmp/agent-device.log', { - maestro: { screenshotCaptureBackend: 'runner' }, + screenshotCaptureBackend: 'runner', }); assert.equal(context.screenshotCaptureBackend, 'runner'); }); diff --git a/packages/daemon/src/core/context.ts b/packages/daemon/src/core/context.ts index 0748c89ee8..83ed12b593 100644 --- a/packages/daemon/src/core/context.ts +++ b/packages/daemon/src/core/context.ts @@ -39,7 +39,6 @@ export function contextFromFlags( logPath, traceLogPath, ...dispatchContextFlags(flags), - screenshotCaptureBackend: flags?.maestro?.screenshotCaptureBackend, ...screenshotFlagsFromOptions(flags), clickButton: resolveClickButton(flags), }; diff --git a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts index 2866b2bd12..e7b046f238 100644 --- a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts +++ b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts @@ -251,6 +251,40 @@ test('rejects --pixel-density outside iOS-family simulators before inspecting ow expect(fixture.binds).toHaveLength(0); }); +test('refuses --capture-backend runner outside Apple targets, with --pixel-density, and any other backend, before inspecting owner facts', async () => { + const refused = async ( + device: SessionState['device'], + // The wire flags are untyped, so a request can name a backend the type does not list. + flags: Record, + code: string, + ): Promise => { + const fixture = screenshotRuntimeFixture(); + await expect( + resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req: screenshotRequest({ + positionals: ['/tmp/runner.png'], + flags: flags as DaemonRequest['flags'], + }), + session: makeSession('default', { device }), + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }), + ).rejects.toMatchObject({ code }); + expect(fixture.binds).toHaveLength(0); + }; + + await refused(ANDROID_EMULATOR, { screenshotCaptureBackend: 'runner' }, 'UNSUPPORTED_OPERATION'); + await refused( + IOS_SIMULATOR, + { screenshotCaptureBackend: 'runner', screenshotPixelDensity: 3 }, + 'INVALID_ARGS', + ); + await refused(IOS_SIMULATOR, { screenshotCaptureBackend: 'simctl' }, 'INVALID_ARGS'); +}); + const CROP_TREE_ROOT = { index: 0, depth: 0, diff --git a/packages/daemon/src/execution/screenshot-runtime.ts b/packages/daemon/src/execution/screenshot-runtime.ts index 1b0196ab1c..8f2e14c802 100644 --- a/packages/daemon/src/execution/screenshot-runtime.ts +++ b/packages/daemon/src/execution/screenshot-runtime.ts @@ -2,12 +2,13 @@ import { expandSessionPath } from '@agent-device/host-kit/session-paths'; import type { CommandFlags } from '@agent-device/contracts/command'; import { retiredScreenshotMaxSizeFlagError, + SCREENSHOT_CAPTURE_BACKENDS, screenshotFlagsFromOptions, screenshotOptionsFromFlags, } from '@agent-device/contracts/capture'; import type { DeviceRotation } from '@agent-device/contracts/device'; import type { ScreenshotRuntimeExecution } from '@agent-device/contracts/screenshot-runtime'; -import { isIosFamily, publicPlatformString } from '@agent-device/kernel/device'; +import { isApplePlatform, isIosFamily, publicPlatformString } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import type { ScreenshotOverlayRef } from '@agent-device/kernel/snapshot'; import { promises as fs } from 'node:fs'; @@ -64,6 +65,7 @@ export async function resolveScreenshotGenericExecution( const retiredMaxSize = retiredScreenshotMaxSizeFlagError('screenshot', req.flags); if (retiredMaxSize) throw new AppError('INVALID_ARGS', retiredMaxSize); assertSupportedScreenshotPixelDensity(session.device, req.flags?.screenshotPixelDensity); + assertSupportedScreenshotCaptureBackend(session.device, req.flags); const request = readScreenshotRequest(req, params.hostCwd); const cropOn = @@ -317,6 +319,33 @@ async function annotateScreenshotWithSessionRefs( }); } +/** The runner capture exists on Apple targets only, and it encodes the device's own density. */ +function assertSupportedScreenshotCaptureBackend( + device: SessionState['device'], + flags: DaemonRequest['flags'], +): void { + const backend: unknown = flags?.screenshotCaptureBackend; + if (backend === undefined) return; + if (!(SCREENSHOT_CAPTURE_BACKENDS as readonly unknown[]).includes(backend)) { + throw new AppError( + 'INVALID_ARGS', + `--capture-backend takes ${SCREENSHOT_CAPTURE_BACKENDS.join(', ')}`, + ); + } + if (!isApplePlatform(device.platform)) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + '--capture-backend runner is supported only on Apple targets', + ); + } + if (flags?.screenshotPixelDensity !== undefined) { + throw new AppError( + 'INVALID_ARGS', + '--pixel-density cannot be combined with --capture-backend runner: the runner captures at the device density', + ); + } +} + /** * `screenshot [path]` and `--out ` are the same destination expressed two ways, and both are * user-typed paths resolved against the client's cwd. The normalized form is also what the session @@ -372,7 +401,7 @@ function createBoundScreenshotBackend( skipIosSimulatorBootCheck: dispatchContext.skipIosSimulatorBootCheck ?? (isIosFamily(session.device) && session.device.kind === 'simulator'), - captureBackend: dispatchContext.screenshotCaptureBackend, + captureBackend: resolved.captureBackend, }, execution: screenshotExecutionFromContext(dispatchContext), }); diff --git a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port.test.ts b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port.test.ts index 2fad6af325..3aec4dc2ae 100644 --- a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port.test.ts +++ b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port.test.ts @@ -661,9 +661,7 @@ test('waitForAnimationToEnd uses two unstabilized screenshot captures', async () expect(requests.map(({ command }) => command)).toEqual(['screenshot', 'screenshot']); expect(requests.every(({ flags }) => flags?.screenshotNoStabilize === true)).toBe(true); - expect( - requests.every(({ flags }) => flags?.maestro?.screenshotCaptureBackend === undefined), - ).toBe(true); + expect(requests.every(({ flags }) => flags?.screenshotCaptureBackend === undefined)).toBe(true); expect(result).not.toHaveProperty('visualStabilityReached'); }); @@ -692,9 +690,7 @@ test('waitForAnimationToEnd uses the persistent runner capture backend on iOS', expect(requests).toHaveLength(2); expect(requests.every(({ flags }) => flags?.screenshotNoStabilize === true)).toBe(true); - expect(requests.every(({ flags }) => flags?.maestro?.screenshotCaptureBackend === 'runner')).toBe( - true, - ); + expect(requests.every(({ flags }) => flags?.screenshotCaptureBackend === 'runner')).toBe(true); }); test('waitForAnimationToEnd between two taps does not throw a stability-generation mismatch', async () => { diff --git a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-public-operation.test.ts b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-public-operation.test.ts index 15ec97c117..7487881bb8 100644 --- a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-public-operation.test.ts +++ b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-public-operation.test.ts @@ -192,7 +192,7 @@ describe('Maestro public operation projection', () => { positionals: ['/tmp/animation-runner.png'], flags: { screenshotNoStabilize: true, - maestro: { screenshotCaptureBackend: 'runner' }, + screenshotCaptureBackend: 'runner', }, }, }, diff --git a/packages/maestro/src/daemon-port/daemon-runtime-public-operation.ts b/packages/maestro/src/daemon-port/daemon-runtime-public-operation.ts index 36cddf63ba..e5ea3df359 100644 --- a/packages/maestro/src/daemon-port/daemon-runtime-public-operation.ts +++ b/packages/maestro/src/daemon-port/daemon-runtime-public-operation.ts @@ -398,7 +398,7 @@ function projectCaptureOperation( flags: { ...(operation.stabilize === false ? { screenshotNoStabilize: true } : {}), ...(operation.captureBackend === 'runner' - ? { maestro: { screenshotCaptureBackend: 'runner' as const } } + ? { screenshotCaptureBackend: 'runner' as const } : {}), }, } diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 7bc99ffd24..55e83e8213 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -380,7 +380,12 @@ function summarizeProviderScenarioFlagExclusions() { { name: 'Apple simulator screenshot rendering options', owner: 'iOS platform and screenshot-diff runtime tests', - keys: ['screenshotNormalizeStatusBar', 'screenshotPixelDensity', 'screenshotScreenMask'], + keys: [ + 'screenshotNormalizeStatusBar', + 'screenshotPixelDensity', + 'screenshotScreenMask', + 'screenshotCaptureBackend', + ], }, { // Reading accessibility custom actions has no provider-scenario surface: diff --git a/src/cli/parser/__tests__/args-parse-interaction.test.ts b/src/cli/parser/__tests__/args-parse-interaction.test.ts index 0aa7872877..ca383234d6 100644 --- a/src/cli/parser/__tests__/args-parse-interaction.test.ts +++ b/src/cli/parser/__tests__/args-parse-interaction.test.ts @@ -265,6 +265,8 @@ test('parseArgs recognizes screenshot flags', () => { '0.3', '--no-stabilize', '--normalize-status-bar', + '--capture-backend', + 'runner', ], { strictFlags: true, @@ -275,6 +277,7 @@ test('parseArgs recognizes screenshot flags', () => { assert.equal(parsed.flags.screenshotPixelDensity, 2); assert.equal(parsed.flags.screenshotScreenMask, 'black'); assert.equal(parsed.flags.screenshotFullscreen, true); + assert.equal(parsed.flags.screenshotCaptureBackend, 'runner'); assert.equal(parsed.flags.screenshotScale, 0.3); assert.equal(parsed.flags.screenshotNoStabilize, true); assert.equal(parsed.flags.screenshotNormalizeStatusBar, true); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index cf533bfb00..1776d3c902 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1027,6 +1027,7 @@ agent-device screenshot page.png --overlay-refs # Draw current @eN refs and tar agent-device screenshot page.png --crop-on 'label="Save"' # Crop the capture to the frame the selector resolves on the same screen agent-device screenshot baseline.png --normalize-status-bar # Normalize iOS simulator or ADB-backed Android chrome for reusable diff baselines agent-device screenshot baseline.png --screen-mask black --normalize-status-bar # Capture an iOS simulator baseline with a black screen outline +agent-device screenshot page.png --capture-backend runner # Apple: capture through the XCTest runner (no simulator-drawn Dynamic Island) agent-device screenshot page.png --platform web --fullscreen # On web, --fullscreen/--full/-f captures the entire document agent-device viewport 1280 900 --platform web # Resize the active web viewport for fixed-layout or 100vh apps agent-device screenshot textedit.png # App-session window capture on macOS @@ -1051,6 +1052,7 @@ agent-device record stop # Stop active recording - `--overlay-refs` takes a fresh snapshot and burns visible `@eN` refs and their rectangles into the PNG. With `--scale`, refs are drawn at the final size, so avoid scales where labels become unreadable. In `--json` mode each ref also has a screenshot-space `center` for `press ` fallback. - `--crop-on ` crops the capture to the frame the selector resolves on the same screen. It works only on iOS simulators, Android emulators, and physical iPhones. A physical iPhone crop is verified only in portrait using CoreDevice/devicectl; the XCTest backend and runner fallback are not accepted. Physical iPads remain rejected for crop, and landscape iPhone crop remains unverified. It needs exactly one framed node, cannot be combined with `--overlay-refs` or `--fullscreen`, and warns when the frame is clipped. The crop is re-encoded, so do not byte-compare it with an older crop. - `--fullscreen` on macOS applies only to app sessions. Other surfaces (`desktop`, `menubar`, `frontmost-app`) always capture the main display and refuse `--fullscreen` with `INVALID_ARGS`. +- `--capture-backend runner` captures an Apple target through the XCTest runner (`XCUIScreen`), as XCTest-based tools such as Maestro do, instead of `simctl io screenshot`. On iOS simulators with a Dynamic Island, `simctl` draws the island into every capture and no `--mask` policy removes it; the runner capture leaves it out. The image is at the device's own density, so `--pixel-density` is refused with it (`INVALID_ARGS`), and other platforms refuse it with `UNSUPPORTED_OPERATION`. Node and MCP take `captureBackend: 'runner'`. - `--normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for reusable baselines. Each status-bar read or preparation command has a 10-second timeout. Each restore command has a 30-second timeout, so clearing and reapplying overrides can take up to 60 seconds. The request timeout includes the wait and can expire before these steps finish. - `--screen-mask black` fills the iOS simulator screen outline, including rounded corners, with black for this capture. Node and MCP use `screenMask: "black"`. Omit it for the default image. Support is limited to handheld iOS/iPadOS simulators. It requires simctl capture; physical devices, tvOS and visionOS simulators, other platforms, provider captures, and XCTest runner captures are refused. If simctl fails, a masked request fails instead of falling back to an unmasked runner capture. This option does not redact app content, remove a Dynamic Island drawn into the framebuffer, or change comparison thresholds. - On an iOS simulator, a screenshot waits until no other screenshot or `settings wifi` / `settings airplane` write is using that simulator's status bar, so overlapping captures cannot see each other's temporary chrome. A request you cancel while it is waiting fails with the cancellation instead of capturing. From 42cb771e0240ccd877869270600472600974408a Mon Sep 17 00:00:00 2001 From: Vitalii Yehorov Date: Wed, 4 Nov 2026 07:45:40 +0100 Subject: [PATCH 2/8] fix(screenshot): refuse a runner capture on a surface the helper owns The screenshot admission accepted --capture-backend runner on every Apple surface, but runAppleScreenshot returns the macOS helper capture before it reaches the runner branch, so the request reported success with another backend. The Apple binding now declares runner support from the selected surface and app backend, and assertScreenshotCaptureSupport refuses it. --normalize-status-bar was dropped the same way, so it is refused with the runner like --pixel-density. The MCP input gains its schema description. Co-Authored-By: Claude Opus 5.5 --- .../src/capture/screenshot-options.test.ts | 11 ++++ .../command-surface/src/capture/screenshot.ts | 5 +- packages/contracts/src/client-capture.ts | 3 +- packages/contracts/src/screenshot-runtime.ts | 8 +++ packages/contracts/src/screenshot.ts | 2 +- packages/daemon/src/core/dispatch-context.ts | 6 +- .../screenshot-runtime-execution.test.ts | 28 +++++++++ .../src/execution/screenshot-runtime.ts | 6 +- .../interactor-screenshot-backend.test.ts | 25 ++++++++ packages/platform-apple/src/interactor.ts | 2 +- packages/platform-apple/src/runtime.ts | 10 +++- .../src/screenshot-support.test.ts | 59 +++++++++++++++++++ .../platform-apple/src/screenshot-support.ts | 3 + scripts/integration-progress-model.ts | 1 + website/docs/docs/commands.md | 2 +- 15 files changed, 157 insertions(+), 14 deletions(-) diff --git a/packages/command-surface/src/capture/screenshot-options.test.ts b/packages/command-surface/src/capture/screenshot-options.test.ts index 3fcf2dfa07..f07a6a60b6 100644 --- a/packages/command-surface/src/capture/screenshot-options.test.ts +++ b/packages/command-surface/src/capture/screenshot-options.test.ts @@ -176,6 +176,17 @@ test('screenshot script flags use the shared recorded flag contract', () => { ]); }); +test('the capture backend stays out of recorded scripts', () => { + const parts: string[] = []; + appendScreenshotScriptFlags(parts, { screenshotCaptureBackend: 'runner' }); + assert.deepEqual(parts, []); + assert.equal( + SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS.find(({ key }) => key === 'screenshotCaptureBackend') + ?.recorded, + false, + ); +}); + test('explicit screenshot stabilization survives SDK projection and script recording', () => { const projected = screenshotFlagsFromOptions({ stabilize: true }); assert.deepEqual(screenshotOptionsFromFlags(projected), { stabilize: true }); diff --git a/packages/command-surface/src/capture/screenshot.ts b/packages/command-surface/src/capture/screenshot.ts index 17e1b81b5c..94b4f15d48 100644 --- a/packages/command-surface/src/capture/screenshot.ts +++ b/packages/command-surface/src/capture/screenshot.ts @@ -51,7 +51,10 @@ const screenshotCommandMetadata = defineFieldCommandMetadata( maxSize: retiredField(RETIRED_SCREENSHOT_MAX_SIZE.migration.screenshot), stabilize: booleanField(), normalizeStatusBar: booleanField(), - captureBackend: enumField(SCREENSHOT_CAPTURE_BACKENDS), + captureBackend: enumField( + SCREENSHOT_CAPTURE_BACKENDS, + 'Capture through the XCTest runner instead of the platform default, which leaves out the simulator-drawn Dynamic Island. Apple targets only; refused on a macOS surface the helper captures, and with pixelDensity, screenMask or normalizeStatusBar.', + ), surface: enumField(SESSION_SURFACES), }, ); diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index ecf4cde0d7..3a1952b7fd 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -1,7 +1,6 @@ -import type { ScreenshotScreenMask } from './screenshot.ts'; +import type { ScreenshotCaptureBackend, ScreenshotScreenMask } from './screenshot.ts'; // The public API vocabulary for snapshot, screenshot and diff capture. -import type { ScreenshotCaptureBackend } from './screenshot.ts'; import type { SessionSurface } from './session-surface.ts'; import type { PublicSnapshotCaptureAnnotations } from './snapshot-capture-annotations.ts'; import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts'; diff --git a/packages/contracts/src/screenshot-runtime.ts b/packages/contracts/src/screenshot-runtime.ts index f63f0c65d6..b3c00c44c3 100644 --- a/packages/contracts/src/screenshot-runtime.ts +++ b/packages/contracts/src/screenshot-runtime.ts @@ -36,6 +36,7 @@ export type ScreenshotRuntimeOperations = Readonly<{ export type ScreenshotCaptureCapabilities = Readonly<{ screenMask?: boolean; + runner?: boolean; }>; type ScreenshotCaptureBindingOptions = Readonly<{ @@ -47,6 +48,13 @@ export async function assertScreenshotCaptureSupport( options: Readonly | undefined, capabilities: ScreenshotCaptureCapabilities = {}, ): Promise { + if (options?.captureBackend === 'runner' && capabilities.runner === false) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + '--capture-backend runner cannot capture this surface: the macOS helper owns it', + { reason: 'screenshot-capture-backend-unsupported' }, + ); + } if (options?.screenMask === undefined) return; const { readScreenshotScreenMask } = await import('./screenshot.ts'); const screenMask = readScreenshotScreenMask(options?.screenMask); diff --git a/packages/contracts/src/screenshot.ts b/packages/contracts/src/screenshot.ts index d4e661a1e5..77b0186f22 100644 --- a/packages/contracts/src/screenshot.ts +++ b/packages/contracts/src/screenshot.ts @@ -214,7 +214,7 @@ export const SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS: readonly ScreenshotSpecificFl enumValues: SCREENSHOT_CAPTURE_BACKENDS, usageLabel: '--capture-backend runner', usageDescription: - 'Screenshot: on Apple targets capture through the XCTest runner (XCUIScreen), which leaves out the simulator-drawn Dynamic Island, instead of simctl; output is at the device density, so --pixel-density is refused with it', + 'Screenshot: on Apple targets capture through the XCTest runner (XCUIScreen), which leaves out the simulator-drawn Dynamic Island, instead of simctl; refused on macOS surfaces the helper captures, and with --pixel-density, --screen-mask and --normalize-status-bar', projectConfig: true, recorded: false, }, diff --git a/packages/daemon/src/core/dispatch-context.ts b/packages/daemon/src/core/dispatch-context.ts index 3625426184..46a2bc09fa 100644 --- a/packages/daemon/src/core/dispatch-context.ts +++ b/packages/daemon/src/core/dispatch-context.ts @@ -20,9 +20,9 @@ import type { Point } from '@agent-device/kernel/snapshot'; * anything this list does not. * * The flags that DO change on the way through stay hand-written at the mapper, - * where the decision lives: `clickButton` (resolved from several flags), the - * screenshot family (`screenshotFlagsFromOptions`), and the maestro-nested - * capture backend. Order follows the flag order in `CliFlags`. + * where the decision lives: `clickButton` (resolved from several flags) and the + * screenshot family (`screenshotFlagsFromOptions`), which includes the capture + * backend. Order follows the flag order in `CliFlags`. */ // Exported only because `DispatchContextFlags` says `typeof` it. // fallow-ignore-next-line unused-export diff --git a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts index e7b046f238..1cf953bf5f 100644 --- a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts +++ b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts @@ -12,6 +12,7 @@ import { mkdtempForTestSync, } from '@agent-device/test-kit'; import { makeSession } from '../__tests__/support/session-factories.ts'; +import { contextFromFlags } from '../core/context.ts'; import type { GenericPlatformExecutionParams } from '../core/generic-leaf-execution.ts'; import { resolveScreenshotGenericExecution } from './screenshot-runtime.ts'; import { @@ -282,9 +283,36 @@ test('refuses --capture-backend runner outside Apple targets, with --pixel-densi { screenshotCaptureBackend: 'runner', screenshotPixelDensity: 3 }, 'INVALID_ARGS', ); + await refused( + IOS_SIMULATOR, + { screenshotCaptureBackend: 'runner', screenshotNormalizeStatusBar: true }, + 'INVALID_ARGS', + ); await refused(IOS_SIMULATOR, { screenshotCaptureBackend: 'simctl' }, 'INVALID_ARGS'); }); +test('a --capture-backend runner request reaches the bound capture', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('ios', { device: IOS_SIMULATOR }); + const req = screenshotRequest({ + positionals: [path.join(mkdtempForTestSync('agent-device-runner-capture'), 'runner.png')], + flags: { screenshotCaptureBackend: 'runner' }, + }); + + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + await executeResult(resolved, session, req, contextFromFlags('/tmp/daemon.log', req.flags)); + expect(fixture.captureScreenshot.mock.calls[0]?.[0]?.options?.captureBackend).toBe('runner'); +}); + const CROP_TREE_ROOT = { index: 0, depth: 0, diff --git a/packages/daemon/src/execution/screenshot-runtime.ts b/packages/daemon/src/execution/screenshot-runtime.ts index 8f2e14c802..7e53621e9b 100644 --- a/packages/daemon/src/execution/screenshot-runtime.ts +++ b/packages/daemon/src/execution/screenshot-runtime.ts @@ -319,7 +319,7 @@ async function annotateScreenshotWithSessionRefs( }); } -/** The runner capture exists on Apple targets only, and it encodes the device's own density. */ +/** The runner capture exists on Apple targets only, and skips the simctl capture pipeline. */ function assertSupportedScreenshotCaptureBackend( device: SessionState['device'], flags: DaemonRequest['flags'], @@ -338,10 +338,10 @@ function assertSupportedScreenshotCaptureBackend( '--capture-backend runner is supported only on Apple targets', ); } - if (flags?.screenshotPixelDensity !== undefined) { + if (flags?.screenshotPixelDensity !== undefined || flags?.screenshotNormalizeStatusBar) { throw new AppError( 'INVALID_ARGS', - '--pixel-density cannot be combined with --capture-backend runner: the runner captures at the device density', + '--pixel-density and --normalize-status-bar cannot be combined with --capture-backend runner: they belong to the simctl capture', ); } } diff --git a/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts b/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts index 57e6215928..a18f45e750 100644 --- a/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts @@ -10,7 +10,13 @@ vi.mock('../core/screenshot.ts', async (importOriginal) => { }; }); +vi.mock('../os/macos/helper.ts', async (importOriginal) => ({ + ...(await importOriginal()), + runMacOsScreenshotAction: vi.fn(), +})); + import { createAppleInteractor } from '../interactor.ts'; +import { runMacOsScreenshotAction } from '../os/macos/helper.ts'; import { captureScreenshotViaRunner, screenshotIos } from '../core/screenshot.ts'; const device: DeviceInfo = { @@ -88,3 +94,22 @@ test.each([ expect(screenshotIos).not.toHaveBeenCalled(); expect(captureScreenshotViaRunner).not.toHaveBeenCalled(); }); + +test('an explicit runner capture never succeeds through the macOS helper', async () => { + const macOs: DeviceInfo = { ...device, appleOs: 'macos', kind: 'device', target: 'desktop' }; + const refused = async (surface: 'desktop' | undefined, env: NodeJS.ProcessEnv) => { + const interactor = createAppleInteractor(macOs, {}, { env, cwd: process.cwd() }); + await expect( + interactor.screenshot('/tmp/refused.png', { captureBackend: 'runner', surface }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-capture-backend-unsupported' }, + }); + }; + + await refused('desktop', {}); + await refused(undefined, { AGENT_DEVICE_MACOS_APP_BACKEND: 'native' }); + + expect(runMacOsScreenshotAction).not.toHaveBeenCalled(); + expect(captureScreenshotViaRunner).not.toHaveBeenCalled(); +}); diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index a645f7927f..2efbfb0d68 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -555,7 +555,7 @@ async function runAppleScreenshot( ): Promise { await assertScreenshotCaptureSupport( options, - appleScreenshotCaptureCapabilities(device, options), + appleScreenshotCaptureCapabilities(device, options, helper), ); if (helper) { if (options.fullscreen) { diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index 19e2cd3c5b..efe68a92f6 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -63,7 +63,7 @@ import { createAppleSystemOperations } from './system/runtime.ts'; import { appleFoldableFacts, createAppleFoldableOperations } from './foldable/runtime.ts'; import { bindAppleFindTextRuntime, bindAppleSnapshotRuntime } from './runtime-snapshot.ts'; import { bindAppleQuerySelectorRuntime } from './runtime-selector-query.ts'; -import type { MacOsAppBackend } from '@agent-device/contracts/session'; +import { macOsHelperSurface, type MacOsAppBackend } from '@agent-device/contracts/session'; import { hostMacOsAppBackend } from './os/macos/app-backend.ts'; import { macOsNativeBackendFacts } from './os/macos/native-backend-facts.ts'; import { createAppleSnapshotRoute } from './snapshot-route.ts'; @@ -325,7 +325,13 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR signal: request.scope.signal, resolveInteractor: host.localInteractors.resolve, captureCapabilities: (options) => - appleScreenshotCaptureCapabilities(request.device, options), + appleScreenshotCaptureCapabilities( + request.device, + options, + isMacOs(request.device) + ? macOsHelperSurface(options.surface, macOsAppBackend(request.device)) + : undefined, + ), }), ), ...whenAdmitted(facts.operations.focusPoint, () => diff --git a/packages/platform-apple/src/screenshot-support.test.ts b/packages/platform-apple/src/screenshot-support.test.ts index 6f87abb83d..2160fa5300 100644 --- a/packages/platform-apple/src/screenshot-support.test.ts +++ b/packages/platform-apple/src/screenshot-support.test.ts @@ -1,6 +1,7 @@ import { expect, test, vi } from 'vitest'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { Interactor, ScreenshotOptions } from '@agent-device/contracts/interactor-types'; +import type { MacOsAppBackend, SessionSurface } from '@agent-device/contracts/session'; import { createApplePlatformRuntime } from './runtime.ts'; import { platformRuntimeHostFixture } from './runtime.fixtures.ts'; @@ -57,3 +58,61 @@ test.each([ } }, ); + +const macOs: DeviceInfo = { + platform: 'apple', + appleOs: 'macos', + id: 'host-macos-local', + name: 'Mac', + kind: 'device', + target: 'desktop', + booted: true, +}; + +test.each([ + [simulator, undefined, undefined, true], + [{ ...simulator, kind: 'device' }, undefined, undefined, true], + [{ ...simulator, appleOs: 'tvos', target: 'tv' }, undefined, undefined, true], + [macOs, undefined, undefined, true], + [macOs, 'app', 'xctest', true], + [macOs, undefined, 'native', false], + [macOs, 'app', 'native', false], + [macOs, 'desktop', undefined, false], + [macOs, 'menubar', undefined, false], + [macOs, 'frontmost-app', undefined, false], +] satisfies [DeviceInfo, SessionSurface | undefined, MacOsAppBackend | undefined, boolean][])( + 'the bound Apple capture declares runner support for %j on surface %s with app backend %s: %s', + async (device, surface, appBackend, supported) => { + const screenshot = vi.fn(async () => ({})); + const resolve = vi.fn(async () => ({ screenshot }) as unknown as Interactor); + const host = { + ...platformRuntimeHostFixture(), + hostEnvironment: { + env: appBackend ? { AGENT_DEVICE_MACOS_APP_BACKEND: appBackend } : {}, + cwd: process.cwd(), + }, + localInteractors: { resolve }, + }; + const binding = await createApplePlatformRuntime(host).bind({ + device, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + const options = { captureBackend: 'runner', surface } as const; + const result = binding.operations.captureScreenshot!({ outPath: '/tmp/r.png', options }); + if (supported) { + await expect(result).resolves.toEqual({}); + expect(screenshot).toHaveBeenCalledWith('/tmp/r.png', options); + } else { + await expect(result).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-capture-backend-unsupported' }, + }); + expect(resolve).not.toHaveBeenCalled(); + } + }, +); diff --git a/packages/platform-apple/src/screenshot-support.ts b/packages/platform-apple/src/screenshot-support.ts index 9a24a726f8..8fbcafae10 100644 --- a/packages/platform-apple/src/screenshot-support.ts +++ b/packages/platform-apple/src/screenshot-support.ts @@ -1,4 +1,5 @@ import { isHandheldAppleSimulator, type DeviceInfo } from '@agent-device/kernel/device'; +import type { MacOsHelperSurface } from '@agent-device/contracts/session'; import { resolveAppleToolProvider } from './core/tool-provider.ts'; import type { ScreenshotCaptureCapabilities, @@ -8,11 +9,13 @@ import type { export function appleScreenshotCaptureCapabilities( device: DeviceInfo, options: Pick, + helper?: MacOsHelperSurface, ): ScreenshotCaptureCapabilities { return { screenMask: isHandheldAppleSimulator(device) && options.captureBackend !== 'runner' && resolveAppleToolProvider().simulatorScreenshotBackend !== 'runner', + runner: helper === undefined, }; } diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 55e83e8213..9ba042188c 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -378,6 +378,7 @@ function summarizeProviderScenarioFlagExclusions() { keys: ['waitMs'], }, { + // A provider scenario has no XCTest runner capture to select, so these stay on the owner's tests. name: 'Apple simulator screenshot rendering options', owner: 'iOS platform and screenshot-diff runtime tests', keys: [ diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 1776d3c902..2c57d91e30 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1052,7 +1052,7 @@ agent-device record stop # Stop active recording - `--overlay-refs` takes a fresh snapshot and burns visible `@eN` refs and their rectangles into the PNG. With `--scale`, refs are drawn at the final size, so avoid scales where labels become unreadable. In `--json` mode each ref also has a screenshot-space `center` for `press ` fallback. - `--crop-on ` crops the capture to the frame the selector resolves on the same screen. It works only on iOS simulators, Android emulators, and physical iPhones. A physical iPhone crop is verified only in portrait using CoreDevice/devicectl; the XCTest backend and runner fallback are not accepted. Physical iPads remain rejected for crop, and landscape iPhone crop remains unverified. It needs exactly one framed node, cannot be combined with `--overlay-refs` or `--fullscreen`, and warns when the frame is clipped. The crop is re-encoded, so do not byte-compare it with an older crop. - `--fullscreen` on macOS applies only to app sessions. Other surfaces (`desktop`, `menubar`, `frontmost-app`) always capture the main display and refuse `--fullscreen` with `INVALID_ARGS`. -- `--capture-backend runner` captures an Apple target through the XCTest runner (`XCUIScreen`), as XCTest-based tools such as Maestro do, instead of `simctl io screenshot`. On iOS simulators with a Dynamic Island, `simctl` draws the island into every capture and no `--mask` policy removes it; the runner capture leaves it out. The image is at the device's own density, so `--pixel-density` is refused with it (`INVALID_ARGS`), and other platforms refuse it with `UNSUPPORTED_OPERATION`. Node and MCP take `captureBackend: 'runner'`. +- `--capture-backend runner` captures an Apple target through the XCTest runner (`XCUIScreen`), as XCTest-based tools such as Maestro do, instead of `simctl io screenshot`. On iOS simulators with a Dynamic Island, `simctl` draws the island into every capture and no `--mask` policy removes it; the runner capture leaves it out. The capture skips the simctl pipeline, so `--pixel-density` and `--normalize-status-bar` are refused with it (`INVALID_ARGS`). Other platforms, and macOS surfaces the helper captures (`desktop`, `menubar`, `frontmost-app`, and app sessions with the native app backend), refuse it with `UNSUPPORTED_OPERATION`. Node and MCP take `captureBackend: 'runner'`. - `--normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for reusable baselines. Each status-bar read or preparation command has a 10-second timeout. Each restore command has a 30-second timeout, so clearing and reapplying overrides can take up to 60 seconds. The request timeout includes the wait and can expire before these steps finish. - `--screen-mask black` fills the iOS simulator screen outline, including rounded corners, with black for this capture. Node and MCP use `screenMask: "black"`. Omit it for the default image. Support is limited to handheld iOS/iPadOS simulators. It requires simctl capture; physical devices, tvOS and visionOS simulators, other platforms, provider captures, and XCTest runner captures are refused. If simctl fails, a masked request fails instead of falling back to an unmasked runner capture. This option does not redact app content, remove a Dynamic Island drawn into the framebuffer, or change comparison thresholds. - On an iOS simulator, a screenshot waits until no other screenshot or `settings wifi` / `settings airplane` write is using that simulator's status bar, so overlapping captures cannot see each other's temporary chrome. A request you cancel while it is waiting fails with the cancellation instead of capturing. From f614bcb502ae30092ff8e06444200377ffd14934 Mon Sep 17 00:00:00 2001 From: Vitalii Yehorov Date: Wed, 4 Nov 2026 09:52:12 +0100 Subject: [PATCH 3/8] fix(screenshot): make every capture owner declare its capture backends An owner that declared nothing accepted --capture-backend runner, so a provider iOS session such as Limrun reported success with a provider screenshot. Every screenshot binding now has to declare its backends and its own explanation, and the existing admission check refuses a backend the owner does not list. The daemon's Apple-only check moves into that one place. Maestro's comparison capture asked for the runner through the public flag, which would now fail on a provider. It goes back to its internal maestro.screenshotCaptureBackend, read as a preference: the runner where the owner declares it, otherwise the owner's default capture, as before. Co-Authored-By: Claude Opus 5.5 --- .../command-surface/src/capture/screenshot.ts | 2 +- packages/contracts/src/command-flags.ts | 1 + packages/contracts/src/interactor-types.ts | 2 + .../src/local-interactor-operation-set.ts | 8 ++- .../contracts/src/screenshot-runtime.test.ts | 55 ++++++++++++++++++- packages/contracts/src/screenshot-runtime.ts | 39 ++++++++++--- packages/contracts/src/screenshot.ts | 2 +- .../daemon/src/core/__tests__/context.test.ts | 12 +++- packages/daemon/src/core/context.ts | 1 + packages/daemon/src/core/dispatch-context.ts | 7 ++- .../screenshot-runtime-execution.test.ts | 28 +++++++++- .../src/execution/screenshot-runtime.ts | 18 ++---- .../__tests__/daemon-runtime-port.test.ts | 8 ++- .../daemon-runtime-public-operation.test.ts | 2 +- .../daemon-runtime-public-operation.ts | 2 +- packages/platform-android/src/runtime.test.ts | 25 +++++++++ packages/platform-android/src/runtime.ts | 6 +- .../platform-apple/src/screenshot-support.ts | 3 +- packages/platform-harmonyos/src/runtime.ts | 6 +- packages/platform-linux/src/runtime.ts | 6 +- packages/platform-web/src/runtime.ts | 2 + .../src/interaction-operations.test.ts | 33 ++++++++++- .../src/interaction-operations.ts | 12 +++- .../src/interaction-operations.ts | 12 +++- .../src/platform-runtime.ts | 8 ++- packages/tv-cdp/src/runtime.ts | 8 ++- website/docs/docs/commands.md | 2 +- 27 files changed, 260 insertions(+), 50 deletions(-) diff --git a/packages/command-surface/src/capture/screenshot.ts b/packages/command-surface/src/capture/screenshot.ts index 94b4f15d48..d67287e69e 100644 --- a/packages/command-surface/src/capture/screenshot.ts +++ b/packages/command-surface/src/capture/screenshot.ts @@ -53,7 +53,7 @@ const screenshotCommandMetadata = defineFieldCommandMetadata( normalizeStatusBar: booleanField(), captureBackend: enumField( SCREENSHOT_CAPTURE_BACKENDS, - 'Capture through the XCTest runner instead of the platform default, which leaves out the simulator-drawn Dynamic Island. Apple targets only; refused on a macOS surface the helper captures, and with pixelDensity, screenMask or normalizeStatusBar.', + 'Capture through the XCTest runner instead of the platform default, which leaves out the simulator-drawn Dynamic Island. Local Apple targets only; refused on provider sessions and on a macOS surface the helper captures, and with pixelDensity, screenMask or normalizeStatusBar.', ), surface: enumField(SESSION_SURFACES), }, diff --git a/packages/contracts/src/command-flags.ts b/packages/contracts/src/command-flags.ts index 1986a47c17..7b56cc0e6d 100644 --- a/packages/contracts/src/command-flags.ts +++ b/packages/contracts/src/command-flags.ts @@ -22,6 +22,7 @@ export type MaestroRuntimeFlags = { allowNonHittableCoordinateFallback?: boolean; expectedTapPoint?: Point; prewarmRunnerBeforeOpen?: boolean; + screenshotCaptureBackend?: 'runner'; }; export type CommandFlags = Omit & { diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index dce79ef80b..04f956e8e4 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -69,6 +69,8 @@ export type ScreenshotOptions = { surface?: SessionSurface; skipIosSimulatorBootCheck?: boolean; captureBackend?: 'runner'; + /** Maestro's comparison capture: the runner where the owner declares it, else the owner's default. */ + preferredCaptureBackend?: 'runner'; }; /** diff --git a/packages/contracts/src/local-interactor-operation-set.ts b/packages/contracts/src/local-interactor-operation-set.ts index 7e49d48a6f..9e25554c35 100644 --- a/packages/contracts/src/local-interactor-operation-set.ts +++ b/packages/contracts/src/local-interactor-operation-set.ts @@ -26,6 +26,7 @@ export function bindLocalInteractorOperationSet( resolveInteractor: LocalInteractorOperationResolver; facts: RuntimeFacts['operations']; pause: (milliseconds: number) => Promise; + captureCapabilities: Parameters[0]['captureCapabilities']; }>, ): Partial { const { facts, pause } = params; @@ -35,7 +36,12 @@ export function bindLocalInteractorOperationSet( resolveInteractor: params.resolveInteractor, }; return { - ...(facts.captureScreenshot.available ? bindLocalScreenshotInteractor(resolver) : {}), + ...(facts.captureScreenshot.available + ? bindLocalScreenshotInteractor({ + ...resolver, + captureCapabilities: params.captureCapabilities, + }) + : {}), ...(facts.focusPoint.available ? bindLocalFocusInteractor(resolver) : {}), ...bindLocalGestureInteractor({ ...resolver, facts }), ...(facts.scrollDirection.available ? bindLocalScrollInteractor(resolver) : {}), diff --git a/packages/contracts/src/screenshot-runtime.test.ts b/packages/contracts/src/screenshot-runtime.test.ts index 89a6b2f857..cb48f3d7f9 100644 --- a/packages/contracts/src/screenshot-runtime.test.ts +++ b/packages/contracts/src/screenshot-runtime.test.ts @@ -3,6 +3,7 @@ import type { Interactor } from './interactor-types.ts'; import { bindLocalScreenshotInteractor, bindProviderScreenshotInteractor, + defaultCaptureOnly, screenshotRuntimeOperationFacts, } from './screenshot-runtime.ts'; @@ -24,7 +25,12 @@ test('a local binding hands the interactor the destination, options, and the req const resolveInteractor = vi.fn(async () => ({ screenshot }) as unknown as Interactor); const signal = new AbortController().signal; - const operations = bindLocalScreenshotInteractor({ device, signal, resolveInteractor }); + const operations = bindLocalScreenshotInteractor({ + device, + signal, + resolveInteractor, + captureCapabilities: defaultCaptureOnly('test'), + }); await operations.captureScreenshot({ outPath: '/tmp/out.png', options: { appBundleId: 'com.example.app', fullscreen: true }, @@ -47,6 +53,7 @@ test('a provider binding fails closed when its exact owner exposes no interactor device, signal: new AbortController().signal, resolveInteractor: () => undefined, + captureCapabilities: defaultCaptureOnly('test'), }); await expect(operations.captureScreenshot({ outPath: '/tmp/out.png' })).rejects.toMatchObject({ @@ -63,6 +70,7 @@ test.each([bindLocalScreenshotInteractor, bindProviderScreenshotInteractor])( device: { ...device, platform: 'apple', appleOs: 'ios', kind: 'simulator' }, signal: new AbortController().signal, resolveInteractor, + captureCapabilities: defaultCaptureOnly('test'), }); await expect( operations.captureScreenshot({ @@ -92,3 +100,48 @@ test('the selected capture capability admits a mask for one invocation without c expect(screenshot.mock.lastCall).toEqual(['/tmp/capture.png', { screenMask }]); } }); + +test.each([bindLocalScreenshotInteractor, bindProviderScreenshotInteractor])( + 'an owner declaring no capture backend refuses an explicit one with its own explanation', + async (bind) => { + const resolveInteractor = vi.fn(); + const operations = bind({ + device, + signal: new AbortController().signal, + resolveInteractor, + captureCapabilities: defaultCaptureOnly('this owner has one capture path'), + }); + await expect( + operations.captureScreenshot({ + outPath: '/tmp/capture.png', + options: { captureBackend: 'runner' }, + }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + message: expect.stringContaining('this owner has one capture path'), + details: { reason: 'screenshot-capture-backend-unsupported' }, + }); + expect(resolveInteractor).not.toHaveBeenCalled(); + }, +); + +test.each([ + [['runner'] as const, { captureBackend: 'runner' }], + [[] as const, {}], +])( + 'a preferred backend selects a declared one and otherwise keeps the default capture: %j', + async (captureBackends, expected) => { + const screenshot = vi.fn(async () => ({})); + const operations = bindLocalScreenshotInteractor({ + device, + signal: new AbortController().signal, + resolveInteractor: async () => ({ screenshot }) as unknown as Interactor, + captureCapabilities: () => ({ captureBackends }), + }); + await operations.captureScreenshot({ + outPath: '/tmp/capture.png', + options: { preferredCaptureBackend: 'runner' }, + }); + expect(screenshot).toHaveBeenCalledWith('/tmp/capture.png', expected); + }, +); diff --git a/packages/contracts/src/screenshot-runtime.ts b/packages/contracts/src/screenshot-runtime.ts index b3c00c44c3..5c0c508deb 100644 --- a/packages/contracts/src/screenshot-runtime.ts +++ b/packages/contracts/src/screenshot-runtime.ts @@ -13,6 +13,7 @@ import type { ScreenshotOptions, } from './interactor-types.ts'; import type { RuntimeOperationFact } from './platform-runtime.ts'; +import type { ScreenshotCaptureBackend } from './screenshot.ts'; export type { ScreenshotCaptureFacts, ScreenshotOptions } from './interactor-types.ts'; @@ -36,22 +37,33 @@ export type ScreenshotRuntimeOperations = Readonly<{ export type ScreenshotCaptureCapabilities = Readonly<{ screenMask?: boolean; - runner?: boolean; + /** The backends a request can select on this owner; an absent list selects none. */ + captureBackends?: readonly ScreenshotCaptureBackend[]; + /** Why this owner cannot serve a selected backend. */ + captureBackendRefusal?: string; }>; type ScreenshotCaptureBindingOptions = Readonly<{ signal: AbortSignal; - captureCapabilities?: (options: Readonly) => ScreenshotCaptureCapabilities; + captureCapabilities: (options: Readonly) => ScreenshotCaptureCapabilities; }>; +/** The declaration of an owner that captures one way only, and why. */ +export function defaultCaptureOnly( + captureBackendRefusal: string, +): (options: Readonly) => ScreenshotCaptureCapabilities { + return () => ({ captureBackendRefusal }); +} + export async function assertScreenshotCaptureSupport( options: Readonly | undefined, capabilities: ScreenshotCaptureCapabilities = {}, ): Promise { - if (options?.captureBackend === 'runner' && capabilities.runner === false) { + const backend = options?.captureBackend; + if (backend !== undefined && !capabilities.captureBackends?.includes(backend)) { throw new AppError( 'UNSUPPORTED_OPERATION', - '--capture-backend runner cannot capture this surface: the macOS helper owns it', + `--capture-backend ${backend} is not supported: ${capabilities.captureBackendRefusal ?? 'this capture owner declares no such backend'}`, { reason: 'screenshot-capture-backend-unsupported' }, ); } @@ -78,6 +90,17 @@ export function screenshotRuntimeOperationFacts( return Object.freeze({ captureScreenshot: input.capture }); } +/** A preference, unlike `captureBackend`, keeps the owner's default where it declares no such backend. */ +function resolvePreferredCaptureBackend( + input: Readonly | undefined, + binding: ScreenshotCaptureBindingOptions, +): Readonly { + const { preferredCaptureBackend: preferred, ...options } = input ?? {}; + return preferred && binding.captureCapabilities(options).captureBackends?.includes(preferred) + ? { captureBackend: preferred, ...options } + : options; +} + /** * Captures one selected owner's interactor authority for the lifetime of a request binding. The * owner is already chosen by the time a binder is called, so each entry point supplies its own @@ -89,16 +112,14 @@ function bindScreenshotCapture( ): ScreenshotRuntimeOperations { return Object.freeze({ captureScreenshot: async (input: CaptureScreenshotInput) => { - await assertScreenshotCaptureSupport( - input.options, - binding.captureCapabilities?.(input.options ?? {}), - ); + const options = resolvePreferredCaptureBackend(input.options, binding); + await assertScreenshotCaptureSupport(options, binding.captureCapabilities(options)); const interactor = await resolveInteractor({ ...input.execution, appBundleId: input.options?.appBundleId, signal: binding.signal, }); - return (await interactor.screenshot(input.outPath, input.options)) ?? {}; + return (await interactor.screenshot(input.outPath, options)) ?? {}; }, }); } diff --git a/packages/contracts/src/screenshot.ts b/packages/contracts/src/screenshot.ts index 77b0186f22..7b06ba3c6b 100644 --- a/packages/contracts/src/screenshot.ts +++ b/packages/contracts/src/screenshot.ts @@ -214,7 +214,7 @@ export const SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS: readonly ScreenshotSpecificFl enumValues: SCREENSHOT_CAPTURE_BACKENDS, usageLabel: '--capture-backend runner', usageDescription: - 'Screenshot: on Apple targets capture through the XCTest runner (XCUIScreen), which leaves out the simulator-drawn Dynamic Island, instead of simctl; refused on macOS surfaces the helper captures, and with --pixel-density, --screen-mask and --normalize-status-bar', + 'Screenshot: on Apple targets capture through the XCTest runner (XCUIScreen), which leaves out the simulator-drawn Dynamic Island, instead of simctl; refused on other platforms, provider sessions and macOS surfaces the helper captures, and with --pixel-density, --screen-mask and --normalize-status-bar', projectConfig: true, recorded: false, }, diff --git a/packages/daemon/src/core/__tests__/context.test.ts b/packages/daemon/src/core/__tests__/context.test.ts index 5718b9cc26..a9b82155cf 100644 --- a/packages/daemon/src/core/__tests__/context.test.ts +++ b/packages/daemon/src/core/__tests__/context.test.ts @@ -8,11 +8,17 @@ test('contextFromFlags propagates back mode into the dispatch context', () => { assert.equal(context.backMode, 'system'); }); -test('contextFromFlags forwards the runner screenshot capture backend', () => { - const context = contextFromFlags('/tmp/agent-device.log', { +test('contextFromFlags keeps an explicit capture backend apart from the Maestro preference', () => { + const explicit = contextFromFlags('/tmp/agent-device.log', { screenshotCaptureBackend: 'runner', }); - assert.equal(context.screenshotCaptureBackend, 'runner'); + assert.equal(explicit.screenshotCaptureBackend, 'runner'); + assert.equal(explicit.preferredScreenshotCaptureBackend, undefined); + const preferred = contextFromFlags('/tmp/agent-device.log', { + maestro: { screenshotCaptureBackend: 'runner' }, + }); + assert.equal(preferred.screenshotCaptureBackend, undefined); + assert.equal(preferred.preferredScreenshotCaptureBackend, 'runner'); }); test('contextFromFlags forwards scroll pixels from CLI flags', () => { diff --git a/packages/daemon/src/core/context.ts b/packages/daemon/src/core/context.ts index 83ed12b593..ca55f9207e 100644 --- a/packages/daemon/src/core/context.ts +++ b/packages/daemon/src/core/context.ts @@ -39,6 +39,7 @@ export function contextFromFlags( logPath, traceLogPath, ...dispatchContextFlags(flags), + preferredScreenshotCaptureBackend: flags?.maestro?.screenshotCaptureBackend, ...screenshotFlagsFromOptions(flags), clickButton: resolveClickButton(flags), }; diff --git a/packages/daemon/src/core/dispatch-context.ts b/packages/daemon/src/core/dispatch-context.ts index 46a2bc09fa..52b6ff7aec 100644 --- a/packages/daemon/src/core/dispatch-context.ts +++ b/packages/daemon/src/core/dispatch-context.ts @@ -20,9 +20,9 @@ import type { Point } from '@agent-device/kernel/snapshot'; * anything this list does not. * * The flags that DO change on the way through stay hand-written at the mapper, - * where the decision lives: `clickButton` (resolved from several flags) and the - * screenshot family (`screenshotFlagsFromOptions`), which includes the capture - * backend. Order follows the flag order in `CliFlags`. + * where the decision lives: `clickButton` (resolved from several flags), the + * screenshot family (`screenshotFlagsFromOptions`), and the maestro-nested + * capture backend. Order follows the flag order in `CliFlags`. */ // Exported only because `DispatchContextFlags` says `typeof` it. // fallow-ignore-next-line unused-export @@ -85,6 +85,7 @@ export type DispatchContext = ScreenshotDispatchFlags & traceLogPath?: string; runnerLeaseContext?: RunnerLogicalLeaseContext; screenshotCaptureBackend?: 'runner'; + preferredScreenshotCaptureBackend?: 'runner'; snapshotIncludeRects?: boolean; skipIosSimulatorBootCheck?: boolean; /** Maestro replay compatibility for coordinate-resolved fill targets. */ diff --git a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts index 1cf953bf5f..34c518e800 100644 --- a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts +++ b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts @@ -252,7 +252,7 @@ test('rejects --pixel-density outside iOS-family simulators before inspecting ow expect(fixture.binds).toHaveLength(0); }); -test('refuses --capture-backend runner outside Apple targets, with --pixel-density, and any other backend, before inspecting owner facts', async () => { +test('refuses --capture-backend runner with --pixel-density or --normalize-status-bar, and any other backend, before inspecting owner facts', async () => { const refused = async ( device: SessionState['device'], // The wire flags are untyped, so a request can name a backend the type does not list. @@ -277,7 +277,6 @@ test('refuses --capture-backend runner outside Apple targets, with --pixel-densi expect(fixture.binds).toHaveLength(0); }; - await refused(ANDROID_EMULATOR, { screenshotCaptureBackend: 'runner' }, 'UNSUPPORTED_OPERATION'); await refused( IOS_SIMULATOR, { screenshotCaptureBackend: 'runner', screenshotPixelDensity: 3 }, @@ -291,6 +290,31 @@ test('refuses --capture-backend runner outside Apple targets, with --pixel-densi await refused(IOS_SIMULATOR, { screenshotCaptureBackend: 'simctl' }, 'INVALID_ARGS'); }); +test('a Maestro comparison capture prefers the runner without selecting it', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('ios', { device: IOS_SIMULATOR }); + const req = screenshotRequest({ + positionals: [path.join(mkdtempForTestSync('agent-device-maestro-capture'), 'maestro.png')], + flags: { maestro: { screenshotCaptureBackend: 'runner' } }, + }); + + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + await executeResult(resolved, session, req, contextFromFlags('/tmp/daemon.log', req.flags)); + expect(fixture.captureScreenshot.mock.calls[0]?.[0]?.options).toMatchObject({ + preferredCaptureBackend: 'runner', + }); + expect(fixture.captureScreenshot.mock.calls[0]?.[0]?.options?.captureBackend).toBeUndefined(); +}); + test('a --capture-backend runner request reaches the bound capture', async () => { const fixture = screenshotRuntimeFixture(); const session = makeSession('ios', { device: IOS_SIMULATOR }); diff --git a/packages/daemon/src/execution/screenshot-runtime.ts b/packages/daemon/src/execution/screenshot-runtime.ts index 7e53621e9b..5f9f2f7bc3 100644 --- a/packages/daemon/src/execution/screenshot-runtime.ts +++ b/packages/daemon/src/execution/screenshot-runtime.ts @@ -8,7 +8,7 @@ import { } from '@agent-device/contracts/capture'; import type { DeviceRotation } from '@agent-device/contracts/device'; import type { ScreenshotRuntimeExecution } from '@agent-device/contracts/screenshot-runtime'; -import { isApplePlatform, isIosFamily, publicPlatformString } from '@agent-device/kernel/device'; +import { isIosFamily, publicPlatformString } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import type { ScreenshotOverlayRef } from '@agent-device/kernel/snapshot'; import { promises as fs } from 'node:fs'; @@ -65,7 +65,7 @@ export async function resolveScreenshotGenericExecution( const retiredMaxSize = retiredScreenshotMaxSizeFlagError('screenshot', req.flags); if (retiredMaxSize) throw new AppError('INVALID_ARGS', retiredMaxSize); assertSupportedScreenshotPixelDensity(session.device, req.flags?.screenshotPixelDensity); - assertSupportedScreenshotCaptureBackend(session.device, req.flags); + assertSupportedScreenshotCaptureBackend(req.flags); const request = readScreenshotRequest(req, params.hostCwd); const cropOn = @@ -319,11 +319,8 @@ async function annotateScreenshotWithSessionRefs( }); } -/** The runner capture exists on Apple targets only, and skips the simctl capture pipeline. */ -function assertSupportedScreenshotCaptureBackend( - device: SessionState['device'], - flags: DaemonRequest['flags'], -): void { +/** Each capture owner declares its backends; what stays here is the shape of the request. */ +function assertSupportedScreenshotCaptureBackend(flags: DaemonRequest['flags']): void { const backend: unknown = flags?.screenshotCaptureBackend; if (backend === undefined) return; if (!(SCREENSHOT_CAPTURE_BACKENDS as readonly unknown[]).includes(backend)) { @@ -332,12 +329,6 @@ function assertSupportedScreenshotCaptureBackend( `--capture-backend takes ${SCREENSHOT_CAPTURE_BACKENDS.join(', ')}`, ); } - if (!isApplePlatform(device.platform)) { - throw new AppError( - 'UNSUPPORTED_OPERATION', - '--capture-backend runner is supported only on Apple targets', - ); - } if (flags?.screenshotPixelDensity !== undefined || flags?.screenshotNormalizeStatusBar) { throw new AppError( 'INVALID_ARGS', @@ -402,6 +393,7 @@ function createBoundScreenshotBackend( dispatchContext.skipIosSimulatorBootCheck ?? (isIosFamily(session.device) && session.device.kind === 'simulator'), captureBackend: resolved.captureBackend, + preferredCaptureBackend: dispatchContext.preferredScreenshotCaptureBackend, }, execution: screenshotExecutionFromContext(dispatchContext), }); diff --git a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port.test.ts b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port.test.ts index 3aec4dc2ae..2fad6af325 100644 --- a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port.test.ts +++ b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port.test.ts @@ -661,7 +661,9 @@ test('waitForAnimationToEnd uses two unstabilized screenshot captures', async () expect(requests.map(({ command }) => command)).toEqual(['screenshot', 'screenshot']); expect(requests.every(({ flags }) => flags?.screenshotNoStabilize === true)).toBe(true); - expect(requests.every(({ flags }) => flags?.screenshotCaptureBackend === undefined)).toBe(true); + expect( + requests.every(({ flags }) => flags?.maestro?.screenshotCaptureBackend === undefined), + ).toBe(true); expect(result).not.toHaveProperty('visualStabilityReached'); }); @@ -690,7 +692,9 @@ test('waitForAnimationToEnd uses the persistent runner capture backend on iOS', expect(requests).toHaveLength(2); expect(requests.every(({ flags }) => flags?.screenshotNoStabilize === true)).toBe(true); - expect(requests.every(({ flags }) => flags?.screenshotCaptureBackend === 'runner')).toBe(true); + expect(requests.every(({ flags }) => flags?.maestro?.screenshotCaptureBackend === 'runner')).toBe( + true, + ); }); test('waitForAnimationToEnd between two taps does not throw a stability-generation mismatch', async () => { diff --git a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-public-operation.test.ts b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-public-operation.test.ts index 7487881bb8..15ec97c117 100644 --- a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-public-operation.test.ts +++ b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-public-operation.test.ts @@ -192,7 +192,7 @@ describe('Maestro public operation projection', () => { positionals: ['/tmp/animation-runner.png'], flags: { screenshotNoStabilize: true, - screenshotCaptureBackend: 'runner', + maestro: { screenshotCaptureBackend: 'runner' }, }, }, }, diff --git a/packages/maestro/src/daemon-port/daemon-runtime-public-operation.ts b/packages/maestro/src/daemon-port/daemon-runtime-public-operation.ts index e5ea3df359..36cddf63ba 100644 --- a/packages/maestro/src/daemon-port/daemon-runtime-public-operation.ts +++ b/packages/maestro/src/daemon-port/daemon-runtime-public-operation.ts @@ -398,7 +398,7 @@ function projectCaptureOperation( flags: { ...(operation.stabilize === false ? { screenshotNoStabilize: true } : {}), ...(operation.captureBackend === 'runner' - ? { screenshotCaptureBackend: 'runner' as const } + ? { maestro: { screenshotCaptureBackend: 'runner' as const } } : {}), }, } diff --git a/packages/platform-android/src/runtime.test.ts b/packages/platform-android/src/runtime.test.ts index 883ced36b7..4ad8e7738f 100644 --- a/packages/platform-android/src/runtime.test.ts +++ b/packages/platform-android/src/runtime.test.ts @@ -1,4 +1,5 @@ import { expect, test, vi } from 'vitest'; +import type { Interactor } from '@agent-device/contracts/interactor-types'; import { HARDWARE_BUTTON_NAMES, parseHardwareButtonPositionals, @@ -663,3 +664,27 @@ function deviceSerial(invocation: AndroidAdbInvocation): string { if (invocation.target.selector.kind !== 'serial') throw new Error('expected a serial target'); return invocation.target.selector.serial; } + +test('an Android capture refuses an explicit runner request at its own binding', async () => { + const screenshot = vi.fn(async () => ({})); + const resolve = vi.fn(async () => ({ screenshot }) as unknown as Interactor); + const binding = await bindOrdinary( + createAndroidPlatformRuntime({ + ...androidRuntimeHost(), + localInteractors: { resolve }, + }), + ANDROID_EMULATOR, + ); + + await expect( + binding.operations.captureScreenshot!({ + outPath: '/tmp/runner.png', + options: { captureBackend: 'runner' }, + }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + message: expect.stringContaining('adb'), + details: { reason: 'screenshot-capture-backend-unsupported' }, + }); + expect(resolve).not.toHaveBeenCalled(); +}); diff --git a/packages/platform-android/src/runtime.ts b/packages/platform-android/src/runtime.ts index cdecf763ad..020efe3a71 100644 --- a/packages/platform-android/src/runtime.ts +++ b/packages/platform-android/src/runtime.ts @@ -29,7 +29,10 @@ import { } from '@agent-device/contracts/platform-runtime'; import { audioProbeRuntimeOperationFacts } from '@agent-device/contracts/audio-probe-runtime'; import { perfRuntimeOperationFacts } from '@agent-device/contracts/perf-runtime'; -import { screenshotRuntimeOperationFacts } from '@agent-device/contracts/screenshot-runtime'; +import { + screenshotRuntimeOperationFacts, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import { selectorObservationRuntimeOperationFacts } from '@agent-device/contracts/selector-observation-runtime'; import { bindLocalSnapshotInteractor, @@ -571,6 +574,7 @@ function androidInteractionOperations( ...bindLocalInteractorOperationSet({ ...resolver, facts: facts.operations, + captureCapabilities: defaultCaptureOnly('an Android device captures through adb'), pause: async (milliseconds) => await host.clock.sleep(milliseconds, request.scope.signal), }), }; diff --git a/packages/platform-apple/src/screenshot-support.ts b/packages/platform-apple/src/screenshot-support.ts index 8fbcafae10..5397c5bf60 100644 --- a/packages/platform-apple/src/screenshot-support.ts +++ b/packages/platform-apple/src/screenshot-support.ts @@ -16,6 +16,7 @@ export function appleScreenshotCaptureCapabilities( isHandheldAppleSimulator(device) && options.captureBackend !== 'runner' && resolveAppleToolProvider().simulatorScreenshotBackend !== 'runner', - runner: helper === undefined, + captureBackends: helper === undefined ? ['runner'] : [], + captureBackendRefusal: 'the macOS helper captures this surface', }; } diff --git a/packages/platform-harmonyos/src/runtime.ts b/packages/platform-harmonyos/src/runtime.ts index 0325c77f0b..52c3aa837e 100644 --- a/packages/platform-harmonyos/src/runtime.ts +++ b/packages/platform-harmonyos/src/runtime.ts @@ -29,7 +29,10 @@ import { unavailableFact, whenAdmitted, } from '@agent-device/contracts/platform-runtime'; -import { screenshotRuntimeOperationFacts } from '@agent-device/contracts/screenshot-runtime'; +import { + screenshotRuntimeOperationFacts, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import { selectorObservationRuntimeOperationFacts } from '@agent-device/contracts/selector-observation-runtime'; import { bindLocalSnapshotInteractor, @@ -341,6 +344,7 @@ export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): Platfor signal: request.scope.signal, resolveInteractor: host.localInteractors.resolve, facts: facts.operations, + captureCapabilities: defaultCaptureOnly('a HarmonyOS device captures through hdc'), pause: async (milliseconds) => await host.clock.sleep(milliseconds, request.scope.signal), }), diff --git a/packages/platform-linux/src/runtime.ts b/packages/platform-linux/src/runtime.ts index c04c74bfd2..7f0230b711 100644 --- a/packages/platform-linux/src/runtime.ts +++ b/packages/platform-linux/src/runtime.ts @@ -28,7 +28,10 @@ import { unavailableFact, } from '@agent-device/contracts/platform-runtime'; import { createUnavailablePlatformRuntimeFacts } from '@agent-device/contracts/platform-runtime-unavailable'; -import { screenshotRuntimeOperationFacts } from '@agent-device/contracts/screenshot-runtime'; +import { + screenshotRuntimeOperationFacts, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import { captureSnapshotSignal, snapshotRuntimeOperationFacts, @@ -179,6 +182,7 @@ function linuxInteractionOperations( return bindLocalInteractorOperationSet({ ...resolver, facts: facts.operations, + captureCapabilities: defaultCaptureOnly('a Linux target captures through its desktop tools'), pause: async (milliseconds) => await host.clock.sleep(milliseconds, request.scope.signal), }); } diff --git a/packages/platform-web/src/runtime.ts b/packages/platform-web/src/runtime.ts index b571fb1a36..1e26106963 100644 --- a/packages/platform-web/src/runtime.ts +++ b/packages/platform-web/src/runtime.ts @@ -30,6 +30,7 @@ import { import { bindLocalScreenshotInteractor, screenshotRuntimeOperationFacts, + defaultCaptureOnly, } from '@agent-device/contracts/screenshot-runtime'; import { selectorObservationRuntimeOperationFacts } from '@agent-device/contracts/selector-observation-runtime'; import { @@ -247,6 +248,7 @@ function bindWebRuntime( device, signal, resolveInteractor: host.localInteractors.resolve, + captureCapabilities: defaultCaptureOnly('a web target captures through its browser'), }) : {}), ...(facts.operations.focusPoint.available diff --git a/packages/provider-limrun/src/interaction-operations.test.ts b/packages/provider-limrun/src/interaction-operations.test.ts index 3d8917f710..7fdcf47978 100644 --- a/packages/provider-limrun/src/interaction-operations.test.ts +++ b/packages/provider-limrun/src/interaction-operations.test.ts @@ -6,8 +6,9 @@ import { import type { Interactor, RunnerContext } from '@agent-device/contracts/interactor-types'; import { bindAdmittedProviderInteractorOperations } from '@agent-device/contracts/interactor-operation-catalog'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { expect, test } from 'vitest'; +import { expect, test, vi } from 'vitest'; import { + bindLimrunInteractionOperations, limrunAppEventOperationFacts, limrunAlertOperationFacts, limrunSettingsOperationFacts, @@ -271,3 +272,33 @@ test('long press is admitted on both direct-session legs and closed with a dead liveSessionUnavailable, ); }); + +test('a provider iOS capture refuses an explicit runner request instead of returning a provider screenshot', async () => { + const screenshot = vi.fn(async () => ({})); + const operations = bindLimrunInteractionOperations({ + device: iosDevice, + signal: new AbortController().signal, + getInteractor: () => ({ screenshot }) as unknown as Interactor, + presentIosAcquisition: (() => { + throw new Error('unused'); + }) as never, + }); + + await expect( + operations.captureScreenshot({ + outPath: '/tmp/runner.png', + options: { captureBackend: 'runner' }, + }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + message: expect.stringContaining('Limrun'), + details: { reason: 'screenshot-capture-backend-unsupported' }, + }); + expect(screenshot).not.toHaveBeenCalled(); + + await operations.captureScreenshot({ + outPath: '/tmp/maestro.png', + options: { preferredCaptureBackend: 'runner' }, + }); + expect(screenshot).toHaveBeenCalledWith('/tmp/maestro.png', {}); +}); diff --git a/packages/provider-limrun/src/interaction-operations.ts b/packages/provider-limrun/src/interaction-operations.ts index 77c4a5c88b..dd04a3668c 100644 --- a/packages/provider-limrun/src/interaction-operations.ts +++ b/packages/provider-limrun/src/interaction-operations.ts @@ -28,7 +28,10 @@ import { import { clipboardRuntimeOperationFacts } from '@agent-device/contracts/clipboard-runtime'; import { keyboardRuntimeOperationFacts } from '@agent-device/contracts/keyboard-runtime'; import { orientationRuntimeOperationFacts } from '@agent-device/contracts/orientation-runtime'; -import { bindProviderScreenshotInteractor } from '@agent-device/contracts/screenshot-runtime'; +import { + bindProviderScreenshotInteractor, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import { bindProviderSnapshotInteractor, type SnapshotRuntimePresenter, @@ -205,7 +208,12 @@ export function bindLimrunInteractionOperations( facts: limrunInteractionOperationFacts(device), pause: async (milliseconds) => await sleep(milliseconds, undefined, { signal }), }), - ...bindProviderScreenshotInteractor({ device, signal, resolveInteractor }), + ...bindProviderScreenshotInteractor({ + device, + signal, + resolveInteractor, + captureCapabilities: defaultCaptureOnly('Limrun captures through its provider endpoint'), + }), ...bindProviderGestureInteractor({ device, signal, diff --git a/packages/provider-revyl/src/interaction-operations.ts b/packages/provider-revyl/src/interaction-operations.ts index cdac7a94d2..85129752dd 100644 --- a/packages/provider-revyl/src/interaction-operations.ts +++ b/packages/provider-revyl/src/interaction-operations.ts @@ -25,7 +25,10 @@ import { bindProviderTouchInteractor, touchRuntimeOperationFacts, } from '@agent-device/contracts/touch-runtime'; -import { bindProviderScreenshotInteractor } from '@agent-device/contracts/screenshot-runtime'; +import { + bindProviderScreenshotInteractor, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import { bindProviderSnapshotInteractor, type SnapshotRuntimePresenter, @@ -239,7 +242,12 @@ export function bindRevylInteractionOperations( resolveInteractor, presentIosAcquisition: params.presentIosAcquisition, }), - ...bindProviderScreenshotInteractor({ device, signal, resolveInteractor }), + ...bindProviderScreenshotInteractor({ + device, + signal, + resolveInteractor, + captureCapabilities: defaultCaptureOnly('Revyl captures through its provider endpoint'), + }), ...bindProviderFocusInteractor({ device, signal, resolveInteractor }), ...bindProviderTypeTextInteractor({ device, signal, resolveInteractor }), ...bindProviderTouchInteractor({ diff --git a/packages/provider-webdriver/src/platform-runtime.ts b/packages/provider-webdriver/src/platform-runtime.ts index b4ebe45a58..78e1d32886 100644 --- a/packages/provider-webdriver/src/platform-runtime.ts +++ b/packages/provider-webdriver/src/platform-runtime.ts @@ -52,6 +52,7 @@ import { import { bindProviderScreenshotInteractor, screenshotRuntimeOperationFacts, + defaultCaptureOnly, } from '@agent-device/contracts/screenshot-runtime'; import { bindProviderSnapshotInteractor, @@ -389,7 +390,12 @@ function webDriverInteractionOperations( return { ...(facts.operations.captureSnapshot.available ? bindProviderSnapshotInteractor(resolver) : {}), ...(facts.operations.captureScreenshot.available - ? bindProviderScreenshotInteractor(resolver) + ? bindProviderScreenshotInteractor({ + ...resolver, + captureCapabilities: defaultCaptureOnly( + 'a WebDriver session captures through its provider', + ), + }) : {}), ...(facts.operations.focusPoint.available ? bindProviderFocusInteractor(resolver) : {}), ...(facts.operations.typeText.available ? bindProviderTypeTextInteractor(resolver) : {}), diff --git a/packages/tv-cdp/src/runtime.ts b/packages/tv-cdp/src/runtime.ts index 84001544a4..09fc795e0a 100644 --- a/packages/tv-cdp/src/runtime.ts +++ b/packages/tv-cdp/src/runtime.ts @@ -33,6 +33,7 @@ import { createUnavailablePlatformRuntimeFacts } from '@agent-device/contracts/p import { bindProviderScreenshotInteractor, screenshotRuntimeOperationFacts, + defaultCaptureOnly, } from '@agent-device/contracts/screenshot-runtime'; import { bindProviderScrollInteractor, @@ -355,7 +356,12 @@ function createTvPlatformRuntime( resolveInteractor, presentIosAcquisition: host.snapshot.presentIosAcquisition, }), - ...bindProviderScreenshotInteractor({ device, signal, resolveInteractor }), + ...bindProviderScreenshotInteractor({ + device, + signal, + resolveInteractor, + captureCapabilities: defaultCaptureOnly('a TV target captures through CDP'), + }), ...bindProviderTouchInteractor({ device, signal, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 2c57d91e30..1cc639da01 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1052,7 +1052,7 @@ agent-device record stop # Stop active recording - `--overlay-refs` takes a fresh snapshot and burns visible `@eN` refs and their rectangles into the PNG. With `--scale`, refs are drawn at the final size, so avoid scales where labels become unreadable. In `--json` mode each ref also has a screenshot-space `center` for `press ` fallback. - `--crop-on ` crops the capture to the frame the selector resolves on the same screen. It works only on iOS simulators, Android emulators, and physical iPhones. A physical iPhone crop is verified only in portrait using CoreDevice/devicectl; the XCTest backend and runner fallback are not accepted. Physical iPads remain rejected for crop, and landscape iPhone crop remains unverified. It needs exactly one framed node, cannot be combined with `--overlay-refs` or `--fullscreen`, and warns when the frame is clipped. The crop is re-encoded, so do not byte-compare it with an older crop. - `--fullscreen` on macOS applies only to app sessions. Other surfaces (`desktop`, `menubar`, `frontmost-app`) always capture the main display and refuse `--fullscreen` with `INVALID_ARGS`. -- `--capture-backend runner` captures an Apple target through the XCTest runner (`XCUIScreen`), as XCTest-based tools such as Maestro do, instead of `simctl io screenshot`. On iOS simulators with a Dynamic Island, `simctl` draws the island into every capture and no `--mask` policy removes it; the runner capture leaves it out. The capture skips the simctl pipeline, so `--pixel-density` and `--normalize-status-bar` are refused with it (`INVALID_ARGS`). Other platforms, and macOS surfaces the helper captures (`desktop`, `menubar`, `frontmost-app`, and app sessions with the native app backend), refuse it with `UNSUPPORTED_OPERATION`. Node and MCP take `captureBackend: 'runner'`. +- `--capture-backend runner` captures an Apple target through the XCTest runner (`XCUIScreen`), as XCTest-based tools such as Maestro do, instead of `simctl io screenshot`. On iOS simulators with a Dynamic Island, `simctl` draws the island into every capture and no `--mask` policy removes it; the runner capture leaves it out. The capture skips the simctl pipeline, so `--pixel-density` and `--normalize-status-bar` are refused with it (`INVALID_ARGS`). Each capture owner declares the backends it serves: other platforms, provider-backed sessions, and macOS surfaces the helper captures (`desktop`, `menubar`, `frontmost-app`, and app sessions with the native app backend) refuse it with `UNSUPPORTED_OPERATION`. Node and MCP take `captureBackend: 'runner'`. - `--normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for reusable baselines. Each status-bar read or preparation command has a 10-second timeout. Each restore command has a 30-second timeout, so clearing and reapplying overrides can take up to 60 seconds. The request timeout includes the wait and can expire before these steps finish. - `--screen-mask black` fills the iOS simulator screen outline, including rounded corners, with black for this capture. Node and MCP use `screenMask: "black"`. Omit it for the default image. Support is limited to handheld iOS/iPadOS simulators. It requires simctl capture; physical devices, tvOS and visionOS simulators, other platforms, provider captures, and XCTest runner captures are refused. If simctl fails, a masked request fails instead of falling back to an unmasked runner capture. This option does not redact app content, remove a Dynamic Island drawn into the framebuffer, or change comparison thresholds. - On an iOS simulator, a screenshot waits until no other screenshot or `settings wifi` / `settings airplane` write is using that simulator's status bar, so overlapping captures cannot see each other's temporary chrome. A request you cancel while it is waiting fails with the cancellation instead of capturing. From 2b12a1a94dfd7c103b3a31470fef796fbad43095 Mon Sep 17 00:00:00 2001 From: Vitalii Yehorov Date: Wed, 4 Nov 2026 11:26:30 +0100 Subject: [PATCH 4/8] fix(screenshot): keep Maestro's runner preference on the daemon path The daemon passes an own captureBackend key, undefined for a Maestro request, and it overrode the preference in the binding, so a local iOS comparison capture fell back to simctl and drew the Dynamic Island back in. An explicit backend still wins. The preference moves off the shared ScreenshotOptions onto the binding input, since the binding strips it before the interactor. The daemon-level Android refusal case is back, now at the capture owner, and WebDriver's refusal has a case. Co-Authored-By: Claude Opus 5.5 --- packages/contracts/src/interactor-types.ts | 2 - .../src/local-interactor-operation-set.ts | 4 +- .../contracts/src/screenshot-runtime.test.ts | 6 +- packages/contracts/src/screenshot-runtime.ts | 15 ++-- .../screenshot-runtime-execution.test.ts | 80 ++++++++++++++++--- .../src/screenshot-support.test.ts | 24 ++++++ .../src/platform-runtime.test.ts | 9 +++ 7 files changed, 116 insertions(+), 24 deletions(-) diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index 04f956e8e4..dce79ef80b 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -69,8 +69,6 @@ export type ScreenshotOptions = { surface?: SessionSurface; skipIosSimulatorBootCheck?: boolean; captureBackend?: 'runner'; - /** Maestro's comparison capture: the runner where the owner declares it, else the owner's default. */ - preferredCaptureBackend?: 'runner'; }; /** diff --git a/packages/contracts/src/local-interactor-operation-set.ts b/packages/contracts/src/local-interactor-operation-set.ts index 9e25554c35..e90552079f 100644 --- a/packages/contracts/src/local-interactor-operation-set.ts +++ b/packages/contracts/src/local-interactor-operation-set.ts @@ -13,8 +13,8 @@ import { bindLocalTypeTextInteractor } from './type-text-runtime.ts'; /** * The whole local interactor-backed operation set a pointer-driving family binds: the uniform - * one-fact-one-bind leaves the interactor catalog walks, plus the four that carry extra input of their own — gesture and touch read the - * fact map, and touch needs a pause clock. Android, Linux, and HarmonyOS bind this one list, so + * one-fact-one-bind leaves the interactor catalog walks, plus the ones that carry extra input of their own — gesture and touch read the + * fact map, touch needs a pause clock, and screenshot takes the owner's capture declaration. Android, Linux, and HarmonyOS bind this one list, so * no pointer-driving family can drift from another by forgetting a member. Capture stays out: an * owner's snapshot mechanics are its own (Linux captures a surface through its host, Android * drives the interactor), so there is no shared reading to make. diff --git a/packages/contracts/src/screenshot-runtime.test.ts b/packages/contracts/src/screenshot-runtime.test.ts index cb48f3d7f9..85d9648861 100644 --- a/packages/contracts/src/screenshot-runtime.test.ts +++ b/packages/contracts/src/screenshot-runtime.test.ts @@ -131,7 +131,7 @@ test.each([ ])( 'a preferred backend selects a declared one and otherwise keeps the default capture: %j', async (captureBackends, expected) => { - const screenshot = vi.fn(async () => ({})); + const screenshot = vi.fn(async (_outPath: string, _options: unknown) => ({})); const operations = bindLocalScreenshotInteractor({ device, signal: new AbortController().signal, @@ -140,8 +140,8 @@ test.each([ }); await operations.captureScreenshot({ outPath: '/tmp/capture.png', - options: { preferredCaptureBackend: 'runner' }, + options: { captureBackend: undefined, preferredCaptureBackend: 'runner' }, }); - expect(screenshot).toHaveBeenCalledWith('/tmp/capture.png', expected); + expect(screenshot.mock.lastCall?.[1]).toEqual(expected); }, ); diff --git a/packages/contracts/src/screenshot-runtime.ts b/packages/contracts/src/screenshot-runtime.ts index 5c0c508deb..472be6fcc6 100644 --- a/packages/contracts/src/screenshot-runtime.ts +++ b/packages/contracts/src/screenshot-runtime.ts @@ -27,7 +27,7 @@ export type ScreenshotRuntimeExecution = Readonly; + options?: Readonly; execution?: ScreenshotRuntimeExecution; }>; @@ -90,14 +90,15 @@ export function screenshotRuntimeOperationFacts( return Object.freeze({ captureScreenshot: input.capture }); } -/** A preference, unlike `captureBackend`, keeps the owner's default where it declares no such backend. */ -function resolvePreferredCaptureBackend( - input: Readonly | undefined, +/** A preference, unlike `captureBackend`, keeps the owner's default where it declares no such backend. Maestro's comparison capture asks for one. */ +function resolveCaptureOptions( + input: CaptureScreenshotInput['options'], binding: ScreenshotCaptureBindingOptions, ): Readonly { - const { preferredCaptureBackend: preferred, ...options } = input ?? {}; + const { captureBackend: explicit, preferredCaptureBackend: preferred, ...options } = input ?? {}; + if (explicit) return { ...options, captureBackend: explicit }; return preferred && binding.captureCapabilities(options).captureBackends?.includes(preferred) - ? { captureBackend: preferred, ...options } + ? { ...options, captureBackend: preferred } : options; } @@ -112,7 +113,7 @@ function bindScreenshotCapture( ): ScreenshotRuntimeOperations { return Object.freeze({ captureScreenshot: async (input: CaptureScreenshotInput) => { - const options = resolvePreferredCaptureBackend(input.options, binding); + const options = resolveCaptureOptions(input.options, binding); await assertScreenshotCaptureSupport(options, binding.captureCapabilities(options)); const interactor = await resolveInteractor({ ...input.execution, diff --git a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts index 34c518e800..bd0835f731 100644 --- a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts +++ b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts @@ -2,8 +2,13 @@ import { daemonPlatformServicesFixture } from '../__tests__/support/platform-ser import fs from 'node:fs'; import path from 'node:path'; -import { expect, test } from 'vitest'; +import { expect, test, vi } from 'vitest'; import { SCREENSHOT_CROP_REASONS } from '@agent-device/contracts/capture'; +import type { Interactor, ScreenshotOptions } from '@agent-device/contracts/interactor-types'; +import { + bindLocalScreenshotInteractor, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { ANDROID_EMULATOR, @@ -290,14 +295,31 @@ test('refuses --capture-backend runner with --pixel-density or --normalize-statu await refused(IOS_SIMULATOR, { screenshotCaptureBackend: 'simctl' }, 'INVALID_ARGS'); }); -test('a Maestro comparison capture prefers the runner without selecting it', async () => { +async function captureThroughRealBinding( + params: Readonly<{ + device: SessionState['device']; + flags: DaemonRequest['flags']; + captureCapabilities: Parameters[0]['captureCapabilities']; + }>, +) { const fixture = screenshotRuntimeFixture(); - const session = makeSession('ios', { device: IOS_SIMULATOR }); + const screenshot = vi.fn(async (outPath: string, _options?: ScreenshotOptions) => { + writeSolidPng(outPath); + return {}; + }); + fixture.captureScreenshot.mockImplementation( + bindLocalScreenshotInteractor({ + device: params.device, + signal: new AbortController().signal, + resolveInteractor: async () => ({ screenshot }) as unknown as Interactor, + captureCapabilities: params.captureCapabilities, + }).captureScreenshot, + ); + const session = makeSession('capture', { device: params.device }); const req = screenshotRequest({ - positionals: [path.join(mkdtempForTestSync('agent-device-maestro-capture'), 'maestro.png')], - flags: { maestro: { screenshotCaptureBackend: 'runner' } }, + positionals: [path.join(mkdtempForTestSync('agent-device-bound-capture'), 'capture.png')], + flags: params.flags, }); - const resolved = await resolveScreenshotGenericExecution({ hostCwd: process.cwd(), createCommandSurface: testCreateCommandSurface, @@ -307,12 +329,50 @@ test('a Maestro comparison capture prefers the runner without selecting it', asy inspectFacts: fixture.inspectFacts, bindDevice: fixture.bindDevice, }); + return { + screenshot, + execute: async () => + await executeResult(resolved, session, req, contextFromFlags('/tmp/daemon.log', req.flags)), + }; +} - await executeResult(resolved, session, req, contextFromFlags('/tmp/daemon.log', req.flags)); - expect(fixture.captureScreenshot.mock.calls[0]?.[0]?.options).toMatchObject({ - preferredCaptureBackend: 'runner', +test('a Maestro comparison capture reaches the interactor as a runner capture where the owner declares it', async () => { + const { screenshot, execute } = await captureThroughRealBinding({ + device: IOS_SIMULATOR, + flags: { maestro: { screenshotCaptureBackend: 'runner' } }, + captureCapabilities: () => ({ captureBackends: ['runner'] }), + }); + + await execute(); + + expect(screenshot.mock.calls[0]?.[1]).toMatchObject({ captureBackend: 'runner' }); + expect(screenshot.mock.calls[0]?.[1]).not.toHaveProperty('preferredCaptureBackend'); +}); + +test('a Maestro comparison capture keeps the default capture of an owner that declares no runner', async () => { + const { screenshot, execute } = await captureThroughRealBinding({ + device: IOS_SIMULATOR, + flags: { maestro: { screenshotCaptureBackend: 'runner' } }, + captureCapabilities: defaultCaptureOnly('this owner has one capture path'), + }); + + await execute(); + + expect(screenshot.mock.calls[0]?.[1]).not.toHaveProperty('captureBackend'); +}); + +test('an Android session refuses --capture-backend runner at its capture owner', async () => { + const { screenshot, execute } = await captureThroughRealBinding({ + device: ANDROID_EMULATOR, + flags: { screenshotCaptureBackend: 'runner' }, + captureCapabilities: defaultCaptureOnly('an Android device captures through adb'), + }); + + await expect(execute()).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-capture-backend-unsupported' }, }); - expect(fixture.captureScreenshot.mock.calls[0]?.[0]?.options?.captureBackend).toBeUndefined(); + expect(screenshot).not.toHaveBeenCalled(); }); test('a --capture-backend runner request reaches the bound capture', async () => { diff --git a/packages/platform-apple/src/screenshot-support.test.ts b/packages/platform-apple/src/screenshot-support.test.ts index 2160fa5300..9ccd0cab1d 100644 --- a/packages/platform-apple/src/screenshot-support.test.ts +++ b/packages/platform-apple/src/screenshot-support.test.ts @@ -116,3 +116,27 @@ test.each([ } }, ); + +test('the shape the daemon passes for a Maestro preference selects the runner on an iOS simulator', async () => { + const screenshot = vi.fn(async (_outPath: string, _options: ScreenshotOptions) => ({})); + const host = { + ...platformRuntimeHostFixture(), + localInteractors: { resolve: async () => ({ screenshot }) as unknown as Interactor }, + }; + const binding = await createApplePlatformRuntime(host).bind({ + device: simulator, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + + await binding.operations.captureScreenshot!({ + outPath: '/tmp/maestro.png', + options: { captureBackend: undefined, preferredCaptureBackend: 'runner' }, + }); + + expect(screenshot.mock.lastCall?.[1]).toEqual({ captureBackend: 'runner' }); +}); diff --git a/packages/provider-webdriver/src/platform-runtime.test.ts b/packages/provider-webdriver/src/platform-runtime.test.ts index 0bf08f14a4..21d5603747 100644 --- a/packages/provider-webdriver/src/platform-runtime.test.ts +++ b/packages/provider-webdriver/src/platform-runtime.test.ts @@ -293,6 +293,15 @@ test('captures through only the active exact WebDriver interactor', async () => expect(binding.operations.setViewport).toBeUndefined(); expect(binding.facts.operations.captureScreenshot).toEqual({ available: true }); expect(binding.operations.captureScreenshot).toBeTypeOf('function'); + await expect( + binding.operations.captureScreenshot!({ + outPath: '/tmp/runner.png', + options: { captureBackend: 'runner' }, + }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-capture-backend-unsupported' }, + }); // R40/R41: interaction cells share the captures' reachability gate and interactor. expect(binding.facts.operations.focusPoint).toEqual({ available: true }); expect(binding.facts.operations.typeText).toEqual({ available: true }); From 6d237c3329bf00fd6a9c2e6ab808e8d029debb42 Mon Sep 17 00:00:00 2001 From: Vitalii Yehorov Date: Wed, 4 Nov 2026 12:49:17 +0100 Subject: [PATCH 5/8] test(screenshot): check the destination of a preferred-backend capture Co-Authored-By: Claude Opus 5.5 --- packages/contracts/src/screenshot-runtime.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/contracts/src/screenshot-runtime.test.ts b/packages/contracts/src/screenshot-runtime.test.ts index 85d9648861..20bbd531d1 100644 --- a/packages/contracts/src/screenshot-runtime.test.ts +++ b/packages/contracts/src/screenshot-runtime.test.ts @@ -142,6 +142,6 @@ test.each([ outPath: '/tmp/capture.png', options: { captureBackend: undefined, preferredCaptureBackend: 'runner' }, }); - expect(screenshot.mock.lastCall?.[1]).toEqual(expected); + expect(screenshot).toHaveBeenLastCalledWith('/tmp/capture.png', expected); }, ); From 8797656349f1d837e5eb5276fa574c394b6d10fb Mon Sep 17 00:00:00 2001 From: Vitalii Yehorov Date: Wed, 4 Nov 2026 18:43:00 +0100 Subject: [PATCH 6/8] fix(screenshot): align explicit runner capture with the default under a runner-backed provider Under a tool provider whose simulator screenshots go through the runner, the default capture asked the runner for fullscreen, but an explicit --capture-backend runner did not. The default now lives in the runner capture, so both routes share it. An explicit fullscreen value still wins, and captures outside such a provider are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../src/core/__tests__/screenshot.test.ts | 50 +++++++++++++++++++ .../platform-apple/src/core/screenshot.ts | 11 ++-- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/packages/platform-apple/src/core/__tests__/screenshot.test.ts b/packages/platform-apple/src/core/__tests__/screenshot.test.ts index 0ccc8457c2..4b08e26d97 100644 --- a/packages/platform-apple/src/core/__tests__/screenshot.test.ts +++ b/packages/platform-apple/src/core/__tests__/screenshot.test.ts @@ -889,6 +889,56 @@ test.each([MACOS_TEST_DEVICE, { ...IOS_TEST_SIMULATOR, kind: 'device' as const } }, ); +test.each([ + ['the default capture', undefined, true], + ['an explicit runner capture', undefined, true], + ['an explicit runner capture', false, false], +] as const)( + '%s under a runner-backed tool provider asks the runner with fullscreen %s: %s', + async (route, fullscreen, expected) => { + mockRunAppleRunnerCommand.mockRejectedValue(new Error('stop after the runner request')); + const capture = async () => + route === 'the default capture' + ? await captureSimulatorScreenshotWithFallback(IOS_TEST_SIMULATOR, '/tmp/runner.png', { + env: {}, + fullscreen, + skipIosSimulatorBootCheck: true, + deps: { resolveCaptureTarget: async () => ({}) }, + }) + : await captureScreenshotViaRunner( + IOS_TEST_SIMULATOR, + '/tmp/runner.png', + undefined, + fullscreen, + undefined, + {}, + ); + await withAppleToolProvider( + { ...resolveAppleToolProvider(), simulatorScreenshotBackend: 'runner' }, + async () => { + await assert.rejects(capture(), /stop after the runner request/); + }, + ); + assert.equal(mockRunAppleRunnerCommand.mock.calls[0]?.[1]?.fullscreen, expected); + }, +); + +test('an explicit runner capture outside a runner-backed tool provider leaves fullscreen unset', async () => { + mockRunAppleRunnerCommand.mockRejectedValue(new Error('stop after the runner request')); + await assert.rejects( + captureScreenshotViaRunner( + IOS_TEST_SIMULATOR, + '/tmp/runner.png', + undefined, + undefined, + undefined, + {}, + ), + /stop after the runner request/, + ); + assert.equal(mockRunAppleRunnerCommand.mock.calls[0]?.[1]?.fullscreen, undefined); +}); + test('screenshotIos rejects a screen mask before a provider-owned runner capture', async () => { mockRunCmd.mockRejectedValue(new Error('unexpected native capture')); mockRunAppleRunnerCommand.mockRejectedValue(new Error('unexpected runner capture')); diff --git a/packages/platform-apple/src/core/screenshot.ts b/packages/platform-apple/src/core/screenshot.ts index 8dcdc9a275..563872c05e 100644 --- a/packages/platform-apple/src/core/screenshot.ts +++ b/packages/platform-apple/src/core/screenshot.ts @@ -210,8 +210,7 @@ export async function captureSimulatorScreenshotWithFallback( device, outPath, options.appBundleId, - options.fullscreen ?? - (resolveAppleToolProvider().simulatorScreenshotBackend === 'runner' ? true : undefined), + options.fullscreen, options.runnerOptions, options.env, ); @@ -287,10 +286,16 @@ export async function captureScreenshotViaRunner( device: DeviceInfo, outPath: string, appBundleId: string | undefined, - fullscreen: boolean | undefined, + requestedFullscreen: boolean | undefined, runnerOptions: AppleRunnerCommandOptions | undefined, env: Readonly, ): Promise { + const fullscreen = + requestedFullscreen ?? + (device.kind === 'simulator' && + resolveAppleToolProvider().simulatorScreenshotBackend === 'runner' + ? true + : undefined); if (!capturesThroughSimulatorStatusBar(device)) { return await captureRunnerScreenshot( device, From b3da8c0d3a2a3f7e7ea602ef9bb38729abf8c0c1 Mon Sep 17 00:00:00 2001 From: Vitalii Yehorov Date: Wed, 4 Nov 2026 19:03:02 +0100 Subject: [PATCH 7/8] fix(screenshot): keep a macOS runner capture out of the provider fullscreen default The default checked only device.kind, so a macOS device shaped as a simulator got fullscreen under a runner-backed provider. It now uses the same predicate as the capture path. Co-Authored-By: Claude Opus 5.5 --- .../src/core/__tests__/screenshot.test.ts | 21 +++++++++++++++++++ .../platform-apple/src/core/screenshot.ts | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/platform-apple/src/core/__tests__/screenshot.test.ts b/packages/platform-apple/src/core/__tests__/screenshot.test.ts index 4b08e26d97..f31eeaa539 100644 --- a/packages/platform-apple/src/core/__tests__/screenshot.test.ts +++ b/packages/platform-apple/src/core/__tests__/screenshot.test.ts @@ -923,6 +923,27 @@ test.each([ }, ); +test('a macOS runner capture under a runner-backed tool provider keeps its requested fullscreen', async () => { + mockRunAppleRunnerCommand.mockRejectedValue(new Error('stop after the runner request')); + await withAppleToolProvider( + { ...resolveAppleToolProvider(), simulatorScreenshotBackend: 'runner' }, + async () => { + await assert.rejects( + captureScreenshotViaRunner( + { ...MACOS_TEST_DEVICE, kind: 'simulator' }, + '/tmp/runner.png', + undefined, + undefined, + undefined, + {}, + ), + /stop after the runner request/, + ); + }, + ); + assert.equal(mockRunAppleRunnerCommand.mock.calls[0]?.[1]?.fullscreen, undefined); +}); + test('an explicit runner capture outside a runner-backed tool provider leaves fullscreen unset', async () => { mockRunAppleRunnerCommand.mockRejectedValue(new Error('stop after the runner request')); await assert.rejects( diff --git a/packages/platform-apple/src/core/screenshot.ts b/packages/platform-apple/src/core/screenshot.ts index 563872c05e..e63a543cc7 100644 --- a/packages/platform-apple/src/core/screenshot.ts +++ b/packages/platform-apple/src/core/screenshot.ts @@ -292,7 +292,7 @@ export async function captureScreenshotViaRunner( ): Promise { const fullscreen = requestedFullscreen ?? - (device.kind === 'simulator' && + (capturesThroughSimulatorStatusBar(device) && resolveAppleToolProvider().simulatorScreenshotBackend === 'runner' ? true : undefined); From fe817f630c12e3859bb7f5b4df409e14c234e8b1 Mon Sep 17 00:00:00 2001 From: Vitalii Yehorov Date: Wed, 4 Nov 2026 19:16:56 +0100 Subject: [PATCH 8/8] test(screenshot): name what the macOS fullscreen case checks Co-Authored-By: Claude Opus 5.5 --- packages/platform-apple/src/core/__tests__/screenshot.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/platform-apple/src/core/__tests__/screenshot.test.ts b/packages/platform-apple/src/core/__tests__/screenshot.test.ts index f31eeaa539..9f33fd2698 100644 --- a/packages/platform-apple/src/core/__tests__/screenshot.test.ts +++ b/packages/platform-apple/src/core/__tests__/screenshot.test.ts @@ -923,7 +923,7 @@ test.each([ }, ); -test('a macOS runner capture under a runner-backed tool provider keeps its requested fullscreen', async () => { +test('a macOS runner capture under a runner-backed tool provider skips the fullscreen default', async () => { mockRunAppleRunnerCommand.mockRejectedValue(new Error('stop after the runner request')); await withAppleToolProvider( { ...resolveAppleToolProvider(), simulatorScreenshotBackend: 'runner' },