diff --git a/packages/command-surface/src/capture/index.test.ts b/packages/command-surface/src/capture/index.test.ts index 28a6aeac72..308a763174 100644 --- a/packages/command-surface/src/capture/index.test.ts +++ b/packages/command-surface/src/capture/index.test.ts @@ -92,6 +92,15 @@ describe('capture command interface', () => { }); }); + test('reads and writes the screenshot capture backend', () => { + const input = screenshotCliReader(['page.png'], flags({ screenshotCaptureBackend: 'runner' })); + expect(input).toMatchObject({ path: 'page.png', captureBackend: 'runner' }); + expect(screenshotDaemonWriter(input)).toMatchObject({ + command: 'screenshot', + options: { screenshotCaptureBackend: 'runner' }, + }); + }); + test('reads diff snapshot input only', () => { expect( diffCliReader(['snapshot'], flags({ snapshotDepth: 4, out: './diff.json' })), diff --git a/packages/command-surface/src/capture/screenshot-options.test.ts b/packages/command-surface/src/capture/screenshot-options.test.ts index 4140eea6a3..f07a6a60b6 100644 --- a/packages/command-surface/src/capture/screenshot-options.test.ts +++ b/packages/command-surface/src/capture/screenshot-options.test.ts @@ -25,6 +25,7 @@ test('screenshot flag projection maps CLI flags to runtime options', () => { screenshotScale: 0.3, screenshotNoStabilize: true, screenshotNormalizeStatusBar: true, + screenshotCaptureBackend: 'runner', }), { overlayRefs: true, @@ -35,6 +36,7 @@ test('screenshot flag projection maps CLI flags to runtime options', () => { scale: 0.3, stabilize: false, normalizeStatusBar: true, + captureBackend: 'runner', }, ); }); @@ -49,6 +51,7 @@ test('screenshot flag projection maps public options to request flags', () => { fullscreen: true, stabilize: false, normalizeStatusBar: true, + captureBackend: 'runner', }), { overlayRefs: true, @@ -58,6 +61,7 @@ test('screenshot flag projection maps public options to request flags', () => { screenshotFullscreen: true, screenshotNoStabilize: true, screenshotNormalizeStatusBar: true, + screenshotCaptureBackend: 'runner', }, ); }); @@ -152,6 +156,7 @@ test('screenshot script flags use the shared recorded flag contract', () => { 'screenshotScale', 'screenshotNoStabilize', 'screenshotNormalizeStatusBar', + 'screenshotCaptureBackend', ]); assert.deepEqual( SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS.map((definition) => definition.key), @@ -167,9 +172,21 @@ test('screenshot script flags use the shared recorded flag contract', () => { 'screenshotScale', 'screenshotNoStabilize', 'screenshotNormalizeStatusBar', + 'screenshotCaptureBackend', ]); }); +test('the capture backend stays out of recorded scripts', () => { + const parts: string[] = []; + appendScreenshotScriptFlags(parts, { screenshotCaptureBackend: 'runner' }); + assert.deepEqual(parts, []); + assert.equal( + SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS.find(({ key }) => key === 'screenshotCaptureBackend') + ?.recorded, + false, + ); +}); + test('explicit screenshot stabilization survives SDK projection and script recording', () => { const projected = screenshotFlagsFromOptions({ stabilize: true }); assert.deepEqual(screenshotOptionsFromFlags(projected), { stabilize: true }); diff --git a/packages/command-surface/src/capture/screenshot.ts b/packages/command-surface/src/capture/screenshot.ts index df08b13bc4..d67287e69e 100644 --- a/packages/command-surface/src/capture/screenshot.ts +++ b/packages/command-surface/src/capture/screenshot.ts @@ -3,6 +3,7 @@ import type { CaptureScreenshotOptions } from '@agent-device/contracts/client'; import { SESSION_SURFACES } from '@agent-device/contracts/session'; import { RETIRED_SCREENSHOT_MAX_SIZE, + SCREENSHOT_CAPTURE_BACKENDS, SCREENSHOT_COMMAND_FLAG_KEYS, SCREENSHOT_SCALE_LIMITS, SCREENSHOT_SCREEN_MASKS, @@ -50,6 +51,10 @@ const screenshotCommandMetadata = defineFieldCommandMetadata( maxSize: retiredField(RETIRED_SCREENSHOT_MAX_SIZE.migration.screenshot), stabilize: booleanField(), normalizeStatusBar: booleanField(), + captureBackend: enumField( + SCREENSHOT_CAPTURE_BACKENDS, + 'Capture through the XCTest runner instead of the platform default, which leaves out the simulator-drawn Dynamic Island. Local Apple targets only; refused on provider sessions and on a macOS surface the helper captures, and with pixelDensity, screenMask or normalizeStatusBar.', + ), surface: enumField(SESSION_SURFACES), }, ); diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index 0df298488c..3a1952b7fd 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -1,4 +1,4 @@ -import type { ScreenshotScreenMask } from './screenshot.ts'; +import type { ScreenshotCaptureBackend, ScreenshotScreenMask } from './screenshot.ts'; // The public API vocabulary for snapshot, screenshot and diff capture. import type { SessionSurface } from './session-surface.ts'; @@ -118,6 +118,8 @@ export type CaptureScreenshotOptions = AgentDeviceRequestOverrides & { scale?: number; stabilize?: boolean; normalizeStatusBar?: boolean; + /** `runner`: capture an Apple target through the XCTest runner instead of `simctl`. */ + captureBackend?: ScreenshotCaptureBackend; surface?: SessionSurface; }; diff --git a/packages/contracts/src/facades/capture.ts b/packages/contracts/src/facades/capture.ts index ca05162baf..5245d1da06 100644 --- a/packages/contracts/src/facades/capture.ts +++ b/packages/contracts/src/facades/capture.ts @@ -2,6 +2,7 @@ export type { DiffSnapshotCommandResult, SnapshotDiffLine, SnapshotDiffSummary } export { RETIRED_SCREENSHOT_MAX_SIZE, SCREENSHOT_ACTION_FLAG_KEYS, + SCREENSHOT_CAPTURE_BACKENDS, SCREENSHOT_COMMAND_FLAG_KEYS, SCREENSHOT_CROP_REASONS, SCREENSHOT_FULLSCREEN_REASONS, @@ -19,6 +20,7 @@ export { validateScreenshotScale, } from '../screenshot.ts'; export type { + ScreenshotCaptureBackend, ScreenshotCropReason, ScreenshotScreenMask, ScreenshotDispatchFlags, diff --git a/packages/contracts/src/local-interactor-operation-set.ts b/packages/contracts/src/local-interactor-operation-set.ts index 7e49d48a6f..e90552079f 100644 --- a/packages/contracts/src/local-interactor-operation-set.ts +++ b/packages/contracts/src/local-interactor-operation-set.ts @@ -13,8 +13,8 @@ import { bindLocalTypeTextInteractor } from './type-text-runtime.ts'; /** * The whole local interactor-backed operation set a pointer-driving family binds: the uniform - * one-fact-one-bind leaves the interactor catalog walks, plus the four that carry extra input of their own — gesture and touch read the - * fact map, and touch needs a pause clock. Android, Linux, and HarmonyOS bind this one list, so + * one-fact-one-bind leaves the interactor catalog walks, plus the ones that carry extra input of their own — gesture and touch read the + * fact map, touch needs a pause clock, and screenshot takes the owner's capture declaration. Android, Linux, and HarmonyOS bind this one list, so * no pointer-driving family can drift from another by forgetting a member. Capture stays out: an * owner's snapshot mechanics are its own (Linux captures a surface through its host, Android * drives the interactor), so there is no shared reading to make. @@ -26,6 +26,7 @@ export function bindLocalInteractorOperationSet( resolveInteractor: LocalInteractorOperationResolver; facts: RuntimeFacts['operations']; pause: (milliseconds: number) => Promise; + captureCapabilities: Parameters[0]['captureCapabilities']; }>, ): Partial { const { facts, pause } = params; @@ -35,7 +36,12 @@ export function bindLocalInteractorOperationSet( resolveInteractor: params.resolveInteractor, }; return { - ...(facts.captureScreenshot.available ? bindLocalScreenshotInteractor(resolver) : {}), + ...(facts.captureScreenshot.available + ? bindLocalScreenshotInteractor({ + ...resolver, + captureCapabilities: params.captureCapabilities, + }) + : {}), ...(facts.focusPoint.available ? bindLocalFocusInteractor(resolver) : {}), ...bindLocalGestureInteractor({ ...resolver, facts }), ...(facts.scrollDirection.available ? bindLocalScrollInteractor(resolver) : {}), diff --git a/packages/contracts/src/screenshot-runtime.test.ts b/packages/contracts/src/screenshot-runtime.test.ts index 89a6b2f857..20bbd531d1 100644 --- a/packages/contracts/src/screenshot-runtime.test.ts +++ b/packages/contracts/src/screenshot-runtime.test.ts @@ -3,6 +3,7 @@ import type { Interactor } from './interactor-types.ts'; import { bindLocalScreenshotInteractor, bindProviderScreenshotInteractor, + defaultCaptureOnly, screenshotRuntimeOperationFacts, } from './screenshot-runtime.ts'; @@ -24,7 +25,12 @@ test('a local binding hands the interactor the destination, options, and the req const resolveInteractor = vi.fn(async () => ({ screenshot }) as unknown as Interactor); const signal = new AbortController().signal; - const operations = bindLocalScreenshotInteractor({ device, signal, resolveInteractor }); + const operations = bindLocalScreenshotInteractor({ + device, + signal, + resolveInteractor, + captureCapabilities: defaultCaptureOnly('test'), + }); await operations.captureScreenshot({ outPath: '/tmp/out.png', options: { appBundleId: 'com.example.app', fullscreen: true }, @@ -47,6 +53,7 @@ test('a provider binding fails closed when its exact owner exposes no interactor device, signal: new AbortController().signal, resolveInteractor: () => undefined, + captureCapabilities: defaultCaptureOnly('test'), }); await expect(operations.captureScreenshot({ outPath: '/tmp/out.png' })).rejects.toMatchObject({ @@ -63,6 +70,7 @@ test.each([bindLocalScreenshotInteractor, bindProviderScreenshotInteractor])( device: { ...device, platform: 'apple', appleOs: 'ios', kind: 'simulator' }, signal: new AbortController().signal, resolveInteractor, + captureCapabilities: defaultCaptureOnly('test'), }); await expect( operations.captureScreenshot({ @@ -92,3 +100,48 @@ test('the selected capture capability admits a mask for one invocation without c expect(screenshot.mock.lastCall).toEqual(['/tmp/capture.png', { screenMask }]); } }); + +test.each([bindLocalScreenshotInteractor, bindProviderScreenshotInteractor])( + 'an owner declaring no capture backend refuses an explicit one with its own explanation', + async (bind) => { + const resolveInteractor = vi.fn(); + const operations = bind({ + device, + signal: new AbortController().signal, + resolveInteractor, + captureCapabilities: defaultCaptureOnly('this owner has one capture path'), + }); + await expect( + operations.captureScreenshot({ + outPath: '/tmp/capture.png', + options: { captureBackend: 'runner' }, + }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + message: expect.stringContaining('this owner has one capture path'), + details: { reason: 'screenshot-capture-backend-unsupported' }, + }); + expect(resolveInteractor).not.toHaveBeenCalled(); + }, +); + +test.each([ + [['runner'] as const, { captureBackend: 'runner' }], + [[] as const, {}], +])( + 'a preferred backend selects a declared one and otherwise keeps the default capture: %j', + async (captureBackends, expected) => { + const screenshot = vi.fn(async (_outPath: string, _options: unknown) => ({})); + const operations = bindLocalScreenshotInteractor({ + device, + signal: new AbortController().signal, + resolveInteractor: async () => ({ screenshot }) as unknown as Interactor, + captureCapabilities: () => ({ captureBackends }), + }); + await operations.captureScreenshot({ + outPath: '/tmp/capture.png', + options: { captureBackend: undefined, preferredCaptureBackend: 'runner' }, + }); + expect(screenshot).toHaveBeenLastCalledWith('/tmp/capture.png', expected); + }, +); diff --git a/packages/contracts/src/screenshot-runtime.ts b/packages/contracts/src/screenshot-runtime.ts index f63f0c65d6..472be6fcc6 100644 --- a/packages/contracts/src/screenshot-runtime.ts +++ b/packages/contracts/src/screenshot-runtime.ts @@ -13,6 +13,7 @@ import type { ScreenshotOptions, } from './interactor-types.ts'; import type { RuntimeOperationFact } from './platform-runtime.ts'; +import type { ScreenshotCaptureBackend } from './screenshot.ts'; export type { ScreenshotCaptureFacts, ScreenshotOptions } from './interactor-types.ts'; @@ -26,7 +27,7 @@ export type ScreenshotRuntimeExecution = Readonly; + options?: Readonly; execution?: ScreenshotRuntimeExecution; }>; @@ -36,17 +37,36 @@ export type ScreenshotRuntimeOperations = Readonly<{ export type ScreenshotCaptureCapabilities = Readonly<{ screenMask?: boolean; + /** The backends a request can select on this owner; an absent list selects none. */ + captureBackends?: readonly ScreenshotCaptureBackend[]; + /** Why this owner cannot serve a selected backend. */ + captureBackendRefusal?: string; }>; type ScreenshotCaptureBindingOptions = Readonly<{ signal: AbortSignal; - captureCapabilities?: (options: Readonly) => ScreenshotCaptureCapabilities; + captureCapabilities: (options: Readonly) => ScreenshotCaptureCapabilities; }>; +/** The declaration of an owner that captures one way only, and why. */ +export function defaultCaptureOnly( + captureBackendRefusal: string, +): (options: Readonly) => ScreenshotCaptureCapabilities { + return () => ({ captureBackendRefusal }); +} + export async function assertScreenshotCaptureSupport( options: Readonly | undefined, capabilities: ScreenshotCaptureCapabilities = {}, ): Promise { + const backend = options?.captureBackend; + if (backend !== undefined && !capabilities.captureBackends?.includes(backend)) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + `--capture-backend ${backend} is not supported: ${capabilities.captureBackendRefusal ?? 'this capture owner declares no such backend'}`, + { reason: 'screenshot-capture-backend-unsupported' }, + ); + } if (options?.screenMask === undefined) return; const { readScreenshotScreenMask } = await import('./screenshot.ts'); const screenMask = readScreenshotScreenMask(options?.screenMask); @@ -70,6 +90,18 @@ export function screenshotRuntimeOperationFacts( return Object.freeze({ captureScreenshot: input.capture }); } +/** A preference, unlike `captureBackend`, keeps the owner's default where it declares no such backend. Maestro's comparison capture asks for one. */ +function resolveCaptureOptions( + input: CaptureScreenshotInput['options'], + binding: ScreenshotCaptureBindingOptions, +): Readonly { + const { captureBackend: explicit, preferredCaptureBackend: preferred, ...options } = input ?? {}; + if (explicit) return { ...options, captureBackend: explicit }; + return preferred && binding.captureCapabilities(options).captureBackends?.includes(preferred) + ? { ...options, captureBackend: preferred } + : options; +} + /** * Captures one selected owner's interactor authority for the lifetime of a request binding. The * owner is already chosen by the time a binder is called, so each entry point supplies its own @@ -81,16 +113,14 @@ function bindScreenshotCapture( ): ScreenshotRuntimeOperations { return Object.freeze({ captureScreenshot: async (input: CaptureScreenshotInput) => { - await assertScreenshotCaptureSupport( - input.options, - binding.captureCapabilities?.(input.options ?? {}), - ); + const options = resolveCaptureOptions(input.options, binding); + await assertScreenshotCaptureSupport(options, binding.captureCapabilities(options)); const interactor = await resolveInteractor({ ...input.execution, appBundleId: input.options?.appBundleId, signal: binding.signal, }); - return (await interactor.screenshot(input.outPath, input.options)) ?? {}; + return (await interactor.screenshot(input.outPath, options)) ?? {}; }, }); } diff --git a/packages/contracts/src/screenshot.ts b/packages/contracts/src/screenshot.ts index be62912d25..7b06ba3c6b 100644 --- a/packages/contracts/src/screenshot.ts +++ b/packages/contracts/src/screenshot.ts @@ -88,6 +88,11 @@ export const SCREENSHOT_FULLSCREEN_REASONS = { export type ScreenshotFullscreenReason = (typeof SCREENSHOT_FULLSCREEN_REASONS)[keyof typeof SCREENSHOT_FULLSCREEN_REASONS]; +/** The capture paths a screenshot can name instead of the platform default. */ +export const SCREENSHOT_CAPTURE_BACKENDS = ['runner'] as const; + +export type ScreenshotCaptureBackend = (typeof SCREENSHOT_CAPTURE_BACKENDS)[number]; + export const SCREENSHOT_COMMAND_FLAG_KEYS = [ 'out', 'overlayRefs', @@ -98,6 +103,7 @@ export const SCREENSHOT_COMMAND_FLAG_KEYS = [ 'screenshotScale', 'screenshotNoStabilize', 'screenshotNormalizeStatusBar', + 'screenshotCaptureBackend', ] as const; export const SCREENSHOT_ACTION_FLAG_KEYS = [ @@ -108,6 +114,7 @@ export const SCREENSHOT_ACTION_FLAG_KEYS = [ 'screenshotScale', 'screenshotNoStabilize', 'screenshotNormalizeStatusBar', + 'screenshotCaptureBackend', ] as const; type ScreenshotSpecificFlagKey = (typeof SCREENSHOT_ACTION_FLAG_KEYS)[number]; @@ -200,6 +207,17 @@ export const SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS: readonly ScreenshotSpecificFl projectConfig: true, recorded: true, }, + { + key: 'screenshotCaptureBackend', + names: ['--capture-backend'], + type: 'enum', + enumValues: SCREENSHOT_CAPTURE_BACKENDS, + usageLabel: '--capture-backend runner', + usageDescription: + 'Screenshot: on Apple targets capture through the XCTest runner (XCUIScreen), which leaves out the simulator-drawn Dynamic Island, instead of simctl; refused on other platforms, provider sessions and macOS surfaces the helper captures, and with --pixel-density, --screen-mask and --normalize-status-bar', + projectConfig: true, + recorded: false, + }, ]; const SCREENSHOT_SCRIPT_BOOLEAN_FLAGS = [ @@ -244,6 +262,7 @@ export type ScreenshotRequestFlags = { screenshotScale?: number; screenshotNoStabilize?: boolean; screenshotNormalizeStatusBar?: boolean; + screenshotCaptureBackend?: ScreenshotCaptureBackend; }; export type ScreenshotDispatchFlags = Pick< @@ -253,6 +272,7 @@ export type ScreenshotDispatchFlags = Pick< | 'screenshotFullscreen' | 'screenshotNoStabilize' | 'screenshotNormalizeStatusBar' + | 'screenshotCaptureBackend' >; export type ScreenshotRuntimeFlags = Pick< @@ -264,6 +284,7 @@ export type ScreenshotRuntimeFlags = Pick< | 'screenshotScale' | 'screenshotNoStabilize' | 'screenshotNormalizeStatusBar' + | 'screenshotCaptureBackend' >; export type ScreenshotPublicOptions = { @@ -275,6 +296,7 @@ export type ScreenshotPublicOptions = { scale?: number; stabilize?: boolean; normalizeStatusBar?: boolean; + captureBackend?: ScreenshotCaptureBackend; }; export type ScreenshotRuntimeOptions = { @@ -286,6 +308,7 @@ export type ScreenshotRuntimeOptions = { scale?: number; stabilize?: boolean; normalizeStatusBar?: boolean; + captureBackend?: ScreenshotCaptureBackend; }; export function screenshotOptionsFromFlags( @@ -301,6 +324,7 @@ export function screenshotOptionsFromFlags( stabilize: flags?.screenshotNoStabilize === undefined ? undefined : !flags.screenshotNoStabilize, normalizeStatusBar: flags?.screenshotNormalizeStatusBar, + captureBackend: flags?.screenshotCaptureBackend, }); } @@ -321,6 +345,7 @@ export function screenshotFlagsFromOptions( (options.stabilize === undefined ? undefined : !options.stabilize), screenshotNormalizeStatusBar: options.screenshotNormalizeStatusBar ?? options.normalizeStatusBar, + screenshotCaptureBackend: options.screenshotCaptureBackend ?? options.captureBackend, }); } diff --git a/packages/daemon/src/core/__tests__/context.test.ts b/packages/daemon/src/core/__tests__/context.test.ts index 92ead9909a..a9b82155cf 100644 --- a/packages/daemon/src/core/__tests__/context.test.ts +++ b/packages/daemon/src/core/__tests__/context.test.ts @@ -8,11 +8,17 @@ test('contextFromFlags propagates back mode into the dispatch context', () => { assert.equal(context.backMode, 'system'); }); -test('contextFromFlags forwards the Maestro runner screenshot backend', () => { - const context = contextFromFlags('/tmp/agent-device.log', { +test('contextFromFlags keeps an explicit capture backend apart from the Maestro preference', () => { + const explicit = contextFromFlags('/tmp/agent-device.log', { + screenshotCaptureBackend: 'runner', + }); + assert.equal(explicit.screenshotCaptureBackend, 'runner'); + assert.equal(explicit.preferredScreenshotCaptureBackend, undefined); + const preferred = contextFromFlags('/tmp/agent-device.log', { maestro: { screenshotCaptureBackend: 'runner' }, }); - assert.equal(context.screenshotCaptureBackend, 'runner'); + assert.equal(preferred.screenshotCaptureBackend, undefined); + assert.equal(preferred.preferredScreenshotCaptureBackend, 'runner'); }); test('contextFromFlags forwards scroll pixels from CLI flags', () => { diff --git a/packages/daemon/src/core/context.ts b/packages/daemon/src/core/context.ts index 0748c89ee8..ca55f9207e 100644 --- a/packages/daemon/src/core/context.ts +++ b/packages/daemon/src/core/context.ts @@ -39,7 +39,7 @@ export function contextFromFlags( logPath, traceLogPath, ...dispatchContextFlags(flags), - screenshotCaptureBackend: flags?.maestro?.screenshotCaptureBackend, + preferredScreenshotCaptureBackend: flags?.maestro?.screenshotCaptureBackend, ...screenshotFlagsFromOptions(flags), clickButton: resolveClickButton(flags), }; diff --git a/packages/daemon/src/core/dispatch-context.ts b/packages/daemon/src/core/dispatch-context.ts index 3625426184..52b6ff7aec 100644 --- a/packages/daemon/src/core/dispatch-context.ts +++ b/packages/daemon/src/core/dispatch-context.ts @@ -85,6 +85,7 @@ export type DispatchContext = ScreenshotDispatchFlags & traceLogPath?: string; runnerLeaseContext?: RunnerLogicalLeaseContext; screenshotCaptureBackend?: 'runner'; + preferredScreenshotCaptureBackend?: 'runner'; snapshotIncludeRects?: boolean; skipIosSimulatorBootCheck?: boolean; /** Maestro replay compatibility for coordinate-resolved fill targets. */ diff --git a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts index 2866b2bd12..bd0835f731 100644 --- a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts +++ b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts @@ -2,8 +2,13 @@ import { daemonPlatformServicesFixture } from '../__tests__/support/platform-ser import fs from 'node:fs'; import path from 'node:path'; -import { expect, test } from 'vitest'; +import { expect, test, vi } from 'vitest'; import { SCREENSHOT_CROP_REASONS } from '@agent-device/contracts/capture'; +import type { Interactor, ScreenshotOptions } from '@agent-device/contracts/interactor-types'; +import { + bindLocalScreenshotInteractor, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { ANDROID_EMULATOR, @@ -12,6 +17,7 @@ import { mkdtempForTestSync, } from '@agent-device/test-kit'; import { makeSession } from '../__tests__/support/session-factories.ts'; +import { contextFromFlags } from '../core/context.ts'; import type { GenericPlatformExecutionParams } from '../core/generic-leaf-execution.ts'; import { resolveScreenshotGenericExecution } from './screenshot-runtime.ts'; import { @@ -251,6 +257,146 @@ test('rejects --pixel-density outside iOS-family simulators before inspecting ow expect(fixture.binds).toHaveLength(0); }); +test('refuses --capture-backend runner with --pixel-density or --normalize-status-bar, and any other backend, before inspecting owner facts', async () => { + const refused = async ( + device: SessionState['device'], + // The wire flags are untyped, so a request can name a backend the type does not list. + flags: Record, + code: string, + ): Promise => { + const fixture = screenshotRuntimeFixture(); + await expect( + resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req: screenshotRequest({ + positionals: ['/tmp/runner.png'], + flags: flags as DaemonRequest['flags'], + }), + session: makeSession('default', { device }), + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }), + ).rejects.toMatchObject({ code }); + expect(fixture.binds).toHaveLength(0); + }; + + await refused( + IOS_SIMULATOR, + { screenshotCaptureBackend: 'runner', screenshotPixelDensity: 3 }, + 'INVALID_ARGS', + ); + await refused( + IOS_SIMULATOR, + { screenshotCaptureBackend: 'runner', screenshotNormalizeStatusBar: true }, + 'INVALID_ARGS', + ); + await refused(IOS_SIMULATOR, { screenshotCaptureBackend: 'simctl' }, 'INVALID_ARGS'); +}); + +async function captureThroughRealBinding( + params: Readonly<{ + device: SessionState['device']; + flags: DaemonRequest['flags']; + captureCapabilities: Parameters[0]['captureCapabilities']; + }>, +) { + const fixture = screenshotRuntimeFixture(); + const screenshot = vi.fn(async (outPath: string, _options?: ScreenshotOptions) => { + writeSolidPng(outPath); + return {}; + }); + fixture.captureScreenshot.mockImplementation( + bindLocalScreenshotInteractor({ + device: params.device, + signal: new AbortController().signal, + resolveInteractor: async () => ({ screenshot }) as unknown as Interactor, + captureCapabilities: params.captureCapabilities, + }).captureScreenshot, + ); + const session = makeSession('capture', { device: params.device }); + const req = screenshotRequest({ + positionals: [path.join(mkdtempForTestSync('agent-device-bound-capture'), 'capture.png')], + flags: params.flags, + }); + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + return { + screenshot, + execute: async () => + await executeResult(resolved, session, req, contextFromFlags('/tmp/daemon.log', req.flags)), + }; +} + +test('a Maestro comparison capture reaches the interactor as a runner capture where the owner declares it', async () => { + const { screenshot, execute } = await captureThroughRealBinding({ + device: IOS_SIMULATOR, + flags: { maestro: { screenshotCaptureBackend: 'runner' } }, + captureCapabilities: () => ({ captureBackends: ['runner'] }), + }); + + await execute(); + + expect(screenshot.mock.calls[0]?.[1]).toMatchObject({ captureBackend: 'runner' }); + expect(screenshot.mock.calls[0]?.[1]).not.toHaveProperty('preferredCaptureBackend'); +}); + +test('a Maestro comparison capture keeps the default capture of an owner that declares no runner', async () => { + const { screenshot, execute } = await captureThroughRealBinding({ + device: IOS_SIMULATOR, + flags: { maestro: { screenshotCaptureBackend: 'runner' } }, + captureCapabilities: defaultCaptureOnly('this owner has one capture path'), + }); + + await execute(); + + expect(screenshot.mock.calls[0]?.[1]).not.toHaveProperty('captureBackend'); +}); + +test('an Android session refuses --capture-backend runner at its capture owner', async () => { + const { screenshot, execute } = await captureThroughRealBinding({ + device: ANDROID_EMULATOR, + flags: { screenshotCaptureBackend: 'runner' }, + captureCapabilities: defaultCaptureOnly('an Android device captures through adb'), + }); + + await expect(execute()).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-capture-backend-unsupported' }, + }); + expect(screenshot).not.toHaveBeenCalled(); +}); + +test('a --capture-backend runner request reaches the bound capture', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('ios', { device: IOS_SIMULATOR }); + const req = screenshotRequest({ + positionals: [path.join(mkdtempForTestSync('agent-device-runner-capture'), 'runner.png')], + flags: { screenshotCaptureBackend: 'runner' }, + }); + + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + await executeResult(resolved, session, req, contextFromFlags('/tmp/daemon.log', req.flags)); + expect(fixture.captureScreenshot.mock.calls[0]?.[0]?.options?.captureBackend).toBe('runner'); +}); + const CROP_TREE_ROOT = { index: 0, depth: 0, diff --git a/packages/daemon/src/execution/screenshot-runtime.ts b/packages/daemon/src/execution/screenshot-runtime.ts index 1b0196ab1c..5f9f2f7bc3 100644 --- a/packages/daemon/src/execution/screenshot-runtime.ts +++ b/packages/daemon/src/execution/screenshot-runtime.ts @@ -2,6 +2,7 @@ import { expandSessionPath } from '@agent-device/host-kit/session-paths'; import type { CommandFlags } from '@agent-device/contracts/command'; import { retiredScreenshotMaxSizeFlagError, + SCREENSHOT_CAPTURE_BACKENDS, screenshotFlagsFromOptions, screenshotOptionsFromFlags, } from '@agent-device/contracts/capture'; @@ -64,6 +65,7 @@ export async function resolveScreenshotGenericExecution( const retiredMaxSize = retiredScreenshotMaxSizeFlagError('screenshot', req.flags); if (retiredMaxSize) throw new AppError('INVALID_ARGS', retiredMaxSize); assertSupportedScreenshotPixelDensity(session.device, req.flags?.screenshotPixelDensity); + assertSupportedScreenshotCaptureBackend(req.flags); const request = readScreenshotRequest(req, params.hostCwd); const cropOn = @@ -317,6 +319,24 @@ async function annotateScreenshotWithSessionRefs( }); } +/** Each capture owner declares its backends; what stays here is the shape of the request. */ +function assertSupportedScreenshotCaptureBackend(flags: DaemonRequest['flags']): void { + const backend: unknown = flags?.screenshotCaptureBackend; + if (backend === undefined) return; + if (!(SCREENSHOT_CAPTURE_BACKENDS as readonly unknown[]).includes(backend)) { + throw new AppError( + 'INVALID_ARGS', + `--capture-backend takes ${SCREENSHOT_CAPTURE_BACKENDS.join(', ')}`, + ); + } + if (flags?.screenshotPixelDensity !== undefined || flags?.screenshotNormalizeStatusBar) { + throw new AppError( + 'INVALID_ARGS', + '--pixel-density and --normalize-status-bar cannot be combined with --capture-backend runner: they belong to the simctl capture', + ); + } +} + /** * `screenshot [path]` and `--out ` are the same destination expressed two ways, and both are * user-typed paths resolved against the client's cwd. The normalized form is also what the session @@ -372,7 +392,8 @@ function createBoundScreenshotBackend( skipIosSimulatorBootCheck: dispatchContext.skipIosSimulatorBootCheck ?? (isIosFamily(session.device) && session.device.kind === 'simulator'), - captureBackend: dispatchContext.screenshotCaptureBackend, + captureBackend: resolved.captureBackend, + preferredCaptureBackend: dispatchContext.preferredScreenshotCaptureBackend, }, execution: screenshotExecutionFromContext(dispatchContext), }); diff --git a/packages/platform-android/src/runtime.test.ts b/packages/platform-android/src/runtime.test.ts index 883ced36b7..4ad8e7738f 100644 --- a/packages/platform-android/src/runtime.test.ts +++ b/packages/platform-android/src/runtime.test.ts @@ -1,4 +1,5 @@ import { expect, test, vi } from 'vitest'; +import type { Interactor } from '@agent-device/contracts/interactor-types'; import { HARDWARE_BUTTON_NAMES, parseHardwareButtonPositionals, @@ -663,3 +664,27 @@ function deviceSerial(invocation: AndroidAdbInvocation): string { if (invocation.target.selector.kind !== 'serial') throw new Error('expected a serial target'); return invocation.target.selector.serial; } + +test('an Android capture refuses an explicit runner request at its own binding', async () => { + const screenshot = vi.fn(async () => ({})); + const resolve = vi.fn(async () => ({ screenshot }) as unknown as Interactor); + const binding = await bindOrdinary( + createAndroidPlatformRuntime({ + ...androidRuntimeHost(), + localInteractors: { resolve }, + }), + ANDROID_EMULATOR, + ); + + await expect( + binding.operations.captureScreenshot!({ + outPath: '/tmp/runner.png', + options: { captureBackend: 'runner' }, + }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + message: expect.stringContaining('adb'), + details: { reason: 'screenshot-capture-backend-unsupported' }, + }); + expect(resolve).not.toHaveBeenCalled(); +}); diff --git a/packages/platform-android/src/runtime.ts b/packages/platform-android/src/runtime.ts index cdecf763ad..020efe3a71 100644 --- a/packages/platform-android/src/runtime.ts +++ b/packages/platform-android/src/runtime.ts @@ -29,7 +29,10 @@ import { } from '@agent-device/contracts/platform-runtime'; import { audioProbeRuntimeOperationFacts } from '@agent-device/contracts/audio-probe-runtime'; import { perfRuntimeOperationFacts } from '@agent-device/contracts/perf-runtime'; -import { screenshotRuntimeOperationFacts } from '@agent-device/contracts/screenshot-runtime'; +import { + screenshotRuntimeOperationFacts, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import { selectorObservationRuntimeOperationFacts } from '@agent-device/contracts/selector-observation-runtime'; import { bindLocalSnapshotInteractor, @@ -571,6 +574,7 @@ function androidInteractionOperations( ...bindLocalInteractorOperationSet({ ...resolver, facts: facts.operations, + captureCapabilities: defaultCaptureOnly('an Android device captures through adb'), pause: async (milliseconds) => await host.clock.sleep(milliseconds, request.scope.signal), }), }; diff --git a/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts b/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts index 57e6215928..a18f45e750 100644 --- a/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts @@ -10,7 +10,13 @@ vi.mock('../core/screenshot.ts', async (importOriginal) => { }; }); +vi.mock('../os/macos/helper.ts', async (importOriginal) => ({ + ...(await importOriginal()), + runMacOsScreenshotAction: vi.fn(), +})); + import { createAppleInteractor } from '../interactor.ts'; +import { runMacOsScreenshotAction } from '../os/macos/helper.ts'; import { captureScreenshotViaRunner, screenshotIos } from '../core/screenshot.ts'; const device: DeviceInfo = { @@ -88,3 +94,22 @@ test.each([ expect(screenshotIos).not.toHaveBeenCalled(); expect(captureScreenshotViaRunner).not.toHaveBeenCalled(); }); + +test('an explicit runner capture never succeeds through the macOS helper', async () => { + const macOs: DeviceInfo = { ...device, appleOs: 'macos', kind: 'device', target: 'desktop' }; + const refused = async (surface: 'desktop' | undefined, env: NodeJS.ProcessEnv) => { + const interactor = createAppleInteractor(macOs, {}, { env, cwd: process.cwd() }); + await expect( + interactor.screenshot('/tmp/refused.png', { captureBackend: 'runner', surface }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-capture-backend-unsupported' }, + }); + }; + + await refused('desktop', {}); + await refused(undefined, { AGENT_DEVICE_MACOS_APP_BACKEND: 'native' }); + + expect(runMacOsScreenshotAction).not.toHaveBeenCalled(); + expect(captureScreenshotViaRunner).not.toHaveBeenCalled(); +}); diff --git a/packages/platform-apple/src/core/__tests__/screenshot.test.ts b/packages/platform-apple/src/core/__tests__/screenshot.test.ts index 0ccc8457c2..9f33fd2698 100644 --- a/packages/platform-apple/src/core/__tests__/screenshot.test.ts +++ b/packages/platform-apple/src/core/__tests__/screenshot.test.ts @@ -889,6 +889,77 @@ test.each([MACOS_TEST_DEVICE, { ...IOS_TEST_SIMULATOR, kind: 'device' as const } }, ); +test.each([ + ['the default capture', undefined, true], + ['an explicit runner capture', undefined, true], + ['an explicit runner capture', false, false], +] as const)( + '%s under a runner-backed tool provider asks the runner with fullscreen %s: %s', + async (route, fullscreen, expected) => { + mockRunAppleRunnerCommand.mockRejectedValue(new Error('stop after the runner request')); + const capture = async () => + route === 'the default capture' + ? await captureSimulatorScreenshotWithFallback(IOS_TEST_SIMULATOR, '/tmp/runner.png', { + env: {}, + fullscreen, + skipIosSimulatorBootCheck: true, + deps: { resolveCaptureTarget: async () => ({}) }, + }) + : await captureScreenshotViaRunner( + IOS_TEST_SIMULATOR, + '/tmp/runner.png', + undefined, + fullscreen, + undefined, + {}, + ); + await withAppleToolProvider( + { ...resolveAppleToolProvider(), simulatorScreenshotBackend: 'runner' }, + async () => { + await assert.rejects(capture(), /stop after the runner request/); + }, + ); + assert.equal(mockRunAppleRunnerCommand.mock.calls[0]?.[1]?.fullscreen, expected); + }, +); + +test('a macOS runner capture under a runner-backed tool provider skips the fullscreen default', async () => { + mockRunAppleRunnerCommand.mockRejectedValue(new Error('stop after the runner request')); + await withAppleToolProvider( + { ...resolveAppleToolProvider(), simulatorScreenshotBackend: 'runner' }, + async () => { + await assert.rejects( + captureScreenshotViaRunner( + { ...MACOS_TEST_DEVICE, kind: 'simulator' }, + '/tmp/runner.png', + undefined, + undefined, + undefined, + {}, + ), + /stop after the runner request/, + ); + }, + ); + assert.equal(mockRunAppleRunnerCommand.mock.calls[0]?.[1]?.fullscreen, undefined); +}); + +test('an explicit runner capture outside a runner-backed tool provider leaves fullscreen unset', async () => { + mockRunAppleRunnerCommand.mockRejectedValue(new Error('stop after the runner request')); + await assert.rejects( + captureScreenshotViaRunner( + IOS_TEST_SIMULATOR, + '/tmp/runner.png', + undefined, + undefined, + undefined, + {}, + ), + /stop after the runner request/, + ); + assert.equal(mockRunAppleRunnerCommand.mock.calls[0]?.[1]?.fullscreen, undefined); +}); + test('screenshotIos rejects a screen mask before a provider-owned runner capture', async () => { mockRunCmd.mockRejectedValue(new Error('unexpected native capture')); mockRunAppleRunnerCommand.mockRejectedValue(new Error('unexpected runner capture')); diff --git a/packages/platform-apple/src/core/screenshot.ts b/packages/platform-apple/src/core/screenshot.ts index 8dcdc9a275..e63a543cc7 100644 --- a/packages/platform-apple/src/core/screenshot.ts +++ b/packages/platform-apple/src/core/screenshot.ts @@ -210,8 +210,7 @@ export async function captureSimulatorScreenshotWithFallback( device, outPath, options.appBundleId, - options.fullscreen ?? - (resolveAppleToolProvider().simulatorScreenshotBackend === 'runner' ? true : undefined), + options.fullscreen, options.runnerOptions, options.env, ); @@ -287,10 +286,16 @@ export async function captureScreenshotViaRunner( device: DeviceInfo, outPath: string, appBundleId: string | undefined, - fullscreen: boolean | undefined, + requestedFullscreen: boolean | undefined, runnerOptions: AppleRunnerCommandOptions | undefined, env: Readonly, ): Promise { + const fullscreen = + requestedFullscreen ?? + (capturesThroughSimulatorStatusBar(device) && + resolveAppleToolProvider().simulatorScreenshotBackend === 'runner' + ? true + : undefined); if (!capturesThroughSimulatorStatusBar(device)) { return await captureRunnerScreenshot( device, diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index d58e99ef25..15e63d15b1 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -558,7 +558,7 @@ async function runAppleScreenshot( ): Promise { await assertScreenshotCaptureSupport( options, - appleScreenshotCaptureCapabilities(device, options), + appleScreenshotCaptureCapabilities(device, options, helper), ); if (helper) { if (options.fullscreen) { diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index 19e2cd3c5b..efe68a92f6 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -63,7 +63,7 @@ import { createAppleSystemOperations } from './system/runtime.ts'; import { appleFoldableFacts, createAppleFoldableOperations } from './foldable/runtime.ts'; import { bindAppleFindTextRuntime, bindAppleSnapshotRuntime } from './runtime-snapshot.ts'; import { bindAppleQuerySelectorRuntime } from './runtime-selector-query.ts'; -import type { MacOsAppBackend } from '@agent-device/contracts/session'; +import { macOsHelperSurface, type MacOsAppBackend } from '@agent-device/contracts/session'; import { hostMacOsAppBackend } from './os/macos/app-backend.ts'; import { macOsNativeBackendFacts } from './os/macos/native-backend-facts.ts'; import { createAppleSnapshotRoute } from './snapshot-route.ts'; @@ -325,7 +325,13 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR signal: request.scope.signal, resolveInteractor: host.localInteractors.resolve, captureCapabilities: (options) => - appleScreenshotCaptureCapabilities(request.device, options), + appleScreenshotCaptureCapabilities( + request.device, + options, + isMacOs(request.device) + ? macOsHelperSurface(options.surface, macOsAppBackend(request.device)) + : undefined, + ), }), ), ...whenAdmitted(facts.operations.focusPoint, () => diff --git a/packages/platform-apple/src/screenshot-support.test.ts b/packages/platform-apple/src/screenshot-support.test.ts index 6f87abb83d..9ccd0cab1d 100644 --- a/packages/platform-apple/src/screenshot-support.test.ts +++ b/packages/platform-apple/src/screenshot-support.test.ts @@ -1,6 +1,7 @@ import { expect, test, vi } from 'vitest'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { Interactor, ScreenshotOptions } from '@agent-device/contracts/interactor-types'; +import type { MacOsAppBackend, SessionSurface } from '@agent-device/contracts/session'; import { createApplePlatformRuntime } from './runtime.ts'; import { platformRuntimeHostFixture } from './runtime.fixtures.ts'; @@ -57,3 +58,85 @@ test.each([ } }, ); + +const macOs: DeviceInfo = { + platform: 'apple', + appleOs: 'macos', + id: 'host-macos-local', + name: 'Mac', + kind: 'device', + target: 'desktop', + booted: true, +}; + +test.each([ + [simulator, undefined, undefined, true], + [{ ...simulator, kind: 'device' }, undefined, undefined, true], + [{ ...simulator, appleOs: 'tvos', target: 'tv' }, undefined, undefined, true], + [macOs, undefined, undefined, true], + [macOs, 'app', 'xctest', true], + [macOs, undefined, 'native', false], + [macOs, 'app', 'native', false], + [macOs, 'desktop', undefined, false], + [macOs, 'menubar', undefined, false], + [macOs, 'frontmost-app', undefined, false], +] satisfies [DeviceInfo, SessionSurface | undefined, MacOsAppBackend | undefined, boolean][])( + 'the bound Apple capture declares runner support for %j on surface %s with app backend %s: %s', + async (device, surface, appBackend, supported) => { + const screenshot = vi.fn(async () => ({})); + const resolve = vi.fn(async () => ({ screenshot }) as unknown as Interactor); + const host = { + ...platformRuntimeHostFixture(), + hostEnvironment: { + env: appBackend ? { AGENT_DEVICE_MACOS_APP_BACKEND: appBackend } : {}, + cwd: process.cwd(), + }, + localInteractors: { resolve }, + }; + const binding = await createApplePlatformRuntime(host).bind({ + device, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + const options = { captureBackend: 'runner', surface } as const; + const result = binding.operations.captureScreenshot!({ outPath: '/tmp/r.png', options }); + if (supported) { + await expect(result).resolves.toEqual({}); + expect(screenshot).toHaveBeenCalledWith('/tmp/r.png', options); + } else { + await expect(result).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-capture-backend-unsupported' }, + }); + expect(resolve).not.toHaveBeenCalled(); + } + }, +); + +test('the shape the daemon passes for a Maestro preference selects the runner on an iOS simulator', async () => { + const screenshot = vi.fn(async (_outPath: string, _options: ScreenshotOptions) => ({})); + const host = { + ...platformRuntimeHostFixture(), + localInteractors: { resolve: async () => ({ screenshot }) as unknown as Interactor }, + }; + const binding = await createApplePlatformRuntime(host).bind({ + device: simulator, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + + await binding.operations.captureScreenshot!({ + outPath: '/tmp/maestro.png', + options: { captureBackend: undefined, preferredCaptureBackend: 'runner' }, + }); + + expect(screenshot.mock.lastCall?.[1]).toEqual({ captureBackend: 'runner' }); +}); diff --git a/packages/platform-apple/src/screenshot-support.ts b/packages/platform-apple/src/screenshot-support.ts index 9a24a726f8..5397c5bf60 100644 --- a/packages/platform-apple/src/screenshot-support.ts +++ b/packages/platform-apple/src/screenshot-support.ts @@ -1,4 +1,5 @@ import { isHandheldAppleSimulator, type DeviceInfo } from '@agent-device/kernel/device'; +import type { MacOsHelperSurface } from '@agent-device/contracts/session'; import { resolveAppleToolProvider } from './core/tool-provider.ts'; import type { ScreenshotCaptureCapabilities, @@ -8,11 +9,14 @@ import type { export function appleScreenshotCaptureCapabilities( device: DeviceInfo, options: Pick, + helper?: MacOsHelperSurface, ): ScreenshotCaptureCapabilities { return { screenMask: isHandheldAppleSimulator(device) && options.captureBackend !== 'runner' && resolveAppleToolProvider().simulatorScreenshotBackend !== 'runner', + captureBackends: helper === undefined ? ['runner'] : [], + captureBackendRefusal: 'the macOS helper captures this surface', }; } diff --git a/packages/platform-harmonyos/src/runtime.ts b/packages/platform-harmonyos/src/runtime.ts index 0325c77f0b..52c3aa837e 100644 --- a/packages/platform-harmonyos/src/runtime.ts +++ b/packages/platform-harmonyos/src/runtime.ts @@ -29,7 +29,10 @@ import { unavailableFact, whenAdmitted, } from '@agent-device/contracts/platform-runtime'; -import { screenshotRuntimeOperationFacts } from '@agent-device/contracts/screenshot-runtime'; +import { + screenshotRuntimeOperationFacts, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import { selectorObservationRuntimeOperationFacts } from '@agent-device/contracts/selector-observation-runtime'; import { bindLocalSnapshotInteractor, @@ -341,6 +344,7 @@ export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): Platfor signal: request.scope.signal, resolveInteractor: host.localInteractors.resolve, facts: facts.operations, + captureCapabilities: defaultCaptureOnly('a HarmonyOS device captures through hdc'), pause: async (milliseconds) => await host.clock.sleep(milliseconds, request.scope.signal), }), diff --git a/packages/platform-linux/src/runtime.ts b/packages/platform-linux/src/runtime.ts index c04c74bfd2..7f0230b711 100644 --- a/packages/platform-linux/src/runtime.ts +++ b/packages/platform-linux/src/runtime.ts @@ -28,7 +28,10 @@ import { unavailableFact, } from '@agent-device/contracts/platform-runtime'; import { createUnavailablePlatformRuntimeFacts } from '@agent-device/contracts/platform-runtime-unavailable'; -import { screenshotRuntimeOperationFacts } from '@agent-device/contracts/screenshot-runtime'; +import { + screenshotRuntimeOperationFacts, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import { captureSnapshotSignal, snapshotRuntimeOperationFacts, @@ -179,6 +182,7 @@ function linuxInteractionOperations( return bindLocalInteractorOperationSet({ ...resolver, facts: facts.operations, + captureCapabilities: defaultCaptureOnly('a Linux target captures through its desktop tools'), pause: async (milliseconds) => await host.clock.sleep(milliseconds, request.scope.signal), }); } diff --git a/packages/platform-web/src/runtime.ts b/packages/platform-web/src/runtime.ts index b571fb1a36..1e26106963 100644 --- a/packages/platform-web/src/runtime.ts +++ b/packages/platform-web/src/runtime.ts @@ -30,6 +30,7 @@ import { import { bindLocalScreenshotInteractor, screenshotRuntimeOperationFacts, + defaultCaptureOnly, } from '@agent-device/contracts/screenshot-runtime'; import { selectorObservationRuntimeOperationFacts } from '@agent-device/contracts/selector-observation-runtime'; import { @@ -247,6 +248,7 @@ function bindWebRuntime( device, signal, resolveInteractor: host.localInteractors.resolve, + captureCapabilities: defaultCaptureOnly('a web target captures through its browser'), }) : {}), ...(facts.operations.focusPoint.available diff --git a/packages/provider-limrun/src/interaction-operations.test.ts b/packages/provider-limrun/src/interaction-operations.test.ts index c485f27e53..8f8c3d4a2e 100644 --- a/packages/provider-limrun/src/interaction-operations.test.ts +++ b/packages/provider-limrun/src/interaction-operations.test.ts @@ -7,8 +7,12 @@ import { import type { Interactor, RunnerContext } from '@agent-device/contracts/interactor-types'; import { bindAdmittedProviderInteractorOperations } from '@agent-device/contracts/interactor-operation-catalog'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { expect, test } from 'vitest'; -import { limrunAndroidSettingsFacts, limrunAppleSettingsFacts } from './interaction-operations.ts'; +import { expect, test, vi } from 'vitest'; +import { + bindLimrunInteractionOperations, + limrunAndroidSettingsFacts, + limrunAppleSettingsFacts, +} from './interaction-operations.ts'; import { limrunAppLogFacts, limrunAppLogRecoveryFacts, @@ -264,3 +268,34 @@ test('long press is admitted on both direct-session legs and closed with a dead }); expect(factsFor(iosDevice, false).longPressPoint).toEqual(liveSessionUnavailable); }); + +test('a provider iOS capture refuses an explicit runner request instead of returning a provider screenshot', async () => { + const screenshot = vi.fn(async () => ({})); + const operations = bindLimrunInteractionOperations({ + device: iosDevice, + signal: new AbortController().signal, + getInteractor: () => ({ screenshot }) as unknown as Interactor, + presentIosAcquisition: (() => { + throw new Error('unused'); + }) as never, + facts: factsFor(iosDevice), + }); + + await expect( + operations.captureScreenshot({ + outPath: '/tmp/runner.png', + options: { captureBackend: 'runner' }, + }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + message: expect.stringContaining('Limrun'), + details: { reason: 'screenshot-capture-backend-unsupported' }, + }); + expect(screenshot).not.toHaveBeenCalled(); + + await operations.captureScreenshot({ + outPath: '/tmp/maestro.png', + options: { preferredCaptureBackend: 'runner' }, + }); + expect(screenshot).toHaveBeenCalledWith('/tmp/maestro.png', {}); +}); diff --git a/packages/provider-limrun/src/interaction-operations.ts b/packages/provider-limrun/src/interaction-operations.ts index 0922f8d53c..09e5657e6f 100644 --- a/packages/provider-limrun/src/interaction-operations.ts +++ b/packages/provider-limrun/src/interaction-operations.ts @@ -33,7 +33,10 @@ import { import { clipboardRuntimeOperationFacts } from '@agent-device/contracts/clipboard-runtime'; import { keyboardRuntimeOperationFacts } from '@agent-device/contracts/keyboard-runtime'; import { orientationRuntimeOperationFacts } from '@agent-device/contracts/orientation-runtime'; -import { bindProviderScreenshotInteractor } from '@agent-device/contracts/screenshot-runtime'; +import { + bindProviderScreenshotInteractor, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import { bindProviderSnapshotInteractor, type SnapshotRuntimePresenter, @@ -150,7 +153,12 @@ export function bindLimrunInteractionOperations( facts: params.facts, pause: async (milliseconds) => await sleep(milliseconds, undefined, { signal }), }), - ...bindProviderScreenshotInteractor({ device, signal, resolveInteractor }), + ...bindProviderScreenshotInteractor({ + device, + signal, + resolveInteractor, + captureCapabilities: defaultCaptureOnly('Limrun captures through its provider endpoint'), + }), ...bindProviderGestureInteractor({ device, signal, diff --git a/packages/provider-revyl/src/interaction-operations.ts b/packages/provider-revyl/src/interaction-operations.ts index cdac7a94d2..85129752dd 100644 --- a/packages/provider-revyl/src/interaction-operations.ts +++ b/packages/provider-revyl/src/interaction-operations.ts @@ -25,7 +25,10 @@ import { bindProviderTouchInteractor, touchRuntimeOperationFacts, } from '@agent-device/contracts/touch-runtime'; -import { bindProviderScreenshotInteractor } from '@agent-device/contracts/screenshot-runtime'; +import { + bindProviderScreenshotInteractor, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import { bindProviderSnapshotInteractor, type SnapshotRuntimePresenter, @@ -239,7 +242,12 @@ export function bindRevylInteractionOperations( resolveInteractor, presentIosAcquisition: params.presentIosAcquisition, }), - ...bindProviderScreenshotInteractor({ device, signal, resolveInteractor }), + ...bindProviderScreenshotInteractor({ + device, + signal, + resolveInteractor, + captureCapabilities: defaultCaptureOnly('Revyl captures through its provider endpoint'), + }), ...bindProviderFocusInteractor({ device, signal, resolveInteractor }), ...bindProviderTypeTextInteractor({ device, signal, resolveInteractor }), ...bindProviderTouchInteractor({ diff --git a/packages/provider-webdriver/src/platform-runtime.test.ts b/packages/provider-webdriver/src/platform-runtime.test.ts index 0bf08f14a4..21d5603747 100644 --- a/packages/provider-webdriver/src/platform-runtime.test.ts +++ b/packages/provider-webdriver/src/platform-runtime.test.ts @@ -293,6 +293,15 @@ test('captures through only the active exact WebDriver interactor', async () => expect(binding.operations.setViewport).toBeUndefined(); expect(binding.facts.operations.captureScreenshot).toEqual({ available: true }); expect(binding.operations.captureScreenshot).toBeTypeOf('function'); + await expect( + binding.operations.captureScreenshot!({ + outPath: '/tmp/runner.png', + options: { captureBackend: 'runner' }, + }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-capture-backend-unsupported' }, + }); // R40/R41: interaction cells share the captures' reachability gate and interactor. expect(binding.facts.operations.focusPoint).toEqual({ available: true }); expect(binding.facts.operations.typeText).toEqual({ available: true }); diff --git a/packages/provider-webdriver/src/platform-runtime.ts b/packages/provider-webdriver/src/platform-runtime.ts index b4ebe45a58..78e1d32886 100644 --- a/packages/provider-webdriver/src/platform-runtime.ts +++ b/packages/provider-webdriver/src/platform-runtime.ts @@ -52,6 +52,7 @@ import { import { bindProviderScreenshotInteractor, screenshotRuntimeOperationFacts, + defaultCaptureOnly, } from '@agent-device/contracts/screenshot-runtime'; import { bindProviderSnapshotInteractor, @@ -389,7 +390,12 @@ function webDriverInteractionOperations( return { ...(facts.operations.captureSnapshot.available ? bindProviderSnapshotInteractor(resolver) : {}), ...(facts.operations.captureScreenshot.available - ? bindProviderScreenshotInteractor(resolver) + ? bindProviderScreenshotInteractor({ + ...resolver, + captureCapabilities: defaultCaptureOnly( + 'a WebDriver session captures through its provider', + ), + }) : {}), ...(facts.operations.focusPoint.available ? bindProviderFocusInteractor(resolver) : {}), ...(facts.operations.typeText.available ? bindProviderTypeTextInteractor(resolver) : {}), diff --git a/packages/tv-cdp/src/runtime.ts b/packages/tv-cdp/src/runtime.ts index 84001544a4..09fc795e0a 100644 --- a/packages/tv-cdp/src/runtime.ts +++ b/packages/tv-cdp/src/runtime.ts @@ -33,6 +33,7 @@ import { createUnavailablePlatformRuntimeFacts } from '@agent-device/contracts/p import { bindProviderScreenshotInteractor, screenshotRuntimeOperationFacts, + defaultCaptureOnly, } from '@agent-device/contracts/screenshot-runtime'; import { bindProviderScrollInteractor, @@ -355,7 +356,12 @@ function createTvPlatformRuntime( resolveInteractor, presentIosAcquisition: host.snapshot.presentIosAcquisition, }), - ...bindProviderScreenshotInteractor({ device, signal, resolveInteractor }), + ...bindProviderScreenshotInteractor({ + device, + signal, + resolveInteractor, + captureCapabilities: defaultCaptureOnly('a TV target captures through CDP'), + }), ...bindProviderTouchInteractor({ device, signal, diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 2cb9b2f827..470b501425 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -378,9 +378,15 @@ function summarizeProviderScenarioFlagExclusions() { keys: ['waitMs'], }, { + // A provider scenario has no XCTest runner capture to select, so these stay on the owner's tests. name: 'Apple simulator screenshot rendering options', owner: 'iOS platform and screenshot-diff runtime tests', - keys: ['screenshotNormalizeStatusBar', 'screenshotPixelDensity', 'screenshotScreenMask'], + keys: [ + 'screenshotNormalizeStatusBar', + 'screenshotPixelDensity', + 'screenshotScreenMask', + 'screenshotCaptureBackend', + ], }, { // Reading accessibility custom actions has no provider-scenario surface: diff --git a/src/cli/parser/__tests__/args-parse-interaction.test.ts b/src/cli/parser/__tests__/args-parse-interaction.test.ts index 0aa7872877..ca383234d6 100644 --- a/src/cli/parser/__tests__/args-parse-interaction.test.ts +++ b/src/cli/parser/__tests__/args-parse-interaction.test.ts @@ -265,6 +265,8 @@ test('parseArgs recognizes screenshot flags', () => { '0.3', '--no-stabilize', '--normalize-status-bar', + '--capture-backend', + 'runner', ], { strictFlags: true, @@ -275,6 +277,7 @@ test('parseArgs recognizes screenshot flags', () => { assert.equal(parsed.flags.screenshotPixelDensity, 2); assert.equal(parsed.flags.screenshotScreenMask, 'black'); assert.equal(parsed.flags.screenshotFullscreen, true); + assert.equal(parsed.flags.screenshotCaptureBackend, 'runner'); assert.equal(parsed.flags.screenshotScale, 0.3); assert.equal(parsed.flags.screenshotNoStabilize, true); assert.equal(parsed.flags.screenshotNormalizeStatusBar, true); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 06bbb0c339..3590e3b83f 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1026,6 +1026,7 @@ agent-device screenshot page.png --overlay-refs # Draw current @eN refs and tar agent-device screenshot page.png --crop-on 'label="Save"' # Crop the capture to the frame the selector resolves on the same screen agent-device screenshot baseline.png --normalize-status-bar # Normalize iOS simulator or ADB-backed Android chrome for reusable diff baselines agent-device screenshot baseline.png --screen-mask black --normalize-status-bar # Capture an iOS simulator baseline with a black screen outline +agent-device screenshot page.png --capture-backend runner # Apple: capture through the XCTest runner (no simulator-drawn Dynamic Island) agent-device screenshot page.png --platform web --fullscreen # On web, --fullscreen/--full/-f captures the entire document agent-device viewport 1280 900 --platform web # Resize the active web viewport for fixed-layout or 100vh apps agent-device screenshot textedit.png # App-session window capture on macOS @@ -1050,6 +1051,7 @@ agent-device record stop # Stop active recording - `--overlay-refs` takes a fresh snapshot and burns visible `@eN` refs and their rectangles into the PNG. With `--scale`, refs are drawn at the final size, so avoid scales where labels become unreadable. In `--json` mode each ref also has a screenshot-space `center` for `press ` fallback. - `--crop-on ` crops the capture to the frame the selector resolves on the same screen. It works only on iOS simulators, Android emulators, and physical iPhones. A physical iPhone crop is verified only in portrait using CoreDevice/devicectl; the XCTest backend and runner fallback are not accepted. Physical iPads remain rejected for crop, and landscape iPhone crop remains unverified. It needs exactly one framed node, cannot be combined with `--overlay-refs` or `--fullscreen`, and warns when the frame is clipped. The crop is re-encoded, so do not byte-compare it with an older crop. - `--fullscreen` on macOS applies only to app sessions. Other surfaces (`desktop`, `menubar`, `frontmost-app`) always capture the main display and refuse `--fullscreen` with `INVALID_ARGS`. +- `--capture-backend runner` captures an Apple target through the XCTest runner (`XCUIScreen`), as XCTest-based tools such as Maestro do, instead of `simctl io screenshot`. On iOS simulators with a Dynamic Island, `simctl` draws the island into every capture and no `--mask` policy removes it; the runner capture leaves it out. The capture skips the simctl pipeline, so `--pixel-density` and `--normalize-status-bar` are refused with it (`INVALID_ARGS`). Each capture owner declares the backends it serves: other platforms, provider-backed sessions, and macOS surfaces the helper captures (`desktop`, `menubar`, `frontmost-app`, and app sessions with the native app backend) refuse it with `UNSUPPORTED_OPERATION`. Node and MCP take `captureBackend: 'runner'`. - `--normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for reusable baselines. Each status-bar read or preparation command has a 10-second timeout. Each restore command has a 30-second timeout, so clearing and reapplying overrides can take up to 60 seconds. The request timeout includes the wait and can expire before these steps finish. - `--screen-mask black` fills the iOS simulator screen outline, including rounded corners, with black for this capture. Node and MCP use `screenMask: "black"`. Omit it for the default image. Support is limited to handheld iOS/iPadOS simulators. It requires simctl capture; physical devices, tvOS and visionOS simulators, other platforms, provider captures, and XCTest runner captures are refused. If simctl fails, a masked request fails instead of falling back to an unmasked runner capture. This option does not redact app content, remove a Dynamic Island drawn into the framebuffer, or change comparison thresholds. - On an iOS simulator, a screenshot waits until no other screenshot or `settings wifi` / `settings airplane` write is using that simulator's status bar, so overlapping captures cannot see each other's temporary chrome. A request you cancel while it is waiting fails with the cancellation instead of capturing.