diff --git a/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.test.ts b/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.test.ts index 3a18ed1631..346df406c9 100644 --- a/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.test.ts @@ -143,7 +143,7 @@ test('runtime diff screenshot enforces the normalized comparison pixel budget', } }); -test('runtime diff screenshot attaches overlay refs to live mismatch regions', async () => { +test('runtime diff screenshot keeps the screen mask on current and overlay captures', async () => { const dir = mkdtempForTestSync('runtime-diff-screenshot-'); const baseline = path.join(dir, 'baseline.png'); const diffOut = path.join(dir, 'diff.png'); @@ -153,6 +153,7 @@ test('runtime diff screenshot attaches overlay refs to live mismatch regions', a try { const device = createAgentDevice({ backend: createScreenshotBackend((outPath, options) => { + assert.equal(options?.screenMask, 'black'); fs.writeFileSync(outPath, solidPngBuffer(20, 20, { r: 255, g: 255, b: 255 })); return { path: outPath, @@ -182,6 +183,7 @@ test('runtime diff screenshot attaches overlay refs to live mismatch regions', a out: { kind: 'path', path: diffOut }, threshold: 0, overlayRefs: true, + screenMask: 'black', }); assert.equal(result.currentOverlayPath, overlayOut); @@ -230,3 +232,22 @@ function solidPngBuffer( } return PNG.sync.write(png); } + +test('runtime diff refuses a screen mask for saved images before reading or capturing', async () => { + const device = createAgentDevice({ + backend: createScreenshotBackend(() => { + throw new Error('must not capture'); + }), + artifacts: createLocalArtifactAdapter(), + sessions, + policy: localCommandPolicy(), + }); + await assert.rejects( + device.capture.diffScreenshot({ + baseline: { kind: 'path', path: '/missing/baseline.png' }, + current: { kind: 'path', path: '/missing/current.png' }, + screenMask: 'black', + }), + { code: 'INVALID_ARGS', message: /requires a live screenshot/ }, + ); +}); diff --git a/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.ts b/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.ts index 656acd361f..3103458e9a 100644 --- a/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.ts @@ -12,6 +12,10 @@ import type { ResolvedInputFile, } from '@agent-device/contracts/artifact-adapter'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; +import { + readScreenshotScreenMask, + type ScreenshotScreenMask, +} from '@agent-device/contracts/capture'; import { AppError } from '@agent-device/kernel/errors'; import { compareScreenshots, @@ -37,6 +41,7 @@ export type DiffScreenshotCommandOptions = CommandContext & { threshold?: number; overlayRefs?: boolean; normalizeStatusBar?: boolean; + screenMask?: ScreenshotScreenMask; surface?: BackendScreenshotOptions['surface']; }; @@ -56,12 +61,7 @@ export const diffScreenshotCommand: RuntimeCommand< const threshold = normalizeThreshold(options.threshold); const currentRef = options.current ?? { kind: 'live' }; - if (options.overlayRefs && !isLiveCurrentRef(currentRef)) { - throw new AppError( - 'INVALID_ARGS', - 'diff screenshot cannot use --overlay-refs because saved-image comparisons have no live accessibility refs', - ); - } + validateLiveCaptureOptions(options, currentRef); const baseline = await resolveCommandInput(runtime, options.baseline, { usage: 'diff screenshot baseline', @@ -121,6 +121,26 @@ export const diffScreenshotCommand: RuntimeCommand< } }; +function validateLiveCaptureOptions( + options: DiffScreenshotCommandOptions, + currentRef: FileInputRef | LiveScreenshotInputRef, +): void { + readScreenshotScreenMask(options.screenMask); + if (isLiveCurrentRef(currentRef)) return; + if (options.overlayRefs) { + throw new AppError( + 'INVALID_ARGS', + 'diff screenshot cannot use --overlay-refs because saved-image comparisons have no live accessibility refs', + ); + } + if (options.screenMask !== undefined) { + throw new AppError( + 'INVALID_ARGS', + '--screen-mask requires a live screenshot, not a saved current image', + ); + } +} + function normalizeThreshold(threshold: unknown): number { if (threshold == null || threshold === '') return DEFAULT_SCREENSHOT_DIFF_THRESHOLD; const value = Number(threshold); @@ -222,10 +242,11 @@ async function captureScreenshot( } function liveDiffScreenshotOptions( - options: Pick, + options: Pick, ): BackendScreenshotOptions { return { normalizeStatusBar: options.normalizeStatusBar ?? true, + ...(options.screenMask !== undefined ? { screenMask: options.screenMask } : {}), ...(options.surface ? { surface: options.surface } : {}), }; } diff --git a/packages/command-runtime/src/commands/capture/runtime/screenshot.ts b/packages/command-runtime/src/commands/capture/runtime/screenshot.ts index 33a2f03cff..f75e49ae51 100644 --- a/packages/command-runtime/src/commands/capture/runtime/screenshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/screenshot.ts @@ -1,5 +1,5 @@ import { AppError } from '@agent-device/kernel/errors'; -import { validateScreenshotScale } from '@agent-device/contracts/capture'; +import { validateScreenshotScale, readScreenshotScreenMask } from '@agent-device/contracts/capture'; import { successText } from '@agent-device/kernel/success-text'; import { resizePngFileToScale } from '@agent-device/capture-kit/png-resize'; import type { DeviceRotation } from '@agent-device/contracts/device'; @@ -46,6 +46,7 @@ export const screenshotCommand: RuntimeCommand< fullscreen: options.fullscreen, overlayRefs: options.overlayRefs, pixelDensity: options.pixelDensity, + screenMask: readScreenshotScreenMask(options.screenMask), stabilize: options.stabilize, normalizeStatusBar: options.normalizeStatusBar, surface: options.surface, diff --git a/packages/command-runtime/src/commands/runtime-types.ts b/packages/command-runtime/src/commands/runtime-types.ts index 5268cb5a4e..a3e3009dc8 100644 --- a/packages/command-runtime/src/commands/runtime-types.ts +++ b/packages/command-runtime/src/commands/runtime-types.ts @@ -1,3 +1,4 @@ +import type { ScreenshotScreenMask } from '@agent-device/contracts/capture'; import type { FileOutputRef } from '@agent-device/contracts/artifact-adapter'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; import type { SessionSurface } from '@agent-device/contracts/session'; @@ -50,6 +51,7 @@ export type ScreenshotCommandOptions = CommandContext & { fullscreen?: boolean; overlayRefs?: boolean; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; scale?: number; stabilize?: boolean; normalizeStatusBar?: boolean; diff --git a/packages/command-surface/src/capture/diff.ts b/packages/command-surface/src/capture/diff.ts index 0c81e593fb..a2cc3134a7 100644 --- a/packages/command-surface/src/capture/diff.ts +++ b/packages/command-surface/src/capture/diff.ts @@ -76,10 +76,18 @@ const diffCommandMetadata = defineFieldCommandMetadata(DIFF_COMMAND_NAME, diffCo const diffCliSchema = { usageOverride: - 'diff snapshot | diff screenshot --baseline [current.png] [--out ] [--threshold <0-1>] [--overlay-refs]', + 'diff snapshot | diff screenshot --baseline [current.png] [--out ] [--threshold <0-1>] [--overlay-refs] [--screen-mask ]', usageFlags: [], positionalArgs: ['kind', 'current?'], - allowedFlags: [...SNAPSHOT_FLAGS, 'baseline', 'threshold', 'out', 'overlayRefs'], + flagsByAction: { snapshot: [], screenshot: ['screenshotScreenMask'] }, + allowedFlags: [ + ...SNAPSHOT_FLAGS, + 'baseline', + 'threshold', + 'out', + 'overlayRefs', + 'screenshotScreenMask', + ], } as const; export const diffCliReader: CliReader = (positionals, flags) => { @@ -106,7 +114,7 @@ export const diffCommandFacet = defineCommandFacet({ text: { summary: 'Diff snapshot or screenshot', cliDetail: - 'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; the current image scales proportionally to the baseline width and aligns top-left. Scaled height rounds to the nearest pixel (minimum 1). Extra or missing bottom rows always count as differences, even at threshold 1. The originals remain unchanged. When dimensions differ, normalization metadata reports original, scaled and compared sizes; region coordinates use comparison pixels. Both screenshot inputs are decoded from their bytes, so a baseline or current image may be PNG or JPEG, and the diff image is always PNG. JPEG is lossy, so keep --threshold above 0 whenever either input is JPEG. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines.', + 'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; the current image scales proportionally to the baseline width and aligns top-left. Scaled height rounds to the nearest pixel (minimum 1). Extra or missing bottom rows always count as differences, even at threshold 1. The originals remain unchanged. When dimensions differ, normalization metadata reports original, scaled and compared sizes; region coordinates use comparison pixels. Both screenshot inputs are decoded from their bytes, so a baseline or current image may be PNG or JPEG, and the diff image is always PNG. JPEG is lossy, so keep --threshold above 0 whenever either input is JPEG. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines. Use --screen-mask black for a live iOS simulator capture with a black screen outline, and use the same option when creating its baseline. --screen-mask is refused when current.png is supplied; saved images are compared unchanged.', }, metadata: diffCommandMetadata, run: (client, input) => client.capture.diff(input), diff --git a/packages/command-surface/src/capture/index.test.ts b/packages/command-surface/src/capture/index.test.ts index 11f67a91b3..28a6aeac72 100644 --- a/packages/command-surface/src/capture/index.test.ts +++ b/packages/command-surface/src/capture/index.test.ts @@ -2,7 +2,11 @@ import { describe, expect, test } from 'vitest'; import type { CliFlags } from '@agent-device/contracts/command'; import { alertCliReader, alertDaemonWriter } from './alert.ts'; import { diffCliReader } from './diff.ts'; -import { screenshotCliReader, screenshotDaemonWriter } from './screenshot.ts'; +import { + screenshotCliReader, + screenshotDaemonWriter, + screenshotCommandFacet, +} from './screenshot.ts'; import { snapshotCliReader } from './snapshot.ts'; import { waitCliReader, waitDaemonWriter } from './wait.ts'; @@ -48,11 +52,17 @@ describe('capture command interface', () => { test('reads screenshot path and writes screenshot flags', () => { const input = screenshotCliReader( ['page.png'], - flags({ screenshotPixelDensity: 2, screenshotFullscreen: true, screenshotScale: 0.3 }), + flags({ + screenshotScreenMask: 'black', + screenshotPixelDensity: 2, + screenshotFullscreen: true, + screenshotScale: 0.3, + }), ); expect(input).toMatchObject({ path: 'page.png', pixelDensity: 2, + screenMask: 'black', fullscreen: true, scale: 0.3, }); @@ -61,6 +71,7 @@ describe('capture command interface', () => { positionals: ['page.png'], options: { screenshotPixelDensity: 2, + screenshotScreenMask: 'black', screenshotFullscreen: true, screenshotScale: 0.3, }, @@ -173,3 +184,17 @@ describe('capture command interface', () => { }); }); }); + +test('refuses invalid screen-mask inputs from structured clients', () => { + expectInvalidArgs( + () => screenshotDaemonWriter({ screenMask: 'alpha' }), + '--screen-mask must be black', + ); +}); + +test('screenshot exposes the screen mask enum in the shared Node and MCP input schema', () => { + expect(screenshotCommandFacet.metadata.inputSchema.properties?.screenMask).toMatchObject({ + type: 'string', + enum: ['black'], + }); +}); diff --git a/packages/command-surface/src/capture/screenshot-options.test.ts b/packages/command-surface/src/capture/screenshot-options.test.ts index 254e65ebcc..4140eea6a3 100644 --- a/packages/command-surface/src/capture/screenshot-options.test.ts +++ b/packages/command-surface/src/capture/screenshot-options.test.ts @@ -20,6 +20,7 @@ test('screenshot flag projection maps CLI flags to runtime options', () => { overlayRefs: true, screenshotCropOn: 'role=button label=Save', screenshotPixelDensity: 2, + screenshotScreenMask: 'black', screenshotFullscreen: true, screenshotScale: 0.3, screenshotNoStabilize: true, @@ -29,6 +30,7 @@ test('screenshot flag projection maps CLI flags to runtime options', () => { overlayRefs: true, cropOn: 'role=button label=Save', pixelDensity: 2, + screenMask: 'black', fullscreen: true, scale: 0.3, stabilize: false, @@ -43,6 +45,7 @@ test('screenshot flag projection maps public options to request flags', () => { overlayRefs: true, cropOn: 'label="Network & internet"', pixelDensity: 3, + screenMask: 'black', fullscreen: true, stabilize: false, normalizeStatusBar: true, @@ -51,6 +54,7 @@ test('screenshot flag projection maps public options to request flags', () => { overlayRefs: true, screenshotCropOn: 'label="Network & internet"', screenshotPixelDensity: 3, + screenshotScreenMask: 'black', screenshotFullscreen: true, screenshotNoStabilize: true, screenshotNormalizeStatusBar: true, @@ -117,11 +121,23 @@ test('screenshot script flags use the shared recorded flag contract', () => { message: /requires a selector expression/, }); + result = readScreenshotScriptFlag({ args: ['--screen-mask', 'black'], index: 0, flags }); + assert.deepEqual(result, { handled: true, nextIndex: 1 }); + assert.equal(flags.screenshotScreenMask, 'black'); + assert.throws( + () => readScreenshotScriptFlag({ args: ['--screen-mask', 'alpha'], index: 0, flags: {} }), + { code: 'INVALID_ARGS' }, + ); + assert.throws(() => readScreenshotScriptFlag({ args: ['--screen-mask'], index: 0, flags: {} }), { + code: 'INVALID_ARGS', + }); appendScreenshotScriptFlags(parts, flags); assert.deepEqual(parts, [ '--pixel-density', '3', + '--screen-mask', + 'black', '--fullscreen', '--scale', '0.3', @@ -131,6 +147,7 @@ test('screenshot script flags use the shared recorded flag contract', () => { assert.deepEqual(SCREENSHOT_ACTION_FLAG_KEYS, [ 'screenshotCropOn', 'screenshotPixelDensity', + 'screenshotScreenMask', 'screenshotFullscreen', 'screenshotScale', 'screenshotNoStabilize', @@ -145,6 +162,7 @@ test('screenshot script flags use the shared recorded flag contract', () => { 'overlayRefs', 'screenshotCropOn', 'screenshotPixelDensity', + 'screenshotScreenMask', 'screenshotFullscreen', 'screenshotScale', 'screenshotNoStabilize', diff --git a/packages/command-surface/src/capture/screenshot.ts b/packages/command-surface/src/capture/screenshot.ts index 2fd871e96d..df08b13bc4 100644 --- a/packages/command-surface/src/capture/screenshot.ts +++ b/packages/command-surface/src/capture/screenshot.ts @@ -5,6 +5,7 @@ import { RETIRED_SCREENSHOT_MAX_SIZE, SCREENSHOT_COMMAND_FLAG_KEYS, SCREENSHOT_SCALE_LIMITS, + SCREENSHOT_SCREEN_MASKS, screenshotFlagsFromPublicOptions, screenshotOptionsFromFlags, validateNoRetiredScreenshotMaxSize, @@ -40,6 +41,10 @@ const screenshotCommandMetadata = defineFieldCommandMetadata( pixelDensity: integerField('Output screenshot pixel density in pixels per logical point.', { min: 1, }), + screenMask: enumField( + SCREENSHOT_SCREEN_MASKS, + 'Fill the iOS simulator screen outline with black for this capture. Requires simctl; omit for the default image.', + ), fullscreen: booleanField(), scale: numberField('Screenshot scale factor.', SCREENSHOT_SCALE_LIMITS), maxSize: retiredField(RETIRED_SCREENSHOT_MAX_SIZE.migration.screenshot), @@ -74,7 +79,7 @@ export const screenshotCommandFacet = defineCommandFacet({ text: { summary: 'Capture a screenshot', cliDetail: - "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.", + "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. Use --screen-mask black to fill the iOS simulator screen outline, including rounded corners, with black for this capture. It requires simctl: physical devices, tvOS and visionOS simulators, other platforms, provider captures and XCTest runner captures are refused. A masked capture fails if simctl fails instead of falling back to an unmasked runner capture. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.", }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input), diff --git a/packages/contracts/src/backend.ts b/packages/contracts/src/backend.ts index ae835bcc9b..b0b5f8d7a3 100644 --- a/packages/contracts/src/backend.ts +++ b/packages/contracts/src/backend.ts @@ -1,3 +1,4 @@ +import type { ScreenshotScreenMask } from './screenshot.ts'; import type { ScreenshotResultData } from './snapshot-types.ts'; import type { SnapshotCaptureAnnotations } from './snapshot-capture-annotations.ts'; import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts'; @@ -71,6 +72,7 @@ export type BackendScreenshotOptions = { fullscreen?: boolean; overlayRefs?: boolean; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; stabilize?: boolean; normalizeStatusBar?: boolean; surface?: SessionSurface; diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index 6024cbb0b0..0df298488c 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -1,3 +1,4 @@ +import type { ScreenshotScreenMask } from './screenshot.ts'; // The public API vocabulary for snapshot, screenshot and diff capture. import type { SessionSurface } from './session-surface.ts'; @@ -112,6 +113,7 @@ export type CaptureScreenshotOptions = AgentDeviceRequestOverrides & { /** Crop the capture to the frame of the selector resolved on the same screen. */ cropOn?: string; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; fullscreen?: boolean; scale?: number; stabilize?: boolean; diff --git a/packages/contracts/src/facades/capture.ts b/packages/contracts/src/facades/capture.ts index fa1318d02a..ca05162baf 100644 --- a/packages/contracts/src/facades/capture.ts +++ b/packages/contracts/src/facades/capture.ts @@ -6,6 +6,8 @@ export { SCREENSHOT_CROP_REASONS, SCREENSHOT_FULLSCREEN_REASONS, SCREENSHOT_SCALE_LIMITS, + SCREENSHOT_SCREEN_MASKS, + readScreenshotScreenMask, SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS, appendScreenshotScriptFlags, readScreenshotScriptFlag, @@ -18,6 +20,7 @@ export { } from '../screenshot.ts'; export type { ScreenshotCropReason, + ScreenshotScreenMask, ScreenshotDispatchFlags, ScreenshotFullscreenReason, ScreenshotPublicOptions, diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index 0de826f22b..dce79ef80b 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -1,3 +1,4 @@ +import type { ScreenshotScreenMask } from './screenshot.ts'; import type { AppStateRuntimeResult } from './app-state-runtime.ts'; import type { ElementTextTarget } from './element-text-target.ts'; import type { BackMode } from './back-mode.ts'; @@ -61,6 +62,7 @@ export type { BackMode }; export type ScreenshotOptions = { appBundleId?: string; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; fullscreen?: boolean; normalizeStatusBar?: boolean; stabilize?: boolean; diff --git a/packages/contracts/src/screenshot-runtime.test.ts b/packages/contracts/src/screenshot-runtime.test.ts index a4e884ae4a..89a6b2f857 100644 --- a/packages/contracts/src/screenshot-runtime.test.ts +++ b/packages/contracts/src/screenshot-runtime.test.ts @@ -54,3 +54,41 @@ test('a provider binding fails closed when its exact owner exposes no interactor details: { reason: 'provider-runtime-interactor-missing' }, }); }); + +test.each([bindLocalScreenshotInteractor, bindProviderScreenshotInteractor])( + 'a capture without a declared mask capability refuses before resolving its interactor', + async (bind) => { + const resolveInteractor = vi.fn(); + const operations = bind({ + device: { ...device, platform: 'apple', appleOs: 'ios', kind: 'simulator' }, + signal: new AbortController().signal, + resolveInteractor, + }); + await expect( + operations.captureScreenshot({ + outPath: '/tmp/masked.png', + options: { screenMask: 'black' }, + }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-screen-mask-unsupported' }, + }); + expect(resolveInteractor).not.toHaveBeenCalled(); + }, +); + +test('the selected capture capability admits a mask for one invocation without changing the next', async () => { + const screenshot = vi.fn(async () => ({ displayRotation: 'portrait' as const })); + const operations = bindLocalScreenshotInteractor({ + device, + signal: new AbortController().signal, + resolveInteractor: async () => ({ screenshot }) as unknown as Interactor, + captureCapabilities: () => ({ screenMask: true }), + }); + for (const screenMask of ['black', undefined] as const) { + await expect( + operations.captureScreenshot({ outPath: '/tmp/capture.png', options: { screenMask } }), + ).resolves.toEqual({ displayRotation: 'portrait' }); + expect(screenshot.mock.lastCall).toEqual(['/tmp/capture.png', { screenMask }]); + } +}); diff --git a/packages/contracts/src/screenshot-runtime.ts b/packages/contracts/src/screenshot-runtime.ts index eae0fef67e..f63f0c65d6 100644 --- a/packages/contracts/src/screenshot-runtime.ts +++ b/packages/contracts/src/screenshot-runtime.ts @@ -1,3 +1,4 @@ +import { AppError } from '@agent-device/kernel/errors'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { localInteractorSource, @@ -33,6 +34,31 @@ export type ScreenshotRuntimeOperations = Readonly<{ captureScreenshot(input: CaptureScreenshotInput): Promise; }>; +export type ScreenshotCaptureCapabilities = Readonly<{ + screenMask?: boolean; +}>; + +type ScreenshotCaptureBindingOptions = Readonly<{ + signal: AbortSignal; + captureCapabilities?: (options: Readonly) => ScreenshotCaptureCapabilities; +}>; + +export async function assertScreenshotCaptureSupport( + options: Readonly | undefined, + capabilities: ScreenshotCaptureCapabilities = {}, +): Promise { + if (options?.screenMask === undefined) return; + const { readScreenshotScreenMask } = await import('./screenshot.ts'); + const screenMask = readScreenshotScreenMask(options?.screenMask); + if (screenMask !== undefined && capabilities.screenMask !== true) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + '--screen-mask requires a handheld iOS simulator simctl capture', + { reason: 'screenshot-screen-mask-unsupported' }, + ); + } +} + export type ScreenshotRuntimeOperationFacts = Readonly<{ captureScreenshot: RuntimeOperationFact; }>; @@ -50,15 +76,19 @@ export function screenshotRuntimeOperationFacts( * resolution and this holds only what both share: the runner context and the capture itself. */ function bindScreenshotCapture( - signal: AbortSignal, + binding: ScreenshotCaptureBindingOptions, resolveInteractor: (runner: RunnerContext) => Promise, ): ScreenshotRuntimeOperations { return Object.freeze({ captureScreenshot: async (input: CaptureScreenshotInput) => { + await assertScreenshotCaptureSupport( + input.options, + binding.captureCapabilities?.(input.options ?? {}), + ); const interactor = await resolveInteractor({ ...input.execution, appBundleId: input.options?.appBundleId, - signal, + signal: binding.signal, }); return (await interactor.screenshot(input.outPath, input.options)) ?? {}; }, @@ -66,25 +96,25 @@ function bindScreenshotCapture( } export function bindLocalScreenshotInteractor( - params: Readonly<{ - device: DeviceInfo; - signal: AbortSignal; - resolveInteractor: LocalInteractorOperationResolver; - }>, + params: ScreenshotCaptureBindingOptions & + Readonly<{ + device: DeviceInfo; + resolveInteractor: LocalInteractorOperationResolver; + }>, ): ScreenshotRuntimeOperations { - return bindScreenshotCapture(params.signal, localInteractorSource(params)); + return bindScreenshotCapture(params, localInteractorSource(params)); } /** Provider bindings fail closed when their exact owner no longer exposes its interactor. */ export function bindProviderScreenshotInteractor( - params: Readonly<{ - device: DeviceInfo; - signal: AbortSignal; - resolveInteractor: ProviderInteractorOperationResolver; - }>, + params: ScreenshotCaptureBindingOptions & + Readonly<{ + device: DeviceInfo; + resolveInteractor: ProviderInteractorOperationResolver; + }>, ): ScreenshotRuntimeOperations { return bindScreenshotCapture( - params.signal, + params, providerInteractorSource({ ...params, operation: 'screenshot' }), ); } diff --git a/packages/contracts/src/screenshot.ts b/packages/contracts/src/screenshot.ts index 48e16a4373..be62912d25 100644 --- a/packages/contracts/src/screenshot.ts +++ b/packages/contracts/src/screenshot.ts @@ -1,5 +1,13 @@ import { AppError } from '@agent-device/kernel/errors'; +export const SCREENSHOT_SCREEN_MASKS = ['black'] as const; +export type ScreenshotScreenMask = (typeof SCREENSHOT_SCREEN_MASKS)[number]; + +export function readScreenshotScreenMask(value: unknown): ScreenshotScreenMask | undefined { + if (value === undefined || value === 'black') return value; + throw new AppError('INVALID_ARGS', '--screen-mask must be black'); +} + export const SCREENSHOT_SCALE_LIMITS = { min: 0.01, max: 1 } as const; /** @@ -85,6 +93,7 @@ export const SCREENSHOT_COMMAND_FLAG_KEYS = [ 'overlayRefs', 'screenshotCropOn', 'screenshotPixelDensity', + 'screenshotScreenMask', 'screenshotFullscreen', 'screenshotScale', 'screenshotNoStabilize', @@ -94,6 +103,7 @@ export const SCREENSHOT_COMMAND_FLAG_KEYS = [ export const SCREENSHOT_ACTION_FLAG_KEYS = [ 'screenshotCropOn', 'screenshotPixelDensity', + 'screenshotScreenMask', 'screenshotFullscreen', 'screenshotScale', 'screenshotNoStabilize', @@ -105,7 +115,8 @@ type ScreenshotSpecificFlagKey = (typeof SCREENSHOT_ACTION_FLAG_KEYS)[number]; type ScreenshotSpecificFlagDefinition = { key: ScreenshotSpecificFlagKey; names: readonly string[]; - type: 'boolean' | 'int' | 'number' | 'string'; + type: 'boolean' | 'int' | 'number' | 'string' | 'enum'; + enumValues?: readonly string[]; min?: number; max?: number; usageLabel: string; @@ -136,6 +147,17 @@ export const SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS: readonly ScreenshotSpecificFl projectConfig: true, recorded: true, }, + { + key: 'screenshotScreenMask', + names: ['--screen-mask'], + type: 'enum', + enumValues: SCREENSHOT_SCREEN_MASKS, + usageLabel: '--screen-mask ', + usageDescription: + 'Screenshot or live screenshot diff: fill the iOS simulator screen outline with black; requires simctl capture', + projectConfig: false, + recorded: true, + }, { key: 'screenshotFullscreen', names: ['--fullscreen', '--full', '-f'], @@ -217,6 +239,7 @@ export type ScreenshotRequestFlags = { overlayRefs?: boolean; screenshotCropOn?: string; screenshotPixelDensity?: number; + screenshotScreenMask?: ScreenshotScreenMask; screenshotFullscreen?: boolean; screenshotScale?: number; screenshotNoStabilize?: boolean; @@ -226,6 +249,7 @@ export type ScreenshotRequestFlags = { export type ScreenshotDispatchFlags = Pick< ScreenshotRequestFlags, | 'screenshotPixelDensity' + | 'screenshotScreenMask' | 'screenshotFullscreen' | 'screenshotNoStabilize' | 'screenshotNormalizeStatusBar' @@ -235,6 +259,7 @@ export type ScreenshotRuntimeFlags = Pick< ScreenshotRequestFlags, | 'screenshotCropOn' | 'screenshotPixelDensity' + | 'screenshotScreenMask' | 'screenshotFullscreen' | 'screenshotScale' | 'screenshotNoStabilize' @@ -245,6 +270,7 @@ export type ScreenshotPublicOptions = { overlayRefs?: boolean; cropOn?: string; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; fullscreen?: boolean; scale?: number; stabilize?: boolean; @@ -255,6 +281,7 @@ export type ScreenshotRuntimeOptions = { overlayRefs?: boolean; cropOn?: string; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; fullscreen?: boolean; scale?: number; stabilize?: boolean; @@ -268,6 +295,7 @@ export function screenshotOptionsFromFlags( overlayRefs: flags?.overlayRefs, cropOn: flags?.screenshotCropOn, pixelDensity: flags?.screenshotPixelDensity, + screenMask: readScreenshotScreenMask(flags?.screenshotScreenMask), fullscreen: flags?.screenshotFullscreen, scale: flags?.screenshotScale, stabilize: @@ -283,6 +311,9 @@ export function screenshotFlagsFromOptions( overlayRefs: options.overlayRefs, screenshotCropOn: options.screenshotCropOn ?? options.cropOn, screenshotPixelDensity: options.screenshotPixelDensity ?? options.pixelDensity, + screenshotScreenMask: readScreenshotScreenMask( + options.screenshotScreenMask ?? options.screenMask, + ), screenshotFullscreen: options.screenshotFullscreen ?? options.fullscreen, screenshotScale: options.screenshotScale, screenshotNoStabilize: @@ -324,6 +355,9 @@ export function appendScreenshotScriptFlags( if (typeof flags?.screenshotPixelDensity === 'number') { parts.push('--pixel-density', String(flags.screenshotPixelDensity)); } + if (flags?.screenshotScreenMask !== undefined) { + parts.push('--screen-mask', readScreenshotScreenMask(flags.screenshotScreenMask)!); + } if (flags?.screenshotFullscreen) parts.push('--fullscreen'); if (typeof flags?.screenshotScale === 'number') { parts.push('--scale', String(flags.screenshotScale)); @@ -340,6 +374,12 @@ export function readScreenshotScriptFlag(params: { }): { handled: true; nextIndex: number } | { handled: false } { const { args, flags, index } = params; const token = args[index]; + if (token === '--screen-mask') { + const value = args[index + 1]; + if (value === undefined) throw new AppError('INVALID_ARGS', '--screen-mask requires black'); + flags.screenshotScreenMask = readScreenshotScreenMask(value); + return { handled: true, nextIndex: index + 1 }; + } if (token === RETIRED_SCREENSHOT_MAX_SIZE.cliToken) { throw new AppError('INVALID_ARGS', RETIRED_SCREENSHOT_MAX_SIZE.migration.screenshot); } diff --git a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts index 7c1d97abdb..2866b2bd12 100644 --- a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts +++ b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts @@ -479,3 +479,26 @@ for (const backend of ['coredevice', 'xctest', undefined] as const) { }); } } + +test('a request screen mask reaches the bound simulator capture without changing later requests', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('ios-mask', { device: IOS_SIMULATOR }); + const outPath = path.join(mkdtempForTestSync('screen-mask-request'), 'screen.png'); + for (const screenMask of ['black', undefined] as const) { + const req = screenshotRequest({ + positionals: [outPath], + flags: screenMask ? { screenshotScreenMask: screenMask } : {}, + }); + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + await executeResult(resolved, session, req, req.flags); + expect(fixture.captureScreenshot.mock.lastCall?.[0].options?.screenMask).toBe(screenMask); + } +}); diff --git a/packages/daemon/src/execution/screenshot-runtime.ts b/packages/daemon/src/execution/screenshot-runtime.ts index 1df252ff60..1b0196ab1c 100644 --- a/packages/daemon/src/execution/screenshot-runtime.ts +++ b/packages/daemon/src/execution/screenshot-runtime.ts @@ -362,6 +362,7 @@ function createBoundScreenshotBackend( options: { appBundleId: dispatchContext.appBundleId, pixelDensity: resolved.pixelDensity, + screenMask: resolved.screenMask, fullscreen: resolved.fullscreen, normalizeStatusBar: resolved.normalizeStatusBar, stabilize: resolved.stabilize, diff --git a/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts b/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts index 9f7d7c6939..57e6215928 100644 --- a/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts @@ -56,3 +56,35 @@ test('routes an internal runner screenshot without changing the default backend' process.env, ); }); + +test('forwards the request mask and refuses an explicit runner capture with it', async () => { + const interactor = createAppleInteractor(device, {}, { env: {}, cwd: process.cwd() }); + await interactor.screenshot('/tmp/masked.png', { screenMask: 'black' }); + expect(screenshotIos).toHaveBeenCalledWith( + device, + '/tmp/masked.png', + expect.objectContaining({ screenMask: 'black' }), + ); + await expect( + interactor.screenshot('/tmp/runner.png', { screenMask: 'black', captureBackend: 'runner' }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-screen-mask-unsupported' }, + }); + expect(captureScreenshotViaRunner).not.toHaveBeenCalled(); +}); + +test.each([ + { ...device, kind: 'device' as const }, + { ...device, appleOs: 'macos' as const, kind: 'device' as const, target: 'desktop' as const }, +])('direct Apple $appleOs $kind captures refuse masks before native execution', async (target) => { + const interactor = createAppleInteractor(target, {}, { env: {}, cwd: process.cwd() }); + await expect( + interactor.screenshot('/tmp/masked.png', { screenMask: 'black' }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-screen-mask-unsupported' }, + }); + expect(screenshotIos).not.toHaveBeenCalled(); + expect(captureScreenshotViaRunner).not.toHaveBeenCalled(); +}); diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index e243ad1390..b2b0c612d0 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -508,12 +508,18 @@ describe('captureSimulatorScreenshotWithRetry display targeting', () => { }); test('names the lit panel explicitly instead of accepting the black default', async () => { - await captureSimulatorScreenshotWithRetry(IOS_TEST_SIMULATOR, '/tmp/duo.png', outerPanel); + await captureSimulatorScreenshotWithRetry( + IOS_TEST_SIMULATOR, + '/tmp/duo.png', + 'black', + outerPanel, + ); assert.equal(mockRunSimctlForDevice.mock.calls.length, 1); assert.deepEqual(mockRunSimctlForDevice.mock.calls[0]![1], [ 'io', IOS_TEST_SIMULATOR.id, 'screenshot', + '--mask=black', '--display=LCD', '/tmp/duo.png', ]); diff --git a/packages/platform-apple/src/core/__tests__/screenshot-simulator-capture.test.ts b/packages/platform-apple/src/core/__tests__/screenshot-simulator-capture.test.ts index 2ad3791535..f6cafa44e3 100644 --- a/packages/platform-apple/src/core/__tests__/screenshot-simulator-capture.test.ts +++ b/packages/platform-apple/src/core/__tests__/screenshot-simulator-capture.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, test, vi } from 'vitest'; +import { afterEach, beforeEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; import { promises as fs, writeFileSync } from 'node:fs'; import path from 'node:path'; @@ -26,6 +26,10 @@ beforeEach(() => { mockRetryWithPolicy.mockImplementation(retryActual.retryWithPolicy); }); +afterEach(() => { + vi.unstubAllEnvs(); +}); + const BOOTED_SIM_LIST_JSON = JSON.stringify({ devices: { 'com.apple.CoreSimulator.SimRuntime.iOS-18-0': [{ udid: 'sim-1', state: 'Booted' }] }, }); @@ -96,13 +100,16 @@ test('screenshotIos retries simulator capture timeouts and eventually succeeds', exitCode: 60, }; } - writeFileSync(args[4] ?? '', sourcePng); + writeFileSync(args.at(-1) ?? '', sourcePng); return ''; } return unexpectedArgs(args); }, async ({ calls }) => { - await screenshotIos(IOS_TEST_SIMULATOR, outPath, { env: process.env }); + await screenshotIos(IOS_TEST_SIMULATOR, outPath, { + env: {}, + screenMask: 'black', + }); const png = PNG.sync.read(await fs.readFile(outPath)); assert.equal(png.width, 42); assert.equal(png.height, 91); @@ -110,7 +117,7 @@ test('screenshotIos retries simulator capture timeouts and eventually succeeds', const flat = calls.map((args) => args.join(' ')); assert.equal( - flat.filter((line) => line === `simctl io sim-1 screenshot ${outPath}`).length, + flat.filter((line) => line === `simctl io sim-1 screenshot --mask=black ${outPath}`).length, 3, 'should retry screenshot command until success', ); @@ -123,31 +130,49 @@ test('screenshotIos retries simulator capture timeouts and eventually succeeds', ); }); -test('screenshotIos keeps requested simulator pixel density', async () => { - const tmpDir = await mkdtempForTest('agent-device-ios-screenshot-density-'); - const outPath = path.join(tmpDir, 'screen.png'); - - // Both dimensions divisible by 3 so the 2/3 rescale (density 2 against the stub's native - // scale 3) stays exact. What this pins is that ratio, not an absolute resolution: it used - // to allocate and resize a 1206x2622 iPhone 16 Pro frame — 3.2 megapixels, ~90x more than - // the arithmetic needs. - const sourcePng = PNG.sync.write(new PNG({ width: 126, height: 273 })); - - await withFakeAppleTool( - (args) => { - if (isSimctlListDevices(args)) return BOOTED_SIM_LIST_JSON; - if (isSimctlMainScreenScale(args)) return '3\n'; - if (isSimctlScreenshot(args)) { - writeFileSync(args[4] ?? '', sourcePng); - return ''; - } - return unexpectedArgs(args); - }, - async () => { - await screenshotIos(IOS_TEST_SIMULATOR, outPath, { env: process.env, pixelDensity: 2 }); - const png = PNG.sync.read(await fs.readFile(outPath)); - assert.equal(png.width, 84); - assert.equal(png.height, 182); - }, - ); -}); +test.each([undefined, 'black'] as const)( + 'screenshotIos keeps density and applies only the request screen mask %s', + async (mask) => { + vi.stubEnv('AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK', 'black'); + const tmpDir = await mkdtempForTest('agent-device-ios-screenshot-density-'); + const outPath = path.join(tmpDir, 'screen.png'); + + // Both dimensions divisible by 3 so the 2/3 rescale (density 2 against the stub's native + // scale 3) stays exact. What this pins is that ratio, not an absolute resolution: it used + // to allocate and resize a 1206x2622 iPhone 16 Pro frame — 3.2 megapixels, ~90x more than + // the arithmetic needs. + const sourcePng = PNG.sync.write(new PNG({ width: 126, height: 273 })); + + await withFakeAppleTool( + (args) => { + if (isSimctlListDevices(args)) return BOOTED_SIM_LIST_JSON; + if (isSimctlMainScreenScale(args)) return '3\n'; + if (isSimctlScreenshot(args)) { + writeFileSync(args.at(-1) ?? '', sourcePng); + return ''; + } + return unexpectedArgs(args); + }, + async ({ calls }) => { + await screenshotIos(IOS_TEST_SIMULATOR, outPath, { + env: process.env, + screenMask: mask, + pixelDensity: 2, + }); + assert.deepEqual(calls.filter(isSimctlScreenshot), [ + [ + 'simctl', + 'io', + 'sim-1', + 'screenshot', + ...(mask === 'black' ? ['--mask=black'] : []), + outPath, + ], + ]); + const png = PNG.sync.read(await fs.readFile(outPath)); + assert.equal(png.width, 84); + assert.equal(png.height, 182); + }, + ); + }, +); diff --git a/packages/platform-apple/src/core/__tests__/screenshot.test.ts b/packages/platform-apple/src/core/__tests__/screenshot.test.ts index bc559ef93c..398eff074d 100644 --- a/packages/platform-apple/src/core/__tests__/screenshot.test.ts +++ b/packages/platform-apple/src/core/__tests__/screenshot.test.ts @@ -49,6 +49,7 @@ import { captureSimulatorScreenshotWithFallback, captureSimulatorScreenshotWithRetry, captureScreenshotViaRunner, + screenshotIos, resolveSimulatorRunnerScreenshotCandidatePaths, shouldRetryIosSimulatorScreenshot, } from '../screenshot.ts'; @@ -857,3 +858,42 @@ test('captureSimulatorScreenshotWithFallback keeps the panel rotation for a runn }); assert.deepEqual(facts, { displayRotation: 'landscape-right' }); }); + +test('an explicit screen mask refuses an unmasked runner fallback after simctl exhaustion', async () => { + const failure = new AppError('COMMAND_FAILED', 'capture failed', { + stderr: 'Timeout waiting for screen surfaces', + }); + mockRetryWithPolicy.mockRejectedValue(failure); + await assert.rejects( + captureSimulatorScreenshotWithFallback(IOS_TEST_SIMULATOR, '/tmp/masked.png', { + env: {}, + screenMask: 'black', + skipIosSimulatorBootCheck: true, + deps: { + resolveCaptureTarget: async () => ({}), + captureWithRetry: captureSimulatorScreenshotWithRetry, + captureWithRunner: captureScreenshotViaRunner, + normalizeDensity: async () => {}, + }, + }), + (error) => error === failure, + ); + assert.equal(mockRunAppleRunnerCommand.mock.calls.length, 0); +}); + +test.each([MACOS_TEST_DEVICE, { ...IOS_TEST_SIMULATOR, kind: 'device' as const }])( + 'screenshotIos refuses a masked $kind capture outside simctl before native execution', + async (device) => { + mockRunCmd.mockRejectedValue(new Error('unexpected native capture')); + mockRunAppleRunnerCommand.mockRejectedValue(new Error('unexpected runner capture')); + await assert.rejects( + screenshotIos(device, '/tmp/masked.png', { env: {}, screenMask: 'black' }), + { + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-screen-mask-unsupported' }, + }, + ); + assert.equal(mockRunCmd.mock.calls.length, 0); + assert.equal(mockRunAppleRunnerCommand.mock.calls.length, 0); + }, +); diff --git a/packages/platform-apple/src/core/screenshot.ts b/packages/platform-apple/src/core/screenshot.ts index a564ee0ecc..40369a0702 100644 --- a/packages/platform-apple/src/core/screenshot.ts +++ b/packages/platform-apple/src/core/screenshot.ts @@ -1,5 +1,8 @@ import path from 'node:path'; import { isMacOs, type DeviceInfo } from '@agent-device/kernel/device'; +import type { ScreenshotScreenMask } from '@agent-device/contracts/capture'; +import { assertScreenshotCaptureSupport } from '@agent-device/contracts/screenshot-runtime'; +import { appleScreenshotCaptureCapabilities } from '../screenshot-support.ts'; import type { EnvMap } from '@agent-device/kernel/source-value'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { copyHostFile } from '@agent-device/host-kit/host-file'; @@ -66,6 +69,7 @@ type SimulatorScreenshotFlowDeps = { captureWithRetry: ( device: DeviceInfo, outPath: string, + screenMask: ScreenshotScreenMask | undefined, display: AppleDeviceDisplay | undefined, ) => Promise; /** @@ -98,6 +102,7 @@ type SimulatorScreenshotFlowOptions = { appBundleId?: string; fullscreen?: boolean; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; normalizeStatusBar?: boolean; runnerOptions?: AppleRunnerCommandOptions; skipIosSimulatorBootCheck?: boolean; @@ -123,6 +128,7 @@ export async function screenshotIos( outPath: string, options: Omit, ): Promise { + await assertScreenshotCaptureSupport(options, appleScreenshotCaptureCapabilities(device, {})); if (isMacOs(device)) { await captureScreenshotViaRunner( device, @@ -152,6 +158,7 @@ export async function captureSimulatorScreenshotWithFallback( outPath: string, options: SimulatorScreenshotFlowOptions, ): Promise { + await assertScreenshotCaptureSupport(options, appleScreenshotCaptureCapabilities(device, {})); if (device.kind !== 'simulator') { throw new AppError( 'UNSUPPORTED_OPERATION', @@ -169,7 +176,7 @@ export async function captureSimulatorScreenshotWithFallback( const { display, displayRotation } = await deps.resolveCaptureTarget(device); const captureFacts: ScreenshotCaptureFacts = displayRotation ? { displayRotation } : {}; const captureAndNormalize = async () => { - await deps.captureWithRetry(device, outPath, display); + await deps.captureWithRetry(device, outPath, options.screenMask, display); await deps.normalizeDensity(device, outPath, options.pixelDensity, display?.pointScale); }; const captureWithFallback = async (): Promise => { @@ -190,7 +197,7 @@ export async function captureSimulatorScreenshotWithFallback( screenshotError = retryError; } } - if (!deps.shouldFallbackToRunner(screenshotError)) { + if (options.screenMask !== undefined || !deps.shouldFallbackToRunner(screenshotError)) { throw screenshotError; } emitScreenshotFallbackDiagnostic(device, 'simctl_screenshot', screenshotError); @@ -219,6 +226,7 @@ export async function captureSimulatorScreenshotWithFallback( export async function captureSimulatorScreenshotWithRetry( device: DeviceInfo, outPath: string, + screenMask: ScreenshotScreenMask | undefined, display?: AppleDeviceDisplay, ): Promise { const deadline = Deadline.fromTimeoutMs(IOS_SIMULATOR_SCREENSHOT_TIMEOUT_MS); @@ -226,6 +234,7 @@ export async function captureSimulatorScreenshotWithRetry( 'io', device.id, 'screenshot', + ...(screenMask === 'black' ? ['--mask=black'] : []), ...appleSimulatorDisplayArgvFragment(display), outPath, ]; diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index 87b3e40013..530b148b22 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -1,3 +1,5 @@ +import { assertScreenshotCaptureSupport } from '@agent-device/contracts/screenshot-runtime'; +import { appleScreenshotCaptureCapabilities } from './screenshot-support.ts'; import { closeIosApp, openIosApp, openIosDevice } from './core/app-launch.ts'; import { readIosClipboardText, writeIosClipboardText } from './core/app-device-io.ts'; import { readIosSetting, setIosSetting } from './core/app-settings.ts'; @@ -493,6 +495,10 @@ async function runAppleScreenshot( helper: MacOsHelperSurface | undefined, env: Readonly, ): Promise { + await assertScreenshotCaptureSupport( + options, + appleScreenshotCaptureCapabilities(device, options), + ); if (helper) { if (options.fullscreen) { throw new AppError( @@ -530,6 +536,7 @@ async function runAppleScreenshot( env, appBundleId: options.appBundleId, pixelDensity: options.pixelDensity, + screenMask: options.screenMask, fullscreen: options.fullscreen, runnerOptions: runnerOpts, normalizeStatusBar: options.normalizeStatusBar, diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index 7b3f8a67d5..c2caf245a9 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -8,6 +8,7 @@ import { import type { NetworkDumpInput } from '@agent-device/contracts/network-runtime'; import type { AppStateRuntimeOperations } from '@agent-device/contracts/app-state-runtime'; import { bindAppleAppStateRuntime } from './app-state-runtime.ts'; +import { appleScreenshotCaptureCapabilities } from './screenshot-support.ts'; import type { PlatformRuntimeHost, PlatformRuntimeOperations, @@ -409,6 +410,8 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR device: request.device, signal: request.scope.signal, resolveInteractor: host.localInteractors.resolve, + captureCapabilities: (options) => + appleScreenshotCaptureCapabilities(request.device, options), }), ), ...whenAdmitted(facts.operations.focusPoint, () => diff --git a/packages/platform-apple/src/screenshot-support.test.ts b/packages/platform-apple/src/screenshot-support.test.ts new file mode 100644 index 0000000000..6f87abb83d --- /dev/null +++ b/packages/platform-apple/src/screenshot-support.test.ts @@ -0,0 +1,59 @@ +import { expect, test, vi } from 'vitest'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { Interactor, ScreenshotOptions } from '@agent-device/contracts/interactor-types'; +import { createApplePlatformRuntime } from './runtime.ts'; +import { platformRuntimeHostFixture } from './runtime.fixtures.ts'; + +const simulator: DeviceInfo = { + platform: 'apple', + appleOs: 'ios', + id: 'sim-1', + name: 'iPhone', + kind: 'simulator', + booted: true, +}; + +test.each([ + [simulator, {}, true], + [{ ...simulator, appleOs: 'ipados' }, {}, true], + [simulator, { captureBackend: 'runner' }, false], + [{ ...simulator, kind: 'device' }, {}, false], + [{ ...simulator, appleOs: 'macos', kind: 'device', target: 'desktop' }, {}, false], + [{ ...simulator, appleOs: 'tvos', target: 'tv' }, {}, false], + [{ ...simulator, appleOs: 'visionos' }, {}, false], +] satisfies [DeviceInfo, Partial, boolean][])( + 'the bound Apple capture declares mask support for %j with %j: %s', + async (device, options, supported) => { + const screenshot = vi.fn(async () => ({ displayRotation: 'portrait' as const })); + const resolve = vi.fn(async () => ({ screenshot }) as unknown as Interactor); + const host = { ...platformRuntimeHostFixture(), localInteractors: { resolve } }; + const binding = await createApplePlatformRuntime(host).bind({ + device, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + const capture = binding.operations.captureScreenshot; + expect(capture).toBeDefined(); + const result = capture!({ + outPath: '/tmp/masked.png', + options: { ...options, screenMask: 'black' }, + }); + if (supported) { + await expect(result).resolves.toEqual({ displayRotation: 'portrait' }); + expect(screenshot).toHaveBeenCalledWith('/tmp/masked.png', { + ...options, + screenMask: 'black', + }); + } else { + await expect(result).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-screen-mask-unsupported' }, + }); + expect(resolve).not.toHaveBeenCalled(); + } + }, +); diff --git a/packages/platform-apple/src/screenshot-support.ts b/packages/platform-apple/src/screenshot-support.ts new file mode 100644 index 0000000000..858fcb8759 --- /dev/null +++ b/packages/platform-apple/src/screenshot-support.ts @@ -0,0 +1,14 @@ +import { isHandheldAppleSimulator, type DeviceInfo } from '@agent-device/kernel/device'; +import type { + ScreenshotCaptureCapabilities, + ScreenshotOptions, +} from '@agent-device/contracts/screenshot-runtime'; + +export function appleScreenshotCaptureCapabilities( + device: DeviceInfo, + options: Pick, +): ScreenshotCaptureCapabilities { + return { + screenMask: isHandheldAppleSimulator(device) && options.captureBackend !== 'runner', + }; +} diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 87b87d92bc..7bc99ffd24 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -380,7 +380,7 @@ function summarizeProviderScenarioFlagExclusions() { { name: 'Apple simulator screenshot rendering options', owner: 'iOS platform and screenshot-diff runtime tests', - keys: ['screenshotNormalizeStatusBar', 'screenshotPixelDensity'], + keys: ['screenshotNormalizeStatusBar', 'screenshotPixelDensity', 'screenshotScreenMask'], }, { // Reading accessibility custom actions has no provider-scenario surface: diff --git a/src/cli/commands/__tests__/screenshot.test.ts b/src/cli/commands/__tests__/screenshot.test.ts index b54f97a429..626b916b45 100644 --- a/src/cli/commands/__tests__/screenshot.test.ts +++ b/src/cli/commands/__tests__/screenshot.test.ts @@ -118,6 +118,7 @@ test.each([false, true])( { transport: async (req): Promise => { assert.equal(req.command, 'screenshot'); + assert.equal(req.flags?.screenshotScreenMask, 'black'); captures += 1; const out = req.positionals?.[0]; assert.equal(typeof out, 'string'); @@ -130,7 +131,13 @@ test.each([false, true])( diffCommand({ host: createTestHostEnvironment(), positionals: ['screenshot', ...(live ? [] : [current])], - flags: { json: true, baseline, out: diff, threshold: '1' } as CliFlags, + flags: { + json: true, + baseline, + out: diff, + threshold: '1', + ...(live ? { screenshotScreenMask: 'black' } : {}), + } as CliFlags, client, }), ); @@ -144,3 +151,18 @@ test.each([false, true])( assert.deepEqual([image.width, image.height], [2, 3]); }, ); + +test('the Node screenshot client sends the mask in its daemon request', async () => { + const client = createAgentDeviceClient( + { session: 'qa' }, + { + transport: async (req): Promise => { + assert.equal(req.command, 'screenshot'); + assert.equal(req.flags?.screenshotScreenMask, 'black'); + return { ok: true, data: { path: '/tmp/masked.png' } }; + }, + }, + ); + const result = await client.capture.screenshot({ path: '/tmp/masked.png', screenMask: 'black' }); + assert.equal(result.path, '/tmp/masked.png'); +}); diff --git a/src/cli/commands/screenshot.ts b/src/cli/commands/screenshot.ts index b1afc33b70..ec43610e0e 100644 --- a/src/cli/commands/screenshot.ts +++ b/src/cli/commands/screenshot.ts @@ -102,6 +102,7 @@ export const diffCommand: ClientCommandHandler = async ({ positionals, flags, cl threshold: parseCliThreshold(flags.threshold), overlayRefs: flags.overlayRefs, normalizeStatusBar: flags.screenshotNormalizeStatusBar, + screenMask: flags.screenshotScreenMask, surface: flags.surface, }); @@ -125,6 +126,7 @@ function createClientScreenshotBackend( pixelDensity: options?.pixelDensity, fullscreen: options?.fullscreen, normalizeStatusBar: options?.normalizeStatusBar, + screenMask: options?.screenMask, stabilize: options?.stabilize, surface: options?.surface, }); diff --git a/src/cli/parser/__tests__/args-parse-interaction.test.ts b/src/cli/parser/__tests__/args-parse-interaction.test.ts index f5cfbd58a4..0aa7872877 100644 --- a/src/cli/parser/__tests__/args-parse-interaction.test.ts +++ b/src/cli/parser/__tests__/args-parse-interaction.test.ts @@ -256,6 +256,8 @@ test('parseArgs recognizes screenshot flags', () => { 'page.png', '--pixel-density', '2', + '--screen-mask', + 'black', '--full', '-f', '--fullscreen', @@ -271,6 +273,7 @@ test('parseArgs recognizes screenshot flags', () => { assert.equal(parsed.command, 'screenshot'); assert.deepEqual(parsed.positionals, ['page.png']); assert.equal(parsed.flags.screenshotPixelDensity, 2); + assert.equal(parsed.flags.screenshotScreenMask, 'black'); assert.equal(parsed.flags.screenshotFullscreen, true); assert.equal(parsed.flags.screenshotScale, 0.3); assert.equal(parsed.flags.screenshotNoStabilize, true); @@ -445,3 +448,19 @@ test('parseArgs maps scroll until and max passes', () => { assert.equal(parsed.flags.until, 'id="row-512"'); assert.equal(parsed.flags.maxPasses, 90); }); + +test('screen-mask is accepted for live screenshot diffs and refuses unknown values', () => { + const parsed = parseArgs( + ['diff', 'screenshot', '--baseline', 'base.png', '--screen-mask', 'black'], + { strictFlags: true }, + ); + assert.equal(parsed.flags.screenshotScreenMask, 'black'); + assert.throws(() => parseArgs(['screenshot', '--screen-mask', 'alpha'], { strictFlags: true })); +}); + +test('screen-mask is refused for diff snapshot', () => { + assert.throws( + () => parseArgs(['diff', 'snapshot', '--screen-mask', 'black'], { strictFlags: true }), + { code: 'INVALID_ARGS' }, + ); +}); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 2d0864ea38..b861ae36e0 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1023,6 +1023,7 @@ agent-device screenshot page.png --scale 0.3 # Resize both dimensions to 30% fo agent-device screenshot page.png --overlay-refs # Draw current @eN refs and target rectangles onto the PNG agent-device screenshot page.png --crop-on 'label="Save"' # Crop the capture to the frame the selector resolves on the same screen agent-device screenshot baseline.png --normalize-status-bar # Normalize iOS simulator or ADB-backed Android chrome for reusable diff baselines +agent-device screenshot baseline.png --screen-mask black --normalize-status-bar # Capture an iOS simulator baseline with a black screen outline agent-device screenshot page.png --platform web --fullscreen # On web, --fullscreen/--full/-f captures the entire document agent-device viewport 1280 900 --platform web # Resize the active web viewport for fixed-layout or 100vh apps agent-device screenshot textedit.png # App-session window capture on macOS @@ -1048,6 +1049,7 @@ agent-device record stop # Stop active recording - `--crop-on ` crops the capture to the frame the selector resolves on the same screen. It works only on iOS simulators, Android emulators, and physical iPhones. A physical iPhone crop is verified only in portrait using CoreDevice/devicectl; the XCTest backend and runner fallback are not accepted. Physical iPads remain rejected for crop, and landscape iPhone crop remains unverified. It needs exactly one framed node, cannot be combined with `--overlay-refs` or `--fullscreen`, and warns when the frame is clipped. The crop is re-encoded, so do not byte-compare it with an older crop. - `--fullscreen` on macOS applies only to app sessions. Other surfaces (`desktop`, `menubar`, `frontmost-app`) always capture the main display and refuse `--fullscreen` with `INVALID_ARGS`. - `--normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for reusable baselines. Each status-bar read or preparation command has a 10-second timeout. Each restore command has a 30-second timeout, so clearing and reapplying overrides can take up to 60 seconds. The request timeout includes the wait and can expire before these steps finish. +- `--screen-mask black` fills the iOS simulator screen outline, including rounded corners, with black for this capture. Node and MCP use `screenMask: "black"`. Omit it for the default image. Support is limited to handheld iOS/iPadOS simulators. It requires simctl capture; physical devices, tvOS and visionOS simulators, other platforms, provider captures, and XCTest runner captures are refused. If simctl fails, a masked request fails instead of falling back to an unmasked runner capture. This option does not redact app content, remove a Dynamic Island drawn into the framebuffer, or change comparison thresholds. - On an iOS simulator, a screenshot waits until no other screenshot or `settings wifi` / `settings airplane` write is using that simulator's status bar, so overlapping captures cannot see each other's temporary chrome. A request you cancel while it is waiting fails with the cancellation instead of capturing. - On single-display iOS simulators and Android devices, the result may report `displayRotation` (`portrait`, `portrait-upside-down`, `landscape-left`, or `landscape-right`). It is absent when the device returns no reading in time, on other targets, and on multi-display devices such as foldables. The saved image is always upright. @@ -1058,7 +1060,7 @@ agent-device record stop # Stop active recording - The current image scales proportionally to the baseline width and aligns at the top left. Its scaled height rounds to the nearest pixel, with a minimum of one row. The comparison includes the full height of both images: extra or missing bottom rows count as differences, including blank or transparent rows, even at `--threshold 1`. Your original files stay unchanged. This does not crop system bars or move content into alignment. - When dimensions differ, JSON `normalization` reports `baseline`, `current`, `scaledCurrent`, and `compared` dimensions. `currentScale.x` and `.y` map original current pixels to comparison pixels; the height scale includes pixel rounding. Region rectangles and ref-match rectangles use comparison pixels, and normalized region rectangles use percentages of the compared canvas. The separate current-screen overlay keeps its original capture size. - `--overlay-refs` also writes a separate current-screen ref overlay and lists refs under changed regions. It needs a live capture, so it is unavailable with a saved `current.png`. -- Live iOS simulator diffs normalize status-bar chrome by default. +- Live iOS simulator diffs normalize status-bar chrome by default. Use `diff screenshot --baseline baseline.png --screen-mask black` to apply the same screen outline as a baseline captured with `screenshot --screen-mask black`. The mask affects only this live capture, including its optional ref overlay. It is refused with a saved current image or `diff snapshot`; saved-image comparisons do not apply a mask. **Recordings**