From c7ef22504af3eb3f616665da5e87c686eb47f483 Mon Sep 17 00:00:00 2001 From: Anthony Mittaz Date: Wed, 4 Nov 2026 10:40:43 +1100 Subject: [PATCH 1/7] feat(screenshot): opt into black iOS simulator screen outlines --- .../command-surface/src/capture/screenshot.ts | 2 +- .../core/__tests__/display-inventory.test.ts | 10 ++- .../screenshot-simulator-capture.test.ts | 87 ++++++++++++------- .../platform-apple/src/core/screenshot.ts | 7 +- website/docs/docs/commands.md | 1 + 5 files changed, 70 insertions(+), 37 deletions(-) diff --git a/packages/command-surface/src/capture/screenshot.ts b/packages/command-surface/src/capture/screenshot.ts index 2fd871e96d..ed928797be 100644 --- a/packages/command-surface/src/capture/screenshot.ts +++ b/packages/command-surface/src/capture/screenshot.ts @@ -74,7 +74,7 @@ export const screenshotCommandFacet = defineCommandFacet({ text: { summary: 'Capture a screenshot', cliDetail: - "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.", + "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. Set AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK=black before starting the daemon to black out the screen outline in simctl captures. Unset or other values keep the default; runner captures are unaffected. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.", }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input), diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index e243ad1390..85a4a723b6 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -508,19 +508,25 @@ describe('captureSimulatorScreenshotWithRetry display targeting', () => { }); test('names the lit panel explicitly instead of accepting the black default', async () => { - await captureSimulatorScreenshotWithRetry(IOS_TEST_SIMULATOR, '/tmp/duo.png', outerPanel); + await captureSimulatorScreenshotWithRetry( + IOS_TEST_SIMULATOR, + '/tmp/duo.png', + { AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK: 'black' }, + outerPanel, + ); assert.equal(mockRunSimctlForDevice.mock.calls.length, 1); assert.deepEqual(mockRunSimctlForDevice.mock.calls[0]![1], [ 'io', IOS_TEST_SIMULATOR.id, 'screenshot', + '--mask=black', '--display=LCD', '/tmp/duo.png', ]); }); test('omits the display flag for a single-panel device', async () => { - await captureSimulatorScreenshotWithRetry(IOS_TEST_SIMULATOR, '/tmp/duo.png', undefined); + await captureSimulatorScreenshotWithRetry(IOS_TEST_SIMULATOR, '/tmp/duo.png', {}); assert.deepEqual(mockRunSimctlForDevice.mock.calls[0]![1], [ 'io', IOS_TEST_SIMULATOR.id, diff --git a/packages/platform-apple/src/core/__tests__/screenshot-simulator-capture.test.ts b/packages/platform-apple/src/core/__tests__/screenshot-simulator-capture.test.ts index 2ad3791535..661f9b2066 100644 --- a/packages/platform-apple/src/core/__tests__/screenshot-simulator-capture.test.ts +++ b/packages/platform-apple/src/core/__tests__/screenshot-simulator-capture.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, test, vi } from 'vitest'; +import { afterEach, beforeEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; import { promises as fs, writeFileSync } from 'node:fs'; import path from 'node:path'; @@ -26,6 +26,10 @@ beforeEach(() => { mockRetryWithPolicy.mockImplementation(retryActual.retryWithPolicy); }); +afterEach(() => { + vi.unstubAllEnvs(); +}); + const BOOTED_SIM_LIST_JSON = JSON.stringify({ devices: { 'com.apple.CoreSimulator.SimRuntime.iOS-18-0': [{ udid: 'sim-1', state: 'Booted' }] }, }); @@ -96,13 +100,15 @@ test('screenshotIos retries simulator capture timeouts and eventually succeeds', exitCode: 60, }; } - writeFileSync(args[4] ?? '', sourcePng); + writeFileSync(args.at(-1) ?? '', sourcePng); return ''; } return unexpectedArgs(args); }, async ({ calls }) => { - await screenshotIos(IOS_TEST_SIMULATOR, outPath, { env: process.env }); + await screenshotIos(IOS_TEST_SIMULATOR, outPath, { + env: { AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK: 'black' }, + }); const png = PNG.sync.read(await fs.readFile(outPath)); assert.equal(png.width, 42); assert.equal(png.height, 91); @@ -110,7 +116,7 @@ test('screenshotIos retries simulator capture timeouts and eventually succeeds', const flat = calls.map((args) => args.join(' ')); assert.equal( - flat.filter((line) => line === `simctl io sim-1 screenshot ${outPath}`).length, + flat.filter((line) => line === `simctl io sim-1 screenshot --mask=black ${outPath}`).length, 3, 'should retry screenshot command until success', ); @@ -123,31 +129,48 @@ test('screenshotIos retries simulator capture timeouts and eventually succeeds', ); }); -test('screenshotIos keeps requested simulator pixel density', async () => { - const tmpDir = await mkdtempForTest('agent-device-ios-screenshot-density-'); - const outPath = path.join(tmpDir, 'screen.png'); - - // Both dimensions divisible by 3 so the 2/3 rescale (density 2 against the stub's native - // scale 3) stays exact. What this pins is that ratio, not an absolute resolution: it used - // to allocate and resize a 1206x2622 iPhone 16 Pro frame — 3.2 megapixels, ~90x more than - // the arithmetic needs. - const sourcePng = PNG.sync.write(new PNG({ width: 126, height: 273 })); - - await withFakeAppleTool( - (args) => { - if (isSimctlListDevices(args)) return BOOTED_SIM_LIST_JSON; - if (isSimctlMainScreenScale(args)) return '3\n'; - if (isSimctlScreenshot(args)) { - writeFileSync(args[4] ?? '', sourcePng); - return ''; - } - return unexpectedArgs(args); - }, - async () => { - await screenshotIos(IOS_TEST_SIMULATOR, outPath, { env: process.env, pixelDensity: 2 }); - const png = PNG.sync.read(await fs.readFile(outPath)); - assert.equal(png.width, 84); - assert.equal(png.height, 182); - }, - ); -}); +test.each([undefined, '', 'alpha', 'BLACK', 'black'])( + 'screenshotIos keeps density and opts into a black outline only for host setting %s', + async (mask) => { + vi.stubEnv('AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK', mask === 'black' ? 'alpha' : 'black'); + const tmpDir = await mkdtempForTest('agent-device-ios-screenshot-density-'); + const outPath = path.join(tmpDir, 'screen.png'); + + // Both dimensions divisible by 3 so the 2/3 rescale (density 2 against the stub's native + // scale 3) stays exact. What this pins is that ratio, not an absolute resolution: it used + // to allocate and resize a 1206x2622 iPhone 16 Pro frame — 3.2 megapixels, ~90x more than + // the arithmetic needs. + const sourcePng = PNG.sync.write(new PNG({ width: 126, height: 273 })); + + await withFakeAppleTool( + (args) => { + if (isSimctlListDevices(args)) return BOOTED_SIM_LIST_JSON; + if (isSimctlMainScreenScale(args)) return '3\n'; + if (isSimctlScreenshot(args)) { + writeFileSync(args.at(-1) ?? '', sourcePng); + return ''; + } + return unexpectedArgs(args); + }, + async ({ calls }) => { + await screenshotIos(IOS_TEST_SIMULATOR, outPath, { + env: { AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK: mask }, + pixelDensity: 2, + }); + assert.deepEqual(calls.filter(isSimctlScreenshot), [ + [ + 'simctl', + 'io', + 'sim-1', + 'screenshot', + ...(mask === 'black' ? ['--mask=black'] : []), + outPath, + ], + ]); + const png = PNG.sync.read(await fs.readFile(outPath)); + assert.equal(png.width, 84); + assert.equal(png.height, 182); + }, + ); + }, +); diff --git a/packages/platform-apple/src/core/screenshot.ts b/packages/platform-apple/src/core/screenshot.ts index a564ee0ecc..abab7d814b 100644 --- a/packages/platform-apple/src/core/screenshot.ts +++ b/packages/platform-apple/src/core/screenshot.ts @@ -66,6 +66,7 @@ type SimulatorScreenshotFlowDeps = { captureWithRetry: ( device: DeviceInfo, outPath: string, + env: Readonly, display: AppleDeviceDisplay | undefined, ) => Promise; /** @@ -93,7 +94,7 @@ type SimulatorScreenshotFlowDeps = { }; type SimulatorScreenshotFlowOptions = { - /** The calling runtime's host environment; runner-container configuration is read from it. */ + /** The calling runtime's host environment for simulator capture and runner-container configuration. */ env: Readonly; appBundleId?: string; fullscreen?: boolean; @@ -169,7 +170,7 @@ export async function captureSimulatorScreenshotWithFallback( const { display, displayRotation } = await deps.resolveCaptureTarget(device); const captureFacts: ScreenshotCaptureFacts = displayRotation ? { displayRotation } : {}; const captureAndNormalize = async () => { - await deps.captureWithRetry(device, outPath, display); + await deps.captureWithRetry(device, outPath, options.env, display); await deps.normalizeDensity(device, outPath, options.pixelDensity, display?.pointScale); }; const captureWithFallback = async (): Promise => { @@ -219,6 +220,7 @@ export async function captureSimulatorScreenshotWithFallback( export async function captureSimulatorScreenshotWithRetry( device: DeviceInfo, outPath: string, + env: Readonly, display?: AppleDeviceDisplay, ): Promise { const deadline = Deadline.fromTimeoutMs(IOS_SIMULATOR_SCREENSHOT_TIMEOUT_MS); @@ -226,6 +228,7 @@ export async function captureSimulatorScreenshotWithRetry( 'io', device.id, 'screenshot', + ...(env.AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK === 'black' ? ['--mask=black'] : []), ...appleSimulatorDisplayArgvFragment(display), outPath, ]; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 2d0864ea38..4c72224993 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1048,6 +1048,7 @@ agent-device record stop # Stop active recording - `--crop-on ` crops the capture to the frame the selector resolves on the same screen. It works only on iOS simulators, Android emulators, and physical iPhones. A physical iPhone crop is verified only in portrait using CoreDevice/devicectl; the XCTest backend and runner fallback are not accepted. Physical iPads remain rejected for crop, and landscape iPhone crop remains unverified. It needs exactly one framed node, cannot be combined with `--overlay-refs` or `--fullscreen`, and warns when the frame is clipped. The crop is re-encoded, so do not byte-compare it with an older crop. - `--fullscreen` on macOS applies only to app sessions. Other surfaces (`desktop`, `menubar`, `frontmost-app`) always capture the main display and refuse `--fullscreen` with `INVALID_ARGS`. - `--normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for reusable baselines. Each status-bar read or preparation command has a 10-second timeout. Each restore command has a 30-second timeout, so clearing and reapplying overrides can take up to 60 seconds. The request timeout includes the wait and can expire before these steps finish. +- Set `AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK=black` in the daemon host environment before starting the daemon to fill the simulator screen outline, including rounded corners, with black in `simctl` captures. Restart an existing daemon for the setting to take effect. Unset or other values keep the default capture. This does not redact app content, remove a Dynamic Island drawn into the framebuffer, or change comparison thresholds. Physical-device and XCTest runner captures, including fallback captures, are unaffected. - On an iOS simulator, a screenshot waits until no other screenshot or `settings wifi` / `settings airplane` write is using that simulator's status bar, so overlapping captures cannot see each other's temporary chrome. A request you cancel while it is waiting fails with the cancellation instead of capturing. - On single-display iOS simulators and Android devices, the result may report `displayRotation` (`portrait`, `portrait-upside-down`, `landscape-left`, or `landscape-right`). It is absent when the device returns no reading in time, on other targets, and on multi-display devices such as foldables. The saved image is always upright. From 35fba700249f74fb3cd3752b3b536863c40ac0f3 Mon Sep 17 00:00:00 2001 From: Anthony Mittaz Date: Wed, 4 Nov 2026 11:56:59 +1100 Subject: [PATCH 2/7] feat(screenshot): make the screen mask a per-request option --- .../capture/runtime/diff-screenshot.test.ts | 23 +++++++++- .../capture/runtime/diff-screenshot.ts | 16 ++++++- .../commands/capture/runtime/screenshot.ts | 3 +- .../src/commands/runtime-types.ts | 2 + packages/command-surface/src/capture/diff.ts | 14 ++++-- .../command-surface/src/capture/index.test.ts | 29 ++++++++++++- .../src/capture/screenshot-options.test.ts | 18 ++++++++ .../command-surface/src/capture/screenshot.ts | 4 +- packages/contracts/src/backend.ts | 2 + packages/contracts/src/client-capture.ts | 2 + packages/contracts/src/facades/capture.ts | 3 ++ packages/contracts/src/interactor-types.ts | 2 + .../contracts/src/screenshot-runtime.test.ts | 13 ++++++ packages/contracts/src/screenshot-runtime.ts | 14 +++++- packages/contracts/src/screenshot.ts | 42 +++++++++++++++++- .../screenshot-runtime-execution.test.ts | 43 +++++++++++++++++++ .../src/execution/screenshot-runtime.ts | 13 ++++++ .../interactor-screenshot-backend.test.ts | 14 ++++++ .../core/__tests__/display-inventory.test.ts | 4 +- .../screenshot-simulator-capture.test.ts | 12 +++--- .../src/core/__tests__/screenshot.test.ts | 22 ++++++++++ .../platform-apple/src/core/screenshot.ts | 14 +++--- packages/platform-apple/src/interactor.ts | 7 +++ scripts/integration-progress-model.ts | 2 +- src/cli/commands/__tests__/screenshot.test.ts | 24 ++++++++++- src/cli/commands/screenshot.ts | 2 + .../__tests__/args-parse-interaction.test.ts | 19 ++++++++ website/docs/docs/commands.md | 6 +-- 28 files changed, 340 insertions(+), 29 deletions(-) diff --git a/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.test.ts b/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.test.ts index 3a18ed1631..346df406c9 100644 --- a/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.test.ts @@ -143,7 +143,7 @@ test('runtime diff screenshot enforces the normalized comparison pixel budget', } }); -test('runtime diff screenshot attaches overlay refs to live mismatch regions', async () => { +test('runtime diff screenshot keeps the screen mask on current and overlay captures', async () => { const dir = mkdtempForTestSync('runtime-diff-screenshot-'); const baseline = path.join(dir, 'baseline.png'); const diffOut = path.join(dir, 'diff.png'); @@ -153,6 +153,7 @@ test('runtime diff screenshot attaches overlay refs to live mismatch regions', a try { const device = createAgentDevice({ backend: createScreenshotBackend((outPath, options) => { + assert.equal(options?.screenMask, 'black'); fs.writeFileSync(outPath, solidPngBuffer(20, 20, { r: 255, g: 255, b: 255 })); return { path: outPath, @@ -182,6 +183,7 @@ test('runtime diff screenshot attaches overlay refs to live mismatch regions', a out: { kind: 'path', path: diffOut }, threshold: 0, overlayRefs: true, + screenMask: 'black', }); assert.equal(result.currentOverlayPath, overlayOut); @@ -230,3 +232,22 @@ function solidPngBuffer( } return PNG.sync.write(png); } + +test('runtime diff refuses a screen mask for saved images before reading or capturing', async () => { + const device = createAgentDevice({ + backend: createScreenshotBackend(() => { + throw new Error('must not capture'); + }), + artifacts: createLocalArtifactAdapter(), + sessions, + policy: localCommandPolicy(), + }); + await assert.rejects( + device.capture.diffScreenshot({ + baseline: { kind: 'path', path: '/missing/baseline.png' }, + current: { kind: 'path', path: '/missing/current.png' }, + screenMask: 'black', + }), + { code: 'INVALID_ARGS', message: /requires a live screenshot/ }, + ); +}); diff --git a/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.ts b/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.ts index 656acd361f..e9f4940c32 100644 --- a/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.ts @@ -12,6 +12,10 @@ import type { ResolvedInputFile, } from '@agent-device/contracts/artifact-adapter'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; +import { + readScreenshotScreenMask, + type ScreenshotScreenMask, +} from '@agent-device/contracts/capture'; import { AppError } from '@agent-device/kernel/errors'; import { compareScreenshots, @@ -37,6 +41,7 @@ export type DiffScreenshotCommandOptions = CommandContext & { threshold?: number; overlayRefs?: boolean; normalizeStatusBar?: boolean; + screenMask?: ScreenshotScreenMask; surface?: BackendScreenshotOptions['surface']; }; @@ -54,6 +59,7 @@ export const diffScreenshotCommand: RuntimeCommand< throw new AppError('INVALID_ARGS', 'diff screenshot requires a baseline image'); } + readScreenshotScreenMask(options.screenMask); const threshold = normalizeThreshold(options.threshold); const currentRef = options.current ?? { kind: 'live' }; if (options.overlayRefs && !isLiveCurrentRef(currentRef)) { @@ -63,6 +69,13 @@ export const diffScreenshotCommand: RuntimeCommand< ); } + if (options.screenMask !== undefined && !isLiveCurrentRef(currentRef)) { + throw new AppError( + 'INVALID_ARGS', + '--screen-mask requires a live screenshot, not a saved current image', + ); + } + const baseline = await resolveCommandInput(runtime, options.baseline, { usage: 'diff screenshot baseline', field: 'baseline', @@ -222,10 +235,11 @@ async function captureScreenshot( } function liveDiffScreenshotOptions( - options: Pick, + options: Pick, ): BackendScreenshotOptions { return { normalizeStatusBar: options.normalizeStatusBar ?? true, + ...(options.screenMask !== undefined ? { screenMask: options.screenMask } : {}), ...(options.surface ? { surface: options.surface } : {}), }; } diff --git a/packages/command-runtime/src/commands/capture/runtime/screenshot.ts b/packages/command-runtime/src/commands/capture/runtime/screenshot.ts index 33a2f03cff..f75e49ae51 100644 --- a/packages/command-runtime/src/commands/capture/runtime/screenshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/screenshot.ts @@ -1,5 +1,5 @@ import { AppError } from '@agent-device/kernel/errors'; -import { validateScreenshotScale } from '@agent-device/contracts/capture'; +import { validateScreenshotScale, readScreenshotScreenMask } from '@agent-device/contracts/capture'; import { successText } from '@agent-device/kernel/success-text'; import { resizePngFileToScale } from '@agent-device/capture-kit/png-resize'; import type { DeviceRotation } from '@agent-device/contracts/device'; @@ -46,6 +46,7 @@ export const screenshotCommand: RuntimeCommand< fullscreen: options.fullscreen, overlayRefs: options.overlayRefs, pixelDensity: options.pixelDensity, + screenMask: readScreenshotScreenMask(options.screenMask), stabilize: options.stabilize, normalizeStatusBar: options.normalizeStatusBar, surface: options.surface, diff --git a/packages/command-runtime/src/commands/runtime-types.ts b/packages/command-runtime/src/commands/runtime-types.ts index 5268cb5a4e..a3e3009dc8 100644 --- a/packages/command-runtime/src/commands/runtime-types.ts +++ b/packages/command-runtime/src/commands/runtime-types.ts @@ -1,3 +1,4 @@ +import type { ScreenshotScreenMask } from '@agent-device/contracts/capture'; import type { FileOutputRef } from '@agent-device/contracts/artifact-adapter'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; import type { SessionSurface } from '@agent-device/contracts/session'; @@ -50,6 +51,7 @@ export type ScreenshotCommandOptions = CommandContext & { fullscreen?: boolean; overlayRefs?: boolean; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; scale?: number; stabilize?: boolean; normalizeStatusBar?: boolean; diff --git a/packages/command-surface/src/capture/diff.ts b/packages/command-surface/src/capture/diff.ts index 0c81e593fb..a2cc3134a7 100644 --- a/packages/command-surface/src/capture/diff.ts +++ b/packages/command-surface/src/capture/diff.ts @@ -76,10 +76,18 @@ const diffCommandMetadata = defineFieldCommandMetadata(DIFF_COMMAND_NAME, diffCo const diffCliSchema = { usageOverride: - 'diff snapshot | diff screenshot --baseline [current.png] [--out ] [--threshold <0-1>] [--overlay-refs]', + 'diff snapshot | diff screenshot --baseline [current.png] [--out ] [--threshold <0-1>] [--overlay-refs] [--screen-mask ]', usageFlags: [], positionalArgs: ['kind', 'current?'], - allowedFlags: [...SNAPSHOT_FLAGS, 'baseline', 'threshold', 'out', 'overlayRefs'], + flagsByAction: { snapshot: [], screenshot: ['screenshotScreenMask'] }, + allowedFlags: [ + ...SNAPSHOT_FLAGS, + 'baseline', + 'threshold', + 'out', + 'overlayRefs', + 'screenshotScreenMask', + ], } as const; export const diffCliReader: CliReader = (positionals, flags) => { @@ -106,7 +114,7 @@ export const diffCommandFacet = defineCommandFacet({ text: { summary: 'Diff snapshot or screenshot', cliDetail: - 'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; the current image scales proportionally to the baseline width and aligns top-left. Scaled height rounds to the nearest pixel (minimum 1). Extra or missing bottom rows always count as differences, even at threshold 1. The originals remain unchanged. When dimensions differ, normalization metadata reports original, scaled and compared sizes; region coordinates use comparison pixels. Both screenshot inputs are decoded from their bytes, so a baseline or current image may be PNG or JPEG, and the diff image is always PNG. JPEG is lossy, so keep --threshold above 0 whenever either input is JPEG. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines.', + 'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; the current image scales proportionally to the baseline width and aligns top-left. Scaled height rounds to the nearest pixel (minimum 1). Extra or missing bottom rows always count as differences, even at threshold 1. The originals remain unchanged. When dimensions differ, normalization metadata reports original, scaled and compared sizes; region coordinates use comparison pixels. Both screenshot inputs are decoded from their bytes, so a baseline or current image may be PNG or JPEG, and the diff image is always PNG. JPEG is lossy, so keep --threshold above 0 whenever either input is JPEG. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines. Use --screen-mask black for a live iOS simulator capture with a black screen outline, and use the same option when creating its baseline. --screen-mask is refused when current.png is supplied; saved images are compared unchanged.', }, metadata: diffCommandMetadata, run: (client, input) => client.capture.diff(input), diff --git a/packages/command-surface/src/capture/index.test.ts b/packages/command-surface/src/capture/index.test.ts index 11f67a91b3..28a6aeac72 100644 --- a/packages/command-surface/src/capture/index.test.ts +++ b/packages/command-surface/src/capture/index.test.ts @@ -2,7 +2,11 @@ import { describe, expect, test } from 'vitest'; import type { CliFlags } from '@agent-device/contracts/command'; import { alertCliReader, alertDaemonWriter } from './alert.ts'; import { diffCliReader } from './diff.ts'; -import { screenshotCliReader, screenshotDaemonWriter } from './screenshot.ts'; +import { + screenshotCliReader, + screenshotDaemonWriter, + screenshotCommandFacet, +} from './screenshot.ts'; import { snapshotCliReader } from './snapshot.ts'; import { waitCliReader, waitDaemonWriter } from './wait.ts'; @@ -48,11 +52,17 @@ describe('capture command interface', () => { test('reads screenshot path and writes screenshot flags', () => { const input = screenshotCliReader( ['page.png'], - flags({ screenshotPixelDensity: 2, screenshotFullscreen: true, screenshotScale: 0.3 }), + flags({ + screenshotScreenMask: 'black', + screenshotPixelDensity: 2, + screenshotFullscreen: true, + screenshotScale: 0.3, + }), ); expect(input).toMatchObject({ path: 'page.png', pixelDensity: 2, + screenMask: 'black', fullscreen: true, scale: 0.3, }); @@ -61,6 +71,7 @@ describe('capture command interface', () => { positionals: ['page.png'], options: { screenshotPixelDensity: 2, + screenshotScreenMask: 'black', screenshotFullscreen: true, screenshotScale: 0.3, }, @@ -173,3 +184,17 @@ describe('capture command interface', () => { }); }); }); + +test('refuses invalid screen-mask inputs from structured clients', () => { + expectInvalidArgs( + () => screenshotDaemonWriter({ screenMask: 'alpha' }), + '--screen-mask must be black', + ); +}); + +test('screenshot exposes the screen mask enum in the shared Node and MCP input schema', () => { + expect(screenshotCommandFacet.metadata.inputSchema.properties?.screenMask).toMatchObject({ + type: 'string', + enum: ['black'], + }); +}); diff --git a/packages/command-surface/src/capture/screenshot-options.test.ts b/packages/command-surface/src/capture/screenshot-options.test.ts index 254e65ebcc..4140eea6a3 100644 --- a/packages/command-surface/src/capture/screenshot-options.test.ts +++ b/packages/command-surface/src/capture/screenshot-options.test.ts @@ -20,6 +20,7 @@ test('screenshot flag projection maps CLI flags to runtime options', () => { overlayRefs: true, screenshotCropOn: 'role=button label=Save', screenshotPixelDensity: 2, + screenshotScreenMask: 'black', screenshotFullscreen: true, screenshotScale: 0.3, screenshotNoStabilize: true, @@ -29,6 +30,7 @@ test('screenshot flag projection maps CLI flags to runtime options', () => { overlayRefs: true, cropOn: 'role=button label=Save', pixelDensity: 2, + screenMask: 'black', fullscreen: true, scale: 0.3, stabilize: false, @@ -43,6 +45,7 @@ test('screenshot flag projection maps public options to request flags', () => { overlayRefs: true, cropOn: 'label="Network & internet"', pixelDensity: 3, + screenMask: 'black', fullscreen: true, stabilize: false, normalizeStatusBar: true, @@ -51,6 +54,7 @@ test('screenshot flag projection maps public options to request flags', () => { overlayRefs: true, screenshotCropOn: 'label="Network & internet"', screenshotPixelDensity: 3, + screenshotScreenMask: 'black', screenshotFullscreen: true, screenshotNoStabilize: true, screenshotNormalizeStatusBar: true, @@ -117,11 +121,23 @@ test('screenshot script flags use the shared recorded flag contract', () => { message: /requires a selector expression/, }); + result = readScreenshotScriptFlag({ args: ['--screen-mask', 'black'], index: 0, flags }); + assert.deepEqual(result, { handled: true, nextIndex: 1 }); + assert.equal(flags.screenshotScreenMask, 'black'); + assert.throws( + () => readScreenshotScriptFlag({ args: ['--screen-mask', 'alpha'], index: 0, flags: {} }), + { code: 'INVALID_ARGS' }, + ); + assert.throws(() => readScreenshotScriptFlag({ args: ['--screen-mask'], index: 0, flags: {} }), { + code: 'INVALID_ARGS', + }); appendScreenshotScriptFlags(parts, flags); assert.deepEqual(parts, [ '--pixel-density', '3', + '--screen-mask', + 'black', '--fullscreen', '--scale', '0.3', @@ -131,6 +147,7 @@ test('screenshot script flags use the shared recorded flag contract', () => { assert.deepEqual(SCREENSHOT_ACTION_FLAG_KEYS, [ 'screenshotCropOn', 'screenshotPixelDensity', + 'screenshotScreenMask', 'screenshotFullscreen', 'screenshotScale', 'screenshotNoStabilize', @@ -145,6 +162,7 @@ test('screenshot script flags use the shared recorded flag contract', () => { 'overlayRefs', 'screenshotCropOn', 'screenshotPixelDensity', + 'screenshotScreenMask', 'screenshotFullscreen', 'screenshotScale', 'screenshotNoStabilize', diff --git a/packages/command-surface/src/capture/screenshot.ts b/packages/command-surface/src/capture/screenshot.ts index ed928797be..13d0debb18 100644 --- a/packages/command-surface/src/capture/screenshot.ts +++ b/packages/command-surface/src/capture/screenshot.ts @@ -5,6 +5,7 @@ import { RETIRED_SCREENSHOT_MAX_SIZE, SCREENSHOT_COMMAND_FLAG_KEYS, SCREENSHOT_SCALE_LIMITS, + SCREENSHOT_SCREEN_MASKS, screenshotFlagsFromPublicOptions, screenshotOptionsFromFlags, validateNoRetiredScreenshotMaxSize, @@ -40,6 +41,7 @@ const screenshotCommandMetadata = defineFieldCommandMetadata( pixelDensity: integerField('Output screenshot pixel density in pixels per logical point.', { min: 1, }), + screenMask: enumField(SCREENSHOT_SCREEN_MASKS), fullscreen: booleanField(), scale: numberField('Screenshot scale factor.', SCREENSHOT_SCALE_LIMITS), maxSize: retiredField(RETIRED_SCREENSHOT_MAX_SIZE.migration.screenshot), @@ -74,7 +76,7 @@ export const screenshotCommandFacet = defineCommandFacet({ text: { summary: 'Capture a screenshot', cliDetail: - "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. Set AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK=black before starting the daemon to black out the screen outline in simctl captures. Unset or other values keep the default; runner captures are unaffected. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.", + "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. Use --screen-mask black to fill the iOS simulator screen outline, including rounded corners, with black for this capture. It requires simctl: physical devices, other platforms, provider captures and XCTest runner captures are refused. A masked capture fails if simctl fails instead of falling back to an unmasked runner capture. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.", }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input), diff --git a/packages/contracts/src/backend.ts b/packages/contracts/src/backend.ts index ae835bcc9b..b0b5f8d7a3 100644 --- a/packages/contracts/src/backend.ts +++ b/packages/contracts/src/backend.ts @@ -1,3 +1,4 @@ +import type { ScreenshotScreenMask } from './screenshot.ts'; import type { ScreenshotResultData } from './snapshot-types.ts'; import type { SnapshotCaptureAnnotations } from './snapshot-capture-annotations.ts'; import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts'; @@ -71,6 +72,7 @@ export type BackendScreenshotOptions = { fullscreen?: boolean; overlayRefs?: boolean; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; stabilize?: boolean; normalizeStatusBar?: boolean; surface?: SessionSurface; diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index 6024cbb0b0..0df298488c 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -1,3 +1,4 @@ +import type { ScreenshotScreenMask } from './screenshot.ts'; // The public API vocabulary for snapshot, screenshot and diff capture. import type { SessionSurface } from './session-surface.ts'; @@ -112,6 +113,7 @@ export type CaptureScreenshotOptions = AgentDeviceRequestOverrides & { /** Crop the capture to the frame of the selector resolved on the same screen. */ cropOn?: string; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; fullscreen?: boolean; scale?: number; stabilize?: boolean; diff --git a/packages/contracts/src/facades/capture.ts b/packages/contracts/src/facades/capture.ts index fa1318d02a..ca05162baf 100644 --- a/packages/contracts/src/facades/capture.ts +++ b/packages/contracts/src/facades/capture.ts @@ -6,6 +6,8 @@ export { SCREENSHOT_CROP_REASONS, SCREENSHOT_FULLSCREEN_REASONS, SCREENSHOT_SCALE_LIMITS, + SCREENSHOT_SCREEN_MASKS, + readScreenshotScreenMask, SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS, appendScreenshotScriptFlags, readScreenshotScriptFlag, @@ -18,6 +20,7 @@ export { } from '../screenshot.ts'; export type { ScreenshotCropReason, + ScreenshotScreenMask, ScreenshotDispatchFlags, ScreenshotFullscreenReason, ScreenshotPublicOptions, diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index 0de826f22b..dce79ef80b 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -1,3 +1,4 @@ +import type { ScreenshotScreenMask } from './screenshot.ts'; import type { AppStateRuntimeResult } from './app-state-runtime.ts'; import type { ElementTextTarget } from './element-text-target.ts'; import type { BackMode } from './back-mode.ts'; @@ -61,6 +62,7 @@ export type { BackMode }; export type ScreenshotOptions = { appBundleId?: string; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; fullscreen?: boolean; normalizeStatusBar?: boolean; stabilize?: boolean; diff --git a/packages/contracts/src/screenshot-runtime.test.ts b/packages/contracts/src/screenshot-runtime.test.ts index a4e884ae4a..bf1e1ee754 100644 --- a/packages/contracts/src/screenshot-runtime.test.ts +++ b/packages/contracts/src/screenshot-runtime.test.ts @@ -54,3 +54,16 @@ test('a provider binding fails closed when its exact owner exposes no interactor details: { reason: 'provider-runtime-interactor-missing' }, }); }); + +test('a provider capture refuses a simctl screen mask before calling its interactor', async () => { + const resolveInteractor = vi.fn(); + const operations = bindProviderScreenshotInteractor({ + device: { ...device, platform: 'apple', appleOs: 'ios', kind: 'simulator' }, + signal: new AbortController().signal, + resolveInteractor, + }); + await expect( + operations.captureScreenshot({ outPath: '/tmp/masked.png', options: { screenMask: 'black' } }), + ).rejects.toMatchObject({ code: 'UNSUPPORTED_OPERATION' }); + expect(resolveInteractor).not.toHaveBeenCalled(); +}); diff --git a/packages/contracts/src/screenshot-runtime.ts b/packages/contracts/src/screenshot-runtime.ts index eae0fef67e..337c5960b4 100644 --- a/packages/contracts/src/screenshot-runtime.ts +++ b/packages/contracts/src/screenshot-runtime.ts @@ -1,3 +1,4 @@ +import { AppError } from '@agent-device/kernel/errors'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { localInteractorSource, @@ -83,8 +84,19 @@ export function bindProviderScreenshotInteractor( resolveInteractor: ProviderInteractorOperationResolver; }>, ): ScreenshotRuntimeOperations { - return bindScreenshotCapture( + const capture = bindScreenshotCapture( params.signal, providerInteractorSource({ ...params, operation: 'screenshot' }), ); + return Object.freeze({ + captureScreenshot: async (input: CaptureScreenshotInput) => { + if (input.options?.screenMask !== undefined) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + '--screen-mask requires local simctl capture, not a provider capture', + ); + } + return await capture.captureScreenshot(input); + }, + }); } diff --git a/packages/contracts/src/screenshot.ts b/packages/contracts/src/screenshot.ts index 48e16a4373..be62912d25 100644 --- a/packages/contracts/src/screenshot.ts +++ b/packages/contracts/src/screenshot.ts @@ -1,5 +1,13 @@ import { AppError } from '@agent-device/kernel/errors'; +export const SCREENSHOT_SCREEN_MASKS = ['black'] as const; +export type ScreenshotScreenMask = (typeof SCREENSHOT_SCREEN_MASKS)[number]; + +export function readScreenshotScreenMask(value: unknown): ScreenshotScreenMask | undefined { + if (value === undefined || value === 'black') return value; + throw new AppError('INVALID_ARGS', '--screen-mask must be black'); +} + export const SCREENSHOT_SCALE_LIMITS = { min: 0.01, max: 1 } as const; /** @@ -85,6 +93,7 @@ export const SCREENSHOT_COMMAND_FLAG_KEYS = [ 'overlayRefs', 'screenshotCropOn', 'screenshotPixelDensity', + 'screenshotScreenMask', 'screenshotFullscreen', 'screenshotScale', 'screenshotNoStabilize', @@ -94,6 +103,7 @@ export const SCREENSHOT_COMMAND_FLAG_KEYS = [ export const SCREENSHOT_ACTION_FLAG_KEYS = [ 'screenshotCropOn', 'screenshotPixelDensity', + 'screenshotScreenMask', 'screenshotFullscreen', 'screenshotScale', 'screenshotNoStabilize', @@ -105,7 +115,8 @@ type ScreenshotSpecificFlagKey = (typeof SCREENSHOT_ACTION_FLAG_KEYS)[number]; type ScreenshotSpecificFlagDefinition = { key: ScreenshotSpecificFlagKey; names: readonly string[]; - type: 'boolean' | 'int' | 'number' | 'string'; + type: 'boolean' | 'int' | 'number' | 'string' | 'enum'; + enumValues?: readonly string[]; min?: number; max?: number; usageLabel: string; @@ -136,6 +147,17 @@ export const SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS: readonly ScreenshotSpecificFl projectConfig: true, recorded: true, }, + { + key: 'screenshotScreenMask', + names: ['--screen-mask'], + type: 'enum', + enumValues: SCREENSHOT_SCREEN_MASKS, + usageLabel: '--screen-mask ', + usageDescription: + 'Screenshot or live screenshot diff: fill the iOS simulator screen outline with black; requires simctl capture', + projectConfig: false, + recorded: true, + }, { key: 'screenshotFullscreen', names: ['--fullscreen', '--full', '-f'], @@ -217,6 +239,7 @@ export type ScreenshotRequestFlags = { overlayRefs?: boolean; screenshotCropOn?: string; screenshotPixelDensity?: number; + screenshotScreenMask?: ScreenshotScreenMask; screenshotFullscreen?: boolean; screenshotScale?: number; screenshotNoStabilize?: boolean; @@ -226,6 +249,7 @@ export type ScreenshotRequestFlags = { export type ScreenshotDispatchFlags = Pick< ScreenshotRequestFlags, | 'screenshotPixelDensity' + | 'screenshotScreenMask' | 'screenshotFullscreen' | 'screenshotNoStabilize' | 'screenshotNormalizeStatusBar' @@ -235,6 +259,7 @@ export type ScreenshotRuntimeFlags = Pick< ScreenshotRequestFlags, | 'screenshotCropOn' | 'screenshotPixelDensity' + | 'screenshotScreenMask' | 'screenshotFullscreen' | 'screenshotScale' | 'screenshotNoStabilize' @@ -245,6 +270,7 @@ export type ScreenshotPublicOptions = { overlayRefs?: boolean; cropOn?: string; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; fullscreen?: boolean; scale?: number; stabilize?: boolean; @@ -255,6 +281,7 @@ export type ScreenshotRuntimeOptions = { overlayRefs?: boolean; cropOn?: string; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; fullscreen?: boolean; scale?: number; stabilize?: boolean; @@ -268,6 +295,7 @@ export function screenshotOptionsFromFlags( overlayRefs: flags?.overlayRefs, cropOn: flags?.screenshotCropOn, pixelDensity: flags?.screenshotPixelDensity, + screenMask: readScreenshotScreenMask(flags?.screenshotScreenMask), fullscreen: flags?.screenshotFullscreen, scale: flags?.screenshotScale, stabilize: @@ -283,6 +311,9 @@ export function screenshotFlagsFromOptions( overlayRefs: options.overlayRefs, screenshotCropOn: options.screenshotCropOn ?? options.cropOn, screenshotPixelDensity: options.screenshotPixelDensity ?? options.pixelDensity, + screenshotScreenMask: readScreenshotScreenMask( + options.screenshotScreenMask ?? options.screenMask, + ), screenshotFullscreen: options.screenshotFullscreen ?? options.fullscreen, screenshotScale: options.screenshotScale, screenshotNoStabilize: @@ -324,6 +355,9 @@ export function appendScreenshotScriptFlags( if (typeof flags?.screenshotPixelDensity === 'number') { parts.push('--pixel-density', String(flags.screenshotPixelDensity)); } + if (flags?.screenshotScreenMask !== undefined) { + parts.push('--screen-mask', readScreenshotScreenMask(flags.screenshotScreenMask)!); + } if (flags?.screenshotFullscreen) parts.push('--fullscreen'); if (typeof flags?.screenshotScale === 'number') { parts.push('--scale', String(flags.screenshotScale)); @@ -340,6 +374,12 @@ export function readScreenshotScriptFlag(params: { }): { handled: true; nextIndex: number } | { handled: false } { const { args, flags, index } = params; const token = args[index]; + if (token === '--screen-mask') { + const value = args[index + 1]; + if (value === undefined) throw new AppError('INVALID_ARGS', '--screen-mask requires black'); + flags.screenshotScreenMask = readScreenshotScreenMask(value); + return { handled: true, nextIndex: index + 1 }; + } if (token === RETIRED_SCREENSHOT_MAX_SIZE.cliToken) { throw new AppError('INVALID_ARGS', RETIRED_SCREENSHOT_MAX_SIZE.migration.screenshot); } diff --git a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts index 7c1d97abdb..db64341a94 100644 --- a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts +++ b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts @@ -479,3 +479,46 @@ for (const backend of ['coredevice', 'xctest', undefined] as const) { }); } } + +test('a request screen mask reaches the bound simulator capture without changing later requests', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('ios-mask', { device: IOS_SIMULATOR }); + const outPath = path.join(mkdtempForTestSync('screen-mask-request'), 'screen.png'); + for (const screenMask of ['black', undefined] as const) { + const req = screenshotRequest({ + positionals: [outPath], + flags: screenMask ? { screenshotScreenMask: screenMask } : {}, + }); + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + await executeResult(resolved, session, req, req.flags); + expect(fixture.captureScreenshot.mock.lastCall?.[0].options?.screenMask).toBe(screenMask); + } +}); + +test.each([ANDROID_EMULATOR, IOS_DEVICE])( + 'refuses a screen mask on $kind $platform before binding', + async (device) => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('unsupported-mask', { device }); + await expect( + resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req: screenshotRequest({ flags: { screenshotScreenMask: 'black' } }), + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }), + ).rejects.toMatchObject({ code: 'UNSUPPORTED_OPERATION' }); + expect(fixture.binds).toHaveLength(0); + }, +); diff --git a/packages/daemon/src/execution/screenshot-runtime.ts b/packages/daemon/src/execution/screenshot-runtime.ts index 1df252ff60..b95653eb75 100644 --- a/packages/daemon/src/execution/screenshot-runtime.ts +++ b/packages/daemon/src/execution/screenshot-runtime.ts @@ -3,6 +3,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import { retiredScreenshotMaxSizeFlagError, screenshotFlagsFromOptions, + readScreenshotScreenMask, screenshotOptionsFromFlags, } from '@agent-device/contracts/capture'; import type { DeviceRotation } from '@agent-device/contracts/device'; @@ -65,6 +66,17 @@ export async function resolveScreenshotGenericExecution( if (retiredMaxSize) throw new AppError('INVALID_ARGS', retiredMaxSize); assertSupportedScreenshotPixelDensity(session.device, req.flags?.screenshotPixelDensity); + const screenMask = readScreenshotScreenMask(req.flags?.screenshotScreenMask); + if ( + screenMask !== undefined && + (!isIosFamily(session.device) || session.device.kind !== 'simulator') + ) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + '--screen-mask requires an iOS simulator simctl capture', + ); + } + const request = readScreenshotRequest(req, params.hostCwd); const cropOn = typeof req.flags?.screenshotCropOn === 'string' && req.flags.screenshotCropOn.length > 0 @@ -362,6 +374,7 @@ function createBoundScreenshotBackend( options: { appBundleId: dispatchContext.appBundleId, pixelDensity: resolved.pixelDensity, + screenMask: resolved.screenMask, fullscreen: resolved.fullscreen, normalizeStatusBar: resolved.normalizeStatusBar, stabilize: resolved.stabilize, diff --git a/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts b/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts index 9f7d7c6939..61bdd7fa7c 100644 --- a/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts @@ -56,3 +56,17 @@ test('routes an internal runner screenshot without changing the default backend' process.env, ); }); + +test('forwards the request mask and refuses an explicit runner capture with it', async () => { + const interactor = createAppleInteractor(device, {}, { env: {}, cwd: process.cwd() }); + await interactor.screenshot('/tmp/masked.png', { screenMask: 'black' }); + expect(screenshotIos).toHaveBeenCalledWith( + device, + '/tmp/masked.png', + expect.objectContaining({ screenMask: 'black' }), + ); + await expect( + interactor.screenshot('/tmp/runner.png', { screenMask: 'black', captureBackend: 'runner' }), + ).rejects.toMatchObject({ code: 'UNSUPPORTED_OPERATION' }); + expect(captureScreenshotViaRunner).not.toHaveBeenCalled(); +}); diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index 85a4a723b6..b2b0c612d0 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -511,7 +511,7 @@ describe('captureSimulatorScreenshotWithRetry display targeting', () => { await captureSimulatorScreenshotWithRetry( IOS_TEST_SIMULATOR, '/tmp/duo.png', - { AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK: 'black' }, + 'black', outerPanel, ); assert.equal(mockRunSimctlForDevice.mock.calls.length, 1); @@ -526,7 +526,7 @@ describe('captureSimulatorScreenshotWithRetry display targeting', () => { }); test('omits the display flag for a single-panel device', async () => { - await captureSimulatorScreenshotWithRetry(IOS_TEST_SIMULATOR, '/tmp/duo.png', {}); + await captureSimulatorScreenshotWithRetry(IOS_TEST_SIMULATOR, '/tmp/duo.png', undefined); assert.deepEqual(mockRunSimctlForDevice.mock.calls[0]![1], [ 'io', IOS_TEST_SIMULATOR.id, diff --git a/packages/platform-apple/src/core/__tests__/screenshot-simulator-capture.test.ts b/packages/platform-apple/src/core/__tests__/screenshot-simulator-capture.test.ts index 661f9b2066..f6cafa44e3 100644 --- a/packages/platform-apple/src/core/__tests__/screenshot-simulator-capture.test.ts +++ b/packages/platform-apple/src/core/__tests__/screenshot-simulator-capture.test.ts @@ -107,7 +107,8 @@ test('screenshotIos retries simulator capture timeouts and eventually succeeds', }, async ({ calls }) => { await screenshotIos(IOS_TEST_SIMULATOR, outPath, { - env: { AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK: 'black' }, + env: {}, + screenMask: 'black', }); const png = PNG.sync.read(await fs.readFile(outPath)); assert.equal(png.width, 42); @@ -129,10 +130,10 @@ test('screenshotIos retries simulator capture timeouts and eventually succeeds', ); }); -test.each([undefined, '', 'alpha', 'BLACK', 'black'])( - 'screenshotIos keeps density and opts into a black outline only for host setting %s', +test.each([undefined, 'black'] as const)( + 'screenshotIos keeps density and applies only the request screen mask %s', async (mask) => { - vi.stubEnv('AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK', mask === 'black' ? 'alpha' : 'black'); + vi.stubEnv('AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK', 'black'); const tmpDir = await mkdtempForTest('agent-device-ios-screenshot-density-'); const outPath = path.join(tmpDir, 'screen.png'); @@ -154,7 +155,8 @@ test.each([undefined, '', 'alpha', 'BLACK', 'black'])( }, async ({ calls }) => { await screenshotIos(IOS_TEST_SIMULATOR, outPath, { - env: { AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK: mask }, + env: process.env, + screenMask: mask, pixelDensity: 2, }); assert.deepEqual(calls.filter(isSimctlScreenshot), [ diff --git a/packages/platform-apple/src/core/__tests__/screenshot.test.ts b/packages/platform-apple/src/core/__tests__/screenshot.test.ts index bc559ef93c..0ec2ea9d07 100644 --- a/packages/platform-apple/src/core/__tests__/screenshot.test.ts +++ b/packages/platform-apple/src/core/__tests__/screenshot.test.ts @@ -857,3 +857,25 @@ test('captureSimulatorScreenshotWithFallback keeps the panel rotation for a runn }); assert.deepEqual(facts, { displayRotation: 'landscape-right' }); }); + +test('an explicit screen mask refuses an unmasked runner fallback after simctl exhaustion', async () => { + const failure = new AppError('COMMAND_FAILED', 'capture failed', { + stderr: 'Timeout waiting for screen surfaces', + }); + mockRetryWithPolicy.mockRejectedValue(failure); + await assert.rejects( + captureSimulatorScreenshotWithFallback(IOS_TEST_SIMULATOR, '/tmp/masked.png', { + env: {}, + screenMask: 'black', + skipIosSimulatorBootCheck: true, + deps: { + resolveCaptureTarget: async () => ({}), + captureWithRetry: captureSimulatorScreenshotWithRetry, + captureWithRunner: captureScreenshotViaRunner, + normalizeDensity: async () => {}, + }, + }), + (error) => error === failure, + ); + assert.equal(mockRunAppleRunnerCommand.mock.calls.length, 0); +}); diff --git a/packages/platform-apple/src/core/screenshot.ts b/packages/platform-apple/src/core/screenshot.ts index abab7d814b..a7d86494aa 100644 --- a/packages/platform-apple/src/core/screenshot.ts +++ b/packages/platform-apple/src/core/screenshot.ts @@ -1,5 +1,6 @@ import path from 'node:path'; import { isMacOs, type DeviceInfo } from '@agent-device/kernel/device'; +import type { ScreenshotScreenMask } from '@agent-device/contracts/capture'; import type { EnvMap } from '@agent-device/kernel/source-value'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { copyHostFile } from '@agent-device/host-kit/host-file'; @@ -66,7 +67,7 @@ type SimulatorScreenshotFlowDeps = { captureWithRetry: ( device: DeviceInfo, outPath: string, - env: Readonly, + screenMask: ScreenshotScreenMask | undefined, display: AppleDeviceDisplay | undefined, ) => Promise; /** @@ -94,11 +95,12 @@ type SimulatorScreenshotFlowDeps = { }; type SimulatorScreenshotFlowOptions = { - /** The calling runtime's host environment for simulator capture and runner-container configuration. */ + /** The calling runtime's host environment; runner-container configuration is read from it. */ env: Readonly; appBundleId?: string; fullscreen?: boolean; pixelDensity?: number; + screenMask?: ScreenshotScreenMask; normalizeStatusBar?: boolean; runnerOptions?: AppleRunnerCommandOptions; skipIosSimulatorBootCheck?: boolean; @@ -170,7 +172,7 @@ export async function captureSimulatorScreenshotWithFallback( const { display, displayRotation } = await deps.resolveCaptureTarget(device); const captureFacts: ScreenshotCaptureFacts = displayRotation ? { displayRotation } : {}; const captureAndNormalize = async () => { - await deps.captureWithRetry(device, outPath, options.env, display); + await deps.captureWithRetry(device, outPath, options.screenMask, display); await deps.normalizeDensity(device, outPath, options.pixelDensity, display?.pointScale); }; const captureWithFallback = async (): Promise => { @@ -191,7 +193,7 @@ export async function captureSimulatorScreenshotWithFallback( screenshotError = retryError; } } - if (!deps.shouldFallbackToRunner(screenshotError)) { + if (options.screenMask !== undefined || !deps.shouldFallbackToRunner(screenshotError)) { throw screenshotError; } emitScreenshotFallbackDiagnostic(device, 'simctl_screenshot', screenshotError); @@ -220,7 +222,7 @@ export async function captureSimulatorScreenshotWithFallback( export async function captureSimulatorScreenshotWithRetry( device: DeviceInfo, outPath: string, - env: Readonly, + screenMask: ScreenshotScreenMask | undefined, display?: AppleDeviceDisplay, ): Promise { const deadline = Deadline.fromTimeoutMs(IOS_SIMULATOR_SCREENSHOT_TIMEOUT_MS); @@ -228,7 +230,7 @@ export async function captureSimulatorScreenshotWithRetry( 'io', device.id, 'screenshot', - ...(env.AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK === 'black' ? ['--mask=black'] : []), + ...(screenMask === 'black' ? ['--mask=black'] : []), ...appleSimulatorDisplayArgvFragment(display), outPath, ]; diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index 87b3e40013..fdfa1169ed 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -493,6 +493,12 @@ async function runAppleScreenshot( helper: MacOsHelperSurface | undefined, env: Readonly, ): Promise { + if (options.screenMask !== undefined && (helper || options.captureBackend === 'runner')) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + '--screen-mask requires simctl capture, not an XCTest runner or macOS capture', + ); + } if (helper) { if (options.fullscreen) { throw new AppError( @@ -530,6 +536,7 @@ async function runAppleScreenshot( env, appBundleId: options.appBundleId, pixelDensity: options.pixelDensity, + screenMask: options.screenMask, fullscreen: options.fullscreen, runnerOptions: runnerOpts, normalizeStatusBar: options.normalizeStatusBar, diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 87b87d92bc..7bc99ffd24 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -380,7 +380,7 @@ function summarizeProviderScenarioFlagExclusions() { { name: 'Apple simulator screenshot rendering options', owner: 'iOS platform and screenshot-diff runtime tests', - keys: ['screenshotNormalizeStatusBar', 'screenshotPixelDensity'], + keys: ['screenshotNormalizeStatusBar', 'screenshotPixelDensity', 'screenshotScreenMask'], }, { // Reading accessibility custom actions has no provider-scenario surface: diff --git a/src/cli/commands/__tests__/screenshot.test.ts b/src/cli/commands/__tests__/screenshot.test.ts index b54f97a429..626b916b45 100644 --- a/src/cli/commands/__tests__/screenshot.test.ts +++ b/src/cli/commands/__tests__/screenshot.test.ts @@ -118,6 +118,7 @@ test.each([false, true])( { transport: async (req): Promise => { assert.equal(req.command, 'screenshot'); + assert.equal(req.flags?.screenshotScreenMask, 'black'); captures += 1; const out = req.positionals?.[0]; assert.equal(typeof out, 'string'); @@ -130,7 +131,13 @@ test.each([false, true])( diffCommand({ host: createTestHostEnvironment(), positionals: ['screenshot', ...(live ? [] : [current])], - flags: { json: true, baseline, out: diff, threshold: '1' } as CliFlags, + flags: { + json: true, + baseline, + out: diff, + threshold: '1', + ...(live ? { screenshotScreenMask: 'black' } : {}), + } as CliFlags, client, }), ); @@ -144,3 +151,18 @@ test.each([false, true])( assert.deepEqual([image.width, image.height], [2, 3]); }, ); + +test('the Node screenshot client sends the mask in its daemon request', async () => { + const client = createAgentDeviceClient( + { session: 'qa' }, + { + transport: async (req): Promise => { + assert.equal(req.command, 'screenshot'); + assert.equal(req.flags?.screenshotScreenMask, 'black'); + return { ok: true, data: { path: '/tmp/masked.png' } }; + }, + }, + ); + const result = await client.capture.screenshot({ path: '/tmp/masked.png', screenMask: 'black' }); + assert.equal(result.path, '/tmp/masked.png'); +}); diff --git a/src/cli/commands/screenshot.ts b/src/cli/commands/screenshot.ts index b1afc33b70..ec43610e0e 100644 --- a/src/cli/commands/screenshot.ts +++ b/src/cli/commands/screenshot.ts @@ -102,6 +102,7 @@ export const diffCommand: ClientCommandHandler = async ({ positionals, flags, cl threshold: parseCliThreshold(flags.threshold), overlayRefs: flags.overlayRefs, normalizeStatusBar: flags.screenshotNormalizeStatusBar, + screenMask: flags.screenshotScreenMask, surface: flags.surface, }); @@ -125,6 +126,7 @@ function createClientScreenshotBackend( pixelDensity: options?.pixelDensity, fullscreen: options?.fullscreen, normalizeStatusBar: options?.normalizeStatusBar, + screenMask: options?.screenMask, stabilize: options?.stabilize, surface: options?.surface, }); diff --git a/src/cli/parser/__tests__/args-parse-interaction.test.ts b/src/cli/parser/__tests__/args-parse-interaction.test.ts index f5cfbd58a4..0aa7872877 100644 --- a/src/cli/parser/__tests__/args-parse-interaction.test.ts +++ b/src/cli/parser/__tests__/args-parse-interaction.test.ts @@ -256,6 +256,8 @@ test('parseArgs recognizes screenshot flags', () => { 'page.png', '--pixel-density', '2', + '--screen-mask', + 'black', '--full', '-f', '--fullscreen', @@ -271,6 +273,7 @@ test('parseArgs recognizes screenshot flags', () => { assert.equal(parsed.command, 'screenshot'); assert.deepEqual(parsed.positionals, ['page.png']); assert.equal(parsed.flags.screenshotPixelDensity, 2); + assert.equal(parsed.flags.screenshotScreenMask, 'black'); assert.equal(parsed.flags.screenshotFullscreen, true); assert.equal(parsed.flags.screenshotScale, 0.3); assert.equal(parsed.flags.screenshotNoStabilize, true); @@ -445,3 +448,19 @@ test('parseArgs maps scroll until and max passes', () => { assert.equal(parsed.flags.until, 'id="row-512"'); assert.equal(parsed.flags.maxPasses, 90); }); + +test('screen-mask is accepted for live screenshot diffs and refuses unknown values', () => { + const parsed = parseArgs( + ['diff', 'screenshot', '--baseline', 'base.png', '--screen-mask', 'black'], + { strictFlags: true }, + ); + assert.equal(parsed.flags.screenshotScreenMask, 'black'); + assert.throws(() => parseArgs(['screenshot', '--screen-mask', 'alpha'], { strictFlags: true })); +}); + +test('screen-mask is refused for diff snapshot', () => { + assert.throws( + () => parseArgs(['diff', 'snapshot', '--screen-mask', 'black'], { strictFlags: true }), + { code: 'INVALID_ARGS' }, + ); +}); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 4c72224993..a91c89af9a 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1022,7 +1022,7 @@ agent-device screenshot page.png # Explicit screenshot path agent-device screenshot page.png --scale 0.3 # Resize both dimensions to 30% for agent context agent-device screenshot page.png --overlay-refs # Draw current @eN refs and target rectangles onto the PNG agent-device screenshot page.png --crop-on 'label="Save"' # Crop the capture to the frame the selector resolves on the same screen -agent-device screenshot baseline.png --normalize-status-bar # Normalize iOS simulator or ADB-backed Android chrome for reusable diff baselines +agent-device screenshot baseline.png --screen-mask black --normalize-status-bar # Normalize iOS simulator or ADB-backed Android chrome for reusable diff baselines agent-device screenshot page.png --platform web --fullscreen # On web, --fullscreen/--full/-f captures the entire document agent-device viewport 1280 900 --platform web # Resize the active web viewport for fixed-layout or 100vh apps agent-device screenshot textedit.png # App-session window capture on macOS @@ -1048,7 +1048,7 @@ agent-device record stop # Stop active recording - `--crop-on ` crops the capture to the frame the selector resolves on the same screen. It works only on iOS simulators, Android emulators, and physical iPhones. A physical iPhone crop is verified only in portrait using CoreDevice/devicectl; the XCTest backend and runner fallback are not accepted. Physical iPads remain rejected for crop, and landscape iPhone crop remains unverified. It needs exactly one framed node, cannot be combined with `--overlay-refs` or `--fullscreen`, and warns when the frame is clipped. The crop is re-encoded, so do not byte-compare it with an older crop. - `--fullscreen` on macOS applies only to app sessions. Other surfaces (`desktop`, `menubar`, `frontmost-app`) always capture the main display and refuse `--fullscreen` with `INVALID_ARGS`. - `--normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for reusable baselines. Each status-bar read or preparation command has a 10-second timeout. Each restore command has a 30-second timeout, so clearing and reapplying overrides can take up to 60 seconds. The request timeout includes the wait and can expire before these steps finish. -- Set `AGENT_DEVICE_IOS_SIMULATOR_SCREENSHOT_MASK=black` in the daemon host environment before starting the daemon to fill the simulator screen outline, including rounded corners, with black in `simctl` captures. Restart an existing daemon for the setting to take effect. Unset or other values keep the default capture. This does not redact app content, remove a Dynamic Island drawn into the framebuffer, or change comparison thresholds. Physical-device and XCTest runner captures, including fallback captures, are unaffected. +- `--screen-mask black` fills the iOS simulator screen outline, including rounded corners, with black for this capture. Node and MCP use `screenMask: "black"`. Omit it for the default image. It requires simctl capture; physical devices, other platforms, provider captures, and XCTest runner captures are refused. If simctl fails, a masked request fails instead of falling back to an unmasked runner capture. This option does not redact app content, remove a Dynamic Island drawn into the framebuffer, or change comparison thresholds. - On an iOS simulator, a screenshot waits until no other screenshot or `settings wifi` / `settings airplane` write is using that simulator's status bar, so overlapping captures cannot see each other's temporary chrome. A request you cancel while it is waiting fails with the cancellation instead of capturing. - On single-display iOS simulators and Android devices, the result may report `displayRotation` (`portrait`, `portrait-upside-down`, `landscape-left`, or `landscape-right`). It is absent when the device returns no reading in time, on other targets, and on multi-display devices such as foldables. The saved image is always upright. @@ -1059,7 +1059,7 @@ agent-device record stop # Stop active recording - The current image scales proportionally to the baseline width and aligns at the top left. Its scaled height rounds to the nearest pixel, with a minimum of one row. The comparison includes the full height of both images: extra or missing bottom rows count as differences, including blank or transparent rows, even at `--threshold 1`. Your original files stay unchanged. This does not crop system bars or move content into alignment. - When dimensions differ, JSON `normalization` reports `baseline`, `current`, `scaledCurrent`, and `compared` dimensions. `currentScale.x` and `.y` map original current pixels to comparison pixels; the height scale includes pixel rounding. Region rectangles and ref-match rectangles use comparison pixels, and normalized region rectangles use percentages of the compared canvas. The separate current-screen overlay keeps its original capture size. - `--overlay-refs` also writes a separate current-screen ref overlay and lists refs under changed regions. It needs a live capture, so it is unavailable with a saved `current.png`. -- Live iOS simulator diffs normalize status-bar chrome by default. +- Live iOS simulator diffs normalize status-bar chrome by default. Use `diff screenshot --baseline baseline.png --screen-mask black` to apply the same screen outline as a baseline captured with `screenshot --screen-mask black`. The mask affects only this live capture, including its optional ref overlay. It is refused with a saved current image or `diff snapshot`; saved-image comparisons do not apply a mask. **Recordings** From e3f5b750b7c9a6682c4645912321ce94460c2f81 Mon Sep 17 00:00:00 2001 From: Anthony Mittaz Date: Wed, 4 Nov 2026 12:03:16 +1100 Subject: [PATCH 3/7] refactor(screenshot): group live diff option validation --- .../capture/runtime/diff-screenshot.ts | 35 +++++++++++-------- website/docs/docs/commands.md | 3 +- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.ts b/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.ts index e9f4940c32..3103458e9a 100644 --- a/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/diff-screenshot.ts @@ -59,22 +59,9 @@ export const diffScreenshotCommand: RuntimeCommand< throw new AppError('INVALID_ARGS', 'diff screenshot requires a baseline image'); } - readScreenshotScreenMask(options.screenMask); const threshold = normalizeThreshold(options.threshold); const currentRef = options.current ?? { kind: 'live' }; - if (options.overlayRefs && !isLiveCurrentRef(currentRef)) { - throw new AppError( - 'INVALID_ARGS', - 'diff screenshot cannot use --overlay-refs because saved-image comparisons have no live accessibility refs', - ); - } - - if (options.screenMask !== undefined && !isLiveCurrentRef(currentRef)) { - throw new AppError( - 'INVALID_ARGS', - '--screen-mask requires a live screenshot, not a saved current image', - ); - } + validateLiveCaptureOptions(options, currentRef); const baseline = await resolveCommandInput(runtime, options.baseline, { usage: 'diff screenshot baseline', @@ -134,6 +121,26 @@ export const diffScreenshotCommand: RuntimeCommand< } }; +function validateLiveCaptureOptions( + options: DiffScreenshotCommandOptions, + currentRef: FileInputRef | LiveScreenshotInputRef, +): void { + readScreenshotScreenMask(options.screenMask); + if (isLiveCurrentRef(currentRef)) return; + if (options.overlayRefs) { + throw new AppError( + 'INVALID_ARGS', + 'diff screenshot cannot use --overlay-refs because saved-image comparisons have no live accessibility refs', + ); + } + if (options.screenMask !== undefined) { + throw new AppError( + 'INVALID_ARGS', + '--screen-mask requires a live screenshot, not a saved current image', + ); + } +} + function normalizeThreshold(threshold: unknown): number { if (threshold == null || threshold === '') return DEFAULT_SCREENSHOT_DIFF_THRESHOLD; const value = Number(threshold); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index a91c89af9a..1090992b6b 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1022,7 +1022,8 @@ agent-device screenshot page.png # Explicit screenshot path agent-device screenshot page.png --scale 0.3 # Resize both dimensions to 30% for agent context agent-device screenshot page.png --overlay-refs # Draw current @eN refs and target rectangles onto the PNG agent-device screenshot page.png --crop-on 'label="Save"' # Crop the capture to the frame the selector resolves on the same screen -agent-device screenshot baseline.png --screen-mask black --normalize-status-bar # Normalize iOS simulator or ADB-backed Android chrome for reusable diff baselines +agent-device screenshot baseline.png --normalize-status-bar # Normalize iOS simulator or ADB-backed Android chrome for reusable diff baselines +agent-device screenshot baseline.png --screen-mask black --normalize-status-bar # Capture an iOS simulator baseline with a black screen outline agent-device screenshot page.png --platform web --fullscreen # On web, --fullscreen/--full/-f captures the entire document agent-device viewport 1280 900 --platform web # Resize the active web viewport for fixed-layout or 100vh apps agent-device screenshot textedit.png # App-session window capture on macOS From 0bbe8a6f1148daa7bb743c85d739c2c273d6946b Mon Sep 17 00:00:00 2001 From: Anthony Mittaz Date: Wed, 4 Nov 2026 12:09:04 +1100 Subject: [PATCH 4/7] docs(screenshot): describe the screen mask tool input --- packages/command-surface/src/capture/screenshot.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/command-surface/src/capture/screenshot.ts b/packages/command-surface/src/capture/screenshot.ts index 13d0debb18..eeb7ae8288 100644 --- a/packages/command-surface/src/capture/screenshot.ts +++ b/packages/command-surface/src/capture/screenshot.ts @@ -41,7 +41,10 @@ const screenshotCommandMetadata = defineFieldCommandMetadata( pixelDensity: integerField('Output screenshot pixel density in pixels per logical point.', { min: 1, }), - screenMask: enumField(SCREENSHOT_SCREEN_MASKS), + screenMask: enumField( + SCREENSHOT_SCREEN_MASKS, + 'Fill the iOS simulator screen outline with black for this capture. Requires simctl; omit for the default image.', + ), fullscreen: booleanField(), scale: numberField('Screenshot scale factor.', SCREENSHOT_SCALE_LIMITS), maxSize: retiredField(RETIRED_SCREENSHOT_MAX_SIZE.migration.screenshot), From 79104333c6e731fb61b54241774c6fee7f492f7b Mon Sep 17 00:00:00 2001 From: Anthony Mittaz Date: Wed, 4 Nov 2026 14:03:10 +1100 Subject: [PATCH 5/7] fix(screenshot): enforce mask support at the capture owner --- .../command-surface/src/capture/screenshot.ts | 2 +- .../contracts/src/screenshot-runtime.test.ts | 43 +++++++++--- packages/contracts/src/screenshot-runtime.ts | 69 ++++++++++++------- .../screenshot-runtime-execution.test.ts | 20 ------ .../src/execution/screenshot-runtime.ts | 12 ---- .../interactor-screenshot-backend.test.ts | 15 ++++ .../src/core/__tests__/screenshot.test.ts | 18 +++++ .../platform-apple/src/core/screenshot.ts | 4 ++ packages/platform-apple/src/interactor.ts | 9 +-- packages/platform-apple/src/runtime.ts | 3 + .../src/screenshot-support.test.ts | 59 ++++++++++++++++ .../platform-apple/src/screenshot-support.ts | 14 ++++ website/docs/docs/commands.md | 2 +- 13 files changed, 195 insertions(+), 75 deletions(-) create mode 100644 packages/platform-apple/src/screenshot-support.test.ts create mode 100644 packages/platform-apple/src/screenshot-support.ts diff --git a/packages/command-surface/src/capture/screenshot.ts b/packages/command-surface/src/capture/screenshot.ts index eeb7ae8288..df08b13bc4 100644 --- a/packages/command-surface/src/capture/screenshot.ts +++ b/packages/command-surface/src/capture/screenshot.ts @@ -79,7 +79,7 @@ export const screenshotCommandFacet = defineCommandFacet({ text: { summary: 'Capture a screenshot', cliDetail: - "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. Use --screen-mask black to fill the iOS simulator screen outline, including rounded corners, with black for this capture. It requires simctl: physical devices, other platforms, provider captures and XCTest runner captures are refused. A masked capture fails if simctl fails instead of falling back to an unmasked runner capture. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.", + "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. Use --screen-mask black to fill the iOS simulator screen outline, including rounded corners, with black for this capture. It requires simctl: physical devices, tvOS and visionOS simulators, other platforms, provider captures and XCTest runner captures are refused. A masked capture fails if simctl fails instead of falling back to an unmasked runner capture. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.", }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input), diff --git a/packages/contracts/src/screenshot-runtime.test.ts b/packages/contracts/src/screenshot-runtime.test.ts index bf1e1ee754..89a6b2f857 100644 --- a/packages/contracts/src/screenshot-runtime.test.ts +++ b/packages/contracts/src/screenshot-runtime.test.ts @@ -55,15 +55,40 @@ test('a provider binding fails closed when its exact owner exposes no interactor }); }); -test('a provider capture refuses a simctl screen mask before calling its interactor', async () => { - const resolveInteractor = vi.fn(); - const operations = bindProviderScreenshotInteractor({ - device: { ...device, platform: 'apple', appleOs: 'ios', kind: 'simulator' }, +test.each([bindLocalScreenshotInteractor, bindProviderScreenshotInteractor])( + 'a capture without a declared mask capability refuses before resolving its interactor', + async (bind) => { + const resolveInteractor = vi.fn(); + const operations = bind({ + device: { ...device, platform: 'apple', appleOs: 'ios', kind: 'simulator' }, + signal: new AbortController().signal, + resolveInteractor, + }); + await expect( + operations.captureScreenshot({ + outPath: '/tmp/masked.png', + options: { screenMask: 'black' }, + }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-screen-mask-unsupported' }, + }); + expect(resolveInteractor).not.toHaveBeenCalled(); + }, +); + +test('the selected capture capability admits a mask for one invocation without changing the next', async () => { + const screenshot = vi.fn(async () => ({ displayRotation: 'portrait' as const })); + const operations = bindLocalScreenshotInteractor({ + device, signal: new AbortController().signal, - resolveInteractor, + resolveInteractor: async () => ({ screenshot }) as unknown as Interactor, + captureCapabilities: () => ({ screenMask: true }), }); - await expect( - operations.captureScreenshot({ outPath: '/tmp/masked.png', options: { screenMask: 'black' } }), - ).rejects.toMatchObject({ code: 'UNSUPPORTED_OPERATION' }); - expect(resolveInteractor).not.toHaveBeenCalled(); + for (const screenMask of ['black', undefined] as const) { + await expect( + operations.captureScreenshot({ outPath: '/tmp/capture.png', options: { screenMask } }), + ).resolves.toEqual({ displayRotation: 'portrait' }); + expect(screenshot.mock.lastCall).toEqual(['/tmp/capture.png', { screenMask }]); + } }); diff --git a/packages/contracts/src/screenshot-runtime.ts b/packages/contracts/src/screenshot-runtime.ts index 337c5960b4..e8b330c2a9 100644 --- a/packages/contracts/src/screenshot-runtime.ts +++ b/packages/contracts/src/screenshot-runtime.ts @@ -1,4 +1,5 @@ import { AppError } from '@agent-device/kernel/errors'; +import { readScreenshotScreenMask } from './screenshot.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { localInteractorSource, @@ -34,6 +35,29 @@ export type ScreenshotRuntimeOperations = Readonly<{ captureScreenshot(input: CaptureScreenshotInput): Promise; }>; +export type ScreenshotCaptureCapabilities = Readonly<{ + screenMask?: boolean; +}>; + +type ScreenshotCaptureBindingOptions = Readonly<{ + signal: AbortSignal; + captureCapabilities?: (options: Readonly) => ScreenshotCaptureCapabilities; +}>; + +export function assertScreenshotCaptureSupport( + options: Readonly | undefined, + capabilities: ScreenshotCaptureCapabilities = {}, +): void { + const screenMask = readScreenshotScreenMask(options?.screenMask); + if (screenMask !== undefined && capabilities.screenMask !== true) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + '--screen-mask requires a handheld iOS simulator simctl capture', + { reason: 'screenshot-screen-mask-unsupported' }, + ); + } +} + export type ScreenshotRuntimeOperationFacts = Readonly<{ captureScreenshot: RuntimeOperationFact; }>; @@ -51,15 +75,19 @@ export function screenshotRuntimeOperationFacts( * resolution and this holds only what both share: the runner context and the capture itself. */ function bindScreenshotCapture( - signal: AbortSignal, + binding: ScreenshotCaptureBindingOptions, resolveInteractor: (runner: RunnerContext) => Promise, ): ScreenshotRuntimeOperations { return Object.freeze({ captureScreenshot: async (input: CaptureScreenshotInput) => { + assertScreenshotCaptureSupport( + input.options, + binding.captureCapabilities?.(input.options ?? {}), + ); const interactor = await resolveInteractor({ ...input.execution, appBundleId: input.options?.appBundleId, - signal, + signal: binding.signal, }); return (await interactor.screenshot(input.outPath, input.options)) ?? {}; }, @@ -67,36 +95,25 @@ function bindScreenshotCapture( } export function bindLocalScreenshotInteractor( - params: Readonly<{ - device: DeviceInfo; - signal: AbortSignal; - resolveInteractor: LocalInteractorOperationResolver; - }>, + params: ScreenshotCaptureBindingOptions & + Readonly<{ + device: DeviceInfo; + resolveInteractor: LocalInteractorOperationResolver; + }>, ): ScreenshotRuntimeOperations { - return bindScreenshotCapture(params.signal, localInteractorSource(params)); + return bindScreenshotCapture(params, localInteractorSource(params)); } /** Provider bindings fail closed when their exact owner no longer exposes its interactor. */ export function bindProviderScreenshotInteractor( - params: Readonly<{ - device: DeviceInfo; - signal: AbortSignal; - resolveInteractor: ProviderInteractorOperationResolver; - }>, + params: ScreenshotCaptureBindingOptions & + Readonly<{ + device: DeviceInfo; + resolveInteractor: ProviderInteractorOperationResolver; + }>, ): ScreenshotRuntimeOperations { - const capture = bindScreenshotCapture( - params.signal, + return bindScreenshotCapture( + params, providerInteractorSource({ ...params, operation: 'screenshot' }), ); - return Object.freeze({ - captureScreenshot: async (input: CaptureScreenshotInput) => { - if (input.options?.screenMask !== undefined) { - throw new AppError( - 'UNSUPPORTED_OPERATION', - '--screen-mask requires local simctl capture, not a provider capture', - ); - } - return await capture.captureScreenshot(input); - }, - }); } diff --git a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts index db64341a94..2866b2bd12 100644 --- a/packages/daemon/src/execution/screenshot-runtime-execution.test.ts +++ b/packages/daemon/src/execution/screenshot-runtime-execution.test.ts @@ -502,23 +502,3 @@ test('a request screen mask reaches the bound simulator capture without changing expect(fixture.captureScreenshot.mock.lastCall?.[0].options?.screenMask).toBe(screenMask); } }); - -test.each([ANDROID_EMULATOR, IOS_DEVICE])( - 'refuses a screen mask on $kind $platform before binding', - async (device) => { - const fixture = screenshotRuntimeFixture(); - const session = makeSession('unsupported-mask', { device }); - await expect( - resolveScreenshotGenericExecution({ - hostCwd: process.cwd(), - createCommandSurface: testCreateCommandSurface, - platformServices: daemonPlatformServicesFixture(), - req: screenshotRequest({ flags: { screenshotScreenMask: 'black' } }), - session, - inspectFacts: fixture.inspectFacts, - bindDevice: fixture.bindDevice, - }), - ).rejects.toMatchObject({ code: 'UNSUPPORTED_OPERATION' }); - expect(fixture.binds).toHaveLength(0); - }, -); diff --git a/packages/daemon/src/execution/screenshot-runtime.ts b/packages/daemon/src/execution/screenshot-runtime.ts index b95653eb75..1b0196ab1c 100644 --- a/packages/daemon/src/execution/screenshot-runtime.ts +++ b/packages/daemon/src/execution/screenshot-runtime.ts @@ -3,7 +3,6 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import { retiredScreenshotMaxSizeFlagError, screenshotFlagsFromOptions, - readScreenshotScreenMask, screenshotOptionsFromFlags, } from '@agent-device/contracts/capture'; import type { DeviceRotation } from '@agent-device/contracts/device'; @@ -66,17 +65,6 @@ export async function resolveScreenshotGenericExecution( if (retiredMaxSize) throw new AppError('INVALID_ARGS', retiredMaxSize); assertSupportedScreenshotPixelDensity(session.device, req.flags?.screenshotPixelDensity); - const screenMask = readScreenshotScreenMask(req.flags?.screenshotScreenMask); - if ( - screenMask !== undefined && - (!isIosFamily(session.device) || session.device.kind !== 'simulator') - ) { - throw new AppError( - 'UNSUPPORTED_OPERATION', - '--screen-mask requires an iOS simulator simctl capture', - ); - } - const request = readScreenshotRequest(req, params.hostCwd); const cropOn = typeof req.flags?.screenshotCropOn === 'string' && req.flags.screenshotCropOn.length > 0 diff --git a/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts b/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts index 61bdd7fa7c..27bedde93e 100644 --- a/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts @@ -70,3 +70,18 @@ test('forwards the request mask and refuses an explicit runner capture with it', ).rejects.toMatchObject({ code: 'UNSUPPORTED_OPERATION' }); expect(captureScreenshotViaRunner).not.toHaveBeenCalled(); }); + +test.each([ + { ...device, kind: 'device' as const }, + { ...device, appleOs: 'macos' as const, kind: 'device' as const, target: 'desktop' as const }, +])('direct Apple $appleOs $kind captures refuse masks before native execution', async (target) => { + const interactor = createAppleInteractor(target, {}, { env: {}, cwd: process.cwd() }); + await expect( + interactor.screenshot('/tmp/masked.png', { screenMask: 'black' }), + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-screen-mask-unsupported' }, + }); + expect(screenshotIos).not.toHaveBeenCalled(); + expect(captureScreenshotViaRunner).not.toHaveBeenCalled(); +}); diff --git a/packages/platform-apple/src/core/__tests__/screenshot.test.ts b/packages/platform-apple/src/core/__tests__/screenshot.test.ts index 0ec2ea9d07..398eff074d 100644 --- a/packages/platform-apple/src/core/__tests__/screenshot.test.ts +++ b/packages/platform-apple/src/core/__tests__/screenshot.test.ts @@ -49,6 +49,7 @@ import { captureSimulatorScreenshotWithFallback, captureSimulatorScreenshotWithRetry, captureScreenshotViaRunner, + screenshotIos, resolveSimulatorRunnerScreenshotCandidatePaths, shouldRetryIosSimulatorScreenshot, } from '../screenshot.ts'; @@ -879,3 +880,20 @@ test('an explicit screen mask refuses an unmasked runner fallback after simctl e ); assert.equal(mockRunAppleRunnerCommand.mock.calls.length, 0); }); + +test.each([MACOS_TEST_DEVICE, { ...IOS_TEST_SIMULATOR, kind: 'device' as const }])( + 'screenshotIos refuses a masked $kind capture outside simctl before native execution', + async (device) => { + mockRunCmd.mockRejectedValue(new Error('unexpected native capture')); + mockRunAppleRunnerCommand.mockRejectedValue(new Error('unexpected runner capture')); + await assert.rejects( + screenshotIos(device, '/tmp/masked.png', { env: {}, screenMask: 'black' }), + { + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-screen-mask-unsupported' }, + }, + ); + assert.equal(mockRunCmd.mock.calls.length, 0); + assert.equal(mockRunAppleRunnerCommand.mock.calls.length, 0); + }, +); diff --git a/packages/platform-apple/src/core/screenshot.ts b/packages/platform-apple/src/core/screenshot.ts index a7d86494aa..d6dae705a8 100644 --- a/packages/platform-apple/src/core/screenshot.ts +++ b/packages/platform-apple/src/core/screenshot.ts @@ -1,6 +1,8 @@ import path from 'node:path'; import { isMacOs, type DeviceInfo } from '@agent-device/kernel/device'; import type { ScreenshotScreenMask } from '@agent-device/contracts/capture'; +import { assertScreenshotCaptureSupport } from '@agent-device/contracts/screenshot-runtime'; +import { appleScreenshotCaptureCapabilities } from '../screenshot-support.ts'; import type { EnvMap } from '@agent-device/kernel/source-value'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { copyHostFile } from '@agent-device/host-kit/host-file'; @@ -126,6 +128,7 @@ export async function screenshotIos( outPath: string, options: Omit, ): Promise { + assertScreenshotCaptureSupport(options, appleScreenshotCaptureCapabilities(device, {})); if (isMacOs(device)) { await captureScreenshotViaRunner( device, @@ -155,6 +158,7 @@ export async function captureSimulatorScreenshotWithFallback( outPath: string, options: SimulatorScreenshotFlowOptions, ): Promise { + assertScreenshotCaptureSupport(options, appleScreenshotCaptureCapabilities(device, {})); if (device.kind !== 'simulator') { throw new AppError( 'UNSUPPORTED_OPERATION', diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index fdfa1169ed..f5841b5254 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -1,3 +1,5 @@ +import { assertScreenshotCaptureSupport } from '@agent-device/contracts/screenshot-runtime'; +import { appleScreenshotCaptureCapabilities } from './screenshot-support.ts'; import { closeIosApp, openIosApp, openIosDevice } from './core/app-launch.ts'; import { readIosClipboardText, writeIosClipboardText } from './core/app-device-io.ts'; import { readIosSetting, setIosSetting } from './core/app-settings.ts'; @@ -493,12 +495,7 @@ async function runAppleScreenshot( helper: MacOsHelperSurface | undefined, env: Readonly, ): Promise { - if (options.screenMask !== undefined && (helper || options.captureBackend === 'runner')) { - throw new AppError( - 'UNSUPPORTED_OPERATION', - '--screen-mask requires simctl capture, not an XCTest runner or macOS capture', - ); - } + assertScreenshotCaptureSupport(options, appleScreenshotCaptureCapabilities(device, options)); if (helper) { if (options.fullscreen) { throw new AppError( diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index 7b3f8a67d5..c2caf245a9 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -8,6 +8,7 @@ import { import type { NetworkDumpInput } from '@agent-device/contracts/network-runtime'; import type { AppStateRuntimeOperations } from '@agent-device/contracts/app-state-runtime'; import { bindAppleAppStateRuntime } from './app-state-runtime.ts'; +import { appleScreenshotCaptureCapabilities } from './screenshot-support.ts'; import type { PlatformRuntimeHost, PlatformRuntimeOperations, @@ -409,6 +410,8 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR device: request.device, signal: request.scope.signal, resolveInteractor: host.localInteractors.resolve, + captureCapabilities: (options) => + appleScreenshotCaptureCapabilities(request.device, options), }), ), ...whenAdmitted(facts.operations.focusPoint, () => diff --git a/packages/platform-apple/src/screenshot-support.test.ts b/packages/platform-apple/src/screenshot-support.test.ts new file mode 100644 index 0000000000..6f87abb83d --- /dev/null +++ b/packages/platform-apple/src/screenshot-support.test.ts @@ -0,0 +1,59 @@ +import { expect, test, vi } from 'vitest'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { Interactor, ScreenshotOptions } from '@agent-device/contracts/interactor-types'; +import { createApplePlatformRuntime } from './runtime.ts'; +import { platformRuntimeHostFixture } from './runtime.fixtures.ts'; + +const simulator: DeviceInfo = { + platform: 'apple', + appleOs: 'ios', + id: 'sim-1', + name: 'iPhone', + kind: 'simulator', + booted: true, +}; + +test.each([ + [simulator, {}, true], + [{ ...simulator, appleOs: 'ipados' }, {}, true], + [simulator, { captureBackend: 'runner' }, false], + [{ ...simulator, kind: 'device' }, {}, false], + [{ ...simulator, appleOs: 'macos', kind: 'device', target: 'desktop' }, {}, false], + [{ ...simulator, appleOs: 'tvos', target: 'tv' }, {}, false], + [{ ...simulator, appleOs: 'visionos' }, {}, false], +] satisfies [DeviceInfo, Partial, boolean][])( + 'the bound Apple capture declares mask support for %j with %j: %s', + async (device, options, supported) => { + const screenshot = vi.fn(async () => ({ displayRotation: 'portrait' as const })); + const resolve = vi.fn(async () => ({ screenshot }) as unknown as Interactor); + const host = { ...platformRuntimeHostFixture(), localInteractors: { resolve } }; + const binding = await createApplePlatformRuntime(host).bind({ + device, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + const capture = binding.operations.captureScreenshot; + expect(capture).toBeDefined(); + const result = capture!({ + outPath: '/tmp/masked.png', + options: { ...options, screenMask: 'black' }, + }); + if (supported) { + await expect(result).resolves.toEqual({ displayRotation: 'portrait' }); + expect(screenshot).toHaveBeenCalledWith('/tmp/masked.png', { + ...options, + screenMask: 'black', + }); + } else { + await expect(result).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-screen-mask-unsupported' }, + }); + expect(resolve).not.toHaveBeenCalled(); + } + }, +); diff --git a/packages/platform-apple/src/screenshot-support.ts b/packages/platform-apple/src/screenshot-support.ts new file mode 100644 index 0000000000..858fcb8759 --- /dev/null +++ b/packages/platform-apple/src/screenshot-support.ts @@ -0,0 +1,14 @@ +import { isHandheldAppleSimulator, type DeviceInfo } from '@agent-device/kernel/device'; +import type { + ScreenshotCaptureCapabilities, + ScreenshotOptions, +} from '@agent-device/contracts/screenshot-runtime'; + +export function appleScreenshotCaptureCapabilities( + device: DeviceInfo, + options: Pick, +): ScreenshotCaptureCapabilities { + return { + screenMask: isHandheldAppleSimulator(device) && options.captureBackend !== 'runner', + }; +} diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 1090992b6b..a9fe2c7d16 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1049,7 +1049,7 @@ agent-device record stop # Stop active recording - `--crop-on ` crops the capture to the frame the selector resolves on the same screen. It works only on iOS simulators, Android emulators, and physical iPhones. A physical iPhone crop is verified only in portrait using CoreDevice/devicectl; the XCTest backend and runner fallback are not accepted. Physical iPads remain rejected for crop, and landscape iPhone crop remains unverified. It needs exactly one framed node, cannot be combined with `--overlay-refs` or `--fullscreen`, and warns when the frame is clipped. The crop is re-encoded, so do not byte-compare it with an older crop. - `--fullscreen` on macOS applies only to app sessions. Other surfaces (`desktop`, `menubar`, `frontmost-app`) always capture the main display and refuse `--fullscreen` with `INVALID_ARGS`. - `--normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for reusable baselines. Each status-bar read or preparation command has a 10-second timeout. Each restore command has a 30-second timeout, so clearing and reapplying overrides can take up to 60 seconds. The request timeout includes the wait and can expire before these steps finish. -- `--screen-mask black` fills the iOS simulator screen outline, including rounded corners, with black for this capture. Node and MCP use `screenMask: "black"`. Omit it for the default image. It requires simctl capture; physical devices, other platforms, provider captures, and XCTest runner captures are refused. If simctl fails, a masked request fails instead of falling back to an unmasked runner capture. This option does not redact app content, remove a Dynamic Island drawn into the framebuffer, or change comparison thresholds. +- `--screen-mask black` fills the iOS simulator screen outline, including rounded corners, with black for this capture. Node and MCP use `screenMask: "black"`. Omit it for the default image. Support is limited to handheld iOS/iPadOS simulators. tvOS and visionOS mask behavior has not been verified. It requires simctl capture; physical devices, tvOS and visionOS simulators, other platforms, provider captures, and XCTest runner captures are refused. If simctl fails, a masked request fails instead of falling back to an unmasked runner capture. This option does not redact app content, remove a Dynamic Island drawn into the framebuffer, or change comparison thresholds. - On an iOS simulator, a screenshot waits until no other screenshot or `settings wifi` / `settings airplane` write is using that simulator's status bar, so overlapping captures cannot see each other's temporary chrome. A request you cancel while it is waiting fails with the cancellation instead of capturing. - On single-display iOS simulators and Android devices, the result may report `displayRotation` (`portrait`, `portrait-upside-down`, `landscape-left`, or `landscape-right`). It is absent when the device returns no reading in time, on other targets, and on multi-display devices such as foldables. The saved image is always upright. From da8b7036d8874d7322e07026d50ab7c4a18b759f Mon Sep 17 00:00:00 2001 From: Anthony Mittaz Date: Wed, 4 Nov 2026 14:18:07 +1100 Subject: [PATCH 6/7] fix(screenshot): load mask validation only for masked captures --- packages/contracts/src/screenshot-runtime.ts | 9 +++++---- packages/platform-apple/src/core/screenshot.ts | 4 ++-- packages/platform-apple/src/interactor.ts | 5 ++++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/contracts/src/screenshot-runtime.ts b/packages/contracts/src/screenshot-runtime.ts index e8b330c2a9..f63f0c65d6 100644 --- a/packages/contracts/src/screenshot-runtime.ts +++ b/packages/contracts/src/screenshot-runtime.ts @@ -1,5 +1,4 @@ import { AppError } from '@agent-device/kernel/errors'; -import { readScreenshotScreenMask } from './screenshot.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { localInteractorSource, @@ -44,10 +43,12 @@ type ScreenshotCaptureBindingOptions = Readonly<{ captureCapabilities?: (options: Readonly) => ScreenshotCaptureCapabilities; }>; -export function assertScreenshotCaptureSupport( +export async function assertScreenshotCaptureSupport( options: Readonly | undefined, capabilities: ScreenshotCaptureCapabilities = {}, -): void { +): Promise { + if (options?.screenMask === undefined) return; + const { readScreenshotScreenMask } = await import('./screenshot.ts'); const screenMask = readScreenshotScreenMask(options?.screenMask); if (screenMask !== undefined && capabilities.screenMask !== true) { throw new AppError( @@ -80,7 +81,7 @@ function bindScreenshotCapture( ): ScreenshotRuntimeOperations { return Object.freeze({ captureScreenshot: async (input: CaptureScreenshotInput) => { - assertScreenshotCaptureSupport( + await assertScreenshotCaptureSupport( input.options, binding.captureCapabilities?.(input.options ?? {}), ); diff --git a/packages/platform-apple/src/core/screenshot.ts b/packages/platform-apple/src/core/screenshot.ts index d6dae705a8..40369a0702 100644 --- a/packages/platform-apple/src/core/screenshot.ts +++ b/packages/platform-apple/src/core/screenshot.ts @@ -128,7 +128,7 @@ export async function screenshotIos( outPath: string, options: Omit, ): Promise { - assertScreenshotCaptureSupport(options, appleScreenshotCaptureCapabilities(device, {})); + await assertScreenshotCaptureSupport(options, appleScreenshotCaptureCapabilities(device, {})); if (isMacOs(device)) { await captureScreenshotViaRunner( device, @@ -158,7 +158,7 @@ export async function captureSimulatorScreenshotWithFallback( outPath: string, options: SimulatorScreenshotFlowOptions, ): Promise { - assertScreenshotCaptureSupport(options, appleScreenshotCaptureCapabilities(device, {})); + await assertScreenshotCaptureSupport(options, appleScreenshotCaptureCapabilities(device, {})); if (device.kind !== 'simulator') { throw new AppError( 'UNSUPPORTED_OPERATION', diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index f5841b5254..530b148b22 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -495,7 +495,10 @@ async function runAppleScreenshot( helper: MacOsHelperSurface | undefined, env: Readonly, ): Promise { - assertScreenshotCaptureSupport(options, appleScreenshotCaptureCapabilities(device, options)); + await assertScreenshotCaptureSupport( + options, + appleScreenshotCaptureCapabilities(device, options), + ); if (helper) { if (options.fullscreen) { throw new AppError( From 71739389ed2b21d7188d11a7055fdcea797e58cc Mon Sep 17 00:00:00 2001 From: Anthony Mittaz Date: Wed, 4 Nov 2026 15:55:53 +1100 Subject: [PATCH 7/7] test(screenshot): assert mask refusal reason and clarify support --- .../src/__tests__/interactor-screenshot-backend.test.ts | 5 ++++- website/docs/docs/commands.md | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts b/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts index 27bedde93e..57e6215928 100644 --- a/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-screenshot-backend.test.ts @@ -67,7 +67,10 @@ test('forwards the request mask and refuses an explicit runner capture with it', ); await expect( interactor.screenshot('/tmp/runner.png', { screenMask: 'black', captureBackend: 'runner' }), - ).rejects.toMatchObject({ code: 'UNSUPPORTED_OPERATION' }); + ).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'screenshot-screen-mask-unsupported' }, + }); expect(captureScreenshotViaRunner).not.toHaveBeenCalled(); }); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index a9fe2c7d16..b861ae36e0 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1049,7 +1049,7 @@ agent-device record stop # Stop active recording - `--crop-on ` crops the capture to the frame the selector resolves on the same screen. It works only on iOS simulators, Android emulators, and physical iPhones. A physical iPhone crop is verified only in portrait using CoreDevice/devicectl; the XCTest backend and runner fallback are not accepted. Physical iPads remain rejected for crop, and landscape iPhone crop remains unverified. It needs exactly one framed node, cannot be combined with `--overlay-refs` or `--fullscreen`, and warns when the frame is clipped. The crop is re-encoded, so do not byte-compare it with an older crop. - `--fullscreen` on macOS applies only to app sessions. Other surfaces (`desktop`, `menubar`, `frontmost-app`) always capture the main display and refuse `--fullscreen` with `INVALID_ARGS`. - `--normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for reusable baselines. Each status-bar read or preparation command has a 10-second timeout. Each restore command has a 30-second timeout, so clearing and reapplying overrides can take up to 60 seconds. The request timeout includes the wait and can expire before these steps finish. -- `--screen-mask black` fills the iOS simulator screen outline, including rounded corners, with black for this capture. Node and MCP use `screenMask: "black"`. Omit it for the default image. Support is limited to handheld iOS/iPadOS simulators. tvOS and visionOS mask behavior has not been verified. It requires simctl capture; physical devices, tvOS and visionOS simulators, other platforms, provider captures, and XCTest runner captures are refused. If simctl fails, a masked request fails instead of falling back to an unmasked runner capture. This option does not redact app content, remove a Dynamic Island drawn into the framebuffer, or change comparison thresholds. +- `--screen-mask black` fills the iOS simulator screen outline, including rounded corners, with black for this capture. Node and MCP use `screenMask: "black"`. Omit it for the default image. Support is limited to handheld iOS/iPadOS simulators. It requires simctl capture; physical devices, tvOS and visionOS simulators, other platforms, provider captures, and XCTest runner captures are refused. If simctl fails, a masked request fails instead of falling back to an unmasked runner capture. This option does not redact app content, remove a Dynamic Island drawn into the framebuffer, or change comparison thresholds. - On an iOS simulator, a screenshot waits until no other screenshot or `settings wifi` / `settings airplane` write is using that simulator's status bar, so overlapping captures cannot see each other's temporary chrome. A request you cancel while it is waiting fails with the cancellation instead of capturing. - On single-display iOS simulators and Android devices, the result may report `displayRotation` (`portrait`, `portrait-upside-down`, `landscape-left`, or `landscape-right`). It is absent when the device returns no reading in time, on other targets, and on multi-display devices such as foldables. The saved image is always upright.