feat(screenshot): add --crop-on to crop captures to a selector frame

contracts/fixtures/screenshot-crop-geometry.json

@@ -0,0 +1,92 @@
+[
+  {
+    "name": "iOS simulator 1x capture: viewport-space rect projects 1:1 against a zero-origin bounds",
+    "space": "viewport-points",
+    "bounds": { "x": 0, "y": 0, "width": 402, "height": 874 },
+    "rect": { "x": 16, "y": 293.33, "width": 370, "height": 52 },
+    "image": { "width": 402, "height": 874 },
+    "expectedProjection": { "x": 16, "y": 293, "width": 370, "height": 52 },
+    "expectedIntersection": { "x": 16, "y": 293, "width": 370, "height": 52 }
+  },
+  {
+    "name": "iOS simulator 3x capture: the same points rect scales by imgW/bounds.width and rounds",
+    "space": "viewport-points",
+    "bounds": { "x": 0, "y": 0, "width": 402, "height": 874 },
+    "rect": { "x": 16, "y": 293.33, "width": 370, "height": 52 },
+    "image": { "width": 1206, "height": 2622 },
+    "expectedProjection": { "x": 48, "y": 880, "width": 1110, "height": 156 },
+    "expectedIntersection": { "x": 48, "y": 880, "width": 1110, "height": 156 }
+  },
+  {
+    "name": "Android: device-pixel rects project 1:1 with no bounds lookup",
+    "space": "device-pixels",
+    "bounds": null,
+    "rect": { "x": 210, "y": 678, "width": 436, "height": 71 },
+    "image": { "width": 1080, "height": 2400 },
+    "expectedProjection": { "x": 210, "y": 678, "width": 436, "height": 71 },
+    "expectedIntersection": { "x": 210, "y": 678, "width": 436, "height": 71 }
+  },
+  {
+    "name": "macOS app window: non-zero-origin bounds shift the rect before the 2x scale",
+    "space": "viewport-points",
+    "bounds": { "x": 898, "y": 74, "width": 586, "height": 488 },
+    "rect": { "x": 906, "y": 114, "width": 20, "height": 20 },
+    "image": { "width": 1172, "height": 976 },
+    "expectedProjection": { "x": 16, "y": 80, "width": 40, "height": 40 },
+    "expectedIntersection": { "x": 16, "y": 80, "width": 40, "height": 40 }
+  },
+  {
+    "name": "viewport-points without bounds degrades to a rounded 1:1 rect instead of dividing by zero",
+    "space": "viewport-points",
+    "bounds": null,
+    "rect": { "x": 16.6, "y": 293.33, "width": 370, "height": 52 },
+    "image": { "width": 402, "height": 874 },
+    "expectedProjection": { "x": 17, "y": 293, "width": 370, "height": 52 },
+    "expectedIntersection": { "x": 17, "y": 293, "width": 370, "height": 52 }
+  },
+  {
+    "name": "scrolled-out row above the image floor intersects nothing",
+    "space": "viewport-points",
+    "bounds": { "x": 0, "y": 0, "width": 402, "height": 874 },
+    "rect": { "x": 16, "y": 900, "width": 370, "height": 52 },
+    "image": { "width": 402, "height": 874 },
+    "expectedProjection": { "x": 16, "y": 900, "width": 370, "height": 52 },
+    "expectedIntersection": null
+  },
+  {
+    "name": "row grazing the bottom edge clips to the image and stays partial",
+    "space": "device-pixels",
+    "bounds": null,
+    "rect": { "x": 210, "y": 2360, "width": 436, "height": 111 },
+    "image": { "width": 1080, "height": 2400 },
+    "expectedProjection": { "x": 210, "y": 2360, "width": 436, "height": 111 },
+    "expectedIntersection": { "x": 210, "y": 2360, "width": 436, "height": 40 }
+  },
+  {
+    "name": "rect left of the image edge clips to x=0",
+    "space": "device-pixels",
+    "bounds": null,
+    "rect": { "x": -40, "y": 100, "width": 200, "height": 50 },
+    "image": { "width": 1080, "height": 2400 },
+    "expectedProjection": { "x": -40, "y": 100, "width": 200, "height": 50 },
+    "expectedIntersection": { "x": 0, "y": 100, "width": 160, "height": 50 }
+  },
+  {
+    "name": "rect fully past the right edge intersects nothing",
+    "space": "device-pixels",
+    "bounds": null,
+    "rect": { "x": 1080, "y": 100, "width": 100, "height": 50 },
+    "image": { "width": 1080, "height": 2400 },
+    "expectedProjection": { "x": 1080, "y": 100, "width": 100, "height": 50 },
+    "expectedIntersection": null
+  },
+  {
+    "name": "zero-area intersection (edge touch only) is empty, not a 1px sliver",
+    "space": "device-pixels",
+    "bounds": null,
+    "rect": { "x": 1079, "y": 2400, "width": 10, "height": 10 },
+    "image": { "width": 1080, "height": 2400 },
+    "expectedProjection": { "x": 1079, "y": 2400, "width": 10, "height": 10 },
+    "expectedIntersection": null
+  }
+]

packages/ad-script/src/internal/__tests__/script.test.ts

@@ -135,6 +135,34 @@ test('screenshot replay script round-trips screenshot flags', () => {
   assert.equal(parsed[0]?.flags.screenshotNoStabilize, true);
 });
 
+test('screenshot replay script round-trips a quoted --crop-on selector', () => {
+  const actions: SessionAction[] = [
+    {
+      ts: Date.now(),
+      command: 'screenshot',
+      positionals: ['./page.png'],
+      flags: {
+        screenshotCropOn: 'role=cell label=General || role=button label=General',
+        screenshotScale: 0.3,
+      },
+    },
+  ];
+
+  const script = formatReplayScriptForTest(actions);
+  assert.match(
+    script,
+    /screenshot "\.\/page\.png" --crop-on "role=cell label=General \|\| role=button label=General" --scale 0\.3/,
+  );
+
+  const parsed = parseReplayScriptDetailed(script).actions;
+  assert.deepEqual(parsed[0]?.positionals, ['./page.png']);
+  assert.equal(
+    parsed[0]?.flags.screenshotCropOn,
+    'role=cell label=General || role=button label=General',
+  );
+  assert.equal(parsed[0]?.flags.screenshotScale, 0.3);
+});
+
 test('snapshot replay script parses full refresh flags', () => {
   const ignoredLegacyFlag = '-' + 'c';
   const parsed = parseReplayScriptDetailed(

packages/ad-script/src/internal/script-utils.ts

@@ -227,6 +227,10 @@ export function appendScreenshotActionScriptArgs(parts: string[], action: Sessio
   for (const positional of action.positionals ?? []) {
     parts.push(formatScriptArg(positional));
   }
+  const cropOn = action.flags?.screenshotCropOn;
+  if (typeof cropOn === 'string' && cropOn.length > 0) {
+    parts.push('--crop-on', formatScriptArgQuoteIfNeeded(cropOn));
+  }
   appendScreenshotScriptFlags(parts, action.flags);
 }
 

packages/capture-kit/package.json

@@ -34,6 +34,10 @@
       "types": "./src/png.ts",
       "default": "./src/png.ts"
     },
+    "./png-crop": {
+      "types": "./src/png-crop.ts",
+      "default": "./src/png-crop.ts"
+    },
     "./png-resize": {
       "types": "./src/png-resize.ts",
       "default": "./src/png-resize.ts"
@@ -73,6 +77,10 @@
     "./snapshot-quality-verdict": {
       "types": "./src/snapshot-quality-verdict.ts",
       "default": "./src/snapshot-quality-verdict.ts"
+    },
+    "./snapshot-rect-projection": {
+      "types": "./src/snapshot-rect-projection.ts",
+      "default": "./src/snapshot-rect-projection.ts"
     }
   },
   "devDependencies": {

packages/capture-kit/src/png-crop.test.ts

@@ -0,0 +1,97 @@
+import { afterAll, test } from 'vitest';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import { PNG } from './png.ts';
+import { cropPngFile } from './png-crop.ts';
+import { terminatePngWorker } from './png-worker-client.ts';
+import { mkdtempForTestSync } from './tmp-dir.fixtures.ts';
+
+afterAll(async () => {
+  await terminatePngWorker();
+});
+
+test('cropPngFile keeps only the box rows and columns, in place', async () => {
+  const filePath = writeCheckedPng();
+  await cropPngFile(filePath, { x: 2, y: 1, width: 3, height: 2 });
+
+  const cropped = PNG.sync.read(fs.readFileSync(filePath));
+  assert.equal(cropped.width, 3);
+  assert.equal(cropped.height, 2);
+  assert.deepEqual(readPngPixel(cropped, 0, 0), pixel(2, 1));
+  assert.deepEqual(readPngPixel(cropped, 2, 1), pixel(4, 2));
+});
+
+test('a box grazing the image edges crops to the edge without clamping the origin', async () => {
+  const filePath = writeCheckedPng();
+  await cropPngFile(filePath, { x: 4, y: 2, width: 2, height: 2 });
+
+  const cropped = PNG.sync.read(fs.readFileSync(filePath));
+  assert.equal(cropped.width, 2);
+  assert.equal(cropped.height, 2);
+  assert.deepEqual(readPngPixel(cropped, 1, 1), pixel(5, 3));
+});
+
+test('a full-image box is a no-op that leaves the file decodable at the same size', async () => {
+  const filePath = writeCheckedPng();
+  const before = fs.readFileSync(filePath);
+  await cropPngFile(filePath, { x: 0, y: 0, width: 6, height: 4 });
+  assert.deepEqual(fs.readFileSync(filePath), before);
+});
+
+test('boxes that exceed the image refuse instead of clamping', async () => {
+  const filePath = writeCheckedPng();
+  await assert.rejects(
+    () => cropPngFile(filePath, { x: 5, y: 3, width: 3, height: 3 }),
+    (error: unknown) => error instanceof Error && error.message.includes('exceeds'),
+  );
+});
+
+test('non-integer or non-positive boxes refuse', async () => {
+  const filePath = writeCheckedPng();
+  const boxes = [
+    { x: 1.5, y: 0, width: 2, height: 2 },
+    { x: 0, y: -1, width: 2, height: 2 },
+    { x: 0, y: 0, width: 0, height: 2 },
+    { x: 0, y: 0, width: 2, height: 0 },
+  ] as const;
+  for (const box of boxes) {
+    await assert.rejects(
+      () => cropPngFile(filePath, box),
+      (error: unknown) => error instanceof Error && error.message.includes('positive integer'),
+      JSON.stringify(box),
+    );
+  }
+});
+
+// A 6x4 grid whose pixel (x, y) carries (x*10, y*10) so a wrong source offset
+// is caught by the value, not just the size.
+function writeCheckedPng(): string {
+  const filePath = path.join(mkdtempForTestSync('agent-device-png-crop-'), 'image.png');
+  const png = new PNG({ width: 6, height: 4 });
+  for (let y = 0; y < png.height; y += 1) {
+    for (let x = 0; x < png.width; x += 1) {
+      const offset = (y * png.width + x) * 4;
+      png.data[offset] = x * 10;
+      png.data[offset + 1] = y * 10;
+      png.data[offset + 2] = 0;
+      png.data[offset + 3] = 255;
+    }
+  }
+  fs.writeFileSync(filePath, PNG.sync.write(png));
+  return filePath;
+}
+
+function pixel(x: number, y: number): number[] {
+  return [x * 10, y * 10, 0, 255];
+}
+
+function readPngPixel(png: PNG, x: number, y: number): number[] {
+  const offset = (y * png.width + x) * 4;
+  return [
+    png.data[offset] ?? 0,
+    png.data[offset + 1] ?? 0,
+    png.data[offset + 2] ?? 0,
+    png.data[offset + 3] ?? 0,
+  ];
+}

packages/capture-kit/src/png-crop.ts

@@ -0,0 +1,54 @@
+import { promises as fs } from 'node:fs';
+import { AppError } from '@agent-device/kernel/errors';
+import type { Rect } from '@agent-device/kernel/snapshot';
+import { PNG } from './png.ts';
+import { decodePngAsync, encodePngAsync } from './png-worker-client.ts';
+
+/**
+ * Crops `filePath` in place to `box` (positive integer pixels). `box` is the caller's
+ * already-intersected region, so one outside the image is a caller bug — refused, not clamped.
+ * Decode and encode run on the PNG worker thread; a full-image box is a no-op.
+ */
+export async function cropPngFile(filePath: string, box: Rect): Promise<void> {
+  if (!isCropBox(box)) {
+    throw new AppError(
+      'INVALID_ARGS',
+      'Screenshot crop box must be positive integer pixel offsets',
+    );
+  }
+
+  const source = await decodePngAsync(await fs.readFile(filePath), 'screenshot');
+  if (box.x + box.width > source.width || box.y + box.height > source.height) {
+    throw new AppError(
+      'INVALID_ARGS',
+      `Screenshot crop box ${box.width}x${box.height} at (${box.x}, ${box.y}) exceeds the ${source.width}x${source.height} image`,
+    );
+  }
+  if (box.x === 0 && box.y === 0 && box.width === source.width && box.height === source.height) {
+    return;
+  }
+
+  await fs.writeFile(filePath, await encodePngAsync(cropPngBox(source, box)));
+}
+
+function isCropBox(box: Rect): boolean {
+  return (
+    Number.isInteger(box.x) &&
+    box.x >= 0 &&
+    Number.isInteger(box.y) &&
+    box.y >= 0 &&
+    Number.isInteger(box.width) &&
+    box.width > 0 &&
+    Number.isInteger(box.height) &&
+    box.height > 0
+  );
+}
+
+function cropPngBox(source: PNG, box: Rect): PNG {
+  const output = new PNG({ width: box.width, height: box.height });
+  for (let row = 0; row < box.height; row += 1) {
+    const sourceStart = ((row + box.y) * source.width + box.x) * 4;
+    source.data.copy(output.data, row * output.width * 4, sourceStart, sourceStart + box.width * 4);
+  }
+  return output;
+}