Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/adr/0004-ios-snapshot-backend-strategy.md
Original file line number Diff line number Diff line change
Expand Up @@ -623,7 +623,7 @@ Thus, the pass returns that tree exactly as reported.
This is a structural consequence of the same rule applied uniformly.
It is not the per-tier omission that a hand-threaded space would have made it.

The provider producers (`appium-source`, `limrun-ios-tree`) never see the app's windows either.
The provider producers (`appium-source`) never see the app's windows either.
A capture from any of them publishes no keyboard fact.
A consumer reads that silence as "this producer did not measure".
Those paths publish what the platform reported.
Expand Down Expand Up @@ -728,7 +728,7 @@ Producers that report no UIKit class names or no dimming-view interaction state
That is a fact about their output, not a backend exception in the fold:

- The runner already omits modal-contained content from its own queries.
- `appium-source` and `limrun-ios-tree` report element types and no classes.
- `appium-source` reports element types and no classes.
- The macOS desktop surface arrives already presented.

A scope that names a modal-contained screen now publishes an empty projection.
Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,7 @@
"oxc-parser": "^0.146.0",
"oxfmt": "^0.67.0",
"oxlint": "^1.79.0",
"plist": "5.0.0",
"pngjs": "^7.0.0",
"publint": "^0.3.24",
"tar-stream": "^3.2.0",
Expand Down
15 changes: 15 additions & 0 deletions packages/apple-runner/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,16 @@
* imports so the entry evaluates only what a host-bound operation needs.
*/

export { RUNNER_STARTUP_TIMEOUT_MS } from './runner-startup-transport.ts';
export async function createLaunchedAppleRunnerProvider(
...args: Parameters<
typeof import('./runner-launcher-provider.ts').createLaunchedAppleRunnerProvider
>
) {
const { createLaunchedAppleRunnerProvider } = await import('./runner-launcher-provider.ts');
return createLaunchedAppleRunnerProvider(...args);
}

export {
bindAppleRunnerHost,
type AppleRunnerHost,
Expand All @@ -22,6 +32,7 @@ export type {
IosPhysicalDeviceRunnerControl,
} from './runner-physical-device.ts';
export {
mintRunnerToken,
RUNNER_READY_PREFLIGHT_TIMEOUT_MS,
SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE,
buildRunnerSequenceCommand,
Expand Down Expand Up @@ -371,3 +382,7 @@ export async function restoreLegacyXctestDeviceSetRedirect(
await import('./legacy-xctest-device-set.ts');
await restore(...args);
}

export { runnerListenerEnv } from './runner-contract.ts';

export { RUNNER_XCTESTRUN_CAPTURE_OPTIONS } from './runner-artifact-env.ts';
2 changes: 1 addition & 1 deletion packages/apple-runner/src/runner-artifact-env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ function toXctestrunTarget(
return target;
}

const RUNNER_XCTESTRUN_CAPTURE_OPTIONS = {
export const RUNNER_XCTESTRUN_CAPTURE_OPTIONS = {
PreferredScreenCaptureFormat: 'screenshots',
SystemAttachmentLifetime: 'keepNever',
UserAttachmentLifetime: 'keepNever',
Expand Down
2 changes: 1 addition & 1 deletion packages/apple-runner/src/runner-launched-session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export async function launchRunnerSession(
let launch;
try {
signal?.throwIfAborted();
launch = await launcher.launch(endpoint, abort.signal);
launch = await launcher.launch(endpoint, abort.signal, budget.phase.deadline);
} finally {
signal?.removeEventListener('abort', cancel);
}
Expand Down
7 changes: 6 additions & 1 deletion packages/apple-runner/src/runner-launcher.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { DeadlineClock } from '@agent-device/host-kit/retry';
import type { AppError } from '@agent-device/kernel/errors';
import type { IosRunnerDeviceStates } from './runner-error-classification.ts';
import type { LocalRunnerLaunchResources, RunnerDisposalOptions } from './runner-disposal.ts';
Expand All @@ -13,7 +14,11 @@ export type RunnerLaunch = {
/** Launch mechanisms supply resources; the session owner controls their lifetime. */
export type RunnerLauncher = {
endpoint: Omit<RunnerEndpoint, 'token'>;
launch(endpoint: RunnerEndpoint, signal: AbortSignal): Promise<RunnerLaunch>;
launch(
endpoint: RunnerEndpoint,
signal: AbortSignal,
deadline?: DeadlineClock,
): Promise<RunnerLaunch>;
};

export type RunnerLaunchObservation = {
Expand Down
4 changes: 2 additions & 2 deletions packages/capture-kit/src/ios-ax-tree.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ function readIosNodeIdentifier(node: IosTreeNode): string | undefined {
* The node's screen-space rect, or undefined when the provider omitted a complete one. A component that
* is not a finite number (an AX sentinel such as an infinite origin) makes the frame unusable.
*/
export function readIosNodeRect(node: IosTreeNode): RawSnapshotNode['rect'] {
function readIosNodeRect(node: IosTreeNode): RawSnapshotNode['rect'] {
const rect = node.rect ?? node.frame;
if (
!rect ||
Expand All @@ -137,7 +137,7 @@ function isFiniteNumber(value: unknown): value is number {
return typeof value === 'number' && Number.isFinite(value);
}

export function readIosNodeChildren(node: IosTreeNode): IosTreeNode[] {
function readIosNodeChildren(node: IosTreeNode): IosTreeNode[] {
return node.children ?? node.nodes ?? node.elements ?? [];
}

Expand Down
2 changes: 0 additions & 2 deletions packages/capture-kit/src/ios-snapshot-acquisition.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ const TREE_SOURCE_CAPABILITIES = Object.freeze({
*/
const IOS_PROVIDER_ACQUISITION_CAPABILITY_VALUES = {
'appium-source': { producer: 'appium-source', ...TREE_SOURCE_CAPABILITIES },
'limrun-ios-tree': { producer: 'limrun-ios-tree', ...TREE_SOURCE_CAPABILITIES },
'revyl-ios-tree': { producer: 'revyl-ios-tree', ...TREE_SOURCE_CAPABILITIES },
} as const satisfies Record<IosProviderAcquisitionProducer, IosProviderAcquisitionCapabilities>;

Expand All @@ -63,7 +62,6 @@ const IOS_SNAPSHOT_TRUNCATION_EVIDENCE = {
'apple-runner': 'available',
'simulator-ax-bridge': 'available',
'appium-source': 'unavailable',
'limrun-ios-tree': 'unavailable',
'revyl-ios-tree': 'unavailable',
} as const satisfies Record<IosSnapshotProducer, IosSnapshotEvidenceAvailability>;

Expand Down
5 changes: 2 additions & 3 deletions packages/capture-kit/src/ios-snapshot-planning.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ test('normalization makes raw, scope, and absent values explicit', () => {
});

test('acquisition derives unavailable provider facts from the registry', () => {
for (const producer of ['appium-source', 'limrun-ios-tree'] as const) {
for (const producer of ['appium-source', 'revyl-ios-tree'] as const) {
assert.deepEqual(
createIosSnapshotAcquisition({
producer,
Expand Down Expand Up @@ -102,7 +102,6 @@ test('truncation evidence is declared for every iOS producer', () => {
'apple-runner': 'available',
'simulator-ax-bridge': 'available',
'appium-source': 'unavailable',
'limrun-ios-tree': 'unavailable',
'revyl-ios-tree': 'unavailable',
} as const satisfies Record<IosSnapshotProducer, IosSnapshotEvidenceAvailability>;

Expand All @@ -114,7 +113,7 @@ test('truncation evidence is declared for every iOS producer', () => {
test('comparison identity rejects every identity axis and residue mismatch', () => {
const base = comparisonIdentity();
const mismatches: IosSnapshotComparisonIdentity[] = [
{ ...base, producer: 'limrun-ios-tree' },
{ ...base, producer: 'revyl-ios-tree' },
{ ...base, intent: 'surface-observation' },
{ ...base, lineage: { targetId: 'simulator-1', generation: 'generation-2' } },
{ ...base, presentationKey: { ...base.presentationKey, depth: 1 } },
Expand Down
34 changes: 0 additions & 34 deletions packages/capture-kit/src/png-worker-client.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import {
encodePngAsync,
retainPngWorker,
terminatePngWorker,
transcodeScreenshotToPngAsync,
} from './png-worker-client.ts';
import { cropPngBytes } from './png-crop-bytes.ts';
import { computeScreenshotDiffPixels } from './screenshot-diff-pixels.ts';
Expand Down Expand Up @@ -148,39 +147,6 @@ test('decodePngAsync rejects invalid PNG data with the canonical decode AppError
);
});

test('transcodeScreenshotToPngAsync matches the synchronous transcoder byte for byte', async () => {
const { encode } = await import('jpeg-js');
const { transcodeScreenshotToPng } = await import('./screenshot-image.ts');
const rgba = Buffer.alloc(9 * 7 * 4, 0x66);
const jpeg = encode({ width: 9, height: 7, data: rgba }, 90).data;

const fromWorker = await runOnWorkerThread('jpeg-to-png', () =>
transcodeScreenshotToPngAsync(jpeg, 'fixture'),
);

assert.deepEqual(fromWorker, transcodeScreenshotToPng(jpeg, 'fixture'));
assert.equal(PNG.sync.read(fromWorker).width, 9);
});

test('transcodeScreenshotToPngAsync returns a PNG untouched without a worker round trip', async () => {
const png = PNG.sync.write(new PNG({ width: 2, height: 2 }));

assert.equal(await transcodeScreenshotToPngAsync(png, 'fixture'), png);
});

test('transcodeScreenshotToPngAsync rejects a corrupt JPEG with the canonical decode AppError', async () => {
const corrupt = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(16, 0)]);

await assert.rejects(transcodeScreenshotToPngAsync(corrupt, 'fixture'), (error) => {
assert.equal(error instanceof AppError, true);
assert.equal((error as AppError).code, 'COMMAND_FAILED');
assert.match((error as AppError).message, /Failed to decode fixture as JPEG/);
assert.equal((error as AppError).details?.label, 'fixture');
assert.ok(String((error as AppError).details?.reason).length > 0);
return true;
});
});

test('decodeScreenshotImageAsync answers the synchronous decoder for both containers', async () => {
const { encode } = await import('jpeg-js');
const { decodeScreenshotImage } = await import('./screenshot-image.ts');
Expand Down
18 changes: 1 addition & 17 deletions packages/capture-kit/src/png-worker-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { Worker } from 'node:worker_threads';
import { AppError, toAppErrorCode } from '@agent-device/kernel/errors';
import type { Rect } from '@agent-device/kernel/snapshot';
import { resolveInternalEntryModulePath } from './internal-entry.ts';
import { decodePng, hasPngSignature, PNG } from './png.ts';
import { decodePng, PNG } from './png.ts';
import {
computeScreenshotDiffPixels,
type ScreenshotDiffPixelsJob,
Expand Down Expand Up @@ -315,19 +315,3 @@ export async function computeScreenshotDiffPixelsAsync(
}));
return { ...result, diffData: toBuffer(result.diffData) };
}

/**
* PNG bytes for a provider screenshot in whatever container it arrived in. A PNG returns as is
* without a worker round trip; a JPEG is decoded and re-encoded on the worker so a full-resolution
* capture never blocks the daemon event loop. Decode failures carry the canonical `AppError`.
*/
export async function transcodeScreenshotToPngAsync(bytes: Buffer, label: string): Promise<Buffer> {
if (hasPngSignature(bytes)) return bytes;
const result = await runPngJob({ kind: 'jpeg-to-png', image: bytes, label }, async () => {
// Read on demand so the JPEG decoder stays out of the import closure of every entry that only
// needs the worker's other jobs.
const { transcodeScreenshotToPng } = await import('./screenshot-image.ts');
return { kind: 'jpeg-to-png', png: transcodeScreenshotToPng(bytes, label) };
});
return toBuffer(result.png);
}
9 changes: 3 additions & 6 deletions packages/capture-kit/src/png-worker-contract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import type { PngRgbDifferenceResult } from './png-rgb-difference.ts';
/**
* Message contract between the daemon-side PNG worker client
* (`png-worker-client.ts`) and the worker thread entry (`png-worker.ts`).
* One message = one decode, encode, crop, diff, or transcode job. Binary payloads cross the
* One message = one decode, encode, crop, or diff job. Binary payloads cross the
* thread boundary via structured clone (or transfer), so `Buffer` fields
* arrive as plain `Uint8Array` views on the receiving side.
*/
Expand All @@ -21,9 +21,7 @@ export type PngWorkerJob =
| { kind: 'encode'; width: number; height: number; data: Uint8Array }
| { kind: 'crop'; png: Uint8Array; label: string; box: Rect }
| { kind: 'rgb-difference'; firstPng: Uint8Array; secondPng: Uint8Array; label: string }
| ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsJob)
// A provider screenshot in whatever container it arrived in; answers PNG bytes.
| { kind: 'jpeg-to-png'; image: Uint8Array; label: string };
| ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsJob);

export type PngWorkerJobResult =
| { kind: 'decode'; width: number; height: number; data: Uint8Array }
Expand All @@ -32,8 +30,7 @@ export type PngWorkerJobResult =
// A crop answers `null` when the box already covers the image, so the caller keeps the file.
| { kind: 'crop'; png: Uint8Array | null }
| ({ kind: 'rgb-difference' } & PngRgbDifferenceResult)
| ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsResult)
| { kind: 'jpeg-to-png'; png: Uint8Array };
| ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsResult);

export type PngWorkerJobKind = PngWorkerJob['kind'];

Expand Down
9 changes: 0 additions & 9 deletions packages/capture-kit/src/png-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,6 @@ async function runJob(request: PngWorkerRequest): Promise<PngWorkerJobResult> {
case 'diff-pixels': {
return { kind: 'diff-pixels', ...computeScreenshotDiffPixels(request) };
}
case 'jpeg-to-png': {
const { transcodeScreenshotToPng } = await import('./screenshot-image.ts');
return {
kind: 'jpeg-to-png',
png: transcodeScreenshotToPng(toBuffer(request.image), request.label),
};
}
}
}

Expand Down Expand Up @@ -102,8 +95,6 @@ function resultBufferViews(result: PngWorkerJobResult): Uint8Array[] {
return [];
case 'diff-pixels':
return [result.diffData, result.diffMask];
case 'jpeg-to-png':
return [result.png];
}
}

Expand Down
37 changes: 4 additions & 33 deletions packages/capture-kit/src/screenshot-image.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,7 @@ import { expect, test } from 'vitest';
import fc from 'fast-check';
import { encode as encodeJpeg } from 'jpeg-js';
import { PNG } from './png.ts';
import {
decodeScreenshotImage,
detectScreenshotImageFormat,
transcodeScreenshotToPng,
} from './screenshot-image.ts';
import { decodeScreenshotImage, detectScreenshotImageFormat } from './screenshot-image.ts';

function solidRgba(width: number, height: number, rgba: readonly [number, number, number, number]) {
const data = Buffer.alloc(width * height * 4);
Expand All @@ -24,32 +20,9 @@ function thrownBy(run: () => unknown): unknown {
throw new Error('expected the call to throw');
}

test('a JPEG screenshot becomes a PNG of the same size with the same picture', () => {
const jpeg = encodeJpeg({ width: 6, height: 4, data: solidRgba(6, 4, [200, 30, 30, 255]) }, 100);
expect(detectScreenshotImageFormat(jpeg.data)).toBe('jpeg');

const png = transcodeScreenshotToPng(jpeg.data, 'test screenshot');

expect(detectScreenshotImageFormat(png)).toBe('png');
const decoded = PNG.sync.read(png);
expect([decoded.width, decoded.height]).toEqual([6, 4]);
const [r = -1, g = -1, b = -1, a = -1] = decoded.data.subarray(0, 4);
// JPEG is lossy; a flat field survives within a few levels per channel.
expect(Math.abs(r - 200)).toBeLessThanOrEqual(4);
expect(Math.abs(g - 30)).toBeLessThanOrEqual(4);
expect(Math.abs(b - 30)).toBeLessThanOrEqual(4);
expect(a).toBe(255);
});

test('a PNG screenshot passes through byte for byte', () => {
const png = PNG.sync.write(new PNG({ width: 3, height: 2 }));

expect(transcodeScreenshotToPng(png, 'test screenshot')).toBe(png);
});

test('bytes that are neither container are refused with a typed error carrying the label', () => {
expect(
thrownBy(() => transcodeScreenshotToPng(Buffer.from('GIF89a'), 'Limrun iOS screenshot')),
thrownBy(() => decodeScreenshotImage(Buffer.from('GIF89a'), 'Limrun iOS screenshot')),
).toMatchObject({
code: 'COMMAND_FAILED',
message: 'Limrun iOS screenshot is neither PNG nor JPEG',
Expand All @@ -61,7 +34,7 @@ test('a valid SOI marker followed by garbage is refused with the typed decode er
const garbage = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(32, 0x41)]);
expect(detectScreenshotImageFormat(garbage)).toBe('jpeg');

expect(thrownBy(() => transcodeScreenshotToPng(garbage, 'Limrun iOS screenshot'))).toMatchObject({
expect(thrownBy(() => decodeScreenshotImage(garbage, 'Limrun iOS screenshot'))).toMatchObject({
code: 'COMMAND_FAILED',
message: 'Failed to decode Limrun iOS screenshot as JPEG',
details: { label: 'Limrun iOS screenshot', reason: expect.any(String) },
Expand All @@ -72,9 +45,7 @@ test('a truncated JPEG body is refused with the same typed decode error', () =>
const jpeg = encodeJpeg({ width: 6, height: 4, data: solidRgba(6, 4, [10, 200, 30, 255]) }, 100);
const truncated = jpeg.data.subarray(0, 24);

expect(
thrownBy(() => transcodeScreenshotToPng(truncated, 'Limrun iOS screenshot')),
).toMatchObject({
expect(thrownBy(() => decodeScreenshotImage(truncated, 'Limrun iOS screenshot'))).toMatchObject({
code: 'COMMAND_FAILED',
message: 'Failed to decode Limrun iOS screenshot as JPEG',
});
Expand Down
16 changes: 0 additions & 16 deletions packages/capture-kit/src/screenshot-image.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,22 +30,6 @@ export function decodeScreenshotImage(bytes: Buffer, label: string, maxPixels?:
throw unsupportedContainer(label, bytes);
}

/**
* Returns PNG bytes for a screenshot a provider handed back in whatever container it prefers. PNG
* passes through untouched; JPEG is decoded and re-encoded losslessly from the decoded pixels, so
* every PNG-only reader downstream (size, crop, overlay) sees the format the path promises. Those
* readers rewrite the file in place, which is why a JPEG cannot be handed to them undecoded.
*
* Synchronous and CPU-bound: daemon request paths reach it through the PNG worker
* (`transcodeScreenshotToPngAsync` in `png-worker-client.ts`), like every other codec job.
*/
export function transcodeScreenshotToPng(bytes: Buffer, label: string): Buffer {
const format = detectScreenshotImageFormat(bytes);
if (format === 'png') return bytes;
if (format === 'jpeg') return PNG.sync.write(decodeJpegImage(bytes, label));
throw unsupportedContainer(label, bytes);
}

/** A JPEG signature does not prove a decodable body; a failure keeps the label and the decoder's reason. */
function decodeJpegImage(bytes: Buffer, label: string, maxPixels?: number): PNG {
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ const ROW: Rect = { x: 16, y: 80, width: 288, height: 52 };
const ACQUISITION_PRODUCERS: readonly IosAcquisitionProducer[] = [
'simulator-ax-bridge',
'appium-source',
'limrun-ios-tree',
'revyl-ios-tree',
];
const PRODUCERS: readonly IosSnapshotProducer[] = ['apple-runner', ...ACQUISITION_PRODUCERS];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,7 @@ const fixture = JSON.parse(
};

test('host iOS presentation publishes shared clipping notes without inventing a capture verdict', () => {
for (const producer of [
'simulator-ax-bridge',
'appium-source',
'limrun-ios-tree',
'revyl-ios-tree',
] as const) {
for (const producer of ['simulator-ax-bridge', 'appium-source', 'revyl-ios-tree'] as const) {
for (const row of fixture.cases) {
const acquired = createIosSnapshotAcquisition({
producer: producer === 'simulator-ax-bridge' ? 'appium-source' : producer,
Expand Down
Loading
Loading