Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions packages/apple-runner/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,21 @@
*
* The package entry names its whole surface. Host-bound operations evaluate the implementation, so
* they reach their host capabilities through the `AppleRunnerHost` port, which `platform-apple`
* binds once through `bindAppleRunnerHost` before any operation runs. The one operation outside
* this closure, `runApplePressSeries`, and the legacy device-set restore stay behind function-scoped
* imports so the entry evaluates only what a host-bound operation needs.
* binds once through `bindAppleRunnerHost` before any operation runs. `runApplePressSeries`,
* `createLaunchedAppleRunnerProvider`, and the legacy device-set restore stay behind
* function-scoped imports so the entry evaluates only what a host-bound operation needs.
*/

export { RUNNER_STARTUP_TIMEOUT_MS } from './runner-startup-transport.ts';
export async function createLaunchedAppleRunnerProvider(
Comment thread
thymikee marked this conversation as resolved.
Resolved
...args: Parameters<
typeof import('./runner-launcher-provider.ts').createLaunchedAppleRunnerProvider
>
) {
const { createLaunchedAppleRunnerProvider } = await import('./runner-launcher-provider.ts');
return createLaunchedAppleRunnerProvider(...args);
}

export {
bindAppleRunnerHost,
type AppleRunnerHost,
Expand All @@ -22,6 +32,7 @@ export type {
IosPhysicalDeviceRunnerControl,
} from './runner-physical-device.ts';
export {
mintRunnerToken,
RUNNER_READY_PREFLIGHT_TIMEOUT_MS,
SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE,
buildRunnerSequenceCommand,
Expand Down Expand Up @@ -371,3 +382,7 @@ export async function restoreLegacyXctestDeviceSetRedirect(
await import('./legacy-xctest-device-set.ts');
await restore(...args);
}

export { runnerListenerEnv } from './runner-contract.ts';

export { RUNNER_XCTESTRUN_CAPTURE_OPTIONS } from './runner-artifact-env.ts';
2 changes: 1 addition & 1 deletion packages/apple-runner/src/runner-artifact-env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ function toXctestrunTarget(
return target;
}

const RUNNER_XCTESTRUN_CAPTURE_OPTIONS = {
export const RUNNER_XCTESTRUN_CAPTURE_OPTIONS = {
PreferredScreenCaptureFormat: 'screenshots',
SystemAttachmentLifetime: 'keepNever',
UserAttachmentLifetime: 'keepNever',
Expand Down
4 changes: 2 additions & 2 deletions packages/capture-kit/src/ios-ax-tree.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ function readIosNodeIdentifier(node: IosTreeNode): string | undefined {
* The node's screen-space rect, or undefined when the provider omitted a complete one. A component that
* is not a finite number (an AX sentinel such as an infinite origin) makes the frame unusable.
*/
export function readIosNodeRect(node: IosTreeNode): RawSnapshotNode['rect'] {
function readIosNodeRect(node: IosTreeNode): RawSnapshotNode['rect'] {
const rect = node.rect ?? node.frame;
if (
!rect ||
Expand All @@ -137,7 +137,7 @@ function isFiniteNumber(value: unknown): value is number {
return typeof value === 'number' && Number.isFinite(value);
}

export function readIosNodeChildren(node: IosTreeNode): IosTreeNode[] {
function readIosNodeChildren(node: IosTreeNode): IosTreeNode[] {
return node.children ?? node.nodes ?? node.elements ?? [];
}

Expand Down
34 changes: 0 additions & 34 deletions packages/capture-kit/src/png-worker-client.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import {
encodePngAsync,
retainPngWorker,
terminatePngWorker,
transcodeScreenshotToPngAsync,
} from './png-worker-client.ts';
import { cropPngBytes } from './png-crop-bytes.ts';
import { computeScreenshotDiffPixels } from './screenshot-diff-pixels.ts';
Expand Down Expand Up @@ -148,39 +147,6 @@ test('decodePngAsync rejects invalid PNG data with the canonical decode AppError
);
});

test('transcodeScreenshotToPngAsync matches the synchronous transcoder byte for byte', async () => {
const { encode } = await import('jpeg-js');
const { transcodeScreenshotToPng } = await import('./screenshot-image.ts');
const rgba = Buffer.alloc(9 * 7 * 4, 0x66);
const jpeg = encode({ width: 9, height: 7, data: rgba }, 90).data;

const fromWorker = await runOnWorkerThread('jpeg-to-png', () =>
transcodeScreenshotToPngAsync(jpeg, 'fixture'),
);

assert.deepEqual(fromWorker, transcodeScreenshotToPng(jpeg, 'fixture'));
assert.equal(PNG.sync.read(fromWorker).width, 9);
});

test('transcodeScreenshotToPngAsync returns a PNG untouched without a worker round trip', async () => {
const png = PNG.sync.write(new PNG({ width: 2, height: 2 }));

assert.equal(await transcodeScreenshotToPngAsync(png, 'fixture'), png);
});

test('transcodeScreenshotToPngAsync rejects a corrupt JPEG with the canonical decode AppError', async () => {
const corrupt = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(16, 0)]);

await assert.rejects(transcodeScreenshotToPngAsync(corrupt, 'fixture'), (error) => {
assert.equal(error instanceof AppError, true);
assert.equal((error as AppError).code, 'COMMAND_FAILED');
assert.match((error as AppError).message, /Failed to decode fixture as JPEG/);
assert.equal((error as AppError).details?.label, 'fixture');
assert.ok(String((error as AppError).details?.reason).length > 0);
return true;
});
});

test('decodeScreenshotImageAsync answers the synchronous decoder for both containers', async () => {
const { encode } = await import('jpeg-js');
const { decodeScreenshotImage } = await import('./screenshot-image.ts');
Expand Down
18 changes: 1 addition & 17 deletions packages/capture-kit/src/png-worker-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { Worker } from 'node:worker_threads';
import { AppError, toAppErrorCode } from '@agent-device/kernel/errors';
import type { Rect } from '@agent-device/kernel/snapshot';
import { resolveInternalEntryModulePath } from './internal-entry.ts';
import { decodePng, hasPngSignature, PNG } from './png.ts';
import { decodePng, PNG } from './png.ts';
import {
computeScreenshotDiffPixels,
type ScreenshotDiffPixelsJob,
Expand Down Expand Up @@ -315,19 +315,3 @@ export async function computeScreenshotDiffPixelsAsync(
}));
return { ...result, diffData: toBuffer(result.diffData) };
}

/**
* PNG bytes for a provider screenshot in whatever container it arrived in. A PNG returns as is
* without a worker round trip; a JPEG is decoded and re-encoded on the worker so a full-resolution
* capture never blocks the daemon event loop. Decode failures carry the canonical `AppError`.
*/
export async function transcodeScreenshotToPngAsync(bytes: Buffer, label: string): Promise<Buffer> {
if (hasPngSignature(bytes)) return bytes;
const result = await runPngJob({ kind: 'jpeg-to-png', image: bytes, label }, async () => {
// Read on demand so the JPEG decoder stays out of the import closure of every entry that only
// needs the worker's other jobs.
const { transcodeScreenshotToPng } = await import('./screenshot-image.ts');
return { kind: 'jpeg-to-png', png: transcodeScreenshotToPng(bytes, label) };
});
return toBuffer(result.png);
}
9 changes: 3 additions & 6 deletions packages/capture-kit/src/png-worker-contract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import type { PngRgbDifferenceResult } from './png-rgb-difference.ts';
/**
* Message contract between the daemon-side PNG worker client
* (`png-worker-client.ts`) and the worker thread entry (`png-worker.ts`).
* One message = one decode, encode, crop, diff, or transcode job. Binary payloads cross the
* One message = one decode, encode, crop, or diff job. Binary payloads cross the
* thread boundary via structured clone (or transfer), so `Buffer` fields
* arrive as plain `Uint8Array` views on the receiving side.
*/
Expand All @@ -21,9 +21,7 @@ export type PngWorkerJob =
| { kind: 'encode'; width: number; height: number; data: Uint8Array }
| { kind: 'crop'; png: Uint8Array; label: string; box: Rect }
| { kind: 'rgb-difference'; firstPng: Uint8Array; secondPng: Uint8Array; label: string }
| ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsJob)
// A provider screenshot in whatever container it arrived in; answers PNG bytes.
| { kind: 'jpeg-to-png'; image: Uint8Array; label: string };
| ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsJob);

export type PngWorkerJobResult =
| { kind: 'decode'; width: number; height: number; data: Uint8Array }
Expand All @@ -32,8 +30,7 @@ export type PngWorkerJobResult =
// A crop answers `null` when the box already covers the image, so the caller keeps the file.
| { kind: 'crop'; png: Uint8Array | null }
| ({ kind: 'rgb-difference' } & PngRgbDifferenceResult)
| ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsResult)
| { kind: 'jpeg-to-png'; png: Uint8Array };
| ({ kind: 'diff-pixels' } & ScreenshotDiffPixelsResult);

export type PngWorkerJobKind = PngWorkerJob['kind'];

Expand Down
9 changes: 0 additions & 9 deletions packages/capture-kit/src/png-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,6 @@ async function runJob(request: PngWorkerRequest): Promise<PngWorkerJobResult> {
case 'diff-pixels': {
return { kind: 'diff-pixels', ...computeScreenshotDiffPixels(request) };
}
case 'jpeg-to-png': {
const { transcodeScreenshotToPng } = await import('./screenshot-image.ts');
return {
kind: 'jpeg-to-png',
png: transcodeScreenshotToPng(toBuffer(request.image), request.label),
};
}
}
}

Expand Down Expand Up @@ -102,8 +95,6 @@ function resultBufferViews(result: PngWorkerJobResult): Uint8Array[] {
return [];
case 'diff-pixels':
return [result.diffData, result.diffMask];
case 'jpeg-to-png':
return [result.png];
}
}

Expand Down
37 changes: 4 additions & 33 deletions packages/capture-kit/src/screenshot-image.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,7 @@ import { expect, test } from 'vitest';
import fc from 'fast-check';
import { encode as encodeJpeg } from 'jpeg-js';
import { PNG } from './png.ts';
import {
decodeScreenshotImage,
detectScreenshotImageFormat,
transcodeScreenshotToPng,
} from './screenshot-image.ts';
import { decodeScreenshotImage, detectScreenshotImageFormat } from './screenshot-image.ts';

function solidRgba(width: number, height: number, rgba: readonly [number, number, number, number]) {
const data = Buffer.alloc(width * height * 4);
Expand All @@ -24,32 +20,9 @@ function thrownBy(run: () => unknown): unknown {
throw new Error('expected the call to throw');
}

test('a JPEG screenshot becomes a PNG of the same size with the same picture', () => {
const jpeg = encodeJpeg({ width: 6, height: 4, data: solidRgba(6, 4, [200, 30, 30, 255]) }, 100);
expect(detectScreenshotImageFormat(jpeg.data)).toBe('jpeg');

const png = transcodeScreenshotToPng(jpeg.data, 'test screenshot');

expect(detectScreenshotImageFormat(png)).toBe('png');
const decoded = PNG.sync.read(png);
expect([decoded.width, decoded.height]).toEqual([6, 4]);
const [r = -1, g = -1, b = -1, a = -1] = decoded.data.subarray(0, 4);
// JPEG is lossy; a flat field survives within a few levels per channel.
expect(Math.abs(r - 200)).toBeLessThanOrEqual(4);
expect(Math.abs(g - 30)).toBeLessThanOrEqual(4);
expect(Math.abs(b - 30)).toBeLessThanOrEqual(4);
expect(a).toBe(255);
});

test('a PNG screenshot passes through byte for byte', () => {
const png = PNG.sync.write(new PNG({ width: 3, height: 2 }));

expect(transcodeScreenshotToPng(png, 'test screenshot')).toBe(png);
});

test('bytes that are neither container are refused with a typed error carrying the label', () => {
expect(
thrownBy(() => transcodeScreenshotToPng(Buffer.from('GIF89a'), 'Limrun iOS screenshot')),
thrownBy(() => decodeScreenshotImage(Buffer.from('GIF89a'), 'Limrun iOS screenshot')),
).toMatchObject({
code: 'COMMAND_FAILED',
message: 'Limrun iOS screenshot is neither PNG nor JPEG',
Expand All @@ -61,7 +34,7 @@ test('a valid SOI marker followed by garbage is refused with the typed decode er
const garbage = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(32, 0x41)]);
expect(detectScreenshotImageFormat(garbage)).toBe('jpeg');

expect(thrownBy(() => transcodeScreenshotToPng(garbage, 'Limrun iOS screenshot'))).toMatchObject({
expect(thrownBy(() => decodeScreenshotImage(garbage, 'Limrun iOS screenshot'))).toMatchObject({
code: 'COMMAND_FAILED',
message: 'Failed to decode Limrun iOS screenshot as JPEG',
details: { label: 'Limrun iOS screenshot', reason: expect.any(String) },
Expand All @@ -72,9 +45,7 @@ test('a truncated JPEG body is refused with the same typed decode error', () =>
const jpeg = encodeJpeg({ width: 6, height: 4, data: solidRgba(6, 4, [10, 200, 30, 255]) }, 100);
const truncated = jpeg.data.subarray(0, 24);

expect(
thrownBy(() => transcodeScreenshotToPng(truncated, 'Limrun iOS screenshot')),
).toMatchObject({
expect(thrownBy(() => decodeScreenshotImage(truncated, 'Limrun iOS screenshot'))).toMatchObject({
code: 'COMMAND_FAILED',
message: 'Failed to decode Limrun iOS screenshot as JPEG',
});
Expand Down
16 changes: 0 additions & 16 deletions packages/capture-kit/src/screenshot-image.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,22 +30,6 @@ export function decodeScreenshotImage(bytes: Buffer, label: string, maxPixels?:
throw unsupportedContainer(label, bytes);
}

/**
* Returns PNG bytes for a screenshot a provider handed back in whatever container it prefers. PNG
* passes through untouched; JPEG is decoded and re-encoded losslessly from the decoded pixels, so
* every PNG-only reader downstream (size, crop, overlay) sees the format the path promises. Those
* readers rewrite the file in place, which is why a JPEG cannot be handed to them undecoded.
*
* Synchronous and CPU-bound: daemon request paths reach it through the PNG worker
* (`transcodeScreenshotToPngAsync` in `png-worker-client.ts`), like every other codec job.
*/
export function transcodeScreenshotToPng(bytes: Buffer, label: string): Buffer {
const format = detectScreenshotImageFormat(bytes);
if (format === 'png') return bytes;
if (format === 'jpeg') return PNG.sync.write(decodeJpegImage(bytes, label));
throw unsupportedContainer(label, bytes);
}

/** A JPEG signature does not prove a decodable body; a failure keeps the label and the decoder's reason. */
function decodeJpegImage(bytes: Buffer, label: string, maxPixels?: number): PNG {
try {
Expand Down
2 changes: 1 addition & 1 deletion packages/command-registry/src/flag-definitions-target.ts
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,7 @@ export const TARGET_FLAG_DEFINITIONS: readonly FlagDefinition[] = [
type: 'string',
usageLabel: '--launch-console <path>',
usageDescription:
'open: capture the initial local iOS simulator launch console window to a file',
'open: capture the initial local or Limrun iOS simulator launch console window to a file',
projectConfig: false,
recorded: false,
},
Expand Down
2 changes: 1 addition & 1 deletion packages/command-surface/src/capture/alert.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { AppError } from '@agent-device/kernel/errors';
const ALERT_COMMAND_NAME = 'alert';

const alertCommandDescription =
"Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. Local Apple XCTest and Limrun iOS choose recognized English labels by preference (Allow While Using App before Allow Once); on iOS simulators, local XCTest also recognizes the device's preferred languages' wording of those labels from the OS permission-prompt strings, on any alert, while Limrun iOS stays English; equal-ranked choices and unknown labels refuse, including a lone unknown button. Open, Done, and Confirm… accept; Keep Browsing, Deny, and Skip dismiss. On the Apple XCUITest runner, dismiss taps only a recognized decline label (Cancel, Don't Allow, Ask App Not to Track, and similar) or the close element of a popover whose accessibility marker reads exactly \"dismiss popup\"; when no button fits the action, accept and dismiss answer ALERT_BUTTON_NOT_FOUND having tapped nothing — read labels with alert get, then press the button by label. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.";
"Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. Apple XCTest and Limrun iOS choose recognized English labels by preference (Allow While Using App before Allow Once); on iOS simulators, the runner also recognizes the device's preferred languages' wording of those labels from the OS permission-prompt strings, on any alert; equal-ranked choices and unknown labels refuse, including a lone unknown button. Open, Done, and Confirm… accept; Keep Browsing, Deny, and Skip dismiss. On the Apple XCUITest runner, dismiss taps only a recognized decline label (Cancel, Don't Allow, Ask App Not to Track, and similar) or the close element of a popover whose accessibility marker reads exactly \"dismiss popup\"; when no button fits the action, accept and dismiss answer ALERT_BUTTON_NOT_FOUND having tapped nothing — read labels with alert get, then press the button by label. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.";

const alertCommandMetadata = defineFieldCommandMetadata(
ALERT_COMMAND_NAME,
Expand Down
Loading
Loading