From 1600d40d077b02a547349396b0677d15902bb7ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 4 Sep 2026 14:30:46 +0200 Subject: [PATCH] feat(ios): route simulator snapshots through AX bridge --- README.md | 2 +- .../adr/0004-ios-snapshot-backend-strategy.md | 34 +-- .../capture-kit/src/ios-snapshot-planning.ts | 10 + packages/contracts/src/interactor-types.ts | 8 +- packages/kernel/src/snapshot.ts | 7 +- .../platform-apple/src/runtime-snapshot.ts | 11 +- packages/platform-apple/src/runtime.ts | 18 +- .../platform-apple/src/snapshot-route.test.ts | 190 ++++++++++++++++ packages/platform-apple/src/snapshot-route.ts | 214 ++++++++++++++++++ .../src/snapshot-source-facade.ts | 4 +- .../src/snapshot-target.test.ts | 53 +++++ .../platform-apple/src/snapshot-target.ts | 91 ++++++++ src/core/__tests__/snapshot-state.test.ts | 28 +++ src/core/snapshot-state.ts | 12 +- src/daemon/deferred-interaction-outcome.ts | 8 +- .../interaction-ios-tap-outcome.test.ts | 28 +++ .../internal/interaction-ios-tap-outcome.ts | 26 +++ src/daemon/types.ts | 4 +- src/snapshot/ios-snapshot-runtime.ts | 7 +- 19 files changed, 720 insertions(+), 35 deletions(-) create mode 100644 packages/platform-apple/src/snapshot-route.test.ts create mode 100644 packages/platform-apple/src/snapshot-route.ts create mode 100644 packages/platform-apple/src/snapshot-target.test.ts create mode 100644 packages/platform-apple/src/snapshot-target.ts diff --git a/README.md b/README.md index fc6a25032e..a5f9cb069c 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ The same session and evidence model works at every step: the agent explores the ## How it works -`agent-device` keeps device state in sessions. It sends commands to XCTest on iOS and tvOS, ADB and the snapshot helper on Android, HDC and ArkUI `uitest` on HarmonyOS, Vega CLI/VDA on the Vega Virtual Device, a local helper on macOS, and AT-SPI on Linux. +`agent-device` keeps device state in sessions. It uses a local accessibility bridge for iOS Simulator snapshots and XCTest for iOS interactions, physical iOS, and tvOS; ADB and the snapshot helper on Android; HDC and ArkUI `uitest` on HarmonyOS; Vega CLI/VDA on the Vega Virtual Device; a local helper on macOS; and AT-SPI on Linux. Support depth varies by target. Newer backends such as HarmonyOS and Vega OS cover a subset of commands; run `agent-device capabilities --platform ` to see what a target supports. diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index b47e9afcbe..7eb1aa6912 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -2,17 +2,18 @@ ## Status -Accepted. Amended after iOS snapshot capture was simplified to two public modes: -regular interactive snapshots and raw diagnostic snapshots. +Accepted. Amended after local iOS Simulator acquisition moved to the host AX bridge while the +public surface remained two modes: regular interactive snapshots and raw diagnostic snapshots. -The runner owns capture-plan acquisition and backend fallback. Host-side iOS validation, semantic -presentation, and publication are owned by `@agent-device/capture-kit`; structured snapshot quality -verdicts make degraded or recovered output observable end to end. +The Apple platform runtime owns acquisition routing and its generation-scoped XCTest fallback. +Host-side iOS validation, semantic presentation, and publication are owned by +`@agent-device/capture-kit`; structured snapshot quality verdicts and fallback warnings make +degraded or recovered output observable end to end. ## Context -Agent Device exposes iOS UI state through snapshots produced by the long-lived XCTest runner. The -runner has two durable snapshot needs: +Agent Device exposes iOS UI state through host AX acquisition on local Simulators and the long-lived +XCTest runner everywhere else. The snapshot surface has two durable needs: - agent-facing regular context, where the important contract is the effective user-visible UI, fixed controls such as tab bars, and scroll-hidden hints for content outside visible scroll @@ -35,8 +36,13 @@ predictable. ## Decision -Keep XCTest as the default iOS automation runner and split iOS snapshot capture into explicit -strategies: +Keep XCTest as the iOS automation runner. Route eligible local iOS Simulator snapshots through the +host AX bridge, present them once through the shared TypeScript engine, and use one typed XCTest +fallback when bridge acquisition or presentation fails. Disable the bridge for that app generation +after fallback; a new app generation re-enables it. Physical devices, providers, custom-action +captures, and interactions remain on their existing owners. + +Keep the two public snapshot strategies explicit: - **Regular visible strategy**: use recursive XCTest snapshots, emit the effective user-visible tree plus visible ancestors and scroll-hidden hints, and fall back through the capture plan when @@ -51,12 +57,10 @@ strategies: carry the response, fail explicitly instead of silently truncating the tree at a hard node count. If XCTest reports a real AX serialization failure, preserve that error instead of pretending the UI is empty. -- **Future AX-service strategy**: treat Bluesky-class failures as evidence that XCTest is - not a complete semantic snapshot backend. A robust semantic fix should add a host-side simulator - accessibility backend, similar in role to existing simulator accessibility inspection tools, - and acquire its output as `RawAXNode` values. Every backend crosses the same - `SnapshotPresentation` construction boundary before producing wire-facing `PresentedNode` values. - That backend can be simulator-only; physical devices should use an equivalent non-XCTest semantic +- **Host AX strategy**: acquire local Simulator trees as raw facts through the bounded host bridge. + Every result crosses the same presentation boundary before publication. XCTest fallback carries + explicit source residue, and comparisons require matching producer, intent, app generation, + presentation key, and residue. Physical devices should use an equivalent non-XCTest semantic backend only if Apple exposes a supported channel. The daemon should make degraded output observable. If an iOS interactive snapshot contains only the diff --git a/packages/capture-kit/src/ios-snapshot-planning.ts b/packages/capture-kit/src/ios-snapshot-planning.ts index 3a4db636aa..11adc1f2b1 100644 --- a/packages/capture-kit/src/ios-snapshot-planning.ts +++ b/packages/capture-kit/src/ios-snapshot-planning.ts @@ -90,6 +90,16 @@ export function areIosSnapshotComparisonIdentitiesEqual( ); } +export function iosSnapshotComparisonIdentityKey(identity: IosSnapshotComparisonIdentity): string { + return JSON.stringify({ + producer: identity.producer, + intent: identity.intent, + lineage: identity.lineage, + presentationKey: identity.presentationKey, + residue: identity.residue.map(residueIdentity).sort(), + }); +} + export function buildIosSnapshotComparisonIdentity( input: IosSnapshotInput, request: IosSnapshotRequest, diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index 5f6cd6ed1f..7de5a223b2 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -9,8 +9,9 @@ import type { SessionSurface } from './session-surface.ts'; import type { BackendSnapshotResult } from './snapshot-types.ts'; import type { RunnerLogicalLeaseContext } from './runner-lease-context.ts'; import type { - IosProviderAcquisitionProducer, + IosAcquisitionProducer, IosSnapshotAcquisitionFacts, + IosSnapshotComparisonIdentity, } from './ios-snapshot.ts'; import type { RawSnapshotNode, @@ -182,6 +183,8 @@ export type SnapshotOptions = BaseSnapshotOptions & { includeRects?: boolean; includeHiddenContentHints?: boolean; surface?: SessionSurface; + /** Internal capture purpose; action outcomes always require the full tree. */ + acquisitionIntent?: 'full' | 'surface-observation'; }; /** @@ -251,11 +254,12 @@ export type KeyboardEnterResult = */ export type SnapshotResult = Omit & { nodes?: RawSnapshotNode[]; + comparisonIdentity?: IosSnapshotComparisonIdentity; } & SnapshotProvenance; export type SnapshotRuntimeAcquiredResult = Readonly<{ stage: 'acquired'; - acquisition: IosSnapshotAcquisitionFacts & Readonly<{ producer: IosProviderAcquisitionProducer }>; + acquisition: IosSnapshotAcquisitionFacts & Readonly<{ producer: IosAcquisitionProducer }>; }>; export type SnapshotRuntimeResult = SnapshotResult | SnapshotRuntimeAcquiredResult; diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 4fc96bcfdb..1a72384d7c 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -162,7 +162,10 @@ export type SnapshotNode = RawSnapshotNode & { * snapshot-provenance.test.ts). */ export type SnapshotProvenance = - | { backend: 'xctest'; producer: 'apple-runner' | 'appium-source' | 'limrun-ios-tree' } + | { + backend: 'xctest'; + producer: 'apple-runner' | 'simulator-ax-bridge' | 'appium-source' | 'limrun-ios-tree'; + } | { backend: 'android'; producer: 'android-uiautomator' | 'appium-source' } | { backend: 'harmonyos-arkui'; producer: 'harmonyos-uitest' } | { backend: 'macos-helper'; producer: 'macos-helper' } @@ -238,6 +241,8 @@ export type SnapshotState = { snapshotQuality?: SnapshotQualityVerdict; comparisonSafe?: boolean; presentationKey?: string; + /** Opaque equality key for iOS acquisition and presentation lineage. */ + comparisonKey?: string; /** * Android: the capture is an occluding system surface (notification shade, quick settings) * rather than app content. Consumers that surface this tree to the agent must disclose the diff --git a/packages/platform-apple/src/runtime-snapshot.ts b/packages/platform-apple/src/runtime-snapshot.ts index 245b445614..f31b419db8 100644 --- a/packages/platform-apple/src/runtime-snapshot.ts +++ b/packages/platform-apple/src/runtime-snapshot.ts @@ -14,11 +14,13 @@ import type { PlatformRuntimeOperations, } from '@agent-device/contracts/platform-runtime-operations'; import { isMacOs, type DeviceInfo } from '@agent-device/kernel/device'; +import type { AppleSnapshotRoute } from './snapshot-route.ts'; /** Apple-owned selection between app snapshots and explicit macOS surface snapshots. */ export function bindAppleSnapshotRuntime( host: PlatformRuntimeHost, request: Readonly<{ device: DeviceInfo; signal: AbortSignal }>, + route?: AppleSnapshotRoute, ): SnapshotRuntimeOperation { const appSnapshot = bindLocalSnapshotInteractor({ device: request.device, @@ -37,7 +39,14 @@ export function bindAppleSnapshotRuntime( captureSnapshotSignal(request.signal, input), ); } - return await appSnapshot.captureSnapshot(input); + if (!route) return await appSnapshot.captureSnapshot(input); + const signal = captureSnapshotSignal(request.signal, input); + return await route.capture( + request.device, + input, + signal, + async (fallbackInput) => await appSnapshot.captureSnapshot(fallbackInput), + ); }; return Object.freeze({ captureSnapshot, diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index 1ba6824045..d9d2992374 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -70,6 +70,7 @@ import { bindAppleFindTextRuntime, bindAppleSnapshotRuntime, } from './runtime-snapshot.ts'; +import { createAppleSnapshotRoute } from './snapshot-route.ts'; const owner = localRuntimeOwner('apple'); const available = Object.freeze({ available: true } as const); @@ -268,6 +269,7 @@ function appleFocusFact(device: DeviceInfo): RuntimeOperationFact { export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformRuntimeOwner { const appLogs = createAppleAppLogRuntime(host); + const snapshotRoute = createAppleSnapshotRoute(host); const inspectFacts = async (device: DeviceInfo) => { const logs = await appLogs.inspectFacts(device); const deployment = appleAppDeploymentFacts(device); @@ -375,10 +377,14 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR }), ), ...whenAdmitted(facts.operations.captureSnapshot, () => - bindAppleSnapshotRuntime(host, { - device: request.device, - signal: request.scope.signal, - }), + bindAppleSnapshotRuntime( + host, + { + device: request.device, + signal: request.scope.signal, + }, + snapshotRoute, + ), ), ...whenAdmitted(facts.operations.captureScreenshot, () => bindLocalScreenshotInteractor({ @@ -490,7 +496,9 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR [Symbol.asyncDispose]: async () => await logs[Symbol.asyncDispose](), }) satisfies DeviceBinding; }, - shutdown: async () => await appLogs.shutdown(), + shutdown: async () => { + await Promise.all([appLogs.shutdown(), snapshotRoute.shutdown()]); + }, }); } diff --git a/packages/platform-apple/src/snapshot-route.test.ts b/packages/platform-apple/src/snapshot-route.test.ts new file mode 100644 index 0000000000..a527c48c4e --- /dev/null +++ b/packages/platform-apple/src/snapshot-route.test.ts @@ -0,0 +1,190 @@ +import { expect, test, vi } from 'vitest'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { platformRuntimeHostFixture } from './runtime.fixtures.ts'; +import { createAppleSnapshotRoute } from './snapshot-route.ts'; +import type { SimulatorSnapshotSource, SnapshotSourceOutcome } from './snapshot-source-facade.ts'; + +const ios = { + platform: 'apple', + appleOs: 'ios', + id: 'ios-1', + name: 'iPhone', + kind: 'simulator', + target: 'mobile', + booted: true, +} as const satisfies DeviceInfo; + +const target = { + udid: ios.id, + runtime: 'iOS 26.0', + pid: 42, + generation: '42:launch-a', + targetId: `${ios.id}:com.example.app`, +} as const; + +const input = { options: { appBundleId: 'com.example.app' } } as const; + +test('eligible simulator capture publishes bridge acquisition without touching XCTest', async () => { + const acquired = bridgeAcquisition(); + const source = sourceReturning(acquired); + const presentIosAcquisition = vi.fn(async () => ({ + backend: 'xctest' as const, + producer: 'simulator-ax-bridge' as const, + nodes: [{ index: 0, type: 'Application' }], + })); + const fallback = vi.fn(async () => runnerResult()); + const route = createAppleSnapshotRoute( + { + ...platformRuntimeHostFixture(), + snapshot: { captureSurface: vi.fn(), presentIosAcquisition }, + }, + { source, resolveTarget: vi.fn(async () => target) }, + ); + + await expect(route.capture(ios, input, signal(), fallback)).resolves.toMatchObject({ + producer: 'simulator-ax-bridge', + }); + expect(presentIosAcquisition).toHaveBeenCalledWith(acquired, input.options); + expect(fallback).not.toHaveBeenCalled(); +}); + +test('typed bridge failure falls back once and disables retries for that app generation', async () => { + const source = sourceReturning({ + stage: 'failed', + failure: { kind: 'transport-failure', code: 'bridge-disconnected' }, + }); + const fallback = vi.fn(async () => runnerResult()); + const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { + source, + resolveTarget: vi.fn(async () => target), + }); + + const first = await route.capture(ios, input, signal(), fallback); + const second = await route.capture(ios, input, signal(), fallback); + + expect(source.acquire).toHaveBeenCalledOnce(); + expect(fallback).toHaveBeenCalledTimes(2); + expect(first.warnings).toEqual([ + 'Simulator AX snapshot unavailable (bridge-disconnected); used XCTest for this app generation.', + ]); + expect(first.comparisonIdentity).toMatchObject({ + producer: 'apple-runner', + lineage: { generation: target.generation }, + residue: [{ kind: 'fallback-source', producer: 'apple-runner' }], + }); + expect(second.comparisonIdentity).toMatchObject({ + producer: 'apple-runner', + lineage: { generation: target.generation }, + }); +}); + +test('a new app generation re-enables the bridge', async () => { + const source = sourceReturning({ + stage: 'failed', + failure: { kind: 'stale-target', code: 'target-generation-changed' }, + }); + const resolveTarget = vi + .fn() + .mockResolvedValueOnce(target) + .mockResolvedValueOnce(target) + .mockResolvedValueOnce({ ...target, pid: 84, generation: '84:launch-b' }); + const fallback = vi.fn(async () => runnerResult()); + const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { + source, + resolveTarget, + }); + + await route.capture(ios, input, signal(), fallback); + await route.capture(ios, input, signal(), fallback); + await route.capture(ios, input, signal(), fallback); + + expect(source.acquire).toHaveBeenCalledTimes(2); +}); + +test('target-resolution fallback remains incomparable with a bridge publication', async () => { + const source = sourceReturning(bridgeAcquisition()); + const fallback = vi.fn(async () => runnerResult()); + const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { + source, + resolveTarget: vi.fn(async () => { + throw new Error('launch job unavailable'); + }), + }); + + const result = await route.capture(ios, input, signal(), fallback); + + expect(source.acquire).not.toHaveBeenCalled(); + expect(result.comparisonIdentity).toMatchObject({ + producer: 'apple-runner', + lineage: { targetId: target.targetId }, + residue: [{ kind: 'fallback-source', producer: 'apple-runner' }], + }); +}); + +test('runtime shutdown closes the process-owned bridge source', async () => { + const source = sourceReturning(bridgeAcquisition()); + const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { source }); + + await route.shutdown(); + + expect(source.close).toHaveBeenCalledOnce(); +}); + +test('cancelled acquisition does not start a fallback after the request aborts', async () => { + const controller = new AbortController(); + const source = sourceReturning({ + stage: 'failed', + failure: { kind: 'cancelled', code: 'abort-signal' }, + }); + vi.mocked(source.acquire).mockImplementation(async () => { + controller.abort(new DOMException('request ended', 'AbortError')); + return { stage: 'failed', failure: { kind: 'cancelled', code: 'abort-signal' } }; + }); + const fallback = vi.fn(async () => runnerResult()); + const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { + source, + resolveTarget: vi.fn(async () => target), + }); + + await expect(route.capture(ios, input, controller.signal, fallback)).rejects.toThrow( + 'request ended', + ); + expect(fallback).not.toHaveBeenCalled(); +}); + +function bridgeAcquisition(): Extract { + return { + stage: 'acquired', + acquisition: { + producer: 'simulator-ax-bridge', + intent: 'full', + hint: { + projection: 'regular', + rawTraversalDepth: null, + regularPresentedDepth: null, + interactiveOnly: false, + customActions: false, + acquisitionIntent: 'full', + }, + nodes: [{ index: 0, type: 'Application' }], + truncated: false, + viewport: { kind: 'reported', rect: { x: 0, y: 0, width: 100, height: 200 } }, + lineage: { targetId: target.targetId, generation: target.generation }, + residue: [], + }, + }; +} + +function sourceReturning( + outcome: Awaited>, +): SimulatorSnapshotSource { + return { acquire: vi.fn(async () => outcome), close: vi.fn(async () => {}) }; +} + +function runnerResult() { + return { backend: 'xctest' as const, producer: 'apple-runner' as const, nodes: [] }; +} + +function signal(): AbortSignal { + return new AbortController().signal; +} diff --git a/packages/platform-apple/src/snapshot-route.ts b/packages/platform-apple/src/snapshot-route.ts new file mode 100644 index 0000000000..6a8c7ce9ab --- /dev/null +++ b/packages/platform-apple/src/snapshot-route.ts @@ -0,0 +1,214 @@ +import type { PlatformRuntimeHost } from '@agent-device/contracts/platform-runtime-operations'; +import type { + CaptureSnapshotInput, + SnapshotResult, + SnapshotRuntimeAcquiredResult, +} from '@agent-device/contracts/snapshot-runtime'; +import type { + IosSnapshotComparisonIdentity, + IosSnapshotLineage, +} from '@agent-device/contracts/ios-snapshot'; +import { + buildIosSnapshotPresentationKey, + createIosSnapshotRequest, + deriveIosCaptureHint, +} from '@agent-device/capture-kit/ios-snapshot-planning'; +import { emitDiagnostic, withDiagnosticTimer } from '@agent-device/host-kit/diagnostics'; +import { AppError } from '@agent-device/kernel/errors'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { + createSimulatorSnapshotSource, + type SimulatorSnapshotSource, + type SnapshotSourceFailure, +} from './snapshot-source-facade.ts'; +import { resolveSimulatorSnapshotTarget, type SimulatorSnapshotTarget } from './snapshot-target.ts'; + +type SnapshotFallback = (input: CaptureSnapshotInput) => Promise; + +export type AppleSnapshotRoute = Readonly<{ + capture( + device: DeviceInfo, + input: CaptureSnapshotInput, + signal: AbortSignal, + fallback: SnapshotFallback, + ): Promise; + shutdown(): Promise; +}>; + +export function createAppleSnapshotRoute( + host: PlatformRuntimeHost, + options: Readonly<{ + source?: SimulatorSnapshotSource; + resolveTarget?: typeof resolveSimulatorSnapshotTarget; + }> = {}, +): AppleSnapshotRoute { + const source = options.source ?? createSimulatorSnapshotSource(); + const resolveTarget = options.resolveTarget ?? resolveSimulatorSnapshotTarget; + const disabledGenerations = new Set(); + const latestGeneration = new Map(); + + return Object.freeze({ + shutdown: async () => await source.close(), + capture: async (device, input, signal, fallback) => { + if (!isEligible(device, input)) return await fallback(input); + let target: SimulatorSnapshotTarget; + try { + target = await resolveTarget(device, input.options!.appBundleId!, signal); + } catch (error) { + emitRouteDiagnostic('target-resolution-failed', device, undefined, error); + return await runFallback( + input, + fallback, + { targetId: `${device.id}:${input.options!.appBundleId!}` }, + requestFor(input), + 'target-resolution-failed', + ); + } + rebaselineGeneration(target, latestGeneration, disabledGenerations); + const circuitKey = generationKey(target); + if (disabledGenerations.has(circuitKey)) { + return await runFallback(input, fallback, target, requestFor(input), 'circuit-disabled'); + } + + const request = requestFor(input); + const outcome = await source.acquire({ + target, + hint: deriveIosCaptureHint(request), + signal, + }); + if (outcome.stage === 'failed') { + if (outcome.failure.kind === 'cancelled') { + signal.throwIfAborted(); + throw new AppError('COMMAND_FAILED', 'Simulator AX snapshot acquisition was cancelled.', { + reason: outcome.failure.code, + ...outcome.failure.details, + }); + } + return await fallbackAfterFailure( + input, + fallback, + target, + request, + outcome.failure, + disabledGenerations, + ); + } + try { + return await withDiagnosticTimer( + 'ios.snapshot-source.present', + async () => + await host.snapshot.presentIosAcquisition( + outcome as SnapshotRuntimeAcquiredResult, + input.options, + ), + { producer: 'simulator-ax-bridge' }, + ); + } catch (error) { + return await fallbackAfterFailure( + input, + fallback, + target, + request, + { kind: 'malformed-tree', code: 'presentation-invariant' }, + disabledGenerations, + error, + ); + } + }, + }); +} + +function isEligible(device: DeviceInfo, input: CaptureSnapshotInput): boolean { + return ( + device.platform === 'apple' && + device.appleOs === 'ios' && + device.kind === 'simulator' && + Boolean(input.options?.appBundleId) && + input.options?.customActions !== true && + input.options?.preferredBackend === undefined + ); +} + +async function fallbackAfterFailure( + input: CaptureSnapshotInput, + fallback: SnapshotFallback, + target: SimulatorSnapshotTarget, + request: ReturnType, + failure: SnapshotSourceFailure, + disabledGenerations: Set, + cause?: unknown, +): Promise { + disabledGenerations.add(generationKey(target)); + emitRouteDiagnostic(failure.code, { id: target.udid }, target.generation, cause, failure.details); + return await runFallback(input, fallback, target, request, failure.code); +} + +async function runFallback( + input: CaptureSnapshotInput, + fallback: SnapshotFallback, + lineage: IosSnapshotLineage, + request: ReturnType, + reason: string, +): Promise { + const result = await fallback(input); + const comparisonIdentity: IosSnapshotComparisonIdentity = Object.freeze({ + producer: 'apple-runner', + intent: request.acquisitionIntent, + lineage: Object.freeze({ ...lineage }), + presentationKey: buildIosSnapshotPresentationKey(request), + residue: Object.freeze([{ kind: 'fallback-source', producer: 'apple-runner' } as const]), + }); + const warning = `Simulator AX snapshot unavailable (${reason}); used XCTest for this app generation.`; + return { + ...result, + comparisonIdentity, + warnings: [...(result.warnings ?? []), warning], + }; +} + +function requestFor(input: CaptureSnapshotInput) { + return createIosSnapshotRequest({ + raw: input.options?.raw, + interactiveOnly: input.options?.interactiveOnly, + depth: input.options?.depth, + scope: input.options?.scope, + customActions: input.options?.customActions, + acquisitionIntent: input.options?.acquisitionIntent, + }); +} + +function rebaselineGeneration( + target: SimulatorSnapshotTarget, + latestGeneration: Map, + disabledGenerations: Set, +): void { + const previous = latestGeneration.get(target.targetId); + if (previous && previous !== target.generation) { + disabledGenerations.delete(`${target.targetId}:${previous}`); + } + latestGeneration.set(target.targetId, target.generation); +} + +function generationKey(target: SimulatorSnapshotTarget): string { + return `${target.targetId}:${target.generation}`; +} + +function emitRouteDiagnostic( + reason: string, + device: Pick, + generation?: string, + error?: unknown, + details?: Readonly>, +): void { + emitDiagnostic({ + level: 'debug', + phase: 'ios_snapshot_route_fallback', + data: { + reason, + deviceId: device.id, + ...(generation ? { generation } : {}), + ...(error ? { error: error instanceof Error ? error.message : String(error) } : {}), + ...(details ? { details } : {}), + }, + }); +} diff --git a/packages/platform-apple/src/snapshot-source-facade.ts b/packages/platform-apple/src/snapshot-source-facade.ts index eb056ea388..dc58e1edcd 100644 --- a/packages/platform-apple/src/snapshot-source-facade.ts +++ b/packages/platform-apple/src/snapshot-source-facade.ts @@ -1,6 +1,6 @@ /** - * Dormant Simulator AX acquisition. The implementation is loaded only when a caller explicitly - * creates the source; importing this facet keeps the platform package's startup surface inert. + * Lazy Simulator AX acquisition. The implementation is loaded on the first eligible local iOS + * Simulator snapshot; importing this facet keeps the platform package's startup surface inert. */ export type { SnapshotSourceFailure, diff --git a/packages/platform-apple/src/snapshot-target.test.ts b/packages/platform-apple/src/snapshot-target.test.ts new file mode 100644 index 0000000000..e3c9b070ff --- /dev/null +++ b/packages/platform-apple/src/snapshot-target.test.ts @@ -0,0 +1,53 @@ +import { expect, test, vi } from 'vitest'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { createLocalAppleToolProvider, withAppleToolProvider } from './core/tool-provider.ts'; +import { resolveSimulatorSnapshotTarget } from './snapshot-target.ts'; + +const ios = { + platform: 'apple', + appleOs: 'ios', + id: 'ios-1', + name: 'iPhone', + kind: 'simulator', + target: 'mobile', + booted: true, +} as const satisfies DeviceInfo; + +test('resolves exact app pid, runtime, and launch generation from simctl', async () => { + const run = vi.fn(async (args: string[]) => { + if (args[0] === 'spawn') { + return { + stdout: [ + '90\t0\tUIKitApplication:com.example.app.beta[wrong][rb-legacy]', + '42\t0\tUIKitApplication:com.example.app[launch-a][rb-legacy]', + ].join('\n'), + stderr: '', + exitCode: 0, + }; + } + return { + stdout: JSON.stringify({ + devices: { + 'com.apple.CoreSimulator.SimRuntime.iOS-26-0': [{ udid: ios.id }], + }, + }), + stderr: '', + exitCode: 0, + }; + }); + const provider = createLocalAppleToolProvider({ simctl: { run } }); + + const result = await withAppleToolProvider( + provider, + async () => + await resolveSimulatorSnapshotTarget(ios, 'com.example.app', new AbortController().signal), + ); + + expect(result).toEqual({ + udid: 'ios-1', + runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-26-0', + pid: 42, + generation: '42:UIKitApplication:com.example.app[launch-a][rb-legacy]', + targetId: 'ios-1:com.example.app', + }); +}); diff --git a/packages/platform-apple/src/snapshot-target.ts b/packages/platform-apple/src/snapshot-target.ts new file mode 100644 index 0000000000..8507822c91 --- /dev/null +++ b/packages/platform-apple/src/snapshot-target.ts @@ -0,0 +1,91 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; +import { runSimctl } from './core/apps-simctl.ts'; + +const TARGET_PROBE_TIMEOUT_MS = 3_000; +const runtimeByDevice = new Map>(); + +export type SimulatorSnapshotTarget = Readonly<{ + udid: string; + runtime: string; + pid: number; + generation: string; + targetId: string; +}>; + +export async function resolveSimulatorSnapshotTarget( + device: DeviceInfo, + appBundleId: string, + signal: AbortSignal, +): Promise { + const [jobs, runtime] = await Promise.all([ + runSimctl(device, ['spawn', device.id, 'launchctl', 'list'], { + allowFailure: true, + signal, + timeoutMs: TARGET_PROBE_TIMEOUT_MS, + }), + readSimulatorRuntime(device, signal), + ]); + if (jobs.exitCode !== 0) { + throw targetError('simulator-target-probe-failed', device, appBundleId); + } + const job = readApplicationJob(jobs.stdout, appBundleId); + if (!job) { + throw targetError('simulator-target-unavailable', device, appBundleId); + } + return Object.freeze({ + udid: device.id, + runtime, + pid: job.pid, + generation: `${job.pid}:${job.label}`, + targetId: `${device.id}:${appBundleId}`, + }); +} + +async function readSimulatorRuntime(device: DeviceInfo, signal: AbortSignal): Promise { + const existing = runtimeByDevice.get(device.id); + if (existing) return await existing; + const pending = runSimctl(device, ['list', 'devices', '-j'], { + allowFailure: true, + signal, + timeoutMs: TARGET_PROBE_TIMEOUT_MS, + }).then((result) => { + if (result.exitCode !== 0) throw targetError('simulator-runtime-probe-failed', device, ''); + const payload = JSON.parse(result.stdout) as { + devices?: Record>; + }; + const runtime = Object.entries(payload.devices ?? {}).find(([, devices]) => + devices.some((candidate) => candidate.udid === device.id), + )?.[0]; + if (!runtime) throw targetError('simulator-runtime-unavailable', device, ''); + return runtime; + }); + runtimeByDevice.set(device.id, pending); + try { + return await pending; + } catch (error) { + runtimeByDevice.delete(device.id); + throw error; + } +} + +function readApplicationJob( + output: string, + appBundleId: string, +): { pid: number; label: string } | undefined { + for (const line of output.split('\n')) { + const [pidText, , label] = line.trim().split(/\s+/); + if (!pidText || !label || !label.startsWith(`UIKitApplication:${appBundleId}[`)) continue; + const pid = Number(pidText); + if (Number.isSafeInteger(pid) && pid > 0) return { pid, label }; + } + return undefined; +} + +function targetError(reason: string, device: DeviceInfo, appBundleId: string): AppError { + return new AppError('COMMAND_FAILED', 'Unable to resolve the running iOS Simulator app.', { + reason, + deviceId: device.id, + appBundleId, + }); +} diff --git a/src/core/__tests__/snapshot-state.test.ts b/src/core/__tests__/snapshot-state.test.ts index 47c4021509..1380e5a5d2 100644 --- a/src/core/__tests__/snapshot-state.test.ts +++ b/src/core/__tests__/snapshot-state.test.ts @@ -60,6 +60,34 @@ test('buildSnapshotState carries the acquisition producer beside the channel', ( expect(state.producer).toBe('appium-source'); }); +test('buildSnapshotState preserves the full iOS comparison identity as one opaque key', () => { + const comparisonIdentity = { + producer: 'simulator-ax-bridge' as const, + intent: 'full' as const, + lineage: { targetId: 'ios-1:com.example.app', generation: 'launch-a' }, + presentationKey: { + projection: 'regular' as const, + interactiveOnly: false, + depth: null, + scope: null, + customActions: false, + }, + residue: [], + }; + const state = buildSnapshotState( + { + nodes: [{ index: 0, type: 'Application' }], + backend: 'xctest', + producer: 'simulator-ax-bridge', + comparisonIdentity, + }, + undefined, + ); + + expect(state.comparisonKey).toContain('simulator-ax-bridge'); + expect(state.comparisonKey).toContain('launch-a'); +}); + test('buildSnapshotState preserves Android effective geometry for post-wire consumers', () => { const xml = ` diff --git a/src/core/snapshot-state.ts b/src/core/snapshot-state.ts index 5745034c1c..d81ee54706 100644 --- a/src/core/snapshot-state.ts +++ b/src/core/snapshot-state.ts @@ -23,6 +23,8 @@ import { scopeSnapshotNodes } from '@agent-device/capture-kit/snapshot-desktop-p import { normalizeSnapshotTree, pruneGroupNodes } from '../core/snapshot-tree-ingestion.ts'; import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; import { IOS_SNAPSHOT_PRODUCER_CAPABILITIES } from '@agent-device/capture-kit/ios-snapshot-acquisition'; +import { iosSnapshotComparisonIdentityKey } from '@agent-device/capture-kit/ios-snapshot-planning'; +import type { IosSnapshotComparisonIdentity } from '@agent-device/contracts/ios-snapshot'; /** * The ONE daemon assembly of a captured tree (ADR 0004 / #1797): normalize, group prune, @@ -38,6 +40,7 @@ export function buildSnapshotState( nodes?: RawSnapshotNode[]; truncated?: boolean; quality?: unknown; + comparisonIdentity?: IosSnapshotComparisonIdentity; } & SnapshotStateProvenance, flags: | (Pick & @@ -73,6 +76,9 @@ export function buildSnapshotState( createdAt: Date.now(), ...snapshotStateProvenance(data), ...(snapshotQuality ? { snapshotQuality } : {}), + ...(data.comparisonIdentity + ? { comparisonKey: iosSnapshotComparisonIdentityKey(data.comparisonIdentity) } + : {}), presentationKey: buildSnapshotPresentationKey(snapshotPresentationOptionsFromFlags(flags)), // Only broad Android snapshots become freshness baselines. If the user asked for a scoped // or filtered view, preserve that output contract but avoid pretending it is safe for @@ -153,7 +159,11 @@ function iosSnapshotPresentationOwner( function iosSnapshotCapabilities(provenance: SnapshotStateProvenance) { if (provenance.backend !== 'xctest' || provenance.producer === undefined) return undefined; return IOS_SNAPSHOT_PRODUCER_CAPABILITIES[ - provenance.producer as 'apple-runner' | 'appium-source' | 'limrun-ios-tree' + provenance.producer as + | 'apple-runner' + | 'simulator-ax-bridge' + | 'appium-source' + | 'limrun-ios-tree' ]; } diff --git a/src/daemon/deferred-interaction-outcome.ts b/src/daemon/deferred-interaction-outcome.ts index e00b6aaaf5..a7549cb1ab 100644 --- a/src/daemon/deferred-interaction-outcome.ts +++ b/src/daemon/deferred-interaction-outcome.ts @@ -124,7 +124,7 @@ function markPostGestureStabilization( baselineSignature, // Recorded so the loop can tell a comparable quiet capture from one // served by a different backend, which is not comparable at all. - baselineBackend: session.snapshot?.snapshotQuality?.backend, + baselineBackend: snapshotComparisonKey(session.snapshot), } : {}), }; @@ -350,7 +350,7 @@ export async function capturePostGestureStabilizedResult(params: { const snapshot = readSnapshot(value); return { signature: buildInteractionSurfaceSignature(snapshot.nodes), - backend: snapshot.snapshotQuality?.backend, + backend: snapshotComparisonKey(snapshot), }; }, signaturesStable: areInteractionSurfaceSignaturesStable, @@ -363,6 +363,10 @@ export async function capturePostGestureStabilizedResult(params: { return outcome; } +function snapshotComparisonKey(snapshot: SnapshotState | undefined): string | undefined { + return snapshot?.comparisonKey ?? snapshot?.snapshotQuality?.backend; +} + function isPostGestureStabilizingAction( action: string, positionals: string[], diff --git a/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts index 07297050b9..550cba3b82 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts @@ -30,6 +30,7 @@ import { resetGetRuntimeFixture, } from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { captureSnapshotWithInteractor } from '../../../snapshot-interactor-capture.ts'; +import { corroborateIosTapFailure } from '../interaction-ios-tap-outcome.ts'; vi.mock('../../../snapshot-interactor-capture.ts', async () => { const fixture = await import('../../../__tests__/legacy-snapshot-capture-fixture.ts'); @@ -288,6 +289,33 @@ test('a changed capture from a different iOS backend keeps the tap failure', asy expect(sessionStore.get(sessionName)?.actions).toHaveLength(0); }); +test('a producer or generation switch cannot corroborate a failed tap', async () => { + const sessionName = 'ios-comparison-identity-mismatch'; + const sessionStore = makeSessionStore(); + const baseline = snapshot(profileNodes); + baseline.comparisonKey = 'simulator-ax-bridge:launch-a'; + const session = makeIosSession(sessionName, { + appBundleId: 'com.example.app', + snapshot: baseline, + }); + sessionStore.set(sessionName, session); + const after = snapshot(imageViewerNodes); + after.comparisonKey = 'apple-runner:launch-a'; + + await expect( + corroborateIosTapFailure({ + error: new AppError('XCTEST_RECORDED_FAILURE', 'tap failed'), + command: 'click', + requestId: undefined, + flags: {}, + session, + sessionStore, + contextFromFlags, + captureSnapshotForSession: async () => after, + }), + ).resolves.toBeUndefined(); +}); + test('a sparse changed capture keeps the tap failure', async () => { const sessionName = 'ios-sparse-tap-corroboration'; const sessionStore = makeSessionStore(); diff --git a/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts b/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts index edaaf2873d..ac3b68318f 100644 --- a/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts +++ b/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts @@ -152,6 +152,32 @@ function hasMatchingPresentation( baseline: SnapshotState, after: SnapshotState, command: string, +): boolean { + const identityMatch = compareSnapshotIdentity(baseline, after); + if (identityMatch !== undefined) { + if (identityMatch) return true; + emitDiagnostic({ + level: 'debug', + phase: 'ios_tap_failure_corroboration_identity_mismatch', + data: { command }, + }); + return false; + } + return hasMatchingLegacyPresentation(baseline, after, command); +} + +function compareSnapshotIdentity( + baseline: SnapshotState, + after: SnapshotState, +): boolean | undefined { + if (baseline.comparisonKey === undefined && after.comparisonKey === undefined) return undefined; + return baseline.comparisonKey !== undefined && baseline.comparisonKey === after.comparisonKey; +} + +function hasMatchingLegacyPresentation( + baseline: SnapshotState, + after: SnapshotState, + command: string, ): boolean { const baselineBackend = baseline.snapshotQuality?.backend; const afterBackend = after.snapshotQuality?.backend; diff --git a/src/daemon/types.ts b/src/daemon/types.ts index 7e78f05e69..49de26a870 100644 --- a/src/daemon/types.ts +++ b/src/daemon/types.ts @@ -13,7 +13,7 @@ import type { DaemonRequest as WireRequest, } from '@agent-device/kernel/contracts'; import type { DeviceInfo, PlatformSelector } from '@agent-device/kernel/device'; -import type { Rect, SnapshotState, SnapshotCaptureBackend } from '@agent-device/kernel/snapshot'; +import type { Rect, SnapshotState } from '@agent-device/kernel/snapshot'; import type { SnapshotFreshnessWindow } from '../snapshot/snapshot-freshness/index.ts'; // Type-only import; erased at runtime. ref-frame.ts imports SessionState from // here, so this back-edge must stay type-only to avoid a runtime cycle. @@ -213,7 +213,7 @@ export type PostGestureStabilization = { * a different backend can only be re-baselined against, never concluded from * (#1569). */ - baselineBackend?: SnapshotCaptureBackend; + baselineBackend?: string; }; export type PendingInteractionOutcome = { diff --git a/src/snapshot/ios-snapshot-runtime.ts b/src/snapshot/ios-snapshot-runtime.ts index 43ba31732b..aaa0a3ad41 100644 --- a/src/snapshot/ios-snapshot-runtime.ts +++ b/src/snapshot/ios-snapshot-runtime.ts @@ -1,6 +1,6 @@ import { IosSnapshotEngineError, - presentIosSnapshot, + publishIosSnapshot, toIosSnapshotEngineErrorDetails, } from '@agent-device/capture-kit/ios-snapshot-engine'; import { @@ -43,11 +43,12 @@ export function presentIosSnapshotAcquisition( const input = iosSnapshotInput(acquired, request); try { - const presentation = presentIosSnapshot(input, request); + const presentation = publishIosSnapshot(input, request); return { backend: 'xctest', producer: acquired.acquisition.producer, - nodes: presentation.nodes, + nodes: [...presentation.payload.nodes], + comparisonIdentity: presentation.comparisonIdentity, ...(acquired.acquisition.truncated === undefined ? {} : { truncated: acquired.acquisition.truncated }),