diff --git a/.gitignore b/.gitignore index 99f444376..136ad8226 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,6 @@ android/ime-helper/dist/ # Workspace package declaration output (tsc -b project references) packages/*/dist-types/ *.tsbuildinfo + +# Reproducible AX bridge captures are published on the evidence branch. +docs/evidence/ios-simulator-ax-bridge-*.json.gz diff --git a/docs/evidence/ios-simulator-ax-bridge-2026-09-02-corrected.md b/docs/evidence/ios-simulator-ax-bridge-2026-09-02-corrected.md new file mode 100644 index 000000000..dc1d24b51 --- /dev/null +++ b/docs/evidence/ios-simulator-ax-bridge-2026-09-02-corrected.md @@ -0,0 +1,124 @@ +# iOS Simulator AX bridge corrected evidence + +- Decision: **GO** +- Interpretation: **maintainer-corrected** +- Revision: 44995806ea3be09f3c48ceac50ac3cab18462c35 (detached) +- Target: ad-2237-axbridge (8CDB4DF1-3A3E-4FB1-AF89-B3D3A17647D5, com.apple.CoreSimulator.SimRuntime.iOS-26-2) +- Generated: 2026-09-03T20:40:27.697Z +- Immutable broad raw artifact: `evidence/ios-snapshot/44995806ea:ios-simulator-ax-bridge-broad-268a90275.json.gz` (SHA-256 `309f974b1dcb90768548a189f6af58b493b5d7b9d56a5bfad060d4335139eb7b`; original NO-GO; interpretation superseded to stretch-only; host client persistent-in-repository-reader) +- Narrow targeted raw artifact: `evidence/ios-snapshot/44995806ea:ios-simulator-ax-bridge-targeted-44995806ea.json.gz` (SHA-256 `3440d066cb7eea33c4715fece838b5185c5d209694b097e54f5536d48d4984ad`; host client node-direct-socket) +- Corrected raw report: `evidence/ios-snapshot/44995806ea:ios-simulator-ax-bridge-corrected-44995806ea.json.gz` (SHA-256 `0a34a84402e85154e177adef5122101b26bfd04d74714a0a9b6f0795270edc41`) +- Host at generation: load average 4.84 on 12 cores + +The broad raw corpus is preserved unchanged. Its old NO-GO used readiness-inclusive first-look and stretch thresholds; this report evaluates the corrected hard contract. The broad warm cells remain conservative upper bounds around the same in-Simulator reader. Relaunch uses the new Node-direct corpus below and does not rely on the legacy relaunch samples. + +## Evaluated guest mechanism + +- Guest reader: idb v1.5.2 `Resources/SimulatorFrameworkBridge` (observed SHA-256 `3545621d2dc98de32879ebac55e8b0c33dc8eb7cc2bfbc2d0d2d21a002c8de58`; required SHA-256 `3545621d2dc98de32879ebac55e8b0c33dc8eb7cc2bfbc2d0d2d21a002c8de58`) from `idb-companion.macos-arm64.tar.gz` (SHA-256 `f17b718a513931705542a7fbfa9cfc11895ee191562c9ffd2343cf7f8254bc08`). +- Transport: xcrun simctl spawn SimulatorFrameworkBridge accessibility serve --idle-timeout 300 --exit-on-disconnect true; UNIX socket frames are a 4-byte big-endian length + JSON. +- Traversal: describe with snapshotTree=true (one XCTest snapshot fetch per read) and automationMode=true asserted per request; no idb_companion, gRPC, or Python client. + +## Hard gates + +| Gate | Status | Target | Evidence | +|---|---|---|---| +| warm | **PASS** | p50 <300 ms and p95 <500 ms per screen | 6/6 warm screen cells passed; quiet p50/p95=8.6 ms/9.3 ms ready=1/20; list p50/p95=118.2 ms/120.9 ms ready=1/20; nested-scroll p50/p95=15.1 ms/15.8 ms ready=1/20; alert p50/p95=41.6 ms/43.3 ms ready=1/20; system-surface p50/p95=37.2 ms/39.6 ms ready=1/20; xctest-stress p50/p95=39.6 ms/41.1 ms ready=1/20 | +| relaunch | **PASS** | p95 <500 ms per screen after independently observed new-generation readiness | 6/6 relaunch screen cells passed; quiet p50/p95=77.4 ms/133.9 ms ready=20/20; list p50/p95=193.4 ms/234.7 ms ready=20/20; nested-scroll p50/p95=84.5 ms/93.3 ms ready=20/20; alert p50/p95=113.7 ms/124.0 ms ready=20/20; system-surface p50/p95=108.9 ms/116.6 ms ready=20/20; xctest-stress p50/p95=117.4 ms/126.0 ms ready=20/20; 120/120 Node-direct samples across 6/6 screens | +| nonresidentBootstrap | **PASS** | nonresident companion + reader bootstrap and first usable tree p95 <2,000 ms | 5/5 usable trees; p95=1136.6 ms; the timer covered guest spawn, socket connect, and the first tree after a throwaway probe observed the relaunched app's readiness (readiness p95=1220.6 ms), with no resident bridge, xcodebuild, XCTest, or agent-device runner in the timed path | +| boundedResources | **PASS** | guest CPU <=2000 ms and RSS <=268435456 bytes per successful read | 129/129 successful reads measured within bounds; max CPU=220.0 ms; max RSS=89407488 bytes | +| liveRecovery | **PASS** | live crash, timeout, cancellation, and honest target-generation handling | 4/4 probes returned a typed failure or typed unavailable-generation residue and a usable recovered response | +| hierarchy | **PASS** | structural hierarchy acquired with typed truncation, or its absence typed as residue | nested tree with traversal depth 29 in 5/5 samples; truncated=false | +| preferenceControl | **PASS** | task-owned Simulator accessibility preferences applied preboot and restored | applied=true; restored=true; enabled keys=AutomationEnabled, IgnoreAXServerEntitlements; fixture launch compatible=true | + +## Readiness boundary and candidate-owned latency + +Warm and relaunch timing starts at bridge acquisition after fixture/app readiness admission. Every relaunch row comes from the Node-direct route and is paired with a separate probe that observed the exact relaunched process generation and expected screen anchor. The old first-look value includes Simulator, app, daemon, and runner costs. + +| State | Screen | Samples | Readable | Ready generation | Candidate p50/p95 ms | Readiness p95 ms | Old first-look p95 ms | Generations | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| warm | quiet | 20 | 20 | 1 | 8.6/9.3 | 0.0 | 9.3 | 1 | +| warm | list | 20 | 20 | 1 | 118.2/120.9 | 0.0 | 120.9 | 1 | +| warm | nested-scroll | 20 | 20 | 1 | 15.1/15.8 | 0.0 | 15.8 | 1 | +| warm | alert | 20 | 20 | 1 | 41.6/43.3 | 0.0 | 43.3 | 1 | +| warm | system-surface | 20 | 20 | 1 | 37.2/39.6 | 0.0 | 39.6 | 1 | +| warm | xctest-stress | 20 | 20 | 1 | 39.6/41.1 | 0.0 | 41.1 | 1 | +| relaunch | quiet | 20 | 20 | 20 | 77.4/133.9 | 1090.5 | 1174.6 | 20 | +| relaunch | list | 20 | 20 | 20 | 193.4/234.7 | 1180.5 | 1374.0 | 20 | +| relaunch | nested-scroll | 20 | 20 | 20 | 84.5/93.3 | 1111.3 | 1208.6 | 20 | +| relaunch | alert | 20 | 20 | 20 | 113.7/124.0 | 1136.9 | 1283.5 | 20 | +| relaunch | system-surface | 20 | 20 | 20 | 108.9/116.6 | 1114.5 | 1224.1 | 20 | +| relaunch | xctest-stress | 20 | 20 | 20 | 117.4/126.0 | 1122.3 | 1244.3 | 20 | + +## Cold diagnostics + +Cold and cold-cold first-look measurements remain visible for diagnosis, but are excluded from the candidate-owned hard verdict because they combine environment and readiness boundaries with bridge work. + +| State | Screen | Preparation p95 ms | First-look p95 ms | Interpretation | +|---|---|---:|---:|---| +| cold-cold | quiet | 16151.2 | 16575.5 | excluded runner/app readiness costs | +| cold-cold | list | 19475.9 | 20062.0 | excluded runner/app readiness costs | +| cold-cold | nested-scroll | 18561.5 | 19089.0 | excluded runner/app readiness costs | +| cold-cold | alert | 17112.3 | 17549.4 | excluded runner/app readiness costs | +| cold-cold | system-surface | 18355.9 | 18866.3 | excluded runner/app readiness costs | +| cold-cold | xctest-stress | 16930.2 | 17368.5 | excluded runner/app readiness costs | +| cold | quiet | 7138.6 | 7147.0 | excluded runner/app readiness costs | +| cold | list | 6985.7 | 7104.5 | excluded runner/app readiness costs | +| cold | nested-scroll | 6930.9 | 6947.1 | excluded runner/app readiness costs | +| cold | alert | 6855.4 | 6895.5 | excluded runner/app readiness costs | +| cold | system-surface | 7145.6 | 7181.9 | excluded runner/app readiness costs | +| cold | xctest-stress | 6918.1 | 6959.1 | excluded runner/app readiness costs | + +## Nonresident bootstrap + +- 5/5 usable trees; p95=1136.6 ms; the timer covered guest spawn, socket connect, and the first tree after a throwaway probe observed the relaunched app's readiness (readiness p95=1220.6 ms), with no resident bridge, xcodebuild, XCTest, or agent-device runner in the timed path. +- 129/129 successful reads measured within bounds; max CPU=220.0 ms; max RSS=89407488 bytes. +- The timed boundary begins with no resident bridge and ends at the first usable guest tree. Before each timer the fixture app was relaunched and a throwaway probe bridge polled until the new generation answered with a tree (readiness), then exited. + +| Sample | Duration ms | CPU ms | RSS MiB | Usable tree | Nodes | Depth | Generation | Readiness ms | Attempts | Host load | +|---:|---:|---:|---:|---|---:|---:|---|---:|---:|---:| +| 1 | 1095.6 | 220.0 | 77.1 | true | 155 | 29 | pid:43541 | 1171 | 1 | 21.27 | +| 2 | 1071.2 | 210.0 | 77.5 | true | 155 | 29 | pid:44026 | 1080 | 1 | 23.17 | +| 3 | 1094.4 | 200.0 | 78.5 | true | 155 | 29 | pid:44524 | 1221 | 1 | 24.33 | +| 4 | 1043.5 | 220.0 | 77.9 | true | 155 | 29 | pid:45028 | 1119 | 1 | 22.17 | +| 5 | 1136.6 | 210.0 | 77.4 | true | 155 | 29 | pid:45518 | 1202 | 1 | 20.71 | + +## Live candidate recovery + +- 4/4 probes returned a typed failure or typed unavailable-generation residue and a usable recovered response. + +| Operation | Observed failure | Recovery response | Recovered tree | +|---|---|---|---| +| process-crash | process-crash/guest-exited | ok | 135 nodes | +| timeout | timeout/batch-duration-limit | ok | 135 nodes | +| cancelled | cancelled/abort-signal | ok | 135 nodes | +| stale-generation | stale-generation/target-generation-mismatch | ok | 135 nodes | + +## Hierarchy + +- nested tree with traversal depth 29 in 5/5 samples; truncated=false. +- Observed traversal depth: 29; depth complete: **true**; interpretation: nested-tree. + +## Simulator preference control + +- applied=true; restored=true; enabled keys=AutomationEnabled, IgnoreAXServerEntitlements; fixture launch compatible=true. +- The broad capture applied its accessibility preference changes only to the disposable benchmark Simulator before boot, verified fixture launch compatibility, and restored the prior preference files and Simulator state afterward. + +## Private-interface compatibility risk + +- SimulatorFrameworkBridge and its accessibility wire protocol are private idb/Apple implementation details with no compatibility guarantee. +- Control: Pin the official idb release and observed guest SHA-256, keep this route Simulator-only behind the acquisition adapter, and re-run this verifier for every idb, Xcode, or Simulator runtime change before production adoption. + +## Stretch findings + +- Original broad-run finding: guest-simulator-framework-bridge cold-cold first look missed the 5 second target. +- Original broad-run finding: guest-simulator-framework-bridge cold prepared first look missed the 1.5 second target. +- Original broad-run finding: guest-simulator-framework-bridge warm/list acquisition missed the 75/150 ms target. +- Original broad-run finding: guest-simulator-framework-bridge relaunch first look missed the 250 ms target. +- Cold and cold-cold first-look measurements include Simulator, app, daemon, and runner readiness costs; they are diagnostics, not candidate-owned hard gates. +- The former warm 75/150 ms and relaunch 250 ms thresholds are stretch findings under the corrected contract. +- Nonresident bootstrap samples were taken on a host with 1-minute load average 4.84 on 12 cores; per-sample load is recorded with each sample. + +## Production boundary + +- No production backend selection, fallback, runner-demand, open/relaunch, proxy, XCTest interaction, or public CLI changes were made. +- The corrected result is evidence for the #2192 decision boundary only; it does not start production routing. diff --git a/package.json b/package.json index 5551f4e14..d349a7043 100644 --- a/package.json +++ b/package.json @@ -124,6 +124,7 @@ "bench:ios-snapshot": "node --experimental-strip-types scripts/ios-snapshot-benchmark/run.ts", "bench:ios-snapshot:deep-button": "node --experimental-strip-types scripts/ios-snapshot-benchmark/deep-button.ts", "bench:ios-snapshot:evidence": "node --experimental-strip-types scripts/ios-snapshot-benchmark/evidence.ts", + "bench:ios-ax-bridge:targeted": "node --experimental-strip-types scripts/ios-ax-bridge-spike/targeted-run.ts", "mutation:run": "node --experimental-strip-types scripts/mutation/run.ts", "mutation:check": "node --experimental-strip-types scripts/mutation/run.ts --no-run", "mutation:affected": "node --experimental-strip-types scripts/mutation/run.ts --affected", diff --git a/scripts/ios-ax-bridge-spike/README.md b/scripts/ios-ax-bridge-spike/README.md new file mode 100644 index 000000000..3a83436aa --- /dev/null +++ b/scripts/ios-ax-bridge-spike/README.md @@ -0,0 +1,28 @@ +# iOS Simulator AX bridge decision verifier + +This narrow harness supplies the decisive live evidence for [#2192](https://github.com/callstack/agent-device/issues/2192). It drives idb v1.5.2's in-Simulator `Resources/SimulatorFrameworkBridge` directly from Node over a private UNIX socket. It does not use `idb_companion`, gRPC, or Python, and it does not change production routing. + +The September 1 broad corpus is retained for its warm measurements and diagnostics. Its one-off Python runner and generated NO-GO reports were removed after the corrected contract made them obsolete. The corrected relaunch corpus is Node-direct. Raw artifacts are kept off-tree at immutable tag `evidence/ios-snapshot/44995806ea` (commit `f8b2fab28b8604f20094785c16e16a79fdc651a3`); their SHA-256 hashes are recorded in the evidence branch README. + +Obtain the guest executable from the official arm64 idb v1.5.2 release. The archive SHA-256 is `f17b718a513931705542a7fbfa9cfc11895ee191562c9ffd2343cf7f8254bc08`; `Resources/SimulatorFrameworkBridge` is `3545621d2dc98de32879ebac55e8b0c33dc8eb7cc2bfbc2d0d2d21a002c8de58`. The verifier hashes the supplied `--guest-bridge` before launching it and fails if it is not that binary. + +Fetch the broad input before rerunning: + +```sh +git fetch origin refs/tags/evidence/ios-snapshot/44995806ea +git show evidence/ios-snapshot/44995806ea:ios-simulator-ax-bridge-broad-268a90275.json.gz \ + > docs/evidence/ios-simulator-ax-bridge-2026-09-01-final.json.gz +shasum -a 256 docs/evidence/ios-simulator-ax-bridge-2026-09-01-final.json.gz +``` + +The expected broad artifact hash is `309f974b1dcb90768548a189f6af58b493b5d7b9d56a5bfad060d4335139eb7b`. + +Run the verifier from a clean commit using the dedicated Simulator with the fixture app installed: + +```sh +pnpm bench:ios-ax-bridge:targeted -- \ + --udid SIMULATOR_UDID \ + --guest-bridge /path/to/Resources/SimulatorFrameworkBridge +``` + +It captures five nonresident bootstrap samples after independently observing application readiness. It then captures 20 relaunches on each of the six representative screens through the Node-direct route; every timed read is paired with a separate readiness probe for the exact relaunched PID and expected screen anchor. Finally it exercises crash, timeout, cancellation, and stale-generation recovery. Successful reads record guest CPU time and resident memory, and the corrected report fails closed if those metrics are missing or exceed the declared bounds. diff --git a/scripts/ios-ax-bridge-spike/adapter.test.ts b/scripts/ios-ax-bridge-spike/adapter.test.ts new file mode 100644 index 000000000..da5a8186e --- /dev/null +++ b/scripts/ios-ax-bridge-spike/adapter.test.ts @@ -0,0 +1,30 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createGuestSimulatorFrameworkBridgeAdapter } from './guest-adapter.ts'; + +test('guest adapter fails closed when the guest bridge executable is not configured', async () => { + const adapter = createGuestSimulatorFrameworkBridgeAdapter({ repoRoot: '/repo' }); + const result = await adapter.acquireBatch([ + { + version: 1, + id: 'guest-unavailable', + candidate: 'guest-simulator-framework-bridge', + simulatorUdid: 'simulator', + state: 'warm', + screen: 'quiet', + limits: { + maxRequestBytes: 64 * 1024, + maxResponseBytes: 4 * 1024 * 1024, + maxNodes: 1500, + maxTraversalDepth: 12, + maxCpuMs: 2000, + maxMemoryBytes: 256 * 1024 * 1024, + maxDurationMs: 5000, + }, + }, + ]); + assert.deepEqual(result.responses[0]?.failure, { + kind: 'unsupported-mechanism', + code: 'guest-tool-unavailable', + }); +}); diff --git a/scripts/ios-ax-bridge-spike/adapter.ts b/scripts/ios-ax-bridge-spike/adapter.ts new file mode 100644 index 000000000..78a739d45 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/adapter.ts @@ -0,0 +1,16 @@ +import type { CandidateId, ResourceLimits, SpikeRequest, SpikeResponse } from './types.ts'; + +export type AcquisitionAdapter = Readonly<{ + candidate: CandidateId; + acquireBatch( + requests: readonly SpikeRequest[], + options?: Readonly<{ signal?: AbortSignal }>, + ): Promise>; + close?: () => Promise; + evidence?: Readonly<{ terminateReaderOnNextBatch?: () => void }>; +}>; + +export type AdapterOptions = Readonly<{ + limits?: ResourceLimits; + guestBridge?: string; +}>; diff --git a/scripts/ios-ax-bridge-spike/config.ts b/scripts/ios-ax-bridge-spike/config.ts new file mode 100644 index 000000000..4712101e8 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/config.ts @@ -0,0 +1,87 @@ +import path from 'node:path'; +import { resolveRepoRoot } from '../ios-snapshot-benchmark/host.ts'; +import { createBenchmarkStateRoot } from '../ios-snapshot-benchmark/state-ownership.ts'; +import { DEFAULT_SPIKE_LIMITS } from './limits.ts'; +import type { ResourceLimits } from './types.ts'; + +class SpikeConfigurationError extends Error { + constructor(message: string) { + super(message); + this.name = 'SpikeConfigurationError'; + } +} + +export type SpikeConfig = Readonly<{ + repoRoot: string; + udid: string; + guestBridge: string; + stateDir: string; + derivedPath: string; + limits: ResourceLimits; + keepDevice: boolean; +}>; + +export function parseConfig(argv: readonly string[]): SpikeConfig { + const args = argv[0] === '--' ? argv.slice(1) : argv; + if (args.some((argument) => argument === '--help' || argument === '-h')) { + printHelp(); + process.exit(0); + } + const parsed = parseArguments(args); + const udid = required(parsed.values, '--udid'); + const guestBridge = required(parsed.values, '--guest-bridge'); + const stateDir = createBenchmarkStateRoot(); + return { + repoRoot: resolveRepoRoot(), + udid, + guestBridge, + stateDir, + derivedPath: path.join(stateDir, 'derived-data'), + limits: DEFAULT_SPIKE_LIMITS, + keepDevice: parsed.keepDevice, + }; +} + +function parseArguments(args: readonly string[]): { + values: ReadonlyMap; + keepDevice: boolean; +} { + const values = new Map(); + let keepDevice = false; + for (let index = 0; index < args.length; index += 1) { + const flag = args[index]; + if (flag === '--keep-device') { + keepDevice = true; + continue; + } + assertValueFlag(flag); + values.set(flag, readValue(args[index + 1], flag)); + index += 1; + } + return { values, keepDevice }; +} + +function assertValueFlag(flag: string | undefined): asserts flag is '--udid' | '--guest-bridge' { + if (flag !== '--udid' && flag !== '--guest-bridge') { + throw new SpikeConfigurationError(`Unknown option: ${String(flag)}`); + } +} + +function readValue(value: string | undefined, flag: string): string { + if (!value || value.startsWith('--')) { + throw new SpikeConfigurationError(`${flag} requires a value.`); + } + return value; +} + +function required(values: ReadonlyMap, flag: string): string { + const value = values.get(flag); + if (!value) throw new SpikeConfigurationError(`${flag} is required.`); + return value; +} + +function printHelp(): void { + process.stdout.write( + 'Usage: pnpm bench:ios-ax-bridge:targeted -- --udid --guest-bridge [--keep-device]\n', + ); +} diff --git a/scripts/ios-ax-bridge-spike/corrected-markdown.ts b/scripts/ios-ax-bridge-spike/corrected-markdown.ts new file mode 100644 index 000000000..03878e168 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/corrected-markdown.ts @@ -0,0 +1,126 @@ +import type { CorrectedReport, GateResult, LatencySummary } from './corrected-types.ts'; + +export function renderCorrectedMarkdown(report: CorrectedReport): string { + const lines = [ + '# iOS Simulator AX bridge corrected evidence', + '', + `- Decision: **${report.decision}**`, + '- Interpretation: **maintainer-corrected**', + `- Revision: ${report.revision.commit} (${report.revision.branch})`, + `- Target: ${report.target.name} (${report.target.udid}, ${report.target.runtime})`, + `- Generated: ${report.generatedAt}`, + `- Immutable broad raw artifact: \`${report.sourceArtifact.path}\` (original ${report.sourceArtifact.originalDecision}; interpretation superseded to stretch-only; host client ${report.sourceArtifact.hostClient})`, + `- Narrow targeted raw artifact: \`${report.targetedArtifact.path}\` (host client ${report.guestMechanism.client})`, + `- Host at generation: load average ${report.host.loadAverage1m} on ${report.host.cpuCores} cores`, + '', + 'The broad raw corpus is preserved unchanged. Its old NO-GO used readiness-inclusive first-look and stretch thresholds; this report evaluates the corrected hard contract. The broad warm cells remain conservative upper bounds around the same in-Simulator reader. Relaunch uses the new Node-direct corpus below and does not rely on the legacy relaunch samples.', + '', + '## Evaluated guest mechanism', + '', + `- Guest reader: ${report.guestMechanism.implementation} ${report.guestMechanism.release} \`${report.guestMechanism.guestBinary}\` (observed SHA-256 \`${report.guestMechanism.guestBinarySha256}\`; required SHA-256 \`${report.guestMechanism.guestBinaryExpectedSha256 ?? 'not recorded'}\`) from \`${report.guestMechanism.companionArchive}\` (SHA-256 \`${report.guestMechanism.companionSha256}\`).`, + `- Transport: ${report.guestMechanism.transport}.`, + `- Traversal: ${report.guestMechanism.traversal}.`, + '', + '## Hard gates', + '', + '| Gate | Status | Target | Evidence |', + '|---|---|---|---|', + ...Object.entries(report.hardGates).map(([name, gate]) => gateLine(name, gate)), + '', + '## Readiness boundary and candidate-owned latency', + '', + 'Warm and relaunch timing starts at bridge acquisition after fixture/app readiness admission. Every relaunch row comes from the Node-direct route and is paired with a separate probe that observed the exact relaunched process generation and expected screen anchor. The old first-look value includes Simulator, app, daemon, and runner costs.', + '', + '| State | Screen | Samples | Readable | Ready generation | Candidate p50/p95 ms | Readiness p95 ms | Old first-look p95 ms | Generations |', + '|---|---|---:|---:|---:|---:|---:|---:|---:|', + ...report.readiness.map(readinessLine), + '', + '## Cold diagnostics', + '', + 'Cold and cold-cold first-look measurements remain visible for diagnosis, but are excluded from the candidate-owned hard verdict because they combine environment and readiness boundaries with bridge work.', + '', + '| State | Screen | Preparation p95 ms | First-look p95 ms | Interpretation |', + '|---|---|---:|---:|---|', + ...report.coldDiagnostics.map(coldDiagnosticLine), + '', + '## Nonresident bootstrap', + '', + `- ${report.hardGates.nonresidentBootstrap.evidence}.`, + `- ${report.hardGates.boundedResources.evidence}.`, + '- The timed boundary begins with no resident bridge and ends at the first usable guest tree. Before each timer the fixture app was relaunched and a throwaway probe bridge polled until the new generation answered with a tree (readiness), then exited.', + '', + '| Sample | Duration ms | CPU ms | RSS MiB | Usable tree | Nodes | Depth | Generation | Readiness ms | Attempts | Host load |', + '|---:|---:|---:|---:|---|---:|---:|---|---:|---:|---:|', + ...report.bootstrap.map(bootstrapLine), + '', + '## Live candidate recovery', + '', + `- ${report.hardGates.liveRecovery.evidence}.`, + '', + '| Operation | Observed failure | Recovery response | Recovered tree |', + '|---|---|---|---|', + ...report.liveRecovery.map(recoveryLine), + '', + '## Hierarchy', + '', + `- ${report.hardGates.hierarchy.evidence}.`, + `- Observed traversal depth: ${report.hierarchy.observedTraversalDepth}; depth complete: **${report.hierarchy.depthComplete}**; interpretation: ${report.hierarchy.interpretation}.`, + '', + '## Simulator preference control', + '', + `- ${report.hardGates.preferenceControl.evidence}.`, + '- The broad capture applied its accessibility preference changes only to the disposable benchmark Simulator before boot, verified fixture launch compatibility, and restored the prior preference files and Simulator state afterward.', + '', + '## Private-interface compatibility risk', + '', + `- ${report.compatibilityRisk.assessment}`, + `- Control: ${report.compatibilityRisk.control}`, + '', + '## Stretch findings', + '', + ...report.stretchFindings.map((finding) => `- ${finding}`), + '', + '## Production boundary', + '', + '- No production backend selection, fallback, runner-demand, open/relaunch, proxy, XCTest interaction, or public CLI changes were made.', + '- The corrected result is evidence for the #2192 decision boundary only; it does not start production routing.', + ]; + return `${lines.join('\n')}\n`; +} + +function gateLine(name: string, gate: GateResult): string { + return `| ${name} | **${gate.status}** | ${gate.target} | ${gate.evidence} |`; +} + +function readinessLine(summary: LatencySummary): string { + return `| ${summary.state} | ${summary.screen} | ${summary.samples} | ${summary.readableSamples} | ${summary.readinessObservedSamples} | ${formatMs(summary.candidateP50Ms)}/${formatMs(summary.candidateP95Ms)} | ${formatMs(summary.preparationP95Ms)} | ${formatMs(summary.firstLookP95Ms)} | ${summary.generationCount} |`; +} + +function coldDiagnosticLine(diagnostic: CorrectedReport['coldDiagnostics'][number]): string { + return `| ${diagnostic.state} | ${diagnostic.screen} | ${formatMs(diagnostic.preparationP95Ms)} | ${formatMs(diagnostic.firstLookP95Ms)} | excluded runner/app readiness costs |`; +} + +function bootstrapLine(sample: CorrectedReport['bootstrap'][number]): string { + const response = sample.response; + return `| ${sample.index} | ${sample.durationMs.toFixed(1)} | ${formatMs(response.metrics.cpuMs)} | ${formatMib(response.metrics.memoryBytes)} | ${sample.usableTree} | ${response.metrics.nodeCount} | ${response.metrics.maxTraversalDepth} | ${response.acquisition?.targetGeneration ?? '–'} | ${sample.readinessMs.toFixed(0)} | ${sample.readinessAttempts} | ${sample.host.loadAverage1m} |`; +} + +function recoveryLine(probe: CorrectedReport['liveRecovery'][number]): string { + const recovery = probe.recoveredResponse; + const status = recovery.ok ? 'ok' : 'failed'; + const nodes = recovery.acquisition?.nodes.length ?? 0; + return `| ${probe.operation} | ${failureText(probe.response.failure)} | ${status} | ${nodes} nodes |`; +} + +function failureText(failure: CorrectedReport['bootstrap'][number]['response']['failure']): string { + if (!failure) return 'none/none'; + return `${failure.kind}/${failure.code ?? 'none'}`; +} + +function formatMs(value: number | null): string { + return value === null ? '–' : value.toFixed(1); +} + +function formatMib(value: number | null): string { + return value === null ? '–' : (value / 1024 / 1024).toFixed(1); +} diff --git a/scripts/ios-ax-bridge-spike/corrected-report.test.ts b/scripts/ios-ax-bridge-spike/corrected-report.test.ts new file mode 100644 index 000000000..b38291dff --- /dev/null +++ b/scripts/ios-ax-bridge-spike/corrected-report.test.ts @@ -0,0 +1,263 @@ +import { describe, expect, test } from 'vitest'; +import { buildCorrectedReport } from './corrected-report.ts'; +import { renderCorrectedMarkdown } from './corrected-markdown.ts'; +import { DEFAULT_SPIKE_LIMITS } from './limits.ts'; +import type { TargetedRawArtifact, TargetedRecoveryProbe } from './corrected-types.ts'; +import type { SpikeReport, SpikeResponse } from './types.ts'; + +const revision = { commit: 'abc123', branch: 'test', dirty: false } as const; +const mechanism = { + implementation: 'idb', + release: 'v1.5.2', + companionArchive: 'idb-companion.macos-arm64.tar.gz', + companionSha256: 'archive', + guestBinary: 'Resources/SimulatorFrameworkBridge', + guestBinaryExpectedSha256: 'guest', + guestBinarySha256: 'guest', + transport: 'socket', + traversal: 'tree', + client: 'node-direct-socket', +} as const; + +describe('corrected Simulator AX bridge report', () => { + test('counts retained generation evidence honestly and uses targeted readiness and resources', () => { + const source = broadReport(); + const targeted = targetedArtifact(); + const report = buildCorrectedReport({ + sourcePath: 'broad.json.gz', + source, + targetedPath: 'targeted.json.gz', + targeted, + }); + + expect( + report.readiness.every( + (cell) => cell.readinessObservedSamples === (cell.state === 'warm' ? 1 : 2), + ), + ).toBe(true); + expect(report.hardGates.relaunch.status).toBe('PASS'); + expect(report.hardGates.boundedResources.status).toBe('PASS'); + expect(report.hardGates.liveRecovery.status).toBe('PASS'); + expect(report.hardGates.hierarchy.status).toBe('PASS'); + expect(report.decision).toBe('GO'); + expect(renderCorrectedMarkdown(report)).toContain('Decision: **GO**'); + }); + + test('fails closed when a successful Node-direct read lacks resource metrics', () => { + const targeted = targetedArtifact(); + const first = targeted.bootstrap[0]!; + const report = buildCorrectedReport({ + sourcePath: 'broad.json.gz', + source: broadReport(), + targetedPath: 'targeted.json.gz', + targeted: { + ...targeted, + bootstrap: [ + { + ...first, + response: { + ...first.response, + metrics: { ...first.response.metrics, cpuMs: null, memoryBytes: null }, + }, + }, + ...targeted.bootstrap.slice(1), + ], + }, + }); + + expect(report.hardGates.boundedResources.status).toBe('FAIL'); + expect(report.decision).toBe('NO-GO'); + }); + + test('fails relaunch when a timed read is not paired to its expected generation', () => { + const targeted = targetedArtifact(); + const first = targeted.relaunch[0]!; + const report = buildCorrectedReport({ + sourcePath: 'broad.json.gz', + source: broadReport(), + targetedPath: 'targeted.json.gz', + targeted: { + ...targeted, + relaunch: [ + { + ...first, + response: successfulResponse('pid:999'), + }, + ...targeted.relaunch.slice(1), + ], + }, + }); + + expect(report.hardGates.relaunch.status).toBe('FAIL'); + expect(report.decision).toBe('NO-GO'); + }); + + test('fails relaunch when the configured corpus is incomplete', () => { + const targeted = targetedArtifact(); + const report = buildCorrectedReport({ + sourcePath: 'broad.json.gz', + source: broadReport(), + targetedPath: 'targeted.json.gz', + targeted: { ...targeted, relaunch: targeted.relaunch.slice(1) }, + }); + + expect(report.hardGates.relaunch.status).toBe('FAIL'); + expect(report.hardGates.relaunch.evidence).toContain('1/2 Node-direct samples'); + }); +}); + +function broadReport(): SpikeReport { + const samples = [ + { ok: true, firstTree: 'readable', wallClockMs: 40, target: 'pid:1' }, + { ok: true, firstTree: 'readable', wallClockMs: 60 }, + ] as const; + return { + revision, + guestMechanism: mechanism, + target: { udid: 'sim', name: 'simulator', runtime: 'iOS' }, + toolchain: { + node: 'node', + pnpm: 'pnpm', + xcode: 'xcode', + simctl: 'simctl', + os: 'macOS', + arch: 'arm64', + }, + cells: (['warm', 'relaunch'] as const).map((state) => ({ + candidate: 'guest-simulator-framework-bridge', + state, + screen: 'list', + acquisitionSamples: samples.map((sample) => ({ + ok: sample.ok, + firstTree: sample.firstTree, + wallClockMs: sample.wallClockMs, + ...(sample.target + ? { acquisition: { ...acquisition(), targetGeneration: sample.target } } + : {}), + })), + })), + decisionReasons: [], + preferenceEvidence: { + applied: true, + restored: true, + fixtureLaunchCompatible: true, + simulatorStateBefore: 'Shutdown', + diffs: [ + { + changes: [ + { key: 'AutomationEnabled', before: 0, after: true }, + { key: 'IgnoreAXServerEntitlements', after: true }, + ], + }, + ], + }, + }; +} + +function targetedArtifact(): TargetedRawArtifact { + const bootstrap = Array.from({ length: 5 }, (_, offset) => { + const appPid = 100 + offset; + return { + index: offset + 1, + durationMs: 100, + usableTree: true, + response: successfulResponse(`pid:${appPid}`), + stderr: '', + appPid, + readinessMs: 50, + readinessAttempts: 1, + host: { loadAverage1m: 1, cpuCores: 12 }, + }; + }); + const operations = ['process-crash', 'timeout', 'cancelled', 'stale-generation'] as const; + const recovery: TargetedRecoveryProbe[] = operations.map((operation) => ({ + operation, + request: request(`request-${operation}`), + response: { + ...successfulResponse('pid:100'), + ok: false, + acquisition: undefined, + failure: { kind: operation, code: 'probe' }, + }, + recoveredResponse: successfulResponse('pid:104'), + })); + const relaunch = Array.from({ length: 2 }, (_, offset) => { + const appPid = 200 + offset; + return { + index: offset + 1, + screen: 'list' as const, + expectedAnchor: 'Item 20', + appPid, + readinessMs: 40, + readinessAttempts: 1, + durationMs: 80, + response: successfulResponse(`pid:${appPid}`), + stderr: '', + }; + }); + return { + schemaVersion: 'ios-simulator-ax-bridge-targeted.v3', + generatedAt: '2026-09-03T00:00:00.000Z', + revision, + command: 'targeted', + sourceArtifact: { path: 'broad.json.gz', revision, hostClient: 'legacy' }, + target: { udid: 'sim', name: 'simulator', runtime: 'iOS' }, + toolchain: { + node: 'node', + pnpm: 'pnpm', + xcode: 'xcode', + simctl: 'simctl', + os: 'macOS', + arch: 'arm64', + }, + host: { loadAverage1m: 1, cpuCores: 12 }, + guestMechanism: mechanism, + limits: DEFAULT_SPIKE_LIMITS, + config: { states: ['warm', 'relaunch'], screens: ['list'], samples: 2, bootstrapSamples: 5 }, + bootstrap, + relaunch, + recovery, + }; +} + +function successfulResponse(generation: string): SpikeResponse { + return { + version: 1, + id: 'response', + candidate: 'guest-simulator-framework-bridge', + ok: true, + acquisition: { ...acquisition(), targetGeneration: generation }, + metrics: { + requestBytes: 100, + responseBytes: 1_000, + nodeCount: 2, + maxTraversalDepth: 1, + cpuMs: 20, + memoryBytes: 10_000, + durationMs: 100, + }, + }; +} + +function acquisition() { + return { + targetId: 'simulator:sim', + targetGeneration: 'pid:100', + nodes: [{ id: 'root' }, { id: 'child', parentId: 'root' }], + viewport: { kind: 'reported', rect: { x: 0, y: 0, width: 100, height: 100 } }, + truncated: false, + residue: [], + } as const; +} + +function request(id: string) { + return { + version: 1, + id, + candidate: 'guest-simulator-framework-bridge', + simulatorUdid: 'sim', + state: 'warm', + screen: 'list', + limits: DEFAULT_SPIKE_LIMITS, + } as const; +} diff --git a/scripts/ios-ax-bridge-spike/corrected-report.ts b/scripts/ios-ax-bridge-spike/corrected-report.ts new file mode 100644 index 000000000..561e8f204 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/corrected-report.ts @@ -0,0 +1,453 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { gunzipSync, gzipSync } from 'node:zlib'; +import type { + CorrectedReport, + GateResult, + LatencySummary, + TargetedRawArtifact, +} from './corrected-types.ts'; +import { renderCorrectedMarkdown } from './corrected-markdown.ts'; +import type { SpikeCell, SpikeReport } from './types.ts'; + +export function readSpikeReport(filePath: string): SpikeReport { + return JSON.parse(gunzipSync(fs.readFileSync(filePath)).toString('utf8')) as SpikeReport; +} + +export function readTargetedArtifact(filePath: string): TargetedRawArtifact { + return JSON.parse(gunzipSync(fs.readFileSync(filePath)).toString('utf8')) as TargetedRawArtifact; +} + +export function buildCorrectedReport(options: { + sourcePath: string; + source: SpikeReport; + targetedPath: string; + targeted: TargetedRawArtifact; +}): CorrectedReport { + const readiness = [ + ...options.source.cells + .filter( + (cell) => cell.candidate === 'guest-simulator-framework-bridge' && cell.state === 'warm', + ) + .map(summarizeLatency), + ...summarizeTargetedRelaunch(options.targeted), + ]; + const coldDiagnostics = options.source.cells + .filter( + (cell) => + cell.candidate === 'guest-simulator-framework-bridge' && + (cell.state === 'cold' || cell.state === 'cold-cold'), + ) + .map(summarizeColdDiagnostic); + const hierarchy = hierarchyEvidence(options.targeted); + const hardGates = { + warm: latencyGate(readiness, 'warm', 'p50 <300 ms and p95 <500 ms per screen', false), + relaunch: relaunchGate(readiness, options.targeted), + nonresidentBootstrap: bootstrapGate(options.targeted), + boundedResources: resourceGate(options.targeted), + liveRecovery: recoveryGate(options.targeted), + hierarchy: hierarchy.gate, + preferenceControl: preferenceGate(options.source), + } as const; + const failedGates = Object.entries(hardGates).filter(([, gate]) => gate.status === 'FAIL'); + return { + schemaVersion: 'ios-simulator-ax-bridge-corrected.v3', + interpretation: 'maintainer-corrected', + generatedAt: new Date().toISOString(), + revision: options.targeted.revision, + sourceArtifact: { + path: options.sourcePath, + revision: options.source.revision, + originalDecision: 'NO-GO', + interpretation: 'superseded-stretch-only', + hostClient: options.targeted.sourceArtifact.hostClient, + }, + targetedArtifact: { + path: options.targetedPath, + revision: options.targeted.revision, + }, + target: options.targeted.target, + toolchain: options.targeted.toolchain, + host: options.targeted.host, + guestMechanism: options.targeted.guestMechanism, + readiness, + hardGates, + coldDiagnostics, + stretchFindings: [ + ...options.source.decisionReasons.map((reason) => `Original broad-run finding: ${reason}`), + 'Cold and cold-cold first-look measurements include Simulator, app, daemon, and runner readiness costs; they are diagnostics, not candidate-owned hard gates.', + 'The former warm 75/150 ms and relaunch 250 ms thresholds are stretch findings under the corrected contract.', + `Nonresident bootstrap samples were taken on a host with 1-minute load average ${options.targeted.host.loadAverage1m} on ${options.targeted.host.cpuCores} cores; per-sample load is recorded with each sample.`, + ], + decision: failedGates.length === 0 ? 'GO' : 'NO-GO', + decisionReasons: failedGates.map( + ([name, gate]) => `${name} hard gate failed: ${gate.evidence}.`, + ), + liveRecovery: options.targeted.recovery, + bootstrap: options.targeted.bootstrap, + hierarchy: hierarchy.value, + compatibilityRisk: { + interface: 'private-idb-simulator-guest', + assessment: + 'SimulatorFrameworkBridge and its accessibility wire protocol are private idb/Apple implementation details with no compatibility guarantee.', + control: + 'Pin the official idb release and observed guest SHA-256, keep this route Simulator-only behind the acquisition adapter, and re-run this verifier for every idb, Xcode, or Simulator runtime change before production adoption.', + }, + productionBoundary: 'no-production-routing-changes', + }; +} + +export function writeCorrectedReport(outputPath: string, report: CorrectedReport): void { + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(outputPath, gzipSync(`${JSON.stringify(report)}\n`, { level: 9 })); + fs.writeFileSync(markdownPath(outputPath), renderCorrectedMarkdown(report)); +} + +function markdownPath(outputPath: string): string { + const replaced = outputPath.replace(/\.json(?:\.gz)?$/u, '.md'); + return replaced === outputPath ? `${outputPath}.md` : replaced; +} + +function summarizeLatency(cell: SpikeCell): LatencySummary { + const samples = cell.acquisitionSamples; + const readable = samples.filter((sample) => sample.ok && sample.firstTree === 'readable'); + const observedGenerations = readable.flatMap((sample) => + observedGeneration(sample.acquisition?.targetGeneration), + ); + return { + state: cell.state as 'warm' | 'relaunch', + screen: cell.screen, + samples: samples.length, + readableSamples: readable.length, + readinessObservedSamples: observedGenerations.length, + generationCount: new Set(observedGenerations).size, + candidateP50Ms: optionalPercentile( + readable.map((sample) => sample.wallClockMs), + 50, + ), + candidateP95Ms: optionalPercentile( + readable.map((sample) => sample.wallClockMs), + 95, + ), + preparationP95Ms: optionalPercentile( + readable.map((sample) => sample.preparationMs), + 95, + ), + firstLookP95Ms: optionalPercentile( + readable.map((sample) => sample.firstLookMs), + 95, + ), + }; +} + +function summarizeColdDiagnostic(cell: SpikeCell): CorrectedReport['coldDiagnostics'][number] { + const readable = cell.acquisitionSamples.filter( + (sample) => sample.ok && sample.firstTree === 'readable', + ); + return { + state: cell.state as 'cold' | 'cold-cold', + screen: cell.screen, + preparationP95Ms: optionalPercentile( + readable.map((sample) => sample.preparationMs), + 95, + ), + firstLookP95Ms: optionalPercentile( + readable.map((sample) => sample.firstLookMs), + 95, + ), + interpretation: 'excluded-runner-and-app-readiness-costs', + }; +} + +function summarizeTargetedRelaunch(targeted: TargetedRawArtifact): LatencySummary[] { + return targeted.config.screens.map((screen) => { + const samples = targeted.relaunch.filter((sample) => sample.screen === screen); + const readable = samples.filter( + (sample) => + sample.response.ok && + sample.response.acquisition !== undefined && + sample.response.acquisition.nodes.length > 0 && + sample.response.acquisition.targetGeneration === `pid:${sample.appPid}`, + ); + const observedGenerations = readable.flatMap((sample) => + observedGeneration(sample.response.acquisition?.targetGeneration), + ); + return { + state: 'relaunch', + screen, + samples: samples.length, + readableSamples: readable.length, + readinessObservedSamples: readable.filter((sample) => sample.readinessAttempts > 0).length, + generationCount: new Set(observedGenerations).size, + candidateP50Ms: optionalPercentile( + readable.map((sample) => sample.durationMs), + 50, + ), + candidateP95Ms: optionalPercentile( + readable.map((sample) => sample.durationMs), + 95, + ), + preparationP95Ms: optionalPercentile( + readable.map((sample) => sample.readinessMs), + 95, + ), + firstLookP95Ms: optionalPercentile( + readable.map((sample) => sample.readinessMs + sample.durationMs), + 95, + ), + }; + }); +} + +function latencyGate( + readiness: readonly LatencySummary[], + state: 'warm' | 'relaunch', + target: string, + requireReadiness: boolean, +): GateResult { + const cells = readiness.filter((summary) => summary.state === state); + const passed = cells.filter((summary) => latencyPassed(summary, requireReadiness)); + return { + status: cells.length > 0 && passed.length === cells.length ? 'PASS' : 'FAIL', + target, + evidence: `${passed.length}/${cells.length} ${state} screen cells passed; ${cells.map(formatLatency).join('; ') || 'no cells'}`, + }; +} + +function relaunchGate( + readiness: readonly LatencySummary[], + targeted: TargetedRawArtifact, +): GateResult { + const latency = latencyGate( + readiness, + 'relaunch', + 'p95 <500 ms per representative screen with every read paired to its expected generation', + true, + ); + const complete = readiness + .filter((summary) => summary.state === 'relaunch') + .every((summary) => summary.samples === targeted.config.samples); + const expectedSampleCount = targeted.config.screens.length * targeted.config.samples; + const representativeScreens = new Set(targeted.relaunch.map((sample) => sample.screen)); + const corpusComplete = + targeted.relaunch.length === expectedSampleCount && + targeted.config.screens.every((screen) => representativeScreens.has(screen)); + return { + status: latency.status === 'PASS' && complete && corpusComplete ? 'PASS' : 'FAIL', + target: 'p95 <500 ms per screen after independently observed new-generation readiness', + evidence: `${latency.evidence}; ${targeted.relaunch.length}/${expectedSampleCount} Node-direct samples across ${representativeScreens.size}/${targeted.config.screens.length} screens`, + }; +} + +function preferenceGate(source: SpikeReport): GateResult { + const evidence = source.preferenceEvidence ?? missingPreferenceEvidence(); + const changes = evidence.diffs.flatMap((diff) => diff.changes); + const required = ['AutomationEnabled', 'IgnoreAXServerEntitlements']; + const applied = required.filter((key) => hasEnabledPreference(changes, key)); + const passed = [evidence.applied, evidence.restored, applied.length === required.length].every( + Boolean, + ); + return { + status: gateStatus(passed), + target: 'task-owned Simulator accessibility preferences applied preboot and restored', + evidence: `applied=${String(evidence.applied)}; restored=${String(evidence.restored)}; enabled keys=${applied.join(', ')}; fixture launch compatible=${String(evidence.fixtureLaunchCompatible)}`, + }; +} + +function missingPreferenceEvidence(): NonNullable { + return { + applied: false, + restored: false, + fixtureLaunchCompatible: null, + simulatorStateBefore: 'unknown', + diffs: [], + }; +} + +function gateStatus(passed: boolean): GateResult['status'] { + return passed ? 'PASS' : 'FAIL'; +} + +function hasEnabledPreference( + changes: NonNullable['diffs'][number]['changes'], + key: string, +): boolean { + return changes.some((change) => change.key === key && change.after === true); +} + +function bootstrapGate(targeted: TargetedRawArtifact): GateResult { + const usable = targeted.bootstrap.filter((sample) => sample.usableTree); + const p95 = optionalPercentile( + targeted.bootstrap.map((sample) => sample.durationMs), + 95, + ); + const passed = [ + targeted.bootstrap.length === 5, + usable.length === targeted.bootstrap.length, + p95 !== null, + p95 !== null && p95 < 2_000, + ].every(Boolean); + return { + status: passed ? 'PASS' : 'FAIL', + target: 'nonresident companion + reader bootstrap and first usable tree p95 <2,000 ms', + evidence: `${usable.length}/${targeted.bootstrap.length} usable trees; p95=${formatMs(p95)}; the timer covered guest spawn, socket connect, and the first tree after a throwaway probe observed the relaunched app's readiness (readiness p95=${formatMs( + optionalPercentile( + targeted.bootstrap.map((sample) => sample.readinessMs), + 95, + ), + )}), with no resident bridge, xcodebuild, XCTest, or agent-device runner in the timed path`, + }; +} + +function recoveryGate(targeted: TargetedRawArtifact): GateResult { + const passed = targeted.recovery.filter( + (probe) => + recoveryOutcomeMatches(probe) && + probe.recoveredResponse.ok === true && + probe.recoveredResponse.acquisition?.nodes.length !== 0, + ); + return { + status: targeted.recovery.length === 4 && passed.length === 4 ? 'PASS' : 'FAIL', + target: 'live crash, timeout, cancellation, and honest target-generation handling', + evidence: `${passed.length}/${targeted.recovery.length} probes returned a typed failure or typed unavailable-generation residue and a usable recovered response`, + }; +} + +function resourceGate(targeted: TargetedRawArtifact): GateResult { + const responses = [ + ...targeted.bootstrap.map((sample) => sample.response), + ...targeted.relaunch.map((sample) => sample.response), + ...targeted.recovery.map((probe) => probe.recoveredResponse), + ].filter((response) => response.ok); + const measured = responses.filter( + (response) => response.metrics.cpuMs !== null && response.metrics.memoryBytes !== null, + ); + const withinBounds = measured.filter( + (response) => + response.metrics.cpuMs! <= targeted.limits.maxCpuMs && + response.metrics.memoryBytes! <= targeted.limits.maxMemoryBytes, + ); + const maxCpuMs = Math.max(0, ...measured.map((response) => response.metrics.cpuMs!)); + const maxMemoryBytes = Math.max(0, ...measured.map((response) => response.metrics.memoryBytes!)); + return { + status: responses.length > 0 && withinBounds.length === responses.length ? 'PASS' : 'FAIL', + target: `guest CPU <=${targeted.limits.maxCpuMs} ms and RSS <=${targeted.limits.maxMemoryBytes} bytes per successful read`, + evidence: `${withinBounds.length}/${responses.length} successful reads measured within bounds; max CPU=${maxCpuMs.toFixed(1)} ms; max RSS=${String(maxMemoryBytes)} bytes`, + }; +} + +function recoveryOutcomeMatches(probe: TargetedRawArtifact['recovery'][number]): boolean { + const failure = probe.response.failure; + if (failure) return failure.kind === probe.operation; + if (probe.operation !== 'stale-generation') return false; + return hasUnavailableGeneration(probe.response); +} + +function hasUnavailableGeneration( + response: TargetedRawArtifact['recovery'][number]['response'], +): boolean { + const acquisition = response.acquisition; + if (!acquisition) return false; + return [ + response.ok, + acquisition.targetGeneration === null, + acquisition.residue.some(isUnavailableGeneration), + ].every(Boolean); +} + +function isUnavailableGeneration( + residue: NonNullable< + TargetedRawArtifact['bootstrap'][number]['response']['acquisition'] + >['residue'][number], +): boolean { + return residue.kind === 'unavailable-fact' && residue.fact === 'generation'; +} + +/** + * Hierarchy is a hard fact, not a presentation: the guest must either return structural depth with + * an honest truncation flag, or type its absence as provider-pruned residue. A flat tree claiming + * completeness fails. + */ +function hierarchyEvidence(targeted: TargetedRawArtifact): { + gate: GateResult; + value: CorrectedReport['hierarchy']; +} { + const usable = targeted.bootstrap.filter((sample) => sample.usableTree); + const depth = Math.max(0, ...usable.map((sample) => sample.response.metrics.maxTraversalDepth)); + const truncated = usable.some((sample) => sample.response.acquisition?.truncated === true); + const typedFlat = usable.some((sample) => + (sample.response.acquisition?.residue ?? []).some( + (residue) => residue.kind === 'provider-pruned' && residue.fields.includes('depth'), + ), + ); + if (usable.length > 0 && depth > 0) { + return { + gate: { + status: 'PASS', + target: + 'structural hierarchy acquired with typed truncation, or its absence typed as residue', + evidence: `nested tree with traversal depth ${depth} in ${usable.length}/${targeted.bootstrap.length} samples; truncated=${truncated}`, + }, + value: { + observedTraversalDepth: depth, + depthComplete: !truncated, + interpretation: 'nested-tree', + }, + }; + } + return { + gate: { + status: typedFlat ? 'PASS' : 'FAIL', + target: + 'structural hierarchy acquired with typed truncation, or its absence typed as residue', + evidence: typedFlat + ? 'flat response with typed provider-pruned/depth residue; depth is not treated as complete' + : 'no hierarchy and no typed residue observed', + }, + value: { + observedTraversalDepth: 0, + depthComplete: false, + interpretation: usable.length === 0 ? 'not-observed' : 'flat-provider-response', + }, + }; +} + +function formatLatency(summary: LatencySummary): string { + return `${summary.screen} p50/p95=${formatMs(summary.candidateP50Ms)}/${formatMs(summary.candidateP95Ms)} ready=${summary.readinessObservedSamples}/${summary.samples}`; +} + +function optionalPercentile( + values: readonly (number | undefined)[], + percentage: number, +): number | null { + const finite = values.filter( + (value): value is number => typeof value === 'number' && Number.isFinite(value), + ); + if (finite.length === 0) return null; + return percentile(finite, percentage); +} + +function observedGeneration(value: string | null | undefined): readonly string[] { + return typeof value === 'string' ? [value] : []; +} + +function latencyPassed(summary: LatencySummary, requireReadiness: boolean): boolean { + return [ + summary.readableSamples === summary.samples, + !requireReadiness || summary.readinessObservedSamples === summary.samples, + summary.candidateP50Ms !== null, + summary.candidateP50Ms !== null && summary.candidateP50Ms < 300, + summary.candidateP95Ms !== null, + summary.candidateP95Ms !== null && summary.candidateP95Ms < 500, + ].every(Boolean); +} + +function formatMs(value: number | null): string { + return value === null ? '–' : `${value.toFixed(1)} ms`; +} + +function percentile(values: readonly number[], percentage: number): number { + const sorted = [...values].sort((left, right) => left - right); + const index = Math.ceil((percentage / 100) * sorted.length) - 1; + return sorted[Math.max(0, index)]!; +} diff --git a/scripts/ios-ax-bridge-spike/corrected-types.ts b/scripts/ios-ax-bridge-spike/corrected-types.ts new file mode 100644 index 000000000..8c7bb6c32 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/corrected-types.ts @@ -0,0 +1,144 @@ +import type { + ResourceLimits, + SpikeCell, + SpikeReport, + SpikeRequest, + SpikeResponse, +} from './types.ts'; + +const CORRECTED_SCHEMA_VERSION = 'ios-simulator-ax-bridge-corrected.v3' as const; +export const TARGETED_SCHEMA_VERSION = 'ios-simulator-ax-bridge-targeted.v3' as const; + +export type TargetedRevision = SpikeReport['revision']; + +export type HostLoad = Readonly<{ loadAverage1m: number; cpuCores: number }>; + +export type TargetedBootstrapSample = Readonly<{ + index: number; + /** Candidate-owned: fresh guest spawn + connect + first usable tree, after readiness. */ + durationMs: number; + usableTree: boolean; + response: SpikeResponse; + stderr: string; + /** Fixture-owned: app relaunch until a throwaway probe first read a tree; not charged. */ + appPid: number; + readinessMs: number; + readinessAttempts: number; + host: HostLoad; +}>; + +export type TargetedRecoveryProbe = Readonly<{ + operation: 'process-crash' | 'timeout' | 'cancelled' | 'stale-generation'; + request: SpikeRequest; + response: SpikeResponse; + recoveredResponse: SpikeResponse; +}>; + +export type TargetedRelaunchSample = Readonly<{ + index: number; + screen: SpikeCell['screen']; + expectedAnchor: string; + appPid: number; + readinessMs: number; + readinessAttempts: number; + durationMs: number; + response: SpikeResponse; + stderr: string; +}>; + +export type TargetedRawArtifact = Readonly<{ + schemaVersion: typeof TARGETED_SCHEMA_VERSION; + generatedAt: string; + revision: TargetedRevision; + command: string; + sourceArtifact: Readonly<{ + path: string; + revision: TargetedRevision; + hostClient: string; + }>; + target: SpikeReport['target']; + toolchain: SpikeReport['toolchain']; + host: HostLoad; + guestMechanism: SpikeReport['guestMechanism']; + limits: ResourceLimits; + config: Readonly<{ + states: readonly SpikeCell['state'][]; + screens: readonly SpikeCell['screen'][]; + samples: number; + bootstrapSamples: number; + }>; + bootstrap: readonly TargetedBootstrapSample[]; + relaunch: readonly TargetedRelaunchSample[]; + recovery: readonly TargetedRecoveryProbe[]; +}>; + +export type LatencySummary = Readonly<{ + state: 'warm' | 'relaunch'; + screen: SpikeCell['screen']; + samples: number; + readableSamples: number; + readinessObservedSamples: number; + generationCount: number; + candidateP50Ms: number | null; + candidateP95Ms: number | null; + preparationP95Ms: number | null; + firstLookP95Ms: number | null; +}>; + +export type GateResult = Readonly<{ + status: 'PASS' | 'FAIL'; + target: string; + evidence: string; +}>; + +export type CorrectedReport = Readonly<{ + schemaVersion: typeof CORRECTED_SCHEMA_VERSION; + interpretation: 'maintainer-corrected'; + generatedAt: string; + revision: TargetedRevision; + sourceArtifact: Readonly<{ + path: string; + revision: TargetedRevision; + originalDecision: 'NO-GO'; + interpretation: 'superseded-stretch-only'; + hostClient: string; + }>; + targetedArtifact: Readonly<{ path: string; revision: TargetedRevision }>; + target: SpikeReport['target']; + toolchain: SpikeReport['toolchain']; + host: HostLoad; + guestMechanism: SpikeReport['guestMechanism']; + readiness: readonly LatencySummary[]; + hardGates: Readonly<{ + warm: GateResult; + relaunch: GateResult; + nonresidentBootstrap: GateResult; + boundedResources: GateResult; + liveRecovery: GateResult; + hierarchy: GateResult; + preferenceControl: GateResult; + }>; + coldDiagnostics: readonly Readonly<{ + state: 'cold-cold' | 'cold'; + screen: SpikeCell['screen']; + preparationP95Ms: number | null; + firstLookP95Ms: number | null; + interpretation: 'excluded-runner-and-app-readiness-costs'; + }>[]; + stretchFindings: readonly string[]; + decision: 'GO' | 'NO-GO'; + decisionReasons: readonly string[]; + liveRecovery: readonly TargetedRecoveryProbe[]; + bootstrap: readonly TargetedBootstrapSample[]; + hierarchy: Readonly<{ + observedTraversalDepth: number; + depthComplete: boolean; + interpretation: 'nested-tree' | 'flat-provider-response' | 'not-observed'; + }>; + compatibilityRisk: Readonly<{ + interface: 'private-idb-simulator-guest'; + assessment: string; + control: string; + }>; + productionBoundary: 'no-production-routing-changes'; +}>; diff --git a/scripts/ios-ax-bridge-spike/guest-adapter.ts b/scripts/ios-ax-bridge-spike/guest-adapter.ts new file mode 100644 index 000000000..b5da0ec2c --- /dev/null +++ b/scripts/ios-ax-bridge-spike/guest-adapter.ts @@ -0,0 +1,237 @@ +import fs from 'node:fs'; +import { performance } from 'node:perf_hooks'; +import { GuestConnection, GuestConnectionError, processAlive } from './guest-connection.ts'; +import { processUsageDelta } from './guest-process-metrics.ts'; +import { + acquisitionFromEnvelope, + encodeGuestFrame, + failureFromEnvelope, + guestDescribeRequest, + GuestWireError, + isTargetNotReady, + type GuestEnvelope, +} from './guest-wire.ts'; +import { DEFAULT_SPIKE_LIMITS, validateRawAcquisition } from './limits.ts'; +import { failureResponse } from './protocol.ts'; +import type { AcquisitionAdapter, AdapterOptions } from './adapter.ts'; +import type { ResourceLimits, SpikeFailure, SpikeRequest, SpikeResponse } from './types.ts'; + +const CANDIDATE = 'guest-simulator-framework-bridge' as const; +const TARGET_NOT_READY_RETRY_MS = 150; +const TARGET_NOT_READY_RETRIES = 2; + +export function createGuestSimulatorFrameworkBridgeAdapter( + options: AdapterOptions, +): AcquisitionAdapter { + const limits = options.limits ?? DEFAULT_SPIKE_LIMITS; + if (!options.guestBridge || !fs.existsSync(options.guestBridge)) { + return unavailableAdapter('guest-tool-unavailable'); + } + const session = new GuestSession(options.guestBridge, limits); + return { + candidate: CANDIDATE, + acquireBatch: (requests, acquireOptions) => session.acquireBatch(requests, acquireOptions), + close: () => session.close(), + evidence: { + terminateReaderOnNextBatch: () => session.killGuestOnNextRequestForEvidence(), + }, + }; +} + +function unavailableAdapter(code: string): AcquisitionAdapter { + return { + candidate: CANDIDATE, + async acquireBatch(requests) { + return { + responses: requests.map((request) => + failureResponse(request, { kind: 'unsupported-mechanism', code }), + ), + stderr: '', + }; + }, + }; +} + +class GuestSession { + private readonly connection: GuestConnection; + private readonly limits: ResourceLimits; + private closed = false; + private serial: Promise = Promise.resolve(); + + constructor(bridgePath: string, limits: ResourceLimits) { + this.limits = limits; + this.connection = new GuestConnection(bridgePath, limits.maxResponseBytes); + } + + acquireBatch( + requests: readonly SpikeRequest[], + options: Readonly<{ signal?: AbortSignal }> = {}, + ): Promise<{ responses: readonly SpikeResponse[]; stderr: string }> { + const operation = this.serial.then(() => this.execute(requests, options.signal)); + this.serial = operation.catch(() => undefined); + return operation; + } + + async close(): Promise { + this.closed = true; + await this.connection.close(); + } + + killGuestOnNextRequestForEvidence(): void { + this.connection.killOnNextRequest(); + } + + private async execute( + requests: readonly SpikeRequest[], + signal: AbortSignal | undefined, + ): Promise<{ responses: readonly SpikeResponse[]; stderr: string }> { + const responses: SpikeResponse[] = []; + for (const request of requests) { + const deadline = performance.now() + request.limits.maxDurationMs; + responses.push(await this.acquire(request, deadline, signal)); + } + return { responses, stderr: this.connection.takeLog() }; + } + + private async acquire( + request: SpikeRequest, + deadline: number, + signal: AbortSignal | undefined, + ): Promise { + const started = performance.now(); + const frame = encodeGuestFrame(guestDescribeRequest(request, request.limits)); + if (frame.length > request.limits.maxRequestBytes) { + return failureResponse(request, { + kind: 'transport-failure', + code: 'request-limit-exceeded', + }); + } + try { + return await this.acquireConnected(request, frame, deadline, started, signal); + } catch (error) { + return failedGuestRequest(request, frame.length, started, error); + } + } + + private async acquireConnected( + request: SpikeRequest, + frame: Buffer, + deadline: number, + started: number, + signal: AbortSignal | undefined, + ): Promise { + assertRequestActive(signal, this.closed); + const wasConnected = await this.connection.ensureConnected( + request.simulatorUdid, + deadline, + signal, + ); + const before = wasConnected ? this.connection.processSample() : undefined; + const { envelope, responseBytes } = await this.readWithReadinessRetry(frame, deadline, signal); + const resources = processUsageDelta(before, this.connection.processSample()); + return this.responseFor(request, envelope, { + requestBytes: frame.length, + responseBytes, + durationMs: performance.now() - started, + cpuMs: resources?.cpuMs ?? null, + memoryBytes: resources?.memoryBytes ?? null, + }); + } + + private async readWithReadinessRetry( + frame: Buffer, + deadline: number, + signal: AbortSignal | undefined, + ): Promise<{ envelope: GuestEnvelope; responseBytes: number }> { + let attempt = await this.connection.roundTrip(frame, deadline, signal); + for (let retry = 0; retry < TARGET_NOT_READY_RETRIES; retry += 1) { + if (attempt.envelope.ok === true || !isTargetNotReady(attempt.envelope)) break; + if (performance.now() + TARGET_NOT_READY_RETRY_MS * 2 > deadline) break; + await sleep(TARGET_NOT_READY_RETRY_MS); + attempt = await this.connection.roundTrip(frame, deadline, signal); + } + return attempt; + } + + private responseFor( + request: SpikeRequest, + envelope: GuestEnvelope, + metrics: { + requestBytes: number; + responseBytes: number; + durationMs: number; + cpuMs: number | null; + memoryBytes: number | null; + }, + ): SpikeResponse { + if (envelope.ok !== true) { + return failureResponse( + request, + failureFromEnvelope(envelope, request, processAlive), + metrics, + ); + } + const parsed = acquisitionFromEnvelope(envelope, request, request.limits); + if ('kind' in parsed) return failureResponse(request, parsed, metrics); + const validated = validateRawAcquisition(parsed.acquisition, request.limits); + if (!validated.ok) { + return failureResponse(request, { kind: 'malformed-tree', code: validated.code }, metrics); + } + const resourceFailure = resourceLimitFailure(metrics, request.limits); + if (resourceFailure) return failureResponse(request, resourceFailure, metrics); + return { + version: 1, + id: request.id, + candidate: request.candidate, + ok: true, + acquisition: parsed.acquisition, + metrics: { + ...metrics, + nodeCount: parsed.acquisition.nodes.length, + maxTraversalDepth: validated.maxTraversalDepth, + }, + }; + } +} + +function asGuestError(error: unknown): GuestConnectionError { + if (error instanceof GuestConnectionError) return error; + if (error instanceof GuestWireError) return new GuestConnectionError(error.kind, error.code); + return new GuestConnectionError('transport-failure', 'guest-unexpected-error'); +} + +function assertRequestActive(signal: AbortSignal | undefined, closed: boolean): void { + if (signal?.aborted) throw new GuestConnectionError('cancelled', 'abort-signal'); + if (closed) throw new GuestConnectionError('transport-failure', 'guest-adapter-closed'); +} + +function failedGuestRequest( + request: SpikeRequest, + requestBytes: number, + started: number, + error: unknown, +): SpikeResponse { + const guestError = asGuestError(error); + return failureResponse( + request, + { kind: guestError.kind, code: guestError.code }, + { requestBytes, durationMs: performance.now() - started }, + ); +} + +function resourceLimitFailure( + metrics: { cpuMs: number | null; memoryBytes: number | null }, + limits: ResourceLimits, +): SpikeFailure | undefined { + if (metrics.cpuMs !== null && metrics.cpuMs > limits.maxCpuMs) { + return { kind: 'transport-failure', code: 'cpu-limit-exceeded' }; + } + if (metrics.memoryBytes !== null && metrics.memoryBytes > limits.maxMemoryBytes) { + return { kind: 'transport-failure', code: 'memory-limit-exceeded' }; + } + return undefined; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/scripts/ios-ax-bridge-spike/guest-binary.test.ts b/scripts/ios-ax-bridge-spike/guest-binary.test.ts new file mode 100644 index 000000000..74982e99a --- /dev/null +++ b/scripts/ios-ax-bridge-spike/guest-binary.test.ts @@ -0,0 +1,32 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { describe, expect, test } from 'vitest'; +import { + EXPECTED_GUEST_BINARY_SHA256, + readVerifiedGuestMechanism, + sha256File, +} from './guest-binary.ts'; + +describe('guest binary provenance', () => { + test('hashes the supplied file bytes', () => { + const file = temporaryFile('verified bytes'); + expect(sha256File(file)).toBe( + '186287b2d987891f027b4bc8baaf621a3e5a4a73ec78e04b0f65dc309b1ccc03', + ); + }); + + test('rejects a supplied binary that is not the pinned idb guest', () => { + const file = temporaryFile('arbitrary bridge'); + expect(() => readVerifiedGuestMechanism(file)).toThrow( + `expected ${EXPECTED_GUEST_BINARY_SHA256}`, + ); + }); +}); + +function temporaryFile(contents: string): string { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'guest-binary-test-')); + const file = path.join(directory, 'SimulatorFrameworkBridge'); + fs.writeFileSync(file, contents); + return file; +} diff --git a/scripts/ios-ax-bridge-spike/guest-binary.ts b/scripts/ios-ax-bridge-spike/guest-binary.ts new file mode 100644 index 000000000..6a53b22ce --- /dev/null +++ b/scripts/ios-ax-bridge-spike/guest-binary.ts @@ -0,0 +1,33 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import type { GuestMechanismEvidence } from './types.ts'; + +export const EXPECTED_GUEST_BINARY_SHA256 = + '3545621d2dc98de32879ebac55e8b0c33dc8eb7cc2bfbc2d0d2d21a002c8de58'; + +export function readVerifiedGuestMechanism(guestBridge: string): GuestMechanismEvidence { + const observed = sha256File(guestBridge); + if (observed !== EXPECTED_GUEST_BINARY_SHA256) { + throw new Error( + `Guest bridge SHA-256 mismatch: expected ${EXPECTED_GUEST_BINARY_SHA256}, observed ${observed}.`, + ); + } + return { + implementation: 'idb', + release: 'v1.5.2', + companionArchive: 'idb-companion.macos-arm64.tar.gz', + companionSha256: 'f17b718a513931705542a7fbfa9cfc11895ee191562c9ffd2343cf7f8254bc08', + guestBinary: 'Resources/SimulatorFrameworkBridge', + guestBinaryExpectedSha256: EXPECTED_GUEST_BINARY_SHA256, + guestBinarySha256: observed, + transport: + 'xcrun simctl spawn SimulatorFrameworkBridge accessibility serve --idle-timeout 300 --exit-on-disconnect true; UNIX socket frames are a 4-byte big-endian length + JSON', + traversal: + 'describe with snapshotTree=true (one XCTest snapshot fetch per read) and automationMode=true asserted per request; no idb_companion, gRPC, or Python client', + client: 'node-direct-socket', + }; +} + +export function sha256File(filePath: string): string { + return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex'); +} diff --git a/scripts/ios-ax-bridge-spike/guest-connection.ts b/scripts/ios-ax-bridge-spike/guest-connection.ts new file mode 100644 index 000000000..f9af137c6 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/guest-connection.ts @@ -0,0 +1,290 @@ +import fs from 'node:fs'; +import net from 'node:net'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; +import { performance } from 'node:perf_hooks'; +import { readGuestProcessSample, type GuestProcessSample } from './guest-process-metrics.ts'; +import { GuestFrameDecoder, GuestWireError, type GuestEnvelope } from './guest-wire.ts'; +import type { SpikeFailure } from './types.ts'; + +const GUEST_IDLE_TIMEOUT_SECONDS = 300; +const CONNECT_POLL_MS = 15; + +type Pending = Readonly<{ + resolve: (frame: Buffer) => void; + reject: (error: GuestConnectionError) => void; +}>; + +export class GuestConnectionError extends Error { + readonly kind: SpikeFailure['kind']; + readonly code: string; + + constructor(kind: SpikeFailure['kind'], code: string) { + super(`${kind}/${code}`); + this.name = 'GuestConnectionError'; + this.kind = kind; + this.code = code; + } +} + +export class GuestConnection { + private readonly bridgePath: string; + private readonly maxResponseBytes: number; + private readonly socketPath = path.join( + socketDirectory(), + `${process.pid.toString(36)}-${Math.random().toString(36).slice(2, 8)}.sock`, + ); + private child?: ChildProcess; + private socket?: net.Socket; + private decoder: GuestFrameDecoder; + private pending?: Pending; + private udid?: string; + private log = ''; + private killNext = false; + + constructor(bridgePath: string, maxResponseBytes: number) { + this.bridgePath = bridgePath; + this.maxResponseBytes = maxResponseBytes; + this.decoder = new GuestFrameDecoder(maxResponseBytes); + } + + async ensureConnected( + udid: string, + deadline: number, + signal: AbortSignal | undefined, + ): Promise { + if (this.socket && !this.socket.destroyed) { + if (this.udid !== udid) { + throw new GuestConnectionError('transport-failure', 'guest-udid-changed'); + } + return true; + } + this.udid = udid; + this.spawnGuest(udid); + this.socket = await this.connect(deadline, signal); + this.decoder = new GuestFrameDecoder(this.maxResponseBytes); + const socket = this.socket; + socket.on('data', (chunk: Buffer) => this.consume(chunk)); + socket.on('close', () => { + if (this.socket === socket) this.socket = undefined; + this.failPending(new GuestConnectionError('process-crash', 'guest-exited')); + }); + socket.on('error', () => { + this.failPending(new GuestConnectionError('transport-failure', 'guest-socket-error')); + }); + return false; + } + + processSample(): GuestProcessSample | undefined { + return readGuestProcessSample(this.socketPath); + } + + roundTrip( + frame: Buffer, + deadline: number, + signal: AbortSignal | undefined, + ): Promise<{ envelope: GuestEnvelope; responseBytes: number }> { + return new Promise((resolve, reject) => { + const socket = this.socket; + if (!socket) { + reject(new GuestConnectionError('transport-failure', 'guest-not-connected')); + return; + } + const timer = setTimeout( + () => this.drop(new GuestConnectionError('timeout', 'batch-duration-limit')), + Math.max(0, deadline - performance.now()), + ); + const onAbort = (): void => this.drop(new GuestConnectionError('cancelled', 'abort-signal')); + signal?.addEventListener('abort', onAbort, { once: true }); + const settle = (): void => { + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + }; + this.pending = { + resolve: (body) => { + settle(); + try { + resolve({ + envelope: JSON.parse(body.toString('utf8')) as GuestEnvelope, + responseBytes: body.length + 4, + }); + } catch { + reject(new GuestConnectionError('malformed-tree', 'invalid-json')); + } + }, + reject: (error) => { + settle(); + reject(error); + }, + }; + socket.write(frame); + if (this.killNext) { + this.killNext = false; + killGuestProcesses(this.socketPath); + } + }); + } + + killOnNextRequest(): void { + this.killNext = true; + } + + takeLog(): string { + const log = this.log; + this.log = ''; + return log; + } + + async close(): Promise { + this.drop(new GuestConnectionError('cancelled', 'process-closed')); + await this.reapChild(); + fs.rmSync(this.socketPath, { force: true }); + } + + private spawnGuest(udid: string): void { + fs.rmSync(this.socketPath, { force: true }); + const child = spawn( + 'xcrun', + [ + 'simctl', + 'spawn', + udid, + this.bridgePath, + 'accessibility', + 'serve', + this.socketPath, + '--idle-timeout', + String(GUEST_IDLE_TIMEOUT_SECONDS), + '--exit-on-disconnect', + 'true', + ], + { stdio: ['ignore', 'pipe', 'pipe'] }, + ); + child.stdout?.on('data', (chunk: Buffer) => this.appendLog(chunk)); + child.stderr?.on('data', (chunk: Buffer) => this.appendLog(chunk)); + child.on('error', (error) => this.appendLog(Buffer.from(`${error.message}\n`))); + child.on('exit', () => { + if (this.child === child) this.child = undefined; + }); + this.child = child; + } + + private connect(deadline: number, signal: AbortSignal | undefined): Promise { + return new Promise((resolve, reject) => { + const attempt = (): void => { + if (signal?.aborted) { + void this.reapChild(); + reject(new GuestConnectionError('cancelled', 'abort-signal')); + return; + } + if (performance.now() > deadline) { + void this.reapChild(); + reject(new GuestConnectionError('timeout', 'guest-connect-timeout')); + return; + } + if (!this.child) { + reject(new GuestConnectionError('transport-failure', 'guest-exited-before-ready')); + return; + } + const socket = net.createConnection(this.socketPath); + socket.once('connect', () => { + socket.removeAllListeners('error'); + resolve(socket); + }); + socket.once('error', () => { + socket.destroy(); + setTimeout(attempt, CONNECT_POLL_MS); + }); + }; + attempt(); + }); + } + + private consume(chunk: Buffer): void { + let frames: Buffer[]; + try { + frames = this.decoder.push(chunk); + } catch (error) { + const wire = error instanceof GuestWireError ? error : undefined; + this.drop( + new GuestConnectionError( + wire?.kind ?? 'malformed-tree', + wire?.code ?? 'frame-limit-exceeded', + ), + ); + return; + } + for (const frame of frames) { + const pending = this.pending; + this.pending = undefined; + pending?.resolve(frame); + } + } + + private failPending(error: GuestConnectionError): void { + const pending = this.pending; + this.pending = undefined; + pending?.reject(error); + } + + private drop(error: GuestConnectionError): void { + this.failPending(error); + const socket = this.socket; + this.socket = undefined; + socket?.destroy(); + killGuestProcesses(this.socketPath); + } + + private async reapChild(): Promise { + const child = this.child; + if (!child || child.exitCode !== null) return; + const exited = new Promise((resolve) => child.once('exit', () => resolve())); + child.kill('SIGTERM'); + await Promise.race([exited, sleep(1_000)]); + if (child.exitCode === null) child.kill('SIGKILL'); + } + + private appendLog(chunk: Buffer): void { + if (this.log.length >= 64 * 1024) return; + this.log += chunk.toString('utf8').slice(0, 64 * 1024 - this.log.length); + } +} + +export function processAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function socketDirectory(): string { + const directory = path.join(os.tmpdir(), 'agent-device-ax'); + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + fs.chmodSync(directory, 0o700); + if (directory.length + 24 >= 104) { + throw new Error(`Socket directory path is too long for a UNIX socket: ${directory}`); + } + return directory; +} + +function killGuestProcesses(socketPath: string): void { + const found = spawnSync('pgrep', ['-f', `accessibility serve ${socketPath}`], { + encoding: 'utf8', + }); + for (const line of (found.stdout ?? '').split('\n')) { + const pid = Number(line.trim()); + if (!Number.isSafeInteger(pid) || pid <= 0) continue; + try { + process.kill(pid, 'SIGKILL'); + } catch { + // already gone + } + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/scripts/ios-ax-bridge-spike/guest-process-metrics.test.ts b/scripts/ios-ax-bridge-spike/guest-process-metrics.test.ts new file mode 100644 index 000000000..85ad5946a --- /dev/null +++ b/scripts/ios-ax-bridge-spike/guest-process-metrics.test.ts @@ -0,0 +1,17 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { parseProcessTime, processUsageDelta } from './guest-process-metrics.ts'; + +test('parses macOS process CPU time and derives one request resource sample', () => { + assert.equal(parseProcessTime('0:00.37'), 370); + assert.equal(parseProcessTime('1:02:03.50'), 3_723_500); + assert.equal(parseProcessTime('2-01:02:03.50'), 176_523_500); + assert.deepEqual( + processUsageDelta({ cpuMs: 120, memoryBytes: 10 }, { cpuMs: 350, memoryBytes: 42 }), + { cpuMs: 230, memoryBytes: 42 }, + ); + assert.deepEqual(processUsageDelta(undefined, { cpuMs: 350, memoryBytes: 42 }), { + cpuMs: 350, + memoryBytes: 42, + }); +}); diff --git a/scripts/ios-ax-bridge-spike/guest-process-metrics.ts b/scripts/ios-ax-bridge-spike/guest-process-metrics.ts new file mode 100644 index 000000000..fa5219bc7 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/guest-process-metrics.ts @@ -0,0 +1,55 @@ +import { spawnSync } from 'node:child_process'; + +export type GuestProcessSample = Readonly<{ cpuMs: number; memoryBytes: number }>; + +export function readGuestProcessSample(socketPath: string): GuestProcessSample | undefined { + const found = spawnSync( + 'pgrep', + ['-f', `SimulatorFrameworkBridge accessibility serve ${socketPath}`], + { encoding: 'utf8' }, + ); + const samples = (found.stdout ?? '') + .split('\n') + .map((value) => Number(value.trim())) + .filter((pid) => Number.isSafeInteger(pid) && pid > 0) + .flatMap(readProcessSample); + if (samples.length === 0) return undefined; + return { + cpuMs: samples.reduce((total, sample) => total + sample.cpuMs, 0), + memoryBytes: samples.reduce((maximum, sample) => Math.max(maximum, sample.memoryBytes), 0), + }; +} + +export function processUsageDelta( + before: GuestProcessSample | undefined, + after: GuestProcessSample | undefined, +): GuestProcessSample | undefined { + if (!after) return undefined; + return { + cpuMs: Math.max(0, after.cpuMs - (before?.cpuMs ?? 0)), + memoryBytes: after.memoryBytes, + }; +} + +export function parseProcessTime(value: string): number | undefined { + const match = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+(?:\.\d+)?)$/u.exec(value.trim()); + if (!match) return undefined; + const days = Number(match[1] ?? 0); + const hours = Number(match[2] ?? 0); + const minutes = Number(match[3]); + const seconds = Number(match[4]); + return (((days * 24 + hours) * 60 + minutes) * 60 + seconds) * 1_000; +} + +function readProcessSample(pid: number): GuestProcessSample[] { + const result = spawnSync('ps', ['-o', 'time=', '-o', 'rss=', '-o', 'args=', '-p', String(pid)], { + encoding: 'utf8', + }); + const fields = (result.stdout ?? '').trim().split(/\s+/u); + if (!fields[2]?.endsWith('SimulatorFrameworkBridge')) return []; + const cpuMs = parseProcessTime(fields[0] ?? ''); + const memoryKb = Number(fields[1]); + return cpuMs === undefined || !Number.isFinite(memoryKb) || memoryKb < 0 + ? [] + : [{ cpuMs, memoryBytes: memoryKb * 1_024 }]; +} diff --git a/scripts/ios-ax-bridge-spike/guest-wire.test.ts b/scripts/ios-ax-bridge-spike/guest-wire.test.ts new file mode 100644 index 000000000..c4b09e720 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/guest-wire.test.ts @@ -0,0 +1,206 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { DEFAULT_SPIKE_LIMITS } from './limits.ts'; +import { + acquisitionFromEnvelope, + elementTypeName, + encodeGuestFrame, + failureFromEnvelope, + flattenGuestTree, + GuestFrameDecoder, + guestDescribeRequest, + isTargetNotReady, + parseExpectedPid, +} from './guest-wire.ts'; +import type { SpikeRequest } from './types.ts'; + +function request(overrides: Partial = {}): SpikeRequest { + return { + version: 1, + id: 'one', + candidate: 'guest-simulator-framework-bridge', + simulatorUdid: '00000000-0000-0000-0000-000000000000', + state: 'warm', + screen: 'list', + limits: DEFAULT_SPIKE_LIMITS, + ...overrides, + }; +} + +const tree = { + XC_kAXXCAttributeElementType: 'UIApplication', + XC_kAXXCAttributeElementBaseType: 'UIApplication', + XC_kAXXCAttributeAutomationType: 1, + XC_kAXXCAttributeLabel: 'Agent Device Tester', + XC_kAXXCAttributeFrame: { X: 0, Y: 0, Width: 402, Height: 874 }, + XC_kAXXCAttributeChildren: [ + { + XC_kAXXCAttributeElementType: 'UIWindow', + XC_kAXXCAttributeElementBaseType: 'UIWindow', + XC_kAXXCAttributeAutomationType: 2, + XC_kAXXCAttributeFrame: { X: 0, Y: 0, Width: 402, Height: 874 }, + XC_kAXXCAttributeChildren: [ + { + XC_kAXXCAttributeElementType: '_UITabButton', + XC_kAXXCAttributeElementBaseType: 'UIControl', + XC_kAXXCAttributeAutomationType: 9, + XC_kAXXCAttributeLabel: 'Catalog', + XC_kAXXCAttributeIdentifier: 'tab-catalog', + XC_kAXXCAttributeValue: 1, + XC_kAXXCAttributeFrame: { X: 10, Y: 800, Width: 60, Height: 50 }, + XC_kAXXCAttributeChildren: [], + }, + ], + }, + ], +}; + +test('frames round-trip through the 4-byte big-endian length prefix, even when split', () => { + const frame = encodeGuestFrame({ verb: 'describe', pid: 7 }); + const decoder = new GuestFrameDecoder(); + assert.deepEqual(decoder.push(frame.subarray(0, 3)), []); + const frames = decoder.push(Buffer.concat([frame.subarray(3), frame])); + assert.equal(frames.length, 2); + assert.deepEqual(JSON.parse(frames[1]!.toString('utf8')), { + verb: 'describe', + pid: 7, + }); + assert.throws(() => new GuestFrameDecoder(8).push(encodeGuestFrame({ padding: 'x'.repeat(32) }))); +}); + +test('a known generation reads by pid and an unknown one resolves the frontmost app in-guest', () => { + assert.equal(parseExpectedPid('pid:4242'), 4242); + assert.equal(parseExpectedPid('gen-1'), undefined); + const byPid = guestDescribeRequest( + request({ expectedTargetGeneration: 'pid:4242' }), + DEFAULT_SPIKE_LIMITS, + ); + assert.equal(byPid.pid, 4242); + assert.equal(byPid.snapshotTree, true); + assert.equal(byPid.automationMode, true); + assert.equal(byPid.maxDepth, DEFAULT_SPIKE_LIMITS.maxTraversalDepth); + const frontmost = guestDescribeRequest(request(), DEFAULT_SPIKE_LIMITS); + assert.equal(frontmost.pid, undefined); + assert.equal(frontmost.method, 'runningboard'); +}); + +test('nested XC_kAXXC trees flatten to parent-linked raw nodes with XCTest type names', () => { + const nodes = flattenGuestTree([tree]); + assert.deepEqual( + nodes.map((node) => [node.id, node.parentId, node.type, node.role]), + [ + ['n0', undefined, 'Application', 'UIApplication'], + ['n1', 'n0', 'Window', 'UIWindow'], + ['n2', 'n1', 'Button', '_UITabButton'], + ], + ); + assert.equal(nodes[2]?.subrole, 'UIControl'); + assert.equal(nodes[2]?.value, '1'); + assert.deepEqual(nodes[2]?.frame, { x: 10, y: 800, width: 60, height: 50 }); + assert.equal(elementTypeName(undefined, 48), 'StaticText'); + assert.equal(elementTypeName(undefined, 0), 'Other'); + assert.equal(elementTypeName(undefined, 999), 'Other'); +}); + +test('a successful envelope becomes an acquisition with generation, viewport, and typed truncation', () => { + const parsed = acquisitionFromEnvelope( + { ok: true, tree: [tree], pid: 4242, truncated: true }, + request(), + DEFAULT_SPIKE_LIMITS, + ); + assert.ok('acquisition' in parsed); + if (!('acquisition' in parsed)) return; + assert.equal(parsed.acquisition.targetGeneration, 'pid:4242'); + assert.deepEqual(parsed.acquisition.viewport, { + kind: 'reported', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }); + assert.equal(parsed.acquisition.truncated, true); + assert.deepEqual(parsed.acquisition.residue, [ + { + kind: 'truncated', + dimension: 'depth', + limit: DEFAULT_SPIKE_LIMITS.maxTraversalDepth, + }, + ]); +}); + +test('an observed pid that differs from the expected generation is a typed stale generation', () => { + const parsed = acquisitionFromEnvelope( + { ok: true, tree: [tree], pid: 4243 }, + request({ expectedTargetGeneration: 'pid:4242' }), + DEFAULT_SPIKE_LIMITS, + ); + assert.deepEqual(parsed, { + kind: 'stale-generation', + code: 'target-generation-mismatch', + expectedTargetGeneration: 'pid:4242', + observedTargetGeneration: 'pid:4243', + }); +}); + +test('guest errors map to typed failures without reading free text as truth', () => { + const notReady = { + ok: false, + error_kind: 'application_not_responding', + error: 'Error kAXErrorServerNotFound getting snapshot', + }; + assert.equal(isTargetNotReady(notReady), true); + assert.deepEqual( + failureFromEnvelope(notReady, request({ expectedTargetGeneration: 'pid:1' }), () => true), + { + kind: 'transport-failure', + code: 'target-application-unavailable', + }, + ); + assert.deepEqual( + failureFromEnvelope( + { + ok: false, + error_kind: 'application_unavailable', + error: 'pid 1 has no accessibility server', + }, + request({ expectedTargetGeneration: 'pid:1' }), + () => false, + ), + { + kind: 'stale-generation', + code: 'target-generation-mismatch', + expectedTargetGeneration: 'pid:1', + }, + ); + assert.deepEqual( + failureFromEnvelope( + { ok: false, error_kind: 'application_not_responding', error: 'pid 9 did not answer' }, + request({ expectedTargetGeneration: 'pid:9' }), + () => false, + ), + { + kind: 'stale-generation', + code: 'target-generation-mismatch', + expectedTargetGeneration: 'pid:9', + }, + ); + assert.deepEqual( + failureFromEnvelope( + { ok: false, error_kind: 'application_not_responding', error: 'pid 9 did not answer' }, + request({ expectedTargetGeneration: 'pid:9' }), + () => true, + ), + { kind: 'timeout', code: 'application-not-responding' }, + ); + assert.deepEqual( + failureFromEnvelope({ ok: false, error_kind: 'reader_unavailable' }, request(), () => true), + { + kind: 'unsupported-mechanism', + code: 'reader-unavailable', + }, + ); + assert.deepEqual( + failureFromEnvelope({ ok: false, error_kind: 'bad_request' }, request(), () => true), + { + kind: 'transport-failure', + code: 'bad-request', + }, + ); +}); diff --git a/scripts/ios-ax-bridge-spike/guest-wire.ts b/scripts/ios-ax-bridge-spike/guest-wire.ts new file mode 100644 index 000000000..a3e467b5a --- /dev/null +++ b/scripts/ios-ax-bridge-spike/guest-wire.ts @@ -0,0 +1,408 @@ +import type { + RawAcquiredNode, + RawAcquisition, + ResourceLimits, + SpikeFailure, + SpikeRequest, +} from './types.ts'; + +/** + * Host side of the guest `accessibility serve` wire contract, byte-matching + * `SimulatorFrameworkBridge/AccessibilityService.m` in idb v1.5.2: frames are a 4-byte big-endian + * length prefix followed by one JSON object; the guest answers `{ ok, tree | error, error_kind, pid, + * truncated, phases, automation }`. + */ +const GUEST_FRAME_HEADER_BYTES = 4; +const GUEST_MAX_FRAME_BYTES = 16 * 1024 * 1024; + +const ATTRIBUTE = { + elementType: 'XC_kAXXCAttributeElementType', + elementBaseType: 'XC_kAXXCAttributeElementBaseType', + label: 'XC_kAXXCAttributeLabel', + value: 'XC_kAXXCAttributeValue', + identifier: 'XC_kAXXCAttributeIdentifier', + frame: 'XC_kAXXCAttributeFrame', + automationType: 'XC_kAXXCAttributeAutomationType', + children: 'XC_kAXXCAttributeChildren', +} as const; + +/** `XCUIElementType` raw values, from Xcode's `XCUIElementTypes.h`; `Any` reads as `Other`. */ +const ELEMENT_TYPE_NAMES: readonly string[] = [ + 'Other', + 'Other', + 'Application', + 'Group', + 'Window', + 'Sheet', + 'Drawer', + 'Alert', + 'Dialog', + 'Button', + 'RadioButton', + 'RadioGroup', + 'CheckBox', + 'DisclosureTriangle', + 'PopUpButton', + 'ComboBox', + 'MenuButton', + 'ToolbarButton', + 'Popover', + 'Keyboard', + 'Key', + 'NavigationBar', + 'TabBar', + 'TabGroup', + 'Toolbar', + 'StatusBar', + 'Table', + 'TableRow', + 'TableColumn', + 'Outline', + 'OutlineRow', + 'Browser', + 'CollectionView', + 'Slider', + 'PageIndicator', + 'ProgressIndicator', + 'ActivityIndicator', + 'SegmentedControl', + 'Picker', + 'PickerWheel', + 'Switch', + 'Toggle', + 'Link', + 'Image', + 'Icon', + 'SearchField', + 'ScrollView', + 'ScrollBar', + 'StaticText', + 'TextField', + 'SecureTextField', + 'DatePicker', + 'TextView', + 'Menu', + 'MenuItem', + 'MenuBar', + 'MenuBarItem', + 'Map', + 'WebView', + 'IncrementArrow', + 'DecrementArrow', + 'Timeline', + 'RatingIndicator', + 'ValueIndicator', + 'SplitGroup', + 'Splitter', + 'RelevanceIndicator', + 'ColorWell', + 'HelpTag', + 'Matte', + 'DockItem', + 'Ruler', + 'RulerMarker', + 'Grid', + 'LevelIndicator', + 'Cell', + 'LayoutArea', + 'LayoutItem', + 'Handle', + 'Stepper', + 'Tab', + 'TouchBar', + 'StatusItem', +]; + +/** + * XCTest names the application and window elements by class rather than by automation type; the + * XCTest control tree reports `Application`/`Window` where the guest attribute says 1/2. Observed + * node-for-node against the control on the catalog fixture (279/279 aligned). + */ +const CLASS_PROMOTED_TYPES: Readonly> = { + UIApplication: 'Application', + UIWindow: 'Window', +}; + +export type GuestEnvelope = Readonly>; + +export type GuestErrorKind = + | 'application_unavailable' + | 'application_not_responding' + | 'frontmost_unresolved' + | 'reader_unavailable' + | 'bad_request' + | 'assertion_failed'; + +export function encodeGuestFrame(value: unknown): Buffer { + const body = Buffer.from(JSON.stringify(value), 'utf8'); + const header = Buffer.alloc(GUEST_FRAME_HEADER_BYTES); + header.writeUInt32BE(body.length, 0); + return Buffer.concat([header, body]); +} + +/** Reassembles length-prefixed frames from a byte stream; oversize frames throw. */ +export class GuestFrameDecoder { + private buffer = Buffer.alloc(0); + private readonly maxFrameBytes: number; + + constructor(maxFrameBytes = GUEST_MAX_FRAME_BYTES) { + this.maxFrameBytes = maxFrameBytes; + } + + push(chunk: Buffer): Buffer[] { + this.buffer = Buffer.concat([this.buffer, chunk]); + const frames: Buffer[] = []; + while (this.buffer.length >= GUEST_FRAME_HEADER_BYTES) { + const length = this.buffer.readUInt32BE(0); + if (length === 0 || length > this.maxFrameBytes) { + throw new GuestWireError('malformed-tree', 'frame-limit-exceeded'); + } + if (this.buffer.length < GUEST_FRAME_HEADER_BYTES + length) break; + frames.push( + this.buffer.subarray(GUEST_FRAME_HEADER_BYTES, GUEST_FRAME_HEADER_BYTES + length), + ); + this.buffer = this.buffer.subarray(GUEST_FRAME_HEADER_BYTES + length); + } + return frames; + } +} + +export class GuestWireError extends Error { + readonly kind: SpikeFailure['kind']; + readonly code: string; + + constructor(kind: SpikeFailure['kind'], code: string) { + super(`${kind}/${code}`); + this.name = 'GuestWireError'; + this.kind = kind; + this.code = code; + } +} + +export function parseExpectedPid(generation: string | undefined): number | undefined { + if (!generation?.startsWith('pid:')) return undefined; + const pid = Number(generation.slice('pid:'.length).split(':', 1)[0]); + return Number.isSafeInteger(pid) && pid > 0 ? pid : undefined; +} + +/** + * The guest `describe` request for one spike request. A known target generation names the app by + * pid; otherwise the guest resolves the foreground app in-guest through RunningBoard (the anchor is + * required by the wire even when the method ignores it). Automation mode is asserted on every read so + * the target exposes its accessibility server without preboot preference edits; the single-fetch + * traversal keeps one Mach round trip per read. + */ +export function guestDescribeRequest( + request: SpikeRequest, + limits: ResourceLimits, +): Record { + const pid = parseExpectedPid(request.expectedTargetGeneration); + return { + verb: 'describe', + ...(pid === undefined ? { x: 1, y: 1, method: 'runningboard' } : { pid }), + snapshotTree: true, + automationMode: true, + maxDepth: limits.maxTraversalDepth, + maxNodes: limits.maxNodes, + }; +} + +function guestErrorKind(envelope: GuestEnvelope): GuestErrorKind | undefined { + const kind = envelope.error_kind; + return typeof kind === 'string' ? (kind as GuestErrorKind) : undefined; +} + +function guestErrorText(envelope: GuestEnvelope): string { + return typeof envelope.error === 'string' ? envelope.error : ''; +} + +/** + * Whether a failed read is the target's accessibility server not being reachable yet, which is what + * a freshly launched app (or one whose automation mode was just asserted) reports for a moment. + */ +export function isTargetNotReady(envelope: GuestEnvelope): boolean { + return ( + guestErrorKind(envelope) === 'application_unavailable' || + guestErrorText(envelope).includes('kAXErrorServerNotFound') + ); +} + +export function failureFromEnvelope( + envelope: GuestEnvelope, + request: SpikeRequest, + targetAlive: (pid: number) => boolean, +): SpikeFailure { + const expectedPid = parseExpectedPid(request.expectedTargetGeneration); + const kind = guestErrorKind(envelope); + const targetSymptom = + kind === 'application_unavailable' || + kind === 'application_not_responding' || + isTargetNotReady(envelope); + // The expected generation is provably gone: whatever symptom the guest saw while reaching for the + // dead pid, the honest answer is a stale generation, never a timeout the caller might retry. + if (targetSymptom && expectedPid !== undefined && !targetAlive(expectedPid)) { + return { + kind: 'stale-generation', + code: 'target-generation-mismatch', + expectedTargetGeneration: request.expectedTargetGeneration!, + }; + } + if (isTargetNotReady(envelope)) { + return { kind: 'transport-failure', code: 'target-application-unavailable' }; + } + switch (kind) { + case 'application_unavailable': + return { kind: 'transport-failure', code: 'target-application-unavailable' }; + case 'application_not_responding': + return { kind: 'timeout', code: 'application-not-responding' }; + case 'frontmost_unresolved': + return { kind: 'transport-failure', code: 'frontmost-unresolved' }; + case 'reader_unavailable': + return { kind: 'unsupported-mechanism', code: 'reader-unavailable' }; + case 'bad_request': + return { kind: 'transport-failure', code: 'bad-request' }; + default: + return { kind: 'transport-failure', code: 'guest-error' }; + } +} + +export type GuestAcquisition = Readonly<{ + acquisition: RawAcquisition; + observedPid: number | undefined; +}>; + +export function acquisitionFromEnvelope( + envelope: GuestEnvelope, + request: SpikeRequest, + limits: ResourceLimits, +): GuestAcquisition | SpikeFailure { + const expectedPid = parseExpectedPid(request.expectedTargetGeneration); + const observedPid = typeof envelope.pid === 'number' ? envelope.pid : undefined; + if (expectedPid !== undefined && observedPid !== undefined && observedPid !== expectedPid) { + return { + kind: 'stale-generation', + code: 'target-generation-mismatch', + expectedTargetGeneration: request.expectedTargetGeneration!, + observedTargetGeneration: `pid:${observedPid}`, + }; + } + const roots = guestRoots(envelope.tree); + if (!roots) return { kind: 'malformed-tree', code: 'guest-tree-shape' }; + const nodes = flattenGuestTree(roots); + const truncated = envelope.truncated === true; + const pid = expectedPid ?? observedPid; + const viewport = viewportFromRoot(roots[0]); + return { + observedPid, + acquisition: { + targetId: `simulator:${request.simulatorUdid}`, + targetGeneration: pid === undefined ? null : `pid:${pid}`, + nodes, + viewport, + truncated, + residue: [ + ...(truncated ? [truncationResidue(nodes.length, limits)] : []), + ...(pid === undefined ? [{ kind: 'unavailable-fact', fact: 'generation' } as const] : []), + ], + }, + }; +} + +function truncationResidue( + nodeCount: number, + limits: ResourceLimits, +): RawAcquisition['residue'][number] { + return nodeCount >= limits.maxNodes + ? { kind: 'truncated', dimension: 'nodes', limit: limits.maxNodes } + : { + kind: 'truncated', + dimension: 'depth', + limit: limits.maxTraversalDepth, + }; +} + +function guestRoots(tree: unknown): Record[] | undefined { + if (Array.isArray(tree)) return tree.every(isRecord) ? tree : undefined; + return isRecord(tree) ? [tree] : undefined; +} + +/** Depth-first flattening; ids follow traversal order and parents precede children. */ +export function flattenGuestTree(roots: readonly Record[]): RawAcquiredNode[] { + const nodes: RawAcquiredNode[] = []; + const visit = (element: Record, parentId: string | undefined): void => { + const id = `n${nodes.length}`; + nodes.push({ + id, + ...(parentId === undefined ? {} : { parentId }), + ...nodeFacts(element), + }); + const children = element[ATTRIBUTE.children]; + if (!Array.isArray(children)) return; + for (const child of children) if (isRecord(child)) visit(child, id); + }; + for (const root of roots) visit(root, undefined); + return nodes; +} + +function nodeFacts(element: Record): Omit { + const elementClass = optionalString(element[ATTRIBUTE.elementType]); + const baseClass = optionalString(element[ATTRIBUTE.elementBaseType]); + const type = elementTypeName(elementClass, element[ATTRIBUTE.automationType]); + const label = optionalString(element[ATTRIBUTE.label]); + const value = optionalScalar(element[ATTRIBUTE.value]); + const identifier = optionalString(element[ATTRIBUTE.identifier]); + const frame = frameFromGuest(element[ATTRIBUTE.frame]); + return { + ...(type === undefined ? {} : { type }), + ...(elementClass === undefined ? {} : { role: elementClass }), + ...(baseClass === undefined || baseClass === elementClass ? {} : { subrole: baseClass }), + ...(label === undefined ? {} : { label }), + ...(value === undefined ? {} : { value }), + ...(identifier === undefined ? {} : { identifier }), + ...(frame === undefined ? {} : { frame }), + }; +} + +export function elementTypeName( + elementClass: string | undefined, + automationType: unknown, +): string | undefined { + if (elementClass !== undefined && CLASS_PROMOTED_TYPES[elementClass]) { + return CLASS_PROMOTED_TYPES[elementClass]; + } + if (typeof automationType !== 'number' || !Number.isInteger(automationType)) return undefined; + return ELEMENT_TYPE_NAMES[automationType] ?? 'Other'; +} + +function frameFromGuest(value: unknown): RawAcquiredNode['frame'] | undefined { + if (!isRecord(value)) return undefined; + const numbers = ['X', 'Y', 'Width', 'Height'].map((key) => value[key]); + if (!numbers.every((number) => typeof number === 'number' && Number.isFinite(number))) { + return undefined; + } + const [x, y, width, height] = numbers as [number, number, number, number]; + return { x, y, width, height }; +} + +function viewportFromRoot(root: Record | undefined): RawAcquisition['viewport'] { + const frame = root === undefined ? undefined : frameFromGuest(root[ATTRIBUTE.frame]); + const isApplication = + root !== undefined && optionalString(root[ATTRIBUTE.elementType]) === 'UIApplication'; + return isApplication && frame + ? { kind: 'reported', rect: frame } + : { kind: 'missing', reason: 'not-provided' }; +} + +function optionalString(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function optionalScalar(value: unknown): string | undefined { + if (typeof value === 'string') return value.length > 0 ? value : undefined; + if (typeof value === 'number' || typeof value === 'boolean') return String(value); + return undefined; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} diff --git a/scripts/ios-ax-bridge-spike/limits.test.ts b/scripts/ios-ax-bridge-spike/limits.test.ts new file mode 100644 index 000000000..85b1dc5b5 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/limits.test.ts @@ -0,0 +1,70 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { DEFAULT_SPIKE_LIMITS, encodeFrame, validateRawAcquisition } from './limits.ts'; + +const acquisition = { + targetId: 'simulator:test', + targetGeneration: 'generation-1', + nodes: [ + { id: 'n0', type: 'XCUIElementTypeApplication', role: 'AXApplication' }, + { id: 'n1', parentId: 'n0', role: 'AXWindow', frame: { x: 0, y: 0, width: 100, height: 200 } }, + ], + viewport: { kind: 'reported', rect: { x: 0, y: 0, width: 100, height: 200 } }, + truncated: false, + residue: [], +} as const; + +test('accepts a raw tree and reports structural depth without publishing it on nodes', () => { + const result = validateRawAcquisition(acquisition, DEFAULT_SPIKE_LIMITS); + assert.equal(result.ok, true); + if (result.ok) { + assert.equal(result.maxTraversalDepth, 1); + assert.equal(result.acquisition.nodes[0]?.type, 'XCUIElementTypeApplication'); + } + assert.equal('depth' in acquisition.nodes[0], false); + assert.equal('hittable' in acquisition.nodes[0], false); +}); + +test('rejects presentation facts in the acquisition reader', () => { + const result = validateRawAcquisition( + { + ...acquisition, + nodes: [{ id: 'n0', visibleToUser: true }], + }, + DEFAULT_SPIKE_LIMITS, + ); + assert.deepEqual(result, { ok: false, code: 'node-contains-presentation-fact' }); +}); + +test('rejects cycles, missing parents, and resource-limit violations', () => { + assert.deepEqual( + validateRawAcquisition( + { ...acquisition, nodes: [{ id: 'n0', parentId: 'missing' }] }, + DEFAULT_SPIKE_LIMITS, + ), + { ok: false, code: 'parent-node-missing' }, + ); + assert.deepEqual( + validateRawAcquisition( + { ...acquisition, nodes: [{ id: 'n0', parentId: 'n0' }] }, + DEFAULT_SPIKE_LIMITS, + ), + { ok: false, code: 'traversal-depth-exceeded' }, + ); + assert.deepEqual( + validateRawAcquisition( + { ...acquisition, nodes: Array.from({ length: 3 }, (_, index) => ({ id: `n${index}` })) }, + { + ...DEFAULT_SPIKE_LIMITS, + maxNodes: 2, + }, + ), + { ok: false, code: 'node-limit-exceeded' }, + ); +}); + +test('frames are newline-delimited and byte bounded', () => { + const frame = encodeFrame({ id: 'one', text: 'ok' }); + assert.equal(frame.line.endsWith('\n'), true); + assert.equal(frame.bytes, Buffer.byteLength(frame.line)); +}); diff --git a/scripts/ios-ax-bridge-spike/limits.ts b/scripts/ios-ax-bridge-spike/limits.ts new file mode 100644 index 000000000..8a84ef303 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/limits.ts @@ -0,0 +1,230 @@ +import type { RawAcquiredNode, RawAcquisition, ResourceLimits, SpikeRect } from './types.ts'; + +export const DEFAULT_SPIKE_LIMITS: ResourceLimits = Object.freeze({ + maxRequestBytes: 64 * 1024, + maxResponseBytes: 4 * 1024 * 1024, + maxNodes: 1500, + maxTraversalDepth: 64, + maxCpuMs: 2_000, + maxMemoryBytes: 256 * 1024 * 1024, + maxDurationMs: 5_000, +}); + +const NODE_KEYS = new Set([ + 'id', + 'type', + 'parentId', + 'role', + 'subrole', + 'label', + 'value', + 'identifier', + 'frame', + 'enabled', + 'selected', + 'focused', +]); + +export type TreeValidation = + | Readonly<{ ok: true; acquisition: RawAcquisition; maxTraversalDepth: number }> + | Readonly<{ ok: false; code: string }>; + +export function encodeFrame(value: unknown): { bytes: number; line: string } { + const line = `${JSON.stringify(value)}\n`; + return { bytes: Buffer.byteLength(line), line }; +} + +export function validateRawAcquisition(value: unknown, limits: ResourceLimits): TreeValidation { + const record = readAcquisitionRecord(value); + if (!record.ok) return record; + const envelope = validateAcquisitionEnvelope(record.value); + if (!envelope.ok) return envelope; + if (record.value.nodes.length > limits.maxNodes) + return { ok: false, code: 'node-limit-exceeded' }; + const nodes = validateNodes(record.value.nodes); + if (!nodes.ok) return nodes; + const maxTraversalDepth = treeDepth(nodes.nodes); + if (maxTraversalDepth > limits.maxTraversalDepth) { + return { ok: false, code: 'traversal-depth-exceeded' }; + } + return { + ok: true, + acquisition: record.value as unknown as RawAcquisition, + maxTraversalDepth, + }; +} + +function readAcquisitionRecord( + value: unknown, +): + | { ok: true; value: Record & { nodes: unknown[] } } + | { ok: false; code: string } { + if (!isRecord(value)) return { ok: false, code: 'acquisition-not-object' }; + if (typeof value.targetId !== 'string' || value.targetId.length === 0) { + return { ok: false, code: 'target-id-missing' }; + } + if (value.targetGeneration !== null && typeof value.targetGeneration !== 'string') { + return { ok: false, code: 'target-generation-invalid' }; + } + if (!Array.isArray(value.nodes)) return { ok: false, code: 'nodes-not-array' }; + return { ok: true, value: value as Record & { nodes: unknown[] } }; +} + +function validateAcquisitionEnvelope( + value: Record, +): { ok: true } | { ok: false; code: string } { + if (typeof value.truncated !== 'boolean') return { ok: false, code: 'truncated-invalid' }; + if (!validateViewport(value.viewport)) return { ok: false, code: 'viewport-invalid' }; + if (!validateResidue(value.residue)) return { ok: false, code: 'residue-invalid' }; + return { ok: true }; +} + +function validateNodes( + rawNodes: readonly unknown[], +): { ok: true; nodes: RawAcquiredNode[] } | { ok: false; code: string } { + const nodes: RawAcquiredNode[] = []; + const ids = new Set(); + for (const rawNode of rawNodes) { + const node = validateNode(rawNode); + if (!node.ok) return node; + if (ids.has(node.node.id)) return { ok: false, code: 'duplicate-node-id' }; + ids.add(node.node.id); + nodes.push(node.node); + } + return validateParents(nodes, ids); +} + +function validateParents( + nodes: readonly RawAcquiredNode[], + ids: ReadonlySet, +): { ok: true; nodes: RawAcquiredNode[] } | { ok: false; code: string } { + for (const node of nodes) { + if (node.parentId !== undefined && !ids.has(node.parentId)) { + return { ok: false, code: 'parent-node-missing' }; + } + } + return { ok: true, nodes: [...nodes] }; +} + +function validateNode( + value: unknown, +): { ok: true; node: RawAcquiredNode } | { ok: false; code: string } { + const node = asNodeRecord(value); + if (!node) return { ok: false, code: 'node-not-object' }; + const code = nodeValidationCode(node); + return code === undefined ? { ok: true, node: node as RawAcquiredNode } : { ok: false, code }; +} + +function asNodeRecord(value: unknown): Record | undefined { + return isRecord(value) ? value : undefined; +} + +function nodeValidationCode(value: Record): string | undefined { + const identityCode = nodeIdentityCode(value); + if (identityCode) return identityCode; + return nodeFactCode(value); +} + +function nodeIdentityCode(value: Record): string | undefined { + if (Object.keys(value).some((key) => !NODE_KEYS.has(key))) { + return 'node-contains-presentation-fact'; + } + if (typeof value.id !== 'string' || value.id.length === 0) return 'node-id-missing'; + if (value.parentId !== undefined && typeof value.parentId !== 'string') { + return 'parent-id-invalid'; + } + return undefined; +} + +function nodeFactCode(value: Record): string | undefined { + const textCode = optionalStringCode(value); + if (textCode) return textCode; + const booleanCode = optionalBooleanCode(value); + if (booleanCode) return booleanCode; + return value.frame !== undefined && !validateRect(value.frame) ? 'frame-invalid' : undefined; +} + +function optionalStringCode(value: Record): string | undefined { + for (const key of ['type', 'role', 'subrole', 'label', 'value', 'identifier'] as const) { + if (value[key] !== undefined && typeof value[key] !== 'string') return `${key}-invalid`; + } + return undefined; +} + +function optionalBooleanCode(value: Record): string | undefined { + for (const key of ['enabled', 'selected', 'focused'] as const) { + if (value[key] !== undefined && typeof value[key] !== 'boolean') return `${key}-invalid`; + } + return undefined; +} + +function validateViewport(value: unknown): boolean { + if (!isRecord(value) || typeof value.kind !== 'string') return false; + if (value.kind === 'missing') { + return ( + value.reason === 'not-provided' || + value.reason === 'not-supported' || + value.reason === 'invalid' + ); + } + return (value.kind === 'reported' || value.kind === 'derived') && validateRect(value.rect); +} + +function validateRect(value: unknown): value is SpikeRect { + if (!isRecord(value)) return false; + for (const key of ['x', 'y', 'width', 'height']) { + if (typeof value[key] !== 'number' || !Number.isFinite(value[key])) return false; + } + const width = value.width; + const height = value.height; + return typeof width === 'number' && typeof height === 'number' && width >= 0 && height >= 0; +} + +function validateResidue(value: unknown): boolean { + if (!Array.isArray(value)) return false; + for (const residue of value) { + if (!validateResidueItem(residue)) return false; + } + return true; +} + +const RESIDUE_VALIDATORS: Readonly) => boolean>> = { + 'provider-pruned': (value) => + Array.isArray(value.fields) && value.fields.every((field) => typeof field === 'string'), + 'missing-viewport': (value) => + ['not-provided', 'not-supported', 'invalid'].includes(String(value.reason)), + truncated: (value) => ['nodes', 'depth', 'payload'].includes(String(value.dimension)), + 'stale-generation': (value) => value.expected === undefined || typeof value.expected === 'string', + 'unavailable-fact': (value) => typeof value.fact === 'string', + 'fallback-source': (value) => typeof value.producer === 'string', +}; + +function validateResidueItem(value: unknown): boolean { + if (!isRecord(value) || typeof value.kind !== 'string') return false; + return RESIDUE_VALIDATORS[value.kind]?.(value) ?? false; +} + +function treeDepth(nodes: readonly RawAcquiredNode[]): number { + const byId = new Map(nodes.map((node) => [node.id, node])); + const memo = new Map(); + const visiting = new Set(); + let maximum = 0; + for (const node of nodes) maximum = Math.max(maximum, depth(node.id)); + return maximum; + + function depth(id: string): number { + const cached = memo.get(id); + if (cached !== undefined) return cached; + if (visiting.has(id)) return Number.POSITIVE_INFINITY; + visiting.add(id); + const parentId = byId.get(id)?.parentId; + const result = parentId === undefined ? 0 : depth(parentId) + 1; + visiting.delete(id); + memo.set(id, result); + return result; + } +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} diff --git a/scripts/ios-ax-bridge-spike/protocol.ts b/scripts/ios-ax-bridge-spike/protocol.ts new file mode 100644 index 000000000..4ba7e560d --- /dev/null +++ b/scripts/ios-ax-bridge-spike/protocol.ts @@ -0,0 +1,24 @@ +import type { ResourceMetrics, SpikeFailure, SpikeRequest, SpikeResponse } from './types.ts'; + +export function failureResponse( + request: SpikeRequest, + failure: SpikeFailure, + metrics: Partial = {}, +): SpikeResponse { + return { + version: 1, + id: request.id, + candidate: request.candidate, + ok: false, + failure, + metrics: { + requestBytes: metrics.requestBytes ?? 0, + responseBytes: metrics.responseBytes ?? 0, + nodeCount: metrics.nodeCount ?? 0, + maxTraversalDepth: metrics.maxTraversalDepth ?? 0, + cpuMs: metrics.cpuMs ?? null, + memoryBytes: metrics.memoryBytes ?? null, + durationMs: metrics.durationMs ?? 0, + }, + }; +} diff --git a/scripts/ios-ax-bridge-spike/targeted-evidence.ts b/scripts/ios-ax-bridge-spike/targeted-evidence.ts new file mode 100644 index 000000000..2d98c3c51 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/targeted-evidence.ts @@ -0,0 +1,247 @@ +import { execFileSync } from 'node:child_process'; +import os from 'node:os'; +import { performance } from 'node:perf_hooks'; +import { createGuestSimulatorFrameworkBridgeAdapter } from './guest-adapter.ts'; +import type { SpikeConfig } from './config.ts'; +import { runTargetedRelaunch } from './targeted-relaunch.ts'; +import { awaitAppReadiness } from './targeted-readiness.ts'; +import { + adapterOptions, + missingResponse, + targetedRequest, + usableTree, +} from './targeted-request.ts'; +import { + bootSimulator, + readRunningAppPids, + shutdownSimulator, + terminateApp, +} from '../ios-snapshot-benchmark/lifecycle.ts'; +import type { SpikeRequest, SpikeResponse } from './types.ts'; +import type { + HostLoad, + TargetedBootstrapSample, + TargetedRelaunchSample, + TargetedRecoveryProbe, +} from './corrected-types.ts'; + +const APP_ID = 'com.callstack.agentdevicelab'; +const BOOTSTRAP_SAMPLES = 5; + +type GuestAdapter = ReturnType; + +export type TargetedRunResult = Readonly<{ + bootstrap: readonly TargetedBootstrapSample[]; + relaunch: readonly TargetedRelaunchSample[]; + recovery: readonly TargetedRecoveryProbe[]; + host: HostLoad; +}>; + +/** + * Live nonresident-bootstrap and recovery evidence for the guest bridge. + * + * Boundary: the Simulator is booted and the fixture app is relaunched for every bootstrap sample. + * App readiness is *observed*, not assumed from a pid: a throwaway probe bridge polls until the new + * app generation answers with a tree, then is torn down, so the timed sample starts with no resident + * bridge and a ready app. Automation mode is asserted per request by the guest. + */ +export async function runTargetedEvidence(config: SpikeConfig): Promise { + bootSimulator(config.udid); + try { + const bootstrap = await runNonresidentBootstrap(config); + return { + bootstrap, + relaunch: await runTargetedRelaunch(config), + recovery: await runLiveRecovery(config, bootstrap), + host: hostLoad(), + }; + } finally { + if (!config.keepDevice) shutdownSimulator(config.udid); + } +} + +async function runNonresidentBootstrap( + config: SpikeConfig, +): Promise { + const samples: TargetedBootstrapSample[] = []; + for (let index = 1; index <= BOOTSTRAP_SAMPLES; index += 1) { + samples.push(await captureBootstrap(config, index)); + } + return samples; +} + +async function captureBootstrap( + config: SpikeConfig, + index: number, +): Promise { + const appPid = await relaunchApp(config.udid); + const readiness = await awaitAppReadiness(config, appPid, 'list'); + await assertNoResidentGuest(); + const adapter = createGuestSimulatorFrameworkBridgeAdapter(adapterOptions(config)); + const started = performance.now(); + const result = await adapter.acquireBatch([ + targetedRequest(config, `bootstrap-${index}`, { + expectedTargetGeneration: `pid:${appPid}`, + }), + ]); + const durationMs = performance.now() - started; + await adapter.close?.(); + const response = result.responses[0] ?? missingResponse(`bootstrap-${index}`); + return { + index, + durationMs, + usableTree: usableTree(response), + response, + stderr: result.stderr, + appPid, + readinessMs: readiness.readinessMs, + readinessAttempts: readiness.attempts, + host: hostLoad(), + }; +} + +/** A killed guest can linger for a moment while launchd_sim reaps it; wait briefly, then fail closed. */ +async function assertNoResidentGuest(): Promise { + const deadline = Date.now() + 5_000; + let found = residentGuestPids(); + while (found.length > 0 && Date.now() < deadline) { + await sleep(100); + found = residentGuestPids(); + } + if (found.length > 0) { + throw new Error(`A guest bridge is still resident before a nonresident sample: pids ${found}`); + } +} + +function residentGuestPids(): number[] { + return execFileSync( + 'sh', + ['-c', 'pgrep -f "SimulatorFrameworkBridge accessibility serve" || true'], + { encoding: 'utf8' }, + ) + .split('\n') + .map((value) => Number(value.trim())) + .filter((value) => Number.isSafeInteger(value) && value > 0); +} + +async function runLiveRecovery( + config: SpikeConfig, + bootstrap: readonly TargetedBootstrapSample[], +): Promise { + const adapter = createGuestSimulatorFrameworkBridgeAdapter(adapterOptions(config)); + try { + const probes: TargetedRecoveryProbe[] = []; + await adapter.acquireBatch([targetedRequest(config, 'recovery-prime')]); + adapter.evidence?.terminateReaderOnNextBatch?.(); + probes.push( + await recoveryProbe( + config, + adapter, + 'process-crash', + targetedRequest(config, 'recovery-crash'), + ), + ); + probes.push( + await recoveryProbe( + config, + adapter, + 'timeout', + targetedRequest(config, 'recovery-timeout', { + limits: { ...config.limits, maxDurationMs: 1 }, + }), + ), + ); + probes.push(await cancellationProbe(config, adapter)); + probes.push( + await recoveryProbe( + config, + adapter, + 'stale-generation', + targetedRequest(config, 'recovery-stale-generation', { + expectedTargetGeneration: `pid:${deadGeneration(bootstrap)}`, + }), + ), + ); + return probes; + } finally { + await adapter.close?.(); + } +} + +/** A previous app generation's pid: the bootstrap relaunches guarantee it is dead. */ +function deadGeneration(bootstrap: readonly TargetedBootstrapSample[]): number { + const previous = bootstrap.at(-2)?.appPid; + if (previous === undefined) throw new Error('No previous app generation to test staleness.'); + return previous; +} + +async function recoveryProbe( + config: SpikeConfig, + adapter: GuestAdapter, + operation: TargetedRecoveryProbe['operation'], + probeRequest: SpikeRequest, +): Promise { + const result = await adapter.acquireBatch([probeRequest]); + return { + operation, + request: probeRequest, + response: result.responses[0] ?? missingResponse(probeRequest.id), + recoveredResponse: await healthyResponse(config, adapter, `${probeRequest.id}-recovered`), + }; +} + +async function cancellationProbe( + config: SpikeConfig, + adapter: GuestAdapter, +): Promise { + const probeRequest = targetedRequest(config, 'recovery-cancelled'); + const controller = new AbortController(); + const pending = adapter.acquireBatch([probeRequest], { + signal: controller.signal, + }); + setTimeout(() => controller.abort(), 1); + const result = await pending; + return { + operation: 'cancelled', + request: probeRequest, + response: result.responses[0] ?? missingResponse(probeRequest.id), + recoveredResponse: await healthyResponse(config, adapter, 'recovery-cancelled-recovered'), + }; +} + +async function healthyResponse( + config: SpikeConfig, + adapter: GuestAdapter, + id: string, +): Promise { + const result = await adapter.acquireBatch([targetedRequest(config, id)]); + return result.responses[0] ?? missingResponse(id); +} + +/** A fresh app generation: terminate, launch, and wait for exactly one running pid. */ +async function relaunchApp(udid: string): Promise { + terminateApp(udid, APP_ID); + execFileSync('xcrun', ['simctl', 'launch', udid, APP_ID], { + encoding: 'utf8', + timeout: 60_000, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const pids = readRunningAppPids(udid, APP_ID); + if (pids.length === 1) return pids[0]!; + await sleep(100); + } + throw new Error(`App ${APP_ID} did not start on ${udid}.`); +} + +function hostLoad(): HostLoad { + return { + loadAverage1m: Number(os.loadavg()[0]?.toFixed(2)), + cpuCores: os.cpus().length, + }; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/scripts/ios-ax-bridge-spike/targeted-readiness.ts b/scripts/ios-ax-bridge-spike/targeted-readiness.ts new file mode 100644 index 000000000..832566400 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/targeted-readiness.ts @@ -0,0 +1,71 @@ +import { performance } from 'node:perf_hooks'; +import { createGuestSimulatorFrameworkBridgeAdapter } from './guest-adapter.ts'; +import { + adapterOptions, + missingResponse, + targetedRequest, + usableTree, +} from './targeted-request.ts'; +import type { SpikeConfig } from './config.ts'; +import type { ScreenId } from '../ios-snapshot-benchmark/types.ts'; + +const READINESS_POLL_MS = 200; +const READINESS_DEADLINE_MS = 90_000; +const READINESS_PROBE_REQUEST_MS = 60_000; + +export async function awaitAppReadiness( + config: SpikeConfig, + appPid: number, + screen: ScreenId, + expectedAnchor?: string, +): Promise<{ readinessMs: number; attempts: number }> { + const limits = { ...config.limits, maxDurationMs: READINESS_PROBE_REQUEST_MS }; + const probe = createGuestSimulatorFrameworkBridgeAdapter({ + ...adapterOptions(config), + limits, + }); + const started = performance.now(); + let attempts = 0; + try { + for (;;) { + assertReadinessDeadline(started, appPid, screen); + attempts += 1; + if (await probeReady(config, probe, limits, appPid, screen, expectedAnchor, attempts)) { + return { readinessMs: performance.now() - started, attempts }; + } + await sleep(READINESS_POLL_MS); + } + } finally { + await probe.close?.(); + } +} + +async function probeReady( + config: SpikeConfig, + probe: ReturnType, + limits: SpikeConfig['limits'], + appPid: number, + screen: ScreenId, + expectedAnchor: string | undefined, + attempt: number, +): Promise { + const id = `readiness-${screen}-${appPid}-${attempt}`; + const result = await probe.acquireBatch([ + targetedRequest(config, id, { + state: 'relaunch', + screen, + expectedTargetGeneration: `pid:${appPid}`, + limits, + }), + ]); + return usableTree(result.responses[0] ?? missingResponse(id), `pid:${appPid}`, expectedAnchor); +} + +function assertReadinessDeadline(started: number, appPid: number, screen: ScreenId): void { + if (performance.now() - started < READINESS_DEADLINE_MS) return; + throw new Error(`App generation pid:${appPid} did not expose ${screen} in time.`); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/scripts/ios-ax-bridge-spike/targeted-relaunch.ts b/scripts/ios-ax-bridge-spike/targeted-relaunch.ts new file mode 100644 index 000000000..db8317d71 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/targeted-relaunch.ts @@ -0,0 +1,151 @@ +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { + openFixture, + pressFixtureTarget, + scrollFixtureSetup, + snapshotFixture, + type CliContext, +} from '../ios-snapshot-benchmark/command.ts'; +import { screenFixture } from '../ios-snapshot-benchmark/definitions.ts'; +import { + fixtureOperationFromCli, + prepareFixture, + requireFixtureOperationSuccess, +} from '../ios-snapshot-benchmark/fixture-admission.ts'; +import { readRunningAppPids, stopDaemon } from '../ios-snapshot-benchmark/lifecycle.ts'; +import { closeSession } from '../ios-snapshot-benchmark/local-runner.ts'; +import { createGuestSimulatorFrameworkBridgeAdapter } from './guest-adapter.ts'; +import { awaitAppReadiness } from './targeted-readiness.ts'; +import { adapterOptions, missingResponse, targetedRequest } from './targeted-request.ts'; +import type { SpikeConfig } from './config.ts'; +import type { TargetedRelaunchSample } from './corrected-types.ts'; +import type { ScreenFixture, ScreenId } from '../ios-snapshot-benchmark/types.ts'; + +const RELAUNCH_SAMPLES = 20; +const SCREENS: readonly ScreenId[] = [ + 'quiet', + 'list', + 'nested-scroll', + 'alert', + 'system-surface', + 'xctest-stress', +]; + +export async function runTargetedRelaunch( + config: SpikeConfig, +): Promise { + const samples: TargetedRelaunchSample[] = []; + for (const screen of SCREENS) { + samples.push(...(await runScreen(config, screen))); + } + return samples; +} + +async function runScreen( + config: SpikeConfig, + screen: ScreenId, +): Promise { + const fixture = screenFixture(screen); + const expectedAnchor = fixture.postSetupAnchorText ?? fixture.anchorText; + const context = contextFor(config, screen); + const adapter = createGuestSimulatorFrameworkBridgeAdapter(adapterOptions(config)); + const samples: TargetedRelaunchSample[] = []; + let previousPid: number | undefined; + try { + for (let index = 1; index <= RELAUNCH_SAMPLES; index += 1) { + await prepareRelaunch(context, fixture, screen); + const appPid = exactAppPid(config.udid, fixture.app); + assertNewPid(previousPid, appPid, screen); + previousPid = appPid; + samples.push(await captureRelaunch(config, adapter, screen, expectedAnchor, appPid, index)); + } + return samples; + } finally { + await adapter.close?.(); + closeSession(context); + stopDaemon(config.repoRoot, config.stateDir); + } +} + +async function prepareRelaunch( + context: CliContext, + fixture: ScreenFixture, + screen: ScreenId, +): Promise { + const opened = openFixture(context, fixture, { relaunch: true }); + requireFixtureOperationSuccess( + fixtureOperationFromCli(opened, `relaunch ${screen} setup`), + `relaunch ${screen} setup`, + 'cell-state', + ); + await prepareFixture(fixture, { + observe: () => fixtureOperationFromCli(snapshotFixture(context), 'fixture snapshot'), + scrollToBottom: () => + fixtureOperationFromCli(scrollFixtureSetup(context), 'fixture scroll bottom'), + openAlert: () => + fixtureOperationFromCli( + pressFixtureTarget(context, 'id="automation-open-alert"'), + 'fixture open alert', + ), + }); +} + +async function captureRelaunch( + config: SpikeConfig, + adapter: ReturnType, + screen: ScreenId, + expectedAnchor: string, + appPid: number, + index: number, +): Promise { + const readiness = await awaitAppReadiness(config, appPid, screen, expectedAnchor); + const id = `relaunch-${screen}-${index}`; + const started = performance.now(); + const result = await adapter.acquireBatch([ + targetedRequest(config, id, { + state: 'relaunch', + screen, + expectedTargetGeneration: `pid:${appPid}`, + }), + ]); + return { + index, + screen, + expectedAnchor, + appPid, + readinessMs: readiness.readinessMs, + readinessAttempts: readiness.attempts, + durationMs: performance.now() - started, + response: result.responses[0] ?? missingResponse(id), + stderr: result.stderr, + }; +} + +function assertNewPid(previousPid: number | undefined, appPid: number, screen: ScreenId): void { + if (previousPid !== appPid) return; + throw new Error(`Relaunch ${screen} reused app pid ${appPid}.`); +} + +function exactAppPid(udid: string, appId: string): number { + const pids = readRunningAppPids(udid, appId); + if (pids.length !== 1) { + throw new Error( + `Expected one ${appId} process on ${udid}, observed ${pids.join(', ') || 'none'}.`, + ); + } + return pids[0]!; +} + +function contextFor(config: SpikeConfig, screen: ScreenId): CliContext { + return { + repoRoot: config.repoRoot, + stateDir: config.stateDir, + session: `ax-bridge-relaunch-${screen}`, + udid: config.udid, + derivedPath: path.join(config.derivedPath, screen), + }; +} + +export const TARGETED_RELAUNCH_SCREENS = SCREENS; +export const TARGETED_RELAUNCH_SAMPLES = RELAUNCH_SAMPLES; diff --git a/scripts/ios-ax-bridge-spike/targeted-request.ts b/scripts/ios-ax-bridge-spike/targeted-request.ts new file mode 100644 index 000000000..e0c0c8d7f --- /dev/null +++ b/scripts/ios-ax-bridge-spike/targeted-request.ts @@ -0,0 +1,68 @@ +import type { SpikeConfig } from './config.ts'; +import type { SpikeRequest, SpikeResponse } from './types.ts'; + +export function targetedRequest( + config: SpikeConfig, + id: string, + overrides: Partial = {}, +): SpikeRequest { + return { + version: 1, + id, + candidate: 'guest-simulator-framework-bridge', + simulatorUdid: config.udid, + state: 'warm', + screen: 'list', + limits: config.limits, + ...overrides, + }; +} + +export function missingResponse(id: string): SpikeResponse { + return { + version: 1, + id, + candidate: 'guest-simulator-framework-bridge', + ok: false, + failure: { kind: 'transport-failure', code: 'missing-response' }, + metrics: { + requestBytes: 0, + responseBytes: 0, + nodeCount: 0, + maxTraversalDepth: 0, + cpuMs: null, + memoryBytes: null, + durationMs: 0, + }, + }; +} + +export function usableTree( + response: SpikeResponse, + expectedGeneration?: string, + expectedAnchor?: string, +): boolean { + const acquisition = response.acquisition; + if (!response.ok) return false; + if (!acquisition || acquisition.nodes.length === 0) return false; + return [ + matchesOptional(expectedGeneration, acquisition.targetGeneration), + containsOptionalAnchor(expectedAnchor, acquisition.nodes), + ].every(Boolean); +} + +export function adapterOptions(config: SpikeConfig) { + return { guestBridge: config.guestBridge, limits: config.limits }; +} + +function matchesOptional(expected: string | undefined, observed: string | null): boolean { + return expected === undefined || expected === observed; +} + +function containsOptionalAnchor( + expected: string | undefined, + nodes: NonNullable['nodes'], +): boolean { + if (expected === undefined) return true; + return nodes.some((node) => node.label === expected || node.value === expected); +} diff --git a/scripts/ios-ax-bridge-spike/targeted-run.ts b/scripts/ios-ax-bridge-spike/targeted-run.ts new file mode 100644 index 000000000..3a44abc16 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/targeted-run.ts @@ -0,0 +1,81 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { gzipSync } from 'node:zlib'; +import { fileURLToPath } from 'node:url'; +import { parseConfig } from './config.ts'; +import { + buildCorrectedReport, + readSpikeReport, + readTargetedArtifact, + writeCorrectedReport, +} from './corrected-report.ts'; +import { readVerifiedGuestMechanism } from './guest-binary.ts'; +import { runTargetedEvidence } from './targeted-evidence.ts'; +import { TARGETED_RELAUNCH_SAMPLES, TARGETED_RELAUNCH_SCREENS } from './targeted-relaunch.ts'; +import { TARGETED_SCHEMA_VERSION, type TargetedRawArtifact } from './corrected-types.ts'; +import { readGitRevision, readTarget, readToolchain } from '../ios-snapshot-benchmark/host.ts'; + +const SOURCE = 'docs/evidence/ios-simulator-ax-bridge-2026-09-01-final.json.gz'; +const TARGETED = 'docs/evidence/ios-simulator-ax-bridge-2026-09-02-targeted.json.gz'; +const CORRECTED = 'docs/evidence/ios-simulator-ax-bridge-2026-09-02-corrected.json.gz'; + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + await main(process.argv.slice(2)); +} + +async function main(argv: readonly string[]): Promise { + const config = parseConfig(argv); + try { + await capture(config); + } finally { + fs.rmSync(config.stateDir, { recursive: true, force: true }); + } +} + +async function capture(config: ReturnType): Promise { + const revision = readGitRevision(config.repoRoot); + if (revision.dirty) { + throw new Error('Targeted bridge evidence must be captured from a clean Git revision.'); + } + const guestMechanism = readVerifiedGuestMechanism(config.guestBridge); + const source = readSpikeReport(SOURCE); + const evidence = await runTargetedEvidence(config); + const target = readTarget(config.udid, 'com.callstack.agentdevicelab'); + const artifact: TargetedRawArtifact = { + schemaVersion: TARGETED_SCHEMA_VERSION, + generatedAt: new Date().toISOString(), + revision, + command: + 'pnpm bench:ios-ax-bridge:targeted -- --udid --guest-bridge /Resources/SimulatorFrameworkBridge', + sourceArtifact: { + path: SOURCE, + revision: source.revision, + hostClient: String((source.guestMechanism as { client?: unknown }).client ?? 'unknown'), + }, + target: { udid: target.udid, name: target.name, runtime: target.runtime }, + toolchain: readToolchain(), + host: evidence.host, + guestMechanism, + limits: config.limits, + config: { + states: ['warm', 'relaunch'], + screens: TARGETED_RELAUNCH_SCREENS, + samples: TARGETED_RELAUNCH_SAMPLES, + bootstrapSamples: evidence.bootstrap.length, + }, + bootstrap: evidence.bootstrap, + relaunch: evidence.relaunch, + recovery: evidence.recovery, + }; + fs.writeFileSync(TARGETED, gzipSync(`${JSON.stringify(artifact)}\n`, { level: 9 })); + writeCorrectedReport( + CORRECTED, + buildCorrectedReport({ + sourcePath: SOURCE, + source, + targetedPath: TARGETED, + targeted: readTargetedArtifact(TARGETED), + }), + ); + process.stdout.write(`${CORRECTED}\n`); +} diff --git a/scripts/ios-ax-bridge-spike/types.ts b/scripts/ios-ax-bridge-spike/types.ts new file mode 100644 index 000000000..826b99f72 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/types.ts @@ -0,0 +1,149 @@ +import type { + IosAcquisitionResidue, + IosViewportEvidence, +} from '@agent-device/contracts/ios-snapshot'; +import type { LocalState, ScreenId } from '../ios-snapshot-benchmark/types.ts'; + +export type CandidateId = 'guest-simulator-framework-bridge' | 'xctest-control'; + +export type SpikeFailureKind = + | 'unsupported-mechanism' + | 'malformed-tree' + | 'stale-generation' + | 'timeout' + | 'cancelled' + | 'process-crash' + | 'transport-failure'; + +export type SpikeFailure = Readonly<{ + kind: SpikeFailureKind; + code?: string; + expectedTargetGeneration?: string; + observedTargetGeneration?: string; +}>; + +export type SpikeRect = Readonly<{ x: number; y: number; width: number; height: number }>; + +export type RawAcquiredNode = Readonly<{ + id: string; + type?: string; + parentId?: string; + role?: string; + subrole?: string; + label?: string; + value?: string; + identifier?: string; + frame?: SpikeRect; + enabled?: boolean; + selected?: boolean; + focused?: boolean; +}>; + +export type RawAcquisition = Readonly<{ + targetId: string; + targetGeneration: string | null; + nodes: readonly RawAcquiredNode[]; + viewport: IosViewportEvidence; + truncated: boolean; + residue: readonly IosAcquisitionResidue[]; +}>; + +export type ResourceLimits = Readonly<{ + maxRequestBytes: number; + maxResponseBytes: number; + maxNodes: number; + maxTraversalDepth: number; + maxCpuMs: number; + maxMemoryBytes: number; + maxDurationMs: number; +}>; + +export type ResourceMetrics = Readonly<{ + requestBytes: number; + responseBytes: number; + nodeCount: number; + maxTraversalDepth: number; + cpuMs: number | null; + memoryBytes: number | null; + durationMs: number; +}>; + +export type SpikeRequest = Readonly<{ + version: 1; + id: string; + candidate: CandidateId; + simulatorUdid: string; + state: LocalState; + screen: ScreenId | 'unprepared-surface'; + expectedTargetGeneration?: string; + limits: ResourceLimits; +}>; + +export type SpikeResponse = Readonly<{ + version: 1; + id: string; + candidate: CandidateId; + ok: boolean; + acquisition?: RawAcquisition; + failure?: SpikeFailure; + metrics: ResourceMetrics; +}>; + +export type SpikeSample = Readonly<{ + wallClockMs: number; + preparationMs?: number; + firstLookMs?: number; + firstTree: 'readable' | 'empty' | 'unreadable' | 'not-observed'; + ok: boolean; + acquisition?: RawAcquisition; +}>; + +export type SpikeCell = Readonly<{ + candidate: CandidateId; + state: LocalState; + screen: ScreenId; + acquisitionSamples: readonly SpikeSample[]; +}>; + +export type Revision = Readonly<{ commit: string; branch: string; dirty: boolean }>; +export type Toolchain = Readonly<{ + node: string; + pnpm: string; + xcode: string; + simctl: string; + os: string; + arch: string; +}>; +export type Target = Readonly<{ udid: string; name: string; runtime: string }>; +export type GuestMechanismEvidence = Readonly<{ + implementation: 'idb'; + release: 'v1.5.2'; + companionArchive: 'idb-companion.macos-arm64.tar.gz'; + companionSha256: string; + guestBinary: 'Resources/SimulatorFrameworkBridge'; + guestBinaryExpectedSha256?: string; + guestBinarySha256: string; + transport: string; + traversal: string; + client: 'node-direct-socket'; +}>; + +export type PreferenceEvidence = Readonly<{ + applied: boolean; + restored: boolean; + fixtureLaunchCompatible: boolean | null; + simulatorStateBefore: string; + diffs: readonly Readonly<{ + changes: readonly Readonly<{ key: string; before?: unknown; after?: unknown }>[]; + }>[]; +}>; + +export type SpikeReport = Readonly<{ + revision: Revision; + guestMechanism: GuestMechanismEvidence; + target: Target; + toolchain: Toolchain; + cells: readonly SpikeCell[]; + decisionReasons: readonly string[]; + preferenceEvidence?: PreferenceEvidence; +}>; diff --git a/vitest.config.ts b/vitest.config.ts index e9908d267..953351510 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -148,6 +148,7 @@ export default defineConfig({ // helper families are rebuilt through the shared release/size-report owner. 'scripts/__tests__/prepare-publish-assets.test.ts', 'scripts/ios-snapshot-benchmark/*.test.ts', + 'scripts/ios-ax-bridge-spike/*.test.ts', // Parses CI configuration only, so this action guard needs no device or subprocess lane. 'test/ci/upload-agent-device-artifacts.test.ts', 'test/ci/upload-artifact-hidden-paths.test.ts',