fix(ios): give simulator AX bridge captures a tree-bound keyboard fact

.changeset/ios-ax-bridge-keyboard-fact.md

@@ -0,0 +1,5 @@
+---
+'agent-device': patch
+---
+
+Fix the first read after a drag that only shows or hides the soft keyboard waiting about 3.5 s on the iOS Simulator AX bridge, the default Simulator route. A bridge capture now publishes a keyboard fact read from its own tree: the docked-tree band when the tree holds a docked keyboard, and `absent` only when the tree is complete and holds no keyboard window, `Keyboard` or `Key` node, or keyboard role. A truncated tree, or a keyboard window that exposes no keys (for example, a hardware keyboard that shows only the assistant bar), stays `unmeasurable`.

packages/contracts/src/tap-keyboard-occlusion.test.ts

@@ -9,6 +9,7 @@ import type {
 } from '@agent-device/kernel/snapshot';
 import {
   acquireProviderKeyboard,
+  observeCompleteTreeKeyboard,
   resolveKeyboardTapOcclusion,
   TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS,
   TAP_KEYBOARD_OCCLUDES_TARGET_REASON,
@@ -34,6 +35,8 @@ type FixtureCase = {
   target: { index: number } | { point: { x: number; y: number } };
   keyboard?: SnapshotKeyboardBandFact;
   observationVisibility?: boolean;
+  /** Whether a producer that reads the app's whole in-process tree captured `nodes` uncut. */
+  treeComplete?: boolean;
   expected: { kind: string; frame?: Rect; controlRects?: readonly Rect[] };
 };
 
@@ -84,6 +87,17 @@ test('keyboard tap occlusion agrees with every golden parity table case', () =>
         `${fixture.name}: acquisition observation`,
       );
     }
+    if (fixture.treeComplete !== undefined) {
+      assert.deepEqual(
+        observeCompleteTreeKeyboard({
+          nodes,
+          viewport: fixture.viewport,
+          complete: fixture.treeComplete,
+        }),
+        fixture.keyboard,
+        `${fixture.name}: complete-tree observation`,
+      );
+    }
     const occlusion = resolveKeyboardTapOcclusion({
       nodes,
       viewport: fixture.viewport,

packages/contracts/src/tap-keyboard-occlusion.ts

@@ -66,6 +66,8 @@ export const TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS = Object.freeze({
 });
 
 const IOS_KEYBOARD_TYPE_NAMES: ReadonlySet<string> = new Set(['keyboard', 'key']);
+/** The system window that hosts the remote keyboard, with or without keys in it. */
+const IOS_KEYBOARD_WINDOW_ROLE = 'UIRemoteKeyboardWindow';
 
 /** The occluding band of the visible keyboard, plus the rects that prove a tap meant the keyboard. */
 export type KeyboardSurface = {
@@ -356,6 +358,31 @@ export function acquireProviderKeyboard(params: ProviderKeyboardInput): {
   };
 }
 
+/**
+ * The keyboard fact of a producer that reads the target app's whole in-process tree in one capture,
+ * as the Simulator AX bridge does: the keyboard window is one of the app's own windows there, so a
+ * complete tree with no keyboard window and no keyboard node proves absence the way a full XCTest scan
+ * does. A truncated tree, or a keyboard window that exposes no keys, proves only presence.
+ */
+export function observeCompleteTreeKeyboard(
+  params: Readonly<{ nodes: readonly RawSnapshotNode[]; viewport: Rect | null; complete: boolean }>,
+): SnapshotKeyboardBandFact {
+  const fact = observeProviderKeyboard(params);
+  if (fact.kind !== 'unmeasurable' || !params.complete) return fact;
+  return params.nodes.some(isKeyboardTraceNode) ? fact : { kind: 'absent', treeBound: true };
+}
+
+/** Any node the keyboard leaves in the tree, including its window and the in-app placeholder. */
+function isKeyboardTraceNode(node: RawSnapshotNode): boolean {
+  return (
+    node.role === IOS_KEYBOARD_WINDOW_ROLE ||
+    isKeyboardSurfaceNode(node) ||
+    hasIosKeyboardRole(node, IOS_KEYBOARD_SETTLE_CHROME_ROLES) ||
+    hasIosKeyboardRole(node, IOS_KEYBOARD_SUBTREE_CHROME_ROLES) ||
+    hasIosKeyboardRole(node, IOS_KEYBOARD_CHROME_MARKER_ROLES)
+  );
+}
+
 function observeProviderKeyboard(params: ProviderKeyboardInput): SnapshotKeyboardBandFact {
   const frame = deriveDockedKeyboardFrame(params.nodes, params.viewport);
   // Derived from the nodes this capture returns, so a stale tree derives the stale tree's band.

packages/daemon/src/core/__tests__/post-gesture-stabilization.test.ts

@@ -1,5 +1,8 @@
 import type { CommandFlags } from '@agent-device/contracts/command';
-import { acquireProviderKeyboard } from '@agent-device/contracts/tap-keyboard-occlusion';
+import {
+  acquireProviderKeyboard,
+  observeCompleteTreeKeyboard,
+} from '@agent-device/contracts/tap-keyboard-occlusion';
 import assert from 'node:assert/strict';
 import { afterEach, test, vi } from 'vitest';
 import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures';
@@ -608,6 +611,84 @@ test('capturePostGestureStabilizedResult still distrusts a stale WebDriver sourc
   assert.equal(settled, 0);
 });
 
+/** A Simulator AX bridge capture: one complete read of the app's tree, keyboard window included. */
+function bridgeCapture(keyboard: boolean) {
+  const source = makeSnapshotState([
+    ...pickupSnapshot(500).nodes,
+    ...(keyboard ? keyboardWindowNodes() : []),
+  ]);
+  return {
+    ...source,
+    keyboard: observeCompleteTreeKeyboard({
+      nodes: source.nodes,
+      viewport: { x: 0, y: 0, width: 390, height: 844 },
+      complete: true,
+    }),
+  };
+}
+
+async function stabilizeBridgeCaptures(params: { before: boolean; after: boolean }) {
+  vi.useFakeTimers();
+  const session = makeSession('ios');
+  session.snapshot = bridgeCapture(params.before);
+  markPostGestureStabilization(session, 'swipe');
+  const baselineKeyboard = session.snapshot.keyboard;
+  const capture = vi.fn(async () => bridgeCapture(params.after));
+
+  const resultPromise = withDiagnosticsScope({}, async () => {
+    const result = await capturePostGestureStabilizedResult({
+      session,
+      capture,
+      readSnapshot: (snapshot) => snapshot,
+    });
+    return {
+      result,
+      staleAccepts: countDiagnosticEventsByPhase(['post_gesture_snapshot_stale_accept']),
+      settled: countDiagnosticEventsByPhase(['post_gesture_snapshot_stabilized']),
+    };
+  });
+
+  await vi.advanceTimersByTimeAsync(10_000);
+  const { result, staleAccepts, settled } = await resultPromise;
+  return {
+    staleAccepts,
+    settled,
+    baselineKeyboard,
+    quietKeyboard: result.value.keyboard,
+    postGestureOutcome: result.postGestureOutcome,
+    captures: capture.mock.calls.length,
+  };
+}
+
+test('capturePostGestureStabilizedResult settles when a complete bridge tree loses the keyboard it derived (iOS)', async () => {
+  const outcome = await stabilizeBridgeCaptures({ before: true, after: false });
+
+  assert.deepEqual(outcome.baselineKeyboard, {
+    kind: 'derived',
+    derivation: 'docked-tree',
+    frame: { x: 0, y: 500, width: 390, height: 344 },
+    treeBound: true,
+  });
+  assert.deepEqual(outcome.quietKeyboard, { kind: 'absent', treeBound: true });
+  assert.equal(outcome.settled, 1);
+  assert.equal(outcome.staleAccepts, 0);
+  assert.equal(outcome.postGestureOutcome, undefined);
+  assert.equal(outcome.captures, 2);
+});
+
+test('capturePostGestureStabilizedResult still distrusts bridge trees whose keyboard fact did not change (iOS)', async () => {
+  // The same fact on both sides proves nothing, so the content comparison decides, and it never moved.
+  for (const keyboard of [false, true]) {
+    const outcome = await stabilizeBridgeCaptures({ before: keyboard, after: keyboard });
+
+    assert.equal(outcome.quietKeyboard?.kind, keyboard ? 'derived' : 'absent');
+    assert.deepEqual(outcome.quietKeyboard, outcome.baselineKeyboard);
+    assert.equal(outcome.staleAccepts, 1, `keyboard ${keyboard}`);
+    assert.equal(outcome.settled, 0, `keyboard ${keyboard}`);
+    vi.useRealTimers();
+  }
+});
+
 // --- #1569: a backend swap mid-poll is not comparable evidence ---
 
 test('capturePostGestureStabilizedResult re-baselines instead of concluding when the backend changes (iOS)', async () => {

packages/platform-apple/src/snapshot-source/adapter-keyboard.fixtures.ts

@@ -0,0 +1,110 @@
+/**
+ * Guest trees for the Simulator AX source's keyboard fact: each case names the windows the fake
+ * guest reports under its root, whether it cut the tree, and the fact the capture must publish.
+ */
+
+/** A guest node; `automationType` 19 is `Keyboard` and 20 is `Key`. */
+function guestNode(
+  elementType: string,
+  frame: Readonly<{ X: number; Y: number; Width: number; Height: number }>,
+  children: readonly Record<string, unknown>[] = [],
+  automationType?: number,
+): Record<string, unknown> {
+  return {
+    XC_kAXXCAttributeElementType: elementType,
+    ...(automationType === undefined ? {} : { XC_kAXXCAttributeAutomationType: automationType }),
+    XC_kAXXCAttributeFrame: frame,
+    XC_kAXXCAttributeChildren: children,
+  };
+}
+
+const APP_FRAME = { X: 0, Y: 0, Width: 390, Height: 844 };
+const ASSISTANT_FRAME = { X: 0, Y: 509, Width: 390, Height: 44 };
+// The docked key plane stops 58 pt above the viewport bottom, as on iPhone 17 Pro.
+const DOCKED_KEYBOARD_WINDOW = guestNode('UIRemoteKeyboardWindow', APP_FRAME, [
+  guestNode('TUISystemInputAssistantView', ASSISTANT_FRAME),
+  guestNode(
+    'UIKeyboardAutomatic',
+    { X: 0, Y: 553, Width: 390, Height: 233 },
+    [
+      guestNode('UIKeyboardLayoutStar', { X: 0, Y: 553, Width: 390, Height: 233 }, [
+        guestNode('UIAccessibilityElementKBKey', { X: 0, Y: 560, Width: 195, Height: 54 }, [], 20),
+        guestNode(
+          'UIAccessibilityElementKBKey',
+          { X: 195, Y: 560, Width: 195, Height: 54 },
+          [],
+          20,
+        ),
+      ]),
+    ],
+    19,
+  ),
+]);
+
+const notObserved = { kind: 'unmeasurable', reason: 'keyboard-not-observed' };
+
+export const ADAPTER_KEYBOARD_CASES: readonly Readonly<{
+  name: string;
+  windows: readonly Record<string, unknown>[];
+  truncated: boolean;
+  keyboard: Record<string, unknown>;
+}>[] = [
+  {
+    name: 'a docked keyboard',
+    windows: [DOCKED_KEYBOARD_WINDOW],
+    truncated: false,
+    keyboard: {
+      kind: 'derived',
+      derivation: 'docked-tree',
+      frame: { x: 0, y: 553, width: 390, height: 291 },
+      treeBound: true,
+    },
+  },
+  {
+    name: 'a complete tree',
+    windows: [],
+    truncated: false,
+    keyboard: { kind: 'absent', treeBound: true },
+  },
+  // A cut can drop the keyboard window, so a truncated tree proves only presence.
+  { name: 'a truncated tree', windows: [], truncated: true, keyboard: notObserved },
+  // The in-app placeholder can outlive the key plane during a dismissal.
+  {
+    name: 'a placeholder without keys',
+    windows: [
+      guestNode('UITextEffectsWindow', APP_FRAME, [
+        guestNode('_UIRemoteKeyboardPlaceholderView', { X: 0, Y: 509, Width: 390, Height: 335 }),
+      ]),
+    ],
+    truncated: false,
+    keyboard: notObserved,
+  },
+  // A keyboard window that exposes no keys is not proof that the keyboard is gone.
+  {
+    name: 'a keyboard window without keys',
+    windows: [guestNode('UIRemoteKeyboardWindow', APP_FRAME)],
+    truncated: false,
+    keyboard: notObserved,
+  },
+  {
+    name: 'a keyboard window that holds only a placeholder',
+    windows: [
+      guestNode('UIRemoteKeyboardWindow', APP_FRAME, [
+        guestNode('_UIRemoteKeyboardPlaceholderView', { X: 0, Y: 509, Width: 390, Height: 335 }),
+      ]),
+    ],
+    truncated: false,
+    keyboard: notObserved,
+  },
+  // A minimized or hardware keyboard leaves only its assistant bar.
+  {
+    name: 'an assistant bar without keys',
+    windows: [
+      guestNode('UIRemoteKeyboardWindow', APP_FRAME, [
+        guestNode('TUISystemInputAssistantView', ASSISTANT_FRAME),
+      ]),
+    ],
+    truncated: false,
+    keyboard: notObserved,
+  },
+];

packages/platform-apple/src/snapshot-source/adapter.test.ts

@@ -16,6 +16,7 @@ import { readSnapshotTargetProcessStartTime } from '../snapshot-process.ts';
 import { DEPTH_HINT_PROBE_BACK_AFTER_USES } from './depth-hints.ts';
 import { SNAPSHOT_BRIDGE_SOURCE_FILENAMES } from './cache-identity.ts';
 import { MAXIMUM_REQUEST_LEVELS } from './limits.ts';
+import { ADAPTER_KEYBOARD_CASES } from './adapter-keyboard.fixtures.ts';
 import {
   encodeSnapshotBridgeFrame,
   SNAPSHOT_SOURCE_PROTOCOL_VERSION,
@@ -554,6 +555,8 @@ type AdapterFixture = {
   runningSurfaceHost: boolean;
   /** The surface host check each request carried: its executables and Simulator, or undefined. */
   surfaceHostChecks: Array<Readonly<{ executables: unknown; simulatorUdid: unknown }> | undefined>;
+  /** The windows the fake guest reports under its root when no other tree shape is selected. */
+  windows: readonly Record<string, unknown>[];
   omitRecovery: boolean;
   diagnostics: Record<string, unknown>[];
   startedTargets: Array<Parameters<SnapshotSourceHost['start']>[0]>;
@@ -591,6 +594,7 @@ function createAdapterHost(buildDelayMs = 0): AdapterFixture {
     omitViewport: false,
     runningSurfaceHost: false,
     surfaceHostChecks: [],
+    windows: [],
     omitRecovery: false,
     diagnostics: [],
     startedTargets: [],
@@ -813,7 +817,7 @@ class AdapterSocket extends EventEmitter implements SnapshotSourceSocket {
                                 ],
                               },
                             ]
-                          : [],
+                          : this.fixture.windows,
                 },
           },
           {
@@ -873,3 +877,43 @@ test('the Simulator AX source refuses a window whose coordinate space it cannot
     await rm(root, { recursive: true, force: true });
   }
 });
+
+test('the Simulator AX source publishes absence only from a complete tree with no keyboard trace', async () => {
+  const root = await mkdtempForTest('agent-device-snapshot-adapter-keyboard-');
+  const sourceRoot = path.join(root, 'source');
+  await (await import('@agent-device/host-kit/host-file')).ensureHostDirectory(sourceRoot);
+  for (const name of SNAPSHOT_BRIDGE_SOURCE_FILENAMES) {
+    await writeFile(path.join(sourceRoot, name), 'native source');
+  }
+  const fixture = createAdapterHost();
+  const source = createSimulatorSnapshotSource({
+    host: fixture.host,
+    sourceRoot,
+    cacheRoot: path.join(root, 'cache'),
+  });
+  const target = { ...targetForTest(), generation: 'generation-1', targetId: 'target-1' };
+
+  try {
+    for (const entry of ADAPTER_KEYBOARD_CASES) {
+      fixture.windows = entry.windows;
+      fixture.truncated = entry.truncated;
+      for (const acquisitionIntent of ['full', 'surface-observation'] as const) {
+        const outcome = await source.acquire({
+          target,
+          hint: deriveIosCaptureHint(createIosSnapshotRequest({ acquisitionIntent })),
+        });
+        assert.equal(outcome.stage, 'acquired', `${entry.name} (${acquisitionIntent})`);
+        if (outcome.stage !== 'acquired') continue;
+        assert.equal(outcome.acquisition.truncated, entry.truncated);
+        assert.deepEqual(
+          outcome.acquisition.keyboard,
+          entry.keyboard,
+          `${entry.name} (${acquisitionIntent})`,
+        );
+      }
+    }
+  } finally {
+    await source.close();
+    await rm(root, { recursive: true, force: true });
+  }
+});

packages/platform-apple/src/snapshot-source/adapter.ts

@@ -1,4 +1,5 @@
 import { AppError } from '@agent-device/kernel/errors';
+import { observeCompleteTreeKeyboard } from '@agent-device/contracts/tap-keyboard-occlusion';
 import type {
   CaptureHint,
   IosSnapshotAcquisition,
@@ -246,6 +247,11 @@ function createAcquisition(
     nodes,
     truncated,
     viewport: decoded.viewport,
+    keyboard: observeCompleteTreeKeyboard({
+      nodes,
+      viewport: decoded.viewport.kind === 'reported' ? decoded.viewport.rect : null,
+      complete: !truncated,
+    }),
     lineage,
     residue,
   };

website/docs/docs/commands.md

@@ -607,7 +607,7 @@ agent-device is text 'id="greeting"' "Welcome back"
 - On the web, on-screen text the browser cannot hand an action to (static text, a `status` announcement) is in the tree and readable by `wait text`, `find text`, `get text`, and `text=`, but carries no `@ref` and has no rect: `find ... list` prints it as `(no ref)`, and it is never a tap target. Such text is published on `value`, so `value="X"` can match page text as well as a field holding that text, and a predicate that needs one element then reports `AMBIGUOUS_MATCH`; use `text=` for page text, or add `editable=true` to name the field.
 - The first read after `scroll`, `swipe`, or `gesture swipe` waits for two captures to agree. When that proves something about the gesture, `is`, `get`, `find`, `wait`, interactions that captured it, and `snapshot` add a warning, and all but `snapshot` report `postGestureOutcome` (`{ "kind", "gesture": { "action", "positionals" } }`) in `data` or `error.details`:
   - `kind: "unsettled"`: the screen was still changing. A miss is not proof of absence; read again. `is absent` refuses with `observation: "unsettled"`, and `wait absent` keeps polling.
-  - `kind: "no-effect"`: nothing changed. The list may be at its edge or ignore synthesized scrolls; a raw `swipe x1 y1 x2 y2` inside it moves such lists. The soft keyboard appearing or going away counts as a change only when the pre-gesture baseline and a later quiet capture both read the keyboard from their own tree and disagree: the iOS XCTest runner does, a WebDriver (Appium) visibility probe does not. Otherwise only the content is compared.
+  - `kind: "no-effect"`: nothing changed. The list may be at its edge or ignore synthesized scrolls; a raw `swipe x1 y1 x2 y2` inside it moves such lists. The soft keyboard appearing or going away counts as a change only when the pre-gesture baseline and a later quiet capture both read the keyboard from their own tree and disagree: the iOS XCTest runner and the iOS Simulator AX bridge (the default Simulator route) do, a WebDriver (Appium) visibility probe does not. The AX bridge reads the keyboard as gone only from a complete tree that holds no keyboard window. Otherwise only the content is compared.
 
 ## Replay