fix: improve ref-based interaction recovery

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift

@@ -27,7 +27,7 @@ extension RunnerTests {
       case .synthesisUnavailable:
         return "Show the software keyboard, then retry type or fill."
       case .commitNotObserved:
-        return "The field may hold none, part, or all of the text. Read it back before retrying, and prefer fill, which replaces the whole value, over type, which appends to whatever committed."
+        return "The field may hold none, part, or all of the text. Run snapshot -i and inspect the field: if it already matches, continue; otherwise retry fill with the full text quoted and --delay-ms 80. Do not use type, which appends to whatever committed."
       }
     }
   }

src/__tests__/cli-grammar.test.ts

@@ -36,9 +36,9 @@ test('interaction and fill grammar share ref, selector, and point parsing', () =
   assert.deepEqual(selectorFill.target, { kind: 'selector', selector: 'id=email' });
   assert.equal(selectorFill.text, 'qa@example.com');
 
-  const refFill = readInputFromCli('fill', ['@e4', 'Email', 'qa@example.com'], BASE_FLAGS);
-  assert.deepEqual(refFill.target, { kind: 'ref', ref: '@e4', label: 'Email' });
-  assert.equal(refFill.text, 'qa@example.com');
+  const refFill = readInputFromCli('fill', ['@e4', 'good', 'morning'], BASE_FLAGS);
+  assert.deepEqual(refFill.target, { kind: 'ref', ref: '@e4' });
+  assert.equal(refFill.text, 'good morning');
 
   const pointFill = readInputFromCli('fill', ['10', '20', 'hello'], BASE_FLAGS);
   assert.deepEqual(pointFill.target, { kind: 'point', x: 10, y: 20 });

src/commands/interaction/index.ts

@@ -249,7 +249,7 @@ const fillCommandFacet = defineCommandFacet({
   text: {
     summary: 'Replace text in a UI input',
     cliDetail:
-      'Clear a field with an empty text argument: fill @e57 "" (the argument must be present — fill @e57 alone is a missing argument, not a clear). When visible label text also matches a non-input element, constrain the target with editable=true, for example fill \'label="Email" editable=true\' "qa@example.com".',
+      'Every positional after an @ref is the replacement text, so fill @e57 good morning enters "good morning"; quote the text when the shell must preserve exact whitespace. Clear a field with an empty text argument: fill @e57 "" (the argument must be present — fill @e57 alone is a missing argument, not a clear). When visible label text also matches a non-input element, constrain the target with editable=true, for example fill \'label="Email" editable=true\' "qa@example.com".',
   },
   metadata: metadata('fill'),
   definition: fillCommandDefinition,

src/core/interaction-positionals.test.ts

@@ -99,3 +99,11 @@ test('readFillTargetFromPositionals still parses "x y" points and selectors', ()
     text: 'qa@example.com',
   });
 });
+
+test('readFillTargetFromPositionals treats every token after a ref as text', () => {
+  assert.deepEqual(readFillTargetFromPositionals(['@e29~s317228', 'good', 'morning']), {
+    kind: 'ref',
+    target: { ref: '@e29~s317228' },
+    text: 'good morning',
+  });
+});

src/core/interaction-positionals.ts

@@ -23,7 +23,7 @@ type PositionalInteractionTarget =
  * erase the field instead of being refused.
  */
 export type DecodedFillTarget =
-  | { kind: 'ref'; target: { ref: string; label?: string }; text: string | undefined }
+  | { kind: 'ref'; target: { ref: string }; text: string | undefined }
   | { kind: 'selector'; target: { selector: string }; text: string | undefined }
   | { kind: 'point'; target: { x: number; y: number }; text: string | undefined };
 
@@ -58,14 +58,11 @@ export function readInteractionTargetFromPositionals(
 export function readFillTargetFromPositionals(positionals: string[]): DecodedFillTarget {
   const firstPositional = positionals[0];
   if (firstPositional?.startsWith('@')) {
-    const textPositionals = positionals.length >= 3 ? positionals.slice(2) : positionals.slice(1);
+    const textPositionals = positionals.slice(1);
     const text = textPositionals.length === 0 ? undefined : textPositionals.join(' ');
     return {
       kind: 'ref',
-      target: {
-        ref: firstPositional,
-        label: positionals.length >= 3 ? optionalTrimmedText(positionals.slice(1, 2)) : undefined,
-      },
+      target: { ref: firstPositional },
       text,
     };
   }

src/daemon/handlers/__tests__/interaction-ref-policy.test.ts

@@ -0,0 +1,47 @@
+import { expect, test } from 'vitest';
+import { activatePartialRefFrame } from '../../ref-frame.ts';
+import { refMutationAdmissionResponse } from '../interaction-ref-policy.ts';
+import { makeStaleRefSession } from './interaction-touch-fixtures.ts';
+
+test('a plain ref emitted by the current partial frame suggests its exact pinned form', () => {
+  const session = makeStaleRefSession('partial-frame-suggestion');
+  session.snapshotGeneration = 531_735;
+  activatePartialRefFrame(session, new Set(['e19', 'e20']));
+
+  const response = refMutationAdmissionResponse({
+    session,
+    ref: '@e19',
+    mintedGeneration: undefined,
+    staleRefsWarning: undefined,
+  });
+
+  expect(response?.ok).toBe(false);
+  if (response && !response.ok) {
+    expect(response.error.details).toMatchObject({
+      reason: 'plain_ref_requires_complete_frame',
+      suggestedRef: '@e19~s531735',
+      hint: 'Retry with the exact emitted ref @e19~s531735.',
+    });
+  }
+});
+
+test('a plain ref outside the emitted partial scope keeps the recapture hint', () => {
+  const session = makeStaleRefSession('partial-frame-no-suggestion');
+  session.snapshotGeneration = 531_735;
+  activatePartialRefFrame(session, new Set(['e20']));
+
+  const response = refMutationAdmissionResponse({
+    session,
+    ref: '@e19',
+    mintedGeneration: undefined,
+    staleRefsWarning: undefined,
+  });
+
+  expect(response?.ok).toBe(false);
+  if (response && !response.ok) {
+    expect(response.error.details?.suggestedRef).toBeUndefined();
+    expect(response.error.details?.hint).toBe(
+      'Capture a fresh interactive snapshot (snapshot -i) or use a stable selector, then retry.',
+    );
+  }
+});

src/daemon/handlers/__tests__/interaction-touch-targets.test.ts

@@ -138,13 +138,26 @@ test('parseFillTarget splits a pinned ref and keeps the text intact', () => {
     target: {
       kind: 'ref',
       ref: '@e4',
-      fallbackLabel: '',
     },
     refGeneration: 3,
     text: 'qa@example.com',
   });
 });
 
+test('parseFillTarget does not reinterpret the first word of ref text as a fallback label', () => {
+  const parsed = parseFillTarget(['@e4~s3', 'good', 'morning']);
+
+  expect(parsed).toEqual({
+    ok: true,
+    target: {
+      kind: 'ref',
+      ref: '@e4',
+    },
+    refGeneration: 3,
+    text: 'good morning',
+  });
+});
+
 test('parseFillTarget rejects a malformed pinned ref before reading text', () => {
   const parsed = parseFillTarget(['@e4~x3', 'text']);
 

src/daemon/handlers/interaction-ref-policy.ts

@@ -60,13 +60,24 @@ export function assertRefMutationAdmitted(params: {
   if (admission.admitted) return;
 
   const scope = refFrameScope(params.session);
+  const currentGeneration = refFrameEpoch(params.session);
+  const suggestedRef =
+    admission.reason === 'plain_ref_requires_complete_frame' &&
+    scope !== 'all' &&
+    scope.has(refBody) &&
+    currentGeneration !== undefined
+      ? `@${refBody}~s${currentGeneration}`
+      : undefined;
   throw new AppError('COMMAND_FAILED', rejectionMessage(admission.reason, params.ref), {
     reason: admission.reason,
     ref: params.ref,
-    currentGeneration: refFrameEpoch(params.session),
+    currentGeneration,
     scope: scope === 'all' ? 'all' : Array.from(scope),
     ...(params.mintedGeneration !== undefined ? { mintedGeneration: params.mintedGeneration } : {}),
-    hint: params.staleRefsWarning ?? REJECTION_HINT,
+    ...(suggestedRef ? { suggestedRef } : {}),
+    hint: suggestedRef
+      ? `Retry with the exact emitted ref ${suggestedRef}.`
+      : (params.staleRefsWarning ?? REJECTION_HINT),
   });
 }
 

src/daemon/handlers/interaction-touch-targets.ts

@@ -141,7 +141,6 @@ export function parseFillTarget(positionals: string[]): ParsedFillTarget {
         target: {
           kind: 'ref',
           ref: versioned.ref,
-          fallbackLabel: readRefFallbackLabel(positionals),
         },
         refGeneration: versioned.generation,
         text: decoded.text,
@@ -228,10 +227,6 @@ export function formatTouchTargetLabel(
   return 'target';
 }
 
-function readRefFallbackLabel(positionals: string[]): string {
-  return positionals.length >= 3 ? positionals[1]?.trim() || '' : '';
-}
-
 function splitTrailingDuration(positionals: string[]): {
   targetPositionals: string[];
   duration: { durationMs: number } | Record<string, never>;

website/docs/docs/commands.md

@@ -427,7 +427,7 @@ agent-device gesture transform 200 420 80 -40 2 35 700 # combined pan, zoom, and
 `type` accepts text only. Do not pass `@ref` to `type`; use `fill @ref "text"` to target a field directly, or `press @ref` then `type "text"` to append in the focused field.
 If `type` reports `TEXT_INPUT_NOT_FOCUSED`, focus a visible text input and retry; when accessibility does not expose the input, use a coordinate focus command before typing.
 On iOS, if `type` reports `TEXT_INPUT_SYNTHESIS_UNAVAILABLE` while the software keyboard is hidden, show the software keyboard, then retry `type` or `fill`. The runner reports this error instead of risking partial input through an unreliable text-entry path.
-On iOS, if `type` or `fill` reports `TEXT_INPUT_COMMIT_NOT_OBSERVED`, the runner could not confirm the typed text reached the field — either it did not land before the runner's deadline, or the expected final text is identical to the field's placeholder. In the latter case, accessibility cannot distinguish committed text from an empty field rendering that placeholder, even if the field held content before dispatch. The field may hold none, part, or all of the text: read it back, then retry with `fill`, which replaces the whole value, rather than `type`, which appends to whatever committed. This covers the bare-type route and the coordinate-driven `fill` route taken when the accessibility channel is under load, both of which observe the field after synthesizing; it is not a guarantee that every text-entry route verifies its result.
+On iOS, if `type` or `fill` reports `TEXT_INPUT_COMMIT_NOT_OBSERVED`, the runner could not confirm the typed text reached the field — either it did not land before the runner's deadline, or the expected final text is identical to the field's placeholder. In the latter case, accessibility cannot distinguish committed text from an empty field rendering that placeholder, even if the field held content before dispatch. The field may hold none, part, or all of the text: run `snapshot -i` and inspect it. If it already matches, continue; otherwise retry with the full text quoted and `fill --delay-ms 80`, which replaces the whole value. Do not use `type`, which appends to whatever committed. This covers the bare-type route and the coordinate-driven `fill` route taken when the accessibility channel is under load, both of which observe the field after synthesizing; it is not a guarantee that every text-entry route verifies its result.
 Use plain `fill` or `type` first for ordinary login and form fields. Use `--delay-ms` on `type` or `fill` only when a debounced search field or search-as-you-type input actually misses characters, or when the app must receive incremental updates.
 Delayed typing intentionally prefers paced character entry over clipboard-style fallbacks so the target field receives each incremental update.
 On Android, `fill` also verifies text and treats IME-owned capture as a terminal failure instead of retrying against the wrong field.