diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 661550362f..c3b3ec4b05 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -27,7 +27,7 @@ extension RunnerTests { case .synthesisUnavailable: return "Show the software keyboard, then retry type or fill." case .commitNotObserved: - return "The field may hold none, part, or all of the text. Read it back before retrying, and prefer fill, which replaces the whole value, over type, which appends to whatever committed." + return "The field may hold none, part, or all of the text. Run snapshot -i and inspect the field: if it already matches, continue; otherwise retry fill with the full text quoted and --delay-ms 80. Do not use type, which appends to whatever committed." } } } diff --git a/src/__tests__/cli-grammar.test.ts b/src/__tests__/cli-grammar.test.ts index 4f88198863..221ee66bec 100644 --- a/src/__tests__/cli-grammar.test.ts +++ b/src/__tests__/cli-grammar.test.ts @@ -36,9 +36,9 @@ test('interaction and fill grammar share ref, selector, and point parsing', () = assert.deepEqual(selectorFill.target, { kind: 'selector', selector: 'id=email' }); assert.equal(selectorFill.text, 'qa@example.com'); - const refFill = readInputFromCli('fill', ['@e4', 'Email', 'qa@example.com'], BASE_FLAGS); - assert.deepEqual(refFill.target, { kind: 'ref', ref: '@e4', label: 'Email' }); - assert.equal(refFill.text, 'qa@example.com'); + const refFill = readInputFromCli('fill', ['@e4', 'good', 'morning'], BASE_FLAGS); + assert.deepEqual(refFill.target, { kind: 'ref', ref: '@e4' }); + assert.equal(refFill.text, 'good morning'); const pointFill = readInputFromCli('fill', ['10', '20', 'hello'], BASE_FLAGS); assert.deepEqual(pointFill.target, { kind: 'point', x: 10, y: 20 }); diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index a3f7ca67aa..2367de6690 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -249,7 +249,7 @@ const fillCommandFacet = defineCommandFacet({ text: { summary: 'Replace text in a UI input', cliDetail: - 'Clear a field with an empty text argument: fill @e57 "" (the argument must be present — fill @e57 alone is a missing argument, not a clear). When visible label text also matches a non-input element, constrain the target with editable=true, for example fill \'label="Email" editable=true\' "qa@example.com".', + 'Every positional after an @ref is the replacement text, so fill @e57 good morning enters "good morning"; quote the text when the shell must preserve exact whitespace. Clear a field with an empty text argument: fill @e57 "" (the argument must be present — fill @e57 alone is a missing argument, not a clear). When visible label text also matches a non-input element, constrain the target with editable=true, for example fill \'label="Email" editable=true\' "qa@example.com".', }, metadata: metadata('fill'), definition: fillCommandDefinition, diff --git a/src/core/interaction-positionals.test.ts b/src/core/interaction-positionals.test.ts index 34377d1d99..56c0cc06dc 100644 --- a/src/core/interaction-positionals.test.ts +++ b/src/core/interaction-positionals.test.ts @@ -99,3 +99,11 @@ test('readFillTargetFromPositionals still parses "x y" points and selectors', () text: 'qa@example.com', }); }); + +test('readFillTargetFromPositionals treats every token after a ref as text', () => { + assert.deepEqual(readFillTargetFromPositionals(['@e29~s317228', 'good', 'morning']), { + kind: 'ref', + target: { ref: '@e29~s317228' }, + text: 'good morning', + }); +}); diff --git a/src/core/interaction-positionals.ts b/src/core/interaction-positionals.ts index def7139faa..517f57ab9d 100644 --- a/src/core/interaction-positionals.ts +++ b/src/core/interaction-positionals.ts @@ -23,7 +23,7 @@ type PositionalInteractionTarget = * erase the field instead of being refused. */ export type DecodedFillTarget = - | { kind: 'ref'; target: { ref: string; label?: string }; text: string | undefined } + | { kind: 'ref'; target: { ref: string }; text: string | undefined } | { kind: 'selector'; target: { selector: string }; text: string | undefined } | { kind: 'point'; target: { x: number; y: number }; text: string | undefined }; @@ -58,14 +58,11 @@ export function readInteractionTargetFromPositionals( export function readFillTargetFromPositionals(positionals: string[]): DecodedFillTarget { const firstPositional = positionals[0]; if (firstPositional?.startsWith('@')) { - const textPositionals = positionals.length >= 3 ? positionals.slice(2) : positionals.slice(1); + const textPositionals = positionals.slice(1); const text = textPositionals.length === 0 ? undefined : textPositionals.join(' '); return { kind: 'ref', - target: { - ref: firstPositional, - label: positionals.length >= 3 ? optionalTrimmedText(positionals.slice(1, 2)) : undefined, - }, + target: { ref: firstPositional }, text, }; } diff --git a/src/daemon/handlers/__tests__/interaction-ref-policy.test.ts b/src/daemon/handlers/__tests__/interaction-ref-policy.test.ts new file mode 100644 index 0000000000..d088f873bb --- /dev/null +++ b/src/daemon/handlers/__tests__/interaction-ref-policy.test.ts @@ -0,0 +1,47 @@ +import { expect, test } from 'vitest'; +import { activatePartialRefFrame } from '../../ref-frame.ts'; +import { refMutationAdmissionResponse } from '../interaction-ref-policy.ts'; +import { makeStaleRefSession } from './interaction-touch-fixtures.ts'; + +test('a plain ref emitted by the current partial frame suggests its exact pinned form', () => { + const session = makeStaleRefSession('partial-frame-suggestion'); + session.snapshotGeneration = 531_735; + activatePartialRefFrame(session, new Set(['e19', 'e20'])); + + const response = refMutationAdmissionResponse({ + session, + ref: '@e19', + mintedGeneration: undefined, + staleRefsWarning: undefined, + }); + + expect(response?.ok).toBe(false); + if (response && !response.ok) { + expect(response.error.details).toMatchObject({ + reason: 'plain_ref_requires_complete_frame', + suggestedRef: '@e19~s531735', + hint: 'Retry with the exact emitted ref @e19~s531735.', + }); + } +}); + +test('a plain ref outside the emitted partial scope keeps the recapture hint', () => { + const session = makeStaleRefSession('partial-frame-no-suggestion'); + session.snapshotGeneration = 531_735; + activatePartialRefFrame(session, new Set(['e20'])); + + const response = refMutationAdmissionResponse({ + session, + ref: '@e19', + mintedGeneration: undefined, + staleRefsWarning: undefined, + }); + + expect(response?.ok).toBe(false); + if (response && !response.ok) { + expect(response.error.details?.suggestedRef).toBeUndefined(); + expect(response.error.details?.hint).toBe( + 'Capture a fresh interactive snapshot (snapshot -i) or use a stable selector, then retry.', + ); + } +}); diff --git a/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts b/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts index b33a9c56b3..2e4138db01 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts +++ b/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts @@ -138,13 +138,26 @@ test('parseFillTarget splits a pinned ref and keeps the text intact', () => { target: { kind: 'ref', ref: '@e4', - fallbackLabel: '', }, refGeneration: 3, text: 'qa@example.com', }); }); +test('parseFillTarget does not reinterpret the first word of ref text as a fallback label', () => { + const parsed = parseFillTarget(['@e4~s3', 'good', 'morning']); + + expect(parsed).toEqual({ + ok: true, + target: { + kind: 'ref', + ref: '@e4', + }, + refGeneration: 3, + text: 'good morning', + }); +}); + test('parseFillTarget rejects a malformed pinned ref before reading text', () => { const parsed = parseFillTarget(['@e4~x3', 'text']); diff --git a/src/daemon/handlers/interaction-ref-policy.ts b/src/daemon/handlers/interaction-ref-policy.ts index 951552b699..466ebf94b4 100644 --- a/src/daemon/handlers/interaction-ref-policy.ts +++ b/src/daemon/handlers/interaction-ref-policy.ts @@ -60,13 +60,24 @@ export function assertRefMutationAdmitted(params: { if (admission.admitted) return; const scope = refFrameScope(params.session); + const currentGeneration = refFrameEpoch(params.session); + const suggestedRef = + admission.reason === 'plain_ref_requires_complete_frame' && + scope !== 'all' && + scope.has(refBody) && + currentGeneration !== undefined + ? `@${refBody}~s${currentGeneration}` + : undefined; throw new AppError('COMMAND_FAILED', rejectionMessage(admission.reason, params.ref), { reason: admission.reason, ref: params.ref, - currentGeneration: refFrameEpoch(params.session), + currentGeneration, scope: scope === 'all' ? 'all' : Array.from(scope), ...(params.mintedGeneration !== undefined ? { mintedGeneration: params.mintedGeneration } : {}), - hint: params.staleRefsWarning ?? REJECTION_HINT, + ...(suggestedRef ? { suggestedRef } : {}), + hint: suggestedRef + ? `Retry with the exact emitted ref ${suggestedRef}.` + : (params.staleRefsWarning ?? REJECTION_HINT), }); } diff --git a/src/daemon/handlers/interaction-touch-targets.ts b/src/daemon/handlers/interaction-touch-targets.ts index b51fcab02c..b7f7ea327a 100644 --- a/src/daemon/handlers/interaction-touch-targets.ts +++ b/src/daemon/handlers/interaction-touch-targets.ts @@ -141,7 +141,6 @@ export function parseFillTarget(positionals: string[]): ParsedFillTarget { target: { kind: 'ref', ref: versioned.ref, - fallbackLabel: readRefFallbackLabel(positionals), }, refGeneration: versioned.generation, text: decoded.text, @@ -228,10 +227,6 @@ export function formatTouchTargetLabel( return 'target'; } -function readRefFallbackLabel(positionals: string[]): string { - return positionals.length >= 3 ? positionals[1]?.trim() || '' : ''; -} - function splitTrailingDuration(positionals: string[]): { targetPositionals: string[]; duration: { durationMs: number } | Record; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 454b6f8045..fde9d6df6b 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -427,7 +427,7 @@ agent-device gesture transform 200 420 80 -40 2 35 700 # combined pan, zoom, and `type` accepts text only. Do not pass `@ref` to `type`; use `fill @ref "text"` to target a field directly, or `press @ref` then `type "text"` to append in the focused field. If `type` reports `TEXT_INPUT_NOT_FOCUSED`, focus a visible text input and retry; when accessibility does not expose the input, use a coordinate focus command before typing. On iOS, if `type` reports `TEXT_INPUT_SYNTHESIS_UNAVAILABLE` while the software keyboard is hidden, show the software keyboard, then retry `type` or `fill`. The runner reports this error instead of risking partial input through an unreliable text-entry path. -On iOS, if `type` or `fill` reports `TEXT_INPUT_COMMIT_NOT_OBSERVED`, the runner could not confirm the typed text reached the field — either it did not land before the runner's deadline, or the expected final text is identical to the field's placeholder. In the latter case, accessibility cannot distinguish committed text from an empty field rendering that placeholder, even if the field held content before dispatch. The field may hold none, part, or all of the text: read it back, then retry with `fill`, which replaces the whole value, rather than `type`, which appends to whatever committed. This covers the bare-type route and the coordinate-driven `fill` route taken when the accessibility channel is under load, both of which observe the field after synthesizing; it is not a guarantee that every text-entry route verifies its result. +On iOS, if `type` or `fill` reports `TEXT_INPUT_COMMIT_NOT_OBSERVED`, the runner could not confirm the typed text reached the field — either it did not land before the runner's deadline, or the expected final text is identical to the field's placeholder. In the latter case, accessibility cannot distinguish committed text from an empty field rendering that placeholder, even if the field held content before dispatch. The field may hold none, part, or all of the text: run `snapshot -i` and inspect it. If it already matches, continue; otherwise retry with the full text quoted and `fill --delay-ms 80`, which replaces the whole value. Do not use `type`, which appends to whatever committed. This covers the bare-type route and the coordinate-driven `fill` route taken when the accessibility channel is under load, both of which observe the field after synthesizing; it is not a guarantee that every text-entry route verifies its result. Use plain `fill` or `type` first for ordinary login and form fields. Use `--delay-ms` on `type` or `fill` only when a debounced search field or search-as-you-type input actually misses characters, or when the app must receive incremental updates. Delayed typing intentionally prefers paced character entry over clipboard-style fallbacks so the target field receives each incremental update. On Android, `fill` also verifies text and treats IME-owned capture as a terminal failure instead of retrying against the wrong field.