diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index f9db080912..901d18573c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -11,6 +11,7 @@ #import "AgentDeviceRunnerApp.h" #import #import +#import "UnitTests/AgentDeviceTextEntryFields.h" // Mirrors the App Tracking Transparency prompt's real button order: the affirmative button is // last (#3627). The answer label records which button the app actually took, so a test asserts @@ -35,26 +36,6 @@ return alert; } -// A plain text field whose accessibility value is the masked rendering of its text. This is how -// a React Native `secureTextEntry` input reaches XCTest: `.textField`, not `.secureTextField`, -// with a value of bullets (tester-army/e2e#872). -@interface AgentDeviceMaskedTextField : UITextField -@end - -@implementation AgentDeviceMaskedTextField - -- (NSString *)accessibilityValue { - NSUInteger length = self.text.length; - if (length == 0) { - return [super accessibilityValue]; - } - // Half the typed length, as observed in the report: the mask hides the count as well. - NSUInteger bullets = MAX((NSUInteger)1, length / 2); - return [@"" stringByPaddingToLength:bullets withString:@"\u2022" startingAtIndex:0]; -} - -@end - @interface AgentDeviceRunnerFixtureViewController : AgentDeviceRunnerViewController @end @@ -105,39 +86,6 @@ - (NSString *)accessibilityValue { } @end -// XCTest reads every element's accessibility value each time it snapshots the app, which it does -// for every element query and every attribute read from a live element. Counting this field's -// value reads therefore counts the accessibility queries a command makes. The value is masked, so a -// fill's read-back ends at its first read instead of polling through a stability window whose -// poll count depends on how fast the host answers. -static NSUInteger AgentDeviceTextEntryValueReads = 0; - -@interface AgentDeviceValueReadCountingTextField : AgentDeviceMaskedTextField -@end - -@implementation AgentDeviceValueReadCountingTextField -- (NSString *)accessibilityValue { - AgentDeviceTextEntryValueReads += 1; - return [super accessibilityValue]; -} -@end - -// Reports the read count as its accessibility label and the counted field's text as its value, so -// reading either changes no view state and the masked text stays checkable. -@interface AgentDeviceValueReadCountLabel : UILabel -@property(nonatomic, weak) UITextField *countedField; -@end - -@implementation AgentDeviceValueReadCountLabel -- (NSString *)accessibilityLabel { - return [NSString stringWithFormat:@"%lu", (unsigned long)AgentDeviceTextEntryValueReads]; -} - -- (NSString *)accessibilityValue { - return self.countedField.text; -} -@end - // A field that reformats its own text on every edit into `## ### ## ##` digit groups, the way a // digit-grouping formatter in a Flutter or React Native input does. The accessibility value is // what the user sees — the formatted string — so it never equals the digits a `fill` sent (#2634). @@ -712,6 +660,8 @@ - (void)viewDidLoad { [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-masked-value"]; BOOL countsValueReads = [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-counts-value-reads"]; + BOOL witnessesReplacement = + [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-witness"]; UITextField *textField; if (digitCountValue) { textField = [[AgentDeviceDigitCountTextField alloc] init]; @@ -724,6 +674,8 @@ - (void)viewDidLoad { textField = [[AgentDeviceSubmitClearingTextField alloc] init]; } else if (maskedValue) { textField = [[AgentDeviceMaskedTextField alloc] init]; + } else if (witnessesReplacement) { + textField = [[AgentDeviceTextEntryWitnessField alloc] init]; } else if (countsValueReads) { textField = [[AgentDeviceValueReadCountingTextField alloc] init]; } else { @@ -761,7 +713,7 @@ - (void)viewDidLoad { [textField.widthAnchor constraintEqualToConstant:240], [textField.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight], ]]; - if (countsValueReads) { + if (countsValueReads || witnessesReplacement) { AgentDeviceValueReadCountLabel *valueReads = [[AgentDeviceValueReadCountLabel alloc] init]; valueReads.countedField = textField; valueReads.accessibilityIdentifier = @"agent-device-text-entry-value-reads"; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/UnitTests/AgentDeviceTextEntryFields.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/UnitTests/AgentDeviceTextEntryFields.h new file mode 100644 index 0000000000..3b42987af0 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/UnitTests/AgentDeviceTextEntryFields.h @@ -0,0 +1,18 @@ +#import + +#if TARGET_OS_IOS && defined(AGENT_DEVICE_RUNNER_UNIT_TESTS) +#import + +@interface AgentDeviceMaskedTextField : UITextField +@end + +@interface AgentDeviceValueReadCountingTextField : AgentDeviceMaskedTextField +@end + +@interface AgentDeviceValueReadCountLabel : UILabel +@property(nonatomic, weak) UITextField *countedField; +@end + +@interface AgentDeviceTextEntryWitnessField : UITextField +@end +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/UnitTests/AgentDeviceTextEntryFields.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/UnitTests/AgentDeviceTextEntryFields.m new file mode 100644 index 0000000000..81544e0244 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/UnitTests/AgentDeviceTextEntryFields.m @@ -0,0 +1,99 @@ +#import "AgentDeviceTextEntryFields.h" + +#if TARGET_OS_IOS && defined(AGENT_DEVICE_RUNNER_UNIT_TESTS) +@implementation AgentDeviceMaskedTextField +- (NSString *)accessibilityValue { + NSUInteger length = self.text.length; + if (length == 0) { + return [super accessibilityValue]; + } + NSUInteger bullets = MAX((NSUInteger)1, length / 2); + return [@"" stringByPaddingToLength:bullets withString:@"\u2022" startingAtIndex:0]; +} +@end + +static NSUInteger AgentDeviceTextEntryValueReads = 0; + +@implementation AgentDeviceValueReadCountingTextField +- (NSString *)accessibilityValue { + AgentDeviceTextEntryValueReads += 1; + return [super accessibilityValue]; +} +@end + +@implementation AgentDeviceValueReadCountLabel +- (NSString *)accessibilityLabel { + if ([self.countedField isKindOfClass:[AgentDeviceTextEntryWitnessField class]]) { + return self.countedField.accessibilityHint; + } + return [NSString stringWithFormat:@"%lu", (unsigned long)AgentDeviceTextEntryValueReads]; +} + +- (NSString *)accessibilityValue { + return self.countedField.text; +} + +@end + +@interface AgentDeviceTextEntryWitnessField () +@property(nonatomic, assign) NSUInteger taps; +@property(nonatomic, assign) NSUInteger selections; +@property(nonatomic, assign) NSUInteger rejectedEdits; +@property(nonatomic, assign) BOOL rejectNextSelectionDeletion; +@end + +@implementation AgentDeviceTextEntryWitnessField +- (instancetype)init { + self = [super init]; + if (self) { + self.delegate = self; + self.autocorrectionType = UITextAutocorrectionTypeNo; + self.autocapitalizationType = UITextAutocapitalizationTypeNone; + UITapGestureRecognizer *tap = [[UITapGestureRecognizer alloc] initWithTarget:self action:@selector(recordTap:)]; + tap.cancelsTouchesInView = NO; + tap.delegate = self; + [self addGestureRecognizer:tap]; + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-witness-prefill"]) { + self.text = @"seedvalue"; + } + } + return self; +} + +- (void)recordTap:(UITapGestureRecognizer *)recognizer { + (void)recognizer; + self.taps += 1; +} + +- (BOOL)gestureRecognizer:(UIGestureRecognizer *)gestureRecognizer shouldRecognizeSimultaneouslyWithGestureRecognizer:(UIGestureRecognizer *)otherGestureRecognizer { + (void)gestureRecognizer; + (void)otherGestureRecognizer; + return YES; +} + +- (void)selectAll:(id)sender { + self.selections += 1; + self.rejectNextSelectionDeletion = + [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-witness-refuse-selection"]; + [super selectAll:sender]; +} + +- (BOOL)textField:(UITextField *)textField shouldChangeCharactersInRange:(NSRange)range replacementString:(NSString *)string { + NSArray *arguments = NSProcessInfo.processInfo.arguments; + if ([arguments containsObject:@"--agent-device-text-entry-witness-refuse-edits"] || + (self.rejectNextSelectionDeletion && range.length > 1 && string.length == 0)) { + self.rejectNextSelectionDeletion = NO; + self.rejectedEdits += 1; + UITextPosition *start = textField.beginningOfDocument; + textField.selectedTextRange = [textField textRangeFromPosition:start toPosition:start]; + return NO; + } + return YES; +} + +- (NSString *)accessibilityHint { + return [NSString stringWithFormat:@"taps=%lu selections=%lu rejected=%lu", + (unsigned long)self.taps, (unsigned long)self.selections, (unsigned long)self.rejectedEdits]; +} +@end +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 4bc51d69cc..ca8ec50d42 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -161,6 +161,7 @@ extension RunnerTests { let prefersFocusedElement: Bool let contentTranslation: TextEntryContentTranslation let fromTapWitness: Bool + let reusesCurrentFocus: Bool /// The input the first resolved element was. Once bound, resolution refuses any other input. let boundIdentity: TextEntryInputIdentity? /// The text input that sat under `refreshPoint` before the focus tap, which binds the target to @@ -179,6 +180,7 @@ extension RunnerTests { prefersFocusedElement: Bool, contentTranslation: TextEntryContentTranslation, fromTapWitness: Bool = false, + reusesCurrentFocus: Bool = false, boundIdentity: TextEntryInputIdentity? = nil, inputAtRefreshPoint: TextInputAtPoint? = nil, postFocusRead: ResolvedTextEntryInput? = nil @@ -188,6 +190,7 @@ extension RunnerTests { self.prefersFocusedElement = prefersFocusedElement self.contentTranslation = contentTranslation self.fromTapWitness = fromTapWitness + self.reusesCurrentFocus = reusesCurrentFocus self.boundIdentity = boundIdentity ?? inputAtRefreshPoint?.identity self.inputAtRefreshPoint = inputAtRefreshPoint self.postFocusRead = postFocusRead @@ -206,6 +209,7 @@ extension RunnerTests { prefersFocusedElement: prefersFocusedElement, contentTranslation: contentTranslation, fromTapWitness: fromTapWitness, + reusesCurrentFocus: reusesCurrentFocus, boundIdentity: boundIdentity ?? input.identity, inputAtRefreshPoint: inputAtRefreshPoint ) @@ -411,11 +415,13 @@ extension RunnerTests { if existing?.isEmpty == true { return } - // Read before the caret tap, which dispatches to the app. + let defersCaretTap = reader.target.reusesCurrentFocus var valueLength = normalizedElementText(input.attributes.value).count // The tap also focuses the field, which a clear can reach with focus unconfirmed. #if !os(tvOS) - try reader.dispatch { try moveCaretToEnd(app: app, input: input) } + if !defersCaretTap { + try reader.dispatch { try moveCaretToEnd(app: app, input: input) } + } #endif // An input that stops answering once focused (#3060) cannot be read back, so it gets one burst. guard try reader.input() != nil else { @@ -456,6 +462,19 @@ extension RunnerTests { valueLength = remaining.count } #endif + if defersCaretTap { + guard let fallbackInput = try reader.input() else { + return + } +#if !os(tvOS) + try reader.dispatch { try moveCaretToEnd(app: app, input: fallbackInput) } +#endif + guard try reader.input() != nil else { + let burst = Self.textClearBurst(valueLength: valueLength) + reader.dispatch { fallbackInput.element.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: burst)) } + return + } + } for _ in 0...(valueLength / Self.maxTextClearBurst + 1) { guard let input = try reader.input() else { return diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index fca641cc12..c7b1d777db 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -104,7 +104,7 @@ extension RunnerTests { } @MainActor - func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) throws -> TextEntryTarget { + func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?, reusingFocusedInput: Bool = false) throws -> TextEntryTarget { let contentTranslation = liveContentTranslation(app: app) guard let x, let y else { let softwareKeyboardVisible = isKeyboardVisible(app: app) @@ -135,8 +135,23 @@ extension RunnerTests { return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true, contentTranslation: contentTranslation) } - let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) let input = try textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first +#if os(iOS) + if reusingFocusedInput, let input, let identity = input.identity, identity.isDistinguishable, + let snapshot = input.snapshot, snapshotHasKeyboardFocus(snapshot) { + NSLog("%@current-input-reports-keyboard-focus polls=0", TextEntryFocusEvidence.logMarker) + return TextEntryTarget( + element: input.element, + refreshPoint: textEntryRefreshPoint(frame: input.frameOnScreen, fallbackTo: CGPoint(x: x, y: y)), + prefersFocusedElement: false, + contentTranslation: contentTranslation, + reusesCurrentFocus: true, + boundIdentity: identity, + postFocusRead: input + ) + } +#endif + let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) let target = input?.element let requestedPoint = CGPoint(x: x, y: y) // `tapAt` takes a screen point: with no screen box, tap the caller's own point (#3899, #3060). diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index 7299948cef..1b347d101b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -65,7 +65,7 @@ extension RunnerTests { if let resolvedCoordinateTarget { target = resolvedCoordinateTarget } else { - target = try focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) + target = try focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y, reusingFocusedInput: textEntryMode == .replacement) } NSLog( "AGENT_DEVICE_RUNNER_TEXT_ENTRY_PHASE commandId=%@ phase=focus durationMs=%.1f chars=%d mode=%@", diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift new file mode 100644 index 0000000000..9a5e630e9d --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift @@ -0,0 +1,201 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + @MainActor + func testFocusedReplacementAndClearDispatchNoFocusOrCaretTap() throws { + for (prefilled, text) in [(false, "hello"), (true, "hello"), (true, "")] { + let field = try launchTextEntryWitness(prefilled: prefilled) + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + field.tap() + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: field)) + XCTAssertEqual(try textEntryWitnessCounts().taps, 1) + + let counts = try assertWitnessEntry(field, text: text) + XCTAssertEqual(counts.taps, 1) + XCTAssertEqual(counts.selections, prefilled ? 1 : 0) + XCTAssertEqual(counts.rejected, 0) + } + } + + @MainActor + func testReplacementReadsFocusAgainAfterAnotherInputTakesIt() throws { + let field = try launchTextEntryWitness(extra: ["--agent-device-text-entry-moves-on-focus"]) + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + field.tap() + XCTAssertTrue(try executeWitnessEntry(field, text: "first").ok) + let neighbour = app.textFields["agent-device-text-entry-neighbour"] + neighbour.tap() + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: neighbour)) + XCTAssertFalse(keyboardFocusConfirmed(app: app, element: field)) + + let response = try executeWitnessEntry(field, text: "second") + + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(app.staticTexts["agent-device-text-entry-value-reads"].value as? String, "second") + XCTAssertEqual(neighbour.value as? String, "neighbour") + XCTAssertEqual(try textEntryWitnessCounts().taps, 3) + } + + @MainActor + func testFocusedUnidentifiedReplacementKeepsFocusAndCaretPreparation() throws { + let field = try launchTextEntryWitness(prefilled: true, extra: ["--agent-device-text-entry-unnamed-input"]) + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + field.tap() + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: field)) + + let counts = try assertWitnessEntry(field, text: "hello") + + XCTAssertEqual(counts.taps, 3) + } + + @MainActor + func testNewlyFocusedReplacementKeepsCaretPreparation() throws { + let field = try launchTextEntryWitness(prefilled: true) + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + XCTAssertFalse(keyboardFocusConfirmed(app: app, element: field)) + + let counts = try assertWitnessEntry(field, text: "hello") + XCTAssertEqual(counts.taps, 2) + } + + @MainActor + func testFocusedAppendKeepsItsFocusTap() throws { + let field = try launchTextEntryWitness() + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + field.tap() + + let response = try executeWitnessEntry(field, text: "hello", mode: "append") + + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(app.staticTexts["agent-device-text-entry-value-reads"].value as? String, "hello") + XCTAssertEqual(try textEntryWitnessCounts().taps, 2) + XCTAssertEqual(try textEntryWitnessCounts().selections, 0) + } + + @MainActor + func testFocusedClearMovesCaretForRejectedSelectAllDeletion() throws { + for text in ["", "hello"] { + let field = try launchTextEntryWitness(prefilled: true, extra: ["--agent-device-text-entry-witness-refuse-selection"]) + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + field.tap() + + let counts = try assertWitnessEntry(field, text: text) + XCTAssertEqual(counts.taps, 2) + XCTAssertGreaterThan(counts.rejected, 0) + } + } + + @MainActor + func testFocusedClearDoesNotContinueIntoAReplacementInput() throws { + let field = try launchTextEntryWitness(prefilled: true, extra: ["--agent-device-text-entry-populated-successor"]) + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + field.tap() + let failures = currentXCTestFailureCount() + + let response = try executeWitnessEntry(field, text: "") + + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertFalse(didRecordXCTestFailure(since: failures)) + XCTAssertEqual(app.textFields["agent-device-auto-submit-next-screen-input"].value as? String, "next screen value") + XCTAssertFalse(field.exists) + } + + @MainActor + func testFocusedClearReportsRejectedEditsInsteadOfSuccess() throws { + let field = try launchTextEntryWitness(prefilled: true, extra: ["--agent-device-text-entry-witness-refuse-edits"]) + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + field.tap() + + let response = try executeWitnessEntry(field, text: "") + + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "TEXT_ENTRY_MISMATCH") + XCTAssertEqual(app.staticTexts["agent-device-text-entry-value-reads"].value as? String, "seedvalue") + XCTAssertGreaterThan(try textEntryWitnessCounts().rejected, 0) + } + + @MainActor + func testLandscapeReplacementDispatchPreservesFocusedAndPenalizedRoutes() throws { + for orientation in [UIDeviceOrientation.landscapeLeft, .landscapeRight] { + let field = try launchTextEntryWitness(prefilled: true) + defer { + XCUIDevice.shared.orientation = .portrait + tearDownSynthesizedReplacementField() + } + XCUIDevice.shared.orientation = orientation + let expected = orientation == .landscapeLeft + ? RunnerInterfaceOrientation.landscapeRight : RunnerInterfaceOrientation.landscapeLeft + let landscape = XCTNSPredicateExpectation( + predicate: NSPredicate { _, _ in + Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: self.app)) == expected + }, object: app + ) + XCTAssertEqual(XCTWaiter.wait(for: [landscape], timeout: 5), .completed) + field.tap() + let initialTaps = try textEntryWitnessCounts().taps + + let focused = try executeWitnessEntry(field, text: "hello", prepared: true) + XCTAssertTrue(focused.ok, String(describing: focused.error)) + XCTAssertEqual(try textEntryWitnessCounts().taps, initialTaps) + XCTAssertEqual(app.staticTexts["agent-device-text-entry-value-reads"].value as? String, "hello") + + penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") + let penalized = try executeWitnessEntry(field, text: "fresh", prepared: true) + XCTAssertTrue(penalized.ok, String(describing: penalized.error)) + XCTAssertEqual(penalized.data?.textEntryRoute, "synthesized-first-responder-replacement") + XCTAssertGreaterThan(try textEntryWitnessCounts().taps, initialTaps) + XCTAssertEqual(app.staticTexts["agent-device-text-entry-value-reads"].value as? String, "fresh") + let cleared = try executeWitnessEntry(field, text: "", prepared: true) + XCTAssertTrue(cleared.ok, String(describing: cleared.error)) + XCTAssertEqual(app.staticTexts["agent-device-text-entry-value-reads"].value as? String, "") + } + } + + @MainActor + private func launchTextEntryWitness(prefilled: Bool = false, extra: [String] = []) throws -> XCUIElement { + app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-witness"] + + (prefilled ? ["--agent-device-text-entry-witness-prefill"] : []) + extra + app.launch() + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) + let field = extra.contains("--agent-device-text-entry-unnamed-input") + ? app.textFields.firstMatch : app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) + return field + } + + @MainActor + private func executeWitnessEntry(_ field: XCUIElement, text: String, mode: String = "replace", prepared: Bool = false) throws -> Response { + let frame = field.frame + let command = try runnerCommandFixture( + #"{"command":"type","text":"\#(text)","textEntryMode":"\#(mode)","x":\#(frame.midX),"y":\#(frame.midY)}"# + ) + if prepared { + return try executeOnMainPrepared(command: command, activeApp: app) + } + return try executeTypeCommand(activeApp: app, command: command) + } + + @MainActor + private func assertWitnessEntry(_ field: XCUIElement, text: String) throws -> (taps: Int, selections: Int, rejected: Int) { + let response = try executeWitnessEntry(field, text: text) + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(app.staticTexts["agent-device-text-entry-value-reads"].value as? String, text) + return try textEntryWitnessCounts() + } + + @MainActor + private func textEntryWitnessCounts() throws -> (taps: Int, selections: Int, rejected: Int) { + let hint = app.staticTexts["agent-device-text-entry-value-reads"].label + let counts = try hint.split(separator: " ").map { field in + try XCTUnwrap(Int(try XCTUnwrap(field.split(separator: "=").last).description)) + } + guard counts.count == 3 else { + XCTFail("Missing app-owned text entry counters: \(hint)") + return (0, 0, 0) + } + return (counts[0], counts[1], counts[2]) + } +#endif +} diff --git a/src/commands/schema/cli-help-workflows.ts b/src/commands/schema/cli-help-workflows.ts index 3b85574470..469eceea12 100644 --- a/src/commands/schema/cli-help-workflows.ts +++ b/src/commands/schema/cli-help-workflows.ts @@ -183,6 +183,7 @@ Stabilizers: Re-enable settings you changed before finishing. Text-entry quirks: + Local iOS fill reuses focus when the current command identifies the target input and reads its keyboard focus. Readable focused inputs use select-all/delete with read-back before any fallback caret tap. iOS Allow Paste cannot be exercised under XCUITest; prefill with clipboard write "some text" instead and test the system prompt manually. Android Gboard handwriting/stylus UI can capture text in an IME-owned input instead of the app field. If fill reports that input was captured by the keyboard/IME, use the diagnostic targetInput/actualInput details, inspect keyboard status/get if needed, and switch or disable handwriting outside the command plan before retrying. Do not keep retrying fill/type against the same field while the IME owns focus. If the exact target changes but app-owned formatting prevents raw equality, fill succeeds with verification: "unconfirmed" plus target-bound requested/before/after evidence; inspect that evidence instead of retrying the same mutation. iOS accessibility-driven fill types the replacement once and never clears and retypes a read-back mismatch. Exact settled text confirms the entry. Nonempty replacements wait until the confirmation deadline before reporting TEXT_ENTRY_MISMATCH. A value containing only characters in order from the post-clear value plus the request fails with TEXT_ENTRY_MISMATCH, including plain dropped characters and unchanged values; legitimate trimming or masks can also hit this limit ("5" into "$0.00" displaying "$0.05"). Other settled values return verification: "unconfirmed" with requested/before/after/target evidence, including grouping ("00 062 91 77" for "000629177"), currency ("$10.00" for "1000"), and OTP summaries. Missing, reordered, duplicated or stale text can also be unconfirmed, so assert the expected value with a value selector, get/is, or the resulting screen before continuing. For single-line fills ending in a submit key, any non-matching value observed on the same target succeeds unverified, including partial or cleared text; this exception skips dropped-character checks because the submit may change the field. Assert the resulting screen. Multiline fields compare newlines as content. Unreadable or removed fields remain unverified. The coordinate-driven fill route still requires an exact match. After TEXT_ENTRY_MISMATCH, inspect the field and retry fill only if it needs correction, using the --delay-ms value from the error hint. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index ecce60609b..8fa658edbb 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -492,6 +492,7 @@ A `click` or `press` whose target the daemon resolves to a point (`@ref` or sele ### Text entry - `fill` replaces the field value. `type` appends to the focused field and accepts text only: use `fill @ref "text"` to target a field, or `press @ref` then `type "text"`. +- Local iOS `fill` avoids another focus tap when the current command identifies the target input and reads its keyboard focus. For readable focused inputs, clearing tries select-all/delete and reads the result before moving the caret for delete-burst fallback. Newly focused inputs retain caret preparation. - Use plain `fill` or `type` first. Add `--delay-ms` only when a debounced or search-as-you-type field misses characters, or the app must receive each incremental update; delayed typing enters characters one at a time instead of using a clipboard-style fallback. On Limrun iOS, `--delay-ms` has no effect on `fill`; use `press` to focus the field and `type --delay-ms` for intentional key-event input. - `fill --text-stdin` reads the text from stdin so a secret never appears in process arguments. Pipe it with `printf %s`. It reads at most 64 KiB, removes one trailing newline, and refuses a terminal, empty input, invalid UTF-8, or an extra text argument. Output shows the value as `${VAR}` with `--record-as ` and as `[REDACTED]` otherwise, including in error messages and details, and in the `requested`, `before`, and `after` read-back of an unconfirmed fill, which can hold a reformatted form of the value. With `--settle`, the diff line text and tail labels are replaced whole, so text around the value in the field is not shown either; refs and counts stay. While script recording is armed, pass `--record-as ` or `--no-record`; passing neither is an error. Batch steps don't accept it. - The privacy controls for typed input (`--text-stdin` and `--record-as`) keep the literal out of agent-device's own responses, diagnostics, and recorded scripts. They do not rewrite what the platform records: on iOS, XCTest writes its own diagnostic logs and its `.xcresult` bundle, and typed text can appear in both, on a simulator or a physical device. Treat those as sensitive, and use test-only credentials when they may be kept or shared. See [Security and trust](/docs/security-trust).