diff --git a/packages/maestro/src/internal/engine-types.ts b/packages/maestro/src/internal/engine-types.ts index 705234badf..2e703bde50 100644 --- a/packages/maestro/src/internal/engine-types.ts +++ b/packages/maestro/src/internal/engine-types.ts @@ -105,6 +105,8 @@ export type MaestroRuntimeMetrics = { hierarchyCaptures: number; screenshotCaptures: number; tapRetries: number; + settleLatches: number; + settleTimeouts: number; }; export type MaestroRuntimePort = { diff --git a/packages/maestro/src/internal/facade-execution.ts b/packages/maestro/src/internal/facade-execution.ts index 6eae647fb3..cb5757f733 100644 --- a/packages/maestro/src/internal/facade-execution.ts +++ b/packages/maestro/src/internal/facade-execution.ts @@ -7,6 +7,7 @@ import { isMaestroControlCommandDescriptor, type MaestroEngineEvent, type MaestroEngineObserver, + type MaestroRuntimeMetrics, type MaestroRuntimePort, } from './engine-types.ts'; import { parseMaestroProgram } from './program-ir-parser.ts'; @@ -49,11 +50,7 @@ export type MaestroActionEvent = { export type MaestroCompletedActionEvent = MaestroActionEvent & { readonly durationMs: number; - readonly runtimeMetrics?: { - hierarchyCaptures: number; - screenshotCaptures: number; - tapRetries: number; - }; + readonly runtimeMetrics?: MaestroRuntimeMetrics; readonly data?: Record; }; diff --git a/packages/maestro/src/internal/replay-plan-execution.ts b/packages/maestro/src/internal/replay-plan-execution.ts index f2a45b789b..644262f23f 100644 --- a/packages/maestro/src/internal/replay-plan-execution.ts +++ b/packages/maestro/src/internal/replay-plan-execution.ts @@ -112,6 +112,8 @@ function runtimeMetricsDelta( hierarchyCaptures: after.hierarchyCaptures - before.hierarchyCaptures, screenshotCaptures: after.screenshotCaptures - before.screenshotCaptures, tapRetries: after.tapRetries - before.tapRetries, + settleLatches: after.settleLatches - before.settleLatches, + settleTimeouts: after.settleTimeouts - before.settleTimeouts, }, }; } diff --git a/packages/maestro/test/conformance/differential/flows/settle-after-tap.yaml b/packages/maestro/test/conformance/differential/flows/settle-after-tap.yaml index 00e8c101fc..4143250f77 100644 --- a/packages/maestro/test/conformance/differential/flows/settle-after-tap.yaml +++ b/packages/maestro/test/conformance/differential/flows/settle-after-tap.yaml @@ -9,5 +9,6 @@ appId: com.callstack.agentdevicelab - assertVisible: Agent Device Tester - tapOn: text: Settings + retryTapIfNoChange: true - assertVisible: id: open-inert-surface diff --git a/packages/maestro/test/conformance/differential/invariants.test.ts b/packages/maestro/test/conformance/differential/invariants.test.ts index 22c04b0d06..a11c3d9f1f 100644 --- a/packages/maestro/test/conformance/differential/invariants.test.ts +++ b/packages/maestro/test/conformance/differential/invariants.test.ts @@ -7,6 +7,7 @@ import os from 'node:os'; import path from 'node:path'; import { test } from 'node:test'; import { MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS, parseMaestroConformanceSource } from '../harness.ts'; +import { parseMaestroProgram } from '../../../src/internal/program-ir-parser.ts'; import { DIFFERENTIAL_SCENARIOS } from './scenarios.ts'; import { type Invariant, evaluateInvariant, readTrace } from './invariants.ts'; @@ -25,6 +26,77 @@ const stop = (command: string, durationMs: number, step = 1) => ({ durationMs, }); +const SETTLE_TIMEOUT_INVARIANT: Invariant = { + kind: 'metricAtMost', + command: 'tapOn', + metric: 'settleTimeouts', + max: 0, + because: 'test', +}; + +const SETTLE_LATCH_INVARIANT: Invariant = { + kind: 'metricAtLeast', + command: 'tapOn', + metric: 'settleLatches', + min: 1, + because: 'test', +}; + +const settleStep = ( + command: string, + durationMs: number, + metrics: { settleLatches: number; settleTimeouts: number }, + step = 1, +) => ({ + type: 'replay_action_stop', + step, + command, + ok: true, + durationMs, + resultTiming: { hierarchyCaptures: 1, screenshotCaptures: 0, tapRetries: 0, ...metrics }, +}); + +test('a stability loop that latched holds the settle invariant however slow the step', () => { + const result = evaluateInvariant( + [settleStep('tapOn', 3344, { settleLatches: 1, settleTimeouts: 0 })], + SETTLE_TIMEOUT_INVARIANT, + ); + assert.equal(result.status, 'held'); +}); + +test('a stability loop that never latched violates it however fast the step', () => { + const result = evaluateInvariant( + [settleStep('tapOn', 120, { settleLatches: 0, settleTimeouts: 1 })], + SETTLE_TIMEOUT_INVARIANT, + ); + assert.equal(result.status, 'violated'); + assert.match(result.detail, /settleTimeouts was 1/); +}); + +test('a tap that did not run the loop violates the settle proof-of-life invariant', () => { + const result = evaluateInvariant( + [settleStep('tapOn', 120, { settleLatches: 0, settleTimeouts: 0 })], + SETTLE_LATCH_INVARIANT, + ); + assert.equal(result.status, 'violated'); +}); + +test('another command running out of settle budget does not implicate the tap', () => { + const result = evaluateInvariant( + [ + settleStep('scroll', 900, { settleLatches: 0, settleTimeouts: 2 }), + settleStep('tapOn', 3344, { settleLatches: 1, settleTimeouts: 0 }, 2), + ], + SETTLE_TIMEOUT_INVARIANT, + ); + assert.equal(result.status, 'held'); +}); + +test('a trace with no settle metric reports no-data rather than passing', () => { + const result = evaluateInvariant([stop('tapOn', 350)], SETTLE_TIMEOUT_INVARIANT); + assert.equal(result.status, 'no-data'); +}); + test('a tap that latches early holds the settle invariant', () => { // Healthy: Android ~350ms, iOS ~800-1100ms — well under the 2000ms budget. const result = evaluateInvariant([stop('tapOn', 350)], SETTLE_INVARIANT); @@ -81,12 +153,20 @@ test('readTrace on a missing file returns no events', () => { test('bug class 4 has a machine-checkable invariant, not just outcome parity', () => { const settle = DIFFERENTIAL_SCENARIOS.find((scenario) => scenario.bugClass === 4); - const invariant = settle?.engineInvariants?.[0]; - assert.ok(invariant, 'settle scenario must carry an engine-side invariant'); - assert.equal(invariant?.kind, 'stepDurationBelow'); - assert.equal( - invariant?.kind === 'stepDurationBelow' ? invariant.maxMs : undefined, - MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS, + const invariants = settle?.engineInvariants; + assert.ok(invariants, 'settle scenario must carry engine-side invariants'); + assert.deepEqual( + invariants.map((invariant) => + invariant.kind === 'metricAtLeast' + ? { kind: invariant.kind, metric: invariant.metric, min: invariant.min } + : invariant.kind === 'metricAtMost' + ? { kind: invariant.kind, metric: invariant.metric, max: invariant.max } + : { kind: invariant.kind }, + ), + [ + { kind: 'metricAtLeast', metric: 'settleLatches', min: 1 }, + { kind: 'metricAtMost', metric: 'settleTimeouts', max: 0 }, + ], ); }); @@ -94,6 +174,7 @@ const SETTLE_FLOW_PATH = path.join(import.meta.dirname, 'flows/settle-after-tap. function assertSettleFlowSemantics(source: string): void { const parsed = parseMaestroConformanceSource(source, SETTLE_FLOW_PATH); + const program = parseMaestroProgram(source, { sourcePath: SETTLE_FLOW_PATH }); assert.equal( parsed.commands.some( (command) => command.kind === 'scroll' || command.kind === 'scrollUntilVisible', @@ -104,6 +185,9 @@ function assertSettleFlowSemantics(source: string): void { parsed.commands.filter((command) => command.kind === 'tap'), [{ kind: 'tap', longPress: false, repeat: 1, target: { selector: { text: 'Settings' } } }], ); + const tap = program.commands.find((command) => command.kind === 'tapOn'); + assert.equal(tap?.kind, 'tapOn'); + assert.equal(tap?.retryTapIfNoChange, true); assert.equal( parsed.commands.some( (command) => @@ -119,9 +203,10 @@ test('the settle detector reaches its tap without an unrelated setup command', ( assertSettleFlowSemantics(fs.readFileSync(SETTLE_FLOW_PATH, 'utf8')); }); -test('the settle flow guard rejects a changed tap target or inserted scroll', () => { +test('the settle flow guard rejects a changed tap target, disabled retry, or inserted scroll', () => { const flow = fs.readFileSync(SETTLE_FLOW_PATH, 'utf8'); assert.throws(() => assertSettleFlowSemantics(flow.replace('text: Settings', 'text: Home'))); + assert.throws(() => assertSettleFlowSemantics(flow.replace(/\n\s*retryTapIfNoChange: true/, ''))); assert.throws(() => assertSettleFlowSemantics(flow.replace('- tapOn:', '- scroll\n- tapOn:'))); }); diff --git a/packages/maestro/test/conformance/differential/invariants.ts b/packages/maestro/test/conformance/differential/invariants.ts index 92a8b19fd7..5d11103d3d 100644 --- a/packages/maestro/test/conformance/differential/invariants.ts +++ b/packages/maestro/test/conformance/differential/invariants.ts @@ -7,13 +7,12 @@ // // These invariants read the `replay-timing.ndjson` written by the test runtime // (src/daemon/handlers/session-test-runtime.ts) and assert engine-side facts — -// e.g. a tap must not burn the entire settle budget, which is the signature of a -// stability loop that never latches (a full-budget tap measures ~2093-2117ms -// against a 2000ms budget; a healthy Android tap is ~350ms, iOS ~800-1100ms). +// e.g. a tap's stability loop must latch rather than run out of settle budget. // // The evaluator is pure and unit-tested against synthetic traces; the device run // that produces a real trace happens only on the scheduled workflow. import fs from 'node:fs'; +import type { MaestroRuntimeMetrics } from '../../../src/internal/engine-types.ts'; export type TraceEvent = { type: string; @@ -21,10 +20,12 @@ export type TraceEvent = { command?: string; ok?: boolean; durationMs?: number; - /** Per-step MaestroRuntimeMetrics delta (hierarchyCaptures/screenshotCaptures/tapRetries). */ + /** Per-step MaestroRuntimeMetrics delta. */ resultTiming?: Record; }; +export type MetricKey = keyof MaestroRuntimeMetrics; + export type Invariant = | { kind: 'stepDurationBelow'; @@ -38,10 +39,17 @@ export type Invariant = kind: 'metricAtLeast'; command: string; /** MaestroRuntimeMetrics key, recorded per step as a delta. */ - metric: 'tapRetries' | 'hierarchyCaptures' | 'screenshotCaptures'; + metric: MetricKey; min: number; because: string; } + | { + kind: 'metricAtMost'; + command: string; + metric: MetricKey; + max: number; + because: string; + } | { kind: 'gestureExecutionProfile'; command: string; @@ -76,69 +84,81 @@ function completedSteps(events: TraceEvent[], command: string): TraceEvent[] { return events.filter((event) => event.type === 'replay_action_stop' && event.command === command); } -export function evaluateInvariant(events: TraceEvent[], invariant: Invariant): InvariantResult { - const steps = completedSteps(events, invariant.command); - if (steps.length === 0) { +type MetricBound = Extract; + +function metricBoundTerms(invariant: MetricBound) { + return invariant.kind === 'metricAtMost' + ? { + broken: (peak: number) => peak > invariant.max, + failed: `> ${invariant.max}`, + held: (peak: number) => `stayed at ${peak} (<= ${invariant.max})`, + } + : { + broken: (peak: number) => peak < invariant.min, + failed: `< ${invariant.min}`, + held: (peak: number) => `reached ${peak} (>= ${invariant.min})`, + }; +} + +function evaluateMetricBound(steps: TraceEvent[], invariant: MetricBound): InvariantResult { + const values = steps + .map((step) => step.resultTiming?.[invariant.metric]) + .filter((value): value is number => typeof value === 'number'); + if (values.length === 0) { return { invariant, status: 'no-data', - detail: `no completed ${invariant.command} steps in the trace`, + detail: `no ${invariant.command} step recorded a ${invariant.metric} metric`, }; } - - if (invariant.kind === 'metricAtLeast') { - const values = steps - .map((step) => step.resultTiming?.[invariant.metric]) - .filter((value): value is number => typeof value === 'number'); - if (values.length === 0) { - return { - invariant, - status: 'no-data', - detail: `no ${invariant.command} step recorded a ${invariant.metric} metric`, - }; - } - // Per-step deltas: the strongest single step is what proves the path ran. - const best = Math.max(...values); - if (best < invariant.min) { - return { + const peak = Math.max(...values); + const terms = metricBoundTerms(invariant); + return terms.broken(peak) + ? { invariant, status: 'violated', - detail: `highest ${invariant.command} ${invariant.metric} was ${best} (< ${invariant.min}): ${invariant.because}`, + detail: `highest ${invariant.command} ${invariant.metric} was ${peak} (${terms.failed}): ${invariant.because}`, + } + : { + invariant, + status: 'held', + detail: `${invariant.command} ${invariant.metric} ${terms.held(peak)}`, }; - } +} + +function evaluateGestureProfile( + steps: TraceEvent[], + invariant: Extract, +): InvariantResult { + const profiles = steps + .map((step) => step.resultTiming?.executionProfile) + .filter((value): value is string => typeof value === 'string'); + if (profiles.length === 0) { return { invariant, - status: 'held', - detail: `${invariant.command} ${invariant.metric} reached ${best} (>= ${invariant.min})`, + status: 'no-data', + detail: `no ${invariant.command} step recorded an executionProfile`, }; } - - if (invariant.kind === 'gestureExecutionProfile') { - const profiles = steps - .map((step) => step.resultTiming?.executionProfile) - .filter((value): value is string => typeof value === 'string'); - if (profiles.length === 0) { - return { - invariant, - status: 'no-data', - detail: `no ${invariant.command} step recorded an executionProfile`, - }; - } - const firstMismatch = profiles.find((profile) => profile !== invariant.profile); - if (firstMismatch !== undefined) { - return { - invariant, - status: 'violated', - detail: `${invariant.command} executionProfile was ${firstMismatch} (expected ${invariant.profile}): ${invariant.because}`, - }; - } + const firstMismatch = profiles.find((profile) => profile !== invariant.profile); + if (firstMismatch !== undefined) { return { invariant, - status: 'held', - detail: `${invariant.command} executionProfile is ${invariant.profile} on ${profiles.length} step(s)`, + status: 'violated', + detail: `${invariant.command} executionProfile was ${firstMismatch} (expected ${invariant.profile}): ${invariant.because}`, }; } + return { + invariant, + status: 'held', + detail: `${invariant.command} executionProfile is ${invariant.profile} on ${profiles.length} step(s)`, + }; +} +function evaluateStepDuration( + steps: TraceEvent[], + invariant: Extract, +): InvariantResult { const timed = steps.filter((step) => typeof step.durationMs === 'number'); if (timed.length === 0) { return { @@ -162,6 +182,22 @@ export function evaluateInvariant(events: TraceEvent[], invariant: Invariant): I }; } +export function evaluateInvariant(events: TraceEvent[], invariant: Invariant): InvariantResult { + const steps = completedSteps(events, invariant.command); + if (steps.length === 0) { + return { + invariant, + status: 'no-data', + detail: `no completed ${invariant.command} steps in the trace`, + }; + } + if (invariant.kind === 'metricAtMost' || invariant.kind === 'metricAtLeast') { + return evaluateMetricBound(steps, invariant); + } + if (invariant.kind === 'gestureExecutionProfile') return evaluateGestureProfile(steps, invariant); + return evaluateStepDuration(steps, invariant); +} + export function evaluateInvariants( events: TraceEvent[], invariants: readonly Invariant[], diff --git a/packages/maestro/test/conformance/differential/scenarios.ts b/packages/maestro/test/conformance/differential/scenarios.ts index 95d34bb3f6..a8493a3e65 100644 --- a/packages/maestro/test/conformance/differential/scenarios.ts +++ b/packages/maestro/test/conformance/differential/scenarios.ts @@ -17,7 +17,6 @@ // we do it engine-side via `engineInvariants` over agent-device's own replay // timing trace. Scenarios without invariants prove outcome parity ONLY; do not // read more into them than that. -import { MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS } from '../harness.ts'; import type { Invariant } from './invariants.ts'; /** Bundle id of the fixture app the workflow installs before running scenarios. */ @@ -98,11 +97,19 @@ export const DIFFERENTIAL_SCENARIOS: DifferentialScenario[] = [ // budget still passes. This invariant is the actual bug-class-4 detector. engineInvariants: [ { - kind: 'stepDurationBelow', + kind: 'metricAtLeast', + command: 'tapOn', + metric: 'settleLatches', + min: 1, + because: 'the scenario must execute and latch its inline stability loop', + }, + { + kind: 'metricAtMost', command: 'tapOn', - maxMs: MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS, + metric: 'settleTimeouts', + max: 0, because: - 'a tap consuming the entire settle budget means the stability loop never latched — the signature of a sleep-before-capture ordering regression', + 'a tap whose stability loop ran out of settle budget without the UI going quiet is the sleep-before-capture ordering signature', }, ], divergenceMeans: diff --git a/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-observation.test.ts b/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-observation.test.ts index cb1be72448..790b4111cd 100644 --- a/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-observation.test.ts +++ b/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-observation.test.ts @@ -235,6 +235,27 @@ test('compares snapshots before sleeping and captures once beyond a zero settle expect(result.snapshot.nodes[0]?.value).toBe('settled'); }); +test('reports which exit the stability loop took', async () => { + const clock = { value: 0 }; + + const latched = await waitForTypedSnapshotStability({ + timeoutMs: 1_000, + context: { generation: 0, env: {} }, + snapshot: async () => makeSnapshot([{ index: 0, type: 'Text', value: 'stable' }]), + dependencies: makeDependencies(clock), + }); + expect(latched.settled).toBe(true); + + let value = 0; + const exhausted = await waitForTypedSnapshotStability({ + timeoutMs: 1_000, + context: { generation: 0, env: {} }, + snapshot: async () => makeSnapshot([{ index: 0, type: 'Text', value: `moving ${++value}` }]), + dependencies: makeDependencies(clock), + }); + expect(exhausted.settled).toBe(false); +}); + test('confirms an unchanged hierarchy across one polling interval', async () => { const clock = { value: 0 }; let captures = 0; diff --git a/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-targets.test.ts b/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-targets.test.ts index 7c4649064e..65b94aaa8e 100644 --- a/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-targets.test.ts +++ b/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-targets.test.ts @@ -235,6 +235,8 @@ test('retries an iOS non-hittable coordinate fallback when the hierarchy does no hierarchyCaptures: 5, screenshotCaptures: 2, tapRetries: 1, + settleLatches: 2, + settleTimeouts: 0, }); }); @@ -298,6 +300,8 @@ test('does not retry an iOS tap when only the rendered surface changes', async ( hierarchyCaptures: 3, screenshotCaptures: 2, tapRetries: 0, + settleLatches: 1, + settleTimeouts: 0, }); }); @@ -344,9 +348,54 @@ test('uses screenshot evidence without a redundant hierarchy baseline for iOS po hierarchyCaptures: 4, screenshotCaptures: 2, tapRetries: 1, + settleLatches: 2, + settleTimeouts: 0, }); }); +test('records an exhausted inline tap settle', async () => { + const clock = { value: 0 }; + let snapshots = 0; + const port = createDaemonMaestroRuntimePort({ + baseReq: makeBaseRequest({ flags: { platform: 'android', replayBackend: 'maestro' } }), + invoke: async (request) => { + if (request.command !== 'snapshot') return { ok: true, data: {} }; + snapshots += 1; + return { + ok: true, + data: { + nodes: [ + { index: 0, type: 'Application' }, + { + index: 1, + parentIndex: 0, + type: snapshots === 1 ? 'Button' : 'Text', + ...(snapshots === 1 ? { identifier: 'continue' } : { value: String(snapshots) }), + rect: { x: 20, y: 40, width: 120, height: 44 }, + }, + ], + }, + }; + }, + dependencies: makeDependencies(clock), + platform: 'android', + }); + + await port.execute({ + command: { + kind: 'tapOn', + source: { line: 2 }, + target: { space: 'target', selector: { id: 'continue' } }, + retryTapIfNoChange: true, + }, + generation: 0, + env: {}, + invalidateObservation() {}, + }); + + expect(port.readMetrics?.()).toMatchObject({ settleTimeouts: 1 }); +}); + function solidPng(value: number): Buffer { const image = new PNG({ width: 2, height: 2 }); image.data.fill(value); diff --git a/src/daemon/adapters/maestro/__tests__/daemon-runtime-port.test.ts b/src/daemon/adapters/maestro/__tests__/daemon-runtime-port.test.ts index 4d9cac2e75..353baf882d 100644 --- a/src/daemon/adapters/maestro/__tests__/daemon-runtime-port.test.ts +++ b/src/daemon/adapters/maestro/__tests__/daemon-runtime-port.test.ts @@ -249,6 +249,8 @@ test('uses an observation as the baseline for a later mutation barrier', async ( hierarchyCaptures: 2, screenshotCaptures: 0, tapRetries: 0, + settleLatches: 1, + settleTimeouts: 0, }); expect(clock.value).toBe(MAESTRO_OBSERVATION_POLL_MS); }); diff --git a/src/daemon/adapters/maestro/daemon-runtime-port-observation.ts b/src/daemon/adapters/maestro/daemon-runtime-port-observation.ts index f74b9b2444..a5056036f6 100644 --- a/src/daemon/adapters/maestro/daemon-runtime-port-observation.ts +++ b/src/daemon/adapters/maestro/daemon-runtime-port-observation.ts @@ -43,12 +43,15 @@ export type MaestroSnapshotSource = { readonly requireStability: (generation: number) => void; readonly consumeStabilityFromVisualWait: (context: MaestroRuntimeReadContext) => void; readonly prime: (generation: number, snapshot: SnapshotState) => void; - readonly settlePending: (context: MaestroRuntimeReadContext) => Promise; + readonly settlePending: ( + context: MaestroRuntimeReadContext, + ) => Promise; }; export type StableMaestroSnapshot = { readonly snapshot: SnapshotState; readonly signature: string; + readonly settled: boolean; }; type MaestroTargetResolutionMode = 'tap' | 'swipe' | 'observe'; @@ -237,12 +240,12 @@ export async function waitForTypedSnapshotStability(params: { ); const snapshot = await captureRetriableMaestroSnapshot(params, deadline); const signature = maestroSnapshotSignature(snapshot); - if (signature === previousSignature) return { snapshot, signature }; + if (signature === previousSignature) return { snapshot, signature, settled: true }; previous = snapshot; previousSignature = signature; if (params.dependencies.now() >= deadline) { - return { snapshot: previous, signature: previousSignature }; + return { snapshot: previous, signature: previousSignature, settled: false }; } } } diff --git a/src/daemon/adapters/maestro/daemon-runtime-port-snapshot-source.ts b/src/daemon/adapters/maestro/daemon-runtime-port-snapshot-source.ts index 35ba3ff053..5d792fb80f 100644 --- a/src/daemon/adapters/maestro/daemon-runtime-port-snapshot-source.ts +++ b/src/daemon/adapters/maestro/daemon-runtime-port-snapshot-source.ts @@ -101,7 +101,7 @@ export function createDaemonMaestroSnapshotSource( primed = { generation, snapshot }; }, settlePending: async (context) => { - if (stabilityRequiredGeneration === undefined) return; + if (stabilityRequiredGeneration === undefined) return undefined; if (stabilityRequiredGeneration !== context.generation) { throw new AppError( 'COMMAND_FAILED', @@ -120,6 +120,7 @@ export function createDaemonMaestroSnapshotSource( stabilityRequiredGeneration = undefined; stabilityBaseline = undefined; primed = { generation: context.generation, snapshot: stable.snapshot }; + return stable; }, }; } diff --git a/src/daemon/adapters/maestro/daemon-runtime-port.ts b/src/daemon/adapters/maestro/daemon-runtime-port.ts index 65ed92cccf..70e7c8b066 100644 --- a/src/daemon/adapters/maestro/daemon-runtime-port.ts +++ b/src/daemon/adapters/maestro/daemon-runtime-port.ts @@ -16,6 +16,7 @@ import { observeTypedMaestroCondition, scrollUntilTypedMaestroTarget, waitForTypedSnapshotStability, + type StableMaestroSnapshot, type MaestroSnapshotSource, } from './daemon-runtime-port-observation.ts'; import { createDaemonMaestroSnapshotSource } from './daemon-runtime-port-snapshot-source.ts'; @@ -41,9 +42,19 @@ function createDaemonMaestroRuntimeParts(options: CreateDaemonMaestroRuntimeOper operations: MaestroRuntimeOperations; snapshots: MaestroSnapshotSource; readMetrics: () => MaestroRuntimeMetrics; + recordSettle: (stable: StableMaestroSnapshot) => void; } { const snapshots = createDaemonMaestroSnapshotSource(options); - const metrics = { screenshotCaptures: 0, tapRetries: 0 }; + const metrics: Omit = { + screenshotCaptures: 0, + tapRetries: 0, + settleLatches: 0, + settleTimeouts: 0, + }; + const recordSettle = (stable: StableMaestroSnapshot) => { + if (stable.settled) metrics.settleLatches += 1; + else metrics.settleTimeouts += 1; + }; const platform = options.platform; const invoke = (operation: Operation) => { if (operation.kind === 'screenshot') metrics.screenshotCaptures += 1; @@ -78,6 +89,7 @@ function createDaemonMaestroRuntimeParts(options: CreateDaemonMaestroRuntimeOper snapshot: snapshots.capture, dependencies: options.dependencies, }); + recordSettle(stable); snapshots.prime(context.generation, stable.snapshot); }; @@ -216,14 +228,13 @@ function createDaemonMaestroRuntimeParts(options: CreateDaemonMaestroRuntimeOper }, context, ); - return ( - await waitForTypedSnapshotStability({ - timeoutMs: Math.min(MAESTRO_RUNTIME_ADAPTER_POLICY.settleTimeoutMs, remainingMs), - context, - snapshot: snapshots.capture, - dependencies: options.dependencies, - }) - ).snapshot; + const stable = await waitForTypedSnapshotStability({ + timeoutMs: Math.min(MAESTRO_RUNTIME_ADAPTER_POLICY.settleTimeoutMs, remainingMs), + context, + snapshot: snapshots.capture, + dependencies: options.dependencies, + }); + return stable.snapshot; }, }); if (match.visiblePercentage !== MAESTRO_RUNTIME_ADAPTER_POLICY.scrollUntilVisiblePercentage) { @@ -279,17 +290,20 @@ function createDaemonMaestroRuntimeParts(options: CreateDaemonMaestroRuntimeOper operations, snapshots, readMetrics: () => ({ ...snapshots.readMetrics(), ...metrics }), + recordSettle, }; } export function createDaemonMaestroRuntimePort( options: CreateDaemonMaestroRuntimeOperationsOptions, ): MaestroRuntimePort { - const { operations, snapshots, readMetrics } = createDaemonMaestroRuntimeParts(options); + const { operations, snapshots, readMetrics, recordSettle } = + createDaemonMaestroRuntimeParts(options); return createMaestroRuntimePort(operations, { beforeExecute: async ({ context, requiresSettledPredecessor }) => { if (requiresSettledPredecessor) { - await snapshots.settlePending(context); + const stable = await snapshots.settlePending(context); + if (stable) recordSettle(stable); } }, afterExecute: ({ context, visualStabilityReached }) => { diff --git a/src/daemon/adapters/maestro/daemon-runtime-tap.ts b/src/daemon/adapters/maestro/daemon-runtime-tap.ts index c08d0c9243..31d5036985 100644 --- a/src/daemon/adapters/maestro/daemon-runtime-tap.ts +++ b/src/daemon/adapters/maestro/daemon-runtime-tap.ts @@ -6,6 +6,7 @@ import { type MaestroDispatchSelector, type MaestroRuntimeOperationContext, type MaestroRuntimeOperations, + type MaestroRuntimeMetrics, type MaestroRuntimeReadContext, type MaestroTargetMatch, type MaestroTargetQuery, @@ -71,7 +72,7 @@ export async function resolveDaemonMaestroTarget(params: { export async function tapTargetAndSettle( options: CreateDaemonMaestroRuntimeOperationsOptions, snapshots: MaestroSnapshotSource, - metrics: { screenshotCaptures: number; tapRetries: number }, + metrics: Omit, target: Parameters[0]['target'], context: MaestroRuntimeOperationContext, policy: { click: MaestroClickOptions; retryIfNoChange: boolean }, @@ -119,7 +120,7 @@ export async function tapTargetAndSettle( async function tapTargetWithRetry( options: CreateDaemonMaestroRuntimeOperationsOptions, snapshots: MaestroSnapshotSource, - metrics: { screenshotCaptures: number; tapRetries: number }, + metrics: Omit, target: Parameters[0]['target'], context: MaestroRuntimeOperationContext, flags: MaestroClickOptions, @@ -129,13 +130,17 @@ async function tapTargetWithRetry( const baselineSignature = target.resolution?.surfaceSignature ?? (screenshotBaseline ? undefined : maestroSnapshotSignature(await snapshots.capture(context))); - const settle = async () => - await waitForTypedSnapshotStability({ + const settle = async () => { + const stable = await waitForTypedSnapshotStability({ timeoutMs: MAESTRO_RUNTIME_ADAPTER_POLICY.settleTimeoutMs, context, snapshot: snapshots.capture, dependencies: options.dependencies, }); + if (stable.settled) metrics.settleLatches += 1; + else metrics.settleTimeouts += 1; + return stable; + }; try { const observed = await executeTapRetryLoop({ diff --git a/src/daemon/handlers/__tests__/session-replay-maestro-label.test.ts b/src/daemon/handlers/__tests__/session-replay-maestro-label.test.ts index 6a3ac86c4c..1a49f01b97 100644 --- a/src/daemon/handlers/__tests__/session-replay-maestro-label.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-maestro-label.test.ts @@ -24,7 +24,13 @@ test('forwards command labels to progress and replay trace projection', () => { observer.actionCompleted?.({ ...event, durationMs: 5, - runtimeMetrics: { hierarchyCaptures: 1, screenshotCaptures: 0, tapRetries: 0 }, + runtimeMetrics: { + hierarchyCaptures: 1, + screenshotCaptures: 0, + tapRetries: 0, + settleLatches: 0, + settleTimeouts: 0, + }, }); expect(onStep).toHaveBeenCalledWith({ diff --git a/src/daemon/handlers/__tests__/session-replay-runtime.test.ts b/src/daemon/handlers/__tests__/session-replay-runtime.test.ts index cb713ffa59..e627a5842a 100644 --- a/src/daemon/handlers/__tests__/session-replay-runtime.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-runtime.test.ts @@ -312,7 +312,13 @@ test('typed Maestro writes source-aware redacted step timing traces', async () = command: 'inputText', ok: true, durationMs: expect.any(Number), - resultTiming: { hierarchyCaptures: 2, screenshotCaptures: 0, tapRetries: 0 }, + resultTiming: { + hierarchyCaptures: 2, + screenshotCaptures: 0, + tapRetries: 0, + settleLatches: 1, + settleTimeouts: 0, + }, }), ]); expect(fs.readFileSync(tracePath, 'utf8')).not.toContain('highly-sensitive');