fix(maestro): measure settle outcomes directly

packages/maestro/src/internal/engine-types.ts

@@ -105,6 +105,8 @@ export type MaestroRuntimeMetrics = {
   hierarchyCaptures: number;
   screenshotCaptures: number;
   tapRetries: number;
+  settleLatches: number;
+  settleTimeouts: number;
 };
 
 export type MaestroRuntimePort = {

packages/maestro/src/internal/facade-execution.ts

@@ -7,6 +7,7 @@ import {
   isMaestroControlCommandDescriptor,
   type MaestroEngineEvent,
   type MaestroEngineObserver,
+  type MaestroRuntimeMetrics,
   type MaestroRuntimePort,
 } from './engine-types.ts';
 import { parseMaestroProgram } from './program-ir-parser.ts';
@@ -49,11 +50,7 @@ export type MaestroActionEvent = {
 
 export type MaestroCompletedActionEvent = MaestroActionEvent & {
   readonly durationMs: number;
-  readonly runtimeMetrics?: {
-    hierarchyCaptures: number;
-    screenshotCaptures: number;
-    tapRetries: number;
-  };
+  readonly runtimeMetrics?: MaestroRuntimeMetrics;
   readonly data?: Record<string, unknown>;
 };
 

packages/maestro/src/internal/replay-plan-execution.ts

@@ -112,6 +112,8 @@ function runtimeMetricsDelta(
       hierarchyCaptures: after.hierarchyCaptures - before.hierarchyCaptures,
       screenshotCaptures: after.screenshotCaptures - before.screenshotCaptures,
       tapRetries: after.tapRetries - before.tapRetries,
+      settleLatches: after.settleLatches - before.settleLatches,
+      settleTimeouts: after.settleTimeouts - before.settleTimeouts,
     },
   };
 }

packages/maestro/test/conformance/differential/flows/settle-after-tap.yaml

@@ -9,5 +9,6 @@ appId: com.callstack.agentdevicelab
 - assertVisible: Agent Device Tester
 - tapOn:
     text: Settings
+    retryTapIfNoChange: true
 - assertVisible:
     id: open-inert-surface

packages/maestro/test/conformance/differential/invariants.test.ts

@@ -7,6 +7,7 @@ import os from 'node:os';
 import path from 'node:path';
 import { test } from 'node:test';
 import { MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS, parseMaestroConformanceSource } from '../harness.ts';
+import { parseMaestroProgram } from '../../../src/internal/program-ir-parser.ts';
 import { DIFFERENTIAL_SCENARIOS } from './scenarios.ts';
 import { type Invariant, evaluateInvariant, readTrace } from './invariants.ts';
 
@@ -25,6 +26,77 @@ const stop = (command: string, durationMs: number, step = 1) => ({
   durationMs,
 });
 
+const SETTLE_TIMEOUT_INVARIANT: Invariant = {
+  kind: 'metricAtMost',
+  command: 'tapOn',
+  metric: 'settleTimeouts',
+  max: 0,
+  because: 'test',
+};
+
+const SETTLE_LATCH_INVARIANT: Invariant = {
+  kind: 'metricAtLeast',
+  command: 'tapOn',
+  metric: 'settleLatches',
+  min: 1,
+  because: 'test',
+};
+
+const settleStep = (
+  command: string,
+  durationMs: number,
+  metrics: { settleLatches: number; settleTimeouts: number },
+  step = 1,
+) => ({
+  type: 'replay_action_stop',
+  step,
+  command,
+  ok: true,
+  durationMs,
+  resultTiming: { hierarchyCaptures: 1, screenshotCaptures: 0, tapRetries: 0, ...metrics },
+});
+
+test('a stability loop that latched holds the settle invariant however slow the step', () => {
+  const result = evaluateInvariant(
+    [settleStep('tapOn', 3344, { settleLatches: 1, settleTimeouts: 0 })],
+    SETTLE_TIMEOUT_INVARIANT,
+  );
+  assert.equal(result.status, 'held');
+});
+
+test('a stability loop that never latched violates it however fast the step', () => {
+  const result = evaluateInvariant(
+    [settleStep('tapOn', 120, { settleLatches: 0, settleTimeouts: 1 })],
+    SETTLE_TIMEOUT_INVARIANT,
+  );
+  assert.equal(result.status, 'violated');
+  assert.match(result.detail, /settleTimeouts was 1/);
+});
+
+test('a tap that did not run the loop violates the settle proof-of-life invariant', () => {
+  const result = evaluateInvariant(
+    [settleStep('tapOn', 120, { settleLatches: 0, settleTimeouts: 0 })],
+    SETTLE_LATCH_INVARIANT,
+  );
+  assert.equal(result.status, 'violated');
+});
+
+test('another command running out of settle budget does not implicate the tap', () => {
+  const result = evaluateInvariant(
+    [
+      settleStep('scroll', 900, { settleLatches: 0, settleTimeouts: 2 }),
+      settleStep('tapOn', 3344, { settleLatches: 1, settleTimeouts: 0 }, 2),
+    ],
+    SETTLE_TIMEOUT_INVARIANT,
+  );
+  assert.equal(result.status, 'held');
+});
+
+test('a trace with no settle metric reports no-data rather than passing', () => {
+  const result = evaluateInvariant([stop('tapOn', 350)], SETTLE_TIMEOUT_INVARIANT);
+  assert.equal(result.status, 'no-data');
+});
+
 test('a tap that latches early holds the settle invariant', () => {
   // Healthy: Android ~350ms, iOS ~800-1100ms — well under the 2000ms budget.
   const result = evaluateInvariant([stop('tapOn', 350)], SETTLE_INVARIANT);
@@ -81,19 +153,28 @@ test('readTrace on a missing file returns no events', () => {
 
 test('bug class 4 has a machine-checkable invariant, not just outcome parity', () => {
   const settle = DIFFERENTIAL_SCENARIOS.find((scenario) => scenario.bugClass === 4);
-  const invariant = settle?.engineInvariants?.[0];
-  assert.ok(invariant, 'settle scenario must carry an engine-side invariant');
-  assert.equal(invariant?.kind, 'stepDurationBelow');
-  assert.equal(
-    invariant?.kind === 'stepDurationBelow' ? invariant.maxMs : undefined,
-    MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS,
+  const invariants = settle?.engineInvariants;
+  assert.ok(invariants, 'settle scenario must carry engine-side invariants');
+  assert.deepEqual(
+    invariants.map((invariant) =>
+      invariant.kind === 'metricAtLeast'
+        ? { kind: invariant.kind, metric: invariant.metric, min: invariant.min }
+        : invariant.kind === 'metricAtMost'
+          ? { kind: invariant.kind, metric: invariant.metric, max: invariant.max }
+          : { kind: invariant.kind },
+    ),
+    [
+      { kind: 'metricAtLeast', metric: 'settleLatches', min: 1 },
+      { kind: 'metricAtMost', metric: 'settleTimeouts', max: 0 },
+    ],
   );
 });
 
 const SETTLE_FLOW_PATH = path.join(import.meta.dirname, 'flows/settle-after-tap.yaml');
 
 function assertSettleFlowSemantics(source: string): void {
   const parsed = parseMaestroConformanceSource(source, SETTLE_FLOW_PATH);
+  const program = parseMaestroProgram(source, { sourcePath: SETTLE_FLOW_PATH });
   assert.equal(
     parsed.commands.some(
       (command) => command.kind === 'scroll' || command.kind === 'scrollUntilVisible',
@@ -104,6 +185,9 @@ function assertSettleFlowSemantics(source: string): void {
     parsed.commands.filter((command) => command.kind === 'tap'),
     [{ kind: 'tap', longPress: false, repeat: 1, target: { selector: { text: 'Settings' } } }],
   );
+  const tap = program.commands.find((command) => command.kind === 'tapOn');
+  assert.equal(tap?.kind, 'tapOn');
+  assert.equal(tap?.retryTapIfNoChange, true);
   assert.equal(
     parsed.commands.some(
       (command) =>
@@ -119,9 +203,10 @@ test('the settle detector reaches its tap without an unrelated setup command', (
   assertSettleFlowSemantics(fs.readFileSync(SETTLE_FLOW_PATH, 'utf8'));
 });
 
-test('the settle flow guard rejects a changed tap target or inserted scroll', () => {
+test('the settle flow guard rejects a changed tap target, disabled retry, or inserted scroll', () => {
   const flow = fs.readFileSync(SETTLE_FLOW_PATH, 'utf8');
   assert.throws(() => assertSettleFlowSemantics(flow.replace('text: Settings', 'text: Home')));
+  assert.throws(() => assertSettleFlowSemantics(flow.replace(/\n\s*retryTapIfNoChange: true/, '')));
   assert.throws(() => assertSettleFlowSemantics(flow.replace('- tapOn:', '- scroll\n- tapOn:')));
 });
 

packages/maestro/test/conformance/differential/invariants.ts

@@ -7,24 +7,25 @@
 //
 // These invariants read the `replay-timing.ndjson` written by the test runtime
 // (src/daemon/handlers/session-test-runtime.ts) and assert engine-side facts —
-// e.g. a tap must not burn the entire settle budget, which is the signature of a
-// stability loop that never latches (a full-budget tap measures ~2093-2117ms
-// against a 2000ms budget; a healthy Android tap is ~350ms, iOS ~800-1100ms).
+// e.g. a tap's stability loop must latch rather than run out of settle budget.
 //
 // The evaluator is pure and unit-tested against synthetic traces; the device run
 // that produces a real trace happens only on the scheduled workflow.
 import fs from 'node:fs';
+import type { MaestroRuntimeMetrics } from '../../../src/internal/engine-types.ts';
 
 export type TraceEvent = {
   type: string;
   step?: number;
   command?: string;
   ok?: boolean;
   durationMs?: number;
-  /** Per-step MaestroRuntimeMetrics delta (hierarchyCaptures/screenshotCaptures/tapRetries). */
+  /** Per-step MaestroRuntimeMetrics delta. */
   resultTiming?: Record<string, unknown>;
 };
 
+export type MetricKey = keyof MaestroRuntimeMetrics;
+
 export type Invariant =
   | {
       kind: 'stepDurationBelow';
@@ -38,10 +39,17 @@ export type Invariant =
       kind: 'metricAtLeast';
       command: string;
       /** MaestroRuntimeMetrics key, recorded per step as a delta. */
-      metric: 'tapRetries' | 'hierarchyCaptures' | 'screenshotCaptures';
+      metric: MetricKey;
       min: number;
       because: string;
     }
+  | {
+      kind: 'metricAtMost';
+      command: string;
+      metric: MetricKey;
+      max: number;
+      because: string;
+    }
   | {
       kind: 'gestureExecutionProfile';
       command: string;
@@ -76,69 +84,81 @@ function completedSteps(events: TraceEvent[], command: string): TraceEvent[] {
   return events.filter((event) => event.type === 'replay_action_stop' && event.command === command);
 }
 
-export function evaluateInvariant(events: TraceEvent[], invariant: Invariant): InvariantResult {
-  const steps = completedSteps(events, invariant.command);
-  if (steps.length === 0) {
+type MetricBound = Extract<Invariant, { kind: 'metricAtLeast' | 'metricAtMost' }>;
+
+function metricBoundTerms(invariant: MetricBound) {
+  return invariant.kind === 'metricAtMost'
+    ? {
+        broken: (peak: number) => peak > invariant.max,
+        failed: `> ${invariant.max}`,
+        held: (peak: number) => `stayed at ${peak} (<= ${invariant.max})`,
+      }
+    : {
+        broken: (peak: number) => peak < invariant.min,
+        failed: `< ${invariant.min}`,
+        held: (peak: number) => `reached ${peak} (>= ${invariant.min})`,
+      };
+}
+
+function evaluateMetricBound(steps: TraceEvent[], invariant: MetricBound): InvariantResult {
+  const values = steps
+    .map((step) => step.resultTiming?.[invariant.metric])
+    .filter((value): value is number => typeof value === 'number');
+  if (values.length === 0) {
     return {
       invariant,
       status: 'no-data',
-      detail: `no completed ${invariant.command} steps in the trace`,
+      detail: `no ${invariant.command} step recorded a ${invariant.metric} metric`,
     };
   }
-
-  if (invariant.kind === 'metricAtLeast') {
-    const values = steps
-      .map((step) => step.resultTiming?.[invariant.metric])
-      .filter((value): value is number => typeof value === 'number');
-    if (values.length === 0) {
-      return {
-        invariant,
-        status: 'no-data',
-        detail: `no ${invariant.command} step recorded a ${invariant.metric} metric`,
-      };
-    }
-    // Per-step deltas: the strongest single step is what proves the path ran.
-    const best = Math.max(...values);
-    if (best < invariant.min) {
-      return {
+  const peak = Math.max(...values);
+  const terms = metricBoundTerms(invariant);
+  return terms.broken(peak)
+    ? {
         invariant,
         status: 'violated',
-        detail: `highest ${invariant.command} ${invariant.metric} was ${best} (< ${invariant.min}): ${invariant.because}`,
+        detail: `highest ${invariant.command} ${invariant.metric} was ${peak} (${terms.failed}): ${invariant.because}`,
+      }
+    : {
+        invariant,
+        status: 'held',
+        detail: `${invariant.command} ${invariant.metric} ${terms.held(peak)}`,
       };
-    }
+}
+
+function evaluateGestureProfile(
+  steps: TraceEvent[],
+  invariant: Extract<Invariant, { kind: 'gestureExecutionProfile' }>,
+): InvariantResult {
+  const profiles = steps
+    .map((step) => step.resultTiming?.executionProfile)
+    .filter((value): value is string => typeof value === 'string');
+  if (profiles.length === 0) {
     return {
       invariant,
-      status: 'held',
-      detail: `${invariant.command} ${invariant.metric} reached ${best} (>= ${invariant.min})`,
+      status: 'no-data',
+      detail: `no ${invariant.command} step recorded an executionProfile`,
     };
   }
-
-  if (invariant.kind === 'gestureExecutionProfile') {
-    const profiles = steps
-      .map((step) => step.resultTiming?.executionProfile)
-      .filter((value): value is string => typeof value === 'string');
-    if (profiles.length === 0) {
-      return {
-        invariant,
-        status: 'no-data',
-        detail: `no ${invariant.command} step recorded an executionProfile`,
-      };
-    }
-    const firstMismatch = profiles.find((profile) => profile !== invariant.profile);
-    if (firstMismatch !== undefined) {
-      return {
-        invariant,
-        status: 'violated',
-        detail: `${invariant.command} executionProfile was ${firstMismatch} (expected ${invariant.profile}): ${invariant.because}`,
-      };
-    }
+  const firstMismatch = profiles.find((profile) => profile !== invariant.profile);
+  if (firstMismatch !== undefined) {
     return {
       invariant,
-      status: 'held',
-      detail: `${invariant.command} executionProfile is ${invariant.profile} on ${profiles.length} step(s)`,
+      status: 'violated',
+      detail: `${invariant.command} executionProfile was ${firstMismatch} (expected ${invariant.profile}): ${invariant.because}`,
     };
   }
+  return {
+    invariant,
+    status: 'held',
+    detail: `${invariant.command} executionProfile is ${invariant.profile} on ${profiles.length} step(s)`,
+  };
+}
 
+function evaluateStepDuration(
+  steps: TraceEvent[],
+  invariant: Extract<Invariant, { kind: 'stepDurationBelow' }>,
+): InvariantResult {
   const timed = steps.filter((step) => typeof step.durationMs === 'number');
   if (timed.length === 0) {
     return {
@@ -162,6 +182,22 @@ export function evaluateInvariant(events: TraceEvent[], invariant: Invariant): I
   };
 }
 
+export function evaluateInvariant(events: TraceEvent[], invariant: Invariant): InvariantResult {
+  const steps = completedSteps(events, invariant.command);
+  if (steps.length === 0) {
+    return {
+      invariant,
+      status: 'no-data',
+      detail: `no completed ${invariant.command} steps in the trace`,
+    };
+  }
+  if (invariant.kind === 'metricAtMost' || invariant.kind === 'metricAtLeast') {
+    return evaluateMetricBound(steps, invariant);
+  }
+  if (invariant.kind === 'gestureExecutionProfile') return evaluateGestureProfile(steps, invariant);
+  return evaluateStepDuration(steps, invariant);
+}
+
 export function evaluateInvariants(
   events: TraceEvent[],
   invariants: readonly Invariant[],