diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 0b2ba0e64..063685d2c 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -46,6 +46,10 @@ "types": "./src/screenshot-diff-pixels.ts", "default": "./src/screenshot-diff-pixels.ts" }, + "./snapshot-desktop-projection": { + "types": "./src/snapshot-desktop-projection.ts", + "default": "./src/snapshot-desktop-projection.ts" + }, "./snapshot-occlusion": { "types": "./src/snapshot-occlusion.ts", "default": "./src/snapshot-occlusion.ts" diff --git a/packages/capture-kit/src/snapshot-desktop-projection.test.ts b/packages/capture-kit/src/snapshot-desktop-projection.test.ts new file mode 100644 index 000000000..768f31693 --- /dev/null +++ b/packages/capture-kit/src/snapshot-desktop-projection.test.ts @@ -0,0 +1,42 @@ +import { expect, test } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; +import { scopeSnapshotNodes } from './snapshot-desktop-projection.ts'; + +test('scopeSnapshotNodes agrees with every golden scope-policy table case', () => { + const cases = JSON.parse( + fs.readFileSync( + path.resolve(import.meta.dirname, '../../../contracts/fixtures/snapshot-scope-policy.json'), + 'utf8', + ), + ) as Array<{ + name: string; + scope: string; + nodes: Array<{ + depth: number; + label?: string; + value?: string; + identifier?: string; + presented?: boolean; + }>; + expectedSubtreeIndexes: number[]; + }>; + expect(cases.length).toBeGreaterThan(0); + for (const fixture of cases) { + const parents: number[] = []; + const nodes = fixture.nodes.map((node, index) => { + parents.length = node.depth; + const parentIndex = node.depth > 0 ? parents[node.depth - 1] : undefined; + parents[node.depth] = index; + return { ...node, index, parentIndex, rect: { x: index, y: 0, width: 1, height: 1 } }; + }); + const scoped = scopeSnapshotNodes(nodes, fixture.scope, (range) => + nodes.slice(range.start, range.end).some((node) => node.presented !== false), + ); + expect( + scoped.map((node) => node.rect?.x), + fixture.name, + ).toEqual(fixture.expectedSubtreeIndexes); + if (scoped.length > 0) expect(scoped[0]?.depth, fixture.name).toBe(0); + } +}); diff --git a/packages/capture-kit/src/snapshot-desktop-projection.ts b/packages/capture-kit/src/snapshot-desktop-projection.ts new file mode 100644 index 000000000..0d5b0139e --- /dev/null +++ b/packages/capture-kit/src/snapshot-desktop-projection.ts @@ -0,0 +1,75 @@ +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import type { SnapshotOptions, SnapshotResult } from '@agent-device/contracts/interactor-types'; +import { + findSnapshotScopeRange, + normalizeSnapshotScope, + reindexSnapshotNodes, +} from '@agent-device/contracts/snapshot-scope'; + +const INTERACTIVE_ROLE_TOKENS = [ + 'button', + 'menu', + 'textfield', + 'searchfield', + 'checkbox', + 'radio', + 'switch', +] as const; + +export function shapeDesktopSurfaceSnapshot( + data: SnapshotResult, + options: Pick, +): SnapshotResult { + let nodes = data.nodes ?? []; + if (options.scope) { + nodes = scopeSnapshotNodes(nodes, options.scope, (range) => + options.interactiveOnly + ? nodes.slice(range.start, range.end).some(isInteractiveSnapshotNode) + : true, + ); + } + if (options.interactiveOnly) nodes = filterInteractiveSnapshotNodes(nodes); + if (typeof options.depth === 'number') nodes = filterSnapshotNodesByDepth(nodes, options.depth); + return { ...data, nodes }; +} + +export function scopeSnapshotNodes( + nodes: RawSnapshotNode[], + scope: string, + subtreeContributes?: (range: { start: number; end: number }) => boolean, +): RawSnapshotNode[] { + const normalizedScope = normalizeSnapshotScope(scope); + if (!normalizedScope) return reindexSnapshotNodes(nodes); + const range = findSnapshotScopeRange(nodes, normalizedScope, subtreeContributes); + if (!range) return []; + const slice = nodes.slice(range.start, range.end); + return reindexSnapshotNodes(slice, slice[0]?.depth ?? 0); +} + +function filterInteractiveSnapshotNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] { + if (nodes.length === 0) return nodes; + const byIndex = new Map(nodes.map((node) => [node.index, node])); + const keepIndexes = new Set(); + for (const node of nodes) { + if (!isInteractiveSnapshotNode(node)) continue; + let current: RawSnapshotNode | undefined = node; + while (current) { + if (keepIndexes.has(current.index)) break; + keepIndexes.add(current.index); + current = + typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined; + } + } + if (keepIndexes.size === 0) return nodes; + return reindexSnapshotNodes(nodes.filter((node) => keepIndexes.has(node.index))); +} + +function filterSnapshotNodesByDepth(nodes: RawSnapshotNode[], maxDepth: number): RawSnapshotNode[] { + return reindexSnapshotNodes(nodes.filter((node) => (node.depth ?? 0) <= maxDepth)); +} + +function isInteractiveSnapshotNode(node: RawSnapshotNode): boolean { + if ([node.focused, node.hittable, node.rect].some(Boolean)) return true; + const role = `${node.type ?? ''} ${node.role ?? ''} ${node.subrole ?? ''}`.toLowerCase(); + return INTERACTIVE_ROLE_TOKENS.some((token) => role.includes(token)); +} diff --git a/packages/contracts/package.json b/packages/contracts/package.json index 5df12899f..8513fa9e6 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -127,6 +127,10 @@ "types": "./src/back-runtime.ts", "default": "./src/back-runtime.ts" }, + "./backend-diagnostics": { + "types": "./src/backend-diagnostics.ts", + "default": "./src/backend-diagnostics.ts" + }, "./boot-failure": { "types": "./src/boot-failure.ts", "default": "./src/boot-failure.ts" @@ -423,6 +427,10 @@ "types": "./src/snapshot-runtime.ts", "default": "./src/snapshot-runtime.ts" }, + "./snapshot-scope": { + "types": "./src/snapshot-scope.ts", + "default": "./src/snapshot-scope.ts" + }, "./snapshot-timeout-evidence": { "types": "./src/snapshot-timeout-evidence.ts", "default": "./src/snapshot-timeout-evidence.ts" diff --git a/packages/contracts/src/backend-diagnostics.ts b/packages/contracts/src/backend-diagnostics.ts new file mode 100644 index 000000000..4225c9f51 --- /dev/null +++ b/packages/contracts/src/backend-diagnostics.ts @@ -0,0 +1,39 @@ +import type { NetworkIncludeMode } from '@agent-device/kernel/contracts'; + +export type BackendDiagnosticsTimeWindow = { + since?: string; + until?: string; +}; + +export type BackendDiagnosticsPageOptions = BackendDiagnosticsTimeWindow & { + cursor?: string; + limit?: number; +}; + +export type BackendNetworkIncludeMode = NetworkIncludeMode; + +export type BackendNetworkEntry = { + timestamp?: string; + method?: string; + url?: string; + status?: number; + durationMs?: number; + requestHeaders?: Record; + responseHeaders?: Record; + requestBody?: string; + responseBody?: string; + metadata?: Record; +}; + +export type BackendDumpNetworkOptions = BackendDiagnosticsPageOptions & { + include?: BackendNetworkIncludeMode; +}; + +export type BackendDumpNetworkResult = { + entries: readonly BackendNetworkEntry[]; + nextCursor?: string; + timeWindow?: BackendDiagnosticsTimeWindow; + backend?: string; + redacted?: boolean; + notes?: readonly string[]; +}; diff --git a/packages/platform-android/src/app-state.ts b/packages/platform-android/src/app-state.ts index ae01ce33d..8fb7de478 100644 --- a/packages/platform-android/src/app-state.ts +++ b/packages/platform-android/src/app-state.ts @@ -28,6 +28,34 @@ const ANDROID_FOCUS_MARKERS = [ 'ResumedActivity:', ] as const; +export type AndroidCommandExecutor = ( + args: string[], + options: { allowFailure: boolean }, +) => Promise<{ exitCode: number; stdout?: string; stderr?: string }>; + +export async function readAndroidAppStateWithExecutor( + run: AndroidCommandExecutor, +): Promise { + const windowFocus = await readAndroidFocusWithExecutor(run, FOCUS_COMMANDS); + if (windowFocus) return windowFocus; + + const activityFocus = await readAndroidFocusWithExecutor(run, ACTIVITY_COMMANDS); + if (activityFocus) return activityFocus; + return {}; +} + +async function readAndroidFocusWithExecutor( + run: AndroidCommandExecutor, + commands: readonly (readonly string[])[], +): Promise { + for (const args of commands) { + const result = await run([...args], { allowFailure: true }); + const parsed = parseAndroidForegroundApp(result.stdout ?? ''); + if (parsed) return parsed; + } + return null; +} + export async function readAndroidAppState( host: AndroidAppStateHost, device: DeviceInfo, diff --git a/packages/platform-android/src/index.ts b/packages/platform-android/src/index.ts index 8dbee8bd0..1ca8a52e1 100644 --- a/packages/platform-android/src/index.ts +++ b/packages/platform-android/src/index.ts @@ -55,11 +55,11 @@ export async function readAndroidAppState( return await read(host, device, signal); } -export async function parseAndroidForegroundApp( - text: string, -): Promise | null> { - const { parseAndroidForegroundApp: parse } = await import('./app-state.ts'); - return parse(text); +export async function readAndroidAppStateWithExecutor( + run: import('./app-state.ts').AndroidCommandExecutor, +): Promise { + const { readAndroidAppStateWithExecutor: read } = await import('./app-state.ts'); + return await read(run); } export const runtimeModule = Object.freeze({ diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 42037013e..9d7a153fc 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -75,6 +75,7 @@ const CONTRACT_EXPORTS = [ '@agent-device/contracts/audio-probe-support', '@agent-device/contracts/audio-runtime-plan', '@agent-device/contracts/back-mode', + '@agent-device/contracts/backend-diagnostics', '@agent-device/contracts/back-runtime', '@agent-device/contracts/boot-failure', '@agent-device/contracts/capture', @@ -150,6 +151,7 @@ const CONTRACT_EXPORTS = [ '@agent-device/contracts/snapshot', '@agent-device/contracts/snapshot-presentation', '@agent-device/contracts/snapshot-runtime', + '@agent-device/contracts/snapshot-scope', '@agent-device/contracts/snapshot-timeout-evidence', '@agent-device/contracts/startup-recovery-fence', '@agent-device/contracts/touch-runtime', @@ -463,6 +465,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/png-worker-client', '@agent-device/capture-kit/screenshot-density', '@agent-device/capture-kit/screenshot-diff-pixels', + '@agent-device/capture-kit/snapshot-desktop-projection', '@agent-device/capture-kit/snapshot-occlusion', '@agent-device/capture-kit/snapshot-quality-backend-capabilities', '@agent-device/capture-kit/snapshot-quality-verdict', diff --git a/src/__tests__/eager-closure-budgets.ts b/src/__tests__/eager-closure-budgets.ts index a8cc04988..e4fb35036 100644 --- a/src/__tests__/eager-closure-budgets.ts +++ b/src/__tests__/eager-closure-budgets.ts @@ -130,6 +130,7 @@ export const FACADE_BUDGETS: Readonly> = Object.freeze({ 'packages/capture-kit/src/screenshot-density.ts': 6, 'packages/capture-kit/src/screenshot-diff-pixels.ts': 1, 'packages/capture-kit/src/mobile-snapshot-semantics.ts': 10, + 'packages/capture-kit/src/snapshot-desktop-projection.ts': 2, 'packages/capture-kit/src/snapshot-occlusion.ts': 10, 'packages/capture-kit/src/snapshot-quality-backend-capabilities.ts': 1, 'packages/capture-kit/src/snapshot-quality-verdict.ts': 2, @@ -172,6 +173,7 @@ export const FACADE_BUDGETS: Readonly> = Object.freeze({ 'packages/contracts/src/audio-probe-support.ts': 5, 'packages/contracts/src/audio-runtime-plan.ts': 5, 'packages/contracts/src/back-mode.ts': 1, + 'packages/contracts/src/backend-diagnostics.ts': 1, 'packages/contracts/src/boot-failure.ts': 1, 'packages/contracts/src/click-button.ts': 3, 'packages/contracts/src/clipboard.ts': 1, @@ -239,6 +241,7 @@ export const FACADE_BUDGETS: Readonly> = Object.freeze({ 'packages/contracts/src/settings.ts': 3, 'packages/contracts/src/snapshot-presentation.ts': 2, 'packages/contracts/src/snapshot-runtime.ts': 3, + 'packages/contracts/src/snapshot-scope.ts': 1, 'packages/contracts/src/snapshot-timeout-evidence.ts': 1, 'packages/contracts/src/startup-recovery-fence.ts': 1, 'packages/contracts/src/tv-remote.ts': 3, diff --git a/src/backend.ts b/src/backend.ts index 488fbfcf4..51cedac41 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -15,7 +15,6 @@ import type { ResolvedScrollExecutionOptions } from '@agent-device/contracts/scr import type { TvRemoteButton } from '@agent-device/contracts/tv-remote'; import type { RecordingExportQuality } from '@agent-device/contracts/recording'; import type { SessionSurface } from '@agent-device/contracts/session'; -import type { NetworkIncludeMode } from '@agent-device/kernel/contracts'; import type { DeviceTarget, Platform, @@ -307,14 +306,22 @@ export type BackendTraceResult = Record & { outPath?: string; }; -export type BackendDiagnosticsTimeWindow = { - since?: string; - until?: string; -}; - -export type BackendDiagnosticsPageOptions = BackendDiagnosticsTimeWindow & { - cursor?: string; - limit?: number; +import type { + BackendDiagnosticsPageOptions, + BackendDiagnosticsTimeWindow, + BackendDumpNetworkOptions, + BackendDumpNetworkResult, + BackendNetworkEntry, + BackendNetworkIncludeMode, +} from '@agent-device/contracts/backend-diagnostics'; + +export type { + BackendDiagnosticsPageOptions, + BackendDiagnosticsTimeWindow, + BackendDumpNetworkOptions, + BackendDumpNetworkResult, + BackendNetworkEntry, + BackendNetworkIncludeMode, }; export type BackendLogEntry = { @@ -340,34 +347,6 @@ export type BackendReadLogsResult = { notes?: readonly string[]; }; -export type BackendNetworkIncludeMode = NetworkIncludeMode; - -export type BackendNetworkEntry = { - timestamp?: string; - method?: string; - url?: string; - status?: number; - durationMs?: number; - requestHeaders?: Record; - responseHeaders?: Record; - requestBody?: string; - responseBody?: string; - metadata?: Record; -}; - -export type BackendDumpNetworkOptions = BackendDiagnosticsPageOptions & { - include?: BackendNetworkIncludeMode; -}; - -export type BackendDumpNetworkResult = { - entries: readonly BackendNetworkEntry[]; - nextCursor?: string; - timeWindow?: BackendDiagnosticsTimeWindow; - backend?: string; - redacted?: boolean; - notes?: readonly string[]; -}; - export type BackendPerfMetric = { name: string; value?: number; diff --git a/src/core/interactors/linux.ts b/src/core/interactors/linux.ts index cd8eac4fa..6635740c6 100644 --- a/src/core/interactors/linux.ts +++ b/src/core/interactors/linux.ts @@ -21,7 +21,7 @@ import { } from '../../platforms/linux/input-actions.ts'; import { singlePointerPlanEndpoints } from '@agent-device/contracts/gesture-plan'; import { screenshotLinux } from '../../platforms/linux/screenshot.ts'; -import { captureLinuxSurfaceSnapshot } from '../../snapshot/snapshot-desktop-surface.ts'; +import { captureLinuxSurfaceSnapshot } from '../../platforms/linux/surface-snapshot.ts'; import type { Interactor } from '@agent-device/contracts/interactor-types'; function unsupportedLinuxAlert(): Promise { diff --git a/src/core/snapshot-state.ts b/src/core/snapshot-state.ts index b90f78fcf..5e9a0b83d 100644 --- a/src/core/snapshot-state.ts +++ b/src/core/snapshot-state.ts @@ -19,7 +19,7 @@ import { annotateSnapshotNodesCoveredByPolicy, } from '@agent-device/capture-kit/snapshot-occlusion'; import { coveredAndroidReplacementNodeIndexes } from '../snapshot/android-replacement-surface-occlusion.ts'; -import { scopeSnapshotNodes } from '../snapshot/snapshot-desktop-surface.ts'; +import { scopeSnapshotNodes } from '@agent-device/capture-kit/snapshot-desktop-projection'; import { normalizeSnapshotTree, pruneGroupNodes } from '../core/snapshot-tree-ingestion.ts'; import { presentIosInteractiveSnapshot } from '../snapshot/snapshot-presentation/ios/index.ts'; diff --git a/src/platform-runtime.ts b/src/platform-runtime.ts index fdfe090c3..ae0ea7d73 100644 --- a/src/platform-runtime.ts +++ b/src/platform-runtime.ts @@ -31,7 +31,7 @@ import { import { createAndroidObservationAdapter as createPackageAndroidObservationAdapter, createAndroidInventoryModule, - parseAndroidForegroundApp as parseAndroidPackageForegroundApp, + readAndroidAppStateWithExecutor, readAndroidAppState as readAndroidPackageAppState, loadShutdownRuntime as loadAndroidShutdownRuntime, runtimeModule as androidRuntimeModule, @@ -74,10 +74,10 @@ export async function readAndroidAppStateWithHost( return await readAndroidPackageAppState(host, device, signal); } -export async function parseAndroidForegroundApp( - text: string, -): Promise | null> { - return await parseAndroidPackageForegroundApp(text); +export async function getAndroidAppStateWithAdb( + adb: Parameters[0], +): Promise { + return await readAndroidAppStateWithExecutor(adb); } const androidInventoryModule = createAndroidInventoryModule({ diff --git a/src/platforms/android/__tests__/app-helpers.test.ts b/src/platforms/android/__tests__/app-helpers.test.ts index 7aa09bda2..54480210a 100644 --- a/src/platforms/android/__tests__/app-helpers.test.ts +++ b/src/platforms/android/__tests__/app-helpers.test.ts @@ -4,7 +4,7 @@ import path from 'node:path'; import { test } from 'vitest'; import type { AndroidAdbExecutor } from '../adb-executor.ts'; import { createDeviceAdbExecutor } from '../adb-executor.ts'; -import { getAndroidAppStateWithAdb, listAndroidAppsWithAdb } from '../app-helpers.ts'; +import { listAndroidAppsWithAdb } from '../app-helpers.ts'; import { mkdtempForTest } from '../../../__tests__/test-utils/tmp-dir.ts'; async function withMockedAdbScript(script: string, run: () => Promise): Promise { @@ -92,30 +92,9 @@ test('Android app helpers work with a local ADB provider', async () => { booted: true, }); - const [apps, state] = await Promise.all([ - listAndroidAppsWithAdb(adb, { target: 'mobile' }), - getAndroidAppStateWithAdb(adb), - ]); + const apps = await listAndroidAppsWithAdb(adb, { target: 'mobile' }); assert.deepEqual(apps, [{ package: 'com.example.app', name: 'Example' }]); - assert.deepEqual(state, { package: 'com.example.app', activity: '.MainActivity' }); }, ); }); - -test('getAndroidAppStateWithAdb parses focus output from failed commands', async () => { - const calls: string[][] = []; - const adb: AndroidAdbExecutor = async (args) => { - calls.push(args); - return { - exitCode: 1, - stdout: 'mCurrentFocus=Window{42 u0 com.example.app/.MainActivity}\n', - stderr: 'dumpsys warning', - }; - }; - - const state = await getAndroidAppStateWithAdb(adb); - - assert.deepEqual(state, { package: 'com.example.app', activity: '.MainActivity' }); - assert.deepEqual(calls, [['shell', 'dumpsys', 'window', 'windows']]); -}); diff --git a/src/platforms/android/app-helpers.ts b/src/platforms/android/app-helpers.ts index c42f9c513..845bb2056 100644 --- a/src/platforms/android/app-helpers.ts +++ b/src/platforms/android/app-helpers.ts @@ -1,5 +1,4 @@ import { resolveAppsFilter, type AppsFilter } from '@agent-device/contracts/device'; -import type { AppStateRuntimeResult } from '@agent-device/contracts/app-state-runtime'; import { androidAdbResultError, type AndroidAdbExecutor } from './adb-executor.ts'; import { parseAndroidLaunchablePackages, @@ -33,23 +32,6 @@ export async function listAndroidAppsWithAdb( .sort((a, b) => a.package.localeCompare(b.package)); } -export async function getAndroidAppStateWithAdb( - adb: AndroidAdbExecutor, -): Promise { - const windowFocus = await readAndroidFocusWithAdb(adb, [ - ['shell', 'dumpsys', 'window', 'windows'], - ['shell', 'dumpsys', 'window'], - ]); - if (windowFocus) return windowFocus; - - const activityFocus = await readAndroidFocusWithAdb(adb, [ - ['shell', 'dumpsys', 'activity', 'activities'], - ['shell', 'dumpsys', 'activity'], - ]); - if (activityFocus) return activityFocus; - return {}; -} - async function listAndroidLaunchablePackagesWithAdb( adb: AndroidAdbExecutor, target: AndroidAppListTarget, @@ -103,16 +85,3 @@ async function listAndroidUserInstalledPackagesWithAdb(adb: AndroidAdbExecutor): } return parseAndroidUserInstalledPackages(result.stdout); } - -async function readAndroidFocusWithAdb( - adb: AndroidAdbExecutor, - commands: string[][], -): Promise { - const { parseAndroidForegroundApp } = await import('../../platform-runtime.ts'); - for (const args of commands) { - const result = await adb(args, { allowFailure: true }); - const parsed = await parseAndroidForegroundApp(result.stdout ?? ''); - if (parsed) return parsed; - } - return null; -} diff --git a/src/platforms/apple/interactor.ts b/src/platforms/apple/interactor.ts index 5e19b22ae..5f9593343 100644 --- a/src/platforms/apple/interactor.ts +++ b/src/platforms/apple/interactor.ts @@ -36,7 +36,7 @@ import type { SnapshotOptions, } from '@agent-device/contracts/interactor-types'; import { readSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; -import { captureMacOsSurfaceSnapshot } from '../../snapshot/snapshot-desktop-surface.ts'; +import { captureMacOsSurfaceSnapshot } from './os/macos/surface-snapshot.ts'; export function createAppleInteractor( device: DeviceInfo, diff --git a/src/platforms/apple/os/macos/surface-snapshot.ts b/src/platforms/apple/os/macos/surface-snapshot.ts new file mode 100644 index 000000000..47d0899eb --- /dev/null +++ b/src/platforms/apple/os/macos/surface-snapshot.ts @@ -0,0 +1,20 @@ +import type { CaptureSnapshotInput } from '@agent-device/contracts/snapshot-runtime'; +import { shapeDesktopSurfaceSnapshot } from '@agent-device/capture-kit/snapshot-desktop-projection'; + +type SnapshotSurfaceOptions = NonNullable; + +export async function captureMacOsSurfaceSnapshot( + options: SnapshotSurfaceOptions, + signal?: AbortSignal, +) { + const surface = options.surface; + if (!surface || surface === 'app') { + throw new TypeError('Apple surface capture requires a non-app macOS surface'); + } + const { runMacOsSnapshotAction } = await import('./helper.ts'); + const result = await runMacOsSnapshotAction(surface, { + bundleId: surface === 'menubar' ? options.appBundleId : undefined, + signal, + }); + return shapeDesktopSurfaceSnapshot({ ...result, producer: 'macos-helper' }, options); +} diff --git a/src/platforms/linux/surface-snapshot.ts b/src/platforms/linux/surface-snapshot.ts new file mode 100644 index 000000000..41d9d1d19 --- /dev/null +++ b/src/platforms/linux/surface-snapshot.ts @@ -0,0 +1,19 @@ +import type { CaptureSnapshotInput } from '@agent-device/contracts/snapshot-runtime'; +import { shapeDesktopSurfaceSnapshot } from '@agent-device/capture-kit/snapshot-desktop-projection'; + +export async function captureLinuxSurfaceSnapshot( + options: CaptureSnapshotInput['options'], + signal?: AbortSignal, +) { + const { snapshotLinux } = await import('./snapshot.ts'); + const result = await snapshotLinux(options?.surface, signal); + return shapeDesktopSurfaceSnapshot( + { + nodes: result.nodes, + truncated: result.truncated, + backend: 'linux-atspi', + producer: 'linux-atspi', + }, + options ?? {}, + ); +} diff --git a/src/platforms/web/agent-browser-network.ts b/src/platforms/web/agent-browser-network.ts index 7991c3d0a..e0257d15f 100644 --- a/src/platforms/web/agent-browser-network.ts +++ b/src/platforms/web/agent-browser-network.ts @@ -2,7 +2,7 @@ import type { BackendDumpNetworkOptions, BackendDumpNetworkResult, BackendNetworkEntry, -} from '../../backend.ts'; +} from '@agent-device/contracts/backend-diagnostics'; import { stripUndefined } from '@agent-device/kernel/record'; import { isJsonObject, readNumberProperty, readStringProperty } from './json-utils.ts'; diff --git a/src/platforms/web/provider.ts b/src/platforms/web/provider.ts index 43a1d19f6..1618a5015 100644 --- a/src/platforms/web/provider.ts +++ b/src/platforms/web/provider.ts @@ -2,7 +2,10 @@ import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture'; import type { SessionSurface } from '@agent-device/contracts/session'; import { createScopedProvider } from '@agent-device/kernel/scoped-provider'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import type { BackendDumpNetworkOptions, BackendDumpNetworkResult } from '../../backend.ts'; +import type { + BackendDumpNetworkOptions, + BackendDumpNetworkResult, +} from '@agent-device/contracts/backend-diagnostics'; import type { AudioProbeResult } from '@agent-device/contracts/audio-probe-result'; import { createAgentBrowserWebProvider } from './agent-browser-provider.ts'; diff --git a/src/sdk/android-adb.test.ts b/src/sdk/android-adb.test.ts new file mode 100644 index 000000000..503fb155e --- /dev/null +++ b/src/sdk/android-adb.test.ts @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { AndroidAdbExecutor } from '../platforms/android/adb-executor.ts'; +import { getAndroidAppStateWithAdb } from './android-adb.ts'; + +test('getAndroidAppStateWithAdb parses focus output from failed commands', async () => { + const calls: string[][] = []; + const adb: AndroidAdbExecutor = async (args) => { + calls.push(args); + return { + exitCode: 1, + stdout: 'mCurrentFocus=Window{42 u0 com.example.app/.MainActivity}\n', + stderr: 'dumpsys warning', + }; + }; + + const state = await getAndroidAppStateWithAdb(adb); + + assert.deepEqual(state, { package: 'com.example.app', activity: '.MainActivity' }); + assert.deepEqual(calls, [['shell', 'dumpsys', 'window', 'windows']]); +}); + +test('getAndroidAppStateWithAdb falls back to activity focus and settles empty', async () => { + const calls: string[][] = []; + const adb: AndroidAdbExecutor = async (args) => { + calls.push(args); + return { exitCode: 0, stdout: '', stderr: '' }; + }; + + const state = await getAndroidAppStateWithAdb(adb); + + assert.deepEqual(state, {}); + assert.deepEqual(calls, [ + ['shell', 'dumpsys', 'window', 'windows'], + ['shell', 'dumpsys', 'window'], + ['shell', 'dumpsys', 'activity', 'activities'], + ['shell', 'dumpsys', 'activity'], + ]); +}); diff --git a/src/sdk/android-adb.ts b/src/sdk/android-adb.ts index 74490b4ca..12a0f87c3 100644 --- a/src/sdk/android-adb.ts +++ b/src/sdk/android-adb.ts @@ -5,10 +5,18 @@ export { type AndroidAdbProvider, type AndroidPortReverseEndpoint, } from '../platforms/android/adb-executor.ts'; -export { - getAndroidAppStateWithAdb, - listAndroidAppsWithAdb, -} from '../platforms/android/app-helpers.ts'; +export { listAndroidAppsWithAdb } from '../platforms/android/app-helpers.ts'; + +import type { AndroidAdbExecutor } from '../platforms/android/adb-executor.ts'; +import type { AppStateRuntimeResult } from '@agent-device/contracts/app-state-runtime'; + +export async function getAndroidAppStateWithAdb( + adb: AndroidAdbExecutor, +): Promise { + const { getAndroidAppStateWithAdb: read } = await import('../platform-runtime.ts'); + return await read(adb); +} + export { forceStopAndroidAppWithAdb, openAndroidAppWithAdb, diff --git a/src/sdk/limrun-runtime-dependencies.test.ts b/src/sdk/limrun-runtime-dependencies.test.ts index 4ca6e58eb..d6d8d64a9 100644 --- a/src/sdk/limrun-runtime-dependencies.test.ts +++ b/src/sdk/limrun-runtime-dependencies.test.ts @@ -15,10 +15,6 @@ vi.mock('@limrun/api', () => ({ vi.mock('../platforms/android/app-helpers.ts', () => { moduleLoads.androidAppHelpers += 1; return { - getAndroidAppStateWithAdb: vi.fn(async () => ({ - package: 'com.example.app', - activity: '.MainActivity', - })), listAndroidAppsWithAdb: vi.fn(async () => [{ package: 'com.example.app', name: 'Example' }]), }; }); diff --git a/src/snapshot/snapshot-desktop-surface.test.ts b/src/snapshot/snapshot-desktop-surface.test.ts index b8418a370..d80893042 100644 --- a/src/snapshot/snapshot-desktop-surface.test.ts +++ b/src/snapshot/snapshot-desktop-surface.test.ts @@ -1,6 +1,4 @@ import { beforeEach, expect, test, vi } from 'vitest'; -import fs from 'node:fs'; -import path from 'node:path'; import type { DeviceInfo } from '@agent-device/kernel/device'; const { runMacOsSnapshotAction, snapshotLinux } = vi.hoisted(() => ({ @@ -11,7 +9,7 @@ const { runMacOsSnapshotAction, snapshotLinux } = vi.hoisted(() => ({ vi.mock('../platforms/apple/os/macos/helper.ts', () => ({ runMacOsSnapshotAction })); vi.mock('../platforms/linux/snapshot.ts', () => ({ snapshotLinux })); -import { createSnapshotRuntimeHost, scopeSnapshotNodes } from './snapshot-desktop-surface.ts'; +import { createSnapshotRuntimeHost } from './snapshot-desktop-surface.ts'; const macosDevice = { id: 'desktop', @@ -94,45 +92,3 @@ test('Linux snapshot host preserves interactive ancestor projection before depth ], }); }); - -// Golden scope-policy leg (#1832 C2): the post-wire pass must agree with -// contracts/fixtures/snapshot-scope-policy.json — the same table the Android projection and the -// shared predicate are asserted against. Fixture position rides in `rect.x` so the check is -// text-independent. -test('scopeSnapshotNodes agrees with every golden scope-policy table case', () => { - const cases = JSON.parse( - fs.readFileSync( - path.resolve(import.meta.dirname, '../../contracts/fixtures/snapshot-scope-policy.json'), - 'utf8', - ), - ) as Array<{ - name: string; - scope: string; - nodes: Array<{ - depth: number; - label?: string; - value?: string; - identifier?: string; - presented?: boolean; - }>; - expectedSubtreeIndexes: number[]; - }>; - expect(cases.length).toBeGreaterThan(0); - for (const fixture of cases) { - const parents: number[] = []; - const nodes = fixture.nodes.map((node, index) => { - parents.length = node.depth; - const parentIndex = node.depth > 0 ? parents[node.depth - 1] : undefined; - parents[node.depth] = index; - return { ...node, index, parentIndex, rect: { x: index, y: 0, width: 1, height: 1 } }; - }); - const scoped = scopeSnapshotNodes(nodes, fixture.scope, (range) => - nodes.slice(range.start, range.end).some((node) => node.presented !== false), - ); - expect( - scoped.map((node) => node.rect?.x), - fixture.name, - ).toEqual(fixture.expectedSubtreeIndexes); - if (scoped.length > 0) expect(scoped[0]?.depth, fixture.name).toBe(0); - } -}); diff --git a/src/snapshot/snapshot-desktop-surface.ts b/src/snapshot/snapshot-desktop-surface.ts index 57628d1d7..7d1cc6b16 100644 --- a/src/snapshot/snapshot-desktop-surface.ts +++ b/src/snapshot/snapshot-desktop-surface.ts @@ -1,60 +1,18 @@ -import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import type { SnapshotOptions, SnapshotResult } from '@agent-device/contracts/interactor-types'; -import type { - CaptureSnapshotInput, - SnapshotRuntimeHost, -} from '@agent-device/contracts/snapshot-runtime'; +import type { SnapshotRuntimeHost } from '@agent-device/contracts/snapshot-runtime'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { - findSnapshotScopeRange, - normalizeSnapshotScope, - reindexSnapshotNodes, -} from '@agent-device/contracts/snapshot'; - -type SnapshotSurfaceOptions = NonNullable; export function createSnapshotRuntimeHost(): SnapshotRuntimeHost { return Object.freeze({ captureSurface }); } -export async function captureLinuxSurfaceSnapshot( - options: CaptureSnapshotInput['options'], - signal?: AbortSignal, -) { - const { snapshotLinux } = await import('../platforms/linux/snapshot.ts'); - const result = await snapshotLinux(options?.surface, signal); - return shapeDesktopSurfaceSnapshot( - { - nodes: result.nodes, - truncated: result.truncated, - backend: 'linux-atspi', - producer: 'linux-atspi', - }, - options ?? {}, - ); -} - -export async function captureMacOsSurfaceSnapshot( - options: SnapshotSurfaceOptions, - signal?: AbortSignal, -) { - const surface = options.surface; - if (!surface || surface === 'app') { - throw new TypeError('Apple surface capture requires a non-app macOS surface'); - } - const { runMacOsSnapshotAction } = await import('../platforms/apple/os/macos/helper.ts'); - const result = await runMacOsSnapshotAction(surface, { - bundleId: surface === 'menubar' ? options.appBundleId : undefined, - signal, - }); - return shapeDesktopSurfaceSnapshot({ ...result, producer: 'macos-helper' }, options); -} - const captureSurface: SnapshotRuntimeHost['captureSurface'] = async (device, options, signal) => { if (device.platform === 'linux') { + const { captureLinuxSurfaceSnapshot } = await import('../platforms/linux/surface-snapshot.ts'); return await captureLinuxSurfaceSnapshot(options, signal); } requireMacOsSurfaceDevice(device); + const { captureMacOsSurfaceSnapshot } = + await import('../platforms/apple/os/macos/surface-snapshot.ts'); return await captureMacOsSurfaceSnapshot(options ?? {}, signal); }; @@ -63,73 +21,3 @@ function requireMacOsSurfaceDevice(device: DeviceInfo): void { throw new TypeError('Apple surface capture requires a non-app macOS surface'); } } - -const INTERACTIVE_ROLE_TOKENS = [ - 'button', - 'menu', - 'textfield', - 'searchfield', - 'checkbox', - 'radio', - 'switch', -] as const; - -/** Applies the legacy desktop-surface projection once for both runtime hosts and legacy capture. */ -function shapeDesktopSurfaceSnapshot( - data: SnapshotResult, - options: Pick, -): SnapshotResult { - let nodes = data.nodes ?? []; - if (options.scope) { - nodes = scopeSnapshotNodes(nodes, options.scope, (range) => - options.interactiveOnly - ? nodes.slice(range.start, range.end).some(isInteractiveSnapshotNode) - : true, - ); - } - if (options.interactiveOnly) nodes = filterInteractiveSnapshotNodes(nodes); - if (typeof options.depth === 'number') nodes = filterSnapshotNodesByDepth(nodes, options.depth); - return { ...data, nodes }; -} - -/** The shared scope specification applied post-wire (`@agent-device/contracts/snapshot`). */ -export function scopeSnapshotNodes( - nodes: RawSnapshotNode[], - scope: string, - subtreeContributes?: (range: { start: number; end: number }) => boolean, -): RawSnapshotNode[] { - const normalizedScope = normalizeSnapshotScope(scope); - if (!normalizedScope) return reindexSnapshotNodes(nodes); - const range = findSnapshotScopeRange(nodes, normalizedScope, subtreeContributes); - if (!range) return []; - const slice = nodes.slice(range.start, range.end); - return reindexSnapshotNodes(slice, slice[0]?.depth ?? 0); -} - -function filterInteractiveSnapshotNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] { - if (nodes.length === 0) return nodes; - const byIndex = new Map(nodes.map((node) => [node.index, node])); - const keepIndexes = new Set(); - for (const node of nodes) { - if (!isInteractiveSnapshotNode(node)) continue; - let current: RawSnapshotNode | undefined = node; - while (current) { - if (keepIndexes.has(current.index)) break; - keepIndexes.add(current.index); - current = - typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined; - } - } - if (keepIndexes.size === 0) return nodes; - return reindexSnapshotNodes(nodes.filter((node) => keepIndexes.has(node.index))); -} - -function filterSnapshotNodesByDepth(nodes: RawSnapshotNode[], maxDepth: number): RawSnapshotNode[] { - return reindexSnapshotNodes(nodes.filter((node) => (node.depth ?? 0) <= maxDepth)); -} - -function isInteractiveSnapshotNode(node: RawSnapshotNode): boolean { - if ([node.focused, node.hittable, node.rect].some(Boolean)) return true; - const role = `${node.type ?? ''} ${node.role ?? ''} ${node.subrole ?? ''}`.toLowerCase(); - return INTERACTIVE_ROLE_TOKENS.some((token) => role.includes(token)); -}