From ea3dbea24f2f71f631226c0ced937f7ab37cb505 Mon Sep 17 00:00:00 2001 From: Muvaffak Onus Date: Mon, 2 Nov 2026 18:33:20 +0300 Subject: [PATCH] fix(daemon): retain Apple runners for leases kept on close --- .../__tests__/session-store-lifetime.test.ts | 19 ++ .../src/core/platform-resource-cleanup.ts | 3 +- packages/daemon/src/core/session-store.ts | 38 ++- ...-execution-scope-lease-session-end.test.ts | 256 +++++++++++++++++- .../http/daemon-runtime-lease-expiry.test.ts | 6 + .../daemon/src/server/http/daemon-runtime.ts | 9 +- .../src/server/request-execution-scope.ts | 92 +++++-- .../daemon/src/sessions/lease-lifecycle.ts | 13 + .../session-close-lifecycle-teardown.test.ts | 164 ++++++++++- .../session-close-lifecycle-teardown.ts | 17 +- .../internal/session-close.ts | 30 +- src/__tests__/cli-help.test.ts | 2 +- src/commands/schema/cli-help.ts | 2 +- ...runtime-resource-cleanup-retention.test.ts | 4 +- src/platform-runtime-resource-cleanup.ts | 2 +- website/docs/docs/client-api.md | 2 +- 16 files changed, 605 insertions(+), 54 deletions(-) diff --git a/packages/daemon/src/core/__tests__/session-store-lifetime.test.ts b/packages/daemon/src/core/__tests__/session-store-lifetime.test.ts index 61d9bf852b..c0332342e5 100644 --- a/packages/daemon/src/core/__tests__/session-store-lifetime.test.ts +++ b/packages/daemon/src/core/__tests__/session-store-lifetime.test.ts @@ -78,6 +78,25 @@ test('address reuse with the same record still starts a different lifetime', () assert.equal(store.getRuntimeHints(ADDRESS), undefined); }); +test('retained execution-host ownership follows the current session lifetime', () => { + const store = makeSessionStore(); + const session = makeSession('default', { + lease: { leaseId: 'lease-1', tenantId: 'tenant-a', runId: 'run-1' }, + }); + const old = store.publish(ADDRESS, session); + store.retire(old, { retainExecutionHost: true }); + const [retained] = store.listRetainedExecutionHosts('lease-1'); + assert.ok(retained); + assert.equal(store.ownsRetainedExecutionHost(retained), true); + const successor = store.publish(ADDRESS, session); + assert.equal(store.ownsRetainedExecutionHost(retained), false); + assert.equal(store.retire(old, { retainExecutionHost: true }), false); + assert.deepEqual(store.listRetainedExecutionHosts('lease-1'), []); + store.retire(successor, { retainExecutionHost: true }); + assert.equal(store.releaseRetainedExecutionHost(retained), false); + assert.equal(store.listRetainedExecutionHosts('lease-1').length, 1); +}); + test('retired updates cannot run their derivation or resurrect a record', () => { const store = makeSessionStore(); const ref = store.publish(ADDRESS, makeSession('default')); diff --git a/packages/daemon/src/core/platform-resource-cleanup.ts b/packages/daemon/src/core/platform-resource-cleanup.ts index ca623b0535..d2632ceb6a 100644 --- a/packages/daemon/src/core/platform-resource-cleanup.ts +++ b/packages/daemon/src/core/platform-resource-cleanup.ts @@ -16,6 +16,7 @@ export type PlatformResourceCleanup = Readonly<{ device: DeviceInfo; shutdownRequested: boolean; hasScreenRecording: boolean; - hasLease: boolean; + /** True when the lease is not kept on close, or when its close decision could not be inspected. */ + leaseWillEnd: boolean; }): boolean; }>; diff --git a/packages/daemon/src/core/session-store.ts b/packages/daemon/src/core/session-store.ts index 7536405cd7..e3f40e0a71 100644 --- a/packages/daemon/src/core/session-store.ts +++ b/packages/daemon/src/core/session-store.ts @@ -49,9 +49,15 @@ import { const REPAIR_TOMBSTONE_TTL_MS = 60 * 60_000; type SessionEntry = { current: SessionState }; type SessionPatch = Partial | ((current: SessionState) => Partial); +export type RetainedExecutionHost = Readonly<{ + address: string; + device: SessionState['device']; + lease: NonNullable; +}>; export class SessionStore { private readonly sessions = new Map(); + private readonly retainedExecutionHosts = new Map(); private acceptingSessions = true; private readonly runtimeHints = new Map(); private readonly sessionsDir: string; @@ -105,6 +111,11 @@ export class SessionStore { this.assertPublishable(address); const entry = { current: session }; this.sessions.set(address, entry); + for (const retained of this.retainedExecutionHosts.values()) { + if (devicesOverlap(retained.device.id, session.device.id)) { + this.releaseRetainedExecutionHost(retained); + } + } this.clearIdleExpiryTombstone(address); return this.captureRef(address, entry); } @@ -141,12 +152,35 @@ export class SessionStore { return next; } - retire(ref: SessionRef): boolean { - if (!this.resolveCurrent(ref)) return false; + retire(ref: SessionRef, options: { retainExecutionHost?: boolean } = {}): boolean { + const session = this.resolveCurrent(ref); + if (!session) return false; + if (options.retainExecutionHost && session.lease) { + this.retainedExecutionHosts.set(session.device.id, { + address: ref.address, + device: session.device, + lease: { ...session.lease }, + }); + } this.runtimeHints.delete(ref.address); return this.sessions.delete(ref.address); } + listRetainedExecutionHosts(leaseId: string): RetainedExecutionHost[] { + return Array.from(this.retainedExecutionHosts.values()).filter( + (host) => host.lease.leaseId === leaseId, + ); + } + + ownsRetainedExecutionHost(host: RetainedExecutionHost): boolean { + return this.retainedExecutionHosts.get(host.device.id) === host; + } + + releaseRetainedExecutionHost(host: RetainedExecutionHost): boolean { + if (!this.ownsRetainedExecutionHost(host)) return false; + return this.retainedExecutionHosts.delete(host.device.id); + } + private captureRef(address: string, entry: SessionEntry): SessionRef { return Object.freeze({ address, session: entry.current, lifetime: entry }); } diff --git a/packages/daemon/src/server/__tests__/request-execution-scope-lease-session-end.test.ts b/packages/daemon/src/server/__tests__/request-execution-scope-lease-session-end.test.ts index ece6abafbd..8f681c0a8e 100644 --- a/packages/daemon/src/server/__tests__/request-execution-scope-lease-session-end.test.ts +++ b/packages/daemon/src/server/__tests__/request-execution-scope-lease-session-end.test.ts @@ -16,6 +16,7 @@ import { acquireDeviceClaim } from '@agent-device/daemon-contracts/device-claims import { existingSessionExecutionLockKeys } from '../request-binding.ts'; import { withRequestExecutionLockKeys } from '../request-execution-locks.ts'; import type { SessionStore } from '../../core/session-store.ts'; +import type { PlatformResourceCleanup } from '../../core/platform-resource-cleanup.ts'; import type { SessionTeardown } from '../../sessions/lease-lifecycle.ts'; import { createLeaseSessionEnd } from '../request-execution-scope.ts'; @@ -67,7 +68,13 @@ function holdSessionLocks( return { finish, held }; } -function leaseSessionEnd(sessionStore: SessionStore, teardownSession: SessionTeardown) { +function leaseSessionEnd( + sessionStore: SessionStore, + teardownSession: SessionTeardown, + cleanupSessionlessExecutionHost: PlatformResourceCleanup['cleanupSessionlessExecutionHost'] = vi.fn( + async () => {}, + ), +) { const executionLocks = new Map>(); return { executionLocks, @@ -76,12 +83,259 @@ function leaseSessionEnd(sessionStore: SessionStore, teardownSession: SessionTea sessionStore, executionLocks, teardownSession, + cleanupSessionlessExecutionHost, }), }; } const settle = async (): Promise => await new Promise((resolve) => setImmediate(resolve)); +function retainExecutionHost(sessionStore: SessionStore, name = 'leased', leaseId = 'lease-1') { + const ref = sessionStore.publish( + name, + makeIosSession(name, { + lease: { leaseId, tenantId: 'tenant-a', runId: 'run-1' }, + }), + ); + sessionStore.retire(ref, { retainExecutionHost: true }); + return ref; +} + +test.each(['LEASE_RELEASED', 'LEASE_EXPIRED'] as const)( + '%s stops a retained host after its session has retired', + async (reason) => { + const sessionStore = makeSessionStore(); + const ref = retainExecutionHost(sessionStore); + const cleanup = vi.fn(async () => {}); + const teardown = vi.fn(async () => {}); + const { ender } = leaseSessionEnd(sessionStore, teardown, cleanup); + + await ender.end('lease-1', reason); + + expect(teardown).not.toHaveBeenCalled(); + expect(cleanup).toHaveBeenCalledExactlyOnceWith(ref.session.device); + expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]); + }, +); + +test('expiry waiting behind close cleans the host that close retains before retiring', async () => { + const sessionStore = makeSessionStore(); + const ref = sessionStore.publish( + 'leased', + makeIosSession('leased', { + lease: { leaseId: 'lease-1', tenantId: 'tenant-a', runId: 'run-1' }, + }), + ); + const cleanup = vi.fn(async () => {}); + const teardown = vi.fn(async () => {}); + const { executionLocks, ender } = leaseSessionEnd(sessionStore, teardown, cleanup); + const close = holdSessionLocks(executionLocks, sessionStore, 'leased'); + const ending = ender.end('lease-1', 'LEASE_EXPIRED'); + await settle(); + expect(cleanup).not.toHaveBeenCalled(); + + sessionStore.retire(ref, { retainExecutionHost: true }); + close.finish(); + await close.held; + await ending; + + expect(teardown).not.toHaveBeenCalled(); + expect(cleanup).toHaveBeenCalledExactlyOnceWith(ref.session.device); + expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]); +}); + +test.each([false, true])( + 'lease end preserves a replacement owner with retained=%s', + async (retained) => { + const sessionStore = makeSessionStore(); + const old = retainExecutionHost(sessionStore); + const cleanup = vi.fn(async () => {}); + const { executionLocks, ender } = leaseSessionEnd(sessionStore, async () => {}, cleanup); + let unlock!: () => void; + const held = withRequestExecutionLockKeys( + executionLocks, + existingSessionExecutionLockKeys(old.address, old.session.device.id), + async () => + await new Promise((resolve) => { + unlock = resolve; + }), + ); + await settle(); + const ending = ender.end('lease-1', 'LEASE_RELEASED'); + await settle(); + const replacement = sessionStore.publish( + 'replacement', + makeIosSession('replacement', { + lease: { leaseId: 'lease-2', tenantId: 'tenant-b', runId: 'run-2' }, + }), + ); + if (retained) sessionStore.retire(replacement, { retainExecutionHost: true }); + unlock(); + await held; + await ending; + + expect(cleanup).not.toHaveBeenCalled(); + expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]); + if (retained) expect(sessionStore.listRetainedExecutionHosts('lease-2')).toHaveLength(1); + else expect(sessionStore.resolveCurrent(replacement)).toBeDefined(); + }, +); + +test('release ends a same-lease reopen published at a new address while cleanup waits for its device', async () => { + const sessionStore = makeSessionStore(); + const old = retainExecutionHost(sessionStore); + const cleanup = vi.fn(async () => {}); + const teardown = vi.fn(async () => {}); + const { executionLocks, ender } = leaseSessionEnd(sessionStore, teardown, cleanup); + let finishNativeOpen!: () => void; + const nativeOpen = new Promise((resolve) => { + finishNativeOpen = resolve; + }); + const reopening = withRequestExecutionLockKeys( + executionLocks, + existingSessionExecutionLockKeys('reopened', old.session.device.id), + async () => { + await nativeOpen; + sessionStore.publish('reopened', makeIosSession('reopened', { lease: old.session.lease })); + }, + ); + await settle(); + const ending = ender.end('lease-1', 'LEASE_RELEASED'); + await settle(); + expect(teardown).not.toHaveBeenCalled(); + finishNativeOpen(); + await reopening; + await ending; + + expect(cleanup).not.toHaveBeenCalled(); + expect(teardown).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ address: 'reopened' }), + ); + expect(sessionStore.get('reopened')).toBeUndefined(); + expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]); +}); + +test('failed retained-host cleanup stays owned and throttles retries without another release request', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(0); + try { + const sessionStore = makeSessionStore(); + retainExecutionHost(sessionStore); + const cleanup = vi + .fn(async () => {}) + .mockRejectedValueOnce(new Error('device unreachable')) + .mockRejectedValueOnce(new Error('device still unreachable')); + const { ender } = leaseSessionEnd(sessionStore, async () => {}, cleanup); + + await ender.end('lease-1', 'LEASE_RELEASED'); + expect(sessionStore.listRetainedExecutionHosts('lease-1')).toHaveLength(1); + for (let elapsed = 1_000; elapsed < 10_000; elapsed += 1_000) { + now.mockReturnValue(elapsed); + await ender.retryPending(); + expect(cleanup).toHaveBeenCalledTimes(1); + } + now.mockReturnValue(10_000); + await ender.retryPending(); + expect(cleanup).toHaveBeenCalledTimes(2); + expect(sessionStore.listRetainedExecutionHosts('lease-1')).toHaveLength(1); + now.mockReturnValue(19_999); + await ender.retryPending(); + expect(cleanup).toHaveBeenCalledTimes(2); + now.mockReturnValue(20_000); + await ender.retryPending(); + expect(cleanup).toHaveBeenCalledTimes(3); + expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]); + now.mockReturnValue(30_000); + await ender.retryPending(); + expect(cleanup).toHaveBeenCalledTimes(3); + } finally { + now.mockRestore(); + } +}); + +test('cancel preserves a retained host queued for cleanup for daemon shutdown', async () => { + const sessionStore = makeSessionStore(); + const ref = retainExecutionHost(sessionStore); + const cleanup = vi.fn(async () => {}); + const { executionLocks, ender } = leaseSessionEnd(sessionStore, async () => {}, cleanup); + let unlock!: () => void; + const held = withRequestExecutionLockKeys( + executionLocks, + existingSessionExecutionLockKeys(ref.address, ref.session.device.id), + async () => + await new Promise((resolve) => { + unlock = resolve; + }), + ); + await settle(); + const ending = ender.end('lease-1', 'LEASE_EXPIRED'); + await settle(); + ender.cancel(); + unlock(); + await held; + await ending; + await ender.drain(); + await ender.retryPending(); + + expect(cleanup).not.toHaveBeenCalled(); + expect(sessionStore.listRetainedExecutionHosts('lease-1')).toHaveLength(1); +}); + +test('retry does not queue duplicate cleanup and shutdown drains a running retained-host stop', async () => { + const sessionStore = makeSessionStore(); + retainExecutionHost(sessionStore); + let finish!: () => void; + const cleanup = vi.fn( + async () => + await new Promise((resolve) => { + finish = resolve; + }), + ); + const { ender } = leaseSessionEnd(sessionStore, async () => {}, cleanup); + const ending = ender.end('lease-1', 'LEASE_EXPIRED'); + await settle(); + await ender.retryPending(); + expect(cleanup).toHaveBeenCalledTimes(1); + ender.cancel(); + let drained = false; + const drain = ender.drain().then(() => { + drained = true; + }); + await settle(); + expect(drained).toBe(false); + finish(); + await ending; + await drain; + expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]); +}); + +test('a request holding the device lock can end its lease ahead of a queued background end', async () => { + const sessionStore = makeSessionStore(); + const ref = retainExecutionHost(sessionStore); + const cleanup = vi.fn(async () => {}); + const { executionLocks, ender } = leaseSessionEnd(sessionStore, async () => {}, cleanup); + let release!: () => void; + const continueRequest = new Promise((resolve) => { + release = resolve; + }); + const request = withRequestExecutionLockKeys( + executionLocks, + existingSessionExecutionLockKeys(ref.address, ref.session.device.id), + async () => { + await continueRequest; + await ender.end('lease-1', 'LEASE_RELEASED'); + }, + ); + await settle(); + const expiry = ender.end('lease-1', 'LEASE_EXPIRED'); + await settle(); + release(); + await request; + await expiry; + + expect(cleanup).toHaveBeenCalledTimes(1); + expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]); +}); + test('end publishes the recorded script of the session it ends', async () => { const sessionStore = makeSessionStore('agent-device-lease-end-script-'); await publishLeasedSession(sessionStore, 'leased', 'lease-1'); diff --git a/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts b/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts index 5f224b66cb..2267894580 100644 --- a/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts @@ -6,6 +6,7 @@ import type { DeviceLease } from '@agent-device/contracts/device'; const leaseProbe = vi.hoisted(() => ({ registries: [] as import('../../resources/lease-registry.ts').LeaseRegistry[], ended: [] as Array<[string, string]>, + retryCalls: 0, shutdownSteps: [] as string[], })); vi.mock('../../resources/lease-registry.ts', async (importOriginal) => { @@ -26,6 +27,9 @@ vi.mock('../request-execution-scope.ts', async (importOriginal) => { end: async (leaseId: string, reason: string) => { leaseProbe.ended.push([leaseId, reason]); }, + retryPending: async () => { + leaseProbe.retryCalls += 1; + }, cancel: () => { leaseProbe.shutdownSteps.push('cancel'); }, @@ -46,6 +50,7 @@ afterEach(() => { vi.restoreAllMocks(); leaseProbe.registries.length = 0; leaseProbe.ended.length = 0; + leaseProbe.retryCalls = 0; leaseProbe.shutdownSteps.length = 0; }); @@ -76,6 +81,7 @@ test('an expired lease ends its session without any further request', async () = await vi.advanceTimersByTimeAsync(1_000); assert.deepEqual(leaseProbe.ended, [[lease.leaseId, 'LEASE_EXPIRED']]); + assert.equal(leaseProbe.retryCalls, 1); const shutdown = runtime?.close(); await vi.runOnlyPendingTimersAsync(); diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index ed14188bf8..ff57c9b120 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -300,10 +300,10 @@ async function startDaemonRuntimeOnHost( }); // The registry only notices an expired lease when something calls it; this sweep ends an // abandoned lease (and, through `onLeaseExpired`, its session and claim) without a later request. - const leaseExpirySweep = setInterval( - () => leaseRegistry.consumeExpiredLeases(), - LEASE_EXPIRY_SWEEP_INTERVAL_MS, - ); + const leaseExpirySweep = setInterval(() => { + leaseRegistry.consumeExpiredLeases(); + void leaseSessionEnd.retryPending(); + }, LEASE_EXPIRY_SWEEP_INTERVAL_MS); leaseExpirySweep.unref(); rollback.onRollback(() => clearInterval(leaseExpirySweep)); const dispatchRequest = createRequestHandler({ @@ -420,6 +420,7 @@ async function startDaemonRuntimeOnHost( sessionStore, executionLocks: getLeaseRegistryExecutionLocks(leaseRegistry), teardownSession: settleIdleExpiredSession, + cleanupSessionlessExecutionHost: deps.platformResourceCleanup.cleanupSessionlessExecutionHost, withinDiagnosticsScope: async (run) => await withDaemonDiagnosticsScope({ logPath }, run), }); diff --git a/packages/daemon/src/server/request-execution-scope.ts b/packages/daemon/src/server/request-execution-scope.ts index 86f758e327..d652021267 100644 --- a/packages/daemon/src/server/request-execution-scope.ts +++ b/packages/daemon/src/server/request-execution-scope.ts @@ -806,10 +806,12 @@ export function getLeaseRegistryExecutionLocks( export type LeaseSessionEnd = Readonly<{ /** - * Ends every session opened under `leaseId` and clears its device claim, each inside that - * session's execution lock pair so a command still running on it finishes first. Never rejects. + * Ends sessions and retained execution hosts owned by `leaseId` under their execution locks. + * Clears live sessions' device claims after teardown. Never rejects. */ end: (leaseId: string, reason: LeaseEndReason) => Promise; + /** Retries retained-host cleanup that failed after the registry removed its lease. */ + retryPending: () => Promise; /** * Called when the daemon begins to leave. Shutdown tears every session down without taking an * execution lock, so no lease end may start a teardown after this, including one already queued @@ -820,18 +822,22 @@ export type LeaseSessionEnd = Readonly<{ drain: () => Promise; }>; +const LEASE_END_RETRY_INTERVAL_MS = 10_000; + export function createLeaseSessionEnd(params: { env: HostEnvironment['env']; sessionStore: SessionStore; executionLocks: Map>; teardownSession: SessionTeardown; + cleanupSessionlessExecutionHost: PlatformResourceCleanup['cleanupSessionlessExecutionHost']; withinDiagnosticsScope?: (run: () => Promise) => Promise; }): LeaseSessionEnd { const { sessionStore, executionLocks, teardownSession } = params; let closing = false; - const inFlight = new Set>(); + const inFlight = new Map>>(); + const pending = new Map(); - const endUnderLocks = async (leaseId: string, reason: LeaseEndReason): Promise => { + const endLiveSessions = async (leaseId: string, reason: LeaseEndReason): Promise => { const refs = sessionStore.listRefs().filter((ref) => ref.session.lease?.leaseId === leaseId); await Promise.all( refs.map( @@ -855,6 +861,37 @@ export function createLeaseSessionEnd(params: { ); }; + const endUnderLocks = async (leaseId: string, reason: LeaseEndReason): Promise => { + await endLiveSessions(leaseId, reason); + // A close queued before this lease end can retire its session while we wait for its locks. + const retainedHosts = sessionStore.listRetainedExecutionHosts(leaseId); + await Promise.all( + retainedHosts.map(async (host) => { + try { + await withRequestExecutionLockKeys( + executionLocks, + existingSessionExecutionLockKeys(host.address, host.device.id), + async () => { + if (closing || !sessionStore.ownsRetainedExecutionHost(host)) return; + if (!sessionStore.findByDevice(host.device.id)) { + await params.cleanupSessionlessExecutionHost(host.device); + } + sessionStore.releaseRetainedExecutionHost(host); + }, + ); + } catch (error) { + emitDiagnostic({ + level: 'warn', + phase: 'retained_execution_host_cleanup_failed', + data: { leaseId, reason, deviceId: host.device.id, error: errorMessage(error) }, + }); + } + }), + ); + // A reopen can publish its same-lease session while retained-host cleanup waits for the device. + await endLiveSessions(leaseId, reason); + }; + const run = async (leaseId: string, reason: LeaseEndReason): Promise => { try { await endUnderLocks(leaseId, reason); @@ -864,27 +901,50 @@ export function createLeaseSessionEnd(params: { phase: 'leased_session_end_failed', data: { leaseId, reason, error: errorMessage(error) }, }); + } finally { + if ( + !closing && + sessionStore.listRetainedExecutionHosts(leaseId).length === 0 && + !sessionStore.listRefs().some((ref) => ref.session.lease?.leaseId === leaseId) + ) { + pending.delete(leaseId); + } else if (!closing) { + pending.set(leaseId, { reason, nextRetryAt: Date.now() + LEASE_END_RETRY_INTERVAL_MS }); + } + } + }; + + const end = async (leaseId: string, reason: LeaseEndReason): Promise => { + if (closing) return; + pending.set(leaseId, { reason, nextRetryAt: 0 }); + const ending = params.withinDiagnosticsScope + ? params.withinDiagnosticsScope(async () => await run(leaseId, reason)) + : run(leaseId, reason); + const running = inFlight.get(leaseId) ?? new Set>(); + running.add(ending); + inFlight.set(leaseId, running); + try { + await ending; + } finally { + running.delete(ending); + if (running.size === 0) inFlight.delete(leaseId); } }; return { - end: async (leaseId, reason) => { - if (closing) return; - const ending = params.withinDiagnosticsScope - ? params.withinDiagnosticsScope(async () => await run(leaseId, reason)) - : run(leaseId, reason); - inFlight.add(ending); - try { - await ending; - } finally { - inFlight.delete(ending); - } + end, + retryPending: async () => { + await Promise.all( + Array.from(pending, async ([leaseId, { reason, nextRetryAt }]) => { + if (!inFlight.has(leaseId) && Date.now() >= nextRetryAt) await end(leaseId, reason); + }), + ); }, cancel: () => { closing = true; }, drain: async () => { - await Promise.allSettled(inFlight); + await Promise.allSettled(Array.from(inFlight.values()).flatMap((running) => [...running])); }, }; } diff --git a/packages/daemon/src/sessions/lease-lifecycle.ts b/packages/daemon/src/sessions/lease-lifecycle.ts index 5091c00a00..5afd8919e2 100644 --- a/packages/daemon/src/sessions/lease-lifecycle.ts +++ b/packages/daemon/src/sessions/lease-lifecycle.ts @@ -190,6 +190,19 @@ function closeLeaseDecision( return activeLease && provider?.release ? 'provider-release' : 'local-release'; } +export function inspectSessionLeaseRetainedOnClose(params: { + session: SessionState; + leaseRegistry: LeaseRegistry; +}): boolean { + if (!params.session.lease) return false; + return ( + closeLeaseDecision( + params.leaseRegistry.inspectLeaseForClose(closeReleaseRequest(params.session)), + undefined, + ) === 'retained' + ); +} + export function inspectSessionProviderReleaseOnClose(params: { session: SessionState; leaseRegistry: LeaseRegistry; diff --git a/packages/daemon/src/sessions/session-lifecycle/internal/session-close-lifecycle-teardown.test.ts b/packages/daemon/src/sessions/session-lifecycle/internal/session-close-lifecycle-teardown.test.ts index 8d3f4ae155..3e93a5d9e6 100644 --- a/packages/daemon/src/sessions/session-lifecycle/internal/session-close-lifecycle-teardown.test.ts +++ b/packages/daemon/src/sessions/session-lifecycle/internal/session-close-lifecycle-teardown.test.ts @@ -11,12 +11,17 @@ import { } from '@agent-device/test-kit'; import { SessionStore } from '../../../core/session-store.ts'; import { LeaseRegistry } from '../../../resources/lease-registry.ts'; +import { createLeaseSessionEnd } from '../../../server/request-execution-scope.ts'; +import { handleLeaseCommands } from '../../handlers/lease.ts'; import type { DaemonRequest } from '../../../core/daemon-request.ts'; import type { SessionState } from '../../../core/session-state.ts'; import { daemonPlatformServicesFixture } from '../../../__tests__/support/platform-services-fixture.ts'; import { fakeCreateCommandSurface } from '../../../__tests__/support/command-surface-fake.ts'; import { platformResourceCleanupFixture } from '../../../__tests__/support/platform-resource-cleanup-fixture.ts'; -import { dispatchApplicationLifecycleEffect } from '../../../__tests__/support/application-lifecycle-runtime-fixture.ts'; +import { + clearRuntimeHintsFixture, + dispatchApplicationLifecycleEffect, +} from '../../../__tests__/support/application-lifecycle-runtime-fixture.ts'; import { handleSessionCommands, mockBindDeviceRuntime, @@ -142,7 +147,7 @@ test.each([ device, shutdownRequested: false, hasScreenRecording: false, - hasLease: false, + leaseWillEnd: false, }); expect(finalizations).toEqual([ expect.objectContaining({ retainRunner: retainExecutionHost, shutdownTarget: false }), @@ -204,9 +209,56 @@ test('close hands the session app and surface to the platform finalization', asy expect(sessionStore.get(sessionName)).toBeUndefined(); }); -test('close of a leased session tells the platform that a lease holds the device', async () => { +test.each([false, true])( + 'close reports whether a lease with retainOnClose=%s will end', + async (retainOnClose) => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-leased-simulator-session'; + const leaseRegistry = new LeaseRegistry(); + const lease = leaseRegistry.allocateLease({ + tenantId: 'tenant-a', + runId: 'run-1', + leaseBackend: 'ios-simulator', + deviceKey: 'ios:sim-1', + clientId: 'client-a', + retainOnClose, + }); + publishSession(sessionStore, sessionName, IOS_SIMULATOR, { + lease: { + leaseId: lease.leaseId, + tenantId: lease.tenantId, + runId: lease.runId, + leaseBackend: lease.backend, + deviceKey: lease.deviceKey, + clientId: lease.clientId, + expiresAt: lease.expiresAt, + }, + }); + + const { response, finalizations, platformResourceCleanup } = await closeSession({ + sessionStore, + sessionName, + retainExecutionHost: retainOnClose, + leaseRegistry, + }); + + expect(response?.ok).toBe(true); + expect(platformResourceCleanup.retainExecutionHostAfterClose).toHaveBeenCalledWith( + expect.objectContaining({ leaseWillEnd: !retainOnClose }), + ); + expect(finalizations).toEqual([expect.objectContaining({ retainRunner: retainOnClose })]); + expect(sessionStore.get(sessionName)).toBeUndefined(); + expect(sessionStore.listRetainedExecutionHosts(lease.leaseId)).toEqual( + retainOnClose + ? [expect.objectContaining({ address: sessionName, device: IOS_SIMULATOR })] + : [], + ); + }, +); + +test('releasing a retained client lease stops the host after close retired its session', async () => { const sessionStore = makeSessionStore(); - const sessionName = 'ios-leased-simulator-session'; + const sessionName = 'retained-host-release'; const leaseRegistry = new LeaseRegistry(); const lease = leaseRegistry.allocateLease({ tenantId: 'tenant-a', @@ -214,32 +266,128 @@ test('close of a leased session tells the platform that a lease holds the device leaseBackend: 'ios-simulator', deviceKey: 'ios:sim-1', clientId: 'client-a', + retainOnClose: true, + }); + const leaseScope = { + leaseId: lease.leaseId, + tenantId: lease.tenantId, + runId: lease.runId, + leaseBackend: lease.backend, + deviceKey: lease.deviceKey, + clientId: lease.clientId, + }; + publishSession(sessionStore, sessionName, IOS_SIMULATOR, { lease: leaseScope }); + const closed = await closeSession({ + sessionStore, + sessionName, + retainExecutionHost: true, + leaseRegistry, + }); + expect(closed.response?.ok).toBe(true); + expect(sessionStore.get(sessionName)).toBeUndefined(); + expect(leaseRegistry.listActiveLeases()).toHaveLength(1); + const cleanup = vi.fn(async () => {}); + const teardown = vi.fn(async () => {}); + const ender = createLeaseSessionEnd({ + env: sessionStore.host.env, + sessionStore, + executionLocks: new Map(), + teardownSession: teardown, + cleanupSessionlessExecutionHost: cleanup, + }); + + const released = await handleLeaseCommands({ + host: sessionStore.host, + req: { + token: 't', + command: 'lease_release', + session: sessionName, + positionals: [], + meta: leaseScope, + }, + sessionName, + sessionStore, + leaseRegistry, + endSessionsForLease: async (leaseId) => await ender.end(leaseId, 'LEASE_RELEASED'), + }); + + expect(released?.ok).toBe(true); + expect(leaseRegistry.listActiveLeases()).toEqual([]); + expect(teardown).not.toHaveBeenCalled(); + expect(cleanup).toHaveBeenCalledExactlyOnceWith(IOS_SIMULATOR); + expect(sessionStore.listRetainedExecutionHosts(lease.leaseId)).toEqual([]); +}); + +test('a lease scope mismatch still closes the app and runner before returning a retriable release error', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-close-mismatched-lease'; + const leaseRegistry = new LeaseRegistry(); + const lease = leaseRegistry.allocateLease({ + tenantId: 'tenant-a', + runId: 'run-1', + leaseBackend: 'ios-simulator', + deviceKey: 'ios:sim-1', + clientId: 'client-a', + retainOnClose: true, }); publishSession(sessionStore, sessionName, IOS_SIMULATOR, { + appBundleId: 'com.example.app', lease: { leaseId: lease.leaseId, tenantId: lease.tenantId, runId: lease.runId, leaseBackend: lease.backend, deviceKey: lease.deviceKey, - clientId: lease.clientId, + clientId: 'client-b', expiresAt: lease.expiresAt, }, }); + sessionStore.setRuntimeHints(sessionName, { platform: 'ios', metroHost: '127.0.0.1' }); const { response, finalizations, platformResourceCleanup } = await closeSession({ sessionStore, sessionName, retainExecutionHost: false, + positionals: ['com.example.app'], leaseRegistry, }); - expect(response?.ok).toBe(true); + expect(platformResourceCleanup.stopSnapshotHelper).toHaveBeenCalledWith(IOS_SIMULATOR); + expect(mockDispatch).toHaveBeenCalledWith( + IOS_SIMULATOR, + 'close', + ['com.example.app'], + undefined, + expect.any(Object), + ); + expect(clearRuntimeHintsFixture).toHaveBeenCalledWith({ + device: IOS_SIMULATOR, + appId: 'com.example.app', + values: expect.objectContaining({ metroHost: '127.0.0.1' }), + }); expect(platformResourceCleanup.retainExecutionHostAfterClose).toHaveBeenCalledWith( - expect.objectContaining({ hasLease: true }), + expect.objectContaining({ leaseWillEnd: true }), ); expect(finalizations).toEqual([expect.objectContaining({ retainRunner: false })]); - expect(sessionStore.get(sessionName)).toBeUndefined(); + expect(response).toMatchObject({ + ok: false, + error: { + code: 'UNAUTHORIZED', + details: { reason: 'LEASE_SCOPE_MISMATCH', session: sessionName }, + retriable: true, + }, + }); + expect(sessionStore.get(sessionName)?.lease?.leaseId).toBe(lease.leaseId); + expect( + leaseRegistry.getLease({ + leaseId: lease.leaseId, + tenantId: lease.tenantId, + runId: lease.runId, + leaseBackend: lease.backend, + deviceKey: lease.deviceKey, + clientId: lease.clientId, + }), + ).toMatchObject({ retainOnClose: true }); }); test('an app-only close terminates the app without finalizing or ending the session', async () => { diff --git a/packages/daemon/src/sessions/session-lifecycle/internal/session-close-lifecycle-teardown.ts b/packages/daemon/src/sessions/session-lifecycle/internal/session-close-lifecycle-teardown.ts index 77a14bb12b..f277ec44e1 100644 --- a/packages/daemon/src/sessions/session-lifecycle/internal/session-close-lifecycle-teardown.ts +++ b/packages/daemon/src/sessions/session-lifecycle/internal/session-close-lifecycle-teardown.ts @@ -1,3 +1,5 @@ +import type { LeaseRegistry } from '../../../resources/lease-registry.ts'; +import { inspectSessionLeaseRetainedOnClose } from '../../lease-lifecycle.ts'; import type { CloseApplicationFinalizationResult } from '@agent-device/contracts/application-lifecycle-runtime'; import type { TargetShutdownResult } from '@agent-device/contracts/device'; import type { DaemonRequest } from '../../../core/daemon-request.ts'; @@ -26,6 +28,7 @@ export type PlatformCloseDispatcher = (params: { export type SessionCloseTeardownResult = Readonly<{ platformCloseError: unknown; + retainExecutionHost: boolean; saveScriptError?: Error; shutdownResult?: TargetShutdownResult; }>; @@ -36,6 +39,7 @@ export async function runSessionCloseTeardown(params: { ref: SessionRef; logPath: string; sessionStore: SessionStore; + leaseRegistry: LeaseRegistry; lifecycle: CloseRuntime | CloseRuntimeWithRuntimeHintClear; clearRuntimeHints?: RuntimeHintClearOperation; cleanupFailures: SessionCleanupFailure[]; @@ -78,7 +82,11 @@ export async function runSessionCloseTeardown(params: { device: session.device, shutdownRequested: req.flags?.shutdown === true, hasScreenRecording: Boolean(session.screenRecording), - hasLease: Boolean(session.lease), + leaseWillEnd: + Boolean(session.lease) && + !(await attemptCleanup('lease_retention', async () => + inspectSessionLeaseRetainedOnClose({ session, leaseRegistry: params.leaseRegistry }), + )), }); const configuredRuntimeHints = sessionStore.getRuntimeHints(sessionName); const resourceStops = sessionResourceStopSteps({ @@ -121,7 +129,12 @@ export async function runSessionCloseTeardown(params: { await attemptCleanup('materialized_paths', () => cleanupRetainedMaterializedPathsForSession(sessionName), ); - return { platformCloseError, saveScriptError, shutdownResult: finalization?.shutdown }; + return { + platformCloseError, + retainExecutionHost, + saveScriptError, + shutdownResult: finalization?.shutdown, + }; } export function closeCleanupError( diff --git a/packages/daemon/src/sessions/session-lifecycle/internal/session-close.ts b/packages/daemon/src/sessions/session-lifecycle/internal/session-close.ts index 35f5d9a88f..193128751f 100644 --- a/packages/daemon/src/sessions/session-lifecycle/internal/session-close.ts +++ b/packages/daemon/src/sessions/session-lifecycle/internal/session-close.ts @@ -362,19 +362,21 @@ async function runCloseTeardownAndRelease(params: { } = params; const { address: sessionName } = ref; const cleanupFailures: SessionCleanupFailure[] = []; - const { platformCloseError, saveScriptError, shutdownResult } = await runSessionCloseTeardown({ - req, - ref, - logPath, - sessionStore, - lifecycle, - clearRuntimeHints, - cleanupFailures, - repairArmed: params.repairArmed, - dispatchTargetedPlatformClose, - finalizeOrdinaryCloseScript, - platformResourceCleanup: params.platformResourceCleanup, - }); + const { platformCloseError, retainExecutionHost, saveScriptError, shutdownResult } = + await runSessionCloseTeardown({ + leaseRegistry, + req, + ref, + logPath, + sessionStore, + lifecycle, + clearRuntimeHints, + cleanupFailures, + repairArmed: params.repairArmed, + dispatchTargetedPlatformClose, + finalizeOrdinaryCloseScript, + platformResourceCleanup: params.platformResourceCleanup, + }); let session = sessionStore.requireCurrent(ref); const leaseRelease = await releaseProviderLeaseForClose({ req, @@ -400,7 +402,7 @@ async function runCloseTeardownAndRelease(params: { } else { await clearDeviceClaim(session.deviceClaim, params.env); } - sessionStore.retire(ref); + sessionStore.retire(ref, { retainExecutionHost }); if (deviceClaimBlockingError) throw deviceClaimBlockingError; if (saveScriptError) throw saveScriptError; return { kind: 'closed', providerData: leaseRelease.providerData, shutdownResult }; diff --git a/src/__tests__/cli-help.test.ts b/src/__tests__/cli-help.test.ts index 22526b2b61..fcd08b0faf 100644 --- a/src/__tests__/cli-help.test.ts +++ b/src/__tests__/cli-help.test.ts @@ -165,7 +165,7 @@ test('help physical-device documents the runner/daemon lifecycle detail moved ou result.stdout, /keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator \(same udid in the same simulator set\) skips the runner build/, ); - assert.match(result.stdout, /the session held a device lease/); + assert.match(result.stdout, /its device lease ends on close/); assert.match(result.stdout, /AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS/); assert.match( result.stdout, diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 2871a57041..47b34af2b6 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -480,7 +480,7 @@ Android physical-device prerequisites: Runner and daemon lifecycle (applies to simulators too): open without --relaunch is idempotent-foreground for an already-running app, except native macOS app opens keep it in the background (help macos); neither restarts the process. open --relaunch restarts the app; on iOS simulators this collapses to one simctl launch --terminate-running-process call instead of a separate terminate-then-launch. No runner read launches a session app that is not running: snapshot, wait, is, get, a reading find, and an interaction's leading reads (a gesture's viewport read, the capture that resolves a selector click/fill) answer the retriable APP_NOT_RUNNING instead of bare-launching over a launch SpringBoard still holds behind its deep-link confirmation. Only open, activate, and a command that mutates without a leading read bring a stopped app up. - close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. Only a ready runner with an observed listener is retained; close retires unfinished prewarm recovery. Losing the retained listener stops that generation, and daemon shutdown stops idle retained runners rather than handing them off without supervision. + close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or its device lease ends on close. A lease allocated with retainOnClose can keep the runner between sessions; releasing or expiring that lease stops its runner unless a session outside the lease has taken over the device. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stopping. Lease release, expiry, and graceful daemon shutdown still clean up runners they own. Only a ready runner with an observed listener is retained; close retires unfinished prewarm recovery. Losing the retained listener stops that generation, and graceful daemon shutdown stops idle retained runners rather than handing them off without supervision. Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. A client starting a local daemon waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower, not failed. It stops at a ceiling (default 120 s; set AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS to a positive integer number of milliseconds to override) and names the outcome in error details.reason: startup-exited (the process ended before registering), startup-stalled (alive, no startup progress for a fixed 60 s window the ceiling variable does not change), or startup-ceiling (a startup was still live when the ceiling passed — this client's daemon or another client's; the client stops its own launch and never signals another client's live daemon. Only a progressing own launch outlasts the 60 s stall window, so a ceiling set below 60 s also ends a silent own launch this way). details.phases carries the startup phase timings it observed and details.startupTimeoutMs the effective ceiling. Branch on details.reason, never on the message. The variable is read only when a client must start a daemon: a value that is not a positive integer fails with INVALID_ARGS before the daemon spawns, and a client that only attaches to or reuses a running daemon never reads it. A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon that agreed to leave is waited for up to 60 s and never signaled while it drains; one still running then fails with details.reason daemon-drain-timeout unless it answers nothing and is provably a daemon process, which is force-retired. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir . Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly. diff --git a/src/platform-runtime-resource-cleanup-retention.test.ts b/src/platform-runtime-resource-cleanup-retention.test.ts index fe249ba025..68ad0aef12 100644 --- a/src/platform-runtime-resource-cleanup-retention.test.ts +++ b/src/platform-runtime-resource-cleanup-retention.test.ts @@ -10,7 +10,7 @@ import { createPlatformResourceCleanup } from './platform-runtime-resource-clean const platformResourceCleanup = createPlatformResourceCleanup(createLiveTestHostEnvironment()); -const PLAIN_CLOSE = { shutdownRequested: false, hasScreenRecording: false, hasLease: false }; +const PLAIN_CLOSE = { shutdownRequested: false, hasScreenRecording: false, leaseWillEnd: false }; function retains(device: DeviceInfo, overrides: Partial = {}): boolean { return platformResourceCleanup.retainExecutionHostAfterClose({ @@ -28,7 +28,7 @@ test('a plain close keeps the runner host of an iOS simulator', () => { test.each([ { name: 'a shutdown request', overrides: { shutdownRequested: true } }, { name: 'a live screen recording', overrides: { hasScreenRecording: true } }, - { name: 'a lease', overrides: { hasLease: true } }, + { name: 'a lease that ends on close', overrides: { leaseWillEnd: true } }, ])('$name releases the runner host of an iOS simulator', ({ overrides }) => { expect(retains(IOS_SIMULATOR, overrides)).toBe(false); }); diff --git a/src/platform-runtime-resource-cleanup.ts b/src/platform-runtime-resource-cleanup.ts index df9c17d17d..3dea56560d 100644 --- a/src/platform-runtime-resource-cleanup.ts +++ b/src/platform-runtime-resource-cleanup.ts @@ -108,7 +108,7 @@ export function createPlatformResourceCleanup(host: HostEnvironment): PlatformRe params.device.kind === 'simulator' && !params.shutdownRequested && !params.hasScreenRecording && - !params.hasLease + !params.leaseWillEnd ); }, }); diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index db77da44c5..f7ee37a8fc 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -344,7 +344,7 @@ Domain client methods: - `client.sessions.list()`, `stateDir()`, `close()`, `saveScript()`, `artifacts()` - `client.apps.install()`, `reinstall()`, `uninstall()`, `installFromSource()`, `list()`, `open()`, `close()`, `push()`, `triggerEvent()` - `client.materializations.release()` -- `client.leases.allocate()`, `heartbeat()`, `release()`. Pass `retainOnClose: true` to `allocate()` when you release the lease yourself; session `close` then leaves it active until `release()`, expiry, or daemon shutdown. The returned lease has `retainOnClose: true` only when the daemon honored it. Keep heartbeating a retained lease between sessions: it expires after its `ttlMs` like any other lease. Pass `leaseId` (16 to 128 hex characters) to `allocate()` to name a new lease with an id the daemon has never issued, or to get back your live lease with that id when you retry with the same tenant, run, client, lease backend, and provider (and no device, or the leased one), then use the returned `leaseId`: a run that already holds a lease gets that one back, and an older daemon mints its own. `allocate()` also tells you which of those you got, on the returned lease's `created`: `true` when your request created that lease, `false` when it was already there, and omitted on `heartbeat()` or when an older daemon does not support ownership evidence. The CLI releases an allocate response it discards before recording (a spec refusal, identity refusal, or keyless replacement), unless `created` is `false`. After binding, rollback on a later preparation failure requires `created:true`. Against an older daemon that omits `created`, refused or replaced leases are released, but a bound, unrecorded lease remains until expiry if preparation fails. `allocate()` with a `leaseProvider` the daemon has no runtime for fails with `UNSUPPORTED_OPERATION`, reason `provider-not-registered`, and `details.registeredProviders`; no lease is recorded. Leases with no provider, `proxy`, or `cloud` need no runtime. +- `client.leases.allocate()`, `heartbeat()`, `release()`. Pass `retainOnClose: true` to `allocate()` when you release the lease yourself; session `close` then leaves it active until `release()`, expiry, or graceful daemon shutdown. The returned lease has `retainOnClose: true` only when the daemon honored it. Keep heartbeating a retained lease between sessions: it expires after its `ttlMs` like any other lease. On iOS simulators, a healthy idle XCTest runner can stay available between those sessions. Releasing or expiring the lease stops its runner unless a session outside the lease has taken over the device. Pass `leaseId` (16 to 128 hex characters) to `allocate()` to name a new lease with an id the daemon has never issued, or to get back your live lease with that id when you retry with the same tenant, run, client, lease backend, and provider (and no device, or the leased one), then use the returned `leaseId`: a run that already holds a lease gets that one back, and an older daemon mints its own. `allocate()` also tells you which of those you got, on the returned lease's `created`: `true` when your request created that lease, `false` when it was already there, and omitted on `heartbeat()` or when an older daemon does not support ownership evidence. The CLI releases an allocate response it discards before recording (a spec refusal, identity refusal, or keyless replacement), unless `created` is `false`. After binding, rollback on a later preparation failure requires `created:true`. Against an older daemon that omits `created`, refused or replaced leases are released, but a bound, unrecorded lease remains until expiry if preparation fails. `allocate()` with a `leaseProvider` the daemon has no runtime for fails with `UNSUPPORTED_OPERATION`, reason `provider-not-registered`, and `details.registeredProviders`; no lease is recorded. Leases with no provider, `proxy`, or `cloud` need no runtime. - `client.metro.prepare()`, `reload()` - `client.capture.snapshot()`, `screenshot()`, `diff()` - `client.interactions.gesture()`, `click()`, `press()`, `longPress()`, `swipe()`, `pan()`, `drag()`, `fling()`, `swipeGesture()`, `focus()`, `type()`, `fill()`, `scroll()`, `pinch()`, `rotateGesture()`, `transformGesture()`, `tapGesture()`, `get()`, `is()`, `find()`