Skip to content

Commit a3a8d59

Browse files
committed
improvement(desktop): sign-in prompts for the agent browser — site passwords, passkey accounts, client certificates, and app links
1 parent e985d51 commit a3a8d59

16 files changed

Lines changed: 2258 additions & 32 deletions

File tree

‎apps/desktop/build/entitlements.mac.plist‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,5 @@
1414
Hardened Runtime denies the camera no matter what the user allowed. -->
1515
<key>com.apple.security.device.camera</key>
1616
<true/>
17-
<!-- WebAuthn hybrid transport (passkey on the user's phone via QR) rides
18-
Bluetooth proximity; without this the QR option silently never
19-
completes in signed builds. -->
20-
<key>com.apple.security.device.bluetooth</key>
21-
<true/>
2217
</dict>
2318
</plist>

‎apps/desktop/e2e/browser-sign-in-prompts.spec.ts‎

Lines changed: 554 additions & 0 deletions
Large diffs are not rendered by default.

‎apps/desktop/electron-builder.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@ mac:
7272
NSScreenCaptureUsageDescription: Mothership captures approved app windows to understand and verify computer-use actions.
7373
NSMicrophoneUsageDescription: Sim uses your microphone for voice input in Chat and for meetings you join in the built-in browser.
7474
NSCameraUsageDescription: Sim uses your camera for meetings you join in the built-in browser, such as Google Meet.
75-
NSBluetoothAlwaysUsageDescription: Sim uses Bluetooth to complete passkey sign-ins with a nearby phone in the built-in browser.
7675
entitlements: build/entitlements.mac.plist
7776
entitlementsInherit: build/entitlements.mac.plist
7877
notarize: true

‎apps/desktop/src/main/browser-agent/driver.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,11 @@ import {
8181
setFocusedInputValue,
8282
typeIntoElement,
8383
} from '@/main/browser-agent/page-functions'
84+
import {
85+
appLaunchNoticeForContents,
86+
pagePromptForContents,
87+
respondToPagePrompt,
88+
} from '@/main/browser-agent/page-prompts'
8489
import { isPanelVisible, panelWindow } from '@/main/browser-agent/panel'
8590
import {
8691
withFailedPostActionObservation,
@@ -555,6 +560,8 @@ function pageStateFor(contents: WebContents, tabId: string): BrowserPageState {
555560
const issue = session.pageIssueForContents(contents)
556561
const mediaPermissionRequest = session.mediaPermissionRequestForContents(contents)
557562
const dialog = session.pageDialogForContents(contents)
563+
const prompt = pagePromptForContents(contents)
564+
const appLaunchNotice = appLaunchNoticeForContents(contents)
558565
return {
559566
scopeId: session.getBrowserScopeId(),
560567
tabId,
@@ -566,6 +573,8 @@ function pageStateFor(contents: WebContents, tabId: string): BrowserPageState {
566573
...(issue ? { issue } : {}),
567574
...(mediaPermissionRequest ? { mediaPermissionRequest } : {}),
568575
...(dialog ? { dialog } : {}),
576+
...(prompt ? { prompt } : {}),
577+
...(appLaunchNotice ? { appLaunchNotice } : {}),
569578
}
570579
}
571580

@@ -804,6 +813,7 @@ export function initDriver(
804813
})
805814
},
806815
onDownloadsChanged: (state) => callbacks.onDownloadsChanged?.(state),
816+
onAgentNotice: recordNotice,
807817
},
808818
getMainWindow,
809819
persistence,
@@ -5500,6 +5510,11 @@ export async function handlePanelAction(
55005510
}
55015511
if (action.action === 'enable-page-dialogs') {
55025512
session.enablePageDialogs()
5513+
if (action.pagePrompts === true) session.enablePagePrompts()
5514+
return
5515+
}
5516+
if (action.action === 'respond-prompt') {
5517+
respondToPagePrompt(session.getBrowserScopeId(), action)
55035518
return
55045519
}
55055520
if (action.action === 'respond-dialog') {
Lines changed: 311 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,311 @@
1+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
2+
3+
vi.mock('electron', () => import('@/test/electron-mock'))
4+
5+
import type { BrowserPagePrompt, BrowserPagePromptResponse } from '@sim/browser-protocol'
6+
import type {
7+
AuthenticationResponseDetails,
8+
AuthInfo,
9+
Certificate,
10+
Session,
11+
WebContents,
12+
WebFrameMain,
13+
} from 'electron'
14+
import { app, shell } from 'electron'
15+
import {
16+
appLaunchNoticeForContents,
17+
handleBrowserLogin,
18+
handleSelectClientCertificate,
19+
httpAuthChallenge,
20+
noteMainFrameNavigation,
21+
pagePromptForContents,
22+
requestExternalLaunch,
23+
resetPagePromptDecisions,
24+
respondToPagePrompt,
25+
setPagePromptHost,
26+
} from '@/main/browser-agent/page-prompts'
27+
import { registerAgentWebContents } from '@/main/browser-agent/registry'
28+
29+
const PAGE = 'https://meet.example'
30+
31+
interface Context {
32+
promptsEnabled: boolean
33+
onScreen: boolean
34+
userActivated: boolean
35+
agentActedLast: boolean
36+
agentDrives: boolean
37+
}
38+
39+
let context: Context
40+
41+
function agentContents(session: Session = {} as Session): WebContents {
42+
const contents = {
43+
isDestroyed: () => false,
44+
getURL: () => `${PAGE}/j/1`,
45+
mainFrame: { frameTreeNodeId: 1 },
46+
session,
47+
} as WebContents
48+
registerAgentWebContents(contents)
49+
return contents
50+
}
51+
52+
function shownPrompt(contents: WebContents): BrowserPagePrompt {
53+
const prompt = pagePromptForContents(contents)
54+
if (!prompt) throw new Error('No prompt shown')
55+
return prompt
56+
}
57+
58+
function answer(contents: WebContents, response: BrowserPagePromptResponse, scopeId = 'chat') {
59+
respondToPagePrompt(scopeId, {
60+
action: 'respond-prompt',
61+
requestId: shownPrompt(contents).requestId,
62+
response,
63+
})
64+
}
65+
66+
beforeEach(() => {
67+
vi.useFakeTimers()
68+
context = {
69+
promptsEnabled: true,
70+
onScreen: true,
71+
userActivated: false,
72+
agentActedLast: false,
73+
agentDrives: false,
74+
}
75+
setPagePromptHost({
76+
tabContext: () => ({ scopeId: 'chat', ...context }),
77+
publish: () => {},
78+
focus: () => {},
79+
noticeForAgent: () => {},
80+
})
81+
})
82+
83+
afterEach(() => {
84+
setPagePromptHost(null)
85+
resetPagePromptDecisions()
86+
vi.useRealTimers()
87+
})
88+
89+
describe('httpAuthChallenge', () => {
90+
const url = 'https://intranet.example:8443/reports'
91+
const page = { url, isRequestForNavigation: true }
92+
const server = { isProxy: false, host: 'intranet.example', port: 8443 }
93+
94+
it("asks about the tab's own top-level navigation, challenged by the host it requested", () => {
95+
expect(httpAuthChallenge(page, server, url)).toEqual({
96+
origin: 'https://intranet.example:8443',
97+
proxy: false,
98+
insecure: false,
99+
})
100+
expect(
101+
httpAuthChallenge(
102+
{ url: 'http://router.local/', isRequestForNavigation: true },
103+
{ isProxy: false, host: 'router.local', port: 80 },
104+
'http://router.local/'
105+
)
106+
).toMatchObject({ insecure: true })
107+
})
108+
109+
it('cancels frame, subresource, and cross-origin challenges', () => {
110+
expect(httpAuthChallenge(page, server, 'https://intranet.example:8443/')).toBeNull()
111+
expect(httpAuthChallenge(page, server, undefined)).toBeNull()
112+
expect(httpAuthChallenge({ ...page, isRequestForNavigation: false }, server, url)).toBeNull()
113+
expect(httpAuthChallenge(page, { ...server, host: 'evil.example' }, url)).toBeNull()
114+
expect(httpAuthChallenge(page, { ...server, port: 443 }, url)).toBeNull()
115+
})
116+
117+
it('asks about a proxy for any request', () => {
118+
expect(
119+
httpAuthChallenge(
120+
{ url: 'https://example.com/pixel.png', isRequestForNavigation: false },
121+
{ isProxy: true, host: 'proxy.corp', port: 3128 },
122+
undefined
123+
)
124+
).toEqual({ origin: 'proxy.corp:3128', proxy: true, insecure: false })
125+
})
126+
})
127+
128+
describe('HTTP sign-in', () => {
129+
const url = 'https://intranet.example/'
130+
const login = (contents: WebContents, firstAuthAttempt = true) => {
131+
const callback = vi.fn()
132+
handleBrowserLogin(
133+
{ preventDefault: () => {} },
134+
contents,
135+
{ url, isRequestForNavigation: true, firstAuthAttempt } as AuthenticationResponseDetails,
136+
{
137+
isProxy: false,
138+
scheme: 'basic',
139+
host: 'intranet.example',
140+
port: 443,
141+
realm: '',
142+
} as AuthInfo,
143+
callback
144+
)
145+
return callback
146+
}
147+
148+
it('stops asking a page that keeps reloading into the same challenge', () => {
149+
const contents = agentContents()
150+
for (let attempt = 0; attempt < 3; attempt++) {
151+
noteMainFrameNavigation(contents, url)
152+
const callback = login(contents)
153+
expect(pagePromptForContents(contents)).toMatchObject({ kind: 'http-auth' })
154+
answer(contents, { allowed: false })
155+
expect(callback).toHaveBeenCalledExactlyOnceWith()
156+
}
157+
noteMainFrameNavigation(contents, url)
158+
expect(login(contents)).toHaveBeenCalledExactlyOnceWith()
159+
expect(pagePromptForContents(contents)).toBeUndefined()
160+
161+
vi.advanceTimersByTime(60_000)
162+
login(contents)
163+
expect(pagePromptForContents(contents)).toMatchObject({ kind: 'http-auth' })
164+
})
165+
166+
it('passes on only the credentials of an accepted answer', () => {
167+
const contents = agentContents()
168+
noteMainFrameNavigation(contents, url)
169+
const callback = login(contents)
170+
answer(contents, { allowed: true, kind: 'http-auth', username: 'ada', password: 'pw' })
171+
expect(callback).toHaveBeenCalledExactlyOnceWith('ada', 'pw')
172+
})
173+
})
174+
175+
describe('client certificates', () => {
176+
const certificate = (fingerprint: string, subjectName: string) =>
177+
({ fingerprint, subjectName, issuerName: 'Corp CA', validExpiry: 2_000_000_000 }) as Certificate
178+
const first = certificate('sha256/first', 'Personal')
179+
const work = certificate('sha256/work', 'Work badge')
180+
181+
function request(contents: WebContents | null, list: Certificate[], url = 'https://mtls.corp') {
182+
const event = { preventDefault: vi.fn() }
183+
const callback = vi.fn()
184+
handleSelectClientCertificate(event, contents, url, list, callback)
185+
return { event, callback }
186+
}
187+
188+
it("leaves the main window on Electron's default", () => {
189+
const { event, callback } = request({ isDestroyed: () => false } as WebContents, [first])
190+
expect(event.preventDefault).not.toHaveBeenCalled()
191+
expect(callback).not.toHaveBeenCalled()
192+
})
193+
194+
it('sends no certificate when there is none, or no renderer to ask', () => {
195+
const contents = agentContents()
196+
const empty = request(contents, [])
197+
expect(empty.event.preventDefault).toHaveBeenCalled()
198+
expect(empty.callback).toHaveBeenCalledExactlyOnceWith()
199+
200+
context.promptsEnabled = false
201+
expect(request(contents, [first, work]).callback).toHaveBeenCalledExactlyOnceWith()
202+
})
203+
204+
it('sends only the picked certificate, remembers it per host, and forgets it on reset', () => {
205+
const session = {} as Session
206+
const contents = agentContents(session)
207+
const { callback } = request(contents, [first, work])
208+
expect(shownPrompt(contents)).toMatchObject({ kind: 'client-certificate', host: 'mtls.corp' })
209+
answer(contents, { allowed: true, kind: 'client-certificate', choiceId: 'sha256/work' })
210+
expect(callback).toHaveBeenCalledExactlyOnceWith(work)
211+
212+
expect(request(agentContents(session), [first, work]).callback).toHaveBeenCalledWith(work)
213+
expect(
214+
request(agentContents(session), [first, work], 'https://other.corp').callback
215+
).not.toHaveBeenCalled()
216+
217+
resetPagePromptDecisions()
218+
expect(request(agentContents(session), [first, work]).callback).not.toHaveBeenCalled()
219+
})
220+
221+
it("remembers Don't send, but not a prompt nobody answered", () => {
222+
const declined = agentContents()
223+
const refused = request(declined, [first])
224+
answer(declined, { allowed: false })
225+
expect(refused.callback).toHaveBeenCalledExactlyOnceWith()
226+
expect(request(agentContents(declined.session), [first]).callback).toHaveBeenCalledWith()
227+
228+
const ignored = agentContents()
229+
const unanswered = request(ignored, [first])
230+
vi.advanceTimersByTime(120_000)
231+
expect(unanswered.callback).toHaveBeenCalledExactlyOnceWith()
232+
expect(request(agentContents(ignored.session), [first]).callback).not.toHaveBeenCalled()
233+
})
234+
235+
it('sends nothing for an unknown choice, an off-screen answer, or another chat', () => {
236+
const forged = agentContents()
237+
const unknown = request(forged, [first])
238+
answer(forged, { allowed: true, kind: 'client-certificate', choiceId: 'sha256/forged' })
239+
expect(unknown.callback).toHaveBeenCalledExactlyOnceWith()
240+
241+
const elsewhere = agentContents()
242+
const otherChat = request(elsewhere, [first])
243+
answer(elsewhere, { allowed: true, kind: 'client-certificate', choiceId: 'sha256/first' }, 'x')
244+
expect(otherChat.callback).not.toHaveBeenCalled()
245+
246+
context.onScreen = false
247+
const hidden = agentContents()
248+
const offScreen = request(hidden, [first])
249+
answer(hidden, { allowed: true, kind: 'client-certificate', choiceId: 'sha256/first' })
250+
expect(offScreen.callback).toHaveBeenCalledExactlyOnceWith()
251+
})
252+
})
253+
254+
describe('app links', () => {
255+
const topFrame = (origin = PAGE) => ({ parent: null, frameTreeNodeId: 1, origin }) as WebFrameMain
256+
257+
beforeEach(() => {
258+
app.getApplicationNameForProtocol = vi.fn(() => 'Zoom')
259+
vi.mocked(shell.openExternal).mockClear()
260+
context.userActivated = true
261+
})
262+
263+
it('refuses a launch a frame initiated, even one targeting the top document', () => {
264+
const contents = agentContents()
265+
const frame = {
266+
parent: topFrame(),
267+
frameTreeNodeId: 2,
268+
origin: 'https://ads.example',
269+
} as WebFrameMain
270+
requestExternalLaunch(contents, 'zoommtg://zoom.us/join', { initiator: frame })
271+
requestExternalLaunch(contents, 'zoommtg://zoom.us/join', {
272+
initiator: topFrame('https://ads.example'),
273+
})
274+
requestExternalLaunch(contents, 'zoommtg://zoom.us/join', { initiator: null })
275+
expect(pagePromptForContents(contents)).toBeUndefined()
276+
expect(appLaunchNoticeForContents(contents)).toBeUndefined()
277+
})
278+
279+
it('asks after the top document is clicked, but only offers a popup passively', () => {
280+
const clicked = agentContents()
281+
requestExternalLaunch(clicked, 'zoommtg://zoom.us/join', { initiator: topFrame() })
282+
expect(pagePromptForContents(clicked)).toMatchObject({ kind: 'external-app', appName: 'Zoom' })
283+
284+
const popup = agentContents()
285+
requestExternalLaunch(popup, 'zoommtg://zoom.us/join', 'popup')
286+
expect(pagePromptForContents(popup)).toBeUndefined()
287+
expect(appLaunchNoticeForContents(popup)).toMatchObject({ origin: PAGE, appName: 'Zoom' })
288+
})
289+
290+
it('opens only the first link while its prompt is pending', async () => {
291+
const contents = agentContents()
292+
requestExternalLaunch(contents, 'zoommtg://zoom.us/join?id=1', { initiator: topFrame() })
293+
requestExternalLaunch(contents, 'zoommtg://zoom.us/join?id=2', { initiator: topFrame() })
294+
expect(appLaunchNoticeForContents(contents)).toBeUndefined()
295+
answer(contents, { allowed: true, kind: 'external-app', alwaysAllow: false })
296+
await vi.advanceTimersByTimeAsync(0)
297+
expect(shell.openExternal).toHaveBeenCalledExactlyOnceWith('zoommtg://zoom.us/join?id=1')
298+
})
299+
300+
it('forgets "Always allow" on reset', async () => {
301+
const contents = agentContents()
302+
requestExternalLaunch(contents, 'zoommtg://zoom.us/join?id=1', { initiator: topFrame() })
303+
answer(contents, { allowed: true, kind: 'external-app', alwaysAllow: true })
304+
resetPagePromptDecisions()
305+
vi.advanceTimersByTime(20_000)
306+
requestExternalLaunch(contents, 'zoommtg://zoom.us/join?id=2', { initiator: topFrame() })
307+
await vi.advanceTimersByTimeAsync(0)
308+
expect(pagePromptForContents(contents)).toMatchObject({ kind: 'external-app' })
309+
expect(shell.openExternal).toHaveBeenCalledExactlyOnceWith('zoommtg://zoom.us/join?id=1')
310+
})
311+
})

0 commit comments

Comments
 (0)