From 577d7e8b5b6f1c1d32abd8730a638119b75c4df2 Mon Sep 17 00:00:00 2001 From: Chris Scott <99081550+chriswritescode-dev@users.noreply.github.com> Date: Wed, 21 Oct 2026 14:07:10 -0400 Subject: [PATCH 1/5] fix(sessions): scroll to the latest message once the transcript is ready --- frontend/src/hooks/useAutoScroll.test.tsx | 135 ++++++++++++++---- frontend/src/hooks/useAutoScroll.ts | 33 +++-- frontend/src/pages/SessionDetail.tsx | 14 +- .../SessionDetail.scroll-floating.test.tsx | 25 +++- 4 files changed, 157 insertions(+), 50 deletions(-) diff --git a/frontend/src/hooks/useAutoScroll.test.tsx b/frontend/src/hooks/useAutoScroll.test.tsx index b3d3370bf..4cd38426b 100644 --- a/frontend/src/hooks/useAutoScroll.test.tsx +++ b/frontend/src/hooks/useAutoScroll.test.tsx @@ -76,29 +76,29 @@ describe('useAutoScroll', () => { time: { created: Date.now() }, } as SessionMessageInfo) - const setupHook = (messages: SessionMessageInfo[], sessionId = 'session-1') => { + const setupHook = ( + messages: SessionMessageInfo[], + options: { + sessionId?: string + isContentReady?: boolean + container?: HTMLDivElement | null + } = {} + ) => { containerHarness = createScrollContainer() - const containerRef = { current: containerHarness.div } const onScrollStateChange = vi.fn() + const initialProps = { + container: options.container === undefined ? containerHarness.div : options.container, + messages, + sessionId: options.sessionId ?? 'session-1', + contentVersion: messages.length, + isContentReady: options.isContentReady ?? true, + onScrollStateChange, + } + const renderResult = renderHook( - (props) => - useAutoScroll({ - containerRef: props.containerRef, - messages: props.messages, - sessionId: props.sessionId, - contentVersion: props.contentVersion, - onScrollStateChange: props.onScrollStateChange, - }), - { - initialProps: { - containerRef, - messages, - sessionId, - contentVersion: messages.length, - onScrollStateChange, - }, - } + (props: typeof initialProps) => useAutoScroll(props), + { initialProps } ) return { renderResult, containerHarness, onScrollStateChange } @@ -131,7 +131,8 @@ describe('useAutoScroll', () => { act(() => { renderResult.rerender({ - containerRef: { current: containerHarness.div }, + container: containerHarness.div, + isContentReady: true, messages: newMessages, sessionId: 'session-1', contentVersion: newMessages.length, @@ -165,7 +166,8 @@ describe('useAutoScroll', () => { act(() => { renderResult.rerender({ - containerRef: { current: containerHarness.div }, + container: containerHarness.div, + isContentReady: true, messages, sessionId: 'session-1', contentVersion: messages.length + 1, @@ -362,7 +364,8 @@ describe('useAutoScroll', () => { act(() => { renderResult.rerender({ - containerRef: { current: containerHarness.div }, + container: containerHarness.div, + isContentReady: true, messages, sessionId: 'session-1', contentVersion: messages.length + 1, @@ -401,7 +404,8 @@ describe('useAutoScroll', () => { act(() => { renderResult.rerender({ - containerRef: { current: containerHarness.div }, + container: containerHarness.div, + isContentReady: true, messages, sessionId: 'session-1', contentVersion: messages.length + 1, @@ -436,7 +440,8 @@ describe('useAutoScroll', () => { act(() => { renderResult.rerender({ - containerRef: { current: containerHarness.div }, + container: containerHarness.div, + isContentReady: true, messages, sessionId: 'session-1', contentVersion: messages.length + 1, @@ -495,7 +500,8 @@ describe('useAutoScroll', () => { act(() => { renderResult.rerender({ - containerRef: { current: containerHarness.div }, + container: containerHarness.div, + isContentReady: true, messages: newMessages, sessionId: 'session-1', contentVersion: newMessages.length, @@ -537,7 +543,8 @@ describe('useAutoScroll', () => { act(() => { renderResult.rerender({ - containerRef: { current: containerHarness.div }, + container: containerHarness.div, + isContentReady: true, messages: olderPage, sessionId: 'session-1', contentVersion: olderPage.length, @@ -576,7 +583,8 @@ describe('useAutoScroll', () => { act(() => { renderResult.rerender({ - containerRef: { current: containerHarness.div }, + container: containerHarness.div, + isContentReady: true, messages: appended, sessionId: 'session-1', contentVersion: appended.length, @@ -593,7 +601,7 @@ describe('useAutoScroll', () => { it('resets state when sessionId changes', () => { const messages = [createMessage('1', 'user')] - const { renderResult, containerHarness } = setupHook(messages, 'session-1') + const { renderResult, containerHarness } = setupHook(messages, { sessionId: 'session-1' }) act(() => { containerHarness.div.dispatchEvent( @@ -608,7 +616,8 @@ describe('useAutoScroll', () => { act(() => { renderResult.rerender({ - containerRef: { current: containerHarness.div }, + container: containerHarness.div, + isContentReady: true, messages: newMessages, sessionId: 'session-2', contentVersion: newMessages.length, @@ -622,4 +631,72 @@ describe('useAutoScroll', () => { expect(containerHarness.getScrollTop()).toBe(containerHarness.div.scrollHeight - containerHarness.div.clientHeight) }) + + it('scrolls to bottom and tracks user scroll once the container mounts after messages arrive', () => { + const messages = [createMessage('1', 'user'), createMessage('2', 'assistant')] + const { renderResult, containerHarness, onScrollStateChange } = setupHook(messages, { container: null }) + + act(() => { + vi.advanceTimersByTime(100) + }) + + expect(containerHarness.getScrollTop()).toBe(0) + + act(() => { + renderResult.rerender({ + container: containerHarness.div, + isContentReady: true, + messages, + sessionId: 'session-1', + contentVersion: messages.length, + onScrollStateChange, + }) + }) + + act(() => { + vi.advanceTimersByTime(100) + }) + + expect(containerHarness.getScrollTop()).toBe(containerHarness.div.scrollHeight - containerHarness.div.clientHeight) + + act(() => { + containerHarness.setScrollTop(100) + containerHarness.div.dispatchEvent( + new WheelEvent('wheel', { + deltaY: -50, + bubbles: true, + }) + ) + }) + + expect(onScrollStateChange).toHaveBeenCalledWith(true) + }) + + it('waits for ready content before the initial scroll', () => { + const messages = [createMessage('1', 'user'), createMessage('2', 'assistant')] + const { renderResult, containerHarness, onScrollStateChange } = setupHook(messages, { isContentReady: false }) + + act(() => { + vi.advanceTimersByTime(100) + }) + + expect(containerHarness.getScrollTop()).toBe(0) + + act(() => { + renderResult.rerender({ + container: containerHarness.div, + isContentReady: true, + messages, + sessionId: 'session-1', + contentVersion: messages.length, + onScrollStateChange, + }) + }) + + act(() => { + vi.advanceTimersByTime(100) + }) + + expect(containerHarness.getScrollTop()).toBe(containerHarness.div.scrollHeight - containerHarness.div.clientHeight) + }) }) diff --git a/frontend/src/hooks/useAutoScroll.ts b/frontend/src/hooks/useAutoScroll.ts index e801256e4..daf3140ee 100644 --- a/frontend/src/hooks/useAutoScroll.ts +++ b/frontend/src/hooks/useAutoScroll.ts @@ -7,10 +7,11 @@ const SHOW_SCROLL_BUTTON_THRESHOLD_PX = 120 const SCROLL_TO_BOTTOM_FRAME_COUNT = 2 interface UseAutoScrollOptions { - containerRef?: React.RefObject + container?: HTMLDivElement | null messages?: SessionMessageInfo[] sessionId?: string contentVersion?: number + isContentReady: boolean onScrollStateChange?: (isScrolledUp: boolean) => void } @@ -20,10 +21,11 @@ interface UseAutoScrollReturn { } export function useAutoScroll({ - containerRef, + container, messages, sessionId, contentVersion, + isContentReady, onScrollStateChange }: UseAutoScrollOptions): UseAutoScrollReturn { const lastMessageCountRef = useRef(0) @@ -42,7 +44,7 @@ export function useAutoScroll({ onScrollStateChangeRef.current = onScrollStateChange const scrollToBottom = useCallback(() => { - if (!containerRef?.current) return + if (!container) return userScrolledAtRef.current = 0 userDisengagedRef.current = false isScrollButtonVisibleRef.current = false @@ -52,8 +54,7 @@ export function useAutoScroll({ let frameCount = 0 const scrollAfterLayout = () => { if (scrollRequestIdRef.current !== scrollRequestId) return - if (!containerRef?.current) return - containerRef.current.scrollTop = containerRef.current.scrollHeight + container.scrollTop = container.scrollHeight frameCount += 1 if (frameCount < SCROLL_TO_BOTTOM_FRAME_COUNT) { requestAnimationFrame(scrollAfterLayout) @@ -61,10 +62,9 @@ export function useAutoScroll({ } requestAnimationFrame(scrollAfterLayout) onScrollStateChangeRef.current?.(false) - }, [containerRef]) + }, [container]) const markDisengaged = useCallback(() => { - const container = containerRef?.current if (!container) return const distanceFromBottom = container.scrollHeight - container.scrollTop - container.clientHeight userScrolledAtRef.current = Date.now() @@ -74,16 +74,15 @@ export function useAutoScroll({ onScrollStateChangeRef.current?.(true) } userDisengagedRef.current = true - }, [containerRef]) + }, [container]) const scrollByUser = useCallback((delta: number) => { - const container = containerRef?.current if (!container) return if (delta < 0) { markDisengaged() } container.scrollBy({ top: delta }) - }, [containerRef, markDisengaged]) + }, [container, markDisengaged]) useEffect(() => { lastMessageCountRef.current = 0 @@ -96,12 +95,10 @@ export function useAutoScroll({ }, [sessionId]) useEffect(() => { - const container = containerRef?.current if (!container) return const updateScrollButtonVisibility = () => { - const container = containerRef?.current - if (!container || !userDisengagedRef.current || isScrollButtonVisibleRef.current) return + if (!userDisengagedRef.current || isScrollButtonVisibleRef.current) return const distanceFromBottom = container.scrollHeight - container.scrollTop - container.clientHeight if (distanceFromBottom > SHOW_SCROLL_BUTTON_THRESHOLD_PX) { isScrollButtonVisibleRef.current = true @@ -206,10 +203,10 @@ export function useAutoScroll({ container.removeEventListener('keydown', handleKeyDown) container.removeEventListener('scroll', handleScroll) } - }, [containerRef, sessionId, messages, markDisengaged]) + }, [container, sessionId, messages, markDisengaged]) useEffect(() => { - if (!containerRef?.current || !messages) return + if (!container || !messages) return const currentCount = messages.length const prevCount = lastMessageCountRef.current @@ -219,6 +216,8 @@ export function useAutoScroll({ const previousLastMessageId = lastMessageIdRef.current lastMessageIdRef.current = lastMessage?.id + if (!isContentReady) return + if (!hasInitialScrolledRef.current && currentCount > 0) { hasInitialScrolledRef.current = true scrollToBottom() @@ -238,7 +237,7 @@ export function useAutoScroll({ const recentlyScrolled = timeSinceUserScroll < SCROLL_LOCK_MS if (userDisengagedRef.current) { - const distanceFromBottom = containerRef.current.scrollHeight - containerRef.current.scrollTop - containerRef.current.clientHeight + const distanceFromBottom = container.scrollHeight - container.scrollTop - container.clientHeight if (!isScrollButtonVisibleRef.current && distanceFromBottom > SHOW_SCROLL_BUTTON_THRESHOLD_PX) { isScrollButtonVisibleRef.current = true onScrollStateChangeRef.current?.(true) @@ -251,7 +250,7 @@ export function useAutoScroll({ } scrollToBottom() - }, [messages, containerRef, scrollToBottom, contentVersion]) + }, [messages, container, isContentReady, scrollToBottom, contentVersion]) return { scrollToBottom, scrollByUser } } diff --git a/frontend/src/pages/SessionDetail.tsx b/frontend/src/pages/SessionDetail.tsx index 7cc5eb483..444af48db 100644 --- a/frontend/src/pages/SessionDetail.tsx +++ b/frontend/src/pages/SessionDetail.tsx @@ -136,6 +136,7 @@ export function SessionDetail() { const { preferences, updateSettings } = useSettings(); const { open: openSettings, openTab: openSettingsTab } = useSettingsDialog(); const messageContainerRef = useRef(null); + const [messageContainer, setMessageContainer] = useState(null); const promptInputRef = useRef(null); const [sessionsDialogOpen, setSessionsDialogOpen] = useState(false); const [fileBrowserOpen, setFileBrowserOpen] = useDialogParam('files'); @@ -302,12 +303,19 @@ export function SessionDetail() { }, [messagePickerMode, sessionId, revertMessageID, serverId]); const messagesContentVersion = useMemo(() => getMessagesContentVersion(messages), [messages]); + const isTranscriptLoading = repoLoading || sessionLoading || messagesLoading; + + const attachMessageContainer = useCallback((element: HTMLDivElement | null) => { + messageContainerRef.current = element; + setMessageContainer(element); + }, []); const { scrollToBottom, scrollByUser } = useAutoScroll({ - containerRef: messageContainerRef, + container: messageContainer, messages, sessionId, contentVersion: messagesContentVersion, + isContentReady: !isTranscriptLoading, onScrollStateChange: setShowScrollButton }); @@ -861,8 +869,8 @@ export function SessionDetail() {
-
- {repoLoading || sessionLoading || messagesLoading ? ( +
+ {isTranscriptLoading ? ( ) : sessionDirectory ? ( { showScrollButton: boolean promptContent?: boolean sessionActive?: boolean + messagesLoading?: boolean }) => { mocks.useMobile.mockReturnValue(opts.mobile) mocks.useSessionStatusForSession.mockReturnValue({ type: opts.sessionActive ? 'busy' : 'idle' }) @@ -302,7 +303,7 @@ describe('SessionDetail scroll floating button', () => { messages: [], pending: [], status: opts.sessionActive ? 'busy' : 'idle', - isLoading: false, + isLoading: opts.messagesLoading ?? false, fetchOlder: vi.fn(), hasOlder: false, }) @@ -460,4 +461,26 @@ describe('SessionDetail scroll floating button', () => { reportObservedHeight(140) expect(paddingBottom()).toBe('156px') }) + + it('marks the transcript content as not ready while messages load and ready afterwards', async () => { + const { buildTree, rerender } = renderWith({ mobile: true, showScrollButton: false, messagesLoading: true }) + + await waitFor(() => { + expect(mocks.useAutoScroll.mock.calls.at(-1)?.[0]).toMatchObject({ isContentReady: false }) + }) + + mocks.useSessionTranscript.mockReturnValue({ + messages: [], + pending: [], + status: 'idle', + isLoading: false, + fetchOlder: vi.fn(), + hasOlder: false, + }) + rerender(buildTree()) + + await waitFor(() => { + expect(mocks.useAutoScroll.mock.calls.at(-1)?.[0]).toMatchObject({ isContentReady: true }) + }) + }) }) From a2ae98107d3aa9369d0ee78643e4ac3aa0520d51 Mon Sep 17 00:00:00 2001 From: Chris Scott <99081550+chriswritescode-dev@users.noreply.github.com> Date: Wed, 21 Oct 2026 14:07:10 -0400 Subject: [PATCH 2/5] feat(frontend): replace the mobile voice overlay with an in-row recorder --- .../message/PromptInput.stt.test.tsx | 157 +++++++++- .../src/components/message/PromptInput.tsx | 273 ++++++++++++------ .../components/message/VoiceRecorderStrip.tsx | 104 +++++++ .../components/message/VoiceStatusOverlay.tsx | 77 ----- frontend/src/index.css | 10 +- 5 files changed, 453 insertions(+), 168 deletions(-) create mode 100644 frontend/src/components/message/VoiceRecorderStrip.tsx delete mode 100644 frontend/src/components/message/VoiceStatusOverlay.tsx diff --git a/frontend/src/components/message/PromptInput.stt.test.tsx b/frontend/src/components/message/PromptInput.stt.test.tsx index ce6fbcd4d..5ed0b6084 100644 --- a/frontend/src/components/message/PromptInput.stt.test.tsx +++ b/frontend/src/components/message/PromptInput.stt.test.tsx @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { render, screen, fireEvent, act, waitFor } from '@testing-library/react' +import { render, screen, fireEvent, act, waitFor, within } from '@testing-library/react' import { QueryClient, QueryClientProvider } from '@tanstack/react-query' import { PromptInput } from './PromptInput' import { createCommandActionsMock } from '@/test/test-utils' @@ -466,7 +466,7 @@ describe('PromptInput STT Gesture Tests', () => { }) }) - it('pointer down while recording sets up swipe gesture', async () => { + it('pointer down while recording starts a drag gesture', async () => { renderComponent({ isRecording: true }) const container = getMobileVoiceButtonContainer() @@ -588,4 +588,157 @@ describe('PromptInput STT Gesture Tests', () => { expect(queryMobileVoiceButton()).not.toBeNull() }) }) + + describe('in-row mobile voice recorder', () => { + const renderRecorder = (sttOverrides: Partial = {}) => { + const queryClient = createTestQueryClient() + const buildTree = (overrides: Partial) => { + mocks.useSTT.mockReturnValue({ + isRecording: false, + isProcessing: false, + isSupported: true, + isEnabled: true, + interimTranscript: '', + transcript: '', + startRecording: mockStartRecording, + stopRecording: mockStopRecording, + abortRecording: mockAbortRecording, + reset: mockReset, + clear: mockClear, + ...overrides, + } as unknown as MockSTTReturn) + return ( + + defaultProps.onPromptChange(hasContent)} /> + + ) + } + + const view = render(buildTree(sttOverrides)) + return { rerender: (overrides: Partial) => view.rerender(buildTree(overrides)) } + } + + const getControlsRow = () => screen.getByTestId('composer-controls-row') + + it('swaps the controls row for discard, strip, mic and send while recording', () => { + renderComponent({ isRecording: true }) + + const row = getControlsRow() + + expect(within(row).getByRole('button', { name: 'Discard recording' })).toBeInTheDocument() + expect(within(row).getByRole('button', { name: 'Send recording' })).toBeInTheDocument() + expect(within(row).getByText('0:00')).toBeInTheDocument() + expect(within(row).getByRole('status')).toHaveTextContent('‹ discard · send ›') + }) + + it('hides the mobile insert keys and renders nothing above the mic', () => { + renderComponent({ isRecording: true }) + + fireEvent.focus(screen.getByPlaceholderText('Send a message...')) + + expect(screen.queryByRole('toolbar', { name: 'Insert' })).not.toBeInTheDocument() + expect(getMobileVoiceButtonContainer().children).toHaveLength(1) + }) + + it('stops recording and marks the transcript for auto-send when send is tapped', async () => { + const view = renderRecorder({ isRecording: true }) + + fireEvent.click(screen.getByRole('button', { name: 'Send recording' })) + + expect(mockStopRecording).toHaveBeenCalledTimes(1) + + view.rerender({ isRecording: false, isProcessing: true }) + + expect(screen.getByText('Transcribing & sending…')).toBeInTheDocument() + + view.rerender({ isRecording: false, isProcessing: false, transcript: 'hello there' }) + + await waitFor(() => { + const [variables] = mocks.useSendPromptMutate.mock.calls[0] as [{ text: string }] + expect(variables.text).toBe('hello there') + }) + }) + + it('upgrades a transcription in progress to send when send is tapped', async () => { + const view = renderRecorder({ isRecording: false, isProcessing: true }) + + expect(screen.getByText('Transcribing…')).toBeInTheDocument() + + fireEvent.click(screen.getByRole('button', { name: 'Send recording' })) + + expect(mockStopRecording).not.toHaveBeenCalled() + + view.rerender({ isRecording: false, isProcessing: false, transcript: 'upgraded text' }) + + await waitFor(() => { + const [variables] = mocks.useSendPromptMutate.mock.calls[0] as [{ text: string }] + expect(variables.text).toBe('upgraded text') + }) + }) + + it('aborts recording when discard is tapped', () => { + renderComponent({ isRecording: true }) + mockAbortRecording.mockClear() + + const discardButton = screen.getByRole('button', { name: 'Discard recording' }) + + fireEvent.pointerDown(discardButton) + fireEvent.click(discardButton) + + expect(mockAbortRecording).toHaveBeenCalledTimes(1) + expect(mockStopRecording).not.toHaveBeenCalled() + }) + + it('arms send on a right drag and sends on release', async () => { + renderComponent({ isRecording: true }) + + const container = getMobileVoiceButtonContainer() + + await act(async () => { + fireEvent.pointerDown(container, { clientX: 100 }) + fireEvent.pointerMove(container, { clientX: 130 }) + }) + + expect(screen.getByText('Release to send')).toBeInTheDocument() + expect(mockStopRecording).not.toHaveBeenCalled() + + await act(async () => { + fireEvent.pointerUp(container, { clientX: 130 }) + }) + + expect(mockStopRecording).toHaveBeenCalledTimes(1) + }) + + it('arms discard on a left drag past the threshold and aborts on release', async () => { + renderComponent({ isRecording: true }) + mockAbortRecording.mockClear() + + const container = getMobileVoiceButtonContainer() + + await act(async () => { + fireEvent.pointerDown(container, { clientX: 200 }) + fireEvent.pointerMove(container, { clientX: 100 }) + }) + + expect(screen.getByText('Release to discard')).toBeInTheDocument() + + await act(async () => { + fireEvent.pointerUp(container, { clientX: 100 }) + }) + + expect(mockAbortRecording).toHaveBeenCalledTimes(1) + expect(mockStopRecording).not.toHaveBeenCalled() + }) + + it('leaves the desktop composer without the recorder row', () => { + mocks.useMobile.mockReturnValue(false) + + renderComponent({ isRecording: true }) + + expect(screen.queryByRole('button', { name: 'Discard recording' })).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Send recording' })).not.toBeInTheDocument() + expect(screen.queryByText('Release to send')).not.toBeInTheDocument() + expect(document.querySelector('[data-voice-variant="desktop"]')).not.toBeNull() + }) + }) }) diff --git a/frontend/src/components/message/PromptInput.tsx b/frontend/src/components/message/PromptInput.tsx index df0be4e56..2b9d0a55b 100644 --- a/frontend/src/components/message/PromptInput.tsx +++ b/frontend/src/components/message/PromptInput.tsx @@ -19,7 +19,7 @@ import { FINE_POINTER_MEDIA_QUERY, useMediaQuery } from '@/hooks/useMediaQuery' import { usePermissions } from '@/contexts/EventContext' import { isLocalOpenCodeServer, useOpenCodeServerId } from '@/contexts/OpenCodeServerContext' -import { ArrowDown, Upload, X, Mic, MicOff, Target } from 'lucide-react' +import { ArrowDown, ArrowUp, Upload, X, Mic, MicOff, Target } from 'lucide-react' import { SquareFill } from '@/components/ui/square-fill' import { IconToggleButton, getIconToggleButtonClassName } from '@/components/ui/icon-toggle-button' @@ -28,7 +28,7 @@ import { PromptSuggestions, type PromptSuggestion } from './PromptSuggestions' import { ComposerActivityBar } from './ComposerActivityBar' import { ModelQuickSelect } from '@/components/model/ModelQuickSelect' import { AgentQuickSelect } from '@/components/agent/AgentQuickSelect' -import { VoiceStatusOverlay, type VoiceStatusOverlayState } from './VoiceStatusOverlay' +import { VoiceRecorderStrip, type VoiceRecorderStripState } from './VoiceRecorderStrip' import { PermissionModeToggle } from '@/components/session/PermissionModeToggle' import { SessionSandboxMenu } from '@/components/session/SessionSandboxMenu' import { ComposerToolsMenu } from './ComposerToolsMenu' @@ -104,8 +104,11 @@ const COMPOSER_MODE_STYLES: Record ranges.map(([start, end]) => [start + offset, end + offset]) -const VOICE_SEND_SWIPE_ARM_THRESHOLD = 24 -const VOICE_SEND_SWIPE_DISARM_THRESHOLD = 8 +const VOICE_SEND_DRAG_ARM_THRESHOLD = 24 +const VOICE_SEND_DRAG_DISARM_THRESHOLD = 8 +const VOICE_CANCEL_DRAG_ARM_THRESHOLD = 96 +const VOICE_CANCEL_DRAG_DISARM_THRESHOLD = VOICE_CANCEL_DRAG_ARM_THRESHOLD - 8 +const VOICE_CLICK_SUPPRESSION_MS = 400 type VoiceButtonVariant = 'desktop' | 'mobile' @@ -156,15 +159,18 @@ export const PromptInput = memo(forwardRef( const [selectedCommandIndex, setSelectedCommandIndex] = useState(0) const [localMode, setLocalMode] = useState(null) const [isTogglingRecording, setIsTogglingRecording] = useState(false) - const [isVoiceSwipeArmed, setIsVoiceSwipeArmed] = useState(false) + const [isVoiceSendArmed, setIsVoiceSendArmed] = useState(false) + const [isVoiceCancelArmed, setIsVoiceCancelArmed] = useState(false) const [isVoiceAutoSendPending, setIsVoiceAutoSendPending] = useState(false) const [isVoiceAutoSendWaitingForTranscript, setIsVoiceAutoSendWaitingForTranscript] = useState(false) + const [voiceRecordingStartedAt, setVoiceRecordingStartedAt] = useState(null) const [isPromptFocused, setIsPromptFocused] = useState(false) const promptBlurTimerRef = useRef | null>(null) const pendingCommandSubmitRef = useRef(false) const lastAddedTranscriptRef = useRef('') - const voiceGestureStartYRef = useRef(null) - const voiceSwipeArmedRef = useRef(false) + const voiceGestureStartXRef = useRef(null) + const voiceSendArmedRef = useRef(false) + const voiceCancelArmedRef = useRef(false) const pendingVoiceAutoSubmitRef = useRef(false) const ignoreVoiceClickUntilRef = useRef(0) const voiceStartRequestRef = useRef(0) @@ -189,7 +195,9 @@ export const PromptInput = memo(forwardRef( const textareaRef = useRef(null) const fileInputRef = useRef(null) const voiceButtonContainerRef = useRef(null) + const composerControlsRowRef = useRef(null) const hasFinePointer = useMediaQuery(FINE_POINTER_MEDIA_QUERY) + const isMobile = useMobile() useEffect(() => { if (!hasFinePointer) return @@ -199,11 +207,13 @@ export const PromptInput = memo(forwardRef( }, [hasFinePointer, sessionID]) const resetVoiceGestureState = useCallback(() => { - voiceGestureStartYRef.current = null - voiceSwipeArmedRef.current = false + voiceGestureStartXRef.current = null + voiceSendArmedRef.current = false + voiceCancelArmedRef.current = false pendingVoiceAutoSubmitRef.current = false ignoreVoiceClickUntilRef.current = 0 - setIsVoiceSwipeArmed(false) + setIsVoiceSendArmed(false) + setIsVoiceCancelArmed(false) setIsVoiceAutoSendPending(false) setIsVoiceAutoSendWaitingForTranscript(false) }, []) @@ -617,8 +627,10 @@ export const PromptInput = memo(forwardRef( pendingVoiceAutoSubmitRef.current = false setIsVoiceAutoSendPending(false) setIsVoiceAutoSendWaitingForTranscript(false) - setIsVoiceSwipeArmed(false) - voiceSwipeArmedRef.current = false + setIsVoiceSendArmed(false) + voiceSendArmedRef.current = false + setIsVoiceCancelArmed(false) + voiceCancelArmedRef.current = false setIsTogglingRecording(true) const started = await startRecording() @@ -650,8 +662,10 @@ export const PromptInput = memo(forwardRef( pendingVoiceAutoSubmitRef.current = false setIsVoiceAutoSendPending(false) setIsVoiceAutoSendWaitingForTranscript(false) - setIsVoiceSwipeArmed(false) - voiceSwipeArmedRef.current = false + setIsVoiceSendArmed(false) + voiceSendArmedRef.current = false + setIsVoiceCancelArmed(false) + voiceCancelArmedRef.current = false stopRecording() return } @@ -659,15 +673,54 @@ export const PromptInput = memo(forwardRef( await startVoiceRecording() } + const cancelVoiceInput = useCallback(() => { + voiceStartRequestRef.current += 1 + resetVoiceGestureState() + setIsTogglingRecording(false) + + if (isRecording || isTogglingRecording || isProcessing) { + abortRecording() + } + }, [abortRecording, isProcessing, isRecording, isTogglingRecording, resetVoiceGestureState]) + + const clearVoiceGestureArmedState = useCallback(() => { + ignoreVoiceClickUntilRef.current = Date.now() + VOICE_CLICK_SUPPRESSION_MS + voiceSendArmedRef.current = false + voiceCancelArmedRef.current = false + setIsVoiceSendArmed(false) + setIsVoiceCancelArmed(false) + }, []) + + const armVoiceAutoSend = useCallback(() => { + pendingVoiceAutoSubmitRef.current = true + setIsVoiceAutoSendPending(true) + setIsVoiceAutoSendWaitingForTranscript(true) + + if (typeof navigator !== 'undefined' && navigator.vibrate) { + navigator.vibrate([10, 20, 10]) + } + }, []) + + const handleVoiceSendRequest = useCallback(() => { + clearVoiceGestureArmedState() + armVoiceAutoSend() + + if (isRecording) { + stopRecording() + } + }, [armVoiceAutoSend, clearVoiceGestureArmedState, isRecording, stopRecording]) + const handleVoicePointerDown = (event: ReactPointerEvent) => { if ((event.pointerType === 'mouse' && event.button !== 0) || isProcessing || !isRecording) { return } event.preventDefault() - voiceGestureStartYRef.current = event.clientY - voiceSwipeArmedRef.current = false - setIsVoiceSwipeArmed(false) + voiceGestureStartXRef.current = event.clientX + voiceSendArmedRef.current = false + voiceCancelArmedRef.current = false + setIsVoiceSendArmed(false) + setIsVoiceCancelArmed(false) if (event.currentTarget.setPointerCapture) { event.currentTarget.setPointerCapture(event.pointerId) @@ -675,21 +728,33 @@ export const PromptInput = memo(forwardRef( } const handleVoicePointerMove = (event: ReactPointerEvent) => { - if (!isRecording || voiceGestureStartYRef.current === null) { + if (!isRecording || voiceGestureStartXRef.current === null) { return } event.preventDefault() - const deltaY = voiceGestureStartYRef.current - event.clientY - const nextIsSwipeArmed = voiceSwipeArmedRef.current - ? deltaY >= VOICE_SEND_SWIPE_DISARM_THRESHOLD - : deltaY >= VOICE_SEND_SWIPE_ARM_THRESHOLD + const deltaX = event.clientX - voiceGestureStartXRef.current + const nextIsSendArmed = voiceSendArmedRef.current + ? deltaX >= VOICE_SEND_DRAG_DISARM_THRESHOLD + : deltaX >= VOICE_SEND_DRAG_ARM_THRESHOLD + const nextIsCancelArmed = !nextIsSendArmed && (voiceCancelArmedRef.current + ? deltaX <= -VOICE_CANCEL_DRAG_DISARM_THRESHOLD + : deltaX <= -VOICE_CANCEL_DRAG_ARM_THRESHOLD) + + if (nextIsSendArmed !== voiceSendArmedRef.current) { + voiceSendArmedRef.current = nextIsSendArmed + setIsVoiceSendArmed(nextIsSendArmed) + + if (nextIsSendArmed && typeof navigator !== 'undefined' && navigator.vibrate) { + navigator.vibrate(10) + } + } - if (nextIsSwipeArmed !== voiceSwipeArmedRef.current) { - voiceSwipeArmedRef.current = nextIsSwipeArmed - setIsVoiceSwipeArmed(nextIsSwipeArmed) + if (nextIsCancelArmed !== voiceCancelArmedRef.current) { + voiceCancelArmedRef.current = nextIsCancelArmed + setIsVoiceCancelArmed(nextIsCancelArmed) - if (nextIsSwipeArmed && typeof navigator !== 'undefined' && navigator.vibrate) { + if (nextIsCancelArmed && typeof navigator !== 'undefined' && navigator.vibrate) { navigator.vibrate(10) } } @@ -700,17 +765,18 @@ export const PromptInput = memo(forwardRef( event.currentTarget.releasePointerCapture(event.pointerId) } - if (voiceGestureStartYRef.current === null) { + if (voiceGestureStartXRef.current === null) { return } event.preventDefault() - voiceGestureStartYRef.current = null + voiceGestureStartXRef.current = null - if (canceled || !voiceSwipeArmedRef.current) { - ignoreVoiceClickUntilRef.current = Date.now() + 400 - voiceSwipeArmedRef.current = false - setIsVoiceSwipeArmed(false) + const wasSendArmed = voiceSendArmedRef.current + const wasCancelArmed = voiceCancelArmedRef.current + clearVoiceGestureArmedState() + + if (canceled || (!wasSendArmed && !wasCancelArmed)) { pendingVoiceAutoSubmitRef.current = false setIsVoiceAutoSendPending(false) setIsVoiceAutoSendWaitingForTranscript(false) @@ -718,30 +784,14 @@ export const PromptInput = memo(forwardRef( return } - ignoreVoiceClickUntilRef.current = Date.now() + 400 - voiceSwipeArmedRef.current = false - setIsVoiceSwipeArmed(false) - pendingVoiceAutoSubmitRef.current = true - setIsVoiceAutoSendPending(true) - setIsVoiceAutoSendWaitingForTranscript(true) - - if (typeof navigator !== 'undefined' && navigator.vibrate) { - navigator.vibrate([10, 20, 10]) + if (wasCancelArmed) { + cancelVoiceInput() + return } - stopRecording() + handleVoiceSendRequest() } - const cancelVoiceInput = useCallback(() => { - voiceStartRequestRef.current += 1 - resetVoiceGestureState() - setIsTogglingRecording(false) - - if (isRecording || isTogglingRecording || isProcessing) { - abortRecording() - } - }, [abortRecording, isProcessing, isRecording, isTogglingRecording, resetVoiceGestureState]) - useEffect(() => { const isVoiceFeedbackVisible = isRecording || isTogglingRecording || isProcessing || isVoiceAutoSendPending @@ -760,11 +810,17 @@ export const PromptInput = memo(forwardRef( return } + if (isMobile && composerControlsRowRef.current?.contains(target)) { + return + } + if (isRecording) { - voiceGestureStartYRef.current = null - voiceSwipeArmedRef.current = false + voiceGestureStartXRef.current = null + voiceSendArmedRef.current = false + voiceCancelArmedRef.current = false pendingVoiceAutoSubmitRef.current = false - setIsVoiceSwipeArmed(false) + setIsVoiceSendArmed(false) + setIsVoiceCancelArmed(false) setIsVoiceAutoSendPending(false) setIsVoiceAutoSendWaitingForTranscript(false) stopRecording() @@ -786,7 +842,7 @@ export const PromptInput = memo(forwardRef( return () => { document.removeEventListener('pointerdown', handleOutsidePointerDown, true) } - }, [cancelVoiceInput, isProcessing, isRecording, isTogglingRecording, isVoiceAutoSendPending, stopRecording]) + }, [cancelVoiceInput, isMobile, isProcessing, isRecording, isTogglingRecording, isVoiceAutoSendPending, stopRecording]) useEffect(() => { const textToUse = transcript || interimTranscript @@ -821,6 +877,10 @@ export const PromptInput = memo(forwardRef( } }, [isTogglingRecording]) + useEffect(() => { + setVoiceRecordingStartedAt(isRecording ? Date.now() : null) + }, [isRecording]) + useEffect(() => { if (!pendingVoiceAutoSubmitRef.current || isVoiceAutoSendWaitingForTranscript || isRecording || isProcessing || !prompt.trim()) { return @@ -832,6 +892,14 @@ export const PromptInput = memo(forwardRef( handleSubmitRef.current() }, [prompt, isRecording, isProcessing, isVoiceAutoSendWaitingForTranscript]) + useEffect(() => { + if (!isVoiceAutoSendWaitingForTranscript || isRecording || isProcessing || !transcript.trim()) { + return + } + + setIsVoiceAutoSendWaitingForTranscript(false) + }, [isRecording, isProcessing, transcript, isVoiceAutoSendWaitingForTranscript]) + useEffect(() => { if (pendingVoiceAutoSubmitRef.current && !isRecording && !isProcessing && !transcript.trim() && !interimTranscript.trim()) { pendingVoiceAutoSubmitRef.current = false @@ -1255,7 +1323,6 @@ if (isIOS && isSecureContext && navigator.clipboard && navigator.clipboard.read) return info?.name || model.modelID || currentModel }, [currentModel, model, providersData]) - const isMobile = useMobile() const { setShowDialog, hasForSession: hasPermissionsForSession } = usePermissions() const hasPendingPermissionForSession = hasPermissionsForSession(sessionID) const { hasVariants, currentVariant, cycleVariant } = useVariants(directory, modelSelectionSession) @@ -1354,34 +1421,25 @@ if (isIOS && isSecureContext && navigator.clipboard && navigator.clipboard.read) : 'Goal mode: the next message becomes the objective' const isGoalModeDisabled = lockedReason !== null || hasOpenGoal || isBashMode || startGoal.isPending const toggleGoalArmed = () => setIsGoalArmed((value) => !value) - const voiceFeedbackState: VoiceStatusOverlayState | null = isTogglingRecording + const voiceFeedbackState: VoiceRecorderStripState | null = isTogglingRecording ? 'starting' : isProcessing ? isVoiceAutoSendPending ? 'sending' : 'processing' : isRecording - ? isVoiceSwipeArmed - ? 'readyToSend' - : 'recording' + ? isVoiceCancelArmed + ? 'cancelArmed' + : isVoiceSendArmed + ? 'readyToSend' + : 'recording' : isVoiceAutoSendPending ? 'sending' : null - const showVoiceFeedback = voiceFeedbackState !== null + const mobileVoiceRecorderState = isMobile ? voiceFeedbackState : null const hasPromptContent = prompt.trim().length > 0 || imageAttachments.length > 0 const canUseMobileVoice = isMobile && sttEnabled && sttSupported && !hasPendingPermissionForSession - const showMobileKeys = isMobile && isPromptFocused && !hasPendingPermissionForSession - const voiceFeedbackLabel = voiceFeedbackState === 'starting' - ? 'Starting microphone...' - : voiceFeedbackState === 'sending' - ? 'Transcribing and sending...' - : voiceFeedbackState === 'processing' - ? 'Transcribing...' - : voiceFeedbackState === 'readyToSend' - ? 'Release to send' - : voiceFeedbackState === 'recording' - ? 'Recording... swipe up to send' - : null + const showMobileKeys = isMobile && isPromptFocused && !hasPendingPermissionForSession && mobileVoiceRecorderState === null const voiceButtonTitle = voiceFeedbackState === 'starting' ? 'Starting microphone' : voiceFeedbackState === 'sending' @@ -1390,9 +1448,22 @@ if (isIOS && isSecureContext && navigator.clipboard && navigator.clipboard.read) ? 'Transcribing speech' : voiceFeedbackState === 'readyToSend' ? 'Release to send' - : voiceFeedbackState === 'recording' - ? 'Tap to transcribe' - : 'Tap to speak' + : voiceFeedbackState === 'cancelArmed' + ? 'Release to discard' + : voiceFeedbackState === 'recording' + ? 'Tap to transcribe' + : 'Tap to speak' + + const handleSendVoiceRecording = () => { + if (isRecording) { + handleVoiceSendRequest() + return + } + + if (isProcessing && !isVoiceAutoSendPending) { + armVoiceAutoSend() + } + } @@ -1427,10 +1498,6 @@ if (isIOS && isSecureContext && navigator.clipboard && navigator.clipboard.read) className={containerClassName} {...voiceGestureHandlers} > - {!isDesktop && showVoiceFeedback && ( -