From 471be26dfa0a1bfe2122fc04167d8c9adf603017 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 31 Aug 2026 13:53:59 -0700 Subject: [PATCH] fix(logs): retry failed runs from failed block --- .../components/log-details/log-details.tsx | 3 +- .../log-row-context-menu.test.tsx | 10 ++ .../log-row-context-menu.tsx | 5 +- .../app/workspace/[workspaceId]/logs/logs.tsx | 13 +- apps/sim/hooks/queries/logs.test.tsx | 157 +++++++++++++++++- apps/sim/hooks/queries/logs.ts | 47 +++++- apps/sim/lib/logs/retry.test.ts | 114 +++++++++++++ apps/sim/lib/logs/retry.ts | 75 +++++++++ 8 files changed, 409 insertions(+), 15 deletions(-) create mode 100644 apps/sim/lib/logs/retry.test.ts create mode 100644 apps/sim/lib/logs/retry.ts diff --git a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/log-details.tsx b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/log-details.tsx index 1d9348154b8..054129b5509 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/log-details.tsx +++ b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/log-details.tsx @@ -828,7 +828,8 @@ export const LogDetails = memo(function LogDetails({

Log Details

- {log.status === 'failed' && + {onRetryExecution && + log.status === 'failed' && (log.workflow?.id || log.workflowId) && log.trigger !== 'mothership' && ( diff --git a/apps/sim/app/workspace/[workspaceId]/logs/components/log-row-context-menu/log-row-context-menu.test.tsx b/apps/sim/app/workspace/[workspaceId]/logs/components/log-row-context-menu/log-row-context-menu.test.tsx index 764695270ea..85a3e9d5158 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/components/log-row-context-menu/log-row-context-menu.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/logs/components/log-row-context-menu/log-row-context-menu.test.tsx @@ -89,6 +89,7 @@ function renderMenu( props: Partial<{ log: WorkflowLogSummary canCancelExecution: boolean + canRetryExecution: boolean isCancelPending: boolean cancelPendingExecutionId: string }> = {} @@ -100,6 +101,7 @@ function renderMenu( position={{ x: 0, y: 0 }} log={props.log ?? LOG} canCancelExecution={props.canCancelExecution ?? true} + canRetryExecution={props.canRetryExecution ?? true} isCancelPending={props.isCancelPending} cancelPendingExecutionId={props.cancelPendingExecutionId} isFilteredByThisWorkflow={false} @@ -152,3 +154,11 @@ describe('LogRowContextMenu cancellation action', () => { expect(findButton('Stopping…')?.disabled).toBe(true) }) }) + +describe('LogRowContextMenu retry action', () => { + it('hides Retry without edit permission', () => { + renderMenu({ log: { ...LOG, status: 'failed' }, canRetryExecution: false }) + + expect(findButton('Retry')).toBeUndefined() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/logs/components/log-row-context-menu/log-row-context-menu.tsx b/apps/sim/app/workspace/[workspaceId]/logs/components/log-row-context-menu/log-row-context-menu.tsx index 89f26ba71a4..b9bc03fee3b 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/components/log-row-context-menu/log-row-context-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/logs/components/log-row-context-menu/log-row-context-menu.tsx @@ -32,6 +32,7 @@ interface LogRowContextMenuProps { onCancelExecution: () => void onRetryExecution: () => void canCancelExecution: boolean + canRetryExecution: boolean isCancelPending?: boolean cancelPendingExecutionId?: string isRetryPending?: boolean @@ -57,6 +58,7 @@ export const LogRowContextMenu = memo(function LogRowContextMenu({ onCancelExecution, onRetryExecution, canCancelExecution, + canRetryExecution, isCancelPending = false, cancelPendingExecutionId, isRetryPending = false, @@ -78,7 +80,8 @@ export const LogRowContextMenu = memo(function LogRowContextMenu({ (isCancelPending && cancelPendingExecutionId === log?.executionId) const showCancelAction = canCancelExecution && hasExecutionId && hasWorkflow && (isCancellable || isStopping) - const isRetryable = log?.status === 'failed' && hasWorkflow && log?.trigger !== 'mothership' + const isRetryable = + canRetryExecution && log?.status === 'failed' && hasWorkflow && log?.trigger !== 'mothership' return ( !open && onClose()} modal={false}> diff --git a/apps/sim/app/workspace/[workspaceId]/logs/logs.tsx b/apps/sim/app/workspace/[workspaceId]/logs/logs.tsx index 7cdc892da44..f679a6993c2 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/logs.tsx +++ b/apps/sim/app/workspace/[workspaceId]/logs/logs.tsx @@ -597,7 +597,7 @@ export default function Logs() { }, [contextMenuLog]) const cancelExecution = useCancelExecution(workspaceId) - const retryExecution = useRetryExecution() + const retryExecution = useRetryExecution(workspaceId) const handleCancelExecution = useCallback(async () => { const workflowId = contextMenuLog?.workflow?.id || contextMenuLog?.workflowId @@ -617,17 +617,17 @@ export default function Logs() { async (log: WorkflowLogRow | null) => { const workflowId = log?.workflow?.id || log?.workflowId const executionId = log?.executionId - if (!workflowId || !executionId) return + if (!userPermissions.canEdit || !workflowId || !executionId) return try { await retryExecution.mutateAsync({ workflowId, executionId }) toast.success('Retry started') - } catch { - toast.error('Failed to retry execution') + } catch (error) { + toast.error(getErrorMessage(error, 'Failed to retry execution')) } }, // eslint-disable-next-line react-hooks/exhaustive-deps - [] + [userPermissions.canEdit] ) const handleRetryExecution = useCallback(() => { @@ -862,7 +862,7 @@ export default function Logs() { onNavigatePrev={handleNavigatePrev} hasNext={selectedLogIndex >= 0 && selectedLogIndex < logs.length - 1} hasPrev={selectedLogIndex > 0} - onRetryExecution={handleRetrySidebarExecution} + onRetryExecution={userPermissions.canEdit ? handleRetrySidebarExecution : undefined} isRetryPending={retryExecution.isPending} onActiveTabChange={handleActiveTabChange} /> @@ -1270,6 +1270,7 @@ export default function Logs() { onCancelExecution={handleCancelExecution} onRetryExecution={handleRetryExecution} canCancelExecution={userPermissions.canEdit} + canRetryExecution={userPermissions.canEdit} isCancelPending={cancelExecution.isPending} cancelPendingExecutionId={cancelExecution.variables?.executionId} isRetryPending={retryExecution.isPending} diff --git a/apps/sim/hooks/queries/logs.test.tsx b/apps/sim/hooks/queries/logs.test.tsx index 7ec36e6bf3b..9aef29b4cd6 100644 --- a/apps/sim/hooks/queries/logs.test.tsx +++ b/apps/sim/hooks/queries/logs.test.tsx @@ -6,7 +6,8 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockRequestJson } = vi.hoisted(() => ({ +const { mockFetch, mockRequestJson } = vi.hoisted(() => ({ + mockFetch: vi.fn(), mockRequestJson: vi.fn(), })) @@ -16,7 +17,7 @@ vi.mock('@/lib/api/client/request', () => ({ import { getLogByExecutionIdContract } from '@/lib/api/contracts/logs' import { cancelWorkflowExecutionContract } from '@/lib/api/contracts/workflows' -import { useCancelExecution } from '@/hooks/queries/logs' +import { useCancelExecution, useRetryExecution } from '@/hooks/queries/logs' function renderHookWithClient(useHook: () => T): { result: () => T @@ -198,3 +199,155 @@ describe('useCancelExecution', () => { unmount() }) }) + +function failedLogDetail( + children = [ + { + id: 'failed-span', + name: 'Failed block', + type: 'function', + status: 'error', + blockId: 'failed-block', + }, + ] +) { + return { + data: { + executionData: { + workflowInput: { prompt: 'original input' }, + traceSpans: [ + { + id: 'workflow-execution', + name: 'Workflow Execution', + type: 'workflow', + status: 'error', + children, + }, + ], + }, + }, + } +} + +function executionStream(events: object[]): ReadableStream { + return new ReadableStream({ + start(controller) { + for (const event of events) { + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(event)}\n\n`)) + } + controller.close() + }, + }) +} + +describe('useRetryExecution', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('starts the retry from the failed block using the source execution state', async () => { + mockRequestJson.mockResolvedValue(failedLogDetail()) + mockFetch.mockResolvedValue({ + ok: true, + body: executionStream([ + { type: 'execution:started', data: { startTime: '2026-08-31T00:00:00.000Z' } }, + { type: 'block:started', data: { blockId: 'different-block' } }, + { type: 'block:started', data: { blockId: 'failed-block' } }, + ]), + }) + + const { result, unmount } = renderHookWithClient(() => useRetryExecution('workspace-1')) + + await act(async () => { + await result().mutateAsync({ workflowId: 'workflow-1', executionId: 'execution-1' }) + }) + + expect(mockRequestJson).toHaveBeenCalledWith(getLogByExecutionIdContract, { + params: { executionId: 'execution-1' }, + query: { workspaceId: 'workspace-1' }, + signal: undefined, + }) + expect(mockFetch).toHaveBeenCalledWith('/api/workflows/workflow-1/execute', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + inputFromExecutionId: 'execution-1', + triggerType: 'manual', + stream: true, + runFromBlock: { startBlockId: 'failed-block', executionId: 'execution-1' }, + }), + }) + + unmount() + }) + + it('surfaces a streamed run-from-block validation error', async () => { + mockRequestJson.mockResolvedValue(failedLogDetail()) + mockFetch.mockResolvedValue({ + ok: true, + body: executionStream([ + { type: 'execution:started', data: { startTime: '2026-08-31T00:00:00.000Z' } }, + { + type: 'execution:error', + data: { error: 'The failed block no longer exists in the current workflow' }, + }, + ]), + }) + + const { result, unmount } = renderHookWithClient(() => useRetryExecution('workspace-1')) + + await act(async () => { + await expect( + result().mutateAsync({ workflowId: 'workflow-1', executionId: 'execution-1' }) + ).rejects.toThrow('The failed block no longer exists in the current workflow') + }) + + unmount() + }) + + it('does not report success when the selected failed block never starts', async () => { + mockRequestJson.mockResolvedValue(failedLogDetail()) + mockFetch.mockResolvedValue({ + ok: true, + body: executionStream([ + { type: 'execution:started', data: { startTime: '2026-08-31T00:00:00.000Z' } }, + { type: 'execution:completed', data: { success: true } }, + ]), + }) + + const { result, unmount } = renderHookWithClient(() => useRetryExecution('workspace-1')) + + await act(async () => { + await expect( + result().mutateAsync({ workflowId: 'workflow-1', executionId: 'execution-1' }) + ).rejects.toThrow('Retry execution ended before the failed block could start') + }) + + unmount() + }) + + it('does not execute when the source run has multiple terminating failures', async () => { + mockRequestJson.mockResolvedValue( + failedLogDetail([ + { id: 'failure-1', name: 'One', type: 'function', status: 'error', blockId: 'one' }, + { id: 'failure-2', name: 'Two', type: 'function', status: 'error', blockId: 'two' }, + ]) + ) + + const { result, unmount } = renderHookWithClient(() => useRetryExecution('workspace-1')) + + await act(async () => { + await expect( + result().mutateAsync({ workflowId: 'workflow-1', executionId: 'execution-1' }) + ).rejects.toThrow('multiple terminating failures') + }) + expect(mockFetch).not.toHaveBeenCalled() + + unmount() + }) +}) diff --git a/apps/sim/hooks/queries/logs.ts b/apps/sim/hooks/queries/logs.ts index b1d357e2ed3..cabac5c3b81 100644 --- a/apps/sim/hooks/queries/logs.ts +++ b/apps/sim/hooks/queries/logs.ts @@ -24,8 +24,11 @@ import { type WorkflowStats, } from '@/lib/api/contracts/logs' import { cancelWorkflowExecutionContract } from '@/lib/api/contracts/workflows' +import { readSSEEvents } from '@/lib/core/utils/sse' import { getEndDateFromTimeRange, getStartDateFromTimeRange } from '@/lib/logs/filters' import { parseQuery, queryToApiParams } from '@/lib/logs/query-parser' +import { resolveRetryTarget } from '@/lib/logs/retry' +import type { ExecutionEvent } from '@/lib/workflows/executor/execution-events' import type { TimeRange } from '@/stores/logs/filters/types' export type { DashboardStatsResponse, WorkflowStats } @@ -430,7 +433,7 @@ export function useCancelExecution(workspaceId: string) { }) } -export function useRetryExecution() { +export function useRetryExecution(workspaceId: string) { const queryClient = useQueryClient() return useMutation({ mutationFn: async ({ @@ -440,6 +443,12 @@ export function useRetryExecution() { workflowId: string executionId: string }) => { + const detail = await fetchLogByExecutionId(workspaceId, executionId) + const retryTarget = resolveRetryTarget(detail.executionData) + if (!retryTarget.success) { + throw new Error(retryTarget.error) + } + // boundary-raw-fetch: stream response, body is a ReadableStream consumed one chunk at a time const res = await fetch(`/api/workflows/${workflowId}/execute`, { method: 'POST', @@ -448,16 +457,44 @@ export function useRetryExecution() { inputFromExecutionId: executionId, triggerType: 'manual', stream: true, + runFromBlock: { + startBlockId: retryTarget.startBlockId, + executionId, + }, }), }) if (!res.ok) { const data = await res.json().catch(() => ({})) throw new Error(data.error || 'Failed to retry execution') } - const reader = res.body?.getReader() - if (reader) { - await reader.read() - reader.cancel() + if (!res.body) { + throw new Error('Retry execution did not return a stream') + } + + const reader = res.body.getReader() + let retryStarted = false + try { + await readSSEEvents(reader, { + onEvent: (event) => { + if (event.type === 'execution:error') { + throw new Error(event.data.error) + } + if (event.type === 'block:started' && event.data.blockId === retryTarget.startBlockId) { + retryStarted = true + return true + } + if (event.type === 'execution:completed' || event.type === 'execution:paused') { + return true + } + }, + }) + } finally { + await reader.cancel().catch(() => undefined) + reader.releaseLock() + } + + if (!retryStarted) { + throw new Error('Retry execution ended before the failed block could start') } return { started: true } }, diff --git a/apps/sim/lib/logs/retry.test.ts b/apps/sim/lib/logs/retry.test.ts new file mode 100644 index 00000000000..61d15a0491c --- /dev/null +++ b/apps/sim/lib/logs/retry.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from 'vitest' +import type { LogTraceSpan, WorkflowLogDetail } from '@/lib/api/contracts/logs' +import { resolveRetryTarget } from '@/lib/logs/retry' + +function span(overrides: Partial = {}): LogTraceSpan { + return { + id: 'span-1', + name: 'Block', + type: 'function', + ...overrides, + } +} + +function workflowTrace(children: LogTraceSpan[]): LogTraceSpan[] { + return [ + span({ + id: 'workflow-execution', + name: 'Workflow Execution', + type: 'workflow', + status: 'error', + children, + }), + ] +} + +describe('resolveRetryTarget', () => { + it('selects the sole top-level unhandled failure and ignores handled errors', () => { + const executionData: WorkflowLogDetail['executionData'] = { + traceSpans: workflowTrace([ + span({ id: 'success', status: 'success', blockId: 'upstream' }), + span({ id: 'handled', status: 'error', errorHandled: true, blockId: 'handled-block' }), + span({ id: 'failure', status: 'error', blockId: 'failed-block' }), + ]), + } + + expect(resolveRetryTarget(executionData)).toEqual({ + success: true, + startBlockId: 'failed-block', + }) + + expect( + resolveRetryTarget({ + workflowInput: { prompt: 'original input' }, + traceSpans: workflowTrace([ + span({ id: 'trigger-failure', type: 'starter', status: 'error', blockId: 'trigger' }), + ]), + }) + ).toEqual({ success: true, startBlockId: 'trigger' }) + }) + + it('rejects unsupported retry targets', () => { + const unsupportedCases: { + name: string + executionData: WorkflowLogDetail['executionData'] + error: string + }[] = [ + { + name: 'missing trace history', + executionData: {}, + error: 'This run does not include enough execution history to retry from the failed block.', + }, + { + name: 'multiple terminating failures', + executionData: { + traceSpans: workflowTrace([ + span({ id: 'failure-1', status: 'error', blockId: 'failed-1' }), + span({ id: 'failure-2', status: 'error', blockId: 'failed-2' }), + ]), + }, + error: + 'This run has multiple terminating failures and cannot be retried from a single block.', + }, + { + name: 'a grouped parallel failure', + executionData: { + traceSpans: workflowTrace([ + span({ + id: 'parallel-execution', + type: 'parallel', + status: 'error', + blockId: 'parallel-block', + }), + ]), + }, + error: 'Retrying failures inside loops or parallel groups is not supported yet.', + }, + { + name: 'a trigger failure without its original input', + executionData: { + traceSpans: workflowTrace([ + span({ id: 'trigger-failure', type: 'starter', status: 'error', blockId: 'trigger' }), + ]), + }, + error: + 'The original input for this failed trigger is unavailable, so it cannot be retried safely.', + }, + { + name: 'a trigger failure with compacted input', + executionData: { + workflowInput: { _truncated: true, reason: 'execution_data_size_limit' }, + traceSpans: workflowTrace([ + span({ id: 'trigger-failure', type: 'starter', status: 'error', blockId: 'trigger' }), + ]), + }, + error: + 'The original input for this failed trigger is unavailable, so it cannot be retried safely.', + }, + ] + + for (const { name, executionData, error } of unsupportedCases) { + expect(resolveRetryTarget(executionData), name).toEqual({ success: false, error }) + } + }) +}) diff --git a/apps/sim/lib/logs/retry.ts b/apps/sim/lib/logs/retry.ts new file mode 100644 index 00000000000..b8e8cf15361 --- /dev/null +++ b/apps/sim/lib/logs/retry.ts @@ -0,0 +1,75 @@ +import type { LogTraceSpan, WorkflowLogDetail } from '@/lib/api/contracts/logs' +import { isTriggerBlockType } from '@/executor/constants' + +export type RetryTargetResolution = + | { success: true; startBlockId: string } + | { success: false; error: string } + +const MISSING_HISTORY_ERROR = + 'This run does not include enough execution history to retry from the failed block.' +const MULTIPLE_FAILURES_ERROR = + 'This run has multiple terminating failures and cannot be retried from a single block.' +const UNSUPPORTED_NESTED_FAILURE_ERROR = + 'Retrying failures inside loops or parallel groups is not supported yet.' +const MISSING_BLOCK_ERROR = 'The failed block could not be identified safely for this run.' +const MISSING_TRIGGER_INPUT_ERROR = + 'The original input for this failed trigger is unavailable, so it cannot be retried safely.' + +function executionSpans(traceSpans: LogTraceSpan[]): LogTraceSpan[] { + if (traceSpans.length !== 1) return traceSpans + + const [rootSpan] = traceSpans + if (rootSpan.type === 'workflow' && !rootSpan.blockId) { + return rootSpan.children ?? [] + } + + return traceSpans +} + +function isTruncatedExecutionValue(value: unknown): boolean { + return ( + typeof value === 'object' && + value !== null && + '_truncated' in value && + value._truncated === true + ) +} + +/** Resolves the one top-level block that safely represents a failed run's terminating error. */ +export function resolveRetryTarget( + executionData: WorkflowLogDetail['executionData'] +): RetryTargetResolution { + const traceSpans = executionData.traceSpans + if (!traceSpans?.length) { + return { success: false, error: MISSING_HISTORY_ERROR } + } + + const failures = executionSpans(traceSpans).filter( + (span) => span.status === 'error' && span.errorHandled !== true + ) + + if (failures.length === 0) { + return { success: false, error: MISSING_HISTORY_ERROR } + } + if (failures.length > 1) { + return { success: false, error: MULTIPLE_FAILURES_ERROR } + } + + const [failedSpan] = failures + if (failedSpan.type === 'loop' || failedSpan.type === 'parallel') { + return { success: false, error: UNSUPPORTED_NESTED_FAILURE_ERROR } + } + if (!failedSpan.blockId) { + return { success: false, error: MISSING_BLOCK_ERROR } + } + + if ( + isTriggerBlockType(failedSpan.type) && + (executionData.workflowInput === undefined || + isTruncatedExecutionValue(executionData.workflowInput)) + ) { + return { success: false, error: MISSING_TRIGGER_INPUT_ERROR } + } + + return { success: true, startBlockId: failedSpan.blockId } +}