From e1970c429619e93106dfc0fc54fabf7c8d945648 Mon Sep 17 00:00:00 2001 From: tamirms Date: Mon, 26 Oct 2026 00:31:42 +0000 Subject: [PATCH] Report a failed block's error instead of the cancellation it causes In a parallel SortedBuilder or UnsortedBuilder build, a block that fails cancels the workers' context and reaches the writer, which reports it on writerDone. The goroutine dispatching blocks waits on both, and when it saw the context first it returned context.Canceled: about one build in eight with a duplicate key in its first block did. It now returns the context's cause, which is the failure's error, or the build context's error when the build was canceled. The writer now cancels with its error as the cause. Its own errors need that, and so does a block's error when the writer's cancel lands before the failed worker's, which 9 of 14,000 sorted builds did under stress. The teardown harness's duplicate-key scenario no longer accepts context.Canceled; the old code failed it 15 times in 3 runs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DjQed1VX9uGU9PWpAjBiz7 --- builder.go | 2 +- builder_parallel.go | 21 ++++++++++++++++----- builder_unsorted_finish.go | 2 +- parallel_pipeline_fuzz_test.go | 11 ++--------- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/builder.go b/builder.go index 2023327..21231bc 100644 --- a/builder.go +++ b/builder.go @@ -49,7 +49,7 @@ type builder struct { resultChan chan blockResult workerGroup *errgroup.Group workerCtx context.Context - workerCancel context.CancelFunc // Cancels workerCtx to unblock stuck workers + workerCancel context.CancelCauseFunc // Cancels workerCtx to unblock stuck workers writerDone chan error writerErr error workersShutDown atomic.Bool // True once the worker pipeline has been torn down diff --git a/builder_parallel.go b/builder_parallel.go index b67a89a..8ce968e 100644 --- a/builder_parallel.go +++ b/builder_parallel.go @@ -70,7 +70,7 @@ func (b *builder) startPipeline(worker func() error) { } // Wrap in explicit cancel so shutdownWorkers can unblock workers stuck on resultChan. - ctx, cancel := context.WithCancel(b.ctx) + ctx, cancel := context.WithCancelCause(b.ctx) b.workerCancel = cancel b.workerGroup, b.workerCtx = errgroup.WithContext(ctx) for range b.workers { @@ -151,7 +151,7 @@ func (b *builder) dispatchBlockWork(blockID uint32, entries []routedEntry, poole case b.workChan <- work: return nil case <-b.workerCtx.Done(): - return b.workerCtx.Err() + return b.pipelineErr() case err := <-b.writerDone: // A nil read here means writerDone was closed (pipeline torn down); do // not drop this block — consumePipelineError surfaces the real error. @@ -165,6 +165,15 @@ func (b *builder) dispatchEmptyBlock(blockID uint32) error { return b.dispatchBlockWork(blockID, nil, false) } +// pipelineErr returns why workerCtx is done: the build context's error if the +// build was canceled, otherwise the error of the worker or writer that failed. +func (b *builder) pipelineErr() error { + if err := b.ctx.Err(); err != nil { + return err + } + return context.Cause(b.workerCtx) +} + // blockSolver is a worker's block builder, with the payload buffer it reuses // across blocks. type blockSolver struct { @@ -293,10 +302,12 @@ func (b *builder) runWorker() error { // drainParallelPipeline's workerGroup.Wait — would hang forever. Combined with // runWorker returning buildErr (which cancels via the errgroup), this guarantees // the invariant: ANY internal failure cancels b.workerCtx exactly once. -// workerCancel is idempotent (also called by shutdownWorkers). +// workerCancel is idempotent (also called by shutdownWorkers). err is its +// cause, so the context carries the failure even when this cancel lands +// before a failed worker's own. func (b *builder) failWriter(err error) { b.writerDone <- err - b.workerCancel() + b.workerCancel(err) } func (b *builder) runWriter() { @@ -425,7 +436,7 @@ func (b *builder) shutdownWorkers() { } b.workersShutDown.Store(true) if b.workerCancel != nil { - b.workerCancel() + b.workerCancel(nil) } close(b.workChan) _ = b.workerGroup.Wait() diff --git a/builder_unsorted_finish.go b/builder_unsorted_finish.go index 44a9002..ce27038 100644 --- a/builder_unsorted_finish.go +++ b/builder_unsorted_finish.go @@ -665,7 +665,7 @@ func (ub *UnsortedBuilder) finishUnsortedParallel() error { case b.workChan <- work: case <-b.workerCtx.Done(): slotFences[slotIdx].Done() - return fail(b.workerCtx.Err()) + return fail(b.pipelineErr()) case err := <-b.writerDone: b.writerErr = err slotFences[slotIdx].Done() diff --git a/parallel_pipeline_fuzz_test.go b/parallel_pipeline_fuzz_test.go index bfc3f79..fca65bd 100644 --- a/parallel_pipeline_fuzz_test.go +++ b/parallel_pipeline_fuzz_test.go @@ -72,7 +72,7 @@ type pipeScenario uint8 const ( scenValid pipeScenario = iota // well-formed input: build must succeed + index is a bijection - scenDuplicate // a duplicate key: build must fail, not hang + scenDuplicate // a duplicate key: build must fail with ErrDuplicateKey, not hang scenCancel // ctx cancelled mid-build: build must return, not hang scenCount ) @@ -209,14 +209,7 @@ func runPipelineScenario(t *testing.T, cfg pipeConfig) { assertBijection(t, indexPath, keys) case scenDuplicate: - if buildErr == nil { - t.Fatalf("%s: expected a duplicate-key error, got nil", cfg) - } - // The precise cause is ErrDuplicateKey; in some interleavings the - // pipeline surfaces a context cancellation from its own teardown first. - // Both mean the build failed cleanly; only nil or an unrelated error is - // a bug. - if !errors.Is(buildErr, sherr.ErrDuplicateKey) && !errors.Is(buildErr, context.Canceled) { + if !errors.Is(buildErr, sherr.ErrDuplicateKey) { t.Fatalf("%s: expected ErrDuplicateKey, got: %v", cfg, buildErr) }