From c888bceb1f5a00735ba0720425455054b8a760c9 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Wed, 18 Nov 2026 22:47:11 -0600 Subject: [PATCH] fix(engine): make Start idempotent and retryable after a failure Engine.Start had no guard against a second call once it had succeeded. A repeat Start ran wfRunner.ResumeAll again, re-executing workflow runs in "running" state that this process was still executing, and called StartWakeListener again, overwriting wakeStop and leaking the first listener (a Postgres LISTEN connection and its goroutine). The forge extension guards with IsStarted, but direct engine users did not. Start now records success under the lifecycle gate and a later Start returns nil without doing anything. Start after Stop or StopWorkers still returns durable.ErrClosed. A Start that failed after launching the durable worker cleaned up with stopDurable, which marks durable workflows permanently stopped, so every later Start returned durable.ErrClosed. The worker itself cannot be reused either: drt.Worker runs once and its Stop is final. Start now checks up front that the durable worker can launch and only launches it after every fallible step (resume, cron scheduler, dispatcher pool) has succeeded. A failed Start therefore never touches the durable worker, and its cleanup only stops the cron scheduler, so a retry starts everything normally. Stop's terminal semantics are unchanged. --- engine/durable.go | 24 +++-- engine/engine.go | 26 ++++-- engine/start_idempotent_test.go | 151 ++++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 14 deletions(-) create mode 100644 engine/start_idempotent_test.go diff --git a/engine/durable.go b/engine/durable.go index 4ac5c8d..51aa8b7 100644 --- a/engine/durable.go +++ b/engine/durable.go @@ -138,7 +138,9 @@ func (eng *Engine) RequestCancelDurableWorkflow(ctx context.Context, request dur return worker.RequestCancelExecution(ctx, request) } -func (eng *Engine) startDurable(ctx context.Context) error { +// checkDurableStart reports why the durable worker cannot be launched. Start +// calls it before any fallible step, so a refusal leaves nothing to undo. +func (eng *Engine) checkDurableStart(ctx context.Context) error { d := eng.durable if d == nil { return nil @@ -148,11 +150,22 @@ func (eng *Engine) startDurable(ctx context.Context) error { if d.stopped { return durable.ErrClosed } - if ctx.Err() != nil { - return ctx.Err() + return ctx.Err() +} + +// launchDurable starts the worker's polling goroutine. It cannot fail, so +// Start runs it only after every step that can: drt.Worker runs once and its +// Stop is final, so a worker launched by a Start that then failed could never +// poll again, and a retried Start would have no durable worker. +func (eng *Engine) launchDurable(ctx context.Context) { + d := eng.durable + if d == nil { + return } - if d.done != nil { - return d.err + d.mu.Lock() + defer d.mu.Unlock() + if d.stopped || d.done != nil { + return } // The worker polls for the life of the engine; it must not keep the // boot span Start's ctx carries, nor its cancellation: Start's ctx can @@ -170,7 +183,6 @@ func (eng *Engine) startDurable(ctx context.Context) error { } close(d.done) }() - return nil } func (eng *Engine) stopDurable(ctx context.Context) error { diff --git a/engine/engine.go b/engine/engine.go index 7b24837..6ba5bf3 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -95,6 +95,7 @@ type Engine struct { // and retains its completion channel across timed-out Stop calls. lifecycleOnce sync.Once lifecycleGate chan struct{} + started bool // a Start has succeeded; later Starts are no-ops stopping bool quiesced chan struct{} quiesceErr error @@ -774,6 +775,12 @@ func (eng *Engine) Health(ctx context.Context) error { // Start begins job processing by starting the worker pool and cron scheduler. // It also resumes any workflow runs left in "running" state (crash recovery). +// +// Start is idempotent: once a Start has succeeded, later calls return nil +// without doing anything, so runs already executing are not resumed twice. +// A Start that fails can be retried: it launches the durable worker only +// once every step that can fail has succeeded. Start after Stop or +// StopWorkers returns durable.ErrClosed. func (eng *Engine) Start(ctx context.Context) (startErr error) { if err := eng.lockLifecycle(ctx); err != nil { return err @@ -782,21 +789,19 @@ func (eng *Engine) Start(ctx context.Context) (startErr error) { if eng.stopping { return durable.ErrClosed } + if eng.started { + return nil + } - if err := eng.startDurable(ctx); err != nil { + if err := eng.checkDurableStart(ctx); err != nil { return err } schedulerStarted := false defer func() { - if startErr != nil { + if startErr != nil && schedulerStarted { cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) defer cancel() - if schedulerStarted { - startErr = errors.Join(startErr, eng.scheduler.Stop(cleanupCtx)) - } - if eng.durable != nil { - startErr = errors.Join(startErr, eng.stopDurable(cleanupCtx)) - } + startErr = errors.Join(startErr, eng.scheduler.Stop(cleanupCtx)) } }() // Resume any interrupted workflow runs (best-effort, non-fatal). @@ -816,6 +821,10 @@ func (eng *Engine) Start(ctx context.Context) (startErr error) { return err } + // Nothing below may fail: the durable worker cannot be restarted once + // launched, so it starts only after every fallible step has succeeded. + eng.launchDurable(ctx) + // Stores that support push notifications (store.WakeNotifier, e.g. // Postgres LISTEN/NOTIFY) wake the worker pool on cross-instance // enqueues so jobs are picked up without waiting out the idle poll @@ -838,6 +847,7 @@ func (eng *Engine) Start(ctx context.Context) (startErr error) { // Keep this worker's cluster row alive. See startHeartbeat. eng.startHeartbeat(ctx) + eng.started = true return nil } diff --git a/engine/start_idempotent_test.go b/engine/start_idempotent_test.go new file mode 100644 index 0000000..13794ed --- /dev/null +++ b/engine/start_idempotent_test.go @@ -0,0 +1,151 @@ +package engine_test + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/xraph/dispatch" + "github.com/xraph/dispatch/durable" + drt "github.com/xraph/dispatch/durable/runtime" + "github.com/xraph/dispatch/engine" + "github.com/xraph/dispatch/store/memory" + "github.com/xraph/dispatch/workflow" +) + +// startCountingStore counts the store calls only Start makes: the resume +// scan for runs left "running" and the wake listener subscription. +type startCountingStore struct { + *memory.Store + resumeScans atomic.Int32 + listeners atomic.Int32 + stops atomic.Int32 +} + +func (s *startCountingStore) ListRuns(ctx context.Context, opts workflow.ListOpts) ([]*workflow.Run, error) { + if opts.State == workflow.RunStateRunning { + s.resumeScans.Add(1) + } + return s.Store.ListRuns(ctx, opts) +} + +func (s *startCountingStore) StartWakeListener(context.Context, func()) (func(), error) { + s.listeners.Add(1) + return func() { s.stops.Add(1) }, nil +} + +// A second Start on a running engine must not resume the runs it is already +// executing, nor subscribe a second wake listener whose stop func is lost. +func TestStartTwiceIsNoOp(t *testing.T) { + s := &startCountingStore{Store: memory.New()} + d, err := dispatch.New(dispatch.WithStore(s), dispatch.WithConcurrency(1), dispatch.WithPollInterval(time.Millisecond)) + if err != nil { + t.Fatal(err) + } + eng, err := engine.Build(d) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if err = eng.Start(ctx); err != nil { + t.Fatalf("first Start: %v", err) + } + if got := s.resumeScans.Load(); got != 1 { + t.Fatalf("resume scans after first Start = %d, want 1", got) + } + if got := s.listeners.Load(); got != 1 { + t.Fatalf("wake listeners after first Start = %d, want 1", got) + } + + if err = eng.Start(ctx); err != nil { + t.Fatalf("second Start: %v", err) + } + if got := s.resumeScans.Load(); got != 1 { + t.Errorf("second Start resumed running workflow runs again: scans = %d, want 1", got) + } + if got := s.listeners.Load(); got != 1 { + t.Errorf("second Start subscribed another wake listener: listeners = %d, want 1", got) + } + + stopCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err = eng.Stop(stopCtx); err != nil { + t.Fatalf("Stop: %v", err) + } + if got := s.stops.Load(); got != 1 { + t.Fatalf("wake listeners stopped = %d, want 1", got) + } + if err = eng.Start(ctx); !errors.Is(err, durable.ErrClosed) { + t.Fatalf("Start after Stop = %v, want ErrClosed", err) + } +} + +// failOncePool fails its first Start, as a pool that could not come up would, +// and starts cleanly on the next attempt. +type failOncePool struct{ starts atomic.Int32 } + +func (p *failOncePool) Start(context.Context) error { + if p.starts.Add(1) == 1 { + return errors.New("pool start failed") + } + return nil +} + +func (p *failOncePool) Stop(context.Context) error { return nil } + +// A Start that fails partway must leave durable workflows restartable, so the +// next Start brings the durable worker up instead of returning ErrClosed. +func TestFailedStartCanBeRetriedWithDurableWorkflows(t *testing.T) { + d, err := dispatch.New(dispatch.WithStore(memory.New()), dispatch.WithConcurrency(1), dispatch.WithPollInterval(time.Millisecond)) + if err != nil { + t.Fatal(err) + } + o := drt.Options{Namespace: "retry", Queue: "work", BuildID: "v1", Owner: "worker", PollInterval: time.Millisecond, + Workflows: map[string]drt.WorkflowFunc{"noop": func(*drt.Workflow, []byte) ([]byte, error) { return nil, nil }}} + eng, err := engine.Build(d, engine.WithDurableWorkflows(o)) + if err != nil { + t.Fatal(err) + } + pool := &failOncePool{} + d.SetPool(pool) + + ctx := context.Background() + if err = eng.Start(ctx); err == nil { + t.Fatal("first Start succeeded, want the injected pool failure") + } + if status, e := eng.DurableStatus(); e != nil || status.State != drt.WorkerNotStarted { + t.Errorf("durable worker after failed Start: state %s %v, want not started", status.State, e) + } + + if err = eng.Start(ctx); err != nil { + t.Fatalf("retried Start: %v", err) + } + t.Cleanup(func() { + stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = eng.Stop(stopCtx) + }) + if got := pool.starts.Load(); got != 2 { + t.Fatalf("pool starts = %d, want 2", got) + } + + deadline := time.Now().Add(2 * time.Second) + for { + status, e := eng.DurableStatus() + if e != nil { + t.Fatal(e) + } + if status.State == drt.WorkerRunning { + break + } + if time.Now().After(deadline) { + t.Fatalf("durable worker after retried Start: %+v, want running", status) + } + time.Sleep(5 * time.Millisecond) + } + if err = eng.Health(ctx); err != nil { + t.Fatalf("Health after retried Start: %v", err) + } +}