diff --git a/engine/durable.go b/engine/durable.go index 4ac5c8d..51aa8b7 100644 --- a/engine/durable.go +++ b/engine/durable.go @@ -138,7 +138,9 @@ func (eng *Engine) RequestCancelDurableWorkflow(ctx context.Context, request dur return worker.RequestCancelExecution(ctx, request) } -func (eng *Engine) startDurable(ctx context.Context) error { +// checkDurableStart reports why the durable worker cannot be launched. Start +// calls it before any fallible step, so a refusal leaves nothing to undo. +func (eng *Engine) checkDurableStart(ctx context.Context) error { d := eng.durable if d == nil { return nil @@ -148,11 +150,22 @@ func (eng *Engine) startDurable(ctx context.Context) error { if d.stopped { return durable.ErrClosed } - if ctx.Err() != nil { - return ctx.Err() + return ctx.Err() +} + +// launchDurable starts the worker's polling goroutine. It cannot fail, so +// Start runs it only after every step that can: drt.Worker runs once and its +// Stop is final, so a worker launched by a Start that then failed could never +// poll again, and a retried Start would have no durable worker. +func (eng *Engine) launchDurable(ctx context.Context) { + d := eng.durable + if d == nil { + return } - if d.done != nil { - return d.err + d.mu.Lock() + defer d.mu.Unlock() + if d.stopped || d.done != nil { + return } // The worker polls for the life of the engine; it must not keep the // boot span Start's ctx carries, nor its cancellation: Start's ctx can @@ -170,7 +183,6 @@ func (eng *Engine) startDurable(ctx context.Context) error { } close(d.done) }() - return nil } func (eng *Engine) stopDurable(ctx context.Context) error { diff --git a/engine/engine.go b/engine/engine.go index 7b24837..6ba5bf3 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -95,6 +95,7 @@ type Engine struct { // and retains its completion channel across timed-out Stop calls. lifecycleOnce sync.Once lifecycleGate chan struct{} + started bool // a Start has succeeded; later Starts are no-ops stopping bool quiesced chan struct{} quiesceErr error @@ -774,6 +775,12 @@ func (eng *Engine) Health(ctx context.Context) error { // Start begins job processing by starting the worker pool and cron scheduler. // It also resumes any workflow runs left in "running" state (crash recovery). +// +// Start is idempotent: once a Start has succeeded, later calls return nil +// without doing anything, so runs already executing are not resumed twice. +// A Start that fails can be retried: it launches the durable worker only +// once every step that can fail has succeeded. Start after Stop or +// StopWorkers returns durable.ErrClosed. func (eng *Engine) Start(ctx context.Context) (startErr error) { if err := eng.lockLifecycle(ctx); err != nil { return err @@ -782,21 +789,19 @@ func (eng *Engine) Start(ctx context.Context) (startErr error) { if eng.stopping { return durable.ErrClosed } + if eng.started { + return nil + } - if err := eng.startDurable(ctx); err != nil { + if err := eng.checkDurableStart(ctx); err != nil { return err } schedulerStarted := false defer func() { - if startErr != nil { + if startErr != nil && schedulerStarted { cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) defer cancel() - if schedulerStarted { - startErr = errors.Join(startErr, eng.scheduler.Stop(cleanupCtx)) - } - if eng.durable != nil { - startErr = errors.Join(startErr, eng.stopDurable(cleanupCtx)) - } + startErr = errors.Join(startErr, eng.scheduler.Stop(cleanupCtx)) } }() // Resume any interrupted workflow runs (best-effort, non-fatal). @@ -816,6 +821,10 @@ func (eng *Engine) Start(ctx context.Context) (startErr error) { return err } + // Nothing below may fail: the durable worker cannot be restarted once + // launched, so it starts only after every fallible step has succeeded. + eng.launchDurable(ctx) + // Stores that support push notifications (store.WakeNotifier, e.g. // Postgres LISTEN/NOTIFY) wake the worker pool on cross-instance // enqueues so jobs are picked up without waiting out the idle poll @@ -838,6 +847,7 @@ func (eng *Engine) Start(ctx context.Context) (startErr error) { // Keep this worker's cluster row alive. See startHeartbeat. eng.startHeartbeat(ctx) + eng.started = true return nil } diff --git a/engine/start_idempotent_test.go b/engine/start_idempotent_test.go new file mode 100644 index 0000000..13794ed --- /dev/null +++ b/engine/start_idempotent_test.go @@ -0,0 +1,151 @@ +package engine_test + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/xraph/dispatch" + "github.com/xraph/dispatch/durable" + drt "github.com/xraph/dispatch/durable/runtime" + "github.com/xraph/dispatch/engine" + "github.com/xraph/dispatch/store/memory" + "github.com/xraph/dispatch/workflow" +) + +// startCountingStore counts the store calls only Start makes: the resume +// scan for runs left "running" and the wake listener subscription. +type startCountingStore struct { + *memory.Store + resumeScans atomic.Int32 + listeners atomic.Int32 + stops atomic.Int32 +} + +func (s *startCountingStore) ListRuns(ctx context.Context, opts workflow.ListOpts) ([]*workflow.Run, error) { + if opts.State == workflow.RunStateRunning { + s.resumeScans.Add(1) + } + return s.Store.ListRuns(ctx, opts) +} + +func (s *startCountingStore) StartWakeListener(context.Context, func()) (func(), error) { + s.listeners.Add(1) + return func() { s.stops.Add(1) }, nil +} + +// A second Start on a running engine must not resume the runs it is already +// executing, nor subscribe a second wake listener whose stop func is lost. +func TestStartTwiceIsNoOp(t *testing.T) { + s := &startCountingStore{Store: memory.New()} + d, err := dispatch.New(dispatch.WithStore(s), dispatch.WithConcurrency(1), dispatch.WithPollInterval(time.Millisecond)) + if err != nil { + t.Fatal(err) + } + eng, err := engine.Build(d) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if err = eng.Start(ctx); err != nil { + t.Fatalf("first Start: %v", err) + } + if got := s.resumeScans.Load(); got != 1 { + t.Fatalf("resume scans after first Start = %d, want 1", got) + } + if got := s.listeners.Load(); got != 1 { + t.Fatalf("wake listeners after first Start = %d, want 1", got) + } + + if err = eng.Start(ctx); err != nil { + t.Fatalf("second Start: %v", err) + } + if got := s.resumeScans.Load(); got != 1 { + t.Errorf("second Start resumed running workflow runs again: scans = %d, want 1", got) + } + if got := s.listeners.Load(); got != 1 { + t.Errorf("second Start subscribed another wake listener: listeners = %d, want 1", got) + } + + stopCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err = eng.Stop(stopCtx); err != nil { + t.Fatalf("Stop: %v", err) + } + if got := s.stops.Load(); got != 1 { + t.Fatalf("wake listeners stopped = %d, want 1", got) + } + if err = eng.Start(ctx); !errors.Is(err, durable.ErrClosed) { + t.Fatalf("Start after Stop = %v, want ErrClosed", err) + } +} + +// failOncePool fails its first Start, as a pool that could not come up would, +// and starts cleanly on the next attempt. +type failOncePool struct{ starts atomic.Int32 } + +func (p *failOncePool) Start(context.Context) error { + if p.starts.Add(1) == 1 { + return errors.New("pool start failed") + } + return nil +} + +func (p *failOncePool) Stop(context.Context) error { return nil } + +// A Start that fails partway must leave durable workflows restartable, so the +// next Start brings the durable worker up instead of returning ErrClosed. +func TestFailedStartCanBeRetriedWithDurableWorkflows(t *testing.T) { + d, err := dispatch.New(dispatch.WithStore(memory.New()), dispatch.WithConcurrency(1), dispatch.WithPollInterval(time.Millisecond)) + if err != nil { + t.Fatal(err) + } + o := drt.Options{Namespace: "retry", Queue: "work", BuildID: "v1", Owner: "worker", PollInterval: time.Millisecond, + Workflows: map[string]drt.WorkflowFunc{"noop": func(*drt.Workflow, []byte) ([]byte, error) { return nil, nil }}} + eng, err := engine.Build(d, engine.WithDurableWorkflows(o)) + if err != nil { + t.Fatal(err) + } + pool := &failOncePool{} + d.SetPool(pool) + + ctx := context.Background() + if err = eng.Start(ctx); err == nil { + t.Fatal("first Start succeeded, want the injected pool failure") + } + if status, e := eng.DurableStatus(); e != nil || status.State != drt.WorkerNotStarted { + t.Errorf("durable worker after failed Start: state %s %v, want not started", status.State, e) + } + + if err = eng.Start(ctx); err != nil { + t.Fatalf("retried Start: %v", err) + } + t.Cleanup(func() { + stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = eng.Stop(stopCtx) + }) + if got := pool.starts.Load(); got != 2 { + t.Fatalf("pool starts = %d, want 2", got) + } + + deadline := time.Now().Add(2 * time.Second) + for { + status, e := eng.DurableStatus() + if e != nil { + t.Fatal(e) + } + if status.State == drt.WorkerRunning { + break + } + if time.Now().After(deadline) { + t.Fatalf("durable worker after retried Start: %+v, want running", status) + } + time.Sleep(5 * time.Millisecond) + } + if err = eng.Health(ctx); err != nil { + t.Fatalf("Health after retried Start: %v", err) + } +}