Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 18 additions & 6 deletions engine/durable.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,9 @@ func (eng *Engine) RequestCancelDurableWorkflow(ctx context.Context, request dur
return worker.RequestCancelExecution(ctx, request)
}

func (eng *Engine) startDurable(ctx context.Context) error {
// checkDurableStart reports why the durable worker cannot be launched. Start
// calls it before any fallible step, so a refusal leaves nothing to undo.
func (eng *Engine) checkDurableStart(ctx context.Context) error {
d := eng.durable
if d == nil {
return nil
Expand All @@ -148,11 +150,22 @@ func (eng *Engine) startDurable(ctx context.Context) error {
if d.stopped {
return durable.ErrClosed
}
if ctx.Err() != nil {
return ctx.Err()
return ctx.Err()
}

// launchDurable starts the worker's polling goroutine. It cannot fail, so
// Start runs it only after every step that can: drt.Worker runs once and its
// Stop is final, so a worker launched by a Start that then failed could never
// poll again, and a retried Start would have no durable worker.
func (eng *Engine) launchDurable(ctx context.Context) {
d := eng.durable
if d == nil {
return
}
if d.done != nil {
return d.err
d.mu.Lock()
defer d.mu.Unlock()
if d.stopped || d.done != nil {
return
}
// The worker polls for the life of the engine; it must not keep the
// boot span Start's ctx carries, nor its cancellation: Start's ctx can
Expand All @@ -170,7 +183,6 @@ func (eng *Engine) startDurable(ctx context.Context) error {
}
close(d.done)
}()
return nil
}

func (eng *Engine) stopDurable(ctx context.Context) error {
Expand Down
26 changes: 18 additions & 8 deletions engine/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ type Engine struct {
// and retains its completion channel across timed-out Stop calls.
lifecycleOnce sync.Once
lifecycleGate chan struct{}
started bool // a Start has succeeded; later Starts are no-ops
stopping bool
quiesced chan struct{}
quiesceErr error
Expand Down Expand Up @@ -774,6 +775,12 @@ func (eng *Engine) Health(ctx context.Context) error {

// Start begins job processing by starting the worker pool and cron scheduler.
// It also resumes any workflow runs left in "running" state (crash recovery).
//
// Start is idempotent: once a Start has succeeded, later calls return nil
// without doing anything, so runs already executing are not resumed twice.
// A Start that fails can be retried: it launches the durable worker only
// once every step that can fail has succeeded. Start after Stop or
// StopWorkers returns durable.ErrClosed.
func (eng *Engine) Start(ctx context.Context) (startErr error) {
if err := eng.lockLifecycle(ctx); err != nil {
return err
Expand All @@ -782,21 +789,19 @@ func (eng *Engine) Start(ctx context.Context) (startErr error) {
if eng.stopping {
return durable.ErrClosed
}
if eng.started {
return nil
}

if err := eng.startDurable(ctx); err != nil {
if err := eng.checkDurableStart(ctx); err != nil {
return err
}
schedulerStarted := false
defer func() {
if startErr != nil {
if startErr != nil && schedulerStarted {
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
if schedulerStarted {
startErr = errors.Join(startErr, eng.scheduler.Stop(cleanupCtx))
}
if eng.durable != nil {
startErr = errors.Join(startErr, eng.stopDurable(cleanupCtx))
}
startErr = errors.Join(startErr, eng.scheduler.Stop(cleanupCtx))
}
}()
// Resume any interrupted workflow runs (best-effort, non-fatal).
Expand All @@ -816,6 +821,10 @@ func (eng *Engine) Start(ctx context.Context) (startErr error) {
return err
}

// Nothing below may fail: the durable worker cannot be restarted once
// launched, so it starts only after every fallible step has succeeded.
eng.launchDurable(ctx)

// Stores that support push notifications (store.WakeNotifier, e.g.
// Postgres LISTEN/NOTIFY) wake the worker pool on cross-instance
// enqueues so jobs are picked up without waiting out the idle poll
Expand All @@ -838,6 +847,7 @@ func (eng *Engine) Start(ctx context.Context) (startErr error) {
// Keep this worker's cluster row alive. See startHeartbeat.
eng.startHeartbeat(ctx)

eng.started = true
return nil
}

Expand Down
151 changes: 151 additions & 0 deletions engine/start_idempotent_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
package engine_test

import (
"context"
"errors"
"sync/atomic"
"testing"
"time"

"github.com/xraph/dispatch"
"github.com/xraph/dispatch/durable"
drt "github.com/xraph/dispatch/durable/runtime"
"github.com/xraph/dispatch/engine"
"github.com/xraph/dispatch/store/memory"
"github.com/xraph/dispatch/workflow"
)

// startCountingStore counts the store calls only Start makes: the resume
// scan for runs left "running" and the wake listener subscription.
type startCountingStore struct {
*memory.Store
resumeScans atomic.Int32
listeners atomic.Int32
stops atomic.Int32
}

func (s *startCountingStore) ListRuns(ctx context.Context, opts workflow.ListOpts) ([]*workflow.Run, error) {
if opts.State == workflow.RunStateRunning {
s.resumeScans.Add(1)
}
return s.Store.ListRuns(ctx, opts)
}

func (s *startCountingStore) StartWakeListener(context.Context, func()) (func(), error) {
s.listeners.Add(1)
return func() { s.stops.Add(1) }, nil
}

// A second Start on a running engine must not resume the runs it is already
// executing, nor subscribe a second wake listener whose stop func is lost.
func TestStartTwiceIsNoOp(t *testing.T) {
s := &startCountingStore{Store: memory.New()}
d, err := dispatch.New(dispatch.WithStore(s), dispatch.WithConcurrency(1), dispatch.WithPollInterval(time.Millisecond))
if err != nil {
t.Fatal(err)
}
eng, err := engine.Build(d)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
if err = eng.Start(ctx); err != nil {
t.Fatalf("first Start: %v", err)
}
if got := s.resumeScans.Load(); got != 1 {
t.Fatalf("resume scans after first Start = %d, want 1", got)
}
if got := s.listeners.Load(); got != 1 {
t.Fatalf("wake listeners after first Start = %d, want 1", got)
}

if err = eng.Start(ctx); err != nil {
t.Fatalf("second Start: %v", err)
}
if got := s.resumeScans.Load(); got != 1 {
t.Errorf("second Start resumed running workflow runs again: scans = %d, want 1", got)
}
if got := s.listeners.Load(); got != 1 {
t.Errorf("second Start subscribed another wake listener: listeners = %d, want 1", got)
}

stopCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if err = eng.Stop(stopCtx); err != nil {
t.Fatalf("Stop: %v", err)
}
if got := s.stops.Load(); got != 1 {
t.Fatalf("wake listeners stopped = %d, want 1", got)
}
if err = eng.Start(ctx); !errors.Is(err, durable.ErrClosed) {
t.Fatalf("Start after Stop = %v, want ErrClosed", err)
}
}

// failOncePool fails its first Start, as a pool that could not come up would,
// and starts cleanly on the next attempt.
type failOncePool struct{ starts atomic.Int32 }

func (p *failOncePool) Start(context.Context) error {
if p.starts.Add(1) == 1 {
return errors.New("pool start failed")
}
return nil
}

func (p *failOncePool) Stop(context.Context) error { return nil }

// A Start that fails partway must leave durable workflows restartable, so the
// next Start brings the durable worker up instead of returning ErrClosed.
func TestFailedStartCanBeRetriedWithDurableWorkflows(t *testing.T) {
d, err := dispatch.New(dispatch.WithStore(memory.New()), dispatch.WithConcurrency(1), dispatch.WithPollInterval(time.Millisecond))
if err != nil {
t.Fatal(err)
}
o := drt.Options{Namespace: "retry", Queue: "work", BuildID: "v1", Owner: "worker", PollInterval: time.Millisecond,
Workflows: map[string]drt.WorkflowFunc{"noop": func(*drt.Workflow, []byte) ([]byte, error) { return nil, nil }}}
eng, err := engine.Build(d, engine.WithDurableWorkflows(o))
if err != nil {
t.Fatal(err)
}
pool := &failOncePool{}
d.SetPool(pool)

ctx := context.Background()
if err = eng.Start(ctx); err == nil {
t.Fatal("first Start succeeded, want the injected pool failure")
}
if status, e := eng.DurableStatus(); e != nil || status.State != drt.WorkerNotStarted {
t.Errorf("durable worker after failed Start: state %s %v, want not started", status.State, e)
}

if err = eng.Start(ctx); err != nil {
t.Fatalf("retried Start: %v", err)
}
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = eng.Stop(stopCtx)
})
if got := pool.starts.Load(); got != 2 {
t.Fatalf("pool starts = %d, want 2", got)
}

deadline := time.Now().Add(2 * time.Second)
for {
status, e := eng.DurableStatus()
if e != nil {
t.Fatal(e)
}
if status.State == drt.WorkerRunning {
break
}
if time.Now().After(deadline) {
t.Fatalf("durable worker after retried Start: %+v, want running", status)
}
time.Sleep(5 * time.Millisecond)
}
if err = eng.Health(ctx); err != nil {
t.Fatalf("Health after retried Start: %v", err)
}
}
Loading