From ab2c130f254ab53360587c098dc5023647e8a942 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Wed, 18 Nov 2026 22:50:29 -0600 Subject: [PATCH] fix(worker): give each pool run fresh lifecycle state Pool created its stop channel once in NewPool and closed it in Stop. After Stop and a later Start, every loop of the new run selected on the already-closed channel and returned at once, so the pool reported itself running while processing nothing, and the next Stop panicked with "close of closed channel". Stop also released the lock before waiting for the old run, so a concurrent Start could launch a new run whose fields raced the draining goroutines. Follow the cron.Scheduler fix: a lifecycleMu serializes Start and Stop. Start is a no-op while running and otherwise builds fresh per-run state (stop channel, cancel context, job hand-off and slot channels), resets the readiness poll result, and drops any stale Wake nudge. Stop is a no-op when not running, otherwise closes and cancels, waits for every goroutine of the run while still holding lifecycleMu, and then discards any in-flight record left behind so no lease timer outlives its run. Calling Stop more than once, as Engine and Dispatcher both do, stays a no-op. Tests cover a restart processing a newly enqueued job, Stop twice after each of several restarts, and concurrent Start/Stop calls under -race. --- worker/pool.go | 95 +++++++++++++--- worker/pool_restart_test.go | 219 ++++++++++++++++++++++++++++++++++++ 2 files changed, 296 insertions(+), 18 deletions(-) create mode 100644 worker/pool_restart_test.go diff --git a/worker/pool.go b/worker/pool.go index 378f975..770aaa4 100644 --- a/worker/pool.go +++ b/worker/pool.go @@ -146,17 +146,29 @@ type Pool struct { // and starts spans on the global tracer provider. tel *telemetry.Telemetry + // Per-run state. Start makes a fresh set of these for every run and + // Stop retires them, so a pool that is stopped and started again never + // hands its new goroutines a channel or context the old run closed. + // They are written only by Start, under lifecycleMu, before any loop + // goroutine of that run exists, and read by those goroutines, which + // Stop waits out before it releases lifecycleMu. stopCh chan struct{} - wakeCh chan struct{} // nudges the fetcher out of its idle backoff jobCh chan admitted // hand-off from the fetcher to the workers slots chan struct{} // free-worker tokens; capacity == concurrency cancelCtx context.Context // Cancelled on Stop to interrupt in-flight store operations cancelFunc context.CancelFunc // Cancels cancelCtx - wg sync.WaitGroup - mu sync.Mutex - running bool - activeJobs map[string]*inflight - activeMu sync.Mutex + + wakeCh chan struct{} // nudges the fetcher out of its idle backoff + + // lifecycleMu serializes Start and Stop. Stop holds it until every + // goroutine of the run has exited, so a concurrent Start waits for the + // old run to finish instead of racing it for the per-run fields above. + lifecycleMu sync.Mutex + wg sync.WaitGroup + mu sync.Mutex + running bool // written under lifecycleMu and mu; read under either + activeJobs map[string]*inflight + activeMu sync.Mutex } // PoolOption configures a Pool. @@ -344,7 +356,6 @@ func NewPool( maxPollInterval: 30 * time.Second, workerID: id.NewWorkerID(), logger: logger, - stopCh: make(chan struct{}), wakeCh: make(chan struct{}, 1), activeJobs: make(map[string]*inflight), } @@ -377,17 +388,37 @@ func (p *Pool) Wake() { // WorkerID returns the pool's unique worker identifier. func (p *Pool) WorkerID() id.WorkerID { return p.workerID } -// Start launches the worker goroutines. It returns immediately. +// Start launches the worker goroutines. It returns immediately. Calling it +// while the pool is running is a no-op; calling it after Stop begins a +// fresh run with its own stop channel, context and worker slots. func (p *Pool) Start(ctx context.Context) error { - p.mu.Lock() + p.lifecycleMu.Lock() + defer p.lifecycleMu.Unlock() if p.running { - p.mu.Unlock() return nil } - p.running = true + + // Fresh per-run state. The previous run, if any, closed its stop + // channel and cancelled its context, and every loop that saw them has + // exited: Stop waited for them before releasing lifecycleMu. + p.stopCh = make(chan struct{}) p.cancelCtx, p.cancelFunc = context.WithCancel(context.Background()) + // A nudge left over from the previous run, or sent while stopped, is + // already satisfied: the new fetcher polls as soon as it starts. + select { + case <-p.wakeCh: + default: + } + + // Readiness describes this run's polls, not the last run's. + p.mu.Lock() + p.running = true + p.polled = false + p.pollErr = nil + p.mu.Unlock() + p.logger.Info("worker pool starting", log.String("worker_id", p.workerID.String()), log.Int("concurrency", p.concurrency), @@ -424,8 +455,6 @@ func (p *Pool) Start(ctx context.Context) error { go p.reaperLoop() } - p.mu.Unlock() - // Sweep sandboxes this worker left behind across a restart before it // takes new work. In-process reclaim is a no-op; an out-of-process rung // would otherwise keep orphaned children or pods alive indefinitely. @@ -435,9 +464,8 @@ func (p *Pool) Start(ctx context.Context) error { // Run in the background, tracked by p.wg like every other pool // goroutine, so Start returns immediately as documented even when a // rung's Reclaim does real (and potentially slow) process or - // filesystem I/O. Outside p.mu: nothing else needs the lock held for - // this, and holding it here would block Stop and every other pool - // method on a sweep that has no bound of its own. + // filesystem I/O. Running it inline would hold lifecycleMu, and so + // block Stop, on a sweep that has no bound of its own. if p.executor != nil { p.wg.Add(1) go p.runReclaimSweep(ctx) @@ -495,12 +523,18 @@ func mergeDone(a, b context.Context) (context.Context, context.CancelFunc) { // Stop signals all workers to stop and waits for them to finish. // If the context has a deadline, active jobs are cancelled when time runs out. +// Calling it while the pool is not running is a no-op, so it is safe to call +// more than once. A later Start begins a fresh run. func (p *Pool) Stop(ctx context.Context) error { - p.mu.Lock() + p.lifecycleMu.Lock() + defer p.lifecycleMu.Unlock() + if !p.running { - p.mu.Unlock() return nil } + + // Ready reports stopped from here on, while the run drains. + p.mu.Lock() p.running = false p.mu.Unlock() @@ -528,9 +562,34 @@ func (p *Pool) Stop(ctx context.Context) error { p.wg.Wait() } + // Every worker untracks its own job on the way out, so this is empty + // by now. Clear it anyway so a record can never outlive its run: its + // lease timer would otherwise fire into the next one. + p.dropActiveJobs() + return nil } +// dropActiveJobs stops and discards any in-flight record still tracked +// after a run's goroutines have exited. +func (p *Pool) dropActiveJobs() { + p.activeMu.Lock() + defer p.activeMu.Unlock() + + if len(p.activeJobs) == 0 { + return + } + + for jobID, rec := range p.activeJobs { + if rec.leaseTimer != nil { + rec.leaseTimer.Stop() + } + rec.cancel(context.Canceled) + delete(p.activeJobs, jobID) + } + p.tel.Jobs().Active(0) +} + // callCtx wraps p.cancelCtx with the configured per-call timeout so a // stalled driver session can't pin a pool connection indefinitely. // Returns the unwrapped cancelCtx + a no-op cancel when the timeout diff --git a/worker/pool_restart_test.go b/worker/pool_restart_test.go new file mode 100644 index 0000000..a2bf574 --- /dev/null +++ b/worker/pool_restart_test.go @@ -0,0 +1,219 @@ +package worker_test + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/xraph/dispatch/backoff" + "github.com/xraph/dispatch/dlq" + "github.com/xraph/dispatch/ext" + "github.com/xraph/dispatch/job" + "github.com/xraph/dispatch/middleware" + "github.com/xraph/dispatch/store/memory" + "github.com/xraph/dispatch/worker" + + log "github.com/xraph/go-utils/log" +) + +// setupRestartPool builds a pool with every background loop enabled +// (fetcher, workers, heartbeat, reaper, reclaim sweep) so a restart has to +// give each of them fresh per-run state. Jobs named "count" bump the +// returned counter. +func setupRestartPool(t *testing.T) (*worker.Pool, *memory.Store, *atomic.Int64) { + t.Helper() + logger := log.NewNoopLogger() + s := memory.New() + reg := job.NewRegistry() + extensions := ext.NewRegistry(logger) + + var ran atomic.Int64 + job.RegisterDefinition(reg, job.NewDefinition("count", func(context.Context, struct{}) error { + ran.Add(1) + return nil + })) + + executor := worker.NewExecutor( + reg, extensions, s, dlq.NewService(s, s), backoff.NewConstant(10*time.Millisecond), logger, + middleware.Recover(logger), + ) + + pool := worker.NewPool(s, executor, extensions, logger, + worker.WithPoolConcurrency(2), + worker.WithPollInterval(10*time.Millisecond), + worker.WithMaxPollInterval(20*time.Millisecond), + worker.WithHeartbeatInterval(10*time.Millisecond), + worker.WithStaleJobThreshold(time.Minute), + worker.WithReapInterval(10*time.Millisecond), + worker.WithPoolQueues([]string{"default"}), + ) + + return pool, s, &ran +} + +func enqueueCountJob(t *testing.T, s *memory.Store) *job.Job { + t.Helper() + payload, _ := json.Marshal(struct{}{}) + now := time.Now().UTC() + j := &job.Job{ + ID: newTestJobID(), + Name: "count", + Queue: "default", + Payload: payload, + State: job.StatePending, + MaxRetries: 1, + RunAt: now, + CreatedAt: now, + UpdatedAt: now, + } + if err := s.EnqueueJob(context.Background(), j); err != nil { + t.Fatalf("enqueue: %v", err) + } + return j +} + +func waitForCount(t *testing.T, ran *atomic.Int64, want int64) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for ran.Load() < want { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %d jobs to run, %d ran", want, ran.Load()) + } + time.Sleep(5 * time.Millisecond) + } +} + +// safeCall runs fn and turns a panic into an error, so a lifecycle bug +// fails the test instead of crashing the test binary. +func safeCall(fn func() error) (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() + return fn() +} + +func stopPool(t *testing.T, p *worker.Pool) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := safeCall(func() error { return p.Stop(ctx) }); err != nil { + t.Fatalf("stop: %v", err) + } +} + +// A pool that is stopped and started again must actually run again: the +// second run's loops must not see the first run's closed stop channel or +// cancelled context and exit at once. +func TestPool_RestartProcessesNewJob(t *testing.T) { + pool, s, ran := setupRestartPool(t) + + if err := pool.Start(context.Background()); err != nil { + t.Fatalf("start: %v", err) + } + enqueueCountJob(t, s) + pool.Wake() + waitForCount(t, ran, 1) + stopPool(t, pool) + + if err := pool.Ready(); err == nil { + t.Fatal("Ready() = nil after Stop, want an error") + } + + if err := pool.Start(context.Background()); err != nil { + t.Fatalf("restart: %v", err) + } + t.Cleanup(func() { stopPool(t, pool) }) + + j := enqueueCountJob(t, s) + pool.Wake() + waitForCount(t, ran, 2) + + got, err := s.GetJob(context.Background(), j.ID) + if err != nil { + t.Fatalf("get job: %v", err) + } + deadline := time.Now().Add(3 * time.Second) + for got.State != job.StateCompleted && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + if got, err = s.GetJob(context.Background(), j.ID); err != nil { + t.Fatalf("get job: %v", err) + } + } + if got.State != job.StateCompleted { + t.Fatalf("job state after restart = %q, want %q", got.State, job.StateCompleted) + } + + if err := pool.Ready(); err != nil { + t.Fatalf("Ready() after restart = %v, want nil", err) + } +} + +// Stop is called more than once on shutdown (Engine and Dispatcher both +// stop the pool). That has to stay a no-op after a restart too, rather +// than closing the run's stop channel a second time. +func TestPool_StopTwiceAfterRestart(t *testing.T) { + pool, _, _ := setupRestartPool(t) + ctx := context.Background() + + for run := range 3 { + if err := safeCall(func() error { return pool.Start(ctx) }); err != nil { + t.Fatalf("start (run %d): %v", run, err) + } + stopPool(t, pool) + stopPool(t, pool) + } +} + +// Start and Stop may race from different goroutines. Every call must be +// safe, and the pool must still work afterwards. +func TestPool_ConcurrentStartStop(t *testing.T) { + pool, s, ran := setupRestartPool(t) + ctx := context.Background() + + var wg sync.WaitGroup + errs := make(chan error, 64) + for g := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for i := range 20 { + var err error + if (g+i)%2 == 0 { + err = safeCall(func() error { return pool.Start(ctx) }) + } else { + stopCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + err = safeCall(func() error { return pool.Stop(stopCtx) }) + cancel() + } + if err != nil { + errs <- err + return + } + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Errorf("concurrent lifecycle call: %v", err) + } + if t.Failed() { + return + } + + stopPool(t, pool) + if err := pool.Start(ctx); err != nil { + t.Fatalf("start after concurrent calls: %v", err) + } + t.Cleanup(func() { stopPool(t, pool) }) + + enqueueCountJob(t, s) + pool.Wake() + waitForCount(t, ran, 1) +}