diff --git a/worker/pool.go b/worker/pool.go index 378f975..770aaa4 100644 --- a/worker/pool.go +++ b/worker/pool.go @@ -146,17 +146,29 @@ type Pool struct { // and starts spans on the global tracer provider. tel *telemetry.Telemetry + // Per-run state. Start makes a fresh set of these for every run and + // Stop retires them, so a pool that is stopped and started again never + // hands its new goroutines a channel or context the old run closed. + // They are written only by Start, under lifecycleMu, before any loop + // goroutine of that run exists, and read by those goroutines, which + // Stop waits out before it releases lifecycleMu. stopCh chan struct{} - wakeCh chan struct{} // nudges the fetcher out of its idle backoff jobCh chan admitted // hand-off from the fetcher to the workers slots chan struct{} // free-worker tokens; capacity == concurrency cancelCtx context.Context // Cancelled on Stop to interrupt in-flight store operations cancelFunc context.CancelFunc // Cancels cancelCtx - wg sync.WaitGroup - mu sync.Mutex - running bool - activeJobs map[string]*inflight - activeMu sync.Mutex + + wakeCh chan struct{} // nudges the fetcher out of its idle backoff + + // lifecycleMu serializes Start and Stop. Stop holds it until every + // goroutine of the run has exited, so a concurrent Start waits for the + // old run to finish instead of racing it for the per-run fields above. + lifecycleMu sync.Mutex + wg sync.WaitGroup + mu sync.Mutex + running bool // written under lifecycleMu and mu; read under either + activeJobs map[string]*inflight + activeMu sync.Mutex } // PoolOption configures a Pool. @@ -344,7 +356,6 @@ func NewPool( maxPollInterval: 30 * time.Second, workerID: id.NewWorkerID(), logger: logger, - stopCh: make(chan struct{}), wakeCh: make(chan struct{}, 1), activeJobs: make(map[string]*inflight), } @@ -377,17 +388,37 @@ func (p *Pool) Wake() { // WorkerID returns the pool's unique worker identifier. func (p *Pool) WorkerID() id.WorkerID { return p.workerID } -// Start launches the worker goroutines. It returns immediately. +// Start launches the worker goroutines. It returns immediately. Calling it +// while the pool is running is a no-op; calling it after Stop begins a +// fresh run with its own stop channel, context and worker slots. func (p *Pool) Start(ctx context.Context) error { - p.mu.Lock() + p.lifecycleMu.Lock() + defer p.lifecycleMu.Unlock() if p.running { - p.mu.Unlock() return nil } - p.running = true + + // Fresh per-run state. The previous run, if any, closed its stop + // channel and cancelled its context, and every loop that saw them has + // exited: Stop waited for them before releasing lifecycleMu. + p.stopCh = make(chan struct{}) p.cancelCtx, p.cancelFunc = context.WithCancel(context.Background()) + // A nudge left over from the previous run, or sent while stopped, is + // already satisfied: the new fetcher polls as soon as it starts. + select { + case <-p.wakeCh: + default: + } + + // Readiness describes this run's polls, not the last run's. + p.mu.Lock() + p.running = true + p.polled = false + p.pollErr = nil + p.mu.Unlock() + p.logger.Info("worker pool starting", log.String("worker_id", p.workerID.String()), log.Int("concurrency", p.concurrency), @@ -424,8 +455,6 @@ func (p *Pool) Start(ctx context.Context) error { go p.reaperLoop() } - p.mu.Unlock() - // Sweep sandboxes this worker left behind across a restart before it // takes new work. In-process reclaim is a no-op; an out-of-process rung // would otherwise keep orphaned children or pods alive indefinitely. @@ -435,9 +464,8 @@ func (p *Pool) Start(ctx context.Context) error { // Run in the background, tracked by p.wg like every other pool // goroutine, so Start returns immediately as documented even when a // rung's Reclaim does real (and potentially slow) process or - // filesystem I/O. Outside p.mu: nothing else needs the lock held for - // this, and holding it here would block Stop and every other pool - // method on a sweep that has no bound of its own. + // filesystem I/O. Running it inline would hold lifecycleMu, and so + // block Stop, on a sweep that has no bound of its own. if p.executor != nil { p.wg.Add(1) go p.runReclaimSweep(ctx) @@ -495,12 +523,18 @@ func mergeDone(a, b context.Context) (context.Context, context.CancelFunc) { // Stop signals all workers to stop and waits for them to finish. // If the context has a deadline, active jobs are cancelled when time runs out. +// Calling it while the pool is not running is a no-op, so it is safe to call +// more than once. A later Start begins a fresh run. func (p *Pool) Stop(ctx context.Context) error { - p.mu.Lock() + p.lifecycleMu.Lock() + defer p.lifecycleMu.Unlock() + if !p.running { - p.mu.Unlock() return nil } + + // Ready reports stopped from here on, while the run drains. + p.mu.Lock() p.running = false p.mu.Unlock() @@ -528,9 +562,34 @@ func (p *Pool) Stop(ctx context.Context) error { p.wg.Wait() } + // Every worker untracks its own job on the way out, so this is empty + // by now. Clear it anyway so a record can never outlive its run: its + // lease timer would otherwise fire into the next one. + p.dropActiveJobs() + return nil } +// dropActiveJobs stops and discards any in-flight record still tracked +// after a run's goroutines have exited. +func (p *Pool) dropActiveJobs() { + p.activeMu.Lock() + defer p.activeMu.Unlock() + + if len(p.activeJobs) == 0 { + return + } + + for jobID, rec := range p.activeJobs { + if rec.leaseTimer != nil { + rec.leaseTimer.Stop() + } + rec.cancel(context.Canceled) + delete(p.activeJobs, jobID) + } + p.tel.Jobs().Active(0) +} + // callCtx wraps p.cancelCtx with the configured per-call timeout so a // stalled driver session can't pin a pool connection indefinitely. // Returns the unwrapped cancelCtx + a no-op cancel when the timeout diff --git a/worker/pool_restart_test.go b/worker/pool_restart_test.go new file mode 100644 index 0000000..a2bf574 --- /dev/null +++ b/worker/pool_restart_test.go @@ -0,0 +1,219 @@ +package worker_test + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/xraph/dispatch/backoff" + "github.com/xraph/dispatch/dlq" + "github.com/xraph/dispatch/ext" + "github.com/xraph/dispatch/job" + "github.com/xraph/dispatch/middleware" + "github.com/xraph/dispatch/store/memory" + "github.com/xraph/dispatch/worker" + + log "github.com/xraph/go-utils/log" +) + +// setupRestartPool builds a pool with every background loop enabled +// (fetcher, workers, heartbeat, reaper, reclaim sweep) so a restart has to +// give each of them fresh per-run state. Jobs named "count" bump the +// returned counter. +func setupRestartPool(t *testing.T) (*worker.Pool, *memory.Store, *atomic.Int64) { + t.Helper() + logger := log.NewNoopLogger() + s := memory.New() + reg := job.NewRegistry() + extensions := ext.NewRegistry(logger) + + var ran atomic.Int64 + job.RegisterDefinition(reg, job.NewDefinition("count", func(context.Context, struct{}) error { + ran.Add(1) + return nil + })) + + executor := worker.NewExecutor( + reg, extensions, s, dlq.NewService(s, s), backoff.NewConstant(10*time.Millisecond), logger, + middleware.Recover(logger), + ) + + pool := worker.NewPool(s, executor, extensions, logger, + worker.WithPoolConcurrency(2), + worker.WithPollInterval(10*time.Millisecond), + worker.WithMaxPollInterval(20*time.Millisecond), + worker.WithHeartbeatInterval(10*time.Millisecond), + worker.WithStaleJobThreshold(time.Minute), + worker.WithReapInterval(10*time.Millisecond), + worker.WithPoolQueues([]string{"default"}), + ) + + return pool, s, &ran +} + +func enqueueCountJob(t *testing.T, s *memory.Store) *job.Job { + t.Helper() + payload, _ := json.Marshal(struct{}{}) + now := time.Now().UTC() + j := &job.Job{ + ID: newTestJobID(), + Name: "count", + Queue: "default", + Payload: payload, + State: job.StatePending, + MaxRetries: 1, + RunAt: now, + CreatedAt: now, + UpdatedAt: now, + } + if err := s.EnqueueJob(context.Background(), j); err != nil { + t.Fatalf("enqueue: %v", err) + } + return j +} + +func waitForCount(t *testing.T, ran *atomic.Int64, want int64) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for ran.Load() < want { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %d jobs to run, %d ran", want, ran.Load()) + } + time.Sleep(5 * time.Millisecond) + } +} + +// safeCall runs fn and turns a panic into an error, so a lifecycle bug +// fails the test instead of crashing the test binary. +func safeCall(fn func() error) (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() + return fn() +} + +func stopPool(t *testing.T, p *worker.Pool) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := safeCall(func() error { return p.Stop(ctx) }); err != nil { + t.Fatalf("stop: %v", err) + } +} + +// A pool that is stopped and started again must actually run again: the +// second run's loops must not see the first run's closed stop channel or +// cancelled context and exit at once. +func TestPool_RestartProcessesNewJob(t *testing.T) { + pool, s, ran := setupRestartPool(t) + + if err := pool.Start(context.Background()); err != nil { + t.Fatalf("start: %v", err) + } + enqueueCountJob(t, s) + pool.Wake() + waitForCount(t, ran, 1) + stopPool(t, pool) + + if err := pool.Ready(); err == nil { + t.Fatal("Ready() = nil after Stop, want an error") + } + + if err := pool.Start(context.Background()); err != nil { + t.Fatalf("restart: %v", err) + } + t.Cleanup(func() { stopPool(t, pool) }) + + j := enqueueCountJob(t, s) + pool.Wake() + waitForCount(t, ran, 2) + + got, err := s.GetJob(context.Background(), j.ID) + if err != nil { + t.Fatalf("get job: %v", err) + } + deadline := time.Now().Add(3 * time.Second) + for got.State != job.StateCompleted && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + if got, err = s.GetJob(context.Background(), j.ID); err != nil { + t.Fatalf("get job: %v", err) + } + } + if got.State != job.StateCompleted { + t.Fatalf("job state after restart = %q, want %q", got.State, job.StateCompleted) + } + + if err := pool.Ready(); err != nil { + t.Fatalf("Ready() after restart = %v, want nil", err) + } +} + +// Stop is called more than once on shutdown (Engine and Dispatcher both +// stop the pool). That has to stay a no-op after a restart too, rather +// than closing the run's stop channel a second time. +func TestPool_StopTwiceAfterRestart(t *testing.T) { + pool, _, _ := setupRestartPool(t) + ctx := context.Background() + + for run := range 3 { + if err := safeCall(func() error { return pool.Start(ctx) }); err != nil { + t.Fatalf("start (run %d): %v", run, err) + } + stopPool(t, pool) + stopPool(t, pool) + } +} + +// Start and Stop may race from different goroutines. Every call must be +// safe, and the pool must still work afterwards. +func TestPool_ConcurrentStartStop(t *testing.T) { + pool, s, ran := setupRestartPool(t) + ctx := context.Background() + + var wg sync.WaitGroup + errs := make(chan error, 64) + for g := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for i := range 20 { + var err error + if (g+i)%2 == 0 { + err = safeCall(func() error { return pool.Start(ctx) }) + } else { + stopCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + err = safeCall(func() error { return pool.Stop(stopCtx) }) + cancel() + } + if err != nil { + errs <- err + return + } + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Errorf("concurrent lifecycle call: %v", err) + } + if t.Failed() { + return + } + + stopPool(t, pool) + if err := pool.Start(ctx); err != nil { + t.Fatalf("start after concurrent calls: %v", err) + } + t.Cleanup(func() { stopPool(t, pool) }) + + enqueueCountJob(t, s) + pool.Wake() + waitForCount(t, ran, 1) +}