Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 77 additions & 18 deletions worker/pool.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,17 +146,29 @@ type Pool struct {
// and starts spans on the global tracer provider.
tel *telemetry.Telemetry

// Per-run state. Start makes a fresh set of these for every run and
// Stop retires them, so a pool that is stopped and started again never
// hands its new goroutines a channel or context the old run closed.
// They are written only by Start, under lifecycleMu, before any loop
// goroutine of that run exists, and read by those goroutines, which
// Stop waits out before it releases lifecycleMu.
stopCh chan struct{}
wakeCh chan struct{} // nudges the fetcher out of its idle backoff
jobCh chan admitted // hand-off from the fetcher to the workers
slots chan struct{} // free-worker tokens; capacity == concurrency
cancelCtx context.Context // Cancelled on Stop to interrupt in-flight store operations
cancelFunc context.CancelFunc // Cancels cancelCtx
wg sync.WaitGroup
mu sync.Mutex
running bool
activeJobs map[string]*inflight
activeMu sync.Mutex

wakeCh chan struct{} // nudges the fetcher out of its idle backoff

// lifecycleMu serializes Start and Stop. Stop holds it until every
// goroutine of the run has exited, so a concurrent Start waits for the
// old run to finish instead of racing it for the per-run fields above.
lifecycleMu sync.Mutex
wg sync.WaitGroup
mu sync.Mutex
running bool // written under lifecycleMu and mu; read under either
activeJobs map[string]*inflight
activeMu sync.Mutex
}

// PoolOption configures a Pool.
Expand Down Expand Up @@ -344,7 +356,6 @@ func NewPool(
maxPollInterval: 30 * time.Second,
workerID: id.NewWorkerID(),
logger: logger,
stopCh: make(chan struct{}),
wakeCh: make(chan struct{}, 1),
activeJobs: make(map[string]*inflight),
}
Expand Down Expand Up @@ -377,17 +388,37 @@ func (p *Pool) Wake() {
// WorkerID returns the pool's unique worker identifier.
func (p *Pool) WorkerID() id.WorkerID { return p.workerID }

// Start launches the worker goroutines. It returns immediately.
// Start launches the worker goroutines. It returns immediately. Calling it
// while the pool is running is a no-op; calling it after Stop begins a
// fresh run with its own stop channel, context and worker slots.
func (p *Pool) Start(ctx context.Context) error {
p.mu.Lock()
p.lifecycleMu.Lock()
defer p.lifecycleMu.Unlock()

if p.running {
p.mu.Unlock()
return nil
}
p.running = true

// Fresh per-run state. The previous run, if any, closed its stop
// channel and cancelled its context, and every loop that saw them has
// exited: Stop waited for them before releasing lifecycleMu.
p.stopCh = make(chan struct{})
p.cancelCtx, p.cancelFunc = context.WithCancel(context.Background())

// A nudge left over from the previous run, or sent while stopped, is
// already satisfied: the new fetcher polls as soon as it starts.
select {
case <-p.wakeCh:
default:
}

// Readiness describes this run's polls, not the last run's.
p.mu.Lock()
p.running = true
p.polled = false
p.pollErr = nil
p.mu.Unlock()

p.logger.Info("worker pool starting",
log.String("worker_id", p.workerID.String()),
log.Int("concurrency", p.concurrency),
Expand Down Expand Up @@ -424,8 +455,6 @@ func (p *Pool) Start(ctx context.Context) error {
go p.reaperLoop()
}

p.mu.Unlock()

// Sweep sandboxes this worker left behind across a restart before it
// takes new work. In-process reclaim is a no-op; an out-of-process rung
// would otherwise keep orphaned children or pods alive indefinitely.
Expand All @@ -435,9 +464,8 @@ func (p *Pool) Start(ctx context.Context) error {
// Run in the background, tracked by p.wg like every other pool
// goroutine, so Start returns immediately as documented even when a
// rung's Reclaim does real (and potentially slow) process or
// filesystem I/O. Outside p.mu: nothing else needs the lock held for
// this, and holding it here would block Stop and every other pool
// method on a sweep that has no bound of its own.
// filesystem I/O. Running it inline would hold lifecycleMu, and so
// block Stop, on a sweep that has no bound of its own.
if p.executor != nil {
p.wg.Add(1)
go p.runReclaimSweep(ctx)
Expand Down Expand Up @@ -495,12 +523,18 @@ func mergeDone(a, b context.Context) (context.Context, context.CancelFunc) {

// Stop signals all workers to stop and waits for them to finish.
// If the context has a deadline, active jobs are cancelled when time runs out.
// Calling it while the pool is not running is a no-op, so it is safe to call
// more than once. A later Start begins a fresh run.
func (p *Pool) Stop(ctx context.Context) error {
p.mu.Lock()
p.lifecycleMu.Lock()
defer p.lifecycleMu.Unlock()

if !p.running {
p.mu.Unlock()
return nil
}

// Ready reports stopped from here on, while the run drains.
p.mu.Lock()
p.running = false
p.mu.Unlock()

Expand Down Expand Up @@ -528,9 +562,34 @@ func (p *Pool) Stop(ctx context.Context) error {
p.wg.Wait()
}

// Every worker untracks its own job on the way out, so this is empty
// by now. Clear it anyway so a record can never outlive its run: its
// lease timer would otherwise fire into the next one.
p.dropActiveJobs()

return nil
}

// dropActiveJobs stops and discards any in-flight record still tracked
// after a run's goroutines have exited.
func (p *Pool) dropActiveJobs() {
p.activeMu.Lock()
defer p.activeMu.Unlock()

if len(p.activeJobs) == 0 {
return
}

for jobID, rec := range p.activeJobs {
if rec.leaseTimer != nil {
rec.leaseTimer.Stop()
}
rec.cancel(context.Canceled)
delete(p.activeJobs, jobID)
}
p.tel.Jobs().Active(0)
}

// callCtx wraps p.cancelCtx with the configured per-call timeout so a
// stalled driver session can't pin a pool connection indefinitely.
// Returns the unwrapped cancelCtx + a no-op cancel when the timeout
Expand Down
219 changes: 219 additions & 0 deletions worker/pool_restart_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
package worker_test

import (
"context"
"encoding/json"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"

"github.com/xraph/dispatch/backoff"
"github.com/xraph/dispatch/dlq"
"github.com/xraph/dispatch/ext"
"github.com/xraph/dispatch/job"
"github.com/xraph/dispatch/middleware"
"github.com/xraph/dispatch/store/memory"
"github.com/xraph/dispatch/worker"

log "github.com/xraph/go-utils/log"
)

// setupRestartPool builds a pool with every background loop enabled
// (fetcher, workers, heartbeat, reaper, reclaim sweep) so a restart has to
// give each of them fresh per-run state. Jobs named "count" bump the
// returned counter.
func setupRestartPool(t *testing.T) (*worker.Pool, *memory.Store, *atomic.Int64) {
t.Helper()
logger := log.NewNoopLogger()
s := memory.New()
reg := job.NewRegistry()
extensions := ext.NewRegistry(logger)

var ran atomic.Int64
job.RegisterDefinition(reg, job.NewDefinition("count", func(context.Context, struct{}) error {
ran.Add(1)
return nil
}))

executor := worker.NewExecutor(
reg, extensions, s, dlq.NewService(s, s), backoff.NewConstant(10*time.Millisecond), logger,
middleware.Recover(logger),
)

pool := worker.NewPool(s, executor, extensions, logger,
worker.WithPoolConcurrency(2),
worker.WithPollInterval(10*time.Millisecond),
worker.WithMaxPollInterval(20*time.Millisecond),
worker.WithHeartbeatInterval(10*time.Millisecond),
worker.WithStaleJobThreshold(time.Minute),
worker.WithReapInterval(10*time.Millisecond),
worker.WithPoolQueues([]string{"default"}),
)

return pool, s, &ran
}

func enqueueCountJob(t *testing.T, s *memory.Store) *job.Job {
t.Helper()
payload, _ := json.Marshal(struct{}{})
now := time.Now().UTC()
j := &job.Job{
ID: newTestJobID(),
Name: "count",
Queue: "default",
Payload: payload,
State: job.StatePending,
MaxRetries: 1,
RunAt: now,
CreatedAt: now,
UpdatedAt: now,
}
if err := s.EnqueueJob(context.Background(), j); err != nil {
t.Fatalf("enqueue: %v", err)
}
return j
}

func waitForCount(t *testing.T, ran *atomic.Int64, want int64) {
t.Helper()
deadline := time.Now().Add(3 * time.Second)
for ran.Load() < want {
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for %d jobs to run, %d ran", want, ran.Load())
}
time.Sleep(5 * time.Millisecond)
}
}

// safeCall runs fn and turns a panic into an error, so a lifecycle bug
// fails the test instead of crashing the test binary.
func safeCall(fn func() error) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("panic: %v", r)
}
}()
return fn()
}

func stopPool(t *testing.T, p *worker.Pool) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
if err := safeCall(func() error { return p.Stop(ctx) }); err != nil {
t.Fatalf("stop: %v", err)
}
}

// A pool that is stopped and started again must actually run again: the
// second run's loops must not see the first run's closed stop channel or
// cancelled context and exit at once.
func TestPool_RestartProcessesNewJob(t *testing.T) {
pool, s, ran := setupRestartPool(t)

if err := pool.Start(context.Background()); err != nil {
t.Fatalf("start: %v", err)
}
enqueueCountJob(t, s)
pool.Wake()
waitForCount(t, ran, 1)
stopPool(t, pool)

if err := pool.Ready(); err == nil {
t.Fatal("Ready() = nil after Stop, want an error")
}

if err := pool.Start(context.Background()); err != nil {
t.Fatalf("restart: %v", err)
}
t.Cleanup(func() { stopPool(t, pool) })

j := enqueueCountJob(t, s)
pool.Wake()
waitForCount(t, ran, 2)

got, err := s.GetJob(context.Background(), j.ID)
if err != nil {
t.Fatalf("get job: %v", err)
}
deadline := time.Now().Add(3 * time.Second)
for got.State != job.StateCompleted && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
if got, err = s.GetJob(context.Background(), j.ID); err != nil {
t.Fatalf("get job: %v", err)
}
}
if got.State != job.StateCompleted {
t.Fatalf("job state after restart = %q, want %q", got.State, job.StateCompleted)
}

if err := pool.Ready(); err != nil {
t.Fatalf("Ready() after restart = %v, want nil", err)
}
}

// Stop is called more than once on shutdown (Engine and Dispatcher both
// stop the pool). That has to stay a no-op after a restart too, rather
// than closing the run's stop channel a second time.
func TestPool_StopTwiceAfterRestart(t *testing.T) {
pool, _, _ := setupRestartPool(t)
ctx := context.Background()

for run := range 3 {
if err := safeCall(func() error { return pool.Start(ctx) }); err != nil {
t.Fatalf("start (run %d): %v", run, err)
}
stopPool(t, pool)
stopPool(t, pool)
}
}

// Start and Stop may race from different goroutines. Every call must be
// safe, and the pool must still work afterwards.
func TestPool_ConcurrentStartStop(t *testing.T) {
pool, s, ran := setupRestartPool(t)
ctx := context.Background()

var wg sync.WaitGroup
errs := make(chan error, 64)
for g := range 8 {
wg.Add(1)
go func() {
defer wg.Done()
for i := range 20 {
var err error
if (g+i)%2 == 0 {
err = safeCall(func() error { return pool.Start(ctx) })
} else {
stopCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
err = safeCall(func() error { return pool.Stop(stopCtx) })
cancel()
}
if err != nil {
errs <- err
return
}
}
}()
}
wg.Wait()
close(errs)
for err := range errs {
t.Errorf("concurrent lifecycle call: %v", err)
}
if t.Failed() {
return
}

stopPool(t, pool)
if err := pool.Start(ctx); err != nil {
t.Fatalf("start after concurrent calls: %v", err)
}
t.Cleanup(func() { stopPool(t, pool) })

enqueueCountJob(t, s)
pool.Wake()
waitForCount(t, ran, 1)
}
Loading