From 72b8e330f227631e1367d1bba7bb4133a44ffbae Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Wed, 18 Nov 2026 22:55:01 -0600 Subject: [PATCH] fix(dwp): make federation lifecycle, reconnects and writes concurrency-safe Federation had several races that made Stop hang and could corrupt the peer stream. - Start overwrote f.cancel unconditionally, so a second Start left the first heartbeat and health loops running with no way to cancel them, and Stop then waited on them forever. Start is now a no-op while running. Stop is final: Start and AddPeer return ErrFederationStopped afterwards. - Stop waited on the WaitGroup before closing peer connections, but the group held peer read loops blocked in ReadServerText and reconnects sleeping on time.Sleep with context.Background. Stop now cancels a federation-owned context, closes every peer connection, and only then waits. Reconnects wait on a timer selected against that context, and connectPeer ties the dial and the auth exchange to it (closing the socket to interrupt the auth read), with a 30s handshake bound. Installing a connection and adding its read loop happen under the lifecycle lock, so nothing is added to the WaitGroup once Stop has begun and a connection cannot be installed after Stop or RemovePeer has retired the peer. - On a heartbeat timeout, checkPeerHealth started a reconnect and the read loop of the connection it closed started a second one; both dialled, leaking a connection and running two read loops on one conn. A per-peer reconnecting flag now allows a single reconnect at a time, the health check tears the connection down in the same critical section as its check, and each read loop owns only the conn it was started with, so a stale loop exits quietly instead of clobbering the new connection's state. - writeJSONFrame writes a WebSocket frame as a header and a payload with no lock, and heartbeats and forwarded requests write concurrently, so frames interleaved and the peer dropped the connection. Writes now go through a per-peer write mutex. - GenerateFrameID was a bare nanosecond timestamp. Clock resolution is coarser than that on some platforms, so concurrent requests shared an ID and one of them lost its response. IDs now carry a process-wide sequence number. - GetPeer and Peers handed out the live *Peer whose State and LastSeen are written under its lock. They now return copies taken under the lock; the type and signatures are unchanged. AddPeer with an existing ID now disconnects the peer it replaces, and a reconnect stops once its peer is removed or replaced. Tests use a fake WebSocket peer and cover Stop with a connected peer, in reconnect backoff and in a stalled reconnect handshake; double Start; Stop being final; a timeout producing exactly one reconnect; concurrent forwards arriving as well-formed frames; race-free peer snapshots; and unique frame IDs under concurrency. --- dwp/federation.go | 364 ++++++++++++++++++------ dwp/federation_lifecycle_test.go | 473 +++++++++++++++++++++++++++++++ dwp/frame.go | 13 +- dwp/frame_test.go | 31 ++ 4 files changed, 789 insertions(+), 92 deletions(-) create mode 100644 dwp/federation_lifecycle_test.go diff --git a/dwp/federation.go b/dwp/federation.go index f5be934..3d73313 100644 --- a/dwp/federation.go +++ b/dwp/federation.go @@ -3,7 +3,9 @@ package dwp import ( "context" "encoding/json" + "errors" "fmt" + "maps" "net" "sync" "sync/atomic" @@ -41,10 +43,12 @@ type Peer struct { // Token is the authentication token for the peer. Token string - // State tracks the connection state. + // State tracks the connection state. On a *Peer returned by + // Federation.GetPeer or Federation.Peers it is a point-in-time copy. State PeerState - // LastSeen is the timestamp of the last heartbeat. + // LastSeen is the timestamp of the last heartbeat. On a *Peer returned + // by Federation.GetPeer or Federation.Peers it is a point-in-time copy. LastSeen time.Time // Metadata carries arbitrary peer information. @@ -53,13 +57,48 @@ type Peer struct { // conn is the underlying WebSocket connection. conn net.Conn + // reconnecting is set while a reconnect goroutine owns this peer, so a + // read error and a health-check timeout cannot both start one. + reconnecting bool + // pending tracks request-response correlation. pending sync.Map // frameID → chan *Frame - // mu protects state updates. + // mu protects State, LastSeen, conn and reconnecting. mu sync.RWMutex + + // wmu serialises frame writes on conn. A WebSocket frame is written as + // a header and a payload, so concurrent writers would interleave them. + wmu sync.Mutex +} + +// snapshot returns a copy of the peer's exported fields, taken under its +// lock, that callers can read without synchronisation. +func (p *Peer) snapshot() *Peer { + p.mu.RLock() + defer p.mu.RUnlock() + return &Peer{ + ID: p.ID, + URL: p.URL, + Token: p.Token, + State: p.State, + LastSeen: p.LastSeen, + Metadata: maps.Clone(p.Metadata), + } +} + +// writeFrame writes one frame to conn, serialised with every other writer +// on this peer. +func (p *Peer) writeFrame(conn net.Conn, frame *Frame) error { + p.wmu.Lock() + defer p.wmu.Unlock() + return writeJSONFrame(conn, frame) } +// ErrFederationStopped is returned by Start and AddPeer once Stop has been +// called. A stopped Federation cannot be restarted; create a new one. +var ErrFederationStopped = errors.New("dwp/federation: federation stopped") + // ── Federation ───────────────────────────────── // Federation manages server-to-server communication between Dispatch @@ -90,9 +129,20 @@ type Federation struct { jobsForwarded atomic.Int64 eventsForwarded atomic.Int64 - // Lifecycle. - cancel context.CancelFunc - wg sync.WaitGroup + // handshakeTimeout bounds the dial-and-auth exchange with a peer. + handshakeTimeout time.Duration + + // Lifecycle. mu guards started, stopped, stopLoops and every wg.Add, + // so no goroutine is added once Stop has begun waiting. ctx is the + // federation's own lifetime: it scopes peer read loops and reconnects + // and is cancelled by Stop. + mu sync.Mutex + started bool + stopped bool + ctx context.Context + cancel context.CancelFunc + stopLoops context.CancelFunc + wg sync.WaitGroup } // FederationOption configures a Federation. @@ -133,16 +183,32 @@ func NewFederation(eng *engine.Engine, broker *stream.Broker, logger log.Logger, heartbeatTimeout: 45 * time.Second, reconnectBackoff: 2 * time.Second, maxReconnect: 60 * time.Second, + handshakeTimeout: 30 * time.Second, } + f.ctx, f.cancel = context.WithCancel(context.Background()) for _, opt := range opts { opt(f) } return f } -// Start begins heartbeat monitoring and peer health checks. -func (f *Federation) Start(ctx context.Context) { - ctx, f.cancel = context.WithCancel(ctx) +// Start begins heartbeat monitoring and peer health checks. The loops run +// until ctx is cancelled or Stop is called. Calling Start again while the +// federation is running is a no-op; calling it after Stop returns +// ErrFederationStopped. +func (f *Federation) Start(ctx context.Context) error { + f.mu.Lock() + defer f.mu.Unlock() + + if f.stopped { + return ErrFederationStopped + } + if f.started { + return nil + } + f.started = true + + ctx, f.stopLoops = context.WithCancel(ctx) // Heartbeat sender. f.wg.Add(1) @@ -157,25 +223,61 @@ func (f *Federation) Start(ctx context.Context) { defer f.wg.Done() f.healthCheckLoop(ctx) }() + return nil } -// Stop gracefully shuts down all peer connections. +// Stop shuts down the federation: it stops the heartbeat and health loops, +// cancels pending reconnects, closes every peer connection and waits for +// all federation goroutines to exit. Stop is final and safe to call more +// than once. func (f *Federation) Stop() { - if f.cancel != nil { + f.mu.Lock() + if !f.stopped { + f.stopped = true f.cancel() + if f.stopLoops != nil { + f.stopLoops() + } } - f.wg.Wait() + f.mu.Unlock() + // Closing the connections unblocks the read loops. No connection can be + // installed after this point: connectPeer checks f.stopped under f.mu. f.peers.Range(func(_, value any) bool { peer := value.(*Peer) //nolint:errcheck // sync.Map always stores *Peer f.disconnectPeer(peer) return true }) + + f.wg.Wait() +} + +// goTracked runs fn in a goroutine tracked by f.wg, passing the federation +// context. It reports false, without running fn, once Stop has been called. +func (f *Federation) goTracked(fn func(ctx context.Context)) bool { + f.mu.Lock() + defer f.mu.Unlock() + if f.stopped { + return false + } + f.wg.Add(1) + go func() { + defer f.wg.Done() + fn(f.ctx) + }() + return true +} + +// isCurrent reports whether peer is still the registered peer for its ID. +func (f *Federation) isCurrent(peer *Peer) bool { + val, ok := f.peers.Load(peer.ID) + return ok && val.(*Peer) == peer //nolint:errcheck // sync.Map always stores *Peer } // ── Peer Management ──────────────────────────── -// AddPeer registers and connects to a remote peer. +// AddPeer registers and connects to a remote peer. A peer already +// registered under peerID is disconnected and replaced. func (f *Federation) AddPeer(ctx context.Context, peerID, url, token string, metadata map[string]string) error { peer := &Peer{ ID: peerID, @@ -184,34 +286,52 @@ func (f *Federation) AddPeer(ctx context.Context, peerID, url, token string, met State: PeerStateDisconnected, Metadata: metadata, } - f.peers.Store(peerID, peer) + + f.mu.Lock() + if f.stopped { + f.mu.Unlock() + return ErrFederationStopped + } + old, replaced := f.peers.Swap(peerID, peer) + f.mu.Unlock() + + if replaced { + f.disconnectPeer(old.(*Peer)) //nolint:errcheck // sync.Map always stores *Peer + } return f.connectPeer(ctx, peer) } // RemovePeer disconnects and removes a peer. func (f *Federation) RemovePeer(peerID string) { + // Delete under f.mu so a concurrent connectPeer either installs its + // connection before the removal (and it is closed below) or sees the + // peer gone and discards the connection. + f.mu.Lock() val, ok := f.peers.LoadAndDelete(peerID) + f.mu.Unlock() if !ok { return } f.disconnectPeer(val.(*Peer)) //nolint:errcheck // sync.Map always stores *Peer } -// GetPeer returns a peer by ID. +// GetPeer returns a snapshot of a peer by ID. The returned value is a copy +// taken under the peer's lock; it does not track later state changes. func (f *Federation) GetPeer(peerID string) (*Peer, bool) { val, ok := f.peers.Load(peerID) if !ok { return nil, false } - return val.(*Peer), true //nolint:errcheck // sync.Map always stores *Peer + return val.(*Peer).snapshot(), true //nolint:errcheck // sync.Map always stores *Peer } -// Peers returns a snapshot of all peers. +// Peers returns a snapshot of all peers. Each element is a copy taken under +// the peer's lock; it does not track later state changes. func (f *Federation) Peers() []*Peer { var peers []*Peer f.peers.Range(func(_, value any) bool { - peers = append(peers, value.(*Peer)) //nolint:errcheck // sync.Map always stores *Peer + peers = append(peers, value.(*Peer).snapshot()) //nolint:errcheck // sync.Map always stores *Peer return true }) return peers @@ -372,14 +492,31 @@ func (f *Federation) connectPeer(ctx context.Context, peer *Peer) error { peer.State = PeerStateConnecting peer.mu.Unlock() - conn, _, _, err := ws.Dial(ctx, peer.URL) - if err != nil { + fail := func(conn net.Conn, err error) error { + if conn != nil { + _ = conn.Close() + } peer.mu.Lock() peer.State = PeerStateDisconnected peer.mu.Unlock() - return fmt.Errorf("dwp/federation: dial %q: %w", peer.ID, err) + return err + } + + // Stop must be able to interrupt a handshake in progress, so tie the + // exchange to the federation context as well as the caller's. + hctx, cancel := context.WithTimeout(ctx, f.handshakeTimeout) + defer cancel() + stopWithFed := context.AfterFunc(f.ctx, cancel) + defer stopWithFed() + + conn, _, _, err := ws.Dial(hctx, peer.URL) + if err != nil { + return fail(nil, fmt.Errorf("dwp/federation: dial %q: %w", peer.ID, err)) } + // Closing the connection is the only way to unblock the auth read. + stopClose := context.AfterFunc(hctx, func() { _ = conn.Close() }) + // Authenticate with the peer. authFrame := &Frame{ ID: GenerateFrameID(), @@ -389,56 +526,64 @@ func (f *Federation) connectPeer(ctx context.Context, peer *Peer) error { } if writeErr := writeJSONFrame(conn, authFrame); writeErr != nil { - _ = conn.Close() - peer.mu.Lock() - peer.State = PeerStateDisconnected - peer.mu.Unlock() - return fmt.Errorf("dwp/federation: auth write %q: %w", peer.ID, writeErr) + stopClose() + return fail(conn, fmt.Errorf("dwp/federation: auth write %q: %w", peer.ID, writeErr)) } // Read auth response. data, err := wsutil.ReadServerText(conn) + if !stopClose() { + // hctx fired: the connection was closed under the read, or is + // about to be, so the result cannot be used either way. + return fail(conn, fmt.Errorf("dwp/federation: auth read %q: %w", peer.ID, hctx.Err())) + } if err != nil { - _ = conn.Close() - peer.mu.Lock() - peer.State = PeerStateDisconnected - peer.mu.Unlock() - return fmt.Errorf("dwp/federation: auth read %q: %w", peer.ID, err) + return fail(conn, fmt.Errorf("dwp/federation: auth read %q: %w", peer.ID, err)) } var resp Frame if err := json.Unmarshal(data, &resp); err != nil { - _ = conn.Close() - peer.mu.Lock() - peer.State = PeerStateDisconnected - peer.mu.Unlock() - return fmt.Errorf("dwp/federation: auth parse %q: %w", peer.ID, err) + return fail(conn, fmt.Errorf("dwp/federation: auth parse %q: %w", peer.ID, err)) } if resp.Type == FrameErr { - _ = conn.Close() - peer.mu.Lock() - peer.State = PeerStateDisconnected - peer.mu.Unlock() msg := "auth failed" if resp.Error != nil { msg = resp.Error.Message } - return fmt.Errorf("dwp/federation: %s at %q", msg, peer.ID) + return fail(conn, fmt.Errorf("dwp/federation: %s at %q", msg, peer.ID)) + } + + // Install the connection and start its read loop atomically with + // respect to Stop and RemovePeer, which both hold f.mu while they + // retire the federation or the peer. + f.mu.Lock() + if f.stopped { + f.mu.Unlock() + return fail(conn, ErrFederationStopped) + } + if !f.isCurrent(peer) { + f.mu.Unlock() + return fail(conn, fmt.Errorf("dwp/federation: peer %q removed", peer.ID)) } peer.mu.Lock() + if peer.conn != nil { + // Should not happen while reconnects are exclusive, but never + // orphan a live connection. + _ = peer.conn.Close() + } peer.conn = conn peer.State = PeerStateConnected peer.LastSeen = time.Now().UTC() peer.mu.Unlock() - // Start read loop for this peer. f.wg.Add(1) go func() { defer f.wg.Done() - f.peerReadLoop(peer) + f.peerReadLoop(peer, conn) }() + f.mu.Unlock() f.logger.Info("federation peer connected", log.String("peer_id", peer.ID), @@ -458,33 +603,30 @@ func (f *Federation) disconnectPeer(peer *Peer) { peer.State = PeerStateDisconnected } -func (f *Federation) peerReadLoop(peer *Peer) { +// peerReadLoop reads frames from conn until it fails. If conn is still the +// peer's active connection at that point, the loop tears it down and asks +// for a reconnect; if someone else already replaced or closed it (health +// check, RemovePeer, Stop), they own what happens next. +func (f *Federation) peerReadLoop(peer *Peer, conn net.Conn) { for { - peer.mu.RLock() - conn := peer.conn - state := peer.State - peer.mu.RUnlock() - - if conn == nil || state != PeerStateConnected { - return - } - data, err := wsutil.ReadServerText(conn) if err != nil { - f.logger.Warn("federation peer read error", - log.String("peer_id", peer.ID), - log.String("error", err.Error()), - ) peer.mu.Lock() - peer.State = PeerStateDisconnected + owned := peer.conn == conn + if owned { + _ = conn.Close() + peer.conn = nil + peer.State = PeerStateDisconnected + } peer.mu.Unlock() - // Attempt reconnect. - f.wg.Add(1) - go func() { - defer f.wg.Done() - f.reconnectPeer(peer) - }() + if owned { + f.logger.Warn("federation peer read error", + log.String("peer_id", peer.ID), + log.String("error", err.Error()), + ) + f.startReconnect(peer) + } return } @@ -511,30 +653,65 @@ func (f *Federation) peerReadLoop(peer *Peer) { } } -func (f *Federation) reconnectPeer(peer *Peer) { +// startReconnect launches a reconnect for peer unless one is already +// running, so at most one goroutine dials a given peer at a time. +func (f *Federation) startReconnect(peer *Peer) { + peer.mu.Lock() + if peer.reconnecting { + peer.mu.Unlock() + return + } + peer.reconnecting = true + peer.mu.Unlock() + + release := func() { + peer.mu.Lock() + peer.reconnecting = false + peer.mu.Unlock() + } + + if !f.goTracked(func(ctx context.Context) { + defer release() + f.reconnectPeer(ctx, peer) + }) { + release() + } +} + +// reconnectPeer redials peer with exponential backoff until it connects, +// the peer is removed, or ctx (the federation context) is cancelled. +func (f *Federation) reconnectPeer(ctx context.Context, peer *Peer) { backoff := f.reconnectBackoff for { - peer.mu.RLock() - state := peer.State - peer.mu.RUnlock() + f.logger.Info("federation reconnecting to peer", + log.String("peer_id", peer.ID), + log.Duration("backoff", backoff), + ) - // Don't reconnect if peer was removed. - if _, exists := f.peers.Load(peer.ID); !exists { + timer := time.NewTimer(backoff) + select { + case <-ctx.Done(): + timer.Stop() return + case <-timer.C: } - if state == PeerStateConnected { + // Don't reconnect if the peer was removed or replaced. + if !f.isCurrent(peer) { return } - f.logger.Info("federation reconnecting to peer", - log.String("peer_id", peer.ID), - log.Duration("backoff", backoff), - ) - - time.Sleep(backoff) + peer.mu.RLock() + state := peer.State + peer.mu.RUnlock() + if state == PeerStateConnected { + return + } - if err := f.connectPeer(context.Background(), peer); err != nil { + if err := f.connectPeer(ctx, peer); err != nil { + if ctx.Err() != nil || errors.Is(err, ErrFederationStopped) { + return + } backoff = min(backoff*2, f.maxReconnect) continue } @@ -571,7 +748,7 @@ func (f *Federation) peerRequest(ctx context.Context, peer *Peer, method string, return nil, fmt.Errorf("peer not connected") } - if err := writeJSONFrame(conn, frame); err != nil { + if err := peer.writeFrame(conn, frame); err != nil { return nil, err } @@ -621,7 +798,7 @@ func (f *Federation) sendHeartbeats() { Data: mustMarshalJSON(FederationHeartbeatRequest{PeerID: f.localID}), } - if err := writeJSONFrame(conn, heartbeat); err != nil { + if err := peer.writeFrame(conn, heartbeat); err != nil { f.logger.Warn("federation heartbeat failed", log.String("peer_id", peer.ID), log.String("error", err.Error()), @@ -649,23 +826,28 @@ func (f *Federation) checkPeerHealth() { now := time.Now().UTC() f.peers.Range(func(_, value any) bool { peer := value.(*Peer) //nolint:errcheck // sync.Map always stores *Peer - peer.mu.RLock() - state := peer.State + + // Check and tear down in one critical section, so the read loop of + // the closed connection sees it is no longer current and leaves + // the reconnect to us. + peer.mu.Lock() lastSeen := peer.LastSeen - peer.mu.RUnlock() + timedOut := peer.State == PeerStateConnected && now.Sub(lastSeen) > f.heartbeatTimeout + if timedOut { + if peer.conn != nil { + _ = peer.conn.Close() + peer.conn = nil + } + peer.State = PeerStateDisconnected + } + peer.mu.Unlock() - if state == PeerStateConnected && now.Sub(lastSeen) > f.heartbeatTimeout { + if timedOut { f.logger.Warn("federation peer timed out", log.String("peer_id", peer.ID), log.Duration("since_last_seen", now.Sub(lastSeen)), ) - f.disconnectPeer(peer) - - f.wg.Add(1) - go func() { - defer f.wg.Done() - f.reconnectPeer(peer) - }() + f.startReconnect(peer) } return true }) diff --git a/dwp/federation_lifecycle_test.go b/dwp/federation_lifecycle_test.go new file mode 100644 index 0000000..cd5b048 --- /dev/null +++ b/dwp/federation_lifecycle_test.go @@ -0,0 +1,473 @@ +package dwp + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gobwas/ws" + "github.com/gobwas/ws/wsutil" +) + +// ── Fake peer harness ────────────────────────────── + +// fakePeerConn is the server side of one federation connection. Writes are +// serialised so a test handler may reply from several goroutines. +type fakePeerConn struct { + conn net.Conn + wmu sync.Mutex +} + +func (c *fakePeerConn) read() (*Frame, error) { + data, err := wsutil.ReadClientText(c.conn) + if err != nil { + return nil, err + } + var f Frame + if err := json.Unmarshal(data, &f); err != nil { + return nil, fmt.Errorf("invalid frame payload %q: %w", data, err) + } + return &f, nil +} + +func (c *fakePeerConn) write(f *Frame) error { + data, err := json.Marshal(f) + if err != nil { + return err + } + c.wmu.Lock() + defer c.wmu.Unlock() + return wsutil.WriteServerText(c.conn, data) +} + +// replyOK answers req with an empty success response. +func (c *fakePeerConn) replyOK(req *Frame) error { + return c.write(&Frame{ + ID: GenerateFrameID(), + Type: FrameResponse, + CorrelID: req.ID, + Data: json.RawMessage(`{"ok":true}`), + Timestamp: time.Now().UTC(), + }) +} + +// authenticate reads the auth frame and accepts it. +func (c *fakePeerConn) authenticate() error { + req, err := c.read() + if err != nil { + return err + } + if req.Method != MethodAuth { + return fmt.Errorf("first frame method = %q, want %q", req.Method, MethodAuth) + } + return c.replyOK(req) +} + +// fakePeer is a minimal DWP endpoint for federation tests. handle is called +// once per accepted connection with its 1-based sequence number; the +// connection is closed when handle returns. +type fakePeer struct { + srv *httptest.Server + accepted atomic.Int64 + open atomic.Int64 + handle func(n int64, c *fakePeerConn) +} + +func newFakePeer(t *testing.T, handle func(n int64, c *fakePeerConn)) *fakePeer { + t.Helper() + p := &fakePeer{handle: handle} + p.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, _, _, err := ws.UpgradeHTTP(r, w) + if err != nil { + return + } + n := p.accepted.Add(1) + p.open.Add(1) + defer p.open.Add(-1) + defer conn.Close() + p.handle(n, &fakePeerConn{conn: conn}) + })) + t.Cleanup(func() { + p.srv.CloseClientConnections() + p.srv.Close() + }) + return p +} + +func (p *fakePeer) url() string { + return "ws" + strings.TrimPrefix(p.srv.URL, "http") +} + +// serveRequests authenticates and then answers every request until the +// connection fails. Malformed frames are reported through bad. +func serveRequests(c *fakePeerConn, bad *atomic.Int64) { + if err := c.authenticate(); err != nil { + return + } + for { + req, err := c.read() + if err != nil { + if !isClosedErr(err) { + bad.Add(1) + } + return + } + if req.Type == FrameRequest { + if err := c.replyOK(req); err != nil { + return + } + } + } +} + +func isClosedErr(err error) bool { + var closed wsutil.ClosedError + if errors.As(err, &closed) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, net.ErrClosed) { + return true + } + return strings.Contains(err.Error(), "connection reset") +} + +func newTestFederation(opts ...FederationOption) *Federation { + all := make([]FederationOption, 0, 3+len(opts)) + all = append(all, + WithLocalID("local"), + WithHeartbeatInterval(time.Hour), + WithHeartbeatTimeout(time.Hour), + ) + return NewFederation(nil, nil, testLogger(), append(all, opts...)...) +} + +// stopTimeout bounds Federation.Stop in tests; the old code hung forever. +const stopTimeout = 2 * time.Second + +// stopWithin calls f.Stop and fails the test if it does not return within +// stopTimeout. +func stopWithin(t *testing.T, f *Federation) { + t.Helper() + d := stopTimeout + done := make(chan struct{}) + go func() { + f.Stop() + close(done) + }() + select { + case <-done: + case <-time.After(d): + t.Fatalf("Federation.Stop did not return within %s", d) + } +} + +func waitFor(t *testing.T, d time.Duration, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(d) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(5 * time.Millisecond) + } +} + +// ── Lifecycle ────────────────────────────────────── + +func TestFederation_StopWithConnectedPeer(t *testing.T) { + var bad atomic.Int64 + peer := newFakePeer(t, func(_ int64, c *fakePeerConn) { serveRequests(c, &bad) }) + + f := newTestFederation() + f.Start(t.Context()) + if err := f.AddPeer(t.Context(), "remote", peer.url(), "tok", nil); err != nil { + t.Fatalf("AddPeer: %v", err) + } + + stopWithin(t, f) + + if s := f.Stats(); s.ConnectedPeers != 0 { + t.Errorf("ConnectedPeers after Stop = %d, want 0", s.ConnectedPeers) + } +} + +func TestFederation_StopDuringReconnectBackoff(t *testing.T) { + var bad atomic.Int64 + peer := newFakePeer(t, func(n int64, c *fakePeerConn) { + if n == 1 { + // Accept, then drop the connection to force a reconnect. + _ = c.authenticate() + return + } + serveRequests(c, &bad) + }) + + f := newTestFederation() + f.reconnectBackoff = 10 * time.Second + f.Start(t.Context()) + if err := f.AddPeer(t.Context(), "remote", peer.url(), "tok", nil); err != nil { + t.Fatalf("AddPeer: %v", err) + } + waitFor(t, 2*time.Second, "peer to drop", func() bool { return f.Stats().ConnectedPeers == 0 }) + + stopWithin(t, f) + + if got := peer.accepted.Load(); got != 1 { + t.Errorf("connections after Stop = %d, want 1 (no reconnect after Stop)", got) + } +} + +func TestFederation_StopDuringReconnectHandshake(t *testing.T) { + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + + peer := newFakePeer(t, func(n int64, c *fakePeerConn) { + if n == 1 { + _ = c.authenticate() + return + } + // Swallow the auth frame and never answer it. + _, _ = c.read() + <-release + }) + + f := newTestFederation() + f.reconnectBackoff = time.Millisecond + f.Start(t.Context()) + if err := f.AddPeer(t.Context(), "remote", peer.url(), "tok", nil); err != nil { + t.Fatalf("AddPeer: %v", err) + } + waitFor(t, 2*time.Second, "reconnect attempt", func() bool { return peer.accepted.Load() >= 2 }) + + stopWithin(t, f) +} + +func TestFederation_DoubleStart(t *testing.T) { + var pings atomic.Int64 + peer := newFakePeer(t, func(_ int64, c *fakePeerConn) { + if err := c.authenticate(); err != nil { + return + } + for { + req, err := c.read() + if err != nil { + return + } + if req.Type == FramePing { + pings.Add(1) + } + } + }) + + f := newTestFederation(WithHeartbeatInterval(20 * time.Millisecond)) + f.Start(t.Context()) + f.Start(t.Context()) + if err := f.AddPeer(t.Context(), "remote", peer.url(), "tok", nil); err != nil { + t.Fatalf("AddPeer: %v", err) + } + + stopWithin(t, f) + + // No heartbeat loop may survive Stop. + before := pings.Load() + time.Sleep(100 * time.Millisecond) + if after := pings.Load(); after != before { + t.Errorf("heartbeats after Stop: %d -> %d", before, after) + } +} + +func TestFederation_StopIsFinal(t *testing.T) { + var bad atomic.Int64 + peer := newFakePeer(t, func(_ int64, c *fakePeerConn) { serveRequests(c, &bad) }) + + f := newTestFederation() + if err := f.Start(t.Context()); err != nil { + t.Fatalf("Start: %v", err) + } + if err := f.Start(t.Context()); err != nil { + t.Fatalf("second Start: %v", err) + } + stopWithin(t, f) + stopWithin(t, f) // idempotent + + if err := f.Start(t.Context()); !errors.Is(err, ErrFederationStopped) { + t.Errorf("Start after Stop = %v, want ErrFederationStopped", err) + } + if err := f.AddPeer(t.Context(), "remote", peer.url(), "tok", nil); !errors.Is(err, ErrFederationStopped) { + t.Errorf("AddPeer after Stop = %v, want ErrFederationStopped", err) + } + if got := peer.accepted.Load(); got != 0 { + t.Errorf("connections after Stop = %d, want 0", got) + } +} + +// ── Peer health ──────────────────────────────────── + +func TestFederation_TimeoutReconnectsOnce(t *testing.T) { + peer := newFakePeer(t, func(n int64, c *fakePeerConn) { + if err := c.authenticate(); err != nil { + return + } + if n == 1 { + // Stay silent so the health check times the peer out, and keep + // the socket open until the client closes it. + for { + if _, err := c.read(); err != nil { + return + } + } + } + // Later connections keep the peer alive. + stop := make(chan struct{}) + defer close(stop) + go func() { + tick := time.NewTicker(20 * time.Millisecond) + defer tick.Stop() + for { + select { + case <-stop: + return + case <-tick.C: + if c.write(&Frame{ID: GenerateFrameID(), Type: FramePong, Timestamp: time.Now().UTC()}) != nil { + return + } + } + } + }() + for { + if _, err := c.read(); err != nil { + return + } + } + }) + + f := newTestFederation(WithHeartbeatTimeout(200 * time.Millisecond)) + f.reconnectBackoff = 20 * time.Millisecond + f.Start(t.Context()) + t.Cleanup(func() { stopWithin(t, f) }) + if err := f.AddPeer(t.Context(), "remote", peer.url(), "tok", nil); err != nil { + t.Fatalf("AddPeer: %v", err) + } + + waitFor(t, 3*time.Second, "reconnect", func() bool { return peer.accepted.Load() >= 2 }) + // Give a duplicate reconnect time to show up. + time.Sleep(300 * time.Millisecond) + + if got := peer.accepted.Load(); got != 2 { + t.Errorf("connections accepted = %d, want 2 (one reconnect)", got) + } + if got := peer.open.Load(); got != 1 { + t.Errorf("connections open = %d, want 1", got) + } + if s := f.Stats(); s.ConnectedPeers != 1 { + t.Errorf("ConnectedPeers = %d, want 1", s.ConnectedPeers) + } +} + +// ── Writes ───────────────────────────────────────── + +func TestFederation_ConcurrentWritesDoNotInterleave(t *testing.T) { + var bad atomic.Int64 + peer := newFakePeer(t, func(_ int64, c *fakePeerConn) { serveRequests(c, &bad) }) + + f := newTestFederation(WithHeartbeatInterval(time.Millisecond)) + f.Start(t.Context()) + t.Cleanup(func() { stopWithin(t, f) }) + if err := f.AddPeer(t.Context(), "remote", peer.url(), "tok", nil); err != nil { + t.Fatalf("AddPeer: %v", err) + } + + const ( + workers = 32 + perW = 25 + ) + payload := json.RawMessage(`{"blob":"` + strings.Repeat("x", 4096) + `"}`) + + // A desynchronised stream loses responses, so bound the wait rather than + // sitting out the 30s request timeout. + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + + var wg sync.WaitGroup + var failed atomic.Int64 + var errOnce sync.Once + var firstErr error + for w := range workers { + wg.Add(1) + go func() { + defer wg.Done() + for i := range perW { + var err error + if (w+i)%2 == 0 { + err = f.ForwardEvent(ctx, "remote", "evt", payload) + } else { + err = f.ForwardJob(ctx, "remote", "job", payload) + } + if err != nil { + failed.Add(1) + errOnce.Do(func() { firstErr = err }) + } + } + }() + } + wg.Wait() + + if n := failed.Load(); n != 0 { + t.Errorf("%d of %d forwards failed, first: %v", n, workers*perW, firstErr) + } + if n := bad.Load(); n != 0 { + t.Errorf("peer saw %d malformed frames", n) + } + if got := peer.accepted.Load(); got != 1 { + t.Errorf("connections accepted = %d, want 1", got) + } +} + +// ── Snapshots ────────────────────────────────────── + +func TestFederation_PeerSnapshotsDoNotRace(t *testing.T) { + peer := newFakePeer(t, func(_ int64, c *fakePeerConn) { + if err := c.authenticate(); err != nil { + return + } + for { + if err := c.write(&Frame{ID: GenerateFrameID(), Type: FramePong, Timestamp: time.Now().UTC()}); err != nil { + return + } + time.Sleep(time.Millisecond) + } + }) + + f := newTestFederation() + f.Start(t.Context()) + t.Cleanup(func() { stopWithin(t, f) }) + if err := f.AddPeer(t.Context(), "remote", peer.url(), "tok", map[string]string{"region": "eu"}); err != nil { + t.Fatalf("AddPeer: %v", err) + } + + deadline := time.Now().Add(100 * time.Millisecond) + for time.Now().Before(deadline) { + p, ok := f.GetPeer("remote") + if !ok { + t.Fatal("GetPeer: not found") + } + _ = p.LastSeen + _ = p.State + for _, p := range f.Peers() { + _ = p.LastSeen + _ = p.State + } + } + + p, _ := f.GetPeer("remote") + if p.State != PeerStateConnected || p.Metadata["region"] != "eu" || p.URL != peer.url() { + t.Errorf("snapshot = %+v", p) + } +} diff --git a/dwp/frame.go b/dwp/frame.go index 0d14db2..a1941aa 100644 --- a/dwp/frame.go +++ b/dwp/frame.go @@ -6,6 +6,8 @@ package dwp import ( "encoding/json" + "strconv" + "sync/atomic" "time" ) @@ -261,9 +263,18 @@ func NewEventFrame(channel string, data any) (*Frame, error) { // GenerateFrameID returns a new unique frame ID. // Uses a simple timestamp + counter approach for performance. +// +// The ID is a UTC timestamp followed by a process-wide sequence number. +// The timestamp alone is not unique: clock resolution is often coarser +// than a nanosecond, so concurrent callers could get the same ID and +// cross their request/response correlation. func GenerateFrameID() string { - return time.Now().UTC().Format("20060102150405.000000000") + seq := frameIDSeq.Add(1) + return time.Now().UTC().Format("20060102150405.000000000") + "-" + strconv.FormatUint(seq, 36) } +// frameIDSeq disambiguates frame IDs generated within one clock tick. +var frameIDSeq atomic.Uint64 + // generateFrameID is an internal alias for backward compatibility. func generateFrameID() string { return GenerateFrameID() } diff --git a/dwp/frame_test.go b/dwp/frame_test.go index b2b6485..0968da3 100644 --- a/dwp/frame_test.go +++ b/dwp/frame_test.go @@ -2,6 +2,7 @@ package dwp import ( "encoding/json" + "sync" "testing" "time" ) @@ -109,6 +110,36 @@ func TestGenerateFrameID(t *testing.T) { } } +func TestGenerateFrameIDConcurrentUnique(t *testing.T) { + t.Parallel() + + const ( + workers = 16 + perW = 1000 + ) + ids := make(chan string, workers*perW) + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + for range perW { + ids <- GenerateFrameID() + } + }() + } + wg.Wait() + close(ids) + + seen := make(map[string]struct{}, workers*perW) + for id := range ids { + if _, dup := seen[id]; dup { + t.Fatalf("duplicate frame ID %q", id) + } + seen[id] = struct{}{} + } +} + func TestCodecJSONRoundtrip(t *testing.T) { t.Parallel()