From 12f250381b090520b80c26fc0fdea64af80a4240 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 09:19:05 +0000 Subject: [PATCH 01/15] raftengine,encryption: retarget cleanup WP-3 tests to live paths Delete TestMaxAppliedIndexStartsFromSnapshotIndex (subsumed by TestOpenInitializesAppliedIndexFromPersistedSnapshot), replace the run-time AppliedIndexWriter check with a compile-time assertion, and drop TestHeaderAADBytes_Layout (same bytes as the Append variant). Point the three persistConfigSnapshot tests at persistConfigState, the function production calls, and delete the orphaned persistConfigSnapshot. Give the over-cap bootstrap batch test all 16385 registration rows so only the cap can reject it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- internal/encryption/envelope_test.go | 8 ---- internal/encryption/fsmwire/wire_test.go | 15 ++++++-- internal/raftengine/etcd/engine.go | 21 ----------- .../etcd/engine_applied_index_test.go | 7 +--- internal/raftengine/etcd/engine_test.go | 37 ++++++++++--------- 5 files changed, 31 insertions(+), 57 deletions(-) diff --git a/internal/encryption/envelope_test.go b/internal/encryption/envelope_test.go index 49a550b6a..42deb6897 100644 --- a/internal/encryption/envelope_test.go +++ b/internal/encryption/envelope_test.go @@ -190,14 +190,6 @@ func TestEnvelope_Decode_DoesNotAliasInput(t *testing.T) { } } -func TestHeaderAADBytes_Layout(t *testing.T) { - got := encryption.HeaderAADBytes(encryption.EnvelopeVersionV2, encryption.FlagCompressed, 0x12345678) - want := []byte{0x02, 0x01, 0x12, 0x34, 0x56, 0x78} - if !bytes.Equal(got, want) { - t.Fatalf("HeaderAADBytes layout mismatch:\n got %x\n want %x", got, want) - } -} - func TestAppendHeaderAADBytes_AppendsAndAllocFree(t *testing.T) { // Append onto an existing buffer. The result should be the original // bytes followed by the 6-byte header. diff --git a/internal/encryption/fsmwire/wire_test.go b/internal/encryption/fsmwire/wire_test.go index 1454c797f..eff57acc1 100644 --- a/internal/encryption/fsmwire/wire_test.go +++ b/internal/encryption/fsmwire/wire_test.go @@ -2,6 +2,7 @@ package fsmwire_test import ( "bytes" + "strings" "testing" "github.com/bootjp/elastickv/internal/encryption/fsmwire" @@ -165,17 +166,23 @@ func TestBootstrap_RejectsOverCapBatchCount(t *testing.T) { t.Fatalf("encoded bootstrap shorter than expected: %d bytes, want >= %d", got, countOffset+4) } // Overwrite count with maxBootstrapBatchCount + 1 = 16385. - // The cap check fires before the wire-bytes check, so we do - // NOT need to extend the payload with 16385 * 14 trailing - // bytes — the decoder must reject on the cap regardless. good[countOffset+0] = 0x00 good[countOffset+1] = 0x00 good[countOffset+2] = 0x40 // 0x4001 = 16385 = (1<<14) + 1 good[countOffset+3] = 0x01 - _, err := fsmwire.DecodeBootstrap(good) + // Supply all 16385 × 14 registration bytes so the remaining-bytes + // check admits the count and only the cap can reject it. + const overCapCount, registrationWireSize = 16385, 14 + payload := make([]byte, 0, countOffset+4+overCapCount*registrationWireSize) + payload = append(payload, good[:countOffset+4]...) + payload = append(payload, make([]byte, overCapCount*registrationWireSize)...) + _, err := fsmwire.DecodeBootstrap(payload) if !errors.Is(err, fsmwire.ErrFSMWireMalformed) { t.Fatalf("expected ErrFSMWireMalformed for over-cap batch count, got %v", err) } + if !strings.Contains(err.Error(), "exceeds cap") { + t.Fatalf("expected the batch-count cap to reject, got %v", err) + } } // TestBootstrap_RejectsHugeBatchCount is the regression for the diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 05adf1d2d..88f5a2b7e 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -3419,27 +3419,6 @@ func applyUpdatedPeerToMap(peers map[uint64]Peer, peer Peer, ok bool) { } } -func (e *Engine) persistConfigSnapshot(index uint64, confState raftpb.ConfState) error { - // Fast path (lock-free): avoid unnecessary FSM snapshot when config is already current. - if upToDate, err := e.configSnapshotUpToDate(index, confState); err != nil { - return err - } else if upToDate { - return nil - } - - e.snapshotMu.Lock() - defer e.snapshotMu.Unlock() - - payload, err := e.snapshotPayloadLocked(index) - if err != nil { - if snapshotDeferred(err, "config snapshot", index) { - return nil - } - return err - } - return e.persistConfigSnapshotPayloadLocked(index, confState, payload) -} - func (e *Engine) persistConfigState(index uint64, confState raftpb.ConfState, peers []Peer) error { // Config changes are committed in strictly increasing raft index order. Use // the cached durable config index to avoid re-reading peer metadata from diff --git a/internal/raftengine/etcd/engine_applied_index_test.go b/internal/raftengine/etcd/engine_applied_index_test.go index 107dcaefa..20c5040b4 100644 --- a/internal/raftengine/etcd/engine_applied_index_test.go +++ b/internal/raftengine/etcd/engine_applied_index_test.go @@ -340,12 +340,7 @@ func TestReleaseIgnoredReceivedFSMSnapshotStepsKeepsSnapshotReadyProtected(t *te // TestRecordingFSM_SatisfiesAppliedIndexWriter is a compile-time- // adjacent assertion: the recording FSM MUST satisfy the writer // seam so the engine hook actually fires for it. -func TestRecordingFSM_SatisfiesAppliedIndexWriter(t *testing.T) { - rec := &applyIndexOrderRecorder{} - var f any = &recordingAppliedIndexFSM{rec: rec} - _, ok := f.(raftengine.AppliedIndexWriter) - require.True(t, ok, "recordingAppliedIndexFSM must implement raftengine.AppliedIndexWriter") -} +var _ raftengine.AppliedIndexWriter = (*recordingAppliedIndexFSM)(nil) // TestPersistCreatedSnapshot_BumpsAppliedIndex exercises Site 1 of // the persist hook. We invoke (*Engine).persistCreatedSnapshot diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index d09af4448..d80f94b90 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -1204,7 +1204,7 @@ func TestMaybePersistLocalSnapshotReturnsWhenWorkerIsStopped(t *testing.T) { } } -func TestPersistConfigSnapshotSkipsSerializationWhenNewerSnapshotExists(t *testing.T) { +func TestPersistConfigStateSkipsSerializationWhenNewerSnapshotExists(t *testing.T) { storage := etcdraft.NewMemoryStorage() snap := raftTestSnapshot(9, 2, []uint64{1, 2}, []byte("newer")) require.NoError(t, storage.ApplySnapshot(&snap)) @@ -1213,13 +1213,17 @@ func TestPersistConfigSnapshotSkipsSerializationWhenNewerSnapshotExists(t *testi engine := &Engine{ storage: storage, fsm: fsm, + dataDir: t.TempDir(), } - require.NoError(t, engine.persistConfigSnapshot(7, raftpb.ConfState{Voters: []uint64{1}})) + require.NoError(t, engine.persistConfigState(7, raftpb.ConfState{Voters: []uint64{1}}, []Peer{ + {NodeID: 1, ID: "n1", Address: "127.0.0.1:7001"}, + })) require.Zero(t, fsm.snapshotCalls.Load()) + require.Equal(t, uint64(7), engine.currentConfigIndex()) } -func TestPersistConfigSnapshotDoesNotCompactLogs(t *testing.T) { +func TestPersistConfigStateDoesNotCompactLogs(t *testing.T) { storage := etcdraft.NewMemoryStorage() snap := raftTestSnapshot(1, 1, []uint64{1}, nil) require.NoError(t, storage.ApplySnapshot(&snap)) @@ -1233,16 +1237,20 @@ func TestPersistConfigSnapshotDoesNotCompactLogs(t *testing.T) { storage: storage, persist: mockstorage.NewStorageRecorder(""), fsm: &testStateMachine{}, + dataDir: t.TempDir(), } - require.NoError(t, engine.persistConfigSnapshot(4, raftpb.ConfState{Voters: []uint64{1, 2}})) + require.NoError(t, engine.persistConfigState(4, raftpb.ConfState{Voters: []uint64{1, 2}}, []Peer{ + {NodeID: 1, ID: "n1", Address: "127.0.0.1:7001"}, + {NodeID: 2, ID: "n2", Address: "127.0.0.1:7002"}, + })) firstIndex, err := storage.FirstIndex() require.NoError(t, err) require.Equal(t, uint64(2), firstIndex) } -func TestPersistConfigSnapshotWaitsForDurablePublication(t *testing.T) { +func TestPersistConfigStateWaitsForDurablePublication(t *testing.T) { storage := etcdraft.NewMemoryStorage() snap := raftTestSnapshot(1, 1, []uint64{1}, nil) require.NoError(t, storage.ApplySnapshot(&snap)) @@ -1270,7 +1278,10 @@ func TestPersistConfigSnapshotWaitsForDurablePublication(t *testing.T) { done := make(chan error, 1) go func() { - done <- engine.persistConfigSnapshot(4, raftpb.ConfState{Voters: []uint64{1, 2}}) + done <- engine.persistConfigState(4, raftpb.ConfState{Voters: []uint64{1, 2}}, []Peer{ + {NodeID: 1, ID: "n1", Address: "127.0.0.1:7001"}, + {NodeID: 2, ID: "n2", Address: "127.0.0.1:7002"}, + }) }() select { @@ -1282,7 +1293,7 @@ func TestPersistConfigSnapshotWaitsForDurablePublication(t *testing.T) { select { case err := <-done: require.NoError(t, err) - t.Fatal("persistConfigSnapshot returned before snapshot serialization completed") + t.Fatal("persistConfigState returned before snapshot serialization completed") case <-time.After(100 * time.Millisecond): } @@ -1293,7 +1304,7 @@ func TestPersistConfigSnapshotWaitsForDurablePublication(t *testing.T) { case err := <-done: require.NoError(t, err) case <-time.After(time.Second): - t.Fatal("persistConfigSnapshot did not finish after snapshot serialization was released") + t.Fatal("persistConfigState did not finish after snapshot serialization was released") } actions := persist.Action() @@ -1707,16 +1718,6 @@ func TestEnqueueDispatchMessagePreservesIncomingReadIndexHeartbeatWhenRespLaneFu require.Equal(t, []byte("read-index-c"), req.msg.Context) } -func TestMaxAppliedIndexStartsFromSnapshotIndex(t *testing.T) { - storage := etcdraft.NewMemoryStorage() - snap := raftTestSnapshot(5, 2, []uint64{1}, nil) - require.NoError(t, storage.ApplySnapshot(&snap)) - hs := testHardState(2, 9) - require.NoError(t, storage.SetHardState(&hs)) - - require.Equal(t, uint64(5), maxAppliedIndex(raftTestSnapshot(5, 0, nil, nil))) -} - func TestOpenRestoresLegacySnapshotState(t *testing.T) { dir := t.TempDir() require.NoError(t, saveStateFile(stateFilePath(dir), persistedState{ From bcf431412660ddabe9e4b9d2df689186238aa707 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 09:21:13 +0000 Subject: [PATCH 02/15] test: delete redundant admin, distribution and keyviz tests (cleanup WP-7) TestServer_WriteRejectsMissingCSRF accepted any of 401/403/405 and still passed with CSRF removed; TestServer_DynamoCreateTable_MissingCSRFRejected covers the chain. TestKeyVizFanoutRunPeerOverlargeBody never exceeded the cap; its PR #686 reference moves onto TestKeyVizFanoutRunPeerExceedsBodyLimit. TestCatalogWatcherLoggerOption and TestAllLabelConstantsInAllLabels only read back what they set. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- distribution/watcher_test.go | 14 --------- internal/admin/keyviz_fanout_test.go | 43 ++-------------------------- internal/admin/server_test.go | 19 ------------ keyviz/labels_test.go | 17 ----------- 4 files changed, 3 insertions(+), 90 deletions(-) diff --git a/distribution/watcher_test.go b/distribution/watcher_test.go index c45b580ff..30f82c851 100644 --- a/distribution/watcher_test.go +++ b/distribution/watcher_test.go @@ -3,8 +3,6 @@ package distribution import ( "bytes" "context" - "io" - "log/slog" "sync" "testing" "time" @@ -220,18 +218,6 @@ func TestCatalogWatcherRetriesOnTransientReadError(t *testing.T) { }, time.Second, 10*time.Millisecond) } -func TestCatalogWatcherLoggerOption(t *testing.T) { - t.Parallel() - - logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - watcher := NewCatalogWatcher( - NewCatalogStore(store.NewMVCCStore()), - NewEngine(), - WithCatalogWatcherLogger(logger), - ) - require.Same(t, logger, watcher.logger) -} - type transientVersionReadStore struct { store.MVCCStore mu sync.Mutex diff --git a/internal/admin/keyviz_fanout_test.go b/internal/admin/keyviz_fanout_test.go index ec36d54a1..51c5814ea 100644 --- a/internal/admin/keyviz_fanout_test.go +++ b/internal/admin/keyviz_fanout_test.go @@ -750,7 +750,9 @@ func TestKeyVizFanoutRunPeerOrder(t *testing.T) { } // TestKeyVizFanoutRunPeerExceedsBodyLimit pins the over-cap path -// (Claude bot round-2 on PR #686). Lowering the per-peer limit to a +// (security-high review item and Claude bot round-2 on PR #686): a +// peer that streams more than keyVizPeerResponseBodyLimit bytes must +// not pin a goroutine on the JSON decoder or balloon memory. Lowering the per-peer limit to a // small test value lets us drive the path without serving 64 MiB. // The peer streams a body deliberately larger than the cap; the // aggregator's countingReader must: @@ -844,45 +846,6 @@ func TestKeyVizFanoutRunPeerNearCapSucceedsWithWarning(t *testing.T) { merged.Fanout.Nodes[1].Error) } -// TestKeyVizFanoutRunPeerOverlargeBody pins the security-high -// review item on PR #686: a peer that streams more than -// keyVizPeerResponseBodyLimit bytes must not pin a goroutine on the -// JSON decoder or balloon memory. The aggregator caps the decode at -// the configured limit and surfaces a warning log rather than -// silently accepting a truncated matrix. -func TestKeyVizFanoutRunPeerOverlargeBody(t *testing.T) { - t.Parallel() - // Build a JSON payload whose `rows` array is enormous: many - // rows of 4096 zeroed values. We do not actually need to exceed - // the production cap (64 MiB) — we just need to assert that the - // decode completes promptly and that the peer call ends up - // reporting OK with a row count that matches what was on the wire - // up to the cap. - hugeRow := KeyVizRow{ - BucketID: "route:big", - Values: make([]uint64, 4096), - } - rows := make([]KeyVizRow, 64) - for i := range rows { - rows[i] = hugeRow - } - body := KeyVizMatrix{ - ColumnUnixMs: make([]int64, 4096), - Rows: rows, - Series: keyVizSeriesReads, - } - peer := newKeyVizPeerStub(t, body) - defer peer.Close() - - f := NewKeyVizFanout("self:8080", []string{peer.URL}).WithHTTPClient(peer.Client()) - - start := time.Now() - merged := f.Run(context.Background(), keyVizParams{series: keyVizSeriesReads, rows: 1024}, KeyVizMatrix{Series: keyVizSeriesReads}, nil) - require.Less(t, time.Since(start), 5*time.Second, "decode must respect the size cap and complete promptly") - require.NotNil(t, merged.Fanout) - require.True(t, merged.Fanout.Nodes[1].OK, "in-cap response must succeed; cap is 64 MiB and the synthetic body is well under that") -} - // newKeyVizPeerStub spins up an httptest.Server that answers // /admin/api/v1/keyviz/matrix with a fixed 200 JSON body. Anything // else returns 404 — which surfaces as "peer status 404" in the diff --git a/internal/admin/server_test.go b/internal/admin/server_test.go index de655af8f..8a3ff3e7c 100644 --- a/internal/admin/server_test.go +++ b/internal/admin/server_test.go @@ -503,22 +503,3 @@ func TestServer_ServerDepsForwarderIsWired(t *testing.T) { "forwarder must be invoked when source returns ErrTablesNotLeader") _ = resp.Body.Close() } - -func TestServer_WriteRejectsMissingCSRF(t *testing.T) { - // Login to obtain a session, then hit cluster with POST to trigger - // CSRF on what the router normally rejects as method_not_allowed. - // Since /cluster rejects POST at the handler level, we use an - // endpoint that would exercise the write path. For P1 we do not - // have one yet, so assert via a direct middleware-level test that - // composing Server routes a write path through CSRF. This is - // already covered by TestCSRF_* in middleware_test.go; here we - // spot-check the compose order by sending POST /cluster and - // expecting 405 (CSRF runs before method check so we get 403 - // instead). Either 403 or 405 is acceptable; the important bit - // is that no handler state is mutated. - srv := newServerForTest(t) - req := httptest.NewRequest(http.MethodPost, "/admin/api/v1/cluster", nil) - rec := httptest.NewRecorder() - srv.Handler().ServeHTTP(rec, req) - require.Contains(t, []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusMethodNotAllowed}, rec.Code) -} diff --git a/keyviz/labels_test.go b/keyviz/labels_test.go index 44de712fd..8032b9764 100644 --- a/keyviz/labels_test.go +++ b/keyviz/labels_test.go @@ -14,14 +14,6 @@ func TestAllLabelsAvoidBucketIDSeparator(t *testing.T) { } } -func TestAllLabelConstantsInAllLabels(t *testing.T) { - t.Parallel() - want := []Label{LabelDynamo, LabelRedis, LabelS3, LabelSQS, LabelRawKV} - for _, label := range want { - require.True(t, labelInAllLabels(label), "AllLabels must include %q", label) - } -} - func TestAllLabelsWithLegacyFreshSlice(t *testing.T) { t.Parallel() labels := allLabelsWithLegacy() @@ -99,12 +91,3 @@ func TestCoarsenedRouteEmitsLegacyLabel(t *testing.T) { require.Equal(t, LabelLegacy, rows[0].Label) require.Equal(t, uint64(1), rows[0].Writes) } - -func labelInAllLabels(label Label) bool { - for _, candidate := range AllLabels { - if candidate == label { - return true - } - } - return false -} From a14ec0cd0e8d50b47af357732fca67f129cf7623 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 09:29:13 +0000 Subject: [PATCH 03/15] main,cmd,proxy: retarget cleanup WP-6 tests to live paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delete tests that duplicate a survivor or cannot fail: the Phase-0 registration gate (its §2.3 note moves onto the table row), the buildLeaderForwarder happy path, the pebbleOptions pair, the EncryptionObserver and capability-fanout non-nil checks, the encryptionAdminWiringForGroup tests together with that uncalled function, the loadToken copies, the racy fanout dedup twin, the proxy map-only pubsub tests, and a duplicate encode-info helper. Retarget the KEK unwrap timing test onto loadKEKAndRunStartupGuards for a fresh node, drive the orphan branch of installOrAttach directly, and point the S3 and registry tests at newS3Server and registryLastSeen. Delete the uncalled startPprofServer, startMetricsServer, startS3Server, prepareS3Server and readWriterRegistryLastSeen wrappers; serverwiring already tests the live prepare functions. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- cmd/elastickv-admin/main_test.go | 123 +++--------------- .../main_fifo_unix_test.go | 22 +--- main.go | 18 --- main_admin_forward_test.go | 10 -- main_encryption_admin.go | 14 -- main_encryption_admin_test.go | 62 --------- main_encryption_fanout_test.go | 12 -- main_encryption_metrics_wiring_test.go | 59 ++++----- main_encryption_registration.go | 13 -- main_encryption_registration_test.go | 56 +++----- main_encryption_write_wiring_test.go | 28 ---- main_metrics_test.go | 17 --- main_pprof_test.go | 22 ---- main_s3.go | 65 --------- main_s3_test.go | 11 +- main_sqs.go | 2 +- proxy/pubsub_test.go | 49 ------- 17 files changed, 64 insertions(+), 519 deletions(-) delete mode 100644 main_metrics_test.go delete mode 100644 main_pprof_test.go diff --git a/cmd/elastickv-admin/main_test.go b/cmd/elastickv-admin/main_test.go index c52e3107b..ef91a1546 100644 --- a/cmd/elastickv-admin/main_test.go +++ b/cmd/elastickv-admin/main_test.go @@ -26,6 +26,7 @@ import ( pb "github.com/bootjp/elastickv/proto" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" ) @@ -90,35 +91,6 @@ func TestLoadTokenRequiresFileOrInsecure(t *testing.T) { } } -func TestLoadTokenReadsAndTrims(t *testing.T) { - t.Parallel() - dir := t.TempDir() - path := filepath.Join(dir, "token") - if err := os.WriteFile(path, []byte("\n s3cret \n"), 0o600); err != nil { - t.Fatal(err) - } - tok, err := loadToken(path, false) - if err != nil { - t.Fatalf("loadToken: %v", err) - } - if tok != "s3cret" { - t.Fatalf("tok = %q, want s3cret", tok) - } -} - -func TestLoadTokenRejectsEmptyFile(t *testing.T) { - t.Parallel() - dir := t.TempDir() - path := filepath.Join(dir, "empty") - if err := os.WriteFile(path, []byte(" \n"), 0o600); err != nil { - t.Fatal(err) - } - _, err := loadToken(path, false) - if err == nil || !strings.Contains(err.Error(), "empty") { - t.Fatalf("expected empty-file error, got %v", err) - } -} - func TestLoadTokenRejectsInsecureWithFile(t *testing.T) { t.Parallel() dir := t.TempDir() @@ -361,11 +333,11 @@ func TestFanoutClientForDeduplicatesConcurrentDials(t *testing.T) { } } -// TestFanoutClientForOrphanedDialClosed asserts that when two non-overlapping -// clientFor calls dial the same address (so singleflight runs the dial fn -// twice with two different *grpc.ClientConn results), the second call's -// orphaned conn is closed instead of leaking. Codex P2 on 1492fdae: the -// orphan path previously dropped the loser conn on the floor. +// TestFanoutClientForOrphanedDialClosed asserts that when a dial +// completes after a non-overlapping earlier clientFor already installed a +// different conn for the same address, installOrAttach closes the orphaned +// conn and hands back the cached client. Codex P2 on 1492fdae: the orphan +// path previously dropped the loser conn on the floor. func TestFanoutClientForOrphanedDialClosed(t *testing.T) { t.Parallel() peer := &fakeAdminServer{members: []string{"m:1"}} @@ -374,39 +346,32 @@ func TestFanoutClientForOrphanedDialClosed(t *testing.T) { f := newFanout([]string{addr}, "", time.Second, insecure.NewCredentials(), defaultMaxDiscoveredNodes) defer f.Close() - // First clientFor — installs into cache. + // First clientFor installs c1 into the cache. c1, rel1, err := f.clientFor(addr) if err != nil { t.Fatal(err) } + defer rel1() - // Hand-craft an orphaned conn by simulating "post-singleflight, cache - // already has different conn". We bypass singleflight to deterministically - // produce a second *grpc.ClientConn that won't equal c1.conn, then drive - // the cache-hit-after-dial branch via a second clientFor call. + // A second dial that lost the race to the cache: its conn differs from + // the cached one, so installOrAttach must close it. conn2, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { t.Fatal(err) } - // Stuff conn2 into the singleflight cache slot via the public API path - // is not feasible; instead, verify the live path: a second concurrent - // clientFor should not leak — the dedup test already covered the - // happy path. Here we just assert that the orphan branch's Close() - // does not panic on a fresh conn (nil-safe Close behavior on - // already-closed conn is what protects the path). - if err := conn2.Close(); err != nil { - t.Fatalf("conn2.Close: %v", err) - } - - rel1() - // Second clientFor should still succeed (cache hit) and not panic. - c2, rel2, err := f.clientFor(addr) + c2, rel2, err := f.installOrAttach(addr, conn2) if err != nil { t.Fatal(err) } defer rel2() if c1 != c2 { - t.Fatalf("nodeClient pointer mismatch — cache lookup did not return the cached entry") + t.Fatalf("installOrAttach returned %p, want the cached client %p", c2, c1) + } + if c2.conn == conn2 { + t.Fatal("installOrAttach replaced the cached conn with the orphan") + } + if got := conn2.GetState(); got != connectivity.Shutdown { + t.Fatalf("orphaned conn state = %v, want %v (closed)", got, connectivity.Shutdown) } } @@ -806,58 +771,6 @@ func TestHandleOverviewUsesProtojson(t *testing.T) { } } -// TestFanoutClientForRaceDeduplicates exercises the dial-outside-the-lock -// path in clientFor: many goroutines racing for the same addr must all -// converge on a single cached *grpc.ClientConn (the loser of each race -// closes its just-dialed conn). Pre-fix, the dial happened under the -// lock so the race was impossible by construction; post-fix, the race -// is intentional but bounded. -func TestFanoutClientForRaceDeduplicates(t *testing.T) { - t.Parallel() - peer := &fakeAdminServer{members: []string{"m:1"}} - addr := startFakeAdmin(t, peer) - f := newFanout([]string{addr}, "", time.Second, insecure.NewCredentials(), defaultMaxDiscoveredNodes) - defer f.Close() - - const racers = 32 - var wg sync.WaitGroup - wg.Add(racers) - clients := make([]*nodeClient, racers) - releases := make([]func(), racers) - for i := 0; i < racers; i++ { - go func(i int) { - defer wg.Done() - c, release, err := f.clientFor(addr) - if err != nil { - t.Errorf("racer %d clientFor: %v", i, err) - return - } - clients[i] = c - releases[i] = release - }(i) - } - wg.Wait() - - for _, release := range releases { - if release != nil { - release() - } - } - - first := clients[0] - for i, c := range clients { - if c != first { - t.Fatalf("racer %d got distinct nodeClient %p, want %p — clientFor de-duplication broke", i, c, first) - } - } - f.mu.Lock() - size := len(f.clients) - f.mu.Unlock() - if size != 1 { - t.Fatalf("cache size after race = %d, want 1 (race created %d duplicates)", size, size-1) - } -} - // TestFanoutCloseDoesNotHoldLockDuringConnClose pins the round-5 fix: // fanout.Close must release f.mu before invoking conn.Close on each // cached connection. The test populates the cache, takes the lock from diff --git a/cmd/elastickv-snapshot-prepare-restore/main_fifo_unix_test.go b/cmd/elastickv-snapshot-prepare-restore/main_fifo_unix_test.go index c66127e9e..5242038ed 100644 --- a/cmd/elastickv-snapshot-prepare-restore/main_fifo_unix_test.go +++ b/cmd/elastickv-snapshot-prepare-restore/main_fifo_unix_test.go @@ -9,7 +9,6 @@ import ( "path/filepath" "syscall" "testing" - "time" "github.com/bootjp/elastickv/internal/backup" "github.com/bootjp/elastickv/internal/raftengine/etcd" @@ -20,7 +19,7 @@ func TestRunPrepareRestoreRejectsFIFOInputBeforeOpen(t *testing.T) { dir := t.TempDir() input := filepath.Join(dir, "encoded.fsm") require.NoError(t, syscall.Mkfifo(input, 0o600)) - writeEncodeInfoWithRecordedSHA(t, input, "cluster-a", "unused") + writeEncodeInfoWithSHA(t, input, "cluster-a", true, true, "unused") code, err := run([]string{ "--input", input, @@ -66,22 +65,3 @@ func TestRunPrepareRestoreRejectsSymlinkInputBeforeOpen(t *testing.T) { require.ErrorIs(t, err, etcd.ErrExternalSnapshotRestoreInvalid) require.Equal(t, exitUserErr, code) } - -func writeEncodeInfoWithRecordedSHA(t *testing.T, input, clusterID, shaHex string) { - t.Helper() - info := backup.NewEncodeInfo(time.Unix(0, 0)) - info.EncoderVersion = "test" - info.InputRoot = "dump" - info.OutputFSMPath = input - info.OutputFSMSHA256 = shaHex - info.LastCommitTS = 1234 - info.ManifestLastCommitTS = 1234 - info.ManifestClusterID = clusterID - info.AdaptersEnabled = []string{"redis"} - info.SelfTest = backup.EncodeInfoSelfTest{Ran: true, Matched: true} - - f, err := os.Create(backup.EncodeInfoSidecarPath(input)) - require.NoError(t, err) - require.NoError(t, backup.WriteEncodeInfo(f, info)) - require.NoError(t, f.Close()) -} diff --git a/main.go b/main.go index 878617495..542415e73 100644 --- a/main.go +++ b/main.go @@ -4485,15 +4485,6 @@ func runDynamoDBServer(ctx context.Context, eg *errgroup.Group, dynamoServer *ad }) } -func startPprofServer(ctx context.Context, lc *net.ListenConfig, eg *errgroup.Group, pprofAddr string, pprofToken string) error { - pprofServer, pprofL, bindAddr, err := serverwiring.PreparePprofServer(ctx, lc, pprofAddr, pprofToken) - if err != nil { - return errors.WithStack(err) - } - runPreparedPprofServer(ctx, eg, pprofServer, pprofL, bindAddr) - return nil -} - func runPreparedPprofServer(ctx context.Context, eg *errgroup.Group, pprofServer *http.Server, pprofL net.Listener, pprofAddr string) { if pprofServer == nil || pprofL == nil { return @@ -4502,15 +4493,6 @@ func runPreparedPprofServer(ctx context.Context, eg *errgroup.Group, pprofServer eg.Go(monitoring.PprofServeTask(pprofServer, pprofL, pprofAddr)) } -func startMetricsServer(ctx context.Context, lc *net.ListenConfig, eg *errgroup.Group, metricsAddr string, metricsToken string, handler http.Handler) error { - metricsServer, metricsL, bindAddr, err := serverwiring.PrepareMetricsServer(ctx, lc, metricsAddr, metricsToken, handler) - if err != nil { - return errors.WithStack(err) - } - runPreparedMetricsServer(ctx, eg, metricsServer, metricsL, bindAddr) - return nil -} - func runPreparedMetricsServer(ctx context.Context, eg *errgroup.Group, metricsServer *http.Server, metricsL net.Listener, metricsAddr string) { if metricsServer == nil || metricsL == nil { return diff --git a/main_admin_forward_test.go b/main_admin_forward_test.go index bc83fd569..fbd986f16 100644 --- a/main_admin_forward_test.go +++ b/main_admin_forward_test.go @@ -37,16 +37,6 @@ func TestBuildLeaderForwarder_RejectsMissingDeps(t *testing.T) { } } -func TestBuildLeaderForwarder_HappyPathReturnsForwarder(t *testing.T) { - // The production bridge does not dial during construction — - // resolver / dial calls only happen on the first Forward — so - // passing real (zero-value) collaborators is enough to confirm - // the wiring itself is well-formed. - fwd, err := buildLeaderForwarder(&kv.Coordinate{}, &kv.GRPCConnCache{}, "n1") - require.NoError(t, err) - require.NotNil(t, fwd) -} - func TestAdminForwardConnFactory_RejectsEmptyAddr(t *testing.T) { // kv.GRPCConnCache.ConnFor returns ErrLeaderNotFound on "". The // LeaderForwarder catches the empty address before this layer is diff --git a/main_encryption_admin.go b/main_encryption_admin.go index 2f1dfd94c..673b88a38 100644 --- a/main_encryption_admin.go +++ b/main_encryption_admin.go @@ -79,20 +79,6 @@ type encryptionAdminEngine interface { raftengine.LeaderView } -// encryptionAdminWiringForGroup keeps the read-only capability service on the -// dedicated TSO listener while withholding every mutating proposal path. The -// TSO FSM has no encryption applier or writer registry, so allowing a mutator -// to propose there would commit an entry that cannot update dedicated TSO -// state. The engine remains available as a LeaderView so ResyncSidecar keeps -// its quorum-confirmed leader-only freshness contract. Data groups retain the -// normal flag-driven mutator wiring. -func encryptionAdminWiringForGroup(groupID uint64, enableMutators bool, engine encryptionAdminEngine) (bool, encryptionAdminEngine) { - if groupID == dedicatedTSORaftGroupID { - return false, engine - } - return enableMutators, engine -} - func encryptionAdminWriterRegistry(sidecarPath string, st store.MVCCStore, groupID uint64) (encryption.WriterRegistryStore, error) { if sidecarPath == "" { return nil, nil diff --git a/main_encryption_admin_test.go b/main_encryption_admin_test.go index 2c9f45bcc..5cb9bfd12 100644 --- a/main_encryption_admin_test.go +++ b/main_encryption_admin_test.go @@ -45,10 +45,6 @@ func (stubEncryptionAdminEngine) LinearizableRead(context.Context) (uint64, erro return 0, nil } -type followerEncryptionAdminEngine struct{ stubEncryptionAdminEngine } - -func (followerEncryptionAdminEngine) State() raftengine.State { return raftengine.StateFollower } - // TestEncryptionAdminFullNodeID_DistinctPerRaftId pins the // PR760 r1 Codex P1 regression: full_node_id must be derived from // --raftId (per-node-stable) and NOT from the Raft group id @@ -252,42 +248,6 @@ func TestEncryptionAdmin_MutatingRPCEnabledWhenGateOn(t *testing.T) { } } -func TestEncryptionAdmin_DedicatedTSOGroupAlwaysRefusesMutators(t *testing.T) { - want := stubEncryptionAdminEngine{} - enabled, engine := encryptionAdminWiringForGroup( - dedicatedTSORaftGroupID, - true, - want, - ) - if enabled || engine == nil { - t.Fatalf("dedicated TSO mutator wiring = (%v, %T), want (false, leader view)", enabled, engine) - } - err := callBootstrapAgainstNewServer(t, enabled, engine) - if got := status.Code(err); got != codes.FailedPrecondition { - t.Fatalf("BootstrapEncryption status=%v, want FailedPrecondition; err=%v", got, err) - } -} - -func TestEncryptionAdmin_DedicatedTSOGroupResyncRejectsFollower(t *testing.T) { - enabled, engine := encryptionAdminWiringForGroup( - dedicatedTSORaftGroupID, - true, - followerEncryptionAdminEngine{}, - ) - err := callResyncAgainstNewServer(t, enabled, engine) - if got := status.Code(err); got != codes.FailedPrecondition { - t.Fatalf("ResyncSidecar status=%v, want FailedPrecondition; err=%v", got, err) - } -} - -func TestEncryptionAdmin_DataGroupPreservesMutatorGate(t *testing.T) { - want := stubEncryptionAdminEngine{} - enabled, engine := encryptionAdminWiringForGroup(1, true, want) - if !enabled || engine == nil { - t.Fatalf("data-group mutator wiring = (%v, %T), want (true, non-nil)", enabled, engine) - } -} - func callBootstrapAgainstNewServer(t *testing.T, enableMutators bool, engine encryptionAdminEngine) error { t.Helper() listener := bufconn.Listen(1024 * 1024) @@ -316,28 +276,6 @@ func callBootstrapAgainstNewServer(t *testing.T, enableMutators bool, engine enc return err } -func callResyncAgainstNewServer(t *testing.T, enableMutators bool, engine encryptionAdminEngine) error { - t.Helper() - listener := bufconn.Listen(1024 * 1024) - gs := grpc.NewServer() - registerEncryptionAdminServer(gs, newEncryptionAdminServer(1, "", enableMutators, engine, nil, nil, nil)) - go func() { _ = gs.Serve(listener) }() - t.Cleanup(gs.Stop) - conn, err := grpc.NewClient( - "passthrough:///bufnet", - grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) { - return listener.Dial() - }), - grpc.WithTransportCredentials(insecure.NewCredentials()), - ) - if err != nil { - t.Fatalf("grpc.NewClient: %v", err) - } - t.Cleanup(func() { _ = conn.Close() }) - _, err = pb.NewEncryptionAdminClient(conn).ResyncSidecar(context.Background(), &pb.ResyncSidecarRequest{}) - return err -} - func TestRegisterEncryptionAdminServer_Registers(t *testing.T) { gs := grpc.NewServer() registerEncryptionAdminServer(gs, newEncryptionAdminServer(1, "", false, nil, nil, nil, nil)) diff --git a/main_encryption_fanout_test.go b/main_encryption_fanout_test.go index 8f7b9b755..1f4f00914 100644 --- a/main_encryption_fanout_test.go +++ b/main_encryption_fanout_test.go @@ -135,18 +135,6 @@ func TestRouteGroupFromConfiguration_Empty(t *testing.T) { } } -// TestBuildCapabilityFanoutFn_NonNil pins that the builder returns a -// usable closure (it is only wired when encryption mutators are -// enabled; the closure's fan-out behavior is exercised end-to-end in -// 6D-6c-3b). -func TestBuildCapabilityFanoutFn_NonNil(t *testing.T) { - t.Parallel() - fn := buildCapabilityFanoutFn(nil, nil, capabilityFanoutTimeout) - if fn == nil { - t.Fatal("buildCapabilityFanoutFn returned nil") - } -} - // TestBuildEncryptionCapabilityFanout_DisabledReturnsNil pins that the // closure is not built when encryption mutators are off — the cutover // RPC is then unreachable, so wiring a fan-out would be dead weight. diff --git a/main_encryption_metrics_wiring_test.go b/main_encryption_metrics_wiring_test.go index 7ef6127aa..c7edcd565 100644 --- a/main_encryption_metrics_wiring_test.go +++ b/main_encryption_metrics_wiring_test.go @@ -3,13 +3,14 @@ package main import ( "bytes" "context" + "encoding/base64" + "path/filepath" "sync" "testing" "time" "github.com/bootjp/elastickv/internal/encryption" "github.com/bootjp/elastickv/internal/encryption/kek" - "github.com/bootjp/elastickv/monitoring" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" ) @@ -109,22 +110,6 @@ func TestEncryptionObserverReachesStoreThroughProductionWiring(t *testing.T) { } } -// TestMetricsRegistryEncryptionObserverSatisfiesStoreInterface pins the -// production type: main.go passes metricsRegistry.EncryptionObserver() -// where a store.EncryptionObserver is required, so the monitoring -// implementation must satisfy the store's interface. A compile-time -// assertion here fails loudly if either side's method set drifts. -func TestMetricsRegistryEncryptionObserverSatisfiesStoreInterface(t *testing.T) { - t.Parallel() - - var obs store.EncryptionObserver = monitoring.NewRegistry("n1", "127.0.0.1:1").EncryptionObserver() - if obs == nil { - t.Fatal("registry returned a nil encryption observer") - } - obs.ObserveEncryptionWrite(1, 10, 42) - obs.ObserveEncryptionDecryptFailure(encryption.DecryptFailureReasonTagMismatch) -} - // countingKEKObserver records how many unwraps were timed. type countingKEKObserver struct { mu sync.Mutex @@ -151,27 +136,29 @@ func (o *countingKEKObserver) total() int { // elastickv_encryption_kek_unwrap_seconds empty despite completed KMS // calls, so the decorator must wrap the source before they run. func TestTimedKEKUnwrapperCoversStartupPreflightUnwraps(t *testing.T) { - t.Parallel() + previousEnabled := *encryptionEnabled + previousSidecar := *encryptionSidecarPath + previousFile := *kekFile + previousURI := *kekURI + t.Cleanup(func() { + *encryptionEnabled = previousEnabled + *encryptionSidecarPath = previousSidecar + *kekFile = previousFile + *kekURI = previousURI + }) + + // A fresh node: the KEK comes from the environment and the sidecar + // does not exist yet, so the KEK preflight is the only unwrap. + t.Setenv(kek.EnvVar, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0x5a}, encryption.KeySize))) + *kekFile = "" + *kekURI = "" + *encryptionEnabled = true + *encryptionSidecarPath = filepath.Join(t.TempDir(), encryption.SidecarFilename) obs := &countingKEKObserver{} - timed := monitoring.NewTimedKEKUnwrapper(preflightKEK{}, obs) - require.NotNil(t, timed) - - // Both preflight paths unwrap through the decorated source. - require.NoError(t, kek.VerifyWrapper(timed)) + wrapper, err := loadKEKAndRunStartupGuards(obs) + require.NoError(t, err) + require.NotNil(t, wrapper) require.Positive(t, obs.total(), "the startup preflight unwrap must be timed, not bypass the decorator") } - -// preflightKEK is a minimal kek.Wrapper for the decoration-order test. -type preflightKEK struct{} - -func (preflightKEK) Name() string { return "preflight-fake" } - -func (preflightKEK) Wrap(dek []byte) ([]byte, error) { - return append([]byte("w:"), dek...), nil -} - -func (preflightKEK) Unwrap(wrapped []byte) ([]byte, error) { - return bytes.TrimPrefix(wrapped, []byte("w:")), nil -} diff --git a/main_encryption_registration.go b/main_encryption_registration.go index 7cdb53f9e..0213a8587 100644 --- a/main_encryption_registration.go +++ b/main_encryption_registration.go @@ -506,19 +506,6 @@ func buildProcessStartRegistrationGate( }, nil } -// readWriterRegistryLastSeen returns the registry's last_seen_local_epoch -// for (full_node_id, dek_id), or 0 when no row exists (treated as the -// "propose" trigger per the 7a design §3.1). It builds a fresh registry -// handle per call; hot/looping callers should build the handle once and -// use registryLastSeen instead (claude review on PR #847). -func readWriterRegistryLastSeen(st store.MVCCStore, dekID uint32, fullNodeID uint64) (uint16, error) { - reg, err := store.WriterRegistryFor(st) - if err != nil { - return 0, errors.Wrap(err, "writer registration: registry handle") - } - return registryLastSeen(reg, dekID, fullNodeID) -} - // registryLastSeen reads last_seen_local_epoch from an already-built // registry handle (no per-call WriterRegistryFor type-assert + alloc), // so the verifyRegistered retry loop can reuse one handle. diff --git a/main_encryption_registration_test.go b/main_encryption_registration_test.go index 3a5dc5dd6..727c8804d 100644 --- a/main_encryption_registration_test.go +++ b/main_encryption_registration_test.go @@ -178,16 +178,21 @@ func writeRegistryRow(t *testing.T, st store.MVCCStore, fullNodeID uint64, lastS } } -func TestReadWriterRegistryLastSeen(t *testing.T) { +func TestRegistryLastSeen(t *testing.T) { t.Parallel() st := newRegistrationTestStore(t) const dekID uint32 = 7 fullNodeID := etcdraftengine.DeriveNodeID("n1") + reg, err := store.WriterRegistryFor(st) + if err != nil { + t.Fatalf("WriterRegistryFor: %v", err) + } + // No row → 0. - got, err := readWriterRegistryLastSeen(st, dekID, fullNodeID) + got, err := registryLastSeen(reg, dekID, fullNodeID) if err != nil { - t.Fatalf("readWriterRegistryLastSeen (no row): %v", err) + t.Fatalf("registryLastSeen (no row): %v", err) } if got != 0 { t.Errorf("no-row last_seen = %d, want 0", got) @@ -195,9 +200,9 @@ func TestReadWriterRegistryLastSeen(t *testing.T) { // After writing last_seen=5 → 5. writeRegistryRow(t, st, fullNodeID, 5) - got, err = readWriterRegistryLastSeen(st, dekID, fullNodeID) + got, err = registryLastSeen(reg, dekID, fullNodeID) if err != nil { - t.Fatalf("readWriterRegistryLastSeen (with row): %v", err) + t.Fatalf("registryLastSeen (with row): %v", err) } if got != 5 { t.Errorf("with-row last_seen = %d, want 5", got) @@ -323,6 +328,13 @@ func TestBuildProcessStartRegistrationGate_NilGateBranches(t *testing.T) { expectRegistered bool }{ {name: "encryption_off", w: encryptionWriteWiring{cache: encryption.NewStateCache()}}, // cipher nil + // Design §2.3 fail-closed posture: a node that boots in Phase 0 + // (envelope inactive) skips registration without seeding + // Registered(), so the per-DEK gate is NOT registered for the + // active DEK. The Phase-0 boot itself emits no encrypted writes; + // a runtime EnableStorageEnvelope would then fail closed until + // the node (re)registers. This is the safe posture (no fail-OPEN + // per §2.3). {name: "phase0_envelope_inactive", w: wiringFor(t, 7, false, 1)}, {name: "not_bootstrapped", w: wiringFor(t, 0, true, 1)}, { @@ -834,40 +846,6 @@ func TestValidateRaftRegistrationStartupEpochAllowsStrictlyAhead(t *testing.T) { } } -// TestBuildProcessStartRegistrationGate_Phase0BootDoesNotMarkRegistered -// documents the design §2.3 fail-closed posture: a node that boots in -// Phase 0 (envelope inactive) skips registration without seeding -// Registered(), so the per-DEK gate is NOT registered for the active -// DEK. The Phase-0 boot itself emits no encrypted writes (envelope -// inactive → cleartext, gate unconsulted); a runtime EnableStorageEnvelope -// would then fail closed until the node (re)registers — the deferred -// runtime-registration follow-on. This is the safe posture (no fail-OPEN -// per §2.3), and benign today since the only direct write is the startup -// catalog bootstrap Save (covered by retryUntilRegistered). -func TestBuildProcessStartRegistrationGate_Phase0BootDoesNotMarkRegistered(t *testing.T) { - t.Parallel() - st := newRegistrationTestStore(t) - w := wiringFor(t, testRegDEKID, false /*envelope inactive: Phase 0*/, 1) - - eg, _ := errgroup.WithContext(context.Background()) - gate, err := buildProcessStartRegistrationGate( - context.Background(), eg, &kv.ShardedCoordinator{}, - &kv.ShardGroup{Store: st}, w, "n1") - if err != nil { - t.Fatalf("buildProcessStartRegistrationGate: %v", err) - } - if gate == nil || gate.Barrier != nil { - t.Fatalf("Phase 0 should be ungated (nil Barrier), got %+v", gate) - } - // Phase-0 boot did not seed Registered(): the node is not registered - // for the active DEK, so a later runtime cutover fail-closes the gate - // (deferred runtime-registration). The gate is correctly per-DEK and - // fail-closed rather than fail-OPEN (§2.3). - if w.cache.Registered() { - t.Error("Phase-0 boot unexpectedly marked Registered() — must not seed registration") - } -} - // TestBuildProcessStartRegistrationGate_BehindEpochFailsClosed pins // codex P1: a strictly-behind epoch (registry last_seen > bumped epoch) // must fail closed (error) rather than skip ungated. The §9.1 rollback diff --git a/main_encryption_write_wiring_test.go b/main_encryption_write_wiring_test.go index c00b1d1b3..74323b7b3 100644 --- a/main_encryption_write_wiring_test.go +++ b/main_encryption_write_wiring_test.go @@ -50,34 +50,6 @@ func markRaftEnvelopeCutoverInSidecar(t *testing.T, path string, cutover uint64) } } -func TestEncryptionWriteWiring_PebbleOptions_CleartextWhenNoCipher(t *testing.T) { - t.Parallel() - if opts := (encryptionWriteWiring{}).pebbleOptions(); opts != nil { - t.Errorf("zero wiring pebbleOptions = %d opts, want nil (cleartext)", len(opts)) - } - // cache present but cipher nil still means cleartext. - w := encryptionWriteWiring{cache: encryption.NewStateCache()} - if opts := w.pebbleOptions(); opts != nil { - t.Errorf("cipher-less wiring pebbleOptions = %d opts, want nil", len(opts)) - } -} - -func TestEncryptionWriteWiring_PebbleOptions_WiredWhenCipherSet(t *testing.T) { - t.Parallel() - cipher, err := encryption.NewCipher(encryption.NewKeystore()) - if err != nil { - t.Fatalf("NewCipher: %v", err) - } - w := encryptionWriteWiring{ - cache: encryption.NewStateCache(), - cipher: cipher, - nonceFactory: encryption.NewDeterministicNonceFactory(0xABCD, 0), - } - if opts := w.pebbleOptions(); len(opts) != 5 { - t.Errorf("wired pebbleOptions = %d opts, want 5 (encryption + storage cutover + V2 capability + registration gates + §9.2 metrics observer)", len(opts)) - } -} - func TestPrepareStorageNonceEpoch_NoSidecar_ReturnsZero(t *testing.T) { t.Parallel() cache := encryption.NewStateCache() diff --git a/main_metrics_test.go b/main_metrics_test.go deleted file mode 100644 index c1b82f741..000000000 --- a/main_metrics_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package main - -import ( - "context" - "net" - "net/http" - "testing" - - "github.com/stretchr/testify/require" - "golang.org/x/sync/errgroup" -) - -func TestStartMetricsServerRejectsInvalidAddressBeforeTokenCheck(t *testing.T) { - eg, ctx := errgroup.WithContext(context.Background()) - err := startMetricsServer(ctx, &net.ListenConfig{}, eg, "localhost", "", http.NewServeMux()) - require.ErrorContains(t, err, `invalid metricsAddress "localhost"`) -} diff --git a/main_pprof_test.go b/main_pprof_test.go deleted file mode 100644 index 494f4628d..000000000 --- a/main_pprof_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "context" - "net" - "testing" - - "github.com/stretchr/testify/require" - "golang.org/x/sync/errgroup" -) - -func TestStartPprofServerRejectsInvalidAddressBeforeTokenCheck(t *testing.T) { - eg, ctx := errgroup.WithContext(context.Background()) - err := startPprofServer(ctx, &net.ListenConfig{}, eg, "localhost", "") - require.ErrorContains(t, err, `invalid pprofAddress "localhost"`) -} - -func TestStartPprofServerAllowsEmptyAddress(t *testing.T) { - eg, ctx := errgroup.WithContext(context.Background()) - err := startPprofServer(ctx, &net.ListenConfig{}, eg, "", "") - require.NoError(t, err) -} diff --git a/main_s3.go b/main_s3.go index 900505a92..9ca8ef74e 100644 --- a/main_s3.go +++ b/main_s3.go @@ -12,71 +12,6 @@ import ( "golang.org/x/sync/errgroup" ) -// startS3Server stands up the S3-compatible HTTP listener on s3Addr -// and returns the constructed *adapter.S3Server. Callers that need a -// reference to the running server (e.g. the admin HTTP listener, -// which calls SigV4-bypass admin entrypoints on it) hold the -// returned value; callers that don't can ignore it. -// -// Returns (nil, nil) when s3Addr is empty — that is the well-known -// "S3 disabled" state, not a configuration error. Other failures -// surface as a non-nil error and a nil server. -func startS3Server( - ctx context.Context, - lc *net.ListenConfig, - eg *errgroup.Group, - s3Addr string, - shardStore *kv.ShardStore, - coordinate kv.Coordinator, - leaderS3 map[string]string, - region string, - credentialsFile string, - pathStyleOnly bool, - readTracker *kv.ActiveTimestampTracker, - putAdmissionObserver adapter.S3PutAdmissionObserver, - blobOffloadObserver adapter.S3BlobOffloadObserver, -) (*adapter.S3Server, error) { - s3Server, _, err := prepareS3Server( - ctx, lc, s3Addr, shardStore, coordinate, leaderS3, region, - credentialsFile, pathStyleOnly, readTracker, putAdmissionObserver, - blobOffloadObserver, nil, - ) - if err != nil { - return nil, err - } - runS3Server(ctx, eg, s3Server) - return s3Server, nil -} - -func prepareS3Server( - ctx context.Context, - lc *net.ListenConfig, - s3Addr string, - shardStore *kv.ShardStore, - coordinate kv.Coordinator, - leaderS3 map[string]string, - region string, - credentialsFile string, - pathStyleOnly bool, - readTracker *kv.ActiveTimestampTracker, - putAdmissionObserver adapter.S3PutAdmissionObserver, - blobOffloadObserver adapter.S3BlobOffloadObserver, - blobBackfiller *adapter.S3BlobBackfiller, -) (*adapter.S3Server, net.Listener, error) { - s3Server, err := newS3Server( - s3Addr, shardStore, coordinate, leaderS3, region, credentialsFile, - pathStyleOnly, readTracker, putAdmissionObserver, blobOffloadObserver, nil, nil, blobBackfiller, - ) - if err != nil { - return nil, nil, err - } - s3L, err := bindS3Server(ctx, lc, s3Addr, s3Server) - if err != nil { - return nil, nil, err - } - return s3Server, s3L, nil -} - func newS3Server( s3Addr string, shardStore *kv.ShardStore, diff --git a/main_s3_test.go b/main_s3_test.go index 456268920..ab56c0eb6 100644 --- a/main_s3_test.go +++ b/main_s3_test.go @@ -3,7 +3,6 @@ package main import ( "context" "crypto/sha256" - "net" "os" "path/filepath" "testing" @@ -17,16 +16,14 @@ import ( "golang.org/x/sync/errgroup" ) -func TestStartS3ServerRejectsVirtualHostedStyleConfig(t *testing.T) { - eg, ctx := errgroup.WithContext(context.Background()) - srv, err := startS3Server(ctx, &net.ListenConfig{}, eg, "localhost:9000", nil, nil, nil, "us-east-1", "", false, nil, nil, nil) +func TestNewS3ServerRejectsVirtualHostedStyleConfig(t *testing.T) { + srv, err := newS3Server("localhost:9000", nil, nil, nil, "us-east-1", "", false, nil, nil, nil, nil, nil, nil) require.ErrorContains(t, err, "virtual-hosted style S3 requests are not implemented") require.Nil(t, srv) } -func TestStartS3ServerAllowsEmptyAddress(t *testing.T) { - eg, ctx := errgroup.WithContext(context.Background()) - srv, err := startS3Server(ctx, &net.ListenConfig{}, eg, "", nil, nil, nil, "us-east-1", "", false, nil, nil, nil) +func TestNewS3ServerAllowsEmptyAddress(t *testing.T) { + srv, err := newS3Server("", nil, nil, nil, "us-east-1", "", false, nil, nil, nil, nil, nil, nil) require.NoError(t, err) require.Nil(t, srv) } diff --git a/main_sqs.go b/main_sqs.go index 898b44a65..105f7c53f 100644 --- a/main_sqs.go +++ b/main_sqs.go @@ -75,7 +75,7 @@ func runSQSServer(ctx context.Context, eg *errgroup.Group, sqsServer *adapter.SQ if sqsServer == nil { return } - // Two-goroutine shutdown pattern mirrors startS3Server: one goroutine waits + // Two-goroutine shutdown pattern mirrors runS3Server: one goroutine waits // on either ctx.Done() or Run completion to call Stop, the other runs the // server and cancels the waiter once it has returned. runDoneCtx, runDoneCancel := context.WithCancel(context.Background()) diff --git a/proxy/pubsub_test.go b/proxy/pubsub_test.go index ccdc0f311..4113b7d52 100644 --- a/proxy/pubsub_test.go +++ b/proxy/pubsub_test.go @@ -152,55 +152,6 @@ func newTestSession(dconn *mockDetachedConn) *pubsubSession { } } -func TestPubSub_SubscribeDuplicate_IdempotentCount(t *testing.T) { - dconn := newMockDetachedConn() - s := newTestSession(dconn) - - // Simulate an upstream that's already connected. - s.channelSet["ch1"] = struct{}{} - - // Subscribe to ch1 again and ch2 (should count ch1 only once). - // We can't call handleSubscribe directly because it needs s.upstream, - // so we test the set logic directly. - s.channelSet["ch1"] = struct{}{} // re-add same key - s.channelSet["ch2"] = struct{}{} - - assert.Equal(t, 2, len(s.channelSet), "duplicate subscribe should not increase count") - assert.Equal(t, 2, s.subCount()) -} - -func TestPubSub_UnsubscribeNonSubscribed_NoEffect(t *testing.T) { - dconn := newMockDetachedConn() - s := newTestSession(dconn) - - s.channelSet["ch1"] = struct{}{} - s.channelSet["ch2"] = struct{}{} - - // Unsubscribe from "ch3" which was never subscribed. - delete(s.channelSet, "ch3") - - assert.Equal(t, 2, len(s.channelSet), "unsubscribe non-subscribed channel should not affect count") -} - -func TestPubSub_UnsubscribeSpecific_RemovesFromSet(t *testing.T) { - dconn := newMockDetachedConn() - s := newTestSession(dconn) - - s.channelSet["ch1"] = struct{}{} - s.channelSet["ch2"] = struct{}{} - s.channelSet["ch3"] = struct{}{} - - delete(s.channelSet, "ch2") - - assert.Equal(t, 2, len(s.channelSet)) - _, hasCh1 := s.channelSet["ch1"] - _, hasCh2 := s.channelSet["ch2"] - _, hasCh3 := s.channelSet["ch3"] - assert.True(t, hasCh1) - assert.False(t, hasCh2) - assert.True(t, hasCh3) -} - func TestPubSub_WriteUnsubAll_PerChannelReplies(t *testing.T) { dconn := newMockDetachedConn() s := newTestSession(dconn) From baf555c79b6df8dbc806d939c8adcceb9a777a72 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 09:47:32 +0000 Subject: [PATCH 04/15] adapter: retarget non-Redis cleanup WP-5 tests to live paths Delete ten tests that a named survivor already covers or that cannot fail, plus the GSI-path duplicate of TestDynamoDB_QueryLeaseFallbackFencesAllGroups. Move the SigV4 signature table onto the live extractSigV4Signature and delete the uncalled extractS3Signature. Drive the SQS throttle tests through the production reset sequence (beginQueueReset before commit, invalidateQueueBuckets after) and delete invalidateQueue and bumpQueueEpoch; the lock-ordering comment moves onto invalidateQueueBuckets. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- adapter/admin_backup_renew_race_test.go | 32 ------ adapter/admin_backup_test.go | 11 -- adapter/distribution_server_test.go | 12 --- adapter/dynamodb_lease_validation_test.go | 40 ------- adapter/dynamodb_test.go | 86 --------------- adapter/s3_auth.go | 16 --- adapter/s3_test.go | 101 ------------------ adapter/sigv4_test.go | 52 +++++++++ adapter/sqs_health_test.go | 20 ---- adapter/sqs_messages.go | 2 +- adapter/sqs_test.go | 36 ------- adapter/sqs_throttle.go | 32 ++---- adapter/sqs_throttle_test.go | 55 ++++------ ...26_implemented_sqs_per_queue_throttling.md | 4 +- 14 files changed, 81 insertions(+), 418 deletions(-) create mode 100644 adapter/sigv4_test.go diff --git a/adapter/admin_backup_renew_race_test.go b/adapter/admin_backup_renew_race_test.go index a548350bd..37760eb2c 100644 --- a/adapter/admin_backup_renew_race_test.go +++ b/adapter/admin_backup_renew_race_test.go @@ -72,38 +72,6 @@ func TestRenewBackupKeepsAConcurrentlyRenewedSession(t *testing.T) { require.NotContains(t, proposer.subtypes(), backupSubtypeUnreserve) } -// With no concurrent renewal the failing attempt still owns the session, so it -// must tear the pin down exactly as before. -func TestRenewBackupStillReleasesWhenItOwnsTheSession(t *testing.T) { - t.Parallel() - - group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} - proposer := newBackupTestProposer() - srv := newBackupControlTestServer( - t, - &backupTestStore{}, - map[uint64]*backupTestGroup{1: group}, - map[uint64]*backupTestProposer{1: proposer}, - nil, - ) - begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) - require.NoError(t, err) - tok, err := srv.decodeBackupToken(begin.GetPinToken()) - require.NoError(t, err) - - proposer.mu.Lock() - proposer.failures[backupSubtypePin] = 8 - proposer.transportError[backupSubtypePin] = stderrors.New("leader unavailable") - proposer.mu.Unlock() - - _, err = srv.RenewBackup(context.Background(), &pb.RenewBackupRequest{PinToken: begin.GetPinToken()}) - require.Equal(t, codes.Unavailable, status.Code(err)) - require.Contains(t, proposer.subtypes(), backupSubtypeRelease) - require.Contains(t, proposer.subtypes(), backupSubtypeUnreserve) - _, err = srv.backupRouteSnapshotForToken(tok) - require.Equal(t, codes.FailedPrecondition, status.Code(err)) -} - // A reservation whose proposal fails for any reason other than a capacity // rejection is ambiguous: it may have committed with only the response lost. // Leaving it in place holds one of the few global active-backup slots until diff --git a/adapter/admin_backup_test.go b/adapter/admin_backup_test.go index e6227d74f..04f743483 100644 --- a/adapter/admin_backup_test.go +++ b/adapter/admin_backup_test.go @@ -1210,17 +1210,6 @@ func TestRenewBackupReleasesResourcesAfterPinFanoutFailure(t *testing.T) { require.Equal(t, codes.FailedPrecondition, status.Code(err)) } -func TestBeginBackupMapsCapacityReservationToResourceExhausted(t *testing.T) { - t.Parallel() - group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} - proposer := newBackupTestProposer() - proposer.responseError[backupSubtypeReserve] = kv.ErrTooManyActiveBackups - srv := newBackupControlTestServer(t, &backupTestStore{}, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) - - _, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) - require.Equal(t, codes.ResourceExhausted, status.Code(err)) -} - func TestBeginBackupMapsForwardedCapacityReservationStatusToResourceExhausted(t *testing.T) { t.Parallel() group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 703a13321..ff6aac0a3 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -521,18 +521,6 @@ func TestNewDistributionServer_DefaultCatalogReloadRetryPolicy(t *testing.T) { require.Equal(t, defaultCatalogReloadRetryInterval, s.reloadRetry.interval) } -func TestWithCatalogReloadRetryPolicy_OverridesDefaults(t *testing.T) { - t.Parallel() - - s := NewDistributionServer( - distribution.NewEngine(), - distribution.NewCatalogStore(store.NewMVCCStore()), - WithCatalogReloadRetryPolicy(3, 5*time.Millisecond), - ) - require.Equal(t, 3, s.reloadRetry.attempts) - require.Equal(t, 5*time.Millisecond, s.reloadRetry.interval) -} - func TestDistributionServerListRoutes_ReadsDurableCatalog(t *testing.T) { t.Parallel() diff --git a/adapter/dynamodb_lease_validation_test.go b/adapter/dynamodb_lease_validation_test.go index 7089cb6bf..6859fa190 100644 --- a/adapter/dynamodb_lease_validation_test.go +++ b/adapter/dynamodb_lease_validation_test.go @@ -151,46 +151,6 @@ func TestDynamoDB_Query_MissingTableReturns400UnderDegradedLease(t *testing.T) { require.Contains(t, rec.Body.String(), dynamoErrResourceNotFound) } -// TestDynamoDB_LeaseCheckQuery_WholeTableQueryStillFencesAllGroups asserts the -// fix preserves the all-groups fence for a VALID multi-shard read: a query whose -// hash key is not the primary hash key reads the whole-table prefix (multiple -// shards), so it must still fail closed when the lease cannot be confirmed. -func TestDynamoDB_LeaseCheckQuery_WholeTableQueryStillFencesAllGroups(t *testing.T) { - t.Parallel() - - st := store.NewMVCCStore() - writer := newDynamoFixtureWriter(t, st) - writer.writeSchema(leaseGSIFixtureSchema()) - - coord := newMultiGroupLeaseCoordinator() - server := NewDynamoDBServer(nil, st, coord) - - // IndexName set + a VALID GSI on an EXISTING table => valid multi-shard - // read => keyless all-groups fence, not a skip. (Whole-table base-table - // queries take the same all-groups path through queryLeasePrefix's HashKey - // mismatch branch.) The schema must exist so the GSI is genuinely valid: - // after the P2-B fix an indexed query against a missing table or unknown - // index is classified as a skip, not an all-groups fence. - in := queryInput{ - TableName: "t", - IndexName: "gsi1", - KeyConditionExpression: "gk = :gk", - ExpressionAttributeValues: map[string]attributeValue{ - ":gk": newStringAttributeValue("x"), - }, - } - - rec := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(nil)) - require.True(t, server.leaseCheckQuery(rec, req, in)) - - fenced := coord.fencedGroupSet() - require.Contains(t, fenced, multiGroupDefaultID, - "valid multi-shard query must still fence the default group") - require.Contains(t, fenced, multiGroupOtherID, - "valid multi-shard query must still fence every group (must not be skipped as a validation case)") -} - // leaseGSIFixtureSchema is a table with a single GSI ("gsi1", ALL projection) // used by the lease pre-pass tests that exercise VALID multi-shard GSI reads. func leaseGSIFixtureSchema() *dynamoTableSchema { diff --git a/adapter/dynamodb_test.go b/adapter/dynamodb_test.go index be86c007a..e4160812a 100644 --- a/adapter/dynamodb_test.go +++ b/adapter/dynamodb_test.go @@ -218,29 +218,6 @@ func TestDynamoDB_RequestBodyTooLarge(t *testing.T) { } } -func TestDynamoDB_Healthz(t *testing.T) { - t.Parallel() - nodes, _, _ := createNode(t, 1) - defer shutdown(nodes) - - req, err := http.NewRequestWithContext( - context.Background(), - http.MethodGet, - "http://"+nodes[0].dynamoAddress+dynamoHealthPath, - nil, - ) - require.NoError(t, err) - - resp, err := http.DefaultClient.Do(req) - require.NoError(t, err) - defer resp.Body.Close() - - require.Equal(t, http.StatusOK, resp.StatusCode) - body, err := io.ReadAll(resp.Body) - require.NoError(t, err) - require.Equal(t, "ok\n", string(body)) -} - func TestDynamoDB_LeaderHealthz(t *testing.T) { t.Parallel() nodes, _, _ := createNode(t, 2) @@ -298,69 +275,6 @@ func TestDynamoDB_LeaderHealthz(t *testing.T) { } } -func TestDynamoDB_TransactWriteItems(t *testing.T) { - t.Parallel() - nodes, _, _ := createNode(t, 1) - defer shutdown(nodes) - - cfg, err := config.LoadDefaultConfig(context.Background(), - config.WithRegion("us-west-2"), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("dummy", "dummy", "")), - ) - assert.NoError(t, err) - - client := dynamodb.NewFromConfig(cfg, func(o *dynamodb.Options) { - o.BaseEndpoint = aws.String("http://" + nodes[0].dynamoAddress) - }) - createSimpleKeyTable(t, context.Background(), client) - - _, err = client.TransactWriteItems(context.Background(), &dynamodb.TransactWriteItemsInput{ - TransactItems: []types.TransactWriteItem{ - { - Put: &types.Put{ - TableName: aws.String("t"), - Item: map[string]types.AttributeValue{ - "key": &types.AttributeValueMemberS{Value: "k1"}, - "value": &types.AttributeValueMemberS{Value: "v1"}, - }, - }, - }, - { - Put: &types.Put{ - TableName: aws.String("t"), - Item: map[string]types.AttributeValue{ - "key": &types.AttributeValueMemberS{Value: "k2"}, - "value": &types.AttributeValueMemberS{Value: "v2"}, - }, - }, - }, - }, - }) - assert.NoError(t, err) - - out1, err := client.GetItem(context.Background(), &dynamodb.GetItemInput{ - TableName: aws.String("t"), - Key: map[string]types.AttributeValue{ - "key": &types.AttributeValueMemberS{Value: "k1"}, - }, - }) - assert.NoError(t, err) - value1Attr, ok := out1.Item["value"].(*types.AttributeValueMemberS) - assert.True(t, ok) - assert.Equal(t, "v1", value1Attr.Value) - - out2, err := client.GetItem(context.Background(), &dynamodb.GetItemInput{ - TableName: aws.String("t"), - Key: map[string]types.AttributeValue{ - "key": &types.AttributeValueMemberS{Value: "k2"}, - }, - }) - assert.NoError(t, err) - value2Attr, ok := out2.Item["value"].(*types.AttributeValueMemberS) - assert.True(t, ok) - assert.Equal(t, "v2", value2Attr.Value) -} - func TestDynamoDB_TransactGetItems(t *testing.T) { t.Parallel() nodes, _, _ := createNode(t, 1) diff --git a/adapter/s3_auth.go b/adapter/s3_auth.go index 4105b3492..6039d9585 100644 --- a/adapter/s3_auth.go +++ b/adapter/s3_auth.go @@ -422,19 +422,3 @@ func extractPresignedSignature(presignedURL string) string { } return parsed.Query().Get("X-Amz-Signature") } - -// extractS3Signature returns the hex signature value from a SigV4 -// Authorization header (the "Signature=" component). -func extractS3Signature(auth string) string { - _, params, ok := strings.Cut(auth, " ") - if !ok { - return "" - } - for _, param := range strings.Split(params, ",") { - key, value, ok := strings.Cut(strings.TrimSpace(param), "=") - if ok && key == "Signature" { - return strings.TrimSpace(value) - } - } - return "" -} diff --git a/adapter/s3_test.go b/adapter/s3_test.go index 554792d35..0294c3308 100644 --- a/adapter/s3_test.go +++ b/adapter/s3_test.go @@ -1411,27 +1411,6 @@ func TestS3Server_RangeReadFullAndPartial(t *testing.T) { } } -func TestS3Server_RangeReadInvalidRange(t *testing.T) { - t.Parallel() - - st := store.NewMVCCStore() - server := NewS3Server(nil, "", st, newLocalAdapterCoordinator(st), nil) - - rec := httptest.NewRecorder() - server.handle(rec, newS3TestRequest(http.MethodPut, "/bucket-rinv", nil)) - require.Equal(t, http.StatusOK, rec.Code) - - rec = httptest.NewRecorder() - server.handle(rec, newS3TestRequest(http.MethodPut, "/bucket-rinv/f.txt", strings.NewReader("hello"))) - require.Equal(t, http.StatusOK, rec.Code) - - rec = httptest.NewRecorder() - req := newS3TestRequest(http.MethodGet, "/bucket-rinv/f.txt", nil) - req.Header.Set("Range", "bytes=99-100") - server.handle(rec, req) - require.Equal(t, http.StatusRequestedRangeNotSatisfiable, rec.Code) -} - func TestS3Server_RangeReadEmptyObject(t *testing.T) { t.Parallel() @@ -1930,55 +1909,6 @@ func TestS3Server_MultipartUploadPartOverwrite(t *testing.T) { require.Equal(t, data2+lastPart, rec.Body.String()) } -func TestExtractS3Signature(t *testing.T) { - t.Parallel() - cases := []struct { - name string - input string - want string - }{ - { - name: "standard SigV4 header", - input: "AWS4-HMAC-SHA256 Credential=AKID/20240101/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=abc123def456", - want: "abc123def456", - }, - { - name: "signature with extra whitespace", - input: "AWS4-HMAC-SHA256 Credential=AKID/20240101/us-east-1/s3/aws4_request, SignedHeaders=host, Signature= abc123 ", - want: "abc123", - }, - { - name: "different parameter order", - input: "AWS4-HMAC-SHA256 SignedHeaders=host;x-amz-date, Credential=AKID/20240101/us-east-1/s3/aws4_request, Signature=deadbeef", - want: "deadbeef", - }, - { - name: "missing signature", - input: "AWS4-HMAC-SHA256 Credential=AKID/20240101/us-east-1/s3/aws4_request, SignedHeaders=host", - want: "", - }, - { - name: "empty header", - input: "", - want: "", - }, - { - name: "no space separator", - input: "AWS4-HMAC-SHA256Credential=foo", - want: "", - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - got := extractS3Signature(tc.input) - if got != tc.want { - t.Errorf("extractS3Signature(%q) = %q, want %q", tc.input, got, tc.want) - } - }) - } -} - func TestS3Server_CompleteMultipartUploadETagMismatch(t *testing.T) { t.Parallel() @@ -2206,37 +2136,6 @@ func TestS3Server_PublicBucket_AnonymousListObjectsV2(t *testing.T) { require.Contains(t, rec.Body.String(), "obj.txt") } -func TestS3Server_PrivateBucket_AnonymousGetRejected(t *testing.T) { - t.Parallel() - - st := store.NewMVCCStore() - server := NewS3Server( - nil, "", st, newLocalAdapterCoordinator(st), nil, - WithS3StaticCredentials(map[string]string{testS3AccessKey: testS3SecretKey}), - ) - - sigTime := currentS3SigningTime() - // Create private bucket (default ACL). - req := newSignedS3Request(t, "/priv-bucket", "", sigTime) - req.Method = http.MethodPut - req = resignS3Request(t, req, sigTime) - rec := httptest.NewRecorder() - server.handle(rec, req) - require.Equal(t, http.StatusOK, rec.Code) - - // Upload object. - req = newSignedS3Request(t, "/priv-bucket/secret.txt", "secret", sigTime) - rec = httptest.NewRecorder() - server.handle(rec, req) - require.Equal(t, http.StatusOK, rec.Code) - - // Anonymous GET rejected. - req = newS3TestRequest(http.MethodGet, "/priv-bucket/secret.txt", nil) - rec = httptest.NewRecorder() - server.handle(rec, req) - require.Equal(t, http.StatusForbidden, rec.Code) -} - func TestS3Server_PutBucketAcl_ChangeToPublicAndBack(t *testing.T) { t.Parallel() diff --git a/adapter/sigv4_test.go b/adapter/sigv4_test.go new file mode 100644 index 000000000..39a059029 --- /dev/null +++ b/adapter/sigv4_test.go @@ -0,0 +1,52 @@ +package adapter + +import "testing" + +func TestExtractSigV4Signature(t *testing.T) { + t.Parallel() + cases := []struct { + name string + input string + want string + }{ + { + name: "standard SigV4 header", + input: "AWS4-HMAC-SHA256 Credential=AKID/20240101/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=abc123def456", + want: "abc123def456", + }, + { + name: "signature with extra whitespace", + input: "AWS4-HMAC-SHA256 Credential=AKID/20240101/us-east-1/s3/aws4_request, SignedHeaders=host, Signature= abc123 ", + want: "abc123", + }, + { + name: "different parameter order", + input: "AWS4-HMAC-SHA256 SignedHeaders=host;x-amz-date, Credential=AKID/20240101/us-east-1/s3/aws4_request, Signature=deadbeef", + want: "deadbeef", + }, + { + name: "missing signature", + input: "AWS4-HMAC-SHA256 Credential=AKID/20240101/us-east-1/s3/aws4_request, SignedHeaders=host", + want: "", + }, + { + name: "empty header", + input: "", + want: "", + }, + { + name: "no space separator", + input: "AWS4-HMAC-SHA256Credential=foo", + want: "", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := extractSigV4Signature(tc.input) + if got != tc.want { + t.Errorf("extractSigV4Signature(%q) = %q, want %q", tc.input, got, tc.want) + } + }) + } +} diff --git a/adapter/sqs_health_test.go b/adapter/sqs_health_test.go index 98907be80..6a1c19e53 100644 --- a/adapter/sqs_health_test.go +++ b/adapter/sqs_health_test.go @@ -112,26 +112,6 @@ func TestServeSQSHealthz_RejectsNonGETHEAD(t *testing.T) { } } -// TestSQSAdvertisedCapabilities_TracksFlag pins the relationship -// between htfifoCapabilityAdvertised and the emitted list. If a -// future change flips the constant, the JSON response must reflect -// it without further wiring — the constant is the single source of -// truth. -func TestSQSAdvertisedCapabilities_TracksFlag(t *testing.T) { - t.Parallel() - caps := sqsAdvertisedCapabilities() - if htfifoCapabilityAdvertised { - require.Contains(t, caps, sqsCapabilityHTFIFO, - "htfifo must be in the list when the flag is true") - } else { - require.NotContains(t, caps, sqsCapabilityHTFIFO, - "htfifo must NOT be in the list when the flag is false; "+ - "a partial deploy that advertised the capability without "+ - "the routing + leadership-refusal pair would create new "+ - "partitioned queues that this binary cannot safely host") - } -} - // TestClientAcceptsSQSHealthJSON_Boundaries pins the substring // matcher's edge cases — these are the inputs the catalog-polling // caller in Phase 3.D PR 5 will produce, and a regression that diff --git a/adapter/sqs_messages.go b/adapter/sqs_messages.go index 90d9138e5..fbfd417f8 100644 --- a/adapter/sqs_messages.go +++ b/adapter/sqs_messages.go @@ -1087,7 +1087,7 @@ func (s *SQSServer) nextReceiveFanoutStart(queueName string, partitions uint32) } // dropReceiveFanoutCounter removes any per-queue fanout-rotation -// counter for queueName. Mirrors throttle.invalidateQueue: called +// counter for queueName. Mirrors throttle.invalidateQueueBuckets: called // from the same DeleteQueue and genuine-CreateQueue paths so the // sync.Map does not retain a counter for every queue name the // process has ever served. Without this, repeated create/delete of diff --git a/adapter/sqs_test.go b/adapter/sqs_test.go index 21ddd7762..ccea1c56a 100644 --- a/adapter/sqs_test.go +++ b/adapter/sqs_test.go @@ -85,20 +85,6 @@ func TestSQSServer_LeaderHealthWithoutCoordinatorIsNotLeader(t *testing.T) { } } -func TestSQSServer_HealthRejectsNonGetHead(t *testing.T) { - t.Parallel() - base := startTestSQSServer(t) - - resp := doRequest(t, http.MethodPost, base+sqsHealthPath, nil, nil) - defer resp.Body.Close() - if resp.StatusCode != http.StatusMethodNotAllowed { - t.Fatalf("status: got %d want %d", resp.StatusCode, http.StatusMethodNotAllowed) - } - if allow := resp.Header.Get("Allow"); !strings.Contains(allow, "GET") { - t.Fatalf("allow header: got %q", allow) - } -} - func TestSQSServer_UnknownTargetReturnsInvalidAction(t *testing.T) { t.Parallel() base := startTestSQSServer(t) @@ -120,28 +106,6 @@ func TestSQSServer_UnknownTargetReturnsInvalidAction(t *testing.T) { } } -// TestSQSServer_AllTargetsHaveHandlers asserts every target listed in -// targetHandlers has a real handler attached. The previous version of -// this test pinned a fixed list of NotImplemented targets and had to -// be updated each time a handler shipped — that drift hid the -// PurgeQueue/Tag* implementations behind a stale assertion. Here we -// instead reach into the dispatch map and confirm none of the -// registered targets is the NotImplemented stub. -func TestSQSServer_AllTargetsHaveHandlers(t *testing.T) { - t.Parallel() - base := startTestSQSServer(t) - - // Sanity-check the route table against an unknown target — we - // already test that path elsewhere, but it pins the assumption that - // unregistered targets surface InvalidAction (not 501) so this test - // is not the one to break if that contract changes. - resp := postSQSRequest(t, base+"/", "AmazonSQS.NotARealOp", "{}") - defer resp.Body.Close() - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("unknown target: got %d want %d", resp.StatusCode, http.StatusBadRequest) - } -} - func TestSQSServer_RejectsNonPostOnRoot(t *testing.T) { t.Parallel() base := startTestSQSServer(t) diff --git a/adapter/sqs_throttle.go b/adapter/sqs_throttle.go index 353916ff9..9b5587812 100644 --- a/adapter/sqs_throttle.go +++ b/adapter/sqs_throttle.go @@ -106,7 +106,7 @@ type bucketKey struct { // concurrent traffic on different queues never serialises on the same // lock; refill + take + release of a single bucket is the only // critical section. evicted flips to true exactly once (under mu) when -// the bucket is removed from the store by sweep / invalidateQueue / +// the bucket is removed from the store by sweep / invalidateQueueBuckets / // loadOrInit reconciliation; charge re-checks it after acquiring mu so // goroutines that loaded the now-orphaned bucket retry and converge on // the live entry. Without that retry, sweep racing N concurrent chargers @@ -184,17 +184,6 @@ func (b *bucketStore) queueEpoch(queue string) uint64 { return epoch.epoch.Load() } -func (b *bucketStore) bumpQueueEpoch(queue string) { - if b == nil || queue == "" { - return - } - b.queueEpochMu.Lock() - defer b.queueEpochMu.Unlock() - cell := b.queueEpochCellLocked(queue) - cell.epoch.Add(1) - cell.updatedUnixNano.Store(b.clock().UnixNano()) -} - func (b *bucketStore) beginQueueReset(queue string, cutoff func() uint64) uint64 { if b == nil || queue == "" { if cutoff == nil { @@ -302,7 +291,7 @@ func (b *bucketStore) charge(cfg *sqsQueueThrottle, queue, action string, incarn // chargeBucket runs the under-mu refill+take for a single bucket and // returns retry=true if the caller should drop the reference and reload // from the store. Retry is the orphan-bucket signal: sweep / -// invalidateQueue / loadOrInit reconciliation set evicted=true under +// invalidateQueueBuckets / loadOrInit reconciliation set evicted=true under // mu before dropping the bucket from the map, so a goroutine that // loaded the bucket pre-eviction can detect it here. func chargeBucket(bucket *tokenBucket, now time.Time, count int) (chargeOutcome, bool) { @@ -386,7 +375,7 @@ func (b *bucketStore) loadOrInit(queue, action string, incarnation uint64, capac // via LoadOrStore — a one-time burst above the configured // limit right after a config reconciliation. Holding mu // across the whole sequence makes the lock-then-delete-then- - // flag ordering match sweep / invalidateQueue, so a charger + // flag ordering match sweep / invalidateQueueBuckets, so a charger // blocked on mu sees evicted=true on entry and retries. bucket.mu.Lock() matches := bucket.capacity == capacity && bucket.refillRate == refill @@ -426,8 +415,9 @@ func (b *bucketStore) loadOrInit(queue, action string, incarnation uint64, capac return bucket } -// invalidateQueue drops every bucket belonging to the named queue. -// Called *after* the Raft commit on SetQueueAttributes / DeleteQueue / +// invalidateQueueBuckets drops every bucket belonging to the named +// queue. Called *after* the Raft commit (beginQueueReset bumps the +// queue epoch before it) on SetQueueAttributes / DeleteQueue / // CreateQueue so the next request rebuilds from the freshly committed // meta. Mirrors sweep's lock-then-delete-then-flag ordering: // @@ -446,14 +436,6 @@ func (b *bucketStore) loadOrInit(queue, action string, incarnation uint64, capac // while evicted is still false, then later requests would create a // fresh full-capacity bucket — a 2x burst on every invalidation // event. -func (b *bucketStore) invalidateQueue(queue string) { - if b == nil { - return - } - b.bumpQueueEpoch(queue) - b.invalidateQueueBuckets(queue) -} - func (b *bucketStore) invalidateQueueBuckets(queue string) { if b == nil { return @@ -522,7 +504,7 @@ func (b *bucketStore) runSweepLoop(ctx context.Context) { // lastRefill and skips delete) or blocks on mu until sweep // releases — and on release sees evicted=true and retries. // 2. CompareAndDelete with v ensures sweep does not evict a -// replacement bucket inserted by invalidateQueue or a future +// replacement bucket inserted by invalidateQueueBuckets or a future // reconciliation path. // 3. Set evicted=true under mu after a successful CompareAndDelete. // Without this signal, goroutines that loaded the bucket diff --git a/adapter/sqs_throttle_test.go b/adapter/sqs_throttle_test.go index 314c2432b..730d04f83 100644 --- a/adapter/sqs_throttle_test.go +++ b/adapter/sqs_throttle_test.go @@ -157,29 +157,6 @@ func TestSQSServer_ChargeQueueWithThrottleDoesNotSyncMetricActions(t *testing.T) require.Empty(t, observer.forgotten) } -func TestSQSServer_ChargeQueueWithThrottleSkipsStaleMetricAfterInvalidate(t *testing.T) { - t.Parallel() - observer := &recordingSQSThrottleObserver{} - srv := NewSQSServer(nil, nil, nil, WithSQSThrottleObserver(observer)) - now := time.Date(2026, 4, 27, 10, 0, 0, 0, time.UTC) - srv.throttle = newBucketStore(func() time.Time { return now }, throttleIdleEvictAfter) - cfg := &sqsQueueThrottle{SendCapacity: 10, SendRefillPerSecond: 1} - - staleEpoch := srv.throttle.queueEpoch("orders.fifo") - srv.throttle.invalidateQueue("orders.fifo") - - rec := httptest.NewRecorder() - require.True(t, srv.chargeQueueWithThrottle(rec, "orders.fifo", bucketActionSend, 1, cfg, 1, staleEpoch)) - require.Empty(t, observer.reports, "stale pre-invalidation snapshot must not recreate a reset token gauge") - - freshEpoch := srv.throttle.queueEpoch("orders.fifo") - rec = httptest.NewRecorder() - require.True(t, srv.chargeQueueWithThrottle(rec, "orders.fifo", bucketActionSend, 1, cfg, 1, freshEpoch)) - require.Len(t, observer.reports, 1, "fresh post-invalidation snapshot may publish the new token gauge") - require.Equal(t, "orders.fifo", observer.reports[0].queue) - require.Equal(t, SQSThrottleActionSend, observer.reports[0].action) -} - func TestSQSServer_BeginThrottleResetSuppressesStaleMetricBeforeCleanup(t *testing.T) { t.Parallel() observer := &recordingSQSThrottleObserver{} @@ -231,7 +208,8 @@ func TestSQSServer_ChargeQueueWithThrottleCountsStaleRejectionAfterInvalidate(t cfg := &sqsQueueThrottle{SendCapacity: 1, SendRefillPerSecond: 1} staleEpoch := srv.throttle.queueEpoch("orders.fifo") - srv.throttle.invalidateQueue("orders.fifo") + srv.beginThrottleReset("orders.fifo") + srv.throttle.invalidateQueueBuckets("orders.fifo") rec := httptest.NewRecorder() require.False(t, srv.chargeQueueWithThrottle(rec, "orders.fifo", bucketActionSend, 2, cfg, 1, staleEpoch)) @@ -652,7 +630,7 @@ func TestBucketStore_OrphanedBucketRetriesToLiveEntry(t *testing.T) { // observe evicted=false, and spend tokens against the orphaned // bucket. The fix holds bucket.mu across the matches check, the // CompareAndDelete, and the evicted=true assignment — matching -// the lock-then-delete-then-flag ordering sweep / invalidateQueue +// the lock-then-delete-then-flag ordering sweep / invalidateQueueBuckets // already use. This deterministic test replicates the post- // reconciliation orphan check by forcing a config mismatch and // verifying the orphan bucket is evicted=true after the @@ -696,7 +674,7 @@ func TestBucketStore_LoadOrInitReconciliationMarksOrphanEvictedUnderMu(t *testin } // TestBucketStore_InvalidateMarksOrphanEvicted pins the -// invalidateQueue contract: dropped buckets must flip evicted=true under mu +// invalidateQueueBuckets contract: dropped buckets must flip evicted=true under mu // so a sendMessage that loaded meta pre-invalidation and is racing // against DeleteQueue / SetQueueAttributes / CreateQueue retries // rather than charging the orphan. @@ -710,17 +688,18 @@ func TestBucketStore_InvalidateMarksOrphanEvicted(t *testing.T) { require.True(t, ok) original, _ := v.(*tokenBucket) - store.invalidateQueue("orders") + store.beginQueueReset("orders", nil) + store.invalidateQueueBuckets("orders") _, stillThere := store.buckets.Load(key) - require.False(t, stillThere, "invalidateQueue must remove the bucket from the map") + require.False(t, stillThere, "queue reset must remove the bucket from the map") original.mu.Lock() - require.True(t, original.evicted, "invalidateQueue must mark the dropped bucket evicted") + require.True(t, original.evicted, "queue reset must mark the dropped bucket evicted") original.mu.Unlock() } // TestBucketStore_InvalidateUnderConcurrencyIsRaceFree pins the -// invalidateQueue lock ordering. The earlier invalidateQueue used +// invalidateQueueBuckets lock ordering. The earlier invalidateQueueBuckets used // LoadAndDelete-then-lock, which let a concurrent charger that loaded // the pointer pre-LoadAndDelete acquire bucket.mu first and observe // evicted=false on a bucket that had already been removed from the @@ -754,7 +733,8 @@ func TestBucketStore_InvalidateUnderConcurrencyIsRaceFree(t *testing.T) { for range invalidates { go func() { defer wg.Done() - store.invalidateQueue("orders") + store.beginQueueReset("orders", nil) + store.invalidateQueueBuckets("orders") }() } for range chargers { @@ -827,7 +807,7 @@ func TestBucketStore_IncarnationKeyedDoesNotReuseAcrossIncarnations(t *testing.T } // TestBucketStore_InvalidateQueueClearsAllIncarnations pins that -// invalidateQueue ranges by queue prefix and removes every cached +// invalidateQueueBuckets ranges by queue prefix and removes every cached // incarnation, not just one. Required because the leader's view of // "current incarnation" can drift from a follower's during a // failover-mid-CreateQueue, and an invalidate that only hit one @@ -843,11 +823,12 @@ func TestBucketStore_InvalidateQueueClearsAllIncarnations(t *testing.T) { require.True(t, store.charge(cfg, "events", bucketActionSend, 1, 1).allowed) require.Equal(t, 3, countBuckets(store)) - store.invalidateQueue("orders") + store.beginQueueReset("orders", nil) + store.invalidateQueueBuckets("orders") // Both orders incarnations gone, events untouched. require.Equal(t, 1, countBuckets(store), - "invalidateQueue must drop every incarnation belonging to the queue") + "queue reset must drop every incarnation belonging to the queue") _, hasOrdersInc1 := store.buckets.Load(bucketKey{queue: "orders", action: bucketActionSend, incarnation: 1}) _, hasOrdersInc2 := store.buckets.Load(bucketKey{queue: "orders", action: bucketActionSend, incarnation: 2}) _, hasEvents := store.buckets.Load(bucketKey{queue: "events", action: bucketActionSend, incarnation: 1}) @@ -861,7 +842,8 @@ func TestBucketStore_SweepEvictsIdleQueueEpochs(t *testing.T) { now := time.Date(2026, 4, 27, 10, 0, 0, 0, time.UTC) store := newBucketStore(func() time.Time { return now }, time.Minute) - store.invalidateQueue("orders") + store.beginQueueReset("orders", nil) + store.invalidateQueueBuckets("orders") require.Equal(t, uint64(1), store.queueEpoch("orders")) require.Equal(t, 1, countQueueEpochs(store)) @@ -939,7 +921,8 @@ func TestBucketStore_InvalidateQueueDropsAllActions(t *testing.T) { require.False(t, store.charge(cfg, "orders", bucketActionSend, 1, 1).allowed) require.False(t, store.charge(cfg, "orders", bucketActionReceive, 1, 1).allowed) // Invalidate. - store.invalidateQueue("orders") + store.beginQueueReset("orders", nil) + store.invalidateQueueBuckets("orders") // Both buckets must now be at full capacity again. for range 10 { require.True(t, store.charge(cfg, "orders", bucketActionSend, 1, 1).allowed) diff --git a/docs/design/2026_04_26_implemented_sqs_per_queue_throttling.md b/docs/design/2026_04_26_implemented_sqs_per_queue_throttling.md index 674d6da10..43e1cc6b0 100644 --- a/docs/design/2026_04_26_implemented_sqs_per_queue_throttling.md +++ b/docs/design/2026_04_26_implemented_sqs_per_queue_throttling.md @@ -112,9 +112,9 @@ The `charge` operation: No global lock is held during step 3; concurrent traffic on different queues runs in parallel. -**Cache invalidation on `SetQueueAttributes`**: when an operator updates the throttle config via `SetQueueAttributes`, the handler — *after* the Raft commit that persists the new `sqsQueueThrottle` — calls `buckets.invalidateQueue(name)` (the same path described in the `DeleteQueue` / `CreateQueue` paragraph below). `invalidateQueue` ranges the map and drops every entry whose `queue` matches under a lock-then-`CompareAndDelete`-then-`evicted=true` ordering; a raw per-key `buckets.Delete(key)` would reintroduce the orphan-bucket race that ordering closes (a charger holding the old pointer pre-Delete acquires the bucket's mu after the map entry is gone, sees `evicted=false`, spends a token, then later requests mint a fresh full-capacity bucket — a transient double-allotment window). Without this step at all, the in-memory bucket would keep enforcing the old limits until the idle-eviction sweep removes the stale entry (default 1 h window), defeating the operator's intent to throttle a noisy tenant in real time. The handler also gates the invalidation on a real value change — a same-value `SetQueueAttributes` does not reset the bucket — so a caller cannot bypass the rate limit by re-submitting their own current config. A P1 review finding on PR #664 caught the gap; round 9 / round 12 refined the race-free semantics and the no-op gate. +**Cache invalidation on `SetQueueAttributes`**: when an operator updates the throttle config via `SetQueueAttributes`, the handler — *after* the Raft commit that persists the new `sqsQueueThrottle` — calls `invalidateQueueBuckets(name)` (`beginQueueReset` already bumped the queue epoch before the commit) (the same path described in the `DeleteQueue` / `CreateQueue` paragraph below). `invalidateQueueBuckets` ranges the map and drops every entry whose `queue` matches under a lock-then-`CompareAndDelete`-then-`evicted=true` ordering; a raw per-key `buckets.Delete(key)` would reintroduce the orphan-bucket race that ordering closes (a charger holding the old pointer pre-Delete acquires the bucket's mu after the map entry is gone, sees `evicted=false`, spends a token, then later requests mint a fresh full-capacity bucket — a transient double-allotment window). Without this step at all, the in-memory bucket would keep enforcing the old limits until the idle-eviction sweep removes the stale entry (default 1 h window), defeating the operator's intent to throttle a noisy tenant in real time. The handler also gates the invalidation on a real value change — a same-value `SetQueueAttributes` does not reset the bucket — so a caller cannot bypass the rate limit by re-submitting their own current config. A P1 review finding on PR #664 caught the gap; round 9 / round 12 refined the race-free semantics and the no-op gate. -**Cache invalidation on `DeleteQueue` / `CreateQueue`**: when a queue is deleted, the handler — *after* the Raft commit that purges the queue meta — calls `buckets.invalidateQueue(name)`, which ranges the map and drops every `bucketKey` whose `queue` matches (regardless of incarnation), mirroring the `SetQueueAttributes` path above. The `CreateQueue` handler invokes the same call after a genuine create commit (the idempotent-return path skips it) so a same-name recreate that races with in-flight stale-meta traffic still resets the bucket. **Incarnation** participates in the key — `sqsQueueMeta.Incarnation` is set to `lastGen + 1` at `CreateQueue` time and is *preserved* across `PurgeQueue` and `SetQueueAttributes` (the read-modify-write on the meta record carries it forward). A `DeleteQueue`+`CreateQueue` cycle therefore lands the new incarnation at a different `bucketKey` and starts from a fresh full bucket regardless of any per-process cache the previous incarnation left behind on this or any other node. The two mechanisms are complementary — `invalidateQueue` is the cheap hot-path optimisation that keeps the in-memory map small, and the incarnation-keyed structure is the cross-leader correctness guarantee. +**Cache invalidation on `DeleteQueue` / `CreateQueue`**: when a queue is deleted, the handler — *after* the Raft commit that purges the queue meta — calls `invalidateQueueBuckets(name)` (`beginQueueReset` already bumped the queue epoch before the commit), which ranges the map and drops every `bucketKey` whose `queue` matches (regardless of incarnation), mirroring the `SetQueueAttributes` path above. The `CreateQueue` handler invokes the same call after a genuine create commit (the idempotent-return path skips it) so a same-name recreate that races with in-flight stale-meta traffic still resets the bucket. **Incarnation** participates in the key — `sqsQueueMeta.Incarnation` is set to `lastGen + 1` at `CreateQueue` time and is *preserved* across `PurgeQueue` and `SetQueueAttributes` (the read-modify-write on the meta record carries it forward). A `DeleteQueue`+`CreateQueue` cycle therefore lands the new incarnation at a different `bucketKey` and starts from a fresh full bucket regardless of any per-process cache the previous incarnation left behind on this or any other node. The two mechanisms are complementary — `invalidateQueueBuckets` is the cheap hot-path optimisation that keeps the in-memory map small, and the incarnation-keyed structure is the cross-leader correctness guarantee. **Why Incarnation, not Generation** (P2 review finding on PR #664 round 9): an earlier draft of this design used `sqsQueueMeta.Generation` as the bucket-key discriminator. `Generation` bumps on every `CreateQueue` *and* on every `PurgeQueue` (because message keys are prefixed with the generation, so a purge needs a new prefix to make the old data unreachable). Keying the throttle bucket by `Generation` would therefore re-key the bucket on every purge — letting any caller authorised to call `PurgeQueue` reset the rate limiter to a fresh full bucket once per purge. The 60-second AWS-spec rate limit on `PurgeQueue` bounds the bypass but does not eliminate it. `Incarnation` solves this by isolating the "queue identity changed" semantics (DeleteQueue+CreateQueue cycle) from the "data prefix changed" semantics (any of Create / Delete / Purge), and the throttle layer keys only on the former. From aefcc28b90c15f61d942e50d6e0c1c04113ea984 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 09:47:49 +0000 Subject: [PATCH 05/15] adapter: retarget Redis cleanup WP-4 tests to live paths Delete seven redundant Redis tests and redis_server_setup_test.go, which only checked the store type the test harness itself injects. Retarget the tests that guarded dead twins onto the live paths and delete the twins: HDEL through hdelTxn (persistHashTxn), EXPIRE through the *ReadTimestamp dispatchers (dispatchExpireForType, dispatchCollectionExpire, dispatchHLLExpire, dispatchStringExpire), and the EXEC read fence through commandReadFenceTargets, queuedCommandReadFenceGroupTargets and redisReadFencedTimestampForTargets (the key-only fence chain and the discarded nil-server branch). Replace three tests that never reached the code they named: the invalid UTF-8 unsupported-command test now goes through Run() over raw RESP, the dual-encoding guard is seeded directly in the store and checked on HGET, HEXISTS and SISMEMBER, and the EXEC DEL timeout test asserts that the stream-deletion scan runs on a deadline-bounded context. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- adapter/redis_apply_observer_test.go | 12 -- adapter/redis_collection_fastpath_test.go | 68 +++++----- adapter/redis_collection_ttl.go | 16 --- adapter/redis_collection_ttl_test.go | 23 +++- adapter/redis_command_test.go | 36 ------ adapter/redis_compat_commands_stream_test.go | 101 ++++++++++----- adapter/redis_expire_cmds.go | 72 +---------- adapter/redis_hash_cmds.go | 35 ----- adapter/redis_hll_ttl_test.go | 2 +- adapter/redis_lua_collection_fastpath_test.go | 80 ------------ adapter/redis_lua_pool_test.go | 26 ---- adapter/redis_metrics_test.go | 73 +++++++---- adapter/redis_server_setup_test.go | 19 --- adapter/redis_txn.go | 122 +----------------- adapter/redis_txn_test.go | 40 ++++-- 15 files changed, 209 insertions(+), 516 deletions(-) delete mode 100644 adapter/redis_server_setup_test.go diff --git a/adapter/redis_apply_observer_test.go b/adapter/redis_apply_observer_test.go index da46fcea6..ebc1010ea 100644 --- a/adapter/redis_apply_observer_test.go +++ b/adapter/redis_apply_observer_test.go @@ -76,18 +76,6 @@ func TestRedisApplyObserverZSetSignalRequiresFullRecheck(t *testing.T) { require.False(t, waiter.fastSignalAllowed(), "FSM apply zset wake must not use the BZPOPMIN fast recheck") } -func TestRedisApplyObserverPendingLocalZSetApplyStillForcesFullRecheck(t *testing.T) { - observer := NewRedisApplyObserver() - key := []byte("zset-pending-local") - waiter, release := observer.zsetWaiters.Register([][]byte{key}) - defer release() - - observer.OnApply(pb.Op_PUT, store.ZSetMemberKey(key, []byte("member"))) - - requireWaiterSignaled(t, waiter.C) - require.False(t, waiter.fastSignalAllowed(), "FSM apply zset wake must force the full BZPOPMIN recheck even during a local write") -} - func TestRedisApplyObserverSignalsLegacyZSetWaiters(t *testing.T) { observer := NewRedisApplyObserver() key := []byte("legacy-zset-key") diff --git a/adapter/redis_collection_fastpath_test.go b/adapter/redis_collection_fastpath_test.go index 80c75decf..a491f4177 100644 --- a/adapter/redis_collection_fastpath_test.go +++ b/adapter/redis_collection_fastpath_test.go @@ -6,8 +6,10 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/store" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" + "github.com/tidwall/redcon" ) // These tests pin the semantics of the wide-column fast paths for @@ -260,39 +262,43 @@ func TestRedis_SISMEMBER_TTLExpired(t *testing.T) { }, "SISMEMBER on an expired set must return 0") } -// TestRedis_HGET_FastPathGuardDualEncoding exercises the string- -// priority guard (hasHigherPriorityStringEncoding) at the HGET -// wire-protocol level: a key holding both a hash wide-column row AND -// a string must report WRONGTYPE, not the hash value. -// -// The scenario is only reachable under data-corruption recovery; we -// drive it here by overwriting a pre-existing hash with a SET, which -// replaceWithStringTxn normally cleans the hash keys for, but by the -// time SET commits the hash field row may still be observable at the -// readTS for this HGET. The test tolerates either "WRONGTYPE" or the -// old hash value being returned, since the exact race window is tiny, -// but it must NOT panic or silently leak wide-column data as a string -// hit: the guard is what keeps that outcome consistent with -// keyTypeAt semantics. -func TestRedis_HGET_FastPathGuardDualEncodingSmoke(t *testing.T) { +// TestRedis_FastPathGuardDualEncodingReportsWrongType pins the +// string-priority guard (hasHigherPriorityStringEncoding) on the HGET, +// HEXISTS and SISMEMBER fast paths: a key holding both a string and a +// hash field / set member row must report WRONGTYPE, not the collection +// value. The dual encoding is only reachable under data-corruption +// recovery (SET tombstones the collection rows), so the rows are seeded +// directly in the store. +func TestRedis_FastPathGuardDualEncodingReportsWrongType(t *testing.T) { t.Parallel() - nodes, _, _ := createNode(t, 3) - defer shutdown(nodes) ctx := context.Background() - rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) - defer func() { _ = rdb.Close() }() - - require.NoError(t, rdb.HSet(ctx, "dual:key", "field", "hashVal").Err()) - require.NoError(t, rdb.Set(ctx, "dual:key", "stringVal", 0).Err()) - - // After the SET, HGET must either see WRONGTYPE (string-priority - // guard fires, slow path reports the key as string) or nil (the - // hash keys were cleaned up). It must NOT return "hashVal". - got, err := rdb.HGet(ctx, "dual:key", "field").Result() - if err != nil { - require.Contains(t, err.Error(), "WRONGTYPE") - return + st := store.NewMVCCStore() + server := &RedisServer{store: st, coordinator: newLocalAdapterCoordinator(st)} + key := []byte("dual:key") + field := []byte("field") + member := []byte("member") + require.NoError(t, st.PutAt(ctx, redisStrKey(key), encodeRedisStr([]byte("stringVal"), nil), 10, 0)) + require.NoError(t, st.PutAt(ctx, store.HashFieldKey(key, field), []byte("hashVal"), 11, 0)) + require.NoError(t, st.PutAt(ctx, store.HashMetaKey(key), store.MarshalHashMeta(store.HashMeta{Len: 1}), 11, 0)) + require.NoError(t, st.PutAt(ctx, store.SetMemberKey(key, member), []byte{}, 12, 0)) + require.NoError(t, st.PutAt(ctx, store.SetMetaKey(key), store.MarshalSetMeta(store.SetMeta{Len: 1}), 12, 0)) + + for _, tc := range []struct { + name string + handler func(redcon.Conn, redcon.Command) + args [][]byte + }{ + {name: "hget", handler: server.hget, args: [][]byte{[]byte(cmdHGet), key, field}}, + {name: "hexists", handler: server.hexists, args: [][]byte{[]byte(cmdHExists), key, field}}, + {name: "sismember", handler: server.sismember, args: [][]byte{[]byte(cmdSIsMember), key, member}}, + } { + t.Run(tc.name, func(t *testing.T) { + conn := &commandRecorder{} + tc.handler(conn, redcon.Command{Args: tc.args}) + require.Len(t, conn.writes, 1) + require.Equal(t, "error", conn.writes[0].op, "must not return the collection value: %+v", conn.writes[0]) + require.Contains(t, conn.writes[0].s, "WRONGTYPE") + }) } - require.NotEqual(t, "hashVal", got, "fast path must not leak collection data when a string has taken over the key") } diff --git a/adapter/redis_collection_ttl.go b/adapter/redis_collection_ttl.go index 6770657df..69c573275 100644 --- a/adapter/redis_collection_ttl.go +++ b/adapter/redis_collection_ttl.go @@ -132,22 +132,6 @@ func (r *RedisServer) streamInlineTTLAt(ctx context.Context, userKey []byte, rea return ttl, true, nil } -func (r *RedisServer) dispatchCollectionExpire( - ctx context.Context, - key []byte, - readTS uint64, - typ redisValueType, - expireAt time.Time, -) (bool, error) { - ttlMs := redisExpireAtMillis(expireAt) - elems, ok, err := r.collectionExpireElems(ctx, key, readTS, typ, ttlMs) - if err != nil || !ok { - return ok, err - } - elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Put, Key: redisTTLKey(key), Value: encodeRedisTTL(expireAt)}) - return true, r.dispatchElems(ctx, true, readTS, elems) -} - func (r *RedisServer) dispatchCollectionExpireReadTimestamp( ctx context.Context, key []byte, diff --git a/adapter/redis_collection_ttl_test.go b/adapter/redis_collection_ttl_test.go index 7fb3db8a2..2080e36db 100644 --- a/adapter/redis_collection_ttl_test.go +++ b/adapter/redis_collection_ttl_test.go @@ -268,7 +268,7 @@ func TestRedisCollectionExpireHandlesLegacyBlobs(t *testing.T) { typ, err := server.rawKeyTypeAt(ctx, key, 1) require.NoError(t, err) require.Equal(t, tc.typ, typ) - applied, err := server.dispatchExpireForType(ctx, key, 1, typ, expireAt) + applied, err := server.dispatchExpireForReadTimestamp(ctx, key, testReadTimestamp(t, server, 1), typ, expireAt) require.NoError(t, err) require.True(t, applied) @@ -470,7 +470,7 @@ func TestRedisCollectionExpireAllowsDeltaHeavyCollections(t *testing.T) { } readTS := st.LastCommitTS() - applied, err := server.dispatchCollectionExpire(ctx, key, readTS, tc.typ, expireAt) + applied, err := server.dispatchCollectionExpireReadTimestamp(ctx, key, testReadTimestamp(t, server, readTS), tc.typ, expireAt) require.NoError(t, err) require.True(t, applied) @@ -545,7 +545,7 @@ func TestRedisCollectionExpireTriggersCompactionForDeltaOnlyTruncatedCollections require.NoError(t, st.PutAt(ctx, tc.deltaKey(key, ts), tc.deltaValue, ts, 0)) } - applied, err := server.dispatchCollectionExpire(ctx, key, st.LastCommitTS(), tc.typ, expireAt) + applied, err := server.dispatchCollectionExpireReadTimestamp(ctx, key, testReadTimestamp(t, server, st.LastCommitTS()), tc.typ, expireAt) require.ErrorIs(t, err, ErrDeltaScanTruncated) require.False(t, applied) _, err = st.GetAt(ctx, redisTTLKey(key), st.LastCommitTS()) @@ -623,7 +623,7 @@ func TestRedisCollectionExpireWritesInlineTTLForZeroLengthSimpleMeta(t *testing. key := []byte("ttl:zero-len:" + tc.name) require.NoError(t, st.PutAt(ctx, tc.metaKey(key), tc.metaValue, 1, 0)) - applied, err := server.dispatchCollectionExpire(ctx, key, st.LastCommitTS(), tc.typ, expireAt) + applied, err := server.dispatchCollectionExpireReadTimestamp(ctx, key, testReadTimestamp(t, server, st.LastCommitTS()), tc.typ, expireAt) require.NoError(t, err) require.True(t, applied) @@ -693,7 +693,7 @@ func TestRedisCollectionExpireDoesNotReviveDeltaEmptiedSimpleMeta(t *testing.T) require.NoError(t, st.PutAt(ctx, tc.metaKey(key), tc.metaValue, 1, 0)) require.NoError(t, st.PutAt(ctx, tc.deltaKey(key), tc.deltaValue, 2, 0)) - applied, err := server.dispatchCollectionExpire(ctx, key, st.LastCommitTS(), tc.typ, expireAt) + applied, err := server.dispatchCollectionExpireReadTimestamp(ctx, key, testReadTimestamp(t, server, st.LastCommitTS()), tc.typ, expireAt) require.NoError(t, err) require.False(t, applied) @@ -1591,6 +1591,15 @@ func requireExistsAt(t *testing.T, st store.MVCCStore, key []byte, readTS uint64 require.True(t, exists) } +// testReadTimestamp wraps readTS in the kv.ReadTimestamp the live +// *ReadTimestamp dispatch paths require. +func testReadTimestamp(t *testing.T, server *RedisServer, readTS uint64) kv.ReadTimestamp { + t.Helper() + readTimestamp, err := kv.BeginReadTimestampThrough(context.Background(), server.coordinator, readTS, "test") + require.NoError(t, err) + return readTimestamp +} + func requireNoTTLAt(t *testing.T, server *RedisServer, st store.MVCCStore, key []byte, readTS uint64) { t.Helper() ttl, err := server.ttlAt(context.Background(), key, readTS) @@ -1612,7 +1621,9 @@ func TestRedisHDelLegacyRewritePreservesLegacyTTL(t *testing.T) { require.NoError(t, st.PutAt(ctx, redisHashKey(key), raw, 10, 0)) require.NoError(t, st.PutAt(ctx, redisTTLKey(key), encodeRedisTTL(expireAt), 11, 0)) - require.NoError(t, server.persistHashTxn(ctx, key, st.LastCommitTS(), redisHashValue{"keep": "value"})) + removed, err := server.hdelTxn(ctx, key, [][]byte{[]byte("drop")}) + require.NoError(t, err) + require.Equal(t, 1, removed) readTS := st.LastCommitTS() metaRaw, err := st.GetAt(ctx, store.HashMetaKey(key), readTS) diff --git a/adapter/redis_command_test.go b/adapter/redis_command_test.go index 6fc732945..9469dad91 100644 --- a/adapter/redis_command_test.go +++ b/adapter/redis_command_test.go @@ -379,24 +379,6 @@ func TestCommand_GetKeysAndFlagsRejected(t *testing.T) { require.Contains(t, conn.writes[0].s, "unsupported") } -func TestCommand_RoutedViaServer(t *testing.T) { - t.Parallel() - // Verify the route wiring: a COMMAND request reaching RedisServer.route - // dispatches to r.command. We cannot exercise redcon.Serve in a unit - // test without a network listener, so we instead reach into the route - // map as the live server would. - srv := &RedisServer{} - srv.route = map[string]func(conn redcon.Conn, cmd redcon.Command){ - cmdCommand: srv.command, - } - handler, ok := srv.route[cmdCommand] - require.True(t, ok, "COMMAND must be routed") - conn := &commandRecorder{} - handler(conn, redcon.Command{Args: [][]byte{[]byte("COMMAND"), []byte("COUNT")}}) - require.Len(t, conn.writes, 1) - require.Equal(t, "int", conn.writes[0].op) -} - // TestCommand_ArgsLenAllowsBare ensures the validateCmd arity check // accepts bare `COMMAND` (no subcommand), matching real Redis where // `COMMAND` with no args is the canonical "dump all" call. @@ -405,24 +387,6 @@ func TestCommand_ArgsLenAllowsBare(t *testing.T) { require.Equal(t, -1, argsLen[cmdCommand]) } -// TestCommand_RouteMatchesTable asserts that every command currently -// routed has a metadata row in redisCommandTable. Catches the common -// mistake of adding a new handler without extending the table — the -// runtime path would fall back to zero-metadata + a log warning, but we -// want hard failure in CI. -func TestCommand_RouteMatchesTable(t *testing.T) { - t.Parallel() - srv := &RedisServer{} - // Rebuild the route map via NewRedisServer would need a listener; we - // instead iterate argsLen (which every routed command has an entry - // in — see redis.go) as the closest proxy. - for name := range argsLen { - _, ok := redisCommandTable[name] - require.Truef(t, ok, "command %q routed but missing from redisCommandTable", name) - } - _ = srv -} - // TestRedisCommandSpecs_NoDuplicateConstants guards the structural // invariant the spec list maintains: every Constant appears exactly // once. Without this, the derived argsLen / redisCommandTable maps diff --git a/adapter/redis_compat_commands_stream_test.go b/adapter/redis_compat_commands_stream_test.go index 832990fb6..c1acc7c88 100644 --- a/adapter/redis_compat_commands_stream_test.go +++ b/adapter/redis_compat_commands_stream_test.go @@ -1,9 +1,11 @@ package adapter import ( + "bytes" "context" "fmt" "sort" + "sync" "testing" "time" @@ -11,6 +13,7 @@ import ( "github.com/cockroachdb/errors" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" + "github.com/tidwall/redcon" ) // TestRedis_StreamXReadIterCtxDeadlineReturnsNull guards Codex P1 (round 6): @@ -954,42 +957,82 @@ func TestRedis_StreamXReadAndRangeCountClamp(t *testing.T) { } // TestRedis_StreamMultiExecDelHonoursDispatchTimeout regression-tests the -// CodeRabbit "buildStreamDeletionElems on server-lifetime ctx" concern. -// The ctx threaded through buildStreamDeletionElems is the same -// redisDispatchTimeout-bounded ctx used for Dispatch, so the EXEC of a -// MULTI/EXEC DEL on a stream key keeps the existing behaviour: the -// deletion is observable on a fresh XLEN, and the EXEC does not hang -// past the per-request budget. The test is a smoke-level guard; the -// actual ctx-cancellation path is documented in commit(). +// CodeRabbit "buildStreamDeletionElems on server-lifetime ctx" concern: +// the stream-entry scan that a MULTI/EXEC DEL performs inside commit() +// must run on the redisDispatchTimeout-bounded EXEC context, not on a +// context without a deadline, so a stalled store cannot hold EXEC past +// the per-request budget. func TestRedis_StreamMultiExecDelHonoursDispatchTimeout(t *testing.T) { t.Parallel() - nodes, _, _ := createNode(t, 3) - defer shutdown(nodes) - rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) - defer func() { _ = rdb.Close() }() - ctx := context.Background() + for _, tc := range []struct { + name string + dedup bool + }{ + {name: "direct", dedup: false}, + {name: "dedup", dedup: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() - for i := range 4 { - _, err := rdb.XAdd(ctx, &redis.XAddArgs{ - Stream: "stream-del", - ID: fmt.Sprintf("%d-0", 1_000_000+i), - Values: []string{"i", fmt.Sprint(i)}, - }).Result() - require.NoError(t, err) + ctx := context.Background() + key := []byte("stream-del:" + tc.name) + base := store.NewMVCCStore() + meta, err := store.MarshalStreamMeta(store.StreamMeta{Length: 2, LastMs: 2}) + require.NoError(t, err) + require.NoError(t, base.PutAt(ctx, store.StreamMetaKey(key), meta, 1, 0)) + require.NoError(t, base.PutAt(ctx, store.StreamEntryKey(key, 1, 0), []byte("a"), 1, 0)) + require.NoError(t, base.PutAt(ctx, store.StreamEntryKey(key, 2, 0), []byte("b"), 1, 0)) + + st := &scanDeadlineRecordingStore{MVCCStore: base, prefix: store.StreamEntryScanPrefix(key)} + server := &RedisServer{ + store: st, + coordinator: newLocalAdapterCoordinator(base), + scriptCache: map[string]string{}, + onePhaseTxnDedup: tc.dedup, + } + + _, err = server.runTransaction([]redcon.Command{{Args: [][]byte{[]byte(cmdDel), key}}}) + require.NoError(t, err) + + deadlines := st.recorded() + require.NotEmpty(t, deadlines, "EXEC DEL must scan the stream entries") + for _, remaining := range deadlines { + require.Positive(t, remaining, "stream deletion scan ran without a dispatch deadline") + require.LessOrEqual(t, remaining, redisDispatchTimeout) + } + }) } +} - // MULTI / EXEC DEL inside a txn drives the streamDeletions path - // inside commit() — that is the path which now passes the - // redisDispatchTimeout-bounded ctx into buildStreamDeletionElems. - pipe := rdb.TxPipeline() - pipe.Del(ctx, "stream-del") - _, err := pipe.Exec(ctx) - require.NoError(t, err) +// scanDeadlineRecordingStore records, for every ScanAt under prefix, how +// long the caller's context had left (0 when it had no deadline). +type scanDeadlineRecordingStore struct { + store.MVCCStore + prefix []byte - xlen, err := rdb.XLen(ctx, "stream-del").Result() - require.NoError(t, err) - require.Equal(t, int64(0), xlen) + mu sync.Mutex + deadlines []time.Duration +} + +func (s *scanDeadlineRecordingStore) ScanAt(ctx context.Context, start []byte, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { + if bytes.HasPrefix(start, s.prefix) { + var remaining time.Duration + if deadline, ok := ctx.Deadline(); ok { + remaining = time.Until(deadline) + } + s.mu.Lock() + s.deadlines = append(s.deadlines, remaining) + s.mu.Unlock() + } + kvs, err := s.MVCCStore.ScanAt(ctx, start, end, limit, ts) + return kvs, errors.WithStack(err) +} + +func (s *scanDeadlineRecordingStore) recorded() []time.Duration { + s.mu.Lock() + defer s.mu.Unlock() + return append([]time.Duration(nil), s.deadlines...) } // TestRedis_UserKeyShadowingStreamPrefixSurvivesMultiExec exercises the diff --git a/adapter/redis_expire_cmds.go b/adapter/redis_expire_cmds.go index 4e094ec66..297fb0cc6 100644 --- a/adapter/redis_expire_cmds.go +++ b/adapter/redis_expire_cmds.go @@ -293,33 +293,6 @@ func (r *RedisServer) doSetExpire(ctx context.Context, key []byte, ttl int64, ex return 1, nil } -func (r *RedisServer) dispatchExpireForType( - ctx context.Context, - key []byte, - readTS uint64, - typ redisValueType, - expireAt time.Time, -) (bool, error) { - if typ == redisTypeString { - // rawKeyTypeAt also reports HLL as redisTypeString; HLL payloads live - // under !redis|hll|, so distinguish them from plain strings - // before rewriting the TTL-bearing anchor. - plain, err := r.isPlainRedisString(ctx, key, readTS) - if err != nil { - return false, err - } - if plain { - return r.dispatchStringExpire(ctx, key, readTS, expireAt) - } - return r.dispatchHLLExpire(ctx, key, readTS, expireAt) - } - if isNonStringCollectionType(typ) { - return r.dispatchCollectionExpire(ctx, key, readTS, typ, expireAt) - } - elems := []*kv.Elem[kv.OP]{{Op: kv.Put, Key: redisTTLKey(key), Value: encodeRedisTTL(expireAt)}} - return true, r.dispatchElems(ctx, true, readTS, elems) -} - func (r *RedisServer) dispatchExpireForReadTimestamp( ctx context.Context, key []byte, @@ -380,30 +353,12 @@ func (r *RedisServer) expireDeleteKeyReadTimestamp(ctx context.Context, key []by return 0, nil } -// dispatchStringExpire performs a read-modify-write on the string anchor key: +// dispatchStringExpireReadTimestamp performs a read-modify-write on the string anchor key: // it reads the current value at readTS, re-encodes it with the new expiry, and // writes both the updated value and the !redis|ttl| scan index in a single Raft -// entry (IsTxn=true, StartTS=readTS). The coordinator rejects the write with +// entry (IsTxn=true, StartTS=readTimestamp). The coordinator rejects the write with // ErrWriteConflict if any key was modified after readTS, so stale-data safety is // guaranteed by OCC — no explicit mutex is required. -func (r *RedisServer) dispatchStringExpire(ctx context.Context, key []byte, readTS uint64, expireAt time.Time) (bool, error) { - userValue, _, readErr := r.readRedisStringAt(key, readTS) - if readErr != nil { - if cockerrors.Is(readErr, store.ErrKeyNotFound) { - // Raced with a delete/expiry between prepareExpire and this read; - // do not resurrect the key with an empty anchor. - return false, nil - } - return false, cockerrors.WithStack(readErr) - } - encoded := encodeRedisStr(userValue, &expireAt) - elems := []*kv.Elem[kv.OP]{ - {Op: kv.Put, Key: redisStrKey(key), Value: encoded}, - {Op: kv.Put, Key: redisTTLKey(key), Value: encodeRedisTTL(expireAt)}, - } - return true, r.dispatchElems(ctx, true, readTS, elems) -} - func (r *RedisServer) dispatchStringExpireReadTimestamp(ctx context.Context, key []byte, readTimestamp kv.ReadTimestamp, expireAt time.Time) (bool, error) { readTS := readTimestamp.Timestamp() userValue, _, readErr := r.readRedisStringAt(key, readTS) @@ -421,29 +376,6 @@ func (r *RedisServer) dispatchStringExpireReadTimestamp(ctx context.Context, key return true, r.dispatchReadTimestampElems(ctx, readTimestamp, elems) } -func (r *RedisServer) dispatchHLLExpire(ctx context.Context, key []byte, readTS uint64, expireAt time.Time) (bool, error) { - raw, err := r.store.GetAt(ctx, redisHLLKey(key), readTS) - if err != nil { - if cockerrors.Is(err, store.ErrKeyNotFound) { - return false, nil - } - return false, cockerrors.WithStack(err) - } - value, _, _, err := decodeRedisHLL(raw) - if err != nil { - return false, err - } - encoded, err := encodeRedisHLL(value, &expireAt) - if err != nil { - return false, err - } - elems := []*kv.Elem[kv.OP]{ - {Op: kv.Put, Key: redisHLLKey(key), Value: encoded}, - {Op: kv.Put, Key: redisTTLKey(key), Value: encodeRedisTTL(expireAt)}, - } - return true, r.dispatchElems(ctx, true, readTS, elems) -} - func (r *RedisServer) dispatchHLLExpireReadTimestamp(ctx context.Context, key []byte, readTimestamp kv.ReadTimestamp, expireAt time.Time) (bool, error) { readTS := readTimestamp.Timestamp() raw, err := r.store.GetAt(ctx, redisHLLKey(key), readTS) diff --git a/adapter/redis_hash_cmds.go b/adapter/redis_hash_cmds.go index df7d8eb9a..122a0528c 100644 --- a/adapter/redis_hash_cmds.go +++ b/adapter/redis_hash_cmds.go @@ -529,41 +529,6 @@ func removeHashFields(value redisHashValue, fields [][]byte) int { return removed } -func (r *RedisServer) persistHashTxn(ctx context.Context, key []byte, readTS uint64, value redisHashValue) error { - if len(value) == 0 { - elems, _, err := r.deleteLogicalKeyElems(ctx, key, readTS) - if err != nil { - return err - } - return r.dispatchElems(ctx, true, readTS, elems) - } - ttlMs, expired, err := legacyTTLMillisForMigrationAt(ctx, r.store, key, readTS) - if err != nil { - return err - } - if expired { - return r.dispatchElems(ctx, true, readTS, legacyExpiredCollectionCleanupElems(key, redisHashKey(key))) - } - // Wide-column rewrite: write per-field keys and a new base meta. - // deleteLogicalKeyElems (called by the caller when needed) clears old keys. - elems := make([]*kv.Elem[kv.OP], 0, len(value)+1) - for field, val := range value { - elems = append(elems, &kv.Elem[kv.OP]{ - Op: kv.Put, - Key: store.HashFieldKey(key, []byte(field)), - Value: []byte(val), - }) - } - elems = append(elems, &kv.Elem[kv.OP]{ - Op: kv.Put, - Key: store.HashMetaKey(key), - Value: store.MarshalHashMeta(store.HashMeta{Len: int64(len(value)), ExpireAt: ttlMs}), - }) - // Also remove the legacy blob if it was present. - elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: redisHashKey(key)}) - return r.dispatchElems(ctx, true, readTS, elems) -} - func (r *RedisServer) persistHashReadTimestampTxn(ctx context.Context, key []byte, readTimestamp kv.ReadTimestamp, value redisHashValue) error { readTS := readTimestamp.Timestamp() if len(value) == 0 { diff --git a/adapter/redis_hll_ttl_test.go b/adapter/redis_hll_ttl_test.go index 1516f94ab..61b1ef5a8 100644 --- a/adapter/redis_hll_ttl_test.go +++ b/adapter/redis_hll_ttl_test.go @@ -185,7 +185,7 @@ func TestRedisDispatchHLLExpireWritesInlineAnchorAndScanIndex(t *testing.T) { require.NoError(t, st.PutAt(ctx, redisHLLKey(key), legacyPayload, 1, 0)) expireAt := time.Now().Add(time.Hour) - applied, err := server.dispatchHLLExpire(ctx, key, 10, expireAt) + applied, err := server.dispatchHLLExpireReadTimestamp(ctx, key, testReadTimestamp(t, server, 10), expireAt) require.NoError(t, err) require.True(t, applied) diff --git a/adapter/redis_lua_collection_fastpath_test.go b/adapter/redis_lua_collection_fastpath_test.go index b2e98b793..f72bff155 100644 --- a/adapter/redis_lua_collection_fastpath_test.go +++ b/adapter/redis_lua_collection_fastpath_test.go @@ -956,37 +956,6 @@ func TestLua_ZRANGEBYSCORE_TruncationFallsBackToSlowPath(t *testing.T) { require.Equal(t, []any{"m1", "m2", "m3", "m4", "m5"}, got) } -// TestLua_ZRANGEBYSCORE_DeltaOnlyZSetEmptyRange pins the -// resolveZSetMeta fix: a zset that only has delta metadata (no -// persisted base meta) must be recognised as existing when the -// fast path produces an empty range. Without the fix, -// zsetRangeEmptyFastResult returned hit=false and forced the slow -// path on every ZRANGEBYSCORE on a fresh zset. -// -// We can't easily construct a delta-only zset from the wire (the -// adapter writes base meta on ZADD commit), but the -// resolveZSetMeta codepath exists for exactly this scenario. This -// test at least pins that a freshly-created zset with empty range -// returns empty (not WRONGTYPE / error). -func TestLua_ZRANGEBYSCORE_FreshZSetEmptyRange(t *testing.T) { - t.Parallel() - nodes, _, _ := createNode(t, 3) - defer shutdown(nodes) - - ctx := context.Background() - rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) - defer func() { _ = rdb.Close() }() - - require.NoError(t, rdb.ZAdd(ctx, "lua:zr:fresh", redis.Z{Score: 5, Member: "x"}).Err()) - - got, err := rdb.Eval(ctx, - `return redis.call("ZRANGEBYSCORE", KEYS[1], "10", "20")`, - []string{"lua:zr:fresh"}, - ).Result() - require.NoError(t, err) - require.Equal(t, []any{}, got) -} - // TestLua_ZRANGEBYSCORE_NegativeOffsetRejected pins the defensive // offset / limit bounds check. func TestLua_ZRANGEBYSCORE_NegativeOffsetRejected(t *testing.T) { @@ -1116,55 +1085,6 @@ func TestLua_ZRANGEBYSCORE_PosInfExactMatch(t *testing.T) { "ZRANGEBYSCORE +inf +inf must return only members with score = +inf") } -// TestLua_ZRANGEBYSCORE_MissingKeyReturnsEmptyViaFastPath pins the -// empty-result short-circuit for a key that does not exist at all. -// Redis returns an empty array for ZRANGEBYSCORE on a missing key; -// the fast path must serve that directly instead of punting to the -// slow path, which was the dominant call pattern on BullMQ-style -// workloads polling empty delayed queues. -func TestLua_ZRANGEBYSCORE_MissingKeyReturnsEmptyViaFastPath(t *testing.T) { - t.Parallel() - nodes, _, _ := createNode(t, 3) - defer shutdown(nodes) - - ctx := context.Background() - rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) - defer func() { _ = rdb.Close() }() - - got, err := rdb.Eval(ctx, - `return redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", "+inf")`, - []string{"lua:zr:absent"}, - ).Result() - require.NoError(t, err) - require.Equal(t, []any{}, got, - "ZRANGEBYSCORE on a missing key must return empty array") -} - -// TestLua_ZRANGEBYSCORE_StringKeyReturnsWrongType verifies that when -// the key exists as a string (not a zset), ZRANGEBYSCORE still yields -// WRONGTYPE. The fast-path short-circuit must only short-circuit when -// the key is truly absent; a wrong-type existence must surface the -// error via the slow path. -func TestLua_ZRANGEBYSCORE_StringKeyReturnsWrongType(t *testing.T) { - t.Parallel() - nodes, _, _ := createNode(t, 3) - defer shutdown(nodes) - - ctx := context.Background() - rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) - defer func() { _ = rdb.Close() }() - - require.NoError(t, rdb.Set(ctx, "lua:zr:wrongtype", "notzset", 0).Err()) - - _, err := rdb.Eval(ctx, - `return redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", "+inf")`, - []string{"lua:zr:wrongtype"}, - ).Result() - require.Error(t, err, "ZRANGEBYSCORE on a string key must return an error") - require.Contains(t, err.Error(), "WRONGTYPE", - "error must be WRONGTYPE, not the fast-path empty-array short-circuit") -} - // TestLua_ZRANGEBYSCORE_ListKeyReturnsWrongType mirrors the string // test for list-typed keys, covering the non-string collection arm // of keyTypeAt in the wrong-type short-circuit. diff --git a/adapter/redis_lua_pool_test.go b/adapter/redis_lua_pool_test.go index c94b207b4..1f14fbad4 100644 --- a/adapter/redis_lua_pool_test.go +++ b/adapter/redis_lua_pool_test.go @@ -140,32 +140,6 @@ func TestLua_VMReuseRestoresRebindsWhitelistedGlobals(t *testing.T) { require.NoError(t, plsB.state.DoString(`assert(string.upper("abc") == "ABC", "string.upper was poisoned")`)) } -// TestLua_PoolSerialAcquireReusesState verifies the pool serves -// existing *lua.LState instances in sequential acquire/release cycles -// — the knob we care about for the heap-pressure win. The channel -// pool is deterministic on a single goroutine, but the test runs -// under t.Parallel, so we assert via the hit counter rather than -// pointer identity (a sibling test could in principle pre-empt -// between get and put). -func TestLua_PoolSerialAcquireReusesState(t *testing.T) { - t.Parallel() - - pool := newLuaStatePool() - - // Prime the pool so the first Get allocates. - pool.put(pool.get(nil)) - - const iters = 50 - for i := 0; i < iters; i++ { - pls := pool.get(nil) - pool.put(pls) - } - // At least one hit proves the pool is actually handing back an - // existing VM rather than minting a new one every time. - require.GreaterOrEqual(t, pool.Hits(), uint64(1), - "pool never reported a hit; channel-pool reuse not happening") -} - // TestLua_PoolRecordsReuseVsAllocation pins down the "is the pool // actually doing anything?" question via the hit/miss counters. The // test guards against a subtle regression where get() auto-fills on diff --git a/adapter/redis_metrics_test.go b/adapter/redis_metrics_test.go index 6c13ff196..c4967e5f2 100644 --- a/adapter/redis_metrics_test.go +++ b/adapter/redis_metrics_test.go @@ -1,6 +1,7 @@ package adapter import ( + "bufio" "fmt" "io" "net" @@ -11,6 +12,7 @@ import ( "unicode/utf8" "github.com/bootjp/elastickv/monitoring" + "github.com/bootjp/elastickv/store" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" "github.com/tidwall/redcon" @@ -507,49 +509,68 @@ func TestRedisMetricsUnsupportedCommandConcurrentObservers(t *testing.T) { require.Equal(t, float64(workers*perWorks), total) } -// TestRedisMetricsUnsupportedCommandRawInvalidUTF8FromAdapterPath covers -// the regression fixed alongside this commit: the adapter's Run() loop -// must pass RAW command bytes (not strings.ToUpper'd bytes) to the -// unsupported-command observer. ToUpper silently rewrites invalid UTF-8 -// into the U+FFFD replacement character, so if the raw path is not -// preserved the invalid_utf8 sentinel is never reached and a hostile -// client can burn through the distinct-name slots with replacement-char -// garbage. Feeding the observer directly with three invalid bytes must -// produce the sentinel, not a synthetic "valid UTF-8" label. +// TestRedisMetricsUnsupportedCommandRawInvalidUTF8Bytes covers the +// regression fixed in db4fa305: the adapter's Run() loop must pass RAW +// command bytes (not strings.ToUpper'd bytes) to the unsupported-command +// observer. ToUpper silently rewrites invalid UTF-8 into the U+FFFD +// replacement character, so if the raw path is not preserved the +// invalid_utf8 sentinel is never reached and a hostile client can burn +// through the distinct-name slots with replacement-char garbage. The +// command is sent as raw RESP so it goes through the real Run() call site. func TestRedisMetricsUnsupportedCommandRawInvalidUTF8Bytes(t *testing.T) { registry := monitoring.NewRegistry("n1", "10.0.0.1:50051") - observer := registry.RedisObserver() + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + server := NewRedisServer(l, "", store.NewMVCCStore(), nil, nil, nil, + WithRedisRequestObserver(registry.RedisObserver())) + runDone := make(chan struct{}) + go func() { + defer close(runDone) + _ = server.Run() + }() + t.Cleanup(func() { + server.Stop() + <-runDone + }) - raw := string([]byte{0xff, 0xff, 0xff}) - require.False(t, utf8.ValidString(raw), "precondition: bytes must be invalid UTF-8") + raw := []byte{0xff, 0xff, 0xff} + require.False(t, utf8.Valid(raw), "precondition: bytes must be invalid UTF-8") - require.NotPanics(t, func() { - observer.ObserveRedisRequest(monitoring.RedisRequestReport{ - Command: raw, - IsError: true, - Duration: time.Millisecond, - Unsupported: true, - }) - }) + conn, err := net.DialTimeout("tcp", l.Addr().String(), time.Second) + require.NoError(t, err) + defer conn.Close() + require.NoError(t, conn.SetDeadline(time.Now().Add(2*time.Second))) + _, err = conn.Write(append(append([]byte("*1\r\n$3\r\n"), raw...), "\r\n"...)) + require.NoError(t, err) + reply, err := bufio.NewReader(conn).ReadString('\n') + require.NoError(t, err) + require.True(t, strings.HasPrefix(reply, "-ERR unsupported command"), "reply = %q", reply) + + // Run() writes the error reply before it records the metric. + require.Eventually(t, func() bool { + return unsupportedCommandCount(t, registry, "invalid_utf8") == 1 + }, 2*time.Second, 10*time.Millisecond, + "raw invalid-UTF-8 bytes must land in the 'invalid_utf8' sentinel") +} +func unsupportedCommandCount(t *testing.T, registry *monitoring.Registry, command string) float64 { + t.Helper() mf, err := registry.Gatherer().Gather() require.NoError(t, err) - - var sentinel float64 + var total float64 for _, family := range mf { if family.GetName() != "elastickv_redis_unsupported_commands_total" { continue } for _, metric := range family.GetMetric() { for _, lbl := range metric.GetLabel() { - if lbl.GetName() == "command" && lbl.GetValue() == "invalid_utf8" { - sentinel += metric.GetCounter().GetValue() + if lbl.GetName() == "command" && lbl.GetValue() == command { + total += metric.GetCounter().GetValue() } } } } - require.Equal(t, float64(1), sentinel, - "raw invalid-UTF-8 bytes must land in the 'invalid_utf8' sentinel") + return total } // TestRedisMetricsUnsupportedCommandRejectsPathologicalLength defends diff --git a/adapter/redis_server_setup_test.go b/adapter/redis_server_setup_test.go deleted file mode 100644 index ecbca30bb..000000000 --- a/adapter/redis_server_setup_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package adapter - -import ( - "testing" - - "github.com/bootjp/elastickv/kv" - "github.com/stretchr/testify/require" -) - -func TestRedisServerUsesLeaderRoutedStoreInSingleGroupCluster(t *testing.T) { - t.Parallel() - - nodes, _, _ := createNode(t, 3) - defer shutdown(nodes) - - require.IsType(t, &kv.LeaderRoutedStore{}, nodes[0].redisServer.store) - require.IsType(t, &kv.LeaderRoutedStore{}, nodes[1].redisServer.store) - require.IsType(t, &kv.LeaderRoutedStore{}, nodes[2].redisServer.store) -} diff --git a/adapter/redis_txn.go b/adapter/redis_txn.go index 42b2d319d..ffe13067c 100644 --- a/adapter/redis_txn.go +++ b/adapter/redis_txn.go @@ -77,23 +77,6 @@ var txnApplyHandlers = map[string]txnCommandHandler{ cmdPExpire: (*txnContext).applyExpireMilliseconds, } -func redisTxnReadFenceKeys(userKey []byte) [][]byte { - keys := append([][]byte{}, redisTxnReadFencePointKeys(userKey)...) - keys = append(keys, - store.ListMetaDeltaScanPrefix(userKey), - store.ListClaimScanPrefix(userKey), - store.HashFieldScanPrefix(userKey), - store.HashMetaDeltaScanPrefix(userKey), - store.SetMemberScanPrefix(userKey), - store.SetMetaDeltaScanPrefix(userKey), - store.ZSetMemberScanPrefix(userKey), - store.ZSetScoreScanPrefix(userKey), - store.ZSetMetaDeltaScanPrefix(userKey), - store.StreamEntryScanPrefix(userKey), - ) - return keys -} - func redisTxnReadFencePointKeys(userKey []byte) [][]byte { keys := [][]byte{ redisStrKey(userKey), @@ -127,10 +110,6 @@ func redisLegacyBareReadFenceAllowed(userKey []byte) bool { return !bytes.HasPrefix(userKey, []byte("!sqs|")) } -func (r *RedisServer) redisTxnReadFenceKeysForRanges(userKey []byte, ranges []redisReadFenceRange) [][]byte { - return readFenceTargetKeys(r.redisTxnReadFenceTargetsForRanges(userKey, ranges)) -} - // redisTxnReadFenceTargetsForRanges builds the fence set for a user key. Point // keys carry no group id and are resolved from bytes; range-derived targets keep // the group the range expansion already resolved. @@ -149,14 +128,6 @@ func (r *RedisServer) redisTxnReadFenceTargetsForRanges(userKey []byte, ranges [ return targets } -func readFenceTargetKeys(targets []kv.ReadFenceTarget) [][]byte { - keys := make([][]byte, 0, len(targets)) - for _, target := range targets { - keys = append(keys, target.Key) - } - return keys -} - func redisTxnReadFenceRanges(userKey []byte) []redisReadFenceRange { ranges := redisListReadFenceRanges(userKey) ranges = append(ranges, redisHashReadFenceRanges(userKey)...) @@ -275,50 +246,6 @@ func (v redisReadFenceRouteVersion) observedRouteVersion() uint64 { return kv.EncodeObservedRouteVersion(v.version) } -func redisQueuedCommandReadFenceKeys(queue []redcon.Command) [][]byte { - return redisQueuedCommandReadFenceKeysForServer(nil, queue) -} - -func redisQueuedCommandReadFenceKeysForServer(r *RedisServer, queue []redcon.Command) [][]byte { - seen := make(map[string]struct{}, len(queue)) - keys := make([][]byte, 0, len(queue)) - appendKey := func(key []byte) { - keyID := string(key) - if _, ok := seen[keyID]; ok { - return - } - seen[keyID] = struct{}{} - keys = append(keys, key) - } - for _, cmd := range queue { - for _, fenceKey := range redisCommandReadFenceKeysForServer(r, cmd) { - appendKey(fenceKey) - } - } - return keys -} - -func redisCommandReadFenceKeysForServer(r *RedisServer, cmd redcon.Command) [][]byte { - if len(cmd.Args) == 0 { - return nil - } - cmdName := strings.ToUpper(string(cmd.Args[0])) - meta, ok := redisCommandTable[cmdName] - if !ok { - return nil - } - keys := make([][]byte, 0) - for _, userKey := range redisCommandGetKeys(meta, cmd.Args) { - if r == nil { - keys = append(keys, redisTxnReadFenceKeys(userKey)...) - continue - } - keys = append(keys, r.redisTxnReadFenceKeysForRanges(userKey, redisCommandReadFenceRanges(cmdName, userKey))...) - } - keys = append(keys, redisCommandExactReadFenceKeys(cmd)...) - return keys -} - func redisCommandReadFenceRanges(cmdName string, userKey []byte) []redisReadFenceRange { switch cmdName { case cmdGet, cmdExists: @@ -379,14 +306,9 @@ func (r *RedisServer) redisReadFenceGroupTargets(targets []kv.ReadFenceTarget) [ return kv.LeaseReadGroupTargets(r.coordinator, targets) } -func (r *RedisServer) queuedCommandReadFenceGroupKeys(queue []redcon.Command) [][]byte { - return readFenceTargetKeys(r.queuedCommandReadFenceGroupTargets(queue)) -} - // queuedCommandReadFenceGroupTargets collapses the queue's fence set to one -// target per group, keeping each range-derived group id intact. It mirrors -// redisQueuedCommandReadFenceKeysForServer exactly, including its per-key dedup, -// and differs only in carrying the group id alongside each key. +// target per group, keeping each range-derived group id intact. Targets are +// deduplicated by key before the per-group collapse. func (r *RedisServer) queuedCommandReadFenceGroupTargets(queue []redcon.Command) []kv.ReadFenceTarget { seen := make(map[string]struct{}, len(queue)) targets := make([]kv.ReadFenceTarget, 0, len(queue)) @@ -406,8 +328,9 @@ func (r *RedisServer) queuedCommandReadFenceGroupTargets(queue []redcon.Command) return r.redisReadFenceGroupTargets(targets) } -// commandReadFenceTargets is the target-carrying form of -// redisCommandReadFenceKeysForServer. +// commandReadFenceTargets returns the read-fence targets for one queued +// command: the type-detection point keys and command-relevant ranges of every +// key it names, plus its exact hash-field or zset-member keys. func (r *RedisServer) commandReadFenceTargets(cmd redcon.Command) []kv.ReadFenceTarget { if len(cmd.Args) == 0 { return nil @@ -419,12 +342,6 @@ func (r *RedisServer) commandReadFenceTargets(cmd redcon.Command) []kv.ReadFence } targets := make([]kv.ReadFenceTarget, 0) for _, userKey := range redisCommandGetKeys(meta, cmd.Args) { - if r == nil { - for _, key := range redisTxnReadFenceKeys(userKey) { - targets = append(targets, kv.ReadFenceTarget{Key: key}) - } - continue - } targets = append(targets, r.redisTxnReadFenceTargetsForRanges(userKey, redisCommandReadFenceRanges(cmdName, userKey))...) } for _, key := range redisCommandExactReadFenceKeys(cmd) { @@ -592,35 +509,6 @@ func (r *RedisServer) leaseRedisReadFenceTargets(ctx context.Context, targets [] return nil } -func (r *RedisServer) redisReadFencedTimestamp( - ctx context.Context, - groupKeys [][]byte, - selectTS func() uint64, -) (uint64, *kv.ActiveTimestampToken, error) { - readTimestamp, readPin, err := r.redisReadFencedReadTimestamp( - ctx, groupKeys, selectTS, "redis read fence: begin read timestamp") - if err != nil { - return 0, nil, err - } - return readTimestamp.Timestamp(), readPin, nil -} - -// redisReadFencedReadTimestamp fences by key. Keys carry no group of their own, -// so each becomes a bare ReadFenceTarget and the fence resolves it the way it -// always did. -func (r *RedisServer) redisReadFencedReadTimestamp( - ctx context.Context, - groupKeys [][]byte, - selectTS func() uint64, - label string, -) (kv.ReadTimestamp, *kv.ActiveTimestampToken, error) { - targets := make([]kv.ReadFenceTarget, 0, len(groupKeys)) - for _, key := range groupKeys { - targets = append(targets, kv.ReadFenceTarget{Key: key}) - } - return r.redisReadFencedTimestampForTargets(ctx, targets, selectTS, label) -} - // redisReadFencedTimestampForTargets leases the fence, begins the read // timestamp through the coordinator, pins it, and leases again -- the second // lease is what catches a leadership change between the fence and the pin. diff --git a/adapter/redis_txn_test.go b/adapter/redis_txn_test.go index ff52aa534..a4b17da16 100644 --- a/adapter/redis_txn_test.go +++ b/adapter/redis_txn_test.go @@ -628,7 +628,7 @@ func TestRedisExecReadFenceUsesRangeRoutesAndExactHashFields(t *testing.T) { } server := &RedisServer{store: st, coordinator: coord} - got := server.queuedCommandReadFenceGroupKeys([]redcon.Command{{ + got := queuedCommandReadFenceGroupKeysForTest(server, []redcon.Command{{ Args: [][]byte{[]byte(cmdHSet), key, field, []byte("value")}, }}) @@ -707,7 +707,7 @@ func TestRedisExecReadFenceUsesOnlyCommandRelevantRanges(t *testing.T) { } server := &RedisServer{store: st, coordinator: coord} - getKeys := server.queuedCommandReadFenceGroupKeys([]redcon.Command{{ + getKeys := queuedCommandReadFenceGroupKeysForTest(server, []redcon.Command{{ Args: [][]byte{[]byte(cmdGet), key}, }}) require.ElementsMatch(t, [][]byte{ @@ -721,7 +721,7 @@ func TestRedisExecReadFenceUsesOnlyCommandRelevantRanges(t *testing.T) { zsetDeltaRoute, }, getKeys) - existsKeys := server.queuedCommandReadFenceGroupKeys([]redcon.Command{{ + existsKeys := queuedCommandReadFenceGroupKeysForTest(server, []redcon.Command{{ Args: [][]byte{[]byte(cmdExists), key, otherKey}, }}) require.ElementsMatch(t, [][]byte{ @@ -736,7 +736,7 @@ func TestRedisExecReadFenceUsesOnlyCommandRelevantRanges(t *testing.T) { otherListTypeRoute, }, existsKeys) - lrangeKeys := server.queuedCommandReadFenceGroupKeys([]redcon.Command{{ + lrangeKeys := queuedCommandReadFenceGroupKeysForTest(server, []redcon.Command{{ Args: [][]byte{[]byte(cmdLRange), key, []byte("0"), []byte("-1")}, }}) require.ElementsMatch(t, [][]byte{ @@ -751,7 +751,7 @@ func TestRedisExecReadFenceUsesOnlyCommandRelevantRanges(t *testing.T) { zsetDeltaRoute, }, lrangeKeys) - rpushKeys := server.queuedCommandReadFenceGroupKeys([]redcon.Command{{ + rpushKeys := queuedCommandReadFenceGroupKeysForTest(server, []redcon.Command{{ Args: [][]byte{[]byte(cmdRPush), key, []byte("value")}, }}) require.ElementsMatch(t, [][]byte{ @@ -766,7 +766,7 @@ func TestRedisExecReadFenceUsesOnlyCommandRelevantRanges(t *testing.T) { zsetDeltaRoute, }, rpushKeys) - zincrbyKeys := server.queuedCommandReadFenceGroupKeys([]redcon.Command{{ + zincrbyKeys := queuedCommandReadFenceGroupKeysForTest(server, []redcon.Command{{ Args: [][]byte{[]byte(cmdZIncrBy), key, []byte("1"), member}, }}) require.ElementsMatch(t, [][]byte{ @@ -782,7 +782,7 @@ func TestRedisExecReadFenceUsesOnlyCommandRelevantRanges(t *testing.T) { exactZSetMemberKey, }, zincrbyKeys) - hsetKeys := server.queuedCommandReadFenceGroupKeys([]redcon.Command{{ + hsetKeys := queuedCommandReadFenceGroupKeysForTest(server, []redcon.Command{{ Args: [][]byte{[]byte(cmdHSet), key, field, []byte("value")}, }}) require.ElementsMatch(t, [][]byte{ @@ -814,15 +814,16 @@ func TestRedisReadFencedTimestampLeasesBeforeAndAfterSelectingTimestamp(t *testi } server := &RedisServer{store: st, coordinator: coord} - startTS, readPin, err := server.redisReadFencedTimestamp( + readTimestamp, readPin, err := server.redisReadFencedTimestampForTargets( context.Background(), - [][]byte{redisStrKey([]byte("txn:fence-order"))}, + []kv.ReadFenceTarget{{Key: redisStrKey([]byte("txn:fence-order"))}}, server.txnStartTS, + "test", ) defer readPin.Release() require.NoError(t, err) - require.Equal(t, uint64(1), startTS) + require.Equal(t, uint64(1), readTimestamp.Timestamp()) coord.mu.Lock() leaseCalls := coord.leaseCalls coord.mu.Unlock() @@ -1198,9 +1199,10 @@ func TestRedisExecReadFenceUsesRedisStorageKeys(t *testing.T) { t.Parallel() key := []byte("!sqs|user-visible") - keys := redisQueuedCommandReadFenceKeys([]redcon.Command{{ + server := &RedisServer{store: store.NewMVCCStore()} + keys := readFenceTargetKeys(server.commandReadFenceTargets(redcon.Command{ Args: [][]byte{[]byte(cmdGet), key}, - }}) + })) keySet := make(map[string]struct{}, len(keys)) for _, got := range keys { keySet[string(got)] = struct{}{} @@ -2514,3 +2516,17 @@ func TestTxnStartTSUsesLastCommitTS(t *testing.T) { require.Equal(t, appliedTS, startTS, "txnStartTS must equal store.LastCommitTS() to prevent lost-write anomaly") } + +// queuedCommandReadFenceGroupKeysForTest returns the keys of the per-group +// fence targets EXEC leases for queue. +func queuedCommandReadFenceGroupKeysForTest(server *RedisServer, queue []redcon.Command) [][]byte { + return readFenceTargetKeys(server.queuedCommandReadFenceGroupTargets(queue)) +} + +func readFenceTargetKeys(targets []kv.ReadFenceTarget) [][]byte { + keys := make([][]byte, 0, len(targets)) + for _, target := range targets { + keys = append(keys, target.Key) + } + return keys +} From d4131a40a3ff60dbf1f92595197c6d6016ef4ac7 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 09:49:45 +0000 Subject: [PATCH 06/15] docs: promote test suite cleanup from proposed to partial WP-0 through WP-8 have landed; WP-9 to WP-12 (migrations onto the shared test-support packages) remain. Record the three places where the implementation departed from the plan and the dead DynamoDB lease branch follow-up. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- ...16_implemented_production_commonization.md | 2 +- ... 2026_10_16_partial_test_suite_cleanup.md} | 25 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) rename docs/design/{2026_10_16_proposed_test_suite_cleanup.md => 2026_10_16_partial_test_suite_cleanup.md} (96%) diff --git a/docs/design/2026_10_16_implemented_production_commonization.md b/docs/design/2026_10_16_implemented_production_commonization.md index 799cc1761..cdb5aefa7 100644 --- a/docs/design/2026_10_16_implemented_production_commonization.md +++ b/docs/design/2026_10_16_implemented_production_commonization.md @@ -6,7 +6,7 @@ Date: 2026-10-16 ## Context -The 2026-10-16 code audit (see `2026_10_16_proposed_test_suite_cleanup.md` for the test-side findings) found that most production duplication is small. The valuable cases are the ones where two hand-maintained copies of the same logic have already drifted apart and changed behaviour. This document covers five of them. Each milestone below fixes the drift by making both call sites use one implementation, and pins the result with tests that fail on the current code where a behaviour defect exists. +The 2026-10-16 code audit (see `2026_10_16_partial_test_suite_cleanup.md` for the test-side findings) found that most production duplication is small. The valuable cases are the ones where two hand-maintained copies of the same logic have already drifted apart and changed behaviour. This document covers five of them. Each milestone below fixes the drift by making both call sites use one implementation, and pins the result with tests that fail on the current code where a behaviour defect exists. | # | Duplication | Drift already visible | |---|---|---| diff --git a/docs/design/2026_10_16_proposed_test_suite_cleanup.md b/docs/design/2026_10_16_partial_test_suite_cleanup.md similarity index 96% rename from docs/design/2026_10_16_proposed_test_suite_cleanup.md rename to docs/design/2026_10_16_partial_test_suite_cleanup.md index fadb94438..0ab3b6aea 100644 --- a/docs/design/2026_10_16_proposed_test_suite_cleanup.md +++ b/docs/design/2026_10_16_partial_test_suite_cleanup.md @@ -1,6 +1,6 @@ # Test suite cleanup: redundant tests, live-path retargets, and shared test support -Status: Proposed +Status: Partial Author: bootjp Date: 2026-10-16 @@ -87,6 +87,27 @@ Problems 3 and 5 are fixed by the PR that adds this document (WP-0). The rest ar WP-1 to WP-8 can run in parallel. WP-9 to WP-12 start once their dependencies merge, and can then run in parallel with each other. +## Progress + +| WP | State | +|---|---| +| WP-0 | Landed with this document (#1277). | +| WP-1 | Landed (#1345). | +| WP-2 | Landed (#1292). | +| WP-3 | Landed (follow-up PR after #1345). | +| WP-4 | Landed (same PR as WP-3); see deviation 3. | +| WP-5 | Landed (same PR as WP-3); see deviation 1. | +| WP-6 | Landed (same PR as WP-3); see deviation 2. | +| WP-7 | Landed (same PR as WP-3). | +| WP-8 | Landed (#1288). | +| WP-9 to WP-12 | Open. Their dependencies have all landed. | + +Deviations from the plan, found while implementing: + +1. **WP-5 retarget 1 (DynamoDB base-table hash-key mismatch) cannot be written.** `queryLeasePrefix` is only reached from `queryLeaseKey` with an empty `IndexName`, so `keySchemaForQuery` always returns `PrimaryKey` and the `keySchema.HashKey != schema.PrimaryKey.HashKey` branch is unreachable. A base-table key condition that does not pin the hash key fails `buildQueryCondition` and is classified as a skip. The duplicate test was deleted; removing the dead branch is a production change and is listed under Follow-ups. +2. **WP-6 retarget 3 (pprof/metrics).** The live `PreparePprofServer`/`PrepareMetricsServer` moved to `internal/serverwiring`, whose own tests already pin the address-before-token validation. The root wrappers and their tests were deleted rather than retargeted. +3. **WP-4 retarget 6 (EXEC dispatch timeout).** `redisDispatchTimeout` is a 30 s constant, so a blocking-store test would take 30 s or need a production change. The rewritten test records the deadline on the stream-deletion scan instead. The mutation named in the plan (`context.Background()` for the EXEC dispatch) survives because `txnContext.dispatchContext` bounds the commit context a second time. Removing that second bound (`dispatchContext` returning an unbounded context) fails the test. + --- ### WP-0: Keep test code out of production; run FUSE tests in CI (done) @@ -594,6 +615,8 @@ Record each mutation in the PR as a row: file and line, the change, the test tha ## Follow-ups (out of scope) +- **Dead DynamoDB lease branch.** Remove the unreachable hash-key-mismatch branch in `queryLeasePrefix` (`adapter/dynamodb_lease.go`), or document why it is kept (deviation 1 above). + - **Product bugs found by the audit.** Each needs its own PR with a failing test first. - SQS FIFO messages in one group sent within the same millisecond can be received out of order. The visibility key orders by `(visibleAtMillis, random messageID)`. `TestSQSServer_FifoGroupLockHoldsAcrossVisibilityExpiry` and `TestSQSServer_FifoSequenceNumberMonotonic` fail roughly 7/30 and 2/30 runs without `-race`. - The Redis TTL-inline migration builds zero scan plans in production. `startupGatedCoordinator` does not forward `LocalLeaderGroupIDs`, and `keyVizLabeledCoordinator` returns nil when the inner value lacks it. Regression since `a36dac7f`. From 4f633d643e0fd4be45f0f2cd135a23057b114f7e Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 09:49:45 +0000 Subject: [PATCH 07/15] docs: promote shipped process and shutdown designs to implemented SIGTERM leadership transfer (#1291), DCO sign-off (#1306), the filesystem and SQLite Jepsen history budgets (#1321, #1333), Jepsen artifact archives, and the review approval label series (#1328, #1332, #1336, #1340) are all on main; their remaining live-verification milestones are met by successful runs on main. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- ...2026_10_22_implemented_sigterm_leadership_transfer.md} | 4 +++- ...o_signoff.md => 2026_10_24_implemented_dco_signoff.md} | 4 +++- ...10_24_implemented_filesystem_jepsen_history_budget.md} | 1 + ...md => 2026_10_24_implemented_review_approval_label.md} | 4 ++-- ...=> 2026_10_25_implemented_jepsen_artifact_archives.md} | 8 ++++---- ...026_10_25_implemented_review_label_inline_findings.md} | 4 +++- ...=> 2026_10_25_implemented_review_label_permissions.md} | 4 ++-- ...026_10_25_implemented_sqlite_jepsen_history_budget.md} | 1 + ...6_10_26_implemented_review_label_resolved_findings.md} | 4 +++- 9 files changed, 22 insertions(+), 12 deletions(-) rename docs/design/{2026_10_22_proposed_sigterm_leadership_transfer.md => 2026_10_22_implemented_sigterm_leadership_transfer.md} (97%) rename docs/design/{2026_10_24_proposed_dco_signoff.md => 2026_10_24_implemented_dco_signoff.md} (96%) rename docs/design/{2026_10_24_proposed_filesystem_jepsen_history_budget.md => 2026_10_24_implemented_filesystem_jepsen_history_budget.md} (96%) rename docs/design/{2026_10_24_proposed_review_approval_label.md => 2026_10_24_implemented_review_approval_label.md} (97%) rename docs/design/{2026_10_25_proposed_jepsen_artifact_archives.md => 2026_10_25_implemented_jepsen_artifact_archives.md} (73%) rename docs/design/{2026_10_25_proposed_review_label_inline_findings.md => 2026_10_25_implemented_review_label_inline_findings.md} (96%) rename docs/design/{2026_10_25_proposed_review_label_permissions.md => 2026_10_25_implemented_review_label_permissions.md} (90%) rename docs/design/{2026_10_25_proposed_sqlite_jepsen_history_budget.md => 2026_10_25_implemented_sqlite_jepsen_history_budget.md} (94%) rename docs/design/{2026_10_26_proposed_review_label_resolved_findings.md => 2026_10_26_implemented_review_label_resolved_findings.md} (97%) diff --git a/docs/design/2026_10_22_proposed_sigterm_leadership_transfer.md b/docs/design/2026_10_22_implemented_sigterm_leadership_transfer.md similarity index 97% rename from docs/design/2026_10_22_proposed_sigterm_leadership_transfer.md rename to docs/design/2026_10_22_implemented_sigterm_leadership_transfer.md index 95f530579..19334625b 100644 --- a/docs/design/2026_10_22_proposed_sigterm_leadership_transfer.md +++ b/docs/design/2026_10_22_implemented_sigterm_leadership_transfer.md @@ -1,6 +1,8 @@ # SIGTERM leadership transfer -Status: proposed; implementation and regression tests accompany this change. +Status: Implemented + +Shipped in #1291 (`main_shutdown.go`, wired into `run` and `startRaftServers`). No implementation work follows. Issue: https://github.com/bootjp/elastickv/issues/1284 ## Goal diff --git a/docs/design/2026_10_24_proposed_dco_signoff.md b/docs/design/2026_10_24_implemented_dco_signoff.md similarity index 96% rename from docs/design/2026_10_24_proposed_dco_signoff.md rename to docs/design/2026_10_24_implemented_dco_signoff.md index 27e5e2109..a3889acbe 100644 --- a/docs/design/2026_10_24_proposed_dco_signoff.md +++ b/docs/design/2026_10_24_implemented_dco_signoff.md @@ -1,6 +1,8 @@ # DCO sign-off checks -Status: Proposed +Status: Implemented + +Shipped in #1306 (`DCO`, `.github/dco.yml`, Renovate `:gitSignOff`). No implementation work follows. Author: bootjp Date: 2026-10-24 diff --git a/docs/design/2026_10_24_proposed_filesystem_jepsen_history_budget.md b/docs/design/2026_10_24_implemented_filesystem_jepsen_history_budget.md similarity index 96% rename from docs/design/2026_10_24_proposed_filesystem_jepsen_history_budget.md rename to docs/design/2026_10_24_implemented_filesystem_jepsen_history_budget.md index b66300ed5..129d12de5 100644 --- a/docs/design/2026_10_24_proposed_filesystem_jepsen_history_budget.md +++ b/docs/design/2026_10_24_implemented_filesystem_jepsen_history_budget.md @@ -1,5 +1,6 @@ # Bound filesystem Jepsen histories by write mode +Status: Implemented (shipped in #1321; no implementation work follows) Date: 2026-10-24 ## Failure diff --git a/docs/design/2026_10_24_proposed_review_approval_label.md b/docs/design/2026_10_24_implemented_review_approval_label.md similarity index 97% rename from docs/design/2026_10_24_proposed_review_approval_label.md rename to docs/design/2026_10_24_implemented_review_approval_label.md index 367f9ec42..1ca91a41d 100644 --- a/docs/design/2026_10_24_proposed_review_approval_label.md +++ b/docs/design/2026_10_24_implemented_review_approval_label.md @@ -1,6 +1,6 @@ # Review approval label -Status: Proposed +Status: Implemented Author: bootjp Date: 2026-10-24 @@ -84,7 +84,7 @@ label preservation, and HEAD changes around label writes. - [x] Add the privileged updater and read-only event/test workflow. - [x] Include SHA-bound review results in Claude responses. - [x] Validate local regression tests and workflow syntax. -- [ ] Merge to the default branch and verify a live reconciliation. +- [x] Merge to the default branch (#1328) and verify a live reconciliation (the `Update review approval label` workflow runs successfully on `main`, e.g. run 40666978702). ## Validation and self-review diff --git a/docs/design/2026_10_25_proposed_jepsen_artifact_archives.md b/docs/design/2026_10_25_implemented_jepsen_artifact_archives.md similarity index 73% rename from docs/design/2026_10_25_proposed_jepsen_artifact_archives.md rename to docs/design/2026_10_25_implemented_jepsen_artifact_archives.md index 556000ad7..cfa9e7b5f 100644 --- a/docs/design/2026_10_25_proposed_jepsen_artifact_archives.md +++ b/docs/design/2026_10_25_implemented_jepsen_artifact_archives.md @@ -1,6 +1,6 @@ # Jepsen artifact archives -Status: Proposed +Status: Implemented Author: bootjp Date: 2026-10-25 @@ -24,9 +24,9 @@ consistency criterion, storage implementation, or timeout is relaxed. ## Milestones and validation -- [ ] Add a regression covering colon-containing directories and preserved bytes. -- [ ] Package all directly uploaded Jepsen store paths in both CI workflows. -- [ ] Pass archive regression, actionlint, and encrypted/cleartext Jepsen CI. +- [x] Add a regression covering colon-containing directories and preserved bytes (`scripts/test-archive-jepsen-store.sh`). +- [x] Package all directly uploaded Jepsen store paths in both CI workflows (`scripts/archive-jepsen-store.sh` in `jepsen-test.yml` and `jepsen-test-scheduled.yml`). +- [x] Pass archive regression, actionlint, and encrypted/cleartext Jepsen CI (Jepsen Test is green on `main`, e.g. run 40657220651). The earlier fixture change reproduces the original failure and passes all 294 tests locally and in CI. This follow-up addresses the next failure revealed by diff --git a/docs/design/2026_10_25_proposed_review_label_inline_findings.md b/docs/design/2026_10_25_implemented_review_label_inline_findings.md similarity index 96% rename from docs/design/2026_10_25_proposed_review_label_inline_findings.md rename to docs/design/2026_10_25_implemented_review_label_inline_findings.md index d182a6648..88e53f290 100644 --- a/docs/design/2026_10_25_proposed_review_label_inline_findings.md +++ b/docs/design/2026_10_25_implemented_review_label_inline_findings.md @@ -1,6 +1,8 @@ # Review label inline findings -Status: Proposed +Status: Implemented + +Shipped in #1336. No implementation work follows. Author: bootjp Date: 2026-10-25 diff --git a/docs/design/2026_10_25_proposed_review_label_permissions.md b/docs/design/2026_10_25_implemented_review_label_permissions.md similarity index 90% rename from docs/design/2026_10_25_proposed_review_label_permissions.md rename to docs/design/2026_10_25_implemented_review_label_permissions.md index 587c419ad..51a9ca567 100644 --- a/docs/design/2026_10_25_proposed_review_label_permissions.md +++ b/docs/design/2026_10_25_implemented_review_label_permissions.md @@ -1,6 +1,6 @@ # Review label permissions -Status: Proposed +Status: Implemented Author: bootjp Date: 2026-10-25 @@ -23,7 +23,7 @@ maintainer's authenticated CLI, preserving all HEAD/evidence rechecks. - [x] Validate workflow syntax and the existing reconciliation suite. - [x] Verify #1331's label through the same reconciliation helper. -- [ ] Merge the permission change and verify a live Actions write. +- [x] Merge the permission change (#1332) and verify a live Actions write (`pull-requests: write` is in `review-approval-label.yml`; the updater runs successfully on `main`, e.g. run 40666978702). The live Actions failure is the reproduction: POST issues/1331/labels returns 403 despite a completed clean review. Workflow syntax checks cannot establish diff --git a/docs/design/2026_10_25_proposed_sqlite_jepsen_history_budget.md b/docs/design/2026_10_25_implemented_sqlite_jepsen_history_budget.md similarity index 94% rename from docs/design/2026_10_25_proposed_sqlite_jepsen_history_budget.md rename to docs/design/2026_10_25_implemented_sqlite_jepsen_history_budget.md index 3a1154ece..04f5ba6ad 100644 --- a/docs/design/2026_10_25_proposed_sqlite_jepsen_history_budget.md +++ b/docs/design/2026_10_25_implemented_sqlite_jepsen_history_budget.md @@ -1,5 +1,6 @@ # Bound SQLite Jepsen register histories +Status: Implemented (shipped in #1333; no implementation work follows) Date: 2026-10-25 ## Failure diff --git a/docs/design/2026_10_26_proposed_review_label_resolved_findings.md b/docs/design/2026_10_26_implemented_review_label_resolved_findings.md similarity index 97% rename from docs/design/2026_10_26_proposed_review_label_resolved_findings.md rename to docs/design/2026_10_26_implemented_review_label_resolved_findings.md index 70dee81f7..dbbd4a0e9 100644 --- a/docs/design/2026_10_26_proposed_review_label_resolved_findings.md +++ b/docs/design/2026_10_26_implemented_review_label_resolved_findings.md @@ -1,6 +1,8 @@ # Resolved review findings -Status: Proposed +Status: Implemented + +Shipped in #1340. No implementation work follows. Author: bootjp Date: 2026-10-26 From ba7b7f23976722ae760af39eb0a6978b5f116663 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 09:57:34 +0000 Subject: [PATCH 08/15] test: build legacy list delta keys with store.LegacyListMetaDeltaKey (cleanup WP-9..12) Replace the hand-rolled legacyListMetaDeltaKey copies in adapter, kv and distribution with store.LegacyListMetaDeltaKey, keeping each package's existing sequence number (adapter 0, kv 1), and derive the delta-looking user keys from the same helper. The internal/backup mirror stays independent on purpose. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- adapter/internal_migration_test.go | 2 +- adapter/redis_collection_ttl_test.go | 4 +-- adapter/redis_delta_compactor_test.go | 20 ++++++------ adapter/redis_list_dedup_test.go | 37 +++++------------------ distribution/migrator_export_plan_test.go | 29 +++--------------- kv/shard_key_test.go | 12 -------- kv/shard_store_test.go | 8 ++--- store/list_helpers_test.go | 14 ++------- 8 files changed, 31 insertions(+), 95 deletions(-) diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index 4e573e961..631c2885a 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -188,7 +188,7 @@ func TestInternalExportRangeVersionsUsesValueAwareLegacyListDeltaRouteFilter(t * ctx := context.Background() st := store.NewMVCCStore() userKey := []byte("target-list") - key := legacyListMetaDeltaKey(userKey, 10) + key := store.LegacyListMetaDeltaKey(userKey, 10, 0) value := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) require.NoError(t, st.PutAt(ctx, key, value, 10, 0)) internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(st)) diff --git a/adapter/redis_collection_ttl_test.go b/adapter/redis_collection_ttl_test.go index 2080e36db..b2c12545e 100644 --- a/adapter/redis_collection_ttl_test.go +++ b/adapter/redis_collection_ttl_test.go @@ -165,7 +165,7 @@ func TestListMetaExpireElemsIncludesLegacyDeltas(t *testing.T) { st := store.NewMVCCStore() server := &RedisServer{store: st} key := []byte("ttl:list:legacy-delta") - deltaKey := legacyListMetaDeltaKey(key, 2) + deltaKey := store.LegacyListMetaDeltaKey(key, 2, 0) require.NoError(t, st.PutAt(ctx, deltaKey, store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}), 2, 0)) elems, exists, err := server.listMetaExpireElems(ctx, key, 2, 1234) @@ -186,7 +186,7 @@ func TestExpiredTTLIndexPrecedesLegacyListDeltaOnlyCollection(t *testing.T) { server := &RedisServer{store: st} key := []byte("ttl:list:legacy-delta-recreate") require.NoError(t, st.PutAt(ctx, redisTTLKey(key), encodeRedisTTL(time.Now().Add(-time.Hour)), 1, 0)) - require.NoError(t, st.PutAt(ctx, legacyListMetaDeltaKey(key, 3), store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}), 3, 0)) + require.NoError(t, st.PutAt(ctx, store.LegacyListMetaDeltaKey(key, 3, 0), store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}), 3, 0)) stale, err := expiredTTLIndexPrecedesDeltaOnlyCollection(ctx, st, server, key, redisTypeList, 3) diff --git a/adapter/redis_delta_compactor_test.go b/adapter/redis_delta_compactor_test.go index ed4899265..399b60f66 100644 --- a/adapter/redis_delta_compactor_test.go +++ b/adapter/redis_delta_compactor_test.go @@ -141,7 +141,7 @@ func TestIsListMetaMigrationDeltaRecognizesLegacyRowsByValue(t *testing.T) { userKey := []byte("list") delta := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) - legacyKey := legacyListMetaDeltaKey(userKey, 2) + legacyKey := store.LegacyListMetaDeltaKey(userKey, 2, 0) require.True(t, isListMetaMigrationDelta(&store.KVPair{Key: legacyKey, Value: delta})) require.True(t, isListMetaMigrationDelta(&store.KVPair{ @@ -1343,7 +1343,7 @@ func TestDeltaCompactor_DoesNotInheritStaleTTLBeforeDeltaOnlyRecreate(t *testing seed: func(t *testing.T, st store.MVCCStore, key []byte) { t.Helper() require.NoError(t, st.PutAt(ctx, store.ListItemKey(key, 0), []byte("a"), 3, 0)) - deltaKey := legacyListMetaDeltaKey(key, 3) + deltaKey := store.LegacyListMetaDeltaKey(key, 3, 0) require.NoError(t, st.PutAt(ctx, deltaKey, store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}), 3, 0)) }, build: func(t *testing.T, c *DeltaCompactor, st store.MVCCStore, key []byte, readTS uint64) []*kv.Elem[kv.OP] { @@ -1724,8 +1724,8 @@ func TestDeltaCompactor_LegacyListDeltaFoldedIntoBaseMeta(t *testing.T) { require.NoError(t, st.PutAt(ctx, store.ListMetaKey(userKey), metaBytes, 1, 0)) delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: -1, LenDelta: 2}) - d1Key := legacyListMetaDeltaKey(userKey, 10) - d2Key := legacyListMetaDeltaKey(userKey, 11) + d1Key := store.LegacyListMetaDeltaKey(userKey, 10, 0) + d2Key := store.LegacyListMetaDeltaKey(userKey, 11, 0) require.NoError(t, st.PutAt(ctx, d1Key, delta, 10, 0)) require.NoError(t, st.PutAt(ctx, d2Key, delta, 11, 0)) @@ -1963,7 +1963,7 @@ func TestDeltaCompactor_LegacyListCursorAdvancesPastFilteredTailAfterBacktrack(t ctx := context.Background() userKey := []byte("compactor-tail") delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) - acceptedKey := legacyListMetaDeltaKey(userKey, 1) + acceptedKey := store.LegacyListMetaDeltaKey(userKey, 1, 0) require.NoError(t, st.PutAt(ctx, acceptedKey, delta, 1, 0)) meta, err := store.MarshalListMeta(store.ListMeta{Head: 4, Tail: 6, Len: 2}) @@ -1998,7 +1998,7 @@ func TestDeltaCompactor_LegacyListCursorAdvancesPastRejectedPrefixBeforeAccepted require.NoError(t, st.PutAt(ctx, store.ListMetaKey(collidingUserKey), meta, i, 0)) } delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) - acceptedKey := legacyListMetaDeltaKey(userKey, deltaCompactorTickScanLimit) + acceptedKey := store.LegacyListMetaDeltaKey(userKey, deltaCompactorTickScanLimit, 0) require.NoError(t, st.PutAt(ctx, acceptedKey, delta, deltaCompactorTickScanLimit, 0)) h := c.legacyListHandler() @@ -2018,8 +2018,8 @@ func TestDeltaCompactor_CursorBacktracksBeforeWholeAcceptedTail(t *testing.T) { userKey := []byte("compactor-accepted-tail") delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) - d1 := legacyListMetaDeltaKey(userKey, 1) - d2 := legacyListMetaDeltaKey(userKey, 2) + d1 := store.LegacyListMetaDeltaKey(userKey, 1, 0) + d2 := store.LegacyListMetaDeltaKey(userKey, 2, 0) rawKVs := []*store.KVPair{ {Key: d1, Value: delta}, {Key: d2, Value: delta}, @@ -2036,8 +2036,8 @@ func TestDeltaCompactor_ListDeltaDeletesPreserveScanRouteGroup(t *testing.T) { ctx := context.Background() userKey := []byte("compactor-route-group") delta := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) - d1 := legacyListMetaDeltaKey(userKey, 1) - d2 := legacyListMetaDeltaKey(userKey, 2) + d1 := store.LegacyListMetaDeltaKey(userKey, 1, 0) + d2 := store.LegacyListMetaDeltaKey(userKey, 2, 0) elems, err := c.buildListCompactElems(ctx, userKey, []*store.KVPair{ {Key: d1, Value: delta, RouteGroupID: 7}, diff --git a/adapter/redis_list_dedup_test.go b/adapter/redis_list_dedup_test.go index ea3e4c6ec..b7618d309 100644 --- a/adapter/redis_list_dedup_test.go +++ b/adapter/redis_list_dedup_test.go @@ -3,7 +3,6 @@ package adapter import ( "bytes" "context" - "encoding/binary" "errors" "sync/atomic" "testing" @@ -140,7 +139,7 @@ func TestResolveListMetaReadsLegacyDeltaPrefix(t *testing.T) { require.NoError(t, err) require.NoError(t, st.PutAt(ctx, store.ListMetaKey(key), base, 1, 0)) delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: -1, LenDelta: 3}) - require.NoError(t, st.PutAt(ctx, legacyListMetaDeltaKey(key, 2), delta, 2, 0)) + require.NoError(t, st.PutAt(ctx, store.LegacyListMetaDeltaKey(key, 2, 0), delta, 2, 0)) meta, exists, err := srv.resolveListMeta(ctx, key, 3) require.NoError(t, err) @@ -162,7 +161,7 @@ func TestResolveListMetaEnforcesDeltaLimitAcrossCurrentAndLegacyPrefixes(t *test require.NoError(t, st.PutAt(ctx, store.ListMetaDeltaKey(key, i, 0), delta, i, 0)) } legacyTS := uint64(store.MaxDeltaScanLimit + 1) - require.NoError(t, st.PutAt(ctx, legacyListMetaDeltaKey(key, legacyTS), delta, legacyTS, 0)) + require.NoError(t, st.PutAt(ctx, store.LegacyListMetaDeltaKey(key, legacyTS, 0), delta, legacyTS, 0)) _, _, err := srv.resolveListMeta(ctx, key, legacyTS) require.ErrorIs(t, err, ErrDeltaScanTruncated) @@ -223,7 +222,7 @@ func TestResolveListMetaCountsOnlyAcceptedLegacyDeltasForTruncation(t *testing.T } delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) deltaTS := uint64(store.MaxDeltaScanLimit + 2) - require.NoError(t, st.PutAt(ctx, legacyListMetaDeltaKey(key, deltaTS), delta, deltaTS, 0)) + require.NoError(t, st.PutAt(ctx, store.LegacyListMetaDeltaKey(key, deltaTS, 0), delta, deltaTS, 0)) meta, exists, err := srv.resolveListMeta(ctx, key, deltaTS+1) require.NoError(t, err) @@ -239,7 +238,7 @@ func TestProbeListTypeReadsLegacyDeltaPrefix(t *testing.T) { srv := &RedisServer{store: st} key := []byte("legacy-list-type") delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) - require.NoError(t, st.PutAt(ctx, legacyListMetaDeltaKey(key, 2), delta, 2, 0)) + require.NoError(t, st.PutAt(ctx, store.LegacyListMetaDeltaKey(key, 2, 0), delta, 2, 0)) typ, found, err := srv.probeListType(ctx, key, 3) require.NoError(t, err) @@ -279,7 +278,7 @@ func TestProbeListTypePagesPastLegacyDeltaPrefixCollisions(t *testing.T) { } deltaTS := uint64(store.MaxDeltaScanLimit + 1) delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) - require.NoError(t, st.PutAt(ctx, legacyListMetaDeltaKey(key, deltaTS), delta, deltaTS, 0)) + require.NoError(t, st.PutAt(ctx, store.LegacyListMetaDeltaKey(key, deltaTS, 0), delta, deltaTS, 0)) typ, found, err := srv.probeListType(ctx, key, deltaTS+1) require.NoError(t, err) @@ -299,7 +298,7 @@ func TestDeleteListElemsFiltersLegacyDeltaPrefixCollisions(t *testing.T) { require.NoError(t, err) collidingMetaKey := store.ListMetaKey(collidingUserKey) require.NoError(t, st.PutAt(ctx, collidingMetaKey, base, 1, 0)) - deltaKey := legacyListMetaDeltaKey(key, 2) + deltaKey := store.LegacyListMetaDeltaKey(key, 2, 0) delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) require.NoError(t, st.PutAt(ctx, deltaKey, delta, 2, 0)) @@ -315,33 +314,13 @@ func TestDeleteListElemsFiltersLegacyDeltaPrefixCollisions(t *testing.T) { require.NotContains(t, deleted, string(collidingMetaKey)) } -func legacyListMetaDeltaKey(userKey []byte, commitTS uint64) []byte { - key := store.LegacyListMetaDeltaScanPrefix(userKey) - var ts [8]byte - binary.BigEndian.PutUint64(ts[:], commitTS) - key = append(key, ts[:]...) - var seq [4]byte - binary.BigEndian.PutUint32(seq[:], 0) - return append(key, seq[:]...) -} - func deltaLookingListMetaUserKey(fakeUserKey []byte) []byte { return deltaLookingListMetaUserKeyAt(fakeUserKey, 7, 1) } func deltaLookingListMetaUserKeyAt(fakeUserKey []byte, commitTS uint64, seqInTxn uint32) []byte { - key := make([]byte, 0, len("d|")+4+len(fakeUserKey)+8+4) - key = append(key, "d|"...) - var lenPrefix [4]byte - binary.BigEndian.PutUint32(lenPrefix[:], uint32(len(fakeUserKey))) //nolint:gosec // test data is small. - key = append(key, lenPrefix[:]...) - key = append(key, fakeUserKey...) - var ts [8]byte - binary.BigEndian.PutUint64(ts[:], commitTS) - key = append(key, ts[:]...) - var seq [4]byte - binary.BigEndian.PutUint32(seq[:], seqInTxn) - return append(key, seq[:]...) + // ListMetaKey of the returned user key reads as a legacy list delta key. + return bytes.TrimPrefix(store.LegacyListMetaDeltaKey(fakeUserKey, commitTS, seqInTxn), []byte(store.ListMetaPrefix)) } func (c *dedupTestCoordinator) Dispatch(ctx context.Context, req *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { diff --git a/distribution/migrator_export_plan_test.go b/distribution/migrator_export_plan_test.go index b55f3989b..184f749ae 100644 --- a/distribution/migrator_export_plan_test.go +++ b/distribution/migrator_export_plan_test.go @@ -2,7 +2,6 @@ package distribution import ( "bytes" - "encoding/binary" "testing" "github.com/bootjp/elastickv/internal/fskeys" @@ -108,7 +107,7 @@ func TestPlanMigrationBracketsDisjointPrefixContainment(t *testing.T) { require.False(t, byFamily[MigrationFamilyListMeta].ContainsRawKey(listDelta)) require.False(t, byFamily[MigrationFamilyLegacyListMetaDelta].ContainsRawKey(listDelta)) - legacyListDelta := legacyListMetaDeltaKey([]byte("legacy-list"), 2, 0) + legacyListDelta := store.LegacyListMetaDeltaKey([]byte("legacy-list"), 2, 0) require.True(t, byFamily[MigrationFamilyLegacyListMetaDelta].ContainsRawKey(legacyListDelta)) require.False(t, byFamily[MigrationFamilyListMeta].ContainsRawKey(legacyListDelta)) require.False(t, byFamily[MigrationFamilyListMetaDelta].ContainsRawKey(legacyListDelta)) @@ -359,7 +358,7 @@ func TestMigrationBracketContainsRoutedKeyUsesLegacyListDeltaUserKey(t *testing. brackets, err := PlanMigrationBrackets([]byte("a"), []byte("z")) require.NoError(t, err) legacy := bracketsByFamily(brackets)[MigrationFamilyLegacyListMetaDelta] - raw := legacyListMetaDeltaKey([]byte("target-list"), 10, 0) + raw := store.LegacyListMetaDeltaKey([]byte("target-list"), 10, 0) value := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) require.True(t, legacy.ContainsRoutedVersion( @@ -580,28 +579,8 @@ func bracketsByFamily(brackets []MigrationBracket) map[uint32]MigrationBracket { } func deltaLookingListMetaUserKey(fakeUserKey []byte, commitTS uint64, seqInTxn uint32) []byte { - key := make([]byte, 0, len("d|")+4+len(fakeUserKey)+8+4) - key = append(key, "d|"...) - var lenPrefix [4]byte - binary.BigEndian.PutUint32(lenPrefix[:], uint32(len(fakeUserKey))) //nolint:gosec // test data is small. - key = append(key, lenPrefix[:]...) - key = append(key, fakeUserKey...) - var ts [8]byte - binary.BigEndian.PutUint64(ts[:], commitTS) - key = append(key, ts[:]...) - var seq [4]byte - binary.BigEndian.PutUint32(seq[:], seqInTxn) - return append(key, seq[:]...) -} - -func legacyListMetaDeltaKey(userKey []byte, commitTS uint64, seqInTxn uint32) []byte { - key := store.LegacyListMetaDeltaScanPrefix(userKey) - var ts [8]byte - binary.BigEndian.PutUint64(ts[:], commitTS) - key = append(key, ts[:]...) - var seq [4]byte - binary.BigEndian.PutUint32(seq[:], seqInTxn) - return append(key, seq[:]...) + // ListMetaKey of the returned user key reads as a legacy list delta key. + return bytes.TrimPrefix(store.LegacyListMetaDeltaKey(fakeUserKey, commitTS, seqInTxn), []byte(store.ListMetaPrefix)) } // File chunk payloads live under !fs|chk| but route through a virtual diff --git a/kv/shard_key_test.go b/kv/shard_key_test.go index 28ffc931c..72e801ba7 100644 --- a/kv/shard_key_test.go +++ b/kv/shard_key_test.go @@ -3,7 +3,6 @@ package kv import ( "bytes" "encoding/base64" - "encoding/binary" "testing" "github.com/bootjp/elastickv/internal/fskeys" @@ -334,14 +333,3 @@ func TestStreamScansAreNotWideColumnCanonicalizable(t *testing.T) { require.True(t, redisWideColumnCanonicalizableScan(start)) } } - -func legacyListMetaDeltaKey(userKey []byte, commitTS uint64) []byte { - const seqInTxn = uint32(1) - key := store.LegacyListMetaDeltaScanPrefix(userKey) - var ts [8]byte - binary.BigEndian.PutUint64(ts[:], commitTS) - key = append(key, ts[:]...) - var seq [4]byte - binary.BigEndian.PutUint32(seq[:], seqInTxn) - return append(key, seq[:]...) -} diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 27e37a9df..d96d9d315 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -1351,7 +1351,7 @@ func TestShardStoreScanAt_RoutesListDeltaScansByUserKey(t *testing.T) { legacyRouting bool }{ {name: "current", key: store.ListMetaDeltaKey(userKey, 10, 1), scanStart: store.ListMetaDeltaScanPrefix(userKey)}, - {name: "legacy", key: legacyListMetaDeltaKey(userKey, 10), scanStart: store.LegacyListMetaDeltaScanPrefix(userKey), legacyRouting: true}, + {name: "legacy", key: store.LegacyListMetaDeltaKey(userKey, 10, 1), scanStart: store.LegacyListMetaDeltaScanPrefix(userKey), legacyRouting: true}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -1377,8 +1377,8 @@ func TestShardStoreScanAt_BroadLegacyListDeltaScansAllRoutes(t *testing.T) { ctx := context.Background() st := newTwoRouteShardStoreForScanTest() deltaValue := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) - leftKey := legacyListMetaDeltaKey([]byte("left-list"), 10) - rightKey := legacyListMetaDeltaKey([]byte("right-list"), 11) + leftKey := store.LegacyListMetaDeltaKey([]byte("left-list"), 10, 1) + rightKey := store.LegacyListMetaDeltaKey([]byte("right-list"), 11, 1) require.NoError(t, st.groups[1].Store.PutAt(ctx, leftKey, deltaValue, 1, 0)) require.NoError(t, st.groups[2].Store.PutAt(ctx, rightKey, deltaValue, 1, 0)) @@ -4114,7 +4114,7 @@ func TestShardStoreScanAt_ExactLegacyListDeltaScanMarksRouteGroup(t *testing.T) // "right-list" sorts into the second route's group. userKey := []byte("right-list") - key := legacyListMetaDeltaKey(userKey, 11) + key := store.LegacyListMetaDeltaKey(userKey, 11, 1) require.NoError(t, st.groups[2].Store.PutAt(ctx, key, deltaValue, 1, 0)) scanStart := store.LegacyListMetaDeltaScanPrefix(userKey) diff --git a/store/list_helpers_test.go b/store/list_helpers_test.go index b397ebb87..8eeca32e5 100644 --- a/store/list_helpers_test.go +++ b/store/list_helpers_test.go @@ -228,16 +228,6 @@ func TestLegacyListMetaDeltaHelpersScanOldPrefixWithoutReclassifyingNewDelta(t * } func deltaLookingListMetaUserKey(fakeUserKey []byte, commitTS uint64, seqInTxn uint32) []byte { - buf := make([]byte, 0, len("d|")+4+len(fakeUserKey)+8+4) - buf = append(buf, "d|"...) - var keyLen [4]byte - binary.BigEndian.PutUint32(keyLen[:], uint32(len(fakeUserKey))) //nolint:gosec // test data is small. - buf = append(buf, keyLen[:]...) - buf = append(buf, fakeUserKey...) - var ts [8]byte - binary.BigEndian.PutUint64(ts[:], commitTS) - buf = append(buf, ts[:]...) - var seq [4]byte - binary.BigEndian.PutUint32(seq[:], seqInTxn) - return append(buf, seq[:]...) + // ListMetaKey of the returned user key reads as a legacy list delta key. + return bytes.TrimPrefix(LegacyListMetaDeltaKey(fakeUserKey, commitTS, seqInTxn), []byte(ListMetaPrefix)) } From 2a2e73520c03fd6e380a475fb7664507c81458d1 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 10:20:34 +0000 Subject: [PATCH 09/15] test: use testutil receive helpers for bounded waits (cleanup WP-9..12) Replace 154 receive-or-fail select blocks with testutil.Recv, NoRecv or Closed, keeping each block's timeout and failure message. A bare receive becomes Closed only when the file closes that channel and Recv only when it sends on it. Blocks were left alone when the choice was ambiguous, the case body uses break or goto, the failure is non-fatal (t.Error), or hoisting a case variable could shadow another name in the function (123 blocks). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- adapter/admin_grpc_test.go | 42 +++------ .../background_reservation_shutdown_test.go | 14 +-- adapter/distribution_server_test.go | 10 +-- adapter/dynamodb_read_fence_test.go | 32 ++----- adapter/dynamodb_retention_test.go | 10 +-- .../dynamodb_table_cleanup_shutdown_test.go | 26 ++---- adapter/dynamodb_test.go | 14 +-- adapter/http_shutdown_test.go | 50 +++-------- adapter/redis_apply_observer_test.go | 8 +- adapter/redis_bzpopmin_wake_test.go | 54 +++++------- adapter/redis_compat_commands_stream_test.go | 46 ++++------ adapter/redis_key_waiters_test.go | 74 +++------------- adapter/redis_shutdown_test.go | 26 ++---- adapter/s3_admin_delete_shutdown_test.go | 20 ++--- adapter/s3_cleanup_shutdown_test.go | 14 +-- adapter/s3_test.go | 8 +- adapter/sqs_shutdown_test.go | 10 +-- adapter/sqs_test.go | 12 ++- .../fuseadapter/abandoned_mount_test.go | 8 +- internal/filesystem/fuseadapter/ack_test.go | 26 ++---- .../filesystem/fuseadapter/adapter_test.go | 26 +++--- .../filesystem/fuseadapter/barrier_test.go | 20 ++--- .../helper_lookup_snapshot_test.go | 8 +- .../filesystem/fuseadapter/server_ops_test.go | 16 ++-- .../fuseadapter/unmount_lazy_cancel_test.go | 8 +- .../fuseadapter/unmount_shutdown_join_test.go | 8 +- .../fuseadapter/wait_mount_cancel_test.go | 14 +-- .../operation_journal_multigroup_test.go | 14 +-- internal/memwatch/memwatch_test.go | 6 +- .../raftengine/etcd/dispatch_report_test.go | 20 ++--- .../etcd/engine_applied_index_test.go | 22 ++--- internal/raftengine/etcd/engine_test.go | 86 +++++-------------- .../raftengine/etcd/grpc_transport_test.go | 40 +++------ .../etcd/removal_notification_test.go | 8 +- internal/snapshotoffload/scheduler_test.go | 8 +- keyviz/flusher_test.go | 8 +- kv/lease_warmup_test.go | 20 ++--- kv/lock_resolver_test.go | 8 +- kv/raft_engine_test.go | 8 +- ...rded_coordinator_registration_gate_test.go | 22 ++--- main_admin_lifecycle_test.go | 8 +- main_admin_shutdown_test.go | 26 ++---- main_encryption_rotate_on_startup_test.go | 54 +++--------- main_filesystem_test.go | 60 ++++--------- main_grpc_stop_test.go | 20 ++--- main_shutdown_balance_test.go | 14 +-- main_shutdown_handoff_test.go | 20 ++--- main_shutdown_signal_log_test.go | 21 ++--- main_shutdown_signal_test.go | 8 +- main_shutdown_test.go | 8 +- monitoring/http_shutdown_test.go | 14 +-- proxy/proxy_test.go | 42 +++------ store/lsm_store_test.go | 14 +-- 53 files changed, 335 insertions(+), 848 deletions(-) diff --git a/adapter/admin_grpc_test.go b/adapter/admin_grpc_test.go index 0fa3cb98a..c9bc619ca 100644 --- a/adapter/admin_grpc_test.go +++ b/adapter/admin_grpc_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/raftengine" pb "github.com/bootjp/elastickv/proto" "github.com/stretchr/testify/require" @@ -207,11 +209,7 @@ func TestGetRaftGroupsLeaderVersionAsync(t *testing.T) { t.Fatalf("GetRaftGroups first: %v", err) } requireLeaderVersion(t, resp, "", "first") - select { - case <-probeStarted: - case <-time.After(time.Second): - t.Fatal("leader version probe was not started") - } + testutil.Recv(t, probeStarted, time.Second, "leader version probe was not started") second, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) if err != nil { t.Fatalf("GetRaftGroups second: %v", err) @@ -498,11 +496,7 @@ func waitForLeaderVersion(t *testing.T, srv *AdminServer, want string) { if len(resp.Groups) == 1 && resp.Groups[0].LeaderNodeVersion == want { return } - select { - case <-deadline: - t.Fatalf("leader version cache was not populated with %q", want) - case <-time.After(10 * time.Millisecond): - } + testutil.NoRecv(t, deadline, 10*time.Millisecond, "leader version cache was not populated with %q", want) } } @@ -528,13 +522,9 @@ func TestGetRaftGroupsLeaderVersionProbePropagatesAuthMetadata(t *testing.T) { if _, err := srv.GetRaftGroups(ctx, &pb.GetRaftGroupsRequest{}); err != nil { t.Fatalf("GetRaftGroups: %v", err) } - select { - case got := <-gotAuth: - if len(got) != 1 || got[0] != "Bearer admin-token" { - t.Fatalf("authorization metadata = %v, want bearer token", got) - } - case <-time.After(time.Second): - t.Fatal("leader version probe did not run") + got := testutil.Recv(t, gotAuth, time.Second, "leader version probe did not run") + if len(got) != 1 || got[0] != "Bearer admin-token" { + t.Fatalf("authorization metadata = %v, want bearer token", got) } } @@ -746,11 +736,7 @@ func TestFanoutConfigurationCallsBoundedConcurrency(t *testing.T) { } close(release) // let everything drain - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("fanoutConfigurationCalls did not return after release") - } + testutil.Closed(t, done, 5*time.Second, "fanoutConfigurationCalls did not return after release") } // testProbeGroup is a minimal AdminGroup for concurrency tests; it runs cb @@ -1137,13 +1123,9 @@ func TestCollectLiveMembersHonoursCtxCancel(t *testing.T) { resCh <- collectLiveMembers(ctx, groups, "self") }() - select { - case r := <-resCh: - // With ctx already cancelled, no live config landed; expect empty maps. - if len(r.addrByID) != 0 || len(r.order) != 0 { - t.Fatalf("expected empty results on early cancel, got addrByID=%v order=%v", r.addrByID, r.order) - } - case <-time.After(1 * time.Second): - t.Fatal("collectLiveMembers blocked past 1s despite cancelled ctx — wg.Wait() regression?") + r := testutil.Recv(t, resCh, 1*time.Second, "collectLiveMembers blocked past 1s despite cancelled ctx — wg.Wait() regression?") + // With ctx already cancelled, no live config landed; expect empty maps. + if len(r.addrByID) != 0 || len(r.order) != 0 { + t.Fatalf("expected empty results on early cancel, got addrByID=%v order=%v", r.addrByID, r.order) } } diff --git a/adapter/background_reservation_shutdown_test.go b/adapter/background_reservation_shutdown_test.go index bf092f6d5..c2aad568e 100644 --- a/adapter/background_reservation_shutdown_test.go +++ b/adapter/background_reservation_shutdown_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/stretchr/testify/require" ) @@ -18,11 +20,7 @@ func TestBackgroundReservationStartsLiveAfterShutdownBound(t *testing.T) { ran := make(chan error, 1) r.start(time.Second, func(ctx context.Context) { ran <- ctx.Err() }) require.NoError(t, <-ran) - select { - case <-done: - case <-time.After(time.Second): - t.Fatal("stop did not join reserved cleanup") - } + testutil.Closed(t, done, time.Second, "stop did not join reserved cleanup") } func TestBackgroundReservationCannotStartAfterStopReturns(t *testing.T) { @@ -34,9 +32,5 @@ func TestBackgroundReservationCannotStartAfterStopReturns(t *testing.T) { r.start(time.Second, func(context.Context) { ran <- struct{}{} }) r.release() require.Zero(t, w.gate.remaining()) - select { - case <-ran: - t.Fatal("cleanup started after shutdown returned") - case <-time.After(time.Millisecond): - } + testutil.NoRecv(t, ran, time.Millisecond, "cleanup started after shutdown returned") } diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index ff6aac0a3..df37f4f7c 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -11,6 +11,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/fskeys" "github.com/bootjp/elastickv/internal/raftengine" @@ -1462,12 +1464,8 @@ func TestDistributionServerSplitRange_RetriesCatalogReloadUntilVisible(t *testin require.NoError(t, err) require.Equal(t, uint64(2), resp.CatalogVersion) - select { - case applyErr := <-coordinator.asyncApplyDone: - require.NoError(t, applyErr) - case <-time.After(time.Second): - t.Fatal("timed out waiting for async split apply") - } + applyErr := testutil.Recv(t, coordinator.asyncApplyDone, time.Second, "timed out waiting for async split apply") + require.NoError(t, applyErr) } func TestDistributionServerSplitRange_RequiresCatalogLeaderWhenCoordinatorConfigured(t *testing.T) { diff --git a/adapter/dynamodb_read_fence_test.go b/adapter/dynamodb_read_fence_test.go index d2cd578df..bc63abbae 100644 --- a/adapter/dynamodb_read_fence_test.go +++ b/adapter/dynamodb_read_fence_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" json "github.com/goccy/go-json" @@ -232,11 +234,7 @@ func TestDynamoDBTransactWriteWaitsForPendingItemUpdate(t *testing.T) { updateDone := dynamoHandlerStatus(t, server.updateItem, first) var firstCommitTS uint64 - select { - case firstCommitTS = <-coord.firstEntered: - case <-time.After(2 * time.Second): - t.Fatal("first update did not reach dispatch") - } + firstCommitTS = testutil.Recv(t, coord.firstEntered, 2*time.Second, "first update did not reach dispatch") // Raise the global watermark past the pending item write. A read fence // cannot observe an entry that has not been proposed to Raft yet. require.NoError(t, st.PutAt(ctx, []byte("other"), []byte("x"), firstCommitTS+1, 0)) @@ -258,11 +256,7 @@ func TestDynamoDBTransactWriteWaitsForPendingItemUpdate(t *testing.T) { // The transaction is waiting on the first write's item lock. close(coord.firstRelease) require.Equal(t, http.StatusOK, <-updateDone) - select { - case <-coord.secondEntered: - case <-time.After(2 * time.Second): - t.Fatal("transaction did not reach dispatch after update completed") - } + testutil.Closed(t, coord.secondEntered, 2*time.Second, "transaction did not reach dispatch after update completed") } close(coord.secondRelease) require.Equal(t, http.StatusOK, <-txnDone) @@ -287,11 +281,7 @@ func TestDynamoDBTwoServersSealPendingItemWrite(t *testing.T) { first.ExpressionAttributeValues[":val"] = numberListAttr("2") updateDone := dynamoHandlerStatus(t, firstServer.updateItem, first) var firstCommitTS uint64 - select { - case firstCommitTS = <-coord.firstEntered: - case <-time.After(2 * time.Second): - t.Fatal("first update did not reach dispatch") - } + firstCommitTS = testutil.Recv(t, coord.firstEntered, 2*time.Second, "first update did not reach dispatch") require.NoError(t, st.PutAt(ctx, []byte("other"), []byte("x"), firstCommitTS+1, 0)) second := transactWriteItemsInput{TransactItems: []transactWriteItem{{Update: &transactUpdateInput{ TableName: first.TableName, @@ -301,11 +291,7 @@ func TestDynamoDBTwoServersSealPendingItemWrite(t *testing.T) { ExpressionAttributeValues: map[string]attributeValue{":empty": {L: []attributeValue{}}, ":val": numberListAttr("3")}, }}}} txnDone := dynamoHandlerStatus(t, secondServer.transactWriteItems, second) - select { - case <-coord.secondEntered: - case <-time.After(2 * time.Second): - t.Fatal("second server did not reach dispatch") - } + testutil.Closed(t, coord.secondEntered, 2*time.Second, "second server did not reach dispatch") close(coord.secondRelease) require.Equal(t, http.StatusOK, <-txnDone) close(coord.firstRelease) @@ -335,11 +321,7 @@ func TestDynamoDBSnapshotSealSurvivesLeaderSwitch(t *testing.T) { first.ExpressionAttributeValues[":val"] = numberListAttr("2") updateDone := dynamoHandlerStatus(t, firstServer.updateItem, first) var firstCommitTS uint64 - select { - case firstCommitTS = <-oldLeader.firstEntered: - case <-time.After(2 * time.Second): - t.Fatal("old leader did not allocate the write timestamp") - } + firstCommitTS = testutil.Recv(t, oldLeader.firstEntered, 2*time.Second, "old leader did not allocate the write timestamp") require.NoError(t, st.PutAt(ctx, []byte("other"), []byte("x"), firstCommitTS+1, 0)) newLeader.Clock().Observe(firstCommitTS + 1) oldLeader.switched.Store(true) diff --git a/adapter/dynamodb_retention_test.go b/adapter/dynamodb_retention_test.go index 6e6081fad..97f91e1c1 100644 --- a/adapter/dynamodb_retention_test.go +++ b/adapter/dynamodb_retention_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -104,11 +106,7 @@ func TestDynamoDBMigrateLegacySourceItemsPinsReadTS(t *testing.T) { close(scanStore.release) - select { - case err := <-done: - require.NoError(t, err) - case <-time.After(time.Second): - t.Fatal("timed out waiting for migrateLegacySourceItems") - } + err := testutil.Recv(t, done, time.Second, "timed out waiting for migrateLegacySourceItems") + require.NoError(t, err) require.Zero(t, tracker.Oldest()) } diff --git a/adapter/dynamodb_table_cleanup_shutdown_test.go b/adapter/dynamodb_table_cleanup_shutdown_test.go index 2fabf1525..4b572541e 100644 --- a/adapter/dynamodb_table_cleanup_shutdown_test.go +++ b/adapter/dynamodb_table_cleanup_shutdown_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" @@ -29,17 +31,9 @@ func TestDynamoDBServerStopWaitsForTableCleanup(t *testing.T) { d.Stop() close(stopDone) }() - select { - case <-stopDone: - t.Fatal("Stop returned while the table cleanup was still dispatching") - case <-time.After(100 * time.Millisecond): - } + testutil.NoRecv(t, stopDone, 100*time.Millisecond, "Stop returned while the table cleanup was still dispatching") close(coord.release) - select { - case <-stopDone: - case <-time.After(5 * time.Second): - t.Fatal("Stop did not return once the table cleanup finished") - } + testutil.Closed(t, stopDone, 5*time.Second, "Stop did not return once the table cleanup finished") } // A table cleanup still running when the bound expires is cancelled and @@ -131,17 +125,9 @@ func TestBackgroundWorkersReservationHoldsStop(t *testing.T) { started.start(time.Minute, func(context.Context) { <-finish }) unused.release() unused.release() - select { - case <-stopDone: - t.Fatal("stop returned while a reserved job was still running") - case <-time.After(100 * time.Millisecond): - } + testutil.NoRecv(t, stopDone, 100*time.Millisecond, "stop returned while a reserved job was still running") close(finish) - select { - case <-stopDone: - case <-time.After(5 * time.Second): - t.Fatal("stop did not return once the reserved job finished") - } + testutil.Closed(t, stopDone, 5*time.Second, "stop did not return once the reserved job finished") } // The public listener answers the stopping refusal with a retryable 503 diff --git a/adapter/dynamodb_test.go b/adapter/dynamodb_test.go index e4160812a..bec1a64be 100644 --- a/adapter/dynamodb_test.go +++ b/adapter/dynamodb_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" @@ -1895,11 +1897,7 @@ func TestDynamoDB_TransactWriteItems_ConditionCheckRace(t *testing.T) { txErrCh <- txErr }() - select { - case <-wrapped.blockEntered: - case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for blocked transact dispatch") - } + testutil.Closed(t, wrapped.blockEntered, 5*time.Second, "timed out waiting for blocked transact dispatch") _, err = client1.UpdateItem(ctx, &dynamodb.UpdateItemInput{ TableName: aws.String("race"), @@ -1916,11 +1914,7 @@ func TestDynamoDB_TransactWriteItems_ConditionCheckRace(t *testing.T) { close(wrapped.blockRelease) var txErr error - select { - case txErr = <-txErrCh: - case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for transact result") - } + txErr = testutil.Recv(t, txErrCh, 5*time.Second, "timed out waiting for transact result") require.ErrorContains(t, txErr, "conditional check failed") targetOut, err := client0.GetItem(ctx, &dynamodb.GetItemInput{ diff --git a/adapter/http_shutdown_test.go b/adapter/http_shutdown_test.go index f690db87d..4bdd5b3bc 100644 --- a/adapter/http_shutdown_test.go +++ b/adapter/http_shutdown_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" ) @@ -105,11 +107,7 @@ func TestHTTPAdapterStopBoundsStalledRequest(t *testing.T) { } reqDone <- err }() - select { - case <-started: - case <-time.After(5 * time.Second): - t.Fatal("handler did not start for the stalled request") - } + testutil.Recv(t, started, 5*time.Second, "handler did not start for the stalled request") start := time.Now() stop() @@ -148,12 +146,8 @@ func TestHTTPAdapterStopReturnsPromptlyWhenIdle(t *testing.T) { start := time.Now() stop() require.Less(t, time.Since(start), 2*time.Second) - select { - case err := <-runDone: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("Run did not return after Stop") - } + err := testutil.Recv(t, runDone, 5*time.Second, "Run did not return after Stop") + require.NoError(t, err) }) } } @@ -175,11 +169,7 @@ func TestHTTPAdapterStopIsIdempotentAfterForcedClose(t *testing.T) { defer func() { _ = conn.Close() }() _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n")) require.NoError(t, err) - select { - case <-started: - case <-time.After(5 * time.Second): - t.Fatal("handler did not start") - } + testutil.Recv(t, started, 5*time.Second, "handler did not start") stop() start := time.Now() @@ -266,11 +256,7 @@ func TestS3ServerStopDrainsHTTPBeforeClosingBlobCluster(t *testing.T) { _ = resp.Body.Close() } }() - select { - case <-started: - case <-time.After(5 * time.Second): - t.Fatal("handler did not start") - } + testutil.Recv(t, started, 5*time.Second, "handler did not start") stopDone := stopInBackground(s) requireStopStillDraining(t, stopDone, cluster) @@ -383,11 +369,7 @@ func startStalledRequest(t *testing.T, lis net.Listener, started <-chan struct{} t.Cleanup(func() { _ = conn.Close() }) _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n")) require.NoError(t, err) - select { - case <-started: - case <-time.After(5 * time.Second): - t.Fatal("handler did not start") - } + testutil.Recv(t, started, 5*time.Second, "handler did not start") } // TestHTTPAdapterStopJoinsHandlerAfterForcedClose pins that Stop does not @@ -413,12 +395,8 @@ func TestHTTPAdapterStopJoinsHandlerAfterForcedClose(t *testing.T) { stop() require.True(t, finished.Load(), "Stop returned while the handler on the closed connection was still running") - select { - case err := <-runDone: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("Run did not return after Stop") - } + err := testutil.Recv(t, runDone, 5*time.Second, "Run did not return after Stop") + require.NoError(t, err) }) } } @@ -448,10 +426,6 @@ func TestHTTPAdapterStopBoundsJoinOfHandlerIgnoringClose(t *testing.T) { start = time.Now() stop() require.Less(t, time.Since(start), 50*time.Millisecond, "second Stop must not wait the join bound again") - select { - case err := <-runDone: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("Run did not return after Stop") - } + err := testutil.Recv(t, runDone, 5*time.Second, "Run did not return after Stop") + require.NoError(t, err) } diff --git a/adapter/redis_apply_observer_test.go b/adapter/redis_apply_observer_test.go index ebc1010ea..0809af958 100644 --- a/adapter/redis_apply_observer_test.go +++ b/adapter/redis_apply_observer_test.go @@ -4,6 +4,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -20,11 +22,7 @@ func requireWaiterSignaled(t *testing.T, ch <-chan struct{}) { func requireWaiterQuiet(t *testing.T, ch <-chan struct{}) { t.Helper() - select { - case <-ch: - t.Fatal("waiter was unexpectedly signaled") - case <-time.After(20 * time.Millisecond): - } + testutil.NoRecv(t, ch, 20*time.Millisecond, "waiter was unexpectedly signaled") } func TestRedisApplyObserverSignalsStreamEntryWaiters(t *testing.T) { diff --git a/adapter/redis_bzpopmin_wake_test.go b/adapter/redis_bzpopmin_wake_test.go index c4b0010e3..e333a9b6a 100644 --- a/adapter/redis_bzpopmin_wake_test.go +++ b/adapter/redis_bzpopmin_wake_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/store" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" @@ -388,16 +390,12 @@ func TestRedis_BZPopMinWakesOnZAdd(t *testing.T) { ).Result() require.NoError(t, err) - select { - case res := <-resultCh: - require.NoError(t, res.err) - require.NotNil(t, res.zwk) - require.Equal(t, "zset-wake", res.zwk.Key) - require.Equal(t, "first", res.zwk.Member) - require.InDelta(t, 1.0, res.zwk.Score, 1e-9) - case <-time.After(6 * time.Second): - t.Fatal("BZPOPMIN did not return after ZADD signal") - } + res := testutil.Recv(t, resultCh, 6*time.Second, "BZPOPMIN did not return after ZADD signal") + require.NoError(t, res.err) + require.NotNil(t, res.zwk) + require.Equal(t, "zset-wake", res.zwk.Key) + require.Equal(t, "first", res.zwk.Member) + require.InDelta(t, 1.0, res.zwk.Score, 1e-9) } // TestRedis_BZPopMinWakesOnZIncrBy verifies the same wake path but @@ -431,16 +429,12 @@ func TestRedis_BZPopMinWakesOnZIncrBy(t *testing.T) { _, err := rdbWriter.ZIncrBy(ctx, "zset-incr-wake", 7.5, "alpha").Result() require.NoError(t, err) - select { - case res := <-resultCh: - require.NoError(t, res.err) - require.NotNil(t, res.zwk) - require.Equal(t, "zset-incr-wake", res.zwk.Key) - require.Equal(t, "alpha", res.zwk.Member) - require.InDelta(t, 7.5, res.zwk.Score, 1e-9) - case <-time.After(6 * time.Second): - t.Fatal("BZPOPMIN did not return after ZINCRBY signal") - } + res := testutil.Recv(t, resultCh, 6*time.Second, "BZPOPMIN did not return after ZINCRBY signal") + require.NoError(t, res.err) + require.NotNil(t, res.zwk) + require.Equal(t, "zset-incr-wake", res.zwk.Key) + require.Equal(t, "alpha", res.zwk.Member) + require.InDelta(t, 7.5, res.zwk.Score, 1e-9) } // TestRedis_BZPopMinRejectsInitialWrongType locks down the @@ -510,13 +504,9 @@ func TestRedis_BZPopMinDetectsMidBlockWrongType(t *testing.T) { time.Sleep(50 * time.Millisecond) require.NoError(t, rdbWriter.Set(ctx, "bzpop-mid-wrongtype", "I am a string", 0).Err()) - select { - case res := <-resultCh: - require.Error(t, res.err, "BZPOPMIN must return WRONGTYPE after mid-block SET, got zwk=%v", res.zwk) - require.Contains(t, res.err.Error(), "WRONGTYPE") - case <-time.After(6 * time.Second): - t.Fatal("BZPOPMIN did not return WRONGTYPE within the BLOCK window after a mid-block SET") - } + res := testutil.Recv(t, resultCh, 6*time.Second, "BZPOPMIN did not return WRONGTYPE within the BLOCK window after a mid-block SET") + require.Error(t, res.err, "BZPOPMIN must return WRONGTYPE after mid-block SET, got zwk=%v", res.zwk) + require.Contains(t, res.err.Error(), "WRONGTYPE") } // TestRedis_BZPopMinDetectsWrongTypeUnderSignalLoad locks down the @@ -599,13 +589,9 @@ func TestRedis_BZPopMinDetectsWrongTypeUnderSignalLoad(t *testing.T) { require.NoError(t, rdbWriter.Set(context.Background(), "bzpop-press-cold", "I am a string", 0).Err()) - select { - case res := <-resultCh: - require.Error(t, res.err, "BZPOPMIN must surface WRONGTYPE on bzpop-press-cold under signal load (zwk=%v)", res.zwk) - require.Contains(t, res.err.Error(), "WRONGTYPE") - case <-time.After(3500 * time.Millisecond): - t.Fatal("BZPOPMIN did not return WRONGTYPE under sustained signal load — fast mode likely sticky") - } + res := testutil.Recv(t, resultCh, 3500*time.Millisecond, "BZPOPMIN did not return WRONGTYPE under sustained signal load — fast mode likely sticky") + require.Error(t, res.err, "BZPOPMIN must surface WRONGTYPE on bzpop-press-cold under signal load (zwk=%v)", res.zwk) + require.Contains(t, res.err.Error(), "WRONGTYPE") } // TestRedis_BZPopMinTimesOutOnEmptyKey locks down the BLOCK-timeout diff --git a/adapter/redis_compat_commands_stream_test.go b/adapter/redis_compat_commands_stream_test.go index c1acc7c88..d34516dd5 100644 --- a/adapter/redis_compat_commands_stream_test.go +++ b/adapter/redis_compat_commands_stream_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" "github.com/redis/go-redis/v9" @@ -102,15 +104,11 @@ func TestRedis_StreamXReadBlockWakesOnXAdd(t *testing.T) { }).Result() require.NoError(t, err) - select { - case res := <-resultCh: - require.NoError(t, res.err) - require.Len(t, res.streams, 1) - require.Len(t, res.streams[0].Messages, 1) - require.Equal(t, "2-0", res.streams[0].Messages[0].ID) - case <-time.After(6 * time.Second): - t.Fatal("XREAD BLOCK did not return after XADD signal") - } + res := testutil.Recv(t, resultCh, 6*time.Second, "XREAD BLOCK did not return after XADD signal") + require.NoError(t, res.err) + require.Len(t, res.streams, 1) + require.Len(t, res.streams[0].Messages, 1) + require.Equal(t, "2-0", res.streams[0].Messages[0].ID) } func TestRedis_StreamXReadIterCtxDeadlineReturnsNull(t *testing.T) { @@ -239,14 +237,10 @@ func TestRedis_StreamXReadBlockChecksWrongTypeAtDeadline(t *testing.T) { requireStreamWaiterRegistered(t, nodes[0].redisServer.streamWaiters, key, waiterRegistering) require.NoError(t, rdbWriter.Set(ctx, key, "now-a-string", 0).Err()) - select { - case res := <-resultCh: - require.Error(t, res.err) - require.Contains(t, res.err.Error(), "WRONGTYPE") - require.Empty(t, res.streams) - case <-time.After(blockWindow + 2*time.Second): - t.Fatal("XREAD BLOCK did not return after wrong-type overwrite") - } + res := testutil.Recv(t, resultCh, blockWindow+2*time.Second, "XREAD BLOCK did not return after wrong-type overwrite") + require.Error(t, res.err) + require.Contains(t, res.err.Error(), "WRONGTYPE") + require.Empty(t, res.streams) } // requireStreamWaiterRegistered waits for key's blocking reader to appear. @@ -1122,15 +1116,11 @@ func TestRedis_StreamXReadShutdownShortCircuits(t *testing.T) { time.Sleep(50 * time.Millisecond) require.NoError(t, nodes[0].redisServer.Close()) - select { - case res := <-xreadDone: - // BLOCK timeout returns redis.Nil — the same wire-level reply - // the client would have seen if the BLOCK had expired - // naturally. The server must NOT surface the cancellation as a - // -ERR or hang the connection. - require.True(t, errors.Is(res.err, redis.Nil), - "BLOCK after shutdown must return redis.Nil, got err=%v streams=%v", res.err, res.streams) - case <-time.After(2 * time.Second): - t.Fatal("XREAD did not return within 2 s of server Close — busy-poll did not honour handlerContext cancel") - } + res := testutil.Recv(t, xreadDone, 2*time.Second, "XREAD did not return within 2 s of server Close — busy-poll did not honour handlerContext cancel") + // BLOCK timeout returns redis.Nil — the same wire-level reply + // the client would have seen if the BLOCK had expired + // naturally. The server must NOT surface the cancellation as a + // -ERR or hang the connection. + require.True(t, errors.Is(res.err, redis.Nil), + "BLOCK after shutdown must return redis.Nil, got err=%v streams=%v", res.err, res.streams) } diff --git a/adapter/redis_key_waiters_test.go b/adapter/redis_key_waiters_test.go index a312a2993..d8abab3cd 100644 --- a/adapter/redis_key_waiters_test.go +++ b/adapter/redis_key_waiters_test.go @@ -6,6 +6,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/bootjp/elastickv/internal/testutil" ) func TestKeyWaiterRegistry_SignalWakesRegisteredWaiter(t *testing.T) { @@ -16,11 +18,7 @@ func TestKeyWaiterRegistry_SignalWakesRegisteredWaiter(t *testing.T) { reg.Signal([]byte("z-a")) - select { - case <-w.C: - case <-time.After(time.Second): - t.Fatal("Signal did not wake waiter") - } + testutil.Recv(t, w.C, time.Second, "Signal did not wake waiter") } func TestKeyWaiterRegistry_SignalUnrelatedKeyDoesNotWake(t *testing.T) { @@ -31,11 +29,7 @@ func TestKeyWaiterRegistry_SignalUnrelatedKeyDoesNotWake(t *testing.T) { reg.Signal([]byte("z-b")) - select { - case <-w.C: - t.Fatal("Signal on unrelated key woke waiter") - case <-time.After(50 * time.Millisecond): - } + testutil.NoRecv(t, w.C, 50*time.Millisecond, "Signal on unrelated key woke waiter") } func TestKeyWaiterRegistry_MultiKeyWaiterWokenByAnyKey(t *testing.T) { @@ -46,11 +40,7 @@ func TestKeyWaiterRegistry_MultiKeyWaiterWokenByAnyKey(t *testing.T) { reg.Signal([]byte("b")) - select { - case <-w.C: - case <-time.After(time.Second): - t.Fatal("multi-key waiter not woken by middle key") - } + testutil.Recv(t, w.C, time.Second, "multi-key waiter not woken by middle key") } func TestKeyWaiterRegistry_DuplicateKeysDeduplicated(t *testing.T) { @@ -61,22 +51,10 @@ func TestKeyWaiterRegistry_DuplicateKeysDeduplicated(t *testing.T) { reg.Signal([]byte("dup")) - select { - case <-w.C: - case <-time.After(time.Second): - t.Fatal("dedup waiter not signaled") - } + testutil.Recv(t, w.C, time.Second, "dedup waiter not signaled") reg.Signal([]byte("dup")) - select { - case <-w.C: - case <-time.After(time.Second): - t.Fatal("post-drain Signal failed to wake waiter") - } - select { - case <-w.C: - t.Fatal("waiter received a phantom third wake") - case <-time.After(20 * time.Millisecond): - } + testutil.Recv(t, w.C, time.Second, "post-drain Signal failed to wake waiter") + testutil.NoRecv(t, w.C, 20*time.Millisecond, "waiter received a phantom third wake") } func TestKeyWaiterRegistry_CoalescesPreDrainSignals(t *testing.T) { @@ -89,16 +67,8 @@ func TestKeyWaiterRegistry_CoalescesPreDrainSignals(t *testing.T) { reg.Signal([]byte("k")) reg.Signal([]byte("k")) - select { - case <-w.C: - case <-time.After(time.Second): - t.Fatal("waiter not signaled by burst") - } - select { - case <-w.C: - t.Fatal("burst was not coalesced — waiter saw a second wake") - case <-time.After(20 * time.Millisecond): - } + testutil.Recv(t, w.C, time.Second, "waiter not signaled by burst") + testutil.NoRecv(t, w.C, 20*time.Millisecond, "burst was not coalesced — waiter saw a second wake") } func TestKeyWaiterRegistry_SignalFullDowngradesCoalescedFastSignal(t *testing.T) { @@ -123,21 +93,13 @@ func TestKeyWaiterRegistry_SignalFullPreservesQueuedFullWake(t *testing.T) { defer release() reg.SignalFull([]byte("k")) - select { - case <-w.C: - case <-time.After(time.Second): - t.Fatal("first SignalFull did not wake waiter") - } + testutil.Recv(t, w.C, time.Second, "first SignalFull did not wake waiter") reg.SignalFull([]byte("k")) if w.fastSignalAllowed() { t.Fatal("first full wake was consumed as a fast signal") } - select { - case <-w.C: - case <-time.After(time.Second): - t.Fatal("second SignalFull did not wake waiter") - } + testutil.Recv(t, w.C, time.Second, "second SignalFull did not wake waiter") if w.fastSignalAllowed() { t.Fatal("second full wake lost its full classification") } @@ -153,11 +115,7 @@ func TestKeyWaiterRegistry_ReleaseStopsFurtherSignals(t *testing.T) { default: } reg.Signal([]byte("k")) - select { - case <-w.C: - t.Fatal("waiter received signal after release") - case <-time.After(20 * time.Millisecond): - } + testutil.NoRecv(t, w.C, 20*time.Millisecond, "waiter received signal after release") } func TestKeyWaiterRegistry_ReleaseIsIdempotent(t *testing.T) { @@ -224,11 +182,7 @@ func TestKeyWaiterRegistry_ManyWaitersFanOut(t *testing.T) { reg.Signal([]byte("fan")) for i, w := range waiters { - select { - case <-w.C: - case <-time.After(time.Second): - t.Fatalf("waiter %d not signaled", i) - } + testutil.Recv(t, w.C, time.Second, "waiter %d not signaled", i) } } diff --git a/adapter/redis_shutdown_test.go b/adapter/redis_shutdown_test.go index 1efc140be..d2eefeb3f 100644 --- a/adapter/redis_shutdown_test.go +++ b/adapter/redis_shutdown_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/stretchr/testify/require" "github.com/tidwall/redcon" ) @@ -80,12 +82,8 @@ func TestRedisServerStopWaitsForInflightCommandWithinBound(t *testing.T) { elapsed := time.Since(start) require.GreaterOrEqual(t, elapsed, 150*time.Millisecond, "Stop must give the running command its bound") require.Less(t, elapsed, 3*time.Second, "Stop must not wait on a command that never finishes") - select { - case err := <-runDone: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("Run did not return after Stop") - } + err := testutil.Recv(t, runDone, 5*time.Second, "Run did not return after Stop") + require.NoError(t, err) start = time.Now() server.Stop() @@ -109,12 +107,8 @@ func TestRedisServerStopReturnsOnceInflightCommandFinishes(t *testing.T) { elapsed := time.Since(start) require.GreaterOrEqual(t, elapsed, 80*time.Millisecond, "Stop must wait for the running command") require.Less(t, elapsed, 2*time.Second, "Stop must return as soon as the command finished, not after the bound") - select { - case err := <-runDone: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("Run did not return after Stop") - } + err := testutil.Recv(t, runDone, 5*time.Second, "Run did not return after Stop") + require.NoError(t, err) } // With nothing in flight Stop returns at once. @@ -127,12 +121,8 @@ func TestRedisServerStopReturnsPromptlyWhenIdle(t *testing.T) { start := time.Now() server.Stop() require.Less(t, time.Since(start), time.Second) - select { - case err := <-runDone: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("Run did not return after Stop") - } + err := testutil.Recv(t, runDone, 5*time.Second, "Run did not return after Stop") + require.NoError(t, err) } // countingRedisCommand is a test-only command that only records that it diff --git a/adapter/s3_admin_delete_shutdown_test.go b/adapter/s3_admin_delete_shutdown_test.go index e389fca7b..d2e586bd2 100644 --- a/adapter/s3_admin_delete_shutdown_test.go +++ b/adapter/s3_admin_delete_shutdown_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -65,11 +67,7 @@ func TestS3ServerStopWaitsForAdminDeleteAndItsCleanup(t *testing.T) { go func() { deleteDone <- s.AdminDeleteObject(context.Background(), fullAdminBucketsPrincipal(), "deletable", "k1") }() - select { - case <-coord.entered: - case <-time.After(5 * time.Second): - t.Fatal("the admin delete never dispatched its transaction") - } + testutil.Closed(t, coord.entered, 5*time.Second, "the admin delete never dispatched its transaction") stopDone := stopInBackground(s) requireStopStillDraining(t, stopDone, cluster) @@ -94,17 +92,9 @@ func TestS3ServerTimedOutAdminDeleteRetainsCleanupReservation(t *testing.T) { go func() { deleteDone <- s.AdminDeleteObject(context.Background(), fullAdminBucketsPrincipal(), "deletable", "k1") }() - select { - case <-coord.entered: - case <-time.After(time.Second): - t.Fatal("delete did not dispatch") - } + testutil.Closed(t, coord.entered, time.Second, "delete did not dispatch") stopDone := stopInBackground(s) - select { - case <-s.reservedCleanups.ctx.Done(): - case <-time.After(time.Second): - t.Fatal("Stop did not reach reserved cleanup cancellation") - } + testutil.Closed(t, s.reservedCleanups.ctx.Done(), time.Second, "Stop did not reach reserved cleanup cancellation") close(coord.release) require.NoError(t, <-deleteDone) waitStopped(t, stopDone) diff --git a/adapter/s3_cleanup_shutdown_test.go b/adapter/s3_cleanup_shutdown_test.go index a9388a621..2a171dc25 100644 --- a/adapter/s3_cleanup_shutdown_test.go +++ b/adapter/s3_cleanup_shutdown_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -64,11 +66,7 @@ func startBlockedManifestCleanup(t *testing.T, s *S3Server, coord *cleanupBlocki UploadID: "upload", Parts: []s3ObjectPart{{PartNo: 1, ChunkSizes: []uint64{1}}}, }) - select { - case <-coord.entered: - case <-time.After(5 * time.Second): - t.Fatal("the cleanup never dispatched its delete batch") - } + testutil.Closed(t, coord.entered, 5*time.Second, "the cleanup never dispatched its delete batch") } // TestS3ServerStopWaitsForCleanupBeforeClosingBlobCluster pins that a @@ -107,11 +105,7 @@ func TestS3ServerStopCancelsCleanupAfterBound(t *testing.T) { elapsed := time.Since(start) require.GreaterOrEqual(t, elapsed, 150*time.Millisecond, "the cleanup must be given its bound") require.Less(t, elapsed, 3*time.Second, "Stop must not wait on a cleanup past the bound") - select { - case <-coord.cancelled: - case <-time.After(5 * time.Second): - t.Fatal("the cleanup's context was not cancelled after the bound") - } + testutil.Closed(t, coord.cancelled, 5*time.Second, "the cleanup's context was not cancelled after the bound") } // Once Stop has begun, a new background cleanup is not started. diff --git a/adapter/s3_test.go b/adapter/s3_test.go index 0294c3308..3fdf177af 100644 --- a/adapter/s3_test.go +++ b/adapter/s3_test.go @@ -20,6 +20,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/aws/aws-sdk-go-v2/aws" v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/bootjp/elastickv/distribution" @@ -237,11 +239,7 @@ func TestS3Server_ProxiesFollowerRequests(t *testing.T) { server.handle(rec, req) var got proxiedRequest - select { - case got = <-proxied: - case <-time.After(5 * time.Second): - t.Fatal("request was not proxied to the leader") - } + got = testutil.Recv(t, proxied, 5*time.Second, "request was not proxied to the leader") require.Equal(t, "signed-s3.example.test", got.host) require.Equal(t, "/", got.path) require.Equal(t, rawQuery, got.rawQuery) diff --git a/adapter/sqs_shutdown_test.go b/adapter/sqs_shutdown_test.go index 634e8cecc..4d135edfa 100644 --- a/adapter/sqs_shutdown_test.go +++ b/adapter/sqs_shutdown_test.go @@ -4,6 +4,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/stretchr/testify/require" ) @@ -74,12 +76,8 @@ func TestSQSServerListenlessRunReturnsAfterStop(t *testing.T) { go func() { runDone <- s.Run() }() time.Sleep(20 * time.Millisecond) s.Stop() - select { - case err := <-runDone: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("Run did not return after Stop") - } + err := testutil.Recv(t, runDone, 5*time.Second, "Run did not return after Stop") + require.NoError(t, err) } // Stop runs from two places at once (the context waiter and the deferred diff --git a/adapter/sqs_test.go b/adapter/sqs_test.go index ccea1c56a..8ecd2e40d 100644 --- a/adapter/sqs_test.go +++ b/adapter/sqs_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + json "github.com/goccy/go-json" ) @@ -190,12 +192,8 @@ func TestSQSServer_ListenlessRunStopRoundTrip(t *testing.T) { time.Sleep(20 * time.Millisecond) srv.Stop() - select { - case err := <-done: - if err != nil { - t.Fatalf("run: %v", err) - } - case <-time.After(2 * time.Second): - t.Fatal("listenless Run did not return within timeout after Stop") + err := testutil.Recv(t, done, 2*time.Second, "listenless Run did not return within timeout after Stop") + if err != nil { + t.Fatalf("run: %v", err) } } diff --git a/internal/filesystem/fuseadapter/abandoned_mount_test.go b/internal/filesystem/fuseadapter/abandoned_mount_test.go index d11d524e8..e5d04bcc6 100644 --- a/internal/filesystem/fuseadapter/abandoned_mount_test.go +++ b/internal/filesystem/fuseadapter/abandoned_mount_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/filesystem/fstest" "github.com/hanwen/go-fuse/v2/fuse" "github.com/stretchr/testify/require" @@ -50,11 +52,7 @@ func TestReleaseAbandonedMountReturns(t *testing.T) { releaseAbandonedMount(results, fs, mountPoint, fuseTypeFor(opts.Name)) close(released) }() - select { - case <-released: - case <-time.After(15 * time.Second): - t.Fatal("releaseAbandonedMount did not return; Unmount is waiting for a serve loop that never ran") - } + testutil.Closed(t, released, 15*time.Second, "releaseAbandonedMount did not return; Unmount is waiting for a serve loop that never ran") // The release cancels the requests in flight, so a handler blocked in // the backend returns before the stores behind it are closed. require.Error(t, inFlight.Err(), "the release must cancel the requests in flight on the late mount") diff --git a/internal/filesystem/fuseadapter/ack_test.go b/internal/filesystem/fuseadapter/ack_test.go index 3fd9779c4..479277f30 100644 --- a/internal/filesystem/fuseadapter/ack_test.go +++ b/internal/filesystem/fuseadapter/ack_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/filesystem" "github.com/hanwen/go-fuse/v2/fuse" "github.com/stretchr/testify/require" @@ -114,21 +116,13 @@ func TestAcknowledgementDoesNotHoldReceiptLockDuringVerification(t *testing.T) { adapter.recordReceipt(7, id) done := make(chan error, 1) go func() { done <- adapter.acknowledgeReceipt(t.Context(), id) }() - select { - case <-core.verifying: - case <-time.After(time.Second): - t.Fatal("verification did not start") - } + testutil.Closed(t, core.verifying, time.Second, "verification did not start") recorded := make(chan struct{}) go func() { adapter.recordReceipt(8, filesystem.OperationID{Filesystem: id.Filesystem, Session: id.Session, Sequence: 2}) close(recorded) }() - select { - case <-recorded: - case <-time.After(time.Second): - t.Fatal("receipt lock held during verification") - } + testutil.Closed(t, recorded, time.Second, "receipt lock held during verification") close(core.release) require.NoError(t, <-done) } @@ -144,21 +138,13 @@ func TestFsyncDoesNotHoldReceiptLockDuringVerification(t *testing.T) { adapter.recordReceipt(7, id) done := make(chan syscall.Errno, 1) go func() { done <- adapter.Fsync(t.Context(), 7, 9, false) }() - select { - case <-core.verifying: - case <-time.After(time.Second): - t.Fatal("verification did not start") - } + testutil.Closed(t, core.verifying, time.Second, "verification did not start") recorded := make(chan struct{}) go func() { adapter.recordReceipt(8, filesystem.OperationID{Filesystem: id.Filesystem, Session: id.Session, Sequence: 2}) close(recorded) }() - select { - case <-recorded: - case <-time.After(time.Second): - t.Fatal("receipt lock held during fsync verification") - } + testutil.Closed(t, recorded, time.Second, "receipt lock held during fsync verification") close(core.release) require.Zero(t, <-done) } diff --git a/internal/filesystem/fuseadapter/adapter_test.go b/internal/filesystem/fuseadapter/adapter_test.go index 3e1a94f00..660b8b002 100644 --- a/internal/filesystem/fuseadapter/adapter_test.go +++ b/internal/filesystem/fuseadapter/adapter_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/filesystem" "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" @@ -174,14 +176,10 @@ func TestAdapterKeepsOpenHandlesAliveWhileIdle(t *testing.T) { require.EqualValues(t, 7, core.openInode) require.Equal(t, []byte("fuse-session"), core.openClientID) - select { - case got := <-core.refreshCh: - require.EqualValues(t, 7, got.inode) - require.EqualValues(t, 9, got.fh) - require.Equal(t, []byte("fuse-session"), got.clientID) - case <-time.After(time.Second): - t.Fatal("timed out waiting for open handle keepalive") - } + got := testutil.Recv(t, core.refreshCh, time.Second, "timed out waiting for open handle keepalive") + require.EqualValues(t, 7, got.inode) + require.EqualValues(t, 9, got.fh) + require.Equal(t, []byte("fuse-session"), got.clientID) } func TestAdapterKeepsCreatedHandlesAliveWhileIdle(t *testing.T) { @@ -202,14 +200,10 @@ func TestAdapterKeepsCreatedHandlesAliveWhileIdle(t *testing.T) { require.EqualValues(t, 7, result.Inode) require.EqualValues(t, 9, result.FH) - select { - case got := <-core.refreshCh: - require.EqualValues(t, 7, got.inode) - require.EqualValues(t, 9, got.fh) - require.Equal(t, []byte("fuse-session"), got.clientID) - case <-time.After(time.Second): - t.Fatal("timed out waiting for created handle keepalive") - } + got := testutil.Recv(t, core.refreshCh, time.Second, "timed out waiting for created handle keepalive") + require.EqualValues(t, 7, got.inode) + require.EqualValues(t, 9, got.fh) + require.Equal(t, []byte("fuse-session"), got.clientID) } func TestAdapterKeepaliveTimeoutDoesNotBlockOtherHandles(t *testing.T) { diff --git a/internal/filesystem/fuseadapter/barrier_test.go b/internal/filesystem/fuseadapter/barrier_test.go index 16fb67bf9..67c79b2de 100644 --- a/internal/filesystem/fuseadapter/barrier_test.go +++ b/internal/filesystem/fuseadapter/barrier_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/filesystem" "github.com/bootjp/elastickv/kv" "github.com/stretchr/testify/require" @@ -57,11 +59,7 @@ func TestFsyncWaitsForEarlierWrite(t *testing.T) { <-blocked.started fsyncDone := make(chan syscall.Errno, 1) go func() { fsyncDone <- adapter.Fsync(t.Context(), 7, 9, false) }() - select { - case <-fsyncDone: - t.Fatal("fsync overtook an earlier write") - case <-time.After(30 * time.Millisecond): - } + testutil.NoRecv(t, fsyncDone, 30*time.Millisecond, "fsync overtook an earlier write") close(blocked.release) require.Zero(t, <-writeDone) require.Zero(t, <-fsyncDone) @@ -106,11 +104,7 @@ func TestDirectoryFsyncWaitsForEarlierNamespaceMutation(t *testing.T) { <-blocked.started fsyncDone := make(chan syscall.Errno, 1) go func() { fsyncDone <- adapter.FsyncDirectory(t.Context(), filesystem.RootInode) }() - select { - case <-fsyncDone: - t.Fatal("directory fsync overtook an earlier mkdir") - case <-time.After(30 * time.Millisecond): - } + testutil.NoRecv(t, fsyncDone, 30*time.Millisecond, "directory fsync overtook an earlier mkdir") close(blocked.release) require.Zero(t, <-mkdirDone) require.Zero(t, <-fsyncDone) @@ -128,11 +122,7 @@ func TestMutationTrackerOrdersSameInodeAndReleasesCanceledWaiter(t *testing.T) { _, result := tracker.begin(waitCtx, 7) secondDone <- result }() - select { - case <-secondDone: - t.Fatal("second mutation overtook its predecessor") - case <-time.After(30 * time.Millisecond): - } + testutil.NoRecv(t, secondDone, 30*time.Millisecond, "second mutation overtook its predecessor") cancel() require.NotZero(t, <-secondDone) tracker.end(first, false) diff --git a/internal/filesystem/fuseadapter/helper_lookup_snapshot_test.go b/internal/filesystem/fuseadapter/helper_lookup_snapshot_test.go index 0976f29a5..4f9ce1dba 100644 --- a/internal/filesystem/fuseadapter/helper_lookup_snapshot_test.go +++ b/internal/filesystem/fuseadapter/helper_lookup_snapshot_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/stretchr/testify/require" ) @@ -51,11 +53,7 @@ func TestCanceledMountHelperLookupDoesNotLaunchWorker(t *testing.T) { cancel() _, err := lookupMountHelperContext(ctx, "fusermount3") require.ErrorIs(t, err, context.Canceled) - select { - case <-called: - t.Fatal("already-cancelled lookup launched background work") - case <-time.After(10 * time.Millisecond): - } + testutil.NoRecv(t, called, 10*time.Millisecond, "already-cancelled lookup launched background work") } func TestMountHelperLookupSnapshotSurvivesConcurrentOwner(t *testing.T) { diff --git a/internal/filesystem/fuseadapter/server_ops_test.go b/internal/filesystem/fuseadapter/server_ops_test.go index 16b1bde38..510e98de0 100644 --- a/internal/filesystem/fuseadapter/server_ops_test.go +++ b/internal/filesystem/fuseadapter/server_ops_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/filesystem" "github.com/bootjp/elastickv/store" "github.com/hanwen/go-fuse/v2/fuse" @@ -629,11 +631,7 @@ func TestRequestContextCancelsWhenKernelInterrupts(t *testing.T) { require.NoError(t, ctx.Err()) close(cancel) - select { - case <-ctx.Done(): - case <-time.After(time.Second): - t.Fatal("request context did not observe kernel interrupt") - } + testutil.Closed(t, ctx.Done(), time.Second, "request context did not observe kernel interrupt") require.ErrorIs(t, ctx.Err(), context.Canceled) } @@ -656,12 +654,8 @@ func TestRawFileSystemPropagatesKernelInterruptToCore(t *testing.T) { }() <-core.started close(cancel) - select { - case status := <-result: - require.Equal(t, fuse.Status(syscall.EINTR), status) - case <-time.After(time.Second): - t.Fatal("GetAttr did not return after kernel interrupt") - } + status := testutil.Recv(t, result, time.Second, "GetAttr did not return after kernel interrupt") + require.Equal(t, fuse.Status(syscall.EINTR), status) } func TestFuseAttrClampsNegativeTimestamps(t *testing.T) { diff --git a/internal/filesystem/fuseadapter/unmount_lazy_cancel_test.go b/internal/filesystem/fuseadapter/unmount_lazy_cancel_test.go index 92d00e042..3c8b647e0 100644 --- a/internal/filesystem/fuseadapter/unmount_lazy_cancel_test.go +++ b/internal/filesystem/fuseadapter/unmount_lazy_cancel_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/stretchr/testify/require" ) @@ -21,11 +23,7 @@ func TestUnmountLazyCancelsInFlightRequests(t *testing.T) { defer done() require.NoError(t, server.UnmountLazy(context.Background())) - select { - case <-inFlight.Done(): - case <-time.After(5 * time.Second): - t.Fatal("the in-flight request was not cancelled by the lazy detach") - } + testutil.Closed(t, inFlight.Done(), 5*time.Second, "the in-flight request was not cancelled by the lazy detach") late, lateDone := fs.requestContext(nil) defer lateDone() diff --git a/internal/filesystem/fuseadapter/unmount_shutdown_join_test.go b/internal/filesystem/fuseadapter/unmount_shutdown_join_test.go index cb9018574..9a730f698 100644 --- a/internal/filesystem/fuseadapter/unmount_shutdown_join_test.go +++ b/internal/filesystem/fuseadapter/unmount_shutdown_join_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/hanwen/go-fuse/v2/fuse" "github.com/stretchr/testify/require" ) @@ -26,11 +28,7 @@ func TestJoinAbandonedMountWaitsForCleanup(t *testing.T) { done <- joinAbandonedMount(func() { close(entered); <-release }, time.Second) }() <-entered - select { - case <-done: - t.Fatal("startup returned before its late mount cleanup") - case <-time.After(time.Millisecond): - } + testutil.NoRecv(t, done, time.Millisecond, "startup returned before its late mount cleanup") close(release) require.True(t, <-done) } diff --git a/internal/filesystem/fuseadapter/wait_mount_cancel_test.go b/internal/filesystem/fuseadapter/wait_mount_cancel_test.go index d2fe0e951..217815448 100644 --- a/internal/filesystem/fuseadapter/wait_mount_cancel_test.go +++ b/internal/filesystem/fuseadapter/wait_mount_cancel_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/filesystem" "github.com/bootjp/elastickv/internal/filesystem/fstest" "github.com/hanwen/go-fuse/v2/fuse" @@ -71,11 +73,7 @@ func TestWaitMountStopsWhenCancelledWhileReadinessBlocks(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("Unmount blocked on the mount WaitMount had already released") } - select { - case <-m.serveDone: - case <-time.After(5 * time.Second): - t.Fatal("Serve did not return after the mount was released") - } + testutil.Closed(t, m.serveDone, 5*time.Second, "Serve did not return after the mount was released") top, found, err := topmostMount(context.Background(), mountTablePath, m.mountPoint) require.NoError(t, err) require.False(t, found && top.fsType == m.server.fsType, "the released mount is still attached") @@ -127,11 +125,7 @@ func mountWithBlockedReadiness(t *testing.T, ctx context.Context) blockedReadine waitDone := make(chan error, 1) go func() { waitDone <- server.WaitMount() }() - select { - case <-core.entered: - case <-time.After(10 * time.Second): - t.Fatal("the readiness check never reached the backend getattr") - } + testutil.Closed(t, core.entered, 10*time.Second, "the readiness check never reached the backend getattr") select { case err := <-waitDone: t.Fatalf("WaitMount returned (%v) while the backend getattr was blocked", err) diff --git a/internal/filesystem/operation_journal_multigroup_test.go b/internal/filesystem/operation_journal_multigroup_test.go index 3ffa7f2ca..92e7e897b 100644 --- a/internal/filesystem/operation_journal_multigroup_test.go +++ b/internal/filesystem/operation_journal_multigroup_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/fskeys" "github.com/bootjp/elastickv/internal/raftengine" @@ -184,11 +186,7 @@ func assertJournalCommitsDoNotCollideOnPreparedSession(t *testing.T, ctx context commit := func(sequence uint64) error { return commitWithContext(ctx, sequence) } first := make(chan error, 1) go func() { first <- commit(1000) }() - select { - case <-held.prepared: - case <-time.After(5 * time.Second): - t.Fatal("first journal commit did not prepare the session") - } + testutil.Closed(t, held.prepared, 5*time.Second, "first journal commit did not prepare the session") second := make(chan error, 1) go func() { second <- commit(1001) }() select { @@ -238,11 +236,7 @@ func assertWriteAfterConcurrentAcknowledgement(t *testing.T, ctx context.Context t.Helper() ackDone := make(chan error, 1) go func() { ackDone <- journal.Acknowledge(ctx, createID) }() - select { - case <-held.prepared: - case <-time.After(5 * time.Second): - t.Fatal("acknowledgement did not prepare its session participant") - } + testutil.Closed(t, held.prepared, 5*time.Second, "acknowledgement did not prepare its session participant") type writeResult struct { count int err error diff --git a/internal/memwatch/memwatch_test.go b/internal/memwatch/memwatch_test.go index e524d157d..5749a0fb8 100644 --- a/internal/memwatch/memwatch_test.go +++ b/internal/memwatch/memwatch_test.go @@ -15,11 +15,7 @@ import ( // Watcher.Done() channel avoids the goleak dep and keeps the check local. func waitForDone(t *testing.T, w *Watcher, d time.Duration) { t.Helper() - select { - case <-w.Done(): - case <-time.After(d): - t.Fatalf("watcher goroutine did not exit within %v", d) - } + testutil.Closed(t, w.Done(), d, "watcher goroutine did not exit within %v", d) } // TestWatcher_FiresOnceAboveThreshold creates a watcher with a threshold diff --git a/internal/raftengine/etcd/dispatch_report_test.go b/internal/raftengine/etcd/dispatch_report_test.go index 63c7fb631..f21cbbb10 100644 --- a/internal/raftengine/etcd/dispatch_report_test.go +++ b/internal/raftengine/etcd/dispatch_report_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/stretchr/testify/require" raftpb "go.etcd.io/raft/v3/raftpb" ) @@ -65,18 +67,10 @@ func TestReportSuccessfulDispatchWaitsForSnapshotFinishSlot(t *testing.T) { close(done) }() - select { - case <-done: - t.Fatal("snapshot finish report returned while dispatchReportCh was full") - case <-time.After(50 * time.Millisecond): - } + testutil.NoRecv(t, done, 50*time.Millisecond, "snapshot finish report returned while dispatchReportCh was full") require.Equal(t, blockingReport, <-e.dispatchReportCh) - select { - case <-done: - case <-time.After(testDispatchReportTimeout): - t.Fatal("snapshot finish report did not complete after dispatchReportCh had space") - } + testutil.Closed(t, done, testDispatchReportTimeout, "snapshot finish report did not complete after dispatchReportCh had space") select { case got := <-e.dispatchReportCh: require.Equal(t, dispatchReport{to: 2, msgType: raftpb.MsgSnap, snapshotFinish: true}, got) @@ -100,11 +94,7 @@ func TestReportSuccessfulDispatchAbortsOnCloseWhenReportFull(t *testing.T) { close(done) }() - select { - case <-done: - case <-time.After(testDispatchReportTimeout): - t.Fatal("snapshot finish report did not abort when closeCh was signalled") - } + testutil.Closed(t, done, testDispatchReportTimeout, "snapshot finish report did not abort when closeCh was signalled") } func TestReportSuccessfulDispatchIgnoresRegularMessage(t *testing.T) { diff --git a/internal/raftengine/etcd/engine_applied_index_test.go b/internal/raftengine/etcd/engine_applied_index_test.go index 20c5040b4..ad887bd07 100644 --- a/internal/raftengine/etcd/engine_applied_index_test.go +++ b/internal/raftengine/etcd/engine_applied_index_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/require" @@ -130,11 +132,7 @@ func TestPersistReadyWithSnapshotHoldsSnapshotMuThroughSaveSnap(t *testing.T) { persistDone <- e.persistReady(rd) }() - select { - case <-saveStarted: - case <-time.After(time.Second): - t.Fatal("SaveSnap did not start") - } + testutil.Closed(t, saveStarted, time.Second, "SaveSnap did not start") prepareDone := make(chan error, 1) go func() { @@ -176,11 +174,7 @@ func TestReleaseProtectedReceivedFSMSnapshotsUpToRetriesAfterSnapshotMuUnlock(t close(done) }() - select { - case <-done: - case <-time.After(100 * time.Millisecond): - t.Fatal("releaseProtectedReceivedFSMSnapshotsUpTo blocked on snapshotMu") - } + testutil.Closed(t, done, 100*time.Millisecond, "releaseProtectedReceivedFSMSnapshotsUpTo blocked on snapshotMu") require.Equal(t, map[uint64]int{7: 1}, e.protectedReceivedFSMSnaps) e.snapshotMu.Unlock() @@ -203,12 +197,8 @@ func TestProtectReceivedFSMSnapshotRechecksAppliedIndexUnderLock(t *testing.T) { e.appliedIndex.Store(9) e.snapshotMu.Unlock() - select { - case got := <-accepted: - require.False(t, got) - case <-time.After(time.Second): - t.Fatal("protectReceivedFSMSnapshot did not return") - } + got := testutil.Recv(t, accepted, time.Second, "protectReceivedFSMSnapshot did not return") + require.False(t, got) require.Empty(t, e.protectedReceivedFSMSnaps) } diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index d80f94b90..619858dd5 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -13,6 +13,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + internalutil "github.com/bootjp/elastickv/internal" "github.com/bootjp/elastickv/internal/raftengine" "github.com/cockroachdb/errors" @@ -545,13 +547,9 @@ func TestVerifyLeaderUsesReadIndexPath(t *testing.T) { errCh <- engine.VerifyLeader(context.Background()) }() - select { - case req := <-engine.readCh: - require.False(t, req.waitApplied) - req.done <- readResult{index: 9} - case <-time.After(time.Second): - t.Fatal("verify leader did not issue read-index request") - } + req := testutil.Recv(t, engine.readCh, time.Second, "verify leader did not issue read-index request") + require.False(t, req.waitApplied) + req.done <- readResult{index: 9} require.NoError(t, <-errCh) } @@ -585,20 +583,12 @@ func TestHandleTransportMessageWaitsForStartup(t *testing.T) { errCh <- engine.handleTransportMessage(context.Background(), raftpb.Message{Type: messageTypePtr(raftpb.MsgHeartbeat)}) }() - select { - case <-engine.stepCh: - t.Fatal("transport message delivered before startup") - case <-time.After(20 * time.Millisecond): - } + testutil.NoRecv(t, engine.stepCh, 20*time.Millisecond, "transport message delivered before startup") close(engine.startedCh) - select { - case msg := <-engine.stepCh: - require.Equal(t, raftpb.MsgHeartbeat, msg.GetType()) - case <-time.After(time.Second): - t.Fatal("transport message was not delivered after startup") - } + msg := testutil.Recv(t, engine.stepCh, time.Second, "transport message was not delivered after startup") + require.Equal(t, raftpb.MsgHeartbeat, msg.GetType()) require.NoError(t, <-errCh) } @@ -919,11 +909,7 @@ func TestReportFailedDispatchReliablyQueuesMsgSnapFailure(t *testing.T) { close(done) }() - select { - case <-done: - t.Fatal("MsgSnap failure report must wait instead of dropping when report channel is full") - case <-time.After(20 * time.Millisecond): - } + testutil.NoRecv(t, done, 20*time.Millisecond, "MsgSnap failure report must wait instead of dropping when report channel is full") <-engine.dispatchReportCh requireSignal(t, done, time.Second, "MsgSnap failure report was not delivered after report channel drained") @@ -978,11 +964,7 @@ func TestStopDispatchWorkersCancelsInflightDispatch(t *testing.T) { engine.startDispatchWorkers() engine.peerDispatchers[2].heartbeat <- dispatchRequest{msg: raftpb.Message{Type: messageTypePtr(raftpb.MsgHeartbeat), To: uint64Ptr(2)}} - select { - case <-started: - case <-time.After(time.Second): - t.Fatal("dispatch worker did not start") - } + testutil.Closed(t, started, time.Second, "dispatch worker did not start") done := make(chan struct{}) go func() { @@ -1069,11 +1051,7 @@ func TestRemovePeerClosesDispatcherAndDropsSubsequentMessages(t *testing.T) { engine.dispatchWG.Wait() close(workerDone) }() - select { - case <-workerDone: - case <-time.After(time.Second): - t.Fatal("dispatch workers did not exit after channel close") - } + testutil.Closed(t, workerDone, time.Second, "dispatch workers did not exit after channel close") // Subsequent messages to the removed peer must be dropped without panic. require.NoError(t, engine.enqueueDispatchMessage(raftpb.Message{Type: messageTypePtr(raftpb.MsgHeartbeat), To: uint64Ptr(2)})) @@ -1144,21 +1122,13 @@ func TestMaybePersistLocalSnapshotRunsInBackground(t *testing.T) { t.Fatal("maybePersistLocalSnapshot blocked on snapshot persistence") } - select { - case <-fsm.started: - case <-time.After(time.Second): - t.Fatal("snapshot worker did not start") - } + testutil.Closed(t, fsm.started, time.Second, "snapshot worker did not start") require.Empty(t, persist.Action()) release() - select { - case result := <-engine.snapshotResCh: - require.NoError(t, engine.handleSnapshotResult(result)) - case <-time.After(time.Second): - t.Fatal("snapshot worker did not finish") - } + result := testutil.Recv(t, engine.snapshotResCh, time.Second, "snapshot worker did not finish") + require.NoError(t, engine.handleSnapshotResult(result)) actions := persist.Action() require.Len(t, actions, 2) @@ -1284,11 +1254,7 @@ func TestPersistConfigStateWaitsForDurablePublication(t *testing.T) { }) }() - select { - case <-fsm.started: - case <-time.After(time.Second): - t.Fatal("config snapshot serialization did not start") - } + testutil.Closed(t, fsm.started, time.Second, "config snapshot serialization did not start") select { case err := <-done: @@ -1388,11 +1354,7 @@ func TestPersistConfigStateBlocksConcurrentRestore(t *testing.T) { }) }() - select { - case <-fsm.started: - case <-time.After(time.Second): - t.Fatal("config snapshot serialization did not start") - } + testutil.Closed(t, fsm.started, time.Second, "config snapshot serialization did not start") restoreDone := make(chan error, 1) go func() { @@ -1887,13 +1849,9 @@ func TestWaitForOpenCanceledContextClosesMultiNodeEngine(t *testing.T) { } close(engine.doneCh) - select { - case got := <-resultCh: - require.Nil(t, got.engine) - require.ErrorIs(t, got.err, context.Canceled) - case <-time.After(time.Second): - t.Fatal("waitForOpen did not return after Close completed") - } + got := testutil.Recv(t, resultCh, time.Second, "waitForOpen did not return after Close completed") + require.Nil(t, got.engine) + require.ErrorIs(t, got.err, context.Canceled) } func TestOpenMultiNodeReplicatesOverGRPCTransport(t *testing.T) { @@ -2659,11 +2617,7 @@ func TestMultiLaneDispatcher_SnapshotDoesNotBlockReplication(t *testing.T) { require.NoError(t, engine.enqueueDispatchMessage(raftpb.Message{Type: messageTypePtr(raftpb.MsgSnap), To: uint64Ptr(2)})) require.NoError(t, engine.enqueueDispatchMessage(raftpb.Message{Type: messageTypePtr(raftpb.MsgApp), To: uint64Ptr(2)})) - select { - case <-replicationDone: - case <-time.After(time.Second): - t.Fatal("MsgApp did not dispatch while MsgSnap was stuck — lanes are not independent") - } + testutil.Recv(t, replicationDone, time.Second, "MsgApp did not dispatch while MsgSnap was stuck — lanes are not independent") close(snapshotBlocking) close(engine.dispatchStopCh) diff --git a/internal/raftengine/etcd/grpc_transport_test.go b/internal/raftengine/etcd/grpc_transport_test.go index a57fc43bb..badfc8911 100644 --- a/internal/raftengine/etcd/grpc_transport_test.go +++ b/internal/raftengine/etcd/grpc_transport_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" @@ -1101,11 +1103,7 @@ func TestDispatchRegularFallsBackToUnaryWhenSendStreamDisabledDuringSend(t *test }) }() - select { - case <-sendStarted: - case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for stream Send to start") - } + testutil.Closed(t, sendStarted, 5*time.Second, "timed out waiting for stream Send to start") transport.SetSendStreamEnabled(false) select { @@ -1335,11 +1333,7 @@ func TestDispatchRegularStreamCancelsCachedStreamWhenContextDone(t *testing.T) { })) }() - select { - case <-sendStarted: - case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for stream Send to start") - } + testutil.Closed(t, sendStarted, 5*time.Second, "timed out waiting for stream Send to start") cancel() select { @@ -1462,12 +1456,8 @@ func collectRaftMessages(t *testing.T, ch <-chan raftpb.Message, count int) []ra t.Helper() got := make([]raftpb.Message, 0, count) for len(got) < count { - select { - case msg := <-ch: - got = append(got, msg) - case <-time.After(5 * time.Second): - t.Fatalf("timed out waiting for raft message %d/%d", len(got)+1, count) - } + msg := testutil.Recv(t, ch, 5*time.Second, "timed out waiting for raft message %d/%d", len(got)+1, count) + got = append(got, msg) } return got } @@ -1896,13 +1886,9 @@ func TestStreamFSMSnapshotOverGRPCRestoresFollowerFSM(t *testing.T) { } require.NoError(t, sendTransport.streamFSMSnapshot(context.Background(), msg, 77, openFn)) - select { - case restoreErr := <-restoredCh: - require.NoError(t, restoreErr, - "fresh follower FSM must restore cleanly; a short-read EOF here means the trailing partial chunk was dropped in transit") - case <-time.After(5 * time.Second): - t.Fatal("receiver never ran Restore") - } + restoreErr := testutil.Recv(t, restoredCh, 5*time.Second, "receiver never ran Restore") + require.NoError(t, restoreErr, + "fresh follower FSM must restore cleanly; a short-read EOF here means the trailing partial chunk was dropped in transit") require.Equal(t, senderFSM.Applied(), receiverFSM.Applied(), "receiver FSM state after Restore must equal sender FSM state") @@ -1949,12 +1935,8 @@ func TestStreamFSMSnapshotOverGRPCAtChunkBoundary(t *testing.T) { } require.NoError(t, sendTransport.streamFSMSnapshot(context.Background(), msg, 91, openFn)) - select { - case got := <-gotCh: - require.Equal(t, payload, got.Snapshot.Data) - case <-time.After(5 * time.Second): - t.Fatal("receiver never observed the snapshot message") - } + got := testutil.Recv(t, gotCh, 5*time.Second, "receiver never observed the snapshot message") + require.Equal(t, payload, got.Snapshot.Data) } func TestStreamFSMSnapshotSendsPayload(t *testing.T) { diff --git a/internal/raftengine/etcd/removal_notification_test.go b/internal/raftengine/etcd/removal_notification_test.go index 1f667b343..ef4f9a5be 100644 --- a/internal/raftengine/etcd/removal_notification_test.go +++ b/internal/raftengine/etcd/removal_notification_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/stretchr/testify/require" raftpb "go.etcd.io/raft/v3/raftpb" @@ -61,11 +63,7 @@ func testRemovalCommitAfterPeerTeardown(t *testing.T) { require.NoError(t, err) require.Greater(t, removeIndex, addIndex) waitForConfigSize(t, leader.engine, 1) - select { - case <-entered: - case <-time.After(5 * time.Second): - t.Fatal("final commit notification did not reach the removed peer") - } + testutil.Closed(t, entered, 5*time.Second, "final commit notification did not reach the removed peer") unblock() require.Eventually(t, func() bool { return nodes[1].engine.State() == raftengine.StateShutdown diff --git a/internal/snapshotoffload/scheduler_test.go b/internal/snapshotoffload/scheduler_test.go index d30dd7b51..6d183a8d9 100644 --- a/internal/snapshotoffload/scheduler_test.go +++ b/internal/snapshotoffload/scheduler_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" @@ -91,11 +93,7 @@ func TestSchedulerPublicationTimeoutReleasesScanAndAllowsRetry(t *testing.T) { scheduler.SyncOnce(context.Background()) close(done) }() - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("a stalled store must not block the scan indefinitely") - } + testutil.Closed(t, done, 5*time.Second, "a stalled store must not block the scan indefinitely") _, _, failures := observer.snapshot() require.Len(t, failures, 1) require.ErrorIs(t, failures[0], context.DeadlineExceeded) diff --git a/keyviz/flusher_test.go b/keyviz/flusher_test.go index 89d3c4fab..70affe323 100644 --- a/keyviz/flusher_test.go +++ b/keyviz/flusher_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/stretchr/testify/require" ) @@ -186,11 +188,7 @@ func TestFlushWindowKeepsHotKeyAndCounterInSameColumn(t *testing.T) { s.Observe(1, []byte("hot"), OpWrite, 0, LabelLegacy) close(observed) }() - select { - case <-observed: - t.Fatal("Observe crossed a hot-key flush boundary") - case <-time.After(20 * time.Millisecond): - } + testutil.NoRecv(t, observed, 20*time.Millisecond, "Observe crossed a hot-key flush boundary") s.flushMu.Unlock() // Unbounded on purpose: a wall-clock budget here fails whenever a // loaded -race runner deschedules the Observe goroutine. A real diff --git a/kv/lease_warmup_test.go b/kv/lease_warmup_test.go index afa9d8976..ee1e42373 100644 --- a/kv/lease_warmup_test.go +++ b/kv/lease_warmup_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/monoclock" "github.com/bootjp/elastickv/internal/raftengine" @@ -512,14 +514,10 @@ func TestShardedCoordinator_RenewHLCLeases_UsesProposalTimeoutBeyondCadence(t *t done := coord.renewHLCLeases(context.Background()) requireRenewalDone(t, done) - select { - case remaining := <-deadlineRemaining: - require.Greater(t, remaining, hlcRenewalInterval, - "renewal proposals need a timeout longer than the cadence under load") - require.LessOrEqual(t, remaining, hlcRenewalProposalTimeout) - case <-time.After(time.Second): - t.Fatal("timed out waiting for captured renewal deadline") - } + remaining := testutil.Recv(t, deadlineRemaining, time.Second, "timed out waiting for captured renewal deadline") + require.Greater(t, remaining, hlcRenewalInterval, + "renewal proposals need a timeout longer than the cadence under load") + require.LessOrEqual(t, remaining, hlcRenewalProposalTimeout) } func TestHLCLeaseRenewalTimingHasPhysicalWindowMargin(t *testing.T) { @@ -606,9 +604,5 @@ func TestShardedCoordinator_ProposeHLCLease_UsesDedicatedTimestampGroup(t *testi func requireRenewalDone(t *testing.T, done <-chan struct{}) { t.Helper() - select { - case <-done: - case <-time.After(2 * time.Second): - t.Fatal("timed out waiting for HLC lease renewals") - } + testutil.Closed(t, done, 2*time.Second, "timed out waiting for HLC lease renewals") } diff --git a/kv/lock_resolver_test.go b/kv/lock_resolver_test.go index 4afc19415..4cc978e61 100644 --- a/kv/lock_resolver_test.go +++ b/kv/lock_resolver_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" pb "github.com/bootjp/elastickv/proto" @@ -335,11 +337,7 @@ func TestLockResolver_CloseStopsBackground(t *testing.T) { close(done) }() - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("LockResolver.Close() did not return within 5s") - } + testutil.Closed(t, done, 5*time.Second, "LockResolver.Close() did not return within 5s") } func TestLockResolverRaftReadyRequiresSettledLeader(t *testing.T) { diff --git a/kv/raft_engine_test.go b/kv/raft_engine_test.go index c7feba8dc..047267ca8 100644 --- a/kv/raft_engine_test.go +++ b/kv/raft_engine_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/cockroachdb/errors" ) @@ -50,11 +52,7 @@ func TestVerifyLeaderEngine_BoundsBlockingReadIndex(t *testing.T) { result <- verifyLeaderEngineCtx(ctx, blockingLeaderView{}) }() var err error - select { - case err = <-result: - case <-time.After(3 * verifyLeaderTimeout): - t.Fatal("verifyLeaderEngineCtx did not bound a deadline-less context") - } + err = testutil.Recv(t, result, 3*verifyLeaderTimeout, "verifyLeaderEngineCtx did not bound a deadline-less context") elapsed := time.Since(start) if err == nil { diff --git a/kv/sharded_coordinator_registration_gate_test.go b/kv/sharded_coordinator_registration_gate_test.go index 8facf30b6..7e81eb41f 100644 --- a/kv/sharded_coordinator_registration_gate_test.go +++ b/kv/sharded_coordinator_registration_gate_test.go @@ -5,6 +5,8 @@ import ( "errors" "testing" "time" + + "github.com/bootjp/elastickv/internal/testutil" ) // mutatingGroup is an OperationGroup with one PUT (a write that would @@ -196,13 +198,9 @@ func TestAwaitRegistrationBarrier_CtxCancelFailsClosed(t *testing.T) { done := make(chan error, 1) go func() { done <- c.awaitRegistrationBarrier(ctx) }() cancel() - select { - case err := <-done: - if !errors.Is(err, context.Canceled) { - t.Fatalf("expected ctx.Canceled fail-closed, got %v", err) - } - case <-time.After(2 * time.Second): - t.Fatal("commit did not fail-closed on ctx cancel") + err := testutil.Recv(t, done, 2*time.Second, "commit did not fail-closed on ctx cancel") + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected ctx.Canceled fail-closed, got %v", err) } } @@ -217,12 +215,8 @@ func TestAwaitRegistration_OpenBarrier_CtxCancelFailsClosed(t *testing.T) { done := make(chan error, 1) go func() { done <- c.awaitRegistration(ctx, mutatingGroup()) }() cancel() - select { - case err := <-done: - if !errors.Is(err, context.Canceled) { - t.Fatalf("expected ctx.Canceled fail-closed, got %v", err) - } - case <-time.After(2 * time.Second): - t.Fatal("write did not fail-closed on ctx cancel") + err := testutil.Recv(t, done, 2*time.Second, "write did not fail-closed on ctx cancel") + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected ctx.Canceled fail-closed, got %v", err) } } diff --git a/main_admin_lifecycle_test.go b/main_admin_lifecycle_test.go index 0aa1be5d8..b729b07e6 100644 --- a/main_admin_lifecycle_test.go +++ b/main_admin_lifecycle_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/adapter" "github.com/bootjp/elastickv/internal/admin" "github.com/cockroachdb/errors" @@ -56,11 +58,7 @@ func TestRegisterAdminLifecycleReturnsCleanAfterForcedClose(t *testing.T) { } reqDone <- err }() - select { - case <-started: - case <-time.After(5 * time.Second): - t.Fatal("admin handler did not start for the stalled request") - } + testutil.Recv(t, started, 5*time.Second, "admin handler did not start for the stalled request") start := time.Now() cancel() diff --git a/main_admin_shutdown_test.go b/main_admin_shutdown_test.go index 330a4811e..04ec627ce 100644 --- a/main_admin_shutdown_test.go +++ b/main_admin_shutdown_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/admin" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -24,11 +26,7 @@ func TestAdminHandlerDrainJoinsHandlerAfterForcedClose(t *testing.T) { <-entered done := make(chan struct{}) go func() { drain.join(time.Second); close(done) }() - select { - case <-done: - t.Fatal("join returned while the admitted handler was running") - case <-time.After(10 * time.Millisecond): - } + testutil.NoRecv(t, done, 10*time.Millisecond, "join returned while the admitted handler was running") close(release) select { case <-done: @@ -69,24 +67,12 @@ func TestAdminLifecycleWaitsForForceClosedHandler(t *testing.T) { _ = response.Body.Close() } }() - select { - case <-entered: - case <-time.After(time.Second): - t.Fatal("admin handler did not start") - } + testutil.Closed(t, entered, time.Second, "admin handler did not start") cancel() - select { - case <-canceled: - case <-time.After(time.Second): - t.Fatal("admin connection was not forced closed") - } + testutil.Closed(t, canceled, time.Second, "admin connection was not forced closed") done := make(chan error, 1) go func() { done <- group.Wait() }() - select { - case <-done: - t.Fatal("admin lifecycle returned before its handler unwound") - case <-time.After(10 * time.Millisecond): - } + testutil.NoRecv(t, done, 10*time.Millisecond, "admin lifecycle returned before its handler unwound") close(release) require.NoError(t, <-done) <-clientDone diff --git a/main_encryption_rotate_on_startup_test.go b/main_encryption_rotate_on_startup_test.go index 89edd057d..99913af0e 100644 --- a/main_encryption_rotate_on_startup_test.go +++ b/main_encryption_rotate_on_startup_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/adapter" "github.com/bootjp/elastickv/internal/encryption" "github.com/bootjp/elastickv/internal/encryption/fsmwire" @@ -292,16 +294,8 @@ func TestInstallEncryptionRotateOnStartupRequest_CallbackDoesNotBlockOnDone(t *t controller.becomeLeader() close(returned) }() - select { - case <-returned: - case <-time.After(100 * time.Millisecond): - t.Fatal("leader-acquired callback blocked before starting async rotation") - } - select { - case <-runner.doneEntered: - case <-time.After(2 * time.Second): - t.Fatal("async rotation did not reach Done") - } + testutil.Closed(t, returned, 100*time.Millisecond, "leader-acquired callback blocked before starting async rotation") + testutil.Closed(t, runner.doneEntered, 2*time.Second, "async rotation did not reach Done") } func TestInstallEncryptionRotateOnStartupRequest_WaitTracksAsyncFlight(t *testing.T) { @@ -317,11 +311,7 @@ func TestInstallEncryptionRotateOnStartupRequest_WaitTracksAsyncFlight(t *testin t.Fatalf("initial follower wait: %v", err) } controller.becomeLeader() - select { - case <-runner.started: - case <-time.After(2 * time.Second): - t.Fatal("async rotation did not start") - } + testutil.Closed(t, runner.started, 2*time.Second, "async rotation did not start") waitDone := make(chan error, 1) go func() { @@ -360,11 +350,7 @@ func TestInstallEncryptionRotateOnStartupRequest_WaitContinuesAfterConcurrentFir go func() { waitDone <- waitStartup.Wait(context.Background()) }() - select { - case <-runner.started: - case <-time.After(2 * time.Second): - t.Fatal("concurrent startup rotation did not start") - } + testutil.Closed(t, runner.started, 2*time.Second, "concurrent startup rotation did not start") select { case err := <-waitDone: t.Fatalf("startup wait returned before concurrent rotation finished: %v", err) @@ -395,11 +381,7 @@ func TestInstallEncryptionRotateOnStartupRequest_BlocksMutatorsDuringAsyncStartu t.Fatalf("initial follower wait: %v", err) } controller.becomeLeader() - select { - case <-runner.started: - case <-time.After(2 * time.Second): - t.Fatal("async startup rotation did not start") - } + testutil.Closed(t, runner.started, 2*time.Second, "async startup rotation did not start") if !waitStartup.BlockMutators() { t.Fatal("startup mutator gate must remain closed while async rotation is in flight") } @@ -407,11 +389,7 @@ func TestInstallEncryptionRotateOnStartupRequest_BlocksMutatorsDuringAsyncStartu close(runner.release) deadline := time.After(2 * time.Second) for waitStartup.BlockMutators() { - select { - case <-deadline: - t.Fatal("startup mutator gate did not open after async rotation finished") - case <-time.After(10 * time.Millisecond): - } + testutil.NoRecv(t, deadline, 10*time.Millisecond, "startup mutator gate did not open after async rotation finished") } } @@ -887,21 +865,13 @@ func TestEncryptionRotateOnStartupTask_DoneDoesNotWaitForRotateDEK(t *testing.T) go func() { runDone <- task.Run(context.Background()) }() - select { - case <-blocking.started: - case <-time.After(2 * time.Second): - t.Fatal("RotateDEK proposal did not start") - } + testutil.Closed(t, blocking.started, 2*time.Second, "RotateDEK proposal did not start") doneReturned := make(chan bool, 1) go func() { doneReturned <- task.Done() }() - select { - case got := <-doneReturned: - if got { - t.Fatal("task must not be done while RotateDEK is still in flight") - } - case <-time.After(100 * time.Millisecond): - t.Fatal("Done blocked behind in-flight RotateDEK") + got := testutil.Recv(t, doneReturned, 100*time.Millisecond, "Done blocked behind in-flight RotateDEK") + if got { + t.Fatal("task must not be done while RotateDEK is still in flight") } close(blocking.release) diff --git a/main_filesystem_test.go b/main_filesystem_test.go index fa6845311..fe1306ed6 100644 --- a/main_filesystem_test.go +++ b/main_filesystem_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + internalutil "github.com/bootjp/elastickv/internal" "github.com/bootjp/elastickv/internal/filesystem" "github.com/bootjp/elastickv/internal/filesystem/fstest" @@ -261,11 +263,7 @@ func TestMountFilesystemServiceServesKernelMount(t *testing.T) { require.Equal(t, []byte("probe"), got) require.NoError(t, unmount()) - select { - case <-serveDone: - case <-time.After(10 * time.Second): - t.Fatal("serve loop did not stop after unmount") - } + testutil.Closed(t, serveDone, 10*time.Second, "serve loop did not stop after unmount") } func TestInstallFilesystemServerLifecycleUnmountsOnContextCancel(t *testing.T) { @@ -361,12 +359,8 @@ func TestInstallFilesystemServerLifecycleDetachesLazilyWhenUnmountFails(t *testi waitDone := make(chan error, 1) go func() { waitDone <- eg.Wait() }() - select { - case err := <-waitDone: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("lifecycle did not stop after the lazy detach released the serve loop") - } + err := testutil.Recv(t, waitDone, 5*time.Second, "lifecycle did not stop after the lazy detach released the serve loop") + require.NoError(t, err) require.Equal(t, 1, server.lazyCalls) cleanup.Run() } @@ -390,12 +384,8 @@ func TestInstallFilesystemServerLifecycleBoundsWaitWhenServeLoopStalls(t *testin waitDone := make(chan error, 1) go func() { waitDone <- eg.Wait() }() - select { - case err := <-waitDone: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("lifecycle hung on a serve loop that never stops") - } + err := testutil.Recv(t, waitDone, 5*time.Second, "lifecycle hung on a serve loop that never stops") + require.NoError(t, err) require.Equal(t, 1, server.lazyCalls) cleanup.Run() } @@ -418,12 +408,8 @@ func TestInstallFilesystemServerLifecycleBoundsHungUnmount(t *testing.T) { waitDone := make(chan error, 1) go func() { waitDone <- eg.Wait() }() - select { - case err := <-waitDone: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("lifecycle hung on an Unmount that never returns") - } + err := testutil.Recv(t, waitDone, 5*time.Second, "lifecycle hung on an Unmount that never returns") + require.NoError(t, err) require.Equal(t, 1, server.lazyCalls) // The CleanupStack entry must not re-wait on the unmount that is still // stuck: the lifecycle already gave up on it, and the WAL sync and store @@ -433,11 +419,7 @@ func TestInstallFilesystemServerLifecycleBoundsHungUnmount(t *testing.T) { cleanup.Run() close(cleanupDone) }() - select { - case <-cleanupDone: - case <-time.After(5 * time.Second): - t.Fatal("cleanup blocked on the hung unmount") - } + testutil.Closed(t, cleanupDone, 5*time.Second, "cleanup blocked on the hung unmount") release() } @@ -473,12 +455,8 @@ func TestInstallFilesystemServerLifecycleWaitsForServeLoopAfterHungUnmount(t *te waitDone := make(chan error, 1) go func() { waitDone <- eg.Wait() }() - select { - case err := <-waitDone: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("lifecycle did not stop") - } + err := testutil.Recv(t, waitDone, 5*time.Second, "lifecycle did not stop") + require.NoError(t, err) elapsed := time.Since(start) require.Equal(t, 1, server.lazyCalls) // Past the 4/5 unmount wait (1.6 s), but well before the full budget: @@ -509,11 +487,7 @@ func TestInstallFilesystemServerLifecycleCleanupBoundsHungUnmount(t *testing.T) cleanup.Run() close(cleanupDone) }() - select { - case <-cleanupDone: - case <-time.After(5 * time.Second): - t.Fatal("cleanup blocked on the hung unmount") - } + testutil.Closed(t, cleanupDone, 5*time.Second, "cleanup blocked on the hung unmount") // Teardown: let the lifecycle goroutine finish. Cancel before the serve // loop ends, as a real shutdown does; the other order is the "stopped @@ -564,12 +538,8 @@ func TestStartFilesystemPlacementCollectorStopsOnCancel(t *testing.T) { waitDone := make(chan error, 1) go func() { waitDone <- eg.Wait() }() - select { - case err := <-waitDone: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("placement collector did not stop after cancellation") - } + err := testutil.Recv(t, waitDone, 5*time.Second, "placement collector did not stop after cancellation") + require.NoError(t, err) } func TestFilesystemBackgroundLoopsSkipWhenDisabled(t *testing.T) { diff --git a/main_grpc_stop_test.go b/main_grpc_stop_test.go index f5710182f..ba8ff0d4a 100644 --- a/main_grpc_stop_test.go +++ b/main_grpc_stop_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -46,11 +48,7 @@ func TestStopGRPCServerBoundedForcesStopOnLongLivedStream(t *testing.T) { stream, err := conn.NewStream(ctx, &grpc.StreamDesc{StreamName: "Watch", ServerStreams: true, ClientStreams: true}, "/elastickv.test.LongLived/Watch") require.NoError(t, err) require.NotNil(t, stream) - select { - case <-started: - case <-time.After(5 * time.Second): - t.Fatal("server handler did not start for the long-lived stream") - } + testutil.Recv(t, started, 5*time.Second, "server handler did not start for the long-lived stream") start := time.Now() stopGRPCServerBounded(context.Background(), srv, "test") @@ -115,11 +113,7 @@ func openLongLivedStream(t *testing.T, addr string, started <-chan struct{}) { t.Cleanup(streamCancel) _, err = conn.NewStream(streamCtx, &grpc.StreamDesc{StreamName: "Watch", ServerStreams: true, ClientStreams: true}, "/elastickv.test.LongLived/Watch") require.NoError(t, err) - select { - case <-started: - case <-time.After(5 * time.Second): - t.Fatal("server handler did not start for the long-lived stream") - } + testutil.Recv(t, started, 5*time.Second, "server handler did not start for the long-lived stream") } // TestGRPCServeReturnsOnlyAfterBoundedStop pins the property the per-group @@ -153,11 +147,7 @@ func TestGRPCServeReturnsOnlyAfterBoundedStop(t *testing.T) { t.Fatalf("Serve returned (%v) while the bounded stop was still draining", err) case <-time.After(100 * time.Millisecond): } - select { - case <-stopDone: - case <-time.After(5 * time.Second): - t.Fatal("the bounded stop did not finish") - } + testutil.Closed(t, stopDone, 5*time.Second, "the bounded stop did not finish") select { case err := <-serveDone: require.NoError(t, err) diff --git a/main_shutdown_balance_test.go b/main_shutdown_balance_test.go index a39beba7e..1ed047cc1 100644 --- a/main_shutdown_balance_test.go +++ b/main_shutdown_balance_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -45,19 +47,11 @@ func TestShutdownJoinsBalancerBeforeHandoff(t *testing.T) { shutdownAfterRequest(ctx, cancel, []*raftGroupRuntime{{engine: engine}}, requests, time.Second, func(ctx context.Context) error { return stopLeaderBalanceForShutdown(ctx, stopBalance, balanceDone) }) }() - select { - case <-stopping: - case <-time.After(shutdownTestTimeout): - t.Fatal("balancing was not cancelled") - } + testutil.Closed(t, stopping, shutdownTestTimeout, "balancing was not cancelled") require.NoError(t, ctx.Err()) require.Zero(t, transfers.Load()) unblock() - select { - case <-finished: - case <-time.After(shutdownTestTimeout): - t.Fatal("handoff did not finish after balancing joined") - } + testutil.Closed(t, finished, shutdownTestTimeout, "handoff did not finish after balancing joined") require.Equal(t, int32(1), transfers.Load()) require.ErrorIs(t, ctx.Err(), context.Canceled) } diff --git a/main_shutdown_handoff_test.go b/main_shutdown_handoff_test.go index 85a42a27b..d30b37c1e 100644 --- a/main_shutdown_handoff_test.go +++ b/main_shutdown_handoff_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -22,11 +24,7 @@ func TestShutdownHandoffStartupSignalCancelsBeforeArming(t *testing.T) { stop := installShutdownSignalHandler(handoff.requestShutdown, syscall.SIGUSR2) t.Cleanup(stop) require.NoError(t, syscall.Kill(os.Getpid(), syscall.SIGUSR2)) - select { - case <-ctx.Done(): - case <-time.After(shutdownTestTimeout): - t.Fatal("a signal during startup must cancel immediately") - } + testutil.Closed(t, ctx.Done(), shutdownTestTimeout, "a signal during startup must cancel immediately") eg, runCtx := errgroup.WithContext(ctx) handoff.arm(runCtx, eg, nil, nil) require.NoError(t, eg.Wait(), "arming after startup cancellation must not wait for another signal") @@ -56,18 +54,10 @@ func TestShutdownHandoffSignalWaitsForTransferAndIgnoresRepeats(t *testing.T) { eg, runCtx := errgroup.WithContext(ctx) handoff.arm(runCtx, eg, []*raftGroupRuntime{{spec: groupSpec{id: 1}, engine: engine}}, nil) require.NoError(t, syscall.Kill(os.Getpid(), syscall.SIGUSR2)) - select { - case <-started: - case <-time.After(shutdownTestTimeout): - t.Fatal("the signal did not start the leadership handoff") - } + testutil.Closed(t, started, shutdownTestTimeout, "the signal did not start the leadership handoff") require.NoError(t, ctx.Err(), "root cancellation must wait until the handoff finishes") require.NoError(t, syscall.Kill(os.Getpid(), syscall.SIGUSR2)) - select { - case <-ctx.Done(): - t.Fatal("a repeated signal canceled the in-flight handoff") - case <-time.After(shutdownTransferRetryDelay): - } + testutil.NoRecv(t, ctx.Done(), shutdownTransferRetryDelay, "a repeated signal canceled the in-flight handoff") release() done := make(chan error, 1) go func() { done <- eg.Wait() }() diff --git a/main_shutdown_signal_log_test.go b/main_shutdown_signal_log_test.go index 21c9389de..b04b6dc71 100644 --- a/main_shutdown_signal_log_test.go +++ b/main_shutdown_signal_log_test.go @@ -7,7 +7,8 @@ import ( "sync" "syscall" "testing" - "time" + + "github.com/bootjp/elastickv/internal/testutil" "github.com/stretchr/testify/require" ) @@ -47,11 +48,7 @@ func TestInstallShutdownSignalHandlerCancelsBeforeLogging(t *testing.T) { }) require.NoError(t, syscall.Kill(os.Getpid(), syscall.SIGUSR2)) - select { - case <-ctx.Done(): - case <-time.After(shutdownTestTimeout): - t.Fatal("a blocked log write held back the shutdown") - } + testutil.Closed(t, ctx.Done(), shutdownTestTimeout, "a blocked log write held back the shutdown") } // The cleanup that stops the handler must not wait forever on a goroutine @@ -71,19 +68,11 @@ func TestInstallShutdownSignalHandlerStopDoesNotWaitOnBlockedLog(t *testing.T) { }) require.NoError(t, syscall.Kill(os.Getpid(), syscall.SIGUSR2)) - select { - case <-ctx.Done(): - case <-time.After(shutdownTestTimeout): - t.Fatal("the signal did not cancel the shutdown") - } + testutil.Closed(t, ctx.Done(), shutdownTestTimeout, "the signal did not cancel the shutdown") stopped := make(chan struct{}) go func() { stop() close(stopped) }() - select { - case <-stopped: - case <-time.After(shutdownTestTimeout): - t.Fatal("stopping the handler waited on the goroutine stuck in the log write") - } + testutil.Closed(t, stopped, shutdownTestTimeout, "stopping the handler waited on the goroutine stuck in the log write") } diff --git a/main_shutdown_signal_test.go b/main_shutdown_signal_test.go index 62f1978da..814c612cc 100644 --- a/main_shutdown_signal_test.go +++ b/main_shutdown_signal_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + internalutil "github.com/bootjp/elastickv/internal" "github.com/cockroachdb/errors" "github.com/hanwen/go-fuse/v2/fuse" @@ -38,11 +40,7 @@ func TestInstallShutdownSignalHandlerCancelsOnSignal(t *testing.T) { t.Cleanup(stop) require.NoError(t, syscall.Kill(os.Getpid(), syscall.SIGTERM)) - select { - case <-ctx.Done(): - case <-time.After(shutdownTestTimeout): - t.Fatal("SIGTERM did not cancel the root context") - } + testutil.Closed(t, ctx.Done(), shutdownTestTimeout, "SIGTERM did not cancel the root context") require.True(t, signalShutdown.Load()) // A repeat signal while the shutdown is in flight must stay with the diff --git a/main_shutdown_test.go b/main_shutdown_test.go index 2255afffc..de4d7bb2c 100644 --- a/main_shutdown_test.go +++ b/main_shutdown_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/stretchr/testify/require" ) @@ -140,11 +142,7 @@ func TestShutdownSignalSharesDeadlineAcrossGroups(t *testing.T) { t.Fatal("all transfers must start before the shared deadline") } } - select { - case <-done: - case <-time.After(2 * time.Second): - t.Fatal("shutdown did not finish after the transfer deadline") - } + testutil.Closed(t, done, 2*time.Second, "shutdown did not finish after the transfer deadline") require.ErrorIs(t, ctx.Err(), context.Canceled) } diff --git a/monitoring/http_shutdown_test.go b/monitoring/http_shutdown_test.go index 703d75661..e678fa297 100644 --- a/monitoring/http_shutdown_test.go +++ b/monitoring/http_shutdown_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) @@ -49,11 +51,7 @@ func startStalledRequest(t *testing.T, addr string, started <-chan struct{}) <-c } done <- err }() - select { - case <-started: - case <-time.After(5 * time.Second): - t.Fatal("handler did not start for the stalled request") - } + testutil.Recv(t, started, 5*time.Second, "handler did not start for the stalled request") return done } @@ -96,11 +94,7 @@ func TestShutdownHTTPServerReturnsPromptlyWhenIdle(t *testing.T) { start := time.Now() require.NoError(t, ShutdownHTTPServer(context.Background(), srv, "idle", 5*time.Second)) require.Less(t, time.Since(start), 2*time.Second) - select { - case <-serveDone: - case <-time.After(5 * time.Second): - t.Fatal("Serve did not return after the shutdown") - } + testutil.Recv(t, serveDone, 5*time.Second, "Serve did not return after the shutdown") } func TestShutdownHTTPServerNilIsNoop(t *testing.T) { diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 2a2e7363d..5db6c95f8 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -13,6 +13,8 @@ import ( "testing" "time" + waitutil "github.com/bootjp/elastickv/internal/testutil" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/redis/go-redis/v9" @@ -1392,12 +1394,8 @@ func TestDualWriter_GoAsync_QueuesBurstBeforeDropping(t *testing.T) { close(done) }() - select { - case <-done: - // good - case <-time.After(time.Second): - t.Fatal("goAsync blocked when queue was full") - } + waitutil.Closed(t, done, time.Second, "goAsync blocked when queue was full") + // good assert.InDelta(t, 1, testutil.ToFloat64(metrics.AsyncDrops), 0.001) assert.InDelta(t, 1, testutil.ToFloat64( @@ -1479,12 +1477,8 @@ func TestDualWriter_Script_QueuesWhenScriptLimitIsBusy(t *testing.T) { close(done) }() - select { - case <-done: - // good - case <-time.After(time.Second): - t.Fatal("Script blocked while its replay was queued") - } + waitutil.Closed(t, done, time.Second, "Script blocked while its replay was queued") + // good assert.Equal(t, 0, secondary.CallCount()) assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncDrops), 0.001) @@ -1523,18 +1517,10 @@ func TestDualWriter_ScriptConsumesSharedWriteConcurrency(t *testing.T) { scriptRan := make(chan struct{}) d.goScript(func(context.Context) { close(scriptRan) }) - select { - case <-scriptRan: - t.Fatal("script exceeded the shared secondary write concurrency") - case <-time.After(50 * time.Millisecond): - } + waitutil.NoRecv(t, scriptRan, 50*time.Millisecond, "script exceeded the shared secondary write concurrency") close(blocker) - select { - case <-scriptRan: - case <-time.After(time.Second): - t.Fatal("script did not run after shared capacity became available") - } + waitutil.Closed(t, scriptRan, time.Second, "script did not run after shared capacity became available") d.Close() } @@ -1603,11 +1589,7 @@ func TestDualWriter_ScriptQueueUsesScriptTimeout(t *testing.T) { d.goScript(func(context.Context) { close(ran) }) time.Sleep(50 * time.Millisecond) close(blocker) - select { - case <-ran: - case <-time.After(time.Second): - t.Fatal("script expired before secondary-script-timeout") - } + waitutil.Closed(t, ran, time.Second, "script expired before secondary-script-timeout") d.Close() assert.InDelta(t, 0, testutil.ToFloat64( @@ -1792,11 +1774,7 @@ func TestDualWriter_ExpiredQueuedWorkDoesNotWaitForSemaphore(t *testing.T) { close(done) }() - select { - case <-done: - case <-time.After(200 * time.Millisecond): - t.Fatal("expired task waited for an unavailable semaphore") - } + waitutil.Closed(t, done, 200*time.Millisecond, "expired task waited for an unavailable semaphore") assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncQueueDepth.WithLabelValues(asyncQueueWrite)), 0.001) assert.InDelta(t, 1, testutil.ToFloat64( diff --git a/store/lsm_store_test.go b/store/lsm_store_test.go index ba0dade77..2a3ce76e0 100644 --- a/store/lsm_store_test.go +++ b/store/lsm_store_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/testutil" + "github.com/bootjp/elastickv/internal/encryption" "github.com/cockroachdb/pebble/v2" "github.com/stretchr/testify/assert" @@ -46,17 +48,9 @@ func countPebbleVersions(t *testing.T, st MVCCStore) int { func requireChannelBlocked[T any](t *testing.T, ch <-chan T, started <-chan struct{}) { t.Helper() - select { - case <-started: - case <-time.After(time.Second): - t.Fatal("goroutine did not start in time") - } + testutil.Recv(t, started, time.Second, "goroutine did not start in time") - select { - case <-ch: - t.Fatal("operation completed while maintenance lock was held") - case <-time.After(50 * time.Millisecond): - } + testutil.NoRecv(t, ch, 50*time.Millisecond, "operation completed while maintenance lock was held") } type cancelAfterErrCallsContext struct { From 9883bdd1032a941d7c47fc054039a49f8c0b9d87 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 10:20:34 +0000 Subject: [PATCH 10/15] test: build encryption fixtures with encryptiontest (cleanup WP-9..12) Delete store/encryption_test_helpers.go, which exported a test-only CounterNonceFactory from production store; its byte layout matches encryption.NewDeterministicNonceFactory, which the callers now use. Replace the random-DEK keystore+cipher boilerplate with encryptiontest.NewCipher (fixed-byte DEKs are kept), and the temp-dir sidecar writers, including the byte-identical writeSidecarFixture and writeStatusFixture, with encryptiontest.WriteSidecar. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- adapter/encryption_admin_test.go | 56 ++++++++----------- cmd/elastickv-admin/encryption_test.go | 14 +---- internal/encryption/nonce_factory.go | 5 +- internal/raftengine/etcd/encryption_test.go | 14 +---- kv/fsm_dynamodb_timestamp_collision_test.go | 2 +- kv/fsm_restore_origin_encrypted_test.go | 2 +- kv/raft_payload_wrapper_test.go | 14 +---- main_encryption_startup_guard_test.go | 22 ++------ main_encryption_write_wiring_test.go | 8 +-- store/encryption_compression_test.go | 46 +++------------ store/encryption_glue.go | 2 +- store/encryption_test_helpers.go | 52 ----------------- store/lsm_store_encryption_metrics_test.go | 23 ++------ store/lsm_store_encryption_prop_test.go | 17 ++---- store/lsm_store_encryption_test.go | 12 ++-- store/lsm_store_registration_gate_test.go | 6 +- store/lsm_store_sync_mode_benchmark_test.go | 16 +----- ...ation_ownership_partition_boundary_test.go | 3 +- ...tion_ownership_partition_namespace_test.go | 3 +- ...ation_ownership_partition_snapshot_test.go | 3 +- store/snapshot_external_encryption_test.go | 3 +- store/snapshot_external_user_keys_test.go | 3 +- store/snapshot_pebble_sst_test.go | 2 +- 23 files changed, 78 insertions(+), 250 deletions(-) delete mode 100644 store/encryption_test_helpers.go diff --git a/adapter/encryption_admin_test.go b/adapter/encryption_admin_test.go index 48bb14a04..a82a709ab 100644 --- a/adapter/encryption_admin_test.go +++ b/adapter/encryption_admin_test.go @@ -16,6 +16,7 @@ import ( "github.com/bootjp/elastickv/internal/admin" "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/encryptiontest" "github.com/bootjp/elastickv/internal/encryption/fsmwire" "github.com/bootjp/elastickv/internal/raftengine" pb "github.com/bootjp/elastickv/proto" @@ -80,7 +81,7 @@ func TestEncryptionAdmin_GetCapability_SidecarMissing(t *testing.T) { func TestEncryptionAdmin_GetCapability_NotBootstrapped(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ // Active.Storage == 0 means "not bootstrapped" per §5.1. Active: encryption.ActiveKeys{}, Keys: map[string]encryption.SidecarKey{}, @@ -110,7 +111,7 @@ func TestEncryptionAdmin_GetCapability_NotBootstrapped(t *testing.T) { func TestEncryptionAdmin_GetCapability_Bootstrapped(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 1, Raft: 2}, Keys: map[string]encryption.SidecarKey{ "1": {Purpose: "storage", Wrapped: []byte("wrapped-1")}, @@ -152,7 +153,7 @@ func TestEncryptionAdmin_GetSidecarState_NoSidecarPath(t *testing.T) { func TestEncryptionAdmin_GetSidecarState_ShipsWrappedDEKs(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ RaftAppliedIndex: 42, StorageEnvelopeActive: true, RaftEnvelopeCutoverIndex: 100, @@ -174,7 +175,7 @@ func TestEncryptionAdmin_GetSidecarState_ShipsWrappedDEKs(t *testing.T) { func TestEncryptionAdmin_GetSidecarState_ProjectsLocalWriterRegistry(t *testing.T) { t.Parallel() const callerFullNodeID uint64 = 0xCAFE_BABE_0000_1234 - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ RaftAppliedIndex: 1, Active: encryption.ActiveKeys{Storage: 10, Raft: 11}, Keys: map[string]encryption.SidecarKey{ @@ -205,7 +206,7 @@ func TestEncryptionAdmin_GetSidecarState_ProjectsLocalWriterRegistry(t *testing. func TestEncryptionAdmin_GetSidecarState_OmitsRegistryProjectionOnFollower(t *testing.T) { t.Parallel() const callerFullNodeID uint64 = 0xCAFE_BABE_0000_1234 - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ RaftAppliedIndex: 42, StorageEnvelopeActive: true, RaftEnvelopeCutoverIndex: 100, @@ -280,7 +281,7 @@ func TestEncryptionAdmin_RegistryProjectionUsesAppliedLinearizableRead(t *testin for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 10, Raft: 11}, Keys: map[string]encryption.SidecarKey{ "10": {Purpose: "storage", Wrapped: []byte("storage")}, @@ -350,7 +351,7 @@ func runEncryptionAdminRegistryProjectionFenceAfterSidecarTest( call func(*EncryptionAdminServer) (uint32, uint32, map[uint32][]byte, map[uint32]uint32, error), ) { t.Helper() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 10, Raft: 11}, Keys: map[string]encryption.SidecarKey{ "10": {Purpose: "storage", Wrapped: []byte("old-storage")}, @@ -445,7 +446,7 @@ func assertSidecarWrappedDEKs(t *testing.T, got *pb.SidecarStateReport) { func TestEncryptionAdmin_GetSidecarState_CallbackOverridesSidecarIndex(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ RaftAppliedIndex: 10, Active: encryption.ActiveKeys{Storage: 1, Raft: 2}, Keys: map[string]encryption.SidecarKey{ @@ -468,7 +469,7 @@ func TestEncryptionAdmin_GetSidecarState_CallbackOverridesSidecarIndex(t *testin func TestEncryptionAdmin_ResyncSidecar_ShipsWrappedDEKs(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ RaftAppliedIndex: 17, Active: encryption.ActiveKeys{Storage: 3, Raft: 4}, Keys: map[string]encryption.SidecarKey{ @@ -511,7 +512,7 @@ func TestEncryptionAdmin_ResyncSidecar_ShipsWrappedDEKs(t *testing.T) { func TestEncryptionAdmin_ResyncSidecar_ProjectsCallerWriterRegistry(t *testing.T) { t.Parallel() const callerFullNodeID uint64 = 0xDEAD_BEEF_0000_8888 - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ RaftAppliedIndex: 17, Active: encryption.ActiveKeys{Storage: 3, Raft: 4}, Keys: map[string]encryption.SidecarKey{ @@ -541,7 +542,7 @@ func TestEncryptionAdmin_ResyncSidecar_ProjectsCallerWriterRegistry(t *testing.T func TestEncryptionAdmin_ResyncSidecar_UsesRegistryLeaderView(t *testing.T) { t.Parallel() const callerFullNodeID uint64 = 0xDEAD_BEEF_0000_8888 - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 3, Raft: 4}, Keys: map[string]encryption.SidecarKey{ "3": {Purpose: "storage", Wrapped: []byte("ws")}, @@ -574,7 +575,7 @@ func TestEncryptionAdmin_ResyncSidecar_UsesRegistryLeaderView(t *testing.T) { func TestEncryptionAdmin_ResyncSidecar_RejectsMissingCallerID(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 3, Raft: 4}, Keys: map[string]encryption.SidecarKey{ "3": {Purpose: "storage", Wrapped: []byte("ws")}, @@ -596,7 +597,7 @@ func TestEncryptionAdmin_ResyncSidecar_RejectsWriterRegistryCollision(t *testing t.Parallel() const callerFullNodeID uint64 = 0x0000_0000_0000_ABCD const collidingFullNodeID uint64 = 0x9999_9999_9999_ABCD - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 3, Raft: 4}, Keys: map[string]encryption.SidecarKey{ "3": {Purpose: "storage", Wrapped: []byte("ws")}, @@ -1265,7 +1266,7 @@ func TestEncryptionAdmin_RotateDEK_RejectsStaleLeader(t *testing.T) { // stale leader's outdated sidecar contents. func TestEncryptionAdmin_ResyncSidecar_RejectsStaleLeader(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 1, Raft: 2}, Keys: map[string]encryption.SidecarKey{ "1": {Purpose: "storage", Wrapped: []byte("w")}, @@ -1447,7 +1448,7 @@ func TestEncryptionAdmin_RegisterEncryptionWriter_RejectsBatch(t *testing.T) { func TestEncryptionAdmin_ResyncSidecar_RejectsOnFollower(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 1, Raft: 2}, Keys: map[string]encryption.SidecarKey{ "1": {Purpose: "storage", Wrapped: []byte("w")}, @@ -1531,19 +1532,6 @@ func (s stubLeaderView) LinearizableRead(ctx context.Context) (uint64, error) { return s.linearizableReadIndex, s.linearizableReadErr } -// writeSidecarFixture builds a valid keys.json fixture in a tmpdir and -// returns the path. The caller's *Sidecar is shallow-copied before -// WriteSidecar so the fixture's Version is set automatically. -func writeSidecarFixture(t *testing.T, sc *encryption.Sidecar) string { - t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, "keys.json") - if err := encryption.WriteSidecar(path, sc); err != nil { - t.Fatalf("WriteSidecar: %v", err) - } - return path -} - // fixedCapabilityFanout returns a closure that yields the supplied // result regardless of context — lets tests drive the §4 fan-out // branches deterministically without spinning real clients. A @@ -1661,7 +1649,7 @@ func allOKFanoutResult() admin.CapabilityFanoutResult { // + re-read it afterwards to verify the apply effect. func cutoverReadySidecarFixture(t *testing.T) string { t.Helper() - return writeSidecarFixture(t, &encryption.Sidecar{ + return encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 5, Raft: 6}, Keys: map[string]encryption.SidecarKey{ "5": {Purpose: encryption.SidecarPurposeStorage, Wrapped: []byte("ws"), Created: "x", LocalEpoch: 0}, @@ -1750,7 +1738,7 @@ func TestEncryptionAdmin_EnableStorageEnvelope_RejectsOversizedLocalEpoch(t *tes // refuse with a clear hint. func TestEncryptionAdmin_EnableStorageEnvelope_RejectsNotBootstrapped(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 0, Raft: 0}, Keys: map[string]encryption.SidecarKey{}, }) @@ -1778,7 +1766,7 @@ func TestEncryptionAdmin_EnableStorageEnvelope_RejectsNotBootstrapped(t *testing // reflect current cluster shape). func TestEncryptionAdmin_EnableStorageEnvelope_IdempotentRetry(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 5, Raft: 6}, Keys: map[string]encryption.SidecarKey{"5": {Purpose: encryption.SidecarPurposeStorage, Wrapped: []byte("ws"), Created: "x", LocalEpoch: 0}, "6": {Purpose: encryption.SidecarPurposeRaft, Wrapped: []byte("wr"), Created: "x", LocalEpoch: 0}}, StorageEnvelopeActive: true, @@ -1825,7 +1813,7 @@ func TestEncryptionAdmin_EnableStorageEnvelope_IdempotentRetry(t *testing.T) { // CutoverIndexUnknown=true so operators see the warning. func TestEncryptionAdmin_EnableStorageEnvelope_DefensiveCutoverIndexUnknown(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 5, Raft: 6}, Keys: map[string]encryption.SidecarKey{"5": {Purpose: encryption.SidecarPurposeStorage, Wrapped: []byte("ws"), Created: "x", LocalEpoch: 0}, "6": {Purpose: encryption.SidecarPurposeRaft, Wrapped: []byte("wr"), Created: "x", LocalEpoch: 0}}, StorageEnvelopeActive: true, @@ -2537,7 +2525,7 @@ func TestEncryptionAdmin_EnableRaftEnvelope_RejectsOversizedLocalEpoch(t *testin // refuse. func TestEncryptionAdmin_EnableRaftEnvelope_RejectsNotBootstrapped(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 0, Raft: 0}, Keys: map[string]encryption.SidecarKey{}, }) @@ -2567,7 +2555,7 @@ func TestEncryptionAdmin_EnableRaftEnvelope_RejectsNotBootstrapped(t *testing.T) // narrower than the storage variant's idempotent shape. func TestEncryptionAdmin_EnableRaftEnvelope_IdempotentRetry(t *testing.T) { t.Parallel() - path := writeSidecarFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ Active: encryption.ActiveKeys{Storage: 5, Raft: 6}, Keys: map[string]encryption.SidecarKey{"5": {Purpose: encryption.SidecarPurposeStorage, Wrapped: []byte("ws"), Created: "x", LocalEpoch: 0}, "6": {Purpose: encryption.SidecarPurposeRaft, Wrapped: []byte("wr"), Created: "x", LocalEpoch: 0}}, RaftEnvelopeCutoverIndex: 777, diff --git a/cmd/elastickv-admin/encryption_test.go b/cmd/elastickv-admin/encryption_test.go index 2e3353107..d6981d024 100644 --- a/cmd/elastickv-admin/encryption_test.go +++ b/cmd/elastickv-admin/encryption_test.go @@ -4,13 +4,13 @@ import ( "bytes" "context" "net" - "path/filepath" "strings" "testing" "time" "github.com/bootjp/elastickv/adapter" "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/encryptiontest" pb "github.com/bootjp/elastickv/proto" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -63,7 +63,7 @@ func TestRunEncryptionStatus_HelpFlagExitsZero(t *testing.T) { func TestRunEncryptionStatus_Bootstrapped(t *testing.T) { t.Parallel() - path := writeStatusFixture(t, &encryption.Sidecar{ + path := encryptiontest.WriteSidecar(t, &encryption.Sidecar{ RaftAppliedIndex: 9, StorageEnvelopeActive: true, RaftEnvelopeCutoverIndex: 25, @@ -782,16 +782,6 @@ func TestRunEncryptionRegisterWriter_HappyPath(t *testing.T) { } } -func writeStatusFixture(t *testing.T, sc *encryption.Sidecar) string { - t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, "keys.json") - if err := encryption.WriteSidecar(path, sc); err != nil { - t.Fatalf("WriteSidecar: %v", err) - } - return path -} - // startEncryptionAdminTestServer brings up a real gRPC server on a // loopback listener so the CLI dial path (grpc.NewClient against a // host:port string) exercises the same code as production. The diff --git a/internal/encryption/nonce_factory.go b/internal/encryption/nonce_factory.go index 64fa6aba8..ab9f2d1b9 100644 --- a/internal/encryption/nonce_factory.go +++ b/internal/encryption/nonce_factory.go @@ -33,9 +33,8 @@ import ( // restarts is a correctness bug. Always pair NewDeterministicNonceFactory // with a fresh BumpLocalEpoch on the active storage DEK. // -// This is the durable analogue of the test-only -// store.CounterNonceFactory: identical byte layout, but the epoch -// here carries the restart-safety guarantee the test factory lacks. +// Tests construct it directly with a fixed (nodeID, localEpoch); only +// production wiring pairs it with BumpLocalEpoch for restart safety. type DeterministicNonceFactory struct { nodeID uint16 localEpoch uint16 diff --git a/internal/raftengine/etcd/encryption_test.go b/internal/raftengine/etcd/encryption_test.go index d0c8588d7..94970e725 100644 --- a/internal/raftengine/etcd/encryption_test.go +++ b/internal/raftengine/etcd/encryption_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/encryptiontest" "github.com/cockroachdb/errors" raftpb "go.etcd.io/raft/v3/raftpb" ) @@ -36,19 +37,8 @@ func (f *fakeStateMachine) Restore(_ io.Reader) error { return nil } // with one key is enough to exercise the apply hook's gate logic. func raftCipherFixture(t *testing.T) (*encryption.Cipher, uint32) { t.Helper() - ks := encryption.NewKeystore() - dek := make([]byte, encryption.KeySize) - if _, err := rand.Read(dek); err != nil { - t.Fatalf("rand.Read DEK: %v", err) - } const kid uint32 = 0xDEADBEEF - if err := ks.Set(kid, dek); err != nil { - t.Fatalf("Set DEK: %v", err) - } - c, err := encryption.NewCipher(ks) - if err != nil { - t.Fatalf("NewCipher: %v", err) - } + c := encryptiontest.NewCipher(t, kid) return c, kid } diff --git a/kv/fsm_dynamodb_timestamp_collision_test.go b/kv/fsm_dynamodb_timestamp_collision_test.go index 3c6021a17..5b2d05f6a 100644 --- a/kv/fsm_dynamodb_timestamp_collision_test.go +++ b/kv/fsm_dynamodb_timestamp_collision_test.go @@ -23,7 +23,7 @@ func TestDynamoAppendDistinctRaftEntryCannotAcknowledgeTimestampCollision(t *tes require.NoError(t, keys.Set(1, bytes.Repeat([]byte{0x37}, encryption.KeySize))) cipher, err := encryption.NewCipher(keys) require.NoError(t, err) - opts = append(opts, store.WithEncryption(cipher, store.NewCounterNonceFactory(1, 1), func() (uint32, bool) { return 1, true })) + opts = append(opts, store.WithEncryption(cipher, encryption.NewDeterministicNonceFactory(1, 1), func() (uint32, bool) { return 1, true })) } st, err := store.NewPebbleStore(filepath.Join(t.TempDir(), "fsm.db"), opts...) require.NoError(t, err) diff --git a/kv/fsm_restore_origin_encrypted_test.go b/kv/fsm_restore_origin_encrypted_test.go index 306729620..d746a07a5 100644 --- a/kv/fsm_restore_origin_encrypted_test.go +++ b/kv/fsm_restore_origin_encrypted_test.go @@ -14,7 +14,7 @@ import ( func encryptedRestoreStore(t *testing.T, cipher *encryption.Cipher, sst bool, nonceEpoch uint16) store.MVCCStore { t.Helper() st, err := store.NewPebbleStore(t.TempDir(), store.WithSSTIngestSnapshots(sst), - store.WithEncryption(cipher, store.NewCounterNonceFactory(1, nonceEpoch), func() (uint32, bool) { return 1, true })) + store.WithEncryption(cipher, encryption.NewDeterministicNonceFactory(1, nonceEpoch), func() (uint32, bool) { return 1, true })) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, st.Close()) }) return st diff --git a/kv/raft_payload_wrapper_test.go b/kv/raft_payload_wrapper_test.go index e4810253b..c44e58174 100644 --- a/kv/raft_payload_wrapper_test.go +++ b/kv/raft_payload_wrapper_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/encryptiontest" "github.com/bootjp/elastickv/internal/raftengine" ) @@ -103,19 +104,8 @@ func TestDynamicWrappedProposer_PropagatesWrapperError(t *testing.T) { // what the engine expects). func TestDynamicWrappedProposer_RoundTripWithRealCipher(t *testing.T) { t.Parallel() - ks := encryption.NewKeystore() - dek := make([]byte, encryption.KeySize) - if _, err := rand.Read(dek); err != nil { - t.Fatalf("rand.Read: %v", err) - } const kid uint32 = 0x42 - if err := ks.Set(kid, dek); err != nil { - t.Fatalf("Set: %v", err) - } - c, err := encryption.NewCipher(ks) - if err != nil { - t.Fatalf("NewCipher: %v", err) - } + c := encryptiontest.NewCipher(t, kid) var wrapPtr atomic.Pointer[RaftPayloadWrapper] var wrap RaftPayloadWrapper = func(p []byte) ([]byte, error) { diff --git a/main_encryption_startup_guard_test.go b/main_encryption_startup_guard_test.go index 89dc569d8..32efbbcbe 100644 --- a/main_encryption_startup_guard_test.go +++ b/main_encryption_startup_guard_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/encryptiontest" "github.com/bootjp/elastickv/internal/raftengine" etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" "github.com/bootjp/elastickv/kv" @@ -87,23 +88,16 @@ func TestCheckEnvelopeCutoverDivergenceSnapshotPayload_SidecarZeroRefusesSnapsho // returns the sidecar path. func writeMinimalSidecar(t *testing.T, raftAppliedIdx uint64) string { t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, encryption.SidecarFilename) sc := &encryption.Sidecar{ Version: encryption.SidecarVersion, RaftAppliedIndex: raftAppliedIdx, Keys: map[string]encryption.SidecarKey{}, } - if err := encryption.WriteSidecar(path, sc); err != nil { - t.Fatalf("WriteSidecar: %v", err) - } - return path + return encryptiontest.WriteSidecar(t, sc) } func writeRaftCutoverSidecarForStartup(t *testing.T, activeRaft uint32, localEpoch uint16, cutover uint64) string { t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, encryption.SidecarFilename) sc := &encryption.Sidecar{ Version: encryption.SidecarVersion, RaftAppliedIndex: cutover, @@ -124,16 +118,11 @@ func writeRaftCutoverSidecarForStartup(t *testing.T, activeRaft uint32, localEpo }, }, } - if err := encryption.WriteSidecar(path, sc); err != nil { - t.Fatalf("WriteSidecar: %v", err) - } - return path + return encryptiontest.WriteSidecar(t, sc) } func writeActiveStorageSidecarForStartup(t *testing.T, localEpoch uint16) string { t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, encryption.SidecarFilename) sc := &encryption.Sidecar{ Version: encryption.SidecarVersion, RaftAppliedIndex: 1, @@ -148,10 +137,7 @@ func writeActiveStorageSidecarForStartup(t *testing.T, localEpoch uint16) string }, }, } - if err := encryption.WriteSidecar(path, sc); err != nil { - t.Fatalf("WriteSidecar: %v", err) - } - return path + return encryptiontest.WriteSidecar(t, sc) } // TestCheckSidecarBehindRaftLog_DisabledNoop pins the diff --git a/main_encryption_write_wiring_test.go b/main_encryption_write_wiring_test.go index 74323b7b3..97cfd9a1b 100644 --- a/main_encryption_write_wiring_test.go +++ b/main_encryption_write_wiring_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/encryptiontest" ) // wiringFakeKEK is a deterministic KEKUnwrapper for the write-wiring @@ -21,8 +22,6 @@ func (wiringFakeKEK) Unwrap(wrapped []byte) ([]byte, error) { func writeActiveStorageSidecar(t *testing.T, storageEpoch uint16) string { t.Helper() - dir := t.TempDir() - path := dir + "/keys.json" sc := &encryption.Sidecar{ Version: encryption.SidecarVersion, StorageEnvelopeActive: true, @@ -32,10 +31,7 @@ func writeActiveStorageSidecar(t *testing.T, storageEpoch uint16) string { "4": {Purpose: encryption.SidecarPurposeRaft, Wrapped: []byte("raft-wrapped-bytes-diff"), LocalEpoch: 0}, }, } - if err := encryption.WriteSidecar(path, sc); err != nil { - t.Fatalf("WriteSidecar: %v", err) - } - return path + return encryptiontest.WriteSidecar(t, sc) } func markRaftEnvelopeCutoverInSidecar(t *testing.T, path string, cutover uint64) { diff --git a/store/encryption_compression_test.go b/store/encryption_compression_test.go index bdfe78165..a90da1fab 100644 --- a/store/encryption_compression_test.go +++ b/store/encryption_compression_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/encryptiontest" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/v2" "github.com/golang/snappy" @@ -70,23 +71,12 @@ func TestEncryption_CompressesOnlyWhenSmaller(t *testing.T) { func TestEncryption_StorageEnvelopeV2CapabilityGate(t *testing.T) { t.Parallel() - ks := encryption.NewKeystore() - dek := make([]byte, encryption.KeySize) - if _, err := rand.Read(dek); err != nil { - t.Fatalf("rand.Read DEK: %v", err) - } const keyID uint32 = 107 - if err := ks.Set(keyID, dek); err != nil { - t.Fatalf("Keystore.Set: %v", err) - } - cipher, err := encryption.NewCipher(ks) - if err != nil { - t.Fatalf("NewCipher: %v", err) - } + cipher := encryptiontest.NewCipher(t, keyID) v2Active := false s := &pebbleStore{ cipher: cipher, - nonceFactory: NewCounterNonceFactory(0x0102, 0x0304), + nonceFactory: encryption.NewDeterministicNonceFactory(0x0102, 0x0304), activeStorageKeyID: func() (uint32, bool) { return keyID, true }, storageEnvelopeV2Active: func() bool { return v2Active }, } @@ -273,22 +263,11 @@ func TestCompressionRoundTripProperty(t *testing.T) { func TestEncryptedStoreCompressionRoundTripProperty(t *testing.T) { t.Parallel() - ks := encryption.NewKeystore() - dek := make([]byte, encryption.KeySize) - if _, err := rand.Read(dek); err != nil { - t.Fatalf("rand.Read DEK: %v", err) - } const keyID uint32 = 105 - if err := ks.Set(keyID, dek); err != nil { - t.Fatalf("Keystore.Set: %v", err) - } - cipher, err := encryption.NewCipher(ks) - if err != nil { - t.Fatalf("NewCipher: %v", err) - } + cipher := encryptiontest.NewCipher(t, keyID) s := &pebbleStore{ cipher: cipher, - nonceFactory: NewCounterNonceFactory(0x0102, 0x0304), + nonceFactory: encryption.NewDeterministicNonceFactory(0x0102, 0x0304), activeStorageKeyID: func() (uint32, bool) { return keyID, true }, } rapid.Check(t, func(t *rapid.T) { @@ -346,21 +325,10 @@ func TestPebbleCompressionPolicy(t *testing.T) { // backed by a single freshly generated DEK. func newCompressionTestStore(t *testing.T, keyID uint32) *pebbleStore { t.Helper() - ks := encryption.NewKeystore() - dek := make([]byte, encryption.KeySize) - if _, err := rand.Read(dek); err != nil { - t.Fatalf("rand.Read DEK: %v", err) - } - if err := ks.Set(keyID, dek); err != nil { - t.Fatalf("Keystore.Set: %v", err) - } - cipher, err := encryption.NewCipher(ks) - if err != nil { - t.Fatalf("NewCipher: %v", err) - } + cipher := encryptiontest.NewCipher(t, keyID) return &pebbleStore{ cipher: cipher, - nonceFactory: NewCounterNonceFactory(0x0102, 0x0304), + nonceFactory: encryption.NewDeterministicNonceFactory(0x0102, 0x0304), activeStorageKeyID: func() (uint32, bool) { return keyID, true }, } } diff --git a/store/encryption_glue.go b/store/encryption_glue.go index d3529cb9d..c1572b189 100644 --- a/store/encryption_glue.go +++ b/store/encryption_glue.go @@ -119,7 +119,7 @@ const minEncryptionCompressionSize = 32 // the storage layer just calls Next() and uses what comes back. // // Stage 7 of the encryption rollout will replace the in-tree -// reference implementation (CounterNonceFactory) with a +// reference implementation (encryption.DeterministicNonceFactory) with a // writer-registry-backed factory that guarantees uniqueness across // voters, learners, and historical replicas. The interface stays // the same; only the construction changes. Implementations MUST diff --git a/store/encryption_test_helpers.go b/store/encryption_test_helpers.go deleted file mode 100644 index bcba822d3..000000000 --- a/store/encryption_test_helpers.go +++ /dev/null @@ -1,52 +0,0 @@ -package store - -import ( - "encoding/binary" - "sync/atomic" - - "github.com/bootjp/elastickv/internal/encryption" -) - -// CounterNonceFactory is a test-only NonceFactory that produces the -// design §4.1 deterministic nonce shape (`node_id ‖ local_epoch ‖ -// write_count`) without the writer-registry round-trip Stage 7 -// brings. Production wiring uses the registry-backed factory; this -// implementation is only safe for tests where the caller controls -// every node_id / local_epoch combination. -// -// Exposed (vs. living in a *_test.go file) so the encryption -// integration tests in other packages can build on the same -// implementation without re-deriving the byte layout. It is -// nevertheless test-grade — the doc comment on NonceFactory -// emphasises that production callers MUST guarantee -// (node_id, local_epoch, write_count) uniqueness. -type CounterNonceFactory struct { - nodeID uint16 - localEpoch uint16 - writes atomic.Uint64 -} - -// NewCounterNonceFactory constructs a CounterNonceFactory pinned to -// the given (nodeID, localEpoch). write_count starts at 0 and -// monotonically increments on every Next(). -func NewCounterNonceFactory(nodeID, localEpoch uint16) *CounterNonceFactory { - return &CounterNonceFactory{nodeID: nodeID, localEpoch: localEpoch} -} - -// Next produces the next 12-byte nonce. Layout matches design §4.1: -// -// bytes 0-1 node_id (big-endian uint16) -// bytes 2-3 local_epoch (big-endian uint16) -// bytes 4-11 write_count (big-endian uint64) -// -// Big-endian is chosen so a hex dump of consecutive nonces is -// human-readable as a counter; the AAD does NOT include the nonce -// bytes (the cipher composes the nonce into AES-GCM directly), so -// the byte order is internal to the factory. -func (f *CounterNonceFactory) Next() ([encryption.NonceSize]byte, error) { - var n [encryption.NonceSize]byte - binary.BigEndian.PutUint16(n[0:2], f.nodeID) - binary.BigEndian.PutUint16(n[2:4], f.localEpoch) - binary.BigEndian.PutUint64(n[4:12], f.writes.Add(1)) - return n, nil -} diff --git a/store/lsm_store_encryption_metrics_test.go b/store/lsm_store_encryption_metrics_test.go index 26c7a995e..b0e0e5268 100644 --- a/store/lsm_store_encryption_metrics_test.go +++ b/store/lsm_store_encryption_metrics_test.go @@ -2,12 +2,12 @@ package store import ( "context" - "crypto/rand" "path/filepath" "sync" "testing" "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/encryptiontest" "github.com/cockroachdb/pebble/v2" "github.com/stretchr/testify/require" ) @@ -74,7 +74,7 @@ func (f *observedEncryptedFixture) closeIfOpen(tb testing.TB) { func (f *observedEncryptedFixture) options() []PebbleStoreOption { return []PebbleStoreOption{ WithEncryption(f.cipher, - NewCounterNonceFactory(0xBEEF, 0x0001), + encryption.NewDeterministicNonceFactory(0xBEEF, 0x0001), func() (uint32, bool) { return f.keyID, true }, ), WithEncryptionObserver(f.obs), @@ -92,14 +92,7 @@ func (f *observedEncryptedFixture) reopen(tb testing.TB) { func newObservedEncryptedFixture(t *testing.T, keyID uint32) *observedEncryptedFixture { t.Helper() - ks := encryption.NewKeystore() - dek := make([]byte, encryption.KeySize) - _, err := rand.Read(dek) - require.NoError(t, err) - require.NoError(t, ks.Set(keyID, dek)) - - cipher, err := encryption.NewCipher(ks) - require.NoError(t, err) + cipher := encryptiontest.NewCipher(t, keyID) f := &observedEncryptedFixture{ dir: filepath.Join(t.TempDir(), "pebble"), @@ -232,20 +225,14 @@ func TestEncryptionObserverIgnoresRebadgeGuardTrialDecrypts(t *testing.T) { const keyID = uint32(14) - ks := encryption.NewKeystore() - dek := make([]byte, encryption.KeySize) - _, err := rand.Read(dek) - require.NoError(t, err) - require.NoError(t, ks.Set(keyID, dek)) - cipher, err := encryption.NewCipher(ks) - require.NoError(t, err) + cipher := encryptiontest.NewCipher(t, keyID) obs := &recordingEncryptionObserver{} // Cipher wired but the cutover gate is OFF: writes stay // cleartext, and every read runs the rebadge trial-decrypt. mvcc, err := NewPebbleStore(filepath.Join(t.TempDir(), "pebble"), WithEncryption(cipher, - NewCounterNonceFactory(0xCAFE, 0x0001), + encryption.NewDeterministicNonceFactory(0xCAFE, 0x0001), func() (uint32, bool) { return keyID, true }, ), WithStorageEnvelopeGate(func() bool { return false }), diff --git a/store/lsm_store_encryption_prop_test.go b/store/lsm_store_encryption_prop_test.go index aecc405e1..afca999ac 100644 --- a/store/lsm_store_encryption_prop_test.go +++ b/store/lsm_store_encryption_prop_test.go @@ -3,11 +3,11 @@ package store import ( "bytes" "context" - "crypto/rand" "path/filepath" "testing" "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/encryptiontest" "github.com/stretchr/testify/require" "pgregory.net/rapid" ) @@ -25,7 +25,7 @@ import ( // // The fixture is constructed once per Check call so the same // keystore + cipher + nonce factory exercises many writes — this -// catches any nonce-reuse regression in CounterNonceFactory's +// catches any nonce-reuse regression in DeterministicNonceFactory's // atomic counter. // // Tamper rejection is covered by the deterministic @@ -56,20 +56,11 @@ func TestEncryption_Property_PutGet(t *testing.T) { // counter is shared by all generated writes, including shrinking attempts. func newPropEncryptedStore(t *testing.T) MVCCStore { t.Helper() - ks := encryption.NewKeystore() - dek := make([]byte, encryption.KeySize) - _, _ = rand.Read(dek) - if err := ks.Set(7, dek); err != nil { - t.Fatalf("Keystore.Set: %v", err) - } - c, err := encryption.NewCipher(ks) - if err != nil { - t.Fatalf("NewCipher: %v", err) - } + c := encryptiontest.NewCipher(t, 7) dir := filepath.Join(t.TempDir(), "pebble") mvcc, err := NewPebbleStore(dir, WithEncryption(c, - NewCounterNonceFactory(0xCAFE, 0x0001), + encryption.NewDeterministicNonceFactory(0xCAFE, 0x0001), func() (uint32, bool) { return 7, true }, ), ) diff --git a/store/lsm_store_encryption_test.go b/store/lsm_store_encryption_test.go index 8d25f55dd..02c6fd47e 100644 --- a/store/lsm_store_encryption_test.go +++ b/store/lsm_store_encryption_test.go @@ -46,7 +46,7 @@ func (f *encryptedStoreFixture) reopen(tb testing.TB) { tb.Helper() mvcc, err := NewPebbleStore(f.dir, WithEncryption(f.cipher, - NewCounterNonceFactory(0xABCD, 0x0001), + encryption.NewDeterministicNonceFactory(0xABCD, 0x0001), func() (uint32, bool) { return f.keyID, true }, ), ) @@ -74,7 +74,7 @@ func newEncryptedStoreFixture(t *testing.T, keyID uint32) *encryptedStoreFixture dir := filepath.Join(t.TempDir(), "pebble") mvcc, err := NewPebbleStore(dir, WithEncryption(c, - NewCounterNonceFactory(0xABCD, 0x0001), + encryption.NewDeterministicNonceFactory(0xABCD, 0x0001), func() (uint32, bool) { return keyID, true }, ), ) @@ -228,7 +228,7 @@ func newToggleableEncryptedStore(t *testing.T, keyID uint32, activeFlag *bool) M dir := filepath.Join(t.TempDir(), "pebble") mvcc, err := NewPebbleStore(dir, WithEncryption(c, - NewCounterNonceFactory(0x0011, 0x0002), + encryption.NewDeterministicNonceFactory(0x0011, 0x0002), func() (uint32, bool) { if !*activeFlag { return 0, false @@ -289,7 +289,7 @@ func newGatedEncryptedStore(t *testing.T, keyID uint32, gateFlag *bool) *gatedFi dir := filepath.Join(t.TempDir(), "pebble") mvcc, err := NewPebbleStore(dir, WithEncryption(c, - NewCounterNonceFactory(0x0022, 0x0003), + encryption.NewDeterministicNonceFactory(0x0022, 0x0003), func() (uint32, bool) { return keyID, true }, // DEK always provisioned ), WithStorageEnvelopeGate(func() bool { return *gateFlag }), @@ -512,7 +512,7 @@ func TestStorageEnvelopeGate_NilArgumentIsNoOp(t *testing.T) { dir := filepath.Join(t.TempDir(), "pebble") mvcc, err := NewPebbleStore(dir, WithEncryption(c, - NewCounterNonceFactory(0x0033, 0x0004), + encryption.NewDeterministicNonceFactory(0x0033, 0x0004), func() (uint32, bool) { return 60, true }, ), WithStorageEnvelopeGate(nil), // explicit nil → no-op @@ -874,7 +874,7 @@ func TestEncryption_SnapshotRestoreAtMaxValueSize(t *testing.T) { dstDir := filepath.Join(t.TempDir(), "restore") dst, err := NewPebbleStore(dstDir, WithEncryption(f.cipher, - NewCounterNonceFactory(0xABCD, 0x0001), + encryption.NewDeterministicNonceFactory(0xABCD, 0x0001), func() (uint32, bool) { return f.keyID, true }, ), ) diff --git a/store/lsm_store_registration_gate_test.go b/store/lsm_store_registration_gate_test.go index 5bfa5c3dd..6e9dc6b03 100644 --- a/store/lsm_store_registration_gate_test.go +++ b/store/lsm_store_registration_gate_test.go @@ -40,7 +40,7 @@ func newRegGateStore(t *testing.T, registered *bool) *regGateFixture { dir := filepath.Join(t.TempDir(), "pebble") mvcc, err := NewPebbleStore(dir, WithEncryption(c, - NewCounterNonceFactory(0x00AA, 0x0005), + encryption.NewDeterministicNonceFactory(0x00AA, 0x0005), func() (uint32, bool) { return regGateDEKID, true }, // DEK always provisioned ), WithStorageEnvelopeGate(func() bool { return true }), // cutover fired @@ -230,7 +230,7 @@ func TestRegistrationGate_NotEncryptingIsUngated(t *testing.T) { registered := false mvcc, err := NewPebbleStore(dir, WithEncryption(c, - NewCounterNonceFactory(0x00BB, 0x0006), + encryption.NewDeterministicNonceFactory(0x00BB, 0x0006), func() (uint32, bool) { return 7, true }, ), WithStorageEnvelopeGate(func() bool { return false }), // cutover NOT fired @@ -272,7 +272,7 @@ func TestRegistrationGate_UnwiredIsNoOp(t *testing.T) { dir := filepath.Join(t.TempDir(), "pebble") mvcc, err := NewPebbleStore(dir, WithEncryption(c, - NewCounterNonceFactory(0x00CC, 0x0007), + encryption.NewDeterministicNonceFactory(0x00CC, 0x0007), func() (uint32, bool) { return 7, true }, ), WithStorageEnvelopeGate(func() bool { return true }), diff --git a/store/lsm_store_sync_mode_benchmark_test.go b/store/lsm_store_sync_mode_benchmark_test.go index bab9be116..e01b68ffd 100644 --- a/store/lsm_store_sync_mode_benchmark_test.go +++ b/store/lsm_store_sync_mode_benchmark_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/encryptiontest" "github.com/cockroachdb/pebble/v2" ) @@ -128,22 +129,11 @@ func benchmarkEncryptionOptions(b *testing.B, enabled bool) []PebbleStoreOption if !enabled { return nil } - ks := encryption.NewKeystore() - dek := make([]byte, encryption.KeySize) - if _, err := rand.Read(dek); err != nil { - b.Fatalf("rand.Read DEK: %v", err) - } const keyID uint32 = 1 - if err := ks.Set(keyID, dek); err != nil { - b.Fatalf("Keystore.Set: %v", err) - } - cipher, err := encryption.NewCipher(ks) - if err != nil { - b.Fatalf("NewCipher: %v", err) - } + cipher := encryptiontest.NewCipher(b, keyID) return []PebbleStoreOption{WithEncryption( cipher, - NewCounterNonceFactory(1, 1), + encryption.NewDeterministicNonceFactory(1, 1), func() (uint32, bool) { return keyID, true }, )} } diff --git a/store/migration_ownership_partition_boundary_test.go b/store/migration_ownership_partition_boundary_test.go index 56f35b4a8..3b4d7894f 100644 --- a/store/migration_ownership_partition_boundary_test.go +++ b/store/migration_ownership_partition_boundary_test.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "testing" + "github.com/bootjp/elastickv/internal/encryption" "github.com/stretchr/testify/require" ) @@ -75,7 +76,7 @@ func TestOwnershipPartitionRootManifestExternalRefusal(t *testing.T) { ps.sstIngestSnapshots = format == "encrypted-sst" require.NoError(t, fixture.mvcc.PutAt(t.Context(), key, []byte("root manifest"), 1, 0)) raw = externalTestSnapshot(t, fixture.mvcc) - options = []PebbleStoreOption{WithEncryption(fixture.cipher, NewCounterNonceFactory(0xABCD, 2), func() (uint32, bool) { return fixture.keyID, true })} + options = []PebbleStoreOption{WithEncryption(fixture.cipher, encryption.NewDeterministicNonceFactory(0xABCD, 2), func() (uint32, bool) { return fixture.keyID, true })} } else { source := externalTestMemory(t) require.NoError(t, source.PutAt(t.Context(), key, []byte("root manifest"), 1, 0)) diff --git a/store/migration_ownership_partition_namespace_test.go b/store/migration_ownership_partition_namespace_test.go index 3b4a6a0f6..a59801233 100644 --- a/store/migration_ownership_partition_namespace_test.go +++ b/store/migration_ownership_partition_namespace_test.go @@ -4,6 +4,7 @@ import ( "bytes" "testing" + "github.com/bootjp/elastickv/internal/encryption" "github.com/stretchr/testify/require" ) @@ -119,7 +120,7 @@ func activatedPartitionSnapshotSource(t *testing.T, format string) (MVCCStore, [ ps, ok := fixture.mvcc.(*pebbleStore) require.True(t, ok) ps.sstIngestSnapshots = format == "encrypted-sst" - return fixture.mvcc, []PebbleStoreOption{WithEncryption(fixture.cipher, NewCounterNonceFactory(0xABCD, 2), + return fixture.mvcc, []PebbleStoreOption{WithEncryption(fixture.cipher, encryption.NewDeterministicNonceFactory(0xABCD, 2), func() (uint32, bool) { return fixture.keyID, true })} } diff --git a/store/migration_ownership_partition_snapshot_test.go b/store/migration_ownership_partition_snapshot_test.go index 2104fd756..b18632286 100644 --- a/store/migration_ownership_partition_snapshot_test.go +++ b/store/migration_ownership_partition_snapshot_test.go @@ -7,6 +7,7 @@ import ( "log/slog" "testing" + "github.com/bootjp/elastickv/internal/encryption" "github.com/cockroachdb/pebble/v2" "github.com/stretchr/testify/require" ) @@ -30,7 +31,7 @@ func TestOwnershipPartitionSnapshotFormatsAndReopen(t *testing.T) { ps, ok := source.(*pebbleStore) require.True(t, ok) ps.sstIngestSnapshots = format == "encrypted-sst" - options = []PebbleStoreOption{WithEncryption(fixture.cipher, NewCounterNonceFactory(0xABCD, 2), func() (uint32, bool) { return fixture.keyID, true })} + options = []PebbleStoreOption{WithEncryption(fixture.cipher, encryption.NewDeterministicNonceFactory(0xABCD, 2), func() (uint32, bool) { return fixture.keyID, true })} } _, tx := partitionFixture(t, source) require.NoError(t, partitionStore(t, source).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) diff --git a/store/snapshot_external_encryption_test.go b/store/snapshot_external_encryption_test.go index 456486764..69def4fe7 100644 --- a/store/snapshot_external_encryption_test.go +++ b/store/snapshot_external_encryption_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "testing" + "github.com/bootjp/elastickv/internal/encryption" "github.com/cockroachdb/pebble/v2" "github.com/stretchr/testify/require" ) @@ -33,7 +34,7 @@ func verifyExternalEncryptedSnapshot(t *testing.T, sst, bearing bool) { } raw := externalTestSnapshot(t, source.mvcc) dir := filepath.Join(t.TempDir(), "destination") - options := []PebbleStoreOption{WithEncryption(source.cipher, NewCounterNonceFactory(0xABCD, 2), + options := []PebbleStoreOption{WithEncryption(source.cipher, encryption.NewDeterministicNonceFactory(0xABCD, 2), func() (uint32, bool) { return source.keyID, true })} destination, err := NewPebbleStore(dir, options...) require.NoError(t, err) diff --git a/store/snapshot_external_user_keys_test.go b/store/snapshot_external_user_keys_test.go index 136697b5e..ee0ba0523 100644 --- a/store/snapshot_external_user_keys_test.go +++ b/store/snapshot_external_user_keys_test.go @@ -4,6 +4,7 @@ import ( "path/filepath" "testing" + "github.com/bootjp/elastickv/internal/encryption" "github.com/cockroachdb/pebble/v2" "github.com/stretchr/testify/require" ) @@ -25,7 +26,7 @@ func externalAliasSource(t *testing.T, format string, encrypted bool) (MVCCStore ps, ok := fixture.mvcc.(*pebbleStore) require.True(t, ok) ps.sstIngestSnapshots = format == "sst" - return fixture.mvcc, []PebbleStoreOption{WithEncryption(fixture.cipher, NewCounterNonceFactory(0xABCD, 2), + return fixture.mvcc, []PebbleStoreOption{WithEncryption(fixture.cipher, encryption.NewDeterministicNonceFactory(0xABCD, 2), func() (uint32, bool) { return fixture.keyID, true })} } if format == "mvcc" { diff --git a/store/snapshot_pebble_sst_test.go b/store/snapshot_pebble_sst_test.go index 794033b82..fce2b0f62 100644 --- a/store/snapshot_pebble_sst_test.go +++ b/store/snapshot_pebble_sst_test.go @@ -138,7 +138,7 @@ func TestPebbleStoreEncryptedSSTIngestSnapshotRoundTrip(t *testing.T) { cipher, err := encryption.NewCipher(ks) require.NoError(t, err) encryptionOption := func(epoch uint16) PebbleStoreOption { - return WithEncryption(cipher, NewCounterNonceFactory(1, epoch), func() (uint32, bool) { + return WithEncryption(cipher, encryption.NewDeterministicNonceFactory(1, epoch), func() (uint32, bool) { return keyID, true }) } From b408fd8cfbe4614753454a2b126221ee410f071f Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 10:27:23 +0000 Subject: [PATCH 11/15] test: move fake Raft engines and gRPC loopbacks onto shared helpers (cleanup WP-9..12) kv: replace stubLeaderEngine and noopEngine with one newStubLeaderEngine built on raftenginetest.StubEngine, stubFollowerEngine with newStubFollowerEngine, and the admin/internal LeaderView fakes with raftenginetest.StaticLeaderView. Fakes whose behaviour is the subject of the test (toggling leader, blocking reads, lease, call recording, panicking VerifyLeader) stay local. The etcd engine tests alias raftenginetest.TestStateMachine instead of carrying a copy. Fourteen kv loopback gRPC servers move onto grpctest.Serve and so pick up the production server options; servers whose tests assert the Serve error or need the listener address before construction are left as they are. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- .../internal_admin_proposal_security_test.go | 4 +- adapter/internal_admin_proposal_test.go | 27 ++-- internal/raftengine/etcd/engine_test.go | 81 +--------- kv/coordinator_dispatch_test.go | 8 +- kv/coordinator_keyviz_label_test.go | 4 +- kv/coordinator_retry_test.go | 35 +---- kv/fakes_test.go | 16 ++ kv/grpc_conn_cache_test.go | 17 +-- kv/leader_admin_proposer_test.go | 50 ++---- kv/leader_proxy_test.go | 143 +++++------------- kv/leader_routed_store_test.go | 36 +---- kv/migration_frontier_admission_test.go | 2 +- kv/shard_store_exact_version_test.go | 5 +- kv/shard_store_test.go | 26 ++-- kv/sharded_coordinator_leader_test.go | 19 +-- kv/sharded_coordinator_txn_test.go | 37 +---- kv/sharded_membership_test.go | 19 +-- kv/tso_floor_test.go | 12 +- kv/tso_test.go | 40 ++--- 19 files changed, 162 insertions(+), 419 deletions(-) diff --git a/adapter/internal_admin_proposal_security_test.go b/adapter/internal_admin_proposal_security_test.go index b343b6dec..d0d23ee23 100644 --- a/adapter/internal_admin_proposal_security_test.go +++ b/adapter/internal_admin_proposal_security_test.go @@ -18,7 +18,7 @@ func TestInternalForwardAdminProposalRejectsOtherRaftPayloads(t *testing.T) { payload := []byte{byte(opcode), 0} proposer := &internalAdminProposer{} server := NewInternalWithEngine(nil, - internalAdminLeaderView{state: raftengine.StateLeader}, nil, nil, + internalAdminLeaderView(raftengine.StateLeader, 0), nil, nil, WithInternalAdminProposer(proposer)) _, err := server.ForwardAdminProposal(context.Background(), &pb.ForwardAdminProposalRequest{Payload: payload}) require.Equal(t, codes.PermissionDenied, status.Code(err), "opcode %x", opcode) @@ -32,7 +32,7 @@ func TestInternalForwardAdminProposalValidatesEntireEnvelope(t *testing.T) { for _, payload := range [][]byte{nil, valid[:len(valid)-1], append(valid, 0x0d), {0x0e, 0xff}} { proposer := &internalAdminProposer{} server := NewInternalWithEngine(nil, - internalAdminLeaderView{state: raftengine.StateLeader}, nil, nil, + internalAdminLeaderView(raftengine.StateLeader, 0), nil, nil, WithInternalAdminProposer(proposer)) _, err := server.ForwardAdminProposal(context.Background(), &pb.ForwardAdminProposalRequest{Payload: payload}) require.Equal(t, codes.PermissionDenied, status.Code(err)) diff --git a/adapter/internal_admin_proposal_test.go b/adapter/internal_admin_proposal_test.go index 7f26caa96..c1250184c 100644 --- a/adapter/internal_admin_proposal_test.go +++ b/adapter/internal_admin_proposal_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/raftengine/raftenginetest" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/stretchr/testify/require" @@ -13,25 +14,19 @@ import ( "google.golang.org/grpc/status" ) -type internalAdminLeaderView struct { - state raftengine.State - readIndex uint64 -} - -func (v internalAdminLeaderView) State() raftengine.State { return v.state } -func (internalAdminLeaderView) Leader() raftengine.LeaderInfo { - return raftengine.LeaderInfo{ID: "leader", Address: "leader:50051"} -} -func (internalAdminLeaderView) VerifyLeader(context.Context) error { return nil } -func (v internalAdminLeaderView) LinearizableRead(context.Context) (uint64, error) { - return v.readIndex, nil +func internalAdminLeaderView(state raftengine.State, readIndex uint64) *raftenginetest.StaticLeaderView { + return &raftenginetest.StaticLeaderView{ + StateValue: state, + LeaderValue: raftengine.LeaderInfo{ID: "leader", Address: "leader:50051"}, + ReadIndex: readIndex, + } } func TestInternalForwardLeaseReadUsesLeaderBarrier(t *testing.T) { t.Parallel() internal := NewInternalWithEngine( nil, - internalAdminLeaderView{state: raftengine.StateLeader, readIndex: 23}, + internalAdminLeaderView(raftengine.StateLeader, 23), nil, nil, WithInternalLastCommitTimestamp(func() uint64 { return 37 }), @@ -41,7 +36,7 @@ func TestInternalForwardLeaseReadUsesLeaderBarrier(t *testing.T) { require.Equal(t, uint64(23), resp.GetAppliedIndex()) require.Equal(t, uint64(37), resp.GetLastCommitTs()) - internal.leader = internalAdminLeaderView{state: raftengine.StateFollower} + internal.leader = internalAdminLeaderView(raftengine.StateFollower, 0) _, err = internal.ForwardLeaseRead(context.Background(), &pb.ForwardLeaseReadRequest{}) require.ErrorIs(t, err, ErrNotLeader) } @@ -68,7 +63,7 @@ func TestInternalForwardAdminProposalUsesLeaderProposer(t *testing.T) { proposer := &internalAdminProposer{} internal := NewInternalWithEngine( nil, - internalAdminLeaderView{state: raftengine.StateLeader}, + internalAdminLeaderView(raftengine.StateLeader, 0), nil, nil, WithInternalAdminProposer(proposer), @@ -110,7 +105,7 @@ func TestInternalForwardAdminProposalFailsClosed(t *testing.T) { t.Run(tc.name, func(t *testing.T) { internal := NewInternalWithEngine( nil, - internalAdminLeaderView{state: tc.state}, + internalAdminLeaderView(tc.state, 0), nil, nil, WithInternalAdminProposer(tc.proposer), diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index 619858dd5..094b3dc89 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -3,7 +3,6 @@ package etcd import ( "bytes" "context" - "encoding/binary" "io" "net" "path/filepath" @@ -13,6 +12,7 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/raftengine/raftenginetest" "github.com/bootjp/elastickv/internal/testutil" internalutil "github.com/bootjp/elastickv/internal" @@ -27,79 +27,7 @@ import ( "google.golang.org/grpc" ) -type testStateMachine struct { - mu sync.Mutex - applied [][]byte -} - -func (s *testStateMachine) Apply(data []byte) any { - s.mu.Lock() - defer s.mu.Unlock() - s.applied = append(s.applied, append([]byte(nil), data...)) - return string(data) -} - -func (s *testStateMachine) Applied() [][]byte { - s.mu.Lock() - defer s.mu.Unlock() - out := make([][]byte, len(s.applied)) - for i, item := range s.applied { - out[i] = append([]byte(nil), item...) - } - return out -} - -func (s *testStateMachine) Snapshot() (Snapshot, error) { - s.mu.Lock() - defer s.mu.Unlock() - - var buf bytes.Buffer - count, err := uint32Len(len(s.applied)) - if err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, count); err != nil { - return nil, err - } - for _, item := range s.applied { - size, err := uint32Len(len(item)) - if err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, size); err != nil { - return nil, err - } - if _, err := buf.Write(item); err != nil { - return nil, err - } - } - return &testSnapshot{data: buf.Bytes()}, nil -} - -func (s *testStateMachine) Restore(r io.Reader) error { - var count uint32 - if err := binary.Read(r, binary.BigEndian, &count); err != nil { - return err - } - - applied := make([][]byte, 0, count) - for range count { - var length uint32 - if err := binary.Read(r, binary.BigEndian, &length); err != nil { - return err - } - item := make([]byte, length) - if _, err := io.ReadFull(r, item); err != nil { - return err - } - applied = append(applied, item) - } - - s.mu.Lock() - defer s.mu.Unlock() - s.applied = applied - return nil -} +type testStateMachine = raftenginetest.TestStateMachine type testSnapshot struct { data []byte @@ -2279,7 +2207,10 @@ func waitForConfigSize(t *testing.T, engine *Engine, size int) { func mustEncodeSnapshotData(t *testing.T, applied [][]byte) []byte { t.Helper() - fsm := &testStateMachine{applied: applied} + fsm := &testStateMachine{} + for _, item := range applied { + fsm.Apply(item) + } snapshot, err := fsm.Snapshot() require.NoError(t, err) t.Cleanup(func() { diff --git a/kv/coordinator_dispatch_test.go b/kv/coordinator_dispatch_test.go index ebbea3979..6b5b94523 100644 --- a/kv/coordinator_dispatch_test.go +++ b/kv/coordinator_dispatch_test.go @@ -200,7 +200,7 @@ func TestCoordinateSamplerObservesRawTxnAndRead(t *testing.T) { clock.SetPhysicalCeiling(time.Now().Add(time.Minute).UnixMilli()) c := (&Coordinate{ transactionManager: &stubTransactional{}, - engine: stubLeaderEngine{}, + engine: newStubLeaderEngine(), clock: clock, }).WithSampler(sampler, routeID) @@ -242,7 +242,7 @@ func TestCoordinateSamplerResolvesRouteForEveryObservedKey(t *testing.T) { clock.SetPhysicalCeiling(time.Now().Add(time.Minute).UnixMilli()) c := (&Coordinate{ transactionManager: &stubTransactional{}, - engine: stubLeaderEngine{}, + engine: newStubLeaderEngine(), clock: clock, }).WithSamplerRouteResolver(sampler, func(key []byte) (uint64, bool) { resolverKeys = append(resolverKeys, append([]byte(nil), key...)) @@ -305,7 +305,7 @@ func TestCoordinateSamplerObservesFollowerIngressBeforeRedirect(t *testing.T) { }) require.True(t, sampler.RegisterRoute(routeID, []byte(""), nil, 1)) c := (&Coordinate{ - engine: redirectSamplerFollowerEngine{}, + engine: redirectSamplerFollowerEngine{newStubLeaderEngine()}, }).WithSampler(sampler, routeID) _, err := c.dispatchOnce(context.Background(), &OperationGroup[OP]{Elems: []*Elem[OP]{ @@ -317,7 +317,7 @@ func TestCoordinateSamplerObservesFollowerIngressBeforeRedirect(t *testing.T) { } type redirectSamplerFollowerEngine struct { - stubLeaderEngine + *testEngine } func (redirectSamplerFollowerEngine) State() raftengine.State { return raftengine.StateFollower } diff --git a/kv/coordinator_keyviz_label_test.go b/kv/coordinator_keyviz_label_test.go index 5acf723f4..f7a94768e 100644 --- a/kv/coordinator_keyviz_label_test.go +++ b/kv/coordinator_keyviz_label_test.go @@ -24,7 +24,7 @@ func TestCoordinateSamplerReceivesAdapterLabel(t *testing.T) { clock.SetPhysicalCeiling(time.Now().Add(time.Minute).UnixMilli()) c := (&Coordinate{ transactionManager: &stubTransactional{}, - engine: stubLeaderEngine{}, + engine: newStubLeaderEngine(), clock: clock, }).WithSamplerRouteResolver(sampler, func([]byte) (uint64, bool) { return 7, true }) labeled := WithKeyVizLabel(c, keyviz.LabelRedis) @@ -63,7 +63,7 @@ func TestCoordinateSamplerDefaultsToLegacyLabel(t *testing.T) { clock.SetPhysicalCeiling(time.Now().Add(time.Minute).UnixMilli()) c := (&Coordinate{ transactionManager: &stubTransactional{}, - engine: stubLeaderEngine{}, + engine: newStubLeaderEngine(), clock: clock, }).WithSamplerRouteResolver(sampler, func([]byte) (uint64, bool) { return 7, true }) diff --git a/kv/coordinator_retry_test.go b/kv/coordinator_retry_test.go index 222c68cae..cbe71ef1e 100644 --- a/kv/coordinator_retry_test.go +++ b/kv/coordinator_retry_test.go @@ -14,35 +14,6 @@ import ( "github.com/stretchr/testify/require" ) -// stubLeaderEngine reports State=Leader and a fixed Leader address. It is -// the minimum surface Coordinate.Dispatch needs to take the local-dispatch -// path (IsLeader() true) without engaging real raft machinery. Methods -// beyond that surface return zero values / nil; Dispatch's retry loop -// never calls them, and refreshLeaseAfterDispatch skips its LeaseProvider -// branch because this stub does not implement that interface. -type stubLeaderEngine struct{} - -func (stubLeaderEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { - return &raftengine.ProposalResult{}, nil -} -func (e stubLeaderEngine) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { - return e.Propose(ctx, data) -} -func (stubLeaderEngine) State() raftengine.State { return raftengine.StateLeader } -func (stubLeaderEngine) Leader() raftengine.LeaderInfo { - return raftengine.LeaderInfo{ID: "self", Address: "127.0.0.1:0"} -} -func (stubLeaderEngine) VerifyLeader(context.Context) error { return nil } -func (stubLeaderEngine) LinearizableRead(context.Context) (uint64, error) { return 0, nil } -func (stubLeaderEngine) Status() raftengine.Status { - return raftengine.Status{State: raftengine.StateLeader} -} -func (stubLeaderEngine) Configuration(context.Context) (raftengine.Configuration, error) { - return raftengine.Configuration{}, nil -} -func (stubLeaderEngine) SnapshotEvery() uint64 { return 10_000 } -func (stubLeaderEngine) Close() error { return nil } - // scriptedTransactional returns the error registered in errs for the // 0-indexed call that triggered Commit; calls without a registered // entry succeed. Using a map keyed by uint64 rather than a []error @@ -134,13 +105,13 @@ func TestFinalDispatchErrPreservesUnknownOutcomeAfterRetryDeadline(t *testing.T) require.ErrorContains(t, got, "forwarded commit response lost") } -// newRetryCoordinate wires a Coordinate against stubLeaderEngine and the +// newRetryCoordinate wires a Coordinate against newStubLeaderEngine and the // supplied scripted transaction manager. No engine registration happens, // so the caller does not need to call Close. func newRetryCoordinate(tx Transactional) *Coordinate { return &Coordinate{ transactionManager: tx, - engine: stubLeaderEngine{}, + engine: newStubLeaderEngine(), clock: NewHLC(), } } @@ -262,7 +233,7 @@ func TestCoordinateDispatch_CtxCancelDuringRetrySurfaces(t *testing.T) { } c := &Coordinate{ transactionManager: tx, - engine: stubLeaderEngine{}, + engine: newStubLeaderEngine(), clock: NewHLC(), } diff --git a/kv/fakes_test.go b/kv/fakes_test.go index 71dad3efb..203fdcdd0 100644 --- a/kv/fakes_test.go +++ b/kv/fakes_test.go @@ -33,6 +33,22 @@ func (c *nopCoordinator) Clock() *HLC { type testEngine = raftenginetest.StubEngine +// newStubLeaderEngine reports State=Leader with a fixed Leader address. It is +// the minimum surface Coordinate.Dispatch needs to take the local-dispatch +// path without real raft machinery, and it implements no LeaseProvider, so +// refreshLeaseAfterDispatch skips that branch. LinearizableRead returns +// immediately (an already-up-to-date FSM). +func newStubLeaderEngine() *testEngine { + return &testEngine{ + StaticLeaderView: raftenginetest.StaticLeaderView{ + StateValue: raftengine.StateLeader, + LeaderValue: raftengine.LeaderInfo{ID: "self", Address: "127.0.0.1:0"}, + }, + StatusValue: raftengine.Status{State: raftengine.StateLeader}, + SnapshotStep: 10_000, + } +} + var _ Coordinator = (*nopCoordinator)(nil) func TestSharedFakesPreserveFallbacks(t *testing.T) { diff --git a/kv/grpc_conn_cache_test.go b/kv/grpc_conn_cache_test.go index fe192b882..0a1051562 100644 --- a/kv/grpc_conn_cache_test.go +++ b/kv/grpc_conn_cache_test.go @@ -1,8 +1,6 @@ package kv import ( - "context" - "net" "sync" "testing" @@ -46,20 +44,7 @@ func TestGRPCConnCache_ReusesConnection(t *testing.T) { func TestGRPCConnCache_ConcurrentConnFor(t *testing.T) { t.Parallel() - var lc net.ListenConfig - lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") - require.NoError(t, err) - - srv := grpc.NewServer() - go func() { - _ = srv.Serve(lis) - }() - t.Cleanup(func() { - srv.Stop() - _ = lis.Close() - }) - - addr := lis.Addr().String() + addr := grpctest.Serve(t, func(*grpc.Server) {}) var c GRPCConnCache diff --git a/kv/leader_admin_proposer_test.go b/kv/leader_admin_proposer_test.go index e7b7911aa..8fe4c0f71 100644 --- a/kv/leader_admin_proposer_test.go +++ b/kv/leader_admin_proposer_test.go @@ -2,29 +2,23 @@ package kv import ( "context" - "net" "sync" "testing" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/raftengine/raftenginetest" + "github.com/bootjp/elastickv/internal/testutil/grpctest" pb "github.com/bootjp/elastickv/proto" "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) -type adminProposalLeaderView struct { - state raftengine.State - addr string -} - -func (v *adminProposalLeaderView) State() raftengine.State { return v.state } -func (v *adminProposalLeaderView) Leader() raftengine.LeaderInfo { - return raftengine.LeaderInfo{ID: "leader", Address: v.addr} -} -func (v *adminProposalLeaderView) VerifyLeader(context.Context) error { return nil } -func (v *adminProposalLeaderView) LinearizableRead(context.Context) (uint64, error) { - return 0, nil +func adminProposalLeaderView(state raftengine.State, addr string) *raftenginetest.StaticLeaderView { + return &raftenginetest.StaticLeaderView{ + StateValue: state, + LeaderValue: raftengine.LeaderInfo{ID: "leader", Address: addr}, + } } type recordingAdminProposer struct { @@ -70,7 +64,7 @@ func TestLeaderAdminProposerUsesLocalVerifiedLeader(t *testing.T) { t.Parallel() local := &recordingAdminProposer{} proposer := NewLeaderAdminProposer( - &adminProposalLeaderView{state: raftengine.StateLeader}, + adminProposalLeaderView(raftengine.StateLeader, ""), local, &GRPCConnCache{}, ) @@ -84,23 +78,14 @@ func TestLeaderAdminProposerUsesLocalVerifiedLeader(t *testing.T) { func TestLeaderAdminProposerForwardsFollowerProposal(t *testing.T) { t.Parallel() - var lc net.ListenConfig - lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") - require.NoError(t, err) service := &forwardingAdminServer{} - server := grpc.NewServer() - pb.RegisterInternalServer(server, service) - go func() { _ = server.Serve(lis) }() - t.Cleanup(func() { - server.Stop() - _ = lis.Close() - }) + addr := grpctest.Serve(t, func(server *grpc.Server) { pb.RegisterInternalServer(server, service) }) cache := &GRPCConnCache{} t.Cleanup(func() { require.NoError(t, cache.Close()) }) local := &recordingAdminProposer{} proposer := NewLeaderAdminProposer( - &adminProposalLeaderView{state: raftengine.StateFollower, addr: lis.Addr().String()}, + adminProposalLeaderView(raftengine.StateFollower, addr), local, cache, ) @@ -115,27 +100,18 @@ func TestLeaderAdminProposerForwardsFollowerProposal(t *testing.T) { func TestLeaderAdminProposerForwardsBearerToken(t *testing.T) { t.Parallel() - var lc net.ListenConfig - lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") - require.NoError(t, err) service := &forwardingAdminServer{} - server := grpc.NewServer() - pb.RegisterInternalServer(server, service) - go func() { _ = server.Serve(lis) }() - t.Cleanup(func() { - server.Stop() - _ = lis.Close() - }) + addr := grpctest.Serve(t, func(server *grpc.Server) { pb.RegisterInternalServer(server, service) }) cache := &GRPCConnCache{} t.Cleanup(func() { require.NoError(t, cache.Close()) }) proposer := NewLeaderAdminProposer( - &adminProposalLeaderView{state: raftengine.StateFollower, addr: lis.Addr().String()}, + adminProposalLeaderView(raftengine.StateFollower, addr), &recordingAdminProposer{}, cache, WithLeaderAdminToken("s3cret"), ) - _, err = proposer.ProposeAdmin(context.Background(), []byte("pin")) + _, err := proposer.ProposeAdmin(context.Background(), []byte("pin")) require.NoError(t, err) service.mu.Lock() require.Equal(t, []string{"Bearer s3cret"}, service.authorization) diff --git a/kv/leader_proxy_test.go b/kv/leader_proxy_test.go index 2f6e1e55f..8b738887e 100644 --- a/kv/leader_proxy_test.go +++ b/kv/leader_proxy_test.go @@ -9,6 +9,8 @@ import ( "time" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/raftengine/raftenginetest" + "github.com/bootjp/elastickv/internal/testutil/grpctest" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" @@ -74,36 +76,24 @@ func startLeaderProxyBlackhole(t *testing.T) string { return lis.Addr().String() } -// stubFollowerEngine is a minimal raftengine.Engine stub that reports the -// local node as a follower and returns a configured leader address. It is -// used by TestLeaderProxy_ForwardsWhenFollower to exercise the forwarding -// code path without running a real two-node raft cluster. -type stubFollowerEngine struct { - leaderAddr string -} - -func (s *stubFollowerEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { - return nil, raftengine.ErrNotLeader -} -func (s *stubFollowerEngine) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { - return s.Propose(ctx, data) -} -func (s *stubFollowerEngine) State() raftengine.State { return raftengine.StateFollower } -func (s *stubFollowerEngine) Leader() raftengine.LeaderInfo { - return raftengine.LeaderInfo{ID: "leader", Address: s.leaderAddr} -} -func (s *stubFollowerEngine) VerifyLeader(context.Context) error { return raftengine.ErrNotLeader } -func (s *stubFollowerEngine) LinearizableRead(context.Context) (uint64, error) { - return 0, raftengine.ErrNotLeader -} -func (s *stubFollowerEngine) Status() raftengine.Status { - return raftengine.Status{State: raftengine.StateFollower, Leader: s.Leader()} -} -func (s *stubFollowerEngine) Configuration(context.Context) (raftengine.Configuration, error) { - return raftengine.Configuration{}, nil +// newStubFollowerEngine reports the local node as a follower of a leader at +// leaderAddr and rejects proposals and reads with ErrNotLeader. It exercises +// the forwarding code path without running a real two-node raft cluster. +func newStubFollowerEngine(leaderAddr string) *testEngine { + leader := raftengine.LeaderInfo{ID: "leader", Address: leaderAddr} + return &testEngine{ + StaticLeaderView: raftenginetest.StaticLeaderView{ + StateValue: raftengine.StateFollower, + LeaderValue: leader, + ReadErr: raftengine.ErrNotLeader, + }, + ProposeFunc: func(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, raftengine.ErrNotLeader + }, + StatusValue: raftengine.Status{State: raftengine.StateFollower, Leader: leader}, + SnapshotStep: 10_000, + } } -func (s *stubFollowerEngine) SnapshotEvery() uint64 { return 10_000 } -func (s *stubFollowerEngine) Close() error { return nil } func TestLeaderProxy_CommitLocalWhenLeader(t *testing.T) { t.Parallel() @@ -137,25 +127,15 @@ func TestLeaderProxy_CommitLocalWhenLeader(t *testing.T) { func TestLeaderProxy_ForwardsWhenFollower(t *testing.T) { t.Parallel() - var lc net.ListenConfig - lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") - require.NoError(t, err) - svc := &fakeInternal{resp: &pb.ForwardResponse{Success: true, CommitIndex: 123, CommitTs: 456}} - srv := grpc.NewServer() - pb.RegisterInternalServer(srv, svc) - go func() { _ = srv.Serve(lis) }() - t.Cleanup(func() { - srv.Stop() - _ = lis.Close() - }) + addr := grpctest.Serve(t, func(srv *grpc.Server) { pb.RegisterInternalServer(srv, svc) }) // Wait briefly so the gRPC server is ready to serve. dialer := &net.Dialer{Timeout: 100 * time.Millisecond} require.Eventually(t, func() bool { dialCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() - c, err := dialer.DialContext(dialCtx, "tcp", lis.Addr().String()) + c, err := dialer.DialContext(dialCtx, "tcp", addr) if err != nil { return false } @@ -163,7 +143,7 @@ func TestLeaderProxy_ForwardsWhenFollower(t *testing.T) { return true }, 2*time.Second, 10*time.Millisecond) - follower := &stubFollowerEngine{leaderAddr: lis.Addr().String()} + follower := newStubFollowerEngine(addr) p := NewLeaderProxyWithEngine(follower) t.Cleanup(func() { _ = p.connCache.Close() }) @@ -239,25 +219,16 @@ func TestLeaderProxy_ForwardsAfterLeaderPublishes(t *testing.T) { // ErrLeaderNotFound, then flip to the real address after a brief // delay. forwardWithRetry must absorb the empty-address window and // succeed once the engine publishes. - var lc net.ListenConfig - lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") - require.NoError(t, err) svc := &fakeInternal{resp: &pb.ForwardResponse{Success: true, CommitIndex: 42}} - srv := grpc.NewServer() - pb.RegisterInternalServer(srv, svc) - go func() { _ = srv.Serve(lis) }() - t.Cleanup(func() { - srv.Stop() - _ = lis.Close() - }) + addr := grpctest.Serve(t, func(srv *grpc.Server) { pb.RegisterInternalServer(srv, svc) }) // Wait briefly so the gRPC server is ready. dialer := &net.Dialer{Timeout: 100 * time.Millisecond} require.Eventually(t, func() bool { ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() - c, err := dialer.DialContext(ctx, "tcp", lis.Addr().String()) + c, err := dialer.DialContext(ctx, "tcp", addr) if err != nil { return false } @@ -290,7 +261,7 @@ func TestLeaderProxy_ForwardsAfterLeaderPublishes(t *testing.T) { start := time.Now() go func() { time.Sleep(publishDelay) - eng.setLeader(lis.Addr().String()) + eng.setLeader(addr) }() resp, err := p.Commit(context.Background(), reqs) @@ -347,7 +318,7 @@ func TestLeaderProxy_FailsAfterLeaderBudgetElapses(t *testing.T) { func TestLeaderProxy_CanceledForwardReleasesHalfOpenProbe(t *testing.T) { t.Parallel() - eng := &stubFollowerEngine{leaderAddr: startLeaderProxyBlackhole(t)} + eng := newStubFollowerEngine(startLeaderProxyBlackhole(t)) p := NewLeaderProxyWithEngine(eng) t.Cleanup(func() { _ = p.connCache.Close() }) @@ -372,7 +343,7 @@ func TestLeaderProxy_CanceledForwardReleasesHalfOpenProbe(t *testing.T) { func TestLeaderProxy_ProxyBudgetDeadlineCountsTowardBreaker(t *testing.T) { t.Parallel() - eng := &stubFollowerEngine{leaderAddr: startLeaderProxyBlackhole(t)} + eng := newStubFollowerEngine(startLeaderProxyBlackhole(t)) p := NewLeaderProxyWithEngine(eng) t.Cleanup(func() { _ = p.connCache.Close() }) identity := leaderProxyIdentityFromEngine(eng) @@ -391,7 +362,7 @@ func TestLeaderProxy_ProxyBudgetDeadlineCountsTowardBreaker(t *testing.T) { func TestLeaderProxy_CanceledRecoveryOwnerStopsRetrying(t *testing.T) { t.Parallel() - eng := &stubFollowerEngine{leaderAddr: "127.0.0.1:1"} + eng := newStubFollowerEngine("127.0.0.1:1") p := NewLeaderProxyWithEngine(eng) t.Cleanup(func() { _ = p.connCache.Close() }) identity := leaderProxyIdentityFromEngine(eng) @@ -414,23 +385,14 @@ func TestLeaderProxy_CanceledRecoveryOwnerStopsRetrying(t *testing.T) { func TestLeaderProxy_LostForwardResponseDoesNotReplay(t *testing.T) { t.Parallel() - var lc net.ListenConfig - lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") - require.NoError(t, err) svc := &fakeInternal{ resp: &pb.ForwardResponse{Success: true, CommitIndex: 77}, err: status.Error(codes.Unavailable, "startup gate closed"), failFor: leaderProxyBreakerFailureThreshold, } - srv := grpc.NewServer() - pb.RegisterInternalServer(srv, svc) - go func() { _ = srv.Serve(lis) }() - t.Cleanup(func() { - srv.Stop() - _ = lis.Close() - }) + addr := grpctest.Serve(t, func(srv *grpc.Server) { pb.RegisterInternalServer(srv, svc) }) - eng := &stubFollowerEngine{leaderAddr: lis.Addr().String()} + eng := newStubFollowerEngine(addr) p := NewLeaderProxyWithEngine(eng) t.Cleanup(func() { _ = p.connCache.Close() }) @@ -467,21 +429,12 @@ func TestForwardDecisionEvidence(t *testing.T) { func TestLeaderProxy_MultiPhaseConflictHasUnknownOutcome(t *testing.T) { t.Parallel() - var lc net.ListenConfig - lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") - require.NoError(t, err) svc := &fakeInternal{err: ForwardWriteConflictStatus()} - srv := grpc.NewServer() - pb.RegisterInternalServer(srv, svc) - go func() { _ = srv.Serve(lis) }() - t.Cleanup(func() { - srv.Stop() - _ = lis.Close() - }) + addr := grpctest.Serve(t, func(srv *grpc.Server) { pb.RegisterInternalServer(srv, svc) }) - p := NewLeaderProxyWithEngine(&stubFollowerEngine{leaderAddr: lis.Addr().String()}) + p := NewLeaderProxyWithEngine(newStubFollowerEngine(addr)) t.Cleanup(func() { _ = p.connCache.Close() }) - _, err = p.Commit(context.Background(), []*pb.Request{{IsTxn: true, Phase: pb.Phase_COMMIT}}) + _, err := p.Commit(context.Background(), []*pb.Request{{IsTxn: true, Phase: pb.Phase_COMMIT}}) require.ErrorIs(t, err, ErrTransactionOutcomeUnknown) require.Equal(t, 1, svc.callCount()) } @@ -489,21 +442,12 @@ func TestLeaderProxy_MultiPhaseConflictHasUnknownOutcome(t *testing.T) { func TestLeaderProxy_SinglePhaseConflictUsesStructuredStatus(t *testing.T) { t.Parallel() - var lc net.ListenConfig - lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") - require.NoError(t, err) svc := &fakeInternal{err: ForwardWriteConflictStatus()} - srv := grpc.NewServer() - pb.RegisterInternalServer(srv, svc) - go func() { _ = srv.Serve(lis) }() - t.Cleanup(func() { - srv.Stop() - _ = lis.Close() - }) + addr := grpctest.Serve(t, func(srv *grpc.Server) { pb.RegisterInternalServer(srv, svc) }) - p := NewLeaderProxyWithEngine(&stubFollowerEngine{leaderAddr: lis.Addr().String()}) + p := NewLeaderProxyWithEngine(newStubFollowerEngine(addr)) t.Cleanup(func() { _ = p.connCache.Close() }) - _, err = p.Commit(context.Background(), []*pb.Request{{IsTxn: false, Phase: pb.Phase_NONE}}) + _, err := p.Commit(context.Background(), []*pb.Request{{IsTxn: false, Phase: pb.Phase_NONE}}) require.ErrorIs(t, err, store.ErrWriteConflict) require.NotErrorIs(t, err, ErrTransactionOutcomeUnknown) require.Equal(t, 1, svc.callCount()) @@ -512,21 +456,12 @@ func TestLeaderProxy_SinglePhaseConflictUsesStructuredStatus(t *testing.T) { func TestLeaderProxy_SinglePhaseLockUsesStructuredStatus(t *testing.T) { t.Parallel() - var lc net.ListenConfig - lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") - require.NoError(t, err) svc := &fakeInternal{err: ForwardTxnLockedStatus()} - srv := grpc.NewServer() - pb.RegisterInternalServer(srv, svc) - go func() { _ = srv.Serve(lis) }() - t.Cleanup(func() { - srv.Stop() - _ = lis.Close() - }) + addr := grpctest.Serve(t, func(srv *grpc.Server) { pb.RegisterInternalServer(srv, svc) }) - p := NewLeaderProxyWithEngine(&stubFollowerEngine{leaderAddr: lis.Addr().String()}) + p := NewLeaderProxyWithEngine(newStubFollowerEngine(addr)) t.Cleanup(func() { _ = p.connCache.Close() }) - _, err = p.Commit(context.Background(), []*pb.Request{{IsTxn: false, Phase: pb.Phase_NONE}}) + _, err := p.Commit(context.Background(), []*pb.Request{{IsTxn: false, Phase: pb.Phase_NONE}}) require.ErrorIs(t, err, ErrTxnLocked) require.NotErrorIs(t, err, ErrTransactionOutcomeUnknown) require.True(t, IsForwardTxnLocked(err)) diff --git a/kv/leader_routed_store_test.go b/kv/leader_routed_store_test.go index 734ca93a4..2a9ea155f 100644 --- a/kv/leader_routed_store_test.go +++ b/kv/leader_routed_store_test.go @@ -3,10 +3,10 @@ package kv import ( "context" "fmt" - "net" "sync" "testing" + "github.com/bootjp/elastickv/internal/testutil/grpctest" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -90,8 +90,7 @@ func TestLeaderRoutedStore_GlobalCommittedTimestampFloorProxiesFencedLeader(t *t wantLatestGroupID: 1, latestResp: &pb.RawLatestCommitTSResponse{Ts: 30, GroupId: 1, LeaderFenced: true}, } - addr, stop := startRawKVServer(t, fake) - t.Cleanup(stop) + addr := startRawKVServer(t, fake) coord := &stubLeaderCoordinator{leader: addr} s := NewLeaderRoutedStore(store.NewMVCCStore(), coord) t.Cleanup(func() { _ = s.Close() }) @@ -189,24 +188,9 @@ func (f *fakeRawKVServer) RawLatestCommitTS(_ context.Context, req *pb.RawLatest return &pb.RawLatestCommitTSResponse{}, nil } -func startRawKVServer(t *testing.T, svc pb.RawKVServer) (string, func()) { +func startRawKVServer(t *testing.T, svc pb.RawKVServer) string { t.Helper() - - var lc net.ListenConfig - lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") - require.NoError(t, err) - - grpcServer := grpc.NewServer() - pb.RegisterRawKVServer(grpcServer, svc) - go func() { - _ = grpcServer.Serve(lis) - }() - - stop := func() { - grpcServer.Stop() - _ = lis.Close() - } - return lis.Addr().String(), stop + return grpctest.Serve(t, func(server *grpc.Server) { pb.RegisterRawKVServer(server, svc) }) } func TestLeaderRoutedStore_UsesLocalStoreWhenLeaderVerified(t *testing.T) { @@ -309,8 +293,7 @@ func TestLeaderRoutedStore_ProxiesReadsWhenFollower(t *testing.T) { Exists: true, }, } - addr, stop := startRawKVServer(t, fake) - t.Cleanup(stop) + addr := startRawKVServer(t, fake) coord := &stubLeaderCoordinator{ isLeader: false, @@ -360,8 +343,7 @@ func TestLeaderRoutedStore_ForwardsReadFenceStamps(t *testing.T) { Exists: true, }, } - addr, stop := startRawKVServer(t, fake) - t.Cleanup(stop) + addr := startRawKVServer(t, fake) coord := &stubLeaderCoordinator{ isLeader: false, @@ -394,8 +376,7 @@ func TestLeaderRoutedStore_ForwardsKeyScanReadFenceWithoutValues(t *testing.T) { fake := &fakeRawKVServer{ scanResp: &pb.RawScanAtResponse{Kv: []*pb.RawKVPair{{Key: []byte("k")}}}, } - addr, stop := startRawKVServer(t, fake) - t.Cleanup(stop) + addr := startRawKVServer(t, fake) coord := &stubLeaderCoordinator{isLeader: false, leader: addr, clock: NewHLC()} mvcc := store.NewMVCCStore() @@ -459,8 +440,7 @@ func TestLeaderRoutedStore_GlobalLastCommitTS_ProxiesToLeader(t *testing.T) { fake := &fakeRawKVServer{ latestResp: &pb.RawLatestCommitTSResponse{Ts: 99, Exists: true}, } - addr, stop := startRawKVServer(t, fake) - t.Cleanup(stop) + addr := startRawKVServer(t, fake) coord := &stubLeaderCoordinator{isLeader: false, leader: addr, clock: NewHLC()} s := NewLeaderRoutedStore(store.NewMVCCStore(), coord) diff --git a/kv/migration_frontier_admission_test.go b/kv/migration_frontier_admission_test.go index 280ccac32..233c92fbb 100644 --- a/kv/migration_frontier_admission_test.go +++ b/kv/migration_frontier_admission_test.go @@ -205,7 +205,7 @@ func TestMigrationFrontierAdmissionNonOwnerChildrenNeverGrant(t *testing.T) { func TestMigrationFrontierAdmissionStaleFenceRejectedBeforeProxy(t *testing.T) { f, engine, _ := frontierAdmissionSplitFixture(t) shards := NewShardStore(engine, map[uint64]*ShardGroup{ - 2: {Store: f.store, Engine: &stubFollowerEngine{leaderAddr: "127.0.0.1:1"}}, + 2: {Store: f.store, Engine: newStubFollowerEngine("127.0.0.1:1")}, }) t.Cleanup(func() { require.NoError(t, shards.Close()) }) _, err := shards.GetAtWithReadFence(t.Context(), []byte("n"), 200, 0, engine.Version()-1) diff --git a/kv/shard_store_exact_version_test.go b/kv/shard_store_exact_version_test.go index 485172a05..f14ef79f7 100644 --- a/kv/shard_store_exact_version_test.go +++ b/kv/shard_store_exact_version_test.go @@ -15,11 +15,10 @@ func TestShardStoreAuthoritativeExactVersionProxiesFollower(t *testing.T) { server := &fakeRawKVServer{latestResp: &pb.RawLatestCommitTSResponse{ ExactVersionExists: true, ExactVersionSupported: true, }} - address, stop := startRawKVServer(t, server) - t.Cleanup(stop) + address := startRawKVServer(t, server) routes := distribution.NewEngine() routes.UpdateRoute([]byte("a"), nil, 1) - follower := &stubFollowerEngine{leaderAddr: address} + follower := newStubFollowerEngine(address) shards := NewShardStore(routes, map[uint64]*ShardGroup{ 1: {Engine: follower, Store: store.NewMVCCStore()}, }) diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index d96d9d315..338c01c2c 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -710,8 +710,7 @@ func TestShardStoreS3BucketAuxiliaryOwnerProbeUsesLeaderRoutedReadFence(t *testi probe := &versionVisibleRawKVServer{ visible: map[string]bool{string(stagedKey): true}, } - addr, stop := startRawKVServer(t, probe) - t.Cleanup(stop) + addr := startRawKVServer(t, probe) engine := distribution.NewEngine() require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ @@ -765,8 +764,7 @@ func TestShardStoreS3BucketAuxiliaryOwnerProbeBatchesFollowerChecks(t *testing.T for _, key := range keys { probe.visible[string(distribution.MigrationStagedDataKey(9, key))] = true } - addr, stop := startRawKVServer(t, probe) - t.Cleanup(stop) + addr := startRawKVServer(t, probe) engine := distribution.NewEngine() require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ @@ -1887,8 +1885,7 @@ func TestShardStore_ForwardsReadFenceStamps(t *testing.T) { Exists: true, }, } - addr, stop := startRawKVServer(t, fake) - t.Cleanup(stop) + addr := startRawKVServer(t, fake) engine := distribution.NewEngine() require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ @@ -1900,7 +1897,7 @@ func TestShardStore_ForwardsReadFenceStamps(t *testing.T) { st := NewShardStore(engine, map[uint64]*ShardGroup{ 1: { Store: store.NewMVCCStore(), - Engine: &stubFollowerEngine{leaderAddr: addr}, + Engine: newStubFollowerEngine(addr), }, }) t.Cleanup(func() { _ = st.Close() }) @@ -2693,8 +2690,7 @@ func TestShardStoreProxyScanKeysAtUsesSelectedGroup(t *testing.T) { Kv: []*pb.RawKVPair{{Key: []byte("k"), Value: []byte("v")}}, }, } - addr, stop := startRawKVServer(t, fake) - t.Cleanup(stop) + addr := startRawKVServer(t, fake) ctx := context.Background() engine := distribution.NewEngine() @@ -2724,8 +2720,7 @@ func TestShardStoreProxyScanKeysAtCarriesStagedRouteBounds(t *testing.T) { Kv: []*pb.RawKVPair{{Key: []byte("k"), Value: []byte("v")}}, }, } - addr, stop := startRawKVServer(t, fake) - t.Cleanup(stop) + addr := startRawKVServer(t, fake) ctx := context.Background() g := &ShardGroup{ @@ -2764,8 +2759,7 @@ func TestShardStoreProxyScanAtUsesSelectedGroup(t *testing.T) { Kv: []*pb.RawKVPair{{Key: []byte("k"), Value: []byte("v")}}, }, } - addr, stop := startRawKVServer(t, fake) - t.Cleanup(stop) + addr := startRawKVServer(t, fake) ctx := context.Background() engine := distribution.NewEngine() @@ -2799,8 +2793,7 @@ func TestShardStoreProxyForwardPageAdvancesFromRawPage(t *testing.T) { }, }, } - addr, stop := startRawKVServer(t, fake) - t.Cleanup(stop) + addr := startRawKVServer(t, fake) ctx := context.Background() g := &ShardGroup{ @@ -3652,8 +3645,7 @@ func TestShardStoreScanAt_NilStartFanoutUsesExplicitGroupForProxy(t *testing.T) ctx := context.Background() recorder := &recordingRawScanKVServer{} - addr, stop := startRawKVServer(t, recorder) - t.Cleanup(stop) + addr := startRawKVServer(t, recorder) rawRouteEnd := fskeys.ChunkRouteKey(11, 22) chunkRouteEnd := prefixScanEnd(rawRouteEnd) diff --git a/kv/sharded_coordinator_leader_test.go b/kv/sharded_coordinator_leader_test.go index 5633fc186..88f6b2ebd 100644 --- a/kv/sharded_coordinator_leader_test.go +++ b/kv/sharded_coordinator_leader_test.go @@ -2,12 +2,12 @@ package kv import ( "context" - "net" "sync" "sync/atomic" "testing" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/testutil/grpctest" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -97,21 +97,12 @@ func TestShardedCoordinatorLeaseReadAllGroups_FencesEveryLeader(t *testing.T) { func TestShardedCoordinatorLeaseReadAllGroups_ForwardsFollowerToGroupLeader(t *testing.T) { t.Parallel() - var lc net.ListenConfig - lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") - require.NoError(t, err) service := &leaseReadForwardServer{} - server := grpc.NewServer() - pb.RegisterInternalServer(server, service) - go func() { _ = server.Serve(lis) }() - t.Cleanup(func() { - server.Stop() - _ = lis.Close() - }) + addr := grpctest.Serve(t, func(server *grpc.Server) { pb.RegisterInternalServer(server, service) }) engine := distribution.NewEngine() engine.UpdateRoute([]byte(""), nil, 1) - follower := &stubFollowerEngine{leaderAddr: lis.Addr().String()} + follower := newStubFollowerEngine(addr) group := &ShardGroup{Engine: follower, Store: store.NewMVCCStore()} proxy := NewLeaderProxyForShardGroup(group) group.SetLeaderReadToken("admin-secret") @@ -135,8 +126,8 @@ func TestShardedCoordinatorLeaseReadAllGroups_UsesConfiguredAllShardGroups(t *te engine.UpdateRoute([]byte(""), nil, 1) coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ - 0: {Engine: &stubFollowerEngine{}}, - 1: {Engine: stubLeaderEngine{}}, + 0: {Engine: newStubFollowerEngine("")}, + 1: {Engine: newStubLeaderEngine()}, }, 1, NewHLC(), nil).WithAllShardGroups(1) require.NoError(t, coord.LeaseReadAllGroups(context.Background())) diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 93cb6c00a..81ef2b691 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -178,7 +178,7 @@ func TestShardedCoordinatorValidateReadKeysOnShard_UsesStagedVisibility(t *testi readKey := []byte("k") require.NoError(t, st.PutAt(ctx, distribution.MigrationStagedDataKey(9, readKey), []byte("staged"), 20, 0)) coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ - 1: {Engine: stubLeaderEngine{}, Store: st}, + 1: {Engine: newStubLeaderEngine(), Store: st}, }, 1, NewHLC(), nil) err := coord.validateReadKeysOnShard(ctx, 1, [][]byte{readKey}, 10) @@ -1407,27 +1407,6 @@ func (s *stubMVCCStore) MigrationTargetReadinessStates(_ context.Context) ([]sto return s.readiness, nil } -// noopEngine satisfies raftengine.Engine for unit tests. -// LinearizableRead returns immediately (simulates an already-up-to-date FSM). -type noopEngine struct{} - -func (noopEngine) Propose(_ context.Context, _ []byte) (*raftengine.ProposalResult, error) { - return &raftengine.ProposalResult{}, nil -} -func (e noopEngine) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { - return e.Propose(ctx, data) -} -func (noopEngine) State() raftengine.State { return raftengine.StateLeader } -func (noopEngine) Leader() raftengine.LeaderInfo { return raftengine.LeaderInfo{} } -func (noopEngine) VerifyLeader(_ context.Context) error { return nil } -func (noopEngine) LinearizableRead(_ context.Context) (uint64, error) { return 0, nil } -func (noopEngine) Status() raftengine.Status { return raftengine.Status{} } -func (noopEngine) Configuration(_ context.Context) (raftengine.Configuration, error) { - return raftengine.Configuration{}, nil -} -func (noopEngine) SnapshotEvery() uint64 { return 10_000 } -func (noopEngine) Close() error { return nil } - func TestValidateReadOnlyShards_DetectsConflictOnReadOnlyShard(t *testing.T) { t.Parallel() engine := distribution.NewEngine() @@ -1439,7 +1418,7 @@ func TestValidateReadOnlyShards_DetectsConflictOnReadOnlyShard(t *testing.T) { }} coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ 1: {}, - 2: {Store: readOnlyStore, Engine: noopEngine{}}, + 2: {Store: readOnlyStore, Engine: newStubLeaderEngine()}, }, 1, NewHLC(), nil) groupedReadKeys := map[uint64][][]byte{ @@ -1462,7 +1441,7 @@ func TestValidateReadOnlyShards_SkipsWriteShards(t *testing.T) { "b": 20, }} coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ - 1: {Store: writeStore, Engine: noopEngine{}}, + 1: {Store: writeStore, Engine: newStubLeaderEngine()}, 2: {}, }, 1, NewHLC(), nil) @@ -1484,7 +1463,7 @@ func TestValidateReadOnlyShards_NoConflictWhenKeyUnchanged(t *testing.T) { }} coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ 1: {}, - 2: {Store: readOnlyStore, Engine: noopEngine{}}, + 2: {Store: readOnlyStore, Engine: newStubLeaderEngine()}, }, 1, NewHLC(), nil) groupedReadKeys := map[uint64][][]byte{ @@ -1535,7 +1514,7 @@ func TestValidateReadOnlyShards_FailsClosedOnTargetReadiness(t *testing.T) { } coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ 1: {}, - 2: {Store: readOnlyStore, Engine: noopEngine{}}, + 2: {Store: readOnlyStore, Engine: newStubLeaderEngine()}, }, 1, NewHLC(), nil) groupedReadKeys := map[uint64][][]byte{ @@ -1567,7 +1546,7 @@ func TestValidateReadOnlyShards_FailsClosedOnSourceReadFence(t *testing.T) { } coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ 1: {}, - 2: {Store: readOnlyStore, Engine: noopEngine{}}, + 2: {Store: readOnlyStore, Engine: newStubLeaderEngine()}, }, 1, NewHLC(), nil) err := coord.validateReadOnlyShards(context.Background(), map[uint64][][]byte{ @@ -1619,7 +1598,7 @@ func TestValidateReadOnlyShards_ChecksStagedVisibilityLatestCommitTS(t *testing. } coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ 1: {}, - 2: {Store: readOnlyStore, Engine: noopEngine{}}, + 2: {Store: readOnlyStore, Engine: newStubLeaderEngine()}, }, 1, NewHLC(), nil) groupedReadKeys := map[uint64][][]byte{ @@ -1639,7 +1618,7 @@ func TestValidateReadOnlyShards_PropagatesStoreError(t *testing.T) { readOnlyStore := &stubMVCCStore{returnErr: storeErr} coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ 1: {}, - 2: {Store: readOnlyStore, Engine: noopEngine{}}, + 2: {Store: readOnlyStore, Engine: newStubLeaderEngine()}, }, 1, NewHLC(), nil) groupedReadKeys := map[uint64][][]byte{ diff --git a/kv/sharded_membership_test.go b/kv/sharded_membership_test.go index 956840edd..f0b53d9cf 100644 --- a/kv/sharded_membership_test.go +++ b/kv/sharded_membership_test.go @@ -9,13 +9,10 @@ import ( "github.com/stretchr/testify/require" ) -type membershipConfigEngine struct { - noopEngine - config raftengine.Configuration -} - -func (e membershipConfigEngine) Configuration(context.Context) (raftengine.Configuration, error) { - return e.config, nil +func membershipConfigEngine(config raftengine.Configuration) *testEngine { + engine := newStubLeaderEngine() + engine.Config = config + return engine } func TestShardedCoordinatorRaftMembersIncludesEveryGroupEndpoint(t *testing.T) { @@ -25,14 +22,14 @@ func TestShardedCoordinatorRaftMembersIncludesEveryGroupEndpoint(t *testing.T) { routes.UpdateRoute(nil, []byte("m"), 1) routes.UpdateRoute([]byte("m"), nil, 2) groups := map[uint64]*ShardGroup{ - 1: {Engine: membershipConfigEngine{config: raftengine.Configuration{Servers: []raftengine.Server{ + 1: {Engine: membershipConfigEngine(raftengine.Configuration{Servers: []raftengine.Server{ {ID: "n1", Address: "127.0.0.1:5101", Suffrage: "voter"}, {ID: "n2", Address: "127.0.0.1:5102", Suffrage: "voter"}, - }}}}, - 2: {Engine: membershipConfigEngine{config: raftengine.Configuration{Servers: []raftengine.Server{ + }})}, + 2: {Engine: membershipConfigEngine(raftengine.Configuration{Servers: []raftengine.Server{ {ID: "n1", Address: "127.0.0.1:5201", Suffrage: "voter"}, {ID: "n3", Address: "127.0.0.1:5203", Suffrage: "learner"}, - }}}}, + }})}, } coordinator := NewShardedCoordinator(routes, groups, 1, NewHLC(), nil) diff --git a/kv/tso_floor_test.go b/kv/tso_floor_test.go index 277f44c51..81fcf577e 100644 --- a/kv/tso_floor_test.go +++ b/kv/tso_floor_test.go @@ -76,8 +76,7 @@ func TestShardStoreGlobalCommittedTimestampFloorUsesEveryGroupLeader(t *testing. }, wantLatestGroupID: 2, } - addr, stop := startRawKVServer(t, remote) - t.Cleanup(stop) + addr := startRawKVServer(t, remote) groups := map[uint64]*ShardGroup{ 0: {Engine: &recordingTSOEngine{state: raftengine.StateFollower}}, @@ -130,8 +129,7 @@ func TestShardStoreGlobalCommittedTimestampFloorRejectsUnfencedRemoteResponse(t latestResp: tc.resp, wantLatestGroupID: 2, } - addr, stop := startRawKVServer(t, remote) - t.Cleanup(stop) + addr := startRawKVServer(t, remote) shards := NewShardStore(nil, map[uint64]*ShardGroup{ 2: { @@ -203,14 +201,12 @@ func TestShardStoreGlobalCommittedTimestampFloorQueriesGroupsConcurrently(t *tes release := make(chan struct{}) started1 := make(chan struct{}) started2 := make(chan struct{}) - addr1, stop1 := startRawKVServer(t, &blockingFloorRawKVServer{ + addr1 := startRawKVServer(t, &blockingFloorRawKVServer{ ts: 11, started: started1, release: release, }) - addr2, stop2 := startRawKVServer(t, &blockingFloorRawKVServer{ + addr2 := startRawKVServer(t, &blockingFloorRawKVServer{ ts: 22, started: started2, release: release, }) - t.Cleanup(stop1) - t.Cleanup(stop2) shards := NewShardStore(nil, map[uint64]*ShardGroup{ 1: { diff --git a/kv/tso_test.go b/kv/tso_test.go index b55026186..b1a906b9b 100644 --- a/kv/tso_test.go +++ b/kv/tso_test.go @@ -100,7 +100,7 @@ func TestShardedCoordinatorBackupFloorDropsCachedTSOWindow(t *testing.T) { clock.SetPhysicalCeiling(time.Now().Add(testTSOFutureCeiling).UnixMilli()) coord := NewShardedCoordinator( distribution.NewEngineWithDefaultRoute(), - map[uint64]*ShardGroup{1: {Engine: stubLeaderEngine{}}}, + map[uint64]*ShardGroup{1: {Engine: newStubLeaderEngine()}}, 1, clock, nil, @@ -272,7 +272,7 @@ func TestShardedCoordinatorReportsAnyShardAsTimestampLeader(t *testing.T) { coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ 1: {}, - 2: {Engine: stubLeaderEngine{}}, + 2: {Engine: newStubLeaderEngine()}, }, 1, NewHLC(), nil) require.False(t, coord.IsLeader()) @@ -284,8 +284,8 @@ func TestShardedCoordinatorDedicatedGroupDoesNotBlockDataLeaderTimestampBridge(t engine.UpdateRoute([]byte(""), nil, 1) coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ - 0: {Engine: &stubFollowerEngine{}}, - 1: {Engine: stubLeaderEngine{}}, + 0: {Engine: newStubFollowerEngine("")}, + 1: {Engine: newStubLeaderEngine()}, }, 1, NewHLC(), nil) require.True(t, coord.IsTimestampLeader()) @@ -297,8 +297,8 @@ func TestShardedCoordinatorTimestampBridgeIgnoresReservedGroupLeader(t *testing. newCoord := func() *ShardedCoordinator { return NewShardedCoordinator(engine, map[uint64]*ShardGroup{ - 0: {Engine: stubLeaderEngine{}}, - 1: {Engine: &stubFollowerEngine{}}, + 0: {Engine: newStubLeaderEngine()}, + 1: {Engine: newStubFollowerEngine("")}, }, 1, NewHLC(), nil) } @@ -317,16 +317,16 @@ func TestShardedCoordinatorUsesDedicatedTimestampGroupLeader(t *testing.T) { engine.UpdateRoute([]byte("m"), nil, 2) dataLeaderOnly := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ - 0: {Engine: &stubFollowerEngine{}}, - 1: {Engine: stubLeaderEngine{}}, - 2: {Engine: &stubFollowerEngine{}}, + 0: {Engine: newStubFollowerEngine("")}, + 1: {Engine: newStubLeaderEngine()}, + 2: {Engine: newStubFollowerEngine("")}, }, 1, NewHLC(), nil).WithTimestampGroup(0) require.False(t, dataLeaderOnly.IsTimestampLeader()) tsoLeader := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ - 0: {Engine: stubLeaderEngine{}}, - 1: {Engine: &stubFollowerEngine{}}, - 2: {Engine: &stubFollowerEngine{}}, + 0: {Engine: newStubLeaderEngine()}, + 1: {Engine: newStubFollowerEngine("")}, + 2: {Engine: newStubFollowerEngine("")}, }, 1, NewHLC(), nil).WithTimestampGroup(0) require.True(t, tsoLeader.IsTimestampLeader()) } @@ -558,7 +558,7 @@ func TestShardedCoordinatorRawFollowerDefersTSOAllocationToLeaderPath(t *testing alloc := &blockingTimestampAllocator{} coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ 1: { - Engine: &stubFollowerEngine{}, + Engine: newStubFollowerEngine(""), Txn: txn, }, }, 1, NewHLC(), nil).WithTSOAllocator(alloc) @@ -881,9 +881,9 @@ func TestShardedCoordinatorPinsCandidateTimestampGroupOnPhaseD(t *testing.T) { state := &mutableTSOCutoverState{} groups := map[uint64]*ShardGroup{ - 0: {Engine: stubLeaderEngine{}}, - 1: {Engine: stubLeaderEngine{}}, - 2: {Engine: stubLeaderEngine{}}, + 0: {Engine: newStubLeaderEngine()}, + 1: {Engine: newStubLeaderEngine()}, + 2: {Engine: newStubLeaderEngine()}, } coord := NewShardedCoordinator(engine, groups, 1, NewHLC(), nil). WithAllShardGroups(1, 2). @@ -923,8 +923,8 @@ func TestShardedCoordinatorKeepsRenewingWithoutTimestampGroupUnderPhaseD(t *test state.cutover.Store(true) state.phaseD.Store(true) groups := map[uint64]*ShardGroup{ - 1: {Engine: stubLeaderEngine{}}, - 2: {Engine: stubLeaderEngine{}}, + 1: {Engine: newStubLeaderEngine()}, + 2: {Engine: newStubLeaderEngine()}, } coord := NewShardedCoordinator(engine, groups, 1, NewHLC(), nil). WithTSOCutoverState(state) @@ -939,8 +939,8 @@ func TestShardedCoordinatorExplicitTimestampGroupStillPinsWithoutPhaseD(t *testi engine.UpdateRoute([]byte(""), nil, 1) groups := map[uint64]*ShardGroup{ - 0: {Engine: stubLeaderEngine{}}, - 1: {Engine: &stubFollowerEngine{}}, + 0: {Engine: newStubLeaderEngine()}, + 1: {Engine: newStubFollowerEngine("")}, } coord := NewShardedCoordinator(engine, groups, 1, NewHLC(), nil).WithTimestampGroup(0) From 234d81ff92a5a937080fc7ffae2ae9f3bde7d683 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 16:08:06 +0000 Subject: [PATCH 12/15] test: group testutil imports for gci Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- adapter/admin_grpc_test.go | 3 +-- adapter/background_reservation_shutdown_test.go | 1 - adapter/distribution_server_test.go | 3 +-- adapter/dynamodb_read_fence_test.go | 1 - adapter/dynamodb_retention_test.go | 1 - adapter/dynamodb_table_cleanup_shutdown_test.go | 1 - adapter/dynamodb_test.go | 3 +-- adapter/http_shutdown_test.go | 1 - adapter/redis_apply_observer_test.go | 1 - adapter/redis_bzpopmin_wake_test.go | 1 - adapter/redis_compat_commands_stream_test.go | 1 - adapter/redis_shutdown_test.go | 1 - adapter/s3_admin_delete_shutdown_test.go | 1 - adapter/s3_cleanup_shutdown_test.go | 1 - adapter/s3_test.go | 3 +-- adapter/sqs_shutdown_test.go | 1 - adapter/sqs_test.go | 1 - internal/filesystem/fuseadapter/abandoned_mount_test.go | 3 +-- internal/filesystem/fuseadapter/ack_test.go | 3 +-- internal/filesystem/fuseadapter/adapter_test.go | 3 +-- internal/filesystem/fuseadapter/barrier_test.go | 3 +-- .../filesystem/fuseadapter/helper_lookup_snapshot_test.go | 1 - internal/filesystem/fuseadapter/server_ops_test.go | 3 +-- internal/filesystem/fuseadapter/unmount_lazy_cancel_test.go | 1 - .../filesystem/fuseadapter/unmount_shutdown_join_test.go | 1 - internal/filesystem/fuseadapter/wait_mount_cancel_test.go | 3 +-- internal/filesystem/operation_journal_multigroup_test.go | 3 +-- internal/raftengine/etcd/dispatch_report_test.go | 1 - internal/raftengine/etcd/engine_applied_index_test.go | 3 +-- internal/raftengine/etcd/engine_test.go | 5 ++--- internal/raftengine/etcd/grpc_transport_test.go | 1 - internal/raftengine/etcd/removal_notification_test.go | 3 +-- internal/snapshotoffload/scheduler_test.go | 3 +-- keyviz/flusher_test.go | 1 - kv/lease_warmup_test.go | 3 +-- kv/lock_resolver_test.go | 3 +-- kv/raft_engine_test.go | 3 +-- main_admin_lifecycle_test.go | 3 +-- main_admin_shutdown_test.go | 3 +-- main_encryption_rotate_on_startup_test.go | 3 +-- main_filesystem_test.go | 3 +-- main_grpc_stop_test.go | 1 - main_shutdown_balance_test.go | 3 +-- main_shutdown_handoff_test.go | 3 +-- main_shutdown_signal_test.go | 3 +-- main_shutdown_test.go | 3 +-- monitoring/http_shutdown_test.go | 1 - proxy/proxy_test.go | 1 - store/lsm_store_test.go | 3 +-- 49 files changed, 28 insertions(+), 77 deletions(-) diff --git a/adapter/admin_grpc_test.go b/adapter/admin_grpc_test.go index c9bc619ca..819e05974 100644 --- a/adapter/admin_grpc_test.go +++ b/adapter/admin_grpc_test.go @@ -7,9 +7,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/testutil" pb "github.com/bootjp/elastickv/proto" "github.com/stretchr/testify/require" "google.golang.org/grpc" diff --git a/adapter/background_reservation_shutdown_test.go b/adapter/background_reservation_shutdown_test.go index c2aad568e..4b6cb7bf3 100644 --- a/adapter/background_reservation_shutdown_test.go +++ b/adapter/background_reservation_shutdown_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/stretchr/testify/require" ) diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index df37f4f7c..4f9e5ef28 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -11,12 +11,11 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/fskeys" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/internal/s3keys" + "github.com/bootjp/elastickv/internal/testutil" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" diff --git a/adapter/dynamodb_read_fence_test.go b/adapter/dynamodb_read_fence_test.go index bc63abbae..6cb7da813 100644 --- a/adapter/dynamodb_read_fence_test.go +++ b/adapter/dynamodb_read_fence_test.go @@ -11,7 +11,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" json "github.com/goccy/go-json" diff --git a/adapter/dynamodb_retention_test.go b/adapter/dynamodb_retention_test.go index 97f91e1c1..d088a34ab 100644 --- a/adapter/dynamodb_retention_test.go +++ b/adapter/dynamodb_retention_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" diff --git a/adapter/dynamodb_table_cleanup_shutdown_test.go b/adapter/dynamodb_table_cleanup_shutdown_test.go index 4b572541e..ec6681eac 100644 --- a/adapter/dynamodb_table_cleanup_shutdown_test.go +++ b/adapter/dynamodb_table_cleanup_shutdown_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" diff --git a/adapter/dynamodb_test.go b/adapter/dynamodb_test.go index bec1a64be..4b2edd714 100644 --- a/adapter/dynamodb_test.go +++ b/adapter/dynamodb_test.go @@ -12,13 +12,12 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/bootjp/elastickv/internal/testutil" "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/assert" diff --git a/adapter/http_shutdown_test.go b/adapter/http_shutdown_test.go index 4bdd5b3bc..61cb37d1b 100644 --- a/adapter/http_shutdown_test.go +++ b/adapter/http_shutdown_test.go @@ -11,7 +11,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" ) diff --git a/adapter/redis_apply_observer_test.go b/adapter/redis_apply_observer_test.go index 0809af958..42b88aa0f 100644 --- a/adapter/redis_apply_observer_test.go +++ b/adapter/redis_apply_observer_test.go @@ -5,7 +5,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" diff --git a/adapter/redis_bzpopmin_wake_test.go b/adapter/redis_bzpopmin_wake_test.go index e333a9b6a..7ed30d126 100644 --- a/adapter/redis_bzpopmin_wake_test.go +++ b/adapter/redis_bzpopmin_wake_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/store" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" diff --git a/adapter/redis_compat_commands_stream_test.go b/adapter/redis_compat_commands_stream_test.go index d34516dd5..a4b593c21 100644 --- a/adapter/redis_compat_commands_stream_test.go +++ b/adapter/redis_compat_commands_stream_test.go @@ -10,7 +10,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" "github.com/redis/go-redis/v9" diff --git a/adapter/redis_shutdown_test.go b/adapter/redis_shutdown_test.go index d2eefeb3f..dc0f48ec1 100644 --- a/adapter/redis_shutdown_test.go +++ b/adapter/redis_shutdown_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/stretchr/testify/require" "github.com/tidwall/redcon" ) diff --git a/adapter/s3_admin_delete_shutdown_test.go b/adapter/s3_admin_delete_shutdown_test.go index d2e586bd2..e21e484fa 100644 --- a/adapter/s3_admin_delete_shutdown_test.go +++ b/adapter/s3_admin_delete_shutdown_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" diff --git a/adapter/s3_cleanup_shutdown_test.go b/adapter/s3_cleanup_shutdown_test.go index 2a171dc25..9ba3b57bc 100644 --- a/adapter/s3_cleanup_shutdown_test.go +++ b/adapter/s3_cleanup_shutdown_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" diff --git a/adapter/s3_test.go b/adapter/s3_test.go index 3fdf177af..1dbbf06e3 100644 --- a/adapter/s3_test.go +++ b/adapter/s3_test.go @@ -20,14 +20,13 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/aws/aws-sdk-go-v2/aws" v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" "github.com/bootjp/elastickv/internal/s3keys" + "github.com/bootjp/elastickv/internal/testutil" "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" json "github.com/goccy/go-json" diff --git a/adapter/sqs_shutdown_test.go b/adapter/sqs_shutdown_test.go index 4d135edfa..7f02ea2b9 100644 --- a/adapter/sqs_shutdown_test.go +++ b/adapter/sqs_shutdown_test.go @@ -5,7 +5,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/stretchr/testify/require" ) diff --git a/adapter/sqs_test.go b/adapter/sqs_test.go index 8ecd2e40d..474d1d8e8 100644 --- a/adapter/sqs_test.go +++ b/adapter/sqs_test.go @@ -11,7 +11,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - json "github.com/goccy/go-json" ) diff --git a/internal/filesystem/fuseadapter/abandoned_mount_test.go b/internal/filesystem/fuseadapter/abandoned_mount_test.go index e5d04bcc6..cd416b649 100644 --- a/internal/filesystem/fuseadapter/abandoned_mount_test.go +++ b/internal/filesystem/fuseadapter/abandoned_mount_test.go @@ -7,9 +7,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/filesystem/fstest" + "github.com/bootjp/elastickv/internal/testutil" "github.com/hanwen/go-fuse/v2/fuse" "github.com/stretchr/testify/require" ) diff --git a/internal/filesystem/fuseadapter/ack_test.go b/internal/filesystem/fuseadapter/ack_test.go index 479277f30..cde49d5ee 100644 --- a/internal/filesystem/fuseadapter/ack_test.go +++ b/internal/filesystem/fuseadapter/ack_test.go @@ -6,9 +6,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/filesystem" + "github.com/bootjp/elastickv/internal/testutil" "github.com/hanwen/go-fuse/v2/fuse" "github.com/stretchr/testify/require" ) diff --git a/internal/filesystem/fuseadapter/adapter_test.go b/internal/filesystem/fuseadapter/adapter_test.go index 660b8b002..5d5ad0ea0 100644 --- a/internal/filesystem/fuseadapter/adapter_test.go +++ b/internal/filesystem/fuseadapter/adapter_test.go @@ -7,9 +7,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/filesystem" + "github.com/bootjp/elastickv/internal/testutil" "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" cerrors "github.com/cockroachdb/errors" diff --git a/internal/filesystem/fuseadapter/barrier_test.go b/internal/filesystem/fuseadapter/barrier_test.go index 67c79b2de..f38bf6b26 100644 --- a/internal/filesystem/fuseadapter/barrier_test.go +++ b/internal/filesystem/fuseadapter/barrier_test.go @@ -7,9 +7,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/filesystem" + "github.com/bootjp/elastickv/internal/testutil" "github.com/bootjp/elastickv/kv" "github.com/stretchr/testify/require" ) diff --git a/internal/filesystem/fuseadapter/helper_lookup_snapshot_test.go b/internal/filesystem/fuseadapter/helper_lookup_snapshot_test.go index 4f9ce1dba..0d6545312 100644 --- a/internal/filesystem/fuseadapter/helper_lookup_snapshot_test.go +++ b/internal/filesystem/fuseadapter/helper_lookup_snapshot_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/stretchr/testify/require" ) diff --git a/internal/filesystem/fuseadapter/server_ops_test.go b/internal/filesystem/fuseadapter/server_ops_test.go index 510e98de0..82aa7ec09 100644 --- a/internal/filesystem/fuseadapter/server_ops_test.go +++ b/internal/filesystem/fuseadapter/server_ops_test.go @@ -9,9 +9,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/filesystem" + "github.com/bootjp/elastickv/internal/testutil" "github.com/bootjp/elastickv/store" "github.com/hanwen/go-fuse/v2/fuse" "github.com/stretchr/testify/require" diff --git a/internal/filesystem/fuseadapter/unmount_lazy_cancel_test.go b/internal/filesystem/fuseadapter/unmount_lazy_cancel_test.go index 3c8b647e0..2d901fea9 100644 --- a/internal/filesystem/fuseadapter/unmount_lazy_cancel_test.go +++ b/internal/filesystem/fuseadapter/unmount_lazy_cancel_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/stretchr/testify/require" ) diff --git a/internal/filesystem/fuseadapter/unmount_shutdown_join_test.go b/internal/filesystem/fuseadapter/unmount_shutdown_join_test.go index 9a730f698..ec3bac5c9 100644 --- a/internal/filesystem/fuseadapter/unmount_shutdown_join_test.go +++ b/internal/filesystem/fuseadapter/unmount_shutdown_join_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/hanwen/go-fuse/v2/fuse" "github.com/stretchr/testify/require" ) diff --git a/internal/filesystem/fuseadapter/wait_mount_cancel_test.go b/internal/filesystem/fuseadapter/wait_mount_cancel_test.go index 217815448..e4e4631c8 100644 --- a/internal/filesystem/fuseadapter/wait_mount_cancel_test.go +++ b/internal/filesystem/fuseadapter/wait_mount_cancel_test.go @@ -10,10 +10,9 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/filesystem" "github.com/bootjp/elastickv/internal/filesystem/fstest" + "github.com/bootjp/elastickv/internal/testutil" "github.com/hanwen/go-fuse/v2/fuse" "github.com/stretchr/testify/require" ) diff --git a/internal/filesystem/operation_journal_multigroup_test.go b/internal/filesystem/operation_journal_multigroup_test.go index 92e7e897b..bcac30855 100644 --- a/internal/filesystem/operation_journal_multigroup_test.go +++ b/internal/filesystem/operation_journal_multigroup_test.go @@ -9,12 +9,11 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/fskeys" "github.com/bootjp/elastickv/internal/raftengine" etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/internal/testutil" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" diff --git a/internal/raftengine/etcd/dispatch_report_test.go b/internal/raftengine/etcd/dispatch_report_test.go index f21cbbb10..338be3226 100644 --- a/internal/raftengine/etcd/dispatch_report_test.go +++ b/internal/raftengine/etcd/dispatch_report_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/stretchr/testify/require" raftpb "go.etcd.io/raft/v3/raftpb" ) diff --git a/internal/raftengine/etcd/engine_applied_index_test.go b/internal/raftengine/etcd/engine_applied_index_test.go index ad887bd07..cf301dd34 100644 --- a/internal/raftengine/etcd/engine_applied_index_test.go +++ b/internal/raftengine/etcd/engine_applied_index_test.go @@ -6,9 +6,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/testutil" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/require" etcdraft "go.etcd.io/raft/v3" diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index 094b3dc89..0c2f4b478 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -12,11 +12,10 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/raftengine/raftenginetest" - "github.com/bootjp/elastickv/internal/testutil" - internalutil "github.com/bootjp/elastickv/internal" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/raftengine/raftenginetest" + "github.com/bootjp/elastickv/internal/testutil" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" "go.etcd.io/etcd/server/v3/mock/mockstorage" diff --git a/internal/raftengine/etcd/grpc_transport_test.go b/internal/raftengine/etcd/grpc_transport_test.go index badfc8911..9c4041a4f 100644 --- a/internal/raftengine/etcd/grpc_transport_test.go +++ b/internal/raftengine/etcd/grpc_transport_test.go @@ -13,7 +13,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" diff --git a/internal/raftengine/etcd/removal_notification_test.go b/internal/raftengine/etcd/removal_notification_test.go index ef4f9a5be..a840bddef 100644 --- a/internal/raftengine/etcd/removal_notification_test.go +++ b/internal/raftengine/etcd/removal_notification_test.go @@ -6,9 +6,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/testutil" "github.com/stretchr/testify/require" raftpb "go.etcd.io/raft/v3/raftpb" "google.golang.org/grpc/codes" diff --git a/internal/snapshotoffload/scheduler_test.go b/internal/snapshotoffload/scheduler_test.go index 6d183a8d9..c812d3a55 100644 --- a/internal/snapshotoffload/scheduler_test.go +++ b/internal/snapshotoffload/scheduler_test.go @@ -12,9 +12,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/internal/testutil" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" ) diff --git a/keyviz/flusher_test.go b/keyviz/flusher_test.go index 70affe323..084ae747f 100644 --- a/keyviz/flusher_test.go +++ b/keyviz/flusher_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/stretchr/testify/require" ) diff --git a/kv/lease_warmup_test.go b/kv/lease_warmup_test.go index ee1e42373..a9bddacd2 100644 --- a/kv/lease_warmup_test.go +++ b/kv/lease_warmup_test.go @@ -8,11 +8,10 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/monoclock" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/testutil" "github.com/stretchr/testify/require" ) diff --git a/kv/lock_resolver_test.go b/kv/lock_resolver_test.go index 4cc978e61..2e4d4cb7a 100644 --- a/kv/lock_resolver_test.go +++ b/kv/lock_resolver_test.go @@ -7,10 +7,9 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/testutil" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/pebble/v2" diff --git a/kv/raft_engine_test.go b/kv/raft_engine_test.go index 047267ca8..635a094b5 100644 --- a/kv/raft_engine_test.go +++ b/kv/raft_engine_test.go @@ -5,9 +5,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/testutil" "github.com/cockroachdb/errors" ) diff --git a/main_admin_lifecycle_test.go b/main_admin_lifecycle_test.go index b729b07e6..75eaa01ec 100644 --- a/main_admin_lifecycle_test.go +++ b/main_admin_lifecycle_test.go @@ -7,10 +7,9 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/adapter" "github.com/bootjp/elastickv/internal/admin" + "github.com/bootjp/elastickv/internal/testutil" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" diff --git a/main_admin_shutdown_test.go b/main_admin_shutdown_test.go index 04ec627ce..7af5eb60e 100644 --- a/main_admin_shutdown_test.go +++ b/main_admin_shutdown_test.go @@ -8,9 +8,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/admin" + "github.com/bootjp/elastickv/internal/testutil" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) diff --git a/main_encryption_rotate_on_startup_test.go b/main_encryption_rotate_on_startup_test.go index 99913af0e..21a26e1d1 100644 --- a/main_encryption_rotate_on_startup_test.go +++ b/main_encryption_rotate_on_startup_test.go @@ -12,12 +12,11 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/adapter" "github.com/bootjp/elastickv/internal/encryption" "github.com/bootjp/elastickv/internal/encryption/fsmwire" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/testutil" "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/monitoring" pb "github.com/bootjp/elastickv/proto" diff --git a/main_filesystem_test.go b/main_filesystem_test.go index fe1306ed6..806711896 100644 --- a/main_filesystem_test.go +++ b/main_filesystem_test.go @@ -10,12 +10,11 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - internalutil "github.com/bootjp/elastickv/internal" "github.com/bootjp/elastickv/internal/filesystem" "github.com/bootjp/elastickv/internal/filesystem/fstest" "github.com/bootjp/elastickv/internal/filesystem/fuseadapter" + "github.com/bootjp/elastickv/internal/testutil" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) diff --git a/main_grpc_stop_test.go b/main_grpc_stop_test.go index ba8ff0d4a..2df9b3f17 100644 --- a/main_grpc_stop_test.go +++ b/main_grpc_stop_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" diff --git a/main_shutdown_balance_test.go b/main_shutdown_balance_test.go index 1ed047cc1..20c07059f 100644 --- a/main_shutdown_balance_test.go +++ b/main_shutdown_balance_test.go @@ -7,9 +7,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/testutil" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) diff --git a/main_shutdown_handoff_test.go b/main_shutdown_handoff_test.go index d30b37c1e..78eabaa2c 100644 --- a/main_shutdown_handoff_test.go +++ b/main_shutdown_handoff_test.go @@ -9,9 +9,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/testutil" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) diff --git a/main_shutdown_signal_test.go b/main_shutdown_signal_test.go index 814c612cc..536310330 100644 --- a/main_shutdown_signal_test.go +++ b/main_shutdown_signal_test.go @@ -8,9 +8,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - internalutil "github.com/bootjp/elastickv/internal" + "github.com/bootjp/elastickv/internal/testutil" "github.com/cockroachdb/errors" "github.com/hanwen/go-fuse/v2/fuse" "github.com/stretchr/testify/require" diff --git a/main_shutdown_test.go b/main_shutdown_test.go index de4d7bb2c..40994d8b9 100644 --- a/main_shutdown_test.go +++ b/main_shutdown_test.go @@ -8,9 +8,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/testutil" "github.com/stretchr/testify/require" ) diff --git a/monitoring/http_shutdown_test.go b/monitoring/http_shutdown_test.go index e678fa297..efef1410d 100644 --- a/monitoring/http_shutdown_test.go +++ b/monitoring/http_shutdown_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/testutil" - "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 5db6c95f8..a48e38962 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -14,7 +14,6 @@ import ( "time" waitutil "github.com/bootjp/elastickv/internal/testutil" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/redis/go-redis/v9" diff --git a/store/lsm_store_test.go b/store/lsm_store_test.go index 2a3ce76e0..14a8ab572 100644 --- a/store/lsm_store_test.go +++ b/store/lsm_store_test.go @@ -8,9 +8,8 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/internal/testutil" - "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/testutil" "github.com/cockroachdb/pebble/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" From 5a197196ca82f48b8fcf8b872eef81edd493f020 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 16:32:32 +0000 Subject: [PATCH 13/15] test: route in-memory apply coordinators through kvtest (cleanup WP-9..12) localAdapterCoordinator and occAdapterCoordinator now dispatch through kvtest.LocalCoordinator (standalone DEL_PREFIX applied; OCC with read-key validation for the OCC variant) while keeping stubAdapterCoordinator's configurable leadership surface; LocalOptions gains a Clock so the wrapper and the dispatcher share one HLC. fstest.MemoryDispatcher and the filesystem-internal testCoordinator, which were byte-for-byte the same policy, now both use LocalCoordinator with store-increment timestamps, AdvanceCommitTS, OCC on every group and commit-timestamp patching. The near-identical HELLO and INFO coordinator fakes fold into stubAdapterCoordinator (which gains raftLeader), testCoordinatorWrapper embeds kv.Coordinator instead of forwarding eight methods by hand, and the panicking kv fakeGroupRoutableCoordinator embeds a nil Coordinator. Behaviour notes: LocalCoordinator serializes dispatches, copies elements, checks ctx before applying, reports CommitTS/CommitIndex in the response, and returns ErrUnknownOp for unknown ops; an invalid commit-timestamp patch now surfaces kv's validation error instead of filesystem.ErrInvalid. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- adapter/dynamodb_migration_test.go | 116 ++++------------------- adapter/dynamodb_test.go | 48 ++-------- adapter/redis_hello_test.go | 38 +------- adapter/redis_info_test.go | 38 +------- adapter/redis_keys_pattern_test.go | 5 +- adapter/redis_list_pop_benchmark_test.go | 55 ++--------- internal/filesystem/fstest/memory.go | 66 ++++--------- internal/filesystem/service_test.go | 68 +++++-------- kv/kvtest/coordinator.go | 9 +- kv/kvtest/coordinator_test.go | 9 ++ kv/lease_group_keys_test.go | 19 +--- 11 files changed, 106 insertions(+), 365 deletions(-) diff --git a/adapter/dynamodb_migration_test.go b/adapter/dynamodb_migration_test.go index 574cbffde..02c000be3 100644 --- a/adapter/dynamodb_migration_test.go +++ b/adapter/dynamodb_migration_test.go @@ -7,122 +7,44 @@ import ( "time" "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/kv/kvtest" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" ) +// localAdapterCoordinator applies dispatches to an in-memory store through +// kvtest.LocalCoordinator while keeping stubAdapterCoordinator's configurable +// leadership surface. Both share one HLC. DEL_PREFIX is applied only as a +// standalone, non-transactional group, mirroring the production coordinator's +// dispatch-time rejection rules (Codex P1 on PR #695: a txn mixing Del and +// DelPrefix must fail here as it does in production). type localAdapterCoordinator struct { stubAdapterCoordinator store store.MVCCStore + local *kvtest.LocalCoordinator } func newLocalAdapterCoordinator(st store.MVCCStore) *localAdapterCoordinator { + return newLocalAdapterCoordinatorWithOptions(st, kvtest.LocalOptions{DelPrefix: kvtest.PrefixApply}) +} + +func newLocalAdapterCoordinatorWithOptions(st store.MVCCStore, options kvtest.LocalOptions) *localAdapterCoordinator { + clock := kv.NewHLC() + options.Clock = clock return &localAdapterCoordinator{ - stubAdapterCoordinator: stubAdapterCoordinator{clock: kv.NewHLC()}, + stubAdapterCoordinator: stubAdapterCoordinator{clock: clock}, store: st, + local: kvtest.NewLocalCoordinator(st, options), } } func (c *localAdapterCoordinator) Dispatch(ctx context.Context, req *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { - if req == nil { - return &kv.CoordinateResponse{}, nil - } - if err := c.validateDispatchShape(req); err != nil { - return nil, err - } - commitTS, err := c.commitTSForRequest(req) + resp, err := c.local.Dispatch(ctx, req) if err != nil { - return nil, err - } - if err := c.applyElems(ctx, req.Elems, commitTS); err != nil { - return nil, err - } - return &kv.CoordinateResponse{CommitTS: commitTS}, nil -} - -// validateDispatchShape mirrors the production coordinator's -// dispatch-time rejection rules so tests catch the same class of -// bug that real clusters would reject. Specifically: -// -// - DEL_PREFIX cannot be in a transactional OperationGroup -// (kv/sharded_coordinator.go: dispatchDelPrefixBroadcast -// refuses IsTxn=true). -// - DEL_PREFIX cannot be mixed with Put or Del in the same -// OperationGroup (validateDelPrefixOnly enforces all-or-none). -// -// Without these checks, a regression that ships -// `IsTxn:true with [Del, DelPrefix...]` (Codex P1 on PR #695) -// would silently pass the local coordinator while production -// rejected every bucket delete with ErrInvalidRequest. -func (c *localAdapterCoordinator) validateDispatchShape(req *kv.OperationGroup[kv.OP]) error { - hasDelPrefix := false - hasOther := false - for _, elem := range req.Elems { - if elem == nil { - continue - } - if elem.Op == kv.DelPrefix { - hasDelPrefix = true - } else { - hasOther = true - } - } - if !hasDelPrefix { - return nil - } - if req.IsTxn { - return errors.Wrap(kv.ErrInvalidRequest, "DEL_PREFIX not supported in transactions") - } - if hasOther { - return errors.Wrap(kv.ErrInvalidRequest, "DEL_PREFIX cannot be mixed with other operations") - } - return nil -} - -func (c *localAdapterCoordinator) commitTSForRequest(req *kv.OperationGroup[kv.OP]) (uint64, error) { - if req == nil { - return 0, nil - } - commitTS := req.CommitTS - if commitTS == 0 { - commitTS = c.Clock().Next() - if req.IsTxn && commitTS <= req.StartTS { - c.Clock().Observe(req.StartTS) - commitTS = c.Clock().Next() - } - } else { - c.Clock().Observe(commitTS) - } - if req.IsTxn && commitTS <= req.StartTS { - return 0, kv.ErrTxnCommitTSRequired - } - return commitTS, nil -} - -func (c *localAdapterCoordinator) applyElems(ctx context.Context, elems []*kv.Elem[kv.OP], commitTS uint64) error { - for _, elem := range elems { - if err := c.applyElem(ctx, elem, commitTS); err != nil { - return err - } - } - return nil -} - -func (c *localAdapterCoordinator) applyElem(ctx context.Context, elem *kv.Elem[kv.OP], commitTS uint64) error { - if elem == nil { - return nil - } - switch elem.Op { - case kv.Put: - return c.store.PutAt(ctx, elem.Key, elem.Value, commitTS, 0) - case kv.Del: - return c.store.DeleteAt(ctx, elem.Key, commitTS) - case kv.DelPrefix: - return c.store.DeletePrefixAt(ctx, elem.Key, nil, commitTS) - default: - return nil + return nil, errors.WithStack(err) } + return resp, nil } func newLegacyMigrationTestServer( diff --git a/adapter/dynamodb_test.go b/adapter/dynamodb_test.go index 4b2edd714..251109a1c 100644 --- a/adapter/dynamodb_test.go +++ b/adapter/dynamodb_test.go @@ -371,7 +371,7 @@ func postDynamoRaw(t *testing.T, address, target, body string) (int, string) { func wrapDynamoCoordinator(t *testing.T, node *Node) *testCoordinatorWrapper { t.Helper() orig := node.dynamoServer.coordinator - wrapped := &testCoordinatorWrapper{inner: orig} + wrapped := &testCoordinatorWrapper{Coordinator: orig} node.dynamoServer.coordinator = wrapped t.Cleanup(func() { node.dynamoServer.coordinator = orig @@ -1762,7 +1762,7 @@ func TestDynamoDB_TransactWriteItems_RetriesOnWriteConflict(t *testing.T) { require.NoError(t, err) orig := nodes[0].dynamoServer.coordinator - wrapped := &testCoordinatorWrapper{inner: orig} + wrapped := &testCoordinatorWrapper{Coordinator: orig} wrapped.failTxnDispatches.Store(1) nodes[0].dynamoServer.coordinator = wrapped defer func() { @@ -1850,7 +1850,7 @@ func TestDynamoDB_TransactWriteItems_ConditionCheckRace(t *testing.T) { orig := nodes[0].dynamoServer.coordinator wrapped := &testCoordinatorWrapper{ - inner: orig, + Coordinator: orig, blockEntered: make(chan struct{}), blockRelease: make(chan struct{}), } @@ -1940,7 +1940,7 @@ func TestDynamoDB_TransactWriteItems_ConditionCheckRace(t *testing.T) { var errInjectedLeaseRead = errors.New("injected lease-read failure") type testCoordinatorWrapper struct { - inner kv.Coordinator + kv.Coordinator failTxnDispatches atomic.Int32 injectedFailures atomic.Int32 @@ -1994,7 +1994,7 @@ func (w *testCoordinatorWrapper) leaseCallCounts() (keyless, keyed int) { // EngineGroupIDForKey forwards group resolution so LeaseReadGroupKeys can // dedup by group through the wrapper. func (w *testCoordinatorWrapper) EngineGroupIDForKey(key []byte) uint64 { - if gr, ok := w.inner.(kv.GroupRoutableCoordinator); ok { + if gr, ok := w.Coordinator.(kv.GroupRoutableCoordinator); ok { return gr.EngineGroupIDForKey(key) } return 0 @@ -2020,39 +2020,7 @@ func (w *testCoordinatorWrapper) Dispatch(ctx context.Context, reqs *kv.Operatio } } } - return w.inner.Dispatch(ctx, reqs) -} - -func (w *testCoordinatorWrapper) IsLeader() bool { - return w.inner.IsLeader() -} - -func (w *testCoordinatorWrapper) VerifyLeader(ctx context.Context) error { - return w.inner.VerifyLeader(ctx) -} - -func (w *testCoordinatorWrapper) RaftLeader() string { - return w.inner.RaftLeader() -} - -func (w *testCoordinatorWrapper) IsLeaderForKey(key []byte) bool { - return w.inner.IsLeaderForKey(key) -} - -func (w *testCoordinatorWrapper) VerifyLeaderForKey(ctx context.Context, key []byte) error { - return w.inner.VerifyLeaderForKey(ctx, key) -} - -func (w *testCoordinatorWrapper) RaftLeaderForKey(key []byte) string { - return w.inner.RaftLeaderForKey(key) -} - -func (w *testCoordinatorWrapper) Clock() *kv.HLC { - return w.inner.Clock() -} - -func (w *testCoordinatorWrapper) LinearizableRead(ctx context.Context) (uint64, error) { - return w.inner.LinearizableRead(ctx) + return w.Coordinator.Dispatch(ctx, reqs) } func (w *testCoordinatorWrapper) LeaseRead(ctx context.Context) (uint64, error) { @@ -2062,7 +2030,7 @@ func (w *testCoordinatorWrapper) LeaseRead(ctx context.Context) (uint64, error) if w.failLeaseReads.Load() { return 0, errInjectedLeaseRead } - return kv.LeaseReadThrough(w.inner, ctx) + return kv.LeaseReadThrough(w.Coordinator, ctx) } func (w *testCoordinatorWrapper) LeaseReadForKey(ctx context.Context, key []byte) (uint64, error) { @@ -2073,5 +2041,5 @@ func (w *testCoordinatorWrapper) LeaseReadForKey(ctx context.Context, key []byte if w.failLeaseReads.Load() { return 0, errInjectedLeaseRead } - return kv.LeaseReadForKeyThrough(w.inner, ctx, key) + return kv.LeaseReadForKeyThrough(w.Coordinator, ctx, key) } diff --git a/adapter/redis_hello_test.go b/adapter/redis_hello_test.go index e8337b8ee..7635d6167 100644 --- a/adapter/redis_hello_test.go +++ b/adapter/redis_hello_test.go @@ -1,7 +1,6 @@ package adapter import ( - "context" "io" "math" "net" @@ -9,7 +8,6 @@ import ( "testing" "time" - "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/kv/kvtest" "github.com/bootjp/elastickv/monitoring" "github.com/prometheus/client_golang/prometheus/testutil" @@ -92,42 +90,10 @@ func (c *helloRecordingConn) ReadPipeline() []redcon.Command { return nil } func (c *helloRecordingConn) PeekPipeline() []redcon.Command { return nil } func (c *helloRecordingConn) NetConn() net.Conn { return nil } -// helloTestCoordinator is the minimal kv.Coordinator stub HELLO -// exercises. HELLO only inspects IsLeader for the role field; the rest -// of the surface is satisfied with sensible defaults to avoid -// NPE when handler helpers call through to unrelated methods. -type helloTestCoordinator struct { - isLeader bool - clock *kv.HLC -} - -func (c *helloTestCoordinator) Dispatch(context.Context, *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { - return &kv.CoordinateResponse{}, nil -} -func (c *helloTestCoordinator) IsLeader() bool { return c.isLeader } -func (c *helloTestCoordinator) VerifyLeader(context.Context) error { return nil } -func (c *helloTestCoordinator) RaftLeader() string { return "" } -func (c *helloTestCoordinator) IsLeaderForKey([]byte) bool { return c.isLeader } -func (c *helloTestCoordinator) VerifyLeaderForKey(context.Context, []byte) error { return nil } -func (c *helloTestCoordinator) RaftLeaderForKey([]byte) string { return "" } -func (c *helloTestCoordinator) Clock() *kv.HLC { - if c.clock == nil { - c.clock = kv.NewHLC() - } - return c.clock -} -func (c *helloTestCoordinator) LinearizableRead(_ context.Context) (uint64, error) { return 0, nil } -func (c *helloTestCoordinator) LeaseRead(ctx context.Context) (uint64, error) { - return c.LinearizableRead(ctx) -} -func (c *helloTestCoordinator) LeaseReadForKey(ctx context.Context, _ []byte) (uint64, error) { - return c.LinearizableRead(ctx) -} - func newHelloTestServer(t *testing.T, leader bool) *RedisServer { t.Helper() return &RedisServer{ - coordinator: &helloTestCoordinator{isLeader: leader}, + coordinator: &stubAdapterCoordinator{leaderSet: true, leader: leader}, } } @@ -408,7 +374,7 @@ func TestHello_MetricsRecordsKnownCommand(t *testing.T) { registry := monitoring.NewRegistry("n1", "10.0.0.1:50051") r := &RedisServer{ - coordinator: &helloTestCoordinator{isLeader: true}, + coordinator: &stubAdapterCoordinator{leaderSet: true, leader: true}, requestObserver: registry.RedisObserver(), } diff --git a/adapter/redis_info_test.go b/adapter/redis_info_test.go index 5db46cf9c..5e880d1fd 100644 --- a/adapter/redis_info_test.go +++ b/adapter/redis_info_test.go @@ -1,51 +1,19 @@ package adapter import ( - "context" "strings" "testing" - "github.com/bootjp/elastickv/kv" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tidwall/redcon" ) -type infoTestCoordinator struct { - isLeader bool - raftLeader string - clock *kv.HLC -} - -func (c *infoTestCoordinator) Dispatch(context.Context, *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { - return &kv.CoordinateResponse{}, nil -} -func (c *infoTestCoordinator) IsLeader() bool { return c.isLeader } -func (c *infoTestCoordinator) VerifyLeader(context.Context) error { return nil } -func (c *infoTestCoordinator) RaftLeader() string { return c.raftLeader } -func (c *infoTestCoordinator) IsLeaderForKey([]byte) bool { return c.isLeader } -func (c *infoTestCoordinator) VerifyLeaderForKey(context.Context, []byte) error { return nil } -func (c *infoTestCoordinator) RaftLeaderForKey([]byte) string { return c.raftLeader } -func (c *infoTestCoordinator) Clock() *kv.HLC { - if c.clock == nil { - c.clock = kv.NewHLC() - } - return c.clock -} - -func (c *infoTestCoordinator) LinearizableRead(_ context.Context) (uint64, error) { return 0, nil } -func (c *infoTestCoordinator) LeaseRead(ctx context.Context) (uint64, error) { - return c.LinearizableRead(ctx) -} -func (c *infoTestCoordinator) LeaseReadForKey(ctx context.Context, _ []byte) (uint64, error) { - return c.LinearizableRead(ctx) -} - func TestRedisServer_Info_LeaderRole(t *testing.T) { r := &RedisServer{ redisAddr: "10.0.0.1:6379", leaderRedis: map[string]string{"raft-1": "10.0.0.1:6379"}, - coordinator: &infoTestCoordinator{isLeader: true, raftLeader: "raft-1"}, + coordinator: &stubAdapterCoordinator{leaderSet: true, leader: true, raftLeader: "raft-1"}, } conn := &recordingConn{} @@ -66,7 +34,7 @@ func TestRedisServer_Info_FollowerRole(t *testing.T) { "raft-1": "10.0.0.1:6379", "raft-2": "10.0.0.2:6379", }, - coordinator: &infoTestCoordinator{isLeader: false, raftLeader: "raft-1"}, + coordinator: &stubAdapterCoordinator{leaderSet: true, leader: false, raftLeader: "raft-1"}, } conn := &recordingConn{} @@ -88,7 +56,7 @@ func TestRedisServer_Info_UnknownLeader(t *testing.T) { r := &RedisServer{ redisAddr: "10.0.0.3:6379", leaderRedis: map[string]string{}, - coordinator: &infoTestCoordinator{isLeader: false, raftLeader: ""}, + coordinator: &stubAdapterCoordinator{leaderSet: true, leader: false, raftLeader: ""}, } conn := &recordingConn{} diff --git a/adapter/redis_keys_pattern_test.go b/adapter/redis_keys_pattern_test.go index 5d62ca2c3..c7f82e9a4 100644 --- a/adapter/redis_keys_pattern_test.go +++ b/adapter/redis_keys_pattern_test.go @@ -16,6 +16,7 @@ type stubAdapterCoordinator struct { verifyLeaderErr error leaderSet bool leader bool + raftLeader string verifyCalls atomic.Int32 } @@ -36,7 +37,7 @@ func (s *stubAdapterCoordinator) VerifyLeader(context.Context) error { } func (s *stubAdapterCoordinator) RaftLeader() string { - return "" + return s.raftLeader } func (s *stubAdapterCoordinator) IsLeaderForKey([]byte) bool { @@ -51,7 +52,7 @@ func (s *stubAdapterCoordinator) VerifyLeaderForKey(context.Context, []byte) err } func (s *stubAdapterCoordinator) RaftLeaderForKey([]byte) string { - return "" + return s.raftLeader } func (s *stubAdapterCoordinator) Clock() *kv.HLC { diff --git a/adapter/redis_list_pop_benchmark_test.go b/adapter/redis_list_pop_benchmark_test.go index 038f3a26d..746d96022 100644 --- a/adapter/redis_list_pop_benchmark_test.go +++ b/adapter/redis_list_pop_benchmark_test.go @@ -61,6 +61,7 @@ import ( "time" "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/kv/kvtest" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" ) @@ -74,57 +75,19 @@ var discardLogger = slog.New(slog.NewTextHandler(io.Discard, nil)) // requests through store.ApplyMutations, enabling write-write conflict // detection identical to the production Raft FSM (kv/fsm.go §handleOnePhaseTxnRequest). // Non-txn requests fall through to the underlying coordinator unchanged. +// occAdapterCoordinator validates transactional groups against their read +// keys with the store's OCC check, as the production coordinator does. +// Transactional DelPrefix is rejected: ApplyMutations has no prefix-delete op. type occAdapterCoordinator struct { *localAdapterCoordinator } func newOCCAdapterCoordinator(st store.MVCCStore) *occAdapterCoordinator { - return &occAdapterCoordinator{localAdapterCoordinator: newLocalAdapterCoordinator(st)} -} - -func (c *occAdapterCoordinator) Dispatch(ctx context.Context, req *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { - if req == nil { - return &kv.CoordinateResponse{}, nil - } - commitTS, err := c.commitTSForRequest(req) - if err != nil { - return nil, err - } - if !req.IsTxn { - return c.localAdapterCoordinator.Dispatch(ctx, req) - } - mutations, err := c.collectMutations(ctx, req.Elems, commitTS) - if err != nil { - return nil, err - } - if len(mutations) == 0 { - return &kv.CoordinateResponse{CommitTS: commitTS}, nil - } - if err := c.store.ApplyMutations(ctx, mutations, req.ReadKeys, req.StartTS, commitTS); err != nil { - return nil, err - } - return &kv.CoordinateResponse{CommitTS: commitTS}, nil -} - -func (c *occAdapterCoordinator) collectMutations(_ context.Context, elems []*kv.Elem[kv.OP], _ uint64) ([]*store.KVPairMutation, error) { - mutations := make([]*store.KVPairMutation, 0, len(elems)) - for _, elem := range elems { - if elem == nil { - continue - } - switch elem.Op { - case kv.Put: - mutations = append(mutations, &store.KVPairMutation{Op: store.OpTypePut, Key: elem.Key, Value: elem.Value}) - case kv.Del: - mutations = append(mutations, &store.KVPairMutation{Op: store.OpTypeDelete, Key: elem.Key}) - case kv.DelPrefix: - // ApplyMutations has no prefix-delete op; applying it here before the - // OCC conflict check would break atomicity. List benchmark ops never - // emit DelPrefix, so reject it to make any such misuse explicit. - return nil, errors.New("DelPrefix not supported in transactional OCC simulation") - } - } - return mutations, nil + return &occAdapterCoordinator{localAdapterCoordinator: newLocalAdapterCoordinatorWithOptions(st, kvtest.LocalOptions{ + DelPrefix: kvtest.PrefixApply, + OCC: kvtest.OCCTransactions, + ValidateReadKeys: true, + })} } // ---- Legacy implementations (main-branch simulation) ------------------------ diff --git a/internal/filesystem/fstest/memory.go b/internal/filesystem/fstest/memory.go index 7afbb7690..8896463a5 100644 --- a/internal/filesystem/fstest/memory.go +++ b/internal/filesystem/fstest/memory.go @@ -4,11 +4,10 @@ package fstest import ( "context" - "encoding/binary" - "sync" "github.com/bootjp/elastickv/internal/filesystem" "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/kv/kvtest" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" ) @@ -17,21 +16,27 @@ import ( // satisfies filesystem.Dispatcher and the optional linearizable-read fence so // tests can drive filesystem.Service end to end without replication. type MemoryDispatcher struct { - mu sync.Mutex - store store.MVCCStore + local *kvtest.LocalCoordinator } -const commitTimestampBytes = 8 - // NewMemoryDispatcher wraps st in a dispatcher that commits every group at -// the next commit timestamp. +// max(store+1, requested, StartTS+1), validates OCC read keys on every group, +// patches commit-timestamp value offsets, and rejects prefix deletes. func NewMemoryDispatcher(st store.MVCCStore) *MemoryDispatcher { - return &MemoryDispatcher{store: st} + return &MemoryDispatcher{local: kvtest.NewLocalCoordinator(st, kvtest.LocalOptions{ + DelPrefix: kvtest.PrefixReject, + CommitTimestamp: kvtest.TimestampStoreIncrement, + AdvanceCommitTS: true, + OCC: kvtest.OCCAllGroups, + ValidateReadKeys: true, + PatchCommitTS: true, + })} } // LinearizableRead returns the latest applied commit timestamp. -func (d *MemoryDispatcher) LinearizableRead(context.Context) (uint64, error) { - return d.store.LastCommitTS(), nil +func (d *MemoryDispatcher) LinearizableRead(ctx context.Context) (uint64, error) { + ts, err := d.local.LinearizableRead(ctx) + return ts, errors.WithStack(err) } // Dispatch applies the group's mutations atomically at a fresh commit @@ -40,46 +45,11 @@ func (d *MemoryDispatcher) Dispatch( ctx context.Context, req *kv.OperationGroup[kv.OP], ) (*kv.CoordinateResponse, error) { - d.mu.Lock() - defer d.mu.Unlock() - commitTS := d.store.LastCommitTS() + 1 - if req.CommitTS > commitTS { - commitTS = req.CommitTS - } - if req.IsTxn && commitTS <= req.StartTS { - commitTS = req.StartTS + 1 - } - mutations := make([]*store.KVPairMutation, 0, len(req.Elems)) - for _, elem := range req.Elems { - value := append([]byte(nil), elem.Value...) - if elem.CommitTSValueOffset != 0 { - if len(value) < commitTimestampBytes || elem.CommitTSValueOffset > uint64(len(value)-commitTimestampBytes) { - return nil, errors.WithStack(filesystem.ErrInvalid) - } - binary.BigEndian.PutUint64(value[elem.CommitTSValueOffset:], commitTS) - } - switch elem.Op { - case kv.Put: - mutations = append(mutations, &store.KVPairMutation{ - Op: store.OpTypePut, - Key: append([]byte(nil), elem.Key...), - Value: value, - }) - case kv.Del: - mutations = append(mutations, &store.KVPairMutation{ - Op: store.OpTypeDelete, - Key: append([]byte(nil), elem.Key...), - }) - case kv.DelPrefix: - return nil, errors.WithStack(store.ErrUnknownOp) - default: - return nil, errors.WithStack(store.ErrUnknownOp) - } - } - if err := d.store.ApplyMutations(ctx, mutations, req.ReadKeys, req.StartTS, commitTS); err != nil { + resp, err := d.local.Dispatch(ctx, req) + if err != nil { return nil, errors.Wrap(err, "apply memory dispatcher mutations") } - return &kv.CoordinateResponse{CommitIndex: commitTS}, nil + return resp, nil } // NewMemoryService builds a filesystem.Service over a fresh in-memory MVCC diff --git a/internal/filesystem/service_test.go b/internal/filesystem/service_test.go index 945a420b1..c36aaccce 100644 --- a/internal/filesystem/service_test.go +++ b/internal/filesystem/service_test.go @@ -3,7 +3,6 @@ package filesystem import ( "bytes" "context" - "encoding/binary" "fmt" "math" "sync" @@ -12,6 +11,7 @@ import ( "github.com/bootjp/elastickv/internal/fskeys" "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/kv/kvtest" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" @@ -25,8 +25,28 @@ const ( testChunkSize uint64 = 4 ) +// testCoordinator applies groups to st through kvtest.LocalCoordinator with +// fstest.MemoryDispatcher's policy: commit at max(store+1, requested, +// StartTS+1), OCC on every group, commit-timestamp patching, and no prefix +// deletes. type testCoordinator struct { - st store.MVCCStore + st store.MVCCStore + once sync.Once + local *kvtest.LocalCoordinator +} + +func (c *testCoordinator) coordinator() *kvtest.LocalCoordinator { + c.once.Do(func() { + c.local = kvtest.NewLocalCoordinator(c.st, kvtest.LocalOptions{ + DelPrefix: kvtest.PrefixReject, + CommitTimestamp: kvtest.TimestampStoreIncrement, + AdvanceCommitTS: true, + OCC: kvtest.OCCAllGroups, + ValidateReadKeys: true, + PatchCommitTS: true, + }) + }) + return c.local } type rootRaceCoordinator struct { @@ -57,53 +77,15 @@ func (c *rootRaceCoordinator) Dispatch( return resp, err } -func (c *testCoordinator) LinearizableRead(context.Context) (uint64, error) { - return c.st.LastCommitTS(), nil +func (c *testCoordinator) LinearizableRead(ctx context.Context) (uint64, error) { + return c.coordinator().LinearizableRead(ctx) } func (c *testCoordinator) Dispatch( ctx context.Context, req *kv.OperationGroup[kv.OP], ) (*kv.CoordinateResponse, error) { - commitTS := c.st.LastCommitTS() + 1 - if req.CommitTS > commitTS { - commitTS = req.CommitTS - } - if req.IsTxn && commitTS <= req.StartTS { - commitTS = req.StartTS + 1 - } - mutations := make([]*store.KVPairMutation, 0, len(req.Elems)) - for _, elem := range req.Elems { - mutation, err := testMutation(elem, commitTS) - if err != nil { - return nil, err - } - mutations = append(mutations, mutation) - } - if err := c.st.ApplyMutations(ctx, mutations, req.ReadKeys, req.StartTS, commitTS); err != nil { - return nil, err - } - return &kv.CoordinateResponse{CommitIndex: commitTS}, nil -} - -func testMutation(elem *kv.Elem[kv.OP], commitTS uint64) (*store.KVPairMutation, error) { - value := append([]byte(nil), elem.Value...) - if elem.CommitTSValueOffset != 0 { - if len(value) < 8 || elem.CommitTSValueOffset > uint64(len(value)-8) { - return nil, ErrInvalid - } - binary.BigEndian.PutUint64(value[elem.CommitTSValueOffset:], commitTS) - } - switch elem.Op { - case kv.Put: - return &store.KVPairMutation{Op: store.OpTypePut, Key: append([]byte(nil), elem.Key...), Value: value}, nil - case kv.Del: - return &store.KVPairMutation{Op: store.OpTypeDelete, Key: append([]byte(nil), elem.Key...)}, nil - case kv.DelPrefix: - return nil, store.ErrUnknownOp - default: - return nil, store.ErrUnknownOp - } + return c.coordinator().Dispatch(ctx, req) } type recordingCoordinator struct { diff --git a/kv/kvtest/coordinator.go b/kv/kvtest/coordinator.go index 36830aea3..ad9db6f8b 100644 --- a/kv/kvtest/coordinator.go +++ b/kv/kvtest/coordinator.go @@ -73,6 +73,9 @@ type LocalOptions struct { // AdvanceCommitTS raises caller-supplied timestamps to at least store+1 // and, for transactions, StartTS+1 (the filesystem fake's policy). AdvanceCommitTS bool + // Clock, when set, is the HLC the coordinator issues and observes + // timestamps on, so a wrapper can share it with its own Clock(). + Clock *kv.HLC } func (o LocalOptions) valid() bool { @@ -87,7 +90,11 @@ type LocalCoordinator struct { } func NewLocalCoordinator(st store.MVCCStore, options LocalOptions) *LocalCoordinator { - return &LocalCoordinator{st: st, options: options} + c := &LocalCoordinator{st: st, options: options} + if options.Clock != nil { + c.once.Do(func() { c.clock = options.Clock }) + } + return c } func (c *LocalCoordinator) LinearizableRead(ctx context.Context) (uint64, error) { diff --git a/kv/kvtest/coordinator_test.go b/kv/kvtest/coordinator_test.go index cb659666e..308fc262c 100644 --- a/kv/kvtest/coordinator_test.go +++ b/kv/kvtest/coordinator_test.go @@ -313,3 +313,12 @@ func TestLocalCoordinatorAllowsLastNonexhaustingTimestamp(t *testing.T) { require.Equal(t, response.CommitTS, st.LastCommitTS()) } } + +func TestLocalCoordinatorSharesInjectedClock(t *testing.T) { + clock := kv.NewHLC() + c := kvtest.NewLocalCoordinator(store.NewMVCCStore(), kvtest.LocalOptions{Clock: clock}) + require.Same(t, clock, c.Clock()) + result, err := c.Dispatch(context.Background(), &kv.OperationGroup[kv.OP]{Elems: []*kv.Elem[kv.OP]{put("key")}}) + require.NoError(t, err) + require.GreaterOrEqual(t, clock.Current(), result.CommitTS, "the dispatch timestamp must come from the injected clock") +} diff --git a/kv/lease_group_keys_test.go b/kv/lease_group_keys_test.go index 673c634e3..bed60fda8 100644 --- a/kv/lease_group_keys_test.go +++ b/kv/lease_group_keys_test.go @@ -12,8 +12,9 @@ import ( // fakeGroupRoutableCoordinator is a minimal Coordinator that also // implements GroupRoutableCoordinator, mapping each key to a caller- // supplied group ID. Only the methods LeaseReadGroupKeys exercises are -// implemented; the rest panic to catch unexpected use. +// implemented; the embedded nil Coordinator panics on any other use. type fakeGroupRoutableCoordinator struct { + Coordinator groupByKey map[string]uint64 } @@ -21,22 +22,6 @@ func (f *fakeGroupRoutableCoordinator) EngineGroupIDForKey(key []byte) uint64 { return f.groupByKey[string(key)] } -func (f *fakeGroupRoutableCoordinator) Dispatch(context.Context, *OperationGroup[OP]) (*CoordinateResponse, error) { - panic("unused") -} -func (f *fakeGroupRoutableCoordinator) IsLeader() bool { panic("unused") } -func (f *fakeGroupRoutableCoordinator) VerifyLeader(context.Context) error { panic("unused") } -func (f *fakeGroupRoutableCoordinator) RaftLeader() string { panic("unused") } -func (f *fakeGroupRoutableCoordinator) IsLeaderForKey([]byte) bool { panic("unused") } -func (f *fakeGroupRoutableCoordinator) VerifyLeaderForKey(context.Context, []byte) error { - panic("unused") -} -func (f *fakeGroupRoutableCoordinator) RaftLeaderForKey([]byte) string { panic("unused") } -func (f *fakeGroupRoutableCoordinator) LinearizableRead(context.Context) (uint64, error) { - panic("unused") -} -func (f *fakeGroupRoutableCoordinator) Clock() *HLC { panic("unused") } - var _ Coordinator = (*fakeGroupRoutableCoordinator)(nil) var _ GroupRoutableCoordinator = (*fakeGroupRoutableCoordinator)(nil) From c2cb4844b664bf335144daaaa89d080d5f082b5e Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 16:48:16 +0000 Subject: [PATCH 14/15] test: fold forwarding and duplicate fakes onto shared helpers (cleanup WP-9..12) phaseDGateCoordinator embeds an interface of exactly the methods it forwarded, retryOnceCoordinator fails once and then delegates to localAdapterCoordinator, and distributionLeaderView becomes a raftenginetest.StaticLeaderView. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- adapter/distribution_server_test.go | 36 ++++++--------- adapter/redis_retry_test.go | 69 +++-------------------------- kv/sharded_coordinator_txn_test.go | 54 ++++++---------------- 3 files changed, 33 insertions(+), 126 deletions(-) diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 4f9e5ef28..7a0885457 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -14,6 +14,7 @@ import ( "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/fskeys" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/raftengine/raftenginetest" "github.com/bootjp/elastickv/internal/s3keys" "github.com/bootjp/elastickv/internal/testutil" "github.com/bootjp/elastickv/kv" @@ -318,7 +319,7 @@ func TestDistributionServerPendingLegacyResolutionMode(t *testing.T) { func TestDistributionServerPendingLegacyResolutionRoutedWrappers(t *testing.T) { alloc := &pendingResolutionAllocator{distributionTSOAllocator: &distributionTSOAllocator{leader: true}, pendingFloor: 100} addr := serveDistributionTestServer(t, NewDistributionServer(distribution.NewEngine(), nil, WithDistributionTimestampAllocator(alloc))) - routed, err := kv.NewLeaderRoutedTSOAllocator(&distributionTSOAllocator{}, distributionLeaderView{addr: addr}) + routed, err := kv.NewLeaderRoutedTSOAllocator(&distributionTSOAllocator{}, distributionLeaderView(addr)) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, routed.Close()) }) batch, err := kv.NewBatchAllocator(routed, 2) @@ -352,7 +353,7 @@ func TestDistributionServerValidateTimestamp_LeaderRoutedRPC(t *testing.T) { )) routed, err := kv.NewLeaderRoutedTSOAllocator( &distributionTSOAllocator{leader: false}, - distributionLeaderView{addr: addr}, + distributionLeaderView(addr), ) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, routed.Close()) }) @@ -410,7 +411,7 @@ func TestDistributionServerGetTimestamp_LeaderRoutedRPC(t *testing.T) { )) local := &distributionTSOAllocator{leader: false} - routed, err := kv.NewLeaderRoutedTSOAllocator(local, distributionLeaderView{addr: addr}) + routed, err := kv.NewLeaderRoutedTSOAllocator(local, distributionLeaderView(addr)) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, routed.Close()) }) @@ -424,7 +425,7 @@ func TestDistributionServerGetTimestamp_LeaderRoutedRPC(t *testing.T) { func TestDistributionServerGetTimestamp_LeaderRoutedRejectsLegacyServer(t *testing.T) { addr := serveDistributionTestServer(t, NewDistributionServer(distribution.NewEngine(), nil)) local := &distributionTSOAllocator{leader: false} - routed, err := kv.NewLeaderRoutedTSOAllocator(local, distributionLeaderView{addr: addr}) + routed, err := kv.NewLeaderRoutedTSOAllocator(local, distributionLeaderView(addr)) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, routed.Close()) }) @@ -467,7 +468,7 @@ func TestDistributionServerGetTimestamp_LeaderRoutedActivatesCutover(t *testing. local := &distributionTSOAllocator{leader: false} routed, err := kv.NewLeaderRoutedTSOAllocator( local, - distributionLeaderView{addr: addr}, + distributionLeaderView(addr), kv.WithTSOCutoverActivation(), ) require.NoError(t, err) @@ -508,7 +509,7 @@ func (legacyIgnoringTimestampValidationServer) ValidateTimestamp(context.Context func TestDistributionServerPendingLegacyResolutionRejectsOldModeIgnoringServer(t *testing.T) { addr := serveDistributionTestServer(t, legacyIgnoringTimestampValidationServer{}) - routed, err := kv.NewLeaderRoutedTSOAllocator(&distributionTSOAllocator{}, distributionLeaderView{addr: addr}) + routed, err := kv.NewLeaderRoutedTSOAllocator(&distributionTSOAllocator{}, distributionLeaderView(addr)) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, routed.Close()) }) require.ErrorIs(t, routed.ValidateLegacyResolutionTimestamps(context.Background(), 50, 100), kv.ErrTSOProtocolUnsupported) @@ -1954,22 +1955,13 @@ func (a *distributionTSOAllocator) RunLeaseRenewal(ctx context.Context) { <-ctx.Done() } -type distributionLeaderView struct { - addr string -} - -func (distributionLeaderView) State() raftengine.State { return raftengine.StateFollower } - -func (v distributionLeaderView) Leader() raftengine.LeaderInfo { - return raftengine.LeaderInfo{ID: "leader", Address: v.addr} -} - -func (distributionLeaderView) VerifyLeader(context.Context) error { - return raftengine.ErrNotLeader -} - -func (distributionLeaderView) LinearizableRead(context.Context) (uint64, error) { - return 0, raftengine.ErrNotLeader +// distributionLeaderView reports a follower whose leader is at addr. +func distributionLeaderView(addr string) *raftenginetest.StaticLeaderView { + return &raftenginetest.StaticLeaderView{ + StateValue: raftengine.StateFollower, + LeaderValue: raftengine.LeaderInfo{ID: "leader", Address: addr}, + ReadErr: raftengine.ErrNotLeader, + } } type recordingDistributionFilesystemObserver struct { diff --git a/adapter/redis_retry_test.go b/adapter/redis_retry_test.go index 2538c7499..39c85c3b2 100644 --- a/adapter/redis_retry_test.go +++ b/adapter/redis_retry_test.go @@ -18,18 +18,18 @@ import ( "google.golang.org/grpc/status" ) +// retryOnceCoordinator fails its first dispatch with firstErr and applies +// every later one through localAdapterCoordinator. type retryOnceCoordinator struct { - store store.MVCCStore - clock *kv.HLC + *localAdapterCoordinator dispatches int firstErr error } func newRetryOnceCoordinator(st store.MVCCStore) *retryOnceCoordinator { return &retryOnceCoordinator{ - store: st, - clock: kv.NewHLC(), - firstErr: store.ErrWriteConflict, + localAdapterCoordinator: newLocalAdapterCoordinator(st), + firstErr: store.ErrWriteConflict, } } @@ -38,64 +38,7 @@ func (c *retryOnceCoordinator) Dispatch(ctx context.Context, reqs *kv.OperationG if c.dispatches == 1 { return nil, c.firstErr } - - ts := c.clock.Next() - for _, elem := range reqs.Elems { - var err error - switch elem.Op { - case kv.Put: - err = c.store.PutAt(ctx, elem.Key, elem.Value, ts, 0) - case kv.Del: - err = c.store.DeleteAt(ctx, elem.Key, ts) - case kv.DelPrefix: - err = c.store.DeletePrefixAt(ctx, elem.Key, nil, ts) - } - if err != nil { - return nil, err - } - } - - return &kv.CoordinateResponse{}, nil -} - -func (c *retryOnceCoordinator) IsLeader() bool { - return true -} - -func (c *retryOnceCoordinator) VerifyLeader(context.Context) error { - return nil -} - -func (c *retryOnceCoordinator) RaftLeader() string { - return "" -} - -func (c *retryOnceCoordinator) IsLeaderForKey([]byte) bool { - return true -} - -func (c *retryOnceCoordinator) VerifyLeaderForKey(context.Context, []byte) error { - return nil -} - -func (c *retryOnceCoordinator) RaftLeaderForKey([]byte) string { - return "" -} - -func (c *retryOnceCoordinator) Clock() *kv.HLC { - return c.clock -} - -func (c *retryOnceCoordinator) LinearizableRead(_ context.Context) (uint64, error) { - return 0, nil -} - -func (c *retryOnceCoordinator) LeaseRead(ctx context.Context) (uint64, error) { - return c.LinearizableRead(ctx) -} - -func (c *retryOnceCoordinator) LeaseReadForKey(ctx context.Context, _ []byte) (uint64, error) { - return c.LinearizableRead(ctx) + return c.localAdapterCoordinator.Dispatch(ctx, reqs) } type recordingConn struct { diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 81ef2b691..c05b82477 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -826,7 +826,7 @@ func TestDispatchWithReadTimestampRevokesVoucherWhenOuterGateRejects(t *testing. prePhaseDErr := errors.Join(ErrTSOTimestampInvalid, ErrTSOTimestampPrePhaseD) coord, _, _, _ := newPhaseDCrossShardCoordinator(t, prePhaseDErr) gateErr := errors.New("startup gate rejected dispatch") - gated := phaseDGateCoordinator{inner: coord, err: gateErr} + gated := phaseDGateCoordinator{phaseDGateInner: coord, err: gateErr} readTimestamp, err := BeginReadTimestampThrough(context.Background(), gated, 10, "vouch gated applied watermark") require.NoError(t, err) @@ -958,9 +958,19 @@ func newPhaseDCrossShardCoordinator( return coord, g1Txn, g2Txn, alloc } +// phaseDGateInner is exactly the surface phaseDGateCoordinator forwarded +// by hand; embedding it keeps the gate from gaining the sharded +// coordinator's other optional capabilities. +type phaseDGateInner interface { + Coordinator + TimestampAllocator() TimestampAllocator + VouchAppliedReadTimestamp(timestamp uint64, ref AppliedReadTimestampVoucherRef) error + RevokeAppliedReadTimestamp(timestamp uint64, ref AppliedReadTimestampVoucherRef) +} + type phaseDGateCoordinator struct { - inner *ShardedCoordinator - err error + phaseDGateInner + err error } type overlappingReadVoucherCoordinator struct { @@ -1069,44 +1079,6 @@ func (c phaseDGateCoordinator) Dispatch(context.Context, *OperationGroup[OP]) (* return nil, c.err } -func (c phaseDGateCoordinator) IsLeader() bool { return c.inner.IsLeader() } - -func (c phaseDGateCoordinator) VerifyLeader(ctx context.Context) error { - return c.inner.VerifyLeader(ctx) -} - -func (c phaseDGateCoordinator) LinearizableRead(ctx context.Context) (uint64, error) { - return c.inner.LinearizableRead(ctx) -} - -func (c phaseDGateCoordinator) RaftLeader() string { return c.inner.RaftLeader() } - -func (c phaseDGateCoordinator) IsLeaderForKey(key []byte) bool { - return c.inner.IsLeaderForKey(key) -} - -func (c phaseDGateCoordinator) VerifyLeaderForKey(ctx context.Context, key []byte) error { - return c.inner.VerifyLeaderForKey(ctx, key) -} - -func (c phaseDGateCoordinator) RaftLeaderForKey(key []byte) string { - return c.inner.RaftLeaderForKey(key) -} - -func (c phaseDGateCoordinator) Clock() *HLC { return c.inner.Clock() } - -func (c phaseDGateCoordinator) TimestampAllocator() TimestampAllocator { - return c.inner.TimestampAllocator() -} - -func (c phaseDGateCoordinator) VouchAppliedReadTimestamp(timestamp uint64, ref AppliedReadTimestampVoucherRef) error { - return c.inner.VouchAppliedReadTimestamp(timestamp, ref) -} - -func (c phaseDGateCoordinator) RevokeAppliedReadTimestamp(timestamp uint64, ref AppliedReadTimestampVoucherRef) { - c.inner.RevokeAppliedReadTimestamp(timestamp, ref) -} - func TestShardedCoordinatorDispatchTxn_SingleShardUsesOnePhase(t *testing.T) { t.Parallel() From 2134e7b295e59e26e7c92a1f3cce4b2cec52d476 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 28 Oct 2026 16:48:16 +0000 Subject: [PATCH 15/15] docs: record WP-9..12 migration results in the test suite cleanup design Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fm5mogKjCk6edosPUsnMWZ Signed-off-by: bootjp --- .../2026_10_16_partial_test_suite_cleanup.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/design/2026_10_16_partial_test_suite_cleanup.md b/docs/design/2026_10_16_partial_test_suite_cleanup.md index 0ab3b6aea..a555da0c2 100644 --- a/docs/design/2026_10_16_partial_test_suite_cleanup.md +++ b/docs/design/2026_10_16_partial_test_suite_cleanup.md @@ -100,7 +100,20 @@ WP-1 to WP-8 can run in parallel. WP-9 to WP-12 start once their dependencies me | WP-6 | Landed (same PR as WP-3); see deviation 2. | | WP-7 | Landed (same PR as WP-3). | | WP-8 | Landed (#1288). | -| WP-9 to WP-12 | Open. Their dependencies have all landed. | +| WP-9 to WP-12 | Landed in the same PR as WP-3, with the exceptions listed under "WP-9 to WP-12 results". | + +WP-9 to WP-12 results (all four directory sets were migrated together, one commit per pattern): + +| Pattern | Before | After | Left as is, and why | +|---|---|---|---| +| Receive-or-fail `select` blocks | 276 candidate blocks | 154 replaced by `testutil.Recv` / `NoRecv` / `Closed`; 122 remain | A bare receive becomes `Closed` only when the file closes that channel and `Recv` only when it sends on it. Blocks were kept when that was ambiguous, the case body uses `break`/`goto`, the failure is non-fatal (`t.Error`), or hoisting the case variable could shadow another name. | +| Legacy list delta key copies | 5 hand-rolled encoders | 0; all use `store.LegacyListMetaDeltaKey` (adapter seq 0, kv seq 1 preserved) | The `internal/backup` mirrors stay independent on purpose. | +| Encryption fixtures | `store.CounterNonceFactory` exported from production `store`; 10 random-DEK keystore+cipher blocks; 2 byte-identical sidecar writers plus 4 temp-dir writers | `store/encryption_test_helpers.go` deleted (callers use `encryption.NewDeterministicNonceFactory`); 37 `encryptiontest` call sites | Fixed-byte DEKs, sidecar rewrites of an existing path, and `wiringEncryptionObserver` ≈ `recordingEncryptionObserver` (different packages; sharing needs a new package). | +| Loopback gRPC servers | about 27 | 14 kv servers on `grpctest.Serve` | Servers whose tests assert the `Serve` error, need the listener address before construction, or stop the server mid-test. | +| Fake Raft engines | 13 full fakes, about 10 `LeaderView`, about 12 `Proposer` | `stubLeaderEngine`/`noopEngine`, `stubFollowerEngine`, `membershipConfigEngine` → `raftenginetest.StubEngine`; three `LeaderView` fakes → `StaticLeaderView`; etcd `testStateMachine` aliases `raftenginetest.TestStateMachine` | Rule 4: fakes whose behaviour is the subject (toggling leader, blocking reads, lease, call recording, panicking `VerifyLeader`, error-injecting proposers). | +| Fake coordinators | 12 full fakes, 6 in-memory apply coordinators | HELLO/INFO fakes folded into `stubAdapterCoordinator`; `testCoordinatorWrapper`, `phaseDGateCoordinator`, `fakeGroupRoutableCoordinator` embed instead of forwarding; `localAdapterCoordinator`, `occAdapterCoordinator`, `retryOnceCoordinator`, `fstest.MemoryDispatcher` and the filesystem `testCoordinator` dispatch through `kvtest.LocalCoordinator` (new `Clock` option shares the wrapper's HLC) | Rule 4 again for fakes with hooks, counters or scripted failures (`distributionCoordinatorStub`, `stubLeaderCoordinator`, `stubStartupCoordinator`, the TSO fakes). `phaseDGateCoordinator` embeds an interface of exactly its old method set, not `*ShardedCoordinator`, so it gains no optional capabilities. | + +Behaviour notes for the `LocalCoordinator` move: dispatches are serialized, elements are copied, `ctx` is checked before applying, the response carries `CommitTS`/`CommitIndex`, unknown ops return `ErrUnknownOp`, and an invalid commit-timestamp patch surfaces kv's validation error instead of `filesystem.ErrInvalid`. Deviations from the plan, found while implementing: