From 629554c885d9990d155d85ad1eec370f73e38953 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 22 Oct 2026 23:10:19 +0900 Subject: [PATCH 1/6] kv: verify durable primary evidence for legacy commits --- adapter/internal.go | 38 ++++- adapter/internal_commit_evidence_test.go | 101 ++++++++++++ ..._partial_prephase_d_resolution_evidence.md | 49 ++++-- kv/primary_commit_evidence.go | 104 +++++++++++++ kv/primary_commit_evidence_test.go | 145 ++++++++++++++++++ kv/tso.go | 6 + kv/tso_prephase_d.go | 9 +- main.go | 1 + 8 files changed, 435 insertions(+), 18 deletions(-) create mode 100644 adapter/internal_commit_evidence_test.go create mode 100644 kv/primary_commit_evidence.go create mode 100644 kv/primary_commit_evidence_test.go diff --git a/adapter/internal.go b/adapter/internal.go index 75b7c848c..3d51eefd3 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -55,6 +55,12 @@ func WithInternalStore(st store.MVCCStore) InternalOption { } } +func WithInternalPrimaryCommitEvidence(reader kv.PrimaryCommitEvidenceReader) InternalOption { + return func(i *Internal) { + i.primaryCommitEvidence = reader + } +} + func WithInternalMigrationProposer(proposer raftengine.Proposer) InternalOption { return func(i *Internal) { i.migrationProposer = proposer @@ -127,6 +133,7 @@ type Internal struct { lastCommitTimestamp func() uint64 relay *RedisPubSubRelay store store.MVCCStore + primaryCommitEvidence kv.PrimaryCommitEvidenceReader migrationProposer raftengine.Proposer migrationImportGate func(context.Context) error migrationPromoteGate func(context.Context) error @@ -1111,7 +1118,9 @@ func (i *Internal) fillForwardedTxnCommitTS(ctx context.Context, reqs []*pb.Requ } } else if err := kv.ValidateForwardedTxnCommitTimestamp( ctx, i.tsAllocator, startTS, commitTS, resolution, - "fillForwardedTxnCommitTS: forwarded commit ts"); err != nil { + "fillForwardedTxnCommitTS: forwarded commit ts", func(ctx context.Context) error { + return i.verifyLegacyCommitEvidence(ctx, reqs, startTS, commitTS) + }); err != nil { // A commit timestamp that arrived already set in the transaction meta // is the timestamp every mutation in this batch is persisted under, and // it never passed through the coordinator's own validation. A legacy @@ -1130,6 +1139,33 @@ func (i *Internal) fillForwardedTxnCommitTS(ctx context.Context, reqs []*pb.Requ return commitTS, nil } +func (i *Internal) verifyLegacyCommitEvidence(ctx context.Context, reqs []*pb.Request, startTS, commitTS uint64) error { + if i.primaryCommitEvidence == nil { + return nil + } + for _, req := range reqs { + if req == nil || req.Phase != pb.Phase_COMMIT { + continue + } + mutation, ok := forwardedTxnMetaMutation(req, []byte(kv.TxnMetaPrefix)) + if !ok { + continue + } + meta, err := kv.DecodeTxnMeta(mutation.Value) + if err != nil { + return errors.WithStack(err) + } + evidence, err := i.primaryCommitEvidence.PrimaryCommitEvidence(ctx, meta.PrimaryKey, startTS) + if err != nil { + return errors.Wrap(err, "validate legacy primary commit evidence") + } + if err := kv.ClassifyPrePhaseDCommit(evidence, commitTS, false); err != nil { + return errors.Wrap(err, "validate legacy primary commit decision") + } + } + return nil +} + func stampRequestMutationCommitTS(reqs []*pb.Request, commitTS uint64) error { for _, r := range reqs { if r == nil { diff --git a/adapter/internal_commit_evidence_test.go b/adapter/internal_commit_evidence_test.go new file mode 100644 index 000000000..b6c07d72a --- /dev/null +++ b/adapter/internal_commit_evidence_test.go @@ -0,0 +1,101 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +type primaryCommitEvidenceFunc func(context.Context, []byte, uint64) (kv.PrimaryCommitEvidence, error) + +func (f primaryCommitEvidenceFunc) PrimaryCommitEvidence(ctx context.Context, key []byte, startTS uint64) (kv.PrimaryCommitEvidence, error) { + return f(ctx, key, startTS) +} + +func TestFillForwardedTxnCommitTSEnforcesPrimaryEvidence(t *testing.T) { + t.Parallel() + readFailure := errors.New("primary read failed") + for _, test := range []struct { + name string + phase pb.Phase + commitTS uint64 + evidence kv.PrimaryCommitEvidence + readErr error + wantErr error + wantReads int + }{ + {name: "matching durable decision", phase: pb.Phase_COMMIT, commitTS: 60, evidence: kv.PrimaryCommitEvidence{Readable: true, Found: true, CommitTS: 60}, wantReads: 1}, + {name: "missing local record", phase: pb.Phase_COMMIT, commitTS: 60, evidence: kv.PrimaryCommitEvidence{Readable: true}, wantErr: kv.ErrPrePhaseDCommitUnproven, wantReads: 1}, + {name: "mismatched decision", phase: pb.Phase_COMMIT, commitTS: 60, evidence: kv.PrimaryCommitEvidence{Readable: true, Found: true, CommitTS: 61}, wantErr: kv.ErrPrePhaseDCommitUnproven, wantReads: 1}, + {name: "remote primary", phase: pb.Phase_COMMIT, commitTS: 60, wantReads: 1}, + {name: "abort needs no commit record", phase: pb.Phase_ABORT, commitTS: 60}, + {name: "post phase D needs no legacy evidence", phase: pb.Phase_COMMIT, commitTS: 150}, + {name: "read error refuses commit", phase: pb.Phase_COMMIT, commitTS: 60, readErr: readFailure, wantErr: readFailure, wantReads: 1}, + } { + t.Run(test.name, func(t *testing.T) { + reads := 0 + reader := primaryCommitEvidenceFunc(func(_ context.Context, key []byte, startTS uint64) (kv.PrimaryCommitEvidence, error) { + reads++ + require.Equal(t, []byte("primary"), key) + require.Equal(t, uint64(50), startTS) + return test.evidence, test.readErr + }) + i := NewInternalWithEngine(nil, nil, nil, nil, + WithInternalTimestampAllocator(&phaseDForwardAllocator{floor: 100}), + WithInternalPrimaryCommitEvidence(reader)) + reqs := []*pb.Request{{IsTxn: true, Phase: test.phase, Mutations: []*pb.Mutation{{ + Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("primary"), CommitTS: test.commitTS}), + }}}} + got, err := i.fillForwardedTxnCommitTS(context.Background(), reqs, 50) + if test.wantErr != nil { + require.ErrorIs(t, err, test.wantErr) + } else { + require.NoError(t, err) + require.Equal(t, test.commitTS, got) + } + require.Equal(t, test.wantReads, reads) + }) + } +} + +func TestInternalForwardRejectsUnprovenLegacyCommitBeforePropose(t *testing.T) { + t.Parallel() + txn := &forwardObserverTxn{} + i := NewInternalWithEngine(txn, forwardObserverLeader{}, nil, nil, + WithInternalTimestampAllocator(&phaseDForwardAllocator{floor: 100}), + WithInternalPrimaryCommitEvidence(primaryCommitEvidenceFunc(func(context.Context, []byte, uint64) (kv.PrimaryCommitEvidence, error) { + return kv.PrimaryCommitEvidence{Readable: true}, nil + }))) + response, err := i.Forward(context.Background(), &pb.ForwardRequest{IsTxn: true, Requests: []*pb.Request{{ + Ts: 50, IsTxn: true, Phase: pb.Phase_COMMIT, Mutations: []*pb.Mutation{{ + Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("primary"), CommitTS: 60}), + }}, + }}}) + require.ErrorIs(t, err, kv.ErrPrePhaseDCommitUnproven) + require.False(t, response.Success) + require.Empty(t, txn.reqs) +} + +func TestFillForwardedTxnCommitTSChecksEveryPrimary(t *testing.T) { + t.Parallel() + var readKeys []string + i := NewInternalWithEngine(nil, nil, nil, nil, + WithInternalTimestampAllocator(&phaseDForwardAllocator{floor: 100}), + WithInternalPrimaryCommitEvidence(primaryCommitEvidenceFunc(func(_ context.Context, key []byte, _ uint64) (kv.PrimaryCommitEvidence, error) { + readKeys = append(readKeys, string(key)) + return kv.PrimaryCommitEvidence{Readable: true, Found: string(key) == "first", CommitTS: 60}, nil + }))) + var reqs []*pb.Request + for _, key := range []string{"first", "second"} { + reqs = append(reqs, &pb.Request{IsTxn: true, Phase: pb.Phase_COMMIT, Mutations: []*pb.Mutation{{ + Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte(key), CommitTS: 60}), + }}}) + } + _, err := i.fillForwardedTxnCommitTS(context.Background(), reqs, 50) + require.ErrorIs(t, err, kv.ErrPrePhaseDCommitUnproven) + require.Equal(t, []string{"first", "second"}, readKeys) +} diff --git a/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md b/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md index 1dd4750c3..9a186d4d3 100644 --- a/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md +++ b/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md @@ -41,12 +41,11 @@ durable state to confirm a primary ever recorded that commit timestamp. **Right:** the phase flag proves nothing. The exemption's entire evidence is attacker-supplied. -**Overstated:** the summary reads as though this PR opens a hole. It does not. -`kv/tso.go` does not exist on `origin/main`; `Internal.Forward` on main accepts -*any* forwarded start and commit timestamp with no validation at all. This PR -closes the general case and leaves one narrow window open. The window is a -regression only against the PR's own stated invariant, not against shipped -behaviour. +**Historical context:** the original finding was against the Phase-D validation +proposal, before `kv/tso.go` shipped. At that time the main branch accepted +forwarded timestamps without these checks. The general validation and bounded +creation window have since shipped; the remaining local-primary evidence check +is described below. Also worth stating precisely: the invariant `ValidateDurablePersistenceTimestamp` exists to protect — "never persist at a timestamp group 0 has not issued yet" — @@ -155,14 +154,31 @@ The production allocator wrappers expose `PhaseDFloor()`; a Phase-D allocator without a usable floor refuses new pre-D PREPAREs. COMMIT and ABORT bypass the creation window so they can resolve existing intents indefinitely. -§4.1's decision is implemented and tested as `ClassifyPrePhaseDCommit`, but is -**not yet wired**. It needs a signal the commit-validation path does not -currently have: whether this node can definitively read the primary's record, -as distinct from simply not finding one. `ShardStore.primaryTxnRecordedStatus` -already draws that distinction with its `done` return, and exposing it to the -forwarded-commit path is the remaining work. Wiring it without that signal -would make a remote primary indistinguishable from a missing record and refuse -exactly the cross-shard resolutions §4.1 is careful to preserve. +§4.1 is wired through `WithInternalPrimaryCommitEvidence` in the production +Internal service. `ValidateForwardedTxnCommitTimestamp` invokes the evidence +check only when both timestamps enter its legacy resolution carve-out. +`Internal.Forward` refuses a COMMIT before proposing it if the locally readable +primary record is missing or disagrees. ABORT and valid post-D commits do not +perform the additional evidence read. + +`ShardStore.PrimaryCommitEvidence` provides the separate readability signal. +It uses the primary's physical owner, including partition-map routing, requires +a local active/non-staged store and verified leadership, and reads directly +without proxying to another primary. It discards the verdict if catalog version, +partition owner, or leadership changes while reading. Unavailable, remote, +follower, or migration-staged owners return an unknown verdict rather than a +missing record; actual storage or decoding failures return errors. + +The older `primaryTxnRecordedStatus` helper is not this signal: its `done` +return means a transaction decision was found, not that an absent record was +read authoritatively. Reusing it would conflate missing and unavailable records. + +Regression coverage includes missing/matching/mismatched decisions, unknown +and follower owners, partition routing and ownership changes, ABORT/post-D +bypass, every primary in a batch, proposal rejection, corrupt records, and +commit-record persistence across Pebble reopen. Full distributed cutover and +leader-change acceptance have not been demonstrated by these focused tests; +the document remains Partial until that acceptance evidence is available. ## 5. Open questions for review @@ -177,5 +193,6 @@ exactly the cross-shard resolutions §4.1 is careful to preserve. 4. Should the `ABORT` path get its own rule, given its timestamp is synthesised rather than recorded? -Settled in §4a. The carve-out is now bounded by §4.2; §4.1 remains to be wired -behind a local-primary readability signal. +Settled in §4a. Both mechanisms are wired. Remaining acceptance must exercise +the real forwarded path across Phase-D activation and leader changes, including +a remote-primary resolution after the creation window has closed. diff --git a/kv/primary_commit_evidence.go b/kv/primary_commit_evidence.go new file mode 100644 index 000000000..a7e222a30 --- /dev/null +++ b/kv/primary_commit_evidence.go @@ -0,0 +1,104 @@ +package kv + +import ( + "context" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +// PrimaryCommitEvidence reads only a locally authoritative primary record. +// It never proxies to another group or interprets an unavailable route as a +// missing record, so legacy cross-group resolution remains admissible. +func (s *ShardStore) PrimaryCommitEvidence(ctx context.Context, primaryKey []byte, startTS uint64) (PrimaryCommitEvidence, error) { + if len(primaryKey) == 0 || startTS == 0 { + return PrimaryCommitEvidence{}, errors.WithStack(ErrInvalidRequest) + } + route, group, version, ok := s.primaryEvidenceRoute(primaryKey) + if !ok { + return PrimaryCommitEvidence{}, nil + } + readable, err := primaryEvidenceLeaderReady(ctx, group) + if err != nil || !readable { + return PrimaryCommitEvidence{}, err + } + evidence, err := s.readPrimaryCommitEvidence(ctx, group, route, primaryKey, startTS) + if err != nil { + return PrimaryCommitEvidence{}, errors.WithStack(err) + } + if !s.primaryEvidenceOwnershipUnchanged(primaryKey, group, route.GroupID, version) { + return PrimaryCommitEvidence{}, nil + } + return evidence, nil +} + +func (s *ShardStore) primaryEvidenceOwnershipUnchanged(primaryKey []byte, group *ShardGroup, groupID, version uint64) bool { + currentRoute, currentGroup, currentVersion, currentOK := s.primaryEvidenceRoute(primaryKey) + if !currentOK || currentGroup != group || currentVersion != version || currentRoute.GroupID != groupID { + return false + } + engine := engineForGroup(group) + return engine == nil || isLeaderEngine(engine) +} + +func (s *ShardStore) readPrimaryCommitEvidence(ctx context.Context, group *ShardGroup, route distribution.Route, key []byte, startTS uint64) (PrimaryCommitEvidence, error) { + value, err := s.localGetAt(ctx, group, route, txnCommitKey(key, startTS), ^uint64(0)) + if err != nil && !errors.Is(err, store.ErrKeyNotFound) { + return PrimaryCommitEvidence{}, errors.WithStack(err) + } + if errors.Is(err, store.ErrKeyNotFound) { + return PrimaryCommitEvidence{Readable: true}, nil + } + commitTS, err := decodeTxnCommitRecord(value) + if err != nil { + return PrimaryCommitEvidence{}, errors.Wrap(err, "read primary commit evidence") + } + return PrimaryCommitEvidence{Readable: true, Found: true, CommitTS: commitTS}, nil +} + +func (s *ShardStore) primaryEvidenceRoute(key []byte) (distribution.Route, *ShardGroup, uint64, bool) { + route, group, version, ok := s.routeAndGroupForKeyWithVersion(key) + if s != nil && s.partitionResolver != nil { + if groupID, resolved := s.partitionResolver.ResolveGroup(key); resolved { + // Preserve catalog migration fences even when the partition map + // selects a different physical group from the byte-range owner. + route, version, ok = s.partitionPrimaryEvidenceRoute(key) + if !ok { + return distribution.Route{}, nil, version, false + } + route.GroupID = groupID + group, ok = s.groups[groupID] + } else if s.partitionResolver.RecognisesPartitionedKey(key) { + return distribution.Route{}, nil, version, false + } + } + readable := ok && group != nil && group.Store != nil && route.State == distribution.RouteStateActive && !routeHasStagedVisibility(route) + return route, group, version, readable +} + +func (s *ShardStore) partitionPrimaryEvidenceRoute(key []byte) (distribution.Route, uint64, bool) { + if s.engine == nil { + return distribution.Route{}, 0, false + } + return s.engine.GetRouteWithVersion(routeKey(key)) +} + +func primaryEvidenceLeaderReady(ctx context.Context, group *ShardGroup) (bool, error) { + engine := engineForGroup(group) + if engine == nil { + return true, nil + } + if !isLeaderEngine(engine) { + return false, nil + } + if err := verifyLeaderEngineCtx(ctx, engine); err != nil { + if ctx.Err() != nil { + return false, errors.WithStack(ctx.Err()) + } + return false, nil + } + return true, nil +} + +var _ PrimaryCommitEvidenceReader = (*ShardStore)(nil) diff --git a/kv/primary_commit_evidence_test.go b/kv/primary_commit_evidence_test.go new file mode 100644 index 000000000..cb39a1121 --- /dev/null +++ b/kv/primary_commit_evidence_test.go @@ -0,0 +1,145 @@ +package kv + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestPrimaryCommitEvidenceDistinguishesMissingFromUnavailable(t *testing.T) { + t.Parallel() + ctx := context.Background() + for _, test := range []struct { + name string + prepare func(*ShardGroup) + want PrimaryCommitEvidence + wantError bool + }{ + {name: "missing local record", want: PrimaryCommitEvidence{Readable: true}}, + {name: "committed", prepare: func(g *ShardGroup) { + require.NoError(t, g.Store.PutAt(ctx, txnCommitKey([]byte("key"), 50), encodeTxnCommitRecord(60), 60, 0)) + }, want: PrimaryCommitEvidence{Readable: true, Found: true, CommitTS: 60}}, + {name: "rolled back", prepare: func(g *ShardGroup) { + require.NoError(t, g.Store.PutAt(ctx, txnRollbackKey([]byte("key"), 50), encodeTxnRollbackRecord(), 60, 0)) + }, want: PrimaryCommitEvidence{Readable: true}}, + {name: "remote primary", prepare: func(g *ShardGroup) { g.Store = nil }}, + {name: "follower primary", prepare: func(g *ShardGroup) { g.Engine = &followerProxyEngine{leader: "127.0.0.1:1"} }}, + {name: "corrupt record", prepare: func(g *ShardGroup) { + require.NoError(t, g.Store.PutAt(ctx, txnCommitKey([]byte("key"), 50), []byte("bad"), 60, 0)) + }, wantError: true}, + } { + t.Run(test.name, func(t *testing.T) { + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, Routes: []distribution.RouteDescriptor{{RouteID: 1, GroupID: 1, State: distribution.RouteStateActive}}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + if test.prepare != nil { + test.prepare(group) + } + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + evidence, err := shards.PrimaryCommitEvidence(ctx, []byte("key"), 50) + if test.wantError { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, test.want, evidence) + }) + } +} + +func TestPrimaryCommitEvidenceDoesNotReadStagedOwnerAsAbsent(t *testing.T) { + t.Parallel() + shards, _ := newStagedVisibilityShardStore(t) + evidence, err := shards.PrimaryCommitEvidence(context.Background(), []byte("key"), 50) + require.NoError(t, err) + require.False(t, evidence.Readable) +} + +func TestPrimaryCommitEvidenceSurvivesStoreReopen(t *testing.T) { + t.Parallel() + ctx := context.Background() + dir := t.TempDir() + st, err := store.NewPebbleStore(dir) + require.NoError(t, err) + require.NoError(t, st.PutAt(ctx, txnCommitKey([]byte("key"), 50), encodeTxnCommitRecord(60), 60, 0)) + require.NoError(t, st.Close()) + reopened, err := store.NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, Routes: []distribution.RouteDescriptor{{RouteID: 1, GroupID: 1, State: distribution.RouteStateActive}}, + })) + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: {Store: reopened}}) + evidence, err := shards.PrimaryCommitEvidence(ctx, []byte("key"), 50) + require.NoError(t, err) + require.Equal(t, PrimaryCommitEvidence{Readable: true, Found: true, CommitTS: 60}, evidence) +} + +func TestPrimaryCommitEvidenceUsesPartitionOwner(t *testing.T) { + t.Parallel() + ctx := context.Background() + key := []byte("partition:primary") + for _, resolved := range []bool{false, true} { + t.Run(map[bool]string{false: "unresolved", true: "resolved"}[resolved], func(t *testing.T) { + engine := distribution.NewEngine() + engine.UpdateRoute(nil, nil, 1) + owner := store.NewMVCCStore() + require.NoError(t, owner.PutAt(ctx, txnCommitKey(key, 50), encodeTxnCommitRecord(60), 60, 0)) + resolver := &fakePartitionResolver{routes: map[string]uint64{}, recognisedPrefix: []byte("partition:")} + if resolved { + resolver.routes[string(key)] = 2 + } + shards := NewShardStore(engine, map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, 2: {Store: owner}, + }).WithPartitionResolver(resolver) + evidence, err := shards.PrimaryCommitEvidence(ctx, key, 50) + require.NoError(t, err) + if resolved { + require.Equal(t, PrimaryCommitEvidence{Readable: true, Found: true, CommitTS: 60}, evidence) + } else { + require.False(t, evidence.Readable) + } + }) + } +} + +type primaryEvidenceReadHookStore struct { + store.MVCCStore + beforeRead func() +} + +func (s *primaryEvidenceReadHookStore) GetAt(ctx context.Context, key []byte, ts uint64) ([]byte, error) { + s.beforeRead() + return s.MVCCStore.GetAt(ctx, key, ts) +} + +func TestPrimaryCommitEvidenceDiscardsChangedOwnership(t *testing.T) { + t.Parallel() + key := []byte("partition:primary") + for _, change := range []string{"catalog", "partition map"} { + t.Run(change, func(t *testing.T) { + engine := distribution.NewEngine() + engine.UpdateRoute(nil, nil, 1) + resolver := &fakePartitionResolver{routes: map[string]uint64{string(key): 1}, recognisedPrefix: []byte("partition:")} + st := &primaryEvidenceReadHookStore{MVCCStore: store.NewMVCCStore(), beforeRead: func() { + if change == "catalog" { + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, Routes: []distribution.RouteDescriptor{{RouteID: 2, GroupID: 2, State: distribution.RouteStateActive}}, + })) + } else { + resolver.routes[string(key)] = 2 + } + }} + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: {Store: st}, 2: {Store: store.NewMVCCStore()}}).WithPartitionResolver(resolver) + evidence, err := shards.PrimaryCommitEvidence(context.Background(), key, 50) + require.NoError(t, err) + require.False(t, evidence.Readable) + }) + } +} diff --git a/kv/tso.go b/kv/tso.go index 9bab0d22a..2607f7a5a 100644 --- a/kv/tso.go +++ b/kv/tso.go @@ -353,6 +353,8 @@ func ValidateDurablePersistenceTimestamp(ctx context.Context, alloc TimestampAll // ValidateForwardedTxnCommitTimestamp validates the commit timestamp of a // forwarded transaction whose caller already stamped it. +// verifyLegacyResolution, when provided, checks durable primary evidence only +// for the pre-Phase-D resolution carve-out; it is not called for valid new timestamps. // // It is ValidateDurablePersistenceTimestamp with one carve-out: a commit // timestamp that predates Phase D is accepted when the transaction's own start @@ -379,6 +381,7 @@ func ValidateForwardedTxnCommitTimestamp( commitTS uint64, resolution bool, label string, + verifyLegacyResolution func(context.Context) error, ) error { err := ValidateDurablePersistenceTimestamp(ctx, alloc, commitTS, label) if err == nil || !errors.Is(err, ErrTSOTimestampPrePhaseD) { @@ -393,6 +396,9 @@ func ValidateForwardedTxnCommitTimestamp( } startErr := ValidateDurablePersistenceTimestamp(ctx, alloc, startTS, label) if startErr != nil && errors.Is(startErr, ErrTSOTimestampPrePhaseD) { + if verifyLegacyResolution != nil { + return errors.WithStack(verifyLegacyResolution(ctx)) + } return nil } return err diff --git a/kv/tso_prephase_d.go b/kv/tso_prephase_d.go index 02c2e83ca..59b3822b6 100644 --- a/kv/tso_prephase_d.go +++ b/kv/tso_prephase_d.go @@ -1,13 +1,14 @@ package kv import ( + "context" "time" "github.com/cockroachdb/errors" ) // The §4.3 closure of the pre-Phase-D resolution carve-out -// (docs/design/2026_09_02_proposed_prephase_d_resolution_evidence.md). +// (docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md). // // The carve-out exists because a cross-shard transaction that began before the // Phase-D marker can still hold unresolved intents afterwards, and resolving @@ -105,6 +106,12 @@ type PrimaryCommitEvidence struct { CommitTS uint64 } +// PrimaryCommitEvidenceReader distinguishes a missing local record from a +// primary whose authoritative record cannot be read on this node. +type PrimaryCommitEvidenceReader interface { + PrimaryCommitEvidence(context.Context, []byte, uint64) (PrimaryCommitEvidence, error) +} + // ClassifyPrePhaseDCommit decides whether a pre-Phase-D commit resolution may // be admitted. // diff --git a/main.go b/main.go index 6efe3f584..edb064ea6 100644 --- a/main.go +++ b/main.go @@ -3358,6 +3358,7 @@ func startRaftServers( relay, append(internalOpts, adapter.WithInternalStore(rt.store), + adapter.WithInternalPrimaryCommitEvidence(shardStore), adapter.WithInternalMigrationProposer(proposerForGroup(rt, shardGroups)), adapter.WithInternalMigrationExportRouting(rt.spec.id, sqsPartitionResolver), )..., From 2794c8ce739bc0ee805444795e4fdb4e092b322a Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 22 Oct 2026 23:56:00 +0900 Subject: [PATCH 2/6] kv: preserve prepared primary commits across timestamp cutover --- adapter/internal.go | 15 ++++ adapter/internal_commit_evidence_test.go | 41 +++++++++++ ..._partial_prephase_d_resolution_evidence.md | 30 ++++++-- kv/primary_commit_evidence.go | 37 +++++++++- kv/primary_commit_evidence_test.go | 68 +++++++++++++++++++ kv/tso_prephase_d.go | 7 +- 6 files changed, 191 insertions(+), 7 deletions(-) diff --git a/adapter/internal.go b/adapter/internal.go index 3d51eefd3..88fbe58cd 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -1159,6 +1159,9 @@ func (i *Internal) verifyLegacyCommitEvidence(ctx context.Context, reqs []*pb.Re if err != nil { return errors.Wrap(err, "validate legacy primary commit evidence") } + if forwardedTxnHasPreparedPrimary(req, meta.PrimaryKey, evidence) { + continue + } if err := kv.ClassifyPrePhaseDCommit(evidence, commitTS, false); err != nil { return errors.Wrap(err, "validate legacy primary commit decision") } @@ -1166,6 +1169,18 @@ func (i *Internal) verifyLegacyCommitEvidence(ctx context.Context, reqs []*pb.Re return nil } +func forwardedTxnHasPreparedPrimary(req *pb.Request, primaryKey []byte, evidence kv.PrimaryCommitEvidence) bool { + if !evidence.Readable || evidence.Found || !evidence.PrimaryPrepared { + return false + } + for _, mutation := range req.Mutations[1:] { + if mutation != nil && bytes.Equal(mutation.Key, primaryKey) { + return true + } + } + return false +} + func stampRequestMutationCommitTS(reqs []*pb.Request, commitTS uint64) error { for _, r := range reqs { if r == nil { diff --git a/adapter/internal_commit_evidence_test.go b/adapter/internal_commit_evidence_test.go index b6c07d72a..ff4ace716 100644 --- a/adapter/internal_commit_evidence_test.go +++ b/adapter/internal_commit_evidence_test.go @@ -99,3 +99,44 @@ func TestFillForwardedTxnCommitTSChecksEveryPrimary(t *testing.T) { require.ErrorIs(t, err, kv.ErrPrePhaseDCommitUnproven) require.Equal(t, []string{"first", "second"}, readKeys) } + +func TestInternalForwardAllowsPreparedFirstPrimaryCommitAcrossActivation(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + key string + prepared bool + found bool + decision uint64 + wantErr bool + }{ + {name: "first primary commit", key: "primary", prepared: true}, + {name: "unprepared primary", key: "primary", wantErr: true}, + {name: "secondary still requires decision", key: "secondary", prepared: true, wantErr: true}, + {name: "prepared primary cannot replace existing decision", key: "primary", prepared: true, found: true, decision: 61, wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + txn := &forwardObserverTxn{} + i := NewInternalWithEngine(txn, forwardObserverLeader{}, nil, nil, + WithInternalTimestampAllocator(&phaseDForwardAllocator{floor: 100}), + WithInternalPrimaryCommitEvidence(primaryCommitEvidenceFunc(func(context.Context, []byte, uint64) (kv.PrimaryCommitEvidence, error) { + return kv.PrimaryCommitEvidence{Readable: true, Found: test.found, CommitTS: test.decision, PrimaryPrepared: test.prepared}, nil + }))) + response, err := i.Forward(context.Background(), &pb.ForwardRequest{IsTxn: true, Requests: []*pb.Request{{ + Ts: 50, IsTxn: true, Phase: pb.Phase_COMMIT, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("primary"), CommitTS: 60})}, + {Op: pb.Op_PUT, Key: []byte(test.key)}, + }, + }}}) + if test.wantErr { + require.ErrorIs(t, err, kv.ErrPrePhaseDCommitUnproven) + require.Empty(t, txn.reqs) + return + } + require.NoError(t, err) + require.True(t, response.Success) + require.Equal(t, uint64(60), response.CommitTs) + require.Len(t, txn.reqs, 1) + }) + } +} diff --git a/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md b/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md index 9a186d4d3..a7dd35124 100644 --- a/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md +++ b/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md @@ -158,12 +158,13 @@ creation window so they can resolve existing intents indefinitely. Internal service. `ValidateForwardedTxnCommitTimestamp` invokes the evidence check only when both timestamps enter its legacy resolution carve-out. `Internal.Forward` refuses a COMMIT before proposing it if the locally readable -primary record is missing or disagrees. ABORT and valid post-D commits do not +primary record is missing or disagrees, except for the first-primary case +below. ABORT and valid post-D commits do not perform the additional evidence read. `ShardStore.PrimaryCommitEvidence` provides the separate readability signal. It uses the primary's physical owner, including partition-map routing, requires -a local active/non-staged store and verified leadership, and reads directly +a local active/non-staged store and an applied linearizable-read fence, and reads directly without proxying to another primary. It discards the verdict if catalog version, partition owner, or leadership changes while reading. Unavailable, remote, follower, or migration-staged owners return an unknown verdict rather than a @@ -173,10 +174,30 @@ The older `primaryTxnRecordedStatus` helper is not this signal: its `done` return means a transaction decision was found, not that an absent record was read authoritatively. Reusing it would conflate missing and unavailable records. +### First primary COMMIT and the remaining proof gap + +The first primary COMMIT creates `txnCommitKey` itself; demanding that record +already exist would abort a valid transaction whose PREPARE finished before +Phase-D activation but whose primary COMMIT is forwarded afterward. This path +is admitted only when the request actually commits its primary key and this +node finds a matching durable primary lock and intent for the same startTS. +An unprepared primary, secondary resolution without a decision, or a mismatched +existing decision is still rejected. The FSM retains its existing commit versus +rollback and intent checks. + +A prepared primary proves an in-flight transaction, not issuance of its claimed +commitTS. An untrusted caller can still fabricate a fresh primary PREPARE and +first COMMIT inside the bounded creation window. Closing that remaining gap +needs a durable chosen-commit proof or a revised admission protocol; this PR +does not claim the full retroactive-write carve-out is closed. In particular, +silently allocating a replacement timestamp after PREPARE is not a safe shortcut: +prepared values may already embed the original commit timestamp. + Regression coverage includes missing/matching/mismatched decisions, unknown and follower owners, partition routing and ownership changes, ABORT/post-D bypass, every primary in a batch, proposal rejection, corrupt records, and -commit-record persistence across Pebble reopen. Full distributed cutover and +commit-record persistence across Pebble reopen, first-primary admission, and +waiting for a committed decision's read index to apply. Full distributed cutover and leader-change acceptance have not been demonstrated by these focused tests; the document remains Partial until that acceptance evidence is available. @@ -193,6 +214,7 @@ the document remains Partial until that acceptance evidence is available. 4. Should the `ABORT` path get its own rule, given its timestamp is synthesised rather than recorded? -Settled in §4a. Both mechanisms are wired. Remaining acceptance must exercise +Settled in §4a, subject to the first-primary proof gap above. Remaining work +must close that gap without rejecting valid cutover transactions. Acceptance must exercise the real forwarded path across Phase-D activation and leader changes, including a remote-primary resolution after the creation window has closed. diff --git a/kv/primary_commit_evidence.go b/kv/primary_commit_evidence.go index a7e222a30..def1e881a 100644 --- a/kv/primary_commit_evidence.go +++ b/kv/primary_commit_evidence.go @@ -1,6 +1,7 @@ package kv import ( + "bytes" "context" "github.com/bootjp/elastickv/distribution" @@ -48,7 +49,8 @@ func (s *ShardStore) readPrimaryCommitEvidence(ctx context.Context, group *Shard return PrimaryCommitEvidence{}, errors.WithStack(err) } if errors.Is(err, store.ErrKeyNotFound) { - return PrimaryCommitEvidence{Readable: true}, nil + prepared, prepareErr := s.primaryPreparedForEvidence(ctx, group, route, key, startTS) + return PrimaryCommitEvidence{Readable: true, PrimaryPrepared: prepared}, prepareErr } commitTS, err := decodeTxnCommitRecord(value) if err != nil { @@ -57,6 +59,35 @@ func (s *ShardStore) readPrimaryCommitEvidence(ctx context.Context, group *Shard return PrimaryCommitEvidence{Readable: true, Found: true, CommitTS: commitTS}, nil } +func (s *ShardStore) primaryPreparedForEvidence(ctx context.Context, group *ShardGroup, route distribution.Route, key []byte, startTS uint64) (bool, error) { + value, err := s.localGetAt(ctx, group, route, txnLockKey(key), ^uint64(0)) + if errors.Is(err, store.ErrKeyNotFound) { + return false, nil + } + if err != nil { + return false, errors.WithStack(err) + } + lock, err := decodeTxnLock(value) + if err != nil { + return false, errors.Wrap(err, "read prepared primary lock") + } + if !lock.IsPrimaryKey || lock.StartTS != startTS || !bytes.Equal(lock.PrimaryKey, key) { + return false, nil + } + value, err = s.localGetAt(ctx, group, route, txnIntentKey(key), ^uint64(0)) + if errors.Is(err, store.ErrKeyNotFound) { + return false, nil + } + if err != nil { + return false, errors.WithStack(err) + } + intent, err := decodeTxnIntent(value) + if err != nil { + return false, errors.Wrap(err, "read prepared primary intent") + } + return intent.StartTS == startTS, nil +} + func (s *ShardStore) primaryEvidenceRoute(key []byte) (distribution.Route, *ShardGroup, uint64, bool) { route, group, version, ok := s.routeAndGroupForKeyWithVersion(key) if s != nil && s.partitionResolver != nil { @@ -92,7 +123,9 @@ func primaryEvidenceLeaderReady(ctx context.Context, group *ShardGroup) (bool, e if !isLeaderEngine(engine) { return false, nil } - if err := verifyLeaderEngineCtx(ctx, engine); err != nil { + readCtx, cancel := context.WithTimeout(ctx, verifyLeaderTimeout) + defer cancel() + if _, err := linearizableReadEngineCtx(readCtx, engine); err != nil { if ctx.Err() != nil { return false, errors.WithStack(ctx.Err()) } diff --git a/kv/primary_commit_evidence_test.go b/kv/primary_commit_evidence_test.go index cb39a1121..7d1153038 100644 --- a/kv/primary_commit_evidence_test.go +++ b/kv/primary_commit_evidence_test.go @@ -3,8 +3,10 @@ package kv import ( "context" "testing" + "time" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" ) @@ -25,6 +27,16 @@ func TestPrimaryCommitEvidenceDistinguishesMissingFromUnavailable(t *testing.T) {name: "rolled back", prepare: func(g *ShardGroup) { require.NoError(t, g.Store.PutAt(ctx, txnRollbackKey([]byte("key"), 50), encodeTxnRollbackRecord(), 60, 0)) }, want: PrimaryCommitEvidence{Readable: true}}, + {name: "prepared primary", prepare: func(g *ShardGroup) { + require.NoError(t, g.Store.PutAt(ctx, txnLockKey([]byte("key")), encodeTxnLock(txnLock{StartTS: 50, PrimaryKey: []byte("key"), IsPrimaryKey: true}), 50, 0)) + require.NoError(t, g.Store.PutAt(ctx, txnIntentKey([]byte("key")), encodeTxnIntent(txnIntent{StartTS: 50, Op: txnIntentOpPut, Value: []byte("v")}), 50, 0)) + }, want: PrimaryCommitEvidence{Readable: true, PrimaryPrepared: true}}, + {name: "primary lock without intent", prepare: func(g *ShardGroup) { + require.NoError(t, g.Store.PutAt(ctx, txnLockKey([]byte("key")), encodeTxnLock(txnLock{StartTS: 50, PrimaryKey: []byte("key"), IsPrimaryKey: true}), 50, 0)) + }, want: PrimaryCommitEvidence{Readable: true}}, + {name: "different transaction lock", prepare: func(g *ShardGroup) { + require.NoError(t, g.Store.PutAt(ctx, txnLockKey([]byte("key")), encodeTxnLock(txnLock{StartTS: 49, PrimaryKey: []byte("key"), IsPrimaryKey: true}), 49, 0)) + }, want: PrimaryCommitEvidence{Readable: true}}, {name: "remote primary", prepare: func(g *ShardGroup) { g.Store = nil }}, {name: "follower primary", prepare: func(g *ShardGroup) { g.Engine = &followerProxyEngine{leader: "127.0.0.1:1"} }}, {name: "corrupt record", prepare: func(g *ShardGroup) { @@ -143,3 +155,59 @@ func TestPrimaryCommitEvidenceDiscardsChangedOwnership(t *testing.T) { }) } } + +type primaryEvidenceBarrierEngine struct { + *followerProxyEngine + entered chan struct{} + applied <-chan struct{} +} + +func (*primaryEvidenceBarrierEngine) State() raftengine.State { return raftengine.StateLeader } + +func (e *primaryEvidenceBarrierEngine) LinearizableRead(ctx context.Context) (uint64, error) { + close(e.entered) + select { + case <-e.applied: + return 2, nil + case <-ctx.Done(): + return 0, ctx.Err() + } +} + +func TestPrimaryCommitEvidenceWaitsForAppliedReadIndex(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + engine := distribution.NewEngine() + engine.UpdateRoute(nil, nil, 1) + st := store.NewMVCCStore() + applied := make(chan struct{}) + raft := &primaryEvidenceBarrierEngine{followerProxyEngine: &followerProxyEngine{}, entered: make(chan struct{}), applied: applied} + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: {Store: st, Engine: raft}}) + type result struct { + evidence PrimaryCommitEvidence + err error + } + done := make(chan result, 1) + go func() { + evidence, err := shards.PrimaryCommitEvidence(ctx, []byte("key"), 50) + done <- result{evidence, err} + }() + select { + case <-raft.entered: + case <-done: + t.Fatal("read completed without an applied-index fence") + case <-ctx.Done(): + t.Fatal("read did not enter the applied-index fence") + } + select { + case <-done: + t.Fatal("read completed before the committed decision was applied") + default: + } + require.NoError(t, st.PutAt(ctx, txnCommitKey([]byte("key"), 50), encodeTxnCommitRecord(60), 60, 0)) + close(applied) + got := <-done + require.NoError(t, got.err) + require.Equal(t, PrimaryCommitEvidence{Readable: true, Found: true, CommitTS: 60}, got.evidence) +} diff --git a/kv/tso_prephase_d.go b/kv/tso_prephase_d.go index 59b3822b6..7b0d64bea 100644 --- a/kv/tso_prephase_d.go +++ b/kv/tso_prephase_d.go @@ -23,7 +23,8 @@ import ( // timestamp, inserting a write retroactively below timestamps that have already // been read. // -// Two mechanisms close it, because neither covers the whole surface: +// Two mechanisms bound creation and strengthen resolution; the first-primary +// proof gap documented in the design remains open: // // - Durable evidence, where it exists. For a COMMIT whose primary key this // node can read, the primary's own commit record has to agree with the @@ -104,6 +105,10 @@ type PrimaryCommitEvidence struct { Found bool // CommitTS is the timestamp that record carries. CommitTS uint64 + // PrimaryPrepared confirms a matching durable primary lock and intent. + // It permits the first primary COMMIT to create its decision, but is not + // evidence that a claimed commit timestamp was previously issued. + PrimaryPrepared bool } // PrimaryCommitEvidenceReader distinguishes a missing local record from a From 6e2852e83615f82493d3a26cca682d796fff3b4f Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 23 Oct 2026 00:14:37 +0900 Subject: [PATCH 3/6] kv: bound legacy evidence work per forwarded batch --- adapter/internal.go | 42 ------ adapter/internal_commit_evidence.go | 136 ++++++++++++++++++ .../internal_commit_evidence_batch_test.go | 85 +++++++++++ ..._partial_prephase_d_resolution_evidence.md | 9 ++ kv/primary_commit_evidence.go | 61 +++++++- kv/primary_commit_evidence_batch_test.go | 66 +++++++++ kv/tso_prephase_d.go | 6 + 7 files changed, 362 insertions(+), 43 deletions(-) create mode 100644 adapter/internal_commit_evidence.go create mode 100644 adapter/internal_commit_evidence_batch_test.go create mode 100644 kv/primary_commit_evidence_batch_test.go diff --git a/adapter/internal.go b/adapter/internal.go index 88fbe58cd..b8ac37317 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -1139,48 +1139,6 @@ func (i *Internal) fillForwardedTxnCommitTS(ctx context.Context, reqs []*pb.Requ return commitTS, nil } -func (i *Internal) verifyLegacyCommitEvidence(ctx context.Context, reqs []*pb.Request, startTS, commitTS uint64) error { - if i.primaryCommitEvidence == nil { - return nil - } - for _, req := range reqs { - if req == nil || req.Phase != pb.Phase_COMMIT { - continue - } - mutation, ok := forwardedTxnMetaMutation(req, []byte(kv.TxnMetaPrefix)) - if !ok { - continue - } - meta, err := kv.DecodeTxnMeta(mutation.Value) - if err != nil { - return errors.WithStack(err) - } - evidence, err := i.primaryCommitEvidence.PrimaryCommitEvidence(ctx, meta.PrimaryKey, startTS) - if err != nil { - return errors.Wrap(err, "validate legacy primary commit evidence") - } - if forwardedTxnHasPreparedPrimary(req, meta.PrimaryKey, evidence) { - continue - } - if err := kv.ClassifyPrePhaseDCommit(evidence, commitTS, false); err != nil { - return errors.Wrap(err, "validate legacy primary commit decision") - } - } - return nil -} - -func forwardedTxnHasPreparedPrimary(req *pb.Request, primaryKey []byte, evidence kv.PrimaryCommitEvidence) bool { - if !evidence.Readable || evidence.Found || !evidence.PrimaryPrepared { - return false - } - for _, mutation := range req.Mutations[1:] { - if mutation != nil && bytes.Equal(mutation.Key, primaryKey) { - return true - } - } - return false -} - func stampRequestMutationCommitTS(reqs []*pb.Request, commitTS uint64) error { for _, r := range reqs { if r == nil { diff --git a/adapter/internal_commit_evidence.go b/adapter/internal_commit_evidence.go new file mode 100644 index 000000000..923044ccc --- /dev/null +++ b/adapter/internal_commit_evidence.go @@ -0,0 +1,136 @@ +package adapter + +import ( + "bytes" + "context" + "time" + + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" +) + +const legacyCommitEvidenceTimeout = 5 * time.Second + +func (i *Internal) verifyLegacyCommitEvidence(ctx context.Context, reqs []*pb.Request, startTS, commitTS uint64) error { + if i.primaryCommitEvidence == nil { + return nil + } + ctx, cancel := context.WithTimeout(ctx, legacyCommitEvidenceTimeout) + defer cancel() + keys, err := collectLegacyPrimaryKeys(ctx, reqs) + if err != nil { + return err + } + if len(keys) == 0 { + return nil + } + results, err := i.readLegacyPrimaryEvidence(ctx, keys, startTS) + if err != nil { + return err + } + evidence := make(map[string]kv.PrimaryCommitEvidence, len(keys)) + for index, key := range keys { + evidence[string(key)] = results[index] + } + return classifyLegacyRequests(ctx, reqs, evidence, commitTS) +} + +func legacyCommitMeta(req *pb.Request) (kv.TxnMeta, bool, error) { + if req == nil || req.Phase != pb.Phase_COMMIT { + return kv.TxnMeta{}, false, nil + } + mutation, ok := forwardedTxnMetaMutation(req, []byte(kv.TxnMetaPrefix)) + if !ok { + return kv.TxnMeta{}, false, nil + } + meta, err := kv.DecodeTxnMeta(mutation.Value) + return meta, true, errors.WithStack(err) +} + +func collectLegacyPrimaryKeys(ctx context.Context, reqs []*pb.Request) ([][]byte, error) { + seen := make(map[string]struct{}) + var keys [][]byte + for _, req := range reqs { + if err := ctx.Err(); err != nil { + return nil, errors.WithStack(err) + } + meta, ok, err := legacyCommitMeta(req) + if err != nil { + return nil, err + } + if !ok { + continue + } + key := string(meta.PrimaryKey) + if _, exists := seen[key]; exists { + continue + } + if len(keys) == kv.MaxPrimaryCommitEvidenceKeys { + return nil, errors.Wrap(kv.ErrInvalidRequest, "too many legacy primary evidence keys") + } + seen[key] = struct{}{} + keys = append(keys, meta.PrimaryKey) + } + return keys, nil +} + +func (i *Internal) readLegacyPrimaryEvidence(ctx context.Context, keys [][]byte, startTS uint64) ([]kv.PrimaryCommitEvidence, error) { + if reader, ok := i.primaryCommitEvidence.(kv.PrimaryCommitEvidenceBatchReader); ok { + results, err := reader.PrimaryCommitEvidenceBatch(ctx, keys, startTS) + if err != nil { + return nil, errors.Wrap(err, "validate legacy primary commit evidence") + } + if len(results) != len(keys) { + return nil, errors.Wrap(kv.ErrInvalidRequest, "invalid primary evidence batch result") + } + return results, nil + } + results := make([]kv.PrimaryCommitEvidence, len(keys)) + for index, key := range keys { + if err := ctx.Err(); err != nil { + return nil, errors.WithStack(err) + } + evidence, err := i.primaryCommitEvidence.PrimaryCommitEvidence(ctx, key, startTS) + if err != nil { + return nil, errors.Wrap(err, "validate legacy primary commit evidence") + } + results[index] = evidence + } + return results, nil +} + +func classifyLegacyRequests(ctx context.Context, reqs []*pb.Request, evidence map[string]kv.PrimaryCommitEvidence, commitTS uint64) error { + for _, req := range reqs { + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + meta, ok, err := legacyCommitMeta(req) + if err != nil { + return err + } + if !ok { + continue + } + proof := evidence[string(meta.PrimaryKey)] + if forwardedTxnHasPreparedPrimary(req, meta.PrimaryKey, proof) { + continue + } + if err := kv.ClassifyPrePhaseDCommit(proof, commitTS, false); err != nil { + return errors.Wrap(err, "validate legacy primary commit decision") + } + } + return nil +} + +func forwardedTxnHasPreparedPrimary(req *pb.Request, primaryKey []byte, evidence kv.PrimaryCommitEvidence) bool { + if !evidence.Readable || evidence.Found || !evidence.PrimaryPrepared { + return false + } + for _, mutation := range req.Mutations[1:] { + if mutation != nil && bytes.Equal(mutation.Key, primaryKey) { + return true + } + } + return false +} diff --git a/adapter/internal_commit_evidence_batch_test.go b/adapter/internal_commit_evidence_batch_test.go new file mode 100644 index 000000000..ceb905153 --- /dev/null +++ b/adapter/internal_commit_evidence_batch_test.go @@ -0,0 +1,85 @@ +package adapter + +import ( + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" +) + +func legacyEvidenceRequest(primary []byte) *pb.Request { + return &pb.Request{IsTxn: true, Phase: pb.Phase_COMMIT, Mutations: []*pb.Mutation{{ + Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: primary, CommitTS: 60}), + }}} +} + +type primaryEvidenceBatchSpy struct { + singleCalls int + batchCalls int + keys [][]byte +} + +func (s *primaryEvidenceBatchSpy) PrimaryCommitEvidence(context.Context, []byte, uint64) (kv.PrimaryCommitEvidence, error) { + s.singleCalls++ + return kv.PrimaryCommitEvidence{}, nil +} + +func (s *primaryEvidenceBatchSpy) PrimaryCommitEvidenceBatch(_ context.Context, keys [][]byte, _ uint64) ([]kv.PrimaryCommitEvidence, error) { + s.batchCalls++ + s.keys = keys + results := make([]kv.PrimaryCommitEvidence, len(keys)) + for index := range results { + results[index] = kv.PrimaryCommitEvidence{Readable: true, Found: true, CommitTS: 60} + } + return results, nil +} + +func TestLegacyCommitEvidenceDeduplicatesBeforeBatchRead(t *testing.T) { + t.Parallel() + reader := &primaryEvidenceBatchSpy{} + i := NewInternalWithEngine(nil, nil, nil, nil, WithInternalPrimaryCommitEvidence(reader)) + var reqs []*pb.Request + for range 256 { + reqs = append(reqs, legacyEvidenceRequest([]byte("first")), legacyEvidenceRequest([]byte("second"))) + } + require.NoError(t, i.verifyLegacyCommitEvidence(t.Context(), reqs, 50, 60)) + require.Zero(t, reader.singleCalls) + require.Equal(t, 1, reader.batchCalls) + require.Equal(t, [][]byte{[]byte("first"), []byte("second")}, reader.keys) +} + +func TestLegacyCommitEvidenceRejectsTooManyKeysBeforeReading(t *testing.T) { + t.Parallel() + reader := &primaryEvidenceBatchSpy{} + i := NewInternalWithEngine(nil, nil, nil, nil, WithInternalPrimaryCommitEvidence(reader)) + var reqs []*pb.Request + for index := range kv.MaxPrimaryCommitEvidenceKeys + 1 { + reqs = append(reqs, legacyEvidenceRequest([]byte{byte(index + 1)})) + } + require.ErrorIs(t, i.verifyLegacyCommitEvidence(t.Context(), reqs, 50, 60), kv.ErrInvalidRequest) + require.Zero(t, reader.singleCalls) + require.Zero(t, reader.batchCalls) +} + +func TestLegacyCommitEvidenceFallbackSharesOneBudget(t *testing.T) { + t.Parallel() + var deadlines []time.Time + reads := 0 + reader := primaryCommitEvidenceFunc(func(ctx context.Context, _ []byte, _ uint64) (kv.PrimaryCommitEvidence, error) { + deadline, ok := ctx.Deadline() + require.True(t, ok) + require.LessOrEqual(t, time.Until(deadline), legacyCommitEvidenceTimeout) + deadlines = append(deadlines, deadline) + reads++ + return kv.PrimaryCommitEvidence{Readable: true, Found: true, CommitTS: 60}, nil + }) + i := NewInternalWithEngine(nil, nil, nil, nil, WithInternalPrimaryCommitEvidence(reader)) + require.NoError(t, i.verifyLegacyCommitEvidence(t.Context(), []*pb.Request{ + legacyEvidenceRequest([]byte("first")), legacyEvidenceRequest([]byte("first")), legacyEvidenceRequest([]byte("second")), + }, 50, 60)) + require.Equal(t, 2, reads) + require.True(t, deadlines[0].Equal(deadlines[1])) +} diff --git a/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md b/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md index a7dd35124..f05b84c15 100644 --- a/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md +++ b/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md @@ -162,6 +162,15 @@ primary record is missing or disagrees, except for the first-primary case below. ABORT and valid post-D commits do not perform the additional evidence read. +Evidence is bounded per forwarded batch: duplicate primary keys are read once, +at most 64 distinct primary keys are admitted (checked before any evidence I/O), +and the whole evidence phase shares one five-second budget, respecting any +earlier caller deadline. The batch reader shares one applied read fence per +local owning group rather than starting a new quorum round trip per metadata +entry. A route-version change makes that cached owner's verdict unknown instead +of refencing repeatedly. This prevents repeated metadata in a large Forward +request from multiplying quorum reads or resetting a five-second timeout. + `ShardStore.PrimaryCommitEvidence` provides the separate readability signal. It uses the primary's physical owner, including partition-map routing, requires a local active/non-staged store and an applied linearizable-read fence, and reads directly diff --git a/kv/primary_commit_evidence.go b/kv/primary_commit_evidence.go index def1e881a..bde4d899c 100644 --- a/kv/primary_commit_evidence.go +++ b/kv/primary_commit_evidence.go @@ -9,10 +9,55 @@ import ( "github.com/cockroachdb/errors" ) +// MaxPrimaryCommitEvidenceKeys caps distinct primary evidence work per batch. +const MaxPrimaryCommitEvidenceKeys = 64 + +type primaryEvidenceFence struct { + version uint64 + ready bool +} + // PrimaryCommitEvidence reads only a locally authoritative primary record. // It never proxies to another group or interprets an unavailable route as a // missing record, so legacy cross-group resolution remains admissible. func (s *ShardStore) PrimaryCommitEvidence(ctx context.Context, primaryKey []byte, startTS uint64) (PrimaryCommitEvidence, error) { + evidence, err := s.PrimaryCommitEvidenceBatch(ctx, [][]byte{primaryKey}, startTS) + if err != nil { + return PrimaryCommitEvidence{}, err + } + return evidence[0], nil +} + +// PrimaryCommitEvidenceBatch bounds all reads by one deadline and fences each +// owner at most once. A route-version change makes that owner's verdict unknown. +func (s *ShardStore) PrimaryCommitEvidenceBatch(ctx context.Context, primaryKeys [][]byte, startTS uint64) ([]PrimaryCommitEvidence, error) { + if len(primaryKeys) > MaxPrimaryCommitEvidenceKeys || startTS == 0 { + return nil, errors.WithStack(ErrInvalidRequest) + } + readCtx, cancel := context.WithTimeout(ctx, verifyLeaderTimeout) + defer cancel() + fences := make(map[*ShardGroup]primaryEvidenceFence) + cache := make(map[string]PrimaryCommitEvidence) + results := make([]PrimaryCommitEvidence, len(primaryKeys)) + for index, key := range primaryKeys { + if err := readCtx.Err(); err != nil { + return nil, errors.WithStack(err) + } + evidence, cached := cache[string(key)] + if !cached { + var err error + evidence, err = s.primaryCommitEvidenceInBatch(readCtx, key, startTS, fences) + if err != nil { + return nil, err + } + cache[string(key)] = evidence + } + results[index] = evidence + } + return results, nil +} + +func (s *ShardStore) primaryCommitEvidenceInBatch(ctx context.Context, primaryKey []byte, startTS uint64, fences map[*ShardGroup]primaryEvidenceFence) (PrimaryCommitEvidence, error) { if len(primaryKey) == 0 || startTS == 0 { return PrimaryCommitEvidence{}, errors.WithStack(ErrInvalidRequest) } @@ -20,7 +65,7 @@ func (s *ShardStore) PrimaryCommitEvidence(ctx context.Context, primaryKey []byt if !ok { return PrimaryCommitEvidence{}, nil } - readable, err := primaryEvidenceLeaderReady(ctx, group) + readable, err := primaryEvidenceBatchLeaderReady(ctx, group, version, fences) if err != nil || !readable { return PrimaryCommitEvidence{}, err } @@ -34,6 +79,18 @@ func (s *ShardStore) PrimaryCommitEvidence(ctx context.Context, primaryKey []byt return evidence, nil } +func primaryEvidenceBatchLeaderReady(ctx context.Context, group *ShardGroup, version uint64, fences map[*ShardGroup]primaryEvidenceFence) (bool, error) { + if fence, exists := fences[group]; exists { + return fence.ready && fence.version == version, nil + } + ready, err := primaryEvidenceLeaderReady(ctx, group) + if err != nil { + return false, err + } + fences[group] = primaryEvidenceFence{version: version, ready: ready} + return ready, nil +} + func (s *ShardStore) primaryEvidenceOwnershipUnchanged(primaryKey []byte, group *ShardGroup, groupID, version uint64) bool { currentRoute, currentGroup, currentVersion, currentOK := s.primaryEvidenceRoute(primaryKey) if !currentOK || currentGroup != group || currentVersion != version || currentRoute.GroupID != groupID { @@ -135,3 +192,5 @@ func primaryEvidenceLeaderReady(ctx context.Context, group *ShardGroup) (bool, e } var _ PrimaryCommitEvidenceReader = (*ShardStore)(nil) + +var _ PrimaryCommitEvidenceBatchReader = (*ShardStore)(nil) diff --git a/kv/primary_commit_evidence_batch_test.go b/kv/primary_commit_evidence_batch_test.go new file mode 100644 index 000000000..f2250e1fd --- /dev/null +++ b/kv/primary_commit_evidence_batch_test.go @@ -0,0 +1,66 @@ +package kv + +import ( + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +type primaryEvidenceCountingEngine struct { + *followerProxyEngine + reads int +} + +func (*primaryEvidenceCountingEngine) State() raftengine.State { return raftengine.StateLeader } + +func (e *primaryEvidenceCountingEngine) LinearizableRead(context.Context) (uint64, error) { + e.reads++ + return 1, nil +} + +func TestPrimaryCommitEvidenceBatchFencesEachOwnerOnce(t *testing.T) { + t.Parallel() + ctx := t.Context() + routes := distribution.NewEngine() + routes.UpdateRoute(nil, nil, 1) + first := &primaryEvidenceCountingEngine{followerProxyEngine: &followerProxyEngine{}} + second := &primaryEvidenceCountingEngine{followerProxyEngine: &followerProxyEngine{}} + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore(), Engine: first}, + 2: {Store: store.NewMVCCStore(), Engine: second}, + } + owners := map[string]uint64{"a": 1, "b": 1, "c": 2} + for key, groupID := range owners { + require.NoError(t, groups[groupID].Store.PutAt(ctx, txnCommitKey([]byte(key), 50), encodeTxnCommitRecord(60), 60, 0)) + } + shards := NewShardStore(routes, groups).WithPartitionResolver(&fakePartitionResolver{routes: owners}) + results, err := shards.PrimaryCommitEvidenceBatch(ctx, [][]byte{[]byte("a"), []byte("b"), []byte("c"), []byte("a")}, 50) + require.NoError(t, err) + require.Len(t, results, 4) + for _, evidence := range results { + require.Equal(t, PrimaryCommitEvidence{Readable: true, Found: true, CommitTS: 60}, evidence) + } + require.Equal(t, 1, first.reads) + require.Equal(t, 1, second.reads) + _, err = shards.PrimaryCommitEvidenceBatch(ctx, make([][]byte, MaxPrimaryCommitEvidenceKeys+1), 50) + require.ErrorIs(t, err, ErrInvalidRequest) + require.Equal(t, 1, first.reads) + require.Equal(t, 1, second.reads) +} + +func TestPrimaryCommitEvidenceBatchHonorsSharedDeadline(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond) + defer cancel() + routes := distribution.NewEngine() + routes.UpdateRoute(nil, nil, 1) + raft := &primaryEvidenceBarrierEngine{followerProxyEngine: &followerProxyEngine{}, entered: make(chan struct{}), applied: make(chan struct{})} + shards := NewShardStore(routes, map[uint64]*ShardGroup{1: {Store: store.NewMVCCStore(), Engine: raft}}) + _, err := shards.PrimaryCommitEvidenceBatch(ctx, [][]byte{[]byte("a"), []byte("b")}, 50) + require.ErrorIs(t, err, context.DeadlineExceeded) +} diff --git a/kv/tso_prephase_d.go b/kv/tso_prephase_d.go index 7b0d64bea..801b03677 100644 --- a/kv/tso_prephase_d.go +++ b/kv/tso_prephase_d.go @@ -117,6 +117,12 @@ type PrimaryCommitEvidenceReader interface { PrimaryCommitEvidence(context.Context, []byte, uint64) (PrimaryCommitEvidence, error) } +// PrimaryCommitEvidenceBatchReader shares applied read fences between primary +// keys with the same local owner. Results have the same order as primaryKeys. +type PrimaryCommitEvidenceBatchReader interface { + PrimaryCommitEvidenceBatch(context.Context, [][]byte, uint64) ([]PrimaryCommitEvidence, error) +} + // ClassifyPrePhaseDCommit decides whether a pre-Phase-D commit resolution may // be admitted. // From d03cc9658dffaae17859ca681604bea53ab4e616 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 23 Oct 2026 02:44:10 +0900 Subject: [PATCH 4/6] kv: persist prepare evidence before timestamp cutover --- adapter/admin_grpc.go | 1 + adapter/distribution_server.go | 48 +- adapter/distribution_server_test.go | 92 ++- adapter/internal.go | 27 +- adapter/internal_commit_evidence.go | 28 +- adapter/internal_commit_evidence_test.go | 27 +- adapter/internal_pending_resolution_test.go | 70 ++ adapter/internal_phase_d_acceptance_test.go | 452 ++++++++++++ adapter/internal_prepare_evidence.go | 93 +++ adapter/internal_prepare_evidence_test.go | 275 ++++++++ adapter/internal_test.go | 21 +- adapter/internal_timestamp_admission_test.go | 80 +++ distribution/migrator.go | 31 +- ..._partial_prephase_d_resolution_evidence.md | 111 ++- kv/backup_codec.go | 55 +- kv/coordinator_prepare_evidence.go | 87 +++ kv/fsm.go | 56 +- kv/fsm_backup.go | 13 + kv/fsm_migration_import.go | 25 + kv/fsm_prepare_creation_gate.go | 133 ++++ kv/fsm_prepare_evidence.go | 114 +++ kv/fsm_prepare_evidence_test.go | 154 ++++ kv/prepare_evidence_inspect.go | 89 +++ kv/primary_commit_evidence.go | 35 +- kv/primary_commit_evidence_test.go | 6 +- kv/shard_store_prepare_evidence.go | 229 ++++++ kv/shard_store_prepare_evidence_test.go | 661 ++++++++++++++++++ kv/sharded_coordinator.go | 14 +- kv/timestamp_admission.go | 93 +++ kv/timestamp_admission_benchmark_test.go | 49 ++ kv/timestamp_admission_test.go | 266 +++++++ kv/transaction.go | 9 +- kv/transaction_timestamp_admission.go | 110 +++ kv/tso.go | 53 +- kv/tso_applied_leader_test.go | 74 ++ kv/tso_fsm.go | 105 ++- kv/tso_pending.go | 363 ++++++++++ kv/tso_pending_batch.go | 64 ++ kv/tso_pending_routing.go | 96 +++ kv/tso_pending_seal_test.go | 227 ++++++ kv/tso_pending_test.go | 219 ++++++ kv/tso_pending_upgrade_test.go | 123 ++++ kv/tso_prephase_d.go | 7 + kv/tso_raft.go | 81 +-- kv/tso_raft_test.go | 21 +- kv/txn_codec.go | 55 +- kv/txn_keys.go | 9 +- kv/txn_lock_evidence_test.go | 49 ++ main.go | 9 +- main_prepare_evidence.go | 37 + main_prepare_evidence_test.go | 97 +++ main_tso_routing_test.go | 11 +- proto/distribution.pb.go | 49 +- proto/distribution.proto | 7 + proto/internal.pb.go | 294 ++++++-- proto/internal.proto | 21 + proto/internal_grpc.pb.go | 52 +- 57 files changed, 5420 insertions(+), 257 deletions(-) create mode 100644 adapter/internal_pending_resolution_test.go create mode 100644 adapter/internal_phase_d_acceptance_test.go create mode 100644 adapter/internal_prepare_evidence.go create mode 100644 adapter/internal_prepare_evidence_test.go create mode 100644 adapter/internal_timestamp_admission_test.go create mode 100644 kv/coordinator_prepare_evidence.go create mode 100644 kv/fsm_prepare_creation_gate.go create mode 100644 kv/fsm_prepare_evidence.go create mode 100644 kv/fsm_prepare_evidence_test.go create mode 100644 kv/prepare_evidence_inspect.go create mode 100644 kv/shard_store_prepare_evidence.go create mode 100644 kv/shard_store_prepare_evidence_test.go create mode 100644 kv/timestamp_admission.go create mode 100644 kv/timestamp_admission_benchmark_test.go create mode 100644 kv/timestamp_admission_test.go create mode 100644 kv/transaction_timestamp_admission.go create mode 100644 kv/tso_applied_leader_test.go create mode 100644 kv/tso_pending.go create mode 100644 kv/tso_pending_batch.go create mode 100644 kv/tso_pending_routing.go create mode 100644 kv/tso_pending_seal_test.go create mode 100644 kv/tso_pending_test.go create mode 100644 kv/tso_pending_upgrade_test.go create mode 100644 kv/txn_lock_evidence_test.go create mode 100644 main_prepare_evidence.go create mode 100644 main_prepare_evidence_test.go diff --git a/adapter/admin_grpc.go b/adapter/admin_grpc.go index 98dfd6ec4..45ee81681 100644 --- a/adapter/admin_grpc.go +++ b/adapter/admin_grpc.go @@ -763,6 +763,7 @@ var adminMethodPrefix = "/" + pb.Admin_ServiceDesc.ServiceName + "/" func adminTokenProtectedMethod(fullMethod string) bool { return strings.HasPrefix(fullMethod, adminMethodPrefix) || fullMethod == pb.Internal_ForwardAdminProposal_FullMethodName || + fullMethod == pb.Internal_GetPrepareEvidenceState_FullMethodName || fullMethod == pb.Internal_ForwardLeaseRead_FullMethodName } diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 95d7a85c2..bb2e3d481 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -15,6 +15,7 @@ import ( pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" + "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -319,6 +320,12 @@ func (s *DistributionServer) ValidateTimestamp( if req == nil || req.GetTimestamp() == 0 { return nil, errors.WithStack(status.Error(codes.InvalidArgument, "timestamp is required")) } + if req.GetLegacyResolution() { + return s.validateLegacyResolutionTimestamp(ctx, req) + } + if req.GetStartTimestamp() != 0 { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "start timestamp requires legacy resolution mode")) + } validator, ok := s.timestampAllocator.(kv.DurableTimestampValidator) if !ok { return nil, errors.WithStack(status.Error(codes.FailedPrecondition, @@ -338,6 +345,20 @@ func (s *DistributionServer) ValidateTimestamp( return resp, nil } +func (s *DistributionServer) validateLegacyResolutionTimestamp(ctx context.Context, req *pb.ValidateTimestampRequest) (*pb.ValidateTimestampResponse, error) { + if req.GetStartTimestamp() == 0 || req.GetStartTimestamp() >= req.GetTimestamp() { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "legacy resolution requires 0 < start < commit")) + } + validator, ok := s.timestampAllocator.(kv.DurableLegacyResolutionValidator) + if !ok { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "pending legacy resolution validation is not configured")) + } + if err := validator.ValidateLegacyResolutionTimestamps(ctx, req.GetStartTimestamp(), req.GetTimestamp()); err != nil { + return nil, timestampRPCError(err) + } + return &pb.ValidateTimestampResponse{Valid: true, LegacyResolutionValidated: true}, nil +} + func (s *DistributionServer) allocateTimestampReservation( ctx context.Context, count int, @@ -407,6 +428,9 @@ func timestampRPCError(err error) error { if code := status.Code(err); code != codes.Unknown { return err } + if mapped := phaseDActivationRPCError(err); mapped != nil { + return mapped + } switch { case errors.Is(err, context.Canceled): return errors.WithStack(status.Error(codes.Canceled, err.Error())) @@ -420,11 +444,33 @@ func timestampRPCError(err error) error { return errors.WithStack(status.Error(codes.OutOfRange, err.Error())) case errors.Is(err, kv.ErrTSOTimestampInvalid): return errors.WithStack(status.Error(codes.InvalidArgument, err.Error())) + default: + return errors.WithStack(status.Error(codes.Unavailable, err.Error())) + } +} + +func phaseDActivationRPCError(err error) error { + switch { case errors.Is(err, kv.ErrTSOPhaseDInactive): + return inactiveTimestampRPCError(err) + case errors.Is(err, kv.ErrTSOPhaseDActivationPending): + return errors.WithStack(status.Error(codes.ResourceExhausted, err.Error())) + case errors.Is(err, kv.ErrTSOPhaseDActivationGateRequired): return errors.WithStack(status.Error(codes.FailedPrecondition, err.Error())) default: - return errors.WithStack(status.Error(codes.Unavailable, err.Error())) + return nil + } +} + +func inactiveTimestampRPCError(err error) error { + s := status.New(codes.FailedPrecondition, err.Error()) + withDetail, detailErr := s.WithDetails(&errdetails.ErrorInfo{ + Domain: kv.TSOValidationErrorDomain, Reason: kv.TSOPhaseDInactiveReason, + }) + if detailErr != nil { + return errors.WithStack(s.Err()) } + return errors.WithStack(withDetail.Err()) } // ListRoutes returns all durable routes from catalog storage. diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index b348d7cb3..5d2605ade 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -261,6 +261,80 @@ func TestDistributionServerValidateTimestamp(t *testing.T) { require.Equal(t, codes.OutOfRange, status.Code(err)) } +type pendingResolutionAllocator struct { + *distributionTSOAllocator + pendingFloor uint64 +} + +func (a *pendingResolutionAllocator) ValidateDurableTimestamp(ctx context.Context, timestamp uint64) error { + if !a.phaseD { + return kv.ErrTSOPhaseDInactive + } + return a.distributionTSOAllocator.ValidateDurableTimestamp(ctx, timestamp) +} + +func (a *pendingResolutionAllocator) ValidateLegacyResolutionTimestamps(_ context.Context, start, commit uint64) error { + if a.phaseD { + return kv.ErrTSOPhaseDInactive + } + if start == 0 || start >= commit || (a.pendingFloor != 0 && commit > a.pendingFloor) { + return kv.ErrTSOTimestampInvalid + } + return nil +} + +func TestDistributionServerPendingLegacyResolutionMode(t *testing.T) { + alloc := &pendingResolutionAllocator{distributionTSOAllocator: &distributionTSOAllocator{leader: true}, pendingFloor: 100} + s := NewDistributionServer(distribution.NewEngine(), nil, WithDistributionTimestampAllocator(alloc)) + ctx := context.Background() + resp, err := s.ValidateTimestamp(ctx, &pb.ValidateTimestampRequest{Timestamp: 100, StartTimestamp: 50, LegacyResolution: true}) + require.NoError(t, err) + require.True(t, resp.GetValid()) + require.True(t, resp.GetLegacyResolutionValidated()) + require.False(t, resp.GetPhaseDActive()) + for _, req := range []*pb.ValidateTimestampRequest{ + {Timestamp: 100, LegacyResolution: true}, + {Timestamp: 100, StartTimestamp: 100, LegacyResolution: true}, + {Timestamp: 101, StartTimestamp: 50, LegacyResolution: true}, + {Timestamp: 100, StartTimestamp: 50}, + } { + _, err := s.ValidateTimestamp(ctx, req) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + } + _, err = s.ValidateTimestamp(ctx, &pb.ValidateTimestampRequest{Timestamp: 100}) + require.Error(t, err, "ordinary validation must remain closed before M7") + alloc.pendingFloor = 0 + resp, err = s.ValidateTimestamp(ctx, &pb.ValidateTimestampRequest{Timestamp: 101, StartTimestamp: 50, LegacyResolution: true}) + require.NoError(t, err, "pre-pending drain must admit existing resolutions above the future barrier") + require.True(t, resp.GetLegacyResolutionValidated()) + alloc.phaseD = true + _, err = s.ValidateTimestamp(ctx, &pb.ValidateTimestampRequest{Timestamp: 100, StartTimestamp: 50, LegacyResolution: true}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) +} + +func TestDistributionServerPendingLegacyResolutionRoutedWrappers(t *testing.T) { + alloc := &pendingResolutionAllocator{distributionTSOAllocator: &distributionTSOAllocator{leader: true}, pendingFloor: 100} + addr := serveDistributionTestServer(t, NewDistributionServer(distribution.NewEngine(), nil, WithDistributionTimestampAllocator(alloc))) + routed, err := kv.NewLeaderRoutedTSOAllocator(&distributionTSOAllocator{}, distributionLeaderView{addr: addr}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, routed.Close()) }) + batch, err := kv.NewBatchAllocator(routed, 2) + require.NoError(t, err) + dynamic := kv.NewDynamicTimestampAllocator(batch) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.ErrorIs(t, dynamic.ValidateDurableTimestamp(ctx, 100), kv.ErrTSOPhaseDInactive) + require.NoError(t, dynamic.ValidateLegacyResolutionTimestamps(ctx, 50, 100)) + require.NoError(t, ctx.Err(), "inactive must not consume the retry budget before resolution") + require.Error(t, batch.ValidateLegacyResolutionTimestamps(context.Background(), 50, 101)) +} + +func TestDistributionServerPendingLegacyResolutionRejectsUnsupportedAllocator(t *testing.T) { + s := NewDistributionServer(distribution.NewEngine(), nil, WithDistributionTimestampAllocator(&distributionTSOAllocator{})) + _, err := s.ValidateTimestamp(context.Background(), &pb.ValidateTimestampRequest{Timestamp: 100, StartTimestamp: 50, LegacyResolution: true}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) +} + func TestDistributionServerValidateTimestamp_LeaderRoutedRPC(t *testing.T) { serverAlloc := &distributionTSOAllocator{ base: 701, @@ -402,7 +476,7 @@ func TestDistributionServerGetTimestamp_LeaderRoutedActivatesCutover(t *testing. require.True(t, serverAlloc.activate) } -func serveDistributionTestServer(t *testing.T, server *DistributionServer) string { +func serveDistributionTestServer(t *testing.T, server pb.DistributionServer) string { t.Helper() listener, err := new(net.ListenConfig).Listen(context.Background(), "tcp", "127.0.0.1:0") require.NoError(t, err) @@ -421,6 +495,22 @@ func serveDistributionTestServer(t *testing.T, server *DistributionServer) strin return listener.Addr().String() } +type legacyIgnoringTimestampValidationServer struct { + pb.UnimplementedDistributionServer +} + +func (legacyIgnoringTimestampValidationServer) ValidateTimestamp(context.Context, *pb.ValidateTimestampRequest) (*pb.ValidateTimestampResponse, error) { + return &pb.ValidateTimestampResponse{Valid: true, PhaseDActive: true}, nil +} + +func TestDistributionServerPendingLegacyResolutionRejectsOldModeIgnoringServer(t *testing.T) { + addr := serveDistributionTestServer(t, legacyIgnoringTimestampValidationServer{}) + routed, err := kv.NewLeaderRoutedTSOAllocator(&distributionTSOAllocator{}, distributionLeaderView{addr: addr}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, routed.Close()) }) + require.ErrorIs(t, routed.ValidateLegacyResolutionTimestamps(context.Background(), 50, 100), kv.ErrTSOProtocolUnsupported) +} + func TestNewDistributionServer_DefaultCatalogReloadRetryPolicy(t *testing.T) { t.Parallel() diff --git a/adapter/internal.go b/adapter/internal.go index b8ac37317..92ae5045c 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -61,6 +61,10 @@ func WithInternalPrimaryCommitEvidence(reader kv.PrimaryCommitEvidenceReader) In } } +func WithInternalTSOCutoverState(state interface{ CutoverActive() bool }) InternalOption { + return func(i *Internal) { i.tsoCutoverState = state } +} + func WithInternalMigrationProposer(proposer raftengine.Proposer) InternalOption { return func(i *Internal) { i.migrationProposer = proposer @@ -134,6 +138,7 @@ type Internal struct { relay *RedisPubSubRelay store store.MVCCStore primaryCommitEvidence kv.PrimaryCommitEvidenceReader + tsoCutoverState interface{ CutoverActive() bool } migrationProposer raftengine.Proposer migrationImportGate func(context.Context) error migrationPromoteGate func(context.Context) error @@ -199,6 +204,9 @@ func (i *Internal) Forward(ctx context.Context, req *pb.ForwardRequest) (*pb.For CommitIndex: 0, }, errors.WithStack(err) } + if err := kv.StampTimestampAdmission(ctx, i.tsAllocator, i.store, req.GetRequests()); err != nil { + return nil, errors.WithStack(err) + } r, err := i.transactionManager.Commit(ctx, req.Requests) if err != nil { @@ -1005,6 +1013,9 @@ func (i *Internal) stampTxnTimestamps(ctx context.Context, reqs []*pb.Request) ( } } + if err := i.validateForwardedPrepareEvidence(ctx, reqs, startTS); err != nil { + return 0, err + } commitTS, err := i.fillForwardedTxnCommitTS(ctx, reqs, startTS) if err != nil { return 0, err @@ -1017,9 +1028,19 @@ func (i *Internal) stampTxnTimestamps(ctx context.Context, reqs []*pb.Request) ( func (i *Internal) validateForwardedTxnStartForPhase(ctx context.Context, reqs []*pb.Request, startTS uint64) error { const label = "stampTxnTimestamps: forwarded start ts" - if forwardedTxnResolutionOnly(reqs) { + if forwardedTxnCreationsHavePrepareProof(reqs) { err := kv.ValidateDurablePersistenceTimestamp(ctx, i.tsAllocator, startTS, label) if errors.Is(err, kv.ErrTSOTimestampPrePhaseD) { + // Applied-read vouchers can legitimately retain an old startTS. + // The chosen commit proof and current admission epoch are validated + // before proposing; no legacy creation-window exemption is needed. + return nil + } + return errors.WithStack(err) + } + if forwardedTxnResolutionOnly(reqs) { + err := kv.ValidateDurablePersistenceTimestamp(ctx, i.tsAllocator, startTS, label) + if errors.Is(err, kv.ErrTSOTimestampPrePhaseD) || errors.Is(err, kv.ErrTSOPhaseDInactive) { return nil // Existing intents may need COMMIT or ABORT after the creation window closes. } return errors.WithStack(err) @@ -1118,8 +1139,8 @@ func (i *Internal) fillForwardedTxnCommitTS(ctx context.Context, reqs []*pb.Requ } } else if err := kv.ValidateForwardedTxnCommitTimestamp( ctx, i.tsAllocator, startTS, commitTS, resolution, - "fillForwardedTxnCommitTS: forwarded commit ts", func(ctx context.Context) error { - return i.verifyLegacyCommitEvidence(ctx, reqs, startTS, commitTS) + "fillForwardedTxnCommitTS: forwarded commit ts", func(ctx context.Context, pending bool) error { + return i.verifyCommitEvidence(ctx, reqs, startTS, commitTS, pending) }); err != nil { // A commit timestamp that arrived already set in the transaction meta // is the timestamp every mutation in this batch is persisted under, and diff --git a/adapter/internal_commit_evidence.go b/adapter/internal_commit_evidence.go index 923044ccc..3a8462326 100644 --- a/adapter/internal_commit_evidence.go +++ b/adapter/internal_commit_evidence.go @@ -13,9 +13,10 @@ import ( const legacyCommitEvidenceTimeout = 5 * time.Second func (i *Internal) verifyLegacyCommitEvidence(ctx context.Context, reqs []*pb.Request, startTS, commitTS uint64) error { - if i.primaryCommitEvidence == nil { - return nil - } + return i.verifyCommitEvidence(ctx, reqs, startTS, commitTS, false) +} + +func (i *Internal) verifyCommitEvidence(ctx context.Context, reqs []*pb.Request, startTS, commitTS uint64, pending bool) error { ctx, cancel := context.WithTimeout(ctx, legacyCommitEvidenceTimeout) defer cancel() keys, err := collectLegacyPrimaryKeys(ctx, reqs) @@ -25,6 +26,9 @@ func (i *Internal) verifyLegacyCommitEvidence(ctx context.Context, reqs []*pb.Re if len(keys) == 0 { return nil } + if i.primaryCommitEvidence == nil { + return errors.Wrap(kv.ErrTSOProtocolUnsupported, "primary commit evidence reader is required") + } results, err := i.readLegacyPrimaryEvidence(ctx, keys, startTS) if err != nil { return err @@ -33,7 +37,7 @@ func (i *Internal) verifyLegacyCommitEvidence(ctx context.Context, reqs []*pb.Re for index, key := range keys { evidence[string(key)] = results[index] } - return classifyLegacyRequests(ctx, reqs, evidence, commitTS) + return classifyLegacyRequests(ctx, reqs, evidence, commitTS, pending) } func legacyCommitMeta(req *pb.Request) (kv.TxnMeta, bool, error) { @@ -100,7 +104,7 @@ func (i *Internal) readLegacyPrimaryEvidence(ctx context.Context, keys [][]byte, return results, nil } -func classifyLegacyRequests(ctx context.Context, reqs []*pb.Request, evidence map[string]kv.PrimaryCommitEvidence, commitTS uint64) error { +func classifyLegacyRequests(ctx context.Context, reqs []*pb.Request, evidence map[string]kv.PrimaryCommitEvidence, commitTS uint64, pending bool) error { for _, req := range reqs { if err := ctx.Err(); err != nil { return errors.WithStack(err) @@ -113,7 +117,7 @@ func classifyLegacyRequests(ctx context.Context, reqs []*pb.Request, evidence ma continue } proof := evidence[string(meta.PrimaryKey)] - if forwardedTxnHasPreparedPrimary(req, meta.PrimaryKey, proof) { + if forwardedTxnHasPreparedPrimary(req, meta.PrimaryKey, commitTS, proof, pending) { continue } if err := kv.ClassifyPrePhaseDCommit(proof, commitTS, false); err != nil { @@ -123,10 +127,13 @@ func classifyLegacyRequests(ctx context.Context, reqs []*pb.Request, evidence ma return nil } -func forwardedTxnHasPreparedPrimary(req *pb.Request, primaryKey []byte, evidence kv.PrimaryCommitEvidence) bool { +func forwardedTxnHasPreparedPrimary(req *pb.Request, primaryKey []byte, commitTS uint64, evidence kv.PrimaryCommitEvidence, pending bool) bool { if !evidence.Readable || evidence.Found || !evidence.PrimaryPrepared { return false } + if !preparedPrimaryTimestampMatches(evidence, commitTS, pending) { + return false + } for _, mutation := range req.Mutations[1:] { if mutation != nil && bytes.Equal(mutation.Key, primaryKey) { return true @@ -134,3 +141,10 @@ func forwardedTxnHasPreparedPrimary(req *pb.Request, primaryKey []byte, evidence } return false } + +func preparedPrimaryTimestampMatches(evidence kv.PrimaryCommitEvidence, commitTS uint64, pending bool) bool { + if evidence.PreparedCommitTS == commitTS { + return true + } + return evidence.PreparedCommitTS == 0 && (pending || evidence.LegacyPreparedAllowed) +} diff --git a/adapter/internal_commit_evidence_test.go b/adapter/internal_commit_evidence_test.go index ff4ace716..3549971b6 100644 --- a/adapter/internal_commit_evidence_test.go +++ b/adapter/internal_commit_evidence_test.go @@ -10,6 +10,12 @@ import ( "github.com/stretchr/testify/require" ) +func recordedPrimaryCommitEvidence(commitTS uint64) kv.PrimaryCommitEvidenceReader { + return primaryCommitEvidenceFunc(func(context.Context, []byte, uint64) (kv.PrimaryCommitEvidence, error) { + return kv.PrimaryCommitEvidence{Readable: true, Found: true, CommitTS: commitTS}, nil + }) +} + type primaryCommitEvidenceFunc func(context.Context, []byte, uint64) (kv.PrimaryCommitEvidence, error) func (f primaryCommitEvidenceFunc) PrimaryCommitEvidence(ctx context.Context, key []byte, startTS uint64) (kv.PrimaryCommitEvidence, error) { @@ -103,14 +109,19 @@ func TestFillForwardedTxnCommitTSChecksEveryPrimary(t *testing.T) { func TestInternalForwardAllowsPreparedFirstPrimaryCommitAcrossActivation(t *testing.T) { t.Parallel() for _, test := range []struct { - name string - key string - prepared bool - found bool - decision uint64 - wantErr bool + name string + key string + prepared bool + found bool + decision uint64 + proofTS uint64 + legacyAllowed bool + wantErr bool }{ - {name: "first primary commit", key: "primary", prepared: true}, + {name: "unproven legacy primary cannot cross completed activation", key: "primary", prepared: true, wantErr: true}, + {name: "existing active cluster drains old primary before protocol seal", key: "primary", prepared: true, legacyAllowed: true}, + {name: "first primary with durable timestamp", key: "primary", prepared: true, proofTS: 60}, + {name: "first primary timestamp differs from prepare", key: "primary", prepared: true, proofTS: 61, wantErr: true}, {name: "unprepared primary", key: "primary", wantErr: true}, {name: "secondary still requires decision", key: "secondary", prepared: true, wantErr: true}, {name: "prepared primary cannot replace existing decision", key: "primary", prepared: true, found: true, decision: 61, wantErr: true}, @@ -120,7 +131,7 @@ func TestInternalForwardAllowsPreparedFirstPrimaryCommitAcrossActivation(t *test i := NewInternalWithEngine(txn, forwardObserverLeader{}, nil, nil, WithInternalTimestampAllocator(&phaseDForwardAllocator{floor: 100}), WithInternalPrimaryCommitEvidence(primaryCommitEvidenceFunc(func(context.Context, []byte, uint64) (kv.PrimaryCommitEvidence, error) { - return kv.PrimaryCommitEvidence{Readable: true, Found: test.found, CommitTS: test.decision, PrimaryPrepared: test.prepared}, nil + return kv.PrimaryCommitEvidence{Readable: true, Found: test.found, CommitTS: test.decision, PrimaryPrepared: test.prepared, PreparedCommitTS: test.proofTS, LegacyPreparedAllowed: test.legacyAllowed}, nil }))) response, err := i.Forward(context.Background(), &pb.ForwardRequest{IsTxn: true, Requests: []*pb.Request{{ Ts: 50, IsTxn: true, Phase: pb.Phase_COMMIT, Mutations: []*pb.Mutation{ diff --git a/adapter/internal_pending_resolution_test.go b/adapter/internal_pending_resolution_test.go new file mode 100644 index 000000000..ced68ed7e --- /dev/null +++ b/adapter/internal_pending_resolution_test.go @@ -0,0 +1,70 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" +) + +type internalDrainResolutionAllocator struct { + phaseDForwardAllocator + bound uint64 + checks int +} + +func (a *internalDrainResolutionAllocator) ValidateDurableTimestamp(context.Context, uint64) error { + return kv.ErrTSOPhaseDInactive +} + +func (a *internalDrainResolutionAllocator) ValidateLegacyResolutionTimestamps(_ context.Context, start, commit uint64) error { + a.checks++ + if start == 0 || commit <= start || commit > a.bound { + return kv.ErrTSOTimestampInvalid + } + return nil +} + +func TestInternalPendingResolutionDrainsUnprovenPrimaryWithoutRetimestamping(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + phase pb.Phase + commit uint64 + prepared bool + wantErr bool + }{ + {name: "old first primary commit", phase: pb.Phase_COMMIT, commit: 60, prepared: true}, + {name: "old abort", phase: pb.Phase_ABORT, commit: 60}, + {name: "no existing intent", phase: pb.Phase_COMMIT, commit: 60, wantErr: true}, + {name: "above pending barrier", phase: pb.Phase_COMMIT, commit: 101, prepared: true, wantErr: true}, + {name: "one phase gets no pending exception", phase: pb.Phase_NONE, commit: 60, prepared: true, wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + allocator := &internalDrainResolutionAllocator{bound: 100} + txn := &forwardObserverTxn{} + internal := NewInternalWithEngine(txn, forwardObserverLeader{}, nil, nil, + WithInternalTimestampAllocator(allocator), + WithInternalPrimaryCommitEvidence(primaryCommitEvidenceFunc(func(context.Context, []byte, uint64) (kv.PrimaryCommitEvidence, error) { + return kv.PrimaryCommitEvidence{Readable: true, PrimaryPrepared: test.prepared}, nil + }))) + response, err := internal.Forward(t.Context(), &pb.ForwardRequest{IsTxn: true, Requests: []*pb.Request{{ + IsTxn: true, Phase: test.phase, Ts: 50, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("primary"), CommitTS: test.commit})}, + {Op: pb.Op_PUT, Key: []byte("primary")}, + }, + }}}) + if test.wantErr { + require.Error(t, err) + require.Empty(t, txn.reqs) + return + } + require.NoError(t, err) + require.Equal(t, test.commit, response.GetCommitTs()) + require.Equal(t, 1, allocator.checks) + require.Len(t, txn.reqs, 1) + }) + } +} diff --git a/adapter/internal_phase_d_acceptance_test.go b/adapter/internal_phase_d_acceptance_test.go new file mode 100644 index 000000000..e57dd440d --- /dev/null +++ b/adapter/internal_phase_d_acceptance_test.go @@ -0,0 +1,452 @@ +package adapter + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "net" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + etcdraft "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +type phaseDAcceptanceNode struct { + id string + clock *kv.HLC + state *kv.TSOStateMachine + routes *distribution.Engine + groups map[uint64]*kv.ShardGroup + shards *kv.ShardStore + local *kv.RaftTSOAllocator + routed *kv.LeaderRoutedTSOAllocator + batch *kv.BatchAllocator + results [3]*raftengine.FactoryResult + listeners [3]net.Listener + servers [3]*grpc.Server + clients [3]*grpc.ClientConn + txns [3]*kv.TransactionManager +} + +type phaseDAcceptanceCluster struct { + token string + nodes [3]*phaseDAcceptanceNode + releaseBlocked atomic.Bool + adminCalls atomic.Int32 + stateCalls atomic.Int32 + forwardCalls atomic.Int32 + releaseFaults atomic.Int32 +} + +func newPhaseDAcceptanceCluster(t *testing.T) *phaseDAcceptanceCluster { + t.Helper() + credential := make([]byte, 32) + _, err := rand.Read(credential) + require.NoError(t, err) + c := &phaseDAcceptanceCluster{token: hex.EncodeToString(credential)} + t.Cleanup(func() { c.close(t) }) + for n := range c.nodes { + node := &phaseDAcceptanceNode{id: fmt.Sprintf("n%d", n), clock: kv.NewHLC(), + routes: distribution.NewEngine(), groups: make(map[uint64]*kv.ShardGroup)} + c.nodes[n] = node + node.clock.SetPhysicalCeiling(time.Now().Add(2 * time.Minute).UnixMilli()) + node.state = kv.NewTSOStateMachine(node.clock) + node.routes.UpdateRoute(nil, []byte("m"), 1) + node.routes.UpdateRoute([]byte("m"), nil, 2) + for group := range node.listeners { + listener, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + node.listeners[group] = listener + } + } + factory := etcdraft.NewFactory(etcdraft.FactoryConfig{TickInterval: 20 * time.Millisecond, + HeartbeatTick: 1, ElectionTick: 15, MaxSizePerMsg: 1 << 20, MaxInflightMsg: 64}) + for group := range uint64(3) { + peers := make([]raftengine.Server, 0, len(c.nodes)) + for _, node := range c.nodes { + peers = append(peers, raftengine.Server{ID: node.id, Address: node.listeners[group].Addr().String(), Suffrage: "voter"}) + } + for _, node := range c.nodes { + shard := &kv.ShardGroup{} + var fsm raftengine.StateMachine = node.state + if group != 0 { + shard.Store = store.NewMVCCStore() + fsm = kv.NewKvFSMWithHLC(shard.Store, node.clock) + } else { + shard.TSOState = node.state + } + node.groups[group] = shard + result, err := factory.Create(raftengine.FactoryConfig{LocalID: node.id, + LocalAddress: node.listeners[group].Addr().String(), + DataDir: filepath.Join(t.TempDir(), node.id, fmt.Sprintf("g%d", group)), + Peers: peers, Bootstrap: true, StateMachine: fsm}) + require.NoError(t, err) + node.results[group], shard.Engine = result, result.Engine + shard.Txn = kv.NewLeaderProxyWithEngine(result.Engine, + kv.WithTimestampAdmission(shard.Store, func() kv.TimestampAllocator { return node.batch })) + shard.SetLeaderReadToken(c.token) + } + } + for _, node := range c.nodes { + c.serveNode(t, node) + } + for group := range 3 { + c.placeLeader(t, group, group) + } + return c +} + +func (c *phaseDAcceptanceCluster) serveNode(t *testing.T, node *phaseDAcceptanceNode) { + t.Helper() + node.shards = kv.NewShardStore(node.routes, node.groups) + local, err := kv.NewRaftTSOAllocator(node.groups[0], node.clock, + kv.WithTSOCutoverFloorProvider(node.shards), kv.WithTSOPhaseDActivationGate(node.shards)) + require.NoError(t, err) + node.local = local + node.routed, err = kv.NewLeaderRoutedTSOAllocator(local, node.groups[0].Engine, + kv.WithTSOCutoverActivation(), kv.WithTSORoutedClock(node.clock)) + require.NoError(t, err) + node.batch, err = kv.NewBatchAllocator(node.routed, 8) + require.NoError(t, err) + for group := range uint64(3) { + result := node.results[group] + shard := node.groups[group] + node.txns[group] = kv.NewTransactionWithProposer(result.Engine, + kv.WithTimestampAdmission(shard.Store, func() kv.TimestampAllocator { return node.batch })) + options := []InternalOption{WithInternalTimestampAllocator(node.batch), + WithInternalTSOCutoverState(node.state), WithInternalPrimaryCommitEvidence(node.shards), + WithInternalAdminProposer(result.Engine)} + if group == 0 { + options = append(options, WithKVForwardRejected()) + } else { + options = append(options, WithInternalStore(shard.Store)) + } + auth, _ := AdminTokenAuth(c.token) + server := grpc.NewServer(grpc.ChainUnaryInterceptor(auth, c.observeRPC(group))) + node.servers[group] = server + require.NotNil(t, result.RegisterTransport) + result.RegisterTransport(server) + pb.RegisterRawKVServer(server, NewGRPCServer(node.shards, nil)) + pb.RegisterInternalServer(server, NewInternalWithEngine(node.txns[group], result.Engine, node.clock, nil, options...)) + pb.RegisterDistributionServer(server, NewDistributionServer(node.routes, nil, WithDistributionTimestampAllocator(local))) + go func() { _ = server.Serve(node.listeners[group]) }() + node.clients[group], err = grpc.NewClient(node.listeners[group].Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + } +} + +func (c *phaseDAcceptanceCluster) observeRPC(group uint64) grpc.UnaryServerInterceptor { + return func(ctx context.Context, request any, info *grpc.UnaryServerInfo, next grpc.UnaryHandler) (any, error) { + switch info.FullMethod { + case pb.Internal_GetPrepareEvidenceState_FullMethodName: + c.stateCalls.Add(1) + case pb.Internal_Forward_FullMethodName: + c.forwardCalls.Add(1) + case pb.Internal_ForwardAdminProposal_FullMethodName: + c.adminCalls.Add(1) + admin, ok := request.(*pb.ForwardAdminProposalRequest) + if ok && group == 2 && c.releaseBlocked.Load() && len(admin.GetPayload()) == len(kv.EncodePrepareEvidenceRelease(1)) && + bytes.Equal(admin.GetPayload()[:2], kv.EncodePrepareEvidenceRelease(1)[:2]) { + c.releaseFaults.Add(1) + return nil, status.Error(codes.Unavailable, "injected release RPC failure") + } + } + return next(ctx, request) + } +} + +func (c *phaseDAcceptanceCluster) leader(t *testing.T, group int) int { + t.Helper() + leader := -1 + require.Eventually(t, func() bool { + leader = -1 + for n, node := range c.nodes { + if node.results[group].Engine.State() == raftengine.StateLeader { + if leader != -1 { + return false + } + leader = n + } + } + return leader != -1 + }, 10*time.Second, 20*time.Millisecond, "group %d needs exactly one leader", group) + return leader +} + +func (c *phaseDAcceptanceCluster) placeLeader(t *testing.T, group, target int) { + t.Helper() + current := c.leader(t, group) + if current != target { + admin, ok := c.nodes[current].results[group].Engine.(raftengine.Admin) + require.True(t, ok) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + require.NoError(t, admin.TransferLeadershipToServer(ctx, c.nodes[target].id, c.nodes[target].listeners[group].Addr().String())) + } + require.Eventually(t, func() bool { + for _, node := range c.nodes { + if node.results[group].Engine.Leader().ID != c.nodes[target].id { + return false + } + } + return c.nodes[target].results[group].Engine.State() == raftengine.StateLeader + }, 10*time.Second, 20*time.Millisecond, "group %d must be led by %s", group, c.nodes[target].id) + configuration, err := c.nodes[target].results[group].Engine.Configuration(context.Background()) + require.NoError(t, err) + require.Len(t, configuration.Servers, 3) + for _, member := range configuration.Servers { + require.Equal(t, "voter", member.Suffrage) + } +} + +func (c *phaseDAcceptanceCluster) close(t *testing.T) { + t.Helper() + for _, node := range c.nodes { + if node == nil { + continue + } + node.close(t) + } +} + +func (node *phaseDAcceptanceNode) close(t *testing.T) { + t.Helper() + for _, txn := range node.txns { + if txn != nil { + txn.Close() + } + } + node.closeEngines(t) + node.closeRPC(t) + if node.routed != nil { + require.NoError(t, node.routed.Close()) + } + if node.shards != nil { + require.NoError(t, node.shards.Close()) + } + for _, shard := range node.groups { + if shard.Store != nil { + require.NoError(t, shard.Store.Close()) + } + } +} + +func (node *phaseDAcceptanceNode) closeEngines(t *testing.T) { + t.Helper() + for _, result := range node.results { + if result != nil { + require.NoError(t, result.Engine.Close()) + if result.Close != nil { + require.NoError(t, result.Close()) + } + } + } +} + +func (node *phaseDAcceptanceNode) closeRPC(t *testing.T) { + t.Helper() + for group, listener := range node.listeners { + if node.clients[group] != nil { + require.NoError(t, node.clients[group].Close()) + } + if node.servers[group] != nil { + node.servers[group].Stop() + } else if listener != nil { + _ = listener.Close() + } + } +} + +func phaseDAcceptanceRequest(phase pb.Phase, key string, start, commit uint64) *pb.Request { + meta := kv.TxnMeta{PrimaryKey: []byte("b-primary"), LockTTLms: 60000, CommitTS: commit} + if phase == pb.Phase_PREPARE { + meta.CommitTS, meta.PrepareCommitTS = 0, commit + } + return &pb.Request{IsTxn: true, Phase: phase, Ts: start, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(meta)}, + {Op: pb.Op_PUT, Key: []byte(key), Value: []byte("value-" + key)}, + }} +} + +func TestInternalPhaseDAcceptanceThreeVotersThreeGroups(t *testing.T) { + c := newPhaseDAcceptanceCluster(t) + ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second) + defer cancel() + auth := c.adoptCutover(t, ctx) + start, proof := c.prepare(t, auth) + window := c.finishAfterLeaderChange(t, auth, proof) + committed := c.resolve(t, auth, start, proof, window) + c.verifyReplicas(t, auth, start, proof, window, committed) + require.Positive(t, c.adminCalls.Load()) + require.Positive(t, c.stateCalls.Load()) + require.GreaterOrEqual(t, c.forwardCalls.Load(), int32(6)) + t.Logf("3 voters x 3 groups; B=%d start=%d proof=%d read=%d; Forward=%d Admin=%d Capability=%d ReleaseFaults=%d", + window.GetPhaseDFloor(), start, proof, window.GetTimestamp(), c.forwardCalls.Load(), c.adminCalls.Load(), c.stateCalls.Load(), c.releaseFaults.Load()) +} + +func (c *phaseDAcceptanceCluster) adoptCutover(t *testing.T, ctx context.Context) context.Context { + t.Helper() + auth := metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+c.token) + primaryClient := pb.NewInternalClient(c.nodes[1].clients[1]) + _, err := primaryClient.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{CapabilityOnly: true}) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + _, err = primaryClient.ForwardAdminProposal(ctx, &pb.ForwardAdminProposalRequest{Payload: kv.EncodePrepareEvidenceFence(0)}) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + + tso := pb.NewDistributionClient(c.nodes[0].clients[0]) + _, err = tso.GetTimestamp(auth, &pb.GetTimestampRequest{Count: 8, ActivateCutover: true}) + require.NoError(t, err) + require.Eventually(t, func() bool { + for _, node := range c.nodes { + if !node.state.CutoverActive() { + return false + } + } + return true + }, 5*time.Second, 10*time.Millisecond) + for _, node := range c.nodes { + for _, conn := range node.clients { + capability, err := pb.NewInternalClient(conn).GetPrepareEvidenceState(auth, &pb.PrepareEvidenceStateRequest{CapabilityOnly: true}) + require.NoError(t, err) + require.True(t, capability.GetSupported()) + require.True(t, capability.GetDedicatedTsoConfigured()) + require.True(t, capability.GetCutoverApplied()) + } + } + return auth +} + +func (c *phaseDAcceptanceCluster) prepare(t *testing.T, auth context.Context) (uint64, uint64) { + t.Helper() + require.NoError(t, c.nodes[0].shards.CheckPhaseDProtocol(auth)) + start, err := c.nodes[2].batch.Next(auth) + require.NoError(t, err) + proof, err := c.nodes[2].batch.NextAfter(auth, start) + require.NoError(t, err) + for group, key := range map[uint64]string{1: "b-primary", 2: "x-secondary"} { + response, err := c.nodes[0].groups[group].Txn.Commit(auth, []*pb.Request{phaseDAcceptanceRequest(pb.Phase_PREPARE, key, start, proof)}) + require.NoError(t, err) + require.NotZero(t, response.CommitIndex) + } + unknown, err := c.nodes[2].shards.PrimaryCommitEvidence(auth, []byte("b-primary"), start) + require.NoError(t, err) + require.False(t, unknown.Readable, "the secondary leader must not treat its follower-local primary copy as authoritative") + return start, proof +} + +func (c *phaseDAcceptanceCluster) finishAfterLeaderChange(t *testing.T, auth context.Context, proof uint64) *pb.GetTimestampResponse { + t.Helper() + c.releaseBlocked.Store(true) + previousAllocation := c.nodes[0].state.AllocationFloor() + tso := pb.NewDistributionClient(c.nodes[0].clients[0]) + _, err := tso.GetTimestamp(auth, &pb.GetTimestampRequest{Count: 2, ActivateCutover: true, ActivatePhaseD: true}) + require.Error(t, err, "release failure must withhold the first Phase-D window") + require.True(t, c.nodes[0].state.PhaseDActive()) + require.Equal(t, previousAllocation, c.nodes[0].state.AllocationFloor(), "an unfinished release must not reserve a new window") + barrier := c.nodes[0].state.PhaseDFloor() + require.GreaterOrEqual(t, barrier, proof) + require.Positive(t, c.releaseFaults.Load(), "the fault must hit the actual authenticated remote admin RPC") + + c.placeLeader(t, 0, 1) + require.Eventually(t, func() bool { return c.nodes[1].state.PhaseDActive() }, 5*time.Second, 10*time.Millisecond) + require.Equal(t, barrier, c.nodes[1].state.PhaseDFloor()) + _, err = pb.NewDistributionClient(c.nodes[1].clients[0]).GetTimestamp(auth, &pb.GetTimestampRequest{Count: 2}) + require.Error(t, err, "the replacement TSO leader must also finish release before issuing") + require.Equal(t, previousAllocation, c.nodes[1].state.AllocationFloor()) + c.releaseBlocked.Store(false) + window, err := pb.NewDistributionClient(c.nodes[1].clients[0]).GetTimestamp(auth, &pb.GetTimestampRequest{Count: 2}) + require.NoError(t, err) + require.Equal(t, barrier, window.GetPhaseDFloor()) + require.Greater(t, window.GetTimestamp(), barrier) + require.True(t, window.GetPhaseDActive()) + return window +} + +func (c *phaseDAcceptanceCluster) resolve(t *testing.T, auth context.Context, start, proof uint64, window *pb.GetTimestampResponse) map[uint64]uint64 { + t.Helper() + wrong := phaseDAcceptanceRequest(pb.Phase_COMMIT, "b-primary", start, proof+1) + _, err := c.nodes[0].groups[1].Txn.Commit(auth, []*pb.Request{wrong}) + require.Error(t, err, "first primary commit must match the durable prepare timestamp") + committed := make(map[uint64]uint64) + for group, key := range map[uint64]string{1: "b-primary", 2: "x-secondary"} { + response, err := c.nodes[0].groups[group].Txn.Commit(auth, []*pb.Request{phaseDAcceptanceRequest(pb.Phase_COMMIT, key, start, proof)}) + require.NoError(t, err) + require.NotZero(t, response.CommitIndex) + committed[group] = response.CommitIndex + } + _, err = c.nodes[0].groups[1].Txn.Commit(auth, []*pb.Request{phaseDAcceptanceRequest(pb.Phase_PREPARE, "b-forged", start, proof)}) + require.Error(t, err, "a new pre-Phase-D prepare must not use an old valid timestamp as fresh evidence") + forged := &pb.Request{Ts: ^uint64(0) - 1, TimestampAdmissionFloor: window.GetPhaseDFloor(), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b-forged-epoch"), Value: []byte("forged")}}} + _, err = c.nodes[0].groups[1].Txn.Commit(auth, []*pb.Request{forged}) + require.Error(t, err, "an incoming epoch cannot authorize an unreserved future timestamp") + fresh := &pb.Request{Ts: window.GetTimestamp(), Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte("b-new-local"), Value: []byte("new-local")}, + }} + _, err = c.nodes[1].groups[1].Txn.Commit(auth, []*pb.Request{fresh}) + require.NoError(t, err, "the leader-local transaction manager must stamp the new epoch too") + require.Zero(t, fresh.GetTimestampAdmissionFloor(), "local admission must clone the caller's request before enqueueing") + voucherProof := window.GetTimestamp() + 1 + voucherPrepare := phaseDAcceptanceRequest(pb.Phase_PREPARE, "b-voucher", start, voucherProof) + voucherPrepare.Mutations[0].Value = kv.EncodeTxnMeta(kv.TxnMeta{ + PrimaryKey: []byte("b-voucher"), PrepareCommitTS: voucherProof, LockTTLms: 60000, + }) + _, err = c.nodes[0].groups[1].Txn.Commit(auth, []*pb.Request{voucherPrepare}) + require.NoError(t, err, "a fresh post-D proof may retain a pre-D applied-read voucher") + voucherCommit := phaseDAcceptanceRequest(pb.Phase_COMMIT, "b-voucher", start, voucherProof) + voucherCommit.Mutations[0].Value = kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("b-voucher"), CommitTS: voucherProof}) + voucherResponse, err := c.nodes[0].groups[1].Txn.Commit(auth, []*pb.Request{voucherCommit}) + require.NoError(t, err) + committed[1] = voucherResponse.CommitIndex + return committed +} + +func (c *phaseDAcceptanceCluster) verifyReplicas(t *testing.T, auth context.Context, start, proof uint64, window *pb.GetTimestampResponse, committed map[uint64]uint64) { + t.Helper() + voucherProof := window.GetTimestamp() + 1 + for _, node := range c.nodes { + require.Eventually(t, func() bool { + for group, key := range map[uint64]string{1: "b-primary", 2: "x-secondary"} { + if node.groups[group].Engine.Status().AppliedIndex < committed[group] { + return false + } + value, readErr := node.groups[group].Store.GetAt(auth, []byte(key), proof) + if readErr != nil || !bytes.Equal(value, []byte("value-"+key)) { + return false + } + } + return true + }, 5*time.Second, 10*time.Millisecond, "committed values must reach %s in both data groups", node.id) + for group, key := range map[uint64]string{1: "b-primary", 2: "x-secondary"} { + _, err := node.groups[group].Store.GetAt(auth, []byte(key), start) + require.ErrorIs(t, err, store.ErrKeyNotFound) + } + _, err := node.groups[1].Store.GetAt(auth, []byte("b-forged"), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound) + _, err = node.groups[1].Store.GetAt(auth, []byte("b-forged-epoch"), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound) + require.Eventually(t, func() bool { + value, readErr := node.groups[1].Store.GetAt(auth, []byte("b-new-local"), window.GetTimestamp()) + if readErr != nil || !bytes.Equal(value, []byte("new-local")) { + return false + } + value, readErr = node.groups[1].Store.GetAt(auth, []byte("b-voucher"), voucherProof) + return readErr == nil && bytes.Equal(value, []byte("value-b-voucher")) + }, 5*time.Second, 10*time.Millisecond) + } +} diff --git a/adapter/internal_prepare_evidence.go b/adapter/internal_prepare_evidence.go new file mode 100644 index 000000000..e3ed1c0e3 --- /dev/null +++ b/adapter/internal_prepare_evidence.go @@ -0,0 +1,93 @@ +package adapter + +import ( + "context" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" +) + +// GetPrepareEvidenceState serves read-only protocol negotiation on every +// replica, and authoritative drain inspection only on an applied leader. +func (i *Internal) GetPrepareEvidenceState(ctx context.Context, req *pb.PrepareEvidenceStateRequest) (*pb.PrepareEvidenceStateResponse, error) { + if req == nil { + return nil, errors.WithStack(kv.ErrInvalidRequest) + } + if req.GetCapabilityOnly() { + return i.prepareEvidenceCapabilities(), nil + } + if i.kvForwardRejected || i.store == nil { + return nil, errors.WithStack(ErrKVForwardNotSupported) + } + if i.leader == nil || i.leader.State() != raftengine.StateLeader { + return nil, errors.WithStack(ErrNotLeader) + } + index, err := i.leader.LinearizableRead(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + enabled, unproven, floor, err := kv.InspectPrepareEvidence(ctx, i.store, req.GetFloor()) + if err != nil { + return nil, errors.WithStack(err) + } + if i.leader.State() != raftengine.StateLeader { + return nil, errors.WithStack(ErrNotLeader) + } + return &pb.PrepareEvidenceStateResponse{ + Supported: true, ProofEnabled: enabled, UnprovenLocks: unproven, + FenceFloor: floor, AppliedIndex: index, + }, nil +} + +func (i *Internal) prepareEvidenceCapabilities() *pb.PrepareEvidenceStateResponse { + return &pb.PrepareEvidenceStateResponse{ + Supported: true, DedicatedTsoConfigured: i.tsoCutoverState != nil, + CutoverApplied: i.tsoCutoverState != nil && i.tsoCutoverState.CutoverActive(), + } +} + +func (i *Internal) validateForwardedPrepareEvidence(ctx context.Context, requests []*pb.Request, startTS uint64) error { + ctx, cancel := context.WithTimeout(ctx, legacyCommitEvidenceTimeout) + defer cancel() + seen := make(map[uint64]struct{}) + for _, request := range requests { + timestamp, err := kv.PrepareRequestCommitTimestamp(request) + if err != nil { + return errors.WithStack(err) + } + if timestamp == 0 { + continue + } + if timestamp <= startTS { + return errors.WithStack(kv.ErrTxnInvalidMeta) + } + if _, exists := seen[timestamp]; exists { + continue + } + if len(seen) == kv.MaxPrimaryCommitEvidenceKeys { + return errors.WithStack(kv.ErrInvalidRequest) + } + if err := kv.ReservePrepareEvidenceTimestamp(ctx, i.tsAllocator, timestamp); err != nil { + return errors.WithStack(err) + } + seen[timestamp] = struct{}{} + } + return nil +} + +func forwardedTxnCreationsHavePrepareProof(requests []*pb.Request) bool { + hasPrepare := false + for _, request := range requests { + if request == nil || request.Phase == pb.Phase_COMMIT || request.Phase == pb.Phase_ABORT { + continue + } + timestamp, err := kv.PrepareRequestCommitTimestamp(request) + if err != nil || timestamp == 0 { + return false + } + hasPrepare = true + } + return hasPrepare +} diff --git a/adapter/internal_prepare_evidence_test.go b/adapter/internal_prepare_evidence_test.go new file mode 100644 index 000000000..326057b0d --- /dev/null +++ b/adapter/internal_prepare_evidence_test.go @@ -0,0 +1,275 @@ +package adapter + +import ( + "bytes" + "context" + "errors" + "fmt" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +type prepareSafetyRPCLeader struct { + forwardObserverLeader + follower atomic.Bool + reads atomic.Int32 + loseRead bool +} + +func (l *prepareSafetyRPCLeader) State() raftengine.State { + if l.follower.Load() { + return raftengine.StateFollower + } + return raftengine.StateLeader +} + +func (l *prepareSafetyRPCLeader) LinearizableRead(context.Context) (uint64, error) { + l.reads.Add(1) + if l.loseRead { + l.follower.Store(true) + } + return 23, nil +} + +func prepareSafetyInternalClient(t *testing.T, server *Internal) pb.InternalClient { + t.Helper() + lis, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + auth, _ := AdminTokenAuth("prepare-safety-token") + srv := grpc.NewServer(grpc.UnaryInterceptor(auth)) + pb.RegisterInternalServer(srv, server) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(func() { srv.Stop(); _ = lis.Close() }) + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + return pb.NewInternalClient(conn) +} + +func TestPrepareEvidenceSafetyRPCRequiresTokenAndLeaderFence(t *testing.T) { + st := store.NewMVCCStore() + fsm := kv.NewKvFSMWithHLC(st, kv.NewHLC()) + require.Nil(t, fsm.Apply(kv.EncodePrepareEvidenceFence(0))) + leader := &prepareSafetyRPCLeader{} + leader.follower.Store(true) + client := prepareSafetyInternalClient(t, NewInternalWithEngine(nil, leader, nil, nil, WithInternalStore(st))) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + for _, token := range []string{"", "wrong"} { + unauthorized := metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+token) + _, err := client.GetPrepareEvidenceState(unauthorized, &pb.PrepareEvidenceStateRequest{CapabilityOnly: true}) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + } + authorized := metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer prepare-safety-token") + capability, err := client.GetPrepareEvidenceState(authorized, &pb.PrepareEvidenceStateRequest{CapabilityOnly: true}) + require.NoError(t, err) + require.True(t, capability.GetSupported(), "a follower must advertise decoder support without a leader read") + require.Zero(t, leader.reads.Load()) + _, err = client.GetPrepareEvidenceState(authorized, &pb.PrepareEvidenceStateRequest{}) + require.Error(t, err, "a follower must not attest that legacy locks drained") + require.Zero(t, leader.reads.Load()) + leader.follower.Store(false) + response, err := client.GetPrepareEvidenceState(authorized, &pb.PrepareEvidenceStateRequest{}) + require.NoError(t, err) + require.True(t, response.GetProofEnabled()) + require.False(t, response.GetUnprovenLocks()) + require.Equal(t, uint64(23), response.GetAppliedIndex()) + require.Equal(t, int32(1), leader.reads.Load()) +} + +func TestPrepareEvidenceSafetyRPCDiscardsLeadershipLostDuringRead(t *testing.T) { + leader := &prepareSafetyRPCLeader{loseRead: true} + client := prepareSafetyInternalClient(t, NewInternalWithEngine(nil, leader, nil, nil, WithInternalStore(store.NewMVCCStore()))) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer prepare-safety-token") + response, err := client.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{}) + require.Error(t, err) + require.Nil(t, response) + require.Equal(t, int32(1), leader.reads.Load()) +} + +type prepareSafetyCutoverState struct{ active atomic.Bool } + +func (s *prepareSafetyCutoverState) CutoverActive() bool { return s.active.Load() } + +func TestPrepareEvidenceSafetyCapabilityReportsLocalCutoverOnFollowers(t *testing.T) { + for _, configured := range []bool{false, true} { + t.Run(fmt.Sprintf("configured-%t", configured), func(t *testing.T) { + leader := &prepareSafetyRPCLeader{} + leader.follower.Store(true) + state := &prepareSafetyCutoverState{} + var options []InternalOption + if configured { + options = append(options, WithInternalTSOCutoverState(state)) + } + client := prepareSafetyInternalClient(t, NewInternalWithEngine(nil, leader, nil, nil, options...)) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer prepare-safety-token") + for _, applied := range []bool{false, true} { + state.active.Store(applied) + response, err := client.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{CapabilityOnly: true}) + require.NoError(t, err) + require.True(t, response.GetSupported()) + require.Equal(t, configured, response.GetDedicatedTsoConfigured()) + require.Equal(t, configured && applied, response.GetCutoverApplied()) + require.Zero(t, leader.reads.Load(), "capability must expose this follower's applied state without a leader round trip") + } + }) + } +} + +type prepareSafetyResolutionAllocator struct { + phaseDForwardAllocator + bound uint64 + validations atomic.Int32 +} + +func (a *prepareSafetyResolutionAllocator) Next(context.Context) (uint64, error) { + return 0, kv.ErrTSOPhaseDActivationPending +} + +func (a *prepareSafetyResolutionAllocator) PhaseDActive() bool { return false } + +func (a *prepareSafetyResolutionAllocator) ValidateDurableTimestamp(context.Context, uint64) error { + return kv.ErrTSOPhaseDInactive +} + +func (a *prepareSafetyResolutionAllocator) ValidateLegacyResolutionTimestamps(_ context.Context, start, commit uint64) error { + a.validations.Add(1) + if start == 0 || commit <= start || (a.bound != 0 && commit > a.bound) { + return kv.ErrTSOTimestampInvalid + } + return nil +} + +func prepareSafetyForward(phase pb.Phase, key string, commit uint64) *pb.ForwardRequest { + return &pb.ForwardRequest{IsTxn: true, Requests: []*pb.Request{{Ts: 50, IsTxn: true, Phase: phase, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("primary"), CommitTS: commit})}, + {Op: pb.Op_PUT, Key: []byte(key)}, + }}}} +} + +func TestPrepareEvidenceSafetyExistingResolutionBeforeAndDuringPending(t *testing.T) { + for _, bound := range []uint64{0, 100} { + for _, phase := range []pb.Phase{pb.Phase_COMMIT, pb.Phase_ABORT} { + t.Run(phase.String()+"-"+map[bool]string{true: "before-pending", false: "pending"}[bound == 0], func(t *testing.T) { + allocator := &prepareSafetyResolutionAllocator{bound: bound} + txn := &forwardObserverTxn{} + i := NewInternalWithEngine(txn, forwardObserverLeader{}, nil, nil, + WithInternalTimestampAllocator(allocator), + WithInternalPrimaryCommitEvidence(primaryCommitEvidenceFunc(func(context.Context, []byte, uint64) (kv.PrimaryCommitEvidence, error) { + return kv.PrimaryCommitEvidence{Readable: true, PrimaryPrepared: true}, nil + }))) + response, err := i.Forward(context.Background(), prepareSafetyForward(phase, "primary", 60)) + require.NoError(t, err) + require.True(t, response.GetSuccess()) + require.Equal(t, uint64(60), response.GetCommitTs()) + require.NotZero(t, allocator.validations.Load(), "legacy resolution needs the authoritative dedicated validator") + }) + } + } +} + +func TestPrepareEvidenceSafetyFirstPrimaryUsesActualDurableTimestamp(t *testing.T) { + for _, commit := range []uint64{60, 61} { + t.Run(map[bool]string{true: "matching", false: "forged"}[commit == 60], func(t *testing.T) { + st := store.NewMVCCStore() + fsm := kv.NewKvFSMWithHLC(st, kv.NewHLC()) + require.Nil(t, fsm.Apply(kv.EncodePrepareEvidenceFence(0))) + request := prepareSafetyForward(pb.Phase_PREPARE, "primary", 0).Requests[0] + request.Mutations[0].Value = kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("primary"), PrepareCommitTS: 60}) + request.Mutations[1].Value = []byte("prepared value") + data, err := proto.Marshal(request) + require.NoError(t, err) + require.Nil(t, fsm.Apply(data)) + routes := distribution.NewEngine() + routes.UpdateRoute(nil, nil, 1) + shards := kv.NewShardStore(routes, map[uint64]*kv.ShardGroup{1: {Store: st}}) + t.Cleanup(func() { require.NoError(t, shards.Close()) }) + txn := &forwardObserverTxn{} + i := NewInternalWithEngine(txn, forwardObserverLeader{}, nil, nil, + WithInternalTimestampAllocator(&phaseDForwardAllocator{floor: 100}), WithInternalPrimaryCommitEvidence(shards)) + response, err := i.Forward(context.Background(), prepareSafetyForward(pb.Phase_COMMIT, "primary", commit)) + if commit != 60 { + require.ErrorIs(t, err, kv.ErrPrePhaseDCommitUnproven) + require.Empty(t, txn.reqs) + return + } + require.NoError(t, err) + require.True(t, response.GetSuccess()) + require.Equal(t, uint64(60), response.GetCommitTs()) + }) + } +} + +func TestPrepareEvidenceSafetyPendingRejectsNewTimestampedWork(t *testing.T) { + allocator := &prepareSafetyResolutionAllocator{bound: 100} + txn := &forwardObserverTxn{} + i := NewInternalWithEngine(txn, forwardObserverLeader{}, nil, nil, WithInternalTimestampAllocator(allocator)) + for _, phase := range []pb.Phase{pb.Phase_NONE, pb.Phase_PREPARE} { + _, err := i.Forward(context.Background(), prepareSafetyForward(phase, "fresh", 60)) + require.True(t, errors.Is(err, kv.ErrTSOPhaseDInactive) || errors.Is(err, kv.ErrTSOPhaseDActivationPending), "%s: %v", phase, err) + } + require.Empty(t, txn.reqs) +} + +type prepareSafetyReservationAllocator struct { + phaseDForwardAllocator + minimum uint64 + fail error +} + +func (*prepareSafetyReservationAllocator) PhaseDActive() bool { return false } +func (*prepareSafetyReservationAllocator) PhaseDRequired() bool { return false } + +func (a *prepareSafetyReservationAllocator) NextAfter(_ context.Context, minimum uint64) (uint64, error) { + a.minimum = minimum + if a.fail != nil { + return 0, a.fail + } + return minimum + 1, nil +} + +func TestPrepareEvidenceSafetyV3IngressReservesProofWithoutRetimestamp(t *testing.T) { + reservationFailure := errors.New("durable reservation failed") + for _, failure := range []error{nil, reservationFailure} { + allocator := &prepareSafetyReservationAllocator{fail: failure} + txn := &forwardObserverTxn{} + i := NewInternalWithEngine(txn, forwardObserverLeader{}, nil, nil, WithInternalTimestampAllocator(allocator)) + request := prepareSafetyForward(pb.Phase_PREPARE, "primary", 0) + meta := kv.TxnMeta{PrimaryKey: []byte("primary"), PrepareCommitTS: 60} + request.Requests[0].Mutations[0].Value = kv.EncodeTxnMeta(meta) + request.Requests[0].Mutations[1].Value = []byte("value already stamped at 60") + before := bytes.Clone(request.Requests[0].Mutations[1].Value) + _, err := i.Forward(context.Background(), request) + require.Equal(t, uint64(60), allocator.minimum, "the chosen proof must be included in the durable reservation range") + if failure != nil { + require.ErrorIs(t, err, reservationFailure) + require.Empty(t, txn.reqs, "reservation failure must precede data proposal") + continue + } + require.NoError(t, err) + require.Len(t, txn.reqs, 1) + after, err := kv.DecodeTxnMeta(txn.reqs[0].Mutations[0].Value) + require.NoError(t, err) + require.Equal(t, uint64(60), after.PrepareCommitTS, "reserving past the proof must not substitute the newly allocated value") + require.Equal(t, before, txn.reqs[0].Mutations[1].Value) + } +} diff --git a/adapter/internal_test.go b/adapter/internal_test.go index f176826f8..8e4868206 100644 --- a/adapter/internal_test.go +++ b/adapter/internal_test.go @@ -244,6 +244,7 @@ func TestStampTxnTimestamps_UsesSingleTxnStartTS(t *testing.T) { Phase: pb.Phase_PREPARE, Ts: 0, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("k")})}, {Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("v")}, }, } @@ -671,7 +672,8 @@ func TestFillForwardedTxnCommitTS_AllowsPrePhaseDResolutionReplay(t *testing.T) t.Parallel() alloc := &phaseDForwardAllocator{floor: 100} - i := NewInternalWithEngine(nil, nil, nil, nil, WithInternalTimestampAllocator(alloc)) + i := NewInternalWithEngine(nil, nil, nil, nil, WithInternalTimestampAllocator(alloc), + WithInternalPrimaryCommitEvidence(recordedPrimaryCommitEvidence(60))) resolution := func(commitTS uint64) []*pb.Request { return []*pb.Request{{ @@ -760,10 +762,13 @@ func TestStampTxnTimestamps_ValidatesForwardedStartTS(t *testing.T) { prepare := func(startTS uint64) []*pb.Request { return []*pb.Request{{ - IsTxn: true, - Phase: pb.Phase_PREPARE, - Ts: startTS, - Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("v")}}, + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("k")})}, + {Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("v")}, + }, }} } @@ -792,7 +797,8 @@ func TestFillForwardedTxnCommitTS_MixedPhaseBatchGetsNoLegacyExemption(t *testin t.Parallel() alloc := &phaseDForwardAllocator{floor: 100} - i := NewInternalWithEngine(nil, nil, nil, nil, WithInternalTimestampAllocator(alloc)) + i := NewInternalWithEngine(nil, nil, nil, nil, WithInternalTimestampAllocator(alloc), + WithInternalPrimaryCommitEvidence(recordedPrimaryCommitEvidence(60))) meta := func(phase pb.Phase, commitTS uint64) *pb.Request { return &pb.Request{ @@ -866,7 +872,8 @@ func TestStampTimestamps_RejectsRawRequestInsideTxnEnvelope(t *testing.T) { t.Parallel() alloc := &phaseDForwardAllocator{floor: 100, ceiling: 200} - i := NewInternalWithEngine(nil, nil, nil, nil, WithInternalTimestampAllocator(alloc)) + i := NewInternalWithEngine(nil, nil, nil, nil, WithInternalTimestampAllocator(alloc), + WithInternalPrimaryCommitEvidence(recordedPrimaryCommitEvidence(60))) resolution := &pb.Request{ IsTxn: true, diff --git a/adapter/internal_timestamp_admission_test.go b/adapter/internal_timestamp_admission_test.go new file mode 100644 index 000000000..848085a9a --- /dev/null +++ b/adapter/internal_timestamp_admission_test.go @@ -0,0 +1,80 @@ +package adapter + +import ( + "strconv" + "testing" + + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func releasedAdmissionStore(t *testing.T, floor uint64) store.MVCCStore { + t.Helper() + st := store.NewMVCCStore() + fsm := kv.NewKvFSMWithHLC(st, kv.NewHLC()) + require.Nil(t, fsm.Apply(kv.EncodePrepareEvidenceFence(floor))) + require.Nil(t, fsm.Apply(kv.EncodePrepareEvidenceSeal())) + require.Nil(t, fsm.Apply(kv.EncodePrepareEvidenceRelease(floor))) + return st +} + +func TestInternalProofPrepareRetainsOldReadSnapshotAfterGraceWindow(t *testing.T) { + t.Parallel() + const afterGraceTS uint64 = (26 * 60 * 60 * 1000) << kv.HLCLogicalBits + for _, commitTS := range []uint64{60, 150} { + t.Run(strconv.FormatUint(commitTS, 10), func(t *testing.T) { + alloc := &phaseDForwardAllocator{floor: 100, ceiling: 200, next: afterGraceTS} + nextBefore := alloc.next + txn := &forwardObserverTxn{} + internal := NewInternalWithEngine(txn, forwardObserverLeader{}, nil, nil, + WithInternalTimestampAllocator(alloc), WithInternalStore(releasedAdmissionStore(t, 100))) + _, err := internal.Forward(t.Context(), &pb.ForwardRequest{IsTxn: true, Requests: []*pb.Request{{ + IsTxn: true, Phase: pb.Phase_PREPARE, Ts: 50, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("primary"), PrepareCommitTS: commitTS})}, + {Op: pb.Op_PUT, Key: []byte("primary"), Value: []byte("value")}, + }, + }}}) + if commitTS < 100 { + require.ErrorIs(t, err, kv.ErrTSOTimestampPrePhaseD) + require.Empty(t, txn.reqs) + } else { + require.NoError(t, err) + require.Len(t, txn.reqs, 1) + require.Equal(t, uint64(50), txn.reqs[0].Ts) + require.Equal(t, uint64(100), txn.reqs[0].GetTimestampAdmissionFloor()) + } + require.Equal(t, nextBefore, alloc.next, "a new proven commit does not need the legacy creation window") + }) + } +} + +type laggingAdmissionAllocator struct{ phaseDForwardAllocator } + +func (*laggingAdmissionAllocator) PhaseDActive() bool { return false } +func (*laggingAdmissionAllocator) PhaseDRequired() bool { return false } + +func TestInternalAdmissionDoesNotTrustStaleModeOrIncomingEpoch(t *testing.T) { + t.Parallel() + for _, timestamp := range []uint64{120, 150} { + t.Run(strconv.FormatUint(timestamp, 10), func(t *testing.T) { + allocator := &laggingAdmissionAllocator{phaseDForwardAllocator{floor: 100, ceiling: 140}} + txn := &forwardObserverTxn{} + internal := NewInternalWithEngine(txn, forwardObserverLeader{}, nil, nil, + WithInternalTimestampAllocator(allocator), WithInternalStore(releasedAdmissionStore(t, 100))) + _, err := internal.Forward(t.Context(), &pb.ForwardRequest{Requests: []*pb.Request{{ + Ts: timestamp, TimestampAdmissionFloor: 100, + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("key"), Value: []byte("value")}}, + }}}) + if timestamp > 140 { + require.ErrorIs(t, err, kv.ErrTSOTimestampInvalid) + require.Empty(t, txn.reqs) + return + } + require.NoError(t, err) + require.Greater(t, allocator.validateCalls, 0) + require.Equal(t, uint64(100), txn.reqs[0].GetTimestampAdmissionFloor()) + }) + } +} diff --git a/distribution/migrator.go b/distribution/migrator.go index e3041ee0b..06acdbe7b 100644 --- a/distribution/migrator.go +++ b/distribution/migrator.go @@ -72,16 +72,19 @@ const ( // Keep this literal in sync with kv.backupTimestampFloorKey. distribution // cannot import kv without a cycle, and only this exact control key is // reserved; user keys under !txn|backup| remain migratable. - migrationTxnBackupTimestampFloorKey = "!txn|backup|timestamp_floor" - migrationRedisPrefix = "!redis|" - migrationHashPrefix = "!hs|" - migrationSetPrefix = "!st|" - migrationZSetPrefix = "!zs|" - migrationDynamoMetaPrefix = "!ddb|meta|table|" - migrationDynamoGenPrefix = "!ddb|meta|gen|" - migrationDynamoItemPrefix = "!ddb|item|" - migrationDynamoGSIPrefix = "!ddb|gsi|" - migrationStagedDataPrefix = "!dist|migstage|" + migrationTxnBackupTimestampFloorKey = "!txn|backup|timestamp_floor" + migrationTxnPrepareEvidenceEnabledKey = "!txn|backup|prepare_evidence_enabled" + migrationTxnPrepareEvidenceGateKey = "!txn|backup|prepare_evidence_gate" + migrationTxnPrepareEvidenceFloorKey = "!txn|backup|prepare_evidence_timestamp_floor" + migrationRedisPrefix = "!redis|" + migrationHashPrefix = "!hs|" + migrationSetPrefix = "!st|" + migrationZSetPrefix = "!zs|" + migrationDynamoMetaPrefix = "!ddb|meta|table|" + migrationDynamoGenPrefix = "!ddb|meta|gen|" + migrationDynamoItemPrefix = "!ddb|item|" + migrationDynamoGSIPrefix = "!ddb|gsi|" + migrationStagedDataPrefix = "!dist|migstage|" ) const ( @@ -238,6 +241,9 @@ var migrationInternalFamilyPrefixes = [][]byte{ var migrationInternalExactKeys = [][]byte{ []byte(migrationTxnBackupTimestampFloorKey), + []byte(migrationTxnPrepareEvidenceEnabledKey), + []byte(migrationTxnPrepareEvidenceGateKey), + []byte(migrationTxnPrepareEvidenceFloorKey), } // MigrationBracket is a raw MVCC export or drain slice used by the migrator. @@ -402,6 +408,11 @@ func MigrationStagedDataKeyPrefix(jobID uint64) []byte { return MigrationStagedDataKey(jobID, nil) } +// MigrationStagedDataScanPrefix covers staged rows from every migration job. +func MigrationStagedDataScanPrefix() []byte { + return []byte(migrationStagedDataPrefix) +} + func IsMigrationStagedDataKey(key []byte) bool { return len(key) >= len(migrationStagedDataPrefix)+migrationStagedDataJobIDBytes+1 && bytes.HasPrefix(key, []byte(migrationStagedDataPrefix)) && diff --git a/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md b/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md index f05b84c15..c32ea6200 100644 --- a/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md +++ b/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md @@ -194,11 +194,11 @@ An unprepared primary, secondary resolution without a decision, or a mismatched existing decision is still rejected. The FSM retains its existing commit versus rollback and intent checks. -A prepared primary proves an in-flight transaction, not issuance of its claimed +A prepared primary alone proves an in-flight transaction, not issuance of its claimed commitTS. An untrusted caller can still fabricate a fresh primary PREPARE and first COMMIT inside the bounded creation window. Closing that remaining gap -needs a durable chosen-commit proof or a revised admission protocol; this PR -does not claim the full retroactive-write carve-out is closed. In particular, +needs a durable chosen-commit proof or a revised admission protocol. The +extension accepted below replaces that incomplete first-primary exception. In particular, silently allocating a replacement timestamp after PREPARE is not a safe shortcut: prepared values may already embed the original commit timestamp. @@ -210,6 +210,107 @@ waiting for a committed decision's read index to apply. Full distributed cutover leader-change acceptance have not been demonstrated by these focused tests; the document remains Partial until that acceptance evidence is available. +## 4b. Durable PREPARE evidence and activation gate (2026-10-23) + +The accepted extension persists the selected commit timestamp with PREPARE +and waits for older, unproven PREPAREs to resolve before activating the stronger +protocol. It does not retimestamp an already prepared value or force an +otherwise resolvable transaction to abort merely to finish the rollout. + +### Evidence Format and Rollout + +`TxnMeta.PrepareCommitTS` uses a new V3 metadata format. V1 and V2 encodings +remain unchanged; in particular, a historical V1 PREPARE's `CommitTS` is not +retroactively reinterpreted as evidence when replaying its log. The chosen +timestamp is saved atomically with each primary or secondary lock and intent. +An idempotent PREPARE cannot add, remove, or change this proof, and a COMMIT +cannot substitute a different timestamp. + +Before V3 can mutate a data FSM, an authenticated protocol negotiation checks +every voter and learner in every configured group, including group zero. The +check also requires each existing member to have applied dedicated-TSO cutover; +an outdated mode file cannot justify continued HLC issuance after that marker. +Membership admission checks the same protocol before adding another member. +A fresh learner may join before applying cutover, but a cutover cluster requires +that it configure the dedicated timestamp runtime. Its catch-up must precede +public serving. Old binaries cannot be admitted after this protocol is required. + +The replicated enable marker then requires proof for every new PREPARE in its +data group. An existing unproven lock may retry with its original identity and +may COMMIT or ABORT, but a retry cannot manufacture proof retroactively. +Primary and secondary unproven locks, including staged migration locks, must +all drain. Checking only primaries would not justify the remote-primary unknown +verdict retained by the compatibility classifier. New migration imports cannot +introduce unproven locks after the enable marker. + +Before creating proof, both the local coordinator and forwarded ingress ensure +that the chosen timestamp is covered by the dedicated TSO's durable reservation +range. Before Phase D, reserving and discarding a timestamp above the chosen +value can extend that range without modifying the already stamped values. +After Phase D, ordinary strict timestamp validation applies. This is not a claim +that every numeric slot in a reserved batch was individually issued; the separate +slot-claim design remains responsible for that stronger property. + +### Activation Order + +1. Commit cutover, negotiate support and applied cutover with all members, and + enable required PREPARE evidence on all data groups. +2. Wait for all old unproven locks to resolve. Return a retryable pending result + between attempts instead of holding the allocator mutex while waiting for + COMMIT or ABORT traffic that needs it. +3. Replicate a creation seal on every data group. It rejects raw, one-phase, and + PREPARE creation at FSM apply, while existing COMMIT and ABORT remain usable. + This closes the late-apply race: an old raw timestamp greater than a sampled + floor must not land between that sample and activation. +4. Read fresh authoritative committed floors and compute the final boundary B. + Persist a pending-D marker with that fixed boundary in group zero. Retries, + snapshots, restarts, and leadership changes preserve B. Ordinary timestamp + issuance, including cached batch slots, stops while this marker is pending. +5. Apply the permanent timestamp floor B to every data group, verify the drain + through applied leader fences, and commit Phase D at B in group zero. +6. Replicate the idempotent creation release on every data group, retaining the + permanent floor. Only then return new allocations. Release failures must be + retried even if the group-zero Phase-D marker is already active. + +After release, every creation request also carries the released admission floor +as its epoch. Both forwarded ingress and the local transaction-manager queue +discard any incoming epoch, strictly validate the unchanged persistence timestamp +against group zero, and then stamp the current epoch. The FSM rejects an old +epoch even when its timestamp exceeds B. A temporary seal alone would not stop +a request validated before cutover but delayed until after release. Requests +queued before the seal must retain their original epoch, not observe a later +retry's mutation of a shared request object. + +The cutover timestamp floor is separate from the backup-pin floor. Raw and +one-phase creation must persist above B. A new proof-bearing PREPARE may keep +an older applied-read snapshot as its startTS, but its chosen commit timestamp +must validate above B and its admission epoch must match. Existing applied-read +vouchers remain usable even after the old creation-grace window expires. A real +backup pin retains its original startTS fence; cutover does not weaken it. + +The enable, seal, and release records belong to the local group, not its user +ranges. User mutations cannot forge them, and range migration does not export +or promote them as application data. The seal is a deterministic refusal, not +an operational error that halts a raw-batch replica. + +### Already Active Clusters + +Snapshots from an older implementation can already have Phase D active but no +PREPARE-proof protocol marker. Such a cluster must bootstrap the stronger +protocol without changing its existing Phase-D floor or stranding old locks. +An authoritative primary may keep the old first-COMMIT exception only until its +creation seal or release is durable. Required-proof admission has already stopped +new unproven PREPAREs before that drain finishes. Once sealed or released, a +first primary COMMIT requires the durable chosen timestamp to match exactly. + +### Acceptance Status + +The implementation and focused regression tests are being integrated in the +existing PR. The document remains Partial until end-to-end forwarded cutover, +leader changes, pending restart/snapshot recovery, release retry, and late legacy +creation are verified together. Code presence or a classifier-only test is not +completion evidence for this protocol. + ## 5. Open questions for review 1. Is the acceptable answer §4.2 alone (cheap, no cross-group read, bounded by a @@ -223,7 +324,7 @@ the document remains Partial until that acceptance evidence is available. 4. Should the `ABORT` path get its own rule, given its timestamp is synthesised rather than recorded? -Settled in §4a, subject to the first-primary proof gap above. Remaining work -must close that gap without rejecting valid cutover transactions. Acceptance must exercise +Settled in §4a and extended by §4b. Remaining acceptance work must demonstrate +closure without rejecting valid cutover transactions. Acceptance must exercise the real forwarded path across Phase-D activation and leader changes, including a remote-primary resolution after the creation window has closed. diff --git a/kv/backup_codec.go b/kv/backup_codec.go index ad0edb016..798b002e7 100644 --- a/kv/backup_codec.go +++ b/kv/backup_codec.go @@ -11,11 +11,14 @@ import ( const ( raftEncodeBackup byte = 0x0e - backupSubtypePin byte = 0x01 - backupSubtypeExtend byte = 0x02 - backupSubtypeRelease byte = 0x03 - backupSubtypeReserve byte = 0x04 - backupSubtypeUnreserve byte = 0x05 + backupSubtypePin byte = 0x01 + backupSubtypeExtend byte = 0x02 + backupSubtypeRelease byte = 0x03 + backupSubtypeReserve byte = 0x04 + backupSubtypeUnreserve byte = 0x05 + backupSubtypePrepareEvidence byte = 0x06 + backupSubtypePrepareSeal byte = 0x07 + backupSubtypePrepareRelease byte = 0x08 backupPinIDBytes = 16 backupUint64Size = 8 @@ -63,10 +66,29 @@ type BackupReserveEntry = BackupPinEntry type BackupUnreserveEntry = BackupReleaseEntry type backupEntry struct { - subtype byte - pin BackupPinEntry - extend BackupExtendEntry - release BackupReleaseEntry + subtype byte + pin BackupPinEntry + extend BackupExtendEntry + release BackupReleaseEntry + prepareFloor uint64 +} + +// EncodePrepareEvidenceFence enables durable PREPARE evidence after protocol +// negotiation. A nonzero floor also seals all new writes at or below the cut. +func EncodePrepareEvidenceFence(floor uint64) []byte { + out := []byte{raftEncodeBackup, backupSubtypePrepareEvidence} + return binary.BigEndian.AppendUint64(out, floor) +} + +// EncodePrepareEvidenceSeal stops new write creation at the data FSM while +// retaining existing transaction resolution during the final cutover fence. +func EncodePrepareEvidenceSeal() []byte { + return []byte{raftEncodeBackup, backupSubtypePrepareSeal} +} + +// EncodePrepareEvidenceRelease reopens creation strictly above the Phase-D floor. +func EncodePrepareEvidenceRelease(floor uint64) []byte { + return binary.BigEndian.AppendUint64([]byte{raftEncodeBackup, backupSubtypePrepareRelease}, floor) } func EncodeBackupPinEntry(entry BackupPinEntry) []byte { @@ -120,6 +142,8 @@ func decodeBackupPayload(data []byte) (backupEntry, error) { return backupEntry{}, errors.WithStack(ErrBackupWireMalformed) } switch data[0] { + case backupSubtypePrepareEvidence, backupSubtypePrepareSeal, backupSubtypePrepareRelease: + return decodePrepareEvidenceFencePayload(data) case backupSubtypePin, backupSubtypeReserve: if len(data) != backupPinEntryLen-1 { return backupEntry{}, errors.WithStack(ErrBackupWireMalformed) @@ -164,6 +188,19 @@ func decodeBackupPayload(data []byte) (backupEntry, error) { } } +func decodePrepareEvidenceFencePayload(data []byte) (backupEntry, error) { + if data[0] == backupSubtypePrepareSeal { + if len(data) != 1 { + return backupEntry{}, errors.WithStack(ErrBackupWireMalformed) + } + return backupEntry{subtype: data[0]}, nil + } + if len(data) != 1+backupUint64Size { + return backupEntry{}, errors.WithStack(ErrBackupWireMalformed) + } + return backupEntry{subtype: data[0], prepareFloor: binary.BigEndian.Uint64(data[1:])}, nil +} + func backupDeadlineMillis(deadline time.Time) uint64 { ms := deadline.UnixMilli() if ms <= 0 { diff --git a/kv/coordinator_prepare_evidence.go b/kv/coordinator_prepare_evidence.go new file mode 100644 index 000000000..7b397335e --- /dev/null +++ b/kv/coordinator_prepare_evidence.go @@ -0,0 +1,87 @@ +package kv + +import ( + "context" + + pb "github.com/bootjp/elastickv/proto" +) + +func (c *ShardedCoordinator) prepareEvidenceMetas(ctx context.Context, primaryKey []byte, commitTS uint64, ids []uint64) (map[uint64]*pb.Mutation, error) { + metas := make(map[uint64]*pb.Mutation, len(ids)) + for _, id := range ids { + meta, err := c.prepareEvidenceMeta(ctx, primaryKey, commitTS, []uint64{id}) + if err != nil { + return nil, err + } + metas[id] = meta + } + return metas, nil +} + +func (c *ShardedCoordinator) prepareEvidenceMeta(ctx context.Context, primaryKey []byte, commitTS uint64, groupIDs []uint64) (*pb.Mutation, error) { + meta := TxnMeta{PrimaryKey: primaryKey, LockTTLms: defaultTxnLockTTLms, PrepareCommitTS: commitTS} + for _, id := range groupIDs { + group, err := c.txnGroupForID(id) + if err != nil { + return nil, err + } + if group.Store == nil { + meta.PrepareCommitTS = 0 + break + } + enabled, err := PrepareEvidenceEnabled(ctx, group.Store) + if err != nil { + return nil, err + } + if !enabled { + meta.PrepareCommitTS = 0 + break + } + } + if meta.PrepareCommitTS != 0 { + if err := reservePrepareEvidenceTimestamp(ctx, c.tsAllocator, commitTS); err != nil { + return nil, err + } + } + return &pb.Mutation{Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(meta)}, nil +} + +// Reservation advances the durable TSO range without changing values already +// stamped by the coordinator. It is needed before Phase D too: a proof must +// not bind a future timestamp beyond the activation barrier's allocation floor. +func reservePrepareEvidenceTimestamp(ctx context.Context, allocator TimestampAllocator, timestamp uint64) error { + if allocator == nil { + return ErrTSOAllocatorRequired + } + if _, ok := allocator.(DurableTimestampValidator); !ok { + return ErrTSOProtocolUnsupported + } + _, _, required, err := phaseDTimestampValidator(allocator) + if err != nil { + return err + } + if required { + return ValidateDurablePersistenceTimestamp(ctx, allocator, timestamp, "validate prepared commit timestamp") + } + _, err = nextTimestampAfterFromAllocator(ctx, allocator, timestamp, "reserve prepared commit timestamp bound") + return err +} + +// ReservePrepareEvidenceTimestamp protects the internal PREPARE ingress using +// the same durable reservation contract as the local coordinator. +func ReservePrepareEvidenceTimestamp(ctx context.Context, allocator TimestampAllocator, timestamp uint64) error { + return reservePrepareEvidenceTimestamp(ctx, allocator, timestamp) +} + +func (c *ShardedCoordinator) bindTxnLogPrepareEvidence(ctx context.Context, logs []*pb.Request, commitTS uint64, ids []uint64) ([]*pb.Request, error) { + meta, _, err := extractTxnMeta(logs[0].Mutations) + if err != nil { + return nil, err + } + mutation, err := c.prepareEvidenceMeta(ctx, meta.PrimaryKey, commitTS, ids) + if err != nil { + return nil, err + } + logs[0].Mutations[0] = mutation + return logs, nil +} diff --git a/kv/fsm.go b/kv/fsm.go index 15eeaffc2..42253d97c 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -110,8 +110,12 @@ type kvFSM struct { // backupTimestampFloor is the highest replicated BackupPin read timestamp. // New raw, one-phase, and PREPARE entries at or below this floor were // timestamped before the backup cut and reached apply too late. - backupTimestampFloor atomic.Uint64 - backupFloorLoadErr error + backupTimestampFloor atomic.Uint64 + backupFloorLoadErr error + prepareEvidenceEnabled atomic.Bool + prepareCreationSealed atomic.Bool + prepareEvidenceReleaseFloor atomic.Uint64 + prepareEvidenceTimestampFloor atomic.Uint64 } // RouteHistory is the kv-side interface to the route catalog's @@ -450,6 +454,7 @@ func isRawApplyVerdict(err error) bool { return errors.Is(err, ErrInvalidRequest) || errors.Is(err, ErrUnknownRequestType) || errors.Is(err, ErrBackupTimestampFenced) || + errors.Is(err, ErrPrepareCreationSealed) || errors.Is(err, ErrRouteWriteFenced) || errors.Is(err, ErrRouteWriteTimestampTooLow) || errors.Is(err, ErrComposed1VersionGCd) || @@ -622,6 +627,9 @@ func (f *kvFSM) applyRequest(ctx context.Context, r *pb.Request) any { } func (f *kvFSM) applyRequestErr(ctx context.Context, r *pb.Request) error { + if err := f.verifyPrepareCreationGate(r); err != nil { + return err + } commitTS, err := requestCommitTS(r) if err != nil { return err @@ -1452,6 +1460,9 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { } startTS := r.Ts + if err := f.validatePrepareEvidenceMeta(meta, startTS); err != nil { + return err + } uniq, err := f.uniqueMutationsAboveFloorForRequest(r, muts, startTS) if err != nil { return err @@ -1462,7 +1473,7 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { expireAt := txnLockExpireAt(meta.LockTTLms) - storeMuts, err := f.buildPrepareStoreMutations(ctx, uniq, meta.PrimaryKey, startTS, expireAt) + storeMuts, err := f.buildPrepareStoreMutations(ctx, uniq, meta.PrimaryKey, startTS, expireAt, meta.PrepareCommitTS) if err != nil { return err } @@ -1855,10 +1866,10 @@ func (f *kvFSM) handleAbortRequest(ctx context.Context, r *pb.Request, abortTS u return errors.WithStack(f.store.ApplyMutationsRaftAt(ctx, storeMuts, nil, startTS, abortTS, f.pendingApplyIdx)) } -func (f *kvFSM) buildPrepareStoreMutations(ctx context.Context, muts []*pb.Mutation, primaryKey []byte, startTS, expireAt uint64) ([]*store.KVPairMutation, error) { +func (f *kvFSM) buildPrepareStoreMutations(ctx context.Context, muts []*pb.Mutation, primaryKey []byte, startTS, expireAt, commitTS uint64) ([]*store.KVPairMutation, error) { storeMuts := make([]*store.KVPairMutation, 0, len(muts)*txnPrepareStoreMutationFactor) for _, mut := range muts { - preparedMuts, err := f.prepareTxnMutation(ctx, mut, primaryKey, startTS, expireAt) + preparedMuts, err := f.prepareTxnMutation(ctx, mut, primaryKey, startTS, expireAt, commitTS) if err != nil { return nil, err } @@ -1893,7 +1904,7 @@ func (f *kvFSM) buildCommitStoreMutations(ctx context.Context, muts []*pb.Mutati committingPrimary = true } - keyMuts, err := f.commitTxnKeyMutations(ctx, key, meta.PrimaryKey, startTS, readKeys) + keyMuts, err := f.commitTxnKeyMutations(ctx, key, meta.PrimaryKey, startTS, commitTS, readKeys) if err != nil { return nil, err } @@ -2031,13 +2042,17 @@ func (f *kvFSM) txnCommitTSAtKey(ctx context.Context, key []byte) (uint64, bool, return commitTS, true, nil } -func (f *kvFSM) prepareTxnMutation(ctx context.Context, mut *pb.Mutation, primaryKey []byte, startTS, expireAt uint64) ([]*store.KVPairMutation, error) { - if err := f.assertNoConflictingTxnLock(ctx, mut.Key, primaryKey, startTS); err != nil { +func (f *kvFSM) prepareTxnMutation(ctx context.Context, mut *pb.Mutation, primaryKey []byte, startTS, expireAt, commitTS uint64) ([]*store.KVPairMutation, error) { + if isTxnInternalKey(mut.Key) || distribution.IsMigrationOnlyControlKey(mut.Key) { + return nil, errors.WithStack(ErrInvalidRequest) + } + if err := f.validatePrepareLock(ctx, mut.Key, primaryKey, startTS, commitTS); err != nil { return nil, err } lockVal := encodeTxnLock(txnLock{ StartTS: startTS, + CommitTS: commitTS, TTLExpireAt: expireAt, PrimaryKey: primaryKey, IsPrimaryKey: bytes.Equal(mut.Key, primaryKey), @@ -2139,7 +2154,7 @@ func storeMutationForIntent(key []byte, intent txnIntent) (*store.KVPairMutation } } -func (f *kvFSM) commitTxnKeyMutations(ctx context.Context, key, primaryKey []byte, startTS uint64, readKeys [][]byte) ([]*store.KVPairMutation, error) { +func (f *kvFSM) commitTxnKeyMutations(ctx context.Context, key, primaryKey []byte, startTS, commitTS uint64, readKeys [][]byte) ([]*store.KVPairMutation, error) { lock, lockKey, intentKey, ok, err := f.txnLockForMutationKey(ctx, key, readKeys) if err != nil { return nil, err @@ -2148,11 +2163,8 @@ func (f *kvFSM) commitTxnKeyMutations(ctx context.Context, key, primaryKey []byt // Already resolved (committed/rolled back). return nil, nil } - if lock.StartTS != startTS { - return nil, NewTxnLockedError(key) - } - if !bytes.Equal(lock.PrimaryKey, primaryKey) { - return nil, errors.Wrapf(ErrTxnInvalidMeta, "lock primary_key mismatch for key %s", string(key)) + if err := validatePreparedCommitLock(lock, key, primaryKey, startTS, commitTS); err != nil { + return nil, err } intent, ok, err := f.txnIntentForCommit(ctx, intentKey) @@ -2175,6 +2187,22 @@ func (f *kvFSM) commitTxnKeyMutations(ctx context.Context, key, primaryKey []byt return out, nil } +func validatePreparedCommitLock(lock txnLock, key, primaryKey []byte, startTS, commitTS uint64) error { + if isTxnInternalKey(key) || distribution.IsMigrationOnlyControlKey(key) { + return errors.WithStack(ErrInvalidRequest) + } + if lock.StartTS != startTS { + return NewTxnLockedError(key) + } + if !bytes.Equal(lock.PrimaryKey, primaryKey) { + return errors.Wrapf(ErrTxnInvalidMeta, "lock primary_key mismatch for key %s", string(key)) + } + if lock.CommitTS != 0 && lock.CommitTS != commitTS { + return errors.Wrap(ErrTxnInvalidMeta, "commit timestamp does not match durable prepare evidence") + } + return nil +} + // abortCleanupMutationsForKey reports whether this abort request must emit // cleanup (lock+intent Delete) mutations for key. It returns nil // when the lock is already missing: lock/intent are always written diff --git a/kv/fsm_backup.go b/kv/fsm_backup.go index 480350fac..212c7ceac 100644 --- a/kv/fsm_backup.go +++ b/kv/fsm_backup.go @@ -29,6 +29,16 @@ var backupTimestampFloorKey = []byte(txnBackupPrefix + "timestamp_floor") const backupTimestampFloorValueSize = 8 func (f *kvFSM) applyBackup(data []byte) any { + if len(data) != 0 && isPrepareEvidenceControlSubtype(data[0]) { + entry, err := decodeBackupPayload(data) + if err == nil { + err = f.applyPrepareEvidenceControl(entry) + } + if err != nil { + return haltErr(errors.Wrap(err, "apply prepare evidence fence")) + } + return nil + } if f.readTracker == nil { return haltErr(errors.Wrap(ErrBackupApply, "kv/fsm: backup entry arrived but no ActiveTimestampTracker is wired")) } @@ -96,6 +106,9 @@ func (f *kvFSM) reloadBackupTimestampFloor(ctx context.Context) error { if f == nil || f.store == nil { return nil } + if err := f.reloadPrepareEvidence(ctx); err != nil { + return err + } value, err := f.store.GetAt(ctx, backupTimestampFloorKey, ^uint64(0)) if err != nil { if errors.Is(err, store.ErrKeyNotFound) { diff --git a/kv/fsm_migration_import.go b/kv/fsm_migration_import.go index 43dfb285d..870a6110a 100644 --- a/kv/fsm_migration_import.go +++ b/kv/fsm_migration_import.go @@ -36,6 +36,9 @@ func (f *kvFSM) applyMigrationImport(ctx context.Context, data []byte) any { if err := proto.Unmarshal(data, req); err != nil { return haltErr(errors.Wrap(errors.Mark(err, ErrMigrationImportApply), "kv/fsm: decode migration import")) } + if err := f.validateImportedPrepareEvidence(req.GetVersions()); err != nil { + return err + } staged := migrationStoreVersionsFromProto(req.GetJobId(), req.GetVersions()) if err := validateStagedKeySizes(staged); err != nil { // Ordinary, not halting: the batch is rejected and the migrator can @@ -66,6 +69,28 @@ func (f *kvFSM) applyMigrationImport(ctx context.Context, data []byte) any { return result } +func (f *kvFSM) validateImportedPrepareEvidence(versions []*pb.MVCCVersion) error { + for _, version := range versions { + if version == nil { + continue + } + if isTxnPeerLocalControlKey(version.GetKey()) { + return errors.Wrap(ErrInvalidRequest, "migration cannot import peer-local timestamp protocol state") + } + if !f.prepareEvidenceEnabled.Load() || version.GetTombstone() { + continue + } + blocked, err := unprovenPrepareEvidenceRow(version.GetKey(), version.GetValue(), ^uint64(0)) + if err != nil { + return err + } + if blocked { + return errors.Wrap(ErrBackupTimestampFenced, "migration cannot introduce an unproven legacy prepare below the activation fence") + } + } + return nil +} + // isMigrationImportOrdinaryApplyError reports whether err is a verdict on the // request bytes rather than a failure of this replica's store. Only those may // be returned as an ordinary apply error: the engine advances setApplied past diff --git a/kv/fsm_prepare_creation_gate.go b/kv/fsm_prepare_creation_gate.go new file mode 100644 index 000000000..38c72f3ee --- /dev/null +++ b/kv/fsm_prepare_creation_gate.go @@ -0,0 +1,133 @@ +package kv + +import ( + "context" + "encoding/binary" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +var ErrPrepareCreationSealed = errors.New("phase-D cutover temporarily seals new write creation") +var ErrPrepareAdmissionFenced = errors.Wrap(ErrInvalidRequest, "write does not carry current phase-D timestamp admission") + +const ( + prepareCreationSealed byte = 1 + prepareCreationReleased byte = 2 + prepareCreationGateValueSize = 9 +) + +func isPrepareEvidenceControlSubtype(subtype byte) bool { + return subtype >= backupSubtypePrepareEvidence && subtype <= backupSubtypePrepareRelease +} + +func (f *kvFSM) applyPrepareEvidenceControl(entry backupEntry) error { + switch entry.subtype { + case backupSubtypePrepareEvidence: + return f.applyPrepareEvidenceFence(entry.prepareFloor) + case backupSubtypePrepareSeal: + return f.applyPrepareCreationSeal() + case backupSubtypePrepareRelease: + return f.applyPrepareCreationRelease(entry.prepareFloor) + default: + return ErrBackupWireSubtype + } +} + +func (f *kvFSM) verifyPrepareCreationGate(request *pb.Request) error { + if request != nil && !timestampAdmissionCreation(request) { + return nil + } + if f.prepareCreationSealed.Load() { + return errors.WithStack(ErrPrepareCreationSealed) + } + floor := f.prepareEvidenceReleaseFloor.Load() + if floor == 0 { + return nil + } + if request == nil || request.GetTimestampAdmissionFloor() != floor { + return errors.WithStack(ErrPrepareAdmissionFenced) + } + timestamp, err := prepareCreationPersistenceTimestamp(request) + if err != nil { + return err + } + if timestamp <= floor { + return errors.WithStack(ErrPrepareAdmissionFenced) + } + return nil +} + +func prepareCreationPersistenceTimestamp(request *pb.Request) (uint64, error) { + if request.IsTxn && request.Phase == pb.Phase_PREPARE { + meta, _, err := extractTxnMeta(request.Mutations) + return meta.PrepareCommitTS, err + } + return requestCommitTS(request) +} + +func readPrepareEvidenceCreationGate(ctx context.Context, st store.MVCCStore) (bool, uint64, error) { + value, err := st.GetAt(ctx, prepareEvidenceGateKey, ^uint64(0)) + if errors.Is(err, store.ErrKeyNotFound) { + return false, 0, nil + } + if err != nil { + return false, 0, errors.WithStack(err) + } + if len(value) != prepareCreationGateValueSize { + return false, 0, errors.Wrap(ErrTxnInvalidMeta, "invalid prepare creation gate") + } + floor := binary.BigEndian.Uint64(value[1:]) + switch { + case value[0] == prepareCreationSealed && floor == 0: + return true, 0, nil + case value[0] == prepareCreationReleased && floor != 0: + return false, floor, nil + default: + return false, 0, errors.Wrap(ErrTxnInvalidMeta, "invalid prepare creation phase or floor") + } +} + +func (f *kvFSM) persistPrepareCreationGate(phase byte, floor uint64) error { + stamp := max(uint64(1), f.store.LastCommitTS(), floor) + value := binary.BigEndian.AppendUint64([]byte{phase}, floor) + return errors.WithStack(f.store.ApplyMutationsRaftAt(context.Background(), []*store.KVPairMutation{{ + Op: store.OpTypePut, Key: prepareEvidenceGateKey, Value: value, + }}, nil, stamp, stamp, f.pendingApplyIdx)) +} + +func (f *kvFSM) applyPrepareCreationSeal() error { + if !f.prepareEvidenceEnabled.Load() { + return errors.Wrap(ErrTSOProtocolUnsupported, "prepare protocol must be enabled before sealing") + } + if f.prepareCreationSealed.Load() || f.prepareEvidenceReleaseFloor.Load() != 0 { + return nil + } + if err := f.persistPrepareCreationGate(prepareCreationSealed, 0); err != nil { + return err + } + f.prepareCreationSealed.Store(true) + return nil +} + +func (f *kvFSM) applyPrepareCreationRelease(floor uint64) error { + if floor == 0 || f.prepareEvidenceTimestampFloor.Load() < floor || !f.prepareEvidenceEnabled.Load() { + return errors.Wrap(ErrTxnInvalidMeta, "prepare creation release requires its durable timestamp fence") + } + if old := f.prepareEvidenceReleaseFloor.Load(); old != 0 { + if old == floor { + return nil + } + return errors.Wrap(ErrTxnInvalidMeta, "prepare creation release floor cannot change") + } + if !f.prepareCreationSealed.Load() { + return errors.Wrap(ErrTxnInvalidMeta, "prepare creation must be sealed before release") + } + if err := f.persistPrepareCreationGate(prepareCreationReleased, floor); err != nil { + return err + } + f.prepareEvidenceReleaseFloor.Store(floor) + f.prepareCreationSealed.Store(false) + return nil +} diff --git a/kv/fsm_prepare_evidence.go b/kv/fsm_prepare_evidence.go new file mode 100644 index 000000000..d1fff1a82 --- /dev/null +++ b/kv/fsm_prepare_evidence.go @@ -0,0 +1,114 @@ +package kv + +import ( + "bytes" + "context" + "encoding/binary" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +var prepareEvidenceEnabledKey = []byte(txnBackupPrefix + "prepare_evidence_enabled") +var prepareEvidenceEnabledValue = []byte{1} +var prepareEvidenceGateKey = []byte(txnBackupPrefix + "prepare_evidence_gate") +var prepareEvidenceTimestampFloorKey = []byte(txnBackupPrefix + "prepare_evidence_timestamp_floor") + +func (f *kvFSM) validatePrepareEvidenceMeta(meta TxnMeta, startTS uint64) error { + if meta.PrepareCommitTS == 0 { + return nil + } + if !f.prepareEvidenceEnabled.Load() { + return errors.Wrap(ErrTSOProtocolUnsupported, "durable prepare evidence is not enabled") + } + if meta.PrepareCommitTS <= startTS { + return errors.Wrap(ErrTxnInvalidMeta, "prepare commit timestamp must follow start timestamp") + } + return nil +} + +func (f *kvFSM) validatePrepareLock(ctx context.Context, key, primaryKey []byte, startTS, commitTS uint64) error { + lock, exists, err := f.txnLockAtKey(ctx, txnLockKey(key)) + if err != nil { + return err + } + if exists && (lock.StartTS != startTS || !bytes.Equal(lock.PrimaryKey, primaryKey)) { + return NewTxnLockedError(key) + } + if exists && lock.CommitTS != commitTS { + return errors.Wrap(ErrTxnInvalidMeta, "prepare cannot change durable commit timestamp evidence") + } + if f.prepareEvidenceEnabled.Load() && commitTS == 0 && !exists { + return errors.Wrap(ErrTSOProtocolUnsupported, "new prepare requires durable commit timestamp evidence") + } + return nil +} + +// PrepareRequestCommitTimestamp extracts only the durable proof timestamp of +// a PREPARE, leaving resolution and ordinary request metadata untouched. +func PrepareRequestCommitTimestamp(request *pb.Request) (uint64, error) { + if request == nil || !request.IsTxn || request.Phase != pb.Phase_PREPARE || len(request.Mutations) == 0 { + return 0, nil + } + meta, _, err := extractTxnMeta(request.Mutations) + return meta.PrepareCommitTS, err +} + +// PrepareEvidenceEnabled is monotonic for an upgraded data group's store. +func PrepareEvidenceEnabled(ctx context.Context, st store.MVCCStore) (bool, error) { + value, err := st.GetAt(ctx, prepareEvidenceEnabledKey, ^uint64(0)) + if errors.Is(err, store.ErrKeyNotFound) { + return false, nil + } + if err != nil { + return false, errors.WithStack(err) + } + if !bytes.Equal(value, prepareEvidenceEnabledValue) { + return false, errors.Wrap(ErrTxnInvalidMeta, "invalid prepare evidence marker") + } + return true, nil +} + +func (f *kvFSM) reloadPrepareEvidence(ctx context.Context) error { + enabled, err := PrepareEvidenceEnabled(ctx, f.store) + if err != nil { + return err + } + f.prepareEvidenceEnabled.Store(enabled) + sealed, released, err := readPrepareEvidenceCreationGate(ctx, f.store) + if err != nil { + return err + } + f.prepareCreationSealed.Store(sealed) + f.prepareEvidenceReleaseFloor.Store(released) + floor, err := readPrepareEvidenceFenceFloor(ctx, f.store) + if err != nil { + return err + } + f.prepareEvidenceTimestampFloor.Store(floor) + return nil +} + +func (f *kvFSM) applyPrepareEvidenceFence(floor uint64) error { + if released := f.prepareEvidenceReleaseFloor.Load(); released != 0 && floor != 0 && floor != released { + return errors.Wrap(ErrTxnInvalidMeta, "released prepare timestamp floor cannot change") + } + currentFloor := f.prepareEvidenceTimestampFloor.Load() + if f.prepareEvidenceEnabled.Load() && floor <= currentFloor { + return nil + } + stamp := max(uint64(1), f.store.LastCommitTS(), floor) + muts := []*store.KVPairMutation{{Op: store.OpTypePut, Key: prepareEvidenceEnabledKey, Value: prepareEvidenceEnabledValue}} + if floor > currentFloor { + muts = append(muts, &store.KVPairMutation{Op: store.OpTypePut, Key: prepareEvidenceTimestampFloorKey, Value: binary.BigEndian.AppendUint64(nil, floor)}) + } + if err := f.store.ApplyMutationsRaftAt(context.Background(), muts, nil, stamp, stamp, f.pendingApplyIdx); err != nil { + return errors.Wrap(err, "persist prepare evidence fence") + } + f.prepareEvidenceEnabled.Store(true) + if floor > currentFloor { + f.prepareEvidenceTimestampFloor.Store(floor) + } + return nil +} diff --git a/kv/fsm_prepare_evidence_test.go b/kv/fsm_prepare_evidence_test.go new file mode 100644 index 000000000..de98eade4 --- /dev/null +++ b/kv/fsm_prepare_evidence_test.go @@ -0,0 +1,154 @@ +package kv + +import ( + "bytes" + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func preparedEvidenceRequest(phase pb.Phase, commitTS uint64) *pb.Request { + meta := TxnMeta{PrimaryKey: []byte("primary"), LockTTLms: defaultTxnLockTTLms, CommitTS: commitTS} + if phase == pb.Phase_PREPARE { + meta.CommitTS = 0 + meta.PrepareCommitTS = commitTS + } + return &pb.Request{ + IsTxn: true, Phase: phase, Ts: 10, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(meta)}, + {Op: pb.Op_PUT, Key: []byte("primary"), Value: []byte("value")}, + }, + } +} + +func TestPreparedCommitEvidenceCannotBeRetimestamped(t *testing.T) { + t.Parallel() + for _, phase := range []pb.Phase{pb.Phase_PREPARE, pb.Phase_COMMIT} { + t.Run(phase.String(), func(t *testing.T) { + st := store.NewMVCCStore() + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + require.NoError(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_PREPARE, 20))) + require.ErrorIs(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(phase, 21)), ErrTxnInvalidMeta) + _, err := st.GetAt(context.Background(), []byte("primary"), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound) + require.NoError(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_COMMIT, 20))) + value, err := st.GetAt(context.Background(), []byte("primary"), 20) + require.NoError(t, err) + require.Equal(t, []byte("value"), value) + }) + } +} + +func TestPrepareCommitEvidenceCannotBeAddedToLegacyIntent(t *testing.T) { + t.Parallel() + st := store.NewMVCCStore() + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + require.NoError(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_PREPARE, 0))) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + require.ErrorIs(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_PREPARE, 20)), ErrTxnInvalidMeta) + // A blocked upgrade must leave the old transaction free to finish normally. + require.NoError(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_COMMIT, 20))) +} + +func TestPrepareCommitEvidenceSurvivesReopen(t *testing.T) { + t.Parallel() + ctx := context.Background() + dir := t.TempDir() + st, err := store.NewPebbleStore(dir) + require.NoError(t, err) + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + require.NoError(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_PREPARE, 20))) + require.NoError(t, st.Close()) + reopened, err := store.NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + raw, err := reopened.GetAt(ctx, txnLockKey([]byte("primary")), ^uint64(0)) + require.NoError(t, err) + lock, err := decodeTxnLock(raw) + require.NoError(t, err) + require.Equal(t, uint64(20), lock.CommitTS) + restarted, ok := NewKvFSMWithHLC(reopened, NewHLC()).(*kvFSM) + require.True(t, ok) + require.ErrorIs(t, applyFSMRequest(t, restarted, preparedEvidenceRequest(pb.Phase_COMMIT, 21)), ErrTxnInvalidMeta) + require.NoError(t, applyFSMRequest(t, restarted, preparedEvidenceRequest(pb.Phase_COMMIT, 20))) +} + +func TestPrepareCommitEvidenceSurvivesSnapshotRestore(t *testing.T) { + t.Parallel() + st := store.NewMVCCStore() + source, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + require.Nil(t, source.Apply(EncodePrepareEvidenceFence(0))) + require.NoError(t, applyFSMRequest(t, source, preparedEvidenceRequest(pb.Phase_PREPARE, 20))) + snapshot, err := st.Snapshot() + require.NoError(t, err) + defer snapshot.Close() + var raw bytes.Buffer + _, err = snapshot.WriteTo(&raw) + require.NoError(t, err) + destination, ok := NewKvFSMWithHLC(store.NewMVCCStore(), NewHLC()).(*kvFSM) + require.True(t, ok) + require.NoError(t, destination.Restore(bytes.NewReader(raw.Bytes()))) + require.ErrorIs(t, applyFSMRequest(t, destination, preparedEvidenceRequest(pb.Phase_COMMIT, 21)), ErrTxnInvalidMeta) + require.NoError(t, applyFSMRequest(t, destination, preparedEvidenceRequest(pb.Phase_COMMIT, 20))) +} + +func TestPrepareEvidenceRequiresReplicatedEnableMarker(t *testing.T) { + t.Parallel() + fsm, ok := NewKvFSMWithHLC(store.NewMVCCStore(), NewHLC()).(*kvFSM) + require.True(t, ok) + require.ErrorIs(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_PREPARE, 20)), ErrTSOProtocolUnsupported) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + require.NoError(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_PREPARE, 20))) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(30))) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceSeal())) + require.ErrorIs(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_PREPARE, 20)), ErrPrepareCreationSealed) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceRelease(30))) + require.ErrorIs(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_PREPARE, 20)), ErrPrepareAdmissionFenced) + require.NoError(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_COMMIT, 20))) +} + +func TestPrepareAdmissionPreservesAppliedReadStartsAndBackupFence(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + epoch uint64 + proof uint64 + backupFloor uint64 + wantErr error + }{ + {name: "old applied snapshot with a new proven commit", epoch: 30, proof: 40}, + {name: "old epoch cannot create after release", epoch: 0, proof: 40, wantErr: ErrPrepareAdmissionFenced}, + {name: "old commit cannot be relabeled", epoch: 30, proof: 20, wantErr: ErrPrepareAdmissionFenced}, + {name: "real backup pin still fences old prepare starts", epoch: 30, proof: 40, backupFloor: 25, wantErr: ErrBackupTimestampFenced}, + } { + t.Run(test.name, func(t *testing.T) { + fsm, ok := NewKvFSMWithHLC(store.NewMVCCStore(), NewHLC()).(*kvFSM) + require.True(t, ok) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(30))) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceSeal())) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceRelease(30))) + if test.backupFloor != 0 { + require.NoError(t, fsm.persistBackupTimestampFloor(t.Context(), test.backupFloor)) + } + request := preparedEvidenceRequest(pb.Phase_PREPARE, test.proof) + request.TimestampAdmissionFloor = test.epoch + err := applyFSMRequest(t, fsm, request) + if test.wantErr != nil { + require.ErrorIs(t, err, test.wantErr) + return + } + require.NoError(t, err) + require.NoError(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_COMMIT, test.proof))) + }) + } +} diff --git a/kv/prepare_evidence_inspect.go b/kv/prepare_evidence_inspect.go new file mode 100644 index 000000000..0adfcba45 --- /dev/null +++ b/kv/prepare_evidence_inspect.go @@ -0,0 +1,89 @@ +package kv + +import ( + "bytes" + "context" + "encoding/binary" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +// InspectPrepareEvidence must follow an applied leader fence. The durable +// timestamp floor prevents creation of old locks during the paged scan. +func InspectPrepareEvidence(ctx context.Context, st store.MVCCStore, floor uint64) (enabled, unproven bool, fencedFloor uint64, err error) { + enabled, err = PrepareEvidenceEnabled(ctx, st) + if err != nil { + return false, false, 0, err + } + fencedFloor, err = readPrepareEvidenceFenceFloor(ctx, st) + if err != nil { + return false, false, 0, err + } + if !enabled || fencedFloor < floor { + return enabled, false, fencedFloor, nil + } + scanFloor := floor + if floor == 0 { + scanFloor = ^uint64(0) + } + for _, prefix := range [][]byte{txnLockPrefixBytes, distribution.MigrationStagedDataScanPrefix()} { + unproven, err = unprovenPrepareEvidenceInPrefix(ctx, st, prefix, scanFloor) + if err != nil || unproven { + return enabled, unproven, fencedFloor, err + } + } + return enabled, false, fencedFloor, nil +} + +func readPrepareEvidenceFenceFloor(ctx context.Context, st store.MVCCStore) (uint64, error) { + value, err := st.GetAt(ctx, prepareEvidenceTimestampFloorKey, ^uint64(0)) + if errors.Is(err, store.ErrKeyNotFound) { + return 0, nil + } + if err != nil { + return 0, errors.WithStack(err) + } + if len(value) != backupTimestampFloorValueSize { + return 0, errors.Wrap(ErrTxnInvalidMeta, "invalid prepare admission fence") + } + return binary.BigEndian.Uint64(value), nil +} + +func unprovenPrepareEvidenceInPrefix(ctx context.Context, st store.MVCCStore, prefix []byte, floor uint64) (bool, error) { + end := store.PrefixScanEnd(prefix) + for cursor := bytes.Clone(prefix); ; { + if err := ctx.Err(); err != nil { + return false, errors.WithStack(err) + } + rows, err := st.ScanAt(ctx, cursor, end, txnLockBatchScanPageSize, ^uint64(0)) + if err != nil { + return false, errors.WithStack(err) + } + if len(rows) == 0 { + return false, nil + } + for _, row := range rows { + blocked, err := unprovenPrepareEvidenceRow(row.Key, row.Value, floor) + if err != nil || blocked { + return blocked, err + } + } + cursor = nextScanCursor(rows[len(rows)-1].Key) + } +} + +func unprovenPrepareEvidenceRow(key, value []byte, floor uint64) (bool, error) { + if _, raw, staged := distribution.MigrationStagedDataKeyParts(key); staged { + key = raw + } + if !bytes.HasPrefix(key, txnLockPrefixBytes) { + return false, nil + } + lock, err := decodeTxnLock(value) + if err != nil { + return false, errors.Wrap(err, "inspect prepare evidence lock") + } + return lock.StartTS <= floor && lock.CommitTS == 0, nil +} diff --git a/kv/primary_commit_evidence.go b/kv/primary_commit_evidence.go index bde4d899c..60c5a3d8b 100644 --- a/kv/primary_commit_evidence.go +++ b/kv/primary_commit_evidence.go @@ -107,7 +107,8 @@ func (s *ShardStore) readPrimaryCommitEvidence(ctx context.Context, group *Shard } if errors.Is(err, store.ErrKeyNotFound) { prepared, prepareErr := s.primaryPreparedForEvidence(ctx, group, route, key, startTS) - return PrimaryCommitEvidence{Readable: true, PrimaryPrepared: prepared}, prepareErr + prepared.Readable = true + return prepared, prepareErr } commitTS, err := decodeTxnCommitRecord(value) if err != nil { @@ -116,22 +117,42 @@ func (s *ShardStore) readPrimaryCommitEvidence(ctx context.Context, group *Shard return PrimaryCommitEvidence{Readable: true, Found: true, CommitTS: commitTS}, nil } -func (s *ShardStore) primaryPreparedForEvidence(ctx context.Context, group *ShardGroup, route distribution.Route, key []byte, startTS uint64) (bool, error) { +func (s *ShardStore) primaryPreparedForEvidence(ctx context.Context, group *ShardGroup, route distribution.Route, key []byte, startTS uint64) (PrimaryCommitEvidence, error) { value, err := s.localGetAt(ctx, group, route, txnLockKey(key), ^uint64(0)) if errors.Is(err, store.ErrKeyNotFound) { - return false, nil + return PrimaryCommitEvidence{}, nil } if err != nil { - return false, errors.WithStack(err) + return PrimaryCommitEvidence{}, errors.WithStack(err) } lock, err := decodeTxnLock(value) if err != nil { - return false, errors.Wrap(err, "read prepared primary lock") + return PrimaryCommitEvidence{}, errors.Wrap(err, "read prepared primary lock") } if !lock.IsPrimaryKey || lock.StartTS != startTS || !bytes.Equal(lock.PrimaryKey, key) { - return false, nil + return PrimaryCommitEvidence{}, nil + } + matches, err := s.primaryPreparedIntentMatches(ctx, group, route, key, startTS) + if err != nil || !matches { + return PrimaryCommitEvidence{}, err } - value, err = s.localGetAt(ctx, group, route, txnIntentKey(key), ^uint64(0)) + return preparedPrimaryEvidenceWithLegacyGate(ctx, group.Store, lock) +} + +func preparedPrimaryEvidenceWithLegacyGate(ctx context.Context, st store.MVCCStore, lock txnLock) (PrimaryCommitEvidence, error) { + evidence := PrimaryCommitEvidence{PrimaryPrepared: true, PreparedCommitTS: lock.CommitTS} + if lock.CommitTS == 0 { + sealed, released, err := readPrepareEvidenceCreationGate(ctx, st) + if err != nil { + return PrimaryCommitEvidence{}, err + } + evidence.LegacyPreparedAllowed = !sealed && released == 0 + } + return evidence, nil +} + +func (s *ShardStore) primaryPreparedIntentMatches(ctx context.Context, group *ShardGroup, route distribution.Route, key []byte, startTS uint64) (bool, error) { + value, err := s.localGetAt(ctx, group, route, txnIntentKey(key), ^uint64(0)) if errors.Is(err, store.ErrKeyNotFound) { return false, nil } diff --git a/kv/primary_commit_evidence_test.go b/kv/primary_commit_evidence_test.go index 7d1153038..a07aece45 100644 --- a/kv/primary_commit_evidence_test.go +++ b/kv/primary_commit_evidence_test.go @@ -30,7 +30,11 @@ func TestPrimaryCommitEvidenceDistinguishesMissingFromUnavailable(t *testing.T) {name: "prepared primary", prepare: func(g *ShardGroup) { require.NoError(t, g.Store.PutAt(ctx, txnLockKey([]byte("key")), encodeTxnLock(txnLock{StartTS: 50, PrimaryKey: []byte("key"), IsPrimaryKey: true}), 50, 0)) require.NoError(t, g.Store.PutAt(ctx, txnIntentKey([]byte("key")), encodeTxnIntent(txnIntent{StartTS: 50, Op: txnIntentOpPut, Value: []byte("v")}), 50, 0)) - }, want: PrimaryCommitEvidence{Readable: true, PrimaryPrepared: true}}, + }, want: PrimaryCommitEvidence{Readable: true, PrimaryPrepared: true, LegacyPreparedAllowed: true}}, + {name: "primary with durable prepare timestamp", prepare: func(g *ShardGroup) { + require.NoError(t, g.Store.PutAt(ctx, txnLockKey([]byte("key")), encodeTxnLock(txnLock{StartTS: 50, CommitTS: 60, PrimaryKey: []byte("key"), IsPrimaryKey: true}), 50, 0)) + require.NoError(t, g.Store.PutAt(ctx, txnIntentKey([]byte("key")), encodeTxnIntent(txnIntent{StartTS: 50, Op: txnIntentOpPut, Value: []byte("v")}), 50, 0)) + }, want: PrimaryCommitEvidence{Readable: true, PrimaryPrepared: true, PreparedCommitTS: 60}}, {name: "primary lock without intent", prepare: func(g *ShardGroup) { require.NoError(t, g.Store.PutAt(ctx, txnLockKey([]byte("key")), encodeTxnLock(txnLock{StartTS: 50, PrimaryKey: []byte("key"), IsPrimaryKey: true}), 50, 0)) }, want: PrimaryCommitEvidence{Readable: true}}, diff --git a/kv/shard_store_prepare_evidence.go b/kv/shard_store_prepare_evidence.go new file mode 100644 index 000000000..e68733ab9 --- /dev/null +++ b/kv/shard_store_prepare_evidence.go @@ -0,0 +1,229 @@ +package kv + +import ( + "context" + "sort" + + "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/metadata" +) + +// CheckPhaseDProtocol negotiates with every voter and learner, including group +// zero, before any entry that an older FSM cannot decode is proposed. +func (s *ShardStore) CheckPhaseDProtocol(ctx context.Context) error { + if err := s.requireLocalPrepareEvidenceCutover(); err != nil { + return err + } + ids := make([]uint64, 0, len(s.groups)) + for id := range s.groups { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + for _, id := range ids { + if err := s.checkPrepareEvidenceGroupProtocol(ctx, id); err != nil { + return err + } + } + if err := s.fencePrepareEvidenceGroups(ctx, 0); err != nil { + return err + } + return s.inspectPrepareEvidenceGroups(ctx, 0) +} + +func (s *ShardStore) requireLocalPrepareEvidenceCutover() error { + if s == nil || s.groups[0] == nil { + return errors.WithStack(ErrTSOProtocolUnsupported) + } + if state := s.groups[0].TSOState; state == nil || !state.CutoverActive() { + return errors.Wrap(ErrTSOPhaseDActivationPending, "local dedicated timestamp cutover has not applied") + } + return nil +} + +func (s *ShardStore) checkPrepareEvidenceGroupProtocol(ctx context.Context, id uint64) error { + group := s.groups[id] + engine := engineForGroup(group) + if engine == nil { + return errors.Wrapf(ErrTSOProtocolUnsupported, "group %d has no engine", id) + } + before := engine.Status() + if before.PendingConfChange { + return errors.WithStack(ErrTSOPhaseDActivationPending) + } + configuration, err := engine.Configuration(ctx) + if err != nil || len(configuration.Servers) == 0 { + return errors.Wrapf(ErrTSOProtocolUnsupported, "group %d membership unavailable: %v", id, err) + } + eg, probeCtx := errgroup.WithContext(ctx) + eg.SetLimit(maxTSOCommitFloorConcurrency) + for _, member := range configuration.Servers { + if isLeaderEngine(engine) && member.ID == before.Leader.ID { + continue + } + eg.Go(func() error { + return s.checkPrepareEvidenceMember(probeCtx, id, member.Address, true) + }) + } + if err := eg.Wait(); err != nil { + return errors.WithStack(err) + } + return prepareEvidenceMembershipStable(before, engine.Status()) +} + +func prepareEvidenceMembershipStable(before, after raftengine.Status) error { + if after.ConfigurationIndex != before.ConfigurationIndex || after.PendingConfChange { + return errors.WithStack(ErrTSOPhaseDActivationPending) + } + return nil +} + +// CheckPrepareEvidenceMember is also the membership admission hook. Requiring +// support on every newly admitted member closes the negotiation/add race. +func (s *ShardStore) CheckPrepareEvidenceMember(ctx context.Context, groupID uint64, address string) error { + if s == nil || s.groups[0] == nil || s.groups[0].TSOState == nil || !s.groups[0].TSOState.CutoverActive() { + return nil + } + return s.checkPrepareEvidenceMember(ctx, groupID, address, false) +} + +func (s *ShardStore) checkPrepareEvidenceMember(ctx context.Context, groupID uint64, address string, requireCutover bool) error { + group := s.groups[groupID] + if group == nil || address == "" { + return errors.WithStack(ErrTSOProtocolUnsupported) + } + conn, err := s.connCache.ConnFor(address) + if err != nil { + return errors.Wrap(err, "prepare evidence capability connection") + } + probeCtx, cancel := context.WithTimeout(ctx, verifyLeaderTimeout) + defer cancel() + probeCtx = prepareEvidencePeerContext(probeCtx, group) + response, err := pb.NewInternalClient(conn).GetPrepareEvidenceState(probeCtx, &pb.PrepareEvidenceStateRequest{CapabilityOnly: true}) + if err != nil { + return errors.Wrapf(ErrTSOProtocolUnsupported, "group %d member %s lacks prepare evidence protocol: %v", groupID, address, err) + } + return s.validatePrepareEvidenceMemberResponse(groupID, address, response, requireCutover) +} + +func (s *ShardStore) validatePrepareEvidenceMemberResponse(groupID uint64, address string, response *pb.PrepareEvidenceStateResponse, requireCutover bool) error { + if !response.GetSupported() { + return errors.Wrapf(ErrTSOProtocolUnsupported, "group %d member %s lacks prepare evidence protocol", groupID, address) + } + if timestampGroup := s.groups[0]; timestampGroup != nil && timestampGroup.TSOState != nil && timestampGroup.TSOState.CutoverActive() && !response.GetDedicatedTsoConfigured() { + return errors.Wrapf(ErrTSOProtocolUnsupported, "group %d member %s must configure the dedicated timestamp runtime before joining a cutover cluster", groupID, address) + } + if requireCutover && !response.GetCutoverApplied() { + return errors.Wrapf(ErrTSOPhaseDActivationPending, "group %d member %s has not applied dedicated timestamp cutover", groupID, address) + } + return nil +} + +// PreparePhaseD keeps COMMIT/ABORT available while refusing activation until +// all old unproven locks, including secondaries and staged locks, have drained. +func (s *ShardStore) PreparePhaseD(ctx context.Context, floor uint64) error { + if floor == 0 { + return errors.WithStack(ErrInvalidRequest) + } + if err := s.CheckPhaseDProtocol(ctx); err != nil { + return err + } + if err := s.fencePrepareEvidenceGroups(ctx, floor); err != nil { + return err + } + return s.inspectPrepareEvidenceGroups(ctx, floor) +} + +func (s *ShardStore) inspectPrepareEvidenceGroups(ctx context.Context, floor uint64) error { + ids, err := s.tsoCommitFloorGroupIDs() + if err != nil { + return err + } + for _, id := range ids { + if err := s.inspectPrepareEvidenceGroup(ctx, id, floor); err != nil { + return err + } + } + return nil +} + +func (s *ShardStore) fencePrepareEvidenceGroups(ctx context.Context, floor uint64) error { + return s.proposePrepareEvidenceGroups(ctx, EncodePrepareEvidenceFence(floor)) +} + +// SealPhaseDCreation closes the late-apply race before sampling the final +// committed timestamp floor. Existing COMMIT and ABORT requests remain usable. +func (s *ShardStore) SealPhaseDCreation(ctx context.Context) error { + return s.proposePrepareEvidenceGroups(ctx, EncodePrepareEvidenceSeal()) +} + +// FinishPhaseD must run after the durable group-zero marker, including after +// leadership changes and retries of a partially completed release fanout. +func (s *ShardStore) FinishPhaseD(ctx context.Context, floor uint64) error { + return s.proposePrepareEvidenceGroups(ctx, EncodePrepareEvidenceRelease(floor)) +} + +func (s *ShardStore) proposePrepareEvidenceGroups(ctx context.Context, command []byte) error { + ids, err := s.tsoCommitFloorGroupIDs() + if err != nil { + return err + } + for _, id := range ids { + group := s.groups[id] + proposer := NewLeaderAdminProposer(group.Engine, group.Proposer(), &s.connCache, WithLeaderAdminToken(group.forwardedLeaderReadToken())) + result, err := proposer.ProposeAdmin(ctx, command) + if err != nil { + return errors.Wrapf(err, "prepare evidence control group %d", id) + } + if result == nil { + return errors.Wrapf(ErrTSOProtocolUnsupported, "group %d returned no prepare control receipt", id) + } + if applyErr, ok := result.Response.(error); ok && applyErr != nil { + return errors.Wrapf(applyErr, "prepare evidence control group %d apply", id) + } + } + return nil +} + +func (s *ShardStore) inspectPrepareEvidenceGroup(ctx context.Context, id, floor uint64) error { + group := s.groups[id] + engine := engineForGroup(group) + if isLeaderEngine(engine) { + if _, err := linearizableReadEngineCtx(ctx, engine); err != nil { + return err + } + enabled, unproven, actualFloor, err := InspectPrepareEvidence(ctx, group.Store, floor) + return prepareEvidenceDrainResult(id, enabled, unproven, actualFloor, floor, err) + } + address := leaderAddrFromEngine(engine) + if address == "" { + return errors.WithStack(ErrLeaderNotFound) + } + conn, err := s.connCache.ConnFor(address) + if err != nil { + return err + } + response, err := pb.NewInternalClient(conn).GetPrepareEvidenceState(prepareEvidencePeerContext(ctx, group), &pb.PrepareEvidenceStateRequest{Floor: floor}) + return prepareEvidenceDrainResult(id, response.GetProofEnabled(), response.GetUnprovenLocks(), response.GetFenceFloor(), floor, err) +} + +func prepareEvidenceDrainResult(id uint64, enabled, unproven bool, actual, expected uint64, err error) error { + if err != nil { + return errors.Wrapf(err, "inspect prepare evidence group %d", id) + } + if !enabled || unproven || expected != 0 && actual != expected { + return errors.Wrapf(ErrTSOPhaseDActivationPending, "group %d is waiting for its prepare evidence fence and legacy lock drain", id) + } + return nil +} + +func prepareEvidencePeerContext(ctx context.Context, group *ShardGroup) context.Context { + if token := group.forwardedLeaderReadToken(); token != "" { + return metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+token) + } + return ctx +} + +var _ TSOPhaseDActivationGate = (*ShardStore)(nil) diff --git a/kv/shard_store_prepare_evidence_test.go b/kv/shard_store_prepare_evidence_test.go new file mode 100644 index 000000000..ce7552824 --- /dev/null +++ b/kv/shard_store_prepare_evidence_test.go @@ -0,0 +1,661 @@ +package kv + +import ( + "bytes" + "context" + "errors" + "fmt" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +type prepareSafetyEngine struct { + *recordingTSOEngine + members []raftengine.Server + version atomic.Uint64 + pending atomic.Bool +} + +func (e *prepareSafetyEngine) Configuration(context.Context) (raftengine.Configuration, error) { + return raftengine.Configuration{Servers: e.members}, nil +} + +func (e *prepareSafetyEngine) Status() raftengine.Status { + s := e.recordingTSOEngine.Status() + s.Leader = e.Leader() + s.ConfigurationIndex, s.PendingConfChange = e.version.Load(), e.pending.Load() + return s +} + +func newPrepareSafetyGate(t *testing.T) (*ShardStore, *prepareSafetyEngine, *kvFSM) { + t.Helper() + st := store.NewMVCCStore() + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + engine := &prepareSafetyEngine{recordingTSOEngine: &recordingTSOEngine{ + state: raftengine.StateLeader, leader: raftengine.LeaderInfo{ID: "self"}, + apply: func(data []byte) error { + result := fsm.Apply(data) + if result == nil { + return nil + } + err, isError := result.(error) + if !isError { + return fmt.Errorf("unexpected fence response %T", result) + } + return err + }, + }, members: []raftengine.Server{{ID: "self", Address: "local", Suffrage: "voter"}}} + state := NewTSOStateMachine(NewHLC()) + require.Nil(t, state.Apply(marshalTSOCutover())) + shards := NewShardStore(nil, map[uint64]*ShardGroup{ + 0: {Engine: engine, TSOState: state}, 1: {Engine: engine, Store: st}, + }) + t.Cleanup(func() { require.NoError(t, shards.Close()) }) + return shards, engine, fsm +} + +type prepareSafetyPeer struct { + pb.UnimplementedInternalServer + probe func(context.Context, *pb.PrepareEvidenceStateRequest) (*pb.PrepareEvidenceStateResponse, error) +} + +func (s *prepareSafetyPeer) GetPrepareEvidenceState(ctx context.Context, req *pb.PrepareEvidenceStateRequest) (*pb.PrepareEvidenceStateResponse, error) { + return s.probe(ctx, req) +} + +func startPrepareSafetyPeer(t *testing.T, peer pb.InternalServer) string { + t.Helper() + lis, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + srv := grpc.NewServer() + pb.RegisterInternalServer(srv, peer) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(func() { srv.Stop(); _ = lis.Close() }) + return lis.Addr().String() +} + +func TestPrepareEvidenceSafetyRejectsEveryUnsupportedMemberBeforeFence(t *testing.T) { + for _, groupID := range []uint64{0, 1} { + for _, role := range []string{"voter", "learner"} { + t.Run(fmt.Sprintf("group-%d-%s", groupID, role), func(t *testing.T) { + shards, engine, _ := newPrepareSafetyGate(t) + old := startPrepareSafetyPeer(t, &pb.UnimplementedInternalServer{}) + other := &prepareSafetyEngine{recordingTSOEngine: &recordingTSOEngine{state: raftengine.StateFollower}, + members: []raftengine.Server{{ID: "old", Address: old, Suffrage: role}}} + shards.groups[groupID].Engine = other + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.ErrorIs(t, shards.CheckPhaseDProtocol(ctx), ErrTSOProtocolUnsupported) + require.Empty(t, engine.proposedPayloads(), "unsupported voter or learner must block even the floor-zero marker") + enabled, err := PrepareEvidenceEnabled(ctx, shards.groups[1].Store) + require.NoError(t, err) + require.False(t, enabled) + }) + } + } +} + +func TestPrepareEvidenceSafetyNegotiatesTokenAndMembershipBeforeMarker(t *testing.T) { + for _, changed := range []bool{false, true} { + t.Run(fmt.Sprintf("configuration-changed-%t", changed), func(t *testing.T) { + shards, engine, _ := newPrepareSafetyGate(t) + var probes atomic.Int32 + peer := &prepareSafetyPeer{probe: func(ctx context.Context, req *pb.PrepareEvidenceStateRequest) (*pb.PrepareEvidenceStateResponse, error) { + md, _ := metadata.FromIncomingContext(ctx) + if got := md.Get("authorization"); len(got) != 1 || got[0] != "Bearer safety-token" { + return nil, status.Error(codes.Unauthenticated, "missing peer token") + } + if !req.GetCapabilityOnly() { + return nil, status.Error(codes.InvalidArgument, "expected capability-only request") + } + probes.Add(1) + if changed { + engine.version.Add(1) + } + return &pb.PrepareEvidenceStateResponse{Supported: true, CutoverApplied: true, DedicatedTsoConfigured: true}, nil + }} + address := startPrepareSafetyPeer(t, peer) + // One shared engine backs groups 0 and 1, so both groups must probe. + engine.members = append(engine.members, raftengine.Server{ID: "learner", Address: address, Suffrage: "learner"}) + for _, group := range shards.groups { + group.SetLeaderReadToken("safety-token") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + err := shards.CheckPhaseDProtocol(ctx) + if changed { + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + require.Empty(t, engine.proposedPayloads()) + } else { + require.NoError(t, err) + require.Equal(t, int32(2), probes.Load()) + require.Equal(t, [][]byte{EncodePrepareEvidenceFence(0)}, engine.proposedPayloads()) + } + }) + } +} + +func TestPrepareEvidenceSafetyFloorZeroDrainsNormalAndStagedLocks(t *testing.T) { + for _, staged := range []bool{false, true} { + t.Run(fmt.Sprintf("staged-%t", staged), func(t *testing.T) { + shards, _, fsm := newPrepareSafetyGate(t) + ctx := context.Background() + // Put the unproven secondary after a full page of proven locks. + for i := 0; i <= txnLockBatchScanPageSize; i++ { + key := txnLockKey([]byte(fmt.Sprintf("key-%06d", i))) + if staged { + key = distribution.MigrationStagedDataKey(7, key) + } + proof := uint64(501) + if i == txnLockBatchScanPageSize { + proof = 0 + } + require.NoError(t, fsm.store.PutAt(ctx, key, encodeTxnLock(txnLock{StartTS: 500, CommitTS: proof, PrimaryKey: []byte("elsewhere")}), 500, 0)) + } + require.ErrorIs(t, shards.CheckPhaseDProtocol(ctx), ErrTSOPhaseDActivationPending) + enabled, unproven, floor, err := InspectPrepareEvidence(ctx, fsm.store, 0) + require.NoError(t, err) + require.True(t, enabled) + require.True(t, unproven, "floor zero must scan all start timestamps and all pages") + require.Zero(t, floor) + key := txnLockKey([]byte(fmt.Sprintf("key-%06d", txnLockBatchScanPageSize))) + if staged { + key = distribution.MigrationStagedDataKey(7, key) + } + require.NoError(t, fsm.store.DeleteAt(ctx, key, 600)) + require.NoError(t, shards.CheckPhaseDProtocol(ctx)) + require.NoError(t, shards.PreparePhaseD(ctx, 700)) + }) + } +} + +func TestPrepareEvidenceSafetyMarkerRejectsNewLegacyButAllowsRetryAndResolution(t *testing.T) { + for _, phase := range []pb.Phase{pb.Phase_COMMIT, pb.Phase_ABORT} { + t.Run(phase.String(), func(t *testing.T) { + _, _, fsm := newPrepareSafetyGate(t) + legacy := preparedEvidenceRequest(pb.Phase_PREPARE, 0) + require.NoError(t, applyFSMRequest(t, fsm, legacy)) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + require.NoError(t, applyFSMRequest(t, fsm, legacy), "same durable legacy lock may retry") + fresh := preparedEvidenceRequest(pb.Phase_PREPARE, 0) + fresh.Ts = 1000 + fresh.Mutations[1].Key = []byte("fresh-primary") + fresh.Mutations[0].Value = EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("fresh-primary"), LockTTLms: defaultTxnLockTTLms}) + require.Error(t, applyFSMRequest(t, fsm, fresh), "marker must reject a new proof-zero transaction even above the old floor") + require.NoError(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(phase, 20))) + _, unproven, _, err := InspectPrepareEvidence(context.Background(), fsm.store, 0) + require.NoError(t, err) + require.False(t, unproven) + }) + } +} + +func TestPrepareEvidenceSafetyMarkerBlocksLateLegacyImportAtAnyStart(t *testing.T) { + for _, floor := range []uint64{0, 100} { + for _, start := range []uint64{10, 1000} { + t.Run(fmt.Sprintf("floor-%d-start-%d", floor, start), func(t *testing.T) { + _, _, fsm := newPrepareSafetyGate(t) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(floor))) + key := txnLockKey([]byte("late-secondary")) + command, err := MarshalMigrationImportCommand(&pb.ImportRangeVersionsRequest{ + JobId: 7, BracketId: 1, BatchSeq: 1, Cursor: []byte("end"), + Versions: []*pb.MVCCVersion{{Key: key, CommitTs: start, Value: encodeTxnLock(txnLock{StartTS: start, PrimaryKey: []byte("primary")})}}, + }) + require.NoError(t, err) + result := fsm.Apply(command) + applyErr, ok := result.(error) + require.True(t, ok, "late import unexpectedly accepted: %v", result) + require.ErrorIs(t, applyErr, ErrBackupTimestampFenced) + _, err = fsm.store.GetAt(context.Background(), distribution.MigrationStagedDataKey(7, key), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound) + }) + } + } +} + +func TestPrepareEvidenceSafetyMarkerIsExcludedFromMigration(t *testing.T) { + brackets, err := distribution.PlanExportBrackets([]byte(txnBackupPrefix), []byte("!txn|backup}")) + require.NoError(t, err) + for _, key := range [][]byte{backupTimestampFloorKey, prepareEvidenceEnabledKey, prepareEvidenceGateKey, prepareEvidenceTimestampFloorKey} { + require.True(t, distribution.IsMigrationKnownInternalKey(key), + "the source must not export a group-local protocol marker as user data") + for _, bracket := range brackets { + require.False(t, bracket.ContainsRawKey(key), "bracket %d exports the protocol marker %q", bracket.BracketID, key) + } + } +} + +func TestPrepareEvidenceSafetyMigrationCannotReplaceLocalProtocolState(t *testing.T) { + for _, enabled := range []bool{false, true} { + for _, tombstone := range []bool{false, true} { + for _, key := range [][]byte{backupTimestampFloorKey, prepareEvidenceEnabledKey, prepareEvidenceGateKey, prepareEvidenceTimestampFloorKey} { + t.Run(fmt.Sprintf("enabled-%t-delete-%t-%s", enabled, tombstone, key), func(t *testing.T) { + _, _, fsm := newPrepareSafetyGate(t) + if enabled { + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + } + ctx := context.Background() + before, beforeErr := fsm.store.GetAt(ctx, key, ^uint64(0)) + userKey := []byte("ordinary-imported-user") + userVersion := &pb.MVCCVersion{Key: userKey, CommitTs: 1000, Value: []byte("value")} + control := &pb.MVCCVersion{Key: key, CommitTs: 1000, Value: []byte{1}, Tombstone: tombstone} + if tombstone { + control.Value = nil + } + request := &pb.ImportRangeVersionsRequest{JobId: 7, BracketId: 1, BatchSeq: 1, + Cursor: []byte("end"), Versions: []*pb.MVCCVersion{userVersion, control}} + command, err := MarshalMigrationImportCommand(request) + require.NoError(t, err) + result := fsm.Apply(command) + applyErr, ok := result.(error) + require.True(t, ok, "a control-key import must be a deterministic rejection, got %T", result) + require.ErrorIs(t, applyErr, ErrInvalidRequest) + for _, rawKey := range [][]byte{key, userKey} { + _, err = fsm.store.GetAt(ctx, distribution.MigrationStagedDataKey(7, rawKey), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound, "the entire rejected batch must remain unapplied") + } + after, afterErr := fsm.store.GetAt(ctx, key, ^uint64(0)) + if beforeErr != nil { + require.ErrorIs(t, beforeErr, store.ErrKeyNotFound) + require.ErrorIs(t, afterErr, store.ErrKeyNotFound) + } else { + require.NoError(t, afterErr) + require.Equal(t, before, after) + } + // Reuse sequence one: rejected protocol rows must not advance import progress. + request.Versions = []*pb.MVCCVersion{userVersion} + command, err = MarshalMigrationImportCommand(request) + require.NoError(t, err) + _, ok = fsm.Apply(command).(store.ImportVersionsResult) + require.True(t, ok, "the same batch sequence must still accept ordinary user data") + value, err := fsm.store.GetAt(ctx, distribution.MigrationStagedDataKey(7, userKey), ^uint64(0)) + require.NoError(t, err) + require.Equal(t, []byte("value"), value) + }) + } + } + } +} + +func TestPrepareEvidenceSafetyMarkerCannotBeForgedByTwoPhaseWrite(t *testing.T) { + _, _, fsm := newPrepareSafetyGate(t) + meta := TxnMeta{PrimaryKey: prepareEvidenceEnabledKey, LockTTLms: defaultTxnLockTTLms} + req := &pb.Request{IsTxn: true, Phase: pb.Phase_PREPARE, Ts: 10, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(TxnMetaPrefix), Value: EncodeTxnMeta(meta)}, + {Op: pb.Op_PUT, Key: prepareEvidenceEnabledKey, Value: []byte{1}}, + }} + prepareErr := applyFSMRequest(t, fsm, req) + if prepareErr == nil { + meta.CommitTS, req.Phase = 20, pb.Phase_COMMIT + req.Mutations[0].Value = EncodeTxnMeta(meta) + require.NoError(t, applyFSMRequest(t, fsm, req)) + } + enabled, err := PrepareEvidenceEnabled(context.Background(), fsm.store) + require.NoError(t, err) + require.False(t, enabled, "a client 2PC must not manufacture the negotiated protocol marker") + require.ErrorIs(t, prepareErr, ErrInvalidRequest) +} + +func TestPrepareEvidenceSafetyExistingForgedIntentCannotMaterializeMarker(t *testing.T) { + _, _, fsm := newPrepareSafetyGate(t) + ctx := context.Background() + // Simulate state prepared before the write-namespace protection existed. + require.NoError(t, fsm.store.PutAt(ctx, txnLockKey(prepareEvidenceEnabledKey), encodeTxnLock(txnLock{ + StartTS: 10, PrimaryKey: prepareEvidenceEnabledKey, IsPrimaryKey: true, + }), 10, 0)) + require.NoError(t, fsm.store.PutAt(ctx, txnIntentKey(prepareEvidenceEnabledKey), encodeTxnIntent(txnIntent{ + StartTS: 10, Op: txnIntentOpPut, Value: []byte{1}, + }), 10, 0)) + commit := &pb.Request{IsTxn: true, Phase: pb.Phase_COMMIT, Ts: 10, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(TxnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: prepareEvidenceEnabledKey, CommitTS: 20})}, + {Op: pb.Op_PUT, Key: prepareEvidenceEnabledKey}, + }} + require.ErrorIs(t, applyFSMRequest(t, fsm, commit), ErrInvalidRequest) + enabled, err := PrepareEvidenceEnabled(ctx, fsm.store) + require.NoError(t, err) + require.False(t, enabled) +} + +func TestPrepareEvidenceSafetyPendingFreezesCachedBatchWindow(t *testing.T) { + gate := &testPhaseDGate{prepareErr: ErrTSOPhaseDActivationPending} + allocator, fsm, _, _ := pendingTestAllocator(t, gate) + batch, err := NewBatchAllocator(allocator, 8) + require.NoError(t, err) + _, err = batch.Next(context.Background()) + require.NoError(t, err, "prime a preactivation batch that still has unused slots") + _, err = allocator.ReserveBatchAfter(context.Background(), 1, 0, true, true) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + require.NotZero(t, fsm.PendingPhaseDFloor()) + _, err = batch.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending, "pending must stop ordinary issuance from cached windows too") +} + +func TestPrepareEvidenceSafetyRequiresEveryMembersCutoverAdoption(t *testing.T) { + for _, groupID := range []uint64{0, 1} { + for _, role := range []string{"voter", "learner"} { + t.Run(fmt.Sprintf("group-%d-%s", groupID, role), func(t *testing.T) { + shards, engine, _ := newPrepareSafetyGate(t) + var adopted atomic.Bool + address := startPrepareSafetyPeer(t, &prepareSafetyPeer{probe: func(context.Context, *pb.PrepareEvidenceStateRequest) (*pb.PrepareEvidenceStateResponse, error) { + return &pb.PrepareEvidenceStateResponse{Supported: true, CutoverApplied: adopted.Load(), DedicatedTsoConfigured: true}, nil + }}) + member := &prepareSafetyEngine{recordingTSOEngine: &recordingTSOEngine{state: raftengine.StateLeader, + leader: raftengine.LeaderInfo{ID: "self"}, apply: engine.apply}, members: []raftengine.Server{ + {ID: "self", Suffrage: "voter"}, {ID: "remote", Address: address, Suffrage: role}, + }} + shards.groups[groupID].Engine = member + require.NoError(t, shards.CheckPrepareEvidenceMember(context.Background(), groupID, address), "admission negotiates support without requiring adoption") + require.ErrorIs(t, shards.CheckPhaseDProtocol(context.Background()), ErrTSOPhaseDActivationPending) + require.Empty(t, engine.proposedPayloads(), "no marker before every member has adopted dedicated TSO") + adopted.Store(true) + require.NoError(t, shards.CheckPhaseDProtocol(context.Background())) + }) + } + } + for _, missing := range []bool{false, true} { + t.Run(fmt.Sprintf("local-missing-%t", missing), func(t *testing.T) { + shards, engine, _ := newPrepareSafetyGate(t) + shards.groups[0].TSOState = NewTSOStateMachine(NewHLC()) + if missing { + shards.groups[0].TSOState = nil + } + require.ErrorIs(t, shards.CheckPhaseDProtocol(context.Background()), ErrTSOPhaseDActivationPending) + require.Empty(t, engine.proposedPayloads()) + }) + } +} + +func prepareSafetyCreationRequest(kind string, timestamp uint64) *pb.Request { + switch kind { + case "onephase": + return onePhaseReq(timestamp-1, timestamp, 0, []byte("late"), []byte("value")) + case "prepare": + request := preparedEvidenceRequest(pb.Phase_PREPARE, timestamp) + request.Ts = timestamp - 1 + return request + default: + return &pb.Request{Ts: timestamp, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("late"), Value: []byte("value")}}} + } +} + +func newPrepareSafetyFSM(t *testing.T, st store.MVCCStore) *kvFSM { + t.Helper() + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + return fsm +} + +func prepareSafetyAdmittedCreation(kind string, timestamp, floor uint64) *pb.Request { + request := prepareSafetyCreationRequest(kind, timestamp) + request.TimestampAdmissionFloor = floor + return request +} + +func TestPrepareEvidenceSafetyCreationSealRejectsLateRequests(t *testing.T) { + for _, kind := range []string{"raw", "onephase", "prepare"} { + t.Run(kind, func(t *testing.T) { + shards, _, fsm := newPrepareSafetyGate(t) + ctx := context.Background() + require.NoError(t, shards.CheckPhaseDProtocol(ctx)) + // The old request already chose a timestamp above the eventual B=100. + payload, err := proto.Marshal(prepareSafetyCreationRequest(kind, 1000)) + require.NoError(t, err) + require.NoError(t, shards.SealPhaseDCreation(ctx)) + require.NoError(t, shards.PreparePhaseD(ctx, 100)) + result, ok := fsm.Apply(payload).(error) + require.True(t, ok) + require.ErrorIs(t, result, ErrPrepareCreationSealed) + for _, key := range [][]byte{[]byte("late"), txnLockKey([]byte("primary")), txnIntentKey([]byte("primary"))} { + _, err = fsm.store.GetAt(ctx, key, ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound, "rejected creation must leave no user data or intent") + } + require.NoError(t, shards.FinishPhaseD(ctx, 100)) + require.ErrorIs(t, applyFSMRequest(t, fsm, prepareSafetyAdmittedCreation(kind, 100, 100)), ErrPrepareAdmissionFenced) + require.NoError(t, applyFSMRequest(t, fsm, prepareSafetyAdmittedCreation(kind, 1001, 100))) + }) + } +} + +func TestPrepareEvidenceSafetyCreationSealAllowsExistingResolution(t *testing.T) { + for _, proof := range []uint64{0, 20} { + for _, phase := range []pb.Phase{pb.Phase_COMMIT, pb.Phase_ABORT} { + t.Run(fmt.Sprintf("proof-%d-%s", proof, phase), func(t *testing.T) { + _, _, fsm := newPrepareSafetyGate(t) + if proof != 0 { + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + } + require.NoError(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_PREPARE, proof))) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceSeal())) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(100))) + require.NoError(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(phase, 20))) + _, err := fsm.store.GetAt(context.Background(), txnLockKey([]byte("primary")), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound) + value, err := fsm.store.GetAt(context.Background(), []byte("primary"), 20) + if phase == pb.Phase_COMMIT { + require.NoError(t, err) + require.Equal(t, []byte("value"), value) + } else { + require.ErrorIs(t, err, store.ErrKeyNotFound) + } + require.ErrorIs(t, applyFSMRequest(t, fsm, prepareSafetyCreationRequest("raw", 1000)), ErrPrepareCreationSealed) + }) + } + } +} + +func TestPrepareEvidenceSafetyCreationGateSurvivesRecovery(t *testing.T) { + for _, snapshot := range []bool{false, true} { + for _, released := range []bool{false, true} { + t.Run(fmt.Sprintf("snapshot-%t-released-%t", snapshot, released), func(t *testing.T) { + dir := t.TempDir() + st, err := store.NewPebbleStore(dir) + require.NoError(t, err) + closed := false + t.Cleanup(func() { + if !closed { + require.NoError(t, st.Close()) + } + }) + fsm := newPrepareSafetyFSM(t, st) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + require.NoError(t, applyFSMRequest(t, fsm, preparedEvidenceRequest(pb.Phase_PREPARE, 20))) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceSeal())) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(100))) + if released { + require.Nil(t, fsm.Apply(EncodePrepareEvidenceRelease(100))) + } + var recovered *kvFSM + if snapshot { + snap, err := fsm.Snapshot() + require.NoError(t, err) + defer snap.Close() + var raw bytes.Buffer + _, err = snap.WriteTo(&raw) + require.NoError(t, err) + destination, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + recovered = newPrepareSafetyFSM(t, destination) + t.Cleanup(func() { require.NoError(t, recovered.store.Close()) }) + // Restore must replace the opposite in-memory gate state, not retain it. + require.Nil(t, recovered.Apply(EncodePrepareEvidenceFence(100))) + require.Nil(t, recovered.Apply(EncodePrepareEvidenceSeal())) + if !released { + require.Nil(t, recovered.Apply(EncodePrepareEvidenceRelease(100))) + } + require.NoError(t, recovered.Restore(bytes.NewReader(raw.Bytes()))) + } else { + require.NoError(t, st.Close()) + closed = true + reopened, err := store.NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + recovered = newPrepareSafetyFSM(t, reopened) + } + creation := prepareSafetyAdmittedCreation("raw", 1000, 100) + if released { + require.Nil(t, recovered.Apply(EncodePrepareEvidenceSeal()), "a stale seal cannot re-close a released gate") + require.NoError(t, applyFSMRequest(t, recovered, creation)) + } else { + require.ErrorIs(t, applyFSMRequest(t, recovered, creation), ErrPrepareCreationSealed) + } + require.NoError(t, applyFSMRequest(t, recovered, preparedEvidenceRequest(pb.Phase_COMMIT, 20))) + require.Nil(t, recovered.Apply(EncodePrepareEvidenceRelease(100))) + require.Nil(t, recovered.Apply(EncodePrepareEvidenceRelease(100)), "release is retryable after recovery") + require.ErrorIs(t, applyFSMRequest(t, recovered, prepareSafetyAdmittedCreation("raw", 100, 100)), ErrPrepareAdmissionFenced) + require.NoError(t, applyFSMRequest(t, recovered, creation)) + value, err := recovered.store.GetAt(context.Background(), []byte("primary"), 20) + require.NoError(t, err) + require.Equal(t, []byte("value"), value) + }) + } + } +} + +func TestPrepareEvidenceSafetyReleaseValidationAndRetry(t *testing.T) { + _, _, fsm := newPrepareSafetyGate(t) + require.ErrorIs(t, haltApplyOf(fsm.Apply(EncodePrepareEvidenceSeal())), ErrTSOProtocolUnsupported) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(100))) + require.ErrorIs(t, haltApplyOf(fsm.Apply(EncodePrepareEvidenceRelease(100))), ErrTxnInvalidMeta, "release before seal must fail") + require.Nil(t, fsm.Apply(EncodePrepareEvidenceSeal())) + for _, floor := range []uint64{0, 101} { + require.ErrorIs(t, haltApplyOf(fsm.Apply(EncodePrepareEvidenceRelease(floor))), ErrTxnInvalidMeta) + require.ErrorIs(t, applyFSMRequest(t, fsm, prepareSafetyCreationRequest("raw", 1000)), ErrPrepareCreationSealed) + } + require.Nil(t, fsm.Apply(EncodePrepareEvidenceRelease(100))) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceRelease(100))) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceSeal())) + for _, floor := range []uint64{99, 101, 200} { + require.ErrorIs(t, haltApplyOf(fsm.Apply(EncodePrepareEvidenceFence(floor))), ErrTxnInvalidMeta, + "a released Phase-D barrier cannot be lowered or raised") + } + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(100))) + for _, floor := range []uint64{0, 99, 101, 200} { + require.ErrorIs(t, haltApplyOf(fsm.Apply(EncodePrepareEvidenceRelease(floor))), ErrTxnInvalidMeta, "release identity must not change on retry") + } + fsm = newPrepareSafetyFSM(t, fsm.store) + enabled, unproven, floor, err := InspectPrepareEvidence(context.Background(), fsm.store, 0) + require.NoError(t, err) + require.True(t, enabled) + require.False(t, unproven) + require.Equal(t, uint64(100), floor, "invalid fence attempts must not change the persisted barrier") + require.NoError(t, applyFSMRequest(t, fsm, prepareSafetyAdmittedCreation("raw", 1000, 100))) + require.ErrorIs(t, applyFSMRequest(t, fsm, prepareSafetyAdmittedCreation("raw", 1001, 200)), ErrPrepareAdmissionFenced) + require.ErrorIs(t, applyFSMRequest(t, fsm, prepareSafetyAdmittedCreation("raw", 100, 100)), ErrPrepareAdmissionFenced) +} + +func TestPrepareEvidenceSafetyPartialReleaseFanoutRetries(t *testing.T) { + shards, _, first := newPrepareSafetyGate(t) + other, engine, second := newPrepareSafetyGate(t) + shards.groups[2] = other.groups[1] + t.Cleanup(func() { delete(shards.groups, 2) }) + ctx := context.Background() + require.NoError(t, shards.CheckPhaseDProtocol(ctx)) + require.NoError(t, shards.SealPhaseDCreation(ctx)) + require.NoError(t, shards.PreparePhaseD(ctx, 100)) + releaseErr := errors.New("second group unavailable during release") + engine.setProposeError(releaseErr) + require.ErrorIs(t, shards.FinishPhaseD(ctx, 100), releaseErr) + require.NoError(t, applyFSMRequest(t, first, prepareSafetyAdmittedCreation("raw", 1000, 100))) + require.ErrorIs(t, applyFSMRequest(t, second, prepareSafetyCreationRequest("raw", 1000)), ErrPrepareCreationSealed) + engine.setProposeError(nil) + require.NoError(t, shards.FinishPhaseD(ctx, 100)) + require.NoError(t, shards.FinishPhaseD(ctx, 100)) + for _, fsm := range []*kvFSM{first, second} { + require.Nil(t, fsm.Apply(EncodePrepareEvidenceSeal())) + require.NoError(t, applyFSMRequest(t, fsm, prepareSafetyAdmittedCreation("raw", 1000, 100))) + require.ErrorIs(t, applyFSMRequest(t, fsm, prepareSafetyAdmittedCreation("raw", 100, 100)), ErrPrepareAdmissionFenced) + } +} + +func TestPrepareEvidenceSafetyPreCutoverRequestsCannotLandAfterRelease(t *testing.T) { + for _, kind := range []string{"raw", "onephase"} { + t.Run(kind, func(t *testing.T) { + shards, _, dataFSM := newPrepareSafetyGate(t) + initial, state, _, _ := pendingTestAllocator(t, shards) + shards.groups[0].TSOState = state + allocator, err := NewRaftTSOAllocator(initial.group, initial.clock, + WithTSOCutoverFloorProvider(shards), WithTSOPhaseDActivationGate(shards)) + require.NoError(t, err) + // A different data leader may have issued from a higher leased HLC + // before cutover. Keep its request off the Raft log until release. + legacyClock := NewHLC() + legacyClock.SetPhysicalCeiling(initial.clock.PhysicalCeiling() + time.Minute.Milliseconds()) + chosen, err := legacyClock.NextFenced() + require.NoError(t, err) + payload, err := proto.Marshal(prepareSafetyCreationRequest(kind, chosen)) + require.NoError(t, err) + window, err := allocator.ReserveBatchAfter(context.Background(), 1, 0, true, true) + require.NoError(t, err) + require.Greater(t, chosen, window.Base, "the fresh committed floor cannot see this uncommitted request") + result := dataFSM.Apply(payload) + applyErr, _ := result.(error) + require.ErrorIs(t, applyErr, ErrPrepareAdmissionFenced, "old %s chosen_ts=%d landed after release above the already-issued read_ts=%d (result %T)", kind, chosen, window.Base, result) + _, err = dataFSM.store.GetAt(context.Background(), []byte("late"), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound) + }) + } +} + +func TestPrepareEvidenceSafetyReleasedGateRequiresExactAdmissionEpoch(t *testing.T) { + for _, kind := range []string{"raw", "onephase", "prepare"} { + t.Run(kind, func(t *testing.T) { + _, _, fsm := newPrepareSafetyGate(t) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(100))) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceSeal())) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceRelease(100))) + for _, epoch := range []uint64{0, 99, 101} { + require.ErrorIs(t, applyFSMRequest(t, fsm, prepareSafetyAdmittedCreation(kind, 1000, epoch)), ErrPrepareAdmissionFenced) + } + require.NoError(t, applyFSMRequest(t, fsm, prepareSafetyAdmittedCreation(kind, 1000, 100))) + }) + } +} + +func TestPrepareEvidenceSafetyPostCutoverReadVoucherDoesNotBypassBackupPin(t *testing.T) { + for _, backup := range []bool{false, true} { + t.Run(fmt.Sprintf("backup-%t", backup), func(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + t.Cleanup(tracker.Close) + fsm := newBackupTestFSM(t, tracker) + t.Cleanup(func() { require.NoError(t, fsm.store.Close()) }) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(100))) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceSeal())) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceRelease(100))) + if backup { + require.Nil(t, fsm.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: backupTrackerTestPinID(1), ReadTS: 75, Deadline: time.Now().Add(time.Minute), + }))) + } + request := prepareSafetyAdmittedCreation("prepare", 101, 100) + request.Ts = 50 + err := applyFSMRequest(t, fsm, request) + if backup { + require.ErrorIs(t, err, ErrBackupTimestampFenced, "a real backup still fences the old read timestamp") + return + } + require.NoError(t, err, "Phase-D B fences the new durable proof, not an old applied-read voucher") + commit := preparedEvidenceRequest(pb.Phase_COMMIT, 101) + commit.Ts = 50 + require.NoError(t, applyFSMRequest(t, fsm, commit)) + value, err := fsm.store.GetAt(context.Background(), []byte("primary"), 101) + require.NoError(t, err) + require.Equal(t, []byte("value"), value) + }) + } +} diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index b19917e6e..2f7888edc 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -975,6 +975,7 @@ func NewShardedCoordinator(engine *distribution.Engine, groups map[uint64]*Shard g.Txn = &leaseRefreshingTxn{inner: g.Txn, g: g, coord: c} } } + c.configureTimestampAdmissionForGroup(g) router.Register(gid, g.Txn, g.Store) // Resolve the optional LeaseProvider capability once so // groupLeaseRead / maybeRefresh test g.lp for nil instead of @@ -2051,7 +2052,10 @@ type preparedGroup struct { } func (c *ShardedCoordinator) prewriteTxn(ctx context.Context, startTS, commitTS uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64, groupedReadKeys map[uint64][][]byte, observedRouteVersion uint64, bypassKeysByGroup map[uint64][][]byte) ([]preparedGroup, error) { - prepareMeta := txnMetaMutation(primaryKey, defaultTxnLockTTLms, 0) + prepareMetas, err := c.prepareEvidenceMetas(ctx, primaryKey, commitTS, gids) + if err != nil { + return nil, err + } prepared := make([]preparedGroup, 0, len(gids)) for _, gid := range gids { @@ -2063,7 +2067,7 @@ func (c *ShardedCoordinator) prewriteTxn(ctx context.Context, startTS, commitTS IsTxn: true, Phase: pb.Phase_PREPARE, Ts: startTS, - Mutations: append([]*pb.Mutation{prepareMeta}, grouped[gid]...), + Mutations: append([]*pb.Mutation{prepareMetas[gid]}, grouped[gid]...), ReadKeys: groupedReadKeys[gid], ObservedRouteVersion: observedRouteVersion, WriteFenceBypassKeys: bypassKeysByGroup[gid], @@ -3267,7 +3271,11 @@ func (c *ShardedCoordinator) txnLogs(ctx context.Context, reqs *OperationGroup[O if err := StampGroupedMutationCommitTS(grouped, commitTS); err != nil { return nil, err } - return buildTxnLogs(reqs.StartTS, commitTS, grouped, gids, reqs.ObservedRouteVersion, bypassKeysByGroup) + logs, err := buildTxnLogs(reqs.StartTS, commitTS, grouped, gids, reqs.ObservedRouteVersion, bypassKeysByGroup) + if err != nil { + return nil, err + } + return c.bindTxnLogPrepareEvidence(ctx, logs, commitTS, gids) } // observeMutation: counted pre-commit, so a mutation that subsequently diff --git a/kv/timestamp_admission.go b/kv/timestamp_admission.go new file mode 100644 index 000000000..200238910 --- /dev/null +++ b/kv/timestamp_admission.go @@ -0,0 +1,93 @@ +package kv + +import ( + "context" + "time" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +const timestampAdmissionValidationBudget = 5 * time.Second + +// StampTimestampAdmission binds creation to the data group's released epoch +// only after validating its unchanged persistence timestamp against group 0. +// Resolution is exempt. Incoming creation epochs are never trusted, even when +// the group has not yet activated the protocol. +func StampTimestampAdmission(ctx context.Context, allocator TimestampAllocator, st store.MVCCStore, reqs []*pb.Request) error { + creation := clearTimestampAdmission(reqs) + if !creation || st == nil { + return nil + } + ctx, cancel := context.WithTimeout(nonNilTSOContext(ctx), timestampAdmissionValidationBudget) + defer cancel() + sealed, floor, err := readPrepareEvidenceCreationGate(ctx, st) + if err != nil { + return errors.Wrap(err, "read timestamp admission epoch") + } + if sealed { + return errors.WithStack(ErrPrepareCreationSealed) + } + if floor == 0 { + return nil + } + if err := validateTimestampAdmission(ctx, allocator, reqs, floor); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return errors.Wrap(err, "timestamp admission budget exhausted") + } + for _, request := range reqs { + if timestampAdmissionCreation(request) { + request.TimestampAdmissionFloor = floor + } + } + return nil +} + +func timestampAdmissionCreation(request *pb.Request) bool { + return request != nil && (!request.IsTxn || request.Phase == pb.Phase_NONE || request.Phase == pb.Phase_PREPARE) +} + +func clearTimestampAdmission(reqs []*pb.Request) bool { + creation := false + for _, request := range reqs { + if timestampAdmissionCreation(request) { + request.TimestampAdmissionFloor = 0 + creation = true + } + } + return creation +} + +func validateTimestampAdmission(ctx context.Context, allocator TimestampAllocator, reqs []*pb.Request, floor uint64) error { + validator, ok := allocator.(DurableTimestampValidator) + if !ok { + return errors.WithStack(ErrTSOProtocolUnsupported) + } + validated := make(map[uint64]struct{}) + for _, request := range reqs { + if err := ctx.Err(); err != nil { + return errors.Wrap(err, "timestamp admission budget exhausted") + } + if !timestampAdmissionCreation(request) { + continue + } + timestamp, err := prepareCreationPersistenceTimestamp(request) + if err != nil { + return errors.WithStack(err) + } + if timestamp <= floor { + return errors.Wrapf(ErrTSOTimestampInvalid, "creation timestamp %d does not exceed admission floor %d", timestamp, floor) + } + if _, exists := validated[timestamp]; exists { + continue + } + if err := validator.ValidateDurableTimestamp(ctx, timestamp); err != nil { + return errors.Wrap(err, "validate creation timestamp admission") + } + validated[timestamp] = struct{}{} + } + return nil +} diff --git a/kv/timestamp_admission_benchmark_test.go b/kv/timestamp_admission_benchmark_test.go new file mode 100644 index 000000000..bb484b8c0 --- /dev/null +++ b/kv/timestamp_admission_benchmark_test.go @@ -0,0 +1,49 @@ +package kv + +import ( + "context" + "encoding/binary" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" +) + +var timestampAdmissionBenchmarkOwned []*pb.Request + +func BenchmarkTimestampAdmissionOwnership(b *testing.B) { + for _, size := range []struct { + name string + bytes int + }{{"32B", 32}, {"1MiB", 1 << 20}} { + b.Run(size.name, func(b *testing.B) { + requests := []*pb.Request{{Ts: 110, Mutations: []*pb.Mutation{{ + Key: []byte("key"), Value: make([]byte, size.bytes), + }}}} + legacy := &transactionTimestampAdmission{store: store.NewMVCCStore()} + allocator := &phaseDTestAllocator{} + released := &transactionTimestampAdmission{store: store.NewMVCCStore(), + allocator: func() TimestampAllocator { return allocator }} + value := binary.BigEndian.AppendUint64([]byte{prepareCreationReleased}, 100) + if err := released.store.PutAt(context.Background(), prepareEvidenceGateKey, value, 1, 0); err != nil { + b.Fatal(err) + } + for _, variant := range []struct { + name string + admission *transactionTimestampAdmission + }{{"before_unguarded", nil}, {"after_legacy", legacy}, {"after_released", released}} { + b.Run(variant.name, func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(size.bytes)) + for range b.N { + owned, err := variant.admission.prepare(context.Background(), requests) + if err != nil { + b.Fatal(err) + } + timestampAdmissionBenchmarkOwned = owned + } + }) + } + }) + } +} diff --git a/kv/timestamp_admission_test.go b/kv/timestamp_admission_test.go new file mode 100644 index 000000000..0762d0aa9 --- /dev/null +++ b/kv/timestamp_admission_test.go @@ -0,0 +1,266 @@ +package kv + +import ( + "context" + "encoding/binary" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/keyviz" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func admissionTestStore(t *testing.T, phase byte, floor uint64) store.MVCCStore { + t.Helper() + st := store.NewMVCCStore() + if phase != 0 { + value := binary.BigEndian.AppendUint64([]byte{phase}, floor) + require.NoError(t, st.PutAt(context.Background(), prepareEvidenceGateKey, value, 1, 0)) + } + return st +} + +func admissionTestRequests() []*pb.Request { + return []*pb.Request{ + {Ts: 110, TimestampAdmissionFloor: 999}, + {IsTxn: true, Phase: pb.Phase_NONE, Ts: 5, TimestampAdmissionFloor: 999, + Mutations: []*pb.Mutation{{Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{CommitTS: 120})}}}, + {IsTxn: true, Phase: pb.Phase_PREPARE, Ts: 6, TimestampAdmissionFloor: 999, + Mutations: []*pb.Mutation{{Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrepareCommitTS: 130})}}}, + } +} + +func TestStampTimestampAdmissionClearsUntrustedEpochBeforeRelease(t *testing.T) { + for _, st := range []store.MVCCStore{nil, admissionTestStore(t, 0, 0)} { + requests := admissionTestRequests() + require.NoError(t, StampTimestampAdmission(context.Background(), nil, st, requests)) + for _, request := range requests { + require.Zero(t, request.GetTimestampAdmissionFloor()) + } + } +} + +func TestStampTimestampAdmissionValidatesPersistenceNotPrepareStart(t *testing.T) { + st := admissionTestStore(t, prepareCreationReleased, 100) + for i, timestamp := range []uint64{110, 120, 130} { + request := admissionTestRequests()[i] + original := proto.CloneOf(request) + allocator := &phaseDTestAllocator{} + require.False(t, allocator.PhaseDRequired()) + require.NoError(t, StampTimestampAdmission(context.Background(), allocator, st, []*pb.Request{request})) + require.Equal(t, uint64(1), allocator.validateCalls.Load()) + require.Equal(t, timestamp, allocator.validated.Load()) + require.Zero(t, allocator.nextCalls.Load(), "admission must never mint or retimestamp") + require.Equal(t, uint64(100), request.GetTimestampAdmissionFloor()) + original.TimestampAdmissionFloor = 100 + require.True(t, proto.Equal(original, request), "start timestamps, metadata and values must remain unchanged") + } +} + +func TestStampTimestampAdmissionResolutionRemainsAvailableWhileSealed(t *testing.T) { + st := admissionTestStore(t, prepareCreationSealed, 0) + resolution := []*pb.Request{nil, + {IsTxn: true, Phase: pb.Phase_COMMIT, TimestampAdmissionFloor: 777}, + {IsTxn: true, Phase: pb.Phase_ABORT, TimestampAdmissionFloor: 888}, + } + require.NoError(t, StampTimestampAdmission(context.Background(), nil, st, resolution)) + require.Equal(t, uint64(777), resolution[1].GetTimestampAdmissionFloor()) + require.Equal(t, uint64(888), resolution[2].GetTimestampAdmissionFloor()) + requests := append([]*pb.Request{}, resolution...) + requests = append(requests, admissionTestRequests()...) + require.ErrorIs(t, StampTimestampAdmission(context.Background(), nil, st, requests), ErrPrepareCreationSealed) + for _, request := range requests[3:] { + require.Zero(t, request.GetTimestampAdmissionFloor()) + } +} + +func TestStampTimestampAdmissionRejectsMissingValidatorAndUnissuedValues(t *testing.T) { + st := admissionTestStore(t, prepareCreationReleased, 100) + requests := admissionTestRequests() + require.ErrorIs(t, StampTimestampAdmission(context.Background(), &fakeTSOAllocator{}, st, requests), ErrTSOProtocolUnsupported) + allocator := &phaseDTestAllocator{validateErr: ErrTSOTimestampInvalid} + require.ErrorIs(t, StampTimestampAdmission(context.Background(), allocator, st, requests), ErrTSOTimestampInvalid) + for _, request := range requests { + require.Zero(t, request.GetTimestampAdmissionFloor()) + } + allocator.validateErr = nil + requests = append(admissionTestRequests(), &pb.Request{Ts: 100, TimestampAdmissionFloor: 999}) + require.ErrorIs(t, StampTimestampAdmission(context.Background(), allocator, st, requests), ErrTSOTimestampInvalid) + for _, request := range requests { + require.Zero(t, request.GetTimestampAdmissionFloor(), "a partial validation must not admit a request prefix") + } +} + +func TestTimestampAdmissionOwnedRequestCannotBeRetaggedByRetry(t *testing.T) { + st := admissionTestStore(t, 0, 0) + allocator := &phaseDTestAllocator{} + admission := &transactionTimestampAdmission{store: st, allocator: func() TimestampAllocator { return allocator }} + request := &pb.Request{Ts: 110, TimestampAdmissionFloor: 999, + Mutations: []*pb.Mutation{{Key: []byte("key"), Value: []byte("old")}}} + owned, err := admission.prepare(context.Background(), []*pb.Request{request}) + require.NoError(t, err) + value := binary.BigEndian.AppendUint64([]byte{prepareCreationReleased}, 100) + require.NoError(t, st.PutAt(context.Background(), prepareEvidenceGateKey, value, 2, 0)) + require.NoError(t, StampTimestampAdmission(context.Background(), allocator, st, []*pb.Request{request})) + require.Equal(t, uint64(100), request.GetTimestampAdmissionFloor()) + require.Zero(t, owned[0].GetTimestampAdmissionFloor(), "a queued legacy request must retain its old admission epoch") + request.Ts = 150 + request.Mutations[0].Value[0] = 'n' + require.Equal(t, uint64(110), owned[0].GetTs()) + require.Equal(t, []byte("old"), owned[0].Mutations[0].Value, "queued values must not be changed by a later retry") +} + +func TestStampTimestampAdmissionDeduplicatesValidationWithinCommit(t *testing.T) { + st := admissionTestStore(t, prepareCreationReleased, 100) + allocator := &phaseDTestAllocator{} + requests := []*pb.Request{{Ts: 110}, {Ts: 110}} + require.NoError(t, StampTimestampAdmission(context.Background(), allocator, st, requests)) + require.Equal(t, uint64(1), allocator.validateCalls.Load()) + for _, request := range requests { + require.Equal(t, uint64(100), request.GetTimestampAdmissionFloor()) + } +} + +func TestTimestampAdmissionTransactionManagerAndLocalProxyAreWired(t *testing.T) { + for _, proxy := range []bool{false, true} { + t.Run(map[bool]string{false: "manager", true: "sharded_proxy"}[proxy], func(t *testing.T) { + st := admissionTestStore(t, prepareCreationReleased, 100) + allocator := &phaseDTestAllocator{} + engine := &recordingTSOEngine{state: raftengine.StateLeader, term: 1} + var tx Transactional + if proxy { + group := &ShardGroup{Engine: engine, Store: st} + group.Txn = NewLeaderProxyForShardGroup(group) + groups := map[uint64]*ShardGroup{0: {Engine: engine, TSOState: NewTSOStateMachine(NewHLC())}, 1: group} + coord := NewShardedCoordinator(distribution.NewEngineWithDefaultRoute(), groups, 1, NewHLC(), st) + coord.WithTSOAllocator(allocator) + t.Cleanup(func() { require.NoError(t, coord.Close()) }) + tx = group.Txn + } else { + tm := NewTransactionWithProposer(engine, WithTimestampAdmission(st, func() TimestampAllocator { return allocator })) + t.Cleanup(tm.Close) + tx = tm + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + request := &pb.Request{Ts: 110, TimestampAdmissionFloor: 999} + _, err := tx.Commit(ctx, []*pb.Request{request}) + require.NoError(t, err) + payloads := engine.proposedPayloads() + require.Len(t, payloads, 1) + requests, err := decodeRaftRequests(payloads[0]) + require.NoError(t, err) + require.Equal(t, uint64(100), requests[0].GetTimestampAdmissionFloor()) + require.Equal(t, uint64(110), requests[0].GetTs()) + require.Equal(t, uint64(1), allocator.validateCalls.Load()) + require.Equal(t, uint64(999), request.GetTimestampAdmissionFloor(), "queued requests must own their protobuf") + }) + } +} + +func TestTimestampAdmissionAutomaticWiringRequiresDedicatedState(t *testing.T) { + for _, scenario := range []struct { + name string + timestampGroup *ShardGroup + wantAdmission bool + }{ + {name: "no_group_zero"}, + {name: "group_zero_without_tso", timestampGroup: &ShardGroup{Engine: &recordingTSOEngine{}}}, + {name: "dedicated_tso", timestampGroup: &ShardGroup{Engine: &recordingTSOEngine{}, TSOState: NewTSOStateMachine(NewHLC())}, wantAdmission: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + engine := &recordingTSOEngine{state: raftengine.StateLeader, term: 1} + group := &ShardGroup{Engine: engine, Store: admissionTestStore(t, 0, 0)} + proxy := NewLeaderProxyForShardGroup(group) + group.Txn = proxy + groups := map[uint64]*ShardGroup{1: group} + if scenario.timestampGroup != nil { + groups[0] = scenario.timestampGroup + } + coord := NewShardedCoordinator(distribution.NewEngineWithDefaultRoute(), groups, 1, NewHLC(), group.Store) + t.Cleanup(func() { require.NoError(t, coord.Close()) }) + require.Equal(t, scenario.wantAdmission, proxy.tm.timestampAdmission.Load() != nil) + }) + } +} + +func TestTimestampAdmissionExplicitOptionSurvivesLegacyCoordinator(t *testing.T) { + engine := &recordingTSOEngine{state: raftengine.StateLeader, term: 1} + st := admissionTestStore(t, prepareCreationReleased, 100) + allocator := &phaseDTestAllocator{} + group := &ShardGroup{Engine: engine, Store: st} + proxy := NewLeaderProxyForShardGroup(group, WithTimestampAdmission(st, func() TimestampAllocator { return allocator })) + group.Txn = proxy + configured := proxy.tm.timestampAdmission.Load() + coord := NewShardedCoordinator(distribution.NewEngineWithDefaultRoute(), map[uint64]*ShardGroup{1: group}, 1, NewHLC(), st) + t.Cleanup(func() { require.NoError(t, coord.Close()) }) + require.Same(t, configured, proxy.tm.timestampAdmission.Load()) + _, err := group.Txn.Commit(context.Background(), []*pb.Request{{Ts: 110}}) + require.NoError(t, err) + require.Equal(t, uint64(1), allocator.validateCalls.Load()) +} + +func TestTimestampAdmissionSingleCoordinatorUsesCurrentAllocator(t *testing.T) { + st := admissionTestStore(t, prepareCreationReleased, 100) + engine := &recordingTSOEngine{state: raftengine.StateLeader, term: 1} + tm := NewTransactionWithProposer(engine) + t.Cleanup(tm.Close) + allocator := &phaseDTestAllocator{next: 140} + coord := NewCoordinatorWithEngine(tm, engine, WithTimestampAdmissionStore(st), WithTSOAllocator(allocator)) + t.Cleanup(func() { require.NoError(t, coord.Close()) }) + _, err := coord.dispatchRaw(context.Background(), []*Elem[OP]{{Op: Put, Key: []byte("k"), Value: []byte("v")}}, keyviz.LabelLegacy) + require.NoError(t, err) + requests, err := decodeRaftRequests(engine.proposedPayloads()[0]) + require.NoError(t, err) + require.Equal(t, uint64(100), requests[0].GetTimestampAdmissionFloor()) + require.Equal(t, uint64(140), allocator.validated.Load()) +} + +type admissionDeadlineAllocator struct { + phaseDTestAllocator + deadlines []time.Time + validated func() +} + +func (a *admissionDeadlineAllocator) ValidateDurableTimestamp(ctx context.Context, _ uint64) error { + deadline, _ := ctx.Deadline() + a.deadlines = append(a.deadlines, deadline) + if a.validated != nil { + a.validated() + } + return nil +} + +func TestStampTimestampAdmissionSharesOneBudgetAndHonorsEarlierDeadline(t *testing.T) { + st := admissionTestStore(t, prepareCreationReleased, 100) + for _, timeout := range []time.Duration{time.Minute, time.Second} { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + allocator := &admissionDeadlineAllocator{} + started := time.Now() + err := StampTimestampAdmission(ctx, allocator, st, []*pb.Request{{Ts: 110}, {Ts: 110}, {Ts: 120}}) + cancel() + require.NoError(t, err) + require.Len(t, allocator.deadlines, 2) + require.False(t, allocator.deadlines[0].IsZero()) + require.Equal(t, allocator.deadlines[0], allocator.deadlines[1]) + require.WithinDuration(t, started.Add(min(timeout, timestampAdmissionValidationBudget)), allocator.deadlines[0], time.Second/10) + } +} + +func TestStampTimestampAdmissionStopsFurtherValidationAfterBudgetCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + allocator := &admissionDeadlineAllocator{validated: cancel} + requests := []*pb.Request{{Ts: 110, TimestampAdmissionFloor: 999}, {Ts: 120, TimestampAdmissionFloor: 999}} + err := StampTimestampAdmission(ctx, allocator, admissionTestStore(t, prepareCreationReleased, 100), requests) + require.ErrorIs(t, err, context.Canceled) + require.Len(t, allocator.deadlines, 1) + for _, request := range requests { + require.Zero(t, request.GetTimestampAdmissionFloor()) + } +} diff --git a/kv/transaction.go b/kv/transaction.go index 0bd96f334..c490a2503 100644 --- a/kv/transaction.go +++ b/kv/transaction.go @@ -4,6 +4,7 @@ import ( "context" "math" "sync" + "sync/atomic" "time" "github.com/bootjp/elastickv/internal/raftengine" @@ -15,7 +16,8 @@ import ( type TransactionManager struct { proposer raftengine.Proposer - proposalObserver ProposalObserver + proposalObserver ProposalObserver + timestampAdmission atomic.Pointer[transactionTimestampAdmission] mu sync.Mutex rawPending []*rawCommitItem @@ -203,6 +205,11 @@ func (t *TransactionManager) Commit(ctx context.Context, reqs []*pb.Request) (*T if len(reqs) == 0 { return &TransactionResponse{}, nil } + owned, err := t.timestampAdmission.Load().prepare(ctx, reqs) + if err != nil { + return nil, err + } + reqs = owned if hasTxnRequests(reqs) { return t.commitSequential(ctx, reqs) } diff --git a/kv/transaction_timestamp_admission.go b/kv/transaction_timestamp_admission.go new file mode 100644 index 000000000..de6a35c8a --- /dev/null +++ b/kv/transaction_timestamp_admission.go @@ -0,0 +1,110 @@ +package kv + +import ( + "context" + "io" + "sync/atomic" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +type transactionTimestampAdmission struct { + store store.MVCCStore + allocator func() TimestampAllocator +} + +// WithTimestampAdmission guards the transaction manager before requests enter +// its raw queue. The getter follows the coordinator's current runtime allocator. +func WithTimestampAdmission(st store.MVCCStore, allocator func() TimestampAllocator) TransactionOption { + return func(t *TransactionManager) { + t.timestampAdmission.Store(&transactionTimestampAdmission{store: st, allocator: allocator}) + } +} + +// WithTimestampAdmissionStore equips a single-group coordinator with the data +// store required to stamp local requests, including its LeaderProxy local path. +func WithTimestampAdmissionStore(st store.MVCCStore) CoordinatorOption { + return func(c *Coordinate) { + c.transactionManager = configureTimestampAdmission(c.transactionManager, st, func() TimestampAllocator { return c.tsAllocator }) + } +} + +func (c *ShardedCoordinator) configureTimestampAdmissionForGroup(group *ShardGroup) { + timestampGroup := c.groups[0] + if timestampGroup == nil || timestampGroup.TSOState == nil { + // This immutable runtime cannot activate the protocol. Keep the default + // legacy path unchanged; an explicit transaction option remains intact. + return + } + group.Txn = configureTimestampAdmission(group.Txn, group.Store, func() TimestampAllocator { return c.tsAllocator }) +} + +func (a *transactionTimestampAdmission) prepare(ctx context.Context, reqs []*pb.Request) ([]*pb.Request, error) { + if a == nil { + return reqs, nil + } + // Keep the admitted epoch immutable while queued, even if a caller reuses + // the original request for a retry after the data group releases creation. + owned := make([]*pb.Request, len(reqs)) + for i, request := range reqs { + owned[i] = proto.CloneOf(request) + } + var allocator TimestampAllocator + if a.allocator != nil { + allocator = a.allocator() + } + if err := StampTimestampAdmission(ctx, allocator, a.store, owned); err != nil { + return nil, err + } + return owned, nil +} + +func configureTimestampAdmission(tx Transactional, st store.MVCCStore, allocator func() TimestampAllocator) Transactional { + admission := &transactionTimestampAdmission{store: st, allocator: allocator} + switch t := tx.(type) { + case *TransactionManager: + t.timestampAdmission.Store(admission) + case *LeaderProxy: + t.tm.timestampAdmission.Store(admission) + case *leaseRefreshingTxn: + t.inner = configureTimestampAdmission(t.inner, st, allocator) + case *timestampAdmissionTransactional: + t.admission.Store(admission) + case nil: + return nil + default: + wrapped := ×tampAdmissionTransactional{inner: tx} + wrapped.admission.Store(admission) + return wrapped + } + return tx +} + +type timestampAdmissionTransactional struct { + inner Transactional + admission atomic.Pointer[transactionTimestampAdmission] +} + +func (t *timestampAdmissionTransactional) Commit(ctx context.Context, reqs []*pb.Request) (*TransactionResponse, error) { + owned, err := t.admission.Load().prepare(ctx, reqs) + if err != nil { + return nil, err + } + response, err := t.inner.Commit(ctx, owned) + return response, errors.WithStack(err) +} + +func (t *timestampAdmissionTransactional) Abort(ctx context.Context, reqs []*pb.Request) (*TransactionResponse, error) { + response, err := t.inner.Abort(ctx, reqs) + return response, errors.WithStack(err) +} + +func (t *timestampAdmissionTransactional) Close() error { + if closer, ok := t.inner.(io.Closer); ok { + return errors.WithStack(closer.Close()) + } + return nil +} diff --git a/kv/tso.go b/kv/tso.go index 2607f7a5a..214b954e5 100644 --- a/kv/tso.go +++ b/kv/tso.go @@ -381,9 +381,12 @@ func ValidateForwardedTxnCommitTimestamp( commitTS uint64, resolution bool, label string, - verifyLegacyResolution func(context.Context) error, + verifyLegacyResolution func(context.Context, bool) error, ) error { err := ValidateDurablePersistenceTimestamp(ctx, alloc, commitTS, label) + if pendingLegacyResolutionNeedsValidation(err, resolution, startTS) { + return validatePendingLegacyResolution(ctx, alloc, startTS, commitTS, label, verifyLegacyResolution) + } if err == nil || !errors.Is(err, ErrTSOTimestampPrePhaseD) { return err } @@ -397,13 +400,28 @@ func ValidateForwardedTxnCommitTimestamp( startErr := ValidateDurablePersistenceTimestamp(ctx, alloc, startTS, label) if startErr != nil && errors.Is(startErr, ErrTSOTimestampPrePhaseD) { if verifyLegacyResolution != nil { - return errors.WithStack(verifyLegacyResolution(ctx)) + return errors.WithStack(verifyLegacyResolution(ctx, false)) } return nil } return err } +func pendingLegacyResolutionNeedsValidation(err error, resolution bool, startTS uint64) bool { + return resolution && startTS != 0 && errors.Is(err, ErrTSOPhaseDInactive) +} + +func validatePendingLegacyResolution(ctx context.Context, alloc TimestampAllocator, startTS, commitTS uint64, label string, verify func(context.Context, bool) error) error { + validator, ok := alloc.(DurableLegacyResolutionValidator) + if !ok || verify == nil { + return errors.Wrap(ErrTSOProtocolUnsupported, label+": pending resolution proof unavailable") + } + if err := validator.ValidateLegacyResolutionTimestamps(ctx, startTS, commitTS); err != nil { + return errors.Wrap(err, label+": pending resolution timestamp bound") + } + return errors.WithStack(verify(ctx, true)) +} + // ValidateForwardedTxnStartTimestamp validates a start timestamp a forwarded // transaction arrived with. // @@ -900,7 +918,13 @@ func (b *BatchAllocator) FreshAdmissionTimestamp(ctx context.Context) (uint64, e return 0, errors.WithStack(ErrTSOAllocatorRequired) } timestamp, err := b.tso.NextBatch(ctx, 1) - return timestamp, errors.Wrap(err, "issue fresh TSO admission timestamp") + if err == nil { + err = b.rejectPendingPhaseD() + } + if err != nil { + return 0, errors.Wrap(err, "issue fresh TSO admission timestamp") + } + return timestamp, nil } func (b *BatchAllocator) NextAfter(ctx context.Context, min uint64) (uint64, error) { @@ -944,15 +968,11 @@ func (b *BatchAllocator) nextAfter(ctx context.Context, min uint64) (uint64, err if err := ctxErr(ctx); err != nil { return 0, err } - phaseDAtStart, invalidated := b.ensurePhaseDTransition() - if invalidated { - continue + ts, ok, err := b.claimCurrentWindow(min) + if err != nil { + return 0, err } - if ts, ok := b.tryWindowAfter(min); ok { - phaseDAfterClaim, invalidated := b.ensurePhaseDTransition() - if invalidated || phaseDAfterClaim != phaseDAtStart { - continue - } + if ok { return ts, nil } if err := b.refill(ctx, min); err != nil { @@ -1044,6 +1064,17 @@ func (b *BatchAllocator) refill(ctx context.Context, min uint64) error { } func (b *BatchAllocator) nextRefillBatch(ctx context.Context, min uint64) (uint64, error) { + base, err := b.requestRefillBatch(ctx, min) + if err != nil { + return 0, err + } + if err := b.rejectPendingPhaseD(); err != nil { + return 0, err + } + return base, nil +} + +func (b *BatchAllocator) requestRefillBatch(ctx context.Context, min uint64) (uint64, error) { if after, ok := b.tso.(tsoBatchAfterAllocator); ok && min > 0 { base, err := after.NextBatchAfter(ctx, b.batchSize, min) return base, errors.Wrap(err, "tso next batch after observed ts") diff --git a/kv/tso_applied_leader_test.go b/kv/tso_applied_leader_test.go new file mode 100644 index 000000000..bf37e5d72 --- /dev/null +++ b/kv/tso_applied_leader_test.go @@ -0,0 +1,74 @@ +package kv + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestTSOLeaderWaitsForAppliedPendingState(t *testing.T) { + for _, operation := range []string{"reserve", "validate", "resolve"} { + t.Run(operation, func(t *testing.T) { + allocator, state, engine, _ := pendingTestAllocator(t, &testPhaseDGate{}) + require.Nil(t, state.Apply(marshalTSOCutover())) + base, err := allocator.clock.NextFenced() + require.NoError(t, err) + require.Nil(t, state.Apply(marshalTSOAllocationFloor(base))) + floor := base + 100 + entered := make(chan struct{}) + applied := make(chan struct{}) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + engine.readIndex = func(ctx context.Context) (uint64, error) { + close(entered) + select { + case <-applied: + return 3, nil + case <-ctx.Done(): + return 0, ctx.Err() + } + } + done := make(chan error, 1) + go func() { + done <- runAppliedPendingOperation(ctx, allocator, operation, base, floor) + }() + select { + case <-entered: + case <-done: + t.Fatal("TSO operation inspected state without an applied leader fence") + case <-ctx.Done(): + t.Fatal("TSO operation did not enter its applied leader fence") + } + require.Empty(t, engine.proposedPayloads()) + require.Nil(t, state.Apply(marshalTSOPendingPhaseD(floor))) + close(applied) + err = <-done + switch operation { + case "reserve": + require.NoError(t, err) + require.Equal(t, floor, state.PhaseDFloor(), "the committed pending floor must not be replaced") + case "validate": + require.ErrorIs(t, err, ErrTSOPhaseDInactive) + case "resolve": + require.ErrorIs(t, err, ErrTSOTimestampInvalid, "a legacy resolution cannot escape the applied pending bound") + } + }) + } +} + +func runAppliedPendingOperation(ctx context.Context, allocator *RaftTSOAllocator, operation string, base, floor uint64) error { + switch operation { + case "reserve": + _, err := allocator.ReserveBatchAfter(ctx, 1, 0, true, true) + return err + case "validate": + return allocator.ValidateDurableTimestamp(ctx, base+1) + case "resolve": + return allocator.ValidateLegacyResolutionTimestamps(ctx, base+1, floor+1) + default: + return fmt.Errorf("unknown TSO operation %q", operation) + } +} diff --git a/kv/tso_fsm.go b/kv/tso_fsm.go index d61c3aa2a..e708a0f61 100644 --- a/kv/tso_fsm.go +++ b/kv/tso_fsm.go @@ -46,12 +46,13 @@ const ( // TSO FSM's own applied state so unrelated shard-group lease renewals cannot // advance group-0 state outside the group-0 log. type TSOStateMachine struct { - hlc *HLC - ceilingMs atomic.Int64 - allocationFloor atomic.Uint64 - cutoverActive atomic.Bool - phaseDActive atomic.Bool - phaseDFloor atomic.Uint64 + hlc *HLC + ceilingMs atomic.Int64 + allocationFloor atomic.Uint64 + cutoverActive atomic.Bool + phaseDActive atomic.Bool + phaseDFloor atomic.Uint64 + pendingPhaseDFloor atomic.Uint64 // observer publishes durable marker state from the apply and restore // paths. Without it the gauges only move on a successful reservation or a // runtime mode transition, so a replica that applies a marker but serves no @@ -136,6 +137,8 @@ func (f *TSOStateMachine) Apply(data []byte) any { return f.applyCutoverEntry(data) case bytes.HasPrefix(data, []byte(tsoPhaseDEnvelope)): return f.applyPhaseDEntry(data) + case bytes.HasPrefix(data, []byte(tsoPendingPhaseDEnvelope)): + return f.applyPendingPhaseDEntry(data) case data[0] >= fsmwire.OpEncryptionMin && data[0] <= fsmwire.OpEncryptionMax: return rejectLegacyTSOEncryptionEntry(data) default: @@ -197,6 +200,9 @@ func (f *TSOStateMachine) applyAllocationFloorEntry(data []byte) any { return haltErr(errors.Wrap(ErrTSOStateMachineInvalidEntry, "zero TSO allocation floor")) } if f != nil { + if pending := f.PendingPhaseDFloor(); pending != 0 && floor > pending { + return haltErr(errors.Wrap(ErrTSOStateMachineInvalidEntry, "allocation exceeds pending phase-D floor")) + } f.applyAllocationFloor(floor) } return nil @@ -228,6 +234,9 @@ func (f *TSOStateMachine) applyPhaseDEntry(data []byte) any { "TSO phase-D marker requires the durable cutover marker")) } floor := binary.BigEndian.Uint64(data[len(tsoPhaseDEnvelope):]) + if pending := f.pendingPhaseDFloor.Load(); pending != 0 && pending != floor { + return haltErr(errors.Wrap(ErrTSOStateMachineInvalidEntry, "phase-D floor disagrees with pending marker")) + } if f.phaseDActive.Load() && f.phaseDFloor.Load() != floor { return haltErr(errors.Wrapf(ErrTSOStateMachineInvalidEntry, "TSO phase-D floor changed from %d to %d", f.phaseDFloor.Load(), floor)) @@ -281,12 +290,14 @@ func (f *TSOStateMachine) Snapshot() (raftengine.Snapshot, error) { var cutoverActive bool var phaseDActive bool var phaseDFloor uint64 + var pendingFloor uint64 if f != nil { ceilingMs = f.ceilingMs.Load() allocationFloor = f.allocationFloor.Load() cutoverActive = f.cutoverActive.Load() phaseDActive = f.phaseDActive.Load() phaseDFloor = f.phaseDFloor.Load() + pendingFloor = f.pendingPhaseDFloor.Load() } return &tsoFSMSnapshot{ ceilingMs: ceilingMs, @@ -294,6 +305,7 @@ func (f *TSOStateMachine) Snapshot() (raftengine.Snapshot, error) { cutoverActive: cutoverActive, phaseDActive: phaseDActive, phaseDFloor: phaseDFloor, + pendingFloor: pendingFloor, }, nil } @@ -305,11 +317,14 @@ func (f *TSOStateMachine) Restore(r io.Reader) error { if legacy, err := restoreLegacyKVFSMSnapshot(f, br); legacy || err != nil { return err } - ceilingMs, allocationFloor, cutoverActive, phaseDActive, phaseDFloor, err := readTSOSnapshotState(br) + ceilingMs, allocationFloor, cutoverActive, phaseDActive, phaseDFloor, pendingFloor, err := readTSOSnapshotState(br) if err != nil { return err } if f != nil { + if err := f.restorePendingPhaseDFloor(pendingFloor, allocationFloor, phaseDActive, phaseDFloor); err != nil { + return err + } f.restoreSnapshotState(ceilingMs, allocationFloor, cutoverActive, phaseDActive, phaseDFloor) } return nil @@ -322,22 +337,26 @@ func tsoSnapshotReader(r io.Reader) *bufio.Reader { return bufio.NewReader(r) } -func readTSOSnapshotState(br *bufio.Reader) (int64, uint64, bool, bool, uint64, error) { - payload, err := io.ReadAll(io.LimitReader(br, tsoSnapshotV4Len+1)) +func readTSOSnapshotState(br *bufio.Reader) (int64, uint64, bool, bool, uint64, uint64, error) { + payload, err := io.ReadAll(io.LimitReader(br, tsoSnapshotV5Len+1)) + if err != nil { + return 0, 0, false, false, 0, 0, errors.Wrap(err, "restore tso fsm snapshot") + } + pendingFloor, payload, err := decodePendingPhaseDSnapshot(payload) if err != nil { - return 0, 0, false, false, 0, errors.Wrap(err, "restore tso fsm snapshot") + return 0, 0, false, false, 0, 0, err } ceilingMs, allocationFloor, cutoverActive, phaseDActive, phaseDFloor, legacySnapshot, err := decodeTSOSnapshotPayload(payload) if err != nil { - return 0, 0, false, false, 0, err + return 0, 0, false, false, 0, 0, err } - if ceilingMs < 0 { - return 0, 0, false, false, 0, errors.Wrapf(ErrTSOStateMachineInvalidEntry, "tso fsm snapshot: negative ceiling %d", ceilingMs) + if err := validatePendingPhaseDSnapshot(pendingFloor, allocationFloor, cutoverActive, phaseDActive, phaseDFloor); err != nil { + return 0, 0, false, false, 0, 0, err } if legacySnapshot && ceilingMs > 0 { allocationFloor = tsoLeaseAllocationFloor(ceilingMs) } - return ceilingMs, allocationFloor, cutoverActive, phaseDActive, phaseDFloor, nil + return ceilingMs, allocationFloor, cutoverActive, phaseDActive, phaseDFloor, pendingFloor, nil } func decodeTSOSnapshotPayload(payload []byte) (int64, uint64, bool, bool, uint64, bool, error) { @@ -522,6 +541,9 @@ func legacyKVFSMStoreSnapshotMagics() [][]byte { } func (f *TSOStateMachine) IsVolatileOnlyPayload(payload []byte) bool { + if isTSOPendingPhaseDEntry(payload) { + return true + } if bytes.Equal(payload, []byte(tsoCutoverEnvelope)) { return true } @@ -647,38 +669,17 @@ type tsoFSMSnapshot struct { cutoverActive bool phaseDActive bool phaseDFloor uint64 + pendingFloor uint64 } func (s *tsoFSMSnapshot) WriteTo(w io.Writer) (int64, error) { if w == nil { return 0, errors.New("tso fsm snapshot: writer is nil") } - var ceilingMs int64 - var allocationFloor uint64 - var cutoverActive bool - var phaseDActive bool - var phaseDFloor uint64 - if s != nil { - ceilingMs = s.ceilingMs - allocationFloor = s.allocationFloor - cutoverActive = s.cutoverActive - phaseDActive = s.phaseDActive - phaseDFloor = s.phaseDFloor - } - buf := make([]byte, tsoSnapshotLenFor(ceilingMs, allocationFloor, cutoverActive, phaseDActive)) - if err := encodeTSOCeiling(buf, ceilingMs); err != nil { + buf, err := s.encode() + if err != nil { return 0, err } - if len(buf) >= tsoSnapshotV2Len { - binary.BigEndian.PutUint64(buf[hlcLeasePayloadLen:tsoSnapshotV2Len], allocationFloor) - } - if cutoverActive { - buf[tsoSnapshotV2Len] = 1 - } - if phaseDActive { - buf[tsoSnapshotV3Len] = 1 - binary.BigEndian.PutUint64(buf[tsoSnapshotV3Len+1:], phaseDFloor) - } n, err := w.Write(buf) if err != nil { return int64(n), errors.Wrap(err, "write tso fsm snapshot") @@ -689,6 +690,34 @@ func (s *tsoFSMSnapshot) WriteTo(w io.Writer) (int64, error) { return int64(n), nil } +func (s *tsoFSMSnapshot) encode() ([]byte, error) { + if s == nil { + s = &tsoFSMSnapshot{} + } + size := tsoSnapshotLenFor(s.ceilingMs, s.allocationFloor, s.cutoverActive, s.phaseDActive) + if s.pendingFloor != 0 { + size = tsoSnapshotV5Len + } + buf := make([]byte, size) + if err := encodeTSOCeiling(buf, s.ceilingMs); err != nil { + return nil, err + } + if len(buf) >= tsoSnapshotV2Len { + binary.BigEndian.PutUint64(buf[hlcLeasePayloadLen:tsoSnapshotV2Len], s.allocationFloor) + } + if s.cutoverActive { + buf[tsoSnapshotV2Len] = 1 + } + if s.phaseDActive { + buf[tsoSnapshotV3Len] = 1 + binary.BigEndian.PutUint64(buf[tsoSnapshotV3Len+1:], s.phaseDFloor) + } + if s.pendingFloor != 0 { + binary.BigEndian.PutUint64(buf[tsoSnapshotV4Len:], s.pendingFloor) + } + return buf, nil +} + // tsoSnapshotLenFor returns the shortest snapshot layout that can carry this // state. Each layout is its predecessor plus one trailing field, and only the // 8-byte V1 form is one a binary predating the allocation floor will restore: diff --git a/kv/tso_pending.go b/kv/tso_pending.go new file mode 100644 index 000000000..d365ef9a7 --- /dev/null +++ b/kv/tso_pending.go @@ -0,0 +1,363 @@ +package kv + +import ( + "bytes" + "context" + "encoding/binary" + stderrors "errors" + "time" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/cockroachdb/errors" +) + +var ( + ErrTSOPhaseDActivationPending = errors.New("tso: phase-D activation is pending") + ErrTSOPhaseDActivationGateRequired = errors.New("tso: phase-D activation gate is required") +) + +// TSOPhaseDActivationGate drains unproven prepares after all replicas adopt +// cutover, seals creation before sampling the global floor, then fences each +// data group at the fixed floor. FinishPhaseD releases creation after M7. +// Callbacks must return promptly with ErrTSOPhaseDActivationPending while old +// work remains, never wait for work requiring the allocator mutex, and be +// idempotent across retries, restarts, and leader changes. +type TSOPhaseDActivationGate interface { + CheckPhaseDProtocol(context.Context) error + SealPhaseDCreation(context.Context) error + PreparePhaseD(context.Context, uint64) error + FinishPhaseD(context.Context, uint64) error +} + +// TSOPendingPhaseDFloorSource exposes the durable barrier to cached issuers. +// An already reserved Phase-C window must not bypass a pending activation. +type TSOPendingPhaseDFloorSource interface { + PendingPhaseDFloor() uint64 +} + +// DurableLegacyResolutionValidator validates legacy resolution order while +// Phase D is inactive, and additionally the fixed bound once pending exists. +// Callers must prove durable txn evidence in BOTH the pre-pending drain and +// pending stages. Success never authorizes an ordinary write or PREPARE. +type DurableLegacyResolutionValidator interface { + ValidateLegacyResolutionTimestamps(context.Context, uint64, uint64) error +} + +const tsoPendingPhaseDEnvelope = "\x07TSOP\x01" +const tsoSnapshotV5Len = tsoSnapshotV4Len + hlcLeasePayloadLen + +const ( + phaseDBootstrapProbeTimeout = time.Second + phaseDBootstrapRetryInterval = 5 * time.Second +) + +// TSOValidationErrorDomain identifies structured group-0 validation errors. +const TSOValidationErrorDomain = "elastickv.tso" + +// TSOPhaseDInactiveReason preserves the inactive sentinel over gRPC without +// confusing it with a not-leader FailedPrecondition. +const TSOPhaseDInactiveReason = "PHASE_D_INACTIVE" + +func WithTSOPhaseDActivationGate(gate TSOPhaseDActivationGate) RaftTSOAllocatorOption { + return func(a *RaftTSOAllocator) { a.phaseDGate = gate } +} + +// PendingPhaseDFloor is zero when no activation is waiting for its data gates. +// The backing value remains durable after activation for idempotent WAL replay. +func (f *TSOStateMachine) PendingPhaseDFloor() uint64 { + if f == nil || f.PhaseDActive() { + return 0 + } + return f.pendingPhaseDFloor.Load() +} + +// PrepareEvidenceProtocolFloor retains the negotiated barrier even after M7. +// Zero distinguishes historical active snapshots that still need bootstrap. +func (f *TSOStateMachine) PrepareEvidenceProtocolFloor() uint64 { + if f == nil { + return 0 + } + return f.pendingPhaseDFloor.Load() +} + +func marshalTSOPendingPhaseD(floor uint64) []byte { + out := make([]byte, len(tsoPendingPhaseDEnvelope)+hlcLeasePayloadLen) + copy(out, tsoPendingPhaseDEnvelope) + binary.BigEndian.PutUint64(out[len(tsoPendingPhaseDEnvelope):], floor) + return out +} + +func (f *TSOStateMachine) applyPendingPhaseDEntry(data []byte) any { + if len(data) != len(tsoPendingPhaseDEnvelope)+hlcLeasePayloadLen { + return haltErr(errors.Wrap(ErrTSOStateMachineInvalidEntry, "invalid pending phase-D entry length")) + } + floor := binary.BigEndian.Uint64(data[len(tsoPendingPhaseDEnvelope):]) + if f == nil || floor == 0 || floor == ^uint64(0) || !f.CutoverActive() { + return haltErr(errors.Wrap(ErrTSOStateMachineInvalidEntry, "pending phase D requires cutover and a usable floor")) + } + if old := f.pendingPhaseDFloor.Load(); old != 0 && old != floor { + return haltErr(errors.Wrap(ErrTSOStateMachineInvalidEntry, "pending phase-D floor changed")) + } + if err := validatePendingPhaseDSnapshot(floor, f.AllocationFloor(), true, f.PhaseDActive(), f.PhaseDFloor()); err != nil { + return haltErr(err) + } + f.pendingPhaseDFloor.Store(floor) + return nil +} + +func isTSOPendingPhaseDEntry(payload []byte) bool { + return len(payload) == len(tsoPendingPhaseDEnvelope)+hlcLeasePayloadLen && + bytes.HasPrefix(payload, []byte(tsoPendingPhaseDEnvelope)) +} + +func decodePendingPhaseDSnapshot(payload []byte) (uint64, []byte, error) { + if len(payload) != tsoSnapshotV5Len { + return 0, payload, nil + } + floor := binary.BigEndian.Uint64(payload[tsoSnapshotV4Len:]) + if floor == 0 || floor == ^uint64(0) { + return 0, nil, errors.Wrap(ErrTSOStateMachineInvalidEntry, "invalid snapshot pending phase-D floor") + } + return floor, payload[:tsoSnapshotV4Len], nil +} + +func (f *TSOStateMachine) restorePendingPhaseDFloor(pending, allocation uint64, active bool, floor uint64) error { + old := f.pendingPhaseDFloor.Load() + if old != 0 && pending != 0 && old != pending { + return errors.Wrap(ErrTSOStateMachineInvalidEntry, "snapshot changed pending phase-D floor") + } + bound := max(old, pending) + if err := validatePendingPhaseDSnapshot(bound, allocation, true, active, floor); err != nil { + return err + } + if bound != 0 && f.PhaseDActive() && f.PhaseDFloor() != bound { + return errors.Wrap(ErrTSOStateMachineInvalidEntry, "snapshot conflicts with pending phase-D barrier") + } + if pending != 0 { + f.pendingPhaseDFloor.Store(pending) + } + return nil +} + +func validatePendingPhaseDSnapshot(pending, allocation uint64, cutover, active bool, floor uint64) error { + if pending == 0 { + return nil + } + if !cutover || pending == ^uint64(0) { + return errors.Wrap(ErrTSOStateMachineInvalidEntry, "pending phase D requires cutover and a usable floor") + } + if active { + if floor == pending { + return nil + } + return errors.Wrap(ErrTSOStateMachineInvalidEntry, "phase-D floor disagrees with pending marker") + } + if allocation > pending { + return errors.Wrap(ErrTSOStateMachineInvalidEntry, "allocation exceeds pending phase-D floor") + } + return nil +} + +func (a *RaftTSOAllocator) preparePhaseDActivation(ctx context.Context, engine raftengine.Engine, term uint64, activate bool) (uint64, error) { + if a.state.PhaseDActive() { + return 0, a.finishPhaseDActivation(ctx, engine, term) + } + pending := a.state.PendingPhaseDFloor() + if !activate { + if pending != 0 { + return 0, errors.WithStack(ErrTSOPhaseDActivationPending) + } + return 0, nil + } + if a.phaseDGate == nil { + return 0, errors.WithStack(ErrTSOPhaseDActivationGateRequired) + } + return pending, a.sealPhaseDActivation(ctx, engine, term) +} + +func (a *RaftTSOAllocator) sealPhaseDActivation(ctx context.Context, engine raftengine.Engine, term uint64) error { + // All members must apply cutover before their capability responses can + // authorize the seal; otherwise a stale runtime could mint legacy values + // after the data groups are released. + if !a.state.CutoverActive() { + if err := a.commitCutover(ctx, engine); err != nil { + return err + } + } + if err := a.phaseDGate.CheckPhaseDProtocol(ctx); err != nil { + return errors.Wrap(err, "check phase-D protocol") + } + if err := verifyTSOLeaderTerm(ctx, engine, term, true); err != nil { + return err + } + if err := a.phaseDGate.SealPhaseDCreation(ctx); err != nil { + return errors.Wrap(err, "seal phase-D creation") + } + return verifyTSOLeaderTerm(ctx, engine, term, true) +} + +func (a *RaftTSOAllocator) finishPhaseDActivation(ctx context.Context, engine raftengine.Engine, term uint64) error { + // Require the gate even for historical M7 state: no local compatibility + // assumption may bypass an unfinished durable seal after restart. + if a.phaseDGate == nil { + return errors.WithStack(ErrTSOPhaseDActivationGateRequired) + } + if a.phaseDFinishedTerm == term { + return nil + } + if err := verifyTSOLeaderTerm(ctx, engine, term, true); err != nil { + return err + } + if a.state.PrepareEvidenceProtocolFloor() == 0 { + started, err := a.tryBootstrapActivePhaseD(ctx, engine, term) + if err != nil { + return err + } + if !started { + return nil + } + } + return a.resumeActivePhaseDProtocol(ctx, engine, term) +} + +func (a *RaftTSOAllocator) resumeActivePhaseDProtocol(ctx context.Context, engine raftengine.Engine, term uint64) error { + // Historical M7 bootstrap records its pending marker before sealing, so + // raw marker presence alone cannot prove these data-side steps completed. + // Recheck first so a data group added by a restart configuration receives + // its proof-required marker before the seal is proposed. + if err := a.phaseDGate.CheckPhaseDProtocol(ctx); err != nil { + return errors.Wrap(err, "check active phase-D protocol") + } + if err := verifyTSOLeaderTerm(ctx, engine, term, true); err != nil { + return err + } + if err := a.phaseDGate.SealPhaseDCreation(ctx); err != nil { + return errors.Wrap(err, "resume active phase-D seal") + } + if err := verifyTSOLeaderTerm(ctx, engine, term, true); err != nil { + return err + } + if err := a.phaseDGate.PreparePhaseD(ctx, a.state.PhaseDFloor()); err != nil { + return errors.Wrap(err, "resume active phase-D data fence") + } + return a.releasePhaseDActivation(ctx, engine, term) +} + +func (a *RaftTSOAllocator) releasePhaseDActivation(ctx context.Context, engine raftengine.Engine, term uint64) error { + if err := verifyTSOLeaderTerm(ctx, engine, term, true); err != nil { + return err + } + if err := a.phaseDGate.FinishPhaseD(ctx, a.state.PhaseDFloor()); err != nil { + return errors.Wrap(err, "finish phase-D activation") + } + if err := verifyTSOLeaderTerm(ctx, engine, term, true); err != nil { + return err + } + a.phaseDFinishedTerm = term + return nil +} + +func (a *RaftTSOAllocator) tryBootstrapActivePhaseD(ctx context.Context, engine raftengine.Engine, term uint64) (bool, error) { + if a.state.PrepareEvidenceProtocolFloor() != 0 { + return true, nil + } + if a.phaseDBootstrapTerm == term && time.Now().Before(a.phaseDBootstrapRetryAt) { + return false, nil + } + floor := a.state.PhaseDFloor() + if floor == 0 || floor == ^uint64(0) { + return false, errors.Wrap(ErrTSOTimestampInvalid, "historical phase-D floor cannot carry prepare evidence protocol") + } + probeCtx, cancel := context.WithTimeout(ctx, phaseDBootstrapProbeTimeout) + defer cancel() + err := a.phaseDGate.CheckPhaseDProtocol(probeCtx) + if fenceErr := verifyTSOLeaderTerm(ctx, engine, term, true); fenceErr != nil { + return false, fenceErr + } + if a.state.PrepareEvidenceProtocolFloor() != 0 { + return true, nil + } + if err != nil { + // Before the first pending marker, failed capability negotiation or + // legacy-lock drain must not stop an already active rolling upgrade. + a.phaseDBootstrapTerm = term + a.phaseDBootstrapRetryAt = time.Now().Add(phaseDBootstrapRetryInterval) + return false, nil + } + if err := a.commitPendingPhaseD(ctx, engine, term, floor); err != nil { + return false, err + } + return true, nil +} + +func (a *RaftTSOAllocator) beginPendingPhaseD(ctx context.Context, engine raftengine.Engine, term uint64, n int, min, floor uint64) (TSOReservation, error) { + if err := a.commitPendingPhaseD(ctx, engine, term, floor); err != nil { + return TSOReservation{}, err + } + return a.resumePendingPhaseD(ctx, engine, term, n, min, floor) +} + +func (a *RaftTSOAllocator) commitPendingPhaseD(ctx context.Context, engine raftengine.Engine, term, floor uint64) error { + if _, err := a.group.Proposer().Propose(ctx, marshalTSOPendingPhaseD(floor)); err != nil { + if tsoProposalLostLeadership(engine, err) { + return errors.Wrap(stderrors.Join(ErrTSONotLeader, err), "commit pending phase D") + } + return errors.Wrap(err, "commit pending phase D") + } + if a.state.PrepareEvidenceProtocolFloor() != floor { + return errors.New("pending phase-D proposal committed without applied marker") + } + return verifyTSOLeaderTerm(ctx, engine, term, true) +} + +func (a *RaftTSOAllocator) resumePendingPhaseD(ctx context.Context, engine raftengine.Engine, term uint64, n int, min, floor uint64) (TSOReservation, error) { + var empty TSOReservation + if min > floor { + return empty, errors.Wrap(ErrTSOTimestampInvalid, "pending phase-D minimum exceeds fixed floor") + } + if err := validateTSOMinimumWindow(floor, n); err != nil { + return empty, err + } + if err := a.rejectMinimumWindowBeyondCeiling(floor, n); err != nil { + return empty, err + } + if err := a.phaseDGate.PreparePhaseD(ctx, floor); err != nil { + return empty, errors.Wrap(err, "prepare data groups for phase D") + } + if err := verifyTSOLeaderTerm(ctx, engine, term, true); err != nil { + return empty, err + } + if err := a.commitPhaseD(ctx, engine, floor); err != nil { + return empty, err + } + if err := a.releasePhaseDActivation(ctx, engine, term); err != nil { + return empty, err + } + end := floor + positiveIntToUint64(n) + if err := a.commitAllocationFloor(ctx, engine, end); err != nil { + return empty, err + } + if err := verifyTSOLeaderTerm(ctx, engine, term, false); err != nil { + return empty, err + } + a.initializedTerm = term + return TSOReservation{Base: floor + 1, Count: n, PreviousAllocationFloor: floor, + CutoverActive: true, PhaseDActive: true, PhaseDFloor: floor}, nil +} + +func (a *RaftTSOAllocator) ValidateLegacyResolutionTimestamps(ctx context.Context, start, commit uint64) error { + ctx = nonNilTSOContext(ctx) + a.mu.Lock() + defer a.mu.Unlock() + if _, err := a.verifiedLeader(ctx); err != nil { + return err + } + floor := a.state.PendingPhaseDFloor() + if a.state.PhaseDActive() { + return errors.WithStack(ErrTSOPhaseDInactive) + } + if start == 0 || commit <= start || (floor != 0 && commit > floor) { + return errors.Wrap(ErrTSOTimestampInvalid, "legacy resolution is outside pending phase-D floor") + } + return nil +} diff --git a/kv/tso_pending_batch.go b/kv/tso_pending_batch.go new file mode 100644 index 000000000..ac6d4a986 --- /dev/null +++ b/kv/tso_pending_batch.go @@ -0,0 +1,64 @@ +package kv + +import "github.com/cockroachdb/errors" + +func pendingPhaseDFloorThrough(allocator any) uint64 { + state, ok := allocator.(TSOPendingPhaseDFloorSource) + if !ok { + return 0 + } + return state.PendingPhaseDFloor() +} + +func (a *RaftTSOAllocator) PendingPhaseDFloor() uint64 { + if a == nil { + return 0 + } + return a.state.PendingPhaseDFloor() +} + +func (a *LeaderRoutedTSOAllocator) PendingPhaseDFloor() uint64 { + return pendingPhaseDFloorThrough(a.local) +} + +func (b *BatchAllocator) PendingPhaseDFloor() uint64 { + return pendingPhaseDFloorThrough(b.tso) +} + +func (d *DynamicTimestampAllocator) PendingPhaseDFloor() uint64 { + return pendingPhaseDFloorThrough(d.currentTimestampAllocator()) +} + +func (a *ShadowTimestampAllocator) PendingPhaseDFloor() uint64 { + return pendingPhaseDFloorThrough(a.shadow) +} + +func (b *BatchAllocator) rejectPendingPhaseD() error { + if b.PendingPhaseDFloor() == 0 { + return nil + } + b.Invalidate() + return errors.WithStack(ErrTSOPhaseDActivationPending) +} + +func (b *BatchAllocator) claimCurrentWindow(min uint64) (uint64, bool, error) { + if b.PendingPhaseDFloor() != 0 { + // Refill must still reach the underlying allocator: a routed D request + // can retry the activation, whereas an ordinary reservation is refused. + b.Invalidate() + return 0, false, nil + } + phaseDAtStart, invalidated := b.ensurePhaseDTransition() + if invalidated { + return 0, false, nil + } + ts, ok := b.tryWindowAfter(min) + if !ok { + return 0, false, nil + } + if err := b.rejectPendingPhaseD(); err != nil { + return 0, false, err + } + phaseDAfterClaim, invalidated := b.ensurePhaseDTransition() + return ts, !invalidated && phaseDAfterClaim == phaseDAtStart, nil +} diff --git a/kv/tso_pending_routing.go b/kv/tso_pending_routing.go new file mode 100644 index 000000000..5df0968ce --- /dev/null +++ b/kv/tso_pending_routing.go @@ -0,0 +1,96 @@ +package kv + +import ( + "context" + stderrors "errors" + "time" + + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func validateLegacyResolutionThrough(ctx context.Context, allocator any, start, commit uint64) error { + validator, ok := allocator.(DurableLegacyResolutionValidator) + if !ok { + return errors.WithStack(ErrTSOProtocolUnsupported) + } + return errors.WithStack(validator.ValidateLegacyResolutionTimestamps(ctx, start, commit)) +} + +func (b *BatchAllocator) ValidateLegacyResolutionTimestamps(ctx context.Context, start, commit uint64) error { + return validateLegacyResolutionThrough(ctx, b.tso, start, commit) +} + +func (d *DynamicTimestampAllocator) ValidateLegacyResolutionTimestamps(ctx context.Context, start, commit uint64) error { + return validateLegacyResolutionThrough(ctx, d.currentTimestampAllocator(), start, commit) +} + +func (a *ShadowTimestampAllocator) ValidateLegacyResolutionTimestamps(ctx context.Context, start, commit uint64) error { + return validateLegacyResolutionThrough(ctx, a.shadow, start, commit) +} + +func (a *LeaderRoutedTSOAllocator) ValidateLegacyResolutionTimestamps(ctx context.Context, start, commit uint64) error { + if start == 0 || commit <= start { + return errors.WithStack(ErrTSOTimestampInvalid) + } + ctx, cancel := context.WithTimeout(nonNilTSOContext(ctx), a.retryBudget) + defer cancel() + for { + started := time.Now() + path, err := a.tryLegacyResolutionValidation(ctx, start, commit) + a.observeRequest("validate_legacy_resolution", path, err, time.Since(started)) + if err == nil || errors.Is(err, ErrTSOPhaseDInactive) || errors.Is(err, ErrTSOProtocolUnsupported) || !isTransientTSORouteError(err) { + return err + } + if waitErr := waitTSORouteRetry(ctx, a.retryInterval); waitErr != nil { + return errors.Wrap(stderrors.Join(waitErr, err), "tso legacy resolution validation") + } + } +} + +func (a *LeaderRoutedTSOAllocator) tryLegacyResolutionValidation(ctx context.Context, start, commit uint64) (string, error) { + if a.local.IsLeader() { + return "local", validateLegacyResolutionThrough(ctx, a.local, start, commit) + } + addr := leaderAddrFromEngine(a.leader) + if addr == "" { + return "remote", errors.WithStack(ErrLeaderNotFound) + } + return "remote", a.requestRemoteLegacyResolution(ctx, addr, start, commit) +} + +func (a *LeaderRoutedTSOAllocator) requestRemoteLegacyResolution(ctx context.Context, addr string, start, commit uint64) error { + conn, err := a.connCache.ConnFor(addr) + if err != nil { + return errors.Wrap(err, "tso dial legacy resolution leader") + } + resp, err := pb.NewDistributionClient(conn).ValidateTimestamp(ctx, &pb.ValidateTimestampRequest{ + Timestamp: commit, StartTimestamp: start, LegacyResolution: true, + }) + if err != nil { + return tsoValidationRPCError(err) + } + if !resp.GetValid() || !resp.GetLegacyResolutionValidated() { + return errors.Wrap(ErrTSOProtocolUnsupported, "leader did not acknowledge legacy resolution validation") + } + return nil +} + +func tsoValidationRPCError(err error) error { + for _, detail := range status.Convert(err).Details() { + if info, ok := detail.(*errdetails.ErrorInfo); ok && + info.GetDomain() == TSOValidationErrorDomain && info.GetReason() == TSOPhaseDInactiveReason { + return errors.Wrap(stderrors.Join(err, ErrTSOPhaseDInactive), "tso phase D is inactive at leader") + } + } + if status.Code(err) == codes.OutOfRange { + return errors.Wrap(stderrors.Join(err, ErrTSOTimestampInvalid, ErrTSOTimestampPrePhaseD), "tso validate at leader") + } + if status.Code(err) == codes.InvalidArgument { + return errors.Wrap(stderrors.Join(err, ErrTSOTimestampInvalid), "tso validate at leader") + } + return errors.Wrap(err, "tso validate at leader") +} diff --git a/kv/tso_pending_seal_test.go b/kv/tso_pending_seal_test.go new file mode 100644 index 000000000..3b78c8e45 --- /dev/null +++ b/kv/tso_pending_seal_test.go @@ -0,0 +1,227 @@ +package kv + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/stretchr/testify/require" +) + +func TestTSOPendingSealPrecedesFreshFloorAndFinishPrecedesAllocation(t *testing.T) { + gate := &testPhaseDGate{} + a, fsm, engine, provider := pendingTestAllocator(t, gate) + var steps []string + gate.check = func() { + require.True(t, fsm.CutoverActive()) + steps = append(steps, "check") + } + gate.seal = func() { + require.Zero(t, provider.callCount()) + provider.setFloor(200) + steps = append(steps, "seal") + } + provider.afterRead = func() { steps = append(steps, "floor") } + gate.prepare = func(floor uint64) { + require.Greater(t, floor, uint64(200)) + require.Equal(t, floor, fsm.PendingPhaseDFloor()) + steps = append(steps, "prepare") + } + gate.finish = func(floor uint64) { + require.True(t, fsm.PhaseDActive()) + require.Equal(t, floor, fsm.PrepareEvidenceProtocolFloor()) + require.Zero(t, fsm.AllocationFloor()) + steps = append(steps, "finish") + } + result, err := a.ReserveBatchAfter(context.Background(), 2, 0, false, true) + require.NoError(t, err) + require.Equal(t, []string{"check", "seal", "floor", "prepare", "finish"}, steps) + require.Equal(t, result.Base+1, fsm.AllocationFloor()) + require.Len(t, engine.proposedPayloads(), 4) +} + +func TestTSOPendingSealFailureResumesAfterAllocatorRestart(t *testing.T) { + gate := &testPhaseDGate{sealErr: ErrTSOPhaseDActivationPending} + a, fsm, engine, provider := pendingTestAllocator(t, gate) + _, err := a.ReserveBatchAfter(context.Background(), 1, 0, true, true) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + require.Zero(t, provider.callCount()) + require.Zero(t, fsm.PrepareEvidenceProtocolFloor()) + require.Equal(t, [][]byte{marshalTSOCutover()}, engine.proposedPayloads()) + gate.sealErr = nil + engine.setTerm(2) + restarted, err := NewRaftTSOAllocator(a.group, a.clock, + WithTSOCutoverFloorProvider(provider), WithTSOPhaseDActivationGate(gate)) + require.NoError(t, err) + _, err = restarted.ReserveBatchAfter(context.Background(), 1, 0, true, true) + require.NoError(t, err) + require.Equal(t, 1, provider.callCount()) + require.True(t, fsm.PhaseDActive()) +} + +func TestTSOPendingFinishFailureBlocksNewWindowsUntilRetry(t *testing.T) { + gate := &testPhaseDGate{finishErr: ErrTSOPhaseDActivationPending} + a, fsm, engine, _ := pendingTestAllocator(t, gate) + _, err := a.ReserveBatchAfter(context.Background(), 2, 0, true, true) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + require.True(t, fsm.PhaseDActive()) + floor := fsm.PrepareEvidenceProtocolFloor() + require.NotZero(t, floor) + require.Zero(t, fsm.AllocationFloor()) + _, err = a.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + require.Zero(t, fsm.AllocationFloor()) + gate.finishErr = nil + ts, err := a.Next(context.Background()) + require.NoError(t, err) + require.Greater(t, ts, floor) + require.Equal(t, []uint64{floor, floor, floor}, gate.finished) + _, err = a.Next(context.Background()) + require.NoError(t, err) + require.Len(t, gate.finished, 3, "finish is cached only for this leader term") + engine.setTerm(2) + gate.finishErr = ErrTSOPhaseDActivationPending + previous := fsm.AllocationFloor() + _, err = a.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + require.Equal(t, previous, fsm.AllocationFloor()) + gate.finishErr = nil + _, err = a.Next(context.Background()) + require.NoError(t, err) + require.Len(t, gate.finished, 5) +} + +func TestTSOPendingLeaderChangeDuringFinishDoesNotReturnWindow(t *testing.T) { + gate := &testPhaseDGate{} + a, fsm, engine, _ := pendingTestAllocator(t, gate) + gate.finish = func(uint64) { engine.setTerm(2) } + _, err := a.ReserveBatchAfter(context.Background(), 1, 0, true, true) + require.ErrorIs(t, err, ErrTSONotLeader) + require.True(t, fsm.PhaseDActive()) + require.Zero(t, fsm.AllocationFloor()) + gate.finish = nil + _, err = a.Next(context.Background()) + require.NoError(t, err) + require.Len(t, gate.finished, 2) +} + +func TestTSOPendingHistoricalActiveSnapshotBootstrapsProtocol(t *testing.T) { + gate := &testPhaseDGate{checkErr: ErrTSOPhaseDActivationPending} + a, fsm, engine, _ := pendingTestAllocator(t, gate) + require.Nil(t, fsm.Apply(marshalTSOCutover())) + require.Nil(t, fsm.Apply(marshalTSOPhaseD(100))) + require.Nil(t, fsm.Apply(marshalTSOAllocationFloor(120))) + snap, err := fsm.Snapshot() + require.NoError(t, err) + var buf bytes.Buffer + _, err = snap.WriteTo(&buf) + require.NoError(t, err) + require.Len(t, buf.Bytes(), tsoSnapshotV4Len) + restored := NewTSOStateMachine(a.clock) + require.NoError(t, restored.Restore(bytes.NewReader(buf.Bytes()))) + engine.apply = applyTSOTestFSM(restored) + a.state = restored + a.group.TSOState = restored + ts, err := a.Next(context.Background()) + require.NoError(t, err, "historical M7 must remain available while old locks drain") + require.Greater(t, ts, uint64(120)) + require.Zero(t, restored.PrepareEvidenceProtocolFloor()) + gate.checkErr = nil + a.phaseDBootstrapRetryAt = time.Time{} + _, err = a.Next(context.Background()) + require.NoError(t, err) + require.Equal(t, uint64(100), restored.PrepareEvidenceProtocolFloor()) + require.Zero(t, restored.PendingPhaseDFloor()) + require.Equal(t, []uint64{100}, gate.floors) + require.Equal(t, []uint64{100}, gate.finished) + require.Equal(t, marshalTSOPendingPhaseD(100), engine.proposedPayloads()[1]) +} + +func TestTSOPendingActiveWithoutGateFailsClosed(t *testing.T) { + a, fsm, engine, _ := pendingTestAllocator(t, nil) + require.Nil(t, fsm.Apply(marshalTSOCutover())) + require.Nil(t, fsm.Apply(marshalTSOPhaseD(100))) + _, err := a.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOPhaseDActivationGateRequired) + require.Empty(t, engine.proposedPayloads()) +} + +func TestTSOPendingActiveSnapshotRetriesDataGatesAfterRestart(t *testing.T) { + gate := &testPhaseDGate{finishErr: ErrTSOPhaseDActivationPending} + a, fsm, _, provider := pendingTestAllocator(t, gate) + _, err := a.ReserveBatchAfter(context.Background(), 2, 0, true, true) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + floor := fsm.PrepareEvidenceProtocolFloor() + snap, err := fsm.Snapshot() + require.NoError(t, err) + var buf bytes.Buffer + _, err = snap.WriteTo(&buf) + require.NoError(t, err) + require.Len(t, buf.Bytes(), tsoSnapshotV5Len) + restored := NewTSOStateMachine(NewHLC()) + require.NoError(t, restored.Restore(bytes.NewReader(buf.Bytes()))) + engine := &recordingTSOEngine{state: raftengine.StateLeader, term: 2, + leader: raftengine.LeaderInfo{Address: "new"}, apply: applyTSOTestFSM(restored)} + gate.checkErr = ErrTSOProtocolUnsupported + gate.sealErr = ErrTSOProtocolUnsupported + gate.finishErr = nil + restarted, err := NewRaftTSOAllocator(&ShardGroup{Engine: engine, TSOState: restored}, restored.hlc, + WithTSOCutoverFloorProvider(provider), WithTSOPhaseDActivationGate(gate)) + require.NoError(t, err) + _, err = restarted.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOProtocolUnsupported, "durable pending marker forbids historical fallback") + require.Zero(t, restored.AllocationFloor()) + gate.checkErr = nil + gate.sealErr = nil + ts, err := restarted.Next(context.Background()) + require.NoError(t, err) + require.Greater(t, ts, floor) + require.Equal(t, []uint64{floor, floor}, gate.finished) + require.Equal(t, []uint64{floor, floor}, gate.floors, "retry must reestablish data gates without moving the floor") +} + +func TestTSOPendingLeaderChangeAfterSealResamplesBeforeFreezing(t *testing.T) { + gate := &testPhaseDGate{} + a, fsm, engine, provider := pendingTestAllocator(t, gate) + provider.afterRead = func() { engine.setTerm(2) } + _, err := a.ReserveBatchAfter(context.Background(), 1, 0, true, true) + require.ErrorIs(t, err, ErrTSONotLeader) + require.Zero(t, fsm.PrepareEvidenceProtocolFloor()) + provider.afterRead = nil + provider.setFloor(1_000) + result, err := a.ReserveBatchAfter(context.Background(), 1, 0, true, true) + require.NoError(t, err) + require.Greater(t, result.PhaseDFloor, uint64(1_000)) + require.Equal(t, 2, provider.callCount()) +} + +func TestTSOPendingCachedWindowsStopButRoutedActivationCanResume(t *testing.T) { + gate := &testPhaseDGate{prepareErr: ErrTSOPhaseDActivationPending} + a, fsm, engine, _ := pendingTestAllocator(t, gate) + routed, err := NewLeaderRoutedTSOAllocator(a, engine) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, routed.Close()) }) + batch, err := NewBatchAllocator(routed, 8) + require.NoError(t, err) + _, err = batch.Next(context.Background()) + require.NoError(t, err) + _, err = a.ReserveBatchAfter(context.Background(), 1, 0, true, true) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + shadow := &ShadowTimestampAllocator{shadow: routed} + dynamic := NewDynamicTimestampAllocator(batch) + require.Equal(t, fsm.PendingPhaseDFloor(), shadow.PendingPhaseDFloor()) + require.Equal(t, fsm.PendingPhaseDFloor(), dynamic.PendingPhaseDFloor()) + _, err = batch.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + _, err = batch.FreshAdmissionTimestamp(context.Background()) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + routed.setActivation(true, true) + gate.prepareErr = nil + ts, err := batch.Next(context.Background()) + require.NoError(t, err) + require.Greater(t, ts, fsm.PhaseDFloor()) + require.True(t, fsm.PhaseDActive()) + require.Equal(t, raftengine.StateLeader, engine.Status().State) +} diff --git a/kv/tso_pending_test.go b/kv/tso_pending_test.go new file mode 100644 index 000000000..e998fc292 --- /dev/null +++ b/kv/tso_pending_test.go @@ -0,0 +1,219 @@ +package kv + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/stretchr/testify/require" +) + +type testPhaseDGate struct { + checks int + checkErr error + sealErr error + prepareErr error + finishErr error + floors []uint64 + finished []uint64 + check func() + seal func() + prepare func(uint64) + finish func(uint64) +} + +func (g *testPhaseDGate) CheckPhaseDProtocol(context.Context) error { + g.checks++ + if g.check != nil { + g.check() + } + return g.checkErr +} + +func (g *testPhaseDGate) PreparePhaseD(_ context.Context, floor uint64) error { + g.floors = append(g.floors, floor) + if g.prepare != nil { + g.prepare(floor) + } + return g.prepareErr +} + +func (g *testPhaseDGate) SealPhaseDCreation(context.Context) error { + if g.seal != nil { + g.seal() + } + return g.sealErr +} + +func (g *testPhaseDGate) FinishPhaseD(_ context.Context, floor uint64) error { + g.finished = append(g.finished, floor) + if g.finish != nil { + g.finish(floor) + } + return g.finishErr +} + +func pendingTestAllocator(t *testing.T, gate TSOPhaseDActivationGate) (*RaftTSOAllocator, *TSOStateMachine, *recordingTSOEngine, *recordingTSOFloorProvider) { + t.Helper() + clock := NewHLC() + clock.SetPhysicalCeiling(time.Now().Add(testTSOFutureCeiling).UnixMilli()) + fsm := NewTSOStateMachine(clock) + engine := &recordingTSOEngine{state: raftengine.StateLeader, term: 1, + leader: raftengine.LeaderInfo{Address: "self"}, apply: applyTSOTestFSM(fsm)} + provider := &recordingTSOFloorProvider{} + a, err := NewRaftTSOAllocator(&ShardGroup{Engine: engine, TSOState: fsm}, clock, + WithTSOCutoverFloorProvider(provider), WithTSOPhaseDActivationGate(gate)) + require.NoError(t, err) + return a, fsm, engine, provider +} + +func TestTSOPendingRequiresGateAndCapabilityBeforeMarker(t *testing.T) { + a, fsm, engine, _ := pendingTestAllocator(t, nil) + _, err := a.ReserveBatchAfter(context.Background(), 2, 0, true, true) + require.ErrorIs(t, err, ErrTSOPhaseDActivationGateRequired) + require.Empty(t, engine.proposedPayloads()) + gate := &testPhaseDGate{checkErr: ErrTSOProtocolUnsupported} + a.phaseDGate = gate + _, err = a.ReserveBatchAfter(context.Background(), 2, 0, true, true) + require.ErrorIs(t, err, ErrTSOProtocolUnsupported) + require.Equal(t, [][]byte{marshalTSOCutover()}, engine.proposedPayloads()) + require.Zero(t, fsm.PendingPhaseDFloor()) +} + +func TestTSOPendingPreBarrierDrainAllowsEvidenceScopedResolutionAndFreshFloor(t *testing.T) { + gate := &testPhaseDGate{checkErr: ErrTSOPhaseDActivationPending} + a, fsm, engine, provider := pendingTestAllocator(t, gate) + _, err := a.ReserveBatchAfter(context.Background(), 2, 0, true, true) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + require.Equal(t, [][]byte{marshalTSOCutover()}, engine.proposedPayloads()) + require.Equal(t, 0, provider.callCount()) + require.NoError(t, a.ValidateLegacyResolutionTimestamps(context.Background(), 50, 1_000_000)) + require.ErrorIs(t, a.ValidateDurableTimestamp(context.Background(), 1_000_000), ErrTSOPhaseDInactive) + provider.setFloor(2_000_000) + gate.checkErr = nil + result, err := a.ReserveBatchAfter(context.Background(), 2, 0, true, true) + require.NoError(t, err) + require.Greater(t, result.PhaseDFloor, uint64(2_000_000)) + require.Equal(t, 1, provider.callCount()) + require.True(t, fsm.PhaseDActive()) +} + +func TestTSOPendingFreezesFloorAndAllowsOnlyScopedResolution(t *testing.T) { + gate := &testPhaseDGate{prepareErr: ErrTSOPhaseDActivationPending} + a, fsm, engine, provider := pendingTestAllocator(t, gate) + gate.check = func() { require.Zero(t, fsm.PendingPhaseDFloor()) } + gate.prepare = func(b uint64) { + require.Equal(t, b, fsm.PendingPhaseDFloor()) + require.False(t, fsm.PhaseDActive()) + require.Zero(t, fsm.AllocationFloor()) + } + _, err := a.ReserveBatchAfter(context.Background(), 3, 0, true, true) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + b := fsm.PendingPhaseDFloor() + require.NotZero(t, b) + require.Len(t, engine.proposedPayloads(), 2) + _, err = a.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + require.NoError(t, a.ValidateLegacyResolutionTimestamps(context.Background(), 1, b)) + require.ErrorIs(t, a.ValidateDurableTimestamp(context.Background(), b), ErrTSOPhaseDInactive) + for _, pair := range [][2]uint64{{0, b}, {b, b}, {1, b + 1}} { + require.ErrorIs(t, a.ValidateLegacyResolutionTimestamps(context.Background(), pair[0], pair[1]), ErrTSOTimestampInvalid) + } + gate.check = nil + gate.prepareErr = nil + engine.setTerm(2) + provider.setFloor(b + 1_000_000) + reservation, err := a.ReserveBatchAfter(context.Background(), 3, 0, true, true) + require.NoError(t, err) + require.Equal(t, b, reservation.PhaseDFloor) + require.Equal(t, b+1, reservation.Base) + require.Equal(t, []uint64{b, b}, gate.floors) + require.Equal(t, 1, provider.callCount(), "retry must not move the barrier") + require.Equal(t, b+3, fsm.AllocationFloor()) + require.ErrorIs(t, a.ValidateLegacyResolutionTimestamps(context.Background(), 1, b), ErrTSOPhaseDInactive) + require.ErrorIs(t, a.ValidateDurableTimestamp(context.Background(), b), ErrTSOTimestampPrePhaseD) + require.NoError(t, a.ValidateDurableTimestamp(context.Background(), b+1)) + payloads := engine.proposedPayloads() + require.Equal(t, marshalTSOPhaseD(b), payloads[2]) + require.Equal(t, marshalTSOAllocationFloor(b+3), payloads[3]) +} + +func TestTSOPendingSnapshotAndLeaderReplacementKeepBarrier(t *testing.T) { + gate := &testPhaseDGate{prepareErr: ErrTSOPhaseDActivationPending} + a, fsm, _, _ := pendingTestAllocator(t, gate) + _, err := a.ReserveBatchAfter(context.Background(), 2, 0, true, true) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + b := fsm.PendingPhaseDFloor() + snap, err := fsm.Snapshot() + require.NoError(t, err) + var buf bytes.Buffer + _, err = snap.WriteTo(&buf) + require.NoError(t, err) + require.Len(t, buf.Bytes(), tsoSnapshotV5Len) + restored := NewTSOStateMachine(NewHLC()) + require.NoError(t, restored.Restore(bytes.NewReader(buf.Bytes()))) + require.Equal(t, b, restored.PendingPhaseDFloor()) + engine := &recordingTSOEngine{state: raftengine.StateLeader, term: 2, + leader: raftengine.LeaderInfo{Address: "new"}, apply: applyTSOTestFSM(restored)} + gate.prepareErr = nil + newAlloc, err := NewRaftTSOAllocator(&ShardGroup{Engine: engine, TSOState: restored}, restored.hlc, + WithTSOCutoverFloorProvider(&recordingTSOFloorProvider{floor: b + 100}), + WithTSOPhaseDActivationGate(gate)) + require.NoError(t, err) + _, err = newAlloc.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + result, err := newAlloc.ReserveBatchAfter(context.Background(), 2, 0, true, true) + require.NoError(t, err) + require.Equal(t, b, result.PhaseDFloor) + require.Equal(t, b+1, result.Base) + activeSnap, err := restored.Snapshot() + require.NoError(t, err) + buf.Reset() + _, err = activeSnap.WriteTo(&buf) + require.NoError(t, err) + after := NewTSOStateMachine(NewHLC()) + require.NoError(t, after.Restore(bytes.NewReader(buf.Bytes()))) + require.True(t, after.PhaseDActive()) + require.Zero(t, after.PendingPhaseDFloor()) + require.Nil(t, after.Apply(marshalTSOPendingPhaseD(b))) +} + +func TestTSOPendingRejectsDifferentFloorAndAllocationBeforeActivation(t *testing.T) { + fsm := NewTSOStateMachine(NewHLC()) + require.Nil(t, fsm.Apply(marshalTSOCutover())) + require.Nil(t, fsm.Apply(marshalTSOPendingPhaseD(100))) + for _, payload := range [][]byte{marshalTSOPendingPhaseD(101), marshalTSOPhaseD(101), marshalTSOAllocationFloor(101)} { + require.Error(t, haltApplyOf(fsm.Apply(payload))) + } + require.True(t, fsm.IsVolatileOnlyPayload(marshalTSOPendingPhaseD(100))) + require.False(t, fsm.PhaseDActive()) +} + +func TestTSOPendingLeaderChangeDuringGateDoesNotActivate(t *testing.T) { + gate := &testPhaseDGate{} + a, fsm, engine, _ := pendingTestAllocator(t, gate) + gate.prepare = func(uint64) { engine.setTerm(2) } + _, err := a.ReserveBatchAfter(context.Background(), 1, 0, true, true) + require.ErrorIs(t, err, ErrTSONotLeader) + require.False(t, fsm.PhaseDActive()) + require.NotZero(t, fsm.PendingPhaseDFloor()) +} + +func TestTSOPendingLegacyValidationThroughDecorators(t *testing.T) { + gate := &testPhaseDGate{prepareErr: ErrTSOPhaseDActivationPending} + a, fsm, _, _ := pendingTestAllocator(t, gate) + _, err := a.ReserveBatchAfter(context.Background(), 1, 0, true, true) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + routed, err := NewLeaderRoutedTSOAllocator(a, engineForGroup(a.group)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, routed.Close()) }) + shadow := &ShadowTimestampAllocator{shadow: routed} + dynamic := NewDynamicTimestampAllocator(shadow) + batch, err := NewBatchAllocator(routed, 2) + require.NoError(t, err) + require.NoError(t, dynamic.ValidateLegacyResolutionTimestamps(context.Background(), 1, fsm.PendingPhaseDFloor())) + require.NoError(t, batch.ValidateLegacyResolutionTimestamps(context.Background(), 1, fsm.PendingPhaseDFloor())) + require.ErrorIs(t, batch.ValidateDurableTimestamp(context.Background(), 2), ErrTSOPhaseDInactive) +} diff --git a/kv/tso_pending_upgrade_test.go b/kv/tso_pending_upgrade_test.go new file mode 100644 index 000000000..6c14f5f47 --- /dev/null +++ b/kv/tso_pending_upgrade_test.go @@ -0,0 +1,123 @@ +package kv + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestTSOPendingHistoricalCheckFailureKeepsIssuingWithBoundedRetry(t *testing.T) { + for _, checkErr := range []error{ErrTSOProtocolUnsupported, ErrTSOPhaseDActivationPending, context.DeadlineExceeded} { + t.Run(checkErr.Error(), func(t *testing.T) { + gate := &testPhaseDGate{checkErr: checkErr} + a, fsm, engine, _ := pendingTestAllocator(t, gate) + require.Nil(t, fsm.Apply(marshalTSOCutover())) + require.Nil(t, fsm.Apply(marshalTSOPhaseD(100))) + for range 2 { + ts, err := a.Next(context.Background()) + require.NoError(t, err) + require.Greater(t, ts, uint64(100)) + } + require.Equal(t, 1, gate.checks, "ordinary requests must not repeatedly probe an incomplete rolling upgrade") + require.Zero(t, fsm.PrepareEvidenceProtocolFloor()) + require.Empty(t, gate.floors) + require.Empty(t, gate.finished) + engine.setTerm(2) + _, err := a.Next(context.Background()) + require.NoError(t, err) + require.Equal(t, 2, gate.checks, "a new term must not inherit the old term's probe cooldown") + gate.checkErr = nil + a.phaseDBootstrapRetryAt = time.Time{} + _, err = a.Next(context.Background()) + require.NoError(t, err) + require.Equal(t, uint64(100), fsm.PrepareEvidenceProtocolFloor()) + }) + } +} + +func TestTSOPendingHistoricalMarkerPrecedesSealAndPermanentlyDisablesFallback(t *testing.T) { + gate := &testPhaseDGate{sealErr: ErrTSOPhaseDActivationPending} + a, fsm, engine, _ := pendingTestAllocator(t, gate) + require.Nil(t, fsm.Apply(marshalTSOCutover())) + require.Nil(t, fsm.Apply(marshalTSOPhaseD(100))) + require.Nil(t, fsm.Apply(marshalTSOAllocationFloor(120))) + gate.seal = func() { require.Equal(t, uint64(100), fsm.PrepareEvidenceProtocolFloor()) } + _, err := a.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + require.Equal(t, [][]byte{marshalTSOPendingPhaseD(100)}, engine.proposedPayloads()) + require.Equal(t, uint64(120), fsm.AllocationFloor()) + engine.setTerm(2) + gate.checkErr = ErrTSOProtocolUnsupported + checks := gate.checks + _, err = a.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOProtocolUnsupported) + require.Equal(t, checks+1, gate.checks, "durable pending retries must recheck support without historical fallback") + gate.checkErr = nil + gate.sealErr = nil + gate.prepareErr = ErrTSOPhaseDActivationPending + _, err = a.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + gate.prepareErr = nil + gate.finishErr = ErrTSOPhaseDActivationPending + _, err = a.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOPhaseDActivationPending) + require.Equal(t, uint64(120), fsm.AllocationFloor()) + gate.finishErr = nil + ts, err := a.Next(context.Background()) + require.NoError(t, err) + require.Greater(t, ts, uint64(120)) + require.Equal(t, uint64(100), fsm.PrepareEvidenceProtocolFloor()) +} + +func TestTSOPendingHistoricalFailedCheckStillFencesLeadership(t *testing.T) { + gate := &testPhaseDGate{checkErr: ErrTSOProtocolUnsupported} + a, fsm, engine, _ := pendingTestAllocator(t, gate) + require.Nil(t, fsm.Apply(marshalTSOCutover())) + require.Nil(t, fsm.Apply(marshalTSOPhaseD(100))) + gate.check = func() { engine.setTerm(2) } + _, err := a.Next(context.Background()) + require.ErrorIs(t, err, ErrTSONotLeader) + require.Empty(t, engine.proposedPayloads()) + require.Zero(t, fsm.AllocationFloor()) +} + +func TestTSOPendingHistoricalLateAppliedMarkerCannotFallBack(t *testing.T) { + gate := &testPhaseDGate{checkErr: ErrTSOProtocolUnsupported, sealErr: ErrTSOPhaseDActivationPending} + a, fsm, _, _ := pendingTestAllocator(t, gate) + require.Nil(t, fsm.Apply(marshalTSOCutover())) + require.Nil(t, fsm.Apply(marshalTSOPhaseD(100))) + gate.check = func() { require.Nil(t, fsm.Apply(marshalTSOPendingPhaseD(100))) } + _, err := a.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOProtocolUnsupported) + require.Zero(t, fsm.AllocationFloor()) +} + +func TestTSOPendingActiveResumeEnablesNewGroupBeforeSeal(t *testing.T) { + gate := &testPhaseDGate{checkErr: ErrTSOProtocolUnsupported} + a, fsm, _, _ := pendingTestAllocator(t, gate) + require.Nil(t, fsm.Apply(marshalTSOCutover())) + require.Nil(t, fsm.Apply(marshalTSOPendingPhaseD(100))) + require.Nil(t, fsm.Apply(marshalTSOPhaseD(100))) + require.Nil(t, fsm.Apply(marshalTSOAllocationFloor(120))) + _, err := a.Next(context.Background()) + require.ErrorIs(t, err, ErrTSOProtocolUnsupported) + require.Equal(t, uint64(120), fsm.AllocationFloor()) + gate.checkErr = nil + var steps []string + enabled := false + gate.check = func() { + enabled = true + steps = append(steps, "check") + } + gate.seal = func() { + require.True(t, enabled, "a newly configured data group must be enabled before seal") + steps = append(steps, "seal") + } + gate.prepare = func(uint64) { steps = append(steps, "prepare") } + gate.finish = func(uint64) { steps = append(steps, "finish") } + _, err = a.Next(context.Background()) + require.NoError(t, err) + require.Equal(t, []string{"check", "seal", "prepare", "finish"}, steps) +} diff --git a/kv/tso_prephase_d.go b/kv/tso_prephase_d.go index 801b03677..9e00ddcdf 100644 --- a/kv/tso_prephase_d.go +++ b/kv/tso_prephase_d.go @@ -109,6 +109,13 @@ type PrimaryCommitEvidence struct { // It permits the first primary COMMIT to create its decision, but is not // evidence that a claimed commit timestamp was previously issued. PrimaryPrepared bool + // PreparedCommitTS is the chosen timestamp atomically persisted with the + // matching lock and intent. Zero identifies an unproven legacy PREPARE. + PreparedCommitTS uint64 + // LegacyPreparedAllowed keeps an already-active old Phase-D cluster + // resolvable while the new protocol drains its preexisting unproven locks. + // The authoritative creation seal or release permanently clears it. + LegacyPreparedAllowed bool } // PrimaryCommitEvidenceReader distinguishes a missing local record from a diff --git a/kv/tso_raft.go b/kv/tso_raft.go index 9b70c9e5a..ca53d9659 100644 --- a/kv/tso_raft.go +++ b/kv/tso_raft.go @@ -59,12 +59,16 @@ type tsoCutoverState interface { // Failed proposals may leak a local window, but they can never expose an // uncommitted timestamp or make a later leader reuse a returned timestamp. type RaftTSOAllocator struct { - group *ShardGroup - clock *HLC - state *TSOStateMachine - floorProvider TSOCutoverFloorProvider - initializedTerm uint64 - mu sync.Mutex + group *ShardGroup + clock *HLC + state *TSOStateMachine + floorProvider TSOCutoverFloorProvider + phaseDGate TSOPhaseDActivationGate + initializedTerm uint64 + phaseDFinishedTerm uint64 + phaseDBootstrapTerm uint64 + phaseDBootstrapRetryAt time.Time + mu sync.Mutex } type RaftTSOAllocatorOption func(*RaftTSOAllocator) @@ -158,6 +162,13 @@ func (a *RaftTSOAllocator) reserveBatchAfterLocked( if term == 0 { return empty, errors.Wrap(ErrTSONotLeader, "tso leader has no active term") } + pending, err := a.preparePhaseDActivation(ctx, engine, term, activatePhaseD) + if err != nil { + return empty, err + } + if pending != 0 { + return a.resumePendingPhaseD(ctx, engine, term, n, min, pending) + } previousFloor, commitPhaseD, err := a.prepareLeaderTermReservation(ctx, engine, term, activateCutover, activatePhaseD) if err != nil { return empty, err @@ -168,10 +179,7 @@ func (a *RaftTSOAllocator) reserveBatchAfterLocked( } reservationFloor := previousFloor if commitPhaseD { - reservationFloor, err = a.commitPhaseDReservation(ctx, engine, term, base) - if err != nil { - return empty, err - } + return a.beginPendingPhaseD(ctx, engine, term, n, min, base-1) } if err := a.commitAllocationFloor(ctx, engine, end); err != nil { return empty, err @@ -201,6 +209,13 @@ func (a *RaftTSOAllocator) reservePreparedLocalWindow( return 0, 0, errors.Wrapf(ErrTSOTimestampInvalid, "phase-D TSO minimum %d exceeds contiguous allocation floor %d", min, previousFloor) } + if commitPhaseD { + if previousFloor == ^uint64(0) { + return 0, 0, errors.WithStack(ErrTxnCommitTSRequired) + } + // Leave room for abortTSFrom(maxObservedStart, 0) below the fixed B. + return a.reserveLocalWindow(n, previousFloor+1, true) + } return a.reserveLocalWindow(n, max(min, previousFloor), phaseDContiguous) } @@ -274,25 +289,6 @@ func (a *RaftTSOAllocator) reservePhaseDWindow(n int, floor uint64) (uint64, uin return base, end, nil } -func (a *RaftTSOAllocator) commitPhaseDReservation( - ctx context.Context, - engine raftengine.Engine, - term uint64, - base uint64, -) (uint64, error) { - if base == 0 { - return 0, errors.Wrap(ErrTxnCommitTSRequired, "tso phase-D reservation base is zero") - } - reservationFloor := base - 1 - if err := a.commitPhaseD(ctx, engine, reservationFloor); err != nil { - return 0, err - } - if err := verifyTSOLeaderTerm(ctx, engine, term, true); err != nil { - return 0, err - } - return reservationFloor, nil -} - func (a *RaftTSOAllocator) prepareLeaderTermReservation( ctx context.Context, engine raftengine.Engine, @@ -359,8 +355,17 @@ func (a *RaftTSOAllocator) verifiedLeader(ctx context.Context) (raftengine.Engin if !isLeaderEngine(engine) { return nil, errors.WithStack(ErrTSONotLeader) } - if err := verifyLeaderEngineCtx(ctx, engine); err != nil { - return nil, errors.Wrap(stderrors.Join(ErrTSONotLeader, err), "tso verify leader") + if _, ok := ctx.Deadline(); !ok { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, verifyLeaderTimeout) + defer cancel() + } + term := engine.Status().Term + if _, err := linearizableReadEngineCtx(ctx, engine); err != nil { + return nil, errors.Wrap(stderrors.Join(ErrTSONotLeader, err), "tso verify applied leader state") + } + if status := engine.Status(); status.State != raftengine.StateLeader || status.Term != term { + return nil, errors.WithStack(ErrTSONotLeader) } return engine, nil } @@ -722,6 +727,9 @@ func (a *LeaderRoutedTSOAllocator) requestRemoteBatch( ActivatePhaseD: activatePhaseD, }) if err != nil { + if status.Code(err) == codes.ResourceExhausted { + return empty, errors.Wrap(ErrTSOPhaseDActivationPending, err.Error()) + } return empty, errors.Wrap(err, "tso request leader batch") } count := uint32(n) //nolint:gosec // n is validated before this request. @@ -780,7 +788,7 @@ func (a *LeaderRoutedTSOAllocator) ValidateDurableTimestamp(ctx context.Context, if lastErr == nil { return nil } - if !isTransientTSORouteError(lastErr) { + if errors.Is(lastErr, ErrTSOPhaseDInactive) || !isTransientTSORouteError(lastErr) { return lastErr } if err := waitTSORouteRetry(ctx, a.retryInterval); err != nil { @@ -796,14 +804,7 @@ func (a *LeaderRoutedTSOAllocator) requestRemoteValidation(ctx context.Context, } resp, err := pb.NewDistributionClient(conn).ValidateTimestamp(ctx, &pb.ValidateTimestampRequest{Timestamp: timestamp}) if err != nil { - code := status.Code(err) - if code == codes.OutOfRange { - return errors.Wrap(stderrors.Join(ErrTSOTimestampInvalid, ErrTSOTimestampPrePhaseD), err.Error()) - } - if code == codes.InvalidArgument { - return errors.Wrap(ErrTSOTimestampInvalid, err.Error()) - } - return errors.Wrap(err, "tso validate timestamp at leader") + return tsoValidationRPCError(err) } if !resp.GetValid() || !resp.GetPhaseDActive() { return errors.Wrapf(ErrTSOTimestampInvalid, diff --git a/kv/tso_raft_test.go b/kv/tso_raft_test.go index 0b6d84a9f..4a5a5b63e 100644 --- a/kv/tso_raft_test.go +++ b/kv/tso_raft_test.go @@ -204,6 +204,7 @@ func TestRaftTSOAllocatorRejectsTermChangeAfterPhaseDMarker(t *testing.T) { &ShardGroup{Engine: engine, TSOState: fsm}, clock, WithTSOCutoverFloorProvider(provider), + WithTSOPhaseDActivationGate(&testPhaseDGate{}), ) require.NoError(t, err) @@ -213,11 +214,12 @@ func TestRaftTSOAllocatorRejectsTermChangeAfterPhaseDMarker(t *testing.T) { require.Zero(t, fsm.AllocationFloor()) payloads := engine.proposedPayloads() - require.Len(t, payloads, 2) + require.Len(t, payloads, 3) require.Equal(t, []byte(tsoCutoverEnvelope), payloads[0]) - require.True(t, bytes.HasPrefix(payloads[1], []byte(tsoPhaseDEnvelope))) + require.Equal(t, marshalTSOPendingPhaseD(fsm.PhaseDFloor()), payloads[1]) + require.True(t, bytes.HasPrefix(payloads[2], []byte(tsoPhaseDEnvelope))) require.Greater(t, fsm.PhaseDFloor(), uint64(0)) - require.Equal(t, marshalTSOPhaseD(fsm.PhaseDFloor()), payloads[1]) + require.Equal(t, marshalTSOPhaseD(fsm.PhaseDFloor()), payloads[2]) } func TestRaftTSOAllocatorCommitsCutoverBeforeProductionWindow(t *testing.T) { @@ -257,6 +259,7 @@ func TestRaftTSOAllocatorCommitsPhaseDBeforeWindowAndValidatesRange(t *testing.T &ShardGroup{Engine: engine, TSOState: fsm}, clock, WithTSOCutoverFloorProvider(provider), + WithTSOPhaseDActivationGate(&testPhaseDGate{}), ) require.NoError(t, err) @@ -269,10 +272,11 @@ func TestRaftTSOAllocatorCommitsPhaseDBeforeWindowAndValidatesRange(t *testing.T require.Greater(t, reservation.Base, reservation.PhaseDFloor) payloads := engine.proposedPayloads() - require.Len(t, payloads, 3) + require.Len(t, payloads, 4) require.Equal(t, []byte(tsoCutoverEnvelope), payloads[0]) - require.Equal(t, marshalTSOPhaseD(reservation.Base-1), payloads[1]) - require.True(t, bytes.HasPrefix(payloads[2], []byte(tsoAllocationFloorEnvelope))) + require.Equal(t, marshalTSOPendingPhaseD(reservation.Base-1), payloads[1]) + require.Equal(t, marshalTSOPhaseD(reservation.Base-1), payloads[2]) + require.True(t, bytes.HasPrefix(payloads[3], []byte(tsoAllocationFloorEnvelope))) end := reservation.Base + positiveIntToUint64(reservation.Count) - 1 require.NoError(t, alloc.ValidateDurableTimestamp(context.Background(), reservation.Base)) @@ -299,6 +303,7 @@ func TestRaftTSOAllocatorPhaseDReservationsFollowWallClockJump(t *testing.T) { &ShardGroup{Engine: engine, TSOState: fsm}, clock, WithTSOCutoverFloorProvider(provider), + WithTSOPhaseDActivationGate(&testPhaseDGate{}), ) require.NoError(t, err) @@ -361,6 +366,7 @@ func TestRaftTSOAllocatorResamplesCommitFloorWhenActivatingPhaseDInInitializedTe clock, WithTSOCutoverFloorProvider(provider), ) + alloc.phaseDGate = &testPhaseDGate{} require.NoError(t, err) _, err = alloc.Next(context.Background()) @@ -862,7 +868,8 @@ type recordingTSOFloorProvider struct { func newTestRaftTSOAllocator(group *ShardGroup, clock *HLC) (*RaftTSOAllocator, error) { return NewRaftTSOAllocator(group, clock, - WithTSOCutoverFloorProvider(&recordingTSOFloorProvider{})) + WithTSOCutoverFloorProvider(&recordingTSOFloorProvider{}), + WithTSOPhaseDActivationGate(&testPhaseDGate{})) } func (p *recordingTSOFloorProvider) GlobalCommittedTimestampFloor(context.Context) (uint64, error) { diff --git a/kv/txn_codec.go b/kv/txn_codec.go index 679340559..87a1a8031 100644 --- a/kv/txn_codec.go +++ b/kv/txn_codec.go @@ -13,6 +13,7 @@ const ( txnMetaVersion byte = 1 txnMetaVersionV1 byte = txnMetaVersion txnMetaVersionV2 byte = 2 + txnMetaVersionV3 byte = 3 txnLockVersion byte = 1 txnIntentVersion byte = 1 txnCommitVersion byte = 1 @@ -20,7 +21,10 @@ const ( txnReadChunkSize = 4096 ) -const txnLockFlagPrimary byte = 0x01 +const ( + txnLockFlagPrimary byte = 0x01 + txnLockFlagCommitTS byte = 0x02 +) const ( txnMetaFlagLockTTL byte = 0x01 @@ -50,9 +54,17 @@ type TxnMeta struct { LockTTLms uint64 CommitTS uint64 PrevCommitTS uint64 + // PrepareCommitTS binds a PREPARE to its chosen commit timestamp. V3 is + // emitted only after all replicas support durable prepare evidence. + PrepareCommitTS uint64 } func EncodeTxnMeta(m TxnMeta) []byte { + if m.PrepareCommitTS != 0 { + encoded := encodeTxnMetaV2(m) + encoded[0] = txnMetaVersionV3 + return binary.BigEndian.AppendUint64(encoded, m.PrepareCommitTS) + } // Keep v1 as the default wire format until the cluster can guarantee that // every node understands v2 during rolling upgrades. The only field that // requires v2 is PrevCommitTS (the one-phase dedup probe), which is set @@ -120,6 +132,19 @@ func DecodeTxnMeta(b []byte) (TxnMeta, error) { return decodeTxnMetaV1(b) case txnMetaVersionV2: return decodeTxnMetaV2(b) + case txnMetaVersionV3: + if len(b) < txnMetaHeaderSize+2*uint64FieldSize { + return TxnMeta{}, errors.New("txn meta: prepared commit timestamp truncated") + } + meta, err := decodeTxnMetaV2(b[:len(b)-uint64FieldSize]) + if err != nil { + return TxnMeta{}, err + } + meta.PrepareCommitTS = binary.BigEndian.Uint64(b[len(b)-uint64FieldSize:]) + if meta.PrepareCommitTS == 0 { + return TxnMeta{}, errors.Wrap(ErrTxnInvalidMeta, "txn meta: prepared commit timestamp is zero") + } + return meta, nil default: return TxnMeta{}, errors.WithStack(errors.Newf("txn meta: unsupported version %d", b[0])) } @@ -222,6 +247,7 @@ func optionalV2Fields(m *TxnMeta) []struct { type txnLock struct { StartTS uint64 + CommitTS uint64 TTLExpireAt uint64 PrimaryKey []byte IsPrimaryKey bool @@ -230,6 +256,9 @@ type txnLock struct { func encodeTxnLock(l txnLock) []byte { // version(1) + StartTS(8) + TTLExpireAt(8) + flags(1) + primaryLen(8) + primaryKey size := 1 + uint64FieldSize + uint64FieldSize + 1 + uint64FieldSize + len(l.PrimaryKey) + if l.CommitTS != 0 { + size += uint64FieldSize + } b := make([]byte, size) b[0] = txnLockVersion binary.BigEndian.PutUint64(b[1:], l.StartTS) @@ -238,6 +267,10 @@ func encodeTxnLock(l txnLock) []byte { if l.IsPrimaryKey { flags |= txnLockFlagPrimary } + if l.CommitTS != 0 { + flags |= txnLockFlagCommitTS + binary.BigEndian.PutUint64(b[size-uint64FieldSize:], l.CommitTS) + } b[17] = flags binary.BigEndian.PutUint64(b[18:], uint64(len(l.PrimaryKey))) copy(b[26:], l.PrimaryKey) @@ -272,12 +305,28 @@ func decodeTxnLock(b []byte) (txnLock, error) { if err != nil { return txnLock{}, err } - return txnLock{ + lock := txnLock{ StartTS: startTS, TTLExpireAt: ttlExpireAt, PrimaryKey: primaryKey, IsPrimaryKey: (flags & txnLockFlagPrimary) != 0, - }, nil + } + if flags&txnLockFlagCommitTS != 0 { + return decodeTxnLockCommitEvidence(r, lock) + } + return lock, nil +} + +func decodeTxnLockCommitEvidence(r *bytes.Reader, lock txnLock) (txnLock, error) { + commitTS, err := readTxnUint64(r, "txn lock: commit ts truncated") + if err != nil { + return txnLock{}, err + } + if commitTS <= lock.StartTS || r.Len() != 0 { + return txnLock{}, errors.Wrap(ErrTxnInvalidMeta, "txn lock: invalid prepared commit timestamp") + } + lock.CommitTS = commitTS + return lock, nil } type txnIntent struct { diff --git a/kv/txn_keys.go b/kv/txn_keys.go index 8119f029d..466675ebd 100644 --- a/kv/txn_keys.go +++ b/kv/txn_keys.go @@ -87,7 +87,14 @@ func isTxnInternalKey(key []byte) bool { bytes.HasPrefix(key, txnRollbackPrefixBytes) || bytes.HasPrefix(key, txnSuccessPrefixBytes) || bytes.HasPrefix(key, txnMetaPrefixBytes) || - bytes.Equal(key, backupTimestampFloorKey) + isTxnPeerLocalControlKey(key) +} + +func isTxnPeerLocalControlKey(key []byte) bool { + return bytes.Equal(key, backupTimestampFloorKey) || + bytes.Equal(key, prepareEvidenceEnabledKey) || + bytes.Equal(key, prepareEvidenceGateKey) || + bytes.Equal(key, prepareEvidenceTimestampFloorKey) } func isTxnMetaKey(key []byte) bool { diff --git a/kv/txn_lock_evidence_test.go b/kv/txn_lock_evidence_test.go new file mode 100644 index 000000000..3e3975889 --- /dev/null +++ b/kv/txn_lock_evidence_test.go @@ -0,0 +1,49 @@ +package kv + +import ( + "encoding/hex" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTxnLockLegacyEncodingRemainsUnchanged(t *testing.T) { + t.Parallel() + lock := txnLock{StartTS: 10, TTLExpireAt: 30, PrimaryKey: []byte("p"), IsPrimaryKey: true} + wire, err := hex.DecodeString("01000000000000000a000000000000001e01000000000000000170") + require.NoError(t, err) + require.Equal(t, wire, encodeTxnLock(lock)) + decoded, err := decodeTxnLock(wire) + require.NoError(t, err) + require.Equal(t, lock, decoded) +} + +func TestTxnLockPreparedCommitEvidenceRoundTrip(t *testing.T) { + t.Parallel() + lock := txnLock{StartTS: 10, CommitTS: 20, TTLExpireAt: 30, PrimaryKey: []byte("p"), IsPrimaryKey: true} + wire := encodeTxnLock(lock) + decoded, err := decodeTxnLock(wire) + require.NoError(t, err) + require.Equal(t, lock, decoded) + for _, malformed := range [][]byte{ + wire[:len(wire)-1], append(append([]byte(nil), wire...), 0), + encodeTxnLock(txnLock{StartTS: 10, CommitTS: 10, PrimaryKey: []byte("p")}), + } { + _, err := decodeTxnLock(malformed) + require.Error(t, err) + } +} + +func TestTxnMetaPreparedCommitEvidenceRequiresV3(t *testing.T) { + t.Parallel() + meta := TxnMeta{PrimaryKey: []byte("p"), LockTTLms: 30, PrepareCommitTS: 20, PrevCommitTS: 15} + wire := EncodeTxnMeta(meta) + require.Equal(t, txnMetaVersionV3, wire[0]) + decoded, err := DecodeTxnMeta(wire) + require.NoError(t, err) + require.Equal(t, meta, decoded) + for _, malformed := range [][]byte{wire[:len(wire)-1], append(append([]byte(nil), wire...), 0)} { + _, err := DecodeTxnMeta(malformed) + require.Error(t, err) + } +} diff --git a/main.go b/main.go index edb064ea6..8f90828dd 100644 --- a/main.go +++ b/main.go @@ -2456,10 +2456,14 @@ func configureDedicatedCoordinatorTSO( observer kv.TSOObserver, ) (coordinatorTSOWiring, error) { var wiring coordinatorTSOWiring + opts := []kv.RaftTSOAllocatorOption{kv.WithTSOCutoverFloorProvider(floorProvider)} + if gate, ok := floorProvider.(kv.TSOPhaseDActivationGate); ok { + opts = append(opts, kv.WithTSOPhaseDActivationGate(gate)) + } local, err := kv.NewRaftTSOAllocator( tsoGroup, coordinate.Clock(), - kv.WithTSOCutoverFloorProvider(floorProvider), + opts..., ) if err != nil { return wiring, errors.Wrap(err, "configure dedicated tso allocator") @@ -3343,6 +3347,7 @@ func startRaftServers( pb.RegisterRawKVServer(gs, grpcSvc) pb.RegisterTransactionalKVServer(gs, grpcSvc) internalOpts := internalServerOptions(coordinate, adminServer, proposerForGroup(rt, shardGroups), shardGroups[rt.spec.id], rt.spec.id) + internalOpts = append(internalOpts, dedicatedTimestampCutoverOptions(shardGroups)...) staticServingServices := registerS3BlobFetchServer( gs, adminGRPCOpts.s3BlobFetchEnabled, @@ -3394,7 +3399,7 @@ func startRaftServers( rt.engine, []string{"RawKV"}, staticServingServices, - confChangeInterceptor, + newPrepareEvidenceMembershipGate(shardStore, rt.spec.id, confChangeInterceptor), ) reflection.Register(gs) diff --git a/main_prepare_evidence.go b/main_prepare_evidence.go new file mode 100644 index 000000000..e8b1ae332 --- /dev/null +++ b/main_prepare_evidence.go @@ -0,0 +1,37 @@ +package main + +import ( + "context" + + "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/internal/raftadmin" + "github.com/bootjp/elastickv/kv" + "github.com/cockroachdb/errors" +) + +func dedicatedTimestampCutoverOptions(groups map[uint64]*kv.ShardGroup) []adapter.InternalOption { + if group := groups[dedicatedTSORaftGroupID]; group != nil { + return []adapter.InternalOption{adapter.WithInternalTSOCutoverState(group.TSOState)} + } + return nil +} + +type prepareEvidenceMembershipGate struct { + probe func(context.Context, uint64, string) error + groupID uint64 + next raftadmin.MembershipChangeInterceptor +} + +func newPrepareEvidenceMembershipGate(shards *kv.ShardStore, groupID uint64, next raftadmin.MembershipChangeInterceptor) raftadmin.MembershipChangeInterceptor { + return &prepareEvidenceMembershipGate{probe: shards.CheckPrepareEvidenceMember, groupID: groupID, next: next} +} + +func (g *prepareEvidenceMembershipGate) PreAddMember(ctx context.Context, id, address string) error { + if err := g.probe(ctx, g.groupID, address); err != nil { + return err + } + if g.next != nil { + return errors.WithStack(g.next.PreAddMember(ctx, id, address)) + } + return nil +} diff --git a/main_prepare_evidence_test.go b/main_prepare_evidence_test.go new file mode 100644 index 000000000..b85c10ef9 --- /dev/null +++ b/main_prepare_evidence_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "context" + "errors" + "net" + "testing" + "time" + + "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type prepareSafetyNextHook func(context.Context, string, string) error + +func (f prepareSafetyNextHook) PreAddMember(ctx context.Context, id, address string) error { + return f(ctx, id, address) +} + +func TestPrepareEvidenceSafetyMainMembershipProbePrecedesExistingHook(t *testing.T) { + probeFailure, nextFailure := errors.New("unsupported member"), errors.New("existing hook rejected") + for _, probeErr := range []error{nil, probeFailure} { + var order []string + gate := &prepareEvidenceMembershipGate{groupID: 7, + probe: func(_ context.Context, group uint64, address string) error { + require.Equal(t, uint64(7), group) + require.Equal(t, "new:50051", address) + order = append(order, "prepare-protocol") + return probeErr + }, + next: prepareSafetyNextHook(func(_ context.Context, id, address string) error { + require.Equal(t, "new-node", id) + require.Equal(t, "new:50051", address) + order = append(order, "existing-hook") + return nextFailure + }), + } + err := gate.PreAddMember(context.Background(), "new-node", "new:50051") + if probeErr != nil { + require.ErrorIs(t, err, probeFailure) + require.Equal(t, []string{"prepare-protocol"}, order) + } else { + require.ErrorIs(t, err, nextFailure) + require.Equal(t, []string{"prepare-protocol", "existing-hook"}, order) + } + } +} + +func TestPrepareEvidenceSafetyMainAdmissionStillRejectsOldMemberAfterMarker(t *testing.T) { + for _, kind := range []string{"old-unsupported", "hlc-only", "fresh-dedicated-learner"} { + t.Run(kind, func(t *testing.T) { + lis, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + auth, _ := adapter.AdminTokenAuth("membership-token") + srv := grpc.NewServer(grpc.UnaryInterceptor(auth)) + if kind == "old-unsupported" { + pb.RegisterInternalServer(srv, &pb.UnimplementedInternalServer{}) + } else { + var options []adapter.InternalOption + if kind == "fresh-dedicated-learner" { + // Configured but not joined to group 0 yet: adoption is false. + options = append(options, adapter.WithInternalTSOCutoverState(kv.NewTSOStateMachine(kv.NewHLC()))) + } + pb.RegisterInternalServer(srv, adapter.NewInternalWithEngine(nil, nil, nil, nil, options...)) + } + go func() { _ = srv.Serve(lis) }() + t.Cleanup(func() { srv.Stop(); _ = lis.Close() }) + st := store.NewMVCCStore() + fsm := kv.NewKvFSMWithHLC(st, kv.NewHLC()) + require.Nil(t, fsm.Apply(kv.EncodePrepareEvidenceFence(0))) + group := &kv.ShardGroup{Store: st, Engine: &mainTSOEngine{state: raftengine.StateLeader}} + group.SetLeaderReadToken("membership-token") + _, _, _, groups := newActiveMainTSOCutover(t) + groups[7] = group + shards := kv.NewShardStore(nil, groups) + t.Cleanup(func() { require.NoError(t, shards.Close()) }) + called := false + next := prepareSafetyNextHook(func(context.Context, string, string) error { called = true; return nil }) + gate := newPrepareEvidenceMembershipGate(shards, 7, next) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + err = gate.PreAddMember(ctx, "later-member", lis.Addr().String()) + if kind == "fresh-dedicated-learner" { + require.NoError(t, err) + require.True(t, called) + } else { + require.ErrorIs(t, err, kv.ErrTSOProtocolUnsupported) + require.False(t, called, "admission after capability scan/marker cannot bypass the protocol hook") + } + }) + } +} diff --git a/main_tso_routing_test.go b/main_tso_routing_test.go index 748a04a04..9147f973b 100644 --- a/main_tso_routing_test.go +++ b/main_tso_routing_test.go @@ -82,8 +82,8 @@ func TestConfigureCoordinatorTSOPhaseDWorksThroughAdapterDecorators(t *testing.T require.NotZero(t, readTS.Timestamp()) require.True(t, fsm.CutoverActive()) require.True(t, fsm.PhaseDActive()) - require.Equal(t, uint64(3), engine.proposals.Load(), - "cutover and phase-D markers must commit before the timestamp window") + require.Equal(t, uint64(4), engine.proposals.Load(), + "cutover, pending, and phase-D markers must commit before the timestamp window") } func TestConfigureCoordinatorTSOPhaseDRequiresCutoverMode(t *testing.T) { @@ -403,6 +403,13 @@ func (p mainTSOFloorProvider) GlobalCommittedTimestampFloor(context.Context) (ui return p.floor, nil } +// These fixtures have no data groups; the tests exercise allocator/decorator +// wiring rather than the production data-group activation gate. +func (mainTSOFloorProvider) CheckPhaseDProtocol(context.Context) error { return nil } +func (mainTSOFloorProvider) SealPhaseDCreation(context.Context) error { return nil } +func (mainTSOFloorProvider) PreparePhaseD(context.Context, uint64) error { return nil } +func (mainTSOFloorProvider) FinishPhaseD(context.Context, uint64) error { return nil } + func (e *mainTSOEngine) Propose(_ context.Context, payload []byte) (*raftengine.ProposalResult, error) { e.proposals.Add(1) if e.tsoState != nil { diff --git a/proto/distribution.pb.go b/proto/distribution.pb.go index e33e05f7f..0cd1da1a2 100644 --- a/proto/distribution.pb.go +++ b/proto/distribution.pb.go @@ -585,10 +585,14 @@ func (x *GetTimestampResponse) GetPhaseDFloor() uint64 { } type ValidateTimestampRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Timestamp uint64 `protobuf:"varint,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Timestamp uint64 `protobuf:"varint,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + // A separate pre-pending drain / pending-D resolution check, never ordinary + // timestamp validity. The caller must verify durable transaction evidence. + LegacyResolution bool `protobuf:"varint,2,opt,name=legacy_resolution,json=legacyResolution,proto3" json:"legacy_resolution,omitempty"` + StartTimestamp uint64 `protobuf:"varint,3,opt,name=start_timestamp,json=startTimestamp,proto3" json:"start_timestamp,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ValidateTimestampRequest) Reset() { @@ -628,14 +632,31 @@ func (x *ValidateTimestampRequest) GetTimestamp() uint64 { return 0 } +func (x *ValidateTimestampRequest) GetLegacyResolution() bool { + if x != nil { + return x.LegacyResolution + } + return false +} + +func (x *ValidateTimestampRequest) GetStartTimestamp() uint64 { + if x != nil { + return x.StartTimestamp + } + return 0 +} + type ValidateTimestampResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Valid bool `protobuf:"varint,1,opt,name=valid,proto3" json:"valid,omitempty"` PhaseDActive bool `protobuf:"varint,2,opt,name=phase_d_active,json=phaseDActive,proto3" json:"phase_d_active,omitempty"` PhaseDFloor uint64 `protobuf:"varint,3,opt,name=phase_d_floor,json=phaseDFloor,proto3" json:"phase_d_floor,omitempty"` AllocationFloor uint64 `protobuf:"varint,4,opt,name=allocation_floor,json=allocationFloor,proto3" json:"allocation_floor,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Explicit acknowledgement prevents old servers ignoring the request mode + // from authorizing a legacy resolution through ordinary validation. + LegacyResolutionValidated bool `protobuf:"varint,5,opt,name=legacy_resolution_validated,json=legacyResolutionValidated,proto3" json:"legacy_resolution_validated,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ValidateTimestampResponse) Reset() { @@ -696,6 +717,13 @@ func (x *ValidateTimestampResponse) GetAllocationFloor() uint64 { return 0 } +func (x *ValidateTimestampResponse) GetLegacyResolutionValidated() bool { + if x != nil { + return x.LegacyResolutionValidated + } + return false +} + type RouteDescriptor struct { state protoimpl.MessageState `protogen:"open.v1"` RouteId uint64 `protobuf:"varint,1,opt,name=route_id,json=routeId,proto3" json:"route_id,omitempty"` @@ -2573,14 +2601,17 @@ const file_distribution_proto_rawDesc = "" + "\x19previous_allocation_floor\x18\x04 \x01(\x04R\x17previousAllocationFloor\x12%\n" + "\x0ecutover_active\x18\x05 \x01(\bR\rcutoverActive\x12$\n" + "\x0ephase_d_active\x18\x06 \x01(\bR\fphaseDActive\x12\"\n" + - "\rphase_d_floor\x18\a \x01(\x04R\vphaseDFloor\"8\n" + + "\rphase_d_floor\x18\a \x01(\x04R\vphaseDFloor\"\x8e\x01\n" + "\x18ValidateTimestampRequest\x12\x1c\n" + - "\ttimestamp\x18\x01 \x01(\x04R\ttimestamp\"\xa6\x01\n" + + "\ttimestamp\x18\x01 \x01(\x04R\ttimestamp\x12+\n" + + "\x11legacy_resolution\x18\x02 \x01(\bR\x10legacyResolution\x12'\n" + + "\x0fstart_timestamp\x18\x03 \x01(\x04R\x0estartTimestamp\"\xe6\x01\n" + "\x19ValidateTimestampResponse\x12\x14\n" + "\x05valid\x18\x01 \x01(\bR\x05valid\x12$\n" + "\x0ephase_d_active\x18\x02 \x01(\bR\fphaseDActive\x12\"\n" + "\rphase_d_floor\x18\x03 \x01(\x04R\vphaseDFloor\x12)\n" + - "\x10allocation_floor\x18\x04 \x01(\x04R\x0fallocationFloor\"\xfe\x02\n" + + "\x10allocation_floor\x18\x04 \x01(\x04R\x0fallocationFloor\x12>\n" + + "\x1blegacy_resolution_validated\x18\x05 \x01(\bR\x19legacyResolutionValidated\"\xfe\x02\n" + "\x0fRouteDescriptor\x12\x19\n" + "\broute_id\x18\x01 \x01(\x04R\arouteId\x12\x14\n" + "\x05start\x18\x02 \x01(\fR\x05start\x12\x10\n" + diff --git a/proto/distribution.proto b/proto/distribution.proto index b494f628c..dc4c7f93d 100644 --- a/proto/distribution.proto +++ b/proto/distribution.proto @@ -69,6 +69,10 @@ message GetTimestampResponse { message ValidateTimestampRequest { uint64 timestamp = 1; + // A separate pre-pending drain / pending-D resolution check, never ordinary + // timestamp validity. The caller must verify durable transaction evidence. + bool legacy_resolution = 2; + uint64 start_timestamp = 3; } message ValidateTimestampResponse { @@ -76,6 +80,9 @@ message ValidateTimestampResponse { bool phase_d_active = 2; uint64 phase_d_floor = 3; uint64 allocation_floor = 4; + // Explicit acknowledgement prevents old servers ignoring the request mode + // from authorizing a legacy resolution through ordinary validation. + bool legacy_resolution_validated = 5; } enum RouteState { diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 7dec69e86..0ec15cd3e 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -227,8 +227,12 @@ type Request struct { // checks only for those exact point mutations; prefixes and unmarked keys // still fail closed. WriteFenceBypassKeys [][]byte `protobuf:"bytes,7,rep,name=write_fence_bypass_keys,json=writeFenceBypassKeys,proto3" json:"write_fence_bypass_keys,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // timestamp_admission_floor identifies the data group's durable Phase-D + // creation epoch. Producers set it only after validating the unchanged + // persistence timestamp against the durable TSO allocation range. + TimestampAdmissionFloor uint64 `protobuf:"varint,8,opt,name=timestamp_admission_floor,json=timestampAdmissionFloor,proto3" json:"timestamp_admission_floor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Request) Reset() { @@ -310,6 +314,13 @@ func (x *Request) GetWriteFenceBypassKeys() [][]byte { return nil } +func (x *Request) GetTimestampAdmissionFloor() uint64 { + if x != nil { + return x.TimestampAdmissionFloor + } + return 0 +} + type RaftCommand struct { state protoimpl.MessageState `protogen:"open.v1"` Requests []*Request `protobuf:"bytes,1,rep,name=requests,proto3" json:"requests,omitempty"` @@ -643,6 +654,151 @@ func (x *ForwardLeaseReadResponse) GetLastCommitTs() uint64 { return 0 } +type PrepareEvidenceStateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Floor uint64 `protobuf:"varint,1,opt,name=floor,proto3" json:"floor,omitempty"` + // Capability probes are read-only and may be answered by any replica. + CapabilityOnly bool `protobuf:"varint,2,opt,name=capability_only,json=capabilityOnly,proto3" json:"capability_only,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PrepareEvidenceStateRequest) Reset() { + *x = PrepareEvidenceStateRequest{} + mi := &file_internal_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PrepareEvidenceStateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PrepareEvidenceStateRequest) ProtoMessage() {} + +func (x *PrepareEvidenceStateRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PrepareEvidenceStateRequest.ProtoReflect.Descriptor instead. +func (*PrepareEvidenceStateRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{9} +} + +func (x *PrepareEvidenceStateRequest) GetFloor() uint64 { + if x != nil { + return x.Floor + } + return 0 +} + +func (x *PrepareEvidenceStateRequest) GetCapabilityOnly() bool { + if x != nil { + return x.CapabilityOnly + } + return false +} + +type PrepareEvidenceStateResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Supported bool `protobuf:"varint,1,opt,name=supported,proto3" json:"supported,omitempty"` + ProofEnabled bool `protobuf:"varint,2,opt,name=proof_enabled,json=proofEnabled,proto3" json:"proof_enabled,omitempty"` + UnprovenLocks bool `protobuf:"varint,3,opt,name=unproven_locks,json=unprovenLocks,proto3" json:"unproven_locks,omitempty"` + FenceFloor uint64 `protobuf:"varint,4,opt,name=fence_floor,json=fenceFloor,proto3" json:"fence_floor,omitempty"` + AppliedIndex uint64 `protobuf:"varint,5,opt,name=applied_index,json=appliedIndex,proto3" json:"applied_index,omitempty"` + CutoverApplied bool `protobuf:"varint,6,opt,name=cutover_applied,json=cutoverApplied,proto3" json:"cutover_applied,omitempty"` + DedicatedTsoConfigured bool `protobuf:"varint,7,opt,name=dedicated_tso_configured,json=dedicatedTsoConfigured,proto3" json:"dedicated_tso_configured,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PrepareEvidenceStateResponse) Reset() { + *x = PrepareEvidenceStateResponse{} + mi := &file_internal_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PrepareEvidenceStateResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PrepareEvidenceStateResponse) ProtoMessage() {} + +func (x *PrepareEvidenceStateResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PrepareEvidenceStateResponse.ProtoReflect.Descriptor instead. +func (*PrepareEvidenceStateResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{10} +} + +func (x *PrepareEvidenceStateResponse) GetSupported() bool { + if x != nil { + return x.Supported + } + return false +} + +func (x *PrepareEvidenceStateResponse) GetProofEnabled() bool { + if x != nil { + return x.ProofEnabled + } + return false +} + +func (x *PrepareEvidenceStateResponse) GetUnprovenLocks() bool { + if x != nil { + return x.UnprovenLocks + } + return false +} + +func (x *PrepareEvidenceStateResponse) GetFenceFloor() uint64 { + if x != nil { + return x.FenceFloor + } + return 0 +} + +func (x *PrepareEvidenceStateResponse) GetAppliedIndex() uint64 { + if x != nil { + return x.AppliedIndex + } + return 0 +} + +func (x *PrepareEvidenceStateResponse) GetCutoverApplied() bool { + if x != nil { + return x.CutoverApplied + } + return false +} + +func (x *PrepareEvidenceStateResponse) GetDedicatedTsoConfigured() bool { + if x != nil { + return x.DedicatedTsoConfigured + } + return false +} + type RelayPublishRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Channel []byte `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` @@ -653,7 +809,7 @@ type RelayPublishRequest struct { func (x *RelayPublishRequest) Reset() { *x = RelayPublishRequest{} - mi := &file_internal_proto_msgTypes[9] + mi := &file_internal_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -665,7 +821,7 @@ func (x *RelayPublishRequest) String() string { func (*RelayPublishRequest) ProtoMessage() {} func (x *RelayPublishRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_msgTypes[9] + mi := &file_internal_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -678,7 +834,7 @@ func (x *RelayPublishRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RelayPublishRequest.ProtoReflect.Descriptor instead. func (*RelayPublishRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_rawDescGZIP(), []int{9} + return file_internal_proto_rawDescGZIP(), []int{11} } func (x *RelayPublishRequest) GetChannel() []byte { @@ -704,7 +860,7 @@ type RelayPublishResponse struct { func (x *RelayPublishResponse) Reset() { *x = RelayPublishResponse{} - mi := &file_internal_proto_msgTypes[10] + mi := &file_internal_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -716,7 +872,7 @@ func (x *RelayPublishResponse) String() string { func (*RelayPublishResponse) ProtoMessage() {} func (x *RelayPublishResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_msgTypes[10] + mi := &file_internal_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -729,7 +885,7 @@ func (x *RelayPublishResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RelayPublishResponse.ProtoReflect.Descriptor instead. func (*RelayPublishResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_rawDescGZIP(), []int{10} + return file_internal_proto_rawDescGZIP(), []int{12} } func (x *RelayPublishResponse) GetSubscribers() int64 { @@ -765,7 +921,7 @@ type ExportRangeVersionsRequest struct { func (x *ExportRangeVersionsRequest) Reset() { *x = ExportRangeVersionsRequest{} - mi := &file_internal_proto_msgTypes[11] + mi := &file_internal_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -777,7 +933,7 @@ func (x *ExportRangeVersionsRequest) String() string { func (*ExportRangeVersionsRequest) ProtoMessage() {} func (x *ExportRangeVersionsRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_msgTypes[11] + mi := &file_internal_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -790,7 +946,7 @@ func (x *ExportRangeVersionsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ExportRangeVersionsRequest.ProtoReflect.Descriptor instead. func (*ExportRangeVersionsRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_rawDescGZIP(), []int{11} + return file_internal_proto_rawDescGZIP(), []int{13} } func (x *ExportRangeVersionsRequest) GetRangeStart() []byte { @@ -888,7 +1044,7 @@ type ExportRangeVersionsResponse struct { func (x *ExportRangeVersionsResponse) Reset() { *x = ExportRangeVersionsResponse{} - mi := &file_internal_proto_msgTypes[12] + mi := &file_internal_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -900,7 +1056,7 @@ func (x *ExportRangeVersionsResponse) String() string { func (*ExportRangeVersionsResponse) ProtoMessage() {} func (x *ExportRangeVersionsResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_msgTypes[12] + mi := &file_internal_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -913,7 +1069,7 @@ func (x *ExportRangeVersionsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ExportRangeVersionsResponse.ProtoReflect.Descriptor instead. func (*ExportRangeVersionsResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_rawDescGZIP(), []int{12} + return file_internal_proto_rawDescGZIP(), []int{14} } func (x *ExportRangeVersionsResponse) GetVersions() []*MVCCVersion { @@ -951,7 +1107,7 @@ type MVCCVersion struct { func (x *MVCCVersion) Reset() { *x = MVCCVersion{} - mi := &file_internal_proto_msgTypes[13] + mi := &file_internal_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -963,7 +1119,7 @@ func (x *MVCCVersion) String() string { func (*MVCCVersion) ProtoMessage() {} func (x *MVCCVersion) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_msgTypes[13] + mi := &file_internal_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -976,7 +1132,7 @@ func (x *MVCCVersion) ProtoReflect() protoreflect.Message { // Deprecated: Use MVCCVersion.ProtoReflect.Descriptor instead. func (*MVCCVersion) Descriptor() ([]byte, []int) { - return file_internal_proto_rawDescGZIP(), []int{13} + return file_internal_proto_rawDescGZIP(), []int{15} } func (x *MVCCVersion) GetKey() []byte { @@ -1034,7 +1190,7 @@ type ImportRangeVersionsRequest struct { func (x *ImportRangeVersionsRequest) Reset() { *x = ImportRangeVersionsRequest{} - mi := &file_internal_proto_msgTypes[14] + mi := &file_internal_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1046,7 +1202,7 @@ func (x *ImportRangeVersionsRequest) String() string { func (*ImportRangeVersionsRequest) ProtoMessage() {} func (x *ImportRangeVersionsRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_msgTypes[14] + mi := &file_internal_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1059,7 +1215,7 @@ func (x *ImportRangeVersionsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ImportRangeVersionsRequest.ProtoReflect.Descriptor instead. func (*ImportRangeVersionsRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_rawDescGZIP(), []int{14} + return file_internal_proto_rawDescGZIP(), []int{16} } func (x *ImportRangeVersionsRequest) GetJobId() uint64 { @@ -1106,7 +1262,7 @@ type ImportRangeVersionsResponse struct { func (x *ImportRangeVersionsResponse) Reset() { *x = ImportRangeVersionsResponse{} - mi := &file_internal_proto_msgTypes[15] + mi := &file_internal_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1118,7 +1274,7 @@ func (x *ImportRangeVersionsResponse) String() string { func (*ImportRangeVersionsResponse) ProtoMessage() {} func (x *ImportRangeVersionsResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_msgTypes[15] + mi := &file_internal_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1131,7 +1287,7 @@ func (x *ImportRangeVersionsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ImportRangeVersionsResponse.ProtoReflect.Descriptor instead. func (*ImportRangeVersionsResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_rawDescGZIP(), []int{15} + return file_internal_proto_rawDescGZIP(), []int{17} } func (x *ImportRangeVersionsResponse) GetAckedCursor() []byte { @@ -1154,7 +1310,7 @@ type PromoteStagedVersionsRequest struct { func (x *PromoteStagedVersionsRequest) Reset() { *x = PromoteStagedVersionsRequest{} - mi := &file_internal_proto_msgTypes[16] + mi := &file_internal_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1166,7 +1322,7 @@ func (x *PromoteStagedVersionsRequest) String() string { func (*PromoteStagedVersionsRequest) ProtoMessage() {} func (x *PromoteStagedVersionsRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_msgTypes[16] + mi := &file_internal_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1179,7 +1335,7 @@ func (x *PromoteStagedVersionsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PromoteStagedVersionsRequest.ProtoReflect.Descriptor instead. func (*PromoteStagedVersionsRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_rawDescGZIP(), []int{16} + return file_internal_proto_rawDescGZIP(), []int{18} } func (x *PromoteStagedVersionsRequest) GetJobId() uint64 { @@ -1229,7 +1385,7 @@ type PromoteStagedVersionsResponse struct { func (x *PromoteStagedVersionsResponse) Reset() { *x = PromoteStagedVersionsResponse{} - mi := &file_internal_proto_msgTypes[17] + mi := &file_internal_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1241,7 +1397,7 @@ func (x *PromoteStagedVersionsResponse) String() string { func (*PromoteStagedVersionsResponse) ProtoMessage() {} func (x *PromoteStagedVersionsResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_msgTypes[17] + mi := &file_internal_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1254,7 +1410,7 @@ func (x *PromoteStagedVersionsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PromoteStagedVersionsResponse.ProtoReflect.Descriptor instead. func (*PromoteStagedVersionsResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_rawDescGZIP(), []int{17} + return file_internal_proto_rawDescGZIP(), []int{19} } func (x *PromoteStagedVersionsResponse) GetNextCursor() []byte { @@ -1294,7 +1450,7 @@ const file_internal_proto_rawDesc = "" + "\x02op\x18\x01 \x01(\x0e2\x03.OpR\x02op\x12\x10\n" + "\x03key\x18\x02 \x01(\fR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value\x123\n" + - "\x16commit_ts_value_offset\x18\x04 \x01(\x04R\x13commitTsValueOffset\"\x81\x02\n" + + "\x16commit_ts_value_offset\x18\x04 \x01(\x04R\x13commitTsValueOffset\"\xbd\x02\n" + "\aRequest\x12\x15\n" + "\x06is_txn\x18\x01 \x01(\bR\x05isTxn\x12\x1c\n" + "\x05phase\x18\x02 \x01(\x0e2\x06.PhaseR\x05phase\x12\x0e\n" + @@ -1302,7 +1458,8 @@ const file_internal_proto_rawDesc = "" + "\tmutations\x18\x04 \x03(\v2\t.MutationR\tmutations\x12\x1b\n" + "\tread_keys\x18\x05 \x03(\fR\breadKeys\x124\n" + "\x16observed_route_version\x18\x06 \x01(\x04R\x14observedRouteVersion\x125\n" + - "\x17write_fence_bypass_keys\x18\a \x03(\fR\x14writeFenceBypassKeys\"3\n" + + "\x17write_fence_bypass_keys\x18\a \x03(\fR\x14writeFenceBypassKeys\x12:\n" + + "\x19timestamp_admission_floor\x18\b \x01(\x04R\x17timestampAdmissionFloor\"3\n" + "\vRaftCommand\x12$\n" + "\brequests\x18\x01 \x03(\v2\b.RequestR\brequests\"M\n" + "\x0eForwardRequest\x12\x15\n" + @@ -1319,7 +1476,19 @@ const file_internal_proto_rawDesc = "" + "\x17ForwardLeaseReadRequest\"e\n" + "\x18ForwardLeaseReadResponse\x12#\n" + "\rapplied_index\x18\x01 \x01(\x04R\fappliedIndex\x12$\n" + - "\x0elast_commit_ts\x18\x02 \x01(\x04R\flastCommitTs\"I\n" + + "\x0elast_commit_ts\x18\x02 \x01(\x04R\flastCommitTs\"\\\n" + + "\x1bPrepareEvidenceStateRequest\x12\x14\n" + + "\x05floor\x18\x01 \x01(\x04R\x05floor\x12'\n" + + "\x0fcapability_only\x18\x02 \x01(\bR\x0ecapabilityOnly\"\xb1\x02\n" + + "\x1cPrepareEvidenceStateResponse\x12\x1c\n" + + "\tsupported\x18\x01 \x01(\bR\tsupported\x12#\n" + + "\rproof_enabled\x18\x02 \x01(\bR\fproofEnabled\x12%\n" + + "\x0eunproven_locks\x18\x03 \x01(\bR\runprovenLocks\x12\x1f\n" + + "\vfence_floor\x18\x04 \x01(\x04R\n" + + "fenceFloor\x12#\n" + + "\rapplied_index\x18\x05 \x01(\x04R\fappliedIndex\x12'\n" + + "\x0fcutover_applied\x18\x06 \x01(\bR\x0ecutoverApplied\x128\n" + + "\x18dedicated_tso_configured\x18\a \x01(\bR\x16dedicatedTsoConfigured\"I\n" + "\x13RelayPublishRequest\x12\x18\n" + "\achannel\x18\x01 \x01(\fR\achannel\x12\x18\n" + "\amessage\x18\x02 \x01(\fR\amessage\"8\n" + @@ -1387,11 +1556,12 @@ const file_internal_proto_rawDesc = "" + "\aPREPARE\x10\x01\x12\n" + "\n" + "\x06COMMIT\x10\x02\x12\t\n" + - "\x05ABORT\x10\x032\x9f\x04\n" + + "\x05ABORT\x10\x032\xf9\x04\n" + "\bInternal\x12.\n" + "\aForward\x12\x0f.ForwardRequest\x1a\x10.ForwardResponse\"\x00\x12U\n" + "\x14ForwardAdminProposal\x12\x1c.ForwardAdminProposalRequest\x1a\x1d.ForwardAdminProposalResponse\"\x00\x12I\n" + - "\x10ForwardLeaseRead\x12\x18.ForwardLeaseReadRequest\x1a\x19.ForwardLeaseReadResponse\"\x00\x12=\n" + + "\x10ForwardLeaseRead\x12\x18.ForwardLeaseReadRequest\x1a\x19.ForwardLeaseReadResponse\"\x00\x12X\n" + + "\x17GetPrepareEvidenceState\x12\x1c.PrepareEvidenceStateRequest\x1a\x1d.PrepareEvidenceStateResponse\"\x00\x12=\n" + "\fRelayPublish\x12\x14.RelayPublishRequest\x1a\x15.RelayPublishResponse\"\x00\x12T\n" + "\x13ExportRangeVersions\x12\x1b.ExportRangeVersionsRequest\x1a\x1c.ExportRangeVersionsResponse\"\x000\x01\x12R\n" + "\x13ImportRangeVersions\x12\x1b.ImportRangeVersionsRequest\x1a\x1c.ImportRangeVersionsResponse\"\x00\x12X\n" + @@ -1410,7 +1580,7 @@ func file_internal_proto_rawDescGZIP() []byte { } var file_internal_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 20) var file_internal_proto_goTypes = []any{ (Op)(0), // 0: Op (Phase)(0), // 1: Phase @@ -1423,15 +1593,17 @@ var file_internal_proto_goTypes = []any{ (*ForwardAdminProposalResponse)(nil), // 8: ForwardAdminProposalResponse (*ForwardLeaseReadRequest)(nil), // 9: ForwardLeaseReadRequest (*ForwardLeaseReadResponse)(nil), // 10: ForwardLeaseReadResponse - (*RelayPublishRequest)(nil), // 11: RelayPublishRequest - (*RelayPublishResponse)(nil), // 12: RelayPublishResponse - (*ExportRangeVersionsRequest)(nil), // 13: ExportRangeVersionsRequest - (*ExportRangeVersionsResponse)(nil), // 14: ExportRangeVersionsResponse - (*MVCCVersion)(nil), // 15: MVCCVersion - (*ImportRangeVersionsRequest)(nil), // 16: ImportRangeVersionsRequest - (*ImportRangeVersionsResponse)(nil), // 17: ImportRangeVersionsResponse - (*PromoteStagedVersionsRequest)(nil), // 18: PromoteStagedVersionsRequest - (*PromoteStagedVersionsResponse)(nil), // 19: PromoteStagedVersionsResponse + (*PrepareEvidenceStateRequest)(nil), // 11: PrepareEvidenceStateRequest + (*PrepareEvidenceStateResponse)(nil), // 12: PrepareEvidenceStateResponse + (*RelayPublishRequest)(nil), // 13: RelayPublishRequest + (*RelayPublishResponse)(nil), // 14: RelayPublishResponse + (*ExportRangeVersionsRequest)(nil), // 15: ExportRangeVersionsRequest + (*ExportRangeVersionsResponse)(nil), // 16: ExportRangeVersionsResponse + (*MVCCVersion)(nil), // 17: MVCCVersion + (*ImportRangeVersionsRequest)(nil), // 18: ImportRangeVersionsRequest + (*ImportRangeVersionsResponse)(nil), // 19: ImportRangeVersionsResponse + (*PromoteStagedVersionsRequest)(nil), // 20: PromoteStagedVersionsRequest + (*PromoteStagedVersionsResponse)(nil), // 21: PromoteStagedVersionsResponse } var file_internal_proto_depIdxs = []int32{ 0, // 0: Mutation.op:type_name -> Op @@ -1439,24 +1611,26 @@ var file_internal_proto_depIdxs = []int32{ 2, // 2: Request.mutations:type_name -> Mutation 3, // 3: RaftCommand.requests:type_name -> Request 3, // 4: ForwardRequest.requests:type_name -> Request - 15, // 5: ExportRangeVersionsResponse.versions:type_name -> MVCCVersion - 15, // 6: ImportRangeVersionsRequest.versions:type_name -> MVCCVersion + 17, // 5: ExportRangeVersionsResponse.versions:type_name -> MVCCVersion + 17, // 6: ImportRangeVersionsRequest.versions:type_name -> MVCCVersion 5, // 7: Internal.Forward:input_type -> ForwardRequest 7, // 8: Internal.ForwardAdminProposal:input_type -> ForwardAdminProposalRequest 9, // 9: Internal.ForwardLeaseRead:input_type -> ForwardLeaseReadRequest - 11, // 10: Internal.RelayPublish:input_type -> RelayPublishRequest - 13, // 11: Internal.ExportRangeVersions:input_type -> ExportRangeVersionsRequest - 16, // 12: Internal.ImportRangeVersions:input_type -> ImportRangeVersionsRequest - 18, // 13: Internal.PromoteStagedVersions:input_type -> PromoteStagedVersionsRequest - 6, // 14: Internal.Forward:output_type -> ForwardResponse - 8, // 15: Internal.ForwardAdminProposal:output_type -> ForwardAdminProposalResponse - 10, // 16: Internal.ForwardLeaseRead:output_type -> ForwardLeaseReadResponse - 12, // 17: Internal.RelayPublish:output_type -> RelayPublishResponse - 14, // 18: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse - 17, // 19: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse - 19, // 20: Internal.PromoteStagedVersions:output_type -> PromoteStagedVersionsResponse - 14, // [14:21] is the sub-list for method output_type - 7, // [7:14] is the sub-list for method input_type + 11, // 10: Internal.GetPrepareEvidenceState:input_type -> PrepareEvidenceStateRequest + 13, // 11: Internal.RelayPublish:input_type -> RelayPublishRequest + 15, // 12: Internal.ExportRangeVersions:input_type -> ExportRangeVersionsRequest + 18, // 13: Internal.ImportRangeVersions:input_type -> ImportRangeVersionsRequest + 20, // 14: Internal.PromoteStagedVersions:input_type -> PromoteStagedVersionsRequest + 6, // 15: Internal.Forward:output_type -> ForwardResponse + 8, // 16: Internal.ForwardAdminProposal:output_type -> ForwardAdminProposalResponse + 10, // 17: Internal.ForwardLeaseRead:output_type -> ForwardLeaseReadResponse + 12, // 18: Internal.GetPrepareEvidenceState:output_type -> PrepareEvidenceStateResponse + 14, // 19: Internal.RelayPublish:output_type -> RelayPublishResponse + 16, // 20: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse + 19, // 21: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse + 21, // 22: Internal.PromoteStagedVersions:output_type -> PromoteStagedVersionsResponse + 15, // [15:23] is the sub-list for method output_type + 7, // [7:15] is the sub-list for method input_type 7, // [7:7] is the sub-list for extension type_name 7, // [7:7] is the sub-list for extension extendee 0, // [0:7] is the sub-list for field type_name @@ -1473,7 +1647,7 @@ func file_internal_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_internal_proto_rawDesc), len(file_internal_proto_rawDesc)), NumEnums: 2, - NumMessages: 18, + NumMessages: 20, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/internal.proto b/proto/internal.proto index 4713cf430..3949d46e5 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -8,6 +8,7 @@ service Internal { rpc Forward(ForwardRequest) returns (ForwardResponse) {} rpc ForwardAdminProposal(ForwardAdminProposalRequest) returns (ForwardAdminProposalResponse) {} rpc ForwardLeaseRead(ForwardLeaseReadRequest) returns (ForwardLeaseReadResponse) {} + rpc GetPrepareEvidenceState(PrepareEvidenceStateRequest) returns (PrepareEvidenceStateResponse) {} rpc RelayPublish(RelayPublishRequest) returns (RelayPublishResponse) {} rpc ExportRangeVersions(ExportRangeVersionsRequest) returns (stream ExportRangeVersionsResponse) {} rpc ImportRangeVersions(ImportRangeVersionsRequest) returns (ImportRangeVersionsResponse) {} @@ -70,6 +71,10 @@ message Request { // checks only for those exact point mutations; prefixes and unmarked keys // still fail closed. repeated bytes write_fence_bypass_keys = 7; + // timestamp_admission_floor identifies the data group's durable Phase-D + // creation epoch. Producers set it only after validating the unchanged + // persistence timestamp against the durable TSO allocation range. + uint64 timestamp_admission_floor = 8; } message RaftCommand { @@ -103,6 +108,22 @@ message ForwardLeaseReadResponse { uint64 last_commit_ts = 2; } +message PrepareEvidenceStateRequest { + uint64 floor = 1; + // Capability probes are read-only and may be answered by any replica. + bool capability_only = 2; +} + +message PrepareEvidenceStateResponse { + bool supported = 1; + bool proof_enabled = 2; + bool unproven_locks = 3; + uint64 fence_floor = 4; + uint64 applied_index = 5; + bool cutover_applied = 6; + bool dedicated_tso_configured = 7; +} + message RelayPublishRequest { bytes channel = 1; bytes message = 2; diff --git a/proto/internal_grpc.pb.go b/proto/internal_grpc.pb.go index 51f41b71b..53778ea87 100644 --- a/proto/internal_grpc.pb.go +++ b/proto/internal_grpc.pb.go @@ -19,13 +19,14 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Internal_Forward_FullMethodName = "/Internal/Forward" - Internal_ForwardAdminProposal_FullMethodName = "/Internal/ForwardAdminProposal" - Internal_ForwardLeaseRead_FullMethodName = "/Internal/ForwardLeaseRead" - Internal_RelayPublish_FullMethodName = "/Internal/RelayPublish" - Internal_ExportRangeVersions_FullMethodName = "/Internal/ExportRangeVersions" - Internal_ImportRangeVersions_FullMethodName = "/Internal/ImportRangeVersions" - Internal_PromoteStagedVersions_FullMethodName = "/Internal/PromoteStagedVersions" + Internal_Forward_FullMethodName = "/Internal/Forward" + Internal_ForwardAdminProposal_FullMethodName = "/Internal/ForwardAdminProposal" + Internal_ForwardLeaseRead_FullMethodName = "/Internal/ForwardLeaseRead" + Internal_GetPrepareEvidenceState_FullMethodName = "/Internal/GetPrepareEvidenceState" + Internal_RelayPublish_FullMethodName = "/Internal/RelayPublish" + Internal_ExportRangeVersions_FullMethodName = "/Internal/ExportRangeVersions" + Internal_ImportRangeVersions_FullMethodName = "/Internal/ImportRangeVersions" + Internal_PromoteStagedVersions_FullMethodName = "/Internal/PromoteStagedVersions" ) // InternalClient is the client API for Internal service. @@ -36,6 +37,7 @@ type InternalClient interface { Forward(ctx context.Context, in *ForwardRequest, opts ...grpc.CallOption) (*ForwardResponse, error) ForwardAdminProposal(ctx context.Context, in *ForwardAdminProposalRequest, opts ...grpc.CallOption) (*ForwardAdminProposalResponse, error) ForwardLeaseRead(ctx context.Context, in *ForwardLeaseReadRequest, opts ...grpc.CallOption) (*ForwardLeaseReadResponse, error) + GetPrepareEvidenceState(ctx context.Context, in *PrepareEvidenceStateRequest, opts ...grpc.CallOption) (*PrepareEvidenceStateResponse, error) RelayPublish(ctx context.Context, in *RelayPublishRequest, opts ...grpc.CallOption) (*RelayPublishResponse, error) ExportRangeVersions(ctx context.Context, in *ExportRangeVersionsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ExportRangeVersionsResponse], error) ImportRangeVersions(ctx context.Context, in *ImportRangeVersionsRequest, opts ...grpc.CallOption) (*ImportRangeVersionsResponse, error) @@ -80,6 +82,16 @@ func (c *internalClient) ForwardLeaseRead(ctx context.Context, in *ForwardLeaseR return out, nil } +func (c *internalClient) GetPrepareEvidenceState(ctx context.Context, in *PrepareEvidenceStateRequest, opts ...grpc.CallOption) (*PrepareEvidenceStateResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(PrepareEvidenceStateResponse) + err := c.cc.Invoke(ctx, Internal_GetPrepareEvidenceState_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *internalClient) RelayPublish(ctx context.Context, in *RelayPublishRequest, opts ...grpc.CallOption) (*RelayPublishResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RelayPublishResponse) @@ -137,6 +149,7 @@ type InternalServer interface { Forward(context.Context, *ForwardRequest) (*ForwardResponse, error) ForwardAdminProposal(context.Context, *ForwardAdminProposalRequest) (*ForwardAdminProposalResponse, error) ForwardLeaseRead(context.Context, *ForwardLeaseReadRequest) (*ForwardLeaseReadResponse, error) + GetPrepareEvidenceState(context.Context, *PrepareEvidenceStateRequest) (*PrepareEvidenceStateResponse, error) RelayPublish(context.Context, *RelayPublishRequest) (*RelayPublishResponse, error) ExportRangeVersions(*ExportRangeVersionsRequest, grpc.ServerStreamingServer[ExportRangeVersionsResponse]) error ImportRangeVersions(context.Context, *ImportRangeVersionsRequest) (*ImportRangeVersionsResponse, error) @@ -160,6 +173,9 @@ func (UnimplementedInternalServer) ForwardAdminProposal(context.Context, *Forwar func (UnimplementedInternalServer) ForwardLeaseRead(context.Context, *ForwardLeaseReadRequest) (*ForwardLeaseReadResponse, error) { return nil, status.Error(codes.Unimplemented, "method ForwardLeaseRead not implemented") } +func (UnimplementedInternalServer) GetPrepareEvidenceState(context.Context, *PrepareEvidenceStateRequest) (*PrepareEvidenceStateResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetPrepareEvidenceState not implemented") +} func (UnimplementedInternalServer) RelayPublish(context.Context, *RelayPublishRequest) (*RelayPublishResponse, error) { return nil, status.Error(codes.Unimplemented, "method RelayPublish not implemented") } @@ -247,6 +263,24 @@ func _Internal_ForwardLeaseRead_Handler(srv interface{}, ctx context.Context, de return interceptor(ctx, in, info, handler) } +func _Internal_GetPrepareEvidenceState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PrepareEvidenceStateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).GetPrepareEvidenceState(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_GetPrepareEvidenceState_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).GetPrepareEvidenceState(ctx, req.(*PrepareEvidenceStateRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Internal_RelayPublish_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RelayPublishRequest) if err := dec(in); err != nil { @@ -331,6 +365,10 @@ var Internal_ServiceDesc = grpc.ServiceDesc{ MethodName: "ForwardLeaseRead", Handler: _Internal_ForwardLeaseRead_Handler, }, + { + MethodName: "GetPrepareEvidenceState", + Handler: _Internal_GetPrepareEvidenceState_Handler, + }, { MethodName: "RelayPublish", Handler: _Internal_RelayPublish_Handler, From 921c164b24d25a3e5a10014c25983bee26b3849a Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 23 Oct 2026 03:56:35 +0900 Subject: [PATCH 5/6] kv: preserve local protocol state during staged promotion --- ..._partial_prephase_d_resolution_evidence.md | 21 +- kv/fsm_migration_promote.go | 19 +- kv/migration_promote_control_state_test.go | 94 ++++++++ kv/prepare_abort_recovery_test.go | 74 +++++++ kv/tso_pending_joint_recovery_test.go | 203 ++++++++++++++++++ kv/tso_pending_routing.go | 2 +- kv/tso_raft.go | 5 +- 7 files changed, 409 insertions(+), 9 deletions(-) create mode 100644 kv/migration_promote_control_state_test.go create mode 100644 kv/prepare_abort_recovery_test.go create mode 100644 kv/tso_pending_joint_recovery_test.go diff --git a/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md b/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md index c32ea6200..98c47fd19 100644 --- a/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md +++ b/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md @@ -305,11 +305,22 @@ first primary COMMIT requires the durable chosen timestamp to match exactly. ### Acceptance Status -The implementation and focused regression tests are being integrated in the -existing PR. The document remains Partial until end-to-end forwarded cutover, -leader changes, pending restart/snapshot recovery, release retry, and late legacy -creation are verified together. Code presence or a classifier-only test is not -completion evidence for this protocol. +The existing PR includes a real nine-engine, three-group gRPC acceptance test +covering forwarded cutover, leadership transfer, release failures, and late +creation. Separate joint-recovery tests restore group-zero pending or active +snapshots together with partially released data groups, keep B fixed, and block +issuance until every release completes. Pebble snapshot/reopen tests cover both +V1 and V3 PREPARE followed by repeated ABORT and rejection of a later COMMIT. + +Restored pre-upgrade staging is checked again at promotion: a peer-local +control row refuses the whole chunk without moving ordinary rows, advancing +the migration checkpoint, or halting replication. The staged input is retained +for explicit recovery rather than silently deleted. Actual storage failures +retain their existing halt classification. + +These tests do not establish a full process-level rolling upgrade or production +stability. The document remains Partial until that deployment acceptance is +available; code presence and in-process tests alone do not establish it. ## 5. Open questions for review diff --git a/kv/fsm_migration_promote.go b/kv/fsm_migration_promote.go index 83365dde4..6f942799f 100644 --- a/kv/fsm_migration_promote.go +++ b/kv/fsm_migration_promote.go @@ -58,8 +58,25 @@ func (f *kvFSM) applyMigrationPromote(ctx context.Context, data []byte) any { if !ok { return haltErr(errors.Wrap(errors.Mark(store.ErrNotSupported, ErrMigrationPromoteApply), "kv/fsm: migration promote store")) } - result, err := promoter.PromoteVersions(ctx, migrationPromoteOptionsFromProto(req, f.pendingApplyIdx)) + opts := migrationPromoteOptionsFromProto(req, f.pendingApplyIdx) + mapTarget := opts.TargetKey + rejectedPeerLocal := false + opts.TargetKey = func(stagedKey []byte) ([]byte, bool) { + key, valid := mapTarget(stagedKey) + if valid && isTxnPeerLocalControlKey(key) { + rejectedPeerLocal = true + return nil, false + } + return key, valid + } + result, err := promoter.PromoteVersions(ctx, opts) if err != nil { + // A legacy staged control row is a deterministic input refusal, not + // a store failure that should halt replication. Promotion plans the + // entire chunk before applying it, so no row has been moved here. + if rejectedPeerLocal { + return errors.Wrap(ErrInvalidRequest, "migration cannot promote peer-local timestamp protocol state") + } if isMigrationPromoteOrdinaryApplyError(err) { return errors.Wrap(err, "kv/fsm: apply migration promote") } diff --git a/kv/migration_promote_control_state_test.go b/kv/migration_promote_control_state_test.go new file mode 100644 index 000000000..c0b037422 --- /dev/null +++ b/kv/migration_promote_control_state_test.go @@ -0,0 +1,94 @@ +package kv + +import ( + "bytes" + "fmt" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestMigrationPromoteRestoredStagingPreservesLocalControl(t *testing.T) { + for _, key := range [][]byte{backupTimestampFloorKey, prepareEvidenceEnabledKey, prepareEvidenceGateKey, prepareEvidenceTimestampFloorKey} { + for _, tombstone := range []bool{false, true} { + t.Run(fmt.Sprintf("%s-delete-%t", key, tombstone), func(t *testing.T) { + assertRestoredPromotionPreservesControl(t, key, tombstone) + }) + } + } +} + +func assertRestoredPromotionPreservesControl(t *testing.T, key []byte, tombstone bool) { + t.Helper() + ctx := t.Context() + source, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Close()) }() + fsm := newPrepareSafetyFSM(t, source) + require.NoError(t, fsm.persistBackupTimestampFloor(ctx, 100)) + for _, command := range [][]byte{EncodePrepareEvidenceFence(0), EncodePrepareEvidenceSeal(), EncodePrepareEvidenceFence(100), EncodePrepareEvidenceRelease(100)} { + require.Nil(t, fsm.Apply(command)) + } + before, err := source.GetAt(ctx, key, ^uint64(0)) + require.NoError(t, err) + // Directly seed the pre-upgrade staging representation, not the new import path. + userKey := []byte("!a-ordinary-user") + stagedUser := distribution.MigrationStagedDataKey(7, userKey) + stagedControl := distribution.MigrationStagedDataKey(7, key) + require.NoError(t, source.PutAt(ctx, stagedUser, []byte("user-value"), 1000, 0)) + if tombstone { + require.NoError(t, source.DeleteAt(ctx, stagedControl, 1000)) + } else { + require.NoError(t, source.PutAt(ctx, stagedControl, []byte("old-peer-state"), 1000, 0)) + } + snapshot, err := fsm.Snapshot() + require.NoError(t, err) + var raw bytes.Buffer + _, err = snapshot.WriteTo(&raw) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + dir := t.TempDir() + restoredStore, err := store.NewPebbleStore(dir) + require.NoError(t, err) + closed := false + t.Cleanup(func() { + if !closed { + require.NoError(t, restoredStore.Close()) + } + }) + restored := newPrepareSafetyFSM(t, restoredStore) + require.NoError(t, restored.Restore(bytes.NewReader(raw.Bytes()))) + command, err := MarshalMigrationPromoteCommand(&pb.PromoteStagedVersionsRequest{JobId: 7, MaxVersions: 10}) + require.NoError(t, err) + response := restored.Apply(command) + require.Nil(t, haltApplyOf(response), "old staging must be refused without halting replication") + applyErr, ok := response.(error) + require.True(t, ok, "unexpected promotion result %T: %v", response, response) + require.ErrorIs(t, applyErr, ErrInvalidRequest) + after, err := restoredStore.GetAt(ctx, key, ^uint64(0)) + require.NoError(t, err) + require.Equal(t, before, after) + _, err = restoredStore.GetAt(ctx, userKey, ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound, "the rejected chunk must not partially promote ordinary rows") + value, err := restoredStore.GetAt(ctx, stagedUser, ^uint64(0)) + require.NoError(t, err) + require.Equal(t, []byte("user-value"), value) + stateReader, ok := restoredStore.(store.MigrationPromotionStateReader) + require.True(t, ok) + _, exists, err := stateReader.MigrationPromotionState(ctx, 7) + require.NoError(t, err) + require.False(t, exists, "rejected staging must not advance migration progress") + require.NoError(t, restoredStore.Close()) + closed = true + reopenedStore, err := store.NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopenedStore.Close()) }) + reopened := newPrepareSafetyFSM(t, reopenedStore) + after, err = reopenedStore.GetAt(ctx, key, ^uint64(0)) + require.NoError(t, err) + require.Equal(t, before, after) + require.NoError(t, applyFSMRequest(t, reopened, prepareSafetyAdmittedCreation("raw", 2000, 100))) +} diff --git a/kv/prepare_abort_recovery_test.go b/kv/prepare_abort_recovery_test.go new file mode 100644 index 000000000..2d835b8b6 --- /dev/null +++ b/kv/prepare_abort_recovery_test.go @@ -0,0 +1,74 @@ +package kv + +import ( + "bytes" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestPreparedAbortRemainsFinalAcrossSnapshotAndReopen(t *testing.T) { + for _, tc := range []struct { + name string + version byte + proof uint64 + }{ + {name: "v1", version: txnMetaVersionV1}, + {name: "v3", version: txnMetaVersionV3, proof: 20}, + } { + t.Run(tc.name, func(t *testing.T) { + assertPreparedAbortRecovery(t, tc.proof, tc.version) + }) + } +} + +func assertPreparedAbortRecovery(t *testing.T, proof uint64, version byte) { + t.Helper() + source, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Close()) }() + fsm := newPrepareSafetyFSM(t, source) + if proof != 0 { + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + } + prepare := preparedEvidenceRequest(pb.Phase_PREPARE, proof) + require.Equal(t, version, prepare.GetMutations()[0].GetValue()[0]) + require.NoError(t, applyFSMRequest(t, fsm, prepare)) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + snapshot, err := fsm.Snapshot() + require.NoError(t, err) + var raw bytes.Buffer + _, err = snapshot.WriteTo(&raw) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + dir := t.TempDir() + restoredStore, err := store.NewPebbleStore(dir) + require.NoError(t, err) + closed := false + t.Cleanup(func() { + if !closed { + require.NoError(t, restoredStore.Close()) + } + }) + restored := newPrepareSafetyFSM(t, restoredStore) + require.NoError(t, restored.Restore(bytes.NewReader(raw.Bytes()))) + for range 2 { + require.NoError(t, applyFSMRequest(t, restored, preparedEvidenceRequest(pb.Phase_ABORT, 20))) + } + require.NoError(t, restoredStore.Close()) + closed = true + reopenedStore, err := store.NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopenedStore.Close()) }) + reopened := newPrepareSafetyFSM(t, reopenedStore) + require.NoError(t, applyFSMRequest(t, reopened, preparedEvidenceRequest(pb.Phase_ABORT, 20))) + require.ErrorIs(t, applyFSMRequest(t, reopened, preparedEvidenceRequest(pb.Phase_COMMIT, 20)), ErrTxnAlreadyAborted) + for _, key := range [][]byte{[]byte("primary"), txnLockKey([]byte("primary")), txnIntentKey([]byte("primary"))} { + _, err = reopenedStore.GetAt(t.Context(), key, ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound) + } + _, err = reopenedStore.GetAt(t.Context(), txnRollbackKey([]byte("primary"), 10), ^uint64(0)) + require.NoError(t, err) +} diff --git a/kv/tso_pending_joint_recovery_test.go b/kv/tso_pending_joint_recovery_test.go new file mode 100644 index 000000000..1cbc4aee8 --- /dev/null +++ b/kv/tso_pending_joint_recovery_test.go @@ -0,0 +1,203 @@ +package kv + +import ( + "bytes" + "context" + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +type jointRecoveryReleaseEngine struct { + *prepareSafetyEngine + floor uint64 + failure error + attempts int +} + +func (e *jointRecoveryReleaseEngine) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { + if bytes.Equal(data, EncodePrepareEvidenceRelease(e.floor)) { + e.attempts++ + if e.failure != nil { + return nil, errors.WithStack(e.failure) + } + } + return e.prepareSafetyEngine.ProposeAdmin(ctx, data) +} + +type jointRecoveryImage struct { + pending []byte + active []byte + data [2][]byte + replay [][]byte + floor uint64 +} + +func jointRecoverySnapshot(t *testing.T, fsm raftengine.StateMachine) []byte { + t.Helper() + snapshot, err := fsm.Snapshot() + require.NoError(t, err) + defer func() { require.NoError(t, snapshot.Close()) }() + var buf bytes.Buffer + _, err = snapshot.WriteTo(&buf) + require.NoError(t, err) + return buf.Bytes() +} + +func jointRecoveryTimestampGroup(state *TSOStateMachine, engine *recordingTSOEngine) *ShardGroup { + engine.leader.ID = "self" + return &ShardGroup{TSOState: state, Engine: &prepareSafetyEngine{ + recordingTSOEngine: engine, + members: []raftengine.Server{{ID: "self", Address: "self", Suffrage: "voter"}}, + }} +} + +func captureJointRecoveryImage(t *testing.T) jointRecoveryImage { + t.Helper() + shards, _, first := newPrepareSafetyGate(t) + other, secondEngine, second := newPrepareSafetyGate(t) + shards.groups[2] = other.groups[1] + t.Cleanup(func() { delete(shards.groups, 2) }) + allocator, state, engine, provider := pendingTestAllocator(t, shards) + allocator.group = jointRecoveryTimestampGroup(state, engine) + shards.groups[0] = allocator.group + provider.setFloor(100) + release := &jointRecoveryReleaseEngine{ + prepareSafetyEngine: secondEngine, + failure: errors.New("second data group release temporarily unavailable"), + } + shards.groups[2].Engine = release + var image jointRecoveryImage + engine.afterPropose = func(data []byte) { + if isTSOPendingPhaseDEntry(data) { + image.floor = state.PendingPhaseDFloor() + release.floor = image.floor + image.pending = jointRecoverySnapshot(t, state) + } + if bytes.HasPrefix(data, []byte(tsoPhaseDEnvelope)) { + image.replay = append(image.replay, bytes.Clone(data)) + } + } + reservation, err := allocator.ReserveBatchAfter(t.Context(), 2, 0, true, true) + require.ErrorIs(t, err, release.failure) + require.Equal(t, TSOReservation{}, reservation) + require.NotEmpty(t, image.pending) + require.Greater(t, image.floor, uint64(100)) + require.Equal(t, [][]byte{marshalTSOPhaseD(image.floor)}, image.replay) + require.True(t, state.PhaseDActive()) + require.Zero(t, state.AllocationFloor(), "partial release must not reserve a timestamp window") + require.Equal(t, 1, release.attempts) + assertJointRecoveryDataGate(t, first, image.floor, true) + assertJointRecoveryDataGate(t, second, image.floor, false) + image.active = jointRecoverySnapshot(t, state) + image.data[0] = jointRecoverySnapshot(t, first) + image.data[1] = jointRecoverySnapshot(t, second) + return image +} + +func assertJointRecoveryDataGate(t *testing.T, fsm *kvFSM, floor uint64, released bool) { + t.Helper() + sealed, releaseFloor, err := readPrepareEvidenceCreationGate(t.Context(), fsm.store) + require.NoError(t, err) + require.Equal(t, !released, sealed) + if released { + require.Equal(t, floor, releaseFloor) + } else { + require.Zero(t, releaseFloor) + } + enabled, unproven, actualFloor, err := InspectPrepareEvidence(t.Context(), fsm.store, floor) + require.NoError(t, err) + require.True(t, enabled) + require.False(t, unproven) + require.Equal(t, floor, actualFloor) +} + +// The pending snapshot predates the active log record. Recovery must replay +// that record before resuming a release which already reached one data group. +func TestTSOPendingJointRecoveryPreservesBarrierUntilAllDataGroupsRelease(t *testing.T) { + image := captureJointRecoveryImage(t) + for _, scenario := range []struct { + name string + snapshot []byte + replay [][]byte + }{ + {name: "pending_snapshot_with_active_log_replay", snapshot: image.pending, replay: image.replay}, + {name: "active_snapshot_with_partial_release", snapshot: image.active}, + } { + t.Run(scenario.name, func(t *testing.T) { + state := NewTSOStateMachine(NewHLC()) + require.NoError(t, state.Restore(bytes.NewReader(scenario.snapshot))) + require.Equal(t, image.floor, state.PrepareEvidenceProtocolFloor()) + require.Zero(t, state.AllocationFloor()) + shards, firstEngine, first := newPrepareSafetyGate(t) + other, secondEngine, second := newPrepareSafetyGate(t) + require.NoError(t, first.Restore(bytes.NewReader(image.data[0]))) + require.NoError(t, second.Restore(bytes.NewReader(image.data[1]))) + assertJointRecoveryDataGate(t, first, image.floor, true) + assertJointRecoveryDataGate(t, second, image.floor, false) + shards.groups[2] = other.groups[1] + t.Cleanup(func() { delete(shards.groups, 2) }) + engine := &recordingTSOEngine{state: raftengine.StateLeader, term: 2, + leader: raftengine.LeaderInfo{Address: "self"}, apply: applyTSOTestFSM(state)} + shards.groups[0] = jointRecoveryTimestampGroup(state, engine) + release := &jointRecoveryReleaseEngine{prepareSafetyEngine: secondEngine, + floor: image.floor, failure: errors.New("restored data group release temporarily unavailable")} + shards.groups[2].Engine = release + provider := &recordingTSOFloorProvider{floor: image.floor + 1000} + allocator, err := NewRaftTSOAllocator(shards.groups[0], state.hlc, + WithTSOCutoverFloorProvider(provider), WithTSOPhaseDActivationGate(shards)) + require.NoError(t, err) + if len(scenario.replay) != 0 { + require.False(t, state.PhaseDActive()) + require.Equal(t, image.floor, state.PendingPhaseDFloor()) + timestamp, issueErr := allocator.Next(t.Context()) + require.ErrorIs(t, issueErr, ErrTSOPhaseDActivationPending) + require.Zero(t, timestamp) + require.Empty(t, engine.proposedPayloads()) + for _, entry := range scenario.replay { + require.Nil(t, state.Apply(entry)) + } + } + require.True(t, state.PhaseDActive()) + require.Equal(t, image.floor, state.PhaseDFloor()) + timestamp, err := allocator.Next(t.Context()) + require.ErrorIs(t, err, release.failure) + require.Zero(t, timestamp) + require.Equal(t, 1, release.attempts, "the retry must reach the actual release fan-out") + require.Zero(t, provider.callCount(), "an unfinished release must not sample a replacement boundary") + require.Zero(t, state.AllocationFloor()) + require.Empty(t, engine.proposedPayloads(), "no allocation may be proposed before every data group releases") + assertJointRecoveryDataGate(t, first, image.floor, true) + assertJointRecoveryDataGate(t, second, image.floor, false) + require.ErrorIs(t, applyFSMRequest(t, second, + prepareSafetyAdmittedCreation("raw", image.floor+1, image.floor)), ErrPrepareCreationSealed) + release.failure = nil + base, err := allocator.NextBatch(t.Context(), 2) + require.NoError(t, err) + require.Greater(t, base, image.floor) + require.Equal(t, base+1, state.AllocationFloor()) + require.Equal(t, image.floor, state.PhaseDFloor()) + require.Equal(t, image.floor, state.PrepareEvidenceProtocolFloor()) + require.Zero(t, state.PendingPhaseDFloor()) + require.Equal(t, 2, release.attempts) + require.Equal(t, 2, jointRecoveryReleaseCount(firstEngine, image.floor), + "an already released group must safely accept both release retries") + for _, fsm := range []*kvFSM{first, second} { + assertJointRecoveryDataGate(t, fsm, image.floor, true) + require.NoError(t, applyFSMRequest(t, fsm, prepareSafetyAdmittedCreation("raw", base, image.floor))) + } + }) + } +} + +func jointRecoveryReleaseCount(engine *prepareSafetyEngine, floor uint64) int { + count := 0 + for _, payload := range engine.proposedPayloads() { + if bytes.Equal(payload, EncodePrepareEvidenceRelease(floor)) { + count++ + } + } + return count +} diff --git a/kv/tso_pending_routing.go b/kv/tso_pending_routing.go index 5df0968ce..d7032d273 100644 --- a/kv/tso_pending_routing.go +++ b/kv/tso_pending_routing.go @@ -53,7 +53,7 @@ func (a *LeaderRoutedTSOAllocator) ValidateLegacyResolutionTimestamps(ctx contex func (a *LeaderRoutedTSOAllocator) tryLegacyResolutionValidation(ctx context.Context, start, commit uint64) (string, error) { if a.local.IsLeader() { - return "local", validateLegacyResolutionThrough(ctx, a.local, start, commit) + return tsoRequestPathLocal, validateLegacyResolutionThrough(ctx, a.local, start, commit) } addr := leaderAddrFromEngine(a.leader) if addr == "" { diff --git a/kv/tso_raft.go b/kv/tso_raft.go index ca53d9659..5e23e7aad 100644 --- a/kv/tso_raft.go +++ b/kv/tso_raft.go @@ -18,6 +18,7 @@ import ( const ( defaultTSORouteRetryBudget = 5 * time.Second defaultTSORouteRetryInterval = 25 * time.Millisecond + tsoRequestPathLocal = "local" ) var ( @@ -656,7 +657,7 @@ func (a *LeaderRoutedTSOAllocator) tryBatch( if a.local.IsLeader() { started := time.Now() reservation, err := a.tryLocalBatch(ctx, n, min, activate, activatePhaseD, requireReservation) - a.observeRequest("reserve", "local", err, time.Since(started)) + a.observeRequest("reserve", tsoRequestPathLocal, err, time.Since(started)) return reservation, err } addr := leaderAddrFromEngine(a.leader) @@ -773,7 +774,7 @@ func (a *LeaderRoutedTSOAllocator) ValidateDurableTimestamp(ctx context.Context, } started := time.Now() lastErr = errors.WithStack(validator.ValidateDurableTimestamp(ctx, timestamp)) - a.observeRequest("validate", "local", lastErr, time.Since(started)) + a.observeRequest("validate", tsoRequestPathLocal, lastErr, time.Since(started)) } else { addr := leaderAddrFromEngine(a.leader) if addr == "" { From e535e5566183169fe68778c335c7c936d3b31701 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 23 Oct 2026 05:05:44 +0900 Subject: [PATCH 6/6] kv: preserve user namespaces and bound prepare evidence inspection --- adapter/admin_grpc.go | 40 ++-- adapter/internal.go | 3 + adapter/internal_phase_d_acceptance_test.go | 1 + adapter/internal_prepare_evidence.go | 35 +++- .../internal_prepare_evidence_budget_test.go | 198 ++++++++++++++++++ adapter/internal_prepare_evidence_test.go | 3 +- distribution/migrator.go | 9 +- ..._partial_prephase_d_resolution_evidence.md | 30 +++ kv/coordinator_prepare_evidence.go | 35 +++- kv/coordinator_prepare_evidence_test.go | 40 ++++ kv/fsm_prepare_evidence.go | 8 +- kv/prepare_evidence_namespace_test.go | 151 +++++++++++++ kv/shard_store_prepare_evidence_test.go | 2 +- main.go | 1 + main_prepare_evidence.go | 2 +- main_prepare_evidence_test.go | 43 +++- 16 files changed, 561 insertions(+), 40 deletions(-) create mode 100644 adapter/internal_prepare_evidence_budget_test.go create mode 100644 kv/coordinator_prepare_evidence_test.go create mode 100644 kv/prepare_evidence_namespace_test.go diff --git a/adapter/admin_grpc.go b/adapter/admin_grpc.go index 45ee81681..86f3f15f7 100644 --- a/adapter/admin_grpc.go +++ b/adapter/admin_grpc.go @@ -791,24 +791,8 @@ func bearerTokenAuth( if token == "" { return nil, nil } - expected := []byte(token) check := func(ctx context.Context) error { - md, ok := metadata.FromIncomingContext(ctx) - if !ok { - return status.Error(codes.Unauthenticated, "missing authorization metadata") - } - values := md.Get("authorization") - if len(values) == 0 { - return status.Error(codes.Unauthenticated, "missing authorization header") - } - got, ok := strings.CutPrefix(values[0], "Bearer ") - if !ok { - return status.Error(codes.Unauthenticated, "authorization is not a bearer token") - } - if subtle.ConstantTimeCompare([]byte(got), expected) != 1 { - return status.Errorf(codes.Unauthenticated, "invalid %s token", credentialName) - } - return nil + return checkBearerToken(ctx, token, credentialName) } unary := func( ctx context.Context, @@ -841,6 +825,28 @@ func bearerTokenAuth( return unary, stream } +func checkBearerToken(ctx context.Context, token, credentialName string) error { + if token == "" { + return status.Errorf(codes.Unauthenticated, "%s credential is not configured", credentialName) + } + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return errors.WithStack(status.Error(codes.Unauthenticated, "missing authorization metadata")) + } + values := md.Get("authorization") + if len(values) == 0 { + return errors.WithStack(status.Error(codes.Unauthenticated, "missing authorization header")) + } + got, ok := strings.CutPrefix(values[0], "Bearer ") + if !ok { + return errors.WithStack(status.Error(codes.Unauthenticated, "authorization is not a bearer token")) + } + if subtle.ConstantTimeCompare([]byte(got), []byte(token)) != 1 { + return status.Errorf(codes.Unauthenticated, "invalid %s token", credentialName) + } + return nil +} + // ErrAdminTokenRequired is returned by NewAdminServer helpers when the operator // failed to supply a token and also did not opt into insecure mode. var ErrAdminTokenRequired = errors.New("admin token file required; pass --adminInsecureNoAuth to run without") diff --git a/adapter/internal.go b/adapter/internal.go index 92ae5045c..f0b95eb3f 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -5,6 +5,7 @@ import ( "context" "os" "strings" + "sync/atomic" "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal" @@ -139,6 +140,8 @@ type Internal struct { store store.MVCCStore primaryCommitEvidence kv.PrimaryCommitEvidenceReader tsoCutoverState interface{ CutoverActive() bool } + prepareEvidenceToken string + prepareEvidenceScanning atomic.Bool migrationProposer raftengine.Proposer migrationImportGate func(context.Context) error migrationPromoteGate func(context.Context) error diff --git a/adapter/internal_phase_d_acceptance_test.go b/adapter/internal_phase_d_acceptance_test.go index e57dd440d..79edd8a56 100644 --- a/adapter/internal_phase_d_acceptance_test.go +++ b/adapter/internal_phase_d_acceptance_test.go @@ -129,6 +129,7 @@ func (c *phaseDAcceptanceCluster) serveNode(t *testing.T, node *phaseDAcceptance node.txns[group] = kv.NewTransactionWithProposer(result.Engine, kv.WithTimestampAdmission(shard.Store, func() kv.TimestampAllocator { return node.batch })) options := []InternalOption{WithInternalTimestampAllocator(node.batch), + WithInternalPrepareEvidenceToken(c.token), WithInternalTSOCutoverState(node.state), WithInternalPrimaryCommitEvidence(node.shards), WithInternalAdminProposer(result.Engine)} if group == 0 { diff --git a/adapter/internal_prepare_evidence.go b/adapter/internal_prepare_evidence.go index e3ed1c0e3..f7e99f577 100644 --- a/adapter/internal_prepare_evidence.go +++ b/adapter/internal_prepare_evidence.go @@ -2,16 +2,31 @@ package adapter import ( "context" + "time" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) +const prepareEvidenceInspectionTimeout = 5 * time.Second + +// WithInternalPrepareEvidenceToken supplies the existing forwarded-control +// credential. An omitted or empty credential leaves this peer RPC disabled, +// independently of whether the Admin service/interceptor is installed. +func WithInternalPrepareEvidenceToken(token string) InternalOption { + return func(i *Internal) { i.prepareEvidenceToken = token } +} + // GetPrepareEvidenceState serves read-only protocol negotiation on every // replica, and authoritative drain inspection only on an applied leader. func (i *Internal) GetPrepareEvidenceState(ctx context.Context, req *pb.PrepareEvidenceStateRequest) (*pb.PrepareEvidenceStateResponse, error) { + if err := checkBearerToken(ctx, i.prepareEvidenceToken, "prepare evidence peer"); err != nil { + return nil, err + } if req == nil { return nil, errors.WithStack(kv.ErrInvalidRequest) } @@ -21,17 +36,33 @@ func (i *Internal) GetPrepareEvidenceState(ctx context.Context, req *pb.PrepareE if i.kvForwardRejected || i.store == nil { return nil, errors.WithStack(ErrKVForwardNotSupported) } + return i.inspectPrepareEvidence(ctx, req.GetFloor()) +} + +func (i *Internal) inspectPrepareEvidence(ctx context.Context, requestedFloor uint64) (*pb.PrepareEvidenceStateResponse, error) { if i.leader == nil || i.leader.State() != raftengine.StateLeader { return nil, errors.WithStack(ErrNotLeader) } + if !i.prepareEvidenceScanning.CompareAndSwap(false, true) { + return nil, errors.WithStack(status.Error(codes.ResourceExhausted, "prepare evidence inspection already running")) + } + defer i.prepareEvidenceScanning.Store(false) + ctx, cancel := context.WithTimeout(ctx, prepareEvidenceInspectionTimeout) + defer cancel() index, err := i.leader.LinearizableRead(ctx) if err != nil { return nil, errors.WithStack(err) } - enabled, unproven, floor, err := kv.InspectPrepareEvidence(ctx, i.store, req.GetFloor()) + if err := ctx.Err(); err != nil { + return nil, errors.WithStack(status.FromContextError(err).Err()) + } + enabled, unproven, floor, err := kv.InspectPrepareEvidence(ctx, i.store, requestedFloor) if err != nil { return nil, errors.WithStack(err) } + if err := ctx.Err(); err != nil { + return nil, errors.WithStack(status.FromContextError(err).Err()) + } if i.leader.State() != raftengine.StateLeader { return nil, errors.WithStack(ErrNotLeader) } @@ -43,7 +74,7 @@ func (i *Internal) GetPrepareEvidenceState(ctx context.Context, req *pb.PrepareE func (i *Internal) prepareEvidenceCapabilities() *pb.PrepareEvidenceStateResponse { return &pb.PrepareEvidenceStateResponse{ - Supported: true, DedicatedTsoConfigured: i.tsoCutoverState != nil, + Supported: true, DedicatedTsoConfigured: i.tsoCutoverState != nil && i.tsAllocator != nil, CutoverApplied: i.tsoCutoverState != nil && i.tsoCutoverState.CutoverActive(), } } diff --git a/adapter/internal_prepare_evidence_budget_test.go b/adapter/internal_prepare_evidence_budget_test.go new file mode 100644 index 000000000..52d0be242 --- /dev/null +++ b/adapter/internal_prepare_evidence_budget_test.go @@ -0,0 +1,198 @@ +package adapter + +import ( + "context" + "errors" + "net" + "testing" + "time" + + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +const prepareEvidenceBudgetToken = "peer-token" + +func unfilteredPrepareEvidenceClient(t *testing.T, server *Internal) pb.InternalClient { + t.Helper() + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + srv := grpc.NewServer() + pb.RegisterInternalServer(srv, server) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(func() { srv.Stop(); _ = lis.Close() }) + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + return pb.NewInternalClient(conn) +} + +func TestPrepareEvidenceRPCWithoutAdminInterceptorRequiresCredential(t *testing.T) { + leader := &prepareSafetyRPCLeader{} + client := unfilteredPrepareEvidenceClient(t, + NewInternalWithEngine(nil, leader, nil, nil, WithInternalStore(store.NewMVCCStore()))) + for _, capability := range []bool{false, true} { + for _, token := range []string{"", "unconfigured-token"} { + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+token) + response, err := client.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{CapabilityOnly: capability}) + cancel() + require.Equal(t, codes.Unauthenticated, status.Code(err)) + require.Nil(t, response) + } + } + require.Zero(t, leader.reads.Load(), "missing server credential must refuse before quorum work") +} + +func TestPrepareEvidenceRPCChecksConfiguredCredentialWithoutInterceptor(t *testing.T) { + leader := &prepareSafetyRPCLeader{} + client := unfilteredPrepareEvidenceClient(t, NewInternalWithEngine(nil, leader, nil, nil, + WithInternalStore(store.NewMVCCStore()), WithInternalPrepareEvidenceToken(prepareEvidenceBudgetToken))) + for _, token := range []string{"", "wrong", "unrelated-blob-token"} { + ctx := metadata.AppendToOutgoingContext(t.Context(), "authorization", "Bearer "+token) + response, err := client.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + require.Nil(t, response) + } + require.Zero(t, leader.reads.Load()) + ctx := metadata.AppendToOutgoingContext(t.Context(), "authorization", "Bearer "+prepareEvidenceBudgetToken) + response, err := client.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{}) + require.NoError(t, err) + require.Equal(t, uint64(23), response.GetAppliedIndex()) + require.Equal(t, int32(1), leader.reads.Load()) +} + +type prepareEvidenceBudgetLeader struct { + prepareSafetyRPCLeader + read func(context.Context) (uint64, error) +} + +func (l *prepareEvidenceBudgetLeader) LinearizableRead(ctx context.Context) (uint64, error) { + l.reads.Add(1) + return l.read(ctx) +} + +func TestPrepareEvidenceRPCBoundsFenceAndPreservesEarlierDeadline(t *testing.T) { + for _, timeout := range []time.Duration{0, time.Second, time.Hour} { + t.Run(timeout.String(), func(t *testing.T) { + observed := make(chan time.Time, 1) + leader := &prepareEvidenceBudgetLeader{read: func(ctx context.Context) (uint64, error) { + deadline, _ := ctx.Deadline() + observed <- deadline + return 0, errors.New("stop after observing fence budget") + }} + client := unfilteredPrepareEvidenceClient(t, NewInternalWithEngine(nil, leader, nil, nil, + WithInternalStore(store.NewMVCCStore()), WithInternalPrepareEvidenceToken(prepareEvidenceBudgetToken))) + ctx := t.Context() + if timeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+prepareEvidenceBudgetToken) + response, err := client.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{}) + require.Error(t, err) + require.Nil(t, response) + deadline := <-observed + require.False(t, deadline.IsZero(), "the server must supply a deadline even when the caller does not") + require.Positive(t, time.Until(deadline)) + require.LessOrEqual(t, time.Until(deadline), 5*time.Second) + if timeout == time.Second { + require.LessOrEqual(t, time.Until(deadline), time.Second) + } + }) + } +} + +func TestPrepareEvidenceRPCRejectsConcurrentScansBeforeQuorumWork(t *testing.T) { + started := make(chan struct{}, 2) + release := make(chan struct{}, 2) + defer close(release) + leader := &prepareEvidenceBudgetLeader{read: func(ctx context.Context) (uint64, error) { + started <- struct{}{} + select { + case <-release: + return 23, nil + case <-ctx.Done(): + return 0, ctx.Err() + } + }} + client := unfilteredPrepareEvidenceClient(t, NewInternalWithEngine(nil, leader, nil, nil, + WithInternalStore(store.NewMVCCStore()), WithInternalPrepareEvidenceToken(prepareEvidenceBudgetToken))) + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+prepareEvidenceBudgetToken) + finished := make(chan error, 1) + go func() { + _, err := client.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{}) + finished <- err + }() + select { + case <-started: + case <-ctx.Done(): + t.Fatal("first inspection did not reach its leader fence") + } + response, err := client.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{}) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) + require.Nil(t, response) + require.Equal(t, int32(1), leader.reads.Load()) + capability, err := client.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{CapabilityOnly: true}) + require.NoError(t, err) + require.True(t, capability.GetSupported(), "bounded inspection must not stall capability negotiation") + release <- struct{}{} + require.NoError(t, <-finished) + release <- struct{}{} + _, err = client.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{}) + require.NoError(t, err, "a completed scan must release its concurrency permit") + require.Equal(t, int32(2), leader.reads.Load()) +} + +type prepareEvidenceBudgetStore struct { + store.MVCCStore + scan func(context.Context, []byte, []byte, int, uint64) ([]*store.KVPair, error) +} + +func (s *prepareEvidenceBudgetStore) ScanAt(ctx context.Context, start, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { + return s.scan(ctx, start, end, limit, ts) +} + +func TestPrepareEvidenceRPCScanSharesFenceDeadlineAndNeverReturnsPartialDrain(t *testing.T) { + base := store.NewMVCCStore() + require.Nil(t, kv.NewKvFSMWithHLC(base, kv.NewHLC()).Apply(kv.EncodePrepareEvidenceFence(0))) + st := &prepareEvidenceBudgetStore{MVCCStore: base} + observed := make(chan time.Time, 2) + scans := make(chan struct{}, 2) + st.scan = func(ctx context.Context, start, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { + scans <- struct{}{} + if len(scans) == 1 { + return base.ScanAt(ctx, start, end, limit, ts) + } + deadline, _ := ctx.Deadline() + observed <- deadline + <-ctx.Done() + return nil, ctx.Err() + } + leader := &prepareEvidenceBudgetLeader{read: func(ctx context.Context) (uint64, error) { + deadline, _ := ctx.Deadline() + observed <- deadline + return 23, nil + }} + client := unfilteredPrepareEvidenceClient(t, NewInternalWithEngine(nil, leader, nil, nil, + WithInternalStore(st), WithInternalPrepareEvidenceToken(prepareEvidenceBudgetToken))) + ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+prepareEvidenceBudgetToken) + response, err := client.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{}) + require.Equal(t, codes.DeadlineExceeded, status.Code(err)) + require.Nil(t, response, "an incomplete staged-lock inspection cannot attest that locks drained") + require.Eventually(t, func() bool { return len(observed) == 2 }, time.Second, time.Millisecond) + require.Equal(t, <-observed, <-observed, "the staged scan must not reset the leader fence's budget") + require.Len(t, scans, 2) +} diff --git a/adapter/internal_prepare_evidence_test.go b/adapter/internal_prepare_evidence_test.go index 326057b0d..5d5b4e56a 100644 --- a/adapter/internal_prepare_evidence_test.go +++ b/adapter/internal_prepare_evidence_test.go @@ -51,6 +51,7 @@ func prepareSafetyInternalClient(t *testing.T, server *Internal) pb.InternalClie lis, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0") require.NoError(t, err) auth, _ := AdminTokenAuth("prepare-safety-token") + WithInternalPrepareEvidenceToken("prepare-safety-token")(server) srv := grpc.NewServer(grpc.UnaryInterceptor(auth)) pb.RegisterInternalServer(srv, server) go func() { _ = srv.Serve(lis) }() @@ -116,7 +117,7 @@ func TestPrepareEvidenceSafetyCapabilityReportsLocalCutoverOnFollowers(t *testin state := &prepareSafetyCutoverState{} var options []InternalOption if configured { - options = append(options, WithInternalTSOCutoverState(state)) + options = append(options, WithInternalTSOCutoverState(state), WithInternalTimestampAllocator(&phaseDForwardAllocator{})) } client := prepareSafetyInternalClient(t, NewInternalWithEngine(nil, leader, nil, nil, options...)) ctx, cancel := context.WithTimeout(context.Background(), time.Second) diff --git a/distribution/migrator.go b/distribution/migrator.go index 06acdbe7b..b45cb2360 100644 --- a/distribution/migrator.go +++ b/distribution/migrator.go @@ -73,9 +73,9 @@ const ( // cannot import kv without a cycle, and only this exact control key is // reserved; user keys under !txn|backup| remain migratable. migrationTxnBackupTimestampFloorKey = "!txn|backup|timestamp_floor" - migrationTxnPrepareEvidenceEnabledKey = "!txn|backup|prepare_evidence_enabled" - migrationTxnPrepareEvidenceGateKey = "!txn|backup|prepare_evidence_gate" - migrationTxnPrepareEvidenceFloorKey = "!txn|backup|prepare_evidence_timestamp_floor" + migrationTxnPrepareEvidenceEnabledKey = migrationTxnMetaPrefix + "prepare_evidence/enabled" + migrationTxnPrepareEvidenceGateKey = migrationTxnMetaPrefix + "prepare_evidence/gate" + migrationTxnPrepareEvidenceFloorKey = migrationTxnMetaPrefix + "prepare_evidence/timestamp_floor" migrationRedisPrefix = "!redis|" migrationHashPrefix = "!hs|" migrationSetPrefix = "!st|" @@ -321,6 +321,9 @@ func SplitJobBracketProgressForPlan(brackets []MigrationBracket, phase SplitJobE // interval after applying bracket-local exclusions. Route ownership still // requires the caller's RouteKeyFilter for every bracket. func (b MigrationBracket) ContainsRawKey(rawKey []byte) bool { + if hasAnyExactKey(rawKey, migrationInternalExactKeys) { + return false + } if bytes.Compare(rawKey, b.Start) < 0 { return false } diff --git a/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md b/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md index 98c47fd19..dcea3f548 100644 --- a/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md +++ b/docs/design/2026_09_02_partial_prephase_d_resolution_evidence.md @@ -234,6 +234,24 @@ Membership admission checks the same protocol before adding another member. A fresh learner may join before applying cutover, but a cutover cluster requires that it configure the dedicated timestamp runtime. Its catch-up must precede public serving. Old binaries cannot be admitted after this protocol is required. +Dedicated-runtime capability requires both a non-nil group-zero state and an +allocator actually installed in the serving coordinator. Merely hosting the TSO +FSM in legacy warm-up is not readiness, even if it later applies cutover. + +The prepare-evidence RPC checks the configured forwarded-control credential at +the handler boundary, including capability-only probes. An absent credential +disables this RPC even when Admin is disabled or explicitly insecure; disabling +Admin authentication does not make peer inspection public. The current runtime +uses its existing leader-read/admin control credential, not the S3 blob token or +an unmerged migration-controller credential. Every per-group registration must +provide that credential, and peer callers already attach it through their group. +Each data-group handler permits at most one authoritative inspection at a time; +additional requests return a retryable resource-exhausted result before a quorum +read or scan. The applied read fence and all scan pages share one server-owned +five-second deadline, shortened by an earlier caller deadline. Capability-only +negotiation performs no scan and does not wait behind an inspection. Exhaustion, +cancellation and scan errors never attest that old locks have drained; callers +retry activation rather than using a partial result. The replicated enable marker then requires proof for every new PREPARE in its data group. An existing unproven lock may retry with its original identity and @@ -292,6 +310,18 @@ The enable, seal, and release records belong to the local group, not its user ranges. User mutations cannot forge them, and range migration does not export or promote them as application data. The seal is a deterministic refusal, not an operational error that halts a raw-batch replica. +Their exact keys live under the already-reserved `!txn|meta|prepare_evidence/` +namespace. The earlier draft's `!txn|backup|prepare_evidence_*` names were valid +user keys before this change; their values are never decoded as protocol state, +overwritten by activation, or excluded from migration. No data rewrite or backup +floor wire-format change is needed. Migration excludes the exact peer-local +keys even from the transaction-metadata family bracket. +The earlier layout was unreleased and was not deployed to production. There is +no content-based fallback: genuine prototype protocol rows cannot be +distinguished from valid legacy user rows, even when their bytes match exactly. +A known deployment of that unreleased layout must stop and use an explicitly +validated offline recovery or rebuild; silently reinterpreting or dropping its +active state during an in-place upgrade is not supported. ### Already Active Clusters diff --git a/kv/coordinator_prepare_evidence.go b/kv/coordinator_prepare_evidence.go index 7b397335e..ce460f35b 100644 --- a/kv/coordinator_prepare_evidence.go +++ b/kv/coordinator_prepare_evidence.go @@ -8,22 +8,42 @@ import ( func (c *ShardedCoordinator) prepareEvidenceMetas(ctx context.Context, primaryKey []byte, commitTS uint64, ids []uint64) (map[uint64]*pb.Mutation, error) { metas := make(map[uint64]*pb.Mutation, len(ids)) + needsReservation := false for _, id := range ids { - meta, err := c.prepareEvidenceMeta(ctx, primaryKey, commitTS, []uint64{id}) + meta, err := c.prepareEvidenceTxnMeta(ctx, primaryKey, commitTS, []uint64{id}) if err != nil { return nil, err } - metas[id] = meta + needsReservation = needsReservation || meta.PrepareCommitTS != 0 + metas[id] = &pb.Mutation{Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(meta)} + } + if needsReservation { + if err := reservePrepareEvidenceTimestamp(ctx, c.tsAllocator, commitTS); err != nil { + return nil, err + } } return metas, nil } func (c *ShardedCoordinator) prepareEvidenceMeta(ctx context.Context, primaryKey []byte, commitTS uint64, groupIDs []uint64) (*pb.Mutation, error) { + meta, err := c.prepareEvidenceTxnMeta(ctx, primaryKey, commitTS, groupIDs) + if err != nil { + return nil, err + } + if meta.PrepareCommitTS != 0 { + if err := reservePrepareEvidenceTimestamp(ctx, c.tsAllocator, commitTS); err != nil { + return nil, err + } + } + return &pb.Mutation{Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(meta)}, nil +} + +func (c *ShardedCoordinator) prepareEvidenceTxnMeta(ctx context.Context, primaryKey []byte, commitTS uint64, groupIDs []uint64) (TxnMeta, error) { meta := TxnMeta{PrimaryKey: primaryKey, LockTTLms: defaultTxnLockTTLms, PrepareCommitTS: commitTS} for _, id := range groupIDs { group, err := c.txnGroupForID(id) if err != nil { - return nil, err + return TxnMeta{}, err } if group.Store == nil { meta.PrepareCommitTS = 0 @@ -31,19 +51,14 @@ func (c *ShardedCoordinator) prepareEvidenceMeta(ctx context.Context, primaryKey } enabled, err := PrepareEvidenceEnabled(ctx, group.Store) if err != nil { - return nil, err + return TxnMeta{}, err } if !enabled { meta.PrepareCommitTS = 0 break } } - if meta.PrepareCommitTS != 0 { - if err := reservePrepareEvidenceTimestamp(ctx, c.tsAllocator, commitTS); err != nil { - return nil, err - } - } - return &pb.Mutation{Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(meta)}, nil + return meta, nil } // Reservation advances the durable TSO range without changing values already diff --git a/kv/coordinator_prepare_evidence_test.go b/kv/coordinator_prepare_evidence_test.go new file mode 100644 index 000000000..a247472bb --- /dev/null +++ b/kv/coordinator_prepare_evidence_test.go @@ -0,0 +1,40 @@ +package kv + +import ( + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestPrepareEvidenceMetasReservesSharedTimestampOnce(t *testing.T) { + for _, phaseD := range []bool{false, true} { + t.Run(map[bool]string{false: "pre-phase-d", true: "phase-d"}[phaseD], func(t *testing.T) { + groups := make(map[uint64]*ShardGroup) + for _, id := range []uint64{1, 2, 3} { + st := store.NewMVCCStore() + groups[id] = &ShardGroup{Store: st, Txn: NewTransactionWithProposer(&recordingTSOEngine{})} + if id != 3 { + require.Nil(t, NewKvFSMWithHLC(st, NewHLC()).Apply(EncodePrepareEvidenceFence(0))) + } + } + allocator := &phaseDTestAllocator{phaseDActive: phaseD, phaseDRequired: phaseD} + coord := NewShardedCoordinator(distribution.NewEngine(), groups, 1, NewHLC(), nil).WithTSOAllocator(allocator) + metas, err := coord.prepareEvidenceMetas(t.Context(), []byte("primary"), 100, []uint64{1, 2, 3}) + require.NoError(t, err) + require.Len(t, metas, 3) + for id, mutation := range metas { + meta, err := DecodeTxnMeta(mutation.Value) + require.NoError(t, err) + if id == 3 { + require.Zero(t, meta.PrepareCommitTS) + } else { + require.Equal(t, uint64(100), meta.PrepareCommitTS) + } + } + require.Equal(t, uint64(1), allocator.nextCalls.Load()+allocator.validateCalls.Load(), + "one transaction shares one reservation/validation across enabled groups") + }) + } +} diff --git a/kv/fsm_prepare_evidence.go b/kv/fsm_prepare_evidence.go index d1fff1a82..5591f6512 100644 --- a/kv/fsm_prepare_evidence.go +++ b/kv/fsm_prepare_evidence.go @@ -10,10 +10,12 @@ import ( "github.com/cockroachdb/errors" ) -var prepareEvidenceEnabledKey = []byte(txnBackupPrefix + "prepare_evidence_enabled") +// This entire namespace was reserved before the proof protocol. The old +// !txn|backup|prepare_evidence_* names remain ordinary user data. +var prepareEvidenceEnabledKey = []byte(txnMetaPrefix + "prepare_evidence/enabled") var prepareEvidenceEnabledValue = []byte{1} -var prepareEvidenceGateKey = []byte(txnBackupPrefix + "prepare_evidence_gate") -var prepareEvidenceTimestampFloorKey = []byte(txnBackupPrefix + "prepare_evidence_timestamp_floor") +var prepareEvidenceGateKey = []byte(txnMetaPrefix + "prepare_evidence/gate") +var prepareEvidenceTimestampFloorKey = []byte(txnMetaPrefix + "prepare_evidence/timestamp_floor") func (f *kvFSM) validatePrepareEvidenceMeta(meta TxnMeta, startTS uint64) error { if meta.PrepareCommitTS == 0 { diff --git a/kv/prepare_evidence_namespace_test.go b/kv/prepare_evidence_namespace_test.go new file mode 100644 index 000000000..70d1fdea6 --- /dev/null +++ b/kv/prepare_evidence_namespace_test.go @@ -0,0 +1,151 @@ +package kv + +import ( + "bytes" + "encoding/binary" + "sync" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func legacyPrepareEvidenceUserKeys() [][]byte { + return [][]byte{[]byte("!txn|backup|prepare_evidence_enabled"), + []byte("!txn|backup|prepare_evidence_gate"), []byte("!txn|backup|prepare_evidence_timestamp_floor")} +} + +func prepareNamespaceStore(t *testing.T, pebble bool, dir string) (store.MVCCStore, func()) { + t.Helper() + var st store.MVCCStore + if pebble { + var err error + st, err = store.NewPebbleStore(dir) + require.NoError(t, err) + } else { + st = store.NewMVCCStore() + } + closeStore := sync.OnceFunc(func() { require.NoError(t, st.Close()) }) + t.Cleanup(closeStore) + return st, closeStore +} + +func prepareNamespaceHistory(t *testing.T, st store.MVCCStore, key []byte) []store.MVCCVersion { + t.Helper() + exported, err := st.ExportVersions(t.Context(), store.ExportVersionsOptions{ + StartKey: key, EndKey: append(bytes.Clone(key), 0), MaxVersions: 10, + }) + require.NoError(t, err) + return exported.Versions +} + +func assertPrepareNamespaceHistory(t *testing.T, want, got []store.MVCCVersion) { + t.Helper() + require.Len(t, got, len(want)) + for n, version := range want { + require.Equal(t, version.Key, got[n].Key) + require.Equal(t, version.CommitTS, got[n].CommitTS) + require.Equal(t, version.Tombstone, got[n].Tombstone) + require.Equal(t, version.ExpireAt, got[n].ExpireAt) + require.Equal(t, version.KeyFamily, got[n].KeyFamily) + // Snapshot encoding need not distinguish nil from empty tombstone bytes. + require.True(t, bytes.Equal(version.Value, got[n].Value)) + } +} + +func TestPrepareEvidenceUpgradePreservesPreviouslyValidUserNames(t *testing.T) { + for _, pebble := range []bool{false, true} { + t.Run(map[bool]string{false: "memory", true: "pebble"}[pebble], func(t *testing.T) { + for _, profile := range []string{"ordinary-values", "wire-shaped-values", "tombstones"} { + t.Run(profile, func(t *testing.T) { assertPrepareNamespaceUpgrade(t, pebble, profile) }) + } + }) + } +} + +func assertPrepareNamespaceUpgrade(t *testing.T, pebble bool, profile string) { + t.Helper() + dir := t.TempDir() + st, closeSource := prepareNamespaceStore(t, pebble, dir) + keys := legacyPrepareEvidenceUserKeys() + values := [][]byte{[]byte("user-enabled"), []byte("user-gate"), []byte("user-floor")} + if profile != "ordinary-values" { + values = [][]byte{{1}, binary.BigEndian.AppendUint64([]byte{prepareCreationReleased}, 100), + binary.BigEndian.AppendUint64(nil, 100)} + } + histories := make(map[string][]store.MVCCVersion) + for n, key := range keys { + require.NoError(t, st.PutAt(t.Context(), key, []byte("previous-user-version"), 1000, 0)) + require.NoError(t, st.PutAt(t.Context(), key, values[n], 2000, ^uint64(0))) + if profile == "tombstones" { + require.NoError(t, st.DeleteAt(t.Context(), key, 3000)) + } + histories[string(key)] = prepareNamespaceHistory(t, st, key) + } + fsm := newPrepareSafetyFSM(t, st) + require.NoError(t, fsm.backupFloorLoadErr) + enabled, err := PrepareEvidenceEnabled(t.Context(), st) + require.NoError(t, err) + require.False(t, enabled, "user bytes must not manufacture an enable marker") + sealed, released, err := readPrepareEvidenceCreationGate(t.Context(), st) + require.NoError(t, err) + require.False(t, sealed) + require.Zero(t, released) + floor, err := readPrepareEvidenceFenceFloor(t.Context(), st) + require.NoError(t, err) + require.Zero(t, floor) + for _, command := range [][]byte{EncodePrepareEvidenceFence(0), EncodePrepareEvidenceSeal(), + EncodePrepareEvidenceFence(100), EncodePrepareEvidenceRelease(100)} { + require.Nil(t, fsm.Apply(command)) + } + image := jointRecoverySnapshot(t, fsm) + closeSource() + reopened, _ := prepareNamespaceStore(t, pebble, dir) + restarted := newPrepareSafetyFSM(t, reopened) + require.NoError(t, restarted.backupFloorLoadErr) + require.NoError(t, restarted.Restore(bytes.NewReader(image))) + assertJointRecoveryDataGate(t, restarted, 100, true) + for n, key := range keys { + assertPrepareNamespaceHistory(t, histories[string(key)], prepareNamespaceHistory(t, reopened, key)) + value, err := reopened.GetAt(t.Context(), key, 2000) + require.NoError(t, err) + require.Equal(t, values[n], value) + require.False(t, isTxnInternalKey(key)) + require.False(t, distribution.IsMigrationKnownInternalKey(key)) + } + assertPrepareNamespaceMigration(t, pebble, histories) + for _, key := range keys { + require.NoError(t, applyFSMRequest(t, restarted, &pb.Request{Ts: 4000, TimestampAdmissionFloor: 100, + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("updated-user-value")}}})) + value, err := reopened.GetAt(t.Context(), key, ^uint64(0)) + require.NoError(t, err) + require.Equal(t, []byte("updated-user-value"), value) + } +} + +func assertPrepareNamespaceMigration(t *testing.T, pebble bool, histories map[string][]store.MVCCVersion) { + t.Helper() + target, _ := prepareNamespaceStore(t, pebble, t.TempDir()) + fsm := newPrepareSafetyFSM(t, target) + request := &pb.ImportRangeVersionsRequest{JobId: 7, BracketId: 1, BatchSeq: 1, Cursor: []byte("end")} + for _, key := range legacyPrepareEvidenceUserKeys() { + for _, version := range histories[string(key)] { + request.Versions = append(request.Versions, &pb.MVCCVersion{Key: version.Key, CommitTs: version.CommitTS, + Value: version.Value, Tombstone: version.Tombstone, ExpireAt: version.ExpireAt, KeyFamily: version.KeyFamily}) + } + } + command, err := MarshalMigrationImportCommand(request) + require.NoError(t, err) + require.IsType(t, store.ImportVersionsResult{}, fsm.Apply(command)) + command, err = MarshalMigrationPromoteCommand(&pb.PromoteStagedVersionsRequest{JobId: 7, MaxVersions: 100}) + require.NoError(t, err) + require.IsType(t, store.PromoteVersionsResult{}, fsm.Apply(command)) + for key, history := range histories { + assertPrepareNamespaceHistory(t, history, prepareNamespaceHistory(t, target, []byte(key))) + } + enabled, err := PrepareEvidenceEnabled(t.Context(), target) + require.NoError(t, err) + require.False(t, enabled, "migrated user bytes must not enable the protocol") +} diff --git a/kv/shard_store_prepare_evidence_test.go b/kv/shard_store_prepare_evidence_test.go index ce7552824..a0809be18 100644 --- a/kv/shard_store_prepare_evidence_test.go +++ b/kv/shard_store_prepare_evidence_test.go @@ -228,7 +228,7 @@ func TestPrepareEvidenceSafetyMarkerBlocksLateLegacyImportAtAnyStart(t *testing. } func TestPrepareEvidenceSafetyMarkerIsExcludedFromMigration(t *testing.T) { - brackets, err := distribution.PlanExportBrackets([]byte(txnBackupPrefix), []byte("!txn|backup}")) + brackets, err := distribution.PlanExportBrackets([]byte(txnBackupPrefix), []byte("!txn|meta}")) require.NoError(t, err) for _, key := range [][]byte{backupTimestampFloorKey, prepareEvidenceEnabledKey, prepareEvidenceGateKey, prepareEvidenceTimestampFloorKey} { require.True(t, distribution.IsMigrationKnownInternalKey(key), diff --git a/main.go b/main.go index c84508bc1..cafd5ea87 100644 --- a/main.go +++ b/main.go @@ -3704,6 +3704,7 @@ func startRaftServers( coordinate.Clock(), relay, append(internalOpts, + adapter.WithInternalPrepareEvidenceToken(adminGRPCOpts.token), adapter.WithInternalStore(rt.store), adapter.WithInternalPrimaryCommitEvidence(shardStore), adapter.WithInternalMigrationProposer(proposerForGroup(rt, shardGroups)), diff --git a/main_prepare_evidence.go b/main_prepare_evidence.go index e8b1ae332..43e85e4c7 100644 --- a/main_prepare_evidence.go +++ b/main_prepare_evidence.go @@ -10,7 +10,7 @@ import ( ) func dedicatedTimestampCutoverOptions(groups map[uint64]*kv.ShardGroup) []adapter.InternalOption { - if group := groups[dedicatedTSORaftGroupID]; group != nil { + if group := groups[dedicatedTSORaftGroupID]; group != nil && group.TSOState != nil { return []adapter.InternalOption{adapter.WithInternalTSOCutoverState(group.TSOState)} } return nil diff --git a/main_prepare_evidence_test.go b/main_prepare_evidence_test.go index b85c10ef9..42bc34301 100644 --- a/main_prepare_evidence_test.go +++ b/main_prepare_evidence_test.go @@ -14,8 +14,46 @@ import ( "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" "google.golang.org/grpc" + "google.golang.org/grpc/metadata" ) +func TestPrepareEvidenceCapabilityRequiresCoordinatorRuntime(t *testing.T) { + for _, routed := range []bool{false, true} { + t.Run(map[bool]string{false: "legacy-warmup", true: "wired-cutover"}[routed], func(t *testing.T) { + setTSOModeFlags(t, routed, false) + clock := kv.NewHLC() + clock.SetPhysicalCeiling(time.Now().Add(time.Minute).UnixMilli()) + state := kv.NewTSOStateMachine(clock) + engine := &mainTSOEngine{state: raftengine.StateLeader, tsoState: state} + groups := map[uint64]*kv.ShardGroup{0: {Engine: engine, TSOState: state}} + coord := newMainTSOCoordinator(clock, groups) + wiring, err := configureCoordinatorTSO(coord, groups, mainTSOFloorProvider{}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, wiring.Close()) }) + options := append(internalTimestampOptions(newStartupGatedCoordinator(coord, nil)), dedicatedTimestampCutoverOptions(groups)...) + options = append(options, adapter.WithInternalPrepareEvidenceToken("runtime-token")) + server := adapter.NewInternalWithEngine(nil, nil, nil, nil, options...) + ctx := metadata.NewIncomingContext(t.Context(), metadata.Pairs("authorization", "Bearer runtime-token")) + for _, cutover := range []bool{false, true} { + if cutover { + allocator, ok := wiring.serverAllocator.(kv.TSOReservationAllocator) + require.True(t, ok) + _, err = allocator.ReserveBatchAfter(t.Context(), 1, 0, true, false) + require.NoError(t, err) + } + response, err := server.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{CapabilityOnly: true}) + require.NoError(t, err) + require.Equal(t, routed, response.GetDedicatedTsoConfigured(), "a durable marker must not invent missing coordinator wiring") + require.Equal(t, cutover, response.GetCutoverApplied()) + } + }) + } +} + +func TestDedicatedTimestampCutoverOptionsRejectsNilState(t *testing.T) { + require.Empty(t, dedicatedTimestampCutoverOptions(map[uint64]*kv.ShardGroup{0: {}})) +} + type prepareSafetyNextHook func(context.Context, string, string) error func (f prepareSafetyNextHook) PreAddMember(ctx context.Context, id, address string) error { @@ -61,10 +99,11 @@ func TestPrepareEvidenceSafetyMainAdmissionStillRejectsOldMemberAfterMarker(t *t if kind == "old-unsupported" { pb.RegisterInternalServer(srv, &pb.UnimplementedInternalServer{}) } else { - var options []adapter.InternalOption + options := []adapter.InternalOption{adapter.WithInternalPrepareEvidenceToken("membership-token")} if kind == "fresh-dedicated-learner" { // Configured but not joined to group 0 yet: adoption is false. - options = append(options, adapter.WithInternalTSOCutoverState(kv.NewTSOStateMachine(kv.NewHLC()))) + options = append(options, adapter.WithInternalTSOCutoverState(kv.NewTSOStateMachine(kv.NewHLC())), + adapter.WithInternalTimestampAllocator(&mainTimestampAllocator{})) } pb.RegisterInternalServer(srv, adapter.NewInternalWithEngine(nil, nil, nil, nil, options...)) }