diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 18ba03baa..ae1189df0 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -12,6 +12,8 @@ permissions: contents: read jobs: test: + env: + GRPC_SEQUENCE_TESTS: '^(Test_consistency_satisfy_write_after_read_sequence|Test_grpc_transaction)$' strategy: matrix: os: [ [ubuntu-latest] ] @@ -27,7 +29,12 @@ jobs: run: sudo apt-get update && sudo apt-get install -y fuse3 - name: Reject production imports of test support run: bash scripts/check-production-imports.sh - - run: go test -race ./... + - name: Run race tests outside full-length gRPC sequences + run: go test -race ./... -skip "$GRPC_SEQUENCE_TESTS" env: # Fail, rather than skip, the kernel-mount tests when FUSE is unusable. ELASTICKV_REQUIRE_FUSE: "1" + # These unchanged 10,000-iteration tests need a fresh package budget + # after the serial migration/snapshot acceptance tests have finished. + - name: Run full-length gRPC consistency sequences + run: go test -race ./adapter -run "$GRPC_SEQUENCE_TESTS" diff --git a/adapter/distribution_job_read_gate_test.go b/adapter/distribution_job_read_gate_test.go new file mode 100644 index 000000000..b8e52caed --- /dev/null +++ b/adapter/distribution_job_read_gate_test.go @@ -0,0 +1,38 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestDistributionServerJobReadsHonorStartupGate(t *testing.T) { + t.Parallel() + ctx := context.Background() + catalog := distribution.NewCatalogStore(store.NewMVCCStore()) + job := sampleDistributionSplitJob(13) + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + blocked := true + server := NewDistributionServer(nil, catalog, WithDistributionReadGate(func() bool { return blocked })) + _, err := server.GetSplitJob(ctx, &pb.GetSplitJobRequest{JobId: job.JobID}) + require.Equal(t, codes.Unavailable, status.Code(err)) + _, err = server.GetSplitJob(ctx, &pb.GetSplitJobRequest{JobId: 99}) + require.Equal(t, codes.Unavailable, status.Code(err), "catch-up must not return a stale NotFound") + _, err = server.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{}) + require.Equal(t, codes.Unavailable, status.Code(err)) + blocked = false + got, err := server.GetSplitJob(ctx, &pb.GetSplitJobRequest{JobId: job.JobID}) + require.NoError(t, err) + require.Equal(t, job.JobID, got.GetJob().GetJobId()) + listed, err := server.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{}) + require.NoError(t, err) + require.Len(t, listed.GetJobs(), 1) + _, err = server.GetSplitJob(ctx, &pb.GetSplitJobRequest{JobId: 99}) + require.Equal(t, codes.NotFound, status.Code(err)) +} diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index bb2e3d481..42e1de783 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -3,37 +3,53 @@ package adapter import ( "bytes" "context" + "encoding/binary" "log/slog" "math" + "sort" "strings" "sync" "time" "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/fskeys" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // DistributionServer serves distribution related gRPC APIs. type DistributionServer struct { - mu sync.Mutex - engine *distribution.Engine - catalog *distribution.CatalogStore - coordinator kv.Coordinator - timestampAllocator kv.TSOAllocator - readTracker *kv.ActiveTimestampTracker - watchInterval time.Duration - watchLeader func() bool - tsoActivationGate func(cutover, phaseD bool) error - fsObserver DistributionFilesystemObserver - readBlocked func() bool - reloadRetry struct { + mu sync.Mutex + engine *distribution.Engine + catalog *distribution.CatalogStore + coordinator kv.Coordinator + timestampAllocator kv.TSOAllocator + readTracker *kv.ActiveTimestampTracker + watchInterval time.Duration + watchLeader func() bool + tsoActivationGate func(cutover, phaseD bool) error + fsObserver DistributionFilesystemObserver + readBlocked func() bool + migrationCapabilityGate SplitMigrationCapabilityGate + splitJobRunnerReady bool + splitJobRunnerReadinessGate SplitMigrationCapabilityGate + splitPromotionClientFactory SplitPromotionClientFactory + splitMigrationClientFactory SplitMigrationClientFactory + splitMigrationGroupFactory SplitMigrationGroupClientFactory + splitMigrationVoterFactory SplitMigrationVoterFactory + splitApplyMemberFactory SplitMigrationApplyMemberSnapshotFactory + ownershipRefresh *migrationOwnershipRefresh + knownRaftGroups map[uint64]struct{} + splitJobHistoryGCLast time.Time + splitJobFailures map[uint64]splitJobFailureState + reloadRetry struct { attempts int interval time.Duration } @@ -49,6 +65,50 @@ type DistributionFilesystemObserver interface { const DistributionFilePinnedHotspotSplitBoundary = "split_boundary" +// SplitMigrationCapabilityGate reports whether this node can safely create +// migration-only side effects. A nil gate keeps StartSplitMigration fail-closed. +type SplitMigrationCapabilityGate func(context.Context) error + +// SplitPromotionClient is the subset of the internal gRPC client the split +// job runner needs to promote target-local staged rows. +type SplitPromotionClient interface { + PromoteStagedVersions(context.Context, *pb.PromoteStagedVersionsRequest, ...grpc.CallOption) (*pb.PromoteStagedVersionsResponse, error) +} + +// SplitPromotionClientFactory dials the node currently leading a split job's +// target range and returns the internal client used by the runner. +type SplitPromotionClientFactory func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) + +// SplitMigrationClient is the internal data-plane client used for a split +// job's source export, target import, and durable source/target guards. +type SplitMigrationClient interface { + SplitPromotionClient + ExportRangeVersions(context.Context, *pb.ExportRangeVersionsRequest, ...grpc.CallOption) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) + ImportRangeVersions(context.Context, *pb.ImportRangeVersionsRequest, ...grpc.CallOption) (*pb.ImportRangeVersionsResponse, error) + ApplyTargetStagedReadiness(context.Context, *pb.TargetStagedReadinessRequest, ...grpc.CallOption) (*pb.TargetStagedReadinessResponse, error) + ProbeMigrationLocks(context.Context, *pb.ProbeMigrationLocksRequest, ...grpc.CallOption) (*pb.ProbeMigrationLocksResponse, error) + CleanupMigration(context.Context, *pb.CleanupMigrationRequest, ...grpc.CallOption) (*pb.CleanupMigrationResponse, error) + ProbeMigrationState(context.Context, *pb.ProbeMigrationStateRequest, ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) + IssueMigrationTimestamp(context.Context, *pb.IssueMigrationTimestampRequest, ...grpc.CallOption) (*pb.IssueMigrationTimestampResponse, error) +} + +// SplitMigrationClientFactory resolves both participating group leaders. +type SplitMigrationClientFactory func(context.Context, distribution.SplitJob, uint64) (source SplitMigrationClient, target SplitMigrationClient, err error) + +// SplitMigrationGroupClientFactory resolves one group without requiring the +// other participant to be available during source recovery. +type SplitMigrationGroupClientFactory func(context.Context, uint64) (SplitMigrationClient, error) + +type SplitMigrationVoter struct { + ID string + Address string + Client SplitMigrationClient +} + +// SplitMigrationVoterFactory resolves the current voter set and a direct +// client for each voter. The runner re-resolves it before every barrier. +type SplitMigrationVoterFactory func(context.Context, uint64) ([]SplitMigrationVoter, error) + // WithDistributionTSOActivationGate authorizes a caller's request to activate // the durable cutover or Phase-D marker against this node's own rollout // configuration. @@ -104,6 +164,71 @@ func WithDistributionReadGate(blocked func() bool) DistributionServerOption { } } +func WithSplitMigrationCapabilityGate(gate SplitMigrationCapabilityGate) DistributionServerOption { + return func(s *DistributionServer) { + s.migrationCapabilityGate = gate + } +} + +// WithSplitJobRunnerReady opens the split migration capability probe once this +// node can advance planned split jobs. +func WithSplitJobRunnerReady() DistributionServerOption { + return func(s *DistributionServer) { + s.splitJobRunnerReady = true + } +} + +// WithSplitJobRunnerReadinessGate configures local runtime gates that must be +// open before this node advertises split migration capability. +func WithSplitJobRunnerReadinessGate(gate SplitMigrationCapabilityGate) DistributionServerOption { + return func(s *DistributionServer) { + s.splitJobRunnerReadinessGate = gate + } +} + +// WithSplitPromotionClientFactory configures the client factory used by the +// split job runner to promote target-local staged versions. +func WithSplitPromotionClientFactory(factory SplitPromotionClientFactory) DistributionServerOption { + return func(s *DistributionServer) { + s.splitPromotionClientFactory = factory + } +} + +// WithSplitMigrationClientFactory configures the source/target clients used by +// the production split job state machine. +func WithSplitMigrationClientFactory(factory SplitMigrationClientFactory) DistributionServerOption { + return func(s *DistributionServer) { + s.splitMigrationClientFactory = factory + } +} + +func WithSplitMigrationGroupClientFactory(factory SplitMigrationGroupClientFactory) DistributionServerOption { + return func(s *DistributionServer) { + s.splitMigrationGroupFactory = factory + } +} + +func WithSplitMigrationVoterFactory(factory SplitMigrationVoterFactory) DistributionServerOption { + return func(s *DistributionServer) { + s.splitMigrationVoterFactory = factory + } +} + +// WithDistributionKnownRaftGroups configures the Raft group IDs this node can +// route migration work to. +func WithDistributionKnownRaftGroups(groupIDs ...uint64) DistributionServerOption { + return func(s *DistributionServer) { + groups := make(map[uint64]struct{}, len(groupIDs)) + for _, groupID := range groupIDs { + if groupID == 0 { + continue + } + groups[groupID] = struct{}{} + } + s.knownRaftGroups = groups + } +} + // WithCatalogReloadRetryPolicy configures the retry policy used after split // commit when waiting for the local catalog snapshot to become visible. func WithCatalogReloadRetryPolicy(attempts int, interval time.Duration) DistributionServerOption { @@ -137,14 +262,33 @@ func WithCatalogWatchLeaderCheck(check func() bool) DistributionServerOption { } const ( - childRouteCount = 2 - splitMutationOpCount = childRouteCount + 3 + childRouteCount = 2 + splitMutationOpCount = childRouteCount + 3 + listSplitJobsDefaultPageSize = 200 + splitJobListCursorVersion = byte(1) + splitJobListCursorTerminalOff = 1 + splitJobListCursorJobIDOff = splitJobListCursorTerminalOff + 8 + splitJobListCursorEncodedBytes = splitJobListCursorJobIDOff + 8 + SplitMigrationCapabilityV2 = "cap_migration_v2" + splitMigrationCapabilityV2 = SplitMigrationCapabilityV2 + + splitPromotionDefaultMaxVersions = 1024 + splitPromotionBytesPerMiB = 1024 * 1024 + splitPromotionMaxBytesMiB = 4 + splitPromotionMaxScannedMiB = 16 + splitPromotionAttemptTimeoutSecs = 2 + promotionCompleteBaseOpCount = 2 ) var ( - defaultCatalogReloadRetryAttempts = 20 - defaultCatalogReloadRetryInterval = 10 * time.Millisecond - defaultCatalogWatchInterval = 100 * time.Millisecond + defaultCatalogReloadRetryAttempts = 20 + defaultCatalogReloadRetryInterval = 10 * time.Millisecond + defaultSplitJobRunnerInterval = time.Second + defaultSplitPromotionMaxVersions = uint32(splitPromotionDefaultMaxVersions) + defaultSplitPromotionMaxBytes = uint64(splitPromotionMaxBytesMiB * splitPromotionBytesPerMiB) + defaultSplitPromotionMaxScanned = uint64(splitPromotionMaxScannedMiB * splitPromotionBytesPerMiB) + defaultSplitPromotionAttemptTimeout = time.Duration(splitPromotionAttemptTimeoutSecs) * time.Second + defaultCatalogWatchInterval = 100 * time.Millisecond errDistributionCatalogNotConfigured = errors.New("route catalog is not configured") errDistributionUnknownRoute = errors.New("unknown route") @@ -156,6 +300,10 @@ var ( errDistributionCoordinatorRequired = errors.New("distribution coordinator is not configured") errDistributionEngineNotConfigured = errors.New("distribution engine is not configured") errDistributionCatalogVersionNotFound = errors.New("route catalog version not found") + errDistributionClusterNotReady = errors.New("cluster is not ready for split migration") + errDistributionRaftGroupsNotKnown = errors.New("distribution raft groups are not configured") + errDistributionUnknownTargetGroup = errors.New("unknown target group") + errDistributionSourceRouteNotActive = errors.New("source route is not active") errDistributionCatalogMutationInvalid = errors.New("catalog store mutation is invalid") ) @@ -190,6 +338,231 @@ func (s *DistributionServer) requireReadReady() error { return nil } +func (s *DistributionServer) RunSplitJobRunner(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + ticker := time.NewTicker(defaultSplitJobRunnerInterval) + defer ticker.Stop() + for { + if err := s.RunSplitJobRunnerOnce(ctx); err != nil { + if !splitJobRunnerContextDone(err) { + slog.Warn("split job runner tick failed", "err", err) + } + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} + +func (s *DistributionServer) RunSplitJobRunnerOnce(ctx context.Context) error { + if !s.splitJobRunnerConfigured() { + return nil + } + leader, err := s.verifySplitJobRunnerLeader(ctx) + if err != nil || !leader { + return err + } + jobs, err := s.catalog.ListSplitJobs(ctx) + if err != nil { + return splitJobCatalogStatusError(err) + } + if job, ok := nextRunnableSplitJob(jobs); ok { + if err := s.verifySplitJobCleanupDispatch(ctx, job); err != nil { + return err + } + if job.Phase != distribution.SplitJobPhaseAbandoning { + ready, gateErr := s.splitJobCapabilityReady(ctx, job) + if gateErr != nil || !ready { + return gateErr + } + } + return s.runSplitJobAttempt(ctx, job) + } + return s.gcSplitJobHistory(ctx, jobs, time.Now()) +} + +func (s *DistributionServer) splitJobCapabilityReady(ctx context.Context, job distribution.SplitJob) (bool, error) { + if s.migrationCapabilityGate == nil { + return true, nil + } + gateErr := s.migrationCapabilityGate(ctx) + if gateErr != nil { + return s.handleSplitJobCapabilityRegression(ctx, job, gateErr) + } + if !job.CapabilityRegressed { + return true, nil + } + return s.clearSplitJobCapabilityRegression(ctx, job) +} + +func (s *DistributionServer) handleSplitJobCapabilityRegression(ctx context.Context, job distribution.SplitJob, cause error) (bool, error) { + markedJob, marked, err := s.markSplitJobCapabilityRegressed(ctx, job, cause) + if err != nil || !marked { + return false, err + } + if splitJobCapabilityRegressionRequiresFailClosed(markedJob) { + closed, err := s.ensureSplitJobCapabilityRegressionFailClosed(ctx, markedJob) + if err != nil || !closed { + return false, err + } + } + return false, nil +} + +func (s *DistributionServer) markSplitJobCapabilityRegressed(ctx context.Context, job distribution.SplitJob, cause error) (distribution.SplitJob, bool, error) { + marked := false + markedJob := job + err := s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != job.Phase { + return current, nil + } + marked = true + if !current.CapabilityRegressed { + current.CapabilityRegressed = true + current.LastError = "cluster migration capability regressed: " + cause.Error() + current.UpdatedAtMs = time.Now().UnixMilli() + } + markedJob = distribution.CloneSplitJob(current) + return current, nil + }) + if err != nil || !marked { + return job, marked, err + } + return markedJob, true, nil +} + +func (s *DistributionServer) clearSplitJobCapabilityRegression(ctx context.Context, job distribution.SplitJob) (bool, error) { + if splitJobCapabilityRegressionRequiresFailClosed(job) { + restored, err := s.restoreSplitJobCapabilityRegressionGuards(ctx, job) + if err != nil || !restored { + return false, err + } + } + err := s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == job.Phase && current.CapabilityRegressed { + current.CapabilityRegressed = false + current.LastError = "" + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + return false, err +} + +func (s *DistributionServer) splitJobRunnerConfigured() bool { + return s != nil && s.catalog != nil && s.coordinator != nil && s.splitPromotionClientFactory != nil && s.splitMigrationClientFactory != nil +} + +func (s *DistributionServer) verifySplitJobRunnerLeader(ctx context.Context) (bool, error) { + if !s.coordinator.IsLeaderForKey(distribution.CatalogVersionKey()) { + return false, nil + } + if err := s.coordinator.VerifyLeaderForKey(ctx, distribution.CatalogVersionKey()); err != nil { + return false, errors.WithStack(err) + } + return true, nil +} + +func nextRunnableSplitJob(jobs []distribution.SplitJob) (distribution.SplitJob, bool) { + for _, job := range jobs { + switch job.Phase { + case distribution.SplitJobPhasePlanned, + distribution.SplitJobPhaseBackfill, + distribution.SplitJobPhaseFence, + distribution.SplitJobPhaseDeltaCopy, + distribution.SplitJobPhaseCutover, + distribution.SplitJobPhaseCleanup, + distribution.SplitJobPhaseAbandoning: + return job, true + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoned: + continue + } + } + return distribution.SplitJob{}, false +} + +func (s *DistributionServer) promoteSplitJobTargetAndComplete(ctx context.Context, job distribution.SplitJob) error { + if !job.TargetPromotionDone { + client, err := s.splitPromotionClientFactory(ctx, job) + if err != nil { + return errors.WithStack(err) + } + if client == nil { + return errors.New("split promotion client is nil") + } + if err := promoteSplitJobTarget(ctx, client, job); err != nil { + return errors.WithStack(err) + } + } + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return err + } + current, found, err := s.catalog.SplitJobAt(ctx, job.JobID, snapshot.ReadTS) + if err != nil { + return splitJobPromotionStatusError(err) + } + if !found { + return splitJobPromotionStatusError(distribution.ErrCatalogSplitJobConflict) + } + completed, _, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, snapshot.Version, current, time.Now().UnixMilli()) + if err != nil { + return err + } + return s.applyEngineSnapshot(completed) +} + +func promoteSplitJobTarget(ctx context.Context, client SplitPromotionClient, job distribution.SplitJob) error { + var cursor []byte + for { + attemptCtx, cancel := splitPromotionAttemptContext(ctx) + resp, err := client.PromoteStagedVersions(attemptCtx, &pb.PromoteStagedVersionsRequest{ + JobId: job.JobID, + Cursor: cursor, + MaxVersions: defaultSplitPromotionMaxVersions, + MaxBytes: defaultSplitPromotionMaxBytes, + MaxScannedBytes: defaultSplitPromotionMaxScanned, + }) + cancel() + if err != nil { + return errors.WithStack(err) + } + if resp.GetDone() { + return nil + } + next := resp.GetNextCursor() + if len(next) == 0 || bytes.Equal(next, cursor) { + return errors.New("split promotion made no cursor progress") + } + cursor = distribution.CloneBytes(next) + } +} + +func splitPromotionAttemptContext(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx == nil { + ctx = context.Background() + } + if defaultSplitPromotionAttemptTimeout <= 0 { + return ctx, func() {} + } + return context.WithTimeout(ctx, defaultSplitPromotionAttemptTimeout) +} + +func splitJobRunnerContextDone(err error) bool { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return true + } + code := status.Code(errors.Cause(err)) + return code == codes.Canceled || code == codes.DeadlineExceeded +} + // UpdateRoute allows updating route information. func (s *DistributionServer) UpdateRoute(start, end []byte, group uint64) { s.engine.UpdateRoute(start, end, group) @@ -539,104 +912,319 @@ func (s *DistributionServer) GetIntersectingRoutes(ctx context.Context, req *pb. }, nil } -func (s *DistributionServer) routeSnapshotAt(version uint64) (distribution.RouteHistorySnapshot, error) { - if s.engine == nil { - return distribution.RouteHistorySnapshot{}, grpcStatusError(codes.FailedPrecondition, errDistributionEngineNotConfigured.Error()) - } - snapshot, ok := s.engine.SnapshotAt(version) - if !ok { - return distribution.RouteHistorySnapshot{}, grpcStatusErrorf(codes.NotFound, "%s: %d", errDistributionCatalogVersionNotFound, version) +func (s *DistributionServer) StartSplitMigration(ctx context.Context, req *pb.StartSplitMigrationRequest) (*pb.StartSplitMigrationResponse, error) { + if err := s.verifyStartSplitMigrationPreflight(ctx, req); err != nil { + return nil, err } - return snapshot, nil -} -// GetCatalogCapabilities negotiates the durable delta-watch protocol. -func (s *DistributionServer) GetCatalogCapabilities(ctx context.Context, _ *pb.CatalogCapabilitiesRequest) (*pb.CatalogCapabilitiesResponse, error) { - if s.catalog == nil { - return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + s.mu.Lock() + defer s.mu.Unlock() + + if err := s.verifyStartSplitMigrationCatalogReady(ctx); err != nil { + return nil, err } - version, err := s.catalog.Version(ctx) + snapshot, err := s.loadCatalogSnapshot(ctx) if err != nil { - return nil, grpcStatusErrorf(codes.Internal, "load catalog version: %v", err) + return nil, err } - floor, err := s.catalog.DeltaFloor(ctx) + defer releaseReadPin(s.pinReadTS(snapshot.ReadTS)) + + parent, err := s.startSplitMigrationParent(ctx, snapshot, req) if err != nil { - return nil, grpcStatusErrorf(codes.Internal, "load catalog delta floor: %v", err) + return nil, err } - return &pb.CatalogCapabilitiesResponse{ - SupportedProtocolVersions: []uint32{distribution.CatalogWatchProtocolVersion}, - CurrentVersion: version, - OldestDeltaVersion: floor, - MaxBatchSize: distribution.MaxCatalogDeltaBatchSize, + job, err := s.newSplitMigrationJob(ctx, snapshot, parent, req) + if err != nil { + return nil, err + } + if err := s.createSplitJobViaCoordinator(ctx, snapshot.ReadTS, job); err != nil { + return nil, err + } + return &pb.StartSplitMigrationResponse{ + CatalogVersion: snapshot.Version, + JobId: job.JobID, }, nil } -// WatchCatalog streams contiguous deltas and emits a snapshot reset when the -// requested reconnect cursor predates retained history. -func (s *DistributionServer) WatchCatalog(req *pb.CatalogWatchRequest, stream pb.Distribution_WatchCatalogServer) error { - config, err := s.catalogWatchConfig(req) - if err != nil { - return err +func (s *DistributionServer) GetSplitMigrationCapability(ctx context.Context, _ *pb.GetSplitMigrationCapabilityRequest) (*pb.GetSplitMigrationCapabilityResponse, error) { + if !s.splitMigrationCapabilityReady(ctx) { + return &pb.GetSplitMigrationCapabilityResponse{}, nil } - for { - if s.watchLeader != nil && !s.watchLeader() { - return grpcStatusError(codes.Unavailable, "catalog watch requires the catalog-group leader") - } - nextCursor, sent, err := s.sendCatalogChanges(stream, config.cursor, config.batchSize) - if err != nil { - return err - } - config.cursor = nextCursor - if sent { - continue - } - stop, err := waitCatalogStream(stream.Context(), config.interval) - if err != nil { - return err - } - if stop { - return nil - } + return &pb.GetSplitMigrationCapabilityResponse{ + MigrationCapable: true, + Capabilities: []string{splitMigrationCapabilityV2}, + }, nil +} + +func (s *DistributionServer) splitMigrationCapabilityReady(ctx context.Context) bool { + if !s.splitJobRunnerReady { + return false + } + if s.splitJobRunnerReadinessGate == nil { + return true } + return s.splitJobRunnerReadinessGate(ctx) == nil } -type catalogWatchConfig struct { - cursor uint64 - batchSize int - interval time.Duration +func (s *DistributionServer) verifyStartSplitMigrationPreflight(ctx context.Context, req *pb.StartSplitMigrationRequest) error { + if req.GetTargetGroupId() == 0 { + return grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobTargetGroupRequired.Error()) + } + if err := s.verifyStartSplitMigrationCatalogReady(ctx); err != nil { + return err + } + return s.verifySplitMigrationCapability(ctx) } -func (s *DistributionServer) catalogWatchConfig(req *pb.CatalogWatchRequest) (catalogWatchConfig, error) { +func (s *DistributionServer) verifyStartSplitMigrationCatalogReady(ctx context.Context) error { + if err := s.verifyCatalogLeader(ctx); err != nil { + return err + } if s.catalog == nil { - return catalogWatchConfig{}, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + return grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) } - if req.GetProtocolVersion() != distribution.CatalogWatchProtocolVersion { - return catalogWatchConfig{}, grpcStatusErrorf(codes.FailedPrecondition, "unsupported catalog watch protocol %d", req.GetProtocolVersion()) + return nil +} + +func (s *DistributionServer) startSplitMigrationParent( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + req *pb.StartSplitMigrationRequest, +) (distribution.RouteDescriptor, error) { + if err := validateExpectedCatalogVersion(snapshot.Version, req.GetExpectedCatalogVersion()); err != nil { + return distribution.RouteDescriptor{}, err } - batchSize := int(req.GetMaxBatchSize()) - if batchSize <= 0 { - batchSize = distribution.DefaultCatalogDeltaBatchSize + parent, found := findRouteByID(snapshot.Routes, req.GetRouteId()) + if !found { + return distribution.RouteDescriptor{}, grpcStatusError(codes.NotFound, errDistributionUnknownRoute.Error()) } - if batchSize > distribution.MaxCatalogDeltaBatchSize { - batchSize = distribution.MaxCatalogDeltaBatchSize + if parent.State != distribution.RouteStateActive { + return distribution.RouteDescriptor{}, grpcStatusError(codes.FailedPrecondition, errDistributionSourceRouteNotActive.Error()) } - interval := s.watchInterval - if interval <= 0 { - interval = defaultCatalogWatchInterval + splitKey := normalizeSplitMigrationSplitKey(req) + if err := validateSplitKey(parent, splitKey); err != nil { + return distribution.RouteDescriptor{}, err } - return catalogWatchConfig{cursor: req.GetAfterVersion(), batchSize: batchSize, interval: interval}, nil + if err := distribution.ValidateSplitMigrationRouteRange(splitKey, parent.End); err != nil { + return distribution.RouteDescriptor{}, splitMigrationRangeStatusError(err) + } + if parent.GroupID == req.GetTargetGroupId() { + return distribution.RouteDescriptor{}, grpcStatusError(codes.InvalidArgument, "target group must differ from source route group") + } + if err := s.verifyKnownTargetGroup(req.GetTargetGroupId()); err != nil { + return distribution.RouteDescriptor{}, err + } + if err := s.rejectLiveSplitJob(ctx, snapshot, parent); err != nil { + return distribution.RouteDescriptor{}, err + } + return parent, nil } -func (s *DistributionServer) sendCatalogChanges( - stream pb.Distribution_WatchCatalogServer, - cursor uint64, - batchSize int, -) (uint64, bool, error) { - changes, err := s.catalog.ChangesSince(stream.Context(), cursor, batchSize) +func (s *DistributionServer) newSplitMigrationJob( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + parent distribution.RouteDescriptor, + req *pb.StartSplitMigrationRequest, +) (distribution.SplitJob, error) { + jobID, err := s.catalog.NextSplitJobIDAt(ctx, snapshot.ReadTS) if err != nil { - if errors.Is(err, distribution.ErrCatalogDeltaVersionFuture) { - return cursor, false, grpcStatusError(codes.InvalidArgument, err.Error()) - } + return distribution.SplitJob{}, splitJobCatalogStatusError(err) + } + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: jobID, + SourceRouteID: parent.RouteID, + SplitKey: normalizeSplitMigrationSplitKey(req), + TargetGroupID: req.GetTargetGroupId(), + }, parent, time.Now().UnixMilli()) + if err != nil { + return distribution.SplitJob{}, splitJobCatalogStatusError(err) + } + return job, nil +} + +func normalizeSplitMigrationSplitKey(req *pb.StartSplitMigrationRequest) []byte { + return distribution.CloneBytes(fskeys.NormalizeSplitBoundary(kv.RouteKey(req.GetSplitKey()))) +} + +func (s *DistributionServer) GetSplitJob(ctx context.Context, req *pb.GetSplitJobRequest) (*pb.GetSplitJobResponse, error) { + if err := s.requireReadReady(); err != nil { + return nil, err + } + if req.GetJobId() == 0 { + return nil, grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobIDRequired.Error()) + } + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + job, found, err := s.catalog.SplitJob(ctx, req.GetJobId()) + if err != nil { + return nil, splitJobCatalogStatusError(err) + } + if !found { + return nil, grpcStatusError(codes.NotFound, distribution.ErrCatalogSplitJobNotFound.Error()) + } + return &pb.GetSplitJobResponse{Job: distribution.SplitJobToProto(job)}, nil +} + +func (s *DistributionServer) ListSplitJobs(ctx context.Context, req *pb.ListSplitJobsRequest) (*pb.ListSplitJobsResponse, error) { + if err := s.requireReadReady(); err != nil { + return nil, err + } + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + if req == nil { + req = &pb.ListSplitJobsRequest{} + } + phaseFilter := normalizeSplitJobPhaseFilter(req.GetPhase()) + if phaseFilter != "" && !validSplitJobPhaseFilter(phaseFilter) { + return nil, grpcStatusErrorf(codes.InvalidArgument, "unknown split job phase %q", req.GetPhase()) + } + jobs, err := s.catalog.ListSplitJobs(ctx) + if err != nil { + return nil, splitJobCatalogStatusError(err) + } + resp, err := splitJobListPage(jobs, req, phaseFilter) + if err != nil { + return nil, err + } + return resp, nil +} + +func (s *DistributionServer) AbandonSplitJob(ctx context.Context, req *pb.AbandonSplitJobRequest) (*pb.AbandonSplitJobResponse, error) { + if req.GetJobId() == 0 { + return nil, grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobIDRequired.Error()) + } + if err := s.verifyCatalogLeader(ctx); err != nil { + return nil, err + } + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + if err := s.updateSplitJobViaCoordinator(ctx, req.GetJobId(), func(job distribution.SplitJob) (distribution.SplitJob, error) { + return distribution.BeginSplitJobAbandon(job, time.Now().UnixMilli()) + }); err != nil { + return nil, err + } + return &pb.AbandonSplitJobResponse{}, nil +} + +func (s *DistributionServer) RetrySplitJob(ctx context.Context, req *pb.RetrySplitJobRequest) (*pb.RetrySplitJobResponse, error) { + if req.GetJobId() == 0 { + return nil, grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobIDRequired.Error()) + } + if err := s.verifyCatalogLeader(ctx); err != nil { + return nil, err + } + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + if err := s.updateSplitJobViaCoordinator(ctx, req.GetJobId(), func(job distribution.SplitJob) (distribution.SplitJob, error) { + return distribution.RetrySplitJobState(job, time.Now().UnixMilli()) + }); err != nil { + return nil, err + } + return &pb.RetrySplitJobResponse{}, nil +} + +func (s *DistributionServer) routeSnapshotAt(version uint64) (distribution.RouteHistorySnapshot, error) { + if s.engine == nil { + return distribution.RouteHistorySnapshot{}, grpcStatusError(codes.FailedPrecondition, errDistributionEngineNotConfigured.Error()) + } + snapshot, ok := s.engine.SnapshotAt(version) + if !ok { + return distribution.RouteHistorySnapshot{}, grpcStatusErrorf(codes.NotFound, "%s: %d", errDistributionCatalogVersionNotFound, version) + } + return snapshot, nil +} + +// GetCatalogCapabilities negotiates the durable delta-watch protocol. +func (s *DistributionServer) GetCatalogCapabilities(ctx context.Context, _ *pb.CatalogCapabilitiesRequest) (*pb.CatalogCapabilitiesResponse, error) { + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + version, err := s.catalog.Version(ctx) + if err != nil { + return nil, grpcStatusErrorf(codes.Internal, "load catalog version: %v", err) + } + floor, err := s.catalog.DeltaFloor(ctx) + if err != nil { + return nil, grpcStatusErrorf(codes.Internal, "load catalog delta floor: %v", err) + } + return &pb.CatalogCapabilitiesResponse{ + SupportedProtocolVersions: []uint32{distribution.CatalogWatchProtocolVersion}, + CurrentVersion: version, + OldestDeltaVersion: floor, + MaxBatchSize: distribution.MaxCatalogDeltaBatchSize, + }, nil +} + +// WatchCatalog streams contiguous deltas and emits a snapshot reset when the +// requested reconnect cursor predates retained history. +func (s *DistributionServer) WatchCatalog(req *pb.CatalogWatchRequest, stream pb.Distribution_WatchCatalogServer) error { + config, err := s.catalogWatchConfig(req) + if err != nil { + return err + } + for { + if s.watchLeader != nil && !s.watchLeader() { + return grpcStatusError(codes.Unavailable, "catalog watch requires the catalog-group leader") + } + nextCursor, sent, err := s.sendCatalogChanges(stream, config.cursor, config.batchSize) + if err != nil { + return err + } + config.cursor = nextCursor + if sent { + continue + } + stop, err := waitCatalogStream(stream.Context(), config.interval) + if err != nil { + return err + } + if stop { + return nil + } + } +} + +type catalogWatchConfig struct { + cursor uint64 + batchSize int + interval time.Duration +} + +func (s *DistributionServer) catalogWatchConfig(req *pb.CatalogWatchRequest) (catalogWatchConfig, error) { + if s.catalog == nil { + return catalogWatchConfig{}, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + if req.GetProtocolVersion() != distribution.CatalogWatchProtocolVersion { + return catalogWatchConfig{}, grpcStatusErrorf(codes.FailedPrecondition, "unsupported catalog watch protocol %d", req.GetProtocolVersion()) + } + batchSize := int(req.GetMaxBatchSize()) + if batchSize <= 0 { + batchSize = distribution.DefaultCatalogDeltaBatchSize + } + if batchSize > distribution.MaxCatalogDeltaBatchSize { + batchSize = distribution.MaxCatalogDeltaBatchSize + } + interval := s.watchInterval + if interval <= 0 { + interval = defaultCatalogWatchInterval + } + return catalogWatchConfig{cursor: req.GetAfterVersion(), batchSize: batchSize, interval: interval}, nil +} + +func (s *DistributionServer) sendCatalogChanges( + stream pb.Distribution_WatchCatalogServer, + cursor uint64, + batchSize int, +) (uint64, bool, error) { + changes, err := s.catalog.ChangesSince(stream.Context(), cursor, batchSize) + if err != nil { + if errors.Is(err, distribution.ErrCatalogDeltaVersionFuture) { + return cursor, false, grpcStatusError(codes.InvalidArgument, err.Error()) + } return cursor, false, grpcStatusErrorf(codes.Internal, "read catalog changes: %v", err) } if changes.Reset != nil { @@ -732,8 +1320,7 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR if err != nil { return nil, err } - readPin := s.pinReadTS(snapshot.ReadTS) - defer readPin.Release() + defer releaseReadPin(s.pinReadTS(snapshot.ReadTS)) if err := validateExpectedCatalogVersion(snapshot.Version, req.GetExpectedCatalogVersion()); err != nil { return nil, err } @@ -742,12 +1329,13 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR if err != nil { return nil, err } - - saved, err := s.saveSplitResultViaCoordinator(ctx, readTimestamp, req.GetExpectedCatalogVersion(), plan.parentID, plan.readKeys, plan.left, plan.right) + partitionGroups, err := s.prepareSplitRangePartitions(ctx, snapshot, plan.parentID) if err != nil { return nil, err } - if err := s.applyEngineSnapshot(saved); err != nil { + + saved, err := s.commitSplitRangePlan(ctx, readTimestamp, req.GetExpectedCatalogVersion(), plan, partitionGroups) + if err != nil { return nil, err } savedLeft, savedRight, err := splitChildrenFromSnapshot(saved, plan.left.RouteID, plan.right.RouteID) @@ -769,6 +1357,17 @@ type splitRangePlan struct { right distribution.RouteDescriptor } +func (s *DistributionServer) commitSplitRangePlan(ctx context.Context, read kv.ReadTimestamp, version uint64, plan splitRangePlan, groups []uint64) (distribution.CatalogSnapshot, error) { + saved, err := s.saveSplitResultViaCoordinator(ctx, read, version, plan.parentID, plan.readKeys, plan.left, plan.right) + if err != nil { + return saved, err + } + if err := s.reconcileSplitRangePartitions(ctx, groups); err != nil { + return saved, err + } + return saved, s.applyEngineSnapshot(saved) +} + func (s *DistributionServer) planSplitRange(ctx context.Context, snapshot distribution.CatalogSnapshot, req *pb.SplitRangeRequest) (splitRangePlan, error) { parent, found := findRouteByID(snapshot.Routes, req.GetRouteId()) if !found { @@ -795,7 +1394,7 @@ func (s *DistributionServer) planSplitRange(ctx context.Context, snapshot distri func (s *DistributionServer) pinReadTS(ts uint64) *kv.ActiveTimestampToken { if s == nil || s.readTracker == nil { - return nil + return &kv.ActiveTimestampToken{} } return s.readTracker.Pin(ts) } @@ -822,6 +1421,13 @@ func isSplitBoundaryError(err error) bool { strings.Contains(err.Error(), errDistributionSplitKeyAtBoundary.Error()) } +func releaseReadPin(token *kv.ActiveTimestampToken) { + if token == nil { + return + } + token.Release() +} + func (s *DistributionServer) verifyCatalogLeader(ctx context.Context) error { if s.coordinator == nil { return grpcStatusError(codes.FailedPrecondition, errDistributionCoordinatorRequired.Error()) @@ -844,6 +1450,17 @@ func (s *DistributionServer) saveSplitResultViaCoordinator( readKeys [][]byte, left distribution.RouteDescriptor, right distribution.RouteDescriptor, +) (distribution.CatalogSnapshot, error) { + return s.saveSplitPartitionViaCoordinator(ctx, readTimestamp, expectedVersion, parentID, readKeys, left, right, nil) +} + +func (s *DistributionServer) saveSplitPartitionViaCoordinator( + ctx context.Context, + readTimestamp kv.ReadTimestamp, + expectedVersion, parentID uint64, + readKeys [][]byte, + left, right distribution.RouteDescriptor, + workingJob *distribution.SplitJob, ) (distribution.CatalogSnapshot, error) { nextVersion, nextRouteID, err := nextCatalogSplitIDs(expectedVersion, right.RouteID) if err != nil { @@ -872,15 +1489,18 @@ func (s *DistributionServer) saveSplitResultViaCoordinator( {Op: distribution.CatalogMutationUpsert, RouteID: right.RouteID, Route: right}, }, } - deltaMutations, err := s.catalog.BuildDeltaMutationsAt(ctx, readTS, delta) + deltaOps, err := s.catalogDeltaMutationOps(ctx, readTS, delta) if err != nil { - return distribution.CatalogSnapshot{}, grpcStatusErrorf(codes.Internal, "build catalog delta mutations: %v", err) + return distribution.CatalogSnapshot{}, err } - deltaOps, err := catalogStoreMutationsToOps(deltaMutations) + ops = append(ops, deltaOps...) + partitionOps, partitionReads, err := s.catalogSplitPartitionOps(ctx, readTS, parentID, left, right, expectedVersion, workingJob) if err != nil { - return distribution.CatalogSnapshot{}, grpcStatusErrorf(codes.Internal, "convert catalog delta mutations: %v", err) + return distribution.CatalogSnapshot{}, err } - ops = append(ops, deltaOps...) + ops = append(ops, partitionOps...) + readKeys = append(readKeys, distribution.CatalogVersionKey(), distribution.CatalogRouteKey(parentID), distribution.CatalogNextRouteIDKey()) + readKeys = append(readKeys, partitionReads...) dispatchCtx := readTimestamp.WithDispatchVoucher(ctx) resp, err := kv.DispatchWithReadTimestamp(dispatchCtx, s.coordinator, &kv.OperationGroup[kv.OP]{ Elems: ops, @@ -901,6 +1521,18 @@ func (s *DistributionServer) saveSplitResultViaCoordinator( return s.loadCatalogSnapshotAtVersion(ctx, resp.CommitTS, nextVersion) } +func (s *DistributionServer) catalogDeltaMutationOps(ctx context.Context, readTS uint64, delta distribution.CatalogDelta) ([]*kv.Elem[kv.OP], error) { + mutations, err := s.catalog.BuildDeltaMutationsAt(ctx, readTS, delta) + if err != nil { + return nil, grpcStatusErrorf(codes.Internal, "build catalog delta mutations: %v", err) + } + ops, err := catalogStoreMutationsToOps(mutations) + if err != nil { + return nil, grpcStatusErrorf(codes.Internal, "convert catalog delta mutations: %v", err) + } + return ops, nil +} + func nextCatalogSplitIDs(expectedVersion uint64, rightRouteID uint64) (uint64, uint64, error) { if expectedVersion == math.MaxUint64 { return 0, 0, grpcStatusError(codes.Internal, "catalog version overflow") @@ -998,6 +1630,47 @@ func (s *DistributionServer) loadCatalogSnapshot(ctx context.Context) (distribut return snapshot, nil } +func (s *DistributionServer) loadCatalogSnapshotAtLeastVersion( + ctx context.Context, + minVersion uint64, +) (distribution.CatalogSnapshot, error) { + if s.catalog == nil { + return distribution.CatalogSnapshot{}, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + attempts := s.reloadRetry.attempts + if attempts <= 0 { + attempts = defaultCatalogReloadRetryAttempts + } + interval := s.reloadRetry.interval + if interval <= 0 { + interval = defaultCatalogReloadRetryInterval + } + + var last distribution.CatalogSnapshot + for attempt := 0; attempt < attempts; attempt++ { + snapshot, err := s.catalog.Snapshot(ctx) + if err != nil { + return distribution.CatalogSnapshot{}, grpcStatusErrorf(codes.Internal, "reload route catalog: %v", err) + } + if snapshot.Version >= minVersion { + return snapshot, nil + } + last = snapshot + if attempt == attempts-1 { + break + } + if err := waitWithContext(ctx, interval); err != nil { + return distribution.CatalogSnapshot{}, err + } + } + return distribution.CatalogSnapshot{}, grpcStatusErrorf( + codes.Internal, + "catalog split committed but local snapshot is stale: got %d, want at least %d", + last.Version, + minVersion, + ) +} + func (s *DistributionServer) loadCatalogSnapshotAt(ctx context.Context, readTS uint64) (distribution.CatalogSnapshot, error) { if s.catalog == nil { return distribution.CatalogSnapshot{}, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) @@ -1076,6 +1749,313 @@ func (s *DistributionServer) applyEngineSnapshot(snapshot distribution.CatalogSn return nil } +func (s *DistributionServer) updateSplitJobViaCoordinator( + ctx context.Context, + jobID uint64, + transition func(distribution.SplitJob) (distribution.SplitJob, error), +) error { + expected, readTS, err := s.catalog.LiveSplitJobForUpdate(ctx, jobID) + if err != nil { + return splitJobCatalogStatusError(err) + } + next, err := transition(expected) + if err != nil { + return splitJobCatalogStatusError(err) + } + if distribution.SplitJobsEquivalent(expected, next) { + return nil + } + encoded, err := distribution.EncodeSplitJob(next) + if err != nil { + return splitJobCatalogStatusError(err) + } + key := distribution.CatalogSplitJobKey(jobID) + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{{ + Op: kv.Put, + Key: key, + Value: encoded, + }}, + IsTxn: true, + StartTS: readTS, + ReadKeys: [][]byte{key}, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + return nil +} + +func (s *DistributionServer) completeSplitJobTargetPromotionViaCoordinator( + ctx context.Context, + expectedVersion uint64, + expected distribution.SplitJob, + nowMs int64, +) (distribution.CatalogSnapshot, distribution.SplitJob, error) { + if err := s.requirePromotionCompletionDeps(); err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, err + } + snapshot, current, alreadyDone, err := s.loadPromotionCompletionCandidate(ctx, expectedVersion, expected, nowMs) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, err + } + if alreadyDone { + return snapshot, current, nil + } + completion, err := distribution.CompleteTargetPromotionState(current, snapshot.Routes, nowMs) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, splitJobPromotionStatusError(err) + } + return s.dispatchPromotionCompletion(ctx, snapshot, expectedVersion, expected.JobID, completion) +} + +func (s *DistributionServer) requirePromotionCompletionDeps() error { + if s.catalog == nil { + return grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + if s.coordinator == nil { + return grpcStatusError(codes.FailedPrecondition, errDistributionCoordinatorRequired.Error()) + } + return nil +} + +func (s *DistributionServer) loadPromotionCompletionCandidate( + ctx context.Context, + expectedVersion uint64, + expected distribution.SplitJob, + nowMs int64, +) (distribution.CatalogSnapshot, distribution.SplitJob, bool, error) { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, err + } + current, found, err := s.catalog.SplitJobAt(ctx, expected.JobID, snapshot.ReadTS) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(err) + } + if !found { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(distribution.ErrCatalogSplitJobConflict) + } + if snapshot.Version == expectedVersion && distribution.SplitJobsEquivalent(current, expected) { + return snapshot, current, false, nil + } + completion, err := completedPromotionRetry(snapshot.Routes, expected, current, nowMs) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(err) + } + if completion.Job.TargetPromotionDone { + return snapshot, current, !completion.Changed, nil + } + if snapshot.Version != expectedVersion { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(distribution.ErrCatalogVersionMismatch) + } + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(distribution.ErrCatalogSplitJobConflict) +} + +func completedPromotionRetry( + routes []distribution.RouteDescriptor, + expected distribution.SplitJob, + current distribution.SplitJob, + nowMs int64, +) (distribution.TargetPromotionCompletion, error) { + matches, err := splitJobPromotionMatchesExpected(expected, current) + if err != nil { + return distribution.TargetPromotionCompletion{}, errors.WithStack(err) + } + if !matches { + return distribution.TargetPromotionCompletion{}, nil + } + completion, err := distribution.CompleteTargetPromotionState(current, routes, nowMs) + if err != nil { + return distribution.TargetPromotionCompletion{}, errors.WithStack(err) + } + return completion, nil +} + +func (s *DistributionServer) dispatchPromotionCompletion( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + expectedVersion uint64, + jobID uint64, + completion distribution.TargetPromotionCompletion, +) (distribution.CatalogSnapshot, distribution.SplitJob, error) { + if !completion.Changed { + return snapshot, completion.Job, nil + } + commitTS, err := s.nextPromotionCompleteCommitTS(ctx, snapshot.ReadTS) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, err + } + if completion.Job.PromotionCompletedTS == 0 { + completion.Job.PromotionCompletedTS = commitTS + } + ops, err := s.buildPromotionCompleteOps(expectedVersion, completion) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, splitJobPromotionStatusError(err) + } + jobKey := distribution.CatalogSplitJobKey(jobID) + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: ops, + IsTxn: true, + StartTS: snapshot.ReadTS, + CommitTS: commitTS, + ReadKeys: [][]byte{ + distribution.CatalogVersionKey(), + jobKey, + }, + }); err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, splitJobCoordinatorStatusError(err) + } + loaded, err := s.loadCatalogSnapshotAtLeastVersion(ctx, expectedVersion+1) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, err + } + return loaded, completion.Job, nil +} + +func (s *DistributionServer) nextPromotionCompleteCommitTS(ctx context.Context, readTS uint64) (uint64, error) { + if readTS == math.MaxUint64 { + return 0, grpcStatusError(codes.Internal, distribution.ErrCatalogVersionOverflow.Error()) + } + commitTS, err := kv.NextTimestampAfterThrough(ctx, s.coordinator, readTS, "allocate promotion completion timestamp") + if err != nil { + return 0, grpcStatusErrorf(codes.FailedPrecondition, "allocate promotion completion timestamp: %v", err) + } + if commitTS <= readTS { + return 0, grpcStatusError(codes.Internal, "promotion completion timestamp did not advance") + } + return commitTS, nil +} + +func (s *DistributionServer) buildPromotionCompleteOps( + expectedVersion uint64, + completion distribution.TargetPromotionCompletion, +) ([]*kv.Elem[kv.OP], error) { + if expectedVersion == math.MaxUint64 { + return nil, errors.WithStack(distribution.ErrCatalogVersionOverflow) + } + ops := make([]*kv.Elem[kv.OP], 0, len(completion.ClearedRouteIDs)+promotionCompleteBaseOpCount) + for _, routeID := range completion.ClearedRouteIDs { + route, ok := promotionCompleteRouteByID(completion.Routes, routeID) + if !ok { + return nil, errors.WithStack(distribution.ErrMigrationPromotionTargetAbsent) + } + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(route, s.catalog.AllowsRouteDescriptorV2Writes()) + if err != nil { + return nil, errors.WithStack(err) + } + ops = append(ops, &kv.Elem[kv.OP]{ + Op: kv.Put, + Key: distribution.CatalogRouteKey(route.RouteID), + Value: encoded, + }) + } + ops = append(ops, &kv.Elem[kv.OP]{ + Op: kv.Put, + Key: distribution.CatalogVersionKey(), + Value: distribution.EncodeCatalogVersion(expectedVersion + 1), + }) + encodedJob, err := distribution.EncodeSplitJob(completion.Job) + if err != nil { + return nil, errors.WithStack(err) + } + ops = append(ops, &kv.Elem[kv.OP]{ + Op: kv.Put, + Key: distribution.CatalogSplitJobKey(completion.Job.JobID), + Value: encodedJob, + }) + return ops, nil +} + +func promotionCompleteRouteByID(routes []distribution.RouteDescriptor, routeID uint64) (distribution.RouteDescriptor, bool) { + for _, route := range routes { + if route.RouteID == routeID { + return route, true + } + } + return distribution.RouteDescriptor{}, false +} + +func splitJobPromotionMatchesExpected(expected, current distribution.SplitJob) (bool, error) { + if expected.TargetPromotionDone || + !current.TargetPromotionDone || + current.PromotionCompletedTS == 0 || + (current.Phase != expected.Phase && current.Phase != distribution.SplitJobPhaseDone) { + return false, nil + } + normalized := distribution.CloneSplitJob(current) + normalized.Phase = expected.Phase + normalized.TargetPromotionDone = expected.TargetPromotionDone + normalized.PromotionCompletedTS = expected.PromotionCompletedTS + normalized.TerminalAtMs = expected.TerminalAtMs + normalized.UpdatedAtMs = expected.UpdatedAtMs + expectedRaw, err := distribution.EncodeSplitJob(expected) + if err != nil { + return false, errors.WithStack(err) + } + normalizedRaw, err := distribution.EncodeSplitJob(normalized) + if err != nil { + return false, errors.WithStack(err) + } + return bytes.Equal(expectedRaw, normalizedRaw), nil +} + +func (s *DistributionServer) verifyKnownTargetGroup(groupID uint64) error { + if len(s.knownRaftGroups) == 0 { + return grpcStatusError(codes.FailedPrecondition, errDistributionRaftGroupsNotKnown.Error()) + } + if _, ok := s.knownRaftGroups[groupID]; !ok { + return grpcStatusError(codes.InvalidArgument, errDistributionUnknownTargetGroup.Error()) + } + return nil +} + +func (s *DistributionServer) createSplitJobViaCoordinator(ctx context.Context, readTS uint64, job distribution.SplitJob) error { + if job.JobID == math.MaxUint64 { + return splitJobCatalogStatusError(distribution.ErrCatalogSplitJobIDOverflow) + } + encoded, err := distribution.EncodeSplitJob(job) + if err != nil { + return splitJobCatalogStatusError(err) + } + jobKey := distribution.CatalogSplitJobKey(job.JobID) + nextIDKey := distribution.CatalogNextSplitJobIDKey() + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{ + { + Op: kv.Put, + Key: jobKey, + Value: encoded, + }, + { + Op: kv.Put, + Key: nextIDKey, + Value: distribution.EncodeCatalogNextSplitJobID(job.JobID + 1), + }, + }, + IsTxn: true, + StartTS: readTS, + ReadKeys: [][]byte{ + jobKey, + nextIDKey, + distribution.CatalogVersionKey(), + distribution.CatalogRouteKey(job.SourceRouteID), + }, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + return nil +} + +func (s *DistributionServer) verifySplitMigrationCapability(ctx context.Context) error { + if s.migrationCapabilityGate == nil { + return grpcStatusError(codes.FailedPrecondition, errDistributionClusterNotReady.Error()) + } + if err := s.migrationCapabilityGate(ctx); err != nil { + return err + } + return nil +} + func validateExpectedCatalogVersion(currentVersion, expectedVersion uint64) error { if currentVersion != expectedVersion { return grpcStatusError(codes.Aborted, errDistributionCatalogConflict.Error()) @@ -1135,6 +2115,29 @@ func splitJobReadFenceKeys(jobs []distribution.SplitJob) [][]byte { return readKeys } +func (s *DistributionServer) rejectLiveSplitJob(ctx context.Context, snapshot distribution.CatalogSnapshot, parent distribution.RouteDescriptor) error { + jobs, err := s.catalog.ListSplitJobsAt(ctx, snapshot.ReadTS) + if err != nil { + return grpcStatusErrorf(codes.Internal, "load split jobs: %v", err) + } + liveJobs := 0 + for _, job := range jobs { + if !splitJobIsLive(job) { + continue + } + liveJobs++ + for _, interval := range liveSplitJobIntervals(job, snapshot.Routes) { + if routeRangeIntersects(parent.Start, parent.End, interval.start, interval.end) { + return grpcStatusError(codes.Aborted, distribution.ErrSplitJobOverlap.Error()) + } + } + } + if liveJobs > 0 { + return grpcStatusError(codes.ResourceExhausted, distribution.ErrTooManyInFlightSplitJobs.Error()) + } + return nil +} + func splitJobIsLive(job distribution.SplitJob) bool { return job.Phase != distribution.SplitJobPhaseDone && job.Phase != distribution.SplitJobPhaseAbandoned } @@ -1180,6 +2183,231 @@ func routeRangeIntersects(aStart, aEnd, bStart, bEnd []byte) bool { return true } +func splitJobPassesListFilter(job distribution.SplitJob, sinceTerminalAtMs uint64, phaseFilter string) bool { + if sinceTerminalAtMs > 0 && job.TerminalAtMs > 0 && uint64(job.TerminalAtMs) < sinceTerminalAtMs { + return false + } + if phaseFilter == "" { + return true + } + return normalizeSplitJobPhaseFilter(pb.SplitJobPhase(job.Phase).String()) == phaseFilter || + normalizeSplitJobPhaseFilter(strings.TrimPrefix(pb.SplitJobPhase(job.Phase).String(), "SPLIT_JOB_PHASE_")) == phaseFilter +} + +func normalizeSplitJobPhaseFilter(phase string) string { + phase = strings.TrimSpace(phase) + if phase == "" { + return "" + } + phase = strings.ReplaceAll(phase, "-", "_") + phase = strings.ReplaceAll(phase, " ", "_") + return strings.ToUpper(phase) +} + +func validSplitJobPhaseFilter(phase string) bool { + for _, candidate := range pb.SplitJobPhase_name { + full := normalizeSplitJobPhaseFilter(candidate) + short := normalizeSplitJobPhaseFilter(strings.TrimPrefix(candidate, "SPLIT_JOB_PHASE_")) + if phase == full || phase == short { + return true + } + } + return false +} + +func splitJobCatalogStatusError(err error) error { + switch { + case errors.Is(err, distribution.ErrCatalogSplitJobIDRequired): + return grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobIDRequired.Error()) + case errors.Is(err, distribution.ErrCatalogSplitJobNotFound): + return grpcStatusError(codes.NotFound, distribution.ErrCatalogSplitJobNotFound.Error()) + case errors.Is(err, distribution.ErrCatalogSplitJobCannotRetry), + errors.Is(err, distribution.ErrCatalogSplitJobCannotAbandon): + return grpcStatusError(codes.FailedPrecondition, err.Error()) + case errors.Is(err, distribution.ErrCatalogSplitJobConflict): + return grpcStatusError(codes.Aborted, distribution.ErrCatalogSplitJobConflict.Error()) + default: + return grpcStatusErrorf(codes.Internal, "split job catalog: %v", err) + } +} + +func splitMigrationRangeStatusError(err error) error { + switch { + case errors.Is(err, distribution.ErrMigrationResolverOwnedRange): + return grpcStatusError(codes.FailedPrecondition, err.Error()) + case errors.Is(err, distribution.ErrMigrationReservedRange), + errors.Is(err, distribution.ErrMigrationInvalidRoute): + return grpcStatusError(codes.InvalidArgument, err.Error()) + default: + return grpcStatusErrorf(codes.Internal, "validate split migration range: %v", err) + } +} + +func splitJobCoordinatorStatusError(err error) error { + switch { + case errors.Is(err, store.ErrWriteConflict): + return grpcStatusError(codes.Aborted, distribution.ErrCatalogSplitJobConflict.Error()) + case splitJobCoordinatorLeadershipError(err): + return grpcStatusError(codes.FailedPrecondition, errDistributionNotLeader.Error()) + default: + return grpcStatusErrorf(codes.Internal, "commit split job mutation: %v", err) + } +} + +func splitJobPromotionStatusError(err error) error { + switch { + case errors.Is(err, distribution.ErrCatalogVersionMismatch), + errors.Is(err, distribution.ErrCatalogSplitJobConflict), + errors.Is(err, store.ErrWriteConflict): + return grpcStatusError(codes.Aborted, err.Error()) + case errors.Is(err, distribution.ErrMigrationPromotionNotReady), + errors.Is(err, distribution.ErrMigrationPromotionTargetAbsent): + return grpcStatusError(codes.FailedPrecondition, err.Error()) + case errors.Is(err, distribution.ErrMigrationInvalidRoute), + errors.Is(err, distribution.ErrCatalogRouteV2WriteDisabled): + return grpcStatusError(codes.InvalidArgument, err.Error()) + default: + return splitJobCatalogStatusError(err) + } +} + +func splitJobListPage(jobs []distribution.SplitJob, req *pb.ListSplitJobsRequest, phaseFilter string) (*pb.ListSplitJobsResponse, error) { + filtered := make([]distribution.SplitJob, 0, len(jobs)) + for _, job := range jobs { + if splitJobPassesListFilter(job, req.GetSinceTerminalAtMs(), phaseFilter) { + filtered = append(filtered, job) + } + } + sortSplitJobsForList(filtered) + + start, err := splitJobListStartIndex(filtered, req.GetPageCursor()) + if err != nil { + return nil, err + } + end := start + listSplitJobsDefaultPageSize + if end > len(filtered) { + end = len(filtered) + } + + resp := &pb.ListSplitJobsResponse{ + Jobs: make([]*pb.SplitJob, 0, end-start), + } + for _, job := range filtered[start:end] { + resp.Jobs = append(resp.Jobs, distribution.SplitJobToProto(job)) + } + if end < len(filtered) { + resp.NextPageCursor = encodeSplitJobListCursor(filtered[end-1]) + } + return resp, nil +} + +func sortSplitJobsForList(jobs []distribution.SplitJob) { + sort.Slice(jobs, func(i, j int) bool { + left, right := jobs[i], jobs[j] + leftLive := left.TerminalAtMs <= 0 + rightLive := right.TerminalAtMs <= 0 + if leftLive != rightLive { + return leftLive + } + if leftLive { + if left.UpdatedAtMs != right.UpdatedAtMs { + return left.UpdatedAtMs > right.UpdatedAtMs + } + return left.JobID > right.JobID + } + if left.TerminalAtMs != right.TerminalAtMs { + return left.TerminalAtMs > right.TerminalAtMs + } + return left.JobID > right.JobID + }) +} + +func splitJobListStartIndex(jobs []distribution.SplitJob, cursor []byte) (int, error) { + if len(cursor) == 0 { + return 0, nil + } + terminalAtMs, jobID, err := decodeSplitJobListCursor(cursor) + if err != nil { + return 0, err + } + for i, job := range jobs { + if job.JobID == jobID && splitJobListCursorTerminalAtMs(job) == terminalAtMs { + return i + 1, nil + } + if terminalAtMs > 0 && splitJobHistoryAfterCursor(job, terminalAtMs, jobID) { + return i, nil + } + } + if terminalAtMs > 0 { + return len(jobs), nil + } + return 0, grpcStatusError(codes.InvalidArgument, "live split job page cursor is no longer available") +} + +func splitJobHistoryAfterCursor(job distribution.SplitJob, terminalAtMs, jobID uint64) bool { + jobTerminalAtMs := splitJobListCursorTerminalAtMs(job) + return jobTerminalAtMs > 0 && (jobTerminalAtMs < terminalAtMs || (jobTerminalAtMs == terminalAtMs && job.JobID < jobID)) +} + +func encodeSplitJobListCursor(job distribution.SplitJob) []byte { + cursor := make([]byte, splitJobListCursorEncodedBytes) + cursor[0] = splitJobListCursorVersion + binary.BigEndian.PutUint64( + cursor[splitJobListCursorTerminalOff:splitJobListCursorJobIDOff], + splitJobListCursorTerminalAtMs(job), + ) + binary.BigEndian.PutUint64(cursor[splitJobListCursorJobIDOff:], job.JobID) + return cursor +} + +func decodeSplitJobListCursor(cursor []byte) (uint64, uint64, error) { + if len(cursor) != splitJobListCursorEncodedBytes || cursor[0] != splitJobListCursorVersion { + return 0, 0, grpcStatusError(codes.InvalidArgument, "invalid split job page cursor") + } + terminalAtMs := binary.BigEndian.Uint64(cursor[splitJobListCursorTerminalOff:splitJobListCursorJobIDOff]) + jobID := binary.BigEndian.Uint64(cursor[splitJobListCursorJobIDOff:]) + if jobID == 0 { + return 0, 0, grpcStatusError(codes.InvalidArgument, "invalid split job page cursor") + } + return terminalAtMs, jobID, nil +} + +func splitJobListCursorTerminalAtMs(job distribution.SplitJob) uint64 { + if job.TerminalAtMs <= 0 { + return 0 + } + return uint64(job.TerminalAtMs) +} + +func splitJobCoordinatorLeadershipError(err error) bool { + if err == nil { + return false + } + if errors.Is(err, kv.ErrLeaderNotFound) || + errors.Is(err, raftengine.ErrNotLeader) || + errors.Is(err, raftengine.ErrLeadershipLost) || + errors.Is(err, raftengine.ErrLeadershipTransferInProgress) { + return true + } + return hasSplitJobLeaderErrorSuffix(err.Error()) +} + +var splitJobLeaderErrorPhrases = []string{ + ErrNotLeader.Error(), + "leader not found", + "leadership lost", + "leadership transfer in progress", +} + +func hasSplitJobLeaderErrorSuffix(msg string) bool { + for _, phrase := range splitJobLeaderErrorPhrases { + if len(msg) >= len(phrase) && strings.EqualFold(msg[len(msg)-len(phrase):], phrase) { + return true + } + } + return false +} + func splitCatalogRoutes( parent distribution.RouteDescriptor, splitKey []byte, diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 5d2605ade..703a13321 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -5,6 +5,8 @@ import ( "context" "encoding/binary" stderrors "errors" + "io" + "math" "net" "testing" "time" @@ -1615,10 +1617,21 @@ type distributionCoordinatorStub struct { // dispatchErr, when set, fails every Dispatch so callers' retry and // error-propagation paths can be exercised. dispatchErr error + // timestampNext/timestampErr/timestampCalls drive the split-job runner's + // direct timestamp probes, which do not go through Dispatch. + timestampNext uint64 + timestampErr error + timestampCalls int } func (s *distributionCoordinatorStub) TimestampAllocator() kv.TimestampAllocator { - return s.allocator + if s.allocator != nil { + return s.allocator + } + // With no explicit allocator the stub allocates for itself, so tests that + // drive timestampNext / assert on timestampCalls see their own values + // rather than a real clock. + return s } func (s *distributionCoordinatorStub) VouchAppliedReadTimestamp(uint64, kv.AppliedReadTimestampVoucherRef) error { @@ -2160,3 +2173,1476 @@ func TestDistributionServerGetRouteOwnership_NormalizesS3BucketAuxiliaryKeys(t * require.Equal(t, uint64(2), resp.Route.RouteId) } } + +func TestDistributionServerPinReadTSWithoutTrackerReturnsReleasableToken(t *testing.T) { + t.Parallel() + + s := NewDistributionServer(distribution.NewEngine(), nil) + token := s.pinReadTS(10) + require.NotNil(t, token) + require.NotPanics(t, func() { + token.Release() + }) + + var nilServer *DistributionServer + nilToken := nilServer.pinReadTS(10) + require.NotNil(t, nilToken) + require.NotPanics(t, func() { + nilToken.Release() + }) +} + +func TestDistributionServerGetSplitMigrationCapabilityReportsNotReadyUntilRunnerReady(t *testing.T) { + t.Parallel() + + s := NewDistributionServer(distribution.NewEngine(), nil) + resp, err := s.GetSplitMigrationCapability(context.Background(), &pb.GetSplitMigrationCapabilityRequest{}) + require.NoError(t, err) + require.False(t, resp.GetMigrationCapable()) + require.NotContains(t, resp.GetCapabilities(), splitMigrationCapabilityV2) +} + +func TestDistributionServerGetSplitMigrationCapabilityReportsReadyWhenRunnerReady(t *testing.T) { + t.Parallel() + + s := NewDistributionServer(distribution.NewEngine(), nil, WithSplitJobRunnerReady()) + resp, err := s.GetSplitMigrationCapability(context.Background(), &pb.GetSplitMigrationCapabilityRequest{}) + require.NoError(t, err) + require.True(t, resp.GetMigrationCapable()) + require.Contains(t, resp.GetCapabilities(), splitMigrationCapabilityV2) +} + +func TestDistributionServerGetSplitMigrationCapabilityRespectsReadinessGate(t *testing.T) { + t.Parallel() + + s := NewDistributionServer( + distribution.NewEngine(), + nil, + WithSplitJobRunnerReady(), + WithSplitJobRunnerReadinessGate(func(context.Context) error { + return status.Error(codes.FailedPrecondition, "migration opcode disabled") + }), + ) + resp, err := s.GetSplitMigrationCapability(context.Background(), &pb.GetSplitMigrationCapabilityRequest{}) + require.NoError(t, err) + require.False(t, resp.GetMigrationCapable()) + require.NotContains(t, resp.GetCapabilities(), splitMigrationCapabilityV2) +} + +func TestDistributionServerStartSplitMigration_FailsClosedUntilCapabilityGate(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + + _, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: 1, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, errDistributionClusterNotReady.Error()) + require.Zero(t, coordinator.dispatchCalls) + + jobs, listErr := catalog.ListSplitJobs(ctx) + require.NoError(t, listErr) + require.Empty(t, jobs) +} + +func TestDistributionServerStartSplitMigrationReturnsCapabilityGateError(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitMigrationCapabilityGate(func(context.Context) error { + return status.Error(codes.Unavailable, "split migration capability not ready") + }), + ) + + _, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: 1, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.ErrorContains(t, err, "split migration capability not ready") + require.Zero(t, coordinator.dispatchCalls) +} + +func TestDistributionServerStartSplitMigrationCapabilityGateRunsOutsideCatalogLock(t *testing.T) { + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + gateEntered := make(chan struct{}) + releaseGate := make(chan struct{}) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { + close(gateEntered) + <-releaseGate + return status.Error(codes.Unavailable, "split migration capability not ready") + }), + ) + + startDone := make(chan error, 1) + go func() { + _, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + startDone <- err + }() + <-gateEntered + + splitDone := make(chan error, 1) + go func() { + _, err := s.SplitRange(ctx, &pb.SplitRangeRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + }) + splitDone <- err + }() + select { + case err := <-splitDone: + require.NoError(t, err) + case <-time.After(500 * time.Millisecond): + t.Fatal("SplitRange blocked behind StartSplitMigration capability gate") + } + + close(releaseGate) + err = <-startDone + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) +} + +func TestDistributionServerStartSplitMigration_CreatesPlannedJobWhenGateOpen(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + gateCalls := 0 + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { + gateCalls++ + return nil + }), + ) + + resp, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + require.NoError(t, err) + require.Equal(t, saved.Version, resp.CatalogVersion) + require.Equal(t, uint64(1), resp.JobId) + require.Equal(t, 1, gateCalls) + require.Equal(t, 1, coordinator.dispatchCalls) + require.ElementsMatch(t, []string{ + string(distribution.CatalogSplitJobKey(1)), + string(distribution.CatalogNextSplitJobIDKey()), + string(distribution.CatalogVersionKey()), + string(distribution.CatalogRouteKey(1)), + }, byteSliceStrings(coordinator.lastReadKeys)) + + jobs, err := catalog.ListSplitJobs(ctx) + require.NoError(t, err) + require.Len(t, jobs, 1) + job := jobs[0] + require.Equal(t, uint64(1), job.JobID) + require.Equal(t, uint64(1), job.SourceRouteID) + require.Equal(t, []byte("g"), job.SplitKey) + require.Equal(t, uint64(2), job.TargetGroupID) + require.Equal(t, distribution.SplitJobPhasePlanned, job.Phase) + require.NotZero(t, job.StartedAtMs) + require.NotZero(t, job.UpdatedAtMs) + require.NotEmpty(t, job.BracketProgress) + next, err := catalog.NextSplitJobID(ctx) + require.NoError(t, err) + require.Equal(t, uint64(2), next) +} + +func TestDistributionServerStartSplitMigration_NormalizesFilesystemChunkSplitKey(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: fskeys.ChunkRouteKey(11, 21), + End: fskeys.ChunkRouteKey(11, 23), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(newDistributionCoordinatorStub(baseStore, true)), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + wantBoundary := fskeys.ChunkRouteKey(11, 22) + + resp, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: fskeys.ChunkKey(11, 22, 99), + TargetGroupId: 2, + }) + require.NoError(t, err) + require.Equal(t, uint64(1), resp.JobId) + + jobs, err := catalog.ListSplitJobs(ctx) + require.NoError(t, err) + require.Len(t, jobs, 1) + require.Equal(t, wantBoundary, jobs[0].SplitKey) + require.NotEqual(t, fskeys.ChunkKey(11, 22, 99), jobs[0].SplitKey) +} + +func TestDistributionServerStartSplitMigration_RejectsUnknownTargetGroup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + + _, err = s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 3, + }) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + require.ErrorContains(t, err, errDistributionUnknownTargetGroup.Error()) + require.Zero(t, coordinator.dispatchCalls) +} + +func TestDistributionServerStartSplitMigration_RejectsNonActiveSourceRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + for _, tc := range []struct { + name string + state distribution.RouteState + }{ + {name: "write_fenced", state: distribution.RouteStateWriteFenced}, + {name: "migrating_source", state: distribution.RouteStateMigratingSource}, + {name: "migrating_target", state: distribution.RouteStateMigratingTarget}, + } { + t.Run(tc.name, func(t *testing.T) { + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: tc.state, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + + _, err = s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, errDistributionSourceRouteNotActive.Error()) + require.Zero(t, coordinator.dispatchCalls) + + jobs, listErr := catalog.ListSplitJobs(ctx) + require.NoError(t, listErr) + require.Empty(t, jobs) + }) + } +} + +func TestDistributionServerStartSplitMigration_RejectsSecondLiveJob(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }, + { + RouteID: 2, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }, + }) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, distribution.SplitJob{ + JobID: 1, + SourceRouteID: 2, + SplitKey: []byte("t"), + TargetGroupID: 3, + Phase: distribution.SplitJobPhaseBackfill, + StartedAtMs: 1000, + UpdatedAtMs: 1000, + })) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2, 3, 4), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + + _, err = s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 4, + }) + require.Error(t, err) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) + require.ErrorContains(t, err, distribution.ErrTooManyInFlightSplitJobs.Error()) + require.Zero(t, coordinator.dispatchCalls) +} + +func TestDistributionServerStartSplitMigration_RejectsReservedMigrationRange(t *testing.T) { + t.Parallel() + + ctx := context.Background() + for _, tc := range []struct { + name string + splitKey []byte + end []byte + }{ + {name: "dist catalog", splitKey: []byte("!dist|"), end: nil}, + {name: "dist staged catalog", splitKey: []byte("!dist|migstage|"), end: nil}, + {name: "migration staged", splitKey: []byte("!migstage|ready|1"), end: nil}, + {name: "migration tracker", splitKey: []byte("!migwrite|"), end: nil}, + {name: "migration fence", splitKey: []byte("!migfence|"), end: nil}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte(""), + End: tc.end, + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + + _, err = s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: tc.splitKey, + TargetGroupId: 2, + }) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + require.ErrorContains(t, err, distribution.ErrMigrationReservedRange.Error()) + require.Zero(t, coordinator.dispatchCalls) + + jobs, listErr := catalog.ListSplitJobs(ctx) + require.NoError(t, listErr) + require.Empty(t, jobs) + }) + } +} + +func TestDistributionServerSplitJobRPCs_ReadAndListCatalogJobs(t *testing.T) { + t.Parallel() + + ctx := context.Background() + catalog := distribution.NewCatalogStore(store.NewMVCCStore()) + live := sampleDistributionSplitJob(10) + live.Phase = distribution.SplitJobPhaseDeltaCopy + require.NoError(t, catalog.CreateSplitJob(ctx, live)) + history := sampleDistributionSplitJob(11) + history.Phase = distribution.SplitJobPhaseDone + history.TerminalAtMs = 2000 + require.NoError(t, catalog.CreateSplitJob(ctx, history)) + require.NoError(t, catalog.MoveSplitJobToHistory(ctx, history, history)) + + s := NewDistributionServer(distribution.NewEngine(), catalog) + got, err := s.GetSplitJob(ctx, &pb.GetSplitJobRequest{JobId: live.JobID}) + require.NoError(t, err) + require.Equal(t, live.JobID, got.Job.JobId) + require.Equal(t, pb.SplitJobPhase_SPLIT_JOB_PHASE_DELTA_COPY, got.Job.Phase) + + listAll, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{}) + require.NoError(t, err) + require.Len(t, listAll.Jobs, 2) + require.Equal(t, []uint64{live.JobID, history.JobID}, []uint64{listAll.Jobs[0].JobId, listAll.Jobs[1].JobId}) + + listDone, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{Phase: "done"}) + require.NoError(t, err) + require.Len(t, listDone.Jobs, 1) + require.Equal(t, history.JobID, listDone.Jobs[0].JobId) + + _, err = s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{Phase: "not-a-phase"}) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} + +func TestDistributionServerListSplitJobs_PaginatesNewestHistory(t *testing.T) { + t.Parallel() + + ctx := context.Background() + catalog := distribution.NewCatalogStore(store.NewMVCCStore()) + for jobID, terminalAtMs := uint64(1), int64(1001); jobID <= 205; jobID, terminalAtMs = jobID+1, terminalAtMs+1 { + job := sampleDistributionSplitJob(jobID) + job.Phase = distribution.SplitJobPhaseDone + job.TerminalAtMs = terminalAtMs + job.UpdatedAtMs = job.TerminalAtMs + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + require.NoError(t, catalog.MoveSplitJobToHistory(ctx, job, job)) + } + + s := NewDistributionServer(distribution.NewEngine(), catalog) + first, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{}) + require.NoError(t, err) + require.Len(t, first.Jobs, listSplitJobsDefaultPageSize) + require.NotEmpty(t, first.NextPageCursor) + require.Equal(t, uint64(205), first.Jobs[0].JobId) + require.Equal(t, uint64(6), first.Jobs[len(first.Jobs)-1].JobId) + + second, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{PageCursor: first.NextPageCursor}) + require.NoError(t, err) + require.Empty(t, second.NextPageCursor) + require.Equal(t, []uint64{5, 4, 3, 2, 1}, splitJobIDs(second.Jobs)) +} + +func TestDistributionServerListSplitJobs_RejectsInvalidCursor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + catalog := distribution.NewCatalogStore(store.NewMVCCStore()) + job := sampleDistributionSplitJob(1) + job.Phase = distribution.SplitJobPhaseDone + job.TerminalAtMs = 1000 + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + require.NoError(t, catalog.MoveSplitJobToHistory(ctx, job, job)) + + s := NewDistributionServer(distribution.NewEngine(), catalog) + _, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{PageCursor: []byte("bad")}) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + + missing := sampleDistributionSplitJob(999) + missing.TerminalAtMs = 9999 + page, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{PageCursor: encodeSplitJobListCursor(missing)}) + require.NoError(t, err) + require.Equal(t, []uint64{1}, splitJobIDs(page.Jobs)) +} + +func TestDistributionServerRetrySplitJob_UsesCoordinatorCAS(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + job := sampleDistributionSplitJob(12) + job.Phase = distribution.SplitJobPhaseFailed + job.RetryPhase = distribution.SplitJobPhaseFence + job.LastError = "retry me" + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + _, err := s.RetrySplitJob(ctx, &pb.RetrySplitJobRequest{JobId: job.JobID}) + require.NoError(t, err) + require.Equal(t, 1, coordinator.dispatchCalls) + + got, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseFence, got.Phase) + require.Equal(t, distribution.SplitJobPhaseNone, got.RetryPhase) + require.Empty(t, got.LastError) +} + +func TestDistributionServerRetrySplitJob_MapsDispatchLeadershipLoss(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + }{ + {name: "leader not found", err: kv.ErrLeaderNotFound}, + {name: "raft not leader", err: raftengine.ErrNotLeader}, + {name: "leadership lost", err: raftengine.ErrLeadershipLost}, + {name: "transfer in progress", err: raftengine.ErrLeadershipTransferInProgress}, + {name: "wrapped grpc detail", err: errors.New("rpc error: code = Unknown desc = raft engine: leadership lost")}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + job := sampleDistributionSplitJob(15) + job.Phase = distribution.SplitJobPhaseFailed + job.RetryPhase = distribution.SplitJobPhaseFence + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + coordinator.dispatchErr = tc.err + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + _, err := s.RetrySplitJob(ctx, &pb.RetrySplitJobRequest{JobId: job.JobID}) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, errDistributionNotLeader.Error()) + require.Equal(t, 1, coordinator.dispatchCalls) + }) + } +} + +func TestDistributionServerAbandonSplitJob_RecordsAbandoningViaCoordinator(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + job := sampleDistributionSplitJob(13) + job.Phase = distribution.SplitJobPhaseFailed + job.RetryPhase = distribution.SplitJobPhaseBackfill + job.AbandonFromPhase = distribution.SplitJobPhaseNone + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + _, err := s.AbandonSplitJob(ctx, &pb.AbandonSplitJobRequest{JobId: job.JobID}) + require.NoError(t, err) + require.Equal(t, 1, coordinator.dispatchCalls) + + got, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseAbandoning, got.Phase) + require.Equal(t, distribution.SplitJobPhaseNone, got.RetryPhase) + require.Equal(t, distribution.SplitJobPhaseBackfill, got.AbandonFromPhase) +} + +func TestDistributionServerCompleteSplitJobTargetPromotion_UsesCoordinatorCAS(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, promotionCompleteDistributionRoutes()) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + before, err := catalog.Snapshot(ctx) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + coordinator.timestampNext = before.ReadTS + 10 + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + + snapshot, completed, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, saved.Version, job, 2000) + require.NoError(t, err) + require.Equal(t, saved.Version+1, snapshot.Version) + require.True(t, completed.TargetPromotionDone) + require.Equal(t, distribution.SplitJobPhaseCleanup, completed.Phase) + require.Equal(t, before.ReadTS+10, completed.PromotionCompletedTS) + require.Equal(t, int64(2000), completed.UpdatedAtMs) + require.Zero(t, completed.TerminalAtMs) + require.Equal(t, 1, coordinator.dispatchCalls) + require.Equal(t, 1, coordinator.timestampCalls) + require.Equal(t, before.ReadTS, coordinator.lastStartTS) + require.Equal(t, completed.PromotionCompletedTS, coordinator.lastCommitTS) + requireReadKeysContain(t, coordinator.lastReadKeys, distribution.CatalogVersionKey()) + requireReadKeysContain(t, coordinator.lastReadKeys, distribution.CatalogSplitJobKey(job.JobID)) + + loadedJob, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, completed, loadedJob) + + loaded, err := catalog.Snapshot(ctx) + require.NoError(t, err) + target := distributionRouteByID(t, loaded.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) +} + +func TestDistributionServerCompleteSplitJobTargetPromotion_RetryAfterCommitIsIdempotent(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, promotionCompleteDistributionRoutes()) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + + firstSnapshot, firstCompleted, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, saved.Version, job, 2000) + require.NoError(t, err) + retrySnapshot, retryCompleted, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, saved.Version, job, 3000) + require.NoError(t, err) + require.Equal(t, firstSnapshot.Version, retrySnapshot.Version) + require.Equal(t, firstCompleted, retryCompleted) + require.Equal(t, 1, coordinator.dispatchCalls) +} + +func TestDistributionServerCompleteSplitJobTargetPromotion_PersistsLegacyTerminalTime(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + routes := promotionCompleteDistributionRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + saved, err := catalog.Save(ctx, 0, routes) + require.NoError(t, err) + expected := promotionCompleteDistributionJob() + legacy := distribution.CloneSplitJob(expected) + legacy.Phase = distribution.SplitJobPhaseDone + legacy.TargetPromotionDone = true + legacy.PromotionCompletedTS = saved.ReadTS + 1 + require.NoError(t, catalog.CreateSplitJob(ctx, legacy)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + + snapshot, completed, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, saved.Version, expected, 3000) + require.NoError(t, err) + require.Equal(t, saved.Version+1, snapshot.Version) + require.Equal(t, int64(3000), completed.TerminalAtMs) + require.Equal(t, int64(3000), completed.UpdatedAtMs) + require.Equal(t, 1, coordinator.dispatchCalls) + + loaded, found, err := catalog.SplitJob(ctx, legacy.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, completed, loaded) +} + +func TestDistributionServerRunSplitJobRunnerOnce_PromotesCleanupJob(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, promotionCompleteDistributionRoutes()) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + client := &splitPromotionClientStub{ + responses: []*pb.PromoteStagedVersionsResponse{ + {NextCursor: []byte("cursor-1")}, + {Done: true}, + }, + } + s, _, _ := newSplitRuntimeRunner(t, baseStore, catalog, job) + coordinator, ok := s.coordinator.(*ownershipCatalogCoordinator) + require.True(t, ok) + s.splitPromotionClientFactory = func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { return client, nil } + + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + require.Equal(t, 2, client.calls) + require.Equal(t, job.JobID, client.requests[0].GetJobId()) + require.Empty(t, client.requests[0].GetCursor()) + require.Equal(t, defaultSplitPromotionMaxVersions, client.requests[0].GetMaxVersions()) + require.Equal(t, defaultSplitPromotionMaxBytes, client.requests[0].GetMaxBytes()) + require.Equal(t, defaultSplitPromotionMaxScanned, client.requests[0].GetMaxScannedBytes()) + require.Equal(t, []byte("cursor-1"), client.requests[1].GetCursor()) + require.Equal(t, 1, coordinator.dispatchCalls) + + loadedJob, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.True(t, loadedJob.TargetPromotionDone) + require.Equal(t, distribution.SplitJobPhaseCleanup, loadedJob.Phase) + require.NotZero(t, loadedJob.PromotionCompletedTS) + require.Zero(t, loadedJob.TerminalAtMs) + + loaded, err := catalog.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, saved.Version+1, loaded.Version) + target := distributionRouteByID(t, loaded.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) + + route, ok := s.engine.GetRoute([]byte("z")) + require.True(t, ok) + require.False(t, route.StagedVisibilityActive) + require.Equal(t, uint64(0), route.MigrationJobID) + require.Equal(t, saved.Version+1, s.engine.Version()) +} + +func TestDistributionServerRunSplitJobRunnerOnce_TerminalizesPromotedCleanupJob(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + routes := promotionCompleteDistributionRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + saved, err := catalog.Save(ctx, 0, routes) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + job.TargetPromotionDone = true + job.PromotionCompletedTS = saved.ReadTS + 1 + job.UpdatedAtMs = 2000 + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + s, _, target := newSplitRuntimeRunner(t, baseStore, catalog, job) + + for range 200 { + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + current, found, loadErr := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, loadErr) + require.True(t, found) + if current.Phase == distribution.SplitJobPhaseDone { + break + } + } + require.Zero(t, target.calls) + + loadedJob, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseDone, loadedJob.Phase) + require.True(t, loadedJob.TargetPromotionDone) + require.Equal(t, job.PromotionCompletedTS, loadedJob.PromotionCompletedTS) + + loaded, err := catalog.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, saved.Version, loaded.Version) +} + +func TestDistributionServerRunSplitJobRunnerOnce_CompletesCrossGroupMigration(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 1, + SourceRouteID: 1, + SplitKey: []byte("m"), + TargetGroupID: 2, + }, saved.Routes[0], time.Now().UnixMilli()) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + s, source, target := newSplitRuntimeRunner(t, baseStore, catalog, job) + + for range 200 { + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + current, found, loadErr := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, loadErr) + require.True(t, found) + if current.Phase == distribution.SplitJobPhaseDone { + break + } + } + + completed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseDone, completed.Phase) + require.True(t, completed.WriteTrackerArmed) + require.True(t, completed.PostFenceDrainCompleted) + require.NotZero(t, completed.SnapshotTS) + require.NotZero(t, completed.FenceTS) + require.NotZero(t, completed.CutoverVersion) + require.True(t, completed.TargetPromotionDone) + require.NotEmpty(t, source.controls) + require.NotEmpty(t, target.controls) + require.NotEmpty(t, source.exports) + require.True(t, completed.TransferPreflightDone) + preflightExports := 0 + for _, req := range source.exports { + if req.GetMaxCommitTs() == completed.TransferPreflightTS { + require.Zero(t, req.GetMinCommitTs()) + preflightExports++ + } + } + require.Equal(t, len(completed.TransferPreflightProgress), preflightExports) + require.Len(t, target.imports, len(source.exports)-preflightExports) + require.GreaterOrEqual(t, len(source.events), 4) + require.Equal(t, []string{"control", "guard", "locks", "timestamp"}, source.events[:4], "tracker and transfer guard barriers must precede lock drain and snapshot timestamp selection") + + finalSnapshot, err := catalog.Snapshot(ctx) + require.NoError(t, err) + require.Len(t, finalSnapshot.Routes, 2) + right := distributionRouteByID(t, finalSnapshot.Routes, 3) + require.Equal(t, uint64(2), right.GroupID) + require.Equal(t, distribution.RouteStateActive, right.State) + require.False(t, right.StagedVisibilityActive) + require.Equal(t, completed.FenceTS, right.MinWriteTSExclusive) + value, err := target.internal.store.GetAt(ctx, []byte("n"), ^uint64(0)) + require.NoError(t, err) + require.Equal(t, []byte("v"), value, "the real import and promotion must retain source data") + _, err = source.internal.store.GetAt(ctx, []byte("n"), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound, "source cleanup must remove the transferred version") +} + +func TestDistributionServerRunSplitJobRunnerOnce_CompletesAbandonCleanup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 7, + SourceRouteID: 1, + SplitKey: []byte("m"), + TargetGroupID: 2, + }, saved.Routes[0], time.Now().UnixMilli()) + require.NoError(t, err) + job.Phase = distribution.SplitJobPhaseAbandoning + job.AbandonFromPhase = distribution.SplitJobPhaseBackfill + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + s, source, target := newSplitRuntimeRunner(t, baseStore, catalog, job) + + for range 10 { + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + current, found, loadErr := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, loadErr) + require.True(t, found) + if current.Phase == distribution.SplitJobPhaseAbandoned { + break + } + } + completed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseAbandoned, completed.Phase) + require.Positive(t, completed.TerminalAtMs) + require.NotEmpty(t, source.cleanups) + require.NotEmpty(t, target.cleanups) + require.Equal(t, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, target.cleanups[0].GetMode()) +} + +func TestDistributionServerRunSplitJobRunnerOnce_PausesOnCapabilityRegression(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 9, + SourceRouteID: 1, + SplitKey: []byte("m"), + TargetGroupID: 2, + }, saved.Routes[0], time.Now().UnixMilli()) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + capabilityErr := errors.New("node capability missing") + s, source, _ := newSplitRuntimeRunner(t, baseStore, catalog, job) + s.migrationCapabilityGate = func(context.Context) error { return capabilityErr } + + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + paused, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.True(t, paused.CapabilityRegressed) + require.Equal(t, distribution.SplitJobPhasePlanned, paused.Phase) + require.Empty(t, source.controls) + + s.migrationCapabilityGate = func(context.Context) error { return nil } + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + resumed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.False(t, resumed.CapabilityRegressed) + require.Equal(t, distribution.SplitJobPhasePlanned, resumed.Phase) + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + armed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseBackfill, armed.Phase) + require.NotZero(t, armed.SnapshotTS) +} + +func TestDistributionServerCleanupSplitJobSourceProofsWaitsForMetadataBarrier(t *testing.T) { + t.Parallel() + testSourceMetadataOwnershipBarrier(t) +} + +func TestDistributionServerRunSplitJobRunnerOnce_FailClosesCapabilityRegressionAfterSideEffects(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 10, + SourceRouteID: 1, + SplitKey: []byte("m"), + TargetGroupID: 2, + }, saved.Routes[0], time.Now().UnixMilli()) + require.NoError(t, err) + job.Phase = distribution.SplitJobPhaseBackfill + job.SnapshotTS = 50 + job.WriteTrackerArmed = true + job.SourceRetentionPinTS = initialMigrationRetentionPinTS + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + source := &splitMigrationClientStub{} + target := &splitMigrationClientStub{} + capabilityErr := errors.New("node capability missing") + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(newDistributionCoordinatorStub(baseStore, true)), + WithSplitMigrationCapabilityGate(func(context.Context) error { return capabilityErr }), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return target, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, target, nil + }), + ) + + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + regressed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.True(t, regressed.CapabilityRegressed) + require.Len(t, source.controls, 1) + require.Len(t, target.controls, 1) + sourceFailClosed := source.controls[0] + require.Equal(t, job.JobID, sourceFailClosed.GetJobId()) + require.Equal(t, []byte("m"), sourceFailClosed.GetRouteStart()) + require.Equal(t, []byte("z"), sourceFailClosed.GetRouteEnd()) + require.True(t, sourceFailClosed.GetSourceWriteFence()) + require.True(t, sourceFailClosed.GetSourceReadFence()) + require.True(t, sourceFailClosed.GetTrackWrites()) + require.Equal(t, uint64(50), sourceFailClosed.GetMinWriteTsExclusive()) + targetFailClosed := target.controls[0] + require.False(t, targetFailClosed.GetSourceWriteFence()) + require.False(t, targetFailClosed.GetSourceReadFence()) + require.False(t, targetFailClosed.GetTrackWrites()) + require.Equal(t, uint64(50), targetFailClosed.GetMinWriteTsExclusive()) + + s.migrationCapabilityGate = func(context.Context) error { return nil } + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + resumed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.False(t, resumed.CapabilityRegressed) + require.Len(t, source.controls, 2) + sourceRestored := source.controls[1] + require.False(t, sourceRestored.GetSourceWriteFence()) + require.False(t, sourceRestored.GetSourceReadFence()) + require.True(t, sourceRestored.GetTrackWrites()) + require.Equal(t, initialMigrationRetentionPinTS, sourceRestored.GetMinWriteTsExclusive()) + require.Len(t, target.controls, 1) +} + +func TestSplitJobHistoryGCKeysAppliesTTLAndCountBound(t *testing.T) { + t.Parallel() + + now := time.UnixMilli(10 * splitJobHistoryTTL.Milliseconds()) + jobs := make([]distribution.SplitJob, 0, splitJobHistoryLimit+3) + jobs = append(jobs, distribution.SplitJob{ + JobID: 1, + Phase: distribution.SplitJobPhaseDone, + TerminalAtMs: now.Add(-splitJobHistoryTTL - time.Second).UnixMilli(), + }) + for i := 0; i < splitJobHistoryLimit+2; i++ { + jobs = append(jobs, distribution.SplitJob{ + JobID: uint64(i) + 2, //nolint:gosec // loop bound is splitJobHistoryLimit. + Phase: distribution.SplitJobPhaseDone, + TerminalAtMs: now.Add(-time.Hour).UnixMilli() + int64(i), + }) + } + + keys := splitJobHistoryGCKeys(jobs, now) + require.Len(t, keys, 3) + require.Equal(t, distribution.CatalogSplitJobHistoryKey(jobs[0].TerminalAtMs, jobs[0].JobID), keys[0]) +} + +func TestDistributionServerRunSplitJobRunnerOnce_SkipsFollower(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + _, err := catalog.Save(ctx, 0, promotionCompleteDistributionRoutes()) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, promotionCompleteDistributionJob())) + + client := &splitPromotionClientStub{ + responses: []*pb.PromoteStagedVersionsResponse{{Done: true}}, + } + coordinator := newDistributionCoordinatorStub(baseStore, false) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return client, nil + }), + ) + + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + require.Zero(t, client.calls) + require.Zero(t, coordinator.dispatchCalls) +} + +func TestSplitJobRunnerContextDoneRecognizesWrappedGRPCStatus(t *testing.T) { + t.Parallel() + + require.True(t, splitJobRunnerContextDone(context.Canceled)) + require.True(t, splitJobRunnerContextDone(context.DeadlineExceeded)) + require.True(t, splitJobRunnerContextDone(errors.WithStack(status.Error(codes.Canceled, "stopping")))) + require.True(t, splitJobRunnerContextDone(errors.WithStack(status.Error(codes.DeadlineExceeded, "stopping")))) + require.False(t, splitJobRunnerContextDone(errors.WithStack(status.Error(codes.Unavailable, "not leader")))) +} + +func TestPromoteSplitJobTargetRejectsNoCursorProgress(t *testing.T) { + t.Parallel() + + client := &splitPromotionClientStub{ + responses: []*pb.PromoteStagedVersionsResponse{{NextCursor: []byte("same")}}, + } + err := promoteSplitJobTarget(context.Background(), client, promotionCompleteDistributionJob()) + require.NoError(t, err) + + client = &splitPromotionClientStub{ + responses: []*pb.PromoteStagedVersionsResponse{ + {NextCursor: []byte("same")}, + {NextCursor: []byte("same")}, + }, + } + err = promoteSplitJobTarget(context.Background(), client, promotionCompleteDistributionJob()) + require.Error(t, err) + require.ErrorContains(t, err, "split promotion made no cursor progress") +} + +func TestDistributionServerRetrySplitJob_RequiresCatalogLeader(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + job := sampleDistributionSplitJob(14) + job.Phase = distribution.SplitJobPhaseFailed + job.RetryPhase = distribution.SplitJobPhaseBackfill + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, false) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + _, err := s.RetrySplitJob(ctx, &pb.RetrySplitJobRequest{JobId: job.JobID}) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Zero(t, coordinator.dispatchCalls) +} + +func sampleDistributionSplitJob(jobID uint64) distribution.SplitJob { + return distribution.SplitJob{ + JobID: jobID, + SourceRouteID: 1, + SplitKey: []byte("g"), + TargetGroupID: 2, + Phase: distribution.SplitJobPhaseBackfill, + RetryPhase: distribution.SplitJobPhaseNone, + StartedAtMs: 1000, + UpdatedAtMs: 1000, + } +} + +func promotionCompleteDistributionJob() distribution.SplitJob { + return distribution.SplitJob{ + JobID: 99, + SourceRouteID: 42, + SplitKey: []byte("m"), + TargetGroupID: 2, + Phase: distribution.SplitJobPhaseCleanup, + FenceTS: 777, + CutoverVersion: 1, + // Set because InitializeSplitJobPlan records it for every job it + // creates, so a job that reached CLEANUP always carries it. Leaving it + // zero made this fixture a legacy job, which runSplitJobPhase repairs + // with an extra catalog write -- the dispatch count below is about + // promotion, not about that repair. The repair itself is covered by + // TestSplitJobSourceGroupBackfillRecordsTheGroupWhileTheShapeStillAnswers. + SourceGroupID: 1, + } +} + +func promotionCompleteDistributionRoutes() []distribution.RouteDescriptor { + return []distribution.RouteDescriptor{ + { + RouteID: 2, + Start: []byte(""), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 42, + }, + { + RouteID: 3, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: distribution.RouteStateActive, + ParentRouteID: 42, + StagedVisibilityActive: true, + MigrationJobID: 99, + MinWriteTSExclusive: 777, + }, + } +} + +func distributionRouteByID(t *testing.T, routes []distribution.RouteDescriptor, routeID uint64) distribution.RouteDescriptor { + t.Helper() + for _, route := range routes { + if route.RouteID == routeID { + return route + } + } + t.Fatalf("route %d not found in %+v", routeID, routes) + return distribution.RouteDescriptor{} +} + +func splitJobIDs(jobs []*pb.SplitJob) []uint64 { + ids := make([]uint64, 0, len(jobs)) + for _, job := range jobs { + ids = append(ids, job.GetJobId()) + } + return ids +} + +func byteSliceStrings(in [][]byte) []string { + out := make([]string, 0, len(in)) + for _, item := range in { + out = append(out, string(item)) + } + return out +} + +type splitPromotionClientStub struct { + responses []*pb.PromoteStagedVersionsResponse + err error + requests []*pb.PromoteStagedVersionsRequest + calls int +} + +type splitMigrationClientStub struct { + splitPromotionClientStub + exports []*pb.ExportRangeVersionsRequest + imports []*pb.ImportRangeVersionsRequest + controls []*pb.TargetStagedReadinessRequest + cleanups []*pb.CleanupMigrationRequest + probes []*pb.ProbeMigrationStateRequest + probeFn func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) + nextTS uint64 + events []string +} + +func (s *splitMigrationClientStub) ExportRangeVersions( + _ context.Context, + req *pb.ExportRangeVersionsRequest, + _ ...grpc.CallOption, +) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) { + cloned := &pb.ExportRangeVersionsRequest{ + RangeStart: distribution.CloneBytes(req.GetRangeStart()), + RangeEnd: distribution.CloneBytes(req.GetRangeEnd()), + MaxCommitTs: req.GetMaxCommitTs(), + MinCommitTs: req.GetMinCommitTs(), + Cursor: distribution.CloneBytes(req.GetCursor()), + ChunkBytes: req.GetChunkBytes(), + RouteStart: distribution.CloneBytes(req.GetRouteStart()), + RouteEnd: distribution.CloneBytes(req.GetRouteEnd()), + MaxScannedBytes: req.GetMaxScannedBytes(), + KeyFamily: req.GetKeyFamily(), + ExcludeKnownInternal: req.GetExcludeKnownInternal(), + ExcludePrefixes: req.GetExcludePrefixes(), + } + s.exports = append(s.exports, cloned) + phaseByte := byte(0) + if req.GetMinCommitTs() != 0 { + phaseByte = 1 + } + family := req.GetKeyFamily() + if family > math.MaxUint8 { + return nil, errors.New("test key family overflows cleanup cursor") + } + cursor := []byte{byte(family), phaseByte} + return &splitMigrationStreamStub{responses: []*pb.ExportRangeVersionsResponse{{ + Versions: []*pb.MVCCVersion{{ + Key: []byte("n"), + CommitTs: 10, + Value: []byte("v"), + KeyFamily: req.GetKeyFamily(), + }}, + NextCursor: cursor, + Done: true, + }}}, nil +} + +func (s *splitMigrationClientStub) ImportRangeVersions( + _ context.Context, + req *pb.ImportRangeVersionsRequest, + _ ...grpc.CallOption, +) (*pb.ImportRangeVersionsResponse, error) { + s.imports = append(s.imports, req) + return &pb.ImportRangeVersionsResponse{AckedCursor: distribution.CloneBytes(req.GetCursor())}, nil +} + +func (s *splitMigrationClientStub) ApplyTargetStagedReadiness( + _ context.Context, + req *pb.TargetStagedReadinessRequest, + _ ...grpc.CallOption, +) (*pb.TargetStagedReadinessResponse, error) { + s.controls = append(s.controls, req) + s.events = append(s.events, "control") + minAdmittedTS := uint64(0) + if req.GetTrackWrites() { + minAdmittedTS = 10 + } + return &pb.TargetStagedReadinessResponse{MinAdmittedTs: minAdmittedTS}, nil +} + +func (s *splitMigrationClientStub) ProbeMigrationLocks( + context.Context, + *pb.ProbeMigrationLocksRequest, + ...grpc.CallOption, +) (*pb.ProbeMigrationLocksResponse, error) { + return &pb.ProbeMigrationLocksResponse{}, nil +} + +func (s *splitMigrationClientStub) CleanupMigration( + _ context.Context, + req *pb.CleanupMigrationRequest, + _ ...grpc.CallOption, +) (*pb.CleanupMigrationResponse, error) { + s.cleanups = append(s.cleanups, req) + return &pb.CleanupMigrationResponse{Done: true, NextCursor: distribution.CloneBytes(req.GetCursor())}, nil +} + +func (s *splitMigrationClientStub) ProbeMigrationState( + _ context.Context, + req *pb.ProbeMigrationStateRequest, + _ ...grpc.CallOption, +) (*pb.ProbeMigrationStateResponse, error) { + s.probes = append(s.probes, req) + if s.probeFn != nil { + return s.probeFn(req) + } + return &pb.ProbeMigrationStateResponse{Ready: true, CatalogVersion: req.GetExpectedCatalogVersion(), AppliedConfigurationIndex: 1}, nil +} + +func (s *splitMigrationClientStub) IssueMigrationTimestamp( + context.Context, + *pb.IssueMigrationTimestampRequest, + ...grpc.CallOption, +) (*pb.IssueMigrationTimestampResponse, error) { + s.events = append(s.events, "timestamp") + if s.nextTS == 0 { + s.nextTS = 100 + } + s.nextTS++ + return &pb.IssueMigrationTimestampResponse{Timestamp: s.nextTS, LastCommitTs: s.nextTS - 1}, nil +} + +type splitMigrationStreamStub struct { + grpc.ClientStream + responses []*pb.ExportRangeVersionsResponse +} + +func (s *splitMigrationStreamStub) Recv() (*pb.ExportRangeVersionsResponse, error) { + if len(s.responses) == 0 { + return nil, io.EOF + } + resp := s.responses[0] + s.responses = s.responses[1:] + return resp, nil +} + +func (s *splitPromotionClientStub) PromoteStagedVersions( + _ context.Context, + req *pb.PromoteStagedVersionsRequest, + _ ...grpc.CallOption, +) (*pb.PromoteStagedVersionsResponse, error) { + s.calls++ + s.requests = append(s.requests, &pb.PromoteStagedVersionsRequest{ + JobId: req.GetJobId(), + Cursor: distribution.CloneBytes(req.GetCursor()), + MaxVersions: req.GetMaxVersions(), + MaxBytes: req.GetMaxBytes(), + MaxScannedBytes: req.GetMaxScannedBytes(), + }) + if s.err != nil { + return nil, s.err + } + if len(s.responses) == 0 { + return &pb.PromoteStagedVersionsResponse{Done: true}, nil + } + resp := s.responses[0] + s.responses = s.responses[1:] + return resp, nil +} + +func (s *distributionCoordinatorStub) Next(ctx context.Context) (uint64, error) { + return s.NextAfter(ctx, 0) +} + +func (s *distributionCoordinatorStub) NextAfter(_ context.Context, min uint64) (uint64, error) { + s.timestampCalls++ + if s.timestampErr != nil { + return 0, s.timestampErr + } + next := s.timestampNext + if next == 0 { + next = s.store.LastCommitTS() + 1 + } + if next <= min { + next = min + 1 + } + s.timestampNext = next + 1 + return next, nil +} diff --git a/adapter/grpc.go b/adapter/grpc.go index 40d965ee7..4d4399d89 100644 --- a/adapter/grpc.go +++ b/adapter/grpc.go @@ -611,6 +611,9 @@ func storeKeysFromKVPairs(kvs []*store.KVPair) [][]byte { } func (r *GRPCServer) RawPut(ctx context.Context, req *pb.RawPutRequest) (*pb.RawPutResponse, error) { + if err := validatePublicMutationKey(req.GetKey()); err != nil { + return nil, err + } m, err := r.grpcTranscoder.RawPutToRequest(req) if err != nil { return nil, errors.WithStack(err) @@ -631,6 +634,9 @@ func (r *GRPCServer) RawPut(ctx context.Context, req *pb.RawPutRequest) (*pb.Raw } func (r *GRPCServer) RawDelete(ctx context.Context, req *pb.RawDeleteRequest) (*pb.RawDeleteResponse, error) { + if err := validatePublicMutationKey(req.GetKey()); err != nil { + return nil, err + } m, err := r.grpcTranscoder.RawDeleteToRequest(req) if err != nil { return nil, errors.WithStack(err) @@ -663,6 +669,9 @@ func (r *GRPCServer) Rollback(ctx context.Context, req *pb.RollbackRequest) (*pb } func (r *GRPCServer) Put(ctx context.Context, req *pb.PutRequest) (*pb.PutResponse, error) { + if err := validatePublicMutationKey(req.GetKey()); err != nil { + return nil, err + } reqs, err := r.grpcTranscoder.TransactionalPutToRequests(req) if err != nil { return nil, errors.WithStack(err) @@ -713,6 +722,9 @@ func (r *GRPCServer) Get(ctx context.Context, req *pb.GetRequest) (*pb.GetRespon } func (r *GRPCServer) Delete(ctx context.Context, req *pb.DeleteRequest) (*pb.DeleteResponse, error) { + if err := validatePublicMutationKey(req.GetKey()); err != nil { + return nil, err + } reqs, err := r.grpcTranscoder.TransactionalDeleteToRequests(req) if err != nil { return nil, errors.WithStack(err) diff --git a/adapter/grpc_mutation_auth.go b/adapter/grpc_mutation_auth.go new file mode 100644 index 000000000..4edeaa5f7 --- /dev/null +++ b/adapter/grpc_mutation_auth.go @@ -0,0 +1,16 @@ +package adapter + +import ( + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func validatePublicMutationKey(key []byte) error { + if distribution.IsReservedControlKey(key) || kv.IsTxnInternalKey(key) { + return errors.WithStack(status.Error(codes.PermissionDenied, "public mutations cannot write internal control keys")) + } + return nil +} diff --git a/adapter/grpc_mutation_auth_test.go b/adapter/grpc_mutation_auth_test.go new file mode 100644 index 000000000..26919b7a9 --- /dev/null +++ b/adapter/grpc_mutation_auth_test.go @@ -0,0 +1,37 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestGRPCPublicMutationsRejectControlKeys(t *testing.T) { + t.Parallel() + // No coordinator is configured: rejected writes must never reach dispatch. + server := NewGRPCServer(nil, nil) + ctx := context.Background() + for _, key := range [][]byte{ + distribution.CatalogSplitJobKey(7), distribution.CatalogRouteKey(1), distribution.CatalogVersionKey(), + []byte("!txn|int|!dist|job|forged"), []byte("!txn|meta|primary"), []byte("!dist|migstage|forged"), + } { + t.Run(string(key), func(t *testing.T) { + _, err := server.RawPut(ctx, &pb.RawPutRequest{Key: key}) + require.Equal(t, codes.PermissionDenied, status.Code(err)) + _, err = server.RawDelete(ctx, &pb.RawDeleteRequest{Key: key}) + require.Equal(t, codes.PermissionDenied, status.Code(err)) + _, err = server.Put(ctx, &pb.PutRequest{Key: key}) + require.Equal(t, codes.PermissionDenied, status.Code(err)) + _, err = server.Delete(ctx, &pb.DeleteRequest{Key: key}) + require.Equal(t, codes.PermissionDenied, status.Code(err)) + }) + } + for _, key := range []string{"user", "!txn|foo", "!txn|backup|customer", "!ddb|customer", "!s3|customer"} { + require.NoError(t, validatePublicMutationKey([]byte(key))) + } +} diff --git a/adapter/internal.go b/adapter/internal.go index f0b95eb3f..fdbaeb4da 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -6,6 +6,7 @@ import ( "os" "strings" "sync/atomic" + "time" "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal" @@ -20,6 +21,18 @@ import ( "google.golang.org/grpc/status" ) +const ( + // MigrationImportOpcodeEnv enables the internal staged import opcode after + // every voter is running a compatible build. + MigrationImportOpcodeEnv = "ELASTICKV_ENABLE_MIGRATION_IMPORT_OPCODE" + // MigrationPromoteOpcodeEnv enables the internal staged promotion opcode + // after every voter is running a compatible build. + MigrationPromoteOpcodeEnv = "ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE" + // MigrationCleanupOpcodeEnv enables the internal migration cleanup opcode + // after every voter is running a compatible build. + MigrationCleanupOpcodeEnv = "ELASTICKV_ENABLE_MIGRATION_CLEANUP_OPCODE" +) + type InternalOption func(*Internal) // WithKVForwardRejected refuses Forward before it can propose. @@ -84,6 +97,24 @@ func WithInternalMigrationPromoteGate(gate func(context.Context) error) Internal } } +func WithInternalMigrationCleanupGate(gate func(context.Context) error) InternalOption { + return func(i *Internal) { + i.migrationCleanupGate = gate + } +} + +func WithInternalRouteEngine(engine *distribution.Engine) InternalOption { + return func(i *Internal) { + i.routeEngine = engine + } +} + +func WithInternalActiveTimestampTracker(tracker *kv.ActiveTimestampTracker) InternalOption { + return func(i *Internal) { + i.readTracker = tracker + } +} + func WithInternalMigrationExportRouting(groupID uint64, resolver kv.PartitionResolver) InternalOption { return func(i *Internal) { i.migrationExportGroupID = groupID @@ -122,6 +153,7 @@ func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, c relay: relay, migrationImportGate: defaultMigrationImportGate, migrationPromoteGate: defaultMigrationPromoteGate, + migrationCleanupGate: defaultMigrationCleanupGate, } for _, opt := range opts { opt(i) @@ -130,24 +162,31 @@ func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, c } type Internal struct { - leader raftengine.LeaderView - transactionManager kv.Transactional - clock *kv.HLC - tsAllocator kv.TimestampAllocator - adminProposer raftengine.Proposer - lastCommitTimestamp func() uint64 - relay *RedisPubSubRelay - store store.MVCCStore - primaryCommitEvidence kv.PrimaryCommitEvidenceReader - tsoCutoverState interface{ CutoverActive() bool } - prepareEvidenceToken string - prepareEvidenceScanning atomic.Bool - migrationProposer raftengine.Proposer - migrationImportGate func(context.Context) error - migrationPromoteGate func(context.Context) error - migrationExportGroupID uint64 - migrationExportResolver kv.PartitionResolver - writeGate kv.MutationWriteGate + leader raftengine.LeaderView + transactionManager kv.Transactional + clock *kv.HLC + tsAllocator kv.TimestampAllocator + adminProposer raftengine.Proposer + lastCommitTimestamp func() uint64 + relay *RedisPubSubRelay + store store.MVCCStore + primaryCommitEvidence kv.PrimaryCommitEvidenceReader + tsoCutoverState interface{ CutoverActive() bool } + prepareEvidenceToken string + prepareEvidenceScanning atomic.Bool + migrationProposer raftengine.Proposer + migrationImportGate func(context.Context) error + migrationPromoteGate func(context.Context) error + migrationCleanupGate func(context.Context) error + migrationOwnershipReady func() bool + migrationPartitionProtocolReady func() bool + migrationNamespaceReady func() bool + transferSizeReady func() bool + routeEngine *distribution.Engine + readTracker *kv.ActiveTimestampTracker + migrationExportGroupID uint64 + migrationExportResolver kv.PartitionResolver + writeGate kv.MutationWriteGate // kvForwardRejected marks a runtime whose state machine cannot apply KV // entries -- the dedicated TSO group. See WithKVForwardRejected. kvForwardRejected bool @@ -184,7 +223,7 @@ const ( // entries come to tens of kilobytes, so a mebibyte covers them with room // to spare. migrationExportPageFramingHeadroom = 1 << 20 - migrationExportPageByteBudget = internal.GRPCMaxMessageBytes - migrationExportPageFramingHeadroom + migrationExportPageByteBudget = store.MigrationPagePayloadBudget ) func (i *Internal) Forward(ctx context.Context, req *pb.ForwardRequest) (*pb.ForwardResponse, error) { @@ -229,6 +268,305 @@ func (i *Internal) Forward(ctx context.Context, req *pb.ForwardRequest) (*pb.For }, nil } +func (i *Internal) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + if req == nil || req.GetJobId() == 0 { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "migration probe job_id is required")) + } + resp, err := i.probeMigrationStateKind(ctx, req) + if err != nil { + return nil, err + } + resp.NamespaceAdmissionVersion = i.migrationNamespaceProtocolVersion() + i.decorateMigrationTransferCapability(req, resp) + i.bindMigrationProbeConfiguration(req, resp) + return resp, nil +} + +func (i *Internal) decorateMigrationTransferCapability(req *pb.ProbeMigrationStateRequest, resp *pb.ProbeMigrationStateResponse) { + if i.MigrationTransferSizeGuardSupported() { + resp.TransferSizeGuardVersion = store.MigrationTransferSizeGuardVersion + } + if req.GetTransferSizeGuardVersion() != 0 && + (req.GetTransferSizeGuardVersion() != resp.GetTransferSizeGuardVersion() || !i.MigrationTransferSizeGuardReady()) { + resp.Ready = false + } +} + +func (i *Internal) bindMigrationProbeConfiguration(req *pb.ProbeMigrationStateRequest, resp *pb.ProbeMigrationStateResponse) { + if !req.GetRequireConfigurationIndex() { + return + } + reader, supported := i.leader.(raftengine.AppliedConfigurationReader) + if !supported { + resp.Ready = false + return + } + index, supported := reader.AppliedConfigurationIndex() + resp.AppliedConfigurationIndex = index + resp.Ready = resp.GetReady() && supported && index != 0 && + (req.GetExpectedConfigurationIndex() == 0 || req.GetExpectedConfigurationIndex() == index) +} + +func (i *Internal) probeMigrationStateKind(ctx context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + if req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_NAMESPACE_ADMISSION { + return i.probeMigrationNamespace(ctx, req) + } + if req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_PARTITION_APPLIED { + return i.probeMigrationPartition(ctx, req) + } + if migrationOwnershipProbeKind(req.GetKind()) { + return i.probeMigrationOwnership(ctx, req) + } + return i.probeLegacyMigrationStateKind(ctx, req) +} + +func migrationOwnershipProbeKind(kind pb.MigrationStateProbeKind) bool { + return kind == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY || + kind == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED +} + +func (i *Internal) probeLegacyMigrationStateKind(ctx context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + switch req.GetKind() { + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY, + pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED, + pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_NAMESPACE_ADMISSION, + pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_PARTITION_APPLIED: + return i.probeMigrationStateKind(ctx, req) + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED: + return i.probeMigrationControl(ctx, req) + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_GUARD_PRESENT: + return i.probeMigrationSourceGuard(ctx, req) + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED: + return i.probeMigrationRoute(req, true), nil + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED: + return i.probeMigrationRoute(req, false), nil + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED: + return i.probeMigrationReadDrain(req), nil + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED: + return i.probeMigrationMetadataCleared(ctx, req.GetJobId()) + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_UNSPECIFIED: + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "migration probe kind is required")) + default: + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "unknown migration probe kind")) + } +} + +func (i *Internal) probeMigrationReadDrain(req *pb.ProbeMigrationStateRequest) *pb.ProbeMigrationStateResponse { + return &pb.ProbeMigrationStateResponse{ + Ready: time.Now().UnixMilli() >= req.GetReadDrainNotBeforeMs() && i.sourceReadsDrained(req.GetReadDrainMinTs()), + CatalogVersion: i.migrationCatalogVersion(), + } +} + +func (i *Internal) probeMigrationSourceGuard(ctx context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + if len(req.GetRouteStart()) == 0 { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "source guard probe requires route_start")) + } + if err := i.verifyInternalLeaderApplied(ctx); err != nil { + return nil, err + } + reader, ok := i.store.(store.MigrationTargetReadinessReader) + if !ok { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration readiness reader is not configured")) + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + for _, state := range states { + if migrationSourceGuardMatches(state, req) { + return &pb.ProbeMigrationStateResponse{Ready: true, CatalogVersion: i.migrationCatalogVersion()}, nil + } + } + return &pb.ProbeMigrationStateResponse{}, nil +} + +func migrationSourceGuardMatches(state store.TargetStagedReadinessState, req *pb.ProbeMigrationStateRequest) bool { + return state.JobID == req.GetJobId() && state.MigrationJobID == req.GetJobId() && state.Armed && + (state.TrackWrites || state.SourceWriteFence || state.SourceReadFence) && + bytes.Equal(state.RouteStart, req.GetRouteStart()) && bytes.Equal(state.RouteEnd, normalizeOptionalEnd(req.GetRouteEnd())) +} + +// sourceReadsDrained reports whether every read that could still observe the +// migrating range on this node has finished. The read tracker is process-wide, +// so unrelated frontend reads would otherwise keep it non-empty forever on a +// busy node. Reads issued after the source read fence armed carry a timestamp +// above drainMinTS and are already rejected for the moving range, so only +// older pins block the drain. A zero drainMinTS means the control plane did +// not supply the fence point (pre-upgrade coordinator), so fall back to +// requiring a fully empty tracker. +func (i *Internal) sourceReadsDrained(drainMinTS uint64) bool { + if i.readTracker == nil { + return true + } + oldest := i.readTracker.Oldest() + if oldest == 0 { + return true + } + if drainMinTS == 0 { + return false + } + return oldest > drainMinTS +} + +func (i *Internal) IssueMigrationTimestamp(ctx context.Context, _ *pb.IssueMigrationTimestampRequest) (*pb.IssueMigrationTimestampResponse, error) { + if err := i.verifyInternalLeaderApplied(ctx); err != nil { + return nil, err + } + if i.store == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration timestamp store is not configured")) + } + lastCommitTS := i.store.LastCommitTS() + ts, err := i.nextTimestampAfter(ctx, lastCommitTS, "issue migration timestamp") + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.IssueMigrationTimestampResponse{Timestamp: ts, LastCommitTs: lastCommitTS}, nil +} + +func (i *Internal) probeMigrationControl(ctx context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + reader, ok := i.store.(store.MigrationTargetReadinessReader) + if !ok { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration readiness reader is not configured")) + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + for _, state := range states { + if state.JobID != req.GetJobId() { + continue + } + ready := migrationControlMatches(state, req) + return &pb.ProbeMigrationStateResponse{Ready: ready, CatalogVersion: i.migrationCatalogVersion(), MinAdmittedTs: state.MinAdmittedTS}, nil + } + return &pb.ProbeMigrationStateResponse{CatalogVersion: i.migrationCatalogVersion()}, nil +} + +func migrationControlMatches(state store.TargetStagedReadinessState, req *pb.ProbeMigrationStateRequest) bool { + return state.Armed && + bytes.Equal(state.RouteStart, req.GetRouteStart()) && + bytes.Equal(state.RouteEnd, normalizeOptionalEnd(req.GetRouteEnd())) && + state.ExpectedCutoverVersion == req.GetExpectedCatalogVersion() && + state.MigrationJobID == req.GetMigrationJobId() && + state.MinWriteTSExclusive == req.GetMinWriteTsExclusive() && + migrationControlModesMatch(state, req) && + (req.GetTransferSizeGuardVersion() == 0 || state.TransferSizeGuardVersion == req.GetTransferSizeGuardVersion()) && + state.RetentionPinTS == req.GetRetentionPinTs() +} + +func migrationControlModesMatch(state store.TargetStagedReadinessState, req *pb.ProbeMigrationStateRequest) bool { + return state.SourceWriteFence == req.GetSourceWriteFence() && state.SourceReadFence == req.GetSourceReadFence() && + state.TrackWrites == req.GetTrackWrites() +} + +func (i *Internal) probeMigrationRoute(req *pb.ProbeMigrationStateRequest, requireCleared bool) *pb.ProbeMigrationStateResponse { + version := i.migrationCatalogVersion() + if i.routeEngine == nil || version < req.GetExpectedCatalogVersion() { + return &pb.ProbeMigrationStateResponse{CatalogVersion: version} + } + route, found := i.routeEngine.GetRoute(req.GetRouteStart()) + ready := found && route.GroupID == req.GetExpectedGroupId() && + bytes.Equal(route.Start, req.GetRouteStart()) && + bytes.Equal(route.End, normalizeOptionalEnd(req.GetRouteEnd())) + if requireCleared { + ready = ready && !route.StagedVisibilityActive && route.MigrationJobID == 0 && + route.MinWriteTSExclusive == req.GetMinWriteTsExclusive() + } + return &pb.ProbeMigrationStateResponse{Ready: ready, CatalogVersion: version} +} + +func (i *Internal) probeMigrationMetadataCleared(ctx context.Context, jobID uint64) (*pb.ProbeMigrationStateResponse, error) { + reader, supported := i.store.(store.MigrationRetirementReader) + if !supported { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration retirement proof is unavailable")) + } + retired, err := reader.MigrationRetired(ctx, jobID) + if err != nil { + return nil, errors.WithStack(err) + } + if !retired { + return &pb.ProbeMigrationStateResponse{CatalogVersion: i.migrationCatalogVersion()}, nil + } + present, err := i.migrationReadinessMetadataPresent(ctx, jobID) + if err != nil { + return nil, err + } + if present { + return &pb.ProbeMigrationStateResponse{CatalogVersion: i.migrationCatalogVersion()}, nil + } + present, err = i.migrationPromotionMetadataPresent(ctx, jobID) + if err != nil { + return nil, err + } + if present { + return &pb.ProbeMigrationStateResponse{CatalogVersion: i.migrationCatalogVersion()}, nil + } + present, err = i.migrationImportMetadataPresent(ctx, jobID) + if err != nil { + return nil, err + } + if present { + return &pb.ProbeMigrationStateResponse{CatalogVersion: i.migrationCatalogVersion()}, nil + } + return &pb.ProbeMigrationStateResponse{Ready: true, CatalogVersion: i.migrationCatalogVersion()}, nil +} + +func (i *Internal) migrationReadinessMetadataPresent(ctx context.Context, jobID uint64) (bool, error) { + readiness, ok := i.store.(store.MigrationTargetReadinessReader) + if !ok { + return false, errors.WithStack(status.Error(codes.FailedPrecondition, "migration readiness reader is not configured")) + } + states, err := readiness.MigrationTargetReadinessStates(ctx) + if err != nil { + return false, errors.WithStack(err) + } + for _, state := range states { + if state.JobID == jobID { + return true, nil + } + } + return false, nil +} + +func (i *Internal) migrationPromotionMetadataPresent(ctx context.Context, jobID uint64) (bool, error) { + promotion, ok := i.store.(store.MigrationPromotionStateReader) + if !ok { + return false, errors.WithStack(status.Error(codes.FailedPrecondition, "migration promotion reader is not configured")) + } + _, found, err := promotion.MigrationPromotionState(ctx, jobID) + if err != nil { + return false, errors.WithStack(err) + } + return found, nil +} + +func (i *Internal) migrationImportMetadataPresent(ctx context.Context, jobID uint64) (bool, error) { + importMetadata, ok := i.store.(store.MigrationImportMetadataReader) + if !ok { + return false, errors.WithStack(status.Error(codes.FailedPrecondition, "migration import metadata reader is not configured")) + } + present, err := importMetadata.MigrationImportMetadataPresent(ctx, jobID) + if err != nil { + return false, errors.WithStack(err) + } + return present, nil +} + +func (i *Internal) migrationCatalogVersion() uint64 { + if i.routeEngine == nil { + return 0 + } + return i.routeEngine.Version() +} + +func normalizeOptionalEnd(end []byte) []byte { + if len(end) == 0 { + return nil + } + return end +} + func (i *Internal) ForwardAdminProposal( ctx context.Context, req *pb.ForwardAdminProposalRequest, @@ -242,6 +580,9 @@ func (i *Internal) ForwardAdminProposal( if i.adminProposer == nil { return nil, errors.New("admin proposer is unavailable") } + if err := kv.ValidateForwardedAdminPayload(req.GetPayload()); err != nil { + return nil, errors.WithStack(status.Error(codes.PermissionDenied, "admin forwarding requires a supported typed admin entry")) + } result, err := i.adminProposer.ProposeAdmin(ctx, req.GetPayload()) if err != nil { return nil, errors.WithStack(err) @@ -368,6 +709,11 @@ func (i *Internal) streamExportRangeVersions(req *pb.ExportRangeVersionsRequest, // Carrying values that large needs a chunked migration wire format, which is a // protocol change rather than a fix here. func checkMigrationExportPageSize(resp *pb.ExportRangeVersionsResponse) error { + for _, version := range resp.GetVersions() { + if !store.MigrationVersionFitsTransfer(version.GetKey(), version.GetValue()) { + return migrationOversizedVersionError(version) + } + } if migrationExportPagePayloadBytes(resp) <= migrationExportPageByteBudget { return nil } @@ -406,13 +752,10 @@ func (i *Internal) ImportRangeVersions(ctx context.Context, req *pb.ImportRangeV if err := validateImportRangeVersionsRequest(req); err != nil { return nil, err } - if i.migrationProposer == nil { - return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration import proposer is not configured")) - } - if err := i.verifyInternalLeader(ctx); err != nil { + if err := i.verifyMigrationImportProposal(ctx); err != nil { return nil, err } - if err := i.verifyMigrationImportEnabled(ctx); err != nil { + if err := i.requireMigrationNamespaceAdmission(ctx); err != nil { return nil, err } result, err := i.proposeMigrationImport(ctx, req) @@ -422,7 +765,7 @@ func (i *Internal) ImportRangeVersions(ctx context.Context, req *pb.ImportRangeV if i.clock != nil && result.MaxImportedTS > 0 { i.clock.Observe(result.MaxImportedTS) } - return &pb.ImportRangeVersionsResponse{AckedCursor: result.AckedCursor}, nil + return &pb.ImportRangeVersionsResponse{AckedCursor: result.AckedCursor, Duplicate: result.Duplicate}, nil } func validateImportRangeVersionsRequest(req *pb.ImportRangeVersionsRequest) error { @@ -452,18 +795,15 @@ func (i *Internal) PromoteStagedVersions(ctx context.Context, req *pb.PromoteSta if req.GetJobId() == 0 { return nil, errors.WithStack(status.Error(codes.InvalidArgument, "promote staged versions job_id is required")) } - if i.migrationProposer == nil { - return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration promote proposer is not configured")) - } - if err := i.verifyInternalLeader(ctx); err != nil { - return nil, err - } - if err := i.verifyMigrationPromoteEnabled(ctx); err != nil { + if err := i.verifyMigrationPromoteProposal(ctx); err != nil { return nil, err } if err := validatePromoteStagedVersionsRequest(req); err != nil { return nil, errors.WithStack(err) } + if err := i.requireMigrationNamespaceAdmission(ctx); err != nil { + return nil, err + } result, err := i.proposeMigrationPromote(ctx, req) if err != nil { return nil, errors.WithStack(err) @@ -479,6 +819,153 @@ func (i *Internal) PromoteStagedVersions(ctx context.Context, req *pb.PromoteSta }, nil } +func (i *Internal) ApplyTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest) (*pb.TargetStagedReadinessResponse, error) { + if err := validateTargetStagedReadinessRequest(req); err != nil { + return nil, err + } + if err := i.validateTransferSizeGuardActivation(ctx, req); err != nil { + return nil, err + } + if i.migrationProposer == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "target staged readiness proposer is not configured")) + } + if err := i.verifyInternalLeader(ctx); err != nil { + return nil, err + } + if err := i.requireMigrationNamespaceAdmission(ctx); err != nil { + return nil, err + } + if err := i.proposeTargetStagedReadiness(ctx, req); err != nil { + return nil, errors.WithStack(err) + } + if req.GetArmed() && i.clock != nil { + floor := req.GetMinWriteTsExclusive() + if i.store != nil { + floor = max(floor, i.store.LastCommitTS()) + } + i.clock.Observe(floor) + } + return &pb.TargetStagedReadinessResponse{MinAdmittedTs: i.migrationMinAdmittedTS(ctx, req.GetJobId())}, nil +} + +func (i *Internal) CleanupMigration(ctx context.Context, req *pb.CleanupMigrationRequest) (*pb.CleanupMigrationResponse, error) { + if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION { + return i.applyMigrationNamespaceAdmission(ctx, req) + } + if err := i.validateMigrationCleanupAdmission(ctx, req); err != nil { + return nil, err + } + if err := i.verifyInternalLeader(ctx); err != nil { + return nil, err + } + if err := i.verifyMigrationCleanupEnabled(ctx); err != nil { + return nil, err + } + result, err := i.proposeMigrationCleanup(ctx, req) + if err != nil { + if errors.Is(err, kv.ErrMigrationConfigurationChanged) || errors.Is(err, store.ErrMigrationOwnership) { + return nil, errors.WithStack(status.Error(codes.Aborted, err.Error())) + } + return nil, errors.WithStack(err) + } + return &pb.CleanupMigrationResponse{ + NextCursor: result.NextCursor, + Done: result.Done, + DeletedRows: result.DeletedRows, + DeletedBytes: result.DeletedBytes, + ScannedBytes: result.ScannedBytes, + }, nil +} + +func validateCleanupMigrationRequest(req *pb.CleanupMigrationRequest) error { + switch { + case req == nil: + return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup request is nil")) + case req.GetJobId() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup job_id is required")) + case req.GetMode() != pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS && + req.GetMode() != pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA && !ownershipCleanupMode(req.GetMode()): + return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup mode is invalid")) + case req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS && req.GetKeyFamily() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup key_family is required")) + } + if ownershipCleanupMode(req.GetMode()) { + return validateOwnershipCleanupRequest(req) + } + return nil +} + +func validateOwnershipCleanupRequest(req *pb.CleanupMigrationRequest) error { + if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION { + return validatePartitionCleanupRequest(req) + } + if !req.GetRequireConfigurationIndex() || req.GetExpectedConfigurationIndex() == 0 { + return errors.WithStack(status.Error(codes.InvalidArgument, "ownership control requires applied configuration")) + } + if _, err := kv.MigrationOwnershipStateFromProto(req.GetJobId(), req.GetOwnership()); err != nil { + return errors.WithStack(status.Error(codes.InvalidArgument, "invalid migration ownership proof")) + } + return nil +} + +func (i *Internal) migrationMinAdmittedTS(ctx context.Context, jobID uint64) uint64 { + reader, ok := i.store.(store.MigrationTargetReadinessReader) + if !ok { + return 0 + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return 0 + } + for _, state := range states { + if state.JobID == jobID { + return state.MinAdmittedTS + } + } + return 0 +} + +func validateTargetStagedReadinessRequest(req *pb.TargetStagedReadinessRequest) error { + switch { + case req == nil: + return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness request is nil")) + case req.GetJobId() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness job_id is required")) + case req.GetMigrationJobId() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness migration_job_id is required")) + case req.GetArmed() && req.GetMinWriteTsExclusive() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness min_write_ts_exclusive is required when armed")) + } + return validateSourceMigrationControlRequest(req) +} + +func validateSourceMigrationControlRequest(req *pb.TargetStagedReadinessRequest) error { + if req.GetSourceReadFence() && !req.GetSourceWriteFence() { + return errors.WithStack(status.Error(codes.InvalidArgument, "source read fence requires source write fence")) + } + if (req.GetSourceWriteFence() || req.GetSourceReadFence()) && req.GetRetentionPinTs() == 0 { + return errors.WithStack(status.Error(codes.InvalidArgument, "source migration control retention_pin_ts is required")) + } + return nil +} + +func (i *Internal) ProbeMigrationLocks(ctx context.Context, req *pb.ProbeMigrationLocksRequest) (*pb.ProbeMigrationLocksResponse, error) { + if req == nil || len(req.GetRouteStart()) == 0 { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "migration lock probe route_start is required")) + } + if i.store == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration lock probe store is not configured")) + } + if err := i.verifyInternalLeaderApplied(ctx); err != nil { + return nil, err + } + locks, err := kv.PendingTxnLocksInRoute(ctx, i.store, req.GetRouteStart(), req.GetRouteEnd(), ^uint64(0), int(req.GetLimit())) + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.ProbeMigrationLocksResponse{PendingCount: uint32(len(locks))}, nil //nolint:gosec // result is bounded by uint32 wire limit +} + func (i *Internal) verifyMigrationPromoteEnabled(ctx context.Context) error { if i.migrationPromoteGate == nil { return nil @@ -489,6 +976,16 @@ func (i *Internal) verifyMigrationPromoteEnabled(ctx context.Context) error { return nil } +func (i *Internal) verifyMigrationCleanupEnabled(ctx context.Context) error { + if i.migrationCleanupGate == nil { + return nil + } + if err := i.migrationCleanupGate(ctx); err != nil { + return errors.WithStack(err) + } + return nil +} + func validatePromoteStagedVersionsRequest(req *pb.PromoteStagedVersionsRequest) error { prefix := distribution.MigrationStagedDataKeyPrefix(req.GetJobId()) if err := store.ValidatePromotionCursorForRange(req.GetCursor(), prefix, store.PrefixScanEnd(prefix)); err != nil { @@ -508,7 +1005,7 @@ func defaultMigrationImportGate(context.Context) error { } func migrationImportOpcodeEnabledFromEnv() bool { - return envFlagEnabled("ELASTICKV_ENABLE_MIGRATION_IMPORT_OPCODE") + return MigrationImportOpcodeEnabledFromEnv() } func defaultMigrationPromoteGate(context.Context) error { @@ -518,8 +1015,33 @@ func defaultMigrationPromoteGate(context.Context) error { return errors.WithStack(status.Error(codes.FailedPrecondition, "migration promote opcode is disabled; enable after every voter is running a build that supports migration promotion")) } +func defaultMigrationCleanupGate(context.Context) error { + if MigrationCleanupOpcodeEnabledFromEnv() { + return nil + } + return errors.WithStack(status.Error(codes.FailedPrecondition, "migration cleanup opcode is disabled; enable after every voter is running a build that supports migration cleanup")) +} + func migrationPromoteOpcodeEnabledFromEnv() bool { - return envFlagEnabled("ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE") + return MigrationPromoteOpcodeEnabledFromEnv() +} + +// MigrationImportOpcodeEnabledFromEnv reports whether the staged import opcode +// is enabled for this process. +func MigrationImportOpcodeEnabledFromEnv() bool { + return envFlagEnabled(MigrationImportOpcodeEnv) +} + +// MigrationPromoteOpcodeEnabledFromEnv reports whether the staged promotion +// opcode is enabled for this process. +func MigrationPromoteOpcodeEnabledFromEnv() bool { + return envFlagEnabled(MigrationPromoteOpcodeEnv) +} + +// MigrationCleanupOpcodeEnabledFromEnv reports whether the migration cleanup +// opcode is enabled for this process. +func MigrationCleanupOpcodeEnabledFromEnv() bool { + return envFlagEnabled(MigrationCleanupOpcodeEnv) } func envFlagEnabled(name string) bool { @@ -603,6 +1125,51 @@ func (i *Internal) proposeMigrationPromote(ctx context.Context, req *pb.PromoteS } } +func (i *Internal) proposeMigrationCleanup(ctx context.Context, req *pb.CleanupMigrationRequest) (store.CleanupVersionsResult, error) { + cmd, err := kv.MarshalMigrationCleanupCommand(req) + if err != nil { + return store.CleanupVersionsResult{}, errors.WithStack(err) + } + resp, err := i.proposeMigrationCommand(ctx, cmd, "migration cleanup") + if err != nil { + return store.CleanupVersionsResult{}, errors.WithStack(err) + } + if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA && resp == nil { + return store.CleanupVersionsResult{Done: true}, nil + } + switch resp := resp.(type) { + case store.CleanupVersionsResult: + return resp, nil + case *store.CleanupVersionsResult: + if resp == nil { + return store.CleanupVersionsResult{}, errors.New("migration cleanup apply returned nil result") + } + return *resp, nil + case error: + return store.CleanupVersionsResult{}, errors.WithStack(resp) + default: + return store.CleanupVersionsResult{}, errors.WithStack(errors.Newf("unexpected migration cleanup apply response type %T", resp)) + } +} + +func (i *Internal) proposeTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest) error { + cmd, err := kv.MarshalTargetStagedReadinessCommand(req) + if err != nil { + return errors.WithStack(err) + } + resp, err := i.proposeMigrationCommand(ctx, cmd, "target staged readiness") + if err != nil { + return errors.WithStack(err) + } + if resp == nil { + return nil + } + if err, ok := resp.(error); ok { + return errors.WithStack(err) + } + return errors.WithStack(errors.Newf("unexpected target readiness apply response type %T", resp)) +} + func (i *Internal) proposeMigrationCommand(ctx context.Context, cmd []byte, label string) (any, error) { result, err := i.migrationProposer.Propose(ctx, cmd) if err != nil { @@ -768,9 +1335,11 @@ func decodedS3BucketRouteFilter(family uint32, routeStart, routeEnd []byte) func } } +// decodedS3BucketRouteSelected delegates to the one definition of bucket +// auxiliary ownership. Keeping a second copy here is how the export filter and +// the cleanup bracket drifted apart in the first place. func decodedS3BucketRouteSelected(bucket string, routeStart, routeEnd []byte) bool { - bucketRouteStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) - return keyInRouteRange(bucketRouteStart, routeStart, routeEnd) + return distribution.S3BucketAuxiliaryRouteSelected(bucket, routeStart, routeEnd) } func rangesIntersect(aStart, aEnd, bStart, bEnd []byte) bool { diff --git a/adapter/internal_admin_proposal_security_test.go b/adapter/internal_admin_proposal_security_test.go new file mode 100644 index 000000000..b343b6dec --- /dev/null +++ b/adapter/internal_admin_proposal_security_test.go @@ -0,0 +1,41 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestInternalForwardAdminProposalRejectsOtherRaftPayloads(t *testing.T) { + t.Parallel() + for opcode := 0; opcode <= 255; opcode++ { + payload := []byte{byte(opcode), 0} + proposer := &internalAdminProposer{} + server := NewInternalWithEngine(nil, + internalAdminLeaderView{state: raftengine.StateLeader}, nil, nil, + WithInternalAdminProposer(proposer)) + _, err := server.ForwardAdminProposal(context.Background(), &pb.ForwardAdminProposalRequest{Payload: payload}) + require.Equal(t, codes.PermissionDenied, status.Code(err), "opcode %x", opcode) + require.Nil(t, proposer.payload) + } +} + +func TestInternalForwardAdminProposalValidatesEntireEnvelope(t *testing.T) { + t.Parallel() + valid := kv.EncodeBackupReleaseEntry(kv.BackupReleaseEntry{}) + for _, payload := range [][]byte{nil, valid[:len(valid)-1], append(valid, 0x0d), {0x0e, 0xff}} { + proposer := &internalAdminProposer{} + server := NewInternalWithEngine(nil, + internalAdminLeaderView{state: raftengine.StateLeader}, nil, nil, + WithInternalAdminProposer(proposer)) + _, err := server.ForwardAdminProposal(context.Background(), &pb.ForwardAdminProposalRequest{Payload: payload}) + require.Equal(t, codes.PermissionDenied, status.Code(err)) + require.Nil(t, proposer.payload) + } +} diff --git a/adapter/internal_admin_proposal_test.go b/adapter/internal_admin_proposal_test.go index 1294d2d57..7f26caa96 100644 --- a/adapter/internal_admin_proposal_test.go +++ b/adapter/internal_admin_proposal_test.go @@ -76,11 +76,11 @@ func TestInternalForwardAdminProposalUsesLeaderProposer(t *testing.T) { resp, err := internal.ForwardAdminProposal( context.Background(), - &pb.ForwardAdminProposalRequest{Payload: []byte("pin")}, + &pb.ForwardAdminProposalRequest{Payload: kv.EncodeBackupReleaseEntry(kv.BackupReleaseEntry{})}, ) require.NoError(t, err) require.Equal(t, uint64(17), resp.GetCommitIndex()) - require.Equal(t, []byte("pin"), proposer.payload) + require.Equal(t, kv.EncodeBackupReleaseEntry(kv.BackupReleaseEntry{}), proposer.payload) } func TestInternalForwardAdminProposalFailsClosed(t *testing.T) { @@ -115,7 +115,9 @@ func TestInternalForwardAdminProposalFailsClosed(t *testing.T) { nil, WithInternalAdminProposer(tc.proposer), ) - _, err := internal.ForwardAdminProposal(context.Background(), &pb.ForwardAdminProposalRequest{}) + _, err := internal.ForwardAdminProposal(context.Background(), &pb.ForwardAdminProposalRequest{ + Payload: kv.EncodeBackupReleaseEntry(kv.BackupReleaseEntry{}), + }) if tc.errTarget != nil { require.ErrorIs(t, err, tc.errTarget) } diff --git a/adapter/internal_migration_namespace.go b/adapter/internal_migration_namespace.go new file mode 100644 index 000000000..80d0fa658 --- /dev/null +++ b/adapter/internal_migration_namespace.go @@ -0,0 +1,134 @@ +package adapter + +import ( + "context" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +var errMigrationNamespaceAdmissionNotConfigured = status.Error(codes.FailedPrecondition, "migration namespace admission is not configured") + +func (i *Internal) verifyMigrationImportProposal(ctx context.Context) error { + if i.migrationProposer == nil { + return errors.WithStack(status.Error(codes.FailedPrecondition, "migration import proposer is not configured")) + } + if i.migrationImportGate == nil { + return errors.WithStack(status.Error(codes.FailedPrecondition, "migration import opcode gate is not configured")) + } + if err := i.verifyInternalLeader(ctx); err != nil { + return err + } + return i.verifyMigrationImportEnabled(ctx) +} + +func (i *Internal) verifyMigrationPromoteProposal(ctx context.Context) error { + if i.migrationProposer == nil { + return errors.WithStack(status.Error(codes.FailedPrecondition, "migration promote proposer is not configured")) + } + if i.migrationPromoteGate == nil { + return errors.WithStack(status.Error(codes.FailedPrecondition, "migration promote opcode gate is not configured")) + } + if err := i.verifyInternalLeader(ctx); err != nil { + return err + } + return i.verifyMigrationPromoteEnabled(ctx) +} + +func (i *Internal) requireMigrationNamespaceAdmission(ctx context.Context) error { + if err := kv.RequireMigrationNamespaceAdmission(ctx, i.store); err != nil { + return errors.WithStack(status.Error(codes.FailedPrecondition, err.Error())) + } + if !i.MigrationNamespaceAdmissionSupported() { + return errors.WithStack(errMigrationNamespaceAdmissionNotConfigured) + } + return nil +} + +func WithMigrationNamespaceAdmissionReady(ready func() bool) InternalOption { + return func(i *Internal) { i.migrationNamespaceReady = ready } +} + +// Installed support is safe to query before a candidate has any ConfState or +// marker. The callback must describe actual public/FSM module installation. +func (i *Internal) MigrationNamespaceAdmissionSupported() bool { + return i != nil && i.store != nil && i.migrationProposer != nil && i.migrationNamespaceReady != nil && i.migrationNamespaceReady() +} + +func (i *Internal) probeMigrationNamespace(ctx context.Context, _ *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + response := &pb.ProbeMigrationStateResponse{} + if !i.MigrationNamespaceAdmissionSupported() { + return response, nil + } + response.NamespaceAdmissionVersion = kv.MigrationNamespaceVersion + reader, ok := i.leader.(raftengine.AppliedConfigurationReader) + if !ok { + return response, nil + } + index, supported := reader.AppliedConfigurationIndex() + response.AppliedConfigurationIndex = index + if !supported || index == 0 { + return response, nil + } + version, err := kv.MigrationNamespaceAdmissionVersion(ctx, i.store) + if err != nil { + return nil, errors.WithStack(err) + } + configuration, err := kv.MigrationConfigurationIndex(ctx, i.store) + if err != nil { + return nil, errors.WithStack(err) + } + response.Ready = version == kv.MigrationNamespaceVersion && configuration == index + return response, nil +} + +// MODE8 is the sole exception to the marker-required migration mutation gate. +// Do not route it through cleanup/partition/Draft operational readiness gates: +// those controls require the very marker this control is installing. +func (i *Internal) applyMigrationNamespaceAdmission(ctx context.Context, request *pb.CleanupMigrationRequest) (*pb.CleanupMigrationResponse, error) { + if !i.MigrationNamespaceAdmissionSupported() { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "namespace admission modules are not installed")) + } + if err := kv.ValidateNamespaceAdmissionRequest(request); err != nil { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, err.Error())) + } + if err := i.verifyInternalLeaderApplied(ctx); err != nil { + return nil, err + } + configuration, err := kv.MigrationConfigurationIndex(ctx, i.store) + if err != nil { + return nil, errors.WithStack(err) + } + if configuration != request.GetExpectedConfigurationIndex() { + return nil, errors.WithStack(status.Error(codes.Aborted, "namespace configuration changed")) + } + if err := i.proposeNamespaceAdmission(ctx, request); err != nil { + return nil, err + } + if err := kv.RequireMigrationNamespaceAdmission(ctx, i.store); err != nil { + return nil, errors.WithStack(err) + } + return &pb.CleanupMigrationResponse{Done: true}, nil +} + +func (i *Internal) proposeNamespaceAdmission(ctx context.Context, request *pb.CleanupMigrationRequest) error { + encoded, err := kv.MarshalMigrationCleanupCommand(request) + if err != nil { + return errors.WithStack(err) + } + response, err := i.proposeMigrationCommand(ctx, encoded, "namespace admission") + if err != nil { + return err + } + if verdict, ok := response.(error); ok { + return errors.WithStack(verdict) + } + if response != nil { + return errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + return nil +} diff --git a/adapter/internal_migration_namespace_mutation_test.go b/adapter/internal_migration_namespace_mutation_test.go new file mode 100644 index 000000000..1ff13f3fb --- /dev/null +++ b/adapter/internal_migration_namespace_mutation_test.go @@ -0,0 +1,200 @@ +package adapter + +import ( + "bytes" + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func activateInternalMigrationTestNamespace(t *testing.T, internal *Internal, proposer *applyingMigrationProposer) { + t.Helper() + inventory, ok := proposer.fsm.(raftengine.NamespaceConfigurationAware) + require.True(t, ok) + member := etcd.DeriveNodeID("voter") + require.NoError(t, inventory.RecordAppliedNamespaceConfiguration(9, []uint64{member})) + proposer.index = max(proposer.index, 9) + require.NoError(t, kv.ActivateMigrationNamespace(t.Context(), internal.store, proposer, kv.NamespaceAdmissionCommand{ + Version: kv.MigrationNamespaceVersion, ConfigurationIndex: 9, ApplyMembers: []uint64{member}, + Members: []kv.NamespaceMemberCapability{{MemberID: member, Version: kv.MigrationNamespaceVersion, ConfigurationIndex: 9}}, + })) + WithMigrationNamespaceAdmissionReady(func() bool { + return kv.MigrationNamespaceInstalledVersion(internal.store, proposer.fsm) == kv.MigrationNamespaceVersion + })(internal) +} + +func namespaceMutationFixture(t *testing.T, active, configured bool) (*Internal, *ownershipRuntimeProposer) { + t.Helper() + st := store.NewMVCCStore() + fsm := kv.NewKvFSMWithHLC(st, nil) + inventory, ok := fsm.(raftengine.NamespaceConfigurationAware) + require.True(t, ok) + member := etcd.DeriveNodeID("voter") + require.NoError(t, inventory.RecordAppliedNamespaceConfiguration(9, []uint64{member})) + proposer := &ownershipRuntimeProposer{fsm: fsm, index: 9} + if active { + require.NoError(t, kv.ActivateMigrationNamespace(t.Context(), st, proposer, kv.NamespaceAdmissionCommand{ + Version: kv.MigrationNamespaceVersion, ConfigurationIndex: 9, ApplyMembers: []uint64{member}, + Members: []kv.NamespaceMemberCapability{{MemberID: member, Version: kv.MigrationNamespaceVersion, ConfigurationIndex: 9}}, + })) + } + i := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(st), WithInternalMigrationProposer(proposer), + WithInternalMigrationImportGate(func(context.Context) error { return nil }), + WithInternalMigrationPromoteGate(func(context.Context) error { return nil })) + if configured { + WithMigrationNamespaceAdmissionReady(func() bool { return kv.MigrationNamespaceInstalledVersion(st, fsm) == kv.MigrationNamespaceVersion })(i) + } + require.NoError(t, st.PutAt(t.Context(), distribution.MigrationStagedDataKey(7, []byte("n")), []byte("staged"), 20, 0)) + return i, proposer +} + +func namespaceMutationRPCs() map[string]func(context.Context, *Internal) error { + return map[string]func(context.Context, *Internal) error{ + "import-unversioned": func(ctx context.Context, i *Internal) error { + _, err := i.ImportRangeVersions(ctx, &pb.ImportRangeVersionsRequest{JobId: 7, BracketId: 1, BatchSeq: 1, + Versions: []*pb.MVCCVersion{{Key: []byte("k"), Value: []byte("value"), CommitTs: 30}}}) + return err + }, + "promote-unversioned": func(ctx context.Context, i *Internal) error { + _, err := i.PromoteStagedVersions(ctx, &pb.PromoteStagedVersionsRequest{JobId: 7, MaxVersions: 10}) + return err + }, + "readiness-unversioned": func(ctx context.Context, i *Internal) error { + _, err := i.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{JobId: 7, MigrationJobId: 7, + RouteStart: []byte("a"), RouteEnd: []byte("z"), Armed: true, MinWriteTsExclusive: 100, ExpectedCutoverVersion: 1}) + return err + }, + "readiness-unarmed-unversioned": func(ctx context.Context, i *Internal) error { + _, err := i.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{JobId: 7, MigrationJobId: 7, + RouteStart: []byte("a"), RouteEnd: []byte("z"), ExpectedCutoverVersion: 1}) + return err + }, + } +} + +func TestInternalMigrationDataOpcodesRequireExplicitGateAfterActivation(t *testing.T) { + for _, name := range []string{"import-unversioned", "promote-unversioned"} { + t.Run(name, func(t *testing.T) { + i, proposer := namespaceMutationFixture(t, true, true) + i.migrationImportGate, i.migrationPromoteGate = nil, nil + before, index := namespaceMutationSnapshot(t, i.store), proposer.index + require.Error(t, namespaceMutationRPCs()[name](t.Context(), i)) + require.Equal(t, index, proposer.index) + require.Equal(t, before, namespaceMutationSnapshot(t, i.store)) + }) + } +} + +func TestInternalMigrationMutationsRetainDedicatedRPCAuthentication(t *testing.T) { + methods := map[string]string{"import-unversioned": pb.Internal_ImportRangeVersions_FullMethodName, + "promote-unversioned": pb.Internal_PromoteStagedVersions_FullMethodName, + "readiness-unversioned": pb.Internal_ApplyTargetStagedReadiness_FullMethodName} + for name, method := range methods { + t.Run(name, func(t *testing.T) { + for _, authorized := range []bool{false, true} { + i, proposer := namespaceMutationFixture(t, true, true) + auth, _ := SplitMigrationTokenAuth(t.Name(), false, false) + before := proposer.index + _, err := auth(migrationNamespacePeerContext(t, authorized), nil, &grpc.UnaryServerInfo{FullMethod: method}, func(ctx context.Context, _ any) (any, error) { + return nil, namespaceMutationRPCs()[name](ctx, i) + }) + if authorized { + require.NoError(t, err) + require.Equal(t, before+1, proposer.index) + } else { + require.Equal(t, codes.Unauthenticated, status.Code(err)) + require.Equal(t, before, proposer.index) + } + } + }) + } +} + +func TestNamespaceRuntimeFixturesUseActualActivationAndMonotonicProposals(t *testing.T) { + for _, split := range []bool{false, true} { + st := store.NewMVCCStore() + var client *ownershipRuntimeClient + if split { + client = newSplitRuntimeClient(t, distribution.NewEngine(), st, 1).ownershipRuntimeClient + } else { + client, _ = newOwnershipRuntimeClient(t, st, 1) + } + require.True(t, client.internal.MigrationNamespaceAdmissionSupported()) + require.NoError(t, kv.RequireMigrationNamespaceAdmission(t.Context(), st)) + value, err := st.GetAt(t.Context(), store.MigrationConfigurationIndexKey(), ^uint64(0)) + require.NoError(t, err) + state, err := store.DecodeMigrationNamespaceConfiguration(value) + require.NoError(t, err) + require.Equal(t, uint64(9), state.Index) + require.Equal(t, []uint64{etcd.DeriveNodeID("voter")}, state.Members) + proposer, ok := client.internal.migrationProposer.(*ownershipRuntimeProposer) + require.True(t, ok) + before := proposer.index + require.Greater(t, before, uint64(9)) + _, err = client.ApplyTargetStagedReadiness(t.Context(), &pb.TargetStagedReadinessRequest{JobId: 7, MigrationJobId: 7, + RouteStart: []byte("a"), RouteEnd: []byte("z"), Armed: true, MinWriteTsExclusive: 100, ExpectedCutoverVersion: 1}) + require.NoError(t, err) + require.Equal(t, before+1, proposer.index) + } +} + +func namespaceMutationSnapshot(t *testing.T, st store.MVCCStore) []byte { + t.Helper() + snapshot, err := st.Snapshot() + require.NoError(t, err) + defer snapshot.Close() + var raw bytes.Buffer + _, err = snapshot.WriteTo(&raw) + require.NoError(t, err) + return raw.Bytes() +} + +func TestInternalMigrationMutationsRequireNamespaceBeforeProposal(t *testing.T) { + for name, call := range namespaceMutationRPCs() { + for _, configured := range []bool{false, true} { + t.Run(name+map[bool]string{false: "/unconfigured", true: "/inactive"}[configured], func(t *testing.T) { + i, proposer := namespaceMutationFixture(t, false, configured) + before := namespaceMutationSnapshot(t, i.store) + index := proposer.index + err := call(t.Context(), i) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Equal(t, index, proposer.index, "no unversioned control may be proposed") + require.Equal(t, before, namespaceMutationSnapshot(t, i.store)) + }) + } + } +} + +func TestInternalMigrationMutationsRequireInstalledHooksAfterActivation(t *testing.T) { + for name, call := range namespaceMutationRPCs() { + t.Run(name, func(t *testing.T) { + i, proposer := namespaceMutationFixture(t, true, false) + before, index := namespaceMutationSnapshot(t, i.store), proposer.index + require.Error(t, call(t.Context(), i)) + require.Equal(t, index, proposer.index) + require.Equal(t, before, namespaceMutationSnapshot(t, i.store)) + }) + } +} + +func TestInternalMigrationMutationsAcceptActivatedProtocol(t *testing.T) { + for name, call := range namespaceMutationRPCs() { + t.Run(name, func(t *testing.T) { + i, proposer := namespaceMutationFixture(t, true, true) + index := proposer.index + require.NoError(t, call(t.Context(), i)) + require.Equal(t, index+1, proposer.index) + }) + } +} diff --git a/adapter/internal_migration_ownership.go b/adapter/internal_migration_ownership.go new file mode 100644 index 000000000..2f019245a --- /dev/null +++ b/adapter/internal_migration_ownership.go @@ -0,0 +1,105 @@ +package adapter + +import ( + "context" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +const migrationOwnershipProtocol = 2 + +// WithInternalMigrationOwnershipReady must be wired to the real public +// proposal/read and deterministic-apply integrations, not an environment flag. +// Until then this binary must not advertise complete ownership readiness. +func WithInternalMigrationOwnershipReady(ready func() bool) InternalOption { + return func(i *Internal) { i.migrationOwnershipReady = ready } +} + +func (i *Internal) ownershipAdmissionReady() bool { + return i.migrationOwnershipReady != nil && i.migrationOwnershipReady() +} + +func ownershipCleanupMode(mode pb.MigrationCleanupMode) bool { + return mode == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION || + mode == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_ACQUIRE || + mode == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR || + mode == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_SOURCE_OWNERSHIP_ACQUIRE || + mode == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH +} + +func (i *Internal) probeMigrationOwnership(ctx context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + resp := &pb.ProbeMigrationStateResponse{OwnershipPartitionProtocol: i.implementedPartitionProtocol()} + if !i.ownershipAdmissionReady() { + return resp, nil + } + engine, ok := i.leader.(raftengine.AppliedConfigurationReader) + if !ok { + return resp, nil + } + index, supported := engine.AppliedConfigurationIndex() + if !supported || index == 0 { + return resp, nil + } + reader, ok := i.store.(store.MigrationOwnershipStore) + if !ok { + return resp, nil + } + if _, ok := i.store.(store.MigrationOwnershipRefresher); !ok { + return resp, nil + } + view, err := reader.MigrationServingState(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + generation, valid := migrationOwnershipGeneration(view.Ownership) + if !valid { + return resp, nil + } + resp.OwnershipGeneration, resp.OwnershipProtocol, resp.AppliedConfigurationIndex = generation, migrationOwnershipProtocol, index + if req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY { + resp.Ready = true + return resp, nil + } + return i.probeMigrationOwnershipApplied(ctx, req, resp, view) +} + +func migrationOwnershipGeneration(states []store.MigrationOwnershipState) (string, bool) { + var generation string + for _, state := range states { + if !store.MigrationOwnershipEvidenceValid(state) { + return "", false + } + if generation != "" && state.Generation != generation { + return "", false + } + generation = state.Generation + } + return generation, true +} + +func (i *Internal) probeMigrationOwnershipApplied(ctx context.Context, req *pb.ProbeMigrationStateRequest, resp *pb.ProbeMigrationStateResponse, view store.MigrationServingState) (*pb.ProbeMigrationStateResponse, error) { + for _, state := range view.Ownership { + if state.JobID != req.GetJobId() { + continue + } + resp.Ownership = kv.MigrationOwnershipProofFromState(state) + resp.Ready = req.GetOwnership() == nil || proto.Equal(req.GetOwnership(), resp.GetOwnership()) + retirement, ok := i.store.(store.MigrationRetirementReader) + if !ok { + resp.Ready = false + return resp, nil + } + retired, err := retirement.MigrationRetired(ctx, state.JobID) + if err != nil { + return nil, errors.WithStack(err) + } + resp.OwnershipCleared = retired + return resp, nil + } + return resp, nil +} diff --git a/adapter/internal_migration_probe_test.go b/adapter/internal_migration_probe_test.go new file mode 100644 index 000000000..97f02ce10 --- /dev/null +++ b/adapter/internal_migration_probe_test.go @@ -0,0 +1,247 @@ +package adapter + +import ( + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestInternalProbeMigrationStateUsesLocalFSMAndCatalog(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 9, + Routes: []distribution.RouteDescriptor{{ + RouteID: 2, + Start: []byte("m"), + GroupID: 2, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 88, + }}, + })) + tracker := kv.NewActiveTimestampTracker() + internal := NewInternalWithEngine(nil, nil, nil, nil, + WithInternalStore(st), + WithInternalRouteEngine(engine), + WithInternalActiveTimestampTracker(tracker), + ) + + state := store.TargetStagedReadinessState{ + JobID: 7, + RouteStart: []byte("m"), + ExpectedCutoverVersion: 9, + MigrationJobID: 7, + MinWriteTSExclusive: 88, + Armed: true, + } + readinessWriter, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, readinessWriter.ApplyTargetStagedReadiness(ctx, state)) + control, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: []byte("m"), + ExpectedCatalogVersion: 9, + MigrationJobId: 7, + MinWriteTsExclusive: 88, + }) + require.NoError(t, err) + require.True(t, control.Ready) + + cleared, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED, + RouteStart: []byte("m"), + ExpectedCatalogVersion: 9, + ExpectedGroupId: 2, + MinWriteTsExclusive: 88, + }) + require.NoError(t, err) + require.True(t, cleared.Ready) + + pin := tracker.Pin(100) + drained, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED, + ReadDrainNotBeforeMs: time.Now().Add(-time.Second).UnixMilli(), + }) + require.NoError(t, err) + require.False(t, drained.Ready) + pin.Release() + drained, err = internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED, + ReadDrainNotBeforeMs: time.Now().Add(-time.Second).UnixMilli(), + }) + require.NoError(t, err) + require.True(t, drained.Ready) + + metadata, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + require.NoError(t, err) + require.False(t, metadata.Ready) + cleaner, ok := st.(store.MigrationCleaner) + require.True(t, ok) + require.NoError(t, cleaner.ClearMigrationState(ctx, 7, 0)) + metadata, err = internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + require.NoError(t, err) + require.True(t, metadata.Ready) +} + +func TestInternalProbeMigrationMetadataClearedWaitsForImportMetadata(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + internal := NewInternalWithEngine(nil, nil, nil, nil, WithInternalStore(st)) + _, err := st.ImportVersions(ctx, store.ImportVersionsOptions{ + JobID: 7, + BracketID: 1, + BatchSeq: 1, + Cursor: []byte("cursor-1"), + Versions: []store.MVCCVersion{{ + Key: []byte("m/key"), + Value: []byte("value"), + CommitTS: 42, + }}, + }) + require.NoError(t, err) + + metadata, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + require.NoError(t, err) + require.False(t, metadata.Ready) + + cleaner, ok := st.(store.MigrationCleaner) + require.True(t, ok) + require.NoError(t, cleaner.ClearMigrationState(ctx, 7, 0)) + metadata, err = internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + require.NoError(t, err) + require.True(t, metadata.Ready) +} + +func TestInternalIssueMigrationTimestampFollowsSourceLastCommit(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + require.NoError(t, st.ApplyMutations(ctx, []*store.KVPairMutation{{Key: []byte("m"), Value: []byte("value")}}, nil, 50, 50)) + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(st)) + + resp, err := internal.IssueMigrationTimestamp(ctx, &pb.IssueMigrationTimestampRequest{}) + require.NoError(t, err) + require.Equal(t, uint64(50), resp.GetLastCommitTs()) + require.Greater(t, resp.GetTimestamp(), resp.GetLastCommitTs()) +} + +func TestInternalProbeSourceReadDrainIgnoresPostCutoverReads(t *testing.T) { + t.Parallel() + + const cutoverTS = uint64(500) + + tests := []struct { + name string + pins []uint64 + drainMinTS uint64 + wantReady bool + }{ + { + name: "no active reads", + drainMinTS: cutoverTS, + wantReady: true, + }, + { + name: "pre-cutover read still in flight", + pins: []uint64{cutoverTS - 1}, + drainMinTS: cutoverTS, + wantReady: false, + }, + { + name: "read pinned exactly at the cutover", + pins: []uint64{cutoverTS}, + drainMinTS: cutoverTS, + wantReady: false, + }, + { + name: "unrelated post-cutover reads do not block", + pins: []uint64{cutoverTS + 1, cutoverTS + 9}, + drainMinTS: cutoverTS, + wantReady: true, + }, + { + name: "one pre-cutover read among newer ones still blocks", + pins: []uint64{cutoverTS + 9, cutoverTS - 1}, + drainMinTS: cutoverTS, + wantReady: false, + }, + { + name: "missing drain floor falls back to requiring an empty tracker", + pins: []uint64{cutoverTS + 1}, + drainMinTS: 0, + wantReady: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + tracker := kv.NewActiveTimestampTracker() + internal := NewInternalWithEngine(nil, nil, nil, nil, + WithInternalStore(store.NewMVCCStore()), + WithInternalActiveTimestampTracker(tracker), + ) + for _, ts := range tt.pins { + defer tracker.Pin(ts).Release() + } + + got, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED, + ReadDrainNotBeforeMs: time.Now().Add(-time.Second).UnixMilli(), + ReadDrainMinTs: tt.drainMinTS, + }) + require.NoError(t, err) + require.Equal(t, tt.wantReady, got.GetReady()) + }) + } +} + +func TestInternalProbeSourceReadDrainHonorsGracePeriod(t *testing.T) { + t.Parallel() + + ctx := context.Background() + internal := NewInternalWithEngine(nil, nil, nil, nil, + WithInternalStore(store.NewMVCCStore()), + WithInternalActiveTimestampTracker(kv.NewActiveTimestampTracker()), + ) + + got, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED, + ReadDrainNotBeforeMs: time.Now().Add(time.Hour).UnixMilli(), + ReadDrainMinTs: 500, + }) + require.NoError(t, err) + require.False(t, got.GetReady()) +} diff --git a/adapter/internal_migration_size_guard.go b/adapter/internal_migration_size_guard.go new file mode 100644 index 000000000..d0aae6999 --- /dev/null +++ b/adapter/internal_migration_size_guard.go @@ -0,0 +1,79 @@ +package adapter + +import ( + "context" + + "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" +) + +const MigrationTransferSizeCapability = "cap_migration_transfer_size_v1" + +// WithInternalMigrationTransferSizeGuardReady opts into capability advertising +// only after runtime wiring has verified public writers and member admission. +// Ownership protocol 2 alone is not evidence that size admission is supported. +func WithInternalMigrationTransferSizeGuardReady(ready func() bool) InternalOption { + return func(i *Internal) { i.transferSizeReady = ready } +} + +// MigrationTransferSizeGuardSupported can be true on a fresh learner before +// its first configuration is applied. It is never an active-guard ACK. +func (i *Internal) MigrationTransferSizeGuardSupported() bool { + if i.transferSizeReady == nil || !i.transferSizeReady() || + i.migrationProposer == nil { + return false + } + if _, ok := i.store.(store.MigrationTargetReadinessReader); !ok { + return false + } + if _, ok := i.store.(store.MigrationTargetReadinessRaftWriter); !ok { + return false + } + return true +} + +// MigrationTransferSizeGuardReady additionally requires an applied membership +// epoch. Activation still compares that epoch at proposal and deterministic apply. +func (i *Internal) MigrationTransferSizeGuardReady() bool { + if !i.MigrationTransferSizeGuardSupported() || !i.ownershipAdmissionReady() { + return false + } + reader, ok := i.leader.(raftengine.AppliedConfigurationReader) + if !ok { + return false + } + index, supported := reader.AppliedConfigurationIndex() + return supported && index != 0 +} + +func (i *Internal) validateTransferSizeGuardActivation(ctx context.Context, req *pb.TargetStagedReadinessRequest) error { + if req.GetTransferSizeGuardVersion() == 0 { + return nil + } + if !validTransferSizeGuardActivation(req) { + return grpcStatusErrorf(codes.InvalidArgument, "transfer size guard requires supported version, source tracker and configuration") + } + if !i.MigrationTransferSizeGuardReady() { + return grpcStatusErrorf(codes.FailedPrecondition, "migration transfer size admission is not wired") + } + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + reader, ok := i.leader.(raftengine.AppliedConfigurationReader) + if !ok { + return grpcStatusErrorf(codes.FailedPrecondition, "migration configuration reader is not wired") + } + index, supported := reader.AppliedConfigurationIndex() + if !supported || index != req.GetExpectedConfigurationIndex() { + return grpcStatusErrorf(codes.FailedPrecondition, "migration transfer size configuration changed") + } + return nil +} + +func validTransferSizeGuardActivation(req *pb.TargetStagedReadinessRequest) bool { + return req.GetTransferSizeGuardVersion() == store.MigrationTransferSizeGuardVersion && req.GetArmed() && req.GetTrackWrites() && + req.GetRequireConfigurationIndex() && req.GetExpectedConfigurationIndex() != 0 +} diff --git a/adapter/internal_migration_size_guard_test.go b/adapter/internal_migration_size_guard_test.go new file mode 100644 index 000000000..fe7ddec0c --- /dev/null +++ b/adapter/internal_migration_size_guard_test.go @@ -0,0 +1,110 @@ +package adapter + +import ( + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func newTransferGuardInternal(t *testing.T, index uint64, ready func() bool) (*Internal, raftengine.AppliedConfigurationAware) { + t.Helper() + st := store.NewMVCCStore() + t.Cleanup(func() { require.NoError(t, st.Close()) }) + fsm := kv.NewKvFSMWithHLC(st, nil, kv.WithRouteHistory(nil, 1)) + configuration, ok := fsm.(raftengine.AppliedConfigurationAware) + require.True(t, ok) + inventory, ok := fsm.(raftengine.NamespaceConfigurationAware) + require.True(t, ok) + member := etcd.DeriveNodeID("voter") + configuration = ownershipRuntimeConfiguration{AppliedConfigurationAware: configuration, inventory: inventory, members: []uint64{member}} + proposer := &ownershipRuntimeProposer{fsm: fsm, index: 20} + if index != 0 { + require.NoError(t, configuration.RecordAppliedConfiguration(index)) + require.NoError(t, kv.ActivateMigrationNamespace(t.Context(), st, proposer, kv.NamespaceAdmissionCommand{ + Version: kv.MigrationNamespaceVersion, ConfigurationIndex: index, ApplyMembers: []uint64{member}, + Members: []kv.NamespaceMemberCapability{{MemberID: member, Version: kv.MigrationNamespaceVersion, ConfigurationIndex: index}}, + })) + } + i := NewInternalWithEngine(nil, ownershipRuntimeLeader{configuration: configuration}, nil, nil, + WithInternalStore(st), WithInternalMigrationProposer(proposer), + WithMigrationNamespaceAdmissionReady(func() bool { return kv.MigrationNamespaceInstalledVersion(st, fsm) == kv.MigrationNamespaceVersion }), + WithInternalMigrationOwnershipReady(func() bool { return true }), WithInternalMigrationTransferSizeGuardReady(ready)) + return i, configuration +} + +func TestMigrationSizeGuardCapabilitySeparatesSupportFromEpoch(t *testing.T) { + for _, index := range []uint64{0, 9} { + for _, ready := range []bool{false, true} { + i, _ := newTransferGuardInternal(t, index, func() bool { return ready }) + require.Equal(t, ready, i.MigrationTransferSizeGuardSupported()) + require.Equal(t, ready && index != 0, i.MigrationTransferSizeGuardReady()) + resp, err := i.ProbeMigrationState(t.Context(), &pb.ProbeMigrationStateRequest{JobId: 11, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY, + RequireConfigurationIndex: true}) + require.NoError(t, err) + if ready { + require.Equal(t, uint32(1), resp.GetTransferSizeGuardVersion(), "fresh learner can advertise implemented support before joining") + } else { + require.Zero(t, resp.GetTransferSizeGuardVersion(), "ownership protocol 2 alone must not advertise the guard") + } + if index == 0 { + require.False(t, resp.GetReady(), "a static capability is never an applied epoch ACK") + } + } + } +} + +func TestMigrationSizeGuardStaticSupportSurvivesClosedOperationalGate(t *testing.T) { + i, _ := newTransferGuardInternal(t, 0, func() bool { return true }) + WithInternalMigrationOwnershipReady(func() bool { return false })(i) + require.True(t, i.MigrationTransferSizeGuardSupported(), "installed hooks do not disappear when the cluster activation gate closes") + require.False(t, i.MigrationTransferSizeGuardReady()) + resp, err := i.ProbeMigrationState(t.Context(), &pb.ProbeMigrationStateRequest{JobId: 11, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY, RequireConfigurationIndex: true}) + require.NoError(t, err) + require.Equal(t, uint32(1), resp.GetTransferSizeGuardVersion()) + require.False(t, resp.GetReady()) +} + +func TestMigrationSizeGuardActivationAndAppliedProof(t *testing.T) { + ready := false + i, configuration := newTransferGuardInternal(t, 9, func() bool { return ready }) + req := &pb.TargetStagedReadinessRequest{JobId: 11, MigrationJobId: 11, RouteStart: []byte("m"), RouteEnd: []byte("z"), + Armed: true, TrackWrites: true, MinWriteTsExclusive: 1, RetentionPinTs: 1, + TransferSizeGuardVersion: 1, RequireConfigurationIndex: true, ExpectedConfigurationIndex: 9} + _, err := i.ApplyTargetStagedReadiness(t.Context(), req) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + ready = true + probe := &pb.ProbeMigrationStateRequest{JobId: 11, MigrationJobId: 11, RouteStart: []byte("m"), RouteEnd: []byte("z"), + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + TrackWrites: true, MinWriteTsExclusive: 1, RetentionPinTs: 1, + TransferSizeGuardVersion: 1, RequireConfigurationIndex: true, ExpectedConfigurationIndex: 9} + resp, err := i.ProbeMigrationState(t.Context(), probe) + require.NoError(t, err) + require.Equal(t, uint32(1), resp.GetTransferSizeGuardVersion()) + require.False(t, resp.GetReady(), "capability alone cannot certify an unapplied guard") + _, err = i.ApplyTargetStagedReadiness(t.Context(), req) + require.NoError(t, err) + resp, err = i.ProbeMigrationState(t.Context(), probe) + require.NoError(t, err) + require.True(t, resp.GetReady()) + WithInternalMigrationOwnershipReady(func() bool { return false })(i) + resp, err = i.ProbeMigrationState(t.Context(), probe) + require.NoError(t, err) + require.Equal(t, uint32(1), resp.GetTransferSizeGuardVersion()) + require.False(t, resp.GetReady(), "a closed operational gate must block active-guard ACKs, not static support") + WithInternalMigrationOwnershipReady(func() bool { return true })(i) + require.NoError(t, configuration.RecordAppliedConfiguration(12)) + resp, err = i.ProbeMigrationState(t.Context(), probe) + require.NoError(t, err) + require.False(t, resp.GetReady(), "old configuration ACK cannot survive a topology change") + _, err = i.ApplyTargetStagedReadiness(t.Context(), req) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) +} diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index 47448aaf9..4e573e961 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -60,12 +60,17 @@ func (l *recordingInternalLeader) LinearizableRead(context.Context) (uint64, err type applyingMigrationProposer struct { fsm raftengine.StateMachine calls uint64 + index uint64 } func (p *applyingMigrationProposer) Propose(_ context.Context, data []byte) (*raftengine.ProposalResult, error) { p.calls++ + p.index++ + if aware, ok := p.fsm.(raftengine.ApplyIndexAware); ok { + aware.SetApplyIndex(p.index) + } return &raftengine.ProposalResult{ - CommitIndex: p.calls, + CommitIndex: p.index, Response: p.fsm.Apply(data), }, nil } @@ -661,6 +666,8 @@ func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { WithInternalMigrationProposer(proposer), WithInternalMigrationImportGate(func(context.Context) error { return nil }), ) + activateInternalMigrationTestNamespace(t, internal, proposer) + before := proposer.calls resp, err := internal.ImportRangeVersions(ctx, &pb.ImportRangeVersionsRequest{ JobId: 7, @@ -673,7 +680,7 @@ func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { }) require.NoError(t, err) require.Equal(t, []byte("cursor-1"), resp.GetAckedCursor()) - require.Equal(t, uint64(1), proposer.calls) + require.Equal(t, before+1, proposer.calls) staged := distribution.MigrationStagedDataKey(7, []byte("k")) got, err := st.GetAt(ctx, staged, 30) @@ -836,6 +843,8 @@ func TestInternalPromoteStagedVersionsAppliesStoreBatch(t *testing.T) { WithInternalMigrationProposer(proposer), WithInternalMigrationPromoteGate(func(context.Context) error { return nil }), ) + activateInternalMigrationTestNamespace(t, internal, proposer) + before := proposer.calls staged := distribution.MigrationStagedDataKey(7, []byte("k")) require.NoError(t, st.PutAt(ctx, staged, []byte("v"), 30, 0)) @@ -848,7 +857,7 @@ func TestInternalPromoteStagedVersionsAppliesStoreBatch(t *testing.T) { require.True(t, resp.GetDone()) require.Equal(t, uint64(1), resp.GetPromotedRows()) require.Equal(t, uint64(30), resp.GetMaxPromotedTs()) - require.Equal(t, uint64(1), proposer.calls) + require.Equal(t, before+1, proposer.calls) got, err := st.GetAt(ctx, []byte("k"), 30) require.NoError(t, err) @@ -888,3 +897,144 @@ func TestInternalExportRangeVersionsRefusesOversizedPage(t *testing.T) { require.Contains(t, err.Error(), `"a"`) require.Empty(t, stream.responses, "an unsendable page must not be sent") } + +func TestInternalApplyTargetStagedReadinessBypassesOpcodeGateForSafetyGuard(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, nil), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + WithInternalMigrationPromoteGate(func(context.Context) error { + return status.Error(codes.FailedPrecondition, "migration opcode disabled for test") + }), + ) + activateInternalMigrationTestNamespace(t, internal, proposer) + before := proposer.calls + + resp, err := internal.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + MinWriteTsExclusive: 100, + Armed: true, + }) + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, before+1, proposer.calls) + + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []store.TargetStagedReadinessState{{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }}, states) +} + +func TestInternalApplyTargetStagedReadinessProposesThroughRaft(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + clock := kv.NewHLC() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, nil), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, clock, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + WithInternalMigrationPromoteGate(func(context.Context) error { return nil }), + ) + activateInternalMigrationTestNamespace(t, internal, proposer) + before := proposer.calls + + _, err := internal.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + MinWriteTsExclusive: 100, + Armed: true, + }) + require.NoError(t, err) + require.Equal(t, before+1, proposer.calls) + require.Equal(t, uint64(100), st.LastCommitTS()) + require.GreaterOrEqual(t, clock.Current(), uint64(100)) + + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []store.TargetStagedReadinessState{{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }}, states) +} + +func TestInternalApplyTargetStagedReadinessRejectsArmedZeroMinWriteTS(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, nil), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + WithInternalMigrationPromoteGate(func(context.Context) error { return nil }), + ) + + resp, err := internal.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + Armed: true, + }) + require.Nil(t, resp) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + require.Equal(t, uint64(0), proposer.calls) +} + +func TestInternalCleanupMigrationRejectsWhenOpcodeGateClosed(t *testing.T) { + t.Parallel() + + proposer := &applyingMigrationProposer{fsm: kv.NewKvFSMWithHLC(store.NewMVCCStore(), nil)} + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, + WithInternalMigrationProposer(proposer), + WithInternalMigrationCleanupGate(func(context.Context) error { + return status.Error(codes.FailedPrecondition, "migration cleanup disabled for test") + }), + ) + + resp, err := internal.CleanupMigration(context.Background(), &pb.CleanupMigrationRequest{ + JobId: 9, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, + KeyFamily: distribution.MigrationFamilyUser, + }) + require.Nil(t, resp) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Zero(t, proposer.calls) +} diff --git a/adapter/internal_partition_control.go b/adapter/internal_partition_control.go new file mode 100644 index 000000000..0861c6d9c --- /dev/null +++ b/adapter/internal_partition_control.go @@ -0,0 +1,119 @@ +package adapter + +import ( + "bytes" + "context" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func validatePartitionCleanupRequest(req *pb.CleanupMigrationRequest) error { + if err := kv.ValidateMigrationPartitionRequest(req); err != nil { + return errors.WithStack(status.Error(codes.InvalidArgument, "invalid partition control")) + } + return nil +} + +func (i *Internal) validateMigrationCleanupAdmission(ctx context.Context, req *pb.CleanupMigrationRequest) error { + if err := validateCleanupMigrationRequest(req); err != nil { + return errors.WithStack(err) + } + if ownershipCleanupMode(req.GetMode()) && !i.ownershipAdmissionReady() { + return errors.WithStack(status.Error(codes.FailedPrecondition, "migration ownership data admission is not wired")) + } + if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION && i.implementedPartitionProtocol() == 0 { + return errors.WithStack(status.Error(codes.FailedPrecondition, "partition control admission is not installed")) + } + if i.migrationProposer == nil { + return errors.WithStack(status.Error(codes.FailedPrecondition, "migration cleanup proposer is not configured")) + } + if !i.MigrationNamespaceAdmissionSupported() { + return errors.WithStack(status.Error(codes.FailedPrecondition, "migration namespace admission is not installed")) + } + if err := kv.RequireMigrationNamespaceAdmission(ctx, i.store); err != nil { + return errors.WithStack(err) + } + return nil +} + +func (i *Internal) migrationNamespaceProtocolVersion() uint32 { + if i.MigrationNamespaceAdmissionSupported() { + return kv.MigrationNamespaceVersion + } + return 0 +} + +func (i *Internal) probeMigrationPartition(ctx context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + resp := &pb.ProbeMigrationStateResponse{OwnershipPartitionProtocol: i.implementedPartitionProtocol()} + if !i.partitionProbeAdmissionReady(req) { + return resp, nil + } + var transition store.MigrationOwnershipPartitionTransition + if err := kv.DecodeMigrationPartitionJSON(req.GetPartitionTransition(), &transition); err != nil { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "invalid partition probe")) + } + index, ready, err := i.partitionProbeEpoch(ctx, req, transition) + if err != nil { + return nil, errors.WithStack(err) + } + if !ready { + return resp, nil + } + reader, ok := i.store.(store.MigrationOwnershipPartitionStore) + if !ok { + return resp, nil + } + state, err := reader.MigrationOwnershipPartitions(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + resp.OwnershipGeneration, resp.OwnershipProtocol, resp.AppliedConfigurationIndex = state.Generation, migrationOwnershipProtocol, index + if state.Generation != transition.Generation { + return resp, nil + } + id, err := store.MigrationOwnershipPartitionReceiptIDFor(transition) + if err != nil { + return nil, errors.WithStack(err) + } + if migrationPartitionReceiptPresent(state, id) { + resp.PartitionReceiptId, resp.Ready = bytes.Clone(id), true + } + return resp, nil +} + +func (i *Internal) partitionProbeAdmissionReady(req *pb.ProbeMigrationStateRequest) bool { + return i.implementedPartitionProtocol() != 0 && i.ownershipAdmissionReady() && req.GetRequireConfigurationIndex() && req.GetExpectedConfigurationIndex() != 0 +} + +func (i *Internal) partitionProbeEpoch(ctx context.Context, req *pb.ProbeMigrationStateRequest, t store.MigrationOwnershipPartitionTransition) (uint64, bool, error) { + engine, ok := i.leader.(raftengine.AppliedConfigurationReader) + if !ok { + return 0, false, nil + } + index, supported := engine.AppliedConfigurationIndex() + if !supported || index == 0 || index != req.GetExpectedConfigurationIndex() || index != t.ConfigurationIndex { + return 0, false, nil + } + durable, err := kv.MigrationConfigurationIndex(ctx, i.store) + return index, durable == index, errors.WithStack(err) +} + +func migrationPartitionReceiptPresent(state store.MigrationOwnershipPartitionState, id []byte) bool { + for _, receipt := range state.Receipts { + if !bytes.Equal(receipt.ReceiptID, id) || receipt.AppliedIndex == 0 { + continue + } + actual, hashErr := store.MigrationOwnershipPartitionReceiptIDFor(receipt.Transition) + if hashErr != nil || !bytes.Equal(actual, id) { + continue + } + return true + } + return false +} diff --git a/adapter/internal_partition_protocol.go b/adapter/internal_partition_protocol.go new file mode 100644 index 000000000..1749ae085 --- /dev/null +++ b/adapter/internal_partition_protocol.go @@ -0,0 +1,33 @@ +package adapter + +import ( + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" +) + +const ( + MigrationPartitionCapability = "cap_migration_partition_v1" + MigrationOwnershipPartitionProtocolVersion uint32 = 1 +) + +// WithInternalMigrationPartitionProtocolReady attests installed partition +// support, independently of an active Raft configuration. A candidate learner +// must be able to answer this before being added. The callback must check the +// real control, public admission and committed-root consumer integrations. +// It never enables migration activation or operational ownership readiness. +func WithInternalMigrationPartitionProtocolReady(ready func() bool) InternalOption { + return func(i *Internal) { i.migrationPartitionProtocolReady = ready } +} + +func (i *Internal) implementedPartitionProtocol() uint32 { + if i.migrationPartitionProtocolReady == nil || !i.migrationPartitionProtocolReady() { + return 0 + } + if i.migrationProposer == nil || !kv.MigrationPartitionRequestAdmissionReady(i.transactionManager) { + return 0 + } + if _, ok := i.store.(store.MigrationOwnershipPartitionStore); !ok { + return 0 + } + return MigrationOwnershipPartitionProtocolVersion +} diff --git a/adapter/internal_partition_protocol_test.go b/adapter/internal_partition_protocol_test.go new file mode 100644 index 000000000..d28f06376 --- /dev/null +++ b/adapter/internal_partition_protocol_test.go @@ -0,0 +1,89 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/reflect/protoreflect" +) + +type partitionProtocolStore struct{ store.MVCCStore } + +type legacyPartitionProtocolStore struct{ store.MVCCStore } + +func (s *partitionProtocolStore) MigrationOwnershipPartitions(context.Context) (store.MigrationOwnershipPartitionState, error) { + return store.MigrationOwnershipPartitionState{}, nil +} + +func (s *partitionProtocolStore) ApplyMigrationOwnershipPartition(context.Context, store.MigrationOwnershipPartitionTransition, uint64) error { + return store.ErrNotSupported +} + +func TestMigrationPartitionProtocolSupportsUnconfiguredCandidate(t *testing.T) { + st := store.NewMVCCStore() + proposer := &ownershipRuntimeProposer{fsm: kv.NewKvFSMWithHLC(st, nil)} + tx := kv.NewTransactionWithProposer(proposer, kv.WithMigrationRequestAdmission(st, 7)) + t.Cleanup(tx.Close) + i := NewInternalWithEngine(tx, migrationConfigurationLeader{}, nil, nil, + WithInternalStore(st), WithInternalMigrationProposer(proposer), + WithInternalMigrationPartitionProtocolReady(func() bool { return true })) + resp, err := i.ProbeMigrationState(t.Context(), &pb.ProbeMigrationStateRequest{JobId: 1, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY, + RequireConfigurationIndex: true}) + require.NoError(t, err) + field := resp.ProtoReflect().Descriptor().Fields().ByNumber(10) + require.NotNil(t, field, "partition protocol field allocation must stay disjoint from transfer size") + require.Equal(t, protoreflect.Name("ownership_partition_protocol"), field.Name()) + require.Equal(t, uint64(1), resp.ProtoReflect().Get(field).Uint()) + require.False(t, resp.GetReady(), "static support is not operational admission") + require.Zero(t, resp.GetAppliedConfigurationIndex(), "a candidate has no applied configuration yet") + require.Zero(t, resp.GetOwnershipProtocol()) +} + +func TestMigrationPartitionProtocolRequiresProposalAdmission(t *testing.T) { + st := &partitionProtocolStore{MVCCStore: store.NewMVCCStore()} + proposer := &ownershipRuntimeProposer{fsm: kv.NewKvFSMWithHLC(st, nil)} + tx := kv.NewTransactionWithProposer(proposer) + t.Cleanup(tx.Close) + i := NewInternalWithEngine(tx, migrationConfigurationLeader{}, nil, nil, + WithInternalStore(st), WithInternalMigrationProposer(proposer), + WithInternalMigrationPartitionProtocolReady(func() bool { return true })) + require.Zero(t, i.implementedPartitionProtocol(), "an explicit callback cannot replace a missing proposal hook") + readyTx := kv.NewTransactionWithProposer(proposer, kv.WithMigrationRequestAdmission(st, 7)) + t.Cleanup(readyTx.Close) + i.transactionManager = readyTx + i.migrationProposer = nil + require.Zero(t, i.implementedPartitionProtocol(), "a concrete control proposer is required") +} + +func TestMigrationPartitionProtocolDefaultDenyAndStorePairing(t *testing.T) { + for _, tc := range []struct { + name string + store store.MVCCStore + ready func() bool + }{ + {name: "missing callback", store: &partitionProtocolStore{MVCCStore: store.NewMVCCStore()}}, + {name: "not installed", store: &partitionProtocolStore{MVCCStore: store.NewMVCCStore()}, ready: func() bool { return false }}, + {name: "missing store", ready: func() bool { return true }}, + {name: "unsupported store", store: &legacyPartitionProtocolStore{MVCCStore: store.NewMVCCStore()}, ready: func() bool { return true }}, + } { + t.Run(tc.name, func(t *testing.T) { + proposer := &ownershipRuntimeProposer{fsm: kv.NewKvFSMWithHLC(store.NewMVCCStore(), nil)} + tx := kv.NewTransactionWithProposer(proposer, kv.WithMigrationRequestAdmission(tc.store, 7)) + t.Cleanup(tx.Close) + i := NewInternalWithEngine(tx, migrationConfigurationLeader{index: 9}, nil, nil, + WithInternalStore(tc.store), WithInternalMigrationProposer(proposer), WithInternalMigrationPartitionProtocolReady(tc.ready)) + resp, err := i.ProbeMigrationState(t.Context(), &pb.ProbeMigrationStateRequest{JobId: 1, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY}) + require.NoError(t, err) + field := resp.ProtoReflect().Descriptor().Fields().ByNumber(10) + require.NotNil(t, field) + require.Zero(t, resp.ProtoReflect().Get(field).Uint()) + require.False(t, resp.GetReady()) + }) + } +} diff --git a/adapter/migration_moveback_acceptance_test.go b/adapter/migration_moveback_acceptance_test.go new file mode 100644 index 000000000..1ff48ee2b --- /dev/null +++ b/adapter/migration_moveback_acceptance_test.go @@ -0,0 +1,36 @@ +package adapter + +import ( + "testing" + + "github.com/bootjp/elastickv/kv" + "github.com/stretchr/testify/require" +) + +func TestMigrationResplitMoveBackReacquiresHistoricalNonOwner(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + first := r.migrate(t, 1, 2, "m") + second := r.migrate(t, r.ownership(t, 2)[0].RouteID, 3, "t") + require.NoError(t, r.server.RunMigrationOwnershipRefreshOnce(t.Context())) + third := r.migrate(t, r.ownership(t, 3)[0].RouteID, 1, "x") + require.NoError(t, r.server.RunMigrationOwnershipRefreshOnce(t.Context())) + require.GreaterOrEqual(t, third.FenceTS, second.FenceTS) + r.read(t, "n", "original-n") + r.read(t, "y", "original-y") + require.ErrorIs(t, r.put(t, 1, "y", "too-old", third.FenceTS), kv.ErrRouteWriteTimestampTooLow) + require.NoError(t, r.put(t, 1, "y", "returned-owner", third.FenceTS+100)) + r.read(t, "y", "returned-owner") + require.NoError(t, r.put(t, 2, "n", "retained-left", third.FenceTS+101)) + r.read(t, "n", "retained-left") + require.ErrorIs(t, r.put(t, 2, "n", "too-old", first.FenceTS), kv.ErrRouteWriteTimestampTooLow) + require.Error(t, r.put(t, 3, "y", "retired-owner", third.FenceTS+102)) +} + +func TestMigrationResplitMoveBackDoesNotRequireBackgroundGeometryRefresh(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + r.migrate(t, 1, 2, "m") + r.migrate(t, r.ownership(t, 2)[0].RouteID, 3, "t") + returned := r.migrate(t, r.ownership(t, 3)[0].RouteID, 1, "x") + require.NoError(t, r.put(t, 1, "y", "returned-without-background", returned.FenceTS+100)) + r.read(t, "y", "returned-without-background") +} diff --git a/adapter/migration_namespace_activation.go b/adapter/migration_namespace_activation.go new file mode 100644 index 000000000..d49d50813 --- /dev/null +++ b/adapter/migration_namespace_activation.go @@ -0,0 +1,207 @@ +package adapter + +import ( + "context" + "slices" + "time" + + "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" +) + +const namespaceProbeTimeout = 2 * time.Second + +type MigrationNamespaceProbeClient interface { + ProbeMigrationState(context.Context, *pb.ProbeMigrationStateRequest, ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) +} + +type NamespaceAdmissionPeer struct { + MemberID uint64 + Address string + Client MigrationNamespaceProbeClient +} + +type NamespaceAdmissionSnapshot struct { + ConfigurationIndex uint64 + Members []NamespaceAdmissionPeer +} + +// EnsureMigrationNamespaceAdmission precedes partition/ownership controls. It +// must receive the FULL apply-member inventory, not the historical voter-only +// subset. The FSM independently compares this roster to committed ConfState. +func EnsureMigrationNamespaceAdmission(ctx context.Context, client SplitMigrationClient, members []SplitMigrationVoter) error { + if client == nil { + return errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + snapshot := func(ctx context.Context) (NamespaceAdmissionSnapshot, error) { + state := NamespaceAdmissionSnapshot{} + for _, member := range members { + if member.ID == "" { + return state, errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + peer := NamespaceAdmissionPeer{MemberID: etcd.DeriveNodeID(member.ID), Address: member.Address, Client: member.Client} + response, err := probeNamespacePeer(ctx, peer, 0, false) + if err != nil { + return state, err + } + index := response.GetAppliedConfigurationIndex() + if index == 0 || (state.ConfigurationIndex != 0 && state.ConfigurationIndex != index) { + return state, errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + state.ConfigurationIndex = index + state.Members = append(state.Members, peer) + } + return state, nil + } + activate := func(ctx context.Context, command kv.NamespaceAdmissionCommand) error { + request, err := kv.NewNamespaceAdmissionRequest(1, command) + if err != nil { + return errors.WithStack(err) + } + _, err = cleanupSplitMigration(ctx, client, request) + return errors.WithStack(err) + } + return EnsureMigrationNamespaceAdmissionWithSnapshot(ctx, snapshot, activate, nil) +} + +// EnsureMigrationNamespaceAdmissionWithSnapshot also supports durable prejoin +// registration. It never infers a peer's version from the local binary. Static +// support is independent of Ready so this can bootstrap an inactive group. +func EnsureMigrationNamespaceAdmissionWithSnapshot(ctx context.Context, + snapshot func(context.Context) (NamespaceAdmissionSnapshot, error), + activate func(context.Context, kv.NamespaceAdmissionCommand) error, + joining []NamespaceAdmissionPeer, +) error { + if snapshot == nil || activate == nil { + return errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + before, err := snapshot(ctx) + if err != nil { + return errors.WithStack(err) + } + if err := activateNamespaceWithSnapshot(ctx, snapshot, before, activate, joining); err != nil { + return err + } + for _, member := range before.Members { + if _, err := probeNamespacePeer(ctx, member, before.ConfigurationIndex, true); err != nil { + return err + } + } + return revalidateNamespaceSnapshot(ctx, snapshot, before) +} + +func activateNamespaceWithSnapshot(ctx context.Context, + snapshot func(context.Context) (NamespaceAdmissionSnapshot, error), before NamespaceAdmissionSnapshot, + activate func(context.Context, kv.NamespaceAdmissionCommand) error, joining []NamespaceAdmissionPeer, +) error { + command, alreadyActive, err := namespaceNegotiation(ctx, before, joining) + if err != nil { + return err + } + if err := revalidateNamespaceSnapshot(ctx, snapshot, before); err != nil { + return err + } + if !alreadyActive || len(joining) != 0 { + if err := activate(ctx, command); err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func namespaceNegotiation(ctx context.Context, snapshot NamespaceAdmissionSnapshot, joining []NamespaceAdmissionPeer) (kv.NamespaceAdmissionCommand, bool, error) { + command := kv.NamespaceAdmissionCommand{Version: kv.MigrationNamespaceVersion, ConfigurationIndex: snapshot.ConfigurationIndex} + if snapshot.ConfigurationIndex == 0 || len(snapshot.Members) == 0 { + return command, false, errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + alreadyActive := true + addresses := make(map[string]bool) + ids := make(map[uint64]bool) + for _, member := range snapshot.Members { + if ids[member.MemberID] || addresses[member.Address] { + return command, false, errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + response, err := probeNamespacePeer(ctx, member, snapshot.ConfigurationIndex, false) + if err != nil { + return command, false, err + } + alreadyActive = alreadyActive && response.GetReady() + ids[member.MemberID], addresses[member.Address] = true, true + command.ApplyMembers = append(command.ApplyMembers, member.MemberID) + command.Members = append(command.Members, kv.NamespaceMemberCapability{MemberID: member.MemberID, Version: response.GetNamespaceAdmissionVersion(), ConfigurationIndex: snapshot.ConfigurationIndex}) + } + var err error + command.Joining, err = probeNamespaceJoining(ctx, snapshot.ConfigurationIndex, joining, ids, addresses) + if err != nil { + return command, false, err + } + slices.Sort(command.ApplyMembers) + return command, alreadyActive, nil +} + +func probeNamespaceJoining(ctx context.Context, configuration uint64, joining []NamespaceAdmissionPeer, ids map[uint64]bool, addresses map[string]bool) ([]kv.NamespaceMemberCapability, error) { + var capabilities []kv.NamespaceMemberCapability + for _, member := range joining { + if ids[member.MemberID] || addresses[member.Address] { + return nil, errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + // A candidate does not yet have a source-group configuration. Only its + // installed support is probed; authorization is bound to source cfg. + response, err := probeNamespacePeer(ctx, member, 0, false) + if err != nil { + return nil, err + } + ids[member.MemberID], addresses[member.Address] = true, true + capabilities = append(capabilities, kv.NamespaceMemberCapability{MemberID: member.MemberID, Version: response.GetNamespaceAdmissionVersion(), ConfigurationIndex: configuration}) + } + return capabilities, nil +} + +func probeNamespacePeer(ctx context.Context, member NamespaceAdmissionPeer, configuration uint64, activated bool) (*pb.ProbeMigrationStateResponse, error) { + if member.MemberID == 0 || member.Address == "" || member.Client == nil { + return nil, errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + attempt, cancel := context.WithTimeout(ctx, namespaceProbeTimeout) + defer cancel() + response, err := member.Client.ProbeMigrationState(attempt, &pb.ProbeMigrationStateRequest{JobId: 1, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_NAMESPACE_ADMISSION, + RequireConfigurationIndex: configuration != 0, ExpectedConfigurationIndex: configuration}) + if err != nil { + return nil, errors.Wrapf(err, "namespace member %d", member.MemberID) + } + if response.GetNamespaceAdmissionVersion() < kv.MigrationNamespaceVersion || + (configuration != 0 && response.GetAppliedConfigurationIndex() != configuration) { + return nil, errors.Wrapf(kv.ErrMigrationNamespaceProtocol, "namespace member %d support/configuration/activation unproven", member.MemberID) + } + if activated && !response.GetReady() { + return nil, errors.Wrapf(errSplitMigrationVoterBarrierIncomplete, "namespace member %d awaits activation ACK", member.MemberID) + } + return response, nil +} + +func revalidateNamespaceSnapshot(ctx context.Context, snapshot func(context.Context) (NamespaceAdmissionSnapshot, error), before NamespaceAdmissionSnapshot) error { + after, err := snapshot(ctx) + if err != nil { + return errors.WithStack(err) + } + canonical := func(members []NamespaceAdmissionPeer) []NamespaceAdmissionPeer { + members = slices.Clone(members) + slices.SortFunc(members, func(a, b NamespaceAdmissionPeer) int { + if a.MemberID < b.MemberID { + return -1 + } + if a.MemberID > b.MemberID { + return 1 + } + return 0 + }) + return members + } + if before.ConfigurationIndex != after.ConfigurationIndex || !slices.EqualFunc(canonical(before.Members), canonical(after.Members), func(a, b NamespaceAdmissionPeer) bool { return a.MemberID == b.MemberID && a.Address == b.Address }) { + return errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + return nil +} diff --git a/adapter/migration_namespace_activation_deadline_test.go b/adapter/migration_namespace_activation_deadline_test.go new file mode 100644 index 000000000..ef328c58c --- /dev/null +++ b/adapter/migration_namespace_activation_deadline_test.go @@ -0,0 +1,103 @@ +package adapter + +import ( + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type namespaceDeadlineAttempt struct { + ctx context.Context + request *pb.CleanupMigrationRequest +} + +type namespaceDeadlineClient struct { + splitMigrationClientStub + started chan namespaceDeadlineAttempt + stop chan struct{} +} + +func (*namespaceDeadlineClient) ProbeMigrationState(context.Context, *pb.ProbeMigrationStateRequest, ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + return &pb.ProbeMigrationStateResponse{NamespaceAdmissionVersion: kv.MigrationNamespaceVersion, AppliedConfigurationIndex: 9}, nil +} + +func (c *namespaceDeadlineClient) CleanupMigration(ctx context.Context, req *pb.CleanupMigrationRequest, _ ...grpc.CallOption) (*pb.CleanupMigrationResponse, error) { + c.started <- namespaceDeadlineAttempt{ctx: ctx, request: req} + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-c.stop: + return nil, context.Canceled + } +} + +func startNamespaceDeadlineAttempt(t *testing.T, ctx context.Context) (namespaceDeadlineAttempt, <-chan error) { + t.Helper() + client := &namespaceDeadlineClient{started: make(chan namespaceDeadlineAttempt, 1), stop: make(chan struct{})} + result, finished := make(chan error, 1), make(chan struct{}) + go func() { + defer close(finished) + result <- EnsureMigrationNamespaceAdmission(ctx, client, []SplitMigrationVoter{{ID: "n1", Address: "n1:7001", Client: client}}) + }() + // Release and join even on the unfixed path, which has no RPC deadline. + t.Cleanup(func() { close(client.stop); <-finished }) + select { + case attempt := <-client.started: + require.NoError(t, kv.ValidateNamespaceAdmissionRequest(attempt.request)) + require.Equal(t, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION, attempt.request.GetMode()) + return attempt, result + case err := <-result: + t.Fatalf("namespace activation returned before the MODE8 RPC: %v", err) + case <-time.After(3 * defaultSplitMigrationAttemptTimeout): + t.Fatal("namespace activation did not reach the MODE8 RPC") + } + return namespaceDeadlineAttempt{}, nil +} + +func namespaceDeadlineResult(t *testing.T, result <-chan error) error { + t.Helper() + select { + case err := <-result: + return err + case <-time.After(3 * defaultSplitMigrationAttemptTimeout): + t.Fatal("namespace activation did not release the controller") + return nil + } +} + +func TestNamespaceAdmissionActivationAttemptBoundsHungRPC(t *testing.T) { + parent := context.Background() + attempt, result := startNamespaceDeadlineAttempt(t, parent) + deadline, bounded := attempt.ctx.Deadline() + require.True(t, bounded, "MODE8 needs its own deadline even when the controller context lives forever") + require.LessOrEqual(t, time.Until(deadline), defaultSplitMigrationAttemptTimeout) + require.ErrorIs(t, namespaceDeadlineResult(t, result), context.DeadlineExceeded) + require.ErrorIs(t, attempt.ctx.Err(), context.DeadlineExceeded) + require.NoError(t, parent.Err(), "one stalled member must not cancel the controller lifecycle") +} + +func TestNamespaceAdmissionActivationAttemptHonorsParentCancellation(t *testing.T) { + parent, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + attempt, result := startNamespaceDeadlineAttempt(t, parent) + cancel() + require.ErrorIs(t, namespaceDeadlineResult(t, result), context.Canceled) + require.ErrorIs(t, attempt.ctx.Err(), context.Canceled) +} + +func TestNamespaceAdmissionActivationAttemptKeepsEarlierParentDeadline(t *testing.T) { + parent, cancel := context.WithTimeout(context.Background(), namespaceProbeTimeout/10) + t.Cleanup(cancel) + attempt, result := startNamespaceDeadlineAttempt(t, parent) + want, bounded := parent.Deadline() + require.True(t, bounded) + got, bounded := attempt.ctx.Deadline() + require.True(t, bounded) + require.Equal(t, want, got, "a per-attempt budget must never extend the caller deadline") + require.ErrorIs(t, namespaceDeadlineResult(t, result), context.DeadlineExceeded) +} diff --git a/adapter/migration_namespace_activation_test.go b/adapter/migration_namespace_activation_test.go new file mode 100644 index 000000000..7332f76f1 --- /dev/null +++ b/adapter/migration_namespace_activation_test.go @@ -0,0 +1,266 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type namespaceRuntimeClient struct { + splitMigrationClientStub + internal *Internal + version uint32 + configuration uint64 + proposals int + probes int +} + +func (c *namespaceRuntimeClient) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest, _ ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + c.probes++ + response, err := c.internal.probeMigrationNamespace(ctx, req) + if err != nil { + return nil, err + } + response.NamespaceAdmissionVersion = c.version + if c.configuration != 0 { + response.AppliedConfigurationIndex = c.configuration + } + return response, nil +} + +func (c *namespaceRuntimeClient) CleanupMigration(ctx context.Context, req *pb.CleanupMigrationRequest, _ ...grpc.CallOption) (*pb.CleanupMigrationResponse, error) { + c.proposals++ + return c.internal.applyMigrationNamespaceAdmission(ctx, req) +} + +func namespaceRuntimeFixture(t *testing.T) ([]SplitMigrationVoter, []*namespaceRuntimeClient) { + t.Helper() + ids := []uint64{etcd.DeriveNodeID("n1"), etcd.DeriveNodeID("n2"), etcd.DeriveNodeID("learner")} + var clients []*namespaceRuntimeClient + var machines []kv.FSM + for range ids { + st := store.NewMVCCStore() + fsm := kv.NewKvFSMWithHLC(st, nil) + aware, ok := fsm.(raftengine.NamespaceConfigurationAware) + require.True(t, ok) + require.NoError(t, aware.RecordAppliedNamespaceConfiguration(9, ids)) + configuration, ok := fsm.(raftengine.AppliedConfigurationAware) + require.True(t, ok) + i := NewInternalWithEngine(nil, ownershipRuntimeLeader{configuration: configuration}, nil, nil, + WithInternalStore(st), WithMigrationNamespaceAdmissionReady(func() bool { return true })) + clients = append(clients, &namespaceRuntimeClient{internal: i, version: kv.MigrationNamespaceVersion}) + machines = append(machines, fsm) + } + proposer := &ownershipRuntimeProposer{fsm: machines[0], followers: machines[1:], index: 10} + var members []SplitMigrationVoter + for index, name := range []string{"n1", "n2", "learner"} { + clients[index].internal.migrationProposer = proposer + members = append(members, SplitMigrationVoter{ID: name, Address: name + ":7001", Client: clients[index]}) + } + return members, clients +} + +func TestNamespaceAdmissionControllerBootstrapsBeforeOperationalGuards(t *testing.T) { + members, clients := namespaceRuntimeFixture(t) + for _, client := range clients { + probe, err := client.ProbeMigrationState(t.Context(), &pb.ProbeMigrationStateRequest{}) + require.NoError(t, err) + require.False(t, probe.GetReady()) + require.Equal(t, kv.MigrationNamespaceVersion, probe.GetNamespaceAdmissionVersion()) + } + require.NoError(t, EnsureMigrationNamespaceAdmission(t.Context(), clients[0], members)) + require.Equal(t, 1, clients[0].proposals) + for _, client := range clients { + require.NoError(t, kv.RequireMigrationNamespaceAdmission(t.Context(), client.internal.store)) + } +} + +func TestNamespaceAdmissionControllerActiveGroupDoesNotPropose(t *testing.T) { + members, clients := namespaceRuntimeFixture(t) + require.NoError(t, EnsureMigrationNamespaceAdmission(t.Context(), clients[0], members)) + for range 3 { + before := make([]int, len(clients)) + for index, client := range clients { + before[index] = client.probes + } + require.NoError(t, EnsureMigrationNamespaceAdmission(t.Context(), clients[0], members)) + require.Equal(t, 1, clients[0].proposals) + for index, client := range clients { + // Three inventory reads, negotiation, and the final durable ACK. + require.Equal(t, before[index]+5, client.probes) + } + } +} + +type namespaceProbeFunc func(context.Context, *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) + +func (f namespaceProbeFunc) ProbeMigrationState(ctx context.Context, request *pb.ProbeMigrationStateRequest, _ ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + return f(ctx, request) +} + +func namespaceRuntimeSnapshot(members []SplitMigrationVoter) NamespaceAdmissionSnapshot { + snapshot := NamespaceAdmissionSnapshot{ConfigurationIndex: 9} + for _, member := range members { + snapshot.Members = append(snapshot.Members, NamespaceAdmissionPeer{ + MemberID: etcd.DeriveNodeID(member.ID), Address: member.Address, Client: member.Client, + }) + } + return snapshot +} + +func TestNamespaceAdmissionNegotiationRequiresEveryMemberReady(t *testing.T) { + members, clients := namespaceRuntimeFixture(t) + require.NoError(t, EnsureMigrationNamespaceAdmission(t.Context(), clients[0], members)) + state := namespaceRuntimeSnapshot(members) + state.Members[2].Client = namespaceProbeFunc(func(ctx context.Context, request *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + response, err := clients[2].ProbeMigrationState(ctx, request) + if err == nil { + response.Ready = false + } + return response, err + }) + _, alreadyActive, err := namespaceNegotiation(t.Context(), state, nil) + require.NoError(t, err) + require.False(t, alreadyActive) + require.NoError(t, kv.RequireMigrationNamespaceAdmission(t.Context(), clients[0].internal.store)) +} + +func TestNamespaceAdmissionActiveGroupJoiningStillRefreshes(t *testing.T) { + members, clients := namespaceRuntimeFixture(t) + require.NoError(t, EnsureMigrationNamespaceAdmission(t.Context(), clients[0], members)) + state := namespaceRuntimeSnapshot(members) + snapshot := func(context.Context) (NamespaceAdmissionSnapshot, error) { return state, nil } + activate := func(ctx context.Context, command kv.NamespaceAdmissionCommand) error { + request, err := kv.NewNamespaceAdmissionRequest(1, command) + if err != nil { + return err + } + _, err = clients[0].CleanupMigration(ctx, request) + return err + } + candidateStore := store.NewMVCCStore() + candidateFSM := kv.NewKvFSMWithHLC(candidateStore, nil) + candidate := NewInternalWithEngine(nil, nil, nil, nil, WithInternalStore(candidateStore), + WithMigrationNamespaceAdmissionReady(func() bool { + return kv.MigrationNamespaceInstalledVersion(candidateStore, candidateFSM) == kv.MigrationNamespaceVersion + })) + candidate.migrationProposer = clients[0].internal.migrationProposer + joiningID := etcd.DeriveNodeID("joining") + joining := []NamespaceAdmissionPeer{{MemberID: joiningID, Address: "joining:7001", + Client: namespaceProbeFunc(candidate.probeMigrationNamespace)}} + for round := range 2 { + require.NoError(t, EnsureMigrationNamespaceAdmissionWithSnapshot(t.Context(), snapshot, activate, joining)) + require.Equal(t, 2+round, clients[0].proposals) + for _, client := range clients { + require.NoError(t, kv.CheckMigrationNamespacePrejoin(t.Context(), client.internal.store, 9, joiningID, kv.MigrationNamespaceVersion)) + } + } +} + +func TestNamespaceAdmissionActiveGroupStillRequiresFinalACK(t *testing.T) { + for _, failure := range []string{"ready", "version", "configuration"} { + t.Run(failure, func(t *testing.T) { + members, clients := namespaceRuntimeFixture(t) + require.NoError(t, EnsureMigrationNamespaceAdmission(t.Context(), clients[0], members)) + state := namespaceRuntimeSnapshot(members) + probes := 0 + state.Members[2].Client = namespaceProbeFunc(func(ctx context.Context, request *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + response, err := clients[2].ProbeMigrationState(ctx, request) + probes++ + if probes == 2 { + switch failure { + case "ready": + response.Ready = false + case "version": + response.NamespaceAdmissionVersion = 0 + case "configuration": + response.AppliedConfigurationIndex = 8 + } + } + return response, err + }) + snapshot := func(context.Context) (NamespaceAdmissionSnapshot, error) { return state, nil } + activate := func(context.Context, kv.NamespaceAdmissionCommand) error { + t.Fatal("active group must not propose") + return nil + } + err := EnsureMigrationNamespaceAdmissionWithSnapshot(t.Context(), snapshot, activate, nil) + if failure == "ready" { + require.ErrorIs(t, err, errSplitMigrationVoterBarrierIncomplete) + require.True(t, splitJobPhaseErrorRetryable(err)) + } else { + require.ErrorIs(t, err, kv.ErrMigrationNamespaceProtocol) + require.False(t, splitJobPhaseErrorRetryable(err)) + } + require.Equal(t, 2, probes) + require.Equal(t, 1, clients[0].proposals) + }) + } +} + +func TestNamespaceAdmissionActiveGroupRevalidatesSnapshots(t *testing.T) { + for _, changeAt := range []int{2, 3} { + members, clients := namespaceRuntimeFixture(t) + require.NoError(t, EnsureMigrationNamespaceAdmission(t.Context(), clients[0], members)) + state := namespaceRuntimeSnapshot(members) + calls := 0 + snapshot := func(context.Context) (NamespaceAdmissionSnapshot, error) { + calls++ + if calls == changeAt { + state.ConfigurationIndex++ + } + return state, nil + } + activate := func(context.Context, kv.NamespaceAdmissionCommand) error { + t.Fatal("active group must not propose") + return nil + } + require.ErrorIs(t, EnsureMigrationNamespaceAdmissionWithSnapshot(t.Context(), snapshot, activate, nil), kv.ErrMigrationNamespaceProtocol) + require.Equal(t, changeAt, calls) + require.Equal(t, 1, clients[0].proposals) + } +} + +func TestNamespaceAdmissionControllerDoesNotProposeForOldLearnerOrDifferentEpoch(t *testing.T) { + for _, old := range []bool{false, true} { + members, clients := namespaceRuntimeFixture(t) + if old { + clients[2].version = 0 + } else { + clients[2].configuration = 8 + } + require.Error(t, EnsureMigrationNamespaceAdmission(t.Context(), clients[0], members)) + require.Zero(t, clients[0].proposals) + for _, client := range clients { + require.ErrorIs(t, kv.RequireMigrationNamespaceAdmission(t.Context(), client.internal.store), kv.ErrMigrationNamespaceProtocol) + } + } +} + +func TestNamespaceAdmissionControllerRejectsPartialRosterAtFSM(t *testing.T) { + members, clients := namespaceRuntimeFixture(t) + require.Error(t, EnsureMigrationNamespaceAdmission(t.Context(), clients[0], members[:2])) + for _, client := range clients { + require.ErrorIs(t, kv.RequireMigrationNamespaceAdmission(t.Context(), client.internal.store), kv.ErrMigrationNamespaceProtocol) + } +} + +func TestNamespaceAdmissionSnapshotChangePreventsControl(t *testing.T) { + members, clients := namespaceRuntimeFixture(t) + peer := NamespaceAdmissionPeer{MemberID: etcd.DeriveNodeID("n1"), Address: members[0].Address, Client: clients[0]} + calls, controls := 0, 0 + snapshot := func(context.Context) (NamespaceAdmissionSnapshot, error) { + calls++ + return NamespaceAdmissionSnapshot{ConfigurationIndex: uint64(8 + calls), Members: []NamespaceAdmissionPeer{peer}}, nil + } + activate := func(context.Context, kv.NamespaceAdmissionCommand) error { controls++; return nil } + require.Error(t, EnsureMigrationNamespaceAdmissionWithSnapshot(t.Context(), snapshot, activate, nil)) + require.Zero(t, controls) +} diff --git a/adapter/migration_namespace_pending_test.go b/adapter/migration_namespace_pending_test.go new file mode 100644 index 000000000..31ffd91b1 --- /dev/null +++ b/adapter/migration_namespace_pending_test.go @@ -0,0 +1,63 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type pendingNamespaceACKClient struct { + SplitMigrationClient + pending bool + probes int +} + +func (c *pendingNamespaceACKClient) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest, opts ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + response, err := c.SplitMigrationClient.ProbeMigrationState(ctx, req, opts...) + if err == nil && response != nil && c.pending && req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_NAMESPACE_ADMISSION { + c.probes++ + response.Ready = false + } + return response, err +} + +func TestNamespaceAdmissionPendingACKDoesNotExhaustRunnerFailureBudget(t *testing.T) { + runtime := newResplitAcceptanceRuntime(t) + server, catalog := runtime.server, runtime.server.catalog + snapshot, err := catalog.Snapshot(t.Context()) + require.NoError(t, err) + started, err := runtime.client.StartSplitMigration(t.Context(), &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: snapshot.Version, RouteId: 1, TargetGroupId: 2, SplitKey: []byte("m")}) + require.NoError(t, err) + job := loadTransferJob(t, t.Context(), catalog, started.GetJobId()) + factory := server.splitApplyMemberFactory + var pending *pendingNamespaceACKClient + server.splitApplyMemberFactory = func(ctx context.Context, groupID uint64) (SplitMigrationApplyMemberSnapshot, error) { + snapshot, err := factory(ctx, groupID) + if groupID == job.SourceGroupID { + if pending == nil { + pending = &pendingNamespaceACKClient{SplitMigrationClient: snapshot.Members[0].Client, pending: true} + } + snapshot.Members[0].Client = pending + } + return snapshot, err + } + for range maxSplitJobPhaseFailures + 1 { + err := server.RunSplitJobRunnerOnce(t.Context()) + require.ErrorIs(t, err, errSplitMigrationVoterBarrierIncomplete) + current := loadTransferJob(t, t.Context(), catalog, job.JobID) + require.Equal(t, distribution.SplitJobPhasePlanned, current.Phase) + require.Zero(t, current.WorkingSourceRouteID, "pending namespace ACK must precede geometry effects") + require.Empty(t, server.splitJobFailures) + } + require.Positive(t, pending.probes) + pending.pending = false + require.NoError(t, server.RunSplitJobRunnerOnce(t.Context())) + current := loadTransferJob(t, t.Context(), catalog, job.JobID) + require.Equal(t, distribution.SplitJobPhaseBackfill, current.Phase) + require.NotZero(t, current.SnapshotTS) +} diff --git a/adapter/migration_ownership_attestation.go b/adapter/migration_ownership_attestation.go new file mode 100644 index 000000000..4925a95ab --- /dev/null +++ b/adapter/migration_ownership_attestation.go @@ -0,0 +1,33 @@ +package adapter + +import ( + "context" + + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// AttestMigrationOwnership is a privileged root-leader peer RPC. Unlike route +// read RPCs, its producer waits for the committed catalog's applied boundary. +func (s *DistributionServer) AttestMigrationOwnership(ctx context.Context, req *pb.AttestMigrationOwnershipRequest) (*pb.AttestMigrationOwnershipResponse, error) { + if len(req.GetPriorPartition()) != 0 || len(req.GetPriorOwnershipState()) != 0 { + return s.attestMigrationPartition(ctx, req) + } + if req.GetJobId() == 0 { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "ownership attestation requires job_id")) + } + if _, err := kv.MigrationOwnershipStateFromProto(req.GetJobId(), req.GetPrior()); err != nil { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "invalid prior ownership proof")) + } + if err := s.requireReadReady(); err != nil { + return nil, err + } + proof, err := s.attestUnchangedOwnershipAtCommittedRoot(ctx, req.GetPrior()) + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.AttestMigrationOwnershipResponse{OwnershipProtocol: migrationOwnershipProtocol, Ownership: proof}, nil +} diff --git a/adapter/migration_ownership_refresh.go b/adapter/migration_ownership_refresh.go new file mode 100644 index 000000000..1e7487d98 --- /dev/null +++ b/adapter/migration_ownership_refresh.go @@ -0,0 +1,381 @@ +package adapter + +import ( + "context" + "log/slog" + "slices" + "sync" + "time" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" +) + +const ( + ownershipRefreshInterval = 5 * time.Second + ownershipRefreshBudget = 5 * time.Second + ownershipRefreshBatch = 4 + ownershipRefreshRetry = time.Minute +) + +type MigrationOwnershipAttestor interface { + AttestMigrationOwnership(context.Context, *pb.AttestMigrationOwnershipRequest, ...grpc.CallOption) (*pb.AttestMigrationOwnershipResponse, error) +} + +type MigrationOwnershipAttestorFactory func(context.Context) (MigrationOwnershipAttestor, error) + +type migrationOwnershipRefresh struct { + mu sync.Mutex + groups map[uint64]*kv.ShardGroup + root MigrationOwnershipAttestorFactory + next map[uint64]int + retry map[ownershipRefreshKey]ownershipRefreshFailure + acked map[ownershipRefreshKey]ownershipRefreshAck + groupNext int + partitionRuntime *migrationPartitionRuntime +} + +type ownershipRefreshKey struct{ group, job uint64 } + +type ownershipRefreshFailure struct { + prior *pb.MigrationOwnershipProof + mirror uint64 + after time.Time +} + +// ACK caching suppresses redundant probes only. No serving/proposal verdict +// consumes this process-local scheduling state; restart probes the proof again. +type ownershipRefreshAck struct { + proof *pb.MigrationOwnershipProof + index, term uint64 +} + +// WithMigrationOwnershipRefresh installs leader-local enumeration, not a root +// proof registry. Native proofs survive both job GC and guard retirement. +func WithMigrationOwnershipRefresh(groups map[uint64]*kv.ShardGroup, root MigrationOwnershipAttestorFactory) DistributionServerOption { + return func(s *DistributionServer) { + local := make(map[uint64]*kv.ShardGroup, len(groups)) + for id, group := range groups { + local[id] = group + } + s.ownershipRefresh = &migrationOwnershipRefresh{groups: local, root: root, next: make(map[uint64]int), retry: make(map[ownershipRefreshKey]ownershipRefreshFailure), acked: make(map[ownershipRefreshKey]ownershipRefreshAck)} + } +} + +// MigrationOwnershipRefreshConfigured reports installation only, not a serving +// verdict. Runtime capability must also check the actual read/proposal hooks. +func (s *DistributionServer) MigrationOwnershipRefreshConfigured(groupID uint64) bool { + if s == nil || groupID == 0 || s.ownershipRefresh == nil || s.ownershipRefresh.root == nil { + return false + } + group := s.ownershipRefresh.groups[groupID] + return group != nil && group.Store != nil && group.Engine != nil && s.splitMigrationGroupFactory != nil && s.splitMigrationVoterFactory != nil +} + +func (s *DistributionServer) RunMigrationOwnershipRefresh(ctx context.Context) error { + if s.ownershipRefresh == nil { + return nil + } + ticker := time.NewTicker(ownershipRefreshInterval) + defer ticker.Stop() + for { + if err := s.RunMigrationOwnershipRefreshOnce(ctx); err != nil && ctx.Err() == nil { + slog.Debug("migration ownership refresh deferred", "err", err) + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} + +func (s *DistributionServer) RunMigrationOwnershipRefreshOnce(ctx context.Context) error { + if s.ownershipRefresh == nil || s.ownershipRefresh.root == nil || s.splitJobRunnerReadinessGate == nil { + return errors.WithStack(store.ErrNotSupported) + } + if err := s.requireReadReady(); err != nil { + return err + } + if err := s.splitJobRunnerReadinessGate(ctx); err != nil { + return err + } + if s.ownershipRefresh.partitionRuntime != nil { + return s.runMigrationPartitionRefreshOnce(ctx) + } + attempt, cancel := context.WithTimeout(ctx, ownershipRefreshBudget) + defer cancel() + r := s.ownershipRefresh + r.mu.Lock() + defer r.mu.Unlock() + ids := make([]uint64, 0, len(r.groups)) + for id := range r.groups { + ids = append(ids, id) + } + slices.Sort(ids) + return s.refreshMigrationOwnershipGroups(attempt, ids) +} + +func (s *DistributionServer) refreshMigrationOwnershipGroups(ctx context.Context, ids []uint64) error { + if len(ids) == 0 { + return nil + } + r := s.ownershipRefresh + start := r.groupNext % len(ids) + var result error + for n := range len(ids) { + at := (start + n) % len(ids) + r.groupNext = (at + 1) % len(ids) + id := ids[at] + if err := s.refreshMigrationOwnershipGroup(ctx, id, r.groups[id]); err != nil { + result = errors.CombineErrors(result, err) + } + if ctx.Err() != nil { + break + } + } + return errors.WithStack(result) +} + +func migrationOwnershipRefreshLeader(id uint64, group *kv.ShardGroup) bool { + return id != 0 && group != nil && group.Engine != nil && group.Store != nil && group.Engine.State() == raftengine.StateLeader +} + +func (s *DistributionServer) refreshMigrationOwnershipGroup(ctx context.Context, id uint64, group *kv.ShardGroup) error { + if !migrationOwnershipRefreshLeader(id, group) { + return nil + } + reader, ok := group.Store.(store.MigrationOwnershipStore) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + if _, err := group.Engine.LinearizableRead(ctx); err != nil { + return errors.WithStack(err) + } + view, err := reader.MigrationServingState(ctx) + if err != nil { + return errors.WithStack(err) + } + if len(view.Ownership) == 0 { + return nil + } + if _, valid := migrationOwnershipGeneration(view.Ownership); !valid { + return errors.WithStack(store.ErrMigrationOwnership) + } + index, err := ownershipRefreshConfiguration(ctx, group) + if err != nil { + return err + } + return s.refreshMigrationOwnershipBatch(ctx, id, group, view.Ownership, index) +} + +func (s *DistributionServer) refreshMigrationOwnershipBatch(ctx context.Context, id uint64, group *kv.ShardGroup, states []store.MigrationOwnershipState, index uint64) error { + r := s.ownershipRefresh + start := r.next[id] % len(states) + var result error + for n := range min(ownershipRefreshBatch, len(states)) { + at := (start + n) % len(states) + r.next[id] = (at + 1) % len(states) + if err := s.refreshMigrationOwnershipProof(ctx, id, group, states[at], index); err != nil { + result = errors.CombineErrors(result, err) + } + if ctx.Err() != nil { + break + } + } + return errors.WithStack(result) +} + +func ownershipRefreshConfiguration(ctx context.Context, group *kv.ShardGroup) (uint64, error) { + reader, ok := group.Engine.(raftengine.AppliedConfigurationReader) + if !ok { + return 0, errors.WithStack(store.ErrNotSupported) + } + index, supported := reader.AppliedConfigurationIndex() + if !supported || index == 0 { + return 0, errors.WithStack(kv.ErrMigrationConfigurationChanged) + } + durable, err := kv.MigrationConfigurationIndex(ctx, group.Store) + if err != nil { + return 0, errors.WithStack(err) + } + if durable != index { + return 0, errors.WithStack(kv.ErrMigrationConfigurationChanged) + } + return index, nil +} + +func (s *DistributionServer) refreshMigrationOwnershipProof(ctx context.Context, id uint64, group *kv.ShardGroup, state store.MigrationOwnershipState, index uint64) error { + if state.LocalGroupID != id { + return errors.WithStack(store.ErrMigrationOwnership) + } + prior := kv.MigrationOwnershipProofFromState(state) + var mirror uint64 + if s.engine != nil { + mirror = s.engine.Version() + } + key := ownershipRefreshKey{group: id, job: state.JobID} + term := group.Engine.Status().Term + if s.ownershipRefresh.deferred(key, prior, mirror) || s.ownershipRefresh.alreadyAcknowledged(key, prior, mirror, index, term) { + return nil + } + if err := s.verifyOwnershipRefreshVoters(ctx, state.JobID, id, index, prior, false); err != nil { + return err + } + // A mirror can trigger an attempt, but cannot authorize changing any field. + if mirror <= state.CatalogVersion { + return s.acknowledgeOwnershipRefresh(ctx, key, group, index, term, prior) + } + proof, err := s.fetchOwnershipRefreshAttempt(ctx, key, prior, mirror) + if err != nil { + return err + } + return s.finishOwnershipRefresh(ctx, key, group, index, term, proof) +} + +func (s *DistributionServer) finishOwnershipRefresh(ctx context.Context, key ownershipRefreshKey, group *kv.ShardGroup, index, term uint64, proof *pb.MigrationOwnershipProof) error { + if err := group.Engine.VerifyLeader(ctx); err != nil { + return errors.WithStack(err) + } + current, err := ownershipRefreshConfiguration(ctx, group) + if err != nil { + return err + } + if current != index { + return errors.WithStack(kv.ErrMigrationConfigurationChanged) + } + if err := s.replicateOwnershipRefresh(ctx, key.job, key.group, index, proof); err != nil { + return err + } + return s.acknowledgeOwnershipRefresh(ctx, key, group, index, term, proof) +} + +func (s *DistributionServer) fetchOwnershipRefreshAttempt(ctx context.Context, key ownershipRefreshKey, prior *pb.MigrationOwnershipProof, mirror uint64) (*pb.MigrationOwnershipProof, error) { + proof, err := s.fetchOwnershipRefresh(ctx, key.job, prior) + if err != nil { + s.ownershipRefresh.retry[key] = ownershipRefreshFailure{prior: proto.CloneOf(prior), mirror: mirror, after: time.Now().Add(ownershipRefreshRetry)} + return nil, err + } + delete(s.ownershipRefresh.retry, key) + return proof, nil +} + +func (s *DistributionServer) acknowledgeOwnershipRefresh(ctx context.Context, key ownershipRefreshKey, group *kv.ShardGroup, index, term uint64, proof *pb.MigrationOwnershipProof) error { + if err := s.verifyOwnershipRefreshVoters(ctx, key.job, key.group, index, proof, true); err != nil { + return err + } + current, err := ownershipRefreshConfiguration(ctx, group) + if err != nil { + return err + } + if current != index || group.Engine.Status().Term != term { + return errors.WithStack(kv.ErrMigrationConfigurationChanged) + } + s.ownershipRefresh.acked[key] = ownershipRefreshAck{proof: proto.CloneOf(proof), index: index, term: term} + return nil +} + +func (r *migrationOwnershipRefresh) deferred(key ownershipRefreshKey, prior *pb.MigrationOwnershipProof, mirror uint64) bool { + failed, found := r.retry[key] + return found && failed.mirror == mirror && proto.Equal(failed.prior, prior) && time.Now().Before(failed.after) +} + +func (r *migrationOwnershipRefresh) alreadyAcknowledged(key ownershipRefreshKey, proof *pb.MigrationOwnershipProof, mirror, index, term uint64) bool { + ack, found := r.acked[key] + return found && mirror <= proof.GetCatalogVersion() && ack.index == index && ack.term == term && proto.Equal(ack.proof, proof) +} + +func (s *DistributionServer) fetchOwnershipRefresh(ctx context.Context, jobID uint64, prior *pb.MigrationOwnershipProof) (*pb.MigrationOwnershipProof, error) { + client, err := s.ownershipRefresh.root(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + if client == nil { + return nil, errors.WithStack(store.ErrNotSupported) + } + attempt, cancel := context.WithTimeout(ctx, defaultSplitMigrationAttemptTimeout) + defer cancel() + resp, err := client.AttestMigrationOwnership(attempt, &pb.AttestMigrationOwnershipRequest{JobId: jobID, Prior: prior}) + if err != nil { + return nil, errors.WithStack(err) + } + if resp.GetOwnershipProtocol() != migrationOwnershipProtocol || resp.GetOwnership() == nil { + return nil, errors.WithStack(store.ErrNotSupported) + } + proof := resp.GetOwnership() + expected := proto.CloneOf(prior) + expected.CatalogVersion = proof.GetCatalogVersion() + if proof.GetCatalogVersion() <= prior.GetCatalogVersion() || !proto.Equal(expected, proof) { + return nil, errors.WithStack(store.ErrMigrationOwnership) + } + return proof, nil +} + +func (s *DistributionServer) replicateOwnershipRefresh(ctx context.Context, jobID, groupID, index uint64, proof *pb.MigrationOwnershipProof) error { + if s.splitMigrationGroupFactory == nil { + return errors.WithStack(store.ErrNotSupported) + } + client, err := s.splitMigrationGroupFactory(ctx, groupID) + if err != nil { + return err + } + if client == nil { + return errors.WithStack(store.ErrNotSupported) + } + resp, err := cleanupSplitMigration(ctx, client, &pb.CleanupMigrationRequest{JobId: jobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH, Ownership: proof, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: index}) + if err != nil { + return err + } + if !resp.GetDone() { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + return nil +} + +func (s *DistributionServer) verifyOwnershipRefreshVoters(ctx context.Context, jobID, groupID, index uint64, proof *pb.MigrationOwnershipProof, applied bool) error { + if s.splitMigrationVoterFactory == nil { + return errors.WithStack(store.ErrNotSupported) + } + voters, err := s.splitMigrationVoters(ctx, groupID, nil) + if err != nil { + return err + } + for _, voter := range voters { + if err := probeOwnershipRefreshVoter(ctx, voter, jobID, index, proof, applied); err != nil { + return err + } + } + return nil +} + +func probeOwnershipRefreshVoter(ctx context.Context, voter SplitMigrationVoter, jobID, index uint64, proof *pb.MigrationOwnershipProof, applied bool) error { + kind := pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY + if applied { + kind = pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED + } + req := &pb.ProbeMigrationStateRequest{JobId: jobID, Kind: kind, RequireConfigurationIndex: true, ExpectedConfigurationIndex: index} + if applied { + req.Ownership = proof + } + attempt, cancel := context.WithTimeout(ctx, defaultSplitMigrationVoterProbeTimeout) + defer cancel() + resp, err := voter.Client.ProbeMigrationState(attempt, req) + if err != nil { + return errors.WithStack(err) + } + if !resp.GetReady() || resp.GetOwnershipProtocol() != migrationOwnershipProtocol || + resp.GetAppliedConfigurationIndex() != index || resp.GetOwnershipGeneration() != proof.GetGeneration() { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + if applied && !proto.Equal(resp.GetOwnership(), proof) { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + return nil +} diff --git a/adapter/migration_ownership_refresh_attestation.go b/adapter/migration_ownership_refresh_attestation.go new file mode 100644 index 000000000..0642b3a4b --- /dev/null +++ b/adapter/migration_ownership_refresh_attestation.go @@ -0,0 +1,187 @@ +package adapter + +import ( + "bytes" + "context" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +// loadCommittedMigrationPartitionCatalog pins a leader-issued timestamp only +// after a root applied-read barrier. Root and data leadership are independent. +func (s *DistributionServer) loadCommittedMigrationPartitionCatalog(ctx context.Context) (distribution.CatalogSnapshot, func(), error) { + if err := s.migrationPartitionRootBarrier(ctx); err != nil { + return distribution.CatalogSnapshot{}, nil, errors.WithStack(err) + } + readTS, err := s.issueMigrationPartitionReadTS(ctx) + if err != nil { + return distribution.CatalogSnapshot{}, nil, errors.WithStack(err) + } + pin := s.pinReadTS(readTS) + release := func() { releaseReadPin(pin) } + snapshot, err := s.loadCatalogSnapshotAt(ctx, readTS) + if err == nil { + err = s.verifyCatalogLeader(ctx) + } + if err != nil { + release() + return distribution.CatalogSnapshot{}, nil, errors.WithStack(err) + } + return snapshot, release, nil +} + +func (s *DistributionServer) attestUnchangedOwnershipAtCommittedRoot(ctx context.Context, prior *pb.MigrationOwnershipProof) (*pb.MigrationOwnershipProof, error) { + snapshot, release, err := s.loadCommittedMigrationPartitionCatalog(ctx) + if err != nil { + return nil, err + } + defer release() + if snapshot.Version < prior.GetCatalogVersion() { + return nil, errors.WithStack(store.ErrMigrationOwnership) + } + route, found := findRouteByID(snapshot.Routes, prior.GetRouteId()) + if !found { + return nil, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + hash, err := distribution.OwnershipPartitionDescriptorHash(route) + if err != nil { + return nil, errors.WithStack(err) + } + if route.GroupID != prior.GetOwnerGroupId() || !bytes.Equal(hash, prior.GetDescriptorHash()) { + return nil, errors.New("committed ownership descriptor changed; reacquisition required") + } + proof := proto.CloneOf(prior) + proof.CatalogVersion = snapshot.Version + return proof, nil +} + +func (s *DistributionServer) migrationPartitionRootBarrier(ctx context.Context) error { + if err := s.requireReadReady(); err != nil { + return errors.WithStack(err) + } + if err := s.verifyCatalogLeader(ctx); err != nil { + return errors.WithStack(err) + } + reader, ok := s.coordinator.(catalogAppliedReader) + if !ok || s.catalog == nil { + return errors.WithStack(store.ErrNotSupported) + } + _, err := reader.LinearizableReadForKey(ctx, distribution.CatalogVersionKey()) + return errors.WithStack(err) +} + +func (s *DistributionServer) issueMigrationPartitionReadTS(ctx context.Context) (uint64, error) { + _, allocator := kv.TimestampAllocatorThrough(s.coordinator) + if !allocator && s.coordinator.Clock() == nil { + return 0, errors.WithStack(store.ErrNotSupported) + } + issued, err := kv.NextTimestampAfterThrough(ctx, s.coordinator, s.catalog.LatestCommitTS(), "partition attestation: issue root read timestamp") + if err != nil { + return 0, errors.WithStack(err) + } + ts, err := kv.BeginReadTimestampThrough(ctx, s.coordinator, issued, "partition attestation: pinned root read") + return ts.Timestamp(), errors.WithStack(err) +} + +func (s *DistributionServer) attestMigrationPartition(ctx context.Context, req *pb.AttestMigrationOwnershipRequest) (*pb.AttestMigrationOwnershipResponse, error) { + partitions, legacy, err := decodeMigrationPartitionPriors(req) + if err != nil { + return nil, errors.WithStack(err) + } + prior, configuration := migrationPartitionAttestationPrior(partitions, legacy) + snapshot, release, err := s.loadCommittedMigrationPartitionCatalog(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + defer release() + if snapshot.Version < prior.CatalogVersion { + return nil, errors.WithStack(store.ErrMigrationOwnership) + } + raw, unchanged, err := s.migrationPartitionManifestForPriors(ctx, snapshot, prior, partitions, legacy) + if err != nil { + return nil, errors.WithStack(err) + } + control, err := kv.BuildMigrationPartitionControl(partitions, legacy, raw, unchanged, snapshot.Version, configuration) + if err != nil { + if errors.Is(err, store.ErrNotSupported) { + return nil, errors.WithStack(status.Error(codes.Unimplemented, "unchanged legacy ownership uses per-job attestation")) + } + return nil, errors.WithStack(err) + } + encoded, err := kv.EncodeMigrationPartitionControl(control) + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.AttestMigrationOwnershipResponse{OwnershipProtocol: migrationOwnershipProtocol, PartitionTransition: encoded}, nil +} + +func decodeMigrationPartitionPriors(req *pb.AttestMigrationOwnershipRequest) ([]store.MigrationOwnershipPartition, []store.MigrationOwnershipState, error) { + if req.GetPrior() != nil { + return nil, nil, errors.WithStack(kv.ErrInvalidRequest) + } + var partitions []store.MigrationOwnershipPartition + var legacy []store.MigrationOwnershipState + if len(req.GetPriorPartition()) != 0 { + if err := kv.DecodeMigrationPartitionJSON(req.GetPriorPartition(), &partitions); err != nil { + return nil, nil, errors.WithStack(err) + } + } + if len(req.GetPriorOwnershipState()) != 0 { + if err := kv.DecodeMigrationPartitionJSON(req.GetPriorOwnershipState(), &legacy); err != nil { + return nil, nil, errors.WithStack(err) + } + } + if len(req.GetPriorPartition())+len(req.GetPriorOwnershipState()) > kv.MaxMigrationPartitionControlBytes || len(partitions)+len(legacy) == 0 { + return nil, nil, errors.WithStack(kv.ErrInvalidRequest) + } + return partitions, legacy, nil +} + +func migrationPartitionAttestationPrior(partitions []store.MigrationOwnershipPartition, legacy []store.MigrationOwnershipState) (store.MigrationOwnershipPartition, uint64) { + var prior store.MigrationOwnershipPartition + var configuration uint64 + for _, p := range partitions { + if prior.CatalogVersion < p.CatalogVersion { + prior = p + } + configuration = max(configuration, p.ConfigurationIndex) + } + for _, state := range legacy { + p := kv.MigrationPartitionFromLegacy(state) + if prior.CatalogVersion < p.CatalogVersion { + prior = p + } + configuration = max(configuration, p.ConfigurationIndex) + } + return prior, configuration +} + +func (s *DistributionServer) migrationPartitionManifestAt(ctx context.Context, snapshot distribution.CatalogSnapshot, prior store.MigrationOwnershipPartition) ([]byte, *distribution.RouteDescriptor, error) { + manifest, found, err := s.catalog.OwnershipPartitionAt(ctx, prior.RouteID, prior.DescriptorHash, snapshot.ReadTS) + if err != nil { + return nil, nil, errors.WithStack(err) + } + if found { + raw, err := distribution.EncodeOwnershipPartitionManifest(manifest) + return raw, nil, errors.WithStack(err) + } + route, exists := findRouteByID(snapshot.Routes, prior.RouteID) + if !exists { + return nil, nil, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + hash, hashErr := distribution.OwnershipPartitionDescriptorHash(route) + if hashErr != nil { + return nil, nil, errors.WithStack(hashErr) + } + if !bytes.Equal(hash, prior.DescriptorHash) { + return nil, nil, errors.WithStack(errMigrationPartitionDescriptorChanged) + } + return nil, &route, nil +} diff --git a/adapter/migration_ownership_refresh_candidates_test.go b/adapter/migration_ownership_refresh_candidates_test.go new file mode 100644 index 000000000..02020ad58 --- /dev/null +++ b/adapter/migration_ownership_refresh_candidates_test.go @@ -0,0 +1,53 @@ +package adapter + +import ( + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestPartitionCandidatesGuardedNewJobIsNotDuplicated(t *testing.T) { + h := newPartitionRefreshHarness(t, 7) + children := h.split(t, h.parent, 2, 3, "m") + require.NoError(t, h.consumer.reconcileMigrationPartitions(t.Context(), 7)) + right := children[1] + hash, err := distribution.OwnershipPartitionDescriptorHash(right) + require.NoError(t, err) + guard := store.TargetStagedReadinessState{JobID: 12, MigrationJobID: 12, Armed: true, TrackWrites: true, + RouteStart: right.Start, RouteEnd: right.End, ExpectedCutoverVersion: 3, MinWriteTSExclusive: 100, RetentionPinTS: 1} + guardWriter, ok := h.stores[0].(store.MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.NoError(t, guardWriter.ApplyTargetStagedReadinessAt(t.Context(), guard, 100)) + writer, ok := h.stores[0].(store.MigrationOwnershipStore) + require.True(t, ok) + state := store.MigrationOwnershipState{JobID: 12, Generation: "0123456789abcdef0123456789abcdef", LocalGroupID: 7, OwnerGroupID: 7, + CatalogVersion: 3, ConfigurationIndex: 9, RouteID: right.RouteID, ParentRouteID: right.ParentRouteID, RouteStart: right.Start, RouteEnd: right.End, + DescriptorHash: hash, Role: store.MigrationOwner, MinWriteTSExclusive: 100, WriteFloorKnown: true, SourceTrackerVersion: 3} + require.NoError(t, writer.AcquireMigrationOwnership(t.Context(), state, 101)) + view, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Len(t, view.Ownership, 2, "the immutable per-job ledger remains complete") + frontier := h.frontier(t, 0) + require.Len(t, frontier.Partitions, 2) + candidates := migrationPartitionCandidates(view, frontier) + require.Len(t, candidates, 2, "a guarded job receipt adopts job 12 even without a LegacyJobID Before reference") + for _, candidate := range candidates { + require.Nil(t, candidate.legacy) + } + invalid := frontier + invalid.Receipts = append([]store.MigrationOwnershipPartitionReceipt(nil), frontier.Receipts...) + for i := range invalid.Receipts { + if invalid.Receipts[i].Transition.Operation == store.MigrationOwnershipPartitionJobOperation { + invalid.Receipts[i].ReceiptID = []byte("corrupt") + } + } + require.Len(t, migrationPartitionCandidates(view, invalid), 3, "an invalid internal receipt cannot adopt a legacy job") + unknown := view.Ownership[1] + unknown.DescriptorHash = nil + view.Ownership = append(view.Ownership, unknown) + candidates = migrationPartitionCandidates(view, frontier) + require.Len(t, candidates, 3, "unknown legacy evidence must remain a refusal barrier") + require.NotNil(t, candidates[2].legacy) +} diff --git a/adapter/migration_ownership_refresh_legacy.go b/adapter/migration_ownership_refresh_legacy.go new file mode 100644 index 000000000..d013082fe --- /dev/null +++ b/adapter/migration_ownership_refresh_legacy.go @@ -0,0 +1,106 @@ +package adapter + +import ( + "bytes" + "context" + + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +// Legacy records with unchanged geometry keep their immutable per-job ledger. +// This path never manufactures a partition ID, clips a proof or trusts a cache. +func (s *DistributionServer) reconcileLegacyOwnershipPartition(ctx context.Context, id uint64, g *kv.ShardGroup, index uint64, members []SplitMigrationVoter, before store.MigrationOwnershipState) error { + proof, err := s.fetchUnchangedLegacyPartition(ctx, before) + if err != nil { + return err + } + term := g.Engine.Status().Term + if err := verifyPartitionConsumerEpoch(ctx, g, index, term); err != nil { + return err + } + if proof.GetCatalogVersion() != before.CatalogVersion || before.ConfigurationIndex != index { + if err := s.replicateOwnershipRefresh(ctx, before.JobID, id, index, proof); err != nil { + return err + } + } + for _, member := range members { + if err := probeOwnershipRefreshVoter(ctx, member, before.JobID, index, proof, true); err != nil { + return err + } + } + if err := verifyLegacyPartitionLocalRecord(ctx, g.Store, before, proof, index); err != nil { + return err + } + if err := verifyPartitionConsumerEpoch(ctx, g, index, term); err != nil { + return err + } + return s.verifyMigrationPartitionMemberSnapshot(ctx, id, members) +} + +func (s *DistributionServer) fetchUnchangedLegacyPartition(ctx context.Context, before store.MigrationOwnershipState) (*pb.MigrationOwnershipProof, error) { + if _, err := store.MigrationOwnershipLegacyReferenceFor(before); err != nil { + return nil, errors.WithStack(err) + } + client, err := s.ownershipRefresh.root(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + if client == nil { + return nil, errors.WithStack(store.ErrNotSupported) + } + prior := kv.MigrationOwnershipProofFromState(before) + resp, err := client.AttestMigrationOwnership(ctx, &pb.AttestMigrationOwnershipRequest{JobId: before.JobID, Prior: prior}) + if err != nil { + return nil, errors.WithStack(err) + } + if resp.GetOwnershipProtocol() != migrationOwnershipProtocol || resp.GetOwnership() == nil { + return nil, errors.WithStack(store.ErrNotSupported) + } + proof := resp.GetOwnership() + expected := proto.CloneOf(prior) + expected.CatalogVersion = proof.GetCatalogVersion() + if proof.GetCatalogVersion() < prior.GetCatalogVersion() || !proto.Equal(expected, proof) { + return nil, errors.WithStack(store.ErrMigrationOwnership) + } + return proof, nil +} + +func verifyLegacyPartitionLocalRecord(ctx context.Context, st store.MVCCStore, before store.MigrationOwnershipState, proof *pb.MigrationOwnershipProof, index uint64) error { + reader, ok := st.(store.MigrationOwnershipStore) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + view, err := reader.MigrationServingState(ctx) + if err != nil { + return errors.WithStack(err) + } + for _, actual := range view.Ownership { + if actual.JobID != before.JobID { + continue + } + if !legacyPartitionRecordMatches(actual, proof, index) { + return errors.WithStack(store.ErrMigrationOwnership) + } + x, err := kv.EncodeMigrationPartitionJSON(before.Receipts) + if err != nil { + return errors.WithStack(err) + } + y, err := kv.EncodeMigrationPartitionJSON(actual.Receipts) + if err != nil { + return errors.WithStack(err) + } + if !bytes.Equal(x, y) { + return errors.WithStack(store.ErrMigrationOwnership) + } + return nil + } + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) +} + +func legacyPartitionRecordMatches(actual store.MigrationOwnershipState, proof *pb.MigrationOwnershipProof, index uint64) bool { + return store.MigrationOwnershipEvidenceValid(actual) && actual.ConfigurationIndex == index && proto.Equal(kv.MigrationOwnershipProofFromState(actual), proof) +} diff --git a/adapter/migration_ownership_refresh_negative.go b/adapter/migration_ownership_refresh_negative.go new file mode 100644 index 000000000..e58932cd1 --- /dev/null +++ b/adapter/migration_ownership_refresh_negative.go @@ -0,0 +1,63 @@ +package adapter + +import ( + "context" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ErrMigrationNegativePartitionDeferred never means a new receipt or serving +// authority. Foreground callers remain pending; background retains the denial. +var ErrMigrationNegativePartitionDeferred = errors.New("negative partition retained; no committed geometry to install") + +var errMigrationPartitionDescriptorChanged = errors.New("committed partition descriptor changed; no manifest") + +func (s *DistributionServer) migrationPartitionManifestForPriors(ctx context.Context, snapshot distribution.CatalogSnapshot, prior store.MigrationOwnershipPartition, partitions []store.MigrationOwnershipPartition, legacy []store.MigrationOwnershipState) ([]byte, *distribution.RouteDescriptor, error) { + raw, unchanged, exactErr := s.migrationPartitionManifestAt(ctx, snapshot, prior) + if exactErr == nil { + return raw, unchanged, nil + } + if !negativePartitionLookupMissing(exactErr) || !kv.MigrationPartitionNegativePriors(partitions, legacy) { + return nil, nil, exactErr + } + manifest, found, err := s.catalog.OwnershipPartitionNegativeAt(ctx, prior.RouteID, prior.ParentRouteID, prior.RouteStart, prior.RouteEnd, snapshot.ReadTS) + if err != nil { + return nil, nil, errors.WithStack(err) + } + if found { + if !kv.MigrationPartitionNegativePriorsMatchParent(partitions, legacy, manifest.Parent) { + return nil, nil, errors.WithStack(store.ErrMigrationOwnership) + } + encoded, err := distribution.EncodeOwnershipPartitionManifest(manifest) + return encoded, nil, errors.WithStack(err) + } + return negativePartitionGeometryUnchanged(snapshot, prior, partitions, legacy, exactErr) +} + +func negativePartitionLookupMissing(err error) bool { + return errors.Is(err, distribution.ErrMigrationSourceRouteChanged) || errors.Is(err, errMigrationPartitionDescriptorChanged) +} + +func negativePartitionGeometryUnchanged(snapshot distribution.CatalogSnapshot, prior store.MigrationOwnershipPartition, partitions []store.MigrationOwnershipPartition, legacy []store.MigrationOwnershipState, exactErr error) ([]byte, *distribution.RouteDescriptor, error) { + route, found := findRouteByID(snapshot.Routes, prior.RouteID) + if !found || !kv.MigrationPartitionNegativePriorsMatchParent(partitions, legacy, route) { + return nil, nil, exactErr + } + return nil, nil, errors.WithStack(status.Error(codes.NotFound, "negative ownership geometry unchanged; no committed partition manifest")) +} + +func deferredMigrationNegativePartition(err error, partitions []store.MigrationOwnershipPartition, legacy []store.MigrationOwnershipState) bool { + return status.Code(err) == codes.NotFound && kv.MigrationPartitionNegativePriors(partitions, legacy) +} + +func migrationPartitionBackgroundResult(err error, background bool) error { + if background && errors.Is(err, ErrMigrationNegativePartitionDeferred) { + return nil + } + return err +} diff --git a/adapter/migration_ownership_refresh_negative_test.go b/adapter/migration_ownership_refresh_negative_test.go new file mode 100644 index 000000000..7090db601 --- /dev/null +++ b/adapter/migration_ownership_refresh_negative_test.go @@ -0,0 +1,113 @@ +package adapter + +import ( + "bytes" + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestPartitionNegativeDeferredUntilCommittedGeometry(t *testing.T) { + h := newPartitionRefreshHarness(t, 9) + changed := h.parent + changed.GroupID, changed.MinWriteTSExclusive = 8, 200 + _, err := h.root.catalog.Save(t.Context(), 1, []distribution.RouteDescriptor{changed}) + require.NoError(t, err) + require.ErrorIs(t, h.consumer.reconcileMigrationPartitions(t.Context(), 9), ErrMigrationNegativePartitionDeferred) + for range 12 { + before := h.rootClient.calls + require.NoError(t, h.consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + require.LessOrEqual(t, h.rootClient.calls-before, ownershipRefreshBatch) + } + require.Empty(t, partitionOnlyCleanups(h.clients[0])) + require.Empty(t, h.frontier(t, 0).Partitions) + h.split(t, changed, 2, 3, "m") + require.NoError(t, h.consumer.reconcileMigrationPartitions(t.Context(), 9)) + state := h.frontier(t, 0) + require.Len(t, state.Partitions, 2) + for _, p := range state.Partitions { + require.Equal(t, store.MigrationNonOwner, p.Role) + require.EqualValues(t, 7, p.OwnerGroupID) + require.EqualValues(t, 100, p.MinWriteTSExclusive) + require.EqualValues(t, 11, p.MigrationJobID) + } + for range 12 { + before := h.rootClient.calls + require.NoError(t, h.consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + require.LessOrEqual(t, h.rootClient.calls-before, ownershipRefreshBatch) + } + for i := range h.stores { + require.Equal(t, state, h.frontier(t, i)) + } +} + +func TestPartitionNegativeCannotCatchUpToLocalOwner(t *testing.T) { + h := newPartitionRefreshHarness(t, 9) + changed := h.parent + changed.GroupID, changed.MinWriteTSExclusive = 9, 200 + _, err := h.root.catalog.Save(t.Context(), 1, []distribution.RouteDescriptor{changed}) + require.NoError(t, err) + err = h.consumer.reconcileMigrationPartitions(t.Context(), 9) + require.Error(t, err) + require.NotErrorIs(t, err, ErrMigrationNegativePartitionDeferred) + h.split(t, changed, 2, 3, "m") + require.Error(t, h.consumer.reconcileMigrationPartitions(t.Context(), 9)) + require.Empty(t, partitionOnlyCleanups(h.clients[0])) +} + +func TestPartitionNegativeFallbackDoesNotMaskLookupFailure(t *testing.T) { + for _, failure := range []error{errors.New("storage I/O failed"), distribution.ErrOwnershipPartitionManifest} { + t.Run(failure.Error(), func(t *testing.T) { + h := newPartitionRefreshHarness(t, 9) + h.split(t, h.parent, 2, 3, "m") + hash, err := distribution.OwnershipPartitionDescriptorHash(h.parent) + require.NoError(t, err) + key, err := distribution.CatalogOwnershipPartitionKey(h.parent.RouteID, hash) + require.NoError(t, err) + h.root.catalog = distribution.NewCatalogStore(&partitionNegativeReadFailure{MVCCStore: h.rootStore, key: key, failure: failure}) + require.ErrorIs(t, h.consumer.reconcileMigrationPartitions(t.Context(), 9), failure) + require.Empty(t, partitionOnlyCleanups(h.clients[0]), "a discoverable manifest must not mask exact-lookup failure") + }) + } +} + +type partitionNegativeReadFailure struct { + store.MVCCStore + key []byte + failure error +} + +func (s *partitionNegativeReadFailure) GetAt(ctx context.Context, key []byte, ts uint64) ([]byte, error) { + if bytes.Equal(key, s.key) { + return nil, s.failure + } + return s.MVCCStore.GetAt(ctx, key, ts) +} + +func TestPartitionNegativeMoveBackRepeatedRefresh(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + r.migrate(t, 1, 2, "m") + r.migrate(t, r.ownership(t, 2)[0].RouteID, 3, "t") + require.NoError(t, r.server.RunMigrationOwnershipRefreshOnce(t.Context())) + third := r.migrate(t, r.ownership(t, 3)[0].RouteID, 1, "x") + rootCalls := 0 + r.server.ownershipRefresh.root = func(context.Context) (MigrationOwnershipAttestor, error) { + rootCalls++ + return r.client, nil + } + for range 16 { + before := rootCalls + require.NoError(t, r.server.RunMigrationOwnershipRefreshOnce(t.Context())) + require.LessOrEqual(t, rootCalls-before, ownershipRefreshBatch) + require.Error(t, r.put(t, 2, "y", "retired-two", third.FenceTS+100)) + require.Error(t, r.put(t, 3, "y", "retired-three", third.FenceTS+100)) + require.ErrorIs(t, r.put(t, 1, "y", "old-floor", third.FenceTS), kv.ErrRouteWriteTimestampTooLow) + } + require.NoError(t, r.put(t, 1, "y", "returned-owner", third.FenceTS+100)) + r.read(t, "y", "returned-owner") +} diff --git a/adapter/migration_ownership_refresh_partition.go b/adapter/migration_ownership_refresh_partition.go new file mode 100644 index 000000000..6e729df62 --- /dev/null +++ b/adapter/migration_ownership_refresh_partition.go @@ -0,0 +1,602 @@ +package adapter + +import ( + "bytes" + "context" + "slices" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type migrationPartitionRuntime struct { + reads *kv.ShardStore + members SplitMigrationVoterFactory + next map[uint64]int +} + +// WithMigrationPartitionRuntime requires the actual public read/store pairing +// and apply-member discovery (voters AND learners). Install after refresh. +// No boolean activation flag can stand in for these installed integrations. +func WithMigrationPartitionRuntime(reads *kv.ShardStore, members SplitMigrationVoterFactory) DistributionServerOption { + return func(s *DistributionServer) { + if s.ownershipRefresh == nil { + return + } + s.ownershipRefresh.partitionRuntime = &migrationPartitionRuntime{reads: reads, members: members, next: make(map[uint64]int)} + } +} + +func (s *DistributionServer) MigrationPartitionRuntimeConfigured(groupID uint64) bool { + if !s.MigrationOwnershipRefreshConfigured(groupID) { + return false + } + r := s.ownershipRefresh.partitionRuntime + if r == nil || r.reads == nil || r.members == nil || !r.reads.MigrationPartitionReadAdmissionReady(groupID) { + return false + } + g := s.ownershipRefresh.groups[groupID] + if !kv.MigrationPartitionRequestAdmissionReady(g.Txn) { + return false + } + _, storeOK := g.Store.(store.MigrationOwnershipPartitionStore) + _, engineOK := g.Engine.(raftengine.AppliedConfigurationReader) + return storeOK && engineOK +} + +// reconcileMigrationPartitions is the committed-root-ACK -> SOURCE handoff. +// It never assumes this server is root leader. Success includes exact receipt +// ACKs from every current apply member; a changed epoch requires a fresh retry. +func (s *DistributionServer) reconcileMigrationPartitions(ctx context.Context, groupID uint64) error { + if !s.MigrationPartitionRuntimeConfigured(groupID) { + return errors.WithStack(store.ErrNotSupported) + } + attempt, cancel := context.WithTimeout(ctx, ownershipRefreshBudget) + defer cancel() + r := s.ownershipRefresh + if !r.mu.TryLock() { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + defer r.mu.Unlock() + reader, ok := r.groups[groupID].Store.(store.MigrationOwnershipPartitionStore) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + // Offline groups may need several retained manifests. Do not ACK a single + // intermediate generation as reconciled with the current root geometry. + for attempt.Err() == nil { + before, err := reader.MigrationOwnershipPartitions(attempt) + if err != nil { + return errors.WithStack(err) + } + if _, err := s.reconcileMigrationPartitionGroup(attempt, groupID, 0); err != nil { + return errors.WithStack(err) + } + after, err := reader.MigrationOwnershipPartitions(attempt) + if err != nil { + return errors.WithStack(err) + } + same, err := sameMigrationPartitionFrontier(before, after) + if err != nil { + return err + } + if same { + return nil + } + } + return errors.WithStack(attempt.Err()) +} + +func sameMigrationPartitionFrontier(before, after store.MigrationOwnershipPartitionState) (bool, error) { + x, err := kv.EncodeMigrationPartitionJSON(before.Partitions) + if err != nil { + return false, errors.WithStack(err) + } + y, err := kv.EncodeMigrationPartitionJSON(after.Partitions) + return bytes.Equal(x, y), errors.WithStack(err) +} + +// Background work has its own frontier cursor, independent of immutable job +// enumeration and retry/ACK caches. Job GC cannot remove this obligation. +func (s *DistributionServer) runMigrationPartitionRefreshOnce(ctx context.Context) error { + attempt, cancel := context.WithTimeout(ctx, ownershipRefreshBudget) + defer cancel() + r := s.ownershipRefresh + if !r.mu.TryLock() { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + defer r.mu.Unlock() + ids := make([]uint64, 0, len(r.groups)) + for id := range r.groups { + ids = append(ids, id) + } + slices.Sort(ids) + if len(ids) == 0 { + return nil + } + start, remaining := r.groupNext%len(ids), ownershipRefreshBatch + var result error + for n := 0; n < len(ids) && remaining > 0 && attempt.Err() == nil; n++ { + at := (start + n) % len(ids) + r.groupNext = (at + 1) % len(ids) + id := ids[at] + if !migrationOwnershipRefreshLeader(id, r.groups[id]) { + continue + } + if !s.MigrationPartitionRuntimeConfigured(id) { + result = errors.CombineErrors(result, store.ErrNotSupported) + continue + } + used, err := s.reconcileMigrationPartitionGroup(attempt, id, remaining) + remaining -= used + result = errors.CombineErrors(result, err) + } + return errors.WithStack(errors.CombineErrors(result, attempt.Err())) +} + +type migrationPartitionPrior struct { + partition *store.MigrationOwnershipPartition + legacy *store.MigrationOwnershipState +} + +func (p migrationPartitionPrior) geometry() store.MigrationOwnershipPartition { + if p.partition != nil { + return *p.partition + } + return kv.MigrationPartitionFromLegacy(*p.legacy) +} + +func migrationPartitionCandidates(view store.MigrationServingState, state store.MigrationOwnershipPartitionState) []migrationPartitionPrior { + var result []migrationPartitionPrior + consumed := make(map[uint64]bool) + for _, r := range state.Receipts { + for _, ref := range r.Transition.Before { + if ref.LegacyJobID != 0 { + consumed[ref.LegacyJobID] = true + } + } + consumeMigrationPartitionJobOperation(state.Generation, r, consumed) + } + for i := range state.Partitions { + result = append(result, migrationPartitionPrior{partition: &state.Partitions[i]}) + } + for i := range view.Ownership { + if !consumed[view.Ownership[i].JobID] || !store.MigrationOwnershipEvidenceValid(view.Ownership[i]) { + result = append(result, migrationPartitionPrior{legacy: &view.Ownership[i]}) + } + } + return result +} + +func consumeMigrationPartitionJobOperation(generation string, receipt store.MigrationOwnershipPartitionReceipt, consumed map[uint64]bool) { + if receipt.Transition.Operation != store.MigrationOwnershipPartitionJobOperation { + return + } + // Validate historical outputs against their own immutable receipt, not the + // current frontier: a later split may already have replaced these children. + historical := store.MigrationOwnershipPartitionState{Generation: generation, Partitions: receipt.Transition.After, Receipts: []store.MigrationOwnershipPartitionReceipt{receipt}} + for _, p := range receipt.Transition.After { + if store.MigrationOwnershipPartitionEvidenceValid(historical, p) { + consumed[p.MigrationJobID] = true + } + } +} + +func partitionRangesOverlap(a, b store.MigrationOwnershipPartition) bool { + return (len(a.RouteEnd) == 0 || bytes.Compare(b.RouteStart, a.RouteEnd) < 0) && (len(b.RouteEnd) == 0 || bytes.Compare(a.RouteStart, b.RouteEnd) < 0) +} + +func migrationPartitionPriorClosure(candidates []migrationPartitionPrior, at int) ([]store.MigrationOwnershipPartition, []store.MigrationOwnershipState, []int) { + selected := map[int]bool{at: true} + for changed := true; changed; { + changed = false + for i, p := range candidates { + if selected[i] { + continue + } + for j := range selected { + if partitionRangesOverlap(p.geometry(), candidates[j].geometry()) { + selected[i], changed = true, true + break + } + } + } + } + var partitions []store.MigrationOwnershipPartition + var legacy []store.MigrationOwnershipState + var indices []int + for i, p := range candidates { + if !selected[i] { + continue + } + indices = append(indices, i) + if p.partition != nil { + partitions = append(partitions, *p.partition) + } else { + legacy = append(legacy, *p.legacy) + } + } + return partitions, legacy, indices +} + +func (s *DistributionServer) reconcileMigrationPartitionGroup(ctx context.Context, id uint64, limit int) (int, error) { + g := s.ownershipRefresh.groups[id] + if !migrationPartitionLocalGroupPresent(id, g) { + return 0, errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + view, state, index, err := readMigrationPartitionGroup(ctx, g) + if err != nil { + return 0, err + } + candidates := migrationPartitionCandidates(view, state) + if len(candidates) == 0 { + return 0, s.verifyEmptyMigrationPartitionGroup(ctx, id, index) + } + r := s.ownershipRefresh.partitionRuntime + start := r.next[id] % len(candidates) + background := limit > 0 + if limit == 0 { + limit = len(candidates) + } + visited := make(map[int]bool) + used := 0 + var result error + for n := 0; n < len(candidates) && used < limit; n++ { + at := (start + n) % len(candidates) + r.next[id] = (at + 1) % len(candidates) + if visited[at] { + continue + } + partitions, legacy, indices := migrationPartitionPriorClosure(candidates, at) + for _, i := range indices { + visited[i] = true + } + used++ + err := s.reconcileMigrationPartitionPrior(ctx, id, g, index, state, partitions, legacy) + result = errors.CombineErrors(result, migrationPartitionBackgroundResult(err, background)) + if ctx.Err() != nil { + break + } + } + return used, errors.WithStack(errors.CombineErrors(result, ctx.Err())) +} + +func migrationPartitionLocalGroupPresent(id uint64, g *kv.ShardGroup) bool { + return id != 0 && g != nil && g.Store != nil && g.Engine != nil +} + +func readMigrationPartitionGroup(ctx context.Context, g *kv.ShardGroup) (store.MigrationServingState, store.MigrationOwnershipPartitionState, uint64, error) { + var view store.MigrationServingState + var state store.MigrationOwnershipPartitionState + if g.Engine.State() == raftengine.StateLeader { + if _, err := g.Engine.LinearizableRead(ctx); err != nil { + return view, state, 0, errors.WithStack(err) + } + } + index, err := ownershipRefreshConfiguration(ctx, g) + if err != nil { + return view, state, 0, err + } + reader, ok := g.Store.(store.MigrationOwnershipStore) + if !ok { + return view, state, 0, errors.WithStack(store.ErrNotSupported) + } + view, err = reader.MigrationServingState(ctx) + if err != nil { + return view, state, 0, errors.WithStack(err) + } + partitionReader, ok := g.Store.(store.MigrationOwnershipPartitionStore) + if !ok { + return view, state, 0, errors.WithStack(store.ErrNotSupported) + } + state, err = partitionReader.MigrationOwnershipPartitions(ctx) + return view, state, index, errors.WithStack(err) +} + +func (s *DistributionServer) partitionApplyMembers(ctx context.Context, id uint64) ([]SplitMigrationVoter, error) { + r := s.ownershipRefresh.partitionRuntime + if r == nil || r.members == nil { + return nil, errors.WithStack(store.ErrNotSupported) + } + members, err := r.members(ctx, id) + if err != nil { + return nil, errors.WithStack(err) + } + members = slices.Clone(members) + if len(members) == 0 { + return nil, errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + if err := validateSplitMigrationVoters(members); err != nil { + return nil, errors.WithStack(err) + } + return members, nil +} + +func (s *DistributionServer) reconcileMigrationPartitionPrior(ctx context.Context, id uint64, g *kv.ShardGroup, index uint64, state store.MigrationOwnershipPartitionState, partitions []store.MigrationOwnershipPartition, legacy []store.MigrationOwnershipState) error { + var prior store.MigrationOwnershipPartition + if len(partitions) > 0 { + prior = partitions[0] + } else { + prior = kv.MigrationPartitionFromLegacy(legacy[0]) + } + if prior.LocalGroupID != id { + return errors.WithStack(store.ErrMigrationOwnership) + } + members, err := s.partitionMembersWithNamespace(ctx, id) + if err != nil { + return errors.WithStack(err) + } + if err := verifyPartitionApplyCapabilities(ctx, members, prior.MigrationJobID, index, prior.Generation); err != nil { + return err + } + c, err := s.fetchMigrationPartitionAttestation(ctx, partitions, legacy, index) + if err != nil { + return s.reconcileMigrationPartitionFailure(ctx, id, g, index, members, legacy, err, partitions) + } + term := g.Engine.Status().Term + transition := c.Transition + unchanged := migrationPartitionControlUnchanged(c, len(partitions), len(legacy), prior, index) + if unchanged { + transition, err = currentMigrationPartitionReceipt(state, prior.ReceiptID) + if err != nil { + return err + } + } else { + if err := verifyPartitionConsumerEpoch(ctx, g, index, term); err != nil { + return errors.WithStack(err) + } + if err := s.replicateMigrationPartition(ctx, id, index, prior.MigrationJobID, c); err != nil { + return err + } + } + return s.finishMigrationPartitionAcknowledgement(ctx, id, g, index, term, members, prior.MigrationJobID, transition) +} + +func (s *DistributionServer) reconcileMigrationPartitionFailure(ctx context.Context, id uint64, g *kv.ShardGroup, index uint64, members []SplitMigrationVoter, legacy []store.MigrationOwnershipState, err error, partitions []store.MigrationOwnershipPartition) error { + if deferredMigrationNegativePartition(err, partitions, legacy) { + return errors.WithStack(ErrMigrationNegativePartitionDeferred) + } + if unchangedLegacyPartitionFallback(err, len(partitions), len(legacy)) { + return s.reconcileLegacyOwnershipPartition(ctx, id, g, index, members, legacy[0]) + } + return errors.WithStack(err) +} + +func unchangedLegacyPartitionFallback(err error, partitions, legacy int) bool { + return partitions == 0 && legacy == 1 && status.Code(err) == codes.Unimplemented +} + +func migrationPartitionControlUnchanged(c kv.MigrationPartitionControl, partitions, legacy int, prior store.MigrationOwnershipPartition, index uint64) bool { + return c.UnchangedDescriptor != nil && partitions == 1 && legacy == 0 && prior.CatalogVersion == c.Transition.CatalogVersion && prior.ConfigurationIndex == index +} + +func (s *DistributionServer) finishMigrationPartitionAcknowledgement(ctx context.Context, id uint64, g *kv.ShardGroup, index, term uint64, members []SplitMigrationVoter, job uint64, transition store.MigrationOwnershipPartitionTransition) error { + if err := s.acknowledgeMigrationPartition(ctx, members, index, job, transition); err != nil { + return errors.WithStack(err) + } + if err := verifyLocalMigrationPartitionReceipt(ctx, g.Store, transition); err != nil { + return errors.WithStack(err) + } + if err := verifyPartitionConsumerEpoch(ctx, g, index, term); err != nil { + return errors.WithStack(err) + } + return s.verifyMigrationPartitionMemberSnapshot(ctx, id, members) +} + +func (s *DistributionServer) verifyMigrationPartitionMemberSnapshot(ctx context.Context, id uint64, members []SplitMigrationVoter) error { + current, err := s.partitionApplyMembers(ctx, id) + if err != nil { + return errors.WithStack(err) + } + if len(members) != len(current) { + return errors.WithStack(kv.ErrMigrationConfigurationChanged) + } + for i := range members { + if members[i].ID != current[i].ID || members[i].Address != current[i].Address { + return errors.WithStack(kv.ErrMigrationConfigurationChanged) + } + } + return nil +} + +func currentMigrationPartitionReceipt(state store.MigrationOwnershipPartitionState, id []byte) (store.MigrationOwnershipPartitionTransition, error) { + for _, receipt := range state.Receipts { + if bytes.Equal(receipt.ReceiptID, id) { + return receipt.Transition, nil + } + } + return store.MigrationOwnershipPartitionTransition{}, errors.WithStack(store.ErrMigrationOwnership) +} + +func verifyPartitionApplyCapabilities(ctx context.Context, members []SplitMigrationVoter, job, index uint64, generation string) error { + for _, member := range members { + resp, err := probeSplitOwnershipCapability(ctx, member, job) + if err != nil { + return err + } + if resp.GetOwnershipPartitionProtocol() != MigrationOwnershipPartitionProtocolVersion || resp.GetAppliedConfigurationIndex() != index || resp.GetOwnershipGeneration() != generation { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + } + return nil +} + +func (s *DistributionServer) replicateMigrationPartition(ctx context.Context, id, index, job uint64, c kv.MigrationPartitionControl) error { + client, err := s.splitMigrationGroupFactory(ctx, id) + if err != nil { + return err + } + if client == nil { + return errors.WithStack(store.ErrNotSupported) + } + raw, err := kv.EncodeMigrationPartitionControl(c) + if err != nil { + return errors.WithStack(err) + } + resp, err := cleanupSplitMigration(ctx, client, &pb.CleanupMigrationRequest{JobId: job, Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: index, PartitionTransition: raw}) + if err != nil { + return err + } + if !resp.GetDone() { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + return nil +} + +func verifyLocalMigrationPartitionReceipt(ctx context.Context, st store.MVCCStore, t store.MigrationOwnershipPartitionTransition) error { + reader, ok := st.(store.MigrationOwnershipPartitionStore) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + state, err := reader.MigrationOwnershipPartitions(ctx) + if err != nil { + return errors.WithStack(err) + } + id, err := store.MigrationOwnershipPartitionReceiptIDFor(t) + if err != nil { + return errors.WithStack(err) + } + for _, receipt := range state.Receipts { + if receipt.AppliedIndex != 0 && bytes.Equal(receipt.ReceiptID, id) { + return nil + } + } + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) +} + +func verifyPartitionConsumerEpoch(ctx context.Context, g *kv.ShardGroup, index, term uint64) error { + if g.Engine.State() == raftengine.StateLeader { + if err := g.Engine.VerifyLeader(ctx); err != nil { + return errors.WithStack(err) + } + } + actual, err := ownershipRefreshConfiguration(ctx, g) + if err != nil { + return errors.WithStack(err) + } + if actual != index || g.Engine.Status().Term != term { + return errors.WithStack(kv.ErrMigrationConfigurationChanged) + } + return nil +} + +func (s *DistributionServer) verifyEmptyMigrationPartitionGroup(ctx context.Context, id, index uint64) error { + members, err := s.partitionMembersWithNamespace(ctx, id) + if err != nil { + return errors.WithStack(err) + } + for _, member := range members { + resp, err := probeSplitOwnershipCapability(ctx, member, 1) + if err != nil { + return errors.WithStack(err) + } + if resp.GetOwnershipPartitionProtocol() != MigrationOwnershipPartitionProtocolVersion || resp.GetAppliedConfigurationIndex() != index || resp.GetOwnershipGeneration() != "" { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + } + return nil +} + +func (s *DistributionServer) partitionMembersWithNamespace(ctx context.Context, id uint64) ([]SplitMigrationVoter, error) { + members, err := s.partitionApplyMembers(ctx, id) + if err != nil { + return nil, err + } + client, err := s.splitMigrationGroupFactory(ctx, id) + if err != nil { + return nil, err + } + if err := EnsureMigrationNamespaceAdmission(ctx, client, members); err != nil { + return nil, err + } + return members, nil +} + +func (s *DistributionServer) fetchMigrationPartitionAttestation(ctx context.Context, partitions []store.MigrationOwnershipPartition, legacy []store.MigrationOwnershipState, index uint64) (kv.MigrationPartitionControl, error) { + var zero kv.MigrationPartitionControl + client, err := s.ownershipRefresh.root(ctx) + if err != nil { + return zero, errors.WithStack(err) + } + if client == nil { + return zero, errors.WithStack(store.ErrNotSupported) + } + req, err := migrationPartitionAttestationRequest(partitions, legacy) + if err != nil { + return zero, err + } + resp, err := client.AttestMigrationOwnership(ctx, req) + if err != nil { + return zero, errors.WithStack(err) + } + if resp.GetOwnershipProtocol() != migrationOwnershipProtocol || resp.GetOwnership() != nil { + return zero, errors.WithStack(store.ErrNotSupported) + } + c, err := kv.DecodeMigrationPartitionControl(resp.GetPartitionTransition()) + if err != nil { + return zero, errors.WithStack(err) + } + if err := matchMigrationPartitionAttestationPriors(req, c); err != nil { + return zero, err + } + result, err := kv.BuildMigrationPartitionControl(partitions, legacy, c.Manifest, c.UnchangedDescriptor, c.Transition.CatalogVersion, index) + return result, errors.WithStack(err) +} + +func migrationPartitionAttestationRequest(partitions []store.MigrationOwnershipPartition, legacy []store.MigrationOwnershipState) (*pb.AttestMigrationOwnershipRequest, error) { + p, err := kv.EncodeMigrationPartitionJSON(partitions) + if err != nil { + return nil, errors.WithStack(err) + } + l, err := kv.EncodeMigrationPartitionJSON(legacy) + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.AttestMigrationOwnershipRequest{PriorPartition: p, PriorOwnershipState: l}, nil +} + +func matchMigrationPartitionAttestationPriors(req *pb.AttestMigrationOwnershipRequest, c kv.MigrationPartitionControl) error { + actual, err := migrationPartitionAttestationRequest(c.PriorPartitions, c.PriorOwnership) + if err != nil { + return err + } + if !bytes.Equal(req.GetPriorPartition(), actual.GetPriorPartition()) || !bytes.Equal(req.GetPriorOwnershipState(), actual.GetPriorOwnershipState()) { + return errors.WithStack(store.ErrMigrationOwnership) + } + return nil +} + +func (s *DistributionServer) acknowledgeMigrationPartition(ctx context.Context, members []SplitMigrationVoter, index, job uint64, t store.MigrationOwnershipPartitionTransition) error { + raw, err := kv.EncodeMigrationPartitionJSON(t) + if err != nil { + return errors.WithStack(err) + } + id, err := store.MigrationOwnershipPartitionReceiptIDFor(t) + if err != nil { + return errors.WithStack(err) + } + for _, member := range members { + attempt, cancel := context.WithTimeout(ctx, defaultSplitMigrationVoterProbeTimeout) + resp, probeErr := member.Client.ProbeMigrationState(attempt, &pb.ProbeMigrationStateRequest{JobId: job, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_PARTITION_APPLIED, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: index, PartitionTransition: raw}) + cancel() + if probeErr != nil { + return errors.WithStack(probeErr) + } + if !migrationPartitionAcknowledged(resp, index, t.Generation, id) { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + } + return nil +} + +func migrationPartitionAcknowledged(resp *pb.ProbeMigrationStateResponse, index uint64, generation string, id []byte) bool { + return resp.GetReady() && resp.GetOwnershipPartitionProtocol() == MigrationOwnershipPartitionProtocolVersion && resp.GetOwnershipProtocol() == migrationOwnershipProtocol && + resp.GetAppliedConfigurationIndex() == index && resp.GetOwnershipGeneration() == generation && bytes.Equal(resp.GetPartitionReceiptId(), id) +} diff --git a/adapter/migration_ownership_refresh_partition_test.go b/adapter/migration_ownership_refresh_partition_test.go new file mode 100644 index 000000000..55b2408c9 --- /dev/null +++ b/adapter/migration_ownership_refresh_partition_test.go @@ -0,0 +1,330 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type partitionRootClient struct { + root *DistributionServer + calls int +} + +func (c *partitionRootClient) AttestMigrationOwnership(ctx context.Context, req *pb.AttestMigrationOwnershipRequest, _ ...grpc.CallOption) (*pb.AttestMigrationOwnershipResponse, error) { + c.calls++ + return c.root.AttestMigrationOwnership(ctx, req) +} + +type partitionProbeClient struct { + *ownershipRuntimeClient + old, wrong, deny bool +} + +func (c *partitionProbeClient) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest, opts ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + resp, err := c.ownershipRuntimeClient.ProbeMigrationState(ctx, req, opts...) + if err != nil { + return nil, err + } + if c.old { + resp.OwnershipPartitionProtocol = 0 + } + if req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_PARTITION_APPLIED { + if c.wrong { + resp.PartitionReceiptId = []byte("wrong receipt") + } + if c.deny { + resp.Ready = false + } + } + return resp, nil +} + +type partitionRefreshHarness struct { + consumer, root *DistributionServer + rootStore store.MVCCStore + rootClient *partitionRootClient + clients []*partitionProbeClient + stores []store.MVCCStore + parent distribution.RouteDescriptor +} + +func newPartitionRefreshHarness(t *testing.T, local uint64) *partitionRefreshHarness { + t.Helper() + h := &partitionRefreshHarness{rootStore: store.NewMVCCStore()} + h.parent = distribution.RouteDescriptor{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 7, State: distribution.RouteStateActive, MinWriteTSExclusive: 100} + catalog := distribution.NewCatalogStore(h.rootStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + _, err := catalog.Save(t.Context(), 0, []distribution.RouteDescriptor{h.parent}) + require.NoError(t, err) + coord := &ownershipCatalogCoordinator{distributionCoordinatorStub: newDistributionCoordinatorStub(h.rootStore, true)} + h.root = NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coord)) + h.rootClient = &partitionRootClient{root: h.root} + hash, err := distribution.OwnershipPartitionDescriptorHash(h.parent) + require.NoError(t, err) + role := store.MigrationOwner + if local != 7 { + role = store.MigrationNonOwner + } + state := store.MigrationOwnershipState{JobID: 11, Generation: "0123456789abcdef0123456789abcdef", LocalGroupID: local, OwnerGroupID: 7, + CatalogVersion: 1, ConfigurationIndex: 9, RouteID: 1, RouteStart: h.parent.Start, RouteEnd: h.parent.End, DescriptorHash: hash, Role: role, WriteFloorKnown: true, MinWriteTSExclusive: 100} + for range 3 { + st := store.NewMVCCStore() + client := newPartitionRuntimeClient(t, st, local) + writer, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + require.NoError(t, writer.ClearMigrationStateWithOwnership(t.Context(), state, 10)) + tx := kv.NewTransactionWithProposer(client.internal.migrationProposer, kv.WithMigrationRequestAdmission(st, local)) + t.Cleanup(tx.Close) + client.internal.transactionManager = tx + client.internal.migrationPartitionProtocolReady = func() bool { return true } + h.stores = append(h.stores, st) + h.clients = append(h.clients, &partitionProbeClient{ownershipRuntimeClient: client}) + } + proposer, ok := h.clients[0].internal.migrationProposer.(*ownershipRuntimeProposer) + require.True(t, ok) + for _, client := range h.clients[1:] { + p, ok := client.internal.migrationProposer.(*ownershipRuntimeProposer) + require.True(t, ok) + proposer.followers = append(proposer.followers, p.fsm) + } + ids := []uint64{etcd.DeriveNodeID("voter1"), etcd.DeriveNodeID("voter2"), etcd.DeriveNodeID("learner")} + for _, client := range h.clients { + p, ok := client.internal.migrationProposer.(*ownershipRuntimeProposer) + require.True(t, ok) + aware, ok := p.fsm.(raftengine.NamespaceConfigurationAware) + require.True(t, ok) + require.NoError(t, aware.RecordAppliedNamespaceConfiguration(9, ids)) + } + configuration, ok := proposer.fsm.(interface { + AppliedConfigurationIndex() uint64 + RecordAppliedConfiguration(uint64) error + }) + require.True(t, ok) + groups := map[uint64]*kv.ShardGroup{local: {Store: h.stores[0], Txn: h.clients[0].internal.transactionManager, Engine: ownershipRefreshEngine{configuration: configuration}}} + members := func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "voter1", Address: "one", Client: h.clients[0]}, {ID: "voter2", Address: "two", Client: h.clients[1]}, {ID: "learner", Address: "three", Client: h.clients[2]}}, nil + } + mirror := distribution.NewEngine() + h.consumer = NewDistributionServer(mirror, nil, + WithSplitJobRunnerReadinessGate(func(context.Context) error { return nil }), + WithSplitMigrationGroupClientFactory(func(context.Context, uint64) (SplitMigrationClient, error) { return h.clients[0], nil }), + WithSplitMigrationVoterFactory(members), + WithMigrationOwnershipRefresh(groups, func(context.Context) (MigrationOwnershipAttestor, error) { return h.rootClient, nil }), + WithMigrationPartitionRuntime(kv.NewShardStore(mirror, groups), members)) + require.True(t, h.consumer.MigrationPartitionRuntimeConfigured(local)) + return h +} + +func newPartitionRuntimeClient(t *testing.T, st store.MVCCStore, group uint64) *ownershipRuntimeClient { + t.Helper() + fsm := kv.NewKvFSMWithHLC(st, nil, kv.WithRouteHistory(nil, group)) + configuration, ok := fsm.(raftengine.AppliedConfigurationAware) + require.True(t, ok) + require.NoError(t, configuration.RecordAppliedConfiguration(9)) + proposer := &ownershipRuntimeProposer{fsm: fsm, index: 20} + i := NewInternalWithEngine(nil, ownershipRuntimeLeader{configuration: configuration}, nil, nil, + WithInternalStore(st), WithInternalMigrationProposer(proposer), WithInternalMigrationOwnershipReady(func() bool { return true }), + WithInternalMigrationCleanupGate(func(context.Context) error { return nil }), + WithMigrationNamespaceAdmissionReady(func() bool { return kv.MigrationNamespaceInstalledVersion(st, fsm) == kv.MigrationNamespaceVersion })) + return &ownershipRuntimeClient{internal: i} +} + +func partitionOnlyCleanups(client *partitionProbeClient) []*pb.CleanupMigrationRequest { + var result []*pb.CleanupMigrationRequest + for _, req := range client.cleanups { + if req.GetMode() != pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION { + result = append(result, req) + } + } + return result +} + +func (h *partitionRefreshHarness) split(t *testing.T, parent distribution.RouteDescriptor, leftID, rightID uint64, key string) []distribution.RouteDescriptor { + t.Helper() + snapshot, err := h.root.catalog.Snapshot(t.Context()) + require.NoError(t, err) + left, right := parent, parent + left.RouteID, left.ParentRouteID, left.End, left.SplitAtHLC = leftID, parent.RouteID, []byte(key), parent.SplitAtHLC+1 + right.RouteID, right.ParentRouteID, right.Start, right.SplitAtHLC = rightID, parent.RouteID, []byte(key), parent.SplitAtHLC+1 + m := distribution.OwnershipPartitionManifest{PreviousVersion: snapshot.Version, Version: snapshot.Version + 1, Parent: parent, Children: []distribution.RouteDescriptor{left, right}} + readTS := h.rootStore.LastCommitTS() + manifest, err := h.root.catalog.BuildOwnershipPartitionMutationAt(t.Context(), readTS, m) + require.NoError(t, err) + mutations := []*store.KVPairMutation{manifest, {Op: store.OpTypeDelete, Key: distribution.CatalogRouteKey(parent.RouteID)}, + {Op: store.OpTypePut, Key: distribution.CatalogVersionKey(), Value: distribution.EncodeCatalogVersion(m.Version)}} + for _, route := range m.Children { + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(route, true) + require.NoError(t, err) + mutations = append(mutations, &store.KVPairMutation{Op: store.OpTypePut, Key: distribution.CatalogRouteKey(route.RouteID), Value: encoded}) + } + require.NoError(t, h.rootStore.ApplyMutations(t.Context(), mutations, [][]byte{distribution.CatalogVersionKey(), manifest.Key}, readTS, readTS+1)) + return m.Children +} + +func (h *partitionRefreshHarness) frontier(t *testing.T, i int) store.MigrationOwnershipPartitionState { + t.Helper() + reader, ok := h.stores[i].(store.MigrationOwnershipPartitionStore) + require.True(t, ok) + state, err := reader.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + return state +} + +func TestPartitionRefreshThreeApplyMembersAndHistoricalNonowner(t *testing.T) { + for _, local := range []uint64{7, 9} { + t.Run(map[uint64]string{7: "owner", 9: "historical-nonowner"}[local], func(t *testing.T) { + h := newPartitionRefreshHarness(t, local) + h.split(t, h.parent, 2, 3, "m") + require.NoError(t, h.consumer.reconcileMigrationPartitions(t.Context(), local)) + state := h.frontier(t, 0) + require.Len(t, state.Partitions, 2) + require.Len(t, state.Receipts, 1, "unchanged polling must not grow receipt history") + for i := 1; i < len(h.stores); i++ { + require.Equal(t, state, h.frontier(t, i)) + } + for _, p := range state.Partitions { + require.EqualValues(t, 2, p.CatalogVersion) + require.EqualValues(t, 100, p.MinWriteTSExclusive) + if local == 9 { + require.Equal(t, store.MigrationNonOwner, p.Role) + } + } + // No root leader/data leader equivalence or runner activation gate. + require.Nil(t, h.consumer.coordinator) + require.NoError(t, h.consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + require.Equal(t, state, h.frontier(t, 0)) + }) + } +} + +func TestPartitionRefreshLearnerCapabilityAndReceiptAreMandatory(t *testing.T) { + h := newPartitionRefreshHarness(t, 7) + h.split(t, h.parent, 2, 3, "m") + h.clients[2].old = true + require.Error(t, h.consumer.reconcileMigrationPartitions(t.Context(), 7)) + require.Empty(t, partitionOnlyCleanups(h.clients[0]), "no all-member proof means no partition proposal") + h.clients[2].old = false + h.clients[2].wrong = true + require.Error(t, h.consumer.reconcileMigrationPartitions(t.Context(), 7)) + require.Len(t, h.frontier(t, 0).Partitions, 2, "a failed ACK does not undo committed apply") + h.clients[2].wrong = false + require.NoError(t, h.consumer.reconcileMigrationPartitions(t.Context(), 7)) + require.Len(t, h.frontier(t, 0).Receipts, 1) +} + +func TestPartitionRefreshRetainedManifestChainAndCurrentC(t *testing.T) { + h := newPartitionRefreshHarness(t, 7) + children := h.split(t, h.parent, 2, 3, "m") + h.split(t, children[1], 4, 5, "t") + // Both ledger steps predate the data group's first attempt; reconciliation + // must finish all geometry, not ACK the first intermediate manifest. + require.NoError(t, h.consumer.reconcileMigrationPartitions(t.Context(), 7)) + state := h.frontier(t, 0) + require.Len(t, state.Partitions, 3) + for _, p := range state.Partitions { + require.EqualValues(t, 3, p.CatalogVersion) + } + require.Len(t, state.Receipts, 2) + ids := make([][]byte, len(state.Partitions)) + for i, p := range state.Partitions { + ids[i] = p.PartitionID + } + // An unrelated root C requires unchanged refresh, never descriptor guesses. + snapshot, err := h.root.catalog.Snapshot(t.Context()) + require.NoError(t, err) + _, err = h.root.catalog.Save(t.Context(), snapshot.Version, snapshot.Routes) + require.NoError(t, err) + require.NoError(t, h.consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + state = h.frontier(t, 0) + for i, p := range state.Partitions { + require.Equal(t, ids[i], p.PartitionID) + require.EqualValues(t, 4, p.CatalogVersion) + } +} + +func TestPartitionRefreshRuntimeGroundedAndChangedDescriptorRefuses(t *testing.T) { + h := newPartitionRefreshHarness(t, 7) + h.consumer.ownershipRefresh.partitionRuntime.members = nil + require.False(t, h.consumer.MigrationPartitionRuntimeConfigured(7)) + require.Error(t, h.consumer.reconcileMigrationPartitions(t.Context(), 7)) + h = newPartitionRefreshHarness(t, 7) + changed := h.parent + changed.End = []byte("y") + _, err := h.root.catalog.Save(t.Context(), 1, []distribution.RouteDescriptor{changed}) + require.NoError(t, err) + require.Error(t, h.consumer.reconcileMigrationPartitions(t.Context(), 7)) + require.Empty(t, h.frontier(t, 0).Partitions) +} + +func TestPartitionRefreshForegroundCanDriveRemoteSourceLeader(t *testing.T) { + h := newPartitionRefreshHarness(t, 7) + h.split(t, h.parent, 2, 3, "m") + g := h.consumer.ownershipRefresh.groups[7] + engine, ok := g.Engine.(ownershipRefreshEngine) + require.True(t, ok) + engine.follower = true + g.Engine = engine + require.NoError(t, h.consumer.reconcileMigrationPartitions(t.Context(), 7)) + require.Len(t, h.frontier(t, 0).Partitions, 2) + // A lagging local view cannot turn an absent local candidate into success. + g.Store = store.NewMVCCStore() + leader, ok := h.clients[0].internal.leader.(ownershipRuntimeLeader) + require.True(t, ok) + configuration := leader.configuration + localFSM := kv.NewKvFSMWithHLC(g.Store, nil) + aware, ok := localFSM.(interface{ RecordAppliedConfiguration(uint64) error }) + require.True(t, ok) + require.NoError(t, aware.RecordAppliedConfiguration(configuration.AppliedConfigurationIndex())) + require.Error(t, h.consumer.reconcileMigrationPartitions(t.Context(), 7)) +} + +func TestPartitionRefreshUnadoptedLegacyUsesRootAndPreservesReceipts(t *testing.T) { + h := newPartitionRefreshHarness(t, 7) + writer, ok := h.stores[0].(store.MigrationOwnershipStore) + require.True(t, ok) + before, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + require.NoError(t, h.consumer.reconcileMigrationPartitions(t.Context(), 7), "same C still requires root and all-member ACK") + require.Empty(t, partitionOnlyCleanups(h.clients[0])) + require.NotZero(t, h.rootClient.calls) + _, err = h.root.catalog.Save(t.Context(), 1, []distribution.RouteDescriptor{h.parent}) + require.NoError(t, err) + require.NoError(t, h.consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + view, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + require.EqualValues(t, 2, view.Ownership[0].CatalogVersion) + require.Equal(t, before.Ownership[0].Receipts, view.Ownership[0].Receipts) + require.Empty(t, h.frontier(t, 0).Partitions, "ordinary legacy refresh never manufactures child authority") + cleanups := partitionOnlyCleanups(h.clients[0]) + require.Len(t, cleanups, 1) + require.Equal(t, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH, cleanups[0].GetMode()) +} + +func TestPartitionRefreshNamespaceNeedsEveryApplyMember(t *testing.T) { + h := newPartitionRefreshHarness(t, 7) + h.split(t, h.parent, 2, 3, "m") + h.clients[2].internal.migrationNamespaceReady = nil + require.Error(t, h.consumer.reconcileMigrationPartitions(t.Context(), 7)) + require.Empty(t, h.clients[0].cleanups, "missing member static namespace proof forbids even MODE8") + require.Empty(t, h.frontier(t, 0).Partitions) +} + +func TestPartitionRefreshCleanupMarkerRequiredExceptNamespaceBootstrap(t *testing.T) { + h := newPartitionRefreshHarness(t, 7) + _, err := h.clients[0].internal.CleanupMigration(t.Context(), &pb.CleanupMigrationRequest{JobId: 11, Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA}) + require.Error(t, err, "ordinary cleanup cannot bootstrap its own marker") + require.Empty(t, h.clients[0].cleanups) + h.split(t, h.parent, 2, 3, "m") + require.NoError(t, h.consumer.reconcileMigrationPartitions(t.Context(), 7), "MODE8 must bypass ordinary cleanup/ownership readiness") + for _, st := range h.stores { + require.NoError(t, kv.RequireMigrationNamespaceAdmission(t.Context(), st)) + } +} diff --git a/adapter/migration_ownership_refresh_test.go b/adapter/migration_ownership_refresh_test.go new file mode 100644 index 000000000..9a7d64f86 --- /dev/null +++ b/adapter/migration_ownership_refresh_test.go @@ -0,0 +1,511 @@ +package adapter + +import ( + "bytes" + "context" + "crypto/sha256" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +func TestOwnershipRefreshConsumerObsoleteProofDoesNotStarveCurrentOwner(t *testing.T) { + root, active := ownershipRefreshCatalog(t) + snapshot, err := root.catalog.Snapshot(t.Context()) + require.NoError(t, err) + for snapshot.Version < 16 { + snapshot.Routes[0].MinWriteTSExclusive++ + _, err = root.catalog.Save(t.Context(), snapshot.Version, snapshot.Routes) + require.NoError(t, err) + snapshot, err = root.catalog.Snapshot(t.Context()) + require.NoError(t, err) + } + active.CatalogVersion = 15 + obsolete := proto.CloneOf(active) + obsolete.CatalogVersion, obsolete.RouteId, obsolete.OwnerGroupId = 12, 999, 9 + obsolete.Role = pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_NONOWNER + obsolete.DescriptorHash = bytes.Repeat([]byte{1}, sha256.Size) + st := store.NewMVCCStore() + _, _ = newOwnershipRuntimeClient(t, st, 7) + writer, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + seedOwnershipRefreshState(t, writer, 6, obsolete, 10) + seedOwnershipRefreshState(t, writer, 7, active, 11) + rootClient, closeClient := newBufconnDistributionClient(t, root) + defer closeClient() + consumer, client, _ := ownershipRefreshConsumer(t, st, rootClient, active) + require.Error(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context()), "old removed route must not be refreshed") + view, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Len(t, view.Ownership, 2) + require.EqualValues(t, 12, view.Ownership[0].CatalogVersion) + require.Equal(t, store.MigrationNonOwner, view.Ownership[0].Role) + require.EqualValues(t, 16, view.Ownership[1].CatalogVersion) + require.Equal(t, store.MigrationOwner, view.Ownership[1].Role) + require.Len(t, client.cleanups, 1) + coordinator, ok := root.coordinator.(*ownershipCatalogCoordinator) + require.True(t, ok) + reads := coordinator.reads + require.NoError(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + require.Equal(t, reads, coordinator.reads, "unchanged C must not poll root once per failed historical proof") + require.Len(t, client.cleanups, 1) + snapshot.Routes[0].MinWriteTSExclusive++ + _, err = root.catalog.Save(t.Context(), snapshot.Version, snapshot.Routes) + require.NoError(t, err) + require.NoError(t, consumer.engine.ApplySnapshot(distribution.CatalogSnapshot{Version: 17, Routes: []distribution.RouteDescriptor{{RouteID: 100, GroupID: 100}}})) + require.Error(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + view, err = writer.MigrationServingState(t.Context()) + require.NoError(t, err) + require.EqualValues(t, 12, view.Ownership[0].CatalogVersion) + require.EqualValues(t, 17, view.Ownership[1].CatalogVersion, "new C bypasses the historical proof's retry delay") +} + +func seedOwnershipRefreshState(t *testing.T, writer store.MigrationOwnershipStore, jobID uint64, proof *pb.MigrationOwnershipProof, index uint64) { + t.Helper() + state, err := kv.MigrationOwnershipStateFromProto(jobID, proof) + require.NoError(t, err) + state.ConfigurationIndex = 9 + require.NoError(t, writer.ClearMigrationStateWithOwnership(t.Context(), state, index)) +} + +func TestOwnershipRefreshConsumerSequentialSplitRequiresRemainingRangeReacquisition(t *testing.T) { + root, original := ownershipRefreshCatalog(t) + snapshot := advanceOwnershipRefreshCatalog(t, root, 12) + original.CatalogVersion = snapshot.Version + // Job 1 owned [m,z]. Job 2 keeps the same left route ID but shrinks its + // descriptor to [m,x], moving [x,z] to another group. This is not an + // unchanged-descriptor refresh of Job 1, even though its left side remains. + snapshot.Routes[1].End = []byte("x") + right := distribution.RouteDescriptor{RouteID: 4, ParentRouteID: 3, + Start: []byte("x"), End: []byte("z"), GroupID: 9, + State: distribution.RouteStateActive, MinWriteTSExclusive: 200} + snapshot.Routes = append(snapshot.Routes, right) + _, err := root.catalog.Save(t.Context(), snapshot.Version, snapshot.Routes) + require.NoError(t, err) + snapshot = advanceOwnershipRefreshCatalog(t, root, 15) + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(right, true) + require.NoError(t, err) + digest := sha256.Sum256(encoded) + nonowner := proto.CloneOf(original) + nonowner.CatalogVersion, nonowner.RouteId, nonowner.ParentRouteId = snapshot.Version, right.RouteID, right.ParentRouteID + nonowner.RouteStart, nonowner.RouteEnd = bytes.Clone(right.Start), bytes.Clone(right.End) + nonowner.OwnerGroupId, nonowner.MinWriteTsExclusive = right.GroupID, right.MinWriteTSExclusive + nonowner.Role, nonowner.DescriptorHash = pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_NONOWNER, digest[:] + st := store.NewMVCCStore() + _, _ = newOwnershipRuntimeClient(t, st, 7) + writer, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + seedOwnershipRefreshState(t, writer, 7, original, 10) + seedOwnershipRefreshState(t, writer, 8, nonowner, 11) + before, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + advanceOwnershipRefreshCatalog(t, root, 16) + rootClient, closeClient := newBufconnDistributionClient(t, root) + defer closeClient() + consumer, client, _ := ownershipRefreshConsumer(t, st, rootClient, nonowner) + require.ErrorContains(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context()), "descriptor changed") + require.Len(t, client.cleanups, 1, "no proposal may clip or refresh the hash-changed old proof") + require.EqualValues(t, 8, client.cleanups[0].GetJobId()) + after, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Len(t, after.Ownership, 2) + assertSequentialSplitOwnershipRetained(t, before.Ownership, after.Ownership, nonowner) + retirement, ok := st.(store.MigrationRetirementReader) + require.True(t, ok) + retired, err := retirement.MigrationRetired(t.Context(), 8) + require.NoError(t, err) + require.True(t, retired, "refresh must not reopen the retired nonowner job") + for _, key := range []string{"n", "y"} { + err := kv.CheckMigrationRequest(t.Context(), st, 7, &pb.Request{Ts: 300, ObservedRouteVersion: 16, + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte(key), Value: []byte("not admitted")}}}) + require.ErrorIs(t, err, kv.ErrRouteCutoverPending, "remaining-left proof needs reacquisition; moved-right stays nonowner") + } +} + +func advanceOwnershipRefreshCatalog(t *testing.T, root *DistributionServer, version uint64) distribution.CatalogSnapshot { + t.Helper() + snapshot, err := root.catalog.Snapshot(t.Context()) + require.NoError(t, err) + for snapshot.Version < version { + snapshot.Routes[0].MinWriteTSExclusive++ + _, err = root.catalog.Save(t.Context(), snapshot.Version, snapshot.Routes) + require.NoError(t, err) + snapshot, err = root.catalog.Snapshot(t.Context()) + require.NoError(t, err) + } + return snapshot +} + +func assertSequentialSplitOwnershipRetained(t *testing.T, before, after []store.MigrationOwnershipState, nonowner *pb.MigrationOwnershipProof) { + t.Helper() + prior := make(map[uint64]store.MigrationOwnershipState, len(before)) + for _, state := range before { + prior[state.JobID] = state + } + for _, state := range after { + if state.JobID == 7 { + require.Equal(t, prior[7], state, "old full [m,z] OWNER proof must remain byte-for-byte intact at C12") + continue + } + require.EqualValues(t, 8, state.JobID) + expected := proto.CloneOf(nonowner) + expected.CatalogVersion = 16 + require.True(t, proto.Equal(expected, kv.MigrationOwnershipProofFromState(state))) + require.Equal(t, prior[8].Receipts, state.Receipts, "NONOWNER receipt C15 must survive its valid unchanged-descriptor refresh") + } +} + +type ownershipRefreshEngine struct { + raftengine.Engine + configuration raftengine.AppliedConfigurationAware + follower bool + read func(context.Context) error +} + +func (e ownershipRefreshEngine) State() raftengine.State { + if e.follower { + return raftengine.StateFollower + } + return raftengine.StateLeader +} + +func (e ownershipRefreshEngine) VerifyLeader(context.Context) error { return nil } +func (e ownershipRefreshEngine) LinearizableRead(ctx context.Context) (uint64, error) { + if e.read != nil { + return 0, e.read(ctx) + } + return 1, nil +} +func (e ownershipRefreshEngine) AppliedConfigurationIndex() (uint64, bool) { + index := e.configuration.AppliedConfigurationIndex() + return index, index != 0 +} + +func (e ownershipRefreshEngine) Status() raftengine.Status { + return raftengine.Status{State: e.State(), Term: 1} +} + +func ownershipRefreshConsumer(t *testing.T, st store.MVCCStore, root pb.DistributionClient, prior *pb.MigrationOwnershipProof) (*DistributionServer, *ownershipRuntimeClient, raftengine.AppliedConfigurationAware) { + t.Helper() + id := prior.GetLocalGroupId() + client, configuration := newOwnershipRuntimeClient(t, st, id) + writer, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + view, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + if len(view.Ownership) == 0 { + state, err := kv.MigrationOwnershipStateFromProto(7, prior) + require.NoError(t, err) + state.ConfigurationIndex = 9 + require.NoError(t, writer.ClearMigrationStateWithOwnership(t.Context(), state, 10)) + } + mirror := distribution.NewEngine() + require.NoError(t, mirror.ApplySnapshot(distribution.CatalogSnapshot{Version: prior.GetCatalogVersion() + 1, Routes: []distribution.RouteDescriptor{ + {RouteID: 99, Start: []byte("a"), End: []byte("z"), GroupID: 99}, + }})) + server := NewDistributionServer(mirror, nil, + WithSplitJobRunnerReadinessGate(func(context.Context) error { return nil }), + WithSplitMigrationGroupClientFactory(func(context.Context, uint64) (SplitMigrationClient, error) { return client, nil }), + WithSplitMigrationVoterFactory(func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "data", Address: "data", Client: client}}, nil + }), + WithMigrationOwnershipRefresh(map[uint64]*kv.ShardGroup{id: {Store: st, Engine: ownershipRefreshEngine{configuration: configuration}}}, + func(context.Context) (MigrationOwnershipAttestor, error) { return root, nil }), + ) + return server, client, configuration +} + +func TestOwnershipRefreshConsumerRejectsConfigurationInterleaving(t *testing.T) { + root, prior := ownershipRefreshCatalog(t) + rootClient, closeClient := newBufconnDistributionClient(t, root) + defer closeClient() + st := store.NewMVCCStore() + consumer, client, configuration := ownershipRefreshConsumer(t, st, rootClient, prior) + client.beforeCleanup = func() { require.NoError(t, configuration.RecordAppliedConfiguration(11)) } + require.Error(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + writer, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + view, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + require.EqualValues(t, 1, view.Ownership[0].CatalogVersion) + client.beforeCleanup = nil + require.NoError(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + require.Len(t, client.cleanups, 2) + require.EqualValues(t, 11, client.cleanups[1].GetExpectedConfigurationIndex()) +} + +type refreshProbeClient struct { + SplitMigrationClient + probe func(*pb.ProbeMigrationStateRequest, *pb.ProbeMigrationStateResponse) +} + +func (c refreshProbeClient) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest, opts ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + resp, err := c.SplitMigrationClient.ProbeMigrationState(ctx, req, opts...) + if err == nil { + c.probe(req, resp) + } + return resp, err +} + +func TestOwnershipRefreshConsumerRetriesExactAppliedAckWithoutNewProposal(t *testing.T) { + root, prior := ownershipRefreshCatalog(t) + rootClient, closeClient := newBufconnDistributionClient(t, root) + defer closeClient() + consumer, client, _ := ownershipRefreshConsumer(t, store.NewMVCCStore(), rootClient, prior) + blocked := true + probes := 0 + follower := refreshProbeClient{SplitMigrationClient: client, probe: func(req *pb.ProbeMigrationStateRequest, resp *pb.ProbeMigrationStateResponse) { + probes++ + if req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED && blocked { + resp.Ready = false + } + }} + consumer.splitMigrationVoterFactory = func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "leader", Address: "leader", Client: client}, {ID: "follower", Address: "follower", Client: follower}}, nil + } + require.ErrorIs(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context()), errSplitMigrationVoterBarrierIncomplete) + require.Len(t, client.cleanups, 1) + blocked = false + require.NoError(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + require.Len(t, client.cleanups, 1) + previous := probes + require.NoError(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + require.Equal(t, previous, probes, "same proof/config/term and C must not poll each voter again") +} + +func TestOwnershipRefreshConsumerRejectsLegacyAndGenerationMismatch(t *testing.T) { + for _, mismatch := range []string{"legacy", "legacy v1", "generation", "configuration", "exact proof", "closed readiness"} { + t.Run(mismatch, func(t *testing.T) { + root, prior := ownershipRefreshCatalog(t) + rootClient, closeClient := newBufconnDistributionClient(t, root) + defer closeClient() + consumer, client, _ := ownershipRefreshConsumer(t, store.NewMVCCStore(), rootClient, prior) + peerClient := refreshProbeClient{SplitMigrationClient: client, probe: func(req *pb.ProbeMigrationStateRequest, resp *pb.ProbeMigrationStateResponse) { + switch mismatch { + case "legacy": + resp.OwnershipProtocol = 0 + case "legacy v1": + resp.OwnershipProtocol = 1 + case "generation": + resp.OwnershipGeneration = "different-native-generation" + case "configuration": + resp.AppliedConfigurationIndex++ + case "exact proof": + if req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED { + resp.Ownership.CatalogVersion++ + } + case "closed readiness": + resp.Ready = false + } + }} + consumer.splitMigrationVoterFactory = func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "voter", Address: "voter", Client: peerClient}}, nil + } + require.Error(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + if mismatch != "exact proof" { + require.Empty(t, client.cleanups) + } + }) + } +} + +func TestOwnershipRefreshConsumerRetainsNonownerAndNativeStateOnRestart(t *testing.T) { + root, prior := ownershipRefreshCatalog(t) + prior = proto.CloneOf(prior) + prior.LocalGroupId, prior.Role = 9, pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_NONOWNER + rootClient, closeClient := newBufconnDistributionClient(t, root) + defer closeClient() + dir := t.TempDir() + st, err := store.NewPebbleStore(dir) + require.NoError(t, err) + consumer, _, _ := ownershipRefreshConsumer(t, st, rootClient, prior) + require.NoError(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + writer, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + before, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, store.MigrationNonOwner, before.Ownership[0].Role) + require.NoError(t, st.Close()) + reopened, err := store.NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + consumer, client, _ := ownershipRefreshConsumer(t, reopened, rootClient, prior) + require.NoError(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + require.Empty(t, client.cleanups) + writer, ok = reopened.(store.MigrationOwnershipStore) + require.True(t, ok) + after, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, before.Ownership, after.Ownership) + retirement, ok := reopened.(store.MigrationRetirementReader) + require.True(t, ok) + retired, err := retirement.MigrationRetired(t.Context(), 7) + require.NoError(t, err) + require.True(t, retired) +} + +func TestOwnershipRefreshConsumerRejectsUnsupportedRoot(t *testing.T) { + _, prior := ownershipRefreshCatalog(t) + rootClient, closeClient := newBufconnDistributionClient(t, &pb.UnimplementedDistributionServer{}) + defer closeClient() + consumer, client, _ := ownershipRefreshConsumer(t, store.NewMVCCStore(), rootClient, prior) + err := consumer.RunMigrationOwnershipRefreshOnce(t.Context()) + require.Equal(t, codes.Unimplemented, status.Code(err)) + require.Empty(t, client.cleanups) +} + +func TestOwnershipRefreshConsumerRotatesGroupsAfterExhaustedBudget(t *testing.T) { + root, prior := ownershipRefreshCatalog(t) + rootClient, closeClient := newBufconnDistributionClient(t, root) + defer closeClient() + consumer, client, _ := ownershipRefreshConsumer(t, store.NewMVCCStore(), rootClient, prior) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + consumer.ownershipRefresh.groups[1] = &kv.ShardGroup{Store: store.NewMVCCStore(), Engine: ownershipRefreshEngine{read: func(context.Context) error { + cancel() + return context.Canceled + }}} + require.Error(t, consumer.RunMigrationOwnershipRefreshOnce(ctx)) + require.Empty(t, client.cleanups) + ctx, cancel = context.WithCancel(t.Context()) + defer cancel() + require.Error(t, consumer.RunMigrationOwnershipRefreshOnce(ctx)) + require.Len(t, client.cleanups, 1, "an earlier slow group cannot consume every pass's budget") +} + +func TestOwnershipRefreshConsumerReplicatesWithoutJobHistory(t *testing.T) { + root, prior := ownershipRefreshCatalog(t) + rootClient, closeClient := newBufconnDistributionClient(t, root) + defer closeClient() + st := store.NewMVCCStore() + consumer, client, _ := ownershipRefreshConsumer(t, st, rootClient, prior) + require.NoError(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + require.Len(t, client.cleanups, 1) + require.Equal(t, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH, client.cleanups[0].GetMode()) + require.EqualValues(t, 9, client.cleanups[0].GetExpectedConfigurationIndex()) + writer, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + view, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + require.EqualValues(t, 2, view.Ownership[0].CatalogVersion) + require.Equal(t, prior.GetGeneration(), view.Ownership[0].Generation) + require.Len(t, view.Ownership[0].Receipts, 1) + require.EqualValues(t, 1, view.Ownership[0].Receipts[0].CatalogVersion) + require.NoError(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context())) + require.Len(t, client.cleanups, 1, "mirror shape is only a trigger, never a replacement descriptor") +} + +func ownershipRefreshCatalog(t *testing.T) (*DistributionServer, *pb.MigrationOwnershipProof) { + t.Helper() + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st, distribution.WithCatalogRouteDescriptorV2Writes(true)) + routes := []distribution.RouteDescriptor{ + {RouteID: 2, ParentRouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 7}, + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 7, MinWriteTSExclusive: 100}, + } + _, err := catalog.Save(t.Context(), 0, routes) + require.NoError(t, err) + snapshot, err := catalog.Snapshot(t.Context()) + require.NoError(t, err) + job := sampleDistributionSplitJob(7) + job.SourceRouteID, job.SourceGroupID, job.TargetGroupID, job.SplitKey = 1, 7, 9, []byte("m") + job.Phase, job.SnapshotTS, job.FenceTS = distribution.SplitJobPhaseAbandoning, 80, 100 + prior, err := splitOwnershipProof(snapshot, job, 7, "0123456789abcdef0123456789abcdef") + require.NoError(t, err) + routes[0].GroupID = 11 + _, err = catalog.Save(t.Context(), snapshot.Version, routes) + require.NoError(t, err) + coordinator := &ownershipCatalogCoordinator{distributionCoordinatorStub: newDistributionCoordinatorStub(st, true)} + return NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)), prior +} + +func TestOwnershipAttestationRPCUsesCommittedCatalogWithoutJobHistory(t *testing.T) { + server, prior := ownershipRefreshCatalog(t) + client, closeClient := newBufconnDistributionClient(t, server) + defer closeClient() + resp, err := client.AttestMigrationOwnership(t.Context(), &pb.AttestMigrationOwnershipRequest{JobId: 7, Prior: prior}) + require.NoError(t, err) + require.EqualValues(t, 2, resp.GetOwnershipProtocol()) + require.Equal(t, prior.GetCatalogVersion()+1, resp.GetOwnership().GetCatalogVersion()) + require.Equal(t, prior.GetDescriptorHash(), resp.GetOwnership().GetDescriptorHash()) + server.coordinator = newDistributionCoordinatorStub(store.NewMVCCStore(), false) + _, err = client.AttestMigrationOwnership(t.Context(), &pb.AttestMigrationOwnershipRequest{JobId: 7, Prior: prior}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) +} + +func TestOwnershipAttestationRPCPreservesProtocolTwoSourceBoundary(t *testing.T) { + server, prior := ownershipRefreshCatalog(t) + prior.WriteFloorKnown, prior.SourceTrackerVersion, prior.SourceAppliedIndex = true, 1, 10 + client, closeClient := newBufconnDistributionClient(t, server) + defer closeClient() + resp, err := client.AttestMigrationOwnership(t.Context(), &pb.AttestMigrationOwnershipRequest{JobId: 7, Prior: prior}) + require.NoError(t, err) + require.EqualValues(t, 2, resp.GetOwnershipProtocol()) + require.True(t, resp.GetOwnership().GetWriteFloorKnown()) + require.EqualValues(t, 1, resp.GetOwnership().GetSourceTrackerVersion()) + require.EqualValues(t, 10, resp.GetOwnership().GetSourceAppliedIndex()) + expected := proto.CloneOf(prior) + expected.CatalogVersion = 2 + require.True(t, proto.Equal(expected, resp.GetOwnership()), "only C may advance across the authenticated producer RPC") +} + +type legacyOwnershipAttestationServer struct { + pb.UnimplementedDistributionServer + proof *pb.MigrationOwnershipProof +} + +func (s legacyOwnershipAttestationServer) AttestMigrationOwnership(context.Context, *pb.AttestMigrationOwnershipRequest) (*pb.AttestMigrationOwnershipResponse, error) { + return &pb.AttestMigrationOwnershipResponse{OwnershipProtocol: 1, Ownership: s.proof}, nil +} + +func TestOwnershipRefreshConsumerRejectsProtocolOneRoot(t *testing.T) { + _, prior := ownershipRefreshCatalog(t) + refreshed := proto.CloneOf(prior) + refreshed.CatalogVersion = 2 + client, closeClient := newBufconnDistributionClient(t, legacyOwnershipAttestationServer{proof: refreshed}) + defer closeClient() + consumer, data, _ := ownershipRefreshConsumer(t, store.NewMVCCStore(), client, prior) + require.ErrorIs(t, consumer.RunMigrationOwnershipRefreshOnce(t.Context()), store.ErrNotSupported) + require.Empty(t, data.cleanups, "protocol 1 cannot attest the protocol 2 source boundary") +} + +func TestOwnershipAttestationRPCRequiresMigrationCredential(t *testing.T) { + for _, enforced := range []bool{false, true} { + unary, _ := SplitMigrationTokenAuth(t.Name(), false, enforced) + for _, token := range []string{"", "other", t.Name()} { + ctx := metadata.NewIncomingContext(t.Context(), metadata.Pairs("authorization", "Bearer "+token)) + ctx = peer.NewContext(ctx, &peer.Peer{AuthInfo: credentials.TLSInfo{}}) + called := false + _, err := unary(ctx, &pb.AttestMigrationOwnershipRequest{}, &grpc.UnaryServerInfo{ + FullMethod: pb.Distribution_AttestMigrationOwnership_FullMethodName, + }, func(context.Context, any) (any, error) { called = true; return nil, nil }) + if token == t.Name() { + require.NoError(t, err) + require.True(t, called) + } else { + require.Equal(t, codes.Unauthenticated, status.Code(err)) + require.False(t, called) + } + } + plain := metadata.NewIncomingContext(t.Context(), metadata.Pairs("authorization", "Bearer "+t.Name())) + _, err := unary(plain, &pb.AttestMigrationOwnershipRequest{}, &grpc.UnaryServerInfo{ + FullMethod: pb.Distribution_AttestMigrationOwnership_FullMethodName, + }, func(context.Context, any) (any, error) { + t.Fatal("plaintext must not reach the root producer") + return nil, nil + }) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + } +} diff --git a/adapter/migration_quorum_leader_churn_test.go b/adapter/migration_quorum_leader_churn_test.go new file mode 100644 index 000000000..dfd84ff02 --- /dev/null +++ b/adapter/migration_quorum_leader_churn_test.go @@ -0,0 +1,130 @@ +package adapter + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestMigrationQuorumCommitFollowsForcedLeadershipTransfer(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second) + defer cancel() + c := newQuorumAcceptanceCluster(t, ctx) + initial := c.leader(t) + epoch := c.waitConfiguration(t, 3) + members, err := c.members(ctx, quorumAcceptanceGroup) + require.NoError(t, err) + quorumAcceptanceEventually(t, func() error { return EnsureMigrationNamespaceAdmission(ctx, initial.client, members) }) + _, root := c.partitionedSourceForDrain(t, ctx, initial, epoch) + control := quorumAcceptanceAcquire(t, ctx, initial, 12, epoch, quorumAcceptanceCatalogVersion, root.later.Children[1], "t") + large := make([]byte, store.MigrationPagePayloadBudget+(1<<19)) + _, prepared, err := c.commitToCurrentLeader(ctx, []*pb.Request{quorumAcceptanceDrainRequest(pb.Phase_PREPARE, "u", large)}) + require.NoError(t, err) + require.Positive(t, prepared.CommitIndex) + old, _, err := c.commitToCurrentLeader(ctx, []*pb.Request{quorumAcceptanceWriteRequest("a-ready", []byte("ready"), 90)}) + require.NoError(t, err) + consumer := c.consumer(t, old, root) + job := distribution.SplitJob{JobID: 12, SourceRouteID: 5, SplitKey: []byte("t")} + quorumAcceptanceEventually(t, func() error { + _, guardErr := consumer.ensureSplitJobTransferControl(ctx, job, quorumAcceptanceGroup, old.client, control, nil) + return guardErr + }) + current := transferQuorumLeaderForChurn(t, ctx, c, old) + stale := quorumAcceptanceWriteRequest("b-stale", []byte("must-not-land"), 100) + _, err = old.tx.Commit(ctx, []*pb.Request{stale}) + require.True(t, errors.Is(err, raftengine.ErrNotLeader), "the cached old transaction handle must reproduce the CI failure") + t.Logf("forced transfer %s -> %s: old direct transaction returns ErrNotLeader", old.peer.ID, current.peer.ID) + followedAt, _, err := c.commitToCurrentLeader(ctx, []*pb.Request{quorumAcceptanceWriteRequest("b-handoff", []byte("new leader"), 105)}) + require.NoError(t, err) + require.Same(t, current, followedAt, "the first post-transfer commit must follow the actual new voter") + committedAt, written, err := c.commitToCurrentLeader(ctx, []*pb.Request{quorumAcceptanceWriteRequest("b", large, 110)}) + require.NoError(t, err) + requireRealQuorumChurnVoter(t, c, committedAt) + require.Greater(t, written.CommitIndex, prepared.CommitIndex) + verdictAt, _, err := c.commitToCurrentLeader(ctx, []*pb.Request{quorumAcceptanceWriteRequest("w", large, 120)}) + require.ErrorIs(t, err, kv.ErrMigrationTransferTooLarge) + requireRealQuorumChurnVoter(t, c, verdictAt) + require.NoError(t, ctx.Err(), "a typed domain verdict must return without exhausting the original context") + staleVersion := quorumAcceptanceWriteRequest("b-wrong-version", []byte("must-not-land"), 130) + staleVersion.ObservedRouteVersion = kv.EncodeObservedRouteVersion(quorumAcceptanceCatalogVersion - 1) + _, _, err = c.commitToCurrentLeader(ctx, []*pb.Request{staleVersion}) + require.ErrorIs(t, err, kv.ErrRouteCutoverPending, "leader retry must not relax a stale catalog verdict") + require.NoError(t, ctx.Err()) + resolvedAt, resolved, err := c.commitToCurrentLeader(ctx, []*pb.Request{quorumAcceptanceDrainRequest(pb.Phase_COMMIT, "u", nil)}) + require.NoError(t, err, "the new leader must resolve the pre-transfer oversized intent using its stored bytes") + requireRealQuorumChurnVoter(t, c, resolvedAt) + require.Greater(t, resolved.CommitIndex, written.CommitIndex) + // Follow the returned node into later phase construction, just as the + // acceptance harness must after a large write or prepared resolution. + consumer = c.consumer(t, resolvedAt, root) + witness, err := consumer.ensureSplitJobTransferControl(ctx, job, quorumAcceptanceGroup, resolvedAt.client, control, nil) + require.NoError(t, err) + require.NoError(t, consumer.revalidateSplitTransferAdmission(ctx, witness)) + require.Equal(t, epoch, c.waitConfiguration(t, 3), "leadership transfer must not manufacture a configuration epoch") + checkQuorumChurnData(t, ctx, c, control, resolved.CommitIndex, large) +} + +func requireRealQuorumChurnVoter(t *testing.T, c *quorumAcceptanceCluster, n *quorumAcceptanceNode) { + t.Helper() + voters := map[*quorumAcceptanceNode]bool{c.nodes[0]: true, c.nodes[1]: true, c.nodes[2]: true} + require.True(t, voters[n], "every result must identify an actual configured data voter, even if leadership changes again") +} + +func transferQuorumLeaderForChurn(t *testing.T, ctx context.Context, c *quorumAcceptanceCluster, old *quorumAcceptanceNode) *quorumAcceptanceNode { + t.Helper() + var target *quorumAcceptanceNode + for _, n := range c.nodes[:3] { + if n != old { + target = n + break + } + } + require.NotNil(t, target) + index, err := old.engine.LinearizableRead(ctx) + require.NoError(t, err) + require.Eventually(t, func() bool { return target.engine.AppliedIndex() >= index }, 5*time.Second, 20*time.Millisecond) + require.NoError(t, old.engine.TransferLeadershipToServer(ctx, target.peer.ID, target.peer.Address)) + require.Eventually(t, func() bool { + return old.engine.State() == raftengine.StateFollower && target.engine.State() == raftengine.StateLeader + }, 5*time.Second, 20*time.Millisecond) + return target +} + +func checkQuorumChurnData(t *testing.T, ctx context.Context, c *quorumAcceptanceCluster, control *pb.TargetStagedReadinessRequest, index uint64, large []byte) { + t.Helper() + require.Eventually(t, func() bool { + for _, n := range c.nodes[:3] { + if n.engine.AppliedIndex() < index { + return false + } + } + return true + }, 5*time.Second, 20*time.Millisecond) + probe := probeMigrationControlRequest(control) + probe.RequireConfigurationIndex, probe.ExpectedConfigurationIndex = true, control.GetExpectedConfigurationIndex() + probe.TransferSizeGuardVersion = 1 + for _, n := range c.nodes[:3] { + for _, key := range []string{"b", "u"} { + value, err := n.store.GetAt(ctx, []byte(key), 1000) + require.NoError(t, err) + require.True(t, bytes.Equal(large, value), "all real voters must retain exactly the admitted and resolved bytes") + } + for _, key := range []string{"b-stale", "b-wrong-version", "w"} { + _, err := n.store.GetAt(ctx, []byte(key), 1000) + require.ErrorIs(t, err, store.ErrKeyNotFound) + } + ack, err := n.client.ProbeMigrationState(ctx, probe) + require.NoError(t, err) + require.True(t, ack.GetReady(), "every actual voter must preserve the exact configured guard through handoff") + require.Equal(t, control.GetExpectedConfigurationIndex(), ack.GetAppliedConfigurationIndex()) + } +} diff --git a/adapter/migration_quorum_partition_acceptance_test.go b/adapter/migration_quorum_partition_acceptance_test.go new file mode 100644 index 000000000..dcc0e61d7 --- /dev/null +++ b/adapter/migration_quorum_partition_acceptance_test.go @@ -0,0 +1,956 @@ +package adapter + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + internalutil "github.com/bootjp/elastickv/internal" + "github.com/bootjp/elastickv/internal/raftengine" + etcdraft "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/internal/raftengine/raftenginetest" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +const ( + quorumAcceptanceGroup = uint64(7) + quorumAcceptanceCatalogVersion = uint64(4) + quorumAcceptanceTick = 10 * time.Millisecond + quorumAcceptanceElectionTicks = 500 + quorumAcceptanceHeartbeatTicks = 10 + quorumAcceptanceElectionWait = 3 * quorumAcceptanceTick * quorumAcceptanceElectionTicks + quorumAcceptanceSnapshotEvery = 8 +) + +// Real data-group elections, WAL, transport, ConfChanges and native snapshots. +// Internal handlers are called directly; this is NOT peer-auth/TLS acceptance. +// The root below is an authoritative immutable manifest stub, NOT a root Raft +// quorum or proof of atomic root-catalog publication. No production gate opens. +type quorumAcceptanceNode struct { + peer etcdraft.Peer + dir string + transport *etcdraft.GRPCTransport + server *grpc.Server + listener net.Listener + engine *etcdraft.Engine + store store.MVCCStore + fsm raftengine.StateMachine + tx *kv.TransactionManager + internal *Internal + client *quorumAcceptanceClient + namespace, partition, size atomic.Bool + mode7, mode8, guards atomic.Uint64 + writeFences, timestamps atomic.Uint64 + exports []*pb.ExportRangeVersionsRequest +} + +type quorumAcceptanceClient struct { + SplitMigrationClient + node *quorumAcceptanceNode +} + +func (c *quorumAcceptanceClient) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest, _ ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + return c.node.internal.ProbeMigrationState(ctx, req) +} + +func (c *quorumAcceptanceClient) CleanupMigration(ctx context.Context, req *pb.CleanupMigrationRequest, _ ...grpc.CallOption) (*pb.CleanupMigrationResponse, error) { + if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION { + c.node.mode7.Add(1) + } + if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION { + c.node.mode8.Add(1) + } + return c.node.internal.CleanupMigration(ctx, req) +} + +func (c *quorumAcceptanceClient) ApplyTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest, _ ...grpc.CallOption) (*pb.TargetStagedReadinessResponse, error) { + if req.GetTransferSizeGuardVersion() != 0 { + c.node.guards.Add(1) + } + if req.GetSourceWriteFence() { + c.node.writeFences.Add(1) + } + return c.node.internal.ApplyTargetStagedReadiness(ctx, req) +} + +func (c *quorumAcceptanceClient) ProbeMigrationLocks(ctx context.Context, req *pb.ProbeMigrationLocksRequest, _ ...grpc.CallOption) (*pb.ProbeMigrationLocksResponse, error) { + return c.node.internal.ProbeMigrationLocks(ctx, req) +} + +func (c *quorumAcceptanceClient) IssueMigrationTimestamp(ctx context.Context, req *pb.IssueMigrationTimestampRequest, _ ...grpc.CallOption) (*pb.IssueMigrationTimestampResponse, error) { + c.node.timestamps.Add(1) + return c.node.internal.IssueMigrationTimestamp(ctx, req) +} + +func (c *quorumAcceptanceClient) ExportRangeVersions(ctx context.Context, req *pb.ExportRangeVersionsRequest, _ ...grpc.CallOption) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) { + c.node.exports = append(c.node.exports, proto.CloneOf(req)) + capture := &captureExportRangeVersionsStream{ctx: ctx} + err := c.node.internal.ExportRangeVersions(req, capture) + return &quorumAcceptanceExportStream{splitMigrationStreamStub: &splitMigrationStreamStub{responses: capture.responses}, terminal: err}, nil +} + +type quorumAcceptanceExportStream struct { + *splitMigrationStreamStub + terminal error +} + +func (s *quorumAcceptanceExportStream) Recv() (*pb.ExportRangeVersionsResponse, error) { + response, err := s.splitMigrationStreamStub.Recv() + if errors.Is(err, io.EOF) && s.terminal != nil { + return nil, s.terminal + } + return response, err +} + +type quorumAcceptanceCluster struct { + nodes []*quorumAcceptanceNode + lifetime context.Context +} + +func newQuorumAcceptanceCluster(t *testing.T, ctx context.Context) *quorumAcceptanceCluster { + t.Helper() + c := &quorumAcceptanceCluster{lifetime: ctx} + for index := range 4 { + listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", "127.0.0.1:0") + require.NoError(t, err) + id := fmt.Sprintf("quorum-%d", index+1) + n := &quorumAcceptanceNode{peer: etcdraft.Peer{NodeID: etcdraft.DeriveNodeID(id), ID: id, Address: listener.Addr().String()}, dir: t.TempDir(), listener: listener} + n.namespace.Store(true) + n.partition.Store(true) + n.size.Store(true) + c.nodes = append(c.nodes, n) + } + peers := make([]etcdraft.Peer, len(c.nodes)) + for index, n := range c.nodes { + peers[index] = n.peer + } + for _, n := range c.nodes { + n.transport = etcdraft.NewGRPCTransport(peers) + n.server = grpc.NewServer(internalutil.GRPCServerOptions()...) + n.transport.Register(n.server) + go func() { _ = n.server.Serve(n.listener) }() + } + t.Cleanup(func() { + for _, n := range c.nodes { + if n.tx != nil { + n.tx.Close() + } + if n.engine != nil { + require.NoError(t, n.engine.Close()) + } + n.server.Stop() + require.NoError(t, n.transport.Close()) + if n.store != nil { + require.NoError(t, n.store.Close()) + } + } + }) + var wg sync.WaitGroup + errs := make(chan error, 3) + for _, n := range c.nodes[:3] { + wg.Add(1) + go func() { defer wg.Done(); errs <- n.open(ctx, peers[:3], true) }() + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + c.leader(t) + c.waitConfiguration(t, 3) + return c +} + +func (n *quorumAcceptanceNode) open(ctx context.Context, peers []etcdraft.Peer, bootstrap bool) error { + n.store = store.NewMVCCStore() + n.fsm = kv.NewKvFSMWithHLC(n.store, nil, kv.WithRouteHistory(nil, quorumAcceptanceGroup)) + engine, err := etcdraft.Open(ctx, etcdraft.OpenConfig{NodeID: n.peer.NodeID, LocalID: n.peer.ID, LocalAddress: n.peer.Address, + DataDir: n.dir, Peers: peers, Bootstrap: bootstrap, JoinAsLearner: !bootstrap && len(peers) != 0, Transport: n.transport, StateMachine: n.fsm, + // Large-value acceptance is not a one-second failover latency test. + // Race-instrumented 64 MiB applies on a small CI runner must fit the + // modeled election window. Explicit transfers still exercise churn. + TickInterval: quorumAcceptanceTick, ElectionTick: quorumAcceptanceElectionTicks, HeartbeatTick: quorumAcceptanceHeartbeatTicks}) + if err != nil { + return errors.WithStack(err) + } + n.engine = engine + n.tx = kv.NewTransactionWithProposer(engine, kv.WithMigrationRequestAdmission(n.store, quorumAcceptanceGroup)) + n.internal = NewInternalWithEngine(n.tx, engine, nil, nil, WithInternalStore(n.store), WithInternalMigrationProposer(engine), + WithInternalMigrationCleanupGate(func(context.Context) error { return nil }), + WithInternalMigrationOwnershipReady(func() bool { return kv.MigrationRequestAdmissionReady(n.tx) }), + WithMigrationNamespaceAdmissionReady(func() bool { + return n.namespace.Load() && kv.MigrationNamespaceInstalledVersion(n.store, n.fsm) == kv.MigrationNamespaceVersion + }), + WithInternalMigrationPartitionProtocolReady(func() bool { return n.partition.Load() && kv.MigrationPartitionRequestAdmissionReady(n.tx) }), + WithInternalMigrationTransferSizeGuardReady(func() bool { return n.size.Load() && kv.MigrationRequestAdmissionReady(n.tx) })) + n.client = &quorumAcceptanceClient{node: n} + return nil +} + +func (c *quorumAcceptanceCluster) leader(t *testing.T) *quorumAcceptanceNode { + t.Helper() + var views []raftengine.LeaderView + var active []*quorumAcceptanceNode + for _, n := range c.nodes { + if n.engine != nil { + views = append(views, n.engine) + active = append(active, n) + } + } + return active[raftenginetest.WaitForLeader(t, quorumAcceptanceElectionWait, views...)] +} + +func (c *quorumAcceptanceCluster) waitConfiguration(t *testing.T, count int) uint64 { + t.Helper() + var epoch uint64 + require.Eventually(t, func() bool { + epoch = 0 + for _, n := range c.nodes[:count] { + cfg, err := n.engine.Configuration(t.Context()) + index, supported := n.engine.AppliedConfigurationIndex() + if err != nil || !supported || index == 0 || len(cfg.Servers) != count || n.engine.Status().PendingConfChange { + return false + } + if epoch != 0 && epoch != index { + return false + } + epoch = index + } + return true + }, 5*time.Second, 20*time.Millisecond) + return epoch +} + +func (c *quorumAcceptanceCluster) members(ctx context.Context, groupID uint64) ([]SplitMigrationVoter, error) { + if groupID != quorumAcceptanceGroup { + return nil, errors.New("quorum fixture has no runtime for the requested group") + } + var engine *etcdraft.Engine + for _, n := range c.nodes { + if n.engine != nil && n.engine.State() == raftengine.StateLeader { + engine = n.engine + break + } + } + if engine == nil { + return nil, errors.WithStack(raftengine.ErrNotLeader) + } + cfg, err := engine.Configuration(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + var members []SplitMigrationVoter + for _, configured := range cfg.Servers { + member, err := c.configuredMember(configured.ID) + if err != nil { + return nil, err + } + members = append(members, member) + } + return members, nil +} + +func (c *quorumAcceptanceCluster) configuredMember(id string) (SplitMigrationVoter, error) { + for _, n := range c.nodes { + if n.peer.ID == id { + return SplitMigrationVoter{ID: n.peer.ID, Address: n.peer.Address, Client: n.client}, nil + } + } + return SplitMigrationVoter{}, errors.New("configured apply member missing from real cluster") +} + +// The stub models a committed immutable root manifest and current descriptors. +// All data-group controls/ACKs still go through real Raft and real stores. +type quorumAcceptanceRoot struct { + manifest distribution.OwnershipPartitionManifest + later *distribution.OwnershipPartitionManifest +} + +func (r quorumAcceptanceRoot) version() uint64 { + if r.later != nil { + return r.later.Version + } + return r.manifest.Version +} +func (r quorumAcceptanceRoot) routes() []distribution.RouteDescriptor { + if r.later == nil { + return r.manifest.Children + } + return append([]distribution.RouteDescriptor{r.manifest.Children[0]}, r.later.Children...) +} + +func (r quorumAcceptanceRoot) AttestMigrationOwnership(_ context.Context, req *pb.AttestMigrationOwnershipRequest, _ ...grpc.CallOption) (*pb.AttestMigrationOwnershipResponse, error) { + partitions, legacy, err := decodeMigrationPartitionPriors(req) + if err != nil { + return nil, err + } + prior, epoch := migrationPartitionAttestationPrior(partitions, legacy) + var raw []byte + var unchanged *distribution.RouteDescriptor + switch { + case prior.RouteID == r.manifest.Parent.RouteID: + raw, err = distribution.EncodeOwnershipPartitionManifest(r.manifest) + case r.later != nil && prior.RouteID == r.later.Parent.RouteID: + raw, err = distribution.EncodeOwnershipPartitionManifest(*r.later) + default: + for _, route := range r.routes() { + if route.RouteID == prior.RouteID { + copy := route + unchanged = © + } + } + } + if err != nil { + return nil, err + } + control, err := kv.BuildMigrationPartitionControl(partitions, legacy, raw, unchanged, r.version(), epoch) + if err != nil { + return nil, err + } + encoded, err := kv.EncodeMigrationPartitionControl(control) + return &pb.AttestMigrationOwnershipResponse{OwnershipProtocol: migrationOwnershipProtocol, PartitionTransition: encoded}, err +} + +func (c *quorumAcceptanceCluster) consumer(t *testing.T, leader *quorumAcceptanceNode, root quorumAcceptanceRoot) *DistributionServer { + t.Helper() + mirror := distribution.NewEngine() + require.NoError(t, mirror.ApplySnapshot(distribution.CatalogSnapshot{Version: root.version(), Routes: root.routes()})) + groups := map[uint64]*kv.ShardGroup{quorumAcceptanceGroup: {Store: leader.store, Txn: leader.tx, Engine: leader.engine}} + return NewDistributionServer(mirror, nil, WithSplitMigrationGroupClientFactory(func(_ context.Context, groupID uint64) (SplitMigrationClient, error) { + if groupID != quorumAcceptanceGroup { + return nil, errors.New("quorum fixture has no client for the requested group") + } + return leader.client, nil + }), + WithSplitMigrationVoterFactory(c.members), WithMigrationOwnershipRefresh(groups, func(context.Context) (MigrationOwnershipAttestor, error) { return root, nil }), + WithMigrationPartitionRuntime(kv.NewShardStore(mirror, groups), c.members), + WithSplitMigrationApplyMemberSnapshotFactory(func(ctx context.Context, groupID uint64) (SplitMigrationApplyMemberSnapshot, error) { + members, err := c.members(ctx, groupID) + index, supported := leader.engine.AppliedConfigurationIndex() + if !supported || leader.engine.Status().PendingConfChange { + return SplitMigrationApplyMemberSnapshot{}, errors.WithStack(kv.ErrMigrationConfigurationChanged) + } + return SplitMigrationApplyMemberSnapshot{ConfigurationIndex: index, Members: members}, err + })) +} + +func quorumAcceptanceEventually(t *testing.T, operation func() error) { + t.Helper() + var last error + require.Eventually(t, func() bool { last = operation(); return last == nil }, 5*time.Second, 20*time.Millisecond, "last protocol error: %v", &last) + require.NoError(t, last) +} + +func quorumAcceptanceAcquire(t *testing.T, ctx context.Context, n *quorumAcceptanceNode, id, epoch, version uint64, route distribution.RouteDescriptor, start string) *pb.TargetStagedReadinessRequest { + t.Helper() + control := splitMigrationControlRequest(distribution.SplitJob{JobID: id, SplitKey: []byte(start)}, route.End, version, 1, false, false, true, 1) + control.RequireConfigurationIndex, control.ExpectedConfigurationIndex = true, epoch + _, err := n.client.ApplyTargetStagedReadiness(ctx, control) + require.NoError(t, err) + hash, err := distribution.OwnershipPartitionDescriptorHash(route) + require.NoError(t, err) + _, err = n.client.CleanupMigration(ctx, &pb.CleanupMigrationRequest{JobId: id, Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_SOURCE_OWNERSHIP_ACQUIRE, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: epoch, + Ownership: &pb.MigrationOwnershipProof{Generation: "0123456789abcdef0123456789abcdef", LocalGroupId: quorumAcceptanceGroup, OwnerGroupId: quorumAcceptanceGroup, + CatalogVersion: version, RouteId: route.RouteID, ParentRouteId: route.ParentRouteID, RouteStart: []byte(start), RouteEnd: route.End, + MinWriteTsExclusive: route.MinWriteTSExclusive, WriteFloorKnown: true, SourceTrackerVersion: version, DescriptorHash: hash, Role: pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER}}) + require.NoError(t, err) + return control +} + +func (c *quorumAcceptanceCluster) completeInitialOwner(t *testing.T, ctx context.Context, leader *quorumAcceptanceNode, epoch uint64, route distribution.RouteDescriptor) { + t.Helper() + quorumAcceptanceAcquire(t, ctx, leader, 11, epoch, 1, route, "a") + hash, err := distribution.OwnershipPartitionDescriptorHash(route) + require.NoError(t, err) + // The authoritative root stub has committed the unchanged-owner rollback + // at V2. Retire the SOURCE tracker atomically with its terminal OWNER proof; + // leaving it armed would model an invalid pair of overlapping live jobs. + proof := &pb.MigrationOwnershipProof{Generation: "0123456789abcdef0123456789abcdef", LocalGroupId: quorumAcceptanceGroup, OwnerGroupId: quorumAcceptanceGroup, + CatalogVersion: 2, RouteId: route.RouteID, RouteStart: route.Start, RouteEnd: route.End, MinWriteTsExclusive: route.MinWriteTSExclusive, + WriteFloorKnown: true, DescriptorHash: hash, Role: pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER} + _, err = leader.client.CleanupMigration(ctx, &pb.CleanupMigrationRequest{JobId: 11, Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: epoch, Ownership: proof}) + require.NoError(t, err) + quorumAcceptanceEventually(t, func() error { + for _, n := range c.nodes[:3] { + ack, err := n.client.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{JobId: 11, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: epoch, Ownership: proof}) + if err != nil { + return err + } + if !ack.GetReady() || !ack.GetOwnershipCleared() || ack.GetAppliedConfigurationIndex() != epoch { + return errors.New("completed initial owner is not ACKed by every actual voter") + } + } + return nil + }) + for _, n := range c.nodes[:3] { + reader, ok := n.store.(store.MigrationOwnershipStore) + require.True(t, ok) + view, err := reader.MigrationServingState(ctx) + require.NoError(t, err) + require.Empty(t, view.Readiness, "native ownership clear must remove the old guard, not raw fixture cleanup") + require.Len(t, view.Ownership, 1) + require.True(t, store.MigrationOwnershipEvidenceValid(view.Ownership[0])) + require.Zero(t, view.Ownership[0].SourceTrackerVersion) + } +} + +func TestMigrationQuorumPartitionAcceptance(t *testing.T) { + t.Setenv("ELASTICKV_RAFT_SNAPSHOT_COUNT", fmt.Sprint(quorumAcceptanceSnapshotEvery)) + lifetime, stop := context.WithTimeout(t.Context(), 3*time.Minute) + defer stop() + ctx, cancel := context.WithTimeout(lifetime, 60*time.Second) + defer cancel() + // Bound each costly acceptance phase independently; request cancellation + // must not double as the lifetime of the cluster needed by the next phase. + c := newQuorumAcceptanceCluster(t, lifetime) + leader := c.leader(t) + epoch := c.waitConfiguration(t, 3) + members, err := c.members(ctx, quorumAcceptanceGroup) + require.NoError(t, err) + c.nodes[2].namespace.Store(false) + require.Error(t, EnsureMigrationNamespaceAdmission(ctx, leader.client, members)) + require.Zero(t, leader.mode8.Load(), "unsupported member must prevent namespace proposal") + c.nodes[2].namespace.Store(true) + quorumAcceptanceEventually(t, func() error { return EnsureMigrationNamespaceAdmission(ctx, leader.client, members) }) + parent := distribution.RouteDescriptor{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: quorumAcceptanceGroup, MinWriteTSExclusive: 1} + c.completeInitialOwner(t, ctx, leader, epoch, parent) + left, right := parent, parent + left.RouteID, left.ParentRouteID, left.End = 2, 1, []byte("m") + right.RouteID, right.ParentRouteID, right.Start = 3, 1, []byte("m") + left.SplitAtHLC, right.SplitAtHLC = 1, 1 + root := quorumAcceptanceRoot{manifest: distribution.OwnershipPartitionManifest{PreviousVersion: 2, Version: 3, Parent: parent, Children: []distribution.RouteDescriptor{left, right}}} + consumer := c.consumer(t, leader, root) + c.nodes[2].partition.Store(false) + require.Error(t, consumer.reconcileMigrationPartitions(ctx, quorumAcceptanceGroup)) + require.Zero(t, leader.mode7.Load(), "field10=0 must prevent partition proposal despite real voter quorum") + c.nodes[2].partition.Store(true) + quorumAcceptanceEventually(t, func() error { return consumer.reconcileMigrationPartitions(ctx, quorumAcceptanceGroup) }) + _, err = leader.tx.Commit(ctx, []*pb.Request{quorumAcceptancePrepare(pb.Phase_PREPARE, 3)}) + require.NoError(t, err, "retained-left PREPARE precedes the later split, learner join and restart") + kept, moving := right, right + kept.RouteID, kept.ParentRouteID, kept.End, kept.SplitAtHLC = 4, 3, []byte("t"), 2 + moving.RouteID, moving.ParentRouteID, moving.Start, moving.SplitAtHLC = 5, 3, []byte("t"), 2 + root.later = &distribution.OwnershipPartitionManifest{PreviousVersion: 3, Version: quorumAcceptanceCatalogVersion, Parent: right, Children: []distribution.RouteDescriptor{kept, moving}} + consumer = c.consumer(t, leader, root) + quorumAcceptanceEventually(t, func() error { return consumer.reconcileMigrationPartitions(ctx, quorumAcceptanceGroup) }) + control := quorumAcceptanceAcquire(t, ctx, leader, 12, epoch, quorumAcceptanceCatalogVersion, moving, "t") + job := distribution.SplitJob{JobID: 12, SourceRouteID: 5, SplitKey: []byte("t")} + c.nodes[2].size.Store(false) + _, err = consumer.ensureSplitJobTransferControl(ctx, job, quorumAcceptanceGroup, leader.client, control, nil) + require.Error(t, err) + require.Zero(t, leader.guards.Load(), "field9=0 must prevent guard proposal") + c.nodes[2].size.Store(true) + var witness *splitTransferAdmissionWitness + quorumAcceptanceEventually(t, func() error { + witness, err = consumer.ensureSplitJobTransferControl(ctx, job, quorumAcceptanceGroup, leader.client, control, nil) + return err + }) + require.NoError(t, consumer.revalidateSplitTransferAdmission(ctx, witness)) + c.exerciseLearner(t, ctx, leader, consumer, job, control, witness, root) +} + +func (c *quorumAcceptanceCluster) exerciseLearner(t *testing.T, ctx context.Context, leader *quorumAcceptanceNode, consumer *DistributionServer, job distribution.SplitJob, control *pb.TargetStagedReadinessRequest, oldWitness *splitTransferAdmissionWitness, root quorumAcceptanceRoot) { + t.Helper() + joining := c.nodes[3] + peers := make([]etcdraft.Peer, len(c.nodes)) + for index, n := range c.nodes { + peers[index] = n.peer + } + require.NoError(t, joining.open(c.lifetime, peers, false)) + joiningEpoch, joiningSupported := joining.engine.AppliedConfigurationIndex() + require.False(t, joiningSupported) + require.Zero(t, joiningEpoch, "prejoin negotiation must not invent a candidate epoch") + installed, err := joining.client.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{JobId: 1, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_NAMESPACE_ADMISSION}) + require.NoError(t, err) + require.Equal(t, kv.MigrationNamespaceVersion, installed.GetNamespaceAdmissionVersion()) + require.False(t, installed.GetReady()) + oldEpoch, supported := leader.engine.AppliedConfigurationIndex() + require.True(t, supported) + quorumAcceptanceEventually(t, func() error { + return EnsureMigrationNamespaceAdmissionWithSnapshot(ctx, c.namespaceSnapshot, func(ctx context.Context, command kv.NamespaceAdmissionCommand) error { + req, err := kv.NewNamespaceAdmissionRequest(1, command) + if err != nil { + return err + } + _, err = leader.client.CleanupMigration(ctx, req) + return err + }, []NamespaceAdmissionPeer{{MemberID: joining.peer.NodeID, Address: joining.peer.Address, Client: joining.client}}) + }) + require.NoError(t, kv.CheckMigrationNamespacePrejoin(ctx, leader.store, oldEpoch, joining.peer.NodeID, kv.MigrationNamespaceVersion)) + newEpoch, err := leader.engine.AddLearner(ctx, joining.peer.ID, joining.peer.Address, oldEpoch) + require.NoError(t, err) + require.Greater(t, newEpoch, oldEpoch, "test must observe a real ConfChange, not reuse old ACKs") + t.Logf("real ConfChange: epoch %d -> %d; three voters plus one learner", oldEpoch, newEpoch) + require.Equal(t, newEpoch, c.waitConfiguration(t, 4)) + cfg, err := leader.engine.Configuration(ctx) + require.NoError(t, err) + var voters, learners int + for _, member := range cfg.Servers { + if member.Suffrage == etcdraft.SuffrageVoter { + voters++ + } + if member.Suffrage == etcdraft.SuffrageLearner { + learners++ + require.Equal(t, joining.peer.ID, member.ID) + } + } + require.Equal(t, 3, voters) + require.Equal(t, 1, learners) + require.Error(t, consumer.revalidateSplitTransferAdmission(ctx, oldWitness), "persisted guard does not authorize an old-epoch witness") + members, err := c.members(ctx, quorumAcceptanceGroup) + require.NoError(t, err) + require.Len(t, members, 4) + joining.namespace.Store(false) + before := leader.mode8.Load() + require.Error(t, EnsureMigrationNamespaceAdmission(ctx, leader.client, members)) + require.Equal(t, before, leader.mode8.Load()) + joining.namespace.Store(true) + quorumAcceptanceEventually(t, func() error { return EnsureMigrationNamespaceAdmission(ctx, leader.client, members) }) + joining.partition.Store(false) + before = leader.mode7.Load() + require.Error(t, consumer.reconcileMigrationPartitions(ctx, quorumAcceptanceGroup)) + require.Equal(t, before, leader.mode7.Load()) + joining.partition.Store(true) + quorumAcceptanceEventually(t, func() error { return consumer.reconcileMigrationPartitions(ctx, quorumAcceptanceGroup) }) + joining.size.Store(false) + before = leader.guards.Load() + fresh := splitMigrationControlRequest(job, control.GetRouteEnd(), control.GetExpectedCutoverVersion(), control.GetMinWriteTsExclusive(), false, false, true, 1) + _, err = consumer.ensureSplitJobTransferControl(ctx, job, quorumAcceptanceGroup, leader.client, fresh, nil) + require.Error(t, err) + require.Equal(t, before, leader.guards.Load()) + joining.size.Store(true) + var witness *splitTransferAdmissionWitness + quorumAcceptanceEventually(t, func() error { + witness, err = consumer.ensureSplitJobTransferControl(ctx, job, quorumAcceptanceGroup, leader.client, fresh, nil) + return err + }) + require.Equal(t, newEpoch, witness.snapshot.ConfigurationIndex) + require.Len(t, witness.snapshot.Members, 4) + c.checkReplicas(t, ctx, newEpoch) + require.NoError(t, ctx.Err(), "configuration/guard phase must finish within its own budget") + c.retainedAndRestart(t, root, newEpoch) +} + +func (c *quorumAcceptanceCluster) namespaceSnapshot(ctx context.Context) (NamespaceAdmissionSnapshot, error) { + members, err := c.members(ctx, quorumAcceptanceGroup) + if err != nil { + return NamespaceAdmissionSnapshot{}, err + } + result := NamespaceAdmissionSnapshot{} + for _, member := range members { + response, err := member.Client.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{JobId: 1, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_NAMESPACE_ADMISSION}) + if err != nil { + return result, err + } + if result.ConfigurationIndex != 0 && result.ConfigurationIndex != response.GetAppliedConfigurationIndex() { + return result, errors.WithStack(kv.ErrMigrationConfigurationChanged) + } + result.ConfigurationIndex = response.GetAppliedConfigurationIndex() + result.Members = append(result.Members, NamespaceAdmissionPeer{MemberID: etcdraft.DeriveNodeID(member.ID), Address: member.Address, Client: member.Client}) + } + return result, nil +} + +func (c *quorumAcceptanceCluster) checkReplicas(t *testing.T, ctx context.Context, epoch uint64) { + t.Helper() + var first store.MigrationOwnershipPartitionState + for index, n := range c.nodes { + require.NoError(t, kv.RequireMigrationNamespaceAdmission(ctx, n.store)) + actual, err := kv.MigrationConfigurationIndex(ctx, n.store) + require.NoError(t, err) + require.Equal(t, epoch, actual) + reader, ok := n.store.(store.MigrationOwnershipPartitionStore) + require.True(t, ok) + state, err := reader.MigrationOwnershipPartitions(ctx) + require.NoError(t, err) + require.NotEmpty(t, state.Partitions) + require.NotEmpty(t, state.Receipts) + for _, partition := range state.Partitions { + require.Equal(t, epoch, partition.ConfigurationIndex) + } + if index == 0 { + first = state + } else { + require.Equal(t, first, state) + } + response, err := n.client.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{JobId: 12, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY, RequireConfigurationIndex: true, ExpectedConfigurationIndex: epoch}) + require.NoError(t, err) + require.True(t, response.GetReady()) + require.Equal(t, uint32(1), response.GetTransferSizeGuardVersion()) + require.Equal(t, uint32(1), response.GetOwnershipPartitionProtocol()) + require.Equal(t, uint32(1), response.GetNamespaceAdmissionVersion()) + guard := &pb.ProbeMigrationStateRequest{JobId: 12, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: []byte("t"), RouteEnd: []byte("z"), ExpectedCatalogVersion: quorumAcceptanceCatalogVersion, MigrationJobId: 12, + MinWriteTsExclusive: 1, RetentionPinTs: 1, TrackWrites: true, TransferSizeGuardVersion: 1, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: epoch} + response, err = n.client.ProbeMigrationState(ctx, guard) + require.NoError(t, err) + require.True(t, response.GetReady()) + require.Equal(t, epoch, response.GetAppliedConfigurationIndex()) + // Advisory admission checks ownership, not the encoded-row size verdict. + // The public leader commits below exercise the real FSM's typed guard. + require.NoError(t, kv.CheckMigrationRequest(ctx, n.store, quorumAcceptanceGroup, quorumAcceptanceWriteRequest("u", []byte("moving-small"), 101)), + "every configured member must restore coherent moving-range ownership admission") + require.NoError(t, kv.CheckMigrationRequest(ctx, n.store, quorumAcceptanceGroup, quorumAcceptanceWriteRequest("b", []byte("left"), 110))) + guards, ok := n.store.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := guards.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Len(t, states, 1, "completed job11 must not remain a live overlapping SOURCE guard") + require.Equal(t, uint64(12), states[0].JobID) + require.Equal(t, uint32(1), states[0].TransferSizeGuardVersion) + } +} + +func quorumAcceptanceWriteRequest(key string, value []byte, timestamp uint64) *pb.Request { + return &pb.Request{Ts: timestamp, ObservedRouteVersion: kv.EncodeObservedRouteVersion(quorumAcceptanceCatalogVersion), Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte(key), Value: value}}} +} + +func (c *quorumAcceptanceCluster) commitToCurrentLeader(ctx context.Context, requests []*pb.Request) (*quorumAcceptanceNode, *kv.TransactionResponse, error) { + ticker := time.NewTicker(splitPartitionApplyPoll) + defer ticker.Stop() + for { + if leader := c.currentLeader(); leader != nil { + // Selection is not a read barrier. A newly elected voter must apply + // its committed ownership controls before local request admission. + _, err := leader.engine.LinearizableRead(ctx) + if err == nil { + response, commitErr := leader.tx.Commit(ctx, requests) + if !errors.Is(commitErr, raftengine.ErrNotLeader) { + return leader, response, commitErr + } + } else if !errors.Is(err, raftengine.ErrNotLeader) { + return leader, nil, err + } + } + select { + case <-ctx.Done(): + return nil, nil, errors.WithStack(ctx.Err()) + case <-ticker.C: + } + } +} + +func (c *quorumAcceptanceCluster) currentLeader() *quorumAcceptanceNode { + for _, node := range c.nodes { + if node.engine != nil && node.engine.State() == raftengine.StateLeader { + return node + } + } + return nil +} + +func (c *quorumAcceptanceCluster) write(ctx context.Context, key string, value []byte, timestamp uint64) (*quorumAcceptanceNode, uint64, error) { + leader, response, err := c.commitToCurrentLeader(ctx, []*pb.Request{quorumAcceptanceWriteRequest(key, value, timestamp)}) + if err != nil { + return leader, 0, err + } + return leader, response.CommitIndex, nil +} + +func quorumAcceptancePrepare(phase pb.Phase, version uint64) *pb.Request { + return &pb.Request{Ts: 200, IsTxn: true, Phase: phase, ObservedRouteVersion: kv.EncodeObservedRouteVersion(version), Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("c"), CommitTS: 210, LockTTLms: 60000})}, + {Op: pb.Op_PUT, Key: []byte("c"), Value: []byte("prepared-left")}, + }} +} + +func (c *quorumAcceptanceCluster) retainedAndRestart(t *testing.T, root quorumAcceptanceRoot, epoch uint64) { + t.Helper() + ctx, cancel := context.WithTimeout(c.lifetime, 60*time.Second) + defer cancel() + started := time.Now() + large := make([]byte, store.MigrationPagePayloadBudget+(1<<19)) + _, _, err := c.write(ctx, "u", large, 100) + require.ErrorIs(t, err, kv.ErrMigrationTransferTooLarge) + _, _, err = c.write(ctx, "u", []byte("moving-small"), 101) + require.NoError(t, err, "moving-range creation must remain available below the transfer budget") + leader, target, err := c.write(ctx, "b", large, 110) + require.NoError(t, err, "retained-left writes keep the global size contract") + require.Positive(t, target) + t.Logf("retained write ACK index %d; cached status applied %d; current applied %d", target, leader.engine.Status().AppliedIndex, leader.engine.AppliedIndex()) + // Cross a snapshot interval after the value's ACK without snapshotting + // 64 MiB on every control, rejection and post-restart probe in this test. + for i := range quorumAcceptanceSnapshotEvery { + _, _, err = c.write(ctx, "a-snapshot", []byte("snapshot barrier"), uint64(111+i)) + require.NoError(t, err) + } + // All members, including the learner, must have an actual persisted native + // snapshot covering the ACKed write itself. Status() is a periodically + // refreshed diagnostic and can lag the proposal's returned CommitIndex. + require.Eventually(t, func() bool { + for _, n := range c.nodes { + if n.engine.Status().LastSnapshotIndex < target { + return false + } + } + return true + }, 5*time.Second, 20*time.Millisecond) + t.Logf("all four members persisted native snapshots covering ACKed write index %d", target) + for _, n := range c.nodes { + n.tx.Close() + n.tx = nil + require.NoError(t, n.engine.Close()) + n.engine = nil + require.NoError(t, n.store.Close()) + n.store = nil + } + require.NoError(t, ctx.Err(), "large-write/snapshot phase must finish within its own budget") + t.Logf("large-write/snapshot phase completed in %s", time.Since(started)) + cancel() + ctx, cancel = context.WithTimeout(c.lifetime, 60*time.Second) + defer cancel() + started = time.Now() + for _, n := range c.nodes { + require.NoError(t, n.open(c.lifetime, nil, false)) + } + leader = c.leader(t) + require.Equal(t, epoch, c.waitConfiguration(t, 4)) + // Election reports leadership, not completion of a linearizable data read. + readIndex, err := leader.engine.LinearizableRead(ctx) + require.NoError(t, err) + require.GreaterOrEqual(t, readIndex, target) + require.GreaterOrEqual(t, leader.engine.AppliedIndex(), readIndex) + c.checkReplicas(t, ctx, epoch) + for _, n := range c.nodes { + value, err := n.store.GetAt(ctx, []byte("b"), 1000) + require.NoError(t, err, "every voter and learner must restore the ACKed retained-left value") + require.True(t, bytes.Equal(large, value)) + } + mirror := distribution.NewEngine() + require.NoError(t, mirror.ApplySnapshot(distribution.CatalogSnapshot{Version: root.version(), Routes: root.routes()})) + reader := kv.NewShardStore(mirror, map[uint64]*kv.ShardGroup{quorumAcceptanceGroup: {Store: leader.store, Engine: leader.engine, Txn: leader.tx}}) + value, err := reader.GetGroupAt(ctx, quorumAcceptanceGroup, []byte("b"), 1000) + require.NoError(t, err) + require.True(t, bytes.Equal(large, value), "native restart must restore the ACKed retained-left value") + _, _, err = c.write(ctx, "u", large, 100) + require.ErrorIs(t, err, kv.ErrMigrationTransferTooLarge) + t.Log("typed moving-range size rejection and unrestricted retained-left oversized data survive native restart on all four members") + leader, _, err = c.commitToCurrentLeader(ctx, []*pb.Request{quorumAcceptancePrepare(pb.Phase_COMMIT, quorumAcceptanceCatalogVersion)}) + require.NoError(t, err, "native snapshot must retain PREPARE resolution evidence") + reader = kv.NewShardStore(mirror, map[uint64]*kv.ShardGroup{quorumAcceptanceGroup: {Store: leader.store, Engine: leader.engine, Txn: leader.tx}}) + value, err = reader.GetGroupAt(ctx, quorumAcceptanceGroup, []byte("c"), 1000) + require.NoError(t, err) + require.Equal(t, []byte("prepared-left"), value) + leader, _, err = c.write(ctx, "d", []byte("left-after-restart"), 1000) + require.NoError(t, err) + consumer := c.consumer(t, leader, root) + quorumAcceptanceEventually(t, func() error { return consumer.reconcileMigrationPartitions(ctx, quorumAcceptanceGroup) }) + c.checkNoQuorum(t, ctx, leader) + require.NoError(t, ctx.Err(), "restart/admission phase must finish within its own budget") + t.Logf("restart/admission phase completed in %s", time.Since(started)) +} + +func (c *quorumAcceptanceCluster) checkNoQuorum(t *testing.T, ctx context.Context, leader *quorumAcceptanceNode) { + t.Helper() + // Keep the leader AND the real learner running, but remove the other two + // voters. A learner must never turn this into a two-of-four write quorum. + for _, n := range c.nodes[:3] { + if n != leader { + n.tx.Close() + n.tx = nil + require.NoError(t, n.engine.Close()) + n.engine = nil + } + } + attempt, cancel := context.WithTimeout(ctx, 250*time.Millisecond) + defer cancel() + // Exercise proposal quorum directly: a preliminary read fence would reject + // before proposing and would not establish that the learner cannot vote. + _, err := leader.tx.Commit(attempt, []*pb.Request{quorumAcceptanceWriteRequest("e", []byte("must-not-commit"), 1100)}) + require.Error(t, err) + _, err = leader.store.GetAt(ctx, []byte("e"), 2000) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +func TestMigrationQuorumPreparedDrainAndHistoricalPreflight(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second) + defer cancel() + c := newQuorumAcceptanceCluster(t, ctx) + leader := c.leader(t) + epoch := c.waitConfiguration(t, 3) + members, err := c.members(ctx, quorumAcceptanceGroup) + require.NoError(t, err) + quorumAcceptanceEventually(t, func() error { return EnsureMigrationNamespaceAdmission(ctx, leader.client, members) }) + consumer, root := c.partitionedSourceForDrain(t, ctx, leader, epoch) + moving := root.later.Children[1] + control := quorumAcceptanceAcquire(t, ctx, leader, 12, epoch, quorumAcceptanceCatalogVersion, moving, "t") + large := make([]byte, store.MigrationPagePayloadBudget+(1<<19)) + for _, key := range []string{"u", "w"} { + prepare := quorumAcceptanceDrainRequest(pb.Phase_PREPARE, key, large) + require.Less(t, proto.Size(prepare), internalutil.GRPCMaxMessageBytes, "legacy PREPARE fits the unchanged 64 MiB transport limit") + leader, _, err = c.commitToCurrentLeader(ctx, []*pb.Request{prepare}) + require.NoError(t, err, "oversized PREPARE predates transfer-size activation") + } + locks, err := leader.client.ProbeMigrationLocks(ctx, &pb.ProbeMigrationLocksRequest{RouteStart: moving.Start, RouteEnd: moving.End, Limit: 10}) + require.NoError(t, err) + require.Equal(t, uint32(2), locks.GetPendingCount()) + job := distribution.SplitJob{JobID: 12, SourceRouteID: 5, SplitKey: []byte("t")} + consumer = c.consumer(t, leader, root) + var witness *splitTransferAdmissionWitness + quorumAcceptanceEventually(t, func() error { + witness, err = consumer.ensureSplitJobTransferControl(ctx, job, quorumAcceptanceGroup, leader.client, control, nil) + return err + }) + require.NoError(t, consumer.revalidateSplitTransferAdmission(ctx, witness)) + for _, resolution := range []struct { + phase pb.Phase + key string + }{{pb.Phase_COMMIT, "u"}, {pb.Phase_ABORT, "w"}} { + // No replacement value is sent: COMMIT must use authenticated bytes + // from the oversized old intent, not admit a new oversized creation. + leader, _, err = c.commitToCurrentLeader(ctx, []*pb.Request{quorumAcceptanceDrainRequest(resolution.phase, resolution.key, nil)}) + require.NoError(t, err, "existing oversized PREPARE must drain after guard activation") + } + locks, err = leader.client.ProbeMigrationLocks(ctx, &pb.ProbeMigrationLocksRequest{RouteStart: moving.Start, RouteEnd: moving.End, Limit: 10}) + require.NoError(t, err) + require.Zero(t, locks.GetPendingCount()) + committed, err := leader.store.GetAt(ctx, []byte("u"), 60) + require.NoError(t, err) + require.True(t, bytes.Equal(large, committed)) + _, err = leader.store.GetAt(ctx, []byte("w"), 1000) + require.ErrorIs(t, err, store.ErrKeyNotFound, "ABORT must not publish its prepared oversized value") + leader, index, err := c.write(ctx, "u", []byte("small latest"), 90) + require.NoError(t, err) + require.Eventually(t, func() bool { + for _, n := range c.nodes[:3] { + if n.engine.AppliedIndex() < index { + return false + } + } + return true + }, 5*time.Second, 20*time.Millisecond) + consumer = c.consumer(t, leader, root) + c.historicalPreflightBeforeFence(t, ctx, leader, consumer, root, large) +} + +func quorumAcceptanceDrainRequest(phase pb.Phase, key string, value []byte) *pb.Request { + return &pb.Request{Ts: 50, IsTxn: true, Phase: phase, ObservedRouteVersion: kv.EncodeObservedRouteVersion(quorumAcceptanceCatalogVersion), Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte(key), CommitTS: 60, LockTTLms: 60000})}, + {Op: pb.Op_PUT, Key: []byte(key), Value: value}, + }} +} + +func (c *quorumAcceptanceCluster) partitionedSourceForDrain(t *testing.T, ctx context.Context, leader *quorumAcceptanceNode, epoch uint64) (*DistributionServer, quorumAcceptanceRoot) { + t.Helper() + parent := distribution.RouteDescriptor{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: quorumAcceptanceGroup, MinWriteTSExclusive: 1} + c.completeInitialOwner(t, ctx, leader, epoch, parent) + left, right := parent, parent + left.RouteID, left.ParentRouteID, left.End, left.SplitAtHLC = 2, 1, []byte("m"), 1 + right.RouteID, right.ParentRouteID, right.Start, right.SplitAtHLC = 3, 1, []byte("m"), 1 + root := quorumAcceptanceRoot{manifest: distribution.OwnershipPartitionManifest{PreviousVersion: 2, Version: 3, Parent: parent, Children: []distribution.RouteDescriptor{left, right}}} + consumer := c.consumer(t, leader, root) + quorumAcceptanceEventually(t, func() error { return consumer.reconcileMigrationPartitions(ctx, quorumAcceptanceGroup) }) + kept, moving := right, right + kept.RouteID, kept.ParentRouteID, kept.End, kept.SplitAtHLC = 4, 3, []byte("t"), 2 + moving.RouteID, moving.ParentRouteID, moving.Start, moving.SplitAtHLC = 5, 3, []byte("t"), 2 + root.later = &distribution.OwnershipPartitionManifest{PreviousVersion: 3, Version: quorumAcceptanceCatalogVersion, Parent: right, Children: []distribution.RouteDescriptor{kept, moving}} + consumer = c.consumer(t, leader, root) + quorumAcceptanceEventually(t, func() error { return consumer.reconcileMigrationPartitions(ctx, quorumAcceptanceGroup) }) + return consumer, root +} + +func (c *quorumAcceptanceCluster) historicalPreflightBeforeFence(t *testing.T, ctx context.Context, leader *quorumAcceptanceNode, consumer *DistributionServer, root quorumAcceptanceRoot, large []byte) { + t.Helper() + // Only root metadata uses the explicit catalog/coordinator stub. Every + // data-group request, control, timestamp, export and verdict remains real. + metadata := store.NewMVCCStore() + t.Cleanup(func() { require.NoError(t, metadata.Close()) }) + catalog := distribution.NewCatalogStore(metadata, distribution.WithCatalogRouteDescriptorV2Writes(true)) + var version uint64 + for _, routes := range [][]distribution.RouteDescriptor{{root.manifest.Parent}, {root.manifest.Parent}, root.manifest.Children, root.routes()} { + snapshot, err := catalog.Save(ctx, version, routes) + require.NoError(t, err) + version = snapshot.Version + } + require.Equal(t, quorumAcceptanceCatalogVersion, version) + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{JobID: 12, SourceRouteID: 3, SplitKey: []byte("t"), TargetGroupID: 9}, root.later.Parent, 1000) + require.NoError(t, err) + job.WorkingSourceRouteID, job.SnapshotTS, job.Phase = 5, 40, distribution.SplitJobPhaseFence + job.WriteTrackerArmed, job.SourceRetentionPinTS = true, 1 + job.Cursor = []byte("saved-copy-progress") + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + consumer.catalog, consumer.coordinator = catalog, newDistributionCoordinatorStub(metadata, true) + _, err = consumer.splitMigrationGroupFactory(ctx, job.TargetGroupID) + require.Error(t, err, "a source-only fixture must never substitute its leader for a missing target group") + for range len(job.BracketProgress) + 3 { + // The whole runner additionally negotiates the target namespace. This + // source-only harness exercises the real preflight/fence phase and its + // compensation helper, not a fictitious cross-group runner acceptance. + err = consumer.finalizeSplitJobFence(ctx, job) + if errors.Is(err, errMigrationOversizedSourceVersion) { + require.NoError(t, consumer.abandonOversizedSplitTransfer(ctx, job, err)) + } + job = loadTransferJob(t, ctx, catalog, job.JobID) + if job.Phase == distribution.SplitJobPhaseAbandoning { + break + } + require.NoError(t, err) + } + require.True(t, errors.Is(err, errMigrationOversizedSourceVersion), "only the authenticated structured source verdict authorizes compensation") + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + details := status.Convert(err).Details() + require.Len(t, details, 1) + info, ok := details[0].(*errdetails.ErrorInfo) + require.True(t, ok) + require.Equal(t, migrationTransferErrorDomain, info.GetDomain()) + require.Equal(t, migrationOversizedVersionReason, info.GetReason()) + require.Equal(t, "60", info.GetMetadata()["commit_ts"], "the rejected historical committed version predates the latest small value") + require.Equal(t, distribution.SplitJobPhaseAbandoning, job.Phase) + require.Equal(t, distribution.SplitJobPhaseFence, job.AbandonFromPhase) + require.False(t, job.TransferPreflightDone) + require.Positive(t, job.TransferPreflightProgress[0].AcceptedRows, "preserve the valid streamed page before the later historical error") + require.NotEmpty(t, job.TransferPreflightProgress[0].Cursor) + require.Positive(t, job.TransferPreflightTS) + require.Zero(t, job.FenceTS) + require.Equal(t, []byte("saved-copy-progress"), job.Cursor) + require.NotEmpty(t, job.LastError) + snapshot, err := catalog.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, quorumAcceptanceCatalogVersion, snapshot.Version) + require.Equal(t, root.routes(), snapshot.Routes, "historical oversize must be found before publishing a catalog write fence") + require.NotEmpty(t, leader.exports) + require.Positive(t, leader.timestamps.Load()) + for _, request := range leader.exports { + require.Zero(t, request.GetMinCommitTs(), "preflight must include retained history, not only the latest small value") + require.Equal(t, job.TransferPreflightTS, request.GetMaxCommitTs()) + } + for _, n := range c.nodes[:3] { + require.Zero(t, n.writeFences.Load(), "no source write-fence control may precede full-history preflight") + reader, ok := n.store.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Len(t, states, 1) + require.Equal(t, uint32(1), states[0].TransferSizeGuardVersion) + require.False(t, states[0].SourceWriteFence) + require.False(t, states[0].SourceReadFence) + historical, err := n.store.GetAt(ctx, []byte("u"), 60) + require.NoError(t, err) + require.True(t, bytes.Equal(large, historical), "every data replica must preserve the oversized pre-guard COMMIT") + _, err = n.store.GetAt(ctx, []byte("w"), 1000) + require.ErrorIs(t, err, store.ErrKeyNotFound, "every data replica must preserve the old PREPARE's ABORT") + latest, err := n.store.GetAt(ctx, []byte("u"), 1000) + require.NoError(t, err) + require.Equal(t, []byte("small latest"), latest) + } +} diff --git a/adapter/migration_resplit_acceptance_test.go b/adapter/migration_resplit_acceptance_test.go new file mode 100644 index 000000000..30e6b6148 --- /dev/null +++ b/adapter/migration_resplit_acceptance_test.go @@ -0,0 +1,270 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// The existing runtime client executes real Internal/FSM/store commands, not +// Raft elections. Only catalog RPCs use bufconn; this is not quorum acceptance. +type resplitAcceptanceRuntime struct { + server *DistributionServer + client pb.DistributionClient + clients map[uint64]*resplitAcceptanceClient + groups map[uint64]*kv.ShardGroup + reader *kv.ShardStore +} + +type resplitAcceptanceClient struct{ *splitRuntimeClient } + +func (c *resplitAcceptanceClient) IssueMigrationTimestamp(ctx context.Context, req *pb.IssueMigrationTimestampRequest, _ ...grpc.CallOption) (*pb.IssueMigrationTimestampResponse, error) { + return c.internal.IssueMigrationTimestamp(ctx, req) +} + +func (c *resplitAcceptanceClient) ProbeMigrationLocks(ctx context.Context, req *pb.ProbeMigrationLocksRequest, _ ...grpc.CallOption) (*pb.ProbeMigrationLocksResponse, error) { + return c.internal.ProbeMigrationLocks(ctx, req) +} + +func newResplitAcceptanceRuntime(t *testing.T) *resplitAcceptanceRuntime { + t.Helper() + root, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, root.Close()) }) + catalog := distribution.NewCatalogStore(root, distribution.WithCatalogRouteDescriptorV2Writes(true)) + _, err = catalog.Save(t.Context(), 0, []distribution.RouteDescriptor{{ + RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 1}}) + require.NoError(t, err) + engine := distribution.NewEngine() + r := &resplitAcceptanceRuntime{clients: make(map[uint64]*resplitAcceptanceClient), groups: make(map[uint64]*kv.ShardGroup)} + for _, id := range []uint64{1, 2, 3} { + local, openErr := store.NewPebbleStore(t.TempDir()) + require.NoError(t, openErr) + t.Cleanup(func() { require.NoError(t, local.Close()) }) + client := &resplitAcceptanceClient{newSplitRuntimeClient(t, engine, local, id)} + r.clients[id] = client + proposer, ok := client.internal.migrationProposer.(*ownershipRuntimeProposer) + require.True(t, ok) + configuration, ok := proposer.fsm.(raftengine.AppliedConfigurationAware) + require.True(t, ok) + tx := kv.NewTransactionWithProposer(proposer, kv.WithMigrationRequestAdmission(client.internal.store, id)) + t.Cleanup(tx.Close) + r.groups[id] = &kv.ShardGroup{Store: client.internal.store, Txn: tx, + Engine: ownershipRefreshEngine{configuration: configuration}} + } + for _, key := range []string{"n", "y"} { + require.NoError(t, r.clients[1].internal.store.PutAt(t.Context(), []byte(key), []byte("original-"+key), 10, 0)) + } + r.server = NewDistributionServer(engine, catalog, + WithDistributionCoordinator(&ownershipCatalogCoordinator{distributionCoordinatorStub: newDistributionCoordinatorStub(root, true)}), + WithDistributionKnownRaftGroups(1, 2, 3), + WithSplitMigrationCapabilityGate(r.capability), + WithSplitJobRunnerReady(), WithSplitJobRunnerReadinessGate(func(context.Context) error { return nil }), + WithSplitMigrationClientFactory(func(_ context.Context, job distribution.SplitJob, _ uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return r.clients[job.SourceGroupID], r.clients[job.TargetGroupID], nil + }), + WithSplitMigrationGroupClientFactory(func(_ context.Context, id uint64) (SplitMigrationClient, error) { return r.clients[id], nil }), + WithSplitMigrationVoterFactory(func(_ context.Context, id uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "voter", Address: "voter:1", Client: r.clients[id]}}, nil + }), + WithSplitMigrationApplyMemberSnapshotFactory(func(_ context.Context, id uint64) (SplitMigrationApplyMemberSnapshot, error) { + reader, ok := r.clients[id].internal.leader.(raftengine.AppliedConfigurationReader) + if !ok { + return SplitMigrationApplyMemberSnapshot{}, store.ErrNotSupported + } + index, supported := reader.AppliedConfigurationIndex() + if !supported { + return SplitMigrationApplyMemberSnapshot{}, store.ErrNotSupported + } + return SplitMigrationApplyMemberSnapshot{ConfigurationIndex: index, + Members: []SplitMigrationVoter{{ID: "voter", Address: "voter:1", Client: r.clients[id]}}}, nil + }), + WithSplitPromotionClientFactory(func(_ context.Context, job distribution.SplitJob) (SplitPromotionClient, error) { + return r.clients[job.TargetGroupID], nil + })) + client, closeClient := newBufconnDistributionClient(t, r.server) + r.client = client + t.Cleanup(closeClient) + WithMigrationOwnershipRefresh(r.groups, func(context.Context) (MigrationOwnershipAttestor, error) { return r.client, nil })(r.server) + r.reader = kv.NewShardStore(engine, r.groups) + WithMigrationPartitionRuntime(r.reader, func(_ context.Context, id uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "voter", Address: "voter:1", Client: r.clients[id]}}, nil + })(r.server) + for id, client := range r.clients { + client.internal.migrationPartitionProtocolReady = func() bool { return r.server.MigrationPartitionRuntimeConfigured(id) } + client.internal.transferSizeReady = func() bool { + return r.server.MigrationPartitionRuntimeConfigured(id) && kv.MigrationPartitionRequestAdmissionReady(r.groups[id].Txn) + } + client.internal.migrationNamespaceReady = func() bool { + proposer, ok := client.internal.migrationProposer.(*ownershipRuntimeProposer) + return ok && r.server.MigrationPartitionRuntimeConfigured(id) && + kv.MigrationNamespaceInstalledVersion(client.internal.store, proposer.fsm) == kv.MigrationNamespaceVersion + } + } + return r +} + +func (r *resplitAcceptanceRuntime) capability(ctx context.Context) error { + for _, client := range r.clients { + response, err := client.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{JobId: 1, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY, RequireConfigurationIndex: true}) + if err != nil { + return errors.WithStack(err) + } + if !response.GetReady() || response.GetOwnershipProtocol() < 2 || response.GetAppliedConfigurationIndex() == 0 { + return errors.WithStack(store.ErrNotSupported) + } + } + return nil +} + +func (r *resplitAcceptanceRuntime) migrate(t *testing.T, route, target uint64, split string, afterTick ...func(distribution.SplitJob)) distribution.SplitJob { + t.Helper() + snapshot, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + started, err := r.client.StartSplitMigration(t.Context(), &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: snapshot.Version, RouteId: route, TargetGroupId: target, SplitKey: []byte(split)}) + require.NoError(t, err) + for range 200 { + require.NoError(t, r.server.RunSplitJobRunnerOnce(t.Context())) + job, found, loadErr := r.server.catalog.SplitJob(t.Context(), started.GetJobId()) + require.NoError(t, loadErr) + require.True(t, found) + for _, check := range afterTick { + check(job) + } + if job.Phase == distribution.SplitJobPhaseDone { + return job + } + } + t.Fatal("bounded runner did not finish migration") + return distribution.SplitJob{} +} + +func (r *resplitAcceptanceRuntime) ownership(t *testing.T, id uint64) []store.MigrationOwnershipState { + t.Helper() + reader, ok := r.groups[id].Store.(store.MigrationOwnershipStore) + require.True(t, ok) + view, err := reader.MigrationServingState(t.Context()) + require.NoError(t, err) + require.NotEmpty(t, view.Ownership) + for _, proof := range view.Ownership { + require.True(t, store.MigrationOwnershipEvidenceValid(proof)) + require.Positive(t, proof.AppliedIndex) + require.NotEmpty(t, proof.Receipts, "proof must originate from committed typed FSM controls") + } + return view.Ownership +} + +func (r *resplitAcceptanceRuntime) put(t *testing.T, group uint64, key, value string, ts uint64) error { + t.Helper() + snapshot, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + _, err = r.groups[group].Txn.Commit(t.Context(), []*pb.Request{{Ts: ts, + ObservedRouteVersion: kv.EncodeObservedRouteVersion(snapshot.Version), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte(key), Value: []byte(value)}}}}) + return err +} + +func (r *resplitAcceptanceRuntime) read(t *testing.T, key, expected string) { + t.Helper() + value, err := r.reader.GetAt(t.Context(), []byte(key), ^uint64(0)) + if assert.NoError(t, err, "public routed read of %q", key) { + assert.Equal(t, expected, string(value)) + } +} + +func TestMigrationResplitAcceptanceBaselineMigratedOwner(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + job := r.migrate(t, 1, 2, "m") + owners, nonowners := r.ownership(t, 2), r.ownership(t, 1) + require.Equal(t, store.MigrationOwner, owners[0].Role) + require.Equal(t, store.MigrationNonOwner, nonowners[0].Role) + for _, key := range []string{"n", "y"} { + r.read(t, key, "original-"+key) + require.ErrorIs(t, r.put(t, 2, key, "too-old", job.FenceTS), kv.ErrRouteWriteTimestampTooLow) + require.NoError(t, r.put(t, 2, key, "baseline", job.FenceTS+10)) + r.read(t, key, "baseline") + require.Error(t, r.put(t, 1, key, "stale-source", job.FenceTS+20)) + _, err := r.groups[1].Store.GetAt(t.Context(), []byte(key), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound) + } +} + +func TestMigrationResplitAcceptanceSameGroupChildren(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + job := r.migrate(t, 1, 2, "m") + owners, nonowners := r.ownership(t, 2), r.ownership(t, 1) + require.Equal(t, store.MigrationOwner, owners[0].Role) + require.Equal(t, store.MigrationNonOwner, nonowners[0].Role) + r.read(t, "n", "original-n") + r.read(t, "y", "original-y") + require.NoError(t, r.put(t, 2, "n", "original-n", job.FenceTS+10), "baseline migrated owner must accept writes before the re-split") + snapshot, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + split, err := r.client.SplitRange(t.Context(), &pb.SplitRangeRequest{ + ExpectedCatalogVersion: snapshot.Version, RouteId: owners[0].RouteID, SplitKey: []byte("t")}) + require.NoError(t, err) + require.Equal(t, job.FenceTS, split.GetLeft().GetMinWriteTsExclusive()) + require.Equal(t, job.FenceTS, split.GetRight().GetMinWriteTsExclusive()) + assert.NoError(t, r.server.RunMigrationOwnershipRefreshOnce(t.Context()), "re-split partition must be recoverable from committed catalog") + for _, key := range []string{"n", "y"} { + t.Run(key, func(t *testing.T) { + r.read(t, key, "original-"+key) + assert.ErrorIs(t, r.put(t, 2, key, "too-old", job.FenceTS), kv.ErrRouteWriteTimestampTooLow) + if assert.NoError(t, r.put(t, 2, key, "after-split", job.FenceTS+100)) { + r.read(t, key, "after-split") + } + assert.Error(t, r.put(t, 1, key, "stale-source", job.FenceTS+200)) + _, getErr := r.groups[1].Store.GetAt(t.Context(), []byte(key), ^uint64(0)) + assert.ErrorIs(t, getErr, store.ErrKeyNotFound) + }) + } +} + +func TestMigrationResplitAcceptanceLaterCrossGroupRetainsLeft(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + first := r.migrate(t, 1, 2, "m") + owner := r.ownership(t, 2)[0] + require.NoError(t, r.put(t, 2, "n", "original-n", first.FenceTS+10), "baseline migrated owner must accept writes before the second move") + checkedTrackerWrites := false + second := r.migrate(t, owner.RouteID, 3, "t", func(job distribution.SplitJob) { + if checkedTrackerWrites || !job.WriteTrackerArmed { + return + } + for _, key := range []string{"n", "y"} { + ts := r.groups[2].Store.LastCommitTS() + 1 + require.NoError(t, r.put(t, 2, key, "original-"+key, ts), "source tracker must preserve full-parent creation before fencing") + } + checkedTrackerWrites = true + }) + require.True(t, checkedTrackerWrites) + require.GreaterOrEqual(t, second.FenceTS, first.FenceTS) + assert.NoError(t, r.server.RunMigrationOwnershipRefreshOnce(t.Context())) + r.read(t, "n", "original-n") + r.read(t, "y", "original-y") + for _, tc := range []struct { + key string + group uint64 + floor uint64 + }{{"n", 2, first.FenceTS}, {"y", 3, second.FenceTS}} { + t.Run(tc.key, func(t *testing.T) { + assert.ErrorIs(t, r.put(t, tc.group, tc.key, "too-old", tc.floor), kv.ErrRouteWriteTimestampTooLow) + if assert.NoError(t, r.put(t, tc.group, tc.key, "second-move", second.FenceTS+100)) { + r.read(t, tc.key, "second-move") + } + }) + } + assert.Error(t, r.put(t, 2, "y", "retired-owner", second.FenceTS+200)) + _, err := r.groups[2].Store.GetAt(t.Context(), []byte("y"), ^uint64(0)) + assert.ErrorIs(t, err, store.ErrKeyNotFound) +} diff --git a/adapter/migration_retirement_test.go b/adapter/migration_retirement_test.go new file mode 100644 index 000000000..c483ac593 --- /dev/null +++ b/adapter/migration_retirement_test.go @@ -0,0 +1,190 @@ +package adapter + +import ( + "bytes" + "context" + "slices" + "sync" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type retirementVoterProposer struct { + voters []raftengine.StateMachine + index uint64 +} + +type retirementUnsupportedStore struct{ store.MVCCStore } + +type retirementContradictingStore struct{ store.MVCCStore } + +func (s retirementContradictingStore) MigrationRetired(context.Context, uint64) (bool, error) { + return true, nil +} + +func (s retirementContradictingStore) MigrationTargetReadinessStates(context.Context) ([]store.TargetStagedReadinessState, error) { + return []store.TargetStagedReadinessState{{JobID: 7}}, nil +} + +func TestMigrationRetirementProbeRequiresWitnessAndAbsentActiveMetadata(t *testing.T) { + t.Parallel() + request := &pb.ProbeMigrationStateRequest{JobId: 7, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED} + server := NewInternalWithEngine(nil, nil, nil, nil, WithInternalStore(store.NewMVCCStore())) + response, err := server.ProbeMigrationState(t.Context(), request) + require.NoError(t, err) + require.False(t, response.GetReady(), "absence alone is not retirement") + server = NewInternalWithEngine(nil, nil, nil, nil, WithInternalStore(retirementUnsupportedStore{store.NewMVCCStore()})) + _, err = server.ProbeMigrationState(t.Context(), request) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + server = NewInternalWithEngine(nil, nil, nil, nil, WithInternalStore(retirementContradictingStore{store.NewMVCCStore()})) + response, err = server.ProbeMigrationState(t.Context(), request) + require.NoError(t, err) + require.False(t, response.GetReady(), "retirement cannot hide an active guard") +} + +func (p *retirementVoterProposer) Propose(_ context.Context, data []byte) (*raftengine.ProposalResult, error) { + p.index++ + var response any + for _, voter := range p.voters { + aware, ok := voter.(raftengine.ApplyIndexAware) + if !ok { + return nil, store.ErrNotSupported + } + aware.SetApplyIndex(p.index) + response = voter.Apply(data) + } + return &raftengine.ProposalResult{CommitIndex: p.index, Response: response}, nil +} + +func (p *retirementVoterProposer) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { + return p.Propose(ctx, data) +} + +type heldRetirementProposer struct { + *retirementVoterProposer + command []byte + started chan struct{} + release chan struct{} +} + +func (p *heldRetirementProposer) Propose(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { + if bytes.Equal(data, p.command) { + close(p.started) + select { + case <-p.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return p.retirementVoterProposer.Propose(ctx, data) +} + +func (p *heldRetirementProposer) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { + return p.Propose(ctx, data) +} + +func TestMigrationRetirementRejectsOldRunnerAfterAllVoterCleanupProbes(t *testing.T) { + t.Parallel() + ctx := t.Context() + engine := distribution.NewEngine() + catalog := distribution.NewCatalogStore(store.NewMVCCStore(), distribution.WithCatalogRouteDescriptorV2Writes(true)) + route := distribution.RouteDescriptor{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 7, State: distribution.RouteStateWriteFenced} + snapshot, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{route}) + require.NoError(t, err) + require.NoError(t, engine.ApplySnapshot(snapshot)) + proposer := &retirementVoterProposer{index: 20} + stores := make([]store.MVCCStore, 3) + servers := make([]*Internal, 3) + members := []uint64{etcd.DeriveNodeID("voter1"), etcd.DeriveNodeID("voter2"), etcd.DeriveNodeID("voter3")} + slices.Sort(members) + for index := range stores { + st := store.NewMVCCStore() + stores[index] = st + writer, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, store.TargetStagedReadinessState{JobID: 7, MigrationJobID: 7, + RouteStart: []byte("m"), RouteEnd: []byte("z"), Armed: true, TrackWrites: true, MinWriteTSExclusive: 1, RetentionPinTS: 1})) + fsm := kv.NewKvFSMWithHLC(st, kv.NewHLC(), kv.WithRouteHistory(kv.WrapDistributionEngine(engine), 7)) + configuration, ok := fsm.(raftengine.AppliedConfigurationAware) + require.True(t, ok) + inventory, ok := fsm.(raftengine.NamespaceConfigurationAware) + require.True(t, ok) + require.NoError(t, inventory.RecordAppliedNamespaceConfiguration(9, members)) + proposer.voters = append(proposer.voters, fsm) + servers[index] = NewInternalWithEngine(nil, ownershipRuntimeLeader{configuration: configuration}, nil, nil, WithInternalStore(st), + WithInternalMigrationProposer(proposer), WithInternalMigrationOwnershipReady(func() bool { return true }), + WithMigrationNamespaceAdmissionReady(func() bool { return kv.MigrationNamespaceInstalledVersion(st, fsm) == kv.MigrationNamespaceVersion }), + WithInternalMigrationCleanupGate(func(context.Context) error { return nil })) + } + commandActivation := kv.NamespaceAdmissionCommand{Version: kv.MigrationNamespaceVersion, ConfigurationIndex: 9, ApplyMembers: members} + for _, member := range members { + commandActivation.Members = append(commandActivation.Members, kv.NamespaceMemberCapability{MemberID: member, Version: kv.MigrationNamespaceVersion, ConfigurationIndex: 9}) + } + require.NoError(t, kv.ActivateMigrationNamespace(ctx, stores[0], proposer, commandActivation)) + late := &pb.TargetStagedReadinessRequest{JobId: 7, MigrationJobId: 7, RouteStart: []byte("m"), RouteEnd: []byte("z"), + Armed: true, SourceWriteFence: true, MinWriteTsExclusive: 500, RetentionPinTs: 1} + command, err := kv.MarshalTargetStagedReadinessCommand(late) + require.NoError(t, err) + held := &heldRetirementProposer{retirementVoterProposer: proposer, command: command, started: make(chan struct{}), release: make(chan struct{})} + release := sync.OnceFunc(func() { close(held.release) }) + t.Cleanup(release) + oldRunner := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(stores[0]), + WithInternalMigrationProposer(held), WithInternalMigrationImportGate(func(context.Context) error { return nil }), + WithMigrationNamespaceAdmissionReady(func() bool { + return kv.MigrationNamespaceInstalledVersion(stores[0], proposer.voters[0]) == kv.MigrationNamespaceVersion + })) + result := make(chan error, 1) + go func() { _, callErr := oldRunner.ApplyTargetStagedReadiness(ctx, late); result <- callErr }() + select { + case <-held.started: + case callErr := <-result: + t.Fatalf("old RPC did not reach the proposal barrier: %v", callErr) + } + // Commit the rollback descriptor before atomically replacing the guard. + route.State, route.MinWriteTSExclusive = distribution.RouteStateActive, 1 + snapshot, err = catalog.Save(ctx, snapshot.Version, []distribution.RouteDescriptor{route}) + require.NoError(t, err) + require.NoError(t, engine.ApplySnapshot(snapshot)) + job := distribution.SplitJob{JobID: 7, SourceRouteID: 1, SourceGroupID: 7, TargetGroupID: 9, + SplitKey: []byte("m"), Phase: distribution.SplitJobPhaseAbandoning} + ownership, err := splitOwnershipProof(snapshot, job, 7, "0123456789abcdef0123456789abcdef") + require.NoError(t, err) + _, err = servers[0].CleanupMigration(ctx, &pb.CleanupMigrationRequest{JobId: 7, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR, Ownership: ownership, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: 9}) + require.NoError(t, err) + for _, server := range servers { + proof, probeErr := server.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{JobId: 7, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED}) + require.NoError(t, probeErr) + require.True(t, proof.GetReady()) + } + release() + require.ErrorIs(t, <-result, store.ErrMigrationRetired) + for _, st := range stores { + require.Equal(t, uint64(1), st.LastCommitTS(), "a late control must not raise the clock floor") + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, readErr := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, readErr) + require.Empty(t, states, "no source guard or retention pin may return") + } + txn := kv.NewTransactionWithProposer(proposer) + t.Cleanup(txn.Close) + _, err = txn.Commit(ctx, []*pb.Request{{Ts: 100, ObservedRouteVersion: kv.EncodeObservedRouteVersion(snapshot.Version), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("active")}}}}) + require.NoError(t, err) + for _, st := range stores { + value, readErr := st.GetAt(ctx, []byte("n"), 100) + require.NoError(t, readErr) + require.Equal(t, []byte("active"), value) + } +} diff --git a/adapter/migration_transfer_error.go b/adapter/migration_transfer_error.go new file mode 100644 index 000000000..462281ef9 --- /dev/null +++ b/adapter/migration_transfer_error.go @@ -0,0 +1,52 @@ +package adapter + +import ( + "crypto/sha256" + "fmt" + "strconv" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const migrationTransferErrorDomain = "elastickv.migration.transfer" +const migrationOversizedVersionReason = "OVERSIZED_MVCC_VERSION" + +var errMigrationOversizedSourceVersion = errors.New("authenticated source version exceeds migration transfer budget") + +func migrationOversizedVersionError(v *pb.MVCCVersion) error { + digest := sha256.Sum256(v.GetKey()) + // Preserve the legacy runner's explicit-failure classification. Only the + // new structured detail authorizes the new runner's compensation path. + s := status.New(codes.FailedPrecondition, fmt.Sprintf("migration version exceeds transfer budget; key %.128q commit_ts=%d value_bytes=%d key_sha256=%x", v.GetKey(), v.GetCommitTs(), len(v.GetValue()), digest)) + withDetail, err := s.WithDetails(&errdetails.ErrorInfo{ + Domain: migrationTransferErrorDomain, Reason: migrationOversizedVersionReason, + Metadata: map[string]string{ + "key_sha256": fmt.Sprintf("%x", digest), "commit_ts": strconv.FormatUint(v.GetCommitTs(), 10), + "value_bytes": strconv.Itoa(len(v.GetValue())), "budget_bytes": strconv.Itoa(store.MigrationPagePayloadBudget), + }, + }) + if err != nil { + return errors.WithStack(s.Err()) + } + return errors.WithStack(withDetail.Err()) +} + +// Only call at the authenticated source ExportRangeVersions boundary. Errors +// from target import, local I/O, transport limits or generic backpressure must +// never acquire the authority to cancel a migration. +func migrationSourceExportError(err error) error { + if status.Code(err) != codes.FailedPrecondition { + return err + } + for _, detail := range status.Convert(err).Details() { + if info, ok := detail.(*errdetails.ErrorInfo); ok && info.GetDomain() == migrationTransferErrorDomain && info.GetReason() == migrationOversizedVersionReason { + return errors.WithStack(errors.Mark(err, errMigrationOversizedSourceVersion)) + } + } + return err +} diff --git a/adapter/migration_transfer_error_test.go b/adapter/migration_transfer_error_test.go new file mode 100644 index 000000000..a5ad4f10d --- /dev/null +++ b/adapter/migration_transfer_error_test.go @@ -0,0 +1,132 @@ +package adapter + +import ( + "context" + "io" + "testing" + + "github.com/bootjp/elastickv/distribution" + internalutil "github.com/bootjp/elastickv/internal" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +func TestMigrationExportOversizeHasExplicitDetail(t *testing.T) { + resp := &pb.ExportRangeVersionsResponse{Versions: []*pb.MVCCVersion{{ + Key: []byte("n"), Value: make([]byte, store.MigrationPagePayloadBudget+1), CommitTs: 17, + }}} + err := checkMigrationExportPageSize(resp) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + details := status.Convert(err).Details() + require.Len(t, details, 1) + info, ok := details[0].(*errdetails.ErrorInfo) + require.True(t, ok) + require.Equal(t, "elastickv.migration.transfer", info.GetDomain()) + require.Equal(t, "OVERSIZED_MVCC_VERSION", info.GetReason()) +} + +func TestMigrationTransferBoundaryFitsExportAndImportRequests(t *testing.T) { + key := make([]byte, store.MaxSnapshotKeySize) + value := make([]byte, store.MigrationPagePayloadBudget-2*len(key)-128) + version := &pb.MVCCVersion{Key: key, Value: value, CommitTs: ^uint64(0), KeyFamily: distribution.MigrationFamilyUser} + export := &pb.ExportRangeVersionsResponse{Versions: []*pb.MVCCVersion{version}, NextCursor: append(make([]byte, 12), key...)} + require.True(t, store.MigrationVersionFitsTransfer(key, value)) + require.NoError(t, checkMigrationExportPageSize(export)) + require.Less(t, proto.Size(export), internalutil.GRPCMaxMessageBytes) + request := &pb.ImportRangeVersionsRequest{JobId: ^uint64(0), Versions: export.Versions, Cursor: export.NextCursor, BracketId: ^uint64(0), BatchSeq: ^uint64(0)} + require.Less(t, proto.Size(request), internalutil.GRPCMaxMessageBytes) + encoded, err := proto.Marshal(request) + require.NoError(t, err) + require.Len(t, encoded, proto.Size(request)) + version.Value = append(version.Value, 0) + require.Equal(t, codes.FailedPrecondition, status.Code(checkMigrationExportPageSize(export))) +} + +func TestMigrationTransferErrorDetailsAreNotGenericBackpressure(t *testing.T) { + typed := migrationOversizedVersionError(&pb.MVCCVersion{Key: []byte("n"), CommitTs: 17}) + wire, err := proto.Marshal(status.Convert(typed).Proto()) + require.NoError(t, err) + decoded := status.New(codes.OK, "").Proto() + require.NoError(t, proto.Unmarshal(wire, decoded)) + require.True(t, errors.Is(migrationSourceExportError(status.FromProto(decoded).Err()), errMigrationOversizedSourceVersion)) + wrong, err := status.New(codes.FailedPrecondition, "oversized").WithDetails(&errdetails.ErrorInfo{Domain: "other", Reason: migrationOversizedVersionReason}) + require.NoError(t, err) + for _, candidate := range []error{status.Error(codes.ResourceExhausted, "disk full"), status.Error(codes.FailedPrecondition, "migration export page exceeds limit"), wrong.Err(), context.Canceled} { + require.False(t, errors.Is(migrationSourceExportError(candidate), errMigrationOversizedSourceVersion)) + } +} + +type transferFailureStream struct { + grpc.ClientStream + first *pb.ExportRangeVersionsResponse + err error +} + +func (s *transferFailureStream) Recv() (*pb.ExportRangeVersionsResponse, error) { + if s.first != nil { + first := s.first + s.first = nil + return first, nil + } + return nil, s.err +} + +func TestMigrationTypedSourceFailureAfterAcknowledgedPageAutoAbandons(t *testing.T) { + ctx, server, catalog, job, source := transferPreflightFixture(t) + job.Phase = distribution.SplitJobPhaseBackfill + require.NoError(t, server.updateSplitJobViaCoordinator(ctx, job.JobID, func(distribution.SplitJob) (distribution.SplitJob, error) { return job, nil })) + source.exportFn = func(context.Context, *pb.ExportRangeVersionsRequest) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) { + return &transferFailureStream{first: &pb.ExportRangeVersionsResponse{Versions: []*pb.MVCCVersion{{Key: []byte("n"), Value: []byte("v"), CommitTs: 5}}, NextCursor: []byte("acked")}, err: migrationOversizedVersionError(&pb.MVCCVersion{Key: []byte("o"), CommitTs: 7})}, nil + } + err := server.runSplitJobAttempt(ctx, job) + require.True(t, errors.Is(err, errMigrationOversizedSourceVersion)) + current := loadTransferJob(t, ctx, catalog, job.JobID) + require.Equal(t, distribution.SplitJobPhaseAbandoning, current.Phase) + require.Equal(t, []byte("acked"), current.Cursor, "retain current durable copy progress, not the stale attempt's cursor") + require.Contains(t, current.LastError, "transfer budget") +} + +type transferTargetFailure struct { + splitMigrationClientStub + err error +} + +func (c *transferTargetFailure) ImportRangeVersions(context.Context, *pb.ImportRangeVersionsRequest, ...grpc.CallOption) (*pb.ImportRangeVersionsResponse, error) { + return nil, c.err +} + +func TestMigrationTargetFailureCannotAuthorizeAbandon(t *testing.T) { + ctx, server, _, job, _, _ := importReplayFixture(t) + err := migrationOversizedVersionError(&pb.MVCCVersion{Key: []byte("n")}) + target := &transferTargetFailure{err: err} + stream := &transferFailureStream{first: &pb.ExportRangeVersionsResponse{Versions: []*pb.MVCCVersion{{Key: []byte("n"), CommitTs: 5}}, Done: true}, err: io.EOF} + copyErr := server.copySplitJobStream(ctx, job, 0, distribution.MigrationBracket{}, job.BracketProgress[0], stream, target) + require.Error(t, copyErr) + require.False(t, errors.Is(copyErr, errMigrationOversizedSourceVersion)) +} + +func TestMigrationLegacySourceErrorKeepsExplicitFailure(t *testing.T) { + ctx, server, catalog, job, source := transferPreflightFixture(t) + require.NoError(t, server.finalizeSplitJobFence(ctx, job)) + job = loadTransferJob(t, ctx, catalog, job.JobID) + source.exportFn = func(context.Context, *pb.ExportRangeVersionsRequest) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) { + return nil, status.Error(codes.FailedPrecondition, "migration export page exceeds limit") + } + for range maxSplitJobPhaseFailures { + err := server.runSplitJobAttempt(ctx, job) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.False(t, errors.Is(err, errMigrationOversizedSourceVersion)) + } + failed := loadTransferJob(t, ctx, catalog, job.JobID) + require.Equal(t, distribution.SplitJobPhaseFailed, failed.Phase) + require.Equal(t, distribution.SplitJobPhaseFence, failed.RetryPhase) + require.Equal(t, distribution.SplitJobPhaseNone, failed.AbandonFromPhase) + require.Contains(t, failed.LastError, "migration export page exceeds limit") +} diff --git a/adapter/redis_compat_commands_stream_test.go b/adapter/redis_compat_commands_stream_test.go index 3aaf554da..832990fb6 100644 --- a/adapter/redis_compat_commands_stream_test.go +++ b/adapter/redis_compat_commands_stream_test.go @@ -361,19 +361,27 @@ func TestRedis_StreamXReadLatencyIsConstant(t *testing.T) { rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) defer func() { _ = rdb.Close() }() - ctx := context.Background() + ctx := t.Context() const probes = 100 lastID := seedStreamEntriesForXReadLatency(t, nodes[0].redisServer, ctx, "stream-lat") measure := func() time.Duration { - start := time.Now() - streams, err := rdb.XRead(ctx, &redis.XReadArgs{ - Streams: []string{"stream-lat", lastID}, - Count: 10, - Block: 10 * time.Millisecond, - }).Result() - elapsed := time.Since(start) + var ( + streams []redis.XStream + elapsed time.Duration + ) + err := retryNotLeader(ctx, func() error { + start := time.Now() + var xerr error + streams, xerr = rdb.XRead(ctx, &redis.XReadArgs{ + Streams: []string{"stream-lat", lastID}, + Count: 10, + Block: 10 * time.Millisecond, + }).Result() + elapsed = time.Since(start) + return xerr + }) require.True(t, errors.Is(err, redis.Nil) || err == nil) require.Empty(t, streams) return elapsed diff --git a/adapter/redis_delta_compactor_test.go b/adapter/redis_delta_compactor_test.go index b6fedb4e6..9bb69e9b0 100644 --- a/adapter/redis_delta_compactor_test.go +++ b/adapter/redis_delta_compactor_test.go @@ -2122,8 +2122,11 @@ func TestDeltaCompactor_UrgentCompactionPagination(t *testing.T) { require.NoError(t, st.PutAt(ctx, dKey, delta, i+1, 0)) } - // Queue and process the urgent compaction. - c.TriggerUrgentCompaction("hash", userKey) + // Exercise the urgent pagination loop directly. Running through c.Run would + // also start an initial background SyncOnce; the local test coordinator applies + // elems one-by-one instead of atomically, so a test read can observe the meta + // update before all delete elems have been applied under the race detector. + c.compactUrgentKey(ctx, urgentCompactionRequest{typeName: "hash", userKey: userKey}) runCtx, cancel := context.WithCancel(ctx) defer cancel() diff --git a/adapter/split_job_abandon_source_test.go b/adapter/split_job_abandon_source_test.go new file mode 100644 index 000000000..58b30f6a0 --- /dev/null +++ b/adapter/split_job_abandon_source_test.go @@ -0,0 +1,100 @@ +package adapter + +import ( + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type abandonUnavailableTarget struct{ splitMigrationClientStub } + +func (c *abandonUnavailableTarget) CleanupMigration(context.Context, *pb.CleanupMigrationRequest, ...grpc.CallOption) (*pb.CleanupMigrationResponse, error) { + return nil, status.Error(codes.Unavailable, "target offline") +} + +func TestSplitJobAbandonRestoresSourceBeforeUnavailableTarget(t *testing.T) { + t.Parallel() + for _, phase := range []distribution.SplitJobPhase{distribution.SplitJobPhaseFence, distribution.SplitJobPhaseDeltaCopy} { + for _, barrierReady := range []bool{false, true} { + t.Run(map[distribution.SplitJobPhase]string{distribution.SplitJobPhaseFence: "fence", distribution.SplitJobPhaseDeltaCopy: "delta"}[phase]+map[bool]string{false: "/pending-voter", true: "/ready"}[barrierReady], func(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st, distribution.WithCatalogRouteDescriptorV2Writes(true)) + parent := distribution.RouteDescriptor{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 7} + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 7, SourceRouteID: 1, TargetGroupID: 9, SplitKey: []byte("m"), + }, parent, time.Now().UnixMilli()) + require.NoError(t, err) + job.Phase, job.AbandonFromPhase = distribution.SplitJobPhaseAbandoning, phase + _, err = catalog.Save(ctx, 0, []distribution.RouteDescriptor{ + {RouteID: 2, ParentRouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 7}, + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 7, State: distribution.RouteStateWriteFenced}, + }) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + sourceStore := store.NewMVCCStore() + writer, ok := sourceStore.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, store.TargetStagedReadinessState{ + JobID: job.JobID, MigrationJobID: job.JobID, Armed: true, SourceWriteFence: true, + RouteStart: job.SplitKey, RouteEnd: []byte("z"), MinWriteTSExclusive: 1, RetentionPinTS: 1, + })) + source, _ := newOwnershipRuntimeClient(t, sourceStore, 7) + source.blockGuard = !barrierReady + source.beforeCleanup = func() { + snapshot, loadErr := catalog.Snapshot(ctx) + require.NoError(t, loadErr) + require.Nil(t, abandonedSplitJobFencedRoute(snapshot.Routes, job), "actual rollback must precede atomic OWNER/guard replacement") + } + target := &abandonUnavailableTarget{} + server := NewDistributionServer(distribution.NewEngine(), catalog, + WithDistributionCoordinator(&ownershipCatalogCoordinator{distributionCoordinatorStub: newDistributionCoordinatorStub(st, true)}), + WithSplitMigrationVoterFactory(func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "source", Address: "source:1", Client: source}}, nil + }), + WithSplitMigrationGroupClientFactory(func(_ context.Context, groupID uint64) (SplitMigrationClient, error) { + require.Equal(t, uint64(7), groupID) + return source, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + require.NotEmpty(t, source.cleanups, "target lookup must follow source cleanup") + return source, target, nil + }), + ) + for range 3 { + _ = server.abandonSplitJob(ctx, job) + var found bool + job, found, err = catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + } + snapshot, err := catalog.Snapshot(ctx) + require.NoError(t, err) + reader, ok := sourceStore.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + if barrierReady { + require.Empty(t, states, "a proven source recovers without waiting for the target") + require.Nil(t, abandonedSplitJobFencedRoute(snapshot.Routes, job)) + require.Equal(t, splitCleanupDoneCursor, job.SourceCutoverAckCursor) + require.ErrorContains(t, server.abandonSplitJob(ctx, job), "target offline") + } else { + require.Len(t, states, 1, "an unproven voter must retain the source guard") + require.NotNil(t, abandonedSplitJobFencedRoute(snapshot.Routes, job)) + require.Empty(t, job.SourceCutoverAckCursor) + } + require.Equal(t, distribution.SplitJobPhaseAbandoning, job.Phase) + }) + } + } +} diff --git a/adapter/split_job_cleanup_membership_test.go b/adapter/split_job_cleanup_membership_test.go new file mode 100644 index 000000000..f24e9ccda --- /dev/null +++ b/adapter/split_job_cleanup_membership_test.go @@ -0,0 +1,70 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestSplitJobSourceCleanupReopensCompletedMembershipProofs(t *testing.T) { + for _, dataDone := range []bool{false, true} { + t.Run(map[bool]string{false: "before data deletion", true: "before guard release"}[dataDone], func(t *testing.T) { + ctx := context.Background() + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st) + _, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 7}}) + require.NoError(t, err) + job := sampleDistributionSplitJob(17) + job.Phase = distribution.SplitJobPhaseCleanup + job.SourceGroupID = 7 + job.TargetPromotionDone = true + job.TargetClearedDescriptorAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + job.SourceCutoverAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + job.SourceReadDrainCursor = distribution.CloneBytes(splitCleanupDoneCursor) + job.SourceRetentionPinTS = 90 + if dataDone { + job.Cursor = distribution.CloneBytes(splitSourceCleanupDoneCursor) + } + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + source := &splitMigrationClientStub{} + n1 := &splitMigrationClientStub{} + n2 := &splitMigrationClientStub{} + voters := []SplitMigrationVoter{{ID: "n1", Address: "n1:50051", Client: n1}} + server := NewDistributionServer(distribution.NewEngine(), catalog, + WithDistributionCoordinator(newDistributionCoordinatorStub(st, true)), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, &splitMigrationClientStub{}, nil + }), + WithSplitMigrationVoterFactory(func(context.Context, uint64) ([]SplitMigrationVoter, error) { return voters, nil })) + complete, err := server.refreshSplitJobSourceCleanupBarriers(ctx, job) + require.NoError(t, err) + require.True(t, complete) + job, _, err = catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + acks, err := decodeSplitVoterAckCursor(job.SourceCutoverAckCursor) + require.NoError(t, err) + require.Len(t, acks, 1, "completion must retain voter identity, not a done sentinel") + n2.probeFn = func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + return &pb.ProbeMigrationStateResponse{}, nil + } + voters = append(voters, SplitMigrationVoter{ID: "n2", Address: "n2:50051", Client: n2}) + require.NoError(t, server.cleanupVerifiedSplitJob(ctx, job)) + require.Empty(t, source.cleanups, "a new stale voter must block data deletion and guard removal") + job, _, err = catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.Equal(t, uint64(90), job.SourceRetentionPinTS) + acks, err = decodeSplitVoterAckCursor(job.SourceCutoverAckCursor) + require.NoError(t, err) + require.Len(t, acks, 2) + require.False(t, acks[1].acked) + n2.probeFn = nil + complete, err = server.refreshSplitJobSourceCleanupBarriers(ctx, job) + require.NoError(t, err) + require.True(t, complete) + }) + } +} diff --git a/adapter/split_job_configuration.go b/adapter/split_job_configuration.go new file mode 100644 index 000000000..0d41f90a7 --- /dev/null +++ b/adapter/split_job_configuration.go @@ -0,0 +1,89 @@ +package adapter + +import ( + "context" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" +) + +func splitCleanupAckConfiguration(acks []splitVoterAck) uint64 { + if len(acks) == 0 { + return 0 + } + index := acks[0].configurationIndex + for _, ack := range acks { + if !ack.acked || ack.configurationIndex != index { + return 0 + } + } + return index +} + +func splitCleanupCursorConfiguration(cursor []byte) (uint64, error) { + acks, err := decodeSplitVoterAckCursor(reusableSplitCleanupAckCursor(cursor)) + if err != nil { + return 0, err + } + index := splitCleanupAckConfiguration(acks) + if index == 0 { + return 0, errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + return index, nil +} + +func (s *DistributionServer) bindSplitSourceCleanup(ctx context.Context, job distribution.SplitJob, source SplitMigrationClient, routeEnd []byte, req *pb.CleanupMigrationRequest) error { + var index uint64 + var err error + if job.Phase == distribution.SplitJobPhaseCleanup { + index, err = s.splitCompletedSourceConfiguration(ctx, job) + } else { + index, err = s.splitAbandonedSourceConfiguration(ctx, job, source, routeEnd) + } + if err != nil { + return err + } + req.RequireConfigurationIndex = true + req.ExpectedConfigurationIndex = index + return nil +} + +func (s *DistributionServer) splitCompletedSourceConfiguration(ctx context.Context, job distribution.SplitJob) (uint64, error) { + complete, err := s.refreshSplitJobSourceCleanupBarriers(ctx, job) + if err != nil { + return 0, err + } + if !complete { + return 0, errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + current, found, err := s.catalog.SplitJob(ctx, job.JobID) + if err != nil || !found { + return 0, errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + return splitCleanupCursorConfiguration(current.SourceReadDrainCursor) +} + +func (s *DistributionServer) splitAbandonedSourceConfiguration(ctx context.Context, job distribution.SplitJob, source SplitMigrationClient, routeEnd []byte) (uint64, error) { + groupID, err := s.splitJobSourceGroupID(ctx, job) + if err != nil { + return 0, err + } + // A repeated clear may already have retired this job. Both proofs require + // an upgraded all-voter configuration attestation; zero never authorizes it. + for _, kind := range []pb.MigrationStateProbeKind{ + pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_GUARD_PRESENT, + pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + } { + cursor, complete, err := s.syncSplitMigrationVoterBarrier(ctx, groupID, nil, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, Kind: kind, RouteStart: job.SplitKey, RouteEnd: routeEnd, RequireConfigurationIndex: true, + }) + if err != nil { + return 0, err + } + if complete { + return splitCleanupCursorConfiguration(cursor) + } + } + return 0, errors.WithStack(errSplitMigrationVoterBarrierIncomplete) +} diff --git a/adapter/split_job_configuration_test.go b/adapter/split_job_configuration_test.go new file mode 100644 index 000000000..9f660fa73 --- /dev/null +++ b/adapter/split_job_configuration_test.go @@ -0,0 +1,72 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +type migrationConfigurationLeader struct { + mockInternalLeader + index uint64 +} + +func (l migrationConfigurationLeader) AppliedConfigurationIndex() (uint64, bool) { + return l.index, l.index != 0 +} + +func TestMigrationConfigurationProbeRequiresOrderedRuntimeHook(t *testing.T) { + ctx := context.Background() + st := store.NewMVCCStore() + fsm := kv.NewKvFSMWithHLC(st, nil) + aware, ok := fsm.(raftengine.AppliedConfigurationAware) + require.True(t, ok) + require.NoError(t, aware.RecordAppliedConfiguration(9)) + req := &pb.ProbeMigrationStateRequest{JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED, RequireConfigurationIndex: true} + unsupported := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(st)) + resp, err := unsupported.ProbeMigrationState(ctx, req) + require.NoError(t, err) + require.False(t, resp.GetReady(), "a copied durable row cannot make an unsupported runtime attest") + require.Zero(t, resp.GetAppliedConfigurationIndex()) + supported := NewInternalWithEngine(nil, migrationConfigurationLeader{index: 9}, nil, nil, WithInternalStore(st)) + resp, err = supported.ProbeMigrationState(ctx, req) + require.NoError(t, err) + require.True(t, resp.GetReady()) + require.Equal(t, uint64(9), resp.GetAppliedConfigurationIndex()) + req.ExpectedConfigurationIndex = 8 + resp, err = supported.ProbeMigrationState(ctx, req) + require.NoError(t, err) + require.False(t, resp.GetReady()) +} + +func TestSplitCleanupConfigurationBarrierRejectsMixedAndMissingEpochs(t *testing.T) { + for _, secondIndex := range []uint64{0, 9, 10} { + t.Run(map[uint64]string{0: "old runtime", 9: "same configuration", 10: "new configuration"}[secondIndex], func(t *testing.T) { + n1 := &splitMigrationClientStub{probeFn: func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + return &pb.ProbeMigrationStateResponse{Ready: true, AppliedConfigurationIndex: 9}, nil + }} + n2 := &splitMigrationClientStub{probeFn: func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + return &pb.ProbeMigrationStateResponse{Ready: true, AppliedConfigurationIndex: secondIndex}, nil + }} + server := &DistributionServer{splitMigrationVoterFactory: func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "n1", Address: "n1", Client: n1}, {ID: "n2", Address: "n2", Client: n2}}, nil + }} + cursor, complete, err := server.syncSplitMigrationVoterBarrier(t.Context(), 7, nil, nil, + &pb.ProbeMigrationStateRequest{JobId: 7, RequireConfigurationIndex: true}) + require.NoError(t, err) + require.Equal(t, secondIndex == 9, complete) + _, err = splitCleanupCursorConfiguration(cursor) + if complete { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, errSplitMigrationVoterBarrierIncomplete) + } + }) + } +} diff --git a/adapter/split_job_copy_timeout_test.go b/adapter/split_job_copy_timeout_test.go new file mode 100644 index 000000000..8aad2d934 --- /dev/null +++ b/adapter/split_job_copy_timeout_test.go @@ -0,0 +1,124 @@ +package adapter + +import ( + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type stalledSplitCopyClient struct { + *splitRuntimeClient + export bool + exportStarted bool + importStarted bool +} + +func (s *stalledSplitCopyClient) ExportRangeVersions(ctx context.Context, req *pb.ExportRangeVersionsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) { + if s.export { + s.exportStarted = true + return &stalledSplitCopyStream{ctx: ctx}, nil + } + return s.splitRuntimeClient.ExportRangeVersions(ctx, req, opts...) +} + +func (s *stalledSplitCopyClient) ImportRangeVersions(ctx context.Context, _ *pb.ImportRangeVersionsRequest, _ ...grpc.CallOption) (*pb.ImportRangeVersionsResponse, error) { + s.importStarted = true + <-ctx.Done() + return nil, ctx.Err() +} + +type stalledSplitCopyStream struct { + grpc.ClientStream + ctx context.Context +} + +type freshBudgetImportClient struct { + *realImportTargetStub + t *testing.T +} + +func (s *freshBudgetImportClient) ImportRangeVersions(ctx context.Context, req *pb.ImportRangeVersionsRequest, opts ...grpc.CallOption) (*pb.ImportRangeVersionsResponse, error) { + s.t.Helper() + require.NoError(s.t, ctx.Err()) + deadline, ok := ctx.Deadline() + require.True(s.t, ok) + require.Greater(s.t, time.Until(deadline), defaultSplitMigrationAttemptTimeout/2) + return s.realImportTargetStub.ImportRangeVersions(ctx, req, opts...) +} + +type expiredExportBudgetStream struct { + grpc.ClientStream + ctx context.Context +} + +func (s *expiredExportBudgetStream) Recv() (*pb.ExportRangeVersionsResponse, error) { + <-s.ctx.Done() + return &pb.ExportRangeVersionsResponse{NextCursor: []byte("cursor"), Done: true}, nil +} + +func TestSplitJobImportBudgetIsIndependentOfExport(t *testing.T) { + t.Parallel() + ctx, server, catalog, job, bracket, target := importReplayFixture(t) + exportCtx, cancel := context.WithCancel(ctx) + cancel() + stream := &expiredExportBudgetStream{ctx: exportCtx} + client := &freshBudgetImportClient{realImportTargetStub: target, t: t} + require.NoError(t, server.copySplitJobStream(ctx, job, 0, bracket, job.BracketProgress[0], stream, client)) + persisted, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, []byte("cursor"), persisted.BracketProgress[0].Cursor) +} + +func (s *stalledSplitCopyStream) Recv() (*pb.ExportRangeVersionsResponse, error) { + <-s.ctx.Done() + return nil, s.ctx.Err() +} + +func TestSplitJobCopyAttemptBoundsStalledExportAndImport(t *testing.T) { + t.Parallel() + for _, export := range []bool{true, false} { + t.Run(map[bool]string{true: "export stream", false: "import call"}[export], func(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st, distribution.WithCatalogRouteDescriptorV2Writes(true)) + _, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 7}}) + require.NoError(t, err) + job := splitJobForSourceGroupTest(t) + job.Phase = distribution.SplitJobPhaseBackfill + job.SnapshotTS = catalog.LatestCommitTS() + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + server, source, target := newSplitRuntimeRunner(t, st, catalog, job) + _, complete, err := server.armSplitSourceOwnership(ctx, job, job.SourceGroupID, source) + require.NoError(t, err) + require.True(t, complete) + sourceClient := &stalledSplitCopyClient{splitRuntimeClient: source, export: export} + targetClient := &stalledSplitCopyClient{splitRuntimeClient: target} + server.splitMigrationClientFactory = func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return sourceClient, targetClient, nil + } + start := time.Now() + err = server.copySplitJobPhase(ctx, job, distribution.SplitJobExportPhaseBackfill, 0, job.SnapshotTS) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.NoError(t, ctx.Err()) + require.Less(t, time.Since(start), 5*defaultSplitMigrationAttemptTimeout) + requireRuntimeTransferGuard(t, source) + require.Equal(t, export, sourceClient.exportStarted) + require.Equal(t, !export, targetClient.importStarted) + if !export { + require.NotEmpty(t, source.exports, "the import timeout must follow a real source export") + } + persisted, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, job.BracketProgress, persisted.BracketProgress, "failed attempts must retain resumable durable progress") + }) + } +} diff --git a/adapter/split_job_dispatch_auth_test.go b/adapter/split_job_dispatch_auth_test.go new file mode 100644 index 000000000..442c28d1a --- /dev/null +++ b/adapter/split_job_dispatch_auth_test.go @@ -0,0 +1,113 @@ +package adapter + +import ( + "context" + "math" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestSplitJobForgedCleanupCannotDispatchPrivilegedRPC(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st, distribution.WithCatalogRouteDescriptorV2Writes(true)) + _, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 7}}) + require.NoError(t, err) + job := splitJobForSourceGroupTest(t) + job.Phase = distribution.SplitJobPhaseCleanup + job.FenceTS = math.MaxUint64 + job.CutoverVersion = 1 + job.TargetPromotionDone = true + job.TargetClearedDescriptorAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + job.SourceCutoverAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + job.SourceReadDrainCursor = distribution.CloneBytes(splitCleanupDoneCursor) + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + server := NewDistributionServer(distribution.NewEngine(), catalog, + WithDistributionCoordinator(&ownershipCatalogCoordinator{distributionCoordinatorStub: newDistributionCoordinatorStub(st, true)}), + WithSplitMigrationCapabilityGate(func(context.Context) error { + t.Fatal("forged cleanup must be rejected before capability RPCs") + return nil + }), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + t.Fatal("forged job must not dispatch promotion") + return nil, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + t.Fatal("forged catalog cursors must not authorize privileged RPC dispatch") + return nil, nil, nil + })) + err = server.RunSplitJobRunnerOnce(ctx) + require.ErrorContains(t, err, "cutover witness") +} + +func TestSplitJobCleanupRequiresAuthoritativeCutoverDescriptor(t *testing.T) { + t.Parallel() + for _, name := range []string{"staged target", "promoted target", "source still live", "wrong owner", "wrong floor", "future version", "unpromoted descriptor", "different job"} { + t.Run(name, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st, distribution.WithCatalogRouteDescriptorV2Writes(true)) + job, routes := splitCleanupWitnessFixture(name) + _, err := catalog.Save(ctx, 0, routes) + require.NoError(t, err) + server := NewDistributionServer(distribution.NewEngine(), catalog, + WithDistributionCoordinator(&ownershipCatalogCoordinator{distributionCoordinatorStub: newDistributionCoordinatorStub(st, true)})) + err = server.verifySplitJobCleanupCutover(ctx, job) + if name == "staged target" || name == "promoted target" { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } +} + +func TestSplitJobCleanupDispatchRejectsStaleRootJob(t *testing.T) { + t.Parallel() + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st, distribution.WithCatalogRouteDescriptorV2Writes(true)) + job, routes := splitCleanupWitnessFixture("staged target") + _, err := catalog.Save(t.Context(), 0, routes) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(t.Context(), job)) + stale := distribution.CloneSplitJob(job) + job.CapabilityRegressed = true + require.NoError(t, catalog.SaveSplitJob(t.Context(), stale, job)) + server := NewDistributionServer(distribution.NewEngine(), catalog, + WithDistributionCoordinator(&ownershipCatalogCoordinator{distributionCoordinatorStub: newDistributionCoordinatorStub(st, true)}), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + t.Fatal("a stale cleanup job must not discover privileged clients") + return nil, nil, nil + })) + require.ErrorIs(t, server.runSplitJobPhase(t.Context(), stale), distribution.ErrCatalogSplitJobConflict) +} + +func splitCleanupWitnessFixture(name string) (distribution.SplitJob, []distribution.RouteDescriptor) { + job := promotionCompleteDistributionJob() + routes := promotionCompleteDistributionRoutes() + switch name { + case "staged target": + case "promoted target": + job.TargetPromotionDone = true + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + case "source still live": + routes[0].RouteID = job.SourceRouteID + case "wrong owner": + routes[1].GroupID = job.SourceGroupID + case "wrong floor": + routes[1].MinWriteTSExclusive++ + case "future version": + job.CutoverVersion++ + case "unpromoted descriptor": + job.TargetPromotionDone = true + case "different job": + routes[1].MigrationJobID++ + } + return job, routes +} diff --git a/adapter/split_job_failure.go b/adapter/split_job_failure.go new file mode 100644 index 000000000..63ab7b23f --- /dev/null +++ b/adapter/split_job_failure.go @@ -0,0 +1,90 @@ +package adapter + +import ( + "context" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const maxSplitJobPhaseFailures = 5 + +type splitJobAttemptKey struct { + jobID uint64 + phase distribution.SplitJobPhase +} + +type splitJobFailureState struct { + phase distribution.SplitJobPhase + count int +} + +func (s *DistributionServer) runSplitJobAttempt(ctx context.Context, job distribution.SplitJob) error { + err := s.runSplitJobPhase(ctx, job) + if errors.Is(err, errMigrationOversizedSourceVersion) && splitTransferCanAutoAbandon(job.Phase) { + return errors.WithStack(errors.CombineErrors(err, s.abandonOversizedSplitTransfer(ctx, job, err))) + } + key := splitJobAttemptKey{jobID: job.JobID, phase: job.Phase} + if splitJobPhaseErrorRetryable(err) { + s.resetSplitJobFailures(key) + return err + } + if !s.splitJobFailureLimitReached(key) { + return err + } + persistErr := s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + // A concurrent runner may have made progress or changed phase. Do not + // terminalize it based on a stale failed attempt. + if !distribution.SplitJobsEquivalent(current, job) { + return current, nil + } + return distribution.FailSplitJobState(current, err.Error(), time.Now().UnixMilli()) + }) + if persistErr == nil { + s.resetSplitJobFailures(key) + } + return errors.WithStack(errors.CombineErrors(err, persistErr)) +} + +func splitJobPhaseErrorRetryable(err error) bool { + if err == nil || splitJobRunnerContextDone(err) || splitJobCoordinatorLeadershipError(err) || errors.Is(err, errSplitMigrationVoterBarrierIncomplete) || errors.Is(err, store.ErrWriteConflict) { + return true + } + switch status.Code(err) { + case codes.Unavailable, codes.DeadlineExceeded, codes.Canceled, codes.Aborted, codes.ResourceExhausted: + return true + case codes.OK, codes.Unknown, codes.InvalidArgument, codes.NotFound, + codes.AlreadyExists, codes.PermissionDenied, codes.FailedPrecondition, + codes.OutOfRange, codes.Unimplemented, codes.Internal, codes.DataLoss, + codes.Unauthenticated: + return false + } + return false +} + +func (s *DistributionServer) resetSplitJobFailures(key splitJobAttemptKey) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.splitJobFailures, key.jobID) +} + +func (s *DistributionServer) splitJobFailureLimitReached(key splitJobAttemptKey) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.splitJobFailures == nil { + s.splitJobFailures = make(map[uint64]splitJobFailureState) + } + state := s.splitJobFailures[key.jobID] + if state.phase != key.phase { + state = splitJobFailureState{phase: key.phase} + } + if state.count < maxSplitJobPhaseFailures { + state.count++ + } + s.splitJobFailures[key.jobID] = state + return state.count >= maxSplitJobPhaseFailures +} diff --git a/adapter/split_job_failure_test.go b/adapter/split_job_failure_test.go new file mode 100644 index 000000000..fdb09ec3b --- /dev/null +++ b/adapter/split_job_failure_test.go @@ -0,0 +1,155 @@ +package adapter + +import ( + "context" + "sync" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestSplitJobPhaseErrorClassification(t *testing.T) { + t.Parallel() + for _, err := range []error{nil, context.Canceled, context.DeadlineExceeded, store.ErrWriteConflict, errSplitMigrationVoterBarrierIncomplete, + kv.ErrLeaderNotFound, ErrNotLeader, raftengine.ErrNotLeader, raftengine.ErrLeadershipLost, raftengine.ErrLeadershipTransferInProgress, + status.Error(codes.Unknown, "not leader"), status.Error(codes.Unknown, "leader not found"), + status.Error(codes.Unavailable, "leader"), status.Error(codes.Aborted, "conflict"), status.Error(codes.ResourceExhausted, "busy")} { + require.True(t, splitJobPhaseErrorRetryable(errors.WithStack(err))) + } + for _, err := range []error{errors.New("invalid durable cursor"), status.Error(codes.Unauthenticated, "token"), + status.Error(codes.PermissionDenied, "denied"), status.Error(codes.FailedPrecondition, "witness"), status.Error(codes.DataLoss, "corrupt")} { + require.False(t, splitJobPhaseErrorRetryable(errors.WithStack(err))) + } +} + +func TestSplitJobLeaderChurnNeverConsumesFailureBudget(t *testing.T) { + t.Parallel() + ctx, server, catalog, job := failedCleanupFixture(t) + for _, leaderErr := range []error{kv.ErrLeaderNotFound, ErrNotLeader, status.Error(codes.Unknown, "not leader"), status.Error(codes.Unknown, "leader not found")} { + server.splitMigrationClientFactory = func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return nil, nil, leaderErr + } + for range maxSplitJobPhaseFailures + 1 { + require.Error(t, server.RunSplitJobRunnerOnce(ctx)) + } + } + current, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.True(t, distribution.SplitJobsEquivalent(job, current)) + require.Empty(t, server.splitJobFailures) +} + +func TestSplitJobPersistentFailureIsDurableAndRetryable(t *testing.T) { + t.Parallel() + ctx, server, catalog, job := failedCleanupFixture(t) + for range maxSplitJobPhaseFailures { + require.ErrorContains(t, server.RunSplitJobRunnerOnce(ctx), "invalid split cleanup cursor") + } + failed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseFailed, failed.Phase) + require.Equal(t, distribution.SplitJobPhaseCleanup, failed.RetryPhase) + require.Contains(t, failed.LastError, "invalid split cleanup cursor") + require.Equal(t, job.SourceRetentionPinTS, failed.SourceRetentionPinTS) + require.Equal(t, job.FenceTS, failed.FenceTS) + require.Equal(t, job.Cursor, failed.Cursor) + require.NoError(t, server.RunSplitJobRunnerOnce(ctx), "FAILED jobs must stop automatic phase attempts") + // Losing the in-memory failure budget must not resume a durable FAILED job. + server.splitJobFailures = nil + require.NoError(t, server.RunSplitJobRunnerOnce(ctx)) + _, err = server.RetrySplitJob(ctx, &pb.RetrySplitJobRequest{JobId: job.JobID}) + require.NoError(t, err) + retried, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseCleanup, retried.Phase) + require.Empty(t, retried.LastError) + require.Equal(t, job.Cursor, retried.Cursor) +} + +func TestSplitJobTransientFailureResetsPersistentFailureBudget(t *testing.T) { + t.Parallel() + ctx, server, catalog, job := failedCleanupFixture(t) + for range maxSplitJobPhaseFailures - 1 { + require.Error(t, server.RunSplitJobRunnerOnce(ctx)) + } + original := server.splitMigrationClientFactory + server.splitMigrationClientFactory = func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return nil, nil, status.Error(codes.Unavailable, "leader changing") + } + require.Error(t, server.RunSplitJobRunnerOnce(ctx)) + server.splitMigrationClientFactory = original + for range maxSplitJobPhaseFailures - 1 { + require.Error(t, server.RunSplitJobRunnerOnce(ctx)) + } + current, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseCleanup, current.Phase) +} + +func TestSplitJobStaleFailureDoesNotOverwriteConcurrentProgress(t *testing.T) { + t.Parallel() + ctx, server, catalog, job := failedCleanupFixture(t) + for range maxSplitJobPhaseFailures - 1 { + require.Error(t, server.RunSplitJobRunnerOnce(ctx)) + } + // Fresh route/read-drain ACKs are legitimate durable progress too. Race + // against the current record, not the pre-ACK fixture that CAS must reject. + observed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + job = observed + progressed := distribution.CloneSplitJob(job) + progressed.Cursor = distribution.CloneBytes(splitSourceCleanupDoneCursor) + progressed.UpdatedAtMs++ + original := server.splitMigrationClientFactory + var advance sync.Once + server.splitMigrationClientFactory = func(ctx context.Context, current distribution.SplitJob, group uint64) (SplitMigrationClient, SplitMigrationClient, error) { + advance.Do(func() { require.NoError(t, catalog.SaveSplitJob(ctx, job, progressed)) }) + return original(ctx, current, group) + } + require.ErrorContains(t, server.RunSplitJobRunnerOnce(ctx), "invalid split cleanup cursor") + current, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.True(t, distribution.SplitJobsEquivalent(progressed, current)) +} + +func failedCleanupFixture(t *testing.T) (context.Context, *DistributionServer, *distribution.CatalogStore, distribution.SplitJob) { + t.Helper() + ctx := context.Background() + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st, distribution.WithCatalogRouteDescriptorV2Writes(true)) + routes := promotionCompleteDistributionRoutes() + routes[1].StagedVisibilityActive, routes[1].MigrationJobID = false, 0 + _, err := catalog.Save(ctx, 0, routes) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + job.TargetPromotionDone = true + job.SourceRetentionPinTS = 1 + job.TargetClearedDescriptorAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + job.SourceCutoverAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + job.SourceReadDrainCursor = distribution.CloneBytes(splitCleanupDoneCursor) + job.Cursor = []byte("invalid") + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + client := &splitMigrationClientStub{} + server := NewDistributionServer(distribution.NewEngine(), catalog, + WithDistributionCoordinator(&ownershipCatalogCoordinator{distributionCoordinatorStub: newDistributionCoordinatorStub(st, true)}), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return client, client, nil + }), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { return client, nil }), + ) + return ctx, server, catalog, job +} diff --git a/adapter/split_job_fence_refresh_test.go b/adapter/split_job_fence_refresh_test.go new file mode 100644 index 000000000..5cd28d224 --- /dev/null +++ b/adapter/split_job_fence_refresh_test.go @@ -0,0 +1,111 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type fenceDrainCommitClient struct { + *splitRuntimeClient + drained bool +} + +func (s *fenceDrainCommitClient) ProbeMigrationLocks(ctx context.Context, req *pb.ProbeMigrationLocksRequest, opts ...grpc.CallOption) (*pb.ProbeMigrationLocksResponse, error) { + if !s.drained { + if _, err := s.internal.transactionManager.Commit(ctx, []*pb.Request{fenceLateCommitRequest(pb.Phase_COMMIT)}); err != nil { + return nil, err + } + s.drained = true + } + return s.splitRuntimeClient.ProbeMigrationLocks(ctx, req, opts...) +} + +func fenceLateCommitRequest(phase pb.Phase) *pb.Request { + return &pb.Request{Ts: 4, IsTxn: true, Phase: phase, ObservedRouteVersion: kv.EncodeObservedRouteVersion(1), Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("o"), CommitTS: 5, LockTTLms: 60000})}, + {Op: pb.Op_PUT, Key: []byte("o"), Value: []byte("late-commit")}, + }} +} + +func TestSplitJobFenceRefreshesMinimumAfterDrainingLocks(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st, distribution.WithCatalogRouteDescriptorV2Writes(true)) + _, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 7}}) + require.NoError(t, err) + job := splitJobForSourceGroupTest(t) + job.SnapshotTS = 40 + job.Phase = distribution.SplitJobPhaseFence + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + server, source, target := newSplitRuntimeRunner(t, st, catalog, job) + // This transaction predates tracking, so only its later COMMIT may lower + // the first tracked minimum after the source write fence is installed. + _, err = source.internal.transactionManager.Commit(ctx, []*pb.Request{fenceLateCommitRequest(pb.Phase_PREPARE)}) + require.NoError(t, err) + _, complete, err := server.armSplitSourceOwnership(ctx, job, job.SourceGroupID, source) + require.NoError(t, err) + require.True(t, complete) + _, err = source.internal.transactionManager.Commit(ctx, []*pb.Request{{Ts: 50, ObservedRouteVersion: kv.EncodeObservedRouteVersion(1), Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte("p"), Value: []byte("first-tracked")}, + }}}) + require.NoError(t, err) + reader, ok := source.internal.store.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Len(t, states, 1) + require.Equal(t, uint64(50), states[0].MinAdmittedTS) + client := &fenceDrainCommitClient{splitRuntimeClient: source} + server.splitMigrationClientFactory = func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return client, target, nil + } + server.splitMigrationGroupFactory = func(context.Context, uint64) (SplitMigrationClient, error) { return client, nil } + for range len(job.BracketProgress) + 3 { + require.NoError(t, server.finalizeSplitJobFence(ctx, job)) + job = loadTransferJob(t, ctx, catalog, job.JobID) + if job.Phase == distribution.SplitJobPhaseDeltaCopy { + break + } + } + persisted, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.True(t, client.drained) + require.Equal(t, distribution.SplitJobPhaseDeltaCopy, persisted.Phase) + require.True(t, persisted.PostFenceDrainCompleted) + require.Equal(t, uint64(5), persisted.SnapshotMinAdmittedTS) + require.Equal(t, uint64(4), persisted.DeltaFloor, "the late commit must be included in the durable delta-copy window") + value, err := source.internal.store.GetAt(ctx, []byte("o"), 5) + require.NoError(t, err) + require.Equal(t, []byte("late-commit"), value) + requireRuntimeTransferGuard(t, source) +} + +func requireRuntimeTransferGuard(t *testing.T, source *splitRuntimeClient) { + t.Helper() + require.NoError(t, kv.RequireMigrationNamespaceAdmission(t.Context(), source.internal.store)) + var guard *pb.TargetStagedReadinessRequest + for _, control := range source.controls { + if control.GetTransferSizeGuardVersion() == 1 { + guard = control + } + } + require.NotNil(t, guard, "the real source must apply a guard, not just advertise support") + require.True(t, guard.GetRequireConfigurationIndex()) + require.Equal(t, uint64(9), guard.GetExpectedConfigurationIndex()) + probe := probeMigrationControlRequest(guard) + probe.RequireConfigurationIndex, probe.ExpectedConfigurationIndex = true, guard.GetExpectedConfigurationIndex() + probe.TransferSizeGuardVersion = guard.GetTransferSizeGuardVersion() + ack, err := source.ProbeMigrationState(t.Context(), probe) + require.NoError(t, err) + require.True(t, ack.GetReady(), "the real FSM/store must ACK the persisted guard control") + require.Equal(t, guard.GetExpectedConfigurationIndex(), ack.GetAppliedConfigurationIndex()) +} diff --git a/adapter/split_job_import_replay_test.go b/adapter/split_job_import_replay_test.go new file mode 100644 index 000000000..953d29068 --- /dev/null +++ b/adapter/split_job_import_replay_test.go @@ -0,0 +1,190 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// realImportTargetStub answers ImportRangeVersions out of a real MVCC store, so +// the duplicate detection exercised here is the production +// store.ImportVersions path (validateNextImportBatch and the durable ack it +// reads) rather than a hand-written answer. +type realImportTargetStub struct { + splitMigrationClientStub + st store.MVCCStore +} + +func (s *realImportTargetStub) ImportRangeVersions( + ctx context.Context, + req *pb.ImportRangeVersionsRequest, + _ ...grpc.CallOption, +) (*pb.ImportRangeVersionsResponse, error) { + versions := make([]store.MVCCVersion, 0, len(req.GetVersions())) + for _, version := range req.GetVersions() { + versions = append(versions, store.MVCCVersion{ + Key: distribution.CloneBytes(version.GetKey()), + Value: distribution.CloneBytes(version.GetValue()), + CommitTS: version.GetCommitTs(), + }) + } + result, err := s.st.ImportVersions(ctx, store.ImportVersionsOptions{ + JobID: req.GetJobId(), + BracketID: req.GetBracketId(), + BatchSeq: req.GetBatchSeq(), + Cursor: distribution.CloneBytes(req.GetCursor()), + Versions: versions, + }) + if err != nil { + return nil, err + } + return &pb.ImportRangeVersionsResponse{ + AckedCursor: result.AckedCursor, + Duplicate: result.Duplicate, + }, nil +} + +// importReplayFixture builds a copy-phase job with one bracket plus a target +// backed by its own real store. +func importReplayFixture(t *testing.T) ( + context.Context, + *DistributionServer, + *distribution.CatalogStore, + distribution.SplitJob, + distribution.MigrationBracket, + *realImportTargetStub, +) { + t.Helper() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + t.Cleanup(func() { _ = baseStore.Close() }) + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 1, + SourceRouteID: 1, + SplitKey: []byte("m"), + TargetGroupID: 2, + }, saved.Routes[0], 1000) + require.NoError(t, err) + job.Phase = distribution.SplitJobPhaseBackfill + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + require.NotEmpty(t, job.BracketProgress) + + targetStore := store.NewMVCCStore() + t.Cleanup(func() { _ = targetStore.Close() }) + target := &realImportTargetStub{st: targetStore} + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(newDistributionCoordinatorStub(baseStore, true)), + ) + + bracket := distribution.MigrationBracket{ + BracketID: job.BracketProgress[0].BracketID, + Family: job.BracketProgress[0].Family, + Start: []byte("m"), + End: []byte("z"), + } + return ctx, s, catalog, job, bracket, target +} + +// A runner that dies after the target durably acknowledged a batch, but before +// it persisted its own progress, replays that batch on restart. The source +// keeps taking writes while the copy runs, so the replayed chunk can end on a +// different boundary than the one the target accepted. Requiring the boundary +// to match would fail the job on that replay and on every retry after it, +// wedging the migration with its guards and retention pin held. The target's +// durable acknowledgement is the authority instead. +func TestCopySplitJobStreamAdoptsDuplicateImportAck(t *testing.T) { + t.Parallel() + + ctx, s, catalog, job, bracket, target := importReplayFixture(t) + + // The attempt whose progress was lost: batch 1 accepted at cursor-a. + acceptedCursor := []byte("cursor-a") + _, err := target.ImportRangeVersions(ctx, &pb.ImportRangeVersionsRequest{ + JobId: job.JobID, + BracketId: bracket.BracketID, + BatchSeq: 1, + Cursor: acceptedCursor, + Versions: []*pb.MVCCVersion{{Key: []byte("m1"), Value: []byte("v"), CommitTs: 10}}, + }) + require.NoError(t, err) + + // Restart: progress still reads batch 0, so the runner re-sends batch 1 -- + // this time from a re-export that ran longer and ended at cursor-b. + progress := job.BracketProgress[0] + progress.Cursor = nil + progress.LastAckedBatchSeq = 0 + stream := &splitMigrationStreamStub{responses: []*pb.ExportRangeVersionsResponse{{ + Versions: []*pb.MVCCVersion{{Key: []byte("m1"), Value: []byte("v"), CommitTs: 10}, {Key: []byte("m2"), Value: []byte("v"), CommitTs: 11}}, + NextCursor: []byte("cursor-b"), + Done: true, + }}} + + require.NoError(t, s.copySplitJobStream(ctx, job, 0, bracket, progress, stream, target)) + + loaded, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, acceptedCursor, loaded.BracketProgress[0].Cursor, + "progress must adopt the cursor the target durably acknowledged") + require.Equal(t, uint64(1), loaded.BracketProgress[0].LastAckedBatchSeq) + require.False(t, loaded.BracketProgress[0].Done, + "the replayed export's end-of-range answer describes a chunk the target never took") +} + +// The strict boundary check still fires when the target acknowledges a cursor +// it was never sent and does not claim the batch as a duplicate. +func TestCopySplitJobStreamRejectsUnexplainedCursorMismatch(t *testing.T) { + t.Parallel() + + ctx, s, _, job, bracket, target := importReplayFixture(t) + + // Batch 1 has never been seen by the target, so nothing is duplicated: the + // store acknowledges exactly the cursor it was sent. Rewriting it here + // stands in for a target that answers with a cursor of its own. + lying := &lyingImportTargetStub{realImportTargetStub: target, cursor: []byte("cursor-x")} + progress := job.BracketProgress[0] + stream := &splitMigrationStreamStub{responses: []*pb.ExportRangeVersionsResponse{{ + Versions: []*pb.MVCCVersion{{Key: []byte("m1"), Value: []byte("v"), CommitTs: 10}}, + NextCursor: []byte("cursor-b"), + Done: true, + }}} + + err := s.copySplitJobStream(ctx, job, 0, bracket, progress, stream, lying) + require.ErrorContains(t, err, "acknowledged a different cursor") +} + +type lyingImportTargetStub struct { + *realImportTargetStub + cursor []byte +} + +func (s *lyingImportTargetStub) ImportRangeVersions( + ctx context.Context, + req *pb.ImportRangeVersionsRequest, + opts ...grpc.CallOption, +) (*pb.ImportRangeVersionsResponse, error) { + resp, err := s.realImportTargetStub.ImportRangeVersions(ctx, req, opts...) + if err != nil { + return nil, err + } + resp.AckedCursor = distribution.CloneBytes(s.cursor) + return resp, nil +} diff --git a/adapter/split_job_namespace.go b/adapter/split_job_namespace.go new file mode 100644 index 000000000..6565f91c0 --- /dev/null +++ b/adapter/split_job_namespace.go @@ -0,0 +1,96 @@ +package adapter + +import ( + "context" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +type splitNamespaceGroup struct { + id uint64 + client SplitMigrationClient + members SplitMigrationApplyMemberSnapshot +} + +func splitJobTransferNamespaceRequired(phase distribution.SplitJobPhase) bool { + switch phase { + case distribution.SplitJobPhaseBackfill, distribution.SplitJobPhaseFence, distribution.SplitJobPhaseDeltaCopy, distribution.SplitJobPhaseCutover: + return true + case distribution.SplitJobPhaseNone, distribution.SplitJobPhasePlanned, distribution.SplitJobPhaseCleanup, + distribution.SplitJobPhaseDone, distribution.SplitJobPhaseFailed, distribution.SplitJobPhaseAbandoning, distribution.SplitJobPhaseAbandoned: + return false + } + return false +} + +// Probe both sides before any activation or catalog geometry write. Static +// support is not a guard ACK; MODE8 subsequently proves durable activation. +func (s *DistributionServer) ensureSplitJobNamespaceAdmission(ctx context.Context, job distribution.SplitJob) error { + return s.ensureMigrationNamespaces(ctx, []uint64{job.SourceGroupID, job.TargetGroupID}) +} + +func (s *DistributionServer) ensureMigrationNamespaces(ctx context.Context, ids []uint64) error { + if s.splitMigrationGroupFactory == nil { + return errors.WithStack(store.ErrNotSupported) + } + groups := make([]splitNamespaceGroup, 0, len(ids)) + for _, id := range ids { + group, err := s.splitNamespaceGroup(ctx, id) + if err != nil { + return err + } + groups = append(groups, group) + } + for _, group := range groups { + if err := EnsureMigrationNamespaceAdmission(ctx, group.client, group.members.Members); err != nil { + return err + } + } + for _, group := range groups { + _, err := s.revalidateSplitTransferMembers(ctx, &splitTransferAdmissionWitness{groupID: group.id, snapshot: group.members}) + if err != nil { + return err + } + } + return nil +} + +func (s *DistributionServer) splitNamespaceGroup(ctx context.Context, id uint64) (splitNamespaceGroup, error) { + members, err := s.splitTransferApplyMembers(ctx, id) + if err != nil { + return splitNamespaceGroup{}, err + } + if err := verifySplitNamespaceMembers(ctx, members); err != nil { + return splitNamespaceGroup{}, err + } + client, err := s.splitMigrationGroupFactory(ctx, id) + if err != nil { + return splitNamespaceGroup{}, errors.WithStack(err) + } + if client == nil { + return splitNamespaceGroup{}, errors.WithStack(store.ErrNotSupported) + } + return splitNamespaceGroup{id: id, client: client, members: members}, nil +} + +func verifySplitNamespaceMembers(ctx context.Context, snapshot SplitMigrationApplyMemberSnapshot) error { + for _, member := range snapshot.Members { + attempt, cancel := context.WithTimeout(ctx, defaultSplitMigrationVoterProbeTimeout) + response, err := member.Client.ProbeMigrationState(attempt, &pb.ProbeMigrationStateRequest{JobId: 1, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: snapshot.ConfigurationIndex}) + cancel() + if err != nil { + return errors.WithStack(err) + } + if response.GetTransferSizeGuardVersion() != store.MigrationTransferSizeGuardVersion || + response.GetOwnershipPartitionProtocol() != MigrationOwnershipPartitionProtocolVersion || + response.GetAppliedConfigurationIndex() != snapshot.ConfigurationIndex { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + } + return nil +} diff --git a/adapter/split_job_ownership.go b/adapter/split_job_ownership.go new file mode 100644 index 000000000..4f58576eb --- /dev/null +++ b/adapter/split_job_ownership.go @@ -0,0 +1,275 @@ +package adapter + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +type catalogAppliedReader interface { + LinearizableReadForKey(context.Context, []byte) (uint64, error) +} + +// AttestUnchangedMigrationOwnership is the root-leader producer for a fresh +// replicated attestation. It does not propose to the data group: the consumer +// must bind the resulting REFRESH to that group's current all-voter epoch. +// No watcher, version increment, or same-owner guess substitutes for this read. +func (s *DistributionServer) AttestUnchangedMigrationOwnership(ctx context.Context, prior *pb.MigrationOwnershipProof) (*pb.MigrationOwnershipProof, error) { + if prior == nil || len(prior.GetDescriptorHash()) != sha256.Size || prior.GetGeneration() == "" { + return nil, errors.New("migration ownership attestation requires durable prior proof") + } + attempt, cancel := context.WithTimeout(ctx, defaultSplitMigrationAttemptTimeout) + defer cancel() + snapshot, err := s.loadCommittedMigrationCatalog(attempt) + if err != nil { + return nil, err + } + if snapshot.Version < prior.GetCatalogVersion() { + return nil, errors.New("committed catalog precedes durable ownership proof") + } + route, found := findRouteByID(snapshot.Routes, prior.GetRouteId()) + if !found { + return nil, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(route, true) + if err != nil { + return nil, errors.WithStack(err) + } + digest := sha256.Sum256(encoded) + if !bytes.Equal(digest[:], prior.GetDescriptorHash()) || route.GroupID != prior.GetOwnerGroupId() { + return nil, errors.New("committed ownership descriptor changed; reacquisition required") + } + refreshed := proto.CloneOf(prior) + refreshed.CatalogVersion = snapshot.Version + return refreshed, nil +} + +func (s *DistributionServer) loadCommittedMigrationCatalog(ctx context.Context) (distribution.CatalogSnapshot, error) { + if err := s.verifyCatalogLeader(ctx); err != nil { + return distribution.CatalogSnapshot{}, err + } + reader, ok := s.coordinator.(catalogAppliedReader) + if !ok { + return distribution.CatalogSnapshot{}, errors.New("migration ownership requires root leader applied-read support") + } + if _, err := reader.LinearizableReadForKey(ctx, distribution.CatalogVersionKey()); err != nil { + return distribution.CatalogSnapshot{}, errors.WithStack(err) + } + return s.loadCatalogSnapshot(ctx) +} + +func splitOwnershipProof(snapshot distribution.CatalogSnapshot, job distribution.SplitJob, localGroupID uint64, generation string) (*pb.MigrationOwnershipProof, error) { + for _, route := range snapshot.Routes { + if !splitOwnershipRouteContains(route, job.SplitKey) { + continue + } + if route.State != distribution.RouteStateActive || route.GroupID == 0 { + return nil, errors.New("migration ownership descriptor is not active") + } + if !splitOwnershipDescriptorMatchesJob(route, job) { + return nil, errors.New("committed descriptor does not attest this migration job") + } + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(route, true) + if err != nil { + return nil, errors.WithStack(err) + } + digest := sha256.Sum256(encoded) + role := pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_NONOWNER + if route.GroupID == localGroupID { + role = pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER + } + return &pb.MigrationOwnershipProof{Generation: generation, LocalGroupId: localGroupID, OwnerGroupId: route.GroupID, + WriteFloorKnown: true, + CatalogVersion: snapshot.Version, RouteStart: bytes.Clone(job.SplitKey), RouteEnd: bytes.Clone(route.End), + RouteId: route.RouteID, ParentRouteId: route.ParentRouteID, MinWriteTsExclusive: route.MinWriteTSExclusive, + Staged: localGroupID == route.GroupID && route.StagedVisibilityActive, Role: role, DescriptorHash: digest[:]}, nil + } + return nil, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) +} + +func splitOwnershipRouteContains(route distribution.RouteDescriptor, key []byte) bool { + return bytes.Compare(route.Start, key) <= 0 && (len(route.End) == 0 || bytes.Compare(key, route.End) < 0) +} + +func splitOwnershipDescriptorMatchesJob(route distribution.RouteDescriptor, job distribution.SplitJob) bool { + if job.Phase == distribution.SplitJobPhaseCleanup { + return validSplitJobCleanupWitness(job, ^uint64(0)) && splitJobCleanupTargetAttests(route, job) + } + if job.Phase != distribution.SplitJobPhaseAbandoning || route.GroupID == job.TargetGroupID || route.StagedVisibilityActive { + return false + } + return (route.RouteID == job.SourceRouteID || route.ParentRouteID == job.SourceRouteID) && + route.MinWriteTSExclusive >= max(initialMigrationRetentionPinTS, job.SnapshotTS, job.FenceTS) +} + +func (s *DistributionServer) splitOwnershipConfiguration(ctx context.Context, jobID, groupID uint64, fallback SplitMigrationClient) (uint64, string, error) { + if s.splitMigrationVoterFactory == nil { + return 0, "", errors.Wrap(errSplitMigrationVoterBarrierIncomplete, "ownership requires explicit voter discovery") + } + voters, err := s.splitMigrationVoters(ctx, groupID, fallback) + if err != nil { + return 0, "", err + } + var index uint64 + var generation string + for n, voter := range voters { + resp, err := probeSplitOwnershipCapability(ctx, voter, jobID) + if err != nil { + return 0, "", err + } + if n == 0 { + index, generation = resp.GetAppliedConfigurationIndex(), resp.GetOwnershipGeneration() + } + if index != resp.GetAppliedConfigurationIndex() || generation != resp.GetOwnershipGeneration() { + return 0, "", errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + } + if generation == "" { + // The first native control establishes the namespace in this group's + // replicated ownership record. A lost response rediscovers that record; + // a copied configuration index is never used as the namespace. + const ownershipGenerationBytes = 16 + nonce := make([]byte, ownershipGenerationBytes) + if _, err := rand.Read(nonce); err != nil { + return 0, "", errors.WithStack(err) + } + generation = hex.EncodeToString(nonce) + } + return index, generation, nil +} + +func probeSplitOwnershipCapability(ctx context.Context, voter SplitMigrationVoter, jobID uint64) (*pb.ProbeMigrationStateResponse, error) { + attempt, cancel := context.WithTimeout(ctx, defaultSplitMigrationVoterProbeTimeout) + defer cancel() + resp, err := voter.Client.ProbeMigrationState(attempt, &pb.ProbeMigrationStateRequest{ + JobId: jobID, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY, RequireConfigurationIndex: true, + }) + if err != nil || !resp.GetReady() || resp.GetOwnershipProtocol() != migrationOwnershipProtocol || resp.GetAppliedConfigurationIndex() == 0 { + return nil, errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + return resp, nil +} + +func (s *DistributionServer) applySplitOwnership(ctx context.Context, job distribution.SplitJob, groupID uint64, client SplitMigrationClient, mode pb.MigrationCleanupMode, expectedConfiguration ...uint64) error { + index, generation, err := s.splitOwnershipConfiguration(ctx, job.JobID, groupID, client) + if err != nil { + return err + } + if len(expectedConfiguration) > 0 && expectedConfiguration[0] != index { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + snapshot, err := s.loadCommittedMigrationCatalog(ctx) + if err != nil { + return err + } + proof, err := splitOwnershipProof(snapshot, job, groupID, generation) + if err != nil { + return err + } + mode, err = splitOwnershipOperation(ctx, client, job.JobID, index, mode) + if err != nil { + return err + } + if _, err := cleanupSplitMigration(ctx, client, &pb.CleanupMigrationRequest{JobId: job.JobID, Mode: mode, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: index, Ownership: proof}); err != nil { + return err + } + _, ready, err := s.syncSplitMigrationVoterBarrier(ctx, groupID, nil, client, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: index, Ownership: proof, + }) + if err != nil { + return err + } + if !ready { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + return nil +} + +func splitOwnershipOperation(ctx context.Context, client SplitMigrationClient, jobID, index uint64, requested pb.MigrationCleanupMode) (pb.MigrationCleanupMode, error) { + attempt, cancel := context.WithTimeout(ctx, defaultSplitMigrationAttemptTimeout) + defer cancel() + resp, err := client.ProbeMigrationState(attempt, &pb.ProbeMigrationStateRequest{ + JobId: jobID, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: index, + }) + if err != nil { + return requested, errors.WithStack(err) + } + if resp.GetOwnershipProtocol() != migrationOwnershipProtocol || resp.GetAppliedConfigurationIndex() != index { + return requested, errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + if resp.GetReady() && resp.GetOwnership() != nil && (requested == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_ACQUIRE || resp.GetOwnershipCleared()) { + return pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH, nil + } + return requested, nil +} + +func (s *DistributionServer) prepareSplitOwnershipRollback(ctx context.Context, job distribution.SplitJob) error { + snapshot, err := s.loadCommittedMigrationCatalog(ctx) + if err != nil { + return err + } + groupID, end, ok := splitJobSourceRouteState(snapshot.Routes, job) + if !ok { + return errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + source, err := s.splitJobSourceClient(ctx, job, groupID) + if err != nil { + return err + } + configuration, _, err := s.splitOwnershipConfiguration(ctx, job.JobID, groupID, source) + if err != nil { + return err + } + _, retired, err := s.syncSplitMigrationVoterBarrier(ctx, groupID, nil, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: configuration, + }) + if err != nil { + return err + } + if !retired { + // Backfill tracking allows creation. Close it in the source log before + // the root publishes the rollback floor, without blocking old resolutions. + if _, err := applySplitMigrationControl(ctx, source, job, end, snapshot.Version, + max(initialMigrationRetentionPinTS, job.SnapshotTS, job.FenceTS), true, false, true, initialMigrationRetentionPinTS); err != nil { + return err + } + } + _, err = s.splitAbandonedSourceConfiguration(ctx, job, source, end) + return err +} + +func splitOwnershipRollbackRoute(routes []distribution.RouteDescriptor, job distribution.SplitJob) *distribution.RouteDescriptor { + for _, route := range routes { + if route.GroupID == job.TargetGroupID || route.StagedVisibilityActive { + continue + } + if route.RouteID == job.SourceRouteID || (route.ParentRouteID == job.SourceRouteID && bytes.Equal(route.Start, job.SplitKey)) { + copy := route + return © + } + } + return nil +} + +func (s *DistributionServer) verifySplitOwnershipRuntimes(ctx context.Context, job distribution.SplitJob, sourceGroupID uint64, source SplitMigrationClient) error { + if _, _, err := s.splitOwnershipConfiguration(ctx, job.JobID, sourceGroupID, source); err != nil { + return err + } + _, target, err := s.splitMigrationClientFactory(ctx, job, sourceGroupID) + if err != nil { + return errors.WithStack(err) + } + _, _, err = s.splitOwnershipConfiguration(ctx, job.JobID, job.TargetGroupID, target) + return err +} diff --git a/adapter/split_job_ownership_test.go b/adapter/split_job_ownership_test.go new file mode 100644 index 000000000..d42255dca --- /dev/null +++ b/adapter/split_job_ownership_test.go @@ -0,0 +1,276 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type ownershipRuntimeLeader struct { + mockInternalLeader + configuration raftengine.AppliedConfigurationAware +} + +func (l ownershipRuntimeLeader) AppliedConfigurationIndex() (uint64, bool) { + index := l.configuration.AppliedConfigurationIndex() + return index, index != 0 +} + +// A simulated ConfChange still uses the full ordered inventory hook after +// activation; the legacy index-only hook intentionally cannot replace it. +type ownershipRuntimeConfiguration struct { + raftengine.AppliedConfigurationAware + inventory raftengine.NamespaceConfigurationAware + members []uint64 +} + +func (c ownershipRuntimeConfiguration) RecordAppliedConfiguration(index uint64) error { + return c.inventory.RecordAppliedNamespaceConfiguration(index, c.members) +} + +type ownershipRuntimeProposer struct { + fsm kv.FSM + index uint64 + followers []kv.FSM +} + +func (p *ownershipRuntimeProposer) Propose(_ context.Context, data []byte) (*raftengine.ProposalResult, error) { + p.index++ + aware, ok := p.fsm.(raftengine.ApplyIndexAware) + if !ok { + return nil, store.ErrNotSupported + } + aware.SetApplyIndex(p.index) + response := p.fsm.Apply(data) + for _, follower := range p.followers { + aware, ok := follower.(raftengine.ApplyIndexAware) + if !ok { + return nil, store.ErrNotSupported + } + aware.SetApplyIndex(p.index) + if err, failed := follower.Apply(data).(error); failed { + return nil, err + } + } + return &raftengine.ProposalResult{CommitIndex: p.index, Response: response}, nil +} + +func (p *ownershipRuntimeProposer) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { + return p.Propose(ctx, data) +} + +type ownershipRuntimeClient struct { + splitMigrationClientStub + internal *Internal + beforeCleanup func() + blockGuard bool + blockMetadata bool +} + +func (c *ownershipRuntimeClient) CleanupMigration(ctx context.Context, req *pb.CleanupMigrationRequest, _ ...grpc.CallOption) (*pb.CleanupMigrationResponse, error) { + c.cleanups = append(c.cleanups, req) + if c.beforeCleanup != nil { + c.beforeCleanup() + } + return c.internal.CleanupMigration(ctx, req) +} + +func (c *ownershipRuntimeClient) ApplyTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest, _ ...grpc.CallOption) (*pb.TargetStagedReadinessResponse, error) { + c.controls = append(c.controls, req) + return c.internal.ApplyTargetStagedReadiness(ctx, req) +} + +func (c *ownershipRuntimeClient) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest, _ ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + if c.blockGuard && (req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_GUARD_PRESENT || + req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED) { + return &pb.ProbeMigrationStateResponse{}, nil + } + resp, err := c.internal.ProbeMigrationState(ctx, req) + if c.blockMetadata && req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED { + resp.Ready = false + } + return resp, err +} + +func newOwnershipRuntimeClient(t *testing.T, st store.MVCCStore, groupID uint64) (*ownershipRuntimeClient, raftengine.AppliedConfigurationAware) { + t.Helper() + fsm := kv.NewKvFSMWithHLC(st, nil, kv.WithRouteHistory(nil, groupID)) + configuration, ok := fsm.(raftengine.AppliedConfigurationAware) + require.True(t, ok) + inventory, ok := fsm.(raftengine.NamespaceConfigurationAware) + require.True(t, ok) + member := etcd.DeriveNodeID("voter") + require.NoError(t, inventory.RecordAppliedNamespaceConfiguration(9, []uint64{member})) + configuration = ownershipRuntimeConfiguration{AppliedConfigurationAware: configuration, inventory: inventory, members: []uint64{member}} + proposer := &ownershipRuntimeProposer{fsm: fsm, index: 20} + require.NoError(t, kv.ActivateMigrationNamespace(t.Context(), st, proposer, kv.NamespaceAdmissionCommand{ + Version: kv.MigrationNamespaceVersion, ConfigurationIndex: 9, ApplyMembers: []uint64{member}, + Members: []kv.NamespaceMemberCapability{{MemberID: member, Version: kv.MigrationNamespaceVersion, ConfigurationIndex: 9}}, + })) + i := NewInternalWithEngine(nil, ownershipRuntimeLeader{configuration: configuration}, nil, nil, + WithInternalStore(st), WithInternalMigrationProposer(proposer), WithInternalMigrationOwnershipReady(func() bool { return true }), + WithMigrationNamespaceAdmissionReady(func() bool { return kv.MigrationNamespaceInstalledVersion(st, fsm) == kv.MigrationNamespaceVersion }), + WithInternalMigrationCleanupGate(func(context.Context) error { return nil })) + return &ownershipRuntimeClient{internal: i}, configuration +} + +type ownershipCatalogCoordinator struct { + *distributionCoordinatorStub + reads int +} + +func (c *ownershipCatalogCoordinator) LinearizableReadForKey(ctx context.Context, _ []byte) (uint64, error) { + c.reads++ + return c.LinearizableRead(ctx) +} + +func TestSplitOwnershipRollbackActualCommittedCAndTargetDown(t *testing.T) { + for _, changeConfiguration := range []bool{false, true} { + t.Run(map[bool]string{false: "normal", true: "configuration interleaving"}[changeConfiguration], func(t *testing.T) { + catalogStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(catalogStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + _, err := catalog.Save(t.Context(), 0, []distribution.RouteDescriptor{ + {RouteID: 2, ParentRouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 7}, + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 7, State: distribution.RouteStateWriteFenced}, + }) + require.NoError(t, err) + job := sampleDistributionSplitJob(7) + job.SourceRouteID, job.SourceGroupID, job.TargetGroupID, job.SplitKey = 1, 7, 9, []byte("m") + job.Phase, job.AbandonFromPhase = distribution.SplitJobPhaseAbandoning, distribution.SplitJobPhaseDeltaCopy + job.SnapshotTS, job.FenceTS = 80, 100 + require.NoError(t, catalog.CreateSplitJob(t.Context(), job)) + sourceStore := store.NewMVCCStore() + source, configuration := newOwnershipRuntimeClient(t, sourceStore, 7) + guardWriter, ok := sourceStore.(store.MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.NoError(t, guardWriter.ApplyTargetStagedReadinessAt(t.Context(), store.TargetStagedReadinessState{ + JobID: job.JobID, MigrationJobID: job.JobID, RouteStart: job.SplitKey, RouteEnd: []byte("z"), + Armed: true, SourceWriteFence: true, ExpectedCutoverVersion: 1, MinWriteTSExclusive: 100, RetentionPinTS: 1, + }, 10)) + coordinator := &ownershipCatalogCoordinator{distributionCoordinatorStub: newDistributionCoordinatorStub(catalogStore, true)} + target := &abandonUnavailableTarget{} + targetLookups := 0 + server := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator), + WithSplitMigrationGroupClientFactory(func(context.Context, uint64) (SplitMigrationClient, error) { return source, nil }), + WithSplitMigrationVoterFactory(func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "source", Address: "source:1", Client: source}}, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + targetLookups++ + return source, target, nil + })) + source.beforeCleanup = func() { + current, err := catalog.Snapshot(t.Context()) + require.NoError(t, err) + require.Nil(t, abandonedSplitJobFencedRoute(current.Routes, job), "rollback must already be committed") + reader, ok := sourceStore.(store.MigrationTargetReadinessReader) + require.True(t, ok) + guards, err := reader.MigrationTargetReadinessStates(t.Context()) + require.NoError(t, err) + require.Len(t, guards, 1, "durable guard remains until atomic ownership replacement") + if changeConfiguration { + require.NoError(t, configuration.RecordAppliedConfiguration(11)) + } + } + err = server.abandonSplitJob(t.Context(), job) + if changeConfiguration { + require.Equal(t, codes.Aborted, status.Code(err)) + source.beforeCleanup = nil + require.NoError(t, server.abandonSplitJob(t.Context(), job)) + } else { + require.NoError(t, err) + } + require.Zero(t, targetLookups, "source recovery must not contact the unavailable target") + reader, ok := sourceStore.(store.MigrationOwnershipStore) + require.True(t, ok) + view, err := reader.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Empty(t, view.Readiness) + require.Len(t, view.Ownership, 1) + committed, err := catalog.Snapshot(t.Context()) + require.NoError(t, err) + require.Equal(t, committed.Version, view.Ownership[0].CatalogVersion) + require.Equal(t, store.MigrationOwner, view.Ownership[0].Role) + require.NotEmpty(t, view.Ownership[0].Generation) + require.Positive(t, coordinator.reads) + job, _, err = catalog.SplitJob(t.Context(), job.JobID) + require.NoError(t, err) + require.ErrorContains(t, server.abandonSplitJob(t.Context(), job), "target offline") + }) + } +} + +func TestMigrationOwnershipCapabilityDefaultDenyAndUnknownEpoch(t *testing.T) { + st := store.NewMVCCStore() + client, _ := newOwnershipRuntimeClient(t, st, 7) + req := &pb.ProbeMigrationStateRequest{JobId: 7, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY, RequireConfigurationIndex: true} + resp, err := client.ProbeMigrationState(t.Context(), req) + require.NoError(t, err) + require.True(t, resp.GetReady()) + client.internal.migrationOwnershipReady = nil + resp, err = client.ProbeMigrationState(t.Context(), req) + require.NoError(t, err) + require.False(t, resp.GetReady()) + client.internal.migrationOwnershipReady = func() bool { return true } + client.internal.leader = migrationConfigurationLeader{index: 0} + resp, err = client.ProbeMigrationState(t.Context(), req) + require.NoError(t, err) + require.False(t, resp.GetReady(), "a copied store row cannot replace an actual runtime epoch") + require.Zero(t, resp.GetOwnershipProtocol()) +} + +func TestSplitOwnershipCapabilityRejectsOldUnknownFieldEcho(t *testing.T) { + old := &splitMigrationClientStub{} + server := &DistributionServer{splitMigrationVoterFactory: func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "old", Address: "old:1", Client: old}}, nil + }} + _, _, err := server.splitOwnershipConfiguration(t.Context(), 7, 7, old) + require.ErrorIs(t, err, errSplitMigrationVoterBarrierIncomplete) +} + +func TestAttestUnchangedMigrationOwnershipUsesCommittedDescriptor(t *testing.T) { + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st, distribution.WithCatalogRouteDescriptorV2Writes(true)) + routes := []distribution.RouteDescriptor{ + {RouteID: 2, ParentRouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 7}, + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 7, MinWriteTSExclusive: 100}, + } + _, err := catalog.Save(t.Context(), 0, routes) + require.NoError(t, err) + snapshot, err := catalog.Snapshot(t.Context()) + require.NoError(t, err) + job := sampleDistributionSplitJob(7) + job.SourceRouteID, job.SourceGroupID, job.TargetGroupID, job.SplitKey = 1, 7, 9, []byte("m") + job.Phase, job.SnapshotTS, job.FenceTS = distribution.SplitJobPhaseAbandoning, 80, 100 + prior, err := splitOwnershipProof(snapshot, job, 7, "0123456789abcdef0123456789abcdef") + require.NoError(t, err) + coordinator := &ownershipCatalogCoordinator{distributionCoordinatorStub: newDistributionCoordinatorStub(st, true)} + server := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + // This change concerns a different route; no migration job lookup or + // watcher history is needed to attest the retained descriptor. + routes[0].GroupID = 11 + _, err = catalog.Save(t.Context(), snapshot.Version, routes) + require.NoError(t, err) + refreshed, err := server.AttestUnchangedMigrationOwnership(t.Context(), prior) + require.NoError(t, err) + require.Equal(t, prior.GetCatalogVersion()+1, refreshed.GetCatalogVersion()) + require.Equal(t, prior.GetDescriptorHash(), refreshed.GetDescriptorHash()) + require.Equal(t, prior.GetGeneration(), refreshed.GetGeneration()) + require.Equal(t, snapshot.Version, prior.GetCatalogVersion(), "producer must not mutate durable prior evidence") + require.Equal(t, 1, coordinator.reads) + routes[1].GroupID = 9 + _, err = catalog.Save(t.Context(), refreshed.GetCatalogVersion(), routes) + require.NoError(t, err) + _, err = server.AttestUnchangedMigrationOwnership(t.Context(), refreshed) + require.ErrorContains(t, err, "descriptor changed") +} diff --git a/adapter/split_job_review_regression_test.go b/adapter/split_job_review_regression_test.go new file mode 100644 index 000000000..838d0a483 --- /dev/null +++ b/adapter/split_job_review_regression_test.go @@ -0,0 +1,70 @@ +package adapter + +import ( + "context" + "math" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestSplitJobCapabilityRegressionUsesCASedCleanupState(t *testing.T) { + t.Parallel() + for _, alreadyMarked := range []bool{false, true} { + t.Run(map[bool]string{false: "new regression", true: "existing regression"}[alreadyMarked], func(t *testing.T) { + t.Parallel() + ctx, server, catalog, stale := failedCleanupFixture(t) + stale.CapabilityRegressed = alreadyMarked + current := distribution.CloneSplitJob(stale) + current.SourceRetentionPinTS = math.MaxUint64 + current.UpdatedAtMs++ + expected, found, err := catalog.SplitJob(ctx, stale.JobID) + require.NoError(t, err) + require.True(t, found) + require.NoError(t, catalog.SaveSplitJob(ctx, expected, current)) + server.splitMigrationClientFactory = func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + t.Fatal("completed source/target cleanup must not rearm guards from stale input") + return nil, nil, nil + } + ready, err := server.handleSplitJobCapabilityRegression(ctx, stale, errors.New("voter unavailable")) + require.NoError(t, err) + require.False(t, ready) + marked, found, err := catalog.SplitJob(ctx, stale.JobID) + require.NoError(t, err) + require.True(t, found) + require.True(t, marked.CapabilityRegressed) + require.Equal(t, uint64(math.MaxUint64), marked.SourceRetentionPinTS) + }) + } +} + +func TestSplitJobHistoryPageSurvivesDeletedAnchor(t *testing.T) { + t.Parallel() + jobs := make([]distribution.SplitJob, 0, 205) + for id := uint64(1); id <= 205; id++ { + job := sampleDistributionSplitJob(id) + job.Phase, job.TerminalAtMs = distribution.SplitJobPhaseDone, 1000 + jobs = append(jobs, job) + } + first, err := splitJobListPage(jobs, &pb.ListSplitJobsRequest{}, "") + require.NoError(t, err) + require.NotEmpty(t, first.NextPageCursor) + // The page ended at job 6; GC removes that anchor and another older row. + remaining := make([]distribution.SplitJob, 0, len(jobs)) + for _, job := range jobs { + if job.JobID != 6 && job.JobID != 4 { + remaining = append(remaining, job) + } + } + second, err := splitJobListPage(remaining, &pb.ListSplitJobsRequest{PageCursor: first.NextPageCursor}, "") + require.NoError(t, err) + require.Equal(t, []uint64{5, 3, 2, 1}, splitJobIDs(second.Jobs)) + last := sampleDistributionSplitJob(1) + last.TerminalAtMs = 1000 + end, err := splitJobListPage(remaining[1:], &pb.ListSplitJobsRequest{PageCursor: encodeSplitJobListCursor(last)}, "") + require.NoError(t, err) + require.Empty(t, end.Jobs) +} diff --git a/adapter/split_job_rpc_persistence_test.go b/adapter/split_job_rpc_persistence_test.go new file mode 100644 index 000000000..cccca3fc3 --- /dev/null +++ b/adapter/split_job_rpc_persistence_test.go @@ -0,0 +1,126 @@ +package adapter + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type lateImportAckClient struct { + *realImportTargetStub + attemptCtx context.Context +} + +func (s *lateImportAckClient) ImportRangeVersions(ctx context.Context, req *pb.ImportRangeVersionsRequest, opts ...grpc.CallOption) (*pb.ImportRangeVersionsResponse, error) { + s.attemptCtx = ctx + resp, err := s.realImportTargetStub.ImportRangeVersions(ctx, req, opts...) + if err != nil { + return nil, err + } + <-ctx.Done() + return resp, nil +} + +func TestSplitJobCopyPersistsLateAndDuplicateAcknowledgements(t *testing.T) { + t.Parallel() + for _, duplicate := range []bool{false, true} { + t.Run(map[bool]string{false: "new ack", true: "duplicate ack"}[duplicate], func(t *testing.T) { + t.Parallel() + ctx, server, catalog, job, bracket, target := importReplayFixture(t) + versions := []*pb.MVCCVersion{{Key: []byte("n"), CommitTs: 10, Value: []byte("v")}} + cursor := []byte("new-cursor") + if duplicate { + cursor = []byte("accepted-cursor") + _, err := target.ImportRangeVersions(ctx, &pb.ImportRangeVersionsRequest{JobId: job.JobID, BracketId: bracket.BracketID, BatchSeq: 1, Cursor: cursor, Versions: versions}) + require.NoError(t, err) + } + stream := &splitMigrationStreamStub{responses: []*pb.ExportRangeVersionsResponse{{Versions: versions, NextCursor: []byte("new-cursor"), Done: true}}} + client := &lateImportAckClient{realImportTargetStub: target} + require.NoError(t, server.copySplitJobStream(ctx, job, 0, bracket, job.BracketProgress[0], stream, client)) + require.ErrorIs(t, client.attemptCtx.Err(), context.DeadlineExceeded) + require.NoError(t, ctx.Err()) + persisted, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, cursor, persisted.BracketProgress[0].Cursor) + require.Equal(t, uint64(1), persisted.BracketProgress[0].LastAckedBatchSeq) + }) + } +} + +type stalledCleanupClient struct { + splitMigrationClientStub +} + +func (*stalledCleanupClient) CleanupMigration(ctx context.Context, _ *pb.CleanupMigrationRequest, _ ...grpc.CallOption) (*pb.CleanupMigrationResponse, error) { + <-ctx.Done() + return nil, ctx.Err() +} + +func TestSplitJobCleanupAttemptsHaveIndependentDeadlines(t *testing.T) { + t.Parallel() + for _, mode := range []pb.MigrationCleanupMode{pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA} { + t.Run(mode.String(), func(t *testing.T) { + t.Parallel() + ctx := context.Background() + _, err := cleanupSplitMigration(ctx, &stalledCleanupClient{}, &pb.CleanupMigrationRequest{JobId: 7, Mode: mode}) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.NoError(t, ctx.Err()) + }) + } +} + +type stalledControlClient struct { + splitMigrationClientStub +} + +func (*stalledControlClient) ApplyTargetStagedReadiness(ctx context.Context, _ *pb.TargetStagedReadinessRequest, _ ...grpc.CallOption) (*pb.TargetStagedReadinessResponse, error) { + <-ctx.Done() + return nil, ctx.Err() +} + +func (*stalledControlClient) IssueMigrationTimestamp(ctx context.Context, _ *pb.IssueMigrationTimestampRequest, _ ...grpc.CallOption) (*pb.IssueMigrationTimestampResponse, error) { + <-ctx.Done() + return nil, ctx.Err() +} + +func (*stalledControlClient) ProbeMigrationLocks(ctx context.Context, _ *pb.ProbeMigrationLocksRequest, _ ...grpc.CallOption) (*pb.ProbeMigrationLocksResponse, error) { + <-ctx.Done() + return nil, ctx.Err() +} + +func TestSplitJobControlRPCsHaveIndependentDeadlines(t *testing.T) { + t.Parallel() + job := splitJobForSourceGroupTest(t) + for _, tc := range []struct { + name string + run func(context.Context, SplitMigrationClient) error + }{ + {"fence", func(ctx context.Context, client SplitMigrationClient) error { + _, err := applySplitMigrationControl(ctx, client, job, []byte("z"), 1, 1, true, false, true, 1) + return err + }}, + {"capability guard", func(ctx context.Context, client SplitMigrationClient) error { + _, err := (&DistributionServer{}).applyAndProbeSplitMigrationControl(ctx, 7, client, &pb.TargetStagedReadinessRequest{JobId: 7}) + return err + }}, + {"timestamp", func(ctx context.Context, client SplitMigrationClient) error { + _, err := issueSplitMigrationTimestamp(ctx, client) + return err + }}, + {"lock drain", func(ctx context.Context, client SplitMigrationClient) error { + _, err := splitJobPendingLocks(ctx, client, []byte("m"), []byte("z")) + return err + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + require.ErrorIs(t, tc.run(ctx, &stalledControlClient{}), context.DeadlineExceeded) + require.NoError(t, ctx.Err()) + }) + } +} diff --git a/adapter/split_job_runner.go b/adapter/split_job_runner.go new file mode 100644 index 000000000..8bb8e3bf8 --- /dev/null +++ b/adapter/split_job_runner.go @@ -0,0 +1,2350 @@ +package adapter + +import ( + "bytes" + "context" + "encoding/binary" + "io" + "math" + "sort" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" +) + +const ( + defaultSplitMigrationChunkBytes = 1 << 20 + defaultSplitMigrationMaxScannedBytes = 4 << 20 + defaultSplitMigrationLockProbeLimit = 1024 + defaultSplitMigrationReadFenceGrace = 30 * time.Second + defaultSplitMigrationVoterProbeTimeout = 500 * time.Millisecond + defaultSplitMigrationAttemptTimeout = 2 * time.Second + splitCleanupCursorVersion = byte(1) + splitVoterAckCursorVersion = byte(1) + splitVoterAckConfigurationVersion = byte(2) + splitCleanupCursorHeaderBytes = 5 + splitJobHistoryLimit = 1000 + splitJobHistoryDeleteBatchLimit = 1000 + splitJobHistoryTTL = 7 * 24 * time.Hour + splitJobHistoryGCInterval = time.Minute +) + +var ( + splitCleanupDoneCursor = []byte{splitCleanupCursorVersion, 0xff, 0xff, 0xff, 0xff} + splitSourceCleanupDoneCursor = []byte{2, 0xff, 0xff, 0xff, 0xff} + errSplitMigrationVoterBarrierIncomplete = errors.New("split migration voter barrier incomplete") +) + +type splitVoterAck struct { + id string + address string + acked bool + configurationIndex uint64 +} + +func splitMigrationVoterBarrierIncompleteError(jobID uint64, kind pb.MigrationStateProbeKind) error { + return errors.Wrapf(errSplitMigrationVoterBarrierIncomplete, "job_id=%d kind=%s", jobID, kind.String()) +} + +func (s *DistributionServer) syncSplitMigrationVoterBarrier( + ctx context.Context, + groupID uint64, + existing []byte, + fallback SplitMigrationClient, + req *pb.ProbeMigrationStateRequest, +) ([]byte, bool, error) { + voters, err := s.splitMigrationVoters(ctx, groupID, fallback) + if err != nil { + return nil, false, err + } + if _, err := decodeSplitVoterAckCursor(existing); err != nil { + return nil, false, err + } + acks := make([]splitVoterAck, 0, len(voters)) + complete := true + for _, voter := range voters { + ack := probeSplitMigrationVoter(ctx, voter, req) + complete = complete && ack.acked + acks = append(acks, ack) + } + if req.GetRequireConfigurationIndex() && splitCleanupAckConfiguration(acks) == 0 { + complete = false + } + return encodeSplitVoterAckCursor(acks), complete, nil +} + +func probeSplitMigrationVoter(ctx context.Context, voter SplitMigrationVoter, req *pb.ProbeMigrationStateRequest) splitVoterAck { + attempt, cancel := context.WithTimeout(ctx, defaultSplitMigrationVoterProbeTimeout) + defer cancel() + resp, err := voter.Client.ProbeMigrationState(attempt, req) + index := resp.GetAppliedConfigurationIndex() + acked := err == nil && resp.GetReady() + if req.GetRequireConfigurationIndex() && (index == 0 || + (req.GetExpectedConfigurationIndex() != 0 && index != req.GetExpectedConfigurationIndex())) { + acked = false + } + return splitVoterAck{id: voter.ID, address: voter.Address, acked: acked, configurationIndex: index} +} + +func (s *DistributionServer) splitMigrationVoters(ctx context.Context, groupID uint64, fallback SplitMigrationClient) ([]SplitMigrationVoter, error) { + if s.splitMigrationVoterFactory == nil { + if fallback == nil { + return nil, errors.New("split migration voter factory is not configured") + } + return []SplitMigrationVoter{{ID: "leader", Address: "leader", Client: fallback}}, nil + } + voters, err := s.splitMigrationVoterFactory(ctx, groupID) + if err != nil { + return nil, errors.WithStack(err) + } + if len(voters) == 0 { + return nil, errors.New("split migration voter set is empty") + } + if err := validateSplitMigrationVoters(voters); err != nil { + return nil, err + } + return voters, nil +} + +func validateSplitMigrationVoters(voters []SplitMigrationVoter) error { + sort.Slice(voters, func(i, j int) bool { + if voters[i].ID == voters[j].ID { + return voters[i].Address < voters[j].Address + } + return voters[i].ID < voters[j].ID + }) + for index, voter := range voters { + if voter.ID == "" || voter.Address == "" || voter.Client == nil { + return errors.New("split migration voter is incomplete") + } + if index > 0 && voter.ID == voters[index-1].ID { + return errors.New("split migration voter id is duplicated") + } + } + return nil +} + +func encodeSplitVoterAckCursor(acks []splitVoterAck) []byte { + version := splitVoterAckCursorVersion + for _, ack := range acks { + if ack.configurationIndex != 0 { + version = splitVoterAckConfigurationVersion + } + } + out := []byte{version} + out = binary.AppendUvarint(out, uint64(len(acks))) + for _, ack := range acks { + out = binary.AppendUvarint(out, uint64(len(ack.id))) + out = append(out, ack.id...) + out = binary.AppendUvarint(out, uint64(len(ack.address))) + out = append(out, ack.address...) + if ack.acked { + out = append(out, 1) + } else { + out = append(out, 0) + } + if version == splitVoterAckConfigurationVersion { + out = binary.AppendUvarint(out, ack.configurationIndex) + } + } + return out +} + +func decodeSplitVoterAckCursor(raw []byte) ([]splitVoterAck, error) { + if len(raw) == 0 { + return nil, nil + } + version := raw[0] + if version != splitVoterAckCursorVersion && version != splitVoterAckConfigurationVersion { + return nil, errors.New("invalid split voter ack cursor version") + } + raw = raw[1:] + count, rest, err := consumeSplitCursorUvarint(raw) + if err != nil { + return nil, err + } + raw = rest + if count > uint64(len(raw)) { + return nil, errors.New("split voter ack cursor count exceeds payload") + } + acks := make([]splitVoterAck, 0, count) + for range count { + ack, next, err := consumeSplitVoterAck(raw, version) + if err != nil { + return nil, err + } + raw = next + acks = append(acks, ack) + } + if len(raw) != 0 { + return nil, errors.New("split voter ack cursor has trailing bytes") + } + return acks, nil +} + +func consumeSplitVoterAck(raw []byte, version byte) (splitVoterAck, []byte, error) { + id, next, err := consumeSplitCursorString(raw) + if err != nil { + return splitVoterAck{}, nil, err + } + address, next, err := consumeSplitCursorString(next) + if err != nil || len(next) == 0 || next[0] > 1 { + return splitVoterAck{}, nil, errors.New("invalid split voter ack cursor entry") + } + ack := splitVoterAck{id: id, address: address, acked: next[0] == 1} + next = next[1:] + if version == splitVoterAckConfigurationVersion { + ack.configurationIndex, next, err = consumeSplitCursorUvarint(next) + } + return ack, next, err +} + +func consumeSplitCursorString(raw []byte) (string, []byte, error) { + size, rest, err := consumeSplitCursorUvarint(raw) + if err != nil || size > uint64(len(rest)) { + return "", nil, errors.New("invalid split voter ack cursor string") + } + return string(rest[:size]), rest[size:], nil +} + +func consumeSplitCursorUvarint(raw []byte) (uint64, []byte, error) { + value, size := binary.Uvarint(raw) + if size <= 0 { + return 0, nil, errors.New("invalid split voter ack cursor integer") + } + return value, raw[size:], nil +} + +func (s *DistributionServer) runSplitJobPhase(ctx context.Context, job distribution.SplitJob) error { + if err := s.verifySplitJobCleanupDispatch(ctx, job); err != nil { + return err + } + if s.splitMigrationClientFactory == nil { + return errors.New("split migration client factory is not configured") + } + if err := s.verifySplitJobMigrationRange(ctx, job); err != nil { + return err + } + job, err := s.backfillSplitJobSourceGroup(ctx, job) + if err != nil { + return err + } + if splitJobTransferNamespaceRequired(job.Phase) && s.MigrationPartitionRuntimeConfigured(job.SourceGroupID) { + if err := s.ensureSplitJobNamespaceAdmission(ctx, job); err != nil { + return err + } + } + if job.Phase == distribution.SplitJobPhaseBackfill || job.Phase == distribution.SplitJobPhaseDeltaCopy { + return s.runSplitJobCopyPhase(ctx, job) + } + return s.runSplitJobControlPhase(ctx, job) +} + +func (s *DistributionServer) verifySplitJobMigrationRange(ctx context.Context, job distribution.SplitJob) error { + // Recovery must remain possible for jobs admitted by an older binary. + if job.Phase == distribution.SplitJobPhaseAbandoning || job.Phase == distribution.SplitJobPhaseCleanup { + return nil + } + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return err + } + _, routeEnd, found := splitJobSourceRouteState(snapshot.Routes, job) + if !found { + return errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + if err := distribution.ValidateSplitMigrationRouteRange(job.SplitKey, routeEnd); err != nil { + return splitMigrationRangeStatusError(err) + } + return nil +} + +// backfillSplitJobSourceGroup records SourceGroupID on a job that predates the +// field, using current routes or retained catalog ancestry. +// +// InitializeSplitJobPlan fills the field for every job it creates, but it runs +// only at plan time: a job persisted by an earlier binary decodes as zero and is +// never re-planned, so nothing ever populates it. +// +// That matters because the field is the last-resort handle +// splitJobSourceRouteState falls back to, and it is needed in exactly the state +// where the route shape stops answering -- once FENCE has replaced the parent +// AND the source child has been split again (which SplitRange permits, the new +// split being disjoint from the moving range), the grandchildren no longer name +// the original parent and the parent itself is gone. A legacy job that reached +// that state could never resolve its source again: CLEANUP failed with +// ErrMigrationSourceRouteChanged on every attempt and held the job, its guards +// and its retention pin indefinitely. +// +// The historical fallback also repairs jobs already in that degraded state. +// The write is a no-op once the field is set, and is skipped for phases that +// will never read it. +func (s *DistributionServer) backfillSplitJobSourceGroup( + ctx context.Context, + job distribution.SplitJob, +) (distribution.SplitJob, error) { + if job.SourceGroupID != 0 || !splitJobPhaseNeedsSourceGroup(job.Phase) { + return job, nil + } + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return job, err + } + groupID, needed, err := s.resolveLegacySplitJobSourceGroup(ctx, job, snapshot.Routes) + if err != nil { + return job, err + } + if !needed { + return job, nil + } + if err := s.updateSplitJobViaCoordinator(ctx, job.JobID, + func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.SourceGroupID != 0 { + return current, nil + } + next := distribution.CloneSplitJob(current) + next.SourceGroupID = groupID + next.UpdatedAtMs = time.Now().UnixMilli() + return next, nil + }); err != nil { + return job, err + } + job.SourceGroupID = groupID + return job, nil +} + +func (s *DistributionServer) resolveLegacySplitJobSourceGroup(ctx context.Context, job distribution.SplitJob, routes []distribution.RouteDescriptor) (uint64, bool, error) { + if groupID, needed := splitJobSourceGroupBackfill(routes, job); needed { + return groupID, true, nil + } + groupID, found, historyErr := s.legacySplitJobSourceGroup(ctx, job) + if found { + return groupID, true, nil + } + groupID, found, err := s.discoverSplitJobSourceGroup(ctx, job, routes) + if err != nil { + return 0, false, err + } + if found { + return groupID, true, nil + } + return 0, false, historyErr +} + +// A current predecessor may itself have migrated. Recover the original owner +// from catalog ancestry instead of guessing from its present-day group. +func (s *DistributionServer) legacySplitJobSourceGroup(ctx context.Context, job distribution.SplitJob) (uint64, bool, error) { + var lastErr error + for _, ts := range []uint64{job.SnapshotTS, job.FenceTS} { + if ts == 0 { + continue + } + snapshot, err := s.loadCatalogSnapshotAt(ctx, ts) + if err != nil { + lastErr = err + continue + } + if groupID, needed := splitJobSourceGroupBackfill(snapshot.Routes, job); needed { + return groupID, true, nil + } + } + return 0, false, lastErr +} + +func (s *DistributionServer) discoverSplitJobSourceGroup(ctx context.Context, job distribution.SplitJob, routes []distribution.RouteDescriptor) (uint64, bool, error) { + if s.splitMigrationClientFactory == nil { + return 0, false, nil + } + for _, route := range routes { + if route.ParentRouteID == job.SourceRouteID && bytes.Equal(route.Start, job.SplitKey) { + return s.probeSplitJobSourceGroups(ctx, job, route.End) + } + } + return 0, false, nil +} + +func (s *DistributionServer) probeSplitJobSourceGroups(ctx context.Context, job distribution.SplitJob, routeEnd []byte) (uint64, bool, error) { + var owner uint64 + for groupID := range s.knownRaftGroups { + ready, err := s.probeSplitJobSourceGroup(ctx, job, groupID, routeEnd) + if err != nil { + return 0, false, err + } + if !ready { + continue + } + if owner != 0 { + return 0, false, errors.New("multiple groups report a source guard for the migration job") + } + owner = groupID + } + return owner, owner != 0, nil +} + +func (s *DistributionServer) probeSplitJobSourceGroup(ctx context.Context, job distribution.SplitJob, groupID uint64, routeEnd []byte) (bool, error) { + attemptCtx, cancel := context.WithTimeout(ctx, defaultSplitMigrationVoterProbeTimeout) + defer cancel() + source, err := s.splitJobSourceClient(attemptCtx, job, groupID) + if err != nil { + return false, errors.WithStack(err) + } + if source == nil { + return false, errors.New("split migration source discovery client is nil") + } + resp, err := source.ProbeMigrationState(attemptCtx, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_GUARD_PRESENT, + RouteStart: job.SplitKey, RouteEnd: routeEnd, + }) + if err != nil { + return false, errors.WithStack(err) + } + return resp.GetReady(), nil +} + +// splitJobSourceGroupBackfill decides whether job needs its SourceGroupID +// recorded, and which group to record. +// +// Kept separate from the catalog write so the decision is testable without a +// live catalog or coordinator: every case that must NOT write -- the field +// already set, a phase that will never read it, a route shape that cannot +// answer -- is a pure-function case. +func splitJobSourceGroupBackfill( + routes []distribution.RouteDescriptor, + job distribution.SplitJob, +) (uint64, bool) { + if job.SourceGroupID != 0 || !splitJobPhaseNeedsSourceGroup(job.Phase) { + return 0, false + } + groupID, _, ok := splitJobSourceRouteState(routes, job) + if !ok || groupID == 0 { + // The shape cannot answer either. Leave the job alone and let the phase + // surface the real error rather than masking it as a backfill failure. + return 0, false + } + return groupID, true +} + +// splitJobPhaseNeedsSourceGroup reports whether a job in this phase can still +// reach a source-side operation, and therefore whether it is worth recording the +// source group for it. Terminal and unplanned phases never read the field. +func splitJobPhaseNeedsSourceGroup(phase distribution.SplitJobPhase) bool { + switch phase { + case distribution.SplitJobPhasePlanned, + distribution.SplitJobPhaseBackfill, + distribution.SplitJobPhaseDeltaCopy, + distribution.SplitJobPhaseFence, + distribution.SplitJobPhaseCutover, + distribution.SplitJobPhaseCleanup, + distribution.SplitJobPhaseAbandoning: + return true + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoned: + return false + } + return false +} + +func (s *DistributionServer) runSplitJobCopyPhase(ctx context.Context, job distribution.SplitJob) error { + if job.Phase == distribution.SplitJobPhaseBackfill { + return s.copySplitJobPhase(ctx, job, distribution.SplitJobExportPhaseBackfill, 0, job.SnapshotTS) + } + ready, err := s.preflightSplitJobTransfer(ctx, job) + if err != nil || !ready { + return err + } + return s.copySplitJobPhase(ctx, job, distribution.SplitJobExportPhaseDeltaCopy, job.DeltaFloor, job.FenceTS) +} + +func (s *DistributionServer) runSplitJobControlPhase(ctx context.Context, job distribution.SplitJob) error { + switch job.Phase { + case distribution.SplitJobPhasePlanned: + return s.beginSplitJobBackfill(ctx, job) + case distribution.SplitJobPhaseFence: + return s.finalizeSplitJobFence(ctx, job) + case distribution.SplitJobPhaseCutover: + return s.cutoverSplitJob(ctx, job) + case distribution.SplitJobPhaseCleanup: + return s.cleanupSplitJob(ctx, job) + case distribution.SplitJobPhaseAbandoning: + return s.abandonSplitJob(ctx, job) + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseBackfill, + distribution.SplitJobPhaseDeltaCopy, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoned: + return nil + } + return errors.New("split job phase is not runnable") +} + +func (s *DistributionServer) cleanupSplitJob(ctx context.Context, job distribution.SplitJob) error { + if err := s.verifySplitJobCleanupCutover(ctx, job); err != nil { + return err + } + return s.cleanupVerifiedSplitJob(ctx, job) +} + +func (s *DistributionServer) cleanupVerifiedSplitJob(ctx context.Context, job distribution.SplitJob) error { + if !job.TargetPromotionDone { + _, target, _, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + if err := s.applySplitOwnership(ctx, job, job.TargetGroupID, target, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_ACQUIRE); err != nil { + return err + } + return s.promoteSplitJobTargetAndComplete(ctx, job) + } + if !bytes.Equal(job.TargetClearedDescriptorAckCursor, splitCleanupDoneCursor) { + if err := s.cleanupSplitJobTargetProofs(ctx, job); err != nil { + return err + } + return nil + } + if complete, err := s.refreshSplitJobSourceCleanupBarriers(ctx, job); err != nil || !complete { + return err + } + if !bytes.Equal(job.Cursor, splitSourceCleanupDoneCursor) { + return s.cleanupSplitJobSourceData(ctx, job) + } + if job.SourceRetentionPinTS != math.MaxUint64 { + return s.cleanupSplitJobSourceProofs(ctx, job) + } + return s.finishSplitJobHistory(ctx, job, distribution.SplitJobPhaseDone) +} + +func (s *DistributionServer) verifySplitJobCleanupCutover(ctx context.Context, job distribution.SplitJob) error { + snapshot, err := s.loadCommittedMigrationCatalog(ctx) + if err != nil { + return err + } + return verifySplitJobCleanupSnapshot(job, snapshot) +} + +// Validate before capability recovery or source discovery can dispatch RPCs. +func (s *DistributionServer) verifySplitJobCleanupDispatch(ctx context.Context, job distribution.SplitJob) error { + if job.Phase != distribution.SplitJobPhaseCleanup { + return nil + } + snapshot, err := s.loadCommittedMigrationCatalog(ctx) + if err != nil { + return err + } + current, found, err := s.catalog.SplitJobAt(ctx, job.JobID, snapshot.ReadTS) + if err != nil { + return splitJobCatalogStatusError(err) + } + if !found || !distribution.SplitJobsEquivalent(current, job) { + return errors.WithStack(distribution.ErrCatalogSplitJobConflict) + } + return verifySplitJobCleanupSnapshot(current, snapshot) +} + +func verifySplitJobCleanupSnapshot(job distribution.SplitJob, snapshot distribution.CatalogSnapshot) error { + if !validSplitJobCleanupWitness(job, snapshot.Version) { + return errors.New("split cleanup lacks a valid cutover witness") + } + if _, found := findRouteByID(snapshot.Routes, job.SourceRouteID); found { + return errors.New("split cleanup source route has not been cut over") + } + for _, route := range snapshot.Routes { + if splitJobCleanupTargetAttests(route, job) { + return nil + } + } + return errors.New("split cleanup target descriptor does not attest the cutover") +} + +func validSplitJobCleanupWitness(job distribution.SplitJob, catalogVersion uint64) bool { + return job.CutoverVersion != 0 && job.CutoverVersion <= catalogVersion && + job.FenceTS != 0 && job.FenceTS != math.MaxUint64 && job.FenceTS >= job.SnapshotTS && + job.SourceGroupID != 0 && job.SourceGroupID != job.TargetGroupID +} + +func splitJobCleanupTargetAttests(route distribution.RouteDescriptor, job distribution.SplitJob) bool { + if !splitJobCleanupTargetIdentity(route, job) { + return false + } + if job.TargetPromotionDone { + return !route.StagedVisibilityActive && route.MigrationJobID == 0 + } + return route.StagedVisibilityActive && route.MigrationJobID == job.JobID +} + +func splitJobCleanupTargetIdentity(route distribution.RouteDescriptor, job distribution.SplitJob) bool { + return route.ParentRouteID == job.SourceRouteID && bytes.Equal(route.Start, job.SplitKey) && + route.GroupID == job.TargetGroupID && route.State == distribution.RouteStateActive && route.MinWriteTSExclusive == job.FenceTS +} + +func (s *DistributionServer) splitJobMigrationClients(ctx context.Context, job distribution.SplitJob) (SplitMigrationClient, SplitMigrationClient, []byte, error) { + if s.splitMigrationClientFactory == nil { + return nil, nil, nil, errors.New("split migration client factory is not configured") + } + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return nil, nil, nil, err + } + sourceGroupID, routeEnd, ok := splitJobSourceRouteState(snapshot.Routes, job) + if !ok { + return nil, nil, nil, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + source, target, err := s.splitMigrationClientFactory(ctx, job, sourceGroupID) + if err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if source == nil || target == nil { + return nil, nil, nil, errors.New("split migration source or target client is nil") + } + return source, target, routeEnd, nil +} + +func (s *DistributionServer) cleanupSplitJobTargetProofs(ctx context.Context, job distribution.SplitJob) error { + _, target, routeEnd, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + descriptorCursor, descriptorComplete, err := s.syncSplitMigrationVoterBarrier(ctx, job.TargetGroupID, job.TargetClearedDescriptorAckCursor, target, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED, + RouteStart: job.SplitKey, + RouteEnd: routeEnd, + ExpectedCatalogVersion: job.CutoverVersion, + ExpectedGroupId: job.TargetGroupID, + MinWriteTsExclusive: job.FenceTS, + }) + if err != nil { + return err + } + if !descriptorComplete { + return s.persistSplitJobTargetCleanupCursor(ctx, job.JobID, descriptorCursor) + } + if err := s.applySplitOwnership(ctx, job, job.TargetGroupID, target, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR); err != nil { + return errors.WithStack(err) + } + _, metadataComplete, err := s.syncSplitMigrationVoterBarrier(ctx, job.TargetGroupID, nil, target, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + if err != nil { + return err + } + if !metadataComplete { + return splitMigrationVoterBarrierIncompleteError(job.JobID, pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED) + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup && current.TargetPromotionDone { + current.TargetClearedDescriptorAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) persistSplitJobTargetCleanupCursor(ctx context.Context, jobID uint64, cursor []byte) error { + return s.updateSplitJobViaCoordinator(ctx, jobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup { + current.TargetClearedDescriptorAckCursor = distribution.CloneBytes(cursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) ackSplitJobSourceRouteRemoval(ctx context.Context, job distribution.SplitJob) (bool, error) { + source, _, routeEnd, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return false, err + } + sourceGroupID, err := s.splitJobSourceGroupID(ctx, job) + if err != nil { + return false, err + } + cursor, complete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceGroupID, reusableSplitCleanupAckCursor(job.SourceCutoverAckCursor), source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED, + RouteStart: job.SplitKey, + RouteEnd: routeEnd, + ExpectedCatalogVersion: job.CutoverVersion, + ExpectedGroupId: job.TargetGroupID, + RequireConfigurationIndex: true, + }) + if err != nil { + return false, err + } + if bytes.Equal(cursor, job.SourceCutoverAckCursor) { + return complete, nil + } + err = s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup { + current.SourceCutoverAckCursor = distribution.CloneBytes(cursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + return complete, err +} + +func (s *DistributionServer) ackSplitJobSourceReadDrain(ctx context.Context, job distribution.SplitJob) (bool, error) { + configurationIndex, err := splitCleanupCursorConfiguration(job.SourceCutoverAckCursor) + if err != nil { + return false, err + } + source, _, _, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return false, err + } + sourceGroupID, err := s.splitJobSourceGroupID(ctx, job) + if err != nil { + return false, err + } + notBefore := int64(job.PromotionCompletedTS>>kv.HLCLogicalBits) + defaultSplitMigrationReadFenceGrace.Milliseconds() //nolint:gosec // HLC physical millis fit int64. + cursor, complete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceGroupID, reusableSplitCleanupAckCursor(job.SourceReadDrainCursor), source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED, + ReadDrainNotBeforeMs: notBefore, + ReadDrainMinTs: job.PromotionCompletedTS, + RequireConfigurationIndex: true, + ExpectedConfigurationIndex: configurationIndex, + }) + if err != nil { + return false, err + } + if bytes.Equal(cursor, job.SourceReadDrainCursor) { + return complete, nil + } + err = s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup { + current.SourceReadDrainCursor = distribution.CloneBytes(cursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + return complete, err +} + +func reusableSplitCleanupAckCursor(cursor []byte) []byte { + if bytes.Equal(cursor, splitCleanupDoneCursor) { + return nil + } + return cursor +} + +func (s *DistributionServer) refreshSplitJobSourceCleanupBarriers(ctx context.Context, job distribution.SplitJob) (bool, error) { + complete, err := s.ackSplitJobSourceRouteRemoval(ctx, job) + if err != nil || !complete { + return false, err + } + // Route ACK publication may have replaced a legacy cursor or observed a + // new configuration. Bind the read-drain ACK to that exact durable proof. + job, found, err := s.catalog.SplitJob(ctx, job.JobID) + if err != nil || !found { + return false, errors.Wrap(errSplitMigrationVoterBarrierIncomplete, "source cleanup job proof unavailable") + } + return s.ackSplitJobSourceReadDrain(ctx, job) +} + +func (s *DistributionServer) splitJobSourceGroupID(ctx context.Context, job distribution.SplitJob) (uint64, error) { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return 0, err + } + groupID, _, ok := splitJobSourceRouteState(snapshot.Routes, job) + if !ok { + return 0, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + return groupID, nil +} + +func (s *DistributionServer) cleanupSplitJobSourceData(ctx context.Context, job distribution.SplitJob) error { + source, _, routeEnd, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + brackets, err := distribution.PlanCleanupBrackets(job.SplitKey, routeEnd) + if err != nil { + return errors.WithStack(err) + } + index, cursor, done, err := decodeSplitSourceCleanupCursor(job.Cursor, len(brackets)) + if err != nil || done { + return err + } + bracket := brackets[index] + req := splitMigrationCleanupRequest(job, routeEnd, bracket, cursor, job.FenceTS) + if err := s.bindSplitSourceCleanup(ctx, job, source, routeEnd, req); err != nil { + return err + } + resp, err := cleanupSplitMigration(ctx, source, req) + if err != nil { + return errors.WithStack(err) + } + encoded, err := nextSplitCleanupCursor(resp, cursor, index, len(brackets)) + if err != nil { + return err + } + encoded = splitSourceCleanupCursor(encoded) + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup && bytes.Equal(current.Cursor, job.Cursor) { + current.Cursor = encoded + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func splitSourceCleanupCursor(encoded []byte) []byte { + if bytes.Equal(encoded, splitCleanupDoneCursor) { + return distribution.CloneBytes(splitSourceCleanupDoneCursor) + } + return encoded +} + +func decodeSplitSourceCleanupCursor(raw []byte, bracketCount int) (int, []byte, bool, error) { + if bytes.Equal(raw, splitSourceCleanupDoneCursor) { + return bracketCount, nil, true, nil + } + if bytes.Equal(raw, splitCleanupDoneCursor) { + // A live job completed the old plan before seals had a cleanup bracket. + return bracketCount - 1, nil, false, nil + } + return decodeSplitCleanupCursor(raw, bracketCount) +} + +func nextSplitCleanupCursor(resp *pb.CleanupMigrationResponse, cursor []byte, index, bracketCount int) ([]byte, error) { + if !resp.GetDone() { + next := resp.GetNextCursor() + if len(next) == 0 || bytes.Equal(next, cursor) { + return nil, errors.New("split migration source cleanup made no cursor progress") + } + return encodeSplitCleanupCursor(index, next, false), nil + } + nextIndex := index + 1 + return encodeSplitCleanupCursor(nextIndex, nil, nextIndex >= bracketCount), nil +} + +func splitMigrationCleanupRequest(job distribution.SplitJob, routeEnd []byte, bracket distribution.MigrationBracket, cursor []byte, maxCommitTS uint64) *pb.CleanupMigrationRequest { + return &pb.CleanupMigrationRequest{ + JobId: job.JobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, + RangeStart: bracket.Start, + RangeEnd: bracket.End, + Cursor: cursor, + MaxCommitTs: maxCommitTS, + MaxVersions: defaultSplitPromotionMaxVersions, + MaxBytes: defaultSplitPromotionMaxBytes, + MaxScannedBytes: defaultSplitPromotionMaxScanned, + KeyFamily: bracket.Family, + RouteStart: job.SplitKey, + RouteEnd: routeEnd, + ExcludeKnownInternal: bracket.ExcludeKnownInternal, + ExcludePrefixes: bracket.ExcludePrefixes, + RequiresRouteKeyCheck: bracket.RequiresRouteKeyCheck, + RequiresDecodedS3: bracket.RequiresDecodedS3, + } +} + +func (s *DistributionServer) cleanupSplitJobSourceProofs(ctx context.Context, job distribution.SplitJob) error { + complete, err := s.refreshSplitJobSourceCleanupBarriers(ctx, job) + if err != nil { + return err + } + if !complete { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + if err := s.cleanupSplitJobSourceMetadata(ctx, job); err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup { + current.SourceRetentionPinTS = math.MaxUint64 + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func encodeSplitCleanupCursor(index int, cursor []byte, done bool) []byte { + if done { + return distribution.CloneBytes(splitCleanupDoneCursor) + } + out := make([]byte, splitCleanupCursorHeaderBytes+len(cursor)) + out[0] = splitCleanupCursorVersion + binary.BigEndian.PutUint32(out[1:], uint32(index)) //nolint:gosec // bracket count is bounded. + copy(out[splitCleanupCursorHeaderBytes:], cursor) + return out +} + +func decodeSplitCleanupCursor(raw []byte, bracketCount int) (int, []byte, bool, error) { + if len(raw) == 0 { + return 0, nil, bracketCount == 0, nil + } + if bytes.Equal(raw, splitCleanupDoneCursor) { + return bracketCount, nil, true, nil + } + if len(raw) < splitCleanupCursorHeaderBytes || raw[0] != splitCleanupCursorVersion { + return 0, nil, false, errors.New("invalid split cleanup cursor") + } + index := int(binary.BigEndian.Uint32(raw[1:splitCleanupCursorHeaderBytes])) + if index < 0 || index >= bracketCount { + return 0, nil, false, errors.New("split cleanup cursor bracket is out of range") + } + return index, distribution.CloneBytes(raw[splitCleanupCursorHeaderBytes:]), false, nil +} + +func (s *DistributionServer) abandonSplitJob(ctx context.Context, job distribution.SplitJob) error { + // Publish the actual rollback descriptor while retaining the durable source + // guard. Only then replace that guard with OWNER evidence. Target cleanup + // is deliberately later and may remain unavailable indefinitely. + if !bytes.Equal(job.SourceCutoverAckCursor, splitCleanupDoneCursor) { + if err := s.prepareSplitOwnershipRollback(ctx, job); err != nil { + return err + } + if err := s.rollbackAbandonedSplitJobFence(ctx, job); err != nil { + return err + } + return s.cleanupAbandonedSplitJobSource(ctx, job) + } + if !bytes.Equal(job.TargetClearedDescriptorAckCursor, splitCleanupDoneCursor) { + return s.cleanupAbandonedSplitJobTarget(ctx, job) + } + return s.finishSplitJobHistory(ctx, job, distribution.SplitJobPhaseAbandoned) +} + +func (s *DistributionServer) cleanupAbandonedSplitJobTarget(ctx context.Context, job distribution.SplitJob) error { + _, target, _, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + _, cursor, done, err := decodeSplitCleanupCursor(job.TargetClearedDescriptorAckCursor, 1) + if err != nil { + return err + } + if !done { + complete, err := s.cleanupAbandonedSplitJobTargetVersions(ctx, target, job, cursor) + if err != nil || !complete { + return err + } + } + if err := s.applySplitOwnership(ctx, job, job.TargetGroupID, target, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR); err != nil { + return errors.WithStack(err) + } + _, metadataComplete, err := s.syncSplitMigrationVoterBarrier(ctx, job.TargetGroupID, nil, target, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + if err != nil { + return err + } + if !metadataComplete { + return splitMigrationVoterBarrierIncompleteError(job.JobID, pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED) + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseAbandoning { + current.TargetClearedDescriptorAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) cleanupAbandonedSplitJobTargetVersions( + ctx context.Context, + target SplitMigrationClient, + job distribution.SplitJob, + cursor []byte, +) (bool, error) { + prefix := distribution.MigrationStagedDataKeyPrefix(job.JobID) + resp, err := cleanupSplitMigration(ctx, target, &pb.CleanupMigrationRequest{ + JobId: job.JobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, + RangeStart: prefix, + RangeEnd: store.PrefixScanEnd(prefix), + Cursor: cursor, + MaxCommitTs: math.MaxUint64, + MaxVersions: defaultSplitPromotionMaxVersions, + MaxBytes: defaultSplitPromotionMaxBytes, + MaxScannedBytes: defaultSplitPromotionMaxScanned, + KeyFamily: distribution.MigrationFamilyUser, + }) + if err != nil { + return false, errors.WithStack(err) + } + if resp.GetDone() { + return true, nil + } + if len(resp.GetNextCursor()) == 0 || bytes.Equal(resp.GetNextCursor(), cursor) { + return false, errors.New("split migration abandon cleanup made no cursor progress") + } + encoded := encodeSplitCleanupCursor(0, resp.GetNextCursor(), false) + err = s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseAbandoning && bytes.Equal(current.TargetClearedDescriptorAckCursor, job.TargetClearedDescriptorAckCursor) { + current.TargetClearedDescriptorAckCursor = encoded + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + return false, err +} + +func (s *DistributionServer) cleanupAbandonedSplitJobSource(ctx context.Context, job distribution.SplitJob) error { + if err := s.cleanupSplitJobSourceMetadata(ctx, job); err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseAbandoning { + current.SourceCutoverAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + current.SourceRetentionPinTS = math.MaxUint64 + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) cleanupSplitJobSourceMetadata(ctx context.Context, job distribution.SplitJob) error { + sourceGroupID, err := s.splitJobSourceGroupID(ctx, job) + if err != nil { + return err + } + source, err := s.splitJobSourceClient(ctx, job, sourceGroupID) + if err != nil { + return err + } + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return err + } + _, routeEnd, ok := splitJobSourceRouteState(snapshot.Routes, job) + if !ok { + return errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + req := &pb.CleanupMigrationRequest{ + JobId: job.JobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA, + } + if err := s.bindSplitSourceCleanup(ctx, job, source, routeEnd, req); err != nil { + return err + } + if err := s.applySplitOwnership(ctx, job, sourceGroupID, source, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR, req.GetExpectedConfigurationIndex()); err != nil { + return errors.WithStack(err) + } + _, metadataComplete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceGroupID, nil, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + if err != nil { + return err + } + if !metadataComplete { + return splitMigrationVoterBarrierIncompleteError(job.JobID, pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED) + } + return nil +} + +func (s *DistributionServer) splitJobSourceClient(ctx context.Context, job distribution.SplitJob, groupID uint64) (SplitMigrationClient, error) { + var client SplitMigrationClient + var err error + switch { + case s.splitMigrationGroupFactory != nil: + client, err = s.splitMigrationGroupFactory(ctx, groupID) + case s.splitMigrationClientFactory != nil: + // Preserve existing embedders, including source discovery candidates. + client, _, err = s.splitMigrationClientFactory(ctx, job, groupID) + default: + return nil, errors.New("split migration client factory is not configured") + } + if err != nil { + return nil, errors.WithStack(err) + } + if client == nil { + return nil, errors.New("split migration source client is nil") + } + return client, nil +} + +func (s *DistributionServer) rollbackAbandonedSplitJobFence(ctx context.Context, job distribution.SplitJob) error { + snapshot, err := s.loadCommittedMigrationCatalog(ctx) + if err != nil { + return err + } + fenced := splitOwnershipRollbackRoute(snapshot.Routes, job) + if fenced == nil { + return errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + floor := max(initialMigrationRetentionPinTS, job.SnapshotTS, job.FenceTS) + if fenced.State == distribution.RouteStateActive && fenced.MinWriteTSExclusive >= floor { + return nil + } + if snapshot.Version == math.MaxUint64 { + return errors.New("split migration catalog version overflow") + } + fenced.State = distribution.RouteStateActive + fenced.MinWriteTSExclusive = max(fenced.MinWriteTSExclusive, floor) + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(*fenced, s.catalog.AllowsRouteDescriptorV2Writes()) + if err != nil { + return errors.WithStack(err) + } + nextVersion := snapshot.Version + 1 + if err := s.commitAbandonedSplitJobFenceRollback(ctx, snapshot, *fenced, encoded, nextVersion); err != nil { + return err + } + loaded, err := s.loadCatalogSnapshotAtLeastVersion(ctx, nextVersion) + if err != nil { + return err + } + return s.applyEngineSnapshot(loaded) +} + +func abandonedSplitJobFencedRoute(routes []distribution.RouteDescriptor, job distribution.SplitJob) *distribution.RouteDescriptor { + for i := range routes { + route := &routes[i] + if route.ParentRouteID == job.SourceRouteID && bytes.Equal(route.Start, job.SplitKey) && route.State == distribution.RouteStateWriteFenced { + return route + } + } + return nil +} + +func (s *DistributionServer) commitAbandonedSplitJobFenceRollback( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + fenced distribution.RouteDescriptor, + encoded []byte, + nextVersion uint64, +) error { + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{ + {Op: kv.Put, Key: distribution.CatalogRouteKey(fenced.RouteID), Value: encoded}, + {Op: kv.Put, Key: distribution.CatalogVersionKey(), Value: distribution.EncodeCatalogVersion(nextVersion)}, + }, + IsTxn: true, + StartTS: snapshot.ReadTS, + ReadKeys: [][]byte{ + distribution.CatalogRouteKey(fenced.RouteID), + distribution.CatalogVersionKey(), + }, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + return nil +} + +func (s *DistributionServer) finishSplitJobHistory(ctx context.Context, job distribution.SplitJob, terminalPhase distribution.SplitJobPhase) error { + current, readTS, err := s.catalog.LiveSplitJobForUpdate(ctx, job.JobID) + if err != nil { + return splitJobCatalogStatusError(err) + } + if current.Phase != job.Phase { + return nil + } + nowMs := time.Now().UnixMilli() + next := distribution.CloneSplitJob(current) + next.Phase = terminalPhase + next.RetryPhase = distribution.SplitJobPhaseNone + next.AbandonFromPhase = distribution.SplitJobPhaseNone + next.TerminalAtMs = nowMs + next.UpdatedAtMs = nowMs + encoded, err := distribution.EncodeSplitJob(next) + if err != nil { + return splitJobCatalogStatusError(err) + } + liveKey := distribution.CatalogSplitJobKey(job.JobID) + historyKey := distribution.CatalogSplitJobHistoryKey(nowMs, job.JobID) + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{ + {Op: kv.Put, Key: historyKey, Value: encoded}, + {Op: kv.Del, Key: liveKey}, + }, + IsTxn: true, + StartTS: readTS, + ReadKeys: [][]byte{liveKey, historyKey}, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + return nil +} + +func (s *DistributionServer) gcSplitJobHistory(ctx context.Context, jobs []distribution.SplitJob, now time.Time) error { + s.mu.Lock() + if !s.splitJobHistoryGCLast.IsZero() && now.Sub(s.splitJobHistoryGCLast) < splitJobHistoryGCInterval { + s.mu.Unlock() + return nil + } + s.mu.Unlock() + + keys := splitJobHistoryGCKeys(jobs, now) + if len(keys) > 0 { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return err + } + elems := make([]*kv.Elem[kv.OP], 0, len(keys)) + for _, key := range keys { + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: key}) + } + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: elems, + IsTxn: true, + StartTS: snapshot.ReadTS, + ReadKeys: keys, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + } + s.mu.Lock() + s.splitJobHistoryGCLast = now + s.mu.Unlock() + return nil +} + +func splitJobHistoryGCKeys(jobs []distribution.SplitJob, now time.Time) [][]byte { + history := make([]distribution.SplitJob, 0, len(jobs)) + for _, job := range jobs { + if (job.Phase == distribution.SplitJobPhaseDone || job.Phase == distribution.SplitJobPhaseAbandoned) && job.TerminalAtMs > 0 { + history = append(history, job) + } + } + sort.Slice(history, func(i, j int) bool { + if history[i].TerminalAtMs == history[j].TerminalAtMs { + return history[i].JobID < history[j].JobID + } + return history[i].TerminalAtMs < history[j].TerminalAtMs + }) + cutoff := now.Add(-splitJobHistoryTTL).UnixMilli() + excess := len(history) - splitJobHistoryLimit + keys := make([][]byte, 0, min(len(history), splitJobHistoryDeleteBatchLimit)) + for i, job := range history { + if job.TerminalAtMs >= cutoff && i >= excess { + continue + } + keys = append(keys, distribution.CatalogSplitJobHistoryKey(job.TerminalAtMs, job.JobID)) + if len(keys) == splitJobHistoryDeleteBatchLimit { + break + } + } + return keys +} + +// splitJobSourceRouteState resolves the group that still holds the source side +// of the split, plus the end of the moved range. The live route shape is +// preferred so a job written before source_group_id existed still resolves; +// the job's own recorded source group is the last resort, for the case where +// neither shape survives: once the source child is split again -- which +// SplitRange permits, the new split being disjoint from the moving range -- +// its grandchildren no longer name the original parent, and that parent is +// itself already gone. Without the recorded group, cleanup would fail with +// ErrMigrationSourceRouteChanged on every attempt after cutover and keep the +// job live with its guards and retention pin held. +func splitJobSourceRouteState( + routes []distribution.RouteDescriptor, + job distribution.SplitJob, +) (uint64, []byte, bool) { + sourceGroupID, routeEnd, ok := splitJobSourceSibling(routes, job) + if ok { + return sourceGroupID, routeEnd, true + } + if parent, found := findRouteByID(routes, job.SourceRouteID); found { + return parent.GroupID, distribution.CloneBytes(parent.End), true + } + if job.SourceGroupID != 0 { + return job.SourceGroupID, routeEnd, true + } + return 0, nil, false +} + +func splitJobSourceSibling(routes []distribution.RouteDescriptor, job distribution.SplitJob) (uint64, []byte, bool) { + var sourceGroupID uint64 + var routeEnd []byte + for _, route := range routes { + if route.ParentRouteID != job.SourceRouteID { + continue + } + if bytes.Equal(route.Start, job.SplitKey) { + routeEnd = distribution.CloneBytes(route.End) + continue + } + if bytes.Equal(route.End, job.SplitKey) { + sourceGroupID = route.GroupID + } + } + return sourceGroupID, routeEnd, sourceGroupID != 0 +} + +func (s *DistributionServer) beginSplitJobBackfill(ctx context.Context, job distribution.SplitJob) error { + job, err := s.prepareSplitJobWorkingSource(ctx, job) + if err != nil { + return err + } + parent, source, err := s.loadSplitBackfillSource(ctx, job) + if err != nil { + return err + } + ackCursor, complete, err := s.armSplitJobSourceTracker(ctx, job, parent, source) + if err != nil { + return err + } + if !complete { + return s.persistSplitJobBackfillAck(ctx, job.JobID, ackCursor) + } + witness, err := s.ensureSplitJobTransferPhase(ctx, job, source, ackCursor) + if err != nil { + return err + } + // PREPARE entries predating the tracker may still carry large values. + // Let them commit without imposing a retroactive limit, then choose the + // snapshot so backfill detects every such row before any source fence. + pending, err := splitJobPendingLocks(ctx, source, job.SplitKey, parent.End) + if err != nil || pending { + return err + } + return s.openSplitJobBackfill(ctx, job, source, witness) +} + +func (s *DistributionServer) loadSplitBackfillSource(ctx context.Context, job distribution.SplitJob) (distribution.RouteDescriptor, SplitMigrationClient, error) { + _, parent, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return parent, nil, err + } + source, err := s.splitJobSourceClient(ctx, job, parent.GroupID) + if err != nil { + return parent, nil, errors.WithStack(err) + } + if source == nil { + return parent, nil, errors.New("split migration source client is nil") + } + return parent, source, s.verifySplitOwnershipRuntimes(ctx, job, parent.GroupID, source) +} + +const initialMigrationRetentionPinTS = uint64(1) + +func (s *DistributionServer) armSplitJobSourceTracker( + ctx context.Context, + job distribution.SplitJob, + parent distribution.RouteDescriptor, + source SplitMigrationClient, +) ([]byte, bool, error) { + return s.armSplitSourceOwnership(ctx, job, parent.GroupID, source) +} + +func (s *DistributionServer) persistSplitJobBackfillAck(ctx context.Context, jobID uint64, ackCursor []byte) error { + return s.updateSplitJobViaCoordinator(ctx, jobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhasePlanned { + current.FenceAckCursor = ackCursor + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) openSplitJobBackfill(ctx context.Context, job distribution.SplitJob, source SplitMigrationClient, witness *splitTransferAdmissionWitness) error { + if err := s.revalidateSplitTransferAdmission(ctx, witness); err != nil { + return err + } + issued, err := issueSplitMigrationTimestamp(ctx, source) + if err != nil { + return errors.WithStack(err) + } + snapshotTS := issued.GetTimestamp() + if snapshotTS == 0 || snapshotTS <= issued.GetLastCommitTs() { + return errors.New("split migration source issued an invalid snapshot timestamp") + } + if err := s.revalidateSplitTransferAdmission(ctx, witness); err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != distribution.SplitJobPhasePlanned { + return current, nil + } + current.Phase = distribution.SplitJobPhaseBackfill + current.SnapshotTS = snapshotTS + current.WriteTrackerArmed = true + current.SourceRetentionPinTS = initialMigrationRetentionPinTS + current.FenceAckCursor = nil + current.UpdatedAtMs = time.Now().UnixMilli() + return current, nil + }) +} + +func (s *DistributionServer) copySplitJobPhase( + ctx context.Context, + job distribution.SplitJob, + exportPhase distribution.SplitJobExportPhase, + minCommitTS uint64, + maxCommitTS uint64, +) error { + if maxCommitTS == 0 { + return errors.New("split migration copy upper timestamp is zero") + } + sourceRoute, witness, err := s.splitCopySourceAdmission(ctx, job) + if err != nil { + return err + } + brackets, err := distribution.PlanExportBrackets(job.SplitKey, sourceRoute.End) + if err != nil { + return errors.WithStack(err) + } + progressIndex, bracket, ok := nextSplitJobBracket(job, brackets, exportPhase) + if !ok { + if err := s.revalidateSplitTransferAdmission(ctx, witness); err != nil { + return err + } + return s.advanceCompletedCopyPhase(ctx, job, exportPhase) + } + source, target, err := s.splitCopyClients(ctx, job, sourceRoute.GroupID) + if err != nil { + return err + } + if err := s.revalidateSplitTransferAdmission(ctx, witness); err != nil { + return err + } + progress := job.BracketProgress[progressIndex] + attemptCtx, cancel := context.WithTimeout(ctx, defaultSplitMigrationAttemptTimeout) + defer cancel() + stream, err := source.ExportRangeVersions(attemptCtx, &pb.ExportRangeVersionsRequest{ + RangeStart: bracket.Start, + RangeEnd: bracket.End, + MaxCommitTs: maxCommitTS, + MinCommitTs: minCommitTS, + Cursor: progress.Cursor, + ChunkBytes: defaultSplitMigrationChunkBytes, + RouteStart: job.SplitKey, + RouteEnd: sourceRoute.End, + MaxScannedBytes: defaultSplitMigrationMaxScannedBytes, + KeyFamily: bracket.Family, + ExcludeKnownInternal: bracket.ExcludeKnownInternal, + ExcludePrefixes: bracket.ExcludePrefixes, + }) + if err != nil { + return errors.WithStack(migrationSourceExportError(err)) + } + return s.copySplitJobStream(ctx, job, progressIndex, bracket, progress, stream, target) +} + +func (s *DistributionServer) splitCopySourceAdmission(ctx context.Context, job distribution.SplitJob) (distribution.RouteDescriptor, *splitTransferAdmissionWitness, error) { + _, route, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return route, nil, err + } + witness, err := s.ensureSplitJobTransferPhase(ctx, job, nil, nil) + return route, witness, err +} + +func (s *DistributionServer) splitCopyClients(ctx context.Context, job distribution.SplitJob, groupID uint64) (SplitMigrationClient, SplitMigrationClient, error) { + source, target, err := s.splitMigrationClientFactory(ctx, job, groupID) + if err != nil { + return nil, nil, errors.WithStack(err) + } + if source == nil || target == nil { + return nil, nil, errors.New("split migration source or target client is nil") + } + return source, target, nil +} + +func (s *DistributionServer) copySplitJobStream( + ctx context.Context, + job distribution.SplitJob, + progressIndex int, + bracket distribution.MigrationBracket, + progress distribution.SplitJobBracketProgress, + stream grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], + target SplitMigrationClient, +) error { + for { + resp, recvErr := stream.Recv() + if errors.Is(recvErr, io.EOF) { + return nil + } + if recvErr != nil { + return errors.WithStack(migrationSourceExportError(recvErr)) + } + nextCursor := distribution.CloneBytes(resp.GetNextCursor()) + batchSeq := progress.LastAckedBatchSeq + 1 + ackedCursor, duplicate, err := importSplitJobChunk(ctx, target, job, bracket, batchSeq, resp.GetVersions(), nextCursor) + if err != nil { + return err + } + if duplicate { + // The target already holds this batch durably from an attempt whose + // progress this runner never persisted. Its acknowledgement is the + // authority on where the copy stands: the source keeps taking writes + // while the copy runs, so a replayed chunk can end on a different + // boundary, and demanding the boundary match would fail the job on + // every retry. Adopt the acknowledged cursor and re-export from it -- + // the rest of this stream is cut relative to a boundary the target + // never accepted. + return s.adoptDuplicateImportAck(ctx, job, progressIndex, progress, batchSeq, ackedCursor) + } + progress.Cursor = nextCursor + progress.Done = resp.GetDone() + progress.AcceptedRows += uint64(len(resp.GetVersions())) + progress.LastAckedBatchSeq = batchSeq + for _, version := range resp.GetVersions() { + if version.GetCommitTs() > job.MaxImportedTS { + job.MaxImportedTS = version.GetCommitTs() + } + } + job.BracketProgress[progressIndex] = progress + job.Cursor = nextCursor + job.UpdatedAtMs = time.Now().UnixMilli() + if err := s.persistSplitJobCopyProgress(ctx, job); err != nil { + return err + } + if progress.Done { + return nil + } + } +} + +func cleanupSplitMigration(ctx context.Context, client SplitMigrationClient, req *pb.CleanupMigrationRequest) (*pb.CleanupMigrationResponse, error) { + attemptCtx, cancel := context.WithTimeout(ctx, defaultSplitMigrationAttemptTimeout) + defer cancel() + resp, err := client.CleanupMigration(attemptCtx, req) + return resp, errors.WithStack(err) +} + +// importSplitJobChunk hands one exported chunk to the target and returns the +// cursor the target durably holds. duplicate marks an acknowledgement the +// target had already recorded for this batch sequence, in which case the +// acknowledged cursor is its own and need not match the one just sent. +func importSplitJobChunk( + ctx context.Context, + target SplitMigrationClient, + job distribution.SplitJob, + bracket distribution.MigrationBracket, + batchSeq uint64, + versions []*pb.MVCCVersion, + nextCursor []byte, +) ([]byte, bool, error) { + attemptCtx, cancel := context.WithTimeout(ctx, defaultSplitMigrationAttemptTimeout) + defer cancel() + resp, err := target.ImportRangeVersions(attemptCtx, &pb.ImportRangeVersionsRequest{ + JobId: job.JobID, + Versions: versions, + Cursor: nextCursor, + BracketId: bracket.BracketID, + BatchSeq: batchSeq, + }) + if err != nil { + return nil, false, errors.WithStack(err) + } + if resp.GetDuplicate() { + return resp.GetAckedCursor(), true, nil + } + if !bytes.Equal(resp.GetAckedCursor(), nextCursor) { + return nil, false, errors.New("split migration import acknowledged a different cursor") + } + return resp.GetAckedCursor(), false, nil +} + +// adoptDuplicateImportAck records the target's durable acknowledgement for a +// batch the runner re-sent after losing its own progress. Only the cursor and +// the batch sequence advance: the rows were applied by the earlier attempt, and +// this runner never observed which of them the target actually accepted, so +// AcceptedRows and MaxImportedTS stay where they are rather than counting rows +// twice. Done is likewise left alone -- the replayed export's end-of-range +// answer describes a chunk the target did not take. +func (s *DistributionServer) adoptDuplicateImportAck( + ctx context.Context, + job distribution.SplitJob, + progressIndex int, + progress distribution.SplitJobBracketProgress, + batchSeq uint64, + ackedCursor []byte, +) error { + adopted := distribution.CloneBytes(ackedCursor) + progress.Cursor = adopted + progress.LastAckedBatchSeq = batchSeq + job.BracketProgress[progressIndex] = progress + job.Cursor = adopted + job.UpdatedAtMs = time.Now().UnixMilli() + return s.persistSplitJobCopyProgress(ctx, job) +} + +func nextSplitJobBracket( + job distribution.SplitJob, + brackets []distribution.MigrationBracket, + exportPhase distribution.SplitJobExportPhase, +) (int, distribution.MigrationBracket, bool) { + byID := make(map[uint64]distribution.MigrationBracket, len(brackets)) + for _, bracket := range brackets { + byID[bracket.BracketID] = bracket + } + for i, progress := range job.BracketProgress { + if progress.ExportPhase != exportPhase || progress.Done { + continue + } + bracket, found := byID[progress.BracketID] + if found && bracket.Family == progress.Family { + return i, bracket, true + } + } + return 0, distribution.MigrationBracket{}, false +} + +func (s *DistributionServer) persistSplitJobCopyProgress(ctx context.Context, next distribution.SplitJob) error { + return s.updateSplitJobViaCoordinator(ctx, next.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != next.Phase { + return current, nil + } + return distribution.CloneSplitJob(next), nil + }) +} + +func (s *DistributionServer) advanceCompletedCopyPhase(ctx context.Context, job distribution.SplitJob, phase distribution.SplitJobExportPhase) error { + switch phase { + case distribution.SplitJobExportPhaseBackfill: + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseBackfill { + current.Phase = distribution.SplitJobPhaseFence + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + case distribution.SplitJobExportPhaseDeltaCopy: + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseDeltaCopy { + current.Phase = distribution.SplitJobPhaseCutover + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + case distribution.SplitJobExportPhaseNone: + return errors.New("unknown split migration export phase") + } + return errors.New("unknown split migration export phase") +} + +func (s *DistributionServer) finalizeSplitJobFence(ctx context.Context, job distribution.SplitJob) error { + snapshot, sourceRoute, ready, err := s.prepareSplitJobCatalogFence(ctx, job) + if err != nil || !ready { + return err + } + source, minAdmittedTS, ackCursor, complete, err := s.drainedSplitJobFenceBarrier(ctx, job, snapshot, sourceRoute) + if err != nil { + return err + } + if !complete { + return s.persistSplitJobFenceAck(ctx, job.JobID, ackCursor) + } + witness, err := s.ensureSplitJobTransferPhase(ctx, job, source, nil) + if err != nil { + return err + } + issued, err := issueSplitMigrationTimestamp(ctx, source) + if err != nil { + return errors.WithStack(err) + } + if issued.GetTimestamp() == 0 || issued.GetTimestamp() <= issued.GetLastCommitTs() { + return errors.New("split migration source issued an invalid fence timestamp") + } + if err := s.revalidateSplitTransferAdmission(ctx, witness); err != nil { + return err + } + return s.commitSplitJobFenceState(ctx, job, snapshot.Version, minAdmittedTS, ackCursor, issued.GetTimestamp()) +} + +func (s *DistributionServer) prepareSplitJobCatalogFence(ctx context.Context, job distribution.SplitJob) (distribution.CatalogSnapshot, distribution.RouteDescriptor, bool, error) { + ready, err := s.preflightSplitJobTransfer(ctx, job) + if err != nil || !ready { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, false, err + } + snapshot, route, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return snapshot, route, false, err + } + if _, err := s.ensureSplitJobTransferPhase(ctx, job, nil, nil); err != nil { + return snapshot, route, false, err + } + snapshot, route, err = s.ensureSplitJobCatalogFence(ctx, job, snapshot, route) + return snapshot, route, err == nil, err +} + +func (s *DistributionServer) drainedSplitJobFenceBarrier( + ctx context.Context, + job distribution.SplitJob, + snapshot distribution.CatalogSnapshot, + sourceRoute distribution.RouteDescriptor, +) (SplitMigrationClient, uint64, []byte, bool, error) { + source, minimum, cursor, complete, err := s.applySplitJobFenceBarrier(ctx, job, snapshot, sourceRoute) + if err != nil || !complete { + return source, minimum, cursor, complete, err + } + pending, err := splitJobPendingLocks(ctx, source, job.SplitKey, sourceRoute.End) + if err != nil { + return nil, 0, nil, false, err + } + if pending { + return source, minimum, cursor, false, nil + } + // Reapplying the identical Raft control preserves and reads the tracker + // minimum after every pre-fence transaction has released its lock. + return s.applySplitJobFenceBarrier(ctx, job, snapshot, sourceRoute) +} + +func (s *DistributionServer) applySplitJobFenceBarrier( + ctx context.Context, + job distribution.SplitJob, + snapshot distribution.CatalogSnapshot, + sourceRoute distribution.RouteDescriptor, +) (SplitMigrationClient, uint64, []byte, bool, error) { + source, err := s.splitJobSourceClient(ctx, job, sourceRoute.GroupID) + if err != nil { + return nil, 0, nil, false, errors.WithStack(err) + } + minAdmittedTS, err := applySplitMigrationControl( + ctx, + source, + job, + sourceRoute.End, + snapshot.Version, + job.SnapshotTS, + true, + false, + true, + 1, + ) + if err != nil { + return nil, 0, nil, false, err + } + ackCursor, complete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceRoute.GroupID, job.FenceAckCursor, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: job.SplitKey, + RouteEnd: sourceRoute.End, + ExpectedCatalogVersion: snapshot.Version, + MigrationJobId: job.JobID, + MinWriteTsExclusive: job.SnapshotTS, + SourceWriteFence: true, + TrackWrites: true, + RetentionPinTs: 1, + }) + if err != nil { + return nil, 0, nil, false, err + } + return source, minAdmittedTS, ackCursor, complete, nil +} + +func (s *DistributionServer) persistSplitJobFenceAck(ctx context.Context, jobID uint64, ackCursor []byte) error { + return s.updateSplitJobViaCoordinator(ctx, jobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseFence { + current.FenceAckCursor = distribution.CloneBytes(ackCursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) commitSplitJobFenceState(ctx context.Context, job distribution.SplitJob, fenceCatalogVersion, minAdmittedTS uint64, ackCursor []byte, fenceTS uint64) error { + deltaFloor := job.SnapshotTS + if minAdmittedTS > 0 && minAdmittedTS-1 < deltaFloor { + deltaFloor = minAdmittedTS - 1 + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != distribution.SplitJobPhaseFence { + return current, nil + } + current.Phase = distribution.SplitJobPhaseDeltaCopy + current.PostFenceDrainCompleted = true + current.SnapshotMinAdmittedTS = minAdmittedTS + current.DeltaFloor = deltaFloor + current.FenceTS = fenceTS + current.FenceCatalogVersion = fenceCatalogVersion + current.FenceAckCursor = distribution.CloneBytes(ackCursor) + current.SourceRetentionPinTS = 1 + for i := range current.BracketProgress { + current.BracketProgress[i].ExportPhase = distribution.SplitJobExportPhaseDeltaCopy + current.BracketProgress[i].Cursor = nil + current.BracketProgress[i].Done = false + current.BracketProgress[i].ScannedBytes = 0 + current.BracketProgress[i].AcceptedRows = 0 + } + current.UpdatedAtMs = time.Now().UnixMilli() + return current, nil + }) +} + +func (s *DistributionServer) ensureSplitJobCatalogFence( + ctx context.Context, + job distribution.SplitJob, + snapshot distribution.CatalogSnapshot, + sourceRoute distribution.RouteDescriptor, +) (distribution.CatalogSnapshot, distribution.RouteDescriptor, error) { + if job.WorkingSourceRouteID != 0 { + return s.fenceSplitWorkingSource(ctx, job, snapshot, sourceRoute) + } + if sourceRoute.RouteID != job.SourceRouteID { + return snapshot, sourceRoute, nil + } + current, found, err := s.catalog.SplitJobAt(ctx, job.JobID, snapshot.ReadTS) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, errors.WithStack(err) + } + if !found || current.Phase != distribution.SplitJobPhaseFence { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, errors.WithStack(store.ErrWriteConflict) + } + leftID, rightID, err := s.allocateChildRouteIDs(ctx, snapshot.ReadTS, snapshot.Routes) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, err + } + left, right := splitCatalogRoutes(sourceRoute, job.SplitKey, leftID, rightID, job.FenceTS) + right.State = distribution.RouteStateWriteFenced + // The catalog write goes through the coordinator, so it needs a real read + // timestamp token rather than the bare snapshot.ReadTS: the dispatch below + // consumes its voucher. Pinned at the snapshot the fence was planned from. + readTimestamp, err := kv.BeginReadTimestampThrough( + ctx, + s.coordinator, + snapshot.ReadTS, + "split job catalog fence: begin read timestamp", + ) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, grpcStatusErrorf( + codes.Internal, "begin split job fence snapshot: %v", err) + } + snapshot, err = s.saveSplitResultViaCoordinator(ctx, readTimestamp, snapshot.Version, sourceRoute.RouteID, [][]byte{distribution.CatalogSplitJobKey(job.JobID)}, left, right) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, err + } + if err := s.applyEngineSnapshot(snapshot); err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, err + } + return snapshot, right, nil +} + +func splitJobPendingLocks(ctx context.Context, source SplitMigrationClient, routeStart []byte, routeEnd []byte) (bool, error) { + attemptCtx, cancel := context.WithTimeout(ctx, defaultSplitMigrationAttemptTimeout) + defer cancel() + resp, err := source.ProbeMigrationLocks(attemptCtx, &pb.ProbeMigrationLocksRequest{ + RouteStart: routeStart, + RouteEnd: routeEnd, + Limit: defaultSplitMigrationLockProbeLimit, + }) + if err != nil { + return false, errors.WithStack(err) + } + return resp.GetPendingCount() != 0, nil +} + +func (s *DistributionServer) cutoverSplitJob(ctx context.Context, job distribution.SplitJob) error { + snapshot, sourceRoute, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return err + } + if snapshot.Version == math.MaxUint64 { + return errors.New("split migration catalog version overflow") + } + expectedVersion := snapshot.Version + 1 + if job.CutoverVersion == 0 || job.CutoverVersion != expectedVersion { + return s.armSplitJobCutoverWitness(ctx, job, expectedVersion) + } + targetCursor, sourceCursor, targetComplete, sourceComplete, err := s.applySplitJobCutoverBarriers(ctx, job, sourceRoute, expectedVersion) + if err != nil { + return err + } + if splitJobCutoverBarrierPending(job, targetComplete, sourceComplete) { + return s.persistSplitJobCutoverAcks(ctx, job.JobID, targetCursor, sourceCursor, targetComplete, sourceComplete) + } + return s.commitSplitJobCutover(ctx, snapshot, sourceRoute, job, expectedVersion) +} + +func (s *DistributionServer) applySplitJobCutoverBarriers( + ctx context.Context, + job distribution.SplitJob, + sourceRoute distribution.RouteDescriptor, + expectedVersion uint64, +) ([]byte, []byte, bool, bool, error) { + source, target, err := s.splitMigrationClientFactory(ctx, job, sourceRoute.GroupID) + if err != nil { + return nil, nil, false, false, errors.WithStack(err) + } + if _, err := applySplitMigrationControl(ctx, target, job, sourceRoute.End, expectedVersion, job.FenceTS, false, false, false, 0); err != nil { + return nil, nil, false, false, err + } + if _, err := applySplitMigrationControl(ctx, source, job, sourceRoute.End, expectedVersion, job.FenceTS, true, true, job.WriteTrackerArmed, 1); err != nil { + return nil, nil, false, false, err + } + targetCursor, targetComplete, err := s.syncSplitMigrationVoterBarrier(ctx, job.TargetGroupID, job.TargetStagedReadinessAckCursor, target, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: job.SplitKey, + RouteEnd: sourceRoute.End, + ExpectedCatalogVersion: expectedVersion, + MigrationJobId: job.JobID, + MinWriteTsExclusive: job.FenceTS, + }) + if err != nil { + return nil, nil, false, false, err + } + sourceCursor, sourceComplete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceRoute.GroupID, job.SourceCutoverReadFenceAckCursor, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: job.SplitKey, + RouteEnd: sourceRoute.End, + ExpectedCatalogVersion: expectedVersion, + MigrationJobId: job.JobID, + MinWriteTsExclusive: job.FenceTS, + SourceWriteFence: true, + SourceReadFence: true, + TrackWrites: job.WriteTrackerArmed, + RetentionPinTs: 1, + }) + if err != nil { + return nil, nil, false, false, err + } + return targetCursor, sourceCursor, targetComplete, sourceComplete, nil +} + +func splitJobCutoverBarrierPending(job distribution.SplitJob, targetComplete, sourceComplete bool) bool { + return !targetComplete || !sourceComplete || + job.TargetStagedReadinessState != distribution.SplitJobBarrierArmed || + job.CutoverReadFenceState != distribution.SplitJobBarrierArmed +} + +func (s *DistributionServer) armSplitJobCutoverWitness(ctx context.Context, job distribution.SplitJob, expectedVersion uint64) error { + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != distribution.SplitJobPhaseCutover { + return current, nil + } + current.CutoverVersion = expectedVersion + current.CutoverReadFenceState = distribution.SplitJobBarrierArming + current.TargetStagedReadinessState = distribution.SplitJobBarrierArming + current.SourceCutoverReadFenceAckCursor = nil + current.TargetStagedReadinessAckCursor = nil + current.UpdatedAtMs = time.Now().UnixMilli() + return current, nil + }) +} + +func (s *DistributionServer) persistSplitJobCutoverAcks( + ctx context.Context, + jobID uint64, + targetCursor []byte, + sourceCursor []byte, + targetComplete bool, + sourceComplete bool, +) error { + return s.updateSplitJobViaCoordinator(ctx, jobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != distribution.SplitJobPhaseCutover { + return current, nil + } + current.TargetStagedReadinessAckCursor = distribution.CloneBytes(targetCursor) + current.SourceCutoverReadFenceAckCursor = distribution.CloneBytes(sourceCursor) + current.TargetStagedReadinessState = distribution.SplitJobBarrierArming + if targetComplete { + current.TargetStagedReadinessState = distribution.SplitJobBarrierArmed + } + current.CutoverReadFenceState = distribution.SplitJobBarrierArming + if sourceComplete { + current.CutoverReadFenceState = distribution.SplitJobBarrierArmed + } + current.UpdatedAtMs = time.Now().UnixMilli() + return current, nil + }) +} + +func applySplitMigrationControl( + ctx context.Context, + client SplitMigrationClient, + job distribution.SplitJob, + routeEnd []byte, + expectedVersion uint64, + minWriteTS uint64, + sourceWriteFence bool, + sourceReadFence bool, + trackWrites bool, + retentionPinTS uint64, +) (uint64, error) { + if client == nil { + return 0, errors.New("split migration control client is nil") + } + resp, err := applySplitMigrationControlRequest(ctx, client, splitMigrationControlRequest( + job, + routeEnd, + expectedVersion, + minWriteTS, + sourceWriteFence, + sourceReadFence, + trackWrites, + retentionPinTS, + )) + if err != nil { + return 0, errors.WithStack(err) + } + return resp.GetMinAdmittedTs(), nil +} + +func applySplitMigrationControlRequest(ctx context.Context, client SplitMigrationClient, req *pb.TargetStagedReadinessRequest) (*pb.TargetStagedReadinessResponse, error) { + attemptCtx, cancel := context.WithTimeout(ctx, defaultSplitMigrationAttemptTimeout) + defer cancel() + resp, err := client.ApplyTargetStagedReadiness(attemptCtx, req) + return resp, errors.WithStack(err) +} + +func issueSplitMigrationTimestamp(ctx context.Context, client SplitMigrationClient) (*pb.IssueMigrationTimestampResponse, error) { + attemptCtx, cancel := context.WithTimeout(ctx, defaultSplitMigrationAttemptTimeout) + defer cancel() + resp, err := client.IssueMigrationTimestamp(attemptCtx, &pb.IssueMigrationTimestampRequest{}) + return resp, errors.WithStack(err) +} + +func splitMigrationControlRequest( + job distribution.SplitJob, + routeEnd []byte, + expectedVersion uint64, + minWriteTS uint64, + sourceWriteFence bool, + sourceReadFence bool, + trackWrites bool, + retentionPinTS uint64, +) *pb.TargetStagedReadinessRequest { + return &pb.TargetStagedReadinessRequest{ + JobId: job.JobID, + RouteStart: job.SplitKey, + RouteEnd: routeEnd, + ExpectedCutoverVersion: expectedVersion, + MigrationJobId: job.JobID, + MinWriteTsExclusive: minWriteTS, + Armed: true, + SourceWriteFence: sourceWriteFence, + SourceReadFence: sourceReadFence, + RetentionPinTs: retentionPinTS, + TrackWrites: trackWrites, + } +} + +func probeMigrationControlRequest(req *pb.TargetStagedReadinessRequest) *pb.ProbeMigrationStateRequest { + return &pb.ProbeMigrationStateRequest{ + JobId: req.GetJobId(), + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: distribution.CloneBytes(req.GetRouteStart()), + RouteEnd: distribution.CloneBytes(req.GetRouteEnd()), + ExpectedCatalogVersion: req.GetExpectedCutoverVersion(), + MigrationJobId: req.GetMigrationJobId(), + MinWriteTsExclusive: req.GetMinWriteTsExclusive(), + SourceWriteFence: req.GetSourceWriteFence(), + SourceReadFence: req.GetSourceReadFence(), + TrackWrites: req.GetTrackWrites(), + RetentionPinTs: req.GetRetentionPinTs(), + } +} + +func splitJobCapabilityRegressionRequiresFailClosed(job distribution.SplitJob) bool { + return splitJobCapabilityRegressionSourceGuardRequired(job) || + splitJobCapabilityRegressionTargetGuardRequired(job) +} + +func splitJobCapabilityRegressionSourceGuardRequired(job distribution.SplitJob) bool { + switch job.Phase { + case distribution.SplitJobPhasePlanned: + return job.WriteTrackerArmed || + job.SnapshotTS != 0 || + job.SourceRetentionPinTS != 0 || + len(job.FenceAckCursor) != 0 + case distribution.SplitJobPhaseBackfill, + distribution.SplitJobPhaseFence, + distribution.SplitJobPhaseDeltaCopy, + distribution.SplitJobPhaseCutover: + return true + case distribution.SplitJobPhaseCleanup: + return job.SourceRetentionPinTS != math.MaxUint64 + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoning, + distribution.SplitJobPhaseAbandoned: + return false + } + return false +} + +func splitJobCapabilityRegressionTargetGuardRequired(job distribution.SplitJob) bool { + switch job.Phase { + case distribution.SplitJobPhasePlanned: + return job.WriteTrackerArmed || + job.SnapshotTS != 0 || + job.SourceRetentionPinTS != 0 || + len(job.FenceAckCursor) != 0 + case distribution.SplitJobPhaseBackfill, + distribution.SplitJobPhaseFence, + distribution.SplitJobPhaseDeltaCopy, + distribution.SplitJobPhaseCutover: + return true + case distribution.SplitJobPhaseCleanup: + return !bytes.Equal(job.TargetClearedDescriptorAckCursor, splitCleanupDoneCursor) + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoning, + distribution.SplitJobPhaseAbandoned: + return false + } + return false +} + +type splitJobMigrationControlTargets struct { + snapshot distribution.CatalogSnapshot + sourceGroupID uint64 + routeEnd []byte + source SplitMigrationClient + target SplitMigrationClient +} + +func (s *DistributionServer) splitJobCapabilityControlTargets(ctx context.Context, job distribution.SplitJob) (splitJobMigrationControlTargets, error) { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return splitJobMigrationControlTargets{}, err + } + sourceGroupID, routeEnd, ok := splitJobSourceRouteState(snapshot.Routes, job) + if !ok { + return splitJobMigrationControlTargets{}, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + source, target, err := s.splitMigrationClientFactory(ctx, job, sourceGroupID) + if err != nil { + return splitJobMigrationControlTargets{}, errors.WithStack(err) + } + if source == nil || target == nil { + return splitJobMigrationControlTargets{}, errors.New("split migration source or target client is nil") + } + return splitJobMigrationControlTargets{ + snapshot: snapshot, + sourceGroupID: sourceGroupID, + routeEnd: routeEnd, + source: source, + target: target, + }, nil +} + +func (s *DistributionServer) ensureSplitJobCapabilityRegressionFailClosed(ctx context.Context, job distribution.SplitJob) (bool, error) { + targets, err := s.splitJobCapabilityControlTargets(ctx, job) + if err != nil { + return false, err + } + expectedVersion := splitJobCapabilityRegressionExpectedVersion(job, targets.snapshot.Version) + minWriteTS := splitJobCapabilityRegressionMinWriteTS(job) + retentionPinTS := splitJobCapabilityRegressionRetentionPinTS(job) + sourceComplete := true + if splitJobCapabilityRegressionSourceGuardRequired(job) { + sourceReq := splitMigrationControlRequest(job, targets.routeEnd, expectedVersion, minWriteTS, true, true, true, retentionPinTS) + var err error + sourceComplete, err = s.applyAndProbeSplitMigrationControl(ctx, targets.sourceGroupID, targets.source, sourceReq) + if err != nil { + return false, err + } + } + targetComplete := true + if splitJobCapabilityRegressionTargetGuardRequired(job) { + targetReq := splitMigrationControlRequest(job, targets.routeEnd, expectedVersion, minWriteTS, false, false, false, 0) + var err error + targetComplete, err = s.applyAndProbeSplitMigrationControl(ctx, job.TargetGroupID, targets.target, targetReq) + if err != nil { + return false, err + } + } + return sourceComplete && targetComplete, nil +} + +func (s *DistributionServer) restoreSplitJobCapabilityRegressionGuards(ctx context.Context, job distribution.SplitJob) (bool, error) { + targets, err := s.splitJobCapabilityControlTargets(ctx, job) + if err != nil { + return false, err + } + if splitJobCapabilityRegressionSourceGuardRequired(job) { + sourceReq, ok := splitJobNormalSourceControlRequest(job, targets.routeEnd, targets.snapshot.Version) + if ok { + complete, err := s.applyAndProbeSplitMigrationControl(ctx, targets.sourceGroupID, targets.source, sourceReq) + if err != nil || !complete { + return complete, err + } + } + } + if splitJobCapabilityRegressionTargetGuardRequired(job) { + targetReq, ok := splitJobNormalTargetControlRequest(job, targets.routeEnd) + if ok { + complete, err := s.applyAndProbeSplitMigrationControl(ctx, job.TargetGroupID, targets.target, targetReq) + if err != nil || !complete { + return complete, err + } + } + } + return true, nil +} + +func (s *DistributionServer) applyAndProbeSplitMigrationControl( + ctx context.Context, + groupID uint64, + client SplitMigrationClient, + req *pb.TargetStagedReadinessRequest, +) (bool, error) { + if client == nil { + return false, errors.New("split migration control client is nil") + } + if _, err := applySplitMigrationControlRequest(ctx, client, req); err != nil { + return false, errors.WithStack(err) + } + _, complete, err := s.syncSplitMigrationVoterBarrier(ctx, groupID, nil, client, probeMigrationControlRequest(req)) + return complete, err +} + +func splitJobCapabilityRegressionExpectedVersion(job distribution.SplitJob, snapshotVersion uint64) uint64 { + switch { + case job.CutoverVersion != 0: + return job.CutoverVersion + case job.FenceCatalogVersion != 0: + return job.FenceCatalogVersion + case snapshotVersion < math.MaxUint64: + return snapshotVersion + 1 + default: + return snapshotVersion + } +} + +func splitJobCapabilityRegressionMinWriteTS(job distribution.SplitJob) uint64 { + switch { + case job.FenceTS != 0: + return job.FenceTS + case job.SnapshotTS != 0: + return job.SnapshotTS + default: + return initialMigrationRetentionPinTS + } +} + +func splitJobCapabilityRegressionRetentionPinTS(job distribution.SplitJob) uint64 { + if job.SourceRetentionPinTS != 0 && job.SourceRetentionPinTS != math.MaxUint64 { + return job.SourceRetentionPinTS + } + return initialMigrationRetentionPinTS +} + +func splitJobNormalSourceControlRequest(job distribution.SplitJob, routeEnd []byte, snapshotVersion uint64) (*pb.TargetStagedReadinessRequest, bool) { + switch job.Phase { + case distribution.SplitJobPhasePlanned, distribution.SplitJobPhaseBackfill: + return splitMigrationControlRequest( + job, + routeEnd, + splitJobCapabilityRegressionExpectedVersion(job, snapshotVersion), + initialMigrationRetentionPinTS, + false, + false, + true, + initialMigrationRetentionPinTS, + ), true + case distribution.SplitJobPhaseFence, distribution.SplitJobPhaseDeltaCopy: + expectedVersion := job.FenceCatalogVersion + if expectedVersion == 0 { + expectedVersion = splitJobCapabilityRegressionExpectedVersion(job, snapshotVersion) + } + return splitMigrationControlRequest(job, routeEnd, expectedVersion, job.SnapshotTS, true, false, true, initialMigrationRetentionPinTS), true + case distribution.SplitJobPhaseCutover, distribution.SplitJobPhaseCleanup: + if job.FenceTS == 0 || job.CutoverVersion == 0 { + return nil, false + } + return splitMigrationControlRequest(job, routeEnd, job.CutoverVersion, job.FenceTS, true, true, false, initialMigrationRetentionPinTS), true + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoning, + distribution.SplitJobPhaseAbandoned: + return nil, false + } + return nil, false +} + +func splitJobNormalTargetControlRequest(job distribution.SplitJob, routeEnd []byte) (*pb.TargetStagedReadinessRequest, bool) { + if job.CutoverVersion == 0 || job.FenceTS == 0 { + return nil, false + } + switch job.Phase { + case distribution.SplitJobPhaseCutover, distribution.SplitJobPhaseCleanup: + return splitMigrationControlRequest(job, routeEnd, job.CutoverVersion, job.FenceTS, false, false, false, 0), true + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhasePlanned, + distribution.SplitJobPhaseBackfill, + distribution.SplitJobPhaseFence, + distribution.SplitJobPhaseDeltaCopy, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoning, + distribution.SplitJobPhaseAbandoned: + return nil, false + } + return nil, false +} + +func (s *DistributionServer) splitJobSourceRoute( + ctx context.Context, + job distribution.SplitJob, +) (distribution.CatalogSnapshot, distribution.RouteDescriptor, error) { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, err + } + if parent, found := findRouteByID(snapshot.Routes, sourceLookupRouteID(job)); found { + return snapshot, parent, nil + } + for _, route := range snapshot.Routes { + if route.ParentRouteID == job.SourceRouteID && bytes.Equal(route.Start, job.SplitKey) { + return snapshot, distribution.CloneRouteDescriptor(route), nil + } + } + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, splitJobCatalogStatusError(distribution.ErrMigrationSourceRouteChanged) +} + +func (s *DistributionServer) commitSplitJobCutover( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + right distribution.RouteDescriptor, + job distribution.SplitJob, + nextVersion uint64, +) error { + right.GroupID = job.TargetGroupID + right.State = distribution.RouteStateActive + right.StagedVisibilityActive = true + right.MigrationJobID = job.JobID + right.MinWriteTSExclusive = job.FenceTS + encodedRoute, err := distribution.EncodeRouteDescriptorForCatalogWrite(right, s.catalog.AllowsRouteDescriptorV2Writes()) + if err != nil { + return errors.WithStack(err) + } + nextJob := distribution.CloneSplitJob(job) + nextJob.Phase = distribution.SplitJobPhaseCleanup + nextJob.CutoverVersion = nextVersion + nextJob.CutoverReadFenceState = distribution.SplitJobBarrierArmed + nextJob.TargetStagedReadinessState = distribution.SplitJobBarrierArmed + nextJob.Cursor = nil + nextJob.UpdatedAtMs = time.Now().UnixMilli() + encodedJob, err := distribution.EncodeSplitJob(nextJob) + if err != nil { + return errors.WithStack(err) + } + versionKey := distribution.CatalogVersionKey() + routeKey := distribution.CatalogRouteKey(right.RouteID) + jobKey := distribution.CatalogSplitJobKey(job.JobID) + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{ + {Op: kv.Put, Key: routeKey, Value: encodedRoute}, + {Op: kv.Put, Key: versionKey, Value: distribution.EncodeCatalogVersion(nextVersion)}, + {Op: kv.Put, Key: jobKey, Value: encodedJob}, + }, + IsTxn: true, + StartTS: snapshot.ReadTS, + ReadKeys: [][]byte{routeKey, versionKey, jobKey}, + }); err != nil { + if errors.Is(err, store.ErrWriteConflict) { + return grpcStatusError(codes.Aborted, errDistributionCatalogConflict.Error()) + } + return errors.WithStack(err) + } + updated, err := s.loadCatalogSnapshotAtLeastVersion(ctx, nextVersion) + if err != nil { + return err + } + return s.applyEngineSnapshot(updated) +} diff --git a/adapter/split_job_runtime_fixture_test.go b/adapter/split_job_runtime_fixture_test.go new file mode 100644 index 000000000..d3ae38c16 --- /dev/null +++ b/adapter/split_job_runtime_fixture_test.go @@ -0,0 +1,143 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// Keep the transport recording used by runner assertions, but execute every +// migration mutation against a real store/FSM with an applied configuration. +type splitRuntimeClient struct { + *ownershipRuntimeClient +} + +func (c *splitRuntimeClient) ApplyTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest, opts ...grpc.CallOption) (*pb.TargetStagedReadinessResponse, error) { + event := "control" + if req.GetTransferSizeGuardVersion() != 0 { + event = "guard" + } + c.events = append(c.events, event) + return c.ownershipRuntimeClient.ApplyTargetStagedReadiness(ctx, req, opts...) +} + +func (c *splitRuntimeClient) ProbeMigrationLocks(ctx context.Context, req *pb.ProbeMigrationLocksRequest, _ ...grpc.CallOption) (*pb.ProbeMigrationLocksResponse, error) { + c.events = append(c.events, "locks") + return c.internal.ProbeMigrationLocks(ctx, req) +} + +func (c *splitRuntimeClient) ExportRangeVersions(ctx context.Context, req *pb.ExportRangeVersionsRequest, _ ...grpc.CallOption) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) { + c.exports = append(c.exports, req) + capture := &captureExportRangeVersionsStream{ctx: ctx} + if err := c.internal.ExportRangeVersions(req, capture); err != nil { + return nil, err + } + return &splitMigrationStreamStub{responses: capture.responses}, nil +} + +func (c *splitRuntimeClient) ImportRangeVersions(ctx context.Context, req *pb.ImportRangeVersionsRequest, _ ...grpc.CallOption) (*pb.ImportRangeVersionsResponse, error) { + c.imports = append(c.imports, req) + return c.internal.ImportRangeVersions(ctx, req) +} + +func (c *splitRuntimeClient) PromoteStagedVersions(ctx context.Context, req *pb.PromoteStagedVersionsRequest, _ ...grpc.CallOption) (*pb.PromoteStagedVersionsResponse, error) { + c.calls++ + c.requests = append(c.requests, req) + return c.internal.PromoteStagedVersions(ctx, req) +} + +func newSplitRuntimeClient(t *testing.T, engine *distribution.Engine, st store.MVCCStore, groupID uint64) *splitRuntimeClient { + t.Helper() + client, _ := newOwnershipRuntimeClient(t, st, groupID) + client.internal.routeEngine = engine + client.internal.migrationImportGate = func(context.Context) error { return nil } + client.internal.migrationPromoteGate = func(context.Context) error { return nil } + client.internal.transferSizeReady = func() bool { return true } + client.internal.migrationPartitionProtocolReady = func() bool { return true } + tx := kv.NewTransactionWithProposer(client.internal.migrationProposer, kv.WithMigrationRequestAdmission(st, groupID)) + t.Cleanup(tx.Close) + client.internal.transactionManager = tx + return &splitRuntimeClient{ownershipRuntimeClient: client} +} + +func splitRuntimeOptions(source, target *splitRuntimeClient, sourceGroupID uint64) []DistributionServerOption { + return []DistributionServerOption{ + WithSplitMigrationApplyMemberSnapshotFactory(func(_ context.Context, groupID uint64) (SplitMigrationApplyMemberSnapshot, error) { + client := target + if groupID == sourceGroupID { + client = source + } + reader, ok := client.internal.leader.(raftengine.AppliedConfigurationReader) + if !ok { + return SplitMigrationApplyMemberSnapshot{}, errors.New("runtime fixture lacks applied configuration reader") + } + index, supported := reader.AppliedConfigurationIndex() + if !supported { + return SplitMigrationApplyMemberSnapshot{}, errors.New("runtime fixture configuration is unsupported") + } + return SplitMigrationApplyMemberSnapshot{ConfigurationIndex: index, Members: []SplitMigrationVoter{{ID: "voter", Address: "voter:1", Client: client}}}, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, target, nil + }), + WithSplitMigrationGroupClientFactory(func(_ context.Context, groupID uint64) (SplitMigrationClient, error) { + if groupID == sourceGroupID { + return source, nil + } + return target, nil + }), + WithSplitMigrationVoterFactory(func(_ context.Context, groupID uint64) ([]SplitMigrationVoter, error) { + var client SplitMigrationClient = target + if groupID == sourceGroupID { + client = source + } + return []SplitMigrationVoter{{ID: "voter", Address: "voter:1", Client: client}}, nil + }), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { return target, nil }), + } +} + +func newSplitRuntimeRunner(t *testing.T, root store.MVCCStore, catalog *distribution.CatalogStore, job distribution.SplitJob) (*DistributionServer, *splitRuntimeClient, *splitRuntimeClient) { + t.Helper() + engine := distribution.NewEngine() + snapshot, err := catalog.Snapshot(t.Context()) + require.NoError(t, err) + require.NoError(t, engine.ApplySnapshot(snapshot)) + sourceStore, targetStore := store.NewMVCCStore(), store.NewMVCCStore() + require.NoError(t, sourceStore.PutAt(t.Context(), []byte("n"), []byte("v"), 10, 0)) + source := newSplitRuntimeClient(t, engine, sourceStore, job.SourceGroupID) + target := newSplitRuntimeClient(t, engine, targetStore, job.TargetGroupID) + if job.Phase == distribution.SplitJobPhaseCleanup { + seedSplitRuntimeCleanup(t, source, target, job, snapshot) + } + options := splitRuntimeOptions(source, target, job.SourceGroupID) + options = append(options, WithDistributionCoordinator(&ownershipCatalogCoordinator{ + distributionCoordinatorStub: newDistributionCoordinatorStub(root, true), + })) + return NewDistributionServer(engine, catalog, options...), source, target +} + +func seedSplitRuntimeCleanup(t *testing.T, source, target *splitRuntimeClient, job distribution.SplitJob, snapshot distribution.CatalogSnapshot) { + t.Helper() + _, err := source.ApplyTargetStagedReadiness(t.Context(), &pb.TargetStagedReadinessRequest{ + JobId: job.JobID, MigrationJobId: job.JobID, RouteStart: job.SplitKey, RouteEnd: snapshot.Routes[1].End, + Armed: true, SourceWriteFence: true, SourceReadFence: true, RetentionPinTs: 1, + ExpectedCutoverVersion: snapshot.Version, MinWriteTsExclusive: job.FenceTS, + }) + require.NoError(t, err) + if !job.TargetPromotionDone { + _, err = target.ApplyTargetStagedReadiness(t.Context(), &pb.TargetStagedReadinessRequest{ + JobId: job.JobID, MigrationJobId: job.JobID, RouteStart: job.SplitKey, RouteEnd: snapshot.Routes[1].End, + Armed: true, RetentionPinTs: 1, ExpectedCutoverVersion: snapshot.Version, MinWriteTsExclusive: job.FenceTS, + }) + require.NoError(t, err) + } +} diff --git a/adapter/split_job_seal_cleanup_test.go b/adapter/split_job_seal_cleanup_test.go new file mode 100644 index 000000000..03c8fd012 --- /dev/null +++ b/adapter/split_job_seal_cleanup_test.go @@ -0,0 +1,109 @@ +package adapter + +import ( + "context" + "encoding/base64" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type routedSealCleanupClient struct { + splitMigrationClientStub + server *Internal + families []uint32 +} + +func (c *routedSealCleanupClient) CleanupMigration(ctx context.Context, req *pb.CleanupMigrationRequest, _ ...grpc.CallOption) (*pb.CleanupMigrationResponse, error) { + c.families = append(c.families, req.GetKeyFamily()) + req.MaxVersions = 1 + return c.server.CleanupMigration(ctx, req) +} + +func (c *routedSealCleanupClient) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest, _ ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + return c.server.ProbeMigrationState(ctx, req) +} + +func TestSplitJobSourceCleanupRetiresRoutedDynamoSeals(t *testing.T) { + t.Parallel() + for _, legacyDone := range []bool{false, true} { + t.Run(map[bool]string{false: "appended bracket", true: "legacy done cursor"}[legacyDone], func(t *testing.T) { + t.Parallel() + testSplitJobSealCleanup(t, legacyDone) + }) + } +} + +func testSplitJobSealCleanup(t *testing.T, legacyDone bool) { + t.Helper() + ctx := context.Background() + sourceStore := store.NewMVCCStore() + table := base64.RawURLEncoding.EncodeToString([]byte("foo")) + neighbor := base64.RawURLEncoding.EncodeToString([]byte("fooX")) + seal := []byte(kv.DynamoSealPrefix + table + "|7|0001") + other := []byte(kv.DynamoSealPrefix + neighbor + "|7|0001") + for _, ts := range []uint64{10, 20, 80} { + require.NoError(t, sourceStore.PutAt(ctx, seal, []byte("seal"), ts, 0)) + } + require.NoError(t, sourceStore.PutAt(ctx, other, []byte("neighbor"), 20, 0)) + start := kv.RouteKey(seal) + end := append(append([]byte(nil), start...), 0) + catalogStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(catalogStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + snapshot, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{RouteID: 3, ParentRouteID: 42, Start: start, End: end, GroupID: 9, MinWriteTSExclusive: 50}}) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + job.SourceGroupID, job.TargetGroupID, job.FenceTS = 7, 9, 50 + job.SplitKey = start + brackets, err := distribution.PlanCleanupBrackets(start, end) + require.NoError(t, err) + job.Cursor = encodeSplitCleanupCursor(len(brackets)-1, nil, false) + if legacyDone { + job.Cursor = distribution.CloneBytes(splitCleanupDoneCursor) + } + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + runtime, _ := newOwnershipRuntimeClient(t, sourceStore, 7) + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(snapshot)) + runtime.internal.routeEngine = engine + _, err = runtime.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{JobId: job.JobID, MigrationJobId: job.JobID, + RouteStart: start, RouteEnd: end, Armed: true, SourceWriteFence: true, SourceReadFence: true, + ExpectedCutoverVersion: snapshot.Version, MinWriteTsExclusive: job.FenceTS, RetentionPinTs: 1}) + require.NoError(t, err) + client := &routedSealCleanupClient{server: runtime.internal} + server := NewDistributionServer(distribution.NewEngine(), catalog, + WithDistributionCoordinator(newDistributionCoordinatorStub(catalogStore, true)), + WithSplitMigrationVoterFactory(func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "source", Address: "source:1", Client: client}}, nil + }), + WithSplitMigrationClientFactory(func(_ context.Context, _ distribution.SplitJob, groupID uint64) (SplitMigrationClient, SplitMigrationClient, error) { + require.Equal(t, uint64(7), groupID) + return client, &splitMigrationClientStub{}, nil + })) + for range 10 { + require.NoError(t, server.cleanupSplitJobSourceData(ctx, job)) + var found bool + job, found, err = catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + if string(job.Cursor) == string(splitSourceCleanupDoneCursor) { + break + } + } + require.Equal(t, splitSourceCleanupDoneCursor, job.Cursor) + require.Greater(t, len(client.families), 1, "cleanup must persist and resume bounded pages") + for _, family := range client.families { + require.Equal(t, distribution.MigrationFamilyDynamoSnapshotSeal, family) + } + _, err = sourceStore.GetAt(ctx, seal, 20) + require.ErrorIs(t, err, store.ErrKeyNotFound) + _, err = sourceStore.GetAt(ctx, seal, 80) + require.NoError(t, err, "versions above the fence must survive") + _, err = sourceStore.GetAt(ctx, other, 20) + require.NoError(t, err, "the neighboring table must survive") +} diff --git a/adapter/split_job_source_group_test.go b/adapter/split_job_source_group_test.go new file mode 100644 index 000000000..8c77e6411 --- /dev/null +++ b/adapter/split_job_source_group_test.go @@ -0,0 +1,264 @@ +package adapter + +import ( + "context" + "strconv" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func splitJobForSourceGroupTest(t *testing.T) distribution.SplitJob { + t.Helper() + + parent := distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 7, + State: distribution.RouteStateActive, + } + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 1, + SourceRouteID: parent.RouteID, + SplitKey: []byte("m"), + TargetGroupID: 9, + }, parent, 1000) + require.NoError(t, err) + return job +} + +// FENCE replaces the source route with a left child that keeps the source group +// and a right child that moves. SplitRange then permits another split wholly +// inside the left child, because that range is disjoint from the moving one -- +// and it replaces the left child with grandchildren that name the left child as +// their parent. From then on no route ends at the split key under the original +// parent, and the parent itself is gone, so the group that still holds the +// source data cannot be read back out of the route shape at all. Cleanup would +// fail with ErrMigrationSourceRouteChanged on every attempt after cutover and +// keep the job live with its guards and retention pin held. +func TestSplitJobSourceRouteStateSurvivesDisjointSiblingSplit(t *testing.T) { + t.Parallel() + + job := splitJobForSourceGroupTest(t) + require.Equal(t, uint64(7), job.SourceGroupID, "the plan records the source group") + + routes := []distribution.RouteDescriptor{ + // Grandchildren of the left child: their parent is route 2, not route 1. + {RouteID: 4, ParentRouteID: 2, Start: []byte("a"), End: []byte("f"), GroupID: 7}, + {RouteID: 5, ParentRouteID: 2, Start: []byte("f"), End: []byte("m"), GroupID: 7}, + // The moved child still names the original parent. + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 9}, + } + + groupID, routeEnd, ok := splitJobSourceRouteState(routes, job) + require.True(t, ok) + require.Equal(t, uint64(7), groupID) + require.Equal(t, []byte("z"), routeEnd, "the moved range still bounds the cleanup") +} + +// The live route shape stays authoritative while it is intact, so a job written +// before the source group was recorded resolves exactly as it did before. +func TestSplitJobSourceRouteStateStillReadsTheRouteShape(t *testing.T) { + t.Parallel() + + job := splitJobForSourceGroupTest(t) + job.SourceGroupID = 0 + + routes := []distribution.RouteDescriptor{ + {RouteID: 2, ParentRouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 7}, + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 9}, + } + + groupID, routeEnd, ok := splitJobSourceRouteState(routes, job) + require.True(t, ok) + require.Equal(t, uint64(7), groupID) + require.Equal(t, []byte("z"), routeEnd) +} + +// Pre-fence the parent is still present and answers for both. +func TestSplitJobSourceRouteStateFallsBackToTheParent(t *testing.T) { + t.Parallel() + + job := splitJobForSourceGroupTest(t) + job.SourceGroupID = 0 + + routes := []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 7}, + } + + groupID, routeEnd, ok := splitJobSourceRouteState(routes, job) + require.True(t, ok) + require.Equal(t, uint64(7), groupID) + require.Equal(t, []byte("z"), routeEnd) +} + +// With neither a recorded group nor a route to read it from, the caller must +// still be told the source route changed rather than handed group 0. +func TestSplitJobSourceRouteStateReportsAnUnresolvableSource(t *testing.T) { + t.Parallel() + + job := splitJobForSourceGroupTest(t) + job.SourceGroupID = 0 + + routes := []distribution.RouteDescriptor{ + {RouteID: 4, ParentRouteID: 2, Start: []byte("a"), End: []byte("f"), GroupID: 7}, + {RouteID: 5, ParentRouteID: 2, Start: []byte("f"), End: []byte("m"), GroupID: 7}, + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 9}, + } + + _, _, ok := splitJobSourceRouteState(routes, job) + require.False(t, ok) +} + +// TestSplitJobSourceGroupBackfillRecordsTheGroupWhileTheShapeStillAnswers pins +// the repair for a job persisted before source_group_id existed. +// +// InitializeSplitJobPlan records the field for every job it creates, but it runs +// only at plan time: a legacy job decodes as zero and is never re-planned, so +// nothing populated it. It is needed in exactly the state where the route shape +// stops answering, so it has to be written while the shape still does. +func TestSplitJobSourceGroupBackfillRecordsTheGroupWhileTheShapeStillAnswers(t *testing.T) { + t.Parallel() + + legacy := splitJobForSourceGroupTest(t) + legacy.SourceGroupID = 0 + legacy.Phase = distribution.SplitJobPhaseBackfill + + intact := []distribution.RouteDescriptor{ + {RouteID: 2, ParentRouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 7}, + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 9}, + } + + groupID, needed := splitJobSourceGroupBackfill(intact, legacy) + require.True(t, needed, "a legacy job must be repaired while the shape answers") + require.Equal(t, uint64(7), groupID) + + // The degraded shape from TestSplitJobSourceRouteStateReportsAnUnresolvableSource, + // which that test pins as unresolvable for a zero-field job. + degraded := []distribution.RouteDescriptor{ + {RouteID: 4, ParentRouteID: 2, Start: []byte("a"), End: []byte("f"), GroupID: 7}, + {RouteID: 5, ParentRouteID: 2, Start: []byte("f"), End: []byte("m"), GroupID: 7}, + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 9}, + } + _, _, ok := splitJobSourceRouteState(degraded, legacy) + require.False(t, ok, "precondition: the zero-field job cannot resolve the degraded shape") + + // Once the backfill has been recorded, the same degraded shape resolves, so + // CLEANUP no longer fails with ErrMigrationSourceRouteChanged forever while + // holding the job's guards and retention pin. + repaired := legacy + repaired.SourceGroupID = groupID + resolvedGroup, routeEnd, ok := splitJobSourceRouteState(degraded, repaired) + require.True(t, ok) + require.Equal(t, uint64(7), resolvedGroup) + require.Equal(t, []byte("z"), routeEnd, "the moved range still bounds the cleanup") +} + +// TestSplitJobSourceGroupBackfillSkipsWhatItMustNotWrite covers every case that +// must not produce a catalog write. +func TestSplitJobSourceGroupBackfillSkipsWhatItMustNotWrite(t *testing.T) { + t.Parallel() + + intact := []distribution.RouteDescriptor{ + {RouteID: 2, ParentRouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 7}, + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 9}, + } + t.Run("the field is already recorded", func(t *testing.T) { + t.Parallel() + job := splitJobForSourceGroupTest(t) + job.Phase = distribution.SplitJobPhaseCleanup + require.Equal(t, uint64(7), job.SourceGroupID) + + _, needed := splitJobSourceGroupBackfill(intact, job) + require.False(t, needed) + }) + + for _, phase := range []distribution.SplitJobPhase{ + distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoned, + } { + t.Run("terminal phase never reads the field", func(t *testing.T) { + t.Parallel() + job := splitJobForSourceGroupTest(t) + job.SourceGroupID = 0 + job.Phase = phase + + _, needed := splitJobSourceGroupBackfill(intact, job) + require.False(t, needed, "phase %v", phase) + }) + } + +} + +func TestSplitJobSourceGroupBackfillRecoversAlreadyDegradedLegacyJob(t *testing.T) { + t.Parallel() + for _, predecessorGroup := range []uint64{7, 4} { + t.Run(strconv.FormatUint(predecessorGroup, 10), func(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st) + initial, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 7}}) + require.NoError(t, err) + job := splitJobForSourceGroupTest(t) + job.SourceGroupID = 0 + job.SnapshotTS = catalog.LatestCommitTS() + job.Phase = distribution.SplitJobPhaseCleanup + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + current, err := catalog.Save(ctx, initial.Version, []distribution.RouteDescriptor{ + {RouteID: 4, ParentRouteID: 2, Start: []byte("a"), End: []byte("f"), GroupID: predecessorGroup}, + {RouteID: 5, ParentRouteID: 2, Start: []byte("f"), End: []byte("m"), GroupID: predecessorGroup}, + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 9}, + }) + require.NoError(t, err) + server := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(newDistributionCoordinatorStub(st, true))) + repaired, err := server.backfillSplitJobSourceGroup(ctx, job) + require.NoError(t, err) + require.Equal(t, uint64(7), repaired.SourceGroupID, "a migrated predecessor must not redirect source cleanup") + persisted, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint64(7), persisted.SourceGroupID) + group, end, ok := splitJobSourceRouteState(current.Routes, persisted) + require.True(t, ok) + require.Equal(t, uint64(7), group) + require.Equal(t, []byte("z"), end) + }) + } +} + +func TestSplitJobSourceGroupRecoveryUsesRetainedFenceTimestamp(t *testing.T) { + t.Parallel() + ctx := context.Background() + catalog := distribution.NewCatalogStore(store.NewMVCCStore()) + initial, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{ + {RouteID: 2, ParentRouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 7}, + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 9}, + }) + require.NoError(t, err) + fenceTS := catalog.LatestCommitTS() + _, err = catalog.Save(ctx, initial.Version, []distribution.RouteDescriptor{ + {RouteID: 4, ParentRouteID: 2, Start: []byte("a"), End: []byte("f"), GroupID: 4}, + {RouteID: 5, ParentRouteID: 2, Start: []byte("f"), End: []byte("m"), GroupID: 4}, + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 9}, + }) + require.NoError(t, err) + job := splitJobForSourceGroupTest(t) + job.SourceGroupID = 0 + job.Phase = distribution.SplitJobPhaseCleanup + job.FenceTS = fenceTS + server := &DistributionServer{catalog: catalog} + group, found, err := server.legacySplitJobSourceGroup(ctx, job) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint64(7), group) + job.FenceTS = 0 + _, found, err = server.legacySplitJobSourceGroup(ctx, job) + require.NoError(t, err) + require.False(t, found, "never guess the old source from a different current owner") +} diff --git a/adapter/split_job_source_guard_test.go b/adapter/split_job_source_guard_test.go new file mode 100644 index 000000000..e14b40a7f --- /dev/null +++ b/adapter/split_job_source_guard_test.go @@ -0,0 +1,127 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type sourceGuardProbeClient struct { + splitMigrationClientStub + server *Internal +} + +func (s *sourceGuardProbeClient) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest, _ ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + return s.server.ProbeMigrationState(ctx, req) +} + +func TestSplitJobSourceRecoverySurvivesCatalogCompaction(t *testing.T) { + t.Parallel() + for _, ambiguous := range []bool{false, true} { + t.Run(map[bool]string{false: "original source", true: "ambiguous guards"}[ambiguous], func(t *testing.T) { + t.Parallel() + ctx := context.Background() + catalogStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(catalogStore) + initial, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 7}}) + require.NoError(t, err) + job := splitJobForSourceGroupTest(t) + job.SourceGroupID = 0 + job.Phase = distribution.SplitJobPhaseCleanup + job.SnapshotTS = catalog.LatestCommitTS() + job.FenceTS = job.SnapshotTS + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + _, err = catalog.Save(ctx, initial.Version, []distribution.RouteDescriptor{ + {RouteID: 4, ParentRouteID: 2, Start: []byte("a"), End: []byte("f"), GroupID: 4}, + {RouteID: 5, ParentRouteID: 2, Start: []byte("f"), End: []byte("m"), GroupID: 4}, + {RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 9}, + }) + require.NoError(t, err) + require.NoError(t, catalogStore.Compact(ctx, catalog.LatestCommitTS())) + _, err = catalog.SnapshotAt(ctx, job.SnapshotTS) + require.ErrorIs(t, err, store.ErrReadTSCompacted) + clients := make(map[uint64]*sourceGuardProbeClient) + for _, groupID := range []uint64{4, 7, 9} { + st := store.NewMVCCStore() + state := store.TargetStagedReadinessState{ + JobID: job.JobID, MigrationJobID: job.JobID, RouteStart: job.SplitKey, RouteEnd: []byte("z"), + Armed: true, MinWriteTSExclusive: 1, RetentionPinTS: 1, TrackWrites: groupID == 7 || (groupID == 4 && ambiguous), + } + writer, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, state)) + clients[groupID] = &sourceGuardProbeClient{server: NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(st))} + } + server := NewDistributionServer(distribution.NewEngine(), catalog, + WithDistributionCoordinator(newDistributionCoordinatorStub(catalogStore, true)), + WithDistributionKnownRaftGroups(4, 7, 9), + WithSplitMigrationClientFactory(func(_ context.Context, _ distribution.SplitJob, groupID uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return clients[groupID], clients[9], nil + })) + _, err = server.backfillSplitJobSourceGroup(ctx, job) + if ambiguous { + require.ErrorContains(t, err, "multiple groups") + } else { + require.NoError(t, err) + } + persisted, found, loadErr := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, loadErr) + require.True(t, found) + if ambiguous { + require.Zero(t, persisted.SourceGroupID) + } else { + require.Equal(t, uint64(7), persisted.SourceGroupID, "source metadata outlives catalog MVCC ancestry") + } + }) + } +} + +func TestInternalSourceGuardProbeValidatesDurableIdentity(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + ready bool + }{ + {"source tracker", true}, {"source fence", true}, {"target only", false}, {"wrong job", false}, + {"wrong migration", false}, {"wrong start", false}, {"wrong end", false}, {"disarmed", false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + state := store.TargetStagedReadinessState{JobID: 7, MigrationJobID: 7, RouteStart: []byte("m"), RouteEnd: []byte("z"), Armed: true, MinWriteTSExclusive: 1, RetentionPinTS: 1, TrackWrites: true} + switch tc.name { + case "source tracker": + case "source fence": + state.TrackWrites = false + state.SourceWriteFence = true + case "target only": + state.TrackWrites = false + case "wrong job": + state.JobID = 8 + case "wrong migration": + state.MigrationJobID = 8 + case "wrong start": + state.RouteStart = []byte("n") + case "wrong end": + state.RouteEnd = []byte("zz") + case "disarmed": + state.Armed = false + } + writer, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, state)) + leader := &recordingInternalLeader{} + server := NewInternalWithEngine(nil, leader, nil, nil, WithInternalStore(st)) + resp, err := server.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{JobId: 7, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_GUARD_PRESENT, RouteStart: []byte("m"), RouteEnd: []byte("z")}) + require.NoError(t, err) + require.Equal(t, tc.ready, resp.GetReady()) + require.Equal(t, 1, leader.linearizableReadCalls) + }) + } +} diff --git a/adapter/split_job_source_metadata_test.go b/adapter/split_job_source_metadata_test.go new file mode 100644 index 000000000..3d32d75d8 --- /dev/null +++ b/adapter/split_job_source_metadata_test.go @@ -0,0 +1,66 @@ +package adapter + +import ( + "context" + "math" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func testSourceMetadataOwnershipBarrier(t *testing.T) { + t.Helper() + ctx := t.Context() + base := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(base, distribution.WithCatalogRouteDescriptorV2Writes(true)) + routes := promotionCompleteDistributionRoutes() + routes[1].StagedVisibilityActive, routes[1].MigrationJobID = false, 0 + snapshot, err := catalog.Save(ctx, 0, routes) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + job.TargetPromotionDone, job.SourceRetentionPinTS = true, 90 + job.TargetClearedDescriptorAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + job.Cursor = distribution.CloneBytes(splitSourceCleanupDoneCursor) + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + source, _ := newOwnershipRuntimeClient(t, store.NewMVCCStore(), job.SourceGroupID) + follower, _ := newOwnershipRuntimeClient(t, store.NewMVCCStore(), job.SourceGroupID) + primaryApply, ok := source.internal.migrationProposer.(*ownershipRuntimeProposer) + require.True(t, ok) + followerApply, ok := follower.internal.migrationProposer.(*ownershipRuntimeProposer) + require.True(t, ok) + primaryApply.followers = append(primaryApply.followers, followerApply.fsm) + for _, voter := range []*ownershipRuntimeClient{source, follower} { + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(snapshot)) + voter.internal.routeEngine = engine + } + _, err = source.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{JobId: job.JobID, MigrationJobId: job.JobID, + Armed: true, SourceWriteFence: true, SourceReadFence: true, RouteStart: job.SplitKey, + ExpectedCutoverVersion: snapshot.Version, MinWriteTsExclusive: job.FenceTS, RetentionPinTs: 1}) + require.NoError(t, err) + follower.blockMetadata = true + server := NewDistributionServer(distribution.NewEngine(), catalog, + WithDistributionCoordinator(&ownershipCatalogCoordinator{distributionCoordinatorStub: newDistributionCoordinatorStub(base, true)}), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, &splitMigrationClientStub{}, nil + }), + WithSplitMigrationVoterFactory(func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "source", Address: "source:1", Client: source}, {ID: "follower", Address: "follower:1", Client: follower}}, nil + })) + require.ErrorIs(t, server.cleanupSplitJobSourceProofs(ctx, job), errSplitMigrationVoterBarrierIncomplete) + require.Len(t, source.cleanups, 1) + require.Equal(t, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR, source.cleanups[0].GetMode()) + current, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint64(90), current.SourceRetentionPinTS, "no job pin release until every metadata ACK") + follower.blockMetadata = false + require.NoError(t, server.cleanupSplitJobSourceProofs(ctx, current)) + current, found, err = catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint64(math.MaxUint64), current.SourceRetentionPinTS) +} diff --git a/adapter/split_job_source_ownership.go b/adapter/split_job_source_ownership.go new file mode 100644 index 000000000..257719f74 --- /dev/null +++ b/adapter/split_job_source_ownership.go @@ -0,0 +1,117 @@ +package adapter + +import ( + "bytes" + "context" + "crypto/sha256" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +// The tracker precedes its certificate in the same data log. Creation is +// refused between these entries; authenticated old resolutions still drain. +func (s *DistributionServer) armSplitSourceOwnership(ctx context.Context, job distribution.SplitJob, groupID uint64, source SplitMigrationClient) ([]byte, bool, error) { + configuration, generation, err := s.splitOwnershipConfiguration(ctx, job.JobID, groupID, source) + if err != nil { + return nil, false, err + } + snapshot, err := s.loadCommittedMigrationCatalog(ctx) + if err != nil { + return nil, false, err + } + proof, err := splitSourceOwnershipProof(snapshot, job, groupID, generation) + if err != nil { + return nil, false, err + } + prior, err := probeSplitSourceOwnershipState(ctx, source, job.JobID, configuration) + if err != nil { + return nil, false, errors.WithStack(err) + } + mode, err := splitSourceOwnershipOperation(proof, prior) + if err != nil { + return nil, false, err + } + minimum := max(initialMigrationRetentionPinTS, proof.GetMinWriteTsExclusive()) + if _, err := applySplitMigrationControl(ctx, source, job, proof.GetRouteEnd(), proof.GetSourceTrackerVersion(), minimum, + false, false, true, initialMigrationRetentionPinTS); err != nil { + return nil, false, err + } + acks, complete, err := s.syncSplitMigrationVoterBarrier(ctx, groupID, job.FenceAckCursor, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: job.SplitKey, RouteEnd: proof.GetRouteEnd(), ExpectedCatalogVersion: proof.GetSourceTrackerVersion(), + MigrationJobId: job.JobID, MinWriteTsExclusive: minimum, TrackWrites: true, RetentionPinTs: initialMigrationRetentionPinTS, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: configuration}) + if err != nil || !complete { + return acks, false, err + } + if _, err := cleanupSplitMigration(ctx, source, &pb.CleanupMigrationRequest{JobId: job.JobID, Mode: mode, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: configuration, Ownership: proof}); err != nil { + return nil, false, err + } + return s.ackSplitSourceOwnership(ctx, job, groupID, configuration, source, proof) +} + +func splitSourceOwnershipProof(snapshot distribution.CatalogSnapshot, job distribution.SplitJob, groupID uint64, generation string) (*pb.MigrationOwnershipProof, error) { + route, found := findRouteByID(snapshot.Routes, sourceLookupRouteID(job)) + if !found || route.GroupID != groupID || route.State != distribution.RouteStateActive || route.StagedVisibilityActive || + !splitOwnershipRouteContains(route, job.SplitKey) || snapshot.Version == 0 { + return nil, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(route, true) + if err != nil { + return nil, errors.WithStack(err) + } + digest := sha256.Sum256(encoded) + return &pb.MigrationOwnershipProof{Generation: generation, LocalGroupId: groupID, OwnerGroupId: groupID, + CatalogVersion: snapshot.Version, RouteId: route.RouteID, ParentRouteId: route.ParentRouteID, + RouteStart: bytes.Clone(job.SplitKey), RouteEnd: bytes.Clone(route.End), MinWriteTsExclusive: route.MinWriteTSExclusive, + WriteFloorKnown: true, SourceTrackerVersion: snapshot.Version, Role: pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER, + DescriptorHash: digest[:]}, nil +} + +func splitSourceOwnershipOperation(proof *pb.MigrationOwnershipProof, prior *pb.ProbeMigrationStateResponse) (pb.MigrationCleanupMode, error) { + mode := pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_SOURCE_OWNERSHIP_ACQUIRE + if prior.GetOwnershipProtocol() != migrationOwnershipProtocol { + return mode, errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + old := prior.GetOwnership() + if old == nil { + return mode, nil + } + if !prior.GetReady() || old.GetSourceAppliedIndex() == 0 || old.GetGeneration() != proof.GetGeneration() || + !bytes.Equal(old.GetDescriptorHash(), proof.GetDescriptorHash()) { + return mode, errors.New("source tracker requires unchanged committed ownership") + } + proof.SourceTrackerVersion, proof.SourceAppliedIndex = old.GetSourceTrackerVersion(), old.GetSourceAppliedIndex() + return pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH, nil +} + +func (s *DistributionServer) ackSplitSourceOwnership(ctx context.Context, job distribution.SplitJob, groupID, configuration uint64, source SplitMigrationClient, proof *pb.MigrationOwnershipProof) ([]byte, bool, error) { + resp, err := probeSplitSourceOwnershipState(ctx, source, job.JobID, configuration) + if err != nil { + return nil, false, errors.WithStack(err) + } + if !resp.GetReady() || resp.GetOwnership().GetSourceAppliedIndex() == 0 { + return nil, false, errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + expected := proto.CloneOf(proof) + expected.SourceAppliedIndex = resp.GetOwnership().GetSourceAppliedIndex() + if !proto.Equal(expected, resp.GetOwnership()) { + return nil, false, errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + return s.syncSplitMigrationVoterBarrier(ctx, groupID, nil, source, &pb.ProbeMigrationStateRequest{JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED, Ownership: expected, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: configuration}) +} + +func probeSplitSourceOwnershipState(ctx context.Context, source SplitMigrationClient, jobID, configuration uint64) (*pb.ProbeMigrationStateResponse, error) { + attempt, cancel := context.WithTimeout(ctx, defaultSplitMigrationAttemptTimeout) + defer cancel() + resp, err := source.ProbeMigrationState(attempt, &pb.ProbeMigrationStateRequest{JobId: jobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: configuration}) + return resp, errors.WithStack(err) +} diff --git a/adapter/split_job_source_ownership_test.go b/adapter/split_job_source_ownership_test.go new file mode 100644 index 000000000..6422fb205 --- /dev/null +++ b/adapter/split_job_source_ownership_test.go @@ -0,0 +1,61 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestSplitSourceOwnershipCommitsActualZeroFloorAndDataBoundary(t *testing.T) { + root := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(root, distribution.WithCatalogRouteDescriptorV2Writes(true)) + snapshot, err := catalog.Save(t.Context(), 0, []distribution.RouteDescriptor{{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 7}}) + require.NoError(t, err) + job := sampleDistributionSplitJob(11) + job.Phase, job.SourceRouteID, job.SourceGroupID, job.TargetGroupID, job.SplitKey = distribution.SplitJobPhasePlanned, 1, 7, 9, []byte("m") + local := store.NewMVCCStore() + source, _ := newOwnershipRuntimeClient(t, local, 7) + server := NewDistributionServer(distribution.NewEngine(), catalog, + WithDistributionCoordinator(&ownershipCatalogCoordinator{distributionCoordinatorStub: newDistributionCoordinatorStub(root, true)}), + WithSplitMigrationVoterFactory(func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "source", Address: "source:1", Client: source}}, nil + })) + _, complete, err := server.armSplitSourceOwnership(t.Context(), job, 7, source) + require.NoError(t, err) + require.True(t, complete) + proofs, ok := local.(store.MigrationOwnershipStore) + require.True(t, ok) + view, err := proofs.MigrationServingState(t.Context()) + require.NoError(t, err) + proof := view.Ownership[0] + require.Equal(t, snapshot.Version, proof.CatalogVersion, "never the predicted next catalog version") + require.True(t, proof.WriteFloorKnown) + require.Zero(t, proof.MinWriteTSExclusive) + require.Positive(t, proof.SourceAppliedIndex) + require.Equal(t, uint64(9), proof.ConfigurationIndex) + require.True(t, store.MigrationOwnershipEvidenceValid(proof)) + tx := kv.NewTransactionWithProposer(source.internal.migrationProposer, kv.WithMigrationRequestAdmission(local, 7)) + t.Cleanup(tx.Close) + _, err = tx.Commit(t.Context(), []*pb.Request{{Ts: 50, ObservedRouteVersion: kv.EncodeObservedRouteVersion(snapshot.Version), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("tracked")}}}}) + require.NoError(t, err) + view, err = proofs.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, uint64(50), view.Readiness[0].MinAdmittedTS) + _, err = catalog.Save(t.Context(), snapshot.Version, append(snapshot.Routes, + distribution.RouteDescriptor{RouteID: 2, Start: []byte("z"), GroupID: 9})) + require.NoError(t, err) + _, complete, err = server.armSplitSourceOwnership(t.Context(), job, 7, source) + require.NoError(t, err) + require.True(t, complete) + view, err = proofs.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, snapshot.Version+1, view.Ownership[0].CatalogVersion) + require.Equal(t, proof.SourceAppliedIndex, view.Ownership[0].SourceAppliedIndex) + require.Equal(t, proof.Receipts, view.Ownership[0].Receipts) +} diff --git a/adapter/split_job_source_ownership_timeout_test.go b/adapter/split_job_source_ownership_timeout_test.go new file mode 100644 index 000000000..8ad011916 --- /dev/null +++ b/adapter/split_job_source_ownership_timeout_test.go @@ -0,0 +1,130 @@ +package adapter + +import ( + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type blockedSourceOwnershipProbe struct { + *ownershipRuntimeClient + blockAt int + probes int + blocked context.Context + onBlocked func() +} + +func (c *blockedSourceOwnershipProbe) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest, opts ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + if req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED { + c.probes++ + if c.probes == c.blockAt { + c.blocked = ctx + if c.onBlocked != nil { + c.onBlocked() + } + <-ctx.Done() + return nil, ctx.Err() + } + } + return c.ownershipRuntimeClient.ProbeMigrationState(ctx, req, opts...) +} + +func sourceOwnershipTimeoutFixture(t *testing.T, blockAt int) (*DistributionServer, distribution.SplitJob, *blockedSourceOwnershipProbe) { + t.Helper() + root := store.NewMVCCStore() + t.Cleanup(func() { _ = root.Close() }) + catalog := distribution.NewCatalogStore(root, distribution.WithCatalogRouteDescriptorV2Writes(true)) + _, err := catalog.Save(t.Context(), 0, []distribution.RouteDescriptor{{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 7}}) + require.NoError(t, err) + job := sampleDistributionSplitJob(11) + job.Phase, job.SourceRouteID, job.SourceGroupID, job.TargetGroupID, job.SplitKey = distribution.SplitJobPhasePlanned, 1, 7, 9, []byte("m") + require.NoError(t, catalog.CreateSplitJob(t.Context(), job)) + local := store.NewMVCCStore() + t.Cleanup(func() { _ = local.Close() }) + runtime, _ := newOwnershipRuntimeClient(t, local, 7) + source := &blockedSourceOwnershipProbe{ownershipRuntimeClient: runtime, blockAt: blockAt} + server := NewDistributionServer(distribution.NewEngine(), catalog, + WithDistributionCoordinator(&ownershipCatalogCoordinator{distributionCoordinatorStub: newDistributionCoordinatorStub(root, true)}), + WithSplitMigrationVoterFactory(func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "source", Address: "source:1", Client: source}}, nil + })) + return server, job, source +} + +func sourceOwnershipTimeoutView(t *testing.T, source *blockedSourceOwnershipProbe) store.MigrationServingState { + t.Helper() + reader, ok := source.internal.store.(store.MigrationOwnershipStore) + require.True(t, ok) + view, err := reader.MigrationServingState(t.Context()) + require.NoError(t, err) + return view +} + +func TestSplitSourceOwnershipBoundsStalledProbes(t *testing.T) { + t.Parallel() + for index, name := range []string{"prior", "ack"} { + t.Run(name, func(t *testing.T) { + t.Parallel() + server, job, source := sourceOwnershipTimeoutFixture(t, index+1) + ctx, cancel := context.WithTimeout(t.Context(), 3*defaultSplitMigrationAttemptTimeout) + defer cancel() + cursor, complete, err := server.armSplitSourceOwnership(ctx, job, 7, source) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.NoError(t, ctx.Err(), "a stalled RPC must not consume the runner's parent lifetime") + require.True(t, splitJobPhaseErrorRetryable(err)) + require.False(t, complete) + require.Nil(t, cursor) + deadline, bounded := source.blocked.Deadline() + require.True(t, bounded) + require.True(t, time.Now().After(deadline)) + before := sourceOwnershipTimeoutView(t, source) + require.Len(t, before.Ownership, index) + require.Len(t, before.Readiness, index) + saved, found, err := server.catalog.SplitJob(t.Context(), job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhasePlanned, saved.Phase) + source.blockAt = 0 + _, complete, err = server.armSplitSourceOwnership(ctx, job, 7, source) + require.NoError(t, err) + require.True(t, complete, "the next attempt must rediscover installed ownership") + after := sourceOwnershipTimeoutView(t, source) + require.Len(t, after.Ownership, 1) + require.Len(t, after.Readiness, 1) + require.True(t, store.MigrationOwnershipEvidenceValid(after.Ownership[0])) + if index == 1 { + require.True(t, before.Readiness[0].Armed) + require.True(t, before.Readiness[0].TrackWrites) + require.Equal(t, initialMigrationRetentionPinTS, before.Readiness[0].RetentionPinTS) + require.Equal(t, before.Ownership[0].SourceAppliedIndex, after.Ownership[0].SourceAppliedIndex) + require.Equal(t, before.Ownership[0].Receipts, after.Ownership[0].Receipts) + require.Equal(t, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH, source.cleanups[len(source.cleanups)-1].GetMode()) + } + }) + } +} + +func TestSplitSourceOwnershipBlockedProbesHonorParentCancellation(t *testing.T) { + t.Parallel() + for index, name := range []string{"prior", "ack"} { + t.Run(name, func(t *testing.T) { + t.Parallel() + server, job, source := sourceOwnershipTimeoutFixture(t, index+1) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + source.onBlocked = cancel + cursor, complete, err := server.armSplitSourceOwnership(ctx, job, 7, source) + require.ErrorIs(t, err, context.Canceled) + require.True(t, splitJobPhaseErrorRetryable(err)) + require.False(t, complete) + require.Nil(t, cursor) + require.ErrorIs(t, source.blocked.Err(), context.Canceled) + }) + } +} diff --git a/adapter/split_job_sqs_range_test.go b/adapter/split_job_sqs_range_test.go new file mode 100644 index 000000000..7a31885d0 --- /dev/null +++ b/adapter/split_job_sqs_range_test.go @@ -0,0 +1,132 @@ +package adapter + +import ( + "context" + "strconv" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestDistributionServerStartSplitMigrationRejectsResolverOwnedSQSRange(t *testing.T) { + t.Parallel() + ctx := context.Background() + require.Equal(t, []byte("!sqs|route|global"), kv.RouteKey([]byte("!sqs|msg|data|p|orders|3"))) + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st) + snapshot, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, Start: []byte("!sqs|route|"), End: []byte("!sqs|route|z"), GroupID: 7, + }}) + require.NoError(t, err) + coordinator := newDistributionCoordinatorStub(st, true) + server := NewDistributionServer(distribution.NewEngine(), catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(7, 9), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + _, err = server.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + RouteId: 1, SplitKey: []byte("!sqs|route|global"), TargetGroupId: 9, ExpectedCatalogVersion: snapshot.Version, + }) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "resolver-owned SQS") + require.Zero(t, coordinator.dispatchCalls) + jobs, err := catalog.ListSplitJobs(ctx) + require.NoError(t, err) + require.Empty(t, jobs) +} + +func TestSplitJobSQSOverlapDoesNotBlockRecovery(t *testing.T) { + t.Parallel() + server := NewDistributionServer(distribution.NewEngine(), nil) + for _, phase := range []distribution.SplitJobPhase{distribution.SplitJobPhaseAbandoning, distribution.SplitJobPhaseCleanup} { + job := distribution.SplitJob{Phase: phase, SplitKey: []byte("!sqs|route|global")} + require.NoError(t, server.verifySplitJobMigrationRange(context.Background(), job)) + } +} + +func TestSplitJobAbandonsLegacySQSOverlapWithoutLosingData(t *testing.T) { + t.Parallel() + ctx := context.Background() + st, sourceStore := store.NewMVCCStore(), store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st, distribution.WithCatalogRouteDescriptorV2Writes(true)) + global, end := []byte("!sqs|route|global"), []byte("!sqs|route|z") + snapshot, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{ + {RouteID: 2, ParentRouteID: 1, Start: []byte("!sqs|route|"), End: global, GroupID: 7}, + {RouteID: 3, ParentRouteID: 1, Start: global, End: end, GroupID: 7, State: distribution.RouteStateWriteFenced}, + }) + require.NoError(t, err) + job := splitJobForSourceGroupTest(t) + job.SplitKey, job.Phase, job.AbandonFromPhase = global, distribution.SplitJobPhaseAbandoning, distribution.SplitJobPhaseFence + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + key := []byte("!sqs|msg|data|p|orders|3") + require.NoError(t, sourceStore.PutAt(ctx, key, []byte("retained"), 1, 0)) + writer, ok := sourceStore.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, store.TargetStagedReadinessState{ + JobID: job.JobID, MigrationJobID: job.JobID, RouteStart: global, RouteEnd: end, + Armed: true, SourceWriteFence: true, MinWriteTSExclusive: 1, RetentionPinTS: 1, + })) + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(snapshot)) + source := newSplitRuntimeClient(t, engine, sourceStore, job.SourceGroupID) + target := newSplitRuntimeClient(t, engine, store.NewMVCCStore(), job.TargetGroupID) + options := splitRuntimeOptions(source, target, job.SourceGroupID) + options = append(options, WithDistributionCoordinator(&ownershipCatalogCoordinator{ + distributionCoordinatorStub: newDistributionCoordinatorStub(st, true), + })) + server := NewDistributionServer(engine, catalog, options...) + for range 8 { + require.NoError(t, server.RunSplitJobRunnerOnce(ctx)) + } + completed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseAbandoned, completed.Phase) + sharded := kv.NewShardStore(server.engine, map[uint64]*kv.ShardGroup{7: {Store: sourceStore}}) + value, err := sharded.GetGroupAt(ctx, 7, key, 100) + require.NoError(t, err) + require.Equal(t, []byte("retained"), value) + txn := kv.NewTransactionWithProposer(source.internal.migrationProposer, kv.WithMigrationRequestAdmission(sourceStore, 7)) + t.Cleanup(txn.Close) + _, err = txn.Commit(ctx, []*pb.Request{{Ts: 100, ObservedRouteVersion: kv.EncodeObservedRouteVersion(server.engine.Version()), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("available")}}}}) + require.NoError(t, err) + value, err = sharded.GetGroupAt(ctx, 7, key, 100) + require.NoError(t, err) + require.Equal(t, []byte("available"), value) +} + +func TestSplitJobRunnerRejectsPersistedResolverOwnedSQSRange(t *testing.T) { + t.Parallel() + for _, phase := range []distribution.SplitJobPhase{ + distribution.SplitJobPhasePlanned, distribution.SplitJobPhaseBackfill, distribution.SplitJobPhaseFence, + distribution.SplitJobPhaseDeltaCopy, distribution.SplitJobPhaseCutover, + } { + t.Run(strconv.Itoa(int(phase)), func(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st) + _, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, Start: []byte("!sqs|route|"), End: []byte("!sqs|route|z"), GroupID: 7, + }}) + require.NoError(t, err) + job := splitJobForSourceGroupTest(t) + job.SplitKey, job.Phase = []byte("!sqs|route|global"), phase + server := NewDistributionServer(distribution.NewEngine(), catalog, + WithDistributionCoordinator(newDistributionCoordinatorStub(st, true)), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + t.Fatal("unsupported catalog interval must be rejected before migration RPCs") + return nil, nil, nil + }), + ) + require.ErrorContains(t, server.runSplitJobPhase(ctx, job), "resolver-owned SQS") + }) + } +} diff --git a/adapter/split_job_transfer_admission.go b/adapter/split_job_transfer_admission.go new file mode 100644 index 000000000..3fdb7030a --- /dev/null +++ b/adapter/split_job_transfer_admission.go @@ -0,0 +1,198 @@ +package adapter + +import ( + "bytes" + "context" + "slices" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +// This witness is local and immutable, not catalog evidence. Recreate it when +// controls change, and revalidate it before advancing a phase. Never reuse an +// ownership ACK by rebinding its control to a newly observed configuration. +type splitTransferAdmissionWitness struct { + groupID uint64 + snapshot SplitMigrationApplyMemberSnapshot + control *pb.TargetStagedReadinessRequest + ownershipProof *pb.MigrationOwnershipProof +} + +// The caller has already ACKed the full same-owner partition at the control's +// configuration. This must finish before preflight timestamps/catalog writes, +// history export, or source write fencing. No leader-only fallback is allowed. +func (s *DistributionServer) ensureSplitTransferAdmission(ctx context.Context, job distribution.SplitJob, groupID uint64, source SplitMigrationClient, control *pb.TargetStagedReadinessRequest, ownershipProof *pb.MigrationOwnershipProof) (*splitTransferAdmissionWitness, error) { + if err := validateSplitTransferAdmission(job, groupID, source, control, ownershipProof); err != nil { + return nil, err + } + snapshot, err := s.splitTransferApplyMembers(ctx, groupID) + if err != nil { + return nil, err + } + if snapshot.ConfigurationIndex != control.GetExpectedConfigurationIndex() { + return nil, errors.New("transfer guard configuration differs from ownership ACK") + } + witness := &splitTransferAdmissionWitness{groupID: groupID, snapshot: snapshot, + control: proto.CloneOf(control), ownershipProof: proto.CloneOf(ownershipProof)} + witness.control.TransferSizeGuardVersion = store.MigrationTransferSizeGuardVersion + if err := probeSplitTransferOwnership(ctx, witness, snapshot.Members); err != nil { + return nil, err + } + if _, err := s.revalidateSplitTransferMembers(ctx, witness); err != nil { + return nil, err + } + attempt, cancel := context.WithTimeout(ctx, defaultSplitMigrationAttemptTimeout) + defer cancel() + if _, err := source.ApplyTargetStagedReadiness(attempt, proto.CloneOf(witness.control)); err != nil { + return nil, errors.WithStack(err) + } + if err := s.revalidateSplitTransferAdmission(ctx, witness); err != nil { + return nil, err + } + return witness, nil +} + +func validateSplitTransferAdmission(job distribution.SplitJob, groupID uint64, source SplitMigrationClient, control *pb.TargetStagedReadinessRequest, proof *pb.MigrationOwnershipProof) error { + if source == nil || !validSplitTransferControl(job, groupID, control) { + return errors.New("transfer guard requires configuration-bound armed source controls") + } + if !validSplitTransferOwner(groupID, proof) || + bytes.Compare(proof.GetRouteStart(), control.GetRouteStart()) > 0 || + !splitTransferEndContains(proof.GetRouteEnd(), control.GetRouteEnd()) { + return errors.New("transfer guard requires the ACKed same-owner source proof") + } + return nil +} + +func validSplitTransferControl(job distribution.SplitJob, groupID uint64, control *pb.TargetStagedReadinessRequest) bool { + if control == nil { + return false + } + return job.JobID != 0 && groupID != 0 && control.GetJobId() == job.JobID && + control.GetMigrationJobId() == job.JobID && len(job.SplitKey) != 0 && bytes.Equal(control.GetRouteStart(), job.SplitKey) && + validTransferSizeGuardActivation(transferGuardControl(control)) && + (control.GetTransferSizeGuardVersion() == 0 || control.GetTransferSizeGuardVersion() == store.MigrationTransferSizeGuardVersion) +} + +func validSplitTransferOwner(groupID uint64, proof *pb.MigrationOwnershipProof) bool { + return proof != nil && proof.GetGeneration() != "" && proof.GetSourceAppliedIndex() != 0 && + proof.GetSourceTrackerVersion() != 0 && proof.GetLocalGroupId() == groupID && proof.GetOwnerGroupId() == groupID && + !proof.GetStaged() && proof.GetRole() == pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER +} + +func transferGuardControl(control *pb.TargetStagedReadinessRequest) *pb.TargetStagedReadinessRequest { + guarded := proto.CloneOf(control) + guarded.TransferSizeGuardVersion = store.MigrationTransferSizeGuardVersion + return guarded +} + +func splitTransferEndContains(outer, inner []byte) bool { + return len(outer) == 0 || (len(inner) != 0 && bytes.Compare(outer, inner) >= 0) +} + +func (s *DistributionServer) splitTransferApplyMembers(ctx context.Context, groupID uint64) (SplitMigrationApplyMemberSnapshot, error) { + if err := ctx.Err(); err != nil { + return SplitMigrationApplyMemberSnapshot{}, errors.WithStack(err) + } + if s.splitApplyMemberFactory == nil { + return SplitMigrationApplyMemberSnapshot{}, errors.New("transfer guard all-member snapshot factory is not configured") + } + snapshot, err := s.splitApplyMemberFactory(ctx, groupID) + if err != nil { + return SplitMigrationApplyMemberSnapshot{}, errors.WithStack(err) + } + if snapshot.ConfigurationIndex == 0 || len(snapshot.Members) == 0 { + return SplitMigrationApplyMemberSnapshot{}, errors.New("transfer guard requires a nonempty applied configuration") + } + snapshot.Members = slices.Clone(snapshot.Members) + if err := validateSplitMigrationVoters(snapshot.Members); err != nil { + return SplitMigrationApplyMemberSnapshot{}, err + } + addresses := make(map[string]struct{}, len(snapshot.Members)) + for _, member := range snapshot.Members { + if _, duplicate := addresses[member.Address]; duplicate { + return SplitMigrationApplyMemberSnapshot{}, errors.New("transfer guard member address is duplicated") + } + addresses[member.Address] = struct{}{} + } + return snapshot, nil +} + +func (s *DistributionServer) revalidateSplitTransferMembers(ctx context.Context, witness *splitTransferAdmissionWitness) (SplitMigrationApplyMemberSnapshot, error) { + snapshot, err := s.splitTransferApplyMembers(ctx, witness.groupID) + if err != nil { + return SplitMigrationApplyMemberSnapshot{}, err + } + if snapshot.ConfigurationIndex != witness.snapshot.ConfigurationIndex || !slices.EqualFunc(snapshot.Members, witness.snapshot.Members, + func(a, b SplitMigrationVoter) bool { return a.ID == b.ID && a.Address == b.Address }) { + return SplitMigrationApplyMemberSnapshot{}, errors.New("transfer guard apply-member configuration changed") + } + return snapshot, nil +} + +func probeSplitTransferMember(ctx context.Context, member SplitMigrationVoter, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + attempt, cancel := context.WithTimeout(ctx, defaultSplitMigrationVoterProbeTimeout) + defer cancel() + resp, err := member.Client.ProbeMigrationState(attempt, proto.CloneOf(req)) + if err != nil { + return nil, errors.Wrapf(err, "transfer guard member %s", member.ID) + } + if resp == nil { + return nil, errors.Errorf("transfer guard member %s returned no probe response", member.ID) + } + if !resp.GetReady() { + return nil, errors.Wrapf(errSplitMigrationVoterBarrierIncomplete, "transfer guard member %s awaits applied configuration or size admission ACK", member.ID) + } + if resp.GetAppliedConfigurationIndex() != req.GetExpectedConfigurationIndex() || + resp.GetTransferSizeGuardVersion() != store.MigrationTransferSizeGuardVersion { + return nil, errors.Errorf("transfer guard member %s lacks applied configuration or size admission ACK", member.ID) + } + return resp, nil +} + +func probeSplitTransferOwnership(ctx context.Context, witness *splitTransferAdmissionWitness, members []SplitMigrationVoter) error { + req := &pb.ProbeMigrationStateRequest{JobId: witness.control.GetJobId(), + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: witness.snapshot.ConfigurationIndex, + Ownership: witness.ownershipProof, TransferSizeGuardVersion: store.MigrationTransferSizeGuardVersion} + for _, member := range members { + resp, err := probeSplitTransferMember(ctx, member, req) + if err != nil { + return err + } + if resp.GetOwnershipProtocol() != migrationOwnershipProtocol || + resp.GetOwnershipPartitionProtocol() != MigrationOwnershipPartitionProtocolVersion || + resp.GetOwnershipCleared() || !proto.Equal(resp.GetOwnership(), witness.ownershipProof) { + return errors.Errorf("transfer guard member %s lacks current partition ownership ACK", member.ID) + } + } + return nil +} + +func (s *DistributionServer) revalidateSplitTransferAdmission(ctx context.Context, witness *splitTransferAdmissionWitness) error { + if witness == nil || witness.control == nil || witness.ownershipProof == nil { + return errors.New("transfer guard admission witness is missing") + } + snapshot, err := s.revalidateSplitTransferMembers(ctx, witness) + if err != nil { + return err + } + if err := probeSplitTransferOwnership(ctx, witness, snapshot.Members); err != nil { + return err + } + req := probeMigrationControlRequest(witness.control) + req.RequireConfigurationIndex = true + req.ExpectedConfigurationIndex = witness.snapshot.ConfigurationIndex + req.TransferSizeGuardVersion = store.MigrationTransferSizeGuardVersion + for _, member := range snapshot.Members { + if _, err := probeSplitTransferMember(ctx, member, req); err != nil { + return err + } + } + _, err = s.revalidateSplitTransferMembers(ctx, witness) + return err +} diff --git a/adapter/split_job_transfer_admission_test.go b/adapter/split_job_transfer_admission_test.go new file mode 100644 index 000000000..7204c1369 --- /dev/null +++ b/adapter/split_job_transfer_admission_test.go @@ -0,0 +1,219 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" +) + +type transferAdmissionClient struct { + SplitMigrationClient + probe func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) + apply func(*pb.TargetStagedReadinessRequest) (*pb.TargetStagedReadinessResponse, error) +} + +func (c *transferAdmissionClient) ProbeMigrationState(_ context.Context, req *pb.ProbeMigrationStateRequest, _ ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + return c.probe(req) +} + +func (c *transferAdmissionClient) ApplyTargetStagedReadiness(_ context.Context, req *pb.TargetStagedReadinessRequest, _ ...grpc.CallOption) (*pb.TargetStagedReadinessResponse, error) { + return c.apply(req) +} + +type transferAdmissionFixture struct { + server *DistributionServer + job distribution.SplitJob + control *pb.TargetStagedReadinessRequest + proof *pb.MigrationOwnershipProof + snapshot SplitMigrationApplyMemberSnapshot + source *transferAdmissionClient + probes []string + applied []*pb.TargetStagedReadinessRequest +} + +func newTransferAdmissionFixture(t *testing.T) *transferAdmissionFixture { + t.Helper() + f := &transferAdmissionFixture{ + job: distribution.SplitJob{JobID: 11, SourceRouteID: 3, SplitKey: []byte("m")}, + control: &pb.TargetStagedReadinessRequest{JobId: 11, MigrationJobId: 11, Armed: true, TrackWrites: true, + RouteStart: []byte("m"), RouteEnd: []byte("z"), ExpectedCutoverVersion: 7, RetentionPinTs: 1, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: 41}, + proof: &pb.MigrationOwnershipProof{Generation: "root", LocalGroupId: 2, OwnerGroupId: 2, RouteId: 3, + RouteStart: []byte("a"), RouteEnd: []byte("z"), SourceTrackerVersion: 7, SourceAppliedIndex: 9, + Role: pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER}, + } + f.snapshot.ConfigurationIndex = 41 + for _, id := range []string{"voter", "learner"} { + client := &transferAdmissionClient{probe: func(req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + f.probes = append(f.probes, id) + require.True(t, req.GetRequireConfigurationIndex()) + require.Equal(t, uint64(41), req.GetExpectedConfigurationIndex()) + if req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED { + require.Equal(t, uint32(1), req.GetTransferSizeGuardVersion(), "field 18 must bind persisted guard ACK") + require.Equal(t, f.control.GetSourceWriteFence(), req.GetSourceWriteFence()) + } + return &pb.ProbeMigrationStateResponse{Ready: true, AppliedConfigurationIndex: 41, + OwnershipProtocol: 2, OwnershipPartitionProtocol: 1, TransferSizeGuardVersion: 1, + Ownership: proto.CloneOf(f.proof)}, nil + }} + f.snapshot.Members = append(f.snapshot.Members, SplitMigrationVoter{ID: id, Address: id + ":9000", Client: client}) + } + f.source = &transferAdmissionClient{apply: func(req *pb.TargetStagedReadinessRequest) (*pb.TargetStagedReadinessResponse, error) { + require.Contains(t, f.probes, "learner", "all members must attest before activation") + require.Equal(t, uint32(1), req.GetTransferSizeGuardVersion()) + f.applied = append(f.applied, proto.CloneOf(req)) + return &pb.TargetStagedReadinessResponse{}, nil + }} + f.server = NewDistributionServer(nil, nil, WithSplitMigrationApplyMemberSnapshotFactory(func(context.Context, uint64) (SplitMigrationApplyMemberSnapshot, error) { + return f.snapshot, nil + })) + return f +} + +func TestSplitTransferAdmissionAllMembersAndRevalidation(t *testing.T) { + f := newTransferAdmissionFixture(t) + witness, err := f.server.ensureSplitTransferAdmission(t.Context(), f.job, 2, f.source, f.control, f.proof) + require.NoError(t, err) + require.Len(t, f.applied, 1) + require.Zero(t, f.control.GetTransferSizeGuardVersion(), "caller controls must not be mutated") + require.NoError(t, f.server.revalidateSplitTransferAdmission(t.Context(), witness)) + f.snapshot.ConfigurationIndex++ + require.Error(t, f.server.revalidateSplitTransferAdmission(t.Context(), witness)) + require.Len(t, f.applied, 1, "revalidation must never silently reapply at a new epoch") +} + +func TestSplitTransferAdmissionFailsClosedBeforeApply(t *testing.T) { + cases := map[string]func(*transferAdmissionFixture){ + "missing factory": func(f *transferAdmissionFixture) { f.server = NewDistributionServer(nil, nil) }, + "empty": func(f *transferAdmissionFixture) { f.snapshot.Members = nil }, + "zero epoch": func(f *transferAdmissionFixture) { f.snapshot.ConfigurationIndex = 0 }, + "changed epoch": func(f *transferAdmissionFixture) { f.snapshot.ConfigurationIndex++ }, + "unbound control": func(f *transferAdmissionFixture) { f.control.RequireConfigurationIndex = false }, + "missing proof": func(f *transferAdmissionFixture) { f.proof = nil }, + "duplicate": func(f *transferAdmissionFixture) { f.snapshot.Members[1] = f.snapshot.Members[0] }, + "missing client": func(f *transferAdmissionFixture) { f.snapshot.Members[1].Client = nil }, + } + for name, change := range cases { + t.Run(name, func(t *testing.T) { + f := newTransferAdmissionFixture(t) + change(f) + _, err := f.server.ensureSplitTransferAdmission(t.Context(), f.job, 2, f.source, f.control, f.proof) + require.Error(t, err) + require.Empty(t, f.applied) + }) + } +} + +func TestSplitTransferAdmissionRejectsMemberEvidence(t *testing.T) { + cases := map[string]func(*pb.ProbeMigrationStateResponse){ + "old size": func(r *pb.ProbeMigrationStateResponse) { r.TransferSizeGuardVersion = 0 }, + "old partition": func(r *pb.ProbeMigrationStateResponse) { r.OwnershipPartitionProtocol = 0 }, + "wrong config": func(r *pb.ProbeMigrationStateResponse) { r.AppliedConfigurationIndex++ }, + "not ready": func(r *pb.ProbeMigrationStateResponse) { r.Ready = false }, + "old proof": func(r *pb.ProbeMigrationStateResponse) { r.Ownership.SourceAppliedIndex++ }, + } + for name, change := range cases { + t.Run(name, func(t *testing.T) { + f := newTransferAdmissionFixture(t) + f.snapshot.Members[1].Client = &transferAdmissionClient{probe: func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + resp := &pb.ProbeMigrationStateResponse{Ready: true, AppliedConfigurationIndex: 41, OwnershipProtocol: 2, + OwnershipPartitionProtocol: 1, TransferSizeGuardVersion: 1, Ownership: proto.CloneOf(f.proof)} + change(resp) + return resp, nil + }} + _, err := f.server.ensureSplitTransferAdmission(t.Context(), f.job, 2, f.source, f.control, f.proof) + require.Error(t, err) + require.Empty(t, f.applied) + }) + } +} + +func TestSplitTransferAdmissionRejectsNilResponseAndInvalidGroup(t *testing.T) { + f := newTransferAdmissionFixture(t) + _, err := f.server.ensureSplitTransferAdmission(t.Context(), f.job, 0, f.source, f.control, f.proof) + require.Error(t, err) + f.snapshot.Members[1].Client = &transferAdmissionClient{probe: func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { return nil, nil }} + _, err = f.server.ensureSplitTransferAdmission(t.Context(), f.job, 2, f.source, f.control, f.proof) + require.Error(t, err) + require.Empty(t, f.applied) +} + +func TestSplitTransferAdmissionRejectsMissingPersistedLearnerACK(t *testing.T) { + f := newTransferAdmissionFixture(t) + learner, ok := f.snapshot.Members[1].Client.(*transferAdmissionClient) + require.True(t, ok) + probe := learner.probe + learner.probe = func(req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + resp, err := probe(req) + if req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED { + resp.Ready = false + } + return resp, err + } + witness, err := f.server.ensureSplitTransferAdmission(t.Context(), f.job, 2, f.source, f.control, f.proof) + require.ErrorIs(t, err, errSplitMigrationVoterBarrierIncomplete) + require.True(t, splitJobPhaseErrorRetryable(err)) + require.Nil(t, witness) + require.Len(t, f.applied, 1, "proposal success alone is not persisted all-member evidence") + learner.probe = probe + witness, err = f.server.ensureSplitTransferAdmission(t.Context(), f.job, 2, f.source, f.control, f.proof) + require.NoError(t, err, "catch-up must allow a fresh admission attempt") + require.NoError(t, f.server.revalidateSplitTransferAdmission(t.Context(), witness)) +} + +func TestSplitTransferAdmissionMembershipChangesBeforeApply(t *testing.T) { + f := newTransferAdmissionFixture(t) + calls := 0 + WithSplitMigrationApplyMemberSnapshotFactory(func(context.Context, uint64) (SplitMigrationApplyMemberSnapshot, error) { + calls++ + if calls > 1 { + f.snapshot.Members = f.snapshot.Members[:1] + } + return f.snapshot, nil + })(f.server) + _, err := f.server.ensureSplitTransferAdmission(t.Context(), f.job, 2, f.source, f.control, f.proof) + require.Error(t, err) + require.Empty(t, f.applied) +} + +func TestSplitTransferAdmissionRevalidationUsesFreshMembers(t *testing.T) { + f := newTransferAdmissionFixture(t) + witness, err := f.server.ensureSplitTransferAdmission(t.Context(), f.job, 2, f.source, f.control, f.proof) + require.NoError(t, err) + f.snapshot.Members[1].Client = &transferAdmissionClient{probe: func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + return &pb.ProbeMigrationStateResponse{}, nil + }} + require.Error(t, f.server.revalidateSplitTransferAdmission(t.Context(), witness), "new factory clients must be probed, not cached ACKs") +} + +func TestSplitTransferAdmissionChangedControlsNeedNewWitness(t *testing.T) { + f := newTransferAdmissionFixture(t) + first, err := f.server.ensureSplitTransferAdmission(t.Context(), f.job, 2, f.source, f.control, f.proof) + require.NoError(t, err) + f.control.SourceWriteFence, f.control.SourceReadFence = true, true + f.control.RetentionPinTs++ + second, err := f.server.ensureSplitTransferAdmission(t.Context(), f.job, 2, f.source, f.control, f.proof) + require.NoError(t, err) + require.False(t, first.control.GetSourceWriteFence()) + require.True(t, second.control.GetSourceWriteFence()) + require.NotEqual(t, first.control.GetRetentionPinTs(), second.control.GetRetentionPinTs()) + require.Len(t, f.applied, 2) + // Mutating caller-owned proof/control must not rewrite the first witness. + f.proof.SourceAppliedIndex++ + require.Equal(t, uint64(9), first.ownershipProof.GetSourceAppliedIndex()) +} + +func TestSplitTransferAdmissionRejectsCanceledContext(t *testing.T) { + f := newTransferAdmissionFixture(t) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err := f.server.ensureSplitTransferAdmission(ctx, f.job, 2, f.source, f.control, f.proof) + require.ErrorIs(t, err, context.Canceled) + require.Empty(t, f.applied) + require.Error(t, f.server.revalidateSplitTransferAdmission(t.Context(), nil)) +} diff --git a/adapter/split_job_transfer_pending_test.go b/adapter/split_job_transfer_pending_test.go new file mode 100644 index 000000000..6ef920f6a --- /dev/null +++ b/adapter/split_job_transfer_pending_test.go @@ -0,0 +1,99 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type pendingTransferControlClient struct { + SplitMigrationClient + pending bool + fencedOnly bool + probes int +} + +func (c *pendingTransferControlClient) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest, opts ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + response, err := c.SplitMigrationClient.ProbeMigrationState(ctx, req, opts...) + if err == nil && response != nil && c.pending && (!c.fencedOnly || req.GetSourceWriteFence()) && + req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED && req.GetTransferSizeGuardVersion() != 0 { + c.probes++ + response.Ready = false + } + return response, err +} + +func TestSplitTransferPendingLearnerACKDoesNotExhaustRunnerFailureBudget(t *testing.T) { + server, catalog, job, source := newTransferRuntimePhaseFixture(t) + learner := installPendingTransferLearner(server, job, source) + for range maxSplitJobPhaseFailures + 1 { + err := server.RunSplitJobRunnerOnce(t.Context()) + require.ErrorIs(t, err, errSplitMigrationVoterBarrierIncomplete) + current := loadTransferJob(t, t.Context(), catalog, job.JobID) + require.Equal(t, distribution.SplitJobPhasePlanned, current.Phase) + require.Zero(t, current.SnapshotTS) + require.Empty(t, server.splitJobFailures) + } + require.Equal(t, maxSplitJobPhaseFailures+1, learner.probes) + learner.pending = false + require.NoError(t, server.RunSplitJobRunnerOnce(t.Context())) + current := loadTransferJob(t, t.Context(), catalog, job.JobID) + require.Equal(t, distribution.SplitJobPhaseBackfill, current.Phase) + require.NotZero(t, current.SnapshotTS) +} + +func installPendingTransferLearner(server *DistributionServer, job distribution.SplitJob, source *splitRuntimeClient) *pendingTransferControlClient { + factory := server.splitApplyMemberFactory + learner := &pendingTransferControlClient{SplitMigrationClient: &transferPartitionProtocolPeer{splitRuntimeClient: source}, pending: true} + server.splitApplyMemberFactory = func(ctx context.Context, groupID uint64) (SplitMigrationApplyMemberSnapshot, error) { + snapshot, err := factory(ctx, groupID) + if groupID == job.SourceGroupID { + snapshot.Members = append(snapshot.Members, SplitMigrationVoter{ID: "learner", Address: "learner:1", Client: learner}) + } + return snapshot, err + } + return learner +} + +func TestSplitTransferPendingLearnerACKKeepsFencedJobRunnable(t *testing.T) { + server, catalog, job, source := newTransferRuntimePhaseFixture(t) + learner := installPendingTransferLearner(server, job, source) + learner.fencedOnly = true + for range len(job.BracketProgress)*2 + 3 { + job = loadTransferJob(t, t.Context(), catalog, job.JobID) + if job.Phase == distribution.SplitJobPhaseFence { + break + } + require.NoError(t, server.RunSplitJobRunnerOnce(t.Context())) + } + require.Equal(t, distribution.SplitJobPhaseFence, job.Phase) + for range len(job.BracketProgress)*2 + 3 { + err := server.RunSplitJobRunnerOnce(t.Context()) + if learner.probes > 0 { + require.ErrorIs(t, err, errSplitMigrationVoterBarrierIncomplete) + break + } + require.NoError(t, err) + } + require.Positive(t, learner.probes, "the full preflight must finish before the fenced guard is tested") + for range maxSplitJobPhaseFailures + 1 { + err := server.RunSplitJobRunnerOnce(t.Context()) + require.ErrorIs(t, err, errSplitMigrationVoterBarrierIncomplete) + current := loadTransferJob(t, t.Context(), catalog, job.JobID) + require.Equal(t, distribution.SplitJobPhaseFence, current.Phase) + _, route, err := server.splitJobSourceRoute(t.Context(), current) + require.NoError(t, err) + require.Equal(t, distribution.RouteStateWriteFenced, route.State) + require.Empty(t, server.splitJobFailures) + } + require.GreaterOrEqual(t, learner.probes, maxSplitJobPhaseFailures+2) + learner.pending = false + require.NoError(t, server.RunSplitJobRunnerOnce(t.Context())) + current := loadTransferJob(t, t.Context(), catalog, job.JobID) + require.Equal(t, distribution.SplitJobPhaseDeltaCopy, current.Phase) + require.NotZero(t, current.FenceTS) +} diff --git a/adapter/split_job_transfer_phase.go b/adapter/split_job_transfer_phase.go new file mode 100644 index 000000000..fcc8871ec --- /dev/null +++ b/adapter/split_job_transfer_phase.go @@ -0,0 +1,117 @@ +package adapter + +import ( + "context" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +// Namespace activation and the full same-owner partition ACK precede the +// caller's source ownership acquisition. Initial arm ACKs bind C explicitly; +// resumed ticks collect a fresh exact all-member ownership ACK, never just +// stamp a newly observed configuration onto a previously saved proof. +func (s *DistributionServer) ensureSplitJobTransferControl(ctx context.Context, job distribution.SplitJob, groupID uint64, source SplitMigrationClient, control *pb.TargetStagedReadinessRequest, ownershipACK []byte) (*splitTransferAdmissionWitness, error) { + if source == nil || control == nil { + return nil, errors.New("transfer phase source or control is missing") + } + snapshot, err := s.splitTransferApplyMembers(ctx, groupID) + if err != nil { + return nil, err + } + configuration, err := splitTransferPhaseConfiguration(snapshot.ConfigurationIndex, control, ownershipACK) + if err != nil { + return nil, err + } + state, err := probeSplitSourceOwnershipState(ctx, source, job.JobID, configuration) + if err != nil { + return nil, err + } + if !validSplitTransferPhaseOwnership(state, groupID, configuration) { + return nil, errors.New("transfer phase requires acquired source ownership at the ACK configuration") + } + proof := proto.CloneOf(state.GetOwnership()) + bound := proto.CloneOf(control) + // This temporary witness only collects the fresh ownership ACK. It is not + // returned or used as control evidence until ensure completes guard ACKs. + ack := &splitTransferAdmissionWitness{groupID: groupID, snapshot: snapshot, control: bound, ownershipProof: proof} + if err := probeSplitTransferOwnership(ctx, ack, snapshot.Members); err != nil { + return nil, err + } + bound.RequireConfigurationIndex, bound.ExpectedConfigurationIndex = true, configuration + if bound.ExpectedCutoverVersion == 0 { + bound.ExpectedCutoverVersion = proof.GetSourceTrackerVersion() + } + bound.MinWriteTsExclusive = max(bound.GetMinWriteTsExclusive(), proof.GetMinWriteTsExclusive()) + return s.ensureSplitTransferAdmission(ctx, job, groupID, source, bound, proof) +} + +func splitTransferPhaseConfiguration(configuration uint64, control *pb.TargetStagedReadinessRequest, ownershipACK []byte) (uint64, error) { + if control.GetRequireConfigurationIndex() && control.GetExpectedConfigurationIndex() != configuration { + return 0, errors.New("transfer phase cannot rebind an existing control configuration") + } + if len(ownershipACK) == 0 { + return configuration, nil + } + index, err := splitCleanupCursorConfiguration(ownershipACK) + if err != nil { + return 0, err + } + if index != configuration { + return 0, errors.New("transfer phase ownership ACK configuration changed") + } + return configuration, nil +} + +func validSplitTransferPhaseOwnership(state *pb.ProbeMigrationStateResponse, groupID, configuration uint64) bool { + return state.GetReady() && state.GetAppliedConfigurationIndex() == configuration && !state.GetOwnershipCleared() && + validSplitTransferOwner(groupID, state.GetOwnership()) +} + +func (s *DistributionServer) ensureSplitJobTransferPhase(ctx context.Context, job distribution.SplitJob, source SplitMigrationClient, ownershipACK []byte) (*splitTransferAdmissionWitness, error) { + snapshot, route, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return nil, err + } + if source == nil { + source, err = s.splitJobSourceClient(ctx, job, route.GroupID) + if err != nil { + return nil, err + } + } + control, err := splitTransferPhaseControl(job, route, snapshot.Version) + if err != nil { + return nil, err + } + return s.ensureSplitJobTransferControl(ctx, job, route.GroupID, source, control, ownershipACK) +} + +func splitTransferPhaseControl(job distribution.SplitJob, route distribution.RouteDescriptor, catalogVersion uint64) (*pb.TargetStagedReadinessRequest, error) { + // A zero expected version is filled from the freshly ACKed source tracker, + // not predicted from the root catalog. Full-history preflight must not itself + // introduce a source write fence before all history has been checked. + control := splitMigrationControlRequest(job, route.End, 0, initialMigrationRetentionPinTS, false, false, true, initialMigrationRetentionPinTS) + switch job.Phase { + case distribution.SplitJobPhasePlanned, distribution.SplitJobPhaseBackfill: + return control, nil + case distribution.SplitJobPhaseFence: + if route.State != distribution.RouteStateWriteFenced { + return control, nil + } + control.ExpectedCutoverVersion = catalogVersion + case distribution.SplitJobPhaseDeltaCopy: + if job.FenceCatalogVersion == 0 { + return nil, errors.New("delta transfer requires the committed fence catalog version") + } + control.ExpectedCutoverVersion = job.FenceCatalogVersion + case distribution.SplitJobPhaseNone, distribution.SplitJobPhaseCutover, distribution.SplitJobPhaseCleanup, + distribution.SplitJobPhaseDone, distribution.SplitJobPhaseFailed, distribution.SplitJobPhaseAbandoning, distribution.SplitJobPhaseAbandoned: + return nil, errors.New("transfer admission is not a copy or preflight phase") + default: + return nil, errors.New("unknown transfer admission phase") + } + control.SourceWriteFence, control.MinWriteTsExclusive = true, job.SnapshotTS + return control, nil +} diff --git a/adapter/split_job_transfer_phase_test.go b/adapter/split_job_transfer_phase_test.go new file mode 100644 index 000000000..d6db6e4cd --- /dev/null +++ b/adapter/split_job_transfer_phase_test.go @@ -0,0 +1,195 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" +) + +func TestSplitTransferPhaseMissingFactoryBlocksTimestampAndExport(t *testing.T) { + ctx, server, catalog, job, source := transferPreflightFixture(t) + server.splitApplyMemberFactory = nil + _, err := server.preflightSplitJobTransfer(ctx, job) + require.Error(t, err) + require.Empty(t, source.events) + require.Empty(t, source.exports) + current := loadTransferJob(t, ctx, catalog, job.JobID) + require.Zero(t, current.TransferPreflightTS) + job.Phase, job.SnapshotTS = distribution.SplitJobPhaseBackfill, 10 + err = server.copySplitJobPhase(ctx, job, distribution.SplitJobExportPhaseBackfill, 0, 10) + require.Error(t, err) + require.Empty(t, source.exports) +} + +func installTransferPhaseFixture(t *testing.T, server *DistributionServer, job distribution.SplitJob, source *transferSourceClient) { + t.Helper() + snapshot, route, err := server.splitJobSourceRoute(t.Context(), job) + require.NoError(t, err) + proof, err := splitSourceOwnershipProof(snapshot, job, route.GroupID, "root") + require.NoError(t, err) + proof.SourceAppliedIndex = 21 + source.probeFn = func(req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + resp := &pb.ProbeMigrationStateResponse{Ready: true, AppliedConfigurationIndex: 1, OwnershipProtocol: 2, + OwnershipPartitionProtocol: 1, TransferSizeGuardVersion: 1, OwnershipGeneration: "root"} + if req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED { + resp.Ownership = proto.CloneOf(proof) + } + if req.GetKind() == pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED { + resp.Ready = len(source.controls) != 0 && transferFixtureControlMatches(source.controls[len(source.controls)-1], req) + } + return resp, nil + } + server.splitApplyMemberFactory = func(context.Context, uint64) (SplitMigrationApplyMemberSnapshot, error) { + return SplitMigrationApplyMemberSnapshot{ConfigurationIndex: 1, Members: []SplitMigrationVoter{{ID: "source", Address: "source:1", Client: source}}}, nil + } +} + +func transferFixtureControlMatches(control *pb.TargetStagedReadinessRequest, req *pb.ProbeMigrationStateRequest) bool { + expected := probeMigrationControlRequest(control) + expected.RequireConfigurationIndex, expected.ExpectedConfigurationIndex = req.GetRequireConfigurationIndex(), req.GetExpectedConfigurationIndex() + if req.GetTransferSizeGuardVersion() != 0 { + expected.TransferSizeGuardVersion = control.GetTransferSizeGuardVersion() + } + return proto.Equal(expected, req) +} + +type transferTimestampChangeClient struct { + SplitMigrationClient + onTimestamp func() +} + +func (c *transferTimestampChangeClient) IssueMigrationTimestamp(context.Context, *pb.IssueMigrationTimestampRequest, ...grpc.CallOption) (*pb.IssueMigrationTimestampResponse, error) { + c.onTimestamp() + return &pb.IssueMigrationTimestampResponse{Timestamp: 100, LastCommitTs: 99}, nil +} + +func TestSplitTransferPhaseConfigurationChangeAfterSnapshotDoesNotAdvance(t *testing.T) { + f := newTransferAdmissionFixture(t) + witness, err := f.server.ensureSplitTransferAdmission(t.Context(), f.job, 2, f.source, f.control, f.proof) + require.NoError(t, err) + source := &transferTimestampChangeClient{SplitMigrationClient: f.source, onTimestamp: func() { f.snapshot.ConfigurationIndex++ }} + // No catalog is needed: revalidation must reject before attempting a write. + err = f.server.openSplitJobBackfill(t.Context(), f.job, source, witness) + require.ErrorContains(t, err, "configuration changed") +} + +func TestSplitTransferPhasePreflightGuardPrecedesTimestamp(t *testing.T) { + ctx, server, catalog, job, source := transferPreflightFixture(t) + _, err := server.preflightSplitJobTransfer(ctx, job) + require.NoError(t, err) + require.Equal(t, "guard", source.events[0]) + require.Equal(t, "timestamp", source.events[len(source.events)-1]) + require.Positive(t, loadTransferJob(t, ctx, catalog, job.JobID).TransferPreflightTS) +} + +func newTransferRuntimePhaseFixture(t *testing.T) (*DistributionServer, *distribution.CatalogStore, distribution.SplitJob, *splitRuntimeClient) { + t.Helper() + root := store.NewMVCCStore() + t.Cleanup(func() { require.NoError(t, root.Close()) }) + catalog := distribution.NewCatalogStore(root, distribution.WithCatalogRouteDescriptorV2Writes(true)) + snapshot, err := catalog.Save(t.Context(), 0, []distribution.RouteDescriptor{{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 1}}) + require.NoError(t, err) + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{JobID: 1, SourceRouteID: 1, SplitKey: []byte("m"), TargetGroupID: 2}, snapshot.Routes[0], 1000) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(t.Context(), job)) + server, source, target := newSplitRuntimeRunner(t, root, catalog, job) + // The base has only partition interfaces; its implementation is integrated + // by the parent worker. Stub just that independent static capability here. + // Ownership, actual configuration, guard application and guard ACK remain + // backed by the real source/target stores and FSMs in this focused fixture. + sourcePeer := &transferPartitionProtocolPeer{splitRuntimeClient: source} + targetPeer := &transferPartitionProtocolPeer{splitRuntimeClient: target} + server.splitMigrationClientFactory = func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return sourcePeer, targetPeer, nil + } + server.splitMigrationGroupFactory = func(context.Context, uint64) (SplitMigrationClient, error) { return sourcePeer, nil } + server.splitMigrationVoterFactory = func(_ context.Context, groupID uint64) ([]SplitMigrationVoter, error) { + var peer SplitMigrationClient = targetPeer + if groupID == job.SourceGroupID { + peer = sourcePeer + } + return []SplitMigrationVoter{{ID: "voter", Address: "voter:1", Client: peer}}, nil + } + server.splitApplyMemberFactory = func(context.Context, uint64) (SplitMigrationApplyMemberSnapshot, error) { + return SplitMigrationApplyMemberSnapshot{ConfigurationIndex: 9, Members: []SplitMigrationVoter{{ID: "voter", Address: "voter:1", Client: sourcePeer}}}, nil + } + return server, catalog, job, source +} + +type transferPartitionProtocolPeer struct{ *splitRuntimeClient } + +func (p *transferPartitionProtocolPeer) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest, opts ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + resp, err := p.splitRuntimeClient.ProbeMigrationState(ctx, req, opts...) + if resp != nil { + resp.OwnershipPartitionProtocol = MigrationOwnershipPartitionProtocolVersion + } + return resp, err +} + +func TestSplitTransferPhaseBeginGuardsBeforeLocksAndSnapshot(t *testing.T) { + server, catalog, job, source := newTransferRuntimePhaseFixture(t) + require.NoError(t, server.beginSplitJobBackfill(t.Context(), job)) + require.Equal(t, []string{"control", "guard", "locks", "timestamp"}, source.events) + current := loadTransferJob(t, t.Context(), catalog, job.JobID) + require.Equal(t, distribution.SplitJobPhaseBackfill, current.Phase) + require.Equal(t, uint32(1), source.controls[1].GetTransferSizeGuardVersion()) + require.True(t, source.controls[1].GetRequireConfigurationIndex()) + require.Equal(t, uint64(9), source.controls[1].GetExpectedConfigurationIndex()) +} + +func TestSplitTransferPhaseBeginOldLearnerBlocksLocksAndSnapshot(t *testing.T) { + server, catalog, job, source := newTransferRuntimePhaseFixture(t) + factory := server.splitApplyMemberFactory + server.splitApplyMemberFactory = func(ctx context.Context, groupID uint64) (SplitMigrationApplyMemberSnapshot, error) { + snapshot, err := factory(ctx, groupID) + snapshot.Members = append(snapshot.Members, SplitMigrationVoter{ID: "learner", Address: "learner:1", Client: &splitMigrationClientStub{}}) + return snapshot, err + } + require.Error(t, server.beginSplitJobBackfill(t.Context(), job)) + require.Equal(t, []string{"control"}, source.events) + require.Len(t, source.controls, 1, "plain ownership tracker is not transfer admission evidence") + require.Zero(t, source.controls[0].GetTransferSizeGuardVersion()) + require.Equal(t, distribution.SplitJobPhasePlanned, loadTransferJob(t, t.Context(), catalog, job.JobID).Phase) +} + +func TestSplitTransferPhaseCopyConfigurationChangeAfterGuardBlocksExport(t *testing.T) { + ctx, server, _, job, source := transferPreflightFixture(t) + job.Phase, job.SnapshotTS = distribution.SplitJobPhaseBackfill, 10 + clients := server.splitMigrationClientFactory + members := server.splitApplyMemberFactory + changed, calls := false, 0 + server.splitMigrationClientFactory = func(ctx context.Context, job distribution.SplitJob, groupID uint64) (SplitMigrationClient, SplitMigrationClient, error) { + calls++ + if calls > 1 { + changed = true + } + return clients(ctx, job, groupID) + } + server.splitApplyMemberFactory = func(ctx context.Context, groupID uint64) (SplitMigrationApplyMemberSnapshot, error) { + snapshot, err := members(ctx, groupID) + if changed { + snapshot.ConfigurationIndex++ + } + return snapshot, err + } + err := server.copySplitJobPhase(ctx, job, distribution.SplitJobExportPhaseBackfill, 0, 10) + require.ErrorContains(t, err, "configuration changed") + require.NotEmpty(t, source.controls, "guard was persisted before membership changed") + require.Equal(t, uint32(1), source.controls[0].GetTransferSizeGuardVersion()) + require.Empty(t, source.exports) +} + +func TestSplitTransferPhaseWitnessPreservesInitialOwnershipACKEpoch(t *testing.T) { + f := newTransferAdmissionFixture(t) + // A bound ACK at 40 cannot be relabelled to the observed configuration 41. + ack := encodeSplitVoterAckCursor([]splitVoterAck{{id: "voter", address: "voter:9000", acked: true, configurationIndex: 40}}) + _, err := f.server.ensureSplitJobTransferControl(context.Background(), f.job, 2, f.source, f.control, ack) + require.Error(t, err) + require.Empty(t, f.applied) +} diff --git a/adapter/split_job_transfer_preflight.go b/adapter/split_job_transfer_preflight.go new file mode 100644 index 000000000..2a02c372f --- /dev/null +++ b/adapter/split_job_transfer_preflight.go @@ -0,0 +1,163 @@ +package adapter + +import ( + "bytes" + "context" + "time" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" +) + +const splitTransferPreflightVersion = uint32(1) + +// Each tick validates one bounded export page without importing it. Its cursor +// is independent of copy progress, so a legacy copy cursor is never proof. +func (s *DistributionServer) preflightSplitJobTransfer(ctx context.Context, job distribution.SplitJob) (bool, error) { + _, route, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return false, err + } + brackets, err := distribution.PlanExportBrackets(job.SplitKey, route.End) + if err != nil { + return false, errors.WithStack(err) + } + source, err := s.splitJobSourceClient(ctx, job, route.GroupID) + if err != nil { + return false, err + } + witness, err := s.ensureSplitJobTransferPhase(ctx, job, source, nil) + if err != nil { + return false, err + } + if job.TransferPreflightVersion == 0 { + return false, s.initializeSplitTransferPreflight(ctx, job, source, brackets, witness) + } + if err := validateSplitTransferPreflight(job, brackets); err != nil { + return false, err + } + if job.TransferPreflightDone { + return true, nil + } + for index, progress := range job.TransferPreflightProgress { + if !progress.Done { + return false, s.preflightSplitTransferPage(ctx, job, source, route.End, brackets[index], index, witness) + } + } + next := distribution.CloneSplitJob(job) + next.TransferPreflightDone = true + return false, s.persistSplitTransferPreflight(ctx, job, next, witness) +} + +func (s *DistributionServer) initializeSplitTransferPreflight(ctx context.Context, job distribution.SplitJob, source SplitMigrationClient, brackets []distribution.MigrationBracket, witness *splitTransferAdmissionWitness) error { + issued, err := issueSplitMigrationTimestamp(ctx, source) + if err != nil { + return err + } + if issued.GetTimestamp() == 0 || issued.GetTimestamp() <= issued.GetLastCommitTs() { + return errors.New("invalid migration preflight timestamp") + } + next := distribution.CloneSplitJob(job) + next.TransferPreflightVersion = splitTransferPreflightVersion + next.TransferPreflightTS = issued.GetTimestamp() + next.TransferPreflightProgress = distribution.SplitJobBracketProgressForPlan(brackets, distribution.SplitJobExportPhaseBackfill) + next.TransferPreflightDone = false + return s.persistSplitTransferPreflight(ctx, job, next, witness) +} + +func validateSplitTransferPreflight(job distribution.SplitJob, brackets []distribution.MigrationBracket) error { + if job.TransferPreflightVersion != splitTransferPreflightVersion || job.TransferPreflightTS == 0 || len(job.TransferPreflightProgress) != len(brackets) { + return errors.New("migration preflight lacks a complete versioned plan") + } + for index, progress := range job.TransferPreflightProgress { + bracket := brackets[index] + if progress.BracketID != bracket.BracketID || progress.Family != bracket.Family || + progress.ExportPhase != distribution.SplitJobExportPhaseBackfill || (job.TransferPreflightDone && !progress.Done) { + return errors.New("migration preflight plan or completion witness does not match") + } + } + return nil +} + +func (s *DistributionServer) preflightSplitTransferPage(ctx context.Context, job distribution.SplitJob, source SplitMigrationClient, routeEnd []byte, bracket distribution.MigrationBracket, index int, witness *splitTransferAdmissionWitness) error { + progress := job.TransferPreflightProgress[index] + attemptCtx, cancel := context.WithTimeout(ctx, defaultSplitMigrationAttemptTimeout) + defer cancel() + stream, err := source.ExportRangeVersions(attemptCtx, &pb.ExportRangeVersionsRequest{ + RangeStart: bracket.Start, RangeEnd: bracket.End, MaxCommitTs: job.TransferPreflightTS, + MinCommitTs: 0, Cursor: progress.Cursor, ChunkBytes: defaultSplitMigrationChunkBytes, + RouteStart: job.SplitKey, RouteEnd: routeEnd, MaxScannedBytes: defaultSplitMigrationMaxScannedBytes, + KeyFamily: bracket.Family, ExcludeKnownInternal: bracket.ExcludeKnownInternal, ExcludePrefixes: bracket.ExcludePrefixes, + }) + if err != nil { + return errors.WithStack(migrationSourceExportError(err)) + } + resp, err := stream.Recv() + if err != nil { + return errors.WithStack(migrationSourceExportError(err)) + } + if !resp.GetDone() && (len(resp.GetNextCursor()) == 0 || bytes.Equal(progress.Cursor, resp.GetNextCursor())) { + return errors.New("migration preflight cursor did not progress") + } + if err := checkMigrationExportPageSize(resp); err != nil { + // A local validation error is not an explicit failure from the source. + return err + } + next := distribution.CloneSplitJob(job) + next.TransferPreflightProgress[index].Cursor = distribution.CloneBytes(resp.GetNextCursor()) + next.TransferPreflightProgress[index].Done = resp.GetDone() + if resp.GetDone() { + // Completed families need only their completion witness, not a retained + // full-key cursor for every family in the catalog job record. + next.TransferPreflightProgress[index].Cursor = nil + } + next.TransferPreflightProgress[index].AcceptedRows += uint64(len(resp.GetVersions())) + return s.persistSplitTransferPreflight(ctx, job, next, witness) +} + +func (s *DistributionServer) persistSplitTransferPreflight(ctx context.Context, expected, next distribution.SplitJob, witness *splitTransferAdmissionWitness) error { + if err := s.revalidateSplitTransferAdmission(ctx, witness); err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, expected.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if !distribution.SplitJobsEquivalent(current, expected) { + return current, nil + } + next.UpdatedAtMs = time.Now().UnixMilli() + return next, nil + }) +} + +func (s *DistributionServer) abandonOversizedSplitTransfer(ctx context.Context, job distribution.SplitJob, cause error) error { + if !errors.Is(cause, errMigrationOversizedSourceVersion) { + return cause + } + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if !splitTransferCanAutoAbandon(current.Phase) || current.Phase != job.Phase || current.SnapshotTS != job.SnapshotTS || current.FenceTS != job.FenceTS { + return current, nil + } + next, err := distribution.BeginSplitJobAbandon(current, time.Now().UnixMilli()) + if err != nil { + return current, errors.WithStack(err) + } + next.LastError = errors.Wrapf(cause, "migration transfer failed before cutover in phase %d", current.Phase).Error() + if current.LastError != "" { + const maxPreviousDiagnosticBytes = 512 + previous := current.LastError + if len(previous) > maxPreviousDiagnosticBytes { + previous = previous[:maxPreviousDiagnosticBytes] + } + next.LastError += "; previous: " + previous + } + return next, nil + }) +} + +func splitTransferCanAutoAbandon(phase distribution.SplitJobPhase) bool { + return phase == distribution.SplitJobPhasePlanned || phase == distribution.SplitJobPhaseBackfill || + phase == distribution.SplitJobPhaseFence || phase == distribution.SplitJobPhaseDeltaCopy +} diff --git a/adapter/split_job_transfer_preflight_test.go b/adapter/split_job_transfer_preflight_test.go new file mode 100644 index 000000000..6593534bb --- /dev/null +++ b/adapter/split_job_transfer_preflight_test.go @@ -0,0 +1,308 @@ +package adapter + +import ( + "context" + "io" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +type transferSourceClient struct { + splitMigrationClientStub + exportFn func(context.Context, *pb.ExportRangeVersionsRequest) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) +} + +func (c *transferSourceClient) ApplyTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest, opts ...grpc.CallOption) (*pb.TargetStagedReadinessResponse, error) { + cloned := proto.CloneOf(req) + if len(c.controls) != 0 { + current := c.controls[len(c.controls)-1] + cloned.TransferSizeGuardVersion = max(cloned.GetTransferSizeGuardVersion(), current.GetTransferSizeGuardVersion()) + cloned.SourceWriteFence = cloned.GetSourceWriteFence() || current.GetSourceWriteFence() + cloned.SourceReadFence = cloned.GetSourceReadFence() || current.GetSourceReadFence() + cloned.MinWriteTsExclusive = max(cloned.GetMinWriteTsExclusive(), current.GetMinWriteTsExclusive()) + } + if cloned.GetTransferSizeGuardVersion() != 0 { + c.events = append(c.events, "guard") + } + return c.splitMigrationClientStub.ApplyTargetStagedReadiness(ctx, cloned, opts...) +} + +func (c *transferSourceClient) ProbeMigrationLocks(ctx context.Context, req *pb.ProbeMigrationLocksRequest, opts ...grpc.CallOption) (*pb.ProbeMigrationLocksResponse, error) { + c.events = append(c.events, "locks") + return c.splitMigrationClientStub.ProbeMigrationLocks(ctx, req, opts...) +} + +func (c *transferSourceClient) ExportRangeVersions(ctx context.Context, req *pb.ExportRangeVersionsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) { + if c.exportFn != nil { + return c.exportFn(ctx, req) + } + return c.splitMigrationClientStub.ExportRangeVersions(ctx, req, opts...) +} + +func transferPreflightFixture(t *testing.T) (context.Context, *DistributionServer, *distribution.CatalogStore, distribution.SplitJob, *transferSourceClient) { + t.Helper() + ctx, server, catalog, job, _, target := importReplayFixture(t) + next := distribution.CloneSplitJob(job) + next.Phase = distribution.SplitJobPhaseFence + next.SnapshotTS = 10 + require.NoError(t, catalog.SaveSplitJob(ctx, job, next)) + source := &transferSourceClient{} + server.splitMigrationClientFactory = func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, target, nil + } + installTransferPhaseFixture(t, server, next, source) + return ctx, server, catalog, next, source +} + +func loadTransferJob(t *testing.T, ctx context.Context, catalog *distribution.CatalogStore, id uint64) distribution.SplitJob { + t.Helper() + job, found, err := catalog.SplitJob(ctx, id) + require.NoError(t, err) + require.True(t, found) + return job +} + +func TestTransferPreflightPersistsEveryFamilyBeforeFence(t *testing.T) { + ctx, server, catalog, job, source := transferPreflightFixture(t) + // Old copy cursors, even marked done, do not attest a fresh full-history scan. + for i := range job.BracketProgress { + job.BracketProgress[i].Done = true + } + require.NoError(t, server.updateSplitJobViaCoordinator(ctx, job.JobID, func(distribution.SplitJob) (distribution.SplitJob, error) { return job, nil })) + plan, err := distribution.PlanExportBrackets(job.SplitKey, []byte("z")) + require.NoError(t, err) + for range len(plan) + 2 { + require.NoError(t, server.finalizeSplitJobFence(ctx, job)) + job = loadTransferJob(t, ctx, catalog, job.JobID) + raw, encodeErr := distribution.EncodeSplitJob(job) + require.NoError(t, encodeErr) + job, err = distribution.DecodeSplitJob(raw) + require.NoError(t, err) + snapshot, snapshotErr := catalog.Snapshot(ctx) + require.NoError(t, snapshotErr) + require.Len(t, snapshot.Routes, 1, "no catalog fence before every preflight bracket finishes") + } + require.True(t, job.TransferPreflightDone) + require.Len(t, source.exports, len(plan)) + for index, req := range source.exports { + require.Equal(t, plan[index].Family, req.GetKeyFamily()) + require.Equal(t, uint64(0), req.GetMinCommitTs(), "include retained historical versions") + require.Equal(t, job.TransferPreflightTS, req.GetMaxCommitTs()) + require.Equal(t, uint32(defaultSplitMigrationChunkBytes), req.GetChunkBytes()) + } + require.NotEmpty(t, source.controls) + for _, control := range source.controls { + require.False(t, control.GetSourceWriteFence(), "preflight must not install a source write fence") + require.Equal(t, uint32(1), control.GetTransferSizeGuardVersion()) + } + require.NoError(t, server.finalizeSplitJobFence(ctx, job)) + job = loadTransferJob(t, ctx, catalog, job.JobID) + require.Equal(t, distribution.SplitJobPhaseDeltaCopy, job.Phase) +} + +func TestTransferPreflightRequiresExplicitDoneAndProgress(t *testing.T) { + for _, response := range []*pb.ExportRangeVersionsResponse{nil, {}, {NextCursor: []byte("resume")}} { + ctx, server, catalog, job, source := transferPreflightFixture(t) + require.NoError(t, server.finalizeSplitJobFence(ctx, job)) + job = loadTransferJob(t, ctx, catalog, job.JobID) + source.exportFn = func(context.Context, *pb.ExportRangeVersionsRequest) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) { + var responses []*pb.ExportRangeVersionsResponse + if response != nil { + responses = append(responses, response) + } + return &splitMigrationStreamStub{responses: responses}, nil + } + err := server.finalizeSplitJobFence(ctx, job) + switch { + case response == nil: + require.ErrorIs(t, err, io.EOF) + case len(response.GetNextCursor()) == 0: + require.ErrorContains(t, err, "did not progress") + default: + require.NoError(t, err) + } + current := loadTransferJob(t, ctx, catalog, job.JobID) + require.False(t, current.TransferPreflightDone) + require.False(t, current.TransferPreflightProgress[0].Done) + } +} + +func TestTransferPreflightCannotTrustIncompleteVersionedEvidence(t *testing.T) { + job := splitJobForSourceGroupTest(t) + plan, err := distribution.PlanExportBrackets(job.SplitKey, []byte("z")) + require.NoError(t, err) + job.TransferPreflightVersion = splitTransferPreflightVersion + job.TransferPreflightTS = 100 + job.TransferPreflightProgress = distribution.SplitJobBracketProgressForPlan(plan, distribution.SplitJobExportPhaseBackfill) + require.NoError(t, validateSplitTransferPreflight(job, plan)) + missing := distribution.CloneSplitJob(job) + missing.TransferPreflightProgress = missing.TransferPreflightProgress[1:] + require.Error(t, validateSplitTransferPreflight(missing, plan)) + unknown := distribution.CloneSplitJob(job) + unknown.TransferPreflightVersion++ + require.Error(t, validateSplitTransferPreflight(unknown, plan)) + unproven := distribution.CloneSplitJob(job) + unproven.TransferPreflightDone = true + require.Error(t, validateSplitTransferPreflight(unproven, plan)) +} + +func TestTransferOversizeAutoAbandonRetainsPhaseCursorAndHistory(t *testing.T) { + for _, phase := range []distribution.SplitJobPhase{distribution.SplitJobPhasePlanned, distribution.SplitJobPhaseBackfill, distribution.SplitJobPhaseFence, distribution.SplitJobPhaseDeltaCopy} { + ctx, server, catalog, job, _ := transferPreflightFixture(t) + job.Phase = phase + job.Cursor = []byte("saved-copy-cursor") + require.NoError(t, server.updateSplitJobViaCoordinator(ctx, job.JobID, func(distribution.SplitJob) (distribution.SplitJob, error) { return job, nil })) + cause := errors.Mark(status.Error(codes.ResourceExhausted, "typed source oversize"), errMigrationOversizedSourceVersion) + require.NoError(t, server.abandonOversizedSplitTransfer(ctx, job, cause)) + abandoning := loadTransferJob(t, ctx, catalog, job.JobID) + require.Equal(t, distribution.SplitJobPhaseAbandoning, abandoning.Phase) + require.Equal(t, phase, abandoning.AbandonFromPhase) + require.Equal(t, job.Cursor, abandoning.Cursor) + require.Contains(t, abandoning.LastError, "typed source oversize") + failed, err := distribution.FailSplitJobState(abandoning, "cleanup denied", 20) + require.NoError(t, err) + require.Equal(t, abandoning.LastError, failed.LastError) + retried, err := distribution.RetrySplitJobState(failed, 21) + require.NoError(t, err) + require.Equal(t, abandoning.LastError, retried.LastError) + require.NoError(t, server.finishSplitJobHistory(ctx, abandoning, distribution.SplitJobPhaseAbandoned)) + history, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseAbandoned, history.Phase) + require.Equal(t, abandoning.LastError, history.LastError) + require.Equal(t, job.Cursor, history.Cursor) + } +} + +func TestTransferAutoAbandonCannotCrossCutoverOrCanceledCAS(t *testing.T) { + ctx, server, catalog, job, _ := transferPreflightFixture(t) + canceled, cancel := context.WithCancel(ctx) + cancel() + cause := errors.Mark(status.Error(codes.ResourceExhausted, "oversize"), errMigrationOversizedSourceVersion) + require.Error(t, server.abandonOversizedSplitTransfer(canceled, job, cause)) + require.Equal(t, job.Phase, loadTransferJob(t, ctx, catalog, job.JobID).Phase) + for _, phase := range []distribution.SplitJobPhase{distribution.SplitJobPhaseCutover, distribution.SplitJobPhaseCleanup, distribution.SplitJobPhaseDone} { + require.NoError(t, server.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + current.Phase = phase + return current, nil + })) + require.NoError(t, server.abandonOversizedSplitTransfer(ctx, job, cause)) + require.Equal(t, phase, loadTransferJob(t, ctx, catalog, job.JobID).Phase) + } +} + +func TestTransferStaleFenceCannotRunAfterAbandon(t *testing.T) { + ctx, server, catalog, job, _ := transferPreflightFixture(t) + require.NoError(t, server.abandonOversizedSplitTransfer(ctx, job, errors.Mark(errors.New("oversize"), errMigrationOversizedSourceVersion))) + snapshot, err := catalog.Snapshot(ctx) + require.NoError(t, err) + _, _, err = server.ensureSplitJobCatalogFence(ctx, job, snapshot, snapshot.Routes[0]) + require.ErrorIs(t, err, store.ErrWriteConflict) + snapshot, err = catalog.Snapshot(ctx) + require.NoError(t, err) + require.Len(t, snapshot.Routes, 1) +} + +func TestTransferFenceTransactionDependsOnLiveJob(t *testing.T) { + ctx, server, catalog, job, _ := transferPreflightFixture(t) + coordinator, ok := server.coordinator.(*distributionCoordinatorStub) + require.True(t, ok) + coordinator.beforeApply = func(ctx context.Context, st store.MVCCStore) error { + next, err := distribution.BeginSplitJobAbandon(job, 900) + if err != nil { + return err + } + encoded, err := distribution.EncodeSplitJob(next) + if err != nil { + return err + } + return st.PutAt(ctx, distribution.CatalogSplitJobKey(job.JobID), encoded, coordinator.lastCommitTS+1, 0) + } + snapshot, err := catalog.Snapshot(ctx) + require.NoError(t, err) + _, _, err = server.ensureSplitJobCatalogFence(ctx, job, snapshot, snapshot.Routes[0]) + require.Error(t, err) + require.Contains(t, coordinator.lastReadKeys, distribution.CatalogSplitJobKey(job.JobID)) + snapshot, err = catalog.Snapshot(ctx) + require.NoError(t, err) + require.Len(t, snapshot.Routes, 1, "ABANDON won the transaction; no stale fence can be installed") +} + +func TestTransferPreflightFindsHistoricalOversizeBeforeAnyFence(t *testing.T) { + ctx, server, catalog, job, source := transferPreflightFixture(t) + data := store.NewMVCCStore() + t.Cleanup(func() { _ = data.Close() }) + require.NoError(t, data.PutAt(ctx, []byte("n"), make([]byte, store.MigrationPagePayloadBudget+(1<<19)), 5, 0)) + require.NoError(t, data.PutAt(ctx, []byte("n"), []byte("small latest"), 9, 0)) + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(data)) + source.exportFn = func(ctx context.Context, req *pb.ExportRangeVersionsRequest) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) { + capture := &captureExportRangeVersionsStream{ctx: ctx} + if err := internal.ExportRangeVersions(req, capture); err != nil { + return nil, err + } + return &splitMigrationStreamStub{responses: capture.responses}, nil + } + for range len(job.BracketProgress) + 3 { + err := server.runSplitJobAttempt(ctx, job) + job = loadTransferJob(t, ctx, catalog, job.JobID) + if job.Phase == distribution.SplitJobPhaseAbandoning { + require.True(t, errors.Is(err, errMigrationOversizedSourceVersion)) + break + } + require.NoError(t, err) + } + require.Equal(t, distribution.SplitJobPhaseAbandoning, job.Phase) + snapshot, err := catalog.Snapshot(ctx) + require.NoError(t, err) + require.Len(t, snapshot.Routes, 1) + latest, err := data.GetAt(ctx, []byte("n"), 100) + require.NoError(t, err) + require.Equal(t, []byte("small latest"), latest) +} + +func TestTransferPreflightRaceAfterCompletionAutoAbandonsDelta(t *testing.T) { + ctx, server, catalog, job, source := transferPreflightFixture(t) + data := store.NewMVCCStore() + t.Cleanup(func() { _ = data.Close() }) + require.NoError(t, data.PutAt(ctx, []byte("n"), []byte("small"), 5, 0)) + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(data)) + source.exportFn = func(ctx context.Context, req *pb.ExportRangeVersionsRequest) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) { + capture := &captureExportRangeVersionsStream{ctx: ctx} + if err := internal.ExportRangeVersions(req, capture); err != nil { + return nil, err + } + return &splitMigrationStreamStub{responses: capture.responses}, nil + } + for range len(job.BracketProgress) + 3 { + require.NoError(t, server.finalizeSplitJobFence(ctx, job)) + job = loadTransferJob(t, ctx, catalog, job.JobID) + if job.TransferPreflightDone { + break + } + } + require.True(t, job.TransferPreflightDone) + require.Equal(t, distribution.SplitJobPhaseFence, job.Phase) + // Inject an old authenticated prepared resolution after preflight. The guard + // must not impose a retroactive limit; delta export still compensates it. + require.NoError(t, data.PutAt(ctx, []byte("n"), make([]byte, store.MigrationPagePayloadBudget+(1<<19)), 150, 0)) + source.nextTS = 200 + require.NoError(t, server.finalizeSplitJobFence(ctx, job)) + job = loadTransferJob(t, ctx, catalog, job.JobID) + require.Equal(t, distribution.SplitJobPhaseDeltaCopy, job.Phase) + err := server.runSplitJobAttempt(ctx, job) + require.True(t, errors.Is(err, errMigrationOversizedSourceVersion)) + abandoning := loadTransferJob(t, ctx, catalog, job.JobID) + require.Equal(t, distribution.SplitJobPhaseAbandoning, abandoning.Phase) + require.Equal(t, distribution.SplitJobPhaseDeltaCopy, abandoning.AbandonFromPhase) + require.False(t, abandoning.TargetPromotionDone) +} diff --git a/adapter/split_job_voter_ack_test.go b/adapter/split_job_voter_ack_test.go new file mode 100644 index 000000000..4fa592f98 --- /dev/null +++ b/adapter/split_job_voter_ack_test.go @@ -0,0 +1,131 @@ +package adapter + +import ( + "context" + "errors" + "testing" + "time" + + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +func TestSplitMigrationVoterBarrierReopensForMembershipChanges(t *testing.T) { + t.Parallel() + + ready := func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + return &pb.ProbeMigrationStateResponse{Ready: true}, nil + } + blocked := func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + return &pb.ProbeMigrationStateResponse{}, nil + } + n1 := &splitMigrationClientStub{probeFn: ready} + n2 := &splitMigrationClientStub{probeFn: ready} + n3 := &splitMigrationClientStub{probeFn: blocked} + voters := []SplitMigrationVoter{ + {ID: "n1", Address: "n1:50051", Client: n1}, + {ID: "n2", Address: "n2:50051", Client: n2}, + } + server := &DistributionServer{splitMigrationVoterFactory: func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return voters, nil + }} + req := &pb.ProbeMigrationStateRequest{JobId: 7, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED} + + cursor, complete, err := server.syncSplitMigrationVoterBarrier(context.Background(), 1, nil, nil, req) + require.NoError(t, err) + require.True(t, complete) + acks, err := decodeSplitVoterAckCursor(cursor) + require.NoError(t, err) + require.Len(t, acks, 2) + + voters = append(voters, SplitMigrationVoter{ID: "n3", Address: "n3:50051", Client: n3}) + cursor, complete, err = server.syncSplitMigrationVoterBarrier(context.Background(), 1, cursor, nil, req) + require.NoError(t, err) + require.False(t, complete) + acks, err = decodeSplitVoterAckCursor(cursor) + require.NoError(t, err) + require.Len(t, acks, 3) + require.False(t, acks[2].acked) + + n3.probeFn = ready + _, complete, err = server.syncSplitMigrationVoterBarrier(context.Background(), 1, cursor, nil, req) + require.NoError(t, err) + require.True(t, complete) + + n2.probeFn = blocked + voters[1].Address = "n2:50052" + _, complete, err = server.syncSplitMigrationVoterBarrier(context.Background(), 1, cursor, nil, req) + require.NoError(t, err) + require.False(t, complete, "an address change must be treated as a new voter endpoint") +} + +func TestSplitMigrationVoterBarrierClearsAckWhenFreshProbeFails(t *testing.T) { + t.Parallel() + + client := &splitMigrationClientStub{probeFn: func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + return &pb.ProbeMigrationStateResponse{Ready: true}, nil + }} + server := &DistributionServer{splitMigrationVoterFactory: func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{{ID: "n1", Address: "n1:50051", Client: client}}, nil + }} + req := &pb.ProbeMigrationStateRequest{JobId: 7, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED} + + cursor, complete, err := server.syncSplitMigrationVoterBarrier(context.Background(), 1, nil, nil, req) + require.NoError(t, err) + require.True(t, complete) + + client.probeFn = func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + return nil, errors.New("connection refused") + } + cursor, complete, err = server.syncSplitMigrationVoterBarrier(context.Background(), 1, cursor, nil, req) + require.NoError(t, err) + require.False(t, complete, "a saved ack must not survive a failed fresh probe") + acks, err := decodeSplitVoterAckCursor(cursor) + require.NoError(t, err) + require.Len(t, acks, 1) + require.False(t, acks[0].acked) +} + +func TestSplitVoterAckCursorRejectsCorruption(t *testing.T) { + t.Parallel() + + _, err := decodeSplitVoterAckCursor([]byte{99}) + require.Error(t, err) + _, err = decodeSplitVoterAckCursor([]byte{splitVoterAckCursorVersion, 1, 4, 'n'}) + require.Error(t, err) +} + +type stalledSplitVoterClient struct { + splitMigrationClientStub +} + +func (*stalledSplitVoterClient) ProbeMigrationState(ctx context.Context, _ *pb.ProbeMigrationStateRequest, _ ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + <-ctx.Done() + return nil, ctx.Err() +} + +func TestSplitMigrationVoterBarrierBoundsStalledProbe(t *testing.T) { + t.Parallel() + healthy := &splitMigrationClientStub{} + server := &DistributionServer{splitMigrationVoterFactory: func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return []SplitMigrationVoter{ + {ID: "n1", Address: "n1:50051", Client: &stalledSplitVoterClient{}}, + {ID: "n2", Address: "n2:50051", Client: healthy}, + }, nil + }} + ctx, cancel := context.WithTimeout(context.Background(), 10*defaultSplitMigrationVoterProbeTimeout) + defer cancel() + start := time.Now() + cursor, complete, err := server.syncSplitMigrationVoterBarrier(ctx, 1, nil, nil, &pb.ProbeMigrationStateRequest{JobId: 7}) + require.NoError(t, err) + require.NoError(t, ctx.Err(), "a voter must not consume the runner's parent deadline") + require.Less(t, time.Since(start), 5*defaultSplitMigrationVoterProbeTimeout) + require.False(t, complete) + acks, err := decodeSplitVoterAckCursor(cursor) + require.NoError(t, err) + require.Len(t, acks, 2) + require.False(t, acks[0].acked) + require.True(t, acks[1].acked) + require.Len(t, healthy.probes, 1) +} diff --git a/adapter/split_job_working_source.go b/adapter/split_job_working_source.go new file mode 100644 index 000000000..51a67efb2 --- /dev/null +++ b/adapter/split_job_working_source.go @@ -0,0 +1,206 @@ +package adapter + +import ( + "bytes" + "context" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +// sourceLookupRouteID is geometry, not lineage. SourceRouteID remains the +// original parent in durable job history and in target ancestry checks. +func sourceLookupRouteID(job distribution.SplitJob) uint64 { + if job.WorkingSourceRouteID != 0 { + return job.WorkingSourceRouteID + } + return job.SourceRouteID +} + +func (s *DistributionServer) prepareSplitJobWorkingSource(ctx context.Context, job distribution.SplitJob) (distribution.SplitJob, error) { + if !s.MigrationPartitionRuntimeConfigured(job.SourceGroupID) { + if s.ownershipRefresh != nil { + return job, errors.WithStack(store.ErrNotSupported) + } + return job, nil + } + if err := s.ensureSplitJobNamespaceAdmission(ctx, job); err != nil { + return job, err + } + if err := s.prepareSplitTargetGeometry(ctx, job); err != nil { + return job, err + } + updated, err := s.ensureSplitWorkingSource(ctx, job) + if err != nil { + return job, err + } + if err := s.reconcileMigrationPartitions(ctx, updated.SourceGroupID); err != nil { + return updated, err + } + // A move-back target may retain an older NONOWNER interval covering the + // whole parent. Partition it without changing its role before a guarded + // acquisition can replace only the new moving child. + return updated, s.reconcileMigrationPartitions(ctx, updated.TargetGroupID) +} + +func (s *DistributionServer) prepareSplitTargetGeometry(ctx context.Context, job distribution.SplitJob) error { + if job.WorkingSourceRouteID != 0 { + return nil + } + err := s.reconcileMigrationPartitions(ctx, job.TargetGroupID) + // Replay retained manifests before sizing the future control. An exact + // historical NONOWNER may still await the not-yet-committed working split; + // that is not an ACK. The size preflight and post-commit target barrier + // independently validate its complete priors before any acquisition. + if errors.Is(err, ErrMigrationNegativePartitionDeferred) { + return nil + } + return err +} + +func (s *DistributionServer) ensureSplitWorkingSource(ctx context.Context, job distribution.SplitJob) (distribution.SplitJob, error) { + if job.WorkingSourceRouteID != 0 { + return s.recoverSplitWorkingSource(ctx, job) + } + snapshot, left, right, err := s.planWorkingSourceSplit(ctx, job) + if err != nil { + return job, err + } + groups, err := s.prepareSplitRangePartitions(ctx, snapshot, job.SourceRouteID) + if err != nil { + return job, err + } + read, err := kv.BeginReadTimestampThrough(ctx, s.coordinator, snapshot.ReadTS, "split working source: catalog transaction") + if err != nil { + return job, errors.WithStack(err) + } + saved, err := s.saveSplitPartitionViaCoordinator(ctx, read, snapshot.Version, job.SourceRouteID, + [][]byte{distribution.CatalogSplitJobKey(job.JobID)}, left, right, &job) + if err != nil { + return job, err + } + if err := s.reconcileSplitRangePartitions(ctx, groups); err != nil { + return job, err + } + if err := s.applyEngineSnapshot(saved); err != nil { + return job, err + } + return s.reloadSplitWorkingSourceAt(ctx, job, right.RouteID, saved.ReadTS) +} + +func (s *DistributionServer) recoverSplitWorkingSource(ctx context.Context, job distribution.SplitJob) (distribution.SplitJob, error) { + snapshot, err := s.loadCommittedMigrationCatalog(ctx) + if err != nil { + return job, err + } + route, found := findRouteByID(snapshot.Routes, job.WorkingSourceRouteID) + if !found || !workingSourceFenceRouteMatches(job, route) || route.State != distribution.RouteStateActive || route.ParentRouteID != job.SourceRouteID { + return job, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + // The route ID proves only the root transaction. A previous attempt may + // have stopped before any disjoint owner's new-version ACK completed. + if _, err := s.prepareSplitRangePartitions(ctx, snapshot, route.RouteID); err != nil { + return job, err + } + return job, s.applyEngineSnapshot(snapshot) +} + +func (s *DistributionServer) reloadSplitWorkingSourceAt(ctx context.Context, job distribution.SplitJob, routeID, readTS uint64) (distribution.SplitJob, error) { + updated, exists, err := s.catalog.SplitJobAt(ctx, job.JobID, readTS) + if err != nil { + return job, errors.WithStack(err) + } + if !exists || updated.WorkingSourceRouteID != routeID { + return job, errors.WithStack(distribution.ErrCatalogSplitJobConflict) + } + return updated, nil +} + +func (s *DistributionServer) planWorkingSourceSplit(ctx context.Context, job distribution.SplitJob) (distribution.CatalogSnapshot, distribution.RouteDescriptor, distribution.RouteDescriptor, error) { + snapshot, err := s.loadCommittedMigrationCatalog(ctx) + if err != nil { + return snapshot, distribution.RouteDescriptor{}, distribution.RouteDescriptor{}, err + } + parent, found := findRouteByID(snapshot.Routes, job.SourceRouteID) + if !found || parent.GroupID != job.SourceGroupID || parent.State != distribution.RouteStateActive || parent.StagedVisibilityActive { + return snapshot, distribution.RouteDescriptor{}, distribution.RouteDescriptor{}, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + if err := validateSplitKey(parent, job.SplitKey); err != nil { + return snapshot, distribution.RouteDescriptor{}, distribution.RouteDescriptor{}, err + } + leftID, rightID, err := s.allocateChildRouteIDs(ctx, snapshot.ReadTS, snapshot.Routes) + if err != nil { + return snapshot, distribution.RouteDescriptor{}, distribution.RouteDescriptor{}, err + } + left, right := splitCatalogRoutes(parent, job.SplitKey, leftID, rightID, 0) + return snapshot, left, right, nil +} + +func (s *DistributionServer) fenceSplitWorkingSource(ctx context.Context, job distribution.SplitJob, + snapshot distribution.CatalogSnapshot, route distribution.RouteDescriptor, +) (distribution.CatalogSnapshot, distribution.RouteDescriptor, error) { + if !workingSourceFenceRouteMatches(job, route) { + return snapshot, route, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + if route.State == distribution.RouteStateWriteFenced { + return snapshot, route, nil + } + if route.State != distribution.RouteStateActive { + return snapshot, route, errors.WithStack(store.ErrWriteConflict) + } + if err := s.verifyWorkingSourceFenceJob(ctx, job, snapshot.ReadTS); err != nil { + return snapshot, route, err + } + return s.commitWorkingSourceFence(ctx, job, snapshot, route) +} + +func workingSourceFenceRouteMatches(job distribution.SplitJob, route distribution.RouteDescriptor) bool { + return route.RouteID == job.WorkingSourceRouteID && bytes.Equal(route.Start, job.SplitKey) && + route.GroupID == job.SourceGroupID && !route.StagedVisibilityActive +} + +func (s *DistributionServer) verifyWorkingSourceFenceJob(ctx context.Context, job distribution.SplitJob, readTS uint64) error { + current, found, err := s.catalog.SplitJobAt(ctx, job.JobID, readTS) + if err != nil { + return errors.WithStack(err) + } + if !found || current.Phase != distribution.SplitJobPhaseFence || !distribution.SplitJobsEquivalent(current, job) { + return errors.WithStack(distribution.ErrCatalogSplitJobConflict) + } + return nil +} + +func (s *DistributionServer) commitWorkingSourceFence(ctx context.Context, job distribution.SplitJob, snapshot distribution.CatalogSnapshot, route distribution.RouteDescriptor) (distribution.CatalogSnapshot, distribution.RouteDescriptor, error) { + nextVersion, _, err := nextCatalogSplitIDs(snapshot.Version, route.RouteID) + if err != nil { + return snapshot, route, err + } + route.State = distribution.RouteStateWriteFenced + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(route, s.catalog.AllowsRouteDescriptorV2Writes()) + if err != nil { + return snapshot, route, errors.WithStack(err) + } + read, err := kv.BeginReadTimestampThrough(ctx, s.coordinator, snapshot.ReadTS, "split working source: fence transaction") + if err != nil { + return snapshot, route, errors.WithStack(err) + } + routeKey, versionKey, jobKey := distribution.CatalogRouteKey(route.RouteID), distribution.CatalogVersionKey(), distribution.CatalogSplitJobKey(job.JobID) + resp, err := kv.DispatchWithReadTimestamp(read.WithDispatchVoucher(ctx), s.coordinator, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{{Op: kv.Put, Key: routeKey, Value: encoded}, + {Op: kv.Put, Key: versionKey, Value: distribution.EncodeCatalogVersion(nextVersion)}}, + IsTxn: true, StartTS: read.Timestamp(), ReadKeys: [][]byte{routeKey, versionKey, jobKey}, + }) + if err != nil { + return snapshot, route, errors.WithStack(err) + } + if resp == nil || resp.CommitTS == 0 { + return snapshot, route, errors.New("working source fence commit timestamp missing") + } + updated, err := s.loadCatalogSnapshotAtVersion(ctx, resp.CommitTS, nextVersion) + if err != nil { + return snapshot, route, err + } + return updated, route, s.applyEngineSnapshot(updated) +} diff --git a/adapter/split_migration_apply_members.go b/adapter/split_migration_apply_members.go new file mode 100644 index 000000000..5c44959f6 --- /dev/null +++ b/adapter/split_migration_apply_members.go @@ -0,0 +1,16 @@ +package adapter + +import "context" + +// SplitMigrationApplyMemberSnapshot binds every configured apply member, +// including learners, to an actual applied configuration epoch. +type SplitMigrationApplyMemberSnapshot struct { + ConfigurationIndex uint64 + Members []SplitMigrationVoter +} + +type SplitMigrationApplyMemberSnapshotFactory func(context.Context, uint64) (SplitMigrationApplyMemberSnapshot, error) + +func WithSplitMigrationApplyMemberSnapshotFactory(factory SplitMigrationApplyMemberSnapshotFactory) DistributionServerOption { + return func(s *DistributionServer) { s.splitApplyMemberFactory = factory } +} diff --git a/adapter/split_migration_auth.go b/adapter/split_migration_auth.go new file mode 100644 index 000000000..5744ed9ca --- /dev/null +++ b/adapter/split_migration_auth.go @@ -0,0 +1,152 @@ +package adapter + +import ( + "bytes" + "context" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +func splitMigrationMutatingMethod(method string) bool { + switch method { + case pb.Distribution_SplitRange_FullMethodName, + pb.Distribution_StartSplitMigration_FullMethodName, + pb.Distribution_AbandonSplitJob_FullMethodName, + pb.Distribution_RetrySplitJob_FullMethodName, + pb.Internal_ApplyTargetStagedReadiness_FullMethodName, + pb.Internal_ImportRangeVersions_FullMethodName, + pb.Internal_PromoteStagedVersions_FullMethodName, + pb.Internal_CleanupMigration_FullMethodName: + return true + default: + return false + } +} + +func splitMigrationProtectedMethod(method string) bool { + switch method { + case pb.Internal_ExportRangeVersions_FullMethodName, + pb.Distribution_AttestMigrationOwnership_FullMethodName, + pb.Internal_ProbeMigrationLocks_FullMethodName, + pb.Internal_ProbeMigrationState_FullMethodName, + pb.Internal_IssueMigrationTimestamp_FullMethodName: + return true + default: + return splitMigrationMutatingMethod(method) + } +} + +func splitMigrationAuthenticatedMethod(method string, peerAuthEnforced bool) bool { + // Forward can write arbitrary system/transaction metadata. In a migration + // cluster it is a privileged peer RPC, not an alternative public write API. + return splitMigrationProtectedMethod(method) || (peerAuthEnforced && method == pb.Internal_Forward_FullMethodName) +} + +// SplitMigrationTokenAuth protects catalog splits, migration control and raw MVCC exports. +// Even a proof-free SplitRange advances the global catalog and needs operator authority. +// Without a configured token, only an explicit insecure opt-in enables access. +// Peer enforcement follows a send-first rollout; the runner must stay paused +// until enforcement is enabled (except in explicit isolated-test no-auth mode). +func SplitMigrationTokenAuth(token string, insecureNoAuth, peerAuthEnforced bool) (grpc.UnaryServerInterceptor, grpc.StreamServerInterceptor) { + if token == "" && insecureNoAuth { + return nil, nil + } + authUnary, authStream := bearerTokenAuth(token, func(method string) bool { + return splitMigrationAuthenticatedMethod(method, peerAuthEnforced) + }, "split migration controller") + controlAuth, _ := bearerTokenAuth(token, func(string) bool { return true }, "split migration peer") + unary := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + if info.FullMethod == pb.Internal_Forward_FullMethodName && splitForwardWritesJobControl(req) { + if err := checkSplitMigrationTransport(ctx, info.FullMethod, token, true); err != nil { + return nil, err + } + return controlAuth(ctx, req, info, handler) + } + if err := checkSplitMigrationTransport(ctx, info.FullMethod, token, peerAuthEnforced); err != nil { + return nil, err + } + if authUnary != nil { + return authUnary(ctx, req, info, handler) + } + return handler(ctx, req) + } + stream := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + if err := checkSplitMigrationTransport(ss.Context(), info.FullMethod, token, peerAuthEnforced); err != nil { + return err + } + if authStream != nil { + return authStream(srv, ss, info, handler) + } + return handler(srv, ss) + } + return unary, stream +} + +// Job authority and typed migration metadata must never be planted during +// send-first rollout. Ordinary legacy peer transaction writes remain compatible. +func splitForwardWritesJobControl(req any) bool { + forward, ok := req.(*pb.ForwardRequest) + if !ok { + return false + } + for _, request := range forward.GetRequests() { + for _, mutation := range request.GetMutations() { + if splitJobControlMutation(mutation) { + return true + } + } + } + return false +} + +func splitJobControlMutation(mutation *pb.Mutation) bool { + key := mutation.GetKey() + embedded := kv.ExtractTxnUserKey(key) + if splitMigrationControlKey(key) || splitMigrationControlKey(embedded) { + return true + } + if mutation.GetOp() != pb.Op_DEL_PREFIX { + return false + } + return distribution.MigrationOnlyControlPrefixIntersects(key) || + distribution.MigrationOnlyControlPrefixIntersects(embedded) || + splitPartitionControlPrefix(key, embedded) || + bytes.HasPrefix(distribution.CatalogSplitJobKey(0), key) || + bytes.HasPrefix(distribution.CatalogSplitJobHistoryKey(0, 0), key) || + bytes.HasPrefix(distribution.CatalogNextSplitJobIDKey(), key) +} + +func splitMigrationControlKey(key []byte) bool { + return distribution.IsMigrationOnlyControlKey(key) || distribution.IsCatalogOwnershipPartitionKey(key) || distribution.IsCatalogSplitJobKey(key) || + distribution.IsCatalogSplitJobHistoryKey(key) || bytes.Equal(key, distribution.CatalogNextSplitJobIDKey()) +} + +func splitPartitionControlPrefix(key, embedded []byte) bool { + return distribution.CatalogOwnershipPartitionPrefixIntersects(key) || + (embedded != nil && distribution.CatalogOwnershipPartitionPrefixIntersects(embedded)) +} + +func checkSplitMigrationTransport(ctx context.Context, method, token string, peerAuthEnforced bool) error { + if !splitMigrationAuthenticatedMethod(method, peerAuthEnforced) { + return nil + } + if token == "" { + return errors.WithStack(status.Error(codes.Unauthenticated, "split migration requires a configured migration token")) + } + p, ok := peer.FromContext(ctx) + if !ok { + return errors.WithStack(status.Error(codes.Unauthenticated, "split migration token requires TLS")) + } + if _, ok := p.AuthInfo.(credentials.TLSInfo); !ok { + return errors.WithStack(status.Error(codes.Unauthenticated, "split migration token requires TLS")) + } + return nil +} diff --git a/adapter/split_migration_auth_test.go b/adapter/split_migration_auth_test.go new file mode 100644 index 000000000..64ce57f82 --- /dev/null +++ b/adapter/split_migration_auth_test.go @@ -0,0 +1,205 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +func TestSplitMigrationRolloutRejectsJobInjectionBeforeEnforcement(t *testing.T) { + t.Parallel() + for _, token := range []string{"", "migration-secret"} { + unary, _ := SplitMigrationTokenAuth(token, false, false) + for _, mutation := range []*pb.Mutation{ + {Op: pb.Op_DEL_PREFIX}, + {Op: pb.Op_PUT, Key: distribution.CatalogSplitJobKey(7)}, + {Op: pb.Op_DEL, Key: distribution.CatalogSplitJobHistoryKey(1000, 7)}, + {Op: pb.Op_PUT, Key: distribution.CatalogNextSplitJobIDKey()}, + {Op: pb.Op_DEL_PREFIX, Key: []byte("!dist|j")}, + } { + req := &pb.ForwardRequest{Requests: []*pb.Request{{Mutations: []*pb.Mutation{mutation}}}} + for _, credential := range []string{"", "Bearer wrong", "Bearer migration-secret"} { + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", credential)) + ctx = peer.NewContext(ctx, &peer.Peer{AuthInfo: credentials.TLSInfo{}}) + called := false + _, err := unary(ctx, req, &grpc.UnaryServerInfo{FullMethod: pb.Internal_Forward_FullMethodName}, func(context.Context, any) (any, error) { + called = true + return nil, nil + }) + allowed := token != "" && credential == "Bearer migration-secret" + require.Equal(t, allowed, called) + if allowed { + require.NoError(t, err) + } else { + require.Equal(t, codes.Unauthenticated, status.Code(err)) + } + } + } + legacy := &pb.ForwardRequest{IsTxn: true, Requests: []*pb.Request{{IsTxn: true, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte("!txn|meta")}, {Op: pb.Op_PUT, Key: []byte("customer")}, + }}}} + _, err := unary(context.Background(), legacy, &grpc.UnaryServerInfo{FullMethod: pb.Internal_Forward_FullMethodName}, func(context.Context, any) (any, error) { return nil, nil }) + require.NoError(t, err, "ordinary legacy peer transactions must remain compatible") + } +} + +func TestSplitMigrationTokenAuthProtectsControlAndExport(t *testing.T) { + t.Parallel() + methods := []string{ + pb.Internal_Forward_FullMethodName, + pb.Distribution_SplitRange_FullMethodName, + pb.Distribution_StartSplitMigration_FullMethodName, + pb.Distribution_AttestMigrationOwnership_FullMethodName, + pb.Distribution_AbandonSplitJob_FullMethodName, + pb.Distribution_RetrySplitJob_FullMethodName, + pb.Internal_ApplyTargetStagedReadiness_FullMethodName, + pb.Internal_ImportRangeVersions_FullMethodName, + pb.Internal_PromoteStagedVersions_FullMethodName, + pb.Internal_CleanupMigration_FullMethodName, + pb.Internal_ExportRangeVersions_FullMethodName, + pb.Internal_ProbeMigrationState_FullMethodName, + pb.Internal_ProbeMigrationLocks_FullMethodName, + pb.Internal_IssueMigrationTimestamp_FullMethodName, + } + for _, method := range methods { + t.Run(method, func(t *testing.T) { + unary, stream := SplitMigrationTokenAuth("migration-secret", false, true) + for _, credential := range []string{"", "Bearer wrong", "Bearer migration-secret"} { + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", credential)) + ctx = peer.NewContext(ctx, &peer.Peer{AuthInfo: credentials.TLSInfo{}}) + called := false + _, err := unary(ctx, nil, &grpc.UnaryServerInfo{FullMethod: method}, func(context.Context, any) (any, error) { + called = true + return nil, nil + }) + want := codes.Unauthenticated + if credential == "Bearer migration-secret" { + want = codes.OK + } + require.Equal(t, want, status.Code(err)) + require.Equal(t, want == codes.OK, called) + called = false + err = stream(nil, &adminAuthTestServerStream{ctx: ctx}, &grpc.StreamServerInfo{FullMethod: method}, func(any, grpc.ServerStream) error { + called = true + return nil + }) + require.Equal(t, want, status.Code(err)) + require.Equal(t, want == codes.OK, called) + } + }) + } +} + +func TestSplitMigrationTokenAuthRejectsPlaintextEvenWithValidToken(t *testing.T) { + t.Parallel() + unary, stream := SplitMigrationTokenAuth("migration-secret", false, true) + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer migration-secret")) + _, err := unary(ctx, nil, &grpc.UnaryServerInfo{FullMethod: pb.Internal_CleanupMigration_FullMethodName}, func(context.Context, any) (any, error) { + t.Fatal("plaintext token must not reach the handler") + return nil, nil + }) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + require.ErrorContains(t, err, "requires TLS") + err = stream(nil, &adminAuthTestServerStream{ctx: ctx}, &grpc.StreamServerInfo{FullMethod: pb.Internal_ExportRangeVersions_FullMethodName}, func(any, grpc.ServerStream) error { + t.Fatal("plaintext export must not reach the handler") + return nil + }) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + require.ErrorContains(t, err, "requires TLS") +} + +func TestSplitMigrationTokenAuthNoTokenRequiresExplicitOptIn(t *testing.T) { + t.Parallel() + unary, _ := SplitMigrationTokenAuth("", false, true) + for _, method := range []string{ + pb.Internal_Forward_FullMethodName, + pb.Distribution_SplitRange_FullMethodName, + pb.Distribution_StartSplitMigration_FullMethodName, + pb.Distribution_AttestMigrationOwnership_FullMethodName, + pb.Distribution_AbandonSplitJob_FullMethodName, + pb.Distribution_RetrySplitJob_FullMethodName, + pb.Internal_ApplyTargetStagedReadiness_FullMethodName, + pb.Internal_ImportRangeVersions_FullMethodName, + pb.Internal_PromoteStagedVersions_FullMethodName, + pb.Internal_CleanupMigration_FullMethodName, + pb.Internal_ExportRangeVersions_FullMethodName, + pb.Internal_ProbeMigrationState_FullMethodName, + pb.Internal_ProbeMigrationLocks_FullMethodName, + pb.Internal_IssueMigrationTimestamp_FullMethodName, + } { + _, err := unary(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: method}, func(context.Context, any) (any, error) { + t.Fatal("unauthenticated migration must not reach the handler") + return nil, nil + }) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + } + resp, err := unary(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: pb.Distribution_ListRoutes_FullMethodName}, func(context.Context, any) (any, error) { + return "routes", nil + }) + require.NoError(t, err) + require.Equal(t, "routes", resp) + insecureUnary, insecureStream := SplitMigrationTokenAuth("", true, false) + require.Nil(t, insecureUnary) + require.Nil(t, insecureStream) +} + +func TestSplitMigrationTokenAuthLeavesNormalPeerRPCsUnchanged(t *testing.T) { + t.Parallel() + for _, token := range []string{"", "migration-secret"} { + unary, stream := SplitMigrationTokenAuth(token, false, false) + for _, method := range []string{ + pb.Internal_Forward_FullMethodName, + pb.Distribution_ListRoutes_FullMethodName, + pb.Distribution_GetSplitMigrationCapability_FullMethodName, + } { + ctx := context.Background() + called := false + _, err := unary(ctx, nil, &grpc.UnaryServerInfo{FullMethod: method}, func(context.Context, any) (any, error) { + called = true + return nil, nil + }) + require.NoError(t, err) + require.True(t, called) + called = false + err = stream(nil, &adminAuthTestServerStream{ctx: ctx}, &grpc.StreamServerInfo{FullMethod: method}, func(any, grpc.ServerStream) error { + called = true + return nil + }) + require.NoError(t, err) + require.True(t, called) + } + } +} + +func TestSplitMigrationPeerAuthRolloutAcceptsLegacyForwardBeforeEnforcement(t *testing.T) { + t.Parallel() + for _, enforced := range []bool{false, true} { + unary, _ := SplitMigrationTokenAuth("migration-secret", false, enforced) + called := false + _, err := unary(context.Background(), &pb.ForwardRequest{}, &grpc.UnaryServerInfo{FullMethod: pb.Internal_Forward_FullMethodName}, func(context.Context, any) (any, error) { + called = true + return nil, nil + }) + if enforced { + require.Equal(t, codes.Unauthenticated, status.Code(err)) + require.False(t, called) + } else { + require.NoError(t, err) + require.True(t, called, "send-only stage must accept old peers during rolling upgrade") + } + _, err = unary(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: pb.Internal_CleanupMigration_FullMethodName}, func(context.Context, any) (any, error) { + t.Fatal("rollout never exposes privileged migration RPCs") + return nil, nil + }) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + } +} diff --git a/adapter/split_migration_namespace_test.go b/adapter/split_migration_namespace_test.go new file mode 100644 index 000000000..d83326538 --- /dev/null +++ b/adapter/split_migration_namespace_test.go @@ -0,0 +1,103 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +func migrationNamespacePeerContext(t *testing.T, authorized bool) context.Context { + t.Helper() + ctx := peer.NewContext(t.Context(), &peer.Peer{AuthInfo: credentials.TLSInfo{}}) + if authorized { + ctx = metadata.NewIncomingContext(ctx, metadata.Pairs("authorization", "Bearer "+t.Name())) + } + return ctx +} + +func TestSplitMigrationNamespaceSendFirstAuthentication(t *testing.T) { + for _, key := range [][]byte{[]byte("!migstage|ready|"), []byte("!migstage|ownership"), []byte("!migstage|retired"), + []byte("!migstage|ack|"), []byte("!migwrite|source"), []byte("!migfence|source"), + []byte("!dist|migstage|row"), store.MigrationConfigurationIndexKey(), + []byte("!txn|int|!migstage|ready|"), []byte("!txn|lock|!migstage|ready|")} { + t.Run(string(key), func(t *testing.T) { + auth, _ := SplitMigrationTokenAuth(t.Name(), false, false) + request := &pb.ForwardRequest{Requests: []*pb.Request{{IsTxn: true, + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key}}}}} + for _, authorized := range []bool{false, true} { + called := false + _, err := auth(migrationNamespacePeerContext(t, authorized), request, + &grpc.UnaryServerInfo{FullMethod: pb.Internal_Forward_FullMethodName}, func(context.Context, any) (any, error) { + called = true + return nil, nil + }) + require.Equal(t, authorized, called) + if authorized { + require.NoError(t, err) + } else { + require.Equal(t, codes.Unauthenticated, status.Code(err)) + } + } + }) + } +} + +func TestSplitMigrationNamespaceAuthorizedForwardStillRejectsInternalWrites(t *testing.T) { + st := store.NewMVCCStore() + fsm := kv.NewKvFSMWithHLC(st, kv.NewHLC()) + proposer := &applyingMigrationProposer{fsm: fsm} + tx := kv.NewTransactionWithProposer(proposer, kv.WithMigrationRequestAdmission(st, 1)) + t.Cleanup(tx.Close) + internal := NewInternalWithEngine(tx, forwardObserverLeader{}, nil, nil, WithInternalStore(st)) + auth, _ := SplitMigrationTokenAuth(t.Name(), false, false) + request := &pb.ForwardRequest{IsTxn: true, Requests: []*pb.Request{{IsTxn: true, Ts: 10, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("!migstage|ready|"), CommitTS: 20})}, + {Op: pb.Op_PUT, Key: []byte("!migstage|ready|"), Value: []byte("poison")}, + }}}} + _, err := auth(migrationNamespacePeerContext(t, true), request, + &grpc.UnaryServerInfo{FullMethod: pb.Internal_Forward_FullMethodName}, func(ctx context.Context, req any) (any, error) { + forward, ok := req.(*pb.ForwardRequest) + require.True(t, ok) + return internal.Forward(ctx, forward) + }) + require.ErrorIs(t, err, kv.ErrInvalidRequest) + require.Zero(t, proposer.calls, "privileged transport cannot bypass preproposal namespace admission") +} + +func TestSplitMigrationNamespacePreservesOrdinaryPeerAliases(t *testing.T) { + auth, _ := SplitMigrationTokenAuth(t.Name(), false, false) + for _, key := range []string{"customer", "_migack", "_mighlc", "_migpromote", "!txn|meta", "!txn|foo", + "!txn|backup|customer", "!txn|int|customer", "!migstage", "!migstage-user", "!hs|!migstage|ready||field"} { + request := &pb.ForwardRequest{Requests: []*pb.Request{{Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte(key)}}}}} + called := false + _, err := auth(t.Context(), request, &grpc.UnaryServerInfo{FullMethod: pb.Internal_Forward_FullMethodName}, func(context.Context, any) (any, error) { + called = true + return nil, nil + }) + require.NoError(t, err, key) + require.True(t, called, key) + } +} + +func TestSplitMigrationNamespaceProtectsPrefixDeletes(t *testing.T) { + auth, _ := SplitMigrationTokenAuth(t.Name(), false, false) + for _, key := range []string{"!migs", "!migstage|", "!migwrite|", "!migfence|", "!dist|migstage|"} { + request := &pb.ForwardRequest{Requests: []*pb.Request{{Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte(key)}}}}} + _, err := auth(migrationNamespacePeerContext(t, false), request, + &grpc.UnaryServerInfo{FullMethod: pb.Internal_Forward_FullMethodName}, func(context.Context, any) (any, error) { + t.Fatal("unprivileged prefix delete reached its handler") + return nil, nil + }) + require.Equal(t, codes.Unauthenticated, status.Code(err), key) + } +} diff --git a/adapter/split_migration_split_range_auth_test.go b/adapter/split_migration_split_range_auth_test.go new file mode 100644 index 000000000..4df196081 --- /dev/null +++ b/adapter/split_migration_split_range_auth_test.go @@ -0,0 +1,82 @@ +package adapter + +import ( + "context" + "strconv" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +func TestSplitRangeAuthRejectsProofFreeCatalogMutationDuringRollout(t *testing.T) { + t.Parallel() + cases := []struct { + name, configuredToken, credential string + tls, allowed bool + }{ + {name: "unconfigured", credential: "Bearer migration-secret", tls: true}, + {name: "anonymous", configuredToken: "migration-secret"}, + {name: "token-without-TLS", configuredToken: "migration-secret", credential: "Bearer migration-secret"}, + {name: "TLS-only", configuredToken: "migration-secret", tls: true}, + {name: "read-only-admin-token", configuredToken: "migration-secret", credential: "Bearer admin-secret", tls: true}, + {name: "blob-peer-token", configuredToken: "migration-secret", credential: "Bearer blob-secret", tls: true}, + {name: "migration-token", configuredToken: "migration-secret", credential: "Bearer migration-secret", tls: true, allowed: true}, + } + for _, enforced := range []bool{false, true} { + for _, tc := range cases { + t.Run(tc.name+"/enforced="+strconv.FormatBool(enforced), func(t *testing.T) { + server, coordinator := proofFreeSplitRangeAuthServer(t) + ctx := metadata.NewIncomingContext(t.Context(), metadata.Pairs("authorization", tc.credential)) + if tc.tls { + ctx = peer.NewContext(ctx, &peer.Peer{AuthInfo: credentials.TLSInfo{}}) + } + unary, _ := SplitMigrationTokenAuth(tc.configuredToken, false, enforced) + request := &pb.SplitRangeRequest{ExpectedCatalogVersion: 1, RouteId: 1, SplitKey: []byte("m")} + _, err := unary(ctx, request, &grpc.UnaryServerInfo{FullMethod: pb.Distribution_SplitRange_FullMethodName}, + func(ctx context.Context, _ any) (any, error) { return server.SplitRange(ctx, request) }) + assertSplitRangeAuthResult(t, server, coordinator, tc.allowed, err) + }) + } + } +} + +func proofFreeSplitRangeAuthServer(t *testing.T) (*DistributionServer, *distributionCoordinatorStub) { + t.Helper() + st := store.NewMVCCStore() + t.Cleanup(func() { require.NoError(t, st.Close()) }) + catalog := distribution.NewCatalogStore(st) + saved, err := catalog.Save(t.Context(), 0, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 7, State: distribution.RouteStateActive}, + }) + require.NoError(t, err) + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(saved)) + coordinator := newDistributionCoordinatorStub(st, true) + return NewDistributionServer(engine, catalog, WithDistributionCoordinator(coordinator)), coordinator +} + +func assertSplitRangeAuthResult(t *testing.T, server *DistributionServer, coordinator *distributionCoordinatorStub, allowed bool, err error) { + t.Helper() + snapshot, snapshotErr := server.catalog.Snapshot(t.Context()) + require.NoError(t, snapshotErr) + if allowed { + require.NoError(t, err) + require.EqualValues(t, 2, snapshot.Version) + require.EqualValues(t, 2, server.engine.Version()) + require.Equal(t, 1, coordinator.dispatchCalls) + return + } + require.EqualValues(t, 1, snapshot.Version, "unauthorized SplitRange must not advance the global catalog") + require.EqualValues(t, 1, server.engine.Version()) + require.Zero(t, coordinator.dispatchCalls) + require.Equal(t, codes.Unauthenticated, status.Code(err)) +} diff --git a/adapter/split_partition_barrier.go b/adapter/split_partition_barrier.go new file mode 100644 index 000000000..ee6aa27b2 --- /dev/null +++ b/adapter/split_partition_barrier.go @@ -0,0 +1,68 @@ +package adapter + +import ( + "context" + "time" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +const splitPartitionApplyPoll = 5 * time.Millisecond + +func (s *DistributionServer) splitPartitionAdmissionView(ctx context.Context, id uint64, group *kv.ShardGroup) (store.MigrationServingState, error) { + if group == nil || group.Store == nil || group.Engine == nil { + return store.MigrationServingState{}, errors.WithStack(store.ErrNotSupported) + } + reader, ok := group.Store.(store.MigrationServingAdmissionReader) + if !ok { + return store.MigrationServingState{}, errors.WithStack(store.ErrNotSupported) + } + if err := s.fenceSplitPartitionGroup(ctx, id, group); err != nil { + return store.MigrationServingState{}, err + } + view, err := reader.MigrationServingAdmissionState(ctx) + return view, errors.WithStack(err) +} + +func (s *DistributionServer) fenceSplitPartitionGroup(ctx context.Context, id uint64, group *kv.ShardGroup) error { + attempt, cancel := context.WithTimeout(ctx, ownershipRefreshBudget) + defer cancel() + if group.Engine.State() == raftengine.StateLeader { + _, err := group.Engine.LinearizableRead(attempt) + return errors.WithStack(err) + } + // The root leader need not lead the data group. Obtain that exact group's + // leader fence, then wait for this local native store to have applied it. + // A representative-key fallback could fence a different owner after a split. + leader, ok := s.coordinator.(interface { + LeaseReadForGroup(context.Context, uint64) (uint64, error) + }) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + index, err := leader.LeaseReadForGroup(attempt, id) + if err != nil { + return errors.WithStack(err) + } + return waitSplitPartitionApplied(attempt, group.Engine, index) +} + +func waitSplitPartitionApplied(ctx context.Context, engine raftengine.LeaderView, index uint64) error { + applied, ok := engine.(interface{ AppliedIndex() uint64 }) + if !ok || index == 0 { + return errors.WithStack(store.ErrNotSupported) + } + ticker := time.NewTicker(splitPartitionApplyPoll) + defer ticker.Stop() + for applied.AppliedIndex() < index { + select { + case <-ctx.Done(): + return errors.WithStack(ctx.Err()) + case <-ticker.C: + } + } + return errors.WithStack(ctx.Err()) +} diff --git a/adapter/split_partition_barrier_test.go b/adapter/split_partition_barrier_test.go new file mode 100644 index 000000000..fb1550e8b --- /dev/null +++ b/adapter/split_partition_barrier_test.go @@ -0,0 +1,85 @@ +package adapter + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestSplitRangeCannotTreatUnfencedEmptyFrontierAsProofFree(t *testing.T) { + t.Parallel() + server, version := seededDistributionServer(t) + coordinator, ok := server.coordinator.(*distributionCoordinatorStub) + require.True(t, ok) + before, err := server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + blocked := errors.New("data group cannot establish a current read fence") + groups := map[uint64]*kv.ShardGroup{7: {Store: coordinator.store, + Engine: ownershipRefreshEngine{read: func(context.Context) error { return blocked }}}} + WithMigrationOwnershipRefresh(groups, nil)(server) + _, err = server.SplitRange(t.Context(), &pb.SplitRangeRequest{RouteId: 1, + SplitKey: []byte("g"), ExpectedCatalogVersion: version}) + require.Error(t, err) + after, err := server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + require.Equal(t, before.Version, after.Version) + require.Equal(t, before.Routes, after.Routes) +} + +func TestSplitWorkingSourceCannotDowngradeConfiguredPartitionRuntime(t *testing.T) { + t.Parallel() + server, _ := seededDistributionServer(t) + WithMigrationOwnershipRefresh(map[uint64]*kv.ShardGroup{}, nil)(server) + job := distribution.SplitJob{JobID: 11, SourceRouteID: 1, SourceGroupID: 7, TargetGroupID: 9} + updated, err := server.prepareSplitJobWorkingSource(t.Context(), job) + require.ErrorIs(t, err, store.ErrNotSupported) + require.Equal(t, job, updated) +} + +type splitPartitionFollower struct { + ownershipRefreshEngine + applied atomic.Uint64 +} + +func (e *splitPartitionFollower) AppliedIndex() uint64 { return e.applied.Load() } + +type splitPartitionFenceCoordinator struct { + *distributionCoordinatorStub + group uint64 +} + +func (c *splitPartitionFenceCoordinator) LeaseReadForGroup(_ context.Context, id uint64) (uint64, error) { + c.group = id + return 42, nil +} + +func TestSplitPartitionFollowerWaitsForExactGroupFenceBeforeReading(t *testing.T) { + t.Parallel() + server, _ := seededDistributionServer(t) + base, ok := server.coordinator.(*distributionCoordinatorStub) + require.True(t, ok) + coordinator := &splitPartitionFenceCoordinator{distributionCoordinatorStub: base} + server.coordinator = coordinator + engine := &splitPartitionFollower{ownershipRefreshEngine: ownershipRefreshEngine{follower: true}} + engine.applied.Store(41) + group := &kv.ShardGroup{Engine: engine, Store: store.NewMVCCStore()} + t.Cleanup(func() { require.NoError(t, group.Store.Close()) }) + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond) + defer cancel() + _, err := server.splitPartitionAdmissionView(ctx, 7, group) + require.ErrorIs(t, err, context.DeadlineExceeded, "a leader fence does not imply local catch-up") + require.EqualValues(t, 7, coordinator.group) + engine.applied.Store(42) + view, err := server.splitPartitionAdmissionView(t.Context(), 7, group) + require.NoError(t, err) + require.Empty(t, view.Ownership) + require.Empty(t, view.Partitions.Partitions) +} diff --git a/adapter/split_partition_budget.go b/adapter/split_partition_budget.go new file mode 100644 index 000000000..e68009dc1 --- /dev/null +++ b/adapter/split_partition_budget.go @@ -0,0 +1,108 @@ +package adapter + +import ( + "bytes" + "context" + "encoding/json" + "io" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +const splitPartitionMaxUintDigits = 20 + +// Check the full downstream envelope before publishing geometry. The manifest +// alone is smaller than the control, which repeats bounds in priors and tiles. +func (s *DistributionServer) validateSplitPartitionBudget(ctx context.Context, parent distribution.RouteDescriptor, manifest []byte, version uint64, job *distribution.SplitJob) error { + if s.ownershipRefresh == nil { + return nil + } + groups := []uint64{parent.GroupID} + if job != nil && job.TargetGroupID != parent.GroupID { + groups = append(groups, job.TargetGroupID) + } + for _, id := range groups { + if err := s.validateSplitGroupPartitionBudget(ctx, id, parent, manifest, version); err != nil { + return err + } + } + return nil +} + +func (s *DistributionServer) validateSplitGroupPartitionBudget(ctx context.Context, id uint64, parent distribution.RouteDescriptor, manifest []byte, version uint64) error { + group := s.ownershipRefresh.groups[id] + view, err := s.splitPartitionAdmissionView(ctx, id, group) + if err != nil { + return errors.WithStack(err) + } + partitions, legacy := splitPartitionBudgetPriors(view, parent) + if len(partitions)+len(legacy) == 0 { + return nil + } + configuration, err := kv.MigrationConfigurationIndex(ctx, group.Store) + if err != nil { + return errors.WithStack(err) + } + control, err := kv.BuildMigrationPartitionControl(partitions, legacy, manifest, nil, version, configuration) + if err != nil { + return errors.WithStack(err) + } + return validateSplitPartitionControlSize(control) +} + +func validateSplitPartitionControlSize(control kv.MigrationPartitionControl) error { + encoded, err := kv.EncodeMigrationPartitionControl(control) + if err != nil { + return errors.WithStack(err) + } + maximum, err := maximumSplitPartitionControlSize(encoded) + if err != nil { + return err + } + if maximum > kv.MaxMigrationPartitionControlBytes { + return errors.Wrap(kv.ErrInvalidRequest, "split ownership control exceeds transfer budget") + } + return nil +} + +// Refresh and membership changes may grow every numeric stamp after this +// preflight. Reserve the full uint64 decimal width at every numeric position, +// not a fixed margin per request. Geometry, origins and digest widths remain +// immutable during those operations. This is sizing only, never an attestation. +func maximumSplitPartitionControlSize(encoded []byte) (int, error) { + decoder := json.NewDecoder(bytes.NewReader(encoded)) + decoder.UseNumber() + maximum := len(encoded) + for { + token, err := decoder.Token() + if err == io.EOF { + return maximum, nil + } + if err != nil { + return 0, errors.WithStack(err) + } + if number, ok := token.(json.Number); ok { + maximum += splitPartitionMaxUintDigits - len(number.String()) + } + } +} + +func splitPartitionBudgetPriors(view store.MigrationServingState, parent distribution.RouteDescriptor) ([]store.MigrationOwnershipPartition, []store.MigrationOwnershipState) { + var partitions []store.MigrationOwnershipPartition + var legacy []store.MigrationOwnershipState + for _, candidate := range migrationPartitionCandidates(view, view.Partitions) { + proof := candidate.geometry() + if !routeRangeIntersects(proof.RouteStart, proof.RouteEnd, parent.Start, parent.End) { + continue + } + if candidate.partition != nil { + partitions = append(partitions, *candidate.partition) + } else { + legacy = append(legacy, *candidate.legacy) + } + } + return partitions, legacy +} diff --git a/adapter/split_partition_budget_test.go b/adapter/split_partition_budget_test.go new file mode 100644 index 000000000..dd4a5a9e6 --- /dev/null +++ b/adapter/split_partition_budget_test.go @@ -0,0 +1,97 @@ +package adapter + +import ( + "bytes" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" +) + +func TestSplitPartitionBudgetRejectsBeforeGeometryPublication(t *testing.T) { + for _, working := range []bool{false, true} { + name := "metadata" + if working { + name = "working-source" + } + t.Run(name, func(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + r.migrate(t, 1, 2, "m") + before, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + parent := r.ownership(t, 2)[0].RouteID + key := append([]byte("n"), bytes.Repeat([]byte("k"), 700<<10)...) + if working { + job := distribution.SplitJob{JobID: 100, SourceRouteID: parent, SourceGroupID: 2, + TargetGroupID: 3, SplitKey: key, Phase: distribution.SplitJobPhasePlanned} + require.NoError(t, r.server.catalog.CreateSplitJob(t.Context(), job)) + _, err = r.server.ensureSplitWorkingSource(t.Context(), job) + stored, found, loadErr := r.server.catalog.SplitJob(t.Context(), job.JobID) + require.NoError(t, loadErr) + require.True(t, found) + require.Zero(t, stored.WorkingSourceRouteID) + } else { + _, err = r.client.SplitRange(t.Context(), &pb.SplitRangeRequest{RouteId: parent, + SplitKey: key, ExpectedCatalogVersion: before.Version}) + } + require.Error(t, err, "the manifest fits, but the full partition control does not") + after, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + require.Equal(t, before.Version, after.Version) + require.Equal(t, before.Routes, after.Routes) + require.NoError(t, r.put(t, 2, "n", "still-writable", 1<<40)) + }) + } +} + +func TestSplitRangeWithoutAuthCannotIgnoreDurableOwnership(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + r.migrate(t, 1, 2, "m") + before, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + parent := r.ownership(t, 2)[0].RouteID + WithMigrationOwnershipRefresh(r.groups, nil)(r.server) + require.False(t, r.server.MigrationOwnershipRefreshConfigured(2)) + _, err = r.client.SplitRange(t.Context(), &pb.SplitRangeRequest{RouteId: parent, + SplitKey: []byte("t"), ExpectedCatalogVersion: before.Version}) + require.Error(t, err) + after, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + require.Equal(t, before.Version, after.Version) + require.Equal(t, before.Routes, after.Routes) +} + +func TestSplitRangeCannotTreatMissingOwnerStoreAsProofFree(t *testing.T) { + t.Parallel() + server, version := seededDistributionServer(t) + before, err := server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + WithMigrationOwnershipRefresh(map[uint64]*kv.ShardGroup{}, nil)(server) + _, err = server.SplitRange(t.Context(), &pb.SplitRangeRequest{RouteId: 1, + SplitKey: []byte("g"), ExpectedCatalogVersion: version}) + require.Error(t, err, "unknown ownership is not an empty ownership frontier") + after, err := server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + require.Equal(t, before.Version, after.Version) + require.Equal(t, before.Routes, after.Routes) +} + +func TestSplitPartitionBudgetAccountsForAllNumericStampGrowth(t *testing.T) { + t.Parallel() + // A constant 4 KiB reserve cannot cover even one growing stamp on each + // of 256 priors. Numeric-looking descriptor bytes are strings, not stamps. + row := []byte(`{"AppliedIndex":1,"CatalogVersion":9,"Hash":"123456789"}`) + encoded := append([]byte{'['}, bytes.Repeat(append(bytes.Clone(row), ','), 255)...) + encoded = append(encoded, row...) + encoded = append(encoded, ']') + maximum, err := maximumSplitPartitionControlSize(encoded) + require.NoError(t, err) + require.Equal(t, len(encoded)+256*2*19, maximum) + require.Greater(t, maximum-len(encoded), 4<<10) + wide := []byte(`{"AppliedIndex":18446744073709551615}`) + maximum, err = maximumSplitPartitionControlSize(wide) + require.NoError(t, err) + require.Len(t, wide, maximum) +} diff --git a/adapter/split_partition_catalog.go b/adapter/split_partition_catalog.go new file mode 100644 index 000000000..c187afbf4 --- /dev/null +++ b/adapter/split_partition_catalog.go @@ -0,0 +1,81 @@ +package adapter + +import ( + "context" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +func (s *DistributionServer) catalogSplitPartitionOps(ctx context.Context, readTS, parentID uint64, + left, right distribution.RouteDescriptor, version uint64, workingJob *distribution.SplitJob, +) ([]*kv.Elem[kv.OP], [][]byte, error) { + // A legacy split-and-fence changes authority rather than partitioning it. + // The new working-child flow records geometry while both children are active. + if left.State != distribution.RouteStateActive || right.State != distribution.RouteStateActive { + return nil, nil, nil + } + mutation, err := s.catalogSplitManifest(ctx, readTS, parentID, left, right, version, workingJob) + if err != nil { + return nil, nil, err + } + ops, err := catalogStoreMutationsToOps([]*store.KVPairMutation{mutation}) + if err != nil { + return nil, nil, err + } + reads := [][]byte{mutation.Key} + if workingJob == nil { + return ops, reads, nil + } + jobOp, err := s.catalogWorkingSourceOp(ctx, readTS, parentID, right.RouteID, *workingJob) + if err != nil { + return nil, nil, err + } + return append(ops, jobOp), append(reads, jobOp.Key), nil +} + +func (s *DistributionServer) catalogSplitManifest(ctx context.Context, readTS, parentID uint64, left, right distribution.RouteDescriptor, + version uint64, workingJob *distribution.SplitJob, +) (*store.KVPairMutation, error) { + snapshot, err := s.loadCatalogSnapshotAt(ctx, readTS) + if err != nil { + return nil, err + } + parent, found := findRouteByID(snapshot.Routes, parentID) + if !found || snapshot.Version != version { + return nil, errors.WithStack(store.ErrWriteConflict) + } + manifest := distribution.OwnershipPartitionManifest{PreviousVersion: version, Version: version + 1, + Parent: parent, Children: []distribution.RouteDescriptor{left, right}} + if workingJob != nil { + manifest.JobID = workingJob.JobID + } + mutation, err := s.catalog.BuildOwnershipPartitionMutationAt(ctx, readTS, manifest) + if err != nil { + return nil, errors.WithStack(err) + } + if err := s.validateSplitPartitionBudget(ctx, parent, mutation.Value, manifest.Version, workingJob); err != nil { + return nil, err + } + return mutation, nil +} + +func (s *DistributionServer) catalogWorkingSourceOp(ctx context.Context, readTS, parentID, rightID uint64, workingJob distribution.SplitJob) (*kv.Elem[kv.OP], error) { + current, exists, err := s.catalog.SplitJobAt(ctx, workingJob.JobID, readTS) + if err != nil { + return nil, errors.WithStack(err) + } + if !exists || !distribution.SplitJobsEquivalent(current, workingJob) || current.Phase != distribution.SplitJobPhasePlanned || + current.WorkingSourceRouteID != 0 || parentID != current.SourceRouteID { + return nil, errors.WithStack(distribution.ErrCatalogSplitJobConflict) + } + current.WorkingSourceRouteID = rightID + encoded, err := distribution.EncodeSplitJob(current) + if err != nil { + return nil, errors.WithStack(err) + } + jobKey := distribution.CatalogSplitJobKey(current.JobID) + return &kv.Elem[kv.OP]{Op: kv.Put, Key: jobKey, Value: encoded}, nil +} diff --git a/adapter/split_partition_catalog_test.go b/adapter/split_partition_catalog_test.go new file mode 100644 index 000000000..b47f3dae1 --- /dev/null +++ b/adapter/split_partition_catalog_test.go @@ -0,0 +1,110 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestSplitPartitionManifestConflictDoesNotPublishRoutes(t *testing.T) { + t.Parallel() + s, version := seededDistributionServer(t) + before, err := s.catalog.Snapshot(t.Context()) + require.NoError(t, err) + hash, err := distribution.OwnershipPartitionDescriptorHash(before.Routes[0]) + require.NoError(t, err) + key, err := distribution.CatalogOwnershipPartitionKey(1, hash) + require.NoError(t, err) + coordinator, ok := s.coordinator.(*distributionCoordinatorStub) + require.True(t, ok) + coordinator.beforeApply = func(ctx context.Context, st store.MVCCStore) error { + return st.PutAt(ctx, key, []byte("conflicting ledger publication"), coordinator.lastCommitTS, 0) + } + _, err = s.SplitRange(t.Context(), &pb.SplitRangeRequest{RouteId: 1, SplitKey: []byte("g"), ExpectedCatalogVersion: version}) + require.Error(t, err) + after, err := s.catalog.Snapshot(t.Context()) + require.NoError(t, err) + require.Equal(t, before.Version, after.Version) + require.Equal(t, before.Routes, after.Routes) + requireReadKeysContain(t, coordinator.lastReadKeys, key) +} + +func TestSplitRangeCommitsPartitionManifestWithRoutes(t *testing.T) { + t.Parallel() + s, version := seededDistributionServer(t) + before, err := s.catalog.Snapshot(t.Context()) + require.NoError(t, err) + parent := before.Routes[0] + response, err := s.SplitRange(t.Context(), &pb.SplitRangeRequest{RouteId: parent.RouteID, + SplitKey: []byte("g"), ExpectedCatalogVersion: version}) + require.NoError(t, err) + after, err := s.catalog.Snapshot(t.Context()) + require.NoError(t, err) + hash, err := distribution.OwnershipPartitionDescriptorHash(parent) + require.NoError(t, err) + manifest, exists, err := s.catalog.OwnershipPartitionAt(t.Context(), parent.RouteID, hash, after.ReadTS) + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, response.CatalogVersion, manifest.Version) + require.Equal(t, version, manifest.PreviousVersion) + require.Equal(t, after.Routes, manifest.Children) + _, exists, err = s.catalog.OwnershipPartitionAt(t.Context(), parent.RouteID, hash, before.ReadTS) + require.NoError(t, err) + require.False(t, exists, "lineage must not become visible before the route transaction") +} + +func TestSplitWorkingSourcePersistsJobWithManifestAndRecovers(t *testing.T) { + t.Parallel() + s, _ := seededDistributionServer(t) + coordinator, ok := s.coordinator.(*distributionCoordinatorStub) + require.True(t, ok) + s.coordinator = &ownershipCatalogCoordinator{distributionCoordinatorStub: coordinator} + before, err := s.catalog.Snapshot(t.Context()) + require.NoError(t, err) + job := distribution.SplitJob{JobID: 13, SourceRouteID: 1, SourceGroupID: 7, + TargetGroupID: 9, SplitKey: []byte("g"), Phase: distribution.SplitJobPhasePlanned} + require.NoError(t, s.catalog.CreateSplitJob(t.Context(), job)) + updated, err := s.ensureSplitWorkingSource(t.Context(), job) + require.NoError(t, err) + require.NotZero(t, updated.WorkingSourceRouteID) + require.Equal(t, job.SourceRouteID, updated.SourceRouteID) + after, route, err := s.splitJobSourceRoute(t.Context(), updated) + require.NoError(t, err) + require.Equal(t, updated.WorkingSourceRouteID, route.RouteID) + require.Equal(t, []byte("g"), route.Start) + require.Len(t, after.Routes, 2) + hash, err := distribution.OwnershipPartitionDescriptorHash(before.Routes[0]) + require.NoError(t, err) + manifest, exists, err := s.catalog.OwnershipPartitionAt(t.Context(), 1, hash, after.ReadTS) + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, job.JobID, manifest.JobID) + require.Equal(t, after.Routes, manifest.Children) + loaded, exists, err := s.catalog.SplitJobAt(t.Context(), job.JobID, after.ReadTS) + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, updated, loaded) + resumed, err := s.ensureSplitWorkingSource(t.Context(), loaded) + require.NoError(t, err) + require.Equal(t, updated, resumed) + fencing := distribution.CloneSplitJob(loaded) + fencing.Phase = distribution.SplitJobPhaseFence + fencing.SnapshotTS = 100 + require.NoError(t, s.catalog.SaveSplitJob(t.Context(), loaded, fencing)) + fenceSnapshot, moving, err := s.splitJobSourceRoute(t.Context(), fencing) + require.NoError(t, err) + fenced, moving, err := s.ensureSplitJobCatalogFence(t.Context(), fencing, fenceSnapshot, moving) + require.NoError(t, err) + require.Len(t, fenced.Routes, 2, "the working child must not be split at its start key again") + require.Equal(t, route.RouteID, moving.RouteID) + require.Equal(t, distribution.RouteStateWriteFenced, moving.State) + require.Equal(t, after.Routes[0], fenced.Routes[0], "the retained side keeps its descriptor and floor") + replayed, replayedMoving, err := s.ensureSplitJobCatalogFence(t.Context(), fencing, fenced, moving) + require.NoError(t, err) + require.Equal(t, fenced.Version, replayed.Version) + require.Equal(t, moving, replayedMoving) +} diff --git a/adapter/split_partition_namespace_test.go b/adapter/split_partition_namespace_test.go new file mode 100644 index 000000000..14d8d2089 --- /dev/null +++ b/adapter/split_partition_namespace_test.go @@ -0,0 +1,46 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +func TestSplitPartitionLedgerRequiresMigrationAuthentication(t *testing.T) { + key, err := distribution.CatalogOwnershipPartitionKey(3, make([]byte, 32)) + require.NoError(t, err) + for _, prefix := range []string{"", "!txn|int|", "!txn|lock|"} { + for _, operation := range []pb.Op{pb.Op_PUT, pb.Op_DEL, pb.Op_DEL_PREFIX} { + auth, _ := SplitMigrationTokenAuth(t.Name(), false, false) + request := &pb.ForwardRequest{Requests: []*pb.Request{{Mutations: []*pb.Mutation{ + {Op: operation, Key: append([]byte(prefix), key...)}, + }}}} + for _, authorized := range []bool{false, true} { + called := false + _, err := auth(migrationNamespacePeerContext(t, authorized), request, + &grpc.UnaryServerInfo{FullMethod: pb.Internal_Forward_FullMethodName}, + func(context.Context, any) (any, error) { called = true; return nil, nil }) + require.Equal(t, authorized, called) + if authorized { + require.NoError(t, err) + } else { + require.Error(t, err) + } + } + } + } +} + +func TestSplitPartitionLedgerPrefixAuthorization(t *testing.T) { + for _, key := range []string{"!dist|meta|ownership_partition/", "!dist|meta|ownership_partition", + "!txn|int|!dist|meta|ownership_partition/", "!txn|lock|!dist|meta|ownership_partition"} { + require.True(t, splitJobControlMutation(&pb.Mutation{Op: pb.Op_DEL_PREFIX, Key: []byte(key)}), key) + } + for _, key := range []string{"customer", "!dist|meta|ownership_partition_user", "!txn|int|customer"} { + require.False(t, splitJobControlMutation(&pb.Mutation{Op: pb.Op_DEL_PREFIX, Key: []byte(key)}), key) + } +} diff --git a/adapter/split_partition_owner_groups.go b/adapter/split_partition_owner_groups.go new file mode 100644 index 000000000..22c1d15e1 --- /dev/null +++ b/adapter/split_partition_owner_groups.go @@ -0,0 +1,78 @@ +package adapter + +import ( + "context" + "maps" + "slices" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +func (s *DistributionServer) splitRangeOwnerGroups(ctx context.Context, snapshot distribution.CatalogSnapshot, parent distribution.RouteDescriptor, parentActive bool) ([]uint64, error) { + for _, route := range snapshot.Routes { + if s.ownershipRefresh.groups[route.GroupID] == nil { + return nil, errors.WithStack(store.ErrNotSupported) + } + } + selected := make(map[uint64]bool) + if parentActive { + selected[parent.GroupID] = true + } + for id, group := range s.ownershipRefresh.groups { + if id == 0 { + continue + } + active, err := s.splitRangeOwnerGroupActive(ctx, id, group) + if err != nil { + return nil, err + } + if active { + selected[id] = true + } + } + return slices.Sorted(maps.Keys(selected)), nil +} + +func (s *DistributionServer) splitRangeOwnerGroupActive(ctx context.Context, id uint64, group *kv.ShardGroup) (bool, error) { + if group == nil || group.Store == nil { + return false, errors.WithStack(store.ErrNotSupported) + } + reader, ok := group.Store.(store.MigrationServingAdmissionReader) + if !ok { + return false, errors.WithStack(store.ErrNotSupported) + } + view, err := reader.MigrationServingAdmissionState(ctx) + if err != nil { + return false, errors.WithStack(err) + } + // Known historical NONOWNER closures remain DENY while offline. Empty or + // potentially positive local views still need a current data-group fence. + if splitPartitionKnownNegativeOnly(view) { + return false, nil + } + view, err = s.splitPartitionAdmissionView(ctx, id, group) + if err != nil { + return false, err + } + return splitPartitionHasServingOwner(view), nil +} + +func splitPartitionKnownNegativeOnly(view store.MigrationServingState) bool { + if len(view.Readiness) != 0 || len(view.Ownership)+len(view.Partitions.Partitions) == 0 { + return false + } + for _, proof := range view.Ownership { + if proof.Role != store.MigrationNonOwner || !store.MigrationOwnershipEvidenceValid(proof) { + return false + } + } + for _, proof := range view.Partitions.Partitions { + if proof.Role != store.MigrationNonOwner || !store.MigrationOwnershipPartitionEvidenceValid(view.Partitions, proof) { + return false + } + } + return true +} diff --git a/adapter/split_partition_owner_refresh.go b/adapter/split_partition_owner_refresh.go new file mode 100644 index 000000000..e00296f9e --- /dev/null +++ b/adapter/split_partition_owner_refresh.go @@ -0,0 +1,130 @@ +package adapter + +import ( + "bytes" + "context" + + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +// Refresh only closures needed by current serving owners. Protected SOURCE and +// unrelated negative evidence remain DENY, not an obligation to obtain ACKs. +func (s *DistributionServer) reconcileSplitServingOwners(ctx context.Context, id uint64) error { + if !s.MigrationPartitionRuntimeConfigured(id) { + return errors.WithStack(store.ErrNotSupported) + } + attempt, cancel := context.WithTimeout(ctx, ownershipRefreshBudget) + defer cancel() + r := s.ownershipRefresh + if !r.mu.TryLock() { + return errors.WithStack(errSplitMigrationVoterBarrierIncomplete) + } + defer r.mu.Unlock() + g := r.groups[id] + for attempt.Err() == nil { + if err := s.fenceSplitPartitionGroup(attempt, id, g); err != nil { + return err + } + view, before, index, err := readMigrationPartitionGroup(attempt, g) + if err != nil { + return err + } + view.Partitions = before + if err := s.refreshSplitServingOwnerClosures(attempt, id, g, view, index); err != nil { + return err + } + _, after, _, err := readMigrationPartitionGroup(attempt, g) + if err != nil { + return errors.WithStack(err) + } + same, err := sameMigrationPartitionFrontier(before, after) + if err != nil || same { + return err + } + } + return errors.WithStack(attempt.Err()) +} + +func (s *DistributionServer) refreshSplitServingOwnerClosures(ctx context.Context, id uint64, g *kv.ShardGroup, view store.MigrationServingState, index uint64) error { + candidates := migrationPartitionCandidates(view, view.Partitions) + visited := make(map[int]bool) + for at, candidate := range candidates { + if visited[at] || !splitPartitionProofServing(view, candidate.geometry()) { + continue + } + partitions, legacy, indices := migrationPartitionPriorClosure(candidates, at) + for _, i := range indices { + visited[i] = true + } + // Do not filter the selected closure: unknown legacy barriers and + // negative overlaps must participate in the exact durable prior CAS. + if err := s.reconcileMigrationPartitionPrior(ctx, id, g, index, view.Partitions, partitions, legacy); err != nil { + return errors.WithStack(err) + } + } + return errors.WithStack(ctx.Err()) +} + +func splitPartitionHasServingOwner(view store.MigrationServingState) bool { + for _, candidate := range migrationPartitionCandidates(view, view.Partitions) { + if splitPartitionProofServing(view, candidate.geometry()) { + return true + } + } + return false +} + +func splitPartitionProofServing(view store.MigrationServingState, p store.MigrationOwnershipPartition) bool { + if p.Role != store.MigrationOwner { + return false + } + if p.SourceTrackerVersion == 0 || !splitPartitionSourceEvidenceKnown(view, p) { + return true + } + for _, guard := range view.Readiness { + if splitPartitionSourceFenceMatches(p, guard) { + return false + } + } + return true +} + +func splitPartitionSourceEvidenceKnown(view store.MigrationServingState, p store.MigrationOwnershipPartition) bool { + if len(p.PartitionID) != 0 { + return store.MigrationOwnershipPartitionEvidenceValid(view.Partitions, p) + } + for _, old := range view.Ownership { + if old.JobID == p.MigrationJobID && store.MigrationOwnershipEvidenceValid(old) && old.SourceTrackerVersion == p.SourceTrackerVersion { + return old.SourceAppliedIndex == p.SourceAppliedIndex && bytes.Equal(old.DescriptorHash, p.DescriptorHash) + } + } + return false +} + +func splitPartitionSourceFenceMatches(p store.MigrationOwnershipPartition, guard store.TargetStagedReadinessState) bool { + if !splitPartitionSourceFenceValid(guard) || guard.JobID != p.MigrationJobID || guard.ExpectedCutoverVersion < p.SourceTrackerVersion { + return false + } + return guard.MinWriteTSExclusive >= p.MinWriteTSExclusive && bytes.Compare(guard.RouteStart, p.RouteStart) <= 0 && + (len(guard.RouteEnd) == 0 || (len(p.RouteEnd) != 0 && bytes.Compare(guard.RouteEnd, p.RouteEnd) >= 0)) +} + +func splitPartitionSourceFenceValid(guard store.TargetStagedReadinessState) bool { + if !guard.Armed || guard.JobID == 0 || guard.MigrationJobID != guard.JobID || guard.MinWriteTSExclusive == 0 || guard.RetentionPinTS == 0 { + return false + } + if guard.SourceReadFence && !guard.SourceWriteFence { + return false + } + return guard.SourceWriteFence && splitPartitionSourceGuardBoundsValid(guard) && splitPartitionSourceTransferGuardValid(guard) +} + +func splitPartitionSourceGuardBoundsValid(guard store.TargetStagedReadinessState) bool { + return guard.RouteEnd == nil || bytes.Compare(guard.RouteStart, guard.RouteEnd) < 0 +} + +func splitPartitionSourceTransferGuardValid(guard store.TargetStagedReadinessState) bool { + return guard.TransferSizeGuardVersion == 0 || (guard.TransferSizeGuardVersion == store.MigrationTransferSizeGuardVersion && guard.TrackWrites) +} diff --git a/adapter/split_partition_owner_refresh_test.go b/adapter/split_partition_owner_refresh_test.go new file mode 100644 index 000000000..f0a986c8d --- /dev/null +++ b/adapter/split_partition_owner_refresh_test.go @@ -0,0 +1,134 @@ +package adapter + +import ( + "bytes" + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func splitServingSourceFixture(t *testing.T) (store.MigrationServingState, store.TargetStagedReadinessState) { + t.Helper() + st := store.NewMVCCStore() + guard := store.TargetStagedReadinessState{JobID: 7, MigrationJobID: 7, Armed: true, TrackWrites: true, + RouteStart: []byte("m"), RouteEnd: []byte("z"), ExpectedCutoverVersion: 1, MinWriteTSExclusive: 1, RetentionPinTS: 1} + guards, ok := st.(store.MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.NoError(t, guards.ApplyTargetStagedReadinessAt(t.Context(), guard, 10)) + state := store.MigrationOwnershipState{Generation: "0123456789abcdef0123456789abcdef", JobID: 7, LocalGroupID: 1, OwnerGroupID: 1, + Role: store.MigrationOwner, CatalogVersion: 1, ConfigurationIndex: 9, RouteID: 1, DescriptorHash: bytes.Repeat([]byte{1}, 32), + RouteStart: []byte("m"), RouteEnd: []byte("z"), WriteFloorKnown: true, SourceTrackerVersion: 1} + writer, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + require.NoError(t, writer.AcquireMigrationOwnership(t.Context(), state, 11)) + view, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + require.True(t, store.MigrationOwnershipEvidenceValid(view.Ownership[0])) + return view, guard +} + +func TestSplitServingOwnerSourceFenceSelection(t *testing.T) { + view, guard := splitServingSourceFixture(t) + p := kv.MigrationPartitionFromLegacy(view.Ownership[0]) + require.True(t, splitPartitionHasServingOwner(view), "an unfenced SOURCE still serves") + guard.SourceWriteFence, guard.SourceReadFence, guard.ExpectedCutoverVersion = true, true, 2 + view.Readiness = []store.TargetStagedReadinessState{guard} + require.False(t, splitPartitionHasServingOwner(view), "later durable fence is not a demand for a stale SOURCE refresh") + for _, tc := range []struct { + name string + edit func(*store.TargetStagedReadinessState) + }{ + {"unarmed", func(g *store.TargetStagedReadinessState) { g.Armed = false }}, + {"wrong job", func(g *store.TargetStagedReadinessState) { g.JobID++ }}, + {"unknown pin", func(g *store.TargetStagedReadinessState) { g.RetentionPinTS = 0 }}, + {"partial coverage", func(g *store.TargetStagedReadinessState) { g.RouteEnd = []byte("t") }}, + {"unknown floor", func(g *store.TargetStagedReadinessState) { g.MinWriteTSExclusive = 0 }}, + {"read fence without write fence", func(g *store.TargetStagedReadinessState) { g.SourceWriteFence = false }}, + {"unsupported guard", func(g *store.TargetStagedReadinessState) { g.TransferSizeGuardVersion = 99 }}, + } { + t.Run(tc.name, func(t *testing.T) { + changed := guard + tc.edit(&changed) + view.Readiness = []store.TargetStagedReadinessState{changed} + require.True(t, splitPartitionProofServing(view, p), "an invalid or unrelated guard cannot suppress an owner obligation") + }) + } + p.SourceTrackerVersion = 0 + require.True(t, splitPartitionProofServing(view, p), "ordinary OWNER is always selected") + p.Role = store.MigrationNonOwner + require.False(t, splitPartitionProofServing(view, p)) +} + +func TestSplitServingOwnersRefreshLegacyCurrentRootAndRequireMembers(t *testing.T) { + h := newPartitionRefreshHarness(t, 7) + _, err := h.root.catalog.Save(t.Context(), 1, []distribution.RouteDescriptor{h.parent}) + require.NoError(t, err) + h.clients[2].old = true + require.Error(t, h.consumer.reconcileSplitServingOwners(t.Context(), 7)) + h.clients[2].old = false + require.NoError(t, h.consumer.reconcileSplitServingOwners(t.Context(), 7)) + for _, st := range h.stores { + reader, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + view, readErr := reader.MigrationServingState(t.Context()) + require.NoError(t, readErr) + require.EqualValues(t, 2, view.Ownership[0].CatalogVersion) + } +} + +func TestSplitServingOwnersSkipNegativeOnlyWithoutRemoteEffects(t *testing.T) { + h := newPartitionRefreshHarness(t, 9) + h.consumer.ownershipRefresh.root = func(context.Context) (MigrationOwnershipAttestor, error) { + return nil, errors.New("unrelated negative root must not be queried") + } + h.clients[2].old = true + require.NoError(t, h.consumer.reconcileSplitServingOwners(t.Context(), 9)) + require.Empty(t, h.clients[0].cleanups) +} + +func TestSplitServingOwnersProtectedSourceWithoutRemoteEffects(t *testing.T) { + h := newPartitionRefreshHarness(t, 9) + for _, st := range h.stores { + guards, ok := st.(store.MigrationTargetReadinessRaftWriter) + require.True(t, ok) + guard := store.TargetStagedReadinessState{JobID: 12, MigrationJobID: 12, Armed: true, TrackWrites: true, + RouteStart: []byte("a"), RouteEnd: []byte("z"), ExpectedCutoverVersion: 2, MinWriteTSExclusive: 100, RetentionPinTS: 1} + require.NoError(t, guards.ApplyTargetStagedReadinessAt(t.Context(), guard, 11)) + writer, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + proof := store.MigrationOwnershipState{Generation: "0123456789abcdef0123456789abcdef", JobID: 12, LocalGroupID: 9, OwnerGroupID: 9, + Role: store.MigrationOwner, CatalogVersion: 2, ConfigurationIndex: 9, RouteID: 1, DescriptorHash: bytes.Repeat([]byte{1}, 32), + RouteStart: []byte("a"), RouteEnd: []byte("z"), WriteFloorKnown: true, MinWriteTSExclusive: 100, SourceTrackerVersion: 2} + require.NoError(t, writer.AcquireMigrationOwnership(t.Context(), proof, 12)) + guard.SourceWriteFence, guard.SourceReadFence, guard.ExpectedCutoverVersion = true, true, 3 + require.NoError(t, guards.ApplyTargetStagedReadinessAt(t.Context(), guard, 13)) + view, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + require.False(t, splitPartitionHasServingOwner(view)) + } + h.consumer.ownershipRefresh.root = func(context.Context) (MigrationOwnershipAttestor, error) { + return nil, errors.New("protected SOURCE must not query root") + } + h.clients[2].old = true + require.NoError(t, h.consumer.reconcileSplitServingOwners(t.Context(), 9)) + require.Empty(t, h.clients[0].cleanups) +} + +func TestSplitServingOwnersUnknownBarrierStaysInOwnerClosure(t *testing.T) { + h := newPartitionRefreshHarness(t, 7) + reader, ok := h.stores[0].(store.MigrationOwnershipStore) + require.True(t, ok) + view, err := reader.MigrationServingState(t.Context()) + require.NoError(t, err) + unknown := view.Ownership[0] + unknown.JobID, unknown.Generation = 12, "" + view.Ownership = append(view.Ownership, unknown) + g := h.consumer.ownershipRefresh.groups[7] + require.Error(t, h.consumer.refreshSplitServingOwnerClosures(t.Context(), 7, g, view, 9)) + require.Empty(t, partitionOnlyCleanups(h.clients[0])) +} diff --git a/adapter/split_partition_preflight_test.go b/adapter/split_partition_preflight_test.go new file mode 100644 index 000000000..cee282ffc --- /dev/null +++ b/adapter/split_partition_preflight_test.go @@ -0,0 +1,81 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestMigrationResplitDoesNotWaitForOfflineHistoricalNonOwner(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + first := r.migrate(t, 1, 2, "m") + before, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + factory := r.server.splitApplyMemberFactory + r.server.splitApplyMemberFactory = func(ctx context.Context, id uint64) (SplitMigrationApplyMemberSnapshot, error) { + if id == 1 { + return SplitMigrationApplyMemberSnapshot{}, errors.New("historical nonowner is offline") + } + return factory(ctx, id) + } + owner := r.ownership(t, 2)[0] + _, err = r.client.SplitRange(t.Context(), &pb.SplitRangeRequest{RouteId: owner.RouteID, SplitKey: []byte("t"), ExpectedCatalogVersion: before.Version}) + require.NoError(t, err, "negative groups can reconcile the retained manifest after they return") + for _, key := range []string{"n", "y"} { + require.NoError(t, r.put(t, 2, key, "available", first.FenceTS+100)) + r.read(t, key, "available") + require.Error(t, r.put(t, 1, key, "stale-owner", first.FenceTS+101)) + _, err := r.groups[1].Store.GetAt(t.Context(), []byte(key), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound) + } +} + +func TestSplitRangeWithoutOwnershipDoesNotRequireMigrationProtocols(t *testing.T) { + t.Parallel() + server, version := seededDistributionServer(t) + coordinator, ok := server.coordinator.(*distributionCoordinatorStub) + require.True(t, ok) + groups := map[uint64]*kv.ShardGroup{0: {}, 7: {Store: coordinator.store, Engine: ownershipRefreshEngine{}}} + WithMigrationOwnershipRefresh(groups, func(context.Context) (MigrationOwnershipAttestor, error) { + t.Fatal("ordinary split attempted ownership attestation") + return nil, nil + })(server) + _, err := server.SplitRange(t.Context(), &pb.SplitRangeRequest{RouteId: 1, SplitKey: []byte("g"), ExpectedCatalogVersion: version}) + require.NoError(t, err) +} + +func TestMigrationResplitRejectsUnsupportedMemberBeforeCatalogChange(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + first := r.migrate(t, 1, 2, "m") + owner := r.ownership(t, 2)[0] + before, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + indices := make(map[uint64]uint64) + for id, client := range r.clients { + proposer, ok := client.internal.migrationProposer.(*ownershipRuntimeProposer) + require.True(t, ok) + indices[id] = proposer.index + } + r.clients[2].internal.migrationPartitionProtocolReady = func() bool { return false } + _, err = r.client.SplitRange(t.Context(), &pb.SplitRangeRequest{RouteId: owner.RouteID, SplitKey: []byte("t"), ExpectedCatalogVersion: before.Version}) + require.Error(t, err) + after, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + require.Equal(t, before.Version, after.Version) + require.Equal(t, before.Routes, after.Routes) + for id, client := range r.clients { + proposer, ok := client.internal.migrationProposer.(*ownershipRuntimeProposer) + require.True(t, ok) + require.Equal(t, indices[id], proposer.index, "all static checks must precede even namespace activation") + } + require.NoError(t, r.put(t, 2, "n", "retained", first.FenceTS+10)) + r.read(t, "n", "retained") + _, exists, err := r.server.catalog.OwnershipPartitionAt(t.Context(), owner.RouteID, owner.DescriptorHash, after.ReadTS) + require.NoError(t, err) + require.False(t, exists) +} diff --git a/adapter/split_partition_reconcile.go b/adapter/split_partition_reconcile.go new file mode 100644 index 000000000..87b6b4ac4 --- /dev/null +++ b/adapter/split_partition_reconcile.go @@ -0,0 +1,73 @@ +package adapter + +import ( + "context" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +func (s *DistributionServer) prepareSplitRangePartitions(ctx context.Context, snapshot distribution.CatalogSnapshot, parentID uint64) ([]uint64, error) { + if s.ownershipRefresh == nil { + return nil, nil + } + parent, found := findRouteByID(snapshot.Routes, parentID) + if !found { + return nil, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + active, err := s.splitRangeGroupHasOwnership(ctx, parent, s.ownershipRefresh.groups[parent.GroupID]) + if err != nil { + return nil, err + } + groups, err := s.splitRangeOwnerGroups(ctx, snapshot, parent, active) + if err != nil || len(groups) == 0 { + return nil, err + } + if s.splitJobRunnerReadinessGate == nil { + return nil, errors.WithStack(store.ErrNotSupported) + } + if err := s.splitJobRunnerReadinessGate(ctx); err != nil { + return nil, err + } + if err := s.ensureMigrationNamespaces(ctx, groups); err != nil { + return nil, err + } + return groups, s.reconcileSplitRangePartitions(ctx, groups) +} + +func (s *DistributionServer) splitRangeGroupHasOwnership(ctx context.Context, parent distribution.RouteDescriptor, group *kv.ShardGroup) (bool, error) { + view, err := s.splitPartitionAdmissionView(ctx, parent.GroupID, group) + if err != nil { + return false, errors.WithStack(err) + } + active := splitRangeOwnershipIntersects(view, parent) + if active && !s.MigrationPartitionRuntimeConfigured(parent.GroupID) { + return false, errors.WithStack(store.ErrNotSupported) + } + return active, nil +} + +func splitRangeOwnershipIntersects(view store.MigrationServingState, parent distribution.RouteDescriptor) bool { + for _, proof := range view.Ownership { + if routeRangeIntersects(proof.RouteStart, proof.RouteEnd, parent.Start, parent.End) { + return true + } + } + for _, partition := range view.Partitions.Partitions { + if routeRangeIntersects(partition.RouteStart, partition.RouteEnd, parent.Start, parent.End) { + return true + } + } + return false +} + +func (s *DistributionServer) reconcileSplitRangePartitions(ctx context.Context, groups []uint64) error { + for _, groupID := range groups { + if err := s.reconcileSplitServingOwners(ctx, groupID); err != nil { + return err + } + } + return nil +} diff --git a/adapter/split_partition_target_budget_test.go b/adapter/split_partition_target_budget_test.go new file mode 100644 index 000000000..f2c7164b0 --- /dev/null +++ b/adapter/split_partition_target_budget_test.go @@ -0,0 +1,127 @@ +package adapter + +import ( + "bytes" + "encoding/json" + "reflect" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSplitWorkingSourceRejectsHistoricalTargetControlBudget(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + r.migrate(t, 1, 2, "m") + r.migrate(t, r.ownership(t, 2)[0].RouteID, 3, "t") + require.NoError(t, r.server.RunMigrationOwnershipRefreshOnce(t.Context())) + snapshot, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + parentID := r.ownership(t, 3)[0].RouteID + parent, found := findRouteByID(snapshot.Routes, parentID) + require.True(t, found) + job := distribution.SplitJob{JobID: 100, SourceRouteID: parentID, SourceGroupID: 3, + TargetGroupID: 1, Phase: distribution.SplitJobPhasePlanned} + source := targetBudgetNativePriors(t, r.groups[3].Store, parent, store.MigrationOwner) + target := targetBudgetNativePriors(t, r.groups[1].Store, parent, store.MigrationNonOwner) + leftID, rightID, err := r.server.allocateChildRouteIDs(t.Context(), snapshot.ReadTS, snapshot.Routes) + require.NoError(t, err) + nextRouteID, err := r.server.catalog.NextRouteIDAt(t.Context(), snapshot.ReadTS) + require.NoError(t, err) + measure := func(size int) ([]byte, kv.MigrationPartitionControl, kv.MigrationPartitionControl, int, int) { + t.Helper() + key := append([]byte("x"), bytes.Repeat([]byte("k"), size)...) + left, right := splitCatalogRoutes(parent, key, leftID, rightID, parent.SplitAtHLC+1) + manifest, encodeErr := distribution.EncodeOwnershipPartitionManifest(distribution.OwnershipPartitionManifest{ + PreviousVersion: snapshot.Version, Version: snapshot.Version + 1, JobID: job.JobID, + Parent: parent, Children: []distribution.RouteDescriptor{left, right}}) + require.NoError(t, encodeErr) + sourceControl := targetBudgetControl(t, source, manifest, snapshot.Version+1) + targetControl := targetBudgetControl(t, target, manifest, snapshot.Version+1) + sourceJSON, marshalErr := json.Marshal(sourceControl) + require.NoError(t, marshalErr) + targetJSON, marshalErr := json.Marshal(targetControl) + require.NoError(t, marshalErr) + sourceMaximum, sizeErr := maximumSplitPartitionControlSize(sourceJSON) + require.NoError(t, sizeErr) + targetMaximum, sizeErr := maximumSplitPartitionControlSize(targetJSON) + require.NoError(t, sizeErr) + return key, sourceControl, targetControl, sourceMaximum, targetMaximum + } + _, _, _, sourceSize, targetSize := measure(0) + require.Greater(t, targetSize, sourceSize, "historical target must have the larger valid native envelope") + limit := kv.MaxMigrationPartitionControlBytes + lo, hi := 0, 700<<10 + for lo < hi { + mid := lo + (hi-lo+1)/2 + _, _, _, size, _ := measure(mid) + if size <= limit { + lo = mid + } else { + hi = mid - 1 + } + } + key, sourceControl, targetControl, sourceSize, targetSize := measure(lo) + job.SplitKey = key + require.LessOrEqual(t, sourceSize, limit) + require.Greater(t, targetSize, limit) + sourceJSON, err := kv.EncodeMigrationPartitionControl(sourceControl) + require.NoError(t, err) + targetJSON, err := kv.EncodeMigrationPartitionControl(targetControl) + require.NoError(t, err, "target rejection must concern the growth reserve, not malformed evidence") + require.NoError(t, validateSplitPartitionControlSize(sourceControl)) + require.ErrorIs(t, validateSplitPartitionControlSize(targetControl), kv.ErrInvalidRequest) + t.Logf("split key bytes=%d source encoded=%d maximum=%d target encoded=%d maximum=%d limit=%d", + len(job.SplitKey), len(sourceJSON), sourceSize, len(targetJSON), targetSize, limit) + require.NoError(t, r.server.catalog.CreateSplitJob(t.Context(), job)) + _, err = r.server.ensureSplitWorkingSource(t.Context(), job) + assert.ErrorIs(t, err, kv.ErrInvalidRequest, "the required historical target control must be budgeted before root geometry commits") + after, loadErr := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, loadErr) + assert.Equal(t, snapshot.Version, after.Version, "target budget rejection must not advance the catalog") + assert.True(t, reflect.DeepEqual(snapshot.Routes, after.Routes), "target budget rejection must not publish children") + afterNextRouteID, loadErr := r.server.catalog.NextRouteIDAt(t.Context(), after.ReadTS) + require.NoError(t, loadErr) + assert.Equal(t, nextRouteID, afterNextRouteID, "target budget rejection must not allocate route IDs") + stored, exists, loadErr := r.server.catalog.SplitJob(t.Context(), job.JobID) + require.NoError(t, loadErr) + require.True(t, exists) + assert.Zero(t, stored.WorkingSourceRouteID, "target budget rejection must not publish working-source allocation") +} + +type targetBudgetPriors struct { + partitions []store.MigrationOwnershipPartition + legacy []store.MigrationOwnershipState + configuration uint64 +} + +func targetBudgetNativePriors(t *testing.T, st store.MVCCStore, parent distribution.RouteDescriptor, role store.MigrationOwnershipRole) targetBudgetPriors { + t.Helper() + reader, ok := st.(store.MigrationServingAdmissionReader) + require.True(t, ok) + view, err := reader.MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + partitions, legacy := splitPartitionBudgetPriors(view, parent) + require.NotZero(t, len(partitions)+len(legacy)) + for _, partition := range partitions { + require.True(t, store.MigrationOwnershipPartitionEvidenceValid(view.Partitions, partition)) + require.Equal(t, role, partition.Role) + } + for _, old := range legacy { + require.True(t, store.MigrationOwnershipEvidenceValid(old)) + require.Equal(t, role, old.Role) + } + configuration, err := kv.MigrationConfigurationIndex(t.Context(), st) + require.NoError(t, err) + return targetBudgetPriors{partitions: partitions, legacy: legacy, configuration: configuration} +} + +func targetBudgetControl(t *testing.T, priors targetBudgetPriors, manifest []byte, version uint64) kv.MigrationPartitionControl { + t.Helper() + control, err := kv.BuildMigrationPartitionControl(priors.partitions, priors.legacy, manifest, nil, version, priors.configuration) + require.NoError(t, err, "both controls must have complete valid native prior evidence") + return control +} diff --git a/adapter/split_partition_unrelated_owners_test.go b/adapter/split_partition_unrelated_owners_test.go new file mode 100644 index 000000000..f121eccbb --- /dev/null +++ b/adapter/split_partition_unrelated_owners_test.go @@ -0,0 +1,56 @@ +package adapter + +import ( + "bytes" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" +) + +func TestSplitRangeRefreshesDisjointNativeOwnersBeforeReturning(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + r.migrate(t, 1, 2, "m") + r.migrate(t, r.ownership(t, 2)[0].RouteID, 3, "t") + before, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + parent := splitUnrelatedOwnerParent(t, before) + _, err = r.client.SplitRange(t.Context(), &pb.SplitRangeRequest{RouteId: parent, + SplitKey: []byte("g"), ExpectedCatalogVersion: before.Version}) + require.NoError(t, err) + // Neither native owner shares the split parent's group. No background + // refresh runs between catalog publication and these foreground writes. + require.NoError(t, r.put(t, 2, "n", "disjoint-owner-two", 1<<40)) + require.NoError(t, r.put(t, 3, "y", "disjoint-owner-three", 1<<40)) + r.read(t, "n", "disjoint-owner-two") + r.read(t, "y", "disjoint-owner-three") +} + +func TestSplitRangeRequiresDisjointOwnerSupportBeforeCatalogMutation(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + r.migrate(t, 1, 2, "m") + r.migrate(t, r.ownership(t, 2)[0].RouteID, 3, "t") + before, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + parent := splitUnrelatedOwnerParent(t, before) + r.clients[3].internal.migrationPartitionProtocolReady = func() bool { return false } + _, err = r.client.SplitRange(t.Context(), &pb.SplitRangeRequest{RouteId: parent, + SplitKey: []byte("g"), ExpectedCatalogVersion: before.Version}) + require.Error(t, err) + after, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + require.Equal(t, before.Version, after.Version) + require.Equal(t, before.Routes, after.Routes) +} + +func splitUnrelatedOwnerParent(t *testing.T, snapshot distribution.CatalogSnapshot) uint64 { + t.Helper() + for _, route := range snapshot.Routes { + if route.GroupID == 1 && bytes.Equal(route.End, []byte("m")) { + return route.RouteID + } + } + t.Fatal("retained proof-free parent is absent") + return 0 +} diff --git a/adapter/split_prepare_evidence_auth_integration_test.go b/adapter/split_prepare_evidence_auth_integration_test.go new file mode 100644 index 000000000..f73076e2e --- /dev/null +++ b/adapter/split_prepare_evidence_auth_integration_test.go @@ -0,0 +1,59 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +func TestSplitAndPrepareEvidenceCredentialsRemainIndependent(t *testing.T) { + t.Parallel() + migration, _ := SplitMigrationTokenAuth("migration-secret", false, true) + admin, _ := AdminTokenAuth("evidence-secret") + leader := &prepareSafetyRPCLeader{} + server := NewInternalWithEngine(nil, leader, nil, nil, WithInternalStore(store.NewMVCCStore()), WithInternalPrepareEvidenceToken("evidence-secret")) + for _, tc := range []struct { + name, method, token string + request any + allowed bool + }{ + {"evidence missing", pb.Internal_GetPrepareEvidenceState_FullMethodName, "", &pb.PrepareEvidenceStateRequest{}, false}, + {"migration cannot inspect evidence", pb.Internal_GetPrepareEvidenceState_FullMethodName, "migration-secret", &pb.PrepareEvidenceStateRequest{}, false}, + {"evidence peer", pb.Internal_GetPrepareEvidenceState_FullMethodName, "evidence-secret", &pb.PrepareEvidenceStateRequest{}, true}, + {"admin cannot inject jobs", pb.Internal_Forward_FullMethodName, "evidence-secret", &pb.ForwardRequest{Requests: []*pb.Request{{Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: distribution.CatalogSplitJobKey(7)}}}}}, false}, + {"migration peer", pb.Internal_Forward_FullMethodName, "migration-secret", &pb.ForwardRequest{Requests: []*pb.Request{{Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: distribution.CatalogSplitJobKey(7)}}}}}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := metadata.NewIncomingContext(t.Context(), metadata.Pairs("authorization", "Bearer "+tc.token)) + ctx = peer.NewContext(ctx, &peer.Peer{AuthInfo: credentials.TLSInfo{}}) + info := &grpc.UnaryServerInfo{FullMethod: tc.method} + called := false + _, err := migration(ctx, tc.request, info, func(ctx context.Context, request any) (any, error) { + return admin(ctx, request, info, func(ctx context.Context, _ any) (any, error) { + called = true + if tc.method == pb.Internal_GetPrepareEvidenceState_FullMethodName { + return server.GetPrepareEvidenceState(ctx, &pb.PrepareEvidenceStateRequest{}) + } + return nil, nil + }) + }) + require.Equal(t, tc.allowed, called) + if tc.allowed { + require.NoError(t, err) + } else { + require.Equal(t, codes.Unauthenticated, status.Code(err)) + } + }) + } + require.Equal(t, int32(1), leader.reads.Load(), "only the evidence credential may trigger quorum inspection") +} diff --git a/adapter/split_working_source_recovery_test.go b/adapter/split_working_source_recovery_test.go new file mode 100644 index 000000000..bb305d410 --- /dev/null +++ b/adapter/split_working_source_recovery_test.go @@ -0,0 +1,87 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type splitRecoveryOwnerFailure struct { + MigrationOwnershipAttestor + catalog *distribution.CatalogStore + version uint64 + blocked bool + calls int +} + +func (c *splitRecoveryOwnerFailure) AttestMigrationOwnership(ctx context.Context, request *pb.AttestMigrationOwnershipRequest, options ...grpc.CallOption) (*pb.AttestMigrationOwnershipResponse, error) { + id := request.GetPrior().GetLocalGroupId() + if id == 0 { + partitions, legacy, err := decodeMigrationPartitionPriors(request) + if err != nil { + return nil, err + } + prior, _ := migrationPartitionAttestationPrior(partitions, legacy) + id = prior.LocalGroupID + } + snapshot, err := c.catalog.Snapshot(ctx) + if err != nil { + return nil, err + } + if id == 2 && snapshot.Version > c.version && c.blocked { + c.calls++ + return nil, status.Error(codes.Unavailable, "injected post-commit disjoint owner outage") + } + return c.MigrationOwnershipAttestor.AttestMigrationOwnership(ctx, request, options...) +} + +func TestSplitWorkingSourceRecoveryRetriesDisjointOwnerBarrier(t *testing.T) { + r := newResplitAcceptanceRuntime(t) + r.migrate(t, 1, 2, "m") + r.migrate(t, r.ownership(t, 2)[0].RouteID, 3, "t") + require.NoError(t, r.server.RunMigrationOwnershipRefreshOnce(t.Context())) + before, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + started, err := r.client.StartSplitMigration(t.Context(), &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: before.Version, RouteId: r.ownership(t, 3)[0].RouteID, + TargetGroupId: 1, SplitKey: []byte("x")}) + require.NoError(t, err) + job, exists, err := r.server.catalog.SplitJob(t.Context(), started.GetJobId()) + require.NoError(t, err) + require.True(t, exists) + failure := &splitRecoveryOwnerFailure{MigrationOwnershipAttestor: r.client, catalog: r.server.catalog, version: before.Version, blocked: true} + r.server.ownershipRefresh.root = func(context.Context) (MigrationOwnershipAttestor, error) { return failure, nil } + _, err = r.server.ensureSplitWorkingSource(t.Context(), job) + require.Error(t, err) + require.Positive(t, failure.calls, "the failure must occur after root geometry commits; error: %v", err) + stored, exists, err := r.server.catalog.SplitJob(t.Context(), job.JobID) + require.NoError(t, err) + require.True(t, exists) + require.NotZero(t, stored.WorkingSourceRouteID) + committed, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + require.Greater(t, committed.Version, before.Version) + next, err := r.server.catalog.NextRouteIDAt(t.Context(), committed.ReadTS) + require.NoError(t, err) + _, err = r.server.ensureSplitWorkingSource(t.Context(), stored) + require.Error(t, err, "a persisted route ID is not a completed owner-ACK barrier") + failure.blocked = false + resumed, err := r.server.ensureSplitWorkingSource(t.Context(), stored) + require.NoError(t, err) + require.Equal(t, stored.WorkingSourceRouteID, resumed.WorkingSourceRouteID) + after, err := r.server.catalog.Snapshot(t.Context()) + require.NoError(t, err) + require.Equal(t, committed.Version, after.Version) + require.Equal(t, committed.Routes, after.Routes) + afterNext, err := r.server.catalog.NextRouteIDAt(t.Context(), after.ReadTS) + require.NoError(t, err) + require.Equal(t, next, afterNext) + require.NoError(t, r.put(t, 2, "n", "recovered-disjoint-owner", 1<<40)) + r.read(t, "n", "recovered-disjoint-owner") +} diff --git a/adapter/sqs_partitioned_cleanup_alignment_test.go b/adapter/sqs_partitioned_cleanup_alignment_test.go new file mode 100644 index 000000000..34167dde4 --- /dev/null +++ b/adapter/sqs_partitioned_cleanup_alignment_test.go @@ -0,0 +1,42 @@ +package adapter + +import ( + "testing" + + "github.com/bootjp/elastickv/kv" + "github.com/stretchr/testify/require" +) + +// Migration cleanup runs inside FSM apply and cannot consult the partition +// resolver, so it recognises resolver-owned rows structurally via +// kv.IsPartitionedSQSKey. That predicate carries its own copy of the family +// prefixes, so it has to be pinned against the keys the adapter actually +// writes: a family renamed or added here without updating kv would leave those +// rows exposed to deletion by a catalog route cleanup that never owned them. +func TestPartitionedSQSPrefixesAlign(t *testing.T) { + t.Parallel() + + const ( + queue = "q" + partition = uint32(3) + gen = uint64(1) + ) + for name, key := range map[string][]byte{ + "data": sqsPartitionedMsgDataKey(queue, partition, gen, "m1"), + "vis": sqsPartitionedMsgVisKey(queue, partition, gen, 100, "m1"), + "dedup": sqsPartitionedMsgDedupKey(queue, partition, gen, "g1", "d1"), + "group": sqsPartitionedMsgGroupKey(queue, partition, gen, "g1"), + "byage": sqsPartitionedMsgByAgeKey(queue, partition, gen, 100, "m1"), + } { + require.True(t, kv.IsPartitionedSQSKey(key), "%s must be recognised as partitioned", name) + } + + // The legacy, catalog-routed layouts must stay outside the guard, or + // cleanup would stop reclaiming the rows it does own. + for name, key := range map[string][]byte{ + "legacy data": sqsMsgDataKey(queue, gen, "m1"), + "queue meta": sqsQueueMetaKey(queue), + } { + require.False(t, kv.IsPartitionedSQSKey(key), "%s must not be recognised as partitioned", name) + } +} diff --git a/cmd/elastickv-list-routes/main.go b/cmd/elastickv-list-routes/main.go index 75679b6f4..662b36a7e 100644 --- a/cmd/elastickv-list-routes/main.go +++ b/cmd/elastickv-list-routes/main.go @@ -41,6 +41,7 @@ import ( "os" "time" + internalutil "github.com/bootjp/elastickv/internal" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" "google.golang.org/grpc" @@ -49,7 +50,12 @@ import ( const rpcTimeout = 10 * time.Second -var address = flag.String("address", "127.0.0.1:50051", "gRPC address of an elastickv server (any node works — the Distribution service is replicated)") +var ( + address = flag.String("address", "127.0.0.1:50051", "gRPC address of an elastickv server (any node works — the Distribution service is replicated)") + tlsEnabled = flag.Bool("tls", false, "Use verified TLS with system roots unless tls-ca-file is provided") + tlsCAFile = flag.String("tls-ca-file", "", "PEM CA certificates for verified gRPC TLS; implies TLS") + tlsServerName = flag.String("tls-server-name", "", "Expected TLS server name when different from the address; implies TLS") +) // routeJSON is the on-the-wire shape this tool emits. Field names // are snake_case to match the proto's wire format; Start/End are @@ -76,7 +82,11 @@ func main() { } func run() error { - conn, err := grpc.NewClient(*address, grpc.WithTransportCredentials(insecure.NewCredentials())) + opts, err := routesDialOptions() + if err != nil { + return err + } + conn, err := grpc.NewClient(*address, opts...) if err != nil { return errors.Wrapf(err, "dial %s", *address) } @@ -100,6 +110,22 @@ func run() error { return emit(resp, os.Stdout) } +func routesDialOptions() ([]grpc.DialOption, error) { + opts := []grpc.DialOption{internalutil.GRPCCallOptions()} + if !*tlsEnabled && *tlsCAFile == "" && *tlsServerName == "" { + return append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())), nil + } + creds, err := internalutil.GRPCClientTLSCredentials(*tlsCAFile) + if err != nil { + return nil, errors.Wrap(err, "load route-list TLS credentials") + } + opts = append(opts, grpc.WithTransportCredentials(creds)) + if *tlsServerName != "" { + opts = append(opts, grpc.WithAuthority(*tlsServerName)) + } + return opts, nil +} + // emit serialises resp as JSON to w. Extracted from run() so the // encoding is testable with bytes.Buffer rather than a real // temp file (review feedback on PR #925). io.Writer is the diff --git a/cmd/elastickv-list-routes/tls_test.go b/cmd/elastickv-list-routes/tls_test.go new file mode 100644 index 000000000..c2b108ee4 --- /dev/null +++ b/cmd/elastickv-list-routes/tls_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "crypto/tls" + "encoding/pem" + "net" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +type routesTestServer struct { + pb.UnimplementedDistributionServer +} + +func (*routesTestServer) ListRoutes(context.Context, *pb.ListRoutesRequest) (*pb.ListRoutesResponse, error) { + return &pb.ListRoutesResponse{CatalogVersion: 7}, nil +} + +func TestRoutesDialOptions(t *testing.T) { + oldTLS, oldCA, oldName := *tlsEnabled, *tlsCAFile, *tlsServerName + t.Cleanup(func() { *tlsEnabled, *tlsCAFile, *tlsServerName = oldTLS, oldCA, oldName }) + fixture := httptest.NewTLSServer(nil) + fixture.Close() + caPath := filepath.Join(t.TempDir(), "ca.pem") + require.NoError(t, os.WriteFile(caPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: fixture.Certificate().Raw}), 0o600)) + for _, tc := range []struct { + name string + serverTLS bool + clientTLS bool + ca string + serverName string + wantError string + }{ + {name: "plaintext default"}, + {name: "CA implies TLS", serverTLS: true, ca: caPath}, + {name: "verified server name", serverTLS: true, ca: caPath, serverName: "example.com"}, + {name: "reject wrong name", serverTLS: true, ca: caPath, serverName: "wrong.invalid", wantError: "certificate"}, + {name: "reject untrusted CA", serverTLS: true, clientTLS: true, wantError: "certificate"}, + } { + t.Run(tc.name, func(t *testing.T) { + *tlsEnabled, *tlsCAFile, *tlsServerName = tc.clientTLS, tc.ca, tc.serverName + var lc net.ListenConfig + listener, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + var serverOpts []grpc.ServerOption + if tc.serverTLS { + serverOpts = append(serverOpts, grpc.Creds(credentials.NewTLS(&tls.Config{ + MinVersion: tls.VersionTLS12, Certificates: fixture.TLS.Certificates, + }))) + } + server := grpc.NewServer(serverOpts...) + pb.RegisterDistributionServer(server, &routesTestServer{}) + serveErr := make(chan error, 1) + go func() { serveErr <- server.Serve(listener) }() + t.Cleanup(func() { + server.Stop() + require.NoError(t, <-serveErr) + }) + opts, err := routesDialOptions() + require.NoError(t, err) + conn, err := grpc.NewClient(listener.Addr().String(), opts...) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + resp, err := pb.NewDistributionClient(conn).ListRoutes(ctx, &pb.ListRoutesRequest{}) + if tc.wantError != "" { + require.ErrorContains(t, err, tc.wantError) + return + } + require.NoError(t, err) + require.Equal(t, uint64(7), resp.GetCatalogVersion()) + }) + } + *tlsCAFile = filepath.Join(t.TempDir(), "missing.pem") + _, err := routesDialOptions() + require.ErrorContains(t, err, "CA file") +} diff --git a/cmd/elastickv-split/main.go b/cmd/elastickv-split/main.go index 767359c71..6728c4ff6 100644 --- a/cmd/elastickv-split/main.go +++ b/cmd/elastickv-split/main.go @@ -27,6 +27,7 @@ import ( "os" "time" + internalutil "github.com/bootjp/elastickv/internal" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" "google.golang.org/grpc" @@ -34,7 +35,8 @@ import ( ) const ( - rpcTimeout = 10 * time.Second + rpcTimeout = 10 * time.Second + maxMigrationTokenBytes = 4 << 10 ) var ( @@ -42,6 +44,13 @@ var ( routeID = flag.Uint64("route-id", 0, "RouteID of the route to split (required)") splitKey = flag.String("split-key", "", "Split key — must lie strictly inside the route's [Start, End) range; rejected if == Start or == End by validateSplitKey (required)") expectedVersion = flag.Uint64("expected-version", 0, "Expected catalog version for OCC; obtain by calling ListRoutes first (required, must be >= 1 — catalog version is 1-based)") + targetGroupID = flag.Uint64("target-group-id", 0, "Target Raft group for an asynchronous cross-group migration; zero uses the existing same-group split") + abandonJobID = flag.Uint64("abandon-job-id", 0, "Abandon a pre-cutover migration job; mutually exclusive with split flags") + retryJobID = flag.Uint64("retry-job-id", 0, "Resume a FAILED migration at its recorded phase; mutually exclusive with other job/split flags") + getJobID = flag.Uint64("get-job-id", 0, "Print one migration job; mutually exclusive with mutation flags") + tokenFile = flag.String("token-file", "", "File containing the dedicated migration controller bearer token; not the read-only Admin token") + tlsCAFile = flag.String("tls-ca-file", "", "PEM CA certificates for verified gRPC TLS; token-file always requires TLS (system roots if omitted)") + tlsServerName = flag.String("tls-server-name", "", "Expected TLS server name when different from the address; implies verified TLS") ) func main() { @@ -57,7 +66,11 @@ func run() error { return err } - conn, err := grpc.NewClient(*address, grpc.WithTransportCredentials(insecure.NewCredentials())) + dialOpts, err := splitDialOptions() + if err != nil { + return err + } + conn, err := grpc.NewClient(*address, dialOpts...) if err != nil { return errors.Wrapf(err, "dial %s", *address) } @@ -76,7 +89,19 @@ func run() error { rpcCtx, rpcCancel := context.WithTimeout(context.Background(), rpcTimeout) defer rpcCancel() + if *getJobID != 0 { + return runGetSplitJob(rpcCtx, client) + } + if *abandonJobID != 0 { + return runAbandonSplitJob(rpcCtx, client) + } + if *retryJobID != 0 { + return runRetrySplitJob(rpcCtx, client) + } + if *targetGroupID != 0 { + return runStartSplitMigration(rpcCtx, client) + } req := &pb.SplitRangeRequest{ ExpectedCatalogVersion: *expectedVersion, RouteId: *routeID, @@ -90,7 +115,74 @@ func run() error { return nil } +func splitDialOptions() ([]grpc.DialOption, error) { + opts := []grpc.DialOption{internalutil.GRPCCallOptions()} + if *tokenFile == "" && *tlsCAFile == "" && *tlsServerName == "" { + return append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())), nil + } + creds, err := internalutil.GRPCClientTLSCredentials(*tlsCAFile) + if err != nil { + return nil, errors.Wrap(err, "load split TLS credentials") + } + opts = append(opts, grpc.WithTransportCredentials(creds)) + if *tlsServerName != "" { + opts = append(opts, grpc.WithAuthority(*tlsServerName)) + } + if *tokenFile != "" { + token, err := internalutil.LoadBearerTokenFile(*tokenFile, maxMigrationTokenBytes, "migration token") + if err != nil { + return nil, errors.Wrap(err, "load split migration token") + } + opts = append(opts, grpc.WithPerRPCCredentials(internalutil.GRPCBearerToken(token))) + } + return opts, nil +} + +func runGetSplitJob(ctx context.Context, client pb.DistributionClient) error { + resp, err := client.GetSplitJob(ctx, &pb.GetSplitJobRequest{JobId: *getJobID}) + if err != nil { + return errors.Wrap(err, "GetSplitJob") + } + job := resp.GetJob() + fmt.Printf("job_id: %d\nphase: %s\n", job.GetJobId(), job.GetPhase()) + return nil +} + +func runAbandonSplitJob(ctx context.Context, client pb.DistributionClient) error { + if _, err := client.AbandonSplitJob(ctx, &pb.AbandonSplitJobRequest{JobId: *abandonJobID}); err != nil { + return errors.Wrap(err, "AbandonSplitJob") + } + fmt.Printf("abandoned_job_id: %d\n", *abandonJobID) + return nil +} + +func runRetrySplitJob(ctx context.Context, client pb.DistributionClient) error { + if _, err := client.RetrySplitJob(ctx, &pb.RetrySplitJobRequest{JobId: *retryJobID}); err != nil { + return errors.Wrap(err, "RetrySplitJob") + } + fmt.Printf("retried_job_id: %d\n", *retryJobID) + return nil +} + +func runStartSplitMigration(ctx context.Context, client pb.DistributionClient) error { + resp, err := client.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: *expectedVersion, + RouteId: *routeID, + SplitKey: []byte(*splitKey), + TargetGroupId: *targetGroupID, + }) + if err != nil { + return errors.Wrap(err, "StartSplitMigration") + } + fmt.Printf("catalog_version: %d\njob_id: %d\n", resp.GetCatalogVersion(), resp.GetJobId()) + return nil +} + func validateFlags() error { + jobMode, err := validateJobModeFlags() + if err != nil || jobMode { + return err + } switch { case *routeID == 0: return errors.New("--route-id is required and must be > 0") @@ -102,6 +194,26 @@ func validateFlags() error { return nil } +func validateJobModeFlags() (bool, error) { + modes := 0 + for _, id := range []uint64{*getJobID, *abandonJobID, *retryJobID} { + if id != 0 { + modes++ + } + } + if modes == 0 { + return false, nil + } + if modes > 1 || splitMutationFlagsSet() { + return false, errors.New("--get-job-id, --abandon-job-id and --retry-job-id are mutually exclusive with each other and split flags") + } + return true, nil +} + +func splitMutationFlagsSet() bool { + return *routeID != 0 || *splitKey != "" || *expectedVersion != 0 || *targetGroupID != 0 +} + func printResponse(resp *pb.SplitRangeResponse) { fmt.Printf("catalog_version: %d\n", resp.GetCatalogVersion()) printRoute("left: ", resp.GetLeft()) diff --git a/cmd/elastickv-split/main_test.go b/cmd/elastickv-split/main_test.go index bc9d40769..c2126309c 100644 --- a/cmd/elastickv-split/main_test.go +++ b/cmd/elastickv-split/main_test.go @@ -1,11 +1,22 @@ package main import ( + "context" + "crypto/tls" + "encoding/pem" "flag" + "net" + "net/http/httptest" + "os" + "path/filepath" "testing" + "github.com/bootjp/elastickv/adapter" + pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" ) // TestValidateFlags_RejectsMissingRequired pins the smoke-test @@ -48,6 +59,48 @@ func TestValidateFlags_AcceptsAllPresent(t *testing.T) { require.NoError(t, validateFlags()) } +func TestValidateFlags_AcceptsAbandonOnly(t *testing.T) { + resetFlags(t) + *abandonJobID = 42 + require.NoError(t, validateFlags()) +} + +func TestValidateFlags_RejectsAbandonWithSplitFlags(t *testing.T) { + resetFlags(t) + *abandonJobID = 42 + *routeID = 7 + err := validateFlags() + require.ErrorContains(t, err, "mutually exclusive") +} + +func TestValidateFlags_AcceptsGetOnly(t *testing.T) { + resetFlags(t) + *getJobID = 42 + require.NoError(t, validateFlags()) +} + +func TestValidateFlags_RetryIsExclusive(t *testing.T) { + resetFlags(t) + t.Cleanup(func() { resetFlags(t) }) + *retryJobID = 42 + require.NoError(t, validateFlags()) + for _, other := range []*uint64{routeID, expectedVersion, targetGroupID, abandonJobID, getJobID} { + *other = 1 + require.ErrorContains(t, validateFlags(), "mutually exclusive") + *other = 0 + } + *splitKey = "m" + require.ErrorContains(t, validateFlags(), "mutually exclusive") +} + +func TestValidateFlags_RejectsGetWithMutationFlags(t *testing.T) { + resetFlags(t) + *getJobID = 42 + *abandonJobID = 42 + err := validateFlags() + require.ErrorContains(t, err, "mutually exclusive") +} + // resetFlags reinitialises the package-level flag pointers around // each test so subtests don't leak state through them. We rebind // the globals directly rather than re-parsing argv because the @@ -58,12 +111,77 @@ func resetFlags(t *testing.T) { // init). errors.New rather than t.Fatalf so a future // refactor that drops the package-level flags surfaces here // rather than silently no-oping the resets. - if routeID == nil || splitKey == nil || expectedVersion == nil || address == nil { + if routeID == nil || splitKey == nil || expectedVersion == nil || targetGroupID == nil || abandonJobID == nil || getJobID == nil || address == nil { t.Fatal(errors.New("package-level flag pointers were not initialised")) } *routeID = 0 *splitKey = "" *expectedVersion = 0 + *targetGroupID = 0 + *abandonJobID = 0 + *retryJobID = 0 + *getJobID = 0 *address = "127.0.0.1:50051" + *tokenFile = "" + *tlsCAFile = "" + *tlsServerName = "" _ = flag.CommandLine // touch to avoid unused import on future trims } + +type splitAuthTestServer struct { + pb.UnimplementedDistributionServer + jobs chan uint64 +} + +func (s *splitAuthTestServer) AbandonSplitJob(_ context.Context, req *pb.AbandonSplitJobRequest) (*pb.AbandonSplitJobResponse, error) { + s.jobs <- req.GetJobId() + return &pb.AbandonSplitJobResponse{}, nil +} + +func (s *splitAuthTestServer) RetrySplitJob(_ context.Context, req *pb.RetrySplitJobRequest) (*pb.RetrySplitJobResponse, error) { + s.jobs <- req.GetJobId() + return &pb.RetrySplitJobResponse{}, nil +} + +func TestRunSendsMigrationControllerToken(t *testing.T) { + resetFlags(t) + t.Cleanup(func() { resetFlags(t) }) + var listenConfig net.ListenConfig + listener, err := listenConfig.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + unary, _ := adapter.SplitMigrationTokenAuth("test-migration-token", false, true) + tlsFixture := httptest.NewTLSServer(nil) + tlsFixture.Close() + caPath := filepath.Join(t.TempDir(), "ca.pem") + require.NoError(t, os.WriteFile(caPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: tlsFixture.Certificate().Raw}), 0o600)) + *tlsCAFile = caPath + server := grpc.NewServer(grpc.UnaryInterceptor(unary), grpc.Creds(credentials.NewTLS(&tls.Config{ + MinVersion: tls.VersionTLS12, Certificates: tlsFixture.TLS.Certificates, + }))) + svc := &splitAuthTestServer{jobs: make(chan uint64, 1)} + pb.RegisterDistributionServer(server, svc) + serveResult := make(chan error, 1) + go func() { serveResult <- server.Serve(listener) }() + t.Cleanup(func() { + server.Stop() + require.NoError(t, <-serveResult) + }) + *address = listener.Addr().String() + *abandonJobID = 7 + err = run() + require.ErrorContains(t, err, "Unauthenticated") + require.Empty(t, svc.jobs) + path := filepath.Join(t.TempDir(), "token") + require.NoError(t, os.WriteFile(path, []byte("test-migration-token\n"), 0o600)) + *tokenFile = path + *tlsServerName = "wrong.invalid" + require.ErrorContains(t, run(), "certificate") + require.Empty(t, svc.jobs) + *tlsServerName = "example.com" + require.NoError(t, run()) + require.Equal(t, uint64(7), <-svc.jobs) + *abandonJobID = 0 + *retryJobID = 8 + require.NoError(t, run()) + require.Equal(t, uint64(8), <-svc.jobs) +} diff --git a/distribution/migration_promotion_complete.go b/distribution/migration_promotion_complete.go new file mode 100644 index 000000000..aa4487a9c --- /dev/null +++ b/distribution/migration_promotion_complete.go @@ -0,0 +1,350 @@ +package distribution + +import ( + "bytes" + "context" + "math" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +var ( + ErrMigrationPromotionNotReady = errors.New("migration target promotion is not ready") + ErrMigrationPromotionTargetAbsent = errors.New("migration target promotion route is missing") +) + +// TargetPromotionCompletion is the result of applying the default-group +// promotion-complete transition to a SplitJob and its catalog routes. +type TargetPromotionCompletion struct { + Job SplitJob + Routes []RouteDescriptor + Changed bool + ClearedRouteIDs []uint64 +} + +// CompleteTargetPromotionState clears the target route's staged visibility +// fields after target-local promotion has completed. It deliberately preserves +// MinWriteTSExclusive; the timestamp floor remains a durable route invariant +// after the staged/live merge is no longer needed. +func CompleteTargetPromotionState(job SplitJob, routes []RouteDescriptor, nowMs int64) (TargetPromotionCompletion, error) { + normalized, err := normalizePromotionCompletionInput(job, routes) + if err != nil { + return TargetPromotionCompletion{}, err + } + + out := TargetPromotionCompletion{ + Job: CloneSplitJob(job), + Routes: normalized, + } + if out.Job.TargetPromotionDone { + return completeAlreadyPromotedTarget(out, nowMs) + } + + cleared, err := clearTargetPromotionRoutes(job, out.Routes) + if err != nil { + return TargetPromotionCompletion{}, err + } + if len(cleared) == 0 { + return TargetPromotionCompletion{}, errors.WithStack(ErrMigrationPromotionTargetAbsent) + } + + out.Changed = true + out.ClearedRouteIDs = cleared + out.Job.TargetPromotionDone = true + out.Job.UpdatedAtMs = nowMs + return out, nil +} + +func completeAlreadyPromotedTarget(out TargetPromotionCompletion, nowMs int64) (TargetPromotionCompletion, error) { + if !targetClearedDescriptorPresent(out.Job, out.Routes) { + return TargetPromotionCompletion{}, errors.WithStack(ErrMigrationPromotionTargetAbsent) + } + // Preserve compatibility for jobs terminalized by older binaries, but do + // not move a current CLEANUP job to DONE until source and target cleanup + // proofs have been removed by the runner. + if out.Job.Phase == SplitJobPhaseDone && out.Job.TerminalAtMs <= 0 { + out.Changed = true + out.Job.TerminalAtMs = nowMs + } + if out.Changed { + out.Job.UpdatedAtMs = nowMs + } + return out, nil +} + +func normalizePromotionCompletionInput(job SplitJob, routes []RouteDescriptor) ([]RouteDescriptor, error) { + if err := validateSplitJob(job); err != nil { + return nil, err + } + if job.Phase == SplitJobPhaseDone && job.TargetPromotionDone { + return normalizeRoutes(routes) + } + if job.Phase != SplitJobPhaseCleanup { + return nil, errors.WithStack(ErrMigrationPromotionNotReady) + } + return normalizeRoutes(routes) +} + +func clearTargetPromotionRoutes(job SplitJob, routes []RouteDescriptor) ([]uint64, error) { + cleared := make([]uint64, 0, 1) + for i := range routes { + route := &routes[i] + if route.MigrationJobID != job.JobID { + continue + } + if !route.StagedVisibilityActive || + route.GroupID != job.TargetGroupID || + route.ParentRouteID != job.SourceRouteID || + !bytes.Equal(route.Start, job.SplitKey) { + return nil, errors.WithStack(ErrMigrationInvalidRoute) + } + route.StagedVisibilityActive = false + route.MigrationJobID = 0 + cleared = append(cleared, route.RouteID) + } + return cleared, nil +} + +// CompleteSplitJobTargetPromotion applies the promotion-complete catalog CAS: +// route descriptor staged fields are cleared, catalog version is bumped, and +// the SplitJob witness is updated in the same MVCC batch. +func (s *CatalogStore) CompleteSplitJobTargetPromotion( + ctx context.Context, + expectedVersion uint64, + expected SplitJob, + nowMs int64, +) (CatalogSnapshot, SplitJob, error) { + if err := ensureCatalogStore(s); err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + ctx = contextOrBackground(ctx) + + readTS, currentVersion, routes, currentJob, alreadyApplied, err := s.loadPromotionCompleteInputs(ctx, expectedVersion, expected) + if err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + if alreadyApplied { + return CatalogSnapshot{ + Version: currentVersion, + Routes: cloneRouteDescriptors(routes), + ReadTS: readTS, + }, currentJob, nil + } + completion, err := CompleteTargetPromotionState(expected, routes, nowMs) + if err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + if !completion.Changed { + return CatalogSnapshot{ + Version: currentVersion, + Routes: cloneRouteDescriptors(completion.Routes), + ReadTS: readTS, + }, completion.Job, nil + } + + plan, mutations, commitTS, err := s.buildPromotionCompleteMutations(ctx, readTS, expectedVersion, expected.JobID, &completion) + if err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + if err := s.applyPromotionCompleteMutations(ctx, plan, mutations, expected.JobID, commitTS); err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + + return CatalogSnapshot{ + Version: plan.nextVersion, + Routes: cloneRouteDescriptors(completion.Routes), + }, completion.Job, nil +} + +func (s *CatalogStore) loadPromotionCompleteInputs(ctx context.Context, expectedVersion uint64, expected SplitJob) (uint64, uint64, []RouteDescriptor, SplitJob, bool, error) { + expectedRaw, err := EncodeSplitJob(expected) + if err != nil { + return 0, 0, nil, SplitJob{}, false, err + } + readTS := s.store.LastCommitTS() + currentVersion, err := s.versionAt(ctx, readTS) + if err != nil { + return 0, 0, nil, SplitJob{}, false, err + } + raw, currentJob, err := s.livePromotionCompleteJobAt(ctx, expected.JobID, readTS, expectedVersion, currentVersion) + if err != nil { + return 0, 0, nil, SplitJob{}, false, err + } + if currentVersion != expectedVersion || !bytes.Equal(raw, expectedRaw) { + return s.resolvePromotionCompleteInputConflict(ctx, readTS, currentVersion, expectedVersion, expected, expectedRaw, currentJob) + } + routes, err := s.routesAt(ctx, readTS) + if err != nil { + return 0, 0, nil, SplitJob{}, false, err + } + return readTS, currentVersion, routes, currentJob, false, nil +} + +func (s *CatalogStore) livePromotionCompleteJobAt(ctx context.Context, jobID uint64, ts uint64, expectedVersion uint64, currentVersion uint64) ([]byte, SplitJob, error) { + raw, err := s.store.GetAt(ctx, CatalogSplitJobKey(jobID), ts) + if err != nil { + if errors.Is(err, store.ErrKeyNotFound) { + return nil, SplitJob{}, promotionCompleteMissingJobError(expectedVersion, currentVersion) + } + return nil, SplitJob{}, errors.WithStack(err) + } + currentJob, err := DecodeSplitJob(raw) + if err != nil { + return nil, SplitJob{}, err + } + if currentJob.JobID != jobID { + return nil, SplitJob{}, errors.WithStack(ErrCatalogSplitJobKeyIDMismatch) + } + return raw, currentJob, nil +} + +func promotionCompleteMissingJobError(expectedVersion uint64, currentVersion uint64) error { + if currentVersion != expectedVersion { + return errors.WithStack(ErrCatalogVersionMismatch) + } + return errors.WithStack(ErrCatalogSplitJobConflict) +} + +func (s *CatalogStore) resolvePromotionCompleteInputConflict( + ctx context.Context, + readTS uint64, + currentVersion uint64, + expectedVersion uint64, + expected SplitJob, + expectedRaw []byte, + currentJob SplitJob, +) (uint64, uint64, []RouteDescriptor, SplitJob, bool, error) { + alreadyApplied, routes, err := s.promotionCompleteAlreadyAppliedAt(ctx, readTS, expected, expectedRaw, currentJob) + if err != nil { + return 0, 0, nil, SplitJob{}, false, err + } + if alreadyApplied { + return readTS, currentVersion, routes, currentJob, true, nil + } + if currentVersion != expectedVersion { + return 0, 0, nil, SplitJob{}, false, errors.WithStack(ErrCatalogVersionMismatch) + } + return 0, 0, nil, SplitJob{}, false, errors.WithStack(ErrCatalogSplitJobConflict) +} + +func (s *CatalogStore) promotionCompleteAlreadyAppliedAt(ctx context.Context, ts uint64, expected SplitJob, expectedRaw []byte, current SplitJob) (bool, []RouteDescriptor, error) { + matches, err := promotionCompleteJobMatchesExpected(expected, expectedRaw, current) + if err != nil || !matches { + return false, nil, err + } + routes, err := s.routesAt(ctx, ts) + if err != nil { + return false, nil, err + } + if !targetClearedDescriptorPresent(current, routes) { + return false, nil, errors.WithStack(ErrMigrationPromotionTargetAbsent) + } + return true, routes, nil +} + +func promotionCompleteJobMatchesExpected(expected SplitJob, expectedRaw []byte, current SplitJob) (bool, error) { + if expected.TargetPromotionDone || + !current.TargetPromotionDone || + current.PromotionCompletedTS == 0 || + (current.Phase != expected.Phase && current.Phase != SplitJobPhaseDone) { + return false, nil + } + normalized := CloneSplitJob(current) + normalized.Phase = expected.Phase + normalized.TargetPromotionDone = expected.TargetPromotionDone + normalized.PromotionCompletedTS = expected.PromotionCompletedTS + normalized.TerminalAtMs = expected.TerminalAtMs + normalized.UpdatedAtMs = expected.UpdatedAtMs + raw, err := EncodeSplitJob(normalized) + if err != nil { + return false, err + } + return bytes.Equal(raw, expectedRaw), nil +} + +func (s *CatalogStore) buildPromotionCompleteMutations( + ctx context.Context, + readTS uint64, + expectedVersion uint64, + jobID uint64, + completion *TargetPromotionCompletion, +) (savePlan, []*store.KVPairMutation, uint64, error) { + if expectedVersion == math.MaxUint64 { + return savePlan{}, nil, 0, errors.WithStack(ErrCatalogVersionOverflow) + } + minCommitTS := readTS + 1 + if minCommitTS == 0 { + return savePlan{}, nil, 0, errors.WithStack(ErrCatalogVersionOverflow) + } + + plan := savePlan{ + readTS: readTS, + minCommitTS: minCommitTS, + nextVersion: expectedVersion + 1, + routes: completion.Routes, + } + mutations, err := s.buildSaveMutations(ctx, &plan) + if err != nil { + return savePlan{}, nil, 0, err + } + commitTS, err := s.commitTSForApply(plan.minCommitTS) + if err != nil { + return savePlan{}, nil, 0, err + } + if completion.Job.PromotionCompletedTS == 0 { + completion.Job.PromotionCompletedTS = commitTS + } + encodedJob, err := EncodeSplitJob(completion.Job) + if err != nil { + return savePlan{}, nil, 0, err + } + jobMutations, err := s.buildSplitJobPutMutations(ctx, readTS, CatalogSplitJobKey(jobID), encodedJob, jobID) + if err != nil { + return savePlan{}, nil, 0, err + } + return plan, append(mutations, jobMutations...), commitTS, nil +} + +func (s *CatalogStore) applyPromotionCompleteMutations(ctx context.Context, plan savePlan, mutations []*store.KVPairMutation, jobID uint64, commitTS uint64) error { + readKeys := [][]byte{ + CatalogVersionKey(), + CatalogSplitJobKey(jobID), + } + if err := s.store.ApplyMutations(ctx, mutations, readKeys, plan.readTS, commitTS); err != nil { + if errors.Is(err, store.ErrWriteConflict) { + return s.promotionCompleteWriteConflict(ctx, plan) + } + return errors.WithStack(err) + } + return nil +} + +func (s *CatalogStore) promotionCompleteWriteConflict(ctx context.Context, plan savePlan) error { + currentVersion, err := s.versionAt(ctx, s.store.LastCommitTS()) + if err != nil { + return err + } + if currentVersion != plan.nextVersion-1 { + return errors.WithStack(ErrCatalogVersionMismatch) + } + return errors.WithStack(ErrCatalogSplitJobConflict) +} + +func targetClearedDescriptorPresent(job SplitJob, routes []RouteDescriptor) bool { + for _, route := range routes { + if route.GroupID != job.TargetGroupID { + continue + } + if route.StagedVisibilityActive || route.MigrationJobID != 0 { + continue + } + if route.ParentRouteID != job.SourceRouteID { + continue + } + if bytes.Equal(route.Start, job.SplitKey) { + return true + } + } + return false +} diff --git a/distribution/migration_promotion_complete_test.go b/distribution/migration_promotion_complete_test.go new file mode 100644 index 000000000..96bfc7e57 --- /dev/null +++ b/distribution/migration_promotion_complete_test.go @@ -0,0 +1,258 @@ +package distribution + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestCompleteTargetPromotionStateClearsStagedFieldsAndRetainsFloor(t *testing.T) { + t.Parallel() + + job := promotionCompleteTestJob() + routes := promotionCompleteTestRoutes() + + result, err := CompleteTargetPromotionState(job, routes, 1000) + require.NoError(t, err) + require.True(t, result.Changed) + require.Equal(t, []uint64{3}, result.ClearedRouteIDs) + require.True(t, result.Job.TargetPromotionDone) + require.Zero(t, result.Job.PromotionCompletedTS) + require.Equal(t, int64(1000), result.Job.UpdatedAtMs) + require.Zero(t, result.Job.TerminalAtMs) + require.Equal(t, SplitJobPhaseCleanup, result.Job.Phase) + + target := routeByID(t, result.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) + require.Equal(t, uint64(42), target.ParentRouteID) + + badRoutes := promotionCompleteTestRoutes() + badRoutes[1].ParentRouteID = 777 + _, err = CompleteTargetPromotionState(job, badRoutes, 1000) + require.ErrorIs(t, err, ErrMigrationInvalidRoute) +} + +func TestCompleteTargetPromotionStateAcceptsAlreadyClearedDescriptor(t *testing.T) { + t.Parallel() + + job := promotionCompleteTestJob() + job.TargetPromotionDone = true + job.PromotionCompletedTS = 900 + job.UpdatedAtMs = 1000 + routes := promotionCompleteTestRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + + result, err := CompleteTargetPromotionState(job, routes, 1100) + require.NoError(t, err) + require.False(t, result.Changed) + require.Equal(t, SplitJobPhaseCleanup, result.Job.Phase) + require.Equal(t, uint64(900), result.Job.PromotionCompletedTS) + require.Equal(t, int64(1000), result.Job.UpdatedAtMs) + require.Zero(t, result.Job.TerminalAtMs) + + target := routeByID(t, result.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) +} + +func TestCatalogStoreCompleteSplitJobTargetPromotionCommitsRouteAndJobTogether(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) + saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) + require.NoError(t, err) + job := promotionCompleteTestJob() + require.NoError(t, cs.CreateSplitJob(ctx, job)) + before, err := cs.Snapshot(ctx) + require.NoError(t, err) + + snapshot, completed, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1000) + require.NoError(t, err) + require.Equal(t, saved.Version+1, snapshot.Version) + require.True(t, completed.TargetPromotionDone) + require.Equal(t, SplitJobPhaseCleanup, completed.Phase) + require.Greater(t, completed.PromotionCompletedTS, before.ReadTS) + require.Equal(t, int64(1000), completed.UpdatedAtMs) + + loadedJob, found, err := cs.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + assertSplitJobEqual(t, completed, loadedJob) + + loaded, err := cs.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, saved.Version+1, loaded.Version) + target := routeByID(t, loaded.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) +} + +func TestCatalogStoreCompleteSplitJobTargetPromotionRetryAfterCommit(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) + saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) + require.NoError(t, err) + job := promotionCompleteTestJob() + require.NoError(t, cs.CreateSplitJob(ctx, job)) + + firstSnapshot, firstCompleted, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1000) + require.NoError(t, err) + + retrySnapshot, retryCompleted, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 2000) + require.NoError(t, err) + require.Equal(t, firstSnapshot.Version, retrySnapshot.Version) + assertSplitJobEqual(t, firstCompleted, retryCompleted) + + loaded, err := cs.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, firstSnapshot.Version, loaded.Version) +} + +func TestCatalogStoreCompleteSplitJobTargetPromotionIsIdempotentAfterClearedDescriptor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) + routes := promotionCompleteTestRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + saved, err := cs.Save(ctx, 0, routes) + require.NoError(t, err) + job := promotionCompleteTestJob() + job.TargetPromotionDone = true + job.Phase = SplitJobPhaseCleanup + job.PromotionCompletedTS = 900 + job.UpdatedAtMs = 1000 + require.NoError(t, cs.CreateSplitJob(ctx, job)) + + snapshot, completed, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1100) + require.NoError(t, err) + require.Equal(t, saved.Version, snapshot.Version) + require.Equal(t, SplitJobPhaseCleanup, completed.Phase) + require.Equal(t, uint64(900), completed.PromotionCompletedTS) + require.Equal(t, int64(1000), completed.UpdatedAtMs) + require.Zero(t, completed.TerminalAtMs) + + loaded, err := cs.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, saved.Version, loaded.Version) +} + +func TestCompleteTargetPromotionStateBackfillsMissingTerminalTime(t *testing.T) { + t.Parallel() + + job := promotionCompleteTestJob() + job.TargetPromotionDone = true + job.Phase = SplitJobPhaseDone + routes := promotionCompleteTestRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + + result, err := CompleteTargetPromotionState(job, routes, 1200) + require.NoError(t, err) + require.True(t, result.Changed) + require.Equal(t, int64(1200), result.Job.TerminalAtMs) + require.Equal(t, int64(1200), result.Job.UpdatedAtMs) +} + +func TestCatalogStoreCompleteSplitJobTargetPromotionRejectsStaleInputs(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) + saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) + require.NoError(t, err) + job := promotionCompleteTestJob() + require.NoError(t, cs.CreateSplitJob(ctx, job)) + + _, _, err = cs.CompleteSplitJobTargetPromotion(ctx, saved.Version+1, job, 1000) + require.True(t, errors.Is(err, ErrCatalogVersionMismatch), "got %v", err) + + advanced := job + advanced.Cursor = []byte("advanced") + advanced.UpdatedAtMs++ + require.NoError(t, cs.SaveSplitJob(ctx, job, advanced)) + + _, _, err = cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1000) + require.True(t, errors.Is(err, ErrCatalogSplitJobConflict), "got %v", err) +} + +func TestCatalogStoreCompleteSplitJobTargetPromotionPreservesVersionConflict(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) + saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) + require.NoError(t, err) + job := promotionCompleteTestJob() + require.NoError(t, cs.CreateSplitJob(ctx, job)) + + readTS, _, routes, _, _, err := cs.loadPromotionCompleteInputs(ctx, saved.Version, job) + require.NoError(t, err) + completion, err := CompleteTargetPromotionState(job, routes, 1000) + require.NoError(t, err) + plan, mutations, commitTS, err := cs.buildPromotionCompleteMutations(ctx, readTS, saved.Version, job.JobID, &completion) + require.NoError(t, err) + + _, err = cs.Save(ctx, saved.Version, promotionCompleteTestRoutes()) + require.NoError(t, err) + + err = cs.applyPromotionCompleteMutations(ctx, plan, mutations, job.JobID, commitTS) + require.True(t, errors.Is(err, ErrCatalogVersionMismatch), "got %v", err) +} + +func promotionCompleteTestJob() SplitJob { + return SplitJob{ + JobID: 99, + SourceRouteID: 42, + SplitKey: []byte("m"), + TargetGroupID: 2, + Phase: SplitJobPhaseCleanup, + } +} + +func promotionCompleteTestRoutes() []RouteDescriptor { + return []RouteDescriptor{ + { + RouteID: 2, + Start: []byte(""), + End: []byte("m"), + GroupID: 1, + State: RouteStateActive, + ParentRouteID: 42, + }, + { + RouteID: 3, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: RouteStateActive, + ParentRouteID: 42, + StagedVisibilityActive: true, + MigrationJobID: 99, + MinWriteTSExclusive: 777, + }, + } +} + +func routeByID(t *testing.T, routes []RouteDescriptor, routeID uint64) RouteDescriptor { + t.Helper() + for _, route := range routes { + if route.RouteID == routeID { + return route + } + } + t.Fatalf("route %d not found", routeID) + return RouteDescriptor{} +} diff --git a/distribution/migrator.go b/distribution/migrator.go index b45cb2360..9492ba351 100644 --- a/distribution/migrator.go +++ b/distribution/migrator.go @@ -60,6 +60,7 @@ const ( // would renumber families that peers already use. MigrationFamilyFilesystemChunk MigrationFamilyFilesystemUsage + MigrationFamilyDynamoSnapshotSeal ) const ( @@ -76,6 +77,7 @@ const ( migrationTxnPrepareEvidenceEnabledKey = migrationTxnMetaPrefix + "prepare_evidence/enabled" migrationTxnPrepareEvidenceGateKey = migrationTxnMetaPrefix + "prepare_evidence/gate" migrationTxnPrepareEvidenceFloorKey = migrationTxnMetaPrefix + "prepare_evidence/timestamp_floor" + migrationConfigurationIndexKey = migrationTxnMetaPrefix + "migration_configuration_index" migrationRedisPrefix = "!redis|" migrationHashPrefix = "!hs|" migrationSetPrefix = "!st|" @@ -84,6 +86,7 @@ const ( migrationDynamoGenPrefix = "!ddb|meta|gen|" migrationDynamoItemPrefix = "!ddb|item|" migrationDynamoGSIPrefix = "!ddb|gsi|" + migrationDynamoSealPrefix = "!ddb|seal|" migrationStagedDataPrefix = "!dist|migstage|" ) @@ -104,6 +107,7 @@ const ( var ( ErrMigrationReservedRange = errors.New("migration range intersects reserved control prefix") + ErrMigrationResolverOwnedRange = errors.New("catalog migration range intersects resolver-owned SQS route") ErrMigrationInvalidRoute = errors.New("migration route is invalid") ErrMigrationDataMoveRequired = errors.New("migration data move is not implemented") ErrMigrationSourceRouteChanged = errors.New("migration source route does not match split job") @@ -122,6 +126,7 @@ var ( // coordinator -- SplitRange does exactly that. var migrationOnlyControlPrefixes = [][]byte{ []byte(migrationStagedDataPrefix), + []byte("!migstage|"), []byte("!migwrite|"), []byte("!migfence|"), } @@ -131,14 +136,29 @@ var migrationOnlyControlPrefixes = [][]byte{ // path's reserved-key test: the raw path can refuse every control namespace // because nothing legitimate writes one as a raw mutation, but the catalog // reaches the store through a transaction, so the transactional test has to be -// the narrower one. A staged-data row is the case that matters -- a client -// write landing there is promoted as user data. +// the narrower one. Include the store's typed metadata namespace and exact +// migration configuration key, but not ordinary aliases of those names. func IsMigrationOnlyControlKey(key []byte) bool { for _, prefix := range migrationOnlyControlPrefixes { if bytes.HasPrefix(key, prefix) { return true } } + return bytes.Equal(key, []byte(migrationConfigurationIndexKey)) +} + +// MigrationOnlyControlPrefixIntersects identifies nonempty prefix deletes +// aimed at typed migration namespaces. Exact transaction-local control keys +// remain excluded by the existing transaction prefix exclusion on that path. +func MigrationOnlyControlPrefixIntersects(prefix []byte) bool { + if len(prefix) == 0 { + return false + } + for _, reserved := range migrationOnlyControlPrefixes { + if bytes.HasPrefix(prefix, reserved) || bytes.HasPrefix(reserved, prefix) { + return true + } + } return false } @@ -202,6 +222,7 @@ var migrationInternalFamilyPrefixes = [][]byte{ []byte(migrationDynamoGenPrefix), []byte(migrationDynamoItemPrefix), []byte(migrationDynamoGSIPrefix), + []byte(migrationDynamoSealPrefix), []byte(migrationSQSQueueMetaPrefix), []byte(migrationSQSQueueGenPrefix), []byte(migrationSQSQueueSeqPrefix), @@ -244,6 +265,7 @@ var migrationInternalExactKeys = [][]byte{ []byte(migrationTxnPrepareEvidenceEnabledKey), []byte(migrationTxnPrepareEvidenceGateKey), []byte(migrationTxnPrepareEvidenceFloorKey), + []byte(migrationConfigurationIndexKey), } // MigrationBracket is a raw MVCC export or drain slice used by the migrator. @@ -300,6 +322,20 @@ func PlanExportBrackets(routeStart, routeEnd []byte) ([]MigrationBracket, error) return out, nil } +// PlanCleanupBrackets preserves export-plan cursor indexes and appends OCC +// seals. Seals are local conflict witnesses, not snapshot data to import. +func PlanCleanupBrackets(routeStart, routeEnd []byte) ([]MigrationBracket, error) { + brackets, err := PlanExportBrackets(routeStart, routeEnd) + if err != nil { + return nil, err + } + start := []byte(migrationDynamoSealPrefix) + return append(brackets, MigrationBracket{ + BracketID: uint64(MigrationFamilyDynamoSnapshotSeal), Family: MigrationFamilyDynamoSnapshotSeal, + Start: start, End: prefixScanEnd(start), RequiresRouteKeyCheck: true, + }), nil +} + // SplitJobBracketProgressForPlan creates durable per-bracket resume state for // a SplitJob phase. func SplitJobBracketProgressForPlan(brackets []MigrationBracket, phase SplitJobExportPhase) []SplitJobBracketProgress { @@ -439,8 +475,25 @@ func (b MigrationBracket) containsDecodedS3Route(rawKey, routeStart, routeEnd [] if routeKeyInRange(rawKey, routeStart, routeEnd) { return true } + return S3BucketAuxiliaryRouteSelected(bucket, routeStart, routeEnd) +} + +// S3BucketAuxiliaryRouteSelected reports whether the slice [routeStart,routeEnd) +// is the OWNER of a bucket's auxiliary rows -- its metadata and generation keys. +// +// Ownership is point containment of the bucket's route start, not intersection. +// A split inside a bucket's route interval leaves that bucket's auxiliary rows +// with the preceding slice, so exactly one slice claims them. Intersection would +// let every slice overlapping the bucket claim the same rows, which duplicates +// them on export and -- far worse -- lets CLEANUP delete rows a moving slice +// never exported and does not own. +// +// This is the single definition. adapter's export filter and the migration +// bracket both route through it precisely so they cannot drift apart again: +// export and cleanup disagreeing about ownership is silent data loss. +func S3BucketAuxiliaryRouteSelected(bucket string, routeStart, routeEnd []byte) bool { bucketRouteStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) - return rangesIntersect(routeStart, routeEnd, bucketRouteStart, prefixScanEnd(bucketRouteStart)) + return routeKeyInRange(bucketRouteStart, routeStart, normalizeMigrationRouteEnd(routeEnd)) } func (b MigrationBracket) decodedS3Bucket(rawKey []byte) (string, bool) { @@ -472,12 +525,25 @@ func InitializeSplitJobPlan(job SplitJob, source RouteDescriptor, nowMs int64) ( } routeStart := CloneBytes(job.SplitKey) routeEnd := CloneBytes(normalizeMigrationRouteEnd(source.End)) + if job.TargetGroupID != source.GroupID { + if err := ValidateSplitMigrationRouteRange(routeStart, routeEnd); err != nil { + return SplitJob{}, err + } + } brackets, err := PlanExportBrackets(routeStart, routeEnd) if err != nil { return SplitJob{}, err } out := CloneSplitJob(job) + if out.SourceGroupID == 0 { + // The source side keeps this group for the life of the job. Recording it + // now is the only durable handle on it: the sibling route it would + // otherwise be read back from is replaced by grandchildren as soon as the + // source child is split again, which SplitRange permits because that split + // is disjoint from the moving range. + out.SourceGroupID = source.GroupID + } if out.Phase == SplitJobPhaseNone { out.Phase = SplitJobPhasePlanned } @@ -542,6 +608,19 @@ func ValidateMigrationRouteRange(routeStart, routeEnd []byte) error { return nil } +// ValidateSplitMigrationRouteRange keeps catalog cutover away from the SQS +// partition resolver's ownership domain. A catalog migration cannot update its +// partition map, so even a temporary catalog fence would stop still-owned rows. +func ValidateSplitMigrationRouteRange(routeStart, routeEnd []byte) error { + if err := ValidateMigrationRouteRange(routeStart, routeEnd); err != nil { + return err + } + if routeKeyInRange([]byte("!sqs|route|global"), routeStart, routeEnd) { + return errors.WithStack(ErrMigrationResolverOwnedRange) + } + return nil +} + func migrationFamilyBrackets() []MigrationBracket { defs := []struct { family uint32 diff --git a/distribution/migrator_cleanup_plan_test.go b/distribution/migrator_cleanup_plan_test.go new file mode 100644 index 000000000..8a4b42457 --- /dev/null +++ b/distribution/migrator_cleanup_plan_test.go @@ -0,0 +1,36 @@ +package distribution_test + +import ( + "encoding/base64" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + "github.com/stretchr/testify/require" +) + +func TestPlanCleanupBracketsIncludesRoutedDynamoSealsWithoutExport(t *testing.T) { + t.Parallel() + table := base64.RawURLEncoding.EncodeToString([]byte("foo")) + neighbor := base64.RawURLEncoding.EncodeToString([]byte("fooX")) + seal := []byte(kv.DynamoSealPrefix + table + "|7|0001") + other := []byte(kv.DynamoSealPrefix + neighbor + "|7|0001") + start := kv.RouteKey(seal) + end := append(append([]byte(nil), start...), 0) + export, err := distribution.PlanExportBrackets(start, end) + require.NoError(t, err) + cleanup, err := distribution.PlanCleanupBrackets(start, end) + require.NoError(t, err) + require.Equal(t, export, cleanup[:len(export)], "existing positional cleanup cursors must retain their meaning") + require.Len(t, cleanup, len(export)+1) + bracket := cleanup[len(export)] + require.Equal(t, distribution.MigrationFamilyDynamoSnapshotSeal, bracket.Family) + require.True(t, bracket.ContainsRoutedKey(seal, start, end, kv.RouteKey)) + require.False(t, bracket.ContainsRoutedKey(other, start, end, kv.RouteKey)) + wideStart, wideEnd := []byte("!ddb|"), []byte("!ddc|") + wide, err := distribution.PlanExportBrackets(wideStart, wideEnd) + require.NoError(t, err) + for _, bracket := range wide { + require.False(t, bracket.ContainsRoutedKey(seal, wideStart, wideEnd, kv.RouteKey), "OCC seals must not be imported as snapshot data") + } +} diff --git a/distribution/migrator_export_plan_test.go b/distribution/migrator_export_plan_test.go index 0b4206427..b55f3989b 100644 --- a/distribution/migrator_export_plan_test.go +++ b/distribution/migrator_export_plan_test.go @@ -199,22 +199,36 @@ func TestMigrationBracketContainsRoutedKeyForS3BucketAuxiliaryState(t *testing.T require.NoError(t, err) byFamily := bracketsByFamily(brackets) + // A slice strictly inside bucket-b's route interval. It overlaps the bucket + // but does not contain the bucket's route start, so it is NOT the owner of + // bucket-b's auxiliary rows -- the preceding slice is. + // + // This previously expected true, i.e. any overlapping slice claimed the + // bucket's metadata. That rule cannot be right for a single row describing + // the whole bucket: it cannot travel with a partial object range, and every + // overlapping slice claiming it means the export duplicates it and CLEANUP + // deletes rows a slice never exported. Ownership is point containment of the + // bucket route start; see S3BucketAuxiliaryRouteSelected. routeStart := s3keys.RouteKey("bucket-b", 7, "a") routeEnd := s3keys.RouteKey("bucket-b", 7, "z") + ownerStart := s3keys.RoutePrefixForBucketAnyGeneration("bucket-b") for _, tc := range []struct { - name string - family uint32 - key []byte - want bool + name string + family uint32 + key []byte + start, end []byte + want bool }{ - {name: "meta same bucket", family: MigrationFamilyS3BucketMeta, key: s3keys.BucketMetaKey("bucket-b"), want: true}, - {name: "generation same bucket", family: MigrationFamilyS3BucketGeneration, key: s3keys.BucketGenerationKey("bucket-b"), want: true}, - {name: "meta different bucket", family: MigrationFamilyS3BucketMeta, key: s3keys.BucketMetaKey("bucket-c"), want: false}, - {name: "generation different bucket", family: MigrationFamilyS3BucketGeneration, key: s3keys.BucketGenerationKey("bucket-c"), want: false}, + {name: "meta inside slice is not the owner", family: MigrationFamilyS3BucketMeta, key: s3keys.BucketMetaKey("bucket-b"), start: routeStart, end: routeEnd, want: false}, + {name: "generation inside slice is not the owner", family: MigrationFamilyS3BucketGeneration, key: s3keys.BucketGenerationKey("bucket-b"), start: routeStart, end: routeEnd, want: false}, + {name: "meta owning slice", family: MigrationFamilyS3BucketMeta, key: s3keys.BucketMetaKey("bucket-b"), start: ownerStart, end: routeEnd, want: true}, + {name: "generation owning slice", family: MigrationFamilyS3BucketGeneration, key: s3keys.BucketGenerationKey("bucket-b"), start: ownerStart, end: routeEnd, want: true}, + {name: "meta different bucket", family: MigrationFamilyS3BucketMeta, key: s3keys.BucketMetaKey("bucket-c"), start: routeStart, end: routeEnd, want: false}, + {name: "generation different bucket", family: MigrationFamilyS3BucketGeneration, key: s3keys.BucketGenerationKey("bucket-c"), start: routeStart, end: routeEnd, want: false}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() - got := byFamily[tc.family].ContainsRoutedKey(tc.key, routeStart, routeEnd, s3keys.ExtractRouteKey) + got := byFamily[tc.family].ContainsRoutedKey(tc.key, tc.start, tc.end, s3keys.ExtractRouteKey) require.Equal(t, tc.want, got) }) } diff --git a/distribution/migrator_s3_owner_test.go b/distribution/migrator_s3_owner_test.go new file mode 100644 index 000000000..cb471d898 --- /dev/null +++ b/distribution/migrator_s3_owner_test.go @@ -0,0 +1,46 @@ +package distribution + +import ( + "testing" + + "github.com/bootjp/elastickv/internal/s3keys" + "github.com/stretchr/testify/require" +) + +// Export and cleanup must agree on which slice owns a bucket's auxiliary rows. +// They resolve it through different call paths -- the adapter's export filter +// and this bracket -- so when the export moved to owner semantics and the +// bracket stayed on intersection, a slice that started inside a bucket's route +// interval would refuse to export that bucket's metadata and then happily +// delete it during CLEANUP. Both now share S3BucketAuxiliaryRouteSelected. +func TestS3BucketAuxiliaryOwnershipIsPointContainmentNotIntersection(t *testing.T) { + t.Parallel() + + const bucket = "bucket-owner" + bucketStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + metaKey := s3keys.BucketMetaKey(bucket) + + // A split inside the bucket produces a slice bounded within the bucket's own + // !s3route| interval: it overlaps the bucket but starts after the bucket's + // route start, so the preceding slice still owns the auxiliary rows. The + // bounds also keep the raw !s3| key out of range, which is what isolates the + // ownership rule from the raw-key match both predicates check first. + insideStart := s3keys.RouteKey(bucket, 0, "") + insideEnd := prefixScanEnd(bucketStart) + require.Greater(t, string(insideStart), string(bucketStart), + "fixture must start after the bucket route start") + require.False(t, routeKeyInRange(metaKey, insideStart, insideEnd), + "fixture must not let the raw-key branch decide") + + require.False(t, S3BucketAuxiliaryRouteSelected(bucket, insideStart, insideEnd), + "a slice starting inside the bucket does not own its auxiliary rows") + + bracket := MigrationBracket{Family: MigrationFamilyS3BucketMeta, RequiresDecodedS3: true} + require.False(t, bracket.containsDecodedS3Route(metaKey, insideStart, insideEnd), + "cleanup must not claim auxiliary rows the export left with the previous owner") + + // The owning slice contains the bucket's route start, so both agree it is + // exported and may be cleaned up. + require.True(t, S3BucketAuxiliaryRouteSelected(bucket, bucketStart, insideEnd)) + require.True(t, bracket.containsDecodedS3Route(metaKey, bucketStart, insideEnd)) +} diff --git a/distribution/migrator_sqs_range_test.go b/distribution/migrator_sqs_range_test.go new file mode 100644 index 000000000..bf2197f7a --- /dev/null +++ b/distribution/migrator_sqs_range_test.go @@ -0,0 +1,45 @@ +package distribution + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSplitMigrationRejectsResolverOwnedSQSRoute(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name, start, end string + reject bool + }{ + {"global at start", "!sqs|route|global", "!sqs|route|z", true}, + {"global interior", "!sqs|route|g", "!sqs|route|z", true}, + {"unbounded", "!sqs|route|g", "", true}, + {"exclusive end", "!sqs|route|g", "!sqs|route|global", false}, + {"global stays left", "!sqs|route|global0", "!sqs|route|z", false}, + {"ordinary user interval", "m", "z", false}, + {"SQS-looking ordinary user interval", "!sqs|foo", "!sqs|fop", false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + start, end := []byte(tc.start), []byte(tc.end) + err := ValidateSplitMigrationRouteRange(start, end) + if tc.reject { + require.ErrorIs(t, err, ErrMigrationResolverOwnedRange) + } else { + require.NoError(t, err) + } + source := RouteDescriptor{RouteID: 1, Start: []byte("!sqs|"), End: end, GroupID: 7} + job := SplitJob{JobID: 1, SourceRouteID: 1, SplitKey: start, TargetGroupID: 9} + _, err = InitializeSplitJobPlan(job, source, 1000) + if tc.reject { + require.ErrorIs(t, err, ErrMigrationResolverOwnedRange) + } else { + require.NoError(t, err) + } + job.TargetGroupID = source.GroupID + _, err = AdvanceSameGroupNoop(job, source, 1000) + require.NoError(t, err, "same-group splits do not change partition ownership") + }) + } +} diff --git a/distribution/ownership_partition_catalog.go b/distribution/ownership_partition_catalog.go new file mode 100644 index 000000000..27ef912ed --- /dev/null +++ b/distribution/ownership_partition_catalog.go @@ -0,0 +1,96 @@ +package distribution + +import ( + "bytes" + "context" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +// BuildOwnershipPartitionMutationAt prepares, but does not publish, immutable +// root lineage. The caller MUST include this key and CatalogVersionKey in its +// OCC read set and commit this mutation with the route/version/job mutations. +func (s *CatalogStore) BuildOwnershipPartitionMutationAt(ctx context.Context, readTS uint64, manifest OwnershipPartitionManifest) (*store.KVPairMutation, error) { + if s == nil || s.store == nil { + return nil, errors.WithStack(ErrCatalogStoreRequired) + } + version, err := s.versionAt(ctx, readTS) + if err != nil { + return nil, err + } + if version != manifest.PreviousVersion { + return nil, errors.WithStack(ErrCatalogVersionMismatch) + } + encoded, err := EncodeOwnershipPartitionManifest(manifest) + if err != nil { + return nil, err + } + digest, err := OwnershipPartitionDescriptorHash(manifest.Parent) + if err != nil { + return nil, err + } + key, err := CatalogOwnershipPartitionKey(manifest.Parent.RouteID, digest) + if err != nil { + return nil, err + } + if err := s.validateOwnershipPartitionExistingAt(ctx, readTS, key, encoded); err != nil { + return nil, err + } + return &store.KVPairMutation{Op: store.OpTypePut, Key: key, Value: encoded}, nil +} + +func (s *CatalogStore) validateOwnershipPartitionExistingAt(ctx context.Context, readTS uint64, key, encoded []byte) error { + prior, err := s.store.GetAt(ctx, key, readTS) + if err != nil && !errors.Is(err, store.ErrKeyNotFound) { + return errors.WithStack(err) + } + if err == nil && !bytes.Equal(prior, encoded) { + return errors.WithStack(ErrOwnershipPartitionManifest) + } + return nil +} + +// OwnershipPartitionAt loads historical lineage at a caller's leader-issued +// root read timestamp. Root RPC callers additionally require a linearizable +// applied-read barrier; a cache-backed route listing is not sufficient. +func (s *CatalogStore) OwnershipPartitionAt(ctx context.Context, routeID uint64, descriptorHash []byte, readTS uint64) (OwnershipPartitionManifest, bool, error) { + if s == nil || s.store == nil { + return OwnershipPartitionManifest{}, false, errors.WithStack(ErrCatalogStoreRequired) + } + key, err := CatalogOwnershipPartitionKey(routeID, descriptorHash) + if err != nil { + return OwnershipPartitionManifest{}, false, err + } + raw, err := s.store.GetAt(ctx, key, readTS) + if errors.Is(err, store.ErrKeyNotFound) { + return OwnershipPartitionManifest{}, false, nil + } + if err != nil { + return OwnershipPartitionManifest{}, false, errors.WithStack(err) + } + manifest, err := s.validateOwnershipPartitionRecordAt(ctx, readTS, routeID, descriptorHash, raw) + if err != nil { + return OwnershipPartitionManifest{}, false, err + } + return manifest, true, nil +} + +func (s *CatalogStore) validateOwnershipPartitionRecordAt(ctx context.Context, readTS, routeID uint64, descriptorHash, raw []byte) (OwnershipPartitionManifest, error) { + manifest, err := DecodeOwnershipPartitionManifest(raw) + if err != nil { + return OwnershipPartitionManifest{}, err + } + digest, err := OwnershipPartitionDescriptorHash(manifest.Parent) + if err != nil { + return OwnershipPartitionManifest{}, err + } + version, err := s.versionAt(ctx, readTS) + if err != nil { + return OwnershipPartitionManifest{}, err + } + if manifest.Parent.RouteID != routeID || !bytes.Equal(descriptorHash, digest) || manifest.Version > version { + return OwnershipPartitionManifest{}, errors.WithStack(ErrOwnershipPartitionManifest) + } + return manifest, nil +} diff --git a/distribution/ownership_partition_keys.go b/distribution/ownership_partition_keys.go new file mode 100644 index 000000000..041f1ce26 --- /dev/null +++ b/distribution/ownership_partition_keys.go @@ -0,0 +1,14 @@ +package distribution + +import "bytes" + +// IsCatalogOwnershipPartitionKey includes malformed keys in the reserved ledger +// namespace so that authorization never depends on successful key decoding. +func IsCatalogOwnershipPartitionKey(key []byte) bool { + return bytes.HasPrefix(key, []byte(ownershipPartitionCatalogPrefix)) +} + +func CatalogOwnershipPartitionPrefixIntersects(prefix []byte) bool { + reserved := []byte(ownershipPartitionCatalogPrefix) + return bytes.HasPrefix(prefix, reserved) || bytes.HasPrefix(reserved, prefix) +} diff --git a/distribution/ownership_partition_keys_test.go b/distribution/ownership_partition_keys_test.go new file mode 100644 index 000000000..eed68056f --- /dev/null +++ b/distribution/ownership_partition_keys_test.go @@ -0,0 +1,20 @@ +package distribution + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOwnershipPartitionLedgerNamespace(t *testing.T) { + key, err := CatalogOwnershipPartitionKey(3, make([]byte, 32)) + require.NoError(t, err) + require.True(t, IsCatalogOwnershipPartitionKey(key)) + require.False(t, IsMigrationOnlyControlKey(key), "catalog transactions must retain their typed ledger writes") + for _, prefix := range [][]byte{nil, []byte("!dist|"), []byte("!dist|meta|ownership_partition"), key} { + require.True(t, CatalogOwnershipPartitionPrefixIntersects(prefix)) + } + for _, prefix := range [][]byte{[]byte("customer"), []byte("!dist|meta|ownership_partition_user")} { + require.False(t, CatalogOwnershipPartitionPrefixIntersects(prefix)) + } +} diff --git a/distribution/ownership_partition_manifest.go b/distribution/ownership_partition_manifest.go new file mode 100644 index 000000000..50235710e --- /dev/null +++ b/distribution/ownership_partition_manifest.go @@ -0,0 +1,161 @@ +package distribution + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/json" + "math" + "sort" + + "github.com/cockroachdb/errors" +) + +const ( + ownershipPartitionCatalogPrefix = catalogMetaPrefix + "ownership_partition/" + ownershipPartitionManifestVersion = byte(1) + ownershipPartitionManifestChildCount = 2 + // Three bounded route descriptors, including JSON byte-slice expansion. + // This bounds only the control record, not ordinary client values. + maxOwnershipPartitionManifestBytes = 16 << 20 +) + +var ErrOwnershipPartitionManifest = errors.New("invalid committed ownership partition manifest") + +// OwnershipPartitionManifest records a same-owner geometry operation in the +// same root transaction as its route/version mutations. It is independent of +// compactable catalog deltas and TTL-collected split-job history. +type OwnershipPartitionManifest struct { + PreviousVersion uint64 + Version uint64 + JobID uint64 + Parent RouteDescriptor + Children []RouteDescriptor +} + +// CatalogOwnershipPartitionKey indexes immutable manifests by the full prior +// descriptor identity. A reused route ID or changed bounds cannot alias it. +func CatalogOwnershipPartitionKey(routeID uint64, descriptorHash []byte) ([]byte, error) { + if routeID == 0 || len(descriptorHash) != sha256.Size { + return nil, errors.WithStack(ErrOwnershipPartitionManifest) + } + key := make([]byte, len(ownershipPartitionCatalogPrefix)+catalogUint64Bytes+sha256.Size) + copy(key, ownershipPartitionCatalogPrefix) + binary.BigEndian.PutUint64(key[len(ownershipPartitionCatalogPrefix):], routeID) + copy(key[len(ownershipPartitionCatalogPrefix)+catalogUint64Bytes:], descriptorHash) + return key, nil +} + +// OwnershipPartitionDescriptorHash uses the same descriptor encoding as the +// root ownership producer, including bounds, staged lineage and known floor. +func OwnershipPartitionDescriptorHash(route RouteDescriptor) ([]byte, error) { + encoded, err := EncodeRouteDescriptorForCatalogWrite(route, true) + if err != nil { + return nil, errors.WithStack(err) + } + digest := sha256.Sum256(encoded) + return digest[:], nil +} + +// EncodeOwnershipPartitionManifest gives the operation a deterministic identity +// without mutating or aliasing caller-owned descriptors. +func EncodeOwnershipPartitionManifest(manifest OwnershipPartitionManifest) ([]byte, error) { + manifest.Parent = cloneOwnershipPartitionRoute(manifest.Parent) + manifest.Children = cloneOwnershipPartitionChildren(manifest.Children) + if err := validateOwnershipPartitionManifest(manifest); err != nil { + return nil, err + } + encoded, err := json.Marshal(manifest) + if err != nil { + return nil, errors.WithStack(err) + } + if len(encoded) >= maxOwnershipPartitionManifestBytes { + return nil, errors.WithStack(ErrOwnershipPartitionManifest) + } + return append([]byte{ownershipPartitionManifestVersion}, encoded...), nil +} + +func DecodeOwnershipPartitionManifest(encoded []byte) (OwnershipPartitionManifest, error) { + if len(encoded) == 0 || len(encoded) > maxOwnershipPartitionManifestBytes || encoded[0] != ownershipPartitionManifestVersion { + return OwnershipPartitionManifest{}, errors.WithStack(ErrOwnershipPartitionManifest) + } + var manifest OwnershipPartitionManifest + if err := json.Unmarshal(encoded[1:], &manifest); err != nil { + return OwnershipPartitionManifest{}, errors.Wrap(ErrOwnershipPartitionManifest, err.Error()) + } + manifest.Children = cloneOwnershipPartitionChildren(manifest.Children) + if err := validateOwnershipPartitionManifest(manifest); err != nil { + return OwnershipPartitionManifest{}, err + } + return manifest, nil +} + +// OwnershipPartitionManifestHash identifies the committed operation, not an +// ownership grant or a data-group configuration epoch. +func OwnershipPartitionManifestHash(manifest OwnershipPartitionManifest) ([]byte, error) { + encoded, err := EncodeOwnershipPartitionManifest(manifest) + if err != nil { + return nil, err + } + digest := sha256.Sum256(encoded) + return digest[:], nil +} + +func cloneOwnershipPartitionRoute(route RouteDescriptor) RouteDescriptor { + route.Start = bytes.Clone(route.Start) + route.End = bytes.Clone(route.End) + return route +} + +func cloneOwnershipPartitionChildren(children []RouteDescriptor) []RouteDescriptor { + out := make([]RouteDescriptor, len(children)) + for i, child := range children { + out[i] = cloneOwnershipPartitionRoute(child) + } + sort.Slice(out, func(i, j int) bool { return bytes.Compare(out[i].Start, out[j].Start) < 0 }) + return out +} + +func validateOwnershipPartitionManifest(manifest OwnershipPartitionManifest) error { + if !ownershipPartitionManifestHeaderValid(manifest) { + return errors.WithStack(ErrOwnershipPartitionManifest) + } + if _, err := OwnershipPartitionDescriptorHash(manifest.Parent); err != nil { + return errors.Wrap(ErrOwnershipPartitionManifest, err.Error()) + } + for _, child := range manifest.Children { + if !ownershipPartitionChildIdentity(manifest.Parent, child) || !ownershipPartitionChildLineage(manifest.Parent, child) { + return errors.WithStack(ErrOwnershipPartitionManifest) + } + if _, err := OwnershipPartitionDescriptorHash(child); err != nil { + return errors.Wrap(ErrOwnershipPartitionManifest, err.Error()) + } + } + return validateOwnershipPartitionTiling(manifest) +} + +func ownershipPartitionManifestHeaderValid(manifest OwnershipPartitionManifest) bool { + return manifest.PreviousVersion != 0 && manifest.PreviousVersion != math.MaxUint64 && + manifest.Version == manifest.PreviousVersion+1 && manifest.Parent.State == RouteStateActive && + len(manifest.Children) == ownershipPartitionManifestChildCount +} + +func ownershipPartitionChildIdentity(parent, child RouteDescriptor) bool { + return child.RouteID != parent.RouteID && child.ParentRouteID == parent.RouteID && + child.GroupID == parent.GroupID && child.State == parent.State +} + +func ownershipPartitionChildLineage(parent, child RouteDescriptor) bool { + return child.MinWriteTSExclusive == parent.MinWriteTSExclusive && child.StagedVisibilityActive == parent.StagedVisibilityActive && + child.MigrationJobID == parent.MigrationJobID && child.SplitAtHLC != 0 && child.SplitAtHLC > parent.SplitAtHLC +} + +func validateOwnershipPartitionTiling(manifest OwnershipPartitionManifest) error { + left, right := manifest.Children[0], manifest.Children[1] + if left.RouteID == right.RouteID || !bytes.Equal(left.Start, manifest.Parent.Start) || + !bytes.Equal(right.End, manifest.Parent.End) || len(left.End) == 0 || + !bytes.Equal(left.End, right.Start) || left.SplitAtHLC != right.SplitAtHLC { + return errors.WithStack(ErrOwnershipPartitionManifest) + } + return nil +} diff --git a/distribution/ownership_partition_manifest_test.go b/distribution/ownership_partition_manifest_test.go new file mode 100644 index 000000000..5462f648b --- /dev/null +++ b/distribution/ownership_partition_manifest_test.go @@ -0,0 +1,166 @@ +package distribution + +import ( + "bytes" + "math" + "testing" + + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func ownershipPartitionManifestFixture() OwnershipPartitionManifest { + parent := RouteDescriptor{RouteID: 3, Start: []byte("m"), End: []byte("z"), GroupID: 2, + State: RouteStateActive, MinWriteTSExclusive: 100, SplitAtHLC: 110} + left, right := parent, parent + left.RouteID, right.RouteID = 4, 5 + left.ParentRouteID, right.ParentRouteID = parent.RouteID, parent.RouteID + left.End, right.Start = []byte("x"), []byte("x") + left.SplitAtHLC, right.SplitAtHLC = 200, 200 + return OwnershipPartitionManifest{PreviousVersion: 12, Version: 13, Parent: parent, + Children: []RouteDescriptor{left, right}} +} + +func TestOwnershipPartitionManifestCanonicalCodec(t *testing.T) { + manifest := ownershipPartitionManifestFixture() + encoded, err := EncodeOwnershipPartitionManifest(manifest) + require.NoError(t, err) + decoded, err := DecodeOwnershipPartitionManifest(encoded) + require.NoError(t, err) + require.Equal(t, manifest, decoded) + reversed := manifest + reversed.Children = []RouteDescriptor{manifest.Children[1], manifest.Children[0]} + other, err := EncodeOwnershipPartitionManifest(reversed) + require.NoError(t, err) + require.Equal(t, encoded, other, "child ordering must not change root operation identity") + firstHash, err := OwnershipPartitionManifestHash(manifest) + require.NoError(t, err) + secondHash, err := OwnershipPartitionManifestHash(reversed) + require.NoError(t, err) + require.Equal(t, firstHash, secondHash) + decoded.Children[0].Start[0] = 'n' + require.Equal(t, []byte("m"), manifest.Children[0].Start, "decoding must not alias caller descriptors") +} + +func TestOwnershipPartitionManifestRejectsAuthorityChanges(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*OwnershipPartitionManifest) + }{ + {"future version gap", func(m *OwnershipPartitionManifest) { m.Version++ }}, + {"version overflow", func(m *OwnershipPartitionManifest) { m.PreviousVersion = math.MaxUint64; m.Version = 0 }}, + {"owner change", func(m *OwnershipPartitionManifest) { m.Children[1].GroupID++ }}, + {"left floor change", func(m *OwnershipPartitionManifest) { m.Children[0].MinWriteTSExclusive++ }}, + {"floor decrease", func(m *OwnershipPartitionManifest) { m.Children[1].MinWriteTSExclusive-- }}, + {"hole", func(m *OwnershipPartitionManifest) { m.Children[1].Start = []byte("y") }}, + {"overlap", func(m *OwnershipPartitionManifest) { m.Children[1].Start = []byte("w") }}, + {"enlargement", func(m *OwnershipPartitionManifest) { m.Children[1].End = nil }}, + {"lost staged lineage", func(m *OwnershipPartitionManifest) { + m.Parent.StagedVisibilityActive = true + m.Parent.MigrationJobID = 7 + }}, + {"wrong parent", func(m *OwnershipPartitionManifest) { m.Children[0].ParentRouteID++ }}, + {"duplicate child", func(m *OwnershipPartitionManifest) { m.Children[1].RouteID = m.Children[0].RouteID }}, + {"reuse parent ID", func(m *OwnershipPartitionManifest) { m.Children[0].RouteID = m.Parent.RouteID }}, + {"fenced child", func(m *OwnershipPartitionManifest) { m.Children[1].State = RouteStateWriteFenced }}, + } { + t.Run(tc.name, func(t *testing.T) { + manifest := ownershipPartitionManifestFixture() + tc.mutate(&manifest) + _, err := EncodeOwnershipPartitionManifest(manifest) + require.ErrorIs(t, err, ErrOwnershipPartitionManifest) + }) + } +} + +func TestOwnershipPartitionManifestSurvivesJobGCAndUnrelatedCatalogVersion(t *testing.T) { + st := store.NewMVCCStore() + catalog := NewCatalogStore(st) + manifest := ownershipPartitionManifestFixture() + require.NoError(t, st.PutAt(t.Context(), CatalogVersionKey(), EncodeCatalogVersion(12), 10, 0)) + mutation, err := catalog.BuildOwnershipPartitionMutationAt(t.Context(), 10, manifest) + require.NoError(t, err) + require.NoError(t, st.ApplyMutations(t.Context(), []*store.KVPairMutation{mutation, + {Op: store.OpTypePut, Key: CatalogVersionKey(), Value: EncodeCatalogVersion(13)}}, + [][]byte{CatalogVersionKey(), mutation.Key}, 10, 20)) + require.NoError(t, st.DeletePrefixAt(t.Context(), []byte(catalogSplitJobHistoryPrefix), nil, 30)) + require.NoError(t, st.PutAt(t.Context(), CatalogVersionKey(), EncodeCatalogVersion(14), 40, 0)) + digest, err := OwnershipPartitionDescriptorHash(manifest.Parent) + require.NoError(t, err) + got, found, err := catalog.OwnershipPartitionAt(t.Context(), 3, digest, math.MaxUint64) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, manifest, got) + _, found, err = catalog.OwnershipPartitionAt(t.Context(), 3, digest, 10) + require.NoError(t, err) + require.False(t, found) + wrongDigest := bytes.Repeat([]byte{1}, len(digest)) + _, found, err = catalog.OwnershipPartitionAt(t.Context(), 3, wrongDigest, math.MaxUint64) + require.NoError(t, err) + require.False(t, found, "route ID alone cannot authorize a changed descriptor") + _, err = catalog.BuildOwnershipPartitionMutationAt(t.Context(), 40, manifest) + require.ErrorIs(t, err, ErrCatalogVersionMismatch) +} + +func TestOwnershipPartitionManifestCannotAppearAheadOfCommittedCatalog(t *testing.T) { + st := store.NewMVCCStore() + catalog := NewCatalogStore(st) + manifest := ownershipPartitionManifestFixture() + digest, err := OwnershipPartitionDescriptorHash(manifest.Parent) + require.NoError(t, err) + key, err := CatalogOwnershipPartitionKey(3, digest) + require.NoError(t, err) + encoded, err := EncodeOwnershipPartitionManifest(manifest) + require.NoError(t, err) + require.NoError(t, st.PutAt(t.Context(), key, encoded, 10, 0)) + require.NoError(t, st.PutAt(t.Context(), CatalogVersionKey(), EncodeCatalogVersion(12), 10, 0)) + _, _, err = catalog.OwnershipPartitionAt(t.Context(), 3, digest, math.MaxUint64) + require.ErrorIs(t, err, ErrOwnershipPartitionManifest) +} + +func TestOwnershipPartitionManifestCatalogCASLeavesNoPartialLineage(t *testing.T) { + st := store.NewMVCCStore() + catalog := NewCatalogStore(st) + manifest := ownershipPartitionManifestFixture() + require.NoError(t, st.PutAt(t.Context(), CatalogVersionKey(), EncodeCatalogVersion(12), 10, 0)) + mutation, err := catalog.BuildOwnershipPartitionMutationAt(t.Context(), 10, manifest) + require.NoError(t, err) + require.NoError(t, st.PutAt(t.Context(), CatalogVersionKey(), EncodeCatalogVersion(13), 15, 0)) + err = st.ApplyMutations(t.Context(), []*store.KVPairMutation{mutation, + {Op: store.OpTypePut, Key: CatalogVersionKey(), Value: EncodeCatalogVersion(13)}}, + [][]byte{CatalogVersionKey(), mutation.Key}, 10, 20) + require.ErrorIs(t, err, store.ErrWriteConflict) + _, err = st.GetAt(t.Context(), mutation.Key, math.MaxUint64) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +func TestOwnershipPartitionManifestNativeSnapshotRetainsLineage(t *testing.T) { + st := store.NewMVCCStore() + manifest := ownershipPartitionManifestFixture() + digest, err := OwnershipPartitionDescriptorHash(manifest.Parent) + require.NoError(t, err) + key, err := CatalogOwnershipPartitionKey(3, digest) + require.NoError(t, err) + encoded, err := EncodeOwnershipPartitionManifest(manifest) + require.NoError(t, err) + require.NoError(t, st.PutAt(t.Context(), key, encoded, 20, 0)) + require.NoError(t, st.PutAt(t.Context(), CatalogVersionKey(), EncodeCatalogVersion(13), 20, 0)) + snapshot, err := st.Snapshot() + require.NoError(t, err) + var raw bytes.Buffer + _, err = snapshot.WriteTo(&raw) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + dir := t.TempDir() + destination, err := store.NewPebbleStore(dir) + require.NoError(t, err) + require.NoError(t, destination.Restore(bytes.NewReader(raw.Bytes()))) + require.NoError(t, destination.Close()) + destination, err = store.NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, destination.Close()) }) + got, found, err := NewCatalogStore(destination).OwnershipPartitionAt(t.Context(), 3, digest, math.MaxUint64) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, manifest, got) +} diff --git a/distribution/ownership_partition_negative_catalog.go b/distribution/ownership_partition_negative_catalog.go new file mode 100644 index 000000000..5a627c379 --- /dev/null +++ b/distribution/ownership_partition_negative_catalog.go @@ -0,0 +1,74 @@ +package distribution + +import ( + "bytes" + "context" + "encoding/binary" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +const ownershipPartitionNegativeScanLimit = 2 + +// OwnershipPartitionNegativeAt discovers geometry for historical negative +// evidence only. It is NOT an ownership attestation: the caller must validate +// every prior as NONOWNER and preserve its owner, floor, role and lineage. +// Production route IDs are never reused. Two retained records under one ID +// are therefore ambiguous, even if only one appears to match the bounds. +func (s *CatalogStore) OwnershipPartitionNegativeAt(ctx context.Context, routeID, parentRouteID uint64, start, end []byte, readTS uint64) (OwnershipPartitionManifest, bool, error) { + var zero OwnershipPartitionManifest + if s == nil || s.store == nil { + return zero, false, errors.WithStack(ErrCatalogStoreRequired) + } + if routeID == 0 || readTS == 0 { + return zero, false, errors.WithStack(ErrOwnershipPartitionManifest) + } + prefix := binary.BigEndian.AppendUint64([]byte(ownershipPartitionCatalogPrefix), routeID) + rows, err := s.store.ScanAt(ctx, prefix, store.PrefixScanEnd(prefix), ownershipPartitionNegativeScanLimit, readTS) + if err != nil { + return zero, false, errors.WithStack(err) + } + if len(rows) == 0 { + return zero, false, nil + } + if len(rows) != 1 { + return zero, false, errors.WithStack(ErrOwnershipPartitionManifest) + } + manifest, err := s.validateNegativeOwnershipPartitionRecord(ctx, routeID, rows[0].Key, rows[0].Value, readTS) + if err != nil { + return zero, false, err + } + if !ownershipPartitionNegativeBoundsMatch(manifest.Parent, parentRouteID, start, end) { + return zero, false, nil + } + return manifest, true, nil +} + +func (s *CatalogStore) validateNegativeOwnershipPartitionRecord(ctx context.Context, routeID uint64, key, raw []byte, readTS uint64) (OwnershipPartitionManifest, error) { + var zero OwnershipPartitionManifest + manifest, err := DecodeOwnershipPartitionManifest(raw) + if err != nil { + return zero, err + } + hash, err := OwnershipPartitionDescriptorHash(manifest.Parent) + if err != nil { + return zero, err + } + expected, err := CatalogOwnershipPartitionKey(routeID, hash) + if err != nil { + return zero, err + } + canonical, err := EncodeOwnershipPartitionManifest(manifest) + if err != nil { + return zero, err + } + if !bytes.Equal(key, expected) || !bytes.Equal(raw, canonical) { + return zero, errors.WithStack(ErrOwnershipPartitionManifest) + } + return s.validateOwnershipPartitionRecordAt(ctx, readTS, routeID, hash, raw) +} + +func ownershipPartitionNegativeBoundsMatch(parent RouteDescriptor, parentRouteID uint64, start, end []byte) bool { + return parent.ParentRouteID == parentRouteID && bytes.Equal(parent.Start, start) && bytes.Equal(parent.End, end) +} diff --git a/distribution/ownership_partition_negative_catalog_test.go b/distribution/ownership_partition_negative_catalog_test.go new file mode 100644 index 000000000..37dc8b168 --- /dev/null +++ b/distribution/ownership_partition_negative_catalog_test.go @@ -0,0 +1,92 @@ +package distribution + +import ( + "context" + "math" + "testing" + + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +type negativePartitionScanStore struct { + store.MVCCStore + limit, calls int +} + +func (s *negativePartitionScanStore) ScanAt(ctx context.Context, start, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { + s.limit, s.calls = limit, s.calls+1 + return s.MVCCStore.ScanAt(ctx, start, end, limit, ts) +} + +func putNegativePartitionManifest(t *testing.T, st store.MVCCStore, m OwnershipPartitionManifest, ts uint64) []byte { + t.Helper() + raw, err := EncodeOwnershipPartitionManifest(m) + require.NoError(t, err) + hash, err := OwnershipPartitionDescriptorHash(m.Parent) + require.NoError(t, err) + key, err := CatalogOwnershipPartitionKey(m.Parent.RouteID, hash) + require.NoError(t, err) + require.NoError(t, st.ApplyMutations(t.Context(), []*store.KVPairMutation{{Op: store.OpTypePut, Key: key, Value: raw}, + {Op: store.OpTypePut, Key: CatalogVersionKey(), Value: EncodeCatalogVersion(m.Version)}}, nil, ts-1, ts)) + return key +} + +func TestOwnershipPartitionNegativeLookupPinnedExactAndBounded(t *testing.T) { + st := &negativePartitionScanStore{MVCCStore: store.NewMVCCStore()} + catalog := NewCatalogStore(st) + m := ownershipPartitionManifestFixture() + putNegativePartitionManifest(t, st, m, 20) + _, found, err := catalog.OwnershipPartitionNegativeAt(t.Context(), m.Parent.RouteID, m.Parent.ParentRouteID, m.Parent.Start, m.Parent.End, 19) + require.NoError(t, err) + require.False(t, found) + actual, found, err := catalog.OwnershipPartitionNegativeAt(t.Context(), m.Parent.RouteID, m.Parent.ParentRouteID, m.Parent.Start, m.Parent.End, 20) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, m, actual) + require.Equal(t, 2, st.limit, "never enumerate the immutable ledger without a bound") + _, found, err = catalog.OwnershipPartitionNegativeAt(t.Context(), m.Parent.RouteID, m.Parent.ParentRouteID, []byte("n"), m.Parent.End, 20) + require.NoError(t, err) + require.False(t, found, "containment is not exact prior geometry") + _, found, err = catalog.OwnershipPartitionNegativeAt(t.Context(), m.Parent.RouteID, m.Parent.ParentRouteID+1, m.Parent.Start, m.Parent.End, 20) + require.NoError(t, err) + require.False(t, found) +} + +func TestOwnershipPartitionNegativeLookupRejectsAmbiguityAndUncommittedC(t *testing.T) { + st := store.NewMVCCStore() + catalog := NewCatalogStore(st) + m := ownershipPartitionManifestFixture() + putNegativePartitionManifest(t, st, m, 20) + require.NoError(t, st.PutAt(t.Context(), CatalogVersionKey(), EncodeCatalogVersion(m.PreviousVersion), 21, 0)) + _, _, err := catalog.OwnershipPartitionNegativeAt(t.Context(), m.Parent.RouteID, m.Parent.ParentRouteID, m.Parent.Start, m.Parent.End, 21) + require.ErrorIs(t, err, ErrOwnershipPartitionManifest) + other := ownershipPartitionManifestFixture() + other.Parent.GroupID = 3 + for i := range other.Children { + other.Children[i].GroupID = 3 + } + putNegativePartitionManifest(t, st, other, 30) + _, _, err = catalog.OwnershipPartitionNegativeAt(t.Context(), m.Parent.RouteID, m.Parent.ParentRouteID, m.Parent.Start, m.Parent.End, math.MaxUint64) + require.ErrorIs(t, err, ErrOwnershipPartitionManifest, "two identities for a route cannot be guessed by owner or version") + _, found, err := catalog.OwnershipPartitionNegativeAt(t.Context(), m.Parent.RouteID, m.Parent.ParentRouteID, m.Parent.Start, m.Parent.End, 20) + require.NoError(t, err) + require.True(t, found, "later ambiguity must not contaminate a pinned earlier read") +} + +func TestOwnershipPartitionNegativeLookupRejectsCorruptKeyIdentity(t *testing.T) { + st := store.NewMVCCStore() + catalog := NewCatalogStore(st) + m := ownershipPartitionManifestFixture() + key := putNegativePartitionManifest(t, st, m, 20) + other := ownershipPartitionManifestFixture() + other.Parent.GroupID = 9 + for i := range other.Children { + other.Children[i].GroupID = 9 + } + raw, err := EncodeOwnershipPartitionManifest(other) + require.NoError(t, err) + require.NoError(t, st.PutAt(t.Context(), key, raw, 21, 0)) + _, _, err = catalog.OwnershipPartitionNegativeAt(t.Context(), m.Parent.RouteID, m.Parent.ParentRouteID, m.Parent.Start, m.Parent.End, 21) + require.ErrorIs(t, err, ErrOwnershipPartitionManifest) +} diff --git a/distribution/split_job_catalog.go b/distribution/split_job_catalog.go index 7da615bca..9d4969348 100644 --- a/distribution/split_job_catalog.go +++ b/distribution/split_job_catalog.go @@ -36,6 +36,7 @@ var ( ErrCatalogSplitJobConflict = errors.New("catalog split job conflict") ErrCatalogSplitJobTerminalRequired = errors.New("catalog split job terminal state is required") ErrSplitJobOverlap = errors.New("split job overlaps requested route") + ErrTooManyInFlightSplitJobs = errors.New("too many in-flight split jobs") ) // SplitJobPhase is the durable phase of a split migration job. @@ -134,8 +135,14 @@ type SplitJobBracketProgress struct { // SplitJob is the durable default-group state for a range split migration. type SplitJob struct { + TransferPreflightVersion uint32 + TransferPreflightTS uint64 + TransferPreflightProgress []SplitJobBracketProgress + TransferPreflightDone bool JobID uint64 SourceRouteID uint64 + WorkingSourceRouteID uint64 + SourceGroupID uint64 SplitKey []byte TargetGroupID uint64 Phase SplitJobPhase @@ -167,6 +174,7 @@ type SplitJob struct { StartedAtMs int64 UpdatedAtMs int64 TerminalAtMs int64 + CapabilityRegressed bool } // CatalogNextSplitJobIDKey returns the reserved key used for split-job ID allocation. @@ -585,7 +593,10 @@ func validateSplitJob(job SplitJob) error { if err := validateSplitJobBarriers(job); err != nil { return err } - return validateSplitJobBracketProgress(job.BracketProgress) + if err := validateSplitJobBracketProgress(job.BracketProgress); err != nil { + return err + } + return validateSplitJobBracketProgress(job.TransferPreflightProgress) } func validateSplitJobPhases(job SplitJob) error { @@ -609,7 +620,7 @@ func (job SplitJob) retryPhaseValidForPhase() bool { } func (job SplitJob) abandonFromPhaseValidForPhase() bool { - if job.Phase == SplitJobPhaseAbandoning { + if job.Phase == SplitJobPhaseAbandoning || (job.Phase == SplitJobPhaseFailed && job.RetryPhase == SplitJobPhaseAbandoning) { return job.AbandonFromPhase.abandonable() } return job.AbandonFromPhase == SplitJobPhaseNone @@ -617,17 +628,17 @@ func (job SplitJob) abandonFromPhaseValidForPhase() bool { func (p SplitJobPhase) retryable() bool { switch p { - case SplitJobPhaseBackfill, + case SplitJobPhasePlanned, + SplitJobPhaseAbandoning, + SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy, SplitJobPhaseCutover, SplitJobPhaseCleanup: return true case SplitJobPhaseNone, - SplitJobPhasePlanned, SplitJobPhaseDone, SplitJobPhaseFailed, - SplitJobPhaseAbandoning, SplitJobPhaseAbandoned: return false default: @@ -637,12 +648,12 @@ func (p SplitJobPhase) retryable() bool { func (p SplitJobPhase) abandonable() bool { switch p { - case SplitJobPhaseBackfill, + case SplitJobPhasePlanned, + SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy: return true case SplitJobPhaseNone, - SplitJobPhasePlanned, SplitJobPhaseCutover, SplitJobPhaseCleanup, SplitJobPhaseDone, @@ -921,8 +932,13 @@ func splitJobLess(left, right SplitJob) bool { func CloneSplitJob(job SplitJob) SplitJob { return SplitJob{ + TransferPreflightVersion: job.TransferPreflightVersion, + TransferPreflightTS: job.TransferPreflightTS, + TransferPreflightProgress: cloneSplitJobBracketProgress(job.TransferPreflightProgress), + TransferPreflightDone: job.TransferPreflightDone, JobID: job.JobID, SourceRouteID: job.SourceRouteID, + WorkingSourceRouteID: job.WorkingSourceRouteID, SplitKey: CloneBytes(job.SplitKey), TargetGroupID: job.TargetGroupID, Phase: job.Phase, @@ -954,14 +970,26 @@ func CloneSplitJob(job SplitJob) SplitJob { StartedAtMs: job.StartedAtMs, UpdatedAtMs: job.UpdatedAtMs, TerminalAtMs: job.TerminalAtMs, + CapabilityRegressed: job.CapabilityRegressed, + SourceGroupID: job.SourceGroupID, } } +// SplitJobToProto converts a catalog SplitJob into its wire representation. +func SplitJobToProto(job SplitJob) *pb.SplitJob { + return splitJobToProto(job) +} + func splitJobToProto(job SplitJob) *pb.SplitJob { job = CloneSplitJob(job) return &pb.SplitJob{ + TransferPreflightVersion: job.TransferPreflightVersion, + TransferPreflightTs: job.TransferPreflightTS, + TransferPreflightProgress: splitJobBracketProgressToProto(job.TransferPreflightProgress), + TransferPreflightDone: job.TransferPreflightDone, JobId: job.JobID, SourceRouteId: job.SourceRouteID, + WorkingSourceRouteId: job.WorkingSourceRouteID, SplitKey: job.SplitKey, TargetGroupId: job.TargetGroupID, Phase: pb.SplitJobPhase(job.Phase), @@ -993,6 +1021,8 @@ func splitJobToProto(job SplitJob) *pb.SplitJob { StartedAtMs: job.StartedAtMs, UpdatedAtMs: job.UpdatedAtMs, TerminalAtMs: job.TerminalAtMs, + CapabilityRegressed: job.CapabilityRegressed, + SourceGroupId: job.SourceGroupID, } } @@ -1024,9 +1054,14 @@ func splitJobFromProto(msg *pb.SplitJob) (SplitJob, error) { if err != nil { return SplitJob{}, err } + preflightProgress, err := splitJobBracketProgressFromProto(msg.GetTransferPreflightProgress()) + if err != nil { + return SplitJob{}, err + } return SplitJob{ JobID: msg.GetJobId(), SourceRouteID: msg.GetSourceRouteId(), + WorkingSourceRouteID: msg.GetWorkingSourceRouteId(), SplitKey: CloneBytes(msg.GetSplitKey()), TargetGroupID: msg.GetTargetGroupId(), Phase: phase, @@ -1053,11 +1088,17 @@ func splitJobFromProto(msg *pb.SplitJob) (SplitJob, error) { SourceReadDrainCursor: CloneBytes(msg.GetSourceReadDrainCursor()), TargetClearedDescriptorAckCursor: CloneBytes(msg.GetTargetClearedDescriptorAckCursor()), BracketProgress: bracketProgress, + TransferPreflightVersion: msg.GetTransferPreflightVersion(), + TransferPreflightTS: msg.GetTransferPreflightTs(), + TransferPreflightProgress: preflightProgress, + TransferPreflightDone: msg.GetTransferPreflightDone(), SourceRetentionPinTS: msg.GetSourceRetentionPinTs(), LastError: msg.GetLastError(), StartedAtMs: msg.GetStartedAtMs(), UpdatedAtMs: msg.GetUpdatedAtMs(), TerminalAtMs: msg.GetTerminalAtMs(), + CapabilityRegressed: msg.GetCapabilityRegressed(), + SourceGroupID: msg.GetSourceGroupId(), }, nil } diff --git a/distribution/split_job_catalog_test.go b/distribution/split_job_catalog_test.go index c6cde5a98..45a3f5c74 100644 --- a/distribution/split_job_catalog_test.go +++ b/distribution/split_job_catalog_test.go @@ -119,6 +119,7 @@ func TestSplitJobCodecValidatesRestartPhases(t *testing.T) { } for _, phase := range []SplitJobPhase{ + SplitJobPhasePlanned, SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy, @@ -150,9 +151,9 @@ func TestSplitJobCodecRejectsInvalidRestartPhases(t *testing.T) { retryPhase: SplitJobPhaseNone, }, { - name: "failed retry to planned", + name: "failed abandoning without origin", phase: SplitJobPhaseFailed, - retryPhase: SplitJobPhasePlanned, + retryPhase: SplitJobPhaseAbandoning, }, { name: "failed retry to terminal", @@ -338,6 +339,120 @@ func TestCatalogStoreSaveSplitJobRejectsStaleExpectedJob(t *testing.T) { assertSplitJobEqual(t, advanced, got) } +func TestCatalogStoreRetrySplitJobUsesDurableRetryPhase(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(18) + job.Phase = SplitJobPhaseFailed + job.RetryPhase = SplitJobPhaseDeltaCopy + job.LastError = "transient" + + if err := cs.CreateSplitJob(ctx, job); err != nil { + t.Fatalf("create split job: %v", err) + } + retried, err := cs.RetrySplitJob(ctx, job.JobID, 1200) + if err != nil { + t.Fatalf("retry split job: %v", err) + } + if retried.Phase != SplitJobPhaseDeltaCopy || retried.RetryPhase != SplitJobPhaseNone { + t.Fatalf("unexpected retry transition: %+v", retried) + } + if retried.AbandonFromPhase != SplitJobPhaseNone || retried.LastError != "" || retried.UpdatedAtMs != 1200 { + t.Fatalf("unexpected retry witness cleanup: %+v", retried) + } + got, found, err := cs.SplitJob(ctx, job.JobID) + if err != nil { + t.Fatalf("load retried split job: %v", err) + } + if !found { + t.Fatal("expected retried split job") + } + assertSplitJobEqual(t, retried, got) +} + +func TestCatalogStoreRetrySplitJobRejectsMissingRetryWitness(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(19) + job.Phase = SplitJobPhaseFailed + job.RetryPhase = SplitJobPhaseNone + + if err := cs.CreateSplitJob(ctx, job); !errors.Is(err, ErrCatalogInvalidSplitJobPhase) { + t.Fatalf("expected ErrCatalogInvalidSplitJobPhase, got %v", err) + } +} + +func TestCatalogStoreBeginSplitJobAbandonRecordsPreCutoverPhase(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(20) + job.Phase = SplitJobPhaseFailed + job.RetryPhase = SplitJobPhaseFence + job.AbandonFromPhase = SplitJobPhaseNone + + if err := cs.CreateSplitJob(ctx, job); err != nil { + t.Fatalf("create split job: %v", err) + } + abandoning, err := cs.BeginSplitJobAbandon(ctx, job.JobID, 1300) + if err != nil { + t.Fatalf("begin split job abandon: %v", err) + } + if abandoning.Phase != SplitJobPhaseAbandoning || + abandoning.RetryPhase != SplitJobPhaseNone || + abandoning.AbandonFromPhase != SplitJobPhaseFence || + abandoning.UpdatedAtMs != 1300 { + t.Fatalf("unexpected abandon transition: %+v", abandoning) + } + got, found, err := cs.SplitJob(ctx, job.JobID) + if err != nil { + t.Fatalf("load abandoning split job: %v", err) + } + if !found { + t.Fatal("expected abandoning split job") + } + assertSplitJobEqual(t, abandoning, got) +} + +func TestCatalogStoreBeginSplitJobAbandonAllowsPlannedPhase(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(22) + job.Phase = SplitJobPhasePlanned + job.RetryPhase = SplitJobPhaseNone + job.AbandonFromPhase = SplitJobPhaseNone + + if err := cs.CreateSplitJob(ctx, job); err != nil { + t.Fatalf("create split job: %v", err) + } + abandoning, err := cs.BeginSplitJobAbandon(ctx, job.JobID, 1400) + if err != nil { + t.Fatalf("begin split job abandon: %v", err) + } + if abandoning.Phase != SplitJobPhaseAbandoning || + abandoning.AbandonFromPhase != SplitJobPhasePlanned || + abandoning.UpdatedAtMs != 1400 { + t.Fatalf("unexpected planned abandon transition: %+v", abandoning) + } + if _, err := EncodeSplitJob(abandoning); err != nil { + t.Fatalf("encode planned abandon transition: %v", err) + } +} + +func TestCatalogStoreBeginSplitJobAbandonRejectsPostCutoverPhases(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(21) + job.Phase = SplitJobPhaseCleanup + job.RetryPhase = SplitJobPhaseNone + + if err := cs.CreateSplitJob(ctx, job); err != nil { + t.Fatalf("create split job: %v", err) + } + if _, err := cs.BeginSplitJobAbandon(ctx, job.JobID, 1300); !errors.Is(err, ErrCatalogSplitJobCannotAbandon) { + t.Fatalf("expected ErrCatalogSplitJobCannotAbandon, got %v", err) + } +} + func TestCatalogStoreListSplitJobsIncludesLiveAndHistory(t *testing.T) { cs := NewCatalogStore(store.NewMVCCStore()) ctx := context.Background() diff --git a/distribution/split_job_failure_test.go b/distribution/split_job_failure_test.go new file mode 100644 index 000000000..b2686ec8c --- /dev/null +++ b/distribution/split_job_failure_test.go @@ -0,0 +1,40 @@ +package distribution + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSplitJobFailurePreservesExactRetryPhase(t *testing.T) { + t.Parallel() + for _, phase := range []SplitJobPhase{SplitJobPhasePlanned, SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy, SplitJobPhaseCutover, SplitJobPhaseCleanup, SplitJobPhaseAbandoning} { + t.Run(string(phase), func(t *testing.T) { + t.Parallel() + job := SplitJob{JobID: 7, SourceRouteID: 1, SplitKey: []byte("m"), TargetGroupID: 9, Phase: phase, FenceTS: 50, SourceRetentionPinTS: 1} + if phase == SplitJobPhaseAbandoning { + job.AbandonFromPhase = SplitJobPhaseFence + } + failed, err := FailSplitJobState(job, "broken phase", 10) + require.NoError(t, err) + encoded, err := EncodeSplitJob(failed) + require.NoError(t, err) + failed, err = DecodeSplitJob(encoded) + require.NoError(t, err) + require.Equal(t, phase, failed.RetryPhase) + retried, err := RetrySplitJobState(failed, 20) + require.NoError(t, err) + require.Equal(t, phase, retried.Phase) + require.Equal(t, job.AbandonFromPhase, retried.AbandonFromPhase) + require.Equal(t, job.SourceRetentionPinTS, retried.SourceRetentionPinTS) + require.Equal(t, job.FenceTS, retried.FenceTS) + if phase == SplitJobPhaseAbandoning { + require.Equal(t, "broken phase", retried.LastError, "compensation retries retain the original failure evidence") + } else { + require.Empty(t, retried.LastError) + } + _, err = EncodeSplitJob(retried) + require.NoError(t, err) + }) + } +} diff --git a/distribution/split_job_lifecycle.go b/distribution/split_job_lifecycle.go new file mode 100644 index 000000000..9030b5ddc --- /dev/null +++ b/distribution/split_job_lifecycle.go @@ -0,0 +1,166 @@ +package distribution + +import ( + "bytes" + "context" + + "github.com/cockroachdb/errors" +) + +var ( + ErrCatalogSplitJobNotFound = errors.New("catalog split job is not found") + ErrCatalogSplitJobCannotRetry = errors.New("catalog split job cannot be retried") + ErrCatalogSplitJobCannotAbandon = errors.New("catalog split job cannot be abandoned") +) + +// RetrySplitJobState returns the durable state transition for RetrySplitJob. +// The retry target must come from the recorded RetryPhase witness; callers must +// not infer a phase from cursors, timestamps, or diagnostics after restart. +func RetrySplitJobState(job SplitJob, nowMs int64) (SplitJob, error) { + out := CloneSplitJob(job) + if out.Phase != SplitJobPhaseFailed || !splitJobRetryPhaseAllowed(out.RetryPhase) { + return SplitJob{}, errors.WithStack(ErrCatalogSplitJobCannotRetry) + } + out.Phase = out.RetryPhase + out.RetryPhase = SplitJobPhaseNone + if out.Phase != SplitJobPhaseAbandoning { + out.AbandonFromPhase = SplitJobPhaseNone + } + if out.Phase != SplitJobPhaseAbandoning { + out.LastError = "" + } + out.UpdatedAtMs = nowMs + return out, nil +} + +// FailSplitJobState preserves the exact phase and cleanup witnesses required +// for an explicit operator retry. It never drops fences or retention pins. +func FailSplitJobState(job SplitJob, diagnostic string, nowMs int64) (SplitJob, error) { + if !job.Phase.retryable() { + return SplitJob{}, errors.WithStack(ErrCatalogInvalidSplitJobPhase) + } + out := CloneSplitJob(job) + out.Phase, out.RetryPhase = SplitJobPhaseFailed, job.Phase + if job.Phase == SplitJobPhaseAbandoning && job.LastError != "" { + diagnostic = job.LastError + } + out.LastError, out.UpdatedAtMs = diagnostic, nowMs + return out, nil +} + +// BeginSplitJobAbandon records the durable ABANDONING witness before any +// pre-CUTOVER cleanup side effect is removed. +func BeginSplitJobAbandon(job SplitJob, nowMs int64) (SplitJob, error) { + out := CloneSplitJob(job) + from, ok := splitJobAbandonFromPhase(out) + if !ok { + return SplitJob{}, errors.WithStack(ErrCatalogSplitJobCannotAbandon) + } + if out.Phase == SplitJobPhaseAbandoning { + return out, nil + } + out.Phase = SplitJobPhaseAbandoning + out.RetryPhase = SplitJobPhaseNone + out.AbandonFromPhase = from + out.UpdatedAtMs = nowMs + return out, nil +} + +// RetrySplitJob CASes a FAILED live job back to its recorded retry phase. +func (s *CatalogStore) RetrySplitJob(ctx context.Context, jobID uint64, nowMs int64) (SplitJob, error) { + expected, _, err := s.LiveSplitJobForUpdate(ctx, jobID) + if err != nil { + return SplitJob{}, err + } + next, err := RetrySplitJobState(expected, nowMs) + if err != nil { + return SplitJob{}, err + } + return next, s.SaveSplitJob(ctx, expected, next) +} + +// BeginSplitJobAbandon CASes a live pre-CUTOVER job into ABANDONING. +func (s *CatalogStore) BeginSplitJobAbandon(ctx context.Context, jobID uint64, nowMs int64) (SplitJob, error) { + expected, _, err := s.LiveSplitJobForUpdate(ctx, jobID) + if err != nil { + return SplitJob{}, err + } + next, err := BeginSplitJobAbandon(expected, nowMs) + if err != nil { + return SplitJob{}, err + } + if SplitJobsEquivalent(expected, next) { + return next, nil + } + return next, s.SaveSplitJob(ctx, expected, next) +} + +// LiveSplitJobForUpdate reads the latest live split job and the snapshot +// timestamp used for the read. RPC callers use readTS as a CAS StartTS when +// proposing the update through Raft. +func (s *CatalogStore) LiveSplitJobForUpdate(ctx context.Context, jobID uint64) (SplitJob, uint64, error) { + if err := ensureCatalogStore(s); err != nil { + return SplitJob{}, 0, err + } + if jobID == 0 { + return SplitJob{}, 0, errors.WithStack(ErrCatalogSplitJobIDRequired) + } + ctx = contextOrBackground(ctx) + readTS := s.store.LastCommitTS() + job, found, err := s.liveSplitJobAt(ctx, jobID, readTS) + if err != nil { + return SplitJob{}, 0, err + } + if !found { + return SplitJob{}, 0, errors.WithStack(ErrCatalogSplitJobNotFound) + } + return job, readTS, nil +} + +func splitJobRetryPhaseAllowed(phase SplitJobPhase) bool { + switch phase { + case SplitJobPhasePlanned, SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy, SplitJobPhaseCutover, SplitJobPhaseCleanup, SplitJobPhaseAbandoning: + return true + case SplitJobPhaseNone, SplitJobPhaseDone, SplitJobPhaseFailed, SplitJobPhaseAbandoned: + return false + } + return false +} + +func splitJobAbandonFromPhase(job SplitJob) (SplitJobPhase, bool) { + switch job.Phase { + case SplitJobPhasePlanned, SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy: + return job.Phase, true + case SplitJobPhaseFailed: + if splitJobAbandonRetryPhaseAllowed(job.RetryPhase) { + return job.RetryPhase, true + } + case SplitJobPhaseAbandoning: + if splitJobAbandonRetryPhaseAllowed(job.AbandonFromPhase) { + return job.AbandonFromPhase, true + } + case SplitJobPhaseNone, SplitJobPhaseCutover, SplitJobPhaseCleanup, SplitJobPhaseDone, SplitJobPhaseAbandoned: + } + return SplitJobPhaseNone, false +} + +func splitJobAbandonRetryPhaseAllowed(phase SplitJobPhase) bool { + switch phase { + case SplitJobPhasePlanned, SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy: + return true + case SplitJobPhaseNone, SplitJobPhaseCutover, SplitJobPhaseCleanup, SplitJobPhaseDone, SplitJobPhaseFailed, SplitJobPhaseAbandoning, SplitJobPhaseAbandoned: + return false + } + return false +} + +// SplitJobsEquivalent reports whether two split jobs encode to the same +// durable catalog value. +func SplitJobsEquivalent(left, right SplitJob) bool { + leftRaw, leftErr := EncodeSplitJob(left) + rightRaw, rightErr := EncodeSplitJob(right) + if leftErr != nil || rightErr != nil { + return false + } + return bytes.Equal(leftRaw, rightRaw) +} diff --git a/docs/design/2026_02_18_partial_hotspot_shard_split.md b/docs/design/2026_02_18_partial_hotspot_shard_split.md index 20a0ccdab..5d5aaf854 100644 --- a/docs/design/2026_02_18_partial_hotspot_shard_split.md +++ b/docs/design/2026_02_18_partial_hotspot_shard_split.md @@ -10,7 +10,7 @@ Current implementation status (updated August 22, 2026): `SplitRange` are implemented. - M2 has the SplitJob catalog substrate (`distribution/split_job_catalog.go`) and codec/list/history tests, tracked in - [`2026_06_11_partial_hotspot_split_milestone2_migration.md`](2026_06_11_partial_hotspot_split_milestone2_migration.md); + [`2026_06_11_implemented_hotspot_split_milestone2_migration.md`](2026_06_11_implemented_hotspot_split_milestone2_migration.md); the actual migration RPC, export/import, fence, cutover, promotion, and cleanup workflow are still open. - Standalone M3 auto-split is implemented: keyviz supplies the sole load signal, @@ -299,6 +299,14 @@ Add RPCs: 2. Job phases: BACKFILL/FENCE/DELTA/CUTOVER 3. Manual split with target-group relocation +Status: implemented. The production runner advances durable cross-group jobs +through BACKFILL, FENCE, DELTA_COPY, CUTOVER, CLEANUP, and `DONE`; current-voter +readiness and cleanup barriers protect leadership changes; and the deterministic +Jepsen workload covers the split alongside leader-kill and partition packages. +The completed M2 contract is recorded in +[`2026_06_11_implemented_hotspot_split_milestone2_migration.md`](2026_06_11_implemented_hotspot_split_milestone2_migration.md) +and its final runner/lifecycle slice landed in PR #1096. + ### Milestone 3: Automation 1. Access aggregation diff --git a/docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md b/docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md similarity index 95% rename from docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md rename to docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md index 4743e3e0a..1d1f2db2f 100644 --- a/docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md +++ b/docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md @@ -1,14 +1,42 @@ # Hotspot Shard Split — Milestone 2: Migration Plane -Status: Partial +Status: Implemented Author: bootjp Date: 2026-06-11 Parent: [2026_02_18_partial_hotspot_shard_split.md](2026_02_18_partial_hotspot_shard_split.md). M1 (control plane) is as-built in [2026_02_18_implemented_hotspot_split_milestone1_pr.md](2026_02_18_implemented_hotspot_split_milestone1_pr.md). +Current implementation status: implemented. The M2 stack provides the durable SplitJob catalog, versioned route descriptors, resumable range export/import, staged/live visibility, source write and read fences, current-voter readiness and cleanup barriers, constant-time route cutover, incremental target promotion, bounded source/target cleanup, terminal `DONE` history, and the deterministic cross-group Jepsen workload. PR #1096 completed the production runner and lifecycle wiring. Automatic hotspot detection and split scheduling remain M3 scope; the broader route-shuffle and fault campaign remains M4 scope. + +Migration control RPCs (`StartSplitMigration`, `AbandonSplitJob`, and `RetrySplitJob`), internal migration mutators (`ApplyTargetStagedReadiness`, `ImportRangeVersions`, `PromoteStagedVersions`, and `CleanupMigration`), and raw MVCC export streams (`ExportRangeVersions`) require the dedicated cluster-shared migration controller/node token configured by `--splitMigrationTokenFile`. This privileged credential must differ from the read-only Admin and S3 blob peer tokens; neither existing token implicitly grants migration authority. Restrict its distribution to nodes and privileged migration controllers. Without it, migration access is denied unless `--adminInsecureNoAuth` explicitly enables unauthenticated access. Runner source, target, voter, and promotion clients authenticate unary calls and export streams without modifying shared peer connections. The non-local split Jepsen workload explicitly enables `--adminInsecureNoAuth` only on its isolated migration-test nodes, matching its unauthenticated helper calls. Each voter readiness probe has a 500 ms attempt deadline; a timeout clears that voter's acknowledgement and lets the runner retry on a later tick. + +Token-authenticated migration requires verified TLS: clients never send these bearer credentials on plaintext connections. Configure `--grpcTLSCertFile` and `--grpcTLSKeyFile` on every node to enable TLS for group listeners and peer clients, and `--grpcTLSCAFile` when the peer certificates use a private CA. Certificates must cover the advertised peer hostnames or IP addresses. Roll out this opt-in transport configuration consistently across the cluster; the ordinary peer transport remains unchanged when the TLS flags are absent. + +The `elastickv-split` CLI reads the dedicated migration credential (not the read-only Admin token) from `--token-file`, which automatically requires TLS and validates the server certificate against system roots or `--tls-ca-file`. There is no certificate-verification bypass. Explicit no-auth development clusters can still use plaintext without a token. + +`elastickv-split --tls-server-name ` sets the expected certificate identity when connecting through an IP or alias; it implies verified TLS and never disables SAN verification. Empty-prefix Forward deletion is a privileged full-keyspace operation: it also requires the migration token over TLS before peer enforcement, since it can erase the migration catalog. Ordinary bounded user writes remain compatible; isolated test no-auth is explicit. + +For operator recovery, `elastickv-split --retry-job-id --token-file --tls-ca-file ` invokes authenticated `RetrySplitJob`. Retry, abandon, inspect and split modes are mutually exclusive. During send-first rollout, Forward writes to live/history job keys or the next-job allocator already require the dedicated token over TLS, including prefix deletes intersecting those namespaces. They are never accepted as legacy peer writes, even before token configuration; ordinary forwarded user transactions remain compatible. Explicit isolated-test no-auth remains the only bypass. Keep old binaries behind trusted peer-only listeners throughout the rollout; an older server does not implement this control-write protection. + +Capability-regression guards use the durable record actually read/updated by the CAS, including already-regressed records, so same-phase cleanup progress cannot be undone by stale input. Terminal-history pagination seeks after its `(terminal_at_ms, job_id)` boundary even when GC has removed the anchor; malformed cursors and missing live-job anchors still fail explicitly. + +The read-only `elastickv-list-routes` CLI supports `--tls`, `--tls-ca-file`, and `--tls-server-name` against the same listeners. A CA file or server name implies verified TLS; without these flags, the explicit development/plaintext behavior is unchanged. Job queries (`GetSplitJob` and `ListSplitJobs`) honor the same startup read gate as route queries. + +Export streaming and each import have independent two-second child deadlines; new and duplicate acknowledgements are persisted with the parent runner context, even when the RPC budget expires at acknowledgement. Source/target data and metadata cleanup, fence/guard application, source timestamp issuance and lock probes each have an independent two-second RPC deadline, also separate from durable catalog progress. After the source lock drain completes, the runner reapplies the identical Raft fence control to refresh the write-tracker minimum before persisting the delta floor. All migration-only internal probes and timestamp issuance require the same migration token and TLS as mutations and export. Applying an armed readiness guard atomically advances the durable migration and store clock floors to at least its write floor and observes that floor on every replica before cutover, including for empty ranges. + +Startup rejects a migration token without gRPC TLS and rejects peer enforcement without a token, even when migration opcodes are disabled (explicit isolated-test no-auth is the only exception). The send-first authentication rollout assumes an established TLS transport; switching a plaintext cluster to TLS is a separate coordinated transport configuration change. Source cleanup appends a route-filtered DynamoDB OCC seal bracket without changing existing family IDs or positional cursor indexes. These local conflict witnesses are excluded from copy/export. A separate source-cleanup completion marker ensures live jobs with the old done cursor resume at the appended seal bracket, page durably, and retire only versions at or below the fence before metadata/history cleanup. + +With `--splitMigrationPeerAuthEnforced`, `Internal.Forward` is an authenticated TLS peer RPC too: otherwise arbitrary transaction metadata/catalog writes could turn the runner into a privileged deputy. Common node dial options attach the configured dedicated token only to `Forward`, including on existing cached connections; Admin and other RPC credentials remain independent. Roll authentication out in two stages: first deploy the same token, verified TLS and this binary to every node with enforcement left false (the default), so old/no-token peers can still forward ordinary writes; then, only after every peer sends the credential, roll all nodes to enforcement true. The send-only stage does not start the privileged migration runner and advertises no migration capability, even if opcode flags are enabled. Keep the shared group listeners restricted to trusted peers during rollout; send-only is not secure migration mode. Migration starts only after all voter capability probes pass the enforcement/readiness gate. An already authenticated cluster upgrading from a binary without Forward credentials uses the same send-first stage rather than immediately enforcing on new nodes. Any enabled migration opcode requires the token or explicit isolated-test `--adminInsecureNoAuth`; that test-only mode intentionally bypasses peer enforcement. Ordinary clusters with migration disabled retain their existing peer transport. Public RawKV/TransactionalKV mutations reject reserved control and concrete transaction-internal keys, so they cannot bypass peer authentication through a local coordinator. Before any CLEANUP dispatch, the runner independently requires a monotonic cutover version, a finite fence, removal of the source parent and a matching active target descriptor/write floor/promotion state; persisted done cursors alone are never authority to delete source data. + +Legacy jobs missing their source group first recover it from current routes or catalog MVCC snapshots at the job's snapshot/fence timestamp. When catalog history is already compacted, the runner probes configured groups for an armed durable source guard matching the exact job ID, migration ID and moving range, excluding target-only readiness. This source metadata is independent of the catalog group's retention window; a unique owner is persisted before cleanup resumes. The additive `SOURCE_GUARD_PRESENT` probe requires an applied leader read and bounded attempts. Missing or ambiguous evidence remains fail-closed; a present-day predecessor is never guessed to be the original owner. + ## 1. Background +Metadata cleanup atomically persists a per-job retirement witness with removal of live guards, import/promotion progress and the applied index. All subsequent readiness, import, promotion and version-cleanup mutations for that job are rejected before duplicate or empty/Done shortcuts. `METADATA_CLEARED` requires both a durable witness and absence of every active metadata family; unsupported stores fail closed. Witnesses are neither active guards nor retention pins and are not collected by TTL, job-history GC or ordinary compaction. They survive Pebble reopen, native/SST snapshots and MVCC streaming restore; MVCC snapshots use V5 only when witnesses exist, retaining older formats otherwise. Legacy retirement and repeated metadata cleanup preserve the witnesses. Job IDs must not be reused. + +Catalog cross-group split migration rejects a moving interval containing `!sqs|route|global` with `FailedPrecondition`, before allocating a job or arming source guards. SQS partition ownership is maintained by its separate resolver, and catalog cutover does not update that partition map. The same structural restriction applies to resumed pre-cutover jobs; an older overlapping job must be abandoned rather than retried into a catalog fence. Abandonment and post-cutover cleanup remain available for recovery. Ordinary SQS data, partition ownership, generic export brackets and same-group splits are unchanged. A moving interval whose exclusive end equals the global SQS route, or which leaves that route on the source side, remains supported. + M1 landed the control plane: - `distribution/catalog.go` — durable `RouteDescriptor` records in the default Raft group's MVCC state. @@ -21,14 +49,6 @@ Composed-1 cross-group commit guard (PR #927) and the routeKey normalization fix M2 does **not** ship hotspot detection or auto-split scheduling (M3). -Implementation status (updated 2026-07-23): the durable SplitJob -catalog substrate has landed in `distribution/split_job_catalog.go` -and `proto/distribution.proto`, with codec/list/history tests in -`distribution/split_job_catalog_test.go`. The `StartSplitMigration` -RPC, migrator state machine, source/target export/import, fencing, -cutover, promotion, cleanup, and capability gate remain open, so the -document is partial rather than implemented. - ## 2. Goals and Non-Goals ### 2.1 Goals @@ -92,7 +112,7 @@ Key layout: | `target_cleared_descriptor_ack_cursor` | bytes | Opaque per-voter ACK progress for §6.4 step 5. Each ACK means a target voter has applied a target-group Raft entry proving its local serving route snapshot has observed the default-group descriptor with `staged_visibility_active=false` / `migration_job_id=0` and the retained `min_write_ts_exclusive` at `promotion_completed_ts`. Target-local import ACKs, per-job HLC floor, readiness, and promotion proof records cannot be deleted until every current target voter has ACKed this cleared-descriptor proof. | | `bracket_progress` | repeated `BracketProgress` | Per-family export bracket state for §6.3.1's parallel exports. Each entry carries `(bracket_id uint64, family uint32, export_phase enum, cursor bytes, done bool, scanned_bytes uint64, accepted_rows uint64, last_acked_batch_seq uint64)`. `bracket_id` is stable for the life of the SplitJob and forms part of the §6.1.1 importer idempotency key `(job_id, bracket_id, batch_seq)`. `last_acked_batch_seq` is monotone across the whole job, not reset per phase. When BACKFILL completes, the `BACKFILL -> DELTA_COPY` transition rewrites every bracket to `export_phase=DELTA_COPY`, clears `cursor`, `done`, `scanned_bytes`, and `accepted_rows`, and preserves `last_acked_batch_seq`; the first DELTA_COPY import uses `batch_seq = last_acked_batch_seq + 1`. This explicit phase boundary reopens exhausted BACKFILL brackets without reusing importer idempotency keys. A leader flap resumes the current phase of each bracket independently. `phase` advances from BACKFILL/DELTA_COPY only when every entry for that phase has `done = true`. | | `source_retention_pin_ts` | uint64 | Source-group MVCC retention floor witness (§6.1.0a). The source group installs a route-scoped no-prune pin (`min_ts=0`, the BACKFILL lower bound) before BACKFILL opens and keeps it until every BACKFILL bracket is done, because BACKFILL exports `(0, snapshot_ts]`. After `delta_floor` is finalised, the same source-local pin is relaxed to `delta_floor` for DELTA_COPY's `(delta_floor, fence_ts]` window. Released only after successful source cleanup or the abandon/FAILED cleanup path disarms the job. | -| `last_error` | string | Most recent transient failure (for operator diagnosis). | +| `last_error` | string | Persistent phase-failure diagnostic, recorded atomically with FAILED and cleared on explicit retry. | | `started_at`, `updated_at`, `terminal_at` | int64 | Wall-clock ms; **diagnostic only**, must not be read by any ordering-sensitive logic (CLAUDE.md HLC rule). `terminal_at` is set when phase enters DONE or ABANDONED; FAILED keeps the job live and does not set a terminal history key. | Encoding mirrors `RouteDescriptor` (a single-byte codec version prefix + protobuf body). Reserve a new codec constant `catalogJobCodecVersion = 1`. @@ -103,6 +123,8 @@ All per-voter ACK cursors are keyed by the Raft configuration epoch / voter-set #### 3.1.1 Job retention (bounded storage) +The runner bounds persistent phase errors to five consecutive non-transient attempts in the same phase. Success, cancellation, leadership/RPC unavailability, deadline, resource exhaustion, write conflict, or an incomplete voter barrier resets this in-memory budget. Once exhausted, a default-group transaction CASes the unchanged attempted job to `FAILED`, with its exact phase in `retry_phase` and the error in `last_error`; concurrent catalog progress prevents a stale attempt from failing the newer job. Fences, pins, cursors, and guards remain intact, and automatic attempts stop until operator retry. `PLANNED` and `ABANDONING` are also exact retry targets; failed abandon retains `abandon_from_phase` and resumes abandon cleanup, never the old migration phase. Restarting before exhaustion resets only the in-memory budget; a persisted FAILED job remains stopped after restart. + Without a retention policy the `!dist|job|*` namespace grows unboundedly across a long-lived cluster (gemini medium on PR #945). M2 ships these bounds: - **Live cap and same-group split exclusion**: M2 v1 allows at most `maxLiveJobs = 1` non-terminal SplitJob in the catalog at any time. `StartSplitMigration` rejects a second cross-group request with `ErrTooManyInFlightJobs` while any job is live, including retryable `FAILED` jobs. The existing M1 `SplitRange` same-group path remains the same wire shape, but it must preflight the SplitJob catalog and reject with `ErrSplitJobOverlap` if the requested parent route / resulting child interval intersects any live SplitJob's `(route_start, route_end)` on either source or target side. A disjoint same-group split can proceed; an overlapping one cannot rewrite route IDs/bounds while BACKFILL/FENCE/DELTA_COPY/CLEANUP still rely on the job's original route interval, fence records, staged prefixes, and cleanup cursors. This matches §8's single migrator goroutine and avoids under-specified interactions between overlapping source/target fences, HLC floors, staged-readiness barriers, and cleanup. M3 may raise the cap only after adding explicit conflict detection for overlapping route intervals, target readiness records, and cleanup cursors. @@ -1239,17 +1261,17 @@ Throttling: chunk-bytes default 1 MiB, inter-chunk pacing default 5 ms. Both con Phased into reviewable PRs, each lands behind its own doc-or-test gate: -| PR | Scope | Tests / status | +| PR | Scope | Tests | |---|---|---| | M2-PR0 | Prepare-time `commit_ts` plumbing behind the PR7 cluster capability gate: decoder/helpers understand both legacy and new lock values; while the gate is closed, normal PREPARE/COMMIT remains byte-compatible with M1 and writes neither lock `commit_ts` fields nor per-key `!txn|ok|` markers; once the gate is open, the coordinator allocates/fences `commit_ts` before PREPARE, prepare request and txn metadata carry it to every participant, `handlePrepare` writes the same value into each `txnLock`, the migration write tracker records that value at prepare admission, and COMMIT reuses it while writing length-prefixed per-key `!txn|ok|` success markers for every committed participant. Keep decoder backward-compat for existing lock values without the field (zero → migrator fallback to direct commit-record lookup / pending drain). | `kv/txn_codec_test.go` round-trip + backward-compat case; `kv/txn_prepare_commit_ts_test.go` covers gate-closed mixed-cluster traffic emitting legacy locks/no `!txn|ok|`, then gate-open coordinator → prepare request/meta → `handlePrepare` → tracker path + per-key success marker; red controls for codec-only, commit-time allocation, primary-only commit proof, and writing PR0 side effects before the gate opens | -| M2-PR1 | SplitJob codec + reserved-key layout + catalog read/write helpers | Implemented in `distribution/split_job_catalog.go` and `distribution/split_job_catalog_test.go` | +| M2-PR1 | SplitJob codec + reserved-key layout + catalog read/write helpers | catalog_test | | M2-PR2 | `proto/distribution.proto` + `proto/internal.proto` regen (`MVCCVersion.expire_at`, bracket/batch identity, scan-budget fields); wire fields plumbed but unused | proto regen; build | | M2-PR3 | `store/MVCCStore.ExportVersions` raw committed-version export + `ImportVersions` with idempotency, tombstone/`expire_at` preservation, target-local full-HLC floor persistence, durable RouteDescriptor v2 codec/dual decoder, and descriptor `min_write_ts_exclusive` plumbing, no migrator wiring | store unit incl. tombstone/TTL + HLC-floor restore cases + property; `distribution/catalog_test.go` v1/v2 route descriptor codec matrix | | M2-PR4 | `distribution/migrator.go` state machine + export bracket planner, including list delta/claim, Redis wide-column brackets, concrete-only SQS/S3 brackets, txn success marker routing, drain-only txn locks, and matching `routeKey()` decoder coverage — same-group target (no-op data move) to lock the state shape | migrator unit + `migrator_export_plan_test` | | M2-PR5 | Coordinator + FSM FENCE rejection (`ErrRouteWriteFenced`) with point and `DEL_PREFIX` range-footprint checks + route-faithful txn-lock drain (§3.2a.0a) + same-group `SplitRange` overlap rejection while a SplitJob is live | fsm + coordinator unit + `migrator_lock_drain_test` + `catalog_test` overlap red controls | | M2-PR6 | Cross-group end-to-end: ExportRangeVersions / ImportVersions server-side handlers + migrator BACKFILL/DELTA_COPY + raw-candidate staged/live merge read path in both scan directions and `LatestCommitTS` + §7.2.2e source-side cutover read-fence arm with every-current-source-voter ACK, membership-epoch re-ACK, catalog-version waiter, route-key-normalized scan ownership checks, and server-stamped RawKV read versions | integration incl. delete/TTL DELTA_COPY, Redis list/hash/set/zset/stream migration, HLC restart/fence-floor cases + `kv/fsm_cutover_read_fence_test.go` | | M2-PR7 | Target-local `PromotionState` + background promoter + ordered default-group promotion-complete CAS retaining `min_write_ts_exclusive` + source/target cleanup before DONE history move; readiness guard accepts matching cleared descriptors while retained; `AbandonSplitJob` with durable `ABANDONING` cleanup witness + CLEANUP GC + Jepsen split workload | `kv/fsm_promote_staged_test.go` raw hidden-version/LatestCommitTS merge + jepsen suite | -| M2-PR8 | Rename `*_proposed_*` -> `*_partial_*` after PR1 ships; update parent partial doc M2 status; rename to `*_implemented_*` after PR7 | Partial rename complete; implemented rename remains gated on PR7 | +| M2-PR8 | Rename `*_proposed_*` → `*_partial_*` after PR1 ships; update parent partial doc M2 status; rename to `*_implemented_*` after PR7 | Completed after the runner, voter barriers, cleanup lifecycle, route publication, and Jepsen workload landed in PR #1096. | Each PR follows the five-lens self-review and is gated by its tests + `make lint`. @@ -1289,9 +1311,12 @@ This is independent of the existing rolling-upgrade protocol for unrelated subsy ## 14. Lifecycle -This document began as `*_proposed_*` and is now `*_partial_*` because -M2-PR1's SplitJob catalog substrate has landed. Per `docs/design/README.md`: +This document is `*_implemented_*` because the M2 cross-group migration plane is complete as a central subsystem. The production runner now resumes durable jobs across leadership changes, drives every phase through `CLEANUP`, waits for current source and target voter proofs, publishes the target route, removes bounded migration state, and moves the job to `DONE` history. The deterministic Jepsen split workload exercises the cross-group operation alongside leader-kill and partition fault packages. + +The remaining work is intentionally outside M2's central scope: -- Rename to `*_implemented_*` after M2-PR7 ships and the parent partial doc's M2 row is checked off. +- M3 owns automatic hotspot detection, target selection, and split scheduling. +- M4 owns the broader route-shuffle nemesis matrix and production-scale fault campaigns. +- Reverse migration after CUTOVER and concurrent migration jobs remain future extensions. `git mv` is used so history follows. diff --git a/docs/design/2026_06_11_implemented_hotspot_split_milestone3_automation.md b/docs/design/2026_06_11_implemented_hotspot_split_milestone3_automation.md index e9c3fd670..0cc5be6a9 100644 --- a/docs/design/2026_06_11_implemented_hotspot_split_milestone3_automation.md +++ b/docs/design/2026_06_11_implemented_hotspot_split_milestone3_automation.md @@ -11,7 +11,7 @@ Date: 2026-06-11 Updated: 2026-07-23 Parent: [2026_02_18_partial_hotspot_shard_split.md](2026_02_18_partial_hotspot_shard_split.md) §12 "Milestone 3: Automation" (1. Access aggregation, 2. Hotspot detector, 3. Auto-split scheduler with cooldown/hysteresis). -Sibling: [2026_06_11_partial_hotspot_split_milestone2_migration.md](2026_06_11_partial_hotspot_split_milestone2_migration.md) (M2 — migration plane). M3 composes with M2 but does **not** depend on it for first value (see §2, §6). +Sibling: [2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md) (M2 — migration plane). M3 composes with M2 but does **not** depend on it for first value (see §2, §6). ## 1. Background diff --git a/docs/design/2026_06_12_proposed_scaling_roadmap.md b/docs/design/2026_06_12_proposed_scaling_roadmap.md index 8d66e7ddb..6d3fa34ac 100644 --- a/docs/design/2026_06_12_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_12_proposed_scaling_roadmap.md @@ -4,16 +4,13 @@ **Author:** bootjp **Date:** 2026-06-12 -> **Superseded roadmap index:** > This document is retained as the original sequencing and SLO record. It is > not an implementation specification and must not be used as the scope for a > combined implementation PR. The > [2026-06-23 roadmap](2026_06_23_proposed_scaling_roadmap.md) audits the current > implementation, closes completed prerequisites, and tracks the remaining -> work as separate design documents and milestone PRs. It is the current -> ownership and sequencing authority; this historical document does not -> describe the current implementation status of its milestones. -> +> work as separate design documents and milestone PRs. + > This roadmap captures the scaling work elastickv needs across four > subsystems that today have known ceilings and have not yet been > designed past those ceilings. The roadmap itself is not a single @@ -21,8 +18,8 @@ > sibling `*_proposed_*.md` design that will land later as its own PR. > Per CLAUDE.md the design-doc-first workflow applies: each milestone > below ships its own `*_proposed_*` doc before its implementation -> starts. The June 23 ownership index now records the authoritative -> requirement-by-requirement disposition and sequencing constraints. +> starts; this file is the shared north star and the sequencing +> constraint between them. ## 1. Motivation @@ -33,7 +30,7 @@ production envelope within one growth cycle: | Dimension | Today's comfortable ceiling | Where the next operator wants to go | Triggering signal | | --- | --- | --- | --- | -| Routes (shards) per cluster | ~10 k (binary search + 100 ms watcher + history ring of 32 versions) | 100 k–1 M routes | hotspot-split-M3 automation (`docs/design/2026_06_11_implemented_hotspot_split_milestone3_automation.md`) generates routes faster than the catalog watcher can fan out at scale | +| Routes (shards) per cluster | ~10 k (binary search + 100 ms watcher + history ring of 32 versions) | 100 k–1 M routes | hotspot-split-M3 automation (`docs/design/2026_06_11_partial_hotspot_split_milestone3_automation.md`) generates routes faster than the catalog watcher can fan out at scale | | Region/DC fan-out | Single DC, LAN-tuned etcd/raft (10 ms tick, 100 ms heartbeat, 1 s election), single default-group HLC ceiling | 2–3 regions with active-active per-shard | Disaster-recovery SLO ("survive a region outage with < 30 s data-plane reconvergence") | | Bytes per shard | ~1 TB before snapshot transfer / WAL replay becomes operational pain | 5–10 TB per shard, with shard counts in the 100 k range | DynamoDB-compatibility customers ingesting at multi-GiB/s per table | | Per-node QPS | Leader-bound write, lease-read scales across shards but only on each shard's leader | 5–10x current per-node throughput; sustained reads from non-leader replicas | Redis-compatibility migration projects ingesting from upstream Redis where the upstream serves reads from replicas | @@ -87,13 +84,12 @@ Breakage points: references anywhere in `kv/`, `distribution/`, or `internal/raftengine/etcd/`. Everything is single-region etcd/raft. -HLC is `kv/hlc.go` + `kv/coordinator.go` + -`kv/sharded_coordinator.go`: physical ceiling is Raft-agreed via -`ProposeHLCLease`, `hlcPhysicalWindowMs = 20 s`, renewed every -`hlcRenewalInterval = 1 s` by each led shard group -(`ShardedCoordinator.RunHLCLeaseRenewal`). `NextFenced` fails closed -with `ErrCeilingExpired` once `wall_now >= ceiling`. Wall clock is -`time.Now().UnixMilli()` — assumes NTP-synced hosts. +HLC is `kv/hlc.go` + `kv/coordinator.go`: physical ceiling is +Raft-agreed via `ProposeHLCLease`, `hlcPhysicalWindowMs = 3 s`, +renewed every `hlcRenewalInterval = 1 s` from the **default-group +leader only** (`ShardedCoordinator.RunHLCLeaseRenewal`). `NextFenced` +fails closed with `ErrCeilingExpired` once `wall_now >= ceiling`. +Wall clock is `time.Now().UnixMilli()` — assumes NTP-synced hosts. Raft engine (`internal/raftengine/etcd/engine.go`): `defaultTickInterval = 10 ms`, `defaultHeartbeatTick = 10` (100 ms), @@ -102,11 +98,10 @@ LAN-tuned; would spuriously election-time-out cross-WAN. Multi-region blockers: -- HLC ceiling renewal is **per Raft group** — each shard group leader - proposes every 1 s, so cross-region leadership for that group makes - its persistence-grade ts depend on cross-WAN quorum progress. 1 s WAN - RTT is now less likely to expire the 20 s window immediately, but - every renewal still depends on that group's cross-WAN quorum progress. +- HLC ceiling is **single-point** — default-group leader proposes + every 1 s; cross-region default-group leadership means every + shard's persistence-grade ts depends on a cross-WAN propose. 1 s + WAN RTT + 1 s renewal interval ≈ no margin against the 3 s window. - 100 ms heartbeat / 1 s election timeout cannot run cross-DC. - No TrueTime/clockbound integration. - Cross-shard txns are blocked (`ErrCrossShardTransactionNotSupported`), @@ -114,14 +109,10 @@ Multi-region blockers: ### 2.3 Storage tier (Pebble) -`store/lsm_store.go`: per-shard `pebble.Open`, process-wide shared block cache -sized to 25% of the node's hard memory capacity by default and shared by all -stores in the process. Linux uses the minimum positive cgroup limit found from -the process leaf through every ancestor, bounded by physical RAM. `GOMEMLIMIT` -is not used as a total RSS budget because the Pebble cache is off-heap. -`ELASTICKV_PEBBLE_CACHE_PERCENT` changes the hard-capacity fraction; -`ELASTICKV_PEBBLE_CACHE_MB` is the absolute-capacity override. WAL sync via -`ELASTICKV_FSM_SYNC_MODE` (default `pebble.Sync` on FSM apply; `nosync` opt-in). +`store/lsm_store.go`: per-shard `pebble.Open`, default block cache +`defaultPebbleCacheBytes = 256 MiB` **per store** (not shared). +WAL sync via `ELASTICKV_FSM_SYNC_MODE` (default `pebble.Sync` on +FSM apply; `nosync` opt-in). `store/mvcc_store.go`: encoded as `UserKey ++ 0x00 ++ inverted_TS`, `maxSnapshotVersionCount = 1 M`, `maxSnapshotValueSize = 256 MiB`. @@ -145,9 +136,8 @@ Storage breakage at 1–10 TB/shard: is tens of minutes. - Pebble L0CompactionThreshold / LBaseMaxBytes / compaction concurrency are defaults; write-heavy shards hit stall thresholds. - Shared block cache M1 has landed, so resident block-cache memory is capped per - process rather than N × 256 MiB. Shared memtable / compaction concurrency - budgeting and per-group cache fairness remain open. + 256 MiB block cache per shard × N shards/node = N × 256 MiB + resident memory (shared-cache TODO not landed). - MVCC retention 30 min + per-key 1 M version cap means a key written ≥ 555/s for 30 min trips the cap; compactor runs every 5 min so read tail latency spikes during accumulation. @@ -321,10 +311,10 @@ whole cluster; cross-WAN that is a 1 s RTT cliff. Options surveyed: service.** Operationally heavy (needs reliable atomic-clock reference per DC); rejected as v1. - **Option C — Stretched single ceiling with relaxed window.** - `hlcPhysicalWindowMs = 20 s` is now the local-resilience baseline. - It reduces accidental ceiling expiry under load, but renewal still - depends on each route's shard group quorum and is not a complete - cross-region design by itself. + Increase `hlcPhysicalWindowMs` to 30 s; renewal still cross-WAN + but the failure mode is "slow ts allocation," not "no ts + allocation." Useful as a fallback when option A is partially + shipped. V1 = option A. Carries the existing M2 hotspot-split monotone-merge contract over the region boundary. @@ -353,9 +343,9 @@ control-plane (`*_proposed_*` doc TBD).** - M1 standalone but doesn't enable cross-region writes — it just makes Raft survive cross-WAN partition. -- M2 depends on the M2 hotspot-split migration contract - (`2026_06_11_partial_hotspot_split_milestone2_migration.md`) - being implemented so the monotone-merge primitive exists. +- M2's hotspot-split migration dependency is implemented in + [2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md), including + the monotone-merge primitive. - M3 depends on M1's region-aware membership and M2's per-region ceiling. - M4 depends on M2 and M3. @@ -374,8 +364,7 @@ control-plane (`*_proposed_*` doc TBD).** ### 5.2 Design — milestones -**M1 — SST ingest snapshot transfer -([focused design](2026_07_19_implemented_pebble_sst_ingest_snapshot_transfer.md)).** +**M1 — SST ingest snapshot transfer (`*_proposed_*` doc TBD).** - Replace `pebbleSnapshot.WriteTo`'s full-iter stream with **Pebble SST-level snapshot transfer**: leader takes a Pebble snapshot, copies the live SSTs (+ memtable flush) as @@ -384,12 +373,11 @@ control-plane (`*_proposed_*` doc TBD).** work. - Streamed in parallel across the leader's outgoing transport. -**M2 — Shared block-cache follow-ups + per-shard tuning (`*_proposed_*` doc +**M2 — Shared block cache + per-shard tuning (`*_proposed_*` doc TBD).** -- M1 shared block cache has landed: one `pebble.Cache` per process is shared - across all shards' stores, defaults to 25% of the node's effective memory - budget, and supports percentage or absolute-MiB operator overrides. -- Add shared memtable / compaction concurrency budgets across stores. +- Land the existing shared-cache TODO: one `pebble.Cache` per node + shared across all shards' stores, sized as a per-node config + fraction of available RAM (default 25%). - Surface `L0CompactionThreshold`, `LBaseMaxBytes`, `MaxConcurrentCompactions` as per-shard config so a write-heavy shard can be tuned without touching the cluster default. @@ -413,8 +401,8 @@ TBD).** rows) so they do not pile up in L0 between compactor runs — reuses the existing TTL helper path. -**M4 — Disaster-recovery snapshot offload -([`2026_07_19_partial_physical_snapshot_object_offload.md`](2026_07_19_partial_physical_snapshot_object_offload.md)).** +**M4 — Disaster-recovery snapshot offload (`*_proposed_*` doc +TBD).** - Periodic per-shard Pebble snapshot uploaded to an S3-compatible bucket (the S3 adapter already speaks the protocol). - Restore is `s3 fetch → pebble.Ingest`; combined with M1's @@ -550,7 +538,7 @@ the ceiling shape: Composability invariant: **every monotone-merge happens via the same `SetPhysicalCeiling` + `Observe` primitive**. The M2 hotspot-split contract (§6.2.1 of -`2026_06_11_partial_hotspot_split_milestone2_migration.md`) is the +[2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md)) is the reference implementation; per-region and per-group merges reuse it. ### 7.2 Capability bits diff --git a/docs/design/2026_06_23_proposed_scaling_roadmap.md b/docs/design/2026_06_23_proposed_scaling_roadmap.md index 1900135d3..210d52c14 100644 --- a/docs/design/2026_06_23_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_23_proposed_scaling_roadmap.md @@ -1,240 +1,363 @@ -# Scaling roadmap ownership index +# Scaling Roadmap — what exists, what is proposed, what is missing Status: Proposed -Document type: Roadmap ownership index Author: bootjp Date: 2026-06-23 -Last audited: 2026-08-27 (UTC) against `origin/main` and GitHub pull requests - -## 1. Scope - -This document owns no runtime behavior. It is the canonical ownership and -dependency index for scaling work. A behavior change belongs to the focused -design document or pull request named below, not to this roadmap. - -The audit follows these rules: - -1. One focused design owns one mechanism. Its implementation may use a stacked - pull request series when the change cannot be reviewed safely as one patch. -2. An implemented document and merged implementation are authoritative over a - proposal or historical roadmap description. -3. An open pull request is in flight, not implemented on `main`. -4. A requirement with neither a focused design nor an implementation pull - request is explicitly unowned. This roadmap only defines the boundary of the - missing design; it does not silently become that design. - -## 2. Relationship to the 2026-06-12 roadmap - -This document supersedes -`2026_06_12_proposed_scaling_roadmap.md` as the roadmap index and sequencing -authority. It does not claim that all mechanisms proposed on 2026-06-12 have -been implemented or moved into focused designs. - -The 2026-06-12 document remains historical input. The requirement-by- -requirement disposition in section 4 is authoritative for the subsystem -milestones it enumerates. In particular, "superseded" means ownership moved -here or to a focused owner; it does not mean "implemented". - -Section 4 covers the subsystem milestone tables only. The cross-cutting -contracts in -[`2026_06_12_proposed_scaling_roadmap.md`](2026_06_12_proposed_scaling_roadmap.md) -§7 — the shared `SetPhysicalCeiling` / `Observe` invariant, the per-feature -capability gate, and the required observability keys and metric cardinality -budget — are **not** superseded and remain normative for focused designs written -from this index, whether or not the row repeats them. - -They are not applied retroactively to already-shipped milestones. One shipped -mechanism deviates deliberately: SST ingest snapshot transfer (§4.3, implemented) -gates rollout with a cluster-wide environment variable set after operators deploy -a receiver-capable binary everywhere, rather than the §7.2 `cap__v` -bit and preflight check. There is no SST capability bit in the tree. The env gate -is fail-safe on an invalid value and reversible without touching persisted data, -so this is recorded as a deviation rather than outstanding work; requiring the -bit later is a separate decision. - -## 3. Current canonical owners - -Status is a snapshot from the audit date. Pull request state must be checked -again before merge or deployment. - -| Scaling requirement | Canonical owner | Implementation evidence | Audited status | -|---|---|---|---| -| Multi-node, multi-group bootstrap | `2026_06_14_implemented_multinode_multigroup_bootstrap.md` | PR #1011 merged | Implemented on `main` | -| Learner membership primitive | `2026_04_26_implemented_raft_learner.md` | PR #676 (`51907e6b`) merged the `AddLearner` / `PromoteLearner` engine, admin, protobuf, CLI, and persistence surfaces; PR #1002 merged the later status promotion and cleanup | Implemented on `main`; follower reads are separate | -| Leader balance | `2026_06_11_implemented_leader_balance_scheduler.md` | PR #1012 merged | Implemented on `main`; data placement is separate | -| Hotspot split M1 catalog and same-group split | `2026_02_18_implemented_hotspot_split_milestone1_pr.md` and `2026_02_18_partial_hotspot_shard_split.md` | The durable catalog, engine snapshots, split admin API, watcher, and M1 integration coverage landed as an earlier series; PR #999 (`a28afa25`, "Clean up hotspot split catalog path") is cleanup and status promotion only, not that implementation. The focused owner is the evidence of record — an auditor should start there, not from #999 | Implemented M1; parent design remains partial | -| Hotspot split M2 migration | `2026_06_11_partial_hotspot_split_milestone2_migration.md` | PR #1096 merged the durable `SplitJob` catalog and codec substrate; PRs #1084, #1085, #1088, and #1090 open | The catalog substrate is on `main`; `StartSplitMigration`, the migrator FSM, fencing, cutover, promotion, and cleanup remain in flight, so this is not a complete migration plane | -| Hotspot split M3 automation | `2026_06_11_implemented_hotspot_split_milestone3_automation.md` | PRs #1097 and #1152 merged the detector core and the committed-window reader (`afec0597`, `distribution/autosplit/sampler_reader.go` plus the observe-only detector bridge); PR #1104 (`07c48af2`, "Complete standalone hotspot split automation") merged M3-PR2b Top-K evidence alignment, the leadership watermark, and M3-PR3 scheduler wiring, and renamed the focused owner to `*_implemented_*` | Standalone M3 is implemented on `main`. The focused owner's §8.1 leaves one slice open: M3-PR4 least-loaded `target_group_id` selection, which waits on the M2 migration plane | -| Per-group HLC renewal and default-group allocator bridge | `2026_04_16_partial_centralized_tso.md` | PR #998 merged | Implemented bridge; dedicated TSO remains in flight | -| Dedicated TSO group and durable routing | `2026_04_16_partial_centralized_tso.md` | PRs #1064, #1103, #1108, and #1150 merged the reservation, configuration scaffolding, and ceiling state machine (`0e85c822` added `kv/tso_fsm.go` with snapshot/restore of the physical ceiling); PR #1095 open | Group reservation/scaffolding and the ceiling FSM are on `main`; production group-0 issuance, TSO-leader redirect/routing, and operational exposure remain in flight | -| Shared Pebble block cache | PR #1082 | PR #1082 merged | Implemented on `main`; cache sharing only, not all resource-pool work | -| Raft gRPC streaming transport | `2026_04_18_implemented_raft_grpc_streaming_transport.md` | PR #1006 merged; PR #1048 merged the kill switch | Implemented on `main`; multi-group soak evidence landed with the design's §8 (`cmd/elastickv-raft-stream-soak`, `scripts/run-jepsen-raft-streaming-multigroup-soak.sh`, `docs/evidence/raft_streaming_multigroup_soak.json`) | -| S3 Raft blob offload | `2026_04_25_partial_s3_raft_blob_offload.md` | PRs #1057 and #1063 merged the rollout scaffolding and blob fetch RPC; `77ea547d` merged the local offload decision, PUT/GET path, and peer replication/fetch; #1126 (`5eaaa05d`) merged follower repair and asynchronous backfill | Transport path is on `main`; the focused design names reference counting, GC readiness, and legacy migration as the remaining blockers | -| Live logical backup | `2026_04_29_proposed_logical_backup.md` | PRs #1065 and #1059 merged scan primitives; PR #1056 and PR #1128 are open for the live pin, admin, and producer stack | In flight; distinct from physical SST snapshot offload | - -## 4. 2026-06-12 requirement audit - -### 4.1 Routing scale-out - -| 2026-06-12 milestone | Disposition | Remaining ownership | -|---|---|---| -| M1 versioned catalog delta and streaming watch | Implemented on `main` | `2026_07_18_implemented_route_catalog_delta_watch.md` (PR #1117, `6c7a66e5`) owns the durable delta log, retention fallback, atomic mirror publication, capability negotiation, and stream reconnect semantics | -| M2 indexed route engine and copy-on-write history | Unimplemented and unowned | Write `*_proposed_route_catalog_index.md`; own the primary index, group secondary index, immutable snapshot sharing, memory bound, and migration from the slice representation | -| M3 batched catalog mutation | Unimplemented and unowned | Write `*_proposed_route_catalog_batching.md`; own batch conflict semantics, one-version publication, idempotency, limits, and interaction with the delta watch | - -Hotspot split M2 and M3 do not own these catalog-scale mechanisms. They consume -the existing catalog and must not absorb the three designs above. - -### 4.2 Multi-region - -| 2026-06-12 milestone | Disposition | Remaining ownership | -|---|---|---| -| M1 WAN Raft tuning and region-aware membership | Unimplemented and unowned | Write `*_proposed_wan_raft_membership.md`; own topology, timing bounds, region identity, quorum failure modes, and rollout. No prerequisite inside this subsystem: it makes Raft survive a cross-WAN partition and does not by itself enable cross-region writes | -| M2 region-local HLC | Unimplemented and unowned | Write `*_proposed_regional_timestamp_oracle.md`; reconcile regional issuance with the dedicated TSO invariant before choosing local ceilings or a global oracle. Depends on the hotspot split M2 migration contract (`2026_06_11_partial_hotspot_split_milestone2_migration.md`) being implemented, because the monotone-merge primitive has to exist first; §3 records that plane as still in flight | -| M3 regional catalog mirror | Unimplemented and unowned | Write `*_proposed_regional_catalog_mirror.md`; define freshness and failover contracts. Depends on the region-aware membership from M1, the per-region ceiling from M2, and the catalog delta/watch design (§4.1 M1, implemented) | -| M4 cross-region disaster recovery | Unimplemented and unowned | Write `*_proposed_cross_region_failover.md`; own authority, fencing, data completeness, operator approval, failback, and split-brain prevention. Depends on M2 and M3 | - -Multi-node bootstrap is a prerequisite, not an implementation of multi-region -placement or failover. - -### 4.3 Storage tier - -| 2026-06-12 milestone | Disposition | Remaining ownership | -|---|---|---| -| M1 SST ingest snapshot transfer | Implemented on `main` | `2026_07_19_implemented_pebble_sst_ingest_snapshot_transfer.md` (PR #1130, `910a97e0`) owns checkpoint consistency, file manifest, integrity, transport, ingest, cleanup, and fallback | -| M2 shared block cache | Implemented on `main` | PR #1082 owned only process-wide cache sizing, lifetime, metrics, and tests | -| M2 per-shard Pebble tuning and write admission | Partially addressed operationally, but the proposed per-shard contract is unowned | Write `*_proposed_pebble_resource_governor.md`; own tuning scope, node-wide fairness, stall thresholds, admission errors, and adapter retry semantics | -| M3 sharded retention scheduling | Existing compaction is implemented; jitter, node budget, and hot-key dynamic retention are unimplemented and unowned | Write `*_proposed_sharded_mvcc_retention.md`; preserve the hard retention contract and active timestamp pins | -| M4 physical disaster-recovery snapshot offload | Partially implemented on `main` | `2026_07_19_partial_physical_snapshot_object_offload.md` (PR #1131, `764db2d8`) owns the export/restore substrate and records M0/M1 implemented; that owner still owns the pending leader-only scheduler, retention/GC readiness, and operational validation. Keep it distinct from logical backup and S3 user-payload blob offload | - -### 4.4 Coordinator and API gateway - -| 2026-06-12 milestone | Disposition | Remaining ownership | -|---|---|---| -| M1 timestamp issuance decoupling | Partially implemented and in flight | `2026_04_16_partial_centralized_tso.md`, PR #998, and the #1064/#1095/#1103/#1108 stack own this work | -| M2 follower and learner reads | Unimplemented and unowned | Write `*_proposed_follower_reads.md`; own leader-vouched read timestamps, apply watermarks, staleness/session contract, invalidation, routing, and Jepsen evidence. Multi-node groups and the learner primitive are already available; this row is not gated on dedicated TSO unless the focused design adds cross-shard or session-global timestamp semantics | -| M3 cross-shard 2PC completion | Partially implemented; no focused end-to-end owner | Write `*_proposed_cross_shard_transaction_completion.md`; start from the existing `ShardedCoordinator` 2PC and Composed-1 guard, then own unsupported router paths, read-only validation, recovery, and adapter coverage. Gate completion on the dedicated timestamp invariant and globally comparable issuance path | -| M4 resolver work delegation | Unimplemented and unowned | Write `*_proposed_lock_resolver_delegation.md`; own snapshot assignment, leader-vouched decisions, duplicate work, failover, admission, and Raft apply boundaries. It also needs a focused per-group HLC tick capability and preflight contract, such as `cap_per_group_hlc_v1`, before resolver work can move away from process-local ticking | -| M5 leader-proxy circuit breaker | Implemented on `main` | `2026_07_19_implemented_leader_proxy_circuit_breaker.md` (PR #1132, `56e36e94`) owns the data-plane breaker in `kv/leader_proxy_breaker.go` plus retry budget, leader-identity reset, half-open behavior, and adapter error mapping | - -- **Per-group HLC vs centralized TSO.** The centralized TSO design - (`docs/design/2026_04_16_implemented_centralized_tso.md`) has shipped M1-M8: - all-led-group compatibility renewal, the dedicated group-0 FSM, leader-routed - durable windows, one-way cutover and Phase D, runtime mode reload, and - operational gates. The shared ordering source is available for cross-node - cross-group transactions. Deployments that remain in `legacy` still have - only per-node monotonicity; before enabling multiple coordinator nodes for - cross-group issuance, they must complete the documented `shadow -> cutover` - sequence. `LeaderProxy.Commit` / `Internal.Forward` preserve non-zero - timestamps, and one `startTS` remains shared by every transaction participant. - Phase D additionally validates a caller-supplied cross-shard `StartTS` at the - group-0 leader before commit allocation. -The admin package's existing `ErrLeaderUnavailable` mapping was never evidence -for the data-plane breaker; that gap was closed separately by PR #1132, which -added `kv/leader_proxy_breaker.go` and its adapter error mapping. - -## 5. Additional gaps introduced by the 2026-06-23 roadmap - -| Gap | Disposition | Required owner | -|---|---|---| -| Replica placement | Unimplemented and unowned | `*_proposed_replica_placement.md`; own creation and reshaping of Raft replica sets — which members a group has and where. Listed as its own row because both the region/range balance scheduler and auto group lifecycle name it as a prerequisite and nothing else owns it. Hotspot migration cannot substitute: it moves ownership to an already suitably placed group | -| Region/range balance scheduler | Unimplemented and unowned | `*_proposed_region_balance_scheduler.md`; depend on the replica placement row above, multi-node bootstrap, and the hotspot split M2 migration plane | -| Range merge | Unimplemented and unowned | `*_proposed_range_merge.md`; split same-group and cross-group merge into reviewable milestones. Same-group merge can own local transaction drain/fencing first; cross-group merge additionally waits for the hotspot split M2 migration plane and the Composed-1 cross-group commit guard/drain protocol | -| Streaming transport multi-group soak | Closed | `2026_04_18_implemented_raft_grpc_streaming_transport.md` §8 records the repeatable soak (`cmd/elastickv-raft-stream-soak` plus a fail-closed verifier over `docs/evidence/raft_streaming_multigroup_soak.json`); no protocol semantics changed | -| Auto group lifecycle | Orchestration unimplemented and unowned; the join/replacement substrate is implemented | `2026_07_18_implemented_raft_fresh_learner_join.md` (PR #1118, `6d8ee633`) and `2026_07_18_implemented_fenced_raft_member_replacement.md` (PR #1122, `5d4c5cad`) already own single-group fresh learner joining and resumable fenced same-ID voter replacement, and both explicitly leave automatic decisions and multi-group joining open. Write `*_proposed_auto_group_lifecycle.md` for the automatic creation/orchestration and multi-group extensions only — do not restate the shipped join/replacement mechanics — and depend on replica placement, the hotspot split M2 migration plane, and the range-merge milestones it chooses to automate | - -## 6. Dependency order - -Every entry below states its own prerequisites, taken from the rows in sections -3 to 5. A numbered step is **not** a barrier for the steps after it: two items -are ordered only where one names the other. Anything whose prerequisites are -already satisfied may start immediately and in parallel with the rest. - -The list is written this way because sequencing items by position repeatedly -invented dependencies the rows do not state. - -**Ready now — no unmet prerequisites, may proceed in parallel:** - -1. Hotspot split M2 migration plane (§3): the open #1084/#1085/#1088/#1090 - stack owns `StartSplitMigration`, the migrator FSM, fencing, cutover, - promotion, and cleanup. It is a root prerequisite for later cross-group range - movement, not implemented by this roadmap. -2. Dedicated TSO group-0 issuance, routing, and exposure (§3, §4.4 M1): proceed - as its own open stack. It gates cross-shard transaction completion but not - follower reads. -3. Live logical backup pin, admin, and producer stack (§3): scan primitives are - on `main`; PR #1056 and PR #1128 own the remaining live stack. -4. S3 Raft blob offload follow-ups (§3): reference counting, GC readiness, and - legacy migration remain in the focused S3 owner. -5. Indexed route engine (§4.1 M2) and batched catalog mutation (§4.1 M3), which - are **parallel, not sequential**: the predecessor's §3.3 records M2 as - independent and M3 as depending only on M1's batched apply observation path, - and PR #1117 merged M1. Catalog delta/watch is therefore no longer in this - list either. -6. The remaining physical snapshot offload milestones (§4.3 M4: the leader-only - scheduler, retention/GC readiness, and operational validation) in - [`2026_07_19_partial_physical_snapshot_object_offload.md`](2026_07_19_partial_physical_snapshot_object_offload.md). - Their only prerequisite was SST ingest snapshot transfer, which PR #1130 - merged and §4.3 records as implemented. -7. Per-shard Pebble tuning and write admission (§4.3 M2) and sharded retention - scheduling (§4.3 M3). Independent of - step 6: the predecessor's §5.3 records M4 as depending on M1 alone, and the - focused offload owner names no dependency on either design. -8. Same-group range merge (§5), after the focused range-merge design defines - local drain and fencing. Cross-group merge is separate and gated below. -9. Replica placement (§5). Named as a prerequisite by region balance and by - auto group lifecycle, and owned by nothing else, so it gates those two. -10. Follower and learner reads (§4.4 M2). Multi-node groups and the learner - primitive are on `main`; this is not waiting for dedicated TSO unless the - focused design adds cross-shard or session-global timestamp semantics. -11. WAN Raft tuning and region-aware membership (§4.2 M1). Multi-node bootstrap - is on `main`, and this row has no unmet prerequisite inside the regional - subsystem. - -**Gated — each names what it waits for:** - -12. Hotspot split M3 cross-group targeting (§3), after the hotspot split M2 - migration plane in step 1. Standalone M3 is implemented — the detector core, - committed-window reader, Top-K evidence alignment, leadership watermark, and - scheduler wiring are all on `main`, so none of those is work to schedule. - What remains is M3-PR4 least-loaded `target_group_id` selection, and the - focused owner's §8.1 records it as post-M2 because it is a scheduler action - that moves data. -13. Cross-group range merge (§5), after the hotspot split M2 migration plane - and the Composed-1 cross-group commit guard/drain protocol required by the - range-merge owner. -14. Region/range balance scheduler (§5), after replica placement, multi-node - bootstrap, and the hotspot split M2 migration plane. -15. Auto group lifecycle (§5), after replica placement, the hotspot split M2 - migration plane, and the range-merge milestones it chooses to automate. It - is **not** gated on the regional stack below: its row names only those - prerequisites. -16. Cross-shard transaction completion (§4.4 M3), after the dedicated timestamp - invariant/group-0 issuance and the read-only validation, recovery, and - router owner. -17. Lock-resolver delegation (§4.4 M4), after a focused design/implementation - provides per-group HLC tick capability/preflight (`cap_per_group_hlc_v1` or - equivalent) and safe status-resolver routing. The current process-local - ticker is not enough. -18. Region-local HLC (§4.2 M2), after the hotspot split M2 migration plane and - the dedicated/global timestamp decision. -19. Regional catalog mirror (§4.2 M3), after regional M1, regional M2, and - catalog delta/watch. -20. Cross-region disaster recovery (§4.2 M4), after regional M2 and M3. - -## 7. Completion rule - -This roadmap can be promoted from `proposed` only when every row is either: - -- implemented on `main` with its focused owner promoted to `implemented`, or -- implemented on `main` where the row's canonical owner is a merged pull - request rather than a design document, evidenced by that merge commit, or -- deliberately rejected with a recorded rationale in its focused owner. - -The second clause exists because a pull request has no lifecycle status to -promote. The shared Pebble block cache row names PR #1082 as its only canonical -owner and treats that work as complete, so under the first clause alone this -roadmap could never become eligible for promotion no matter what else shipped. + +This is a roadmap / implementation-plan document (the kind `docs/design/README.md` +explicitly admits under "Concrete implementation plans"). It surveys the +current scaling envelope of elastickv, maps every scaling dimension to the +design that already addresses it, and identifies the gaps where no design +exists yet. Every claim about current behaviour below is anchored to a file +and line on `main` at the propose date; every referenced design doc is one +that actually exists in `docs/design/` (or, for in-flight work, an open PR +branch named explicitly). + +Scope: this doc proposes **no implementation**. It proposes a sequence and a +short list of new design docs to write. It does not duplicate the detail of +the docs it references; read those for mechanism. + +--- + +## 1. Current scaling envelope + +What bounds a single elastickv deployment today: + +1. **Single-process multi-group only.** Multiple Raft groups run in one + process (`--raftGroups id=addr,…`, `shard_config.go:61-99`); each group + gets its own engine and its own gRPC listener at `rt.spec.address` + (`main.go:1606-1620`, listener at `:1613`). But a *multi-group topology whose + per-group voter sets span nodes is not deployable from the startup flags*: + `resolveBootstrapServers` rejects `--raftBootstrapMembers` whenever + `len(groups) != 1` (`main.go:746`, `ErrBootstrapMembersRequireSingleGroup` + at `:736`), and `buildRuntimeForGroup` passes the same `bootstrapServers` + (which is `nil` in any multi-group config) to every group with + `LocalAddress: group.address` (`multiraft_runtime.go:246-254`), so each + group in a multi-group process bootstraps as a **single-member** cluster. + The multi-group story today is "one process hosts every group, each group + has one voter" (the demo / Jepsen M5 model). True multi-node multi-group + is not wired (see §2(b), §2(e)). + +2. **Per-group memory.** Each Raft group owns a private Pebble store, and + `NewPebbleStore` → `defaultPebbleOptionsWithCache` allocates a *fresh* + block cache per store (`store/lsm_store.go:273-279`, `pebble.NewCache(pebbleCacheBytes)` + at `:274`), default 256 MiB (`defaultPebbleCacheBytes` at `:66`). N groups + on a node means N × 256 MiB of block cache alone, plus N memtables, N WALs, + N compaction budgets. The TODO at `store/lsm_store.go:117-120` records the + intended fix (a process-wide shared cache plumbed through `NewPebbleStore`) + but it is a comment, not a design. + +3. **Leader concentration.** Leadership of each group is elected + independently by etcd/raft; nothing spreads leaderships across nodes. One + node can end up leading every group while peers idle, carrying all + leader-only work (write proposals, HLC ceiling renewal, lease reads, OCC + timestamp issuance, route-catalog proposes). This is the explicit + motivation for PR #953 (leader balance scheduler). + +4. **Range granularity is fixed after a manual split.** M1 of the hotspot + shard split shipped a durable, versioned route catalog and a same-group + `SplitRange` RPC (`adapter/distribution_server.go`, `distribution/catalog.go`, + `distribution/engine.go`, `distribution/watcher.go`). There is no data + movement (cross-group migration is M2, in flight), no auto-detection (M3, + in flight), and **no merge** (no design exists). The old + `distribution.Engine.RecordAccess` midpoint path was removed by M3-PR1a + because it was never called from any request path; future auto-detection + uses keyviz. + +5. **Cross-group timestamps via per-group HLC, but no global TSO yet.** + `ShardedCoordinator.RunHLCLeaseRenewal` proposes the physical-ceiling + renewal to every shard group this node currently leads + (`kv/sharded_coordinator.go:1960-2030`), implementing the centralized-TSO + doc's M1 near-term fix. This closes the default-group-only renewal gap for + a node that leads a non-default group. It does **not** create a single + global timestamp oracle: once groups span nodes, cross-group operations + whose timestamps can be allocated by different coordinators still need the + dedicated TSO / single-oracle bridge described in §2(d). + +6. **Large values traverse the Raft log.** Every byte of an S3 object travels + through the Raft log as `s3keys.BlobKey` entries (the S3 raft-blob-offload + doc §1). WAL and snapshot size scale with object bytes, and follower + catch-up re-applies every byte on the single-threaded apply loop. + +7. **FSM apply is serial per group.** `kvFSM.Apply` runs on the group's + single apply goroutine — "Raft applies are serial" (`kv/fsm.go:123-124`, + `Apply` at `:297`). Apply throughput per group is bounded by that one + goroutine; the only way to add apply parallelism today is to add groups. + +--- + +## 2. Dimension-by-dimension analysis + +Each dimension below lists what bounds it and the design (existing, in-flight, +or missing) that addresses it. + +### (a) Data volume per node + +The amount of data one node can hold is bounded by Pebble capacity and by the +memory each group's private cache/memtable pins. + +- **Range split — distribute a range across groups.** Same-group split + shipped in M1 (`distribution/`). Cross-group migration (the part that + actually relocates data and reduces per-node volume) is implemented by the + M2 stack recorded in + [2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md): + a resumable `SplitJob` with + `PLANNED → BACKFILL → FENCE → DELTA_COPY → CUTOVER → CLEANUP → DONE` phases + driven by a migrator on the default-group leader. M2 is the required + ownership-migration mechanism, but it reduces per-node bytes only when the + target group has different replica placement; if every group still shares the + same local voter set, migration changes ownership but cannot move data off an + over-full node. Actual per-node volume relief therefore also depends on Gap 1 + plus the replica-placement / region-balance work in Gap 4. Auto-detection / + scheduling is **PR #951** + (`docs/design/2026_06_11_partial_hotspot_split_milestone3_automation.md`, + branch `design/hotspot-split-m3-automation`), which drives detection off the + already-wired keyviz sampler rather than the dead `RecordAccess` counters. +- **Range MERGE — missing, no design exists.** The inverse of split. Both the + parent hotspot doc (§2.2.2) and M2 (§2.2) list merge as an explicit + non-goal, and no `*_merge_*.md` exists in `docs/design/` (verified). Why it + is needed: without merge, a workload whose hot range cools, or one that + over-splits during a transient spike, accumulates permanent route fragments. + Each fragment costs a route entry, sampler cardinality, and — once a child + lives in its own group — a full group's per-node memory overhead (§1.2). A + long-running cluster's route count is monotonically non-decreasing, which is + not sustainable. Hard parts to sketch: choosing merge candidates (adjacent + routes, both cold, same or adjacent groups); fencing both sides atomically; + reconciling the two MVCC histories and the per-key internal keyspace + (`!txn|…`, list/hash/set/zset meta) without losing a committed write or an + unresolved prepare; cutover that bumps the catalog version exactly like + split; and the Composed-1 cross-group commit guard interaction (the same + apply-time hazard M2 §7.3 closes for split). Merge is strictly harder than + split because it must *unify* two independent commit-timestamp streams, not + bisect one. +- **Blob offload — proposed.** + `docs/design/2026_04_25_proposed_s3_raft_blob_offload.md` keeps large object + payloads out of the Raft log (chunkref through Raft, chunkblob via a + peer-to-peer side channel with semi-synchronous quorum replication). This + bounds WAL/snapshot growth to O(manifest), which is the data-volume lever + for the S3 surface specifically. Still proposed; the legacy `BlobKey`-on-Raft + path is what runs today. +- **Shared Pebble cache / resource pools — TODO only, promoted to a design + item here.** The `store/lsm_store.go:117-120` TODO is the single biggest + per-node memory tax as group count grows: N independent 256 MiB caches with + no shared eviction pool. This deserves a real design (see §3, Gap 2), not a + comment. It blocks high group counts on a single node, which in turn blocks + the "many small ranges" model that split (a) produces. + +### (b) Write throughput + +- **Multi-group** is the primary write-scaling lever: independent Raft + pipelines and independent serial apply loops (§1.7). Already in-tree + structurally. +- **Leader balance — PR #953** + (`docs/design/2026_06_11_implemented_leader_balance_scheduler.md`, branch + `design/leader-balance-scheduler`) spreads group leaderships across nodes so + write-proposal load is not pinned to one node. Count-based v1 (TiKV + balance-leader equivalent), embedded in the default-group leader, default + OFF behind `--leaderBalance`. +- **Multi-node multi-group bootstrap — done.** The implemented + `docs/design/2026_06_14_implemented_multinode_multigroup_bootstrap.md` + adds `--raftGroupPeers`, per-group bootstrap peer wiring, persisted + bootstrap-seed validation, and a 3-node × 2-group integration smoke with + leader transfer. This unblocks the topology prerequisite for leader balance, + write-throughput scaling beyond one node, and the region-balance gap in §3. + +### (c) Read throughput + +- **Lease reads — done.** Leader-served linearizable reads without a Raft + round-trip per read (`kv/lease_state.go`, + `docs/design/2026_04_20_implemented_lease_read.md`; the lease-read observer + is wired in at `main.go:426`). This scales leader read throughput but keeps + all reads on the leader. +- **Learner reads → follower/learner reads.** The learner *primitive* is + implemented in + `2026_04_26_implemented_raft_learner.md` (promoted to implemented status): + `AddLearner` / `PromoteLearner` are + on the engine `Admin` interface (`internal/raftengine/engine.go:233`, `:245`) + and implemented in the etcd backend (`internal/raftengine/etcd/engine.go:1268`, + `:1289`, `ConfChangeAddLearnerNode` handling at `:2550`), the raftadmin CLI + has `add_learner` / `promote_learner` (`cmd/raftadmin/main.go:204-206`), and + `--raftJoinAsLearner` exists (`main.go:104`). The learner doc itself scopes + follower-served reads as an explicit non-goal (§2 "Non-goals", §8 OQ-5): + a `LinearizableRead` against a learner is **not** forwarded by the engine — + `handleRead` returns `ErrNotLeader` for any non-leader state + (`internal/raftengine/etcd/engine.go:1583`), and the *caller* must forward to + the leader (`docs/raft_learner_operations.md`, "Serve linearizable reads from + the learner"). So the read-replica attach primitive exists, but the design + that consumes it for off-leader reads **does not** — **follower-read design + is missing**, and it must supply the forwarding/proxy or replica-read API the + current primitive deliberately omits. Requirements to + sketch (Gap 3): a leader-issued read-timestamp pipeline so a follower/learner + serves a snapshot read at a ts the leader has vouched for (CLAUDE.md HLC + rule: never use the local wall clock or a follower-issued ts for MVCC + visibility / OCC / lease decisions); a staleness bound and a way for the + reader to know it has applied past that ts; lease invalidation interaction; + and adapter routing so reads can be steered to a replica. The low-level + packages must stay generic: `internal/raftengine` / `store` should expose an + optional, nil-safe delegate/interceptor interface for "may serve at this + leader-vouched read timestamp?" and "has this replica applied past it?", + with nil preserving today's leader-only behavior. `kv` / `distribution` + implement the policy above that interface rather than being imported by the + engine/store packages. The centralized TSO doc §9 Q4 and the learner doc + §8 OQ-5 both flag this as the open follow-on. + +### (d) Cross-group transactions at scale + +- **Per-group HLC vs centralized TSO.** Today the ceiling is renewed only on + the default group (§1.5); the TSO doc + (`docs/design/2026_04_16_partial_centralized_tso.md`) has shipped the + near-term fix (renew on all led groups, in parallel — its M1) and still + leaves the dedicated TSO Raft group with a batch allocator open. + **Assessment of whether TSO is still needed once groups multiply:** the + near-term per-group fix (TSO doc §6) was *necessary regardless* — it is the + minimum correctness fix the moment a node leads a non-default group it is + member of, and it has landed before multi-node multi-group bootstrap (b) + makes that topology reachable. The *full* dedicated-TSO component (TSO doc + M6–M7) is a larger component, but the need for a shared ordering source is + not a throughput/amortization question once cross-node cross-group + transactions are possible: with the per-group fix in place, every node's + timestamps are self-monotonic, but **global** monotonicity across coordinator + nodes is still not guaranteed without a shared oracle (TSO doc §6 + "Guarantee"). Cross-group transactions (`kv/transaction.go`, + `kv/txn_codec.go`) whose timestamps can be allocated by different ingress / + coordinator nodes need a single ordering source for OCC commit-ts + comparability, regardless of where the participating groups' current leaders + live. `LeaderProxy.Commit` / `Internal.Forward` preserve non-zero timestamps, + so leader co-location does not prove single-clock allocation. (The shared-snapshot + invariant — every operation in one txn reading at the *same* `startTS` — is + already upheld: `nextStartTS` allocates one `startTS` for the whole txn and + propagates it via `reqs.StartTS` to every participating group; the gap is the + cross-*coordinator* comparability of the per-txn `commitTS`, not the per-txn + `startTS`. See OQ-1.) So: per-group renewal fix is in-scope-soon and + load-bearing for one node leading multiple groups; before enabling any + cross-group transaction mode in which more than one coordinator node can issue + `startTS` / `commitTS`, the roadmap must either pull the dedicated TSO group + forward or land a narrower single-oracle bridge that pins cross-group + timestamp allocation to one designated leader. Until such txns are enabled, + the per-group fix plus the shared-`*HLC` property remains adequate. + +### (e) Cluster size / membership + +- **Learner / conf-change — primitive present.** `AddVoter` / `AddLearner` / + `PromoteLearner` / `RemoveServer` exist on the engine and the raftadmin CLI + (see (c) / §1). Single-step V1 conf changes only; joint consensus is + rejected (`validateConfState`). +- **Multi-node bootstrap — GAP** (same gap as (b)): the membership primitives + exist for *live* expansion (`AddVoter` after bootstrap), but there is no + startup wiring to declare a multi-node voter set per group, and no + integration harness for multi-voter groups across processes (PR #953 §1.1a; + Jepsen M5 runner records "True distributed multi-group is M6+ work", + `scripts/run-jepsen-m5-local.sh`). +- **Auto group creation — explicit non-goal today, long-term.** Both the + hotspot parent doc (§2.2.1) and M2/M3 (§2.2) state automatic Raft group + creation / membership orchestration is out of scope. The scheduler only + targets groups already in `--raftGroups`. Noting it as a long-term item: + elastic scale-out (add a node → automatically create/rebalance groups onto + it) needs an auto group lifecycle, which depends on multi-node bootstrap (b), + region balance (§3 Gap 4), and merge (a) all being in place first. + +### (f) Operational scaling + +- **keyviz** — the per-route load sampler is wired and allocation-free on the + write hot path (`observeMutation`, `kv/sharded_coordinator.go:1841-1846`). + Shipped extensions include cluster fan-out + (`2026_04_27_implemented_keyviz_cluster_fanout.md`), subrange sampling + (`2026_05_25_implemented_keyviz_subrange_sampling.md`), hot-key top-K + (`2026_05_28_implemented_keyviz_hot_key_topk.md`), and per-cell conflict. It + is the detection signal M3 reuses. Current adapter-direct + Redis/DynamoDB/S3 reads that hit `MVCCStore.GetAt` bypass this coordinator + sampler, so read-heavy hotspots remain invisible until read-path sampling or + an equivalent adapter read observation path is added. +- **admin** — admin dashboard / data browser / purge-queue are implemented + (`2026_04_24_implemented_admin_dashboard.md`, + `2026_05_22_implemented_admin_data_browser.md`, + `2026_05_16_implemented_admin_purge_queue.md`). +- **metrics cardinality** — recurring constraint across proposals (S3 + admission `protocol`/`stage` labels, SQS per-queue counters bounded by a + top-N sketch, keyviz `MaxTrackedRoutes` coarsening). Any new per-route / + per-queue / per-peer metric must carry a cardinality bound; this is a + cross-cutting operational-scaling rule, not a single design. +- **workload isolation** — `2026_04_24_implemented_workload_isolation.md` (heavy + command worker pool, optional Raft-thread pinning, per-client admission, + XREAD O(N)→O(new)), S3 PUT admission control + (`2026_04_25_implemented_s3_admission_control.md`), SQS per-queue throttling + (`2026_04_26_implemented_sqs_per_queue_throttling.md`). These bound *one + workload's* impact so it cannot starve the shared runtime / Raft control + plane; they scale a deployment by making it predictable under adversarial or + unbalanced load rather than by raising raw capacity. + +--- + +## 3. Gap list → recommended new designs (ranked by leverage) + +Each gap is a design doc that should be written (`*_proposed_*.md`). Ranked by +how much further scaling it unlocks. "Depends-on" lists hard prerequisites. + +### Gap 1 — Multi-node multi-group bootstrap (closed) +**Status.** Closed by +`docs/design/2026_06_14_implemented_multinode_multigroup_bootstrap.md`. +Startup wiring can now express a *multi-group* topology where each group has a +multi-node voter set via `--raftGroupPeers`, and the in-process 3-node × 2-group +smoke verifies per-group voters, leader transfer, and restart rejoin. The +remaining work from the original roadmap item is the broader Jepsen runner for +true multi-node multi-group workloads, tracked as a follow-on rather than a +bootstrap design blocker. + +### Gap 2 — Shared Pebble cache / resource pools +**Problem.** Each group's store allocates a private 256 MiB block cache +(`store/lsm_store.go:273-279`); N groups = N × 256 MiB plus N memtables/WALs, +with no shared eviction. This caps how many groups (hence how many ranges) one +node can hold, throttling the "many small ranges" model that split produces. +**Rough milestones:** (M1) process-wide shared `pebble.Cache` plumbed through +`NewPebbleStore` (the existing TODO), with per-node sizing. (M2) shared +memtable / compaction concurrency budget across stores. (M3) per-group +fairness so one hot group cannot evict everyone else's working set. +**Depends-on:** none functionally; pairs naturally with Gap 1 (high group +counts only matter once multi-node groups exist). + +### Gap 3 — Follower / learner reads +**Problem.** Reads are leader-only; the learner primitive exists but its +consumer (off-leader reads) has no design. Read throughput cannot scale past +one node's leader capacity. **Rough milestones:** (M1) leader-issued read-ts +pipeline + a replica apply-watermark so a follower/learner can serve a +snapshot read at a leader-vouched timestamp (CLAUDE.md HLC rule), exposed +through optional nil-safe low-level delegate interfaces so raftengine/store +do not import `kv` / `distribution`. (M2) lease invalidation interaction + +staleness bound + adapter read routing to replicas. +(M3) Jepsen stale-read bound workload. **Depends-on:** Gap 1 (multi-node +groups to have a remote replica); the learner primitive (already in-tree). + +### Gap 4 — Region (range) balance scheduler +**Problem.** Leader balance (PR #953) spreads *leaderships*; nothing spreads +*data* (which node holds which range's replicas). After enough splits, ranges +pile up unevenly across nodes. This is TiKV's separate balance-region +scheduler; PR #953 §2.2(3) explicitly excludes replica/data movement. The M2 +migration plane can move *range ownership* between Raft groups, but it does not +by itself move replicas off an over-full node when the source and target groups +share the same voter set. A real region-balance design therefore needs both a +target-group placement constraint and, when no suitable group already exists, a +Raft membership-change / replica-placement step before or during migration. +**Rough milestones:** (M1) data-balance policy that chooses a target group whose +replica set actually reduces node skew, then reuses the M2 migration plane for +the range-ownership move only when that placement predicate holds. (M2) Raft +membership-change primitive for creating or reshaping groups when no existing +group has the needed placement. (M3) compose with leader balance so the two +schedulers don't fight. **Depends-on:** M2 migration plane (PR #945) for +range-ownership movement, a replica-placement / membership-change design for +per-node data movement, and Gap 1 (multi-node bootstrap) for somewhere to place +replicas. Edge: region balance ⟂ depends on M2 + multi-node bootstrap + replica +placement. + +### Gap 5 — Range merge +**Problem.** No way to recombine ranges; route count is monotonically +non-decreasing (§2(a)). Over-split or cooled workloads leak route fragments +and (once children are in their own groups) per-node memory. **Rough +milestones:** (M1) same-group merge of two adjacent cold routes (mirror M1 +split: catalog CAS + version bump). (M2) cross-group merge (relocate one +child's data into the other's group, reusing the M2 migration plane in +reverse). (M3) auto-merge in the M3 detector (cold-route hysteresis). +**Depends-on:** M2 migration plane for cross-group merge; the Composed-1 guard +for cutover coherence. ### Gap 6 — Connection / transport scaling (streaming transport soak) **Problem.** Raft inter-node messages previously used unary gRPC per message @@ -295,14 +418,19 @@ The ordering is driven by unblock-edges, not by perceived value in isolation. 9. **Range merge** (Gap 5). After step 4 for cross-group merge. 10. **Streaming transport** (Gap 6). Any time after step 2 makes inter-node Raft traffic significant; pairs with the S3 blob-fetch RPC. -11. **Dedicated TSO group** (TSO doc M6–M8 / OQ-1 resolved) — implemented as - the shared ordering source for cross-node, cross-group transactions. The - remaining sequencing constraint is operational: deployments still running - in `legacy` retain only the per-node HLC guarantee from step 1, while - multi-node cross-group issuance requires completing the documented - `shadow -> cutover -> phase-d` gate so both `startTS` and `commitTS` come - from group 0 and caller-supplied timestamps are validated against its - durable allocation floor. +11. **Dedicated TSO group or single-oracle bridge** (TSO doc M6–M7 / OQ-1) — + shown late only because the roadmap must first create node-spanning groups. + This placement is not a throughput amortization tradeoff: before enabling + any cross-group transaction mode whose `startTS` / `commitTS` can be + allocated by more than one coordinator / ingress node, step 2 must either + pull the dedicated TSO group forward or land a narrower bridge that + allocates all cross-group `startTS` and `commitTS` values from one + designated oracle. The + per-group fix gives per-node monotonicity only (TSO doc §6 "Guarantee"); + commit-ts comparability across coordinators on different nodes is not + covered by it (OQ-1), even when forwarding preserves timestamps to leaders + that happen to be co-located. Treat step 11 as + "deferred-pending-OQ-1-resolution", not "settled-last". 12. **Auto group lifecycle** (Gap 7) — long-term, after 2/4/8/9. In-flight PRs map cleanly: **#955** is step 2 (Gap 1 bootstrap proposal), @@ -345,24 +473,75 @@ design if needed. The near-term mitigation is layered: ## 5. Open Questions -1. **Centralized TSO ordering status.** OQ-1 is resolved for the timestamp - oracle itself: the dedicated TSO group is now the shared source for - cross-node, cross-group `startTS` / `commitTS` once a deployment completes - the `shadow -> cutover -> phase-d` sequence. Step 1 remains necessary as the - legacy compatibility floor, but it is no longer the answer for multi-node - cross-group issuance. In `legacy`, timestamps are still drawn from each - coordinator node's local HLC and the old per-node limitation applies. After - cutover, coordinator-owned timestamps route through group 0, follower - timestamp requests redirect to the TSO leader, Phase D validates - caller-supplied cross-shard timestamps against the durable allocation floor, - and pre-D applied read watermarks use bounded vouchers at dispatch. - - A separate guardrail remains outside OQ-1: cross-group txns with read-only - participant shards still rely on `validateReadOnlyShards`, whose - linearizable-barrier-to-`LatestCommitTS` check is not made stronger merely by - moving timestamp issuance to group 0. Enabling additional multi-node - read-only-shard shapes must still either reject those shapes or land the - dedicated read-validation phase described above. +1. **Per-group HLC fix vs full TSO ordering.** Is the per-group renewal fix + (step 1) sufficient for cross-group OCC correctness in the multi-node + topology, or does the first node-spanning cross-group transaction force the + dedicated TSO group earlier than step 11? + + **Where the timestamps come from (grounded in code).** A cross-group txn's + coordinator issues *both* of its timestamps from one `*HLC` — `c.clock` on + the coordinator's node: + - `startTS` via `nextStartTS` (`kv/sharded_coordinator.go:1429`): + `Observe(maxLatestCommitTS(keys))` then `c.clock.NextFenced()`. The + `maxLatestCommitTS` floor is a per-key read against the store — it pins + `startTS` above the latest commit on *the keys this txn touches*, nothing + more. + - `commitTS` via `resolveTxnCommitTS` → `nextTxnTSAfter` + (`kv/sharded_coordinator.go:1102`, `:1376`): `c.clock.NextFenced()`, + re-allocated after `Observe(startTS)` if it did not strictly exceed + `startTS`. A caller-supplied `commitTS` is fed through `c.clock.Observe` + to keep the clock monotonic. + + Both calls go through the same `c.clock`. The apply-time OCC/ownership check + then compares these timestamps against stored `CommitTS` values (the + Composed-1 guard, `docs/design/2026_05_29_implemented_composed1_cross_group_commit_guard.md` + §4.2(a)/§4.4; FSM `latest > startTS` write-conflict check). OCC + serializability depends on those `commitTS` values being **mutually + comparable** across all participating groups, and read-only participant shards + need the separate validation gate described below. + + **Why the per-group fix is necessary but not sufficient.** Today this is + safe only because every group shares the *same* process-wide `*HLC` + (single-node groups, §1.1/§1.5): one clock issues every timestamp, so all + `commitTS` are trivially comparable. The per-group renewal fix (step 1) + extends correctness to *one node leading several groups* — but its guarantee + is explicitly scoped: "all timestamps issued by a single node are strictly + monotonic … monotonicity across nodes that lead *different* groups is not + fully guaranteed without a shared TSO" (TSO doc §6 "Guarantee", §1.1). The + moment step 2 (multi-node bootstrap) makes it possible for two clients to + coordinate cross-group txns on *different nodes*, two concurrent cross-group + txns can draw `commitTS` from two different `*HLC` instances whose ceilings + were advanced independently. This is true even if the participating groups' + leaders are later co-located, because `ShardedCoordinator.Dispatch` and + `resolveTxnCommitTS` stamp the requests before `LeaderProxy.Commit` / + `Internal.Forward` forwards them. The `maxLatestCommitTS`/`Observe` floor + does not close this: it orders writes only on the specific keys read, not the + global commit order two unrelated cross-group txns need to be serializable + against each other. + + A second guardrail is independent of the timestamp oracle. Cross-group txns + with read-only participant shards currently rely on `validateReadOnlyShards`; + that path has a documented TOCTOU window between the linearizable barrier and + the `LatestCommitTS` check. A dedicated TSO group or single-oracle bridge + makes commit timestamps comparable, but it does not by itself close that + read-validation gap. The rollout gate must therefore either reject + read-only-shard txn shapes until a dedicated read-validate FSM phase lands, + or land that phase before enabling those shapes in a multi-node topology. + + **Conclusion / trigger.** OQ-1 therefore is *not* a "focused correctness + review deferrable until just before step 2 lands" — the per-group fix cannot + answer it in the affirmative for the cross-node case. The trigger to pull the + dedicated TSO group forward from step 11 is concrete: **the first + cross-group transaction mode whose timestamps may be issued by more than one + coordinator node**, not merely the first case whose participant leaders sit + on different nodes. Until then the per-group fix + shared-`*HLC` property + holds. Open sub-question: whether an interim measure short of the full TSO + group — e.g. pinning every cross-group txn's timestamp allocation to a single + designated group's leader (the default group's `c.clock`), so all cross-group + `startTS` and `commitTS` still come from one clock — can bridge the timestamp + gap between step 2 and step 11 without the full batch-allocator TSO. That interim option + must be paired with the read-only-shard validation gate above; it should be + evaluated as part of the step-2 design (PR #955) rather than left to step 11. 2. **Shared cache (Gap 2) vs per-group isolation (workload isolation doc).** A single shared block cache trades isolation for density: one hot group can evict a latency-sensitive group's working set. How does Gap 2's per-group @@ -403,5 +582,3 @@ design if needed. The near-term mitigation is layered: doc §7 already specifies a phased dual-write/shadow-read/feature-flag cutover for the *timestamp* migration; the bootstrap cutover should mirror that structure.) -An *open* pull request, a code primitive, or a superseded roadmap paragraph is -still not sufficient evidence of completion. diff --git a/docs/design/2026_09_05_proposed_apply_time_readiness_evidence.md b/docs/design/2026_09_05_proposed_apply_time_readiness_evidence.md new file mode 100644 index 000000000..c9389cf99 --- /dev/null +++ b/docs/design/2026_09_05_proposed_apply_time_readiness_evidence.md @@ -0,0 +1,169 @@ +# Apply-time target readiness must not read the catalog watcher + +Status: Proposed +Author: bootjp +Date: 2026-09-05 + +Parent: [2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md) + +## 1. The problem + +`kvFSM.verifyTargetReadinessForRange` runs during Raft apply and resolves the +routes it checks through `currentShardRoutesForRouteRange`, which calls +`f.routes.Current()` — the route engine's *current* cached view. + +That view is filled by the catalog watcher from the default group's catalog, over +a stream with **no ordering relationship to the target group's Raft log**. Two +voters applying the same entry at the same index can therefore hold different +catalog versions: the leader may already see the cutover descriptor while a +follower is still behind. + +This is the failure mode `CLAUDE.md` records for the `8668bdce` revert — apply +must not depend on state that is not replicated through the log it is applying. + +## 2. Why it is not merely a stale read + +`targetReadinessStatesSatisfied` returns `ErrRouteCutoverPending` when the routes +it can see do not yet satisfy the replicated readiness guard. Before this change +that was an ordinary FSM response, so the Raft engine advanced the applied index +and the lagging voter **skipped a committed write** while the leader applied it. +The divergence was silent and permanent: nothing re-delivers the entry. + +## 3. What has already been done + +`applyRequest` now converts `ErrRouteCutoverPending` into a halt +(`ErrTargetReadinessApply`). A voter that cannot prove readiness stops instead of +advancing past the entry, so the divergence becomes a loud, recoverable stop +rather than silent data loss. + +This is a containment fix, not a correctness fix. It converts "wrong answer" into +"no answer", which is the right trade for an apply path, but a follower that is +merely behind on catalog delivery will now halt where it previously (wrongly) +continued. That is strictly safer and strictly noisier. + +## 4. Options for making apply deterministic + +### 4.1 Derive readiness from the entry (preferred) + +Requests already carry `observed_route_version`, and the base branch established +the pattern: `routeFloorSnapshotForRequest` resolves +`f.routes.SnapshotAt(observedVer)` and fails with `ErrComposed1VersionGCd` when +that version is no longer retained. Readiness could resolve the same way, making +the verdict a pure function of (entry, replicated readiness guard, retained +catalog history). + +Open questions: + +- Unpinned entries have no observed version. Do they skip the readiness check + (matching how the write-floor check treats them), or is pinning mandatory once + a migration is armed? +- `SnapshotAt` needs the version to still be retained. What is the retention + bound relative to the apply lag a halted follower can accumulate? + +### 4.2 Replicate the routes the guard needs + +The readiness guard itself is already replicated through the target group +(`raftEncodeTargetReadiness`). The route bounds it is checked against could be +carried in that same entry, so apply never consults the catalog cache at all. +Larger entries, but no dependency on catalog delivery order. + +### 4.3 Gate on the guard alone + +`targetReadinessStatesSatisfied` currently needs both the replicated guard *and* +a matching catalog view. If the guard carried enough to decide by itself, the +catalog read disappears. This is 4.2 with the bounds folded into the existing +state rather than the entry. + +## 5. Recommendation + +4.1, because the mechanism already exists on the write-floor path and reuses the +retention story rather than inventing one. 4.2 is the fallback if the retention +bound in §4.1 cannot be made to cover realistic follower lag. + +## 6. Until then + +The halt in §3 stands. It is not a substitute for this document: a cluster whose +catalog delivery lags will see followers halt, which is safe but disruptive, and +that pressure is the reason to land §4 rather than leave the halt as the answer. + +## 7. Bounded native-lineage implementation checkpoint + +The working implementation explores 4.2/4.3, not a watcher-history fallback. +Permanent per-range OWNER/NONOWNER certificates are separate from active +readiness guards and retention pins. The ownership certificate, job retirement, +guard removal, and applied index share one store lock/batch. MVCC snapshots emit +V6 only when ownership certificates exist; the MVCC-to-Pebble streaming path, +native/SST snapshots, and reopen preserve them. + +The shared serving view prevents a read from observing an old catalog floor +with a newly cleared guard. NONOWNER does not reopen because a cache advances. +New requests lacking an exact ownership proof, or spanning a proved and +uncovered interval, temporarily receive a whole-request retry before mutation. +Independent requests coalesced into one Raft entry retain independent results. +Only a matching persisted COMMIT/ABORT resolution may drain through this gate. +A remote primary reference alone is not a local primary-intent participant. + +This checkpoint is not a complete migration protocol and is not merge-ready: + +- The runner must bind certificates to actual committed descriptors using a + root-leader applied read barrier. VerifyLeader alone does not wait for apply. +- Source guard clear must use the new atomic ownership replacement; rollback + must first commit its actual catalog version while retaining the guard. +- All proposal ingress must share admission and stamp an actual proved route + version. The final apply check is still required for queued requests. +- An OWNER certificate currently proves exactly C. A trusted unchanged- + descriptor refresh/attestation is required for unrelated catalog increments; + blindly accepting every later observed version is not valid evidence. +- Target promotion/metadata clear needs a retained cleared OWNER certificate, + with data-ready/floor evidence, and every new voter must attest support. +- The current record does not identify a verified Raft generation. External + and fresh restore of migration-bearing payloads must refuse before publishing + a new ConfState, including physical offload and cold-skip paths. Proof-free, + epoch-only, and empty-metadata restores must continue to support ordinary KV; + copied configuration indices are not migration lineage evidence. +- The earlier target-only apply-route prototype is not the completed protocol: + its future-version predicate and participant footprint still need replacement + by durable actual-C proofs and context-helper integration. + +These remaining requirements must be completed and tested before publication. + +The checkpoint audit additionally identified three unresolved interactions: +an unrelated range advancing the group floor can invalidate an older exact-C +OWNER proof; the pin check currently runs before establishing whether a request +intersects migration evidence; and helper-level matching-resolution admission +does not prove the subsequent target/composed/floor path or durable idempotent +retry succeeds. Separate immutable operation receipts from refreshable replicated +serving attestations, and test the complete apply path before claiming closure. + +## 8. SOURCE creation and public admission checkpoint + +SOURCE tracking uses the actual root descriptor after a root-leader applied +barrier. Its inherited write floor is separately marked known; a verified zero +is not the tracker's positive retention/HLC floor. The tracker is applied first +in the data log, then a configuration-bound SOURCE acquisition records its own +applied boundary. Until that acquisition, creation retries deterministically. +The immutable acquisition receipt retains this boundary across later unchanged- +descriptor REFRESH attestations and native snapshot/reopen. Source fences still +deny creation, while authenticated persisted COMMIT/ABORT resolutions drain. + +Operational evidence requires generation, descriptor identity/hash, operation +receipt, configuration and applied boundary. The decoder remains compatible +with older records, but such unknown records cannot authorize their range. +Unrelated ordinary KV does not require migration evidence. SOURCE controls use +explicit opcode mode 6 and protocol 2; every voter must attest support before +the runner submits them. An epoch change invalidates SOURCE creation until a +new all-voter, configuration-bound attestation. + +The common TransactionManager proposal path checks final requests after queue +waiting and preserves independent results for coalesced requests. ShardStore +scan/physical/exact-version entry points and coordinator read-key checks share +the durable serving gate. Apply repeats admission independently of proposal. +Unpinned or incompletely covered affected requests retry before any mutation; +there is no watcher or fabricated floor-zero fallback. Production wiring and +the authenticated cross-leader refresh consumer remain integration gates. + +A later split of a certificate-bearing route can change the retained left +descriptor hash even when that prefix keeps its owner. Unchanged-descriptor +REFRESH cannot repair that case. A durable pre-effect catalog mutation gate or +an explicitly replicated retained-prefix reacquisition is still required; +this checkpoint does not claim sequential split availability or merge readiness. diff --git a/docs/design/2026_10_23_proposed_migration_ownership_partitions.md b/docs/design/2026_10_23_proposed_migration_ownership_partitions.md new file mode 100644 index 000000000..6bdacee33 --- /dev/null +++ b/docs/design/2026_10_23_proposed_migration_ownership_partitions.md @@ -0,0 +1,187 @@ +# Replicated ownership partitions for subsequent range splits + +Status: Implementation in PR #1090; final acceptance, review and merge pending +Author: bootjp +Date: 2026-10-23 + +This extends the bounded native-lineage ownership work in +[apply-time readiness evidence](2026_09_05_proposed_apply_time_readiness_evidence.md). +The Milestone 2 design's historical Implemented label does not mean the current +PR #1090 ownership integration has shipped. Final acceptance, latest-HEAD review +and CI are tracked separately; this proposal keeps its filename until merge. + +## Contract + +The canonical Go interface is +`store/migration_ownership_partition_types.go`. Both memory and Pebble stores +implement it. Immutable per-job Acquire/Clear receipts remain keyed by job ID. +The active, non-overlapping serving frontier has independent partition IDs; +several children can retain the same immutable job receipt. Never flatten the +job ledger into child records or deduplicate active children by job ID. + +Same-owner partition controls preserve the complete prior union, role, known +floor, staged-data lineage and protected SOURCE boundary. References identify +the exact prior attestation, including every overlapping negative proof. A +NONOWNER cannot become OWNER through partition or unchanged-descriptor refresh. +Only an ordinary newer guarded acquisition with data-ready evidence can reopen +ownership. Semantic mismatches deterministically retry before effects. + +Root geometry transactions atomically persist a manifest with their descriptor, +catalog-version, route allocation and working-source mutations. Only a manifest +reloaded after the root leader's linearizable applied-read barrier can attest +actual committed C. Data leaders install it in their own logs with generation +and applied-configuration preconditions. Apply never consults a catalog watcher +for authority. Historical NONOWNER groups discover retained manifest chains by +old descriptor identity through the authenticated refresh consumer, including +after job-history GC. Historical all-NONOWNER evidence may discover a manifest +by the never-reused native route ID plus exact full bounds and parent ID when +a remote handoff changed the old descriptor hash. This bounded lookup rejects +ambiguous records, preserves the historical owner, floor, role and origins, +and never substitutes for positive ownership attestation. With no new geometry, +background refresh retains DENY; a foreground move remains pending rather than +claiming an ACK. Negative groups need not be online for a same-owner metadata split; +their old negative proof continues denying until reconciliation/acquisition. + +Before a cross-group suffix move, commit and replicate a full same-owner +partition into retained left and working moving child. SOURCE tracker, catalog +fence, write floor, cutover and rollback then operate only on that moving child. +The job records its working source durably so retries do not allocate children +again. Same-group SplitRange uses this partition protocol too; subsequent splits +are supported, not permanently refused. An abandon may leave the harmless +same-owner metadata split but cannot lower floors or restore a negative owner. +Before publishing the manifest or either route, encode the complete downstream +partition control against the current native frontier and its configuration. +Reject controls exceeding the 4 MiB envelope after reserving the full uint64 +decimal width at every numeric position. Working-source creation checks both +the source and required target, including historical negative evidence. A data +leader read fence, or an exact-group leader fence followed by local applied-index +catch-up, precedes even an empty-frontier decision. The 16 MiB manifest limit +alone does not bound the repeated, base64-encoded +priors and tiles. This is a metadata-operation limit, not a global RawKV limit. + +SplitRange is authenticated control-plane traffic even for a proof-free parent. +Advancing its global catalog version also affects disjoint native OWNER proofs. +Metadata/working-child splits therefore discover and fence serving owners across +groups, establish current-version readiness before mutation, and synchronously +refresh/ACK them at the actual committed new version before returning or updating +the local route engine. A four-item background refresh budget is not the foreground +availability mechanism. Historical negative-only closures may remain offline; +already-fenced SOURCE intervals stay denied rather than acquiring new authority. +The root transaction is still committed before data-group refresh: independent +watchers may briefly observe a version whose owner ACK is pending and retry. +This is not atomic root/data publication or a claim of zero transient retries. +Recovery of a persisted working child repeats the complete serving-owner barrier +against the committed catalog before publishing locally or arming its tracker. +The durable child ID proves the root transaction, not completion of those ACKs; +an unavailable disjoint owner keeps recovery pending without reallocating routes. +Move-back also replays the target's retained negative geometry in the foreground +before control-size preflight. Deferred geometry is not a readiness ACK: complete +size validation and the post-commit target barrier still precede acquisition. + +## Persistence Hook Map + +The implementation uses the following persistence hooks: + +| File | Required hook | +| --- | --- | +| `store/migration_ownership_partition*.go` | New raw container `!migstage|ownership_partition`, codec v1, canonical identities/references, validators, atomic memory/Pebble transition and immutable receipts. | +| `store/migration_ownership.go` | Add `Partitions MigrationOwnershipPartitionState` to the coherent serving view; couple existing guarded Acquire/Clear/Refresh to the frontier in the same lock/batch as the old job ledger and applied index. SOURCE uses this same acquire path. | +| `store/migration_ownership_evidence.go` | Preserve full origin, known-floor and SOURCE data-log-boundary validation. Do not globally reject decodable legacy job records. | +| `store/mvcc_store.go` | New in-memory container, V7 trailer only when needed, V1..V6 compatibility, coherent snapshot capture and atomic restored-tree/metadata publication. | +| `store/snapshot_migration_metadata.go` | Validate on reopen and stage the exact new raw key in streaming MVCC-to-Pebble restore before publication. | +| `store/lsm_store.go` | Pass the V7 metadata into the temporary restore DB; native/SST carry the raw key and must validate it. Preserve encrypted snapshot plumbing. | +| `store/migration_versions.go` | Classify/exclude/protect the new raw metadata key in export/import/cleanup. Do not confuse decoded MVCC user keys with raw metadata. | +| `store/snapshot_external.go` | Refuse nonempty frontier or receipts before tree/DB publication under external rebase, including receipt-only evidence; ordinary proof-free restore remains compatible. | +| `internal/backup/snapshot_reader.go` | Mirror exact raw-key filtering for the standalone offline reader. This is the store worker's single assigned non-store hook. | + +The new container is independent of `!migstage|ownership`. Both native histories +must survive reopen, MVCC streaming, native/SST/encrypted snapshots and voter +join. External restore must not reuse copied configuration epochs as lineage. +Protocol auth, Raft controls and public/FSM admission live in `proto`, +`distribution`, `adapter` and `kv`; process membership wiring lives in main. +The serving cache publishes a coherent native frontier after each durable +metadata commit. Ordinary reads do not enumerate all groups' Pebble metadata +or acquire every apply mutex. Unknown legacy barriers remain in that view. + +## Helper Interfaces + +The following helpers implement the canonical identities alongside the types: + +```go +MigrationOwnershipPartitionReferenceFor(MigrationOwnershipPartition) (MigrationOwnershipPartitionReference, error) +MigrationOwnershipLegacyReferenceFor(MigrationOwnershipState) (MigrationOwnershipPartitionReference, error) +MigrationOwnershipPartitionIDFor(MigrationOwnershipPartitionTransition, int) ([]byte, error) +MigrationOwnershipPartitionReceiptIDFor(MigrationOwnershipPartitionTransition) ([]byte, error) +MigrationOwnershipPartitionEvidenceValid(MigrationOwnershipPartitionState, MigrationOwnershipPartition) bool +MigrationOwnershipPartitionMetadataKey() []byte +IsMigrationOwnershipPartitionMetadataKey([]byte) bool +``` + +IDs/digests are domain-separated SHA-256 with canonical encoding, never ad hoc +JSON hashing by a caller. Partition IDs survive refresh; receipt IDs bind exact +operations. Legacy references require complete operational old evidence and +hash the entire current job state. Complete old positive and negative coverage +must be validated before adoption; unknown legacy records cannot seed authority. +The internal-only JobOperation receipt kind can be produced only by existing +validated Acquire/Clear operations, not external partition controls. + +## Runtime Capability + +The geometry capability is `cap_migration_partition_v1`, version 1. The static +probe is `ProbeMigrationStateResponse.ownership_partition_protocol = 10` and +the option is `WithInternalMigrationPartitionProtocolReady(func() bool)`. + +Static implemented support must be observable on a new learner before it has +an applied configuration. Operational ownership readiness, guard installation +and ACKs independently require a real nonzero applied configuration, generation +and all-member support. Static support never overrides configured authentication, +opcode flags, installed read/proposal/consumer hooks or the all-current-member barrier. + +## Rolling Namespace Activation + +New binaries preserve legacy committed KV/transaction semantics until an +ordered, durable namespace activation command is applied. Every actual apply +member, including learners, must advertise installed support at the same +configuration before activation; a leader-only fallback is not permitted. +The durable marker carries configuration, activation index, member inventory +and authorized joining IDs. Ordered ConfChange application, native snapshot +restore and prejoin hooks preserve this state. Catalog watchers do not authorize +activation. New migration mutation RPCs require the marker; the strictly typed +namespace activation control is the only bootstrap exception. +Activation uses the bounded cleanup RPC attempt, never the runner's process +lifetime alone. Parent cancellation and an earlier parent deadline are retained. +After activation, supported peers at the matching configuration may still await +the entry. Their incomplete ACK is retryable without consuming the phase failure +budget; unsupported versions or mismatched configurations remain refused. + +Forwarded admin proposals use an exact typed allowlist for existing backup and +PREPARE-evidence controls. Migration, encrypted, user and malformed payloads +cannot bypass their authenticated mutation paths through this RPC. + +## Moving-Range Size Guard + +RawKV's global 64 MiB limit is unchanged. A versioned replicated guard limits +only new writes and PREPARE creation in a migrating range to the transfer-page +budget, including intent-key/value overhead and actual command framing. Retained +neighbors remain writable. Existing matching PREPAREs may COMMIT or ABORT without +retroactive size rejection. All actual source members acknowledge the guard +before copying or the write fence. A full historical-value preflight detects +untransferable data before a catalog fence and safely abandons the move. +An otherwise successful probe reporting `Ready: false` is an incomplete voter +barrier, not a terminal phase failure. Learner catch-up therefore does not consume +the failure budget or strand a fenced job in FAILED. Readiness is still required +before advancing; invalid configuration/protocol evidence and authentication +errors do not become successful ACKs. + +## Acceptance Gates + +The independent harness owns existing-public-interface regressions for both +same-group resplit and cross-group suffix migration of a previously migrated +range. Retained-left reads and raw/1PC/PREPARE writes must survive the split and +unrelated later C without descriptor-hash guesses or floor changes. Also require +negative-owner safety, real move-back, full matching COMMIT/ABORT drain, exact and +conflicting retries, leader/config changes at each phase, three voters with +different caches, batch failure, job GC, reopen and every native snapshot path. + +No focused test, type compilation or capability advertisement completes these +gates or declares PR #1090 merge/deployment ready. diff --git a/internal/backup/snapshot_reader.go b/internal/backup/snapshot_reader.go index 719e0fbee..c053df403 100644 --- a/internal/backup/snapshot_reader.go +++ b/internal/backup/snapshot_reader.go @@ -276,6 +276,9 @@ var pebbleOperationalKeys = [][]byte{ []byte("_migack"), []byte("_mighlc"), []byte("_migpromote"), + []byte("!migstage|retired"), + []byte("!migstage|ownership"), + []byte("!migstage|ownership_partition"), } // Writer-registry rows: mirror internal/encryption.WriterRegistryPrefix, @@ -310,7 +313,9 @@ func isPebbleOperationalKey(rawKey []byte) bool { return true } } - return isPebbleWriterRegistryKey(rawKey) + const readinessPrefix = "!migstage|ready|" + return isPebbleWriterRegistryKey(rawKey) || + (len(rawKey) == len(readinessPrefix)+snapshotTSSize && bytes.HasPrefix(rawKey, []byte(readinessPrefix))) } // readOneEntry handles one (key, value) tuple plus the callback diff --git a/internal/backup/snapshot_reader_live_test.go b/internal/backup/snapshot_reader_live_test.go index f2ef47963..26b8b0213 100644 --- a/internal/backup/snapshot_reader_live_test.go +++ b/internal/backup/snapshot_reader_live_test.go @@ -139,6 +139,13 @@ func TestIsPebbleOperationalKey_MatchesStore(t *testing.T) { {name: "migration ack", key: []byte("_migack"), want: true}, {name: "migration hlc floor", key: []byte("_mighlc"), want: true}, {name: "migration promote", key: []byte("_migpromote"), want: true}, + {name: "migration ownership", key: []byte("!migstage|ownership"), want: true}, + {name: "migration partitions", key: []byte("!migstage|ownership_partition"), want: true}, + {name: "partition near miss", key: []byte("!migstage|ownership_partition-user"), want: false}, + {name: "migration retirement", key: []byte("!migstage|retired"), want: true}, + {name: "migration readiness", key: append([]byte("!migstage|ready|"), make([]byte, snapshotTSSize)...), want: true}, + {name: "ownership near miss", key: []byte("!migstage|ownership-user"), want: false}, + {name: "bare readiness prefix", key: []byte("!migstage|ready|"), want: false}, {name: "writer registry row", key: registry, want: true}, {name: "bare registry prefix", key: bytes.Clone(encryption.WriterRegistryPrefix), want: false}, {name: "registry row one byte short", key: registry[:len(registry)-1], want: false}, diff --git a/internal/grpc.go b/internal/grpc.go index c3658622f..8f368aad9 100644 --- a/internal/grpc.go +++ b/internal/grpc.go @@ -10,17 +10,26 @@ const GRPCMaxMessageBytes = 64 << 20 // GRPCServerOptions keeps Raft replication and the public/internal APIs aligned // on the same message-size budget. func GRPCServerOptions() []grpc.ServerOption { - return []grpc.ServerOption{ + opts := []grpc.ServerOption{ grpc.MaxRecvMsgSize(GRPCMaxMessageBytes), grpc.MaxSendMsgSize(GRPCMaxMessageBytes), } + if cfg := grpcTLS.Load(); cfg != nil { + opts = append(opts, grpc.Creds(cfg.server.Clone())) + } + return opts } -// GRPCDialOptions returns the common insecure dial options used by node-local -// and node-to-node traffic. +// GRPCDialOptions returns the common transport and message-size options for +// node-local and node-to-node traffic. TLS is explicitly configured at startup. func GRPCDialOptions() []grpc.DialOption { + creds := insecure.NewCredentials() + if cfg := grpcTLS.Load(); cfg != nil { + creds = cfg.client.Clone() + } return []grpc.DialOption{ - grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithTransportCredentials(creds), + grpc.WithChainUnaryInterceptor(forwardTokenClientInterceptor), grpc.WithDefaultCallOptions( grpc.MaxCallRecvMsgSize(GRPCMaxMessageBytes), grpc.MaxCallSendMsgSize(GRPCMaxMessageBytes), diff --git a/internal/grpc_forward_auth.go b/internal/grpc_forward_auth.go new file mode 100644 index 000000000..f0c58d2cd --- /dev/null +++ b/internal/grpc_forward_auth.go @@ -0,0 +1,34 @@ +package internal + +import ( + "context" + "sync/atomic" + + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +var grpcForwardToken atomic.Pointer[string] + +// ConfigureGRPCForwardToken configures node-to-node Forward calls in clusters +// with authenticated migration enabled. Other RPC credentials are unchanged. +func ConfigureGRPCForwardToken(token string) { + if token == "" { + grpcForwardToken.Store(nil) + return + } + grpcForwardToken.Store(&token) +} + +func forwardTokenClientInterceptor(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if token := grpcForwardToken.Load(); token != nil && method == pb.Internal_Forward_FullMethodName { + md, _ := metadata.FromOutgoingContext(ctx) + md = md.Copy() + md.Delete("authorization") + ctx = metadata.NewOutgoingContext(ctx, md) + opts = append(opts, grpc.PerRPCCredentials(GRPCBearerToken(*token))) + } + return errors.WithStack(invoker(ctx, method, req, reply, cc, opts...)) +} diff --git a/internal/grpc_forward_auth_test.go b/internal/grpc_forward_auth_test.go new file mode 100644 index 000000000..c67b34205 --- /dev/null +++ b/internal/grpc_forward_auth_test.go @@ -0,0 +1,43 @@ +package internal + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +func TestGRPCForwardTokenOnlyAuthenticatesPeerForward(t *testing.T) { + ConfigureGRPCForwardToken("peer-secret") + t.Cleanup(func() { ConfigureGRPCForwardToken("") }) + ctx := metadata.NewOutgoingContext(context.Background(), metadata.Pairs("authorization", "Bearer stale", "trace-id", "keep")) + for _, method := range []string{pb.Internal_Forward_FullMethodName, pb.Internal_ForwardLeaseRead_FullMethodName, pb.Admin_GetNodeVersion_FullMethodName} { + called := false + err := forwardTokenClientInterceptor(ctx, method, nil, nil, nil, func(callCtx context.Context, _ string, _, _ any, _ *grpc.ClientConn, opts ...grpc.CallOption) error { + called = true + md, _ := metadata.FromOutgoingContext(callCtx) + require.Equal(t, []string{"keep"}, md.Get("trace-id")) + if method == pb.Internal_Forward_FullMethodName { + require.Empty(t, md.Get("authorization")) + require.Len(t, opts, 1) + option, ok := opts[0].(grpc.PerRPCCredsCallOption) + require.True(t, ok) + require.True(t, option.Creds.RequireTransportSecurity()) + token, tokenErr := option.Creds.GetRequestMetadata(callCtx) + require.NoError(t, tokenErr) + require.Equal(t, "Bearer peer-secret", token["authorization"]) + } else { + require.Equal(t, []string{"Bearer stale"}, md.Get("authorization")) + require.Empty(t, opts) + } + return nil + }) + require.NoError(t, err) + require.True(t, called) + } + md, _ := metadata.FromOutgoingContext(ctx) + require.Equal(t, []string{"Bearer stale"}, md.Get("authorization")) +} diff --git a/internal/grpc_tls.go b/internal/grpc_tls.go new file mode 100644 index 000000000..7bdabcd4b --- /dev/null +++ b/internal/grpc_tls.go @@ -0,0 +1,70 @@ +package internal + +import ( + "context" + "crypto/tls" + "crypto/x509" + "os" + "sync/atomic" + + "github.com/cockroachdb/errors" + "google.golang.org/grpc/credentials" +) + +type grpcTLSCredentials struct { + server credentials.TransportCredentials + client credentials.TransportCredentials +} + +var grpcTLS atomic.Pointer[grpcTLSCredentials] + +// ConfigureGRPCTLS installs transport credentials before node services start. +// Empty paths preserve the existing plaintext peer transport. +func ConfigureGRPCTLS(caPath, certPath, keyPath string) error { + if caPath == "" && certPath == "" && keyPath == "" { + grpcTLS.Store(nil) + return nil + } + if certPath == "" || keyPath == "" { + return errors.New("gRPC TLS requires both certificate and key files") + } + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return errors.Wrap(err, "load gRPC TLS certificate/key") + } + client, err := GRPCClientTLSCredentials(caPath) + if err != nil { + return err + } + grpcTLS.Store(&grpcTLSCredentials{ + server: credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{cert}}), + client: client, + }) + return nil +} + +// GRPCClientTLSCredentials verifies peers using the given CA or system roots. +// The peer name is taken from the dial target; verification cannot be disabled. +func GRPCClientTLSCredentials(caPath string) (credentials.TransportCredentials, error) { + cfg := &tls.Config{MinVersion: tls.VersionTLS12} + if caPath != "" { + pem, err := os.ReadFile(caPath) + if err != nil { + return nil, errors.Wrap(err, "read gRPC TLS CA file") + } + cfg.RootCAs = x509.NewCertPool() + if !cfg.RootCAs.AppendCertsFromPEM(pem) { + return nil, errors.New("gRPC TLS CA file contains no certificates") + } + } + return credentials.NewTLS(cfg), nil +} + +// GRPCBearerToken is never sent on an insecure gRPC connection. +type GRPCBearerToken string + +func (t GRPCBearerToken) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { + return map[string]string{"authorization": "Bearer " + string(t)}, nil +} + +func (GRPCBearerToken) RequireTransportSecurity() bool { return true } diff --git a/internal/grpc_tls_test.go b/internal/grpc_tls_test.go new file mode 100644 index 000000000..04d64f822 --- /dev/null +++ b/internal/grpc_tls_test.go @@ -0,0 +1,77 @@ +package internal + +import ( + "context" + "crypto/x509" + "encoding/pem" + "net" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +func TestConfigureGRPCTLSValidatesFiles(t *testing.T) { + require.ErrorContains(t, ConfigureGRPCTLS("ca.pem", "", ""), "certificate and key") + require.ErrorContains(t, ConfigureGRPCTLS("", "missing.pem", "missing.key"), "certificate/key") + _, err := GRPCClientTLSCredentials("missing.pem") + require.ErrorContains(t, err, "CA file") + invalid := filepath.Join(t.TempDir(), "invalid.pem") + require.NoError(t, os.WriteFile(invalid, []byte("invalid"), 0o600)) + _, err = GRPCClientTLSCredentials(invalid) + require.ErrorContains(t, err, "no certificates") + require.NoError(t, ConfigureGRPCTLS("", "", "")) + require.Nil(t, grpcTLS.Load()) +} + +func TestGRPCTLSCommonOptionsVerifyPeer(t *testing.T) { + fixture := httptest.NewTLSServer(nil) + fixture.Close() + cert := fixture.Certificate() + key, err := x509.MarshalPKCS8PrivateKey(fixture.TLS.Certificates[0].PrivateKey) + require.NoError(t, err) + dir := t.TempDir() + certPath, keyPath := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") + require.NoError(t, os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}), 0o600)) + require.NoError(t, os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key}), 0o600)) + require.NoError(t, ConfigureGRPCTLS(certPath, certPath, keyPath)) + t.Cleanup(func() { require.NoError(t, ConfigureGRPCTLS("", "", "")) }) + var lc net.ListenConfig + lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + server := grpc.NewServer(GRPCServerOptions()...) + healthpb.RegisterHealthServer(server, health.NewServer()) + serveErr := make(chan error, 1) + go func() { serveErr <- server.Serve(lis) }() + t.Cleanup(func() { + server.Stop() + _ = lis.Close() + require.NoError(t, <-serveErr) + }) + conn, err := grpc.NewClient(lis.Addr().String(), GRPCDialOptions()...) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err = healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}) + require.NoError(t, err) + // System roots do not trust the test CA; never fall back to plaintext. + creds, err := GRPCClientTLSCredentials("") + require.NoError(t, err) + untrusted, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(creds)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, untrusted.Close()) }) + _, err = healthpb.NewHealthClient(untrusted).Check(ctx, &healthpb.HealthCheckRequest{}) + require.ErrorContains(t, err, "certificate") + wrongName, err := grpc.NewClient(lis.Addr().String(), append(GRPCDialOptions(), grpc.WithAuthority("wrong-node.invalid"))...) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, wrongName.Close()) }) + _, err = healthpb.NewHealthClient(wrongName).Check(ctx, &healthpb.HealthCheckRequest{}) + require.ErrorContains(t, err, "certificate") +} diff --git a/internal/raftadmin/interceptor.go b/internal/raftadmin/interceptor.go index 92cb87750..ac5694cc1 100644 --- a/internal/raftadmin/interceptor.go +++ b/internal/raftadmin/interceptor.go @@ -26,3 +26,9 @@ type MembershipChangeInterceptor interface { // target node's Raft/gRPC endpoint and may be used for capability checks. PreAddMember(ctx context.Context, raftID, address string) error } + +// MembershipPromotionInterceptor optionally rechecks a learner before promotion. +// It does not repeat PreAddMember's potentially mutating registration work. +type MembershipPromotionInterceptor interface { + PrePromoteMember(ctx context.Context, raftID, address string) error +} diff --git a/internal/raftadmin/promotion_interceptor_test.go b/internal/raftadmin/promotion_interceptor_test.go new file mode 100644 index 000000000..e630fd531 --- /dev/null +++ b/internal/raftadmin/promotion_interceptor_test.go @@ -0,0 +1,97 @@ +package raftadmin + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type promotionInterceptor struct { + recordingInterceptor + promote func(context.Context, string, string) error +} + +func (p *promotionInterceptor) PrePromoteMember(ctx context.Context, id, address string) error { + return p.promote(ctx, id, address) +} + +func TestPromotionInterceptorRunsBeforeProposal(t *testing.T) { + t.Parallel() + for _, reject := range []bool{false, true} { + t.Run(map[bool]string{false: "accept", true: "reject"}[reject], func(t *testing.T) { + t.Parallel() + engine := &fakeEngine{config: raftengine.Configuration{Servers: []raftengine.Server{ + {ID: "candidate", Address: "candidate:50051", Suffrage: "learner"}, + }}} + calls := 0 + hook := &promotionInterceptor{promote: func(_ context.Context, id, address string) error { + require.Empty(t, engine.promoteLearnerCalls) + require.Equal(t, "candidate", id) + require.Equal(t, "candidate:50051", address) + calls++ + if reject { + return status.Error(codes.FailedPrecondition, "migration protocol unsupported") + } + return nil + }} + server := NewServerWithInterceptor(engine, hook) + resp, err := server.PromoteLearner(t.Context(), &pb.RaftAdminPromoteLearnerRequest{ + Id: "candidate", PreviousIndex: 7, MinAppliedIndex: 99, + }) + require.Equal(t, 1, calls) + if reject { + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Nil(t, resp) + require.Empty(t, engine.promoteLearnerCalls) + return + } + require.NoError(t, err) + require.Equal(t, uint64(44), resp.GetIndex()) + require.Equal(t, []fakePromoteLearnerCall{{id: "candidate", prevIndex: 7, minAppliedIndex: 99}}, engine.promoteLearnerCalls) + require.Empty(t, hook.calls, "promotion must not rerun encryption registration") + }) + } +} + +func TestPromotionInterceptorRejectsMissingAddressOrPendingConfiguration(t *testing.T) { + t.Parallel() + for _, name := range []string{"missing-member", "missing-address", "pending-configuration"} { + t.Run(name, func(t *testing.T) { + t.Parallel() + engine := &fakeEngine{} + if name == "missing-address" { + engine.config.Servers = []raftengine.Server{{ID: "candidate", Suffrage: "learner"}} + } + if name == "pending-configuration" { + engine.status.PendingConfChange = true + engine.config.Servers = []raftengine.Server{{ID: "candidate", Address: "candidate:50051", Suffrage: "learner"}} + } + hook := &promotionInterceptor{promote: func(context.Context, string, string) error { + t.Fatal("invalid configuration must not reach the capability probe") + return nil + }} + server := NewServerWithInterceptor(engine, hook) + resp, err := server.PromoteLearner(t.Context(), &pb.RaftAdminPromoteLearnerRequest{Id: "candidate", MinAppliedIndex: 99}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Nil(t, resp) + require.Empty(t, engine.promoteLearnerCalls) + }) + } +} + +func TestPromotionPreservesAddOnlyInterceptor(t *testing.T) { + t.Parallel() + engine := &fakeEngine{} + hook := &recordingInterceptor{retErr: context.DeadlineExceeded} + server := NewServerWithInterceptor(engine, hook) + resp, err := server.PromoteLearner(t.Context(), &pb.RaftAdminPromoteLearnerRequest{Id: "candidate", MinAppliedIndex: 99}) + require.NoError(t, err) + require.Equal(t, uint64(44), resp.GetIndex()) + require.Len(t, engine.promoteLearnerCalls, 1) + require.Empty(t, hook.calls, "legacy registration hooks must not run on promotion") +} diff --git a/internal/raftadmin/server.go b/internal/raftadmin/server.go index de96041f4..9749abdd5 100644 --- a/internal/raftadmin/server.go +++ b/internal/raftadmin/server.go @@ -152,6 +152,12 @@ func (s *Server) PromoteLearner(ctx context.Context, req *pb.RaftAdminPromoteLea } s.membershipMu.Lock() defer s.membershipMu.Unlock() + if err := s.requireMembershipChangeReady(); err != nil { + return nil, err + } + if err := s.checkPromotionInterceptor(ctx, req.Id); err != nil { + return nil, err + } index, err := s.admin.PromoteLearner(ctx, req.Id, req.PreviousIndex, req.MinAppliedIndex, req.SkipMinAppliedCheck) if err != nil { return nil, adminError(err) @@ -159,6 +165,23 @@ func (s *Server) PromoteLearner(ctx context.Context, req *pb.RaftAdminPromoteLea return &pb.RaftAdminConfigurationChangeResponse{Index: index}, nil } +func (s *Server) checkPromotionInterceptor(ctx context.Context, id string) error { + hook, ok := s.interceptor.(MembershipPromotionInterceptor) + if !ok { + return nil + } + cfg, err := s.engine.Configuration(ctx) + if err != nil { + return adminError(err) + } + for _, member := range cfg.Servers { + if member.ID == id && member.Address != "" { + return adminError(hook.PrePromoteMember(ctx, id, member.Address)) + } + } + return grpcStatus(codes.FailedPrecondition, "promotion capability check requires a configured member address") +} + func (s *Server) RemoveServer(ctx context.Context, req *pb.RaftAdminRemoveServerRequest) (*pb.RaftAdminConfigurationChangeResponse, error) { if s == nil || s.admin == nil { return nil, grpcStatus(codes.Unimplemented, "remove server is not supported by this raft engine") diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 24758dcb2..28bd3161f 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -706,7 +706,7 @@ func Open(ctx context.Context, cfg OpenConfig) (*Engine, error) { } engine.configIndex.Store(maxAppliedIndex(prepared.disk.LocalSnap)) engine.appliedIndex.Store(initialApplied) - if err := engine.restorePendingConfChangeFenceFromStorage(rawNodeApplied); err != nil { + if err := engine.recoverOpenConfiguration(prepared, rawNodeApplied); err != nil { _ = closePersist(prepared.disk.Persist) return nil, err } @@ -740,6 +740,24 @@ func Open(ctx context.Context, cfg OpenConfig) (*Engine, error) { return openedEngine, nil } +func (e *Engine) recoverOpenConfiguration(prepared preparedOpenState, rawNodeApplied uint64) error { + if err := recoverNamespaceConfiguration(e.fsm, prepared.disk.LocalSnap); err != nil { + return errors.Wrap(err, "recover namespace configuration") + } + if err := e.restorePendingConfChangeFenceFromStorage(rawNodeApplied); err != nil { + return err + } + if prepared.disk.ExternalRestoreValidated { + // External preparation seeded this destination ConfState at this index. + // Bind only after store validation and RawNode restoration succeed; never + // reuse an imported epoch or a later native snapshot's data index. + if err := e.recordFSMConfiguration(prepared.disk.LocalSnap.GetMetadata().GetIndex(), confStateValue(prepared.disk.LocalSnap.GetMetadata().GetConfState())); err != nil { + return errors.Wrap(err, "bind external restore destination configuration") + } + } + return completeExternalRestoreOrigin(prepared.cfg.DataDir, prepared.disk.ExternalRestoreValidated) +} + func (e *Engine) warnIfLeaseReadDisabled() { if lease := e.LeaseDuration(); lease <= 0 { slog.Warn("etcd raft engine: lease read disabled (non-positive LeaseDuration)", @@ -2678,19 +2696,8 @@ func (e *Engine) applyReadySnapshotLocked(snapshot *raftpb.Snapshot) error { // Snapshot application is intentionally synchronous with the raft loop: the // local FSM must reflect the incoming raft snapshot before Ready can advance // and later committed entries can be applied safely. - if isSnapshotToken(snapshot.GetData()) { - tok, err := decodeSnapshotToken(snapshot.GetData()) - if err != nil { - return errors.Wrapf(err, "decode snapshot token index=%d", snapshot.GetMetadata().GetIndex()) - } - if err := openAndRestoreFSMSnapshot(e.fsm, fsmSnapPath(e.fsmSnapDir, tok.Index), tok.CRC32C); err != nil { - return errors.Wrapf(err, "restore fsm snapshot file index=%d crc=%08x", tok.Index, tok.CRC32C) - } - } else { - // Legacy format: full FSM payload in snapshot.Data. - if err := e.fsm.Restore(bytes.NewReader(snapshot.GetData())); err != nil { - return errors.Wrapf(err, "restore fsm from legacy snapshot payload index=%d", snapshot.GetMetadata().GetIndex()) - } + if err := e.restoreReadySnapshotFSM(snapshot); err != nil { + return err } if err := e.storage.ApplySnapshot(snapshot); err != nil { return errors.Wrapf(err, "apply snapshot to raft storage index=%d term=%d", @@ -2721,6 +2728,27 @@ func (e *Engine) applyReadySnapshotLocked(snapshot *raftpb.Snapshot) error { return nil } +func (e *Engine) restoreReadySnapshotFSM(snapshot *raftpb.Snapshot) error { + if isSnapshotToken(snapshot.GetData()) { + tok, err := decodeSnapshotToken(snapshot.GetData()) + if err != nil { + return errors.Wrapf(err, "decode snapshot token index=%d", snapshot.GetMetadata().GetIndex()) + } + if err := openAndRestoreFSMSnapshot(e.fsm, fsmSnapPath(e.fsmSnapDir, tok.Index), tok.CRC32C); err != nil { + return errors.Wrapf(err, "restore fsm snapshot file index=%d crc=%08x", tok.Index, tok.CRC32C) + } + } else { + // Legacy format: full FSM payload in snapshot.Data. + if err := e.fsm.Restore(bytes.NewReader(snapshot.GetData())); err != nil { + return errors.Wrapf(err, "restore fsm from legacy snapshot payload index=%d", snapshot.GetMetadata().GetIndex()) + } + } + if err := recoverNamespaceConfiguration(e.fsm, *snapshot); err != nil { + return errors.Wrap(err, "recover snapshot namespace configuration") + } + return nil +} + // savePeersFileForSnapshot writes the v2 peers file with suffrage // drawn from the snapshot's ConfState. Idempotent: savePersistedPeers // short-circuits when the on-disk index already covers `index`. @@ -2998,6 +3026,9 @@ func (e *Engine) applyConfChangeCommitted(entry raftpb.Entry) error { if err != nil { return errors.Wrapf(err, "build peer inventory for config change index=%d", entry.GetIndex()) } + if err := e.recordFSMConfiguration(entry.GetIndex(), confStateValue(confState)); err != nil { + return err + } if err := e.persistConfigState(entry.GetIndex(), confStateValue(confState), nextPeers); err != nil { return err } @@ -3019,6 +3050,9 @@ func (e *Engine) applyConfChangeV2Committed(entry raftpb.Entry) error { if err != nil { return errors.Wrapf(err, "build peer inventory for config change v2 index=%d", entry.GetIndex()) } + if err := e.recordFSMConfiguration(entry.GetIndex(), confStateValue(confState)); err != nil { + return err + } if err := e.persistConfigState(entry.GetIndex(), confStateValue(confState), nextPeers); err != nil { return err } @@ -3028,6 +3062,19 @@ func (e *Engine) applyConfChangeV2Committed(entry raftpb.Entry) error { return nil } +func (e *Engine) recordFSMConfiguration(index uint64, state raftpb.ConfState) error { + return recordNamespaceConfiguration(e.fsm, index, state) +} + +func (e *Engine) AppliedConfigurationIndex() (uint64, bool) { + aware, ok := e.fsm.(raftengine.AppliedConfigurationAware) + if !ok { + return 0, false + } + index := aware.AppliedConfigurationIndex() + return index, index != 0 +} + // applyNormalEntry decodes the raft envelope, optionally unwraps the // cipher payload, and dispatches to fsm.Apply. // @@ -3693,6 +3740,15 @@ func (e *Engine) protectedReceivedFSMSnapshotIndexesLocked() map[uint64]bool { // If fsmSnapDir is not set (e.g., engines created directly in unit tests), // falls back to the legacy in-memory []byte path. func (e *Engine) snapshotPayloadLocked(index uint64) ([]byte, error) { + // Config capture may have waited behind a newer snapshot publication. + // Do not serialize over a retained .fsm before CreateSnapshot rejects it. + current, err := e.storage.Snapshot() + if err != nil { + return nil, errors.WithStack(err) + } + if index <= current.GetMetadata().GetIndex() { + return nil, errors.WithStack(raftengine.ErrSnapshotDeferred) + } if e.fsmSnapDir == "" { snapshot, err := e.fsm.Snapshot() if err != nil { @@ -5460,6 +5516,13 @@ func (e *Engine) persistLocalSnapshot(req snapshotRequest) error { e.snapshotMu.Lock() defer e.snapshotMu.Unlock() + current, err := e.storage.Snapshot() + if err != nil { + return errors.Join(errors.WithStack(err), errors.WithStack(req.snapshot.Close())) + } + if req.index <= current.GetMetadata().GetIndex() { + return errors.WithStack(req.snapshot.Close()) + } if err := e.prepareFSMSnapshotWrite(req.index); err != nil { slog.Warn("failed to prepare fsm snapshot write", "index", req.index, diff --git a/internal/raftengine/etcd/external_origin_configuration_test.go b/internal/raftengine/etcd/external_origin_configuration_test.go new file mode 100644 index 000000000..5f5738c5d --- /dev/null +++ b/internal/raftengine/etcd/external_origin_configuration_test.go @@ -0,0 +1,192 @@ +package etcd + +import ( + "bytes" + "context" + "encoding/binary" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +const externalDestinationConfigurationIndex = uint64(42) + +func prepareExternalConfigurationFixture(t *testing.T, format string, bearing bool) string { + t.Helper() + st := restartOriginStore(t, format) + // Deliberately greater than the destination epoch: max(source, destination) + // and trusting any copied epoch must both fail this regression. + require.NoError(t, st.PutAt(t.Context(), store.MigrationConfigurationIndexKey(), binary.BigEndian.AppendUint64(nil, 9000), 10, 0)) + require.NoError(t, st.PutAt(t.Context(), []byte("ordinary"), []byte("value"), 20, 0)) + if indexed, ok := st.(interface{ SetDurableAppliedIndex(uint64) error }); ok { + require.NoError(t, indexed.SetDurableAppliedIndex(9001)) + } + if bearing { + writer, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(t.Context(), store.TargetStagedReadinessState{ + JobID: 7, MigrationJobID: 7, Armed: true, MinWriteTSExclusive: 100, + RouteStart: []byte("m"), RouteEnd: []byte("z"), + })) + } + var raw bytes.Buffer + var snapshot interface { + io.WriterTo + io.Closer + } + var err error + if format == "external-native" { + snapshot, err = st.Snapshot() + } else { + snapshot, err = kv.NewKvFSMWithHLC(st, nil).Snapshot() + } + require.NoError(t, err) + _, err = snapshot.WriteTo(&raw) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + input := filepath.Join(t.TempDir(), "input") + require.NoError(t, os.WriteFile(input, raw.Bytes(), 0o600)) + dir := filepath.Join(t.TempDir(), "raft") + peers := []Peer{{NodeID: 1, ID: "n1", Address: "127.0.0.1:12001"}} + if format == "external-native" { + _, err = PrepareExternalSnapshotRestore(ExternalSnapshotRestoreOptions{ + InputFSMPath: input, DataDir: dir, Index: externalDestinationConfigurationIndex, Term: 7, Peers: peers, + }) + } else { + _, err = PreparePhysicalSnapshotRestore(PhysicalSnapshotRestoreOptions{ + InputFSMPath: input, DataDir: dir, Index: externalDestinationConfigurationIndex, Term: 7, Peers: peers, + }) + } + require.NoError(t, err) + return dir +} + +func externalConfigurationOpen(ctx context.Context, dir string, fsm StateMachine) (*Engine, error) { + return Open(ctx, OpenConfig{NodeID: 1, LocalID: "n1", LocalAddress: "127.0.0.1:12001", DataDir: dir, + StateMachine: fsm, TickInterval: 10 * time.Millisecond, ElectionTick: 5, HeartbeatTick: 1}) +} + +func assertExternalConfigurationBound(t *testing.T, engine *Engine, st store.MVCCStore) { + t.Helper() + index, supported := engine.AppliedConfigurationIndex() + require.True(t, supported, "destination ConfState must establish its own durable epoch without another ConfChange") + require.Equal(t, externalDestinationConfigurationIndex, index) + durable, err := kv.MigrationConfigurationIndex(t.Context(), st) + require.NoError(t, err) + require.Equal(t, index, durable) + config, err := engine.Configuration(t.Context()) + require.NoError(t, err) + require.Len(t, config.Servers, 1) + require.Equal(t, "n1", config.Servers[0].ID) +} + +func TestExternalOriginDestinationConfigurationAndNativeRestart(t *testing.T) { + t.Setenv(snapshotEveryEnvVar, "10000") + for _, format := range []string{"mvcc", "native", "sst", "external-native"} { + t.Run(format, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) + defer cancel() + dir := prepareExternalConfigurationFixture(t, format, false) + for restart := range 3 { + st := restartOriginStore(t, format) + fsm := kv.NewKvFSMWithHLC(st, nil, kv.WithRouteHistory(nil, 1)) + engine, err := externalConfigurationOpen(ctx, dir, fsm) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, engine.Close()) }) + assertExternalConfigurationBound(t, engine, st) + if restart == 1 { + assertExternalConfigurationDoesNotAuthorizeSource(t, ctx, engine, st) + snapshot, snapErr := fsm.Snapshot() + require.NoError(t, snapErr) + index := engine.appliedIndex.Load() + require.Greater(t, index, externalDestinationConfigurationIndex) + require.NoError(t, engine.persistLocalSnapshot(snapshotRequest{index: index, snapshot: snapshot})) + } + if restart == 2 { + require.Greater(t, engine.Status().LastSnapshotIndex, externalDestinationConfigurationIndex, + "native snapshot data index must not replace the destination's configuration epoch") + require.ErrorIs(t, kv.CheckMigrationRequest(ctx, st, 1, &pb.Request{Ts: 200, + ObservedRouteVersion: kv.EncodeObservedRouteVersion(1), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("blocked")}}, + }), kv.ErrRouteCutoverPending) + } + require.NoError(t, engine.Close()) + } + }) + } +} + +func assertExternalConfigurationDoesNotAuthorizeSource(t *testing.T, ctx context.Context, engine *Engine, st store.MVCCStore) { + t.Helper() + control, err := kv.MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{ + JobId: 7, MigrationJobId: 7, Armed: true, TrackWrites: true, + RouteStart: []byte("m"), RouteEnd: []byte("z"), ExpectedCutoverVersion: 1, + MinWriteTsExclusive: 100, RetentionPinTs: 1, + }) + require.NoError(t, err) + result, err := engine.Propose(ctx, control) + require.NoError(t, err) + require.Nil(t, result.Response) + require.ErrorIs(t, kv.CheckMigrationRequest(ctx, st, 1, &pb.Request{Ts: 200, + ObservedRouteVersion: kv.EncodeObservedRouteVersion(1), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("blocked")}}, + }), kv.ErrRouteCutoverPending, "a configuration epoch alone is not SOURCE ownership evidence") + _, err = st.GetAt(ctx, []byte("n"), 200) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +type externalConfigurationFailureFSM struct { + StateMachine + indices []uint64 + failure error +} + +func (f *externalConfigurationFailureFSM) RestoreWithContext(ctx context.Context, r io.Reader, opts raftengine.SnapshotRestoreOptions) error { + restorer, ok := f.StateMachine.(raftengine.ContextSnapshotRestorer) + if !ok { + return errors.WithStack(ErrExternalSnapshotRestoreUnsupported) + } + return errors.WithStack(restorer.RestoreWithContext(ctx, r, opts)) +} + +func (f *externalConfigurationFailureFSM) RecordAppliedConfiguration(index uint64) error { + f.indices = append(f.indices, index) + return f.failure +} + +func (f *externalConfigurationFailureFSM) AppliedConfigurationIndex() uint64 { return 0 } + +func TestExternalOriginConfigurationFailureStopsOpen(t *testing.T) { + for _, bearing := range []bool{false, true} { + t.Run(map[bool]string{false: "configuration-write", true: "external-validation"}[bearing], func(t *testing.T) { + dir := prepareExternalConfigurationFixture(t, "native", bearing) + st := restartOriginStore(t, "native") + failure := errors.New("destination configuration write failed") + fsm := &externalConfigurationFailureFSM{StateMachine: kv.NewKvFSMWithHLC(st, nil), failure: failure} + engine, err := externalConfigurationOpen(t.Context(), dir, fsm) + if engine != nil { + t.Cleanup(func() { require.NoError(t, engine.Close()) }) + } + if bearing { + require.ErrorIs(t, err, store.ErrExternalSnapshotMigrationState) + require.Empty(t, fsm.indices, "failed external validation must not bind any destination epoch") + } else { + require.ErrorIs(t, err, failure) + require.Equal(t, []uint64{externalDestinationConfigurationIndex}, fsm.indices) + } + require.Nil(t, engine) + marker, readErr := readExternalRestoreOrigin(dir) + require.NoError(t, readErr) + require.False(t, marker.Completed) + }) + } +} diff --git a/internal/raftengine/etcd/external_origin_restart_test.go b/internal/raftengine/etcd/external_origin_restart_test.go new file mode 100644 index 000000000..96e9ac3d1 --- /dev/null +++ b/internal/raftengine/etcd/external_origin_restart_test.go @@ -0,0 +1,256 @@ +package etcd + +import ( + "bytes" + "encoding/binary" + "io" + "os" + "path/filepath" + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "go.etcd.io/etcd/server/v3/etcdserver/api/snap" + raftpb "go.etcd.io/raft/v3/raftpb" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" +) + +func restartOriginStore(t *testing.T, format string) store.MVCCStore { + t.Helper() + if format == "mvcc" { + return store.NewMVCCStore() + } + st, err := store.NewPebbleStore(t.TempDir(), store.WithSSTIngestSnapshots(format == "sst")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + return st +} + +func prepareRestartOrigin(t *testing.T, format string) string { + t.Helper() + st := restartOriginStore(t, format) + require.NoError(t, st.PutAt(t.Context(), store.MigrationConfigurationIndexKey(), binary.BigEndian.AppendUint64(nil, 11), 10, 0)) + require.NoError(t, st.PutAt(t.Context(), []byte("ordinary"), []byte("value"), 20, 0)) + snapshot, err := kv.NewKvFSMWithHLC(st, kv.NewHLC()).Snapshot() + require.NoError(t, err) + var raw bytes.Buffer + _, err = snapshot.WriteTo(&raw) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + input := filepath.Join(t.TempDir(), "original") + require.NoError(t, os.WriteFile(input, raw.Bytes(), 0o600)) + dir := filepath.Join(t.TempDir(), "raft") + _, err = PreparePhysicalSnapshotRestore(PhysicalSnapshotRestoreOptions{ + InputFSMPath: input, DataDir: dir, Index: 42, Term: 7, + Peers: []Peer{{NodeID: 1, ID: "n1", Address: "127.0.0.1:12001"}}, + }) + require.NoError(t, err) + return dir +} + +func assertRestartOriginEpochAbsent(t *testing.T, st store.MVCCStore) { + t.Helper() + _, err := st.GetAt(t.Context(), store.MigrationConfigurationIndexKey(), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound, "a full fallback must not reintroduce the imported epoch") + value, err := st.GetAt(t.Context(), []byte("ordinary"), 20) + require.NoError(t, err) + require.Equal(t, []byte("value"), value) +} + +func TestExternalOriginCrashBeforeAppliedCheckpointRevalidates(t *testing.T) { + for _, format := range []string{"mvcc", "native", "sst"} { + t.Run(format, func(t *testing.T) { + dir := prepareRestartOrigin(t, format) + original, err := readExternalRestoreOrigin(dir) + require.NoError(t, err) + for range 2 { + // A fresh destination has no durable applied index: force the + // full-restore fallback after the first successful validation. + st := restartOriginStore(t, format) + disk, err := loadOriginFixture(dir, kv.NewKvFSMWithHLC(st, kv.NewHLC())) + require.NoError(t, err) + require.NoError(t, closePersist(disk.Persist)) + assertRestartOriginEpochAbsent(t, st) + require.NoError(t, completeExternalRestoreOrigin(dir, disk.ExternalRestoreValidated)) + } + require.FileExists(t, filepath.Join(dir, externalRestoreOriginFile)) + completed, err := readExternalRestoreOrigin(dir) + require.NoError(t, err) + require.True(t, completed.Completed) + completed.Completed = false + require.Equal(t, original, completed, "completion cannot change the original identity") + }) + } +} + +func publishRestartNative(t *testing.T, dir string, disk *diskState, st store.MVCCStore) { + t.Helper() + require.NoError(t, st.PutAt(t.Context(), store.MigrationConfigurationIndexKey(), binary.BigEndian.AppendUint64(nil, 66), 30, 0)) + writer, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(t.Context(), store.TargetStagedReadinessState{ + JobID: 7, MigrationJobID: 7, Armed: true, MinWriteTSExclusive: 100, + RouteStart: []byte("a"), RouteEnd: []byte("z"), + })) + entry := raftpb.Entry{Index: proto.Uint64(43), Term: proto.Uint64(8), Type: raftpb.EntryNormal.Enum()} + require.NoError(t, disk.Persist.Save(&raftpb.HardState{Term: proto.Uint64(8), Commit: proto.Uint64(43)}, []*raftpb.Entry{&entry})) + require.NoError(t, disk.Storage.Append([]*raftpb.Entry{&entry})) + fsm := kv.NewKvFSMWithHLC(st, kv.NewHLC()) + snapshot, err := fsm.Snapshot() + require.NoError(t, err) + engine := &Engine{dataDir: dir, fsmSnapDir: filepath.Join(dir, fsmSnapDirName), + storage: disk.Storage, persist: disk.Persist, fsm: fsm} + require.NoError(t, engine.persistLocalSnapshot(snapshotRequest{index: 43, snapshot: snapshot})) +} + +func TestExternalOriginNativePublicationAndOriginalFallback(t *testing.T) { + for _, format := range []string{"mvcc", "native", "sst"} { + t.Run(format, func(t *testing.T) { + dir := prepareRestartOrigin(t, format) + st := restartOriginStore(t, format) + disk, err := loadOriginFixture(dir, kv.NewKvFSMWithHLC(st, kv.NewHLC())) + require.NoError(t, err) + require.NoError(t, completeExternalRestoreOrigin(dir, disk.ExternalRestoreValidated)) + publishRestartNative(t, dir, disk, st) + require.NoError(t, closePersist(disk.Persist)) + fresh := restartOriginStore(t, format) + disk, err = loadOriginFixture(dir, kv.NewKvFSMWithHLC(fresh, kv.NewHLC())) + require.NoError(t, err, "a WAL-published native snapshot must not be externally sanitized") + require.Equal(t, uint64(43), disk.LocalSnap.GetMetadata().GetIndex()) + require.NoError(t, closePersist(disk.Persist)) + value, err := fresh.GetAt(t.Context(), store.MigrationConfigurationIndexKey(), ^uint64(0)) + require.NoError(t, err) + require.Equal(t, binary.BigEndian.AppendUint64(nil, 66), value) + reader, ok := fresh.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(t.Context()) + require.NoError(t, err) + require.Len(t, states, 1) + newest := "0000000000000008-000000000000002b.snap" + require.NoError(t, os.Remove(filepath.Join(dir, snapDirName, newest))) + fallback := restartOriginStore(t, format) + disk, err = loadOriginFixture(dir, kv.NewKvFSMWithHLC(fallback, kv.NewHLC())) + require.NoError(t, err) + require.Equal(t, uint64(42), disk.LocalSnap.GetMetadata().GetIndex()) + require.NoError(t, closePersist(disk.Persist)) + assertRestartOriginEpochAbsent(t, fallback) + }) + } +} + +type originOwnedSnapshot struct { + wrote bool + closed bool +} + +func (s *originOwnedSnapshot) WriteTo(w io.Writer) (int64, error) { + s.wrote = true + return bytes.NewReader([]byte("must not overwrite the original snapshot")).WriteTo(w) +} + +func (s *originOwnedSnapshot) Close() error { s.closed = true; return nil } + +func TestExternalOriginDuplicateCaptureDoesNotOverwrite(t *testing.T) { + dir, result := prepareOriginFixture(t) + disk, err := loadOriginFixture(dir, &originRecordingFSM{}) + require.NoError(t, err) + defer func() { require.NoError(t, closePersist(disk.Persist)) }() + before, err := os.ReadFile(result.FSMPath) + require.NoError(t, err) + owned := &originOwnedSnapshot{} + engine := &Engine{dataDir: dir, fsmSnapDir: filepath.Join(dir, fsmSnapDirName), storage: disk.Storage, persist: disk.Persist} + require.NoError(t, engine.persistLocalSnapshot(snapshotRequest{index: 42, snapshot: owned})) + require.True(t, owned.closed) + require.False(t, owned.wrote, "duplicate must be rejected before cleanup or serialization") + after, err := os.ReadFile(result.FSMPath) + require.NoError(t, err) + require.Equal(t, before, after) + engine.fsm = &originRecordingFSM{} + engine.snapshotMu.Lock() + _, err = engine.snapshotPayloadLocked(42) + engine.snapshotMu.Unlock() + require.ErrorIs(t, err, raftengine.ErrSnapshotDeferred, "config capture must not overwrite the same retained file") + after, err = os.ReadFile(result.FSMPath) + require.NoError(t, err) + require.Equal(t, before, after) +} + +func TestExternalOriginNativeRequiresCompletion(t *testing.T) { + dir := prepareRestartOrigin(t, "mvcc") + st := restartOriginStore(t, "mvcc") + disk, err := loadOriginFixture(dir, kv.NewKvFSMWithHLC(st, kv.NewHLC())) + require.NoError(t, err) + // Deliberately manufacture publication before Open's durable completion. + // Production cannot do this: completion precedes transport and the run loop. + publishRestartNative(t, dir, disk, st) + require.NoError(t, closePersist(disk.Persist)) + probe := &originRecordingFSM{} + _, err = loadOriginFixture(dir, probe) + require.ErrorIs(t, err, ErrExternalSnapshotRestoreOrigin) + require.Empty(t, probe.origins) + require.Empty(t, probe.bodyBytes) +} + +func TestExternalOriginNativeRequiresWALSelection(t *testing.T) { + dir, _ := prepareOriginFixture(t) + disk, err := loadOriginFixture(dir, &originRecordingFSM{}) + require.NoError(t, err) + require.NoError(t, completeExternalRestoreOrigin(dir, disk.ExternalRestoreValidated)) + require.NoError(t, closePersist(disk.Persist)) + crc, _ := writeFSMFileForTest(t, filepath.Join(dir, fsmSnapDirName), 43, []byte("not published in WAL")) + unpublished := raftpb.Snapshot{Data: encodeSnapshotToken(43, crc), Metadata: testSnapshotMetadata(43, 8, []uint64{1})} + require.NoError(t, snap.New(zap.NewNop(), filepath.Join(dir, snapDirName)).SaveSnap(&unpublished)) + probe := &originRecordingFSM{} + disk, err = loadOriginFixture(dir, probe) + require.NoError(t, err) + require.NoError(t, closePersist(disk.Persist)) + require.Equal(t, uint64(42), disk.LocalSnap.GetMetadata().GetIndex()) + require.Equal(t, []raftengine.SnapshotRestoreOrigin{raftengine.SnapshotRestoreExternalRebase}, probe.origins) + _, _, err = restoreSnapshotWithOrigin(&originRecordingFSM{}, nil, 0, filepath.Join(dir, fsmSnapDirName), nil, nil) + require.ErrorIs(t, err, ErrExternalSnapshotRestoreOrigin) +} + +func TestExternalOriginNativeValidationBeforeSkip(t *testing.T) { + for _, corruption := range []string{"token-index", "token-crc", "payload", "unsupported-legacy"} { + t.Run(corruption, func(t *testing.T) { + dir := prepareRestartOrigin(t, "mvcc") + st := restartOriginStore(t, "mvcc") + disk, err := loadOriginFixture(dir, kv.NewKvFSMWithHLC(st, kv.NewHLC())) + require.NoError(t, err) + require.NoError(t, completeExternalRestoreOrigin(dir, disk.ExternalRestoreValidated)) + publishRestartNative(t, dir, disk, st) + selected, err := disk.Storage.Snapshot() + require.NoError(t, err) + require.NoError(t, closePersist(disk.Persist)) + token, err := decodeSnapshotToken(selected.Data) + require.NoError(t, err) + switch corruption { + case "token-index": + selected.Data = encodeSnapshotToken(44, token.CRC32C) + case "token-crc": + selected.Data = encodeSnapshotToken(43, token.CRC32C^1) + case "payload": + path := fsmSnapPath(filepath.Join(dir, fsmSnapDirName), 43) + data, readErr := os.ReadFile(path) + require.NoError(t, readErr) + data[0] ^= 1 + root, rootErr := os.OpenRoot(dir) + require.NoError(t, rootErr) + require.NoError(t, root.WriteFile("fsm-snap/000000000000002b.fsm", data, 0o600)) + require.NoError(t, root.Close()) + case "unsupported-legacy": + selected.Data = []byte("unrecognized embedded legacy snapshot") + } + require.NoError(t, snap.New(zap.NewNop(), filepath.Join(dir, snapDirName)).SaveSnap(selected)) + probe := &originRecordingFSM{skipGateFSM: skipGateFSM{applied: 100, appliedPresent: true}} + _, err = loadOriginFixture(dir, probe) + require.Error(t, err) + require.Empty(t, probe.origins) + require.Empty(t, probe.bodyBytes) + require.False(t, probe.restoredHeader, "validation must precede even the header-only skip") + }) + } +} diff --git a/internal/raftengine/etcd/external_restore_origin.go b/internal/raftengine/etcd/external_restore_origin.go new file mode 100644 index 000000000..b2b7a52b5 --- /dev/null +++ b/internal/raftengine/etcd/external_restore_origin.go @@ -0,0 +1,214 @@ +package etcd + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "hash/crc32" + "io" + "os" + "path/filepath" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/cockroachdb/errors" + raftpb "go.etcd.io/raft/v3/raftpb" +) + +const externalRestoreOriginFile = "external-restore-origin.json" +const externalRestoreOriginVersion = 1 +const externalRestoreOriginMaxBytes = 4096 +const externalRestoreOriginPerm = 0o600 + +var ErrExternalSnapshotRestoreOrigin = errors.New("external snapshot restore: invalid origin binding") +var ErrExternalSnapshotRestoreUnsupported = errors.New("external snapshot restore: FSM lacks origin-aware validation") + +type externalRestoreOrigin struct { + Version int `json:"version"` + Index uint64 `json:"index"` + Term uint64 `json:"term"` + CRC32C uint32 `json:"crc32c"` + // The entire FSM payload, including its header, excluding only the footer. + PayloadSHA256 string `json:"fsm_payload_sha256"` + Completed bool `json:"completed,omitempty"` +} + +func writeExternalRestoreOrigin(dir string, opts ExternalSnapshotRestoreOptions, crc uint32) error { + f, err := os.Open(fsmSnapPath(filepath.Join(dir, fsmSnapDirName), opts.Index)) + if err != nil { + return errors.WithStack(err) + } + defer f.Close() + digest, _, err := verifiedExternalRestorePayload(f, crc) + if err != nil { + return err + } + data, err := json.Marshal(externalRestoreOrigin{Version: externalRestoreOriginVersion, + Index: opts.Index, Term: opts.Term, CRC32C: crc, PayloadSHA256: digest}) + if err != nil { + return errors.WithStack(err) + } + marker, err := os.OpenFile(filepath.Join(dir, externalRestoreOriginFile), os.O_WRONLY|os.O_CREATE|os.O_EXCL, externalRestoreOriginPerm) + if err != nil { + return errors.WithStack(err) + } + _, writeErr := marker.Write(data) + syncErr := marker.Sync() + closeErr := marker.Close() + if err := errors.Join(writeErr, syncErr, closeErr); err != nil { + return errors.WithStack(err) + } + return syncDir(dir) +} + +func readExternalRestoreOrigin(dir string) (*externalRestoreOrigin, error) { + f, err := openRegularExternalFSMInput(filepath.Join(dir, externalRestoreOriginFile)) + if os.IsNotExist(errors.UnwrapAll(err)) { + return nil, nil + } + if err != nil { + return nil, err + } + defer f.Close() + data, err := io.ReadAll(io.LimitReader(f, externalRestoreOriginMaxBytes+1)) + if err != nil { + return nil, errors.WithStack(err) + } + return decodeExternalRestoreOrigin(data) +} + +func decodeExternalRestoreOrigin(data []byte) (*externalRestoreOrigin, error) { + var marker externalRestoreOrigin + if len(data) > externalRestoreOriginMaxBytes { + return nil, errors.WithStack(ErrExternalSnapshotRestoreOrigin) + } + if err := json.Unmarshal(data, &marker); err != nil { + return nil, errors.Wrap(ErrExternalSnapshotRestoreOrigin, err.Error()) + } + digest, err := hex.DecodeString(marker.PayloadSHA256) + if err != nil || len(digest) != sha256.Size || marker.Version != externalRestoreOriginVersion || marker.Index == 0 || marker.Term == 0 { + return nil, errors.WithStack(ErrExternalSnapshotRestoreOrigin) + } + return &marker, nil +} + +func verifiedExternalRestorePayload(f *os.File, tokenCRC uint32) (string, int64, error) { + info, err := f.Stat() + if err != nil { + return "", 0, errors.WithStack(err) + } + footer, err := verifyFSMSnapshotPrefix(f, info.Size(), f.Name(), tokenCRC) + if err != nil { + return "", 0, err + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return "", 0, errors.WithStack(err) + } + payloadSize := info.Size() - fsmFooterSize + digest, crc := sha256.New(), crc32.New(crc32cTable) + n, err := io.Copy(io.MultiWriter(digest, crc), io.LimitReader(f, payloadSize)) + if err != nil { + return "", 0, errors.WithStack(err) + } + if n != payloadSize || crc.Sum32() != footer { + return "", 0, errors.WithStack(ErrFSMSnapshotFileCRC) + } + return hex.EncodeToString(digest.Sum(nil)), payloadSize, nil +} + +func restoreWithExternalOrigin(fsm StateMachine, snapshot raftpb.Snapshot, dir string, marker *externalRestoreOrigin) (uint64, error) { + token, err := decodeSnapshotToken(snapshot.Data) + if err != nil { + return 0, err + } + if !marker.matchesSnapshot(snapshot, token) { + return 0, errors.WithStack(ErrExternalSnapshotRestoreOrigin) + } + f, err := os.Open(fsmSnapPath(dir, token.Index)) + if err != nil { + return 0, statFSMFileError(err) + } + defer f.Close() + digest, size, err := verifiedExternalRestorePayload(f, token.CRC32C) + if err != nil { + return 0, err + } + if digest != marker.PayloadSHA256 { + return 0, errors.WithStack(ErrExternalSnapshotRestoreOrigin) + } + restorer, ok := fsm.(raftengine.ContextSnapshotRestorer) + if !ok { + return 0, errors.WithStack(ErrExternalSnapshotRestoreUnsupported) + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return 0, errors.WithStack(err) + } + if err := restorer.RestoreWithContext(context.Background(), io.LimitReader(f, size), raftengine.SnapshotRestoreOptions{ + Origin: raftengine.SnapshotRestoreExternalRebase, + }); err != nil { + return 0, errors.Wrap(err, "validate and restore external FSM snapshot") + } + return marker.Index, nil +} + +func (m externalRestoreOrigin) matchesSnapshot(snapshot raftpb.Snapshot, token snapshotToken) bool { + return m.Index == snapshot.GetMetadata().GetIndex() && m.Term == snapshot.GetMetadata().GetTerm() && + token.Index == m.Index && token.CRC32C == m.CRC32C +} + +// Completion permits later WAL-published native snapshots, never ordinary +// restore of the original bytes. Keep the immutable binding for every fallback. +// Open persists this before starting transport or the snapshot/apply workers. +func completeExternalRestoreOrigin(dir string, validated bool) error { + return completeExternalRestoreOriginWithPersistence(dir, validated, writeAndSyncFile, syncDir) +} + +// Dependencies are explicit so tests can reproduce a visible rename whose +// directory flush failed, without process-wide filesystem hooks. +func completeExternalRestoreOriginWithPersistence(dir string, validated bool, + writeCompletion func(string, []byte) error, syncCompletion func(string) error, +) error { + if !validated { + return nil + } + marker, err := readExternalRestoreOrigin(dir) + if err != nil { + return err + } + if marker == nil { + return errors.WithStack(ErrExternalSnapshotRestoreOrigin) + } + if marker.Completed { + // A previous rename may be visible despite a failed directory fsync. + return errors.WithStack(syncCompletion(dir)) + } + marker.Completed = true + data, err := json.Marshal(marker) + if err != nil { + return errors.WithStack(err) + } + return errors.WithStack(writeCompletion(filepath.Join(dir, externalRestoreOriginFile), data)) +} + +// This decision requires the real WAL selection and successful ReadAll, not an +// arbitrary snapshot whose index merely exceeds the imported snapshot's index. +// Preparation seeds a fresh directory with only the original snapshot; Open +// durably completes validation before local/config capture or received Ready +// can publish any later snapshot. The selected WAL boundary is their provenance. +func validateNativeAfterExternalRestore(selected *walRestoreState, marker *externalRestoreOrigin, dir string) error { + snapshot := selected.snapshot + token, err := decodeSnapshotToken(snapshot.Data) + if err != nil { + return errors.Wrap(ErrExternalSnapshotRestoreOrigin, "later native snapshot requires a supported FSM token") + } + meta := snapshot.GetMetadata() + if !marker.Completed || meta.GetIndex() <= marker.Index || meta.GetTerm() < marker.Term || + token.Index != meta.GetIndex() || selected.hardState.GetCommit() < meta.GetIndex() || + selected.hardState.GetTerm() < meta.GetTerm() { + return errors.WithStack(ErrExternalSnapshotRestoreOrigin) + } + // Validate the entire file even when the ordinary cold-start gate would + // use only its header. Neither copied epoch rows nor an applied index are + // evidence that an unvalidated snapshot belongs to this Raft lineage. + return verifyFSMSnapshotFile(fsmSnapPath(dir, token.Index), token.CRC32C) +} diff --git a/internal/raftengine/etcd/external_restore_origin_test.go b/internal/raftengine/etcd/external_restore_origin_test.go new file mode 100644 index 000000000..d84aa3c72 --- /dev/null +++ b/internal/raftengine/etcd/external_restore_origin_test.go @@ -0,0 +1,269 @@ +package etcd + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "hash/crc32" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" + raftpb "go.etcd.io/raft/v3/raftpb" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" +) + +type originRecordingFSM struct { + skipGateFSM + origins []raftengine.SnapshotRestoreOrigin + reject error +} + +func (f *originRecordingFSM) RestoreWithContext(_ context.Context, r io.Reader, opts raftengine.SnapshotRestoreOptions) error { + f.origins = append(f.origins, opts.Origin) + if f.reject != nil { + return f.reject + } + return f.Restore(r) +} + +func prepareOriginFixture(t *testing.T) (string, *ExternalSnapshotRestoreResult) { + t.Helper() + root := t.TempDir() + input := filepath.Join(root, "payload") + require.NoError(t, os.WriteFile(input, []byte("EKVTHLC1proof-free-payload"), 0o600)) + dir := filepath.Join(root, "raft") + result, err := PreparePhysicalSnapshotRestore(PhysicalSnapshotRestoreOptions{ + InputFSMPath: input, DataDir: dir, Index: 42, Term: 7, + Peers: []Peer{{NodeID: 1, ID: "n1", Address: "127.0.0.1:12001"}}, + }) + require.NoError(t, err) + return dir, result +} + +func loadOriginFixture(dir string, fsm StateMachine) (*diskState, error) { + return loadWalState(zap.NewNop(), filepath.Join(dir, walDirName), filepath.Join(dir, snapDirName), + filepath.Join(dir, fsmSnapDirName), fsm, nil) +} + +func TestExternalOriginForcesValidatedFullRestore(t *testing.T) { + dir, _ := prepareOriginFixture(t) + fsm := &originRecordingFSM{skipGateFSM: skipGateFSM{applied: 100, appliedPresent: true}} + disk, err := loadOriginFixture(dir, fsm) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, closePersist(disk.Persist)) }) + require.Equal(t, []raftengine.SnapshotRestoreOrigin{raftengine.SnapshotRestoreExternalRebase}, fsm.origins) + require.Equal(t, uint64(42), disk.EffectiveApplied) + require.Equal(t, []byte("EKVTHLC1proof-free-payload"), fsm.bodyBytes) + require.False(t, fsm.restoredHeader, "external restore must not use the header-only skip") + require.FileExists(t, filepath.Join(dir, externalRestoreOriginFile), "full restore alone cannot consume the marker") +} + +func TestExternalOriginRejectsUnsupportedFSM(t *testing.T) { + dir, _ := prepareOriginFixture(t) + fsm := &skipGateFSM{} + disk, err := loadOriginFixture(dir, fsm) + if disk != nil { + t.Cleanup(func() { require.NoError(t, closePersist(disk.Persist)) }) + } + require.ErrorIs(t, err, ErrExternalSnapshotRestoreUnsupported) + require.Empty(t, fsm.bodyBytes) +} + +func TestExternalOriginBindsWrappedHeaderAndPreservesInputHash(t *testing.T) { + root := t.TempDir() + payload := []byte("EKVPBBL1ordinary-kv") + input := filepath.Join(root, "input") + require.NoError(t, os.WriteFile(input, payload, 0o600)) + dir := filepath.Join(root, "raft") + result, err := PrepareExternalSnapshotRestore(ExternalSnapshotRestoreOptions{ + InputFSMPath: input, DataDir: dir, Index: 42, Term: 7, SnapshotCeilingMs: 123, + Peers: []Peer{{NodeID: 1, ID: "n1", Address: "127.0.0.1:12001"}}, + }) + require.NoError(t, err) + marker, err := readExternalRestoreOrigin(dir) + require.NoError(t, err) + require.NotNil(t, marker) + bytes, err := os.ReadFile(result.FSMPath) + require.NoError(t, err) + whole := sha256.Sum256(bytes[:len(bytes)-fsmFooterSize]) + original := sha256.Sum256(payload) + require.Equal(t, hex.EncodeToString(whole[:]), marker.PayloadSHA256) + require.Equal(t, hex.EncodeToString(original[:]), result.PayloadSHA256) + require.NotEqual(t, result.PayloadSHA256, marker.PayloadSHA256) +} + +func TestExternalOriginRejectsMarkerMismatchBeforeFSM(t *testing.T) { + for _, completed := range []bool{false, true} { + for _, field := range []string{"version", "index", "term", "crc", "sha", "truncated", "oversized"} { + t.Run(field+map[bool]string{false: "/pending", true: "/completed"}[completed], func(t *testing.T) { + dir, _ := prepareOriginFixture(t) + marker, err := readExternalRestoreOrigin(dir) + require.NoError(t, err) + marker.Completed = completed + switch field { + case "version": + marker.Version++ + case "index": + marker.Index++ + case "term": + marker.Term++ + case "crc": + marker.CRC32C++ + case "sha": + marker.PayloadSHA256 = hex.EncodeToString(make([]byte, sha256.Size)) + } + data, err := json.Marshal(marker) + require.NoError(t, err) + if field == "truncated" { + data = []byte("{") + } + if field == "oversized" { + data = make([]byte, externalRestoreOriginMaxBytes+1) + } + require.NoError(t, os.WriteFile(filepath.Join(dir, externalRestoreOriginFile), data, 0o600)) + fsm := &originRecordingFSM{} + _, err = loadOriginFixture(dir, fsm) + require.ErrorIs(t, err, ErrExternalSnapshotRestoreOrigin) + require.Empty(t, fsm.origins) + require.FileExists(t, filepath.Join(dir, externalRestoreOriginFile)) + }) + } + } +} + +func TestExternalOriginVerifiesCRCAndSHAOnSamePayload(t *testing.T) { + for _, repairCRC := range []bool{false, true} { + t.Run(map[bool]string{false: "crc", true: "sha"}[repairCRC], func(t *testing.T) { + dir, result := prepareOriginFixture(t) + marker, err := readExternalRestoreOrigin(dir) + require.NoError(t, err) + data, err := os.ReadFile(result.FSMPath) + require.NoError(t, err) + data[0] ^= 1 + if repairCRC { + marker.CRC32C = crc32.Checksum(data[:len(data)-fsmFooterSize], crc32cTable) + binary.BigEndian.PutUint32(data[len(data)-fsmFooterSize:], marker.CRC32C) + } + root, err := os.OpenRoot(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, root.Close()) }) + require.NoError(t, root.WriteFile("fsm-snap/000000000000002a.fsm", data, 0o600)) + snapshot := raftpb.Snapshot{Data: encodeSnapshotToken(marker.Index, marker.CRC32C), + Metadata: &raftpb.SnapshotMetadata{Index: proto.Uint64(marker.Index), Term: proto.Uint64(marker.Term)}} + fsm := &originRecordingFSM{} + _, err = restoreWithExternalOrigin(fsm, snapshot, filepath.Join(dir, fsmSnapDirName), marker) + if repairCRC { + require.ErrorIs(t, err, ErrExternalSnapshotRestoreOrigin) + } else { + require.ErrorIs(t, err, ErrFSMSnapshotFileCRC) + } + require.Empty(t, fsm.origins) + }) + } +} + +func TestExternalOriginValidationFailureSurvivesRetry(t *testing.T) { + dir, _ := prepareOriginFixture(t) + rejected := errors.New("decoded migration-bearing restore unsupported") + for range 2 { + fsm := &originRecordingFSM{skipGateFSM: skipGateFSM{applied: 100, appliedPresent: true}, reject: rejected} + _, err := loadOriginFixture(dir, fsm) + require.ErrorIs(t, err, rejected) + require.Len(t, fsm.origins, 1) + require.FileExists(t, filepath.Join(dir, externalRestoreOriginFile)) + } +} + +func TestExternalOriginMarkerlessUsesOrdinarySkip(t *testing.T) { + dir, _ := prepareOriginFixture(t) + require.NoError(t, os.Remove(filepath.Join(dir, externalRestoreOriginFile))) + fsm := &originRecordingFSM{skipGateFSM: skipGateFSM{applied: 100, appliedPresent: true}} + disk, err := loadOriginFixture(dir, fsm) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, closePersist(disk.Persist)) }) + require.Empty(t, fsm.origins) + require.True(t, fsm.restoredHeader) + require.Equal(t, uint64(100), disk.EffectiveApplied) +} + +// Embedding the old interface strips the optional capability. Do not unwrap it +// behind a wrapper's back or silently call the legacy Restore method. +type originStrippingFSM struct{ StateMachine } + +func TestExternalOriginWrapperMustForwardCapability(t *testing.T) { + dir, _ := prepareOriginFixture(t) + inner := &originRecordingFSM{} + _, err := loadOriginFixture(dir, &originStrippingFSM{StateMachine: inner}) + require.ErrorIs(t, err, ErrExternalSnapshotRestoreUnsupported) + require.Empty(t, inner.origins) +} + +func TestExternalOriginSuccessfulOpenRetainsCompletedBinding(t *testing.T) { + dir, _ := prepareOriginFixture(t) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + fsm := &originRecordingFSM{} + engine, err := Open(ctx, OpenConfig{NodeID: 1, LocalID: "n1", LocalAddress: "127.0.0.1:12001", DataDir: dir, + StateMachine: fsm, TickInterval: 10 * time.Millisecond, ElectionTick: 5, HeartbeatTick: 1}) + require.NoError(t, err) + marker, err := readExternalRestoreOrigin(dir) + require.NoError(t, err) + require.NotNil(t, marker) + require.True(t, marker.Completed) + require.NoError(t, engine.Close()) + require.Equal(t, []raftengine.SnapshotRestoreOrigin{raftengine.SnapshotRestoreExternalRebase}, fsm.origins) + // Completion never lets the original bytes bypass origin validation. + ordinary := &skipGateFSM{applied: 42, appliedPresent: true} + _, err = loadOriginFixture(dir, ordinary) + require.ErrorIs(t, err, ErrExternalSnapshotRestoreUnsupported) +} + +func TestExternalOriginMissingWALCannotBootstrap(t *testing.T) { + dir, _ := prepareOriginFixture(t) + require.NoError(t, os.RemoveAll(filepath.Join(dir, walDirName))) + _, err := openDiskState(OpenConfig{DataDir: dir, StateMachine: &originRecordingFSM{}, Bootstrap: true}, nil) + require.ErrorIs(t, err, ErrExternalSnapshotRestoreOrigin) +} + +func TestExternalOriginCompletionRetriesDirectoryFlush(t *testing.T) { + dir, _ := prepareOriginFixture(t) + failure := errors.New("completion directory flush failed after rename") + flushes, writes := 0, 0 + flush := func(path string) error { + require.Equal(t, dir, path) + flushes++ + if flushes < 3 { + return failure + } + return syncDir(path) + } + write := func(path string, data []byte) error { + writes++ + // Publish real, valid completed bytes atomically, then inject the + // post-rename error returned by the completion writer's contract. + require.NoError(t, writeAndSyncFile(path, data)) + return flush(filepath.Dir(path)) + } + for attempt := range 3 { + err := completeExternalRestoreOriginWithPersistence(dir, true, write, flush) + if attempt < 2 { + require.ErrorIs(t, err, failure, "Open must remain gated while completion durability is uncertain") + } else { + require.NoError(t, err) + } + marker, readErr := readExternalRestoreOrigin(dir) + require.NoError(t, readErr) + require.True(t, marker.Completed, "a visible completed flag is not proof that its directory flush succeeded") + } + require.Equal(t, 1, writes, "completed retries must not rewrite the immutable binding") + require.Equal(t, 3, flushes, "every retry must re-establish the completion durability barrier") +} diff --git a/internal/raftengine/etcd/external_snapshot_restore.go b/internal/raftengine/etcd/external_snapshot_restore.go index dc3497594..9d070a822 100644 --- a/internal/raftengine/etcd/external_snapshot_restore.go +++ b/internal/raftengine/etcd/external_snapshot_restore.go @@ -132,7 +132,10 @@ func prepareExternalSnapshotRestore( "copied payload has %s, expected %s", payloadSHA, opts.ExpectedPayloadSHA256) } token := encodeSnapshotToken(opts.Index, crc) - if err := seedExternalSnapshotRestoreDir(ctx, tempDir, opts, token); err != nil { + if err := ctx.Err(); err != nil { + return nil, errors.WithStack(err) + } + if err := seedExternalSnapshotRestoreWithOrigin(ctx, tempDir, opts, token, crc); err != nil { return nil, err } if err := finalizeExternalSnapshotRestoreDir(ctx, tempDir, destDataDir); err != nil { @@ -151,6 +154,13 @@ func prepareExternalSnapshotRestore( }, nil } +func seedExternalSnapshotRestoreWithOrigin(ctx context.Context, tempDir string, opts ExternalSnapshotRestoreOptions, token []byte, crc uint32) error { + if err := writeExternalRestoreOrigin(tempDir, opts, crc); err != nil { + return err + } + return seedExternalSnapshotRestoreDir(ctx, tempDir, opts, token) +} + func finalizeExternalSnapshotRestoreDir(ctx context.Context, tempDir string, destDataDir string) error { if err := ctx.Err(); err != nil { return errors.WithStack(err) diff --git a/internal/raftengine/etcd/migration_configuration_test.go b/internal/raftengine/etcd/migration_configuration_test.go new file mode 100644 index 000000000..aba647934 --- /dev/null +++ b/internal/raftengine/etcd/migration_configuration_test.go @@ -0,0 +1,68 @@ +package etcd + +import ( + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestMigrationCleanupConfChangeInterleavingAndRestart(t *testing.T) { + t.Setenv(snapshotEveryEnvVar, "1") + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) + defer cancel() + dir := t.TempDir() + st := store.NewMVCCStore() + require.NoError(t, st.PutAt(ctx, []byte("n"), []byte("customer"), 100, 0)) + fsm := kv.NewKvFSMWithHLC(st, nil) + engine, err := Open(ctx, OpenConfig{NodeID: 1, LocalID: "n1", LocalAddress: "127.0.0.1:7001", + DataDir: dir, Bootstrap: true, StateMachine: fsm, TickInterval: 10 * time.Millisecond, ElectionTick: 5, HeartbeatTick: 1}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, engine.Close()) }) + require.Eventually(t, func() bool { return engine.State() == raftengine.StateLeader }, 3*time.Second, 10*time.Millisecond) + aware, ok := fsm.(raftengine.AppliedConfigurationAware) + require.True(t, ok) + require.NotZero(t, aware.AppliedConfigurationIndex(), "bootstrap ConfChange must install the durable epoch") + control, err := kv.MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{JobId: 7, MigrationJobId: 7, + RouteStart: []byte("m"), RouteEnd: []byte("z"), Armed: true, SourceWriteFence: true, SourceReadFence: true, + MinWriteTsExclusive: 100, RetentionPinTs: 1}) + require.NoError(t, err) + _, err = engine.Propose(ctx, control) + require.NoError(t, err) + oldIndex := aware.AppliedConfigurationIndex() + cleanup, err := kv.MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{JobId: 7, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA, RequireConfigurationIndex: true, ExpectedConfigurationIndex: oldIndex}) + require.NoError(t, err) + index, err := engine.AddLearner(ctx, "n2", "127.0.0.1:7002", 0) + require.NoError(t, err) + require.Greater(t, index, oldIndex) + require.Equal(t, index, aware.AppliedConfigurationIndex()) + result, err := engine.Propose(ctx, cleanup) + require.NoError(t, err) + responseErr, ok := result.Response.(error) + require.True(t, ok) + require.ErrorIs(t, responseErr, kv.ErrMigrationConfigurationChanged) + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Len(t, states, 1, "a committed membership change invalidates the old cleanup before guard removal") + require.Equal(t, raftengine.StateLeader, engine.State(), "configuration mismatch must not halt raft") + require.Eventually(t, func() bool { return engine.Status().LastSnapshotIndex > index }, 3*time.Second, 10*time.Millisecond) + require.NoError(t, engine.Close()) + restored := store.NewMVCCStore() + restoredFSM := kv.NewKvFSMWithHLC(restored, nil) + reopened, err := Open(ctx, OpenConfig{NodeID: 1, LocalID: "n1", LocalAddress: "127.0.0.1:7001", + DataDir: dir, StateMachine: restoredFSM, TickInterval: 10 * time.Millisecond, ElectionTick: 5, HeartbeatTick: 1}) + require.NoError(t, err) + defer reopened.Close() + restoredAware, ok := restoredFSM.(raftengine.AppliedConfigurationAware) + require.True(t, ok) + require.Equal(t, index, restoredAware.AppliedConfigurationIndex(), + "snapshot data index must not replace the actual ConfChange index") +} diff --git a/internal/raftengine/etcd/namespace_configuration.go b/internal/raftengine/etcd/namespace_configuration.go new file mode 100644 index 000000000..e660e8fc9 --- /dev/null +++ b/internal/raftengine/etcd/namespace_configuration.go @@ -0,0 +1,34 @@ +package etcd + +import ( + "slices" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/cockroachdb/errors" + raftpb "go.etcd.io/raft/v3/raftpb" +) + +func namespaceApplyMembers(state raftpb.ConfState) []uint64 { + ids := append(slices.Clone(state.GetVoters()), state.GetVotersOutgoing()...) + ids = append(ids, state.GetLearners()...) + ids = append(ids, state.GetLearnersNext()...) + slices.Sort(ids) + return slices.Compact(ids) +} + +func recordNamespaceConfiguration(fsm StateMachine, index uint64, state raftpb.ConfState) error { + if aware, ok := fsm.(raftengine.NamespaceConfigurationAware); ok { + return errors.WithStack(aware.RecordAppliedNamespaceConfiguration(index, namespaceApplyMembers(state))) + } + if aware, ok := fsm.(raftengine.AppliedConfigurationAware); ok { + return errors.WithStack(aware.RecordAppliedConfiguration(index)) + } + return nil +} + +func recoverNamespaceConfiguration(fsm StateMachine, snapshot raftpb.Snapshot) error { + if aware, ok := fsm.(raftengine.NamespaceConfigurationAware); ok { + return errors.WithStack(aware.RecoverNamespaceConfiguration(snapshot.GetMetadata().GetIndex(), namespaceApplyMembers(confStateValue(snapshot.GetMetadata().GetConfState())))) + } + return nil +} diff --git a/internal/raftengine/etcd/namespace_configuration_test.go b/internal/raftengine/etcd/namespace_configuration_test.go new file mode 100644 index 000000000..be2feaf1a --- /dev/null +++ b/internal/raftengine/etcd/namespace_configuration_test.go @@ -0,0 +1,61 @@ +package etcd + +import ( + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + raftpb "go.etcd.io/raft/v3/raftpb" +) + +func TestNamespaceApplyMembersIncludesJointAndLearnerMembers(t *testing.T) { + state := raftpb.ConfState{Voters: []uint64{1, 2}, VotersOutgoing: []uint64{2, 3}, Learners: []uint64{4}, LearnersNext: []uint64{5}} + require.Equal(t, []uint64{1, 2, 3, 4, 5}, namespaceApplyMembers(state)) +} + +func TestNamespaceActivationRealRaftConfChangeSnapshotRestart(t *testing.T) { + t.Setenv(snapshotEveryEnvVar, "1") + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) + defer cancel() + dir := t.TempDir() + st := store.NewMVCCStore() + fsm := kv.NewKvFSMWithHLC(st, nil) + cfg := OpenConfig{NodeID: 1, LocalID: "n1", LocalAddress: "127.0.0.1:7001", DataDir: dir, Bootstrap: true, + StateMachine: fsm, TickInterval: 10 * time.Millisecond, ElectionTick: 5, HeartbeatTick: 1} + engine, err := Open(ctx, cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, engine.Close()) }) + require.Eventually(t, func() bool { return engine.State() == raftengine.StateLeader }, 3*time.Second, 10*time.Millisecond) + index, err := kv.MigrationConfigurationIndex(ctx, st) + require.NoError(t, err) + capability := kv.NamespaceMemberCapability{MemberID: 1, Version: kv.MigrationNamespaceVersion, ConfigurationIndex: index} + command := kv.NamespaceAdmissionCommand{Version: kv.MigrationNamespaceVersion, ConfigurationIndex: index, + ApplyMembers: []uint64{1}, Members: []kv.NamespaceMemberCapability{capability}} + require.NoError(t, kv.ActivateMigrationNamespace(ctx, st, engine, command)) + joiningID := DeriveNodeID("n2") + require.ErrorIs(t, kv.CheckMigrationNamespacePrejoin(ctx, st, index, joiningID, kv.MigrationNamespaceVersion), kv.ErrMigrationNamespaceProtocol) + command.Joining = []kv.NamespaceMemberCapability{{MemberID: joiningID, Version: kv.MigrationNamespaceVersion, ConfigurationIndex: index}} + require.NoError(t, kv.ActivateMigrationNamespace(ctx, st, engine, command)) + require.NoError(t, kv.CheckMigrationNamespacePrejoin(ctx, st, index, joiningID, kv.MigrationNamespaceVersion)) + require.ErrorIs(t, kv.CheckMigrationNamespacePrejoin(ctx, st, index, joiningID, 0), kv.ErrMigrationNamespaceProtocol) + joinedAt, err := engine.AddLearner(ctx, "n2", "127.0.0.1:7002", 0) + require.NoError(t, err) + require.Greater(t, joinedAt, index) + require.NoError(t, kv.RequireMigrationNamespaceAdmission(ctx, st)) + require.Eventually(t, func() bool { return engine.Status().LastSnapshotIndex >= joinedAt }, 3*time.Second, 10*time.Millisecond) + require.NoError(t, engine.Close()) + restored := store.NewMVCCStore() + restoredFSM := kv.NewKvFSMWithHLC(restored, nil) + cfg.StateMachine, cfg.Bootstrap = restoredFSM, false + reopened, err := Open(ctx, cfg) + require.NoError(t, err) + defer reopened.Close() + require.NoError(t, kv.RequireMigrationNamespaceAdmission(ctx, restored)) + actual, err := kv.MigrationConfigurationIndex(ctx, restored) + require.NoError(t, err) + require.Equal(t, joinedAt, actual) +} diff --git a/internal/raftengine/etcd/wal_store.go b/internal/raftengine/etcd/wal_store.go index 876cc6a1e..1551b11d1 100644 --- a/internal/raftengine/etcd/wal_store.go +++ b/internal/raftengine/etcd/wal_store.go @@ -34,7 +34,8 @@ type diskState struct { // already represented in the FSM. Zero when no override applies // (legacy path / execute-restore path); the engine then falls // back to maxAppliedIndex(LocalSnap). Codex P1 #934. - EffectiveApplied uint64 + EffectiveApplied uint64 + ExternalRestoreValidated bool } func openDiskState(cfg OpenConfig, peers []Peer) (*diskState, error) { @@ -42,6 +43,13 @@ func openDiskState(cfg OpenConfig, peers []Peer) (*diskState, error) { walDir := filepath.Join(cfg.DataDir, walDirName) snapDir := filepath.Join(cfg.DataDir, snapDirName) fsmSnapDir := filepath.Join(cfg.DataDir, fsmSnapDirName) + marker, err := readExternalRestoreOrigin(cfg.DataDir) + if err != nil { + return nil, err + } + if marker != nil && !wal.Exist(walDir) { + return nil, errors.Wrap(ErrExternalSnapshotRestoreOrigin, "external restore requires its prepared WAL") + } if err := prepareDataDirs(cfg.DataDir, snapDir, fsmSnapDir); err != nil { return nil, err @@ -107,6 +115,9 @@ func joinWalState(logger *zap.Logger, walDir, snapDir, fsmSnapDir string) (*disk func bootstrapWalState(logger *zap.Logger, walDir, snapDir, fsmSnapDir string, fsm StateMachine, peers []Peer) (*diskState, error) { // Bootstrap snapshot always has index 1. const bootstrapIndex = uint64(1) + if err := recordNamespaceConfiguration(fsm, bootstrapIndex, confStateForPeers(peers)); err != nil { + return nil, errors.WithStack(err) + } snapshot, err := fsm.Snapshot() if err != nil { return nil, errors.WithStack(err) @@ -124,7 +135,17 @@ func bootstrapWalState(logger *zap.Logger, walDir, snapDir, fsmSnapDir string, f return persistBootState(logger, walDir, snapDir, fsmSnapDir, fsm, boot) } -func loadWalState(logger *zap.Logger, walDir, snapDir, fsmSnapDir string, fsm StateMachine, obs raftengine.ColdStartObserver) (*diskState, error) { +// walRestoreState is constructed only by readWALRestoreState, which selects a +// WAL-valid snapshot and successfully reads the WAL from that exact boundary. +// Its unexported type carries that provenance into the origin-aware restorer. +type walRestoreState struct { + snapshot raftpb.Snapshot + w *wal.WAL + hardState raftpb.HardState + entries []raftpb.Entry +} + +func readWALRestoreState(logger *zap.Logger, walDir, snapDir string) (*walRestoreState, error) { // Scope the repair retry tightly to WAL-only reads: both // loadPersistedSnapshot (scans WAL via wal.ValidSnapshotEntries) // and openAndReadWAL's ReadAll can surface io.ErrUnexpectedEOF @@ -153,10 +174,18 @@ func loadWalState(logger *zap.Logger, walDir, snapDir, fsmSnapDir string, fsm St if err != nil { return nil, err } - committedReplayTarget := coldStartReplayTarget(snapshot, hardState) - effectiveApplied, err := restoreSnapshotState(fsm, snapshot, committedReplayTarget, fsmSnapDir, obs, logger) + return &walRestoreState{snapshot: snapshot, w: w, hardState: hardState, entries: entries}, nil +} + +func loadWalState(logger *zap.Logger, walDir, snapDir, fsmSnapDir string, fsm StateMachine, obs raftengine.ColdStartObserver) (*diskState, error) { + selected, err := readWALRestoreState(logger, walDir, snapDir) if err != nil { - if closeErr := w.Close(); closeErr != nil { + return nil, err + } + committedReplayTarget := coldStartReplayTarget(selected.snapshot, selected.hardState) + effectiveApplied, externalValidated, err := restoreSnapshotWithOrigin(fsm, selected, committedReplayTarget, fsmSnapDir, obs, logger) + if err != nil { + if closeErr := selected.w.Close(); closeErr != nil { logger.Warn("WAL close failed after restoreSnapshotState error", zap.String("dir", walDir), zap.Error(closeErr), @@ -166,12 +195,12 @@ func loadWalState(logger *zap.Logger, walDir, snapDir, fsmSnapDir string, fsm St } storage, err := newMemoryStorage(persistedState{ - HardState: hardState, - Snapshot: snapshot, - Entries: entries, + HardState: selected.hardState, + Snapshot: selected.snapshot, + Entries: selected.entries, }) if err != nil { - if closeErr := w.Close(); closeErr != nil { + if closeErr := selected.w.Close(); closeErr != nil { logger.Warn("WAL close failed after storage init error", zap.String("dir", walDir), zap.Error(closeErr), @@ -181,13 +210,36 @@ func loadWalState(logger *zap.Logger, walDir, snapDir, fsmSnapDir string, fsm St } return &diskState{ - Storage: storage, - Persist: etcdstorage.NewStorage(logger, w, snapshotter), - LocalSnap: snapshot, - EffectiveApplied: effectiveApplied, + Storage: storage, + Persist: etcdstorage.NewStorage(logger, selected.w, snap.New(logger, snapDir)), + LocalSnap: selected.snapshot, + EffectiveApplied: effectiveApplied, + ExternalRestoreValidated: externalValidated, }, nil } +func restoreSnapshotWithOrigin(fsm StateMachine, selected *walRestoreState, target uint64, dir string, obs raftengine.ColdStartObserver, logger *zap.Logger) (uint64, bool, error) { + if selected == nil || selected.w == nil { + return 0, false, errors.Wrap(ErrExternalSnapshotRestoreOrigin, "restore requires WAL-selected state") + } + snapshot := selected.snapshot + marker, err := readExternalRestoreOrigin(filepath.Dir(dir)) + if err != nil { + return 0, false, err + } + if marker != nil && snapshot.GetMetadata().GetIndex() == marker.Index { + index, err := restoreWithExternalOrigin(fsm, snapshot, dir, marker) + return index, err == nil, err + } + if marker != nil { + if err := validateNativeAfterExternalRestore(selected, marker, dir); err != nil { + return 0, false, err + } + } + index, err := restoreSnapshotState(fsm, snapshot, target, dir, obs, logger) + return index, false, err +} + // reportColdStartExecute emits the executed-arm of the cold-start // outcome (callback + structured log). Split out of reportColdStart // so the parent stays under the cyclop limit; the absolute-value @@ -262,8 +314,8 @@ func coldStartApplied(disk *diskState) uint64 { // loadPersistedSnapshotWithRepair wraps loadPersistedSnapshot with one // wal.Repair attempt on io.ErrUnexpectedEOF. The caller passes in a -// shared snapshotter so loadWalState does not instantiate snap.New -// twice per open. +// shared snapshotter so both the initial scan and the repair retry use the +// same instance. func loadPersistedSnapshotWithRepair(logger *zap.Logger, walDir string, snapshotter *snap.Snapshotter) (raftpb.Snapshot, error) { snapshot, err := loadPersistedSnapshot(logger, walDir, snapshotter) if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) { diff --git a/internal/raftengine/namespace_configuration.go b/internal/raftengine/namespace_configuration.go new file mode 100644 index 000000000..7d965138f --- /dev/null +++ b/internal/raftengine/namespace_configuration.go @@ -0,0 +1,10 @@ +package raftengine + +// NamespaceConfigurationAware receives only committed/snapshot ConfState on +// the serial apply loop. Members must include all voters (including outgoing +// joint voters), learners, and learners-next, not just the election quorum. +// A watcher or Status() result cannot substitute for these ordered hooks. +type NamespaceConfigurationAware interface { + RecordAppliedNamespaceConfiguration(index uint64, members []uint64) error + RecoverNamespaceConfiguration(snapshotIndex uint64, members []uint64) error +} diff --git a/internal/raftengine/statemachine.go b/internal/raftengine/statemachine.go index ec14dc517..e1ec120a3 100644 --- a/internal/raftengine/statemachine.go +++ b/internal/raftengine/statemachine.go @@ -1,6 +1,28 @@ package raftengine -import "io" +import ( + "context" + "io" +) + +type SnapshotRestoreOrigin uint8 + +const ( + SnapshotRestoreRaft SnapshotRestoreOrigin = iota + SnapshotRestoreExternalRebase +) + +type SnapshotRestoreOptions struct { + Origin SnapshotRestoreOrigin +} + +// ContextSnapshotRestorer is required for externally rebased snapshots. Success +// must mean the decoded state was validated for this origin and durably restored. +// Wrappers must forward the origin to their underlying implementation, never +// downgrade an external restore to StateMachine.Restore. +type ContextSnapshotRestorer interface { + RestoreWithContext(context.Context, io.Reader, SnapshotRestoreOptions) error +} // Snapshot is an owned export handle from the state machine. Callers are // responsible for closing it after WriteTo completes. @@ -47,6 +69,20 @@ type ApplyIndexAware interface { SetApplyIndex(idx uint64) } +// AppliedConfigurationAware records the actual ConfChange index on the serial +// apply loop before its FSM snapshot is captured. Unlike Status.ConfigurationIndex, +// this value must survive snapshots without becoming the snapshot's data index. +type AppliedConfigurationAware interface { + RecordAppliedConfiguration(index uint64) error + AppliedConfigurationIndex() uint64 +} + +// AppliedConfigurationReader identifies runtimes that supply the ordered hook. +// A copied metadata row alone does not prove that the running engine supports it. +type AppliedConfigurationReader interface { + AppliedConfigurationIndex() (uint64, bool) +} + // AppliedIndexReader is an OPTIONAL extension that lets the engine // query the FSM's durable applied-index for the cold-start skip gate. // See docs/design/2026_06_02_implemented_idempotent_snapshot_restore.md §3. diff --git a/jepsen/SPLIT_REGRESSIONS.md b/jepsen/SPLIT_REGRESSIONS.md new file mode 100644 index 000000000..e25662c7b --- /dev/null +++ b/jepsen/SPLIT_REGRESSIONS.md @@ -0,0 +1,85 @@ +# Bounded Split Regressions + +`elastickv.split-workload --repeat-splits` opts into exactly three splits: + +1. Same-owner split at `m2-split`. +2. Split its right child at `m2-t` into another Raft group. +3. Split the moved right child at `m2-u` back into the original group. + +Each step re-reads the route catalog and uses its current version and child ID. +The next split cannot start until the preceding split completes. A rejected +helper call, FAILED/ABANDONED job, timeout, or missing lifecycle fails the run; +it does not count as successful migration or authorize another split. + +The oracle checks exact partition bounds, unchanged retained/unrelated routes, +owners, increasing catalog versions, distinct jobs, and the next source being +the previous moving child. SourceRouteID remains original lineage. Observed +working children must keep their ID through cutover (e.g. working route 40, +not the original parent 20). Polling can miss FENCE; absence of a sampled fence +does not prove that a durable fence was correct or never occurred. Job-phase +and route queries are separate RPCs, not an atomic evidence snapshot. The helper +does not expose durable SourceRouteID/WorkingSourceRouteID fields; the lineage +check uses the requested source and observed geometry, not decoded job internals. + +Four small integer registers cover all final ranges. Final reads from every +node run after the main phase and fault recovery. The coverage checker requires +an acknowledged write to every register and successful final reads after the +last verification; the existing independent linearizability checker checks values. +The default single-split and explicit abandon workloads are unchanged. + +## Running + +Use an isolated cluster with an ACTIVE route containing all four register keys, +two Raft groups, and the existing split/list-routes binaries. A programmatic test +uses 8 workers by default; CLI invocations must specify `--concurrency 8` (the +shared CLI default is 5). The worker count must be even and at least the node +count and 8. For an already provisioned cluster, for example: + +```sh +lein run -m elastickv.split-workload --local --nodes n1,n2,n3 \ + --raft-groups 1=50051,2=50052 --grpc-host-port n1:50051 \ + --repeat-splits --concurrency 8 --rate 5 --time-limit 90 \ + --split-bin /path/to/elastickv-split --list-routes-bin /path/to/elastickv-list-routes +``` + +There are at most 100 writes/key by default. Verification polls every 250 ms, +times out after 120 s/job, and retains at most 64 distinct observations; exceeding +that cap fails rather than discarding evidence. A short run that misses a split +fails. `--repeat-splits` and `--abandon-at-fence` are mutually exclusive. + +## Size-Guard Limits And Runtime Gate + +The controller/oracle unit cases cover a ResourceExhausted helper rejection, +preserved diagnostics, sticky failure, and preflight ABANDONED not being DONE. +They do **not** construct an oversized value or exercise the replicated guard. +Redis register writes are integers; the helpers expose only job ID/phase, not +guard activation, durable fence indexes, preflight cause, or pending PREPAREs. +Adding payload padding alone cannot establish a pre-fence admission test. + +The Go adapter's `TestMigrationQuorumPartitionAcceptance` separately exercises +three real data voters, a new learner, configuration/leader changes and native +snapshot restart. It asserts typed moving-range rejection, retained-left large +writes and exact restored bytes. `TestMigrationQuorumPreparedDrainAndHistoricalPreflight` +holds large PREPAREs before guard activation, resolves COMMIT and ABORT through +real Raft, and detects a retained oversized historical version beneath a small +latest value before either a source or catalog write fence. Root metadata in +these data-quorum tests is explicitly stubbed; they do not establish TLS/auth +or full cross-group root/data fault-injection acceptance. Deterministic controller +regressions separately cover repeated cross-group moves and retained negative owners. + +Production activation remains opt-in through authentication and opcode flags, +installed read/proposal/consumer hooks, a supported applied configuration and +all-current-apply-member negotiation. Flags alone cannot bypass namespace +activation or authorize imported data. The Jepsen additions here are scheduling +and oracle unit coverage, not a completed real-cluster fault-injection campaign. + +Run focused coverage and the full harness suite from `jepsen/`: + +```sh +lein test elastickv.split-workload-test elastickv.split-regression-test +lein test +``` + +The full suite requires the pinned `jepsen/redis` submodule, unlike the focused +split tests. In a fresh worktree, initialize it or add a clean checkout of that +same pinned revision to Lein's source paths; do not substitute stub namespaces. diff --git a/jepsen/src/elastickv/db.clj b/jepsen/src/elastickv/db.clj index 751e38e93..dac5020be 100644 --- a/jepsen/src/elastickv/db.clj +++ b/jepsen/src/elastickv/db.clj @@ -17,6 +17,8 @@ (def ^:private pid-file "/var/run/elastickv.pid") (def ^:private server-bin (str bin-dir "/elastickv")) (def ^:private raftadmin-bin (str bin-dir "/raftadmin")) +(def ^:private split-bin (str bin-dir "/elastickv-split")) +(def ^:private list-routes-bin (str bin-dir "/elastickv-list-routes")) (def ^:private admin-bin (str bin-dir "/elastickv-admin")) (def ^:private encryption-setup-bin (str bin-dir "/jepsen-encryption-setup")) (def ^:private kek-file (str data-dir "/kek.bin")) @@ -36,6 +38,8 @@ (def ^:private binary-builds [["elastickv" ["go" "build" "-o" (str build-dir "/elastickv") "."]] ["raftadmin" ["go" "build" "-o" (str build-dir "/raftadmin") "./cmd/raftadmin"]] + ["elastickv-split" ["go" "build" "-o" (str build-dir "/elastickv-split") "./cmd/elastickv-split"]] + ["elastickv-list-routes" ["go" "build" "-o" (str build-dir "/elastickv-list-routes") "./cmd/elastickv-list-routes"]] ["elastickv-admin" ["go" "build" "-o" (str build-dir "/elastickv-admin") "./cmd/elastickv-admin"]] ["jepsen-encryption-setup" ["go" "build" "-o" (str build-dir "/jepsen-encryption-setup") "./cmd/jepsen-encryption-setup"]]]) @@ -71,7 +75,7 @@ (c/on node (c/su (c/exec :mkdir :-p bin-dir) - (doseq [bin ["elastickv" "raftadmin" "elastickv-admin" "jepsen-encryption-setup"]] + (doseq [[bin _] binary-builds] (c/upload (str build-dir "/" bin) (str bin-dir "/" bin)) (c/exec :chmod "755" (str bin-dir "/" bin)))))) @@ -206,7 +210,7 @@ "Builds the production server argv for one managed Jepsen node." [{:keys [node grpc redis dynamo s3 sqs sqs-region data-dir raft-engine raft-redis-map raft-dynamo-map raft-s3-map raft-sqs-map raft-groups shard-ranges - filesystem-mount encryption bootstrap? join-as-learner join-members]}] + filesystem-mount encryption migration-enabled bootstrap? join-as-learner join-members]}] (cond-> ["--address" grpc "--redisAddress" redis "--raftId" (name node) @@ -221,6 +225,8 @@ filesystem-mount (conj "--filesystemMount" filesystem-mount) (seq raft-groups) (conj "--raftGroups" (build-raft-groups-arg node raft-groups)) (seq shard-ranges) (conj "--shardRanges" shard-ranges) + ;; Isolated migration tests intentionally use unauthenticated helpers. + migration-enabled (conj "--adminInsecureNoAuth") encryption (conj "--encryptionSidecarPath" sidecar-file "--encryption-enabled" "--kekFile" kek-file) @@ -306,7 +312,7 @@ (c/exec :bash "-c" (unmount-filesystem-cmd mount-dir)))))) (defn- start-node! - [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env filesystem-mount encryption join-as-learner join-members]}] + [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env migration-enabled filesystem-mount encryption join-as-learner join-members]}] (when (and (seq raft-groups) (> (count raft-groups) 1) (nil? shard-ranges)) @@ -331,14 +337,19 @@ :raft-redis-map raft-redis-map :raft-dynamo-map raft-dynamo-map :raft-s3-map raft-s3-map :raft-sqs-map raft-sqs-map :raft-groups raft-groups :shard-ranges shard-ranges - :filesystem-mount mount-dir :encryption encryption - :bootstrap? bootstrap? :join-as-learner join-as-learner - :join-members join-members}) + :filesystem-mount mount-dir :encryption encryption :migration-enabled migration-enabled + :bootstrap? bootstrap? :join-as-learner join-as-learner + :join-members join-members}) + effective-server-env (cond-> (or server-env {}) + migration-enabled + (merge {"ELASTICKV_ENABLE_MIGRATION_IMPORT_OPCODE" "true" + "ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE" "true" + "ELASTICKV_ENABLE_MIGRATION_CLEANUP_OPCODE" "true"})) daemon-opts (cond-> {:chdir bin-dir :logfile log-file :pidfile pid-file :background? true} - (seq server-env) (assoc :env server-env))] + (seq effective-server-env) (assoc :env effective-server-env))] (c/on node (c/su (c/exec :mkdir :-p data-dir) diff --git a/jepsen/src/elastickv/jepsen_test.clj b/jepsen/src/elastickv/jepsen_test.clj index ff97a1c68..3ef8e8e9e 100644 --- a/jepsen/src/elastickv/jepsen_test.clj +++ b/jepsen/src/elastickv/jepsen_test.clj @@ -8,6 +8,7 @@ [elastickv.s3-workload :as s3-workload] [elastickv.filesystem-workload :as filesystem-workload] [elastickv.sqs-htfifo-workload :as sqs-htfifo-workload] + [elastickv.split-workload :as split-workload] [jepsen.cli :as cli])) (defn elastickv-test @@ -38,6 +39,10 @@ ([] (elastickv-zset-safety-test {})) ([opts] (zset-safety-workload/elastickv-zset-safety-test opts))) +(defn elastickv-split-test + ([] (elastickv-split-test {})) + ([opts] (split-workload/elastickv-split-test opts))) + (def ^:private test-fns "Map of user-facing test names to their constructor fns. The first positional CLI arg selects which workload runs; if absent or unknown, @@ -48,6 +53,7 @@ "elastickv-dynamodb-test" elastickv-dynamodb-test "elastickv-s3-test" elastickv-s3-test "elastickv-filesystem-test" elastickv-filesystem-test + "elastickv-split-test" elastickv-split-test "elastickv-learner-test" elastickv-learner-test}) (defn elastickv-sqs-htfifo-test @@ -65,7 +71,7 @@ Supported s: elastickv-test, elastickv-zset-safety-test, elastickv-dynamodb-test, elastickv-s3-test, elastickv-filesystem-test, - elastickv-learner-test. When the first positional arg is not a known test + elastickv-split-test, elastickv-learner-test. When the first positional arg is not a known test name, we default to `elastickv-test` for backward compatibility and forward ALL args to jepsen.cli/run!. diff --git a/jepsen/src/elastickv/split_workload.clj b/jepsen/src/elastickv/split_workload.clj new file mode 100644 index 000000000..531db2f05 --- /dev/null +++ b/jepsen/src/elastickv/split_workload.clj @@ -0,0 +1,516 @@ +(ns elastickv.split-workload + "Bounded M2 split workloads over Redis registers." + (:gen-class) + (:require [clojure.java.shell :as shell] + [clojure.tools.logging :refer [info warn]] + [elastickv.cli :as cli] + [elastickv.composed1-nemesis :as routes] + [elastickv.db :as ekdb] + [jepsen [checker :as checker] + [client :as client] + [generator :as gen] + [independent :as independent] + [net :as net]] + [jepsen.checker.timeline :as timeline] + [jepsen.control :as control] + [jepsen.db :as jdb] + [jepsen.nemesis :as nemesis] + [jepsen.nemesis.combined :as combined] + [jepsen.os :as os] + [jepsen.os.debian :as debian] + [knossos.model :as model] + [taoensso.carmine :as car])) + +(def default-nodes ["n1" "n2" "n3" "n4" "n5"]) +(def ^:private split-key "m2-split") +(def ^:private register-keys ["m2-left" "m2-target"]) +(def ^:private repeated-register-keys ["m2-left" "m2-stay" "m2-target" "m2-z"]) +(def ^:private repeated-splits + [{:split-key "m2-split" :same-owner? true} + {:split-key "m2-t"} + {:split-key "m2-u"}]) + +(defn- helper-path [name] + (str (System/getProperty "user.dir") "/target/elastickv-jepsen/" name)) + +(defrecord SplitRegisterClient [node->port conn-spec] + client/Client + (open! [this test node] + (assoc this :node node :conn-spec + {:pool {} + :spec {:host (or (:redis-host test) (name node)) + :port (get node->port node 6379) + :timeout-ms 10000}})) + (close! [this _test] (assoc this :conn-spec nil)) + (setup! [this _test] + (doseq [k (or (:register-keys this) register-keys)] + (car/wcar conn-spec (car/del k))) + this) + (teardown! [this _test] this) + (invoke! [this _test op] + (let [op (cond-> op (:final op) (assoc :node (:node this)))] + (try + (let [[k value] (:value op) + redis-key (nth (or (:register-keys this) register-keys) k)] + (case (:f op) + :write (do (car/wcar conn-spec (car/set redis-key (str value))) + (assoc op :type :ok)) + :read (let [raw (car/wcar conn-spec (car/get redis-key)) + value (when raw (Long/parseLong (str raw)))] + (assoc op :type :ok :value (independent/tuple k value))))) + (catch java.net.ConnectException _ + (assoc op :type :info :error :connection-refused)) + (catch java.net.SocketTimeoutException _ + (assoc op :type :info :error :socket-timeout)) + (catch Throwable t + (assoc op :type :info :error (or (.getMessage t) (str t)))))))) + +(defn split-register-workload [opts] + (let [keys (if (:repeat-splits opts) repeated-register-keys register-keys) + client (assoc (->SplitRegisterClient + (or (:node->port opts) (zipmap default-nodes (repeat 6379))) + nil) + :register-keys keys) + max-writes (or (:max-writes-per-key opts) 100)] + {:client client + :generator (independent/concurrent-generator + 2 + (range (count keys)) + (fn [_] + (gen/mix [(map (fn [v] {:f :write :value v}) (range max-writes)) + (repeat max-writes {:f :read})]))) + :checker (independent/checker + (checker/compose + {:linear (checker/linearizable {:model (model/register) + :algorithm :competition}) + :timeline (timeline/html)}))})) + +(defn- grpc-address [opts test] + (or (:grpc-host-port opts) + (:grpc-host-port test) + (let [node (name (first (:nodes test))) + groups (:raft-groups test) + port (if (seq groups) + (get groups (first (sort (keys groups)))) + (or (:grpc-port test) 50051))] + (str node ":" port)))) + +(defn- run-helper! [bin & args] + (let [result (apply shell/sh bin args)] + (when-not (zero? (:exit result)) + (throw (ex-info (str bin " failed") result))) + (:out result))) + +(defn- current-plan [opts test] + (let [address (grpc-address opts test) + snapshot (routes/parse-routes-json + (run-helper! (or (:list-routes-bin opts) (helper-path "elastickv-list-routes")) + "--address" address)) + source (routes/route-containing-key (:routes snapshot) split-key) + groups (sort (keys (:raft-groups test))) + target (or (:target-group-id opts) + (first (remove #{(:raft-group-id source)} groups)))] + (when-not (and source target) + (throw (ex-info "split workload requires an active source route and a different target group" + {:source source :groups groups}))) + {:address address + :catalog-version (:catalog-version snapshot) + :route-id (:route-id source) + :target-group-id target})) + +(defn- start-split! [opts test] + (let [{:keys [address catalog-version route-id target-group-id] :as plan} + (current-plan opts test) + output (run-helper! (or (:split-bin opts) (helper-path "elastickv-split")) + "--address" address + "--route-id" (str route-id) + "--split-key" split-key + "--expected-version" (str catalog-version) + "--target-group-id" (str target-group-id)) + job-id (some->> (re-find #"job_id:\s*(\d+)" output) second Long/parseLong)] + (when-not job-id + (throw (ex-info "split helper did not return job_id" {:output output :plan plan}))) + (assoc plan :job-id job-id :output output))) + +(defn- job-phase [opts address job-id] + (let [output (run-helper! (or (:split-bin opts) (helper-path "elastickv-split")) + "--address" address "--get-job-id" (str job-id))] + (some->> (re-find #"phase:\s*(\S+)" output) second))) + +(defn- await-phase! [opts address job-id wanted timeout-ms] + (let [deadline (+ (System/currentTimeMillis) timeout-ms)] + (loop [] + (let [phase (job-phase opts address job-id)] + (cond + (contains? wanted phase) phase + (> (System/currentTimeMillis) deadline) + (throw (ex-info "timed out waiting for split job phase" + {:job-id job-id :phase phase :wanted wanted})) + :else (do (Thread/sleep 100) (recur))))))) + +(defn split-nemesis [opts] + (let [active (atom nil)] + (reify + nemesis/Reflection + (fs [_] #{:start-cross-group-split :verify-cross-group-split}) + nemesis/Nemesis + (setup! [this _test] this) + (invoke! [_ test op] + (try + (case (:f op) + :start-cross-group-split + (let [plan (start-split! opts test)] + (reset! active plan) + (if (:abandon-at-fence opts) + (do + (await-phase! opts (:address plan) (:job-id plan) + #{"SPLIT_JOB_PHASE_FENCE"} 60000) + (run-helper! (or (:split-bin opts) (helper-path "elastickv-split")) + "--address" (:address plan) + "--abandon-job-id" (str (:job-id plan)))) + (info "started M2 split job" (:job-id plan))) + (assoc op :value plan)) + + :verify-cross-group-split + (let [{:keys [address job-id] :as plan} @active + wanted (if (:abandon-at-fence opts) + #{"SPLIT_JOB_PHASE_ABANDONED"} + #{"SPLIT_JOB_PHASE_DONE"}) + phase (await-phase! opts address job-id wanted 120000)] + (assoc op :value (assoc plan :phase phase))) + + op) + (catch Throwable t + (warn t "M2 split nemesis failed") + (assoc op :type :fail :error (or (.getMessage t) (str t)))))) + (teardown! [this _test] this)))) + +(defn split-package [opts] + {:generator (gen/phases + (gen/sleep (or (:split-at-seconds opts) 5)) + (gen/once {:type :info :f :start-cross-group-split}) + (gen/sleep (or (:split-verify-after-seconds opts) 20)) + (gen/once {:type :info :f :verify-cross-group-split})) + :final-generator nil + :nemesis (split-nemesis opts) + :perf #{{:name "cross-group-split" + :fs #{:start-cross-group-split :verify-cross-group-split} + :start #{:start-cross-group-split} + :stop #{:verify-cross-group-split} + :color "#4A90E2"}}}) + +(defn split-lifecycle-checker [opts] + (reify checker/Checker + (check [_ _test history _checker-opts] + (let [operations (filter #(and (= :nemesis (:process %)) + (contains? #{:start-cross-group-split :verify-cross-group-split} (:f %))) + history) + failures (filter #(or (= :fail (:type %)) (:error %)) operations) + ;; Jepsen also logs invocation placeholders with no result value. + completions (vec (filter #(map? (:value %)) operations)) + [start verify] completions + job-id (get-in start [:value :job-id]) + expected (if (:abandon-at-fence opts) + "SPLIT_JOB_PHASE_ABANDONED" + "SPLIT_JOB_PHASE_DONE")] + {:valid? (and (empty? failures) + (= 2 (count completions)) + (= :start-cross-group-split (:f start)) + (= :verify-cross-group-split (:f verify)) + (every? #(contains? #{:info :ok} (:type %)) completions) + (integer? job-id) (pos? job-id) + (= job-id (get-in verify [:value :job-id])) + (= expected (get-in verify [:value :phase]))) + :expected-phase expected + :completions completions + :failures (vec failures)})))) + +(defn- route-snapshot! [opts test] + (update (routes/parse-routes-json + (run-helper! (or (:list-routes-bin opts) (helper-path "elastickv-list-routes")) + "--address" (grpc-address opts test))) + :routes #(vec (sort-by :route-id %)))) + +(defn- exact-range [snapshot start end] + (filter #(and (= start (:start %)) (= end (:end %))) (:routes snapshot))) + +(defn- repeated-plan! [opts test step previous] + (let [{:keys [split-key same-owner?]} (nth repeated-splits step) + before (route-snapshot! opts test) + sources (filter #(routes/route-covers? % split-key) (:routes before)) + source (first sources) + target (if same-owner? (:raft-group-id source) + (if (= 1 step) + (or (:target-group-id opts) + (first (remove #{(:raft-group-id source)} (sort (keys (:raft-groups test)))))) + (:raft-group-id (:source previous))))] + (when-not (and (= 1 (count sources)) + (neg? (compare (:start source) split-key)) + target (or same-owner? (not= target (:raft-group-id source))) + (or (zero? step) (= source (:right previous)))) + (throw (ex-info "repeated split requires the previous moving child and a valid owner" + {:step step :source source :target target :previous previous}))) + {:step step :split-key split-key :same-owner? (boolean same-owner?) + :address (grpc-address opts test) :before before :source source + :route-id (:route-id source) :target-group-id target})) + +(defn- begin-repeated-split! [opts plan] + (let [{:keys [address route-id split-key before same-owner? target-group-id]} plan + args (cond-> ["--address" address "--route-id" (str route-id) + "--split-key" split-key "--expected-version" (str (:catalog-version before))] + (not same-owner?) (into ["--target-group-id" (str target-group-id)])) + output (apply run-helper! (or (:split-bin opts) (helper-path "elastickv-split")) args) + job-id (some->> (re-find #"job_id:\s*(\d+)" output) second Long/parseLong)] + (when (and (not same-owner?) (not (and job-id (pos? job-id)))) + (throw (ex-info "split helper did not return a positive job_id" {:output output :plan plan}))) + (assoc plan :job-id job-id))) + +(declare partition-valid?) + +(defn- finish-repeated-split! [opts test plan] + (let [deadline (+ (System/currentTimeMillis) (or (:split-timeout-ms opts) 120000))] + (loop [observations []] + (let [phase (if (:same-owner? plan) "SPLIT_JOB_PHASE_DONE" + (job-phase opts (:address plan) (:job-id plan))) + snapshot (route-snapshot! opts test) + observation {:phase phase :snapshot snapshot} + observations (if (= observation (peek observations)) observations + (conj observations observation))] + (when (> (count observations) 64) + (throw (ex-info "too many distinct split observations" {:plan plan}))) + (cond + (= "SPLIT_JOB_PHASE_DONE" phase) + (let [result (assoc plan :phase phase :after snapshot :observations observations + :right (first (exact-range snapshot (:split-key plan) (:end (:source plan)))))] + (when-not (partition-valid? result) + (throw (ex-info "split completed with invalid partition geometry" result))) + result) + (or (contains? #{"SPLIT_JOB_PHASE_FAILED" "SPLIT_JOB_PHASE_ABANDONING" + "SPLIT_JOB_PHASE_ABANDONED"} phase) + (> (System/currentTimeMillis) deadline)) + (throw (ex-info "repeated split did not complete" {:plan plan :observations observations})) + :else (do (Thread/sleep 250) (recur observations))))))) + +(defn repeated-split-nemesis [opts] + (let [state (atom {:next-step 0})] + (reify + nemesis/Reflection + (fs [_] #{:start-repeated-split :verify-repeated-split}) + nemesis/Nemesis + (setup! [this _test] this) + (invoke! [_ test op] + (try + (let [{:keys [next-step active previous failed?]} @state + step (get-in op [:value :step])] + (when (or failed? (not= step next-step) (>= next-step (count repeated-splits))) + (throw (ex-info "out-of-order repeated split" {:state @state :op op}))) + (case (:f op) + :start-repeated-split + (do + (when active (throw (ex-info "split already active" {:active active}))) + (let [plan (begin-repeated-split! opts (repeated-plan! opts test step previous))] + (swap! state assoc :active plan) + (assoc op :value plan))) + :verify-repeated-split + (do + (when-not active (throw (ex-info "no active repeated split" {:step step}))) + (let [result (finish-repeated-split! opts test active)] + (reset! state {:next-step (inc next-step) :previous result}) + (assoc op :value result))) + op)) + (catch Throwable t + (swap! state assoc :failed? true) + (assoc op :type :fail :error (or (.getMessage t) (str t)) + :details (ex-data t))))) + (teardown! [this _test] this)))) + +(defn repeated-split-package [opts] + {:generator (apply gen/phases + (gen/sleep (or (:split-at-seconds opts) 5)) + (mapcat (fn [step] + [(gen/once {:type :info :f :start-repeated-split :value {:step step}}) + (gen/once {:type :info :f :verify-repeated-split :value {:step step}})]) + (range (count repeated-splits)))) + :final-generator nil + :nemesis (repeated-split-nemesis opts) + :perf #{}}) + +(defn- partition-valid? [{:keys [before after source split-key target-group-id observations route-id right]}] + (let [lefts (exact-range after (:start source) split-key) + rights (exact-range after split-key (:end source)) + left (first lefts) + positive-id? #(and (integer? %) (pos? %)) + without-source (remove #{source} (:routes before))] + (and (= source (first (filter #(= route-id (:route-id %)) (:routes before)))) + (= route-id (:route-id source)) + (routes/route-covers? source split-key) + (neg? (compare (:start source) split-key)) + (= 1 (count lefts)) (= 1 (count rights)) (= right (first rights)) + (every? positive-id? [route-id (:route-id left) (:route-id right)]) + (= 3 (count (set [route-id (:route-id left) (:route-id right)]))) + (= "ROUTE_STATE_ACTIVE" (:state left) (:state right)) + (= (:raft-group-id source) (:raft-group-id left)) + (= target-group-id (:raft-group-id right)) + (< (:catalog-version before) (:catalog-version after)) + (= (set (:routes after)) (set (concat without-source [left right]))) + (= (inc (count (:routes before))) (count (:routes after))) + (= (count (:routes after)) (count (set (map :route-id (:routes after))))) + (seq observations) + ;; SourceRouteID is lineage. Only the moving child's observed ID is + ;; stable across FENCE/cutover; it must not be replaced at cutover. + (= after (:snapshot (peek observations))) + (= "SPLIT_JOB_PHASE_DONE" (:phase (peek observations))) + (apply <= (:catalog-version before) (map #(get-in % [:snapshot :catalog-version]) observations)) + (every? (fn [{:keys [snapshot]}] + (let [moving (first (exact-range snapshot split-key (:end source)))] + (or (= (set (:routes before)) (set (:routes snapshot))) + (and (= (:route-id right) (:route-id moving)) + (contains? (set [(:raft-group-id source) target-group-id]) (:raft-group-id moving)) + (contains? #{"ROUTE_STATE_ACTIVE" "ROUTE_STATE_WRITE_FENCED"} (:state moving)) + (= (set (:routes snapshot)) + (set (concat without-source [left moving]))) + (= (count (:routes after)) (count (:routes snapshot))))))) + observations)))) + +(defn repeated-split-checker [] + (reify checker/Checker + (check [_ _test history _checker-opts] + (let [ops (filter #(and (= :nemesis (:process %)) + (contains? #{:start-repeated-split :verify-repeated-split} (:f %))) history) + failures (vec (filter #(or (= :fail (:type %)) (:error %)) ops)) + completions (vec (filter #(and (contains? #{:info :ok} (:type %)) + (:source (:value %))) ops)) + pairs (partition 2 completions) + results (mapv (comp :value second) pairs)] + {:valid? (boolean + (and (empty? failures) (= 6 (count completions)) + (every? true? + (map-indexed + (fn [step [start verify]] + (let [plan (:value start) result (:value verify) + same? (zero? step)] + (and (= :start-repeated-split (:f start)) + (= :verify-repeated-split (:f verify)) + (= step (:step plan) (:step result)) + (= (:split-key (nth repeated-splits step)) (:split-key plan)) + (= same? (:same-owner? plan)) + (= plan (select-keys result (keys plan))) + (= "SPLIT_JOB_PHASE_DONE" (:phase result)) + (if same? (and (nil? (:job-id plan)) + (= (:target-group-id plan) (:raft-group-id (:source plan)))) + (and (integer? (:job-id plan)) (pos? (:job-id plan)) + (not= (:target-group-id plan) (:raft-group-id (:source plan))))) + (partition-valid? result)))) pairs)) + (= 2 (count (set (keep :job-id results)))) + (every? (fn [[previous result]] + (and (= (:right previous) (:source result)) + (= (:after previous) (:before result)))) + (partition 2 1 results)) + (= (:raft-group-id (:source (first results))) + (:target-group-id (last results))))) + :completions completions :failures failures})))) + +(defn repeated-final-reads [node-count] + (apply gen/any + (for [node-index (range node-count)] + (gen/on-threads #(and (integer? %) (= node-index (mod % node-count))) + (map (fn [k] {:f :read :value (independent/tuple k nil) :final true}) + (range (count repeated-register-keys))))))) + +(defn repeated-read-checker [] + (reify checker/Checker + (check [_ test history _opts] + (let [writes (set (keep #(when (and (= :ok (:type %)) (= :write (:f %))) + (first (:value %))) history)) + reads (filter #(and (:final %) (= :ok (:type %)) (= :read (:f %))) history) + last-verify (last (filter #(and (= :verify-repeated-split (:f %)) + (= "SPLIT_JOB_PHASE_DONE" (get-in % [:value :phase]))) history)) + observed (set (map #(vector (:node %) (first (:value %))) reads)) + expected (set (for [n (:nodes test), k (range (count repeated-register-keys))] [n k]))] + {:valid? (and (= (set (range (count repeated-register-keys))) writes) + (= expected observed) + (some? last-verify) + (every? #(< (:index last-verify) (:index %)) reads)) + :missing-final-reads (vec (sort (remove observed expected))) + :written-keys writes})))) + +(defn elastickv-split-test + ([] (elastickv-split-test {})) + ([opts] + (when (and (:repeat-splits opts) (:abandon-at-fence opts)) + (throw (ex-info "repeated splits require completed migrations" {}))) + (let [nodes (or (:nodes opts) default-nodes) + concurrency (or (:concurrency opts) (if (:repeat-splits opts) 8 6)) + _ (when (and (:repeat-splits opts) + (or (< concurrency (max (count nodes) 8)) (odd? concurrency))) + (throw (ex-info "repeated splits require even concurrency >= max(node count, 8)" {}))) + ports (or (:redis-ports opts) (repeat (count nodes) (or (:redis-port opts) 6379))) + node->port (or (:node->port opts) (cli/ports->node-map ports nodes)) + local? (:local opts) + db (if local? jdb/noop + (ekdb/db {:grpc-port (or (:grpc-port opts) 50051) + :redis-port node->port + :raft-groups (:raft-groups opts) + :shard-ranges (:shard-ranges opts) + :migration-enabled true})) + fault-package (when-not local? + (combined/nemesis-package + {:db db + :faults (cli/normalize-faults (or (:faults opts) [:partition :kill])) + :interval (or (:fault-interval opts) 10)})) + package (combined/compose-packages + (cond-> [(if (:repeat-splits opts) (repeated-split-package opts) (split-package opts))] + fault-package (conj fault-package))) + workload (split-register-workload (assoc opts :node->port node->port)) + main-gen (->> (:generator workload) + (gen/nemesis (:generator package)) + (gen/stagger (/ (double (or (:rate opts) 10)))) + (gen/time-limit (or (:time-limit opts) 60))) + checkers (cond-> {:register (:checker workload) + :split-lifecycle (if (:repeat-splits opts) + (repeated-split-checker) + (split-lifecycle-checker opts))} + (:repeat-splits opts) (assoc :final-reads (repeated-read-checker))) + generator (if (:repeat-splits opts) + (gen/phases main-gen + (gen/nemesis (:final-generator package)) + (gen/clients (repeated-final-reads (count nodes)))) + main-gen)] + (merge workload + {:name (or (:name opts) (if (:repeat-splits opts) "elastickv-m2-repeated-split" + "elastickv-m2-cross-group-split")) + :nodes nodes + :db db + :raft-groups (:raft-groups opts) + :grpc-port (:grpc-port opts) + :grpc-host-port (:grpc-host-port opts) + :redis-host (:redis-host opts) + :os (if local? os/noop debian/os) + :net (if local? net/noop net/iptables) + :ssh (merge {:username "vagrant" + :private-key-path "/home/vagrant/.ssh/id_rsa" + :strict-host-key-checking false} + (when local? {:dummy true}) (:ssh opts)) + :remote control/ssh + :nemesis (:nemesis package) + :checker (checker/compose checkers) + :final-generator nil + :concurrency concurrency + :generator generator})))) + +(def split-cli-opts + [[nil "--redis-port PORT" "Redis port." :default 6379 :parse-fn #(Integer/parseInt %)] + [nil "--target-group-id ID" "Target Raft group." :parse-fn #(Long/parseLong %)] + [nil "--grpc-host-port HOST:PORT" "Distribution gRPC address."] + [nil "--split-bin PATH" "Path to elastickv-split."] + [nil "--list-routes-bin PATH" "Path to elastickv-list-routes."] + [nil "--split-at-seconds SECONDS" "Workload time to start migration." :default 5 :parse-fn #(Integer/parseInt %)] + [nil "--split-verify-after-seconds SECONDS" "Delay before verifying terminal state." :default 20 :parse-fn #(Integer/parseInt %)] + [nil "--repeat-splits" "Run one same-owner and two successive cross-group splits." :default false] + [nil "--abandon-at-fence" "Abandon after observing FENCE." :default false]]) + +(defn -main [& args] + (cli/run-workload! args + (into cli/common-cli-opts split-cli-opts) + #(cli/parse-common-opts % nil) + elastickv-split-test)) diff --git a/jepsen/test/elastickv/db_test.clj b/jepsen/test/elastickv/db_test.clj index 761394df2..c533c547b 100644 --- a/jepsen/test/elastickv/db_test.clj +++ b/jepsen/test/elastickv/db_test.clj @@ -3,12 +3,30 @@ [clojure.java.shell :as sh] [clojure.string :as str] [clojure.test :refer :all] - [elastickv.db :as db]) + [elastickv.db :as db] + [jepsen.control :as c] + [jepsen.control.util :as cu]) (:import (java.nio.file Files) (java.nio.file.attribute FileAttribute))) (def ^:private awkward-dir "/mnt/elastickv test's dir") +(deftest migration-nodes-explicitly-allow-unauthenticated-test-helpers + (doseq [enabled? [false true]] + (let [started (atom nil)] + (with-redefs [c/session (constantly :test-session) + c/disconnect (constantly nil) + c/exec (constantly nil) + cu/start-daemon! (fn [opts _bin & args] + (reset! started {:opts opts :args (set args)}))] + (#'db/start-node! {:nodes ["n1"]} "n1" + {:bootstrap-node "n1" :grpc-port 50051 :redis-port 6379 + :data-dir "/tmp/test" :migration-enabled enabled?})) + (is (= enabled? (contains? (:args @started) "--adminInsecureNoAuth"))) + (doseq [opcode ["IMPORT" "PROMOTE" "CLEANUP"]] + (is (= (when enabled? "true") + (get-in @started [:opts :env (str "ELASTICKV_ENABLE_MIGRATION_" opcode "_OPCODE")]))))))) + (deftest lease-counters-accept-production-const-labels (is (= {:hit 2 :miss 1} (db/parse-lease-read-counters diff --git a/jepsen/test/elastickv/split_regression_test.clj b/jepsen/test/elastickv/split_regression_test.clj new file mode 100644 index 000000000..87c7cc99c --- /dev/null +++ b/jepsen/test/elastickv/split_regression_test.clj @@ -0,0 +1,256 @@ +(ns elastickv.split-regression-test + "Controller/oracle unit fixtures, not replicated runtime acceptance." + (:require [clojure.data.json :as json] + [clojure.java.shell :as shell] + [clojure.test :refer :all] + [elastickv.split-workload :as split] + [jepsen.checker :as checker] + [jepsen.generator :as gen] + [jepsen.history :as history] + [jepsen.nemesis :as nemesis])) + +(defn- route [id owner start end] + {:route-id id :raft-group-id owner :start start :end end :state "ROUTE_STATE_ACTIVE"}) + +(defn- snapshot [version & routes] + {:catalog-version version :routes (vec routes)}) + +(def root (route 100 1 "" nil)) +(def left (route 10 1 "" "m2-split")) +(def child (route 20 1 "m2-split" nil)) +(def retained (route 30 1 "m2-split" "m2-t")) +(def moving (route 40 2 "m2-t" nil)) +(def retained-again (route 50 2 "m2-t" "m2-u")) +(def moving-again (route 60 1 "m2-u" nil)) +(def snapshots [(snapshot 7 root) + (snapshot 8 left child) + (snapshot 10 left retained moving) + (snapshot 12 left retained retained-again moving-again)]) +(def fences [(snapshot 9 left retained + (assoc moving :raft-group-id 1 :state "ROUTE_STATE_WRITE_FENCED")) + (snapshot 11 left retained retained-again + (assoc moving-again :raft-group-id 2 :state "ROUTE_STATE_WRITE_FENCED"))]) + +(defn- lifecycle-fixture [] + (vec + (mapcat + (fn [step source key target job-id] + (let [plan {:step step :source source :route-id (:route-id source) + :before (nth snapshots step) :address "n1:50051" + :split-key key :same-owner? (zero? step) + :target-group-id target :job-id job-id} + after (nth snapshots (inc step)) + result (assoc plan :phase "SPLIT_JOB_PHASE_DONE" :after after + :right (last (:routes after)) + :observations (cond-> [] + (pos? step) (conj {:phase "SPLIT_JOB_PHASE_FENCE" + :snapshot (nth fences (dec step))}) + true (conj {:phase "SPLIT_JOB_PHASE_DONE" :snapshot after})))] + [{:process :nemesis :type :info :f :start-repeated-split :value plan} + {:process :nemesis :type :info :f :verify-repeated-split :value result}])) + (range 3) [root child moving] ["m2-split" "m2-t" "m2-u"] [1 2 1] [nil 44 45]))) + +(defn- check-lifecycle [events] + (checker/check (split/repeated-split-checker) {} (history/history events) {})) + +(defn- b64 [s] + (.encodeToString (java.util.Base64/getEncoder) (.getBytes (or s "") "UTF-8"))) + +(defn- encode-snapshot [s] + (json/write-str + {:catalog_version (:catalog-version s) + :routes (mapv (fn [r] {:route_id (:route-id r) :raft_group_id (:raft-group-id r) + :start (b64 (:start r)) :end (b64 (:end r)) :state (:state r)}) + (:routes s))})) + +(def opts {:list-routes-bin "elastickv-list-routes" :split-bin "elastickv-split" + :grpc-host-port "n1:50051" :repeat-splits true}) +(def cluster {:nodes ["n1" "n2" "n3"] :raft-groups {1 50051, 2 50052}}) + +(defn- invoke [nem f step] + (nemesis/invoke! nem cluster {:process :nemesis :type :info :f f :value {:step step}})) + +(deftest repeated-split-scheduling-is-bounded-and-ordered + (let [ctx (gen/context {:concurrency 8}) + package (split/repeated-split-package {:split-at-seconds 0}) + ops (loop [g (:generator package), ops []] + (let [[op next-gen] (gen/op g {} ctx)] + (cond + (nil? op) ops + (= :pending op) (throw (ex-info "unexpected pending generator" {})) + (> (count ops) 6) (throw (ex-info "unbounded split generator" {})) + :else (recur next-gen (conj ops op)))))] + (is (= [[:start-repeated-split 0] [:verify-repeated-split 0] + [:start-repeated-split 1] [:verify-repeated-split 1] + [:start-repeated-split 2] [:verify-repeated-split 2]] + (mapv #(vector (:f %) (get-in % [:value :step])) (filter :f ops)))) + (is (= #{:start-repeated-split :verify-repeated-split} (nemesis/fs (:nemesis package)))))) + +(deftest repeated-controller-queries-current-child-and-current-version + (let [current (atom 0), calls (atom [])] + (with-redefs [shell/sh + (fn [bin & args] + (swap! calls conj (into [bin] args)) + {:exit 0 :err "" + :out (cond + (= bin "elastickv-list-routes") (encode-snapshot (nth snapshots @current)) + (= "--get-job-id" (nth args 2)) "phase: SPLIT_JOB_PHASE_DONE\n" + :else (let [step @current] + (swap! current inc) + (str "catalog_version: " (+ 8 step) "\n" + (when (pos? step) (str "job_id: " (+ 43 step) "\n")))))})] + (let [nem (split/repeated-split-nemesis opts) + results (vec (mapcat (fn [step] [(invoke nem :start-repeated-split step) + (invoke nem :verify-repeated-split step)]) (range 3))) + starts (filter #(some #{"--route-id"} %) @calls)] + (is (every? #(= :info (:type %)) results)) + (is (true? (:valid? (check-lifecycle results)))) + (is (= [["elastickv-split" "--address" "n1:50051" "--route-id" "100" + "--split-key" "m2-split" "--expected-version" "7"] + ["elastickv-split" "--address" "n1:50051" "--route-id" "20" + "--split-key" "m2-t" "--expected-version" "8" "--target-group-id" "2"] + ["elastickv-split" "--address" "n1:50051" "--route-id" "40" + "--split-key" "m2-u" "--expected-version" "10" "--target-group-id" "1"]] + starts)) + (is (= :fail (:type (invoke nem :start-repeated-split 3)))))))) + +(deftest repeated-oracle-checks-partitions-lineage-and-stable-moving-id + (let [h (lifecycle-fixture)] + (is (true? (:valid? (check-lifecycle h)))) + (is (= 20 (get-in h [3 :value :route-id])) "SourceRouteID remains original lineage") + (is (= 40 (get-in h [3 :value :right :route-id])) "working moving child is distinct") + (doseq [[label bad] [["empty" []] + ["only first split" (subvec h 0 2)] + ["missing verify" (pop h)] + ["duplicate lifecycle" (into h (subvec h 0 2))] + ["reordered" (assoc h 0 (nth h 1) 1 (nth h 0))] + ["failed start" (assoc-in h [2 :type] :fail)] + ["failed verify" (assoc-in h [3 :type] :fail)] + ["wrong job" (assoc-in h [3 :value :job-id] 999)] + ["job reused" (-> h (assoc-in [4 :value :job-id] 44) + (assoc-in [5 :value :job-id] 44))] + ["zero job" (-> h (assoc-in [2 :value :job-id] 0) + (assoc-in [3 :value :job-id] 0))] + ["wrong step" (assoc-in h [2 :value :step] 0)] + ["still fenced" (assoc-in h [3 :value :phase] "SPLIT_JOB_PHASE_FENCE")] + ["oversized preflight abandoned" (assoc-in h [3 :value :phase] "SPLIT_JOB_PHASE_ABANDONED")] + ["retired source reused" (assoc-in h [4 :value :source] child)] + ["cutover replaced working child" (assoc-in h [3 :value :observations 0 :snapshot :routes 2 :route-id] 99)] + ["whole parent fenced" (assoc-in h [3 :value :observations 0 :snapshot] + (snapshot 9 left (assoc child :state "ROUTE_STATE_WRITE_FENCED")))] + ["retained side moved" (assoc-in h [3 :value :after :routes 1 :raft-group-id] 2)] + ["gap" (assoc-in h [3 :value :after :routes 1 :end] "m2-s")] + ["overlap" (update-in h [3 :value :after :routes] conj child)] + ["unrelated route changed" (assoc-in h [3 :value :after :routes 0 :raft-group-id] 2)] + ["catalog regressed" (assoc-in h [3 :value :after :catalog-version] 7)] + ["no observations" (assoc-in h [3 :value :observations] [])]]] + (is (false? (:valid? (check-lifecycle bad))) label)) + (is (true? (:valid? (check-lifecycle + (into [{:process :nemesis :type :info :f :start-repeated-split :value {:step 0}}] h)))) + "invocation placeholders are not successful completions"))) + +(deftest oversized-helper-rejection-is-not-migration-success + (let [calls (atom 0) + diagnostic "rpc error: code = ResourceExhausted desc = new version exceeds moving-range transfer budget"] + (with-redefs [shell/sh (fn [bin & _] + (swap! calls inc) + (if (= bin "elastickv-list-routes") + {:exit 0 :out (encode-snapshot (first snapshots)) :err ""} + {:exit 1 :out "" :err diagnostic}))] + (let [nem (split/repeated-split-nemesis opts) + start (invoke nem :start-repeated-split 0) + verify (invoke nem :verify-repeated-split 0) + next (invoke nem :start-repeated-split 1)] + (is (= :fail (:type start) (:type verify) (:type next))) + (is (= 1 (get-in start [:details :exit]))) + (is (= diagnostic (get-in start [:details :err]))) + (is (= 2 @calls) "failed sequence must not dispatch any later split") + (is (false? (:valid? (check-lifecycle [start verify next])))))))) + +(deftest preflight-abandonment-cannot-advance-to-another-split + (let [current (atom 0), phase (atom "SPLIT_JOB_PHASE_ABANDONED")] + (with-redefs [shell/sh (fn [bin & args] + {:exit 0 :err "" + :out (cond + (= bin "elastickv-list-routes") (encode-snapshot (nth snapshots @current)) + (= "--get-job-id" (nth args 2)) (str "phase: " @phase "\n") + :else (let [step @current] + (swap! current inc) + (if (zero? step) "catalog_version: 8\n" "job_id: 44\n")))})] + (let [nem (split/repeated-split-nemesis opts)] + (invoke nem :start-repeated-split 0) + (invoke nem :verify-repeated-split 0) + (invoke nem :start-repeated-split 1) + (let [rejected (invoke nem :verify-repeated-split 1)] + (is (= :fail (:type rejected))) + (is (= "SPLIT_JOB_PHASE_ABANDONED" (get-in rejected [:details :observations 0 :phase])))) + (reset! phase "SPLIT_JOB_PHASE_DONE") + (is (= :fail (:type (invoke nem :start-repeated-split 2)))))))) + +(deftest controller-fails-closed-on-incomplete-or-invalid-completion + (doseq [phase ["SPLIT_JOB_PHASE_FENCE" "SPLIT_JOB_PHASE_CUTOVER" + "SPLIT_JOB_PHASE_FAILED" "SPLIT_JOB_PHASE_DONE" "UNKNOWN"]] + (let [current (atom 0)] + (with-redefs [shell/sh + (fn [bin & args] + {:exit 0 :err "" + :out (cond + (= bin "elastickv-list-routes") + (encode-snapshot (if (= 2 @current) + (assoc-in (nth snapshots 2) [:routes 2 :raft-group-id] 1) + (nth snapshots @current))) + (= "--get-job-id" (nth args 2)) (str "phase: " phase "\n") + :else (let [step @current] + (swap! current inc) + (if (zero? step) "catalog_version: 8\n" "job_id: 44\n")))})] + (let [nem (split/repeated-split-nemesis (assoc opts :split-timeout-ms -1))] + (is (= :info (:type (invoke nem :start-repeated-split 0)))) + (is (= :info (:type (invoke nem :verify-repeated-split 0)))) + (is (= :info (:type (invoke nem :start-repeated-split 1)))) + (is (= :fail (:type (invoke nem :verify-repeated-split 1))) phase) + (is (= :fail (:type (invoke nem :start-repeated-split 2))))))))) + +(deftest controller-does-not-dispatch-out-of-order-operations + (doseq [[f step] [[:verify-repeated-split 0] [:start-repeated-split 1]]] + (let [calls (atom 0)] + (with-redefs [shell/sh (fn [& _] (swap! calls inc))] + (is (= :fail (:type (invoke (split/repeated-split-nemesis opts) f step)))) + (is (zero? @calls)))))) + +(deftest repeated-registers-cover-every-final-partition + (let [keys (var-get (ns-resolve 'elastickv.split-workload 'repeated-register-keys))] + (doseq [r (:routes (last snapshots))] + (is (some #(and (not (neg? (compare % (:start r)))) + (or (nil? (:end r)) (neg? (compare % (:end r))))) keys))))) + +(deftest repeated-final-reads-require-every-key-from-every-node + (let [test {:concurrency 8 :nodes ["n1" "n2" "n3"]} + ctx (gen/context test) + reads (loop [g (split/repeated-final-reads 3), ops []] + (let [[op next-gen] (gen/op g test ctx)] + (if (or (nil? op) (= :pending op)) ops + (recur next-gen (conj ops (assoc (dissoc (into {} op) :index) + :type :ok :node (nth (:nodes test) (mod (:process op) 3)))))))) + writes (mapv (fn [k] {:process 0 :type :ok :f :write :value [k 1]}) (range 4)) + prefix (into writes (lifecycle-fixture)) + check #(checker/check (split/repeated-read-checker) test (history/history %) {})] + (is (= 12 (count reads))) + (is (every? :final reads)) + (is (true? (:valid? (check (into prefix reads))))) + (is (false? (:valid? (check (into prefix (pop reads)))))) + (is (false? (:valid? (check (into prefix (assoc-in reads [0 :type] :info)))))) + (is (false? (:valid? (check (into (into (pop writes) (lifecycle-fixture)) reads))))) + (is (false? (:valid? (check (into (into writes reads) (lifecycle-fixture)))))) + (is (false? (:valid? (check (into writes reads))))) + (is (false? (:valid? (check (into prefix (map #(dissoc % :node) reads)))))) + (is (true? (:valid? (check (into prefix (map #(update % :process + 8) reads))))) + "process rollover must not change the observed node"))) + +(deftest repeated-mode-is-opt-in-and-rejects-incompatible-settings + (is (= 6 (:concurrency (split/elastickv-split-test {:local true})))) + (is (= 8 (:concurrency (split/elastickv-split-test {:local true :repeat-splits true})))) + (is (thrown? clojure.lang.ExceptionInfo + (split/elastickv-split-test {:local true :repeat-splits true :abandon-at-fence true}))) + (doseq [concurrency [0 5 6 9]] + (is (thrown? clojure.lang.ExceptionInfo + (split/elastickv-split-test {:local true :repeat-splits true :concurrency concurrency}))))) diff --git a/jepsen/test/elastickv/split_workload_test.clj b/jepsen/test/elastickv/split_workload_test.clj new file mode 100644 index 000000000..74377af52 --- /dev/null +++ b/jepsen/test/elastickv/split_workload_test.clj @@ -0,0 +1,68 @@ +(ns elastickv.split-workload-test + (:require [clojure.java.shell :as shell] + [clojure.test :refer :all] + [elastickv.split-workload :as split] + [jepsen.checker :as checker] + [jepsen.history :as history] + [jepsen.nemesis :as nemesis])) + +(defn- routes-json [] + (str "{\"catalog_version\":7,\"routes\":[" + "{\"route_id\":100,\"raft_group_id\":1,\"start\":\"\",\"end\":\"\",\"state\":\"ROUTE_STATE_ACTIVE\"}" + "]}")) + +(deftest split-package-is-deterministic + (let [package (split/split-package {:split-at-seconds 1 + :split-verify-after-seconds 2})] + (is (some? (:generator package))) + (is (some? (:nemesis package))) + (is (= #{:start-cross-group-split :verify-cross-group-split} + (nemesis/fs (:nemesis package)))))) + +(deftest split-register-keys-cover-both-routes + (let [split-key (var-get (ns-resolve 'elastickv.split-workload 'split-key)) + register-keys (var-get (ns-resolve 'elastickv.split-workload 'register-keys))] + (is (some #(neg? (compare % split-key)) register-keys)) + (is (some #(not (neg? (compare % split-key))) register-keys)))) + +(deftest split-nemesis-starts-cross-group-job + (let [calls (atom [])] + (with-redefs [shell/sh (fn [& args] + (swap! calls conj args) + (if (.contains (first args) "list-routes") + {:exit 0 :out (routes-json) :err ""} + {:exit 0 :out "catalog_version: 8\njob_id: 44\n" :err ""}))] + (let [result (nemesis/invoke! + (split/split-nemesis + {:list-routes-bin "elastickv-list-routes" + :split-bin "elastickv-split" + :target-group-id 2 + :grpc-host-port "n1:50051"}) + {:nodes ["n1"] :raft-groups {1 50051, 2 50052}} + {:type :info :f :start-cross-group-split})] + (is (= 44 (get-in result [:value :job-id]))) + (is (= [["elastickv-list-routes" "--address" "n1:50051"] + ["elastickv-split" "--address" "n1:50051" + "--route-id" "100" "--split-key" "m2-split" + "--expected-version" "7" "--target-group-id" "2"]] + @calls)))))) + +(deftest split-run-requires-successful-lifecycle + (let [test (split/elastickv-split-test {:local true}) + start {:process :nemesis :type :info :f :start-cross-group-split + :value {:job-id 44}} + verify {:process :nemesis :type :info :f :verify-cross-group-split + :value {:job-id 44 :phase "SPLIT_JOB_PHASE_DONE"}} + check-history #(checker/check (:checker test) test (history/history %) {})] + (is (false? (:valid? (check-history [])))) + (is (false? (:valid? (check-history [start])))) + (is (false? (:valid? (check-history [(assoc start :type :fail) verify])))) + (is (false? (:valid? (check-history [start (assoc verify :type :fail)])))) + (is (false? (:valid? (check-history [verify start])))) + (is (false? (:valid? (check-history [start (assoc-in verify [:value :job-id] 45)])))) + (is (false? (:valid? (check-history [start (assoc-in verify [:value :phase] "SPLIT_JOB_PHASE_FENCE")])))) + (is (true? (:valid? (check-history [start verify])))) + (let [test (split/elastickv-split-test {:local true :abandon-at-fence true}) + check-history #(checker/check (:checker test) test (history/history %) {})] + (is (false? (:valid? (check-history [start verify])))) + (is (true? (:valid? (check-history [start (assoc-in verify [:value :phase] "SPLIT_JOB_PHASE_ABANDONED")]))))))) diff --git a/kv/compactor.go b/kv/compactor.go index c021a5b7c..89af85b39 100644 --- a/kv/compactor.go +++ b/kv/compactor.go @@ -234,6 +234,13 @@ func (c *FSMCompactor) compactRuntime(ctx context.Context, runtime FSMCompactRun if !ok { return nil } + safeMinTS, err := migrationRetentionMinTS(ctx, runtime.Store, safeMinTS) + if err != nil { + return errors.Wrapf(err, "read migration retention pins for group %d", runtime.GroupID) + } + if safeMinTS <= retention.MinRetainedTS() { + return nil + } compactCtx, cancel := c.compactContext(ctx, status) defer cancel() @@ -253,6 +260,23 @@ func (c *FSMCompactor) compactRuntime(ctx context.Context, runtime FSMCompactRun return nil } +func migrationRetentionMinTS(ctx context.Context, st store.MVCCStore, safeMinTS uint64) (uint64, error) { + reader, ok := st.(store.MigrationTargetReadinessReader) + if !ok { + return safeMinTS, nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return 0, errors.WithStack(err) + } + for _, state := range states { + if state.Armed && state.RetentionPinTS > 0 && state.RetentionPinTS < safeMinTS { + safeMinTS = state.RetentionPinTS + } + } + return safeMinTS, nil +} + func (c *FSMCompactor) compactionRuntimeReady(runtime FSMCompactRuntime) (store.RetentionController, raftengine.Status, bool) { if runtime.StatusReader == nil || runtime.Store == nil { return nil, raftengine.Status{}, false diff --git a/kv/compactor_test.go b/kv/compactor_test.go index 3c13f1b8a..021a200bb 100644 --- a/kv/compactor_test.go +++ b/kv/compactor_test.go @@ -547,3 +547,75 @@ func TestFSMCompactorCompactsEligiblePebbleRuntime(t *testing.T) { require.NoError(t, err) require.Equal(t, []byte("v30"), val) } + +type migrationPinnedCompactionStore struct { + deadlineCapturingStore + states []store.TargetStagedReadinessState +} + +func (s *migrationPinnedCompactionStore) MigrationTargetReadinessStates(context.Context) ([]store.TargetStagedReadinessState, error) { + return s.states, nil +} + +func TestFSMCompactorRespectsMigrationRetentionPin(t *testing.T) { + t.Parallel() + + st := &migrationPinnedCompactionStore{ + deadlineCapturingStore: deadlineCapturingStore{lastCommitTS: ^uint64(0)}, + states: []store.TargetStagedReadinessState{{ + JobID: 1, + MigrationJobID: 1, + MinWriteTSExclusive: 1, + Armed: true, + RetentionPinTS: 42, + }}, + } + compactor := NewFSMCompactor( + []FSMCompactRuntime{{ + GroupID: 1, + StatusReader: fakeRaftStatus{status: raftengine.Status{ + State: raftengine.StateFollower, + AppliedIndex: 1, + CommitIndex: 1, + }}, + Store: st, + }}, + WithFSMCompactorInterval(time.Hour), + WithFSMCompactorRetentionWindow(time.Millisecond), + ) + + require.NoError(t, compactor.SyncOnce(context.Background())) + require.True(t, st.compactCalled) + require.Equal(t, uint64(42), st.compactMinTS) +} + +func TestFSMCompactorIgnoresDisarmedMigrationRetentionPin(t *testing.T) { + t.Parallel() + + st := &migrationPinnedCompactionStore{ + deadlineCapturingStore: deadlineCapturingStore{lastCommitTS: ^uint64(0)}, + states: []store.TargetStagedReadinessState{{ + JobID: 1, + MigrationJobID: 1, + MinWriteTSExclusive: 1, + RetentionPinTS: 42, + }}, + } + compactor := NewFSMCompactor( + []FSMCompactRuntime{{ + GroupID: 1, + StatusReader: fakeRaftStatus{status: raftengine.Status{ + State: raftengine.StateFollower, + AppliedIndex: 1, + CommitIndex: 1, + }}, + Store: st, + }}, + WithFSMCompactorInterval(time.Hour), + WithFSMCompactorRetentionWindow(time.Millisecond), + ) + + require.NoError(t, compactor.SyncOnce(context.Background())) + require.True(t, st.compactCalled) + require.Greater(t, st.compactMinTS, uint64(42)) +} diff --git a/kv/coordinator_prepare_evidence.go b/kv/coordinator_prepare_evidence.go index ce460f35b..30a7d1eb7 100644 --- a/kv/coordinator_prepare_evidence.go +++ b/kv/coordinator_prepare_evidence.go @@ -39,7 +39,9 @@ func (c *ShardedCoordinator) prepareEvidenceMeta(ctx context.Context, primaryKey } func (c *ShardedCoordinator) prepareEvidenceTxnMeta(ctx context.Context, primaryKey []byte, commitTS uint64, groupIDs []uint64) (TxnMeta, error) { - meta := TxnMeta{PrimaryKey: primaryKey, LockTTLms: defaultTxnLockTTLms, PrepareCommitTS: commitTS} + // The legacy timestamp remains a migration-floor hint, not durable proof. + // Only the negotiated V3 field may be persisted into a prepared lock. + meta := TxnMeta{PrimaryKey: primaryKey, LockTTLms: defaultTxnLockTTLms, CommitTS: commitTS, PrepareCommitTS: commitTS} for _, id := range groupIDs { group, err := c.txnGroupForID(id) if err != nil { diff --git a/kv/forwarded_admin_payload.go b/kv/forwarded_admin_payload.go new file mode 100644 index 000000000..8126fb543 --- /dev/null +++ b/kv/forwarded_admin_payload.go @@ -0,0 +1,9 @@ +package kv + +// ValidateForwardedAdminPayload limits the forwarding RPC to the typed backup +// and PREPARE-evidence entries used by leaderAdminProposer. In particular, the +// RPC is not an alternative authorization path for user or migration proposals. +func ValidateForwardedAdminPayload(payload []byte) error { + _, err := decodeBackupEntry(payload) + return err +} diff --git a/kv/fsm.go b/kv/fsm.go index 40577c6d2..6a92d7258 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -118,6 +118,9 @@ type kvFSM struct { prepareCreationSealed atomic.Bool prepareEvidenceReleaseFloor atomic.Uint64 prepareEvidenceTimestampFloor atomic.Uint64 + migrationConfigurationIndex atomic.Uint64 + migrationNamespaceVersion atomic.Uint32 + migrationNamespaceMembers []uint64 // serial Raft configuration/apply loop only } // RouteHistory is the kv-side interface to the route catalog's @@ -152,9 +155,12 @@ type RouteSnapshot interface { // OwnerOf returns the Raft group ID that owned key at this // snapshot's version. (0, false) when no route covered key. OwnerOf(key []byte) (uint64, bool) - // RouteOf returns the complete route descriptor covering key. + // RouteOf returns the complete route descriptor covering key at this + // snapshot's version. Used by apply-time target-readiness checks to + // prove staged/cleared descriptor state before mutating the store. RouteOf(key []byte) (distribution.Route, bool) - // IntersectingRoutes returns every route intersecting [start, end). + // IntersectingRoutes returns the route descriptors that intersect + // [start, end) at this snapshot's version. A nil end denotes +infinity. IntersectingRoutes(start, end []byte) []distribution.Route // WriteFencedForKey reports whether key is currently inside a // WriteFenced route in this snapshot. @@ -162,6 +168,12 @@ type RouteSnapshot interface { // WriteFencedIntersects reports whether [start, end) intersects // any WriteFenced route in this snapshot. WriteFencedIntersects(start, end []byte) bool + // WriteFloorForKey returns the post-migration write timestamp floor + // for key, when the current route retains one. + WriteFloorForKey(key []byte) (uint64, bool) + // WriteFloorIntersects returns the maximum post-migration write + // timestamp floor across routes intersecting [start, end). + WriteFloorIntersects(start, end []byte) (uint64, bool) } // SetApplyIndex implements raftengine.ApplyIndexAware. The engine @@ -425,6 +437,12 @@ func (f *kvFSM) applyBatchedRequests(ctx context.Context, reqs []*pb.Request, re }) if err != nil { f.discardDeferredApplyNotices() + if classified := haltResponseFor(err); classified != nil { + return haltErr(errors.Wrap( + errors.Mark(classified.HaltApply(), ErrRaftEntryApply), + "kv/fsm: apply batched raft entry", + )), true + } return haltErr(errors.Wrap(errors.Mark(err, ErrRaftEntryApply), "kv/fsm: apply batched raft entry")), true } f.flushDeferredApplyNotices() @@ -453,17 +471,26 @@ func firstOperationalRequestFailure(results []error) error { // isRawApplyVerdict reports whether err is a deterministic verdict of the // raw apply path, on the FSM side or the store side. func isRawApplyVerdict(err error) bool { + if errors.Is(err, errTargetReadinessUnproven) { + return false + } return errors.Is(err, ErrInvalidRequest) || + errors.Is(err, ErrMigrationTransferTooLarge) || + errors.Is(err, ErrRouteCutoverPending) || errors.Is(err, ErrUnknownRequestType) || errors.Is(err, ErrBackupTimestampFenced) || errors.Is(err, ErrPrepareCreationSealed) || - errors.Is(err, ErrRouteWriteFenced) || - errors.Is(err, ErrRouteWriteTimestampTooLow) || - errors.Is(err, ErrComposed1VersionGCd) || + isRawRouteVerdict(err) || errors.Is(err, ErrTxnLocked) || store.IsApplyVerdict(err) } +func isRawRouteVerdict(err error) bool { + return errors.Is(err, ErrRouteWriteFenced) || + errors.Is(err, ErrRouteWriteTimestampTooLow) || + errors.Is(err, ErrComposed1VersionGCd) +} + func (f *kvFSM) applyRequestsOneByOne(ctx context.Context, reqs []*pb.Request, results []error) bool { hasError := false for i, req := range reqs { @@ -512,16 +539,30 @@ func (f *kvFSM) applyReservedOpcode(ctx context.Context, data []byte) (any, bool switch { case data[0] == raftEncodeHLCLease: return f.applyHLCLease(data[1:]), true - case data[0] == raftEncodeMigrationImport: - return f.applyMigrationImport(ctx, data[1:]), true - case data[0] == raftEncodeMigrationRetire: - return f.applyMigrationRetire(ctx, data[1:]), true - case data[0] == raftEncodeMigrationPromote: - return f.applyMigrationPromote(ctx, data[1:]), true case data[0] == raftEncodeBackup: return f.applyBackup(data[1:]), true case data[0] >= fsmwire.OpEncryptionMin && data[0] <= fsmwire.OpEncryptionMax: return f.applyEncryption(f.pendingApplyIdx, data[0], data[1:]), true + default: + return f.applyMigrationOpcode(ctx, data) + } +} + +// applyMigrationOpcode handles the range-migration half of the reserved opcode +// space. Split out of applyReservedOpcode purely to keep that switch inside the +// cyclop budget; the two together are still one flat dispatch. +func (f *kvFSM) applyMigrationOpcode(ctx context.Context, data []byte) (any, bool) { + switch data[0] { + case raftEncodeMigrationImport: + return f.applyMigrationImport(ctx, data[1:]), true + case raftEncodeMigrationRetire: + return f.applyMigrationRetire(ctx, data[1:]), true + case raftEncodeMigrationPromote: + return f.applyMigrationPromote(ctx, data[1:]), true + case raftEncodeTargetReadiness: + return f.applyTargetStagedReadiness(ctx, data[1:]), true + case raftEncodeMigrationCleanup: + return f.applyMigrationCleanup(ctx, data[1:]), true default: return nil, false } @@ -562,6 +603,23 @@ const ( // data promotion chunk. Every target voter atomically copies staged MVCC // versions into the live keyspace and removes the promoted staged rows. raftEncodeMigrationPromote byte = 0x0b + // raftEncodeTargetReadiness carries the target-local staged-readiness + // guard. It must be replicated through the target Raft group before the + // migration controller can treat a target as fail-closed for cutover. + // + // 0x0c was this opcode's value while this branch was open, but the base + // branch independently assigned 0x0c to raftEncodeMigrationRetire. The + // base merges first, so the readiness guard takes the next free value + // rather than aliasing an opcode the apply switch already matches -- a + // collision here misroutes committed entries, which is unrecoverable. + // + // 0x0a is deliberately skipped: a bare proto-marshalled RaftCommand starts + // with 0x0a (field 1, wire type 2), which applyReservedOpcode would then + // swallow. TestFSMApplyBatchKeepsPerRequestResults pins that. + raftEncodeTargetReadiness byte = 0x0f + // raftEncodeMigrationCleanup carries one bounded source/target physical + // cleanup chunk or a per-job proof-record cleanup. + raftEncodeMigrationCleanup byte = 0x0d ) func decodeRaftRequests(data []byte) ([]*pb.Request, error) { @@ -621,14 +679,89 @@ func requestCommitTS(r *pb.Request) (uint64, error) { return commitTS, nil } +// ErrTargetReadinessApply marks an apply that could not prove target-staged +// readiness. It is deliberately a halt rather than an ordinary response: the +// entry is already committed, so a voter that returns an ordinary error here +// advances its applied index and permanently skips a write the rest of the +// cluster performed. +// +// The underlying readiness check still reads the catalog watcher's current view +// (f.routes.Current()), which is process-local: it arrives over the cross-group +// catalog stream with no ordering relationship to this Raft log, so two voters +// at the same index can legitimately disagree. Halting converts that silent +// divergence into a loud stop. It does not make apply deterministic -- see +// docs/design/2026_09_05_proposed_apply_time_readiness_evidence.md for that. +var ErrTargetReadinessApply = errors.New("target staged readiness: FSM apply could not prove readiness; halting apply") + +// errTargetReadinessUnproven marks the ONE ErrRouteCutoverPending verdict that +// depends on process-local state: the target-readiness proof, which reads the +// catalog watcher's current view. +// +// The source-read fence returns the same sentinel but is a replicated, +// deterministic verdict -- every replica reads the same readiness states from +// its own store and reaches the same answer -- so it is an ordinary transaction +// rejection. Halting on it would stop every source replica during a normal +// cutover, which is why the halt keys off this mark rather than off +// ErrRouteCutoverPending itself. +var errTargetReadinessUnproven = errors.New("target staged readiness unproven from the local catalog view") + +// errMigrationTrackerPersistence marks process-local failures while recording +// the minimum admitted write timestamp for an armed migration. A voter that +// advances past the entry without this record can later report a higher safe +// frontier than peers that persisted it. +var errMigrationTrackerPersistence = errors.New("migration write tracker persistence failed") + func (f *kvFSM) applyRequest(ctx context.Context, r *pb.Request) any { if err := f.applyRequestErr(ctx, r); err != nil { - return err + return applyErrorResponse(err) + } + return nil +} + +// applyErrorResponse decides whether an apply error is an ordinary rejection or +// has to halt this replica. +// +// Only a verdict that depended on process-local state halts: the replica cannot +// know whether its peers reached the same answer, so advancing past the entry +// risks silent divergence. Every deterministic rejection -- route fences, +// timestamp floors, malformed requests -- stays ordinary, because every replica +// reaches it for the same entry and halting would stop the group during normal +// operation. +func applyErrorResponse(err error) any { + if halt := haltResponseFor(err); halt != nil { + return halt + } + return err +} + +// haltResponseFor returns the halt response for an apply error that must stop +// this replica, or nil when the error is an ordinary rejection. +// +// Both Apply paths -- the single-request one and the batch loop -- consult this +// so the classification cannot drift between them. It did: the batch loop used +// to record every error as a per-request result, so a coalesced write carrying +// an unproven readiness verdict advanced the applied index instead of halting. +func haltResponseFor(err error) *haltApplyResponse { + if errors.Is(err, errTargetReadinessUnproven) { + return haltErr(errors.Wrap(errors.Mark(err, ErrTargetReadinessApply), "kv/fsm: apply target readiness")) + } + if errors.Is(err, errMigrationTrackerPersistence) { + return haltErr(errors.Wrap(errors.Mark(err, ErrTargetReadinessApply), "kv/fsm: persist migration write tracker")) } return nil } func (f *kvFSM) applyRequestErr(ctx context.Context, r *pb.Request) error { + if f.migrationNamespaceVersion.Load() != 0 { + if err := verifyRequestMigrationControlKeys(r); err != nil { + return err + } + } + var err error + ctx, err = f.withMigrationOwnershipEvidence(ctx, r) + if err != nil { + return err + } if err := f.verifyPrepareCreationGate(r); err != nil { return err } @@ -656,10 +789,10 @@ func (f *kvFSM) handleRequest(ctx context.Context, r *pb.Request, commitTS uint6 } func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS uint64) error { - if err := f.verifyWriteFence(r); err != nil { + if err := f.verifyWriteFence(ctx, r); err != nil { return err } - floorSnap, err := f.routeFloorSnapshotForRequest(r) + floorSnap, err := f.routeFloorSnapshotForRequest(ctx, r) if err != nil { return err } @@ -673,11 +806,16 @@ func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS ui if err := f.validateRawMutationsForApply(ctx, r, commitTS, floorSnap); err != nil { return err } - muts, err := toStoreMutations(r.Mutations) if err != nil { return errors.WithStack(err) } + if err := f.verifyMigrationTransferSize(ctx, muts); err != nil { + return err + } + if err := f.recordMigrationWrite(ctx, r.Mutations, commitTS); err != nil { + return err + } // Raw requests always commit against the latest state; use commitTS as both // the validation snapshot and the commit timestamp. if err := f.store.ApplyMutationsRaftAt(ctx, muts, nil, commitTS, commitTS, f.pendingApplyIdx); err != nil { @@ -712,12 +850,7 @@ func (f *kvFSM) validateRawMutationForApply(ctx context.Context, mut *pb.Mutatio if distribution.IsReservedControlKey(mut.Key) { return errors.WithStack(ErrInvalidRequest) } - if _, bypass := writeFenceBypassKeys[string(mut.Key)]; !bypass { - if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { - return err - } - } - if err := verifyRouteWriteTimestampFloorForKeyFromSnapshot(floorSnap, mut.Key, commitTS); err != nil { + if err := f.verifyRawMutationFencesAndFloors(ctx, mut.Key, writeFenceBypassKeys, commitTS, floorSnap); err != nil { return err } if err := f.assertNoConflictingTxnLock(ctx, mut.Key, nil, 0); err != nil { @@ -726,6 +859,39 @@ func (f *kvFSM) validateRawMutationForApply(ctx context.Context, mut *pb.Mutatio return nil } +// verifyRawMutationFencesAndFloors runs the fence, floor and migration-readiness +// gates for one raw mutation key. Split out of validateRawMutationForApply to +// keep that function inside the cyclop budget. +func (f *kvFSM) verifyRawMutationFencesAndFloors( + ctx context.Context, + key []byte, + writeFenceBypassKeys map[string]struct{}, + commitTS uint64, + floorSnap RouteSnapshot, +) error { + if err := f.verifySourceWriteFenceForRange(ctx, key, nextScanCursor(key)); err != nil { + return err + } + if _, bypass := writeFenceBypassKeys[string(key)]; !bypass { + if err := f.verifyRouteNotFencedForKey(ctx, key); err != nil { + return err + } + } + if err := verifyRouteWriteTimestampFloorForKeyFromSnapshot(floorSnap, key, commitTS); err != nil { + return err + } + // A pinned request is governed by its observed snapshot alone -- that is the + // whole point of pinning, and re-checking the current snapshot would reject + // writes the pinned view admits. An unpinned apply has no observed view, so + // the current snapshot's floor is the only one that can bound it. + if floorSnap == nil { + if err := f.verifyRouteWriteFloorForKey(ctx, key, commitTS); err != nil { + return err + } + } + return f.verifyTargetReadinessForRange(ctx, key, nextScanCursor(key)) +} + // extractDelPrefix checks if the mutations contain a DEL_PREFIX operation. // If found, it validates that no other operation types are mixed in. func extractDelPrefix(muts []*pb.Mutation) (bool, []byte) { @@ -750,17 +916,28 @@ func (f *kvFSM) handleDelPrefixWithFloorSnapshot(ctx context.Context, prefix []b if distribution.ReservedControlPrefixIntersects(prefix) { return errors.WithStack(ErrInvalidRequest) } - if err := f.verifyRouteNotFencedForPrefix(prefix); err != nil { + if err := f.verifyRouteNotFencedForPrefix(ctx, prefix); err != nil { + return err + } + if err := f.verifyTargetReadinessForPrefix(ctx, prefix); err != nil { return err } if err := verifyRouteWriteTimestampFloorForPrefixFromSnapshot(floorSnap, prefix, commitTS); err != nil { return err } + if floorSnap == nil { + if err := f.verifyRouteWriteFloorForPrefix(ctx, prefix, commitTS); err != nil { + return err + } + } + if err := f.recordMigrationWrite(ctx, []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: prefix}}, commitTS); err != nil { + return err + } deletes := []store.PrefixDelete{{ Prefix: prefix, ExcludePrefix: txnCommonPrefix, }} - for _, del := range f.stagedVisibilityPrefixDeletesForApply(prefix, txnCommonPrefix) { + for _, del := range f.stagedVisibilityPrefixDeletesForApply(ctx, prefix, txnCommonPrefix) { deletes = append(deletes, store.PrefixDelete{ Prefix: del.prefix, ExcludePrefix: del.excludePrefix, @@ -773,15 +950,16 @@ func (f *kvFSM) handleDelPrefixWithFloorSnapshot(ctx context.Context, prefix []b return nil } -func (f *kvFSM) routeFloorSnapshotForRequest(r *pb.Request) (RouteSnapshot, error) { - if f.routes == nil || f.shardGroupID == 0 { +func (f *kvFSM) routeFloorSnapshotForRequest(ctx context.Context, r *pb.Request) (RouteSnapshot, error) { + routes := f.routeHistoryForContext(ctx) + if routes == nil || f.shardGroupID == 0 { return nil, nil } observedVer, pinned := DecodeObservedRouteVersion(r.GetObservedRouteVersion()) if !pinned { return nil, nil } - snap, ok := f.routes.SnapshotAt(observedVer) + snap, ok := routes.SnapshotAt(observedVer) if !ok { return nil, errors.WithStack(ErrComposed1VersionGCd) } @@ -793,11 +971,15 @@ type fsmStagedVisibilityPrefixDelete struct { excludePrefix []byte } -func (f *kvFSM) stagedVisibilityPrefixDeletesForApply(prefix []byte, excludePrefix []byte) []fsmStagedVisibilityPrefixDelete { - if f == nil || f.routes == nil || f.shardGroupID == 0 { +func (f *kvFSM) stagedVisibilityPrefixDeletesForApply(ctx context.Context, prefix []byte, excludePrefix []byte) []fsmStagedVisibilityPrefixDelete { + if f == nil || f.shardGroupID == 0 { + return nil + } + history := f.routeHistoryForContext(ctx) + if history == nil { return nil } - snap, ok := f.routes.Current() + snap, ok := history.Current() if !ok { return nil } @@ -824,11 +1006,30 @@ func (f *kvFSM) stagedVisibilityPrefixDeletesForApply(prefix []byte, excludePref return out } -func (f *kvFSM) verifyRouteNotFencedForKey(key []byte) error { - if f.routes == nil { +func (f *kvFSM) verifyRouteNotFencedForMutations(ctx context.Context, muts []*pb.Mutation, bypassKeys map[string]struct{}) error { + for _, mut := range muts { + if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { + continue + } + if err := f.verifySourceWriteFenceForRange(ctx, mut.Key, nextScanCursor(mut.Key)); err != nil { + return err + } + if _, bypass := bypassKeys[string(mut.Key)]; bypass { + continue + } + if err := f.verifyRouteNotFencedForKey(ctx, mut.Key); err != nil { + return err + } + } + return nil +} + +func (f *kvFSM) verifyRouteNotFencedForKey(ctx context.Context, key []byte) error { + routes := f.routeHistoryForContext(ctx) + if routes == nil { return nil } - snap, ok := f.routes.Current() + snap, ok := routes.Current() if !ok { return nil } @@ -846,21 +1047,47 @@ func (f *kvFSM) verifyRouteNotFencedForKey(key []byte) error { return nil } -func (f *kvFSM) verifyRouteNotFencedForPrefix(prefix []byte) error { - if f.routes == nil { +func (f *kvFSM) verifyRouteNotFencedForPrefix(ctx context.Context, prefix []byte) error { + start, end := routePrefixRange(prefix) + if err := f.verifySourceWriteFenceForRouteRange(ctx, start, end); err != nil { + return err + } + routes := f.routeHistoryForContext(ctx) + if routes == nil { return nil } - snap, ok := f.routes.Current() + snap, ok := routes.Current() if !ok { return nil } - start, end := routePrefixRange(prefix) if !snap.WriteFencedIntersects(start, end) { return nil } return errors.Wrapf(ErrRouteWriteFenced, "prefix %q route range [%q,%q)", prefix, start, end) } +func verifyRouteWriteTimestampFloorForRoute(route distribution.Route, key []byte, commitTS uint64) error { + if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { + return nil + } + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", key, routeKey(key), commitTS, route.MinWriteTSExclusive) +} + +func verifyRouteWriteTimestampFloorForRange(route distribution.Route, key, start, end []byte, commitTS uint64) error { + if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { + return nil + } + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q route range [%q,%q) commit_ts=%d floor=%d", key, start, end, commitTS, route.MinWriteTSExclusive) +} + +func uniqueTxnMutations(muts []*pb.Mutation) ([]*pb.Mutation, error) { + uniq, err := uniqueMutations(muts) + if err != nil { + return nil, err + } + return uniq, nil +} + func verifyRouteWriteTimestampFloorForKeyFromSnapshot(snap RouteSnapshot, key []byte, commitTS uint64) error { if snap == nil || commitTS == 0 { return nil @@ -906,18 +1133,257 @@ func verifyRouteWriteTimestampFloorForPrefixFromSnapshot(snap RouteSnapshot, pre return nil } -func verifyRouteWriteTimestampFloorForRoute(route distribution.Route, key []byte, commitTS uint64) error { - if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { +func (f *kvFSM) verifyRouteWriteFloorForMutations(ctx context.Context, muts []*pb.Mutation, commitTS uint64) error { + for _, mut := range muts { + if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { + continue + } + if err := f.verifyRouteWriteFloorForKey(ctx, mut.Key, commitTS); err != nil { + return err + } + } + return nil +} + +func (f *kvFSM) verifySourceWriteFenceForRange(ctx context.Context, start []byte, end []byte) error { + routeStart, routeEnd := readinessRouteRangeForScan(start, end) + return f.verifySourceWriteFenceForRouteRange(ctx, routeStart, routeEnd) +} + +func (f *kvFSM) verifySourceWriteFenceForRouteRange(ctx context.Context, routeStart []byte, routeEnd []byte) error { + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { return nil } - return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", key, routeKey(key), commitTS, route.MinWriteTSExclusive) + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + for _, state := range states { + if state.Armed && state.SourceWriteFence && routeRangeIntersects(routeStart, routeEnd, state.RouteStart, state.RouteEnd) { + return errors.Wrapf(ErrRouteWriteFenced, "source migration fence job %d route range [%q,%q)", state.JobID, routeStart, routeEnd) + } + } + return nil } -func verifyRouteWriteTimestampFloorForRange(route distribution.Route, key, start, end []byte, commitTS uint64) error { - if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { +func (f *kvFSM) verifyTargetReadinessForMutations(ctx context.Context, muts []*pb.Mutation) error { + for _, mut := range muts { + if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { + continue + } + if err := f.verifyTargetReadinessForRange(ctx, mut.Key, nextScanCursor(mut.Key)); err != nil { + return err + } + } + return nil +} + +func (f *kvFSM) verifyTargetReadinessForReadKeys(ctx context.Context, keys [][]byte) error { + for _, key := range keys { + if isTxnInternalKey(key) { + continue + } + if err := f.verifySourceReadFenceForRange(ctx, key, nextScanCursor(key)); err != nil { + return err + } + if err := f.verifyTargetReadinessForRange(ctx, key, nextScanCursor(key)); err != nil { + return err + } + } + return nil +} + +func (f *kvFSM) verifySourceReadFenceForRange(ctx context.Context, start []byte, end []byte) error { + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { return nil } - return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q route range [%q,%q) commit_ts=%d floor=%d", key, start, end, commitTS, route.MinWriteTSExclusive) + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + routeStart, routeEnd := readinessRouteRangeForScan(start, end) + if sourceReadFenceApplies(states, routeStart, routeEnd) { + return errors.WithStack(ErrRouteCutoverPending) + } + return nil +} + +func (f *kvFSM) verifyTargetReadinessForTxnFootprint(ctx context.Context, muts []*pb.Mutation, readKeys [][]byte, primaryKey []byte) error { + if len(primaryKey) != 0 && !isTxnInternalKey(primaryKey) { + if err := f.verifyTargetReadinessForRange(ctx, primaryKey, nextScanCursor(primaryKey)); err != nil { + return err + } + } + if err := f.verifyTargetReadinessForReadKeys(ctx, readKeys); err != nil { + return err + } + return f.verifyTargetReadinessForMutations(ctx, muts) +} + +func (f *kvFSM) verifyTargetReadinessForPrefix(ctx context.Context, prefix []byte) error { + start, end := routePrefixRange(prefix) + return f.verifyTargetReadinessForRouteRange(ctx, start, end) +} + +func (f *kvFSM) verifyTargetReadinessForRange(ctx context.Context, start []byte, end []byte) error { + routeStart, routeEnd := readinessRouteRange(start, end) + return f.verifyTargetReadinessForRouteRange(ctx, routeStart, routeEnd) +} + +func (f *kvFSM) verifyTargetReadinessForRouteRange(ctx context.Context, routeStart []byte, routeEnd []byte) error { + _, err := f.targetReadyRoutesForRouteRange(ctx, routeStart, routeEnd) + return err +} + +func (f *kvFSM) targetReadyRoutesForRange(ctx context.Context, start []byte, end []byte) ([]distribution.Route, error) { + routeStart, routeEnd := readinessRouteRange(start, end) + return f.targetReadyRoutesForRouteRange(ctx, routeStart, routeEnd) +} + +func (f *kvFSM) targetReadyRoutesForRouteRange(ctx context.Context, routeStart []byte, routeEnd []byte) ([]distribution.Route, error) { + routes, catalogVersion, proof := f.currentShardRoutesForRouteRange(ctx, routeStart, routeEnd) + + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { + return routes, nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + // A local store read that failed on this replica only. Returning it as + // an ordinary error lets this voter advance past an entry its healthy + // peers applied, so it is marked for the same halt as the unprovable + // verdict below. + return nil, errors.WithStack(errors.Mark(err, errTargetReadinessUnproven)) + } + if len(states) == 0 { + return routes, nil + } + if targetReadinessStatesSatisfied(states, routes, routeStart, routeEnd, f.shardGroupID, catalogVersion, proof) { + return routes, nil + } + // This verdict rests on routes/catalogVersion, which came from the catalog + // watcher's current view -- process-local. Mark it so apply can halt on this + // one and only this one; see errTargetReadinessUnproven. + return nil, errors.WithStack(errors.Mark(ErrRouteCutoverPending, errTargetReadinessUnproven)) +} + +func (f *kvFSM) currentShardRoutesForRouteRange(ctx context.Context, routeStart []byte, routeEnd []byte) ([]distribution.Route, uint64, bool) { + history := f.routeHistoryForContext(ctx) + if history == nil { + return nil, 0, false + } + snap, ok := history.Current() + if !ok { + return nil, 0, false + } + routes := make([]distribution.Route, 0) + for _, route := range snap.IntersectingRoutes(routeStart, routeEnd) { + if route.GroupID == f.shardGroupID { + routes = append(routes, route) + } + } + return routes, snap.Version(), true +} + +func targetReadinessStatesSatisfied( + states []store.TargetStagedReadinessState, + routes []distribution.Route, + routeStart []byte, + routeEnd []byte, + groupID uint64, + catalogVersion uint64, + proof bool, +) bool { + for _, ready := range states { + if !ready.Armed || ready.SourceWriteFence || ready.SourceReadFence || ready.TrackWrites || + !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { + continue + } + if !proof || !routesSatisfyTargetReadiness(routes, ready, groupID, catalogVersion) { + return false + } + } + return true +} + +func routesSatisfyTargetReadiness(routes []distribution.Route, ready store.TargetStagedReadinessState, groupID uint64, catalogVersion uint64) bool { + matched := false + for _, route := range routes { + if route.GroupID != groupID { + continue + } + if !routeRangeIntersects(route.Start, route.End, ready.RouteStart, ready.RouteEnd) { + continue + } + matched = true + if !routeSatisfiesTargetReadiness(route, ready, catalogVersion) { + return false + } + } + return matched +} + +func (f *kvFSM) verifyRouteWriteFloorForKey(ctx context.Context, key []byte, commitTS uint64) error { + routes := f.routeHistoryForContext(ctx) + if routes == nil { + return nil + } + snap, ok := routes.Current() + if !ok { + return nil + } + if err := f.verifyS3BucketAuxiliaryRouteWriteFloor(snap, key, commitTS); err != nil { + return err + } + if _, _, ok := s3BucketAuxiliaryRouteRange(key); ok { + return nil + } + rkey := routeKey(key) + floor, ok := snap.WriteFloorForKey(rkey) + if ok && commitTS != 0 && commitTS <= floor { + return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d for key %q routeKey %q", commitTS, floor, key, rkey) + } + return nil +} + +// verifyS3BucketAuxiliaryRouteWriteFloor consults only the route that OWNS the +// auxiliary range, not every route the range happens to intersect. A floor on a +// non-owner sibling must not block the owner's writes -- the same correction +// verifyRouteNotFencedForKey makes for fences. +func (f *kvFSM) verifyS3BucketAuxiliaryRouteWriteFloor(snap RouteSnapshot, key []byte, commitTS uint64) error { + if commitTS == 0 { + return nil + } + start, end, ok := s3BucketAuxiliaryRouteRange(key) + if !ok { + return nil + } + route, found := s3BucketAuxiliaryOwnerRouteFromRange(start, end, snap.IntersectingRoutes(start, end)) + if !found || route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { + return nil + } + return errors.Wrapf(ErrRouteWriteBelowFloor, + "commit_ts %d <= floor %d for key %q route range [%q,%q)", + commitTS, route.MinWriteTSExclusive, key, start, end) +} + +func (f *kvFSM) verifyRouteWriteFloorForPrefix(ctx context.Context, prefix []byte, commitTS uint64) error { + routes := f.routeHistoryForContext(ctx) + if routes == nil || commitTS == 0 { + return nil + } + snap, ok := routes.Current() + if !ok { + return nil + } + start, end := routePrefixRange(prefix) + floor, ok := snap.WriteFloorIntersects(start, end) + if !ok || commitTS > floor { + return nil + } + return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d for prefix %q route range [%q,%q)", commitTS, floor, prefix, start, end) } func routePrefixRange(prefix []byte) ([]byte, []byte) { @@ -1050,6 +1516,10 @@ func (f *kvFSM) Snapshot() (raftengine.Snapshot, error) { } func (f *kvFSM) Restore(r io.Reader) error { + return f.restoreSnapshot(r, f.store.Restore) +} + +func (f *kvFSM) restoreSnapshot(r io.Reader, restoreStore func(io.Reader) error) error { // Stage 8a §3.2: the bufio.Reader is owned by the caller (us) and // MUST be passed unchanged to the inner-store path on every branch // — buffered bytes can sit between the header consumption and the @@ -1065,6 +1535,9 @@ func (f *kvFSM) Restore(r io.Reader) error { if err != nil { return errors.WithStack(err) } + if err := restoreStore(io.NopCloser(br)); err != nil { + return errors.WithStack(err) + } if f.hlc != nil && ceilingU > 0 { f.hlc.SetPhysicalCeiling(int64(ceilingU)) //nolint:gosec // ceiling is a Unix ms timestamp encoded as uint64 } @@ -1072,9 +1545,6 @@ func (f *kvFSM) Restore(r io.Reader) error { // 6E's apply-hook to read on the next apply. Stays 0 for v1 and // headerless-legacy restores. f.restoredCutover = cutover - if err := f.store.Restore(io.NopCloser(br)); err != nil { - return errors.WithStack(err) - } checkpoint, err = f.checkpointForDurableFSM(checkpoint, false) if err != nil { return err @@ -1163,10 +1633,7 @@ func (f *kvFSM) IsVolatileOnlyPayload(payload []byte) bool { } func (f *kvFSM) handleTxnRequest(ctx context.Context, r *pb.Request, commitTS uint64) error { - if err := f.verifyWriteFence(r); err != nil { - return err - } - if err := f.verifyComposed1(r); err != nil { + if err := f.verifyTxnRouteAdmission(ctx, r); err != nil { return err } switch r.Phase { @@ -1183,6 +1650,16 @@ func (f *kvFSM) handleTxnRequest(ctx context.Context, r *pb.Request, commitTS ui } } +func (f *kvFSM) verifyTxnRouteAdmission(ctx context.Context, r *pb.Request) error { + if migrationResolutionAuthorized(ctx, r) { + return nil + } + if err := f.verifyWriteFence(ctx, r); err != nil { + return err + } + return f.verifyComposed1(ctx, r) +} + // verifyComposed1 is the M3 apply-time Composed-1 gate per // docs/design/2026_05_29_implemented_composed1_cross_group_commit_guard.md // §4.2(a) + §4.4. Runs two checks before the txn's writes land: @@ -1212,7 +1689,7 @@ func (f *kvFSM) handleTxnRequest(ctx context.Context, r *pb.Request, commitTS ui // Returns ErrComposed1VersionGCd when the observed version is // outside the ring (M4 retry), and ErrComposed1Violation wrapped // with per-key context otherwise. -func (f *kvFSM) verifyComposed1(r *pb.Request) error { +func (f *kvFSM) verifyComposed1(ctx context.Context, r *pb.Request) error { // ABORT requests MUST always reach the abort handler so the // txn's intent locks get released. If a route shifted between // PREPARE and ABORT (or the observed version was evicted), a @@ -1235,7 +1712,8 @@ func (f *kvFSM) verifyComposed1(r *pb.Request) error { // to silently reject every pinned txn — no real Raft group // has ID 0, so OwnerOf would never match (coderabbit MAJOR on // PR #895). - if f.routes == nil || f.shardGroupID == 0 { + routes := f.routeHistoryForContext(ctx) + if routes == nil || f.shardGroupID == 0 { return nil } observedVer, pinned := DecodeObservedRouteVersion(r.GetObservedRouteVersion()) @@ -1245,7 +1723,7 @@ func (f *kvFSM) verifyComposed1(r *pb.Request) error { bypassKeys := writeFenceBypassKeySet(r.GetWriteFenceBypassKeys()) // (a) Observed-version check. - observedSnap, ok := f.routes.SnapshotAt(observedVer) + observedSnap, ok := routes.SnapshotAt(observedVer) if !ok { return errors.WithStack(ErrComposed1VersionGCd) } @@ -1254,7 +1732,7 @@ func (f *kvFSM) verifyComposed1(r *pb.Request) error { } // (b) Current-version cross-version-read fence. - currentSnap, ok := f.routes.Current() + currentSnap, ok := routes.Current() if !ok { // No current snapshot — engine has no history, nothing // to compare against. Fall through (matches the @@ -1264,21 +1742,22 @@ func (f *kvFSM) verifyComposed1(r *pb.Request) error { return f.verifyOwnerFromSnapshot(r.GetMutations(), bypassKeys, currentSnap, currentSnap.Version(), "current") } -func (f *kvFSM) verifyWriteFence(r *pb.Request) error { +func (f *kvFSM) verifyWriteFence(ctx context.Context, r *pb.Request) error { if requestBypassesWriteFence(r) { return nil } observedVer := r.GetObservedRouteVersion() - if !f.writeFenceHistoryReady() { + if !f.writeFenceHistoryReady(ctx) { return nil } - currentSnap, ok := f.routes.Current() + routes := f.routeHistoryForContext(ctx) + currentSnap, ok := routes.Current() if !ok { return nil } if observedVer != 0 { - observedSnap, ok := f.routes.SnapshotAt(observedVer) + observedSnap, ok := routes.SnapshotAt(observedVer) if !ok { return errors.WithStack(ErrComposed1VersionGCd) } @@ -1306,8 +1785,8 @@ func requestBypassesWriteFence(r *pb.Request) bool { return false } -func (f *kvFSM) writeFenceHistoryReady() bool { - return f.routes != nil && f.shardGroupID != 0 +func (f *kvFSM) writeFenceHistoryReady(ctx context.Context) bool { + return f.routeHistoryForContext(ctx) != nil && f.shardGroupID != 0 } func verifyWriteFenceFromSnapshot(mutations []*pb.Mutation, writeFenceBypassKeys [][]byte, snap RouteSnapshot, snapVer uint64, phase string) error { @@ -1419,7 +1898,7 @@ func (f *kvFSM) validateConflicts(ctx context.Context, muts []*pb.Mutation, star } seen[keyStr] = struct{}{} - latest, exists, err := f.store.LatestCommitTS(ctx, mut.Key) + latest, exists, err := f.latestCommitTSForTargetReadyKey(ctx, mut.Key) if err != nil { return errors.WithStack(err) } @@ -1430,6 +1909,64 @@ func (f *kvFSM) validateConflicts(ctx context.Context, muts []*pb.Mutation, star return nil } +func (f *kvFSM) validateReadConflicts(ctx context.Context, keys [][]byte, startTS uint64) error { + seen := make(map[string]struct{}, len(keys)) + for _, key := range keys { + if len(key) == 0 || isTxnInternalKey(key) { + continue + } + keyStr := string(key) + if _, ok := seen[keyStr]; ok { + continue + } + seen[keyStr] = struct{}{} + + latest, exists, err := f.latestCommitTSForTargetReadyKey(ctx, key) + if err != nil { + return errors.WithStack(err) + } + if exists && latest > startTS { + return errors.WithStack(store.NewWriteConflictError(key)) + } + } + return nil +} + +func (f *kvFSM) latestCommitTSForTargetReadyKey(ctx context.Context, key []byte) (uint64, bool, error) { + routes, err := f.targetReadyRoutesForRange(ctx, key, nextScanCursor(key)) + if err != nil { + return 0, false, err + } + liveTS, liveExists, err := f.store.LatestCommitTS(ctx, key) + if err != nil { + return 0, false, errors.WithStack(err) + } + latest, exists := liveTS, liveExists + for _, route := range routes { + if !routeHasStagedVisibility(route) { + continue + } + stagedTS, stagedExists, err := f.store.LatestCommitTS(ctx, distribution.MigrationStagedDataKey(route.MigrationJobID, key)) + if err != nil { + return 0, false, errors.WithStack(err) + } + if stagedExists && (!exists || stagedTS > latest) { + latest, exists = stagedTS, true + } + } + return latest, exists, nil +} + +func (f *kvFSM) validateTxnConflicts(ctx context.Context, muts []*pb.Mutation, readKeys [][]byte, startTS uint64) error { + if err := f.validateConflicts(ctx, muts, startTS); err != nil { + return errors.WithStack(err) + } + if err := f.validateReadConflicts(ctx, readKeys, startTS); err != nil { + return errors.WithStack(err) + } + return nil +} + func uniqueMutations(muts []*pb.Mutation) ([]*pb.Mutation, error) { if len(muts) == 0 { return []*pb.Mutation{}, nil @@ -1474,12 +2011,16 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { if err := f.validatePrepareEvidenceMeta(meta, startTS); err != nil { return err } - uniq, err := f.uniqueMutationsAboveFloorForRequest(r, muts, startTS) + floorTS := prepareRouteFloorTimestamp(meta, startTS) + if err := f.verifyTargetReadinessForTxnFootprint(ctx, muts, r.ReadKeys, nil); err != nil { + return err + } + uniq, err := f.uniqueMutationsAboveFloorForRequest(ctx, r, muts, floorTS) if err != nil { return err } - if err := f.validateConflicts(ctx, uniq, startTS); err != nil { - return errors.WithStack(err) + if err := f.validateTxnConflicts(ctx, uniq, r.ReadKeys, startTS); err != nil { + return err } expireAt := txnLockExpireAt(meta.LockTTLms) @@ -1488,8 +2029,27 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } + return f.recordAndApplyPrepare(ctx, uniq, storeMuts, r.ReadKeys, floorTS, startTS) +} + +func prepareRouteFloorTimestamp(meta TxnMeta, startTS uint64) uint64 { + if meta.PrepareCommitTS != 0 { + return meta.PrepareCommitTS + } + if meta.CommitTS != 0 { + return meta.CommitTS + } + return startTS +} - if err := f.store.ApplyMutationsRaftAt(ctx, storeMuts, r.ReadKeys, startTS, startTS, f.pendingApplyIdx); err != nil { +func (f *kvFSM) recordAndApplyPrepare(ctx context.Context, muts []*pb.Mutation, storeMuts []*store.KVPairMutation, readKeys [][]byte, floorTS, startTS uint64) error { + if err := f.verifyMigrationTransferSize(ctx, storeMuts); err != nil { + return err + } + if err := f.recordMigrationWrite(ctx, muts, floorTS); err != nil { + return err + } + if err := f.store.ApplyMutationsRaftAt(ctx, storeMuts, readKeys, startTS, startTS, f.pendingApplyIdx); err != nil { return errors.WithStack(err) } return nil @@ -1544,53 +2104,125 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } - uniq, err := f.uniqueMutationsAboveFloorForRequest(r, muts, commitTS) + uniq, storeMuts, err := f.onePhaseStoreMutations(ctx, r, muts, commitTS) if err != nil { return err } + return f.recordAndApplyOnePhase(ctx, uniq, storeMuts, r.ReadKeys, startTS, commitTS) +} - storeMuts, err := f.buildOnePhaseStoreMutations(ctx, uniq) +func (f *kvFSM) recordAndApplyOnePhase(ctx context.Context, uniq []*pb.Mutation, storeMuts []*store.KVPairMutation, readKeys [][]byte, startTS, commitTS uint64) error { + if err := f.verifyMigrationTransferSize(ctx, storeMuts); err != nil { + return err + } + readKeys, err := f.onePhaseConflictReadKeys(ctx, uniq, readKeys) if err != nil { return err } - if err := f.store.ApplyMutationsRaftAtFenced(ctx, storeMuts, r.ReadKeys, startTS, commitTS, f.pendingApplyIdx); err != nil { + if err := f.recordMigrationWrite(ctx, uniq, commitTS); err != nil { + return err + } + if err := f.store.ApplyMutationsRaftAtFenced(ctx, storeMuts, readKeys, startTS, commitTS, f.pendingApplyIdx); err != nil { return errors.WithStack(err) } f.notifyApplyObservers(commitTS, uniq) return nil } -func uniqueTxnMutations(muts []*pb.Mutation) ([]*pb.Mutation, error) { - uniq, err := uniqueMutations(muts) +func (f *kvFSM) onePhaseStoreMutations( + ctx context.Context, + r *pb.Request, + muts []*pb.Mutation, + commitTS uint64, +) ([]*pb.Mutation, []*store.KVPairMutation, error) { + uniq, err := f.uniqueMutationsAboveFloorForRequest(ctx, r, muts, commitTS) if err != nil { - return nil, err + return nil, nil, err } - return uniq, nil + storeMuts, err := f.buildOnePhaseStoreMutations(ctx, uniq) + if err != nil { + return nil, nil, err + } + return uniq, storeMuts, nil } -func (f *kvFSM) uniqueMutationsAboveFloor(muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { - return f.uniqueMutationsAboveFloorWithSnapshot(muts, commitTS, nil) +// Check staged versions under the store's atomic conflict/replay boundary. +// Pre-checking live versions here would reject replay of the same Raft entry. +func (f *kvFSM) onePhaseConflictReadKeys(ctx context.Context, muts []*pb.Mutation, readKeys [][]byte) ([][]byte, error) { + keys := append([][]byte(nil), readKeys...) + for _, mut := range muts { + keys = append(keys, mut.Key) + } + out := append([][]byte(nil), readKeys...) + seen := make(map[string]struct{}, len(keys)) + for _, key := range keys { + if len(key) == 0 || isTxnInternalKey(key) { + continue + } + if _, found := seen[string(key)]; found { + continue + } + seen[string(key)] = struct{}{} + routes, err := f.targetReadyRoutesForRange(ctx, key, nextScanCursor(key)) + if err != nil { + return nil, err + } + for _, route := range routes { + if routeHasStagedVisibility(route) { + out = append(out, distribution.MigrationStagedDataKey(route.MigrationJobID, key)) + } + } + } + return out, nil } -func (f *kvFSM) uniqueMutationsAboveFloorForRequest(r *pb.Request, muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { - floorSnap, err := f.routeFloorSnapshotForRequest(r) +func (f *kvFSM) uniqueMutationsAboveFloor(ctx context.Context, muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { + return f.uniqueMutationsAboveFloorWithSnapshot(ctx, muts, nil, commitTS, nil) +} + +func (f *kvFSM) uniqueMutationsAboveFloorForRequest( + ctx context.Context, + r *pb.Request, + muts []*pb.Mutation, + commitTS uint64, +) ([]*pb.Mutation, error) { + floorSnap, err := f.routeFloorSnapshotForRequest(ctx, r) if err != nil { return nil, err } - return f.uniqueMutationsAboveFloorWithSnapshot(muts, commitTS, floorSnap) + return f.uniqueMutationsAboveFloorWithSnapshot(ctx, muts, r.GetWriteFenceBypassKeys(), commitTS, floorSnap) } -func (f *kvFSM) uniqueMutationsAboveFloorWithSnapshot(muts []*pb.Mutation, commitTS uint64, floorSnap RouteSnapshot) ([]*pb.Mutation, error) { +func (f *kvFSM) uniqueMutationsAboveFloorWithSnapshot( + ctx context.Context, + muts []*pb.Mutation, + writeFenceBypassKeys [][]byte, + commitTS uint64, + floorSnap RouteSnapshot, +) ([]*pb.Mutation, error) { uniq, err := uniqueMutations(muts) if err != nil { return nil, err } - if err := rejectReservedControlMutations(uniq); err != nil { + if err := f.rejectReservedControlMutations(uniq); err != nil { + return nil, err + } + if err := f.verifyRouteNotFencedForMutations(ctx, uniq, writeFenceBypassKeySet(writeFenceBypassKeys)); err != nil { + return nil, err + } + if err := f.verifyTargetReadinessForMutations(ctx, uniq); err != nil { return nil, err } if err := verifyRouteWriteTimestampFloorsForMutationsFromSnapshot(floorSnap, uniq, commitTS); err != nil { return nil, err } + // Only for requests that pinned no route version; see + // validateRawMutationForApply for why a pinned view must not be re-checked. + if floorSnap == nil { + if err := f.verifyRouteWriteFloorForMutations(ctx, uniq, commitTS); err != nil { + return nil, err + } + } return uniq, nil } @@ -1606,53 +2238,94 @@ func (f *kvFSM) uniqueMutationsAboveFloorWithSnapshot(muts []*pb.Mutation, commi // The verdict is a pure function of the key, so every replica reaches it for // the same entry. The typed migration commands (import, promote, fence) do not // pass through here and keep writing these namespaces. -func rejectReservedControlMutations(muts []*pb.Mutation) error { +func (f *kvFSM) rejectReservedControlMutations(muts []*pb.Mutation) error { for _, mut := range muts { if mut == nil { continue } - if distribution.IsMigrationOnlyControlKey(mut.Key) { + if f.isMigrationOnlyControlKeyForApply(mut.Key) { return errors.WithStack(ErrInvalidRequest) } } return nil } -func (f *kvFSM) uniqueTxnMutationsAboveFloor(muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { - return f.uniqueTxnMutationsAboveFloorWithSnapshot(muts, commitTS, nil) +func (f *kvFSM) uniqueTxnMutationsAboveFloor(ctx context.Context, muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { + return f.uniqueTxnMutationsAboveFloorWithSnapshot(ctx, muts, commitTS, nil) } -func (f *kvFSM) uniqueTxnMutationsAboveFloorForRequest(r *pb.Request, muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { - floorSnap, err := f.routeFloorSnapshotForRequest(r) +func (f *kvFSM) uniqueTxnMutationsAboveFloorForRequest( + ctx context.Context, + r *pb.Request, + muts []*pb.Mutation, + commitTS uint64, +) ([]*pb.Mutation, error) { + if migrationResolutionAuthorized(ctx, r) { + return f.migrationResolutionMutations(muts) + } + floorSnap, err := f.routeFloorSnapshotForRequest(ctx, r) if err != nil { return nil, err } - return f.uniqueTxnMutationsAboveFloorWithSnapshot(muts, commitTS, floorSnap) + return f.uniqueTxnMutationsAboveFloorWithSnapshot(ctx, muts, commitTS, floorSnap) } -func (f *kvFSM) uniqueTxnMutationsAboveFloorWithSnapshot(muts []*pb.Mutation, commitTS uint64, floorSnap RouteSnapshot) ([]*pb.Mutation, error) { +func (f *kvFSM) migrationResolutionMutations(muts []*pb.Mutation) ([]*pb.Mutation, error) { uniq, err := uniqueTxnMutations(muts) if err != nil { return nil, err } - if err := rejectReservedControlMutations(uniq); err != nil { + if err := f.rejectReservedControlMutations(uniq); err != nil { + return nil, err + } + return uniq, nil +} + +func (f *kvFSM) uniqueTxnMutationsAboveFloorWithSnapshot( + ctx context.Context, + muts []*pb.Mutation, + commitTS uint64, + floorSnap RouteSnapshot, +) ([]*pb.Mutation, error) { + // COMMIT must drain pre-fence locks. Unlike PREPARE/one-phase admission, + // this path skips both write fences; buildCommitStoreMutations validates + // each live lock's start timestamp and primary before applying its intent. + uniq, err := uniqueTxnMutations(muts) + if err != nil { + return nil, err + } + if err := f.rejectReservedControlMutations(uniq); err != nil { + return nil, err + } + if err := f.verifyTargetReadinessForMutations(ctx, uniq); err != nil { return nil, err } if err := verifyRouteWriteTimestampFloorsForMutationsFromSnapshot(floorSnap, uniq, commitTS); err != nil { return nil, err } + // Only for requests that pinned no route version; see + // validateRawMutationForApply for why a pinned view must not be re-checked. + if floorSnap == nil { + if err := f.verifyRouteWriteFloorForMutations(ctx, uniq, commitTS); err != nil { + return nil, err + } + } return uniq, nil } -// dedupProbeOnePhase decides whether handleOnePhaseTxnRequest should no-op -// because the entry is a retry whose prior attempt already landed. Extracted -// to keep handleOnePhaseTxnRequest under the cyclop budget; the determinism -// rationale lives at the call site. +// dedupProbeOnePhase first fail-closes the target readiness footprint, then +// decides whether handleOnePhaseTxnRequest should no-op because the entry is a +// retry whose prior attempt already landed. Extracted to keep +// handleOnePhaseTxnRequest under the cyclop budget; the determinism rationale +// lives at the call site. // // Returns (true, nil) → the entry must no-op (prior attempt landed). // Returns (false, nil) → fall through to normal apply. // Returns (false, err) → propagate err; apply must not proceed. func (f *kvFSM) dedupProbeOnePhase(ctx context.Context, meta TxnMeta, readKeys [][]byte, muts []*pb.Mutation) (bool, error) { + if err := f.verifyTargetReadinessForTxnFootprint(ctx, muts, readKeys, meta.PrimaryKey); err != nil { + return false, err + } if meta.PrevCommitTS == 0 { return false, nil } @@ -1763,17 +2436,24 @@ func (f *kvFSM) handleCommitRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } - uniq, err := f.uniqueTxnMutationsAboveFloorForRequest(r, muts, commitTS) + uniq, err := f.uniqueTxnMutationsAboveFloorForRequest(ctx, r, muts, commitTS) if err != nil { return err } - storeMuts, err := f.buildCommitStoreMutations(ctx, uniq, meta, startTS, commitTS, r.ReadKeys) + storeMuts, err := f.buildCommitStoreMutations(ctx, uniq, meta, startTS, commitTS, applyStartTS == commitTS, r.ReadKeys) if err != nil { return err } + return f.recordAndApplyCommit(ctx, storeMuts, uniq, applyStartTS, commitTS) +} + +func (f *kvFSM) recordAndApplyCommit(ctx context.Context, storeMuts []*store.KVPairMutation, uniq []*pb.Mutation, applyStartTS, commitTS uint64) error { if len(storeMuts) == 0 { return nil } + if err := f.recordMigrationWrite(ctx, uniq, commitTS); err != nil { + return err + } if err := f.applyCommitWithIdempotencyFallback(ctx, storeMuts, uniq, applyStartTS, commitTS); err != nil { return err } @@ -1893,7 +2573,7 @@ func (f *kvFSM) handleAbortRequest(ctx context.Context, r *pb.Request, abortTS u // abortCleanupMutationsForKey (lock-missing ⇒ nothing to do) and for the // rollback-marker Put in appendRollbackRecord. - uniq, err := uniqueMutations(muts) + uniq, err := f.uniqueAbortCleanupMutations(ctx, muts) if err != nil { return err } @@ -1913,6 +2593,10 @@ func (f *kvFSM) handleAbortRequest(ctx context.Context, r *pb.Request, abortTS u return errors.WithStack(f.store.ApplyMutationsRaftAt(ctx, storeMuts, nil, startTS, abortTS, f.pendingApplyIdx)) } +func (f *kvFSM) uniqueAbortCleanupMutations(_ context.Context, muts []*pb.Mutation) ([]*pb.Mutation, error) { + return uniqueMutations(muts) +} + func (f *kvFSM) buildPrepareStoreMutations(ctx context.Context, muts []*pb.Mutation, primaryKey []byte, startTS, expireAt, commitTS uint64) ([]*store.KVPairMutation, error) { storeMuts := make([]*store.KVPairMutation, 0, len(muts)*txnPrepareStoreMutationFactor) for _, mut := range muts { @@ -1941,7 +2625,7 @@ func (f *kvFSM) buildOnePhaseStoreMutations(ctx context.Context, muts []*pb.Muta return storeMuts, nil } -func (f *kvFSM) buildCommitStoreMutations(ctx context.Context, muts []*pb.Mutation, meta TxnMeta, startTS, commitTS uint64, readKeys [][]byte) ([]*store.KVPairMutation, error) { +func (f *kvFSM) buildCommitStoreMutations(ctx context.Context, muts []*pb.Mutation, meta TxnMeta, startTS, commitTS uint64, primaryAlreadyCommitted bool, readKeys [][]byte) ([]*store.KVPairMutation, error) { storeMuts := make([]*store.KVPairMutation, 0, len(muts)*txnCommitStoreMutationFactor+txnCommitStoreMutationSlack) committingPrimary := false @@ -1951,10 +2635,13 @@ func (f *kvFSM) buildCommitStoreMutations(ctx context.Context, muts []*pb.Mutati committingPrimary = true } - keyMuts, err := f.commitTxnKeyMutations(ctx, key, meta.PrimaryKey, startTS, commitTS, readKeys) + keyMuts, prepared, err := f.commitTxnKeyMutations(ctx, key, meta.PrimaryKey, startTS, commitTS, readKeys) if err != nil { return nil, err } + if bytes.Equal(key, meta.PrimaryKey) && !prepared && !primaryAlreadyCommitted { + return nil, errors.Wrapf(ErrTxnInvalidMeta, "primary lock missing for key %s", string(key)) + } storeMuts = append(storeMuts, keyMuts...) } @@ -2090,7 +2777,7 @@ func (f *kvFSM) txnCommitTSAtKey(ctx context.Context, key []byte) (uint64, bool, } func (f *kvFSM) prepareTxnMutation(ctx context.Context, mut *pb.Mutation, primaryKey []byte, startTS, expireAt, commitTS uint64) ([]*store.KVPairMutation, error) { - if isTxnInternalKey(mut.Key) || distribution.IsMigrationOnlyControlKey(mut.Key) { + if isTxnInternalKey(mut.Key) || f.isMigrationOnlyControlKeyForApply(mut.Key) { return nil, errors.WithStack(ErrInvalidRequest) } if err := f.validatePrepareLock(ctx, mut.Key, primaryKey, startTS, commitTS); err != nil { @@ -2201,41 +2888,41 @@ func storeMutationForIntent(key []byte, intent txnIntent) (*store.KVPairMutation } } -func (f *kvFSM) commitTxnKeyMutations(ctx context.Context, key, primaryKey []byte, startTS, commitTS uint64, readKeys [][]byte) ([]*store.KVPairMutation, error) { +func (f *kvFSM) commitTxnKeyMutations(ctx context.Context, key, primaryKey []byte, startTS, commitTS uint64, readKeys [][]byte) ([]*store.KVPairMutation, bool, error) { lock, lockKey, intentKey, ok, err := f.txnLockForMutationKey(ctx, key, readKeys) if err != nil { - return nil, err + return nil, false, err } if !ok { // Already resolved (committed/rolled back). - return nil, nil + return nil, false, nil } - if err := validatePreparedCommitLock(lock, key, primaryKey, startTS, commitTS); err != nil { - return nil, err + if err := f.validatePreparedCommitLock(lock, key, primaryKey, startTS, commitTS); err != nil { + return nil, false, err } intent, ok, err := f.txnIntentForCommit(ctx, intentKey) if err != nil { - return nil, err + return nil, false, err } out := make([]*store.KVPairMutation, 0, txnCommitStoreMutationFactor) if ok { if intent.StartTS != startTS { - return nil, errors.Wrapf(ErrTxnInvalidMeta, "intent start_ts mismatch for key %s", string(key)) + return nil, false, errors.Wrapf(ErrTxnInvalidMeta, "intent start_ts mismatch for key %s", string(key)) } mut, err := storeMutationForIntent(key, intent) if err != nil { - return nil, err + return nil, false, err } out = append(out, mut) } out = append(out, txnCleanupMutationsForKeys(lockKey, intentKey)...) - return out, nil + return out, true, nil } -func validatePreparedCommitLock(lock txnLock, key, primaryKey []byte, startTS, commitTS uint64) error { - if isTxnInternalKey(key) || distribution.IsMigrationOnlyControlKey(key) { +func (f *kvFSM) validatePreparedCommitLock(lock txnLock, key, primaryKey []byte, startTS, commitTS uint64) error { + if isTxnInternalKey(key) || f.isMigrationOnlyControlKeyForApply(key) { return errors.WithStack(ErrInvalidRequest) } if lock.StartTS != startTS { diff --git a/kv/fsm_apply_route_context.go b/kv/fsm_apply_route_context.go new file mode 100644 index 000000000..37b7f5018 --- /dev/null +++ b/kv/fsm_apply_route_context.go @@ -0,0 +1,23 @@ +package kv + +import "context" + +type applyRouteHistoryKey struct{} + +type applyRouteHistory struct { + history RouteHistory +} + +// withApplyRouteHistory scopes a caller-supplied immutable history to an apply +// request. The caller owns proof validation and must not mutate the history or +// its snapshots while the context is in use. This helper does not infer routes. +func withApplyRouteHistory(ctx context.Context, history RouteHistory) context.Context { + return context.WithValue(ctx, applyRouteHistoryKey{}, applyRouteHistory{history: history}) +} + +func (f *kvFSM) routeHistoryForContext(ctx context.Context) RouteHistory { + if routes, ok := ctx.Value(applyRouteHistoryKey{}).(applyRouteHistory); ok { + return routes.history + } + return f.routes +} diff --git a/kv/fsm_apply_route_context_test.go b/kv/fsm_apply_route_context_test.go new file mode 100644 index 000000000..a172bae4c --- /dev/null +++ b/kv/fsm_apply_route_context_test.go @@ -0,0 +1,314 @@ +package kv + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +type immutableApplyRouteHistory struct { + current RouteSnapshot + observed RouteSnapshot +} + +func (h immutableApplyRouteHistory) Current() (RouteSnapshot, bool) { + return h.current, h.current != nil +} + +func (h immutableApplyRouteHistory) SnapshotAt(version uint64) (RouteSnapshot, bool) { + if h.observed != nil && h.observed.Version() == version { + return h.observed, true + } + if h.current != nil && h.current.Version() == version { + return h.current, true + } + return nil, false +} + +func applyRouteContextHistory(t *testing.T, version uint64, route distribution.RouteDescriptor) RouteHistory { + t.Helper() + e := distribution.NewEngine() + applyComposed1Snapshot(t, e, version, []distribution.RouteDescriptor{route}) + snapshot, ok := WrapDistributionEngine(e).Current() + require.True(t, ok) + return immutableApplyRouteHistory{current: snapshot} +} + +func TestApplyRouteHistoryContextIsolation(t *testing.T) { + t.Parallel() + base := context.Background() + legacy := applyRouteContextHistory(t, 1, distribution.RouteDescriptor{RouteID: 1, GroupID: 1, State: distribution.RouteStateActive}) + first := applyRouteContextHistory(t, 2, distribution.RouteDescriptor{RouteID: 2, GroupID: 2, State: distribution.RouteStateActive}) + second := applyRouteContextHistory(t, 3, distribution.RouteDescriptor{RouteID: 3, GroupID: 3, State: distribution.RouteStateActive}) + f := &kvFSM{routes: legacy} + firstCtx := withApplyRouteHistory(base, first) + secondCtx := withApplyRouteHistory(base, second) + require.Equal(t, legacy, f.routeHistoryForContext(base)) + require.Equal(t, first, f.routeHistoryForContext(firstCtx)) + require.Equal(t, second, f.routeHistoryForContext(secondCtx)) + require.Equal(t, second, f.routeHistoryForContext(withApplyRouteHistory(firstCtx, second))) + require.Equal(t, legacy, f.routes) +} + +func applyRouteContextDescriptor() distribution.RouteDescriptor { + return distribution.RouteDescriptor{ + RouteID: 10, Start: []byte("a"), End: []byte("z"), GroupID: 1, + State: distribution.RouteStateActive, MinWriteTSExclusive: 100, + MigrationJobID: 11, StagedVisibilityActive: true, + } +} + +func applyRouteContextWithObserved(t *testing.T, current, observed distribution.RouteDescriptor) RouteHistory { + t.Helper() + currentSnap, ok := applyRouteContextHistory(t, 7, current).Current() + require.True(t, ok) + observedSnap, ok := applyRouteContextHistory(t, 6, observed).Current() + require.True(t, ok) + return immutableApplyRouteHistory{current: currentSnap, observed: observedSnap} +} + +func TestApplyRouteContextHelpersIgnoreLocalViews(t *testing.T) { + t.Parallel() + route := applyRouteContextDescriptor() + observed := route + observed.MinWriteTSExclusive = 50 + ctx := withApplyRouteHistory(context.Background(), applyRouteContextWithObserved(t, route, observed)) + for _, view := range []string{"lagging", "fenced", "other-owner"} { + t.Run(view, func(t *testing.T) { + localRoute := route + localRoute.MinWriteTSExclusive = 900 + version := uint64(7) + switch view { + case "lagging": + version = 5 + localRoute.StagedVisibilityActive = false + localRoute.MigrationJobID = 0 + case "fenced": + localRoute.State = distribution.RouteStateWriteFenced + case "other-owner": + localRoute.GroupID = 2 + } + f := &kvFSM{store: store.NewMVCCStore(), routes: applyRouteContextHistory(t, version, localRoute), shardGroupID: 1} + assertApplyRouteContextHelpers(t, f, ctx) + }) + } +} + +func assertApplyRouteContextHelpers(t *testing.T, f *kvFSM, ctx context.Context) { + t.Helper() + key := []byte("k") + muts := []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("value")}} + r := &pb.Request{ObservedRouteVersion: 6, Mutations: muts} + snapshot, err := f.routeFloorSnapshotForRequest(ctx, r) + require.NoError(t, err) + require.Equal(t, uint64(6), snapshot.Version()) + require.NoError(t, verifyRouteWriteTimestampFloorForKeyFromSnapshot(snapshot, key, 75)) + require.NoError(t, f.verifyRouteNotFencedForKey(ctx, key)) + require.NoError(t, f.verifyRouteNotFencedForMutations(ctx, muts, nil)) + require.NoError(t, f.verifyRouteNotFencedForPrefix(ctx, key)) + require.NoError(t, f.verifyRouteWriteFloorForKey(ctx, key, 150)) + require.ErrorIs(t, f.verifyRouteWriteFloorForKey(ctx, key, 100), ErrRouteWriteBelowFloor) + require.NoError(t, f.verifyRouteWriteFloorForPrefix(ctx, key, 150)) + require.ErrorIs(t, f.verifyRouteWriteFloorForPrefix(ctx, key, 100), ErrRouteWriteBelowFloor) + require.NoError(t, f.verifyRouteWriteFloorForMutations(ctx, muts, 150)) + require.NoError(t, f.verifyWriteFence(ctx, r)) + require.NoError(t, f.verifyComposed1(ctx, r)) + routes, version, proof := f.currentShardRoutesForRouteRange(ctx, key, []byte("l")) + require.True(t, proof) + require.Equal(t, uint64(7), version) + require.Len(t, routes, 1) + require.Equal(t, uint64(11), routes[0].MigrationJobID) + deletes := f.stagedVisibilityPrefixDeletesForApply(ctx, key, txnCommonPrefix) + require.Len(t, deletes, 1) + require.Equal(t, distribution.MigrationStagedDataKey(11, key), deletes[0].prefix) + require.Equal(t, distribution.MigrationStagedDataKey(11, txnCommonPrefix), deletes[0].excludePrefix) +} + +func TestApplyRouteContextReadinessOCCAndPrefixDelete(t *testing.T) { + t.Parallel() + for _, view := range []string{"lagging", "fenced", "other-owner"} { + t.Run(view, func(t *testing.T) { + ctx, f := newApplyRouteContextStagedFSM(t, view) + key := []byte("k") + stagedKey := distribution.MigrationStagedDataKey(11, key) + require.NoError(t, f.store.PutAt(ctx, stagedKey, []byte("staged"), 120, 0)) + require.NoError(t, f.store.PutAt(ctx, key, []byte("live"), 110, 0)) + outside := distribution.MigrationStagedDataKey(11, []byte("n")) + require.NoError(t, f.store.PutAt(ctx, outside, []byte("keep"), 120, 0)) + routes, err := f.targetReadyRoutesForRange(ctx, key, nextScanCursor(key)) + require.NoError(t, err) + require.Len(t, routes, 1) + require.Equal(t, uint64(11), routes[0].MigrationJobID) + latest, exists, err := f.latestCommitTSForTargetReadyKey(ctx, key) + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, uint64(120), latest) + muts := []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("new")}} + require.ErrorIs(t, f.validateConflicts(ctx, muts, 115), store.ErrWriteConflict) + require.ErrorIs(t, f.validateReadConflicts(ctx, [][]byte{key}, 115), store.ErrWriteConflict) + readKeys, err := f.onePhaseConflictReadKeys(ctx, muts, [][]byte{key}) + require.NoError(t, err) + require.Equal(t, [][]byte{key, stagedKey}, readKeys) + require.NoError(t, f.handleDelPrefix(ctx, key, 150)) + _, err = f.store.GetAt(ctx, key, 150) + require.ErrorIs(t, err, store.ErrKeyNotFound) + _, err = f.store.GetAt(ctx, stagedKey, 150) + require.ErrorIs(t, err, store.ErrKeyNotFound) + value, err := f.store.GetAt(ctx, outside, 150) + require.NoError(t, err) + require.Equal(t, []byte("keep"), value) + }) + } +} + +func newApplyRouteContextStagedFSM(t *testing.T, view string) (context.Context, *kvFSM) { + t.Helper() + return newApplyRouteContextStagedFSMWithStore(t, view, store.NewMVCCStore()) +} + +func newApplyRouteContextStagedFSMWithStore(t *testing.T, view string, st store.MVCCStore) (context.Context, *kvFSM) { + t.Helper() + route := applyRouteContextDescriptor() + history := applyRouteContextWithObserved(t, route, route) + ctx := withApplyRouteHistory(context.Background(), history) + local := route + version := uint64(7) + switch view { + case "lagging": + version = 5 + local.MigrationJobID = 0 + local.StagedVisibilityActive = false + case "fenced": + local.State = distribution.RouteStateWriteFenced + case "other-owner": + local.GroupID = 2 + } + f := &kvFSM{store: st, routes: applyRouteContextHistory(t, version, local), shardGroupID: 1} + installStagedOwnershipEvidence(t, f, 7, route) + return ctx, f +} + +func TestApplyRouteContextMissingAndContradictingEvidence(t *testing.T) { + t.Parallel() + route := applyRouteContextDescriptor() + f := &kvFSM{store: store.NewMVCCStore(), routes: applyRouteContextWithObserved(t, route, route), shardGroupID: 1} + r := &pb.Request{ObservedRouteVersion: 6, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("k")}}} + missing := withApplyRouteHistory(context.Background(), applyRouteContextHistory(t, 7, route)) + _, err := f.routeFloorSnapshotForRequest(missing, r) + require.ErrorIs(t, err, ErrComposed1VersionGCd) + require.ErrorIs(t, f.verifyComposed1(missing, r), ErrComposed1VersionGCd) + require.ErrorIs(t, f.verifyWriteFence(missing, r), ErrComposed1VersionGCd) + fenced := route + fenced.State = distribution.RouteStateWriteFenced + for _, history := range []RouteHistory{ + applyRouteContextWithObserved(t, fenced, route), + applyRouteContextWithObserved(t, route, fenced), + } { + ctx := withApplyRouteHistory(context.Background(), history) + require.ErrorIs(t, f.verifyWriteFence(ctx, r), ErrRouteWriteFenced) + } + ctx := withApplyRouteHistory(context.Background(), applyRouteContextHistory(t, 7, fenced)) + require.ErrorIs(t, f.verifyRouteNotFencedForKey(ctx, []byte("k")), ErrRouteWriteFenced) + require.ErrorIs(t, f.verifyRouteNotFencedForPrefix(ctx, []byte("k")), ErrRouteWriteFenced) + other := route + other.GroupID = 2 + for _, history := range []RouteHistory{ + applyRouteContextWithObserved(t, other, route), + applyRouteContextWithObserved(t, route, other), + } { + require.ErrorIs(t, f.verifyComposed1(withApplyRouteHistory(context.Background(), history), r), ErrComposed1Violation) + } +} + +func TestApplyRouteContextTransactionPaths(t *testing.T) { + t.Parallel() + for _, view := range []string{"lagging", "fenced", "other-owner"} { + t.Run(view, func(t *testing.T) { + for _, phase := range []pb.Phase{pb.Phase_NONE, pb.Phase_PREPARE} { + t.Run(phase.String(), func(t *testing.T) { + ctx, f := newApplyRouteContextStagedFSM(t, view) + key := []byte("k") + require.NoError(t, f.store.PutAt(ctx, distribution.MigrationStagedDataKey(11, key), []byte("old"), 120, 0)) + r := applyRouteContextTxn(phase, 130, 200, key) + f.SetApplyIndex(4) + require.NoError(t, f.applyRequestErr(ctx, r)) + if phase == pb.Phase_PREPARE { + commit := applyRouteContextTxn(pb.Phase_COMMIT, 130, 200, key) + f.SetApplyIndex(5) + require.NoError(t, f.applyRequestErr(ctx, commit)) + } + value, err := f.store.GetAt(ctx, key, 200) + require.NoError(t, err) + require.Equal(t, []byte("new"), value) + }) + } + }) + } +} + +func applyRouteContextTxn(phase pb.Phase, startTS, commitTS uint64, key []byte) *pb.Request { + return &pb.Request{ + IsTxn: true, Phase: phase, Ts: startTS, ObservedRouteVersion: 7, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: key, LockTTLms: 1000, CommitTS: commitTS, + })}, + {Op: pb.Op_PUT, Key: key, Value: []byte("new")}, + }, + } +} + +func TestApplyRouteContextBatchVerdictsAndLegacyFallback(t *testing.T) { + t.Parallel() + for _, pebble := range []bool{false, true} { + name := "memory" + if pebble { + name = "pebble" + } + t.Run(name, func(t *testing.T) { + st := store.NewMVCCStore() + if pebble { + var err error + st, err = store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + } + ctx, f := newApplyRouteContextStagedFSMWithStore(t, "fenced", st) + f.pendingApplyIdx = 42 + requests := []*pb.Request{ + {Ts: 100, ObservedRouteVersion: 7, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("too-old")}}}, + {Ts: 150, ObservedRouteVersion: 7, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("valid")}}}, + {Ts: 160, ObservedRouteVersion: 7, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Value: []byte("invalid")}}}, + {Ts: 170, ObservedRouteVersion: 7, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("after-error")}}}, + } + results := make([]error, len(requests)) + halt, hasError := f.applyBatchedRequests(ctx, requests, results) + require.Nil(t, halt) + require.True(t, hasError) + require.ErrorIs(t, results[0], ErrRouteWriteTimestampTooLow) + require.NoError(t, results[1]) + require.ErrorIs(t, results[2], ErrInvalidRequest) + require.NoError(t, results[3]) + value, err := f.store.GetAt(ctx, []byte("k"), 200) + require.NoError(t, err) + require.Equal(t, []byte("valid"), value) + value, err = f.store.GetAt(ctx, []byte("n"), 200) + require.NoError(t, err) + require.Equal(t, []byte("after-error"), value) + require.ErrorIs(t, f.verifyWriteFence(context.Background(), requests[1]), ErrRouteWriteFenced) + require.ErrorIs(t, f.verifyRouteNotFencedForKey(context.Background(), []byte("k")), ErrRouteWriteFenced) + if pebble { + reader, ok := f.store.(interface{ LastAppliedIndex() (uint64, bool, error) }) + require.True(t, ok) + index, present, err := reader.LastAppliedIndex() + require.NoError(t, err) + require.True(t, present) + require.Equal(t, uint64(42), index) + } + }) + } +} diff --git a/kv/fsm_backup.go b/kv/fsm_backup.go index 212c7ceac..02bcefafc 100644 --- a/kv/fsm_backup.go +++ b/kv/fsm_backup.go @@ -106,6 +106,9 @@ func (f *kvFSM) reloadBackupTimestampFloor(ctx context.Context) error { if f == nil || f.store == nil { return nil } + if err := f.reloadMigrationConfiguration(ctx); err != nil { + return err + } if err := f.reloadPrepareEvidence(ctx); err != nil { return err } diff --git a/kv/fsm_commit_fence_drain_test.go b/kv/fsm_commit_fence_drain_test.go new file mode 100644 index 000000000..04204340f --- /dev/null +++ b/kv/fsm_commit_fence_drain_test.go @@ -0,0 +1,90 @@ +package kv + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestFSMPreparedCommitDrainsMigrationFences(t *testing.T) { + t.Parallel() + for _, secondary := range []bool{false, true} { + for _, fence := range []string{"catalog", "durable", "both"} { + t.Run(fence+map[bool]string{false: "/primary", true: "/secondary"}[secondary], func(t *testing.T) { + t.Parallel() + ctx := context.Background() + engine := distribution.NewEngine() + route := distribution.RouteDescriptor{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 1} + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{route}) + fsm := newComposed1FSM(t, engine, 1) + primary := []byte("n") + if secondary { + primary = []byte("b") + } + meta := &pb.Mutation{Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: primary, CommitTS: 60, LockTTLms: defaultTxnLockTTLms, + })} + mutation := &pb.Mutation{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("prepared")} + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{IsTxn: true, Phase: pb.Phase_PREPARE, Ts: 40, + Mutations: []*pb.Mutation{meta, mutation}}, 40)) + if fence != "durable" { + route.State = distribution.RouteStateWriteFenced + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{route}) + } + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 11, MigrationJobID: 11, RouteStart: []byte("m"), RouteEnd: []byte("z"), + Armed: true, SourceWriteFence: fence != "catalog", TrackWrites: true, + MinWriteTSExclusive: 1, RetentionPinTS: 1, + }) + // Lock resolution supplies neither a pinned route version nor bypass keys, + // and must apply the stored intent, not the caller's mutation value. + request := &pb.Request{IsTxn: true, Phase: pb.Phase_COMMIT, Ts: 40, + Mutations: []*pb.Mutation{meta, {Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("untrusted")}}} + require.NoError(t, fsm.handleTxnRequest(ctx, request, 60)) + value, err := fsm.store.GetAt(ctx, []byte("n"), 60) + require.NoError(t, err) + require.Equal(t, []byte("prepared"), value) + _, exists, err := fsm.txnLockAtKey(ctx, txnLockKey([]byte("n"))) + require.NoError(t, err) + require.False(t, exists) + require.Equal(t, uint64(60), migrationTrackerMinimum(t, fsm)) + require.NoError(t, fsm.handleTxnRequest(ctx, request, 60), "resolved retries remain idempotent") + }) + } + } +} + +func TestFSMFencedCommitAuthenticatesLiveLock(t *testing.T) { + t.Parallel() + for _, mismatch := range []string{"start", "primary", "missing"} { + t.Run(mismatch, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + fsm := newMigrationWriteTrackerFSM(t) + meta := &pb.Mutation{Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("n"), CommitTS: 60, LockTTLms: defaultTxnLockTTLms, + })} + mutation := &pb.Mutation{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("prepared")} + if mismatch != "missing" { + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{IsTxn: true, Phase: pb.Phase_PREPARE, Ts: 40, + Mutations: []*pb.Mutation{meta, mutation}}, 40)) + } + applySourceMigrationControlToFSM(t, fsm, false) + request := &pb.Request{IsTxn: true, Phase: pb.Phase_COMMIT, Ts: 40, Mutations: []*pb.Mutation{meta, mutation}} + if mismatch == "start" { + request.Ts = 41 + } + if mismatch == "primary" { + request.Mutations[0] = &pb.Mutation{Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("b"), CommitTS: 60})} + } + require.Error(t, fsm.handleTxnRequest(ctx, request, 60)) + _, err := fsm.store.GetAt(ctx, []byte("n"), 60) + require.ErrorIs(t, err, store.ErrKeyNotFound) + }) + } +} diff --git a/kv/fsm_composed1_test.go b/kv/fsm_composed1_test.go index 05ff2dd67..0b62fbf17 100644 --- a/kv/fsm_composed1_test.go +++ b/kv/fsm_composed1_test.go @@ -199,7 +199,7 @@ func TestVerifyComposed1_EncodedObservedVersionZeroRunsGate(t *testing.T) { }) fsm := newComposed1FSM(t, e, 1) - err := fsm.verifyComposed1(commitTxnRequest(ObservedRouteVersionZero, "k")) + err := fsm.verifyComposed1(context.Background(), commitTxnRequest(ObservedRouteVersionZero, "k")) require.ErrorIs(t, err, ErrComposed1Violation, "an explicitly pinned version-0 observation must run the Composed-1 gate instead of being mistaken for the unpinned zero sentinel") require.Contains(t, err.Error(), "current-version") @@ -291,7 +291,7 @@ func TestVerifyComposed1_ValidOwnershipPassesGate(t *testing.T) { }) fsm := newComposed1FSM(t, e, 1) - require.NoError(t, fsm.verifyComposed1(commitTxnRequest(1, "k")), + require.NoError(t, fsm.verifyComposed1(context.Background(), commitTxnRequest(1, "k")), "verifyComposed1 must return nil when this FSM's shardGroupID matches the owner at BOTH the observed version and the current catalog version — covers the happy path neither the failure tests nor the bypass tests exercise") } @@ -333,7 +333,7 @@ func TestVerifyComposed1_AbortPhaseSkipsGate(t *testing.T) { {Op: pb.Op_DEL, Key: []byte("k")}, }, } - require.NoError(t, fsm.verifyComposed1(abortReq), + require.NoError(t, fsm.verifyComposed1(context.Background(), abortReq), "ABORT requests must bypass the Composed-1 gate even with a non-zero ObservedRouteVersion pinned, so a route shift between PREPARE and ABORT cannot strand the txn's intent locks past lock-resolver TTL") } @@ -373,6 +373,6 @@ func TestVerifyComposed1_DynamoMetaKeyNormalizedToRouteKeyBeforeOwnerOf(t *testi }) fsm := newComposed1FSM(t, e, 2) // this FSM serves group 2 - require.NoError(t, fsm.verifyComposed1(commitTxnRequest(1, string(rawMetaKey))), + require.NoError(t, fsm.verifyComposed1(context.Background(), commitTxnRequest(1, string(rawMetaKey))), "issue #930: verifyOwnerFromSnapshot must routeKey-normalize raw adapter keys before OwnerOf, so the gate routes the same way as ShardRouter.ResolveGroup; without normalization the raw \"!ddb|meta|...\" key sorts below the \"!ddb|route|...\" range and is falsely owned by group 1") } diff --git a/kv/fsm_migration_apply_routes.go b/kv/fsm_migration_apply_routes.go new file mode 100644 index 000000000..902f32d26 --- /dev/null +++ b/kv/fsm_migration_apply_routes.go @@ -0,0 +1,34 @@ +package kv + +import ( + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" +) + +type migrationApplyRange struct{ start, end []byte } + +func migrationApplyFootprint(req *pb.Request) []migrationApplyRange { + ranges := make([]migrationApplyRange, 0, len(req.GetMutations())+len(req.GetReadKeys())) + for _, mut := range req.GetMutations() { + if mut == nil || isTxnInternalKey(mut.GetKey()) { + continue + } + var start, end []byte + if mut.GetOp() == pb.Op_DEL_PREFIX { + start, end = routePrefixRange(mut.GetKey()) + } else { + start, end = readinessRouteRangeForScan(mut.GetKey(), nextScanCursor(mut.GetKey())) + } + ranges = append(ranges, migrationApplyRange{start, end}) + } + for _, key := range req.GetReadKeys() { + if _, raw, staged := distribution.MigrationStagedDataKeyParts(key); staged { + key = raw + } + if !isTxnInternalKey(key) { + start, end := readinessRouteRangeForScan(key, nextScanCursor(key)) + ranges = append(ranges, migrationApplyRange{start, end}) + } + } + return ranges +} diff --git a/kv/fsm_migration_cleanup.go b/kv/fsm_migration_cleanup.go new file mode 100644 index 000000000..b054346b4 --- /dev/null +++ b/kv/fsm_migration_cleanup.go @@ -0,0 +1,170 @@ +package kv + +import ( + "bytes" + "context" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +// ErrMigrationCleanupApply marks a replica-local cleanup failure. Import, +// promote and retire already halt on their equivalents; cleanup returning an +// ordinary error let the Raft engine advance the applied index past a committed +// entry that this voter never performed, so it would silently keep versions the +// rest of the cluster deleted. +var ErrMigrationCleanupApply = errors.New("migration cleanup: FSM apply failed; halting apply") + +// MarshalMigrationCleanupCommand encodes a bounded cleanup operation for Raft. +func MarshalMigrationCleanupCommand(req *pb.CleanupMigrationRequest) ([]byte, error) { + if req == nil { + return nil, errors.WithStack(ErrInvalidRequest) + } + b, err := proto.Marshal(req) + if err != nil { + return nil, errors.WithStack(err) + } + if len(b) >= maxMarshaledCommandSize { + return nil, errors.New("marshaled migration cleanup request too large") + } + return prependByte(raftEncodeMigrationCleanup, b), nil +} + +func (f *kvFSM) applyMigrationCleanup(ctx context.Context, data []byte) any { + req := &pb.CleanupMigrationRequest{} + if err := proto.Unmarshal(data, req); err != nil { + return errors.WithStack(err) + } + if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION { + if !req.GetRequireConfigurationIndex() || req.GetExpectedConfigurationIndex() != f.AppliedConfigurationIndex() { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + return f.applyNamespaceAdmission(ctx, req.GetNamespaceAdmission()) + } + if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION { + return f.applyMigrationPartitionControl(ctx, req) + } + return f.applyMigrationCleanupRequest(ctx, req) +} + +func (f *kvFSM) applyMigrationCleanupRequest(ctx context.Context, req *pb.CleanupMigrationRequest) any { + if err := f.verifyMigrationCleanupConfiguration(ctx, req); err != nil { + if errors.Is(err, ErrMigrationConfigurationChanged) { + return err + } + return haltErr(errors.Wrap(errors.Mark(err, ErrMigrationCleanupApply), "kv/fsm: inspect cleanup configuration")) + } + if migrationOwnershipControlMode(req.GetMode()) { + return f.applyMigrationOwnershipControl(ctx, req) + } + cleaner, ok := f.store.(store.MigrationCleaner) + if !ok { + return haltErr(errors.Wrap(errors.Mark(store.ErrNotSupported, ErrMigrationCleanupApply), "kv/fsm: migration cleanup store")) + } + if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA { + return f.applyLegacyMigrationMetadataClear(ctx, cleaner, req.GetJobId()) + } + result, err := cleaner.CleanupVersions(ctx, migrationCleanupOptionsFromProto(req, f.pendingApplyIdx)) + if err != nil { + if isMigrationCleanupOrdinaryApplyError(err) { + return errors.Wrap(err, "kv/fsm: apply migration cleanup") + } + return haltErr(errors.Wrap(errors.Mark(err, ErrMigrationCleanupApply), "kv/fsm: apply migration cleanup")) + } + return result +} + +func (f *kvFSM) applyLegacyMigrationMetadataClear(ctx context.Context, cleaner store.MigrationCleaner, jobID uint64) any { + if err := f.verifyLegacyMigrationMetadataClear(ctx, jobID); err != nil { + if errors.Is(err, store.ErrMigrationOwnership) || errors.Is(err, store.ErrNotSupported) { + return err + } + return haltErr(errors.Wrap(errors.Mark(err, ErrMigrationCleanupApply), "kv/fsm: inspect legacy metadata clear")) + } + if err := cleaner.ClearMigrationState(ctx, jobID, f.pendingApplyIdx); err != nil { + return haltErr(errors.Wrap(errors.Mark(err, ErrMigrationCleanupApply), "kv/fsm: clear migration state")) + } + return nil +} + +// isMigrationCleanupOrdinaryApplyError separates deterministic verdicts on the +// request itself -- every replica reaches them for the same entry, so they are +// safe to return as an ordinary response -- from replica-local store failures +// (Pebble read, decrypt, batch, commit), which must halt rather than let this +// voter skip a committed cleanup. +func isMigrationCleanupOrdinaryApplyError(err error) bool { + return errors.Is(err, store.ErrMigrationRetired) || + errors.Is(err, ErrInvalidRequest) || + errors.Is(err, store.ErrInvalidImportVersion) || + // A malformed or out-of-range cursor is decided from the request alone, + // so every replica rejects it identically. + errors.Is(err, store.ErrInvalidExportCursor) || + errors.Is(err, store.ErrInvalidExportBudget) || + errors.Is(err, store.ErrValueTooLarge) || + errors.Is(err, store.ErrSnapshotKeyTooLarge) +} + +func migrationCleanupOptionsFromProto(req *pb.CleanupMigrationRequest, appliedIndex uint64) store.CleanupVersionsOptions { + maxVersions := int(req.GetMaxVersions()) + if maxVersions <= 0 { + maxVersions = defaultMigrationPromoteMaxVersions + } + maxBytes := req.GetMaxBytes() + if maxBytes == 0 { + maxBytes = defaultMigrationPromoteMaxBytes + } + maxScannedBytes := req.GetMaxScannedBytes() + if maxScannedBytes == 0 { + maxScannedBytes = defaultMigrationPromoteMaxScannedBytes + } + bracket := distribution.MigrationBracket{ + Family: req.GetKeyFamily(), + Start: bytes.Clone(req.GetRangeStart()), + End: bytes.Clone(req.GetRangeEnd()), + ExcludePrefixes: cloneMigrationByteSlices(req.GetExcludePrefixes()), + ExcludeKnownInternal: req.GetExcludeKnownInternal(), + RequiresRouteKeyCheck: req.GetRequiresRouteKeyCheck(), + RequiresDecodedS3: req.GetRequiresDecodedS3(), + } + return store.CleanupVersionsOptions{ + JobID: req.GetJobId(), + AppliedIndex: appliedIndex, + StartKey: bytes.Clone(req.GetRangeStart()), + EndKey: bytes.Clone(req.GetRangeEnd()), + Cursor: bytes.Clone(req.GetCursor()), + MaxCommitTS: req.GetMaxCommitTs(), + MaxVersions: maxVersions, + MaxBytes: maxBytes, + MaxScannedBytes: maxScannedBytes, + KeyFamily: req.GetKeyFamily(), + AcceptVersion: func(key, value []byte) bool { + if IsPartitionedSQSKey(key) { + // routeKey collapses every partitioned SQS row to the single + // global SQS route key, so a migrated interval covering that + // key would look like it owns every partition -- including the + // ones the resolver still routes to this group, whose rows the + // export never claimed. Deleting those loses messages that are + // still being served from here. Apply cannot ask the resolver + // which partitions moved: it is process-local config + // (--sqsFifoPartitionMap), not replicated, so a per-node answer + // would make apply diverge across replicas. Catalog route + // cleanup therefore deletes only catalog-routed data and leaves + // the resolver-owned keyspace alone; retiring a partition's + // rows belongs to the resolver's own migration path. + return false + } + return bracket.ContainsRoutedVersion(key, value, req.GetRouteStart(), req.GetRouteEnd(), routeKey) + }, + } +} + +func cloneMigrationByteSlices(in [][]byte) [][]byte { + out := make([][]byte, len(in)) + for i := range in { + out[i] = bytes.Clone(in[i]) + } + return out +} diff --git a/kv/fsm_migration_cleanup_halt_test.go b/kv/fsm_migration_cleanup_halt_test.go new file mode 100644 index 000000000..75016a54f --- /dev/null +++ b/kv/fsm_migration_cleanup_halt_test.go @@ -0,0 +1,90 @@ +package kv + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +// cleanupFailureStore fails CleanupVersions and ClearMigrationState with a +// caller-supplied error so apply's halt-vs-ordinary classification can be +// driven directly. +type cleanupFailureStore struct { + store.MVCCStore + err error +} + +func (s *cleanupFailureStore) CleanupVersions(context.Context, store.CleanupVersionsOptions) (store.CleanupVersionsResult, error) { + return store.CleanupVersionsResult{}, s.err +} + +func (s *cleanupFailureStore) ClearMigrationState(context.Context, uint64, uint64) error { + return s.err +} + +func (s *cleanupFailureStore) MigrationTargetReadinessStates(ctx context.Context) ([]store.TargetStagedReadinessState, error) { + reader, ok := s.MVCCStore.(store.MigrationTargetReadinessReader) + if !ok { + return nil, errors.WithStack(store.ErrNotSupported) + } + states, err := reader.MigrationTargetReadinessStates(ctx) + return states, errors.WithStack(err) +} + +func cleanupCommand(t *testing.T, mode pb.MigrationCleanupMode) []byte { + t.Helper() + cmd, err := MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{ + JobId: 7, + Mode: mode, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + }) + require.NoError(t, err) + return cmd +} + +// A replica-local store failure during cleanup must halt apply. Returning an +// ordinary error lets the Raft engine advance the applied index, so this voter +// permanently skips a committed cleanup while healthy voters delete the +// versions -- divergent committed state that no retry repairs. Import, promote +// and retire already halt on their equivalents. +func TestApplyMigrationCleanupHaltsOnLocalStoreFailure(t *testing.T) { + t.Parallel() + + local := errors.New("pebble: commit failed") + for _, tc := range []struct { + name string + mode pb.MigrationCleanupMode + }{ + {"versions", pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_UNSPECIFIED}, + {"metadata", pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + fsm := &kvFSM{store: &cleanupFailureStore{MVCCStore: store.NewMVCCStore(), err: local}} + halt := haltApplyOf(fsm.Apply(cleanupCommand(t, tc.mode))) + require.Error(t, halt, "a replica-local cleanup failure must halt apply, not advance past it") + // cockroachdb errors.Is traverses Mark(); the stdlib one testify + // uses does not. + require.True(t, errors.Is(halt, ErrMigrationCleanupApply), "got %v", halt) + require.ErrorIs(t, halt, local) + }) + } +} + +// A deterministic verdict on the request itself is reached identically by every +// replica, so it stays an ordinary response rather than halting the cluster. +func TestApplyMigrationCleanupKeepsDeterministicErrorsOrdinary(t *testing.T) { + t.Parallel() + + fsm := &kvFSM{store: &cleanupFailureStore{MVCCStore: store.NewMVCCStore(), err: store.ErrValueTooLarge}} + resp := fsm.Apply(cleanupCommand(t, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_UNSPECIFIED)) + require.NoError(t, haltApplyOf(resp), "a deterministic request verdict must not halt apply") + err, ok := resp.(error) + require.True(t, ok) + require.ErrorIs(t, err, store.ErrValueTooLarge) +} diff --git a/kv/fsm_migration_cleanup_sqs_test.go b/kv/fsm_migration_cleanup_sqs_test.go new file mode 100644 index 000000000..2daebafea --- /dev/null +++ b/kv/fsm_migration_cleanup_sqs_test.go @@ -0,0 +1,71 @@ +package kv + +import ( + "encoding/base64" + "encoding/binary" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" +) + +// partitionedSQSMsgDataKey builds an HT-FIFO partitioned message row the way +// the SQS adapter lays one out: family prefix, partition discriminator, +// base64url queue segment, terminator, then the big-endian partition number. +func partitionedSQSMsgDataKey(queue string, partition uint32) []byte { + key := []byte(sqsMsgDataPrefix + sqsPartitionMarker) + key = append(key, base64.RawURLEncoding.AppendEncode(nil, []byte(queue))...) + key = append(key, '|') + return binary.BigEndian.AppendUint32(key, partition) +} + +// Cleanup runs inside FSM apply, where the partition resolver is unavailable: +// it is process-local config (--sqsFifoPartitionMap), so consulting it would +// make apply diverge across replicas. routeKey collapses every partitioned SQS +// row to the one global SQS route key, so a migrated interval covering that key +// would otherwise look like it owns every partition -- including the ones the +// resolver still routes to this group, whose rows the export (which does have +// the resolver) never claimed. Deleting those loses messages still served from +// here. +func TestMigrationCleanupSkipsResolverOwnedSQSRows(t *testing.T) { + t.Parallel() + + // A range that covers the global SQS route key, which is what every + // partitioned row collapses to. + opts := migrationCleanupOptionsFromProto(&pb.CleanupMigrationRequest{ + JobId: 7, + RangeStart: []byte("!sqs|"), + RangeEnd: []byte("!sqs~"), + RouteStart: []byte("!"), + RouteEnd: nil, + RequiresRouteKeyCheck: true, + }, 1) + require.NotNil(t, opts.AcceptVersion) + + partitioned := partitionedSQSMsgDataKey("q", 3) + require.Equal(t, sqsGlobalRouteKey, routeKey(partitioned), + "the route key really does collapse to the global SQS key") + require.False(t, opts.AcceptVersion(partitioned, []byte("body")), + "a resolver-owned partitioned row must not be deleted by catalog cleanup") + + // A non-partitioned SQS row in the same interval is catalog-routed and + // still gets cleaned up, so the guard does not disable cleanup wholesale. + legacy := append([]byte(sqsMsgDataPrefix), []byte("q|1")...) + require.True(t, opts.AcceptVersion(legacy, []byte("body"))) +} + +func TestIsPartitionedSQSKey(t *testing.T) { + t.Parallel() + + for _, family := range []string{ + sqsMsgDataPrefix, sqsMsgVisPrefix, sqsMsgDedupPrefix, + sqsMsgGroupPrefix, sqsMsgByAgePrefix, + } { + require.True(t, IsPartitionedSQSKey([]byte(family+sqsPartitionMarker+"q")), family) + require.False(t, IsPartitionedSQSKey([]byte(family+"q")), family) + } + // Queue metadata is not partitioned. + require.False(t, IsPartitionedSQSKey([]byte(sqsQueueMetaPrefix+sqsPartitionMarker+"q"))) + require.False(t, IsPartitionedSQSKey(nil)) + require.False(t, IsPartitionedSQSKey([]byte("!redis|k"))) +} diff --git a/kv/fsm_migration_cleanup_test.go b/kv/fsm_migration_cleanup_test.go new file mode 100644 index 000000000..70aa9ee74 --- /dev/null +++ b/kv/fsm_migration_cleanup_test.go @@ -0,0 +1,42 @@ +package kv + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestApplyMigrationCleanupCommandDeletesBoundedVersions(t *testing.T) { + ctx := context.Background() + st := store.NewMVCCStore() + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("old"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("new"), 20, 0)) + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + + cmd, err := MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{ + JobId: 1, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, + RangeStart: []byte("a"), + RangeEnd: []byte("b"), + MaxCommitTs: 10, + MaxVersions: 16, + MaxBytes: 1 << 20, + MaxScannedBytes: 1 << 20, + KeyFamily: 1, + }) + require.NoError(t, err) + result, ok := fsm.applyMigrationCleanup(ctx, cmd[1:]).(store.CleanupVersionsResult) + require.True(t, ok) + require.True(t, result.Done) + require.Equal(t, uint64(1), result.DeletedRows) + + value, err := st.GetAt(ctx, []byte("a"), 20) + require.NoError(t, err) + require.Equal(t, []byte("new"), value) + _, err = st.GetAt(ctx, []byte("a"), 10) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} diff --git a/kv/fsm_migration_clock_floor_test.go b/kv/fsm_migration_clock_floor_test.go new file mode 100644 index 000000000..b0df08ab5 --- /dev/null +++ b/kv/fsm_migration_clock_floor_test.go @@ -0,0 +1,63 @@ +package kv + +import ( + "context" + "testing" + "time" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestApplyTargetStagedReadinessAdvancesClockBeforeCutover(t *testing.T) { + t.Parallel() + for _, imported := range []bool{false, true} { + t.Run(map[bool]string{false: "empty range", true: "higher imported floor"}[imported], func(t *testing.T) { + t.Parallel() + ctx := context.Background() + dir := t.TempDir() + st, err := store.NewPebbleStore(dir) + require.NoError(t, err) + clock := NewHLC() + fsm := NewKvFSMWithHLC(st, clock) + fence := nonNegativeUint64(time.Now().Add(time.Hour).UnixMilli()) << HLCLogicalBits + want := fence + if imported { + want += 100 + cmd, marshalErr := MarshalMigrationImportCommand(&pb.ImportRangeVersionsRequest{ + JobId: 7, BracketId: 1, BatchSeq: 1, + Versions: []*pb.MVCCVersion{{Key: []byte("n"), CommitTs: want, Value: []byte("v")}}, + }) + require.NoError(t, marshalErr) + require.IsType(t, store.ImportVersionsResult{}, fsm.Apply(cmd)) + } + cmd, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{ + JobId: 7, MigrationJobId: 7, RouteStart: []byte("m"), RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, MinWriteTsExclusive: fence, Armed: true, + }) + require.NoError(t, err) + require.Nil(t, fsm.Apply(cmd)) + require.Equal(t, want, st.LastCommitTS()) + require.GreaterOrEqual(t, clock.Current(), want) + require.Greater(t, clock.Next(), fence) + require.NoError(t, st.Close()) + reopened, err := store.NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + require.Equal(t, want, reopened.LastCommitTS()) + floor, err := reopened.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Equal(t, want, floor) + reader, ok := reopened.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Len(t, states, 1) + require.Equal(t, fence, states[0].MinWriteTSExclusive) + restartedClock := NewHLC() + _ = NewKvFSMWithHLC(reopened, restartedClock) + require.Greater(t, restartedClock.Next(), fence) + }) + } +} diff --git a/kv/fsm_migration_configuration.go b/kv/fsm_migration_configuration.go new file mode 100644 index 000000000..0fdf5a9d8 --- /dev/null +++ b/kv/fsm_migration_configuration.go @@ -0,0 +1,89 @@ +package kv + +import ( + "context" + "encoding/binary" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +var migrationConfigurationIndexKey = store.MigrationConfigurationIndexKey() + +var ErrMigrationConfigurationChanged = errors.New("migration cleanup configuration changed; retry voter barriers") + +// MigrationConfigurationIndex is absent on stores that have not observed a +// configuration through the upgraded apply loop. Such stores cannot attest a +// configuration-bound cleanup barrier. +func MigrationConfigurationIndex(ctx context.Context, st store.MVCCStore) (uint64, error) { + state, err := readNamespaceConfiguration(ctx, st) + return state.Index, err +} + +func (f *kvFSM) AppliedConfigurationIndex() uint64 { + return f.migrationConfigurationIndex.Load() +} + +func (f *kvFSM) RecordAppliedConfiguration(index uint64) error { + if f.backupFloorLoadErr != nil { + return errors.Wrap(f.backupFloorLoadErr, "cannot record configuration over unreadable FSM control metadata") + } + if index == 0 || index <= f.AppliedConfigurationIndex() { + return nil + } + if f.migrationNamespaceVersion.Load() != 0 { + return errors.Wrap(ErrMigrationNamespaceProtocol, "active namespace protocol requires the full ordered configuration hook") + } + // Reuse the replicated store frontier; configuration bookkeeping must not + // issue a wall-clock timestamp or change user transaction admission. + ts := max(f.store.LastCommitTS(), 1) + value := binary.BigEndian.AppendUint64(nil, index) + // A crash must not replay an older rejected cleanup against a future + // durable epoch. Bundle the epoch with the same applied-index boundary. + if err := f.store.ApplyMutationsRaftAt(context.Background(), []*store.KVPairMutation{{ + Op: store.OpTypePut, Key: migrationConfigurationIndexKey, Value: value, + }}, nil, ts, ts, index); err != nil { + return errors.WithStack(err) + } + f.migrationConfigurationIndex.Store(index) + f.migrationNamespaceMembers = nil // Index-only hooks cannot prove apply membership. + return nil +} + +func (f *kvFSM) reloadMigrationConfiguration(ctx context.Context) error { + state, err := readNamespaceConfiguration(ctx, f.store) + if err == nil { + f.migrationConfigurationIndex.Store(state.Index) + f.migrationNamespaceVersion.Store(state.Version) + f.migrationNamespaceMembers = state.Members + } + return err +} + +func (f *kvFSM) verifyMigrationCleanupConfiguration(ctx context.Context, req *pb.CleanupMigrationRequest) error { + if req.GetRequireConfigurationIndex() { + if req.GetExpectedConfigurationIndex() == 0 || req.GetExpectedConfigurationIndex() != f.AppliedConfigurationIndex() { + return errors.WithStack(ErrMigrationConfigurationChanged) + } + return nil + } + return f.rejectUnboundSourceCleanup(ctx, req.GetJobId()) +} + +func (f *kvFSM) rejectUnboundSourceCleanup(ctx context.Context, jobID uint64) error { + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { + return nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + for _, state := range states { + if state.JobID == jobID && state.Armed && (state.SourceWriteFence || state.SourceReadFence || state.TrackWrites) { + return errors.WithStack(ErrMigrationConfigurationChanged) + } + } + return nil +} diff --git a/kv/fsm_migration_configuration_test.go b/kv/fsm_migration_configuration_test.go new file mode 100644 index 000000000..c5cf544b3 --- /dev/null +++ b/kv/fsm_migration_configuration_test.go @@ -0,0 +1,143 @@ +package kv + +import ( + "bytes" + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestMigrationCleanupConfigurationRejectsBeforeSideEffects(t *testing.T) { + ctx := context.Background() + for _, mode := range []pb.MigrationCleanupMode{pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA} { + t.Run(mode.String(), func(t *testing.T) { + st := store.NewMVCCStore() + fsm, ok := NewKvFSMWithHLC(st, nil).(*kvFSM) + require.True(t, ok) + require.NoError(t, st.PutAt(ctx, []byte("n"), []byte("customer"), 90, 0)) + state := store.TargetStagedReadinessState{JobID: 7, MigrationJobID: 7, RouteStart: []byte("a"), RouteEnd: []byte("z"), + Armed: true, SourceWriteFence: true, SourceReadFence: true, RetentionPinTS: 1, MinWriteTSExclusive: 100} + writer, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, state)) + require.NoError(t, fsm.RecordAppliedConfiguration(10)) + req := &pb.CleanupMigrationRequest{JobId: 7, Mode: mode, RequireConfigurationIndex: true, ExpectedConfigurationIndex: 10, + RangeStart: []byte("a"), RangeEnd: []byte("z"), RouteStart: []byte("a"), RouteEnd: []byte("z"), + KeyFamily: distribution.MigrationFamilyUser, MaxCommitTs: 100, MaxVersions: 10} + cmd, err := MarshalMigrationCleanupCommand(req) + require.NoError(t, err) + // The cleanup was already constructed/proposed from configuration 10; + // a ConfChange ordered before its apply invalidates that old proof. + require.NoError(t, fsm.RecordAppliedConfiguration(11)) + resp := fsm.Apply(cmd) + require.NoError(t, haltApplyOf(resp)) + responseErr, ok := resp.(error) + require.True(t, ok) + require.ErrorIs(t, responseErr, ErrMigrationConfigurationChanged) + value, err := st.GetAt(ctx, []byte("n"), 100) + require.NoError(t, err) + require.Equal(t, []byte("customer"), value) + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []store.TargetStagedReadinessState{state}, states) + retirement, ok := st.(store.MigrationRetirementReader) + require.True(t, ok) + retired, err := retirement.MigrationRetired(ctx, 7) + require.NoError(t, err) + require.False(t, retired) + require.Equal(t, uint64(100), st.LastCommitTS()) + req.RequireConfigurationIndex = false + cmd, err = MarshalMigrationCleanupCommand(req) + require.NoError(t, err) + responseErr, ok = fsm.Apply(cmd).(error) + require.True(t, ok) + require.ErrorIs(t, responseErr, ErrMigrationConfigurationChanged, "an old runner cannot clear a live source guard") + req.RequireConfigurationIndex = true + req.ExpectedConfigurationIndex = 11 + cmd, err = MarshalMigrationCleanupCommand(req) + require.NoError(t, err) + resp = fsm.Apply(cmd) + require.NoError(t, haltApplyOf(resp)) + if mode == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA { + responseErr, ok = resp.(error) + require.True(t, ok) + require.ErrorIs(t, responseErr, store.ErrMigrationOwnership, "the current epoch alone cannot replace a live serving guard") + return + } + if applyErr, ok := resp.(error); ok { + require.NoError(t, applyErr) + } + }) + } +} + +func TestMigrationConfigurationSurvivesSnapshotAndReopen(t *testing.T) { + ctx := context.Background() + st := store.NewMVCCStore() + fsm, ok := NewKvFSMWithHLC(st, nil).(*kvFSM) + require.True(t, ok) + require.NoError(t, fsm.RecordAppliedConfiguration(10)) + require.NoError(t, fsm.RecordAppliedConfiguration(11)) + require.NoError(t, fsm.RecordAppliedConfiguration(5), "old ConfChange replay cannot regress the durable epoch") + snapshot, err := st.Snapshot() + require.NoError(t, err) + defer snapshot.Close() + var raw bytes.Buffer + _, err = snapshot.WriteTo(&raw) + require.NoError(t, err) + dir := t.TempDir() + pebble, err := store.NewPebbleStore(dir) + require.NoError(t, err) + require.NoError(t, pebble.Restore(bytes.NewReader(raw.Bytes()))) + restoredFSM, ok := NewKvFSMWithHLC(pebble, nil).(*kvFSM) + require.True(t, ok) + require.Equal(t, uint64(11), restoredFSM.AppliedConfigurationIndex()) + require.NoError(t, pebble.Close()) + pebble, err = store.NewPebbleStore(dir) + require.NoError(t, err) + defer pebble.Close() + index, err := MigrationConfigurationIndex(ctx, pebble) + require.NoError(t, err) + require.Equal(t, uint64(11), index) +} + +func TestMigrationConfigurationSharesDurableAppliedBoundary(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + st, err := store.NewPebbleStore(dir) + require.NoError(t, err) + fsm, ok := NewKvFSMWithHLC(st, nil).(*kvFSM) + require.True(t, ok) + require.NoError(t, fsm.RecordAppliedConfiguration(10)) + require.NoError(t, fsm.RecordAppliedConfiguration(11)) + reader, ok := st.(interface{ LastAppliedIndex() (uint64, bool, error) }) + require.True(t, ok) + index, present, err := reader.LastAppliedIndex() + require.NoError(t, err) + require.True(t, present) + require.Equal(t, uint64(11), index, "the skip gate cannot replay pre-epoch entries against a future epoch") + require.NoError(t, st.Close()) + st, err = store.NewPebbleStore(dir) + require.NoError(t, err) + defer st.Close() + index, err = MigrationConfigurationIndex(ctx, st) + require.NoError(t, err) + require.Equal(t, uint64(11), index) +} + +func TestMigrationConfigurationCorruptionCannotBeOverwrittenByConfChange(t *testing.T) { + st := store.NewMVCCStore() + require.NoError(t, st.PutAt(t.Context(), migrationConfigurationIndexKey, []byte{1}, 100, 0)) + fsm, ok := NewKvFSMWithHLC(st, nil).(*kvFSM) + require.True(t, ok) + require.ErrorContains(t, fsm.RecordAppliedConfiguration(11), "invalid durable migration configuration index") + value, err := st.GetAt(t.Context(), migrationConfigurationIndexKey, 100) + require.NoError(t, err) + require.Equal(t, []byte{1}, value) +} diff --git a/kv/fsm_migration_deterministic_test.go b/kv/fsm_migration_deterministic_test.go new file mode 100644 index 000000000..60df0e2e4 --- /dev/null +++ b/kv/fsm_migration_deterministic_test.go @@ -0,0 +1,238 @@ +package kv + +import ( + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +type forbiddenApplyWatcher struct{} + +func (forbiddenApplyWatcher) Current() (RouteSnapshot, bool) { + panic("migration apply consulted the catalog watcher") +} + +func (forbiddenApplyWatcher) SnapshotAt(uint64) (RouteSnapshot, bool) { + panic("migration apply consulted watcher history") +} + +func deterministicMigrationGuard(t *testing.T, st store.MVCCStore, source bool) { + t.Helper() + writer, ok := st.(store.MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), store.TargetStagedReadinessState{ + JobID: 7, MigrationJobID: 7, Armed: true, RouteStart: []byte("m"), RouteEnd: []byte("z"), + ExpectedCutoverVersion: 10, MinWriteTSExclusive: 100, TrackWrites: source, RetentionPinTS: 1, + }, 8)) +} + +func deterministicMigrationOwner(t *testing.T, st store.MVCCStore) { + t.Helper() + deterministicMigrationGuard(t, st, false) + writer, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + require.NoError(t, writer.AcquireMigrationOwnership(t.Context(), store.MigrationOwnershipState{ + Generation: "0123456789abcdef0123456789abcdef", RouteID: 3, DescriptorHash: make([]byte, 32), + JobID: 7, LocalGroupID: 1, OwnerGroupID: 1, Role: store.MigrationOwner, + RouteStart: []byte("m"), RouteEnd: []byte("z"), CatalogVersion: 10, + ConfigurationIndex: 3, MinWriteTSExclusive: 100, Staged: true, + }, 9)) +} + +func deterministicRawRequest(version uint64, keys ...string) *pb.Request { + r := &pb.Request{Ts: 200, ObservedRouteVersion: version} + for _, key := range keys { + r.Mutations = append(r.Mutations, &pb.Mutation{Op: pb.Op_PUT, Key: []byte(key), Value: []byte("value")}) + } + return r +} + +func TestDeterministicMigrationApplyRequiresCompleteProof(t *testing.T) { + for _, kind := range []string{"target-guard", "source-tracker", "future-owner", "mixed-owner", "exact-owner"} { + t.Run(kind, func(t *testing.T) { + st := store.NewMVCCStore() + f := &kvFSM{store: st, routes: forbiddenApplyWatcher{}, shardGroupID: 1} + r := deterministicRawRequest(10, "n") + switch kind { + case "target-guard", "source-tracker": + deterministicMigrationGuard(t, st, kind == "source-tracker") + default: + deterministicMigrationOwner(t, st) + } + if kind == "future-owner" { + r.ObservedRouteVersion = 11 + } + if kind == "mixed-owner" { + r = deterministicRawRequest(10, "b", "n") + } + err := f.applyRequestErr(t.Context(), r) + if kind == "exact-owner" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, ErrRouteCutoverPending) + require.NoError(t, haltApplyOf(applyErrorResponse(err))) + exists, err := st.ExistsAt(t.Context(), []byte("b"), 200) + require.NoError(t, err) + require.False(t, exists) + exists, err = st.ExistsAt(t.Context(), []byte("n"), 200) + require.NoError(t, err) + require.False(t, exists) + }) + } +} + +func TestDeterministicMigrationUnaffectedLegacyRequest(t *testing.T) { + st := store.NewMVCCStore() + deterministicMigrationOwner(t, st) + e := distribution.NewEngine() + applyComposed1Snapshot(t, e, 10, []distribution.RouteDescriptor{{RouteID: 1, GroupID: 1, State: distribution.RouteStateActive}}) + f := &kvFSM{store: st, routes: WrapDistributionEngine(e), shardGroupID: 1} + require.NoError(t, f.applyRequestErr(t.Context(), deterministicRawRequest(0, "b"))) +} + +func TestCheckMigrationRequestIsReadOnlyAndAdvisory(t *testing.T) { + st := store.NewMVCCStore() + deterministicMigrationOwner(t, st) + r := deterministicRawRequest(10, "n") + require.NoError(t, CheckMigrationRequest(t.Context(), st, 1, r)) + exists, err := st.ExistsAt(t.Context(), []byte("n"), 200) + require.NoError(t, err) + require.False(t, exists) + require.NoError(t, CheckMigrationRequest(t.Context(), st, 1, deterministicRawRequest(0, "b"))) + r.ObservedRouteVersion = 11 + f := &kvFSM{store: st, routes: forbiddenApplyWatcher{}, shardGroupID: 1} + require.ErrorIs(t, f.applyRequestErr(t.Context(), r), ErrRouteCutoverPending) + require.ErrorIs(t, CheckMigrationRequest(t.Context(), st, 1, r), ErrRouteCutoverPending) + exists, err = st.ExistsAt(t.Context(), []byte("n"), 200) + require.NoError(t, err) + require.False(t, exists) +} + +func TestDeterministicMigrationResolutionDrainsAndRetries(t *testing.T) { + for _, phase := range []pb.Phase{pb.Phase_COMMIT, pb.Phase_ABORT} { + t.Run(phase.String(), func(t *testing.T) { + f := &kvFSM{store: store.NewMVCCStore(), shardGroupID: 1} + r := applyRouteContextTxn(pb.Phase_PREPARE, 20, 60, []byte("n")) + r.ObservedRouteVersion = 0 + require.NoError(t, f.applyRequestErr(t.Context(), r)) + retireServingSource(t, f.store) + f.routes = forbiddenApplyWatcher{} + r.Phase = phase + r.ObservedRouteVersion = 999 + require.NoError(t, f.applyRequestErr(t.Context(), r)) + require.NoError(t, f.applyRequestErr(t.Context(), r), "idempotent resolution cannot require fresh ownership") + value, err := f.store.GetAt(t.Context(), []byte("n"), 200) + if phase == pb.Phase_COMMIT { + require.NoError(t, err) + require.Equal(t, []byte("new"), value) + } else { + require.ErrorIs(t, err, store.ErrKeyNotFound) + } + }) + } +} + +func TestDeterministicMigrationRemotePrimaryAndStagedReadAlias(t *testing.T) { + st := store.NewMVCCStore() + deterministicMigrationOwner(t, st) + f := &kvFSM{store: st, routes: forbiddenApplyWatcher{}, shardGroupID: 1} + r := applyRouteContextTxn(pb.Phase_PREPARE, 150, 200, []byte("n")) + r.ObservedRouteVersion = 10 + r.Mutations[0].Value = EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("b-remote"), LockTTLms: 1000, CommitTS: 200}) + r.ReadKeys = [][]byte{distribution.MigrationStagedDataKey(7, []byte("n"))} + require.NoError(t, f.applyRequestErr(t.Context(), r), "remote primary is not part of this data group's footprint") + r.Phase = pb.Phase_COMMIT + r.ObservedRouteVersion = 999 + require.NoError(t, f.applyRequestErr(t.Context(), r)) + require.NoError(t, f.applyRequestErr(t.Context(), r), "resolved secondary is a no-op without consulting remote ownership") + _, err := st.GetAt(t.Context(), txnCommitKey([]byte("b-remote"), 150), 200) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +func TestDeterministicMigrationResolutionRejectsUnpreparedAndMismatchedKeys(t *testing.T) { + for _, mismatch := range []string{"start", "primary", "chosen", "missing-primary", "extra-key"} { + t.Run(mismatch, func(t *testing.T) { + f := &kvFSM{store: store.NewMVCCStore(), shardGroupID: 1} + r := applyRouteContextTxn(pb.Phase_PREPARE, 20, 60, []byte("n")) + r.ObservedRouteVersion = 0 + r.Mutations[0].Value = EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("n"), PrepareCommitTS: 60}) + // Install the same durable V3 lock as an already upgraded PREPARE; + // this fixture does not activate the separate Phase-D protocol. + lock := txnLock{StartTS: 20, CommitTS: 60, PrimaryKey: []byte("n"), IsPrimaryKey: true} + require.NoError(t, f.store.PutAt(t.Context(), txnLockKey([]byte("n")), encodeTxnLock(lock), 20, 0)) + require.NoError(t, f.store.PutAt(t.Context(), txnIntentKey([]byte("n")), encodeTxnIntent(txnIntent{StartTS: 20, Value: []byte("prepared")}), 20, 0)) + retireServingSource(t, f.store) + f.routes = forbiddenApplyWatcher{} + r.Phase = pb.Phase_COMMIT + r.Mutations[0].Value = EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("n"), CommitTS: 60}) + switch mismatch { + case "start": + r.Ts++ + case "primary": + r.Mutations[0].Value = EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("other"), CommitTS: 60}) + case "chosen": + r.Mutations[0].Value = EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("n"), CommitTS: 61}) + case "missing-primary": + r.Mutations[1].Key = []byte("o") + r.Mutations[0].Value = EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("o"), CommitTS: 60}) + case "extra-key": + r.Mutations = append(r.Mutations, &pb.Mutation{Op: pb.Op_PUT, Key: []byte("o"), Value: []byte("forged")}) + } + err := f.applyRequestErr(t.Context(), r) + switch mismatch { + case "extra-key": + require.NoError(t, err, "missing secondary cannot materialize caller-supplied data") + case "chosen": + require.ErrorIs(t, err, ErrTxnInvalidMeta) + default: + require.ErrorIs(t, err, ErrRouteCutoverPending) + } + _, err = f.store.GetAt(t.Context(), []byte("o"), 200) + require.ErrorIs(t, err, store.ErrKeyNotFound) + }) + } +} + +func TestDeterministicMigrationCoalescedContextIsPerRequest(t *testing.T) { + st, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + deterministicMigrationOwner(t, st) + e := distribution.NewEngine() + applyComposed1Snapshot(t, e, 999, []distribution.RouteDescriptor{{ + RouteID: 1, GroupID: 1, State: distribution.RouteStateWriteFenced, + }}) + f := &kvFSM{store: st, routes: WrapDistributionEngine(e), shardGroupID: 1, pendingApplyIdx: 20} + requests := []*pb.Request{ + deterministicRawRequest(10, "n"), + deterministicRawRequest(0, "b"), + deterministicRawRequest(11, "o"), + deterministicRawRequest(10, "p"), + } + requests[3].Ts = 203 + data, err := proto.Marshal(&pb.RaftCommand{Requests: requests}) + require.NoError(t, err) + response := f.Apply(data) + require.NoError(t, haltApplyOf(response)) + results, ok := response.(*fsmApplyResponse) + require.True(t, ok) + require.NoError(t, results.results[0]) + require.ErrorIs(t, results.results[1], ErrRouteWriteFenced, "legacy request cannot inherit the preceding ownership context") + require.ErrorIs(t, results.results[2], ErrRouteCutoverPending) + require.NoError(t, results.results[3]) + for _, key := range []string{"n", "p"} { + exists, err := st.ExistsAt(t.Context(), []byte(key), 300) + require.NoError(t, err) + require.True(t, exists) + } + for _, key := range []string{"b", "o"} { + exists, err := st.ExistsAt(t.Context(), []byte(key), 300) + require.NoError(t, err) + require.False(t, exists) + } +} diff --git a/kv/fsm_migration_evidence_fixture_test.go b/kv/fsm_migration_evidence_fixture_test.go new file mode 100644 index 000000000..31b60fa80 --- /dev/null +++ b/kv/fsm_migration_evidence_fixture_test.go @@ -0,0 +1,82 @@ +package kv + +import ( + "crypto/sha256" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func installStagedOwnershipEvidence(t *testing.T, f *kvFSM, version uint64, route distribution.RouteDescriptor) store.MigrationOwnershipState { + t.Helper() + require.NoError(t, f.RecordAppliedConfiguration(1)) + guard, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{ + JobId: route.MigrationJobID, MigrationJobId: route.MigrationJobID, Armed: true, + RouteStart: route.Start, RouteEnd: route.End, ExpectedCutoverVersion: version, + MinWriteTsExclusive: route.MinWriteTSExclusive, + }) + require.NoError(t, err) + f.SetApplyIndex(2) + require.Nil(t, f.Apply(guard)) + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(route, true) + require.NoError(t, err) + digest := sha256.Sum256(encoded) + acquire, err := MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{ + JobId: route.MigrationJobID, Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_ACQUIRE, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: 1, + Ownership: &pb.MigrationOwnershipProof{ + Generation: "0123456789abcdef0123456789abcdef", LocalGroupId: f.shardGroupID, OwnerGroupId: route.GroupID, + CatalogVersion: version, RouteId: route.RouteID, RouteStart: route.Start, RouteEnd: route.End, + MinWriteTsExclusive: route.MinWriteTSExclusive, WriteFloorKnown: true, DescriptorHash: digest[:], + Role: pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER, Staged: true, + }, + }) + require.NoError(t, err) + f.SetApplyIndex(3) + response := f.Apply(acquire) + result, ok := response.(store.CleanupVersionsResult) + require.True(t, ok, "ownership control returned %v", response) + require.True(t, result.Done) + reader, ok := f.store.(store.MigrationOwnershipStore) + require.True(t, ok) + view, err := reader.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Len(t, view.Readiness, 1) + require.Len(t, view.Ownership, 1) + proof := view.Ownership[0] + require.True(t, store.MigrationOwnershipEvidenceValid(proof)) + require.True(t, migrationTargetOwnershipMatches(proof, view.Readiness[0])) + require.Len(t, proof.Receipts, 1) + require.Equal(t, store.MigrationOwnershipAcquire, proof.Receipts[0].Operation) + require.Equal(t, uint64(1), proof.ConfigurationIndex) + require.Equal(t, uint64(3), proof.AppliedIndex) + require.Equal(t, digest[:], proof.Receipts[0].DescriptorHash) + return proof +} + +func TestStagedOwnershipEvidenceRequiresDescriptorIdentity(t *testing.T) { + f := &kvFSM{store: store.NewMVCCStore(), shardGroupID: 1} + proof := installStagedOwnershipEvidence(t, f, 7, applyRouteContextDescriptor()) + for _, field := range []string{"generation", "route ID", "descriptor hash", "receipt hash", "receipt"} { + t.Run(field, func(t *testing.T) { + incomplete := proof + incomplete.Receipts = append([]store.MigrationOwnershipReceipt(nil), proof.Receipts...) + switch field { + case "generation": + incomplete.Generation = "" + case "route ID": + incomplete.RouteID = 0 + case "descriptor hash": + incomplete.DescriptorHash = nil + case "receipt hash": + incomplete.Receipts[0].DescriptorHash = nil + case "receipt": + incomplete.Receipts = nil + } + require.False(t, store.MigrationOwnershipEvidenceValid(incomplete), field) + }) + } +} diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 23660780c..b6c7ad9ed 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -1,6 +1,7 @@ package kv import ( + "bytes" "context" "testing" @@ -9,6 +10,7 @@ import ( "github.com/bootjp/elastickv/internal/s3keys" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" ) @@ -541,7 +543,7 @@ func TestFSMComposed1UsesS3BucketAuxiliaryRouteOwner(t *testing.T) { applyComposed1Snapshot(t, engine, 1, s3BucketAuxiliaryStagedRoutes(bucket, 3, 4)) fsm := newComposed1FSM(t, engine, 4) - err := fsm.verifyComposed1(&pb.Request{ + err := fsm.verifyComposed1(context.Background(), &pb.Request{ IsTxn: true, Phase: pb.Phase_PREPARE, Ts: 10, @@ -808,3 +810,976 @@ func TestFSMTimestampFloorUsesObservedSnapshotDuringApply(t *testing.T) { require.NoError(t, getErr) require.Equal(t, []byte("proposed-before-floor"), got) } + +func newTargetReadinessFSM(t *testing.T, route distribution.RouteDescriptor) *kvFSM { + t.Helper() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{route}) + fsm := newComposed1FSM(t, engine, route.GroupID) + writer, ok := fsm.store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + })) + return fsm +} + +func applyTargetReadinessToFSM(t *testing.T, fsm *kvFSM, state store.TargetStagedReadinessState) { + t.Helper() + + writer, ok := fsm.store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), state)) +} + +func applySourceMigrationControlToFSM(t *testing.T, fsm *kvFSM, readFence bool) { + t.Helper() + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 10, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 10, + MinWriteTSExclusive: 50, + Armed: true, + SourceWriteFence: true, + SourceReadFence: readFence, + RetentionPinTS: 40, + }) +} + +func newMigrationWriteTrackerFSM(t *testing.T) *kvFSM { + t.Helper() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + fsm := newComposed1FSM(t, engine, 1) + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 11, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 11, + MinWriteTSExclusive: 1, + Armed: true, + RetentionPinTS: 1, + TrackWrites: true, + }) + return fsm +} + +func migrationTrackerMinimum(t *testing.T, fsm *kvFSM) uint64 { + t.Helper() + reader, ok := fsm.store.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(context.Background()) + require.NoError(t, err) + require.Len(t, states, 1) + return states[0].MinAdmittedTS +} + +func TestFSMMigrationWriteTrackerCoversAllAdmissionPaths(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newMigrationWriteTrackerFSM(t) + require.NoError(t, fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("raw")}}, + }, 80)) + require.Equal(t, uint64(80), migrationTrackerMinimum(t, fsm)) + + require.NoError(t, fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("m")}}, + }, 70)) + require.Equal(t, uint64(70), migrationTrackerMinimum(t, fsm)) + + // Live one-phase commits must be above the store-wide apply-order fence. + // Use a fresh tracker to verify this admission path independently. + fsm = newMigrationWriteTrackerFSM(t) + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Ts: 50, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("o"), CommitTS: 60})}, + {Op: pb.Op_PUT, Key: []byte("o"), Value: []byte("one-phase")}, + }, + }, 60)) + require.Equal(t, uint64(60), migrationTrackerMinimum(t, fsm)) + + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 40, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("p"), CommitTS: 55, LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_PUT, Key: []byte("p"), Value: []byte("prepared")}, + }, + }, 40)) + require.Equal(t, uint64(55), migrationTrackerMinimum(t, fsm)) +} + +func TestFSMMigrationWriteTrackerRecordsCommitPreparedBeforeArm(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + fsm := newComposed1FSM(t, engine, 1) + startTS, commitTS := uint64(40), uint64(60) + meta := &pb.Mutation{ + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("n"), + CommitTS: commitTS, + LockTTLms: defaultTxnLockTTLms, + }), + } + mutation := &pb.Mutation{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("value")} + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: startTS, + Mutations: []*pb.Mutation{meta, mutation}, + }, startTS)) + + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 12, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 12, + MinWriteTSExclusive: 1, + Armed: true, + RetentionPinTS: 1, + TrackWrites: true, + }) + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_COMMIT, + Ts: startTS, + Mutations: []*pb.Mutation{meta, mutation}, + }, commitTS)) + require.Equal(t, commitTS, migrationTrackerMinimum(t, fsm)) +} + +func newReadinessReadKeyFSM(t *testing.T) *kvFSM { + t.Helper() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: []byte("z"), GroupID: 1, State: distribution.RouteStateActive}, + }) + fsm := newComposed1FSM(t, engine, 1) + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + return fsm +} + +func TestFSMRejectsRawPointWriteOnWriteFencedRoute(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) + + _, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMDurableSourceFenceRejectsRawWriteDespiteCatalogBypass(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + applySourceMigrationControlToFSM(t, fsm, false) + key := []byte("n") + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + WriteFenceBypassKeys: [][]byte{key}, + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, + }, 60) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + +func TestFSMDurableSourceFenceRejectsPrefixWrite(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + applySourceMigrationControlToFSM(t, fsm, false) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("m")}}, + }, 60) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + +func TestFSMDurableSourceFenceRejectsPrefixWriteOnActiveRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("m"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + fsm := newComposed1FSM(t, engine, 1) + applySourceMigrationControlToFSM(t, fsm, false) + require.NoError(t, fsm.store.PutAt(ctx, []byte("m/key"), []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("m")}}, + }, 60) + require.ErrorIs(t, err, ErrRouteWriteFenced) + + got, getErr := fsm.store.GetAt(ctx, []byte("m/key"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestFSMWriteFenceBypassAllowsPinnedOnePhaseTxnOnNonOwningGroup(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateWriteFenced}, + }) + fsm := newComposed1FSM(t, engine, 1) + key := []byte("z") + err := fsm.handleTxnRequest(context.Background(), &pb.Request{ + IsTxn: true, + Ts: 10, + ObservedRouteVersion: 1, + WriteFenceBypassKeys: [][]byte{key}, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key})}, + {Op: pb.Op_PUT, Key: key, Value: []byte("v")}, + }, + }, 20) + require.NoError(t, err) + + got, err := fsm.store.GetAt(context.Background(), key, 20) + require.NoError(t, err) + require.Equal(t, []byte("v"), got) +} + +func TestFSMDurableSourceFenceRejectsPinnedOnePhaseTxn(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + applySourceMigrationControlToFSM(t, fsm, false) + key := []byte("n") + err := fsm.handleTxnRequest(context.Background(), &pb.Request{ + IsTxn: true, + Ts: 50, + WriteFenceBypassKeys: [][]byte{key}, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key})}, + {Op: pb.Op_PUT, Key: key, Value: []byte("v")}, + }, + }, 60) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + +func TestFSMRejectsRawPointWriteWithoutTargetReadinessProof(t *testing.T) { + t.Parallel() + + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}}, + }, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMAllowsRawPointWriteWithClearedTargetReadinessProof(t *testing.T) { + t.Parallel() + + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}}, + }, 101) + require.NoError(t, err) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestFSMRejectsTargetReadinessProofFromAnotherGroup(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 2, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + }) + fsm := newComposed1FSM(t, engine, 1) + writer, ok := fsm.store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + })) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}}, + }, 101) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestFSMRejectsS3BucketAuxiliaryPointWriteOnWriteFencedRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const bucket = "bucket-a" + fsm := newS3BucketAuxiliaryWriteFencedFSM(t, bucket) + + for _, key := range [][]byte{ + s3keys.BucketMetaKey(bucket), + s3keys.BucketGenerationKey(bucket), + } { + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) + + _, getErr := fsm.store.GetAt(ctx, key, ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) + } +} + +func TestFSMRejectsS3BucketAuxiliaryPointWriteBelowRouteFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const bucket = "bucket-a" + routeStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + routeEnd := prefixScanEnd(routeStart) + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}) + fsm := newComposed1FSM(t, engine, 1) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: s3keys.BucketMetaKey(bucket), Value: []byte("v")}}, + }, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) +} + +func TestFSMIgnoresRawRouteFloorForS3BucketAuxiliaryPointWrite(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const bucket = "bucket-a" + key := s3keys.BucketMetaKey(bucket) + auxStart, auxEnd, ok := s3BucketAuxiliaryRouteRange(key) + require.True(t, ok) + rawStart := []byte(s3keys.BucketMetaPrefix) + rawEnd := prefixScanEnd(rawStart) + require.Less(t, bytes.Compare(auxEnd, rawStart), 0) + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: auxStart, + End: auxEnd, + GroupID: 1, + State: distribution.RouteStateActive, + }, + { + RouteID: 2, + Start: rawStart, + End: rawEnd, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + }) + fsm := newComposed1FSM(t, engine, 1) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, + }, 100) + require.NoError(t, err) + got, err := fsm.store.GetAt(ctx, key, 100) + require.NoError(t, err) + require.Equal(t, []byte("v"), got) +} + +func TestFSMRejectsS3BucketAuxiliaryPointWriteWithoutTargetReadinessProof(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const bucket = "bucket-a" + routeStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + routeEnd := prefixScanEnd(routeStart) + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 1, + State: distribution.RouteStateActive, + }}) + fsm := newComposed1FSM(t, engine, 1) + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeEnd, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: s3keys.BucketGenerationKey(bucket), Value: []byte("v")}}, + }, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestFSMRejectsDelPrefixIntersectingWriteFencedRoute(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + require.NoError(t, fsm.store.PutAt(context.Background(), []byte("z"), []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("z")}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestFSMRejectsDelPrefixWithoutTargetReadinessProof(t *testing.T) { + t.Parallel() + + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + require.NoError(t, fsm.store.PutAt(context.Background(), []byte("b"), []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("b")}}, + }, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestFSMDelPrefixTombstonesStagedVisibilityRows(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }) + rawKey := []byte("b") + stagedKey := distribution.MigrationStagedDataKey(9, rawKey) + require.NoError(t, fsm.store.PutAt(ctx, stagedKey, []byte("staged"), 110, 0)) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: rawKey}}, + }, 120) + require.NoError(t, err) + + _, err = fsm.store.GetAt(ctx, stagedKey, 130) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +func TestFSMRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + require.NoError(t, fsm.store.PutAt(context.Background(), []byte("z"), []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: nil}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestFSMRejectsRawPointWriteBelowRouteFloor(t *testing.T) { + t.Parallel() + + fsm := newWriteFloorFSM(t) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}}, + }, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + _, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMRejectsDelPrefixBelowRouteFloor(t *testing.T) { + t.Parallel() + + fsm := newWriteFloorFSM(t) + require.NoError(t, fsm.store.PutAt(context.Background(), []byte("b"), []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("b")}}, + }, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestFSMRejectsBroadInternalDelPrefixWhenRouteIsWriteFenced(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + key := []byte("!redis|string|z") + require.NoError(t, fsm.store.PutAt(context.Background(), key, []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("!redis|")}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) + + got, getErr := fsm.store.GetAt(context.Background(), key, ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestFSMRejectsOnePhaseTxnBelowRouteFloor(t *testing.T) { + t.Parallel() + + fsm := newWriteFloorFSM(t) + err := fsm.handleTxnRequest(context.Background(), onePhaseReq(10, 100, 0, []byte("b"), []byte("v")), 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + _, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMOnePhaseDedupChecksTargetReadinessBeforeNoOp(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + require.NoError(t, fsm.store.PutAt(ctx, []byte("b"), []byte("landed"), 20, 0)) + + err := fsm.handleTxnRequest(ctx, onePhaseReq(30, 40, 20, []byte("b"), []byte("retry")), 40) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + landedAt40, probeErr := fsm.store.CommittedVersionAt(ctx, []byte("b"), 40) + require.NoError(t, probeErr) + require.False(t, landedAt40) +} + +func TestFSMOnePhaseTxnChecksReadKeysForTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newReadinessReadKeyFSM(t) + req := onePhaseReq(10, 20, 0, []byte("b"), []byte("v")) + req.ReadKeys = [][]byte{[]byte("n")} + + err := fsm.handleTxnRequest(ctx, req, 20) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, getErr := fsm.store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMPrepareTxnChecksReadKeysForTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newReadinessReadKeyFSM(t) + req := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, + ReadKeys: [][]byte{[]byte("n")}, + Mutations: []*pb.Mutation{ + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("b"), + LockTTLms: defaultTxnLockTTLms, + }), + }, + {Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}, + }, + } + + err := fsm.handleTxnRequest(ctx, req, 10) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, getErr := fsm.store.GetAt(ctx, txnLockKey([]byte("b")), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMOnePhaseTxnChecksSourceReadFenceForReadKeys(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{ + RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 1, State: distribution.RouteStateActive, + }}) + fsm := newComposed1FSM(t, engine, 1) + applySourceMigrationControlToFSM(t, fsm, true) + req := onePhaseReq(10, 20, 0, []byte("b"), []byte("v")) + req.ReadKeys = [][]byte{[]byte("n")} + + err := fsm.handleTxnRequest(ctx, req, 20) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, getErr := fsm.store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMPrepareTxnSkipsRemotePrimaryForTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 2, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: []byte("z"), GroupID: 1, State: distribution.RouteStateActive}, + }) + fsm := newComposed1FSM(t, engine, 1) + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("m"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + + err := fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, + Mutations: []*pb.Mutation{ + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("b"), + LockTTLms: defaultTxnLockTTLms, + }), + }, + {Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("v")}, + }, + }, 10) + require.NoError(t, err) + + _, getErr := fsm.store.GetAt(ctx, txnLockKey([]byte("n")), ^uint64(0)) + require.NoError(t, getErr) +} + +func TestFSMPrepareTxnChecksStagedVisibilityWriteConflicts(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }) + require.NoError(t, fsm.store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b")), []byte("staged"), 20, 0)) + + err := fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, + Mutations: []*pb.Mutation{ + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("b"), + CommitTS: 120, + LockTTLms: defaultTxnLockTTLms, + }), + }, + {Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}, + }, + }, 10) + require.ErrorIs(t, err, store.ErrWriteConflict) + + _, getErr := fsm.store.GetAt(ctx, txnLockKey([]byte("b")), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMOnePhaseTxnChecksStagedVisibilityReadConflicts(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }) + require.NoError(t, fsm.store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("n")), []byte("staged"), 20, 0)) + + req := onePhaseReq(10, 120, 0, []byte("b"), []byte("v")) + req.ReadKeys = [][]byte{[]byte("n")} + + err := fsm.handleTxnRequest(ctx, req, 120) + require.ErrorIs(t, err, store.ErrWriteConflict) + + _, getErr := fsm.store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMPrepareUsesCommitTSForRouteFloorWhenPresent(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newWriteFloorFSM(t) + commitTS := uint64(101) + primaryKey := []byte("b") + err := fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 50, + Mutations: []*pb.Mutation{ + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, LockTTLms: defaultTxnLockTTLms, CommitTS: commitTS}), + }, + {Op: pb.Op_PUT, Key: primaryKey, Value: []byte("v")}, + }, + }, 50) + require.NoError(t, err) + + rawLock, err := fsm.store.GetAt(ctx, txnLockKey(primaryKey), ^uint64(0)) + require.NoError(t, err) + lock, err := decodeTxnLock(rawLock) + require.NoError(t, err) + require.Zero(t, lock.CommitTS, "a legacy floor hint must not become durable prepare evidence") +} + +func TestFSMPrepareWithoutCommitTSUsesStartTSForRouteFloor(t *testing.T) { + t.Parallel() + + fsm := newWriteFloorFSM(t) + err := fsm.handleTxnRequest(context.Background(), &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 100, + Mutations: []*pb.Mutation{ + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("b"), LockTTLms: defaultTxnLockTTLms}), + }, + {Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}, + }, + }, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) +} + +func TestFSMRejectsPrepareOnWriteFencedRouteButAllowsAbort(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newWriteFencedFSM(t) + prepare := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, + }, + } + require.ErrorIs(t, fsm.handleTxnRequest(ctx, prepare, 10), ErrRouteWriteFenced) + + abort := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_ABORT, + Ts: 11, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 11})}, + {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, + }, + } + err := fsm.handleTxnRequest(ctx, abort, 11) + require.False(t, errors.Is(err, ErrRouteWriteFenced), "ABORT must keep the narrow cleanup lane open") +} + +func TestFSMAbortCleanupBypassesRetainedWriteFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newWriteFloorFSM(t) + startTS := uint64(10) + abortTS := uint64(11) + primaryKey := []byte("b") + require.NoError(t, fsm.store.PutAt(ctx, txnLockKey(primaryKey), encodeTxnLock(txnLock{ + StartTS: startTS, + PrimaryKey: primaryKey, + IsPrimaryKey: true, + }), startTS, 0)) + require.NoError(t, fsm.store.PutAt(ctx, txnIntentKey(primaryKey), encodeTxnIntent(txnIntent{ + StartTS: startTS, + Op: txnIntentOpPut, + Value: []byte("v"), + }), startTS, 0)) + + abort := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_ABORT, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, CommitTS: abortTS})}, + {Op: pb.Op_PUT, Key: primaryKey}, + }, + } + require.NoError(t, fsm.handleTxnRequest(ctx, abort, abortTS)) + + _, lockErr := fsm.store.GetAt(ctx, txnLockKey(primaryKey), ^uint64(0)) + require.ErrorIs(t, lockErr, store.ErrKeyNotFound) + _, intentErr := fsm.store.GetAt(ctx, txnIntentKey(primaryKey), ^uint64(0)) + require.ErrorIs(t, intentErr, store.ErrKeyNotFound) +} + +func TestFSMAbortCleanupBypassesTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + startTS := uint64(10) + abortTS := uint64(11) + primaryKey := []byte("b") + require.NoError(t, fsm.store.PutAt(ctx, txnLockKey(primaryKey), encodeTxnLock(txnLock{ + StartTS: startTS, + PrimaryKey: primaryKey, + IsPrimaryKey: true, + }), startTS, 0)) + require.NoError(t, fsm.store.PutAt(ctx, txnIntentKey(primaryKey), encodeTxnIntent(txnIntent{ + StartTS: startTS, + Op: txnIntentOpPut, + Value: []byte("v"), + }), startTS, 0)) + + abort := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_ABORT, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, CommitTS: abortTS})}, + {Op: pb.Op_PUT, Key: primaryKey}, + }, + } + require.NoError(t, fsm.handleTxnRequest(ctx, abort, abortTS)) + + _, lockErr := fsm.store.GetAt(ctx, txnLockKey(primaryKey), ^uint64(0)) + require.ErrorIs(t, lockErr, store.ErrKeyNotFound) + _, intentErr := fsm.store.GetAt(ctx, txnIntentKey(primaryKey), ^uint64(0)) + require.ErrorIs(t, intentErr, store.ErrKeyNotFound) +} diff --git a/kv/fsm_migration_import.go b/kv/fsm_migration_import.go index 870a6110a..dd0e61eff 100644 --- a/kv/fsm_migration_import.go +++ b/kv/fsm_migration_import.go @@ -74,7 +74,7 @@ func (f *kvFSM) validateImportedPrepareEvidence(versions []*pb.MVCCVersion) erro if version == nil { continue } - if isTxnPeerLocalControlKey(version.GetKey()) { + if isTxnPeerLocalControlKey(version.GetKey()) || store.IsMigrationRetirementMetadataKey(version.GetKey()) { return errors.Wrap(ErrInvalidRequest, "migration cannot import peer-local timestamp protocol state") } if !f.prepareEvidenceEnabled.Load() || version.GetTombstone() { @@ -120,7 +120,8 @@ func validateStagedKeySizes(versions []store.MVCCVersion) error { } func isMigrationImportOrdinaryApplyError(err error) bool { - return errors.Is(err, store.ErrImportBatchGap) || + return errors.Is(err, store.ErrMigrationRetired) || + errors.Is(err, store.ErrImportBatchGap) || errors.Is(err, store.ErrInvalidImportVersion) || errors.Is(err, store.ErrValueTooLarge) || errors.Is(err, store.ErrSnapshotKeyTooLarge) diff --git a/kv/fsm_migration_namespace_test.go b/kv/fsm_migration_namespace_test.go new file mode 100644 index 000000000..1d0babeb4 --- /dev/null +++ b/kv/fsm_migration_namespace_test.go @@ -0,0 +1,155 @@ +package kv + +import ( + "bytes" + "context" + "encoding/binary" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func migrationNamespaceKeys() [][]byte { + return [][]byte{[]byte("!migstage|ready|"), []byte("!migstage|ownership"), []byte("!migstage|retired"), + []byte("!migstage|ack|"), []byte("!migwrite|source"), []byte("!migfence|source"), + []byte("!dist|migstage|data"), store.MigrationConfigurationIndexKey(), + []byte("!txn|lock|customer"), backupTimestampFloorKey, prepareEvidenceEnabledKey, prepareEvidenceGateKey, prepareEvidenceTimestampFloorKey} +} + +func migrationNamespaceRequest(phase pb.Phase, key []byte) *pb.Request { + return &pb.Request{IsTxn: true, Phase: phase, Ts: 10, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key, CommitTS: 20})}, + {Op: pb.Op_PUT, Key: key, Value: []byte("not-control-metadata")}, + }} +} + +func TestMigrationNamespaceRejectsAllTransactionPhasesAndAdvisory(t *testing.T) { + for _, phase := range []pb.Phase{pb.Phase_NONE, pb.Phase_PREPARE, pb.Phase_COMMIT, pb.Phase_ABORT} { + for _, key := range migrationNamespaceKeys() { + t.Run(phase.String()+"/"+string(key), func(t *testing.T) { + st := store.NewMVCCStore() + f := &kvFSM{store: st, shardGroupID: 1} + activateMigrationNamespace(t, f) + request := migrationNamespaceRequest(phase, key) + if phase == pb.Phase_COMMIT || phase == pb.Phase_ABORT { + seedLegacyMigrationIntent(t, st, key) + } + require.ErrorIs(t, CheckMigrationRequest(t.Context(), st, 1, request), ErrInvalidRequest) + require.ErrorIs(t, applyFSMRequest(t, f, request), ErrInvalidRequest) + _, err := st.GetAt(t.Context(), key, ^uint64(0)) + if bytes.Equal(key, migrationConfigurationIndexKey) { + require.NoError(t, err) + require.NoError(t, RequireMigrationNamespaceAdmission(t.Context(), st)) + } else { + require.ErrorIs(t, err, store.ErrKeyNotFound) + } + if phase == pb.Phase_COMMIT || phase == pb.Phase_ABORT { + _, err = st.GetAt(t.Context(), txnIntentKey(key), ^uint64(0)) + require.NoError(t, err, "rejected resolution must not mutate the legacy intent") + } + }) + } + } +} + +func seedLegacyMigrationIntent(t *testing.T, st store.MVCCStore, key []byte) { + t.Helper() + require.NoError(t, st.PutAt(t.Context(), txnLockKey(key), encodeTxnLock(txnLock{ + StartTS: 10, CommitTS: 20, PrimaryKey: key, IsPrimaryKey: true}), 10, 0)) + require.NoError(t, st.PutAt(t.Context(), txnIntentKey(key), encodeTxnIntent(txnIntent{ + StartTS: 10, Op: txnIntentOpPut, Value: []byte("forged-before-upgrade")}), 10, 0)) +} + +func TestMigrationNamespacePebbleRemainsReopenableAfterRejectedWrite(t *testing.T) { + dir := t.TempDir() + st, err := store.NewPebbleStore(dir) + require.NoError(t, err) + f, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + activateMigrationNamespace(t, f) + assert.ErrorIs(t, applyFSMRequest(t, f, migrationNamespaceRequest(pb.Phase_NONE, []byte("!migstage|ready|"))), ErrInvalidRequest) + for _, key := range []string{"_migack", "_mighlc", "_migpromote"} { + require.NoError(t, applyFSMRequest(t, f, &pb.Request{Ts: 30, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(key), Value: []byte("ordinary")}, + }})) + } + control, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{ + JobId: 7, MigrationJobId: 7, RouteStart: []byte("b"), RouteEnd: []byte("z"), + ExpectedCutoverVersion: 1, MinWriteTsExclusive: 1, Armed: true}) + require.NoError(t, err) + f.SetApplyIndex(1) + require.Nil(t, f.Apply(control), "typed readiness controls must remain valid") + require.NoError(t, st.Close()) + reopened, err := store.NewPebbleStore(dir) + require.NoError(t, err, "logical MVCC keys must not poison raw readiness metadata on reopen") + reader, ok := reopened.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(t.Context()) + require.NoError(t, err) + require.Len(t, states, 1) + require.Equal(t, uint64(7), states[0].JobID) + require.True(t, states[0].Armed) + for _, key := range []string{"_migack", "_mighlc", "_migpromote"} { + value, getErr := reopened.GetAt(t.Context(), []byte(key), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("ordinary"), value) + } + require.NoError(t, reopened.Close()) +} + +func TestMigrationNamespacePreservesOrdinaryAliases(t *testing.T) { + keys := append(legacyPrepareEvidenceUserKeys(), []byte("_migack"), []byte("_mighlc"), []byte("_migpromote"), + []byte("!migstage-user"), []byte("!migstage"), []byte("!txn|foo"), []byte("!txn|backup|customer"), + []byte("!hs|!migstage|ready||field"), []byte("customer")) + for _, key := range keys { + t.Run(string(key), func(t *testing.T) { + st := store.NewMVCCStore() + f := &kvFSM{store: st, shardGroupID: 1} + activateMigrationNamespace(t, f) + request := migrationNamespaceRequest(pb.Phase_NONE, key) + require.NoError(t, CheckMigrationRequest(context.Background(), st, 1, request)) + require.NoError(t, applyFSMRequest(t, f, request)) + value, err := st.GetAt(t.Context(), key, 20) + require.NoError(t, err) + require.Equal(t, []byte("not-control-metadata"), value) + raw := &pb.Request{Ts: 30, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("raw")}}} + require.NoError(t, CheckMigrationRequest(t.Context(), st, 1, raw)) + require.NoError(t, applyFSMRequest(t, f, raw)) + }) + } +} + +func TestMigrationNamespaceRejectsPrefixDeletesAndCancelledAdvisory(t *testing.T) { + for _, key := range [][]byte{[]byte("!migs"), []byte("!migstage|ready|"), []byte("!dist|migstage|"), []byte("!migwrite|"), []byte("!migfence|")} { + for _, phase := range []pb.Phase{pb.Phase_NONE, pb.Phase_PREPARE, pb.Phase_COMMIT, pb.Phase_ABORT} { + st := store.NewMVCCStore() + f := &kvFSM{store: st, shardGroupID: 1} + activateMigrationNamespace(t, f) + request := migrationNamespaceRequest(phase, key) + request.Mutations[1].Op = pb.Op_DEL_PREFIX + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.ErrorIs(t, CheckMigrationRequest(ctx, st, 1, request), ErrInvalidRequest) + require.ErrorIs(t, applyFSMRequest(t, f, request), ErrInvalidRequest) + } + } +} + +func TestMigrationNamespacePreservesLegacyRawFlush(t *testing.T) { + st := store.NewMVCCStore() + f := &kvFSM{store: st, shardGroupID: 1} + require.NoError(t, st.PutAt(t.Context(), []byte("customer"), []byte("ordinary"), 10, 0)) + configuration := binary.BigEndian.AppendUint64(nil, 5) + require.NoError(t, st.PutAt(t.Context(), store.MigrationConfigurationIndexKey(), configuration, 10, 0)) + request := &pb.Request{Ts: 20, Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX}}} + require.NoError(t, CheckMigrationRequest(t.Context(), st, 1, request)) + require.NoError(t, applyFSMRequest(t, f, request)) + _, err := st.GetAt(t.Context(), []byte("customer"), 20) + require.ErrorIs(t, err, store.ErrKeyNotFound) + control, err := st.GetAt(t.Context(), store.MigrationConfigurationIndexKey(), 20) + require.NoError(t, err) + require.Equal(t, configuration, control) +} diff --git a/kv/fsm_migration_ownership_control.go b/kv/fsm_migration_ownership_control.go new file mode 100644 index 000000000..cf9bf6d1a --- /dev/null +++ b/kv/fsm_migration_ownership_control.go @@ -0,0 +1,143 @@ +package kv + +import ( + "bytes" + "context" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +func MigrationOwnershipStateFromProto(jobID uint64, proof *pb.MigrationOwnershipProof) (store.MigrationOwnershipState, error) { + if proof == nil || len(proof.GetGeneration()) < 16 || len(proof.GetGeneration()) > 128 || proof.GetRouteId() == 0 || len(proof.GetDescriptorHash()) != 32 { + return store.MigrationOwnershipState{}, errors.WithStack(ErrInvalidRequest) + } + var role store.MigrationOwnershipRole + switch proof.GetRole() { + case pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER: + role = store.MigrationOwner + case pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_NONOWNER: + role = store.MigrationNonOwner + case pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_UNSPECIFIED: + return store.MigrationOwnershipState{}, errors.WithStack(ErrInvalidRequest) + default: + return store.MigrationOwnershipState{}, errors.WithStack(ErrInvalidRequest) + } + return store.MigrationOwnershipState{Generation: proof.GetGeneration(), JobID: jobID, + WriteFloorKnown: proof.GetWriteFloorKnown(), SourceTrackerVersion: proof.GetSourceTrackerVersion(), SourceAppliedIndex: proof.GetSourceAppliedIndex(), + LocalGroupID: proof.GetLocalGroupId(), OwnerGroupID: proof.GetOwnerGroupId(), CatalogVersion: proof.GetCatalogVersion(), + MinWriteTSExclusive: proof.GetMinWriteTsExclusive(), RouteStart: bytes.Clone(proof.GetRouteStart()), + RouteEnd: bytes.Clone(proof.GetRouteEnd()), RouteID: proof.GetRouteId(), ParentRouteID: proof.GetParentRouteId(), + Staged: proof.GetStaged(), Role: role, DescriptorHash: bytes.Clone(proof.GetDescriptorHash())}, nil +} + +func MigrationOwnershipProofFromState(state store.MigrationOwnershipState) *pb.MigrationOwnershipProof { + role := pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_NONOWNER + if state.Role == store.MigrationOwner { + role = pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER + } + return &pb.MigrationOwnershipProof{Generation: state.Generation, LocalGroupId: state.LocalGroupID, + WriteFloorKnown: state.WriteFloorKnown, SourceTrackerVersion: state.SourceTrackerVersion, SourceAppliedIndex: state.SourceAppliedIndex, + OwnerGroupId: state.OwnerGroupID, CatalogVersion: state.CatalogVersion, MinWriteTsExclusive: state.MinWriteTSExclusive, + RouteStart: bytes.Clone(state.RouteStart), RouteEnd: bytes.Clone(state.RouteEnd), RouteId: state.RouteID, + ParentRouteId: state.ParentRouteID, Staged: state.Staged, Role: role, DescriptorHash: bytes.Clone(state.DescriptorHash)} +} + +func migrationOwnershipControlMode(mode pb.MigrationCleanupMode) bool { + return mode == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_ACQUIRE || + mode == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR || + mode == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_SOURCE_OWNERSHIP_ACQUIRE || + mode == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH +} + +func (f *kvFSM) verifyLegacyMigrationMetadataClear(ctx context.Context, jobID uint64) error { + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + guards, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + for _, guard := range guards { + if guard.JobID == jobID && guard.Armed { + return errors.WithStack(store.ErrMigrationOwnership) + } + } + return nil +} + +func (f *kvFSM) applyMigrationOwnershipControl(ctx context.Context, req *pb.CleanupMigrationRequest) any { + if !req.GetRequireConfigurationIndex() || req.GetExpectedConfigurationIndex() == 0 { + return errors.WithStack(ErrMigrationConfigurationChanged) + } + state, err := MigrationOwnershipStateFromProto(req.GetJobId(), req.GetOwnership()) + if err != nil { + return err + } + if f.shardGroupID == 0 || state.LocalGroupID != f.shardGroupID { + return errors.WithStack(ErrInvalidRequest) + } + state.ConfigurationIndex = f.AppliedConfigurationIndex() + if !migrationOwnershipSourceModeValid(req.GetMode(), state) { + return errors.WithStack(ErrInvalidRequest) + } + err = f.applyMigrationOwnershipOperation(ctx, req.GetMode(), state) + if err == nil { + return store.CleanupVersionsResult{Done: true} + } + if migrationOwnershipOrdinaryError(err) { + return err + } + return haltErr(errors.Wrap(errors.Mark(err, ErrMigrationCleanupApply), "kv/fsm: ownership control")) +} + +func migrationOwnershipOrdinaryError(err error) bool { + return errors.Is(err, store.ErrMigrationOwnership) || errors.Is(err, store.ErrMigrationRetired) || errors.Is(err, store.ErrNotSupported) +} + +func (f *kvFSM) applyMigrationOwnershipOperation(ctx context.Context, mode pb.MigrationCleanupMode, state store.MigrationOwnershipState) error { + writer, ok := f.store.(store.MigrationOwnershipStore) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + switch mode { + case pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_ACQUIRE, + pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_SOURCE_OWNERSHIP_ACQUIRE: + return errors.WithStack(writer.AcquireMigrationOwnership(ctx, state, f.pendingApplyIdx)) + case pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR: + return errors.WithStack(writer.ClearMigrationStateWithOwnership(ctx, state, f.pendingApplyIdx)) + case pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH: + refresher, ok := f.store.(store.MigrationOwnershipRefresher) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + return errors.WithStack(refresher.RefreshMigrationOwnership(ctx, state, f.pendingApplyIdx)) + case pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_UNSPECIFIED, + pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION, + pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION, + pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, + pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA: + return errors.WithStack(ErrInvalidRequest) + default: + return errors.WithStack(ErrInvalidRequest) + } +} + +func migrationOwnershipSourceModeValid(mode pb.MigrationCleanupMode, state store.MigrationOwnershipState) bool { + switch mode { + case pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_SOURCE_OWNERSHIP_ACQUIRE: + return state.SourceTrackerVersion != 0 && state.SourceAppliedIndex == 0 && state.WriteFloorKnown + case pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH: + return true + case pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_UNSPECIFIED, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, + pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION, + pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION, + pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_ACQUIRE, + pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR: + return state.SourceTrackerVersion == 0 && state.SourceAppliedIndex == 0 + default: + return false + } +} diff --git a/kv/fsm_migration_partition.go b/kv/fsm_migration_partition.go new file mode 100644 index 000000000..609aa5c32 --- /dev/null +++ b/kv/fsm_migration_partition.go @@ -0,0 +1,479 @@ +package kv + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "slices" + "sort" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +const ( + MaxMigrationPartitionControlBytes = 4 << 20 + maxMigrationPartitionPriors = 256 +) + +// MigrationPartitionControl carries root evidence and exact native prior CASes. +// Neither catalog watchers nor immutable job proofs authorize this operation. +type MigrationPartitionControl struct { + Transition store.MigrationOwnershipPartitionTransition + PriorPartitions []store.MigrationOwnershipPartition + PriorOwnership []store.MigrationOwnershipState + Manifest []byte + UnchangedDescriptor *distribution.RouteDescriptor +} + +// DecodeMigrationPartitionJSON accepts only the encoder's exact byte format. +// Re-encoding rejects duplicate keys, case aliases, unknown fields, trailing +// input, insignificant whitespace and alternate number/base64 representations. +func DecodeMigrationPartitionJSON(raw []byte, target any) error { + if len(raw) == 0 || len(raw) > MaxMigrationPartitionControlBytes { + return errors.WithStack(ErrInvalidRequest) + } + d := json.NewDecoder(bytes.NewReader(raw)) + d.DisallowUnknownFields() + if err := d.Decode(target); err != nil { + return errors.Wrap(ErrInvalidRequest, "invalid partition JSON") + } + if err := d.Decode(new(any)); err != io.EOF { + return errors.WithStack(ErrInvalidRequest) + } + canonical, err := json.Marshal(target) + if err != nil || !bytes.Equal(raw, canonical) { + return errors.WithStack(ErrInvalidRequest) + } + return nil +} + +func EncodeMigrationPartitionJSON(value any) ([]byte, error) { + raw, err := json.Marshal(value) + if err != nil { + return nil, errors.WithStack(err) + } + if len(raw) > MaxMigrationPartitionControlBytes { + return nil, errors.WithStack(ErrInvalidRequest) + } + return raw, nil +} + +func DecodeMigrationPartitionControl(raw []byte) (MigrationPartitionControl, error) { + var c MigrationPartitionControl + if err := DecodeMigrationPartitionJSON(raw, &c); err != nil { + return c, errors.WithStack(err) + } + expected, err := BuildMigrationPartitionControl(c.PriorPartitions, c.PriorOwnership, c.Manifest, + c.UnchangedDescriptor, c.Transition.CatalogVersion, c.Transition.ConfigurationIndex) + if err != nil { + return c, errors.WithStack(err) + } + want, err := EncodeMigrationPartitionJSON(expected) + if err != nil || !bytes.Equal(want, raw) { + return c, errors.WithStack(store.ErrMigrationOwnership) + } + return c, nil +} + +func EncodeMigrationPartitionControl(c MigrationPartitionControl) ([]byte, error) { + raw, err := EncodeMigrationPartitionJSON(c) + if err != nil { + return nil, errors.WithStack(err) + } + _, err = DecodeMigrationPartitionControl(raw) + return raw, err +} + +// BuildMigrationPartitionControl is shared by the committed-root producer and +// deterministic data apply. Store apply independently resolves complete prior +// coverage against its coherent durable frontier, including negative records. +func BuildMigrationPartitionControl(partitions []store.MigrationOwnershipPartition, legacy []store.MigrationOwnershipState, + manifest []byte, unchanged *distribution.RouteDescriptor, version, configuration uint64) (MigrationPartitionControl, error) { + c := MigrationPartitionControl{PriorPartitions: partitions, PriorOwnership: legacy, Manifest: manifest, UnchangedDescriptor: unchanged} + if !migrationPartitionControlEnvelopeValid(version, configuration, len(partitions)+len(legacy)) { + return c, errors.WithStack(ErrInvalidRequest) + } + before, refs, err := migrationPartitionPriorEvidence(partitions, legacy) + if err != nil { + return c, errors.WithStack(err) + } + parent, routes, hash, operation, err := migrationPartitionRootEvidence(manifest, unchanged, version) + if err != nil { + return c, errors.WithStack(err) + } + t := store.MigrationOwnershipPartitionTransition{Operation: operation, RootManifestHash: hash, + Generation: before[0].Generation, LocalGroupID: before[0].LocalGroupID, CatalogVersion: version, + ConfigurationIndex: configuration, Before: refs} + if operation == store.MigrationOwnershipPartitionRefresh && len(legacy) != 0 { + return c, errors.WithStack(store.ErrNotSupported) + } + t.After, err = migrationPartitionTiles(before, parent, routes, version, configuration) + if err != nil { + return c, errors.WithStack(err) + } + if err := stampMigrationPartitionIdentity(&t); err != nil { + return c, errors.WithStack(err) + } + c.Transition = t + return c, nil +} + +func migrationPartitionControlEnvelopeValid(version, configuration uint64, count int) bool { + return version != 0 && configuration != 0 && migrationPartitionCountValid(count) +} + +func migrationPartitionCountValid(count int) bool { + return count > 0 && count <= maxMigrationPartitionPriors +} + +func stampMigrationPartitionIdentity(t *store.MigrationOwnershipPartitionTransition) error { + identity, err := EncodeMigrationPartitionJSON(struct { + Root []byte + Before []store.MigrationOwnershipPartitionReference + Version uint64 + }{t.RootManifestHash, t.Before, t.CatalogVersion}) + if err != nil { + return errors.WithStack(err) + } + digest := sha256.Sum256(identity) + t.OperationID = "partition/" + hex.EncodeToString(digest[:]) + for i := range t.After { + if t.Operation == store.MigrationOwnershipPartitionSplit { + t.After[i].PartitionID, err = store.MigrationOwnershipPartitionIDFor(*t, i) + if err != nil { + return errors.WithStack(err) + } + } + } + return nil +} + +func migrationPartitionPriorEvidence(partitions []store.MigrationOwnershipPartition, legacy []store.MigrationOwnershipState) ([]store.MigrationOwnershipPartition, []store.MigrationOwnershipPartitionReference, error) { + before := slices.Clone(partitions) + refs := make([]store.MigrationOwnershipPartitionReference, 0, len(partitions)+len(legacy)) + for _, p := range partitions { + ref, err := store.MigrationOwnershipPartitionReferenceFor(p) + if err != nil { + return nil, nil, errors.WithStack(err) + } + refs = append(refs, ref) + } + for _, p := range legacy { + ref, err := store.MigrationOwnershipLegacyReferenceFor(p) + if err != nil { + return nil, nil, errors.WithStack(err) + } + refs = append(refs, ref) + before = append(before, MigrationPartitionFromLegacy(p)) + } + sort.Slice(refs, func(i, j int) bool { + if refs[i].LegacyJobID != refs[j].LegacyJobID { + return refs[i].LegacyJobID < refs[j].LegacyJobID + } + return bytes.Compare(refs[i].PartitionID, refs[j].PartitionID) < 0 + }) + for i := 1; i < len(refs); i++ { + if refs[i].LegacyJobID == refs[i-1].LegacyJobID && bytes.Equal(refs[i].PartitionID, refs[i-1].PartitionID) { + return nil, nil, errors.WithStack(store.ErrMigrationOwnership) + } + } + return before, refs, nil +} + +// MigrationPartitionFromLegacy preserves the complete operational lineage. +// The original record, not this projection, is used for the legacy CAS hash. +func MigrationPartitionFromLegacy(s store.MigrationOwnershipState) store.MigrationOwnershipPartition { + p := store.MigrationOwnershipPartition{Generation: s.Generation, LocalGroupID: s.LocalGroupID, OwnerGroupID: s.OwnerGroupID, + CatalogVersion: s.CatalogVersion, ConfigurationIndex: s.ConfigurationIndex, AppliedIndex: s.AppliedIndex, + RouteID: s.RouteID, ParentRouteID: s.ParentRouteID, RouteStart: s.RouteStart, RouteEnd: s.RouteEnd, + DescriptorHash: s.DescriptorHash, Role: s.Role, MinWriteTSExclusive: s.MinWriteTSExclusive, + WriteFloorKnown: s.WriteFloorKnown || s.MinWriteTSExclusive != 0, Staged: s.Staged, MigrationJobID: s.JobID, + SourceTrackerVersion: s.SourceTrackerVersion, SourceAppliedIndex: s.SourceAppliedIndex} + for _, r := range s.Receipts { + p.Origins = append(p.Origins, store.MigrationOwnershipOperationReference{JobID: s.JobID, Operation: r.Operation, + CatalogVersion: r.CatalogVersion, ConfigurationIndex: r.ConfigurationIndex, AppliedIndex: r.AppliedIndex, + DescriptorHash: r.DescriptorHash, Staged: r.Staged}) + } + return p +} + +func migrationPartitionRootEvidence(raw []byte, unchanged *distribution.RouteDescriptor, version uint64) (distribution.RouteDescriptor, []distribution.RouteDescriptor, []byte, store.MigrationOwnershipPartitionOperation, error) { + var zero distribution.RouteDescriptor + if len(raw) != 0 && unchanged == nil { + return migrationPartitionSplitEvidence(raw, version) + } + if len(raw) != 0 || unchanged == nil || unchanged.State != distribution.RouteStateActive { + return zero, nil, nil, 0, errors.WithStack(store.ErrMigrationOwnership) + } + descriptor, err := distribution.EncodeRouteDescriptorForCatalogWrite(*unchanged, true) + if err != nil { + return zero, nil, nil, 0, errors.WithStack(err) + } + identity, err := EncodeMigrationPartitionJSON(struct { + Domain string + Version uint64 + Descriptor []byte + }{"elastickv/ownership-partition/unchanged/v1", version, descriptor}) + if err != nil { + return zero, nil, nil, 0, err + } + digest := sha256.Sum256(identity) + return *unchanged, []distribution.RouteDescriptor{*unchanged}, digest[:], store.MigrationOwnershipPartitionRefresh, nil +} + +func migrationPartitionSplitEvidence(raw []byte, version uint64) (distribution.RouteDescriptor, []distribution.RouteDescriptor, []byte, store.MigrationOwnershipPartitionOperation, error) { + var zero distribution.RouteDescriptor + m, err := distribution.DecodeOwnershipPartitionManifest(raw) + if err != nil { + return zero, nil, nil, 0, errors.WithStack(err) + } + canonical, err := distribution.EncodeOwnershipPartitionManifest(m) + if err != nil || !bytes.Equal(canonical, raw) || m.Version > version { + return zero, nil, nil, 0, errors.WithStack(store.ErrMigrationOwnership) + } + hash, err := distribution.OwnershipPartitionManifestHash(m) + return m.Parent, m.Children, hash, store.MigrationOwnershipPartitionSplit, errors.WithStack(err) +} + +func migrationPartitionTiles(before []store.MigrationOwnershipPartition, parent distribution.RouteDescriptor, routes []distribution.RouteDescriptor, version, configuration uint64) ([]store.MigrationOwnershipPartition, error) { + parentHash, err := distribution.OwnershipPartitionDescriptorHash(parent) + if err != nil { + return nil, errors.WithStack(err) + } + negativeOnly := migrationPartitionNegativeOnlyForGeometry(before, parent, routes, parentHash) + boundaries, err := migrationPartitionBoundaries(before, parent, routes, version, configuration) + if err != nil { + return nil, errors.WithStack(err) + } + var after []store.MigrationOwnershipPartition + for i, start := range boundaries { + if len(parent.End) != 0 && bytes.Compare(start, parent.End) >= 0 { + break + } + var end []byte + if i+1 < len(boundaries) { + end = boundaries[i+1] + } + p, found, err := migrationPartitionIntervalTile(before, parent, parentHash, routes, start, end, version, configuration, negativeOnly) + if err != nil { + return nil, errors.WithStack(err) + } + if !found { + continue + } + after = append(after, p) + } + if !migrationPartitionCountValid(len(after)) { + return nil, errors.WithStack(store.ErrMigrationOwnership) + } + return after, nil +} + +func migrationPartitionIntervalTile(before []store.MigrationOwnershipPartition, parent distribution.RouteDescriptor, hash []byte, routes []distribution.RouteDescriptor, start, end []byte, version, configuration uint64, negativeOnly bool) (store.MigrationOwnershipPartition, bool, error) { + p, found, err := migrationPartitionCovering(before, start, end) + if err != nil || !found { + return p, found, err + } + if !migrationPartitionParentMatches(p, parent, hash, negativeOnly) { + return p, false, errors.WithStack(store.ErrMigrationOwnership) + } + p, err = projectMigrationPartitionTile(p, routes, start, end, version, configuration) + return p, true, err +} + +func migrationPartitionBoundaries(before []store.MigrationOwnershipPartition, parent distribution.RouteDescriptor, routes []distribution.RouteDescriptor, version, configuration uint64) ([][]byte, error) { + boundaries := [][]byte{parent.Start} + if len(parent.End) != 0 { + boundaries = append(boundaries, parent.End) + } + for _, p := range before { + if !migrationPartitionPriorEpochMatches(p, before[0], version, configuration) || !migrationPartitionRangeContained(p, parent.Start, parent.End) { + return nil, errors.WithStack(store.ErrMigrationOwnership) + } + boundaries = append(boundaries, p.RouteStart) + if len(p.RouteEnd) != 0 { + boundaries = append(boundaries, p.RouteEnd) + } + } + for _, route := range routes { + boundaries = append(boundaries, route.Start) + } + sort.Slice(boundaries, func(i, j int) bool { return bytes.Compare(boundaries[i], boundaries[j]) < 0 }) + boundaries = slices.CompactFunc(boundaries, bytes.Equal) + return boundaries, nil +} + +func migrationPartitionPriorEpochMatches(p, first store.MigrationOwnershipPartition, version, configuration uint64) bool { + return p.Generation == first.Generation && p.LocalGroupID == first.LocalGroupID && p.ConfigurationIndex <= configuration && p.CatalogVersion <= version +} + +func migrationPartitionRangeContained(p store.MigrationOwnershipPartition, start, end []byte) bool { + return bytes.Compare(p.RouteStart, start) >= 0 && (len(end) == 0 || (len(p.RouteEnd) != 0 && bytes.Compare(p.RouteEnd, end) <= 0)) +} + +func migrationPartitionParentMatches(p store.MigrationOwnershipPartition, parent distribution.RouteDescriptor, hash []byte, negativeOnly bool) bool { + strict := p.RouteID == parent.RouteID && p.OwnerGroupID == parent.GroupID && bytes.Equal(p.DescriptorHash, hash) && + p.MinWriteTSExclusive == parent.MinWriteTSExclusive && (p.Role != store.MigrationOwner || p.Staged == parent.StagedVisibilityActive) + return strict || (negativeOnly && migrationPartitionNegativeParentMatches(p, parent)) +} + +// MigrationPartitionNegativePriors validates complete native references before +// permitting geometry discovery or a non-authorizing background deferral. +func MigrationPartitionNegativePriors(partitions []store.MigrationOwnershipPartition, legacy []store.MigrationOwnershipState) bool { + if !migrationPartitionCountValid(len(partitions) + len(legacy)) { + return false + } + before, _, err := migrationPartitionPriorEvidence(partitions, legacy) + return err == nil && migrationPartitionNegativeBefore(before) +} + +func MigrationPartitionNegativePriorsMatchParent(partitions []store.MigrationOwnershipPartition, legacy []store.MigrationOwnershipState, parent distribution.RouteDescriptor) bool { + if !migrationPartitionCountValid(len(partitions) + len(legacy)) { + return false + } + before, _, err := migrationPartitionPriorEvidence(partitions, legacy) + return err == nil && migrationPartitionNegativeBeforeMatchesParent(before, parent) +} + +func migrationPartitionNegativeBefore(before []store.MigrationOwnershipPartition) bool { + if len(before) == 0 { + return false + } + for _, p := range before { + if !migrationPartitionNegativeRole(p) || p.Generation != before[0].Generation || p.LocalGroupID != before[0].LocalGroupID { + return false + } + } + return true +} + +func migrationPartitionNegativeRole(p store.MigrationOwnershipPartition) bool { + return p.Role == store.MigrationNonOwner && p.SourceTrackerVersion == 0 && p.SourceAppliedIndex == 0 && !p.Staged && p.WriteFloorKnown +} + +func migrationPartitionNegativeBeforeMatchesParent(before []store.MigrationOwnershipPartition, parent distribution.RouteDescriptor) bool { + if !migrationPartitionNegativeBefore(before) { + return false + } + for _, p := range before { + if !migrationPartitionNegativeParentMatches(p, parent) { + return false + } + } + return true +} + +func migrationPartitionNegativeParentMatches(p store.MigrationOwnershipPartition, parent distribution.RouteDescriptor) bool { + return migrationPartitionNegativeRole(p) && parent.GroupID != p.LocalGroupID && parent.MinWriteTSExclusive >= p.MinWriteTSExclusive && + p.RouteID == parent.RouteID && p.ParentRouteID == parent.ParentRouteID && bytes.Equal(p.RouteStart, parent.Start) && bytes.Equal(p.RouteEnd, parent.End) +} + +func migrationPartitionNegativeGeometryAllowed(before []store.MigrationOwnershipPartition, routes []distribution.RouteDescriptor, parentHash []byte) bool { + if len(routes) > 1 { + return true + } + // An unchanged refresh may retain a historical owner/floor, but cannot + // invent a new descriptor hash without a committed geometry manifest. + for _, p := range before { + if !bytes.Equal(p.DescriptorHash, parentHash) { + return false + } + } + return true +} + +func migrationPartitionNegativeOnlyForGeometry(before []store.MigrationOwnershipPartition, parent distribution.RouteDescriptor, routes []distribution.RouteDescriptor, parentHash []byte) bool { + return migrationPartitionNegativeBeforeMatchesParent(before, parent) && migrationPartitionNegativeGeometryAllowed(before, routes, parentHash) +} + +func projectMigrationPartitionTile(p store.MigrationOwnershipPartition, routes []distribution.RouteDescriptor, start, end []byte, version, configuration uint64) (store.MigrationOwnershipPartition, error) { + for _, route := range routes { + if bytes.Compare(start, route.Start) < 0 || (len(route.End) != 0 && (len(end) == 0 || bytes.Compare(end, route.End) > 0)) { + continue + } + p.RouteID, p.ParentRouteID = route.RouteID, route.ParentRouteID + p.RouteStart, p.RouteEnd = start, end + p.CatalogVersion, p.ConfigurationIndex, p.AppliedIndex, p.ReceiptID = version, configuration, 0, nil + hash, err := distribution.OwnershipPartitionDescriptorHash(route) + p.DescriptorHash = hash + return p, errors.WithStack(err) + } + return p, errors.WithStack(store.ErrMigrationOwnership) +} + +func migrationPartitionCovering(before []store.MigrationOwnershipPartition, start, end []byte) (store.MigrationOwnershipPartition, bool, error) { + var chosen store.MigrationOwnershipPartition + found := false + for _, p := range before { + if bytes.Compare(p.RouteStart, start) > 0 || (len(p.RouteEnd) != 0 && (len(end) == 0 || bytes.Compare(p.RouteEnd, end) < 0)) { + continue + } + if found && p.CatalogVersion == chosen.CatalogVersion { + return chosen, false, errors.WithStack(store.ErrMigrationOwnership) + } + if !found || p.CatalogVersion > chosen.CatalogVersion { + chosen, found = p, true + } + } + return chosen, found, nil +} + +func ValidateMigrationPartitionRequest(req *pb.CleanupMigrationRequest) error { + if req.GetMode() != pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION || + !req.GetRequireConfigurationIndex() || req.GetExpectedConfigurationIndex() == 0 || req.GetOwnership() != nil { + return errors.WithStack(ErrInvalidRequest) + } + c, err := DecodeMigrationPartitionControl(req.GetPartitionTransition()) + if err != nil { + return errors.WithStack(err) + } + if c.Transition.ConfigurationIndex != req.GetExpectedConfigurationIndex() { + return errors.WithStack(ErrMigrationConfigurationChanged) + } + return nil +} + +func (f *kvFSM) applyMigrationPartitionControl(ctx context.Context, req *pb.CleanupMigrationRequest) any { + if err := ValidateMigrationPartitionRequest(req); err != nil { + return errors.WithStack(err) + } + c, err := DecodeMigrationPartitionControl(req.GetPartitionTransition()) + if err != nil { + return errors.WithStack(err) + } + if f.shardGroupID == 0 || c.Transition.LocalGroupID != f.shardGroupID { + return errors.WithStack(ErrInvalidRequest) + } + if err := f.verifyMigrationPartitionApplyConfiguration(ctx, c.Transition.ConfigurationIndex); err != nil { + return err + } + writer, ok := f.store.(store.MigrationOwnershipPartitionStore) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + err = writer.ApplyMigrationOwnershipPartition(ctx, c.Transition, f.pendingApplyIdx) + if err == nil { + return store.CleanupVersionsResult{Done: true} + } + if migrationOwnershipOrdinaryError(err) { + return errors.WithStack(err) + } + return haltErr(errors.Wrap(errors.Mark(err, ErrMigrationCleanupApply), "kv/fsm: partition control")) +} + +func (f *kvFSM) verifyMigrationPartitionApplyConfiguration(ctx context.Context, expected uint64) any { + durable, err := MigrationConfigurationIndex(ctx, f.store) + if err != nil { + return haltErr(errors.Mark(err, ErrMigrationCleanupApply)) + } + if durable == 0 || durable != expected || durable != f.AppliedConfigurationIndex() { + return errors.WithStack(ErrMigrationConfigurationChanged) + } + return nil +} diff --git a/kv/fsm_migration_partition_negative_test.go b/kv/fsm_migration_partition_negative_test.go new file mode 100644 index 000000000..12c49a7ff --- /dev/null +++ b/kv/fsm_migration_partition_negative_test.go @@ -0,0 +1,117 @@ +package kv + +import ( + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func changedNegativeManifest(t *testing.T, c MigrationPartitionControl) distribution.OwnershipPartitionManifest { + t.Helper() + m, err := distribution.DecodeOwnershipPartitionManifest(c.Manifest) + require.NoError(t, err) + m.Parent.GroupID, m.Parent.MinWriteTSExclusive = 3, 200 + for i := range m.Children { + m.Children[i].GroupID, m.Children[i].MinWriteTSExclusive = 3, 200 + } + return m +} + +func TestPartitionNegativeGeometryPreservesHistoricalDenialAndRefresh(t *testing.T) { + st := store.NewMVCCStore() + f, old := partitionControlFixture(t, st, 2) + m := changedNegativeManifest(t, old) + raw, err := distribution.EncodeOwnershipPartitionManifest(m) + require.NoError(t, err) + c, err := BuildMigrationPartitionControl(nil, old.PriorOwnership, raw, nil, 40, 9) + require.NoError(t, err) + f.SetApplyIndex(20) + require.IsType(t, store.CleanupVersionsResult{}, f.Apply(partitionControlCommand(t, c))) + reader, ok := st.(store.MigrationOwnershipPartitionStore) + require.True(t, ok) + before, err := reader.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + for _, p := range before.Partitions { + require.Equal(t, store.MigrationNonOwner, p.Role) + require.EqualValues(t, 1, p.OwnerGroupID, "root owner 3 is not an ownership acquisition") + require.EqualValues(t, 100, p.MinWriteTSExclusive, "never replace the native known floor with a guessed one") + require.EqualValues(t, 11, p.MigrationJobID) + require.True(t, store.MigrationOwnershipPartitionEvidenceValid(before, p)) + } + refresh, err := BuildMigrationPartitionControl(before.Partitions[:1], nil, nil, &m.Children[0], 41, 9) + require.NoError(t, err, "already projected negative hash can refresh without gaining the root's owner/floor") + f.SetApplyIndex(21) + require.IsType(t, store.CleanupVersionsResult{}, f.Apply(partitionControlCommand(t, refresh))) + after, err := reader.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + require.Equal(t, before.Partitions[0].PartitionID, after.Partitions[0].PartitionID) + require.Equal(t, before.Partitions[0].Origins, after.Partitions[0].Origins) + require.Equal(t, before.Partitions[0].OwnerGroupID, after.Partitions[0].OwnerGroupID) + require.Equal(t, before.Partitions[0].MinWriteTSExclusive, after.Partitions[0].MinWriteTSExclusive) +} + +func TestPartitionNegativeGeometryRejectsAuthorityAndGeometryChanges(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*distribution.OwnershipPartitionManifest, []store.MigrationOwnershipState) + }{ + {"root owns local", func(m *distribution.OwnershipPartitionManifest, _ []store.MigrationOwnershipState) { + m.Parent.GroupID = 2 + for i := range m.Children { + m.Children[i].GroupID = 2 + } + }}, + {"floor lowered", func(m *distribution.OwnershipPartitionManifest, _ []store.MigrationOwnershipState) { + m.Parent.MinWriteTSExclusive = 99 + for i := range m.Children { + m.Children[i].MinWriteTSExclusive = 99 + } + }}, + {"parent ID changed", func(m *distribution.OwnershipPartitionManifest, _ []store.MigrationOwnershipState) { + m.Parent.ParentRouteID++ + }}, + {"bounds changed", func(m *distribution.OwnershipPartitionManifest, _ []store.MigrationOwnershipState) { + m.Parent.Start = []byte("b") + m.Children[0].Start = []byte("b") + }}, + {"unknown floor", func(_ *distribution.OwnershipPartitionManifest, prior []store.MigrationOwnershipState) { + prior[0].MinWriteTSExclusive = 0 + prior[0].WriteFloorKnown = false + }}, + {"SOURCE record", func(_ *distribution.OwnershipPartitionManifest, prior []store.MigrationOwnershipState) { + prior[0].SourceTrackerVersion = 1 + prior[0].SourceAppliedIndex = 10 + }}, + } { + t.Run(tc.name, func(t *testing.T) { + _, old := partitionControlFixture(t, store.NewMVCCStore(), 2) + m := changedNegativeManifest(t, old) + tc.mutate(&m, old.PriorOwnership) + raw, err := distribution.EncodeOwnershipPartitionManifest(m) + require.NoError(t, err) + _, err = BuildMigrationPartitionControl(nil, old.PriorOwnership, raw, nil, 40, 9) + require.Error(t, err) + }) + } +} + +func TestPartitionNegativeGeometryRejectsAnyOwnerInCompletePrior(t *testing.T) { + _, old := partitionControlFixture(t, store.NewMVCCStore(), 2) + negative := old.PriorOwnership[0] + negative.CatalogVersion = 2 + positive := old.PriorOwnership[0] + positive.JobID, positive.OwnerGroupID, positive.Role = 12, 2, store.MigrationOwner + require.True(t, store.MigrationOwnershipEvidenceValid(positive)) + prior := []store.MigrationOwnershipState{positive, negative} + m := changedNegativeManifest(t, old) + raw, err := distribution.EncodeOwnershipPartitionManifest(m) + require.NoError(t, err) + require.False(t, MigrationPartitionNegativePriors(nil, prior)) + _, err = BuildMigrationPartitionControl(nil, prior, raw, nil, 40, 9) + require.Error(t, err, "an older OWNER cannot be omitted merely because the newest covering record is NONOWNER") + _, owner := partitionControlFixture(t, store.NewMVCCStore(), 1) + _, err = BuildMigrationPartitionControl(nil, owner.PriorOwnership, raw, nil, 40, 9) + require.Error(t, err, "positive proofs retain exact descriptor/owner/floor matching") +} diff --git a/kv/fsm_migration_partition_test.go b/kv/fsm_migration_partition_test.go new file mode 100644 index 000000000..520fffc17 --- /dev/null +++ b/kv/fsm_migration_partition_test.go @@ -0,0 +1,219 @@ +package kv + +import ( + "bytes" + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestPartitionControlStrictCodec(t *testing.T) { + for _, raw := range [][]byte{nil, []byte(`{}`), []byte(`{"Transition":{},"unknown":1}`), []byte(`{} {}`), []byte(`{"Transition":{},"Transition":{}}`)} { + _, err := DecodeMigrationPartitionControl(raw) + require.Error(t, err) + } +} + +func partitionControlFixture(t *testing.T, st store.MVCCStore, local uint64) (*kvFSM, MigrationPartitionControl) { + t.Helper() + f := &kvFSM{store: st, shardGroupID: local, routes: forbiddenApplyWatcher{}} + require.NoError(t, f.RecordAppliedConfiguration(9)) + parent := distribution.RouteDescriptor{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 1, State: distribution.RouteStateActive, MinWriteTSExclusive: 100} + hash, err := distribution.OwnershipPartitionDescriptorHash(parent) + require.NoError(t, err) + role := store.MigrationOwner + if local != 1 { + role = store.MigrationNonOwner + } + state := store.MigrationOwnershipState{JobID: 11, Generation: "0123456789abcdef0123456789abcdef", LocalGroupID: local, OwnerGroupID: 1, + CatalogVersion: 1, ConfigurationIndex: 9, RouteID: 1, RouteStart: parent.Start, RouteEnd: parent.End, + Role: role, MinWriteTSExclusive: 100, WriteFloorKnown: true, DescriptorHash: hash} + writer, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + require.NoError(t, writer.ClearMigrationStateWithOwnership(t.Context(), state, 10)) + view, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + left, right := parent, parent + left.RouteID, left.ParentRouteID, left.End, left.SplitAtHLC = 2, 1, []byte("m"), 20 + right.RouteID, right.ParentRouteID, right.Start, right.SplitAtHLC = 3, 1, []byte("m"), 20 + manifest, err := distribution.EncodeOwnershipPartitionManifest(distribution.OwnershipPartitionManifest{PreviousVersion: 1, Version: 2, Parent: parent, Children: []distribution.RouteDescriptor{left, right}}) + require.NoError(t, err) + c, err := BuildMigrationPartitionControl(nil, view.Ownership, manifest, nil, 40, 9) + require.NoError(t, err) + return f, c +} + +func partitionControlCommand(t *testing.T, c MigrationPartitionControl) []byte { + t.Helper() + raw, err := EncodeMigrationPartitionControl(c) + require.NoError(t, err) + cmd, err := MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{JobId: 11, Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: c.Transition.ConfigurationIndex, PartitionTransition: raw}) + require.NoError(t, err) + return cmd +} + +func TestPartitionControlApplyAndExactReplay(t *testing.T) { + for _, local := range []uint64{1, 2} { + t.Run(map[uint64]string{1: "owner", 2: "nonowner"}[local], func(t *testing.T) { + st := store.NewMVCCStore() + f, c := partitionControlFixture(t, st, local) + cmd := partitionControlCommand(t, c) + f.SetApplyIndex(20) + result, ok := f.Apply(cmd).(store.CleanupVersionsResult) + require.True(t, ok) + require.True(t, result.Done) + reader, ok := st.(store.MigrationOwnershipPartitionStore) + require.True(t, ok) + before, err := reader.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + require.Len(t, before.Partitions, 2) + require.Len(t, before.Receipts, 1) + for _, p := range before.Partitions { + require.Equal(t, c.Transition.After[0].Role, p.Role) + require.EqualValues(t, 100, p.MinWriteTSExclusive) + require.True(t, store.MigrationOwnershipPartitionEvidenceValid(before, p)) + require.EqualValues(t, 40, p.CatalogVersion) + require.EqualValues(t, 20, p.AppliedIndex) + } + f.SetApplyIndex(21) + _, ok = f.Apply(cmd).(store.CleanupVersionsResult) + require.True(t, ok) + after, err := reader.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + require.Equal(t, before, after) + }) + } +} + +func TestPartitionControlRefreshStableIdentityAndNegativeRole(t *testing.T) { + st := store.NewMVCCStore() + f, c := partitionControlFixture(t, st, 2) + f.SetApplyIndex(20) + require.IsType(t, store.CleanupVersionsResult{}, f.Apply(partitionControlCommand(t, c))) + reader, ok := st.(store.MigrationOwnershipPartitionStore) + require.True(t, ok) + before, err := reader.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + m, err := distribution.DecodeOwnershipPartitionManifest(c.Manifest) + require.NoError(t, err) + refresh, err := BuildMigrationPartitionControl(before.Partitions[:1], nil, nil, &m.Children[0], 41, 9) + require.NoError(t, err) + f.SetApplyIndex(21) + require.IsType(t, store.CleanupVersionsResult{}, f.Apply(partitionControlCommand(t, refresh))) + after, err := reader.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + require.Equal(t, before.Partitions[0].PartitionID, after.Partitions[0].PartitionID) + require.Equal(t, store.MigrationNonOwner, after.Partitions[0].Role) + require.Equal(t, before.Partitions[0].Origins, after.Partitions[0].Origins) + require.EqualValues(t, 41, after.Partitions[0].CatalogVersion) + refresh.Transition.After[0].Role = store.MigrationOwner + _, err = EncodeMigrationPartitionControl(refresh) + require.Error(t, err) +} + +func TestPartitionControlRejectsChangedPriorAndConfiguration(t *testing.T) { + st := store.NewMVCCStore() + f, c := partitionControlFixture(t, st, 1) + // A complete valid-looking alternate legacy CAS must still match durable state. + legacy := append([]store.MigrationOwnershipState(nil), c.PriorOwnership...) + legacy[0].AppliedIndex++ + conflict, err := BuildMigrationPartitionControl(nil, legacy, c.Manifest, nil, 40, 9) + require.NoError(t, err) + f.SetApplyIndex(20) + response := f.Apply(partitionControlCommand(t, conflict)) + require.NoError(t, haltApplyOf(response)) + applyErr, ok := response.(error) + require.True(t, ok) + require.ErrorIs(t, applyErr, store.ErrMigrationOwnership) + require.NoError(t, f.RecordAppliedConfiguration(21)) + f.SetApplyIndex(22) + response = f.Apply(partitionControlCommand(t, c)) + require.NoError(t, haltApplyOf(response)) + applyErr, ok = response.(error) + require.True(t, ok) + require.ErrorIs(t, applyErr, ErrMigrationConfigurationChanged) +} + +type partitionFailureStore struct { + store.MVCCStore + err error +} + +func (s *partitionFailureStore) MigrationOwnershipPartitions(context.Context) (store.MigrationOwnershipPartitionState, error) { + return store.MigrationOwnershipPartitionState{}, nil +} +func (s *partitionFailureStore) ApplyMigrationOwnershipPartition(context.Context, store.MigrationOwnershipPartitionTransition, uint64) error { + return s.err +} + +func TestPartitionControlIOHaltsButSemanticRefusalDoesNot(t *testing.T) { + base := store.NewMVCCStore() + _, c := partitionControlFixture(t, base, 1) + for _, failure := range []error{errors.New("partition batch IO failure"), store.ErrMigrationOwnership, store.ErrNotSupported} { + f := &kvFSM{store: &partitionFailureStore{MVCCStore: base, err: failure}, shardGroupID: 1} + require.NoError(t, f.reloadMigrationConfiguration(t.Context())) + f.SetApplyIndex(20) + response := f.Apply(partitionControlCommand(t, c)) + if errors.Is(failure, store.ErrMigrationOwnership) || errors.Is(failure, store.ErrNotSupported) { + require.NoError(t, haltApplyOf(response)) + } else { + require.Error(t, haltApplyOf(response)) + } + } +} + +func TestPartitionControlCanonicalAndBoundedPayload(t *testing.T) { + _, c := partitionControlFixture(t, store.NewMVCCStore(), 1) + raw, err := EncodeMigrationPartitionControl(c) + require.NoError(t, err) + _, err = DecodeMigrationPartitionControl(raw) + require.NoError(t, err) + _, err = DecodeMigrationPartitionControl(append([]byte(" "), raw...)) + require.Error(t, err) + _, err = DecodeMigrationPartitionControl(bytes.Replace(raw, []byte(`"Transition":`), []byte(`"transition":`), 1)) + require.Error(t, err) + _, err = DecodeMigrationPartitionControl(bytes.Repeat([]byte("x"), MaxMigrationPartitionControlBytes+1)) + require.Error(t, err) + c.Transition.Before[0].AttestationHash[0] ^= 1 + _, err = EncodeMigrationPartitionControl(c) + require.Error(t, err) +} + +func TestPartitionControlRequiresBoundConfiguration(t *testing.T) { + err := ValidateMigrationPartitionRequest(&pb.CleanupMigrationRequest{Mode: pb.MigrationCleanupMode(7)}) + require.Error(t, err) +} + +func TestPartitionControlPebbleApplyPersistsReceiptAndIndex(t *testing.T) { + dir := t.TempDir() + st, err := store.NewPebbleStore(dir) + require.NoError(t, err) + f, c := partitionControlFixture(t, st, 1) + f.SetApplyIndex(20) + require.IsType(t, store.CleanupVersionsResult{}, f.Apply(partitionControlCommand(t, c)), "typed control must dispatch before the raw batch scope") + reader, ok := st.(store.MigrationOwnershipPartitionStore) + require.True(t, ok) + before, err := reader.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + require.NoError(t, st.Close()) + reopened, err := store.NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + reader, ok = reopened.(store.MigrationOwnershipPartitionStore) + require.True(t, ok) + after, err := reader.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + require.Equal(t, before, after) + applied, ok := reopened.(interface{ LastAppliedIndex() (uint64, bool, error) }) + require.True(t, ok) + index, present, err := applied.LastAppliedIndex() + require.NoError(t, err) + require.True(t, present) + require.EqualValues(t, 20, index) +} diff --git a/kv/fsm_migration_promote.go b/kv/fsm_migration_promote.go index 6f942799f..423addf8f 100644 --- a/kv/fsm_migration_promote.go +++ b/kv/fsm_migration_promote.go @@ -126,7 +126,7 @@ func clampMigrationPromoteBound(requested, fallback, ceiling uint64) uint64 { } func isMigrationPromoteOrdinaryApplyError(err error) bool { - return errors.Is(err, store.ErrInvalidExportCursor) + return errors.Is(err, store.ErrMigrationRetired) || errors.Is(err, store.ErrInvalidExportCursor) || errors.Is(err, store.ErrInvalidImportVersion) } func migrationPromoteTargetKey(jobID uint64) func([]byte) ([]byte, bool) { diff --git a/kv/fsm_migration_readiness.go b/kv/fsm_migration_readiness.go new file mode 100644 index 000000000..c8223e29a --- /dev/null +++ b/kv/fsm_migration_readiness.go @@ -0,0 +1,230 @@ +package kv + +import ( + "bytes" + "context" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +// MarshalTargetStagedReadinessCommand encodes a target-group readiness guard +// as a Raft FSM command. The target Internal RPC handler uses this instead of +// mutating its local store directly so an acknowledged guard has been applied +// by the target group's voters. +func MarshalTargetStagedReadinessCommand(req *pb.TargetStagedReadinessRequest) ([]byte, error) { + if req == nil { + return nil, errors.WithStack(ErrInvalidRequest) + } + b, err := proto.Marshal(req) + if err != nil { + return nil, errors.WithStack(err) + } + if len(b) >= maxMarshaledCommandSize { + return nil, errors.New("marshaled target readiness request too large") + } + return prependByte(raftEncodeTargetReadiness, b), nil +} + +func (f *kvFSM) applyTargetStagedReadiness(ctx context.Context, data []byte) any { + req := &pb.TargetStagedReadinessRequest{} + if err := proto.Unmarshal(data, req); err != nil { + return errors.WithStack(err) + } + if req.GetTransferSizeGuardVersion() != 0 && + (!req.GetRequireConfigurationIndex() || req.GetExpectedConfigurationIndex() == 0 || + req.GetExpectedConfigurationIndex() != f.AppliedConfigurationIndex()) { + return errors.WithStack(ErrMigrationConfigurationChanged) + } + return f.applyTargetStagedReadinessRequest(ctx, req) +} + +func (f *kvFSM) applyTargetStagedReadinessRequest(ctx context.Context, req *pb.TargetStagedReadinessRequest) any { + writer, ok := f.store.(store.MigrationTargetReadinessWriter) + if !ok { + return haltErr(errors.Wrap(errors.Mark(store.ErrNotSupported, ErrTargetReadinessApply), "kv/fsm: target staged readiness store")) + } + state := targetStagedReadinessStateFromProto(req) + state, err := f.preserveMigrationTrackerMinimum(ctx, state) + if err != nil { + if errors.Is(err, store.ErrMigrationOwnership) { + return err + } + return haltErr(errors.Wrap(errors.Mark(err, ErrTargetReadinessApply), "kv/fsm: inspect source migration control")) + } + if err := applyTargetStagedReadinessAt(ctx, writer, state, f.pendingApplyIdx); err != nil { + // Validation is a pure function of the entry, so every replica rejects + // the same malformed request. Halting on it would let one bad internal + // RPC stop the group instead of returning InvalidArgument to its caller. + if errors.Is(err, store.ErrInvalidReadinessState) || errors.Is(err, store.ErrMigrationRetired) { + return errors.Wrap(err, "kv/fsm: target staged readiness") + } + // Anything else is a local Pebble write. A replica that returns an + // ordinary error here advances past the entry without installing the + // source fence, write tracker or target guard that healthy replicas + // installed, and then accepts user writes they reject -- divergence the + // runner's voter probe cannot repair after the fact. + return haltErr(errors.Wrap(errors.Mark(err, ErrTargetReadinessApply), "kv/fsm: persist target staged readiness")) + } + if state.Armed && f.hlc != nil { + f.hlc.Observe(max(state.MinWriteTSExclusive, f.store.LastCommitTS())) + } + return nil +} + +func (f *kvFSM) preserveMigrationTrackerMinimum(ctx context.Context, state store.TargetStagedReadinessState) (store.TargetStagedReadinessState, error) { + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { + return state, nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return state, errors.WithStack(err) + } + for _, current := range states { + if current.JobID == state.JobID { + return preserveSourceMigrationControl(state, current) + } + } + return state, nil +} + +func preserveSourceMigrationControl(state, current store.TargetStagedReadinessState) (store.TargetStagedReadinessState, error) { + if armedSourceMigrationControl(current) { + if !sameSourceMigrationControlRange(state, current) { + return state, errors.WithStack(store.ErrMigrationOwnership) + } + state = mergeSourceMigrationControl(state, current) + } + if state.TrackWrites && (state.MinAdmittedTS == 0 || current.MinAdmittedTS < state.MinAdmittedTS) { + state.MinAdmittedTS = current.MinAdmittedTS + } + return state, nil +} + +func armedSourceMigrationControl(state store.TargetStagedReadinessState) bool { + return state.Armed && (state.TrackWrites || state.SourceWriteFence || state.SourceReadFence) +} + +func mergeSourceMigrationControl(state, current store.TargetStagedReadinessState) store.TargetStagedReadinessState { + state.SourceWriteFence = state.SourceWriteFence || current.SourceWriteFence + state.SourceReadFence = state.SourceReadFence || current.SourceReadFence + state.TrackWrites = state.TrackWrites || current.TrackWrites + state.TransferSizeGuardVersion = max(state.TransferSizeGuardVersion, current.TransferSizeGuardVersion) + state.MinWriteTSExclusive = max(state.MinWriteTSExclusive, current.MinWriteTSExclusive) + if state.RetentionPinTS == 0 || current.RetentionPinTS < state.RetentionPinTS { + state.RetentionPinTS = current.RetentionPinTS + } + return state +} + +func sameSourceMigrationControlRange(state, current store.TargetStagedReadinessState) bool { + return state.Armed && state.MigrationJobID == current.MigrationJobID && + bytes.Equal(state.RouteStart, current.RouteStart) && bytes.Equal(state.RouteEnd, current.RouteEnd) +} + +func (f *kvFSM) recordMigrationWrite(ctx context.Context, muts []*pb.Mutation, commitTS uint64) error { + if commitTS == 0 { + return nil + } + reader, writer, ok := f.migrationReadinessStore() + if !ok { + return nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(errors.Mark(err, errMigrationTrackerPersistence)) + } + // Tracker updates share the Raft entry with the following user mutation. + // Leave metaAppliedIndex to the data batch so replay cannot skip a write if + // the process stops after only the tracker record is persisted. + if err := recordMigrationWriteInStates(ctx, writer, states, muts, commitTS, 0); err != nil { + return errors.WithStack(errors.Mark(err, errMigrationTrackerPersistence)) + } + return nil +} + +func (f *kvFSM) migrationReadinessStore() (store.MigrationTargetReadinessReader, store.MigrationTargetReadinessWriter, bool) { + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { + return nil, nil, false + } + writer, ok := f.store.(store.MigrationTargetReadinessWriter) + if !ok { + return nil, nil, false + } + return reader, writer, true +} + +func recordMigrationWriteInStates(ctx context.Context, writer store.MigrationTargetReadinessWriter, states []store.TargetStagedReadinessState, muts []*pb.Mutation, commitTS uint64, appliedIndex uint64) error { + for _, state := range states { + if !state.Armed || !state.TrackWrites || !migrationMutationsIntersect(muts, state.RouteStart, state.RouteEnd) { + continue + } + if state.MinAdmittedTS != 0 && state.MinAdmittedTS <= commitTS { + continue + } + state.MinAdmittedTS = commitTS + if err := applyTargetStagedReadinessAt(ctx, writer, state, appliedIndex); err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func applyTargetStagedReadinessAt(ctx context.Context, writer store.MigrationTargetReadinessWriter, state store.TargetStagedReadinessState, appliedIndex uint64) error { + if raftWriter, ok := writer.(store.MigrationTargetReadinessRaftWriter); ok { + if err := raftWriter.ApplyTargetStagedReadinessAt(ctx, state, appliedIndex); err != nil { + return errors.WithStack(err) + } + return nil + } + if err := writer.ApplyTargetStagedReadiness(ctx, state); err != nil { + return errors.WithStack(err) + } + return nil +} + +func migrationMutationsIntersect(muts []*pb.Mutation, routeStart []byte, routeEnd []byte) bool { + for _, mut := range muts { + if mut == nil || isTxnInternalKey(mut.Key) { + continue + } + var start, end []byte + if mut.GetOp() == pb.Op_DEL_PREFIX { + start, end = routePrefixRange(mut.Key) + } else { + if len(mut.Key) == 0 { + continue + } + start, end = readinessRouteRangeForScan(mut.Key, nextScanCursor(mut.Key)) + } + if routeRangeIntersects(start, end, routeStart, routeEnd) { + return true + } + } + return false +} + +func targetStagedReadinessStateFromProto(req *pb.TargetStagedReadinessRequest) store.TargetStagedReadinessState { + if req == nil { + return store.TargetStagedReadinessState{} + } + return store.TargetStagedReadinessState{ + JobID: req.GetJobId(), + RouteStart: bytes.Clone(req.GetRouteStart()), + RouteEnd: bytes.Clone(req.GetRouteEnd()), + ExpectedCutoverVersion: req.GetExpectedCutoverVersion(), + MigrationJobID: req.GetMigrationJobId(), + MinWriteTSExclusive: req.GetMinWriteTsExclusive(), + Armed: req.GetArmed(), + SourceWriteFence: req.GetSourceWriteFence(), + SourceReadFence: req.GetSourceReadFence(), + RetentionPinTS: req.GetRetentionPinTs(), + TrackWrites: req.GetTrackWrites(), + MinAdmittedTS: req.GetMinAdmittedTs(), + TransferSizeGuardVersion: req.GetTransferSizeGuardVersion(), + } +} diff --git a/kv/fsm_migration_readiness_test.go b/kv/fsm_migration_readiness_test.go new file mode 100644 index 000000000..dbb358d46 --- /dev/null +++ b/kv/fsm_migration_readiness_test.go @@ -0,0 +1,156 @@ +package kv + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestApplyTargetStagedReadinessCommandPersistsGuard(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + fsm := &kvFSM{store: st} + + cmd, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + MinWriteTsExclusive: 100, + Armed: true, + }) + require.NoError(t, err) + require.Nil(t, fsm.Apply(cmd)) + + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []store.TargetStagedReadinessState{{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }}, states) +} + +func TestApplyTargetStagedReadinessCommandPersistsAppliedIndex(t *testing.T) { + t.Parallel() + + st, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + fsm := &kvFSM{store: st, pendingApplyIdx: 77} + + cmd, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + MinWriteTsExclusive: 100, + Armed: true, + }) + require.NoError(t, err) + require.Nil(t, fsm.Apply(cmd)) + + appliedReader, ok := st.(interface { + LastAppliedIndex() (uint64, bool, error) + }) + require.True(t, ok) + idx, present, err := appliedReader.LastAppliedIndex() + require.NoError(t, err) + require.True(t, present) + require.Equal(t, uint64(77), idx) +} + +func TestRecordMigrationWriteDoesNotAdvanceAppliedIndexBeforeDataWrite(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + writer, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, store.TargetStagedReadinessState{ + JobID: 11, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 11, + MinWriteTSExclusive: 1, + Armed: true, + TrackWrites: true, + RetentionPinTS: 1, + MinAdmittedTS: 90, + })) + fsm := &kvFSM{store: st, pendingApplyIdx: 77} + + err = fsm.recordMigrationWrite(ctx, []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("v")}}, 50) + require.NoError(t, err) + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Len(t, states, 1) + require.Equal(t, uint64(50), states[0].MinAdmittedTS) + + appliedReader, ok := st.(interface { + LastAppliedIndex() (uint64, bool, error) + }) + require.True(t, ok) + idx, present, err := appliedReader.LastAppliedIndex() + require.NoError(t, err) + require.False(t, present) + require.Zero(t, idx) + + err = st.ApplyMutationsRaftAt(ctx, []*store.KVPairMutation{{ + Op: store.OpTypePut, + Key: []byte("n"), + Value: []byte("v"), + }}, nil, 50, 50, 77) + require.NoError(t, err) + idx, present, err = appliedReader.LastAppliedIndex() + require.NoError(t, err) + require.True(t, present) + require.Equal(t, uint64(77), idx) +} + +func TestRecordMigrationWriteTracksEmptyDelPrefix(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + writer, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + + err := recordMigrationWriteInStates(ctx, writer, []store.TargetStagedReadinessState{{ + JobID: 11, + RouteStart: []byte("m"), + RouteEnd: []byte("n"), + ExpectedCutoverVersion: 2, + MigrationJobID: 11, + MinWriteTSExclusive: 100, + Armed: true, + TrackWrites: true, + RetentionPinTS: 40, + MinAdmittedTS: 90, + }}, []*pb.Mutation{{Op: pb.Op_DEL_PREFIX}}, 50, 0) + require.NoError(t, err) + + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Len(t, states, 1) + require.Equal(t, uint64(50), states[0].MinAdmittedTS) +} diff --git a/kv/fsm_migration_retirement_test.go b/kv/fsm_migration_retirement_test.go new file mode 100644 index 000000000..93b689f1c --- /dev/null +++ b/kv/fsm_migration_retirement_test.go @@ -0,0 +1,68 @@ +package kv + +import ( + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestRetiredMigrationMutationsRejectWithoutRaftHalt(t *testing.T) { + t.Parallel() + st := store.NewMVCCStore() + require.NoError(t, st.PutAt(t.Context(), []byte("customer"), []byte("active"), 100, 0)) + fsm := newPrepareSafetyFSM(t, st) + cleanup, err := MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{JobId: 7, Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA}) + require.NoError(t, err) + require.Nil(t, fsm.Apply(cleanup)) + readiness, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{JobId: 7, MigrationJobId: 7, + Armed: true, SourceWriteFence: true, MinWriteTsExclusive: 1000, RetentionPinTs: 1}) + require.NoError(t, err) + imported, err := MarshalMigrationImportCommand(&pb.ImportRangeVersionsRequest{JobId: 7, BracketId: 1, BatchSeq: 1}) + require.NoError(t, err) + promoted, err := MarshalMigrationPromoteCommand(&pb.PromoteStagedVersionsRequest{JobId: 7}) + require.NoError(t, err) + deleted, err := MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{JobId: 7, Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS}) + require.NoError(t, err) + for _, command := range [][]byte{readiness, imported, promoted, deleted} { + response := fsm.Apply(command) + require.Nil(t, haltApplyOf(response), "a retired job must not stop replication") + applyErr, ok := response.(error) + require.True(t, ok) + require.ErrorIs(t, applyErr, store.ErrMigrationRetired) + require.Equal(t, uint64(100), st.LastCommitTS()) + } + for _, tombstone := range []bool{false, true} { + command, marshalErr := MarshalMigrationImportCommand(&pb.ImportRangeVersionsRequest{JobId: 8, BracketId: 1, BatchSeq: 1, + Versions: []*pb.MVCCVersion{{Key: []byte("!migstage|retired"), CommitTs: 200, Tombstone: tombstone}}}) + require.NoError(t, marshalErr) + response := fsm.Apply(command) + require.Nil(t, haltApplyOf(response)) + applyErr, ok := response.(error) + require.True(t, ok) + require.ErrorIs(t, applyErr, ErrInvalidRequest) + } + retire, err := MarshalMigrationRetireCommand(7) + require.NoError(t, err) + require.Nil(t, fsm.Apply(retire), "legacy retirement remains idempotent") + reader, ok := st.(store.MigrationRetirementReader) + require.True(t, ok) + retired, err := reader.MigrationRetired(t.Context(), 7) + require.NoError(t, err) + require.True(t, retired) + require.NoError(t, applyFSMRequest(t, fsm, &pb.Request{Ts: 200, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("customer"), Value: []byte("still-active")}}})) + staged := distribution.MigrationStagedDataKey(8, []byte("!migstage|retired")) + require.NoError(t, st.PutAt(t.Context(), staged, []byte("forged-retirement"), 300, 0)) + command, err := MarshalMigrationPromoteCommand(&pb.PromoteStagedVersionsRequest{JobId: 8, MaxVersions: 10}) + require.NoError(t, err) + response := fsm.Apply(command) + require.Nil(t, haltApplyOf(response)) + applyErr, ok := response.(error) + require.True(t, ok) + require.ErrorIs(t, applyErr, store.ErrInvalidImportVersion) + retired, err = reader.MigrationRetired(t.Context(), 8) + require.NoError(t, err) + require.False(t, retired, "legacy staging must not forge a retirement witness") +} diff --git a/kv/fsm_migration_serving.go b/kv/fsm_migration_serving.go new file mode 100644 index 000000000..dadc5232f --- /dev/null +++ b/kv/fsm_migration_serving.go @@ -0,0 +1,312 @@ +package kv + +import ( + "bytes" + "context" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +// CheckMigrationRequest performs advisory preproposal admission against the +// group's local durable migration state. req must already have its final +// timestamps, metadata and footprint. The FSM repeats this check at apply; +// a successful check does not authorize a different or later request. +func CheckMigrationRequest(ctx context.Context, localStore store.MVCCStore, groupID uint64, req *pb.Request) error { + if localStore == nil || req == nil { + return errors.WithStack(ErrInvalidRequest) + } + if err := verifyRequestMigrationControlKeys(req); err != nil { + return err + } + f := &kvFSM{store: localStore, shardGroupID: groupID} + index, err := MigrationConfigurationIndex(ctx, localStore) + if err != nil { + return err + } + f.migrationConfigurationIndex.Store(index) + _, err = f.withMigrationOwnershipEvidence(ctx, req) + return err +} + +func (f *kvFSM) withMigrationOwnershipEvidence(ctx context.Context, req *pb.Request) (context.Context, error) { + view, err := readMigrationServingState(ctx, f.store) + if err != nil { + return ctx, errors.WithStack(errors.Mark(err, errTargetReadinessUnproven)) + } + ranges, err := migrationServingFootprint(req) + if err != nil { + return ctx, err + } + if !migrationApplyIntersects(view, ranges) { + return ctx, nil + } + if drain, err := f.matchesMigrationResolution(ctx, req); drain || err != nil { + if err != nil { + return ctx, err + } + return context.WithValue(ctx, migrationResolutionContextKey{}, req), nil + } + return f.withMigrationCreationEvidence(ctx, req, view, ranges) +} + +func (f *kvFSM) withMigrationCreationEvidence(ctx context.Context, req *pb.Request, view store.MigrationServingState, ranges []migrationApplyRange) (context.Context, error) { + if !migrationCreationGuardsProven(view, ranges, f.AppliedConfigurationIndex()) { + return ctx, errors.WithStack(ErrRouteCutoverPending) + } + version, pinned := DecodeObservedRouteVersion(req.GetObservedRouteVersion()) + if !pinned { + return ctx, errors.WithStack(ErrRouteCutoverPending) + } + proofs, err := migrationAdmissionOwnershipProofs(view, ranges, f.shardGroupID, version) + if err != nil { + return ctx, err + } + if len(proofs) == 0 { + return ctx, errors.WithStack(ErrRouteCutoverPending) + } + history, err := migrationAdmissionHistory(proofs, version) + if err != nil { + return ctx, errors.Wrap(ErrRouteCutoverPending, "incomplete migration apply history") + } + return withApplyRouteHistory(ctx, history), nil +} + +type migrationResolutionContextKey struct{} + +func migrationResolutionAuthorized(ctx context.Context, req *pb.Request) bool { + authorized, ok := ctx.Value(migrationResolutionContextKey{}).(*pb.Request) + return ok && authorized == req +} + +func migrationApplyIntersects(view store.MigrationServingState, ranges []migrationApplyRange) bool { + for _, r := range ranges { + for _, proof := range migrationAdmissionProofs(view) { + if routeRangeIntersects(r.start, r.end, proof.RouteStart, proof.RouteEnd) { + return true + } + } + for _, guard := range view.Readiness { + if guard.Armed && routeRangeIntersects(r.start, r.end, guard.RouteStart, guard.RouteEnd) { + return true + } + } + } + return false +} + +func migrationCreationGuardsProven(view store.MigrationServingState, ranges []migrationApplyRange, configuration uint64) bool { + for _, guard := range view.Readiness { + if guard.Armed && migrationGuardIntersects(guard, ranges) && !migrationAdmissionGuardProven(view, guard, ranges, configuration) { + return false + } + } + return true +} + +func migrationAdmissionGuardProven(view store.MigrationServingState, guard store.TargetStagedReadinessState, ranges []migrationApplyRange, configuration uint64) bool { + if guard.SourceReadFence || guard.SourceWriteFence { + return false + } + for _, r := range ranges { + if !routeRangeIntersects(r.start, r.end, guard.RouteStart, guard.RouteEnd) { + continue + } + start, end := prefixDeleteRouteIntersection(distribution.Route{Start: guard.RouteStart, End: guard.RouteEnd}, r.start, r.end) + proofs, err := migrationAdmissionRangeProofs(view, migrationApplyRange{start, end}) + if err != nil || len(proofs) == 0 { + return false + } + for _, proof := range proofs { + if !migrationAdmissionGuardMatches(proof, guard, configuration) { + return false + } + } + } + return true +} + +func migrationAdmissionGuardMatches(proof migrationAdmissionProof, guard store.TargetStagedReadinessState, configuration uint64) bool { + if proof.partition == nil { + return migrationGuardOwnershipProven(guard, []store.MigrationOwnershipState{proof.MigrationOwnershipState}, configuration) + } + if proof.Role != store.MigrationOwner || proof.JobID != guard.JobID || guard.MigrationJobID != guard.JobID { + return false + } + if guard.TrackWrites { + return migrationAdmissionSourceGuardMatches(proof, guard, configuration) + } + return migrationAdmissionTargetGuardMatches(proof, guard) +} + +func migrationAdmissionSourceGuardMatches(proof migrationAdmissionProof, guard store.TargetStagedReadinessState, configuration uint64) bool { + return configuration != 0 && proof.ConfigurationIndex == configuration && !proof.Staged && + guard.RetentionPinTS != 0 && proof.SourceTrackerVersion != 0 && proof.SourceTrackerVersion == guard.ExpectedCutoverVersion +} + +func migrationAdmissionTargetGuardMatches(proof migrationAdmissionProof, guard store.TargetStagedReadinessState) bool { + if !proof.Staged || proof.MinWriteTSExclusive != guard.MinWriteTSExclusive { + return false + } + for _, origin := range proof.partition.Origins { + if origin.JobID == guard.JobID && origin.Staged && origin.CatalogVersion >= guard.ExpectedCutoverVersion { + return true + } + } + return false +} + +func migrationGuardOwnershipProven(guard store.TargetStagedReadinessState, proofs []store.MigrationOwnershipState, configuration uint64) bool { + if !guard.TrackWrites { + return migrationTargetGuardProven(guard, proofs) + } + if guard.SourceWriteFence || guard.SourceReadFence || configuration == 0 { + return false + } + for _, proof := range proofs { + if migrationSourceOwnershipMatches(proof, guard, configuration) { + return true + } + } + return false +} + +func migrationSourceOwnershipMatches(proof store.MigrationOwnershipState, guard store.TargetStagedReadinessState, configuration uint64) bool { + return store.MigrationOwnershipEvidenceValid(proof) && proof.SourceTrackerVersion != 0 && + proof.SourceTrackerVersion == guard.ExpectedCutoverVersion && proof.ConfigurationIndex == configuration && + guard.MigrationJobID == guard.JobID && proof.JobID == guard.JobID && guard.RetentionPinTS != 0 && + bytes.Equal(proof.RouteStart, guard.RouteStart) && bytes.Equal(proof.RouteEnd, guard.RouteEnd) +} + +func migrationGuardIntersects(guard store.TargetStagedReadinessState, ranges []migrationApplyRange) bool { + for _, r := range ranges { + if routeRangeIntersects(r.start, r.end, guard.RouteStart, guard.RouteEnd) { + return true + } + } + return false +} + +func migrationTargetGuardProven(guard store.TargetStagedReadinessState, proofs []store.MigrationOwnershipState) bool { + // Source tracking does not attest the preexisting owner or write floor. + // A target guard alone only predicts a future cutover, not its publication. + if guard.TrackWrites || guard.SourceWriteFence || guard.SourceReadFence { + return false + } + for _, proof := range proofs { + if migrationTargetOwnershipMatches(proof, guard) { + return true + } + } + return false +} + +func migrationTargetOwnershipMatches(proof store.MigrationOwnershipState, guard store.TargetStagedReadinessState) bool { + return store.MigrationOwnershipEvidenceValid(proof) && guard.MigrationJobID == guard.JobID && proof.JobID == guard.JobID && proof.Role == store.MigrationOwner && proof.Staged && + proof.Receipts[0].CatalogVersion >= guard.ExpectedCutoverVersion && proof.MinWriteTSExclusive == guard.MinWriteTSExclusive && + bytes.Equal(proof.RouteStart, guard.RouteStart) && bytes.Equal(proof.RouteEnd, guard.RouteEnd) +} + +func (f *kvFSM) matchesMigrationResolution(ctx context.Context, req *pb.Request) (bool, error) { + if !req.GetIsTxn() || (req.GetPhase() != pb.Phase_COMMIT && req.GetPhase() != pb.Phase_ABORT) { + return false, nil + } + meta, mutations, err := extractTxnMeta(req.GetMutations()) + if err != nil { + return false, err + } + if len(mutations) == 0 || len(meta.PrimaryKey) == 0 { + return false, nil + } + return f.migrationResolutionMutationsMatch(ctx, req, meta, mutations) +} + +func (f *kvFSM) migrationResolutionMutationsMatch(ctx context.Context, req *pb.Request, meta TxnMeta, mutations []*pb.Mutation) (bool, error) { + for _, mut := range mutations { + if mut == nil || len(mut.GetKey()) == 0 || isTxnInternalKey(mut.GetKey()) { + return false, errors.WithStack(ErrInvalidRequest) + } + if err := f.rejectReservedControlMutations([]*pb.Mutation{mut}); err != nil { + return false, err + } + matches, err := f.migrationResolutionKeyMatches(ctx, req, meta, mut.GetKey()) + if err != nil || !matches { + return false, err + } + } + return true, nil +} + +func (f *kvFSM) migrationResolutionKeyMatches(ctx context.Context, req *pb.Request, meta TxnMeta, key []byte) (bool, error) { + lock, _, _, found, err := f.txnLockForMutationKey(ctx, key, req.GetReadKeys()) + if err != nil { + return false, errors.WithStack(errors.Mark(err, errTargetReadinessUnproven)) + } + if !found { + return f.migrationMissingResolutionKey(ctx, req, meta, key) + } + if lock.StartTS != req.GetTs() || !bytes.Equal(lock.PrimaryKey, meta.PrimaryKey) || + lock.IsPrimaryKey != bytes.Equal(key, meta.PrimaryKey) { + return false, nil + } + if req.GetPhase() == pb.Phase_COMMIT && lock.CommitTS != 0 && lock.CommitTS != meta.CommitTS { + return false, errors.WithStack(ErrTxnInvalidMeta) + } + return true, nil +} + +func (f *kvFSM) migrationMissingResolutionKey(ctx context.Context, req *pb.Request, meta TxnMeta, key []byte) (bool, error) { + intentKeys := append([][]byte{txnIntentKey(key)}, stagedVisibilityKeysForReadAlias(req.GetReadKeys(), key, txnIntentKey(key))...) + for _, intentKey := range intentKeys { + exists, err := f.store.ExistsAt(ctx, intentKey, ^uint64(0)) + if err != nil { + return false, errors.WithStack(errors.Mark(err, errTargetReadinessUnproven)) + } + if exists { + return false, nil + } + } + if !bytes.Equal(key, meta.PrimaryKey) { + // No secondary lock/intent means both handlers produce no mutation for + // this key. It cannot authorize creation of a primary terminal record. + return true, nil + } + return f.migrationPrimaryTerminalMatches(ctx, req, meta) +} + +func (f *kvFSM) migrationPrimaryTerminalMatches(ctx context.Context, req *pb.Request, meta TxnMeta) (bool, error) { + commitTS, committed, err := f.txnCommitTS(ctx, meta.PrimaryKey, req.GetTs()) + if err == nil && !committed { + commitTS, committed, err = f.stagedTxnCommitTS(ctx, meta.PrimaryKey, req.GetTs(), req.GetReadKeys()) + } + if err != nil { + return false, errors.WithStack(errors.Mark(err, errTargetReadinessUnproven)) + } + if committed { + if req.GetPhase() == pb.Phase_ABORT { + return false, errors.WithStack(ErrTxnAlreadyCommitted) + } + if commitTS != meta.CommitTS { + return false, errors.WithStack(ErrTxnInvalidMeta) + } + return true, nil + } + return f.migrationPrimaryRollbackMatches(ctx, req, meta) +} + +func (f *kvFSM) migrationPrimaryRollbackMatches(ctx context.Context, req *pb.Request, meta TxnMeta) (bool, error) { + key := txnRollbackKey(meta.PrimaryKey, req.GetTs()) + exists, err := f.store.ExistsAt(ctx, key, ^uint64(0)) + if err == nil && !exists { + exists, err = f.stagedTxnRecordExists(ctx, meta.PrimaryKey, key, req.GetReadKeys()) + } + if err != nil { + return false, errors.WithStack(errors.Mark(err, errTargetReadinessUnproven)) + } + if exists && req.GetPhase() == pb.Phase_COMMIT { + return false, errors.WithStack(ErrTxnAlreadyAborted) + } + return exists, nil +} diff --git a/kv/fsm_migration_size_guard.go b/kv/fsm_migration_size_guard.go new file mode 100644 index 000000000..45a64ff25 --- /dev/null +++ b/kv/fsm_migration_size_guard.go @@ -0,0 +1,57 @@ +package kv + +import ( + "context" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +// ErrMigrationTransferTooLarge is a deterministic rejection of new data in +// an explicitly guarded moving interval, not a replica-local apply failure. +var ErrMigrationTransferTooLarge = errors.Wrap(store.ErrValueTooLarge, "new version exceeds moving-range transfer budget") + +// Validate the actual rows, including encoded PREPARE locks and intents, before +// recording tracker progress or applying any row of the request. COMMIT/ABORT +// intentionally do not call this: already authenticated old resolutions drain. +func (f *kvFSM) verifyMigrationTransferSize(ctx context.Context, muts []*store.KVPairMutation) error { + // Most requests contain only small rows. Avoid a metadata read entirely + // unless a row can violate the per-version transfer budget. + var states []store.TargetStagedReadinessState + loaded := false + for _, mut := range muts { + if mut == nil || store.MigrationVersionFitsTransfer(mut.Key, mut.Value) { + continue + } + if !loaded { + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { + return nil // Stores without readiness cannot activate this guard. + } + var err error + states, err = reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + loaded = true + } + if jobID, guarded := transferSizeGuardJob(states, mut.Key); guarded { + return errors.Wrapf(ErrMigrationTransferTooLarge, + "migration job %d: key bytes %d, encoded value bytes %d", jobID, len(mut.Key), len(mut.Value)) + } + } + return nil +} + +func transferSizeGuardJob(states []store.TargetStagedReadinessState, key []byte) (uint64, bool) { + if embedded, ok := txnRouteKey(key); ok { + key = embedded + } + start, end := readinessRouteRangeForScan(key, nextScanCursor(key)) + for _, state := range states { + if state.Armed && state.TransferSizeGuardVersion != 0 && routeRangeIntersects(start, end, state.RouteStart, state.RouteEnd) { + return state.JobID, true + } + } + return 0, false +} diff --git a/kv/fsm_migration_size_guard_test.go b/kv/fsm_migration_size_guard_test.go new file mode 100644 index 000000000..67e1cbb54 --- /dev/null +++ b/kv/fsm_migration_size_guard_test.go @@ -0,0 +1,250 @@ +package kv + +import ( + "bytes" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func activateTransferSizeGuardForTest(t *testing.T, f *kvFSM) { + t.Helper() + req := &pb.TargetStagedReadinessRequest{JobId: 11, MigrationJobId: 11, + Armed: true, TrackWrites: true, RouteStart: []byte("m"), RouteEnd: []byte("z"), + ExpectedCutoverVersion: 1, MinWriteTsExclusive: 1, RetentionPinTs: 1, + TransferSizeGuardVersion: 1, RequireConfigurationIndex: true, ExpectedConfigurationIndex: 9} + command, err := MarshalTargetStagedReadinessCommand(req) + require.NoError(t, err) + require.Nil(t, f.Apply(command)) +} + +func TestMigrationSizeGuardRawRequestAtomicVerdict(t *testing.T) { + f := newMigrationWriteTrackerFSM(t) + installSourceOwnershipForTest(t, f, 0) + activateTransferSizeGuardForTest(t, f) + request := &pb.Request{Ts: 50, ObservedRouteVersion: EncodeObservedRouteVersion(1), Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("must not land")}, + {Op: pb.Op_PUT, Key: []byte("o"), Value: make([]byte, store.MigrationPagePayloadBudget+(1<<19))}, + }} + err := f.applyRequestErr(t.Context(), request) + require.ErrorIs(t, err, ErrMigrationTransferTooLarge) + require.True(t, isRawApplyVerdict(err), "size rejection must not halt the raw batch") + _, err = f.store.GetAt(t.Context(), []byte("n"), 100) + require.ErrorIs(t, err, store.ErrKeyNotFound) + require.Zero(t, migrationTrackerMinimum(t, f), "failed admission must not change the source tracker") +} + +func TestMigrationSizeGuardLegacyAndOutsideRangeUnchanged(t *testing.T) { + value := make([]byte, store.MigrationPagePayloadBudget+(1<<19)) + for _, test := range []struct { + name string + guard bool + key string + }{{"legacy tracker", false, "n"}, {"outside moving range", true, "b"}} { + t.Run(test.name, func(t *testing.T) { + f := newMigrationWriteTrackerFSM(t) + if test.guard { + require.NoError(t, f.RecordAppliedConfiguration(9)) + } else { + installSourceOwnershipForTest(t, f, 0) + } + if test.guard { + activateTransferSizeGuardForTest(t, f) + } + req := &pb.Request{Ts: 50, ObservedRouteVersion: EncodeObservedRouteVersion(1), Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte(test.key), Value: value}}} + require.NoError(t, f.applyRequestErr(t.Context(), req)) + actual, err := f.store.GetAt(t.Context(), []byte(test.key), 100) + require.NoError(t, err) + require.Len(t, actual, len(value)) + }) + } +} + +func sizeGuardTxnRequest(phase pb.Phase, value []byte) *pb.Request { + key := []byte("n") + return &pb.Request{IsTxn: true, Phase: phase, Ts: 50, ObservedRouteVersion: EncodeObservedRouteVersion(1), Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key, CommitTS: 60, LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_PUT, Key: key, Value: value}, + }} +} + +func TestMigrationSizeGuardChecksEncodedPrepareIntent(t *testing.T) { + value := make([]byte, store.MigrationPagePayloadBudget-130) + require.True(t, store.MigrationVersionFitsTransfer([]byte("n"), value), "the raw version itself fits") + f := newMigrationWriteTrackerFSM(t) + installSourceOwnershipForTest(t, f, 0) + activateTransferSizeGuardForTest(t, f) + err := f.applyRequestErr(t.Context(), sizeGuardTxnRequest(pb.Phase_PREPARE, value)) + require.ErrorIs(t, err, ErrMigrationTransferTooLarge, "the encoded intent and its physical key exceed the transfer budget") + _, err = f.store.GetAt(t.Context(), txnLockKey([]byte("n")), 100) + require.ErrorIs(t, err, store.ErrKeyNotFound, "a rejected PREPARE must not leave even its small lock row") + require.Zero(t, migrationTrackerMinimum(t, f)) + raw := &pb.Request{Ts: 70, ObservedRouteVersion: EncodeObservedRouteVersion(1), Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: value}}} + require.NoError(t, f.applyRequestErr(t.Context(), raw), "encoded-intent overhead must not reduce the raw limit") +} + +func TestMigrationSizeGuardOnePhaseRejectsAtomically(t *testing.T) { + f := newMigrationWriteTrackerFSM(t) + installSourceOwnershipForTest(t, f, 0) + activateTransferSizeGuardForTest(t, f) + req := sizeGuardTxnRequest(pb.Phase_NONE, make([]byte, store.MigrationPagePayloadBudget+(1<<19))) + req.Mutations = append(req.Mutations, &pb.Mutation{Op: pb.Op_PUT, Key: []byte("o"), Value: []byte("no partial transaction")}) + require.ErrorIs(t, f.applyRequestErr(t.Context(), req), ErrMigrationTransferTooLarge) + _, err := f.store.GetAt(t.Context(), []byte("o"), 100) + require.ErrorIs(t, err, store.ErrKeyNotFound) + require.Zero(t, migrationTrackerMinimum(t, f)) +} + +func TestMigrationSizeGuardExistingPrepareResolutionsDrain(t *testing.T) { + value := make([]byte, store.MigrationPagePayloadBudget+(1<<19)) + for _, phase := range []pb.Phase{pb.Phase_COMMIT, pb.Phase_ABORT} { + t.Run(phase.String(), func(t *testing.T) { + f := newMigrationWriteTrackerFSM(t) + installSourceOwnershipForTest(t, f, 0) + require.NoError(t, f.applyRequestErr(t.Context(), sizeGuardTxnRequest(pb.Phase_PREPARE, value))) + activateTransferSizeGuardForTest(t, f) + require.NoError(t, f.applyRequestErr(t.Context(), sizeGuardTxnRequest(phase, nil)), "existing resolution must not be reclassified as new creation") + _, err := f.store.GetAt(t.Context(), txnLockKey([]byte("n")), 100) + require.ErrorIs(t, err, store.ErrKeyNotFound) + actual, err := f.store.GetAt(t.Context(), []byte("n"), 100) + if phase == pb.Phase_COMMIT { + require.NoError(t, err) + require.Len(t, actual, len(value)) + } else { + require.ErrorIs(t, err, store.ErrKeyNotFound) + } + }) + } +} + +func TestMigrationSizeGuardRawBatchKeepsIndependentVerdicts(t *testing.T) { + f := newMigrationWriteTrackerFSM(t) + installSourceOwnershipForTest(t, f, 0) + activateTransferSizeGuardForTest(t, f) + bad := &pb.Request{Ts: 50, ObservedRouteVersion: EncodeObservedRouteVersion(1), Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: make([]byte, store.MigrationPagePayloadBudget+(1<<19))}}} + good := &pb.Request{Ts: 60, ObservedRouteVersion: EncodeObservedRouteVersion(1), Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("o"), Value: []byte("accepted independently")}}} + command, err := marshalRaftCommand([]*pb.Request{bad, good}) + require.NoError(t, err) + f.SetApplyIndex(12) + result, ok := f.Apply(command).(*fsmApplyResponse) + require.True(t, ok, "size rejection must not produce a halt") + require.ErrorIs(t, result.results[0], ErrMigrationTransferTooLarge) + require.NoError(t, result.results[1]) + actual, err := f.store.GetAt(t.Context(), []byte("o"), 100) + require.NoError(t, err) + require.Equal(t, []byte("accepted independently"), actual) +} + +func TestMigrationSizeGuardActivationOrdersAndLegacyRefresh(t *testing.T) { + value := make([]byte, store.MigrationPagePayloadBudget+(1<<19)) + for _, writeFirst := range []bool{false, true} { + f := newMigrationWriteTrackerFSM(t) + installSourceOwnershipForTest(t, f, 0) + req := &pb.Request{Ts: 50, ObservedRouteVersion: EncodeObservedRouteVersion(1), Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: value}}} + if writeFirst { + require.NoError(t, f.applyRequestErr(t.Context(), req), "a write ordered before explicit activation retains the legacy verdict") + } + activateTransferSizeGuardForTest(t, f) + // An older runner's tracker refresh must not silently clear the guard. + refresh, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{JobId: 11, MigrationJobId: 11, + Armed: true, TrackWrites: true, RouteStart: []byte("m"), RouteEnd: []byte("z"), + ExpectedCutoverVersion: 1, MinWriteTsExclusive: 1, RetentionPinTs: 1}) + require.NoError(t, err) + require.Nil(t, f.Apply(refresh)) + req.Ts = 70 + require.ErrorIs(t, f.applyRequestErr(t.Context(), req), ErrMigrationTransferTooLarge, "all writes ordered after activation see the guard") + actual, err := f.store.GetAt(t.Context(), []byte("n"), 100) + if writeFirst { + require.NoError(t, err) + require.Len(t, actual, len(value), "activation must not delete or rewrite existing oversized history") + } else { + require.ErrorIs(t, err, store.ErrKeyNotFound) + } + } +} + +func TestMigrationSizeGuardStaleActivationConfiguration(t *testing.T) { + f := newMigrationWriteTrackerFSM(t) + installSourceOwnershipForTest(t, f, 0) + command, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{JobId: 11, MigrationJobId: 11, + Armed: true, TrackWrites: true, RouteStart: []byte("m"), RouteEnd: []byte("z"), MinWriteTsExclusive: 1, RetentionPinTs: 1, + TransferSizeGuardVersion: 1, RequireConfigurationIndex: true, ExpectedConfigurationIndex: 9}) + require.NoError(t, err) + require.NoError(t, f.RecordAppliedConfiguration(12)) + response, ok := f.Apply(command).(error) + require.True(t, ok) + require.ErrorIs(t, response, ErrMigrationConfigurationChanged) + reader, ok := f.store.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(t.Context()) + require.NoError(t, err) + require.Zero(t, states[0].TransferSizeGuardVersion) +} + +func TestMigrationSizeGuardFullAtomicRequestBoundUnchanged(t *testing.T) { + value := make([]byte, 32<<20) + req := &pb.Request{Ts: 50, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte("n"), Value: value}, {Op: pb.Op_PUT, Key: []byte("o"), Value: value}, + }} + for _, mut := range req.Mutations { + require.True(t, store.MigrationVersionFitsTransfer(mut.Key, mut.Value)) + } + require.GreaterOrEqual(t, proto.Size(req), maxMarshaledCommandSize) + _, err := marshalRaftCommand([]*pb.Request{req}) + require.ErrorContains(t, err, "marshaled request too large", "fitting individual transfer rows cannot bypass the full atomic command limit") + _, err = marshalRaftCommand([]*pb.Request{{Ts: 50, Mutations: req.Mutations[:1]}, {Ts: 60, Mutations: req.Mutations[1:]}}) + require.ErrorContains(t, err, "marshaled request batch too large") +} + +func TestMigrationSizeGuardNormalizedDerivedWriters(t *testing.T) { + value := make([]byte, store.MigrationPagePayloadBudget+(1<<19)) + for _, key := range [][]byte{ + store.HashFieldKey([]byte("n"), []byte("field")), + store.ZSetMemberKey([]byte("n"), []byte("member")), + store.SetMemberKey([]byte("n"), []byte("member")), + store.StreamEntryKey([]byte("n"), 123, 4), + } { + f := newMigrationWriteTrackerFSM(t) + installSourceOwnershipForTest(t, f, 0) + activateTransferSizeGuardForTest(t, f) + req := &pb.Request{Ts: 50, ObservedRouteVersion: EncodeObservedRouteVersion(1), Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: value}}} + require.ErrorIs(t, f.applyRequestErr(t.Context(), req), ErrMigrationTransferTooLarge, "derived physical key %q must use its logical moving range", key) + } +} + +func TestMigrationSizeGuardCommitOnlyUsesPreparedBytes(t *testing.T) { + f := newMigrationWriteTrackerFSM(t) + installSourceOwnershipForTest(t, f, 0) + activateTransferSizeGuardForTest(t, f) + require.NoError(t, f.applyRequestErr(t.Context(), sizeGuardTxnRequest(pb.Phase_PREPARE, []byte("persisted intent")))) + request := sizeGuardTxnRequest(pb.Phase_COMMIT, make([]byte, store.MigrationPagePayloadBudget+(1<<19))) + require.NoError(t, f.applyRequestErr(t.Context(), request), "resolution may carry unused caller bytes, but cannot create an oversized unprepared value") + actual, err := f.store.GetAt(t.Context(), []byte("n"), 100) + require.NoError(t, err) + require.Equal(t, []byte("persisted intent"), actual) +} + +func TestMigrationSizeGuardRestoredFSMAppliesGuard(t *testing.T) { + f := newMigrationWriteTrackerFSM(t) + installSourceOwnershipForTest(t, f, 0) + activateTransferSizeGuardForTest(t, f) + snapshot, err := f.store.Snapshot() + require.NoError(t, err) + var raw bytes.Buffer + _, err = snapshot.WriteTo(&raw) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + st := store.NewMVCCStore() + t.Cleanup(func() { require.NoError(t, st.Close()) }) + require.NoError(t, st.Restore(bytes.NewReader(raw.Bytes()))) + restored := NewKvFSMWithHLC(st, nil, WithRouteHistory(nil, 1)) + command, err := marshalRaftCommand([]*pb.Request{{Ts: 50, ObservedRouteVersion: EncodeObservedRouteVersion(1), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: make([]byte, store.MigrationPagePayloadBudget+(1<<19))}}}}) + require.NoError(t, err) + response, ok := restored.Apply(command).(error) + require.True(t, ok) + require.ErrorIs(t, response, ErrMigrationTransferTooLarge, "restored FSM must reject by the persisted size guard, not missing ownership or epoch") +} diff --git a/kv/fsm_migration_source_ownership_test.go b/kv/fsm_migration_source_ownership_test.go new file mode 100644 index 000000000..763db1a97 --- /dev/null +++ b/kv/fsm_migration_source_ownership_test.go @@ -0,0 +1,94 @@ +package kv + +import ( + "crypto/sha256" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func installSourceOwnershipForTest(t *testing.T, f *kvFSM, floor uint64) { + t.Helper() + require.NoError(t, f.RecordAppliedConfiguration(9)) + guard, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{JobId: 11, MigrationJobId: 11, + Armed: true, TrackWrites: true, RouteStart: []byte("m"), RouteEnd: []byte("z"), + ExpectedCutoverVersion: 1, MinWriteTsExclusive: 1, RetentionPinTs: 1}) + require.NoError(t, err) + f.SetApplyIndex(10) + require.Nil(t, f.Apply(guard)) + route := distribution.RouteDescriptor{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 1, + State: distribution.RouteStateActive, MinWriteTSExclusive: floor} + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(route, true) + require.NoError(t, err) + digest := sha256.Sum256(encoded) + command, err := MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{JobId: 11, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_SOURCE_OWNERSHIP_ACQUIRE, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: 9, + Ownership: &pb.MigrationOwnershipProof{Generation: "0123456789abcdef0123456789abcdef", LocalGroupId: 1, OwnerGroupId: 1, + CatalogVersion: 1, RouteId: 1, RouteStart: []byte("m"), RouteEnd: []byte("z"), WriteFloorKnown: true, + MinWriteTsExclusive: floor, SourceTrackerVersion: 1, DescriptorHash: digest[:], Role: pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER}}) + require.NoError(t, err) + f.SetApplyIndex(11) + response := f.Apply(command) + result, ok := response.(store.CleanupVersionsResult) + require.True(t, ok, "ownership apply returned %v", response) + require.True(t, result.Done) +} + +func TestSourceOwnershipApplyIgnoresWatcherAndBindsConfiguration(t *testing.T) { + for _, watcher := range []RouteHistory{forbiddenApplyWatcher{}, WrapDistributionEngine(distribution.NewEngine()), WrapDistributionEngine(staleServingEngine(t, 999))} { + f := &kvFSM{store: store.NewMVCCStore(), shardGroupID: 1, routes: watcher} + installSourceOwnershipForTest(t, f, 0) + req := &pb.Request{Ts: 50, ObservedRouteVersion: EncodeObservedRouteVersion(1), Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("value")}}} + require.NoError(t, f.applyRequestErr(t.Context(), req)) + require.Equal(t, uint64(50), migrationTrackerMinimum(t, f)) + require.NoError(t, f.RecordAppliedConfiguration(12)) + req.Ts = 60 + require.ErrorIs(t, f.applyRequestErr(t.Context(), req), ErrRouteCutoverPending) + reader, ok := f.store.(store.MigrationOwnershipStore) + require.True(t, ok) + view, err := reader.MigrationServingState(t.Context()) + require.NoError(t, err) + refresh, err := MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{JobId: 11, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH, RequireConfigurationIndex: true, + ExpectedConfigurationIndex: 12, Ownership: MigrationOwnershipProofFromState(view.Ownership[0])}) + require.NoError(t, err) + f.SetApplyIndex(13) + _, ok = f.Apply(refresh).(store.CleanupVersionsResult) + require.True(t, ok) + require.NoError(t, f.applyRequestErr(t.Context(), req)) + } +} + +func TestSourceOwnershipStaleControlsCannotReopenCreation(t *testing.T) { + f := &kvFSM{store: store.NewMVCCStore(), shardGroupID: 1, routes: forbiddenApplyWatcher{}} + installSourceOwnershipForTest(t, f, 0) + control := &pb.TargetStagedReadinessRequest{JobId: 11, MigrationJobId: 11, Armed: true, TrackWrites: true, + RouteStart: []byte("m"), RouteEnd: []byte("z"), ExpectedCutoverVersion: 2, MinWriteTsExclusive: 70, + RetentionPinTs: 1, SourceWriteFence: true, SourceReadFence: true} + command, err := MarshalTargetStagedReadinessCommand(control) + require.NoError(t, err) + f.SetApplyIndex(12) + require.Nil(t, f.Apply(command)) + control.SourceWriteFence, control.SourceReadFence, control.MinWriteTsExclusive = false, false, 1 + control.ExpectedCutoverVersion = 1 + command, err = MarshalTargetStagedReadinessCommand(control) + require.NoError(t, err) + f.SetApplyIndex(13) + require.Nil(t, f.Apply(command)) + require.ErrorIs(t, f.applyRequestErr(t.Context(), &pb.Request{Ts: 80, ObservedRouteVersion: EncodeObservedRouteVersion(1), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("stale")}}}), ErrRouteCutoverPending) + control.RouteEnd = []byte("y") + command, err = MarshalTargetStagedReadinessCommand(control) + require.NoError(t, err) + f.SetApplyIndex(14) + response := f.Apply(command) + require.NoError(t, haltApplyOf(response)) + applyErr, ok := response.(error) + require.True(t, ok) + require.ErrorIs(t, applyErr, store.ErrMigrationOwnership) + require.Equal(t, uint64(70), f.store.LastCommitTS(), "rejection must not change the HLC floor") +} diff --git a/kv/fsm_migration_sqs_availability_test.go b/kv/fsm_migration_sqs_availability_test.go new file mode 100644 index 000000000..de00ca443 --- /dev/null +++ b/kv/fsm_migration_sqs_availability_test.go @@ -0,0 +1,101 @@ +package kv + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestCatalogMigrationPreservesSQSAvailabilityOutsideMovingRange(t *testing.T) { + t.Parallel() + ctx := context.Background() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{RouteID: 1, End: []byte("z"), GroupID: 1}}) + fsm := newComposed1FSM(t, engine, 1) + sharded := NewShardStore(engine, map[uint64]*ShardGroup{1: {Store: fsm.store}}) + keys := [][]byte{ + partitionedSQSMsgDataKey("orders", 3), []byte(sqsMsgVisPrefix + "p|orders|3"), + []byte(sqsMsgDedupPrefix + "p|orders|3"), []byte(sqsMsgGroupPrefix + "p|orders|3"), + []byte(sqsMsgByAgePrefix + "p|orders|3"), []byte(sqsMsgDataPrefix + "orders|legacy"), + []byte(sqsQueueMetaPrefix + "orders"), + } + partitionOwners := make(map[string]uint64) + for _, key := range keys { + if IsPartitionedSQSKey(key) { + partitionOwners[string(key)] = 1 + } + } + resolver := &fakePartitionResolver{routes: partitionOwners} + router := NewShardRouter(engine).WithPartitionResolver(resolver) + sharded.WithPartitionResolver(resolver) + version := uint64(2) + for _, phase := range []string{"FENCE", "CUTOVER", "CLEANUP"} { + t.Run(phase, func(t *testing.T) { + moving := distribution.RouteDescriptor{RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), GroupID: 1, State: distribution.RouteStateWriteFenced} + if phase != "FENCE" { + moving.GroupID, moving.State = 2, distribution.RouteStateActive + } + applyComposed1Snapshot(t, engine, version, []distribution.RouteDescriptor{ + {RouteID: 2, ParentRouteID: 1, End: []byte("m"), GroupID: 1}, moving, + }) + if phase == "CLEANUP" { + cleaner, ok := fsm.store.(store.MigrationCleaner) + require.True(t, ok) + require.NoError(t, cleaner.ClearMigrationState(ctx, 10, 0)) + } else { + applySourceMigrationControlToFSM(t, fsm, phase == "CUTOVER") + } + for _, key := range keys { + if IsPartitionedSQSKey(key) { + groupID, found := router.ResolveGroup(key) + require.True(t, found) + require.Equal(t, uint64(1), groupID, "partition ownership must not change with catalog migration phases") + } + require.Equal(t, sqsGlobalRouteKey, routeKey(key)) + require.NoError(t, distribution.ValidateSplitMigrationRouteRange([]byte("m"), []byte("z"))) + ts := version * 100 + value := []byte(phase) + require.NoError(t, fsm.handleRawRequest(ctx, &pb.Request{Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: value}}}, ts)) + got, err := sharded.GetGroupAt(ctx, 1, key, ts) + require.NoError(t, err) + require.Equal(t, value, got) + owner, found := engine.GetRoute(routeKey(key)) + require.True(t, found) + require.Equal(t, uint64(1), owner.GroupID) + } + }) + version++ + } +} + +func TestSQSShapeDoesNotBypassCatalogOrSourceFences(t *testing.T) { + t.Parallel() + for _, durable := range []bool{false, true} { + t.Run(map[bool]string{false: "catalog", true: "source"}[durable], func(t *testing.T) { + t.Parallel() + ctx := context.Background() + engine := distribution.NewEngine() + state := distribution.RouteStateWriteFenced + if durable { + state = distribution.RouteStateActive + } + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{RouteID: 1, Start: []byte("!sqs|route|"), End: []byte("!sqs|route|z"), GroupID: 1, State: state}}) + fsm := newComposed1FSM(t, engine, 1) + if durable { + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{JobID: 10, MigrationJobID: 10, + RouteStart: []byte("!sqs|route|"), RouteEnd: []byte("!sqs|route|z"), Armed: true, + SourceWriteFence: true, MinWriteTSExclusive: 1, RetentionPinTS: 1}) + } + for _, key := range [][]byte{partitionedSQSMsgDataKey("q", 3), []byte(sqsMsgDataPrefix + "q|legacy"), []byte(sqsQueueMetaPrefix + "q"), []byte("!sqs|route|global-user")} { + err := fsm.handleRawRequest(ctx, &pb.Request{Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("must-not-land")}}}, 100) + require.ErrorIs(t, err, ErrRouteWriteFenced) + _, err = fsm.store.GetAt(ctx, key, 100) + require.ErrorIs(t, err, store.ErrKeyNotFound) + } + }) + } +} diff --git a/kv/fsm_migration_transfer_semantics_test.go b/kv/fsm_migration_transfer_semantics_test.go new file mode 100644 index 000000000..16534ecbd --- /dev/null +++ b/kv/fsm_migration_transfer_semantics_test.go @@ -0,0 +1,37 @@ +package kv + +import ( + "context" + "testing" + + internalutil "github.com/bootjp/elastickv/internal" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func TestMigrationTransferDoesNotChangeLargeWriteAdmission(t *testing.T) { + value := make([]byte, store.MigrationPagePayloadBudget+(1<<19)) + ctx := context.Background() + for _, test := range []struct { + transactional bool + phase pb.Phase + }{{false, pb.Phase_NONE}, {true, pb.Phase_NONE}, {true, pb.Phase_PREPARE}} { + fsm := newMigrationWriteTrackerFSM(t) + installSourceOwnershipForTest(t, fsm, 0) + key := []byte("n") + mutations := []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: value}} + if test.transactional { + mutations = append([]*pb.Mutation{{Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key, CommitTS: 60, LockTTLms: defaultTxnLockTTLms})}}, mutations...) + } + request := &pb.Request{IsTxn: test.transactional, Phase: test.phase, Ts: 50, Mutations: mutations, ObservedRouteVersion: EncodeObservedRouteVersion(1)} + require.Less(t, proto.Size(request), internalutil.GRPCMaxMessageBytes) + require.NoError(t, fsm.applyRequestErr(ctx, request)) + if test.phase == pb.Phase_NONE { + actual, err := fsm.store.GetAt(ctx, key, 100) + require.NoError(t, err) + require.Len(t, actual, len(value)) + } + } +} diff --git a/kv/fsm_onephase_staged_replay_test.go b/kv/fsm_onephase_staged_replay_test.go new file mode 100644 index 000000000..6696fe007 --- /dev/null +++ b/kv/fsm_onephase_staged_replay_test.go @@ -0,0 +1,54 @@ +package kv + +import ( + "context" + "path/filepath" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestFSMOnePhaseStagedConflictsPreserveRaftReplay(t *testing.T) { + t.Parallel() + for _, writeStagedKey := range []bool{false, true} { + t.Run(map[bool]string{false: "read footprint", true: "write footprint"}[writeStagedKey], func(t *testing.T) { + t.Parallel() + ctx := context.Background() + engine := distribution.NewEngine() + route := distribution.RouteDescriptor{ + RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 1, + State: distribution.RouteStateActive, StagedVisibilityActive: true, + MigrationJobID: 9, MinWriteTSExclusive: 100, + } + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{route}) + st, err := store.NewPebbleStore(filepath.Join(t.TempDir(), "fsm.db")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + fsm, ok := NewKvFSMWithHLC(st, NewHLC(), WithRouteHistory(WrapDistributionEngine(engine), 1)).(*kvFSM) + require.True(t, ok) + installStagedOwnershipEvidence(t, fsm, 2, route) + require.NoError(t, st.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("n")), []byte("staged"), 20, 0)) + writeKey := []byte("b") + if writeStagedKey { + writeKey = []byte("n") + } + req := onePhaseReq(10, 120, 0, writeKey, []byte("new")) + req.ObservedRouteVersion = 2 + if !writeStagedKey { + req.ReadKeys = [][]byte{[]byte("n")} + } + fsm.SetApplyIndex(4) + require.ErrorIs(t, applyFSMRequest(t, fsm, req), store.ErrWriteConflict) + _, err = st.GetAt(ctx, writeKey, 120) + require.ErrorIs(t, err, store.ErrKeyNotFound) + req.Ts = 110 + fsm.SetApplyIndex(5) + require.NoError(t, applyFSMRequest(t, fsm, req)) + require.NoError(t, applyFSMRequest(t, fsm, req), "same entry must replay even though its live version exceeds startTS") + fsm.SetApplyIndex(6) + require.ErrorIs(t, applyFSMRequest(t, fsm, req), store.ErrWriteConflict, "a different entry cannot reuse the commit timestamp") + }) + } +} diff --git a/kv/fsm_readiness_halt_boundary_test.go b/kv/fsm_readiness_halt_boundary_test.go new file mode 100644 index 000000000..b6bea5291 --- /dev/null +++ b/kv/fsm_readiness_halt_boundary_test.go @@ -0,0 +1,278 @@ +package kv + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +// readinessWriteFailureStore fails only the readiness persistence write. +type readinessWriteFailureStore struct { + store.MVCCStore + err error + states []store.TargetStagedReadinessState +} + +func (s *readinessWriteFailureStore) ApplyTargetStagedReadiness(context.Context, store.TargetStagedReadinessState) error { + return s.err +} + +func (s *readinessWriteFailureStore) ApplyTargetStagedReadinessAt(context.Context, store.TargetStagedReadinessState, uint64) error { + return s.err +} + +func (s *readinessWriteFailureStore) MigrationTargetReadinessStates(context.Context) ([]store.TargetStagedReadinessState, error) { + return s.states, nil +} + +// Persisting the readiness guard is a local Pebble write. A replica that +// returns an ordinary error advances past the entry without the source fence, +// write tracker or target guard its peers installed, then accepts user writes +// they reject. +func TestApplyTargetStagedReadinessHaltsOnLocalPersistFailure(t *testing.T) { + t.Parallel() + + local := errors.New("pebble: commit failed") + fsm := &kvFSM{store: &readinessWriteFailureStore{MVCCStore: store.NewMVCCStore(), err: local}} + cmd, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + Armed: true, + }) + require.NoError(t, err) + + halt := haltApplyOf(fsm.Apply(cmd)) + require.Error(t, halt, "a local readiness persistence failure must halt apply") + require.True(t, errors.Is(halt, ErrTargetReadinessApply), "got %v", halt) + require.ErrorIs(t, halt, local) +} + +func TestApplyRawRequestHaltsOnMigrationTrackerPersistFailure(t *testing.T) { + t.Parallel() + + local := errors.New("pebble: commit failed") + fsm := &kvFSM{store: &readinessWriteFailureStore{ + MVCCStore: store.NewMVCCStore(), + err: local, + states: []store.TargetStagedReadinessState{{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + Armed: true, + TrackWrites: true, + MinAdmittedTS: 100, + }}, + }} + req := &pb.Request{ + Ts: 50, + Mutations: []*pb.Mutation{{ + Op: pb.Op_PUT, + Key: []byte("k"), + Value: []byte("v"), + }}, + } + // The admission gate now rejects an unproven source tracker before writes. + // Exercise the underlying write/persistence boundary independently. + err := fsm.handleRawRequest(context.Background(), req, req.Ts) + halt := haltApplyOf(applyErrorResponse(err)) + require.Error(t, halt, "a local migration tracker persistence failure must halt apply") + require.True(t, errors.Is(halt, ErrTargetReadinessApply), "got %v", halt) + require.ErrorIs(t, halt, local) +} + +// The source-read fence returns ErrRouteCutoverPending too, but it is a +// replicated, deterministic verdict: every replica reads the same readiness +// states from its own store and rejects the same transactions. Halting on it +// would stop every source replica during a normal cutover. +// +// Both errors below come from the real verifiers and go through the real +// decision function, so this pins the boundary rather than restating it: an +// earlier version keyed the halt off ErrRouteCutoverPending itself, which +// turned the ordinary rejection into a cluster stop. +func TestApplyErrorResponseHaltsOnlyOnUnprovenLocalReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 1, + State: distribution.RouteStateActive, + }) + writer, ok := fsm.store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 50, + })) + + fenceErr := fsm.verifySourceReadFenceForRange(ctx, []byte("b"), nextScanCursor([]byte("b"))) + require.ErrorIs(t, fenceErr, ErrRouteCutoverPending, "the armed fence must reject") + require.NoError(t, haltApplyOf(applyErrorResponse(fenceErr)), + "a replicated fence verdict is an ordinary rejection; halting stops every source replica during cutover") + + // The target-readiness proof reads the catalog watcher's current view, so a + // replica cannot know whether its peers agree. That one must halt. + // + // A guard expecting a cutover version this replica's catalog view has not + // reached is exactly the "peer may be ahead of me" case. It carries no fence + // flags, because targetReadinessStatesSatisfied skips fenced states. + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, store.TargetStagedReadinessState{ + JobID: 10, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 99, + MigrationJobID: 10, + MinWriteTSExclusive: 100, + Armed: true, + })) + _, readinessErr := fsm.targetReadyRoutesForRouteRange(ctx, []byte("a"), []byte("z")) + require.ErrorIs(t, readinessErr, ErrRouteCutoverPending) + halt := haltApplyOf(applyErrorResponse(readinessErr)) + require.Error(t, halt, "an unprovable local readiness verdict must halt") + require.True(t, errors.Is(halt, ErrTargetReadinessApply), "got %v", halt) +} + +// readinessReadFailureStore fails only the readiness states READ. +type readinessReadFailureStore struct { + store.MVCCStore + err error +} + +func (s *readinessReadFailureStore) MigrationTargetReadinessStates(context.Context) ([]store.TargetStagedReadinessState, error) { + return nil, s.err +} + +// The three sites below are the same rule seen from both directions, which is +// why they are pinned together: only a verdict that depended on process-local +// state halts, and every deterministic verdict stays an ordinary rejection. +// Classifying per call site rather than by that rule is what produced a halt +// that was simultaneously too wide (validation, cursors) and too narrow (local +// read failures). +func TestApplyHaltBoundaryFollowsProcessLocalityNotCallSite(t *testing.T) { + t.Parallel() + + ctx := context.Background() + local := errors.New("pebble: background error") + + t.Run("local readiness read failure halts", func(t *testing.T) { + t.Parallel() + fsm := &kvFSM{store: &readinessReadFailureStore{MVCCStore: store.NewMVCCStore(), err: local}} + _, err := fsm.targetReadyRoutesForRouteRange(ctx, []byte("a"), []byte("z")) + require.Error(t, err) + require.True(t, errors.Is(err, errTargetReadinessUnproven), + "a read that failed on this replica only must halt, not advance past the entry") + halt := haltApplyOf(applyErrorResponse(err)) + require.Error(t, halt) + }) + + t.Run("malformed readiness request stays ordinary", func(t *testing.T) { + t.Parallel() + // track_writes without a retention pin: the store rejects it + // deterministically, so every replica reaches the same verdict. + fsm := &kvFSM{store: store.NewMVCCStore()} + cmd, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{ + JobId: 9, + MigrationJobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + TrackWrites: true, + }) + require.NoError(t, err) + + resp := fsm.Apply(cmd) + require.NoError(t, haltApplyOf(resp), + "one malformed internal RPC must not stop the raft group") + respErr, ok := resp.(error) + require.True(t, ok, "the request is still rejected, got %T", resp) + require.True(t, errors.Is(respErr, store.ErrInvalidReadinessState), "got %v", respErr) + }) + + t.Run("invalid cleanup cursor stays ordinary", func(t *testing.T) { + t.Parallel() + require.True(t, isMigrationCleanupOrdinaryApplyError(errors.Mark(errors.New("x"), store.ErrInvalidExportCursor)), + "a cursor verdict is decided from the request alone") + require.False(t, isMigrationCleanupOrdinaryApplyError(local), + "a local store failure is not an ordinary verdict") + }) +} + +// TestApplyHaltsOnUnprovenReadinessInsideABatchedCommand closes the gap +// between the halt decision and the call site that has to honour it. +// +// The tests above exercise applyErrorResponse directly, so they pass whether or +// not Apply actually routes through it. Apply has two paths: a single request +// goes through applyRequest -> applyErrorResponse, while concurrent raw commits +// coalesced into one multi-request Raft command went through applyRequestErr +// and had every error recorded as a per-request result. An unproven local +// readiness verdict in a coalesced write therefore advanced this voter's +// applied index while its peers applied the write -- the committed-state +// divergence the halt exists to prevent, reachable only through batching. +func TestApplyRejectsUnprovenReadinessInsideABatchedCommand(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 1, + State: distribution.RouteStateActive, + }) + writer, ok := fsm.store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + + // A guard expecting a cutover version this replica's catalog view has not + // reached: the "a peer may be ahead of me" case, which cannot be proven + // locally. No fence flags, because targetReadinessStatesSatisfied skips + // fenced states. + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, store.TargetStagedReadinessState{ + JobID: 10, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 99, + MigrationJobID: 10, + MinWriteTSExclusive: 100, + Armed: true, + })) + + // Confirm the precondition through the real verifier, so this test cannot + // pass by failing for some unrelated reason. + _, readinessErr := fsm.targetReadyRoutesForRouteRange(ctx, []byte("a"), []byte("z")) + // cockroachdb's errors.Is, not require.ErrorIs: the sentinel is attached + // with errors.Mark, which the stdlib traversal does not follow. + require.True(t, errors.Is(readinessErr, errTargetReadinessUnproven), + "precondition: the guard must produce an unprovable local verdict, got %v", readinessErr) + + cmd := &pb.RaftCommand{ + Requests: []*pb.Request{ + {Ts: 200, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v1")}}}, + {Ts: 201, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("c"), Value: []byte("v2")}}}, + }, + } + data, err := proto.Marshal(cmd) + require.NoError(t, err) + + resp := fsm.Apply(data) + results, perRequestResults := resp.(*fsmApplyResponse) + require.True(t, perRequestResults, "incomplete replicated proof is a deterministic request verdict") + require.NoError(t, haltApplyOf(resp)) + for _, err := range results.results { + require.ErrorIs(t, err, ErrRouteCutoverPending) + } + for _, key := range []string{"b", "c"} { + exists, err := fsm.store.ExistsAt(ctx, []byte(key), 201) + require.NoError(t, err) + require.False(t, exists) + } +} diff --git a/kv/fsm_reserved_control_test.go b/kv/fsm_reserved_control_test.go index 6a5077f64..0ea483582 100644 --- a/kv/fsm_reserved_control_test.go +++ b/kv/fsm_reserved_control_test.go @@ -108,10 +108,10 @@ func TestTxnMutationHelpersRejectReservedControlKeys(t *testing.T) { []byte("!migfence|7"), } { muts := []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}} - _, err := f.uniqueMutationsAboveFloor(muts, 10) + _, err := f.uniqueMutationsAboveFloor(context.Background(), muts, 10) require.ErrorIs(t, err, ErrInvalidRequest, "prepare/one-phase must refuse %q", key) - _, err = f.uniqueTxnMutationsAboveFloor(muts, 10) + _, err = f.uniqueTxnMutationsAboveFloor(context.Background(), muts, 10) require.ErrorIs(t, err, ErrInvalidRequest, "commit must refuse %q", key) } @@ -129,9 +129,9 @@ func TestTxnMutationHelpersRejectReservedControlKeys(t *testing.T) { []byte("!dist|meta|version"), } { muts := []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}} - _, err := f.uniqueMutationsAboveFloor(muts, 10) + _, err := f.uniqueMutationsAboveFloor(context.Background(), muts, 10) require.NoError(t, err, "key %q must be accepted", key) - _, err = f.uniqueTxnMutationsAboveFloor(muts, 10) + _, err = f.uniqueTxnMutationsAboveFloor(context.Background(), muts, 10) require.NoError(t, err, "key %q must be accepted", key) } } diff --git a/kv/fsm_restore_origin.go b/kv/fsm_restore_origin.go new file mode 100644 index 000000000..c77facd80 --- /dev/null +++ b/kv/fsm_restore_origin.go @@ -0,0 +1,40 @@ +package kv + +import ( + "context" + "io" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +var ErrExternalSnapshotRestoreUnsupported = errors.New("store lacks external snapshot validation") + +var _ raftengine.ContextSnapshotRestorer = (*kvFSM)(nil) + +func (f *kvFSM) RestoreWithContext(ctx context.Context, r io.Reader, opts raftengine.SnapshotRestoreOptions) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + switch opts.Origin { + case raftengine.SnapshotRestoreRaft: + return f.Restore(r) + case raftengine.SnapshotRestoreExternalRebase: + st, ok := f.store.(store.ExternalSnapshotRestorer) + if !ok { + return errors.WithStack(ErrExternalSnapshotRestoreUnsupported) + } + return f.restoreSnapshot(r, func(body io.Reader) error { + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + return st.RestoreExternal(ctx, body) + }) + default: + return errors.New("unknown snapshot restore origin") + } +} diff --git a/kv/fsm_restore_origin_encrypted_test.go b/kv/fsm_restore_origin_encrypted_test.go new file mode 100644 index 000000000..306729620 --- /dev/null +++ b/kv/fsm_restore_origin_encrypted_test.go @@ -0,0 +1,166 @@ +package kv + +import ( + "bytes" + "encoding/binary" + "testing" + + "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func encryptedRestoreStore(t *testing.T, cipher *encryption.Cipher, sst bool, nonceEpoch uint16) store.MVCCStore { + t.Helper() + st, err := store.NewPebbleStore(t.TempDir(), store.WithSSTIngestSnapshots(sst), + store.WithEncryption(cipher, store.NewCounterNonceFactory(1, nonceEpoch), func() (uint32, bool) { return 1, true })) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + return st +} + +func encryptedRestoreSnapshot(t *testing.T, st store.MVCCStore) []byte { + t.Helper() + snapshot, err := st.Snapshot() + require.NoError(t, err) + var body bytes.Buffer + _, err = snapshot.WriteTo(&body) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + return body.Bytes() +} + +func encryptedRestorePayload(t *testing.T, st store.MVCCStore, mode string) []byte { + t.Helper() + require.NoError(t, st.PutAt(t.Context(), []byte("ordinary"), []byte("encrypted !migwrite| payload"), 30, 0)) + if mode == "epoch-only" { + for ts, epoch := range map[uint64]uint64{10: 11, 20: 22} { + require.NoError(t, st.PutAt(t.Context(), store.MigrationConfigurationIndexKey(), binary.BigEndian.AppendUint64(nil, epoch), ts, 0)) + } + } + if mode == "migration-bearing" { + writer, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(t.Context(), store.TargetStagedReadinessState{ + JobID: 7, MigrationJobID: 7, Armed: true, TrackWrites: true, + SourceWriteFence: true, SourceReadFence: true, + ExpectedCutoverVersion: 10, MinWriteTSExclusive: 100, + RouteStart: []byte("a"), RouteEnd: []byte("z"), RetentionPinTS: 1, + })) + } + return encryptedRestoreSnapshot(t, st) +} + +func TestExternalRestoreEncryptedKVAndTSOLineage(t *testing.T) { + for _, format := range []string{"native", "sst"} { + for _, mode := range []string{"proof-free", "epoch-only", "migration-bearing"} { + t.Run(format+"/"+mode, func(t *testing.T) { + keys := encryption.NewKeystore() + require.NoError(t, keys.Set(1, bytes.Repeat([]byte{0x37}, encryption.KeySize))) + cipher, err := encryption.NewCipher(keys) + require.NoError(t, err) + source := encryptedRestoreStore(t, cipher, format == "sst", 1) + body := encryptedRestorePayload(t, source, mode) + assertEncryptedRestoreNeedsCipher(t, body) + for _, header := range []bool{false, true} { + t.Run(map[bool]string{false: "headerless", true: "header"}[header], func(t *testing.T) { + raw := bytes.Clone(body) + if header { + prefix := make([]byte, hlcSnapshotHeaderLen) + copy(prefix, hlcSnapshotMagic[:]) + binary.BigEndian.PutUint64(prefix[len(hlcSnapshotMagic):], 9000) + raw = bytes.Join([][]byte{prefix, body}, nil) + } + destination := encryptedRestoreStore(t, cipher, format == "sst", 2) + verifyEncryptedKVRestore(t, destination, mode, header, raw) + verifyEncryptedTSORestore(t, mode, header, raw) + }) + } + }) + } + } +} + +// A plain store can install the raw snapshot but cannot read its ordinary +// values. TSO validation must therefore work without decrypting user payloads. +func assertEncryptedRestoreNeedsCipher(t *testing.T, body []byte) { + t.Helper() + plain, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + defer func() { require.NoError(t, plain.Close()) }() + require.NoError(t, plain.Restore(bytes.NewReader(body))) + value, err := plain.GetAt(t.Context(), []byte("ordinary"), 30) + require.ErrorContains(t, err, "encrypted value present but no cipher configured") + require.Empty(t, value) +} + +func verifyEncryptedKVRestore(t *testing.T, st store.MVCCStore, mode string, header bool, raw []byte) { + t.Helper() + require.NoError(t, st.PutAt(t.Context(), store.MigrationConfigurationIndexKey(), binary.BigEndian.AppendUint64(nil, 5), 5, 0)) + require.NoError(t, st.PutAt(t.Context(), []byte("sentinel"), []byte("unchanged"), 10, 0)) + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + fsm.restoredCutover = 33 + before := encryptedRestoreSnapshot(t, st) + err := fsm.RestoreWithContext(t.Context(), bytes.NewReader(raw), raftengine.SnapshotRestoreOptions{ + Origin: raftengine.SnapshotRestoreExternalRebase, + }) + if mode == "migration-bearing" { + require.ErrorIs(t, err, store.ErrExternalSnapshotMigrationState) + require.Equal(t, before, encryptedRestoreSnapshot(t, st)) + require.Equal(t, uint64(5), fsm.AppliedConfigurationIndex()) + require.Zero(t, fsm.hlc.PhysicalCeiling()) + require.Equal(t, uint64(33), fsm.RestoredCutover()) + value, readErr := st.GetAt(t.Context(), []byte("sentinel"), 10) + require.NoError(t, readErr) + require.Equal(t, []byte("unchanged"), value) + return + } + require.NoError(t, err) + value, err := st.GetAt(t.Context(), []byte("ordinary"), 30) + require.NoError(t, err) + require.Equal(t, []byte("encrypted !migwrite| payload"), value) + for _, ts := range []uint64{10, 20, ^uint64(0)} { + _, err = st.GetAt(t.Context(), store.MigrationConfigurationIndexKey(), ts) + require.ErrorIs(t, err, store.ErrKeyNotFound) + } + require.Zero(t, fsm.AppliedConfigurationIndex()) + require.Zero(t, fsm.RestoredCutover()) + if header { + require.Equal(t, int64(9000), fsm.hlc.PhysicalCeiling()) + } +} + +func verifyEncryptedTSORestore(t *testing.T, mode string, header bool, raw []byte) { + t.Helper() + fsm := NewTSOStateMachine(nil) + require.Nil(t, fsm.Apply(marshalTSOAllocationFloor(50))) + before := encryptedTSOState(t, fsm) + err := fsm.RestoreWithContext(t.Context(), bytes.NewReader(raw), raftengine.SnapshotRestoreOptions{ + Origin: raftengine.SnapshotRestoreExternalRebase, + }) + if mode == "migration-bearing" { + require.ErrorIs(t, err, store.ErrExternalSnapshotMigrationState) + require.Equal(t, before, encryptedTSOState(t, fsm)) + return + } + require.NoError(t, err, "TSO's cipher-free validator must inspect raw migration metadata, not decrypt user values") + if header { + require.Equal(t, int64(9000), fsm.ceilingMs.Load()) + require.Equal(t, tsoLeaseAllocationFloor(9000), fsm.AllocationFloor()) + } else { + require.Equal(t, before, encryptedTSOState(t, fsm)) + } +} + +func encryptedTSOState(t *testing.T, fsm *TSOStateMachine) []byte { + t.Helper() + snapshot, err := fsm.Snapshot() + require.NoError(t, err) + var payload bytes.Buffer + _, err = snapshot.WriteTo(&payload) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + return payload.Bytes() +} diff --git a/kv/fsm_restore_origin_real_test.go b/kv/fsm_restore_origin_real_test.go new file mode 100644 index 000000000..8aa7c4bef --- /dev/null +++ b/kv/fsm_restore_origin_real_test.go @@ -0,0 +1,103 @@ +package kv + +import ( + "bytes" + "encoding/binary" + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func restoreOriginRealStore(t *testing.T, format string) store.MVCCStore { + t.Helper() + if format == "mvcc" { + return store.NewMVCCStore() + } + st, err := store.NewPebbleStore(t.TempDir(), store.WithSSTIngestSnapshots(format == "sst")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + return st +} + +func restoreOriginRealPayload(t *testing.T, format string, bearing bool) []byte { + t.Helper() + st := restoreOriginRealStore(t, format) + key := store.MigrationConfigurationIndexKey() + require.NoError(t, st.PutAt(t.Context(), key, binary.BigEndian.AppendUint64(nil, 11), 10, 0)) + require.NoError(t, st.PutAt(t.Context(), key, binary.BigEndian.AppendUint64(nil, 22), 20, 0)) + require.NoError(t, st.PutAt(t.Context(), []byte("ordinary"), []byte("literal !migstage|ownership"), 30, 0)) + if bearing { + writer, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(t.Context(), store.TargetStagedReadinessState{ + JobID: 7, MigrationJobID: 7, Armed: true, ExpectedCutoverVersion: 10, + MinWriteTSExclusive: 100, RouteStart: []byte("a"), RouteEnd: []byte("z"), + })) + } + snapshot, err := st.Snapshot() + require.NoError(t, err) + fsmSnapshot := &kvFSMSnapshot{snapshot: snapshot, ceilingMs: 9000, useV2: true, cutover: 77} + var buf bytes.Buffer + _, err = fsmSnapshot.WriteTo(&buf) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + return buf.Bytes() +} + +func TestExternalRestoreRealStoreLineage(t *testing.T) { + for _, format := range []string{"mvcc", "native", "sst"} { + for _, bearing := range []bool{false, true} { + name := format + map[bool]string{false: "/epoch-only", true: "/migration-bearing"}[bearing] + t.Run(name, func(t *testing.T) { + raw := restoreOriginRealPayload(t, format, bearing) + verifyKVExternalRestoreLineage(t, format, bearing, raw) + tso := NewTSOStateMachine(nil) + require.Nil(t, tso.Apply(marshalTSOAllocationFloor(50))) + err := tso.RestoreWithContext(t.Context(), bytes.NewReader(raw), raftengine.SnapshotRestoreOptions{ + Origin: raftengine.SnapshotRestoreExternalRebase, + }) + if bearing { + require.ErrorIs(t, err, store.ErrExternalSnapshotMigrationState) + require.Equal(t, uint64(50), tso.AllocationFloor()) + require.Zero(t, tso.ceilingMs.Load()) + } else { + require.NoError(t, err) + require.Equal(t, int64(9000), tso.ceilingMs.Load()) + } + }) + } + } +} + +func verifyKVExternalRestoreLineage(t *testing.T, format string, bearing bool, raw []byte) { + t.Helper() + st := restoreOriginRealStore(t, format) + require.NoError(t, st.PutAt(t.Context(), store.MigrationConfigurationIndexKey(), binary.BigEndian.AppendUint64(nil, 5), 5, 0)) + require.NoError(t, st.PutAt(t.Context(), []byte("sentinel"), []byte("unchanged"), 10, 0)) + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + require.Equal(t, uint64(5), fsm.AppliedConfigurationIndex()) + err := fsm.RestoreWithContext(t.Context(), bytes.NewReader(raw), raftengine.SnapshotRestoreOptions{ + Origin: raftengine.SnapshotRestoreExternalRebase, + }) + if bearing { + require.ErrorIs(t, err, store.ErrExternalSnapshotMigrationState) + require.Equal(t, uint64(5), fsm.AppliedConfigurationIndex()) + require.Zero(t, fsm.hlc.PhysicalCeiling()) + require.Zero(t, fsm.RestoredCutover()) + value, readErr := st.GetAt(t.Context(), []byte("sentinel"), 10) + require.NoError(t, readErr) + require.Equal(t, []byte("unchanged"), value) + } else { + require.NoError(t, err) + require.Zero(t, fsm.AppliedConfigurationIndex(), "a copied epoch is not a new configuration proof") + value, readErr := st.GetAt(t.Context(), []byte("ordinary"), 30) + require.NoError(t, readErr) + require.Equal(t, []byte("literal !migstage|ownership"), value) + } + require.NoError(t, fsm.Restore(bytes.NewReader(raw)), "same-lineage restore retains its control evidence") + require.Equal(t, uint64(22), fsm.AppliedConfigurationIndex()) + require.Equal(t, uint64(77), fsm.RestoredCutover()) +} diff --git a/kv/fsm_restore_origin_test.go b/kv/fsm_restore_origin_test.go new file mode 100644 index 000000000..585947a54 --- /dev/null +++ b/kv/fsm_restore_origin_test.go @@ -0,0 +1,107 @@ +package kv + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +type restoreOriginStore struct { + store.MVCCStore + legacyCalls int + externalCalls int + externalErr error +} + +func (s *restoreOriginStore) Restore(r io.Reader) error { + s.legacyCalls++ + return s.MVCCStore.Restore(r) +} + +func (s *restoreOriginStore) RestoreExternal(_ context.Context, r io.Reader) error { + s.externalCalls++ + if s.externalErr != nil { + return s.externalErr + } + return s.MVCCStore.Restore(r) +} + +type legacyRestoreOnlyStore struct { + store.MVCCStore + calls int +} + +func (s *legacyRestoreOnlyStore) Restore(r io.Reader) error { + s.calls++ + return s.MVCCStore.Restore(r) +} + +func restoreOriginPayload(t *testing.T) []byte { + t.Helper() + snapshot, err := store.NewMVCCStore().Snapshot() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, snapshot.Close()) }) + fsmSnapshot := &kvFSMSnapshot{snapshot: snapshot, ceilingMs: 9000, useV2: true, cutover: 77} + var buf bytes.Buffer + _, err = fsmSnapshot.WriteTo(&buf) + require.NoError(t, err) + return buf.Bytes() +} + +func TestRestoreWithContextDispatchesValidatedOrigin(t *testing.T) { + payload := restoreOriginPayload(t) + for _, external := range []bool{false, true} { + t.Run(map[bool]string{false: "raft", true: "external"}[external], func(t *testing.T) { + st := &restoreOriginStore{MVCCStore: store.NewMVCCStore()} + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + origin := raftengine.SnapshotRestoreRaft + if external { + origin = raftengine.SnapshotRestoreExternalRebase + } + require.NoError(t, fsm.RestoreWithContext(context.Background(), bytes.NewReader(payload), raftengine.SnapshotRestoreOptions{Origin: origin})) + require.Equal(t, 1, st.legacyCalls+st.externalCalls) + require.Equal(t, external, st.externalCalls == 1) + require.Equal(t, int64(9000), fsm.hlc.PhysicalCeiling()) + require.Equal(t, uint64(77), fsm.RestoredCutover()) + }) + } +} + +func TestExternalRestoreNeverFallsBackOrPublishesRejectedHeader(t *testing.T) { + payload := restoreOriginPayload(t) + st := &restoreOriginStore{MVCCStore: store.NewMVCCStore(), externalErr: errors.New("rejected migration state")} + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + fsm.restoredCutover = 33 + before := fsm.hlc.PhysicalCeiling() + opts := raftengine.SnapshotRestoreOptions{Origin: raftengine.SnapshotRestoreExternalRebase} + require.ErrorIs(t, fsm.RestoreWithContext(context.Background(), bytes.NewReader(payload), opts), st.externalErr) + require.Equal(t, 1, st.externalCalls) + require.Zero(t, st.legacyCalls) + require.Equal(t, before, fsm.hlc.PhysicalCeiling()) + require.Equal(t, uint64(33), fsm.RestoredCutover()) + + legacy := &legacyRestoreOnlyStore{MVCCStore: store.NewMVCCStore()} + fsm, ok = NewKvFSMWithHLC(legacy, NewHLC()).(*kvFSM) + require.True(t, ok) + require.ErrorIs(t, fsm.RestoreWithContext(context.Background(), bytes.NewReader(payload), opts), ErrExternalSnapshotRestoreUnsupported) + require.Zero(t, legacy.calls) +} + +func TestRestoreWithContextRejectsCanceledOrUnknownOrigin(t *testing.T) { + st := &restoreOriginStore{MVCCStore: store.NewMVCCStore()} + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.ErrorIs(t, fsm.RestoreWithContext(ctx, bytes.NewReader(nil), raftengine.SnapshotRestoreOptions{}), context.Canceled) + require.Error(t, fsm.RestoreWithContext(context.Background(), bytes.NewReader(nil), raftengine.SnapshotRestoreOptions{Origin: 255})) + require.Zero(t, st.legacyCalls+st.externalCalls) +} diff --git a/kv/fsm_txn_test.go b/kv/fsm_txn_test.go index 97fb7d17f..f0d96fa77 100644 --- a/kv/fsm_txn_test.go +++ b/kv/fsm_txn_test.go @@ -421,6 +421,33 @@ func TestCommitRejectsMissingPrimaryKey(t *testing.T) { require.ErrorIs(t, err, ErrTxnPrimaryKeyRequired) } +func TestCommitRejectsMissingPrimaryLockWithoutCommitRecord(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + + primary := []byte("p") + startTS := uint64(10) + commitTS := uint64(20) + commit := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_COMMIT, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primary, CommitTS: commitTS})}, + {Op: pb.Op_PUT, Key: primary}, + }, + } + + err := applyFSMRequest(t, fsm, commit) + require.ErrorIs(t, err, ErrTxnInvalidMeta) + _, err = st.GetAt(ctx, txnCommitKey(primary, startTS), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound, "a missing primary lock must not produce only a commit record") +} + func TestCommitRejectsMismatchedPrimaryKey(t *testing.T) { t.Parallel() diff --git a/kv/migration_apply_range_proofs.go b/kv/migration_apply_range_proofs.go new file mode 100644 index 000000000..3949edef4 --- /dev/null +++ b/kv/migration_apply_range_proofs.go @@ -0,0 +1,191 @@ +package kv + +import ( + "bytes" + "slices" + "strconv" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +// Legacy job certificates and active partitions share interval selection, but +// retain separate evidence validation and identities. A staged-data job is not +// a serving partition ID: multiple children can read the same staged lineage. +type migrationAdmissionProof struct { + store.MigrationOwnershipState + partition *store.MigrationOwnershipPartition + identity string +} + +func migrationAdmissionProofs(view store.MigrationServingState) []migrationAdmissionProof { + proofs := make([]migrationAdmissionProof, 0, len(view.Ownership)+len(view.Partitions.Partitions)) + for _, state := range view.Ownership { + proofs = append(proofs, migrationAdmissionProof{MigrationOwnershipState: state, identity: "job/" + strconv.FormatUint(state.JobID, 10)}) + } + for i := range view.Partitions.Partitions { + p := &view.Partitions.Partitions[i] + proofs = append(proofs, migrationAdmissionProof{partition: p, identity: "partition/" + string(p.PartitionID), + MigrationOwnershipState: store.MigrationOwnershipState{ + Generation: p.Generation, JobID: p.MigrationJobID, LocalGroupID: p.LocalGroupID, OwnerGroupID: p.OwnerGroupID, + CatalogVersion: p.CatalogVersion, ConfigurationIndex: p.ConfigurationIndex, AppliedIndex: p.AppliedIndex, + RouteStart: p.RouteStart, RouteEnd: p.RouteEnd, RouteID: p.RouteID, ParentRouteID: p.ParentRouteID, + DescriptorHash: p.DescriptorHash, Role: p.Role, Staged: p.Staged, MinWriteTSExclusive: p.MinWriteTSExclusive, + WriteFloorKnown: p.WriteFloorKnown, SourceTrackerVersion: p.SourceTrackerVersion, SourceAppliedIndex: p.SourceAppliedIndex, + }}) + } + return proofs +} + +func (p migrationAdmissionProof) evidenceValid(view store.MigrationServingState) bool { + if p.partition != nil { + return store.MigrationOwnershipPartitionEvidenceValid(view.Partitions, *p.partition) + } + return store.MigrationOwnershipEvidenceValid(p.MigrationOwnershipState) +} + +func migrationAdmissionFloor(view store.MigrationServingState) uint64 { + floor := migrationServingFloor(view.Ownership) + for _, partition := range view.Partitions.Partitions { + floor = max(floor, partition.CatalogVersion) + } + return floor +} + +func migrationAdmissionBoundaries(proofs []migrationAdmissionProof, r migrationApplyRange) ([][]byte, bool) { + boundaries := [][]byte{r.start} + var intersects bool + for _, proof := range proofs { + if !routeRangeIntersects(r.start, r.end, proof.RouteStart, proof.RouteEnd) { + continue + } + intersects = true + for _, boundary := range [][]byte{proof.RouteStart, proof.RouteEnd} { + if bytes.Compare(boundary, r.start) > 0 && (len(r.end) == 0 || bytes.Compare(boundary, r.end) < 0) { + boundaries = append(boundaries, boundary) + } + } + } + slices.SortFunc(boundaries, bytes.Compare) + return slices.CompactFunc(boundaries, bytes.Equal), intersects +} + +func migrationAdmissionCertificate(proofs []migrationAdmissionProof, start, end []byte) (migrationAdmissionProof, error) { + var selected migrationAdmissionProof + for _, proof := range proofs { + if !routeRangeIntersects(start, end, proof.RouteStart, proof.RouteEnd) { + continue + } + if proof.CatalogVersion == selected.CatalogVersion && proof.identity != selected.identity { + return selected, errors.WithStack(ErrRouteCutoverPending) + } + if proof.CatalogVersion > selected.CatalogVersion { + selected = proof + } + } + if selected.identity == "" || !migrationAdmissionProofCovers(selected, start, end) { + return selected, errors.WithStack(ErrRouteCutoverPending) + } + return selected, nil +} + +func migrationAdmissionProofCovers(proof migrationAdmissionProof, start, end []byte) bool { + return bytes.Compare(proof.RouteStart, start) <= 0 && + (len(proof.RouteEnd) == 0 || (len(end) != 0 && bytes.Compare(end, proof.RouteEnd) <= 0)) +} + +func migrationAdmissionRangeProofs(view store.MigrationServingState, r migrationApplyRange) ([]migrationAdmissionProof, error) { + for _, legacy := range view.Ownership { + if routeRangeIntersects(r.start, r.end, legacy.RouteStart, legacy.RouteEnd) && !store.MigrationOwnershipEvidenceValid(legacy) { + return nil, errors.WithStack(ErrRouteCutoverPending) + } + } + all := migrationAdmissionProofs(view) + intersecting := all[:0] + for _, proof := range all { + if routeRangeIntersects(r.start, r.end, proof.RouteStart, proof.RouteEnd) { + intersecting = append(intersecting, proof) + } + } + slices.SortFunc(intersecting, func(a, b migrationAdmissionProof) int { + return bytes.Compare(a.RouteStart, b.RouteStart) + }) + if migrationAdmissionProofsDisjoint(intersecting) { + return migrationAdmissionDisjointRangeProofs(view, intersecting, r) + } + // Overlapping legacy certificates retain the original conservative tie and + // newest-version selection order, including negative/unknown barriers. + return migrationAdmissionOverlappingRangeProofs(view, migrationAdmissionProofs(view), r) +} + +func migrationAdmissionProofsDisjoint(proofs []migrationAdmissionProof) bool { + for i := 1; i < len(proofs); i++ { + end := proofs[i-1].RouteEnd + if len(end) == 0 || bytes.Compare(end, proofs[i].RouteStart) > 0 { + return false + } + } + return true +} + +func migrationAdmissionDisjointRangeProofs(view store.MigrationServingState, proofs []migrationAdmissionProof, r migrationApplyRange) ([]migrationAdmissionProof, error) { + if len(proofs) == 0 { + return nil, nil + } + cursor := r.start + for _, proof := range proofs { + if bytes.Compare(proof.RouteStart, cursor) > 0 || !proof.evidenceValid(view) { + return nil, errors.WithStack(ErrRouteCutoverPending) + } + if len(proof.RouteEnd) == 0 || (len(r.end) != 0 && bytes.Compare(proof.RouteEnd, r.end) >= 0) { + return proofs, nil + } + cursor = proof.RouteEnd + } + return nil, errors.WithStack(ErrRouteCutoverPending) +} + +func migrationAdmissionOverlappingRangeProofs(view store.MigrationServingState, all []migrationAdmissionProof, r migrationApplyRange) ([]migrationAdmissionProof, error) { + boundaries, intersects := migrationAdmissionBoundaries(all, r) + if !intersects { + return nil, nil + } + proofs := make([]migrationAdmissionProof, 0, len(boundaries)) + for i, start := range boundaries { + end := r.end + if i+1 < len(boundaries) { + end = boundaries[i+1] + } + proof, err := migrationAdmissionCertificate(all, start, end) + if err != nil { + return nil, err + } + if !proof.evidenceValid(view) { + return nil, errors.WithStack(ErrRouteCutoverPending) + } + proofs = append(proofs, proof) + } + return proofs, nil +} + +func migrationAdmissionOwnershipProofs(view store.MigrationServingState, ranges []migrationApplyRange, groupID, version uint64) ([]migrationAdmissionProof, error) { + var proofs []migrationAdmissionProof + var uncovered bool + for _, r := range ranges { + selected, err := migrationAdmissionRangeProofs(view, r) + if err != nil { + return nil, err + } + for _, proof := range selected { + if proof.Role != store.MigrationOwner || proof.LocalGroupID != groupID || proof.CatalogVersion != version { + return nil, errors.WithStack(ErrRouteCutoverPending) + } + } + uncovered = uncovered || len(selected) == 0 + proofs = append(proofs, selected...) + } + if uncovered && len(proofs) != 0 { + return nil, errors.WithStack(ErrRouteCutoverPending) + } + return proofs, nil +} diff --git a/kv/migration_broadcast_admission.go b/kv/migration_broadcast_admission.go new file mode 100644 index 000000000..41dccb765 --- /dev/null +++ b/kv/migration_broadcast_admission.go @@ -0,0 +1,98 @@ +package kv + +import ( + "context" + "maps" + "slices" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +type admittedBroadcastParticipant struct { + group *ShardGroup + requests []*pb.Request +} + +// Route-local preflight is insufficient for a full-prefix broadcast. Finish +// every participant's whole-request admission before calling any Commit. +// Apply-time admission still repeats these checks; this is not distributed +// atomic commit against concurrent state changes or transport failures. +func (c *ShardedCoordinator) admitBroadcastRequests(ctx context.Context, requests []*pb.Request) ([]admittedBroadcastParticipant, error) { + groups, err := c.allShardGroups() + if err != nil { + return nil, err + } + // allShardGroups uses these same immutable, ordered configuration IDs. + ids := c.allShardGroupIDs + if !c.allShardGroupsConfigured { + ids = slices.Sorted(maps.Keys(c.groups)) + } + if len(ids) != len(groups) { + return nil, errors.WithStack(ErrInvalidRequest) + } + migration, err := broadcastIntersectsMigration(ctx, groups, requests) + if err != nil { + return nil, err + } + participants := make([]admittedBroadcastParticipant, 0, len(groups)) + for i, group := range groups { + if c.groups[ids[i]] != group || group.Txn == nil { + return nil, errors.WithStack(ErrLeaderNotFound) + } + owned, err := c.admitBroadcastParticipant(ctx, group, ids[i], requests, migration) + if err != nil { + return nil, err + } + participants = append(participants, admittedBroadcastParticipant{group: group, requests: owned}) + } + return participants, nil +} + +func (c *ShardedCoordinator) admitBroadcastParticipant(ctx context.Context, group *ShardGroup, groupID uint64, requests []*pb.Request, migration bool) ([]*pb.Request, error) { + if migration { + if _, supported := group.Store.(store.MigrationOwnershipStore); !supported { + return nil, errors.WithStack(store.ErrNotSupported) + } + } + owned := make([]*pb.Request, len(requests)) + for i, request := range requests { + owned[i] = proto.CloneOf(request) + } + if err := StampTimestampAdmission(ctx, c.tsAllocator, group.Store, owned); err != nil { + return nil, err + } + // Evidence-free legacy groups without a store keep their existing behavior. + // They cannot participate in a migration-bearing broadcast above. + if group.Store != nil { + for _, request := range owned { + if err := CheckMigrationRequest(ctx, group.Store, groupID, request); err != nil { + return nil, err + } + } + } + return owned, nil +} + +func broadcastIntersectsMigration(ctx context.Context, groups []*ShardGroup, requests []*pb.Request) (bool, error) { + var ranges []migrationApplyRange + for _, request := range requests { + footprint, err := migrationServingFootprint(request) + if err != nil { + return false, err + } + ranges = append(ranges, footprint...) + } + for _, group := range groups { + view, err := readMigrationServingState(ctx, group.Store) + if err != nil { + return false, err + } + if migrationApplyIntersects(view, ranges) { + return true, nil + } + } + return false, nil +} diff --git a/kv/migration_broadcast_admission_test.go b/kv/migration_broadcast_admission_test.go new file mode 100644 index 000000000..28a19cffd --- /dev/null +++ b/kv/migration_broadcast_admission_test.go @@ -0,0 +1,160 @@ +package kv + +import ( + "context" + "crypto/sha256" + "strconv" + "sync/atomic" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type broadcastCommitRecorder struct { + inner Transactional + commits atomic.Int32 +} + +func (r *broadcastCommitRecorder) Commit(ctx context.Context, reqs []*pb.Request) (*TransactionResponse, error) { + r.commits.Add(1) + return r.inner.Commit(ctx, reqs) +} + +func (r *broadcastCommitRecorder) Abort(ctx context.Context, reqs []*pb.Request) (*TransactionResponse, error) { + return r.inner.Abort(ctx, reqs) +} + +func realPrefixBroadcastFixture(t *testing.T, retiredSource bool) (*ShardedCoordinator, map[uint64]*broadcastCommitRecorder) { + t.Helper() + routes := []distribution.RouteDescriptor{ + {RouteID: 1, End: []byte("p"), GroupID: 11, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("p"), End: []byte("q"), GroupID: 22, State: distribution.RouteStateActive, MinWriteTSExclusive: 100}, + {RouteID: 3, Start: []byte("q"), GroupID: 33, State: distribution.RouteStateActive}, + } + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{Version: 7, Routes: routes})) + groups := make(map[uint64]*ShardGroup) + recorders := make(map[uint64]*broadcastCommitRecorder) + for _, gid := range []uint64{11, 22, 33} { + st := store.NewMVCCStore() + f, ok := NewKvFSMWithHLC(st, NewHLC(), WithRouteHistory(WrapDistributionEngine(engine), gid)).(*kvFSM) + require.True(t, ok) + raft, stop := newSingleRaft(t, "prefix-"+strconv.FormatUint(gid, 10), f) + t.Cleanup(stop) + if gid == 22 || (gid == 11 && retiredSource) { + installBroadcastOwnership(t, f, raft, routes[1], gid) + } + tx := NewLeaderProxyWithEngine(raft, WithMigrationRequestAdmission(st, gid)) + t.Cleanup(tx.tm.Close) + recorder := &broadcastCommitRecorder{inner: tx} + recorders[gid] = recorder + groups[gid] = &ShardGroup{Engine: raft, Store: st, Txn: recorder} + } + for gid, key := range map[uint64]string{11: "a", 22: "pa", 33: "z"} { + require.NoError(t, groups[gid].Store.PutAt(t.Context(), []byte(key), []byte(key), 120, 0)) + } + shards := NewShardStore(engine, groups) + c := NewShardedCoordinator(engine, groups, 11, NewHLC(), shards).WithAllShardGroups(33, 11, 22) + t.Cleanup(func() { require.NoError(t, c.Close()) }) + return c, recorders +} + +func installBroadcastOwnership(t *testing.T, f *kvFSM, proposer raftengine.Proposer, route distribution.RouteDescriptor, gid uint64) { + t.Helper() + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(route, true) + require.NoError(t, err) + digest := sha256.Sum256(encoded) + role := pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER + if gid != route.GroupID { + role = pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_NONOWNER + } + command, err := MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{JobId: 7, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: f.AppliedConfigurationIndex(), + Ownership: &pb.MigrationOwnershipProof{Generation: "0123456789abcdef0123456789abcdef", + LocalGroupId: gid, OwnerGroupId: route.GroupID, CatalogVersion: 7, RouteId: route.RouteID, + RouteStart: route.Start, RouteEnd: route.End, MinWriteTsExclusive: 100, + WriteFloorKnown: true, DescriptorHash: digest[:], Role: role}}) + require.NoError(t, err) + result, err := proposer.ProposeAdmin(t.Context(), command) + require.NoError(t, err) + response, ok := result.Response.(store.CleanupVersionsResult) + require.True(t, ok, "%v", result.Response) + require.True(t, response.Done) +} + +func TestMigrationBroadcastRefusesBeforeAnyParticipantCommit(t *testing.T) { + c, recorders := realPrefixBroadcastFixture(t, true) + _, err := c.Dispatch(t.Context(), &OperationGroup[OP]{ObservedRouteVersion: 7, + Elems: []*Elem[OP]{{Op: DelPrefix}}}) + require.ErrorIs(t, err, ErrRouteCutoverPending) + assertBroadcastUnchanged(t, c, recorders) +} + +func assertBroadcastUnchanged(t *testing.T, c *ShardedCoordinator, recorders map[uint64]*broadcastCommitRecorder) { + t.Helper() + for _, gid := range []uint64{11, 22, 33} { + assert.Zero(t, recorders[gid].commits.Load(), "group %d must not receive Commit", gid) + } + for gid, key := range map[uint64]string{11: "a", 22: "pa", 33: "z"} { + value, getErr := c.groups[gid].Store.GetAt(t.Context(), []byte(key), ^uint64(0)) + require.NoError(t, getErr, "group %d must remain unchanged", gid) + require.Equal(t, []byte(key), value) + } +} + +func TestMigrationBroadcastPreservesObservedVersionWithoutRepinning(t *testing.T) { + for _, version := range []uint64{0, 6, 8} { + t.Run(strconv.FormatUint(version, 10), func(t *testing.T) { + c, recorders := realPrefixBroadcastFixture(t, false) + _, err := c.Dispatch(t.Context(), &OperationGroup[OP]{ObservedRouteVersion: version, + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("p")}}}) + require.ErrorIs(t, err, ErrRouteCutoverPending) + assertBroadcastUnchanged(t, c, recorders) + }) + } +} + +func TestMigrationBroadcastTimestampSealAndParticipantSelection(t *testing.T) { + c, recorders := realPrefixBroadcastFixture(t, false) + for _, command := range [][]byte{EncodePrepareEvidenceFence(0), EncodePrepareEvidenceSeal()} { + result, err := c.groups[33].Engine.ProposeAdmin(t.Context(), command) + require.NoError(t, err) + require.Nil(t, result.Response) + } + _, err := c.Dispatch(t.Context(), &OperationGroup[OP]{ObservedRouteVersion: 7, + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("p")}}}) + require.ErrorIs(t, err, ErrPrepareCreationSealed) + assertBroadcastUnchanged(t, c, recorders) + // Explicit participant selection must not probe or commit the excluded + // group's timestamp gate, nor substitute the slice index for its group ID. + c.WithAllShardGroups(22, 11) + _, err = c.Dispatch(t.Context(), &OperationGroup[OP]{ObservedRouteVersion: 7, + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("p")}}}) + require.NoError(t, err) + require.Equal(t, int32(1), recorders[11].commits.Load()) + require.Equal(t, int32(1), recorders[22].commits.Load()) + require.Zero(t, recorders[33].commits.Load()) +} + +func TestMigrationBroadcastFullyAdmittedPrefixCommitsAllParticipants(t *testing.T) { + c, recorders := realPrefixBroadcastFixture(t, false) + _, err := c.Dispatch(t.Context(), &OperationGroup[OP]{ObservedRouteVersion: 7, + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("p")}}}) + require.NoError(t, err) + for _, gid := range []uint64{11, 22, 33} { + require.Equal(t, int32(1), recorders[gid].commits.Load(), "group %d", gid) + } + _, err = c.groups[22].Store.GetAt(t.Context(), []byte("pa"), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound) + for gid, key := range map[uint64]string{11: "a", 33: "z"} { + value, getErr := c.groups[gid].Store.GetAt(t.Context(), []byte(key), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte(key), value) + } +} diff --git a/kv/migration_frontier_admission_benchmark_test.go b/kv/migration_frontier_admission_benchmark_test.go new file mode 100644 index 000000000..d92a82ca9 --- /dev/null +++ b/kv/migration_frontier_admission_benchmark_test.go @@ -0,0 +1,94 @@ +package kv + +import ( + "bytes" + "crypto/sha256" + "fmt" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +// Keep durable setup and the cached-view clone outside the selection timing. +// The shared receipt deliberately has many children, as after a large re-split. +func benchmarkFrontierAdmissionView(b *testing.B, size int) store.MigrationServingState { + b.Helper() + f, _, _, legacy := resplitSourceReceiptFixture(b) + before, err := store.MigrationOwnershipLegacyReferenceFor(legacy) + require.NoError(b, err) + tx := store.MigrationOwnershipPartitionTransition{OperationID: "kv/benchmark/frontier", + Operation: store.MigrationOwnershipPartitionSplit, RootManifestHash: bytes.Repeat([]byte{1}, sha256.Size), + Generation: legacy.Generation, LocalGroupID: legacy.LocalGroupID, CatalogVersion: legacy.CatalogVersion + 1, + ConfigurationIndex: legacy.ConfigurationIndex, Before: []store.MigrationOwnershipPartitionReference{before}} + for i := range size { + start, end := fmt.Appendf(nil, "m%06d", i), fmt.Appendf(nil, "m%06d", i+1) + if i == 0 { + start = legacy.RouteStart + } + if i == size-1 { + end = legacy.RouteEnd + } + route := distribution.RouteDescriptor{RouteID: uint64(i) + 4, ParentRouteID: legacy.RouteID, + Start: start, End: end, GroupID: legacy.OwnerGroupID, MinWriteTSExclusive: legacy.MinWriteTSExclusive} + encoded, encodeErr := distribution.EncodeRouteDescriptorForCatalogWrite(route, true) + require.NoError(b, encodeErr) + hash := sha256.Sum256(encoded) + p := store.MigrationOwnershipPartition{Generation: legacy.Generation, + LocalGroupID: legacy.LocalGroupID, OwnerGroupID: legacy.OwnerGroupID, + CatalogVersion: tx.CatalogVersion, ConfigurationIndex: tx.ConfigurationIndex, + RouteID: route.RouteID, ParentRouteID: route.ParentRouteID, RouteStart: start, RouteEnd: end, + DescriptorHash: hash[:], Role: legacy.Role, Staged: legacy.Staged, MigrationJobID: legacy.JobID, + MinWriteTSExclusive: legacy.MinWriteTSExclusive, WriteFloorKnown: true, + SourceTrackerVersion: legacy.SourceTrackerVersion, SourceAppliedIndex: legacy.SourceAppliedIndex} + for _, receipt := range legacy.Receipts { + p.Origins = append(p.Origins, store.MigrationOwnershipOperationReference{JobID: legacy.JobID, + Operation: receipt.Operation, CatalogVersion: receipt.CatalogVersion, + ConfigurationIndex: receipt.ConfigurationIndex, AppliedIndex: receipt.AppliedIndex, + DescriptorHash: receipt.DescriptorHash, Staged: receipt.Staged}) + } + tx.After = append(tx.After, p) + } + for i := range tx.After { + tx.After[i].PartitionID, err = store.MigrationOwnershipPartitionIDFor(tx, i) + require.NoError(b, err) + } + writer, ok := f.store.(store.MigrationOwnershipPartitionStore) + require.True(b, ok) + require.NoError(b, writer.ApplyMigrationOwnershipPartition(b.Context(), tx, 12)) + reader, ok := f.store.(store.MigrationServingAdmissionReader) + require.True(b, ok) + view, err := reader.MigrationServingAdmissionState(b.Context()) + require.NoError(b, err) + require.Empty(b, view.Ownership) + require.Len(b, view.Partitions.Partitions, size) + return view +} + +func BenchmarkMigrationFrontierAdmission(b *testing.B) { + for _, size := range []int{16, 128, 512} { + b.Run(fmt.Sprintf("partitions=%d", size), func(b *testing.B) { + view := benchmarkFrontierAdmissionView(b, size) + point := view.Partitions.Partitions[size/2].RouteStart + for _, tc := range []struct { + name string + rangeToProve migrationApplyRange + want int + }{ + {name: "selected-point", rangeToProve: migrationApplyRange{point, nextScanCursor(point)}, want: 1}, + {name: "full-scan", rangeToProve: migrationApplyRange{view.Partitions.Partitions[0].RouteStart, view.Partitions.Partitions[size-1].RouteEnd}, want: size}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + proofs, err := migrationAdmissionRangeProofs(view, tc.rangeToProve) + if err != nil || len(proofs) != tc.want { + b.Fatalf("admission selected %d proofs, want %d: %v", len(proofs), tc.want, err) + } + } + }) + } + }) + } +} diff --git a/kv/migration_frontier_admission_performance_test.go b/kv/migration_frontier_admission_performance_test.go new file mode 100644 index 000000000..b01f30d1f --- /dev/null +++ b/kv/migration_frontier_admission_performance_test.go @@ -0,0 +1,127 @@ +package kv + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +type frontierAdmissionCountingStore struct { + store.MVCCStore + store.MigrationOwnershipStore + admissionReads atomic.Int64 + ledgerReads atomic.Int64 +} + +func (s *frontierAdmissionCountingStore) MigrationServingState(ctx context.Context) (store.MigrationServingState, error) { + s.ledgerReads.Add(1) + return s.MigrationOwnershipStore.MigrationServingState(ctx) +} + +func (s *frontierAdmissionCountingStore) MigrationServingAdmissionState(ctx context.Context) (store.MigrationServingState, error) { + s.admissionReads.Add(1) + reader, ok := s.MVCCStore.(store.MigrationServingAdmissionReader) + if !ok { + return store.MigrationServingState{}, store.ErrNotSupported + } + return reader.MigrationServingAdmissionState(ctx) +} + +func (s *frontierAdmissionCountingStore) VersionExistsAtOrBefore(ctx context.Context, key []byte, ts uint64) (bool, error) { + reader, ok := s.MVCCStore.(store.VersionPresenceReader) + if !ok { + return false, store.ErrNotSupported + } + return reader.VersionExistsAtOrBefore(ctx, key, ts) +} + +func TestMigrationFrontierAdmissionOrdinaryReadsOnlyInspectSelectedStore(t *testing.T) { + for _, backend := range []string{"memory", "pebble"} { + t.Run(backend, func(t *testing.T) { frontierAdmissionOrdinaryReadSuite(t, backend) }) + } +} + +func frontierAdmissionOrdinaryReadSuite(t *testing.T, backend string) { + t.Helper() + for _, operation := range []string{"get", "scan", "keys", "reverse", "latest", "exists", "batch-exists", "committed"} { + t.Run(operation, func(t *testing.T) { + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 10, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 1}, + {RouteID: 2, Start: []byte("m"), End: []byte("z"), GroupID: 2}, + }) + groups := make(map[uint64]*ShardGroup) + stores := make([]*frontierAdmissionCountingStore, 0, 2) + for _, id := range []uint64{1, 2} { + st := store.NewMVCCStore() + if backend == "pebble" { + var err error + st, err = store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + } + ownership, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + counting := &frontierAdmissionCountingStore{MVCCStore: st, MigrationOwnershipStore: ownership} + groups[id] = &ShardGroup{Store: counting} + stores = append(stores, counting) + } + require.NoError(t, stores[0].PutAt(t.Context(), []byte("b"), []byte("value"), 5, 0)) + shards := NewShardStore(engine, groups) + frontierAdmissionRead(t, shards, operation) + require.Zero(t, stores[0].ledgerReads.Load(), "the legacy reader may take Pebble applyMu and DB locks") + require.Positive(t, stores[0].admissionReads.Load(), "the selected store must still prove admission") + require.Zero(t, stores[1].ledgerReads.Load(), "unselected shard must not be visited") + require.Zero(t, stores[1].admissionReads.Load(), "common reads must not scale with all groups") + }) + } +} + +func frontierAdmissionRead(t *testing.T, shards *ShardStore, operation string) { + t.Helper() + ctx := t.Context() + key, end := []byte("b"), []byte("c") + switch operation { + case "get": + value, err := shards.GetAt(ctx, key, 20) + require.NoError(t, err) + require.Equal(t, []byte("value"), value) + case "scan": + rows, err := shards.ScanAt(ctx, key, end, 10, 20) + require.NoError(t, err) + require.Len(t, rows, 1) + case "keys": + keys, err := shards.ScanKeysAt(ctx, key, end, 10, 20) + require.NoError(t, err) + require.Equal(t, [][]byte{key}, keys) + case "reverse": + rows, err := shards.ReverseScanAt(ctx, key, end, 10, 20) + require.NoError(t, err) + require.Len(t, rows, 1) + case "latest": + ts, found, err := shards.LatestCommitTS(ctx, key) + require.NoError(t, err) + require.True(t, found) + require.EqualValues(t, 5, ts) + case "exists": + found, answered, err := shards.VersionExistsAtOrBeforeGroupWithReadFence(ctx, key, 1, 20, 0) + require.NoError(t, err) + require.True(t, answered) + require.True(t, found) + case "batch-exists": + found, answered, err := shards.VersionsExistAtOrBeforeGroupWithReadFence(ctx, [][]byte{key}, 1, 20, 0) + require.NoError(t, err) + require.True(t, answered) + require.Equal(t, []bool{true}, found) + case "committed": + found, err := shards.CommittedVersionAt(ctx, key, 5) + require.NoError(t, err) + require.True(t, found) + default: + t.Fatalf("unknown read %q", operation) + } +} diff --git a/kv/migration_frontier_admission_test.go b/kv/migration_frontier_admission_test.go new file mode 100644 index 000000000..280ccac32 --- /dev/null +++ b/kv/migration_frontier_admission_test.go @@ -0,0 +1,234 @@ +package kv + +import ( + "bytes" + "crypto/sha256" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func frontierAdmissionSplitFixture(t *testing.T) (*kvFSM, *distribution.Engine, store.MigrationServingState) { + t.Helper() + f, catalog, engine, legacy := resplitSourceReceiptFixture(t) + children := []distribution.RouteDescriptor{ + {RouteID: 4, ParentRouteID: 3, Start: []byte("m"), End: []byte("t"), GroupID: 2, MinWriteTSExclusive: 100}, + {RouteID: 5, ParentRouteID: 3, Start: []byte("t"), End: []byte("z"), GroupID: 2, MinWriteTSExclusive: 100}, + } + snapshot, err := catalog.Save(t.Context(), legacy.CatalogVersion, children) + require.NoError(t, err) + require.NoError(t, engine.ApplySnapshot(snapshot)) + view := frontierAdmissionSplitLegacy(t, f, legacy, children, snapshot.Version) + return f, engine, view +} + +func frontierAdmissionSplitLegacy(t *testing.T, f *kvFSM, legacy store.MigrationOwnershipState, children []distribution.RouteDescriptor, version uint64) store.MigrationServingState { + t.Helper() + before, err := store.MigrationOwnershipLegacyReferenceFor(legacy) + require.NoError(t, err) + tx := store.MigrationOwnershipPartitionTransition{OperationID: "kv/frontier/split", Operation: store.MigrationOwnershipPartitionSplit, + RootManifestHash: bytes.Repeat([]byte{1}, sha256.Size), Generation: legacy.Generation, LocalGroupID: legacy.LocalGroupID, + CatalogVersion: version, ConfigurationIndex: legacy.ConfigurationIndex, Before: []store.MigrationOwnershipPartitionReference{before}} + for _, child := range children { + encoded, encodeErr := distribution.EncodeRouteDescriptorForCatalogWrite(child, true) + require.NoError(t, encodeErr) + hash := sha256.Sum256(encoded) + p := store.MigrationOwnershipPartition{Generation: legacy.Generation, LocalGroupID: legacy.LocalGroupID, OwnerGroupID: legacy.OwnerGroupID, + CatalogVersion: version, ConfigurationIndex: legacy.ConfigurationIndex, RouteID: child.RouteID, ParentRouteID: child.ParentRouteID, + RouteStart: child.Start, RouteEnd: child.End, DescriptorHash: hash[:], Role: legacy.Role, Staged: legacy.Staged, + MinWriteTSExclusive: legacy.MinWriteTSExclusive, WriteFloorKnown: true, MigrationJobID: legacy.JobID, + SourceTrackerVersion: legacy.SourceTrackerVersion, SourceAppliedIndex: legacy.SourceAppliedIndex} + for _, receipt := range legacy.Receipts { + p.Origins = append(p.Origins, store.MigrationOwnershipOperationReference{JobID: legacy.JobID, + Operation: receipt.Operation, CatalogVersion: receipt.CatalogVersion, ConfigurationIndex: receipt.ConfigurationIndex, + AppliedIndex: receipt.AppliedIndex, DescriptorHash: receipt.DescriptorHash, Staged: receipt.Staged}) + } + tx.After = append(tx.After, p) + } + for i := range tx.After { + id, idErr := store.MigrationOwnershipPartitionIDFor(tx, i) + require.NoError(t, idErr) + tx.After[i].PartitionID = id + } + writer, ok := f.store.(store.MigrationOwnershipPartitionStore) + require.True(t, ok) + require.NoError(t, writer.ApplyMigrationOwnershipPartition(t.Context(), tx, 12)) + reader, ok := f.store.(store.MigrationServingAdmissionReader) + require.True(t, ok) + view, err := reader.MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Empty(t, view.Ownership, "adopted parent is history, not concurrent serving authority") + require.Len(t, view.Partitions.Partitions, 2) + return view +} + +func TestMigrationFrontierAdmissionSameJobChildrenKeepIndependentAuthority(t *testing.T) { + f, engine, _ := frontierAdmissionSplitFixture(t) + request := &pb.Request{Ts: 200, ObservedRouteVersion: EncodeObservedRouteVersion(engine.Version()), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("left")}, + {Op: pb.Op_PUT, Key: []byte("y"), Value: []byte("right")}}} + require.NoError(t, CheckMigrationRequest(t.Context(), f.store, 2, request)) + f.SetApplyIndex(13) + require.NoError(t, applyFSMRequest(t, f, request)) + shards := NewShardStore(engine, map[uint64]*ShardGroup{2: {Store: f.store}}) + for _, key := range []string{"n", "y"} { + value, err := shards.GetAt(t.Context(), []byte(key), 200) + require.NoError(t, err) + require.Equal(t, map[string]string{"n": "left", "y": "right"}[key], string(value)) + } + rows, err := shards.ScanAt(t.Context(), []byte("m"), []byte("z"), 10, 200) + require.NoError(t, err) + require.Len(t, rows, 2) + request.Ts = 100 + require.ErrorIs(t, applyFSMRequest(t, f, request), ErrRouteWriteTimestampTooLow) + request.Ts, request.ObservedRouteVersion = 201, EncodeObservedRouteVersion(engine.Version()-1) + require.ErrorIs(t, CheckMigrationRequest(t.Context(), f.store, 2, request), ErrRouteCutoverPending) + require.ErrorIs(t, applyFSMRequest(t, f, request), ErrRouteCutoverPending) +} + +func TestMigrationFrontierAdmissionRequiresCompleteUnion(t *testing.T) { + f, engine, view := frontierAdmissionSplitFixture(t) + proofs, err := migrationAdmissionRangeProofs(view, migrationApplyRange{[]byte("m"), []byte("z")}) + require.NoError(t, err) + require.Len(t, proofs, 2) + history, err := migrationAdmissionHistory(proofs, engine.Version()) + require.NoError(t, err) + snapshot, ok := history.Current() + require.True(t, ok) + for _, key := range []string{"n", "y"} { + route, found := snapshot.RouteOf([]byte(key)) + require.True(t, found, "same-job children must not collapse during history construction") + require.EqualValues(t, 100, route.MinWriteTSExclusive) + } + view.Partitions.Partitions = view.Partitions.Partitions[:1] + _, err = migrationAdmissionRangeProofs(view, migrationApplyRange{[]byte("m"), []byte("z")}) + require.ErrorIs(t, err, ErrRouteCutoverPending, "historical sibling output cannot fill an inactive frontier hole") + _, err = migrationAdmissionOwnershipProofs(view, []migrationApplyRange{{[]byte("n"), []byte("o")}, {[]byte("y"), []byte("z")}}, 2, engine.Version()) + require.ErrorIs(t, err, ErrRouteCutoverPending, "one covered mutation cannot authorize an uncovered mutation") + _, err = migrationAdmissionOwnershipProofs(view, []migrationApplyRange{{[]byte("n"), []byte("o")}}, 3, engine.Version()) + require.ErrorIs(t, err, ErrRouteCutoverPending) + shards := NewShardStore(engine, map[uint64]*ShardGroup{2: {Store: f.store}}) + _, err = shards.GetAtWithReadFence(t.Context(), []byte("n"), 200, 2, engine.Version()-1) + require.ErrorIs(t, err, ErrRouteCutoverPending, "a newer watcher does not upgrade the caller's stale fence") +} + +func TestMigrationFrontierAdmissionSourceAndStagedBoundaries(t *testing.T) { + f, engine, _ := frontierAdmissionSplitFixture(t) + shards := NewShardStore(engine, map[uint64]*ShardGroup{2: {Store: f.store}}) + applyComposed1Snapshot(t, engine, engine.Version()+1, []distribution.RouteDescriptor{ + {RouteID: 4, Start: []byte("m"), End: []byte("t"), GroupID: 2, MinWriteTSExclusive: 99}, + {RouteID: 5, Start: []byte("t"), End: []byte("z"), GroupID: 2, MinWriteTSExclusive: 100, + MigrationJobID: 99, StagedVisibilityActive: true}, + }) + for _, key := range []string{"n", "y"} { + _, err := shards.GetAt(t.Context(), []byte(key), 200) + require.ErrorIs(t, err, ErrRouteCutoverPending, "floor and lineage cannot be borrowed from a different boundary") + } + writer, ok := f.store.(store.MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), store.TargetStagedReadinessState{ + JobID: 7, MigrationJobID: 7, Armed: true, TrackWrites: true, SourceWriteFence: true, SourceReadFence: true, + RouteStart: []byte("m"), RouteEnd: []byte("z"), ExpectedCutoverVersion: 1, MinWriteTSExclusive: 100, RetentionPinTS: 1, + }, 14)) + request := &pb.Request{Ts: 200, ObservedRouteVersion: EncodeObservedRouteVersion(2), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("fenced")}}} + require.ErrorIs(t, CheckMigrationRequest(t.Context(), f.store, 2, request), ErrRouteCutoverPending) + f.SetApplyIndex(15) + require.ErrorIs(t, applyFSMRequest(t, f, request), ErrRouteCutoverPending) +} + +func TestMigrationFrontierAdmissionPartialRefreshDoesNotUpgradeSibling(t *testing.T) { + f, engine, view := frontierAdmissionSplitFixture(t) + left := view.Partitions.Partitions[0] + ref, err := store.MigrationOwnershipPartitionReferenceFor(left) + require.NoError(t, err) + left.CatalogVersion++ + left.ReceiptID, left.AppliedIndex = nil, 0 + refresh := store.MigrationOwnershipPartitionTransition{OperationID: "kv/frontier/refresh-left", Operation: store.MigrationOwnershipPartitionRefresh, + RootManifestHash: bytes.Repeat([]byte{2}, sha256.Size), Generation: left.Generation, LocalGroupID: 2, + CatalogVersion: left.CatalogVersion, ConfigurationIndex: 9, Before: []store.MigrationOwnershipPartitionReference{ref}, + After: []store.MigrationOwnershipPartition{left}} + writer, ok := f.store.(store.MigrationOwnershipPartitionStore) + require.True(t, ok) + require.NoError(t, writer.ApplyMigrationOwnershipPartition(t.Context(), refresh, 14)) + request := &pb.Request{Ts: 200, ObservedRouteVersion: EncodeObservedRouteVersion(left.CatalogVersion), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("left")}}} + require.NoError(t, CheckMigrationRequest(t.Context(), f.store, 2, request)) + f.SetApplyIndex(15) + require.NoError(t, applyFSMRequest(t, f, request), "a stale watcher cannot override durable child evidence") + request.Mutations = append(request.Mutations, &pb.Mutation{Op: pb.Op_PUT, Key: []byte("y"), Value: []byte("not-refreshed")}) + require.ErrorIs(t, CheckMigrationRequest(t.Context(), f.store, 2, request), ErrRouteCutoverPending) + require.ErrorIs(t, applyFSMRequest(t, f, request), ErrRouteCutoverPending) + request.ObservedRouteVersion = EncodeObservedRouteVersion(engine.Version()) + require.ErrorIs(t, CheckMigrationRequest(t.Context(), f.store, 2, request), ErrRouteCutoverPending, "old active attestation is now history only") + _, err = f.store.GetAt(t.Context(), []byte("y"), 200) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +func TestMigrationFrontierAdmissionNonOwnerChildrenNeverGrant(t *testing.T) { + st := store.NewMVCCStore() + retireServingSource(t, st) + engine := staleServingEngine(t, 11) + f, ok := NewKvFSMWithHLC(st, nil, WithRouteHistory(WrapDistributionEngine(engine), 1)).(*kvFSM) + require.True(t, ok) + require.NoError(t, f.RecordAppliedConfiguration(3)) + reader, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + full, err := reader.MigrationServingState(t.Context()) + require.NoError(t, err) + view := frontierAdmissionSplitLegacy(t, f, full.Ownership[0], []distribution.RouteDescriptor{ + {RouteID: 4, ParentRouteID: 3, Start: []byte("m"), End: []byte("t"), GroupID: 2, MinWriteTSExclusive: 100}, + {RouteID: 5, ParentRouteID: 3, Start: []byte("t"), End: []byte("z"), GroupID: 2, MinWriteTSExclusive: 100}, + }, 11) + for _, p := range view.Partitions.Partitions { + require.Equal(t, store.MigrationNonOwner, p.Role) + require.True(t, store.MigrationOwnershipPartitionEvidenceValid(view.Partitions, p)) + } + require.NoError(t, st.PutAt(t.Context(), []byte("n"), []byte("old-source"), 90, 0)) + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: {Store: st}}) + _, err = shards.GetAt(t.Context(), []byte("n"), 200) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, err = shards.ScanAt(t.Context(), []byte("m"), []byte("z"), 10, 200) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, _, err = shards.VersionExistsAtOrBeforeGroupWithReadFence(t.Context(), []byte("n"), 1, 200, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) + request := &pb.Request{Ts: 200, ObservedRouteVersion: EncodeObservedRouteVersion(11), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("y"), Value: []byte("denied")}}} + require.ErrorIs(t, CheckMigrationRequest(t.Context(), st, 1, request), ErrRouteCutoverPending) + f.SetApplyIndex(13) + require.ErrorIs(t, applyFSMRequest(t, f, request), ErrRouteCutoverPending) +} + +func TestMigrationFrontierAdmissionStaleFenceRejectedBeforeProxy(t *testing.T) { + f, engine, _ := frontierAdmissionSplitFixture(t) + shards := NewShardStore(engine, map[uint64]*ShardGroup{ + 2: {Store: f.store, Engine: &stubFollowerEngine{leaderAddr: "127.0.0.1:1"}}, + }) + t.Cleanup(func() { require.NoError(t, shards.Close()) }) + _, err := shards.GetAtWithReadFence(t.Context(), []byte("n"), 200, 0, engine.Version()-1) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, _, err = shards.LatestCommitTSWithReadFence(t.Context(), []byte("n"), engine.Version()-1) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, _, err = shards.LatestCommitTSGroupWithReadFence(t.Context(), []byte("n"), 2, engine.Version()-1) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, _, err = shards.VersionExistsAtOrBeforeGroupWithReadFence(t.Context(), []byte("n"), 2, 200, engine.Version()-1) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, _, err = shards.VersionsExistAtOrBeforeGroupWithReadFence(t.Context(), [][]byte{[]byte("n")}, 2, 200, engine.Version()-1) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, err = shards.ScanAtWithReadFence(t.Context(), []byte("m"), []byte("t"), 10, 200, false, 0, engine.Version()-1, nil, nil) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestMigrationFrontierAdmissionUnknownLegacyCannotBorrowNewerPartition(t *testing.T) { + _, engine, view := frontierAdmissionSplitFixture(t) + view.Ownership = []store.MigrationOwnershipState{{JobID: 7, LocalGroupID: 2, OwnerGroupID: 2, + CatalogVersion: engine.Version() - 1, ConfigurationIndex: 9, AppliedIndex: 11, Role: store.MigrationOwner, + RouteStart: []byte("m"), RouteEnd: []byte("t"), MinWriteTSExclusive: 100}} + _, err := migrationAdmissionOwnershipProofs(view, []migrationApplyRange{{[]byte("n"), []byte("o")}}, 2, engine.Version()) + require.ErrorIs(t, err, ErrRouteCutoverPending, "receipt-less legacy remains an interval barrier even beside newer valid frontier evidence") + _, err = migrationAdmissionOwnershipProofs(view, []migrationApplyRange{{[]byte("y"), []byte("z")}}, 2, engine.Version()) + require.NoError(t, err, "unknown legacy must not block an unaffected sibling") +} diff --git a/kv/migration_frontier_selection_test.go b/kv/migration_frontier_selection_test.go new file mode 100644 index 000000000..715cd77b1 --- /dev/null +++ b/kv/migration_frontier_selection_test.go @@ -0,0 +1,51 @@ +package kv + +import ( + "testing" + + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestMigrationFrontierSelectionMatchesConservativeIntervals(t *testing.T) { + f, _, original := frontierAdmissionSplitFixture(t) + reader, ok := f.store.(store.MigrationOwnershipStore) + require.True(t, ok) + history, err := reader.MigrationServingState(t.Context()) + require.NoError(t, err) + for _, tc := range []struct { + name string + start, end string + truncate bool + overlap bool + }{ + {name: "point", start: "n", end: "o"}, + {name: "boundary", start: "t", end: "u"}, + {name: "full union", start: "m", end: "z"}, + {name: "uncovered prefix", start: "l", end: "z"}, + {name: "uncovered suffix", start: "m", end: "zz"}, + {name: "unbounded suffix", start: "m"}, + {name: "unaffected legacy", start: "a", end: "b"}, + {name: "missing sibling", start: "m", end: "z", truncate: true}, + {name: "overlapping old ledger", start: "m", end: "z", overlap: true}, + } { + t.Run(tc.name, func(t *testing.T) { + view := original + if tc.truncate { + view.Partitions.Partitions = view.Partitions.Partitions[:1] + } + if tc.overlap { + view.Ownership = history.Ownership + } + r := migrationApplyRange{[]byte(tc.start), []byte(tc.end)} + want, wantErr := migrationAdmissionOverlappingRangeProofs(view, migrationAdmissionProofs(view), r) + got, gotErr := migrationAdmissionRangeProofs(view, r) + if wantErr != nil { + require.ErrorIs(t, gotErr, ErrRouteCutoverPending) + return + } + require.NoError(t, gotErr) + require.Equal(t, want, got) + }) + } +} diff --git a/kv/migration_namespace_activation.go b/kv/migration_namespace_activation.go new file mode 100644 index 000000000..cdecfbfd9 --- /dev/null +++ b/kv/migration_namespace_activation.go @@ -0,0 +1,118 @@ +package kv + +import ( + "context" + "slices" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +// CheckMigrationNamespacePrejoin runs BEFORE a new member receives snapshots +// or joins as a learner. Static installed support alone is insufficient after +// activation: MODE8 must also durably authorize this member at the current cfg. +// The ordered configuration hook repeats the inventory check against races. +func CheckMigrationNamespacePrejoin(ctx context.Context, st store.MVCCStore, configurationIndex, memberID uint64, installedVersion uint32) error { + if st == nil { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + state, err := readNamespaceConfiguration(ctx, st) + if err != nil { + return err + } + if state.Version == 0 { + return nil + } + if configurationIndex != state.Index || memberID == 0 || installedVersion < state.Version || + (!slices.Contains(state.Members, memberID) && !slices.Contains(state.Joining, memberID)) { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + return nil +} + +// MigrationNamespaceInstalledVersion is deliberately independent of activation +// and of the Draft gate. Wrappers must forward the ordered configuration hooks. +func MigrationNamespaceInstalledVersion(st store.MVCCStore, fsm raftengine.StateMachine) uint32 { + if st == nil || fsm == nil { + return 0 + } + if _, ok := fsm.(raftengine.NamespaceConfigurationAware); !ok { + return 0 + } + return MigrationNamespaceVersion +} + +// MigrationNamespaceAdmissionVersion reports durable activation, not installed +// support. It is safe for public handlers to fail closed before proposal using +// this API; FSM user admission consults only its ordered durable counterpart. +func MigrationNamespaceAdmissionVersion(ctx context.Context, st store.MVCCStore) (uint32, error) { + if st == nil { + return 0, errors.WithStack(ErrMigrationNamespaceProtocol) + } + state, err := readNamespaceConfiguration(ctx, st) + return state.Version, err +} + +// RequireMigrationNamespaceAdmission applies to every mutating migration RPC, +// including unversioned import/promote/retire/readiness/cleanup calls. Only the +// namespace activation control is exempt, avoiding a bootstrap dependency. +func RequireMigrationNamespaceAdmission(ctx context.Context, st store.MVCCStore) error { + version, err := MigrationNamespaceAdmissionVersion(ctx, st) + if err != nil { + return err + } + if version != MigrationNamespaceVersion { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + return nil +} + +// ActivateMigrationNamespace is an explicit internal control, never an +// automatic response to installed support or a watcher. The controller gathers +// Members from all current apply-members and Joining from prospective members +// before this call. The FSM rechecks the exact configuration and inventory. +func ActivateMigrationNamespace(ctx context.Context, st store.MVCCStore, proposer raftengine.Proposer, command NamespaceAdmissionCommand) error { + if st == nil || proposer == nil { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + index, err := MigrationConfigurationIndex(ctx, st) + if err != nil { + return err + } + if index != command.ConfigurationIndex { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + data, err := MarshalNamespaceAdmissionCommand(command) + if err != nil { + return err + } + if err := proposeNamespaceAdmission(ctx, proposer, data); err != nil { + return err + } + if err := RequireMigrationNamespaceAdmission(ctx, st); err != nil { + return err + } + index, err = MigrationConfigurationIndex(ctx, st) + if err != nil { + return err + } + if index != command.ConfigurationIndex { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + return nil +} + +func proposeNamespaceAdmission(ctx context.Context, proposer raftengine.Proposer, data []byte) error { + result, err := proposer.ProposeAdmin(ctx, data) + if err != nil { + return errors.WithStack(err) + } + if result == nil { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + if err, ok := result.Response.(error); ok { + return errors.WithStack(err) + } + return nil +} diff --git a/kv/migration_namespace_admission.go b/kv/migration_namespace_admission.go new file mode 100644 index 000000000..38b10ba13 --- /dev/null +++ b/kv/migration_namespace_admission.go @@ -0,0 +1,27 @@ +package kv + +import ( + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" +) + +// Validate supplied user mutations before evidence, dedup or resolution can +// short-circuit a request. Typed control opcodes never enter this path. +func verifyRequestMigrationControlKeys(request *pb.Request) error { + mutations := request.GetMutations() + if request.GetIsTxn() { + _, userMutations, err := extractTxnMeta(mutations) + if err != nil { + return err + } + mutations = userMutations + } + for _, mutation := range mutations { + if isTxnInternalKey(mutation.GetKey()) || distribution.IsMigrationOnlyControlKey(mutation.GetKey()) || + (mutation.GetOp() == pb.Op_DEL_PREFIX && distribution.ReservedControlPrefixIntersects(mutation.GetKey())) { + return errors.WithStack(ErrInvalidRequest) + } + } + return nil +} diff --git a/kv/migration_namespace_compat_test.go b/kv/migration_namespace_compat_test.go new file mode 100644 index 000000000..26822cc88 --- /dev/null +++ b/kv/migration_namespace_compat_test.go @@ -0,0 +1,92 @@ +package kv + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +// This test is also run unchanged at 2d1a36ac. Compare the namespace-compat +// lines: they include the verdict and every resulting MVCC version. The +// preexisting wall-clock-derived lock deadline is normalized, and only that. +func TestMigrationNamespaceCompatibilityMatrix(t *testing.T) { + keys := []string{"!migstage|ready|customer", "!migstage|ownership", "!migstage|retired", "!migstage|ack|", "!migs", "!dist|migstage|x", "!migwrite|x", "!migfence|x", "!txn|lock|x", "!txn|meta|migration_configuration_index", "!txn|meta|prepare_evidence/enabled", "!txn|backup|timestamp_floor", "!txn|foo", "_migack", "user"} + for _, phase := range []pb.Phase{pb.Phase_NONE, pb.Phase_PREPARE, pb.Phase_COMMIT, pb.Phase_ABORT} { + for _, key := range keys { + for _, op := range []pb.Op{pb.Op_PUT, pb.Op_DEL, pb.Op_DEL_PREFIX} { + for _, mixed := range []bool{false, true} { + name := fmt.Sprintf("%s/%q/%s/mixed=%v", phase, key, op, mixed) + st := store.NewMVCCStore() + f := &kvFSM{store: st, shardGroupID: 1} + if phase == pb.Phase_COMMIT || phase == pb.Phase_ABORT { + require.NoError(t, st.PutAt(t.Context(), txnLockKey([]byte(key)), encodeTxnLock(txnLock{StartTS: 10, CommitTS: 20, PrimaryKey: []byte(key), IsPrimaryKey: true}), 10, 0)) + require.NoError(t, st.PutAt(t.Context(), txnIntentKey([]byte(key)), encodeTxnIntent(txnIntent{StartTS: 10, Op: txnIntentOpPut, Value: []byte("before-upgrade")}), 10, 0)) + } + request := &pb.Request{IsTxn: true, Phase: phase, Ts: 10, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte(key), CommitTS: 20})}, + {Op: op, Key: []byte(key), Value: []byte("after-upgrade")}, + }} + if mixed { + request.Mutations = append(request.Mutations, &pb.Mutation{Op: pb.Op_PUT, Key: []byte("neighbor"), Value: []byte("value")}) + } + data, err := proto.Marshal(request) + require.NoError(t, err) + namespaceCompatibilityState(t, name, st, f.Apply(append([]byte{raftEncodeSingle}, data...))) + require.NoError(t, st.Close()) + } + } + } + } + for _, reverse := range []bool{false, true} { + st := store.NewMVCCStore() + f := &kvFSM{store: st} + requests := []*pb.Request{ + {Ts: 10, Mutations: []*pb.Mutation{{Key: []byte("neighbor"), Value: []byte("first")}}}, + {Ts: 20, Mutations: []*pb.Mutation{{Op: pb.Op(99), Key: []byte("!migstage|ready|x")}}}, + {Ts: 30, Mutations: []*pb.Mutation{{Key: []byte("neighbor"), Value: []byte("last")}}}, + } + if reverse { + requests[0], requests[2] = requests[2], requests[0] + } + data, err := proto.Marshal(&pb.RaftCommand{Requests: requests}) + require.NoError(t, err) + namespaceCompatibilityState(t, fmt.Sprintf("batch/reverse=%v", reverse), st, f.Apply(append([]byte{raftEncodeBatch}, data...))) + require.NoError(t, st.Close()) + } +} + +func namespaceCompatibilityState(t *testing.T, name string, st store.MVCCStore, response any) { + t.Helper() + verdict := fmt.Sprint(response) + if batch, ok := response.(*fsmApplyResponse); ok { + verdict = fmt.Sprint(batch.results) + } + exported, err := st.ExportVersions(t.Context(), store.ExportVersionsOptions{MaxCommitTSInclusive: ^uint64(0), MaxVersions: 1000, MaxBytes: 1 << 20, MaxScannedBytes: 1 << 20}) + require.NoError(t, err) + require.True(t, exported.Done) + for i := range exported.Versions { + version := &exported.Versions[i] + if !version.Tombstone && bytesHasTxnLockPrefix(version.Key) { + lock, err := decodeTxnLock(version.Value) + require.NoError(t, err) + lock.TTLExpireAt = 0 + version.Value = encodeTxnLock(lock) + } + } + raw, err := json.Marshal(struct { + LastCommitTS uint64 + Versions []store.MVCCVersion + }{st.LastCommitTS(), exported.Versions}) + require.NoError(t, err) + fmt.Printf("namespace-compat %s verdict=%q state=%x\n", name, verdict, sha256.Sum256(raw)) +} + +func bytesHasTxnLockPrefix(key []byte) bool { + return len(key) >= len(txnLockPrefix) && string(key[:len(txnLockPrefix)]) == txnLockPrefix +} diff --git a/kv/migration_namespace_protocol.go b/kv/migration_namespace_protocol.go new file mode 100644 index 000000000..00f0f8a90 --- /dev/null +++ b/kv/migration_namespace_protocol.go @@ -0,0 +1,284 @@ +package kv + +import ( + "bytes" + "context" + "encoding/json" + "slices" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +// MigrationNamespaceVersion is installed support, not permission to change +// committed-entry admission. Only the configuration-bound control activates it. +const MigrationNamespaceVersion = store.MigrationNamespaceVersion + +var ErrMigrationNamespaceProtocol = errors.New("namespace admission protocol is not proven for this configuration") + +// NamespaceMemberCapability is a trusted peer's acknowledgement of support for +// the protocol at ConfigurationIndex. Include learners, not just voters. +type NamespaceMemberCapability struct { + MemberID uint64 + Version uint32 + ConfigurationIndex uint64 +} + +// NamespaceAdmissionCommand must only be proposed after probing every apply +// member. Joining members must acknowledge support BEFORE AddLearner/AddVoter, +// since catch-up already executes the activated log and installs its snapshots. +// Neither this control nor installed support opens the migration Draft gate. +type NamespaceAdmissionCommand struct { + Version uint32 + ConfigurationIndex uint64 + ApplyMembers []uint64 + Members []NamespaceMemberCapability + Joining []NamespaceMemberCapability +} + +// Reuse the existing exact peer-local MVCC key. Its versions already survive +// native snapshots and are all stripped by external/rebased restore. Until +// activation it retains the legacy eight-byte value and snapshot compatibility. +type namespaceConfiguration = store.MigrationNamespaceConfiguration + +func readNamespaceConfiguration(ctx context.Context, st store.MVCCStore) (namespaceConfiguration, error) { + value, err := st.GetAt(ctx, migrationConfigurationIndexKey, ^uint64(0)) + if errors.Is(err, store.ErrKeyNotFound) { + return namespaceConfiguration{}, nil + } + if err != nil { + return namespaceConfiguration{}, errors.WithStack(err) + } + state, err := store.DecodeMigrationNamespaceConfiguration(value) + return state, errors.WithStack(errors.Mark(err, ErrMigrationNamespaceProtocol)) +} + +func validNamespaceMembers(ids []uint64) bool { + return len(ids) > 0 && ids[0] != 0 && slices.IsSorted(ids) && len(slices.Compact(slices.Clone(ids))) == len(ids) +} + +func validNamespaceJoining(ids, members []uint64) bool { + if len(ids) == 0 { + return true + } + if !validNamespaceMembers(ids) { + return false + } + for _, id := range ids { + if slices.Contains(members, id) { + return false + } + } + return true +} + +func (f *kvFSM) persistNamespaceConfiguration(ctx context.Context, state namespaceConfiguration, applyIndex uint64) error { + value, err := store.EncodeMigrationNamespaceConfiguration(state) + if err != nil { + return errors.WithStack(err) + } + ts := max(f.store.LastCommitTS(), 1) + if err := f.store.ApplyMutationsRaftAt(ctx, []*store.KVPairMutation{{Op: store.OpTypePut, Key: migrationConfigurationIndexKey, Value: value}}, nil, ts, ts, applyIndex); err != nil { + return errors.WithStack(err) + } + f.migrationConfigurationIndex.Store(state.Index) + f.migrationNamespaceVersion.Store(state.Version) + return nil +} + +// RecordAppliedNamespaceConfiguration is called on the serial Raft apply loop, +// never by a local watcher. All current/joint voters and learners are required. +func (f *kvFSM) RecordAppliedNamespaceConfiguration(index uint64, members []uint64) error { + if f.backupFloorLoadErr != nil { + return errors.WithStack(f.backupFloorLoadErr) + } + members = slices.Clone(members) + slices.Sort(members) + if index == 0 || !validNamespaceMembers(members) { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + state, err := readNamespaceConfiguration(context.Background(), f.store) + if err != nil || index < state.Index { + return err + } + state, err = namespaceConfigurationForMembers(state, index, members) + if err != nil { + return err + } + if index == state.Index { + // Cold-start replays ConfChanges below the store's data frontier. Only + // recover the ordered inventory; never rewrite a smaller applied index. + f.migrationNamespaceMembers = members + return nil + } + state.Index, state.Members = index, members + if err := f.persistNamespaceConfiguration(context.Background(), state, index); err != nil { + return err + } + f.migrationNamespaceMembers = members + return nil +} + +func namespaceConfigurationForMembers(state namespaceConfiguration, index uint64, members []uint64) (namespaceConfiguration, error) { + if state.Version == 0 { + return state, nil + } + for _, id := range members { + if !slices.Contains(state.Members, id) && !slices.Contains(state.Joining, id) { + return state, errors.WithStack(ErrMigrationNamespaceProtocol) + } + } + if index == state.Index && !slices.Equal(state.Members, members) { + return state, errors.WithStack(ErrMigrationNamespaceProtocol) + } + if index != state.Index { + state.Joining = nil // Prejoin acknowledgements cannot cross a ConfChange. + } + return state, nil +} + +// RecoverNamespaceConfiguration binds an inactive replica to the actual +// snapshot ConfState. A store ahead of that snapshot must replay its ConfChange +// before activation; never label the snapshot's data index a configuration. +func (f *kvFSM) RecoverNamespaceConfiguration(snapshotIndex uint64, members []uint64) error { + state, err := readNamespaceConfiguration(context.Background(), f.store) + if err != nil || state.Index == 0 || state.Index > snapshotIndex { + return err + } + members = slices.Clone(members) + slices.Sort(members) + if !validNamespaceMembers(members) || (state.Version != 0 && !slices.Equal(state.Members, members)) { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + f.migrationNamespaceMembers = members + return nil +} + +func MarshalNamespaceAdmissionCommand(command NamespaceAdmissionCommand) ([]byte, error) { + request, err := NewNamespaceAdmissionRequest(1, command) + if err != nil { + return nil, err + } + return MarshalMigrationCleanupCommand(request) +} + +func NewNamespaceAdmissionRequest(jobID uint64, command NamespaceAdmissionCommand) (*pb.CleanupMigrationRequest, error) { + if err := validateNamespaceAdmissionCommand(command); err != nil { + return nil, err + } + data, err := json.Marshal(command) + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.CleanupMigrationRequest{ + JobId: jobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION, + ExpectedConfigurationIndex: command.ConfigurationIndex, RequireConfigurationIndex: true, + NamespaceAdmission: data, + }, nil +} + +func ValidateNamespaceAdmissionRequest(request *pb.CleanupMigrationRequest) error { + if request == nil || request.GetMode() != pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION || !request.GetRequireConfigurationIndex() { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + var command NamespaceAdmissionCommand + if err := json.Unmarshal(request.GetNamespaceAdmission(), &command); err != nil { + return errors.WithStack(err) + } + if command.ConfigurationIndex != request.GetExpectedConfigurationIndex() { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + return validateNamespaceAdmissionCommand(command) +} + +func validateNamespaceAdmissionCommand(command NamespaceAdmissionCommand) error { + if command.Version != MigrationNamespaceVersion || command.ConfigurationIndex == 0 || !validNamespaceMembers(command.ApplyMembers) { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + proved, err := namespaceCapabilityIDs(command.Members, command.ConfigurationIndex) + if err != nil || !slices.Equal(proved, command.ApplyMembers) { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + joining, err := namespaceCapabilityIDs(command.Joining, command.ConfigurationIndex) + if err != nil || !validNamespaceJoining(joining, command.ApplyMembers) { + return errors.WithStack(ErrMigrationNamespaceProtocol) + } + return nil +} + +func (f *kvFSM) applyNamespaceAdmission(ctx context.Context, data []byte) any { + var command NamespaceAdmissionCommand + if err := json.Unmarshal(data, &command); err != nil { + return haltErr(errors.Wrap(err, "decode namespace admission control")) + } + state, err := readNamespaceConfiguration(ctx, f.store) + if err != nil { + return haltErr(err) + } + members := f.migrationNamespaceMembers + if state.Version != 0 { + members = state.Members + } + joining, err := validateNamespaceAdmissionState(command, state, members, f.pendingApplyIdx) + if err != nil { + return err + } + // Concurrent controllers may refresh activation while membership admission + // is between its prejoin control and ConfChange. Never revoke an already + // committed permission inside the same configuration. + joining = append(slices.Clone(state.Joining), joining...) + slices.Sort(joining) + state.Version, state.Members, state.Joining = command.Version, members, slices.Compact(joining) + if state.ActivatedAt == 0 { + state.ActivatedAt = state.Index + } + if err := f.persistNamespaceConfiguration(ctx, state, f.pendingApplyIdx); err != nil { + return haltErr(errors.Wrap(err, "persist namespace admission control")) + } + return nil +} + +func validateNamespaceAdmissionState(command NamespaceAdmissionCommand, state namespaceConfiguration, members []uint64, applyIndex uint64) ([]uint64, error) { + if validateNamespaceAdmissionCommand(command) != nil || command.ConfigurationIndex != state.Index || !slices.Equal(command.ApplyMembers, members) || !validNamespaceMembers(members) { + return nil, errors.WithStack(ErrMigrationNamespaceProtocol) + } + proved, err := namespaceCapabilityIDs(command.Members, command.ConfigurationIndex) + if err != nil || !slices.Equal(proved, members) { + return nil, errors.WithStack(ErrMigrationNamespaceProtocol) + } + joining, err := namespaceCapabilityIDs(command.Joining, command.ConfigurationIndex) + if err != nil || !validNamespaceJoining(joining, members) { + return nil, errors.WithStack(ErrMigrationNamespaceProtocol) + } + if applyIndex <= state.Index { + return nil, errors.WithStack(ErrMigrationNamespaceProtocol) + } + return joining, nil +} + +func namespaceCapabilityIDs(capabilities []NamespaceMemberCapability, index uint64) ([]uint64, error) { + ids := make([]uint64, 0, len(capabilities)) + for _, capability := range capabilities { + if capability.Version < MigrationNamespaceVersion || capability.ConfigurationIndex != index { + return nil, errors.WithStack(ErrMigrationNamespaceProtocol) + } + ids = append(ids, capability.MemberID) + } + slices.Sort(ids) + if len(ids) > 0 && !validNamespaceMembers(ids) { + return nil, errors.WithStack(ErrMigrationNamespaceProtocol) + } + return ids, nil +} + +func (f *kvFSM) isMigrationOnlyControlKeyForApply(key []byte) bool { + if f.migrationNamespaceVersion.Load() != 0 { + return distribution.IsMigrationOnlyControlKey(key) + } + // Exact 2d1a36ac classifier. In particular !migstage| was an ordinary + // transaction key, even though raw writes already refused that namespace. + return bytes.HasPrefix(key, []byte("!dist|migstage|")) || bytes.HasPrefix(key, []byte("!migwrite|")) || bytes.HasPrefix(key, []byte("!migfence|")) +} diff --git a/kv/migration_namespace_protocol_test.go b/kv/migration_namespace_protocol_test.go new file mode 100644 index 000000000..15e5a4f33 --- /dev/null +++ b/kv/migration_namespace_protocol_test.go @@ -0,0 +1,217 @@ +package kv + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func namespaceCommand(index uint64, ids ...uint64) NamespaceAdmissionCommand { + command := NamespaceAdmissionCommand{Version: MigrationNamespaceVersion, ConfigurationIndex: index, ApplyMembers: ids} + for _, id := range ids { + command.Members = append(command.Members, NamespaceMemberCapability{MemberID: id, Version: MigrationNamespaceVersion, ConfigurationIndex: index}) + } + return command +} + +func applyNamespaceCommand(t *testing.T, f *kvFSM, index uint64, command NamespaceAdmissionCommand) any { + t.Helper() + // Bypass the preproposal verifier to exercise the actual FSM validation. + data, err := json.Marshal(command) + require.NoError(t, err) + encoded, err := MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{ + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION, RequireConfigurationIndex: true, + ExpectedConfigurationIndex: command.ConfigurationIndex, NamespaceAdmission: data}) + require.NoError(t, err) + f.SetApplyIndex(index) + return f.Apply(encoded) +} + +func TestMigrationNamespaceActivationRequiresEveryApplyMember(t *testing.T) { + for _, kind := range []string{"missing-learner", "old-member", "wrong-member-config", "extra-member", "duplicate", "wrong-roster", "stale-config", "zero-version", "valid"} { + t.Run(kind, func(t *testing.T) { + st := store.NewMVCCStore() + f := &kvFSM{store: st} + require.NoError(t, f.RecordAppliedNamespaceConfiguration(5, []uint64{1, 2, 3})) + before, err := st.GetAt(t.Context(), migrationConfigurationIndexKey, ^uint64(0)) + require.NoError(t, err) + require.Len(t, before, 8, "inactive records keep the old snapshot layout") + command := namespaceActivationTestCommand(t, f, kind) + response := applyNamespaceCommand(t, f, 7, command) + if kind == "valid" { + require.Nil(t, response) + require.NoError(t, RequireMigrationNamespaceAdmission(t.Context(), st)) + return + } + err, ok := response.(error) + require.True(t, ok) + require.ErrorIs(t, err, ErrMigrationNamespaceProtocol) + require.Zero(t, f.migrationNamespaceVersion.Load()) + require.ErrorIs(t, RequireMigrationNamespaceAdmission(t.Context(), st), ErrMigrationNamespaceProtocol) + }) + } +} + +func namespaceActivationTestCommand(t *testing.T, f *kvFSM, kind string) NamespaceAdmissionCommand { + t.Helper() + command := namespaceCommand(5, 1, 2, 3) + switch kind { + case "missing-learner": + command.Members = command.Members[:2] + case "old-member": + command.Members[2].Version = 0 + case "wrong-member-config": + command.Members[2].ConfigurationIndex = 4 + case "extra-member": + command.Members = append(command.Members, NamespaceMemberCapability{4, 1, 5}) + case "duplicate": + command.Members[2] = command.Members[1] + case "wrong-roster": + command = namespaceCommand(5, 1, 2) + case "stale-config": + require.NoError(t, f.RecordAppliedNamespaceConfiguration(6, []uint64{1, 2, 3})) + case "zero-version": + command.Version = 0 + } + return command +} + +func TestMigrationNamespacePrejoinAndConfigurationMonotonicity(t *testing.T) { + st := store.NewMVCCStore() + f := &kvFSM{store: st} + require.NoError(t, f.RecordAppliedNamespaceConfiguration(5, []uint64{1, 2})) + command := namespaceCommand(5, 1, 2) + command.Joining = []NamespaceMemberCapability{{3, MigrationNamespaceVersion, 5}} + require.Nil(t, applyNamespaceCommand(t, f, 6, command)) + require.NoError(t, f.RecordAppliedNamespaceConfiguration(7, []uint64{1, 2, 3})) + require.NoError(t, RequireMigrationNamespaceAdmission(t.Context(), st)) + require.ErrorIs(t, f.RecordAppliedNamespaceConfiguration(8, []uint64{1, 2, 3, 4}), ErrMigrationNamespaceProtocol) + require.Equal(t, uint64(7), f.AppliedConfigurationIndex()) + require.ErrorIs(t, f.RecordAppliedConfiguration(8), ErrMigrationNamespaceProtocol) + // Removing a member never disables admission, and invalidates old prejoins. + require.NoError(t, f.RecordAppliedNamespaceConfiguration(8, []uint64{1, 2})) + require.ErrorIs(t, f.RecordAppliedNamespaceConfiguration(9, []uint64{1, 2, 3}), ErrMigrationNamespaceProtocol) + require.NoError(t, RequireMigrationNamespaceAdmission(t.Context(), st)) + require.NoError(t, f.RecordAppliedNamespaceConfiguration(5, []uint64{99}), "old WAL configuration cannot regress state") + require.Equal(t, uint64(8), f.AppliedConfigurationIndex()) +} + +func TestMigrationNamespaceActivationSnapshotRestartAndExternalRebase(t *testing.T) { + for _, backend := range []string{"mvcc", "pebble"} { + t.Run(backend, func(t *testing.T) { + source := store.NewMVCCStore() + if backend == "pebble" { + var err error + source, err = store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + } + defer source.Close() + f := &kvFSM{store: source} + activateMigrationNamespace(t, f) + snapshot, err := source.Snapshot() + require.NoError(t, err) + defer snapshot.Close() + var raw bytes.Buffer + _, err = snapshot.WriteTo(&raw) + require.NoError(t, err) + for _, external := range []bool{false, true} { + dir := t.TempDir() + target, err := store.NewPebbleStore(dir) + require.NoError(t, err) + if external { + restorer, ok := target.(store.ExternalSnapshotRestorer) + require.True(t, ok) + require.NoError(t, restorer.RestoreExternal(t.Context(), bytes.NewReader(raw.Bytes()))) + } else { + require.NoError(t, target.Restore(bytes.NewReader(raw.Bytes()))) + } + require.NoError(t, target.Close()) + target, err = store.NewPebbleStore(dir) + require.NoError(t, err) + restored, ok := NewKvFSMWithHLC(target, nil).(*kvFSM) + require.True(t, ok) + if external { + require.Zero(t, restored.migrationNamespaceVersion.Load()) + // The destination may reuse exactly the source configuration index. + require.NoError(t, restored.RecordAppliedNamespaceConfiguration(1, []uint64{77})) + require.ErrorIs(t, RequireMigrationNamespaceAdmission(t.Context(), target), ErrMigrationNamespaceProtocol) + } else { + require.Equal(t, MigrationNamespaceVersion, restored.migrationNamespaceVersion.Load()) + require.NoError(t, RequireMigrationNamespaceAdmission(t.Context(), target)) + require.ErrorIs(t, restored.RecordAppliedNamespaceConfiguration(3, []uint64{77}), ErrMigrationNamespaceProtocol) + } + require.NoError(t, target.Close()) + } + // MVCC external restore strips all versions as well, not just Pebble. + if backend != "mvcc" { + return + } + target := store.NewMVCCStore() + defer target.Close() + restorer, ok := target.(store.ExternalSnapshotRestorer) + require.True(t, ok) + require.NoError(t, restorer.RestoreExternal(t.Context(), bytes.NewReader(raw.Bytes()))) + require.ErrorIs(t, RequireMigrationNamespaceAdmission(t.Context(), target), ErrMigrationNamespaceProtocol) + }) + } +} + +type namespaceFailStore struct { + store.MVCCStore + fail bool +} + +func (s *namespaceFailStore) ApplyMutationsRaftAt(ctx context.Context, muts []*store.KVPairMutation, reads [][]byte, start, commit, index uint64) error { + if s.fail { + return errors.New("namespace durability failure") + } + return s.MVCCStore.ApplyMutationsRaftAt(ctx, muts, reads, start, commit, index) +} + +func TestMigrationNamespaceDurabilityFailureHaltsWithoutActivation(t *testing.T) { + st := &namespaceFailStore{MVCCStore: store.NewMVCCStore()} + f := &kvFSM{store: st} + require.NoError(t, f.RecordAppliedNamespaceConfiguration(5, []uint64{1})) + st.fail = true + response := applyNamespaceCommand(t, f, 6, namespaceCommand(5, 1)) + halted, ok := response.(interface{ HaltApply() error }) + require.True(t, ok) + require.ErrorContains(t, halted.HaltApply(), "namespace durability failure") + require.Zero(t, f.migrationNamespaceVersion.Load()) + require.ErrorIs(t, RequireMigrationNamespaceAdmission(t.Context(), st), ErrMigrationNamespaceProtocol) +} + +type namespaceProposer struct { + f *kvFSM + calls int +} + +func (p *namespaceProposer) Propose(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { + return p.ProposeAdmin(ctx, data) +} +func (p *namespaceProposer) ProposeAdmin(_ context.Context, data []byte) (*raftengine.ProposalResult, error) { + p.calls++ + p.f.SetApplyIndex(6) + return &raftengine.ProposalResult{CommitIndex: 6, Response: p.f.Apply(data)}, nil +} + +func TestMigrationNamespaceActivationHelperRefusesIncompleteNegotiationBeforeProposal(t *testing.T) { + st := store.NewMVCCStore() + f := &kvFSM{store: st} + require.NoError(t, f.RecordAppliedNamespaceConfiguration(5, []uint64{1, 2})) + p := &namespaceProposer{f: f} + command := namespaceCommand(5, 1, 2) + command.Members = command.Members[:1] + require.ErrorIs(t, ActivateMigrationNamespace(t.Context(), st, p, command), ErrMigrationNamespaceProtocol) + require.Zero(t, p.calls) + require.NoError(t, ActivateMigrationNamespace(t.Context(), st, p, namespaceCommand(5, 1, 2))) + require.Equal(t, 1, p.calls) + require.Equal(t, MigrationNamespaceVersion, MigrationNamespaceInstalledVersion(st, f)) +} diff --git a/kv/migration_namespace_upgrade_test.go b/kv/migration_namespace_upgrade_test.go new file mode 100644 index 000000000..5f1e84a40 --- /dev/null +++ b/kv/migration_namespace_upgrade_test.go @@ -0,0 +1,103 @@ +package kv + +import ( + "bytes" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func activateMigrationNamespace(t *testing.T, f *kvFSM) { + t.Helper() + require.NoError(t, f.RecordAppliedNamespaceConfiguration(1, []uint64{1})) + command, err := MarshalNamespaceAdmissionCommand(NamespaceAdmissionCommand{ + Version: MigrationNamespaceVersion, ConfigurationIndex: 1, + ApplyMembers: []uint64{1}, + Members: []NamespaceMemberCapability{{MemberID: 1, Version: MigrationNamespaceVersion, ConfigurationIndex: 1}}, + }) + require.NoError(t, err) + f.SetApplyIndex(2) + require.Nil(t, f.Apply(command)) +} + +func TestMigrationNamespaceOrderedBatchIgnoresDifferentWatchers(t *testing.T) { + for _, rejectedPosition := range []int{0, 1, 2} { + var snapshots [][]byte + for _, watcher := range []RouteHistory{nil, forbiddenApplyWatcher{}} { + st := store.NewMVCCStore() + f := &kvFSM{store: st, routes: watcher, shardGroupID: 1} + activateMigrationNamespace(t, f) + deterministicMigrationOwner(t, st) + requests := []*pb.Request{deterministicRawRequest(10, "n"), deterministicRawRequest(10, "p"), migrationNamespaceRequest(pb.Phase_NONE, []byte("!migstage|ready|customer"))} + requests[rejectedPosition], requests[2] = requests[2], requests[rejectedPosition] + encoded, err := proto.Marshal(&pb.RaftCommand{Requests: requests}) + require.NoError(t, err) + f.SetApplyIndex(20) + response, ok := f.Apply(append([]byte{raftEncodeBatch}, encoded...)).(*fsmApplyResponse) + require.True(t, ok) + for i, verdict := range response.results { + if i == rejectedPosition { + require.ErrorIs(t, verdict, ErrInvalidRequest) + } else { + require.NoError(t, verdict) + } + } + snapshot, err := st.Snapshot() + require.NoError(t, err) + var raw bytes.Buffer + _, err = snapshot.WriteTo(&raw) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + snapshots = append(snapshots, raw.Bytes()) + require.NoError(t, st.Close()) + } + require.Equal(t, snapshots[0], snapshots[1], "watcher state cannot authorize namespace admission") + } +} + +func TestMigrationNamespaceSharedPrepareAndCommitValidatorsUseDurableVersion(t *testing.T) { + for _, active := range []bool{false, true} { + f := &kvFSM{store: store.NewMVCCStore()} + if active { + activateMigrationNamespace(t, f) + } + key := []byte("!migstage|ready|customer") + _, err := f.prepareTxnMutation(t.Context(), &pb.Mutation{Key: key, Value: []byte("value")}, key, 10, 0, 20) + lockErr := f.validatePreparedCommitLock(txnLock{StartTS: 10, CommitTS: 20, PrimaryKey: key}, key, key, 10, 20) + _, resolutionErr := f.migrationResolutionMutations([]*pb.Mutation{{Key: key, Value: []byte("value")}}) + if active { + require.ErrorIs(t, err, ErrInvalidRequest) + require.ErrorIs(t, lockErr, ErrInvalidRequest) + require.ErrorIs(t, resolutionErr, ErrInvalidRequest) + } else { + require.NoError(t, err) + require.NoError(t, lockErr) + require.NoError(t, resolutionErr) + } + } +} + +// These entries were admitted by 2d1a36ac. A newly upgraded apply-member must +// produce the same writes, including resolution of pre-upgrade intents. +func TestMigrationNamespaceUpgradeLegacyHistory(t *testing.T) { + for _, phase := range []pb.Phase{pb.Phase_NONE, pb.Phase_PREPARE, pb.Phase_COMMIT, pb.Phase_ABORT} { + t.Run(phase.String(), func(t *testing.T) { + st := store.NewMVCCStore() + f := &kvFSM{store: st, shardGroupID: 1} + key := []byte("!migstage|ready|customer") + if phase == pb.Phase_COMMIT || phase == pb.Phase_ABORT { + seedLegacyMigrationIntent(t, st, key) + } + request := migrationNamespaceRequest(phase, key) + require.ErrorIs(t, CheckMigrationRequest(t.Context(), st, 1, request), ErrInvalidRequest) + require.NoError(t, applyFSMRequest(t, f, request), "committed legacy entry must retain old apply semantics") + if phase == pb.Phase_NONE || phase == pb.Phase_COMMIT { + _, err := st.GetAt(t.Context(), key, 20) + require.NoError(t, err) + } + }) + } +} diff --git a/kv/migration_partition_codec_property_test.go b/kv/migration_partition_codec_property_test.go new file mode 100644 index 000000000..56825fadf --- /dev/null +++ b/kv/migration_partition_codec_property_test.go @@ -0,0 +1,114 @@ +package kv + +import ( + "bytes" + "encoding/json" + "slices" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "pgregory.net/rapid" +) + +func codecPropertyControlSeeds(t *testing.T) []MigrationPartitionControl { + t.Helper() + var seeds []MigrationPartitionControl + for _, group := range []uint64{1, 2} { + st := store.NewMVCCStore() + t.Cleanup(func() { require.NoError(t, st.Close()) }) + _, seed := partitionControlFixture(t, st, group) + seeds = append(seeds, seed) + } + return seeds +} + +func codecPropertyControl(t *rapid.T, seeds []MigrationPartitionControl) MigrationPartitionControl { + t.Helper() + seed := seeds[rapid.IntRange(0, len(seeds)-1).Draw(t, "owner role")] + manifest, err := distribution.DecodeOwnershipPartitionManifest(seed.Manifest) + require.NoError(t, err) + split := append([]byte("m"), rapid.SliceOfN(rapid.Byte(), 0, 32).Draw(t, "split suffix")...) + route := rapid.Uint64Range(10, 1000000).Draw(t, "child route id") + stamp := rapid.Uint64Range(1, ^uint64(0)).Draw(t, "split timestamp") + manifest.Children[0].RouteID, manifest.Children[1].RouteID = route, route+1 + manifest.Children[0].End, manifest.Children[1].Start = split, split + manifest.Children[0].SplitAtHLC, manifest.Children[1].SplitAtHLC = stamp, stamp + raw, err := distribution.EncodeOwnershipPartitionManifest(manifest) + require.NoError(t, err) + slices.Reverse(manifest.Children) + permuted, err := distribution.EncodeOwnershipPartitionManifest(manifest) + require.NoError(t, err) + require.Equal(t, raw, permuted, "manifest encoding canonicalizes child order") + control, err := BuildMigrationPartitionControl(nil, seed.PriorOwnership, raw, nil, + rapid.Uint64Range(2, 1000000).Draw(t, "catalog version"), + rapid.Uint64Range(9, 1000000).Draw(t, "configuration index")) + require.NoError(t, err) + return control +} + +func TestMigrationCodecPropertyControlRoundTrip(t *testing.T) { + seeds := codecPropertyControlSeeds(t) + rapid.Check(t, func(t *rapid.T) { + control := codecPropertyControl(t, seeds) + raw, err := EncodeMigrationPartitionControl(control) + require.NoError(t, err) + decoded, err := DecodeMigrationPartitionControl(raw) + require.NoError(t, err) + require.Equal(t, control, decoded) + reencoded, err := EncodeMigrationPartitionControl(decoded) + require.NoError(t, err) + require.Equal(t, raw, reencoded) + transition, err := EncodeMigrationPartitionJSON(control.Transition) + require.NoError(t, err) + var actual store.MigrationOwnershipPartitionTransition + require.NoError(t, DecodeMigrationPartitionJSON(transition, &actual)) + require.Equal(t, control.Transition, actual) + id, err := store.MigrationOwnershipPartitionReceiptIDFor(control.Transition) + require.NoError(t, err) + slices.Reverse(actual.After) + permutedID, err := store.MigrationOwnershipPartitionReceiptIDFor(actual) + require.NoError(t, err) + require.Equal(t, id, permutedID) + }) +} + +func TestMigrationCodecPropertyControlRejectsMalformed(t *testing.T) { + seeds := codecPropertyControlSeeds(t) + rapid.Check(t, func(t *rapid.T) { + control := codecPropertyControl(t, seeds) + raw, err := EncodeMigrationPartitionControl(control) + require.NoError(t, err) + malformed := [][]byte{ + append([]byte(" "), raw...), append(bytes.Clone(raw), ' '), + append(bytes.Clone(raw), []byte(`{}`)...), raw[:len(raw)-1], + append([]byte(`{"Unknown":true,`), raw[1:]...), + append([]byte(`{"Transition":{},`), raw[1:]...), + bytes.Replace(raw, []byte(`"Transition":`), []byte(`"transition":`), 1), + } + for _, bad := range malformed { + _, err = DecodeMigrationPartitionControl(bad) + require.Error(t, err) + } + mutations := []struct { + name string + mutate func(*MigrationPartitionControl) + }{ + {"child descriptor", func(c *MigrationPartitionControl) { c.Transition.After[0].DescriptorHash[0] ^= 1 }}, + {"prior hash", func(c *MigrationPartitionControl) { c.Transition.Before[0].AttestationHash[0] ^= 1 }}, + {"configuration", func(c *MigrationPartitionControl) { c.Transition.ConfigurationIndex++ }}, + {"operation identity", func(c *MigrationPartitionControl) { c.Transition.OperationID += "/changed" }}, + {"child ordering", func(c *MigrationPartitionControl) { slices.Reverse(c.Transition.After) }}, + } + for _, mutation := range mutations { + bad, decodeErr := DecodeMigrationPartitionControl(raw) + require.NoError(t, decodeErr) + mutation.mutate(&bad) + encoded, encodeErr := json.Marshal(bad) + require.NoError(t, encodeErr) + _, decodeErr = DecodeMigrationPartitionControl(encoded) + require.Error(t, decodeErr, mutation.name) + } + }) +} diff --git a/kv/migration_partition_history.go b/kv/migration_partition_history.go new file mode 100644 index 000000000..1ad3fc040 --- /dev/null +++ b/kv/migration_partition_history.go @@ -0,0 +1,39 @@ +package kv + +import ( + "context" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +// MigrationPartitionHistoryPresent detects durable admission obligations even +// when migration activation is disabled. Receipt-only history still requires +// compatible members. This is not a serving or ownership authorization. +func MigrationPartitionHistoryPresent(ctx context.Context, st store.MVCCStore) (bool, error) { + if st == nil { + return false, errors.WithStack(ErrInvalidRequest) + } + if reader, supported := st.(store.MigrationOwnershipPartitionStore); supported { + state, err := reader.MigrationOwnershipPartitions(ctx) + if err != nil { + return false, errors.WithStack(err) + } + if len(state.Partitions) != 0 || len(state.Receipts) != 0 { + return true, nil + } + } + view, err := readMigrationServingState(ctx, st) + if err != nil { + return false, err + } + if len(view.Ownership) != 0 { + return true, nil + } + for _, guard := range view.Readiness { + if guard.Armed { + return true, nil + } + } + return false, nil +} diff --git a/kv/migration_partition_history_test.go b/kv/migration_partition_history_test.go new file mode 100644 index 000000000..f28d457c3 --- /dev/null +++ b/kv/migration_partition_history_test.go @@ -0,0 +1,68 @@ +package kv + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +type partitionHistoryStore struct { + store.MVCCStore + partitions store.MigrationOwnershipPartitionState + view store.MigrationServingState + err error +} + +func (s *partitionHistoryStore) MigrationOwnershipPartitions(context.Context) (store.MigrationOwnershipPartitionState, error) { + return s.partitions, s.err +} + +func (s *partitionHistoryStore) ApplyMigrationOwnershipPartition(context.Context, store.MigrationOwnershipPartitionTransition, uint64) error { + return store.ErrNotSupported +} + +func (s *partitionHistoryStore) MigrationServingState(context.Context) (store.MigrationServingState, error) { + return s.view, s.err +} + +func (s *partitionHistoryStore) AcquireMigrationOwnership(context.Context, store.MigrationOwnershipState, uint64) error { + return store.ErrNotSupported +} + +func (s *partitionHistoryStore) ClearMigrationStateWithOwnership(context.Context, store.MigrationOwnershipState, uint64) error { + return store.ErrNotSupported +} + +func TestMigrationPartitionHistoryPresent(t *testing.T) { + for _, tc := range []struct { + name string + partitions store.MigrationOwnershipPartitionState + view store.MigrationServingState + want bool + }{ + {name: "empty"}, + {name: "active partition", want: true, partitions: store.MigrationOwnershipPartitionState{Partitions: []store.MigrationOwnershipPartition{{}}}}, + {name: "receipt only", want: true, partitions: store.MigrationOwnershipPartitionState{Receipts: []store.MigrationOwnershipPartitionReceipt{{}}}}, + {name: "legacy history", want: true, view: store.MigrationServingState{Ownership: []store.MigrationOwnershipState{{}}}}, + {name: "armed guard", want: true, view: store.MigrationServingState{Readiness: []store.TargetStagedReadinessState{{Armed: true}}}}, + {name: "unarmed guard", view: store.MigrationServingState{Readiness: []store.TargetStagedReadinessState{{}}}}, + } { + t.Run(tc.name, func(t *testing.T) { + st := &partitionHistoryStore{MVCCStore: store.NewMVCCStore(), partitions: tc.partitions, view: tc.view} + present, err := MigrationPartitionHistoryPresent(t.Context(), st) + require.NoError(t, err) + require.Equal(t, tc.want, present) + }) + } + st := &partitionHistoryStore{MVCCStore: store.NewMVCCStore(), err: errors.New("failed durable read")} + _, err := MigrationPartitionHistoryPresent(t.Context(), st) + require.ErrorIs(t, err, st.err) + _, err = MigrationPartitionHistoryPresent(t.Context(), nil) + require.ErrorIs(t, err, ErrInvalidRequest) + present, err := MigrationPartitionHistoryPresent(t.Context(), store.NewMVCCStore()) + require.NoError(t, err) + require.False(t, present) +} diff --git a/kv/migration_partition_protocol_readiness.go b/kv/migration_partition_protocol_readiness.go new file mode 100644 index 000000000..b4a383fe8 --- /dev/null +++ b/kv/migration_partition_protocol_readiness.go @@ -0,0 +1,67 @@ +package kv + +import ( + "reflect" + + "github.com/bootjp/elastickv/store" +) + +// MigrationPartitionRequestAdmissionReady attests the installed final-request +// proposal hook and its coherent partition store. It neither enables migration +// nor attests an active configuration, root consumer, or member agreement. +func MigrationPartitionRequestAdmissionReady(tx Transactional) bool { + switch t := tx.(type) { + case *TransactionManager: + return migrationPartitionManagerReady(t) + case *LeaderProxy: + return t != nil && MigrationPartitionRequestAdmissionReady(t.tm) + case *leaseRefreshingTxn: + return t != nil && MigrationPartitionRequestAdmissionReady(t.inner) + case *timestampAdmissionTransactional: + return t != nil && MigrationPartitionRequestAdmissionReady(t.inner) + default: + return false + } +} + +func migrationPartitionManagerReady(t *TransactionManager) bool { + if t == nil || migrationPartitionComponentNil(t.proposer) { + return false + } + admission := t.migrationAdmission.Load() + return admission != nil && admission.groupID != 0 && migrationPartitionStoreReady(admission.store) +} + +// MigrationPartitionReadAdmissionReady attests the actual public selected-route +// admission path and coherent partition store. Committed-root consumer and +// all-member protocol/configuration agreement remain separate runtime gates. +func (s *ShardStore) MigrationPartitionReadAdmissionReady(groupID uint64) bool { + return s.MigrationReadAdmissionReady(groupID) && migrationPartitionStoreReady(s.groups[groupID].Store) +} + +func migrationPartitionStoreReady(st store.MVCCStore) bool { + if migrationPartitionComponentNil(st) { + return false + } + _, partitions := st.(store.MigrationOwnershipPartitionStore) + _, legacy := st.(store.MigrationOwnershipStore) + _, coherent := st.(store.MigrationServingAdmissionReader) + return partitions && legacy && coherent +} + +func migrationPartitionComponentNil(component any) bool { + if component == nil { + return true + } + v := reflect.ValueOf(component) + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + return v.IsNil() + case reflect.Invalid, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128, reflect.Array, + reflect.String, reflect.Struct, reflect.UnsafePointer: + return false + } + return false +} diff --git a/kv/migration_partition_protocol_readiness_test.go b/kv/migration_partition_protocol_readiness_test.go new file mode 100644 index 000000000..a81bc11fb --- /dev/null +++ b/kv/migration_partition_protocol_readiness_test.go @@ -0,0 +1,72 @@ +package kv + +import ( + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +type partitionReadinessNilStore struct { + store.MVCCStore + store.MigrationOwnershipStore + store.MigrationOwnershipPartitionStore + store.MigrationServingAdmissionReader +} + +func TestMigrationPartitionProtocolReadinessRequiresInstalledRequestPath(t *testing.T) { + st := store.NewMVCCStore() + f, ok := NewKvFSMWithHLC(st, nil).(*kvFSM) + require.True(t, ok) + proposer := &migrationAdmissionProposer{fsm: f} + var nilProposer *migrationAdmissionProposer + var nilStore *partitionReadinessNilStore + for _, tc := range []struct { + name string + tx *TransactionManager + want bool + }{ + {name: "no proposal hook", tx: NewTransactionWithProposer(proposer)}, + {name: "no proposer", tx: NewTransactionWithProposer(nil, WithMigrationRequestAdmission(st, 7))}, + {name: "typed nil proposer", tx: NewTransactionWithProposer(nilProposer, WithMigrationRequestAdmission(st, 7))}, + {name: "typed nil store", tx: NewTransactionWithProposer(proposer, WithMigrationRequestAdmission(nilStore, 7))}, + {name: "zero group", tx: NewTransactionWithProposer(proposer, WithMigrationRequestAdmission(st, 0))}, + {name: "type-only legacy store", tx: NewTransactionWithProposer(proposer, WithMigrationRequestAdmission(&partitionHistoryStore{MVCCStore: st}, 7))}, + {name: "installed without active epoch", tx: NewTransactionWithProposer(proposer, WithMigrationRequestAdmission(st, 7)), want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Cleanup(tc.tx.Close) + require.Equal(t, tc.want, MigrationPartitionRequestAdmissionReady(tc.tx)) + wrapped := ×tampAdmissionTransactional{inner: &leaseRefreshingTxn{inner: &LeaderProxy{tm: tc.tx}}} + require.Equal(t, tc.want, MigrationPartitionRequestAdmissionReady(wrapped), "wrappers cannot invent an installed hook") + }) + } + var nilManager *TransactionManager + require.False(t, MigrationPartitionRequestAdmissionReady(nilManager)) + require.False(t, MigrationPartitionRequestAdmissionReady(nil)) +} + +func TestMigrationPartitionProtocolReadinessRequiresCachedPublicReadPairing(t *testing.T) { + engine := distribution.NewEngine() + st := store.NewMVCCStore() + var nilStore *partitionReadinessNilStore + for _, tc := range []struct { + name string + reads *ShardStore + group uint64 + want bool + }{ + {name: "nil shard store", group: 7}, + {name: "no route engine", reads: NewShardStore(nil, map[uint64]*ShardGroup{7: {Store: st}}), group: 7}, + {name: "unknown group", reads: NewShardStore(engine, nil), group: 7}, + {name: "zero group", reads: NewShardStore(engine, map[uint64]*ShardGroup{0: {Store: st}})}, + {name: "typed nil store", reads: NewShardStore(engine, map[uint64]*ShardGroup{7: {Store: nilStore}}), group: 7}, + {name: "type-only legacy store", reads: NewShardStore(engine, map[uint64]*ShardGroup{7: {Store: &partitionHistoryStore{MVCCStore: st}}}), group: 7}, + {name: "installed without active epoch", reads: NewShardStore(engine, map[uint64]*ShardGroup{7: {Store: st}}), group: 7, want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, tc.reads.MigrationPartitionReadAdmissionReady(tc.group)) + }) + } +} diff --git a/kv/migration_prefix_ownership_test.go b/kv/migration_prefix_ownership_test.go new file mode 100644 index 000000000..1f8c5f455 --- /dev/null +++ b/kv/migration_prefix_ownership_test.go @@ -0,0 +1,178 @@ +package kv + +import ( + "crypto/sha256" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func prefixOwnershipFixture(t *testing.T, defect string) (*ShardStore, *kvFSM) { + t.Helper() + routes := []distribution.RouteDescriptor{ + {RouteID: 1, End: []byte("pm"), GroupID: 1, State: distribution.RouteStateActive, MinWriteTSExclusive: 100}, + {RouteID: 2, Start: []byte("pm"), GroupID: 1, State: distribution.RouteStateActive, MinWriteTSExclusive: 100}, + } + if defect == "catalog-hole" { + routes[0].End = []byte("pl") + } + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{Version: 7, Routes: routes})) + st := store.NewMVCCStore() + writer, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + for i, route := range routes { + if i == 0 && defect == "legacy-neighbor" { + continue + } + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(route, true) + require.NoError(t, err) + digest := sha256.Sum256(encoded) + proof := store.MigrationOwnershipState{Generation: "0123456789abcdef0123456789abcdef", + JobID: route.RouteID + 10, LocalGroupID: 1, OwnerGroupID: 1, Role: store.MigrationOwner, + RouteID: route.RouteID, DescriptorHash: digest[:], RouteStart: route.Start, RouteEnd: route.End, + CatalogVersion: 7, ConfigurationIndex: 1, MinWriteTSExclusive: 100, WriteFloorKnown: true} + if i == 1 { + alterPrefixOwnershipProof(&proof, defect) + } + require.NoError(t, writer.ClearMigrationStateWithOwnership(t.Context(), proof, route.RouteID+1)) + } + for _, key := range []string{"pa", "pz", "z"} { + require.NoError(t, st.PutAt(t.Context(), []byte(key), []byte(key), 120, 0)) + } + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: {Store: st}}) + return shards, &kvFSM{store: st, shardGroupID: 1, routes: forbiddenApplyWatcher{}} +} + +func alterPrefixOwnershipProof(proof *store.MigrationOwnershipState, defect string) { + switch defect { + case "proof-hole": + proof.RouteStart = []byte("pn") + case "nonowner": + proof.OwnerGroupID, proof.Role = 2, store.MigrationNonOwner + case "wrong-floor": + proof.MinWriteTSExclusive = 150 + case "future-owner": + proof.CatalogVersion = 8 + } +} + +func prefixOwnershipRequest(prefix []byte, ts uint64) *pb.Request { + return &pb.Request{Ts: ts, ObservedRouteVersion: 7, + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: prefix}}} +} + +func assertPrefixOwnershipValues(t *testing.T, st store.MVCCStore, prefix []byte, deleted bool) { + t.Helper() + for _, key := range []string{"pa", "pz", "z"} { + value, err := st.GetAt(t.Context(), []byte(key), ^uint64(0)) + if deleted && (len(prefix) == 0 || key != "z") { + require.ErrorIs(t, err, store.ErrKeyNotFound, key) + } else { + require.NoError(t, err, key) + require.Equal(t, []byte(key), value) + } + } +} + +func TestMigrationPrefixOwnershipPublicDeletesAcrossRoutes(t *testing.T) { + for _, defect := range []string{"", "legacy-neighbor"} { + assertPrefixOwnershipPublicDeletes(t, defect) + } +} + +func assertPrefixOwnershipPublicDeletes(t *testing.T, defect string) { + t.Helper() + for _, prefix := range []string{"p", ""} { + for _, mode := range []string{"normal", "raft", "raft-index", "batch"} { + t.Run(defect+"/"+mode+"/"+prefix, func(t *testing.T) { + shards, f := prefixOwnershipFixture(t, defect) + require.NoError(t, deletePrefixOwnershipFixture(t, shards, []byte(prefix), mode, 200)) + assertPrefixOwnershipValues(t, f.store, []byte(prefix), true) + }) + } + } +} + +func deletePrefixOwnershipFixture(t *testing.T, s *ShardStore, prefix []byte, mode string, ts uint64) error { + t.Helper() + switch mode { + case "normal": + return s.DeletePrefixAt(t.Context(), prefix, nil, ts) + case "raft": + return s.DeletePrefixAtRaft(t.Context(), prefix, nil, ts) + case "raft-index": + return s.DeletePrefixAtRaftAt(t.Context(), prefix, nil, ts, 20) + default: + return s.DeletePrefixesAtRaftAt(t.Context(), []store.PrefixDelete{{Prefix: prefix}}, ts, 20) + } +} + +func TestMigrationPrefixOwnershipFSMAndCoordinatorAcrossRoutes(t *testing.T) { + for _, prefix := range []string{"p", ""} { + t.Run("FSM/"+prefix, func(t *testing.T) { + _, f := prefixOwnershipFixture(t, "") + before, err := readMigrationServingState(t.Context(), f.store) + require.NoError(t, err) + f.SetApplyIndex(20) + require.NoError(t, applyFSMRequest(t, f, prefixOwnershipRequest([]byte(prefix), 200))) + assertPrefixOwnershipValues(t, f.store, []byte(prefix), true) + after, err := readMigrationServingState(t.Context(), f.store) + require.NoError(t, err) + require.Equal(t, before, after, "range partitioning must not clip or rewrite ownership authority") + }) + t.Run("coordinator/"+prefix, func(t *testing.T) { + shards, f := prefixOwnershipFixture(t, "") + tx := NewTransactionWithProposer(&migrationAdmissionProposer{fsm: f}) + t.Cleanup(tx.Close) + shards.groups[1].Txn = tx + coordinator := NewShardedCoordinator(shards.engine, shards.groups, 1, NewHLC(), shards) + t.Cleanup(func() { require.NoError(t, coordinator.Close()) }) + _, err := coordinator.Dispatch(t.Context(), &OperationGroup[OP]{ObservedRouteVersion: 7, + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte(prefix)}}}) + require.NoError(t, err) + assertPrefixOwnershipValues(t, f.store, []byte(prefix), true) + }) + } +} + +func TestMigrationPrefixOwnershipFSMDoesNotInventLegacyNeighborAuthority(t *testing.T) { + _, f := prefixOwnershipFixture(t, "legacy-neighbor") + f.SetApplyIndex(20) + require.ErrorIs(t, applyFSMRequest(t, f, prefixOwnershipRequest(nil, 200)), ErrRouteCutoverPending) + assertPrefixOwnershipValues(t, f.store, nil, false) +} + +func TestMigrationPrefixOwnershipRejectsIncompleteAuthorityBeforeEffects(t *testing.T) { + for _, defect := range []string{"proof-hole", "catalog-hole", "nonowner", "future-owner", "wrong-floor", "low-ts"} { + t.Run(defect, func(t *testing.T) { + for _, prefix := range []string{"p", ""} { + shards, f := prefixOwnershipFixture(t, defect) + ts := uint64(200) + if defect == "low-ts" { + ts = 100 + } + expected := ErrRouteCutoverPending + if defect == "low-ts" { + expected = ErrRouteWriteTimestampTooLow + } + require.ErrorIs(t, deletePrefixOwnershipFixture(t, shards, []byte(prefix), "normal", ts), expected) + assertPrefixOwnershipValues(t, f.store, []byte(prefix), false) + f.SetApplyIndex(20) + if defect == "wrong-floor" { + // A durable floor of 150 is valid authority at 200; the + // public cache's smaller floor must not authorize at 150. + ts = 150 + } + if defect == "wrong-floor" { + expected = ErrRouteWriteTimestampTooLow + } + require.ErrorIs(t, applyFSMRequest(t, f, prefixOwnershipRequest([]byte(prefix), ts)), expected) + assertPrefixOwnershipValues(t, f.store, []byte(prefix), false) + } + }) + } +} diff --git a/kv/migration_prepare_evidence_integration_test.go b/kv/migration_prepare_evidence_integration_test.go new file mode 100644 index 000000000..5e70a6dff --- /dev/null +++ b/kv/migration_prepare_evidence_integration_test.go @@ -0,0 +1,137 @@ +package kv + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestProvenCommitDrainsMigrationAndCreationFences(t *testing.T) { + t.Parallel() + for _, primary := range []string{"n", "b"} { + t.Run(primary, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + engine := distribution.NewEngine() + route := distribution.RouteDescriptor{RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 1} + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{route}) + fsm := newComposed1FSM(t, engine, 1) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + meta := TxnMeta{PrimaryKey: []byte(primary), CommitTS: 999, PrepareCommitTS: 60, LockTTLms: defaultTxnLockTTLms} + request := &pb.Request{IsTxn: true, Phase: pb.Phase_PREPARE, Ts: 40, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(meta)}, + {Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("prepared")}, + }} + require.NoError(t, applyFSMRequest(t, fsm, request)) + lock, found, err := fsm.txnLockAtKey(ctx, txnLockKey([]byte("n"))) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint64(60), lock.CommitTS, "only the V3 proof timestamp may bind the lock") + route.State = distribution.RouteStateWriteFenced + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{route}) + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 11, MigrationJobID: 11, RouteStart: []byte("m"), RouteEnd: []byte("z"), + Armed: true, SourceWriteFence: true, TrackWrites: true, MinWriteTSExclusive: 1, RetentionPinTS: 1, + }) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceSeal())) + request.Phase = pb.Phase_COMMIT + meta.PrepareCommitTS, meta.CommitTS = 0, 61 + request.Mutations[0].Value = EncodeTxnMeta(meta) + require.ErrorIs(t, applyFSMRequest(t, fsm, request), ErrTxnInvalidMeta) + meta.CommitTS = 60 + request.Mutations[0].Value = EncodeTxnMeta(meta) + require.NoError(t, applyFSMRequest(t, fsm, request)) + value, err := fsm.store.GetAt(ctx, []byte("n"), 60) + require.NoError(t, err) + require.Equal(t, []byte("prepared"), value) + require.Equal(t, uint64(60), migrationTrackerMinimum(t, fsm)) + _, found, err = fsm.txnLockAtKey(ctx, txnLockKey([]byte("n"))) + require.NoError(t, err) + require.False(t, found) + }) + } +} + +func TestPrepareProofTimestampControlsMigrationFloor(t *testing.T) { + t.Parallel() + for _, proof := range []uint64{60, 101} { + fsm := newWriteFloorFSM(t) + require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) + hint := uint64(999) + if proof == 101 { + hint = 1 + } + request := &pb.Request{IsTxn: true, Phase: pb.Phase_PREPARE, Ts: 50, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("b"), CommitTS: hint, PrepareCommitTS: proof})}, + {Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("prepared")}, + }} + err := applyFSMRequest(t, fsm, request) + if proof == 60 { + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow, "an inflated legacy hint must not bypass the route floor") + } else { + require.NoError(t, err, "the negotiated proof, not its legacy hint, determines admission") + } + } +} + +func TestMigrationKeepsNewPrepareControlLocalAndOldNamesUserOwned(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + oldKeys := legacyPrepareEvidenceUserKeys() + for _, key := range oldKeys { + require.NoError(t, st.PutAt(ctx, key, []byte{1}, 10, 0)) + } + fsm := newPrepareSafetyFSM(t, st) + require.NoError(t, fsm.persistBackupTimestampFloor(ctx, 100)) + for _, command := range [][]byte{EncodePrepareEvidenceFence(0), EncodePrepareEvidenceSeal(), EncodePrepareEvidenceFence(100), EncodePrepareEvidenceRelease(100)} { + require.Nil(t, fsm.Apply(command)) + } + controls := [][]byte{backupTimestampFloorKey, prepareEvidenceEnabledKey, prepareEvidenceGateKey, prepareEvidenceTimestampFloorKey} + brackets, err := distribution.PlanExportBrackets([]byte("!txn|"), []byte("!txn}")) + require.NoError(t, err) + for _, key := range controls { + for _, bracket := range brackets { + require.False(t, bracket.ContainsRawKey(key), "peer-local control must never enter an export family") + } + command, marshalErr := MarshalMigrationImportCommand(&pb.ImportRangeVersionsRequest{JobId: 99, BracketId: 1, BatchSeq: 1, + Versions: []*pb.MVCCVersion{{Key: key, Value: []byte{1}, CommitTs: 200}}}) + require.NoError(t, marshalErr) + applyErr, ok := fsm.Apply(command).(error) + require.True(t, ok) + require.ErrorIs(t, applyErr, ErrInvalidRequest) + } + cleanup := &pb.CleanupMigrationRequest{JobId: 7, RangeStart: []byte("!txn|"), RangeEnd: []byte("!txn}"), RouteStart: []byte("!"), + KeyFamily: distribution.MigrationFamilyUser, ExcludeKnownInternal: true, RequiresRouteKeyCheck: true, MaxCommitTs: 1000} + opts := migrationCleanupOptionsFromProto(cleanup, 0) + exported, err := st.ExportVersions(ctx, store.ExportVersionsOptions{StartKey: cleanup.RangeStart, EndKey: cleanup.RangeEnd, + MaxVersions: 100, AcceptVersion: opts.AcceptVersion}) + require.NoError(t, err) + require.Len(t, exported.Versions, len(oldKeys)) + histories := make(map[string][]store.MVCCVersion) + for _, version := range exported.Versions { + histories[string(version.Key)] = append(histories[string(version.Key)], version) + } + assertPrepareNamespaceMigration(t, false, histories) + before := make(map[string][]byte) + for _, key := range controls { + before[string(key)], err = st.GetAt(ctx, key, ^uint64(0)) + require.NoError(t, err) + } + command, err := MarshalMigrationCleanupCommand(cleanup) + require.NoError(t, err) + require.IsType(t, store.CleanupVersionsResult{}, fsm.Apply(command)) + for _, key := range oldKeys { + _, err = st.GetAt(ctx, key, ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound, "old names must retire as ordinary migrated user versions") + } + for _, key := range controls { + value, loadErr := st.GetAt(ctx, key, ^uint64(0)) + require.NoError(t, loadErr) + require.Equal(t, before[string(key)], value) + } +} diff --git a/kv/migration_public_admission_test.go b/kv/migration_public_admission_test.go new file mode 100644 index 000000000..18761d23c --- /dev/null +++ b/kv/migration_public_admission_test.go @@ -0,0 +1,134 @@ +package kv + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +type migrationAdmissionProposer struct { + fsm *kvFSM + proposals uint64 +} + +func (p *migrationAdmissionProposer) Propose(_ context.Context, data []byte) (*raftengine.ProposalResult, error) { + p.proposals++ + index := uint64(20) + p.proposals + p.fsm.SetApplyIndex(index) + return &raftengine.ProposalResult{CommitIndex: index, Response: p.fsm.Apply(data)}, nil +} + +func (p *migrationAdmissionProposer) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { + return p.Propose(ctx, data) +} + +func TestMigrationProposalQueueRevalidatesIndependentRequests(t *testing.T) { + f := newComposed1FSM(t, staleServingEngine(t, 1), 1) + p := &migrationAdmissionProposer{fsm: f} + tx := NewTransactionWithProposer(p, WithMigrationRequestAdmission(f.store, 1)) + t.Cleanup(tx.Close) + require.True(t, MigrationRequestAdmissionReady(tx)) + // These are independent queued requests. A whole client request crossing + // both ranges is rejected atomically; unrelated queue items still apply. + rejected := &rawCommitItem{reqs: []*pb.Request{{Ts: 50, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("must-not-leak")}, + {Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("unproven")}, + }}}, done: make(chan rawCommitResult, 1)} + accepted := &rawCommitItem{reqs: []*pb.Request{{Ts: 51, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("c"), Value: []byte("ordinary")}}}}, done: make(chan rawCommitResult, 1)} + // The guard arrives after queuing, before the common proposal hook runs. + deterministicMigrationGuard(t, f.store, true) + tx.applyRawBatch([]*rawCommitItem{rejected, accepted}) + require.ErrorIs(t, (<-rejected.done).err, ErrRouteCutoverPending) + require.NoError(t, (<-accepted.done).err) + require.Equal(t, uint64(1), p.proposals) + _, err := f.store.GetAt(t.Context(), []byte("b"), 100) + require.ErrorIs(t, err, store.ErrKeyNotFound) + value, err := f.store.GetAt(t.Context(), []byte("c"), 100) + require.NoError(t, err) + require.Equal(t, []byte("ordinary"), value) +} + +func TestMigrationProposalAdvisoryPreservesAuthenticatedResolution(t *testing.T) { + for _, phase := range []pb.Phase{pb.Phase_COMMIT, pb.Phase_ABORT} { + t.Run(phase.String(), func(t *testing.T) { + f := &kvFSM{store: store.NewMVCCStore(), shardGroupID: 1} + request := applyRouteContextTxn(pb.Phase_PREPARE, 20, 60, []byte("n")) + require.NoError(t, f.applyRequestErr(t.Context(), request)) + retireServingSource(t, f.store) + f.routes = forbiddenApplyWatcher{} + tx := NewTransactionWithProposer(&migrationAdmissionProposer{fsm: f}, WithMigrationRequestAdmission(f.store, 1)) + t.Cleanup(tx.Close) + request.Phase = phase + _, err := tx.Commit(t.Context(), []*pb.Request{request}) + require.NoError(t, err) + _, err = tx.Commit(t.Context(), []*pb.Request{request}) + require.NoError(t, err, "durable resolution retry must remain a no-op") + }) + } +} + +func TestMigrationPublicReadsHonorNodeFloor(t *testing.T) { + st := store.NewMVCCStore() + retireServingSource(t, st) + shards := NewShardStore(staleServingEngine(t, 9), map[uint64]*ShardGroup{1: {Store: st}}) + require.True(t, shards.MigrationReadAdmissionReady(1)) + checks := []func() error{ + func() error { _, e := shards.ScanAt(t.Context(), []byte("b"), []byte("c"), 1, 200); return e }, + func() error { _, e := shards.ReverseScanAt(t.Context(), []byte("b"), []byte("c"), 1, 200); return e }, + func() error { _, e := shards.ScanGroupAt(t.Context(), 1, []byte("b"), []byte("c"), 1, 200); return e }, + func() error { + _, e := shards.ReverseScanGroupAt(t.Context(), 1, []byte("b"), []byte("c"), 1, 200) + return e + }, + func() error { + _, e := shards.ScanGroupKeysAt(t.Context(), 1, []byte("b"), []byte("c"), 1, 200) + return e + }, + func() error { + _, _, e := shards.ScanAtPhysicalLimit(t.Context(), []byte("b"), []byte("c"), 1, 1, 200) + return e + }, + func() error { + _, _, e := shards.ReverseScanAtPhysicalLimit(t.Context(), []byte("b"), []byte("c"), 1, 1, 200) + return e + }, + func() error { _, e := shards.CommittedVersionAt(t.Context(), []byte("b"), 200); return e }, + } + for i, check := range checks { + require.ErrorIs(t, check(), ErrRouteCutoverPending, "entry point %d", i) + } + coordinator := &ShardedCoordinator{engine: shards.engine, groups: shards.groups} + require.ErrorIs(t, coordinator.verifyTargetReadinessForReadKeyOnShard(t.Context(), 1, shards.groups[1], []byte("b")), ErrRouteCutoverPending) +} + +func TestLegacyUnknownOwnershipBlocksOnlyItsRange(t *testing.T) { + st := store.NewMVCCStore() + writer, ok := st.(store.MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), store.TargetStagedReadinessState{ + JobID: 7, MigrationJobID: 7, Armed: true, TrackWrites: true, SourceWriteFence: true, + RouteStart: []byte("m"), RouteEnd: []byte("z"), ExpectedCutoverVersion: 10, MinWriteTSExclusive: 100, RetentionPinTS: 1, + }, 8)) + proofs, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + require.NoError(t, proofs.ClearMigrationStateWithOwnership(t.Context(), store.MigrationOwnershipState{ + JobID: 7, LocalGroupID: 1, OwnerGroupID: 1, CatalogVersion: 10, ConfigurationIndex: 3, + MinWriteTSExclusive: 100, Role: store.MigrationOwner, RouteStart: []byte("m"), RouteEnd: []byte("z"), + }, 9)) + require.NoError(t, st.PutAt(t.Context(), []byte("b"), []byte("ordinary"), 150, 0)) + engine := staleServingEngine(t, 9) + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: {Store: st}}) + value, err := shards.GetAt(t.Context(), []byte("b"), 200) + require.NoError(t, err) + require.Equal(t, []byte("ordinary"), value) + _, err = shards.GetAt(t.Context(), []byte("n"), 200) + require.ErrorIs(t, err, ErrRouteCutoverPending) + f, ok := NewKvFSMWithHLC(st, nil, WithRouteHistory(WrapDistributionEngine(engine), 1)).(*kvFSM) + require.True(t, ok) + require.NoError(t, f.applyRequestErr(t.Context(), &pb.Request{Ts: 200, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("c"), Value: []byte("ordinary")}}})) + require.ErrorIs(t, f.applyRequestErr(t.Context(), deterministicRawRequest(10, "n")), ErrRouteCutoverPending) +} diff --git a/kv/migration_resplit_acceptance_test.go b/kv/migration_resplit_acceptance_test.go new file mode 100644 index 000000000..7c13a05b8 --- /dev/null +++ b/kv/migration_resplit_acceptance_test.go @@ -0,0 +1,112 @@ +package kv + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +// This is the negative half of the runtime re-split contract. The adapter +// acceptance owns real SplitRange/runner/partition progress; no child authority +// is synthesized here to turn a newer watcher into a committed proof. +func resplitSourceReceiptFixture(t testing.TB) (*kvFSM, *distribution.CatalogStore, *distribution.Engine, store.MigrationOwnershipState) { + t.Helper() + root := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(root, distribution.WithCatalogRouteDescriptorV2Writes(true)) + parent := distribution.RouteDescriptor{RouteID: 3, ParentRouteID: 1, Start: []byte("m"), End: []byte("z"), + GroupID: 2, MinWriteTSExclusive: 100} + snapshot, err := catalog.Save(t.Context(), 0, []distribution.RouteDescriptor{parent}) + require.NoError(t, err) + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(snapshot)) + local, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, local.Close()) }) + fsm, ok := NewKvFSMWithHLC(local, nil, WithRouteHistory(WrapDistributionEngine(engine), 2)).(*kvFSM) + require.True(t, ok) + require.NoError(t, fsm.RecordAppliedConfiguration(9)) + guard, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{JobId: 7, MigrationJobId: 7, + Armed: true, TrackWrites: true, RouteStart: parent.Start, RouteEnd: parent.End, + ExpectedCutoverVersion: snapshot.Version, MinWriteTsExclusive: 100, RetentionPinTs: 1}) + require.NoError(t, err) + fsm.SetApplyIndex(10) + require.Nil(t, fsm.Apply(guard)) + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(parent, true) + require.NoError(t, err) + digest := sha256.Sum256(encoded) + var generation [16]byte + _, err = rand.Read(generation[:]) + require.NoError(t, err) + acquire, err := MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{JobId: 7, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_SOURCE_OWNERSHIP_ACQUIRE, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: 9, + Ownership: &pb.MigrationOwnershipProof{Generation: hex.EncodeToString(generation[:]), + LocalGroupId: 2, OwnerGroupId: 2, CatalogVersion: snapshot.Version, RouteId: parent.RouteID, + ParentRouteId: parent.ParentRouteID, RouteStart: parent.Start, RouteEnd: parent.End, + WriteFloorKnown: true, MinWriteTsExclusive: 100, SourceTrackerVersion: snapshot.Version, + DescriptorHash: digest[:], Role: pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER}}) + require.NoError(t, err) + fsm.SetApplyIndex(11) + result, ok := fsm.Apply(acquire).(store.CleanupVersionsResult) + require.True(t, ok) + require.True(t, result.Done) + reader, ok := local.(store.MigrationOwnershipStore) + require.True(t, ok) + view, err := reader.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Len(t, view.Ownership, 1) + proof := view.Ownership[0] + require.True(t, store.MigrationOwnershipEvidenceValid(proof)) + require.NotEmpty(t, proof.Receipts) + require.EqualValues(t, 11, proof.SourceAppliedIndex) + return fsm, catalog, engine, proof +} + +func TestMigrationResplitAcceptanceNewMirrorCannotReplacePartitionReceipt(t *testing.T) { + for _, rightOwner := range []uint64{2, 3} { + t.Run(map[uint64]string{2: "same-owner children", 3: "moving-right child"}[rightOwner], func(t *testing.T) { + fsm, catalog, engine, proof := resplitSourceReceiptFixture(t) + children := []distribution.RouteDescriptor{ + {RouteID: 4, ParentRouteID: 3, Start: []byte("m"), End: []byte("t"), GroupID: 2, MinWriteTSExclusive: 100}, + {RouteID: 5, ParentRouteID: 3, Start: []byte("t"), End: []byte("z"), GroupID: rightOwner, MinWriteTSExclusive: 100}, + } + snapshot, err := catalog.Save(t.Context(), proof.CatalogVersion, children) + require.NoError(t, err) + require.NoError(t, engine.ApplySnapshot(snapshot)) + for _, key := range []string{"n", "y"} { + request := &pb.Request{Ts: 200, ObservedRouteVersion: EncodeObservedRouteVersion(snapshot.Version), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte(key), Value: []byte("unproved-new-C")}}} + require.ErrorIs(t, CheckMigrationRequest(t.Context(), fsm.store, 2, request), ErrRouteCutoverPending) + require.ErrorIs(t, applyFSMRequest(t, fsm, request), ErrRouteCutoverPending) + _, err = fsm.store.GetAt(t.Context(), []byte(key), ^uint64(0)) + require.ErrorIs(t, err, store.ErrKeyNotFound) + } + }) + } +} + +func TestMigrationResplitAcceptanceLoggedParentEntryKeepsReceiptAndFloor(t *testing.T) { + fsm, catalog, engine, proof := resplitSourceReceiptFixture(t) + snapshot, err := catalog.Save(t.Context(), proof.CatalogVersion, []distribution.RouteDescriptor{ + {RouteID: 4, ParentRouteID: 3, Start: []byte("m"), End: []byte("t"), GroupID: 2, MinWriteTSExclusive: 100}, + {RouteID: 5, ParentRouteID: 3, Start: []byte("t"), End: []byte("z"), GroupID: 2, MinWriteTSExclusive: 100}, + }) + require.NoError(t, err) + require.NoError(t, engine.ApplySnapshot(snapshot)) + for _, key := range []string{"n", "y"} { + request := &pb.Request{Ts: 100, ObservedRouteVersion: EncodeObservedRouteVersion(proof.CatalogVersion), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte(key), Value: []byte("logged-before-split")}}} + require.ErrorIs(t, applyFSMRequest(t, fsm, request), ErrRouteWriteTimestampTooLow) + request.Ts = 200 + require.NoError(t, applyFSMRequest(t, fsm, request), "apply uses immutable logged parent evidence, not the newer mirror") + value, getErr := fsm.store.GetAt(t.Context(), []byte(key), 200) + require.NoError(t, getErr) + require.Equal(t, []byte("logged-before-split"), value) + } +} diff --git a/kv/migration_serving.go b/kv/migration_serving.go new file mode 100644 index 000000000..319d96dd9 --- /dev/null +++ b/kv/migration_serving.go @@ -0,0 +1,213 @@ +package kv + +import ( + "bytes" + "context" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +func migrationServingFloor(states []store.MigrationOwnershipState) uint64 { + var floor uint64 + for _, state := range states { + if !store.MigrationOwnershipEvidenceValid(state) { + continue + } + floor = max(floor, state.CatalogVersion) + } + return floor +} + +func readMigrationServingState(ctx context.Context, st store.MVCCStore) (store.MigrationServingState, error) { + if reader, ok := st.(store.MigrationServingAdmissionReader); ok { + view, err := reader.MigrationServingAdmissionState(ctx) + return view, errors.WithStack(err) + } + if reader, ok := st.(store.MigrationOwnershipStore); ok { + view, err := reader.MigrationServingState(ctx) + return view, errors.WithStack(err) + } + if reader, ok := st.(store.MigrationTargetReadinessReader); ok { + guards, err := reader.MigrationTargetReadinessStates(ctx) + return store.MigrationServingState{Readiness: guards}, errors.WithStack(err) + } + return store.MigrationServingState{}, nil +} + +type migrationServingReadVersionKey struct{} + +func migrationServingReadContext(ctx context.Context, version uint64) context.Context { + if version == 0 { + return ctx + } + if prior, ok := ctx.Value(migrationServingReadVersionKey{}).(uint64); ok && prior <= version { + return ctx + } + return context.WithValue(ctx, migrationServingReadVersionKey{}, version) +} + +func (s *ShardStore) verifyMigrationServingReadFloor(ctx context.Context, view store.MigrationServingState) error { + floor := migrationAdmissionFloor(view) + requested, _ := ctx.Value(migrationServingReadVersionKey{}).(uint64) + if floor > s.ReadRouteVersion() || (requested != 0 && requested < floor) { + return errors.WithStack(ErrRouteCutoverPending) + } + return nil +} + +func (s *ShardStore) verifyMigrationServingFloor(ctx context.Context, requested uint64) error { + for _, group := range s.groups { + if group == nil || group.Store == nil { + continue + } + floor, err := migrationStoreServingFloor(ctx, group.Store) + if err != nil { + return err + } + if floor > s.ReadRouteVersion() || (requested != 0 && requested < floor) { + return errors.WithStack(ErrRouteCutoverPending) + } + } + return nil +} + +func migrationStoreServingFloor(ctx context.Context, st store.MVCCStore) (uint64, error) { + if reader, ok := st.(store.MigrationServingSummaryReader); ok { + summary, err := reader.MigrationServingSummary(ctx) + return summary.CatalogVersionFloor, errors.WithStack(err) + } + view, err := readMigrationServingState(ctx, st) + return migrationAdmissionFloor(view), err +} + +// MigrationReadAdmissionReady reports the concrete read gate/store pairing, +// not an environment flag. The Internal runtime separately attests its epoch. +func (s *ShardStore) MigrationReadAdmissionReady(groupID uint64) bool { + if s == nil || s.engine == nil || groupID == 0 { + return false + } + group := s.groups[groupID] + if group == nil { + return false + } + _, ok := group.Store.(store.MigrationOwnershipStore) + return ok +} + +func (s *ShardStore) verifyMigrationServingRange(view store.MigrationServingState, route distribution.Route, start, end []byte) error { + if s.ReadRouteVersion() < migrationAdmissionFloor(view) { + return errors.WithStack(ErrRouteCutoverPending) + } + proofs, err := migrationAdmissionRangeProofs(view, migrationApplyRange{start, end}) + if err != nil { + return err + } + for _, proof := range proofs { + if proof.LocalGroupID != route.GroupID || proof.Role != store.MigrationOwner { + return errors.WithStack(ErrRouteCutoverPending) + } + proofStart, proofEnd := prefixDeleteRouteIntersection(distribution.Route{Start: proof.RouteStart, End: proof.RouteEnd}, start, end) + if err := s.verifyMigrationAdmissionOwnerRoutes(proof, route, proofStart, proofEnd); err != nil { + return err + } + } + return nil +} + +func (s *ShardStore) verifyMigrationAdmissionOwnerRoutes(proof migrationAdmissionProof, route distribution.Route, start, end []byte) error { + if err := s.verifyMigrationServingOwnerRoutes(proof.MigrationOwnershipState, route, start, end); err != nil { + return err + } + if proof.partition == nil { + return nil + } + routes, _, _ := s.readinessProofRoutes(route, start, end) + cursor := start + for _, candidate := range routes { + if !migrationAdmissionRouteMatches(proof, candidate, cursor) { + return errors.WithStack(ErrRouteCutoverPending) + } + if len(candidate.End) == 0 || (len(end) != 0 && bytes.Compare(candidate.End, end) >= 0) { + return nil + } + cursor = candidate.End + } + return errors.WithStack(ErrRouteCutoverPending) +} + +func migrationAdmissionRouteMatches(proof migrationAdmissionProof, route distribution.Route, cursor []byte) bool { + return bytes.Compare(route.Start, cursor) <= 0 && routeHasStagedVisibility(route) == proof.Staged && + (!proof.Staged || route.MigrationJobID == proof.JobID) +} + +func (s *ShardStore) verifyMigrationServingOwnerRoutes(proof store.MigrationOwnershipState, route distribution.Route, start, end []byte) error { + routes, version, ok := s.readinessProofRoutes(route, start, end) + if !ok || version < proof.CatalogVersion { + return errors.WithStack(ErrRouteCutoverPending) + } + for _, candidate := range routes { + if candidate.GroupID != proof.OwnerGroupID || candidate.MinWriteTSExclusive < proof.MinWriteTSExclusive { + return errors.WithStack(ErrRouteCutoverPending) + } + } + return nil +} + +func migrationServingFootprint(req *pb.Request) ([]migrationApplyRange, error) { + ranges := migrationApplyFootprint(req) + if !req.GetIsTxn() { + return ranges, nil + } + meta, _, err := extractTxnMeta(req.GetMutations()) + if err != nil { + return nil, err + } + if migrationPrimaryParticipates(req, meta.PrimaryKey) { + start, end := readinessRouteRangeForScan(meta.PrimaryKey, nextScanCursor(meta.PrimaryKey)) + ranges = append(ranges, migrationApplyRange{start, end}) + } + return ranges, nil +} + +func migrationPrimaryParticipates(req *pb.Request, primary []byte) bool { + for _, mutation := range req.GetMutations() { + if mutation != nil && bytes.Equal(mutation.GetKey(), primary) { + return true + } + } + for _, key := range req.GetReadKeys() { + if bytes.Equal(key, primary) { + return true + } + } + return false +} + +func migrationAdmissionHistory(proofs []migrationAdmissionProof, version uint64) (RouteHistory, error) { + routes := make([]distribution.RouteDescriptor, 0, len(proofs)) + seen := make(map[string]bool) + for _, proof := range proofs { + if seen[proof.identity] { + continue + } + seen[proof.identity] = true + routeID := proof.RouteID + if proof.partition == nil { + routeID = proof.JobID + } + route := distribution.RouteDescriptor{RouteID: routeID, Start: proof.RouteStart, End: proof.RouteEnd, + GroupID: proof.LocalGroupID, State: distribution.RouteStateActive, MinWriteTSExclusive: proof.MinWriteTSExclusive} + if proof.Staged { + route.MigrationJobID, route.StagedVisibilityActive = proof.JobID, true + } + routes = append(routes, route) + } + engine := distribution.NewEngine() + if err := engine.ApplySnapshot(distribution.CatalogSnapshot{Version: version, Routes: routes}); err != nil { + return nil, errors.WithStack(err) + } + return WrapDistributionEngine(engine), nil +} diff --git a/kv/migration_serving_test.go b/kv/migration_serving_test.go new file mode 100644 index 000000000..eca575777 --- /dev/null +++ b/kv/migration_serving_test.go @@ -0,0 +1,187 @@ +package kv + +import ( + "bytes" + "context" + "strconv" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func retireServingSource(t *testing.T, st store.MVCCStore) { + t.Helper() + writer, ok := st.(store.MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), store.TargetStagedReadinessState{ + JobID: 7, MigrationJobID: 7, Armed: true, SourceWriteFence: true, SourceReadFence: true, + RouteStart: []byte("m"), RouteEnd: []byte("z"), ExpectedCutoverVersion: 10, + MinWriteTSExclusive: 100, RetentionPinTS: 1, + }, 8)) + proofs, ok := st.(store.MigrationOwnershipStore) + require.True(t, ok) + require.NoError(t, proofs.ClearMigrationStateWithOwnership(t.Context(), store.MigrationOwnershipState{ + Generation: "0123456789abcdef0123456789abcdef", RouteID: 3, DescriptorHash: make([]byte, 32), + JobID: 7, LocalGroupID: 1, OwnerGroupID: 2, CatalogVersion: 10, ConfigurationIndex: 3, + MinWriteTSExclusive: 100, Role: store.MigrationNonOwner, RouteStart: []byte("m"), RouteEnd: []byte("z"), + }, 9)) +} + +func staleServingEngine(t *testing.T, version uint64) *distribution.Engine { + t.Helper() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, version, []distribution.RouteDescriptor{{ + RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 1, State: distribution.RouteStateActive, + }}) + return engine +} + +func TestMigrationServingJoinedVoterRejectsStaleCatalogReadsAndApply(t *testing.T) { + source := store.NewMVCCStore() + require.NoError(t, source.PutAt(t.Context(), []byte("n"), []byte("old-source"), 90, 0)) + retireServingSource(t, source) + snapshot, err := source.Snapshot() + require.NoError(t, err) + var raw bytes.Buffer + _, err = snapshot.WriteTo(&raw) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + for _, version := range []uint64{9, 10, 11} { + t.Run(strconv.FormatUint(version, 10), func(t *testing.T) { + joined, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, joined.Close()) }) + require.NoError(t, joined.Restore(bytes.NewReader(raw.Bytes()))) + engine := staleServingEngine(t, version) + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: {Store: joined}}) + _, err = shards.GetAt(t.Context(), []byte("n"), 200) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, err = shards.GetGroupAt(t.Context(), 1, []byte("n"), 200) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, _, err = shards.LatestCommitTSGroupWithReadFence(t.Context(), []byte("n"), 1, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, _, err = shards.VersionExistsAtOrBeforeGroupWithReadFence(t.Context(), []byte("n"), 1, 200, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, _, err = shards.VersionsExistAtOrBeforeGroupWithReadFence(t.Context(), [][]byte{[]byte("n")}, 1, 200, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, err = shards.ScanAt(t.Context(), []byte("m"), []byte("z"), 10, 200) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, err = shards.ScanKeysAtWithReadFence(t.Context(), []byte("m"), []byte("z"), 10, 200, 1, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) + fsm, ok := NewKvFSMWithHLC(joined, nil, WithRouteHistory(WrapDistributionEngine(engine), 1)).(*kvFSM) + require.True(t, ok) + for _, observed := range []uint64{0, 9, 10, 11} { + err := fsm.applyRequestErr(t.Context(), &pb.Request{Ts: 200, ObservedRouteVersion: EncodeObservedRouteVersion(observed), + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("wrong-owner")}}}) + require.ErrorIs(t, err, ErrRouteCutoverPending) + require.NoError(t, haltApplyOf(applyErrorResponse(err))) + } + require.Equal(t, uint64(100), joined.LastCommitTS()) + value, err := joined.GetAt(t.Context(), []byte("n"), 200) + require.NoError(t, err) + require.Equal(t, []byte("old-source"), value) + }) + } +} + +func TestMigrationServingCoalescedRequestsKeepIndependentResults(t *testing.T) { + st, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + retireServingSource(t, st) + engine := staleServingEngine(t, 10) + fsm, ok := NewKvFSMWithHLC(st, nil, WithRouteHistory(WrapDistributionEngine(engine), 1)).(*kvFSM) + require.True(t, ok) + fsm.SetApplyIndex(20) + cmd, err := marshalRaftCommand([]*pb.Request{ + {Ts: 200, ObservedRouteVersion: EncodeObservedRouteVersion(10), Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("good")}}}, + {Ts: 201, ObservedRouteVersion: EncodeObservedRouteVersion(10), Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("bad")}}}, + {Ts: 202, ObservedRouteVersion: EncodeObservedRouteVersion(10), Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("c"), Value: []byte("after")}}}, + }) + require.NoError(t, err) + response := fsm.Apply(cmd) + require.NoError(t, haltApplyOf(response)) + results, ok := response.(*fsmApplyResponse) + require.True(t, ok) + require.NoError(t, results.results[0]) + require.ErrorIs(t, results.results[1], ErrRouteCutoverPending) + require.NoError(t, results.results[2]) + value, err := st.GetAt(t.Context(), []byte("c"), 202) + require.NoError(t, err) + require.Equal(t, []byte("after"), value) +} + +func TestMigrationServingProofRejectsMixedAndFutureVersions(t *testing.T) { + states := []store.MigrationOwnershipState{{JobID: 8, LocalGroupID: 1, OwnerGroupID: 1, Role: store.MigrationOwner, + Generation: "0123456789abcdef0123456789abcdef", RouteID: 3, ConfigurationIndex: 3, AppliedIndex: 9, DescriptorHash: make([]byte, 32), + Receipts: []store.MigrationOwnershipReceipt{{Operation: store.MigrationOwnershipClear, CatalogVersion: 12, ConfigurationIndex: 3, AppliedIndex: 9, DescriptorHash: make([]byte, 32)}}, + CatalogVersion: 12, RouteStart: []byte("m"), RouteEnd: []byte("z"), MinWriteTSExclusive: 100}} + view := store.MigrationServingState{Ownership: states} + for _, version := range []uint64{11, 13} { + _, err := migrationAdmissionOwnershipProofs(view, []migrationApplyRange{{[]byte("n"), []byte("o")}}, 1, version) + require.ErrorIs(t, err, ErrRouteCutoverPending) + } + _, err := migrationAdmissionOwnershipProofs(view, []migrationApplyRange{{[]byte("n"), []byte("o")}, {[]byte("b"), []byte("c")}}, 1, 12) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, err = migrationAdmissionOwnershipProofs(view, []migrationApplyRange{{[]byte("a"), []byte("z")}}, 1, 12) + require.ErrorIs(t, err, ErrRouteCutoverPending) + proofs, err := migrationAdmissionOwnershipProofs(view, []migrationApplyRange{{[]byte("n"), []byte("o")}}, 1, 12) + require.NoError(t, err) + require.Len(t, proofs, 1) + request := &pb.Request{IsTxn: true, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("b")})}, + {Op: pb.Op_PUT, Key: []byte("n")}, + }} + ranges, err := migrationServingFootprint(request) + require.NoError(t, err) + _, err = migrationAdmissionOwnershipProofs(view, ranges, 1, 12) + require.NoError(t, err, "a remote primary reference does not add a local participant") + request.Mutations = append(request.Mutations, &pb.Mutation{Op: pb.Op_PUT, Key: []byte("b")}) + ranges, err = migrationServingFootprint(request) + require.NoError(t, err) + _, err = migrationAdmissionOwnershipProofs(view, ranges, 1, 12) + require.ErrorIs(t, err, ErrRouteCutoverPending, "a participating primary footprint cannot bypass mixed admission") +} + +func TestMigrationServingMatchingCommitAbortDrain(t *testing.T) { + for _, phase := range []pb.Phase{pb.Phase_COMMIT, pb.Phase_ABORT} { + t.Run(phase.String(), func(t *testing.T) { + fsm := newComposed1FSM(t, staleServingEngine(t, 9), 1) + meta := &pb.Mutation{Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("n"), CommitTS: 60, LockTTLms: defaultTxnLockTTLms})} + mutation := &pb.Mutation{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("prepared")} + require.NoError(t, fsm.handleTxnRequest(context.Background(), &pb.Request{IsTxn: true, Phase: pb.Phase_PREPARE, Ts: 40, Mutations: []*pb.Mutation{meta, mutation}}, 40)) + retireServingSource(t, fsm.store) + request := &pb.Request{IsTxn: true, Phase: phase, Ts: 40, Mutations: []*pb.Mutation{meta, mutation}} + _, err := fsm.withMigrationOwnershipEvidence(t.Context(), request) + require.NoError(t, err, "matching persisted preexisting resolution must pass the new gate") + request.Ts++ + _, err = fsm.withMigrationOwnershipEvidence(t.Context(), request) + require.ErrorIs(t, err, ErrRouteCutoverPending, "mismatched resolution cannot use a blanket bypass") + }) + } +} + +func TestMigrationServingScopedHistoryCarriesOwnershipFloor(t *testing.T) { + fsm := newComposed1FSM(t, staleServingEngine(t, 9), 1) + proofs := migrationAdmissionProofs(store.MigrationServingState{Ownership: []store.MigrationOwnershipState{{ + JobID: 8, LocalGroupID: 1, OwnerGroupID: 1, CatalogVersion: 12, + Role: store.MigrationOwner, Staged: true, RouteStart: []byte("m"), RouteEnd: []byte("z"), MinWriteTSExclusive: 200, + }}}) + history, err := migrationAdmissionHistory(proofs, 12) + require.NoError(t, err) + scoped := withApplyRouteHistory(t.Context(), history) + snapshot, ok := fsm.routeHistoryForContext(scoped).Current() + require.True(t, ok) + require.Equal(t, uint64(12), snapshot.Version()) + route, ok := snapshot.RouteOf([]byte("n")) + require.True(t, ok) + require.Equal(t, uint64(200), route.MinWriteTSExclusive) + require.Equal(t, uint64(8), route.MigrationJobID) + require.True(t, route.StagedVisibilityActive) + current, ok := fsm.routeHistoryForContext(t.Context()).Current() + require.True(t, ok) + require.Equal(t, uint64(9), current.Version(), "request-local proof does not mutate the watcher") +} diff --git a/kv/migrator_lock_drain.go b/kv/migrator_lock_drain.go index ef7a7e531..62adb9e05 100644 --- a/kv/migrator_lock_drain.go +++ b/kv/migrator_lock_drain.go @@ -18,6 +18,7 @@ type TxnLockDrainEntry struct { TTLExpireAt uint64 PrimaryKey []byte IsPrimaryKey bool + CommitTS uint64 } // PendingTxnLocksInRoute scans the txn-lock namespace and filters each lock by @@ -88,5 +89,6 @@ func txnLockDrainEntry(kvp *store.KVPair, filter func([]byte) bool) (TxnLockDrai TTLExpireAt: lock.TTLExpireAt, PrimaryKey: bytes.Clone(lock.PrimaryKey), IsPrimaryKey: lock.IsPrimaryKey, + CommitTS: lock.CommitTS, }, true, nil } diff --git a/kv/migrator_lock_drain_test.go b/kv/migrator_lock_drain_test.go index d790e85c0..753aae6d6 100644 --- a/kv/migrator_lock_drain_test.go +++ b/kv/migrator_lock_drain_test.go @@ -5,6 +5,7 @@ import ( "context" "testing" + "github.com/bootjp/elastickv/internal/s3keys" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" ) @@ -27,6 +28,7 @@ func TestPendingTxnLocksInRouteFiltersLocksByRouteKey(t *testing.T) { TTLExpireAt: 99, PrimaryKey: itemKey, IsPrimaryKey: true, + CommitTS: 101, }) require.NoError(t, st.PutAt(ctx, txnLockKey(itemKey), lock, 1, 0)) require.NoError(t, st.PutAt(ctx, txnLockKey(outsideKey), encodeTxnLock(txnLock{ @@ -40,10 +42,33 @@ func TestPendingTxnLocksInRouteFiltersLocksByRouteKey(t *testing.T) { require.Equal(t, itemKey, pending[0].UserKey) require.Equal(t, uint64(11), pending[0].StartTS) require.Equal(t, uint64(99), pending[0].TTLExpireAt) + require.Equal(t, uint64(101), pending[0].CommitTS) require.True(t, pending[0].IsPrimaryKey) require.Equal(t, txnLockKey(itemKey), pending[0].LockKey) } +func TestPendingTxnLocksInRouteIncludesS3AuxiliaryOwnership(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + start := s3keys.RoutePrefixForBucketAnyGeneration("moving-bucket") + end := append(bytes.Clone(start), 0xff) + keys := [][]byte{s3keys.BucketMetaKey("moving-bucket"), s3keys.BucketGenerationKey("moving-bucket")} + for _, key := range append(keys, s3keys.BucketMetaKey("outside-bucket")) { + require.NoError(t, st.PutAt(ctx, txnLockKey(key), encodeTxnLock(txnLock{StartTS: 11, PrimaryKey: key}), 1, 0)) + } + for _, key := range keys { + require.False(t, keyInMigrationRouteRange(routeKey(key), start, end), "raw auxiliary keys must not accidentally satisfy the test") + } + pending, err := PendingTxnLocksInRoute(ctx, st, start, end, ^uint64(0), 10) + require.NoError(t, err) + var got [][]byte + for _, entry := range pending { + got = append(got, entry.UserKey) + } + require.ElementsMatch(t, keys, got) +} + func TestPendingTxnLocksInRouteHonorsCanceledContext(t *testing.T) { t.Parallel() diff --git a/kv/route_history.go b/kv/route_history.go index 8bc50c58e..94eec1076 100644 --- a/kv/route_history.go +++ b/kv/route_history.go @@ -82,3 +82,21 @@ func (s distributionRouteSnapshot) WriteFencedIntersects(start, end []byte) bool } return false } + +func (s distributionRouteSnapshot) WriteFloorForKey(key []byte) (uint64, bool) { + route, ok := s.snap.RouteOf(key) + if !ok || route.MinWriteTSExclusive == 0 { + return 0, false + } + return route.MinWriteTSExclusive, true +} + +func (s distributionRouteSnapshot) WriteFloorIntersects(start, end []byte) (uint64, bool) { + var maxFloor uint64 + for _, route := range s.snap.IntersectingRoutes(start, end) { + if route.MinWriteTSExclusive > maxFloor { + maxFloor = route.MinWriteTSExclusive + } + } + return maxFloor, maxFloor > 0 +} diff --git a/kv/shard_key.go b/kv/shard_key.go index 0cfd1795f..56ce75976 100644 --- a/kv/shard_key.go +++ b/kv/shard_key.go @@ -52,6 +52,7 @@ const ( sqsMsgDedupPrefix = "!sqs|msg|dedup|" sqsMsgGroupPrefix = "!sqs|msg|group|" sqsMsgByAgePrefix = "!sqs|msg|byage|" + sqsPartitionMarker = "p|" ) var ( @@ -63,7 +64,22 @@ var ( dynamoGSIPrefixBytes = []byte(DynamoGSIPrefix) sqsInternalPrefixBytes = []byte(sqsInternalPrefix) sqsGlobalRouteKey = []byte(sqsRoutePrefix + "global") - sqsConcreteInternalPrefixBytes = [][]byte{ + // sqsPartitionedFamilyPrefixBytes is the shape of an HT-FIFO partitioned + // SQS row: a message family prefix followed by the partition + // discriminator. Only the message families are partitioned; queue + // metadata is not. Membership is a pure prefix test over compile-time + // constants -- no routing map, no process-local config -- so every + // replica answers identically and FSM apply may use it. + // TestPartitionedSQSPrefixesAlign pins these against the adapter's + // SqsPartitioned*Prefix constants. + sqsPartitionedFamilyPrefixBytes = [][]byte{ + []byte(sqsMsgDataPrefix + sqsPartitionMarker), + []byte(sqsMsgVisPrefix + sqsPartitionMarker), + []byte(sqsMsgDedupPrefix + sqsPartitionMarker), + []byte(sqsMsgGroupPrefix + sqsPartitionMarker), + []byte(sqsMsgByAgePrefix + sqsPartitionMarker), + } + sqsConcreteInternalPrefixBytes = [][]byte{ []byte(sqsQueueMetaPrefix), []byte(sqsQueueGenPrefix), []byte(sqsQueueSeqPrefix), @@ -477,6 +493,19 @@ func sqsRouteKey(key []byte) []byte { return sqsGlobalRouteKey } +// IsPartitionedSQSKey reports whether the key carries the HT-FIFO partitioned +// shape. Routing for those rows is decided by the partition resolver rather +// than by a catalog route range, and routeKey collapses every one of them to +// the single global SQS route key. +func IsPartitionedSQSKey(key []byte) bool { + for _, prefix := range sqsPartitionedFamilyPrefixBytes { + if bytes.HasPrefix(key, prefix) { + return true + } + } + return false +} + func hasSQSConcreteInternalPrefix(key []byte) bool { for _, prefix := range sqsConcreteInternalPrefixBytes { if bytes.HasPrefix(key, prefix) { diff --git a/kv/shard_store.go b/kv/shard_store.go index 9c4301a99..a42fbf0b5 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -41,10 +41,12 @@ type ShardStore struct { var ( ErrCrossShardMutationBatchNotSupported = errors.New("cross-shard mutation batches are not supported") + ErrRouteCutoverPending = errors.New("route cutover pending") ErrExplicitGroupStagedVisibilityUnresolved = errors.New("explicit group read cannot resolve staged visibility route") ErrExplicitGroupRouteOwnerMismatch = errors.New("explicit group read does not own the requested key range") ErrReadRouteVersionUnavailable = errors.New("read route version is not locally available") ErrFilesystemPlacementTargetNotFound = errors.New("filesystem placement target group has no routable home slot") + ErrRouteWriteBelowFloor = ErrRouteWriteTimestampTooLow ) var ErrTSOCommitFloorUnavailable = errors.New("tso: authoritative data-group commit floor is unavailable") @@ -82,6 +84,8 @@ func (s *ShardStore) ReadRouteVersion() uint64 { } func (s *ShardStore) awaitReadRouteVersion(ctx context.Context, requested uint64) error { + // Migration floors are proved against the selected route's coherent store + // view before data access, including when no catalog version was requested. if requested == 0 || s.ReadRouteVersion() >= requested { return nil } @@ -246,6 +250,7 @@ func (s *ShardStore) GetAt(ctx context.Context, key []byte, ts uint64) ([]byte, } func (s *ShardStore) GetAtWithReadFence(ctx context.Context, key []byte, ts uint64, groupID uint64, readRouteVersion uint64) ([]byte, error) { + ctx = migrationServingReadContext(ctx, readRouteVersion) if err := s.awaitReadRouteVersion(ctx, readRouteVersion); err != nil { return nil, err } @@ -284,6 +289,10 @@ func (s *ShardStore) GetGroupAt(ctx context.Context, groupID uint64, key []byte, } func (s *ShardStore) getGroupAtWithReadFence(ctx context.Context, groupID uint64, key []byte, ts uint64, readRouteVersion uint64) ([]byte, error) { + ctx = migrationServingReadContext(ctx, readRouteVersion) + if err := s.awaitReadRouteVersion(ctx, readRouteVersion); err != nil { + return nil, err + } g, ok := s.groupForID(groupID) if !ok || g.Store == nil { return nil, store.ErrKeyNotFound @@ -311,6 +320,9 @@ func (s *ShardStore) getGroupAt(ctx context.Context, g *ShardGroup, route distri if isLinearizableRaftLeader(ctx, engineForGroup(g)) { return s.leaderGetAt(ctx, g, route, key, ts) } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return nil, err + } return s.proxyRawGet(ctx, g, key, ts, groupID, readRouteVersion) } @@ -371,6 +383,7 @@ func (s *ShardStore) routeHasVersionAtOrBeforeRemote(ctx context.Context, route // could answer authoritatively; a replica that is no longer the group leader // says no rather than guessing. func (s *ShardStore) VersionExistsAtOrBeforeGroupWithReadFence(ctx context.Context, key []byte, groupID uint64, ts uint64, readRouteVersion uint64) (bool, bool, error) { + ctx = migrationServingReadContext(ctx, readRouteVersion) if err := s.awaitReadRouteVersion(ctx, readRouteVersion); err != nil { return false, false, err } @@ -390,6 +403,9 @@ func (s *ShardStore) routeHasVersionAtOrBefore(ctx context.Context, route distri if !ok || g.Store == nil { return false, true, nil } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return false, true, err + } if engine := engineForGroup(g); engine != nil && !isLinearizableRaftLeader(ctx, engine) { return false, false, nil } @@ -483,6 +499,11 @@ func isLinearizableRaftLeader(ctx context.Context, engine raftengine.LeaderView) } func (s *ShardStore) leaderGetAt(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return nil, err + } if !isTxnInternalKey(key) { if err := s.maybeResolveTxnLock(ctx, g, key, ts); err != nil { return nil, err @@ -492,6 +513,11 @@ func (s *ShardStore) leaderGetAt(ctx context.Context, g *ShardGroup, route distr } func (s *ShardStore) localGetAt(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return nil, err + } if routeHasStagedVisibility(route) { return s.getAtWithStagedVisibility(ctx, g, route, key, ts) } @@ -506,6 +532,200 @@ func routeHasStagedVisibility(route distribution.Route) bool { return route.StagedVisibilityActive && route.MigrationJobID != 0 } +func routeSatisfiesTargetReadiness(route distribution.Route, ready store.TargetStagedReadinessState, catalogVersion uint64) bool { + if !routeRangeIntersects(route.Start, route.End, ready.RouteStart, ready.RouteEnd) { + return false + } + if route.MinWriteTSExclusive < ready.MinWriteTSExclusive { + return false + } + if route.StagedVisibilityActive { + return route.MigrationJobID == ready.MigrationJobID + } + if route.MigrationJobID != 0 { + return false + } + return ready.ExpectedCutoverVersion == 0 || catalogVersion >= ready.ExpectedCutoverVersion +} + +func (s *ShardStore) targetReadyRouteForRange(ctx context.Context, g *ShardGroup, route distribution.Route, start []byte, end []byte) (distribution.Route, error) { + routeStart, routeEnd := readinessRouteRangeForScan(start, end) + return s.targetReadyRouteForRouteRange(ctx, g, route, routeStart, routeEnd) +} + +func (s *ShardStore) verifyTargetReadinessForRange(ctx context.Context, g *ShardGroup, route distribution.Route, start []byte, end []byte) error { + _, err := s.targetReadyRouteForRange(ctx, g, route, start, end) + return err +} + +func (s *ShardStore) targetReadyRouteForRouteRange(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte) (distribution.Route, error) { + routes, err := s.targetReadyRoutesForRouteRange(ctx, g, route, routeStart, routeEnd) + if err != nil { + return route, err + } + if len(routes) == 1 { + return routes[0], nil + } + return route, nil +} + +func (s *ShardStore) targetReadyRoutesForRouteRange(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte) ([]distribution.Route, error) { + if g == nil || g.Store == nil { + return []distribution.Route{route}, nil + } + view, err := readMigrationServingState(ctx, g.Store) + if err != nil { + return nil, errors.WithStack(err) + } + if err := s.verifyMigrationServingReadFloor(ctx, view); err != nil { + return nil, err + } + if err := s.verifyMigrationServingRange(view, route, routeStart, routeEnd); err != nil { + return nil, err + } + states := view.Readiness + if sourceReadFenceApplies(states, routeStart, routeEnd) { + return nil, errors.WithStack(ErrRouteCutoverPending) + } + applicable := targetReadinessApplicableStates(states, route, routeStart, routeEnd) + if len(applicable) == 0 { + return []distribution.Route{route}, nil + } + + proofRoutes, catalogVersion, ok := s.readinessProofRoutes(route, routeStart, routeEnd) + if !ok { + return nil, errors.WithStack(ErrRouteCutoverPending) + } + if !readinessProofSatisfiesStates(applicable, proofRoutes, route.GroupID, catalogVersion) { + return nil, errors.WithStack(ErrRouteCutoverPending) + } + return proofRoutes, nil +} + +func targetReadinessApplicableStates( + states []store.TargetStagedReadinessState, + route distribution.Route, + routeStart []byte, + routeEnd []byte, +) []store.TargetStagedReadinessState { + applicable := make([]store.TargetStagedReadinessState, 0, len(states)) + for _, ready := range states { + if ready.SourceWriteFence || ready.SourceReadFence || ready.TrackWrites { + continue + } + if targetReadinessAppliesToRoute(route, routeStart, routeEnd, ready) { + applicable = append(applicable, ready) + } + } + return applicable +} + +func sourceReadFenceApplies(states []store.TargetStagedReadinessState, routeStart []byte, routeEnd []byte) bool { + for _, state := range states { + if state.Armed && state.SourceReadFence && routeRangeIntersects(routeStart, routeEnd, state.RouteStart, state.RouteEnd) { + return true + } + } + return false +} + +func readinessProofSatisfiesStates( + states []store.TargetStagedReadinessState, + routes []distribution.Route, + groupID uint64, + catalogVersion uint64, +) bool { + for _, ready := range states { + if !routesSatisfyTargetReadiness(routes, ready, groupID, catalogVersion) { + return false + } + } + return true +} + +func (s *ShardStore) verifyTargetReadinessForRouteRange(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte) error { + _, err := s.targetReadyRouteForRouteRange(ctx, g, route, routeStart, routeEnd) + return err +} + +func (s *ShardStore) readinessProofRoutes(route distribution.Route, routeStart []byte, routeEnd []byte) ([]distribution.Route, uint64, bool) { + if s == nil || s.engine == nil { + return []distribution.Route{route}, 0, true + } + snap, ok := s.engine.Current() + if !ok { + return nil, 0, false + } + routes := snap.IntersectingRoutes(routeStart, routeEnd) + proof := routes[:0] + for _, candidate := range routes { + if candidate.GroupID != route.GroupID { + continue + } + if (route.Start != nil || route.End != nil || route.RouteID != 0) && + !routeRangeIntersects(candidate.Start, candidate.End, route.Start, route.End) { + continue + } + proof = append(proof, candidate) + } + return proof, snap.Version(), len(proof) > 0 +} + +func targetReadinessAppliesToRoute(route distribution.Route, routeStart []byte, routeEnd []byte, ready store.TargetStagedReadinessState) bool { + if !ready.Armed || !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { + return false + } + if route.RouteID != 0 && !routeRangeIntersects(route.Start, route.End, ready.RouteStart, ready.RouteEnd) { + return false + } + return true +} + +func readinessRouteRange(start []byte, end []byte) ([]byte, []byte) { + if routeStart, routeEnd, ok := s3BucketAuxiliaryRouteRange(start); ok && (end == nil || bytes.Equal(end, nextScanCursor(start))) { + return routeStart, routeEnd + } + routeStart := routeKey(start) + if end == nil { + return routeStart, nil + } + routeEnd := routeKey(end) + if bytes.Compare(routeEnd, routeStart) <= 0 { + routeEnd = nextScanCursor(routeStart) + } + return routeStart, routeEnd +} + +func readinessRouteRangeForScan(start []byte, end []byte) ([]byte, []byte) { + if routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end); ok { + return routeStart, routeEnd + } + if s3BucketAuxiliaryScanBounds(start, end) { + return s3BucketAuxiliaryScanRouteRange(start, end) + } + if routeStart, routeEnd, ok := fskeys.ChunkScanRouteBounds(start, end); ok { + return routeStart, routeEnd + } + return readinessRouteRange(start, end) +} + +func verifyRouteWriteFloor(route distribution.Route, commitTS uint64) error { + if route.MinWriteTSExclusive == 0 || commitTS == 0 || commitTS > route.MinWriteTSExclusive { + return nil + } + return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d", commitTS, route.MinWriteTSExclusive) +} + +func routeRangeIntersects(aStart, aEnd, bStart, bEnd []byte) bool { + if aEnd != nil && bytes.Compare(aEnd, bStart) <= 0 { + return false + } + if bEnd != nil && bytes.Compare(bEnd, aStart) <= 0 { + return false + } + return true +} + func (s *ShardStore) routeForExplicitGroupKey(groupID uint64, key []byte) (distribution.Route, error) { fallback := distribution.Route{GroupID: groupID} if s == nil || s.engine == nil { @@ -620,6 +840,7 @@ func latestMVCCVersionAt(ctx context.Context, st store.MVCCStore, key []byte, ts StartKey: key, EndKey: exactKeyScanEnd(key), MaxCommitTSInclusive: ts, + ReadTS: ts, MaxVersions: 1, MaxScannedBytes: 0, MinCommitTSExclusive: 0, @@ -704,19 +925,23 @@ func (s *ShardStore) ExistsAt(ctx context.Context, key []byte, ts uint64) (bool, // pending.length fast-path during churn. Mirrors LeaderRoutedStore's fix // for codex P1 #796. func (s *ShardStore) CommittedVersionAt(ctx context.Context, key []byte, commitTS uint64) (bool, error) { - g, ok := s.groupForKey(key) + if err := s.awaitReadRouteVersion(ctx, 0); err != nil { + return false, err + } + route, g, ok := s.routeAndGroupForKey(key) if !ok || g.Store == nil { return false, nil } + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return false, err + } // engineForGroup may be nil in test fixtures that wire ShardStore // without raft; preserve the existing local-only fallback there. engine := engineForGroup(g) if engine == nil { - exists, err := g.Store.CommittedVersionAt(ctx, key, commitTS) - if err != nil { - return false, errors.WithStack(err) - } - return exists, nil + return committedVersionAtForRoute(ctx, g.Store, route, key, commitTS) } if !isLinearizableRaftLeader(ctx, engine) && !tryEngineLinearizableFence(ctx, engine) { // Not the linearizable leader for this group AND the ReadIndex @@ -726,7 +951,23 @@ func (s *ShardStore) CommittedVersionAt(ctx context.Context, key []byte, commitT // serialization. return false, nil } - exists, err := g.Store.CommittedVersionAt(ctx, key, commitTS) + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return false, err + } + return committedVersionAtForRoute(ctx, g.Store, route, key, commitTS) +} + +func committedVersionAtForRoute(ctx context.Context, st store.MVCCStore, route distribution.Route, key []byte, commitTS uint64) (bool, error) { + exists, err := st.CommittedVersionAt(ctx, key, commitTS) + if err != nil { + return false, errors.WithStack(err) + } + if exists || !routeHasStagedVisibility(route) { + return exists, nil + } + stagedKey := distribution.MigrationStagedDataKey(route.MigrationJobID, key) + exists, err = st.CommittedVersionAt(ctx, stagedKey, commitTS) if err != nil { return false, errors.WithStack(err) } @@ -759,6 +1000,7 @@ func (s *ShardStore) ScanAt(ctx context.Context, start []byte, end []byte, limit } func (s *ShardStore) ScanAtWithReadFence(ctx context.Context, start []byte, end []byte, limit int, ts uint64, reverse bool, groupID uint64, readRouteVersion uint64, routeStart []byte, routeEnd []byte) ([]*store.KVPair, error) { + ctx = migrationServingReadContext(ctx, readRouteVersion) if limit <= 0 { return []*store.KVPair{}, nil } @@ -775,6 +1017,10 @@ func (s *ShardStore) ScanAtWithReadFence(ctx context.Context, start []byte, end } func (s *ShardStore) scanAtWithReadFence(ctx context.Context, start []byte, end []byte, limit int, ts uint64, groupID uint64, readRouteVersion uint64, routeStart []byte, routeEnd []byte) ([]*store.KVPair, error) { + ctx = migrationServingReadContext(ctx, readRouteVersion) + if err := s.awaitReadRouteVersion(ctx, readRouteVersion); err != nil { + return nil, err + } if limit <= 0 { return []*store.KVPair{}, nil } @@ -804,6 +1050,7 @@ func (s *ShardStore) ScanKeysAt(ctx context.Context, start []byte, end []byte, l } func (s *ShardStore) ScanKeysAtWithReadFence(ctx context.Context, start []byte, end []byte, limit int, ts uint64, groupID uint64, readRouteVersion uint64) ([][]byte, error) { + ctx = migrationServingReadContext(ctx, readRouteVersion) if limit <= 0 { return [][]byte{}, nil } @@ -880,6 +1127,9 @@ func (s *ShardStore) canonicalizeRedisWideColumnScanKeys(ctx context.Context, ke } func (s *ShardStore) ScanAtPhysicalLimit(ctx context.Context, start []byte, end []byte, visibleLimit, physicalLimit int, ts uint64) ([]*store.KVPair, bool, error) { + if err := s.awaitReadRouteVersion(ctx, 0); err != nil { + return nil, false, err + } if visibleLimit <= 0 || physicalLimit <= 0 { return []*store.KVPair{}, false, nil } @@ -927,6 +1177,9 @@ func (s *ShardStore) ScanGroupAt(ctx context.Context, groupID uint64, start []by // on any other node the read is forwarded, and the leader serves it as // ScanGroupAt, which may refuse such a range. func (s *ShardStore) ScanGroupOwnedAt(ctx context.Context, groupID uint64, start []byte, end []byte, limit int, ts uint64) ([]*store.KVPair, []byte, error) { + if err := s.awaitReadRouteVersion(ctx, 0); err != nil { + return nil, nil, err + } if limit <= 0 { return []*store.KVPair{}, nil, nil } @@ -1010,6 +1263,10 @@ func (s *ShardStore) ownerHoldsNoVersionAt(ctx context.Context, owner distributi } func (s *ShardStore) scanExplicitGroupAtWithReadFence(ctx context.Context, groupID uint64, start []byte, end []byte, limit int, ts uint64, reverse bool, readRouteVersion uint64, routeStart []byte, routeEnd []byte) ([]*store.KVPair, error) { + ctx = migrationServingReadContext(ctx, readRouteVersion) + if err := s.awaitReadRouteVersion(ctx, readRouteVersion); err != nil { + return nil, err + } if limit <= 0 { return []*store.KVPair{}, nil } @@ -1017,6 +1274,10 @@ func (s *ShardStore) scanExplicitGroupAtWithReadFence(ctx context.Context, group if err != nil { return nil, err } + readinessStart, readinessEnd := readinessRouteRangeForScan(start, end) + if err := s.verifyExplicitGroupRoutesForRange(ctx, groupID, routes, readinessStart, readinessEnd); err != nil { + return nil, err + } routeFilterPresent := routeScanBoundsPresent(routeStart, routeEnd) dedupeByKey := s3BucketAuxiliaryScanBounds(start, end) if !clampToRoutes && !routeFilterPresent { @@ -1074,6 +1335,9 @@ func (s *ShardStore) ReverseScanGroupAt(ctx context.Context, groupID uint64, sta // ScanGroupKeysAt scans keys on the explicitly selected Raft group without // materializing values over proxy links. func (s *ShardStore) ScanGroupKeysAt(ctx context.Context, groupID uint64, start []byte, end []byte, limit int, ts uint64) ([][]byte, error) { + if err := s.awaitReadRouteVersion(ctx, 0); err != nil { + return nil, err + } if limit <= 0 { return [][]byte{}, nil } @@ -1085,6 +1349,10 @@ func (s *ShardStore) ReverseScanAt(ctx context.Context, start []byte, end []byte } func (s *ShardStore) reverseScanAtWithReadFence(ctx context.Context, start []byte, end []byte, limit int, ts uint64, readRouteVersion uint64, routeStart []byte, routeEnd []byte) ([]*store.KVPair, error) { + ctx = migrationServingReadContext(ctx, readRouteVersion) + if err := s.awaitReadRouteVersion(ctx, readRouteVersion); err != nil { + return nil, err + } if limit <= 0 { return []*store.KVPair{}, nil } @@ -1102,6 +1370,9 @@ func (s *ShardStore) reverseScanAtWithReadFence(ctx context.Context, start []byt } func (s *ShardStore) ReverseScanAtPhysicalLimit(ctx context.Context, start []byte, end []byte, visibleLimit, physicalLimit int, ts uint64) ([]*store.KVPair, bool, error) { + if err := s.awaitReadRouteVersion(ctx, 0); err != nil { + return nil, false, err + } if visibleLimit <= 0 || physicalLimit <= 0 { return []*store.KVPair{}, false, nil } @@ -2077,6 +2348,15 @@ func (s *ShardStore) scanKeyRouteAtWithReadFence( return nil, nil } + // keys_only reads need the same readiness proof as value scans. Without it + // an armed source read fence still hands back old-source keys, and an armed + // target guard hands back live-only or empty keys while the catalog watcher + // lags -- the exact windows scanRouteAtDirection proves its way out of. + route, err := s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { + return nil, err + } + if engineForGroup(g) == nil { if routeHasStagedVisibility(route) { return scanKeysWithRefill(start, end, limit, func(cursor []byte, pageLimit int) ([][]byte, error) { @@ -2691,6 +2971,11 @@ func (s *ShardStore) routeHasVersionsAtOrBefore( if !ok || g.Store == nil { return out, true, nil } + for _, key := range keys { + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return out, true, err + } + } if engine := engineForGroup(g); engine != nil && !isLinearizableRaftLeader(ctx, engine) { return out, false, nil } @@ -2928,6 +3213,9 @@ func (s *ShardStore) scanRouteAtDirectionWithReadFenceRouteFilterProxyPage( routeStart []byte, routeEnd []byte, ) ([]*store.KVPair, []*store.KVPair, error) { + if err := s.verifyTargetReadinessForRouteRange(ctx, g, route, routeStart, routeEnd); err != nil { + return nil, nil, err + } groupID := proxyScanGroupID(route, explicitGroup, readRouteVersion, routeStart, routeEnd) kvs, err := s.proxyRawScanAt(ctx, g, start, end, limit, ts, reverse, groupID, readRouteVersion, routeStart, routeEnd) if err != nil { @@ -3014,6 +3302,9 @@ func (s *ShardStore) scanRouteAtReverse( return markScanRouteGroup(kvs, route.GroupID, markRouteGroup), err } + if err := s.verifyTargetReadinessForRange(ctx, g, route, start, end); err != nil { + return nil, err + } groupID := proxyScanGroupID(route, explicitGroup, readRouteVersion, routeStart, routeEnd) kvs, err := s.proxyRawScanAt(ctx, g, start, end, limit, ts, true, groupID, readRouteVersion, routeStart, routeEnd) if err != nil { @@ -3289,6 +3580,9 @@ func (s *ShardStore) scanRouteAtForwardProxyPage( routeStart []byte, routeEnd []byte, ) (scanRoutePage, error) { + if err := s.verifyTargetReadinessForRange(ctx, g, route, start, end); err != nil { + return scanRoutePage{}, err + } groupID := proxyScanGroupID(route, explicitGroup, readRouteVersion, routeStart, routeEnd) raw, err := s.proxyRawScanAt(ctx, g, start, end, limit, ts, false, groupID, readRouteVersion, routeStart, routeEnd) if err != nil { @@ -3321,6 +3615,11 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( return nil, false, nil } markRouteGroup := shouldMarkRouteGroupOnScan(start, false, nil, nil) + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { + return nil, false, err + } if engineForGroup(g) == nil { if routeHasStagedVisibility(route) { @@ -3335,11 +3634,7 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { - if routeHasStagedVisibility(route) { - kvs, err := s.scanRouteAtLeader(ctx, g, route, start, end, visibleLimit, ts, reverse) - return markScanRouteGroup(kvs, route.GroupID, markRouteGroup), false, err - } - kvs, limitReached, err := s.scanRouteAtLeaderPhysicalLimit(ctx, g, route, start, end, visibleLimit, physicalLimit, ts, reverse) + kvs, limitReached, err := s.scanReadyLeaderPhysicalLimit(ctx, g, route, start, end, visibleLimit, physicalLimit, ts, reverse) return markScanRouteGroup(kvs, route.GroupID, markRouteGroup), limitReached, err } @@ -3348,6 +3643,28 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( return nil, true, nil } +func (s *ShardStore) scanReadyLeaderPhysicalLimit( + ctx context.Context, + g *ShardGroup, + route distribution.Route, + start []byte, + end []byte, + visibleLimit int, + physicalLimit int, + ts uint64, + reverse bool, +) ([]*store.KVPair, bool, error) { + route, err := s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { + return nil, false, err + } + if routeHasStagedVisibility(route) { + kvs, err := s.scanRouteAtLeader(ctx, g, route, start, end, visibleLimit, ts, reverse) + return kvs, false, err + } + return s.scanRouteAtLeaderPhysicalLimit(ctx, g, route, start, end, visibleLimit, physicalLimit, ts, reverse) +} + func scanLocalPhysicalLimit( ctx context.Context, st store.MVCCStore, @@ -3398,6 +3715,11 @@ func (s *ShardStore) scanRouteLocal( ts uint64, reverse bool, ) ([]*store.KVPair, error) { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { + return nil, err + } if routeHasStagedVisibility(route) { return s.scanRouteWithStagedVisibility(ctx, g, route, start, end, limit, ts, reverse) } @@ -3420,6 +3742,11 @@ func (s *ShardStore) scanRouteAtLeaderPhysicalLimit( ts uint64, reverse bool, ) ([]*store.KVPair, bool, error) { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { + return nil, false, err + } kvs, limitReached, err := scanLocalPhysicalLimit(ctx, g.Store, start, end, visibleLimit, physicalLimit, ts, reverse) if err != nil { return nil, limitReached, errors.WithStack(err) @@ -3443,10 +3770,12 @@ func (s *ShardStore) scanRouteAtLeader( ts uint64, reverse bool, ) ([]*store.KVPair, error) { - var ( - kvs []*store.KVPair - err error - ) + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { + return nil, err + } + var kvs []*store.KVPair switch { case routeHasStagedVisibility(route): kvs, err = s.scanRouteWithStagedVisibility(ctx, g, route, start, end, limit, ts, reverse) @@ -4208,9 +4537,14 @@ func clampScanEnd(end []byte, routeEnd []byte) []byte { func (s *ShardStore) PutAt(ctx context.Context, key []byte, value []byte, commitTS uint64, expireAt uint64) error { route, g, ok := s.routeAndGroupForKey(key) - if !ok || g.Store == nil { + if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return err + } if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { return err } @@ -4222,9 +4556,14 @@ func (s *ShardStore) PutAt(ctx context.Context, key []byte, value []byte, commit func (s *ShardStore) DeleteAt(ctx context.Context, key []byte, commitTS uint64) error { route, g, ok := s.routeAndGroupForKey(key) - if !ok || g.Store == nil { + if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return err + } if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { return err } @@ -4236,9 +4575,14 @@ func (s *ShardStore) DeleteAt(ctx context.Context, key []byte, commitTS uint64) func (s *ShardStore) PutWithTTLAt(ctx context.Context, key []byte, value []byte, commitTS uint64, expireAt uint64) error { route, g, ok := s.routeAndGroupForKey(key) - if !ok || g.Store == nil { + if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return err + } if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { return err } @@ -4250,9 +4594,14 @@ func (s *ShardStore) PutWithTTLAt(ctx context.Context, key []byte, value []byte, func (s *ShardStore) ExpireAt(ctx context.Context, key []byte, expireAt uint64, commitTS uint64) error { route, g, ok := s.routeAndGroupForKey(key) - if !ok || g.Store == nil { + if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return err + } if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { return err } @@ -4309,6 +4658,7 @@ func (s *ShardStore) LatestCommitTS(ctx context.Context, key []byte) (uint64, bo } func (s *ShardStore) LatestCommitTSWithReadFence(ctx context.Context, key []byte, readRouteVersion uint64) (uint64, bool, error) { + ctx = migrationServingReadContext(ctx, readRouteVersion) if err := s.awaitReadRouteVersion(ctx, readRouteVersion); err != nil { return 0, false, err } @@ -4339,10 +4689,14 @@ func (s *ShardStore) LatestCommitTSWithReadFence(ctx context.Context, key []byte } } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return 0, false, err + } return s.proxyLatestCommitTS(ctx, g, key, readRouteVersion) } func (s *ShardStore) LatestCommitTSGroupWithReadFence(ctx context.Context, key []byte, groupID uint64, readRouteVersion uint64) (uint64, bool, error) { + ctx = migrationServingReadContext(ctx, readRouteVersion) if groupID == 0 { return s.LatestCommitTSWithReadFence(ctx, key, readRouteVersion) } @@ -4353,23 +4707,31 @@ func (s *ShardStore) LatestCommitTSGroupWithReadFence(ctx context.Context, key [ if !ok || g.Store == nil { return 0, false, nil } + route, err := s.routeForExplicitGroupKey(groupID, key) + if err != nil { + return 0, false, err + } if engineForGroup(g) == nil { - ts, exists, err := g.Store.LatestCommitTS(ctx, key) + ts, exists, err := s.localLatestCommitTS(ctx, g, route, key) return ts, exists, errors.WithStack(err) } if engine := engineForGroup(g); isLeaderEngine(engine) { if _, err := leaseReadEngineCtx(ctx, engine); err == nil { - ts, exists, err := g.Store.LatestCommitTS(ctx, key) + ts, exists, err := s.localLatestCommitTS(ctx, g, route, key) return ts, exists, errors.WithStack(err) } } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return 0, false, err + } return s.proxyLatestCommitTSGroup(ctx, g, key, groupID, readRouteVersion) } func (s *ShardStore) VersionsExistAtOrBeforeGroupWithReadFence(ctx context.Context, keys [][]byte, groupID uint64, ts uint64, readRouteVersion uint64) ([]bool, bool, error) { + ctx = migrationServingReadContext(ctx, readRouteVersion) if groupID == 0 { return s.versionsExistAtOrBeforeWithReadFence(ctx, keys, ts, readRouteVersion) } @@ -4380,6 +4742,7 @@ func (s *ShardStore) VersionsExistAtOrBeforeGroupWithReadFence(ctx context.Conte } func (s *ShardStore) versionsExistAtOrBeforeWithReadFence(ctx context.Context, keys [][]byte, ts uint64, readRouteVersion uint64) ([]bool, bool, error) { + ctx = migrationServingReadContext(ctx, readRouteVersion) out := make([]bool, len(keys)) if err := s.awaitReadRouteVersion(ctx, readRouteVersion); err != nil { return nil, false, err @@ -4399,6 +4762,10 @@ func (s *ShardStore) versionsExistAtOrBeforeWithReadFence(ctx context.Context, k } func (s *ShardStore) localLatestCommitTS(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte) (uint64, bool, error) { + route, err := s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return 0, false, err + } if !routeHasStagedVisibility(route) { liveTS, liveExists, err := g.Store.LatestCommitTS(ctx, key) return liveTS, liveExists, errors.WithStack(err) @@ -5143,11 +5510,48 @@ func (s *ShardStore) ApplyMutations(ctx context.Context, mutations []*store.KVPa if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { return err } + if err := s.verifyMutationRoutes(ctx, mutations, readKeys, commitTS); err != nil { + return err + } readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) readKeys = s.readKeysWithStagedVisibilityMutationAliases(group, readKeys, mutations) return errors.WithStack(group.Store.ApplyMutations(ctx, mutations, readKeys, startTS, commitTS)) } +func (s *ShardStore) verifyMutationRoutes(ctx context.Context, mutations []*store.KVPairMutation, readKeys [][]byte, commitTS uint64) error { + for _, mut := range mutations { + if err := s.verifyMutationWriteRoute(ctx, mut.Key, commitTS); err != nil { + return err + } + } + for _, key := range readKeys { + route, g, ok := s.routeAndGroupForKey(key) + if !ok || g == nil || g.Store == nil { + return store.ErrNotSupported + } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return err + } + } + return nil +} + +func (s *ShardStore) verifyMutationWriteRoute(ctx context.Context, key []byte, commitTS uint64) error { + route, g, ok := s.routeAndGroupForKey(key) + if !ok || g == nil || g.Store == nil { + return store.ErrNotSupported + } + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return err + } + if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { + return err + } + return s.ensureS3BucketAuxiliaryWriteTimestampFloor(key, commitTS) +} + // ApplyMutationsRaft is the raft-apply variant; see store.MVCCStore for the // durability contract. Only the FSM may call this method. func (s *ShardStore) ApplyMutationsRaft(ctx context.Context, mutations []*store.KVPairMutation, readKeys [][]byte, startTS, commitTS uint64) error { @@ -5155,9 +5559,6 @@ func (s *ShardStore) ApplyMutationsRaft(ctx context.Context, mutations []*store. if err != nil || group == nil { return err } - if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { - return err - } readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) readKeys = s.readKeysWithStagedVisibilityMutationAliases(group, readKeys, mutations) return errors.WithStack(group.Store.ApplyMutationsRaft(ctx, mutations, readKeys, startTS, commitTS)) @@ -5171,9 +5572,6 @@ func (s *ShardStore) ApplyMutationsRaftAt(ctx context.Context, mutations []*stor if err != nil || group == nil { return err } - if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { - return err - } readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) readKeys = s.readKeysWithStagedVisibilityMutationAliases(group, readKeys, mutations) return errors.WithStack(group.Store.ApplyMutationsRaftAt(ctx, mutations, readKeys, startTS, commitTS, appliedIndex)) @@ -5471,19 +5869,113 @@ func (s *ShardStore) resolveSingleShardGroup(mutations []*store.KVPairMutation) // DeletePrefixAt applies a prefix delete to every shard in the store. func (s *ShardStore) DeletePrefixAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { + return s.deletePrefixEverywhere(ctx, prefix, excludePrefix, commitTS, + func(g *ShardGroup, p, exclude []byte) error { + return g.Store.DeletePrefixAt(ctx, p, exclude, commitTS) + }) +} + +// DeletePrefixAtRaft is the raft-apply variant of DeletePrefixAt. +func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { + return s.deletePrefixEverywhere(ctx, prefix, excludePrefix, commitTS, + func(g *ShardGroup, p, exclude []byte) error { + return g.Store.DeletePrefixAtRaft(ctx, p, exclude, commitTS) + }) +} + +// deletePrefixEverywhere is the shared body of DeletePrefixAt and +// DeletePrefixAtRaft: prove the floors and migration readiness once, then let +// each group delete its staged aliases before the live prefix. Only the +// per-group delete call differs between the two. +func (s *ShardStore) deletePrefixEverywhere( + ctx context.Context, + prefix []byte, + excludePrefix []byte, + commitTS uint64, + deleteOne func(g *ShardGroup, prefix, excludePrefix []byte) error, +) error { if err := s.ensurePrefixWriteTimestampFloors(prefix, commitTS); err != nil { return err } - for _, g := range s.groups { + routes, err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS) + if err != nil { + return err + } + for groupID, g := range s.groups { if g == nil || g.Store == nil { continue } - if err := g.Store.DeletePrefixAt(ctx, prefix, excludePrefix, commitTS); err != nil { + deleteStagedPrefix := func(stagedPrefix []byte, stagedExcludePrefix []byte) error { + return deleteOne(g, stagedPrefix, stagedExcludePrefix) + } + if err := deleteStagedVisibilityPrefixes(routesForGroupID(routes, groupID), prefix, excludePrefix, deleteStagedPrefix); err != nil { + return err + } + if err := deleteOne(g, prefix, excludePrefix); err != nil { return errors.WithStack(err) } } - for _, del := range s.stagedVisibilityPrefixDeletes(prefix, excludePrefix) { - if err := del.group.Store.DeletePrefixAt(ctx, del.prefix, del.excludePrefix, commitTS); err != nil { + return nil +} + +// DeletePrefixAtRaftAt is the raft-entry-index-aware variant. The +// caller's raft entry index applies only to the local group whose +// FSM is driving this apply; on a multi-group ShardStore, fanning +// the SAME index across other groups would corrupt their +// metaAppliedIndex. The single-group case (the common case for an +// FSM-local DeletePrefixAtRaft path) gets the correct bundling; the +// multi-group broadcast case is treated as "passive" — peer groups +// receive the prefix-delete without a meta-key bump (their own raft +// applies will catch up the index on the next mutation). +// +// In practice the FSM call sites that issue raft-DeletePrefix +// operate against a single group's store; the multi-group ShardStore +// is the receiver only when an aggregate (admin / coordinator) path +// is replaying a global FLUSHALL, which is not raft-applied. +func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS, appliedIndex uint64) error { + return s.DeletePrefixesAtRaftAt(ctx, []store.PrefixDelete{{Prefix: prefix, ExcludePrefix: excludePrefix}}, commitTS, appliedIndex) +} + +func (s *ShardStore) DeletePrefixesAtRaftAt(ctx context.Context, deletes []store.PrefixDelete, commitTS, appliedIndex uint64) error { + if len(deletes) == 0 { + return nil + } + stagedByGroup := make(map[*ShardGroup][]store.PrefixDelete) + for _, del := range deletes { + // Migration readiness and source fences are proved before any floor or + // staged planning: a prefix delete that sweeps a range mid-cutover must + // be refused, not merely floor-checked. + if _, err := s.verifyPrefixDeleteRoutes(ctx, del.Prefix, commitTS); err != nil { + return err + } + if err := s.ensurePrefixWriteTimestampFloors(del.Prefix, commitTS); err != nil { + return err + } + for _, staged := range s.stagedVisibilityPrefixDeletes(del.Prefix, del.ExcludePrefix) { + stagedByGroup[staged.group] = append(stagedByGroup[staged.group], store.PrefixDelete{ + Prefix: staged.prefix, + ExcludePrefix: staged.excludePrefix, + }) + } + } + for _, g := range s.groups { + if g == nil || g.Store == nil { + continue + } + groupDeletes := make([]store.PrefixDelete, 0, len(deletes)+len(stagedByGroup[g])) + groupDeletes = append(groupDeletes, deletes...) + groupDeletes = append(groupDeletes, stagedByGroup[g]...) + // Pass appliedIndex through to every group. In the + // single-group call-path (the production raft-apply case) + // this is correct: appliedIndex IS that group's raft entry + // index. In a hypothetical multi-group call, only one group + // would see the matching index and the rest would treat it + // as a non-monotonic stray write — but the rest of the + // raft-apply contract (single FSM per raft log) makes that + // case impossible to reach in production. Tests that + // exercise ShardStore.DeletePrefixAtRaftAt across multiple + // groups MUST pass appliedIndex=0 to opt out. + if err := g.Store.DeletePrefixesAtRaftAt(ctx, groupDeletes, commitTS, appliedIndex); err != nil { return errors.WithStack(err) } } @@ -5496,6 +5988,19 @@ type stagedVisibilityPrefixDelete struct { excludePrefix []byte } +func (s *ShardStore) ensurePrefixWriteTimestampFloors(prefix []byte, commitTS uint64) error { + if s == nil || s.engine == nil || commitTS == 0 { + return nil + } + start, end := routePrefixRange(prefix) + for _, route := range s.engine.GetIntersectingRoutes(start, end) { + if route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "prefix %q route range [%q,%q) commit_ts=%d floor=%d", prefix, start, end, commitTS, route.MinWriteTSExclusive) + } + } + return nil +} + func (s *ShardStore) stagedVisibilityPrefixDeletes(prefix []byte, excludePrefix []byte) []stagedVisibilityPrefixDelete { if s == nil || s.engine == nil { return nil @@ -5527,92 +6032,116 @@ func (s *ShardStore) stagedVisibilityPrefixDeletes(prefix []byte, excludePrefix return out } -func (s *ShardStore) ensurePrefixWriteTimestampFloors(prefix []byte, commitTS uint64) error { - if s == nil || s.engine == nil || commitTS == 0 { - return nil +func (s *ShardStore) verifyPrefixDeleteRoutes(ctx context.Context, prefix []byte, commitTS uint64) ([]distribution.Route, error) { + if s == nil || s.engine == nil { + return nil, nil } - start, end := routePrefixRange(prefix) - for _, route := range s.engine.GetIntersectingRoutes(start, end) { - if route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { - return errors.Wrapf(ErrRouteWriteTimestampTooLow, "prefix %q route range [%q,%q) commit_ts=%d floor=%d", prefix, start, end, commitTS, route.MinWriteTSExclusive) - } + routeStart, routeEnd := routePrefixRange(prefix) + routes := s.engine.GetIntersectingRoutes(routeStart, routeEnd) + if len(routes) == 0 { + return nil, errors.WithStack(ErrRouteCutoverPending) } - return nil + coverageRequired, err := s.prefixDeleteRequiresCoverage(ctx, routes, routeStart, routeEnd) + if err != nil { + return nil, err + } + return s.verifiedPrefixDeleteRoutes(ctx, routes, routeStart, routeEnd, commitTS, coverageRequired) } -// DeletePrefixAtRaft is the raft-apply variant of DeletePrefixAt. -func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { - if err := s.ensurePrefixWriteTimestampFloors(prefix, commitTS); err != nil { - return err - } - for _, g := range s.groups { - if g == nil || g.Store == nil { - continue +func (s *ShardStore) verifiedPrefixDeleteRoutes(ctx context.Context, routes []distribution.Route, routeStart, routeEnd []byte, commitTS uint64, coverageRequired bool) ([]distribution.Route, error) { + verified := make([]distribution.Route, 0, len(routes)) + cursor := routeStart + for _, route := range routes { + start, end := prefixDeleteRouteIntersection(route, routeStart, routeEnd) + if coverageRequired && bytes.Compare(start, cursor) > 0 { + return nil, errors.WithStack(ErrRouteCutoverPending) } - if err := g.Store.DeletePrefixAtRaft(ctx, prefix, excludePrefix, commitTS); err != nil { - return errors.WithStack(err) + proofRoutes, err := s.verifyPrefixDeleteRoute(ctx, route, start, end, commitTS) + if err != nil { + return nil, err + } + verified = append(verified, proofRoutes...) + if len(end) == 0 || bytes.Equal(end, routeEnd) { + return verified, nil } + cursor = end } - for _, del := range s.stagedVisibilityPrefixDeletes(prefix, excludePrefix) { - if err := del.group.Store.DeletePrefixAtRaft(ctx, del.prefix, del.excludePrefix, commitTS); err != nil { - return errors.WithStack(err) + if coverageRequired { + return nil, errors.WithStack(ErrRouteCutoverPending) + } + return verified, nil +} + +func (s *ShardStore) prefixDeleteRequiresCoverage(ctx context.Context, routes []distribution.Route, start, end []byte) (bool, error) { + for _, route := range routes { + g, ok := s.groupForID(route.GroupID) + if !ok || g == nil || g.Store == nil { + return false, store.ErrNotSupported + } + view, err := readMigrationServingState(ctx, g.Store) + if err != nil { + return false, err + } + if migrationApplyIntersects(view, []migrationApplyRange{{start: start, end: end}}) { + return true, nil } } - return nil + return false, nil } -// DeletePrefixAtRaftAt is the raft-entry-index-aware variant. The -// caller's raft entry index applies only to the local group whose -// FSM is driving this apply; on a multi-group ShardStore, fanning -// the SAME index across other groups would corrupt their -// metaAppliedIndex. The single-group case (the common case for an -// FSM-local DeletePrefixAtRaft path) gets the correct bundling; the -// multi-group broadcast case is treated as "passive" — peer groups -// receive the prefix-delete without a meta-key bump (their own raft -// applies will catch up the index on the next mutation). -// -// In practice the FSM call sites that issue raft-DeletePrefix -// operate against a single group's store; the multi-group ShardStore -// is the receiver only when an aggregate (admin / coordinator) path -// is replaying a global FLUSHALL, which is not raft-applied. -func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS, appliedIndex uint64) error { - return s.DeletePrefixesAtRaftAt(ctx, []store.PrefixDelete{{Prefix: prefix, ExcludePrefix: excludePrefix}}, commitTS, appliedIndex) +func prefixDeleteRouteIntersection(route distribution.Route, start, end []byte) ([]byte, []byte) { + if bytes.Compare(route.Start, start) > 0 { + start = route.Start + } + if len(end) == 0 || (len(route.End) > 0 && bytes.Compare(route.End, end) < 0) { + end = route.End + } + return start, end } -func (s *ShardStore) DeletePrefixesAtRaftAt(ctx context.Context, deletes []store.PrefixDelete, commitTS, appliedIndex uint64) error { - if len(deletes) == 0 { - return nil +func (s *ShardStore) verifyPrefixDeleteRoute(ctx context.Context, route distribution.Route, routeStart []byte, routeEnd []byte, commitTS uint64) ([]distribution.Route, error) { + g, ok := s.groupForID(route.GroupID) + if !ok || g == nil || g.Store == nil { + return nil, store.ErrNotSupported } - stagedByGroup := make(map[*ShardGroup][]store.PrefixDelete) - for _, del := range deletes { - if err := s.ensurePrefixWriteTimestampFloors(del.Prefix, commitTS); err != nil { - return err + proofRoutes, err := s.targetReadyRoutesForRouteRange(ctx, g, route, routeStart, routeEnd) + if err != nil { + return nil, err + } + for _, proofRoute := range proofRoutes { + if err := verifyRouteWriteFloor(proofRoute, commitTS); err != nil { + return nil, err } - for _, staged := range s.stagedVisibilityPrefixDeletes(del.Prefix, del.ExcludePrefix) { - stagedByGroup[staged.group] = append(stagedByGroup[staged.group], store.PrefixDelete{ - Prefix: staged.prefix, - ExcludePrefix: staged.excludePrefix, - }) + } + return proofRoutes, nil +} + +func routesForGroupID(routes []distribution.Route, groupID uint64) []distribution.Route { + out := make([]distribution.Route, 0, len(routes)) + for _, route := range routes { + if route.GroupID == groupID { + out = append(out, route) } } - for _, g := range s.groups { - if g == nil || g.Store == nil { + return out +} + +func deleteStagedVisibilityPrefixes(routes []distribution.Route, prefix []byte, excludePrefix []byte, deletePrefix func([]byte, []byte) error) error { + seen := make(map[uint64]struct{}) + for _, route := range routes { + if !routeHasStagedVisibility(route) { continue } - groupDeletes := make([]store.PrefixDelete, 0, len(deletes)+len(stagedByGroup[g])) - groupDeletes = append(groupDeletes, deletes...) - groupDeletes = append(groupDeletes, stagedByGroup[g]...) - // Pass appliedIndex through to every group. In the - // single-group call-path (the production raft-apply case) - // this is correct: appliedIndex IS that group's raft entry - // index. In a hypothetical multi-group call, only one group - // would see the matching index and the rest would treat it - // as a non-monotonic stray write — but the rest of the - // raft-apply contract (single FSM per raft log) makes that - // case impossible to reach in production. Tests that - // exercise ShardStore.DeletePrefixAtRaftAt across multiple - // groups MUST pass appliedIndex=0 to opt out. - if err := g.Store.DeletePrefixesAtRaftAt(ctx, groupDeletes, commitTS, appliedIndex); err != nil { + if _, ok := seen[route.MigrationJobID]; ok { + continue + } + seen[route.MigrationJobID] = struct{}{} + stagedPrefix := distribution.MigrationStagedDataKey(route.MigrationJobID, prefix) + var stagedExcludePrefix []byte + if len(excludePrefix) > 0 { + stagedExcludePrefix = distribution.MigrationStagedDataKey(route.MigrationJobID, excludePrefix) + } + if err := deletePrefix(stagedPrefix, stagedExcludePrefix); err != nil { return errors.WithStack(err) } } @@ -5975,6 +6504,22 @@ func (s *ShardStore) groupForKey(key []byte) (*ShardGroup, bool) { return g, ok } +func (s *ShardStore) verifyExplicitGroupRoutesForRange(ctx context.Context, groupID uint64, routes []distribution.Route, routeStart []byte, routeEnd []byte) error { + g, ok := s.groupForID(groupID) + if !ok || g == nil || g.Store == nil { + // A store-less group (the dedicated TSO group) holds no rows, so there + // is no migration readiness to prove and nothing a scan could return. + // Refusing here would turn an empty scan into an error. + return nil + } + for _, route := range routes { + if err := s.verifyTargetReadinessForRouteRange(ctx, g, route, routeStart, routeEnd); err != nil { + return err + } + } + return nil +} + func (s *ShardStore) routeAndGroupForKey(key []byte) (distribution.Route, *ShardGroup, bool) { route, g, _, ok := s.routeAndGroupForKeyWithVersion(key) return route, g, ok diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index c01dc0c9e..ef61d831a 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -15,6 +15,7 @@ import ( "github.com/bootjp/elastickv/internal/s3keys" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" ) @@ -939,6 +940,10 @@ func TestShardStoreExplicitGroupReads_MergeStagedVisibility(t *testing.T) { got, err := st.GetGroupAt(ctx, 1, []byte("b"), 25) require.NoError(t, err) require.Equal(t, []byte("staged-b"), got) + latest, exists, err := st.LatestCommitTSGroupWithReadFence(ctx, []byte("b"), 1, 0) + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, uint64(20), latest) kvs, err := st.ScanGroupAt(ctx, 1, []byte("a"), []byte("z"), 10, 35) require.NoError(t, err) @@ -1003,6 +1008,9 @@ func TestShardStoreExplicitGroupReads_FailClosedWhenRouteMovedToStagedGroup(t *t _, err := st.GetGroupAt(ctx, 1, []byte("b"), 25) require.ErrorIs(t, err, ErrExplicitGroupStagedVisibilityUnresolved) + _, _, err = st.LatestCommitTSGroupWithReadFence(ctx, []byte("b"), 1, 0) + require.ErrorIs(t, err, ErrExplicitGroupStagedVisibilityUnresolved) + _, err = st.ScanGroupAt(ctx, 1, []byte("a"), []byte("z"), 10, 25) require.ErrorIs(t, err, ErrExplicitGroupStagedVisibilityUnresolved) } @@ -1227,22 +1235,6 @@ func TestShardStoreRejectsWritesAtMigrationTimestampFloor(t *testing.T) { require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("ok"), 101, 0)) } -func TestShardStoreRaftApplyRejectsMigrationTimestampFloor(t *testing.T) { - t.Parallel() - - ctx := context.Background() - st, _ := newStagedVisibilityShardStore(t) - - require.ErrorIs(t, st.ApplyMutationsRaft(ctx, []*store.KVPairMutation{ - {Op: store.OpTypePut, Key: []byte("k-raft"), Value: []byte("v")}, - }, nil, 90, 100), ErrRouteWriteTimestampTooLow) - require.ErrorIs(t, st.ApplyMutationsRaftAt(ctx, []*store.KVPairMutation{ - {Op: store.OpTypePut, Key: []byte("k-raft-at"), Value: []byte("v")}, - }, nil, 90, 100, 1), ErrRouteWriteTimestampTooLow) - require.ErrorIs(t, st.DeletePrefixAtRaft(ctx, []byte("k-raft"), nil, 100), ErrRouteWriteTimestampTooLow) - require.ErrorIs(t, st.DeletePrefixAtRaftAt(ctx, []byte("k-raft-at"), nil, 100, 2), ErrRouteWriteTimestampTooLow) -} - type followerProxyEngine struct { leader string } @@ -5062,3 +5054,1031 @@ func TestValidateBackupSnapshotAtFencesCapturedRouteReads(t *testing.T) { require.ErrorIs(t, err, ErrLeaderNotFound, "the captured-route status read must be fenced, not served from the local replica") } + +func applyTargetReadiness(t *testing.T, group *ShardGroup) { + t.Helper() + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) +} + +func applyTargetReadinessState(t *testing.T, group *ShardGroup, state store.TargetStagedReadinessState) { + t.Helper() + writer, ok := group.Store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), state)) +} + +func newReadinessShardStore(t *testing.T, route distribution.RouteDescriptor) (*ShardStore, *ShardGroup) { + t.Helper() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{route}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + return NewShardStore(engine, map[uint64]*ShardGroup{route.GroupID: group}), group +} + +type readinessFenceEngine struct { + state raftengine.State + onLinearizableRead func() +} + +func TestShardStoreCommittedVersionAtChecksStagedVisibilityKey(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + rawKey := []byte("k") + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, rawKey), []byte("staged"), 77, 0)) + + landed, err := st.CommittedVersionAt(ctx, rawKey, 77) + require.NoError(t, err) + require.True(t, landed) +} + +func TestShardStoreCommittedVersionAtChecksTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("k"), []byte("live"), 77, 0)) + + landed, err := st.CommittedVersionAt(ctx, []byte("k"), 77) + require.False(t, landed) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreSourceReadFenceRejectsPointRead(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 10, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 10, + MinWriteTSExclusive: 50, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 40, + }) + require.NoError(t, group.Store.PutAt(ctx, []byte("n"), []byte("live"), 60, 0)) + + _, err := st.GetAt(ctx, []byte("n"), 60) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, _, err = st.LatestCommitTSGroupWithReadFence(ctx, []byte("n"), 1, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, err = st.ScanAt(ctx, []byte("m"), []byte("z"), 60, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +// keys_only reads take a different path than value scans and used to skip the +// readiness proof entirely, so an armed source read fence still handed back +// keys from the old source -- and an armed target guard handed back live-only +// or empty keys while the catalog watcher lagged. Both the routed and the +// explicit-group form must fail closed the same way ScanAt does. +func TestShardStoreSourceReadFenceRejectsKeyOnlyScan(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 10, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 10, + MinWriteTSExclusive: 50, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 40, + }) + require.NoError(t, group.Store.PutAt(ctx, []byte("n"), []byte("live"), 60, 0)) + + _, err := st.ScanKeysAt(ctx, []byte("m"), []byte("z"), 60, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, err = st.ScanKeysAtWithReadFence(ctx, []byte("m"), []byte("z"), 60, 0, 1, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending, + "an explicit source group is exactly the case the fence exists for") +} + +func TestShardStoreSourceWriteFenceKeepsReadsAvailable(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 10, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 10, + MinWriteTSExclusive: 50, + Armed: true, + SourceWriteFence: true, + RetentionPinTS: 40, + }) + require.NoError(t, group.Store.PutAt(ctx, []byte("n"), []byte("live"), 60, 0)) + + got, err := st.GetAt(ctx, []byte("n"), 60) + require.NoError(t, err) + require.Equal(t, []byte("live"), got) +} + +func TestShardStoreCommittedVersionAtRechecksTargetReadinessAfterFence(t *testing.T) { + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + group.Engine = &readinessFenceEngine{ + onLinearizableRead: func() { + applyTargetReadiness(t, group) + }, + } + require.NoError(t, group.Store.PutAt(ctx, []byte("k"), []byte("live"), 77, 0)) + + landed, err := st.CommittedVersionAt(ctx, []byte("k"), 77) + require.False(t, landed) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreTargetReadinessFailsClosedWithoutDescriptorProof(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + + _, err := st.GetAt(ctx, []byte("k"), 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, _, err = st.LatestCommitTSGroupWithReadFence(ctx, []byte("k"), 1, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + err = st.PutAt(ctx, []byte("k"), []byte("v"), 120, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreTargetReadinessNormalizesInternalKeyRange(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + + itemKey := store.ListItemKey([]byte("b"), 0) + require.NoError(t, group.Store.PutAt(ctx, itemKey, []byte("item"), 120, 0)) + + _, err := st.GetAt(ctx, itemKey, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreTargetReadinessAcceptsStagedDescriptor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + applyTargetReadiness(t, group) + rawKey := []byte("k") + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, rawKey), []byte("staged"), 120, 0)) + + got, err := st.GetAt(ctx, rawKey, 130) + require.NoError(t, err) + require.Equal(t, []byte("staged"), got) +} + +func TestShardStoreTargetReadinessAcceptsClearedDescriptorAndRetainsFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("k"), []byte("live"), 120, 0)) + + got, err := st.GetAt(ctx, []byte("k"), 130) + require.NoError(t, err) + require.Equal(t, []byte("live"), got) + + err = st.PutAt(ctx, []byte("k"), []byte("low"), 100, 0) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + err = st.ExpireAt(ctx, []byte("k"), 200, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + err = st.ApplyMutations(ctx, []*store.KVPairMutation{{ + Op: store.OpTypePut, + Key: []byte("n"), + Value: []byte("low"), + }}, nil, 0, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + err = st.PutAt(ctx, []byte("k"), []byte("ok"), 101, 0) + require.NoError(t, err) + + err = st.ApplyMutations(ctx, []*store.KVPairMutation{{ + Op: store.OpTypePut, + Key: []byte("n"), + Value: []byte("ok"), + }}, nil, 0, 101) + require.NoError(t, err) +} + +func TestShardStoreTargetReadinessRejectsClearedDescriptorBeforeCutoverVersion(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("k"), []byte("live"), 120, 0)) + + _, err := st.GetAt(ctx, []byte("k"), 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreTargetReadinessUsesSingleCatalogSnapshotProof(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + staleRoute, ok := engine.GetRoute([]byte("b")) + require.True(t, ok) + group := &ShardGroup{Store: store.NewMVCCStore()} + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("live-old"), 80, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b")), []byte("staged-new"), 120, 0)) + + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }}, + })) + + got, err := shards.localGetAt(ctx, group, staleRoute, []byte("b"), 130) + require.NoError(t, err) + require.Equal(t, []byte("staged-new"), got) +} + +func TestShardStoreDeletePrefixAtUsesReadinessProofRouteFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("live"), 80, 0)) + + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }}, + })) + + err := shards.DeletePrefixAt(ctx, []byte("b"), nil, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("live"), got) +} + +func TestShardStoreDeletePrefixAtUsesReadinessProofRouteForStagedCleanup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadiness(t, group) + stagedKey := distribution.MigrationStagedDataKey(9, []byte("b")) + require.NoError(t, group.Store.PutAt(ctx, stagedKey, []byte("staged"), 120, 0)) + + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }}, + })) + + require.NoError(t, shards.DeletePrefixAt(ctx, []byte("b"), nil, 130)) + + _, err := group.Store.GetAt(ctx, stagedKey, 140) + require.ErrorIs(t, err, store.ErrKeyNotFound) + _, err = shards.GetAt(ctx, []byte("b"), 140) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +func TestShardStoreExplicitGroupReadUsesRouteProofForReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 42, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("live"), 120, 0)) + require.NoError(t, group.Store.PutAt(ctx, []byte("c"), []byte("scan"), 121, 0)) + + got, err := st.GetGroupAt(ctx, 42, []byte("b"), 130) + require.NoError(t, err) + require.Equal(t, []byte("live"), got) + + kvs, err := st.ScanGroupAt(ctx, 42, []byte("a"), []byte("z"), 10, 130) + require.NoError(t, err) + require.Len(t, kvs, 2) +} + +func TestShardStoreScanGroupAtChecksEveryCoveredRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 42, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + { + RouteID: 2, + Start: []byte("m"), + End: []byte("z"), + GroupID: 42, + State: distribution.RouteStateActive, + }, + }, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("left"), 120, 0)) + require.NoError(t, group.Store.PutAt(ctx, []byte("x"), []byte("right"), 120, 0)) + + _, err := st.ScanGroupAt(ctx, 42, []byte("a"), []byte("z"), 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreScanGroupAtClampsSingleMatchedRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("m"), + End: []byte("z"), + GroupID: 42, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("outside"), 7, 0)) + require.NoError(t, group.Store.PutAt(ctx, []byte("n"), []byte("inside"), 7, 0)) + + kvs, err := st.ScanGroupAt(ctx, 42, []byte("a"), []byte("z"), 10, 7) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{{Key: []byte("n"), Value: []byte("inside")}}, kvs) +} + +func TestShardStoreScanGroupAtSplitsCoveredRoutesForStagedVisibility(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 42, + State: distribution.RouteStateActive, + }, + { + RouteID: 2, + Start: []byte("m"), + End: []byte("z"), + GroupID: 42, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 10, + }, + }, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("left-live"), 110, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(10, []byte("x")), []byte("right-staged"), 120, 0)) + + kvs, err := st.ScanGroupAt(ctx, 42, []byte("a"), []byte("z"), 10, 130) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{ + {Key: []byte("b"), Value: []byte("left-live")}, + {Key: []byte("x"), Value: []byte("right-staged")}, + }, kvs) +} + +func TestShardStorePhysicalLimitScanChecksTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + + userKey := []byte("b") + start := store.ListItemKey(userKey, 0) + end := store.ListItemKey(userKey, 2) + require.NoError(t, group.Store.PutAt(ctx, start, []byte("v"), 120, 0)) + + _, _, err := st.ScanAtPhysicalLimit(ctx, start, end, 10, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStorePhysicalLimitScanRechecksTargetReadinessAfterFence(t *testing.T) { + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + group.Engine = &readinessFenceEngine{ + state: raftengine.StateLeader, + onLinearizableRead: func() { + applyTargetReadiness(t, group) + }, + } + + userKey := []byte("b") + start := store.ListItemKey(userKey, 0) + end := store.ListItemKey(userKey, 2) + require.NoError(t, group.Store.PutAt(ctx, start, []byte("v"), 120, 0)) + + _, _, err := st.ScanAtPhysicalLimit(ctx, start, end, 10, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreS3ManifestScanChecksTargetReadinessInRouteSpace(t *testing.T) { + t.Parallel() + + ctx := context.Background() + start := s3keys.ObjectManifestScanStart("bucket-a", 1, "z/") + end := prefixScanEnd(start) + routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end) + require.True(t, ok) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 1, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeEnd, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + + key := s3keys.ObjectManifestKey("bucket-a", 1, "z/object-0") + require.NoError(t, group.Store.PutAt(ctx, key, []byte("manifest"), 120, 0)) + + _, err := st.ScanAt(ctx, start, end, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, _, err = st.ScanAtPhysicalLimit(ctx, start, end, 10, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreS3BucketAuxiliaryScanChecksSourceReadFenceInRouteSpace(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const bucket = "bucket-a" + start := []byte(s3keys.BucketMetaPrefix) + end := prefixScanEnd(start) + routeStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + routeEnd := prefixScanEnd(routeStart) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 42, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeEnd, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 90, + }) + require.NoError(t, group.Store.PutAt(ctx, s3keys.BucketMetaKey(bucket), []byte("meta"), 120, 0)) + + _, err := st.ScanGroupAt(ctx, 42, start, end, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreFilesystemChunkScanChecksSourceReadFenceInRouteSpace(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const homeSlot uint64 = 11 + const inode uint64 = 22 + start := fskeys.ChunkPrefix(homeSlot, inode) + end := prefixScanEnd(start) + routeStart := fskeys.ChunkRouteKey(homeSlot, inode) + routeEnd := prefixScanEnd(routeStart) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 42, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeEnd, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 90, + }) + require.NoError(t, group.Store.PutAt(ctx, fskeys.ChunkKey(homeSlot, inode, 0), []byte("chunk"), 120, 0)) + + _, err := st.ScanAt(ctx, start, end, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, err = st.ScanGroupAt(ctx, 42, start, end, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreTargetReadinessSkipsNonOverlappingGuardForScannedRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + start := s3keys.ObjectManifestScanStart("bucket-a", 1, "a/") + end := s3keys.ObjectManifestScanStart("bucket-a", 1, "z/") + routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end) + require.True(t, ok) + routeBoundary, _, ok := s3keys.ManifestScanRouteBounds( + s3keys.ObjectManifestScanStart("bucket-a", 1, "m/"), + end, + ) + require.True(t, ok) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: routeStart, + End: routeBoundary, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + { + RouteID: 2, + Start: routeBoundary, + End: routeEnd, + GroupID: 1, + State: distribution.RouteStateActive, + }, + }, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeBoundary, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + + kvs, err := st.ScanAt(ctx, start, end, 10, 130) + require.NoError(t, err) + require.Empty(t, kvs) +} + +func TestShardStoreExplicitGroupS3ManifestScanUsesRouteProofForReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + start := s3keys.ObjectManifestScanStart("bucket-a", 1, "z/") + end := prefixScanEnd(start) + routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end) + require.True(t, ok) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 42, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeEnd, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + + key := s3keys.ObjectManifestKey("bucket-a", 1, "z/object-0") + require.NoError(t, group.Store.PutAt(ctx, key, []byte("manifest"), 120, 0)) + + kvs, err := st.ScanGroupAt(ctx, 42, start, end, 10, 130) + require.NoError(t, err) + require.Len(t, kvs, 1) + require.Equal(t, key, kvs[0].Key) +} + +func TestShardStoreDeletePrefixAtChecksTargetReadinessBeforeDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("v"), 1, 0)) + + err := st.DeletePrefixAt(ctx, []byte("b"), nil, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestShardStoreDeletePrefixAtRaftChecksTargetReadinessBeforeDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("v"), 1, 0)) + + err := st.DeletePrefixAtRaft(ctx, []byte("b"), nil, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestShardStoreDeletePrefixAtRaftAtChecksTargetReadinessBeforeDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("v"), 1, 0)) + + err := st.DeletePrefixAtRaftAt(ctx, []byte("b"), nil, 120, 10) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestShardStoreDeletePrefixAtFailsClosedWithoutRouteProof(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("m"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("v"), 1, 0)) + + err := st.DeletePrefixAt(ctx, []byte("b"), nil, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestShardStoreDeletePrefixAtTombstonesStagedVisibilityRows(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + rawKey := []byte("b") + stagedKey := distribution.MigrationStagedDataKey(9, rawKey) + require.NoError(t, group.Store.PutAt(ctx, stagedKey, []byte("staged"), 120, 0)) + + require.NoError(t, st.DeletePrefixAt(ctx, rawKey, nil, 130)) + + _, err := st.GetAt(ctx, rawKey, 140) + require.ErrorIs(t, err, store.ErrKeyNotFound) + _, err = group.Store.GetAt(ctx, stagedKey, 140) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +func TestShardStoreStagedVisibilityPreservesCompactionErrors(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + retention, ok := group.Store.(store.RetentionController) + require.True(t, ok) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b")), []byte("staged"), 10, 0)) + retention.SetMinRetainedTS(20) + + _, err := st.GetAt(ctx, []byte("b"), 15) + require.ErrorIs(t, err, store.ErrReadTSCompacted) + + _, err = st.ScanAt(ctx, []byte("a"), []byte("z"), 10, 15) + require.ErrorIs(t, err, store.ErrReadTSCompacted) +} + +func TestShardStoreRaftApplySkipsPointMigrationTimestampFloorButGuardsPrefixDeletes(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, _ := newStagedVisibilityShardStore(t) + + require.NoError(t, st.ApplyMutationsRaft(ctx, []*store.KVPairMutation{ + {Op: store.OpTypePut, Key: []byte("k-raft"), Value: []byte("v")}, + }, nil, 90, 100)) + require.NoError(t, st.ApplyMutationsRaftAt(ctx, []*store.KVPairMutation{ + {Op: store.OpTypePut, Key: []byte("k-raft-at"), Value: []byte("v")}, + }, nil, 90, 100, 1)) + require.ErrorIs(t, st.DeletePrefixAtRaft(ctx, []byte("k-raft"), nil, 100), ErrRouteWriteTimestampTooLow) + require.ErrorIs(t, st.DeletePrefixAtRaftAt(ctx, []byte("k-raft-at"), nil, 100, 2), ErrRouteWriteTimestampTooLow) +} + +func (e *readinessFenceEngine) State() raftengine.State { + if e.state != "" { + return e.state + } + return raftengine.StateFollower +} + +func (e *readinessFenceEngine) Leader() raftengine.LeaderInfo { + return raftengine.LeaderInfo{ID: "leader"} +} + +func (e *readinessFenceEngine) VerifyLeader(context.Context) error { + return nil +} + +func (e *readinessFenceEngine) LinearizableRead(context.Context) (uint64, error) { + if e.onLinearizableRead != nil { + e.onLinearizableRead() + } + return 1, nil +} + +func (e *readinessFenceEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, errors.New("unexpected propose") +} + +func (e *readinessFenceEngine) ProposeAdmin(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, errors.New("unexpected propose admin") +} + +func (e *readinessFenceEngine) Status() raftengine.Status { + return raftengine.Status{State: e.State()} +} + +func (e *readinessFenceEngine) Configuration(context.Context) (raftengine.Configuration, error) { + return raftengine.Configuration{}, nil +} + +func (e *readinessFenceEngine) Close() error { + return nil +} + +func (e *readinessFenceEngine) SnapshotEvery() uint64 { return 0 } diff --git a/kv/shard_store_txn_lock_test.go b/kv/shard_store_txn_lock_test.go index e85918742..d0f9b314a 100644 --- a/kv/shard_store_txn_lock_test.go +++ b/kv/shard_store_txn_lock_test.go @@ -114,6 +114,36 @@ func TestShardStoreGetAt_ReturnsTxnLockedForPendingLock(t *testing.T) { require.True(t, errors.Is(err, ErrTxnLocked), "expected ErrTxnLocked, got %v", err) } +func TestShardStoreLeaderGetAtChecksReadinessBeforeResolvingPendingLock(t *testing.T) { + t.Parallel() + + ctx := context.Background() + route := distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + } + shardStore, group := newReadinessShardStore(t, route) + applyTargetReadiness(t, group) + + lockTS := uint64(1) + key := []byte("b") + require.NoError(t, group.Store.PutAt(ctx, txnLockKey(key), encodeTxnLock(txnLock{ + StartTS: lockTS, + PrimaryKey: key, + }), lockTS, 0)) + + resolvedRoute, _, ok := shardStore.routeAndGroupForKey(key) + require.True(t, ok) + _, err := shardStore.leaderGetAt(ctx, group, resolvedRoute, key, ^uint64(0)) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, lockErr := group.Store.GetAt(ctx, txnLockKey(key), ^uint64(0)) + require.NoError(t, lockErr) +} + func TestShardStoreGetAt_ReturnsTxnLockedForPendingCrossShardTxn(t *testing.T) { t.Parallel() @@ -677,6 +707,104 @@ func TestShardStoreScanAt_ResolvesCommittedSecondaryLockWithoutCommittedValue(t require.Equal(t, []byte("v2"), kvs[0].Value) } +func TestShardStorePhysicalLimitScanPreservesRouteProofDuringLockResolution(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + + st1 := store.NewMVCCStore() + fsm := NewKvFSMWithHLC(st1, NewHLC(), WithRouteHistory(WrapDistributionEngine(engine), 1)) + applyFSM, ok := fsm.(*kvFSM) + require.True(t, ok) + txn := &localApplyTransactional{fsm: applyFSM} + + groups := map[uint64]*ShardGroup{ + 1: {Store: st1, Txn: txn}, + } + shardStore := NewShardStore(engine, groups) + applyTargetReadiness(t, groups[1]) + + startTS := uint64(101) + commitTS := uint64(120) + primaryKey := []byte("b") + secondaryKey := []byte("c") + require.NoError(t, st1.PutAt(ctx, secondaryKey, []byte("old"), 1, 0)) + prepare := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_PUT, Key: primaryKey, Value: []byte("vp")}, + {Op: pb.Op_PUT, Key: secondaryKey, Value: []byte("vs")}, + }, + } + _, err := groups[1].Txn.Commit(ctx, []*pb.Request{prepare}) + require.NoError(t, err) + commitPrimary := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_COMMIT, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, CommitTS: commitTS})}, + {Op: pb.Op_PUT, Key: primaryKey}, + }, + } + _, err = groups[1].Txn.Commit(ctx, []*pb.Request{commitPrimary}) + require.NoError(t, err) + + route, ok := engine.GetRoute(secondaryKey) + require.True(t, ok) + kvs, _, err := shardStore.scanRouteAtLeaderPhysicalLimit(ctx, groups[1], route, []byte(""), nil, 10, 10, commitTS, false) + require.NoError(t, err) + got := map[string]string{} + for _, kvp := range kvs { + got[string(kvp.Key)] = string(kvp.Value) + } + require.Equal(t, "vs", got[string(secondaryKey)]) + + _, lockErr := st1.GetAt(ctx, txnLockKey(secondaryKey), ^uint64(0)) + require.ErrorIs(t, lockErr, store.ErrKeyNotFound) +} + +type localApplyTransactional struct { + fsm *kvFSM +} + +func (t *localApplyTransactional) Commit(ctx context.Context, reqs []*pb.Request) (*TransactionResponse, error) { + for _, req := range reqs { + if err := t.fsm.handleTxnRequest(ctx, req, txnRequestResolveTS(req)); err != nil { + return nil, err + } + } + return &TransactionResponse{}, nil +} + +func (t *localApplyTransactional) Abort(ctx context.Context, reqs []*pb.Request) (*TransactionResponse, error) { + return t.Commit(ctx, reqs) +} + +func txnRequestResolveTS(req *pb.Request) uint64 { + meta, _, err := extractTxnMeta(req.GetMutations()) + if err == nil && meta.CommitTS != 0 { + return meta.CommitTS + } + return req.GetTs() +} + func TestShardStoreScanAt_FiltersTxnInternalKeysWithoutRaft(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 2f7888edc..18dcc0126 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -976,6 +976,9 @@ func NewShardedCoordinator(engine *distribution.Engine, groups map[uint64]*Shard } } c.configureTimestampAdmissionForGroup(g) + if gid != 0 { + configureMigrationRequestAdmission(g.Txn, g.Store, gid) + } router.Register(gid, g.Txn, g.Store) // Resolve the optional LeaseProvider capability once so // groupLeaseRead / maybeRefresh test g.lp for nil instead of @@ -1466,6 +1469,9 @@ func (c *ShardedCoordinator) dispatchDelPrefixBroadcast(ctx context.Context, isT if err := c.rejectWriteTimestampFloorDelPrefixes(elems, ts); err != nil { return nil, err } + if err := c.rejectDelPrefixesWithoutTargetReadinessProof(ctx, elems, ts); err != nil { + return nil, err + } requests := make([]*pb.Request, 0, len(elems)) for _, elem := range elems { mut := elemToMutation(elem) @@ -1587,6 +1593,22 @@ func (c *ShardedCoordinator) rejectWriteFencedDelPrefixes(elems []*Elem[OP]) err return nil } +func (c *ShardedCoordinator) rejectDelPrefixesWithoutTargetReadinessProof(ctx context.Context, elems []*Elem[OP], commitTS uint64) error { + shards, ok := c.store.(*ShardStore) + if !ok || shards == nil { + return nil + } + for _, elem := range elems { + if elem == nil { + continue + } + if _, err := shards.verifyPrefixDeleteRoutes(ctx, elem.Key, commitTS); err != nil { + return errors.Wrapf(err, "prefix %q", elem.Key) + } + } + return nil +} + func (c *ShardedCoordinator) rejectWriteTimestampFloorPointKey(key []byte, commitTS uint64) error { if c == nil || c.engine == nil || commitTS == 0 { return nil @@ -1673,15 +1695,15 @@ func (c *ShardedCoordinator) broadcastToAllGroups(ctx context.Context, requests errMu sync.Mutex wg sync.WaitGroup ) - groups, err := c.allShardGroups() + participants, err := c.admitBroadcastRequests(ctx, requests) if err != nil { return nil, err } - for _, g := range groups { + for _, participant := range participants { wg.Add(1) - go func(g *ShardGroup) { + go func(participant admittedBroadcastParticipant) { defer wg.Done() - r, err := g.Txn.Commit(ctx, requests) + r, err := participant.group.Txn.Commit(ctx, participant.requests) if err != nil { errMu.Lock() if firstErr == nil { @@ -1698,7 +1720,7 @@ func (c *ShardedCoordinator) broadcastToAllGroups(ctx context.Context, requests } } } - }(g) + }(participant) } wg.Wait() @@ -3088,6 +3110,9 @@ func (c *ShardedCoordinator) validateReadKeysOnShard(ctx context.Context, gid ui return errors.WithStack(err) } for _, key := range keys { + if err := c.verifyTargetReadinessForReadKeyOnShard(ctx, gid, g, key); err != nil { + return err + } ts, exists, err := c.latestCommitTSForReadKeyOnShard(ctx, gid, g, key) if err != nil { return errors.WithStack(err) @@ -3099,6 +3124,50 @@ func (c *ShardedCoordinator) validateReadKeysOnShard(ctx context.Context, gid ui return nil } +func (c *ShardedCoordinator) verifyTargetReadinessForReadKeyOnShard(ctx context.Context, gid uint64, g *ShardGroup, key []byte) error { + view, err := readMigrationServingState(ctx, g.Store) + if err != nil { + return err + } + routeStart, routeEnd := readinessRouteRange(key, nextScanCursor(key)) + serving := &ShardStore{engine: c.engine, groups: c.groups} + if err := serving.verifyMigrationServingFloor(ctx, 0); err != nil { + return err + } + if err := serving.verifyMigrationServingRange(view, distribution.Route{GroupID: gid}, routeStart, routeEnd); err != nil { + return err + } + states := view.Readiness + if len(states) == 0 { + return nil + } + if sourceReadFenceApplies(states, routeStart, routeEnd) { + return errors.WithStack(ErrRouteCutoverPending) + } + routes, catalogVersion, proof := c.currentShardRoutesForRouteRange(gid, routeStart, routeEnd) + if targetReadinessStatesSatisfied(states, routes, routeStart, routeEnd, gid, catalogVersion, proof) { + return nil + } + return errors.WithStack(ErrRouteCutoverPending) +} + +func (c *ShardedCoordinator) currentShardRoutesForRouteRange(gid uint64, routeStart []byte, routeEnd []byte) ([]distribution.Route, uint64, bool) { + if c == nil || c.engine == nil { + return nil, 0, false + } + snap, ok := c.engine.Current() + if !ok { + return nil, 0, false + } + routes := make([]distribution.Route, 0) + for _, route := range snap.IntersectingRoutes(routeStart, routeEnd) { + if route.GroupID == gid { + routes = append(routes, route) + } + } + return routes, snap.Version(), true +} + func (c *ShardedCoordinator) latestCommitTSForReadKeyOnShard(ctx context.Context, gid uint64, g *ShardGroup, key []byte) (uint64, bool, error) { route, ok := c.stagedVisibilityRouteForReadKey(gid, key) if !ok { diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index 0cfda39b5..652190989 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -722,6 +722,70 @@ func TestShardedCoordinatorRejectsDelPrefixIntersectingWriteFencedRoute(t *testi require.Empty(t, g2Txn.requests) } +func TestShardedCoordinatorRejectsDelPrefixBelowRouteFloorBeforeBroadcast(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive, MinWriteTSExclusive: ^uint64(0)}, + }, + })) + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("z")}}, + }) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + require.Empty(t, g1Txn.requests) + require.Empty(t, g2Txn.requests) +} + +func TestShardedCoordinatorRejectsDelPrefixWithoutTargetReadinessProofBeforeBroadcast(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive}, + }, + })) + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{} + g1 := &ShardGroup{Txn: g1Txn, Store: store.NewMVCCStore()} + g2 := &ShardGroup{Txn: g2Txn, Store: store.NewMVCCStore()} + applyTargetReadinessState(t, g2, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: nil, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + groups := map[uint64]*ShardGroup{1: g1, 2: g2} + shardStore := NewShardStore(engine, groups) + coord := NewShardedCoordinator(engine, groups, 1, NewHLC(), shardStore) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("z")}}, + }) + require.ErrorIs(t, err, ErrRouteCutoverPending) + require.Empty(t, g1Txn.requests) + require.Empty(t, g2Txn.requests) +} + func TestShardedCoordinatorRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 2e83593fa..8d674a7d0 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -591,8 +591,8 @@ func TestShardedCoordinatorDispatchTxn_CrossShardPhasesAndCommitIndex(t *testing require.Equal(t, []byte("b"), prepareMeta2.PrimaryKey) require.Equal(t, defaultTxnLockTTLms, prepareMeta1.LockTTLms) require.Equal(t, defaultTxnLockTTLms, prepareMeta2.LockTTLms) - require.Zero(t, prepareMeta1.CommitTS) - require.Zero(t, prepareMeta2.CommitTS) + require.Greater(t, prepareMeta1.CommitTS, startTS) + require.Equal(t, prepareMeta1.CommitTS, prepareMeta2.CommitTS) require.Equal(t, pb.Phase_COMMIT, g1Commit.Phase) require.Equal(t, pb.Phase_COMMIT, g2Commit.Phase) @@ -611,7 +611,7 @@ func TestShardedCoordinatorDispatchTxn_CrossShardPhasesAndCommitIndex(t *testing require.Equal(t, []byte("b"), commitMeta2.PrimaryKey) require.Zero(t, commitMeta1.LockTTLms) require.Zero(t, commitMeta2.LockTTLms) - require.Greater(t, commitMeta1.CommitTS, startTS) + require.Equal(t, prepareMeta1.CommitTS, commitMeta1.CommitTS) require.Equal(t, commitMeta1.CommitTS, commitMeta2.CommitTS) require.Equal(t, commitMeta1.CommitTS, binary.BigEndian.Uint64(g1Prepare.Mutations[1].Value[4:12])) require.Equal(t, commitMeta1.CommitTS, binary.BigEndian.Uint64(g2Prepare.Mutations[1].Value[4:12])) @@ -1158,6 +1158,10 @@ func TestShardedCoordinatorDispatchTxn_UsesProvidedCommitTS(t *testing.T) { commitMeta2 := requestTxnMeta(t, g2Txn.requests[1]) require.Equal(t, commitTS, commitMeta1.CommitTS) require.Equal(t, commitTS, commitMeta2.CommitTS) + prepareMeta1 := requestTxnMeta(t, g1Txn.requests[0]) + prepareMeta2 := requestTxnMeta(t, g2Txn.requests[0]) + require.Equal(t, commitTS, prepareMeta1.CommitTS) + require.Equal(t, commitTS, prepareMeta2.CommitTS) } func TestShardedCoordinatorDispatchTxn_RejectsMigrationTimestampFloor(t *testing.T) { @@ -1362,6 +1366,7 @@ func TestGroupReadKeysByShardID_RoutesS3BucketAuxiliaryToPromotedOwner(t *testin type stubMVCCStore struct { store.MVCCStore latestTS map[string]uint64 + readiness []store.TargetStagedReadinessState returnErr error } @@ -1373,6 +1378,10 @@ func (s *stubMVCCStore) LatestCommitTS(_ context.Context, key []byte) (uint64, b return ts, ok, nil } +func (s *stubMVCCStore) MigrationTargetReadinessStates(_ context.Context) ([]store.TargetStagedReadinessState, error) { + return s.readiness, nil +} + // noopEngine satisfies raftengine.Engine for unit tests. // LinearizableRead returns immediately (simulates an already-up-to-date FSM). type noopEngine struct{} @@ -1460,6 +1469,141 @@ func TestValidateReadOnlyShards_NoConflictWhenKeyUnchanged(t *testing.T) { require.NoError(t, err) } +func TestValidateReadOnlyShards_FailsClosedOnTargetReadiness(t *testing.T) { + t.Parallel() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + }, + { + RouteID: 2, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: distribution.RouteStateActive, + }, + }, + })) + + readOnlyStore := &stubMVCCStore{ + latestTS: map[string]uint64{ + "x": 5, + }, + readiness: []store.TargetStagedReadinessState{ + { + JobID: 7, + RouteStart: []byte("m"), + RouteEnd: nil, + ExpectedCutoverVersion: 2, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }, + }, + } + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {}, + 2: {Store: readOnlyStore, Engine: noopEngine{}}, + }, 1, NewHLC(), nil) + + groupedReadKeys := map[uint64][][]byte{ + 2: {[]byte("x")}, + } + err := coord.validateReadOnlyShards(context.Background(), groupedReadKeys, []uint64{1}, 10) + require.Error(t, err) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestValidateReadOnlyShards_FailsClosedOnSourceReadFence(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + engine.UpdateRoute([]byte("a"), []byte("m"), 1) + engine.UpdateRoute([]byte("m"), nil, 2) + readOnlyStore := &stubMVCCStore{ + latestTS: map[string]uint64{"x": 5}, + readiness: []store.TargetStagedReadinessState{{ + JobID: 7, + RouteStart: []byte("m"), + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 10, + }}, + } + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {}, + 2: {Store: readOnlyStore, Engine: noopEngine{}}, + }, 1, NewHLC(), nil) + + err := coord.validateReadOnlyShards(context.Background(), map[uint64][][]byte{ + 2: {[]byte("x")}, + }, []uint64{1}, 10) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestValidateReadOnlyShards_ChecksStagedVisibilityLatestCommitTS(t *testing.T) { + t.Parallel() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + }, + { + RouteID: 2, + Start: []byte("m"), + End: []byte("z"), + GroupID: 2, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 7, + }, + }, + })) + + readOnlyStore := &stubMVCCStore{ + latestTS: map[string]uint64{ + "x": 5, + string(distribution.MigrationStagedDataKey(7, []byte("x"))): 20, + }, + readiness: []store.TargetStagedReadinessState{ + { + JobID: 7, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 7, + Armed: true, + }, + }, + } + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {}, + 2: {Store: readOnlyStore, Engine: noopEngine{}}, + }, 1, NewHLC(), nil) + + groupedReadKeys := map[uint64][][]byte{ + 2: {[]byte("x")}, + } + err := coord.validateReadOnlyShards(context.Background(), groupedReadKeys, []uint64{1}, 10) + require.ErrorIs(t, err, store.ErrWriteConflict) +} + func TestValidateReadOnlyShards_PropagatesStoreError(t *testing.T) { t.Parallel() engine := distribution.NewEngine() diff --git a/kv/transaction.go b/kv/transaction.go index c490a2503..12bae41d7 100644 --- a/kv/transaction.go +++ b/kv/transaction.go @@ -18,6 +18,7 @@ type TransactionManager struct { proposalObserver ProposalObserver timestampAdmission atomic.Pointer[transactionTimestampAdmission] + migrationAdmission atomic.Pointer[transactionMigrationAdmission] mu sync.Mutex rawPending []*rawCommitItem @@ -149,7 +150,10 @@ func prependByte(prefix byte, data []byte) []byte { // HashiCorp Raft delivers FSM responses via ApplyFuture.Response(), not Error(), // so we must inspect the response to avoid silently treating failed writes as // successes. -func applyRequests(ctx context.Context, proposer raftengine.Proposer, reqs []*pb.Request, proposalObserver ProposalObserver) (uint64, []error, error) { +func applyRequests(ctx context.Context, proposer raftengine.Proposer, reqs []*pb.Request, proposalObserver ProposalObserver, admission ...*transactionMigrationAdmission) (uint64, []error, error) { + if len(admission) > 0 && admission[0] != nil { + return admission[0].apply(ctx, proposer, reqs, proposalObserver) + } b, err := marshalRaftCommand(reqs) if err != nil { return 0, nil, errors.WithStack(err) @@ -229,7 +233,7 @@ func (t *TransactionManager) commitSequential(ctx context.Context, reqs []*pb.Re commitIndex, err := func() (uint64, error) { commitIndex := uint64(0) for _, req := range reqs { - idx, results, err := applyRequests(ctx, t.proposer, []*pb.Request{req}, t.proposalObserver) + idx, results, err := applyRequests(ctx, t.proposer, []*pb.Request{req}, t.proposalObserver, t.migrationAdmission.Load()) if err != nil { return 0, err } @@ -416,7 +420,7 @@ func (t *TransactionManager) applyRawBatch(batch []*rawCommitItem) { // see the commitRaw comment. Use Background here; per-caller // cancellation is honoured at the wait site in commitRaw via select // on item.done vs ctx.Done. - idx, results, err := applyRequests(context.Background(), t.proposer, reqs, t.proposalObserver) + idx, results, err := applyRequests(context.Background(), t.proposer, reqs, t.proposalObserver, t.migrationAdmission.Load()) if err != nil { for _, item := range batch { item.done <- rawCommitResult{err: err} @@ -455,7 +459,7 @@ func (t *TransactionManager) Abort(ctx context.Context, reqs []*pb.Request) (*Tr var commitIndex uint64 for _, req := range abortReqs { - idx, results, err := applyRequests(ctx, t.proposer, []*pb.Request{req}, t.proposalObserver) + idx, results, err := applyRequests(ctx, t.proposer, []*pb.Request{req}, t.proposalObserver, t.migrationAdmission.Load()) if err != nil { return nil, err } diff --git a/kv/transaction_migration_admission.go b/kv/transaction_migration_admission.go new file mode 100644 index 000000000..92c5ab774 --- /dev/null +++ b/kv/transaction_migration_admission.go @@ -0,0 +1,81 @@ +package kv + +import ( + "context" + + "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" +) + +type transactionMigrationAdmission struct { + store store.MVCCStore + groupID uint64 +} + +// WithMigrationRequestAdmission rechecks the final request after raw queue +// waiting, immediately before proposal. Apply independently repeats admission. +func WithMigrationRequestAdmission(st store.MVCCStore, groupID uint64) TransactionOption { + return func(t *TransactionManager) { + t.migrationAdmission.Store(&transactionMigrationAdmission{store: st, groupID: groupID}) + } +} + +func MigrationRequestAdmissionReady(tx Transactional) bool { + switch t := tx.(type) { + case *TransactionManager: + return migrationTransactionManagerReady(t) + case *LeaderProxy: + return t != nil && MigrationRequestAdmissionReady(t.tm) + case *leaseRefreshingTxn: + return t != nil && MigrationRequestAdmissionReady(t.inner) + case *timestampAdmissionTransactional: + return t != nil && MigrationRequestAdmissionReady(t.inner) + default: + return false + } +} + +func migrationTransactionManagerReady(t *TransactionManager) bool { + if t == nil { + return false + } + a := t.migrationAdmission.Load() + return a != nil && a.store != nil && a.groupID != 0 +} + +func configureMigrationRequestAdmission(tx Transactional, st store.MVCCStore, groupID uint64) { + switch t := tx.(type) { + case *TransactionManager: + WithMigrationRequestAdmission(st, groupID)(t) + case *LeaderProxy: + configureMigrationRequestAdmission(t.tm, st, groupID) + case *leaseRefreshingTxn: + configureMigrationRequestAdmission(t.inner, st, groupID) + case *timestampAdmissionTransactional: + configureMigrationRequestAdmission(t.inner, st, groupID) + } +} + +func (a *transactionMigrationAdmission) apply(ctx context.Context, proposer raftengine.Proposer, reqs []*pb.Request, observer ProposalObserver) (uint64, []error, error) { + results := make([]error, len(reqs)) + accepted := make([]*pb.Request, 0, len(reqs)) + positions := make([]int, 0, len(reqs)) + for i, req := range reqs { + results[i] = CheckMigrationRequest(ctx, a.store, a.groupID, req) + if results[i] == nil { + accepted, positions = append(accepted, req), append(positions, i) + } + } + if len(accepted) == 0 { + return 0, results, nil + } + index, applied, err := applyRequests(ctx, proposer, accepted, observer) + if err != nil { + return 0, nil, err + } + for i, position := range positions { + results[position] = applied[i] + } + return index, results, nil +} diff --git a/kv/tso_restore_context.go b/kv/tso_restore_context.go new file mode 100644 index 000000000..ced4104c4 --- /dev/null +++ b/kv/tso_restore_context.go @@ -0,0 +1,139 @@ +package kv + +import ( + "bufio" + "context" + "io" + "os" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +var _ raftengine.ContextSnapshotRestorer = (*TSOStateMachine)(nil) + +var ErrTSOExternalLegacySnapshotUnsupported = errors.New("tso fsm: external legacy KV snapshot requires decoded store validation") + +type tsoExternalStoreRestorer interface { + RestoreExternal(context.Context, io.Reader) error +} + +func (f *TSOStateMachine) RestoreWithContext(ctx context.Context, r io.Reader, opts raftengine.SnapshotRestoreOptions) error { + if err := validateTSORestoreContext(ctx, r, opts.Origin); err != nil { + return err + } + return f.restoreWithValidatedTSOContext(ctx, r, opts.Origin) +} + +func validateTSORestoreContext(ctx context.Context, r io.Reader, origin raftengine.SnapshotRestoreOrigin) error { + if ctx == nil || r == nil { + return errors.Wrap(ErrTSOStateMachineInvalidEntry, "restore context and reader are required") + } + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + if origin != raftengine.SnapshotRestoreRaft && origin != raftengine.SnapshotRestoreExternalRebase { + return errors.Wrap(ErrTSOStateMachineInvalidEntry, "unknown snapshot restore origin") + } + return nil +} + +func (f *TSOStateMachine) restoreWithValidatedTSOContext(ctx context.Context, r io.Reader, origin raftengine.SnapshotRestoreOrigin) error { + br := tsoSnapshotReader(tsoContextRestoreReader{ctx: ctx, r: r}) + legacy, err := hasLegacyKVFSMSnapshotHeader(br) + if err != nil { + return err + } + if legacy { + return f.restoreLegacyTSOWithContext(ctx, br, origin) + } + ceiling, allocation, cutover, active, floor, pending, err := readTSOSnapshotState(br) + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + if f != nil { + if err := f.restorePendingPhaseDFloor(pending, allocation, active, floor); err != nil { + return err + } + f.restoreSnapshotState(ceiling, allocation, cutover, active, floor) + } + return nil +} + +func (f *TSOStateMachine) restoreLegacyTSOWithContext(ctx context.Context, br *bufio.Reader, origin raftengine.SnapshotRestoreOrigin) error { + staged := NewTSOStateMachine(nil) + if origin == raftengine.SnapshotRestoreExternalRebase { + ceiling, err := validateExternalLegacyTSO(ctx, br) + if err != nil { + return err + } + staged.restoreSnapshotState(ceiling, tsoLeaseAllocationFloor(ceiling), false, false, 0) + } else if err := staged.Restore(br); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + if f != nil && staged.ceilingMs.Load() > 0 { + f.restoreSnapshotState(staged.ceilingMs.Load(), staged.AllocationFloor(), false, false, 0) + } + return nil +} + +// Pebble's existing restore dispatcher handles native, streaming MVCC and SST +// formats. No external KV payload may reach the legacy io.Discard bridge. +func validateExternalLegacyTSO(ctx context.Context, br *bufio.Reader) (int64, error) { + ceiling, _, err := ReadSnapshotHeader(br) + if err != nil { + return 0, errors.Wrap(err, "read external legacy TSO header") + } + ceilingMs, err := decodeTSOCeiling(ceiling, "external legacy KV snapshot") + if err != nil { + return 0, err + } + dir, err := os.MkdirTemp("", "elastickv-tso-restore-*") + if err != nil { + return 0, errors.WithStack(err) + } + defer os.RemoveAll(dir) + st, err := store.NewPebbleStore(dir) + if err != nil { + return 0, errors.WithStack(err) + } + err = validateTSOLegacyStore(ctx, br, st) + if err := errors.Join(err, st.Close()); err != nil { + return 0, errors.WithStack(err) + } + return ceilingMs, nil +} + +func validateTSOLegacyStore(ctx context.Context, r io.Reader, st store.MVCCStore) error { + gate, ok := st.(tsoExternalStoreRestorer) + if !ok { + return errors.WithStack(ErrTSOExternalLegacySnapshotUnsupported) + } + return errors.WithStack(gate.RestoreExternal(ctx, r)) +} + +type tsoContextRestoreReader struct { + ctx context.Context + r io.Reader +} + +func (r tsoContextRestoreReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, errors.WithStack(err) + } + n, err := r.r.Read(p) + if canceled := r.ctx.Err(); canceled != nil { + return n, errors.WithStack(canceled) + } + if errors.Is(err, io.EOF) { + return n, io.EOF + } + return n, errors.WithStack(err) +} diff --git a/kv/tso_restore_context_test.go b/kv/tso_restore_context_test.go new file mode 100644 index 000000000..a74b26dd1 --- /dev/null +++ b/kv/tso_restore_context_test.go @@ -0,0 +1,194 @@ +package kv + +import ( + "bytes" + "context" + "encoding/binary" + "io" + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestTSORestoreContextRequiresOptionalCapability(t *testing.T) { + fsm := NewTSOStateMachine(nil) + restorer, ok := any(fsm).(raftengine.ContextSnapshotRestorer) + require.True(t, ok, "group-zero external restore needs explicit origin-aware validation") + raw := snapshotBytesFor(t, &tsoFSMSnapshot{ceilingMs: 100}) + require.NoError(t, restorer.RestoreWithContext(t.Context(), bytes.NewReader(raw), raftengine.SnapshotRestoreOptions{ + Origin: raftengine.SnapshotRestoreExternalRebase, + })) + require.Equal(t, tsoLeaseAllocationFloor(100), fsm.AllocationFloor()) +} + +type cancelTSORestoreReader struct { + r io.Reader + cancel context.CancelFunc +} + +func (r cancelTSORestoreReader) Read(p []byte) (int, error) { + n, err := r.r.Read(p) + r.cancel() + return n, err +} + +func TestTSORestoreContextCancellationAndUnknownOriginDoNotMutate(t *testing.T) { + for _, scenario := range []string{"canceled", "during_read", "unknown", "nil_reader"} { + t.Run(scenario, func(t *testing.T) { + fsm := NewTSOStateMachine(nil) + require.Nil(t, fsm.Apply(marshalTSOAllocationFloor(50))) + restorer, ok := any(fsm).(raftengine.ContextSnapshotRestorer) + require.True(t, ok) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var reader io.Reader = bytes.NewReader(snapshotBytesFor(t, &tsoFSMSnapshot{ceilingMs: 100, allocationFloor: 200})) + opts := raftengine.SnapshotRestoreOptions{Origin: raftengine.SnapshotRestoreExternalRebase} + switch scenario { + case "canceled": + cancel() + case "during_read": + reader = cancelTSORestoreReader{r: reader, cancel: cancel} + case "unknown": + opts.Origin = 255 + case "nil_reader": + reader = nil + } + err := restorer.RestoreWithContext(ctx, reader, opts) + require.Error(t, err) + if scenario == "canceled" || scenario == "during_read" { + require.True(t, errors.Is(err, context.Canceled)) + } + require.Equal(t, uint64(50), fsm.AllocationFloor()) + require.Zero(t, fsm.ceilingMs.Load()) + }) + } +} + +func TestTSORestoreContextNativeLayoutsAndPending(t *testing.T) { + for _, snap := range []tsoFSMSnapshot{ + {ceilingMs: 100}, + {ceilingMs: 100, allocationFloor: 200}, + {ceilingMs: 100, allocationFloor: 200, cutoverActive: true}, + {ceilingMs: 100, allocationFloor: 200, cutoverActive: true, phaseDActive: true, phaseDFloor: 200}, + {ceilingMs: 100, allocationFloor: 200, cutoverActive: true, pendingFloor: 200}, + } { + raw := snapshotBytesFor(t, &snap) + want := NewTSOStateMachine(nil) + require.NoError(t, want.Restore(bytes.NewReader(raw))) + for _, origin := range []raftengine.SnapshotRestoreOrigin{raftengine.SnapshotRestoreRaft, raftengine.SnapshotRestoreExternalRebase} { + got := NewTSOStateMachine(nil) + require.NoError(t, got.RestoreWithContext(t.Context(), bytes.NewReader(raw), raftengine.SnapshotRestoreOptions{Origin: origin})) + require.Equal(t, want.ceilingMs.Load(), got.ceilingMs.Load()) + require.Equal(t, want.AllocationFloor(), got.AllocationFloor()) + require.Equal(t, want.CutoverActive(), got.CutoverActive()) + require.Equal(t, want.PhaseDActive(), got.PhaseDActive()) + require.Equal(t, want.PhaseDFloor(), got.PhaseDFloor()) + require.Equal(t, want.PendingPhaseDFloor(), got.PendingPhaseDFloor()) + } + } +} + +func TestTSORestoreContextStrictNativeErrorsPreserveState(t *testing.T) { + valid := snapshotBytesFor(t, &tsoFSMSnapshot{ceilingMs: 100, allocationFloor: 200, cutoverActive: true}) + invalidFlag := bytes.Clone(valid) + invalidFlag[tsoSnapshotV2Len] = 2 + for _, raw := range [][]byte{valid[:2], append(bytes.Clone(valid), 1), invalidFlag, append(bytes.Clone(valid), []byte("EKVPBBL1extra")...)} { + fsm := NewTSOStateMachine(nil) + require.Nil(t, fsm.Apply(marshalTSOAllocationFloor(50))) + require.Error(t, fsm.RestoreWithContext(t.Context(), bytes.NewReader(raw), raftengine.SnapshotRestoreOptions{Origin: raftengine.SnapshotRestoreExternalRebase})) + require.Equal(t, uint64(50), fsm.AllocationFloor()) + require.False(t, fsm.CutoverActive()) + } +} + +func tsoLegacyStorePayload(t *testing.T, format string) []byte { + t.Helper() + var st store.MVCCStore + if format == "memory" { + st = store.NewMVCCStore() + } else { + var err error + st, err = store.NewPebbleStore(t.TempDir(), store.WithSSTIngestSnapshots(format == "sst")) + require.NoError(t, err) + } + t.Cleanup(func() { require.NoError(t, st.Close()) }) + require.NoError(t, st.PutAt(t.Context(), []byte("ordinary"), []byte("value"), 100, 0)) + snap, err := st.Snapshot() + require.NoError(t, err) + var out bytes.Buffer + _, err = snap.WriteTo(&out) + require.NoError(t, err) + require.NoError(t, snap.Close()) + return out.Bytes() +} + +func TestTSORestoreContextLegacyExternalRequiresDecodedGate(t *testing.T) { + probe, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + _, supported := probe.(tsoExternalStoreRestorer) + require.NoError(t, probe.Close()) + for _, format := range []string{"memory", "native", "sst"} { + t.Run(format, func(t *testing.T) { + body := tsoLegacyStorePayload(t, format) + for _, header := range []bool{false, true} { + raw := bytes.Clone(body) + if header { + prefix := make([]byte, hlcSnapshotHeaderLen) + copy(prefix, hlcSnapshotMagic[:]) + binary.BigEndian.PutUint64(prefix[len(hlcSnapshotMagic):], 100) + raw = bytes.Join([][]byte{prefix, raw}, nil) + } + fsm := NewTSOStateMachine(nil) + err := fsm.RestoreWithContext(t.Context(), bytes.NewReader(raw), raftengine.SnapshotRestoreOptions{Origin: raftengine.SnapshotRestoreExternalRebase}) + if supported { + require.NoError(t, err, "proof-free legacy snapshots must use the integrated store detector") + } else { + require.ErrorIs(t, err, ErrTSOExternalLegacySnapshotUnsupported) + require.Zero(t, fsm.AllocationFloor()) + require.Zero(t, fsm.ceilingMs.Load()) + } + ordinary := NewTSOStateMachine(nil) + require.NoError(t, ordinary.RestoreWithContext(t.Context(), bytes.NewReader(raw), raftengine.SnapshotRestoreOptions{Origin: raftengine.SnapshotRestoreRaft})) + if header { + require.Equal(t, int64(100), ordinary.ceilingMs.Load()) + } + } + }) + } +} + +type recordingTSOLegacyGate struct { + store.MVCCStore + called bool + err error +} + +func (s *recordingTSOLegacyGate) RestoreExternal(ctx context.Context, r io.Reader) error { + s.called = true + if s.err != nil { + return s.err + } + return s.Restore(r) +} + +func TestTSORestoreContextLegacyGateDelegation(t *testing.T) { + body := tsoLegacyStorePayload(t, "memory") + rejected := errors.New("migration-bearing external restore unsupported") + for _, failure := range []error{nil, rejected} { + st := &recordingTSOLegacyGate{MVCCStore: store.NewMVCCStore(), err: failure} + err := validateTSOLegacyStore(t.Context(), bytes.NewReader(body), st) + require.True(t, st.called) + if failure != nil { + require.ErrorIs(t, err, rejected) + } else { + require.NoError(t, err) + value, err := st.GetAt(t.Context(), []byte("ordinary"), 100) + require.NoError(t, err) + require.Equal(t, []byte("value"), value) + } + require.NoError(t, st.Close()) + } +} diff --git a/kv/txn_codec.go b/kv/txn_codec.go index 87a1a8031..37d8b9e3a 100644 --- a/kv/txn_codec.go +++ b/kv/txn_codec.go @@ -24,8 +24,11 @@ const ( const ( txnLockFlagPrimary byte = 0x01 txnLockFlagCommitTS byte = 0x02 + txnLockKnownFlags = txnLockFlagPrimary | txnLockFlagCommitTS ) +const txnLockPrimaryOffset = 1 + uint64FieldSize + uint64FieldSize + 1 + uint64FieldSize + const ( txnMetaFlagLockTTL byte = 0x01 txnMetaFlagCommitTS byte = 0x02 @@ -254,7 +257,7 @@ type txnLock struct { } func encodeTxnLock(l txnLock) []byte { - // version(1) + StartTS(8) + TTLExpireAt(8) + flags(1) + primaryLen(8) + primaryKey + // version(1) + StartTS(8) + TTLExpireAt(8) + flags(1) + primaryLen(8) + primaryKey + optional fields size := 1 + uint64FieldSize + uint64FieldSize + 1 + uint64FieldSize + len(l.PrimaryKey) if l.CommitTS != 0 { size += uint64FieldSize @@ -269,11 +272,14 @@ func encodeTxnLock(l txnLock) []byte { } if l.CommitTS != 0 { flags |= txnLockFlagCommitTS - binary.BigEndian.PutUint64(b[size-uint64FieldSize:], l.CommitTS) } b[17] = flags binary.BigEndian.PutUint64(b[18:], uint64(len(l.PrimaryKey))) - copy(b[26:], l.PrimaryKey) + copy(b[txnLockPrimaryOffset:], l.PrimaryKey) + offset := txnLockPrimaryOffset + len(l.PrimaryKey) + if flags&txnLockFlagCommitTS != 0 { + binary.BigEndian.PutUint64(b[offset:], l.CommitTS) + } return b } @@ -285,25 +291,36 @@ func decodeTxnLock(b []byte) (txnLock, error) { return txnLock{}, errors.WithStack(errors.Newf("txn lock: unsupported version %d", b[0])) } r := bytes.NewReader(b[1:]) - var startTS uint64 - var ttlExpireAt uint64 - if err := binary.Read(r, binary.BigEndian, &startTS); err != nil { - return txnLock{}, errors.WithStack(err) + lock, flags, err := decodeTxnLockRequired(r) + if err != nil { + return txnLock{}, err } - if err := binary.Read(r, binary.BigEndian, &ttlExpireAt); err != nil { - return txnLock{}, errors.WithStack(err) + return decodeTxnLockOptional(lock, flags, r) +} + +func decodeTxnLockRequired(r *bytes.Reader) (txnLock, byte, error) { + startTS, err := readTxnUint64(r, "txn lock: start ts truncated") + if err != nil { + return txnLock{}, 0, err + } + ttlExpireAt, err := readTxnUint64(r, "txn lock: ttl expire at truncated") + if err != nil { + return txnLock{}, 0, err } flags, err := r.ReadByte() if err != nil { - return txnLock{}, errors.WithStack(err) + return txnLock{}, 0, errors.WithStack(err) } - var primaryLen uint64 - if err := binary.Read(r, binary.BigEndian, &primaryLen); err != nil { - return txnLock{}, errors.WithStack(err) + if flags&^txnLockKnownFlags != 0 { + return txnLock{}, 0, errors.WithStack(errors.Newf("txn lock: unsupported flags 0x%02x", flags)) + } + primaryLen, err := readTxnUint64(r, "txn lock: primary key length truncated") + if err != nil { + return txnLock{}, 0, err } primaryKey, err := readTxnField(r, primaryLen, "txn lock: primary key truncated") if err != nil { - return txnLock{}, err + return txnLock{}, 0, err } lock := txnLock{ StartTS: startTS, @@ -311,9 +328,16 @@ func decodeTxnLock(b []byte) (txnLock, error) { PrimaryKey: primaryKey, IsPrimaryKey: (flags & txnLockFlagPrimary) != 0, } + return lock, flags, nil +} + +func decodeTxnLockOptional(lock txnLock, flags byte, r *bytes.Reader) (txnLock, error) { if flags&txnLockFlagCommitTS != 0 { return decodeTxnLockCommitEvidence(r, lock) } + if r.Len() != 0 { + return txnLock{}, errors.WithStack(errors.Newf("txn lock: unexpected trailing bytes %d", r.Len())) + } return lock, nil } diff --git a/kv/txn_codec_test.go b/kv/txn_codec_test.go index 6e1a900a9..a52224e3e 100644 --- a/kv/txn_codec_test.go +++ b/kv/txn_codec_test.go @@ -1,6 +1,7 @@ package kv import ( + "encoding/binary" "testing" "github.com/stretchr/testify/require" @@ -127,3 +128,53 @@ func TestDecodeTxnMetaV2RejectsTrailingBytes(t *testing.T) { _, err := DecodeTxnMeta(encoded) require.ErrorContains(t, err, "unexpected trailing bytes") } + +func TestTxnLockCommitTSRoundTrip(t *testing.T) { + t.Parallel() + + lock := txnLock{ + StartTS: 11, + TTLExpireAt: 99, + PrimaryKey: []byte("primary"), + IsPrimaryKey: true, + CommitTS: 101, + } + encoded := encodeTxnLock(lock) + + require.Equal(t, txnLockVersion, encoded[0]) + require.Equal(t, txnLockFlagPrimary|txnLockFlagCommitTS, encoded[17]) + require.Equal(t, lock.CommitTS, binary.BigEndian.Uint64(encoded[len(encoded)-uint64FieldSize:])) + + decoded, err := decodeTxnLock(encoded) + require.NoError(t, err) + require.Equal(t, lock, decoded) +} + +func TestTxnLockLegacyRoundTripOmitsCommitTS(t *testing.T) { + t.Parallel() + + lock := txnLock{ + StartTS: 11, + TTLExpireAt: 99, + PrimaryKey: []byte("primary"), + IsPrimaryKey: true, + } + encoded := encodeTxnLock(lock) + + require.Equal(t, txnLockFlagPrimary, encoded[17]) + require.Len(t, encoded, 1+uint64FieldSize+uint64FieldSize+1+uint64FieldSize+len(lock.PrimaryKey)) + + decoded, err := decodeTxnLock(encoded) + require.NoError(t, err) + require.Equal(t, lock, decoded) +} + +func TestDecodeTxnLockRejectsInvalidOptionalCommitTS(t *testing.T) { + t.Parallel() + + encoded := encodeTxnLock(txnLock{StartTS: 11, PrimaryKey: []byte("primary")}) + encoded[17] |= txnLockFlagCommitTS + + _, err := decodeTxnLock(encoded) + require.ErrorContains(t, err, "commit ts truncated") +} diff --git a/kv/txn_keys.go b/kv/txn_keys.go index 466675ebd..25b081a91 100644 --- a/kv/txn_keys.go +++ b/kv/txn_keys.go @@ -91,12 +91,19 @@ func isTxnInternalKey(key []byte) bool { } func isTxnPeerLocalControlKey(key []byte) bool { - return bytes.Equal(key, backupTimestampFloorKey) || + return bytes.Equal(key, migrationConfigurationIndexKey) || + bytes.Equal(key, backupTimestampFloorKey) || bytes.Equal(key, prepareEvidenceEnabledKey) || bytes.Equal(key, prepareEvidenceGateKey) || bytes.Equal(key, prepareEvidenceTimestampFloorKey) } +// IsTxnInternalKey identifies the concrete namespaces reserved for transaction +// machinery, without reserving ordinary user keys under the !txn| umbrella. +func IsTxnInternalKey(key []byte) bool { + return isTxnInternalKey(key) +} + func isTxnMetaKey(key []byte) bool { return bytes.HasPrefix(key, txnMetaPrefixBytes) } diff --git a/main.go b/main.go index 61f7058f1..b48493495 100644 --- a/main.go +++ b/main.go @@ -71,6 +71,7 @@ const ( defaultFilesystemRootMode = 0o755 defaultFilesystemPlacementScanInterval = 30 * time.Second defaultFilesystemLeaseReapInterval = 30 * time.Second + splitMigrationCapabilityProbeTimeout = 2 * time.Second ) var errInvalidLiveBackupConfig = errors.New("invalid live backup configuration") @@ -178,13 +179,18 @@ var ( // below — both can be enabled simultaneously, and operators can pick // whichever auth path they need (gRPC bearer token vs. HTTP cookies + // SigV4 access keys). - adminTokenFile = flag.String("adminTokenFile", "", "Path to a file containing the read-only bearer token required on the Admin gRPC service (leave blank with --adminInsecureNoAuth off to disable the Admin service)") - adminInsecureNoAuth = flag.Bool("adminInsecureNoAuth", false, "Register the Admin gRPC service without bearer-token authentication; development only") - s3BlobPeerTokenFile = flag.String("s3BlobPeerTokenFile", "", "Path to the cluster-shared bearer token for peer S3 chunkblob RPCs") - backupDefaultTTL = flag.Duration("backupDefaultTTL", defaultBackupTTL, "Default live-backup pin TTL") - backupMaxTTL = flag.Duration("backupMaxTTL", defaultBackupMaxTTL, "Maximum live-backup pin TTL per renewal") - backupBeginDeadline = flag.Duration("backupBeginDeadline", defaultBackupDeadline, "Deadline for live-backup peer gates and pin fan-out") - backupSnapshotHeadroomEntries = flag.Uint64( + adminTokenFile = flag.String("adminTokenFile", "", "Path to the read-only Admin bearer token; does not authorize split migration") + adminInsecureNoAuth = flag.Bool("adminInsecureNoAuth", false, "Register the Admin gRPC service without bearer-token authentication; development only") + splitMigrationTokenFile = flag.String("splitMigrationTokenFile", "", "Path to a dedicated cluster-shared migration controller/node token, distinct from Admin and S3 peer tokens") + splitMigrationPeerAuthEnforced = flag.Bool("splitMigrationPeerAuthEnforced", false, "Require TLS peer Forward credentials and enable migration only after every node sends the migration token") + grpcTLSCAFile = flag.String("grpcTLSCAFile", "", "CA certificates for verifying peer gRPC TLS; requires grpcTLSCertFile and grpcTLSKeyFile") + grpcTLSCertFile = flag.String("grpcTLSCertFile", "", "PEM certificate enabling TLS on group gRPC listeners and peer clients") + grpcTLSKeyFile = flag.String("grpcTLSKeyFile", "", "PEM private key for group gRPC TLS") + s3BlobPeerTokenFile = flag.String("s3BlobPeerTokenFile", "", "Path to the cluster-shared bearer token for peer S3 chunkblob RPCs") + backupDefaultTTL = flag.Duration("backupDefaultTTL", defaultBackupTTL, "Default live-backup pin TTL") + backupMaxTTL = flag.Duration("backupMaxTTL", defaultBackupMaxTTL, "Maximum live-backup pin TTL per renewal") + backupBeginDeadline = flag.Duration("backupBeginDeadline", defaultBackupDeadline, "Deadline for live-backup peer gates and pin fan-out") + backupSnapshotHeadroomEntries = flag.Uint64( "backupSnapshotHeadroomEntries", defaultBackupHeadroom, "Minimum Raft entries remaining before the next snapshot when a live backup begins", ) @@ -477,6 +483,9 @@ func newRedisDeltaCompactorIfEnabled(shardStore *kv.ShardStore, coordinate kv.Co } func run() error { + if err := configureNodeGRPCTransport(); err != nil { + return err + } cfg, engineType, bootstrapCfg, bootstrap, err := resolveValidatedRuntimeInputs() if err != nil { return err @@ -659,6 +668,7 @@ func run() error { runtimes: runtimes, cfg: cfg, coordinate: coordinate, + shardStore: shardStore, defaultGroup: shardGroups[cfg.defaultGroup], encWiring: encWiring, raftID: *raftId, @@ -668,6 +678,7 @@ func run() error { metricsRegistry: metricsRegistry, clock: clock, tsoWiring: tsoWiring, + shardGroups: shardGroups, }) if err != nil { // Members already run on eg (TSO reloader, raft lifecycle @@ -808,6 +819,7 @@ type distributionStartupInput struct { runtimes []*raftGroupRuntime cfg runtimeConfig coordinate *kv.ShardedCoordinator + shardStore *kv.ShardStore defaultGroup *kv.ShardGroup encWiring encryptionWriteWiring raftID string @@ -817,6 +829,9 @@ type distributionStartupInput struct { metricsRegistry *monitoring.Registry clock *kv.HLC tsoWiring coordinatorTSOWiring + // shardGroups feeds the split-migration promotion/voter factories, which + // resolve per-group leaders rather than going through the coordinator. + shardGroups map[uint64]*kv.ShardGroup } type distributionStartup struct { @@ -830,6 +845,11 @@ type distributionStartup struct { } func startDistributionStartup(in distributionStartupInput) (distributionStartup, error) { + auth, err := loadNodeGRPCAuth(*adminTokenFile, *adminInsecureNoAuth, *s3BlobPeerTokenFile, *splitMigrationTokenFile) + if err != nil { + return distributionStartup{}, err + } + internalutil.ConfigureGRPCForwardToken(auth.migrationToken) distCatalog, catalogRuntime, defaultRuntime, err := setupDistributionRuntimeDependencies( in.ctx, in.eg, in.runtimes, in.cfg.engine, in.coordinate, in.defaultGroup, in.cfg.defaultGroup, @@ -854,6 +874,8 @@ func startDistributionStartup(in distributionStartupInput) (distributionStartup, serverwiring.StartFlusher(in.ctx, in.eg, in.sampler) startKeyVizLeaderTermPublisher(in.ctx, in.eg, in.sampler, in.runtimes) startMemoryWatchdog(in.ctx, in.eg, in.cancel) + splitMigrationConnCache := &kv.GRPCConnCache{} + in.cleanup.Add(func() { _ = splitMigrationConnCache.Close() }) distServer := adapter.NewDistributionServer( in.cfg.engine, distCatalog, @@ -866,6 +888,42 @@ func startDistributionStartup(in distributionStartupInput) (distributionStartup, return engine != nil && engine.State() == raftengine.StateLeader }), adapter.WithDistributionFilesystemObserver(in.metricsRegistry.FileSystemObserver()), + adapter.WithDistributionKnownRaftGroups(shardGroupIDs(in.shardGroups)...), + adapter.WithSplitMigrationCapabilityGate(newSplitMigrationCapabilityGate( + splitMigrationCapabilityPeerSourceForRuntimes(in.runtimes), + splitMigrationCapabilityProbeTimeout, + nil, + splitMigrationLocalReadinessGate, + )), + adapter.WithSplitPromotionClientFactory(splitPromotionClientFactory( + splitPromotionTargetLeaderResolver(in.shardGroups), + splitMigrationConnCache, + auth.migrationToken, + )), + adapter.WithSplitMigrationClientFactory(splitMigrationClientFactory( + splitMigrationGroupLeaderResolver(in.shardGroups), + splitMigrationConnCache, + auth.migrationToken, + )), + adapter.WithSplitMigrationVoterFactory(splitMigrationVoterFactory( + in.shardGroups, + splitMigrationConnCache, + auth.migrationToken, + )), + adapter.WithSplitMigrationApplyMemberSnapshotFactory(splitMigrationApplyMemberSnapshotFactory( + in.shardGroups, + splitMigrationConnCache, + auth.migrationToken, + )), + adapter.WithSplitMigrationGroupClientFactory(splitMigrationGroupClientFactory( + splitMigrationGroupLeaderResolver(in.shardGroups), + splitMigrationConnCache, + auth.migrationToken, + )), + // Do not advertise cap_migration_v2 before protocol coverage is accepted. + adapter.WithSplitJobRunnerReadinessGate(splitMigrationLocalReadinessGate), + migrationOwnershipRefreshOption(in.shardGroups, catalogRuntime, splitMigrationConnCache, auth.migrationToken, *splitMigrationPeerAuthEnforced), + migrationPartitionRuntimeOption(in.shardStore, in.shardGroups, splitMigrationConnCache, auth.migrationToken, *splitMigrationPeerAuthEnforced), ) autoSplitRuntime, err := setupDistributionWatcherAndAutoSplit( in.ctx, @@ -1386,6 +1444,152 @@ func cloneRaftServers(in []raftengine.Server) []raftengine.Server { return append([]raftengine.Server(nil), in...) } +func shardGroupIDs(groups map[uint64]*kv.ShardGroup) []uint64 { + ids := make([]uint64, 0, len(groups)) + for id := range groups { + ids = append(ids, id) + } + slices.Sort(ids) + return ids +} + +type splitMigrationCapabilityPeer struct { + ID string + Address string +} + +type splitMigrationCapabilityPeerSource func(context.Context) ([]splitMigrationCapabilityPeer, error) + +type splitMigrationCapabilityProbe func(context.Context, string) error + +func splitMigrationCapabilityPeerSourceForRuntimes(runtimes []*raftGroupRuntime) splitMigrationCapabilityPeerSource { + return func(ctx context.Context) ([]splitMigrationCapabilityPeer, error) { + if len(runtimes) == 0 { + return nil, errors.New("raft group runtimes are not configured") + } + peers := make([]splitMigrationCapabilityPeer, 0) + seen := make(map[string]struct{}) + for _, rt := range runtimes { + if rt == nil { + return nil, errors.New("raft group runtime is not configured") + } + engine := rt.snapshotEngine() + if engine == nil { + return nil, errors.Errorf("raft group %d engine is not configured", rt.spec.id) + } + cfg, err := engine.Configuration(ctx) + if err != nil { + return nil, errors.Wrapf(err, "raft group %d configuration", rt.spec.id) + } + groupPeers := splitMigrationCapabilityPeersFromConfiguration(cfg) + if len(groupPeers) == 0 { + return nil, errors.Errorf("raft group %d configuration has no split migration capability peers", rt.spec.id) + } + for _, peer := range groupPeers { + key := peer.ID + "\x00" + peer.Address + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + peers = append(peers, peer) + } + } + return peers, nil + } +} + +func splitMigrationCapabilityPeersFromConfiguration(cfg raftengine.Configuration) []splitMigrationCapabilityPeer { + servers := cfg.Servers + if len(servers) == 0 { + return nil + } + peers := make([]splitMigrationCapabilityPeer, 0, len(servers)) + seen := make(map[string]struct{}, len(servers)) + for _, server := range servers { + key := server.ID + "\x00" + server.Address + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + id := server.ID + if id == "" { + id = server.Address + } + peers = append(peers, splitMigrationCapabilityPeer{ + ID: id, + Address: server.Address, + }) + } + return peers +} + +func newSplitMigrationCapabilityGate( + source splitMigrationCapabilityPeerSource, + timeout time.Duration, + probe splitMigrationCapabilityProbe, + localGate adapter.SplitMigrationCapabilityGate, +) adapter.SplitMigrationCapabilityGate { + if probe == nil { + probe = probeSplitMigrationCapabilityPeer + } + return func(ctx context.Context) error { + if localGate != nil { + if err := localGate(ctx); err != nil { + return status.Errorf(codes.FailedPrecondition, "split migration local readiness is not available: %v", err) + } + } + if source == nil { + return status.Error(codes.FailedPrecondition, "split migration capability peers are not configured") + } + peers, err := source(ctx) + if err != nil { + return status.Errorf(codes.FailedPrecondition, "split migration capability peers are not available: %v", err) + } + if len(peers) == 0 { + return status.Error(codes.FailedPrecondition, "split migration capability peers are not configured") + } + for _, peer := range peers { + peerCtx := ctx + cancel := func() {} + if timeout > 0 { + var cancelCtx context.CancelFunc + peerCtx, cancelCtx = context.WithTimeout(ctx, timeout) + cancel = cancelCtx + } + err := probe(peerCtx, peer.Address) + cancel() + if err != nil { + return status.Errorf(codes.FailedPrecondition, "split migration capability peer %s is not ready: %v", peer.ID, err) + } + } + return nil + } +} + +func probeSplitMigrationCapabilityPeer(ctx context.Context, address string) error { + if strings.TrimSpace(address) == "" { + return errors.New("empty split migration capability peer address") + } + conn, err := grpc.NewClient(address, internalutil.GRPCDialOptions()...) + if err != nil { + return errors.Wrapf(err, "dial split migration capability peer %s", address) + } + defer func() { + _ = conn.Close() + }() + resp, err := pb.NewDistributionClient(conn).GetSplitMigrationCapability(ctx, &pb.GetSplitMigrationCapabilityRequest{}) + if err != nil { + return errors.Wrapf(err, "probe split migration capability peer %s", address) + } + if resp == nil || !resp.GetMigrationCapable() { + return errors.New("peer does not advertise split migration capability") + } + if !slices.Contains(resp.GetCapabilities(), adapter.SplitMigrationCapabilityV2) { + return errors.Errorf("peer does not advertise %s", adapter.SplitMigrationCapabilityV2) + } + return nil +} + type runtimeConfig struct { groups []groupSpec defaultGroup uint64 @@ -2333,6 +2537,170 @@ type serversInput struct { encryptionConfChangeInterceptor internalraftadmin.MembershipChangeInterceptor } +type splitPromotionLeaderResolver func(distribution.SplitJob) (string, error) +type splitMigrationLeaderResolver func(uint64) (string, error) + +func splitPromotionTargetLeaderResolver(shardGroups map[uint64]*kv.ShardGroup) splitPromotionLeaderResolver { + return func(job distribution.SplitJob) (string, error) { + sg := shardGroups[job.TargetGroupID] + if sg == nil || sg.Engine == nil { + return "", errors.Wrapf(kv.ErrLeaderNotFound, "split promotion target group %d", job.TargetGroupID) + } + addr := strings.TrimSpace(sg.Engine.Leader().Address) + if addr == "" { + return "", errors.Wrapf(kv.ErrLeaderNotFound, "split promotion target group %d", job.TargetGroupID) + } + return addr, nil + } +} + +func splitMigrationGroupLeaderResolver(shardGroups map[uint64]*kv.ShardGroup) splitMigrationLeaderResolver { + return func(groupID uint64) (string, error) { + sg := shardGroups[groupID] + if sg == nil || sg.Engine == nil { + return "", errors.Wrapf(kv.ErrLeaderNotFound, "split migration group %d", groupID) + } + addr := strings.TrimSpace(sg.Engine.Leader().Address) + if addr == "" { + return "", errors.Wrapf(kv.ErrLeaderNotFound, "split migration group %d", groupID) + } + return addr, nil + } +} + +func splitPromotionClientFactory(resolve splitPromotionLeaderResolver, connCache *kv.GRPCConnCache, token string) adapter.SplitPromotionClientFactory { + return func(_ context.Context, job distribution.SplitJob) (adapter.SplitPromotionClient, error) { + if resolve == nil { + return nil, errors.New("split promotion leader resolver is not configured") + } + if connCache == nil { + return nil, errors.New("split promotion gRPC connection cache is not configured") + } + addr, err := resolve(job) + if err != nil { + return nil, errors.WithStack(err) + } + conn, err := connCache.ConnFor(addr) + if err != nil { + return nil, errors.WithStack(err) + } + return newSplitMigrationInternalClient(conn, token), nil + } +} + +func splitMigrationClientFactory(resolve splitMigrationLeaderResolver, connCache *kv.GRPCConnCache, token string) adapter.SplitMigrationClientFactory { + groupClient := splitMigrationGroupClientFactory(resolve, connCache, token) + return func(ctx context.Context, job distribution.SplitJob, sourceGroupID uint64) (adapter.SplitMigrationClient, adapter.SplitMigrationClient, error) { + source, err := groupClient(ctx, sourceGroupID) + if err != nil { + return nil, nil, err + } + target, err := groupClient(ctx, job.TargetGroupID) + if err != nil { + return nil, nil, err + } + return source, target, nil + } +} + +func splitMigrationGroupClientFactory(resolve splitMigrationLeaderResolver, connCache *kv.GRPCConnCache, token string) adapter.SplitMigrationGroupClientFactory { + return func(_ context.Context, groupID uint64) (adapter.SplitMigrationClient, error) { + if resolve == nil { + return nil, errors.New("split migration leader resolver is not configured") + } + if connCache == nil { + return nil, errors.New("split migration gRPC connection cache is not configured") + } + addr, err := resolve(groupID) + if err != nil { + return nil, errors.WithStack(err) + } + conn, err := connCache.ConnFor(addr) + if err != nil { + return nil, errors.WithStack(err) + } + return newSplitMigrationInternalClient(conn, token), nil + } +} + +func splitMigrationVoterFactory(shardGroups map[uint64]*kv.ShardGroup, connCache *kv.GRPCConnCache, token string) adapter.SplitMigrationVoterFactory { + return func(ctx context.Context, groupID uint64) ([]adapter.SplitMigrationVoter, error) { + sg := shardGroups[groupID] + if sg == nil || sg.Engine == nil { + return nil, errors.Wrapf(kv.ErrLeaderNotFound, "split migration group %d", groupID) + } + if connCache == nil { + return nil, errors.New("split migration voter gRPC connection cache is not configured") + } + cfg, err := sg.Engine.Configuration(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + voters := make([]adapter.SplitMigrationVoter, 0, len(cfg.Servers)) + for _, member := range cfg.Servers { + if member.Suffrage != raftMemberSuffrageVoter { + continue + } + voter, err := splitMigrationVoter(groupID, member, connCache, token) + if err != nil { + return nil, err + } + voters = append(voters, voter) + } + if len(voters) == 0 { + return nil, errors.Errorf("split migration group %d has no voters", groupID) + } + return voters, nil + } +} + +func splitMigrationVoter(groupID uint64, member raftengine.Server, connCache *kv.GRPCConnCache, token string) (adapter.SplitMigrationVoter, error) { + addr := strings.TrimSpace(member.Address) + if member.ID == "" || addr == "" { + return adapter.SplitMigrationVoter{}, errors.Errorf("split migration group %d has voter with missing id/address", groupID) + } + conn, err := connCache.ConnFor(addr) + if err != nil { + return adapter.SplitMigrationVoter{}, errors.WithStack(err) + } + return adapter.SplitMigrationVoter{ID: member.ID, Address: addr, Client: newSplitMigrationInternalClient(conn, token)}, nil +} + +func splitMigrationLocalReadinessGate(ctx context.Context) error { + if err := splitMigrationPeerAuthReadinessGate(ctx); err != nil { + return err + } + if !adapter.MigrationImportOpcodeEnabledFromEnv() { + return errors.Errorf("%s is disabled", adapter.MigrationImportOpcodeEnv) + } + if !adapter.MigrationPromoteOpcodeEnabledFromEnv() { + return errors.Errorf("%s is disabled", adapter.MigrationPromoteOpcodeEnv) + } + if !adapter.MigrationCleanupOpcodeEnabledFromEnv() { + return errors.Errorf("%s is disabled", adapter.MigrationCleanupOpcodeEnv) + } + return nil +} + +func splitMigrationPeerAuthReadinessGate(context.Context) error { + if *adminInsecureNoAuth || *splitMigrationPeerAuthEnforced { + return nil + } + return errors.New("migration is paused during peer auth rollout; enable --splitMigrationPeerAuthEnforced only after every node sends the token") +} + +func startDistributionSplitJobRunner(ctx context.Context, eg *errgroup.Group, server *adapter.DistributionServer) { + if eg == nil || server == nil { + return + } + if splitMigrationLocalReadinessGate(ctx) != nil { + return + } + eg.Go(func() error { + return server.RunSplitJobRunner(ctx) + }) +} + // startServersAfterStartupRotation wires up the AdminServer, starts the // per-group Raft listeners needed for quorum traffic, waits for a fresh join // to catch up, prepares the public listeners, waits for any requested startup @@ -2458,6 +2826,8 @@ func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, } startHLCLeaseRenewal(in.ctx, in.eg, in.coordinate) publicKVGate.markReady() + startDistributionSplitJobRunner(in.ctx, in.eg, in.distServer) + startMigrationOwnershipRefresh(in.ctx, in.eg, in.distServer) if err := runner.startPublicServices(); err != nil { return err } @@ -2553,7 +2923,7 @@ func raftJoinRuntimesReady(ctx context.Context, runtimes []*raftGroupRuntime, lo func configurationContainsMember(configuration raftengine.Configuration, localID string) bool { for _, server := range configuration.Servers { - if server.ID == localID && (server.Suffrage == "learner" || server.Suffrage == shutdownVoterSuffrage) { + if server.ID == localID && (server.Suffrage == "learner" || server.Suffrage == raftMemberSuffrageVoter) { return true } } @@ -2874,6 +3244,7 @@ func setupAdminService( *adminTokenFile, *adminInsecureNoAuth, *s3BlobPeerTokenFile, + *splitMigrationTokenFile, adapter.S3BlobOffloadEnabledFromEnv(), adapter.NodeIdentity{NodeID: nodeID, GRPCAddress: selfAddr}, members, @@ -2974,6 +3345,7 @@ type adminGRPCInterceptors struct { unary []grpc.UnaryServerInterceptor stream []grpc.StreamServerInterceptor token string + migrationToken string s3BlobFetchEnabled bool } @@ -3105,9 +3477,19 @@ func (g *startupPublicKVGate) blocked() bool { func startupRotationGatedMethod(fullMethod string) bool { switch fullMethod { case pb.Internal_Forward_FullMethodName, + pb.Internal_ApplyTargetStagedReadiness_FullMethodName, + pb.Internal_ImportRangeVersions_FullMethodName, + pb.Internal_PromoteStagedVersions_FullMethodName, + pb.Internal_CleanupMigration_FullMethodName, + pb.Internal_IssueMigrationTimestamp_FullMethodName, pb.AdminForward_Forward_FullMethodName, pb.Distribution_GetTimestamp_FullMethodName, pb.Distribution_SplitRange_FullMethodName, + pb.Distribution_StartSplitMigration_FullMethodName, + pb.Distribution_GetSplitMigrationCapability_FullMethodName, + pb.Distribution_AttestMigrationOwnership_FullMethodName, + pb.Distribution_AbandonSplitJob_FullMethodName, + pb.Distribution_RetrySplitJob_FullMethodName, pb.RaftAdmin_AddVoter_FullMethodName, pb.RaftAdmin_AddLearner_FullMethodName, pb.RaftAdmin_PromoteLearner_FullMethodName, @@ -3156,13 +3538,14 @@ func configureAdminService( tokenPath string, insecureNoAuth bool, s3BlobPeerTokenPath string, + migrationTokenPath string, s3BlobOffloadEnabled bool, self adapter.NodeIdentity, members []adapter.NodeIdentity, connCache *kv.GRPCConnCache, backupDeps ...adminBackupDependencies, ) (*adapter.AdminServer, adminGRPCInterceptors, error) { - auth, err := loadNodeGRPCAuth(tokenPath, insecureNoAuth, s3BlobPeerTokenPath) + auth, err := loadNodeGRPCAuth(tokenPath, insecureNoAuth, s3BlobPeerTokenPath, migrationTokenPath) if err != nil { return nil, adminGRPCInterceptors{}, err } @@ -3180,9 +3563,11 @@ func configureAdminService( } // token is what setupAdminService hands to SetLeaderReadToken and the // per-group backup proposer, so it has to survive the interceptor build. - icept := adminGRPCInterceptors{token: auth.adminToken} + icept := adminGRPCInterceptors{token: auth.adminToken, migrationToken: auth.migrationToken} adminUnary, adminStream := adapter.AdminTokenAuth(auth.adminToken) appendGRPCBearerInterceptors(&icept, adminUnary, adminStream) + migrationUnary, migrationStream := adapter.SplitMigrationTokenAuth(auth.migrationToken, insecureNoAuth, *splitMigrationPeerAuthEnforced) + appendGRPCBearerInterceptors(&icept, migrationUnary, migrationStream) peerUnary, peerStream := adapter.S3BlobPeerTokenAuth(auth.peerToken) appendGRPCBearerInterceptors(&icept, peerUnary, peerStream) icept.s3BlobFetchEnabled = auth.peerToken != "" @@ -3190,12 +3575,13 @@ func configureAdminService( } type nodeGRPCAuthConfig struct { - adminToken string - peerToken string - adminEnabled bool + adminToken string + peerToken string + migrationToken string + adminEnabled bool } -func loadNodeGRPCAuth(tokenPath string, insecureNoAuth bool, peerTokenPath string) (nodeGRPCAuthConfig, error) { +func loadNodeGRPCAuth(tokenPath string, insecureNoAuth bool, peerTokenPath, migrationTokenPath string) (nodeGRPCAuthConfig, error) { if tokenPath != "" && insecureNoAuth { return nodeGRPCAuthConfig{}, errors.New("--adminInsecureNoAuth and --adminTokenFile are mutually exclusive") } @@ -3215,12 +3601,80 @@ func loadNodeGRPCAuth(tokenPath string, insecureNoAuth bool, peerTokenPath strin } cfg.peerToken = loaded } - if cfg.adminToken != "" && cfg.peerToken == cfg.adminToken { - return nodeGRPCAuthConfig{}, errors.New("S3 blob peer token must differ from the read-only admin token") + if err := validateNodeGRPCPeerToken(cfg.adminToken, cfg.peerToken); err != nil { + return nodeGRPCAuthConfig{}, err + } + migrationToken, err := loadSplitMigrationToken(migrationTokenPath, insecureNoAuth, cfg.adminToken, cfg.peerToken) + if err != nil { + return nodeGRPCAuthConfig{}, err } + cfg.migrationToken = migrationToken return cfg, nil } +func validateMigrationOpcodeAuth(token string, insecureNoAuth bool) error { + if token != "" || insecureNoAuth { + return nil + } + if adapter.MigrationImportOpcodeEnabledFromEnv() || adapter.MigrationPromoteOpcodeEnabledFromEnv() || adapter.MigrationCleanupOpcodeEnabledFromEnv() { + return errors.New("migration opcodes require --splitMigrationTokenFile or explicit test-only --adminInsecureNoAuth") + } + return nil +} + +func validateMigrationStartupAuth() error { + auth, err := loadNodeGRPCAuth(*adminTokenFile, *adminInsecureNoAuth, *s3BlobPeerTokenFile, *splitMigrationTokenFile) + if err != nil { + return err + } + return validateMigrationPeerSecurity(auth.migrationToken, *adminInsecureNoAuth, *splitMigrationPeerAuthEnforced, *grpcTLSCertFile != "" && *grpcTLSKeyFile != "") +} + +func configureNodeGRPCTransport() error { + if err := internalutil.ConfigureGRPCTLS(*grpcTLSCAFile, *grpcTLSCertFile, *grpcTLSKeyFile); err != nil { + return errors.Wrap(err, "configure peer gRPC transport") + } + return validateMigrationStartupAuth() +} + +func validateMigrationPeerSecurity(token string, insecureNoAuth, enforced, tlsEnabled bool) error { + if insecureNoAuth { + return nil + } + if enforced && token == "" { + return errors.New("--splitMigrationPeerAuthEnforced requires --splitMigrationTokenFile") + } + if token != "" && !tlsEnabled { + return errors.New("--splitMigrationTokenFile requires gRPC TLS certificate and key configuration") + } + return nil +} + +func validateNodeGRPCPeerToken(adminToken, peerToken string) error { + if adminToken != "" && peerToken == adminToken { + return errors.New("S3 blob peer token must differ from the read-only admin token") + } + return nil +} + +func loadSplitMigrationToken(path string, insecureNoAuth bool, adminToken, peerToken string) (string, error) { + if path == "" { + return "", validateMigrationOpcodeAuth("", insecureNoAuth) + } + if insecureNoAuth { + return "", errors.New("--adminInsecureNoAuth and --splitMigrationTokenFile are mutually exclusive") + } + const maxMigrationTokenBytes = 4 << 10 + token, err := internalutil.LoadBearerTokenFile(path, maxMigrationTokenBytes, "migration token") + if err != nil { + return "", errors.WithStack(err) + } + if token == adminToken || token == peerToken { + return "", errors.New("migration token must differ from Admin and S3 peer tokens") + } + return token, nil +} + func appendGRPCBearerInterceptors( icept *adminGRPCInterceptors, unary grpc.UnaryServerInterceptor, @@ -3648,9 +4102,9 @@ func raftGRPCServerOptions(adminGRPCOpts adminGRPCInterceptors) []grpc.ServerOpt // (single-interceptor) option added anywhere in this chain cannot // silently overwrite the admin auth gate — gRPC-Go keeps only the // last option of the same type. - if len(adminGRPCOpts.unary) > 0 { - opts = append(opts, grpc.ChainUnaryInterceptor(adminGRPCOpts.unary...)) - } + unary := append([]grpc.UnaryServerInterceptor{}, adminGRPCOpts.unary...) + unary = append(unary, migrationCoverageUnaryGate) + opts = append(opts, grpc.ChainUnaryInterceptor(unary...)) if len(adminGRPCOpts.stream) > 0 { opts = append(opts, grpc.ChainStreamInterceptor(adminGRPCOpts.stream...)) } @@ -3679,6 +4133,8 @@ func startRaftServers( confChangeInterceptor internalraftadmin.MembershipChangeInterceptor, encWiring encryptionWriteWiring, sqsPartitionResolver kv.PartitionResolver, + routeEngine *distribution.Engine, + readTracker *kv.ActiveTimestampTracker, kekConfigured bool, defaultGroupID uint64, s3BlobObserver adapter.S3BlobOffloadObserver, @@ -3710,12 +4166,16 @@ func startRaftServers( ) for _, rt := range runtimes { gs := grpc.NewServer(raftGRPCServerOptions(adminGRPCOpts)...) - trx := kv.NewTransactionWithProposer(proposerForGroup(rt, shardGroups), kv.WithProposalObserver(observerForGroup(proposalObserverForGroup, rt.spec.id))) + trx := newInternalMigrationTransaction(proposerForGroup(rt, shardGroups), observerForGroup(proposalObserverForGroup, rt.spec.id), rt.store, rt.spec.id) grpcSvc := adapter.NewGRPCServer(shardStore, coordinate) pb.RegisterRawKVServer(gs, grpcSvc) pb.RegisterTransactionalKVServer(gs, grpcSvc) internalOpts := internalServerOptions(coordinate, adminServer, proposerForGroup(rt, shardGroups), shardGroups[rt.spec.id], rt.spec.id) internalOpts = append(internalOpts, dedicatedTimestampCutoverOptions(shardGroups)...) + internalOpts = append(internalOpts, adapter.WithInternalMigrationOwnershipReady(func() bool { + return migrationOwnershipRuntimeReady(trx, shardStore, distServer, rt.spec.id, rt.snapshotEngine()) + })) + internalOpts = append(internalOpts, migrationInternalProtocolOptions(trx, shardStore, distServer, rt)...) staticServingServices := registerS3BlobFetchServer( gs, adminGRPCOpts.s3BlobFetchEnabled, @@ -3734,6 +4194,8 @@ func startRaftServers( adapter.WithInternalStore(rt.store), adapter.WithInternalPrimaryCommitEvidence(shardStore), adapter.WithInternalMigrationProposer(proposerForGroup(rt, shardGroups)), + adapter.WithInternalRouteEngine(routeEngine), + adapter.WithInternalActiveTimestampTracker(readTracker), adapter.WithInternalMigrationExportRouting(rt.spec.id, sqsPartitionResolver), )..., )) @@ -3768,7 +4230,7 @@ func startRaftServers( rt.engine, []string{"RawKV"}, staticServingServices, - newPrepareEvidenceMembershipGate(shardStore, rt.spec.id, confChangeInterceptor), + migrationMembershipInterceptor(rt, shardStore, proposerForGroup(rt, shardGroups), adminGRPCOpts.migrationToken, confChangeInterceptor), ) reflection.Register(gs) @@ -3993,6 +4455,7 @@ func distributionCatalogStoreForGroup(runtimes []*raftGroupRuntime, groupID uint } if rt.spec.id == groupID { return distribution.NewCatalogStore(rt.store, + distribution.WithCatalogRouteDescriptorV2Writes(true), distribution.WithCatalogMigrationStoreResolver(catalogMigrationStoreResolver(runtimes)), distribution.WithCatalogMigrationRetireResolver(catalogMigrationRetireResolver(runtimes)), ) @@ -4343,6 +4806,8 @@ func (r *runtimeServerRunner) startRaftTransport() error { r.encryptionConfChangeInterceptor, r.encWiring, sqsPartitionResolver, + r.routeEngine, + r.readTracker, r.kekConfigured, r.defaultGroupID, r.metricsRegistry.S3BlobOffloadObserver(), diff --git a/main_admin_test.go b/main_admin_test.go index b645ae85d..ebc1b6307 100644 --- a/main_admin_test.go +++ b/main_admin_test.go @@ -38,12 +38,12 @@ import ( func TestConfigureAdminServiceDisabledByDefault(t *testing.T) { t.Parallel() - srv, icept, err := configureAdminService("", false, "", false, adapter.NodeIdentity{NodeID: "n1"}, nil, nil) + srv, icept, err := configureAdminService("", false, "", "", false, adapter.NodeIdentity{NodeID: "n1"}, nil, nil) if err != nil { t.Fatalf("disabled-by-default should not error: %v", err) } - if srv != nil || !icept.empty() { - t.Fatalf("disabled service should return nil server and empty interceptors; got %v %+v", srv, icept) + if srv != nil || len(icept.unary) != 1 || len(icept.stream) != 1 { + t.Fatalf("disabled admin service should still deny migration mutations; got %v %+v", srv, icept) } } @@ -163,7 +163,7 @@ func TestConfigureAdminServiceRejectsMutualExclusion(t *testing.T) { if err := os.WriteFile(tokPath, []byte("x"), 0o600); err != nil { t.Fatal(err) } - if _, _, err := configureAdminService(tokPath, true, "", false, adapter.NodeIdentity{}, nil, nil); err == nil { + if _, _, err := configureAdminService(tokPath, true, "", "", false, adapter.NodeIdentity{}, nil, nil); err == nil { t.Fatal("expected mutual-exclusion error") } } @@ -179,15 +179,15 @@ func TestConfigureAdminServiceTokenFile(t *testing.T) { if err := os.WriteFile(peerPath, []byte("peer-secret\n"), 0o600); err != nil { t.Fatal(err) } - srv, icept, err := configureAdminService(tokPath, false, peerPath, true, adapter.NodeIdentity{NodeID: "n1"}, nil, nil) + srv, icept, err := configureAdminService(tokPath, false, peerPath, "", true, adapter.NodeIdentity{NodeID: "n1"}, nil, nil) if err != nil { t.Fatalf("configureAdminService: %v", err) } if srv == nil { t.Fatal("expected an AdminServer instance") } - if len(icept.unary) != 2 || len(icept.stream) != 2 { - t.Fatalf("expected separate admin and peer interceptors, got %d + %d", len(icept.unary), len(icept.stream)) + if len(icept.unary) != 3 || len(icept.stream) != 3 { + t.Fatalf("expected admin, migration and peer interceptors, got %d + %d", len(icept.unary), len(icept.stream)) } if !icept.s3BlobFetchEnabled { t.Fatal("token-authenticated configuration should enable S3BlobFetch") @@ -211,7 +211,7 @@ func TestConfigureAdminServiceRejectsSharedAdminAndPeerToken(t *testing.T) { t.Fatal(err) } } - if _, _, err := configureAdminService(adminPath, false, peerPath, true, adapter.NodeIdentity{}, nil, nil); err == nil { + if _, _, err := configureAdminService(adminPath, false, peerPath, "", true, adapter.NodeIdentity{}, nil, nil); err == nil { t.Fatal("expected shared admin and peer token to be rejected") } } @@ -227,7 +227,7 @@ func TestConfigureAdminServiceDoesNotAdvertiseDisabledS3BlobOffload(t *testing.T if err := os.WriteFile(peerPath, []byte("peer-secret\n"), 0o600); err != nil { t.Fatal(err) } - srv, icept, err := configureAdminService(tokPath, false, peerPath, false, adapter.NodeIdentity{NodeID: "n1"}, nil, nil) + srv, icept, err := configureAdminService(tokPath, false, peerPath, "", false, adapter.NodeIdentity{NodeID: "n1"}, nil, nil) if err != nil { t.Fatalf("configureAdminService: %v", err) } @@ -245,7 +245,7 @@ func TestConfigureAdminServiceDoesNotAdvertiseDisabledS3BlobOffload(t *testing.T func TestConfigureAdminServiceInsecureNoAuth(t *testing.T) { t.Parallel() - srv, icept, err := configureAdminService("", true, "", true, adapter.NodeIdentity{NodeID: "n1"}, nil, nil) + srv, icept, err := configureAdminService("", true, "", "", true, adapter.NodeIdentity{NodeID: "n1"}, nil, nil) if err != nil { t.Fatalf("insecure mode should succeed: %v", err) } @@ -295,6 +295,7 @@ func TestConfigureAdminServiceWiresLeaderVersionProbe(t *testing.T) { "", true, "", + "", false, adapter.NodeIdentity{NodeID: "n1", GRPCAddress: "127.0.0.1:50051"}, nil, @@ -329,6 +330,7 @@ func TestConfigureAdminServiceUsesStampedBuildVersion(t *testing.T) { "", true, "", + "", false, adapter.NodeIdentity{NodeID: "n1", GRPCAddress: "127.0.0.1:50051"}, nil, diff --git a/main_catalog_test.go b/main_catalog_test.go index 193a98219..6cf3a81d9 100644 --- a/main_catalog_test.go +++ b/main_catalog_test.go @@ -2,11 +2,20 @@ package main import ( "context" + "errors" + "net" "testing" + "time" + "github.com/bootjp/elastickv/adapter" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func TestDistributionCatalogGroupID_UsesCatalogKeyRoute(t *testing.T) { @@ -68,3 +77,308 @@ func TestSetupDistributionCatalog_UsesResolvedCatalogGroup(t *testing.T) { require.NoError(t, err) require.NotNil(t, catalog) } + +func TestDistributionCatalogStoreForGroupEnablesRouteDescriptorV2Writes(t *testing.T) { + t.Parallel() + + rt := &raftGroupRuntime{ + spec: groupSpec{id: 5}, + store: store.NewMVCCStore(), + } + t.Cleanup(func() { + rt.Close() + }) + + catalog := distributionCatalogStoreForGroup([]*raftGroupRuntime{rt}, 5) + require.NotNil(t, catalog) + require.True(t, catalog.AllowsRouteDescriptorV2Writes()) +} + +func TestSplitMigrationCapabilityGateChecksAllPeers(t *testing.T) { + t.Parallel() + + peers := []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + {ID: "n2", Address: "10.0.0.12:50051"}, + } + var probed []string + gate := newSplitMigrationCapabilityGate(staticSplitMigrationCapabilityPeerSource(peers), time.Second, func(_ context.Context, address string) error { + probed = append(probed, address) + return nil + }, nil) + + require.NoError(t, gate(context.Background())) + require.Equal(t, []string{"10.0.0.11:50051", "10.0.0.12:50051"}, probed) +} + +func TestSplitMigrationCapabilityGateFailsClosedWithoutPeers(t *testing.T) { + t.Parallel() + + gate := newSplitMigrationCapabilityGate(nil, time.Second, func(context.Context, string) error { + t.Fatal("probe must not run without peers") + return nil + }, nil) + + err := gate(context.Background()) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "peers are not configured") +} + +func TestSplitMigrationCapabilityGateFailsClosedWhenPeerMissingCapability(t *testing.T) { + t.Parallel() + + peers := []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + {ID: "n2", Address: "10.0.0.12:50051"}, + } + gate := newSplitMigrationCapabilityGate(staticSplitMigrationCapabilityPeerSource(peers), time.Second, func(_ context.Context, address string) error { + if address == "10.0.0.12:50051" { + return status.Error(codes.Unimplemented, "method not found") + } + return nil + }, nil) + + err := gate(context.Background()) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "n2") + require.ErrorContains(t, err, "method not found") +} + +func TestSplitMigrationCapabilityGateFailsClosedWhenPeerSourceErrors(t *testing.T) { + t.Parallel() + + errPeerSourceUnavailable := errors.New("configuration unavailable") + gate := newSplitMigrationCapabilityGate(func(context.Context) ([]splitMigrationCapabilityPeer, error) { + return nil, errPeerSourceUnavailable + }, time.Second, func(context.Context, string) error { + t.Fatal("probe must not run when peer source fails") + return nil + }, nil) + + err := gate(context.Background()) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "peers are not available") +} + +func TestSplitMigrationCapabilityGateFailsClosedWhenLocalReadinessFails(t *testing.T) { + t.Parallel() + + peers := []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + } + gate := newSplitMigrationCapabilityGate(staticSplitMigrationCapabilityPeerSource(peers), time.Second, func(context.Context, string) error { + t.Fatal("probe must not run when local readiness is closed") + return nil + }, func(context.Context) error { + return status.Error(codes.FailedPrecondition, "migration opcode disabled") + }) + + err := gate(context.Background()) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "local readiness") + require.ErrorContains(t, err, "migration opcode disabled") +} + +func TestSplitMigrationLocalReadinessGateRequiresMigrationOpcodes(t *testing.T) { + oldEnforced := *splitMigrationPeerAuthEnforced + *splitMigrationPeerAuthEnforced = true + t.Cleanup(func() { *splitMigrationPeerAuthEnforced = oldEnforced }) + t.Setenv(adapter.MigrationImportOpcodeEnv, "") + t.Setenv(adapter.MigrationPromoteOpcodeEnv, "1") + t.Setenv(adapter.MigrationCleanupOpcodeEnv, "1") + err := splitMigrationLocalReadinessGate(context.Background()) + require.Error(t, err) + require.ErrorContains(t, err, adapter.MigrationImportOpcodeEnv) + + t.Setenv(adapter.MigrationImportOpcodeEnv, "1") + t.Setenv(adapter.MigrationPromoteOpcodeEnv, "") + err = splitMigrationLocalReadinessGate(context.Background()) + require.Error(t, err) + require.ErrorContains(t, err, adapter.MigrationPromoteOpcodeEnv) + + t.Setenv(adapter.MigrationPromoteOpcodeEnv, "1") + t.Setenv(adapter.MigrationCleanupOpcodeEnv, "") + err = splitMigrationLocalReadinessGate(context.Background()) + require.Error(t, err) + require.ErrorContains(t, err, adapter.MigrationCleanupOpcodeEnv) + + t.Setenv(adapter.MigrationCleanupOpcodeEnv, "1") + require.NoError(t, splitMigrationLocalReadinessGate(context.Background())) +} + +func TestSplitMigrationCapabilityPeersFromConfigurationUsesCurrentMembers(t *testing.T) { + t.Parallel() + + cfg := raftengine.Configuration{Servers: []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + {Suffrage: "learner", ID: "n2", Address: "10.0.0.12:50051"}, + {Suffrage: "", ID: "n3", Address: "10.0.0.13:50051"}, + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + }} + + require.Equal(t, []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + {ID: "n2", Address: "10.0.0.12:50051"}, + {ID: "n3", Address: "10.0.0.13:50051"}, + }, splitMigrationCapabilityPeersFromConfiguration(cfg)) +} + +func TestSplitMigrationCapabilityPeerSourceForRuntimesChecksAllGroups(t *testing.T) { + t.Parallel() + + runtimes := []*raftGroupRuntime{ + { + spec: groupSpec{id: 1}, + engine: capabilityConfigEngine{cfg: raftengine.Configuration{Servers: []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + {Suffrage: "learner", ID: "n2", Address: "10.0.0.12:50051"}, + }}}, + }, + { + spec: groupSpec{id: 2}, + engine: capabilityConfigEngine{cfg: raftengine.Configuration{Servers: []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + {Suffrage: "voter", ID: "n3", Address: "10.0.0.13:50051"}, + }}}, + }, + } + + peers, err := splitMigrationCapabilityPeerSourceForRuntimes(runtimes)(context.Background()) + require.NoError(t, err) + require.Equal(t, []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + {ID: "n2", Address: "10.0.0.12:50051"}, + {ID: "n3", Address: "10.0.0.13:50051"}, + }, peers) +} + +func TestSplitMigrationCapabilityPeerSourceForRuntimesFailsClosedOnEmptyGroup(t *testing.T) { + t.Parallel() + + runtimes := []*raftGroupRuntime{ + { + spec: groupSpec{id: 1}, + engine: capabilityConfigEngine{cfg: raftengine.Configuration{Servers: []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + }}}, + }, + { + spec: groupSpec{id: 2}, + engine: capabilityConfigEngine{cfg: raftengine.Configuration{}}, + }, + } + + peers, err := splitMigrationCapabilityPeerSourceForRuntimes(runtimes)(context.Background()) + require.Error(t, err) + require.Nil(t, peers) + require.ErrorContains(t, err, "raft group 2") + require.ErrorContains(t, err, "no split migration capability peers") +} + +func TestProbeSplitMigrationCapabilityPeerRequiresDocumentedToken(t *testing.T) { + t.Parallel() + + addr := startSplitMigrationCapabilityTestServer(t, &pb.GetSplitMigrationCapabilityResponse{ + MigrationCapable: true, + Capabilities: []string{"split_migration_v2"}, + }) + + err := probeSplitMigrationCapabilityPeer(context.Background(), addr) + require.Error(t, err) + require.ErrorContains(t, err, adapter.SplitMigrationCapabilityV2) +} + +func TestProbeSplitMigrationCapabilityPeerAcceptsDocumentedToken(t *testing.T) { + t.Parallel() + + addr := startSplitMigrationCapabilityTestServer(t, &pb.GetSplitMigrationCapabilityResponse{ + MigrationCapable: true, + Capabilities: []string{adapter.SplitMigrationCapabilityV2}, + }) + + require.NoError(t, probeSplitMigrationCapabilityPeer(context.Background(), addr)) +} + +func staticSplitMigrationCapabilityPeerSource(peers []splitMigrationCapabilityPeer) splitMigrationCapabilityPeerSource { + return func(context.Context) ([]splitMigrationCapabilityPeer, error) { + out := make([]splitMigrationCapabilityPeer, len(peers)) + copy(out, peers) + return out, nil + } +} + +type splitMigrationCapabilityTestServer struct { + pb.UnimplementedDistributionServer + resp *pb.GetSplitMigrationCapabilityResponse +} + +func (s *splitMigrationCapabilityTestServer) GetSplitMigrationCapability(context.Context, *pb.GetSplitMigrationCapabilityRequest) (*pb.GetSplitMigrationCapabilityResponse, error) { + return s.resp, nil +} + +func startSplitMigrationCapabilityTestServer(t *testing.T, resp *pb.GetSplitMigrationCapabilityResponse) string { + t.Helper() + + var listenConfig net.ListenConfig + listener, err := listenConfig.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + grpcServer := grpc.NewServer() + pb.RegisterDistributionServer(grpcServer, &splitMigrationCapabilityTestServer{resp: resp}) + done := make(chan struct{}) + go func() { + defer close(done) + _ = grpcServer.Serve(listener) + }() + t.Cleanup(func() { + grpcServer.Stop() + <-done + }) + return listener.Addr().String() +} + +type capabilityConfigEngine struct { + cfg raftengine.Configuration + leader raftengine.LeaderInfo +} + +func (e capabilityConfigEngine) SnapshotEvery() uint64 { return 0 } + +func (e capabilityConfigEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, nil +} + +func (e capabilityConfigEngine) ProposeAdmin(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, nil +} + +func (e capabilityConfigEngine) State() raftengine.State { + return raftengine.StateFollower +} + +func (e capabilityConfigEngine) Leader() raftengine.LeaderInfo { + return e.leader +} + +func (e capabilityConfigEngine) VerifyLeader(context.Context) error { + return nil +} + +func (e capabilityConfigEngine) LinearizableRead(context.Context) (uint64, error) { + return 0, nil +} + +func (e capabilityConfigEngine) Status() raftengine.Status { + return raftengine.Status{} +} + +func (e capabilityConfigEngine) Configuration(context.Context) (raftengine.Configuration, error) { + return e.cfg, nil +} + +func (e capabilityConfigEngine) Close() error { + return nil +} diff --git a/main_encryption_rotate_on_startup_test.go b/main_encryption_rotate_on_startup_test.go index 08b2cec7f..89edd057d 100644 --- a/main_encryption_rotate_on_startup_test.go +++ b/main_encryption_rotate_on_startup_test.go @@ -440,9 +440,19 @@ func TestStartupPublicKVGate_BlocksMutatorsUntilReady(t *testing.T) { pb.RawKV_RawGet_FullMethodName, pb.TransactionalKV_Get_FullMethodName, pb.Internal_Forward_FullMethodName, + pb.Internal_ApplyTargetStagedReadiness_FullMethodName, + pb.Internal_ImportRangeVersions_FullMethodName, + pb.Internal_PromoteStagedVersions_FullMethodName, + pb.Internal_CleanupMigration_FullMethodName, + pb.Internal_IssueMigrationTimestamp_FullMethodName, pb.AdminForward_Forward_FullMethodName, pb.Distribution_GetTimestamp_FullMethodName, pb.Distribution_SplitRange_FullMethodName, + pb.Distribution_StartSplitMigration_FullMethodName, + pb.Distribution_GetSplitMigrationCapability_FullMethodName, + pb.Distribution_AttestMigrationOwnership_FullMethodName, + pb.Distribution_AbandonSplitJob_FullMethodName, + pb.Distribution_RetrySplitJob_FullMethodName, pb.RaftAdmin_AddVoter_FullMethodName, pb.RaftAdmin_AddLearner_FullMethodName, pb.RaftAdmin_PromoteLearner_FullMethodName, diff --git a/main_leader_balance.go b/main_leader_balance.go index 5612b9184..3d4e0c740 100644 --- a/main_leader_balance.go +++ b/main_leader_balance.go @@ -24,6 +24,7 @@ const ( defaultLeaderBalanceImbalanceThreshold = 2 defaultLeaderBalanceMaxTargetLag = 1024 defaultLeaderBalanceRPCTimeout = 5 * time.Second + raftMemberSuffrageVoter = "voter" ) type leaderBalanceConfig struct { @@ -345,7 +346,7 @@ func observeLeaderBalance(ctx context.Context, runtimes []*raftGroupRuntime, loc localLeader: status.State == raftengine.StateLeader && status.Leader.ID == localNodeID, } for _, server := range cfg.Servers { - if server.Suffrage != shutdownVoterSuffrage { + if server.Suffrage != raftMemberSuffrageVoter { continue } obs.voters[server.ID] = server diff --git a/main_migration_apply_members.go b/main_migration_apply_members.go new file mode 100644 index 000000000..214add532 --- /dev/null +++ b/main_migration_apply_members.go @@ -0,0 +1,80 @@ +package main + +import ( + "context" + "strings" + + "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + "github.com/cockroachdb/errors" +) + +func splitMigrationApplyMemberSnapshotFactory(groups map[uint64]*kv.ShardGroup, cache *kv.GRPCConnCache, token string) adapter.SplitMigrationApplyMemberSnapshotFactory { + return func(ctx context.Context, groupID uint64) (adapter.SplitMigrationApplyMemberSnapshot, error) { + group := groups[groupID] + if group == nil || group.Engine == nil || cache == nil { + return adapter.SplitMigrationApplyMemberSnapshot{}, errors.New("migration apply-member group or connection cache is missing") + } + engine := group.Engine + before, err := splitMigrationApplyConfiguration(engine) + if err != nil { + return adapter.SplitMigrationApplyMemberSnapshot{}, err + } + cfg, err := engine.Configuration(ctx) + if err != nil { + return adapter.SplitMigrationApplyMemberSnapshot{}, errors.WithStack(err) + } + members, err := splitMigrationApplyMembers(groupID, cfg.Servers, cache, token) + if err != nil { + return adapter.SplitMigrationApplyMemberSnapshot{}, err + } + after, err := splitMigrationApplyConfiguration(engine) + if err != nil { + return adapter.SplitMigrationApplyMemberSnapshot{}, err + } + if before != after { + return adapter.SplitMigrationApplyMemberSnapshot{}, errors.New("migration apply-member configuration changed during resolution") + } + return adapter.SplitMigrationApplyMemberSnapshot{ConfigurationIndex: before, Members: members}, nil + } +} + +func splitMigrationApplyConfiguration(engine raftengine.Engine) (uint64, error) { + reader, ok := engine.(raftengine.AppliedConfigurationReader) + if !ok { + return 0, errors.New("migration apply-member engine lacks applied configuration reader") + } + before := engine.Status() + index, supported := reader.AppliedConfigurationIndex() + after := engine.Status() + if !supported || index == 0 || before.PendingConfChange || after.PendingConfChange || + before.ConfigurationIndex != index || after.ConfigurationIndex != index { + return 0, errors.New("migration apply-member configuration is not stably applied") + } + return index, nil +} + +func splitMigrationApplyMembers(groupID uint64, servers []raftengine.Server, cache *kv.GRPCConnCache, token string) ([]adapter.SplitMigrationVoter, error) { + if len(servers) == 0 { + return nil, errors.New("migration apply-member configuration is empty") + } + members := make([]adapter.SplitMigrationVoter, 0, len(servers)) + ids, addresses := make(map[string]struct{}, len(servers)), make(map[string]struct{}, len(servers)) + for _, server := range servers { + server.ID, server.Address = strings.TrimSpace(server.ID), strings.TrimSpace(server.Address) + if _, duplicate := ids[server.ID]; duplicate { + return nil, errors.New("migration apply-member ID is duplicated") + } + if _, duplicate := addresses[server.Address]; duplicate { + return nil, errors.New("migration apply-member address is duplicated") + } + member, err := splitMigrationVoter(groupID, server, cache, token) + if err != nil { + return nil, err + } + ids[member.ID], addresses[member.Address] = struct{}{}, struct{}{} + members = append(members, member) + } + return members, nil +} diff --git a/main_migration_apply_members_test.go b/main_migration_apply_members_test.go new file mode 100644 index 000000000..c88fa1b64 --- /dev/null +++ b/main_migration_apply_members_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "context" + "net" + "testing" + "time" + + internalutil "github.com/bootjp/elastickv/internal" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +type applyMemberEngine struct { + capabilityConfigEngine + index uint64 + pending bool + configured func() +} + +type applyMemberAuthServer struct{ pb.UnimplementedInternalServer } + +func (applyMemberAuthServer) ProbeMigrationState(ctx context.Context, _ *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + md, _ := metadata.FromIncomingContext(ctx) + if got := md.Get("authorization"); len(got) != 1 || got[0] != "Bearer migration-secret" { + return nil, status.Error(codes.Unauthenticated, "missing migration credential") + } + if got := md.Get("trace-id"); len(got) != 1 || got[0] != "preserved" { + return nil, status.Error(codes.InvalidArgument, "missing caller metadata") + } + return &pb.ProbeMigrationStateResponse{Ready: true}, nil +} + +func TestSplitMigrationApplyMemberSnapshotFactoryPreservesTLSCredentials(t *testing.T) { + cert, key := writeSelfSignedCert(t) + require.NoError(t, internalutil.ConfigureGRPCTLS(cert, cert, key)) + t.Cleanup(func() { require.NoError(t, internalutil.ConfigureGRPCTLS("", "", "")) }) + server := grpc.NewServer(internalutil.GRPCServerOptions()...) + pb.RegisterInternalServer(server, applyMemberAuthServer{}) + engine := newApplyMemberEngine() + for index := range engine.cfg.Servers { + var lc net.ListenConfig + listener, err := lc.Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + engine.cfg.Servers[index].Address = listener.Addr().String() + done := make(chan error, 1) + go func() { done <- server.Serve(listener) }() + t.Cleanup(func() { server.Stop(); require.NoError(t, <-done) }) + } + var cache kv.GRPCConnCache + t.Cleanup(func() { require.NoError(t, cache.Close()) }) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("authorization", "Bearer stale", "trace-id", "preserved")) + snapshot, err := splitMigrationApplyMemberSnapshotFactory(map[uint64]*kv.ShardGroup{2: {Engine: engine}}, &cache, "migration-secret")(ctx, 2) + require.NoError(t, err) + for _, member := range snapshot.Members { + _, err := member.Client.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{JobId: 11}) + require.NoError(t, err, member.ID) + conn, err := cache.ConnFor(member.Address) + require.NoError(t, err) + _, err = pb.NewInternalClient(conn).ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{JobId: 11}) + require.Equal(t, codes.Unauthenticated, status.Code(err), "shared cache must not retain a runner token") + } + md, _ := metadata.FromOutgoingContext(ctx) + require.Equal(t, []string{"Bearer stale"}, md.Get("authorization")) +} + +func (e *applyMemberEngine) AppliedConfigurationIndex() (uint64, bool) { return e.index, true } +func (e *applyMemberEngine) Status() raftengine.Status { + return raftengine.Status{ConfigurationIndex: e.index, PendingConfChange: e.pending} +} +func (e *applyMemberEngine) Configuration(context.Context) (raftengine.Configuration, error) { + if e.configured != nil { + e.configured() + } + return e.cfg, nil +} + +func newApplyMemberEngine() *applyMemberEngine { + return &applyMemberEngine{index: 41, capabilityConfigEngine: capabilityConfigEngine{cfg: raftengine.Configuration{Servers: []raftengine.Server{ + {ID: "voter", Address: "127.0.0.1:19001", Suffrage: raftMemberSuffrageVoter}, + {ID: "learner", Address: "127.0.0.1:19002", Suffrage: "Nonvoter"}, + }}}} +} + +func TestSplitMigrationApplyMemberSnapshotFactoryIncludesLearners(t *testing.T) { + engine := newApplyMemberEngine() + var cache kv.GRPCConnCache + t.Cleanup(func() { require.NoError(t, cache.Close()) }) + factory := splitMigrationApplyMemberSnapshotFactory(map[uint64]*kv.ShardGroup{2: {Engine: engine}}, &cache, "migration-secret") + snapshot, err := factory(t.Context(), 2) + require.NoError(t, err) + require.Equal(t, uint64(41), snapshot.ConfigurationIndex) + require.Len(t, snapshot.Members, 2) + require.Equal(t, "learner", snapshot.Members[1].ID) + for _, member := range snapshot.Members { + require.NotNil(t, member.Client) + } +} + +func TestSplitMigrationApplyMemberSnapshotFactoryRejectsUnstableConfiguration(t *testing.T) { + cases := map[string]func(*applyMemberEngine){ + "zero epoch": func(e *applyMemberEngine) { e.index = 0 }, + "pending before": func(e *applyMemberEngine) { e.pending = true }, + "epoch changes": func(e *applyMemberEngine) { e.configured = func() { e.index++ } }, + "pending after": func(e *applyMemberEngine) { e.configured = func() { e.pending = true } }, + "empty": func(e *applyMemberEngine) { e.cfg.Servers = nil }, + "missing id": func(e *applyMemberEngine) { e.cfg.Servers[1].ID = "" }, + "duplicate id": func(e *applyMemberEngine) { e.cfg.Servers[1].ID = e.cfg.Servers[0].ID }, + "duplicate address": func(e *applyMemberEngine) { e.cfg.Servers[1].Address = e.cfg.Servers[0].Address }, + } + for name, change := range cases { + t.Run(name, func(t *testing.T) { + engine := newApplyMemberEngine() + var cache kv.GRPCConnCache + t.Cleanup(func() { require.NoError(t, cache.Close()) }) + groups := map[uint64]*kv.ShardGroup{2: {Engine: engine}} + change(engine) + _, err := splitMigrationApplyMemberSnapshotFactory(groups, &cache, "migration-secret")(t.Context(), 2) + require.Error(t, err) + }) + } +} + +func TestSplitMigrationApplyMemberSnapshotFactoryRequiresWiring(t *testing.T) { + var cache kv.GRPCConnCache + t.Cleanup(func() { require.NoError(t, cache.Close()) }) + groups := map[uint64]*kv.ShardGroup{2: {Engine: newApplyMemberEngine()}} + _, err := splitMigrationApplyMemberSnapshotFactory(groups, nil, "migration-secret")(t.Context(), 2) + require.Error(t, err) + _, err = splitMigrationApplyMemberSnapshotFactory(groups, &cache, "migration-secret")(t.Context(), 3) + require.Error(t, err) + groups[2].Engine = capabilityConfigEngine{} + _, err = splitMigrationApplyMemberSnapshotFactory(groups, &cache, "migration-secret")(t.Context(), 2) + require.Error(t, err) +} diff --git a/main_migration_membership.go b/main_migration_membership.go new file mode 100644 index 000000000..39aa685f5 --- /dev/null +++ b/main_migration_membership.go @@ -0,0 +1,91 @@ +package main + +import ( + "context" + + "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/internal/raftadmin" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +func migrationMemberProtocolsConfigured() bool { + return adapter.MigrationImportOpcodeEnabledFromEnv() || adapter.MigrationPromoteOpcodeEnabledFromEnv() || + adapter.MigrationCleanupOpcodeEnabledFromEnv() +} + +func migrationMembershipInterceptor(runtime *raftGroupRuntime, shards *kv.ShardStore, proposer raftengine.Proposer, + token string, next raftadmin.MembershipChangeInterceptor, +) raftadmin.MembershipChangeInterceptor { + if runtime == nil { + return next + } + prepare := newPrepareEvidenceMembershipGate(shards, runtime.spec.id, next) + // Dedicated TSO has neither KV migration controls nor an MVCC namespace. + if runtime.spec.id == 0 { + return prepare + } + protocols := newMigrationMembershipGate(runtime.store, migrationMemberProtocolsConfigured(), + newMigrationMemberProtocolProbe(token), prepare) + return newMigrationNamespaceMembershipGate(runtime.store, runtime.snapshotEngine(), proposer, token, protocols) +} + +// Check support before any new member can race a guarded-control proposal. +// Existing durable evidence keeps the requirement after feature flags change. +type migrationMembershipGate struct { + store store.MVCCStore + enabled bool + probe func(context.Context, string) error + next raftadmin.MembershipChangeInterceptor +} + +func newMigrationMembershipGate(st store.MVCCStore, enabled bool, probe func(context.Context, string) error, next raftadmin.MembershipChangeInterceptor) *migrationMembershipGate { + return &migrationMembershipGate{store: st, enabled: enabled, probe: probe, next: next} +} + +func (g *migrationMembershipGate) PreAddMember(ctx context.Context, id, address string) error { + if err := g.check(ctx, address); err != nil { + return err + } + if g.next != nil { + return errors.WithStack(g.next.PreAddMember(ctx, id, address)) + } + return nil +} + +func (g *migrationMembershipGate) PrePromoteMember(ctx context.Context, id, address string) error { + if err := g.check(ctx, address); err != nil { + return err + } + if next, ok := g.next.(raftadmin.MembershipPromotionInterceptor); ok { + return errors.WithStack(next.PrePromoteMember(ctx, id, address)) + } + return nil +} + +func (g *migrationMembershipGate) check(ctx context.Context, address string) error { + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + required, err := g.protocolRequired(ctx) + if err != nil || !required { + return err + } + if g.probe == nil || address == "" { + return errors.New("migration membership capability probe is not configured") + } + return errors.WithStack(g.probe(ctx, address)) +} + +func (g *migrationMembershipGate) protocolRequired(ctx context.Context) (bool, error) { + if g.enabled { + return true, nil + } + if g.store == nil { + return false, nil + } + present, err := kv.MigrationPartitionHistoryPresent(ctx, g.store) + return present, errors.WithStack(err) +} diff --git a/main_migration_membership_probe.go b/main_migration_membership_probe.go new file mode 100644 index 000000000..de5019473 --- /dev/null +++ b/main_migration_membership_probe.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "strings" + "time" + + internalutil "github.com/bootjp/elastickv/internal" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const migrationMemberProbeTimeout = 2 * time.Second + +func newMigrationMemberProtocolProbe(token string) func(context.Context, string) error { + return func(ctx context.Context, address string) error { + if strings.TrimSpace(address) == "" { + return errors.WithStack(status.Error(codes.FailedPrecondition, "migration member address is empty")) + } + conn, err := grpc.NewClient(address, internalutil.GRPCDialOptions()...) + if err != nil { + return errors.Wrap(err, "migration member capability connection") + } + defer func() { _ = conn.Close() }() + return checkMigrationMemberProtocols(ctx, newSplitMigrationInternalClient(conn, token)) + } +} + +func checkMigrationMemberProtocols(ctx context.Context, client pb.InternalClient) error { + attempt, cancel := context.WithTimeout(ctx, migrationMemberProbeTimeout) + defer cancel() + // Capability queries require a nonzero syntactic job ID but do not read or + // mutate that job. A joining learner need not have an applied epoch yet. + response, err := client.ProbeMigrationState(attempt, &pb.ProbeMigrationStateRequest{ + JobId: 1, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY, + }) + if err != nil { + return errors.Wrap(err, "migration member capability probe") + } + return validateMigrationMemberProtocols(response.GetTransferSizeGuardVersion(), response.GetOwnershipPartitionProtocol()) +} + +func validateMigrationMemberProtocols(transferVersion, partitionVersion uint32) error { + const requiredTransferVersion, requiredPartitionVersion = 1, 1 + if transferVersion != requiredTransferVersion || partitionVersion != requiredPartitionVersion { + return errors.WithStack(status.Errorf(codes.FailedPrecondition, + "migration member lacks required protocols: transfer-size=%d ownership-partition=%d", transferVersion, partitionVersion)) + } + return nil +} diff --git a/main_migration_membership_probe_test.go b/main_migration_membership_probe_test.go new file mode 100644 index 000000000..e37713157 --- /dev/null +++ b/main_migration_membership_probe_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/adapter" + internalutil "github.com/bootjp/elastickv/internal" + "github.com/bootjp/elastickv/internal/testutil/grpctest" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +type migrationMemberProbeServer struct { + pb.UnimplementedInternalServer + response *pb.ProbeMigrationStateResponse + err error +} + +func (s migrationMemberProbeServer) ProbeMigrationState(_ context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + if req.GetJobId() == 0 || req.GetKind() != pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY || req.GetRequireConfigurationIndex() { + return nil, status.Error(codes.InvalidArgument, "expected static capability query") + } + return s.response, s.err +} + +func TestMigrationMemberProtocolProbeAcceptsFreshLearnerNotLegacyEcho(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + transfer uint32 + partition uint32 + want codes.Code + }{ + {name: "fresh-learner", transfer: 1, partition: 1, want: codes.OK}, + {name: "legacy-response", want: codes.FailedPrecondition}, + {name: "size-only", transfer: 1, want: codes.FailedPrecondition}, + {name: "partition-only", partition: 1, want: codes.FailedPrecondition}, + {name: "unknown-size-version", transfer: 2, partition: 1, want: codes.FailedPrecondition}, + {name: "unknown-partition-version", transfer: 1, partition: 2, want: codes.FailedPrecondition}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + address := grpctest.Serve(t, func(server *grpc.Server) { + pb.RegisterInternalServer(server, migrationMemberProbeServer{response: &pb.ProbeMigrationStateResponse{ + TransferSizeGuardVersion: test.transfer, OwnershipPartitionProtocol: test.partition, + }}) + }) + probe := newMigrationMemberProtocolProbe("") + err := probe(t.Context(), address) + require.Equal(t, test.want, status.Code(err), "%v", err) + }) + } +} + +func TestMigrationMemberProtocolProbeDoesNotBypassMissingRPCOrCancellation(t *testing.T) { + t.Parallel() + address := grpctest.Serve(t, func(server *grpc.Server) { + pb.RegisterInternalServer(server, &pb.UnimplementedInternalServer{}) + }) + probe := newMigrationMemberProtocolProbe("") + require.Equal(t, codes.Unimplemented, status.Code(probe(t.Context(), address))) + require.Equal(t, codes.FailedPrecondition, status.Code(probe(t.Context(), " "))) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + started := time.Now() + require.Equal(t, codes.Canceled, status.Code(probe(ctx, address))) + require.Less(t, time.Since(started), migrationMemberProbeTimeout) +} + +func TestMigrationMemberProtocolProbeUsesDedicatedTLSCredential(t *testing.T) { + cert, key := writeSelfSignedCert(t) + require.NoError(t, internalutil.ConfigureGRPCTLS(cert, cert, key)) + t.Cleanup(func() { require.NoError(t, internalutil.ConfigureGRPCTLS("", "", "")) }) + auth, _ := adapter.SplitMigrationTokenAuth("migration-secret", false, true) + address := grpctest.Serve(t, func(server *grpc.Server) { + pb.RegisterInternalServer(server, migrationMemberProbeServer{response: &pb.ProbeMigrationStateResponse{ + TransferSizeGuardVersion: 1, OwnershipPartitionProtocol: 1, + }}) + }, grpc.ChainUnaryInterceptor(auth, func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + md, _ := metadata.FromIncomingContext(ctx) + require.Equal(t, []string{"Bearer migration-secret"}, md.Get("authorization")) + require.Equal(t, []string{"preserved"}, md.Get("trace-id")) + return handler(ctx, req) + })) + ctx := metadata.NewOutgoingContext(t.Context(), metadata.Pairs("authorization", "Bearer stale", "trace-id", "preserved")) + require.NoError(t, newMigrationMemberProtocolProbe("migration-secret")(ctx, address)) + require.Equal(t, codes.Unauthenticated, status.Code(newMigrationMemberProtocolProbe("wrong-token")(ctx, address))) + md, _ := metadata.FromOutgoingContext(ctx) + require.Equal(t, []string{"Bearer stale"}, md.Get("authorization")) +} + +func TestMigrationMemberProtocolProbeRefusesCredentialOnPlaintext(t *testing.T) { + t.Parallel() + address := grpctest.Serve(t, func(server *grpc.Server) { + pb.RegisterInternalServer(server, migrationMemberProbeServer{response: &pb.ProbeMigrationStateResponse{ + TransferSizeGuardVersion: 1, OwnershipPartitionProtocol: 1, + }}) + }) + require.Error(t, newMigrationMemberProtocolProbe("migration-secret")(t.Context(), address)) +} diff --git a/main_migration_membership_test.go b/main_migration_membership_test.go new file mode 100644 index 000000000..22a53ab92 --- /dev/null +++ b/main_migration_membership_test.go @@ -0,0 +1,110 @@ +package main + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/internal/raftadmin" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +type migrationMembershipStore struct { + store.MVCCStore + store.MigrationOwnershipStore + view store.MigrationServingState + err error +} + +func (s migrationMembershipStore) MigrationServingState(context.Context) (store.MigrationServingState, error) { + return s.view, s.err +} + +type migrationMembershipNext struct { + add func() + promote func() + err error +} + +func (n migrationMembershipNext) PreAddMember(context.Context, string, string) error { + n.add() + return n.err +} + +func (n migrationMembershipNext) PrePromoteMember(context.Context, string, string) error { + n.promote() + return n.err +} + +func TestMigrationMembershipRequiresSupportBeforeNextHook(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + enabled bool + view store.MigrationServingState + }{ + {name: "enabled-before-first-control", enabled: true}, + {name: "durable-ownership", view: store.MigrationServingState{Ownership: []store.MigrationOwnershipState{{JobID: 7}}}}, + {name: "durable-guard", view: store.MigrationServingState{Readiness: []store.TargetStagedReadinessState{{JobID: 7, Armed: true}}}}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + var order []string + next := migrationMembershipNext{add: func() { order = append(order, "add") }, promote: func() { order = append(order, "promote") }} + gate := newMigrationMembershipGate(migrationMembershipStore{view: test.view}, test.enabled, + func(_ context.Context, address string) error { + require.Equal(t, "candidate:50051", address) + order = append(order, "probe") + return nil + }, next) + require.NoError(t, gate.PreAddMember(t.Context(), "candidate", "candidate:50051")) + require.NoError(t, gate.PrePromoteMember(t.Context(), "candidate", "candidate:50051")) + require.Equal(t, []string{"probe", "add", "probe", "promote"}, order) + }) + } +} + +func TestMigrationMembershipRefusalStopsNextHook(t *testing.T) { + t.Parallel() + sentinel := errors.New("unsupported protocol") + next := migrationMembershipNext{add: func() { t.Fatal("add hook ran") }, promote: func() { t.Fatal("promotion hook ran") }} + gate := newMigrationMembershipGate(nil, true, func(context.Context, string) error { return sentinel }, next) + require.ErrorIs(t, gate.PreAddMember(t.Context(), "candidate", "candidate:50051"), sentinel) + require.ErrorIs(t, gate.PrePromoteMember(t.Context(), "candidate", "candidate:50051"), sentinel) + gate.probe = nil + require.Error(t, gate.PreAddMember(t.Context(), "candidate", "candidate:50051")) +} + +func TestMigrationMembershipPreservesUnconfiguredClusters(t *testing.T) { + t.Parallel() + for _, st := range []store.MVCCStore{nil, store.NewMVCCStore(), migrationMembershipStore{}} { + calls := 0 + var next raftadmin.MembershipChangeInterceptor = migrationMembershipNext{add: func() { calls++ }} + gate := newMigrationMembershipGate(st, false, func(context.Context, string) error { + t.Fatal("ordinary cluster must not probe migration support") + return nil + }, next) + require.NoError(t, gate.PreAddMember(t.Context(), "legacy", "legacy:50051")) + require.Equal(t, 1, calls) + } +} + +func TestMigrationMembershipMetadataErrorAndCancellationStopAdmission(t *testing.T) { + t.Parallel() + sentinel := errors.New("ownership metadata unavailable") + gate := newMigrationMembershipGate(migrationMembershipStore{err: sentinel}, false, nil, nil) + require.ErrorIs(t, gate.PreAddMember(t.Context(), "candidate", "candidate:50051"), sentinel) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.ErrorIs(t, gate.PrePromoteMember(ctx, "candidate", "candidate:50051"), context.Canceled) +} + +func TestMigrationMembershipPreservesNextHookErrors(t *testing.T) { + t.Parallel() + sentinel := errors.New("prepare evidence protocol unsupported") + next := migrationMembershipNext{add: func() {}, promote: func() {}, err: sentinel} + gate := newMigrationMembershipGate(nil, false, nil, next) + require.ErrorIs(t, gate.PreAddMember(t.Context(), "candidate", "candidate:50051"), sentinel) + require.ErrorIs(t, gate.PrePromoteMember(t.Context(), "candidate", "candidate:50051"), sentinel) +} diff --git a/main_migration_namespace_membership.go b/main_migration_namespace_membership.go new file mode 100644 index 000000000..6ccc1a0e1 --- /dev/null +++ b/main_migration_namespace_membership.go @@ -0,0 +1,188 @@ +package main + +import ( + "context" + "strings" + "sync" + "time" + + "github.com/bootjp/elastickv/adapter" + internalutil "github.com/bootjp/elastickv/internal" + "github.com/bootjp/elastickv/internal/raftadmin" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" +) + +const namespaceMembershipHandshakeTimeout = 2 * time.Second + +type migrationNamespaceMembershipGate struct { + mu sync.Mutex + store store.MVCCStore + engine raftengine.Engine + proposer raftengine.Proposer + probe func(context.Context, string, *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) + next raftadmin.MembershipChangeInterceptor +} + +var _ raftadmin.MembershipChangeInterceptor = (*migrationNamespaceMembershipGate)(nil) +var _ raftadmin.MembershipPromotionInterceptor = (*migrationNamespaceMembershipGate)(nil) + +func newMigrationNamespaceMembershipGate(st store.MVCCStore, engine raftengine.Engine, proposer raftengine.Proposer, token string, next raftadmin.MembershipChangeInterceptor) *migrationNamespaceMembershipGate { + return &migrationNamespaceMembershipGate{store: st, engine: engine, proposer: proposer, next: next, + probe: func(ctx context.Context, address string, request *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + conn, err := grpc.NewClient(address, internalutil.GRPCDialOptions()...) + if err != nil { + return nil, errors.WithStack(err) + } + defer func() { _ = conn.Close() }() + response, err := newSplitMigrationInternalClient(conn, token).ProbeMigrationState(ctx, request) + return response, errors.WithStack(err) + }} +} + +func (g *migrationNamespaceMembershipGate) PreAddMember(ctx context.Context, id, address string) error { + g.mu.Lock() + defer g.mu.Unlock() + if err := g.authorize(ctx, id, address); err != nil { + return err + } + if g.next != nil { + return errors.WithStack(g.next.PreAddMember(ctx, id, address)) + } + return nil +} + +func (g *migrationNamespaceMembershipGate) PrePromoteMember(ctx context.Context, id, address string) error { + g.mu.Lock() + defer g.mu.Unlock() + if err := g.authorize(ctx, id, address); err != nil { + return err + } + if next, ok := g.next.(raftadmin.MembershipPromotionInterceptor); ok { + return errors.WithStack(next.PrePromoteMember(ctx, id, address)) + } + return nil +} + +func (g *migrationNamespaceMembershipGate) authorize(ctx context.Context, id, address string) error { + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + version, err := kv.MigrationNamespaceAdmissionVersion(ctx, g.store) + if err != nil || version == 0 { + return errors.WithStack(err) + } + peer, installedVersion, err := g.probeNamespaceCandidate(ctx, id, address, version) + if err != nil { + return err + } + snapshot := func(ctx context.Context) (adapter.NamespaceAdmissionSnapshot, error) { return g.snapshot(ctx) } + current, err := snapshot(ctx) + if err != nil { + return err + } + joining, err := namespaceJoiningCandidate(current, peer) + if err != nil { + return err + } + activate := func(ctx context.Context, command kv.NamespaceAdmissionCommand) error { + return errors.WithStack(kv.ActivateMigrationNamespace(ctx, g.store, g.proposer, command)) + } + if err := adapter.EnsureMigrationNamespaceAdmissionWithSnapshot(ctx, snapshot, activate, joining); err != nil { + return errors.WithStack(err) + } + configuration, err := kv.MigrationConfigurationIndex(ctx, g.store) + if err != nil { + return errors.WithStack(err) + } + return errors.WithStack(kv.CheckMigrationNamespacePrejoin(ctx, g.store, configuration, peer.MemberID, installedVersion)) +} + +func (g *migrationNamespaceMembershipGate) probeNamespaceCandidate(ctx context.Context, id, address string, version uint32) (adapter.NamespaceAdmissionPeer, uint32, error) { + if strings.TrimSpace(id) == "" || id != strings.TrimSpace(id) || address == "" || g.engine == nil || g.proposer == nil || g.probe == nil { + return adapter.NamespaceAdmissionPeer{}, 0, errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + peer := g.peer(etcd.DeriveNodeID(id), address) + attempt, cancel := context.WithTimeout(ctx, namespaceMembershipHandshakeTimeout) + response, err := peer.Client.ProbeMigrationState(attempt, &pb.ProbeMigrationStateRequest{JobId: 1, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_NAMESPACE_ADMISSION}) + cancel() + if err != nil { + return peer, 0, errors.WithStack(err) + } + if response.GetNamespaceAdmissionVersion() < version { + return peer, 0, errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + return peer, response.GetNamespaceAdmissionVersion(), nil +} + +func namespaceJoiningCandidate(current adapter.NamespaceAdmissionSnapshot, peer adapter.NamespaceAdmissionPeer) ([]adapter.NamespaceAdmissionPeer, error) { + joining := []adapter.NamespaceAdmissionPeer{peer} + for _, member := range current.Members { + if member.MemberID == peer.MemberID { + if member.Address != peer.Address { + return nil, errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + joining = nil + } + } + return joining, nil +} + +type namespaceMembershipProbeClient struct { + address string + probe func(context.Context, string, *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) +} + +func (c namespaceMembershipProbeClient) ProbeMigrationState(ctx context.Context, request *pb.ProbeMigrationStateRequest, _ ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) { + return c.probe(ctx, c.address, request) +} + +func (g *migrationNamespaceMembershipGate) peer(id uint64, address string) adapter.NamespaceAdmissionPeer { + return adapter.NamespaceAdmissionPeer{MemberID: id, Address: address, Client: namespaceMembershipProbeClient{address: address, probe: g.probe}} +} + +func (g *migrationNamespaceMembershipGate) snapshot(ctx context.Context) (adapter.NamespaceAdmissionSnapshot, error) { + if _, err := g.engine.LinearizableRead(ctx); err != nil { + return adapter.NamespaceAdmissionSnapshot{}, errors.WithStack(err) + } + reader, ok := g.engine.(raftengine.AppliedConfigurationReader) + if !ok { + return adapter.NamespaceAdmissionSnapshot{}, errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + before, err := g.namespaceSnapshotConfigurationIndex(reader) + if err != nil { + return adapter.NamespaceAdmissionSnapshot{}, err + } + configuration, err := g.engine.Configuration(ctx) + if err != nil { + return adapter.NamespaceAdmissionSnapshot{}, errors.WithStack(err) + } + result := adapter.NamespaceAdmissionSnapshot{ConfigurationIndex: before} + for _, member := range configuration.Servers { + if member.ID == "" || member.ID != strings.TrimSpace(member.ID) { + return result, errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + result.Members = append(result.Members, g.peer(etcd.DeriveNodeID(member.ID), member.Address)) + } + after, err := g.namespaceSnapshotConfigurationIndex(reader) + if err != nil { + return result, err + } + if before != after { + return result, errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + return result, nil +} + +func (g *migrationNamespaceMembershipGate) namespaceSnapshotConfigurationIndex(reader raftengine.AppliedConfigurationReader) (uint64, error) { + index, supported := reader.AppliedConfigurationIndex() + if !supported || index == 0 || g.engine.Status().PendingConfChange { + return 0, errors.WithStack(kv.ErrMigrationNamespaceProtocol) + } + return index, nil +} diff --git a/main_migration_namespace_membership_test.go b/main_migration_namespace_membership_test.go new file mode 100644 index 000000000..ec9b2cd3d --- /dev/null +++ b/main_migration_namespace_membership_test.go @@ -0,0 +1,162 @@ +package main + +import ( + "context" + "slices" + "testing" + + "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +type namespaceGateEngine struct { + raftengine.Engine + store store.MVCCStore + fsm raftengine.StateMachine + configuration raftengine.Configuration + proposals int + index uint64 +} + +func (e *namespaceGateEngine) ProposeAdmin(_ context.Context, data []byte) (*raftengine.ProposalResult, error) { + e.proposals++ + e.index++ + aware, ok := e.fsm.(raftengine.ApplyIndexAware) + if !ok { + return nil, kv.ErrMigrationNamespaceProtocol + } + aware.SetApplyIndex(e.index) + return &raftengine.ProposalResult{CommitIndex: e.index, Response: e.fsm.Apply(data)}, nil +} +func (e *namespaceGateEngine) Configuration(context.Context) (raftengine.Configuration, error) { + return e.configuration, nil +} +func (e *namespaceGateEngine) LinearizableRead(context.Context) (uint64, error) { return e.index, nil } +func (e *namespaceGateEngine) Status() raftengine.Status { + return raftengine.Status{ConfigurationIndex: 7} +} +func (e *namespaceGateEngine) AppliedConfigurationIndex() (uint64, bool) { + index, err := kv.MigrationConfigurationIndex(context.Background(), e.store) + return index, err == nil && index != 0 +} + +type namespaceGateNext struct { + check func() + adds, promotions int +} + +func (n *namespaceGateNext) PreAddMember(context.Context, string, string) error { + n.check() + n.adds++ + return nil +} +func (n *namespaceGateNext) PrePromoteMember(context.Context, string, string) error { + n.check() + n.promotions++ + return nil +} + +func newNamespaceGateFixture(t *testing.T, active bool) (*migrationNamespaceMembershipGate, *namespaceGateEngine, *namespaceGateNext, map[string]uint32, map[string]int) { + t.Helper() + st := store.NewMVCCStore() + fsm := kv.NewKvFSMWithHLC(st, nil) + ids := []uint64{etcd.DeriveNodeID("n1"), etcd.DeriveNodeID("learner")} + aware, ok := fsm.(raftengine.NamespaceConfigurationAware) + require.True(t, ok) + require.NoError(t, aware.RecordAppliedNamespaceConfiguration(7, ids)) + engine := &namespaceGateEngine{store: st, fsm: fsm, index: 7, configuration: raftengine.Configuration{Servers: []raftengine.Server{{ID: "n1", Address: "n1:7001", Suffrage: "voter"}, {ID: "learner", Address: "learner:7001", Suffrage: "learner"}}}} + if active { + slices.Sort(ids) + command := kv.NamespaceAdmissionCommand{Version: kv.MigrationNamespaceVersion, ConfigurationIndex: 7, ApplyMembers: ids} + for _, id := range ids { + command.Members = append(command.Members, kv.NamespaceMemberCapability{MemberID: id, Version: kv.MigrationNamespaceVersion, ConfigurationIndex: 7}) + } + require.NoError(t, kv.ActivateMigrationNamespace(t.Context(), st, engine, command)) + } + next := &namespaceGateNext{check: func() {}} + gate := newMigrationNamespaceMembershipGate(st, engine, engine, "secret", next) + versions := map[string]uint32{"n1:7001": 1, "learner:7001": 1, "candidate:7001": 1} + probes := map[string]int{} + gate.probe = func(ctx context.Context, address string, _ *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + probes[address]++ + version, err := kv.MigrationNamespaceAdmissionVersion(ctx, st) + if err != nil { + return nil, err + } + index := uint64(7) + if address == "candidate:7001" { + index = 0 + } + return &pb.ProbeMigrationStateResponse{NamespaceAdmissionVersion: versions[address], AppliedConfigurationIndex: index, Ready: version == kv.MigrationNamespaceVersion}, nil + } + return gate, engine, next, versions, probes +} + +func TestMigrationNamespaceMembershipRegistersBeforeAddAndProbesEveryMember(t *testing.T) { + gate, engine, next, _, probes := newNamespaceGateFixture(t, true) + next.check = func() { + value, err := engine.store.GetAt(t.Context(), store.MigrationConfigurationIndexKey(), ^uint64(0)) + require.NoError(t, err) + state, err := store.DecodeMigrationNamespaceConfiguration(value) + require.NoError(t, err) + require.Contains(t, state.Joining, etcd.DeriveNodeID("candidate")) + } + require.NoError(t, gate.PreAddMember(t.Context(), "candidate", "candidate:7001")) + require.Equal(t, 1, next.adds) + for _, address := range []string{"n1:7001", "learner:7001", "candidate:7001"} { + require.Positive(t, probes[address], address) + } +} + +func TestMigrationNamespaceMembershipRejectsOldCandidateAndOldLearnerBeforeProposal(t *testing.T) { + for _, old := range []string{"candidate:7001", "learner:7001"} { + gate, engine, next, versions, _ := newNamespaceGateFixture(t, true) + versions[old] = 0 + before := engine.proposals + require.Error(t, gate.PreAddMember(t.Context(), "candidate", "candidate:7001")) + require.Equal(t, before, engine.proposals) + require.Zero(t, next.adds) + } +} + +func TestMigrationNamespaceMembershipPreservesInactiveAndRechecksPromotion(t *testing.T) { + gate, _, next, _, probes := newNamespaceGateFixture(t, false) + require.NoError(t, gate.PreAddMember(t.Context(), "candidate", "candidate:7001")) + require.Empty(t, probes) + require.Equal(t, 1, next.adds) + gate, _, next, versions, _ := newNamespaceGateFixture(t, true) + require.NoError(t, gate.PrePromoteMember(t.Context(), "learner", "learner:7001")) + require.Equal(t, 1, next.promotions) + versions["learner:7001"] = 0 + require.Error(t, gate.PrePromoteMember(t.Context(), "learner", "learner:7001")) + require.Equal(t, 1, next.promotions) +} + +type namespaceHiddenFSM struct{ raftengine.StateMachine } + +func TestMigrationNamespaceProductionFSMInstalledWithEncryptionOptions(t *testing.T) { + for _, encrypted := range []bool{false, true} { + wiring := encryptionWriteWiring{} + if encrypted { + cipher, err := encryption.NewCipher(encryption.NewKeystore()) + require.NoError(t, err) + wiring = encryptionWriteWiring{cache: encryption.NewStateCache(), cipher: cipher, nonceFactory: encryption.NewDeterministicNonceFactory(1, 1)} + } + st, err := store.NewPebbleStore(t.TempDir(), wiring.pebbleOptions()...) + require.NoError(t, err) + registry, err := store.WriterRegistryFor(st) + require.NoError(t, err) + applier, err := encryption.NewApplier(registry) + require.NoError(t, err) + fsm := kv.NewKvFSMWithHLCAndTracker(st, kv.NewHLC(), nil, fsmOptionsForGroup(applier, nil, 1, wiring)...) + require.Equal(t, kv.MigrationNamespaceVersion, kv.MigrationNamespaceInstalledVersion(st, fsm)) + require.Zero(t, kv.MigrationNamespaceInstalledVersion(st, namespaceHiddenFSM{fsm})) + require.Zero(t, kv.MigrationNamespaceInstalledVersion(st, kv.NewTSOStateMachine(kv.NewHLC()))) + require.NoError(t, st.Close()) + } +} diff --git a/main_migration_ownership_readiness.go b/main_migration_ownership_readiness.go new file mode 100644 index 000000000..15b68002d --- /dev/null +++ b/main_migration_ownership_readiness.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + + "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func newInternalMigrationTransaction(proposer raftengine.Proposer, observer kv.ProposalObserver, st store.MVCCStore, groupID uint64) *kv.TransactionManager { + opts := []kv.TransactionOption{kv.WithProposalObserver(observer)} + if groupID != 0 { + opts = append(opts, kv.WithMigrationRequestAdmission(st, groupID)) + } + return kv.NewTransactionWithProposer(proposer, opts...) +} + +func migrationOwnershipRuntimeHooksInstalled(tx kv.Transactional, shards *kv.ShardStore, server *adapter.DistributionServer, groupID uint64) bool { + return kv.MigrationPartitionRequestAdmissionReady(tx) && shards.MigrationPartitionReadAdmissionReady(groupID) && server.MigrationPartitionRuntimeConfigured(groupID) +} + +func migrationOwnershipRuntimeReady(tx kv.Transactional, shards *kv.ShardStore, server *adapter.DistributionServer, groupID uint64, engine raftengine.Engine) bool { + if !migrationOwnershipRuntimeHooksInstalled(tx, shards, server, groupID) || splitMigrationLocalReadinessGate(context.Background()) != nil { + return false + } + reader, ok := engine.(raftengine.AppliedConfigurationReader) + if !ok { + return false + } + index, supported := reader.AppliedConfigurationIndex() + return supported && index != 0 +} + +// Old controllers may cache cap_migration_v2. Refuse their side-effecting RPCs +// when local flags are disabled, while leaving ordinary KV, authenticated 2PC +// resolution, same-group SplitRange, and general Admin traffic unchanged. +func migrationCoverageUnaryGate(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + if info.FullMethod == pb.Internal_CleanupMigration_FullMethodName { + if control, ok := req.(*pb.CleanupMigrationRequest); ok && control.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION { + if err := kv.ValidateNamespaceAdmissionRequest(control); err != nil { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "invalid namespace admission control")) + } + return handler(ctx, req) + } + } + if migrationCoverageMutationMethod(info.FullMethod) { + if err := splitMigrationLocalReadinessGate(ctx); err != nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, err.Error())) + } + } + return handler(ctx, req) +} + +func migrationCoverageMutationMethod(method string) bool { + switch method { + case pb.Distribution_StartSplitMigration_FullMethodName, + pb.Distribution_AbandonSplitJob_FullMethodName, + pb.Distribution_RetrySplitJob_FullMethodName, + pb.Internal_ApplyTargetStagedReadiness_FullMethodName, + pb.Internal_ImportRangeVersions_FullMethodName, + pb.Internal_PromoteStagedVersions_FullMethodName, + pb.Internal_CleanupMigration_FullMethodName, + pb.Internal_IssueMigrationTimestamp_FullMethodName: + return true + default: + return false + } +} diff --git a/main_migration_ownership_readiness_test.go b/main_migration_ownership_readiness_test.go new file mode 100644 index 000000000..ca2b9fb83 --- /dev/null +++ b/main_migration_ownership_readiness_test.go @@ -0,0 +1,263 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +type migrationProviderConfigEngine struct{ capabilityConfigEngine } + +func (migrationProviderConfigEngine) AppliedConfigurationIndex() (uint64, bool) { return 0, false } + +type migrationReadinessConfigurationOverride struct { + raftengine.Engine + index uint64 + supported bool +} + +func (e migrationReadinessConfigurationOverride) AppliedConfigurationIndex() (uint64, bool) { + return e.index, e.supported +} + +func enableMigrationRuntimeFlags(t *testing.T) { + t.Helper() + oldEnforced, oldNoAuth := *splitMigrationPeerAuthEnforced, *adminInsecureNoAuth + *splitMigrationPeerAuthEnforced, *adminInsecureNoAuth = true, false + t.Cleanup(func() { + *splitMigrationPeerAuthEnforced, *adminInsecureNoAuth = oldEnforced, oldNoAuth + }) + for _, env := range []string{adapter.MigrationImportOpcodeEnv, adapter.MigrationPromoteOpcodeEnv, adapter.MigrationCleanupOpcodeEnv} { + t.Setenv(env, "true") + } +} + +func TestMigrationOwnershipProviderWiringRequiresAppliedConfiguration(t *testing.T) { + enableMigrationRuntimeFlags(t) + st := store.NewMVCCStore() + engine := migrationProviderConfigEngine{} + tx := newInternalMigrationTransaction(engine, nil, st, 7) + t.Cleanup(tx.Close) + groups := map[uint64]*kv.ShardGroup{7: {Engine: engine, Store: st, Txn: tx}} + routes := distribution.NewEngine() + shards := kv.NewShardStore(routes, groups) + var cache kv.GRPCConnCache + t.Cleanup(func() { require.NoError(t, cache.Close()) }) + server := adapter.NewDistributionServer(routes, nil, + adapter.WithSplitMigrationGroupClientFactory(splitMigrationGroupClientFactory(splitMigrationGroupLeaderResolver(groups), &cache, t.Name())), + adapter.WithSplitMigrationVoterFactory(splitMigrationVoterFactory(groups, &cache, t.Name())), + migrationOwnershipRefreshOption(groups, &raftGroupRuntime{engine: engine}, &cache, t.Name(), true), + migrationPartitionRuntimeOption(shards, groups, &cache, t.Name(), true)) + require.True(t, kv.MigrationRequestAdmissionReady(tx)) + require.True(t, shards.MigrationReadAdmissionReady(7)) + require.True(t, server.MigrationOwnershipRefreshConfigured(7)) + require.True(t, migrationOwnershipRuntimeHooksInstalled(tx, shards, server, 7)) + require.False(t, migrationOwnershipRuntimeReady(tx, shards, server, 7, engine), "installed hooks alone do not prove an applied configuration") + internal := adapter.NewInternalWithEngine(tx, engine, nil, nil, adapter.WithInternalStore(st), + adapter.WithInternalMigrationOwnershipReady(func() bool { return migrationOwnershipRuntimeReady(tx, shards, server, 7, engine) })) + resp, err := internal.ProbeMigrationState(t.Context(), &pb.ProbeMigrationStateRequest{JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY}) + require.NoError(t, err) + require.False(t, resp.GetReady()) + legacy := kv.NewTransactionWithProposer(engine) + t.Cleanup(legacy.Close) + require.False(t, migrationOwnershipRuntimeHooksInstalled(legacy, shards, server, 7)) + require.False(t, migrationOwnershipRuntimeHooksInstalled(tx, nil, server, 7)) + require.False(t, migrationOwnershipRuntimeHooksInstalled(tx, shards, nil, 7)) + zero := newInternalMigrationTransaction(engine, nil, st, 0) + t.Cleanup(zero.Close) + require.False(t, kv.MigrationRequestAdmissionReady(zero)) +} + +func TestMigrationDisabledFlagsCloseLegacyCapabilityAndRunners(t *testing.T) { + old := *splitMigrationPeerAuthEnforced + *splitMigrationPeerAuthEnforced = true + t.Cleanup(func() { *splitMigrationPeerAuthEnforced = old }) + t.Setenv(adapter.MigrationImportOpcodeEnv, "false") + require.ErrorContains(t, splitMigrationLocalReadinessGate(t.Context()), adapter.MigrationImportOpcodeEnv) + server := adapter.NewDistributionServer(distribution.NewEngine(), nil, adapter.WithSplitJobRunnerReady(), + adapter.WithSplitJobRunnerReadinessGate(splitMigrationLocalReadinessGate)) + resp, err := server.GetSplitMigrationCapability(t.Context(), &pb.GetSplitMigrationCapabilityRequest{}) + require.NoError(t, err) + require.False(t, resp.GetMigrationCapable()) + require.NotContains(t, resp.GetCapabilities(), adapter.SplitMigrationCapabilityV2) + for _, start := range []func(context.Context, *errgroup.Group, *adapter.DistributionServer){startDistributionSplitJobRunner, startMigrationOwnershipRefresh} { + assertMigrationCoverageRunnerNotStarted(t, start, server) + } +} + +func assertMigrationCoverageRunnerNotStarted(t *testing.T, start func(context.Context, *errgroup.Group, *adapter.DistributionServer), server *adapter.DistributionServer) { + t.Helper() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + group := new(errgroup.Group) + start(ctx, group, server) + done := make(chan error, 1) + go func() { done <- group.Wait() }() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(time.Second): + cancel() + <-done + t.Fatal("disabled migration flags must not start a migration runner") + } +} + +func TestMigrationCoverageRPCGateLeavesOrdinaryAndResolutionPathsAvailable(t *testing.T) { + t.Setenv(adapter.MigrationImportOpcodeEnv, "false") + for _, method := range []string{ + pb.Distribution_StartSplitMigration_FullMethodName, pb.Distribution_AbandonSplitJob_FullMethodName, pb.Distribution_RetrySplitJob_FullMethodName, + pb.Internal_ApplyTargetStagedReadiness_FullMethodName, pb.Internal_ImportRangeVersions_FullMethodName, + pb.Internal_PromoteStagedVersions_FullMethodName, pb.Internal_CleanupMigration_FullMethodName, pb.Internal_IssueMigrationTimestamp_FullMethodName, + } { + _, err := migrationCoverageUnaryGate(t.Context(), nil, &grpc.UnaryServerInfo{FullMethod: method}, func(context.Context, any) (any, error) { + t.Fatal("disabled migration reached its handler") + return nil, nil + }) + require.Equal(t, codes.FailedPrecondition, status.Code(err), method) + } + for _, method := range []string{pb.Internal_Forward_FullMethodName, pb.Internal_ForwardAdminProposal_FullMethodName, + pb.Distribution_SplitRange_FullMethodName, pb.Distribution_ListRoutes_FullMethodName, pb.Distribution_AttestMigrationOwnership_FullMethodName, + pb.RawKV_RawPut_FullMethodName} { + called := false + _, err := migrationCoverageUnaryGate(t.Context(), nil, &grpc.UnaryServerInfo{FullMethod: method}, func(context.Context, any) (any, error) { + called = true + return nil, nil + }) + require.NoError(t, err) + require.True(t, called, method) + } +} + +func TestMigrationCoverageRPCGateUsesLocalFlags(t *testing.T) { + enableMigrationRuntimeFlags(t) + for _, method := range []string{ + pb.Distribution_StartSplitMigration_FullMethodName, pb.Distribution_AbandonSplitJob_FullMethodName, pb.Distribution_RetrySplitJob_FullMethodName, + pb.Internal_ApplyTargetStagedReadiness_FullMethodName, pb.Internal_ImportRangeVersions_FullMethodName, + pb.Internal_PromoteStagedVersions_FullMethodName, pb.Internal_CleanupMigration_FullMethodName, pb.Internal_IssueMigrationTimestamp_FullMethodName, + } { + called := false + _, err := migrationCoverageUnaryGate(t.Context(), nil, &grpc.UnaryServerInfo{FullMethod: method}, func(context.Context, any) (any, error) { + called = true + return nil, nil + }) + require.NoError(t, err, method) + require.True(t, called, method) + } +} + +func TestMigrationOwnershipRuntimeRequiresInstalledHooksFlagsAndAppliedConfiguration(t *testing.T) { + enableMigrationRuntimeFlags(t) + st := store.NewMVCCStore() + node := newOwnershipRefreshRaftNode(t, "readiness", 7, st, t.Name()) + engine := node.group.Engine + tx := newInternalMigrationTransaction(engine, nil, st, 7) + t.Cleanup(tx.Close) + node.group.Txn = tx + groups := map[uint64]*kv.ShardGroup{7: node.group} + routes := distribution.NewEngine() + shards := kv.NewShardStore(routes, groups) + var cache kv.GRPCConnCache + t.Cleanup(func() { require.NoError(t, cache.Close()) }) + server := adapter.NewDistributionServer(routes, nil, + adapter.WithSplitMigrationGroupClientFactory(splitMigrationGroupClientFactory(splitMigrationGroupLeaderResolver(groups), &cache, t.Name())), + adapter.WithSplitMigrationVoterFactory(splitMigrationVoterFactory(groups, &cache, t.Name())), + migrationOwnershipRefreshOption(groups, &raftGroupRuntime{engine: engine}, &cache, t.Name(), true), + migrationPartitionRuntimeOption(shards, groups, &cache, t.Name(), true)) + reader, ok := engine.(raftengine.AppliedConfigurationReader) + require.True(t, ok) + index, supported := reader.AppliedConfigurationIndex() + require.True(t, supported) + require.NotZero(t, index) + require.True(t, migrationOwnershipRuntimeReady(tx, shards, server, 7, engine)) + internal := adapter.NewInternalWithEngine(tx, engine, nil, nil, adapter.WithInternalStore(st), + adapter.WithInternalMigrationOwnershipReady(func() bool { return migrationOwnershipRuntimeReady(tx, shards, server, 7, engine) })) + probe, err := internal.ProbeMigrationState(t.Context(), &pb.ProbeMigrationStateRequest{JobId: 1, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: index}) + require.NoError(t, err) + require.True(t, probe.GetReady()) + require.Equal(t, index, probe.GetAppliedConfigurationIndex()) + require.False(t, migrationOwnershipRuntimeReady(tx, nil, server, 7, engine)) + require.False(t, migrationOwnershipRuntimeReady(tx, shards, nil, 7, engine)) + require.False(t, migrationOwnershipRuntimeReady(tx, shards, server, 0, engine)) + for _, missing := range []raftengine.Engine{nil, capabilityConfigEngine{}, migrationProviderConfigEngine{}, + migrationReadinessConfigurationOverride{Engine: engine, supported: true}, + migrationReadinessConfigurationOverride{Engine: engine, index: index}} { + require.False(t, migrationOwnershipRuntimeReady(tx, shards, server, 7, missing)) + } + legacy := kv.NewTransactionWithProposer(engine) + t.Cleanup(legacy.Close) + require.False(t, migrationOwnershipRuntimeReady(legacy, shards, server, 7, engine)) + assertMigrationRuntimeFlagsRequired(t, tx, shards, server, engine) +} + +func assertMigrationRuntimeFlagsRequired(t *testing.T, tx kv.Transactional, shards *kv.ShardStore, server *adapter.DistributionServer, engine raftengine.Engine) { + t.Helper() + for _, env := range []string{adapter.MigrationImportOpcodeEnv, adapter.MigrationPromoteOpcodeEnv, adapter.MigrationCleanupOpcodeEnv} { + t.Run(env, func(t *testing.T) { + t.Setenv(env, "false") + require.False(t, migrationOwnershipRuntimeReady(tx, shards, server, 7, engine)) + _, err := migrationCoverageUnaryGate(t.Context(), nil, &grpc.UnaryServerInfo{FullMethod: pb.Internal_ImportRangeVersions_FullMethodName}, + func(context.Context, any) (any, error) { + t.Fatal("disabled flags reached mutation handler") + return nil, nil + }) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + }) + } + *splitMigrationPeerAuthEnforced = false + require.False(t, migrationOwnershipRuntimeReady(tx, shards, server, 7, engine)) + *splitMigrationPeerAuthEnforced = true +} + +func TestMigrationCoverageEnabledFlagsRetainRPCAuthenticationAndNamespaceGuards(t *testing.T) { + enableMigrationRuntimeFlags(t) + st := store.NewMVCCStore() + node := newOwnershipRefreshRaftNode(t, "namespace-guard", 7, st, t.Name()) + fsm := node.fsm + engine := &namespaceGateEngine{Engine: node.group.Engine, store: st, fsm: fsm} + internal := adapter.NewInternalWithEngine(nil, engine, nil, nil, adapter.WithInternalStore(st), + adapter.WithInternalMigrationProposer(engine), adapter.WithInternalMigrationImportGate(func(context.Context) error { return nil }), + adapter.WithMigrationNamespaceAdmissionReady(func() bool { + return kv.MigrationNamespaceInstalledVersion(st, fsm) == kv.MigrationNamespaceVersion + })) + request := &pb.ImportRangeVersionsRequest{JobId: 7, BracketId: 1, BatchSeq: 1, + Versions: []*pb.MVCCVersion{{Key: []byte("k"), Value: []byte("value"), CommitTs: 30}}} + auth, _ := adapter.SplitMigrationTokenAuth(t.Name(), false, true) + info := &grpc.UnaryServerInfo{FullMethod: pb.Internal_ImportRangeVersions_FullMethodName} + handler := func(ctx context.Context, req any) (any, error) { + return migrationCoverageUnaryGate(ctx, req, info, func(ctx context.Context, _ any) (any, error) { + return internal.ImportRangeVersions(ctx, request) + }) + } + _, err := auth(t.Context(), request, info, handler) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + tlsContext := peer.NewContext(t.Context(), &peer.Peer{AuthInfo: credentials.TLSInfo{}}) + _, err = auth(tlsContext, request, info, handler) + require.Equal(t, codes.Unauthenticated, status.Code(err), "TLS alone cannot authorize migration") + wrongToken := metadata.NewIncomingContext(tlsContext, metadata.Pairs("authorization", "Bearer unrelated")) + _, err = auth(wrongToken, request, info, handler) + require.Equal(t, codes.Unauthenticated, status.Code(err), "unrelated credentials cannot authorize migration") + ctx := metadata.NewIncomingContext(tlsContext, metadata.Pairs("authorization", "Bearer "+t.Name())) + _, err = auth(ctx, request, info, handler) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "namespace admission protocol") + require.Zero(t, engine.proposals, "enabled flags and valid credentials cannot bypass durable namespace activation") +} diff --git a/main_migration_ownership_refresh.go b/main_migration_ownership_refresh.go new file mode 100644 index 000000000..7c20eb490 --- /dev/null +++ b/main_migration_ownership_refresh.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "strings" + + "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "golang.org/x/sync/errgroup" +) + +func migrationOwnershipRootLeaderResolver(runtime *raftGroupRuntime) func() (string, error) { + return func() (string, error) { + if runtime == nil { + return "", errors.WithStack(kv.ErrLeaderNotFound) + } + engine := runtime.snapshotEngine() + if engine == nil { + return "", errors.WithStack(kv.ErrLeaderNotFound) + } + address := strings.TrimSpace(engine.Leader().Address) + if address == "" { + return "", errors.WithStack(kv.ErrLeaderNotFound) + } + return address, nil + } +} + +func migrationOwnershipRootClientFactory(resolve func() (string, error), cache *kv.GRPCConnCache, token string) adapter.MigrationOwnershipAttestorFactory { + return func(context.Context) (adapter.MigrationOwnershipAttestor, error) { + // This endpoint is never downgraded to anonymous catalog reads, even + // during peer send-first rollout or on an unsupported old root binary. + if token == "" || resolve == nil || cache == nil { + return nil, errors.New("ownership refresh requires authenticated root client configuration") + } + address, err := resolve() + if err != nil { + return nil, errors.WithStack(err) + } + conn, err := cache.ConnFor(address) + if err != nil { + return nil, errors.WithStack(err) + } + return pb.NewDistributionClient(splitMigrationAuthConn{ClientConnInterface: conn, token: token}), nil + } +} + +func migrationOwnershipRefreshOption(groups map[uint64]*kv.ShardGroup, root *raftGroupRuntime, cache *kv.GRPCConnCache, token string, enforced bool) adapter.DistributionServerOption { + if token == "" || !enforced { + // Keep durable ownership visible to metadata preflight even when the + // authenticated mutation protocol is disabled. Do not install a root + // client or advertise a configured refresh runtime in this case. + return adapter.WithMigrationOwnershipRefresh(groups, nil) + } + return adapter.WithMigrationOwnershipRefresh(groups, migrationOwnershipRootClientFactory(migrationOwnershipRootLeaderResolver(root), cache, token)) +} + +func startMigrationOwnershipRefresh(ctx context.Context, eg *errgroup.Group, server *adapter.DistributionServer) { + if eg == nil || server == nil || splitMigrationLocalReadinessGate(ctx) != nil { + return + } + eg.Go(func() error { return server.RunMigrationOwnershipRefresh(ctx) }) +} diff --git a/main_migration_ownership_refresh_test.go b/main_migration_ownership_refresh_test.go new file mode 100644 index 000000000..d069746f1 --- /dev/null +++ b/main_migration_ownership_refresh_test.go @@ -0,0 +1,169 @@ +package main + +import ( + "context" + "crypto/sha256" + "net" + "testing" + "time" + + "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/distribution" + internalutil "github.com/bootjp/elastickv/internal" + "github.com/bootjp/elastickv/internal/raftengine" + etcdraft "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +type ownershipRefreshRaftNode struct { + group *kv.ShardGroup + fsm kv.FSM + server *grpc.Server + listener net.Listener +} + +func newOwnershipRefreshRaftNode(t *testing.T, id string, groupID uint64, st store.MVCCStore, token string) ownershipRefreshRaftNode { + t.Helper() + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + factory := etcdraft.NewFactory(etcdraft.FactoryConfig{TickInterval: 20 * time.Millisecond, HeartbeatTick: 1, ElectionTick: 10}) + fsm := kv.NewKvFSMWithHLC(st, kv.NewHLC(), kv.WithRouteHistory(nil, groupID)) + result, err := factory.Create(raftengine.FactoryConfig{LocalID: id, LocalAddress: listener.Addr().String(), + DataDir: t.TempDir(), Peers: []raftengine.Server{{ID: id, Address: listener.Addr().String(), Suffrage: "voter"}}, Bootstrap: true, StateMachine: fsm}) + require.NoError(t, err) + auth, _ := adapter.SplitMigrationTokenAuth(token, false, true) + options := raftGRPCServerOptions(adminGRPCInterceptors{unary: []grpc.UnaryServerInterceptor{auth}}) + server := grpc.NewServer(options...) + if result.RegisterTransport != nil { + result.RegisterTransport(server) + } + t.Cleanup(func() { + server.Stop() + _ = listener.Close() + require.NoError(t, result.Engine.Close()) + if result.Close != nil { + require.NoError(t, result.Close()) + } + }) + return ownershipRefreshRaftNode{group: &kv.ShardGroup{Store: st, Engine: result.Engine}, fsm: fsm, server: server, listener: listener} +} + +func ownershipRefreshRootCatalog(t *testing.T) (store.MVCCStore, *distribution.CatalogStore, *pb.MigrationOwnershipProof) { + t.Helper() + st := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(st, distribution.WithCatalogRouteDescriptorV2Writes(true)) + routes := []distribution.RouteDescriptor{ + {RouteID: 1, End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), GroupID: 7, State: distribution.RouteStateActive, MinWriteTSExclusive: 100}, + } + _, err := catalog.Save(t.Context(), 0, routes) + require.NoError(t, err) + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(routes[1], true) + require.NoError(t, err) + digest := sha256.Sum256(encoded) + proof := &pb.MigrationOwnershipProof{Generation: "native-generation-for-refresh", LocalGroupId: 7, OwnerGroupId: 7, + CatalogVersion: 1, RouteStart: []byte("m"), RouteId: 2, MinWriteTsExclusive: 100, WriteFloorKnown: true, + Role: pb.MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER, DescriptorHash: digest[:]} + // Bootstrap the root store at a later C with the migrated descriptor intact. + // No SplitJob/history is created: the terminal job may already have been GC'd. + routes[0].MinWriteTSExclusive = 80 + _, err = catalog.Save(t.Context(), 1, routes) + require.NoError(t, err) + return st, catalog, proof +} + +func TestOwnershipRefreshDifferentRaftLeadersRespectExplicitClosedGate(t *testing.T) { + cert, key := writeSelfSignedCert(t) + require.NoError(t, internalutil.ConfigureGRPCTLS(cert, cert, key)) + t.Cleanup(func() { require.NoError(t, internalutil.ConfigureGRPCTLS("", "", "")) }) + token := t.Name() + rootStore, catalog, prior := ownershipRefreshRootCatalog(t) + root := newOwnershipRefreshRaftNode(t, "root", 1, rootStore, token) + data := newOwnershipRefreshRaftNode(t, "data", 7, store.NewMVCCStore(), token) + rootRoutes := distribution.NewEngine() + rootRoutes.UpdateRoute(nil, nil, 1) + rootGroups := map[uint64]*kv.ShardGroup{1: root.group} + rootShards := kv.NewShardStore(rootRoutes, rootGroups) + coordinate := kv.NewShardedCoordinator(rootRoutes, rootGroups, 1, kv.NewHLC(), rootShards) + pb.RegisterDistributionServer(root.server, adapter.NewDistributionServer(rootRoutes, catalog, adapter.WithDistributionCoordinator(coordinate))) + groups := map[uint64]*kv.ShardGroup{7: data.group} + dataRoutes := distribution.NewEngine() + dataRoutes.UpdateRoute(nil, nil, 7) + dataShards := kv.NewShardStore(dataRoutes, groups) + dataTx := newInternalMigrationTransaction(data.group.Engine, nil, data.group.Store, 7) + t.Cleanup(dataTx.Close) + data.group.Txn = dataTx + var cache kv.GRPCConnCache + t.Cleanup(func() { require.NoError(t, cache.Close()) }) + groupFactory := splitMigrationGroupClientFactory(splitMigrationGroupLeaderResolver(groups), &cache, token) + rootFactory := migrationOwnershipRootClientFactory(migrationOwnershipRootLeaderResolver(&raftGroupRuntime{engine: root.group.Engine}), &cache, token) + mirror := distribution.NewEngine() + require.NoError(t, mirror.ApplySnapshot(distribution.CatalogSnapshot{Version: 2, Routes: []distribution.RouteDescriptor{{RouteID: 99, GroupID: 99}}})) + closedGate := func(context.Context) error { + return status.Error(codes.FailedPrecondition, "synthetic migration gate closed") + } + consumer := adapter.NewDistributionServer(mirror, nil, + adapter.WithSplitJobRunnerReadinessGate(closedGate), + adapter.WithSplitMigrationGroupClientFactory(groupFactory), + adapter.WithSplitMigrationVoterFactory(splitMigrationVoterFactory(groups, &cache, token)), + adapter.WithMigrationOwnershipRefresh(groups, rootFactory), + migrationPartitionRuntimeOption(dataShards, groups, &cache, token, true)) + require.True(t, migrationOwnershipRuntimeHooksInstalled(dataTx, dataShards, consumer, 7)) + pb.RegisterInternalServer(data.server, adapter.NewInternalWithEngine(nil, data.group.Engine, nil, nil, + adapter.WithInternalStore(data.group.Store), adapter.WithInternalMigrationProposer(data.group.Engine), + adapter.WithInternalMigrationOwnershipReady(func() bool { return closedGate(t.Context()) == nil }))) + go func() { _ = root.server.Serve(root.listener) }() + go func() { _ = data.server.Serve(data.listener) }() + require.Eventually(t, func() bool { + return root.group.Engine.State() == raftengine.StateLeader && data.group.Engine.State() == raftengine.StateLeader + }, 5*time.Second, 10*time.Millisecond) + require.NotEqual(t, root.group.Engine.Leader().ID, data.group.Engine.Leader().ID) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + dataClient, err := groupFactory(ctx, 7) + require.NoError(t, err) + capability, err := dataClient.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{JobId: 77, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY}) + require.NoError(t, err) + require.False(t, capability.GetReady()) + index, err := kv.MigrationConfigurationIndex(ctx, data.group.Store) + require.NoError(t, err) + _, err = dataClient.CleanupMigration(ctx, &pb.CleanupMigrationRequest{JobId: 77, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR, Ownership: prior, + RequireConfigurationIndex: true, ExpectedConfigurationIndex: index}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + // A caller's unrelated credential cannot override the configured peer token. + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("authorization", "Bearer unrelated")) + require.ErrorContains(t, consumer.RunMigrationOwnershipRefreshOnce(ctx), "synthetic migration gate closed") + rootClient, err := rootFactory(ctx) + require.NoError(t, err) + attestation, err := rootClient.AttestMigrationOwnership(ctx, &pb.AttestMigrationOwnershipRequest{JobId: 77, Prior: prior}) + require.NoError(t, err) + require.EqualValues(t, 2, attestation.GetOwnershipProtocol()) + require.EqualValues(t, 2, attestation.GetOwnership().GetCatalogVersion()) + require.True(t, attestation.GetOwnership().GetWriteFloorKnown()) + writer, ok := data.group.Store.(store.MigrationOwnershipStore) + require.True(t, ok) + view, err := writer.MigrationServingState(ctx) + require.NoError(t, err) + require.Empty(t, view.Ownership, "explicit closed gate must prevent all data-side ownership writes") + assertOwnershipRootRejectsAnonymous(t, ctx, &cache, root.listener.Addr().String(), prior) +} + +func assertOwnershipRootRejectsAnonymous(t *testing.T, ctx context.Context, cache *kv.GRPCConnCache, address string, prior *pb.MigrationOwnershipProof) { + t.Helper() + conn, err := cache.ConnFor(address) + require.NoError(t, err) + ctx = migrationCredentialContext(ctx) + _, err = pb.NewDistributionClient(conn).AttestMigrationOwnership(ctx, &pb.AttestMigrationOwnershipRequest{JobId: 77, Prior: prior}) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + _, err = migrationOwnershipRootClientFactory(func() (string, error) { return address, nil }, cache, "")(ctx) + require.Error(t, err, "no anonymous fallback even with a reachable root") +} diff --git a/main_migration_partition_runtime.go b/main_migration_partition_runtime.go new file mode 100644 index 000000000..ac313ee58 --- /dev/null +++ b/main_migration_partition_runtime.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + + "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/kv" +) + +func migrationInternalProtocolOptions(tx kv.Transactional, shards *kv.ShardStore, server *adapter.DistributionServer, runtime *raftGroupRuntime) []adapter.InternalOption { + installed := func() bool { return migrationOwnershipRuntimeHooksInstalled(tx, shards, server, runtime.spec.id) } + return []adapter.InternalOption{ + adapter.WithInternalMigrationTransferSizeGuardReady(installed), + adapter.WithInternalMigrationPartitionProtocolReady(installed), + adapter.WithMigrationNamespaceAdmissionReady(func() bool { + return installed() && kv.MigrationNamespaceInstalledVersion(runtime.store, runtime.stateMachine) == kv.MigrationNamespaceVersion + }), + } +} + +func migrationPartitionRuntimeOption(reads *kv.ShardStore, groups map[uint64]*kv.ShardGroup, cache *kv.GRPCConnCache, + token string, enforced bool, +) adapter.DistributionServerOption { + if token == "" || !enforced || reads == nil { + return nil + } + return adapter.WithMigrationPartitionRuntime(reads, migrationApplyMemberClients(splitMigrationApplyMemberSnapshotFactory(groups, cache, token))) +} + +func migrationApplyMemberClients(factory adapter.SplitMigrationApplyMemberSnapshotFactory) adapter.SplitMigrationVoterFactory { + return func(ctx context.Context, groupID uint64) ([]adapter.SplitMigrationVoter, error) { + snapshot, err := factory(ctx, groupID) + return snapshot.Members, err + } +} diff --git a/main_migration_protocol_wiring_test.go b/main_migration_protocol_wiring_test.go new file mode 100644 index 000000000..832eb2666 --- /dev/null +++ b/main_migration_protocol_wiring_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/internal/raftadmin" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestMigrationMembershipWiringKeepsEnvelopeProposerAndTSOIsolation(t *testing.T) { + t.Parallel() + proposer := &capabilityConfigEngine{} + runtime := &raftGroupRuntime{store: store.NewMVCCStore(), engine: capabilityConfigEngine{}} + runtime.spec.id = 7 + gate, ok := migrationMembershipInterceptor(runtime, nil, proposer, t.Name(), nil).(*migrationNamespaceMembershipGate) + require.True(t, ok) + require.Same(t, proposer, gate.proposer, "namespace prejoin must use the envelope-aware proposer passed by main") + _, ok = gate.next.(*migrationMembershipGate) + require.True(t, ok) + runtime.spec.id = 0 + _, namespace := migrationMembershipInterceptor(runtime, nil, proposer, t.Name(), nil).(*migrationNamespaceMembershipGate) + require.False(t, namespace, "TSO must not require a KV namespace or KV capability") + _, prepare := migrationMembershipInterceptor(runtime, nil, proposer, t.Name(), nil).(*prepareEvidenceMembershipGate) + require.True(t, prepare, "existing TSO membership safety hooks must remain installed") + var next raftadmin.MembershipChangeInterceptor = migrationMembershipNext{} + require.Equal(t, next, migrationMembershipInterceptor(nil, nil, nil, "", next)) +} + +func TestMigrationCoverageGateAllowsOnlyTypedNamespaceBootstrap(t *testing.T) { + t.Parallel() + request, err := kv.NewNamespaceAdmissionRequest(1, kv.NamespaceAdmissionCommand{Version: kv.MigrationNamespaceVersion, + ConfigurationIndex: 9, ApplyMembers: []uint64{7}, + Members: []kv.NamespaceMemberCapability{{MemberID: 7, Version: kv.MigrationNamespaceVersion, ConfigurationIndex: 9}}}) + require.NoError(t, err) + info := &grpc.UnaryServerInfo{FullMethod: pb.Internal_CleanupMigration_FullMethodName} + called := false + _, err = migrationCoverageUnaryGate(t.Context(), request, info, func(context.Context, any) (any, error) { called = true; return nil, nil }) + require.NoError(t, err) + require.True(t, called) + request.NamespaceAdmission = []byte("invalid") + _, err = migrationCoverageUnaryGate(t.Context(), request, info, func(context.Context, any) (any, error) { + t.Fatal("malformed bootstrap reached the handler") + return nil, nil + }) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} diff --git a/main_proposer_for_group_test.go b/main_proposer_for_group_test.go index 017344040..f40f01ee0 100644 --- a/main_proposer_for_group_test.go +++ b/main_proposer_for_group_test.go @@ -3,8 +3,10 @@ package main import ( "testing" + "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" + "github.com/cockroachdb/errors" ) // stubEngineForProposerTest satisfies raftengine.Engine at the @@ -92,3 +94,37 @@ func TestProposerForGroup_FallsBackToEngineWhenShardGroupNil(t *testing.T) { t.Errorf("proposerForGroup did not fall back to rt.engine for nil shardGroup entry; got %T, want raw engine", got) } } + +func TestSplitPromotionTargetLeaderResolverUsesTargetGroup(t *testing.T) { + t.Parallel() + + source := capabilityConfigEngine{leader: raftengine.LeaderInfo{Address: "source:50051"}} + target := capabilityConfigEngine{leader: raftengine.LeaderInfo{Address: "target:50051"}} + resolve := splitPromotionTargetLeaderResolver(map[uint64]*kv.ShardGroup{ + 1: {Engine: source}, + 2: {Engine: target}, + }) + + addr, err := resolve(distribution.SplitJob{ + SplitKey: []byte("m"), + TargetGroupID: 2, + }) + if err != nil { + t.Fatalf("resolve returned error: %v", err) + } + if addr != "target:50051" { + t.Fatalf("resolve returned %q, want target leader address", addr) + } +} + +func TestSplitPromotionTargetLeaderResolverRejectsMissingLeader(t *testing.T) { + t.Parallel() + + resolve := splitPromotionTargetLeaderResolver(map[uint64]*kv.ShardGroup{ + 2: {Engine: capabilityConfigEngine{}}, + }) + _, err := resolve(distribution.SplitJob{TargetGroupID: 2}) + if !errors.Is(err, kv.ErrLeaderNotFound) { + t.Fatalf("resolve error = %v, want ErrLeaderNotFound", err) + } +} diff --git a/main_split_migration_auth.go b/main_split_migration_auth.go new file mode 100644 index 000000000..34e2427b0 --- /dev/null +++ b/main_split_migration_auth.go @@ -0,0 +1,44 @@ +package main + +import ( + "context" + + internalutil "github.com/bootjp/elastickv/internal" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +// Keep migration credentials on runner clients, not on the shared peer cache. +type splitMigrationAuthConn struct { + grpc.ClientConnInterface + token string +} + +func (c splitMigrationAuthConn) Invoke(ctx context.Context, method string, args, reply any, opts ...grpc.CallOption) error { + ctx = migrationCredentialContext(ctx) + opts = append(opts, grpc.PerRPCCredentials(internalutil.GRPCBearerToken(c.token))) + return errors.WithStack(c.ClientConnInterface.Invoke(ctx, method, args, reply, opts...)) +} + +func (c splitMigrationAuthConn) NewStream(ctx context.Context, desc *grpc.StreamDesc, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) { + ctx = migrationCredentialContext(ctx) + opts = append(opts, grpc.PerRPCCredentials(internalutil.GRPCBearerToken(c.token))) + stream, err := c.ClientConnInterface.NewStream(ctx, desc, method, opts...) + return stream, errors.WithStack(err) +} + +func migrationCredentialContext(ctx context.Context) context.Context { + md, _ := metadata.FromOutgoingContext(ctx) + md = md.Copy() + md.Delete("authorization") + return metadata.NewOutgoingContext(ctx, md) +} + +func newSplitMigrationInternalClient(conn grpc.ClientConnInterface, token string) pb.InternalClient { + if token != "" { + conn = splitMigrationAuthConn{ClientConnInterface: conn, token: token} + } + return pb.NewInternalClient(conn) +} diff --git a/main_split_migration_auth_test.go b/main_split_migration_auth_test.go new file mode 100644 index 000000000..9fae3ad3d --- /dev/null +++ b/main_split_migration_auth_test.go @@ -0,0 +1,318 @@ +package main + +import ( + "context" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/distribution" + internalutil "github.com/bootjp/elastickv/internal" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" +) + +type splitRangeAuthRPCServer struct { + pb.UnimplementedDistributionServer +} + +func (splitRangeAuthRPCServer) SplitRange(context.Context, *pb.SplitRangeRequest) (*pb.SplitRangeResponse, error) { + return &pb.SplitRangeResponse{CatalogVersion: 2}, nil +} + +func (splitRangeAuthRPCServer) ListRoutes(context.Context, *pb.ListRoutesRequest) (*pb.ListRoutesResponse, error) { + return &pb.ListRoutesResponse{CatalogVersion: 1}, nil +} + +func TestSplitRangeProductionAuthRequiresDedicatedTokenDuringSendFirst(t *testing.T) { + oldEnforced := *splitMigrationPeerAuthEnforced + *splitMigrationPeerAuthEnforced = false + t.Cleanup(func() { *splitMigrationPeerAuthEnforced = oldEnforced }) + for _, env := range []string{adapter.MigrationImportOpcodeEnv, adapter.MigrationPromoteOpcodeEnv, adapter.MigrationCleanupOpcodeEnv} { + t.Setenv(env, "false") + } + cert, key := writeSelfSignedCert(t) + require.NoError(t, internalutil.ConfigureGRPCTLS(cert, cert, key)) + t.Cleanup(func() { require.NoError(t, internalutil.ConfigureGRPCTLS("", "", "")) }) + dir := t.TempDir() + for name, token := range map[string]string{"admin": "read-only-admin", "peer": "blob-peer", "migration": "migration-secret"} { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(token), 0o600)) + } + adminServer, auth, err := configureAdminService(filepath.Join(dir, "admin"), false, filepath.Join(dir, "peer"), filepath.Join(dir, "migration"), + false, adapter.NodeIdentity{}, nil, nil) + require.NoError(t, err) + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + server := grpc.NewServer(raftGRPCServerOptions(auth)...) + pb.RegisterDistributionServer(server, splitRangeAuthRPCServer{}) + pb.RegisterAdminServer(server, adminServer) + serveResult := make(chan error, 1) + go func() { serveResult <- server.Serve(listener) }() + t.Cleanup(func() { + server.Stop() + _ = listener.Close() + require.NoError(t, <-serveResult) + }) + conn, err := grpc.NewClient(listener.Addr().String(), internalutil.GRPCDialOptions()...) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + client := pb.NewDistributionClient(conn) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + request := &pb.SplitRangeRequest{RouteId: 1, SplitKey: []byte("m"), ExpectedCatalogVersion: 1} + for _, credential := range []string{"", "Bearer read-only-admin", "Bearer blob-peer", "Bearer wrong"} { + callCtx := metadata.NewOutgoingContext(ctx, metadata.Pairs("authorization", credential)) + _, err := client.SplitRange(callCtx, request) + require.Equal(t, codes.Unauthenticated, status.Code(err), credential) + _, err = client.ListRoutes(callCtx, &pb.ListRoutesRequest{}) + require.NoError(t, err, "ordinary catalog reads remain public") + } + migrationCtx := metadata.NewOutgoingContext(ctx, metadata.Pairs("authorization", "Bearer migration-secret")) + response, err := client.SplitRange(migrationCtx, request) + require.NoError(t, err, "operator split is authenticated independently of migration opcode flags") + require.EqualValues(t, 2, response.GetCatalogVersion()) + adminCtx := metadata.NewOutgoingContext(ctx, metadata.Pairs("authorization", "Bearer read-only-admin")) + _, err = pb.NewAdminClient(conn).GetNodeVersion(adminCtx, &pb.GetNodeVersionRequest{}) + require.NoError(t, err) + _, err = pb.NewAdminClient(conn).GetNodeVersion(migrationCtx, &pb.GetNodeVersionRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err), "migration token must not gain unrelated Admin authority") +} + +func TestSplitMigrationFactoriesPropagateCredentials(t *testing.T) { + oldEnforced := *splitMigrationPeerAuthEnforced + *splitMigrationPeerAuthEnforced = true + t.Cleanup(func() { *splitMigrationPeerAuthEnforced = oldEnforced }) + cert, key := writeSelfSignedCert(t) + require.NoError(t, internalutil.ConfigureGRPCTLS(cert, cert, key)) + t.Cleanup(func() { require.NoError(t, internalutil.ConfigureGRPCTLS("", "", "")) }) + internalutil.ConfigureGRPCForwardToken("migration-secret") + t.Cleanup(func() { internalutil.ConfigureGRPCForwardToken("") }) + var lc net.ListenConfig + lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + adminPath := filepath.Join(t.TempDir(), "admin-token") + migrationPath := filepath.Join(t.TempDir(), "migration-token") + require.NoError(t, os.WriteFile(adminPath, []byte("read-only-admin"), 0o600)) + require.NoError(t, os.WriteFile(migrationPath, []byte("migration-secret"), 0o600)) + adminServer, auth, err := configureAdminService(adminPath, false, "", migrationPath, false, adapter.NodeIdentity{}, nil, nil) + require.NoError(t, err) + serverOpts := append(raftGRPCServerOptions(auth), grpc.ChainUnaryInterceptor( + func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + if info.FullMethod == pb.Admin_GetNodeVersion_FullMethodName { + return handler(ctx, req) + } + md, _ := metadata.FromIncomingContext(ctx) + require.Equal(t, []string{"Bearer migration-secret"}, md.Get("authorization")) + require.Equal(t, []string{"preserved"}, md.Get("trace-id")) + return &emptypb.Empty{}, nil + })) + server := grpc.NewServer(serverOpts...) + pb.RegisterInternalServer(server, migrationAuthExportServer{}) + pb.RegisterAdminServer(server, adminServer) + serveErr := make(chan error, 1) + go func() { serveErr <- server.Serve(lis) }() + t.Cleanup(func() { + server.Stop() + _ = lis.Close() + require.NoError(t, <-serveErr) + }) + var cache kv.GRPCConnCache + t.Cleanup(func() { require.NoError(t, cache.Close()) }) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + // A stale caller credential must not override the configured runner token. + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("authorization", "Bearer stale", "trace-id", "preserved")) + job := distribution.SplitJob{TargetGroupID: 2} + resolve := func(uint64) (string, error) { return lis.Addr().String(), nil } + source, target, err := splitMigrationClientFactory(resolve, &cache, "migration-secret")(ctx, job, 1) + require.NoError(t, err) + voters, err := splitMigrationVoterFactory(map[uint64]*kv.ShardGroup{ + 2: {Engine: capabilityConfigEngine{cfg: raftengine.Configuration{Servers: []raftengine.Server{ + {ID: "voter", Address: lis.Addr().String(), Suffrage: raftMemberSuffrageVoter}, + }}}}, + }, &cache, "migration-secret")(ctx, 2) + require.NoError(t, err) + require.Len(t, voters, 1) + sourceOnly, err := splitMigrationGroupClientFactory(func(groupID uint64) (string, error) { + if groupID != 1 { + return "", status.Error(codes.Unavailable, "target leader unavailable") + } + return lis.Addr().String(), nil + }, &cache, "migration-secret")(ctx, 1) + require.NoError(t, err, "source recovery must not resolve the unavailable target") + for _, client := range []adapter.SplitMigrationClient{source, target, voters[0].Client, sourceOnly} { + _, err = client.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + _, err = client.ImportRangeVersions(ctx, &pb.ImportRangeVersionsRequest{}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + _, err = client.PromoteStagedVersions(ctx, &pb.PromoteStagedVersionsRequest{}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + _, err = client.CleanupMigration(ctx, &pb.CleanupMigrationRequest{}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + _, err = client.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{}) + require.NoError(t, err) + _, err = client.ProbeMigrationLocks(ctx, &pb.ProbeMigrationLocksRequest{}) + require.NoError(t, err) + _, err = client.IssueMigrationTimestamp(ctx, &pb.IssueMigrationTimestampRequest{}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + stream, streamErr := client.ExportRangeVersions(ctx, &pb.ExportRangeVersionsRequest{}) + require.NoError(t, streamErr) + page, recvErr := stream.Recv() + require.NoError(t, recvErr) + require.True(t, page.GetDone()) + } + promoter, err := splitPromotionClientFactory(func(distribution.SplitJob) (string, error) { + return lis.Addr().String(), nil + }, &cache, "migration-secret")(ctx, job) + require.NoError(t, err) + _, err = promoter.PromoteStagedVersions(ctx, &pb.PromoteStagedVersionsRequest{}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + md, _ := metadata.FromOutgoingContext(ctx) + require.Equal(t, []string{"Bearer stale"}, md.Get("authorization")) + conn, err := cache.ConnFor(lis.Addr().String()) + require.NoError(t, err) + require.NoError(t, conn.Invoke(ctx, pb.Internal_Forward_FullMethodName, &pb.ForwardRequest{}, &pb.ForwardResponse{}), "normal peer forwarding uses the configured TLS credential") + clientTLS, err := internalutil.GRPCClientTLSCredentials(cert) + require.NoError(t, err) + untrustedConn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(clientTLS)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, untrustedConn.Close()) }) + err = untrustedConn.Invoke(context.Background(), pb.Internal_Forward_FullMethodName, &pb.ForwardRequest{}, &pb.ForwardResponse{}) + require.Equal(t, codes.Unauthenticated, status.Code(err), "untrusted clients must not forge catalog or transaction metadata through Forward") + _, err = pb.NewInternalClient(conn).CleanupMigration(ctx, &pb.CleanupMigrationRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err), "runner token must not leak into cached connections") + adminCtx := metadata.NewOutgoingContext(ctx, metadata.Pairs("authorization", "Bearer read-only-admin")) + _, err = pb.NewInternalClient(conn).ApplyTargetStagedReadiness(adminCtx, &pb.TargetStagedReadinessRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err), "read-only admin token must not authorize migration fences") + _, err = pb.NewAdminClient(conn).GetNodeVersion(adminCtx, &pb.GetNodeVersionRequest{}) + require.NoError(t, err, "read-only admin operations retain their own credential") + migrationCtx := metadata.NewOutgoingContext(ctx, metadata.Pairs("authorization", "Bearer migration-secret")) + _, err = pb.NewAdminClient(conn).GetNodeVersion(migrationCtx, &pb.GetNodeVersionRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err), "migration credential must not gain unrelated Admin access") +} + +func TestSplitMigrationOpcodesRequireExplicitAuthentication(t *testing.T) { + for _, env := range []string{adapter.MigrationImportOpcodeEnv, adapter.MigrationPromoteOpcodeEnv, adapter.MigrationCleanupOpcodeEnv} { + t.Setenv(env, "false") + } + for _, env := range []string{adapter.MigrationImportOpcodeEnv, adapter.MigrationPromoteOpcodeEnv, adapter.MigrationCleanupOpcodeEnv} { + t.Run(env, func(t *testing.T) { + t.Setenv(env, "true") + _, err := loadNodeGRPCAuth("", false, "", "") + require.ErrorContains(t, err, "migration opcodes require") + _, err = loadNodeGRPCAuth("", true, "", "") + require.NoError(t, err) + }) + } +} + +func TestSplitMigrationPeerAuthRolloutPausesRunnerAndCapability(t *testing.T) { + oldEnforced, oldNoAuth := *splitMigrationPeerAuthEnforced, *adminInsecureNoAuth + *splitMigrationPeerAuthEnforced, *adminInsecureNoAuth = false, false + t.Cleanup(func() { + *splitMigrationPeerAuthEnforced, *adminInsecureNoAuth = oldEnforced, oldNoAuth + }) + for _, env := range []string{adapter.MigrationImportOpcodeEnv, adapter.MigrationPromoteOpcodeEnv, adapter.MigrationCleanupOpcodeEnv} { + t.Setenv(env, "true") + } + require.ErrorContains(t, splitMigrationLocalReadinessGate(context.Background()), "peer auth rollout") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + group := new(errgroup.Group) + startDistributionSplitJobRunner(ctx, group, adapter.NewDistributionServer(nil, nil)) + done := make(chan error, 1) + go func() { done <- group.Wait() }() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(time.Second): + cancel() + <-done + t.Fatal("send-only rollout stage must not start a privileged runner") + } + *splitMigrationPeerAuthEnforced = true + require.NoError(t, splitMigrationLocalReadinessGate(context.Background())) + *splitMigrationPeerAuthEnforced, *adminInsecureNoAuth = false, true + require.NoError(t, splitMigrationLocalReadinessGate(context.Background()), "explicit insecure mode retains its existing auth-rollout exception") +} + +func TestSplitMigrationTokenMustBeDedicated(t *testing.T) { + t.Parallel() + dir := t.TempDir() + adminPath, peerPath := filepath.Join(dir, "admin"), filepath.Join(dir, "peer") + require.NoError(t, os.WriteFile(adminPath, []byte("read-only-admin"), 0o600)) + require.NoError(t, os.WriteFile(peerPath, []byte("blob-peer"), 0o600)) + for _, token := range []string{"read-only-admin", "blob-peer", "migration-secret"} { + path := filepath.Join(t.TempDir(), "migration") + require.NoError(t, os.WriteFile(path, []byte(token), 0o600)) + cfg, err := loadNodeGRPCAuth(adminPath, false, peerPath, path) + if token != "migration-secret" { + require.ErrorContains(t, err, "migration token must differ") + continue + } + require.NoError(t, err) + require.Equal(t, "migration-secret", cfg.migrationToken) + _, err = loadNodeGRPCAuth("", true, "", path) + require.ErrorContains(t, err, "mutually exclusive") + } + cfg, err := loadNodeGRPCAuth(adminPath, false, peerPath, "") + require.NoError(t, err) + require.Empty(t, cfg.migrationToken, "an Admin token must never implicitly configure migration authority") +} + +type migrationAuthExportServer struct { + pb.UnimplementedInternalServer +} + +func (migrationAuthExportServer) ExportRangeVersions(_ *pb.ExportRangeVersionsRequest, stream pb.Internal_ExportRangeVersionsServer) error { + md, _ := metadata.FromIncomingContext(stream.Context()) + if got := md.Get("authorization"); len(got) != 1 || got[0] != "Bearer migration-secret" { + return status.Error(codes.Unauthenticated, "missing stream credential") + } + return stream.Send(&pb.ExportRangeVersionsResponse{Done: true}) +} + +func TestSplitMigrationClientNeverSendsTokenOnPlaintext(t *testing.T) { + t.Parallel() + var lc net.ListenConfig + lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + server := grpc.NewServer() + pb.RegisterInternalServer(server, &pb.UnimplementedInternalServer{}) + serveErr := make(chan error, 1) + go func() { serveErr <- server.Serve(lis) }() + t.Cleanup(func() { + server.Stop() + _ = lis.Close() + require.NoError(t, <-serveErr) + }) + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + client := newSplitMigrationInternalClient(conn, "migration-secret") + _, err = client.CleanupMigration(ctx, &pb.CleanupMigrationRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + require.ErrorContains(t, err, "cannot send secure credentials") + stream, err := client.ExportRangeVersions(ctx, &pb.ExportRangeVersionsRequest{}) + if err == nil { + _, err = stream.Recv() + } + require.Equal(t, codes.Unauthenticated, status.Code(err)) + require.ErrorContains(t, err, "cannot send secure credentials") + err = conn.Invoke(ctx, pb.Internal_Forward_FullMethodName, &emptypb.Empty{}, &emptypb.Empty{}) + require.Equal(t, codes.Unimplemented, status.Code(err), "ordinary plaintext peer calls must still reach the handler") +} diff --git a/main_split_migration_config_test.go b/main_split_migration_config_test.go new file mode 100644 index 000000000..805bfed2e --- /dev/null +++ b/main_split_migration_config_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/bootjp/elastickv/adapter" + internalutil "github.com/bootjp/elastickv/internal" + "github.com/stretchr/testify/require" +) + +func TestSplitMigrationPeerSecurityConfiguration(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + token string + noAuth bool + enforced bool + tls bool + wantErr string + }{ + {name: "ordinary plaintext cluster"}, + {name: "TLS send-only", token: "secret", tls: true}, + {name: "TLS enforced", token: "secret", tls: true, enforced: true}, + {name: "plaintext send-only", token: "secret", wantErr: "requires gRPC TLS"}, + {name: "plaintext enforced", token: "secret", enforced: true, wantErr: "requires gRPC TLS"}, + {name: "enforced without token", enforced: true, wantErr: "requires --splitMigrationTokenFile"}, + {name: "explicit isolated no-auth", enforced: true, noAuth: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + err := validateMigrationPeerSecurity(tc.token, tc.noAuth, tc.enforced, tc.tls) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestRunRejectsUnsafeMigrationAuthBeforeStartingServices(t *testing.T) { + paths := []*string{adminTokenFile, s3BlobPeerTokenFile, splitMigrationTokenFile, grpcTLSCAFile, grpcTLSCertFile, grpcTLSKeyFile} + for _, path := range paths { + old := *path + *path = "" + t.Cleanup(func() { *path = old }) + } + oldNoAuth, oldEnforced := *adminInsecureNoAuth, *splitMigrationPeerAuthEnforced + *adminInsecureNoAuth, *splitMigrationPeerAuthEnforced = false, false + t.Cleanup(func() { + *adminInsecureNoAuth, *splitMigrationPeerAuthEnforced = oldNoAuth, oldEnforced + require.NoError(t, internalutil.ConfigureGRPCTLS("", "", "")) + }) + for _, env := range []string{adapter.MigrationImportOpcodeEnv, adapter.MigrationPromoteOpcodeEnv, adapter.MigrationCleanupOpcodeEnv} { + t.Setenv(env, "false") + } + path := filepath.Join(t.TempDir(), "migration-token") + require.NoError(t, os.WriteFile(path, []byte("migration-secret"), 0o600)) + *splitMigrationTokenFile = path + require.ErrorContains(t, run(), "requires gRPC TLS") + *splitMigrationTokenFile, *splitMigrationPeerAuthEnforced = "", true + require.ErrorContains(t, run(), "requires --splitMigrationTokenFile") +} diff --git a/proto/distribution.pb.go b/proto/distribution.pb.go index 0cd1da1a2..18c373035 100644 --- a/proto/distribution.pb.go +++ b/proto/distribution.pb.go @@ -975,8 +975,21 @@ type SplitJob struct { StartedAtMs int64 `protobuf:"varint,31,opt,name=started_at_ms,json=startedAtMs,proto3" json:"started_at_ms,omitempty"` UpdatedAtMs int64 `protobuf:"varint,32,opt,name=updated_at_ms,json=updatedAtMs,proto3" json:"updated_at_ms,omitempty"` TerminalAtMs int64 `protobuf:"varint,33,opt,name=terminal_at_ms,json=terminalAtMs,proto3" json:"terminal_at_ms,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + CapabilityRegressed bool `protobuf:"varint,34,opt,name=capability_regressed,json=capabilityRegressed,proto3" json:"capability_regressed,omitempty"` + // Group that still owns the source side of the split. Recorded at plan + // time because the route shape it can otherwise be recovered from + // disappears once the source child is itself split. + SourceGroupId uint64 `protobuf:"varint,35,opt,name=source_group_id,json=sourceGroupId,proto3" json:"source_group_id,omitempty"` + // Version 1 preflight checks every transfer family and relevant MVCC version. + // Missing fields on legacy jobs never attest that this scan completed. + TransferPreflightVersion uint32 `protobuf:"varint,36,opt,name=transfer_preflight_version,json=transferPreflightVersion,proto3" json:"transfer_preflight_version,omitempty"` + TransferPreflightTs uint64 `protobuf:"varint,37,opt,name=transfer_preflight_ts,json=transferPreflightTs,proto3" json:"transfer_preflight_ts,omitempty"` + TransferPreflightProgress []*SplitJobBracketProgress `protobuf:"bytes,38,rep,name=transfer_preflight_progress,json=transferPreflightProgress,proto3" json:"transfer_preflight_progress,omitempty"` + TransferPreflightDone bool `protobuf:"varint,39,opt,name=transfer_preflight_done,json=transferPreflightDone,proto3" json:"transfer_preflight_done,omitempty"` + // Durable moving child created with its same-owner partition manifest. + WorkingSourceRouteId uint64 `protobuf:"varint,40,opt,name=working_source_route_id,json=workingSourceRouteId,proto3" json:"working_source_route_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SplitJob) Reset() { @@ -1240,6 +1253,55 @@ func (x *SplitJob) GetTerminalAtMs() int64 { return 0 } +func (x *SplitJob) GetCapabilityRegressed() bool { + if x != nil { + return x.CapabilityRegressed + } + return false +} + +func (x *SplitJob) GetSourceGroupId() uint64 { + if x != nil { + return x.SourceGroupId + } + return 0 +} + +func (x *SplitJob) GetTransferPreflightVersion() uint32 { + if x != nil { + return x.TransferPreflightVersion + } + return 0 +} + +func (x *SplitJob) GetTransferPreflightTs() uint64 { + if x != nil { + return x.TransferPreflightTs + } + return 0 +} + +func (x *SplitJob) GetTransferPreflightProgress() []*SplitJobBracketProgress { + if x != nil { + return x.TransferPreflightProgress + } + return nil +} + +func (x *SplitJob) GetTransferPreflightDone() bool { + if x != nil { + return x.TransferPreflightDone + } + return false +} + +func (x *SplitJob) GetWorkingSourceRouteId() uint64 { + if x != nil { + return x.WorkingSourceRouteId + } + return 0 +} + type ListRoutesRequest struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -1994,6 +2056,94 @@ func (x *StartSplitMigrationResponse) GetJobId() uint64 { return 0 } +type GetSplitMigrationCapabilityRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetSplitMigrationCapabilityRequest) Reset() { + *x = GetSplitMigrationCapabilityRequest{} + mi := &file_distribution_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetSplitMigrationCapabilityRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetSplitMigrationCapabilityRequest) ProtoMessage() {} + +func (x *GetSplitMigrationCapabilityRequest) ProtoReflect() protoreflect.Message { + mi := &file_distribution_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetSplitMigrationCapabilityRequest.ProtoReflect.Descriptor instead. +func (*GetSplitMigrationCapabilityRequest) Descriptor() ([]byte, []int) { + return file_distribution_proto_rawDescGZIP(), []int{22} +} + +type GetSplitMigrationCapabilityResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + MigrationCapable bool `protobuf:"varint,1,opt,name=migration_capable,json=migrationCapable,proto3" json:"migration_capable,omitempty"` + Capabilities []string `protobuf:"bytes,2,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetSplitMigrationCapabilityResponse) Reset() { + *x = GetSplitMigrationCapabilityResponse{} + mi := &file_distribution_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetSplitMigrationCapabilityResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetSplitMigrationCapabilityResponse) ProtoMessage() {} + +func (x *GetSplitMigrationCapabilityResponse) ProtoReflect() protoreflect.Message { + mi := &file_distribution_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetSplitMigrationCapabilityResponse.ProtoReflect.Descriptor instead. +func (*GetSplitMigrationCapabilityResponse) Descriptor() ([]byte, []int) { + return file_distribution_proto_rawDescGZIP(), []int{23} +} + +func (x *GetSplitMigrationCapabilityResponse) GetMigrationCapable() bool { + if x != nil { + return x.MigrationCapable + } + return false +} + +func (x *GetSplitMigrationCapabilityResponse) GetCapabilities() []string { + if x != nil { + return x.Capabilities + } + return nil +} + type GetRouteOwnershipRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` @@ -2004,7 +2154,7 @@ type GetRouteOwnershipRequest struct { func (x *GetRouteOwnershipRequest) Reset() { *x = GetRouteOwnershipRequest{} - mi := &file_distribution_proto_msgTypes[22] + mi := &file_distribution_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2016,7 +2166,7 @@ func (x *GetRouteOwnershipRequest) String() string { func (*GetRouteOwnershipRequest) ProtoMessage() {} func (x *GetRouteOwnershipRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[22] + mi := &file_distribution_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2029,7 +2179,7 @@ func (x *GetRouteOwnershipRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetRouteOwnershipRequest.ProtoReflect.Descriptor instead. func (*GetRouteOwnershipRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{22} + return file_distribution_proto_rawDescGZIP(), []int{24} } func (x *GetRouteOwnershipRequest) GetKey() []byte { @@ -2057,7 +2207,7 @@ type GetRouteOwnershipResponse struct { func (x *GetRouteOwnershipResponse) Reset() { *x = GetRouteOwnershipResponse{} - mi := &file_distribution_proto_msgTypes[23] + mi := &file_distribution_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2069,7 +2219,7 @@ func (x *GetRouteOwnershipResponse) String() string { func (*GetRouteOwnershipResponse) ProtoMessage() {} func (x *GetRouteOwnershipResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[23] + mi := &file_distribution_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2082,7 +2232,7 @@ func (x *GetRouteOwnershipResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetRouteOwnershipResponse.ProtoReflect.Descriptor instead. func (*GetRouteOwnershipResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{23} + return file_distribution_proto_rawDescGZIP(), []int{25} } func (x *GetRouteOwnershipResponse) GetRoute() *RouteDescriptor { @@ -2117,7 +2267,7 @@ type GetIntersectingRoutesRequest struct { func (x *GetIntersectingRoutesRequest) Reset() { *x = GetIntersectingRoutesRequest{} - mi := &file_distribution_proto_msgTypes[24] + mi := &file_distribution_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2129,7 +2279,7 @@ func (x *GetIntersectingRoutesRequest) String() string { func (*GetIntersectingRoutesRequest) ProtoMessage() {} func (x *GetIntersectingRoutesRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[24] + mi := &file_distribution_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2142,7 +2292,7 @@ func (x *GetIntersectingRoutesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetIntersectingRoutesRequest.ProtoReflect.Descriptor instead. func (*GetIntersectingRoutesRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{24} + return file_distribution_proto_rawDescGZIP(), []int{26} } func (x *GetIntersectingRoutesRequest) GetStart() []byte { @@ -2176,7 +2326,7 @@ type GetIntersectingRoutesResponse struct { func (x *GetIntersectingRoutesResponse) Reset() { *x = GetIntersectingRoutesResponse{} - mi := &file_distribution_proto_msgTypes[25] + mi := &file_distribution_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2188,7 +2338,7 @@ func (x *GetIntersectingRoutesResponse) String() string { func (*GetIntersectingRoutesResponse) ProtoMessage() {} func (x *GetIntersectingRoutesResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[25] + mi := &file_distribution_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2201,7 +2351,7 @@ func (x *GetIntersectingRoutesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetIntersectingRoutesResponse.ProtoReflect.Descriptor instead. func (*GetIntersectingRoutesResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{25} + return file_distribution_proto_rawDescGZIP(), []int{27} } func (x *GetIntersectingRoutesResponse) GetRoutes() []*RouteDescriptor { @@ -2227,7 +2377,7 @@ type GetSplitJobRequest struct { func (x *GetSplitJobRequest) Reset() { *x = GetSplitJobRequest{} - mi := &file_distribution_proto_msgTypes[26] + mi := &file_distribution_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2239,7 +2389,7 @@ func (x *GetSplitJobRequest) String() string { func (*GetSplitJobRequest) ProtoMessage() {} func (x *GetSplitJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[26] + mi := &file_distribution_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2252,7 +2402,7 @@ func (x *GetSplitJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetSplitJobRequest.ProtoReflect.Descriptor instead. func (*GetSplitJobRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{26} + return file_distribution_proto_rawDescGZIP(), []int{28} } func (x *GetSplitJobRequest) GetJobId() uint64 { @@ -2271,7 +2421,7 @@ type GetSplitJobResponse struct { func (x *GetSplitJobResponse) Reset() { *x = GetSplitJobResponse{} - mi := &file_distribution_proto_msgTypes[27] + mi := &file_distribution_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2283,7 +2433,7 @@ func (x *GetSplitJobResponse) String() string { func (*GetSplitJobResponse) ProtoMessage() {} func (x *GetSplitJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[27] + mi := &file_distribution_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2296,7 +2446,7 @@ func (x *GetSplitJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetSplitJobResponse.ProtoReflect.Descriptor instead. func (*GetSplitJobResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{27} + return file_distribution_proto_rawDescGZIP(), []int{29} } func (x *GetSplitJobResponse) GetJob() *SplitJob { @@ -2317,7 +2467,7 @@ type ListSplitJobsRequest struct { func (x *ListSplitJobsRequest) Reset() { *x = ListSplitJobsRequest{} - mi := &file_distribution_proto_msgTypes[28] + mi := &file_distribution_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2329,7 +2479,7 @@ func (x *ListSplitJobsRequest) String() string { func (*ListSplitJobsRequest) ProtoMessage() {} func (x *ListSplitJobsRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[28] + mi := &file_distribution_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2342,7 +2492,7 @@ func (x *ListSplitJobsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListSplitJobsRequest.ProtoReflect.Descriptor instead. func (*ListSplitJobsRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{28} + return file_distribution_proto_rawDescGZIP(), []int{30} } func (x *ListSplitJobsRequest) GetSinceTerminalAtMs() uint64 { @@ -2376,7 +2526,7 @@ type ListSplitJobsResponse struct { func (x *ListSplitJobsResponse) Reset() { *x = ListSplitJobsResponse{} - mi := &file_distribution_proto_msgTypes[29] + mi := &file_distribution_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2388,7 +2538,7 @@ func (x *ListSplitJobsResponse) String() string { func (*ListSplitJobsResponse) ProtoMessage() {} func (x *ListSplitJobsResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[29] + mi := &file_distribution_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2401,7 +2551,7 @@ func (x *ListSplitJobsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListSplitJobsResponse.ProtoReflect.Descriptor instead. func (*ListSplitJobsResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{29} + return file_distribution_proto_rawDescGZIP(), []int{31} } func (x *ListSplitJobsResponse) GetJobs() []*SplitJob { @@ -2427,7 +2577,7 @@ type AbandonSplitJobRequest struct { func (x *AbandonSplitJobRequest) Reset() { *x = AbandonSplitJobRequest{} - mi := &file_distribution_proto_msgTypes[30] + mi := &file_distribution_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2439,7 +2589,7 @@ func (x *AbandonSplitJobRequest) String() string { func (*AbandonSplitJobRequest) ProtoMessage() {} func (x *AbandonSplitJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[30] + mi := &file_distribution_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2452,7 +2602,7 @@ func (x *AbandonSplitJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AbandonSplitJobRequest.ProtoReflect.Descriptor instead. func (*AbandonSplitJobRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{30} + return file_distribution_proto_rawDescGZIP(), []int{32} } func (x *AbandonSplitJobRequest) GetJobId() uint64 { @@ -2470,7 +2620,7 @@ type AbandonSplitJobResponse struct { func (x *AbandonSplitJobResponse) Reset() { *x = AbandonSplitJobResponse{} - mi := &file_distribution_proto_msgTypes[31] + mi := &file_distribution_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2482,7 +2632,7 @@ func (x *AbandonSplitJobResponse) String() string { func (*AbandonSplitJobResponse) ProtoMessage() {} func (x *AbandonSplitJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[31] + mi := &file_distribution_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2495,7 +2645,7 @@ func (x *AbandonSplitJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AbandonSplitJobResponse.ProtoReflect.Descriptor instead. func (*AbandonSplitJobResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{31} + return file_distribution_proto_rawDescGZIP(), []int{33} } type RetrySplitJobRequest struct { @@ -2507,7 +2657,7 @@ type RetrySplitJobRequest struct { func (x *RetrySplitJobRequest) Reset() { *x = RetrySplitJobRequest{} - mi := &file_distribution_proto_msgTypes[32] + mi := &file_distribution_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2519,7 +2669,7 @@ func (x *RetrySplitJobRequest) String() string { func (*RetrySplitJobRequest) ProtoMessage() {} func (x *RetrySplitJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[32] + mi := &file_distribution_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2532,7 +2682,7 @@ func (x *RetrySplitJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RetrySplitJobRequest.ProtoReflect.Descriptor instead. func (*RetrySplitJobRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{32} + return file_distribution_proto_rawDescGZIP(), []int{34} } func (x *RetrySplitJobRequest) GetJobId() uint64 { @@ -2550,7 +2700,7 @@ type RetrySplitJobResponse struct { func (x *RetrySplitJobResponse) Reset() { *x = RetrySplitJobResponse{} - mi := &file_distribution_proto_msgTypes[33] + mi := &file_distribution_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2562,7 +2712,7 @@ func (x *RetrySplitJobResponse) String() string { func (*RetrySplitJobResponse) ProtoMessage() {} func (x *RetrySplitJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[33] + mi := &file_distribution_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2575,14 +2725,142 @@ func (x *RetrySplitJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RetrySplitJobResponse.ProtoReflect.Descriptor instead. func (*RetrySplitJobResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{33} + return file_distribution_proto_rawDescGZIP(), []int{35} +} + +type AttestMigrationOwnershipRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId uint64 `protobuf:"varint,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + Prior *MigrationOwnershipProof `protobuf:"bytes,2,opt,name=prior,proto3" json:"prior,omitempty"` + PriorPartition []byte `protobuf:"bytes,3,opt,name=prior_partition,json=priorPartition,proto3" json:"prior_partition,omitempty"` + PriorOwnershipState []byte `protobuf:"bytes,4,opt,name=prior_ownership_state,json=priorOwnershipState,proto3" json:"prior_ownership_state,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttestMigrationOwnershipRequest) Reset() { + *x = AttestMigrationOwnershipRequest{} + mi := &file_distribution_proto_msgTypes[36] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttestMigrationOwnershipRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttestMigrationOwnershipRequest) ProtoMessage() {} + +func (x *AttestMigrationOwnershipRequest) ProtoReflect() protoreflect.Message { + mi := &file_distribution_proto_msgTypes[36] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttestMigrationOwnershipRequest.ProtoReflect.Descriptor instead. +func (*AttestMigrationOwnershipRequest) Descriptor() ([]byte, []int) { + return file_distribution_proto_rawDescGZIP(), []int{36} +} + +func (x *AttestMigrationOwnershipRequest) GetJobId() uint64 { + if x != nil { + return x.JobId + } + return 0 +} + +func (x *AttestMigrationOwnershipRequest) GetPrior() *MigrationOwnershipProof { + if x != nil { + return x.Prior + } + return nil +} + +func (x *AttestMigrationOwnershipRequest) GetPriorPartition() []byte { + if x != nil { + return x.PriorPartition + } + return nil +} + +func (x *AttestMigrationOwnershipRequest) GetPriorOwnershipState() []byte { + if x != nil { + return x.PriorOwnershipState + } + return nil +} + +type AttestMigrationOwnershipResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + OwnershipProtocol uint32 `protobuf:"varint,1,opt,name=ownership_protocol,json=ownershipProtocol,proto3" json:"ownership_protocol,omitempty"` + Ownership *MigrationOwnershipProof `protobuf:"bytes,2,opt,name=ownership,proto3" json:"ownership,omitempty"` + PartitionTransition []byte `protobuf:"bytes,3,opt,name=partition_transition,json=partitionTransition,proto3" json:"partition_transition,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttestMigrationOwnershipResponse) Reset() { + *x = AttestMigrationOwnershipResponse{} + mi := &file_distribution_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttestMigrationOwnershipResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttestMigrationOwnershipResponse) ProtoMessage() {} + +func (x *AttestMigrationOwnershipResponse) ProtoReflect() protoreflect.Message { + mi := &file_distribution_proto_msgTypes[37] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttestMigrationOwnershipResponse.ProtoReflect.Descriptor instead. +func (*AttestMigrationOwnershipResponse) Descriptor() ([]byte, []int) { + return file_distribution_proto_rawDescGZIP(), []int{37} +} + +func (x *AttestMigrationOwnershipResponse) GetOwnershipProtocol() uint32 { + if x != nil { + return x.OwnershipProtocol + } + return 0 +} + +func (x *AttestMigrationOwnershipResponse) GetOwnership() *MigrationOwnershipProof { + if x != nil { + return x.Ownership + } + return nil +} + +func (x *AttestMigrationOwnershipResponse) GetPartitionTransition() []byte { + if x != nil { + return x.PartitionTransition + } + return nil } var File_distribution_proto protoreflect.FileDescriptor const file_distribution_proto_rawDesc = "" + "\n" + - "\x12distribution.proto\"#\n" + + "\x12distribution.proto\x1a\x0einternal.proto\"#\n" + "\x0fGetRouteRequest\x12\x10\n" + "\x03key\x18\x01 \x01(\fR\x03key\"^\n" + "\x10GetRouteResponse\x12\x14\n" + @@ -2634,7 +2912,7 @@ const file_distribution_proto_rawDesc = "" + "\x04done\x18\x05 \x01(\bR\x04done\x12#\n" + "\rscanned_bytes\x18\x06 \x01(\x04R\fscannedBytes\x12#\n" + "\raccepted_rows\x18\a \x01(\x04R\facceptedRows\x12/\n" + - "\x14last_acked_batch_seq\x18\b \x01(\x04R\x11lastAckedBatchSeq\"\xe9\f\n" + + "\x14last_acked_batch_seq\x18\b \x01(\x04R\x11lastAckedBatchSeq\"\xff\x0f\n" + "\bSplitJob\x12\x15\n" + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12&\n" + "\x0fsource_route_id\x18\x02 \x01(\x04R\rsourceRouteId\x12\x1b\n" + @@ -2673,7 +2951,14 @@ const file_distribution_proto_rawDesc = "" + "last_error\x18\x1e \x01(\tR\tlastError\x12\"\n" + "\rstarted_at_ms\x18\x1f \x01(\x03R\vstartedAtMs\x12\"\n" + "\rupdated_at_ms\x18 \x01(\x03R\vupdatedAtMs\x12$\n" + - "\x0eterminal_at_ms\x18! \x01(\x03R\fterminalAtMs\"\x13\n" + + "\x0eterminal_at_ms\x18! \x01(\x03R\fterminalAtMs\x121\n" + + "\x14capability_regressed\x18\" \x01(\bR\x13capabilityRegressed\x12&\n" + + "\x0fsource_group_id\x18# \x01(\x04R\rsourceGroupId\x12<\n" + + "\x1atransfer_preflight_version\x18$ \x01(\rR\x18transferPreflightVersion\x122\n" + + "\x15transfer_preflight_ts\x18% \x01(\x04R\x13transferPreflightTs\x12X\n" + + "\x1btransfer_preflight_progress\x18& \x03(\v2\x18.SplitJobBracketProgressR\x19transferPreflightProgress\x126\n" + + "\x17transfer_preflight_done\x18' \x01(\bR\x15transferPreflightDone\x125\n" + + "\x17working_source_route_id\x18( \x01(\x04R\x14workingSourceRouteId\"\x13\n" + "\x11ListRoutesRequest\"g\n" + "\x12ListRoutesResponse\x12'\n" + "\x0fcatalog_version\x18\x01 \x01(\x04R\x0ecatalogVersion\x12(\n" + @@ -2722,7 +3007,11 @@ const file_distribution_proto_rawDesc = "" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"]\n" + "\x1bStartSplitMigrationResponse\x12'\n" + "\x0fcatalog_version\x18\x01 \x01(\x04R\x0ecatalogVersion\x12\x15\n" + - "\x06job_id\x18\x02 \x01(\x04R\x05jobId\"U\n" + + "\x06job_id\x18\x02 \x01(\x04R\x05jobId\"$\n" + + "\"GetSplitMigrationCapabilityRequest\"v\n" + + "#GetSplitMigrationCapabilityResponse\x12+\n" + + "\x11migration_capable\x18\x01 \x01(\bR\x10migrationCapable\x12\"\n" + + "\fcapabilities\x18\x02 \x03(\tR\fcapabilities\"U\n" + "\x18GetRouteOwnershipRequest\x12\x10\n" + "\x03key\x18\x01 \x01(\fR\x03key\x12'\n" + "\x0fcatalog_version\x18\x02 \x01(\x04R\x0ecatalogVersion\"\x82\x01\n" + @@ -2754,7 +3043,16 @@ const file_distribution_proto_rawDesc = "" + "\x17AbandonSplitJobResponse\"-\n" + "\x14RetrySplitJobRequest\x12\x15\n" + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\"\x17\n" + - "\x15RetrySplitJobResponse*\xa3\x01\n" + + "\x15RetrySplitJobResponse\"\xc5\x01\n" + + "\x1fAttestMigrationOwnershipRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12.\n" + + "\x05prior\x18\x02 \x01(\v2\x18.MigrationOwnershipProofR\x05prior\x12'\n" + + "\x0fprior_partition\x18\x03 \x01(\fR\x0epriorPartition\x122\n" + + "\x15prior_ownership_state\x18\x04 \x01(\fR\x13priorOwnershipState\"\xbc\x01\n" + + " AttestMigrationOwnershipResponse\x12-\n" + + "\x12ownership_protocol\x18\x01 \x01(\rR\x11ownershipProtocol\x126\n" + + "\townership\x18\x02 \x01(\v2\x18.MigrationOwnershipProofR\townership\x121\n" + + "\x14partition_transition\x18\x03 \x01(\fR\x13partitionTransition*\xa3\x01\n" + "\n" + "RouteState\x12\x1b\n" + "\x17ROUTE_STATE_UNSPECIFIED\x10\x00\x12\x16\n" + @@ -2787,7 +3085,7 @@ const file_distribution_proto_rawDesc = "" + "\x16CatalogDeltaMutationOp\x12)\n" + "%CATALOG_DELTA_MUTATION_OP_UNSPECIFIED\x10\x00\x12$\n" + " CATALOG_DELTA_MUTATION_OP_UPSERT\x10\x01\x12$\n" + - " CATALOG_DELTA_MUTATION_OP_DELETE\x10\x022\xd9\a\n" + + " CATALOG_DELTA_MUTATION_OP_DELETE\x10\x022\xa8\t\n" + "\fDistribution\x121\n" + "\bGetRoute\x12\x10.GetRouteRequest\x1a\x11.GetRouteResponse\"\x00\x12=\n" + "\fGetTimestamp\x12\x14.GetTimestampRequest\x1a\x15.GetTimestampResponse\"\x00\x12L\n" + @@ -2798,13 +3096,15 @@ const file_distribution_proto_rawDesc = "" + "SplitRange\x12\x12.SplitRangeRequest\x1a\x13.SplitRangeResponse\"\x00\x12U\n" + "\x16GetCatalogCapabilities\x12\x1b.CatalogCapabilitiesRequest\x1a\x1c.CatalogCapabilitiesResponse\"\x00\x12<\n" + "\fWatchCatalog\x12\x14.CatalogWatchRequest\x1a\x12.CatalogWatchEvent\"\x000\x01\x12R\n" + - "\x13StartSplitMigration\x12\x1b.StartSplitMigrationRequest\x1a\x1c.StartSplitMigrationResponse\"\x00\x12L\n" + + "\x13StartSplitMigration\x12\x1b.StartSplitMigrationRequest\x1a\x1c.StartSplitMigrationResponse\"\x00\x12j\n" + + "\x1bGetSplitMigrationCapability\x12#.GetSplitMigrationCapabilityRequest\x1a$.GetSplitMigrationCapabilityResponse\"\x00\x12L\n" + "\x11GetRouteOwnership\x12\x19.GetRouteOwnershipRequest\x1a\x1a.GetRouteOwnershipResponse\"\x00\x12X\n" + "\x15GetIntersectingRoutes\x12\x1d.GetIntersectingRoutesRequest\x1a\x1e.GetIntersectingRoutesResponse\"\x00\x12:\n" + "\vGetSplitJob\x12\x13.GetSplitJobRequest\x1a\x14.GetSplitJobResponse\"\x00\x12@\n" + "\rListSplitJobs\x12\x15.ListSplitJobsRequest\x1a\x16.ListSplitJobsResponse\"\x00\x12F\n" + "\x0fAbandonSplitJob\x12\x17.AbandonSplitJobRequest\x1a\x18.AbandonSplitJobResponse\"\x00\x12@\n" + - "\rRetrySplitJob\x12\x15.RetrySplitJobRequest\x1a\x16.RetrySplitJobResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" + "\rRetrySplitJob\x12\x15.RetrySplitJobRequest\x1a\x16.RetrySplitJobResponse\"\x00\x12a\n" + + "\x18AttestMigrationOwnership\x12 .AttestMigrationOwnershipRequest\x1a!.AttestMigrationOwnershipResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" var ( file_distribution_proto_rawDescOnce sync.Once @@ -2819,48 +3119,53 @@ func file_distribution_proto_rawDescGZIP() []byte { } var file_distribution_proto_enumTypes = make([]protoimpl.EnumInfo, 5) -var file_distribution_proto_msgTypes = make([]protoimpl.MessageInfo, 35) +var file_distribution_proto_msgTypes = make([]protoimpl.MessageInfo, 39) var file_distribution_proto_goTypes = []any{ - (RouteState)(0), // 0: RouteState - (SplitJobPhase)(0), // 1: SplitJobPhase - (SplitJobBarrierState)(0), // 2: SplitJobBarrierState - (SplitJobExportPhase)(0), // 3: SplitJobExportPhase - (CatalogDeltaMutationOp)(0), // 4: CatalogDeltaMutationOp - (*GetRouteRequest)(nil), // 5: GetRouteRequest - (*GetRouteResponse)(nil), // 6: GetRouteResponse - (*GetTimestampRequest)(nil), // 7: GetTimestampRequest - (*GetTimestampResponse)(nil), // 8: GetTimestampResponse - (*ValidateTimestampRequest)(nil), // 9: ValidateTimestampRequest - (*ValidateTimestampResponse)(nil), // 10: ValidateTimestampResponse - (*RouteDescriptor)(nil), // 11: RouteDescriptor - (*SplitJobBracketProgress)(nil), // 12: SplitJobBracketProgress - (*SplitJob)(nil), // 13: SplitJob - (*ListRoutesRequest)(nil), // 14: ListRoutesRequest - (*ListRoutesResponse)(nil), // 15: ListRoutesResponse - (*SplitRangeRequest)(nil), // 16: SplitRangeRequest - (*SplitRangeResponse)(nil), // 17: SplitRangeResponse - (*CatalogCapabilitiesRequest)(nil), // 18: CatalogCapabilitiesRequest - (*CatalogCapabilitiesResponse)(nil), // 19: CatalogCapabilitiesResponse - (*CatalogWatchRequest)(nil), // 20: CatalogWatchRequest - (*CatalogDeltaMutation)(nil), // 21: CatalogDeltaMutation - (*CatalogDeltaRecord)(nil), // 22: CatalogDeltaRecord - (*CatalogSnapshotReset)(nil), // 23: CatalogSnapshotReset - (*CatalogWatchEvent)(nil), // 24: CatalogWatchEvent - (*StartSplitMigrationRequest)(nil), // 25: StartSplitMigrationRequest - (*StartSplitMigrationResponse)(nil), // 26: StartSplitMigrationResponse - (*GetRouteOwnershipRequest)(nil), // 27: GetRouteOwnershipRequest - (*GetRouteOwnershipResponse)(nil), // 28: GetRouteOwnershipResponse - (*GetIntersectingRoutesRequest)(nil), // 29: GetIntersectingRoutesRequest - (*GetIntersectingRoutesResponse)(nil), // 30: GetIntersectingRoutesResponse - (*GetSplitJobRequest)(nil), // 31: GetSplitJobRequest - (*GetSplitJobResponse)(nil), // 32: GetSplitJobResponse - (*ListSplitJobsRequest)(nil), // 33: ListSplitJobsRequest - (*ListSplitJobsResponse)(nil), // 34: ListSplitJobsResponse - (*AbandonSplitJobRequest)(nil), // 35: AbandonSplitJobRequest - (*AbandonSplitJobResponse)(nil), // 36: AbandonSplitJobResponse - (*RetrySplitJobRequest)(nil), // 37: RetrySplitJobRequest - (*RetrySplitJobResponse)(nil), // 38: RetrySplitJobResponse - nil, // 39: StartSplitMigrationRequest.OptionsEntry + (RouteState)(0), // 0: RouteState + (SplitJobPhase)(0), // 1: SplitJobPhase + (SplitJobBarrierState)(0), // 2: SplitJobBarrierState + (SplitJobExportPhase)(0), // 3: SplitJobExportPhase + (CatalogDeltaMutationOp)(0), // 4: CatalogDeltaMutationOp + (*GetRouteRequest)(nil), // 5: GetRouteRequest + (*GetRouteResponse)(nil), // 6: GetRouteResponse + (*GetTimestampRequest)(nil), // 7: GetTimestampRequest + (*GetTimestampResponse)(nil), // 8: GetTimestampResponse + (*ValidateTimestampRequest)(nil), // 9: ValidateTimestampRequest + (*ValidateTimestampResponse)(nil), // 10: ValidateTimestampResponse + (*RouteDescriptor)(nil), // 11: RouteDescriptor + (*SplitJobBracketProgress)(nil), // 12: SplitJobBracketProgress + (*SplitJob)(nil), // 13: SplitJob + (*ListRoutesRequest)(nil), // 14: ListRoutesRequest + (*ListRoutesResponse)(nil), // 15: ListRoutesResponse + (*SplitRangeRequest)(nil), // 16: SplitRangeRequest + (*SplitRangeResponse)(nil), // 17: SplitRangeResponse + (*CatalogCapabilitiesRequest)(nil), // 18: CatalogCapabilitiesRequest + (*CatalogCapabilitiesResponse)(nil), // 19: CatalogCapabilitiesResponse + (*CatalogWatchRequest)(nil), // 20: CatalogWatchRequest + (*CatalogDeltaMutation)(nil), // 21: CatalogDeltaMutation + (*CatalogDeltaRecord)(nil), // 22: CatalogDeltaRecord + (*CatalogSnapshotReset)(nil), // 23: CatalogSnapshotReset + (*CatalogWatchEvent)(nil), // 24: CatalogWatchEvent + (*StartSplitMigrationRequest)(nil), // 25: StartSplitMigrationRequest + (*StartSplitMigrationResponse)(nil), // 26: StartSplitMigrationResponse + (*GetSplitMigrationCapabilityRequest)(nil), // 27: GetSplitMigrationCapabilityRequest + (*GetSplitMigrationCapabilityResponse)(nil), // 28: GetSplitMigrationCapabilityResponse + (*GetRouteOwnershipRequest)(nil), // 29: GetRouteOwnershipRequest + (*GetRouteOwnershipResponse)(nil), // 30: GetRouteOwnershipResponse + (*GetIntersectingRoutesRequest)(nil), // 31: GetIntersectingRoutesRequest + (*GetIntersectingRoutesResponse)(nil), // 32: GetIntersectingRoutesResponse + (*GetSplitJobRequest)(nil), // 33: GetSplitJobRequest + (*GetSplitJobResponse)(nil), // 34: GetSplitJobResponse + (*ListSplitJobsRequest)(nil), // 35: ListSplitJobsRequest + (*ListSplitJobsResponse)(nil), // 36: ListSplitJobsResponse + (*AbandonSplitJobRequest)(nil), // 37: AbandonSplitJobRequest + (*AbandonSplitJobResponse)(nil), // 38: AbandonSplitJobResponse + (*RetrySplitJobRequest)(nil), // 39: RetrySplitJobRequest + (*RetrySplitJobResponse)(nil), // 40: RetrySplitJobResponse + (*AttestMigrationOwnershipRequest)(nil), // 41: AttestMigrationOwnershipRequest + (*AttestMigrationOwnershipResponse)(nil), // 42: AttestMigrationOwnershipResponse + nil, // 43: StartSplitMigrationRequest.OptionsEntry + (*MigrationOwnershipProof)(nil), // 44: MigrationOwnershipProof } var file_distribution_proto_depIdxs = []int32{ 0, // 0: RouteDescriptor.state:type_name -> RouteState @@ -2871,53 +3176,60 @@ var file_distribution_proto_depIdxs = []int32{ 2, // 5: SplitJob.cutover_read_fence_state:type_name -> SplitJobBarrierState 2, // 6: SplitJob.target_staged_readiness_state:type_name -> SplitJobBarrierState 12, // 7: SplitJob.bracket_progress:type_name -> SplitJobBracketProgress - 11, // 8: ListRoutesResponse.routes:type_name -> RouteDescriptor - 11, // 9: SplitRangeResponse.left:type_name -> RouteDescriptor - 11, // 10: SplitRangeResponse.right:type_name -> RouteDescriptor - 4, // 11: CatalogDeltaMutation.op:type_name -> CatalogDeltaMutationOp - 11, // 12: CatalogDeltaMutation.route:type_name -> RouteDescriptor - 21, // 13: CatalogDeltaRecord.mutations:type_name -> CatalogDeltaMutation - 11, // 14: CatalogSnapshotReset.routes:type_name -> RouteDescriptor - 23, // 15: CatalogWatchEvent.snapshot:type_name -> CatalogSnapshotReset - 22, // 16: CatalogWatchEvent.delta:type_name -> CatalogDeltaRecord - 39, // 17: StartSplitMigrationRequest.options:type_name -> StartSplitMigrationRequest.OptionsEntry - 11, // 18: GetRouteOwnershipResponse.route:type_name -> RouteDescriptor - 11, // 19: GetIntersectingRoutesResponse.routes:type_name -> RouteDescriptor - 13, // 20: GetSplitJobResponse.job:type_name -> SplitJob - 13, // 21: ListSplitJobsResponse.jobs:type_name -> SplitJob - 5, // 22: Distribution.GetRoute:input_type -> GetRouteRequest - 7, // 23: Distribution.GetTimestamp:input_type -> GetTimestampRequest - 9, // 24: Distribution.ValidateTimestamp:input_type -> ValidateTimestampRequest - 14, // 25: Distribution.ListRoutes:input_type -> ListRoutesRequest - 16, // 26: Distribution.SplitRange:input_type -> SplitRangeRequest - 18, // 27: Distribution.GetCatalogCapabilities:input_type -> CatalogCapabilitiesRequest - 20, // 28: Distribution.WatchCatalog:input_type -> CatalogWatchRequest - 25, // 29: Distribution.StartSplitMigration:input_type -> StartSplitMigrationRequest - 27, // 30: Distribution.GetRouteOwnership:input_type -> GetRouteOwnershipRequest - 29, // 31: Distribution.GetIntersectingRoutes:input_type -> GetIntersectingRoutesRequest - 31, // 32: Distribution.GetSplitJob:input_type -> GetSplitJobRequest - 33, // 33: Distribution.ListSplitJobs:input_type -> ListSplitJobsRequest - 35, // 34: Distribution.AbandonSplitJob:input_type -> AbandonSplitJobRequest - 37, // 35: Distribution.RetrySplitJob:input_type -> RetrySplitJobRequest - 6, // 36: Distribution.GetRoute:output_type -> GetRouteResponse - 8, // 37: Distribution.GetTimestamp:output_type -> GetTimestampResponse - 10, // 38: Distribution.ValidateTimestamp:output_type -> ValidateTimestampResponse - 15, // 39: Distribution.ListRoutes:output_type -> ListRoutesResponse - 17, // 40: Distribution.SplitRange:output_type -> SplitRangeResponse - 19, // 41: Distribution.GetCatalogCapabilities:output_type -> CatalogCapabilitiesResponse - 24, // 42: Distribution.WatchCatalog:output_type -> CatalogWatchEvent - 26, // 43: Distribution.StartSplitMigration:output_type -> StartSplitMigrationResponse - 28, // 44: Distribution.GetRouteOwnership:output_type -> GetRouteOwnershipResponse - 30, // 45: Distribution.GetIntersectingRoutes:output_type -> GetIntersectingRoutesResponse - 32, // 46: Distribution.GetSplitJob:output_type -> GetSplitJobResponse - 34, // 47: Distribution.ListSplitJobs:output_type -> ListSplitJobsResponse - 36, // 48: Distribution.AbandonSplitJob:output_type -> AbandonSplitJobResponse - 38, // 49: Distribution.RetrySplitJob:output_type -> RetrySplitJobResponse - 36, // [36:50] is the sub-list for method output_type - 22, // [22:36] is the sub-list for method input_type - 22, // [22:22] is the sub-list for extension type_name - 22, // [22:22] is the sub-list for extension extendee - 0, // [0:22] is the sub-list for field type_name + 12, // 8: SplitJob.transfer_preflight_progress:type_name -> SplitJobBracketProgress + 11, // 9: ListRoutesResponse.routes:type_name -> RouteDescriptor + 11, // 10: SplitRangeResponse.left:type_name -> RouteDescriptor + 11, // 11: SplitRangeResponse.right:type_name -> RouteDescriptor + 4, // 12: CatalogDeltaMutation.op:type_name -> CatalogDeltaMutationOp + 11, // 13: CatalogDeltaMutation.route:type_name -> RouteDescriptor + 21, // 14: CatalogDeltaRecord.mutations:type_name -> CatalogDeltaMutation + 11, // 15: CatalogSnapshotReset.routes:type_name -> RouteDescriptor + 23, // 16: CatalogWatchEvent.snapshot:type_name -> CatalogSnapshotReset + 22, // 17: CatalogWatchEvent.delta:type_name -> CatalogDeltaRecord + 43, // 18: StartSplitMigrationRequest.options:type_name -> StartSplitMigrationRequest.OptionsEntry + 11, // 19: GetRouteOwnershipResponse.route:type_name -> RouteDescriptor + 11, // 20: GetIntersectingRoutesResponse.routes:type_name -> RouteDescriptor + 13, // 21: GetSplitJobResponse.job:type_name -> SplitJob + 13, // 22: ListSplitJobsResponse.jobs:type_name -> SplitJob + 44, // 23: AttestMigrationOwnershipRequest.prior:type_name -> MigrationOwnershipProof + 44, // 24: AttestMigrationOwnershipResponse.ownership:type_name -> MigrationOwnershipProof + 5, // 25: Distribution.GetRoute:input_type -> GetRouteRequest + 7, // 26: Distribution.GetTimestamp:input_type -> GetTimestampRequest + 9, // 27: Distribution.ValidateTimestamp:input_type -> ValidateTimestampRequest + 14, // 28: Distribution.ListRoutes:input_type -> ListRoutesRequest + 16, // 29: Distribution.SplitRange:input_type -> SplitRangeRequest + 18, // 30: Distribution.GetCatalogCapabilities:input_type -> CatalogCapabilitiesRequest + 20, // 31: Distribution.WatchCatalog:input_type -> CatalogWatchRequest + 25, // 32: Distribution.StartSplitMigration:input_type -> StartSplitMigrationRequest + 27, // 33: Distribution.GetSplitMigrationCapability:input_type -> GetSplitMigrationCapabilityRequest + 29, // 34: Distribution.GetRouteOwnership:input_type -> GetRouteOwnershipRequest + 31, // 35: Distribution.GetIntersectingRoutes:input_type -> GetIntersectingRoutesRequest + 33, // 36: Distribution.GetSplitJob:input_type -> GetSplitJobRequest + 35, // 37: Distribution.ListSplitJobs:input_type -> ListSplitJobsRequest + 37, // 38: Distribution.AbandonSplitJob:input_type -> AbandonSplitJobRequest + 39, // 39: Distribution.RetrySplitJob:input_type -> RetrySplitJobRequest + 41, // 40: Distribution.AttestMigrationOwnership:input_type -> AttestMigrationOwnershipRequest + 6, // 41: Distribution.GetRoute:output_type -> GetRouteResponse + 8, // 42: Distribution.GetTimestamp:output_type -> GetTimestampResponse + 10, // 43: Distribution.ValidateTimestamp:output_type -> ValidateTimestampResponse + 15, // 44: Distribution.ListRoutes:output_type -> ListRoutesResponse + 17, // 45: Distribution.SplitRange:output_type -> SplitRangeResponse + 19, // 46: Distribution.GetCatalogCapabilities:output_type -> CatalogCapabilitiesResponse + 24, // 47: Distribution.WatchCatalog:output_type -> CatalogWatchEvent + 26, // 48: Distribution.StartSplitMigration:output_type -> StartSplitMigrationResponse + 28, // 49: Distribution.GetSplitMigrationCapability:output_type -> GetSplitMigrationCapabilityResponse + 30, // 50: Distribution.GetRouteOwnership:output_type -> GetRouteOwnershipResponse + 32, // 51: Distribution.GetIntersectingRoutes:output_type -> GetIntersectingRoutesResponse + 34, // 52: Distribution.GetSplitJob:output_type -> GetSplitJobResponse + 36, // 53: Distribution.ListSplitJobs:output_type -> ListSplitJobsResponse + 38, // 54: Distribution.AbandonSplitJob:output_type -> AbandonSplitJobResponse + 40, // 55: Distribution.RetrySplitJob:output_type -> RetrySplitJobResponse + 42, // 56: Distribution.AttestMigrationOwnership:output_type -> AttestMigrationOwnershipResponse + 41, // [41:57] is the sub-list for method output_type + 25, // [25:41] is the sub-list for method input_type + 25, // [25:25] is the sub-list for extension type_name + 25, // [25:25] is the sub-list for extension extendee + 0, // [0:25] is the sub-list for field type_name } func init() { file_distribution_proto_init() } @@ -2925,6 +3237,7 @@ func file_distribution_proto_init() { if File_distribution_proto != nil { return } + file_internal_proto_init() file_distribution_proto_msgTypes[19].OneofWrappers = []any{ (*CatalogWatchEvent_Snapshot)(nil), (*CatalogWatchEvent_Delta)(nil), @@ -2935,7 +3248,7 @@ func file_distribution_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_distribution_proto_rawDesc), len(file_distribution_proto_rawDesc)), NumEnums: 5, - NumMessages: 35, + NumMessages: 39, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/distribution.proto b/proto/distribution.proto index dc4c7f93d..a60cfc000 100644 --- a/proto/distribution.proto +++ b/proto/distribution.proto @@ -1,5 +1,7 @@ syntax = "proto3"; +import "internal.proto"; + option go_package = "github.com/bootjp/elastickv/proto"; service Distribution { @@ -11,12 +13,15 @@ service Distribution { rpc GetCatalogCapabilities (CatalogCapabilitiesRequest) returns (CatalogCapabilitiesResponse) {} rpc WatchCatalog (CatalogWatchRequest) returns (stream CatalogWatchEvent) {} rpc StartSplitMigration (StartSplitMigrationRequest) returns (StartSplitMigrationResponse) {} + rpc GetSplitMigrationCapability (GetSplitMigrationCapabilityRequest) returns (GetSplitMigrationCapabilityResponse) {} rpc GetRouteOwnership (GetRouteOwnershipRequest) returns (GetRouteOwnershipResponse) {} rpc GetIntersectingRoutes (GetIntersectingRoutesRequest) returns (GetIntersectingRoutesResponse) {} rpc GetSplitJob (GetSplitJobRequest) returns (GetSplitJobResponse) {} rpc ListSplitJobs (ListSplitJobsRequest) returns (ListSplitJobsResponse) {} rpc AbandonSplitJob (AbandonSplitJobRequest) returns (AbandonSplitJobResponse) {} rpc RetrySplitJob (RetrySplitJobRequest) returns (RetrySplitJobResponse) {} + // Privileged peer RPC: reads the committed root descriptor, never a watcher. + rpc AttestMigrationOwnership (AttestMigrationOwnershipRequest) returns (AttestMigrationOwnershipResponse) {} } message GetRouteRequest { @@ -178,6 +183,19 @@ message SplitJob { int64 started_at_ms = 31; int64 updated_at_ms = 32; int64 terminal_at_ms = 33; + bool capability_regressed = 34; + // Group that still owns the source side of the split. Recorded at plan + // time because the route shape it can otherwise be recovered from + // disappears once the source child is itself split. + uint64 source_group_id = 35; + // Version 1 preflight checks every transfer family and relevant MVCC version. + // Missing fields on legacy jobs never attest that this scan completed. + uint32 transfer_preflight_version = 36; + uint64 transfer_preflight_ts = 37; + repeated SplitJobBracketProgress transfer_preflight_progress = 38; + bool transfer_preflight_done = 39; + // Durable moving child created with its same-owner partition manifest. + uint64 working_source_route_id = 40; } message ListRoutesRequest {} @@ -257,6 +275,13 @@ message StartSplitMigrationResponse { uint64 job_id = 2; } +message GetSplitMigrationCapabilityRequest {} + +message GetSplitMigrationCapabilityResponse { + bool migration_capable = 1; + repeated string capabilities = 2; +} + message GetRouteOwnershipRequest { bytes key = 1; uint64 catalog_version = 2; @@ -309,3 +334,16 @@ message RetrySplitJobRequest { } message RetrySplitJobResponse {} + +message AttestMigrationOwnershipRequest { + uint64 job_id = 1; + MigrationOwnershipProof prior = 2; + bytes prior_partition = 3; + bytes prior_ownership_state = 4; +} + +message AttestMigrationOwnershipResponse { + uint32 ownership_protocol = 1; + MigrationOwnershipProof ownership = 2; + bytes partition_transition = 3; +} diff --git a/proto/distribution_grpc.pb.go b/proto/distribution_grpc.pb.go index 0e650c806..17954e1d2 100644 --- a/proto/distribution_grpc.pb.go +++ b/proto/distribution_grpc.pb.go @@ -19,20 +19,22 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Distribution_GetRoute_FullMethodName = "/Distribution/GetRoute" - Distribution_GetTimestamp_FullMethodName = "/Distribution/GetTimestamp" - Distribution_ValidateTimestamp_FullMethodName = "/Distribution/ValidateTimestamp" - Distribution_ListRoutes_FullMethodName = "/Distribution/ListRoutes" - Distribution_SplitRange_FullMethodName = "/Distribution/SplitRange" - Distribution_GetCatalogCapabilities_FullMethodName = "/Distribution/GetCatalogCapabilities" - Distribution_WatchCatalog_FullMethodName = "/Distribution/WatchCatalog" - Distribution_StartSplitMigration_FullMethodName = "/Distribution/StartSplitMigration" - Distribution_GetRouteOwnership_FullMethodName = "/Distribution/GetRouteOwnership" - Distribution_GetIntersectingRoutes_FullMethodName = "/Distribution/GetIntersectingRoutes" - Distribution_GetSplitJob_FullMethodName = "/Distribution/GetSplitJob" - Distribution_ListSplitJobs_FullMethodName = "/Distribution/ListSplitJobs" - Distribution_AbandonSplitJob_FullMethodName = "/Distribution/AbandonSplitJob" - Distribution_RetrySplitJob_FullMethodName = "/Distribution/RetrySplitJob" + Distribution_GetRoute_FullMethodName = "/Distribution/GetRoute" + Distribution_GetTimestamp_FullMethodName = "/Distribution/GetTimestamp" + Distribution_ValidateTimestamp_FullMethodName = "/Distribution/ValidateTimestamp" + Distribution_ListRoutes_FullMethodName = "/Distribution/ListRoutes" + Distribution_SplitRange_FullMethodName = "/Distribution/SplitRange" + Distribution_GetCatalogCapabilities_FullMethodName = "/Distribution/GetCatalogCapabilities" + Distribution_WatchCatalog_FullMethodName = "/Distribution/WatchCatalog" + Distribution_StartSplitMigration_FullMethodName = "/Distribution/StartSplitMigration" + Distribution_GetSplitMigrationCapability_FullMethodName = "/Distribution/GetSplitMigrationCapability" + Distribution_GetRouteOwnership_FullMethodName = "/Distribution/GetRouteOwnership" + Distribution_GetIntersectingRoutes_FullMethodName = "/Distribution/GetIntersectingRoutes" + Distribution_GetSplitJob_FullMethodName = "/Distribution/GetSplitJob" + Distribution_ListSplitJobs_FullMethodName = "/Distribution/ListSplitJobs" + Distribution_AbandonSplitJob_FullMethodName = "/Distribution/AbandonSplitJob" + Distribution_RetrySplitJob_FullMethodName = "/Distribution/RetrySplitJob" + Distribution_AttestMigrationOwnership_FullMethodName = "/Distribution/AttestMigrationOwnership" ) // DistributionClient is the client API for Distribution service. @@ -47,12 +49,15 @@ type DistributionClient interface { GetCatalogCapabilities(ctx context.Context, in *CatalogCapabilitiesRequest, opts ...grpc.CallOption) (*CatalogCapabilitiesResponse, error) WatchCatalog(ctx context.Context, in *CatalogWatchRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[CatalogWatchEvent], error) StartSplitMigration(ctx context.Context, in *StartSplitMigrationRequest, opts ...grpc.CallOption) (*StartSplitMigrationResponse, error) + GetSplitMigrationCapability(ctx context.Context, in *GetSplitMigrationCapabilityRequest, opts ...grpc.CallOption) (*GetSplitMigrationCapabilityResponse, error) GetRouteOwnership(ctx context.Context, in *GetRouteOwnershipRequest, opts ...grpc.CallOption) (*GetRouteOwnershipResponse, error) GetIntersectingRoutes(ctx context.Context, in *GetIntersectingRoutesRequest, opts ...grpc.CallOption) (*GetIntersectingRoutesResponse, error) GetSplitJob(ctx context.Context, in *GetSplitJobRequest, opts ...grpc.CallOption) (*GetSplitJobResponse, error) ListSplitJobs(ctx context.Context, in *ListSplitJobsRequest, opts ...grpc.CallOption) (*ListSplitJobsResponse, error) AbandonSplitJob(ctx context.Context, in *AbandonSplitJobRequest, opts ...grpc.CallOption) (*AbandonSplitJobResponse, error) RetrySplitJob(ctx context.Context, in *RetrySplitJobRequest, opts ...grpc.CallOption) (*RetrySplitJobResponse, error) + // Privileged peer RPC: reads the committed root descriptor, never a watcher. + AttestMigrationOwnership(ctx context.Context, in *AttestMigrationOwnershipRequest, opts ...grpc.CallOption) (*AttestMigrationOwnershipResponse, error) } type distributionClient struct { @@ -152,6 +157,16 @@ func (c *distributionClient) StartSplitMigration(ctx context.Context, in *StartS return out, nil } +func (c *distributionClient) GetSplitMigrationCapability(ctx context.Context, in *GetSplitMigrationCapabilityRequest, opts ...grpc.CallOption) (*GetSplitMigrationCapabilityResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetSplitMigrationCapabilityResponse) + err := c.cc.Invoke(ctx, Distribution_GetSplitMigrationCapability_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *distributionClient) GetRouteOwnership(ctx context.Context, in *GetRouteOwnershipRequest, opts ...grpc.CallOption) (*GetRouteOwnershipResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetRouteOwnershipResponse) @@ -212,6 +227,16 @@ func (c *distributionClient) RetrySplitJob(ctx context.Context, in *RetrySplitJo return out, nil } +func (c *distributionClient) AttestMigrationOwnership(ctx context.Context, in *AttestMigrationOwnershipRequest, opts ...grpc.CallOption) (*AttestMigrationOwnershipResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(AttestMigrationOwnershipResponse) + err := c.cc.Invoke(ctx, Distribution_AttestMigrationOwnership_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // DistributionServer is the server API for Distribution service. // All implementations must embed UnimplementedDistributionServer // for forward compatibility. @@ -224,12 +249,15 @@ type DistributionServer interface { GetCatalogCapabilities(context.Context, *CatalogCapabilitiesRequest) (*CatalogCapabilitiesResponse, error) WatchCatalog(*CatalogWatchRequest, grpc.ServerStreamingServer[CatalogWatchEvent]) error StartSplitMigration(context.Context, *StartSplitMigrationRequest) (*StartSplitMigrationResponse, error) + GetSplitMigrationCapability(context.Context, *GetSplitMigrationCapabilityRequest) (*GetSplitMigrationCapabilityResponse, error) GetRouteOwnership(context.Context, *GetRouteOwnershipRequest) (*GetRouteOwnershipResponse, error) GetIntersectingRoutes(context.Context, *GetIntersectingRoutesRequest) (*GetIntersectingRoutesResponse, error) GetSplitJob(context.Context, *GetSplitJobRequest) (*GetSplitJobResponse, error) ListSplitJobs(context.Context, *ListSplitJobsRequest) (*ListSplitJobsResponse, error) AbandonSplitJob(context.Context, *AbandonSplitJobRequest) (*AbandonSplitJobResponse, error) RetrySplitJob(context.Context, *RetrySplitJobRequest) (*RetrySplitJobResponse, error) + // Privileged peer RPC: reads the committed root descriptor, never a watcher. + AttestMigrationOwnership(context.Context, *AttestMigrationOwnershipRequest) (*AttestMigrationOwnershipResponse, error) mustEmbedUnimplementedDistributionServer() } @@ -264,6 +292,9 @@ func (UnimplementedDistributionServer) WatchCatalog(*CatalogWatchRequest, grpc.S func (UnimplementedDistributionServer) StartSplitMigration(context.Context, *StartSplitMigrationRequest) (*StartSplitMigrationResponse, error) { return nil, status.Error(codes.Unimplemented, "method StartSplitMigration not implemented") } +func (UnimplementedDistributionServer) GetSplitMigrationCapability(context.Context, *GetSplitMigrationCapabilityRequest) (*GetSplitMigrationCapabilityResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetSplitMigrationCapability not implemented") +} func (UnimplementedDistributionServer) GetRouteOwnership(context.Context, *GetRouteOwnershipRequest) (*GetRouteOwnershipResponse, error) { return nil, status.Error(codes.Unimplemented, "method GetRouteOwnership not implemented") } @@ -282,6 +313,9 @@ func (UnimplementedDistributionServer) AbandonSplitJob(context.Context, *Abandon func (UnimplementedDistributionServer) RetrySplitJob(context.Context, *RetrySplitJobRequest) (*RetrySplitJobResponse, error) { return nil, status.Error(codes.Unimplemented, "method RetrySplitJob not implemented") } +func (UnimplementedDistributionServer) AttestMigrationOwnership(context.Context, *AttestMigrationOwnershipRequest) (*AttestMigrationOwnershipResponse, error) { + return nil, status.Error(codes.Unimplemented, "method AttestMigrationOwnership not implemented") +} func (UnimplementedDistributionServer) mustEmbedUnimplementedDistributionServer() {} func (UnimplementedDistributionServer) testEmbeddedByValue() {} @@ -440,6 +474,24 @@ func _Distribution_StartSplitMigration_Handler(srv interface{}, ctx context.Cont return interceptor(ctx, in, info, handler) } +func _Distribution_GetSplitMigrationCapability_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetSplitMigrationCapabilityRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DistributionServer).GetSplitMigrationCapability(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Distribution_GetSplitMigrationCapability_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DistributionServer).GetSplitMigrationCapability(ctx, req.(*GetSplitMigrationCapabilityRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Distribution_GetRouteOwnership_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetRouteOwnershipRequest) if err := dec(in); err != nil { @@ -548,6 +600,24 @@ func _Distribution_RetrySplitJob_Handler(srv interface{}, ctx context.Context, d return interceptor(ctx, in, info, handler) } +func _Distribution_AttestMigrationOwnership_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AttestMigrationOwnershipRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DistributionServer).AttestMigrationOwnership(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Distribution_AttestMigrationOwnership_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DistributionServer).AttestMigrationOwnership(ctx, req.(*AttestMigrationOwnershipRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Distribution_ServiceDesc is the grpc.ServiceDesc for Distribution service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -583,6 +653,10 @@ var Distribution_ServiceDesc = grpc.ServiceDesc{ MethodName: "StartSplitMigration", Handler: _Distribution_StartSplitMigration_Handler, }, + { + MethodName: "GetSplitMigrationCapability", + Handler: _Distribution_GetSplitMigrationCapability_Handler, + }, { MethodName: "GetRouteOwnership", Handler: _Distribution_GetRouteOwnership_Handler, @@ -607,6 +681,10 @@ var Distribution_ServiceDesc = grpc.ServiceDesc{ MethodName: "RetrySplitJob", Handler: _Distribution_RetrySplitJob_Handler, }, + { + MethodName: "AttestMigrationOwnership", + Handler: _Distribution_AttestMigrationOwnership_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 0ec15cd3e..133b9a9f3 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -128,6 +128,199 @@ func (Phase) EnumDescriptor() ([]byte, []int) { return file_internal_proto_rawDescGZIP(), []int{1} } +type MigrationCleanupMode int32 + +const ( + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_UNSPECIFIED MigrationCleanupMode = 0 + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS MigrationCleanupMode = 1 + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA MigrationCleanupMode = 2 + // New modes deliberately fail on old runtimes instead of letting unknown + // proof fields silently degrade to an unbound metadata clear. + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_ACQUIRE MigrationCleanupMode = 3 + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR MigrationCleanupMode = 4 + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH MigrationCleanupMode = 5 + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_SOURCE_OWNERSHIP_ACQUIRE MigrationCleanupMode = 6 + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION MigrationCleanupMode = 7 + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION MigrationCleanupMode = 8 +) + +// Enum value maps for MigrationCleanupMode. +var ( + MigrationCleanupMode_name = map[int32]string{ + 0: "MIGRATION_CLEANUP_MODE_UNSPECIFIED", + 1: "MIGRATION_CLEANUP_MODE_VERSIONS", + 2: "MIGRATION_CLEANUP_MODE_METADATA", + 3: "MIGRATION_CLEANUP_MODE_OWNERSHIP_ACQUIRE", + 4: "MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR", + 5: "MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH", + 6: "MIGRATION_CLEANUP_MODE_SOURCE_OWNERSHIP_ACQUIRE", + 7: "MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION", + 8: "MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION", + } + MigrationCleanupMode_value = map[string]int32{ + "MIGRATION_CLEANUP_MODE_UNSPECIFIED": 0, + "MIGRATION_CLEANUP_MODE_VERSIONS": 1, + "MIGRATION_CLEANUP_MODE_METADATA": 2, + "MIGRATION_CLEANUP_MODE_OWNERSHIP_ACQUIRE": 3, + "MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR": 4, + "MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH": 5, + "MIGRATION_CLEANUP_MODE_SOURCE_OWNERSHIP_ACQUIRE": 6, + "MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION": 7, + "MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION": 8, + } +) + +func (x MigrationCleanupMode) Enum() *MigrationCleanupMode { + p := new(MigrationCleanupMode) + *p = x + return p +} + +func (x MigrationCleanupMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MigrationCleanupMode) Descriptor() protoreflect.EnumDescriptor { + return file_internal_proto_enumTypes[2].Descriptor() +} + +func (MigrationCleanupMode) Type() protoreflect.EnumType { + return &file_internal_proto_enumTypes[2] +} + +func (x MigrationCleanupMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MigrationCleanupMode.Descriptor instead. +func (MigrationCleanupMode) EnumDescriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{2} +} + +type MigrationOwnershipRole int32 + +const ( + MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_UNSPECIFIED MigrationOwnershipRole = 0 + MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_OWNER MigrationOwnershipRole = 1 + MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_NONOWNER MigrationOwnershipRole = 2 +) + +// Enum value maps for MigrationOwnershipRole. +var ( + MigrationOwnershipRole_name = map[int32]string{ + 0: "MIGRATION_OWNERSHIP_ROLE_UNSPECIFIED", + 1: "MIGRATION_OWNERSHIP_ROLE_OWNER", + 2: "MIGRATION_OWNERSHIP_ROLE_NONOWNER", + } + MigrationOwnershipRole_value = map[string]int32{ + "MIGRATION_OWNERSHIP_ROLE_UNSPECIFIED": 0, + "MIGRATION_OWNERSHIP_ROLE_OWNER": 1, + "MIGRATION_OWNERSHIP_ROLE_NONOWNER": 2, + } +) + +func (x MigrationOwnershipRole) Enum() *MigrationOwnershipRole { + p := new(MigrationOwnershipRole) + *p = x + return p +} + +func (x MigrationOwnershipRole) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MigrationOwnershipRole) Descriptor() protoreflect.EnumDescriptor { + return file_internal_proto_enumTypes[3].Descriptor() +} + +func (MigrationOwnershipRole) Type() protoreflect.EnumType { + return &file_internal_proto_enumTypes[3] +} + +func (x MigrationOwnershipRole) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MigrationOwnershipRole.Descriptor instead. +func (MigrationOwnershipRole) EnumDescriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{3} +} + +type MigrationStateProbeKind int32 + +const ( + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_UNSPECIFIED MigrationStateProbeKind = 0 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED MigrationStateProbeKind = 1 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED MigrationStateProbeKind = 2 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED MigrationStateProbeKind = 3 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED MigrationStateProbeKind = 4 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED MigrationStateProbeKind = 5 + // Identifies the durable source guard by job and moving range, independently + // of compactable catalog route history. Target-only readiness is excluded. + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_GUARD_PRESENT MigrationStateProbeKind = 6 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY MigrationStateProbeKind = 7 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED MigrationStateProbeKind = 8 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_PARTITION_APPLIED MigrationStateProbeKind = 9 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_NAMESPACE_ADMISSION MigrationStateProbeKind = 10 +) + +// Enum value maps for MigrationStateProbeKind. +var ( + MigrationStateProbeKind_name = map[int32]string{ + 0: "MIGRATION_STATE_PROBE_KIND_UNSPECIFIED", + 1: "MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED", + 2: "MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED", + 3: "MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED", + 4: "MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED", + 5: "MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED", + 6: "MIGRATION_STATE_PROBE_KIND_SOURCE_GUARD_PRESENT", + 7: "MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY", + 8: "MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED", + 9: "MIGRATION_STATE_PROBE_KIND_PARTITION_APPLIED", + 10: "MIGRATION_STATE_PROBE_KIND_NAMESPACE_ADMISSION", + } + MigrationStateProbeKind_value = map[string]int32{ + "MIGRATION_STATE_PROBE_KIND_UNSPECIFIED": 0, + "MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED": 1, + "MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED": 2, + "MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED": 3, + "MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED": 4, + "MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED": 5, + "MIGRATION_STATE_PROBE_KIND_SOURCE_GUARD_PRESENT": 6, + "MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY": 7, + "MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED": 8, + "MIGRATION_STATE_PROBE_KIND_PARTITION_APPLIED": 9, + "MIGRATION_STATE_PROBE_KIND_NAMESPACE_ADMISSION": 10, + } +) + +func (x MigrationStateProbeKind) Enum() *MigrationStateProbeKind { + p := new(MigrationStateProbeKind) + *p = x + return p +} + +func (x MigrationStateProbeKind) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MigrationStateProbeKind) Descriptor() protoreflect.EnumDescriptor { + return file_internal_proto_enumTypes[4].Descriptor() +} + +func (MigrationStateProbeKind) Type() protoreflect.EnumType { + return &file_internal_proto_enumTypes[4] +} + +func (x MigrationStateProbeKind) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MigrationStateProbeKind.Descriptor instead. +func (MigrationStateProbeKind) EnumDescriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{4} +} + type Mutation struct { state protoimpl.MessageState `protogen:"open.v1"` Op Op `protobuf:"varint,1,opt,name=op,proto3,enum=Op" json:"op,omitempty"` @@ -1254,8 +1447,13 @@ func (x *ImportRangeVersionsRequest) GetBatchSeq() uint64 { } type ImportRangeVersionsResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - AckedCursor []byte `protobuf:"bytes,1,opt,name=acked_cursor,json=ackedCursor,proto3" json:"acked_cursor,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + AckedCursor []byte `protobuf:"bytes,1,opt,name=acked_cursor,json=ackedCursor,proto3" json:"acked_cursor,omitempty"` + // Set when the target already held a durable acknowledgement for this + // batch_seq and discarded the resent rows. acked_cursor is then the cursor + // the target recorded for that batch, which need not be the cursor the + // caller just sent. + Duplicate bool `protobuf:"varint,2,opt,name=duplicate,proto3" json:"duplicate,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1297,6 +1495,13 @@ func (x *ImportRangeVersionsResponse) GetAckedCursor() []byte { return nil } +func (x *ImportRangeVersionsResponse) GetDuplicate() bool { + if x != nil { + return x.Duplicate + } + return false +} + type PromoteStagedVersionsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` JobId uint64 `protobuf:"varint,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` @@ -1441,111 +1646,1395 @@ func (x *PromoteStagedVersionsResponse) GetMaxPromotedTs() uint64 { return 0 } -var File_internal_proto protoreflect.FileDescriptor +type TargetStagedReadinessRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId uint64 `protobuf:"varint,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + RouteStart []byte `protobuf:"bytes,2,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` + RouteEnd []byte `protobuf:"bytes,3,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` + ExpectedCutoverVersion uint64 `protobuf:"varint,4,opt,name=expected_cutover_version,json=expectedCutoverVersion,proto3" json:"expected_cutover_version,omitempty"` + MigrationJobId uint64 `protobuf:"varint,5,opt,name=migration_job_id,json=migrationJobId,proto3" json:"migration_job_id,omitempty"` + MinWriteTsExclusive uint64 `protobuf:"varint,6,opt,name=min_write_ts_exclusive,json=minWriteTsExclusive,proto3" json:"min_write_ts_exclusive,omitempty"` + Armed bool `protobuf:"varint,7,opt,name=armed,proto3" json:"armed,omitempty"` + // Source-side controls reuse the same Raft-replicated durable record. They + // are ignored by target staged-readiness checks. + SourceWriteFence bool `protobuf:"varint,8,opt,name=source_write_fence,json=sourceWriteFence,proto3" json:"source_write_fence,omitempty"` + SourceReadFence bool `protobuf:"varint,9,opt,name=source_read_fence,json=sourceReadFence,proto3" json:"source_read_fence,omitempty"` + RetentionPinTs uint64 `protobuf:"varint,10,opt,name=retention_pin_ts,json=retentionPinTs,proto3" json:"retention_pin_ts,omitempty"` + TrackWrites bool `protobuf:"varint,11,opt,name=track_writes,json=trackWrites,proto3" json:"track_writes,omitempty"` + MinAdmittedTs uint64 `protobuf:"varint,12,opt,name=min_admitted_ts,json=minAdmittedTs,proto3" json:"min_admitted_ts,omitempty"` + // Explicit opt-in; zero preserves legacy tracker write admission. + TransferSizeGuardVersion uint32 `protobuf:"varint,13,opt,name=transfer_size_guard_version,json=transferSizeGuardVersion,proto3" json:"transfer_size_guard_version,omitempty"` + RequireConfigurationIndex bool `protobuf:"varint,14,opt,name=require_configuration_index,json=requireConfigurationIndex,proto3" json:"require_configuration_index,omitempty"` + ExpectedConfigurationIndex uint64 `protobuf:"varint,15,opt,name=expected_configuration_index,json=expectedConfigurationIndex,proto3" json:"expected_configuration_index,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TargetStagedReadinessRequest) Reset() { + *x = TargetStagedReadinessRequest{} + mi := &file_internal_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} -const file_internal_proto_rawDesc = "" + - "\n" + - "\x0einternal.proto\"|\n" + - "\bMutation\x12\x13\n" + - "\x02op\x18\x01 \x01(\x0e2\x03.OpR\x02op\x12\x10\n" + - "\x03key\x18\x02 \x01(\fR\x03key\x12\x14\n" + - "\x05value\x18\x03 \x01(\fR\x05value\x123\n" + - "\x16commit_ts_value_offset\x18\x04 \x01(\x04R\x13commitTsValueOffset\"\xbd\x02\n" + - "\aRequest\x12\x15\n" + - "\x06is_txn\x18\x01 \x01(\bR\x05isTxn\x12\x1c\n" + - "\x05phase\x18\x02 \x01(\x0e2\x06.PhaseR\x05phase\x12\x0e\n" + - "\x02ts\x18\x03 \x01(\x04R\x02ts\x12'\n" + - "\tmutations\x18\x04 \x03(\v2\t.MutationR\tmutations\x12\x1b\n" + - "\tread_keys\x18\x05 \x03(\fR\breadKeys\x124\n" + - "\x16observed_route_version\x18\x06 \x01(\x04R\x14observedRouteVersion\x125\n" + - "\x17write_fence_bypass_keys\x18\a \x03(\fR\x14writeFenceBypassKeys\x12:\n" + - "\x19timestamp_admission_floor\x18\b \x01(\x04R\x17timestampAdmissionFloor\"3\n" + - "\vRaftCommand\x12$\n" + - "\brequests\x18\x01 \x03(\v2\b.RequestR\brequests\"M\n" + - "\x0eForwardRequest\x12\x15\n" + - "\x06is_txn\x18\x01 \x01(\bR\x05isTxn\x12$\n" + - "\brequests\x18\x02 \x03(\v2\b.RequestR\brequests\"k\n" + - "\x0fForwardResponse\x12\x18\n" + - "\asuccess\x18\x01 \x01(\bR\asuccess\x12!\n" + - "\fcommit_index\x18\x02 \x01(\x04R\vcommitIndex\x12\x1b\n" + - "\tcommit_ts\x18\x03 \x01(\x04R\bcommitTs\"7\n" + - "\x1bForwardAdminProposalRequest\x12\x18\n" + - "\apayload\x18\x01 \x01(\fR\apayload\"A\n" + - "\x1cForwardAdminProposalResponse\x12!\n" + - "\fcommit_index\x18\x01 \x01(\x04R\vcommitIndex\"\x19\n" + - "\x17ForwardLeaseReadRequest\"e\n" + - "\x18ForwardLeaseReadResponse\x12#\n" + - "\rapplied_index\x18\x01 \x01(\x04R\fappliedIndex\x12$\n" + - "\x0elast_commit_ts\x18\x02 \x01(\x04R\flastCommitTs\"\\\n" + - "\x1bPrepareEvidenceStateRequest\x12\x14\n" + - "\x05floor\x18\x01 \x01(\x04R\x05floor\x12'\n" + - "\x0fcapability_only\x18\x02 \x01(\bR\x0ecapabilityOnly\"\xb1\x02\n" + - "\x1cPrepareEvidenceStateResponse\x12\x1c\n" + - "\tsupported\x18\x01 \x01(\bR\tsupported\x12#\n" + - "\rproof_enabled\x18\x02 \x01(\bR\fproofEnabled\x12%\n" + - "\x0eunproven_locks\x18\x03 \x01(\bR\runprovenLocks\x12\x1f\n" + - "\vfence_floor\x18\x04 \x01(\x04R\n" + - "fenceFloor\x12#\n" + - "\rapplied_index\x18\x05 \x01(\x04R\fappliedIndex\x12'\n" + - "\x0fcutover_applied\x18\x06 \x01(\bR\x0ecutoverApplied\x128\n" + - "\x18dedicated_tso_configured\x18\a \x01(\bR\x16dedicatedTsoConfigured\"I\n" + - "\x13RelayPublishRequest\x12\x18\n" + - "\achannel\x18\x01 \x01(\fR\achannel\x12\x18\n" + - "\amessage\x18\x02 \x01(\fR\amessage\"8\n" + - "\x14RelayPublishResponse\x12 \n" + - "\vsubscribers\x18\x01 \x01(\x03R\vsubscribers\"\xc5\x03\n" + - "\x1aExportRangeVersionsRequest\x12\x1f\n" + - "\vrange_start\x18\x01 \x01(\fR\n" + - "rangeStart\x12\x1b\n" + - "\trange_end\x18\x02 \x01(\fR\brangeEnd\x12\"\n" + - "\rmax_commit_ts\x18\x03 \x01(\x04R\vmaxCommitTs\x12\"\n" + - "\rmin_commit_ts\x18\x04 \x01(\x04R\vminCommitTs\x12\x16\n" + - "\x06cursor\x18\x05 \x01(\fR\x06cursor\x12\x1f\n" + - "\vchunk_bytes\x18\x06 \x01(\rR\n" + - "chunkBytes\x12\x1f\n" + - "\vroute_start\x18\a \x01(\fR\n" + - "routeStart\x12\x1b\n" + - "\troute_end\x18\b \x01(\fR\brouteEnd\x12*\n" + - "\x11max_scanned_bytes\x18\t \x01(\x04R\x0fmaxScannedBytes\x12\x1d\n" + - "\n" + - "key_family\x18\n" + - " \x01(\rR\tkeyFamily\x124\n" + - "\x16exclude_known_internal\x18\v \x01(\bR\x14excludeKnownInternal\x12)\n" + - "\x10exclude_prefixes\x18\f \x03(\fR\x0fexcludePrefixes\"|\n" + - "\x1bExportRangeVersionsResponse\x12(\n" + - "\bversions\x18\x01 \x03(\v2\f.MVCCVersionR\bversions\x12\x1f\n" + - "\vnext_cursor\x18\x02 \x01(\fR\n" + - "nextCursor\x12\x12\n" + - "\x04done\x18\x03 \x01(\bR\x04done\"\xac\x01\n" + - "\vMVCCVersion\x12\x10\n" + - "\x03key\x18\x01 \x01(\fR\x03key\x12\x1b\n" + - "\tcommit_ts\x18\x02 \x01(\x04R\bcommitTs\x12\x1c\n" + - "\ttombstone\x18\x03 \x01(\bR\ttombstone\x12\x14\n" + - "\x05value\x18\x04 \x01(\fR\x05value\x12\x1d\n" + - "\n" + - "key_family\x18\x05 \x01(\rR\tkeyFamily\x12\x1b\n" + - "\texpire_at\x18\x06 \x01(\x04R\bexpireAt\"\xb1\x01\n" + - "\x1aImportRangeVersionsRequest\x12\x15\n" + - "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12(\n" + - "\bversions\x18\x02 \x03(\v2\f.MVCCVersionR\bversions\x12\x16\n" + - "\x06cursor\x18\x03 \x01(\fR\x06cursor\x12\x1d\n" + - "\n" + - "bracket_id\x18\x04 \x01(\x04R\tbracketId\x12\x1b\n" + - "\tbatch_seq\x18\x05 \x01(\x04R\bbatchSeq\"@\n" + - "\x1bImportRangeVersionsResponse\x12!\n" + - "\facked_cursor\x18\x01 \x01(\fR\vackedCursor\"\xb9\x01\n" + - "\x1cPromoteStagedVersionsRequest\x12\x15\n" + - "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12\x16\n" + - "\x06cursor\x18\x02 \x01(\fR\x06cursor\x12!\n" + - "\fmax_versions\x18\x03 \x01(\rR\vmaxVersions\x12\x1b\n" + - "\tmax_bytes\x18\x04 \x01(\x04R\bmaxBytes\x12*\n" + - "\x11max_scanned_bytes\x18\x05 \x01(\x04R\x0fmaxScannedBytes\"\xa1\x01\n" + - "\x1dPromoteStagedVersionsResponse\x12\x1f\n" + - "\vnext_cursor\x18\x01 \x01(\fR\n" + - "nextCursor\x12\x12\n" + - "\x04done\x18\x02 \x01(\bR\x04done\x12#\n" + - "\rpromoted_rows\x18\x03 \x01(\x04R\fpromotedRows\x12&\n" + - "\x0fmax_promoted_ts\x18\x04 \x01(\x04R\rmaxPromotedTs*&\n" + +func (x *TargetStagedReadinessRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TargetStagedReadinessRequest) ProtoMessage() {} + +func (x *TargetStagedReadinessRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TargetStagedReadinessRequest.ProtoReflect.Descriptor instead. +func (*TargetStagedReadinessRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{20} +} + +func (x *TargetStagedReadinessRequest) GetJobId() uint64 { + if x != nil { + return x.JobId + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetRouteStart() []byte { + if x != nil { + return x.RouteStart + } + return nil +} + +func (x *TargetStagedReadinessRequest) GetRouteEnd() []byte { + if x != nil { + return x.RouteEnd + } + return nil +} + +func (x *TargetStagedReadinessRequest) GetExpectedCutoverVersion() uint64 { + if x != nil { + return x.ExpectedCutoverVersion + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetMigrationJobId() uint64 { + if x != nil { + return x.MigrationJobId + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetMinWriteTsExclusive() uint64 { + if x != nil { + return x.MinWriteTsExclusive + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetArmed() bool { + if x != nil { + return x.Armed + } + return false +} + +func (x *TargetStagedReadinessRequest) GetSourceWriteFence() bool { + if x != nil { + return x.SourceWriteFence + } + return false +} + +func (x *TargetStagedReadinessRequest) GetSourceReadFence() bool { + if x != nil { + return x.SourceReadFence + } + return false +} + +func (x *TargetStagedReadinessRequest) GetRetentionPinTs() uint64 { + if x != nil { + return x.RetentionPinTs + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetTrackWrites() bool { + if x != nil { + return x.TrackWrites + } + return false +} + +func (x *TargetStagedReadinessRequest) GetMinAdmittedTs() uint64 { + if x != nil { + return x.MinAdmittedTs + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetTransferSizeGuardVersion() uint32 { + if x != nil { + return x.TransferSizeGuardVersion + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetRequireConfigurationIndex() bool { + if x != nil { + return x.RequireConfigurationIndex + } + return false +} + +func (x *TargetStagedReadinessRequest) GetExpectedConfigurationIndex() uint64 { + if x != nil { + return x.ExpectedConfigurationIndex + } + return 0 +} + +type TargetStagedReadinessResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + MinAdmittedTs uint64 `protobuf:"varint,1,opt,name=min_admitted_ts,json=minAdmittedTs,proto3" json:"min_admitted_ts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TargetStagedReadinessResponse) Reset() { + *x = TargetStagedReadinessResponse{} + mi := &file_internal_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TargetStagedReadinessResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TargetStagedReadinessResponse) ProtoMessage() {} + +func (x *TargetStagedReadinessResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TargetStagedReadinessResponse.ProtoReflect.Descriptor instead. +func (*TargetStagedReadinessResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{21} +} + +func (x *TargetStagedReadinessResponse) GetMinAdmittedTs() uint64 { + if x != nil { + return x.MinAdmittedTs + } + return 0 +} + +type ProbeMigrationLocksRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RouteStart []byte `protobuf:"bytes,1,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` + RouteEnd []byte `protobuf:"bytes,2,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` + Limit uint32 `protobuf:"varint,3,opt,name=limit,proto3" json:"limit,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProbeMigrationLocksRequest) Reset() { + *x = ProbeMigrationLocksRequest{} + mi := &file_internal_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProbeMigrationLocksRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeMigrationLocksRequest) ProtoMessage() {} + +func (x *ProbeMigrationLocksRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeMigrationLocksRequest.ProtoReflect.Descriptor instead. +func (*ProbeMigrationLocksRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{22} +} + +func (x *ProbeMigrationLocksRequest) GetRouteStart() []byte { + if x != nil { + return x.RouteStart + } + return nil +} + +func (x *ProbeMigrationLocksRequest) GetRouteEnd() []byte { + if x != nil { + return x.RouteEnd + } + return nil +} + +func (x *ProbeMigrationLocksRequest) GetLimit() uint32 { + if x != nil { + return x.Limit + } + return 0 +} + +type ProbeMigrationLocksResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + PendingCount uint32 `protobuf:"varint,1,opt,name=pending_count,json=pendingCount,proto3" json:"pending_count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProbeMigrationLocksResponse) Reset() { + *x = ProbeMigrationLocksResponse{} + mi := &file_internal_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProbeMigrationLocksResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeMigrationLocksResponse) ProtoMessage() {} + +func (x *ProbeMigrationLocksResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeMigrationLocksResponse.ProtoReflect.Descriptor instead. +func (*ProbeMigrationLocksResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{23} +} + +func (x *ProbeMigrationLocksResponse) GetPendingCount() uint32 { + if x != nil { + return x.PendingCount + } + return 0 +} + +type MigrationOwnershipProof struct { + state protoimpl.MessageState `protogen:"open.v1"` + Generation string `protobuf:"bytes,1,opt,name=generation,proto3" json:"generation,omitempty"` + LocalGroupId uint64 `protobuf:"varint,2,opt,name=local_group_id,json=localGroupId,proto3" json:"local_group_id,omitempty"` + OwnerGroupId uint64 `protobuf:"varint,3,opt,name=owner_group_id,json=ownerGroupId,proto3" json:"owner_group_id,omitempty"` + CatalogVersion uint64 `protobuf:"varint,4,opt,name=catalog_version,json=catalogVersion,proto3" json:"catalog_version,omitempty"` + MinWriteTsExclusive uint64 `protobuf:"varint,5,opt,name=min_write_ts_exclusive,json=minWriteTsExclusive,proto3" json:"min_write_ts_exclusive,omitempty"` + RouteStart []byte `protobuf:"bytes,6,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` + RouteEnd []byte `protobuf:"bytes,7,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` + RouteId uint64 `protobuf:"varint,8,opt,name=route_id,json=routeId,proto3" json:"route_id,omitempty"` + ParentRouteId uint64 `protobuf:"varint,9,opt,name=parent_route_id,json=parentRouteId,proto3" json:"parent_route_id,omitempty"` + Staged bool `protobuf:"varint,10,opt,name=staged,proto3" json:"staged,omitempty"` + Role MigrationOwnershipRole `protobuf:"varint,11,opt,name=role,proto3,enum=MigrationOwnershipRole" json:"role,omitempty"` + DescriptorHash []byte `protobuf:"bytes,12,opt,name=descriptor_hash,json=descriptorHash,proto3" json:"descriptor_hash,omitempty"` + WriteFloorKnown bool `protobuf:"varint,13,opt,name=write_floor_known,json=writeFloorKnown,proto3" json:"write_floor_known,omitempty"` + SourceTrackerVersion uint64 `protobuf:"varint,14,opt,name=source_tracker_version,json=sourceTrackerVersion,proto3" json:"source_tracker_version,omitempty"` + SourceAppliedIndex uint64 `protobuf:"varint,15,opt,name=source_applied_index,json=sourceAppliedIndex,proto3" json:"source_applied_index,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MigrationOwnershipProof) Reset() { + *x = MigrationOwnershipProof{} + mi := &file_internal_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MigrationOwnershipProof) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MigrationOwnershipProof) ProtoMessage() {} + +func (x *MigrationOwnershipProof) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[24] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MigrationOwnershipProof.ProtoReflect.Descriptor instead. +func (*MigrationOwnershipProof) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{24} +} + +func (x *MigrationOwnershipProof) GetGeneration() string { + if x != nil { + return x.Generation + } + return "" +} + +func (x *MigrationOwnershipProof) GetLocalGroupId() uint64 { + if x != nil { + return x.LocalGroupId + } + return 0 +} + +func (x *MigrationOwnershipProof) GetOwnerGroupId() uint64 { + if x != nil { + return x.OwnerGroupId + } + return 0 +} + +func (x *MigrationOwnershipProof) GetCatalogVersion() uint64 { + if x != nil { + return x.CatalogVersion + } + return 0 +} + +func (x *MigrationOwnershipProof) GetMinWriteTsExclusive() uint64 { + if x != nil { + return x.MinWriteTsExclusive + } + return 0 +} + +func (x *MigrationOwnershipProof) GetRouteStart() []byte { + if x != nil { + return x.RouteStart + } + return nil +} + +func (x *MigrationOwnershipProof) GetRouteEnd() []byte { + if x != nil { + return x.RouteEnd + } + return nil +} + +func (x *MigrationOwnershipProof) GetRouteId() uint64 { + if x != nil { + return x.RouteId + } + return 0 +} + +func (x *MigrationOwnershipProof) GetParentRouteId() uint64 { + if x != nil { + return x.ParentRouteId + } + return 0 +} + +func (x *MigrationOwnershipProof) GetStaged() bool { + if x != nil { + return x.Staged + } + return false +} + +func (x *MigrationOwnershipProof) GetRole() MigrationOwnershipRole { + if x != nil { + return x.Role + } + return MigrationOwnershipRole_MIGRATION_OWNERSHIP_ROLE_UNSPECIFIED +} + +func (x *MigrationOwnershipProof) GetDescriptorHash() []byte { + if x != nil { + return x.DescriptorHash + } + return nil +} + +func (x *MigrationOwnershipProof) GetWriteFloorKnown() bool { + if x != nil { + return x.WriteFloorKnown + } + return false +} + +func (x *MigrationOwnershipProof) GetSourceTrackerVersion() uint64 { + if x != nil { + return x.SourceTrackerVersion + } + return 0 +} + +func (x *MigrationOwnershipProof) GetSourceAppliedIndex() uint64 { + if x != nil { + return x.SourceAppliedIndex + } + return 0 +} + +type CleanupMigrationRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId uint64 `protobuf:"varint,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + Mode MigrationCleanupMode `protobuf:"varint,2,opt,name=mode,proto3,enum=MigrationCleanupMode" json:"mode,omitempty"` + RangeStart []byte `protobuf:"bytes,3,opt,name=range_start,json=rangeStart,proto3" json:"range_start,omitempty"` + RangeEnd []byte `protobuf:"bytes,4,opt,name=range_end,json=rangeEnd,proto3" json:"range_end,omitempty"` + Cursor []byte `protobuf:"bytes,5,opt,name=cursor,proto3" json:"cursor,omitempty"` + MaxCommitTs uint64 `protobuf:"varint,6,opt,name=max_commit_ts,json=maxCommitTs,proto3" json:"max_commit_ts,omitempty"` + MaxVersions uint32 `protobuf:"varint,7,opt,name=max_versions,json=maxVersions,proto3" json:"max_versions,omitempty"` + MaxBytes uint64 `protobuf:"varint,8,opt,name=max_bytes,json=maxBytes,proto3" json:"max_bytes,omitempty"` + MaxScannedBytes uint64 `protobuf:"varint,9,opt,name=max_scanned_bytes,json=maxScannedBytes,proto3" json:"max_scanned_bytes,omitempty"` + KeyFamily uint32 `protobuf:"varint,10,opt,name=key_family,json=keyFamily,proto3" json:"key_family,omitempty"` + RouteStart []byte `protobuf:"bytes,11,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` + RouteEnd []byte `protobuf:"bytes,12,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` + ExcludeKnownInternal bool `protobuf:"varint,13,opt,name=exclude_known_internal,json=excludeKnownInternal,proto3" json:"exclude_known_internal,omitempty"` + ExcludePrefixes [][]byte `protobuf:"bytes,14,rep,name=exclude_prefixes,json=excludePrefixes,proto3" json:"exclude_prefixes,omitempty"` + RequiresRouteKeyCheck bool `protobuf:"varint,15,opt,name=requires_route_key_check,json=requiresRouteKeyCheck,proto3" json:"requires_route_key_check,omitempty"` + RequiresDecodedS3 bool `protobuf:"varint,16,opt,name=requires_decoded_s3,json=requiresDecodedS3,proto3" json:"requires_decoded_s3,omitempty"` + // Source cleanup is conditional on the all-voter ACK configuration, checked + // against the FSM's durable ConfChange index before any cleanup side effect. + ExpectedConfigurationIndex uint64 `protobuf:"varint,17,opt,name=expected_configuration_index,json=expectedConfigurationIndex,proto3" json:"expected_configuration_index,omitempty"` + RequireConfigurationIndex bool `protobuf:"varint,18,opt,name=require_configuration_index,json=requireConfigurationIndex,proto3" json:"require_configuration_index,omitempty"` + Ownership *MigrationOwnershipProof `protobuf:"bytes,19,opt,name=ownership,proto3" json:"ownership,omitempty"` + // Strict canonical bounded JSON control; never a raw metadata mutation. + PartitionTransition []byte `protobuf:"bytes,20,opt,name=partition_transition,json=partitionTransition,proto3" json:"partition_transition,omitempty"` + // Versioned namespace activation/prejoin control. Never propose until every + // apply member (including learners) acknowledged support at this ConfChange. + NamespaceAdmission []byte `protobuf:"bytes,21,opt,name=namespace_admission,json=namespaceAdmission,proto3" json:"namespace_admission,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CleanupMigrationRequest) Reset() { + *x = CleanupMigrationRequest{} + mi := &file_internal_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CleanupMigrationRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CleanupMigrationRequest) ProtoMessage() {} + +func (x *CleanupMigrationRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[25] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CleanupMigrationRequest.ProtoReflect.Descriptor instead. +func (*CleanupMigrationRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{25} +} + +func (x *CleanupMigrationRequest) GetJobId() uint64 { + if x != nil { + return x.JobId + } + return 0 +} + +func (x *CleanupMigrationRequest) GetMode() MigrationCleanupMode { + if x != nil { + return x.Mode + } + return MigrationCleanupMode_MIGRATION_CLEANUP_MODE_UNSPECIFIED +} + +func (x *CleanupMigrationRequest) GetRangeStart() []byte { + if x != nil { + return x.RangeStart + } + return nil +} + +func (x *CleanupMigrationRequest) GetRangeEnd() []byte { + if x != nil { + return x.RangeEnd + } + return nil +} + +func (x *CleanupMigrationRequest) GetCursor() []byte { + if x != nil { + return x.Cursor + } + return nil +} + +func (x *CleanupMigrationRequest) GetMaxCommitTs() uint64 { + if x != nil { + return x.MaxCommitTs + } + return 0 +} + +func (x *CleanupMigrationRequest) GetMaxVersions() uint32 { + if x != nil { + return x.MaxVersions + } + return 0 +} + +func (x *CleanupMigrationRequest) GetMaxBytes() uint64 { + if x != nil { + return x.MaxBytes + } + return 0 +} + +func (x *CleanupMigrationRequest) GetMaxScannedBytes() uint64 { + if x != nil { + return x.MaxScannedBytes + } + return 0 +} + +func (x *CleanupMigrationRequest) GetKeyFamily() uint32 { + if x != nil { + return x.KeyFamily + } + return 0 +} + +func (x *CleanupMigrationRequest) GetRouteStart() []byte { + if x != nil { + return x.RouteStart + } + return nil +} + +func (x *CleanupMigrationRequest) GetRouteEnd() []byte { + if x != nil { + return x.RouteEnd + } + return nil +} + +func (x *CleanupMigrationRequest) GetExcludeKnownInternal() bool { + if x != nil { + return x.ExcludeKnownInternal + } + return false +} + +func (x *CleanupMigrationRequest) GetExcludePrefixes() [][]byte { + if x != nil { + return x.ExcludePrefixes + } + return nil +} + +func (x *CleanupMigrationRequest) GetRequiresRouteKeyCheck() bool { + if x != nil { + return x.RequiresRouteKeyCheck + } + return false +} + +func (x *CleanupMigrationRequest) GetRequiresDecodedS3() bool { + if x != nil { + return x.RequiresDecodedS3 + } + return false +} + +func (x *CleanupMigrationRequest) GetExpectedConfigurationIndex() uint64 { + if x != nil { + return x.ExpectedConfigurationIndex + } + return 0 +} + +func (x *CleanupMigrationRequest) GetRequireConfigurationIndex() bool { + if x != nil { + return x.RequireConfigurationIndex + } + return false +} + +func (x *CleanupMigrationRequest) GetOwnership() *MigrationOwnershipProof { + if x != nil { + return x.Ownership + } + return nil +} + +func (x *CleanupMigrationRequest) GetPartitionTransition() []byte { + if x != nil { + return x.PartitionTransition + } + return nil +} + +func (x *CleanupMigrationRequest) GetNamespaceAdmission() []byte { + if x != nil { + return x.NamespaceAdmission + } + return nil +} + +type CleanupMigrationResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + NextCursor []byte `protobuf:"bytes,1,opt,name=next_cursor,json=nextCursor,proto3" json:"next_cursor,omitempty"` + Done bool `protobuf:"varint,2,opt,name=done,proto3" json:"done,omitempty"` + DeletedRows uint64 `protobuf:"varint,3,opt,name=deleted_rows,json=deletedRows,proto3" json:"deleted_rows,omitempty"` + DeletedBytes uint64 `protobuf:"varint,4,opt,name=deleted_bytes,json=deletedBytes,proto3" json:"deleted_bytes,omitempty"` + ScannedBytes uint64 `protobuf:"varint,5,opt,name=scanned_bytes,json=scannedBytes,proto3" json:"scanned_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CleanupMigrationResponse) Reset() { + *x = CleanupMigrationResponse{} + mi := &file_internal_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CleanupMigrationResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CleanupMigrationResponse) ProtoMessage() {} + +func (x *CleanupMigrationResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[26] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CleanupMigrationResponse.ProtoReflect.Descriptor instead. +func (*CleanupMigrationResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{26} +} + +func (x *CleanupMigrationResponse) GetNextCursor() []byte { + if x != nil { + return x.NextCursor + } + return nil +} + +func (x *CleanupMigrationResponse) GetDone() bool { + if x != nil { + return x.Done + } + return false +} + +func (x *CleanupMigrationResponse) GetDeletedRows() uint64 { + if x != nil { + return x.DeletedRows + } + return 0 +} + +func (x *CleanupMigrationResponse) GetDeletedBytes() uint64 { + if x != nil { + return x.DeletedBytes + } + return 0 +} + +func (x *CleanupMigrationResponse) GetScannedBytes() uint64 { + if x != nil { + return x.ScannedBytes + } + return 0 +} + +type ProbeMigrationStateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId uint64 `protobuf:"varint,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + Kind MigrationStateProbeKind `protobuf:"varint,2,opt,name=kind,proto3,enum=MigrationStateProbeKind" json:"kind,omitempty"` + RouteStart []byte `protobuf:"bytes,3,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` + RouteEnd []byte `protobuf:"bytes,4,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` + ExpectedCatalogVersion uint64 `protobuf:"varint,5,opt,name=expected_catalog_version,json=expectedCatalogVersion,proto3" json:"expected_catalog_version,omitempty"` + ExpectedGroupId uint64 `protobuf:"varint,6,opt,name=expected_group_id,json=expectedGroupId,proto3" json:"expected_group_id,omitempty"` + MigrationJobId uint64 `protobuf:"varint,7,opt,name=migration_job_id,json=migrationJobId,proto3" json:"migration_job_id,omitempty"` + MinWriteTsExclusive uint64 `protobuf:"varint,8,opt,name=min_write_ts_exclusive,json=minWriteTsExclusive,proto3" json:"min_write_ts_exclusive,omitempty"` + SourceWriteFence bool `protobuf:"varint,9,opt,name=source_write_fence,json=sourceWriteFence,proto3" json:"source_write_fence,omitempty"` + SourceReadFence bool `protobuf:"varint,10,opt,name=source_read_fence,json=sourceReadFence,proto3" json:"source_read_fence,omitempty"` + TrackWrites bool `protobuf:"varint,11,opt,name=track_writes,json=trackWrites,proto3" json:"track_writes,omitempty"` + RetentionPinTs uint64 `protobuf:"varint,12,opt,name=retention_pin_ts,json=retentionPinTs,proto3" json:"retention_pin_ts,omitempty"` + ReadDrainNotBeforeMs int64 `protobuf:"varint,13,opt,name=read_drain_not_before_ms,json=readDrainNotBeforeMs,proto3" json:"read_drain_not_before_ms,omitempty"` + // Reads pinned at or below this HLC timestamp were admitted before the + // source read fence armed, so only those still block SOURCE_READ_DRAINED. + // Zero keeps the legacy behaviour of requiring an empty tracker. + ReadDrainMinTs uint64 `protobuf:"varint,14,opt,name=read_drain_min_ts,json=readDrainMinTs,proto3" json:"read_drain_min_ts,omitempty"` + RequireConfigurationIndex bool `protobuf:"varint,15,opt,name=require_configuration_index,json=requireConfigurationIndex,proto3" json:"require_configuration_index,omitempty"` + ExpectedConfigurationIndex uint64 `protobuf:"varint,16,opt,name=expected_configuration_index,json=expectedConfigurationIndex,proto3" json:"expected_configuration_index,omitempty"` + Ownership *MigrationOwnershipProof `protobuf:"bytes,17,opt,name=ownership,proto3" json:"ownership,omitempty"` + TransferSizeGuardVersion uint32 `protobuf:"varint,18,opt,name=transfer_size_guard_version,json=transferSizeGuardVersion,proto3" json:"transfer_size_guard_version,omitempty"` + PartitionTransition []byte `protobuf:"bytes,19,opt,name=partition_transition,json=partitionTransition,proto3" json:"partition_transition,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProbeMigrationStateRequest) Reset() { + *x = ProbeMigrationStateRequest{} + mi := &file_internal_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProbeMigrationStateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeMigrationStateRequest) ProtoMessage() {} + +func (x *ProbeMigrationStateRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeMigrationStateRequest.ProtoReflect.Descriptor instead. +func (*ProbeMigrationStateRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{27} +} + +func (x *ProbeMigrationStateRequest) GetJobId() uint64 { + if x != nil { + return x.JobId + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetKind() MigrationStateProbeKind { + if x != nil { + return x.Kind + } + return MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_UNSPECIFIED +} + +func (x *ProbeMigrationStateRequest) GetRouteStart() []byte { + if x != nil { + return x.RouteStart + } + return nil +} + +func (x *ProbeMigrationStateRequest) GetRouteEnd() []byte { + if x != nil { + return x.RouteEnd + } + return nil +} + +func (x *ProbeMigrationStateRequest) GetExpectedCatalogVersion() uint64 { + if x != nil { + return x.ExpectedCatalogVersion + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetExpectedGroupId() uint64 { + if x != nil { + return x.ExpectedGroupId + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetMigrationJobId() uint64 { + if x != nil { + return x.MigrationJobId + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetMinWriteTsExclusive() uint64 { + if x != nil { + return x.MinWriteTsExclusive + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetSourceWriteFence() bool { + if x != nil { + return x.SourceWriteFence + } + return false +} + +func (x *ProbeMigrationStateRequest) GetSourceReadFence() bool { + if x != nil { + return x.SourceReadFence + } + return false +} + +func (x *ProbeMigrationStateRequest) GetTrackWrites() bool { + if x != nil { + return x.TrackWrites + } + return false +} + +func (x *ProbeMigrationStateRequest) GetRetentionPinTs() uint64 { + if x != nil { + return x.RetentionPinTs + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetReadDrainNotBeforeMs() int64 { + if x != nil { + return x.ReadDrainNotBeforeMs + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetReadDrainMinTs() uint64 { + if x != nil { + return x.ReadDrainMinTs + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetRequireConfigurationIndex() bool { + if x != nil { + return x.RequireConfigurationIndex + } + return false +} + +func (x *ProbeMigrationStateRequest) GetExpectedConfigurationIndex() uint64 { + if x != nil { + return x.ExpectedConfigurationIndex + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetOwnership() *MigrationOwnershipProof { + if x != nil { + return x.Ownership + } + return nil +} + +func (x *ProbeMigrationStateRequest) GetTransferSizeGuardVersion() uint32 { + if x != nil { + return x.TransferSizeGuardVersion + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetPartitionTransition() []byte { + if x != nil { + return x.PartitionTransition + } + return nil +} + +type ProbeMigrationStateResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` + CatalogVersion uint64 `protobuf:"varint,2,opt,name=catalog_version,json=catalogVersion,proto3" json:"catalog_version,omitempty"` + MinAdmittedTs uint64 `protobuf:"varint,3,opt,name=min_admitted_ts,json=minAdmittedTs,proto3" json:"min_admitted_ts,omitempty"` + AppliedConfigurationIndex uint64 `protobuf:"varint,4,opt,name=applied_configuration_index,json=appliedConfigurationIndex,proto3" json:"applied_configuration_index,omitempty"` + OwnershipGeneration string `protobuf:"bytes,5,opt,name=ownership_generation,json=ownershipGeneration,proto3" json:"ownership_generation,omitempty"` + OwnershipProtocol uint32 `protobuf:"varint,6,opt,name=ownership_protocol,json=ownershipProtocol,proto3" json:"ownership_protocol,omitempty"` + Ownership *MigrationOwnershipProof `protobuf:"bytes,7,opt,name=ownership,proto3" json:"ownership,omitempty"` + OwnershipCleared bool `protobuf:"varint,8,opt,name=ownership_cleared,json=ownershipCleared,proto3" json:"ownership_cleared,omitempty"` + // Local support, not proof that the requested job's guard is applied. + TransferSizeGuardVersion uint32 `protobuf:"varint,9,opt,name=transfer_size_guard_version,json=transferSizeGuardVersion,proto3" json:"transfer_size_guard_version,omitempty"` + // Installed support, observable before a candidate has an applied ConfState. + // This is not operational ownership readiness or an activation override. + OwnershipPartitionProtocol uint32 `protobuf:"varint,10,opt,name=ownership_partition_protocol,json=ownershipPartitionProtocol,proto3" json:"ownership_partition_protocol,omitempty"` + // Installed support only; the durable group activation is separately probed. + NamespaceAdmissionVersion uint32 `protobuf:"varint,11,opt,name=namespace_admission_version,json=namespaceAdmissionVersion,proto3" json:"namespace_admission_version,omitempty"` + PartitionReceiptId []byte `protobuf:"bytes,12,opt,name=partition_receipt_id,json=partitionReceiptId,proto3" json:"partition_receipt_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProbeMigrationStateResponse) Reset() { + *x = ProbeMigrationStateResponse{} + mi := &file_internal_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProbeMigrationStateResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeMigrationStateResponse) ProtoMessage() {} + +func (x *ProbeMigrationStateResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[28] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeMigrationStateResponse.ProtoReflect.Descriptor instead. +func (*ProbeMigrationStateResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{28} +} + +func (x *ProbeMigrationStateResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +func (x *ProbeMigrationStateResponse) GetCatalogVersion() uint64 { + if x != nil { + return x.CatalogVersion + } + return 0 +} + +func (x *ProbeMigrationStateResponse) GetMinAdmittedTs() uint64 { + if x != nil { + return x.MinAdmittedTs + } + return 0 +} + +func (x *ProbeMigrationStateResponse) GetAppliedConfigurationIndex() uint64 { + if x != nil { + return x.AppliedConfigurationIndex + } + return 0 +} + +func (x *ProbeMigrationStateResponse) GetOwnershipGeneration() string { + if x != nil { + return x.OwnershipGeneration + } + return "" +} + +func (x *ProbeMigrationStateResponse) GetOwnershipProtocol() uint32 { + if x != nil { + return x.OwnershipProtocol + } + return 0 +} + +func (x *ProbeMigrationStateResponse) GetOwnership() *MigrationOwnershipProof { + if x != nil { + return x.Ownership + } + return nil +} + +func (x *ProbeMigrationStateResponse) GetOwnershipCleared() bool { + if x != nil { + return x.OwnershipCleared + } + return false +} + +func (x *ProbeMigrationStateResponse) GetTransferSizeGuardVersion() uint32 { + if x != nil { + return x.TransferSizeGuardVersion + } + return 0 +} + +func (x *ProbeMigrationStateResponse) GetOwnershipPartitionProtocol() uint32 { + if x != nil { + return x.OwnershipPartitionProtocol + } + return 0 +} + +func (x *ProbeMigrationStateResponse) GetNamespaceAdmissionVersion() uint32 { + if x != nil { + return x.NamespaceAdmissionVersion + } + return 0 +} + +func (x *ProbeMigrationStateResponse) GetPartitionReceiptId() []byte { + if x != nil { + return x.PartitionReceiptId + } + return nil +} + +type IssueMigrationTimestampRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IssueMigrationTimestampRequest) Reset() { + *x = IssueMigrationTimestampRequest{} + mi := &file_internal_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IssueMigrationTimestampRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IssueMigrationTimestampRequest) ProtoMessage() {} + +func (x *IssueMigrationTimestampRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[29] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IssueMigrationTimestampRequest.ProtoReflect.Descriptor instead. +func (*IssueMigrationTimestampRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{29} +} + +type IssueMigrationTimestampResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Timestamp uint64 `protobuf:"varint,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + LastCommitTs uint64 `protobuf:"varint,2,opt,name=last_commit_ts,json=lastCommitTs,proto3" json:"last_commit_ts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IssueMigrationTimestampResponse) Reset() { + *x = IssueMigrationTimestampResponse{} + mi := &file_internal_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IssueMigrationTimestampResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IssueMigrationTimestampResponse) ProtoMessage() {} + +func (x *IssueMigrationTimestampResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[30] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IssueMigrationTimestampResponse.ProtoReflect.Descriptor instead. +func (*IssueMigrationTimestampResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{30} +} + +func (x *IssueMigrationTimestampResponse) GetTimestamp() uint64 { + if x != nil { + return x.Timestamp + } + return 0 +} + +func (x *IssueMigrationTimestampResponse) GetLastCommitTs() uint64 { + if x != nil { + return x.LastCommitTs + } + return 0 +} + +var File_internal_proto protoreflect.FileDescriptor + +const file_internal_proto_rawDesc = "" + + "\n" + + "\x0einternal.proto\"|\n" + + "\bMutation\x12\x13\n" + + "\x02op\x18\x01 \x01(\x0e2\x03.OpR\x02op\x12\x10\n" + + "\x03key\x18\x02 \x01(\fR\x03key\x12\x14\n" + + "\x05value\x18\x03 \x01(\fR\x05value\x123\n" + + "\x16commit_ts_value_offset\x18\x04 \x01(\x04R\x13commitTsValueOffset\"\xbd\x02\n" + + "\aRequest\x12\x15\n" + + "\x06is_txn\x18\x01 \x01(\bR\x05isTxn\x12\x1c\n" + + "\x05phase\x18\x02 \x01(\x0e2\x06.PhaseR\x05phase\x12\x0e\n" + + "\x02ts\x18\x03 \x01(\x04R\x02ts\x12'\n" + + "\tmutations\x18\x04 \x03(\v2\t.MutationR\tmutations\x12\x1b\n" + + "\tread_keys\x18\x05 \x03(\fR\breadKeys\x124\n" + + "\x16observed_route_version\x18\x06 \x01(\x04R\x14observedRouteVersion\x125\n" + + "\x17write_fence_bypass_keys\x18\a \x03(\fR\x14writeFenceBypassKeys\x12:\n" + + "\x19timestamp_admission_floor\x18\b \x01(\x04R\x17timestampAdmissionFloor\"3\n" + + "\vRaftCommand\x12$\n" + + "\brequests\x18\x01 \x03(\v2\b.RequestR\brequests\"M\n" + + "\x0eForwardRequest\x12\x15\n" + + "\x06is_txn\x18\x01 \x01(\bR\x05isTxn\x12$\n" + + "\brequests\x18\x02 \x03(\v2\b.RequestR\brequests\"k\n" + + "\x0fForwardResponse\x12\x18\n" + + "\asuccess\x18\x01 \x01(\bR\asuccess\x12!\n" + + "\fcommit_index\x18\x02 \x01(\x04R\vcommitIndex\x12\x1b\n" + + "\tcommit_ts\x18\x03 \x01(\x04R\bcommitTs\"7\n" + + "\x1bForwardAdminProposalRequest\x12\x18\n" + + "\apayload\x18\x01 \x01(\fR\apayload\"A\n" + + "\x1cForwardAdminProposalResponse\x12!\n" + + "\fcommit_index\x18\x01 \x01(\x04R\vcommitIndex\"\x19\n" + + "\x17ForwardLeaseReadRequest\"e\n" + + "\x18ForwardLeaseReadResponse\x12#\n" + + "\rapplied_index\x18\x01 \x01(\x04R\fappliedIndex\x12$\n" + + "\x0elast_commit_ts\x18\x02 \x01(\x04R\flastCommitTs\"\\\n" + + "\x1bPrepareEvidenceStateRequest\x12\x14\n" + + "\x05floor\x18\x01 \x01(\x04R\x05floor\x12'\n" + + "\x0fcapability_only\x18\x02 \x01(\bR\x0ecapabilityOnly\"\xb1\x02\n" + + "\x1cPrepareEvidenceStateResponse\x12\x1c\n" + + "\tsupported\x18\x01 \x01(\bR\tsupported\x12#\n" + + "\rproof_enabled\x18\x02 \x01(\bR\fproofEnabled\x12%\n" + + "\x0eunproven_locks\x18\x03 \x01(\bR\runprovenLocks\x12\x1f\n" + + "\vfence_floor\x18\x04 \x01(\x04R\n" + + "fenceFloor\x12#\n" + + "\rapplied_index\x18\x05 \x01(\x04R\fappliedIndex\x12'\n" + + "\x0fcutover_applied\x18\x06 \x01(\bR\x0ecutoverApplied\x128\n" + + "\x18dedicated_tso_configured\x18\a \x01(\bR\x16dedicatedTsoConfigured\"I\n" + + "\x13RelayPublishRequest\x12\x18\n" + + "\achannel\x18\x01 \x01(\fR\achannel\x12\x18\n" + + "\amessage\x18\x02 \x01(\fR\amessage\"8\n" + + "\x14RelayPublishResponse\x12 \n" + + "\vsubscribers\x18\x01 \x01(\x03R\vsubscribers\"\xc5\x03\n" + + "\x1aExportRangeVersionsRequest\x12\x1f\n" + + "\vrange_start\x18\x01 \x01(\fR\n" + + "rangeStart\x12\x1b\n" + + "\trange_end\x18\x02 \x01(\fR\brangeEnd\x12\"\n" + + "\rmax_commit_ts\x18\x03 \x01(\x04R\vmaxCommitTs\x12\"\n" + + "\rmin_commit_ts\x18\x04 \x01(\x04R\vminCommitTs\x12\x16\n" + + "\x06cursor\x18\x05 \x01(\fR\x06cursor\x12\x1f\n" + + "\vchunk_bytes\x18\x06 \x01(\rR\n" + + "chunkBytes\x12\x1f\n" + + "\vroute_start\x18\a \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\b \x01(\fR\brouteEnd\x12*\n" + + "\x11max_scanned_bytes\x18\t \x01(\x04R\x0fmaxScannedBytes\x12\x1d\n" + + "\n" + + "key_family\x18\n" + + " \x01(\rR\tkeyFamily\x124\n" + + "\x16exclude_known_internal\x18\v \x01(\bR\x14excludeKnownInternal\x12)\n" + + "\x10exclude_prefixes\x18\f \x03(\fR\x0fexcludePrefixes\"|\n" + + "\x1bExportRangeVersionsResponse\x12(\n" + + "\bversions\x18\x01 \x03(\v2\f.MVCCVersionR\bversions\x12\x1f\n" + + "\vnext_cursor\x18\x02 \x01(\fR\n" + + "nextCursor\x12\x12\n" + + "\x04done\x18\x03 \x01(\bR\x04done\"\xac\x01\n" + + "\vMVCCVersion\x12\x10\n" + + "\x03key\x18\x01 \x01(\fR\x03key\x12\x1b\n" + + "\tcommit_ts\x18\x02 \x01(\x04R\bcommitTs\x12\x1c\n" + + "\ttombstone\x18\x03 \x01(\bR\ttombstone\x12\x14\n" + + "\x05value\x18\x04 \x01(\fR\x05value\x12\x1d\n" + + "\n" + + "key_family\x18\x05 \x01(\rR\tkeyFamily\x12\x1b\n" + + "\texpire_at\x18\x06 \x01(\x04R\bexpireAt\"\xb1\x01\n" + + "\x1aImportRangeVersionsRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12(\n" + + "\bversions\x18\x02 \x03(\v2\f.MVCCVersionR\bversions\x12\x16\n" + + "\x06cursor\x18\x03 \x01(\fR\x06cursor\x12\x1d\n" + + "\n" + + "bracket_id\x18\x04 \x01(\x04R\tbracketId\x12\x1b\n" + + "\tbatch_seq\x18\x05 \x01(\x04R\bbatchSeq\"^\n" + + "\x1bImportRangeVersionsResponse\x12!\n" + + "\facked_cursor\x18\x01 \x01(\fR\vackedCursor\x12\x1c\n" + + "\tduplicate\x18\x02 \x01(\bR\tduplicate\"\xb9\x01\n" + + "\x1cPromoteStagedVersionsRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12\x16\n" + + "\x06cursor\x18\x02 \x01(\fR\x06cursor\x12!\n" + + "\fmax_versions\x18\x03 \x01(\rR\vmaxVersions\x12\x1b\n" + + "\tmax_bytes\x18\x04 \x01(\x04R\bmaxBytes\x12*\n" + + "\x11max_scanned_bytes\x18\x05 \x01(\x04R\x0fmaxScannedBytes\"\xa1\x01\n" + + "\x1dPromoteStagedVersionsResponse\x12\x1f\n" + + "\vnext_cursor\x18\x01 \x01(\fR\n" + + "nextCursor\x12\x12\n" + + "\x04done\x18\x02 \x01(\bR\x04done\x12#\n" + + "\rpromoted_rows\x18\x03 \x01(\x04R\fpromotedRows\x12&\n" + + "\x0fmax_promoted_ts\x18\x04 \x01(\x04R\rmaxPromotedTs\"\xb2\x05\n" + + "\x1cTargetStagedReadinessRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12\x1f\n" + + "\vroute_start\x18\x02 \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\x03 \x01(\fR\brouteEnd\x128\n" + + "\x18expected_cutover_version\x18\x04 \x01(\x04R\x16expectedCutoverVersion\x12(\n" + + "\x10migration_job_id\x18\x05 \x01(\x04R\x0emigrationJobId\x123\n" + + "\x16min_write_ts_exclusive\x18\x06 \x01(\x04R\x13minWriteTsExclusive\x12\x14\n" + + "\x05armed\x18\a \x01(\bR\x05armed\x12,\n" + + "\x12source_write_fence\x18\b \x01(\bR\x10sourceWriteFence\x12*\n" + + "\x11source_read_fence\x18\t \x01(\bR\x0fsourceReadFence\x12(\n" + + "\x10retention_pin_ts\x18\n" + + " \x01(\x04R\x0eretentionPinTs\x12!\n" + + "\ftrack_writes\x18\v \x01(\bR\vtrackWrites\x12&\n" + + "\x0fmin_admitted_ts\x18\f \x01(\x04R\rminAdmittedTs\x12=\n" + + "\x1btransfer_size_guard_version\x18\r \x01(\rR\x18transferSizeGuardVersion\x12>\n" + + "\x1brequire_configuration_index\x18\x0e \x01(\bR\x19requireConfigurationIndex\x12@\n" + + "\x1cexpected_configuration_index\x18\x0f \x01(\x04R\x1aexpectedConfigurationIndex\"G\n" + + "\x1dTargetStagedReadinessResponse\x12&\n" + + "\x0fmin_admitted_ts\x18\x01 \x01(\x04R\rminAdmittedTs\"p\n" + + "\x1aProbeMigrationLocksRequest\x12\x1f\n" + + "\vroute_start\x18\x01 \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\x02 \x01(\fR\brouteEnd\x12\x14\n" + + "\x05limit\x18\x03 \x01(\rR\x05limit\"B\n" + + "\x1bProbeMigrationLocksResponse\x12#\n" + + "\rpending_count\x18\x01 \x01(\rR\fpendingCount\"\xe6\x04\n" + + "\x17MigrationOwnershipProof\x12\x1e\n" + + "\n" + + "generation\x18\x01 \x01(\tR\n" + + "generation\x12$\n" + + "\x0elocal_group_id\x18\x02 \x01(\x04R\flocalGroupId\x12$\n" + + "\x0eowner_group_id\x18\x03 \x01(\x04R\fownerGroupId\x12'\n" + + "\x0fcatalog_version\x18\x04 \x01(\x04R\x0ecatalogVersion\x123\n" + + "\x16min_write_ts_exclusive\x18\x05 \x01(\x04R\x13minWriteTsExclusive\x12\x1f\n" + + "\vroute_start\x18\x06 \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\a \x01(\fR\brouteEnd\x12\x19\n" + + "\broute_id\x18\b \x01(\x04R\arouteId\x12&\n" + + "\x0fparent_route_id\x18\t \x01(\x04R\rparentRouteId\x12\x16\n" + + "\x06staged\x18\n" + + " \x01(\bR\x06staged\x12+\n" + + "\x04role\x18\v \x01(\x0e2\x17.MigrationOwnershipRoleR\x04role\x12'\n" + + "\x0fdescriptor_hash\x18\f \x01(\fR\x0edescriptorHash\x12*\n" + + "\x11write_floor_known\x18\r \x01(\bR\x0fwriteFloorKnown\x124\n" + + "\x16source_tracker_version\x18\x0e \x01(\x04R\x14sourceTrackerVersion\x120\n" + + "\x14source_applied_index\x18\x0f \x01(\x04R\x12sourceAppliedIndex\"\x86\a\n" + + "\x17CleanupMigrationRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12)\n" + + "\x04mode\x18\x02 \x01(\x0e2\x15.MigrationCleanupModeR\x04mode\x12\x1f\n" + + "\vrange_start\x18\x03 \x01(\fR\n" + + "rangeStart\x12\x1b\n" + + "\trange_end\x18\x04 \x01(\fR\brangeEnd\x12\x16\n" + + "\x06cursor\x18\x05 \x01(\fR\x06cursor\x12\"\n" + + "\rmax_commit_ts\x18\x06 \x01(\x04R\vmaxCommitTs\x12!\n" + + "\fmax_versions\x18\a \x01(\rR\vmaxVersions\x12\x1b\n" + + "\tmax_bytes\x18\b \x01(\x04R\bmaxBytes\x12*\n" + + "\x11max_scanned_bytes\x18\t \x01(\x04R\x0fmaxScannedBytes\x12\x1d\n" + + "\n" + + "key_family\x18\n" + + " \x01(\rR\tkeyFamily\x12\x1f\n" + + "\vroute_start\x18\v \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\f \x01(\fR\brouteEnd\x124\n" + + "\x16exclude_known_internal\x18\r \x01(\bR\x14excludeKnownInternal\x12)\n" + + "\x10exclude_prefixes\x18\x0e \x03(\fR\x0fexcludePrefixes\x127\n" + + "\x18requires_route_key_check\x18\x0f \x01(\bR\x15requiresRouteKeyCheck\x12.\n" + + "\x13requires_decoded_s3\x18\x10 \x01(\bR\x11requiresDecodedS3\x12@\n" + + "\x1cexpected_configuration_index\x18\x11 \x01(\x04R\x1aexpectedConfigurationIndex\x12>\n" + + "\x1brequire_configuration_index\x18\x12 \x01(\bR\x19requireConfigurationIndex\x126\n" + + "\townership\x18\x13 \x01(\v2\x18.MigrationOwnershipProofR\townership\x121\n" + + "\x14partition_transition\x18\x14 \x01(\fR\x13partitionTransition\x12/\n" + + "\x13namespace_admission\x18\x15 \x01(\fR\x12namespaceAdmission\"\xbc\x01\n" + + "\x18CleanupMigrationResponse\x12\x1f\n" + + "\vnext_cursor\x18\x01 \x01(\fR\n" + + "nextCursor\x12\x12\n" + + "\x04done\x18\x02 \x01(\bR\x04done\x12!\n" + + "\fdeleted_rows\x18\x03 \x01(\x04R\vdeletedRows\x12#\n" + + "\rdeleted_bytes\x18\x04 \x01(\x04R\fdeletedBytes\x12#\n" + + "\rscanned_bytes\x18\x05 \x01(\x04R\fscannedBytes\"\x9a\a\n" + + "\x1aProbeMigrationStateRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12,\n" + + "\x04kind\x18\x02 \x01(\x0e2\x18.MigrationStateProbeKindR\x04kind\x12\x1f\n" + + "\vroute_start\x18\x03 \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\x04 \x01(\fR\brouteEnd\x128\n" + + "\x18expected_catalog_version\x18\x05 \x01(\x04R\x16expectedCatalogVersion\x12*\n" + + "\x11expected_group_id\x18\x06 \x01(\x04R\x0fexpectedGroupId\x12(\n" + + "\x10migration_job_id\x18\a \x01(\x04R\x0emigrationJobId\x123\n" + + "\x16min_write_ts_exclusive\x18\b \x01(\x04R\x13minWriteTsExclusive\x12,\n" + + "\x12source_write_fence\x18\t \x01(\bR\x10sourceWriteFence\x12*\n" + + "\x11source_read_fence\x18\n" + + " \x01(\bR\x0fsourceReadFence\x12!\n" + + "\ftrack_writes\x18\v \x01(\bR\vtrackWrites\x12(\n" + + "\x10retention_pin_ts\x18\f \x01(\x04R\x0eretentionPinTs\x126\n" + + "\x18read_drain_not_before_ms\x18\r \x01(\x03R\x14readDrainNotBeforeMs\x12)\n" + + "\x11read_drain_min_ts\x18\x0e \x01(\x04R\x0ereadDrainMinTs\x12>\n" + + "\x1brequire_configuration_index\x18\x0f \x01(\bR\x19requireConfigurationIndex\x12@\n" + + "\x1cexpected_configuration_index\x18\x10 \x01(\x04R\x1aexpectedConfigurationIndex\x126\n" + + "\townership\x18\x11 \x01(\v2\x18.MigrationOwnershipProofR\townership\x12=\n" + + "\x1btransfer_size_guard_version\x18\x12 \x01(\rR\x18transferSizeGuardVersion\x121\n" + + "\x14partition_transition\x18\x13 \x01(\fR\x13partitionTransition\"\xfe\x04\n" + + "\x1bProbeMigrationStateResponse\x12\x14\n" + + "\x05ready\x18\x01 \x01(\bR\x05ready\x12'\n" + + "\x0fcatalog_version\x18\x02 \x01(\x04R\x0ecatalogVersion\x12&\n" + + "\x0fmin_admitted_ts\x18\x03 \x01(\x04R\rminAdmittedTs\x12>\n" + + "\x1bapplied_configuration_index\x18\x04 \x01(\x04R\x19appliedConfigurationIndex\x121\n" + + "\x14ownership_generation\x18\x05 \x01(\tR\x13ownershipGeneration\x12-\n" + + "\x12ownership_protocol\x18\x06 \x01(\rR\x11ownershipProtocol\x126\n" + + "\townership\x18\a \x01(\v2\x18.MigrationOwnershipProofR\townership\x12+\n" + + "\x11ownership_cleared\x18\b \x01(\bR\x10ownershipCleared\x12=\n" + + "\x1btransfer_size_guard_version\x18\t \x01(\rR\x18transferSizeGuardVersion\x12@\n" + + "\x1cownership_partition_protocol\x18\n" + + " \x01(\rR\x1aownershipPartitionProtocol\x12>\n" + + "\x1bnamespace_admission_version\x18\v \x01(\rR\x19namespaceAdmissionVersion\x120\n" + + "\x14partition_receipt_id\x18\f \x01(\fR\x12partitionReceiptId\" \n" + + "\x1eIssueMigrationTimestampRequest\"e\n" + + "\x1fIssueMigrationTimestampResponse\x12\x1c\n" + + "\ttimestamp\x18\x01 \x01(\x04R\ttimestamp\x12$\n" + + "\x0elast_commit_ts\x18\x02 \x01(\x04R\flastCommitTs*&\n" + "\x02Op\x12\a\n" + "\x03PUT\x10\x00\x12\a\n" + "\x03DEL\x10\x01\x12\x0e\n" + @@ -1556,7 +3045,34 @@ const file_internal_proto_rawDesc = "" + "\aPREPARE\x10\x01\x12\n" + "\n" + "\x06COMMIT\x10\x02\x12\t\n" + - "\x05ABORT\x10\x032\xf9\x04\n" + + "\x05ABORT\x10\x03*\xa5\x03\n" + + "\x14MigrationCleanupMode\x12&\n" + + "\"MIGRATION_CLEANUP_MODE_UNSPECIFIED\x10\x00\x12#\n" + + "\x1fMIGRATION_CLEANUP_MODE_VERSIONS\x10\x01\x12#\n" + + "\x1fMIGRATION_CLEANUP_MODE_METADATA\x10\x02\x12,\n" + + "(MIGRATION_CLEANUP_MODE_OWNERSHIP_ACQUIRE\x10\x03\x12*\n" + + "&MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR\x10\x04\x12,\n" + + "(MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH\x10\x05\x123\n" + + "/MIGRATION_CLEANUP_MODE_SOURCE_OWNERSHIP_ACQUIRE\x10\x06\x12.\n" + + "*MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION\x10\a\x12.\n" + + "*MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION\x10\b*\x8d\x01\n" + + "\x16MigrationOwnershipRole\x12(\n" + + "$MIGRATION_OWNERSHIP_ROLE_UNSPECIFIED\x10\x00\x12\"\n" + + "\x1eMIGRATION_OWNERSHIP_ROLE_OWNER\x10\x01\x12%\n" + + "!MIGRATION_OWNERSHIP_ROLE_NONOWNER\x10\x02*\xcb\x04\n" + + "\x17MigrationStateProbeKind\x12*\n" + + "&MIGRATION_STATE_PROBE_KIND_UNSPECIFIED\x10\x00\x12.\n" + + "*MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED\x10\x01\x128\n" + + "4MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED\x10\x02\x123\n" + + "/MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED\x10\x03\x122\n" + + ".MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED\x10\x04\x12/\n" + + "+MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED\x10\x05\x123\n" + + "/MIGRATION_STATE_PROBE_KIND_SOURCE_GUARD_PRESENT\x10\x06\x123\n" + + "/MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY\x10\a\x120\n" + + ",MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED\x10\b\x120\n" + + ",MIGRATION_STATE_PROBE_KIND_PARTITION_APPLIED\x10\t\x122\n" + + ".MIGRATION_STATE_PROBE_KIND_NAMESPACE_ADMISSION\x10\n" + + "2\xab\b\n" + "\bInternal\x12.\n" + "\aForward\x12\x0f.ForwardRequest\x1a\x10.ForwardResponse\"\x00\x12U\n" + "\x14ForwardAdminProposal\x12\x1c.ForwardAdminProposalRequest\x1a\x1d.ForwardAdminProposalResponse\"\x00\x12I\n" + @@ -1565,7 +3081,12 @@ const file_internal_proto_rawDesc = "" + "\fRelayPublish\x12\x14.RelayPublishRequest\x1a\x15.RelayPublishResponse\"\x00\x12T\n" + "\x13ExportRangeVersions\x12\x1b.ExportRangeVersionsRequest\x1a\x1c.ExportRangeVersionsResponse\"\x000\x01\x12R\n" + "\x13ImportRangeVersions\x12\x1b.ImportRangeVersionsRequest\x1a\x1c.ImportRangeVersionsResponse\"\x00\x12X\n" + - "\x15PromoteStagedVersions\x12\x1d.PromoteStagedVersionsRequest\x1a\x1e.PromoteStagedVersionsResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" + "\x15PromoteStagedVersions\x12\x1d.PromoteStagedVersionsRequest\x1a\x1e.PromoteStagedVersionsResponse\"\x00\x12]\n" + + "\x1aApplyTargetStagedReadiness\x12\x1d.TargetStagedReadinessRequest\x1a\x1e.TargetStagedReadinessResponse\"\x00\x12R\n" + + "\x13ProbeMigrationLocks\x12\x1b.ProbeMigrationLocksRequest\x1a\x1c.ProbeMigrationLocksResponse\"\x00\x12I\n" + + "\x10CleanupMigration\x12\x18.CleanupMigrationRequest\x1a\x19.CleanupMigrationResponse\"\x00\x12R\n" + + "\x13ProbeMigrationState\x12\x1b.ProbeMigrationStateRequest\x1a\x1c.ProbeMigrationStateResponse\"\x00\x12^\n" + + "\x17IssueMigrationTimestamp\x12\x1f.IssueMigrationTimestampRequest\x1a .IssueMigrationTimestampResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" var ( file_internal_proto_rawDescOnce sync.Once @@ -1579,61 +3100,91 @@ func file_internal_proto_rawDescGZIP() []byte { return file_internal_proto_rawDescData } -var file_internal_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_internal_proto_enumTypes = make([]protoimpl.EnumInfo, 5) +var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 31) var file_internal_proto_goTypes = []any{ - (Op)(0), // 0: Op - (Phase)(0), // 1: Phase - (*Mutation)(nil), // 2: Mutation - (*Request)(nil), // 3: Request - (*RaftCommand)(nil), // 4: RaftCommand - (*ForwardRequest)(nil), // 5: ForwardRequest - (*ForwardResponse)(nil), // 6: ForwardResponse - (*ForwardAdminProposalRequest)(nil), // 7: ForwardAdminProposalRequest - (*ForwardAdminProposalResponse)(nil), // 8: ForwardAdminProposalResponse - (*ForwardLeaseReadRequest)(nil), // 9: ForwardLeaseReadRequest - (*ForwardLeaseReadResponse)(nil), // 10: ForwardLeaseReadResponse - (*PrepareEvidenceStateRequest)(nil), // 11: PrepareEvidenceStateRequest - (*PrepareEvidenceStateResponse)(nil), // 12: PrepareEvidenceStateResponse - (*RelayPublishRequest)(nil), // 13: RelayPublishRequest - (*RelayPublishResponse)(nil), // 14: RelayPublishResponse - (*ExportRangeVersionsRequest)(nil), // 15: ExportRangeVersionsRequest - (*ExportRangeVersionsResponse)(nil), // 16: ExportRangeVersionsResponse - (*MVCCVersion)(nil), // 17: MVCCVersion - (*ImportRangeVersionsRequest)(nil), // 18: ImportRangeVersionsRequest - (*ImportRangeVersionsResponse)(nil), // 19: ImportRangeVersionsResponse - (*PromoteStagedVersionsRequest)(nil), // 20: PromoteStagedVersionsRequest - (*PromoteStagedVersionsResponse)(nil), // 21: PromoteStagedVersionsResponse + (Op)(0), // 0: Op + (Phase)(0), // 1: Phase + (MigrationCleanupMode)(0), // 2: MigrationCleanupMode + (MigrationOwnershipRole)(0), // 3: MigrationOwnershipRole + (MigrationStateProbeKind)(0), // 4: MigrationStateProbeKind + (*Mutation)(nil), // 5: Mutation + (*Request)(nil), // 6: Request + (*RaftCommand)(nil), // 7: RaftCommand + (*ForwardRequest)(nil), // 8: ForwardRequest + (*ForwardResponse)(nil), // 9: ForwardResponse + (*ForwardAdminProposalRequest)(nil), // 10: ForwardAdminProposalRequest + (*ForwardAdminProposalResponse)(nil), // 11: ForwardAdminProposalResponse + (*ForwardLeaseReadRequest)(nil), // 12: ForwardLeaseReadRequest + (*ForwardLeaseReadResponse)(nil), // 13: ForwardLeaseReadResponse + (*PrepareEvidenceStateRequest)(nil), // 14: PrepareEvidenceStateRequest + (*PrepareEvidenceStateResponse)(nil), // 15: PrepareEvidenceStateResponse + (*RelayPublishRequest)(nil), // 16: RelayPublishRequest + (*RelayPublishResponse)(nil), // 17: RelayPublishResponse + (*ExportRangeVersionsRequest)(nil), // 18: ExportRangeVersionsRequest + (*ExportRangeVersionsResponse)(nil), // 19: ExportRangeVersionsResponse + (*MVCCVersion)(nil), // 20: MVCCVersion + (*ImportRangeVersionsRequest)(nil), // 21: ImportRangeVersionsRequest + (*ImportRangeVersionsResponse)(nil), // 22: ImportRangeVersionsResponse + (*PromoteStagedVersionsRequest)(nil), // 23: PromoteStagedVersionsRequest + (*PromoteStagedVersionsResponse)(nil), // 24: PromoteStagedVersionsResponse + (*TargetStagedReadinessRequest)(nil), // 25: TargetStagedReadinessRequest + (*TargetStagedReadinessResponse)(nil), // 26: TargetStagedReadinessResponse + (*ProbeMigrationLocksRequest)(nil), // 27: ProbeMigrationLocksRequest + (*ProbeMigrationLocksResponse)(nil), // 28: ProbeMigrationLocksResponse + (*MigrationOwnershipProof)(nil), // 29: MigrationOwnershipProof + (*CleanupMigrationRequest)(nil), // 30: CleanupMigrationRequest + (*CleanupMigrationResponse)(nil), // 31: CleanupMigrationResponse + (*ProbeMigrationStateRequest)(nil), // 32: ProbeMigrationStateRequest + (*ProbeMigrationStateResponse)(nil), // 33: ProbeMigrationStateResponse + (*IssueMigrationTimestampRequest)(nil), // 34: IssueMigrationTimestampRequest + (*IssueMigrationTimestampResponse)(nil), // 35: IssueMigrationTimestampResponse } var file_internal_proto_depIdxs = []int32{ 0, // 0: Mutation.op:type_name -> Op 1, // 1: Request.phase:type_name -> Phase - 2, // 2: Request.mutations:type_name -> Mutation - 3, // 3: RaftCommand.requests:type_name -> Request - 3, // 4: ForwardRequest.requests:type_name -> Request - 17, // 5: ExportRangeVersionsResponse.versions:type_name -> MVCCVersion - 17, // 6: ImportRangeVersionsRequest.versions:type_name -> MVCCVersion - 5, // 7: Internal.Forward:input_type -> ForwardRequest - 7, // 8: Internal.ForwardAdminProposal:input_type -> ForwardAdminProposalRequest - 9, // 9: Internal.ForwardLeaseRead:input_type -> ForwardLeaseReadRequest - 11, // 10: Internal.GetPrepareEvidenceState:input_type -> PrepareEvidenceStateRequest - 13, // 11: Internal.RelayPublish:input_type -> RelayPublishRequest - 15, // 12: Internal.ExportRangeVersions:input_type -> ExportRangeVersionsRequest - 18, // 13: Internal.ImportRangeVersions:input_type -> ImportRangeVersionsRequest - 20, // 14: Internal.PromoteStagedVersions:input_type -> PromoteStagedVersionsRequest - 6, // 15: Internal.Forward:output_type -> ForwardResponse - 8, // 16: Internal.ForwardAdminProposal:output_type -> ForwardAdminProposalResponse - 10, // 17: Internal.ForwardLeaseRead:output_type -> ForwardLeaseReadResponse - 12, // 18: Internal.GetPrepareEvidenceState:output_type -> PrepareEvidenceStateResponse - 14, // 19: Internal.RelayPublish:output_type -> RelayPublishResponse - 16, // 20: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse - 19, // 21: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse - 21, // 22: Internal.PromoteStagedVersions:output_type -> PromoteStagedVersionsResponse - 15, // [15:23] is the sub-list for method output_type - 7, // [7:15] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 5, // 2: Request.mutations:type_name -> Mutation + 6, // 3: RaftCommand.requests:type_name -> Request + 6, // 4: ForwardRequest.requests:type_name -> Request + 20, // 5: ExportRangeVersionsResponse.versions:type_name -> MVCCVersion + 20, // 6: ImportRangeVersionsRequest.versions:type_name -> MVCCVersion + 3, // 7: MigrationOwnershipProof.role:type_name -> MigrationOwnershipRole + 2, // 8: CleanupMigrationRequest.mode:type_name -> MigrationCleanupMode + 29, // 9: CleanupMigrationRequest.ownership:type_name -> MigrationOwnershipProof + 4, // 10: ProbeMigrationStateRequest.kind:type_name -> MigrationStateProbeKind + 29, // 11: ProbeMigrationStateRequest.ownership:type_name -> MigrationOwnershipProof + 29, // 12: ProbeMigrationStateResponse.ownership:type_name -> MigrationOwnershipProof + 8, // 13: Internal.Forward:input_type -> ForwardRequest + 10, // 14: Internal.ForwardAdminProposal:input_type -> ForwardAdminProposalRequest + 12, // 15: Internal.ForwardLeaseRead:input_type -> ForwardLeaseReadRequest + 14, // 16: Internal.GetPrepareEvidenceState:input_type -> PrepareEvidenceStateRequest + 16, // 17: Internal.RelayPublish:input_type -> RelayPublishRequest + 18, // 18: Internal.ExportRangeVersions:input_type -> ExportRangeVersionsRequest + 21, // 19: Internal.ImportRangeVersions:input_type -> ImportRangeVersionsRequest + 23, // 20: Internal.PromoteStagedVersions:input_type -> PromoteStagedVersionsRequest + 25, // 21: Internal.ApplyTargetStagedReadiness:input_type -> TargetStagedReadinessRequest + 27, // 22: Internal.ProbeMigrationLocks:input_type -> ProbeMigrationLocksRequest + 30, // 23: Internal.CleanupMigration:input_type -> CleanupMigrationRequest + 32, // 24: Internal.ProbeMigrationState:input_type -> ProbeMigrationStateRequest + 34, // 25: Internal.IssueMigrationTimestamp:input_type -> IssueMigrationTimestampRequest + 9, // 26: Internal.Forward:output_type -> ForwardResponse + 11, // 27: Internal.ForwardAdminProposal:output_type -> ForwardAdminProposalResponse + 13, // 28: Internal.ForwardLeaseRead:output_type -> ForwardLeaseReadResponse + 15, // 29: Internal.GetPrepareEvidenceState:output_type -> PrepareEvidenceStateResponse + 17, // 30: Internal.RelayPublish:output_type -> RelayPublishResponse + 19, // 31: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse + 22, // 32: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse + 24, // 33: Internal.PromoteStagedVersions:output_type -> PromoteStagedVersionsResponse + 26, // 34: Internal.ApplyTargetStagedReadiness:output_type -> TargetStagedReadinessResponse + 28, // 35: Internal.ProbeMigrationLocks:output_type -> ProbeMigrationLocksResponse + 31, // 36: Internal.CleanupMigration:output_type -> CleanupMigrationResponse + 33, // 37: Internal.ProbeMigrationState:output_type -> ProbeMigrationStateResponse + 35, // 38: Internal.IssueMigrationTimestamp:output_type -> IssueMigrationTimestampResponse + 26, // [26:39] is the sub-list for method output_type + 13, // [13:26] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name } func init() { file_internal_proto_init() } @@ -1646,8 +3197,8 @@ func file_internal_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_internal_proto_rawDesc), len(file_internal_proto_rawDesc)), - NumEnums: 2, - NumMessages: 20, + NumEnums: 5, + NumMessages: 31, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/internal.proto b/proto/internal.proto index 3949d46e5..ab8edbe2d 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -13,6 +13,11 @@ service Internal { rpc ExportRangeVersions(ExportRangeVersionsRequest) returns (stream ExportRangeVersionsResponse) {} rpc ImportRangeVersions(ImportRangeVersionsRequest) returns (ImportRangeVersionsResponse) {} rpc PromoteStagedVersions(PromoteStagedVersionsRequest) returns (PromoteStagedVersionsResponse) {} + rpc ApplyTargetStagedReadiness(TargetStagedReadinessRequest) returns (TargetStagedReadinessResponse) {} + rpc ProbeMigrationLocks(ProbeMigrationLocksRequest) returns (ProbeMigrationLocksResponse) {} + rpc CleanupMigration(CleanupMigrationRequest) returns (CleanupMigrationResponse) {} + rpc ProbeMigrationState(ProbeMigrationStateRequest) returns (ProbeMigrationStateResponse) {} + rpc IssueMigrationTimestamp(IssueMigrationTimestampRequest) returns (IssueMigrationTimestampResponse) {} } // internal.proto is node to node communication message in raft replication. @@ -179,6 +184,11 @@ message ImportRangeVersionsRequest { message ImportRangeVersionsResponse { bytes acked_cursor = 1; + // Set when the target already held a durable acknowledgement for this + // batch_seq and discarded the resent rows. acked_cursor is then the cursor + // the target recorded for that batch, which need not be the cursor the + // caller just sent. + bool duplicate = 2; } message PromoteStagedVersionsRequest { @@ -195,3 +205,180 @@ message PromoteStagedVersionsResponse { uint64 promoted_rows = 3; uint64 max_promoted_ts = 4; } + +message TargetStagedReadinessRequest { + uint64 job_id = 1; + bytes route_start = 2; + bytes route_end = 3; + uint64 expected_cutover_version = 4; + uint64 migration_job_id = 5; + uint64 min_write_ts_exclusive = 6; + bool armed = 7; + // Source-side controls reuse the same Raft-replicated durable record. They + // are ignored by target staged-readiness checks. + bool source_write_fence = 8; + bool source_read_fence = 9; + uint64 retention_pin_ts = 10; + bool track_writes = 11; + uint64 min_admitted_ts = 12; + // Explicit opt-in; zero preserves legacy tracker write admission. + uint32 transfer_size_guard_version = 13; + bool require_configuration_index = 14; + uint64 expected_configuration_index = 15; +} + +message TargetStagedReadinessResponse { + uint64 min_admitted_ts = 1; +} + +message ProbeMigrationLocksRequest { + bytes route_start = 1; + bytes route_end = 2; + uint32 limit = 3; +} + +message ProbeMigrationLocksResponse { + uint32 pending_count = 1; +} + +enum MigrationCleanupMode { + MIGRATION_CLEANUP_MODE_UNSPECIFIED = 0; + MIGRATION_CLEANUP_MODE_VERSIONS = 1; + MIGRATION_CLEANUP_MODE_METADATA = 2; + // New modes deliberately fail on old runtimes instead of letting unknown + // proof fields silently degrade to an unbound metadata clear. + MIGRATION_CLEANUP_MODE_OWNERSHIP_ACQUIRE = 3; + MIGRATION_CLEANUP_MODE_OWNERSHIP_CLEAR = 4; + MIGRATION_CLEANUP_MODE_OWNERSHIP_REFRESH = 5; + MIGRATION_CLEANUP_MODE_SOURCE_OWNERSHIP_ACQUIRE = 6; + MIGRATION_CLEANUP_MODE_OWNERSHIP_PARTITION = 7; + MIGRATION_CLEANUP_MODE_NAMESPACE_ADMISSION = 8; +} + +enum MigrationOwnershipRole { + MIGRATION_OWNERSHIP_ROLE_UNSPECIFIED = 0; + MIGRATION_OWNERSHIP_ROLE_OWNER = 1; + MIGRATION_OWNERSHIP_ROLE_NONOWNER = 2; +} + +message MigrationOwnershipProof { + string generation = 1; + uint64 local_group_id = 2; + uint64 owner_group_id = 3; + uint64 catalog_version = 4; + uint64 min_write_ts_exclusive = 5; + bytes route_start = 6; + bytes route_end = 7; + uint64 route_id = 8; + uint64 parent_route_id = 9; + bool staged = 10; + MigrationOwnershipRole role = 11; + bytes descriptor_hash = 12; + bool write_floor_known = 13; + uint64 source_tracker_version = 14; + uint64 source_applied_index = 15; +} + +message CleanupMigrationRequest { + uint64 job_id = 1; + MigrationCleanupMode mode = 2; + bytes range_start = 3; + bytes range_end = 4; + bytes cursor = 5; + uint64 max_commit_ts = 6; + uint32 max_versions = 7; + uint64 max_bytes = 8; + uint64 max_scanned_bytes = 9; + uint32 key_family = 10; + bytes route_start = 11; + bytes route_end = 12; + bool exclude_known_internal = 13; + repeated bytes exclude_prefixes = 14; + bool requires_route_key_check = 15; + bool requires_decoded_s3 = 16; + // Source cleanup is conditional on the all-voter ACK configuration, checked + // against the FSM's durable ConfChange index before any cleanup side effect. + uint64 expected_configuration_index = 17; + bool require_configuration_index = 18; + MigrationOwnershipProof ownership = 19; + // Strict canonical bounded JSON control; never a raw metadata mutation. + bytes partition_transition = 20; + // Versioned namespace activation/prejoin control. Never propose until every + // apply member (including learners) acknowledged support at this ConfChange. + bytes namespace_admission = 21; +} + +message CleanupMigrationResponse { + bytes next_cursor = 1; + bool done = 2; + uint64 deleted_rows = 3; + uint64 deleted_bytes = 4; + uint64 scanned_bytes = 5; +} + +enum MigrationStateProbeKind { + MIGRATION_STATE_PROBE_KIND_UNSPECIFIED = 0; + MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED = 1; + MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED = 2; + MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED = 3; + MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED = 4; + MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED = 5; + // Identifies the durable source guard by job and moving range, independently + // of compactable catalog route history. Target-only readiness is excluded. + MIGRATION_STATE_PROBE_KIND_SOURCE_GUARD_PRESENT = 6; + MIGRATION_STATE_PROBE_KIND_OWNERSHIP_CAPABILITY = 7; + MIGRATION_STATE_PROBE_KIND_OWNERSHIP_APPLIED = 8; + MIGRATION_STATE_PROBE_KIND_PARTITION_APPLIED = 9; + MIGRATION_STATE_PROBE_KIND_NAMESPACE_ADMISSION = 10; +} + +message ProbeMigrationStateRequest { + uint64 job_id = 1; + MigrationStateProbeKind kind = 2; + bytes route_start = 3; + bytes route_end = 4; + uint64 expected_catalog_version = 5; + uint64 expected_group_id = 6; + uint64 migration_job_id = 7; + uint64 min_write_ts_exclusive = 8; + bool source_write_fence = 9; + bool source_read_fence = 10; + bool track_writes = 11; + uint64 retention_pin_ts = 12; + int64 read_drain_not_before_ms = 13; + // Reads pinned at or below this HLC timestamp were admitted before the + // source read fence armed, so only those still block SOURCE_READ_DRAINED. + // Zero keeps the legacy behaviour of requiring an empty tracker. + uint64 read_drain_min_ts = 14; + bool require_configuration_index = 15; + uint64 expected_configuration_index = 16; + MigrationOwnershipProof ownership = 17; + uint32 transfer_size_guard_version = 18; + bytes partition_transition = 19; +} + +message ProbeMigrationStateResponse { + bool ready = 1; + uint64 catalog_version = 2; + uint64 min_admitted_ts = 3; + uint64 applied_configuration_index = 4; + string ownership_generation = 5; + uint32 ownership_protocol = 6; + MigrationOwnershipProof ownership = 7; + bool ownership_cleared = 8; + // Local support, not proof that the requested job's guard is applied. + uint32 transfer_size_guard_version = 9; + // Installed support, observable before a candidate has an applied ConfState. + // This is not operational ownership readiness or an activation override. + uint32 ownership_partition_protocol = 10; + // Installed support only; the durable group activation is separately probed. + uint32 namespace_admission_version = 11; + bytes partition_receipt_id = 12; +} + +message IssueMigrationTimestampRequest {} + +message IssueMigrationTimestampResponse { + uint64 timestamp = 1; + uint64 last_commit_ts = 2; +} diff --git a/proto/internal_grpc.pb.go b/proto/internal_grpc.pb.go index 53778ea87..2ef933011 100644 --- a/proto/internal_grpc.pb.go +++ b/proto/internal_grpc.pb.go @@ -19,14 +19,19 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Internal_Forward_FullMethodName = "/Internal/Forward" - Internal_ForwardAdminProposal_FullMethodName = "/Internal/ForwardAdminProposal" - Internal_ForwardLeaseRead_FullMethodName = "/Internal/ForwardLeaseRead" - Internal_GetPrepareEvidenceState_FullMethodName = "/Internal/GetPrepareEvidenceState" - Internal_RelayPublish_FullMethodName = "/Internal/RelayPublish" - Internal_ExportRangeVersions_FullMethodName = "/Internal/ExportRangeVersions" - Internal_ImportRangeVersions_FullMethodName = "/Internal/ImportRangeVersions" - Internal_PromoteStagedVersions_FullMethodName = "/Internal/PromoteStagedVersions" + Internal_Forward_FullMethodName = "/Internal/Forward" + Internal_ForwardAdminProposal_FullMethodName = "/Internal/ForwardAdminProposal" + Internal_ForwardLeaseRead_FullMethodName = "/Internal/ForwardLeaseRead" + Internal_GetPrepareEvidenceState_FullMethodName = "/Internal/GetPrepareEvidenceState" + Internal_RelayPublish_FullMethodName = "/Internal/RelayPublish" + Internal_ExportRangeVersions_FullMethodName = "/Internal/ExportRangeVersions" + Internal_ImportRangeVersions_FullMethodName = "/Internal/ImportRangeVersions" + Internal_PromoteStagedVersions_FullMethodName = "/Internal/PromoteStagedVersions" + Internal_ApplyTargetStagedReadiness_FullMethodName = "/Internal/ApplyTargetStagedReadiness" + Internal_ProbeMigrationLocks_FullMethodName = "/Internal/ProbeMigrationLocks" + Internal_CleanupMigration_FullMethodName = "/Internal/CleanupMigration" + Internal_ProbeMigrationState_FullMethodName = "/Internal/ProbeMigrationState" + Internal_IssueMigrationTimestamp_FullMethodName = "/Internal/IssueMigrationTimestamp" ) // InternalClient is the client API for Internal service. @@ -42,6 +47,11 @@ type InternalClient interface { ExportRangeVersions(ctx context.Context, in *ExportRangeVersionsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ExportRangeVersionsResponse], error) ImportRangeVersions(ctx context.Context, in *ImportRangeVersionsRequest, opts ...grpc.CallOption) (*ImportRangeVersionsResponse, error) PromoteStagedVersions(ctx context.Context, in *PromoteStagedVersionsRequest, opts ...grpc.CallOption) (*PromoteStagedVersionsResponse, error) + ApplyTargetStagedReadiness(ctx context.Context, in *TargetStagedReadinessRequest, opts ...grpc.CallOption) (*TargetStagedReadinessResponse, error) + ProbeMigrationLocks(ctx context.Context, in *ProbeMigrationLocksRequest, opts ...grpc.CallOption) (*ProbeMigrationLocksResponse, error) + CleanupMigration(ctx context.Context, in *CleanupMigrationRequest, opts ...grpc.CallOption) (*CleanupMigrationResponse, error) + ProbeMigrationState(ctx context.Context, in *ProbeMigrationStateRequest, opts ...grpc.CallOption) (*ProbeMigrationStateResponse, error) + IssueMigrationTimestamp(ctx context.Context, in *IssueMigrationTimestampRequest, opts ...grpc.CallOption) (*IssueMigrationTimestampResponse, error) } type internalClient struct { @@ -141,6 +151,56 @@ func (c *internalClient) PromoteStagedVersions(ctx context.Context, in *PromoteS return out, nil } +func (c *internalClient) ApplyTargetStagedReadiness(ctx context.Context, in *TargetStagedReadinessRequest, opts ...grpc.CallOption) (*TargetStagedReadinessResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(TargetStagedReadinessResponse) + err := c.cc.Invoke(ctx, Internal_ApplyTargetStagedReadiness_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *internalClient) ProbeMigrationLocks(ctx context.Context, in *ProbeMigrationLocksRequest, opts ...grpc.CallOption) (*ProbeMigrationLocksResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ProbeMigrationLocksResponse) + err := c.cc.Invoke(ctx, Internal_ProbeMigrationLocks_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *internalClient) CleanupMigration(ctx context.Context, in *CleanupMigrationRequest, opts ...grpc.CallOption) (*CleanupMigrationResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CleanupMigrationResponse) + err := c.cc.Invoke(ctx, Internal_CleanupMigration_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *internalClient) ProbeMigrationState(ctx context.Context, in *ProbeMigrationStateRequest, opts ...grpc.CallOption) (*ProbeMigrationStateResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ProbeMigrationStateResponse) + err := c.cc.Invoke(ctx, Internal_ProbeMigrationState_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *internalClient) IssueMigrationTimestamp(ctx context.Context, in *IssueMigrationTimestampRequest, opts ...grpc.CallOption) (*IssueMigrationTimestampResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(IssueMigrationTimestampResponse) + err := c.cc.Invoke(ctx, Internal_IssueMigrationTimestamp_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // InternalServer is the server API for Internal service. // All implementations must embed UnimplementedInternalServer // for forward compatibility. @@ -154,6 +214,11 @@ type InternalServer interface { ExportRangeVersions(*ExportRangeVersionsRequest, grpc.ServerStreamingServer[ExportRangeVersionsResponse]) error ImportRangeVersions(context.Context, *ImportRangeVersionsRequest) (*ImportRangeVersionsResponse, error) PromoteStagedVersions(context.Context, *PromoteStagedVersionsRequest) (*PromoteStagedVersionsResponse, error) + ApplyTargetStagedReadiness(context.Context, *TargetStagedReadinessRequest) (*TargetStagedReadinessResponse, error) + ProbeMigrationLocks(context.Context, *ProbeMigrationLocksRequest) (*ProbeMigrationLocksResponse, error) + CleanupMigration(context.Context, *CleanupMigrationRequest) (*CleanupMigrationResponse, error) + ProbeMigrationState(context.Context, *ProbeMigrationStateRequest) (*ProbeMigrationStateResponse, error) + IssueMigrationTimestamp(context.Context, *IssueMigrationTimestampRequest) (*IssueMigrationTimestampResponse, error) mustEmbedUnimplementedInternalServer() } @@ -188,6 +253,21 @@ func (UnimplementedInternalServer) ImportRangeVersions(context.Context, *ImportR func (UnimplementedInternalServer) PromoteStagedVersions(context.Context, *PromoteStagedVersionsRequest) (*PromoteStagedVersionsResponse, error) { return nil, status.Error(codes.Unimplemented, "method PromoteStagedVersions not implemented") } +func (UnimplementedInternalServer) ApplyTargetStagedReadiness(context.Context, *TargetStagedReadinessRequest) (*TargetStagedReadinessResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ApplyTargetStagedReadiness not implemented") +} +func (UnimplementedInternalServer) ProbeMigrationLocks(context.Context, *ProbeMigrationLocksRequest) (*ProbeMigrationLocksResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ProbeMigrationLocks not implemented") +} +func (UnimplementedInternalServer) CleanupMigration(context.Context, *CleanupMigrationRequest) (*CleanupMigrationResponse, error) { + return nil, status.Error(codes.Unimplemented, "method CleanupMigration not implemented") +} +func (UnimplementedInternalServer) ProbeMigrationState(context.Context, *ProbeMigrationStateRequest) (*ProbeMigrationStateResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ProbeMigrationState not implemented") +} +func (UnimplementedInternalServer) IssueMigrationTimestamp(context.Context, *IssueMigrationTimestampRequest) (*IssueMigrationTimestampResponse, error) { + return nil, status.Error(codes.Unimplemented, "method IssueMigrationTimestamp not implemented") +} func (UnimplementedInternalServer) mustEmbedUnimplementedInternalServer() {} func (UnimplementedInternalServer) testEmbeddedByValue() {} @@ -346,6 +426,96 @@ func _Internal_PromoteStagedVersions_Handler(srv interface{}, ctx context.Contex return interceptor(ctx, in, info, handler) } +func _Internal_ApplyTargetStagedReadiness_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(TargetStagedReadinessRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).ApplyTargetStagedReadiness(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_ApplyTargetStagedReadiness_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).ApplyTargetStagedReadiness(ctx, req.(*TargetStagedReadinessRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Internal_ProbeMigrationLocks_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ProbeMigrationLocksRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).ProbeMigrationLocks(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_ProbeMigrationLocks_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).ProbeMigrationLocks(ctx, req.(*ProbeMigrationLocksRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Internal_CleanupMigration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CleanupMigrationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).CleanupMigration(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_CleanupMigration_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).CleanupMigration(ctx, req.(*CleanupMigrationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Internal_ProbeMigrationState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ProbeMigrationStateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).ProbeMigrationState(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_ProbeMigrationState_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).ProbeMigrationState(ctx, req.(*ProbeMigrationStateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Internal_IssueMigrationTimestamp_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(IssueMigrationTimestampRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).IssueMigrationTimestamp(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_IssueMigrationTimestamp_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).IssueMigrationTimestamp(ctx, req.(*IssueMigrationTimestampRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Internal_ServiceDesc is the grpc.ServiceDesc for Internal service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -381,6 +551,26 @@ var Internal_ServiceDesc = grpc.ServiceDesc{ MethodName: "PromoteStagedVersions", Handler: _Internal_PromoteStagedVersions_Handler, }, + { + MethodName: "ApplyTargetStagedReadiness", + Handler: _Internal_ApplyTargetStagedReadiness_Handler, + }, + { + MethodName: "ProbeMigrationLocks", + Handler: _Internal_ProbeMigrationLocks_Handler, + }, + { + MethodName: "CleanupMigration", + Handler: _Internal_CleanupMigration_Handler, + }, + { + MethodName: "ProbeMigrationState", + Handler: _Internal_ProbeMigrationState_Handler, + }, + { + MethodName: "IssueMigrationTimestamp", + Handler: _Internal_IssueMigrationTimestamp_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 80bc23a7e..1d2258832 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -25,6 +25,9 @@ func (s *pebbleStore) exportVersionsLocked(ctx context.Context, opts ExportVersi if opts.MaxVersions <= 0 { return ExportVersionsResult{}, errors.WithStack(ErrInvalidExportBudget) } + if readTSCompacted(exportRetentionReadTS(opts), s.effectiveMinRetainedTS()) { + return ExportVersionsResult{}, ErrReadTSCompacted + } iter, err := s.db.NewIter(pebbleExportIterOptions(opts)) if err != nil { @@ -392,6 +395,9 @@ func (s *pebbleStore) commitPebbleImportAppliedIndex(appliedIndex uint64, writeO } func (s *pebbleStore) validatePebbleImportBatch(opts ImportVersionsOptions) (bool, []byte, error) { + if err := s.checkMigrationNotRetiredLocked(opts.JobID); err != nil { + return false, nil, err + } existing, hasExisting, err := s.readMigrationImportAck(opts.JobID, opts.BracketID) if err != nil { return false, nil, err @@ -579,6 +585,28 @@ func (s *pebbleStore) MigrationHLCFloor(_ context.Context, jobID uint64) (uint64 return floor, nil } +func (s *pebbleStore) MigrationImportMetadataPresent(_ context.Context, jobID uint64) (bool, error) { + s.dbMu.RLock() + defer s.dbMu.RUnlock() + floors, err := s.readMigrationHLCFloors() + if err != nil { + return false, err + } + if floors[jobID] != 0 { + return true, nil + } + acks, err := s.readMigrationImportAcks() + if err != nil { + return false, err + } + for id := range acks { + if id.jobID == jobID { + return true, nil + } + } + return false, nil +} + func (s *pebbleStore) RetireMigration(ctx context.Context, jobID uint64) error { return s.retireMigrationWithOpts(ctx, jobID, s.directApplyWriteOpts(), 0) } @@ -593,6 +621,9 @@ func (s *pebbleStore) retireMigrationWithOpts(ctx context.Context, jobID uint64, } s.dbMu.Lock() defer s.dbMu.Unlock() + if _, err := s.readMigrationRetiredJobs(); err != nil { + return err + } batch := s.db.NewBatch() defer batch.Close() diff --git a/store/lsm_store.go b/store/lsm_store.go index 18cfb151d..3cc2faf3a 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -241,19 +241,21 @@ var metaAppliedIndexBytes = []byte(metaAppliedIndex) // DB is being swapped (Restore/Close) or checkpointed, and as a // read-lock by every operation that accesses s.db. // 4. mtx – guards the in-memory metadata fields -// (lastCommitTS, minRetainedTS, pendingMinRetainedTS). +// (lastCommitTS, minRetainedTS, pendingMinRetainedTS, migrationReadinessCache). type pebbleStore struct { - db *pebble.DB - cache *pebble.Cache // owns one ref; Unref on Close / reopen. - dbMu sync.RWMutex // guards s.db pointer – see lock ordering above - log *slog.Logger - lastCommitTS uint64 - minRetainedTS uint64 - pendingMinRetainedTS uint64 - mtx sync.RWMutex - applyMu sync.Mutex // serializes ApplyMutations: conflict check → commit - maintenanceMu sync.Mutex - dir string + db *pebble.DB + cache *pebble.Cache // owns one ref; Unref on Close / reopen. + dbMu sync.RWMutex // guards s.db pointer – see lock ordering above + log *slog.Logger + lastCommitTS uint64 + minRetainedTS uint64 + pendingMinRetainedTS uint64 + migrationReadinessCache []TargetStagedReadinessState + migrationAdmission migrationServingAdmissionIndex + mtx sync.RWMutex + applyMu sync.Mutex // serializes ApplyMutations: conflict check → commit + maintenanceMu sync.Mutex + dir string // sstIngestSnapshots enables the cluster-wide opt-in snapshot format // implemented in snapshot_pebble_sst.go. Receivers always understand the // format; senders keep the legacy stream by default for rolling-upgrade @@ -453,9 +455,15 @@ func NewPebbleStore(dir string, opts ...PebbleStoreOption) (MVCCStore, error) { cleanupOnInitFail() return nil, err } + readinessStates, err := s.loadMigrationServingMetadata() + if err != nil { + cleanupOnInitFail() + return nil, err + } s.lastCommitTS = maxTS s.minRetainedTS = minRetainedTS s.pendingMinRetainedTS = pendingMinRetainedTS + s.migrationReadinessCache = readinessStates // A backup is only stale after the live directory has opened and its // metadata has been read successfully. Before this point it is the recovery // source for a process crash between the two restore renames. @@ -719,6 +727,12 @@ func writeTempDBMetadata(db *pebble.DB, meta streamingMVCCRestoreMetadata) error if err := batch.Set(migrationPromoteMetaKeyBytes, encodeMigrationPromotionStates(meta.migrationPromotions), nil); err != nil { return errors.WithStack(err) } + if err := batch.Set(migrationRetiredMetaKeyBytes, encodeMigrationRetiredJobs(meta.migrationRetired), nil); err != nil { + return errors.WithStack(err) + } + if err := stageSnapshotServingMetadata(batch, meta); err != nil { + return err + } return errors.WithStack(batch.Commit(pebble.Sync)) } @@ -3317,6 +3331,10 @@ func restoreBatchLoopInto(r io.Reader, db *pebble.DB) error { } func (s *pebbleStore) Restore(r io.Reader) error { + return s.restoreSnapshot(r, nil) +} + +func (s *pebbleStore) restoreSnapshot(r io.Reader, validate func(string) error) error { s.maintenanceMu.Lock() defer s.maintenanceMu.Unlock() @@ -3339,14 +3357,14 @@ func (s *pebbleStore) Restore(r io.Reader) error { case bytes.Equal(header, pebbleSnapshotMagic[:]): // Native Pebble format: restorePebbleNativeAtomic performs a temp-dir // swap so the existing DB is preserved if the restore fails midway. - return s.restorePebbleNativeAtomic(br) + return s.restorePebbleNativeAtomic(br, validate) case bytes.Equal(header, pebbleSSTIngestSnapshotMagic[:]): - return s.restorePebbleSSTIngestAtomic(br) + return s.restorePebbleSSTIngestAtomic(br, validate) case bytes.Equal(header, mvccSnapshotMagic[:]): // Streaming MVCC format: restoreFromStreamingMVCC performs an atomic // temp-dir swap, so the existing DB is preserved until checksum // verification succeeds. - return s.restoreFromStreamingMVCC(br) + return s.restoreFromStreamingMVCC(br, validate) default: return errors.WithStack(errors.Newf("unrecognized snapshot format: unknown magic header")) } @@ -3369,6 +3387,8 @@ func (s *pebbleStore) handleRestorePeekError(err error, header []byte) error { s.lastCommitTS = 0 s.minRetainedTS = 0 s.pendingMinRetainedTS = 0 + s.migrationReadinessCache = nil + s.migrationAdmission = migrationServingAdmissionIndex{} if setErr := writePebbleUint64(s.db, metaLastCommitTSBytes, 0, pebble.NoSync); setErr != nil { return errors.WithStack(setErr) } @@ -3409,7 +3429,7 @@ func (s *pebbleStore) reopenFreshDB() error { // It writes all entries into a temporary sibling directory and only swaps the // temp directory into s.dir after a full successful read, preserving the // existing DB on any failure. -func (s *pebbleStore) restorePebbleNativeAtomic(r io.Reader) error { +func (s *pebbleStore) restorePebbleNativeAtomic(r io.Reader, validate func(string) error) error { magic, ts, err := readPebbleNativeHeader(r) if err != nil { return err @@ -3428,6 +3448,9 @@ func (s *pebbleStore) restorePebbleNativeAtomic(r io.Reader) error { return err } + if err := validateRestoreTempDB(tmpDir, validate); err != nil { + return err + } if err := s.swapInTempDB(tmpDir); err != nil { return err } @@ -3503,6 +3526,10 @@ type streamingMVCCRestoreMetadata struct { migrationAcks map[migrationAckID]migrationImportAck migrationHLCFloors map[uint64]uint64 migrationPromotions map[uint64]PromotionState + migrationReadiness []TargetStagedReadinessState + migrationRetired map[uint64]struct{} + migrationOwnership []MigrationOwnershipState + migrationPartitions MigrationOwnershipPartitionState } func readStreamingMVCCRestoreHeader(r io.Reader) (io.Reader, hash.Hash32, uint32, streamingMVCCRestoreMetadata, error) { @@ -3513,16 +3540,20 @@ func readStreamingMVCCRestoreHeader(r io.Reader) (io.Reader, hash.Hash32, uint32 hash := crc32.NewIEEE() body := io.TeeReader(r, hash) - lastCommitTS, minRetainedTS, migrationAcks, migrationHLCFloors, migrationPromotions, err := readMVCCSnapshotMetadata(body, version) + lastCommitTS, minRetainedTS, snapshotMeta, err := readMVCCSnapshotMetadata(body, version) if err != nil { return nil, nil, 0, streamingMVCCRestoreMetadata{}, err } meta := streamingMVCCRestoreMetadata{ lastCommitTS: lastCommitTS, minRetainedTS: minRetainedTS, - migrationAcks: migrationAcks, - migrationHLCFloors: migrationHLCFloors, - migrationPromotions: migrationPromotions, + migrationAcks: snapshotMeta.acks, + migrationHLCFloors: snapshotMeta.floors, + migrationPromotions: snapshotMeta.promotions, + migrationReadiness: snapshotMeta.readiness, + migrationRetired: snapshotMeta.retired, + migrationOwnership: snapshotMeta.ownership, + migrationPartitions: snapshotMeta.partitions, } return body, hash, expectedChecksum, meta, nil } @@ -3566,7 +3597,7 @@ func writeStreamingMVCCRestoreTempDB(dir string, body io.Reader, hash hash.Hash3 return tmpDir, nil } -func (s *pebbleStore) restoreFromStreamingMVCC(r io.Reader) error { +func (s *pebbleStore) restoreFromStreamingMVCC(r io.Reader, validate func(string) error) error { body, hash, expectedChecksum, meta, err := readStreamingMVCCRestoreHeader(r) if err != nil { return err @@ -3576,6 +3607,9 @@ func (s *pebbleStore) restoreFromStreamingMVCC(r io.Reader) error { if err != nil { return err } + if err := validateRestoreTempDB(tmpDir, validate); err != nil { + return err + } return s.swapInTempDB(tmpDir) } @@ -3646,9 +3680,14 @@ func (s *pebbleStore) swapInTempDBWithMetadata(tmpDir string, expected *pebbleSn if err != nil { return errors.Join(err, s.restoreSwapBackup(backupDir)) } + readinessStates, err := s.loadMigrationServingMetadata() + if err != nil { + return errors.Join(err, s.restoreSwapBackup(backupDir)) + } s.lastCommitTS = lastCommitTS s.minRetainedTS = minRetainedTS s.pendingMinRetainedTS = pendingMinRetainedTS + s.migrationReadinessCache = readinessStates if err := os.RemoveAll(backupDir); err != nil { s.log.Warn("failed to remove restore backup", "path", backupDir, "error", err) } diff --git a/store/lsm_store_test.go b/store/lsm_store_test.go index 9c7dbfd3d..ba0dade77 100644 --- a/store/lsm_store_test.go +++ b/store/lsm_store_test.go @@ -924,3 +924,47 @@ func TestPebbleStore_SnapshotRestore_MaxSizeKey(t *testing.T) { require.NoError(t, err) assert.Equal(t, []byte("val"), got) } + +func TestPebbleStore_RestoreFromStreamingMVCCPreservesMigrationImportMetadata(t *testing.T) { + ctx := context.Background() + src := NewMVCCStore() + _, err := src.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 1, + Cursor: []byte("c1"), + Versions: []MVCCVersion{{Key: []byte("snapshotted"), CommitTS: 50, Value: []byte("v50")}}, + }) + require.NoError(t, err) + + snap, err := src.Snapshot() + require.NoError(t, err) + defer snap.Close() + + dir, err := os.MkdirTemp("", "pebble-migrate-import-metadata-*") + require.NoError(t, err) + defer os.RemoveAll(dir) + dst, err := NewPebbleStore(dir) + require.NoError(t, err) + defer dst.Close() + require.NoError(t, dst.Restore(bytes.NewReader(snapshotBytes(t, snap)))) + + val, err := dst.GetAt(ctx, []byte("snapshotted"), 50) + require.NoError(t, err) + require.Equal(t, []byte("v50"), val) + floor, err := dst.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Equal(t, uint64(50), floor) + res, err := dst.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 1, + Cursor: []byte("fresh"), + Versions: []MVCCVersion{{Key: []byte("fresh"), CommitTS: 60, Value: []byte("v60")}}, + }) + require.NoError(t, err) + require.True(t, res.Duplicate) + require.Equal(t, []byte("c1"), res.AckedCursor) + _, err = dst.GetAt(ctx, []byte("fresh"), 60) + require.ErrorIs(t, err, ErrKeyNotFound) +} diff --git a/store/migration_cleanup.go b/store/migration_cleanup.go new file mode 100644 index 000000000..ce1240960 --- /dev/null +++ b/store/migration_cleanup.go @@ -0,0 +1,227 @@ +package store + +import ( + "bytes" + "context" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +func cleanupExportOptions(opts CleanupVersionsOptions) ExportVersionsOptions { + return ExportVersionsOptions{ + StartKey: opts.StartKey, + EndKey: opts.EndKey, + Cursor: opts.Cursor, + MaxCommitTSInclusive: opts.MaxCommitTS, + MaxVersions: opts.MaxVersions, + MaxBytes: opts.MaxBytes, + MaxScannedBytes: opts.MaxScannedBytes, + KeyFamily: opts.KeyFamily, + AcceptVersion: opts.AcceptVersion, + } +} + +func cleanupResult(exported ExportVersionsResult) CleanupVersionsResult { + var deletedBytes uint64 + for _, version := range exported.Versions { + deletedBytes += versionExportSize(version.Key, len(version.Value)) + } + return CleanupVersionsResult{ + NextCursor: bytes.Clone(exported.NextCursor), + Done: exported.Done, + DeletedRows: uint64(len(exported.Versions)), //nolint:gosec // bounded by MaxVersions. + DeletedBytes: deletedBytes, + ScannedBytes: exported.ScannedBytes, + } +} + +func (s *mvccStore) CleanupVersions(ctx context.Context, opts CleanupVersionsOptions) (CleanupVersionsResult, error) { + s.mtx.Lock() + defer s.mtx.Unlock() + if _, retired := s.migrationRetired[opts.JobID]; retired { + return CleanupVersionsResult{}, errors.WithStack(ErrMigrationRetired) + } + if opts.MaxVersions <= 0 { + return CleanupVersionsResult{Done: true}, nil + } + + exportOpts := normalizeExportVersionsOptions(cleanupExportOptions(opts)) + pos, err := decodeExportCursorForOptions(exportOpts) + if err != nil { + return CleanupVersionsResult{}, err + } + exported, err := s.exportVersionsLocked(ctx, exportOpts, pos) + if err != nil { + return CleanupVersionsResult{}, err + } + for _, version := range exported.Versions { + s.removeVersionLocked(version.Key, version.CommitTS) + } + return cleanupResult(exported), nil +} + +func (s *mvccStore) ClearMigrationState(_ context.Context, jobID, _ uint64) error { + if jobID == 0 { + return errors.New("migration cleanup job id is required") + } + s.mtx.Lock() + defer s.mtx.Unlock() + s.clearMigrationStateLocked(jobID) + return nil +} + +func (s *mvccStore) clearMigrationStateLocked(jobID uint64) { + for id := range s.migrationAcks { + if id.jobID == jobID { + delete(s.migrationAcks, id) + } + } + delete(s.migrationHLCFloors, jobID) + delete(s.migrationPromotions, jobID) + delete(s.migrationReadiness, jobID) + s.migrationReadinessCache = upsertReadinessCacheWithoutJob(s.migrationReadinessCache, jobID) + if s.migrationRetired == nil { + s.migrationRetired = make(map[uint64]struct{}) + } + s.migrationRetired[jobID] = struct{}{} +} + +func upsertReadinessCacheWithoutJob(states []TargetStagedReadinessState, jobID uint64) []TargetStagedReadinessState { + out := states[:0] + for _, state := range states { + if state.JobID != jobID { + out = append(out, state) + } + } + return out +} + +func (s *pebbleStore) CleanupVersions(ctx context.Context, opts CleanupVersionsOptions) (CleanupVersionsResult, error) { + s.applyMu.Lock() + defer s.applyMu.Unlock() + s.dbMu.RLock() + defer s.dbMu.RUnlock() + if err := s.checkMigrationNotRetiredLocked(opts.JobID); err != nil { + return CleanupVersionsResult{}, err + } + if opts.MaxVersions <= 0 { + return CleanupVersionsResult{Done: true}, nil + } + + exported, err := s.exportVersionsLocked(ctx, cleanupExportOptions(opts)) + if err != nil { + return CleanupVersionsResult{}, err + } + batch := s.db.NewBatch() + defer batch.Close() + for _, version := range exported.Versions { + if err := batch.Delete(encodeKey(version.Key, version.CommitTS), nil); err != nil { + return CleanupVersionsResult{}, errors.WithStack(err) + } + } + if opts.AppliedIndex > 0 { + if err := setPebbleUint64InBatch(batch, metaAppliedIndexBytes, opts.AppliedIndex); err != nil { + return CleanupVersionsResult{}, err + } + } + if err := batch.Commit(s.cleanupWriteOpts(opts.AppliedIndex)); err != nil { + return CleanupVersionsResult{}, errors.WithStack(err) + } + return cleanupResult(exported), nil +} + +func (s *pebbleStore) ClearMigrationState(_ context.Context, jobID, appliedIndex uint64) error { + if jobID == 0 { + return errors.New("migration cleanup job id is required") + } + s.applyMu.Lock() + defer s.applyMu.Unlock() + s.dbMu.RLock() + defer s.dbMu.RUnlock() + + metadata, err := s.migrationMetadataWithoutJob(jobID) + if err != nil { + return err + } + + batch := s.db.NewBatch() + defer batch.Close() + if err := stageMigrationMetadataCleanup(batch, jobID, appliedIndex, metadata); err != nil { + return err + } + if err := batch.Commit(s.cleanupWriteOpts(appliedIndex)); err != nil { + return errors.WithStack(err) + } + s.mtx.Lock() + s.migrationReadinessCache = upsertReadinessCacheWithoutJob(s.migrationReadinessCache, jobID) + s.mtx.Unlock() + return nil +} + +type migrationMetadataState struct { + acks map[migrationAckID]migrationImportAck + floors map[uint64]uint64 + promotions map[uint64]PromotionState + retired map[uint64]struct{} +} + +func (s *pebbleStore) migrationMetadataWithoutJob(jobID uint64) (migrationMetadataState, error) { + acks, err := s.readMigrationImportAcks() + if err != nil { + return migrationMetadataState{}, err + } + for id := range acks { + if id.jobID == jobID { + delete(acks, id) + } + } + floors, err := s.readMigrationHLCFloors() + if err != nil { + return migrationMetadataState{}, err + } + delete(floors, jobID) + promotions, err := s.readPebblePromotionStates() + if err != nil { + return migrationMetadataState{}, err + } + delete(promotions, jobID) + retired, err := s.readMigrationRetiredJobs() + if err != nil { + return migrationMetadataState{}, err + } + retired[jobID] = struct{}{} + return migrationMetadataState{acks: acks, floors: floors, promotions: promotions, retired: retired}, nil +} + +func stageMigrationMetadataCleanup(batch *pebble.Batch, jobID, appliedIndex uint64, metadata migrationMetadataState) error { + if err := batch.Set(migrationAckMetaKeyBytes, encodeMigrationImportAcks(metadata.acks), nil); err != nil { + return errors.WithStack(err) + } + if err := batch.Set(migrationHLCFloorMetaKeyBytes, encodeMigrationHLCFloors(metadata.floors), nil); err != nil { + return errors.WithStack(err) + } + if err := batch.Set(migrationPromoteMetaKeyBytes, encodeMigrationPromotionStates(metadata.promotions), nil); err != nil { + return errors.WithStack(err) + } + if err := batch.Delete(migrationReadyKey(jobID), nil); err != nil { + return errors.WithStack(err) + } + if err := batch.Set(migrationRetiredMetaKeyBytes, encodeMigrationRetiredJobs(metadata.retired), nil); err != nil { + return errors.WithStack(err) + } + if appliedIndex == 0 { + return nil + } + return setPebbleUint64InBatch(batch, metaAppliedIndexBytes, appliedIndex) +} + +func (s *pebbleStore) cleanupWriteOpts(appliedIndex uint64) *pebble.WriteOptions { + if appliedIndex > 0 { + return s.raftApplyWriteOpts() + } + return s.directApplyWriteOpts() +} + +var _ MigrationCleaner = (*mvccStore)(nil) +var _ MigrationCleaner = (*pebbleStore)(nil) diff --git a/store/migration_configuration_key.go b/store/migration_configuration_key.go new file mode 100644 index 000000000..0a99f16df --- /dev/null +++ b/store/migration_configuration_key.go @@ -0,0 +1,53 @@ +package store + +import ( + "bytes" + "context" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +// MigrationConfigurationIndexKey is the durable Raft configuration epoch, not +// the snapshot's data applied index. Return private bytes for each caller. +func MigrationConfigurationIndexKey() []byte { + return []byte("!txn|meta|migration_configuration_index") +} + +// Only the unpublished external-restore tempDB is writable here. Remove the +// source Raft watermark as well: it cannot suppress destination Raft writes. +// An exact decoded-key comparison preserves adjacent logical keys. +func sanitizeExternalConfigurationEpoch(ctx context.Context, db *pebble.DB) error { + key := MigrationConfigurationIndexKey() + iter, err := db.NewIter(&pebble.IterOptions{LowerBound: key, UpperBound: PrefixScanEnd(key)}) + if err != nil { + return errors.WithStack(err) + } + defer iter.Close() + batch := db.NewBatch() + defer func() { _ = batch.Close() }() + for valid := iter.First(); valid; valid = iter.Next() { + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + logicalKey, _ := decodeKeyView(iter.Key()) + if !bytes.Equal(logicalKey, key) { + continue + } + if err := batch.Delete(iter.Key(), nil); err != nil { + return errors.WithStack(err) + } + if snapshotBatchShouldFlush(batch) { + if err := flushSnapshotBatch(db, &batch, pebble.Sync); err != nil { + return err + } + } + } + if err := errors.Join(errors.WithStack(iter.Error()), errors.WithStack(ctx.Err())); err != nil { + return err + } + if err := batch.Delete(metaAppliedIndexBytes, nil); err != nil { + return errors.WithStack(err) + } + return errors.WithStack(batch.Commit(pebble.Sync)) +} diff --git a/store/migration_namespace_configuration.go b/store/migration_namespace_configuration.go new file mode 100644 index 000000000..e000520b2 --- /dev/null +++ b/store/migration_namespace_configuration.go @@ -0,0 +1,83 @@ +package store + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "slices" + + "github.com/cockroachdb/errors" +) + +const MigrationNamespaceVersion uint32 = 1 + +// MigrationNamespaceConfiguration extends the existing exact peer-local MVCC +// configuration row only after explicit protocol activation. Keep this codec +// shared by FSM and partition storage; neither may assume an eight-byte row. +type MigrationNamespaceConfiguration struct { + Index uint64 + Version uint32 + ActivatedAt uint64 + Members []uint64 + Joining []uint64 +} + +var migrationNamespaceConfigurationMagic = []byte("EKVNS1\x00") + +func DecodeMigrationConfigurationIndex(value []byte) (uint64, error) { + state, err := DecodeMigrationNamespaceConfiguration(value) + return state.Index, err +} + +func DecodeMigrationNamespaceConfiguration(value []byte) (MigrationNamespaceConfiguration, error) { + if len(value) == 8 && binary.BigEndian.Uint64(value) != 0 { + return MigrationNamespaceConfiguration{Index: binary.BigEndian.Uint64(value)}, nil + } + var state MigrationNamespaceConfiguration + if !bytes.HasPrefix(value, migrationNamespaceConfigurationMagic) { + return state, errors.New("invalid durable migration configuration index") + } + if err := json.Unmarshal(value[len(migrationNamespaceConfigurationMagic):], &state); err != nil { + return state, errors.Wrap(err, "invalid durable migration configuration envelope") + } + if !validMigrationNamespaceConfiguration(state) { + return state, errors.New("invalid durable migration namespace configuration") + } + return state, nil +} + +func EncodeMigrationNamespaceConfiguration(state MigrationNamespaceConfiguration) ([]byte, error) { + if state.Version == 0 { + if state.Index == 0 || state.ActivatedAt != 0 || len(state.Joining) != 0 { + return nil, errors.New("invalid legacy migration configuration") + } + return binary.BigEndian.AppendUint64(nil, state.Index), nil + } + if !validMigrationNamespaceConfiguration(state) { + return nil, errors.New("invalid durable migration namespace configuration") + } + encoded, err := json.Marshal(state) + if err != nil { + return nil, errors.WithStack(err) + } + return append(slices.Clone(migrationNamespaceConfigurationMagic), encoded...), nil +} + +func validMigrationNamespaceConfiguration(state MigrationNamespaceConfiguration) bool { + if state.Index == 0 || state.Version != MigrationNamespaceVersion || state.ActivatedAt == 0 || state.ActivatedAt > state.Index || !validMigrationNamespaceMemberIDs(state.Members) { + return false + } + if len(state.Joining) != 0 && !validMigrationNamespaceMemberIDs(state.Joining) { + return false + } + for _, id := range state.Joining { + if slices.Contains(state.Members, id) { + return false + } + } + return true +} + +func validMigrationNamespaceMemberIDs(ids []uint64) bool { + return len(ids) > 0 && ids[0] != 0 && slices.IsSorted(ids) && len(slices.Compact(slices.Clone(ids))) == len(ids) +} diff --git a/store/migration_namespace_configuration_test.go b/store/migration_namespace_configuration_test.go new file mode 100644 index 000000000..4443db7d9 --- /dev/null +++ b/store/migration_namespace_configuration_test.go @@ -0,0 +1,32 @@ +package store + +import ( + "encoding/binary" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMigrationNamespaceSharedConfigurationCodec(t *testing.T) { + for _, active := range []bool{false, true} { + state := MigrationNamespaceConfiguration{Index: 5} + if active { + state.Version, state.ActivatedAt, state.Members, state.Joining = MigrationNamespaceVersion, 3, []uint64{1, 2}, []uint64{3} + } + encoded, err := EncodeMigrationNamespaceConfiguration(state) + require.NoError(t, err) + index, err := DecodeMigrationConfigurationIndex(encoded) + require.NoError(t, err) + require.Equal(t, state.Index, index) + decoded, err := DecodeMigrationNamespaceConfiguration(encoded) + require.NoError(t, err) + require.Equal(t, state, decoded) + if !active { + require.Len(t, encoded, 8) + } + } + for _, malformed := range [][]byte{nil, {1}, binary.BigEndian.AppendUint64(nil, 0), []byte("EKVNS1\x00{}"), []byte("EKVNS1\x00{\"Index\":5,\"Version\":2}"), []byte("EKVNS1\x00{\"Index\":5,\"Version\":1,\"ActivatedAt\":3,\"Members\":[1,1]}")} { + _, err := DecodeMigrationConfigurationIndex(malformed) + require.Error(t, err) + } +} diff --git a/store/migration_namespace_property_test.go b/store/migration_namespace_property_test.go new file mode 100644 index 000000000..9c7e02cfe --- /dev/null +++ b/store/migration_namespace_property_test.go @@ -0,0 +1,113 @@ +package store + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "slices" + "testing" + + "github.com/stretchr/testify/require" + "pgregory.net/rapid" +) + +func codecPropertyNamespaceState(t *rapid.T) MigrationNamespaceConfiguration { + t.Helper() + index := rapid.Uint64Range(1, ^uint64(0)-1).Draw(t, "configuration index") + state := MigrationNamespaceConfiguration{ + Index: index, Version: MigrationNamespaceVersion, + ActivatedAt: rapid.Uint64Range(1, index).Draw(t, "activation index"), + } + base := rapid.Uint64Range(1, ^uint64(0)-100).Draw(t, "member base") + count := rapid.IntRange(2, 8).Draw(t, "member count") + for i := range count { + state.Members = append(state.Members, base+uint64(i)) + } + joining := rapid.IntRange(0, 4).Draw(t, "joining count") + for i := range joining { + state.Joining = append(state.Joining, base+uint64(count+i)) + } + return state +} + +func TestMigrationCodecPropertyNamespaceRoundTrip(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + legacy := MigrationNamespaceConfiguration{Index: rapid.Uint64Range(1, ^uint64(0)).Draw(t, "legacy index")} + active := codecPropertyNamespaceState(t) + for _, state := range []MigrationNamespaceConfiguration{legacy, active} { + encoded, err := EncodeMigrationNamespaceConfiguration(state) + require.NoError(t, err) + decoded, err := DecodeMigrationNamespaceConfiguration(encoded) + require.NoError(t, err) + require.Equal(t, state, decoded) + index, err := DecodeMigrationConfigurationIndex(encoded) + require.NoError(t, err) + require.Equal(t, state.Index, index) + reencoded, err := EncodeMigrationNamespaceConfiguration(decoded) + require.NoError(t, err) + require.Equal(t, encoded, reencoded) + if state.Version == 0 { + require.Equal(t, binary.BigEndian.AppendUint64(nil, state.Index), encoded) + } + } + // This codec intentionally permits unknown JSON fields; unlike the + // partition RPC codec, it has no strict/canonical-input contract. + encoded, err := EncodeMigrationNamespaceConfiguration(active) + require.NoError(t, err) + extended := append(bytes.Clone(encoded[:len(encoded)-1]), []byte(`,"FutureField":{"Enabled":true}}`)...) + decoded, err := DecodeMigrationNamespaceConfiguration(extended) + require.NoError(t, err) + require.Equal(t, active, decoded) + reencoded, err := EncodeMigrationNamespaceConfiguration(decoded) + require.NoError(t, err) + require.Equal(t, encoded, reencoded, "known state is canonicalized without preserving extensions") + }) +} + +func TestMigrationCodecPropertyNamespaceRejectsMalformed(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + state := codecPropertyNamespaceState(t) + mutations := []struct { + name string + mutate func(*MigrationNamespaceConfiguration) + }{ + {"zero index", func(s *MigrationNamespaceConfiguration) { s.Index = 0 }}, + {"unsupported version", func(s *MigrationNamespaceConfiguration) { s.Version++ }}, + {"missing activation", func(s *MigrationNamespaceConfiguration) { s.ActivatedAt = 0 }}, + {"future activation", func(s *MigrationNamespaceConfiguration) { s.ActivatedAt = s.Index + 1 }}, + {"missing members", func(s *MigrationNamespaceConfiguration) { s.Members = nil }}, + {"zero member", func(s *MigrationNamespaceConfiguration) { s.Members[0] = 0 }}, + {"duplicate member", func(s *MigrationNamespaceConfiguration) { s.Members[1] = s.Members[0] }}, + {"unsorted members", func(s *MigrationNamespaceConfiguration) { slices.Reverse(s.Members) }}, + {"joining overlaps", func(s *MigrationNamespaceConfiguration) { s.Joining = []uint64{s.Members[0]} }}, + {"duplicate joining", func(s *MigrationNamespaceConfiguration) { s.Joining = []uint64{1, 1} }}, + } + for _, mutation := range mutations { + bad := state + bad.Members, bad.Joining = slices.Clone(state.Members), slices.Clone(state.Joining) + mutation.mutate(&bad) + _, err := EncodeMigrationNamespaceConfiguration(bad) + require.Error(t, err, mutation.name) + payload, err := json.Marshal(bad) + require.NoError(t, err) + _, err = DecodeMigrationNamespaceConfiguration(append(bytes.Clone(migrationNamespaceConfigurationMagic), payload...)) + require.Error(t, err, mutation.name) + } + encoded, err := EncodeMigrationNamespaceConfiguration(state) + require.NoError(t, err) + for _, bad := range [][]byte{make([]byte, 8), encoded[:len(encoded)-1], append([]byte{0}, encoded[1:]...), append(bytes.Clone(encoded), '{')} { + _, err = DecodeMigrationConfigurationIndex(bad) + require.Error(t, err) + } + legacy := MigrationNamespaceConfiguration{Index: rapid.Uint64Range(1, ^uint64(0)).Draw(t, "legacy index")} + encoded, err = EncodeMigrationNamespaceConfiguration(legacy) + require.NoError(t, err) + for _, bad := range [][]byte{encoded[:7], append(bytes.Clone(encoded), 0)} { + _, err = DecodeMigrationConfigurationIndex(bad) + require.Error(t, err, "legacy configuration requires exactly eight bytes") + } + legacy.ActivatedAt = 1 + _, err = EncodeMigrationNamespaceConfiguration(legacy) + require.Error(t, err, "legacy configuration cannot carry activation state") + }) +} diff --git a/store/migration_ownership.go b/store/migration_ownership.go new file mode 100644 index 000000000..53052161f --- /dev/null +++ b/store/migration_ownership.go @@ -0,0 +1,687 @@ +package store + +import ( + "bytes" + "context" + "encoding/json" + "sort" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +var migrationOwnershipMetaKeyBytes = []byte("!migstage|ownership") + +var ErrMigrationOwnership = errors.New("migration ownership evidence mismatch; retry attestation") + +type MigrationOwnershipRole uint8 + +const ( + MigrationOwner MigrationOwnershipRole = iota + 1 + MigrationNonOwner +) + +// MigrationOwnershipState survives guard retirement. A newer catalog cache is +// not an ownership acquisition: only a newer replicated certificate can replace +// a NONOWNER verdict for the same range. +type MigrationOwnershipState struct { + // Generation is a replicated ownership namespace, never a configuration + // index, snapshot index, peer list, or watcher-derived value. + Generation string + JobID uint64 + LocalGroupID uint64 + OwnerGroupID uint64 + CatalogVersion uint64 + ConfigurationIndex uint64 + AppliedIndex uint64 + MinWriteTSExclusive uint64 + RouteStart []byte + RouteEnd []byte + Role MigrationOwnershipRole + Staged bool + RouteID uint64 + ParentRouteID uint64 + DescriptorHash []byte + Receipts []MigrationOwnershipReceipt + WriteFloorKnown bool + SourceTrackerVersion uint64 + SourceAppliedIndex uint64 +} + +type MigrationOwnershipOperation uint8 + +const ( + MigrationOwnershipAcquire MigrationOwnershipOperation = iota + 1 + MigrationOwnershipClear + MigrationOwnershipRefresh +) + +// Receipts name immutable operations; the outer state's version/index fields +// are the latest separately replicated unchanged-descriptor attestation. +type MigrationOwnershipReceipt struct { + Operation MigrationOwnershipOperation + CatalogVersion uint64 + ConfigurationIndex uint64 + AppliedIndex uint64 + Staged bool + DescriptorHash []byte +} + +type MigrationServingState struct { + Ownership []MigrationOwnershipState + Readiness []TargetStagedReadinessState + Partitions MigrationOwnershipPartitionState +} + +// MigrationOwnershipStore reads guard and serving evidence in one store view. +// ClearMigrationStateWithOwnership replaces a guard, retires its job, and +// installs serving evidence at the same durable applied boundary. +type MigrationOwnershipStore interface { + MigrationServingState(context.Context) (MigrationServingState, error) + AcquireMigrationOwnership(context.Context, MigrationOwnershipState, uint64) error + ClearMigrationStateWithOwnership(context.Context, MigrationOwnershipState, uint64) error +} + +// MigrationOwnershipRefresher is separate so old runtimes cannot accidentally +// advertise refresh support by embedding the original ownership interface. +type MigrationOwnershipRefresher interface { + RefreshMigrationOwnership(context.Context, MigrationOwnershipState, uint64) error +} + +func IsMigrationOwnershipMetadataKey(key []byte) bool { + return bytes.Equal(key, migrationOwnershipMetaKeyBytes) +} + +func validMigrationOwnership(state MigrationOwnershipState) bool { + if !validMigrationOwnershipIdentity(state) || + (len(state.RouteEnd) > 0 && bytes.Compare(state.RouteStart, state.RouteEnd) >= 0) { + return false + } + switch state.Role { + case MigrationOwner: + return state.LocalGroupID == state.OwnerGroupID + case MigrationNonOwner: + return state.LocalGroupID != state.OwnerGroupID && !state.Staged + default: + return false + } +} + +func validMigrationOwnershipIdentity(state MigrationOwnershipState) bool { + return state.JobID != 0 && state.LocalGroupID != 0 && state.OwnerGroupID != 0 && + state.CatalogVersion != 0 && state.ConfigurationIndex != 0 && state.AppliedIndex != 0 && + (state.MinWriteTSExclusive != 0 || state.WriteFloorKnown) +} + +func cloneMigrationOwnership(states []MigrationOwnershipState) []MigrationOwnershipState { + out := make([]MigrationOwnershipState, len(states)) + for i, state := range states { + out[i] = state + out[i].RouteStart = bytes.Clone(state.RouteStart) + out[i].RouteEnd = bytes.Clone(state.RouteEnd) + out[i].Receipts = append([]MigrationOwnershipReceipt(nil), state.Receipts...) + out[i].DescriptorHash = bytes.Clone(state.DescriptorHash) + for j := range out[i].Receipts { + out[i].Receipts[j].DescriptorHash = bytes.Clone(state.Receipts[j].DescriptorHash) + } + } + return out +} + +func encodeMigrationOwnership(states []MigrationOwnershipState) ([]byte, error) { + ordered := cloneMigrationOwnership(states) + sort.Slice(ordered, func(i, j int) bool { return ordered[i].JobID < ordered[j].JobID }) + data, err := json.Marshal(ordered) + if err != nil { + return nil, errors.WithStack(err) + } + return append([]byte{migrationMetadataVersion}, data...), nil +} + +func decodeMigrationOwnership(data []byte) ([]MigrationOwnershipState, error) { + if len(data) == 0 || data[0] != migrationMetadataVersion { + return nil, errors.New("invalid migration ownership encoding") + } + var states []MigrationOwnershipState + if err := json.Unmarshal(data[1:], &states); err != nil { + return nil, errors.Wrap(err, "decode migration ownership") + } + seen := make(map[uint64]struct{}, len(states)) + for _, state := range states { + if _, duplicate := seen[state.JobID]; duplicate || !validMigrationOwnership(state) || !validOwnershipReceipts(state) { + return nil, errors.New("invalid migration ownership certificate") + } + seen[state.JobID] = struct{}{} + } + return states, nil +} + +func validOwnershipReceipts(state MigrationOwnershipState) bool { + const maxOwnershipReceipts = 2 + if len(state.Receipts) > maxOwnershipReceipts { + return false + } + seen := make(map[MigrationOwnershipOperation]bool) + for _, receipt := range state.Receipts { + if seen[receipt.Operation] || !validOwnershipReceipt(state, receipt) { + return false + } + seen[receipt.Operation] = true + } + return true +} + +func validOwnershipReceipt(state MigrationOwnershipState, receipt MigrationOwnershipReceipt) bool { + return (receipt.Operation == MigrationOwnershipAcquire || receipt.Operation == MigrationOwnershipClear) && + receipt.CatalogVersion != 0 && receipt.CatalogVersion <= state.CatalogVersion && receipt.ConfigurationIndex != 0 && + receipt.ConfigurationIndex <= state.ConfigurationIndex && receipt.AppliedIndex != 0 && receipt.AppliedIndex <= state.AppliedIndex +} + +func (s *pebbleStore) readMigrationOwnership() ([]MigrationOwnershipState, error) { + data, closer, err := s.db.Get(migrationOwnershipMetaKeyBytes) + if errors.Is(err, pebble.ErrNotFound) { + return nil, nil + } + if err != nil { + return nil, errors.WithStack(err) + } + defer func() { _ = closer.Close() }() + return decodeMigrationOwnership(data) +} + +func (s *mvccStore) MigrationServingState(_ context.Context) (MigrationServingState, error) { + s.mtx.RLock() + defer s.mtx.RUnlock() + return MigrationServingState{ + Ownership: cloneMigrationOwnership(s.migrationOwnership), + Readiness: cloneReadinessStates(s.migrationReadinessCache), + Partitions: clonePartitionState(s.migrationPartitions), + }, nil +} + +func (s *pebbleStore) MigrationServingState(ctx context.Context) (MigrationServingState, error) { + // The raw batch callback already owns applyMu. Its context is the only + // supported way to read this view without reentering that mutex. + if s.raftEntryScopeFrom(ctx) == nil { + s.applyMu.Lock() + defer s.applyMu.Unlock() + } + s.dbMu.RLock() + defer s.dbMu.RUnlock() + states, err := s.readMigrationOwnership() + if err != nil { + return MigrationServingState{}, err + } + partitions, err := s.readMigrationOwnershipPartitions() + if err != nil { + return MigrationServingState{}, err + } + s.mtx.RLock() + defer s.mtx.RUnlock() + return MigrationServingState{Ownership: states, Readiness: cloneReadinessStates(s.migrationReadinessCache), Partitions: partitions}, nil +} + +func cloneReadinessStates(states []TargetStagedReadinessState) []TargetStagedReadinessState { + out := make([]TargetStagedReadinessState, len(states)) + for i, state := range states { + out[i] = state + out[i].RouteStart = bytes.Clone(state.RouteStart) + out[i].RouteEnd = bytes.Clone(state.RouteEnd) + } + return out +} + +func replaceMigrationOwnership(states []MigrationOwnershipState, state MigrationOwnershipState, operation MigrationOwnershipOperation) ([]MigrationOwnershipState, error) { + if !validMigrationOwnership(state) { + return nil, errors.WithStack(ErrMigrationOwnership) + } + out := cloneMigrationOwnership(states) + for i, old := range out { + if old.LocalGroupID != state.LocalGroupID || old.Generation != state.Generation { + return nil, errors.WithStack(ErrMigrationOwnership) + } + if old.JobID == state.JobID { + updated, err := updateMigrationOwnershipAttestation(old, state, operation) + if err != nil { + return nil, err + } + out[i] = cloneMigrationOwnership([]MigrationOwnershipState{updated})[0] + return out, nil + } + if migrationOwnershipOverlaps(old, state) && old.CatalogVersion >= state.CatalogVersion { + return nil, errors.WithStack(ErrMigrationOwnership) + } + } + if operation == MigrationOwnershipRefresh { + return nil, errors.WithStack(ErrMigrationOwnership) + } + state.Receipts = []MigrationOwnershipReceipt{ownershipOperationReceipt(state, operation)} + return append(out, cloneMigrationOwnership([]MigrationOwnershipState{state})[0]), nil +} + +func ownershipOperationReceipt(state MigrationOwnershipState, operation MigrationOwnershipOperation) MigrationOwnershipReceipt { + return MigrationOwnershipReceipt{Operation: operation, CatalogVersion: state.CatalogVersion, + ConfigurationIndex: state.ConfigurationIndex, AppliedIndex: state.AppliedIndex, Staged: state.Staged, + DescriptorHash: bytes.Clone(state.DescriptorHash)} +} + +func updateMigrationOwnershipAttestation(old, state MigrationOwnershipState, operation MigrationOwnershipOperation) (MigrationOwnershipState, error) { + if operation == MigrationOwnershipClear && ownershipSourceTransition(old, state) { + state.Receipts = append(append([]MigrationOwnershipReceipt(nil), old.Receipts...), ownershipOperationReceipt(state, operation)) + return state, nil + } + if !sameMigrationOwnershipDescriptor(old, state) { + return MigrationOwnershipState{}, errors.WithStack(ErrMigrationOwnership) + } + if operation == MigrationOwnershipRefresh { + return refreshMigrationOwnershipAttestation(old, state) + } + return replayMigrationOwnershipOperation(old, state, operation) +} + +func refreshMigrationOwnershipAttestation(old, state MigrationOwnershipState) (MigrationOwnershipState, error) { + if state.CatalogVersion < old.CatalogVersion || state.ConfigurationIndex < old.ConfigurationIndex || state.AppliedIndex < old.AppliedIndex || + state.Staged != old.Staged || !bytes.Equal(state.DescriptorHash, old.DescriptorHash) { + return MigrationOwnershipState{}, errors.WithStack(ErrMigrationOwnership) + } + state.Receipts = append([]MigrationOwnershipReceipt(nil), old.Receipts...) + return state, nil +} + +func replayMigrationOwnershipOperation(old, state MigrationOwnershipState, operation MigrationOwnershipOperation) (MigrationOwnershipState, error) { + for _, receipt := range old.Receipts { + if receipt.Operation != operation { + continue + } + if receipt.CatalogVersion != state.CatalogVersion || receipt.Staged != state.Staged || !bytes.Equal(receipt.DescriptorHash, state.DescriptorHash) { + return MigrationOwnershipState{}, errors.WithStack(ErrMigrationOwnership) + } + return old, nil + } + if operation != MigrationOwnershipClear || !old.Staged || state.Staged || state.CatalogVersion < old.CatalogVersion { + return MigrationOwnershipState{}, errors.WithStack(ErrMigrationOwnership) + } + state.Receipts = append(append([]MigrationOwnershipReceipt(nil), old.Receipts...), ownershipOperationReceipt(state, operation)) + return state, nil +} + +func sameMigrationOwnershipDescriptor(a, b MigrationOwnershipState) bool { + return sameMigrationOwnershipRange(a, b) && a.Generation == b.Generation && a.Role == b.Role && + ownershipWriteFloorKnown(a) == ownershipWriteFloorKnown(b) && a.SourceTrackerVersion == b.SourceTrackerVersion && a.SourceAppliedIndex == b.SourceAppliedIndex && + a.MinWriteTSExclusive == b.MinWriteTSExclusive && a.RouteID == b.RouteID && a.ParentRouteID == b.ParentRouteID +} + +func sameMigrationOwnershipRange(a, b MigrationOwnershipState) bool { + return a.JobID == b.JobID && a.LocalGroupID == b.LocalGroupID && a.OwnerGroupID == b.OwnerGroupID && + bytes.Equal(a.RouteStart, b.RouteStart) && bytes.Equal(a.RouteEnd, b.RouteEnd) +} + +func migrationOwnershipOverlaps(a, b MigrationOwnershipState) bool { + return (len(a.RouteEnd) == 0 || bytes.Compare(b.RouteStart, a.RouteEnd) < 0) && + (len(b.RouteEnd) == 0 || bytes.Compare(a.RouteStart, b.RouteEnd) < 0) +} + +func validateOwnershipGuard(state MigrationOwnershipState, guards []TargetStagedReadinessState, retired bool, promoted ...bool) error { + if retired { + return nil // The immutable existing certificate is checked by replacement. + } + for _, guard := range guards { + if !ownershipGuardMatchesRange(state, guard) { + continue + } + // Rollback to the source retains its tracked live data. Target ownership + // acquisition needs a separate data-ready control and is not accepted here. + if ownershipClearMatches(state, guard, len(promoted) > 0 && promoted[0]) { + return nil + } + } + if !ownershipJobGuardExists(guards, state.JobID) && !state.Staged && len(state.Generation) >= 16 && len(state.DescriptorHash) == 32 { + return nil + } + return errors.WithStack(ErrMigrationOwnership) +} + +func ownershipClearMatches(state MigrationOwnershipState, guard TargetStagedReadinessState, promoted bool) bool { + return ownershipSourceRemovalMatches(state, guard) || ownershipAbandonedTargetMatches(state, guard) || + ownershipRollbackMatches(state, guard) || (promoted && ownershipPromotedTargetMatches(state, guard)) +} + +func ownershipJobGuardExists(guards []TargetStagedReadinessState, jobID uint64) bool { + for _, guard := range guards { + if guard.JobID == jobID { + return true + } + } + return false +} + +func ownershipPromotedTargetMatches(state MigrationOwnershipState, guard TargetStagedReadinessState) bool { + return state.Role == MigrationOwner && !state.Staged && !guard.TrackWrites && !guard.SourceWriteFence && !guard.SourceReadFence && + state.CatalogVersion >= guard.ExpectedCutoverVersion && state.MinWriteTSExclusive == guard.MinWriteTSExclusive +} + +func ownershipGuardMatchesRange(state MigrationOwnershipState, guard TargetStagedReadinessState) bool { + return guard.JobID == state.JobID && guard.MigrationJobID == state.JobID && guard.Armed && + bytes.Equal(guard.RouteStart, state.RouteStart) && bytes.Equal(guard.RouteEnd, state.RouteEnd) +} + +func ownershipSourceRemovalMatches(state MigrationOwnershipState, guard TargetStagedReadinessState) bool { + return state.Role == MigrationNonOwner && guard.SourceWriteFence && guard.SourceReadFence && + state.CatalogVersion >= guard.ExpectedCutoverVersion && state.MinWriteTSExclusive == guard.MinWriteTSExclusive +} + +func ownershipRollbackMatches(state MigrationOwnershipState, guard TargetStagedReadinessState) bool { + return state.Role == MigrationOwner && (guard.TrackWrites || guard.SourceWriteFence) && !state.Staged && + state.CatalogVersion >= guard.ExpectedCutoverVersion && state.MinWriteTSExclusive >= guard.MinWriteTSExclusive +} + +func ownershipAbandonedTargetMatches(state MigrationOwnershipState, guard TargetStagedReadinessState) bool { + return state.Role == MigrationNonOwner && !guard.TrackWrites && !guard.SourceWriteFence && !guard.SourceReadFence && + state.CatalogVersion >= guard.ExpectedCutoverVersion && state.MinWriteTSExclusive >= guard.MinWriteTSExclusive +} + +func validateOwnershipAcquisition(state MigrationOwnershipState, guards []TargetStagedReadinessState) error { + if !validMigrationOwnership(state) || state.Role != MigrationOwner { + return errors.WithStack(ErrMigrationOwnership) + } + for _, guard := range guards { + if ownershipGuardMatchesRange(state, guard) && (ownershipTargetMatches(state, guard) || ownershipSourceAcquisitionMatches(state, guard)) { + return nil + } + } + return errors.WithStack(ErrMigrationOwnership) +} + +func ownershipTargetMatches(state MigrationOwnershipState, guard TargetStagedReadinessState) bool { + return !guard.TrackWrites && !guard.SourceWriteFence && !guard.SourceReadFence && state.Staged && + ownershipAcquisitionVersionMatches(state, guard) && guard.MinWriteTSExclusive == state.MinWriteTSExclusive +} + +func ownershipAcquisitionVersionMatches(state MigrationOwnershipState, guard TargetStagedReadinessState) bool { + if guard.ExpectedCutoverVersion == state.CatalogVersion { + return true + } + // Only the authenticated committed-descriptor control may attest a later + // version. Legacy unversioned controls never gain this ability implicitly. + return len(state.Generation) >= 16 && len(state.DescriptorHash) == 32 && state.CatalogVersion > guard.ExpectedCutoverVersion +} + +func (s *mvccStore) AcquireMigrationOwnership(_ context.Context, state MigrationOwnershipState, appliedIndex uint64) error { + s.mtx.Lock() + defer s.mtx.Unlock() + state.AppliedIndex = appliedIndex + if state.SourceTrackerVersion != 0 { + state.SourceAppliedIndex = appliedIndex + } + if partitionJobReplay(s.migrationPartitions, s.migrationOwnership, state, MigrationOwnershipAcquire) { + return nil + } + if _, retired := s.migrationRetired[state.JobID]; retired { + return errors.WithStack(ErrMigrationRetired) + } + if err := validateOwnershipAcquisition(state, s.migrationReadinessCache); err != nil { + return err + } + states, err := replaceMigrationOwnership(s.migrationOwnership, state, MigrationOwnershipAcquire) + if err != nil { + return err + } + partitions, err := s.syncPartitionJobLocked(states, state, MigrationOwnershipAcquire, appliedIndex) + if err != nil { + return err + } + s.migrationOwnership, s.migrationPartitions = states, partitions + s.migrationAdmission = buildMigrationServingAdmission(states, partitions) + return nil +} + +func (s *pebbleStore) AcquireMigrationOwnership(ctx context.Context, state MigrationOwnershipState, appliedIndex uint64) error { + if s.raftEntryScopeFrom(ctx) != nil { + return errors.WithStack(ErrMigrationOwnership) + } + s.applyMu.Lock() + defer s.applyMu.Unlock() + s.dbMu.RLock() + defer s.dbMu.RUnlock() + state.AppliedIndex = appliedIndex + if state.SourceTrackerVersion != 0 { + state.SourceAppliedIndex = appliedIndex + } + old, err := s.readMigrationOwnership() + if err != nil { + return err + } + partitions, err := s.readMigrationOwnershipPartitions() + if err != nil { + return err + } + if partitionJobReplay(partitions, old, state, MigrationOwnershipAcquire) { + return s.persistMigrationOwnershipLocked(old, appliedIndex, partitions) + } + if err := s.checkMigrationNotRetiredLocked(state.JobID); err != nil { + return err + } + s.mtx.RLock() + err = validateOwnershipAcquisition(state, s.migrationReadinessCache) + s.mtx.RUnlock() + if err != nil { + return err + } + states, err := replaceMigrationOwnership(old, state, MigrationOwnershipAcquire) + if err != nil { + return err + } + partitions, err = s.syncPartitionJobLocked(ctx, partitions, old, states, state, MigrationOwnershipAcquire, appliedIndex) + if err != nil { + return err + } + return s.persistMigrationOwnershipLocked(states, appliedIndex, partitions) +} + +func (s *mvccStore) RefreshMigrationOwnership(_ context.Context, state MigrationOwnershipState, appliedIndex uint64) error { + s.mtx.Lock() + defer s.mtx.Unlock() + state.AppliedIndex = appliedIndex + if state.Generation == "" || state.RouteID == 0 { + return errors.WithStack(ErrMigrationOwnership) + } + states, err := replaceMigrationOwnership(s.migrationOwnership, state, MigrationOwnershipRefresh) + if err != nil { + return err + } + partitions, err := s.syncPartitionJobLocked(states, state, MigrationOwnershipRefresh, appliedIndex) + if err != nil { + return err + } + s.migrationOwnership, s.migrationPartitions = states, partitions + s.migrationAdmission = buildMigrationServingAdmission(states, partitions) + return nil +} + +func (s *pebbleStore) RefreshMigrationOwnership(ctx context.Context, state MigrationOwnershipState, appliedIndex uint64) error { + if s.raftEntryScopeFrom(ctx) != nil { + return errors.WithStack(ErrMigrationOwnership) + } + s.applyMu.Lock() + defer s.applyMu.Unlock() + s.dbMu.RLock() + defer s.dbMu.RUnlock() + state.AppliedIndex = appliedIndex + if state.Generation == "" || state.RouteID == 0 { + return errors.WithStack(ErrMigrationOwnership) + } + states, err := s.readMigrationOwnership() + if err != nil { + return err + } + old := states + states, err = replaceMigrationOwnership(states, state, MigrationOwnershipRefresh) + if err != nil { + return err + } + partitions, err := s.readMigrationOwnershipPartitions() + if err != nil { + return err + } + partitions, err = s.syncPartitionJobLocked(ctx, partitions, old, states, state, MigrationOwnershipRefresh, appliedIndex) + if err != nil { + return err + } + return s.persistMigrationOwnershipLocked(states, appliedIndex, partitions) +} + +func (s *pebbleStore) persistMigrationOwnershipLocked(states []MigrationOwnershipState, appliedIndex uint64, partitions MigrationOwnershipPartitionState) error { + data, err := encodeMigrationOwnership(states) + if err != nil { + return err + } + batch := s.db.NewBatch() + defer batch.Close() + if err := batch.Set(migrationOwnershipMetaKeyBytes, data, nil); err != nil { + return errors.WithStack(err) + } + if err := stageSnapshotPartitions(batch, partitions); err != nil { + return err + } + if err := s.stagePartitionAppliedIndex(batch, appliedIndex); err != nil { + return err + } + if err := batch.Commit(s.cleanupWriteOpts(appliedIndex)); err != nil { + return errors.WithStack(err) + } + s.mtx.Lock() + s.migrationAdmission = buildMigrationServingAdmission(states, partitions) + s.mtx.Unlock() + return nil +} + +func (s *mvccStore) ClearMigrationStateWithOwnership(_ context.Context, state MigrationOwnershipState, appliedIndex uint64) error { + s.mtx.Lock() + defer s.mtx.Unlock() + state.AppliedIndex = appliedIndex + if partitionJobReplay(s.migrationPartitions, s.migrationOwnership, state, MigrationOwnershipClear) { + return nil + } + _, retired := s.migrationRetired[state.JobID] + if err := validateOwnershipGuard(state, s.migrationReadinessCache, retired, s.migrationPromotions[state.JobID].Done); err != nil { + return err + } + states, err := replaceMigrationOwnership(s.migrationOwnership, state, MigrationOwnershipClear) + if err != nil { + return err + } + if err := validateRetiredOwnership(s.migrationOwnership, state.JobID, retired); err != nil { + return err + } + partitions, err := s.syncPartitionJobLocked(states, state, MigrationOwnershipClear, appliedIndex) + if err != nil { + return err + } + s.migrationOwnership, s.migrationPartitions = states, partitions + s.migrationAdmission = buildMigrationServingAdmission(states, partitions) + s.clearMigrationStateLocked(state.JobID) + return nil +} + +func hasMigrationOwnership(states []MigrationOwnershipState, jobID uint64) bool { + for _, state := range states { + if state.JobID == jobID { + return true + } + } + return false +} + +func validateRetiredOwnership(states []MigrationOwnershipState, jobID uint64, retired bool) error { + if retired && !hasMigrationOwnership(states, jobID) { + return errors.WithStack(ErrMigrationOwnership) + } + return nil +} + +func (s *pebbleStore) ClearMigrationStateWithOwnership(ctx context.Context, state MigrationOwnershipState, appliedIndex uint64) error { + if s.raftEntryScopeFrom(ctx) != nil { + return errors.WithStack(ErrMigrationOwnership) + } + s.applyMu.Lock() + defer s.applyMu.Unlock() + s.dbMu.RLock() + defer s.dbMu.RUnlock() + state.AppliedIndex = appliedIndex + states, err := s.readMigrationOwnership() + if err != nil { + return err + } + partitions, err := s.readMigrationOwnershipPartitions() + if err != nil { + return err + } + if partitionJobReplay(partitions, states, state, MigrationOwnershipClear) { + return s.persistMigrationOwnershipLocked(states, appliedIndex, partitions) + } + if err := s.validateOwnershipClearLocked(state, states); err != nil { + return err + } + old := states + states, err = replaceMigrationOwnership(states, state, MigrationOwnershipClear) + if err != nil { + return err + } + partitions, err = s.syncPartitionJobLocked(ctx, partitions, old, states, state, MigrationOwnershipClear, appliedIndex) + if err != nil { + return err + } + return s.clearMigrationStateWithOwnershipLocked(state.JobID, appliedIndex, states, partitions) +} + +func (s *pebbleStore) validateOwnershipClearLocked(state MigrationOwnershipState, states []MigrationOwnershipState) error { + retired, err := s.readMigrationRetiredJobs() + if err != nil { + return err + } + _, isRetired := retired[state.JobID] + promotions, err := s.readPebblePromotionStates() + if err != nil { + return err + } + s.mtx.RLock() + err = validateOwnershipGuard(state, s.migrationReadinessCache, isRetired, promotions[state.JobID].Done) + s.mtx.RUnlock() + if err != nil { + return err + } + return validateRetiredOwnership(states, state.JobID, isRetired) +} + +func (s *pebbleStore) clearMigrationStateWithOwnershipLocked(jobID, appliedIndex uint64, states []MigrationOwnershipState, partitions MigrationOwnershipPartitionState) error { + metadata, err := s.migrationMetadataWithoutJob(jobID) + if err != nil { + return err + } + data, err := encodeMigrationOwnership(states) + if err != nil { + return err + } + batch := s.db.NewBatch() + defer batch.Close() + if err := stageMigrationMetadataCleanup(batch, jobID, appliedIndex, metadata); err != nil { + return err + } + if err := batch.Set(migrationOwnershipMetaKeyBytes, data, nil); err != nil { + return errors.WithStack(err) + } + if err := stageSnapshotPartitions(batch, partitions); err != nil { + return err + } + if err := s.stagePartitionAppliedIndex(batch, appliedIndex); err != nil { + return err + } + if err := batch.Commit(s.cleanupWriteOpts(appliedIndex)); err != nil { + return errors.WithStack(err) + } + s.mtx.Lock() + s.migrationAdmission = buildMigrationServingAdmission(states, partitions) + s.migrationReadinessCache = upsertReadinessCacheWithoutJob(s.migrationReadinessCache, jobID) + s.mtx.Unlock() + return nil +} diff --git a/store/migration_ownership_evidence.go b/store/migration_ownership_evidence.go new file mode 100644 index 000000000..235dbf805 --- /dev/null +++ b/store/migration_ownership_evidence.go @@ -0,0 +1,60 @@ +package store + +import "bytes" + +// MigrationOwnershipEvidenceValid deliberately does not tighten the snapshot +// codec: legacy records remain recoverable, but cannot authorize serving. +func MigrationOwnershipEvidenceValid(state MigrationOwnershipState) bool { + if !validMigrationOwnership(state) || !validOwnershipReceipts(state) || !ownershipDescriptorKnown(state) || len(state.Receipts) == 0 { + return false + } + if !ownershipLatestReceiptMatches(state) { + return false + } + if state.SourceTrackerVersion != 0 { + return ownershipSourceBoundaryKnown(state) + } + return state.SourceAppliedIndex == 0 && ownershipTerminalReceiptMatches(state) +} + +func ownershipTerminalReceiptMatches(state MigrationOwnershipState) bool { + latest := state.Receipts[len(state.Receipts)-1] + return (state.Staged && latest.Operation == MigrationOwnershipAcquire) || (!state.Staged && latest.Operation == MigrationOwnershipClear) +} + +func ownershipLatestReceiptMatches(state MigrationOwnershipState) bool { + latest := state.Receipts[len(state.Receipts)-1] + return len(latest.DescriptorHash) == 32 && bytes.Equal(latest.DescriptorHash, state.DescriptorHash) && latest.Staged == state.Staged +} + +func ownershipDescriptorKnown(state MigrationOwnershipState) bool { + return len(state.Generation) >= 16 && len(state.Generation) <= 128 && state.RouteID != 0 && len(state.DescriptorHash) == 32 +} + +func ownershipWriteFloorKnown(state MigrationOwnershipState) bool { + return state.WriteFloorKnown || state.MinWriteTSExclusive != 0 +} + +func ownershipSourceBoundaryKnown(state MigrationOwnershipState) bool { + first := state.Receipts[0] + return state.Role == MigrationOwner && !state.Staged && state.WriteFloorKnown && state.SourceAppliedIndex != 0 && + state.SourceAppliedIndex <= state.AppliedIndex && first.Operation == MigrationOwnershipAcquire && first.AppliedIndex == state.SourceAppliedIndex +} + +func ownershipSourceAcquisitionMatches(state MigrationOwnershipState, guard TargetStagedReadinessState) bool { + return ownershipDescriptorKnown(state) && state.WriteFloorKnown && !state.Staged && state.SourceTrackerVersion != 0 && + state.SourceAppliedIndex == state.AppliedIndex && guard.TrackWrites && !guard.SourceWriteFence && !guard.SourceReadFence && + state.SourceTrackerVersion == guard.ExpectedCutoverVersion && guard.RetentionPinTS != 0 +} + +func ownershipSourceTransition(old, state MigrationOwnershipState) bool { + return ownershipSourceTransitionIdentity(old, state) && state.CatalogVersion > old.CatalogVersion && + state.ConfigurationIndex >= old.ConfigurationIndex && state.AppliedIndex > old.AppliedIndex && !state.Staged && + state.MinWriteTSExclusive >= old.MinWriteTSExclusive +} + +func ownershipSourceTransitionIdentity(old, state MigrationOwnershipState) bool { + return old.SourceTrackerVersion != 0 && MigrationOwnershipEvidenceValid(old) && state.SourceTrackerVersion == 0 && state.SourceAppliedIndex == 0 && + old.JobID == state.JobID && old.LocalGroupID == state.LocalGroupID && old.Generation == state.Generation && + bytes.Equal(old.RouteStart, state.RouteStart) && bytes.Equal(old.RouteEnd, state.RouteEnd) +} diff --git a/store/migration_ownership_partition.go b/store/migration_ownership_partition.go new file mode 100644 index 000000000..4807a993e --- /dev/null +++ b/store/migration_ownership_partition.go @@ -0,0 +1,507 @@ +package store + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "slices" + "sort" + + "github.com/cockroachdb/errors" +) + +func partitionHasEvidence(s MigrationOwnershipPartitionState) bool { + return len(s.Partitions)+len(s.Receipts) != 0 +} + +func validPartition(p MigrationOwnershipPartition) bool { + return validPartitionIdentity(p) && validPartitionBoundary(p) && validPartitionRole(p) && validPartitionSource(p) && validPartitionOrigins(p) +} + +func validPartitionIdentity(p MigrationOwnershipPartition) bool { + if len(p.PartitionID) != sha256.Size || len(p.ReceiptID) != sha256.Size || + len(p.Generation) < 16 || len(p.Generation) > 128 || p.LocalGroupID == 0 || p.OwnerGroupID == 0 { + return false + } + return p.RouteID != 0 && p.MigrationJobID != 0 && len(p.DescriptorHash) == sha256.Size +} + +func validPartitionBoundary(p MigrationOwnershipPartition) bool { + return p.CatalogVersion != 0 && p.ConfigurationIndex != 0 && p.AppliedIndex != 0 && p.WriteFloorKnown && + (len(p.RouteEnd) == 0 || bytes.Compare(p.RouteStart, p.RouteEnd) < 0) +} + +func validPartitionRole(p MigrationOwnershipPartition) bool { + if (p.Role == MigrationOwner && p.LocalGroupID != p.OwnerGroupID) || + (p.Role == MigrationNonOwner && (p.LocalGroupID == p.OwnerGroupID || p.Staged)) || + (p.Role != MigrationOwner && p.Role != MigrationNonOwner) { + return false + } + return true +} + +func validPartitionSource(p MigrationOwnershipPartition) bool { + if !validPartitionSourceIdentity(p) { + return false + } + if p.SourceTrackerVersion == 0 { + return true + } + for _, origin := range p.Origins { + if origin.JobID == p.MigrationJobID && origin.Operation == MigrationOwnershipAcquire && origin.AppliedIndex == p.SourceAppliedIndex { + return true + } + } + return false +} + +func validPartitionSourceIdentity(p MigrationOwnershipPartition) bool { + return (p.SourceTrackerVersion == 0) == (p.SourceAppliedIndex == 0) && p.SourceAppliedIndex <= p.AppliedIndex && + (p.SourceTrackerVersion == 0 || (p.Role == MigrationOwner && !p.Staged)) +} + +func validPartitionOrigins(p MigrationOwnershipPartition) bool { + if len(p.Origins) == 0 { + return false + } + seen := make(map[string]bool) + for _, origin := range p.Origins { + if !validPartitionOrigin(p, origin) { + return false + } + data, _ := json.Marshal(origin) + if seen[string(data)] { + return false + } + seen[string(data)] = true + } + return true +} + +func validPartitionOrigin(p MigrationOwnershipPartition, origin MigrationOwnershipOperationReference) bool { + return validPartitionOriginIdentity(origin) && origin.CatalogVersion <= p.CatalogVersion && + origin.ConfigurationIndex <= p.ConfigurationIndex && origin.AppliedIndex != 0 && origin.AppliedIndex <= p.AppliedIndex && + len(origin.DescriptorHash) == sha256.Size +} + +func validPartitionOriginIdentity(origin MigrationOwnershipOperationReference) bool { + return origin.JobID != 0 && (origin.Operation == MigrationOwnershipAcquire || origin.Operation == MigrationOwnershipClear) && + origin.CatalogVersion != 0 && origin.ConfigurationIndex != 0 +} + +func samePartition(a, b MigrationOwnershipPartition) bool { + x, _ := json.Marshal(clonePartition(a)) + y, _ := json.Marshal(clonePartition(b)) + return bytes.Equal(x, y) +} + +func MigrationOwnershipPartitionEvidenceValid(s MigrationOwnershipPartitionState, p MigrationOwnershipPartition) bool { + if s.servingEvidence != nil { + return s.servingEvidence.valid(s, p) + } + return migrationPartitionEvidenceUncached(s, p) +} + +func migrationPartitionEvidenceUncached(s MigrationOwnershipPartitionState, p MigrationOwnershipPartition) bool { + if s.Generation != p.Generation || !validPartition(p) || !partitionIsActive(s, p) { + return false + } + for _, r := range s.Receipts { + if !bytes.Equal(r.ReceiptID, p.ReceiptID) || !validPartitionReceipt(r) { + continue + } + for _, candidate := range r.Transition.After { + if samePartition(p, candidate) { + return true + } + } + } + return false +} + +func partitionIsActive(s MigrationOwnershipPartitionState, p MigrationOwnershipPartition) bool { + for _, active := range s.Partitions { + if samePartition(active, p) { + return true + } + } + return false +} + +func validPartitionReceipt(r MigrationOwnershipPartitionReceipt) bool { + t := r.Transition + if !validPartitionReceiptHeader(r) { + return false + } + id, err := MigrationOwnershipPartitionReceiptIDFor(t) + if err != nil || !bytes.Equal(id, r.ReceiptID) { + return false + } + for _, p := range t.After { + if !validPartition(p) || !partitionReceiptStampMatches(r, p) { + return false + } + } + return true +} + +func validPartitionReceiptHeader(r MigrationOwnershipPartitionReceipt) bool { + t := r.Transition + return r.AppliedIndex != 0 && t.CatalogVersion != 0 && t.ConfigurationIndex != 0 && len(t.After) > 0 && validPartitionReceiptOperation(t) +} + +func validPartitionReceiptOperation(t MigrationOwnershipPartitionTransition) bool { + if t.Operation == MigrationOwnershipPartitionJobOperation { + return len(t.RootManifestHash) == 0 + } + return (t.Operation == MigrationOwnershipPartitionSplit || t.Operation == MigrationOwnershipPartitionRefresh) && len(t.RootManifestHash) == sha256.Size +} + +func partitionReceiptStampMatches(r MigrationOwnershipPartitionReceipt, p MigrationOwnershipPartition) bool { + t := r.Transition + return p.Generation == t.Generation && p.LocalGroupID == t.LocalGroupID && p.CatalogVersion == t.CatalogVersion && + p.ConfigurationIndex == t.ConfigurationIndex && p.AppliedIndex == r.AppliedIndex && bytes.Equal(p.ReceiptID, r.ReceiptID) +} + +func encodeMigrationOwnershipPartitions(s MigrationOwnershipPartitionState) ([]byte, error) { + data, err := json.Marshal(clonePartitionState(s)) + if err != nil { + return nil, errors.WithStack(err) + } + return append([]byte{migrationMetadataVersion}, data...), nil +} + +func decodeMigrationOwnershipPartitions(data []byte) (MigrationOwnershipPartitionState, error) { + var s MigrationOwnershipPartitionState + if len(data) == 0 || data[0] != migrationMetadataVersion { + return s, errors.New("invalid ownership partition encoding") + } + if err := json.Unmarshal(data[1:], &s); err != nil { + return s, errors.Wrap(err, "decode ownership partition") + } + if err := validatePartitionReceipts(s); err != nil { + return s, err + } + s = clonePartitionState(s) + return s, validatePartitionFrontier(s) +} + +func validatePartitionReceipts(s MigrationOwnershipPartitionState) error { + seen := make(map[string]bool) + for _, r := range s.Receipts { + if r.Transition.Generation != s.Generation || seen[r.Transition.OperationID] || !validPartitionReceipt(r) { + return errors.New("invalid ownership partition receipt") + } + seen[r.Transition.OperationID] = true + } + return nil +} + +func partitionFrontierProofs(s MigrationOwnershipPartitionState) (map[string]struct{}, error) { + proofs := make(map[string]struct{}) + for _, r := range s.Receipts { + for _, p := range r.Transition.After { + key, err := partitionDigest("elastickv/ownership-partition/frontier/v1", clonePartition(p)) + if err != nil { + return nil, err + } + proofs[string(key)] = struct{}{} + } + } + return proofs, nil +} + +func validatePartitionFrontier(s MigrationOwnershipPartitionState) error { + // decode validates every receipt once before indexing its outputs, never + // rehashing historical transitions once per active partition. + proofs, err := partitionFrontierProofs(s) + if err != nil { + return err + } + seen := make(map[string]bool) + for i, p := range s.Partitions { + key, err := partitionDigest("elastickv/ownership-partition/frontier/v1", clonePartition(p)) + if err != nil { + return err + } + _, supported := proofs[string(key)] + if p.Generation != s.Generation || !validPartition(p) || !supported || seen[string(p.PartitionID)] || + (i > 0 && rangesOverlap(s.Partitions[i-1], p)) { + return errors.New("invalid ownership partition frontier") + } + seen[string(p.PartitionID)] = true + } + return nil +} + +func rangesOverlap(a, b MigrationOwnershipPartition) bool { + return (len(a.RouteEnd) == 0 || bytes.Compare(b.RouteStart, a.RouteEnd) < 0) && + (len(b.RouteEnd) == 0 || bytes.Compare(a.RouteStart, b.RouteEnd) < 0) +} + +func partitionFromLegacy(s MigrationOwnershipState) MigrationOwnershipPartition { + p := MigrationOwnershipPartition{Generation: s.Generation, LocalGroupID: s.LocalGroupID, OwnerGroupID: s.OwnerGroupID, + CatalogVersion: s.CatalogVersion, ConfigurationIndex: s.ConfigurationIndex, AppliedIndex: s.AppliedIndex, + RouteID: s.RouteID, ParentRouteID: s.ParentRouteID, RouteStart: bytes.Clone(s.RouteStart), RouteEnd: bytes.Clone(s.RouteEnd), + DescriptorHash: bytes.Clone(s.DescriptorHash), Role: s.Role, MinWriteTSExclusive: s.MinWriteTSExclusive, + WriteFloorKnown: ownershipWriteFloorKnown(s), Staged: s.Staged, MigrationJobID: s.JobID, + SourceTrackerVersion: s.SourceTrackerVersion, SourceAppliedIndex: s.SourceAppliedIndex} + for _, r := range s.Receipts { + p.Origins = append(p.Origins, MigrationOwnershipOperationReference{JobID: s.JobID, Operation: r.Operation, + CatalogVersion: r.CatalogVersion, ConfigurationIndex: r.ConfigurationIndex, AppliedIndex: r.AppliedIndex, + DescriptorHash: bytes.Clone(r.DescriptorHash), Staged: r.Staged}) + } + return clonePartition(p) +} + +type partitionCandidate struct { + partition MigrationOwnershipPartition + reference MigrationOwnershipPartitionReference +} + +func partitionCandidates(s MigrationOwnershipPartitionState, legacy []MigrationOwnershipState) ([]partitionCandidate, error) { + var out []partitionCandidate + consumed := partitionAdoptedJobs(s) + for _, p := range s.Partitions { + ref, err := MigrationOwnershipPartitionReferenceFor(p) + if err != nil { + return nil, err + } + out = append(out, partitionCandidate{p, ref}) + } + for _, p := range legacy { + if consumed[p.JobID] && MigrationOwnershipEvidenceValid(p) { + continue + } + // Unknown legacy evidence is kept as a negative candidate. It cannot be + // omitted or used to seed a replacement merely because validation fails. + ref, _ := MigrationOwnershipLegacyReferenceFor(p) + ref.LegacyJobID = p.JobID + out = append(out, partitionCandidate{partitionFromLegacy(p), ref}) + } + return out, nil +} + +func equalPartitionReference(a, b MigrationOwnershipPartitionReference) bool { + return a.LegacyJobID == b.LegacyJobID && bytes.Equal(a.PartitionID, b.PartitionID) && + len(a.AttestationHash) == sha256.Size && bytes.Equal(a.AttestationHash, b.AttestationHash) +} + +func inheritedPartition(a, b MigrationOwnershipPartition) bool { + a = clonePartition(a) + b = clonePartition(b) + a.PartitionID, b.PartitionID = nil, nil + a.ReceiptID, b.ReceiptID = nil, nil + a.CatalogVersion, b.CatalogVersion = 0, 0 + a.ConfigurationIndex, b.ConfigurationIndex = 0, 0 + a.AppliedIndex, b.AppliedIndex = 0, 0 + a.RouteID, b.RouteID = 0, 0 + a.ParentRouteID, b.ParentRouteID = 0, 0 + a.RouteStart, b.RouteStart = nil, nil + a.RouteEnd, b.RouteEnd = nil, nil + a.DescriptorHash, b.DescriptorHash = nil, nil + return samePartition(a, b) +} + +// Check each elementary interval, including the two unbounded intervals. This +// compares complete unions even when old job records overlay one another. +func validatePartitionUnion(before, after []MigrationOwnershipPartition) error { + boundaries := partitionUnionBoundaries(before, after) + for i, start := range boundaries { + var end []byte + if i+1 < len(boundaries) { + end = boundaries[i+1] + } + if !partitionIntervalMatches(before, after, start, end) { + return ErrMigrationOwnership + } + } + return nil +} + +func partitionUnionBoundaries(before, after []MigrationOwnershipPartition) [][]byte { + boundaries := [][]byte{nil} + for _, list := range [][]MigrationOwnershipPartition{before, after} { + for _, p := range list { + if len(p.RouteStart) != 0 { + boundaries = append(boundaries, p.RouteStart) + } + if len(p.RouteEnd) != 0 { + boundaries = append(boundaries, p.RouteEnd) + } + } + } + sort.Slice(boundaries, func(i, j int) bool { return bytes.Compare(boundaries[i], boundaries[j]) < 0 }) + return slices.CompactFunc(boundaries, bytes.Equal) +} + +func partitionIntervalMatches(before, after []MigrationOwnershipPartition, start, end []byte) bool { + old, oldCount, ambiguous := partitionCovering(before, start, end) + child, count, _ := partitionCovering(after, start, end) + if ambiguous || count > 1 || (oldCount == 0) != (count == 0) { + return false + } + return count == 0 || (inheritedPartition(old, child) && child.CatalogVersion >= old.CatalogVersion && + child.ConfigurationIndex >= old.ConfigurationIndex && child.AppliedIndex >= old.AppliedIndex) +} + +func partitionCovering(list []MigrationOwnershipPartition, start, end []byte) (MigrationOwnershipPartition, int, bool) { + var chosen MigrationOwnershipPartition + count := 0 + ambiguous := false + for _, p := range list { + if bytes.Compare(p.RouteStart, start) > 0 || (len(p.RouteEnd) != 0 && (len(end) == 0 || bytes.Compare(p.RouteEnd, end) < 0)) { + continue + } + count++ + if chosen.CatalogVersion == 0 || p.CatalogVersion > chosen.CatalogVersion { + chosen, ambiguous = p, false + } else if p.CatalogVersion == chosen.CatalogVersion && !samePartition(p, chosen) { + ambiguous = true + } + } + return chosen, count, ambiguous +} + +func applyPartitionTransition(s MigrationOwnershipPartitionState, legacy []MigrationOwnershipState, t MigrationOwnershipPartitionTransition, index, configuration uint64) (MigrationOwnershipPartitionState, error) { + t = canonicalPartitionTransition(t) + id, err := MigrationOwnershipPartitionReceiptIDFor(t) + if err != nil || !validPartitionTransitionEnvelope(s, t, index) { + return s, ErrMigrationOwnership + } + if replay, replayErr := partitionTransitionReplay(s, t.OperationID, id); replay || replayErr != nil { + return clonePartitionState(s), replayErr + } + if configuration == 0 || configuration != t.ConfigurationIndex { + return s, ErrMigrationOwnership + } + before, retained, err := resolvePartitionBefore(s, legacy, t) + if err != nil { + return s, err + } + if err := validatePartitionChildren(&t, before, index, id); err != nil { + return s, err + } + if err := validatePartitionUnion(before, t.After); err != nil { + return s, err + } + return appendPartitionReceipt(s, retained, t, index, id) +} + +func validPartitionTransitionEnvelope(s MigrationOwnershipPartitionState, t MigrationOwnershipPartitionTransition, index uint64) bool { + return (t.Operation == MigrationOwnershipPartitionSplit || t.Operation == MigrationOwnershipPartitionRefresh) && + len(t.RootManifestHash) == sha256.Size && len(t.Before) > 0 && len(t.After) > 0 && index != 0 && + (!partitionHasEvidence(s) || s.Generation == t.Generation) +} + +func partitionTransitionReplay(s MigrationOwnershipPartitionState, operation string, id []byte) (bool, error) { + for _, receipt := range s.Receipts { + if receipt.Transition.OperationID != operation { + continue + } + if !bytes.Equal(id, receipt.ReceiptID) { + return false, ErrMigrationOwnership + } + return true, nil + } + return false, nil +} + +func resolvePartitionBefore(s MigrationOwnershipPartitionState, legacy []MigrationOwnershipState, t MigrationOwnershipPartitionTransition) ([]MigrationOwnershipPartition, []MigrationOwnershipPartition, error) { + candidates, err := partitionCandidates(s, legacy) + if err != nil { + return nil, nil, err + } + var before, retained []MigrationOwnershipPartition + used := make([]bool, len(t.Before)) + for _, c := range candidates { + matched, matchErr := matchPartitionReference(t.Before, c.reference, used) + if matchErr != nil { + return nil, nil, matchErr + } + if intersectsPartitionList(c.partition, t.After) != matched { + return nil, nil, ErrMigrationOwnership + } + if matched { + if c.partition.Generation != t.Generation || c.partition.LocalGroupID != t.LocalGroupID { + return nil, nil, ErrMigrationOwnership + } + before = append(before, c.partition) + } else if c.reference.LegacyJobID == 0 { + retained = append(retained, c.partition) + } + } + if slices.Contains(used, false) { + return nil, nil, ErrMigrationOwnership + } + return before, retained, nil +} + +func matchPartitionReference(refs []MigrationOwnershipPartitionReference, candidate MigrationOwnershipPartitionReference, used []bool) (bool, error) { + matched := false + for i, ref := range refs { + if !equalPartitionReference(ref, candidate) { + continue + } + if used[i] || matched { + return false, ErrMigrationOwnership + } + used[i], matched = true, true + } + return matched, nil +} + +func intersectsPartitionList(p MigrationOwnershipPartition, list []MigrationOwnershipPartition) bool { + for _, other := range list { + if rangesOverlap(p, other) { + return true + } + } + return false +} + +func validatePartitionChildren(t *MigrationOwnershipPartitionTransition, before []MigrationOwnershipPartition, index uint64, id []byte) error { + for i := range t.After { + p := &t.After[i] + if p.Generation != t.Generation || p.LocalGroupID != t.LocalGroupID || p.CatalogVersion != t.CatalogVersion || p.ConfigurationIndex != t.ConfigurationIndex { + return ErrMigrationOwnership + } + p.AppliedIndex, p.ReceiptID = index, bytes.Clone(id) + if !validPartition(*p) { + return ErrMigrationOwnership + } + if !partitionChildIdentityMatches(*t, i, before) { + return ErrMigrationOwnership + } + } + return nil +} + +func partitionChildIdentityMatches(t MigrationOwnershipPartitionTransition, i int, before []MigrationOwnershipPartition) bool { + p := t.After[i] + if t.Operation != MigrationOwnershipPartitionRefresh { + expected, err := MigrationOwnershipPartitionIDFor(t, i) + return err == nil && bytes.Equal(p.PartitionID, expected) + } + for _, old := range before { + a := clonePartition(old) + a.CatalogVersion, a.ConfigurationIndex, a.AppliedIndex, a.ReceiptID = p.CatalogVersion, p.ConfigurationIndex, p.AppliedIndex, p.ReceiptID + if samePartition(a, p) { + return true + } + } + return false +} + +func appendPartitionReceipt(s MigrationOwnershipPartitionState, retained []MigrationOwnershipPartition, t MigrationOwnershipPartitionTransition, index uint64, id []byte) (MigrationOwnershipPartitionState, error) { + s = clonePartitionState(s) + s.Generation = t.Generation + s.Partitions = retained + s.Partitions = append(s.Partitions, t.After...) + s.Receipts = append(s.Receipts, MigrationOwnershipPartitionReceipt{ReceiptID: id, Transition: t, AppliedIndex: index}) + data, err := encodeMigrationOwnershipPartitions(s) + if err != nil { + return s, err + } + return decodeMigrationOwnershipPartitions(data) +} diff --git a/store/migration_ownership_partition_boundary_test.go b/store/migration_ownership_partition_boundary_test.go new file mode 100644 index 000000000..56f35b4a8 --- /dev/null +++ b/store/migration_ownership_partition_boundary_test.go @@ -0,0 +1,118 @@ +package store + +import ( + "bytes" + "encoding/binary" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOwnershipPartitionSourceZeroFloorAndConfigurationCAS(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + writer, ok := st.(MigrationTargetReadinessRaftWriter) + require.True(t, ok) + guard := TargetStagedReadinessState{JobID: 7, MigrationJobID: 7, Armed: true, TrackWrites: true, + RouteStart: []byte("m"), RouteEnd: []byte("z"), ExpectedCutoverVersion: 1, MinWriteTSExclusive: 1, RetentionPinTS: 1} + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), guard, 10)) + state := MigrationOwnershipState{Generation: "0123456789abcdef0123456789abcdef", JobID: 7, LocalGroupID: 1, OwnerGroupID: 1, + Role: MigrationOwner, CatalogVersion: 1, ConfigurationIndex: 9, RouteID: 1, DescriptorHash: bytes.Repeat([]byte{1}, 32), + RouteStart: []byte("m"), RouteEnd: []byte("z"), WriteFloorKnown: true, SourceTrackerVersion: 1} + require.NoError(t, ownershipStore(t, st).AcquireMigrationOwnership(t.Context(), state, 11)) + require.NoError(t, st.PutAt(t.Context(), MigrationConfigurationIndexKey(), binary.BigEndian.AppendUint64(nil, 9), 1, 0)) + view, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + ref, err := MigrationOwnershipLegacyReferenceFor(view.Ownership[0]) + require.NoError(t, err) + left, right := partitionFromLegacy(view.Ownership[0]), partitionFromLegacy(view.Ownership[0]) + left.RouteID, left.RouteEnd, left.DescriptorHash = 2, []byte("x"), bytes.Repeat([]byte{2}, 32) + right.RouteID, right.RouteStart, right.DescriptorHash = 3, []byte("x"), bytes.Repeat([]byte{3}, 32) + tx := MigrationOwnershipPartitionTransition{OperationID: "source/split", Operation: MigrationOwnershipPartitionSplit, + RootManifestHash: bytes.Repeat([]byte{4}, 32), Generation: state.Generation, LocalGroupID: 1, CatalogVersion: 2, + ConfigurationIndex: 9, Before: []MigrationOwnershipPartitionReference{ref}, After: []MigrationOwnershipPartition{left, right}} + finalizePartitionTransition(t, &tx) + proofs := partitionStore(t, st) + require.NoError(t, proofs.ApplyMigrationOwnershipPartition(t.Context(), tx, 12)) + frontier, err := proofs.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + for _, p := range frontier.Partitions { + require.Zero(t, p.MinWriteTSExclusive) + require.True(t, p.WriteFloorKnown) + require.Equal(t, uint64(11), p.SourceAppliedIndex) + require.True(t, MigrationOwnershipPartitionEvidenceValid(frontier, p)) + } + ref, err = MigrationOwnershipPartitionReferenceFor(frontier.Partitions[0]) + require.NoError(t, err) + refresh := tx + refresh.OperationID, refresh.Operation, refresh.CatalogVersion = "source/refresh", MigrationOwnershipPartitionRefresh, 3 + refresh.Before, refresh.After = []MigrationOwnershipPartitionReference{ref}, []MigrationOwnershipPartition{frontier.Partitions[0]} + finalizePartitionTransition(t, &refresh) + require.NoError(t, st.PutAt(t.Context(), MigrationConfigurationIndexKey(), binary.BigEndian.AppendUint64(nil, 10), 2, 0)) + require.ErrorIs(t, proofs.ApplyMigrationOwnershipPartition(t.Context(), refresh, 13), ErrMigrationOwnership, "durable epoch changed after proposal") + refresh.ConfigurationIndex = 10 + finalizePartitionTransition(t, &refresh) + require.NoError(t, proofs.ApplyMigrationOwnershipPartition(t.Context(), refresh, 13)) + current, err := proofs.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + require.Equal(t, frontier.Partitions[1], current.Partitions[1], "unrelated right remains unchanged") + require.NoError(t, proofs.ApplyMigrationOwnershipPartition(t.Context(), tx, 14), "old receipt replay cannot roll the new configuration back") + current, err = proofs.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + require.Equal(t, uint64(10), current.Partitions[0].ConfigurationIndex) + }) +} + +func TestOwnershipPartitionRootManifestExternalRefusal(t *testing.T) { + key := append([]byte("!dist|meta|ownership_partition/"), make([]byte, 40)...) + for _, format := range []string{"mvcc", "native", "sst", "encrypted-native", "encrypted-sst"} { + t.Run(format, func(t *testing.T) { + var raw []byte + var options []PebbleStoreOption + if format == "encrypted-native" || format == "encrypted-sst" { + fixture := newEncryptedStoreFixture(t, 7) + ps, ok := fixture.mvcc.(*pebbleStore) + require.True(t, ok) + ps.sstIngestSnapshots = format == "encrypted-sst" + require.NoError(t, fixture.mvcc.PutAt(t.Context(), key, []byte("root manifest"), 1, 0)) + raw = externalTestSnapshot(t, fixture.mvcc) + options = []PebbleStoreOption{WithEncryption(fixture.cipher, NewCounterNonceFactory(0xABCD, 2), func() (uint32, bool) { return fixture.keyID, true })} + } else { + source := externalTestMemory(t) + require.NoError(t, source.PutAt(t.Context(), key, []byte("root manifest"), 1, 0)) + raw = externalTestFormat(t, source, format) + } + destination, err := NewPebbleStore(t.TempDir(), options...) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, destination.Close()) }) + require.NoError(t, destination.PutAt(t.Context(), []byte("sentinel"), []byte("unchanged"), 1, 0)) + before := externalTestSnapshot(t, destination) + require.ErrorIs(t, externalTestRestore(t, destination, raw), ErrExternalSnapshotMigrationState) + require.Equal(t, before, externalTestSnapshot(t, destination)) + if format == "mvcc" { + require.ErrorIs(t, externalTestRestore(t, NewMVCCStore(), raw), ErrExternalSnapshotMigrationState) + } + }) + } + require.False(t, isMigrationMetadataKey(key), "root manifests are logical rows, never raw store containers") + require.NoError(t, validateExternalTypedMigrationKey([]byte("!dist|meta|ownership_partition-user"))) +} + +func TestOwnershipPartitionHashesCanonicalAndBound(t *testing.T) { + st := NewMVCCStore() + _, tx := partitionFixture(t, st) + id, err := MigrationOwnershipPartitionReceiptIDFor(tx) + require.NoError(t, err) + permuted := canonicalPartitionTransition(tx) + permuted.After[0], permuted.After[1] = permuted.After[1], permuted.After[0] + permuted.After[0].AppliedIndex, permuted.After[0].ReceiptID = 99, bytes.Repeat([]byte{9}, 32) + other, err := MigrationOwnershipPartitionReceiptIDFor(permuted) + require.NoError(t, err) + require.Equal(t, id, other, "sorting and store-stamped boundaries do not alter operation identity") + permuted.After[0].Origins[0].AppliedIndex++ + other, err = MigrationOwnershipPartitionReceiptIDFor(permuted) + require.NoError(t, err) + require.NotEqual(t, id, other, "immutable origin boundary is bound") + key := MigrationOwnershipPartitionMetadataKey() + key[0]++ + require.True(t, IsMigrationOwnershipPartitionMetadataKey([]byte("!migstage|ownership_partition"))) +} diff --git a/store/migration_ownership_partition_hash.go b/store/migration_ownership_partition_hash.go new file mode 100644 index 000000000..ba8175df6 --- /dev/null +++ b/store/migration_ownership_partition_hash.go @@ -0,0 +1,147 @@ +package store + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "sort" + + "github.com/cockroachdb/errors" +) + +var migrationOwnershipPartitionMetaKey = []byte("!migstage|ownership_partition") + +func MigrationOwnershipPartitionMetadataKey() []byte { + return bytes.Clone(migrationOwnershipPartitionMetaKey) +} + +func IsMigrationOwnershipPartitionMetadataKey(key []byte) bool { + return bytes.Equal(key, migrationOwnershipPartitionMetaKey) +} + +func partitionDigest(domain string, value any) ([]byte, error) { + data, err := json.Marshal(value) + if err != nil { + return nil, errors.WithStack(err) + } + sum := sha256.Sum256(append([]byte(domain+"\x00"), data...)) + return bytes.Clone(sum[:]), nil +} + +func clonePartition(p MigrationOwnershipPartition) MigrationOwnershipPartition { + p.PartitionID = bytes.Clone(p.PartitionID) + p.ReceiptID = bytes.Clone(p.ReceiptID) + p.RouteStart = bytes.Clone(p.RouteStart) + p.RouteEnd = bytes.Clone(p.RouteEnd) + p.DescriptorHash = bytes.Clone(p.DescriptorHash) + p.Origins = append([]MigrationOwnershipOperationReference(nil), p.Origins...) + for i := range p.Origins { + p.Origins[i].DescriptorHash = bytes.Clone(p.Origins[i].DescriptorHash) + } + sort.Slice(p.Origins, func(i, j int) bool { + a, _ := json.Marshal(p.Origins[i]) + b, _ := json.Marshal(p.Origins[j]) + return bytes.Compare(a, b) < 0 + }) + return p +} + +func canonicalPartitionTransition(t MigrationOwnershipPartitionTransition) MigrationOwnershipPartitionTransition { + t.RootManifestHash = bytes.Clone(t.RootManifestHash) + t.Before = append([]MigrationOwnershipPartitionReference(nil), t.Before...) + for i := range t.Before { + t.Before[i].PartitionID = bytes.Clone(t.Before[i].PartitionID) + t.Before[i].AttestationHash = bytes.Clone(t.Before[i].AttestationHash) + } + sort.Slice(t.Before, func(i, j int) bool { + if t.Before[i].LegacyJobID != t.Before[j].LegacyJobID { + return t.Before[i].LegacyJobID < t.Before[j].LegacyJobID + } + return bytes.Compare(t.Before[i].PartitionID, t.Before[j].PartitionID) < 0 + }) + t.After = append([]MigrationOwnershipPartition(nil), t.After...) + for i := range t.After { + t.After[i] = clonePartition(t.After[i]) + } + sort.Slice(t.After, func(i, j int) bool { + if c := bytes.Compare(t.After[i].RouteStart, t.After[j].RouteStart); c != 0 { + return c < 0 + } + return bytes.Compare(t.After[i].PartitionID, t.After[j].PartitionID) < 0 + }) + return t +} + +func clonePartitionState(s MigrationOwnershipPartitionState) MigrationOwnershipPartitionState { + s.Partitions = append([]MigrationOwnershipPartition(nil), s.Partitions...) + for i := range s.Partitions { + s.Partitions[i] = clonePartition(s.Partitions[i]) + } + sort.Slice(s.Partitions, func(i, j int) bool { + return bytes.Compare(s.Partitions[i].RouteStart, s.Partitions[j].RouteStart) < 0 + }) + s.Receipts = append([]MigrationOwnershipPartitionReceipt(nil), s.Receipts...) + for i := range s.Receipts { + s.Receipts[i].ReceiptID = bytes.Clone(s.Receipts[i].ReceiptID) + s.Receipts[i].Transition = canonicalPartitionTransition(s.Receipts[i].Transition) + } + sort.Slice(s.Receipts, func(i, j int) bool { + return bytes.Compare(s.Receipts[i].ReceiptID, s.Receipts[j].ReceiptID) < 0 + }) + return s +} + +func MigrationOwnershipPartitionReferenceFor(p MigrationOwnershipPartition) (MigrationOwnershipPartitionReference, error) { + if !validPartition(p) { + return MigrationOwnershipPartitionReference{}, ErrMigrationOwnership + } + hash, err := partitionDigest("elastickv/ownership-partition/attestation/v1", clonePartition(p)) + return MigrationOwnershipPartitionReference{PartitionID: bytes.Clone(p.PartitionID), AttestationHash: hash}, err +} + +func MigrationOwnershipLegacyReferenceFor(p MigrationOwnershipState) (MigrationOwnershipPartitionReference, error) { + if !MigrationOwnershipEvidenceValid(p) { + return MigrationOwnershipPartitionReference{}, ErrMigrationOwnership + } + hash, err := partitionDigest("elastickv/ownership-partition/legacy/v1", cloneMigrationOwnership([]MigrationOwnershipState{p})[0]) + return MigrationOwnershipPartitionReference{LegacyJobID: p.JobID, AttestationHash: hash}, err +} + +func MigrationOwnershipPartitionIDFor(t MigrationOwnershipPartitionTransition, childIndex int) ([]byte, error) { + if childIndex < 0 || childIndex >= len(t.After) || t.OperationID == "" || t.Generation == "" || t.LocalGroupID == 0 { + return nil, ErrMigrationOwnership + } + child := t.After[childIndex] + if t.Operation == MigrationOwnershipPartitionRefresh { + if len(child.PartitionID) != sha256.Size { + return nil, ErrMigrationOwnership + } + return bytes.Clone(child.PartitionID), nil + } + parents := canonicalPartitionTransition(t).Before + for i := range parents { + if parents[i].LegacyJobID == 0 { + parents[i].AttestationHash = nil + } + } + return partitionDigest("elastickv/ownership-partition/id/v1", struct { + Generation string + Group uint64 + Operation string + Parents []MigrationOwnershipPartitionReference + Route uint64 + Start, End []byte + }{t.Generation, t.LocalGroupID, t.OperationID, parents, child.RouteID, child.RouteStart, child.RouteEnd}) +} + +func MigrationOwnershipPartitionReceiptIDFor(t MigrationOwnershipPartitionTransition) ([]byte, error) { + t = canonicalPartitionTransition(t) + if t.OperationID == "" || t.Generation == "" || t.LocalGroupID == 0 { + return nil, ErrMigrationOwnership + } + for i := range t.After { + t.After[i].AppliedIndex = 0 + t.After[i].ReceiptID = nil + } + return partitionDigest("elastickv/ownership-partition/receipt/v1", t) +} diff --git a/store/migration_ownership_partition_job.go b/store/migration_ownership_partition_job.go new file mode 100644 index 000000000..e67c804e6 --- /dev/null +++ b/store/migration_ownership_partition_job.go @@ -0,0 +1,190 @@ +package store + +import ( + "bytes" + "encoding/hex" + + "github.com/cockroachdb/errors" +) + +func partitionJobReplay(s MigrationOwnershipPartitionState, ledger []MigrationOwnershipState, requested MigrationOwnershipState, operation MigrationOwnershipOperation) bool { + if !partitionHasEvidence(s) || operation == MigrationOwnershipRefresh { + return false + } + for _, old := range ledger { + if old.JobID != requested.JobID { + continue + } + return partitionLegacyOperationReplay(old, requested, operation) + } + return false +} + +func partitionLegacyOperationReplay(old, requested MigrationOwnershipState, operation MigrationOwnershipOperation) bool { + if requested.SourceTrackerVersion != 0 && requested.SourceAppliedIndex == requested.AppliedIndex { + requested.SourceAppliedIndex = old.SourceAppliedIndex + } + if !sameMigrationOwnershipDescriptor(old, requested) { + return false + } + for _, r := range old.Receipts { + if partitionJobReceiptMatches(r, requested, operation) { + return true + } + } + return false +} + +func partitionJobReceiptMatches(r MigrationOwnershipReceipt, requested MigrationOwnershipState, operation MigrationOwnershipOperation) bool { + return r.Operation == operation && r.CatalogVersion == requested.CatalogVersion && r.ConfigurationIndex == requested.ConfigurationIndex && + r.Staged == requested.Staged && bytes.Equal(r.DescriptorHash, requested.DescriptorHash) +} + +// Only callers that already validated the real job guard may invoke this. +// External partition controls cannot choose the internal JobOperation opcode. +func syncPartitionJob(s MigrationOwnershipPartitionState, oldLedger, ledger []MigrationOwnershipState, requested MigrationOwnershipState, operation MigrationOwnershipOperation, index, configuration uint64, ownerChangeAllowed bool) (MigrationOwnershipPartitionState, error) { + if !partitionHasEvidence(s) { + return s, nil + } + if partitionJobReplay(s, oldLedger, requested, operation) { + return clonePartitionState(s), nil + } + if !partitionJobEpochMatches(s, requested, configuration) { + return s, ErrMigrationOwnership + } + state := partitionJobState(ledger, requested.JobID) + if !MigrationOwnershipEvidenceValid(state) { + return s, ErrMigrationOwnership + } + child := partitionFromLegacy(state) + replacement, err := selectPartitionJobReplacement(s, oldLedger, child, index, ownerChangeAllowed) + if err != nil { + return s, err + } + if operation == MigrationOwnershipRefresh { + var unadopted bool + child, unadopted, err = refreshPartitionJobChild(s, child, replacement.matched) + if err != nil || unadopted { + return clonePartitionState(s), err + } + } + tx, id, err := makePartitionJobTransition(state, operation, replacement.before, child, index) + if err != nil { + return s, err + } + return appendPartitionReceipt(s, replacement.retained, tx, index, id) +} + +func partitionJobEpochMatches(s MigrationOwnershipPartitionState, requested MigrationOwnershipState, configuration uint64) bool { + return configuration != 0 && requested.ConfigurationIndex == configuration && requested.Generation == s.Generation +} + +func partitionJobState(ledger []MigrationOwnershipState, jobID uint64) MigrationOwnershipState { + for _, p := range ledger { + if p.JobID == jobID { + return p + } + } + return MigrationOwnershipState{} +} + +type partitionJobReplacement struct { + before []MigrationOwnershipPartitionReference + retained, matched []MigrationOwnershipPartition +} + +func selectPartitionJobReplacement(s MigrationOwnershipPartitionState, oldLedger []MigrationOwnershipState, child MigrationOwnershipPartition, index uint64, ownerChangeAllowed bool) (partitionJobReplacement, error) { + var result partitionJobReplacement + candidates, err := partitionCandidates(s, oldLedger) + if err != nil { + return result, err + } + for _, c := range candidates { + if !rangesOverlap(c.partition, child) { + if c.reference.LegacyJobID == 0 { + result.retained = append(result.retained, c.partition) + } + continue + } + if !validPartitionJobReplacement(c, child, index) { + return result, ErrMigrationOwnership + } + result.before = append(result.before, c.reference) + result.matched = append(result.matched, c.partition) + if child.Role == MigrationOwner && c.partition.Role == MigrationNonOwner && + (!ownerChangeAllowed || child.CatalogVersion <= c.partition.CatalogVersion) { + return result, ErrMigrationOwnership + } + } + return result, nil +} + +func validPartitionJobReplacement(c partitionCandidate, child MigrationOwnershipPartition, index uint64) bool { + return partitionRangeContains(child, c.partition) && len(c.reference.AttestationHash) == 32 && + c.partition.Generation == child.Generation && c.partition.LocalGroupID == child.LocalGroupID && + c.partition.MinWriteTSExclusive <= child.MinWriteTSExclusive && c.partition.CatalogVersion <= child.CatalogVersion && + c.partition.ConfigurationIndex <= child.ConfigurationIndex && c.partition.AppliedIndex <= index +} + +func partitionRangeContains(outer, inner MigrationOwnershipPartition) bool { + return bytes.Compare(outer.RouteStart, inner.RouteStart) <= 0 && + (len(outer.RouteEnd) == 0 || (len(inner.RouteEnd) != 0 && bytes.Compare(inner.RouteEnd, outer.RouteEnd) <= 0)) +} + +func refreshPartitionJobChild(s MigrationOwnershipPartitionState, child MigrationOwnershipPartition, matched []MigrationOwnershipPartition) (MigrationOwnershipPartition, bool, error) { + // An unadopted legacy interval remains in the per-job view. An adopted + // interval must be exactly one unchanged active descriptor, never first + // match by JobID among multiple descendants. + active := 0 + for _, p := range s.Partitions { + if p.MigrationJobID == child.MigrationJobID { + active++ + } + } + if active == 0 { + return child, true, nil + } + if active != 1 || len(matched) != 1 || len(matched[0].PartitionID) == 0 { + return child, false, ErrMigrationOwnership + } + old := matched[0] + if !samePartitionJobDescriptor(old, child) || !inheritedPartition(old, child) { + return child, false, ErrMigrationOwnership + } + child.PartitionID = bytes.Clone(old.PartitionID) + return child, false, nil +} + +func samePartitionJobDescriptor(a, b MigrationOwnershipPartition) bool { + return bytes.Equal(a.RouteStart, b.RouteStart) && bytes.Equal(a.RouteEnd, b.RouteEnd) && + a.RouteID == b.RouteID && a.ParentRouteID == b.ParentRouteID && bytes.Equal(a.DescriptorHash, b.DescriptorHash) +} + +func makePartitionJobTransition(state MigrationOwnershipState, operation MigrationOwnershipOperation, before []MigrationOwnershipPartitionReference, child MigrationOwnershipPartition, index uint64) (MigrationOwnershipPartitionTransition, []byte, error) { + // Bind the exact operation origins AND the new full attestation. In + // particular distinct configuration-bound refresh attempts cannot rewrite + // the operation receipt from a previous configuration. + identity, err := partitionDigest("elastickv/ownership-partition/job/v1", struct { + Operation MigrationOwnershipOperation + State MigrationOwnershipState + }{operation, state}) + if err != nil { + return MigrationOwnershipPartitionTransition{}, nil, errors.WithStack(err) + } + tx := MigrationOwnershipPartitionTransition{OperationID: "job/" + hex.EncodeToString(identity), Operation: MigrationOwnershipPartitionJobOperation, + Generation: state.Generation, LocalGroupID: state.LocalGroupID, CatalogVersion: state.CatalogVersion, + ConfigurationIndex: state.ConfigurationIndex, Before: before, After: []MigrationOwnershipPartition{child}} + if operation != MigrationOwnershipRefresh { + id, hashErr := MigrationOwnershipPartitionIDFor(tx, 0) + if hashErr != nil { + return tx, nil, hashErr + } + tx.After[0].PartitionID = id + } + receiptID, err := MigrationOwnershipPartitionReceiptIDFor(tx) + if err != nil { + return tx, nil, err + } + tx.After[0].AppliedIndex, tx.After[0].ReceiptID = index, bytes.Clone(receiptID) + return tx, receiptID, nil +} diff --git a/store/migration_ownership_partition_job_test.go b/store/migration_ownership_partition_job_test.go new file mode 100644 index 000000000..aab963dac --- /dev/null +++ b/store/migration_ownership_partition_job_test.go @@ -0,0 +1,143 @@ +package store + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOwnershipPartitionGuardedMoveBackAndClear(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + _, tx := partitionFixture(t, st) + proofs := ownershipStore(t, st) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + initial, err := proofs.MigrationServingState(t.Context()) + require.NoError(t, err) + incoming := MigrationOwnershipState{Generation: tx.Generation, JobID: 8, LocalGroupID: 1, OwnerGroupID: 1, + CatalogVersion: 14, ConfigurationIndex: 3, RouteID: 12, ParentRouteID: 10, RouteStart: []byte("m"), RouteEnd: []byte("z"), + DescriptorHash: bytes.Repeat([]byte{3}, 32), Role: MigrationOwner, MinWriteTSExclusive: 200, WriteFloorKnown: true, SourceTrackerVersion: 14} + guard := TargetStagedReadinessState{JobID: 8, MigrationJobID: 8, Armed: true, TrackWrites: true, + RouteStart: incoming.RouteStart, RouteEnd: incoming.RouteEnd, ExpectedCutoverVersion: 14, MinWriteTSExclusive: 200, RetentionPinTS: 1} + writer, ok := st.(MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.ErrorIs(t, proofs.AcquireMigrationOwnership(t.Context(), incoming, 11), ErrMigrationOwnership) + unguarded := incoming + unguarded.SourceTrackerVersion = 0 + require.ErrorIs(t, proofs.ClearMigrationStateWithOwnership(t.Context(), unguarded, 11), ErrMigrationOwnership, + "an unguarded ordinary Clear must not elevate a durable NONOWNER") + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), guard, 11)) + require.NoError(t, proofs.AcquireMigrationOwnership(t.Context(), incoming, 12)) + acquired, err := proofs.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Len(t, acquired.Partitions.Partitions, 2) + require.Equal(t, initial.Partitions.Partitions[0], acquired.Partitions.Partitions[0], "retained left is neither fenced nor raised") + right := acquired.Partitions.Partitions[1] + require.Equal(t, MigrationOwner, right.Role) + require.Equal(t, uint64(8), right.MigrationJobID) + require.Equal(t, uint64(12), right.SourceAppliedIndex) + require.True(t, MigrationOwnershipPartitionEvidenceValid(acquired.Partitions, right)) + admission, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + for _, legacy := range admission.Ownership { + require.NotEqual(t, incoming.JobID, legacy.JobID, "a job represented by the active frontier must not compete with its own partition") + } + require.NoError(t, proofs.AcquireMigrationOwnership(t.Context(), incoming, 13)) + guard.SourceWriteFence, guard.SourceReadFence, guard.ExpectedCutoverVersion = true, true, 15 + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), guard, 14)) + outgoing := incoming + outgoing.Role, outgoing.OwnerGroupID, outgoing.CatalogVersion, outgoing.SourceTrackerVersion = MigrationNonOwner, 2, 15, 0 + outgoing.DescriptorHash = bytes.Repeat([]byte{9}, 32) + require.NoError(t, proofs.ClearMigrationStateWithOwnership(t.Context(), outgoing, 15)) + cleared, err := proofs.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Empty(t, cleared.Readiness) + require.Equal(t, MigrationNonOwner, cleared.Partitions.Partitions[1].Role) + require.Zero(t, cleared.Partitions.Partitions[1].SourceTrackerVersion) + require.Equal(t, initial.Partitions.Partitions[0], cleared.Partitions.Partitions[0]) + require.Len(t, cleared.Partitions.Receipts, 3) + require.NoError(t, proofs.ClearMigrationStateWithOwnership(t.Context(), outgoing, 16)) + after, err := proofs.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, cleared, after, "retry must not restore the superseded SOURCE OWNER") + }) +} + +func TestOwnershipPartitionPartialGuardCannotClipFrontier(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + _, tx := partitionFixture(t, st) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + incoming := MigrationOwnershipState{Generation: tx.Generation, JobID: 8, LocalGroupID: 1, OwnerGroupID: 1, + CatalogVersion: 14, ConfigurationIndex: 3, RouteID: 20, RouteStart: []byte("n"), RouteEnd: []byte("z"), + DescriptorHash: bytes.Repeat([]byte{5}, 32), Role: MigrationOwner, MinWriteTSExclusive: 200, WriteFloorKnown: true, Staged: true} + guard := TargetStagedReadinessState{JobID: 8, MigrationJobID: 8, Armed: true, RouteStart: incoming.RouteStart, + RouteEnd: incoming.RouteEnd, ExpectedCutoverVersion: 14, MinWriteTSExclusive: 200} + writer, ok := st.(MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), guard, 11)) + before, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + require.ErrorIs(t, ownershipStore(t, st).AcquireMigrationOwnership(t.Context(), incoming, 12), ErrMigrationOwnership) + after, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, before, after, "all CAS/union checks precede any ledger, guard or receipt mutation") + }) +} + +func TestOwnershipPartitionMoveBackRequiresNewerActiveCatalog(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + _, tx := partitionFixture(t, st) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + incoming := MigrationOwnershipState{Generation: tx.Generation, JobID: 8, LocalGroupID: 1, OwnerGroupID: 1, + CatalogVersion: tx.CatalogVersion, ConfigurationIndex: 3, RouteID: 12, ParentRouteID: 10, + RouteStart: []byte("m"), RouteEnd: []byte("z"), DescriptorHash: bytes.Repeat([]byte{3}, 32), + Role: MigrationOwner, MinWriteTSExclusive: 200, WriteFloorKnown: true, Staged: true} + guard := TargetStagedReadinessState{JobID: 8, MigrationJobID: 8, Armed: true, RouteStart: incoming.RouteStart, + RouteEnd: incoming.RouteEnd, ExpectedCutoverVersion: incoming.CatalogVersion, MinWriteTSExclusive: 200} + writer, ok := st.(MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), guard, 11)) + before, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.ErrorIs(t, ownershipStore(t, st).AcquireMigrationOwnership(t.Context(), incoming, 12), ErrMigrationOwnership) + after, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Equal(t, before, after) + incoming.CatalogVersion++ + require.NoError(t, ownershipStore(t, st).AcquireMigrationOwnership(t.Context(), incoming, 13)) + }) +} + +func TestOwnershipPartitionLegacyRefreshRequiresExactSingleFrontier(t *testing.T) { + for _, split := range []bool{false, true} { + t.Run(map[bool]string{false: "one", true: "descendants"}[split], func(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + old, tx := partitionFixture(t, st) + if !split { + tx.After = []MigrationOwnershipPartition{partitionFromLegacy(old)} + finalizePartitionTransition(t, &tx) + } + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + before, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + old.CatalogVersion = 13 + refresher, ok := st.(MigrationOwnershipRefresher) + require.True(t, ok) + err = refresher.RefreshMigrationOwnership(t.Context(), old, 11) + after, readErr := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, readErr) + if split { + require.ErrorIs(t, err, ErrMigrationOwnership) + require.Equal(t, before, after) + } else { + require.NoError(t, err) + require.Equal(t, uint64(13), after.Partitions.Partitions[0].CatalogVersion) + require.Equal(t, before.Partitions.Partitions[0].PartitionID, after.Partitions.Partitions[0].PartitionID) + require.Equal(t, before.Partitions.Partitions[0].Origins, after.Partitions.Partitions[0].Origins) + require.Len(t, after.Partitions.Receipts, 2) + require.True(t, MigrationOwnershipPartitionEvidenceValid(after.Partitions, after.Partitions.Partitions[0])) + } + }) + }) + } +} diff --git a/store/migration_ownership_partition_namespace_test.go b/store/migration_ownership_partition_namespace_test.go new file mode 100644 index 000000000..3b4a6a0f6 --- /dev/null +++ b/store/migration_ownership_partition_namespace_test.go @@ -0,0 +1,167 @@ +package store + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" +) + +func applyPartitionNamespaceMarker(t *testing.T, st MVCCStore, configuration, index uint64) []byte { + t.Helper() + marker, err := EncodeMigrationNamespaceConfiguration(MigrationNamespaceConfiguration{ + Index: configuration, Version: MigrationNamespaceVersion, ActivatedAt: 3, Members: []uint64{1, 2, 3}, Joining: []uint64{4}, + }) + require.NoError(t, err) + require.Greater(t, len(marker), 8, "activated state must exercise the full marker, not a legacy uint64") + ts := st.LastCommitTS() + 1 + require.NoError(t, st.ApplyMutationsRaftAt(t.Context(), []*KVPairMutation{{ + Op: OpTypePut, Key: MigrationConfigurationIndexKey(), Value: marker, + }}, nil, ts, ts, index)) + return marker +} + +func TestOwnershipPartitionActivatedNamespaceGuardedAcquireClear(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + _, tx := partitionFixture(t, st) + applyPartitionNamespaceMarker(t, st, 3, 10) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 11)) + before, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + incoming := MigrationOwnershipState{Generation: tx.Generation, JobID: 8, LocalGroupID: 1, OwnerGroupID: 1, + CatalogVersion: 14, ConfigurationIndex: 3, RouteID: 12, ParentRouteID: 10, RouteStart: []byte("m"), RouteEnd: []byte("z"), + DescriptorHash: bytes.Repeat([]byte{3}, 32), Role: MigrationOwner, MinWriteTSExclusive: 200, + WriteFloorKnown: true, SourceTrackerVersion: 14} + guard := TargetStagedReadinessState{JobID: 8, MigrationJobID: 8, Armed: true, TrackWrites: true, + RouteStart: incoming.RouteStart, RouteEnd: incoming.RouteEnd, ExpectedCutoverVersion: 14, + MinWriteTSExclusive: 200, RetentionPinTS: 1} + writer, ok := st.(MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), guard, 12)) + require.NoError(t, ownershipStore(t, st).AcquireMigrationOwnership(t.Context(), incoming, 13)) + acquired, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Equal(t, before.Partitions.Partitions[0], acquired.Partitions.Partitions[0]) + require.Equal(t, uint64(13), acquired.Partitions.Partitions[1].SourceAppliedIndex) + require.True(t, MigrationOwnershipPartitionEvidenceValid(acquired.Partitions, acquired.Partitions.Partitions[1])) + guard.SourceWriteFence, guard.SourceReadFence, guard.ExpectedCutoverVersion = true, true, 15 + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), guard, 14)) + outgoing := incoming + outgoing.Role, outgoing.OwnerGroupID, outgoing.CatalogVersion, outgoing.SourceTrackerVersion = MigrationNonOwner, 2, 15, 0 + outgoing.DescriptorHash = bytes.Repeat([]byte{9}, 32) + require.NoError(t, ownershipStore(t, st).ClearMigrationStateWithOwnership(t.Context(), outgoing, 15)) + cleared, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Equal(t, MigrationNonOwner, cleared.Partitions.Partitions[1].Role) + require.Empty(t, cleared.Readiness) + require.Equal(t, before.Partitions.Partitions[0], cleared.Partitions.Partitions[0]) + }) +} + +func TestOwnershipPartitionActivatedNamespaceConfigurationCAS(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + _, tx := partitionFixture(t, st) + applyPartitionNamespaceMarker(t, st, 3, 10) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 11)) + before, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + p := before.Partitions.Partitions[0] + ref, err := MigrationOwnershipPartitionReferenceFor(p) + require.NoError(t, err) + refresh := tx + refresh.Operation, refresh.OperationID, refresh.CatalogVersion = MigrationOwnershipPartitionRefresh, "activated/refresh", 13 + refresh.Before, refresh.After = []MigrationOwnershipPartitionReference{ref}, []MigrationOwnershipPartition{p} + finalizePartitionTransition(t, &refresh) + applyPartitionNamespaceMarker(t, st, 4, 12) + require.ErrorIs(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), refresh, 13), ErrMigrationOwnership) + after, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Equal(t, before, after, "a stale configuration does not publish frontier or receipts") + refresh.ConfigurationIndex = 4 + finalizePartitionTransition(t, &refresh) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), refresh, 13)) + after, err = admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Equal(t, uint64(4), after.Partitions.Partitions[0].ConfigurationIndex) + require.Equal(t, before.Partitions.Partitions[1], after.Partitions.Partitions[1]) + }) +} + +func TestOwnershipPartitionActivatedNamespaceCorruptionRejectsBeforeEffects(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + _, tx := partitionFixture(t, st) + marker := applyPartitionNamespaceMarker(t, st, 3, 10) + ts := st.LastCommitTS() + 1 + require.NoError(t, st.ApplyMutationsRaftAt(t.Context(), []*KVPairMutation{{ + Op: OpTypePut, Key: MigrationConfigurationIndexKey(), Value: append(bytes.Clone(marker), '{'), + }}, nil, ts, ts, 11)) + before, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Error(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 12)) + after, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Equal(t, before, after) + }) +} + +func activatedPartitionSnapshotSource(t *testing.T, format string) (MVCCStore, []PebbleStoreOption) { + t.Helper() + if format == "mvcc" { + return NewMVCCStore(), nil + } + if format == "native" || format == "sst" { + source, err := NewPebbleStore(t.TempDir(), WithSSTIngestSnapshots(format == "sst")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, source.Close()) }) + return source, nil + } + fixture := newEncryptedStoreFixture(t, 7) + ps, ok := fixture.mvcc.(*pebbleStore) + require.True(t, ok) + ps.sstIngestSnapshots = format == "encrypted-sst" + return fixture.mvcc, []PebbleStoreOption{WithEncryption(fixture.cipher, NewCounterNonceFactory(0xABCD, 2), + func() (uint32, bool) { return fixture.keyID, true })} +} + +func TestOwnershipPartitionActivatedNamespaceSnapshotReopen(t *testing.T) { + for _, format := range []string{"mvcc", "native", "sst", "encrypted-native", "encrypted-sst"} { + t.Run(format, func(t *testing.T) { + source, options := activatedPartitionSnapshotSource(t, format) + _, tx := partitionFixture(t, source) + marker := applyPartitionNamespaceMarker(t, source, 3, 10) + require.NoError(t, partitionStore(t, source).ApplyMigrationOwnershipPartition(t.Context(), tx, 11)) + expected, err := admissionReader(t, source).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + raw := externalTestSnapshot(t, source) + dir := t.TempDir() + destination, err := NewPebbleStore(dir, options...) + require.NoError(t, err) + require.NoError(t, destination.PutAt(t.Context(), []byte("sentinel"), []byte("unchanged"), 1, 0)) + before := externalTestSnapshot(t, destination) + require.ErrorIs(t, externalTestRestore(t, destination, raw), ErrExternalSnapshotMigrationState) + require.Equal(t, before, externalTestSnapshot(t, destination)) + require.NoError(t, destination.Restore(bytes.NewReader(raw))) + require.NoError(t, destination.Close()) + destination, err = NewPebbleStore(dir, options...) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, destination.Close()) }) + actual, err := admissionReader(t, destination).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Equal(t, expected, actual) + value, err := destination.GetAt(t.Context(), MigrationConfigurationIndexKey(), ^uint64(0)) + require.NoError(t, err) + require.Equal(t, marker, value) + configuration, err := partitionConfiguration(value, nil) + require.NoError(t, err) + require.Equal(t, uint64(3), configuration) + p := actual.Partitions.Partitions[0] + ref, err := MigrationOwnershipPartitionReferenceFor(p) + require.NoError(t, err) + refresh := tx + refresh.Operation, refresh.OperationID, refresh.CatalogVersion = MigrationOwnershipPartitionRefresh, "restored/activated/refresh", 13 + refresh.Before, refresh.After = []MigrationOwnershipPartitionReference{ref}, []MigrationOwnershipPartition{p} + finalizePartitionTransition(t, &refresh) + require.NoError(t, partitionStore(t, destination).ApplyMigrationOwnershipPartition(t.Context(), refresh, 12)) + }) + } +} diff --git a/store/migration_ownership_partition_property_test.go b/store/migration_ownership_partition_property_test.go new file mode 100644 index 000000000..d4dbe6a4c --- /dev/null +++ b/store/migration_ownership_partition_property_test.go @@ -0,0 +1,160 @@ +package store + +import ( + "bytes" + "encoding/hex" + "slices" + "testing" + + "github.com/stretchr/testify/require" + "pgregory.net/rapid" +) + +func codecPropertyPartitionFixture(t *testing.T) ([]MigrationOwnershipState, MigrationOwnershipPartitionTransition) { + t.Helper() + st := NewMVCCStore() + t.Cleanup(func() { require.NoError(t, st.Close()) }) + old, transition := partitionFixture(t, st) + newer := old + newer.JobID, newer.CatalogVersion, newer.OwnerGroupID, newer.Role = 8, 11, 1, MigrationOwner + newer.RouteStart = []byte("m") + newer.DescriptorHash = bytes.Repeat([]byte{5}, 32) + require.NoError(t, ownershipStore(t, st).ClearMigrationStateWithOwnership(t.Context(), newer, 10)) + view, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + ref, err := MigrationOwnershipLegacyReferenceFor(view.Ownership[1]) + require.NoError(t, err) + transition.Before = append(transition.Before, ref) + transition.After[1] = partitionFromLegacy(view.Ownership[1]) + finalizePartitionTransition(t, &transition) + return view.Ownership, transition +} + +func codecPropertyPartitionState(t *rapid.T, ledger []MigrationOwnershipState, seed MigrationOwnershipPartitionTransition) MigrationOwnershipPartitionState { + t.Helper() + transition := canonicalPartitionTransition(seed) + transition.OperationID = "codec/split/" + hex.EncodeToString(rapid.SliceOfN(rapid.Byte(), 0, 24).Draw(t, "operation")) + transition.RootManifestHash = rapid.SliceOfN(rapid.Byte(), 32, 32).Draw(t, "root hash") + transition.CatalogVersion += rapid.Uint64Range(0, 10000).Draw(t, "catalog increment") + transition.ConfigurationIndex += rapid.Uint64Range(0, 10000).Draw(t, "configuration increment") + index := rapid.Uint64Range(100, 1000000).Draw(t, "apply index") + for i := range transition.After { + child := &transition.After[i] + child.CatalogVersion, child.ConfigurationIndex = transition.CatalogVersion, transition.ConfigurationIndex + child.DescriptorHash = rapid.SliceOfN(rapid.Byte(), 32, 32).Draw(t, "child descriptor hash") + id, err := MigrationOwnershipPartitionIDFor(transition, i) + require.NoError(t, err) + child.PartitionID = id + } + state, err := applyPartitionTransition(MigrationOwnershipPartitionState{}, ledger, transition, index, transition.ConfigurationIndex) + require.NoError(t, err) + child := state.Partitions[rapid.IntRange(0, len(state.Partitions)-1).Draw(t, "refreshed child")] + ref, err := MigrationOwnershipPartitionReferenceFor(child) + require.NoError(t, err) + refresh := MigrationOwnershipPartitionTransition{ + OperationID: "codec/refresh", Operation: MigrationOwnershipPartitionRefresh, + Generation: state.Generation, LocalGroupID: child.LocalGroupID, + CatalogVersion: transition.CatalogVersion + 1, ConfigurationIndex: transition.ConfigurationIndex, + RootManifestHash: transition.RootManifestHash, Before: []MigrationOwnershipPartitionReference{ref}, + After: []MigrationOwnershipPartition{clonePartition(child)}, + } + refresh.After[0].CatalogVersion = refresh.CatalogVersion + state, err = applyPartitionTransition(state, ledger, refresh, index+1, refresh.ConfigurationIndex) + require.NoError(t, err) + return state +} + +func codecPropertyPermute[T any](t *rapid.T, values []T) { + t.Helper() + for i := len(values) - 1; i > 0; i-- { + j := rapid.IntRange(0, i).Draw(t, "permutation") + values[i], values[j] = values[j], values[i] + } +} + +func TestMigrationCodecPropertyPartitionRoundTripAndHashes(t *testing.T) { + ledger, seed := codecPropertyPartitionFixture(t) + rapid.Check(t, func(t *rapid.T) { + state := codecPropertyPartitionState(t, ledger, seed) + encoded, err := encodeMigrationOwnershipPartitions(state) + require.NoError(t, err) + decoded, err := decodeMigrationOwnershipPartitions(encoded) + require.NoError(t, err) + require.Equal(t, state, decoded) + for _, child := range decoded.Partitions { + require.True(t, MigrationOwnershipPartitionEvidenceValid(decoded, child)) + } + permuted := clonePartitionState(state) + codecPropertyPermute(t, permuted.Partitions) + codecPropertyPermute(t, permuted.Receipts) + for i := range permuted.Receipts { + transition := &permuted.Receipts[i].Transition + codecPropertyPermute(t, transition.Before) + codecPropertyPermute(t, transition.After) + id, hashErr := MigrationOwnershipPartitionReceiptIDFor(*transition) + require.NoError(t, hashErr) + require.Equal(t, permuted.Receipts[i].ReceiptID, id) + for j, child := range transition.After { + id, hashErr = MigrationOwnershipPartitionIDFor(*transition, j) + require.NoError(t, hashErr) + require.Equal(t, child.PartitionID, id) + } + } + reencoded, err := encodeMigrationOwnershipPartitions(permuted) + require.NoError(t, err) + require.Equal(t, encoded, reencoded) + codecPropertyPartitionHashBinding(t, state.Receipts[0].Transition) + }) +} + +func codecPropertyPartitionHashBinding(t *rapid.T, transition MigrationOwnershipPartitionTransition) { + t.Helper() + id, err := MigrationOwnershipPartitionReceiptIDFor(transition) + require.NoError(t, err) + changed := canonicalPartitionTransition(transition) + changed.RootManifestHash[0] ^= 1 + different, err := MigrationOwnershipPartitionReceiptIDFor(changed) + require.NoError(t, err) + require.NotEqual(t, id, different) + stamped := canonicalPartitionTransition(transition) + for i := range stamped.After { + stamped.After[i].AppliedIndex = rapid.Uint64().Draw(t, "ignored apply stamp") + stamped.After[i].ReceiptID = []byte("ignored receipt stamp") + } + same, err := MigrationOwnershipPartitionReceiptIDFor(stamped) + require.NoError(t, err) + require.Equal(t, id, same, "receipt identity excludes apply-time stamps") +} + +func TestMigrationCodecPropertyPartitionRejectsMalformed(t *testing.T) { + ledger, seed := codecPropertyPartitionFixture(t) + rapid.Check(t, func(t *rapid.T) { + state := codecPropertyPartitionState(t, ledger, seed) + mutations := []struct { + name string + mutate func(*MigrationOwnershipPartitionState) + }{ + {"generation", func(s *MigrationOwnershipPartitionState) { s.Generation += "different" }}, + {"receipt digest", func(s *MigrationOwnershipPartitionState) { s.Receipts[0].ReceiptID[0] ^= 1 }}, + {"duplicate receipt", func(s *MigrationOwnershipPartitionState) { s.Receipts = append(s.Receipts, s.Receipts[0]) }}, + {"missing receipts", func(s *MigrationOwnershipPartitionState) { s.Receipts = nil }}, + {"frontier stamp", func(s *MigrationOwnershipPartitionState) { s.Partitions[0].AppliedIndex++ }}, + {"unknown floor", func(s *MigrationOwnershipPartitionState) { s.Partitions[0].WriteFloorKnown = false }}, + {"duplicate frontier", func(s *MigrationOwnershipPartitionState) { s.Partitions = append(s.Partitions, s.Partitions[0]) }}, + } + for _, mutation := range mutations { + bad := clonePartitionState(state) + mutation.mutate(&bad) + encoded, err := encodeMigrationOwnershipPartitions(bad) + require.NoError(t, err) + _, err = decodeMigrationOwnershipPartitions(encoded) + require.Error(t, err, mutation.name) + } + encoded, err := encodeMigrationOwnershipPartitions(state) + require.NoError(t, err) + for _, bad := range [][]byte{slices.Clone(encoded[:len(encoded)-1]), append([]byte{0}, encoded[1:]...), append(slices.Clone(encoded), '{')} { + _, err = decodeMigrationOwnershipPartitions(bad) + require.Error(t, err) + } + }) +} diff --git a/store/migration_ownership_partition_snapshot_test.go b/store/migration_ownership_partition_snapshot_test.go new file mode 100644 index 000000000..2104fd756 --- /dev/null +++ b/store/migration_ownership_partition_snapshot_test.go @@ -0,0 +1,200 @@ +package store + +import ( + "bytes" + "context" + "encoding/binary" + "log/slog" + "testing" + + "github.com/cockroachdb/pebble/v2" + "github.com/stretchr/testify/require" +) + +func TestOwnershipPartitionSnapshotFormatsAndReopen(t *testing.T) { + for _, format := range []string{"mvcc", "native", "sst", "encrypted-native", "encrypted-sst"} { + t.Run(format, func(t *testing.T) { + var source MVCCStore + var options []PebbleStoreOption + switch format { + case "mvcc": + source = NewMVCCStore() + case "native", "sst": + var err error + source, err = NewPebbleStore(t.TempDir(), WithSSTIngestSnapshots(format == "sst")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, source.Close()) }) + default: + fixture := newEncryptedStoreFixture(t, 7) + source = fixture.mvcc + ps, ok := source.(*pebbleStore) + require.True(t, ok) + ps.sstIngestSnapshots = format == "encrypted-sst" + options = []PebbleStoreOption{WithEncryption(fixture.cipher, NewCounterNonceFactory(0xABCD, 2), func() (uint32, bool) { return fixture.keyID, true })} + } + _, tx := partitionFixture(t, source) + require.NoError(t, partitionStore(t, source).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + require.NoError(t, source.PutAt(t.Context(), []byte("user"), []byte("payload"), 20, 0)) + expected, err := ownershipStore(t, source).MigrationServingState(t.Context()) + require.NoError(t, err) + expectedAdmission, err := admissionReader(t, source).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + raw := externalTestSnapshot(t, source) + if format == "mvcc" { + require.Equal(t, mvccSnapshotVersionV7, binary.LittleEndian.Uint32(raw[len(mvccSnapshotMagic):])) + memory := NewMVCCStore() + require.NoError(t, memory.Restore(bytes.NewReader(raw))) + view, readErr := ownershipStore(t, memory).MigrationServingState(t.Context()) + require.NoError(t, readErr) + require.Equal(t, expected, view) + } + dir := t.TempDir() + destination, err := NewPebbleStore(dir, options...) + require.NoError(t, err) + require.NoError(t, destination.PutAt(t.Context(), []byte("sentinel"), []byte("unchanged"), 1, 0)) + before := externalTestSnapshot(t, destination) + require.ErrorIs(t, externalTestRestore(t, destination, raw), ErrExternalSnapshotMigrationState) + require.Equal(t, before, externalTestSnapshot(t, destination)) + require.NoError(t, destination.Restore(bytes.NewReader(raw))) + require.NoError(t, destination.Close()) + destination, err = NewPebbleStore(dir, options...) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, destination.Close()) }) + actual, err := ownershipStore(t, destination).MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, expected, actual) + actualAdmission, err := admissionReader(t, destination).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Equal(t, expectedAdmission, actualAdmission) + value, err := destination.GetAt(t.Context(), []byte("user"), 20) + require.NoError(t, err) + require.Equal(t, []byte("payload"), value) + // The immutable original receipt remains an idempotency key after + // every format conversion and reopen. + require.NoError(t, partitionStore(t, destination).ApplyMigrationOwnershipPartition(t.Context(), tx, 11)) + }) + } +} + +func TestOwnershipPartitionExternalRestoreRejectsContainerAlone(t *testing.T) { + for _, receiptOnly := range []bool{false, true} { + t.Run(map[bool]string{false: "frontier", true: "receipts-only"}[receiptOnly], func(t *testing.T) { + memory := NewMVCCStore() + _, tx := partitionFixture(t, memory) + require.NoError(t, partitionStore(t, memory).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + ms, ok := memory.(*mvccStore) + require.True(t, ok) + ms.migrationOwnership, ms.migrationRetired = nil, nil + if receiptOnly { + ms.migrationPartitions.Partitions = nil + } + for _, format := range []string{"mvcc", "native", "sst"} { + raw := externalTestFormat(t, ms, format) + runMigrationStoreSuite(t, func(t *testing.T, destination MVCCStore) { + if _, isMemory := destination.(*mvccStore); isMemory && format != "mvcc" { + return + } + require.NoError(t, destination.PutAt(t.Context(), []byte("sentinel"), []byte("unchanged"), 1, 0)) + before := externalTestSnapshot(t, destination) + err := externalTestRestore(t, destination, raw) + require.ErrorIs(t, err, ErrExternalSnapshotMigrationState) + require.Equal(t, before, externalTestSnapshot(t, destination)) + }) + } + }) + } +} + +func TestOwnershipPartitionFailedWALBatchKeepsFrontierReceiptAndIndex(t *testing.T) { + db, err := pebble.Open(t.TempDir(), &pebble.Options{DisableWAL: true}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + st := &pebbleStore{db: db, log: slog.Default(), fsmApplyWriteOpts: pebble.NoSync} + _, tx := partitionFixture(t, st) + st.fsmApplyWriteOpts = pebble.Sync + before, err := st.MigrationServingState(t.Context()) + require.NoError(t, err) + beforeAdmission, err := st.MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.ErrorContains(t, st.ApplyMigrationOwnershipPartition(t.Context(), tx, 10), "WAL disabled") + after, err := st.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, before, after) + afterAdmission, err := st.MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Equal(t, beforeAdmission, afterAdmission) + index, found, err := st.LastAppliedIndex() + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint64(9), index) + st.fsmApplyWriteOpts = pebble.NoSync + require.NoError(t, st.ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + before, err = st.MigrationServingState(t.Context()) + require.NoError(t, err) + beforeAdmission, err = st.MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + st.fsmApplyWriteOpts = pebble.Sync + incoming := before.Ownership[0] + incoming.JobID, incoming.CatalogVersion = 8, 14 + incoming.RouteStart, incoming.RouteEnd = []byte("m"), []byte("z") + incoming.RouteID, incoming.ParentRouteID, incoming.DescriptorHash = 12, 10, bytes.Repeat([]byte{3}, 32) + require.ErrorContains(t, st.ClearMigrationStateWithOwnership(t.Context(), incoming, 11), "WAL disabled") + after, err = st.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, before, after, "failed job cleanup does not retire or publish frontier/receipt") + afterAdmission, err = st.MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Equal(t, beforeAdmission, afterAdmission) + retired, err := st.MigrationRetired(context.Background(), 8) + require.NoError(t, err) + require.False(t, retired) +} + +func TestOwnershipPartitionCorruptionAndNamespaceProtection(t *testing.T) { + dir := t.TempDir() + st, err := NewPebbleStore(dir) + require.NoError(t, err) + _, tx := partitionFixture(t, st) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + _, err = st.ImportVersions(t.Context(), ImportVersionsOptions{JobID: 9, BracketID: 1, BatchSeq: 1, Versions: []MVCCVersion{{ + Key: MigrationOwnershipPartitionMetadataKey(), Value: []byte("forged"), CommitTS: 200, + }}}) + require.ErrorIs(t, err, ErrInvalidImportVersion) + require.True(t, isPebbleMetaKey(MigrationOwnershipPartitionMetadataKey())) + ps, ok := st.(*pebbleStore) + require.True(t, ok) + require.NoError(t, ps.db.Set(migrationOwnershipPartitionMetaKey, []byte("corrupt"), pebble.Sync)) + _, err = ps.MigrationServingState(t.Context()) + require.ErrorContains(t, err, "invalid ownership partition encoding") + require.NoError(t, st.Close()) + _, err = NewPebbleStore(dir) + require.ErrorContains(t, err, "invalid ownership partition encoding") +} + +func TestOwnershipPartitionRefusesRawEntryScopeWithoutReenteringOrEffects(t *testing.T) { + st, err := NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + old, tx := partitionFixture(t, st) + ps, ok := st.(*pebbleStore) + require.True(t, ok) + before, err := ps.MigrationServingState(t.Context()) + require.NoError(t, err) + require.ErrorIs(t, ps.ApplyRaftEntry(t.Context(), 10, func(ctx context.Context) error { + view, readErr := ps.MigrationServingState(ctx) + require.NoError(t, readErr) + require.Equal(t, before, view) + require.ErrorIs(t, ps.ApplyMigrationOwnershipPartition(ctx, tx, 10), ErrMigrationOwnership) + require.ErrorIs(t, ps.AcquireMigrationOwnership(ctx, old, 10), ErrMigrationOwnership) + require.ErrorIs(t, ps.ClearMigrationStateWithOwnership(ctx, old, 10), ErrMigrationOwnership) + require.ErrorIs(t, ps.RefreshMigrationOwnership(ctx, old, 10), ErrMigrationOwnership) + return ErrMigrationOwnership + }), ErrMigrationOwnership) + after, err := ps.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, before, after) + index, found, err := ps.LastAppliedIndex() + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint64(9), index) +} diff --git a/store/migration_ownership_partition_store.go b/store/migration_ownership_partition_store.go new file mode 100644 index 000000000..db53cf2f9 --- /dev/null +++ b/store/migration_ownership_partition_store.go @@ -0,0 +1,189 @@ +package store + +import ( + "context" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +func (s *pebbleStore) readMigrationOwnershipPartitions() (MigrationOwnershipPartitionState, error) { + data, closer, err := s.db.Get(migrationOwnershipPartitionMetaKey) + if errors.Is(err, pebble.ErrNotFound) { + return MigrationOwnershipPartitionState{}, nil + } + if err != nil { + return MigrationOwnershipPartitionState{}, errors.WithStack(err) + } + defer func() { _ = closer.Close() }() + return decodeMigrationOwnershipPartitions(data) +} + +func (s *mvccStore) MigrationOwnershipPartitions(ctx context.Context) (MigrationOwnershipPartitionState, error) { + view, err := s.MigrationServingState(ctx) + return view.Partitions, err +} + +func (s *pebbleStore) MigrationOwnershipPartitions(ctx context.Context) (MigrationOwnershipPartitionState, error) { + view, err := s.MigrationServingState(ctx) + return view.Partitions, err +} + +func partitionConfiguration(value []byte, err error) (uint64, error) { + if errors.Is(err, ErrKeyNotFound) { + return 0, nil + } + if err != nil { + return 0, err + } + index, err := DecodeMigrationConfigurationIndex(value) + if err != nil { + return 0, errors.WithStack(err) + } + return index, nil +} + +func (s *mvccStore) partitionConfigurationLocked() (uint64, error) { + v, ok := s.tree.Get(MigrationConfigurationIndexKey()) + if !ok { + return 0, nil + } + versions, _ := v.([]VersionedValue) + latest, ok := latestVisible(versions, ^uint64(0)) + if !ok || latest.Tombstone { + return 0, nil + } + return partitionConfiguration(latest.Value, nil) +} + +func (s *pebbleStore) partitionConfigurationLocked(ctx context.Context) (uint64, error) { + return partitionConfiguration(s.getAt(ctx, MigrationConfigurationIndexKey(), ^uint64(0))) +} + +func (s *mvccStore) syncPartitionJobLocked(updated []MigrationOwnershipState, requested MigrationOwnershipState, operation MigrationOwnershipOperation, index uint64) (MigrationOwnershipPartitionState, error) { + var cfg uint64 + if partitionHasEvidence(s.migrationPartitions) { + var err error + cfg, err = s.partitionConfigurationLocked() + if err != nil { + return s.migrationPartitions, err + } + } + grant := partitionJobOwnerGrant(requested, operation, s.migrationReadinessCache, s.migrationPromotions[requested.JobID].Done) + return syncPartitionJob(s.migrationPartitions, s.migrationOwnership, updated, requested, operation, index, cfg, grant) +} + +func (s *pebbleStore) syncPartitionJobLocked(ctx context.Context, partitions MigrationOwnershipPartitionState, old, updated []MigrationOwnershipState, requested MigrationOwnershipState, operation MigrationOwnershipOperation, index uint64) (MigrationOwnershipPartitionState, error) { + var cfg uint64 + if partitionHasEvidence(partitions) { + var err error + cfg, err = s.partitionConfigurationLocked(ctx) + if err != nil { + return partitions, err + } + } + s.mtx.RLock() + guards := cloneReadinessStates(s.migrationReadinessCache) + s.mtx.RUnlock() + promotions, err := s.readPebblePromotionStates() + if err != nil { + return partitions, err + } + grant := partitionJobOwnerGrant(requested, operation, guards, promotions[requested.JobID].Done) + return syncPartitionJob(partitions, old, updated, requested, operation, index, cfg, grant) +} + +func partitionJobOwnerGrant(requested MigrationOwnershipState, operation MigrationOwnershipOperation, guards []TargetStagedReadinessState, promoted bool) bool { + if operation == MigrationOwnershipAcquire { + return true + } // acquisition guard was already validated + if operation != MigrationOwnershipClear { + return false + } + for _, guard := range guards { + if ownershipGuardMatchesRange(requested, guard) && ownershipClearMatches(requested, guard, promoted) { + return true + } + } + return false +} + +func (s *mvccStore) ApplyMigrationOwnershipPartition(ctx context.Context, t MigrationOwnershipPartitionTransition, index uint64) error { + s.mtx.Lock() + defer s.mtx.Unlock() + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + cfg, err := s.partitionConfigurationLocked() + if err != nil { + return err + } + updated, err := applyPartitionTransition(s.migrationPartitions, s.migrationOwnership, t, index, cfg) + if err != nil { + return err + } + s.migrationPartitions = updated + s.migrationAdmission = buildMigrationServingAdmission(s.migrationOwnership, updated) + return nil +} + +func (s *pebbleStore) ApplyMigrationOwnershipPartition(ctx context.Context, t MigrationOwnershipPartitionTransition, index uint64) error { + // Typed migration controls are separate Raft entries. A raw entry already + // owns applyMu; neither reenter it nor commit metadata outside its batch. + if s.raftEntryScopeFrom(ctx) != nil { + return errors.WithStack(ErrMigrationOwnership) + } + s.applyMu.Lock() + defer s.applyMu.Unlock() + s.dbMu.RLock() + defer s.dbMu.RUnlock() + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + state, err := s.readMigrationOwnershipPartitions() + if err != nil { + return err + } + legacy, err := s.readMigrationOwnership() + if err != nil { + return err + } + cfg, err := s.partitionConfigurationLocked(ctx) + if err != nil { + return err + } + updated, err := applyPartitionTransition(state, legacy, t, index, cfg) + if err != nil { + return err + } + batch := s.db.NewBatch() + defer batch.Close() + if err := stageSnapshotPartitions(batch, updated); err != nil { + return err + } + if err := s.stagePartitionAppliedIndex(batch, index); err != nil { + return err + } + if err := batch.Commit(s.cleanupWriteOpts(index)); err != nil { + return errors.WithStack(err) + } + s.mtx.Lock() + s.migrationAdmission = buildMigrationServingAdmission(legacy, updated) + s.mtx.Unlock() + return nil +} + +// Late receipt replay must never rewind the durable Raft watermark. +func (s *pebbleStore) stagePartitionAppliedIndex(batch *pebble.Batch, index uint64) error { + persisted, err := readPebbleUint64(s.db, metaAppliedIndexBytes) + if err != nil { + return err + } + if index < persisted { + index = persisted + } + return setPebbleUint64InBatch(batch, metaAppliedIndexBytes, index) +} + +var _ MigrationOwnershipPartitionStore = (*mvccStore)(nil) +var _ MigrationOwnershipPartitionStore = (*pebbleStore)(nil) diff --git a/store/migration_ownership_partition_test.go b/store/migration_ownership_partition_test.go new file mode 100644 index 000000000..a0bc4994f --- /dev/null +++ b/store/migration_ownership_partition_test.go @@ -0,0 +1,189 @@ +package store + +import ( + "bytes" + "encoding/binary" + "testing" + + "github.com/stretchr/testify/require" +) + +func partitionFixture(t *testing.T, st MVCCStore) (MigrationOwnershipState, MigrationOwnershipPartitionTransition) { + t.Helper() + old := ownershipFixture(t, st) + old.Generation = "0123456789abcdef0123456789abcdef" + old.RouteID = 10 + old.DescriptorHash = bytes.Repeat([]byte{1}, 32) + old.WriteFloorKnown = true + require.NoError(t, ownershipStore(t, st).ClearMigrationStateWithOwnership(t.Context(), old, 9)) + require.NoError(t, st.PutAt(t.Context(), MigrationConfigurationIndexKey(), binary.BigEndian.AppendUint64(nil, 3), 1, 0)) + view, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + old = view.Ownership[0] + ref, err := MigrationOwnershipLegacyReferenceFor(old) + require.NoError(t, err) + left, right := partitionFromLegacy(old), partitionFromLegacy(old) + left.RouteID, left.ParentRouteID, left.RouteEnd, left.DescriptorHash = 11, 10, []byte("m"), bytes.Repeat([]byte{2}, 32) + right.RouteID, right.ParentRouteID, right.RouteStart, right.DescriptorHash = 12, 10, []byte("m"), bytes.Repeat([]byte{3}, 32) + tx := MigrationOwnershipPartitionTransition{OperationID: "root/split/10", Operation: MigrationOwnershipPartitionSplit, + RootManifestHash: bytes.Repeat([]byte{4}, 32), Generation: old.Generation, LocalGroupID: 1, CatalogVersion: 12, + ConfigurationIndex: 3, Before: []MigrationOwnershipPartitionReference{ref}, After: []MigrationOwnershipPartition{left, right}} + finalizePartitionTransition(t, &tx) + return old, tx +} + +func finalizePartitionTransition(t *testing.T, tx *MigrationOwnershipPartitionTransition) { + t.Helper() + for i := range tx.After { + tx.After[i].CatalogVersion, tx.After[i].ConfigurationIndex = tx.CatalogVersion, tx.ConfigurationIndex + id, err := MigrationOwnershipPartitionIDFor(*tx, i) + require.NoError(t, err) + tx.After[i].PartitionID = id + tx.After[i].ReceiptID, tx.After[i].AppliedIndex = nil, 0 + } +} + +func partitionStore(t *testing.T, st MVCCStore) MigrationOwnershipPartitionStore { + t.Helper() + p, ok := st.(MigrationOwnershipPartitionStore) + require.True(t, ok) + return p +} + +func TestOwnershipPartitionAtomicSplitRefreshAndLateReplay(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + old, tx := partitionFixture(t, st) + proofs := partitionStore(t, st) + require.NoError(t, proofs.ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + view, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, []MigrationOwnershipState{old}, view.Ownership, "job ledger is not rewritten as children") + require.Len(t, view.Partitions.Partitions, 2) + require.Len(t, view.Partitions.Receipts, 1) + for _, p := range view.Partitions.Partitions { + require.True(t, MigrationOwnershipPartitionEvidenceValid(view.Partitions, p)) + require.Equal(t, MigrationNonOwner, p.Role) + require.Equal(t, old.MinWriteTSExclusive, p.MinWriteTSExclusive) + require.Equal(t, partitionFromLegacy(old).Origins, p.Origins) + } + ref, err := MigrationOwnershipPartitionReferenceFor(view.Partitions.Partitions[0]) + require.NoError(t, err) + refresh := tx + refresh.OperationID, refresh.Operation, refresh.CatalogVersion = "root/refresh/13", MigrationOwnershipPartitionRefresh, 13 + refresh.Before = []MigrationOwnershipPartitionReference{ref} + refresh.After = []MigrationOwnershipPartition{view.Partitions.Partitions[0]} + finalizePartitionTransition(t, &refresh) + require.NoError(t, proofs.ApplyMigrationOwnershipPartition(t.Context(), refresh, 11)) + newer, err := proofs.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + require.False(t, MigrationOwnershipPartitionEvidenceValid(newer, view.Partitions.Partitions[0]), "immutable history does not make a replaced attestation active") + require.NoError(t, proofs.ApplyMigrationOwnershipPartition(t.Context(), tx, 12), "immutable receipt permits late duplicate without rollback") + after, err := proofs.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + require.Equal(t, newer, after) + conflict := canonicalPartitionTransition(tx) + conflict.RootManifestHash[0]++ + require.ErrorIs(t, proofs.ApplyMigrationOwnershipPartition(t.Context(), conflict, 13), ErrMigrationOwnership) + if ps, ok := st.(*pebbleStore); ok { + index, found, readErr := ps.LastAppliedIndex() + require.NoError(t, readErr) + require.True(t, found) + require.Equal(t, uint64(12), index) + require.NoError(t, ps.SetDurableAppliedIndex(99)) + require.NoError(t, proofs.ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + index, found, readErr = ps.LastAppliedIndex() + require.NoError(t, readErr) + require.True(t, found) + require.Equal(t, uint64(99), index, "late duplicate cannot rewind an unrelated later Raft watermark") + } + view.Partitions.Partitions[0].Origins[0].DescriptorHash[0]++ + view.Partitions.Receipts[0].Transition.After[0].DescriptorHash[0]++ + unchanged, err := proofs.MigrationOwnershipPartitions(t.Context()) + require.NoError(t, err) + require.Equal(t, newer, unchanged, "readers own all returned slices") + }) +} + +func TestOwnershipPartitionRejectsBeforeEffects(t *testing.T) { + cases := map[string]func(*MigrationOwnershipPartitionTransition){ + "owner grant": func(tx *MigrationOwnershipPartitionTransition) { + tx.After[0].Role, tx.After[0].OwnerGroupID = MigrationOwner, 1 + }, + "floor decrease": func(tx *MigrationOwnershipPartitionTransition) { tx.After[0].MinWriteTSExclusive-- }, + "floor unknown": func(tx *MigrationOwnershipPartitionTransition) { tx.After[0].WriteFloorKnown = false }, + "floor increase": func(tx *MigrationOwnershipPartitionTransition) { tx.After[0].MinWriteTSExclusive++ }, + "hole": func(tx *MigrationOwnershipPartitionTransition) { tx.After[1].RouteStart = []byte("n") }, + "overlap": func(tx *MigrationOwnershipPartitionTransition) { tx.After[1].RouteStart = []byte("l") }, + "enlargement": func(tx *MigrationOwnershipPartitionTransition) { tx.After[1].RouteEnd = nil }, + "missing before": func(tx *MigrationOwnershipPartitionTransition) { tx.Before = nil }, + "duplicate before": func(tx *MigrationOwnershipPartitionTransition) { tx.Before = append(tx.Before, tx.Before[0]) }, + "forged digest": func(tx *MigrationOwnershipPartitionTransition) { tx.Before[0].AttestationHash[0]++ }, + "forged origin": func(tx *MigrationOwnershipPartitionTransition) { tx.After[0].Origins[0].JobID++ }, + "lost lineage": func(tx *MigrationOwnershipPartitionTransition) { tx.After[0].MigrationJobID++ }, + "generation": func(tx *MigrationOwnershipPartitionTransition) { tx.Generation = "different-generation" }, + "configuration": func(tx *MigrationOwnershipPartitionTransition) { tx.ConfigurationIndex++ }, + "unknown epoch": func(tx *MigrationOwnershipPartitionTransition) { tx.ConfigurationIndex = 0 }, + "manifest": func(tx *MigrationOwnershipPartitionTransition) { tx.RootManifestHash = nil }, + "external job op": func(tx *MigrationOwnershipPartitionTransition) { + tx.Operation = MigrationOwnershipPartitionJobOperation + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + _, tx := partitionFixture(t, st) + before, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + mutate(&tx) + finalizePartitionTransition(t, &tx) + require.ErrorIs(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 10), ErrMigrationOwnership) + after, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, before, after) + }) + }) + } +} + +func TestOwnershipPartitionRequiresAllLegacyOverlays(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + old, tx := partitionFixture(t, st) + newer := old + newer.JobID, newer.CatalogVersion, newer.OwnerGroupID, newer.Role = 8, 11, 1, MigrationOwner + newer.RouteStart, newer.RouteEnd = []byte("m"), []byte("z") + newer.DescriptorHash = bytes.Repeat([]byte{5}, 32) + require.NoError(t, ownershipStore(t, st).ClearMigrationStateWithOwnership(t.Context(), newer, 10)) + require.ErrorIs(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 11), ErrMigrationOwnership, + "omitted newer positive/negative overlays may not disappear") + view, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + ref, err := MigrationOwnershipLegacyReferenceFor(view.Ownership[1]) + require.NoError(t, err) + tx.Before = append(tx.Before, ref) + tx.After[1] = partitionFromLegacy(view.Ownership[1]) + finalizePartitionTransition(t, &tx) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 11)) + }) +} + +func TestOwnershipPartitionUnboundedUnionAndUnknownLegacy(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + old, tx := partitionFixture(t, st) + old.RouteStart, old.RouteEnd, old.JobID, old.CatalogVersion = nil, nil, 8, 11 + require.NoError(t, ownershipStore(t, st).ClearMigrationStateWithOwnership(t.Context(), old, 10)) + view, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + ref, err := MigrationOwnershipLegacyReferenceFor(view.Ownership[1]) + require.NoError(t, err) + tx.Before = append(tx.Before, ref) + tx.After = []MigrationOwnershipPartition{partitionFromLegacy(view.Ownership[1])} + finalizePartitionTransition(t, &tx) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 11)) + }) + st := NewMVCCStore() + _, tx := partitionFixture(t, st) + ms, ok := st.(*mvccStore) + require.True(t, ok) + ms.migrationOwnership[0].Receipts = nil + require.ErrorIs(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 10), ErrMigrationOwnership) +} diff --git a/store/migration_ownership_partition_types.go b/store/migration_ownership_partition_types.go new file mode 100644 index 000000000..facb8ce7e --- /dev/null +++ b/store/migration_ownership_partition_types.go @@ -0,0 +1,104 @@ +package store + +import "context" + +// MigrationOwnershipOperationReference binds a partition to an immutable job +// operation. Partitioning never rewrites the referenced receipt's descriptor. +type MigrationOwnershipOperationReference struct { + JobID uint64 + Operation MigrationOwnershipOperation + CatalogVersion uint64 + ConfigurationIndex uint64 + AppliedIndex uint64 + DescriptorHash []byte + Staged bool +} + +// MigrationOwnershipPartition is active serving evidence, not a job record. +// PartitionID remains stable across unchanged-descriptor attestations; several +// child partitions may share a job receipt or staged-data lineage. +type MigrationOwnershipPartition struct { + PartitionID []byte + ReceiptID []byte + Generation string + LocalGroupID uint64 + OwnerGroupID uint64 + CatalogVersion uint64 + ConfigurationIndex uint64 + AppliedIndex uint64 + RouteID uint64 + ParentRouteID uint64 + RouteStart []byte + RouteEnd []byte + DescriptorHash []byte + Role MigrationOwnershipRole + MinWriteTSExclusive uint64 + WriteFloorKnown bool + Staged bool + MigrationJobID uint64 + SourceTrackerVersion uint64 + SourceAppliedIndex uint64 + Origins []MigrationOwnershipOperationReference +} + +// MigrationOwnershipPartitionReference is an exact prior-attestation CAS. +// Legacy references set LegacyJobID and leave PartitionID empty; they bind the +// complete old job record and do not turn JobID into a child partition key. +type MigrationOwnershipPartitionReference struct { + PartitionID []byte + AttestationHash []byte + LegacyJobID uint64 +} + +type MigrationOwnershipPartitionOperation uint8 + +const ( + MigrationOwnershipPartitionSplit MigrationOwnershipPartitionOperation = iota + 1 + MigrationOwnershipPartitionRefresh + // MigrationOwnershipPartitionJobOperation is an internal receipt kind for + // validated Acquire/Clear operations. External partition controls reject it. + MigrationOwnershipPartitionJobOperation +) + +// MigrationOwnershipPartitionTransition replaces exactly the complete Before +// union. Split preserves ownership and lineage; Refresh preserves geometry too. +// RootManifestHash identifies committed root evidence, not a proposed version. +type MigrationOwnershipPartitionTransition struct { + OperationID string + Operation MigrationOwnershipPartitionOperation + RootManifestHash []byte + Generation string + LocalGroupID uint64 + CatalogVersion uint64 + ConfigurationIndex uint64 + Before []MigrationOwnershipPartitionReference + After []MigrationOwnershipPartition +} + +// MigrationOwnershipPartitionReceipt is immutable operation history. It remains +// after its child frontier is replaced and after split-job history is collected. +type MigrationOwnershipPartitionReceipt struct { + ReceiptID []byte + Transition MigrationOwnershipPartitionTransition + AppliedIndex uint64 +} + +// MigrationOwnershipPartitionState is the independently persisted frontier and +// receipt container. It never replaces the original per-job ownership ledger. +type MigrationOwnershipPartitionState struct { + Generation string + Partitions []MigrationOwnershipPartition + Receipts []MigrationOwnershipPartitionReceipt + + // Only the durable serving-cache builder seals evidence. Shared by clones, + // never serialized, and never derived from caller-owned public fields. + servingEvidence *partitionEvidenceIndex +} + +// MigrationOwnershipPartitionStore atomically replaces serving evidence, +// records its immutable receipt, and advances the supplied Raft applied index. +// Semantic, generation and configuration mismatches refuse before side effects. +type MigrationOwnershipPartitionStore interface { + MigrationOwnershipPartitions(context.Context) (MigrationOwnershipPartitionState, error) + ApplyMigrationOwnershipPartition(context.Context, MigrationOwnershipPartitionTransition, uint64) error +} diff --git a/store/migration_ownership_refresh_test.go b/store/migration_ownership_refresh_test.go new file mode 100644 index 000000000..ed9deb1c3 --- /dev/null +++ b/store/migration_ownership_refresh_test.go @@ -0,0 +1,90 @@ +package store + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMigrationOwnershipRefreshPreservesImmutableReceipt(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + state := ownershipFixture(t, st) + state.Generation, state.RouteID, state.ParentRouteID = "0123456789abcdef0123456789abcdef", 3, 1 + state.DescriptorHash = bytes.Repeat([]byte{1}, 32) + writer := ownershipStore(t, st) + require.NoError(t, writer.ClearMigrationStateWithOwnership(t.Context(), state, 9)) + refresher, ok := st.(MigrationOwnershipRefresher) + require.True(t, ok) + refreshed := state + refreshed.CatalogVersion, refreshed.ConfigurationIndex = 15, 4 + require.NoError(t, refresher.RefreshMigrationOwnership(t.Context(), refreshed, 12)) + view, err := writer.MigrationServingState(t.Context()) + require.NoError(t, err) + proof := view.Ownership[0] + require.Equal(t, uint64(15), proof.CatalogVersion) + require.Equal(t, uint64(4), proof.ConfigurationIndex) + require.Len(t, proof.Receipts, 1) + require.Equal(t, uint64(10), proof.Receipts[0].CatalogVersion) + require.Equal(t, uint64(3), proof.Receipts[0].ConfigurationIndex) + require.Equal(t, uint64(9), proof.Receipts[0].AppliedIndex) + olderEpoch := refreshed + olderEpoch.ConfigurationIndex = 3 + require.ErrorIs(t, refresher.RefreshMigrationOwnership(t.Context(), olderEpoch, 13), ErrMigrationOwnership) + require.ErrorIs(t, refresher.RefreshMigrationOwnership(t.Context(), refreshed, 11), ErrMigrationOwnership) + require.NoError(t, writer.ClearMigrationStateWithOwnership(t.Context(), state, 13), "operation replay must not regress its newer attestation") + for _, mutate := range []func(*MigrationOwnershipState){ + func(s *MigrationOwnershipState) { s.Generation = "different-native-generation" }, + func(s *MigrationOwnershipState) { s.Role, s.OwnerGroupID = MigrationOwner, 1 }, + func(s *MigrationOwnershipState) { s.MinWriteTSExclusive++ }, + func(s *MigrationOwnershipState) { s.RouteID++ }, + func(s *MigrationOwnershipState) { s.DescriptorHash = bytes.Repeat([]byte{2}, 32) }, + } { + wrong := refreshed + mutate(&wrong) + require.ErrorIs(t, refresher.RefreshMigrationOwnership(t.Context(), wrong, 14), ErrMigrationOwnership) + } + snapshot, err := st.Snapshot() + require.NoError(t, err) + raw := snapshotBytes(t, snapshot) + require.NoError(t, snapshot.Close()) + target, err := NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, target.Close()) }) + require.NoError(t, target.Restore(bytes.NewReader(raw))) + restored, err := ownershipStore(t, target).MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, proof, restored.Ownership[0]) + }) +} + +func TestMigrationOwnershipPromotedClearKeepsAcquireReceipt(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + writer, ok := st.(MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), TargetStagedReadinessState{JobID: 8, MigrationJobID: 8, + Armed: true, ExpectedCutoverVersion: 12, MinWriteTSExclusive: 200, RouteStart: []byte("m"), RouteEnd: []byte("z")}, 8)) + state := MigrationOwnershipState{Generation: "0123456789abcdef0123456789abcdef", JobID: 8, + LocalGroupID: 1, OwnerGroupID: 1, Role: MigrationOwner, Staged: true, CatalogVersion: 12, + ConfigurationIndex: 3, RouteID: 3, ParentRouteID: 1, DescriptorHash: bytes.Repeat([]byte{1}, 32), + RouteStart: []byte("m"), RouteEnd: []byte("z"), MinWriteTSExclusive: 200} + proofs := ownershipStore(t, st) + require.NoError(t, proofs.AcquireMigrationOwnership(t.Context(), state, 9)) + cleared := state + cleared.Staged, cleared.CatalogVersion, cleared.DescriptorHash = false, 13, bytes.Repeat([]byte{2}, 32) + require.ErrorIs(t, proofs.ClearMigrationStateWithOwnership(t.Context(), cleared, 10), ErrMigrationOwnership) + promoter, ok := st.(MigrationPromoter) + require.True(t, ok) + _, err := promoter.PromoteVersions(t.Context(), PromoteVersionsOptions{JobID: 8, AppliedIndex: 10, + StartKey: []byte("staged/"), EndKey: []byte("staged0"), MaxVersions: 10, MaxBytes: 1024, + TargetKey: func(key []byte) ([]byte, bool) { return key, true }}) + require.NoError(t, err) + require.NoError(t, proofs.ClearMigrationStateWithOwnership(t.Context(), cleared, 11)) + view, err := proofs.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Empty(t, view.Readiness) + require.Len(t, view.Ownership[0].Receipts, 2) + require.Equal(t, uint64(12), view.Ownership[0].Receipts[0].CatalogVersion) + require.Equal(t, uint64(13), view.Ownership[0].Receipts[1].CatalogVersion) + }) +} diff --git a/store/migration_ownership_test.go b/store/migration_ownership_test.go new file mode 100644 index 000000000..f9e08bafa --- /dev/null +++ b/store/migration_ownership_test.go @@ -0,0 +1,186 @@ +package store + +import ( + "bytes" + "encoding/binary" + "log/slog" + "testing" + + "github.com/cockroachdb/pebble/v2" + "github.com/stretchr/testify/require" +) + +func ownershipFixture(t *testing.T, st MVCCStore) MigrationOwnershipState { + t.Helper() + writer, ok := st.(MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), TargetStagedReadinessState{ + JobID: 7, MigrationJobID: 7, Armed: true, TrackWrites: true, + SourceWriteFence: true, SourceReadFence: true, RouteStart: []byte("a"), RouteEnd: []byte("z"), + ExpectedCutoverVersion: 10, MinWriteTSExclusive: 100, RetentionPinTS: 1, + }, 8)) + return MigrationOwnershipState{JobID: 7, LocalGroupID: 1, OwnerGroupID: 2, CatalogVersion: 10, + ConfigurationIndex: 3, RouteStart: []byte("a"), RouteEnd: []byte("z"), + MinWriteTSExclusive: 100, Role: MigrationNonOwner} +} + +func ownershipStore(t *testing.T, st MVCCStore) MigrationOwnershipStore { + t.Helper() + proofs, ok := st.(MigrationOwnershipStore) + require.True(t, ok) + return proofs +} + +func assertOwnershipCleared(t *testing.T, st MVCCStore) { + t.Helper() + state, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + require.Empty(t, state.Readiness) + require.Len(t, state.Ownership, 1) + require.Equal(t, MigrationNonOwner, state.Ownership[0].Role) + require.Equal(t, uint64(10), state.Ownership[0].CatalogVersion) + require.Equal(t, uint64(9), state.Ownership[0].AppliedIndex) + retirement, ok := st.(MigrationRetirementReader) + require.True(t, ok) + retired, err := retirement.MigrationRetired(t.Context(), 7) + require.NoError(t, err) + require.True(t, retired) +} + +func TestMigrationOwnershipAtomicClearAndImmutableRetry(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + state := ownershipFixture(t, st) + proofs := ownershipStore(t, st) + wrong := state + wrong.RouteEnd = []byte("y") + require.ErrorIs(t, proofs.ClearMigrationStateWithOwnership(t.Context(), wrong, 9), ErrMigrationOwnership) + view, err := proofs.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Len(t, view.Readiness, 1) + require.Empty(t, view.Ownership) + require.NoError(t, proofs.ClearMigrationStateWithOwnership(t.Context(), state, 9)) + assertOwnershipCleared(t, st) + require.NoError(t, proofs.ClearMigrationStateWithOwnership(t.Context(), state, 10)) + state.CatalogVersion++ + require.ErrorIs(t, proofs.ClearMigrationStateWithOwnership(t.Context(), state, 11), ErrMigrationOwnership, + "cache advancement or a retry cannot revise the old ownership certificate") + require.NoError(t, st.Compact(t.Context(), 100)) + assertOwnershipCleared(t, st) + _, err = st.ImportVersions(t.Context(), ImportVersionsOptions{JobID: 8, BracketID: 1, BatchSeq: 1, Versions: []MVCCVersion{{ + Key: migrationOwnershipMetaKeyBytes, Value: []byte("forged"), CommitTS: 200, + }}}) + require.ErrorIs(t, err, ErrInvalidImportVersion) + }) +} + +func TestMigrationOwnershipSnapshotRestoreAndReopen(t *testing.T) { + source := NewMVCCStore() + state := ownershipFixture(t, source) + require.NoError(t, ownershipStore(t, source).ClearMigrationStateWithOwnership(t.Context(), state, 9)) + snapshot, err := source.Snapshot() + require.NoError(t, err) + raw := snapshotBytes(t, snapshot) + require.NoError(t, snapshot.Close()) + require.Equal(t, mvccSnapshotVersionV6, binary.LittleEndian.Uint32(raw[len(mvccSnapshotMagic):])) + runMigrationStoreSuite(t, func(t *testing.T, destination MVCCStore) { + require.NoError(t, destination.Restore(bytes.NewReader(raw))) + assertOwnershipCleared(t, destination) + }) + dir := t.TempDir() + destination, err := NewPebbleStore(dir) + require.NoError(t, err) + require.NoError(t, destination.Restore(bytes.NewReader(raw))) + require.NoError(t, destination.Close()) + destination, err = NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, destination.Close()) }) + assertOwnershipCleared(t, destination) +} + +func TestMigrationOwnershipNativeAndSSTSnapshots(t *testing.T) { + for _, sst := range []bool{false, true} { + t.Run(map[bool]string{false: "native", true: "sst"}[sst], func(t *testing.T) { + source, err := NewPebbleStore(t.TempDir(), WithSSTIngestSnapshots(sst)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, source.Close()) }) + state := ownershipFixture(t, source) + require.NoError(t, ownershipStore(t, source).ClearMigrationStateWithOwnership(t.Context(), state, 9)) + snapshot, err := source.Snapshot() + require.NoError(t, err) + raw := snapshotBytes(t, snapshot) + require.NoError(t, snapshot.Close()) + target, err := NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, target.Close()) }) + require.NoError(t, target.Restore(bytes.NewReader(raw))) + assertOwnershipCleared(t, target) + }) + } +} + +func TestMigrationOwnershipFailedBatchKeepsGuardAndAppliedIndex(t *testing.T) { + db, err := pebble.Open(t.TempDir(), &pebble.Options{DisableWAL: true}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + st := &pebbleStore{db: db, log: slog.Default(), fsmApplyWriteOpts: pebble.NoSync} + state := ownershipFixture(t, st) + st.fsmApplyWriteOpts = pebble.Sync + require.ErrorContains(t, st.ClearMigrationStateWithOwnership(t.Context(), state, 9), "WAL disabled") + view, err := st.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Len(t, view.Readiness, 1) + require.Empty(t, view.Ownership) + retired, err := st.MigrationRetired(t.Context(), 7) + require.NoError(t, err) + require.False(t, retired) + index, found, err := st.LastAppliedIndex() + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint64(8), index) +} + +func TestMigrationOwnershipCorruptionRejectsReopen(t *testing.T) { + dir := t.TempDir() + st, err := NewPebbleStore(dir) + require.NoError(t, err) + ps, ok := st.(*pebbleStore) + require.True(t, ok) + require.NoError(t, ps.db.Set(migrationOwnershipMetaKeyBytes, []byte("corrupt"), pebble.Sync)) + _, err = ps.MigrationServingState(t.Context()) + require.ErrorContains(t, err, "invalid migration ownership encoding") + require.NoError(t, st.Close()) + _, err = NewPebbleStore(dir) + require.ErrorContains(t, err, "invalid migration ownership encoding") +} + +func TestMigrationOwnershipMoveBackRequiresNewerGuardBoundAcquisition(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + state := ownershipFixture(t, st) + proofs := ownershipStore(t, st) + require.NoError(t, proofs.ClearMigrationStateWithOwnership(t.Context(), state, 9)) + writer, ok := st.(MigrationTargetReadinessRaftWriter) + require.True(t, ok) + guard := TargetStagedReadinessState{JobID: 8, MigrationJobID: 8, Armed: true, + RouteStart: []byte("a"), RouteEnd: []byte("z"), ExpectedCutoverVersion: 12, MinWriteTSExclusive: 200} + require.NoError(t, writer.ApplyTargetStagedReadinessAt(t.Context(), guard, 10)) + acquisition := MigrationOwnershipState{JobID: 8, LocalGroupID: 1, OwnerGroupID: 1, CatalogVersion: 12, + ConfigurationIndex: 4, MinWriteTSExclusive: 200, RouteStart: []byte("a"), RouteEnd: []byte("z"), + Role: MigrationOwner, Staged: true} + for _, version := range []uint64{10, 11, 13} { + wrong := acquisition + wrong.CatalogVersion = version + require.ErrorIs(t, proofs.AcquireMigrationOwnership(t.Context(), wrong, 11), ErrMigrationOwnership) + } + wrong := acquisition + wrong.MinWriteTSExclusive-- + require.ErrorIs(t, proofs.AcquireMigrationOwnership(t.Context(), wrong, 11), ErrMigrationOwnership) + view, err := proofs.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Len(t, view.Ownership, 1, "cache/version changes cannot erase NONOWNER evidence") + require.NoError(t, proofs.AcquireMigrationOwnership(t.Context(), acquisition, 11)) + view, err = proofs.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Len(t, view.Ownership, 2, "the newer replicated certificate supersedes, not deletes, the old range proof") + require.NoError(t, proofs.AcquireMigrationOwnership(t.Context(), acquisition, 12), "exact replay is idempotent") + }) +} diff --git a/store/migration_partition_evidence_index.go b/store/migration_partition_evidence_index.go new file mode 100644 index 000000000..58f80c085 --- /dev/null +++ b/store/migration_partition_evidence_index.go @@ -0,0 +1,91 @@ +package store + +import ( + "bytes" + "crypto/sha256" +) + +type partitionEvidenceEntry struct { + digest [sha256.Size]byte + position int +} + +// This index attests immutable durable support, not mutable caller receipt +// copies. Both the supplied proof and its current active member must still +// match the sealed canonical attestation; knowing an ID cannot grant authority. +type partitionEvidenceIndex struct { + generation string + active map[string]partitionEvidenceEntry +} + +type partitionEvidenceKey struct { + receipt, partition string +} + +func partitionEvidenceDigest(p MigrationOwnershipPartition) [sha256.Size]byte { + // MigrationOwnershipPartition has no custom marshalers or unsupported fields. + digest, _ := partitionDigest("elastickv/ownership-partition/attestation/v1", clonePartition(p)) + return [sha256.Size]byte(digest) +} + +func partitionEvidenceSupport(receipts []MigrationOwnershipPartitionReceipt) map[partitionEvidenceKey][sha256.Size]byte { + support := make(map[partitionEvidenceKey][sha256.Size]byte) + for _, receipt := range receipts { + if !validPartitionReceipt(receipt) { + continue + } + for _, p := range receipt.Transition.After { + key := partitionEvidenceKey{string(receipt.ReceiptID), string(p.PartitionID)} + support[key] = partitionEvidenceDigest(p) + } + } + return support +} + +func buildPartitionEvidenceIndex(state MigrationOwnershipPartitionState) *partitionEvidenceIndex { + if len(state.Partitions) == 0 { + return nil + } + support := partitionEvidenceSupport(state.Receipts) + index := &partitionEvidenceIndex{generation: state.Generation, active: make(map[string]partitionEvidenceEntry, len(state.Partitions))} + for position, p := range state.Partitions { + if p.Generation != state.Generation || !validPartition(p) { + continue + } + digest := partitionEvidenceDigest(p) + if supported, ok := support[partitionEvidenceKey{string(p.ReceiptID), string(p.PartitionID)}]; !ok || supported != digest { + continue + } + id := string(p.PartitionID) + if _, duplicate := index.active[id]; duplicate { + position = -1 + } + index.active[id] = partitionEvidenceEntry{digest: digest, position: position} + } + return index +} + +func (i *partitionEvidenceIndex) valid(state MigrationOwnershipPartitionState, p MigrationOwnershipPartition) bool { + if state.Generation != i.generation || p.Generation != i.generation { + return false + } + entry, ok := i.active[string(p.PartitionID)] + if !ok || entry.position < 0 || partitionEvidenceDigest(p) != entry.digest { + return false + } + return partitionEvidenceMemberValid(state.Partitions, p.PartitionID, entry) +} + +func partitionEvidenceMemberValid(partitions []MigrationOwnershipPartition, id []byte, entry partitionEvidenceEntry) bool { + if entry.position < len(partitions) && bytes.Equal(partitions[entry.position].PartitionID, id) { + return partitionEvidenceDigest(partitions[entry.position]) == entry.digest + } + // Caller reordering/truncation cannot introduce new authority. Only this + // exceptional, modified-view path needs to search for an original member. + for _, active := range partitions { + if bytes.Equal(active.PartitionID, id) && partitionEvidenceDigest(active) == entry.digest { + return true + } + } + return false +} diff --git a/store/migration_partition_evidence_index_test.go b/store/migration_partition_evidence_index_test.go new file mode 100644 index 000000000..672ae3108 --- /dev/null +++ b/store/migration_partition_evidence_index_test.go @@ -0,0 +1,97 @@ +package store + +import ( + "bytes" + "encoding/json" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func sealedPartitionEvidenceFixture(t *testing.T, st MVCCStore) MigrationServingState { + t.Helper() + _, tx := partitionFixture(t, st) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + view, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.NotNil(t, view.Partitions.servingEvidence) + return view +} + +func TestPartitionEvidenceIndexRejectsCallerForgedAttestation(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + view := sealedPartitionEvidenceFixture(t, st) + receipt := view.Partitions.Receipts[0] + receipt.Transition.After[0].MinWriteTSExclusive-- + id, err := MigrationOwnershipPartitionReceiptIDFor(receipt.Transition) + require.NoError(t, err) + receipt.ReceiptID = id + for i := range receipt.Transition.After { + receipt.Transition.After[i].ReceiptID = bytes.Clone(id) + } + view.Partitions.Receipts = []MigrationOwnershipPartitionReceipt{receipt} + view.Partitions.Partitions = receipt.Transition.After + forged := view.Partitions.Partitions[0] + require.True(t, migrationPartitionEvidenceUncached(view.Partitions, forged), "the caller recomputed a self-consistent, but never durable, receipt") + require.False(t, MigrationOwnershipPartitionEvidenceValid(view.Partitions, forged), "public receipt copies cannot replace sealed durable authority") + fresh, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.True(t, MigrationOwnershipPartitionEvidenceValid(fresh.Partitions, fresh.Partitions.Partitions[0])) + require.EqualValues(t, 100, fresh.Partitions.Partitions[0].MinWriteTSExclusive) + }) +} + +func TestPartitionEvidenceIndexRequiresCurrentActiveMember(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + view := sealedPartitionEvidenceFixture(t, st) + left, right := view.Partitions.Partitions[0], view.Partitions.Partitions[1] + view.Partitions.Partitions = view.Partitions.Partitions[1:] + require.False(t, MigrationOwnershipPartitionEvidenceValid(view.Partitions, left)) + require.True(t, MigrationOwnershipPartitionEvidenceValid(view.Partitions, right)) + view.Partitions.Partitions[0].Role = MigrationOwner + view.Partitions.Partitions[0].OwnerGroupID = right.LocalGroupID + require.False(t, MigrationOwnershipPartitionEvidenceValid(view.Partitions, right), "an ID match cannot hide a changed active member") + require.False(t, MigrationOwnershipPartitionEvidenceValid(view.Partitions, view.Partitions.Partitions[0])) + view.Partitions.Generation = "different-generation" + require.False(t, MigrationOwnershipPartitionEvidenceValid(view.Partitions, right)) + }) +} + +func TestPartitionEvidenceIndexNeverSerializesTrust(t *testing.T) { + view := sealedPartitionEvidenceFixture(t, NewMVCCStore()) + encoded, err := json.Marshal(view.Partitions) + require.NoError(t, err) + require.NotContains(t, string(encoded), "servingEvidence") + var restored MigrationOwnershipPartitionState + require.NoError(t, json.Unmarshal(encoded, &restored)) + require.Nil(t, restored.servingEvidence) + for _, p := range restored.Partitions { + require.True(t, MigrationOwnershipPartitionEvidenceValid(restored, p), "unsealed state still proves full receipt evidence") + } + restored.Receipts[0].Transition.After[0].Origins[0].JobID++ + index := buildPartitionEvidenceIndex(restored) + restored.servingEvidence = index + for _, p := range restored.Partitions { + require.False(t, MigrationOwnershipPartitionEvidenceValid(restored, p), "cache construction must not seal corrupt support") + } +} + +func TestPartitionEvidenceIndexClonesIsolateConcurrentMutation(t *testing.T) { + st := NewMVCCStore() + mutated := sealedPartitionEvidenceFixture(t, st) + stable, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Same(t, mutated.Partitions.servingEvidence, stable.Partitions.servingEvidence) + var done sync.WaitGroup + done.Go(func() { + for range 100 { + mutated.Partitions.Partitions[0].DescriptorHash[0]++ + mutated.Partitions.Receipts[0].Transition.After[0].Origins[0].DescriptorHash[0]++ + } + }) + for range 100 { + require.True(t, MigrationOwnershipPartitionEvidenceValid(stable.Partitions, stable.Partitions.Partitions[0])) + } + done.Wait() +} diff --git a/store/migration_promote.go b/store/migration_promote.go index b64df958f..6bba5f802 100644 --- a/store/migration_promote.go +++ b/store/migration_promote.go @@ -58,6 +58,11 @@ func promotedVersionsFromStaged(opts PromoteVersionsOptions, versions []MVCCVers } func (s *mvccStore) PromoteVersions(ctx context.Context, opts PromoteVersionsOptions) (PromoteVersionsResult, error) { + s.mtx.Lock() + defer s.mtx.Unlock() + if _, retired := s.migrationRetired[opts.JobID]; retired { + return PromoteVersionsResult{}, errors.WithStack(ErrMigrationRetired) + } if err := validatePromoteVersionsOptions(opts); err != nil { return PromoteVersionsResult{}, err } @@ -65,9 +70,6 @@ func (s *mvccStore) PromoteVersions(ctx context.Context, opts PromoteVersionsOpt return PromoteVersionsResult{Done: true}, nil } - s.mtx.Lock() - defer s.mtx.Unlock() - state, cursor := s.promotionStateAndCursorLocked(opts) if opts.JobID != 0 && state.Done { return PromoteVersionsResult{Done: true, TotalPromotedRows: state.PromotedRows, MaxPromotedTS: state.MaxPromotedTS}, nil @@ -104,7 +106,7 @@ func (s *mvccStore) planMemoryPromotionLocked( if err != nil { return ExportVersionsResult{}, nil, PromoteVersionsResult{}, err } - exported, err := s.exportMemoryVersionsLocked(ctx, exportOpts, pos) + exported, err := s.exportVersionsLocked(ctx, exportOpts, pos) if err != nil { return ExportVersionsResult{}, nil, PromoteVersionsResult{}, err } @@ -142,55 +144,24 @@ func (s *mvccStore) MigrationPromotionState(_ context.Context, jobID uint64) (Pr return clonePromotionState(state), ok, nil } -func (s *mvccStore) exportMemoryVersionsLocked(ctx context.Context, opts ExportVersionsOptions, pos exportCursorPosition) (ExportVersionsResult, error) { - result := newExportVersionsResult(opts.MaxVersions) - it := s.tree.Iterator() - if !s.seekMemoryExportStart(&it, opts.StartKey, pos) { - result.Done = true - return result, nil - } - for ok := true; ok; ok = it.Next() { - key, keyOK := it.Key().([]byte) - if err := checkExportKey(ctx, key, keyOK, opts.EndKey); err != nil { - if errors.Is(err, errExportReachedEnd) { - result.Done = true - result.NextCursor = nil - return result, nil - } - return ExportVersionsResult{}, err - } - if !keyOK { - continue - } - done, err := exportMemoryIteratorKey(ctx, opts, pos, key, it.Value(), &result) - if err != nil || !done { - return result, err - } - } - result.Done = true - result.NextCursor = nil - return result, nil -} - -func (s *mvccStore) removeVersionLocked(key []byte, commitTS uint64) bool { +func (s *mvccStore) removeVersionLocked(key []byte, commitTS uint64) { existing, ok := s.tree.Get(key) if !ok { - return false + return } versions, _ := existing.([]VersionedValue) idx := findVersionIndex(versions, commitTS) if idx < 0 { - return false + return } next := make([]VersionedValue, len(versions)-1) copy(next, versions[:idx]) copy(next[idx:], versions[idx+1:]) if len(next) == 0 { s.tree.Remove(key) - return true + return } s.tree.Put(bytes.Clone(key), next) - return true } func findVersionIndex(versions []VersionedValue, commitTS uint64) int { @@ -203,6 +174,13 @@ func findVersionIndex(versions []VersionedValue, commitTS uint64) int { } func (s *pebbleStore) PromoteVersions(ctx context.Context, opts PromoteVersionsOptions) (PromoteVersionsResult, error) { + s.applyMu.Lock() + defer s.applyMu.Unlock() + s.dbMu.RLock() + defer s.dbMu.RUnlock() + if err := s.checkMigrationNotRetiredLocked(opts.JobID); err != nil { + return PromoteVersionsResult{}, err + } if err := validatePromoteVersionsOptions(opts); err != nil { return PromoteVersionsResult{}, err } @@ -210,12 +188,6 @@ func (s *pebbleStore) PromoteVersions(ctx context.Context, opts PromoteVersionsO return PromoteVersionsResult{Done: true}, nil } - s.applyMu.Lock() - defer s.applyMu.Unlock() - - s.dbMu.RLock() - defer s.dbMu.RUnlock() - state, cursor, err := s.pebblePromotionStateAndCursor(opts) if err != nil { return PromoteVersionsResult{}, err diff --git a/store/migration_readiness.go b/store/migration_readiness.go new file mode 100644 index 000000000..21acf66d5 --- /dev/null +++ b/store/migration_readiness.go @@ -0,0 +1,425 @@ +package store + +import ( + "bytes" + "context" + "encoding/binary" + "sort" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +const ( + targetReadinessCodecVersionV1 byte = 1 + targetReadinessCodecVersion byte = 2 + targetReadinessGuardCodecVersion byte = 3 + targetReadinessArmedFlag byte = 1 + targetReadinessSourceWrite byte = 1 + targetReadinessSourceRead byte = 2 + targetReadinessTrackWrites byte = 4 + targetReadinessTransferSizeGuard byte = 8 +) + +// MigrationTransferSizeGuardVersion is independent of the ownership protocol +// and does not change the admission contract of legacy write trackers. +const MigrationTransferSizeGuardVersion uint32 = 1 + +// invalidReadinessState marks a validation verdict so apply can tell it apart +// from a replica-local persistence failure: the former is a pure function of +// the request and must stay an ordinary rejection, the latter must halt. +func invalidReadinessState(msg string) error { + return errors.WithStack(errors.Mark(errors.New(msg), ErrInvalidReadinessState)) +} + +func validateTargetStagedReadinessState(state TargetStagedReadinessState) error { + switch { + case state.JobID == 0: + return invalidReadinessState("target staged readiness job_id is required") + case state.MigrationJobID == 0: + return invalidReadinessState("target staged readiness migration_job_id is required") + case state.Armed && state.MinWriteTSExclusive == 0: + return invalidReadinessState("target staged readiness min_write_ts_exclusive is required when armed") + case state.RouteEnd != nil && bytes.Compare(state.RouteStart, state.RouteEnd) >= 0: + return invalidReadinessState("target staged readiness route range is invalid") + } + return validateSourceMigrationControlState(state) +} + +func validateSourceMigrationControlState(state TargetStagedReadinessState) error { + if err := validateMigrationTransferSizeGuard(state); err != nil { + return err + } + if state.SourceReadFence && !state.SourceWriteFence { + return invalidReadinessState("source read fence requires source write fence") + } + if (state.SourceWriteFence || state.SourceReadFence) && state.RetentionPinTS == 0 { + return invalidReadinessState("source migration control retention pin is required") + } + if state.TrackWrites && state.RetentionPinTS == 0 { + return invalidReadinessState("migration write tracker retention pin is required") + } + return nil +} + +func validateMigrationTransferSizeGuard(state TargetStagedReadinessState) error { + if state.TransferSizeGuardVersion != 0 && + (state.TransferSizeGuardVersion != MigrationTransferSizeGuardVersion || !state.Armed || !state.TrackWrites) { + return invalidReadinessState("transfer size guard requires supported version and armed source tracker") + } + return nil +} + +func (s *mvccStore) ApplyTargetStagedReadiness(_ context.Context, state TargetStagedReadinessState) error { + if err := validateTargetStagedReadinessState(state); err != nil { + return err + } + s.mtx.Lock() + defer s.mtx.Unlock() + if _, retired := s.migrationRetired[state.JobID]; retired { + return errors.WithStack(ErrMigrationRetired) + } + cloned := cloneTargetStagedReadinessState(state) + s.migrationReadiness[state.JobID] = cloned + s.migrationReadinessCache = upsertTargetStagedReadinessStateCache(s.migrationReadinessCache, cloned) + if state.Armed { + s.lastCommitTS = max(s.lastCommitTS, state.MinWriteTSExclusive) + s.migrationHLCFloors[state.JobID] = max(s.migrationHLCFloors[state.JobID], state.MinWriteTSExclusive) + } + return nil +} + +func (s *mvccStore) ApplyTargetStagedReadinessAt(ctx context.Context, state TargetStagedReadinessState, _ uint64) error { + return s.ApplyTargetStagedReadiness(ctx, state) +} + +func (s *mvccStore) MigrationTargetReadinessStates(_ context.Context) ([]TargetStagedReadinessState, error) { + s.mtx.RLock() + defer s.mtx.RUnlock() + return cloneTargetStagedReadinessStates(s.migrationReadinessCache), nil +} + +func (s *pebbleStore) ApplyTargetStagedReadiness(_ context.Context, state TargetStagedReadinessState) error { + return s.applyTargetStagedReadiness(state, 0, false) +} + +func (s *pebbleStore) ApplyTargetStagedReadinessAt(_ context.Context, state TargetStagedReadinessState, appliedIndex uint64) error { + return s.applyTargetStagedReadiness(state, appliedIndex, true) +} + +func (s *pebbleStore) applyTargetStagedReadiness(state TargetStagedReadinessState, appliedIndex uint64, raftApply bool) error { + if err := validateTargetStagedReadinessState(state); err != nil { + return err + } + s.applyMu.Lock() + defer s.applyMu.Unlock() + s.dbMu.RLock() + defer s.dbMu.RUnlock() + if err := s.checkMigrationNotRetiredLocked(state.JobID); err != nil { + return err + } + + writeAppliedIndex, err := s.shouldWriteAppliedIndexLocked(appliedIndex) + if err != nil { + return err + } + + batch := s.db.NewBatch() + defer func() { _ = batch.Close() }() + if err := stageTargetReadinessGuard(batch, state, appliedIndex, writeAppliedIndex); err != nil { + return err + } + + // The clock floor must survive the same crash boundary as the guard and + // applied index, even when the moved range contains no imported versions. + floor := uint64(0) + if state.Armed { + floor = state.MinWriteTSExclusive + } + unlock, lastTS, err := s.stageMigrationClockMetadata(batch, state.JobID, floor) + if err != nil { + return err + } + defer unlock() + writeOpts := s.directApplyWriteOpts() + if raftApply { + writeOpts = s.raftApplyWriteOpts() + } + if err := batch.Commit(writeOpts); err != nil { + return errors.WithStack(err) + } + s.lastCommitTS = lastTS + s.migrationReadinessCache = upsertTargetStagedReadinessStateCache(s.migrationReadinessCache, state) + return nil +} + +func stageTargetReadinessGuard(batch *pebble.Batch, state TargetStagedReadinessState, appliedIndex uint64, writeAppliedIndex bool) error { + if err := batch.Set(migrationReadyKey(state.JobID), encodeTargetStagedReadinessState(state), nil); err != nil { + return errors.WithStack(err) + } + if writeAppliedIndex { + return setPebbleUint64InBatch(batch, metaAppliedIndexBytes, appliedIndex) + } + return nil +} + +func (s *pebbleStore) MigrationTargetReadinessStates(_ context.Context) ([]TargetStagedReadinessState, error) { + s.mtx.RLock() + defer s.mtx.RUnlock() + return cloneTargetStagedReadinessStates(s.migrationReadinessCache), nil +} + +func (s *pebbleStore) loadPebbleTargetReadinessStatesFromDB() ([]TargetStagedReadinessState, error) { + iter, err := s.db.NewIter(&pebble.IterOptions{ + LowerBound: []byte(migrationReadyPrefix), + UpperBound: PrefixScanEnd([]byte(migrationReadyPrefix)), + }) + if err != nil { + return nil, errors.WithStack(err) + } + defer iter.Close() + + var out []TargetStagedReadinessState + for valid := iter.First(); valid; valid = iter.Next() { + if !isTargetReadinessRecordKey(iter.Key()) { + continue + } + state, ok := decodeTargetStagedReadinessState(iter.Value()) + if !ok { + return nil, errors.New("corrupt target staged readiness state") + } + out = append(out, state) + } + if err := iter.Error(); err != nil { + return nil, errors.WithStack(err) + } + sortTargetStagedReadinessStates(out) + return out, nil +} + +func isTargetReadinessRecordKey(rawKey []byte) bool { + return len(rawKey) == len(migrationReadyPrefix)+migrationUint64Bytes && + bytes.HasPrefix(rawKey, []byte(migrationReadyPrefix)) +} + +func encodeTargetStagedReadinessState(state TargetStagedReadinessState) []byte { + buf := make([]byte, 0, targetReadinessEncodedSize(state)) + buf = append(buf, targetReadinessCodecVersion) + if state.TransferSizeGuardVersion != 0 { + buf[0] = targetReadinessGuardCodecVersion + } + if state.Armed { + buf = append(buf, targetReadinessArmedFlag) + } else { + buf = append(buf, 0) + } + var sourceFlags byte + if state.SourceWriteFence { + sourceFlags |= targetReadinessSourceWrite + } + if state.SourceReadFence { + sourceFlags |= targetReadinessSourceRead + } + if state.TrackWrites { + sourceFlags |= targetReadinessTrackWrites + } + if state.TransferSizeGuardVersion == MigrationTransferSizeGuardVersion { + sourceFlags |= targetReadinessTransferSizeGuard + } + buf = append(buf, sourceFlags) + buf = binary.BigEndian.AppendUint64(buf, state.JobID) + buf = binary.BigEndian.AppendUint64(buf, state.ExpectedCutoverVersion) + buf = binary.BigEndian.AppendUint64(buf, state.MigrationJobID) + buf = binary.BigEndian.AppendUint64(buf, state.MinWriteTSExclusive) + buf = binary.BigEndian.AppendUint64(buf, state.RetentionPinTS) + buf = binary.BigEndian.AppendUint64(buf, state.MinAdmittedTS) + buf = appendVarBytes(buf, state.RouteStart) + if state.RouteEnd == nil { + buf = append(buf, 0) + return buf + } + buf = append(buf, 1) + return appendVarBytes(buf, state.RouteEnd) +} + +func decodeTargetStagedReadinessState(data []byte) (TargetStagedReadinessState, bool) { + state, rest, ok := decodeTargetReadinessHeader(data) + if !ok { + return TargetStagedReadinessState{}, false + } + if !decodeTargetReadinessRange(&state, rest) { + return TargetStagedReadinessState{}, false + } + if err := validateTargetStagedReadinessState(state); err != nil { + return TargetStagedReadinessState{}, false + } + return state, true +} + +func decodeTargetReadinessHeader(data []byte) (TargetStagedReadinessState, []byte, bool) { + if len(data) == 0 { + return TargetStagedReadinessState{}, nil, false + } + if data[0] == targetReadinessCodecVersionV1 { + return decodeTargetReadinessHeaderV1(data) + } + if len(data) < 3+6*migrationUint64Bytes || (data[0] != targetReadinessCodecVersion && data[0] != targetReadinessGuardCodecVersion) { + return TargetStagedReadinessState{}, nil, false + } + if data[1] != 0 && data[1] != targetReadinessArmedFlag { + return TargetStagedReadinessState{}, nil, false + } + sourceFlags := data[2] + if !validTargetReadinessSourceFlags(data[0], sourceFlags) { + return TargetStagedReadinessState{}, nil, false + } + state := TargetStagedReadinessState{ + Armed: data[1] == targetReadinessArmedFlag, + SourceWriteFence: sourceFlags&targetReadinessSourceWrite != 0, + SourceReadFence: sourceFlags&targetReadinessSourceRead != 0, + TrackWrites: sourceFlags&targetReadinessTrackWrites != 0, + } + if sourceFlags&targetReadinessTransferSizeGuard != 0 { + state.TransferSizeGuardVersion = MigrationTransferSizeGuardVersion + } + rest := data[3:] + state.JobID = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.ExpectedCutoverVersion = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MigrationJobID = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MinWriteTSExclusive = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.RetentionPinTS = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MinAdmittedTS = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + return state, rest, true +} + +func validTargetReadinessSourceFlags(version, sourceFlags byte) bool { + allowedFlags := targetReadinessSourceWrite | targetReadinessSourceRead | targetReadinessTrackWrites + if version == targetReadinessGuardCodecVersion { + allowedFlags |= targetReadinessTransferSizeGuard + if sourceFlags&targetReadinessTransferSizeGuard == 0 { + return false + } + } + return sourceFlags & ^allowedFlags == 0 +} + +func decodeTargetReadinessHeaderV1(data []byte) (TargetStagedReadinessState, []byte, bool) { + if len(data) < 2+4*migrationUint64Bytes || data[1] != 0 && data[1] != targetReadinessArmedFlag { + return TargetStagedReadinessState{}, nil, false + } + state := TargetStagedReadinessState{Armed: data[1] == targetReadinessArmedFlag} + rest := data[2:] + state.JobID = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.ExpectedCutoverVersion = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MigrationJobID = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MinWriteTSExclusive = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + return state, rest, true +} + +func decodeTargetReadinessRange(state *TargetStagedReadinessState, data []byte) bool { + routeStart, rest, ok := readVarBytes(data) + if !ok || len(rest) == 0 { + return false + } + state.RouteStart = routeStart + endPresent := rest[0] + rest = rest[1:] + if endPresent == 0 { + return len(rest) == 0 + } + if endPresent != 1 { + return false + } + state.RouteEnd, rest, ok = readVarBytes(rest) + return ok && len(rest) == 0 +} + +func targetReadinessEncodedSize(state TargetStagedReadinessState) int { + size := 3 + 6*migrationUint64Bytes + binary.MaxVarintLen64 + len(state.RouteStart) + 1 + if state.RouteEnd != nil { + size += binary.MaxVarintLen64 + len(state.RouteEnd) + } + return size +} + +func appendVarBytes(dst []byte, value []byte) []byte { + dst = binary.AppendUvarint(dst, lenAsUint64(len(value))) + return append(dst, value...) +} + +func readVarBytes(data []byte) ([]byte, []byte, bool) { + l, n := binary.Uvarint(data) + if n <= 0 { + return nil, nil, false + } + rest := data[n:] + if l > lenAsUint64(len(rest)) { + return nil, nil, false + } + return bytes.Clone(rest[:l]), rest[l:], true +} + +func cloneTargetStagedReadinessState(state TargetStagedReadinessState) TargetStagedReadinessState { + return TargetStagedReadinessState{ + JobID: state.JobID, + RouteStart: bytes.Clone(state.RouteStart), + RouteEnd: bytes.Clone(state.RouteEnd), + ExpectedCutoverVersion: state.ExpectedCutoverVersion, + MigrationJobID: state.MigrationJobID, + MinWriteTSExclusive: state.MinWriteTSExclusive, + Armed: state.Armed, + SourceWriteFence: state.SourceWriteFence, + SourceReadFence: state.SourceReadFence, + RetentionPinTS: state.RetentionPinTS, + TrackWrites: state.TrackWrites, + MinAdmittedTS: state.MinAdmittedTS, + TransferSizeGuardVersion: state.TransferSizeGuardVersion, + } +} + +func cloneTargetStagedReadinessStates(states []TargetStagedReadinessState) []TargetStagedReadinessState { + if len(states) == 0 { + return nil + } + out := make([]TargetStagedReadinessState, 0, len(states)) + for _, state := range states { + out = append(out, cloneTargetStagedReadinessState(state)) + } + return out +} + +func upsertTargetStagedReadinessStateCache(cache []TargetStagedReadinessState, state TargetStagedReadinessState) []TargetStagedReadinessState { + cloned := cloneTargetStagedReadinessState(state) + for i := range cache { + if cache[i].JobID == cloned.JobID { + cache[i] = cloned + return cache + } + } + cache = append(cache, cloned) + sortTargetStagedReadinessStates(cache) + return cache +} + +func sortTargetStagedReadinessStates(states []TargetStagedReadinessState) { + sort.Slice(states, func(i, j int) bool { + return states[i].JobID < states[j].JobID + }) +} + +var _ MigrationTargetReadinessReader = (*mvccStore)(nil) +var _ MigrationTargetReadinessReader = (*pebbleStore)(nil) +var _ MigrationTargetReadinessWriter = (*mvccStore)(nil) +var _ MigrationTargetReadinessWriter = (*pebbleStore)(nil) +var _ MigrationTargetReadinessRaftWriter = (*mvccStore)(nil) +var _ MigrationTargetReadinessRaftWriter = (*pebbleStore)(nil) diff --git a/store/migration_readiness_clock_test.go b/store/migration_readiness_clock_test.go new file mode 100644 index 000000000..9e6609298 --- /dev/null +++ b/store/migration_readiness_clock_test.go @@ -0,0 +1,37 @@ +package store + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTargetReadinessAtomicallyAdvancesClockFloor(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + writer, ok := st.(MigrationTargetReadinessRaftWriter) + require.True(t, ok) + state := TargetStagedReadinessState{ + JobID: 7, MigrationJobID: 7, RouteStart: []byte("m"), RouteEnd: []byte("z"), + Armed: true, MinWriteTSExclusive: 100, + } + require.NoError(t, writer.ApplyTargetStagedReadinessAt(ctx, state, 77)) + require.Equal(t, uint64(100), st.LastCommitTS()) + floor, err := st.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Equal(t, uint64(100), floor) + state.MinWriteTSExclusive = 50 + require.NoError(t, writer.ApplyTargetStagedReadinessAt(ctx, state, 78)) + require.Equal(t, uint64(100), st.LastCommitTS()) + floor, err = st.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Equal(t, uint64(100), floor) + if reader, ok := st.(interface{ LastAppliedIndex() (uint64, bool, error) }); ok { + index, found, readErr := reader.LastAppliedIndex() + require.NoError(t, readErr) + require.True(t, found) + require.Equal(t, uint64(78), index) + } + }) +} diff --git a/store/migration_readiness_test.go b/store/migration_readiness_test.go new file mode 100644 index 000000000..5b3db499b --- /dev/null +++ b/store/migration_readiness_test.go @@ -0,0 +1,79 @@ +package store + +import ( + "encoding/binary" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTargetStagedReadinessCodecRoundTripSourceControls(t *testing.T) { + t.Parallel() + + want := TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 11, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 80, + TrackWrites: true, + MinAdmittedTS: 70, + } + + got, ok := decodeTargetStagedReadinessState(encodeTargetStagedReadinessState(want)) + require.True(t, ok) + require.Equal(t, want, got) +} + +func TestTargetStagedReadinessCodecDecodesV1(t *testing.T) { + t.Parallel() + + data := []byte{targetReadinessCodecVersionV1, targetReadinessArmedFlag} + for _, value := range []uint64{9, 11, 7, 100} { + data = binary.BigEndian.AppendUint64(data, value) + } + data = appendVarBytes(data, []byte("m")) + data = append(data, 1) + data = appendVarBytes(data, []byte("z")) + + got, ok := decodeTargetStagedReadinessState(data) + require.True(t, ok) + require.Equal(t, TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 11, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }, got) +} + +func TestValidateTargetStagedReadinessSourceControls(t *testing.T) { + t.Parallel() + + base := TargetStagedReadinessState{ + JobID: 9, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + } + + readOnly := base + readOnly.SourceReadFence = true + readOnly.RetentionPinTS = 80 + require.ErrorContains(t, validateTargetStagedReadinessState(readOnly), "requires source write fence") + + withoutPin := base + withoutPin.SourceWriteFence = true + require.ErrorContains(t, validateTargetStagedReadinessState(withoutPin), "retention pin is required") + + trackerWithoutPin := base + trackerWithoutPin.TrackWrites = true + require.ErrorContains(t, validateTargetStagedReadinessState(trackerWithoutPin), "tracker retention pin is required") +} diff --git a/store/migration_retirement.go b/store/migration_retirement.go new file mode 100644 index 000000000..72ce0801c --- /dev/null +++ b/store/migration_retirement.go @@ -0,0 +1,99 @@ +package store + +import ( + "bytes" + "context" + "encoding/binary" + "sort" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +var migrationRetiredMetaKeyBytes = []byte("!migstage|retired") + +// ErrMigrationRetired is a replicated per-job refusal, not a storage failure. +var ErrMigrationRetired = errors.New("migration job is retired") + +func IsMigrationRetirementMetadataKey(key []byte) bool { + return bytes.Equal(key, migrationRetiredMetaKeyBytes) +} + +func encodeMigrationRetiredJobs(jobs map[uint64]struct{}) []byte { + ids := make([]uint64, 0, len(jobs)) + for id := range jobs { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + out := binary.AppendUvarint([]byte{migrationMetadataVersion}, lenAsUint64(len(ids))) + for _, id := range ids { + out = binary.BigEndian.AppendUint64(out, id) + } + return out +} + +func decodeMigrationRetiredJobs(data []byte) (map[uint64]struct{}, bool) { + if len(data) == 0 || data[0] != migrationMetadataVersion { + return nil, false + } + count, size := binary.Uvarint(data[1:]) + if size <= 0 { + return nil, false + } + rest := data[1+size:] + if count != lenAsUint64(len(rest))/migrationUint64Bytes || len(rest)%migrationUint64Bytes != 0 { + return nil, false + } + jobs := make(map[uint64]struct{}, count) + for len(rest) != 0 { + id := binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + if _, duplicate := jobs[id]; id == 0 || duplicate { + return nil, false + } + jobs[id] = struct{}{} + rest = rest[migrationUint64Bytes:] + } + return jobs, true +} + +func (s *pebbleStore) readMigrationRetiredJobs() (map[uint64]struct{}, error) { + value, closer, err := s.db.Get(migrationRetiredMetaKeyBytes) + if errors.Is(err, pebble.ErrNotFound) { + return make(map[uint64]struct{}), nil + } + if err != nil { + return nil, errors.WithStack(err) + } + defer func() { _ = closer.Close() }() + jobs, valid := decodeMigrationRetiredJobs(value) + if !valid { + return nil, errors.New("corrupt migration retirement witnesses") + } + return jobs, nil +} + +func (s *pebbleStore) checkMigrationNotRetiredLocked(jobID uint64) error { + jobs, err := s.readMigrationRetiredJobs() + if err != nil { + return err + } + if _, retired := jobs[jobID]; retired { + return errors.WithStack(ErrMigrationRetired) + } + return nil +} + +func (s *mvccStore) MigrationRetired(_ context.Context, jobID uint64) (bool, error) { + s.mtx.RLock() + defer s.mtx.RUnlock() + _, retired := s.migrationRetired[jobID] + return retired, nil +} + +func (s *pebbleStore) MigrationRetired(_ context.Context, jobID uint64) (bool, error) { + s.dbMu.RLock() + defer s.dbMu.RUnlock() + jobs, err := s.readMigrationRetiredJobs() + _, retired := jobs[jobID] + return retired, err +} diff --git a/store/migration_retirement_test.go b/store/migration_retirement_test.go new file mode 100644 index 000000000..aec4a9e0e --- /dev/null +++ b/store/migration_retirement_test.go @@ -0,0 +1,188 @@ +package store + +import ( + "bytes" + "context" + "encoding/binary" + "log/slog" + "testing" + + "github.com/cockroachdb/pebble/v2" + "github.com/stretchr/testify/require" +) + +func TestMetadataCleanupPermanentlyRejectsLateMigrationMutations(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + cleaner, ok := st.(MigrationCleaner) + require.True(t, ok) + require.NoError(t, st.PutAt(ctx, []byte("customer"), []byte("active"), 100, 0)) + _, err := st.ImportVersions(ctx, ImportVersionsOptions{JobID: 7, BracketID: 1, BatchSeq: 1, Cursor: []byte("old-progress")}) + require.NoError(t, err) + require.NoError(t, cleaner.ClearMigrationState(ctx, 7, 10)) + assertLateMigrationRejected(t, st) + require.NoError(t, cleaner.ClearMigrationState(ctx, 7, 11), "cleanup must remain idempotent") + require.NoError(t, cleaner.ClearMigrationState(ctx, 99, 12)) + _, err = st.ImportVersions(ctx, ImportVersionsOptions{JobID: 8, BracketID: 1, BatchSeq: 1}) + require.NoError(t, err, "retirement is per job") + }) +} + +func assertLateMigrationRejected(t *testing.T, st MVCCStore) { + t.Helper() + ctx := context.Background() + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + err := writer.ApplyTargetStagedReadiness(ctx, TargetStagedReadinessState{JobID: 7, MigrationJobID: 7, + RouteStart: []byte("a"), RouteEnd: []byte("z"), Armed: true, SourceWriteFence: true, MinWriteTSExclusive: 1, RetentionPinTS: 1}) + require.ErrorContains(t, err, "migration job is retired") + raftWriter, ok := st.(MigrationTargetReadinessRaftWriter) + require.True(t, ok) + require.ErrorContains(t, raftWriter.ApplyTargetStagedReadinessAt(ctx, TargetStagedReadinessState{JobID: 7, MigrationJobID: 7}, 20), "migration job is retired") + _, err = st.ImportVersions(ctx, ImportVersionsOptions{JobID: 7, BracketID: 1, BatchSeq: 1, + Versions: []MVCCVersion{{Key: []byte("late-staging"), CommitTS: 200, Value: []byte("late")}}}) + require.ErrorContains(t, err, "migration job is retired") + _, err = st.ImportVersionsRaft(ctx, ImportVersionsOptions{JobID: 7, BracketID: 1, BatchSeq: 1, AppliedIndex: 21}) + require.ErrorContains(t, err, "migration job is retired", "a duplicate/empty batch cannot bypass retirement") + promoter, ok := st.(MigrationPromoter) + require.True(t, ok) + _, err = promoter.PromoteVersions(ctx, PromoteVersionsOptions{JobID: 7, MaxVersions: 10, TargetKey: func(key []byte) ([]byte, bool) { return key, true }}) + require.ErrorContains(t, err, "migration job is retired") + _, err = promoter.PromoteVersions(ctx, PromoteVersionsOptions{JobID: 7, TargetKey: func(key []byte) ([]byte, bool) { return key, true }}) + require.ErrorContains(t, err, "migration job is retired", "zero budgets must not report Done") + cleaner, ok := st.(MigrationCleaner) + require.True(t, ok) + _, err = cleaner.CleanupVersions(ctx, CleanupVersionsOptions{JobID: 7, StartKey: []byte("customer"), MaxVersions: 10, MaxCommitTS: 1000}) + require.ErrorContains(t, err, "migration job is retired") + _, err = cleaner.CleanupVersions(ctx, CleanupVersionsOptions{JobID: 7}) + require.ErrorContains(t, err, "migration job is retired") + require.NoError(t, st.RetireMigration(ctx, 7), "legacy retirement must preserve the witness") + retirement, ok := st.(MigrationRetirementReader) + require.True(t, ok) + retired, err := retirement.MigrationRetired(ctx, 7) + require.NoError(t, err) + require.True(t, retired) + reader, ok := st.(MigrationImportMetadataReader) + require.True(t, ok) + present, err := reader.MigrationImportMetadataPresent(ctx, 7) + require.NoError(t, err) + require.False(t, present, "a retirement witness is not live import progress") + readiness, ok := st.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := readiness.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Empty(t, states, "retirement must not retain an active guard or retention pin") + require.Equal(t, uint64(100), st.LastCommitTS(), "rejected requests must not move the clock") + require.NoError(t, st.Compact(ctx, 100), "ordinary compaction must not collect retirement") + retired, err = retirement.MigrationRetired(ctx, 7) + require.NoError(t, err) + require.True(t, retired) + value, err := st.GetAt(ctx, []byte("customer"), 100) + require.NoError(t, err) + require.Equal(t, []byte("active"), value) +} + +func TestMigrationRetirementSurvivesSnapshotRestoreAndReopen(t *testing.T) { + source := NewMVCCStore() + initial, err := source.Snapshot() + require.NoError(t, err) + initialRaw := snapshotBytes(t, initial) + require.NoError(t, initial.Close()) + require.Equal(t, mvccSnapshotVersionV1, binary.LittleEndian.Uint32(initialRaw[len(mvccSnapshotMagic):]), "no witness must retain the old snapshot format") + cleaner, ok := source.(MigrationCleaner) + require.True(t, ok) + require.NoError(t, source.PutAt(t.Context(), []byte("customer"), []byte("active"), 100, 0)) + require.NoError(t, cleaner.ClearMigrationState(t.Context(), 7, 10)) + snapshot, err := source.Snapshot() + require.NoError(t, err) + raw := snapshotBytes(t, snapshot) + require.Equal(t, mvccSnapshotVersionV5, binary.LittleEndian.Uint32(raw[len(mvccSnapshotMagic):])) + require.NoError(t, snapshot.Close()) + runMigrationStoreSuite(t, func(t *testing.T, destination MVCCStore) { + require.NoError(t, destination.Restore(bytes.NewReader(raw))) + assertLateMigrationRejected(t, destination) + }) + dir := t.TempDir() + pebble, err := NewPebbleStore(dir) + require.NoError(t, err) + require.NoError(t, pebble.Restore(bytes.NewReader(raw))) + require.NoError(t, pebble.Close()) + pebble, err = NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, pebble.Close()) }) + assertLateMigrationRejected(t, pebble) +} + +func TestMigrationRetirementNativeAndSSTSnapshots(t *testing.T) { + for _, sst := range []bool{false, true} { + t.Run(map[bool]string{false: "native", true: "sst"}[sst], func(t *testing.T) { + source, err := NewPebbleStore(t.TempDir(), WithSSTIngestSnapshots(sst)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, source.Close()) }) + require.NoError(t, source.PutAt(t.Context(), []byte("customer"), []byte("active"), 100, 0)) + cleaner, ok := source.(MigrationCleaner) + require.True(t, ok) + require.NoError(t, cleaner.ClearMigrationState(t.Context(), 7, 10)) + snapshot, err := source.Snapshot() + require.NoError(t, err) + raw := snapshotBytes(t, snapshot) + require.NoError(t, snapshot.Close()) + target, err := NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, target.Close()) }) + require.NoError(t, target.Restore(bytes.NewReader(raw))) + assertLateMigrationRejected(t, target) + }) + } +} + +func TestMigrationRetirementCleanupBatchFailureIsAtomic(t *testing.T) { + db, err := pebble.Open(t.TempDir(), &pebble.Options{DisableWAL: true}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + s := &pebbleStore{db: db, log: slog.Default(), fsmApplyWriteOpts: pebble.NoSync} + state := TargetStagedReadinessState{JobID: 7, MigrationJobID: 7, Armed: true, SourceWriteFence: true, + MinWriteTSExclusive: 100, RetentionPinTS: 1} + require.NoError(t, s.ApplyTargetStagedReadinessAt(t.Context(), state, 5)) + // A synced commit on a no-WAL DB deterministically fails before publication. + s.fsmApplyWriteOpts = pebble.Sync + require.ErrorContains(t, s.ClearMigrationState(t.Context(), 7, 6), "WAL disabled") + retired, err := s.MigrationRetired(t.Context(), 7) + require.NoError(t, err) + require.False(t, retired) + states, err := s.MigrationTargetReadinessStates(t.Context()) + require.NoError(t, err) + require.Len(t, states, 1) + index, found, err := s.LastAppliedIndex() + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint64(5), index) + s.fsmApplyWriteOpts = pebble.NoSync + require.NoError(t, s.ClearMigrationState(t.Context(), 7, 6)) + retired, err = s.MigrationRetired(t.Context(), 7) + require.NoError(t, err) + require.True(t, retired) + states, err = s.MigrationTargetReadinessStates(t.Context()) + require.NoError(t, err) + require.Empty(t, states) + index, found, err = s.LastAppliedIndex() + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint64(6), index) +} + +func TestCorruptMigrationRetirementFailsClosed(t *testing.T) { + dir := t.TempDir() + st, err := NewPebbleStore(dir) + require.NoError(t, err) + ps, ok := st.(*pebbleStore) + require.True(t, ok) + require.NoError(t, ps.db.Set(migrationRetiredMetaKeyBytes, []byte("corrupt"), pebble.Sync)) + _, err = ps.ImportVersions(t.Context(), ImportVersionsOptions{JobID: 8, BracketID: 1, BatchSeq: 1}) + require.ErrorContains(t, err, "corrupt migration retirement") + require.NotErrorIs(t, err, ErrMigrationRetired, "corruption must not be mistaken for an ordinary retired-job verdict") + require.ErrorContains(t, ps.ClearMigrationState(t.Context(), 8, 10), "corrupt migration retirement") + require.NoError(t, st.Close()) + _, err = NewPebbleStore(dir) + require.ErrorContains(t, err, "corrupt migration retirement") +} diff --git a/store/migration_serving_admission.go b/store/migration_serving_admission.go new file mode 100644 index 000000000..7bbf89042 --- /dev/null +++ b/store/migration_serving_admission.go @@ -0,0 +1,150 @@ +package store + +import ( + "context" + + "github.com/cockroachdb/errors" +) + +// MigrationServingAdmissionReader is an optional compact, coherent serving +// view. It retains unknown legacy barriers, but excludes adopted legacy jobs +// and historical partition receipts not referenced by the active frontier. +// Full-history readers remain available for control CAS and receipt replay. +type MigrationServingAdmissionReader interface { + MigrationServingAdmissionState(context.Context) (MigrationServingState, error) +} + +// MigrationServingSummary is a catalog floor and conservative metadata-presence +// hint, never ownership authority. Present includes unarmed readiness and +// receipt-only history; an empty hint must not bypass route/configuration gates. +type MigrationServingSummary struct { + CatalogVersionFloor uint64 + Present bool +} + +type MigrationServingSummaryReader interface { + MigrationServingSummary(context.Context) (MigrationServingSummary, error) +} + +// Protected by the store metadata mutex. Replaced, never mutated, after a +// successful metadata commit or validated restore. Readiness is kept in the +// existing cache under that same mutex rather than duplicated here. +type migrationServingAdmissionIndex struct { + state MigrationServingState + summary MigrationServingSummary +} + +func buildMigrationServingAdmission(ownership []MigrationOwnershipState, partitions MigrationOwnershipPartitionState) migrationServingAdmissionIndex { + index := migrationServingAdmissionIndex{} + index.summary.Present = len(ownership) != 0 || partitionHasEvidence(partitions) + consumed := partitionAdoptedJobs(partitions) + receipts := make(map[string]MigrationOwnershipPartitionReceipt, len(partitions.Receipts)) + for _, r := range partitions.Receipts { + receipts[string(r.ReceiptID)] = r + } + index.state.Ownership, index.summary.CatalogVersionFloor = admissionLegacyOwnership(ownership, consumed) + index.state.Partitions.Generation = partitions.Generation + index.state.Partitions.Partitions = partitions.Partitions + used := make(map[string]bool) + for _, p := range partitions.Partitions { + index.summary.CatalogVersionFloor = max(index.summary.CatalogVersionFloor, p.CatalogVersion) + id := string(p.ReceiptID) + if !used[id] { + index.state.Partitions.Receipts = append(index.state.Partitions.Receipts, receipts[id]) + used[id] = true + } + } + index.state.Ownership = cloneMigrationOwnership(index.state.Ownership) + index.state.Partitions = clonePartitionState(index.state.Partitions) + index.state.Partitions.servingEvidence = buildPartitionEvidenceIndex(index.state.Partitions) + return index +} + +func partitionAdoptedJobs(state MigrationOwnershipPartitionState) map[uint64]bool { + consumed := make(map[uint64]bool) + for _, receipt := range state.Receipts { + for _, ref := range receipt.Transition.Before { + if ref.LegacyJobID != 0 { + consumed[ref.LegacyJobID] = true + } + } + // Guarded Acquire/Clear creates a new job certificate and frontier in + // the same commit; its Before references name the superseded partition, + // not the newly created job. That new ledger is history, not a second + // certificate competing with the frontier at the same catalog version. + if receipt.Transition.Operation == MigrationOwnershipPartitionJobOperation { + for _, after := range receipt.Transition.After { + consumed[after.MigrationJobID] = true + } + } + } + return consumed +} + +func admissionLegacyOwnership(ownership []MigrationOwnershipState, consumed map[uint64]bool) ([]MigrationOwnershipState, uint64) { + var retained []MigrationOwnershipState + var floor uint64 + for _, old := range ownership { + if !MigrationOwnershipEvidenceValid(old) { + retained = append(retained, old) + continue + } + floor = max(floor, old.CatalogVersion) + if !consumed[old.JobID] { + retained = append(retained, old) + } + } + return retained, floor +} + +func (i migrationServingAdmissionIndex) view(readiness []TargetStagedReadinessState) MigrationServingState { + return MigrationServingState{Ownership: cloneMigrationOwnership(i.state.Ownership), + Partitions: clonePartitionState(i.state.Partitions), Readiness: cloneReadinessStates(readiness)} +} + +func (i migrationServingAdmissionIndex) withReadiness(readiness []TargetStagedReadinessState) MigrationServingSummary { + summary := i.summary + summary.Present = summary.Present || len(readiness) != 0 + return summary +} + +func (s *mvccStore) MigrationServingAdmissionState(ctx context.Context) (MigrationServingState, error) { + if err := ctx.Err(); err != nil { + return MigrationServingState{}, errors.WithStack(err) + } + s.mtx.RLock() + defer s.mtx.RUnlock() + return s.migrationAdmission.view(s.migrationReadinessCache), nil +} + +func (s *pebbleStore) MigrationServingAdmissionState(ctx context.Context) (MigrationServingState, error) { + if err := ctx.Err(); err != nil { + return MigrationServingState{}, errors.WithStack(err) + } + s.mtx.RLock() + defer s.mtx.RUnlock() + return s.migrationAdmission.view(s.migrationReadinessCache), nil +} + +func (s *mvccStore) MigrationServingSummary(ctx context.Context) (MigrationServingSummary, error) { + if err := ctx.Err(); err != nil { + return MigrationServingSummary{}, errors.WithStack(err) + } + s.mtx.RLock() + defer s.mtx.RUnlock() + return s.migrationAdmission.withReadiness(s.migrationReadinessCache), nil +} + +func (s *pebbleStore) MigrationServingSummary(ctx context.Context) (MigrationServingSummary, error) { + if err := ctx.Err(); err != nil { + return MigrationServingSummary{}, errors.WithStack(err) + } + s.mtx.RLock() + defer s.mtx.RUnlock() + return s.migrationAdmission.withReadiness(s.migrationReadinessCache), nil +} + +var _ MigrationServingAdmissionReader = (*mvccStore)(nil) +var _ MigrationServingAdmissionReader = (*pebbleStore)(nil) +var _ MigrationServingSummaryReader = (*mvccStore)(nil) +var _ MigrationServingSummaryReader = (*pebbleStore)(nil) diff --git a/store/migration_serving_admission_test.go b/store/migration_serving_admission_test.go new file mode 100644 index 000000000..ef5f21e1d --- /dev/null +++ b/store/migration_serving_admission_test.go @@ -0,0 +1,231 @@ +package store + +import ( + "bytes" + "context" + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func admissionReader(t *testing.T, st MVCCStore) MigrationServingAdmissionReader { + t.Helper() + reader, ok := st.(MigrationServingAdmissionReader) + require.True(t, ok) + return reader +} + +func servingSummary(t *testing.T, st MVCCStore) MigrationServingSummary { + t.Helper() + reader, ok := st.(MigrationServingSummaryReader) + require.True(t, ok) + summary, err := reader.MigrationServingSummary(t.Context()) + require.NoError(t, err) + return summary +} + +func refreshAdmissionPartition(t *testing.T, st MVCCStore, version uint64) { + t.Helper() + full, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + p := full.Partitions.Partitions[0] + ref, err := MigrationOwnershipPartitionReferenceFor(p) + require.NoError(t, err) + tx := MigrationOwnershipPartitionTransition{OperationID: "admission/refresh/" + strconv.FormatUint(version, 10), + Operation: MigrationOwnershipPartitionRefresh, RootManifestHash: bytes.Repeat([]byte{4}, 32), + Generation: p.Generation, LocalGroupID: p.LocalGroupID, CatalogVersion: version, + ConfigurationIndex: p.ConfigurationIndex, Before: []MigrationOwnershipPartitionReference{ref}, + After: []MigrationOwnershipPartition{p}} + finalizePartitionTransition(t, &tx) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, version)) +} + +func TestMigrationServingAdmissionCompactsHistoryAndOwnsSlices(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + reader := admissionReader(t, st) + require.Equal(t, MigrationServingSummary{}, servingSummary(t, st)) + _, tx := partitionFixture(t, st) + legacy, err := reader.MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Len(t, legacy.Ownership, 1) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + for version := uint64(13); version < 33; version++ { + refreshAdmissionPartition(t, st, version) + } + full, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + require.Len(t, full.Partitions.Receipts, 21) + view, err := reader.MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Empty(t, view.Ownership, "adopted legacy job must not obscure children") + require.Len(t, view.Partitions.Receipts, 2, "cost depends on active support, not immutable history") + require.Equal(t, full.Partitions.Partitions, view.Partitions.Partitions) + for _, p := range view.Partitions.Partitions { + require.True(t, MigrationOwnershipPartitionEvidenceValid(view.Partitions, p)) + } + require.Equal(t, MigrationServingSummary{CatalogVersionFloor: 32, Present: true}, servingSummary(t, st)) + view.Partitions.Partitions[0].DescriptorHash[0]++ + view.Partitions.Receipts[0].Transition.After[0].Origins[0].DescriptorHash[0]++ + after, err := reader.MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Equal(t, full.Partitions.Partitions, after.Partitions.Partitions) + require.True(t, MigrationOwnershipPartitionEvidenceValid(after.Partitions, after.Partitions.Partitions[0])) + }) +} + +func TestMigrationServingAdmissionUnknownLegacyAndReceiptOnlyRestore(t *testing.T) { + for _, receiptOnly := range []bool{false, true} { + source := NewMVCCStore() + _, tx := partitionFixture(t, source) + require.NoError(t, partitionStore(t, source).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + ms, ok := source.(*mvccStore) + require.True(t, ok) + if receiptOnly { + ms.migrationOwnership = nil + ms.migrationPartitions.Partitions = nil + } else { + unknown := ms.migrationOwnership[0] + unknown.JobID, unknown.Receipts = 99, nil + ms.migrationOwnership = append(ms.migrationOwnership, unknown) + } + raw := externalTestSnapshot(t, source) + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + require.NoError(t, st.Restore(bytes.NewReader(raw))) + view, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + if receiptOnly { + require.Empty(t, view.Partitions.Partitions) + require.Empty(t, view.Partitions.Receipts) + } else { + require.Len(t, view.Ownership, 1) + require.Equal(t, uint64(99), view.Ownership[0].JobID) + require.False(t, MigrationOwnershipEvidenceValid(view.Ownership[0])) + } + require.True(t, servingSummary(t, st).Present) + clean := externalTestSnapshot(t, NewMVCCStore()) + require.NoError(t, st.Restore(bytes.NewReader(clean))) + require.Equal(t, MigrationServingSummary{}, servingSummary(t, st)) + view, err = admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Empty(t, view.Ownership) + require.Empty(t, view.Partitions.Partitions) + }) + } +} + +func TestMigrationServingAdmissionDoesNotAcquireDatabaseOrApplyLock(t *testing.T) { + for _, migrated := range []bool{false, true} { + st, err := NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + if migrated { + _, tx := partitionFixture(t, st) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + } + ps, ok := st.(*pebbleStore) + require.True(t, ok) + func() { + ps.applyMu.Lock() + defer ps.applyMu.Unlock() + ps.dbMu.Lock() + defer ps.dbMu.Unlock() + done := make(chan error, 1) + go func() { + _, readErr := ps.MigrationServingAdmissionState(t.Context()) + if readErr == nil { + _, readErr = ps.MigrationServingSummary(t.Context()) + } + done <- readErr + }() + select { + case readErr := <-done: + require.NoError(t, readErr) + case <-time.After(5 * time.Second): + t.Fatal("admission reader acquired DB/apply lock") + } + }() + } +} + +func TestMigrationServingAdmissionConcurrentPublication(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + _, tx := partitionFixture(t, st) + require.NoError(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + var wg sync.WaitGroup + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + wg.Go(func() { + reader := admissionReader(t, st) + for ctx.Err() == nil { + view, err := reader.MigrationServingAdmissionState(ctx) + if ctx.Err() != nil { + return + } + require.NoError(t, err) + for _, p := range view.Partitions.Partitions { + require.True(t, MigrationOwnershipPartitionEvidenceValid(view.Partitions, p)) + } + } + }) + for version := uint64(13); version < 23; version++ { + refreshAdmissionPartition(t, st, version) + } + cancel() + wg.Wait() + }) +} + +func TestMigrationServingAdmissionLegacyRefreshDoesNotBorrowCallerSlices(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + old, _ := partitionFixture(t, st) + old.CatalogVersion++ + refresher, ok := st.(MigrationOwnershipRefresher) + require.True(t, ok) + require.NoError(t, refresher.RefreshMigrationOwnership(t.Context(), old, 10)) + before, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + old.RouteStart[0]++ + old.RouteEnd[0]++ + old.DescriptorHash[0]++ + old.Receipts[0].DescriptorHash[0]++ + after, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, before, after) + admission, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Equal(t, before, admission) + }) +} + +func TestMigrationServingAdmissionDoesNotHideUnknownAdoptedLegacy(t *testing.T) { + source := NewMVCCStore() + _, tx := partitionFixture(t, source) + require.NoError(t, partitionStore(t, source).ApplyMigrationOwnershipPartition(t.Context(), tx, 10)) + ms, ok := source.(*mvccStore) + require.True(t, ok) + // Decodable legacy history cannot become ignorable just because its job + // ID appeared in a historical adoption receipt with a different digest. + ms.migrationOwnership[0].Receipts = nil + raw := externalTestSnapshot(t, source) + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + require.NoError(t, st.Restore(bytes.NewReader(raw))) + view, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Len(t, view.Ownership, 1) + require.False(t, MigrationOwnershipEvidenceValid(view.Ownership[0])) + p := view.Partitions.Partitions[0] + ref, err := MigrationOwnershipPartitionReferenceFor(p) + require.NoError(t, err) + refresh := tx + refresh.Operation, refresh.OperationID, refresh.CatalogVersion = MigrationOwnershipPartitionRefresh, "unknown/refresh", 13 + refresh.Before, refresh.After = []MigrationOwnershipPartitionReference{ref}, []MigrationOwnershipPartition{p} + finalizePartitionTransition(t, &refresh) + require.ErrorIs(t, partitionStore(t, st).ApplyMigrationOwnershipPartition(t.Context(), refresh, 13), ErrMigrationOwnership) + after, err := admissionReader(t, st).MigrationServingAdmissionState(t.Context()) + require.NoError(t, err) + require.Equal(t, view, after) + }) +} diff --git a/store/migration_size_guard_test.go b/store/migration_size_guard_test.go new file mode 100644 index 000000000..8aca8225b --- /dev/null +++ b/store/migration_size_guard_test.go @@ -0,0 +1,89 @@ +package store + +import ( + "bytes" + "encoding/hex" + "testing" + + "github.com/stretchr/testify/require" +) + +func transferSizeReadinessForTest() TargetStagedReadinessState { + return TargetStagedReadinessState{JobID: 11, MigrationJobID: 11, RouteStart: []byte("m"), RouteEnd: []byte("z"), + Armed: true, TrackWrites: true, MinWriteTSExclusive: 1, RetentionPinTS: 1, + TransferSizeGuardVersion: MigrationTransferSizeGuardVersion} +} + +func TestMigrationSizeGuardCodecExplicitVersion(t *testing.T) { + state := transferSizeReadinessForTest() + encoded := encodeTargetStagedReadinessState(state) + require.Equal(t, byte(3), encoded[0]) + decoded, ok := decodeTargetStagedReadinessState(encoded) + require.True(t, ok) + require.Equal(t, state, decoded) + state.TransferSizeGuardVersion = 0 + // Frozen V2 bytes: legacy trackers retain their exact persisted contract. + want, err := hex.DecodeString("020104000000000000000b0000000000000000000000000000000b000000000000000100000000000000010000000000000000016d01017a") + require.NoError(t, err) + require.Equal(t, want, encodeTargetStagedReadinessState(state)) + legacy, ok := decodeTargetStagedReadinessState(want) + require.True(t, ok) + require.Zero(t, legacy.TransferSizeGuardVersion) + encoded[0] = 4 + _, ok = decodeTargetStagedReadinessState(encoded) + require.False(t, ok, "unknown future guard codec must not silently lose the guard") +} + +func TestMigrationSizeGuardSnapshotAndReopen(t *testing.T) { + want := transferSizeReadinessForTest() + runMigrationStoreSuite(t, func(t *testing.T, source MVCCStore) { + writer, ok := source.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(t.Context(), want)) + snapshot, err := source.Snapshot() + require.NoError(t, err) + raw := snapshotBytes(t, snapshot) + require.NoError(t, snapshot.Close()) + // Native Pebble snapshots are not a memory-store input format. + if _, memory := source.(*mvccStore); memory { + destination := NewMVCCStore() + t.Cleanup(func() { require.NoError(t, destination.Close()) }) + require.NoError(t, destination.Restore(bytes.NewReader(raw))) + reader, ok := destination.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(t.Context()) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{want}, states) + } + dir := t.TempDir() + pebble, err := NewPebbleStore(dir) + require.NoError(t, err) + require.NoError(t, pebble.Restore(bytes.NewReader(raw))) + require.NoError(t, pebble.Close()) + pebble, err = NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, pebble.Close()) }) + reader, ok := pebble.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(t.Context()) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{want}, states) + }) +} + +func TestMigrationSizeGuardRejectsInvalidActivation(t *testing.T) { + for _, change := range []func(*TargetStagedReadinessState){ + func(s *TargetStagedReadinessState) { s.TransferSizeGuardVersion = 2 }, + func(s *TargetStagedReadinessState) { s.Armed = false }, + func(s *TargetStagedReadinessState) { s.TrackWrites = false }, + } { + state := transferSizeReadinessForTest() + change(&state) + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + err := writer.ApplyTargetStagedReadiness(t.Context(), state) + require.Error(t, err) + }) + } +} diff --git a/store/migration_source_ownership_test.go b/store/migration_source_ownership_test.go new file mode 100644 index 000000000..8b648daa2 --- /dev/null +++ b/store/migration_source_ownership_test.go @@ -0,0 +1,73 @@ +package store + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSourceOwnershipZeroFloorBoundarySurvivesNativeRestore(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + guards, ok := st.(MigrationTargetReadinessRaftWriter) + require.True(t, ok) + guard := TargetStagedReadinessState{JobID: 7, MigrationJobID: 7, Armed: true, TrackWrites: true, + RouteStart: []byte("m"), RouteEnd: []byte("z"), ExpectedCutoverVersion: 1, MinWriteTSExclusive: 1, RetentionPinTS: 1} + require.NoError(t, guards.ApplyTargetStagedReadinessAt(t.Context(), guard, 10)) + state := MigrationOwnershipState{Generation: "0123456789abcdef0123456789abcdef", JobID: 7, LocalGroupID: 1, OwnerGroupID: 1, + Role: MigrationOwner, CatalogVersion: 1, ConfigurationIndex: 9, RouteID: 1, DescriptorHash: bytes.Repeat([]byte{1}, 32), + RouteStart: []byte("m"), RouteEnd: []byte("z"), WriteFloorKnown: true, SourceTrackerVersion: 1} + proofs := ownershipStore(t, st) + wrong := state + wrong.WriteFloorKnown = false + require.ErrorIs(t, proofs.AcquireMigrationOwnership(t.Context(), wrong, 11), ErrMigrationOwnership) + wrong = state + wrong.SourceTrackerVersion++ + require.ErrorIs(t, proofs.AcquireMigrationOwnership(t.Context(), wrong, 11), ErrMigrationOwnership) + require.NoError(t, proofs.AcquireMigrationOwnership(t.Context(), state, 11)) + view, err := proofs.MigrationServingState(t.Context()) + require.NoError(t, err) + require.True(t, MigrationOwnershipEvidenceValid(view.Ownership[0])) + require.Zero(t, view.Ownership[0].MinWriteTSExclusive) + require.Equal(t, uint64(11), view.Ownership[0].SourceAppliedIndex) + snapshot, err := st.Snapshot() + require.NoError(t, err) + raw := snapshotBytes(t, snapshot) + require.NoError(t, snapshot.Close()) + dir := t.TempDir() + restored, err := NewPebbleStore(dir) + require.NoError(t, err) + require.NoError(t, restored.Restore(bytes.NewReader(raw))) + require.NoError(t, restored.Close()) + restored, err = NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, restored.Close()) }) + view, err = ownershipStore(t, restored).MigrationServingState(t.Context()) + require.NoError(t, err) + require.True(t, MigrationOwnershipEvidenceValid(view.Ownership[0])) + require.Equal(t, uint64(11), view.Ownership[0].SourceAppliedIndex) + guard.SourceWriteFence, guard.SourceReadFence, guard.ExpectedCutoverVersion, guard.MinWriteTSExclusive = true, true, 2, 100 + require.NoError(t, guards.ApplyTargetStagedReadinessAt(t.Context(), guard, 12)) + state.OwnerGroupID, state.Role, state.CatalogVersion, state.MinWriteTSExclusive = 2, MigrationNonOwner, 2, 100 + state.SourceTrackerVersion, state.SourceAppliedIndex, state.DescriptorHash = 0, 0, bytes.Repeat([]byte{2}, 32) + require.NoError(t, proofs.ClearMigrationStateWithOwnership(t.Context(), state, 13)) + view, err = proofs.MigrationServingState(t.Context()) + require.NoError(t, err) + require.Empty(t, view.Readiness) + require.True(t, MigrationOwnershipEvidenceValid(view.Ownership[0])) + require.Len(t, view.Ownership[0].Receipts, 2) + }) +} + +func TestLegacyOwnershipRemainsDecodableButNotOperational(t *testing.T) { + st := NewMVCCStore() + state := ownershipFixture(t, st) + require.NoError(t, ownershipStore(t, st).ClearMigrationStateWithOwnership(t.Context(), state, 9)) + view, err := ownershipStore(t, st).MigrationServingState(t.Context()) + require.NoError(t, err) + encoded, err := encodeMigrationOwnership(view.Ownership) + require.NoError(t, err) + decoded, err := decodeMigrationOwnership(encoded) + require.NoError(t, err) + require.False(t, MigrationOwnershipEvidenceValid(decoded[0])) +} diff --git a/store/migration_transfer_budget_test.go b/store/migration_transfer_budget_test.go new file mode 100644 index 000000000..1006e2e4f --- /dev/null +++ b/store/migration_transfer_budget_test.go @@ -0,0 +1,14 @@ +package store + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMigrationTransferBudgetIncludesFullKeyCursor(t *testing.T) { + key := make([]byte, 1<<20) + value := make([]byte, MigrationPagePayloadBudget-2*len(key)-128) + require.True(t, MigrationVersionFitsTransfer(key, value)) + require.False(t, MigrationVersionFitsTransfer(key, make([]byte, len(value)+1))) +} diff --git a/store/migration_versions.go b/store/migration_versions.go index 32ef2ccc7..38d8f16f7 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -10,6 +10,17 @@ import ( "github.com/emirpasic/gods/maps/treemap" ) +// MigrationPagePayloadBudget leaves framing headroom in the 64 MiB peer +// transport. Transfer checks include a full-key cursor; this is not a limit +// on ordinary writes, which retain their existing admission semantics. +const MigrationPagePayloadBudget = 63 << 20 + +func MigrationVersionFitsTransfer(key, value []byte) bool { + const cursorAndFramingBytes = 128 + return len(key) <= (MigrationPagePayloadBudget-cursorAndFramingBytes)/2 && + len(value) <= MigrationPagePayloadBudget-cursorAndFramingBytes-2*len(key) +} + const ( exportCursorTagEmitted byte = iota exportCursorTagScanned @@ -22,7 +33,9 @@ const ( migrationMetadataVersion = 1 migrationAckPrefix = "!migstage|ack|" + migrationReadyPrefix = "!migstage|ready|" migrationUint64Bytes = 8 + migrationAckKeyIDBytes = 2 * migrationUint64Bytes exportVersionSizeOverhead = 24 defaultSparseExportMaxScannedBytes = 1 << 20 ) @@ -203,10 +216,20 @@ func exportUsesSparseScanBudget(opts ExportVersionsOptions) bool { opts.EndKey != nil } +func migrationReadyKey(jobID uint64) []byte { + key := make([]byte, len(migrationReadyPrefix)+migrationUint64Bytes) + copy(key, migrationReadyPrefix) + binary.BigEndian.PutUint64(key[len(migrationReadyPrefix):], jobID) + return key +} + func isMigrationMetadataKey(rawKey []byte) bool { - return bytes.Equal(rawKey, migrationAckMetaKeyBytes) || + return IsMigrationOwnershipPartitionMetadataKey(rawKey) || bytes.Equal(rawKey, migrationOwnershipMetaKeyBytes) || + bytes.Equal(rawKey, migrationRetiredMetaKeyBytes) || + bytes.Equal(rawKey, migrationAckMetaKeyBytes) || bytes.Equal(rawKey, migrationHLCFloorMetaKeyBytes) || - bytes.Equal(rawKey, migrationPromoteMetaKeyBytes) + bytes.Equal(rawKey, migrationPromoteMetaKeyBytes) || + (len(rawKey) == len(migrationReadyPrefix)+migrationUint64Bytes && bytes.HasPrefix(rawKey, []byte(migrationReadyPrefix))) } func encodeMigrationImportAcks(acks map[migrationAckID]migrationImportAck) []byte { @@ -367,6 +390,12 @@ func decodeMigrationPromotionStates(data []byte) (map[uint64]PromotionState, boo } func validateImportVersion(version MVCCVersion) error { + if IsMigrationOwnershipMetadataKey(version.Key) || IsMigrationOwnershipPartitionMetadataKey(version.Key) { + return errors.Wrap(ErrInvalidImportVersion, "migration cannot import ownership metadata") + } + if bytes.Equal(version.Key, migrationRetiredMetaKeyBytes) { + return errors.Wrap(ErrInvalidImportVersion, "migration cannot import retirement metadata") + } if version.CommitTS == 0 { return errors.Wrap(ErrInvalidImportVersion, "migration import version has zero commit_ts") } @@ -431,7 +460,14 @@ func (s *mvccStore) ExportVersions(ctx context.Context, opts ExportVersionsOptio s.mtx.RLock() defer s.mtx.RUnlock() + if err := s.checkExportReadTSLocked(opts); err != nil { + return ExportVersionsResult{}, err + } + + return s.exportVersionsLocked(ctx, opts, pos) +} +func (s *mvccStore) exportVersionsLocked(ctx context.Context, opts ExportVersionsOptions, pos exportCursorPosition) (ExportVersionsResult, error) { result := newExportVersionsResult(opts.MaxVersions) it := s.tree.Iterator() if !s.seekMemoryExportStart(&it, opts.StartKey, pos) { @@ -462,6 +498,20 @@ func (s *mvccStore) ExportVersions(ctx context.Context, opts ExportVersionsOptio return result, nil } +func (s *mvccStore) checkExportReadTSLocked(opts ExportVersionsOptions) error { + if readTSCompacted(exportRetentionReadTS(opts), s.minRetainedTS) { + return ErrReadTSCompacted + } + return nil +} + +func exportRetentionReadTS(opts ExportVersionsOptions) uint64 { + if opts.ReadTS != 0 { + return opts.ReadTS + } + return opts.MaxCommitTSInclusive +} + var errExportReachedEnd = errors.New("export reached end") var errExportChunkFull = errors.New("export chunk full") @@ -649,6 +699,9 @@ func exportMemoryVersionsForKey( func (s *mvccStore) ImportVersions(_ context.Context, opts ImportVersionsOptions) (ImportVersionsResult, error) { s.mtx.Lock() defer s.mtx.Unlock() + if _, retired := s.migrationRetired[opts.JobID]; retired { + return ImportVersionsResult{}, errors.WithStack(ErrMigrationRetired) + } id := migrationAckID{jobID: opts.JobID, bracketID: opts.BracketID} existing, hasExisting := s.migrationAcks[id] @@ -694,6 +747,20 @@ func (s *mvccStore) MigrationHLCFloor(_ context.Context, jobID uint64) (uint64, return s.migrationHLCFloors[jobID], nil } +func (s *mvccStore) MigrationImportMetadataPresent(_ context.Context, jobID uint64) (bool, error) { + s.mtx.RLock() + defer s.mtx.RUnlock() + if s.migrationHLCFloors[jobID] != 0 { + return true, nil + } + for id := range s.migrationAcks { + if id.jobID == jobID { + return true, nil + } + } + return false, nil +} + func (s *mvccStore) RetireMigration(ctx context.Context, jobID uint64) error { if err := ctx.Err(); err != nil { return errors.WithStack(err) diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index c707ff623..e023f9df1 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -6,7 +6,9 @@ import ( "encoding/binary" "math" "os" + "runtime" "testing" + "time" "github.com/bootjp/elastickv/internal/encryption" "github.com/cockroachdb/pebble/v2" @@ -1583,3 +1585,348 @@ func TestExportVersionsEmitsSingleOversizedRow(t *testing.T) { require.Equal(t, huge, got.Versions[0].Value) }) } + +func TestExportVersionsReadTSPreservesCompactionErrors(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("v10"), 10, 0)) + retention, ok := st.(RetentionController) + require.True(t, ok) + retention.SetMinRetainedTS(20) + + _, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("k"), + EndKey: []byte("l"), + MaxCommitTSInclusive: 15, + ReadTS: 15, + MaxVersions: 10, + }) + require.ErrorIs(t, err, ErrReadTSCompacted) + + _, err = st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("k"), + EndKey: []byte("l"), + MaxCommitTSInclusive: 15, + MaxVersions: 10, + }) + require.ErrorIs(t, err, ErrReadTSCompacted) + + result, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("k"), + EndKey: []byte("l"), + MaxCommitTSInclusive: 25, + MaxVersions: 10, + }) + require.NoError(t, err) + require.Len(t, result.Versions, 1) + }) +} + +func TestMigrationImportMetadataPresentClearsWithMigrationState(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + reader, ok := st.(MigrationImportMetadataReader) + require.True(t, ok) + cleaner, ok := st.(MigrationCleaner) + require.True(t, ok) + + present, err := reader.MigrationImportMetadataPresent(ctx, 1) + require.NoError(t, err) + require.False(t, present) + _, err = st.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 1, + BracketID: 2, + BatchSeq: 1, + Cursor: []byte("ack-only"), + }) + require.NoError(t, err) + present, err = reader.MigrationImportMetadataPresent(ctx, 1) + require.NoError(t, err) + require.True(t, present) + present, err = reader.MigrationImportMetadataPresent(ctx, 2) + require.NoError(t, err) + require.False(t, present) + + require.NoError(t, cleaner.ClearMigrationState(ctx, 1, 0)) + present, err = reader.MigrationImportMetadataPresent(ctx, 1) + require.NoError(t, err) + require.False(t, present) + }) +} + +func TestTargetStagedReadinessStatePersistsAndIsCloned(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + reader, ok := st.(MigrationTargetReadinessReader) + require.True(t, ok) + + state := TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 12, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + } + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, state)) + + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{state}, states) + + states[0].RouteStart[0] = 'x' + states, err = reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []byte("a"), states[0].RouteStart) + + updated := state + updated.MinWriteTSExclusive = 101 + updated.RouteStart = []byte("b") + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, updated)) + + states, err = reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{updated}, states) + + exported, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("!"), + EndKey: []byte("~"), + MaxVersions: 100, + }) + require.NoError(t, err) + require.Empty(t, exported.Versions) + }) +} + +func TestTargetStagedReadinessRejectsArmedStateWithoutWriteFloor(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + + err := writer.ApplyTargetStagedReadiness(context.Background(), TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 12, + MigrationJobID: 9, + Armed: true, + }) + require.ErrorContains(t, err, "min_write_ts_exclusive") + }) +} + +func TestPebbleTargetStagedReadinessPersistsAcrossReopen(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-ready-persist-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + + st, err := NewPebbleStore(dir) + require.NoError(t, err) + state := TargetStagedReadinessState{ + JobID: 11, + RouteStart: []byte("m"), + RouteEnd: nil, + ExpectedCutoverVersion: 22, + MigrationJobID: 11, + MinWriteTSExclusive: 333, + Armed: true, + } + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, state)) + require.NoError(t, st.Close()) + + reopened, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + reader, ok := reopened.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{state}, states) +} + +func TestPebbleTargetStagedReadinessIgnoresNonRecordPrefixKeys(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-ready-prefix-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + + st, err := NewPebbleStore(dir) + require.NoError(t, err) + state := TargetStagedReadinessState{ + JobID: 11, + RouteStart: []byte("m"), + RouteEnd: nil, + ExpectedCutoverVersion: 22, + MigrationJobID: 11, + MinWriteTSExclusive: 333, + Armed: true, + } + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, state)) + pebbleStore, ok := st.(*pebbleStore) + require.True(t, ok) + junkKey := append([]byte(migrationReadyPrefix), []byte("side-record")...) + require.NoError(t, pebbleStore.db.Set(junkKey, []byte("not-readiness-state"), pebbleStore.directApplyWriteOpts())) + require.NoError(t, st.Close()) + + reopened, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + reader, ok := reopened.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{state}, states) +} + +func TestCleanupVersionsRemovesOnlySelectedCommittedVersions(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("v1"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("v2"), 20, 0)) + require.NoError(t, st.PutAt(ctx, []byte("b"), []byte("keep"), 10, 0)) + + cleaner, ok := st.(MigrationCleaner) + require.True(t, ok) + first, err := cleaner.CleanupVersions(ctx, CleanupVersionsOptions{ + JobID: 1, + StartKey: []byte("a"), + EndKey: []byte("b"), + MaxCommitTS: 10, + MaxVersions: 1, + MaxBytes: 1 << 20, + }) + require.NoError(t, err) + require.Equal(t, uint64(1), first.DeletedRows) + + value, err := st.GetAt(ctx, []byte("a"), 20) + require.NoError(t, err) + require.Equal(t, []byte("v2"), value) + _, err = st.GetAt(ctx, []byte("a"), 10) + require.ErrorIs(t, err, ErrKeyNotFound) + value, err = st.GetAt(ctx, []byte("b"), 20) + require.NoError(t, err) + require.Equal(t, []byte("keep"), value) + }) +} + +func TestClearMigrationStateRemovesPerJobProofs(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + _, err := st.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 1, + BatchSeq: 1, + Versions: []MVCCVersion{{ + Key: []byte("!dist|migstage|row"), + CommitTS: 50, + Value: []byte("v"), + }}, + }) + require.NoError(t, err) + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, TargetStagedReadinessState{ + JobID: 7, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 7, + MinWriteTSExclusive: 50, + Armed: true, + })) + + cleaner, ok := st.(MigrationCleaner) + require.True(t, ok) + require.NoError(t, cleaner.ClearMigrationState(ctx, 7, 0)) + floor, err := st.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Zero(t, floor) + reader, ok := st.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + for _, state := range states { + require.NotEqual(t, uint64(7), state.JobID) + } + }) +} + +func TestPebbleMigrationApplyPathsAcquireApplyMuBeforeDBMu(t *testing.T) { + dir := t.TempDir() + st, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + pebble, ok := st.(*pebbleStore) + require.True(t, ok) + + tests := []struct { + name string + run func() error + }{ + { + name: "cleanup versions", + run: func() error { + _, cleanupErr := pebble.CleanupVersions(context.Background(), CleanupVersionsOptions{ + MaxVersions: 1, + }) + return cleanupErr + }, + }, + { + name: "clear migration state", + run: func() error { + return pebble.ClearMigrationState(context.Background(), 1, 0) + }, + }, + { + name: "apply target readiness", + run: func() error { + return pebble.ApplyTargetStagedReadinessAt(context.Background(), TargetStagedReadinessState{ + JobID: 2, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 2, + MinWriteTSExclusive: 10, + Armed: true, + }, 1) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pebble.applyMu.Lock() + started := make(chan struct{}) + done := make(chan error, 1) + go func() { + close(started) + done <- tt.run() + }() + <-started + + deadline := time.Now().Add(100 * time.Millisecond) + dbRemainedAvailable := true + for time.Now().Before(deadline) { + runtime.Gosched() + if !pebble.dbMu.TryLock() { + dbRemainedAvailable = false + break + } + pebble.dbMu.Unlock() + } + pebble.applyMu.Unlock() + + require.True(t, dbRemainedAvailable, "operation held dbMu while waiting for applyMu") + require.NoError(t, <-done) + }) + } +} diff --git a/store/mvcc_store.go b/store/mvcc_store.go index 71943d0f9..5b2bc1451 100644 --- a/store/mvcc_store.go +++ b/store/mvcc_store.go @@ -28,6 +28,10 @@ const ( mvccSnapshotVersionV1 = uint32(1) mvccSnapshotVersionV2 = uint32(2) mvccSnapshotVersionV3 = uint32(3) + mvccSnapshotVersionV4 = uint32(4) + mvccSnapshotVersionV5 = uint32(5) + mvccSnapshotVersionV6 = uint32(6) + mvccSnapshotVersionV7 = uint32(7) maxSnapshotKeySize = 1 << 20 // 1 MiB per logical key // maxSnapshotInternalKeyEnvelope reserves snapshot headroom for internal // stored-key envelopes such as migration staging. Logical source keys keep @@ -86,6 +90,14 @@ type mvccStore struct { migrationAcks map[migrationAckID]migrationImportAck migrationHLCFloors map[uint64]uint64 migrationPromotions map[uint64]PromotionState + migrationReadiness map[uint64]TargetStagedReadinessState + migrationRetired map[uint64]struct{} + migrationOwnership []MigrationOwnershipState + migrationPartitions MigrationOwnershipPartitionState + migrationAdmission migrationServingAdmissionIndex + // migrationReadinessCache is the snapshot-order projection of + // migrationReadiness; it is what the V4 trailer serialises. + migrationReadinessCache []TargetStagedReadinessState // writeConflicts mirrors the per-(kind, key_prefix) counter from // the pebble-backed store so the in-memory implementation shows up // in the same Prometheus series (even if the counts are usually @@ -135,6 +147,8 @@ func NewMVCCStore(opts ...MVCCStoreOption) MVCCStore { migrationAcks: make(map[migrationAckID]migrationImportAck), migrationHLCFloors: make(map[uint64]uint64), migrationPromotions: make(map[uint64]PromotionState), + migrationReadiness: make(map[uint64]TargetStagedReadinessState), + migrationRetired: make(map[uint64]struct{}), writeConflicts: newWriteConflictCounter(), } for _, opt := range opts { @@ -909,6 +923,10 @@ func (s *mvccStore) Snapshot() (Snapshot, error) { } func (s *mvccStore) Restore(r io.Reader) error { + return s.restoreSnapshot(r, nil) +} + +func (s *mvccStore) restoreSnapshot(r io.Reader, validate func(*treemap.Map, mvccSnapshotMigrationMetadata) error) error { br := bufio.NewReader(r) streaming, err := isStreamingMVCCSnapshot(br) if err != nil { @@ -917,7 +935,7 @@ func (s *mvccStore) Restore(r io.Reader) error { if !streaming { return errors.New("unrecognized snapshot format: unknown magic header") } - return s.restoreStreamingSnapshot(br) + return s.restoreStreamingSnapshot(br, validate) } func isStreamingMVCCSnapshot(r *bufio.Reader) (bool, error) { @@ -939,7 +957,14 @@ func (s *mvccStore) writeSnapshotFile(f *os.File) error { s.mtx.RLock() defer s.mtx.RUnlock() - version := mvccSnapshotVersionForState(s.migrationAcks, s.migrationHLCFloors, s.migrationPromotions) + version := mvccSnapshotVersionForState( + s.migrationAcks, s.migrationHLCFloors, s.migrationPromotions, s.migrationReadinessCache, s.migrationRetired) + if len(s.migrationOwnership) > 0 { + version = mvccSnapshotVersionV6 + } + if partitionHasEvidence(s.migrationPartitions) { + version = mvccSnapshotVersionV7 + } checksumOffset, err := writeMVCCSnapshotHeader(f, version) if err != nil { return err @@ -964,8 +989,14 @@ func mvccSnapshotVersionForState( acks map[migrationAckID]migrationImportAck, floors map[uint64]uint64, promotions map[uint64]PromotionState, + readiness []TargetStagedReadinessState, + retired ...map[uint64]struct{}, ) uint32 { switch { + case len(retired) > 0 && len(retired[0]) > 0: + return mvccSnapshotVersionV5 + case len(readiness) > 0: + return mvccSnapshotVersionV4 case len(promotions) > 0: return mvccSnapshotVersionV3 case len(acks) > 0 || len(floors) > 0: @@ -1005,9 +1036,21 @@ func (s *mvccStore) writeSnapshotBodyLocked(f *os.File, version uint32) (uint32, s.migrationAcks, s.migrationHLCFloors, s.migrationPromotions, + s.migrationReadinessCache, + s.migrationRetired, + s.migrationOwnership, ); err != nil { return 0, err } + if version >= mvccSnapshotVersionV7 { + data, err := encodeMigrationOwnershipPartitions(s.migrationPartitions) + if err != nil { + return 0, err + } + if err := writeMVCCSnapshotBytes(w, data); err != nil { + return 0, err + } + } iter := s.tree.Iterator() for iter.Next() { key, ok := iter.Key().([]byte) @@ -1036,6 +1079,9 @@ func writeMVCCSnapshotMetadata( migrationAcks map[migrationAckID]migrationImportAck, migrationHLCFloors map[uint64]uint64, migrationPromotions map[uint64]PromotionState, + migrationReadiness []TargetStagedReadinessState, + migrationRetired map[uint64]struct{}, + migrationOwnership ...[]MigrationOwnershipState, ) error { if err := binary.Write(w, binary.LittleEndian, lastCommitTS); err != nil { return errors.WithStack(err) @@ -1058,6 +1104,49 @@ func writeMVCCSnapshotMetadata( if err := writeMVCCSnapshotBytes(w, encodeMigrationPromotionStates(migrationPromotions)); err != nil { return err } + if version < mvccSnapshotVersionV4 { + return nil + } + if err := writeMVCCSnapshotReadinessStates(w, migrationReadiness); err != nil { + return err + } + return writeMVCCSnapshotServingMetadata(w, version, migrationRetired, migrationOwnership...) +} + +func writeMVCCSnapshotServingMetadata(w io.Writer, version uint32, migrationRetired map[uint64]struct{}, migrationOwnership ...[]MigrationOwnershipState) error { + if version < mvccSnapshotVersionV5 { + return nil + } + if err := writeMVCCSnapshotBytes(w, encodeMigrationRetiredJobs(migrationRetired)); err != nil { + return err + } + if version < mvccSnapshotVersionV6 { + return nil + } + var ownership []MigrationOwnershipState + if len(migrationOwnership) > 0 { + ownership = migrationOwnership[0] + } + data, err := encodeMigrationOwnership(ownership) + if err != nil { + return err + } + return writeMVCCSnapshotBytes(w, data) +} + +// writeMVCCSnapshotReadinessStates emits the V4 trailer: a count followed by +// one length-prefixed encoded state each. The states are stored as a sequence +// rather than one opaque blob so a corrupt entry is localised to its own +// record instead of failing the whole map decode. +func writeMVCCSnapshotReadinessStates(w io.Writer, states []TargetStagedReadinessState) error { + if err := binary.Write(w, binary.LittleEndian, uint64(len(states))); err != nil { + return errors.WithStack(err) + } + for _, state := range states { + if err := writeMVCCSnapshotBytes(w, encodeTargetStagedReadinessState(state)); err != nil { + return err + } + } return nil } @@ -1161,28 +1250,39 @@ func mvccSnapshotTombstoneByte(tombstone bool) byte { return 0 } -func (s *mvccStore) restoreStreamingSnapshot(r io.Reader) error { +func (s *mvccStore) restoreStreamingSnapshot(r io.Reader, validate func(*treemap.Map, mvccSnapshotMigrationMetadata) error) error { version, expected, err := readMVCCSnapshotHeader(r) if err != nil { return err } - tree, lastCommitTS, minRetainedTS, migrationAcks, migrationHLCFloors, migrationPromotions, actual, err := restoreStreamingMVCCSnapshotBody(r, version) + tree, lastCommitTS, minRetainedTS, meta, actual, err := restoreStreamingMVCCSnapshotBody(r, version) if err != nil { return err } if actual != expected { return errors.WithStack(ErrInvalidChecksum) } - s.mtx.Lock() defer s.mtx.Unlock() + if validate != nil { + if err := validate(tree, meta); err != nil { + return err + } + } + s.tree = tree s.lastCommitTS = lastCommitTS s.minRetainedTS = minRetainedTS - s.migrationAcks = migrationAcks - s.migrationHLCFloors = migrationHLCFloors - s.migrationPromotions = migrationPromotions + s.migrationAcks = meta.acks + s.migrationHLCFloors = meta.floors + s.migrationPromotions = meta.promotions + s.migrationReadiness = targetReadinessStateMap(meta.readiness) + s.migrationRetired = meta.retired + s.migrationOwnership = meta.ownership + s.migrationPartitions = meta.partitions + s.migrationAdmission = buildMigrationServingAdmission(meta.ownership, meta.partitions) + s.migrationReadinessCache = cloneTargetStagedReadinessStates(meta.readiness) return nil } @@ -1199,7 +1299,7 @@ func readMVCCSnapshotHeader(r io.Reader) (uint32, uint32, error) { if err := binary.Read(r, binary.LittleEndian, &version); err != nil { return 0, 0, errors.WithStack(err) } - if version != mvccSnapshotVersionV1 && version != mvccSnapshotVersionV2 && version != mvccSnapshotVersionV3 { + if version < mvccSnapshotVersionV1 || version > mvccSnapshotVersionV7 { return 0, 0, errors.WithStack(errors.Newf("unsupported mvcc snapshot version %d", version)) } @@ -1213,68 +1313,183 @@ func readMVCCSnapshotHeader(r io.Reader) (uint32, uint32, error) { func restoreStreamingMVCCSnapshotBody( r io.Reader, version uint32, -) (*treemap.Map, uint64, uint64, map[migrationAckID]migrationImportAck, map[uint64]uint64, map[uint64]PromotionState, uint32, error) { +) (*treemap.Map, uint64, uint64, mvccSnapshotMigrationMetadata, uint32, error) { hash := crc32.NewIEEE() body := io.TeeReader(r, hash) - lastCommitTS, minRetainedTS, migrationAcks, migrationHLCFloors, migrationPromotions, err := readMVCCSnapshotMetadata(body, version) + lastCommitTS, minRetainedTS, meta, err := readMVCCSnapshotMetadata(body, version) if err != nil { - return nil, 0, 0, nil, nil, nil, 0, err + return nil, 0, 0, mvccSnapshotMigrationMetadata{}, 0, err } tree, err := readMVCCSnapshotTree(body) if err != nil { - return nil, 0, 0, nil, nil, nil, 0, err + return nil, 0, 0, mvccSnapshotMigrationMetadata{}, 0, err } - return tree, lastCommitTS, minRetainedTS, migrationAcks, migrationHLCFloors, migrationPromotions, hash.Sum32(), nil + return tree, lastCommitTS, minRetainedTS, meta, hash.Sum32(), nil } func readMVCCSnapshotMetadata( r io.Reader, version uint32, -) (uint64, uint64, map[migrationAckID]migrationImportAck, map[uint64]uint64, map[uint64]PromotionState, error) { +) (uint64, uint64, mvccSnapshotMigrationMetadata, error) { var lastCommitTS uint64 if err := binary.Read(r, binary.LittleEndian, &lastCommitTS); err != nil { - return 0, 0, nil, nil, nil, errors.WithStack(err) + return 0, 0, mvccSnapshotMigrationMetadata{}, errors.WithStack(err) } var minRetainedTS uint64 if err := binary.Read(r, binary.LittleEndian, &minRetainedTS); err != nil { - return 0, 0, nil, nil, nil, errors.WithStack(err) + return 0, 0, mvccSnapshotMigrationMetadata{}, errors.WithStack(err) } if version == mvccSnapshotVersionV1 { - return lastCommitTS, - minRetainedTS, - make(map[migrationAckID]migrationImportAck), - make(map[uint64]uint64), - make(map[uint64]PromotionState), - nil + return lastCommitTS, minRetainedTS, emptyMVCCSnapshotMigrationMetadata(), nil } + migrationAcks, migrationHLCFloors, err := readMVCCSnapshotImportMetadata(r) + if err != nil { + return 0, 0, mvccSnapshotMigrationMetadata{}, err + } + + migrationPromotions, err := readMVCCSnapshotPromotionMetadata(r, version) + if err != nil { + return 0, 0, mvccSnapshotMigrationMetadata{}, err + } + + migrationReadiness, err := readMVCCSnapshotReadinessStates(r, version) + if err != nil { + return 0, 0, mvccSnapshotMigrationMetadata{}, err + } + retired, ownership, err := readMVCCSnapshotServingMetadata(r, version) + if err != nil { + return 0, 0, mvccSnapshotMigrationMetadata{}, err + } + partitions, err := readSnapshotPartitions(r, version) + if err != nil { + return 0, 0, mvccSnapshotMigrationMetadata{}, err + } + return lastCommitTS, minRetainedTS, mvccSnapshotMigrationMetadata{ + acks: migrationAcks, + floors: migrationHLCFloors, + promotions: migrationPromotions, + readiness: migrationReadiness, + retired: retired, + ownership: ownership, + partitions: partitions, + }, nil +} + +func readMVCCSnapshotImportMetadata(r io.Reader) (map[migrationAckID]migrationImportAck, map[uint64]uint64, error) { ackData, err := readMVCCSnapshotBytes(r, "snapshot migration acks") if err != nil { - return 0, 0, nil, nil, nil, err + return nil, nil, err } - migrationAcks, ok := decodeMigrationImportAcks(ackData) + acks, ok := decodeMigrationImportAcks(ackData) if !ok { - return 0, 0, nil, nil, nil, errors.New("invalid snapshot migration acks") + return nil, nil, errors.New("invalid snapshot migration acks") } - floorData, err := readMVCCSnapshotBytes(r, "snapshot migration hlc floors") if err != nil { - return 0, 0, nil, nil, nil, err + return nil, nil, err } - migrationHLCFloors, ok := decodeMigrationHLCFloors(floorData) + floors, ok := decodeMigrationHLCFloors(floorData) if !ok { - return 0, 0, nil, nil, nil, errors.New("invalid snapshot migration hlc floors") + return nil, nil, errors.New("invalid snapshot migration hlc floors") } + return acks, floors, nil +} - migrationPromotions, err := readMVCCSnapshotPromotionMetadata(r, version) - if err != nil { - return 0, 0, nil, nil, nil, err +func readMVCCSnapshotServingMetadata(r io.Reader, version uint32) (map[uint64]struct{}, []MigrationOwnershipState, error) { + retired := make(map[uint64]struct{}) + if version >= mvccSnapshotVersionV5 { + data, readErr := readMVCCSnapshotBytes(r, "snapshot migration retirement") + if readErr != nil { + return nil, nil, readErr + } + var valid bool + retired, valid = decodeMigrationRetiredJobs(data) + if !valid { + return nil, nil, errors.New("invalid snapshot migration retirement") + } } - return lastCommitTS, minRetainedTS, migrationAcks, migrationHLCFloors, migrationPromotions, nil + var ownership []MigrationOwnershipState + if version >= mvccSnapshotVersionV6 { + data, readErr := readMVCCSnapshotBytes(r, "snapshot migration ownership") + if readErr != nil { + return nil, nil, readErr + } + var err error + ownership, err = decodeMigrationOwnership(data) + if err != nil { + return nil, nil, err + } + } + return retired, ownership, nil +} + +// mvccSnapshotMigrationMetadata is the version-layered trailer that follows +// (lastCommitTS, minRetainedTS). Each version adds exactly one field to its +// predecessor: V2 adds acks+floors, V3 adds promotions, V4 adds readiness, +// and V5 adds permanent per-job retirement witnesses. +// Bundling them keeps the reader/restore signatures from growing a return +// value per layer. +type mvccSnapshotMigrationMetadata struct { + acks map[migrationAckID]migrationImportAck + floors map[uint64]uint64 + promotions map[uint64]PromotionState + readiness []TargetStagedReadinessState + retired map[uint64]struct{} + ownership []MigrationOwnershipState + partitions MigrationOwnershipPartitionState +} + +func emptyMVCCSnapshotMigrationMetadata() mvccSnapshotMigrationMetadata { + return mvccSnapshotMigrationMetadata{ + acks: make(map[migrationAckID]migrationImportAck), + floors: make(map[uint64]uint64), + promotions: make(map[uint64]PromotionState), + retired: make(map[uint64]struct{}), + } +} + +// readMVCCSnapshotReadinessStates mirrors writeMVCCSnapshotReadinessStates. +// Pre-V4 snapshots carry no trailer, so the target rebuilds its readiness view +// from the migration catalog instead of the snapshot. +func readMVCCSnapshotReadinessStates(r io.Reader, version uint32) ([]TargetStagedReadinessState, error) { + if version < mvccSnapshotVersionV4 { + return nil, nil + } + var count uint64 + if err := binary.Read(r, binary.LittleEndian, &count); err != nil { + return nil, errors.WithStack(err) + } + if count > maxSnapshotVersionCount { + return nil, errors.Wrapf(ErrSnapshotVersionCountTooLarge, "readiness states %d > %d", count, maxSnapshotVersionCount) + } + states := make([]TargetStagedReadinessState, 0, count) + for range count { + encoded, err := readMVCCSnapshotBytes(r, "snapshot migration readiness state") + if err != nil { + return nil, err + } + state, ok := decodeTargetStagedReadinessState(encoded) + if !ok { + return nil, errors.New("invalid snapshot migration readiness state") + } + states = append(states, state) + } + sortTargetStagedReadinessStates(states) + return states, nil +} + +func targetReadinessStateMap(states []TargetStagedReadinessState) map[uint64]TargetStagedReadinessState { + out := make(map[uint64]TargetStagedReadinessState, len(states)) + for _, state := range states { + cloned := cloneTargetStagedReadinessState(state) + out[cloned.JobID] = cloned + } + return out } func readMVCCSnapshotPromotionMetadata(r io.Reader, version uint32) (map[uint64]PromotionState, error) { diff --git a/store/mvcc_store_snapshot_test.go b/store/mvcc_store_snapshot_test.go index 69e59807c..d4d84fe0a 100644 --- a/store/mvcc_store_snapshot_test.go +++ b/store/mvcc_store_snapshot_test.go @@ -3,6 +3,7 @@ package store import ( "bytes" "context" + "encoding/binary" "testing" "github.com/stretchr/testify/require" @@ -325,3 +326,120 @@ func snapshotBytes(t *testing.T, snap Snapshot) []byte { require.NoError(t, err) return buf.Bytes() } + +func TestMVCCStore_SnapshotRestorePreservesTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + src := newTestMVCCStore(t) + + stale := TargetStagedReadinessState{ + JobID: 8, + RouteStart: []byte("old"), + RouteEnd: []byte("oldz"), + ExpectedCutoverVersion: 1, + MigrationJobID: 8, + MinWriteTSExclusive: 80, + Armed: true, + } + require.NoError(t, src.ApplyTargetStagedReadiness(ctx, stale)) + + snapState := TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 12, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + } + require.NoError(t, src.ApplyTargetStagedReadiness(ctx, snapState)) + + snap, err := src.Snapshot() + require.NoError(t, err) + defer snap.Close() + raw := snapshotBytes(t, snap) + require.Equal(t, mvccSnapshotVersionV4, binary.LittleEndian.Uint32(raw[len(mvccSnapshotMagic):])) + + dst := newTestMVCCStore(t) + require.NoError(t, dst.ApplyTargetStagedReadiness(ctx, TargetStagedReadinessState{ + JobID: 77, + RouteStart: []byte("dst-only"), + RouteEnd: []byte("dst-z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 77, + MinWriteTSExclusive: 700, + Armed: true, + })) + + require.NoError(t, dst.Restore(bytes.NewReader(raw))) + states, err := dst.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{stale, snapState}, states) + + states[0].RouteStart[0] = 'x' + states, err = dst.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []byte("old"), states[0].RouteStart) +} + +func TestMVCCStore_SnapshotRestorePreservesMigrationImportMetadata(t *testing.T) { + t.Parallel() + + ctx := context.Background() + src := newTestMVCCStore(t) + res, err := src.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 1, + Cursor: []byte("c1"), + Versions: []MVCCVersion{{Key: []byte("snapshotted"), CommitTS: 50, Value: []byte("v50")}}, + }) + require.NoError(t, err) + require.False(t, res.Duplicate) + + snap, err := src.Snapshot() + require.NoError(t, err) + defer snap.Close() + raw := snapshotBytes(t, snap) + require.Equal(t, mvccSnapshotVersionV2, binary.LittleEndian.Uint32(raw[len(mvccSnapshotMagic):])) + + dst := newTestMVCCStore(t) + require.NoError(t, dst.Restore(bytes.NewReader(raw))) + val, err := dst.GetAt(ctx, []byte("snapshotted"), 50) + require.NoError(t, err) + require.Equal(t, []byte("v50"), val) + floor, err := dst.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Equal(t, uint64(50), floor) + present, err := dst.MigrationImportMetadataPresent(ctx, 7) + require.NoError(t, err) + require.True(t, present) + + res, err = dst.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 1, + Cursor: []byte("fresh"), + Versions: []MVCCVersion{{Key: []byte("fresh"), CommitTS: 60, Value: []byte("v60")}}, + }) + require.NoError(t, err) + require.True(t, res.Duplicate) + require.Equal(t, []byte("c1"), res.AckedCursor) + _, err = dst.GetAt(ctx, []byte("fresh"), 60) + require.ErrorIs(t, err, ErrKeyNotFound) + + res, err = dst.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 2, + Cursor: []byte("c2"), + Versions: []MVCCVersion{{Key: []byte("fresh"), CommitTS: 60, Value: []byte("v60")}}, + }) + require.NoError(t, err) + require.False(t, res.Duplicate) + require.Equal(t, []byte("c2"), res.AckedCursor) + val, err = dst.GetAt(ctx, []byte("fresh"), 60) + require.NoError(t, err) + require.Equal(t, []byte("v60"), val) +} diff --git a/store/snapshot_external.go b/store/snapshot_external.go new file mode 100644 index 000000000..7ccff451d --- /dev/null +++ b/store/snapshot_external.go @@ -0,0 +1,189 @@ +package store + +import ( + "bytes" + "context" + "io" + "os" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" + "github.com/emirpasic/gods/maps/treemap" +) + +// ExternalSnapshotRestorer rebases a snapshot without importing migration +// evidence from a different Raft lineage. Restore remains the same-lineage API. +type ExternalSnapshotRestorer interface { + RestoreExternal(context.Context, io.Reader) error +} + +var ErrExternalSnapshotMigrationState = errors.New("external snapshot contains migration state") + +func (s *mvccStore) RestoreExternal(ctx context.Context, r io.Reader) error { + return s.restoreSnapshot(&externalSnapshotReader{ctx: ctx, r: r}, func(tree *treemap.Map, meta mvccSnapshotMigrationMetadata) error { + return validateExternalMVCCTree(ctx, tree, meta) + }) +} + +func (s *pebbleStore) RestoreExternal(ctx context.Context, r io.Reader) error { + return s.restoreSnapshot(&externalSnapshotReader{ctx: ctx, r: r}, func(dir string) error { + return validateExternalPebbleDB(ctx, dir) + }) +} + +type externalSnapshotReader struct { + ctx context.Context + r io.Reader +} + +func (r *externalSnapshotReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, errors.WithStack(err) + } + n, err := r.r.Read(p) + return n, errors.WithStack(err) +} + +func validateRestoreTempDB(dir string, validate func(string) error) error { + if validate == nil { + return nil + } + if err := validate(dir); err != nil { + _ = os.RemoveAll(dir) + return err + } + return nil +} + +func validateExternalMVCCTree(ctx context.Context, tree *treemap.Map, meta mvccSnapshotMigrationMetadata) error { + if len(meta.acks)+len(meta.floors)+len(meta.promotions)+len(meta.readiness)+len(meta.retired)+len(meta.ownership) != 0 || partitionHasEvidence(meta.partitions) { + return errors.WithStack(ErrExternalSnapshotMigrationState) + } + iter := tree.Iterator() + for iter.Next() { + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + key, _ := iter.Key().([]byte) + if err := validateExternalTypedMigrationKey(key); err != nil { + return err + } + } + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + tree.Remove(MigrationConfigurationIndexKey()) + return nil +} + +func validateExternalPebbleDB(ctx context.Context, dir string) (retErr error) { + opts, cache := defaultPebbleOptionsWithCache(false) + db, err := pebble.Open(dir, opts) + if err != nil { + cache.Unref() + return errors.WithStack(err) + } + defer cache.Unref() + defer func() { + retErr = errors.Join(retErr, errors.WithStack(db.Close()), errors.WithStack(ctx.Err())) + }() + if err := validateExternalPebbleRows(ctx, db); err != nil { + return err + } + return sanitizeExternalConfigurationEpoch(ctx, db) +} + +func validateExternalPebbleRows(ctx context.Context, db *pebble.DB) error { + iter, err := db.NewIter(nil) + if err != nil { + return errors.WithStack(err) + } + defer iter.Close() + for valid := iter.First(); valid; valid = iter.Next() { + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + if err := validateExternalPebbleRow(iter.Key(), iter.Value()); err != nil { + return err + } + } + return errors.Join(errors.WithStack(iter.Error()), errors.WithStack(ctx.Err())) +} + +func validateExternalPebbleRow(key, value []byte) error { + if isMigrationMetadataKey(key) { + return validateExternalMigrationRow(key, value) + } + if isPebbleMetaKey(key) { + return nil + } + userKey, _ := decodeKeyView(key) + if userKey == nil { + return errors.New("invalid external snapshot MVCC key") + } + // Decoded MVCC keys are logical keys, not the raw metadata namespace. + // In particular, _migack/_mighlc/_migpromote remain ordinary user keys. + return validateExternalTypedMigrationKey(userKey) +} + +func validateExternalMigrationRow(key, value []byte) error { + switch { + case IsMigrationOwnershipPartitionMetadataKey(key): + return validateExternalPartitionRow(value) + case bytes.Equal(key, migrationOwnershipMetaKeyBytes): + states, err := decodeMigrationOwnership(value) + if err != nil { + return err + } + return externalMigrationCollection(len(states), true) + case bytes.Equal(key, migrationRetiredMetaKeyBytes): + states, ok := decodeMigrationRetiredJobs(value) + return externalMigrationCollection(len(states), ok) + case bytes.Equal(key, migrationAckMetaKeyBytes): + states, ok := decodeMigrationImportAcks(value) + return externalMigrationCollection(len(states), ok) + case bytes.Equal(key, migrationHLCFloorMetaKeyBytes): + states, ok := decodeMigrationHLCFloors(value) + return externalMigrationCollection(len(states), ok) + case bytes.Equal(key, migrationPromoteMetaKeyBytes): + states, ok := decodeMigrationPromotionStates(value) + return externalMigrationCollection(len(states), ok) + case isTargetReadinessRecordKey(key): + _, ok := decodeTargetStagedReadinessState(value) + return externalMigrationCollection(1, ok) + default: + return validateExternalTypedMigrationKey(key) + } +} + +func validateExternalPartitionRow(value []byte) error { + state, err := decodeMigrationOwnershipPartitions(value) + if err != nil { + return err + } + return externalMigrationCollection(len(state.Partitions)+len(state.Receipts), true) +} + +func externalMigrationCollection(size int, valid bool) error { + if !valid { + return errors.New("corrupt external snapshot migration metadata") + } + if size != 0 { + return errors.WithStack(ErrExternalSnapshotMigrationState) + } + return nil +} + +func validateExternalTypedMigrationKey(key []byte) error { + // Keep these logical-key guards aligned with distribution's migration-only + // control namespaces. Raw store metadata is decoded separately above. + for _, prefix := range []string{"!dist|migstage|", "!migstage|", "!migwrite|", "!migfence|", "!dist|meta|ownership_partition/"} { + if bytes.HasPrefix(key, []byte(prefix)) { + return errors.WithStack(ErrExternalSnapshotMigrationState) + } + } + return nil +} + +var _ ExternalSnapshotRestorer = (*mvccStore)(nil) +var _ ExternalSnapshotRestorer = (*pebbleStore)(nil) diff --git a/store/snapshot_external_configuration_test.go b/store/snapshot_external_configuration_test.go new file mode 100644 index 000000000..3ff0927d6 --- /dev/null +++ b/store/snapshot_external_configuration_test.go @@ -0,0 +1,58 @@ +package store + +import ( + "bytes" + "encoding/binary" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExternalSnapshotRebasesOnlyRaftLineageMetadata(t *testing.T) { + for _, format := range []string{"native", "sst"} { + t.Run(format, func(t *testing.T) { + st, err := NewPebbleStore(t.TempDir(), WithSSTIngestSnapshots(format == "sst")) + require.NoError(t, err) + source, ok := st.(*pebbleStore) + require.True(t, ok) + defer source.Close() + key := MigrationConfigurationIndexKey() + adjacent := append(bytes.Clone(key), []byte("/user")...) + require.NoError(t, source.PutAt(t.Context(), key, binary.BigEndian.AppendUint64(nil, 9000), 10, 0)) + require.NoError(t, source.PutAt(t.Context(), adjacent, []byte("preserved"), 20, 0)) + require.NoError(t, source.SetDurableAppliedIndex(9001)) + raw := externalTestSnapshot(t, source) + for _, external := range []bool{false, true} { + t.Run(map[bool]string{false: "raft", true: "external"}[external], func(t *testing.T) { + st, openErr := NewPebbleStore(t.TempDir()) + require.NoError(t, openErr) + destination, ok := st.(*pebbleStore) + require.True(t, ok) + defer destination.Close() + if external { + require.NoError(t, destination.RestoreExternal(t.Context(), bytes.NewReader(raw))) + } else { + require.NoError(t, destination.Restore(bytes.NewReader(raw))) + } + applied, present, readErr := destination.LastAppliedIndex() + require.NoError(t, readErr) + value, readErr := destination.GetAt(t.Context(), key, 30) + if external { + require.ErrorIs(t, readErr, ErrKeyNotFound) + require.False(t, present, "source Raft progress must not suppress destination applies") + require.Zero(t, applied) + } else { + require.NoError(t, readErr) + require.Equal(t, uint64(9000), binary.BigEndian.Uint64(value)) + require.True(t, present) + require.Equal(t, uint64(9001), applied) + } + value, readErr = destination.GetAt(t.Context(), adjacent, 30) + require.NoError(t, readErr) + require.Equal(t, []byte("preserved"), value) + require.Equal(t, uint64(20), destination.LastCommitTS()) + }) + } + }) + } +} diff --git a/store/snapshot_external_encryption_test.go b/store/snapshot_external_encryption_test.go new file mode 100644 index 000000000..456486764 --- /dev/null +++ b/store/snapshot_external_encryption_test.go @@ -0,0 +1,106 @@ +package store + +import ( + "bytes" + "context" + "path/filepath" + "testing" + + "github.com/cockroachdb/pebble/v2" + "github.com/stretchr/testify/require" +) + +func TestExternalSnapshotEncryptedPebbleBoundary(t *testing.T) { + for _, sst := range []bool{false, true} { + for _, bearing := range []bool{false, true} { + verifyExternalEncryptedSnapshot(t, sst, bearing) + } + } +} + +func verifyExternalEncryptedSnapshot(t *testing.T, sst, bearing bool) { + t.Helper() + source := newEncryptedStoreFixture(t, 7) + ps, ok := source.mvcc.(*pebbleStore) + require.True(t, ok) + ps.sstIngestSnapshots = sst + require.NoError(t, source.mvcc.PutAt(t.Context(), []byte("encrypted"), []byte("payload !migstage|ownership"), 20, 0)) + seedExternalExistingEpoch(t, source.mvcc) + if bearing { + encoded, err := encodeMigrationOwnership([]MigrationOwnershipState{externalTestOwnership(MigrationOwner)}) + require.NoError(t, err) + require.NoError(t, ps.db.Set(migrationOwnershipMetaKeyBytes, encoded, pebble.Sync)) + } + raw := externalTestSnapshot(t, source.mvcc) + dir := filepath.Join(t.TempDir(), "destination") + options := []PebbleStoreOption{WithEncryption(source.cipher, NewCounterNonceFactory(0xABCD, 2), + func() (uint32, bool) { return source.keyID, true })} + destination, err := NewPebbleStore(dir, options...) + require.NoError(t, err) + defer func() { require.NoError(t, destination.Close()) }() + seedExternalExistingEpoch(t, destination) + require.NoError(t, destination.PutAt(t.Context(), []byte("sentinel"), []byte("unchanged"), 10, 0)) + before := externalTestSnapshot(t, destination) + err = externalTestRestore(t, destination, raw) + if bearing { + require.ErrorIs(t, err, ErrExternalSnapshotMigrationState) + require.Equal(t, before, externalTestSnapshot(t, destination)) + } else { + require.NoError(t, err) + _, readErr := destination.GetAt(t.Context(), MigrationConfigurationIndexKey(), ^uint64(0)) + require.ErrorIs(t, readErr, ErrKeyNotFound) + } + require.NoError(t, destination.Close()) + destination, err = NewPebbleStore(dir, options...) + require.NoError(t, err) + if bearing { + require.Equal(t, before, externalTestSnapshot(t, destination)) + require.NoError(t, destination.Restore(bytes.NewReader(raw))) + } + value, err := destination.GetAt(t.Context(), []byte("encrypted"), 20) + require.NoError(t, err) + require.Equal(t, []byte("payload !migstage|ownership"), value) +} + +type cancelAfterSnapshotReader struct { + reader *bytes.Reader + cancel context.CancelFunc +} + +func (r *cancelAfterSnapshotReader) Read(p []byte) (int, error) { + if len(p) > 1 { + p = p[:1] + } + n, err := r.reader.Read(p) + if r.reader.Len() == 0 { + r.cancel() + } + return n, err +} + +func TestExternalSnapshotCancellationAfterInputConsumed(t *testing.T) { + for _, format := range []string{"mvcc", "native", "sst"} { + source := externalTestMemory(t) + require.NoError(t, source.PutAt(t.Context(), []byte("new"), []byte("value"), 20, 0)) + seedExternalExistingEpoch(t, source) + raw := externalTestFormat(t, source, format) + for _, memory := range []bool{false, true} { + if memory && format != "mvcc" { + continue + } + destination := externalTestDestination(t, memory) + defer destination.Close() + seedExternalExistingEpoch(t, destination) + require.NoError(t, destination.PutAt(t.Context(), []byte("old"), []byte("value"), 10, 0)) + before := externalTestSnapshot(t, destination) + ctx, cancel := context.WithCancel(t.Context()) + reader := &cancelAfterSnapshotReader{reader: bytes.NewReader(raw), cancel: cancel} + restorer, ok := destination.(ExternalSnapshotRestorer) + require.True(t, ok) + require.ErrorIs(t, restorer.RestoreExternal(ctx, reader), context.Canceled) + cancel() + require.Zero(t, reader.reader.Len()) + require.Equal(t, before, externalTestSnapshot(t, destination)) + } + } +} diff --git a/store/snapshot_external_epoch_test.go b/store/snapshot_external_epoch_test.go new file mode 100644 index 000000000..348dae46a --- /dev/null +++ b/store/snapshot_external_epoch_test.go @@ -0,0 +1,76 @@ +package store + +import ( + "bytes" + "encoding/binary" + "testing" + + "github.com/stretchr/testify/require" +) + +func seedExternalExistingEpoch(t *testing.T, s MVCCStore) { + t.Helper() + key := MigrationConfigurationIndexKey() + require.NoError(t, s.PutAt(t.Context(), key, binary.BigEndian.AppendUint64(nil, 11), 10, 0)) + require.NoError(t, s.PutAt(t.Context(), key, binary.BigEndian.AppendUint64(nil, 22), 20, 0)) +} + +func TestMigrationConfigurationIndexKeyReturnsPrivateBytes(t *testing.T) { + key := MigrationConfigurationIndexKey() + key[0] = '?' + require.Equal(t, []byte("!txn|meta|migration_configuration_index"), MigrationConfigurationIndexKey()) +} + +func TestExternalSnapshotDropsOnlyConfigurationEpochAllVersions(t *testing.T) { + source := externalTestMemory(t) + seedExternalExistingEpoch(t, source) + adjacent := append(MigrationConfigurationIndexKey(), []byte("|user")...) + require.NoError(t, source.PutAt(t.Context(), adjacent, []byte("keep"), 30, 0)) + for _, format := range []string{"mvcc", "native", "sst"} { + t.Run(format, func(t *testing.T) { + raw := externalTestFormat(t, source, format) + for _, memory := range []bool{true, false} { + if memory && format != "mvcc" { + continue + } + verifyExternalEpochSanitization(t, raw, memory, adjacent) + } + }) + } +} + +func verifyExternalEpochSanitization(t *testing.T, raw []byte, memory bool, adjacent []byte) { + t.Helper() + destination := externalTestDestination(t, memory) + defer func() { require.NoError(t, destination.Close()) }() + require.NoError(t, externalTestRestore(t, destination, raw)) + assertExternalEpochAbsent(t, destination, adjacent) + destination = reopenExternalDestination(t, destination) + assertExternalEpochAbsent(t, destination, adjacent) + // Same-lineage restore retains both historical epoch versions verbatim. + require.NoError(t, destination.Restore(bytes.NewReader(raw))) + assertExternalEpochRetained(t, destination) + destination = reopenExternalDestination(t, destination) + assertExternalEpochRetained(t, destination) +} + +func assertExternalEpochAbsent(t *testing.T, s MVCCStore, adjacent []byte) { + t.Helper() + for _, ts := range []uint64{10, 20, ^uint64(0)} { + _, err := s.GetAt(t.Context(), MigrationConfigurationIndexKey(), ts) + require.ErrorIs(t, err, ErrKeyNotFound) + } + value, err := s.GetAt(t.Context(), adjacent, ^uint64(0)) + require.NoError(t, err) + require.Equal(t, []byte("keep"), value) + require.Equal(t, uint64(30), s.LastCommitTS(), "do not replace the epoch with the data frontier") +} + +func assertExternalEpochRetained(t *testing.T, s MVCCStore) { + t.Helper() + for ts, epoch := range map[uint64]uint64{10: 11, 20: 22} { + value, err := s.GetAt(t.Context(), MigrationConfigurationIndexKey(), ts) + require.NoError(t, err) + require.Equal(t, binary.BigEndian.AppendUint64(nil, epoch), value) + } +} diff --git a/store/snapshot_external_test.go b/store/snapshot_external_test.go new file mode 100644 index 000000000..bcaf1cb41 --- /dev/null +++ b/store/snapshot_external_test.go @@ -0,0 +1,317 @@ +package store + +import ( + "bytes" + "context" + "encoding/binary" + "hash/crc32" + "os" + "path/filepath" + "testing" + + "github.com/cockroachdb/pebble/v2" + "github.com/stretchr/testify/require" +) + +func externalTestMemory(t *testing.T) *mvccStore { + t.Helper() + s, ok := NewMVCCStore().(*mvccStore) + require.True(t, ok) + return s +} + +func externalTestSnapshot(t *testing.T, s MVCCStore) []byte { + t.Helper() + snap, err := s.Snapshot() + require.NoError(t, err) + defer snap.Close() + return snapshotBytes(t, snap) +} + +func externalTestFormat(t *testing.T, source *mvccStore, format string) []byte { + t.Helper() + raw := externalTestSnapshot(t, source) + if format == "mvcc" { + return raw + } + s, err := NewPebbleStore(t.TempDir(), WithSSTIngestSnapshots(format == "sst")) + require.NoError(t, err) + defer s.Close() + require.NoError(t, s.Restore(bytes.NewReader(raw))) + return externalTestSnapshot(t, s) +} + +func externalTestDestination(t *testing.T, memory bool) MVCCStore { + t.Helper() + if memory { + return NewMVCCStore() + } + s, err := NewPebbleStore(filepath.Join(t.TempDir(), "db")) + require.NoError(t, err) + return s +} + +func externalTestRestore(t *testing.T, s MVCCStore, raw []byte) error { + t.Helper() + restorer, ok := s.(ExternalSnapshotRestorer) + require.True(t, ok) + return restorer.RestoreExternal(t.Context(), bytes.NewReader(raw)) +} + +func externalTestCases() map[string]func(*mvccStore) { + return map[string]func(*mvccStore){ + "acks": func(s *mvccStore) { + s.migrationAcks[migrationAckID{jobID: 7, bracketID: 2}] = migrationImportAck{batchSeq: 3, cursor: []byte("cursor")} + }, + "floors": func(s *mvccStore) { s.migrationHLCFloors[7] = 100 }, + "promotion": func(s *mvccStore) { + s.migrationPromotions[7] = PromotionState{Cursor: []byte("cursor"), PromotedRows: 2, MaxPromotedTS: 100} + }, + "retirement": func(s *mvccStore) { s.migrationRetired[7] = struct{}{} }, + "readiness": func(s *mvccStore) { + s.migrationReadinessCache = []TargetStagedReadinessState{{JobID: 7, MigrationJobID: 7, Armed: true, MinWriteTSExclusive: 100}} + }, + "tracker-and-fences": func(s *mvccStore) { + s.migrationReadinessCache = []TargetStagedReadinessState{{JobID: 7, MigrationJobID: 7, + TrackWrites: true, SourceReadFence: true, SourceWriteFence: true, RetentionPinTS: 10, MinAdmittedTS: 20}} + }, + "owner": func(s *mvccStore) { + s.migrationOwnership = []MigrationOwnershipState{externalTestOwnership(MigrationOwner)} + }, + "nonowner": func(s *mvccStore) { + s.migrationOwnership = []MigrationOwnershipState{externalTestOwnership(MigrationNonOwner)} + }, + } +} + +func externalTestOwnership(role MigrationOwnershipRole) MigrationOwnershipState { + owner := uint64(1) + operation := MigrationOwnershipAcquire + if role == MigrationNonOwner { + owner = 2 + operation = MigrationOwnershipClear + } + state := MigrationOwnershipState{JobID: 7, LocalGroupID: 1, OwnerGroupID: owner, CatalogVersion: 10, + ConfigurationIndex: 3, AppliedIndex: 9, MinWriteTSExclusive: 100, + Generation: "0123456789abcdef0123456789abcdef", DescriptorHash: bytes.Repeat([]byte{1}, 32), RouteID: 3, ParentRouteID: 1, + RouteStart: []byte("a"), RouteEnd: []byte("z"), Role: role, Staged: role == MigrationOwner} + state.Receipts = []MigrationOwnershipReceipt{ownershipOperationReceipt(state, operation)} + return state +} + +func TestExternalSnapshotOwnershipFixtureClone(t *testing.T) { + for _, role := range []MigrationOwnershipRole{MigrationOwner, MigrationNonOwner} { + state := externalTestOwnership(role) + require.True(t, validMigrationOwnership(state)) + require.True(t, validOwnershipReceipts(state)) + encoded, err := encodeMigrationOwnership([]MigrationOwnershipState{state}) + require.NoError(t, err) + decoded, err := decodeMigrationOwnership(encoded) + require.NoError(t, err) + require.Equal(t, []MigrationOwnershipState{state}, decoded) + cloned := cloneMigrationOwnership(decoded) + require.Equal(t, decoded, cloned) + cloned[0].RouteStart[0]++ + cloned[0].RouteEnd[0]++ + cloned[0].DescriptorHash[0]++ + cloned[0].Receipts[0].DescriptorHash[0]++ + cloned[0].Receipts[0].CatalogVersion++ + require.Equal(t, []MigrationOwnershipState{state}, decoded) + require.Equal(t, externalTestOwnership(role), state) + } +} + +func TestExternalSnapshotMigrationMetadataRejectedBeforePublication(t *testing.T) { + for name, populate := range externalTestCases() { + t.Run(name, func(t *testing.T) { + for _, format := range []string{"mvcc", "native", "sst"} { + t.Run(format, func(t *testing.T) { + source := externalTestMemory(t) + populate(source) + raw := externalTestFormat(t, source, format) + for _, memory := range []bool{true, false} { + if memory && format != "mvcc" { + continue + } + verifyExternalMetadataRejection(t, raw, memory, source.migrationOwnership) + } + }) + } + }) + } +} + +func verifyExternalMetadataRejection(t *testing.T, raw []byte, memory bool, ownership []MigrationOwnershipState) { + t.Helper() + destination := externalTestDestination(t, memory) + defer func() { require.NoError(t, destination.Close()) }() + seedExternalExistingEpoch(t, destination) + require.NoError(t, destination.PutAt(t.Context(), []byte("sentinel"), []byte("unchanged"), 50, 0)) + before := externalTestSnapshot(t, destination) + require.ErrorIs(t, externalTestRestore(t, destination, raw), ErrExternalSnapshotMigrationState) + require.Equal(t, before, externalTestSnapshot(t, destination)) + destination = reopenExternalDestination(t, destination) + require.Equal(t, before, externalTestSnapshot(t, destination)) + // Same-lineage restore must retain every decoded field, not strip evidence. + require.NoError(t, destination.Restore(bytes.NewReader(raw))) + assertExternalNativeMetadata(t, destination, raw) + assertExternalOwnershipFields(t, destination, ownership) + destination = reopenExternalDestination(t, destination) + assertExternalNativeMetadata(t, destination, raw) + assertExternalOwnershipFields(t, destination, ownership) +} + +func assertExternalOwnershipFields(t *testing.T, s MVCCStore, ownership []MigrationOwnershipState) { + t.Helper() + if len(ownership) == 0 { + return + } + view, err := ownershipStore(t, s).MigrationServingState(t.Context()) + require.NoError(t, err) + require.Equal(t, ownership, view.Ownership) +} + +func reopenExternalDestination(t *testing.T, s MVCCStore) MVCCStore { + t.Helper() + ps, ok := s.(*pebbleStore) + if !ok { + return s + } + require.NoError(t, ps.Close()) + reopened, err := NewPebbleStore(ps.dir) + require.NoError(t, err) + return reopened +} + +func assertExternalNativeMetadata(t *testing.T, destination MVCCStore, raw []byte) { + t.Helper() + // Converting the original into Pebble gives a common decoder for all formats. + expected := externalTestDestination(t, false) + defer expected.Close() + require.NoError(t, expected.Restore(bytes.NewReader(raw))) + got := externalTestSnapshot(t, destination) + actual := externalTestDestination(t, false) + defer actual.Close() + require.NoError(t, actual.Restore(bytes.NewReader(got))) + require.Equal(t, externalTestSnapshot(t, expected), externalTestSnapshot(t, actual)) +} + +func TestExternalSnapshotTypedMigrationRows(t *testing.T) { + for _, prefix := range []string{"!dist|migstage|", "!migwrite|", "!migfence|"} { + t.Run(prefix, func(t *testing.T) { + source := externalTestMemory(t) + key := binary.BigEndian.AppendUint64([]byte(prefix), 7) + key = append(key, '|', 'k') + require.NoError(t, source.PutAt(t.Context(), key, []byte("payload"), 20, 0)) + for _, format := range []string{"mvcc", "native", "sst"} { + raw := externalTestFormat(t, source, format) + verifyExternalMetadataRejection(t, raw, false, nil) + if format == "mvcc" { + verifyExternalMetadataRejection(t, raw, true, nil) + } + } + }) + } +} + +func TestExternalSnapshotProofFreeAndEpochOnly(t *testing.T) { + source := externalTestMemory(t) + // Metadata-like bytes in ordinary values or non-typed keys are not evidence. + require.NoError(t, source.PutAt(t.Context(), []byte("user"), []byte("!migstage|ownership !migwrite| _migack"), 20, 0)) + seedExternalExistingEpoch(t, source) + for _, format := range []string{"mvcc", "native", "sst"} { + raw := externalTestFormat(t, source, format) + for _, memory := range []bool{false, true} { + if memory && format != "mvcc" { + continue + } + destination := externalTestDestination(t, memory) + defer destination.Close() + require.NoError(t, externalTestRestore(t, destination, raw)) + value, err := destination.GetAt(t.Context(), []byte("user"), 21) + require.NoError(t, err) + require.Equal(t, []byte("!migstage|ownership !migwrite| _migack"), value) + } + } +} + +func TestExternalSnapshotEmptyV6Metadata(t *testing.T) { + s := externalTestMemory(t) + f, err := os.CreateTemp(t.TempDir(), "snapshot") + require.NoError(t, err) + defer f.Close() + offset, err := writeMVCCSnapshotHeader(f, mvccSnapshotVersionV6) + require.NoError(t, err) + sum, err := s.writeSnapshotBodyLocked(f, mvccSnapshotVersionV6) + require.NoError(t, err) + require.NoError(t, finalizeMVCCSnapshotFile(f, offset, sum)) + raw, err := os.ReadFile(f.Name()) + require.NoError(t, err) + for _, memory := range []bool{true, false} { + destination := externalTestDestination(t, memory) + defer destination.Close() + require.NoError(t, externalTestRestore(t, destination, raw)) + } +} + +func TestExternalSnapshotCorruptMetadataLeavesDestinationUnchanged(t *testing.T) { + for _, sst := range []bool{false, true} { + for _, key := range [][]byte{migrationOwnershipMetaKeyBytes, migrationRetiredMetaKeyBytes, + migrationAckMetaKeyBytes, migrationHLCFloorMetaKeyBytes, migrationPromoteMetaKeyBytes, migrationReadyKey(7)} { + s, err := NewPebbleStore(t.TempDir(), WithSSTIngestSnapshots(sst)) + require.NoError(t, err) + ps, ok := s.(*pebbleStore) + require.True(t, ok) + require.NoError(t, ps.db.Set(key, []byte("corrupt"), pebble.Sync)) + raw := externalTestSnapshot(t, s) + require.NoError(t, s.Close()) + destination := externalTestDestination(t, false) + defer destination.Close() + seedExternalExistingEpoch(t, destination) + require.NoError(t, destination.PutAt(t.Context(), []byte("sentinel"), []byte("unchanged"), 50, 0)) + before := externalTestSnapshot(t, destination) + require.Error(t, externalTestRestore(t, destination, raw)) + require.Equal(t, before, externalTestSnapshot(t, destination)) + } + } +} + +func TestExternalSnapshotCanceledBeforePublication(t *testing.T) { + source := externalTestMemory(t) + raw := externalTestSnapshot(t, source) + for _, memory := range []bool{true, false} { + destination := externalTestDestination(t, memory) + defer destination.Close() + seedExternalExistingEpoch(t, destination) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + restorer, ok := destination.(ExternalSnapshotRestorer) + require.True(t, ok) + before := externalTestSnapshot(t, destination) + require.ErrorIs(t, restorer.RestoreExternal(ctx, bytes.NewReader(raw)), context.Canceled) + require.Equal(t, before, externalTestSnapshot(t, destination)) + } +} + +func TestExternalSnapshotCorruptMVCCMetadata(t *testing.T) { + source := externalTestMemory(t) + source.migrationOwnership = []MigrationOwnershipState{externalTestOwnership(MigrationOwner)} + raw := externalTestSnapshot(t, source) + encoded, err := encodeMigrationOwnership(source.migrationOwnership) + require.NoError(t, err) + offset := bytes.Index(raw, encoded) + require.Positive(t, offset) + raw[offset] = 0 // Unknown ownership codec, with a valid outer checksum. + const checksumOffset = 12 + const bodyOffset = 16 + binary.LittleEndian.PutUint32(raw[checksumOffset:bodyOffset], crc32.ChecksumIEEE(raw[bodyOffset:])) + for _, memory := range []bool{true, false} { + destination := externalTestDestination(t, memory) + defer destination.Close() + seedExternalExistingEpoch(t, destination) + require.NoError(t, destination.PutAt(t.Context(), []byte("sentinel"), []byte("unchanged"), 50, 0)) + before := externalTestSnapshot(t, destination) + require.ErrorContains(t, externalTestRestore(t, destination, raw), "invalid migration ownership encoding") + require.Equal(t, before, externalTestSnapshot(t, destination)) + } +} diff --git a/store/snapshot_external_user_keys_test.go b/store/snapshot_external_user_keys_test.go new file mode 100644 index 000000000..136697b5e --- /dev/null +++ b/store/snapshot_external_user_keys_test.go @@ -0,0 +1,174 @@ +package store + +import ( + "path/filepath" + "testing" + + "github.com/cockroachdb/pebble/v2" + "github.com/stretchr/testify/require" +) + +func externalAliasValues() map[string][]byte { + return map[string][]byte{ + "_migack": encodeMigrationImportAcks(map[migrationAckID]migrationImportAck{ + {jobID: 7, bracketID: 1}: {batchSeq: 1}, + }), + "_mighlc": encodeMigrationHLCFloors(map[uint64]uint64{7: 100}), + "_migpromote": encodeMigrationPromotionStates(map[uint64]PromotionState{7: {Cursor: []byte("pending")}}), + } +} + +func externalAliasSource(t *testing.T, format string, encrypted bool) (MVCCStore, []PebbleStoreOption) { + t.Helper() + if encrypted { + fixture := newEncryptedStoreFixture(t, 7) + ps, ok := fixture.mvcc.(*pebbleStore) + require.True(t, ok) + ps.sstIngestSnapshots = format == "sst" + return fixture.mvcc, []PebbleStoreOption{WithEncryption(fixture.cipher, NewCounterNonceFactory(0xABCD, 2), + func() (uint32, bool) { return fixture.keyID, true })} + } + if format == "mvcc" { + return NewMVCCStore(), nil + } + source, err := NewPebbleStore(t.TempDir(), WithSSTIngestSnapshots(format == "sst")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, source.Close()) }) + return source, nil +} + +func TestExternalSnapshotOrdinaryMigrationNamesRoundTrip(t *testing.T) { + for _, format := range []string{"mvcc", "native", "sst"} { + for _, encrypted := range []bool{false, true} { + if encrypted && format == "mvcc" { + continue + } + t.Run(format+map[bool]string{false: "", true: "-encrypted"}[encrypted], func(t *testing.T) { + source, options := externalAliasSource(t, format, encrypted) + seedExternalExistingEpoch(t, source) + for key, value := range externalAliasValues() { + require.NoError(t, source.PutAt(t.Context(), []byte(key), []byte("ordinary"), 10, 0)) + require.NoError(t, source.PutAt(t.Context(), []byte(key), value, 20, 0)) + } + if indexed, ok := source.(interface{ SetDurableAppliedIndex(uint64) error }); ok { + require.NoError(t, indexed.SetDurableAppliedIndex(9001)) + } + if encrypted { + assertExternalAliasEncrypted(t, source) + } + raw := externalTestSnapshot(t, source) + t.Run("pebble", func(t *testing.T) { verifyExternalAliasDestination(t, raw, false, options) }) + if format == "mvcc" { + t.Run("memory", func(t *testing.T) { verifyExternalAliasDestination(t, raw, true, nil) }) + } + }) + } + } +} + +func assertExternalAliasEncrypted(t *testing.T, source MVCCStore) { + t.Helper() + ps, ok := source.(*pebbleStore) + require.True(t, ok) + for key := range externalAliasValues() { + value, closer, err := ps.db.Get(encodeKey([]byte(key), 10)) + require.NoError(t, err) + decoded, err := decodeValue(value) + require.NoError(t, err) + require.Equal(t, encStateEncrypted, decoded.EncState) + require.NoError(t, closer.Close()) + } +} + +func verifyExternalAliasDestination(t *testing.T, raw []byte, memory bool, options []PebbleStoreOption) { + t.Helper() + destination := NewMVCCStore() + dir := filepath.Join(t.TempDir(), "destination") + if !memory { + var err error + destination, err = NewPebbleStore(dir, options...) + require.NoError(t, err) + } + defer func() { require.NoError(t, destination.Close()) }() + require.NoError(t, externalTestRestore(t, destination, raw)) + assertExternalAliasValues(t, destination) + if !memory { + require.NoError(t, destination.Close()) + var err error + destination, err = NewPebbleStore(dir, options...) + require.NoError(t, err) + assertExternalAliasValues(t, destination) + } +} + +func assertExternalAliasValues(t *testing.T, s MVCCStore) { + t.Helper() + for key, value := range externalAliasValues() { + for ts, expected := range map[uint64][]byte{10: []byte("ordinary"), 20: value} { + actual, err := s.GetAt(t.Context(), []byte(key), ts) + require.NoError(t, err) + require.Equal(t, expected, actual) + } + } + _, err := s.GetAt(t.Context(), MigrationConfigurationIndexKey(), ^uint64(0)) + require.ErrorIs(t, err, ErrKeyNotFound) + if indexed, ok := s.(interface{ LastAppliedIndex() (uint64, bool, error) }); ok { + _, present, err := indexed.LastAppliedIndex() + require.NoError(t, err) + require.False(t, present) + } +} + +func TestExternalSnapshotRawMigrationNamesRemainMetadata(t *testing.T) { + for _, format := range []string{"native", "sst"} { + for _, encrypted := range []bool{false, true} { + for key, value := range externalAliasValues() { + t.Run(format+map[bool]string{false: "", true: "-encrypted"}[encrypted]+"/"+key, func(t *testing.T) { + source, options := externalAliasSource(t, format, encrypted) + ps, ok := source.(*pebbleStore) + require.True(t, ok) + require.NoError(t, source.PutAt(t.Context(), []byte("ordinary"), []byte("payload"), 10, 0)) + require.NoError(t, ps.db.Set([]byte(key), value, pebble.Sync)) + raw := externalTestSnapshot(t, source) + verifyExternalAliasRefusal(t, raw, options, ErrExternalSnapshotMigrationState) + require.NoError(t, ps.db.Set([]byte(key), []byte("corrupt"), pebble.Sync)) + verifyExternalAliasRefusal(t, externalTestSnapshot(t, source), options, nil) + }) + } + } + } +} + +func verifyExternalAliasRefusal(t *testing.T, raw []byte, options []PebbleStoreOption, expected error) { + t.Helper() + destination, err := NewPebbleStore(t.TempDir(), options...) + require.NoError(t, err) + defer destination.Close() + seedExternalExistingEpoch(t, destination) + require.NoError(t, destination.PutAt(t.Context(), []byte("old"), []byte("unchanged"), 30, 0)) + before := externalTestSnapshot(t, destination) + err = externalTestRestore(t, destination, raw) + if expected != nil { + require.ErrorIs(t, err, expected) + } else { + require.Error(t, err) + } + require.Equal(t, before, externalTestSnapshot(t, destination)) +} + +func TestExternalSnapshotLogicalMigrationControlKeysRemainReserved(t *testing.T) { + for _, key := range [][]byte{migrationOwnershipMetaKeyBytes, migrationRetiredMetaKeyBytes, migrationReadyKey(7)} { + source := externalTestMemory(t) + require.NoError(t, source.PutAt(t.Context(), key, []byte("legacy control"), 10, 0)) + for _, format := range []string{"mvcc", "native", "sst"} { + raw := externalTestFormat(t, source, format) + verifyExternalAliasRefusal(t, raw, nil, nil) + if format == "mvcc" { + destination := externalTestDestination(t, true) + before := externalTestSnapshot(t, destination) + require.Error(t, externalTestRestore(t, destination, raw)) + require.Equal(t, before, externalTestSnapshot(t, destination)) + } + } + } +} diff --git a/store/snapshot_migration_metadata.go b/store/snapshot_migration_metadata.go new file mode 100644 index 000000000..94f76ce78 --- /dev/null +++ b/store/snapshot_migration_metadata.go @@ -0,0 +1,71 @@ +package store + +import ( + "io" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +func stageSnapshotServingMetadata(batch *pebble.Batch, meta streamingMVCCRestoreMetadata) error { + if err := stageSnapshotOwnership(batch, meta.migrationOwnership); err != nil { + return err + } + if err := stageSnapshotPartitions(batch, meta.migrationPartitions); err != nil { + return err + } + // Readiness has one raw row per job; publish it in the same batch. + for _, state := range meta.migrationReadiness { + if err := batch.Set(migrationReadyKey(state.JobID), encodeTargetStagedReadinessState(state), nil); err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func readSnapshotPartitions(r io.Reader, version uint32) (MigrationOwnershipPartitionState, error) { + if version < mvccSnapshotVersionV7 { + return MigrationOwnershipPartitionState{}, nil + } + data, err := readMVCCSnapshotBytes(r, "snapshot ownership partitions") + if err != nil { + return MigrationOwnershipPartitionState{}, err + } + return decodeMigrationOwnershipPartitions(data) +} + +func (s *pebbleStore) loadMigrationServingMetadata() ([]TargetStagedReadinessState, error) { + readiness, err := s.loadPebbleTargetReadinessStatesFromDB() + if err != nil { + return nil, err + } + if _, err := s.readMigrationRetiredJobs(); err != nil { + return nil, err + } + ownership, err := s.readMigrationOwnership() + if err != nil { + return nil, err + } + partitions, err := s.readMigrationOwnershipPartitions() + if err != nil { + return nil, err + } + s.migrationAdmission = buildMigrationServingAdmission(ownership, partitions) + return readiness, nil +} + +func stageSnapshotPartitions(batch *pebble.Batch, state MigrationOwnershipPartitionState) error { + encoded, err := encodeMigrationOwnershipPartitions(state) + if err != nil { + return err + } + return errors.WithStack(batch.Set(migrationOwnershipPartitionMetaKey, encoded, nil)) +} + +func stageSnapshotOwnership(batch *pebble.Batch, states []MigrationOwnershipState) error { + encoded, err := encodeMigrationOwnership(states) + if err != nil { + return err + } + return errors.WithStack(batch.Set(migrationOwnershipMetaKeyBytes, encoded, nil)) +} diff --git a/store/snapshot_pebble_sst.go b/store/snapshot_pebble_sst.go index e2a36601d..ea4fbcc6d 100644 --- a/store/snapshot_pebble_sst.go +++ b/store/snapshot_pebble_sst.go @@ -589,7 +589,7 @@ func (b *pebbleSSTIngestBundle) Close() error { return errors.WithStack(err) } -func (s *pebbleStore) restorePebbleSSTIngestAtomic(r io.Reader) error { +func (s *pebbleStore) restorePebbleSSTIngestAtomic(r io.Reader, validate func(string) error) error { manifest, err := readPebbleSSTIngestManifest(r) if err != nil { return err @@ -613,6 +613,16 @@ func (s *pebbleStore) restorePebbleSSTIngestAtomic(r io.Reader) error { _ = os.RemoveAll(tmpDir) return err } + if err := validateRestoreTempDB(tmpDir, validate); err != nil { + return err + } + if validate != nil { + // External validation removes the source lineage's Raft watermark. + // Ingest verified the original manifest before this rebasing step; + // publication must verify its absence, not restore the source index. + metadata.AppliedIndex = 0 + metadata.AppliedIndexPresent = false + } return s.swapInTempDBWithMetadata(tmpDir, &metadata) } diff --git a/store/store.go b/store/store.go index d04bf94cf..7fbe1b22b 100644 --- a/store/store.go +++ b/store/store.go @@ -31,6 +31,12 @@ var ErrSnapshotVersionCountTooLarge = errors.New("mvcc snapshot version count to var ErrValueTooLarge = errors.New("value too large") var ErrInvalidExportCursor = errors.New("invalid export cursor") +// ErrInvalidReadinessState marks a target-staged-readiness state that fails +// validation. Validation is a pure function of the request, so every replica +// reaches the same verdict for the same entry: apply must return it as an +// ordinary rejection rather than halting the group over one malformed RPC. +var ErrInvalidReadinessState = errors.New("invalid target staged readiness state") + // ErrInvalidExportBudget rejects an export asked for with no version budget. // Answering Done for a zero budget lets a migration driver record a non-empty // bracket as fully copied and move on to cutover, so the completion signal must @@ -106,13 +112,15 @@ type ExportVersionsOptions struct { EndKey []byte MinCommitTSExclusive uint64 MaxCommitTSInclusive uint64 - Cursor []byte - MaxVersions int - MaxBytes uint64 - MaxScannedBytes uint64 - KeyFamily uint32 - AcceptKey func([]byte) bool - AcceptVersion func(key []byte, value []byte) bool + // ReadTS asks export to enforce the same retention watermark as GetAt/ScanAt. + ReadTS uint64 + Cursor []byte + MaxVersions int + MaxBytes uint64 + MaxScannedBytes uint64 + KeyFamily uint32 + AcceptKey func([]byte) bool + AcceptVersion func(key []byte, value []byte) bool } // ExportVersionsResult is one resumable chunk of raw MVCC versions. @@ -180,17 +188,100 @@ type PromotionState struct { LastError string } +// CleanupVersionsOptions physically removes exact committed versions selected +// by the migration bracket filter. It is cursor-resumable and is used only +// after ownership has moved or while abandoning an unpublished migration. +type CleanupVersionsOptions struct { + JobID uint64 + AppliedIndex uint64 + StartKey []byte + EndKey []byte + Cursor []byte + MaxCommitTS uint64 + MaxVersions int + MaxBytes uint64 + MaxScannedBytes uint64 + KeyFamily uint32 + AcceptVersion func(key, value []byte) bool +} + +// CleanupVersionsResult reports one bounded physical-delete chunk. +type CleanupVersionsResult struct { + NextCursor []byte + Done bool + DeletedRows uint64 + DeletedBytes uint64 + ScannedBytes uint64 +} + +// TargetStagedReadinessState is a target-local guard that makes a target voter +// fail closed for a moving route until it has either the staged descriptor or +// the cleared descriptor with the retained write timestamp floor. +type TargetStagedReadinessState struct { + JobID uint64 + RouteStart []byte + RouteEnd []byte + ExpectedCutoverVersion uint64 + MigrationJobID uint64 + MinWriteTSExclusive uint64 + Armed bool + SourceWriteFence bool + SourceReadFence bool + RetentionPinTS uint64 + TrackWrites bool + MinAdmittedTS uint64 + // TransferSizeGuardVersion explicitly opts this source interval into + // deterministic transfer-size admission; legacy trackers leave it zero. + TransferSizeGuardVersion uint32 +} + // MigrationPromoter is implemented by stores that can promote staged range // migration data into the live keyspace. type MigrationPromoter interface { PromoteVersions(ctx context.Context, opts PromoteVersionsOptions) (PromoteVersionsResult, error) } +// MigrationCleaner removes source or abandoned target versions and the +// per-job migration proof records through the Raft apply path. +type MigrationCleaner interface { + CleanupVersions(ctx context.Context, opts CleanupVersionsOptions) (CleanupVersionsResult, error) + ClearMigrationState(ctx context.Context, jobID, appliedIndex uint64) error +} + +// MigrationRetirementReader proves that cleanup permanently fenced the job. +// Retirement witnesses are independent of live guards, pins and progress. +type MigrationRetirementReader interface { + MigrationRetired(context.Context, uint64) (bool, error) +} + // MigrationPromotionStateReader reads target-local staged promotion state. type MigrationPromotionStateReader interface { MigrationPromotionState(ctx context.Context, jobID uint64) (PromotionState, bool, error) } +// MigrationTargetReadinessWriter persists a target-local staged-readiness +// guard through the target Raft group. +type MigrationTargetReadinessWriter interface { + ApplyTargetStagedReadiness(ctx context.Context, state TargetStagedReadinessState) error +} + +// MigrationTargetReadinessRaftWriter persists a target-local readiness guard +// from the Raft apply loop while bundling the applied index with that update. +type MigrationTargetReadinessRaftWriter interface { + ApplyTargetStagedReadinessAt(ctx context.Context, state TargetStagedReadinessState, appliedIndex uint64) error +} + +// MigrationTargetReadinessReader reads retained target-local readiness guards. +type MigrationTargetReadinessReader interface { + MigrationTargetReadinessStates(ctx context.Context) ([]TargetStagedReadinessState, error) +} + +// MigrationImportMetadataReader reports whether target-local import proof +// metadata remains for a migration job. +type MigrationImportMetadataReader interface { + MigrationImportMetadataPresent(ctx context.Context, jobID uint64) (bool, error) +} + // OpType describes a mutation kind. type OpType int