diff --git a/adapter/redis_background_rotation.go b/adapter/redis_background_rotation.go new file mode 100644 index 000000000..c5f3dfa63 --- /dev/null +++ b/adapter/redis_background_rotation.go @@ -0,0 +1,46 @@ +package adapter + +import ( + "context" + "time" + + "github.com/cockroachdb/errors" +) + +// Delta and migration work share both the timeout and its rotation. Separate +// rotations still starve migration when any delta handler exhausts every tick. +func (c *DeltaCompactor) compactBackgroundRotation(ctx context.Context, readTS uint64, snapshotAt time.Time) error { + if !c.ttlInlineMu.TryLock() { + return nil + } + defer c.ttlInlineMu.Unlock() + plans := c.ttlInlineMigrationTickPlans() + c.pruneTTLInlineMigrationPlanStates(plans) + var handlers []collectionDeltaHandler + if c.coord.IsLeader() { + handlers = c.allHandlers() + } + total := len(handlers) + len(plans) + start := c.rotationStart(&c.backgroundCursor, total) + var combined error + for offset := range total { + if ctx.Err() != nil { + break + } + index := (start + offset) % total + err := c.runBackgroundSlot(ctx, handlers, plans, index, readTS, snapshotAt) + c.advanceRotation(&c.backgroundCursor, index, total) + if err != nil && !errors.Is(err, context.Canceled) { + combined = errors.CombineErrors(combined, err) + } + } + return errors.WithStack(errors.CombineErrors(combined, ctx.Err())) +} + +func (c *DeltaCompactor) runBackgroundSlot(ctx context.Context, handlers []collectionDeltaHandler, plans []ttlInlineMigrationTickPlan, index int, readTS uint64, snapshotAt time.Time) error { + if index < len(handlers) { + return c.compactHandlerSafely(ctx, handlers[index], readTS) + } + plan := plans[index-len(handlers)] + return c.migrateTTLInlineScanPlan(ctx, plan.handler, plan.plan, readTS, snapshotAt) +} diff --git a/adapter/redis_delta_compactor.go b/adapter/redis_delta_compactor.go index dcf867469..777844cf6 100644 --- a/adapter/redis_delta_compactor.go +++ b/adapter/redis_delta_compactor.go @@ -81,10 +81,9 @@ type DeltaCompactor struct { stateMu sync.Mutex backoffUntil time.Time - // handlerCursor and ttlInlinePlanCursor are where the next tick's - // background handlers and TTL-inline scan plans start, so a tick that runs - // out of budget hands the rest of the rotation to the next one. - handlerCursor int + // backgroundCursor rotates delta handlers and migration plans together. + // ttlInlinePlanCursor supports standalone migration passes. + backgroundCursor int ttlInlinePlanCursor int ttlInlineGroupsUnknownOnce sync.Once @@ -350,12 +349,7 @@ func (c *DeltaCompactor) SyncOnce(ctx context.Context) error { tickCtx = readTimestamp.WithDispatchVoucher(tickCtx) readTS := readTimestamp.Timestamp() - combined := c.compactBackgroundHandlers(tickCtx, readTS) - if tickCtx.Err() == nil { - if err := c.migrateTTLInlineOnce(tickCtx, readTS, snapshotAt); err != nil && !errors.Is(err, context.Canceled) { - combined = errors.CombineErrors(combined, err) - } - } + combined := c.compactBackgroundRotation(tickCtx, readTS, snapshotAt) if deltaCompactorBudgetExhausted(combined, tickCtx, ctx) { c.backoffBackgroundCompaction(c.timeoutBackoff) c.logger.WarnContext(ctx, "delta compactor: backing off after timeout", @@ -365,28 +359,6 @@ func (c *DeltaCompactor) SyncOnce(ctx context.Context) error { return errors.WithStack(combined) } -func (c *DeltaCompactor) compactBackgroundHandlers(ctx context.Context, readTS uint64) error { - if !c.coord.IsLeader() { - return nil - } - handlers := c.allHandlers() - start := c.rotationStart(&c.handlerCursor, len(handlers)) - var combined error - for offset := range handlers { - if ctx.Err() != nil { - return errors.WithStack(errors.CombineErrors(combined, ctx.Err())) - } - idx := (start + offset) % len(handlers) - h := handlers[idx] - err := c.compactHandlerSafely(ctx, h, readTS) - c.advanceRotation(&c.handlerCursor, idx, len(handlers)) - if err != nil && !errors.Is(err, context.Canceled) { - combined = errors.CombineErrors(combined, err) - } - } - return errors.WithStack(combined) -} - // rotationStart returns where a rotation over total items starts this tick. // cursor is one of the compactor's rotation cursors, guarded by stateMu. func (c *DeltaCompactor) rotationStart(cursor *int, total int) int { diff --git a/adapter/redis_delta_compactor_test.go b/adapter/redis_delta_compactor_test.go index 9bb69e9b0..ed4899265 100644 --- a/adapter/redis_delta_compactor_test.go +++ b/adapter/redis_delta_compactor_test.go @@ -1636,7 +1636,7 @@ func TestDeltaCompactor_TTLInlineStopsAfterTimeout(t *testing.T) { require.True(t, bytes.HasPrefix(st.starts[0], []byte(redisStrPrefix))) } -func TestDeltaCompactor_RotatesHandlerAfterTimeout(t *testing.T) { +func TestDeltaCompactor_RotatesAllBackgroundWorkAfterTimeout(t *testing.T) { t.Parallel() base := store.NewMVCCStore() @@ -1655,8 +1655,11 @@ func TestDeltaCompactor_RotatesHandlerAfterTimeout(t *testing.T) { store.HashMetaDeltaPrefix, store.SetMetaDeltaPrefix, store.ZSetMetaDeltaPrefix, - store.ListMetaDeltaPrefix, } + for _, handler := range c.ttlInlineMigrationHandlers() { + wantPrefixes = append(wantPrefixes, string(handler.prefix)) + } + wantPrefixes = append(wantPrefixes, store.ListMetaDeltaPrefix) for _, prefix := range wantPrefixes { err := c.SyncOnce(context.Background()) require.ErrorIs(t, err, context.DeadlineExceeded) diff --git a/adapter/redis_ttl_inline_migrator.go b/adapter/redis_ttl_inline_migrator.go index 6d0e4466e..811071f6d 100644 --- a/adapter/redis_ttl_inline_migrator.go +++ b/adapter/redis_ttl_inline_migrator.go @@ -46,8 +46,8 @@ type ttlInlineMigrationTickPlan struct { // migrateTTLInlineOnce runs every handler's scan plans under the tick's // budget. The plans share that budget, so each tick starts at the plan after -// the last one the previous tick ran, as compactBackgroundHandlers rotates its -// handlers; starting from the first plan every tick starved the later handlers +// the last one the previous tick ran. SyncOnce uses the combined background +// rotation instead; starting from the first plan every tick starved later handlers // and groups whenever the earlier plans used up the budget. // // snapshotAt is a wall-clock time no later than the acquisition of readTS. A diff --git a/distribution/migrator.go b/distribution/migrator.go index 9492ba351..5a7d318a3 100644 --- a/distribution/migrator.go +++ b/distribution/migrator.go @@ -357,7 +357,7 @@ func SplitJobBracketProgressForPlan(brackets []MigrationBracket, phase SplitJobE // interval after applying bracket-local exclusions. Route ownership still // requires the caller's RouteKeyFilter for every bracket. func (b MigrationBracket) ContainsRawKey(rawKey []byte) bool { - if hasAnyExactKey(rawKey, migrationInternalExactKeys) { + if store.IsReplicatedControlKey(rawKey) || hasAnyExactKey(rawKey, migrationInternalExactKeys) { return false } if bytes.Compare(rawKey, b.Start) < 0 { diff --git a/docs/design/2026_09_26_proposed_positioning_and_roadmap.md b/docs/design/2026_09_26_proposed_positioning_and_roadmap.md index 01e46236b..9b3541612 100644 --- a/docs/design/2026_09_26_proposed_positioning_and_roadmap.md +++ b/docs/design/2026_09_26_proposed_positioning_and_roadmap.md @@ -245,7 +245,7 @@ one activated but still draining included, runs without these guarantees. The claims also exclude Phase-D clusters (`--tsoPhaseDEnabled`) until both TSO admission proposals (`2026_08_29_proposed_tso_batch_slot_claims.md`, -`2026_09_02_proposed_prephase_d_resolution_evidence.md`) have landed: +`2026_09_02_partial_prephase_d_resolution_evidence.md`) have landed: under Phase D the batch-window validator accepts any timestamp inside a committed window whether or not it was issued, and the pre-Phase-D resolution carve-out accepts a caller-asserted legacy timestamp, so a @@ -330,7 +330,7 @@ failure on every surface (A6, G11). The A0 reproduction test is the first thing its result decides what README may say. The two open TSO proposals (`2026_08_29_proposed_tso_batch_slot_claims.md`, -`2026_09_02_proposed_prephase_d_resolution_evidence.md`) protect the +`2026_09_02_partial_prephase_d_resolution_evidence.md`) protect the timestamp-uniqueness assumption the OCC check relies on. They belong to the same correctness track and are listed as active in §7. Their order relative to the audit's fixes is open (§9), but they gate the README claims for @@ -381,7 +381,7 @@ more. Can proceed in parallel with 6.2 to 6.4. | Track | Docs | |---|---| -| Active, correctness | this doc; `2026_09_26_proposed_serializable_isolation_audit.md`; `2026_08_29_proposed_tso_batch_slot_claims.md`; `2026_09_02_proposed_prephase_d_resolution_evidence.md`; the security and benchmark docs to be written | +| Active, correctness | this doc; `2026_09_26_proposed_serializable_isolation_audit.md`; `2026_08_29_proposed_tso_batch_slot_claims.md`; `2026_09_02_partial_prephase_d_resolution_evidence.md`; the security and benchmark docs to be written | | Active, features | `2026_07_19_partial_physical_snapshot_object_offload.md` (M3); `2026_04_25_partial_s3_raft_blob_offload.md` (M3, M4); `2026_06_11_partial_hotspot_split_milestone2_migration.md` with parent `2026_02_18_partial_hotspot_shard_split.md` | | Future track (horizontal scale) | `2026_06_23_proposed_scaling_roadmap.md` (its 06/12 predecessor is already marked superseded). Referenced from README's future goals; re-evaluated when the active list is empty. | | Deferred | Remaining Stage 9 of `2026_04_29_partial_data_at_rest_encryption.md` (9C+: rotation budget, rewrap / retire / rewrite, `last_proposed_index_per_raft_dek`, encrypted Jepsen). Stage 5E (the capability fan-out helper) is shipped and wired (`buildEncryptionCapabilityFanout` in `main.go`) although that doc's header still says deferred; the header is corrected in the A5 documentation pass. | diff --git a/docs/design/2026_09_26_proposed_serializable_isolation_audit.md b/docs/design/2026_09_26_proposed_serializable_isolation_audit.md index a7c737760..117bda05c 100644 --- a/docs/design/2026_09_26_proposed_serializable_isolation_audit.md +++ b/docs/design/2026_09_26_proposed_serializable_isolation_audit.md @@ -5,7 +5,7 @@ Author: bootjp Date: 2026-09-26 Related: `2026_09_26_proposed_positioning_and_roadmap.md` §5 and §6.3 (the claim this audit backs), `2026_08_29_proposed_tso_batch_slot_claims.md` and -`2026_09_02_proposed_prephase_d_resolution_evidence.md` (timestamp +`2026_09_02_partial_prephase_d_resolution_evidence.md` (timestamp uniqueness, which the conflict check assumes). ## 1. Claim and scope diff --git a/docs/design/2026_10_17_proposed_ttl_inline_cross_leader_legacy_rows.md b/docs/design/2026_10_17_proposed_ttl_inline_cross_leader_legacy_rows.md new file mode 100644 index 000000000..08b245c91 --- /dev/null +++ b/docs/design/2026_10_17_proposed_ttl_inline_cross_leader_legacy_rows.md @@ -0,0 +1,335 @@ +# TTL-inline migration of legacy-route rows across leaders + +Status: Proposed +Author: bootjp +Date: 2026-10-17 + +Related: +- [2026_04_17_implemented_ttl_inline_value.md](2026_04_17_implemented_ttl_inline_value.md): Phase 2 (background migration) and Phase 3 (removing the legacy `!redis|ttl|` read fallback). +- [2026_04_16_implemented_centralized_tso.md](2026_04_16_implemented_centralized_tso.md) §7.4: Phase D. +- [2026_09_26_proposed_serializable_isolation_audit.md](2026_09_26_proposed_serializable_isolation_audit.md): gaps G0 (across groups) and G10, fixes A0 and A0b. +- bootjp/elastickv#1278 (merged to main as `e569513`): the per-group migration this proposal builds on, and the review threads that deferred this case to a separate design. Its `ScanGroupOwnedAt`, `ownerHoldsNoVersionAt` and plan rotation are on main, as is the `SyncOnce` ordering §3 C changes. + +## 1. Problem + +After bootjp/elastickv#1278, each node migrates TTL anchors only in the data groups it leads. Each group's rows come from `ShardStore.ScanGroupOwnedAt`. That scan also serves hash, set, zset and stream metadata still stored at the route of its **raw** key: +- such a row sits in group L, while its user key belongs to group P; +- the scan serves it only when the node leading L also leads P, and P holds no version of the key at the read timestamp (`ownerHoldsNoVersionAt`); +- that is exactly when a point read falls back to the legacy route (`legacyPointRouteKey`, `routeMissStopsPointFallback`). + +When different nodes lead L and P, nothing processes the row: +- L's leader cannot tell whether P holds the key. `routeHasVersionAtOrBefore` answers only on P's leader. +- P's leader plans scans only for the groups it leads, so it never reads L. +- Only P's leader may issue the rewrite's timestamp. This is the repository's HLC rule, which the migrator keeps by dispatching only for keys it leads (`IsLeaderForKey`). + +Reads stay correct meanwhile, through the dual-route point-read fallback and the legacy `!redis|ttl|` fallback. But those rows keep their legacy TTL anchors indefinitely, so **Phase 3 of the TTL-inline design (disabling the fallback) cannot proceed** on a cluster that has any of them. + +How common this is depends on the catalog: +- Before user-key routing for these families, their metadata was routed by its raw bytes. `!hs|meta|…`, `!st|meta|…`, `!zs|meta|…` and `!stream|meta|…` sort into the `!` region of the keyspace, which usually belongs to one route. +- So a cluster's legacy rows typically sit in one group L, and their owners spread over every group. +- With leaders spread across nodes, most of those rows fall into this case. + +Before a36dac7f (2026-07-22), every node's whole-prefix `ScanAt` reached these rows; a36dac7f traded that coverage away to stop whole-keyspace scans from every node. + +## 2. Goals and non-goals + +Goals: +- Every legacy-route row is migrated, or shown not to exist, so Phase 3 can be gated on evidence. +- Keep the HLC rule: no persistence timestamp from a node that does not lead the written key's group (or from anything but the TSO under Phase D). +- Keep the per-tick cost bounded and proportional to the legacy backlog, not to the keyspace. +- Do not reintroduce whole-prefix scans from every node. + +Non-goals: +- Changing how point reads resolve legacy routes. The fallback stays until the rows are gone. +- Families without a legacy point route: strings, HLL, lists, the TTL index. +- Fixing G10 or G0 in general. This design needs neither A0b nor Phase D, because every relocation is dispatched by the owner group's own leader, and it closes G0 only for its own relocations (§3 B). + +## 3. Options + +### A. Owner-side scans of foreign legacy groups + +P's leader also scans the hash, set, zset and stream meta prefixes in every group it does not lead whose routes cover those raw prefixes. It serves the rows whose owner is P (a local presence check) and rewrites them as today. + +- **Timestamps:** compliant in every TSO mode, because P's leader writes P's keys. +- **Cost:** every node pages through every legacy group's four meta ranges over RPC to that group's leader, each tick. The rows it can use are only those owned by groups it leads; the rest are transferred and dropped. These are the remote whole-prefix scans a36dac7f removed. +- **Cost bound.** Take N nodes and G_L groups that hold legacy rows, with one page of at most 512 rows per plan per tick (30s). Each node scans the four meta plans of every legacy group it does not lead. That is up to 4 · N · G_L remote page RPCs per tick across the cluster, carrying up to 2,048 · N · G_L rows. With N = 5 and G_L = 1, that is 20 RPCs and up to 10,240 rows every 30s. All of them compete for the shared 5s tick budget with the node's own plans. A node can use only the rows whose owner it leads, about 1/N of them, so the cluster reads each row about N times per pass. +- **The range never shrinks.** The rewrite goes to P and L's copy stays, so every node rescans it on every pass, forever. That cost lasts after the backlog is migrated. + +#### A′. Owner-side scans that delete L's copy in the same transaction + +The relocation into P and an explicit-group delete of L's raw copy run in one transaction, so L's range shrinks and the scans converge. +- **Convergence:** each row is read about N times per pass until it is deleted, so a backlog of B rows costs about N · B row reads in total. Once the range is empty, each plan reads one empty page per tick. +- **Timestamps:** the transaction writes two groups, so it is a cross-group 2PC with P as primary and L as secondary. Its commit timestamp must come from the primary group's leader or the TSO. The audit lists that as **remaining** A0/A0b work: "allocate the 2PC commit timestamp after all PREPAREs succeed, from the primary group's leader or the TSO". Until it lands, a cross-group commit is coordinator-stamped, which is G10. A′ is therefore compliant only under Phase D. +- **New machinery:** a 2PC participant that writes an explicit group. Today 2PC routes each mutation by its key, and L's raw key routes to P. +- **Still N-fold:** it keeps the per-tick bound above until the backlog is empty, and every node still scans groups it does not lead. + +### B. Legacy group's leader checks the owner and forwards the rewrite + +L's leader already scans L, but `ScanGroupOwnedAt` filters out a legacy-route row whose owner is led elsewhere before the migrator sees it. **The scan must report the rows it skips.** Beside the served rows, the page returns each skipped legacy-route row with its owner group and the reason it was skipped: +- the owner is led elsewhere; +- the owner, led locally, holds a version; +- a route is under staged visibility. + +These rows are candidates and inputs to M1's counts, not rows to rewrite as served. They come from the same stored page, so reporting them costs no extra read. + +For a legacy-route row whose owner P is led elsewhere, L's leader does **not** build the rewrite itself. Building it reads the key's legacy TTL anchor twice: `legacyTTLMillisAt` and `ttlIndexSyncElem` both call `GetAt(redisTTLKey(userKey), readTS)`. That key routes to P, so on L's leader every row would cost up to two sequential remote reads, which no batching of the presence probe bounds. One 512-row page could then use up the 5s tick before its first transaction is dispatched, and the next tick would restart at the same cursor. Instead, P's leader builds and dispatches the rewrite: +1. L's leader groups the rows by owner group. For each group it sends at most 32 rows per request to P's leader, in a new peer-only `Internal` RPC, `RelocateLegacyRows`. A request carries each row's raw key and value as read from L, the `readTS` they were read at, and the catalog version the owners were resolved under. +2. P's leader serves the request locally, behind its read fence: + - it refuses the request if the catalog has moved past that version, or if any key no longer routes to P; + - it checks each key's presence in P at `readTS` (`VersionExistsAtOrBefore`, the check the remote probe already uses); + - it reads the TTL anchors from its own store and builds the elements with the handler's `buildElems`, forcing the metadata `Put`; + - it dispatches one transaction with `StartTS = readTS` and M1b's absence condition on every relocated metadata key. + + It answers per row: relocated, already held, conflicted, or refused. +3. L's leader settles each row from that answer. + +The presence check thus costs no separate round trip, and the TTL-anchor reads are local to P. M3 and D2's `--probe` still probe P from L's leader, because they only need presence. They use the batched form already on main, so they need no new RPC: `RawLatestCommitTSRequest.key_batch` (`pb.EncodeRawLatestCommitTSKeyBatch`) carries many keys, answered in `version_visible_results` by `ShardStore.VersionsExistAtOrBeforeGroupWithReadFence`. It has the same semantics per key as the single-key probe: the same read fence, and "cannot answer" for the whole batch on a non-leader. Its client is `ShardStore.routeHasVersionsAtOrBeforeRemote`, which today serves the S3 bucket-auxiliary owner probe. The server decodes at most `maxGRPCScanLimit` (1,025) keys per request, more than one 512-row page. + +**Timestamps.** P's leader dispatches the relocation as the leader of every key it writes, like any other write it serves. In legacy HLC mode the commit timestamp therefore comes from P's leader, and under Phase D from the TSO. L's leader never dispatches for P's keys, stamped or unstamped, so M2 does not depend on A0b or Phase D. (An earlier revision dispatched from L's leader, which needed A0b's unstamped forwarding or Phase D; stamping on L's leader would be G10.) Every relocation transaction writes only P: the metadata row and its `!redis|ttl|` row both route by user key. + +**A relocation requires absence at apply.** OCC alone does not protect a relocation. The FSM rejects a write when the key's latest version is newer than `StartTS` (`validateConflicts`, `checkConflicts*`). But `readTS` is the global snapshot, the maximum applied watermark over all groups. A user write to P's key with a commit timestamp at or below `readTS` that has not yet applied on P is therefore invisible to both the probe and the OCC check. This is G0 across groups (audit §A0, "Multi-group deployments"). A user write to a legacy-route hash reads the old metadata through the fallback and writes P. If it lands first, the relocation, derived from L's older row, overwrites it at a newer timestamp, and the user's write is lost. + +The relocation's precondition is stronger than OCC: P holds **no** version of the key at all, whatever its timestamp. Milestone M1b makes the FSM check exactly that, on every relocation path; M2 then uses it for cross-leader relocations. Each relocated metadata mutation carries an apply-time condition, "conflict if the key has any version", which P's FSM evaluates at apply, in log order: +- A user write applied before the relocation makes it conflict, whatever its timestamp. +- A user write applied after it conflicts as usual when its `StartTS` is below the relocation's commit timestamp. If its `StartTS` is above that timestamp, it read P before the relocation applied: that is G0 on the user's side, as for any transaction. It still loses nothing, because it was derived from the same legacy row the relocation copied, and the relocation carries no user data of its own. +- The presence probe stays as a pre-filter. Without it, every batch holding a row P already holds would conflict on every pass. +- The condition needs a new mutation field, which is a wire change. An old FSM would ignore the field and silently drop the check, so the gate covers every replica that can apply P's log, not only its voters. A learner still catching up would otherwise replay a relocation without the check. If P already holds the key, that learner overwrites it while the upgraded replicas reject the entry, and its FSM diverges before it is ever promoted. M1b's guarded relocations, and so M2, therefore run only after P replicates a capability marker, which P's leader proposes only once every voter and learner in its configuration reports the capability. Reporting the capability includes a **downgrade fence**: before it reports, each replica persists a new form of its `raft-engine` marker. Today's binaries refuse to open a data directory whose marker they cannot parse (`readRaftEngineMarker` → `parseRaftEngineType`), so an existing voter or learner restarted in place on an old binary fails at startup instead of replaying relocation entries without the check. This is how A7 fences its own FSM changes, and no membership change is needed for it to apply. + + Two more rules close the gaps around membership: + - **The marker is bound to the configuration it was checked against.** A conf change that passed admission before activation can still apply after the leader checked capabilities. So the marker entry carries the configuration it was validated for: the voter and learner IDs and the index of the last applied conf change. P's FSM applies the marker only if that configuration is still current at the marker's log position, and otherwise ignores it; the leader then rechecks and proposes again. Conversely, a leader refuses to propose a conf change that would add a replica lacking the capability once its log contains the marker, applied or not. A new leader sees the marker in its log, so this holds across leader changes. A conf change already included in the configuration the marker records passes the check. A conf change applied after the recorded last-applied conf change makes the marker stale: P ignores it, and the leader rechecks and proposes it again. A conf change ordered after the marker may add a replica only if that replica has the capability. + - **A joining replica fences its directory before its FSM changes.** A node that joins P after activation creates P's data directory then, long after it first advertised the capability. An M1b binary therefore creates every new group directory with the fenced `raft-engine` marker form, before any snapshot restore or log replay touches the FSM. Admission also requires the joining replica to report that P's directory carries the fence. From then on, a membership change that would add a replica lacking the capability is refused: a fresh learner joins by log replay or by snapshot, and either may reach relocation entries. This is the rollout rule the serializable isolation audit's A7 sets for FSM semantic changes. Until the marker is in place, the same-node path keeps today's behaviour, and M3 stays off for keys owned by that group. +- The same-node relocation path of bootjp/elastickv#1278 has the same exposure (its open review thread on `ownerHoldsNoVersionAt`). M1b applies the condition there, independently of M2: the same-node path relocates rows today, and M3 deletes L's copy after such a relocation, so M3 must never ship before the guard does. + +L's side cannot go stale the same way. Nothing writes L's raw metadata copy: user writes route by user key to P, and only M3 deletes the copy. A catalog move of L's range puts the route under staged visibility, and the row is deferred. + +**Batches are partitioned by owner group.** The migrator batches up to 32 keys per OCC transaction, and one scan page can hold rows owned by several remote groups. Each `RelocateLegacyRows` request goes to one owner group's leader, which may stamp only its own keys, so M2 groups rows by owner group and chunks each group's rows into requests of at most 32 keys. A page with many rows of one owner produces several requests for it. + +**Batches are bound to the catalog version they were grouped under.** A split or move between grouping and commit can leave one batch's keys owned by both P and a new group Q. Each batch records the catalog version used to resolve its owners. Before dispatch, the migrator re-resolves the owners. If the catalog has moved past that version, it repartitions the rows under the new catalog. The request carries the version, and P's leader refuses it if any key no longer routes to P. A refused batch leaves its rows unsettled, so the next tick redoes them under the current catalog. + +**A relocation always writes.** The rewrite goes through the handler's `buildElems`, which uses `putIfChanged`. A legacy-route row already in inline format, written after TTL-inline encoding shipped but before user-key routing, produces no mutation, because L's value equals the desired value. The row would then count as settled with nothing copied into P, and reappear on every pass. When the presence probe says P holds no version, M2 forces the metadata `Put` into P, plus the TTL-index sync. Value equality is a valid no-op only once P holds the row. The same-node path of bootjp/elastickv#1278 has the same no-op for already-inline legacy rows, and M2 applies the forced `Put` there too, so M3 can evacuate those rows as well. + +**Progress is checkpointed within the page.** Today a plan's cursor advances only after the whole page has been processed, and every plan shares one 5s tick. A page of remote-owner rows, each needing a probe and a forward, could exhaust the tick before the page ends. The next tick would then restart on the same page and repeat the same probes, starving every row behind them. M2 must: +- send one `RelocateLegacyRows` request per owner group and 32 rows, with the owner groups' requests issued concurrently; +- checkpoint only a contiguous prefix of the page. Owner groups interleave in scan order (`P1, P2, P1, …`) and their batches finish in any order, so the cursor may advance only to the last row of the longest prefix in which every row is settled. A row is settled when its transaction committed, or when it needed no work: owned, skipped, or the owner already holds a version. A tick that runs out of budget resumes behind that prefix and redoes at most the unsettled rows after it. A relocation that committed without being recorded as settled is not written twice: the redo re-probes it, finds P's version, and settles it with no work. A redo that raced past the probe conflicts on the absence condition. + +Properties: +- **Cost:** one `RelocateLegacyRows` request per 32 rows of each owner group, bounded by L's one page per tick. P's leader does the presence check and the TTL-anchor reads locally. +- **Repeated probes.** The work is not done once per row while L's copy exists. After the rewrite, the copy stays in L, and every later pass re-probes it only to learn that P now holds a version. That recurring traffic lasts until M3 removes the copy. §5 budgets it, and M2 and M3 ship together so the recurrence is short-lived. +- **No new scans.** L's leader is already scanning L. + +### C. Evacuate stale legacy copies + +Once P holds a version of the **metadata** key, L's copy of that row is shadowed for good. Compaction keeps the latest version at or below the horizon, tombstones included (`TestMVCCStore_Compact_Delete`), so the fallback never reaches L's copy again. Deleting it costs nothing in correctness, and the migrator stops rescanning and re-probing it. + +This does **not** let the dual-route fallback be retired. A collection created before user-key routing may also have field, member, score or entry rows that live only in L. Evacuating its metadata row moves none of them, and a point read such as `HGET` still needs the legacy route to find them. Retiring the fallback needs a separate migration of every row family, with its own gate. That is out of scope here (§2). + +**Enumerating the candidates.** The rows M3 deletes are exactly the base metadata rows the migrator's scan does not serve: `ScanGroupOwnedAt` skips a legacy-route row once its owner holds any version, a tombstone included. Scans that canonicalize wide-column rows drop it as well, because the owner's tombstone stops the point-read fallback. M3 therefore reads L's raw stored page, `scanRouteAtForwardStoredPage` without the point-read pass, and keeps legacy-route rows whose owner, a group other than L, holds a version at the read timestamp. Rows shadowed by a tombstone at P are candidates like any other. Rows are never candidates while L's route or the owner's route is under staged visibility (`routeHasStagedVisibility`), the same exclusion `ScanGroupOwnedAt` applies. Reading the raw page bypasses that scan's filter, so M3 applies the exclusion itself. The asynchronous local catalog watcher is only a planning hint; it cannot decide whether a replicated purge applies. + +**A replicated permit fences route changes.** Before probing or purging a batch, the catalog group's leader commits a cleanup reservation by CAS at catalog version v, covering L's raw range and each owner range the batch uses. The reservation records an immutable permit ID, v, the bounds, and their owner groups, and is granted only when none has staged visibility. Every catalog mutation path, including direct `CatalogStore.Save`, split, promotion, move and staged-visibility changes, must reject changes overlapping a reservation. L's leader then proposes an `InstallCleanupPermit` entry containing that committed grant. Every L replica persists the identical permit in its FSM and snapshots. A purge carries the permit ID and may apply only to the exact recorded bounds and owners while that permit is active in L's replicated state; it never checks its local watcher. Revocation is ordered the other way: `RevokeCleanupPermit` must apply in L and be acknowledged before the catalog group removes the reservation and permits route changes. A purge ordered before revocation uses the still-reserved mapping; a purge ordered afterward rejects everywhere. Failed coordinators leave the reservation blocking route changes until recovery completes this same revocation protocol; neither side expires a permit using wall time. Permit IDs are `(L, generation)`, with a persisted monotonic counter in the catalog group and at most one outstanding permit per L. L stores only its active permit and a `revokedThrough` high-water mark. Revocation advances that mark even if installation has not arrived; every later install at or below it rejects. A higher generation is granted only after the previous reservation is revoked, so records from old batches need no permanent per-ID tombstones. These counters and the active permit survive snapshots and cannot wrap. M1b includes these entries and their snapshot state, and M3 cannot enable until every route-mutation path enforces the reservation. + +The meta prefixes also contain nested hash, set and zset delta rows. M1, M2, M3 and D2 must classify a row as metadata only when the same base-metadata predicate used by the existing migration extractors accepts it (`extractListMetaMigrationUserKey`, `extractHashMetaMigrationUserKey`, `extractSetMetaMigrationUserKey`, `extractZSetMetaMigrationUserKey`, or `store.ExtractStreamUserKeyFromMeta`, as appropriate). A nested delta row is rejected before counting, forwarding or purging. Delta compaction remains its only owner; such a row is not migration backlog and cannot block M4. + +The delete is a write into **L** at the raw key, issued by L's leader, so it is compliant in every TSO mode. However, the normal dispatch path routes the raw key to P by its user key, and **a plain `Del` would delete P's copy**. Option C needs an explicit-group operation: "remove this raw key from group L, if L's route still covers it". `ShardStore` has explicit-group reads (`GetGroupAt`, `ScanGroupAt`) but no explicit-group write. + +**The removal is a purge, not a tombstone.** An ordinary delete leaves a tombstone in L stamped after P's version. If the user-key range later moves from P back to L, the migration imports P's versions with their original commit timestamps and merges them with L's data by newest version. L's newer tombstone then shadows the imported metadata, and the collection appears deleted after cutover. M3 therefore proposes a replicated `PurgeShadowedVersions(rawKey)` entry in L. Applying it removes L's versions of that raw key physically and writes nothing, so a later import is never shadowed. + +Removing versions changes what a read below P's earliest version would see: such a read falls back to L and would find nothing instead of L's copy. Each replica's own retention floor (`minRetainedTS`) cannot guard this, because floors are local and differ. A lagging replica that later becomes leader would still accept a read between its floor and P's earliest version, and that read needs L's copy. M3 therefore first establishes a **replicated read floor** in L: +- M1b adds a persisted replicated backup-pin set, keyed by pin ID and holding the read timestamp and deadline, plus a monotonically increasing `pinGeneration`. It is separate from the process-local `ActiveTimestampTracker`, is included in FSM snapshots, and changes only by applying `Pin`, `Extend`, `Release`, or `Expire` Raft entries. In particular, neither the tracker sweeper nor an FSM apply path removes a pin by comparing its deadline with `time.Now()`. +- A deadline is only a leader-side liveness trigger. After it passes, the current leader may propose `ExpireBackupPin(pinID, expectedDeadline)`. When that identity has an active replicated record, the FSM removes and retires it only if its expected deadline matches, then increments `pinGeneration`; a concurrently ordered `Extend` makes a stale expiry a no-op without advancing the retirement fence. When no active record exists, expiry of a current-epoch, not-yet-retired identity advances `retiredThrough` and increments `pinGeneration`, even if its initial `Pin` has not arrived. A missing identity already at or below that watermark is an idempotent no-op. No deadline comparison is possible or required for the missing-record case. Until a valid removal entry commits, every replica conservatively treats an active pin as protected. Clock skew can therefore delay reclamation but cannot make replicas disagree about retention. Explicit `Release` uses the same Raft-ordered retirement transition without a deadline condition. +- Renewal uses `Extend`, which requires the pin to exist in the replicated set and preserves its original read timestamp. M1b changes every renewal path, including `renewBackupGroups`, the baseline-building renewal and the background renewal loop: none may replay `BackupPin` to renew. If `Expire` or `Release` applies first, a later `Extend` rejects the missing pin, and the backup run fails without publishing a successful result. `Pin` is used only to begin a new backup with a new identity, never to recover an expired run. A retry of a failed or expired run starts a new backup at a fresh timestamp. +- **Delayed initial pins cannot resurrect retired identities.** A new backup first obtains `(backupEpoch, backupGeneration)` from a persisted monotonic allocator in the control group, committed before fan-out; random pin IDs alone are insufficient. Every pin, renewal, release, expiry and control-capacity reservation carries that identity. Each data group and the control-capacity FSM persist `retiredThrough` for the current epoch, alongside active records. `Release` retires the generation even when its initial `Pin` has not arrived; expiry follows the active-versus-missing rules above, including retiring a missing identity before its initial admission. Retirement removes only that identity's active record and raises `retiredThrough`. An initial `Pin` or `Reserve` with no active matching record rejects at or below that watermark, without consuming capacity or changing `pinGeneration`; an exact duplicate of a still-active record is idempotent and cannot change its read timestamp or extend its deadline. Existing active lower-generation records remain protected and may use `Extend`, even after another backup raises the watermark. A delayed, not-yet-admitted older backup may therefore conservatively fail; its retry allocates a fresh generation and read timestamp. This rejects arbitrary late initial commands without keeping a tombstone for every retired ID. The allocator, watermark and active identities survive replay and snapshots and cannot wrap. Epoch advancement requires the durable drain protocol below, and every command from an older epoch rejects permanently; wall-clock time or an RPC timeout never permits forgetting the rejection fence. +- M1b audits initial admission and compensation too: `pinBackupGroups`, reservation retries, `compensateBackupRelease`, normal end/abort, and every forwarded proposal preserve the allocated identity. Compensation targets every group to which admission was attempted, including ambiguous or unacknowledged proposals, and unreserves through the same control-group retirement fence. A lost compensation can delay reclamation until replicated expiry, but cannot justify publishing a successful backup. Generation allocation adds one control-group consensus step per new backup, not per migration page. +- Compaction obtains its group retention bound from both ordinary process-local read pins and this replicated backup-pin set, taking the oldest nonzero timestamp. It must not use the wall-clock-expiring backup portion of `ActiveTimestampTracker` as a substitute. Restoring a snapshot restores the replicated pins before compaction or backup reads can start. It also closes a per-replica recovery gate: on startup or snapshot installation, compaction and backup reads remain paused until a fresh quorum ReadIndex barrier has supplied a committed target and local apply has reached that target, including the post-snapshot log tail. The snapshot's applied index or a locally cached commit index cannot open the gate. A missing leader or unavailable barrier keeps it closed. This prevents compaction between restoring an empty pin set at index i and replaying a live pin at i+1. The same persisted set guards snapshots that would otherwise omit live backup state. A local sweeper may remove only its local bookkeeping; it cannot remove the retention protection of a replicated pin. +- Quorum freshness is required for every compaction pass, not only recovery. Each pass obtains a fresh ReadIndex barrier for that group and waits for its local apply to reach the returned target before calculating its retention bound. If that fails, the pass performs no deletion. An isolated follower cannot use equal locally cached applied/commit indexes as proof that it has seen current pins. Losing quorum never reuses a previous pass's successful barrier. +- L's leader selects T from its eligible retention horizon (the timestamp policy would permit compacting through), bounded by the oldest pin in that replicated set. This is a candidate horizon, not an already-completed local compaction watermark: compaction cannot run ahead of the replicated floor and then establish it afterward. The wall-clock-expiring backup portion of `ActiveTimestampTracker` is not an input to the replicated predicate. +- The FSM applies the raise only when `pinGeneration` still matches and no pin in the replicated set has a read timestamp below T. It does not evaluate deadlines. A mismatch rejects the raise, and the leader refreshes the pin state and retries. +- Ordinary read pins also constrain the raise. First, a Raft-ordered `PrepareReadFloor(T, prepareGeneration, membershipRevision)` closes admission of new reads below T on every replica of L. `prepareGeneration` increases monotonically for each attempt, including attempts at the same T after cancellation, and is persisted through restart and snapshots. Admission and insertion of an ordinary read pin must check that pending floor atomically, so a read cannot enter behind the closure. Each voter and learner acknowledges only after applying that exact preparation and draining its existing ordinary pins below T. Every acknowledgement and final `RaiseReadFloor` carries T, `prepareGeneration`, membership revision and the observed backup-pin generation. The FSM accepts a final raise only for the exact still-active preparation and its complete matching certificate; acknowledgements from a cancelled or superseded preparation reject even if T and membership are unchanged. An unavailable replica or membership change prevents the raise. The apply verdict depends on this replicated preparation and certificate, never on a replica's instantaneous local pins. Preparing or cancelling a raise is persisted and Raft-ordered; a restart restores the admission closure before reads resume. Cancelling reopens only timestamps allowed by the already-applied floor. A backup-pin generation change rejects the final raise and requires cancellation or a new preparation. M1b includes this read-admission and drain protocol, including every read entry point. +- Applying a replicated backup-pin entry rejects a new pin below the current replicated read floor. Because pin lifecycle changes and floor raises are ordered Raft entries, a pin ordered before the raise makes the raise reject, while a raise ordered before the pin makes the pin reject. There is no ordering in which a pin below an already-applied floor can become active. +- Every replica records T when it applies the entry. From then on, every read in L below T fails with `ErrReadTSCompacted`, on that replica and on any future leader. +- The floor only rises. + +**Compaction uses that same admission boundary.** In every group using the new retention protocol, a pass may physically compact only at or below its applied replicated read floor, further bounded by ordinary local pins and replicated backup pins. To advance beyond it, the leader first completes the prepare/drain/raise protocol above using a fresh eligible policy horizon. No pass may sample `OldestForGroup` and then delete above the applied floor. A read pin racing the sample is either already covered by the drained preparation or admitted only at or above the applied floor, where the retained anchor remains readable. This also closes admission of a new backup pin below a compaction target before deletion begins. Compaction errors do not lower the floor or reopen old read admission. A group whose floor has not activated performs no new-protocol reclamation until its first successful preparation and raise. + +This only makes reads fail earlier than a lagging replica's local floor would, which the retention window already allows. + +Only after the floor has applied does L's leader propose purges, and only for rows whose owner's earliest version is at or below T. A read that could still fall back to L is then below T and refused everywhere. The batched probe M3 uses must return that earliest version timestamp along with presence. This needs a new response field, which old servers leave unset, so a row whose probe does not report it is not purged. Every replica evaluates the entries using only replicated state; no local clock or local retention floor changes the apply verdict. + +**The floor, purge and deterministic pin-lifecycle entries ship in M1b.** `RaiseReadFloor`, `PurgeShadowedVersions`, the persisted pin set, and `ExpireBackupPin` are FSM changes like the absence condition. Their decoders, snapshot state and apply logic are part of M1b, so M1b's capability marker, downgrade fence and admission rules cover them. A replica running an M1b binary can apply every entry an M3 leader proposes. A separate M3 marker would otherwise be needed, and is not. + +**Activation drains legacy backups.** Binary support alone does not activate deterministic retention. After every serving node, voter and learner supports the cutover protocol, operators commit a drain epoch and membership revision in the control group, close logical-backup admission on every node and stop backup producers. Each node persists and fsyncs its closed-admission epoch before acknowledging it; a restart must reload that closure before exposing backup APIs. A node unable to prove its drain/activation state starts with backup admission closed. In-flight baseline builders, renewers, readers and publishers must finish or be cancelled, and every legacy session must end with its releases committed in all participating groups. Drain acknowledgements are tied to that epoch and membership revision; a membership change invalidates the set, and a joining node cannot serve backup APIs until it has durably installed the closed epoch. An empty wall-clock-filtered tracker is not evidence of a drain. During this maintenance interval, a missing node or an outstanding session blocks activation. Mixed-version replicas continue using the legacy format and cannot write the new replicated set. After the drain completes, a Raft-ordered activation records a new backup-token epoch, rejects all pre-activation tokens, and enables the new pin format. Only proof that this activation has applied can reopen admission. `RaiseReadFloor` and M3 remain disabled until then; a restart preserves the closed or activated epoch and cannot accept a delayed legacy renewal or publication. This is a coordinated cutover, not an automatic conversion of process-local pins. + +**M3 progress is checkpointed within the page, like M2's.** A raw page full of shadowed rows needs a presence probe and a delete per row. Done one row at a time, that could exhaust the shared 5s tick before the page ends, and every later tick would restart on the same page. M3 therefore: +- probes in batches per owner group (`RawLatestCommitTS` with a `key_batch`), issued concurrently across groups; +- purges in batches of at most 32 keys. Every key in a batch lives in L, so one explicit-group `PurgeShadowedVersions` entry by L's leader covers it; +- keeps its own cursor per plan and advances it only over the longest contiguous prefix of settled rows. A row is settled when its purge committed or it is not a candidate; +- runs as its own entries in the TTL-inline plan rotation, not after the M2 plans in a fixed order, so M2's recurring probes can never use up every tick before M3 starts (§4). + +**Tick slot.** The plan rotation alone is not enough. Today `DeltaCompactor.SyncOnce` runs the delta-compaction handlers (`compactBackgroundHandlers`) first and calls `migrateTTLInlineOnce` only if the shared 5s budget is still live. Under sustained delta-compaction load, every TTL-inline plan is skipped, M2's and M3's included, and so is the migrator's paging that M1 counts. M1 therefore merges the two loops into one top-level rotation: the delta handlers and the migration plans take turns from one cursor, and a tick that runs out of budget hands the next tick the entry after the last one it ran. A busy delta handler then delays migration by at most one rotation and can no longer starve it. The change also covers bootjp/elastickv#1278's existing plans, which have the same exposure. + +The cursor is in memory, like the existing plan cursors. A leader change restarts the pass, which redoes only idempotent work: purging an absent row is a no-op. + +**Why not delete in the relocation transaction (B′).** L's leader could relocate into P and delete L's copy in one transaction, which would save M3's second probe per row. That transaction writes two groups, so it is a cross-group 2PC with the same blockers as A′: +- its commit timestamp needs the unfinished A0/A0b 2PC allocation, or Phase D; +- it needs a 2PC participant that writes an explicit group. + +A separate single-group delete by L's leader is compliant in every TSO mode and needs no 2PC. The price is one batched probe per owner group per page in the pass after the rewrite. + +### D. Measure first + +Measurement alone migrates nothing, but it tells operators whether options A to C matter for their cluster and gives Phase 3 a concrete gate. Three forms: +- **D1, background counters.** On each group's leader, count the legacy-route rows the scan serves or skips, by owner group and reason, plus a "full pass finished" signal per plan. No extra reads (M1). +- **D2, one-shot audit.** An admin RPC on each group's leader runs the same classification once over the four meta ranges at one `readTS`, paging from start to end with no rewrites. It returns counts by bucket and owner group. It reads the whole legacy range once, locally, and finishes in minutes rather than a background pass spread over many 30s ticks. A run that long must hold its snapshot: + - **Read pin.** The run pins its `readTS` in the node's `ActiveTimestampTracker` (`Pin`, released when the run ends or fails), which the FSM compactor honours. Otherwise compaction could drop a version a later page needs. + - **One catalog version.** The run records the catalog version at its start and classifies every page under it. If the catalog moves before the run ends, the run is discarded and reports "restarted", the same rule M1 applies to a background pass. Otherwise pages classified under different route maps could add up to a false zero. + - **Leadership.** A leadership change of the group mid-run discards the run too. + + A discarded run publishes nothing, so it can never feed M4 or a decision to defer M2. Like D1 it makes no remote probes, so rows whose owner is led elsewhere are counted without the "holds a version" split. With `--probe`, it adds batched presence probes per owner group. Those probes read P, which the audit's local pin does not protect: P's retention horizon can pass the audit's `readTS`, and a probe at that `readTS` would fail with `ErrReadTSCompacted`. So the probes do not use the pinned `readTS`. Each asks P's leader at a fresh read timestamp of its own. Presence only grows: compaction keeps the latest version at or below the horizon, tombstones included, so once P holds a version it keeps one. A later probe can therefore only move a row from "P holds nothing" to "P holds a version", never back. The split is informational. Neither M4 nor the M2-deferral rule uses it, since both require the whole owner-led-elsewhere bucket to be zero. +- **D3, offline count.** `cmd/elastickv-snapshot-decode` already reads a group's `.fsm` files. Counting legacy-route rows also needs the route catalog, which lives in the default group's snapshot. This touches no live node, but the snapshot is only as current as its last checkpoint, and no snapshot can see whether a remote owner holds the key. + +M2 can wait only while no row needs a relocation that the same-node path could miss. A zero owner-led-elsewhere count says only where leaders sit now. If rows are still served (owner led locally and holding nothing) and P's leadership later moves away from L's node, those rows become owner-led-elsewhere rows: the same-node path skips them, and M3 cannot delete them because P is empty. So M2 may be deferred on a cluster only once D1 or D2 reports zero in both the owner-led-elsewhere and the served buckets, and the decision is re-checked at every published M1 snapshot. If either bucket turns nonzero, M2 ships. M4 enforces the same thing, because it requires both buckets to be zero. M3 is still needed in that case. It evacuates the copies the same-node path of bootjp/elastickv#1278 leaves in L, and M4's "owner led locally and holds a version" bucket must reach zero. So M3 would ship without M2, but never without M1b: the absence guard on the same-node path is what keeps those relocations from overwriting a user write that landed below `readTS`. + +## 4. Proposal + +Every milestone below applies only to base metadata rows accepted by the family-specific migration extractor. Nested delta rows are excluded before M1/D2 counting, M2 forwarding and M3 probing or purging. + +| Milestone | Change | Gated on | +|---|---|---| +| M1 | **Measure.** Per-group-leader counts of the legacy-route rows the scan serves or reports as skipped (§3 B), split into: served (owner led locally, holds nothing), owner led locally and holds a version, owner led elsewhere, deferred by staged visibility. Without a remote probe, M1 cannot tell whether a remote owner holds the key; M2 adds that split. Counts accumulate per pass of each plan (one plan per family: hash, set, zset, stream). When that plan's cursor wraps, they are published atomically with the pass's completion epoch and the catalog version the pass ran under. A pass that sees the catalog version change restarts. A leadership change discards the in-progress pass and invalidates the group's published snapshots, since a new leader starts new passes. Expose the published snapshots, keyed by group and plan, as metrics and in an admin RPC. The same RPC runs the one-shot audit (§3 D2). M1 also puts the migration plans into the compactor's top-level rotation (§3 C, "Tick slot"), since its counts come from the migrator's own passes. | bootjp/elastickv#1278 | +| M1b | **Absence guard and deterministic retention state.** The apply-time absence condition (§3 B) as a new mutation field, plus M3's persisted replicated backup-pin set and `Pin`/`Extend`/`Release`/`ExpireBackupPin`, `RaiseReadFloor`, and `PurgeShadowedVersions` entries (§3 C). Pin activity is decided only by applied entries, never replica-local wall time. All decoders, snapshot state and apply logic share one capability marker on every voter and learner, the downgrade fence in each replica's `raft-engine` marker, and the refusal of replicas that lack the capability. The absence condition applies to every relocation, starting with the same-node path of bootjp/elastickv#1278, which also gets the forced `Put`. It needs no cross-leader forwarding and ships on its own. | M1 | +| M2 | **Option B.** L's leader takes the owner-led-elsewhere rows its scan reports as skipped and sends them to each owner group's leader in `RelocateLegacyRows` requests of at most 32 rows, concurrently across owner groups. P's leader checks presence, reads the TTL anchors, builds the rewrite and dispatches it as the leader of its own keys. Each transaction is bound to the catalog version its owners were resolved under, and is repartitioned or refused once the catalog moves. A relocation forces the `Put` when the owner holds nothing, and carries M1b's apply-time absence condition (§3 B). The plan cursor advances only over the contiguous prefix of settled rows. It extends M1's counts with "owner led elsewhere and holds a version". Disabled, with the row skipped as today, until P has replicated M1b's capability marker (§3 B). | M1; M1b; the `RelocateLegacyRows` RPC | +| M3 | **Option C.** By L's leader, in a later pass: enumerate L's raw page and purge L's metadata copy with an explicit-group `PurgeShadowedVersions` entry, which writes no tombstone. It purges when the current owner is a group other than L, holds a version, and that version is at or below L's replicated read floor (`RaiseReadFloor`, proposed first), at the purge's route version. Ships together with M2, which otherwise re-probes every rewritten row on every pass. M3's plans (one per meta family and legacy group) join the same flattened rotation as the TTL-inline plans (`migrateTTLInlineOnce`, `ttlInlinePlanCursor`), so an M2 plan that uses up a tick's budget hands the next tick the plan after it, and M3 gets one page per full rotation. Otherwise a fixed order would let a recurring M2 workload starve M3, and since M2 leaves every row in L, the design would never converge. If M1 shows that M2 can wait (§3 D), M3 ships without M2 for the copies the same-node path leaves, but always after M1b. | M1; M1b; an explicit-group delete path | +| M4 | **Legacy-route gate for Phase 3.** For every group and every plan, the latest published M1 snapshot must meet all of these:
• it is valid and has a completion epoch from after the rollout;
• it reports zero in every bucket: owner led elsewhere (holding a version or not), served but unmigrated, owner led locally and holding a version, and deferred by staged visibility. A nonzero locally shadowed bucket means M3 has not yet evacuated rows whose owner shares L's leader;
• it carries the same catalog version as every other gated snapshot, and that version is still current.
Counts, epoch and catalog version come from the same pass, so in-progress zeroes after a reset never satisfy the gate. A row moved between groups by a catalog change cannot hide between two passes either. This is **one additional** Phase 3 condition, not the whole gate. The TTL-inline design's other prerequisites still apply: every node has migrated, and legacy bare string keys, which no prefix scan covers, have been ruled out or rewritten. | M1 and M3; M2 wherever M1 reports owner-led-elsewhere rows | + +Option A is rejected. It restores the cost profile a36dac7f removed and never converges, because the copies it reads are never removed. A′ converges, but it still reads every row about N times per pass, and it needs the unfinished cross-group 2PC timestamp work or Phase D. + +## 5. Cost + +Per tick (30s, shared 5s budget, at most 512 physical rows per plan page), with N nodes, G groups, and G_L groups holding legacy rows: + +| Option | Who scans | Remote reads per tick, cluster-wide | Writes per tick | Converges | +|---|---|---|---|---| +| A | every node, each legacy group it does not lead | up to 4 · N · G_L page RPCs (≤ 2,048 · N · G_L rows) | per useful page: ≤ Σ_P ⌈n_P / 32⌉ ≤ 16 + G relocation txns, one owner group each | no: L's copies stay, so the cost is permanent | +| A′ | as A | as A, until the range is empty | as A (≤ 16 + G per page), each a cross-group 2PC writing L and one P | yes, after about N · B row reads in total | +| B (M2) | L's leader only, its existing pages | per page: ≤ Σ_P ⌈n_P / 32⌉ ≤ 16 + G `RelocateLegacyRows` requests; P's leader reads presence and TTL anchors locally | per page: one one-phase txn per request, each in one owner group | yes, with M3 | +| C (M3) | L's leader only, its raw page | per page: ≤ min(G, 512) batched probes, plus current-voter/learner floor-drain acknowledgements when the floor advances | per page: ≤ 16 explicit-group purges plus 4 permit-lifecycle consensus writes; a floor advance adds 2 consensus writes (prepare and raise) and any explicit cancellation | yes: each row is purged once; control steps also consume the tick budget | +| D1 (M1) | L's leader only, its existing pages | none | none | not applicable | +| D2 | each group's leader, once | none (or batched probes with `--probe`) | none | not applicable | + +For A, A′, B and C, the 16 is ⌈512 / 32⌉, where n_P is the number of a page's rows owned by group P. The extra G comes from a page split across owner groups, each owner group's last chunk being partial. A page whose 512 rows all belong to one remote owner therefore costs 16 `RelocateLegacyRows` requests, not one. C's probes are not chunked: one batched probe carries up to 1,025 keys, so C needs at most one per owner group per page. Unlike A and A′, neither depends on N: a node with no legacy group does no extra work. + +**Convergence.** B and C keep the existing page cap: one legacy group handles at most 512 rows per meta family per 30s tick, an ideal ceiling of about 61,000 rows per family per hour. A backlog of 1,000,000 hash meta rows therefore needs at least about 1,954 M2 page opportunities, roughly 16 hours, plus M3's work. This is only a lower bound on completion time, not a throughput prediction: the shared rotation, four serial permit-lifecycle writes per M3 page, floor preparation/drain/raise, retries and one-minute timeout backoff can require multiple ticks per page. D2 sizes the backlog, while completed-row and control-phase latency measurements estimate operational convergence before M4 is used. The probes, forwards and control transitions all consume the same five-second tick context; a timed-out step resumes from persisted control state on a later rotation. + +- **M1:** no remote reads. The counters classify rows the page already reads, by an in-memory route lookup plus, for a locally led owner, the local version probe bootjp/elastickv#1278 already performs. Whether a remote owner holds the key is left to M2, which issues the probe anyway. +- **M2:** per page (≤ 512 rows per plan per tick), one `RelocateLegacyRows` request per 32 of each remote owner group's rows, issued concurrently across owner groups. P's leader serves each with local reads only: one presence check and up to two TTL-anchor reads per row, all in its own store. Batching, and building the rewrite on P, are convergence requirements, not optimisations: sequential per-row RPCs, including the TTL-anchor reads `buildElems` makes, could exhaust the tick on one page every time. +- **M2 before M3 evacuates:** every pass sends each rewritten row still present in L to P's leader again, which answers "already held". That is up to Σ_P ⌈n_P / 32⌉ `RelocateLegacyRows` requests per page, the same as the first pass, recurring for the backlog's size until M3 removes the copies. With M2 and M3 shipped together, a row is re-probed at most until the pass after its rewrite. If M3 is disabled or failing, M1's "owner holds a newer version" counter shows the recurring set. +- **M3:** one permit covers the entire raw page of at most 512 rows, not each 32-key purge. The page needs a catalog reservation commit, permit install in L, at most one batched presence RPC per remote owner group, at most 16 purge entries, revocation in L, and a catalog release commit. These are up to 20 consensus writes, or 22 when the page also advances its floor; floor-drain acknowledgements and retries add latency. Persist the active page/permit identity and completed control phase so a tick can stop after any step, including after the last purge but before revocation, without starting another permit or forgetting to release the reservation. Only one permit is outstanding per L; another migration plan may use a later tick while this page waits, but cannot acquire a second permit for L. After restart, recover and finish or revoke the recorded permit before selecting a fresh page. Each metadata row is evacuated once. After evacuation, L's legacy metadata range is empty and the TTL-inline plans read nothing there. Collection field, member, score and entry rows stay in L (§3 C). + +## 6. Correctness + +- **OCC:** the rewrite's `StartTS` is the page's `readTS`. A user write to the key at P after `readTS` makes the rewrite conflict, and the row is reconsidered on a later pass. +- **Writes below `readTS` (G0 across groups):** `readTS` is the maximum applied watermark over all groups. P can therefore accept a user write with a commit timestamp at or below `readTS` after the probe has answered, and `latest > StartTS` would not catch it. The apply-time absence condition (§3 B) rejects the relocation whenever P holds any version. That closes this case for relocations without a per-row read-index round trip, and without waiting for A0's cross-group fix. Rewrites of rows P already owns keep plain OCC, and stay exposed to G0 like every other transaction until A0 lands. +- **Presence-check races:** if P gains a version between the probe and the commit, the rewrite conflicts with it, because it writes the same key. If P's leadership moves, the probe is answered only by a current leader (it reports "cannot answer" otherwise), and the forward is re-routed or fails. +- **Timestamps:** never issued by L's leader for P's key. P's leader dispatches every relocation of its own keys: its HLC stamps them in legacy mode, and the TSO under Phase D. +- **Staged visibility:** rows whose route, at L or P, is under staged visibility stay deferred, as in bootjp/elastickv#1278. M1 counts them, so the M4 gate does not pass while any remain. +- **Catalog changes during M2:** a batch whose owners were resolved under an older catalog version is repartitioned before dispatch, or refused by P's leader. No relocation therefore writes a group other than its dispatching leader's own. +- **Catalog changes during the gate:** each M1 snapshot carries its pass's catalog version, and M4 requires one current version across all of them. A row moved into a group after that group's pass completed fails the gate until every pass has run again under the new version. +- **M3 safety:** delete L's copy only when, at the delete's own read timestamp and route version: + - the key's current owner is a group other than L; + - neither L's route nor the owner's route is under staged visibility; + - that owner holds a version of the key; + - L's route still covers the raw key; + - the owner's earliest version is at or below L's replicated read floor, which every replica and future leader enforces, so no served read can still fall back to L's copy. + + The removal is a purge, never a tombstone, so a later move of the range back into L cannot be shadowed by it. + + The explicit-group purge carries its replicated permit and route version. The catalog reservation blocks overlapping route changes until L applies revocation; a later purge then refuses using L's replicated state. It never makes an apply-time decision from the asynchronous local watcher. This prevents a catalog change from making L the owner while an older decision deletes its only current metadata copy. + +## 7. Test plan + +- `kv`: a real split `ShardStore`, with P behind a `followerProxyEngine` whose leader is a fake RawKV server. Cover the batched probe (for M3) answering "none", "has a version" and "cannot answer". Cover `RelocateLegacyRows` on P's leader: relocated, already held, conflicted, and refused on a catalog move or on a non-leader. The commit timestamp comes from P's leader in legacy mode and from the TSO under Phase D. Building the rewrite reads the TTL anchors only from P's own store: a counting store on L must see no `redisTTLKey` reads for cross-leader rows. +- `kv`, batched probe through M2's client: per-key answers of `RawLatestCommitTS` with a `key_batch` match the single-key probe for a mixed batch (absent, visible, tombstoned, newer than `ts`), and a non-leader reports "cannot answer" for the whole batch. The existing S3 auxiliary probe tests cover the server; this covers the migrator's use of it per owner group. +- `kv`, absence condition (M1b), written first as a failing reproduction in the style of `TestApplyOrder_GlobalSnapshotMissesLaggingGroupWrite`: + - two groups; a user write to P's key at `c1` is parked before its proposal while another group's write at `c2 > c1` applies; + - the migrator takes `readTS = c2`, the probe reports P empty, and the parked write lands on P; + - the relocation must then conflict, and P must keep the user's value. + + Before the condition exists, the test shows the user's write overwritten. Cover the same-node path too. A relocation applied before a parked user write must make that write conflict. A group with a voter or a learner lacking the capability must keep M2 off, and once P's capability marker is in place, adding a replica that lacks the capability is refused. A replica whose `raft-engine` marker carries the fence, restarted on a binary from before M1b, fails at startup. A learner added by a conf change that applies between the leader's capability check and the marker makes the marker apply as a no-op. A node that joins after activation writes the fenced marker into P's new directory before its first snapshot restore. +- `adapter`: compactor tests where L and P have different leaders. The legacy row is migrated into P exactly once; a concurrent user write at P wins. One page holding rows of two remote owner groups produces one transaction per owner group. With interleaved owners (`P1, P2, P1`), a tick whose budget runs out after P1's batch commits but before P2's resumes behind the settled prefix and never skips the P2 row. A group with more than 32 rows on the page commits in several transactions. An already-inline legacy row whose owner holds nothing is copied into P, on both the cross-leader and the same-node path. A split of P between grouping and dispatch repartitions the batch, and P's leader refuses a forward carrying a key it no longer owns. +- `kv`: the group scan reports every skipped legacy-route row with its owner group and reason (owner led elsewhere, owner holds a version, staged) and does not serve it. +- M1: + - a reset pass never publishes zeroes before its completion; + - a leadership change invalidates the snapshots; + - each plan publishes its own snapshot; + - a catalog change during a pass restarts it. +- Plan rotation: with M2 plans whose pages always use up the tick's budget, M3's plans still each run once per rotation, and the backlog of evacuable rows shrinks. +- Top-level rotation (M1): with a delta-compaction handler that always uses up the tick's budget, every migration plan still runs once per top-level rotation, and M1 still publishes its pass snapshots. Before the change, the same setup runs no migration plan at all. +- M1b ordering: with M1b and M3 but no M2, a same-node relocation whose owner received a parked user write below `readTS` conflicts, and M3 then finds the owner holding the user's version and evacuates only L's copy. A build with M3 and without M1b must not be possible to enable. +- M3 progress: a tick whose budget runs out mid-page resumes behind the settled prefix and still reaches the rows after it. A page with more than 32 candidates deletes them in several transactions. +- M4: the gate fails when one plan of a group still has rows while the others report zero, and when only the "owner led locally and holds a version" bucket is nonzero. It also fails when snapshots carry different catalog versions, or a version that is no longer current. It does not replace the bare-string prerequisite. +- M3: + - an explicit-group purge removes L's copy only, never P's, and writes no tombstone; + - it refuses after the route moves, and when the current owner has become L; + - with local catalog watchers deliberately at different versions, L replicas apply the same permit and purge entries identically; overlapping catalog changes block until L's replicated revocation, stale grant installation rejects after revocation, and crash recovery preserves reservations and permits; + - many completed cleanup batches leave only one active permit and one revocation high-water mark per L, while arbitrarily delayed old installs still reject after snapshot restore; + - rows shadowed by a tombstone at P are found by the raw enumeration and evacuated; + - after the purge, moving the user-key range from P back to L imports P's metadata and the collection stays readable (reproduce first with an ordinary delete, which shadows the import); + - a row whose owner's earliest version is above L's replicated read floor is not purged until the floor passes it; + - a replicated backup pin ordered before `RaiseReadFloor(T, pinGeneration)` makes the stale-generation raise reject, while a pin below T ordered after the raise rejects; after refreshing the generation, a raise never passes an older active pin; + - two replicas applying the same pin and floor entries with clocks on opposite sides of the pin deadline reach the same decision: the pin remains active until `ExpireBackupPin` applies; + - an `Extend` ordered before `ExpireBackupPin(pinID, expectedDeadline)` makes that stale expiry a no-op, while an expiry ordered first removes the old pin and increments `pinGeneration`; replay and snapshot restore preserve the same result; + - an expiry followed by each renewal path rejects the missing pin and prevents backup publication, even if compaction has already discarded the original read timestamp; no renewal path emits a new `Pin`; + - order a compensating `Release` before an ambiguous initial `Pin` or its retry, then replay that delayed pin before and after snapshot restore: it rejects without using a backup slot or changing `pinGeneration`; cover expiry and control `Unreserve`/delayed `Reserve` too, including groups whose admission acknowledgement was lost; + - expire a missing current-epoch identity before its initial `Pin`: retirement advances once, repeated expiry is idempotent, and the delayed initial admission rejects before and after restore; contrast an active record whose deadline was extended, where stale expiry neither removes the pin nor advances the watermark; + - retire a higher generation while an older generation is already active: the older pin remains protected and extendable, but an older admission not yet applied rejects and restarts at a fresh generation; many retired backups leave only active records and one watermark per group, with old-epoch commands still rejecting after drain and epoch advancement; + - with a live pre-M1b backup, missing drain acknowledgement or mixed-version member, activation and floor raises remain disabled; after drain and activation, old tokens, delayed renewals and delayed publication attempts reject across restart; + - after acknowledging drain, a node restart keeps backup admission closed before its listener starts; a joining node or membership revision change prevents activation until new durable acknowledgements are collected; + - compaction on replicas whose clocks straddle a pin deadline retains the backup's versions until a replicated release or expiry applies, while still honoring ordinary local read pins; snapshot restore installs the same protection before compaction resumes; + - install a snapshot at i without a pin, pause replay before its committed `Pin` at i+1, and prove that compaction and backup reads stay blocked until a fresh ReadIndex target is applied; no local applied/commit watermark can bypass this gate; + - partition a follower after a successful compaction barrier, create and renew a backup on the quorum beyond the retention window, and prove every follower compaction pass refuses to delete until a new barrier and the missing pins have applied; + - hold an ordinary read pin below T on another replica: preparation blocks new low-timestamp reads, the raise waits for the old pin to drain, and replay/restart/membership changes cannot bypass that wait; + - cancel a fully acknowledged preparation, admit a new read below T, and prepare T again at the same membership revision: the old acknowledgement certificate must reject until the new preparation generation has drained that read; + - race pin admission against compaction target selection: deletion never exceeds the applied replicated floor, and reads admitted during the pass retain their anchor; no standalone sample-then-delete path may bypass the floor protocol; + - exhaust the tick budget after each permit control step, including between final purge and revocation, then restart: the page resumes without another permit for L and eventually releases the catalog reservation. Convergence reporting includes all control writes and drain/probe time; + - after `RaiseReadFloor(T, pinGeneration)` applies, a follower whose local `minRetainedTS` is below T refuses reads below T, and keeps refusing them after it becomes leader; + - an M3 leader's purge applies on a replica running the M1b binary, and the M1b marker does not activate on a group with a replica that cannot decode both new entry types. +- Base-metadata filtering: pages containing hash, set and zset nested delta rows count, forward and purge only the base metadata rows. Delta rows produce no M1/D2 counters, no M2 RPC and no M3 purge, and cannot keep M4 nonzero. +- M1 one-shot audit (D2): on a real split `ShardStore`, it returns the same buckets as a completed background pass over the same data, and it writes nothing. It holds an `ActiveTimestampTracker` pin at its `readTS` for the whole run and releases it on success and on failure; a compaction run during the audit removes no version the audit still reads. A catalog change or leadership change mid-run discards the run and publishes nothing. With `--probe`, a compaction on P past the audit's `readTS` does not fail the run, because the probes read P at fresh timestamps. +- Jepsen: the Redis workloads with a seeded legacy-route backlog and leader shuffles, checking linearizable reads of the affected keys during migration and evacuation. + - The CI workloads write only inline-format data, so they never exercise this path (bootjp/elastickv#1278 notes the same). Any routed write of a legacy key lands in P, so a routed write cannot seed L. M3 adds only an explicit-group purge in L, not a write. Seeding therefore needs either a test-only explicit-group put into L, behind a flag that production builds do not expose, or a pre-built snapshot fixture that already holds the legacy-route rows. + - Place L's and P's leaders on different nodes (`cmd/raftadmin` leadership transfer), then run hash, set, zset and stream operations with TTLs on the seeded keys under the partition and leader-shuffle nemeses. + - Check that reads, including `TTL`, stay linearizable while M2 and M3 run, and that every M1 bucket ends at zero. + - Also run the existing `elastickv.redis-workload` and `elastickv.redis-zset-safety-workload` to show no regression. + +## Decisions + +- **The cross-leader case is handled outside bootjp/elastickv#1278.** That PR keeps leaving such rows untouched, and Phase 3 stays blocked until this design lands. (Maintainer decision, 2026-10-17.) +- Proposed, pending review: reject option A. Implement option D first, then B and C together. B builds and dispatches each relocation on the owner group's leader, so it needs neither A0b nor Phase D. +- Proposed, pending review: reject A′. Deleting in the same transaction makes owner-side scans converge, but they keep the N-fold read cost, and the cross-group 2PC needs commit-timestamp work A0/A0b has not finished. +- Proposed, pending review: evacuation is a separate single-group delete by L's leader (M3), not part of the relocation transaction (B′). It stays compliant in every TSO mode and needs no 2PC, at the cost of one probe per row in a later pass. +- Proposed, pending review: a relocation is guarded by an apply-time absence condition, not only by OCC at `readTS`. This closes G0 across groups for relocations, on both the cross-leader and the same-node path, without a per-row fence. +- Proposed, pending review: M1 ships first, with the one-shot audit (D2) and the top-level rotation. Its results decide whether M2 is needed on the clusters that matter. M3 is needed either way. +- Proposed, pending review: P's leader builds and dispatches cross-leader relocations (`RelocateLegacyRows`), from the raw rows L's leader sends. Building them on L's leader would cost up to two remote TTL-anchor reads per row and would need A0b or Phase D for the commit timestamp. +- Proposed, pending review: M2 is deferred on a cluster only while both the owner-led-elsewhere and the served buckets are zero, re-checked at every published M1 snapshot, because a leadership move turns served rows into cross-leader rows. +- Proposed, pending review: the absence guard is its own milestone (M1b), independent of cross-leader forwarding. M3 never ships before it, because the same-node path already relocates rows that M3 then deletes from L. +- Proposed, pending review: retiring the dual-route fallback is not a goal of this design. It needs a migration of every row family of pre-routing collections, not only their metadata. + +## Open questions + +1. Should `RelocateLegacyRows` be its own `Internal` RPC, or a mode of `Forward` that carries rows instead of mutations? +2. Where does the explicit-group purge belong: a `ShardStore` method behind a coordinator capability, or a dedicated internal RPC? +3. Should the absence condition be a field on `pb.Mutation`, or a transaction-level list of keys that must be absent? The field is simpler; the list keeps the condition out of every other mutation path. +4. Should the replicated read floor be group-wide, as proposed, or scoped to the purged raw keys, so that other reads in L keep their full retention window? The alternative to purging at all is teaching the import path to ignore tombstones written by evacuation. + +## Implementation progress + +This proposal's design review is not implementation acceptance. The runtime changes below are being consolidated in this PR; no complete milestone or operational migration is claimed yet. + +| Component | Current implementation evidence | Remaining acceptance | +|---|---|---| +| M1 physical-page reporting | `ShardStore.ScanGroupOwnedPageAt` returns served rows, four legacy base-metadata dispositions, catalog version and the physical resume key without remote owner probes. Delta and migration plans now share one timeout-aware rotation; the regression exhausts every slot's tick and still reaches every migration family. | Production reporting integration, valid completed snapshots, status and one-shot audit. | +| M1b atomic absence | Store apply-lock checks include old committed versions and tombstones on memory/Pebble and entry-scoped batches. Protobuf, coordinator and 1PC preserve the condition, including duplicate-key mutation reduction. Store/KV race suites and scoped lint passed. | Durable replicated capability/activation is not implemented yet, so guarded requests remain refused by default. Retention state, downgrade fencing, drain, read-admission and compaction requirements above remain mandatory. | +| M1b durable generation pins | Implemented deterministic admission, extension, retirement and expected-deadline expiry transitions, including retirement before initial admission and protection of active lower generations. The durable journal saves state with its Raft index, reloads on an already-applied disk index, and does not publish a failed write. Its control-group allocator saves a monotonic counter before returning an identity, rejects unallocated reservations, and never guesses an old lost allocation response. Memory/Pebble snapshot, reopen, ordering, encrypted-state and race tests cover these primitives. No wall clock or user HLC allocation is used. | Activation, allocator fan-out, token format, all renewal/compensation callers, drain proof, snapshot recovery barriers and read-floor/compaction integration remain mandatory. A test-only seeded epoch is not activation evidence. | +| M1b typed FSM integration | Added a strict versioned `0x16` command decoder (moved from `0x0f`, which main assigned to the migration target-readiness opcode), typed allocation responses, durable pin apply, and journal reload at startup/snapshot restore. The highest admitted pin timestamp persists after release/expiry, fences delayed writes after restart, and invalidates timestamp caches without adding a volatile backup pin. Legacy commands reject once an activated epoch exists. New-epoch state cannot use legacy compaction while its quorum/floor protocol is unavailable. | No production producer or activation path emits this format. Both capability and a persisted activated epoch are required. Capability certificates, downgrade fencing, drain/activation, backup APIs and the full per-pass ReadIndex/prepare/drain/raise protocol are not implemented yet; reclamation for such an epoch remains closed rather than approximating them. | +| M2-M4 | Not implemented. | Peer relocation, bounded checkpointing, reserved purge, completed-snapshot gate and seeded Jepsen acceptance. Both read fallbacks stay enabled. | + +Existing pass metrics and bookkeeping are reused from bootjp/elastickv#1283 as a stack dependency; they do not implement cross-leader migration or M1's classification snapshots. No duplicate metric families are added here. diff --git a/kv/absence_guard.go b/kv/absence_guard.go new file mode 100644 index 000000000..d603c5e39 --- /dev/null +++ b/kv/absence_guard.go @@ -0,0 +1,50 @@ +package kv + +import ( + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" +) + +var ErrTTLInlineCapabilityRequired = errors.New("TTL-inline replicated capability is not active") + +// Relocations are single-owner 1PCs. Refuse other phases explicitly instead +// of allowing an intent encoder or raw request path to discard the condition. +func (f *kvFSM) validateAbsenceRequest(request *pb.Request) error { + for _, mutation := range request.GetMutations() { + if mutation == nil || !mutation.RequireAbsent { + continue + } + if !f.ttlInlineCapability.Load() { + return errors.WithStack(ErrTTLInlineCapabilityRequired) + } + if !request.IsTxn || request.Phase != pb.Phase_NONE || mutation.Op != pb.Op_PUT || isTxnInternalKey(mutation.Key) { + return errors.WithStack(ErrInvalidRequest) + } + } + return nil +} + +func validateAbsenceMutationKinds(mutations []*pb.Mutation) error { + for _, mutation := range mutations { + if mutation.GetRequireAbsent() && (mutation.Op != pb.Op_PUT || isTxnInternalKey(mutation.Key)) { + return errors.WithStack(ErrInvalidRequest) + } + } + return nil +} + +// refuseAbsenceProposal keeps require_absent out of the log entirely until +// M1b fences every voter and learner onto the capability. A pre-capability +// binary decodes the field as unknown and applies the PUT that a current FSM +// refuses, so proposing it in a mixed-version group diverges replicas at the +// same applied index. Internal.Forward hands caller-supplied mutations here. +func refuseAbsenceProposal(reqs []*pb.Request) error { + for _, req := range reqs { + for _, mutation := range req.GetMutations() { + if mutation.GetRequireAbsent() { + return errors.WithStack(ErrTTLInlineCapabilityRequired) + } + } + } + return nil +} diff --git a/kv/absence_guard_test.go b/kv/absence_guard_test.go new file mode 100644 index 000000000..529a1d023 --- /dev/null +++ b/kv/absence_guard_test.go @@ -0,0 +1,159 @@ +package kv + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func absenceRequest(key []byte, guarded bool) *pb.Request { + return &pb.Request{IsTxn: true, Phase: pb.Phase_NONE, Ts: 100, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key, CommitTS: 101})}, + {Op: pb.Op_PUT, Key: key, Value: []byte("migration"), RequireAbsent: guarded}, + }} +} + +func TestAbsenceGuardRejectsBelowSnapshotUserWrite(t *testing.T) { + t.Parallel() + for _, deleted := range []bool{false, true} { + t.Run(map[bool]string{false: "value", true: "tombstone"}[deleted], func(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := store.NewMVCCStore() + f, ok := NewKvFSMWithHLC(s, NewHLC()).(*kvFSM) + require.True(t, ok) + // Fixture for a committed capability; runtime activation is separately gated. + f.ttlInlineCapability.Store(true) + key := []byte("metadata") + reader, ok := s.(store.VersionPresenceReader) + require.True(t, ok) + exists, err := reader.VersionExistsAtOrBefore(ctx, key, 100) + require.NoError(t, err) + require.False(t, exists) + // Another group's watermark gave readTS=100 while this write was parked. + require.NoError(t, s.PutAt(ctx, key, []byte("user"), 5, 0)) + if deleted { + require.NoError(t, s.DeleteAt(ctx, key, 6)) + } + require.ErrorIs(t, applyFSMRequest(t, f, absenceRequest(key, true)), store.ErrWriteConflict) + got, err := s.GetAt(ctx, key, 200) + if deleted { + require.ErrorIs(t, err, store.ErrKeyNotFound) + } else { + require.NoError(t, err) + require.Equal(t, []byte("user"), got) + } + }) + } +} + +func TestAbsenceGuardCapabilityAndPhaseAdmission(t *testing.T) { + t.Parallel() + s := store.NewMVCCStore() + f, ok := NewKvFSMWithHLC(s, NewHLC()).(*kvFSM) + require.True(t, ok) + key := []byte("metadata") + require.ErrorIs(t, applyFSMRequest(t, f, absenceRequest(key, true)), ErrTTLInlineCapabilityRequired) + f.ttlInlineCapability.Store(true) + for _, phase := range []pb.Phase{pb.Phase_PREPARE, pb.Phase_COMMIT, pb.Phase_ABORT} { + r := absenceRequest(key, true) + r.Phase = phase + require.ErrorIs(t, f.handleRequest(context.Background(), r, 101), ErrInvalidRequest) + } + raw := absenceRequest(key, true) + raw.IsTxn = false + require.ErrorIs(t, f.handleRequest(context.Background(), raw, 101), ErrInvalidRequest) + require.NoError(t, applyFSMRequest(t, f, absenceRequest(key, true))) + got, err := s.GetAt(context.Background(), key, 200) + require.NoError(t, err) + require.Equal(t, []byte("migration"), got) +} + +func TestAbsenceGuardRawBatchRefusalDoesNotHalt(t *testing.T) { + t.Parallel() + dir := t.TempDir() + s, err := store.NewPebbleStore(dir) + require.NoError(t, err) + closed := false + t.Cleanup(func() { + if !closed { + _ = s.Close() + } + }) + f := g17FSM(t, s) + guarded := g17RawPut(10, "guarded", "blocked") + guarded.Mutations[0].RequireAbsent = true + response := f.Apply(g17Entry(t, guarded, g17RawPut(11, "ordinary", "accepted"))) + result, ok := response.(*fsmApplyResponse) + require.True(t, ok, "a deterministic capability refusal must not halt apply: %T", response) + require.Len(t, result.results, 2) + require.ErrorIs(t, result.results[0], ErrTTLInlineCapabilityRequired) + require.NoError(t, result.results[1]) + require.False(t, g17Exists(t, s, "guarded")) + require.True(t, g17Exists(t, s, "ordinary")) + index, present := g17AppliedIndex(t, s) + require.True(t, present) + require.Equal(t, g17EntryIndex, index) + require.NoError(t, s.Close()) + closed = true + + reopened := g17OpenStore(t, dir) + index, present = g17AppliedIndex(t, reopened) + require.True(t, present) + require.Equal(t, g17EntryIndex, index, "restart must not replay a poison entry") + require.True(t, g17Exists(t, reopened, "ordinary")) + next := g17FSM(t, reopened) + next.SetApplyIndex(index + 1) + require.Nil(t, next.Apply(g17Entry(t, g17RawPut(12, "next", "accepted")))) +} + +func TestAbsenceGuardLeavesLegacyRequestsUnchanged(t *testing.T) { + t.Parallel() + s := store.NewMVCCStore() + f, ok := NewKvFSMWithHLC(s, NewHLC()).(*kvFSM) + require.True(t, ok) + key := []byte("metadata") + require.NoError(t, applyFSMRequest(t, f, absenceRequest(key, false))) + got, err := s.GetAt(context.Background(), key, 200) + require.NoError(t, err) + require.Equal(t, []byte("migration"), got) +} + +func TestAbsenceGuardRejectsDuplicateThatChangesOperation(t *testing.T) { + t.Parallel() + s := store.NewMVCCStore() + f, ok := NewKvFSMWithHLC(s, NewHLC()).(*kvFSM) + require.True(t, ok) + f.ttlInlineCapability.Store(true) + key := []byte("metadata") + r := absenceRequest(key, true) + r.Mutations = append(r.Mutations, &pb.Mutation{Op: pb.Op_DEL, Key: key}) + require.ErrorIs(t, applyFSMRequest(t, f, r), ErrInvalidRequest) + _, exists, err := s.LatestCommitTS(context.Background(), key) + require.NoError(t, err) + require.False(t, exists) +} + +func TestAbsenceGuardSurvivesWireAndDuplicateMutations(t *testing.T) { + t.Parallel() + first := elemToMutation(&Elem[OP]{Op: Put, Key: []byte("metadata"), Value: []byte("first"), RequireAbsent: true}) + encoded, err := proto.Marshal(first) + require.NoError(t, err) + var decoded pb.Mutation + require.NoError(t, proto.Unmarshal(encoded, &decoded)) + require.True(t, decoded.RequireAbsent) + last := &pb.Mutation{Op: pb.Op_PUT, Key: first.Key, Value: []byte("last")} + mutations, err := uniqueMutations([]*pb.Mutation{&decoded, last}) + require.NoError(t, err) + require.Len(t, mutations, 1) + require.True(t, mutations[0].RequireAbsent) + require.Equal(t, []byte("last"), mutations[0].Value) + require.False(t, last.RequireAbsent, "dedup must not mutate the original request") + stored, err := toStoreMutations(mutations) + require.NoError(t, err) + require.True(t, stored[0].RequireAbsent) +} diff --git a/kv/absence_proposal_gate_test.go b/kv/absence_proposal_gate_test.go new file mode 100644 index 000000000..9d459a25c --- /dev/null +++ b/kv/absence_proposal_gate_test.go @@ -0,0 +1,49 @@ +package kv + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +// Pre-capability binaries decode require_absent as an unknown field and apply +// the PUT, so an entry carrying it must never reach the log before every +// member is fenced onto the capability. Internal.Forward reaches Commit with +// caller-supplied mutations, so the refusal has to happen before proposing. +func TestTransactionManagerRefusesAbsenceGuardBeforeProposal(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + req *pb.Request + }{ + { + name: "one-phase txn", + req: &pb.Request{IsTxn: true, Phase: pb.Phase_NONE, Ts: 10, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("v"), RequireAbsent: true}, + }}, + }, + { + name: "raw", + req: &pb.Request{Phase: pb.Phase_NONE, Ts: 10, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("v"), RequireAbsent: true}, + }}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + proposer := &fakeProposer{} + tm := NewTransactionWithProposer(proposer) + t.Cleanup(tm.Close) + + _, err := tm.Commit(context.Background(), []*pb.Request{tc.req}) + require.True(t, errors.Is(err, ErrTTLInlineCapabilityRequired), "got %v", err) + require.Zero(t, proposer.calls.Load()) + require.Zero(t, proposer.adminCalls.Load()) + }) + } +} diff --git a/kv/active_timestamp_tracker.go b/kv/active_timestamp_tracker.go index 1aa9ef334..9176ce4c2 100644 --- a/kv/active_timestamp_tracker.go +++ b/kv/active_timestamp_tracker.go @@ -119,6 +119,20 @@ func (t *ActiveTimestampTracker) SetBackupTimestampFloorObserver(observer func(u t.mu.Unlock() } +// Replicated pin retention stays in the journal; only its timestamp-cache +// invalidation notification is shared with legacy backup machinery. +func (t *ActiveTimestampTracker) observeReplicatedBackupTimestampFloor(readTS uint64) { + if t == nil { + return + } + t.mu.Lock() + observer := t.backupFloorObserver + t.mu.Unlock() + if observer != nil { + observer(readTS) + } +} + func (t *ActiveTimestampTracker) Pin(ts uint64) *ActiveTimestampToken { if t == nil || ts == 0 || ts == ^uint64(0) { return &ActiveTimestampToken{} diff --git a/kv/compactor.go b/kv/compactor.go index 89af85b39..a5250dfdf 100644 --- a/kv/compactor.go +++ b/kv/compactor.go @@ -244,6 +244,9 @@ func (c *FSMCompactor) compactRuntime(ctx context.Context, runtime FSMCompactRun compactCtx, cancel := c.compactContext(ctx, status) defer cancel() + if err := requireLegacyCompactionState(compactCtx, runtime.Store); err != nil { + return errors.Wrapf(err, "compact group %d retention protocol", runtime.GroupID) + } if err := runtime.Store.Compact(compactCtx, safeMinTS); err != nil { c.handleCompactError(ctx, runtime.GroupID, status, compactCtx, err) @@ -260,6 +263,23 @@ func (c *FSMCompactor) compactRuntime(ctx context.Context, runtime FSMCompactRun return nil } +// New-protocol state must never fall back to the wall-clock-expiring legacy +// tracker. Until the ReadIndex/prepare/drain/floor path is wired, its persisted +// epoch keeps reclamation closed, even with an empty active set or after restore. +func requireLegacyCompactionState(ctx context.Context, st store.MVCCStore) error { + if _, supported := st.(store.ReplicatedControlStore); !supported { + return nil + } + journal, err := openReplicatedPinJournal(ctx, st) + if err != nil { + return errors.Wrap(err, "load replicated retention state") + } + if journal.snapshot().Pins.Epoch != 0 { + return errors.Wrap(ErrTTLInlineCapabilityRequired, "replicated retention requires a fresh quorum barrier and prepared read floor") + } + return nil +} + func migrationRetentionMinTS(ctx context.Context, st store.MVCCStore, safeMinTS uint64) (uint64, error) { reader, ok := st.(store.MigrationTargetReadinessReader) if !ok { diff --git a/kv/coordinator.go b/kv/coordinator.go index 0bd456794..6d508a6d4 100644 --- a/kv/coordinator.go +++ b/kv/coordinator.go @@ -1625,16 +1625,19 @@ func elemToMutation(req *Elem[OP]) *pb.Mutation { Key: req.Key, Value: value, CommitTsValueOffset: req.CommitTSValueOffset, + RequireAbsent: req.RequireAbsent, } case Del: return &pb.Mutation{ - Op: pb.Op_DEL, - Key: req.Key, + Op: pb.Op_DEL, + Key: req.Key, + RequireAbsent: req.RequireAbsent, } case DelPrefix: return &pb.Mutation{ - Op: pb.Op_DEL_PREFIX, - Key: req.Key, // prefix (may be empty for "all keys") + Op: pb.Op_DEL_PREFIX, + Key: req.Key, // prefix (may be empty for "all keys") + RequireAbsent: req.RequireAbsent, } } panic("unreachable") diff --git a/kv/fsm.go b/kv/fsm.go index 6a92d7258..323af3043 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -40,6 +40,8 @@ func observeStoreLastCommitTS(hlc *HLC, st store.MVCCStore) { type kvFSM struct { store store.MVCCStore log *slog.Logger + // Set only by the replicated protocol activation, never by a process flag. + ttlInlineCapability atomic.Bool // hlc is the shared HLC instance updated when a HLC lease entry is applied. // May be nil for nodes that do not participate in physical ceiling tracking. hlc *HLC @@ -114,6 +116,8 @@ type kvFSM struct { // timestamped before the backup cut and reached apply too late. backupTimestampFloor atomic.Uint64 backupFloorLoadErr error + replicatedPins *replicatedPinJournal + replicatedPinLoadErr error prepareEvidenceEnabled atomic.Bool prepareCreationSealed atomic.Bool prepareEvidenceReleaseFloor atomic.Uint64 @@ -313,6 +317,7 @@ func NewKvFSMWithHLC(store store.MVCCStore, hlc *HLC, opts ...FSMOption) FSM { opt(f) } f.backupFloorLoadErr = f.reloadBackupTimestampFloor(context.Background()) + f.replicatedPinLoadErr = f.reloadReplicatedBackupPins(context.Background()) f.snapLatch.log = f.log observeStoreLastCommitTS(hlc, store) return f @@ -374,6 +379,9 @@ type fsmApplyResponse struct { } func (f *kvFSM) Apply(data []byte) any { + if f.replicatedPinLoadErr != nil { + return replicatedBackupHalt(f.replicatedPinLoadErr, "load journal") + } if f.backupFloorLoadErr != nil { return haltErr(errors.Wrap(errors.Mark(f.backupFloorLoadErr, ErrBackupApply), "kv/fsm: load backup timestamp floor")) } @@ -478,6 +486,7 @@ func isRawApplyVerdict(err error) bool { errors.Is(err, ErrMigrationTransferTooLarge) || errors.Is(err, ErrRouteCutoverPending) || errors.Is(err, ErrUnknownRequestType) || + errors.Is(err, ErrTTLInlineCapabilityRequired) || errors.Is(err, ErrBackupTimestampFenced) || errors.Is(err, ErrPrepareCreationSealed) || isRawRouteVerdict(err) || @@ -539,12 +548,13 @@ func (f *kvFSM) applyReservedOpcode(ctx context.Context, data []byte) (any, bool switch { case data[0] == raftEncodeHLCLease: return f.applyHLCLease(data[1:]), true - case data[0] == raftEncodeBackup: - return f.applyBackup(data[1:]), true case data[0] >= fsmwire.OpEncryptionMin && data[0] <= fsmwire.OpEncryptionMax: return f.applyEncryption(f.pendingApplyIdx, data[0], data[1:]), true default: - return f.applyMigrationOpcode(ctx, data) + if resp, handled := f.applyMigrationOpcode(ctx, data); handled { + return resp, true + } + return f.applyBackupOpcode(ctx, data[0], data[1:]) } } @@ -780,6 +790,9 @@ func (f *kvFSM) applyRequestErr(ctx context.Context, r *pb.Request) error { } func (f *kvFSM) handleRequest(ctx context.Context, r *pb.Request, commitTS uint64) error { + if err := f.validateAbsenceRequest(r); err != nil { + return err + } switch { case r.IsTxn: return f.handleTxnRequest(ctx, r, commitTS) @@ -1563,6 +1576,10 @@ func (f *kvFSM) restoreSnapshot(r io.Reader, restoreStore func(io.Reader) error) if f.backupFloorLoadErr != nil { return errors.Wrap(f.backupFloorLoadErr, "restore backup timestamp floor") } + f.replicatedPinLoadErr = f.reloadReplicatedBackupPins(context.Background()) + if f.replicatedPinLoadErr != nil { + return errors.Wrap(f.replicatedPinLoadErr, "restore replicated backup pins") + } observeStoreLastCommitTS(f.hlc, f.store) return nil } @@ -1971,7 +1988,7 @@ func uniqueMutations(muts []*pb.Mutation) ([]*pb.Mutation, error) { if len(muts) == 0 { return []*pb.Mutation{}, nil } - seen := make(map[string]struct{}, len(muts)) + seen := make(map[string]int, len(muts)) reversed := make([]*pb.Mutation, 0, len(muts)) // Keep the last mutation per key to avoid dropping final operations like // PUT followed by DEL in the same transactional batch. @@ -1981,10 +1998,15 @@ func uniqueMutations(muts []*pb.Mutation) ([]*pb.Mutation, error) { return nil, errors.WithStack(ErrInvalidRequest) } keyStr := string(mut.Key) - if _, ok := seen[keyStr]; ok { + if index, ok := seen[keyStr]; ok { + if mut.RequireAbsent { + kept := proto.CloneOf(reversed[index]) + kept.RequireAbsent = true + reversed[index] = kept + } continue } - seen[keyStr] = struct{}{} + seen[keyStr] = len(reversed) reversed = append(reversed, mut) } @@ -2243,7 +2265,7 @@ func (f *kvFSM) rejectReservedControlMutations(muts []*pb.Mutation) error { if mut == nil { continue } - if f.isMigrationOnlyControlKeyForApply(mut.Key) { + if f.isMigrationOnlyControlKeyForApply(mut.Key) || store.IsReplicatedControlKey(mut.Key) { return errors.WithStack(ErrInvalidRequest) } } @@ -3106,19 +3128,24 @@ func txnLockConflictError(lockBytes []byte, key, primaryKey []byte, startTS uint } func toStoreMutations(muts []*pb.Mutation) ([]*store.KVPairMutation, error) { + if err := validateAbsenceMutationKinds(muts); err != nil { + return nil, err + } out := make([]*store.KVPairMutation, 0, len(muts)) for _, mut := range muts { switch mut.Op { case pb.Op_PUT: out = append(out, &store.KVPairMutation{ - Op: store.OpTypePut, - Key: mut.Key, - Value: mut.Value, + Op: store.OpTypePut, + Key: mut.Key, + Value: mut.Value, + RequireAbsent: mut.RequireAbsent, }) case pb.Op_DEL: out = append(out, &store.KVPairMutation{ - Op: store.OpTypeDelete, - Key: mut.Key, + Op: store.OpTypeDelete, + Key: mut.Key, + RequireAbsent: mut.RequireAbsent, }) case pb.Op_DEL_PREFIX: return nil, ErrUnknownRequestType diff --git a/kv/fsm_backup.go b/kv/fsm_backup.go index 02bcefafc..cdc86e3c6 100644 --- a/kv/fsm_backup.go +++ b/kv/fsm_backup.go @@ -29,15 +29,11 @@ var backupTimestampFloorKey = []byte(txnBackupPrefix + "timestamp_floor") const backupTimestampFloorValueSize = 8 func (f *kvFSM) applyBackup(data []byte) any { - if len(data) != 0 && isPrepareEvidenceControlSubtype(data[0]) { - entry, err := decodeBackupPayload(data) - if err == nil { - err = f.applyPrepareEvidenceControl(entry) - } - if err != nil { - return haltErr(errors.Wrap(err, "apply prepare evidence fence")) - } - return nil + if result, handled := f.applyPrepareEvidenceBackup(data); handled { + return result + } + if f.replicatedPins != nil && f.replicatedPins.snapshot().Pins.Epoch != 0 { + return errors.Wrap(ErrInvalidBackupPin, "legacy backup command rejected after epoch activation") } if f.readTracker == nil { return haltErr(errors.Wrap(ErrBackupApply, "kv/fsm: backup entry arrived but no ActiveTimestampTracker is wired")) @@ -56,6 +52,20 @@ func (f *kvFSM) applyBackup(data []byte) any { return haltErr(errors.Wrap(errors.Mark(err, ErrBackupApply), "kv/fsm: apply backup entry")) } +func (f *kvFSM) applyPrepareEvidenceBackup(data []byte) (any, bool) { + if len(data) == 0 || !isPrepareEvidenceControlSubtype(data[0]) { + return nil, false + } + entry, err := decodeBackupPayload(data) + if err == nil { + err = f.applyPrepareEvidenceControl(entry) + } + if err != nil { + return haltErr(errors.Wrap(err, "apply prepare evidence fence")), true + } + return nil, true +} + func (f *kvFSM) applyDecodedBackup(entry backupEntry) error { switch entry.subtype { case backupSubtypePin: diff --git a/kv/fsm_replicated_backup.go b/kv/fsm_replicated_backup.go new file mode 100644 index 000000000..43ecf08ee --- /dev/null +++ b/kv/fsm_replicated_backup.go @@ -0,0 +1,100 @@ +package kv + +import ( + "context" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +var ErrReplicatedBackupApply = errors.New("replicated backup state apply failed") + +// ReplicatedBackupAllocation is returned only after the control-group journal +// has durably saved the identity at this entry's applied index. +type ReplicatedBackupAllocation struct { + Identity BackupIdentity +} + +func (f *kvFSM) applyBackupOpcode(ctx context.Context, opcode byte, payload []byte) (any, bool) { + switch opcode { + case raftEncodeBackup: + return f.applyBackup(payload), true + case raftEncodeReplicatedBackup: + return f.applyReplicatedBackup(ctx, payload), true + default: + return nil, false + } +} + +func (f *kvFSM) reloadReplicatedBackupPins(ctx context.Context) error { + if f.store == nil { + f.replicatedPins = nil + return nil + } + if _, supported := f.store.(store.ReplicatedControlStore); !supported { + f.replicatedPins = nil + return nil + } + journal, err := openReplicatedPinJournal(ctx, f.store) + if err != nil { + return err + } + f.replicatedPins = journal + f.observeReplicatedBackupWriteFloor() + return nil +} + +func (f *kvFSM) observeReplicatedBackupWriteFloor() { + floor := f.replicatedPins.snapshot().Pins.WriteTimestampFloor + if floor <= f.backupTimestampFloor.Load() { + return + } + f.backupTimestampFloor.Store(floor) + _ = f.observeBackupReadTimestamp(floor, nil) + f.readTracker.observeReplicatedBackupTimestampFloor(floor) +} + +func (f *kvFSM) applyReplicatedBackup(ctx context.Context, payload []byte) any { + command, err := decodeReplicatedBackupCommand(payload) + if err != nil { + return replicatedBackupHalt(err, "decode command") + } + if f.replicatedPins == nil || f.pendingApplyIdx == 0 { + return replicatedBackupHalt(errors.New("missing journal or Raft applied index"), "apply command") + } + if !f.ttlInlineCapability.Load() || f.replicatedPins.snapshot().Pins.Epoch == 0 { + // Binary support is not activation. Only a future configuration-bound + // capability marker and complete durable drain may create the epoch. + return errors.WithStack(ErrTTLInlineCapabilityRequired) + } + if command.Operation == ReplicatedBackupAllocate { + identity, err := f.replicatedPins.allocate(ctx, f.pendingApplyIdx, command.Identity.Epoch) + if err != nil { + return replicatedBackupApplyError(err) + } + return &ReplicatedBackupAllocation{Identity: identity} + } + err = f.replicatedPins.apply(ctx, f.pendingApplyIdx, replicatedPinCommand{ + Operation: replicatedPinOperation(command.Operation), Identity: command.Identity, + PinID: command.PinID, ReadTS: command.ReadTS, Deadline: command.DeadlineMillis, + ExpectedDeadline: command.ExpectedDeadlineMillis, + }) + if err == nil { + f.observeReplicatedBackupWriteFloor() + } + return replicatedBackupApplyError(err) +} + +func replicatedBackupApplyError(err error) any { + if err == nil { + return nil + } + if errors.Is(err, ErrReplicatedBackupPin) || errors.Is(err, ErrTooManyActiveBackups) { + return err + } + return replicatedBackupHalt(err, "persist command") +} + +func replicatedBackupHalt(err error, operation string) any { + return haltErr(errors.Wrap(errors.Mark(err, ErrReplicatedBackupApply), "kv/fsm: replicated backup "+operation)) +} diff --git a/kv/fsm_replicated_backup_test.go b/kv/fsm_replicated_backup_test.go new file mode 100644 index 000000000..8089b2afa --- /dev/null +++ b/kv/fsm_replicated_backup_test.go @@ -0,0 +1,239 @@ +package kv + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func applyReplicatedBackupCommand(t *testing.T, f *kvFSM, index uint64, command ReplicatedBackupCommand) any { + t.Helper() + payload, err := EncodeReplicatedBackupCommand(command) + require.NoError(t, err) + f.SetApplyIndex(index) + require.False(t, f.IsVolatileOnlyPayload(payload), "pin commands persist their own state/index atomically") + return f.Apply(payload) +} + +func activatedPinFSM(t *testing.T, st store.MVCCStore) *kvFSM { + t.Helper() + f := g17FSM(t, st) + // Test fixture only: production activation still requires its capability + // certificate and the durable legacy drain, neither supplied by this flag. + f.ttlInlineCapability.Store(true) + return f +} + +func TestFSMReplicatedBackupRequiresCapabilityAndActivatedEpoch(t *testing.T) { + t.Parallel() + st := g17OpenStore(t, t.TempDir()) + f := g17FSM(t, st) + command := replicatedBackupWireFixtures()[0] + response := applyReplicatedBackupCommand(t, f, 2, command) + err, ok := response.(error) + require.True(t, ok) + require.ErrorIs(t, err, ErrTTLInlineCapabilityRequired) + f.ttlInlineCapability.Store(true) + response = applyReplicatedBackupCommand(t, f, 3, command) + err, ok = response.(error) + require.True(t, ok) + require.ErrorIs(t, err, ErrTTLInlineCapabilityRequired, "binary support cannot manufacture an epoch") +} + +func TestFSMReplicatedBackupCoexistsWithPrepareEvidence(t *testing.T) { + t.Parallel() + for _, backend := range []string{"memory", "pebble"} { + t.Run(backend, func(t *testing.T) { + t.Parallel() + open := func() store.MVCCStore { + if backend == "pebble" { + return g17OpenStore(t, t.TempDir()) + } + st := store.NewMVCCStore() + t.Cleanup(func() { require.NoError(t, st.Close()) }) + return st + } + st := open() + seedActivatedPinJournal(t, st) + f := activatedPinFSM(t, st) + pin := replicatedBackupWireFixtures()[0] + payload, err := EncodeReplicatedBackupCommand(pin) + require.NoError(t, err) + commands := [][]byte{EncodePrepareEvidenceFence(0), EncodePrepareEvidenceSeal(), payload, + EncodePrepareEvidenceFence(pin.ReadTS), EncodePrepareEvidenceRelease(pin.ReadTS)} + for index, command := range commands { + f.SetApplyIndex(uint64(index) + 2) + require.Nil(t, f.Apply(command), "timestamp controls are not legacy backup lifecycle commands") + } + snapshot, err := f.Snapshot() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, snapshot.Close()) }) + var image bytes.Buffer + _, err = snapshot.WriteTo(&image) + require.NoError(t, err) + restored := activatedPinFSM(t, open()) + require.NoError(t, restored.Restore(&image)) + for _, current := range []*kvFSM{f, restored} { + require.True(t, current.prepareEvidenceEnabled.Load()) + require.False(t, current.prepareCreationSealed.Load()) + require.Equal(t, pin.ReadTS, current.prepareEvidenceReleaseFloor.Load()) + require.Equal(t, pin.ReadTS, current.prepareEvidenceTimestampFloor.Load()) + require.Equal(t, f.replicatedPins.snapshot(), current.replicatedPins.snapshot()) + legacy := EncodeBackupReleaseEntry(BackupReleaseEntry{PinID: pin.PinID}) + response, ok := current.Apply(legacy).(error) + require.True(t, ok) + require.ErrorIs(t, response, ErrInvalidBackupPin) + } + }) + } +} + +func TestFSMReplicatedBackupPreservesRetentionAndWriteFenceThroughRestore(t *testing.T) { + t.Parallel() + st := g17OpenStore(t, t.TempDir()) + seedActivatedPinJournal(t, st) + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + var observed uint64 + tracker.SetBackupTimestampFloorObserver(func(ts uint64) { observed = ts }) + f := activatedPinFSM(t, st) + f.readTracker = tracker + commands := replicatedBackupWireFixtures() + require.Nil(t, applyReplicatedBackupCommand(t, f, 2, commands[0])) + require.Equal(t, uint64(100), observed) + require.Zero(t, tracker.ActiveBackupPinCount(), "the local sweeper must not own replicated retention") + require.Nil(t, applyReplicatedBackupCommand(t, f, 3, commands[1])) + require.Nil(t, applyReplicatedBackupCommand(t, f, 4, commands[3])) + require.Equal(t, uint64(100), f.replicatedPins.snapshot().Pins.oldest(), "stale expiry must not remove an extended pin") + require.Nil(t, applyReplicatedBackupCommand(t, f, 5, commands[2])) + require.Empty(t, f.replicatedPins.snapshot().Pins.Active) + require.Equal(t, uint64(100), f.replicatedPins.snapshot().Pins.WriteTimestampFloor) + response := applyReplicatedBackupCommand(t, f, 6, commands[0]) + err, ok := response.(error) + require.True(t, ok) + require.ErrorIs(t, err, ErrReplicatedBackupPin) + snapshot, err := f.Snapshot() + require.NoError(t, err) + defer snapshot.Close() + var data bytes.Buffer + _, err = snapshot.WriteTo(&data) + require.NoError(t, err) + target := activatedPinFSM(t, g17OpenStore(t, t.TempDir())) + require.NoError(t, target.Restore(&data)) + require.Equal(t, f.replicatedPins.snapshot(), target.replicatedPins.snapshot()) + for _, current := range []*kvFSM{f, target} { + require.Equal(t, uint64(100), current.backupTimestampFloor.Load()) + current.SetApplyIndex(7) + response := current.Apply(g17Entry(t, g17RawPut(99, "late", "refused"))) + err, ok := response.(error) + require.True(t, ok) + require.ErrorIs(t, err, ErrBackupTimestampFenced) + legacy := EncodeBackupReleaseEntry(BackupReleaseEntry{PinID: commands[0].PinID}) + response = current.Apply(legacy) + err, ok = response.(error) + require.True(t, ok) + require.ErrorIs(t, err, ErrInvalidBackupPin) + } +} + +func TestFSMReplicatedBackupAllocationAndOperationalFailure(t *testing.T) { + t.Parallel() + st := store.NewMVCCStore() + t.Cleanup(func() { _ = st.Close() }) + seedPinJournalRecord(t, st, replicatedPinStateRecord{ + Version: replicatedPinStateVersion, AppliedIndex: 1, Allocator: true, + Pins: replicatedBackupPins{Epoch: 1, Capacity: 1}, + }) + f := activatedPinFSM(t, st) + response := applyReplicatedBackupCommand(t, f, 2, replicatedBackupWireFixtures()[4]) + allocation, ok := response.(*ReplicatedBackupAllocation) + require.True(t, ok) + require.Equal(t, BackupIdentity{Epoch: 1, Generation: 1}, allocation.Identity) + failure := errors.New("disk failed") + broken := activatedPinFSM(t, &failingPinControlStore{MVCCStore: st, err: failure}) + before := broken.replicatedPins.snapshot() + response = applyReplicatedBackupCommand(t, broken, 3, replicatedBackupWireFixtures()[4]) + require.ErrorIs(t, haltApplyOf(response), failure) + require.Equal(t, before, broken.replicatedPins.snapshot()) + require.Equal(t, before, newPinJournal(t, st).snapshot()) +} + +func TestFSMReplicatedBackupMalformedCommandAndCorruptStateHalt(t *testing.T) { + t.Parallel() + st := store.NewMVCCStore() + t.Cleanup(func() { _ = st.Close() }) + f := g17FSM(t, st) + require.True(t, errors.Is(haltApplyOf(f.Apply([]byte{raftEncodeReplicatedBackup, 0xff})), ErrReplicatedBackupApply)) + writer, ok := st.(store.ReplicatedControlStore) + require.True(t, ok) + _, err := writer.ApplyReplicatedControlState(context.Background(), replicatedPinStateKey, []byte("corrupt"), 1) + require.NoError(t, err) + f = g17FSM(t, st) + require.True(t, errors.Is(haltApplyOf(f.Apply(g17TwoPutEntry(t))), ErrReplicatedBackupApply)) +} + +func TestFSMReplicatedBackupReopenKeepsReleasedWriteFloor(t *testing.T) { + t.Parallel() + dir := t.TempDir() + st, err := store.NewPebbleStore(dir) + require.NoError(t, err) + closed := false + t.Cleanup(func() { + if !closed { + _ = st.Close() + } + }) + seedActivatedPinJournal(t, st) + f := activatedPinFSM(t, st) + commands := replicatedBackupWireFixtures() + require.Nil(t, applyReplicatedBackupCommand(t, f, 2, commands[0])) + require.Nil(t, applyReplicatedBackupCommand(t, f, 3, commands[2])) + require.NoError(t, st.Close()) + closed = true + reopened := g17FSM(t, g17OpenStore(t, dir)) + require.False(t, reopened.ttlInlineCapability.Load(), "loading state must not manufacture a capability certificate") + require.Empty(t, reopened.replicatedPins.snapshot().Pins.Active) + require.Equal(t, uint64(100), reopened.backupTimestampFloor.Load()) + reopened.SetApplyIndex(4) + response := reopened.Apply(g17Entry(t, g17RawPut(99, "late", "refused"), g17RawPut(101, "new", "accepted"))) + results, ok := response.(*fsmApplyResponse) + require.True(t, ok) + require.ErrorIs(t, results.results[0], ErrBackupTimestampFenced) + require.NoError(t, results.results[1]) + require.True(t, g17Exists(t, reopened.store, "new")) +} + +func TestFSMCompactorRefusesActivatedJournalWithoutFloorProtocol(t *testing.T) { + t.Parallel() + for _, withPin := range []bool{false, true} { + t.Run(map[bool]string{false: "empty active set", true: "active pin"}[withPin], func(t *testing.T) { + t.Parallel() + st := store.NewMVCCStore() + t.Cleanup(func() { _ = st.Close() }) + seedActivatedPinJournal(t, st) + if withPin { + f := activatedPinFSM(t, st) + require.Nil(t, applyReplicatedBackupCommand(t, f, 2, replicatedBackupWireFixtures()[0])) + } + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("key"), []byte("old"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("key"), []byte("new"), 30, 0)) + compactor := NewFSMCompactor([]FSMCompactRuntime{{ + GroupID: 1, Store: st, + StatusReader: fakeRaftStatus{status: raftengine.Status{ + State: raftengine.StateFollower, AppliedIndex: 10, CommitIndex: 10, + }}, + }}, WithFSMCompactorRetentionWindow(time.Millisecond)) + require.ErrorIs(t, compactor.SyncOnce(ctx), ErrTTLInlineCapabilityRequired, + "matching cached apply/commit indexes cannot replace a fresh quorum barrier") + got, err := st.GetAt(ctx, []byte("key"), 10) + require.NoError(t, err) + require.Equal(t, []byte("old"), got) + }) + } +} diff --git a/kv/fsm_reserved_control_test.go b/kv/fsm_reserved_control_test.go index 0ea483582..06ad88125 100644 --- a/kv/fsm_reserved_control_test.go +++ b/kv/fsm_reserved_control_test.go @@ -31,6 +31,7 @@ func TestValidateRawMutationRejectsReservedControlKeys(t *testing.T) { []byte("!migstage|7|victim"), []byte("!migwrite|7"), []byte("!migfence|7"), + []byte(store.ReplicatedControlPrefix + "ttl-inline"), } { err := f.validateRawMutationForApply(ctx, &pb.Mutation{Op: pb.Op_PUT, Key: key, Value: []byte("v")}, nil, 10, nil) require.ErrorIs(t, err, ErrInvalidRequest, "key %q must be refused", key) @@ -106,6 +107,7 @@ func TestTxnMutationHelpersRejectReservedControlKeys(t *testing.T) { distribution.MigrationStagedDataKey(7, []byte("victim")), []byte("!migwrite|7"), []byte("!migfence|7"), + []byte(store.ReplicatedControlPrefix + "ttl-inline"), } { muts := []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}} _, err := f.uniqueMutationsAboveFloor(context.Background(), muts, 10) @@ -135,3 +137,16 @@ func TestTxnMutationHelpersRejectReservedControlKeys(t *testing.T) { require.NoError(t, err, "key %q must be accepted", key) } } + +func TestWholeKeyspaceDeletePreservesReplicatedControlState(t *testing.T) { + t.Parallel() + st := g17OpenStore(t, t.TempDir()) + seedActivatedPinJournal(t, st) + before := newPinJournal(t, st).snapshot() + f := g17FSM(t, st) + require.Nil(t, f.Apply(g17Entry(t, g17RawPut(10, "ordinary", "value")))) + f.SetApplyIndex(g17EntryIndex + 1) + require.Nil(t, f.Apply(g17Entry(t, g17RawDelPrefix(11, "")))) + require.False(t, g17Exists(t, st, "ordinary")) + require.Equal(t, before, newPinJournal(t, st).snapshot()) +} diff --git a/kv/fsm_reserved_opcode_test.go b/kv/fsm_reserved_opcode_test.go new file mode 100644 index 000000000..3fbf1a3df --- /dev/null +++ b/kv/fsm_reserved_opcode_test.go @@ -0,0 +1,37 @@ +package kv + +import ( + "testing" + + "github.com/bootjp/elastickv/internal/encryption/fsmwire" + "github.com/stretchr/testify/require" +) + +// applyReservedOpcode dispatches on the first byte, so two features sharing a +// value would route committed entries to the wrong apply path. Branches that +// assign opcodes independently collide silently at merge time; this pins them. +func TestKVFSMReservedOpcodesAreDistinct(t *testing.T) { + t.Parallel() + + opcodes := map[string]byte{ + "single": raftEncodeSingle, + "batch": raftEncodeBatch, + "hlc lease": raftEncodeHLCLease, + "migration import": raftEncodeMigrationImport, + "migration promote": raftEncodeMigrationPromote, + "migration retire": raftEncodeMigrationRetire, + "migration cleanup": raftEncodeMigrationCleanup, + "backup": raftEncodeBackup, + "target readiness": raftEncodeTargetReadiness, + "replicated backup": raftEncodeReplicatedBackup, + } + owner := make(map[byte]string, len(opcodes)) + for name, op := range opcodes { + require.Falsef(t, op >= fsmwire.OpEncryptionMin && op <= fsmwire.OpEncryptionMax, + "%s opcode 0x%02x is inside the encryption range", name, op) + if prev, dup := owner[op]; dup { + t.Fatalf("%s and %s share opcode 0x%02x", prev, name, op) + } + owner[op] = name + } +} diff --git a/kv/legacy_metadata_scan.go b/kv/legacy_metadata_scan.go new file mode 100644 index 000000000..74791c7d4 --- /dev/null +++ b/kv/legacy_metadata_scan.go @@ -0,0 +1,184 @@ +package kv + +import ( + "context" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +const legacyMetadataScanMaxRows = 512 + +// LegacyMetadataRowReason describes a legacy-location base metadata row. +type LegacyMetadataRowReason string + +const ( + // LegacyMetadataServed is a fallback row whose locally led owner is empty. + LegacyMetadataServed LegacyMetadataRowReason = "served" + // LegacyMetadataOwnerHasVersion includes an owner's visible tombstone. + LegacyMetadataOwnerHasVersion LegacyMetadataRowReason = "owner_has_version" + // LegacyMetadataOwnerLedElsewhere requires no remote presence probe. + LegacyMetadataOwnerLedElsewhere LegacyMetadataRowReason = "owner_led_elsewhere" + // LegacyMetadataStaged defers a row while either route has staged visibility. + LegacyMetadataStaged LegacyMetadataRowReason = "staged_visibility" +) + +// LegacyMetadataRow reports a physical row, its logical owner and disposition. +// Delta rows and canonical owner-local metadata are never legacy backlog. +type LegacyMetadataRow struct { + Row *store.KVPair + OwnerGroupID uint64 + Reason LegacyMetadataRowReason +} + +// GroupOwnedScanPage contains served rows and legacy diagnostics from one +// physical page. Resume is the last physical key when the page was full, even +// when no row was served. CatalogVersion identifies this page, not a completed +// pass or a gate-eligible snapshot. +type GroupOwnedScanPage struct { + Rows []*store.KVPair + LegacyRows []LegacyMetadataRow + Resume []byte + CatalogVersion uint64 +} + +// ScanGroupOwnedPageAt is the local-leader diagnostic form of ScanGroupOwnedAt. +// It reads at most 512 physical rows, makes no remote owner probes, and checks +// that the catalog did not change while the page was classified. The caller +// must preserve read admission and track leadership/catalog identity across +// pages before publishing a completed pass. This method alone is not an audit +// or evidence that migration has completed. +func (s *ShardStore) ScanGroupOwnedPageAt(ctx context.Context, groupID uint64, start, end []byte, limit int, ts uint64) (GroupOwnedScanPage, error) { + if limit <= 0 || limit > legacyMetadataScanMaxRows { + return GroupOwnedScanPage{}, errors.New("legacy metadata scan limit must be between 1 and 512") + } + g, _ := s.groupForID(groupID) + if g == nil || g.Store == nil { + return GroupOwnedScanPage{}, errors.WithStack(ErrLeaderNotFound) + } + if err := requireLocalMetadataScanLeader(ctx, g); err != nil { + return GroupOwnedScanPage{}, err + } + version := s.ReadRouteVersion() + page, err := s.scanGroupOwnedLocalPage(ctx, groupID, g, start, end, limit, ts) + if err != nil { + return GroupOwnedScanPage{}, err + } + out, err := s.classifyGroupOwnedScanPageAt(ctx, groupID, page, ts) + if err != nil { + return GroupOwnedScanPage{}, err + } + if err := requireLocalMetadataScanLeader(ctx, g); err != nil { + return GroupOwnedScanPage{}, err + } + if s.ReadRouteVersion() != version { + return GroupOwnedScanPage{}, errors.WithStack(ErrReadRouteVersionUnavailable) + } + out.CatalogVersion = version + return out, nil +} + +func (s *ShardStore) scanGroupOwnedLocalPage(ctx context.Context, groupID uint64, g *ShardGroup, start, end []byte, limit int, ts uint64) (scanRoutePage, error) { + route := distribution.Route{GroupID: groupID} + if engineForGroup(g) == nil { + return s.scanRouteAtForwardLocalPage(ctx, g, route, start, end, limit, ts) + } + return s.scanRouteAtForwardLeaderPage(ctx, g, route, start, end, limit, ts) +} + +func (s *ShardStore) classifyGroupOwnedScanPageAt(ctx context.Context, groupID uint64, page scanRoutePage, ts uint64) (GroupOwnedScanPage, error) { + out := GroupOwnedScanPage{Rows: make([]*store.KVPair, 0, len(page.kvs))} + if page.full { + out.Resume = page.advanceKey + } + for _, row := range page.kvs { + if row == nil { + continue + } + legacy, serves, err := s.classifyLegacyMetadataRowAt(ctx, groupID, row, ts) + if err != nil { + return GroupOwnedScanPage{}, err + } + if serves { + out.Rows = append(out.Rows, row) + } + if legacy != nil { + out.LegacyRows = append(out.LegacyRows, *legacy) + } + } + return out, nil +} + +func requireLocalMetadataScanLeader(ctx context.Context, g *ShardGroup) error { + if engine := engineForGroup(g); engine != nil && !isLinearizableRaftLeader(ctx, engine) { + return errors.WithStack(raftengine.ErrNotLeader) + } + return nil +} + +func (s *ShardStore) classifyLegacyMetadataRowAt(ctx context.Context, groupID uint64, row *store.KVPair, ts uint64) (*LegacyMetadataRow, bool, error) { + if !isLegacyMigrationMetadataKey(row.Key) { + serves, err := s.groupServesKeyAt(ctx, groupID, row.Key, ts) + return nil, serves, err + } + owner, _, ok := s.routeAndGroupForKey(row.Key) + if !ok { + return nil, false, errors.WithStack(ErrReadRouteVersionUnavailable) + } + if owner.GroupID == groupID { + return nil, !routeHasStagedVisibility(owner), nil + } + legacy, ok := s.engine.GetRoute(legacyPointRouteKey(row.Key)) + if !ok || legacy.GroupID != groupID { + return nil, false, nil + } + result := &LegacyMetadataRow{Row: row, OwnerGroupID: owner.GroupID} + if routeHasStagedVisibility(owner) || routeHasStagedVisibility(legacy) { + result.Reason = LegacyMetadataStaged + return result, false, nil + } + reason, err := s.legacyMetadataOwnerReasonAt(ctx, owner, row.Key, ts) + if err != nil { + return nil, false, err + } + result.Reason = reason + return result, reason == LegacyMetadataServed, nil +} + +func (s *ShardStore) legacyMetadataOwnerReasonAt(ctx context.Context, owner distribution.Route, key []byte, ts uint64) (LegacyMetadataRowReason, error) { + g, ok := s.groupForID(owner.GroupID) + if !ok || g == nil || g.Store == nil { + return LegacyMetadataOwnerLedElsewhere, nil + } + exists, answered, err := s.routeHasVersionAtOrBefore(ctx, owner, key, ts) + if err != nil { + return "", err + } + if !answered { + if engine := engineForGroup(g); engine != nil && !isLinearizableRaftLeader(ctx, engine) { + return LegacyMetadataOwnerLedElsewhere, nil + } + return "", errors.New("legacy metadata owner cannot answer local version presence") + } + if !exists { + return LegacyMetadataServed, nil + } + return LegacyMetadataOwnerHasVersion, nil +} + +func isLegacyMigrationMetadataKey(key []byte) bool { + switch { + case store.IsHashMetaKey(key): + return !store.IsHashMetaDeltaKey(key) && store.ExtractHashUserKeyFromMeta(key) != nil + case store.IsSetMetaKey(key): + return !store.IsSetMetaDeltaKey(key) && store.ExtractSetUserKeyFromMeta(key) != nil + case store.IsZSetMetaKey(key): + return !store.IsZSetMetaDeltaKey(key) && store.ExtractZSetUserKeyFromMeta(key) != nil + case store.IsStreamMetaKey(key): + return store.ExtractStreamUserKeyFromMeta(key) != nil + default: + return false + } +} diff --git a/kv/legacy_metadata_scan_test.go b/kv/legacy_metadata_scan_test.go new file mode 100644 index 000000000..18fc2a93a --- /dev/null +++ b/kv/legacy_metadata_scan_test.go @@ -0,0 +1,221 @@ +package kv + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +type legacyMetadataPresenceStore struct { + store.MVCCStore + checks atomic.Int64 +} + +func (s *legacyMetadataPresenceStore) VersionExistsAtOrBefore(ctx context.Context, key []byte, ts uint64) (bool, error) { + s.checks.Add(1) + reader, ok := s.MVCCStore.(store.VersionPresenceReader) + if !ok { + return false, fmt.Errorf("test store has no version-presence reader") + } + return reader.VersionExistsAtOrBefore(ctx, key, ts) +} + +func TestScanGroupOwnedPageAt_ClassifiesLegacyMetadata(t *testing.T) { + t.Parallel() + families := []struct { + name string + key func([]byte) []byte + }{ + {"hash", store.HashMetaKey}, + {"set", store.SetMetaKey}, + {"zset", store.ZSetMetaKey}, + {"stream", store.StreamMetaKey}, + } + cases := []struct { + name string + ownerRow bool + tombstone bool + remote bool + staged uint64 + want LegacyMetadataRowReason + checks int64 + }{ + {name: "served", want: LegacyMetadataServed, checks: 1}, + {name: "shadowed", ownerRow: true, want: LegacyMetadataOwnerHasVersion, checks: 1}, + {name: "tombstone", tombstone: true, want: LegacyMetadataOwnerHasVersion, checks: 1}, + {name: "remote owner", remote: true, want: LegacyMetadataOwnerLedElsewhere}, + {name: "staged source", staged: 1, want: LegacyMetadataStaged}, + {name: "staged owner", staged: 2, want: LegacyMetadataStaged}, + } + for _, family := range families { + for _, tc := range cases { + t.Run(family.name+"/"+tc.name, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + st, groups := newSplitRedisShardStoreForGroupScan() + owner := &legacyMetadataPresenceStore{MVCCStore: groups[2].Store} + groups[2].Store = owner + if tc.remote { + groups[2].Engine = &followerProxyEngine{} + } + if tc.staged != 0 { + routes := []distribution.RouteDescriptor{ + {RouteID: 1, End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), GroupID: 2, State: distribution.RouteStateActive}, + } + routes[tc.staged-1].StagedVisibilityActive = true + routes[tc.staged-1].MigrationJobID = 9 + require.NoError(t, st.engine.ApplySnapshot(distribution.CatalogSnapshot{Version: 10, Routes: routes})) + } + key := family.key([]byte("zebra")) + require.NoError(t, groups[1].Store.PutAt(ctx, key, []byte("legacy"), 5, 0)) + if tc.ownerRow { + require.NoError(t, owner.PutAt(ctx, key, []byte("current"), 6, 0)) + } + if tc.tombstone { + require.NoError(t, owner.DeleteAt(ctx, key, 6)) + } + page, err := st.ScanGroupOwnedPageAt(ctx, 1, key, nil, 10, 10) + require.NoError(t, err) + require.Equal(t, st.ReadRouteVersion(), page.CatalogVersion) + require.Equal(t, []LegacyMetadataRow{{Row: &store.KVPair{Key: key, Value: []byte("legacy")}, OwnerGroupID: 2, Reason: tc.want}}, page.LegacyRows) + require.Equal(t, tc.checks, owner.checks.Load()) + wantServed := []*store.KVPair{} + if tc.want == LegacyMetadataServed { + wantServed = append(wantServed, &store.KVPair{Key: key, Value: []byte("legacy")}) + } + require.Equal(t, wantServed, page.Rows) + require.Nil(t, page.Resume) + previous, _, err := st.ScanGroupOwnedAt(ctx, 1, key, nil, 10, 10) + require.NoError(t, err) + require.Equal(t, previous, page.Rows) + }) + } + } +} + +func TestScanGroupOwnedPageAt_ExcludesCanonicalAndDeltaRows(t *testing.T) { + t.Parallel() + ctx := context.Background() + st, groups := newSplitRedisShardStoreForGroupScan() + keys := [][]byte{ + store.HashMetaKey([]byte("apple")), + store.SetMetaKey([]byte("apple")), + store.ZSetMetaKey([]byte("apple")), + store.StreamMetaKey([]byte("apple")), + store.HashMetaDeltaKey([]byte("zebra"), 5, 0), + store.SetMetaDeltaKey([]byte("zebra"), 5, 0), + store.ZSetMetaDeltaKey([]byte("zebra"), 5, 0), + store.ListMetaKey([]byte("zebra")), + } + for _, key := range keys { + require.NoError(t, groups[1].Store.PutAt(ctx, key, []byte("value"), 5, 0)) + } + page, err := st.ScanGroupOwnedPageAt(ctx, 1, nil, nil, 20, 10) + require.NoError(t, err) + require.Empty(t, page.LegacyRows) + previous, _, err := st.ScanGroupOwnedAt(ctx, 1, nil, nil, 20, 10) + require.NoError(t, err) + require.Equal(t, previous, page.Rows) +} + +func TestScanGroupOwnedPageAt_ResumesAfterFilteredPhysicalPage(t *testing.T) { + t.Parallel() + ctx := context.Background() + st, groups := newSplitRedisShardStoreForGroupScan() + source := &rowCountingMVCCStore{MVCCStore: groups[1].Store} + groups[1].Store = source + groups[2].Engine = &followerProxyEngine{} + prefix := []byte(store.HashMetaPrefix) + for i := range 5 { + require.NoError(t, source.PutAt(ctx, store.HashMetaKey([]byte(fmt.Sprintf("zebra%d", i))), []byte("legacy"), 5, 0)) + } + var seen int + start := prefix + for range 3 { + page, err := st.ScanGroupOwnedPageAt(ctx, 1, start, prefixScanEnd(prefix), 2, 10) + require.NoError(t, err) + require.Empty(t, page.Rows) + seen += len(page.LegacyRows) + if page.Resume == nil { + break + } + start = append(append([]byte(nil), page.Resume...), 0) + } + require.Equal(t, 5, seen) + require.Equal(t, 5, source.rowsRead()) +} + +type legacyMetadataChangingStore struct { + store.MVCCStore + afterScan func() +} + +func (s *legacyMetadataChangingStore) ScanAt(ctx context.Context, start, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { + rows, err := s.MVCCStore.ScanAt(ctx, start, end, limit, ts) + s.afterScan() + return rows, err +} + +func TestScanGroupOwnedPageAt_RejectsUnknownOrChangedEvidence(t *testing.T) { + t.Parallel() + ctx := context.Background() + t.Run("catalog changes", func(t *testing.T) { + st, groups := newSplitRedisShardStoreForGroupScan() + groups[1].Store = &legacyMetadataChangingStore{MVCCStore: groups[1].Store, afterScan: func() { + require.NoError(t, st.engine.ApplySnapshot(distribution.CatalogSnapshot{Version: 10, Routes: []distribution.RouteDescriptor{ + {RouteID: 1, End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), GroupID: 2, State: distribution.RouteStateActive}, + }})) + }} + page, err := st.ScanGroupOwnedPageAt(ctx, 1, nil, nil, 10, 10) + require.ErrorIs(t, err, ErrReadRouteVersionUnavailable) + require.Equal(t, GroupOwnedScanPage{}, page) + }) + t.Run("not source leader", func(t *testing.T) { + st, groups := newSplitRedisShardStoreForGroupScan() + groups[1].Engine = &followerProxyEngine{} + page, err := st.ScanGroupOwnedPageAt(ctx, 1, nil, nil, 10, 10) + require.ErrorIs(t, err, raftengine.ErrNotLeader) + require.Equal(t, GroupOwnedScanPage{}, page) + }) + t.Run("source leadership lost during page", func(t *testing.T) { + st, groups := newSplitRedisShardStoreForGroupScan() + groups[1].Store = &legacyMetadataChangingStore{MVCCStore: groups[1].Store, afterScan: func() { + groups[1].Engine = &followerProxyEngine{} + }} + page, err := st.ScanGroupOwnedPageAt(ctx, 1, nil, nil, 10, 10) + require.ErrorIs(t, err, raftengine.ErrNotLeader) + require.Equal(t, GroupOwnedScanPage{}, page) + }) + t.Run("owner lacks local presence reader", func(t *testing.T) { + st, groups := newSplitRedisShardStoreForGroupScan() + // Embedding the base interface deliberately hides optional presence support. + groups[2].Store = &struct{ store.MVCCStore }{groups[2].Store} + key := store.HashMetaKey([]byte("zebra")) + require.NoError(t, groups[1].Store.PutAt(ctx, key, []byte("legacy"), 5, 0)) + page, err := st.ScanGroupOwnedPageAt(ctx, 1, nil, nil, 10, 10) + require.ErrorContains(t, err, "cannot answer local version presence") + require.Equal(t, GroupOwnedScanPage{}, page) + }) + t.Run("missing source", func(t *testing.T) { + st, _ := newSplitRedisShardStoreForGroupScan() + page, err := st.ScanGroupOwnedPageAt(ctx, 99, nil, nil, 10, 10) + require.ErrorIs(t, err, ErrLeaderNotFound) + require.Equal(t, GroupOwnedScanPage{}, page) + }) + for _, limit := range []int{-1, 0, 513} { + t.Run(fmt.Sprintf("invalid limit %d", limit), func(t *testing.T) { + st, _ := newSplitRedisShardStoreForGroupScan() + page, err := st.ScanGroupOwnedPageAt(ctx, 1, nil, nil, limit, 10) + require.Error(t, err) + require.Equal(t, GroupOwnedScanPage{}, page) + }) + } +} diff --git a/kv/migration_promote_control_state_test.go b/kv/migration_promote_control_state_test.go index c0b037422..417b52039 100644 --- a/kv/migration_promote_control_state_test.go +++ b/kv/migration_promote_control_state_test.go @@ -12,7 +12,7 @@ import ( ) func TestMigrationPromoteRestoredStagingPreservesLocalControl(t *testing.T) { - for _, key := range [][]byte{backupTimestampFloorKey, prepareEvidenceEnabledKey, prepareEvidenceGateKey, prepareEvidenceTimestampFloorKey} { + for _, key := range [][]byte{backupTimestampFloorKey, prepareEvidenceEnabledKey, prepareEvidenceGateKey, prepareEvidenceTimestampFloorKey, replicatedPinStateKey} { for _, tombstone := range []bool{false, true} { t.Run(fmt.Sprintf("%s-delete-%t", key, tombstone), func(t *testing.T) { assertRestoredPromotionPreservesControl(t, key, tombstone) @@ -27,6 +27,9 @@ func assertRestoredPromotionPreservesControl(t *testing.T, key []byte, tombstone source, err := store.NewPebbleStore(t.TempDir()) require.NoError(t, err) defer func() { require.NoError(t, source.Close()) }() + if bytes.Equal(key, replicatedPinStateKey) { + seedActivatedPinJournal(t, source) + } fsm := newPrepareSafetyFSM(t, source) require.NoError(t, fsm.persistBackupTimestampFloor(ctx, 100)) for _, command := range [][]byte{EncodePrepareEvidenceFence(0), EncodePrepareEvidenceSeal(), EncodePrepareEvidenceFence(100), EncodePrepareEvidenceRelease(100)} { diff --git a/kv/replicated_backup_codec.go b/kv/replicated_backup_codec.go new file mode 100644 index 000000000..0c86790b9 --- /dev/null +++ b/kv/replicated_backup_codec.go @@ -0,0 +1,156 @@ +package kv + +import ( + "encoding/binary" + "math" + + "github.com/cockroachdb/errors" +) + +// 0x16 (field 2, wire type 6) is an invalid protobuf prefix, so it cannot be +// mistaken for a legacy bare Request or RaftCommand. 0x0f was this opcode's +// value while the branch was open, but main assigned 0x0f to +// raftEncodeTargetReadiness first; aliasing it would misroute committed +// entries. No producer may emit it until the configuration-bound capability +// gate and durable legacy drain activate the new backup epoch. +const raftEncodeReplicatedBackup byte = 0x16 + +const ( + replicatedBackupWireVersion = byte(1) + replicatedBackupPayloadSize = 58 +) + +type replicatedBackupWire struct { + Version byte + Operation ReplicatedBackupOperation + Identity BackupIdentity + PinID BackupPinID + ReadTS uint64 + DeadlineMillis int64 + ExpectedDeadlineMillis int64 +} + +type ReplicatedBackupOperation byte + +const ( + ReplicatedBackupPin ReplicatedBackupOperation = iota + 1 + ReplicatedBackupExtend + ReplicatedBackupRelease + ReplicatedBackupExpire + ReplicatedBackupAllocate +) + +// ReplicatedBackupCommand carries only ordered state; deadline milliseconds +// are identities for Extend/Expire, never a local apply-time expiry clock. +type ReplicatedBackupCommand struct { + Operation ReplicatedBackupOperation + Identity BackupIdentity + PinID BackupPinID + ReadTS uint64 + DeadlineMillis int64 + ExpectedDeadlineMillis int64 +} + +func (c ReplicatedBackupCommand) validate() error { + if c.Identity.Epoch == 0 { + return errors.Wrap(ErrReplicatedBackupPin, "missing backup epoch") + } + if c.Operation == ReplicatedBackupAllocate { + return c.validateAllocation() + } + if c.Identity.Generation == 0 || c.PinID.IsZero() { + return errors.Wrap(ErrReplicatedBackupPin, "missing backup identity or pin ID") + } + return c.validatePinFields() +} + +func (c ReplicatedBackupCommand) validateAllocation() error { + if c.Identity.Generation != 0 || !c.PinID.IsZero() || c.ReadTS != 0 || c.DeadlineMillis != 0 || c.ExpectedDeadlineMillis != 0 { + return errors.Wrap(ErrReplicatedBackupPin, "allocation carries unexpected pin fields") + } + return nil +} + +func (c ReplicatedBackupCommand) validatePinFields() error { + if err := c.validateReadTimestamp(); err != nil { + return err + } + switch c.Operation { + case ReplicatedBackupPin, ReplicatedBackupExtend: + return c.validateLiveDeadline() + case ReplicatedBackupRelease: + return c.validateNoDeadline() + case ReplicatedBackupExpire: + return c.validateExpectedDeadline() + case ReplicatedBackupAllocate: + return c.validateAllocation() + } + return errors.Wrap(ErrReplicatedBackupPin, "unknown backup operation") +} + +func (c ReplicatedBackupCommand) validateReadTimestamp() error { + if c.Operation == ReplicatedBackupPin { + if c.ReadTS == 0 || c.ReadTS == math.MaxUint64 { + return errors.Wrap(ErrReplicatedBackupPin, "invalid pin read timestamp") + } + } else if c.ReadTS != 0 { + return errors.Wrap(ErrReplicatedBackupPin, "only initial pins may carry a read timestamp") + } + return nil +} + +func (c ReplicatedBackupCommand) validateLiveDeadline() error { + if c.DeadlineMillis <= 0 || c.ExpectedDeadlineMillis != 0 { + return errors.Wrap(ErrReplicatedBackupPin, "invalid pin deadline") + } + return nil +} + +func (c ReplicatedBackupCommand) validateNoDeadline() error { + if c.DeadlineMillis != 0 || c.ExpectedDeadlineMillis != 0 { + return errors.Wrap(ErrReplicatedBackupPin, "release carries unexpected deadlines") + } + return nil +} + +func (c ReplicatedBackupCommand) validateExpectedDeadline() error { + if c.DeadlineMillis != 0 || c.ExpectedDeadlineMillis <= 0 { + return errors.Wrap(ErrReplicatedBackupPin, "invalid expiry deadline identity") + } + return nil +} + +func EncodeReplicatedBackupCommand(command ReplicatedBackupCommand) ([]byte, error) { + if err := command.validate(); err != nil { + return nil, err + } + out := make([]byte, 1+replicatedBackupPayloadSize) + out[0] = raftEncodeReplicatedBackup + wire := replicatedBackupWire{ + Version: replicatedBackupWireVersion, Operation: command.Operation, + Identity: command.Identity, PinID: command.PinID, ReadTS: command.ReadTS, + DeadlineMillis: command.DeadlineMillis, ExpectedDeadlineMillis: command.ExpectedDeadlineMillis, + } + if _, err := binary.Encode(out[1:], binary.BigEndian, wire); err != nil { + return nil, errors.Wrap(err, "encode replicated backup command") + } + return out, nil +} + +func decodeReplicatedBackupCommand(payload []byte) (ReplicatedBackupCommand, error) { + if len(payload) != replicatedBackupPayloadSize || payload[0] != replicatedBackupWireVersion { + return ReplicatedBackupCommand{}, errors.New("unknown or malformed replicated backup wire format") + } + var wire replicatedBackupWire + if _, err := binary.Decode(payload, binary.BigEndian, &wire); err != nil { + return ReplicatedBackupCommand{}, errors.Wrap(err, "decode replicated backup command") + } + command := ReplicatedBackupCommand{ + Operation: wire.Operation, Identity: wire.Identity, PinID: wire.PinID, + ReadTS: wire.ReadTS, DeadlineMillis: wire.DeadlineMillis, ExpectedDeadlineMillis: wire.ExpectedDeadlineMillis, + } + if err := command.validate(); err != nil { + return ReplicatedBackupCommand{}, err + } + return command, nil +} diff --git a/kv/replicated_backup_codec_test.go b/kv/replicated_backup_codec_test.go new file mode 100644 index 000000000..9dfb21c24 --- /dev/null +++ b/kv/replicated_backup_codec_test.go @@ -0,0 +1,90 @@ +package kv + +import ( + "encoding/binary" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func replicatedBackupWireFixtures() []ReplicatedBackupCommand { + id := BackupIdentity{Epoch: 1, Generation: 7} + pinID := backupTrackerTestPinID(1) + return []ReplicatedBackupCommand{ + {Operation: ReplicatedBackupPin, Identity: id, PinID: pinID, ReadTS: 100, DeadlineMillis: 200}, + {Operation: ReplicatedBackupExtend, Identity: id, PinID: pinID, DeadlineMillis: 300}, + {Operation: ReplicatedBackupRelease, Identity: id, PinID: pinID}, + {Operation: ReplicatedBackupExpire, Identity: id, PinID: pinID, ExpectedDeadlineMillis: 200}, + {Operation: ReplicatedBackupAllocate, Identity: BackupIdentity{Epoch: 1}}, + } +} + +func TestReplicatedBackupWireRoundTrip(t *testing.T) { + t.Parallel() + require.Equal(t, replicatedBackupPayloadSize, binary.Size(replicatedBackupWire{})) + require.Equal(t, byte(0x16), raftEncodeReplicatedBackup) + require.Equal(t, byte(replicatedPinAdmit), byte(ReplicatedBackupPin)) + require.Equal(t, byte(replicatedPinExtend), byte(ReplicatedBackupExtend)) + require.Equal(t, byte(replicatedPinRelease), byte(ReplicatedBackupRelease)) + require.Equal(t, byte(replicatedPinExpire), byte(ReplicatedBackupExpire)) + for _, command := range replicatedBackupWireFixtures() { + encoded, err := EncodeReplicatedBackupCommand(command) + require.NoError(t, err) + require.Len(t, encoded, 1+replicatedBackupPayloadSize) + require.Equal(t, raftEncodeReplicatedBackup, encoded[0]) + decoded, err := decodeReplicatedBackupCommand(encoded[1:]) + require.NoError(t, err) + require.Equal(t, command, decoded) + require.Error(t, proto.Unmarshal(encoded, new(pb.Request)), "must not overlap a legacy protobuf message") + } +} + +func TestReplicatedBackupWireRejectsMalformedOrIgnoredFields(t *testing.T) { + t.Parallel() + valid := replicatedBackupWireFixtures()[0] + encoded, err := EncodeReplicatedBackupCommand(valid) + require.NoError(t, err) + for length := range replicatedBackupPayloadSize { + _, err := decodeReplicatedBackupCommand(encoded[1 : 1+length]) + require.Error(t, err) + } + _, err = decodeReplicatedBackupCommand(append(encoded[1:], 0)) + require.Error(t, err) + for _, field := range []int{1, 2} { + bad := append([]byte(nil), encoded...) + bad[field] = 0xff + _, err := decodeReplicatedBackupCommand(bad[1:]) + require.Error(t, err) + } + for _, command := range []ReplicatedBackupCommand{ + {Operation: ReplicatedBackupAllocate}, + {Operation: ReplicatedBackupAllocate, Identity: valid.Identity}, + {Operation: ReplicatedBackupPin, Identity: valid.Identity, ReadTS: 100, DeadlineMillis: 200}, + {Operation: ReplicatedBackupPin, Identity: valid.Identity, PinID: valid.PinID, ReadTS: 100, DeadlineMillis: -1}, + {Operation: ReplicatedBackupExtend, Identity: valid.Identity, PinID: valid.PinID, ReadTS: 100, DeadlineMillis: 300}, + {Operation: ReplicatedBackupRelease, Identity: valid.Identity, PinID: valid.PinID, DeadlineMillis: 200}, + {Operation: ReplicatedBackupExpire, Identity: valid.Identity, PinID: valid.PinID, DeadlineMillis: 200, ExpectedDeadlineMillis: 200}, + } { + _, err := EncodeReplicatedBackupCommand(command) + require.Error(t, err) + } +} + +func FuzzReplicatedBackupWire(f *testing.F) { + for _, command := range replicatedBackupWireFixtures() { + encoded, err := EncodeReplicatedBackupCommand(command) + require.NoError(f, err) + f.Add(encoded[1:]) + } + f.Fuzz(func(t *testing.T, payload []byte) { + command, err := decodeReplicatedBackupCommand(payload) + if err != nil { + return + } + encoded, err := EncodeReplicatedBackupCommand(command) + require.NoError(t, err) + require.Equal(t, payload, encoded[1:]) + }) +} diff --git a/kv/replicated_backup_journal.go b/kv/replicated_backup_journal.go new file mode 100644 index 000000000..1be72397c --- /dev/null +++ b/kv/replicated_backup_journal.go @@ -0,0 +1,212 @@ +package kv + +import ( + "bytes" + "context" + "encoding/json" + "io" + "maps" + "math" + "sync" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +const replicatedPinStateVersion = 1 + +var replicatedPinStateKey = []byte(store.ReplicatedControlPrefix + "backup-pins") + +type replicatedPinStateRecord struct { + Version uint32 + AppliedIndex uint64 + Pins replicatedBackupPins + // Only the activated control group's record may allocate identities. + Allocator bool + AllocatedThrough uint64 + LastAllocationIndex uint64 +} + +// The journal installs a new in-memory view only after its state and Raft +// applied index have committed together. Runtime activation must provide the +// initial epoch after the legacy drain; an absent record stays inactive. +type replicatedPinJournal struct { + mu sync.RWMutex + store store.MVCCStore + state replicatedPinStateRecord +} + +func openReplicatedPinJournal(ctx context.Context, st store.MVCCStore) (*replicatedPinJournal, error) { + j := &replicatedPinJournal{store: st} + if err := j.reload(ctx); err != nil { + return nil, err + } + return j, nil +} + +func (j *replicatedPinJournal) reload(ctx context.Context) error { + value, err := j.store.GetAt(ctx, replicatedPinStateKey, math.MaxUint64) + if errors.Is(err, store.ErrKeyNotFound) { + j.state = replicatedPinStateRecord{} + return nil + } + if err != nil { + return errors.Wrap(err, "read replicated backup pins") + } + record, err := decodeReplicatedPinState(value) + if err != nil { + return err + } + j.state = record + return nil +} + +func decodeReplicatedPinState(value []byte) (replicatedPinStateRecord, error) { + var record replicatedPinStateRecord + decoder := json.NewDecoder(bytes.NewReader(value)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&record); err != nil { + return record, errors.Wrap(err, "decode replicated backup pins") + } + if err := decoder.Decode(new(any)); !errors.Is(err, io.EOF) { + return record, errors.New("trailing replicated backup pin state") + } + if err := record.validate(); err != nil { + return record, err + } + return record, nil +} + +func (r replicatedPinStateRecord) validate() error { + if !r.validHeader() { + return errors.New("invalid replicated backup pin state header") + } + if r.Pins.Capacity != 0 && uint64(len(r.Pins.Active)) > uint64(r.Pins.Capacity) { + return errors.New("replicated backup pin state exceeds capacity") + } + if r.Pins.PinGeneration < uint64(len(r.Pins.Active)) { + return errors.New("replicated backup pin generation is behind active set") + } + if err := r.validateAllocator(); err != nil { + return err + } + for generation, pin := range r.Pins.Active { + if generation == 0 || !pin.validAtFloor(r.Pins.ReadFloor) || pin.ReadTS > r.Pins.WriteTimestampFloor { + return errors.New("invalid active replicated backup pin") + } + } + return nil +} + +func (r replicatedPinStateRecord) validateAllocator() error { + if !r.Allocator { + if r.AllocatedThrough != 0 || r.LastAllocationIndex != 0 { + return errors.New("data-group backup pin state cannot allocate identities") + } + return nil + } + if (r.AllocatedThrough == 0) != (r.LastAllocationIndex == 0) || r.LastAllocationIndex > r.AppliedIndex { + return errors.New("invalid backup identity allocation checkpoint") + } + if r.Pins.RetiredThrough > r.AllocatedThrough { + return errors.New("control retirement exceeds allocated backup identities") + } + for generation := range r.Pins.Active { + if generation > r.AllocatedThrough { + return errors.New("control reservation has an unallocated backup identity") + } + } + return nil +} + +func (r replicatedPinStateRecord) validHeader() bool { + return r.Version == replicatedPinStateVersion && r.AppliedIndex != 0 && r.Pins.Epoch != 0 && r.Pins.ReadFloor != math.MaxUint64 && r.Pins.WriteTimestampFloor != math.MaxUint64 +} + +func (p replicatedBackupPin) validAtFloor(floor uint64) bool { + return !p.PinID.IsZero() && p.ReadTS != 0 && p.ReadTS != math.MaxUint64 && p.Deadline > 0 && p.ReadTS >= floor +} + +func (j *replicatedPinJournal) apply(ctx context.Context, index uint64, command replicatedPinCommand) error { + j.mu.Lock() + defer j.mu.Unlock() + if index == 0 { + return errors.Wrap(ErrReplicatedBackupPin, "missing Raft apply index") + } + if index <= j.state.AppliedIndex { + return nil + } + if j.state.Allocator && command.Identity.Generation > j.state.AllocatedThrough { + return errors.Wrap(ErrReplicatedBackupPin, "backup identity was not allocated") + } + pins, _, err := j.state.Pins.transition(command) + if err != nil { + return err + } + record := j.state + record.Version = replicatedPinStateVersion + record.AppliedIndex = index + record.Pins = pins + return j.persist(ctx, record) +} + +func (j *replicatedPinJournal) allocate(ctx context.Context, index, epoch uint64) (BackupIdentity, error) { + j.mu.Lock() + defer j.mu.Unlock() + if !j.state.Allocator || epoch == 0 || epoch != j.state.Pins.Epoch || index == 0 { + return BackupIdentity{}, errors.Wrap(ErrReplicatedBackupPin, "identity allocator is inactive or epoch is stale") + } + if index <= j.state.AppliedIndex { + return j.allocationAt(index) + } + if j.state.AllocatedThrough == math.MaxUint64 { + return BackupIdentity{}, errors.Wrap(ErrReplicatedBackupPin, "backup identity generation exhausted") + } + record := j.state + record.AppliedIndex = index + record.AllocatedThrough++ + record.LastAllocationIndex = index + if err := j.persist(ctx, record); err != nil { + return BackupIdentity{}, err + } + // An already-applied disk index caused a reload. Never return the stale + // computed identity unless that exact allocation is in the durable state. + return j.allocationAt(index) +} + +func (j *replicatedPinJournal) allocationAt(index uint64) (BackupIdentity, error) { + if index != j.state.LastAllocationIndex || j.state.AllocatedThrough == 0 { + // A lost old response may allocate again with a new Raft entry, but may + // never guess an identity that a later allocation could already own. + return BackupIdentity{}, errors.Wrap(ErrReplicatedBackupPin, "allocation result is no longer available; allocate a fresh identity") + } + return BackupIdentity{Epoch: j.state.Pins.Epoch, Generation: j.state.AllocatedThrough}, nil +} + +func (j *replicatedPinJournal) persist(ctx context.Context, record replicatedPinStateRecord) error { + value, err := json.Marshal(record) + if err != nil { + return errors.Wrap(err, "encode replicated backup pins") + } + writer, ok := j.store.(store.ReplicatedControlStore) + if !ok { + return errors.New("store does not support durable replicated control state") + } + applied, err := writer.ApplyReplicatedControlState(ctx, replicatedPinStateKey, value, record.AppliedIndex) + if err != nil { + return errors.Wrap(err, "persist replicated backup pins") + } + if !applied { + return j.reload(ctx) + } + j.state = record + return nil +} + +func (j *replicatedPinJournal) snapshot() replicatedPinStateRecord { + j.mu.RLock() + defer j.mu.RUnlock() + record := j.state + record.Pins.Active = maps.Clone(j.state.Pins.Active) + return record +} diff --git a/kv/replicated_backup_journal_test.go b/kv/replicated_backup_journal_test.go new file mode 100644 index 000000000..9396760a8 --- /dev/null +++ b/kv/replicated_backup_journal_test.go @@ -0,0 +1,277 @@ +package kv + +import ( + "bytes" + "context" + "encoding/json" + "math" + "testing" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func seedActivatedPinJournal(t *testing.T, st store.MVCCStore) { + t.Helper() + // Test-only activation fixture. Production activation must prove the + // durable legacy drain and capability gate before writing this epoch. + seedPinJournalRecord(t, st, replicatedPinStateRecord{ + Version: replicatedPinStateVersion, AppliedIndex: 1, + Pins: replicatedBackupPins{Epoch: 1, Capacity: 4}, + }) +} + +func seedPinJournalRecord(t *testing.T, st store.MVCCStore, record replicatedPinStateRecord) { + t.Helper() + value, err := json.Marshal(record) + require.NoError(t, err) + writer, ok := st.(store.ReplicatedControlStore) + require.True(t, ok) + applied, err := writer.ApplyReplicatedControlState(context.Background(), replicatedPinStateKey, value, record.AppliedIndex) + require.NoError(t, err) + require.True(t, applied) +} + +func TestReplicatedPinJournalAllocatesBeforeReserving(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := g17OpenStore(t, t.TempDir()) + seedPinJournalRecord(t, st, replicatedPinStateRecord{ + Version: replicatedPinStateVersion, AppliedIndex: 1, Allocator: true, + Pins: replicatedBackupPins{Epoch: 1, Capacity: 1}, + }) + journal := newPinJournal(t, st) + require.ErrorIs(t, journal.apply(ctx, 2, pinCommand(replicatedPinAdmit, 1)), ErrReplicatedBackupPin) + first, err := journal.allocate(ctx, 2, 1) + require.NoError(t, err) + require.Equal(t, BackupIdentity{Epoch: 1, Generation: 1}, first) + repeated, err := journal.allocate(ctx, 2, 1) + require.NoError(t, err) + require.Equal(t, first, repeated) + require.NoError(t, journal.apply(ctx, 3, pinCommand(replicatedPinAdmit, first.Generation))) + journal = newPinJournal(t, st) + second, err := journal.allocate(ctx, 4, 1) + require.NoError(t, err) + require.Equal(t, first.Generation+1, second.Generation) + _, err = journal.allocate(ctx, 2, 1) + require.ErrorIs(t, err, ErrReplicatedBackupPin, "an old lost allocation response must not return a newer caller's identity") + require.ErrorIs(t, journal.apply(ctx, 5, pinCommand(replicatedPinAdmit, second.Generation)), ErrTooManyActiveBackups) + require.NoError(t, journal.apply(ctx, 6, pinCommand(replicatedPinRelease, second.Generation))) + require.ErrorIs(t, journal.apply(ctx, 7, pinCommand(replicatedPinAdmit, second.Generation)), ErrReplicatedBackupPin) + _, err = journal.allocate(ctx, 8, 2) + require.ErrorIs(t, err, ErrReplicatedBackupPin) + require.Equal(t, uint64(2), journal.snapshot().AllocatedThrough) +} + +func TestReplicatedPinJournalAllocationFailureNeverReturnsIdentity(t *testing.T) { + t.Parallel() + st := store.NewMVCCStore() + t.Cleanup(func() { _ = st.Close() }) + seedPinJournalRecord(t, st, replicatedPinStateRecord{ + Version: replicatedPinStateVersion, AppliedIndex: 1, Allocator: true, + Pins: replicatedBackupPins{Epoch: 1, Capacity: 1}, + }) + failure := errors.New("disk unavailable") + journal := newPinJournal(t, &failingPinControlStore{MVCCStore: st, err: failure}) + identity, err := journal.allocate(context.Background(), 2, 1) + require.ErrorIs(t, err, failure) + require.Zero(t, identity) + require.Zero(t, journal.snapshot().AllocatedThrough) + journal = newPinJournal(t, st) + journal.state.AllocatedThrough = math.MaxUint64 + journal.state.LastAllocationIndex = 1 + identity, err = journal.allocate(context.Background(), 2, 1) + require.ErrorIs(t, err, ErrReplicatedBackupPin) + require.Zero(t, identity) + require.Equal(t, uint64(math.MaxUint64), journal.snapshot().AllocatedThrough) +} + +func TestReplicatedPinJournalAllocatorSurvivesSnapshotAndReopen(t *testing.T) { + t.Parallel() + ctx := context.Background() + dir := t.TempDir() + st, err := store.NewPebbleStore(dir) + require.NoError(t, err) + closed := false + t.Cleanup(func() { + if !closed { + _ = st.Close() + } + }) + seedPinJournalRecord(t, st, replicatedPinStateRecord{ + Version: replicatedPinStateVersion, AppliedIndex: 1, Allocator: true, + Pins: replicatedBackupPins{Epoch: 1, Capacity: 1}, + }) + journal := newPinJournal(t, st) + first, err := journal.allocate(ctx, 2, 1) + require.NoError(t, err) + snapshot, err := st.Snapshot() + require.NoError(t, err) + defer snapshot.Close() + var data bytes.Buffer + _, err = snapshot.WriteTo(&data) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + require.NoError(t, st.Close()) + closed = true + + reopened := newPinJournal(t, g17OpenStore(t, dir)) + target := g17OpenStore(t, t.TempDir()) + require.NoError(t, target.Restore(&data)) + restored := newPinJournal(t, target) + for _, current := range []*replicatedPinJournal{reopened, restored} { + require.Equal(t, journal.snapshot(), current.snapshot()) + second, err := current.allocate(ctx, 3, 1) + require.NoError(t, err) + require.Equal(t, first.Generation+1, second.Generation) + } +} + +func newPinJournal(t *testing.T, st store.MVCCStore) *replicatedPinJournal { + t.Helper() + j, err := openReplicatedPinJournal(context.Background(), st) + require.NoError(t, err) + return j +} + +func TestReplicatedPinJournalSnapshotPreservesRetirement(t *testing.T) { + t.Parallel() + for _, backend := range []string{"memory", "pebble"} { + t.Run(backend, func(t *testing.T) { + t.Parallel() + open := func() store.MVCCStore { + if backend == "pebble" { + return g17OpenStore(t, t.TempDir()) + } + st := store.NewMVCCStore() + t.Cleanup(func() { _ = st.Close() }) + return st + } + st := open() + seedActivatedPinJournal(t, st) + journal := newPinJournal(t, st) + ctx := context.Background() + require.NoError(t, journal.apply(ctx, 2, pinCommand(replicatedPinAdmit, 1))) + require.NoError(t, journal.apply(ctx, 3, pinCommand(replicatedPinRelease, 5))) + snapshot, err := st.Snapshot() + require.NoError(t, err) + defer snapshot.Close() + var data bytes.Buffer + _, err = snapshot.WriteTo(&data) + require.NoError(t, err) + target := open() + require.NoError(t, target.Restore(&data)) + restored := newPinJournal(t, target) + require.Equal(t, journal.snapshot(), restored.snapshot()) + require.ErrorIs(t, restored.apply(ctx, 4, pinCommand(replicatedPinAdmit, 5)), ErrReplicatedBackupPin) + extend := pinCommand(replicatedPinExtend, 1) + extend.Deadline = 200 + require.NoError(t, restored.apply(ctx, 5, extend)) + require.NoError(t, restored.apply(ctx, 6, pinCommand(replicatedPinExpire, 1))) + require.Equal(t, uint64(10), restored.snapshot().Pins.oldest()) + require.Equal(t, uint64(5), restored.snapshot().Pins.RetiredThrough) + require.Zero(t, target.LastCommitTS()) + }) + } +} + +func TestReplicatedPinJournalReopenAndSkipStaleApply(t *testing.T) { + t.Parallel() + ctx := context.Background() + dir := t.TempDir() + st, err := store.NewPebbleStore(dir) + require.NoError(t, err) + seedActivatedPinJournal(t, st) + older := newPinJournal(t, st) + current := newPinJournal(t, st) + require.NoError(t, current.apply(ctx, 3, pinCommand(replicatedPinAdmit, 3))) + require.NoError(t, older.apply(ctx, 2, pinCommand(replicatedPinRelease, 5))) + require.Equal(t, current.snapshot(), older.snapshot(), "stale durable index must reload, not install its computed replacement") + require.NoError(t, st.Close()) + reopened := g17OpenStore(t, dir) + journal := newPinJournal(t, reopened) + require.Equal(t, current.snapshot(), journal.snapshot()) + require.NoError(t, journal.apply(ctx, 3, pinCommand(replicatedPinRelease, 3))) + require.Len(t, journal.snapshot().Pins.Active, 1, "duplicate index must not apply a different transition") + require.NoError(t, journal.apply(ctx, 4, pinCommand(replicatedPinRelease, 3))) + require.Empty(t, journal.snapshot().Pins.Active) + _, _, err = journal.snapshot().Pins.transition(pinCommand(replicatedPinAdmit, 3)) + require.ErrorIs(t, err, ErrReplicatedBackupPin) +} + +type failingPinControlStore struct { + store.MVCCStore + err error +} + +func (s *failingPinControlStore) ApplyReplicatedControlState(context.Context, []byte, []byte, uint64) (bool, error) { + return false, s.err +} + +func TestReplicatedPinJournalDoesNotPublishFailedWrite(t *testing.T) { + t.Parallel() + st := store.NewMVCCStore() + t.Cleanup(func() { _ = st.Close() }) + seedActivatedPinJournal(t, st) + failure := errors.New("disk unavailable") + journal := newPinJournal(t, &failingPinControlStore{MVCCStore: st, err: failure}) + before := journal.snapshot() + require.ErrorIs(t, journal.apply(context.Background(), 2, pinCommand(replicatedPinAdmit, 1)), failure) + require.Equal(t, before, journal.snapshot()) + require.Equal(t, before, newPinJournal(t, st).snapshot()) +} + +func TestReplicatedPinJournalMissingOrCorruptStateStaysInactive(t *testing.T) { + t.Parallel() + st := store.NewMVCCStore() + t.Cleanup(func() { _ = st.Close() }) + journal := newPinJournal(t, st) + require.Zero(t, journal.snapshot().Pins.Epoch) + require.ErrorIs(t, journal.apply(context.Background(), 1, pinCommand(replicatedPinAdmit, 1)), ErrReplicatedBackupPin) + for _, value := range []string{ + `{}`, `{"Version":2}`, `null`, `{} {}`, + `{"Version":1,"AppliedIndex":1,"Pins":{"Epoch":1},"Unexpected":true}`, + `{"Version":1,"AppliedIndex":1,"Pins":{"Epoch":1,"Active":{"0":{}}}}`, + } { + _, err := decodeReplicatedPinState([]byte(value)) + require.Error(t, err, "corrupt state %s", value) + } +} + +func TestReplicatedPinJournalLeavesLegacyUserNamespaceUntouched(t *testing.T) { + t.Parallel() + for _, deleted := range []bool{false, true} { + t.Run(map[bool]string{false: "value", true: "tombstone"}[deleted], func(t *testing.T) { + t.Parallel() + st := g17OpenStore(t, t.TempDir()) + ctx := context.Background() + legacyKey := []byte("!txn|control|backup-pins") + require.NoError(t, st.PutAt(ctx, legacyKey, []byte("user"), 10, 0)) + if deleted { + require.NoError(t, st.DeleteAt(ctx, legacyKey, 11)) + } + seedActivatedPinJournal(t, st) + journal := newPinJournal(t, st) + require.NoError(t, journal.apply(ctx, 2, pinCommand(replicatedPinAdmit, 1))) + snapshot, err := st.Snapshot() + require.NoError(t, err) + defer snapshot.Close() + var data bytes.Buffer + _, err = snapshot.WriteTo(&data) + require.NoError(t, err) + target := g17OpenStore(t, t.TempDir()) + require.NoError(t, target.Restore(&data)) + require.Equal(t, journal.snapshot(), newPinJournal(t, target).snapshot()) + for _, current := range []store.MVCCStore{st, target} { + got, err := current.GetAt(ctx, legacyKey, ^uint64(0)) + if deleted { + require.ErrorIs(t, err, store.ErrKeyNotFound) + } else { + require.NoError(t, err) + require.Equal(t, []byte("user"), got) + } + } + }) + } +} diff --git a/kv/replicated_backup_pins.go b/kv/replicated_backup_pins.go new file mode 100644 index 000000000..6144f6017 --- /dev/null +++ b/kv/replicated_backup_pins.go @@ -0,0 +1,166 @@ +package kv + +import ( + "maps" + "math" + + "github.com/cockroachdb/errors" +) + +var ErrReplicatedBackupPin = errors.New("replicated backup pin rejected") + +// BackupIdentity is allocated durably before a backup fans out to data groups. +// A generation is never reused within an epoch, including failed admissions. +type BackupIdentity struct { + Epoch uint64 + Generation uint64 +} + +type replicatedBackupPin struct { + PinID BackupPinID + ReadTS uint64 + // Unix milliseconds; apply compares versions, never the local clock. + Deadline int64 +} + +type replicatedBackupPins struct { + Epoch uint64 + RetiredThrough uint64 + PinGeneration uint64 + ReadFloor uint64 + // A released/expired backup never reopens admission of delayed writes. + WriteTimestampFloor uint64 + Capacity uint32 + Active map[uint64]replicatedBackupPin +} + +type replicatedPinOperation byte + +const ( + replicatedPinAdmit replicatedPinOperation = iota + 1 + replicatedPinExtend + replicatedPinRelease + replicatedPinExpire +) + +type replicatedPinCommand struct { + Operation replicatedPinOperation + Identity BackupIdentity + PinID BackupPinID + ReadTS uint64 + Deadline int64 + ExpectedDeadline int64 +} + +// transition has no clock input. Deadlines only identify the expiry version; +// the leader decides when to propose an expiry, never an applying replica. +// The receiver remains unchanged so a failed durable write cannot publish it. +func (s replicatedBackupPins) transition(command replicatedPinCommand) (replicatedBackupPins, bool, error) { + if err := s.validateCommandIdentity(command); err != nil { + return s, false, err + } + next := s + next.Active = maps.Clone(s.Active) + if next.Active == nil { + next.Active = make(map[uint64]replicatedBackupPin) + } + changed, err := next.apply(command) + if err != nil || !changed { + return s, false, err + } + if s.PinGeneration == math.MaxUint64 { + return s, false, errors.Wrap(ErrReplicatedBackupPin, "pin generation exhausted") + } + next.PinGeneration++ + return next, true, nil +} + +func (s replicatedBackupPins) validateCommandIdentity(command replicatedPinCommand) error { + if s.Epoch == 0 || command.Identity.Epoch != s.Epoch || command.Identity.Generation == 0 || command.PinID.IsZero() { + return errors.Wrap(ErrReplicatedBackupPin, "invalid or stale backup identity") + } + if active, ok := s.Active[command.Identity.Generation]; ok && active.PinID != command.PinID { + return errors.Wrap(ErrReplicatedBackupPin, "generation already belongs to another pin") + } + return nil +} + +func (s *replicatedBackupPins) apply(command replicatedPinCommand) (bool, error) { + switch command.Operation { + case replicatedPinAdmit: + return s.admit(command) + case replicatedPinExtend: + return s.extend(command) + case replicatedPinRelease: + return s.retire(command.Identity.Generation), nil + case replicatedPinExpire: + return s.expire(command) + default: + return false, errors.Wrap(ErrReplicatedBackupPin, "unknown pin operation") + } +} + +func (s *replicatedBackupPins) admit(command replicatedPinCommand) (bool, error) { + if command.ReadTS == 0 || command.ReadTS == math.MaxUint64 || command.Deadline <= 0 { + return false, errors.Wrap(ErrReplicatedBackupPin, "invalid pin timestamp or deadline") + } + pin := replicatedBackupPin{PinID: command.PinID, ReadTS: command.ReadTS, Deadline: command.Deadline} + if active, ok := s.Active[command.Identity.Generation]; ok { + if active != pin { + return false, errors.Wrap(ErrReplicatedBackupPin, "initial admission cannot replace an active pin") + } + return false, nil + } + if command.Identity.Generation <= s.RetiredThrough || command.ReadTS < s.ReadFloor { + return false, errors.Wrap(ErrReplicatedBackupPin, "pin is behind retirement or read floor") + } + if s.Capacity != 0 && uint64(len(s.Active)) >= uint64(s.Capacity) { + return false, errors.WithStack(ErrTooManyActiveBackups) + } + s.Active[command.Identity.Generation] = pin + s.WriteTimestampFloor = max(s.WriteTimestampFloor, command.ReadTS) + return true, nil +} + +func (s *replicatedBackupPins) extend(command replicatedPinCommand) (bool, error) { + active, ok := s.Active[command.Identity.Generation] + if !ok || command.Deadline <= 0 { + return false, errors.Wrap(ErrReplicatedBackupPin, "cannot extend a missing pin or invalid deadline") + } + if command.Deadline <= active.Deadline { + return false, nil + } + active.Deadline = command.Deadline + s.Active[command.Identity.Generation] = active + return true, nil +} + +func (s *replicatedBackupPins) expire(command replicatedPinCommand) (bool, error) { + if command.ExpectedDeadline <= 0 { + return false, errors.Wrap(ErrReplicatedBackupPin, "invalid expected deadline") + } + if active, ok := s.Active[command.Identity.Generation]; ok && active.Deadline != command.ExpectedDeadline { + return false, nil + } + return s.retire(command.Identity.Generation), nil +} + +func (s *replicatedBackupPins) retire(generation uint64) bool { + _, active := s.Active[generation] + delete(s.Active, generation) + if generation > s.RetiredThrough { + s.RetiredThrough = generation + return true + } + return active +} + +func (s replicatedBackupPins) oldest() uint64 { + var oldest uint64 + for _, pin := range s.Active { + if oldest == 0 || pin.ReadTS < oldest { + oldest = pin.ReadTS + } + } + return oldest +} diff --git a/kv/replicated_backup_pins_test.go b/kv/replicated_backup_pins_test.go new file mode 100644 index 000000000..c4e777b9c --- /dev/null +++ b/kv/replicated_backup_pins_test.go @@ -0,0 +1,120 @@ +package kv + +import ( + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func pinCommand(operation replicatedPinOperation, generation uint64) replicatedPinCommand { + return replicatedPinCommand{ + Operation: operation, Identity: BackupIdentity{Epoch: 1, Generation: generation}, + PinID: backupTrackerTestPinID(1), ReadTS: 10, Deadline: 100, ExpectedDeadline: 100, + } +} + +func applyPinCommand(t *testing.T, s replicatedBackupPins, command replicatedPinCommand, changed bool) replicatedBackupPins { + t.Helper() + next, didChange, err := s.transition(command) + require.NoError(t, err) + require.Equal(t, changed, didChange) + return next +} + +func TestReplicatedBackupPinRetiresMissingBeforeAdmission(t *testing.T) { + t.Parallel() + for _, operation := range []replicatedPinOperation{replicatedPinRelease, replicatedPinExpire} { + t.Run(map[replicatedPinOperation]string{replicatedPinRelease: "release", replicatedPinExpire: "expire"}[operation], func(t *testing.T) { + t.Parallel() + s := replicatedBackupPins{Epoch: 1, Capacity: 1} + s = applyPinCommand(t, s, pinCommand(operation, 5), true) + require.Equal(t, uint64(5), s.RetiredThrough) + require.Equal(t, uint64(1), s.PinGeneration) + s = applyPinCommand(t, s, pinCommand(operation, 5), false) + for _, generation := range []uint64{1, 5} { + next, changed, err := s.transition(pinCommand(replicatedPinAdmit, generation)) + require.ErrorIs(t, err, ErrReplicatedBackupPin) + require.False(t, changed) + require.Equal(t, s, next) + } + s = applyPinCommand(t, s, pinCommand(replicatedPinAdmit, 6), true) + require.Len(t, s.Active, 1) + }) + } +} + +func TestReplicatedBackupPinExtendAndExpiryOrdering(t *testing.T) { + t.Parallel() + initial := applyPinCommand(t, replicatedBackupPins{Epoch: 1}, pinCommand(replicatedPinAdmit, 1), true) + extend := pinCommand(replicatedPinExtend, 1) + extend.Deadline = 200 + extended := applyPinCommand(t, initial, extend, true) + staleExpiry := applyPinCommand(t, extended, pinCommand(replicatedPinExpire, 1), false) + require.Equal(t, extended, staleExpiry) + require.Zero(t, staleExpiry.RetiredThrough) + require.Equal(t, uint64(10), staleExpiry.oldest(), "a passed deadline alone never removes retention") + + expiry := pinCommand(replicatedPinExpire, 1) + expiry.ExpectedDeadline = 200 + expired := applyPinCommand(t, extended, expiry, true) + require.Zero(t, expired.oldest()) + require.Equal(t, uint64(3), expired.PinGeneration) + _, _, err := expired.transition(extend) + require.ErrorIs(t, err, ErrReplicatedBackupPin, "Extend cannot resurrect a retired identity") + require.Equal(t, int64(100), initial.Active[1].Deadline, "transitions must not mutate their input") +} + +func TestReplicatedBackupPinHigherRetirementPreservesActiveOlderPin(t *testing.T) { + t.Parallel() + s := applyPinCommand(t, replicatedBackupPins{Epoch: 1}, pinCommand(replicatedPinAdmit, 1), true) + s = applyPinCommand(t, s, pinCommand(replicatedPinRelease, 5), true) + s = applyPinCommand(t, s, pinCommand(replicatedPinAdmit, 1), false) + extend := pinCommand(replicatedPinExtend, 1) + extend.Deadline = 200 + s = applyPinCommand(t, s, extend, true) + require.Equal(t, uint64(5), s.RetiredThrough) + require.Equal(t, int64(200), s.Active[1].Deadline) + _, _, err := s.transition(pinCommand(replicatedPinAdmit, 2)) + require.ErrorIs(t, err, ErrReplicatedBackupPin) + s = applyPinCommand(t, s, pinCommand(replicatedPinRelease, 1), true) + require.Empty(t, s.Active) + require.Equal(t, uint64(5), s.RetiredThrough) +} + +func TestReplicatedBackupPinIdentityFloorAndCapacity(t *testing.T) { + t.Parallel() + s := replicatedBackupPins{Epoch: 1, Capacity: 1, ReadFloor: 10} + s = applyPinCommand(t, s, pinCommand(replicatedPinAdmit, 1), true) + _, _, err := s.transition(pinCommand(replicatedPinAdmit, 2)) + require.ErrorIs(t, err, ErrTooManyActiveBackups) + for _, operation := range []replicatedPinOperation{replicatedPinAdmit, replicatedPinExtend, replicatedPinRelease, replicatedPinExpire} { + command := pinCommand(operation, 1) + command.Identity.Epoch = 2 + _, _, err = s.transition(command) + require.ErrorIs(t, err, ErrReplicatedBackupPin) + command.Identity.Epoch = 1 + command.PinID = backupTrackerTestPinID(2) + _, _, err = s.transition(command) + require.ErrorIs(t, err, ErrReplicatedBackupPin) + } + s = applyPinCommand(t, s, pinCommand(replicatedPinRelease, 1), true) + below := pinCommand(replicatedPinAdmit, 2) + below.ReadTS = 9 + _, _, err = s.transition(below) + require.ErrorIs(t, err, ErrReplicatedBackupPin) + duplicate := pinCommand(replicatedPinAdmit, 2) + s = applyPinCommand(t, s, duplicate, true) + duplicate.Deadline++ + _, _, err = s.transition(duplicate) + require.ErrorIs(t, err, ErrReplicatedBackupPin, "initial admission must never renew") +} + +func TestReplicatedBackupPinGenerationDoesNotWrap(t *testing.T) { + t.Parallel() + s := replicatedBackupPins{Epoch: 1, PinGeneration: math.MaxUint64} + next, changed, err := s.transition(pinCommand(replicatedPinRelease, 1)) + require.ErrorIs(t, err, ErrReplicatedBackupPin) + require.False(t, changed) + require.Equal(t, s, next) +} diff --git a/kv/shard_store_prepare_evidence_test.go b/kv/shard_store_prepare_evidence_test.go index a0809be18..1f02d89aa 100644 --- a/kv/shard_store_prepare_evidence_test.go +++ b/kv/shard_store_prepare_evidence_test.go @@ -230,7 +230,8 @@ func TestPrepareEvidenceSafetyMarkerBlocksLateLegacyImportAtAnyStart(t *testing. func TestPrepareEvidenceSafetyMarkerIsExcludedFromMigration(t *testing.T) { brackets, err := distribution.PlanExportBrackets([]byte(txnBackupPrefix), []byte("!txn|meta}")) require.NoError(t, err) - for _, key := range [][]byte{backupTimestampFloorKey, prepareEvidenceEnabledKey, prepareEvidenceGateKey, prepareEvidenceTimestampFloorKey} { + for _, key := range [][]byte{backupTimestampFloorKey, prepareEvidenceEnabledKey, prepareEvidenceGateKey, prepareEvidenceTimestampFloorKey, + replicatedPinStateKey, []byte(store.ReplicatedControlPrefix + "future-state")} { require.True(t, distribution.IsMigrationKnownInternalKey(key), "the source must not export a group-local protocol marker as user data") for _, bracket := range brackets { @@ -239,12 +240,36 @@ func TestPrepareEvidenceSafetyMarkerIsExcludedFromMigration(t *testing.T) { } } +func TestReplicatedControlMigrationPreservesNormalizedUserKey(t *testing.T) { + raw := replicatedPinStateKey + normalized := ExtractTxnUserKey(raw) + require.Equal(t, []byte("ttl-inline-control|backup-pins"), normalized) + end := append(bytes.Clone(normalized), 0) + brackets, err := distribution.PlanExportBrackets(normalized, end) + require.NoError(t, err) + userBracketSeen := false + for _, bracket := range brackets { + require.False(t, bracket.ContainsRawKey(raw), "raw journal must not enter its normalized route's export") + if bracket.Family == distribution.MigrationFamilyUser { + userBracketSeen = true + require.True(t, bracket.ContainsRawKey(normalized), "a literal user key with the normalized name remains migratable") + } + } + require.True(t, userBracketSeen) + require.False(t, isTxnPeerLocalControlKey(normalized)) +} + func TestPrepareEvidenceSafetyMigrationCannotReplaceLocalProtocolState(t *testing.T) { for _, enabled := range []bool{false, true} { for _, tombstone := range []bool{false, true} { - for _, key := range [][]byte{backupTimestampFloorKey, prepareEvidenceEnabledKey, prepareEvidenceGateKey, prepareEvidenceTimestampFloorKey} { + for _, key := range [][]byte{backupTimestampFloorKey, prepareEvidenceEnabledKey, prepareEvidenceGateKey, prepareEvidenceTimestampFloorKey, + replicatedPinStateKey, []byte(store.ReplicatedControlPrefix + "future-state")} { t.Run(fmt.Sprintf("enabled-%t-delete-%t-%s", enabled, tombstone, key), func(t *testing.T) { _, _, fsm := newPrepareSafetyGate(t) + if bytes.Equal(key, replicatedPinStateKey) { + seedActivatedPinJournal(t, fsm.store) + require.NoError(t, fsm.reloadReplicatedBackupPins(t.Context())) + } if enabled { require.Nil(t, fsm.Apply(EncodePrepareEvidenceFence(0))) } diff --git a/kv/transaction.go b/kv/transaction.go index 12bae41d7..f33f98ca1 100644 --- a/kv/transaction.go +++ b/kv/transaction.go @@ -209,6 +209,9 @@ func (t *TransactionManager) Commit(ctx context.Context, reqs []*pb.Request) (*T if len(reqs) == 0 { return &TransactionResponse{}, nil } + if err := refuseAbsenceProposal(reqs); err != nil { + return nil, err + } owned, err := t.timestampAdmission.Load().prepare(ctx, reqs) if err != nil { return nil, err diff --git a/kv/transcoder.go b/kv/transcoder.go index 2eb1d2f1e..9cd8216ef 100644 --- a/kv/transcoder.go +++ b/kv/transcoder.go @@ -54,6 +54,9 @@ type Elem[T OP] struct { Op T Key []byte Value []byte + // RequireAbsent is an apply-time precondition for a capability-gated, + // single-owner TTL-inline relocation. Tombstones also count as present. + RequireAbsent bool // CommitTSValueOffset, when non-zero, asks the coordinator or forwarded // leader to stamp the resolved transaction commit timestamp into Value at // this byte offset before committing the mutation. diff --git a/kv/txn_keys.go b/kv/txn_keys.go index 25b081a91..b785bdf9a 100644 --- a/kv/txn_keys.go +++ b/kv/txn_keys.go @@ -3,6 +3,8 @@ package kv import ( "bytes" "encoding/binary" + + "github.com/bootjp/elastickv/store" ) const ( @@ -91,7 +93,8 @@ func isTxnInternalKey(key []byte) bool { } func isTxnPeerLocalControlKey(key []byte) bool { - return bytes.Equal(key, migrationConfigurationIndexKey) || + return store.IsReplicatedControlKey(key) || + bytes.Equal(key, migrationConfigurationIndexKey) || bytes.Equal(key, backupTimestampFloorKey) || bytes.Equal(key, prepareEvidenceEnabledKey) || bytes.Equal(key, prepareEvidenceGateKey) || diff --git a/kv/txn_keys_test.go b/kv/txn_keys_test.go index adf39e495..19ac98b1b 100644 --- a/kv/txn_keys_test.go +++ b/kv/txn_keys_test.go @@ -20,6 +20,7 @@ func TestIsTxnInternalKeyReservesOnlyKnownFamilies(t *testing.T) { []byte(txnCommitPrefix + "k"), []byte(txnRollbackPrefix + "k"), []byte(txnMetaPrefix + "k"), + []byte("!txn|meta|ttl-inline-control|backup-pins"), backupTimestampFloorKey, } { require.True(t, isTxnInternalKey(key), "reserved key %q", key) @@ -27,6 +28,7 @@ func TestIsTxnInternalKeyReservesOnlyKnownFamilies(t *testing.T) { for _, key := range [][]byte{ []byte("!txn|foo"), + []byte("!txn|control|backup-pins"), []byte("!txn|"), []byte("!txnfoo"), // The backup family reserves the exact floor key, not the prefix: a diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 133b9a9f3..b31b310a8 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -332,8 +332,12 @@ type Mutation struct { // that durably embed their own creation timestamp but still need leader-side // timestamp allocation after redirect/retry. CommitTsValueOffset uint64 `protobuf:"varint,4,opt,name=commit_ts_value_offset,json=commitTsValueOffset,proto3" json:"commit_ts_value_offset,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // require_absent is the TTL-inline relocation precondition: no committed + // version of key, including tombstones, may exist when the 1PC applies. + // It requires the replicated TTL-inline capability on every voter/learner. + RequireAbsent bool `protobuf:"varint,5,opt,name=require_absent,json=requireAbsent,proto3" json:"require_absent,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Mutation) Reset() { @@ -394,6 +398,13 @@ func (x *Mutation) GetCommitTsValueOffset() uint64 { return 0 } +func (x *Mutation) GetRequireAbsent() bool { + if x != nil { + return x.RequireAbsent + } + return false +} + type Request struct { state protoimpl.MessageState `protogen:"open.v1"` IsTxn bool `protobuf:"varint,1,opt,name=is_txn,json=isTxn,proto3" json:"is_txn,omitempty"` @@ -2813,12 +2824,13 @@ var File_internal_proto protoreflect.FileDescriptor const file_internal_proto_rawDesc = "" + "\n" + - "\x0einternal.proto\"|\n" + + "\x0einternal.proto\"\xa3\x01\n" + "\bMutation\x12\x13\n" + "\x02op\x18\x01 \x01(\x0e2\x03.OpR\x02op\x12\x10\n" + "\x03key\x18\x02 \x01(\fR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value\x123\n" + - "\x16commit_ts_value_offset\x18\x04 \x01(\x04R\x13commitTsValueOffset\"\xbd\x02\n" + + "\x16commit_ts_value_offset\x18\x04 \x01(\x04R\x13commitTsValueOffset\x12%\n" + + "\x0erequire_absent\x18\x05 \x01(\bR\rrequireAbsent\"\xbd\x02\n" + "\aRequest\x12\x15\n" + "\x06is_txn\x18\x01 \x01(\bR\x05isTxn\x12\x1c\n" + "\x05phase\x18\x02 \x01(\x0e2\x06.PhaseR\x05phase\x12\x0e\n" + diff --git a/proto/internal.proto b/proto/internal.proto index ab8edbe2d..142f6f501 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -42,6 +42,10 @@ message Mutation { // that durably embed their own creation timestamp but still need leader-side // timestamp allocation after redirect/retry. uint64 commit_ts_value_offset = 4; + // require_absent is the TTL-inline relocation precondition: no committed + // version of key, including tombstones, may exist when the 1PC applies. + // It requires the replicated TTL-inline capability on every voter/learner. + bool require_absent = 5; } enum Phase { diff --git a/store/absence_guard.go b/store/absence_guard.go new file mode 100644 index 000000000..e5b71c32d --- /dev/null +++ b/store/absence_guard.go @@ -0,0 +1,45 @@ +package store + +import ( + "context" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +func hasAbsenceGuards(mutations []*KVPairMutation) bool { + for _, mutation := range mutations { + if mutation.RequireAbsent { + return true + } + } + return false +} + +// Called under applyMu, against the indexed batch for entry-scoped apply so +// earlier requests in that same Raft entry are also visible to the predicate. +func (s *pebbleStore) checkAbsenceGuardsWith(ctx context.Context, r pebble.Reader, mutations []*KVPairMutation) error { + if !hasAbsenceGuards(mutations) { + return nil + } + iter, err := r.NewIter(&pebble.IterOptions{}) + if err != nil { + return errors.WithStack(err) + } + defer iter.Close() + var seekKey, upperBound []byte + for _, mutation := range mutations { + if !mutation.RequireAbsent { + continue + } + _, exists, err := s.latestCommitTSWithIter(ctx, iter, mutation.Key, &seekKey, &upperBound, false, 0) + if err != nil { + return err + } + if exists { + s.writeConflicts.record(WriteConflictKindWrite, classifyWriteConflictKey(mutation.Key)) + return NewWriteConflictError(mutation.Key) + } + } + return nil +} diff --git a/store/absence_guard_test.go b/store/absence_guard_test.go new file mode 100644 index 000000000..937482a61 --- /dev/null +++ b/store/absence_guard_test.go @@ -0,0 +1,106 @@ +package store + +import ( + "context" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func absenceTestStores(t *testing.T) map[string]func(*testing.T) MVCCStore { + t.Helper() + return map[string]func(*testing.T) MVCCStore{ + "memory": func(t *testing.T) MVCCStore { t.Helper(); return NewMVCCStore() }, + "pebble": func(t *testing.T) MVCCStore { + t.Helper() + s, err := NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + return s + }, + } +} + +func TestApplyMutationsRequireAbsentRejectsOldVersionsAtomically(t *testing.T) { + t.Parallel() + for name, factory := range absenceTestStores(t) { + for _, deleted := range []bool{false, true} { + t.Run(name+map[bool]string{false: "/value", true: "/tombstone"}[deleted], func(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := factory(t) + key := []byte("metadata") + require.NoError(t, s.PutAt(ctx, key, []byte("user"), 5, 0)) + if deleted { + require.NoError(t, s.DeleteAt(ctx, key, 6)) + } + // Ordinary OCC at StartTS=100 does not reject this older version. + err := s.ApplyMutationsRaftAt(ctx, []*KVPairMutation{ + {Op: OpTypePut, Key: []byte("other"), Value: []byte("must not land")}, + {Op: OpTypePut, Key: key, Value: []byte("migration"), RequireAbsent: true}, + }, nil, 100, 101, 1) + require.Error(t, err) + _, err = s.GetAt(ctx, []byte("other"), 200) + require.ErrorIs(t, err, ErrKeyNotFound) + got, err := s.GetAt(ctx, key, 200) + if deleted { + require.ErrorIs(t, err, ErrKeyNotFound) + } else { + require.NoError(t, err) + require.Equal(t, []byte("user"), got) + } + }) + } + } +} + +func TestApplyMutationsRequireAbsentAllowsOnlyOneConcurrentWinner(t *testing.T) { + t.Parallel() + for name, factory := range absenceTestStores(t) { + t.Run(name, func(t *testing.T) { + t.Parallel() + s := factory(t) + start := make(chan struct{}) + results := make(chan error, 2) + var workers sync.WaitGroup + for _, startTS := range []uint64{100, 200} { + workers.Add(1) + go func() { + defer workers.Done() + <-start + results <- s.ApplyMutations(context.Background(), []*KVPairMutation{{Op: OpTypePut, Key: []byte("key"), Value: []byte("value"), RequireAbsent: true}}, nil, startTS, startTS+1) + }() + } + close(start) + workers.Wait() + close(results) + var successes int + for err := range results { + if err == nil { + successes++ + } + } + require.Equal(t, 1, successes) + }) + } +} + +func TestApplyRaftEntryAbsenceGuardSeesEarlierRequest(t *testing.T) { + ctx := context.Background() + s := newApplyIndexPebbleStore(t) + applier := raftEntryApplier(t, s) + require.NoError(t, applier.ApplyRaftEntry(ctx, 7, func(ctx context.Context) error { + require.NoError(t, s.ApplyMutationsRaftAt(ctx, raftEntryPut("metadata", "user"), nil, 4, 5, 7)) + err := s.ApplyMutationsRaftAt(ctx, []*KVPairMutation{ + {Op: OpTypePut, Key: []byte("other"), Value: []byte("must not land")}, + {Op: OpTypePut, Key: []byte("metadata"), Value: []byte("migration"), RequireAbsent: true}, + }, nil, 100, 101, 7) + require.ErrorIs(t, err, ErrWriteConflict) + return nil + })) + require.False(t, raftEntryExists(t, s, "other")) + got, err := s.GetAt(ctx, []byte("metadata"), 200) + require.NoError(t, err) + require.Equal(t, []byte("user"), got) +} diff --git a/store/lsm_store.go b/store/lsm_store.go index 3cc2faf3a..765dfa3a7 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -2597,6 +2597,9 @@ func (s *pebbleStore) checkApplyConflicts(ctx context.Context, mutations []*KVPa // batch and a predicate that also counts the entry's staged commits inside // a Raft entry scope. func (s *pebbleStore) checkApplyConflictsWith(ctx context.Context, r pebble.Reader, hasCommitsAfter func(uint64) bool, mutations []*KVPairMutation, readKeys [][]byte, startTS uint64, force bool) error { + if err := s.checkAbsenceGuardsWith(ctx, r, mutations); err != nil { + return err + } if !force && !hasCommitsAfter(startTS) { return nil } diff --git a/store/mvcc_store.go b/store/mvcc_store.go index 5b2bc1451..cc8293c5a 100644 --- a/store/mvcc_store.go +++ b/store/mvcc_store.go @@ -733,7 +733,7 @@ func (s *mvccStore) applyMutations(ctx context.Context, mutations []*KVPairMutat return err } - if s.lastCommitTS > startTS { + if s.lastCommitTS > startTS || hasAbsenceGuards(mutations) { if err := s.checkConflictsLocked(mutations, readKeys, startTS); err != nil { return err } @@ -800,7 +800,7 @@ func (s *mvccStore) ApplyMutationsPreservingLastCommitTS(ctx context.Context, mu func (s *mvccStore) checkConflictsLocked(mutations []*KVPairMutation, readKeys [][]byte, startTS uint64) error { for _, mut := range mutations { - if latestVer, ok := s.latestVersionLocked(mut.Key); ok && latestVer.TS > startTS { + if latestVer, ok := s.latestVersionLocked(mut.Key); ok && (mut.RequireAbsent || latestVer.TS > startTS) { s.writeConflicts.record(WriteConflictKindWrite, classifyWriteConflictKey(mut.Key)) return NewWriteConflictError(mut.Key) } diff --git a/store/replicated_control.go b/store/replicated_control.go new file mode 100644 index 000000000..5e5cfcbce --- /dev/null +++ b/store/replicated_control.go @@ -0,0 +1,127 @@ +package store + +import ( + "bytes" + "context" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +// This is a child of the already-reserved transaction metadata namespace. +// Reserving a new !txn| family would hide valid pre-upgrade user keys. +const ReplicatedControlPrefix = "!txn|meta|ttl-inline-control|" + +var ErrReplicatedControlCollision = errors.New("replicated control state collides with existing MVCC versions") + +// ReplicatedControlStore persists one FSM control-state replacement and its +// applied index atomically. False means the durable store already applied that +// index; a caller must reload its control state rather than install its stale +// computed replacement. The memory backend, like ApplyMutationsRaftAt, has no +// durable applied-index reader; its FSM must deduplicate via its saved state. +type ReplicatedControlStore interface { + ApplyReplicatedControlState(ctx context.Context, key, value []byte, appliedIndex uint64) (bool, error) +} + +// IsReplicatedControlKey identifies state reserved for typed FSM commands. +func IsReplicatedControlKey(key []byte) bool { + return bytes.HasPrefix(key, []byte(ReplicatedControlPrefix)) +} + +func validateReplicatedControlState(ctx context.Context, key, value []byte) error { + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + if !IsReplicatedControlKey(key) || len(key) == len(ReplicatedControlPrefix) { + return errors.New("replicated control state requires a reserved control key") + } + return validateValueSize(value) +} + +func (s *mvccStore) ApplyReplicatedControlState(ctx context.Context, key, value []byte, _ uint64) (bool, error) { + if err := validateReplicatedControlState(ctx, key, value); err != nil { + return false, err + } + s.mtx.Lock() + defer s.mtx.Unlock() + if value, exists := s.tree.Get(key); exists { + versions, _ := value.([]VersionedValue) + for _, version := range versions { + if version.TS != 0 || version.Tombstone || version.ExpireAt != 0 { + return false, errors.WithStack(ErrReplicatedControlCollision) + } + } + } + // Version zero is a fixed internal metadata slot, not a user timestamp. + // Replacing it allocates no HLC value and cannot raise LastCommitTS. + s.putVersionLocked(key, value, 0, 0) + return true, nil +} + +func (s *pebbleStore) ApplyReplicatedControlState(ctx context.Context, key, value []byte, appliedIndex uint64) (bool, error) { + if err := validateReplicatedControlState(ctx, key, value); err != nil { + return false, err + } + s.applyMu.Lock() + defer s.applyMu.Unlock() + s.dbMu.RLock() + defer s.dbMu.RUnlock() + if appliedIndex != 0 { + current, present, err := s.readAppliedIndexLocked() + if err != nil { + return false, err + } + if present && current >= appliedIndex { + return false, nil + } + } + if err := s.checkReplicatedControlCollision(ctx, key); err != nil { + return false, err + } + return s.writeReplicatedControlBatch(key, value, appliedIndex) +} + +func (s *pebbleStore) checkReplicatedControlCollision(ctx context.Context, key []byte) error { + iter, err := s.db.NewIter(&pebble.IterOptions{}) + if err != nil { + return errors.WithStack(err) + } + defer iter.Close() + var seekKey, upperBound []byte + ts, exists, err := s.latestCommitTSWithIter(ctx, iter, key, &seekKey, &upperBound, false, 0) + if err != nil { + return err + } + if !exists { + return nil + } + if ts != 0 { + return errors.WithStack(ErrReplicatedControlCollision) + } + tombstone, expireAt, _, err := decodeValueHeader(iter.Value()) + if err != nil { + return errors.WithStack(err) + } + if tombstone || expireAt != 0 { + return errors.WithStack(ErrReplicatedControlCollision) + } + return nil +} + +func (s *pebbleStore) writeReplicatedControlBatch(key, value []byte, appliedIndex uint64) (bool, error) { + batch := s.db.NewBatch() + defer batch.Close() + if err := s.applyMutationsBatch(batch, []*KVPairMutation{{Op: OpTypePut, Key: key, Value: value}}, 0, false); err != nil { + return false, err + } + if appliedIndex != 0 { + if err := setPebbleUint64InBatch(batch, metaAppliedIndexBytes, appliedIndex); err != nil { + return false, err + } + } + // Control state is a recovery/admission fence even with FSM nosync enabled. + if err := batch.Commit(pebble.Sync); err != nil { + return false, errors.WithStack(err) + } + return true, nil +} diff --git a/store/replicated_control_test.go b/store/replicated_control_test.go new file mode 100644 index 000000000..376a6b78e --- /dev/null +++ b/store/replicated_control_test.go @@ -0,0 +1,142 @@ +package store + +import ( + "bytes" + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func controlStore(t *testing.T, s MVCCStore) ReplicatedControlStore { + t.Helper() + w, ok := s.(ReplicatedControlStore) + require.True(t, ok) + return w +} + +func TestReplicatedControlStateSurvivesSnapshotWithoutTimestampAllocation(t *testing.T) { + t.Parallel() + ctx := context.Background() + for name, factory := range absenceTestStores(t) { + t.Run(name, func(t *testing.T) { + t.Parallel() + s := factory(t) + key := []byte(ReplicatedControlPrefix + "ttl-inline") + applied, err := controlStore(t, s).ApplyReplicatedControlState(ctx, key, []byte("first"), 1) + require.NoError(t, err) + require.True(t, applied) + require.Zero(t, s.LastCommitTS()) + applied, err = controlStore(t, s).ApplyReplicatedControlState(ctx, key, []byte("second"), 2) + require.NoError(t, err) + require.True(t, applied) + snapshot, err := s.Snapshot() + require.NoError(t, err) + defer snapshot.Close() + var data bytes.Buffer + _, err = snapshot.WriteTo(&data) + require.NoError(t, err) + target := factory(t) + require.NoError(t, target.Restore(&data)) + got, err := target.GetAt(ctx, key, ^uint64(0)) + require.NoError(t, err) + require.Equal(t, []byte("second"), got) + require.Zero(t, target.LastCommitTS()) + }) + } +} + +func TestReplicatedControlStateReopenAndIndexFence(t *testing.T) { + t.Parallel() + ctx := context.Background() + dir := t.TempDir() + s, err := NewPebbleStore(dir) + require.NoError(t, err) + key := []byte(ReplicatedControlPrefix + "ttl-inline") + w := controlStore(t, s) + applied, err := w.ApplyReplicatedControlState(ctx, key, []byte("current"), 7) + require.NoError(t, err) + require.True(t, applied) + applied, err = w.ApplyReplicatedControlState(ctx, key, []byte("stale"), 6) + require.NoError(t, err) + require.False(t, applied) + require.NoError(t, s.Close()) + s, err = NewPebbleStore(dir) + require.NoError(t, err) + defer s.Close() + got, err := s.GetAt(ctx, key, ^uint64(0)) + require.NoError(t, err) + require.Equal(t, []byte("current"), got) + index, present, err := pebbleStoreApplied(t, s).LastAppliedIndex() + require.NoError(t, err) + require.True(t, present) + require.Equal(t, uint64(7), index) + require.Zero(t, s.LastCommitTS()) + _, err = controlStore(t, s).ApplyReplicatedControlState(ctx, []byte("ordinary"), []byte("invalid"), 8) + require.Error(t, err) +} + +func TestReplicatedControlStateEncryptedSnapshotAndReopen(t *testing.T) { + t.Parallel() + ctx := context.Background() + fixture := newEncryptedStoreFixture(t, 1) + key := []byte(ReplicatedControlPrefix + "backup-pins") + value := []byte("replicated epoch and active pin state") + applied, err := controlStore(t, fixture.mvcc).ApplyReplicatedControlState(ctx, key, value, 7) + require.NoError(t, err) + require.True(t, applied) + snapshot, err := fixture.mvcc.Snapshot() + require.NoError(t, err) + defer snapshot.Close() + var data bytes.Buffer + _, err = snapshot.WriteTo(&data) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + fixture.closeIfOpen(t) + fixture.reopen(t) + got, err := fixture.mvcc.GetAt(ctx, key, ^uint64(0)) + require.NoError(t, err) + require.Equal(t, value, got) + require.NoError(t, fixture.mvcc.Restore(&data)) + got, err = fixture.mvcc.GetAt(ctx, key, ^uint64(0)) + require.NoError(t, err) + require.Equal(t, value, got) + require.Zero(t, fixture.mvcc.LastCommitTS()) + index, present, err := pebbleStoreApplied(t, fixture.mvcc).LastAppliedIndex() + require.NoError(t, err) + require.True(t, present) + require.Equal(t, uint64(7), index) +} + +func TestReplicatedControlStateRejectsExistingVersionOrTombstone(t *testing.T) { + t.Parallel() + ctx := context.Background() + for name, factory := range absenceTestStores(t) { + for _, deleted := range []bool{false, true} { + t.Run(name+map[bool]string{false: "/value", true: "/tombstone"}[deleted], func(t *testing.T) { + t.Parallel() + st := factory(t) + key := []byte(ReplicatedControlPrefix + "backup-pins") + require.NoError(t, st.PutAt(ctx, key, []byte("preexisting"), 10, 0)) + if deleted { + require.NoError(t, st.DeleteAt(ctx, key, 11)) + } + applied, err := controlStore(t, st).ApplyReplicatedControlState(ctx, key, []byte("control"), 7) + require.ErrorIs(t, err, ErrReplicatedControlCollision) + require.False(t, applied) + got, err := st.GetAt(ctx, key, ^uint64(0)) + if deleted { + require.ErrorIs(t, err, ErrKeyNotFound) + } else { + require.NoError(t, err) + require.Equal(t, []byte("preexisting"), got) + } + if reader, ok := st.(interface{ LastAppliedIndex() (uint64, bool, error) }); ok { + _, present, err := reader.LastAppliedIndex() + require.NoError(t, err) + require.False(t, present, "a refused installation must not publish its applied index") + } + }) + } + } +} diff --git a/store/store.go b/store/store.go index 7fbe1b22b..bf5cd47c3 100644 --- a/store/store.go +++ b/store/store.go @@ -472,6 +472,10 @@ type KVPairMutation struct { Op OpType Key []byte Value []byte + // RequireAbsent rejects the entire batch if Key has any committed version, + // including a tombstone or a version at/below StartTS. Checked atomically + // with application; ordinary mutations leave this false. + RequireAbsent bool // ExpireAt is an HLC timestamp; 0 means no TTL. ExpireAt uint64 }