From 03ff0a9c6b213b49b9bcfa30f9da18fdcef122dc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 17 Nov 2026 15:04:26 +0000 Subject: [PATCH 1/3] Fix listdir limit pushdown, LSM density, and memtable backpressure. Stop directory scans once the requested live rows are found, rebuild locality density from SSTables on open, stream compaction instead of loading each SST into one map, and honor memtable_size_bytes plus the immutable-table cap. Co-authored-by: RustFS --- crates/metadb-raft/src/state_machine.rs | 90 ++++- crates/metadb-server/src/config.rs | 10 + crates/metadb-server/src/lib.rs | 31 +- crates/metadb-storage/src/backend.rs | 79 ++--- crates/metadb-storage/src/block.rs | 34 ++ crates/metadb-storage/src/compaction.rs | 19 ++ crates/metadb-storage/src/engine.rs | 314 +++++++++++++++++- crates/metadb-storage/src/lib.rs | 4 +- crates/metadb-storage/src/mvcc.rs | 50 ++- crates/metadb-storage/src/sstable.rs | 141 +++++++- crates/metadb-storage/src/version_set.rs | 213 +++++++++++- .../tests/first_ship_storage_regressions.rs | 274 +++++++++++++++ crates/metadb-tx/src/pushdown.rs | 105 +++++- crates/metadb-tx/src/two_pc.rs | 37 ++- docs/designs/correctness_hardening.md | 2 +- docs/en/architecture.md | 2 +- docs/en/configuration.md | 5 +- docs/en/operations.md | 2 +- docs/guides/en/deployment.md | 3 +- docs/guides/en/operations-and-tuning.md | 2 +- 20 files changed, 1312 insertions(+), 105 deletions(-) create mode 100644 crates/metadb-storage/tests/first_ship_storage_regressions.rs diff --git a/crates/metadb-raft/src/state_machine.rs b/crates/metadb-raft/src/state_machine.rs index 8aec598..d1142d5 100644 --- a/crates/metadb-raft/src/state_machine.rs +++ b/crates/metadb-raft/src/state_machine.rs @@ -24,7 +24,7 @@ use metadb_core::{ }; use metadb_storage::sstable::{sst_file_name, sync_dir}; use metadb_storage::{ - DurableShardCheckpoint, LifecycleStorageEngine, LsmEngineConfig, PreFlushHook, + DurableShardCheckpoint, FlushIdentity, LifecycleStorageEngine, LsmEngineConfig, PreFlushHook, ShardedBlockCache, }; @@ -141,6 +141,37 @@ struct PersistedSnapshotMeta { file_specs: Vec<(String, u64, u32)>, } +fn attach_flush_hook(engine: &LifecycleStorageEngine, handler: &Arc) { + engine.set_pre_flush_hook(Arc::new(HandlerFlushHook { + handler: handler.clone(), + })); +} + +struct HandlerFlushHook { + handler: Arc, +} + +impl PreFlushHook for HandlerFlushHook { + fn on_seal_memtable( + &self, + sealed_raft_index: u64, + sealed_max_hlc: HlcTimestamp, + view_at_m: &metadb_storage::ViewAtRaftIndex<'_>, + ) -> Result, HlcTimestamp, metadb_storage::MvccValue)>> { + match self.handler.pre_flush_hook() { + Some(hook) => hook.on_seal_memtable(sealed_raft_index, sealed_max_hlc, view_at_m), + None => Ok(Vec::new()), + } + } + + fn on_flush_durable(&self, durable_raft_index: u64, durable_max_hlc: HlcTimestamp) -> u64 { + match self.handler.pre_flush_hook() { + Some(hook) => hook.on_flush_durable(durable_raft_index, durable_max_hlc), + None => 0, + } + } +} + fn checkpoint_log_id(checkpoint: &DurableShardCheckpoint) -> Option> { if ( checkpoint.flushed_raft_term, @@ -223,6 +254,41 @@ impl StateMachineShared { Ok((state_flushed, updated_cp)) } + fn refresh_flushed_checkpoint( + &self, + engine: &LifecycleStorageEngine, + log_id: &LogIdOf, + applied_hlc: HlcTimestamp, + ) -> Result<()> { + let current = engine.durable_checkpoint(); + // Only the apply that flushed this index may stamp its term and descriptor. + if current.flushed_raft_index != log_id.index { + return Ok(()); + } + let membership = self.last_membership.read().unwrap().clone(); + let membership_bytes = bincode::serialize(&membership).unwrap_or_default(); + let descriptor = self.handler.encode_descriptor_bytes(); + let corrected = DurableShardCheckpoint { + flushed_raft_term: log_id.committed_leader_id().term, + flushed_raft_leader: log_id.committed_leader_id().node_id, + flushed_raft_index: current.flushed_raft_index, + flushed_max_hlc: current.flushed_max_hlc.max(applied_hlc), + contiguous_aux_acked_index: current.contiguous_aux_acked_index, + membership_bytes, + descriptor_bytes: descriptor, + }; + if corrected != current { + engine.persist_checkpoint(corrected.clone())?; + } + self.watermarks + .advance_state_flushed(corrected.flushed_raft_index); + if self.handler.pre_flush_hook().is_some() { + self.watermarks + .advance_aux_outbox_durable(engine.last_aux_frontier()); + } + Ok(()) + } + fn build_snapshot_sync(&self) -> io::Result> { let _operation = self.operation_lock.lock().unwrap(); let current_idx = self @@ -454,6 +520,7 @@ impl ShardStateMachine { block_cache.clone(), lsm_config.clone(), )?); + attach_flush_hook(&engine, &handler); let cp = engine.durable_checkpoint(); // Report `MANIFEST`'s `flushed_raft_log_id` (`F`) as `applied_state` on startup so OpenRaft @@ -632,10 +699,30 @@ impl RaftStateMachine for ShardStateMachine { actual: cur_epoch, }) } else { + let membership = self.shared.last_membership.read().unwrap().clone(); + engine.set_flush_identity(FlushIdentity { + term: log_id.committed_leader_id().term, + leader: log_id.committed_leader_id().node_id, + membership: bincode::serialize(&membership).unwrap_or_default(), + descriptor: self.shared.handler.encode_descriptor_bytes(), + aux_index: self.shared.watermarks.snapshot_values().1, + }); self.shared .handler .apply_command(log_id.index, commit_hlc, &req, &engine) }; + let apply_res = match apply_res { + Ok(bytes) => self + .shared + .refresh_flushed_checkpoint(&engine, &log_id, commit_hlc) + .map(|()| bytes), + Err(error) => { + let _ = self + .shared + .refresh_flushed_checkpoint(&engine, &log_id, commit_hlc); + Err(error) + } + }; let updated_epoch = self.shared.handler.current_epoch(); MetaDbRaftResponse { @@ -852,6 +939,7 @@ impl RaftStateMachine for ShardStateMachine { ) .map_err(|e| io::Error::other(e.to_string()))?, ); + attach_flush_hook(&reopened, &self.shared.handler); *self.shared.engine.write().unwrap() = reopened.clone(); *self.shared.last_applied_log_id.write().unwrap() = meta.last_log_id; diff --git a/crates/metadb-server/src/config.rs b/crates/metadb-server/src/config.rs index e7f73ed..e51afe8 100644 --- a/crates/metadb-server/src/config.rs +++ b/crates/metadb-server/src/config.rs @@ -75,6 +75,9 @@ pub struct StorageSection { pub bloom_bits_per_key: usize, pub locality_tombstone_threshold: usize, pub locality_tombstone_ratio: f64, + /// Queued immutable memtables allowed before writers wait for a flush. + /// The storage design documents a backpressure limit of four. + pub max_imm_memtables: usize, } impl Default for StorageSection { @@ -86,6 +89,7 @@ impl Default for StorageSection { bloom_bits_per_key: 10, locality_tombstone_threshold: 128, locality_tombstone_ratio: 0.50, + max_imm_memtables: 4, } } } @@ -170,6 +174,8 @@ impl ServerTomlConfig { tombstone_min_ratio_permille: ratio_permille, bloom_bits_per_key: self.storage.bloom_bits_per_key, auto_locality_compaction: true, + memtable_size_bytes: self.storage.memtable_size_bytes.max(1), + max_imm_memtables: self.storage.max_imm_memtables.max(1), }) } } @@ -212,5 +218,9 @@ mod tests { .unwrap(); assert_eq!(cfg.storage.memtable_size_bytes, 1024); assert_eq!(cfg.storage.block_cache_bytes, 65536); + assert_eq!(cfg.storage.max_imm_memtables, 4); + let lsm = cfg.to_lsm_config().unwrap(); + assert_eq!(lsm.memtable_size_bytes, 1024); + assert_eq!(lsm.max_imm_memtables, 4); } } diff --git a/crates/metadb-server/src/lib.rs b/crates/metadb-server/src/lib.rs index 8fd724e..48b0a0f 100644 --- a/crates/metadb-server/src/lib.rs +++ b/crates/metadb-server/src/lib.rs @@ -367,9 +367,21 @@ impl MetaDbEngine { /// Lists all live child entries in directory `pid`. pub fn listdir(&self, pid: InodeId) -> Result> { + self.listdir_limited(pid, None, None, usize::MAX) + } + + /// Pinned directory page. `limit == usize::MAX` reads the whole directory. + pub fn listdir_limited( + &self, + pid: InodeId, + after_name: Option<&str>, + name_prefix: Option<&str>, + limit: usize, + ) -> Result> { let hlc = self.clock.now(); let (_, _, storage) = self.global_manager.route_pid(pid)?; - self.pushdown.listdir(&storage, pid, hlc) + self.pushdown + .listdir_limited(&storage, pid, after_name, name_prefix, limit, hlc) } /// Executes a lightweight metadata SQL template statement directly without full SQL optimizer overhead. @@ -411,16 +423,13 @@ impl MetaDbEngine { name_prefix, limit, } => { - let mut rows = self.listdir(pid)?; - if let Some(after) = after_name { - rows.retain(|e| e.key.name > after); - } - if let Some(pfx) = name_prefix { - rows.retain(|e| e.key.name.starts_with(&pfx)); - } - if limit > 0 && rows.len() > limit { - rows.truncate(limit); - } + let page_limit = if limit == 0 { usize::MAX } else { limit }; + let rows = self.listdir_limited( + pid, + after_name.as_deref(), + name_prefix.as_deref(), + page_limit, + )?; Ok(SqlQueryResult::Entries(rows)) } MetadataSqlCommand::Rename { diff --git a/crates/metadb-storage/src/backend.rs b/crates/metadb-storage/src/backend.rs index 9a0ce10..a906cfa 100644 --- a/crates/metadb-storage/src/backend.rs +++ b/crates/metadb-storage/src/backend.rs @@ -12,7 +12,7 @@ use crate::block::BlockEntryValue; use crate::cache::ShardedBlockCache; use crate::manifest::{DurableShardCheckpoint, ManifestLog, VersionEdit}; use crate::sstable::{sst_file_name, sync_dir, SstFileMeta, SstableBuilder, SstableReader}; -use crate::version_set::{Version, NUM_LSM_LEVELS}; +use crate::version_set::{MergedCursor, Version, NUM_LSM_LEVELS}; pub const DEFAULT_L0_COMPACTION_TRIGGER: usize = 4; pub const TARGET_L1_SST_BYTES: usize = 64 * 1024 * 1024; @@ -469,63 +469,36 @@ impl StorageBackend for LsmStorageBackend { lvl.sort_by(|a, b| a.meta().min_key.cmp(&b.meta().min_key)); } - // Step 3: Merge 100% of keys from all participating SSTables in deterministic precedence order! - let mut merged: BTreeMap, BlockEntryValue> = BTreeMap::new(); + // Step 3: Stream every key from the participating SSTables. Earlier readers + // win (`L0` newest flush generation, then `L1..L3`). One data block per + // input file is resident; the output is flushed in `TARGET_L1_SST_BYTES` chunks. let mut max_raft_idx = 0u64; let mut max_hlc = metadb_core::HlcTimestamp::ZERO; - let mut input_tombstones_in_range = 0usize; - + let mut readers = Vec::new(); for sst in &participating[0] { max_raft_idx = max_raft_idx.max(sst.meta().max_raft_index); max_hlc = max_hlc.max(sst.meta().max_hlc); - for (k, v) in sst.read_all_entries()? { - merged.entry(k).or_insert(v); - } + readers.push(sst.clone()); } for lvl in &participating[1..] { for sst in lvl { max_raft_idx = max_raft_idx.max(sst.meta().max_raft_index); max_hlc = max_hlc.max(sst.meta().max_hlc); - for (k, v) in sst.read_all_entries()? { - merged.entry(k).or_insert(v); - } + readers.push(sst.clone()); } } + let mut cursor = MergedCursor::open(readers, &[], None)?; // Step 4: Bottommost Tombstone Drop Invariant (`Resolution B4`): // Drop `PhysicalTombstone` iff zero non-participating SSTables in levels `> target_level` // overlap `K`. For `K ∈ [start_key, end_key)`, `target_level` already absorbed the deepest // overlapping SSTable so `has_overlap_above` is guaranteed `false`; for outer boundary keys, // `base_version.has_overlap_above(target_level, &k)` preserves tombstones whenever needed! - let mut output_entries: Vec<(Vec, BlockEntryValue)> = Vec::with_capacity(merged.len()); - let mut purged_tombstones = 0usize; - - for (k, v) in merged { - match v { - BlockEntryValue::Value(bytes) => { - output_entries.push((k, BlockEntryValue::Value(bytes))); - } - BlockEntryValue::PhysicalTombstone => { - let in_target_range = k.as_slice() >= start_key && k.as_slice() < end_key; - if in_target_range { - input_tombstones_in_range += 1; - } - if !base_version.has_overlap_above(target_level, &k) { - purged_tombstones += 1; - } else { - output_entries.push((k, BlockEntryValue::PhysicalTombstone)); - } - } - } - } - let _ = input_tombstones_in_range; - - // Partition output into <= TARGET_L1_SST_BYTES non-overlapping SSTables at `target_level`. let mut new_metas: Vec = Vec::new(); let mut new_readers: Vec> = Vec::new(); - let mut chunk: Vec<(Vec, BlockEntryValue)> = Vec::new(); let mut chunk_bytes = 0usize; + let mut purged_tombstones = 0usize; let write_chunk = |items: &[(Vec, BlockEntryValue)]| -> Result<(SstFileMeta, Arc)> { @@ -549,27 +522,41 @@ impl StorageBackend for LsmStorageBackend { Ok((meta, reader)) }; - for (k, v) in output_entries { + while let Some((k, v)) = cursor.next_owned()? { + let emitted = match v { + BlockEntryValue::Value(bytes) => Some(BlockEntryValue::Value(bytes)), + BlockEntryValue::PhysicalTombstone => { + if !base_version.has_overlap_above(target_level, &k) { + purged_tombstones += 1; + None + } else { + Some(BlockEntryValue::PhysicalTombstone) + } + } + }; + let Some(value) = emitted else { + continue; + }; let sz = k.len() - + match &v { - BlockEntryValue::Value(b) => b.len(), + + match &value { + BlockEntryValue::Value(bytes) => bytes.len(), BlockEntryValue::PhysicalTombstone => 0, } + 16; - chunk.push((k, v)); + chunk.push((k, value)); chunk_bytes += sz; if chunk_bytes >= TARGET_L1_SST_BYTES { - let (m, r) = write_chunk(&chunk)?; - new_metas.push(m); - new_readers.push(r); + let (meta, reader) = write_chunk(&chunk)?; + new_metas.push(meta); + new_readers.push(reader); chunk.clear(); chunk_bytes = 0; } } if !chunk.is_empty() { - let (m, r) = write_chunk(&chunk)?; - new_metas.push(m); - new_readers.push(r); + let (meta, reader) = write_chunk(&chunk)?; + new_metas.push(meta); + new_readers.push(reader); } // Step 5: Atomically publish `VersionEdit` (taking care to retain any newer L0 files flushed diff --git a/crates/metadb-storage/src/block.rs b/crates/metadb-storage/src/block.rs index 796fbbb..c6d79bd 100644 --- a/crates/metadb-storage/src/block.rs +++ b/crates/metadb-storage/src/block.rs @@ -229,6 +229,40 @@ impl DataBlock { Ok(()) } + /// Byte offset of the restart point that may contain `start_key`. + /// An empty `start_key` selects the first restart. + pub(crate) fn restart_offset_for(&self, start_key: &[u8]) -> Result { + if start_key.is_empty() || self.restarts.is_empty() { + return Ok(self.restarts.first().copied().unwrap_or(0) as usize); + } + let mut left = 0usize; + let mut right = self.restarts.len(); + while left + 1 < right { + let mid = left + (right - left) / 2; + let restart_key = self.decode_restart_key(self.restarts[mid] as usize)?; + if restart_key.as_slice() <= start_key { + left = mid; + } else { + right = mid; + } + } + Ok(self.restarts[left] as usize) + } + + /// Decodes the entry at `offset`, advancing `offset` and `key`. + /// Returns `None` at the end of the block. + pub(crate) fn read_entry( + &self, + offset: &mut usize, + key: &mut Vec, + ) -> Result> { + if *offset >= self.entries_end { + return Ok(None); + } + let value = self.decode_next_entry(offset, key)?; + Ok(Some(value)) + } + /// Decodes every entry in the block (used by compaction iterators). pub fn iter_all(&self) -> Result, BlockEntryValue)>> { let mut offset = 0usize; diff --git a/crates/metadb-storage/src/compaction.rs b/crates/metadb-storage/src/compaction.rs index 670b50f..795b3ef 100644 --- a/crates/metadb-storage/src/compaction.rs +++ b/crates/metadb-storage/src/compaction.rs @@ -54,6 +54,25 @@ impl LocalityCompactor { self.density_threshold } + /// Replaces in-memory bucket counters. Used when opening an LSM so compaction + /// decisions follow the tombstones and live keys already on disk. + pub fn restore_counts(&self, counts: BTreeMap) { + let mut guard = self.inner.write().unwrap(); + guard.buckets.clear(); + for (prefix, (tombstones, live_writes)) in counts { + if tombstones == 0 && live_writes == 0 { + continue; + } + guard.buckets.insert( + prefix, + BucketDensity { + tombstones, + live_writes, + }, + ); + } + } + /// Extracts the 9-byte `[table_prefix][group_key: 8B]` bucket prefix from a raw key. pub fn bucket_for_key(key: &[u8]) -> Option { if key.len() >= 9 { diff --git a/crates/metadb-storage/src/engine.rs b/crates/metadb-storage/src/engine.rs index 464d104..3d64a1b 100644 --- a/crates/metadb-storage/src/engine.rs +++ b/crates/metadb-storage/src/engine.rs @@ -1,16 +1,40 @@ +use std::collections::BTreeMap; use std::path::Path; -use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::{Duration, Instant}; -use metadb_core::{HlcTimestamp, Result, ShardId}; +use metadb_core::{HlcTimestamp, MetaDbError, Result, ShardId}; pub use crate::backend::PersistentEntry; use crate::backend::{InMemoryBackend, LsmStorageBackend, StorageBackend}; +use crate::block::BlockEntryValue; use crate::bloom::DEFAULT_BLOOM_BITS_PER_KEY; use crate::cache::ShardedBlockCache; -use crate::compaction::{CompactionStats, LocalityCompactor}; +use crate::compaction::{CompactionStats, LocalityCompactor, RangeBucketPrefix}; use crate::manifest::DurableShardCheckpoint; -use crate::mvcc::{MemMvccStore, MvccValue, PreFlushHook, SnapshotGuard}; +use crate::mvcc::{ + MemMvccStore, MvccValue, PinnedSnapshotData, PreFlushHook, ReadCut, SnapshotGuard, +}; use crate::raft_log::{LogSequenceNumber, RaftLog, RaftLogPayload}; +use crate::version_set::Version; + +/// Caller of [`LifecycleStorageEngine::scan_live_until`] stops or keeps scanning. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ScanControl { + Continue, + Stop, +} + +/// Raft identity stamped onto a size-triggered flush checkpoint. +#[derive(Debug, Clone)] +pub struct FlushIdentity { + pub term: u64, + pub leader: u64, + pub membership: Vec, + pub descriptor: Vec, + pub aux_index: u64, +} /// Operational metrics demonstrating the benefits of In-Memory MVCC and Locality-Aware Compaction. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -20,6 +44,8 @@ pub struct StorageMetrics { pub persistent_kv_writes: u64, pub active_physical_tombstones: usize, pub tombstones_traversed_on_scans: u64, + /// Keys the limited scanner actually stepped to, including physical tombstones. + pub entries_examined_on_scans: u64, pub compaction_stats: CompactionStats, } @@ -31,6 +57,10 @@ pub struct LsmEngineConfig { pub tombstone_min_ratio_permille: usize, pub bloom_bits_per_key: usize, pub auto_locality_compaction: bool, + /// Seal and flush the active memtable once its accounted bytes reach this size. + pub memtable_size_bytes: usize, + /// Immutable memtables queued above this cap block writers until one flushes. + pub max_imm_memtables: usize, } impl Default for LsmEngineConfig { @@ -41,6 +71,8 @@ impl Default for LsmEngineConfig { tombstone_min_ratio_permille: 500, bloom_bits_per_key: DEFAULT_BLOOM_BITS_PER_KEY, auto_locality_compaction: true, + memtable_size_bytes: 32 * 1024 * 1024, + max_imm_memtables: 4, } } } @@ -56,6 +88,13 @@ pub struct LifecycleStorageEngine { compactor: Arc, backend: Arc, auto_locality_compaction: bool, + memtable_size_bytes: usize, + max_imm_memtables: usize, + budget_mu: Mutex<()>, + flush_hook: RwLock>>, + flush_identity: RwLock>, + entries_examined: AtomicU64, + last_aux_frontier: AtomicU64, } impl LifecycleStorageEngine { @@ -73,6 +112,13 @@ impl LifecycleStorageEngine { compactor: Arc::new(LocalityCompactor::new(tombstone_density_threshold)), backend: Arc::new(InMemoryBackend::new()), auto_locality_compaction, + memtable_size_bytes: 32 * 1024 * 1024, + max_imm_memtables: 4, + budget_mu: Mutex::new(()), + flush_hook: RwLock::new(None), + flush_identity: RwLock::new(None), + entries_examined: AtomicU64::new(0), + last_aux_frontier: AtomicU64::new(0), } } @@ -85,7 +131,7 @@ impl LifecycleStorageEngine { ) -> Result { let lsm = LsmStorageBackend::open(shard_id, lsm_dir, block_cache, config.bloom_bits_per_key)?; - Ok(Self::with_lsm_backend(shard_id, lsm, config)) + Self::with_lsm_backend(shard_id, lsm, config) } /// Opens a complete existing checkpoint without creating or repairing its manifest. @@ -101,14 +147,14 @@ impl LifecycleStorageEngine { block_cache, config.bloom_bits_per_key, )?; - Ok(Self::with_lsm_backend(shard_id, lsm, config)) + Self::with_lsm_backend(shard_id, lsm, config) } fn with_lsm_backend( shard_id: ShardId, lsm: LsmStorageBackend, config: LsmEngineConfig, - ) -> Self { + ) -> Result { let lsm = Arc::new(lsm); let cp = lsm.current_version().checkpoint.clone(); let mvcc = Arc::new(MemMvccStore::with_recovered_watermarks( @@ -120,14 +166,42 @@ impl LifecycleStorageEngine { config.tombstone_density_threshold, config.tombstone_min_ratio_permille, )); - Self { + rebuild_density(&compactor, &lsm.current_version())?; + Ok(Self { shard_id, raft_log: Arc::new(RaftLog::new(shard_id)), mvcc, compactor, backend: lsm, auto_locality_compaction: config.auto_locality_compaction, - } + memtable_size_bytes: config.memtable_size_bytes.max(1), + max_imm_memtables: config.max_imm_memtables.max(1), + budget_mu: Mutex::new(()), + flush_hook: RwLock::new(None), + flush_identity: RwLock::new(None), + entries_examined: AtomicU64::new(0), + last_aux_frontier: AtomicU64::new(0), + }) + } + + /// Installs the Raft pre-flush hook used by size-triggered seals. + pub fn set_pre_flush_hook(&self, hook: Arc) { + *self.flush_hook.write().unwrap() = Some(hook); + } + + /// Records the Raft log identity to stamp on the next size-triggered checkpoint. + pub fn set_flush_identity(&self, identity: FlushIdentity) { + *self.flush_identity.write().unwrap() = Some(identity); + } + + pub fn last_aux_frontier(&self) -> u64 { + self.last_aux_frontier.load(Ordering::Acquire) + } + + /// Rewrites only the durable checkpoint. SSTables are left in place. + pub fn persist_checkpoint(&self, checkpoint: DurableShardCheckpoint) -> Result<()> { + self.backend.persist_checkpoint_only(checkpoint)?; + Ok(()) } pub fn shard_id(&self) -> ShardId { @@ -163,6 +237,7 @@ impl LifecycleStorageEngine { raft_index: LogSequenceNumber, ) { self.mvcc.put(key, value, hlc, raft_index); + self.maintain_memtables().expect("memtable size flush"); } /// Applies an externally Raft-logged `Delete` at `(raft_index, hlc)` directly into `MemMvccStore`. @@ -173,6 +248,7 @@ impl LifecycleStorageEngine { raft_index: LogSequenceNumber, ) { self.mvcc.delete(key, hlc, raft_index); + self.maintain_memtables().expect("memtable size flush"); } /// Appends a `Put` to the internal `RaftLog` and applies it to `MemMvccStore`. @@ -185,6 +261,7 @@ impl LifecycleStorageEngine { }, ); self.mvcc.put(key, value, hlc, lsn); + self.maintain_memtables().expect("memtable size flush"); lsn } @@ -194,6 +271,7 @@ impl LifecycleStorageEngine { .raft_log .append(hlc, RaftLogPayload::Delete { key: key.clone() }); self.mvcc.delete(key, hlc, lsn); + self.maintain_memtables().expect("memtable size flush"); lsn } @@ -322,6 +400,99 @@ impl LifecycleStorageEngine { Ok(out) } + /// Visits live keys in `[start_key, end_key)` in order and stops when `visit` returns + /// [`ScanControl::Stop`]. Physical tombstones before that point are counted; keys after + /// the stop are not read. + pub fn scan_live_until( + &self, + start_key: &[u8], + end_key: &[u8], + snap: &SnapshotGuard, + mut visit: impl FnMut(&[u8], &[u8]) -> Result, + ) -> Result<()> { + if start_key >= end_key { + return Ok(()); + } + let mut disk_from = start_key.to_vec(); + let mut mvcc_from = start_key.to_vec(); + let cut = snap.cut(); + loop { + let pulled = snap.with_step_lease(|pinned| { + self.pull_next(pinned, cut, &mut disk_from, &mut mvcc_from, end_key) + })?; + match pulled { + Pulled::Done => break, + Pulled::MvccTombstone => { + self.entries_examined.fetch_add(1, Ordering::Relaxed); + } + Pulled::PhysicalTombstone => { + self.entries_examined.fetch_add(1, Ordering::Relaxed); + self.backend.record_scanned_tombstones(1); + } + Pulled::Live(key, value) => { + self.entries_examined.fetch_add(1, Ordering::Relaxed); + if visit(&key, &value)? == ScanControl::Stop { + break; + } + } + } + } + Ok(()) + } + + fn pull_next( + &self, + pinned: &PinnedSnapshotData, + cut: ReadCut, + disk_from: &mut Vec, + mvcc_from: &mut Vec, + end_key: &[u8], + ) -> Result { + if mvcc_from.as_slice() >= end_key && disk_from.as_slice() >= end_key { + return Ok(Pulled::Done); + } + let mvcc_item = if mvcc_from.as_slice() >= end_key { + None + } else { + self.mvcc + .range_page_with_pinned(mvcc_from, end_key, cut, pinned, 1) + .into_iter() + .next() + }; + let disk_item = if disk_from.as_slice() >= end_key { + None + } else { + let (page, _) = + self.backend + .scan_in_version_page(&pinned.version, disk_from, end_key, 1)?; + page.into_iter().next() + }; + match (mvcc_item, disk_item) { + (None, None) => Ok(Pulled::Done), + (Some((key, value)), None) => { + *mvcc_from = key_successor(&key); + Ok(pulled_from_mvcc(key, value)) + } + (None, Some((key, value))) => { + *disk_from = key_successor(&key); + Ok(pulled_from_disk(key, value)) + } + (Some((mvcc_key, mvcc_value)), Some((disk_key, disk_value))) => { + if mvcc_key < disk_key { + *mvcc_from = key_successor(&mvcc_key); + Ok(pulled_from_mvcc(mvcc_key, mvcc_value)) + } else if disk_key < mvcc_key { + *disk_from = key_successor(&disk_key); + Ok(pulled_from_disk(disk_key, disk_value)) + } else { + *mvcc_from = key_successor(&mvcc_key); + *disk_from = key_successor(&disk_key); + Ok(pulled_from_mvcc(mvcc_key, mvcc_value)) + } + } + } + } + /// Point lookup at `read_hlc` consulting `MemMvccStore` first and falling back to persistent storage. pub fn get(&self, key: &[u8], read_hlc: HlcTimestamp) -> Result>> { if let Some(mvcc_val) = self.mvcc.get_at(key, read_hlc)? { @@ -380,9 +551,8 @@ impl LifecycleStorageEngine { checkpoint: DurableShardCheckpoint, hook: Option<&dyn PreFlushHook>, ) -> Result<(LogSequenceNumber, LogSequenceNumber)> { - self.mvcc - .seal_active_memtable(self.backend.as_ref(), checkpoint, hook)?; - self.flush_queued_imms_with_hook(hook) + let _budget = self.budget_mu.lock().unwrap(); + self.seal_and_flush_locked(checkpoint, hook) } /// Flushes already-sealed `ImmMemTable`s that pass the `ActiveSnapshotRegistry` Safe-Flush Gate. @@ -471,7 +641,127 @@ impl LifecycleStorageEngine { persistent_kv_writes: kv_writes, active_physical_tombstones: active_tombs, tombstones_traversed_on_scans: tombs_scanned, + entries_examined_on_scans: self.entries_examined.load(Ordering::Relaxed), compaction_stats: self.compactor.stats(), } } + + fn maintain_memtables(&self) -> Result<()> { + if self.mvcc.active_memtable_bytes() < self.memtable_size_bytes { + return Ok(()); + } + let _budget = self.budget_mu.lock().unwrap(); + if self.mvcc.active_memtable_bytes() < self.memtable_size_bytes { + return Ok(()); + } + let (lsn, hlc, _, _) = self.mvcc.latest_watermarks(); + let checkpoint = self.checkpoint_for_flush(lsn, hlc); + let hook = self.flush_hook.read().unwrap().clone(); + let hook_ref = hook.as_ref().map(|installed| installed.as_ref()); + self.seal_and_flush_locked(checkpoint, hook_ref)?; + Ok(()) + } + + fn seal_and_flush_locked( + &self, + checkpoint: DurableShardCheckpoint, + hook: Option<&dyn PreFlushHook>, + ) -> Result<(LogSequenceNumber, LogSequenceNumber)> { + self.ensure_imm_capacity(hook)?; + self.mvcc + .seal_active_memtable(self.backend.as_ref(), checkpoint, hook)?; + let flushed = self.flush_queued_imms_with_hook(hook)?; + self.last_aux_frontier.store(flushed.1, Ordering::Release); + Ok(flushed) + } + + fn ensure_imm_capacity(&self, hook: Option<&dyn PreFlushHook>) -> Result<()> { + let started = Instant::now(); + let limit = Duration::from_millis(self.mvcc.retention_ms().saturating_add(1_000)); + while self.mvcc.imm_queue_len() >= self.max_imm_memtables { + self.flush_queued_imms_with_hook(hook)?; + if self.mvcc.imm_queue_len() < self.max_imm_memtables { + return Ok(()); + } + if started.elapsed() > limit { + self.mvcc.snapshot_registry().expire_stale_and_min_bounds(); + self.flush_queued_imms_with_hook(hook)?; + if self.mvcc.imm_queue_len() >= self.max_imm_memtables { + return Err(MetaDbError::StorageIoError(format!( + "immutable memtable cap {} is pinned past the MVCC retention window", + self.max_imm_memtables + ))); + } + return Ok(()); + } + self.mvcc + .snapshot_registry() + .wait_for_flush_progress(Duration::from_millis(20)); + } + Ok(()) + } + + fn checkpoint_for_flush(&self, lsn: u64, hlc: HlcTimestamp) -> DurableShardCheckpoint { + let current = self.durable_checkpoint(); + match self.flush_identity.read().unwrap().clone() { + Some(identity) => DurableShardCheckpoint { + flushed_raft_term: identity.term, + flushed_raft_leader: identity.leader, + flushed_raft_index: lsn, + flushed_max_hlc: hlc, + contiguous_aux_acked_index: identity.aux_index, + membership_bytes: identity.membership, + descriptor_bytes: identity.descriptor, + }, + None => DurableShardCheckpoint { + flushed_raft_index: lsn, + flushed_max_hlc: hlc, + ..current + }, + } + } +} + +enum Pulled { + Done, + MvccTombstone, + PhysicalTombstone, + Live(Vec, Vec), +} + +fn pulled_from_mvcc(key: Vec, value: MvccValue) -> Pulled { + match value { + MvccValue::Value(bytes) => Pulled::Live(key, bytes), + MvccValue::Tombstone => Pulled::MvccTombstone, + } +} + +fn pulled_from_disk(key: Vec, value: PersistentEntry) -> Pulled { + match value { + PersistentEntry::Value(bytes) => Pulled::Live(key, bytes), + PersistentEntry::PhysicalTombstone => Pulled::PhysicalTombstone, + } +} + +fn key_successor(key: &[u8]) -> Vec { + let mut next = key.to_vec(); + next.push(0x00); + next +} + +fn rebuild_density(compactor: &LocalityCompactor, version: &Version) -> Result<()> { + let mut cursor = version.merged_cursor(&[], None)?; + let mut counts: BTreeMap = BTreeMap::new(); + while let Some((key, value)) = cursor.next_owned()? { + let Some(bucket) = LocalityCompactor::bucket_for_key(&key) else { + continue; + }; + let slot = counts.entry(bucket).or_insert((0, 0)); + match value { + BlockEntryValue::PhysicalTombstone => slot.0 += 1, + BlockEntryValue::Value(_) => slot.1 += 1, + } + } + compactor.restore_counts(counts); + Ok(()) } diff --git a/crates/metadb-storage/src/lib.rs b/crates/metadb-storage/src/lib.rs index 968c8f7..9c72b3f 100644 --- a/crates/metadb-storage/src/lib.rs +++ b/crates/metadb-storage/src/lib.rs @@ -26,7 +26,9 @@ pub mod version_set; pub use backend::{InMemoryBackend, LsmStorageBackend, PersistentEntry, StorageBackend}; pub use cache::ShardedBlockCache; pub use compaction::{CompactionStats, LocalityCompactor}; -pub use engine::{LifecycleStorageEngine, LsmEngineConfig, StorageMetrics}; +pub use engine::{ + FlushIdentity, LifecycleStorageEngine, LsmEngineConfig, ScanControl, StorageMetrics, +}; pub use manifest::{DurableShardCheckpoint, ManifestLog, VersionEdit}; pub use mvcc::{ ActiveMemTable, ActiveSnapshotRegistry, ImmMemTable, MemMvccStore, MvccValue, MvccVersion, diff --git a/crates/metadb-storage/src/mvcc.rs b/crates/metadb-storage/src/mvcc.rs index e8c068f..41a87f9 100644 --- a/crates/metadb-storage/src/mvcc.rs +++ b/crates/metadb-storage/src/mvcc.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, VecDeque}; use std::ops::Bound; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Condvar, Mutex, RwLock}; use std::time::{Duration, Instant}; use metadb_core::{HlcTimestamp, MetaDbError, Result}; @@ -168,6 +168,8 @@ pub struct ActiveSnapshotRegistry { next_id: AtomicU64, retention: Duration, slots: Mutex>>, + flush_mu: Mutex<()>, + flush_cv: Condvar, } impl ActiveSnapshotRegistry { @@ -176,6 +178,8 @@ impl ActiveSnapshotRegistry { next_id: AtomicU64::new(1), retention: Duration::from_millis(retention_ms.max(1)), slots: Mutex::new(BTreeMap::new()), + flush_mu: Mutex::new(()), + flush_cv: Condvar::new(), } } @@ -198,10 +202,23 @@ impl ActiveSnapshotRegistry { slot } + /// Wakes writers blocked on the immutable-memtable cap. + pub fn notify_flush_waiters(&self) { + let _guard = self.flush_mu.lock().unwrap(); + self.flush_cv.notify_all(); + } + + /// Parks until a snapshot is dropped or `timeout` elapses. + pub fn wait_for_flush_progress(&self, timeout: Duration) { + let guard = self.flush_mu.lock().unwrap(); + let _ = self.flush_cv.wait_timeout(guard, timeout); + } + pub fn unregister(&self, id: u64) { let removed = self.slots.lock().unwrap().remove(&id); if let Some(slot) = removed { *slot.state.write().unwrap() = None; + self.notify_flush_waiters(); } } @@ -231,9 +248,14 @@ impl ActiveSnapshotRegistry { }); } } + let expired_any = !expired_ids.is_empty(); for id in expired_ids { guard.remove(&id); } + drop(guard); + if expired_any { + self.notify_flush_waiters(); + } min_hlc.zip(min_flushed) } @@ -316,6 +338,7 @@ struct MemMvccInner { last_flushed_lsn: LogSequenceNumber, total_mvcc_writes: u64, total_pruned_versions: u64, + active_bytes: usize, } /// In-Memory MVCC store coordinating active version chains, `ImmMemTable` queue, @@ -351,6 +374,7 @@ impl MemMvccStore { last_flushed_lsn: flushed_lsn, total_mvcc_writes: 0, total_pruned_versions: 0, + active_bytes: 0, }), flush_mutex: std::sync::Mutex::new(()), } @@ -361,10 +385,12 @@ impl MemMvccStore { } pub fn put(&self, key: Vec, value: Vec, hlc: HlcTimestamp, lsn: LogSequenceNumber) { + let accounted = key.len().saturating_add(value.len()).max(1); let mut guard = self.inner.write().unwrap(); guard.latest_hlc = guard.latest_hlc.max(hlc); guard.latest_lsn = guard.latest_lsn.max(lsn); guard.total_mvcc_writes += 1; + guard.active_bytes = guard.active_bytes.saturating_add(accounted); guard.hlc_to_lsn.push_back((hlc, lsn)); guard .active_versions @@ -380,10 +406,12 @@ impl MemMvccStore { } pub fn delete(&self, key: Vec, hlc: HlcTimestamp, lsn: LogSequenceNumber) { + let accounted = key.len().max(1); let mut guard = self.inner.write().unwrap(); guard.latest_hlc = guard.latest_hlc.max(hlc); guard.latest_lsn = guard.latest_lsn.max(lsn); guard.total_mvcc_writes += 1; + guard.active_bytes = guard.active_bytes.saturating_add(accounted); guard.hlc_to_lsn.push_back((hlc, lsn)); guard .active_versions @@ -398,6 +426,14 @@ impl MemMvccStore { }); } + pub fn active_memtable_bytes(&self) -> usize { + self.inner.read().unwrap().active_bytes + } + + pub fn retention_ms(&self) -> u64 { + self.retention_ms + } + pub fn acquire_linearizable_snapshot( &self, min_read_index: LogSequenceNumber, @@ -701,10 +737,11 @@ impl MemMvccStore { } } - if guard.active_versions.read().unwrap().is_empty() - && sealed_index == guard.last_flushed_lsn - { - return Ok(None); + if guard.active_versions.read().unwrap().is_empty() { + guard.active_bytes = 0; + if sealed_index == guard.last_flushed_lsn { + return Ok(None); + } } let seal_id = guard.next_seal_id; @@ -713,6 +750,7 @@ impl MemMvccStore { &mut guard.active_versions, Arc::new(RwLock::new(BTreeMap::new())), ); + guard.active_bytes = 0; let entries: BTreeMap, Vec> = old_active .read() .unwrap() @@ -801,6 +839,7 @@ impl MemMvccStore { .unwrap_or(false) { guard.imm_queue.pop_front(); + self.registry.notify_flush_waiters(); } guard.disk_flushed_max_hlc = guard.disk_flushed_max_hlc.max(candidate.max_hlc); guard.last_flushed_lsn = guard.last_flushed_lsn.max(candidate.max_raft_index); @@ -858,6 +897,7 @@ impl MemMvccStore { g.active_versions = Arc::new(RwLock::new(BTreeMap::new())); g.hlc_to_lsn.clear(); g.imm_queue.clear(); + g.active_bytes = 0; g.latest_lsn = snapshot_lsn; g.latest_hlc = snapshot_hlc; g.last_flushed_lsn = snapshot_lsn; diff --git a/crates/metadb-storage/src/sstable.rs b/crates/metadb-storage/src/sstable.rs index 1679d38..4b96c8d 100644 --- a/crates/metadb-storage/src/sstable.rs +++ b/crates/metadb-storage/src/sstable.rs @@ -1,5 +1,6 @@ //! Immutable SSTable builder and reader (`.sst`) with 64-byte magic/CRC32C footer. +use std::cell::Cell; use std::fs::{self, File, OpenOptions}; use std::io::Write; use std::os::unix::fs::FileExt; @@ -7,6 +8,35 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +thread_local! { + static FULL_RANGE_MATERIALIZATIONS: Cell = const { Cell::new(0) }; + static DATA_BLOCKS_TOUCHED: Cell = const { Cell::new(0) }; +} + +/// Test probe: how many times `read_all_entries` ran on this thread. +pub fn full_range_materializations() -> u64 { + FULL_RANGE_MATERIALIZATIONS.with(|counter| counter.get()) +} + +/// Test probe: how many data blocks this thread has loaded or cache-hit. +pub fn data_blocks_touched() -> u64 { + DATA_BLOCKS_TOUCHED.with(|counter| counter.get()) +} + +/// Clears the thread-local compaction and scan probes. +pub fn reset_scan_probes() { + FULL_RANGE_MATERIALIZATIONS.with(|counter| counter.set(0)); + DATA_BLOCKS_TOUCHED.with(|counter| counter.set(0)); +} + +fn note_full_range_materialization() { + FULL_RANGE_MATERIALIZATIONS.with(|counter| counter.set(counter.get().saturating_add(1))); +} + +fn note_data_block_touch() { + DATA_BLOCKS_TOUCHED.with(|counter| counter.set(counter.get().saturating_add(1))); +} + use serde::{Deserialize, Serialize}; use metadb_core::{ @@ -475,8 +505,11 @@ impl SstableReader { Ok(out) } - /// Reads every entry across all data blocks in this SSTable (used during compaction). + /// Reads every entry across all data blocks in this SSTable. + /// + /// Compaction must not call this: it materializes the whole file before merge. pub fn read_all_entries(&self) -> Result, BlockEntryValue)>> { + note_full_range_materialization(); let mut out = Vec::new(); for ie in &self.index { let block = self.load_data_block(ie)?; @@ -531,6 +564,7 @@ impl SstableReader { } fn load_data_block(&self, ie: &IndexBlockEntry) -> Result> { + note_data_block_touch(); let cache_key = BlockCacheKey { shard_id: self.shard_id, sst_file_id: self.meta.file_id, @@ -666,3 +700,108 @@ impl SstableReader { }) } } + +/// Forward cursor over one SSTable. Decodes a single entry at a time and loads +/// the next data block only when the current block is exhausted. +pub(crate) struct SstCursor { + reader: Arc, + start_key: Vec, + end_key: Option>, + block_idx: usize, + block: Option>, + offset: usize, + key: Vec, + pending: Option<(Vec, BlockEntryValue)>, + needs_fill: bool, + exhausted: bool, +} + +impl SstCursor { + pub(crate) fn open( + reader: Arc, + start_key: &[u8], + end_key: Option<&[u8]>, + ) -> Result { + let block_idx = reader + .index + .partition_point(|entry| entry.last_key.as_slice() < start_key); + Ok(Self { + reader, + start_key: start_key.to_vec(), + end_key: end_key.map(|key| key.to_vec()), + block_idx, + block: None, + offset: 0, + key: Vec::new(), + pending: None, + needs_fill: true, + exhausted: false, + }) + } + + pub(crate) fn peek(&mut self) -> Result, BlockEntryValue)>> { + if self.needs_fill { + self.fill_one()?; + self.needs_fill = false; + } + Ok(self.pending.clone()) + } + + /// Drops the current entry without decoding the successor. + pub(crate) fn advance(&mut self) { + self.pending = None; + self.needs_fill = !self.exhausted; + } + + fn fill_one(&mut self) -> Result<()> { + if self.exhausted { + self.pending = None; + return Ok(()); + } + loop { + if self.block.is_none() { + if self.block_idx >= self.reader.index.len() { + self.exhausted = true; + self.pending = None; + return Ok(()); + } + let index_entry = &self.reader.index[self.block_idx]; + if index_entry.last_key.as_slice() < self.start_key.as_slice() { + self.block_idx += 1; + continue; + } + let block = self.reader.load_data_block(index_entry)?; + self.offset = block.restart_offset_for(&self.start_key)?; + self.key.clear(); + self.block = Some(block); + } + let block = self.block.as_ref().unwrap().clone(); + match block.read_entry(&mut self.offset, &mut self.key)? { + None => { + self.block = None; + self.block_idx += 1; + self.key.clear(); + self.offset = 0; + continue; + } + Some(value) => { + if self.key.as_slice() < self.start_key.as_slice() { + continue; + } + if self + .end_key + .as_ref() + .is_some_and(|end| self.key.as_slice() >= end.as_slice()) + { + self.exhausted = true; + self.pending = None; + self.block = None; + return Ok(()); + } + self.pending = Some((self.key.clone(), value)); + return Ok(()); + } + } + } + } +} diff --git a/crates/metadb-storage/src/version_set.rs b/crates/metadb-storage/src/version_set.rs index d7aabc7..3e10b99 100644 --- a/crates/metadb-storage/src/version_set.rs +++ b/crates/metadb-storage/src/version_set.rs @@ -1,13 +1,14 @@ //! Immutable `Version` views and deterministic `MergingIterator` across `L0..L3` SSTables. -use std::collections::BTreeMap; +use std::cmp::Ordering; +use std::collections::{BTreeMap, BinaryHeap}; use std::sync::Arc; use metadb_core::Result; use crate::block::BlockEntryValue; use crate::manifest::DurableShardCheckpoint; -use crate::sstable::{SstFileMeta, SstableReader}; +use crate::sstable::{SstCursor, SstFileMeta, SstableReader}; pub const NUM_LSM_LEVELS: usize = 4; @@ -199,4 +200,212 @@ impl Version { } samples } + + /// Streaming merge of the newest value of each key in `[start, end)`. + /// `end == None` reads through the last key of every selected SSTable. + pub(crate) fn merged_cursor(&self, start: &[u8], end: Option<&[u8]>) -> Result { + let mut readers = Vec::new(); + for sst in &self.levels[0] { + readers.push(sst.clone()); + } + for level in &self.levels[1..] { + let first = if start.is_empty() { + 0 + } else { + level.partition_point(|sst| sst.meta().max_key.as_slice() < start) + }; + for sst in &level[first..] { + if end.is_some_and(|bound| sst.meta().min_key.as_slice() >= bound) { + break; + } + readers.push(sst.clone()); + } + } + MergedCursor::open(readers, start, end) + } +} + +/// K-way merge. Earlier readers win on equal keys (`L0` newest-first, then `L1..L3`). +pub(crate) struct MergedCursor { + sources: Vec, + heap: BinaryHeap, + staged: Option, + exhausted: bool, +} + +struct MergeSource { + cursor: SstCursor, + precedence: u32, + generation: u64, + needs_reheap: bool, +} + +struct StagedEntry { + key: Vec, + value: BlockEntryValue, + source_ids: Vec, +} + +#[derive(Clone, Eq, PartialEq)] +struct HeapNode { + key: Vec, + precedence: u32, + cursor_id: u32, + generation: u64, +} + +impl PartialOrd for HeapNode { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for HeapNode { + fn cmp(&self, other: &Self) -> Ordering { + // `BinaryHeap` pops the greatest node. Invert key order so the smallest key wins. + other + .key + .cmp(&self.key) + .then_with(|| other.precedence.cmp(&self.precedence)) + .then_with(|| self.cursor_id.cmp(&other.cursor_id)) + .then_with(|| self.generation.cmp(&other.generation)) + } +} + +impl MergedCursor { + pub(crate) fn open( + readers: Vec>, + start: &[u8], + end: Option<&[u8]>, + ) -> Result { + let mut sources = Vec::with_capacity(readers.len()); + let mut heap = BinaryHeap::new(); + for (index, reader) in readers.into_iter().enumerate() { + let mut cursor = SstCursor::open(reader, start, end)?; + let cursor_id = index as u32; + if let Some((key, _)) = cursor.peek()? { + heap.push(HeapNode { + key, + precedence: cursor_id, + cursor_id, + generation: 0, + }); + } + sources.push(MergeSource { + cursor, + precedence: cursor_id, + generation: 0, + needs_reheap: false, + }); + } + Ok(Self { + sources, + heap, + staged: None, + exhausted: false, + }) + } + + pub(crate) fn peek(&mut self) -> Result, BlockEntryValue)>> { + self.ensure_staged()?; + Ok(self + .staged + .as_ref() + .map(|entry| (entry.key.clone(), entry.value.clone()))) + } + + /// Drops the current winner. The successor is not decoded until the next peek. + pub(crate) fn advance(&mut self) -> Result<()> { + self.ensure_staged()?; + let Some(staged) = self.staged.take() else { + return Ok(()); + }; + for source_id in staged.source_ids { + let source = &mut self.sources[source_id as usize]; + source.cursor.advance(); + source.generation = source.generation.wrapping_add(1); + source.needs_reheap = true; + } + Ok(()) + } + + pub(crate) fn next_owned(&mut self) -> Result, BlockEntryValue)>> { + let Some((key, value)) = self.peek()? else { + return Ok(None); + }; + self.advance()?; + Ok(Some((key, value))) + } + + fn ensure_staged(&mut self) -> Result<()> { + if self.staged.is_some() || self.exhausted { + return Ok(()); + } + self.reheap_pending()?; + loop { + let Some(node) = self.heap.pop() else { + self.exhausted = true; + return Ok(()); + }; + let cursor_id = node.cursor_id as usize; + if self.sources[cursor_id].generation != node.generation { + continue; + } + let Some((key, value)) = self.sources[cursor_id].cursor.peek()? else { + continue; + }; + if key != node.key { + continue; + } + let mut best_precedence = node.precedence; + let mut best_value = value; + let mut source_ids = vec![node.cursor_id]; + while self.heap.peek().is_some_and(|next| next.key == node.key) { + let other = self.heap.pop().unwrap(); + let other_index = other.cursor_id as usize; + if self.sources[other_index].generation != other.generation { + continue; + } + let Some((other_key, other_value)) = self.sources[other_index].cursor.peek()? + else { + continue; + }; + if other_key != node.key { + continue; + } + source_ids.push(other.cursor_id); + if other.precedence < best_precedence { + best_precedence = other.precedence; + best_value = other_value; + } + } + self.staged = Some(StagedEntry { + key: node.key, + value: best_value, + source_ids, + }); + return Ok(()); + } + } + + fn reheap_pending(&mut self) -> Result<()> { + for cursor_id in 0..self.sources.len() { + if !self.sources[cursor_id].needs_reheap { + continue; + } + self.sources[cursor_id].needs_reheap = false; + let generation = self.sources[cursor_id].generation; + let precedence = self.sources[cursor_id].precedence; + let Some((key, _)) = self.sources[cursor_id].cursor.peek()? else { + continue; + }; + self.heap.push(HeapNode { + key, + precedence, + cursor_id: cursor_id as u32, + generation, + }); + } + Ok(()) + } } diff --git a/crates/metadb-storage/tests/first_ship_storage_regressions.rs b/crates/metadb-storage/tests/first_ship_storage_regressions.rs new file mode 100644 index 0000000..669bee7 --- /dev/null +++ b/crates/metadb-storage/tests/first_ship_storage_regressions.rs @@ -0,0 +1,274 @@ +//! Regressions for the first-ship storage and listdir failures on main `f325f6a`. + +use std::sync::mpsc; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +use metadb_core::{HlcTimestamp, MetaKey}; +use metadb_storage::sstable::{ + data_blocks_touched, full_range_materializations, reset_scan_probes, +}; +use metadb_storage::{LifecycleStorageEngine, LsmEngineConfig, ScanControl, ShardedBlockCache}; +use tempfile::tempdir; + +fn engine(dir: &std::path::Path, config: LsmEngineConfig) -> LifecycleStorageEngine { + let cache = Arc::new(ShardedBlockCache::new(8 * 1024 * 1024)); + LifecycleStorageEngine::open_lsm(1, dir, cache, config).unwrap() +} + +#[test] +fn listdir_limit_does_not_walk_the_remaining_tombstone_range() { + let dir = tempdir().unwrap(); + let lsm = dir.path().join("lsm"); + let engine = engine( + &lsm, + LsmEngineConfig { + auto_locality_compaction: false, + tombstone_density_threshold: 100_000, + memtable_size_bytes: 32 * 1024 * 1024, + ..LsmEngineConfig::default() + }, + ); + + let live = MetaKey::child(7, "a").encode(); + engine.apply_committed_put( + live.clone(), + b"live".to_vec(), + HlcTimestamp::new(1_000, 1), + 1, + ); + for index in 0..2048u64 { + let key = MetaKey::child(7, format!("t{index:04}")).encode(); + engine.apply_committed_delete(key, HlcTimestamp::new(1_000, 2), index + 2); + } + engine.flush_mvcc_to_persistent(); + + let version = engine.backend().current_version(); + let blocks: usize = version + .levels + .iter() + .flat_map(|level| level.iter()) + .map(|sst| sst.index_entries().len()) + .sum(); + assert!( + blocks > 4, + "the fixture must span multiple data blocks, got {blocks}" + ); + + reset_scan_probes(); + let before = engine.metrics(); + let snap = engine.acquire_linearizable_snapshot(0).unwrap(); + let mut rows = Vec::new(); + engine + .scan_live_until( + &MetaKey::encode_pid_prefix_start(7), + &MetaKey::encode_pid_prefix_end(7), + &snap, + |key, value| { + rows.push((key.to_vec(), value.to_vec())); + Ok(ScanControl::Stop) + }, + ) + .unwrap(); + drop(snap); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].0, live); + let after = engine.metrics(); + assert_eq!( + after.tombstones_traversed_on_scans - before.tombstones_traversed_on_scans, + 0, + "limit=1 must not count the 2048 tombstones that sort after the live row" + ); + assert_eq!( + after.entries_examined_on_scans - before.entries_examined_on_scans, + 1 + ); + let touched = data_blocks_touched(); + assert!( + touched < blocks as u64, + "limit=1 touched {touched} data blocks of {blocks}" + ); +} + +#[test] +fn locality_density_survives_reopen() { + let dir = tempdir().unwrap(); + let lsm = dir.path().join("lsm"); + let config = LsmEngineConfig { + auto_locality_compaction: true, + tombstone_density_threshold: 10_000, + ..LsmEngineConfig::default() + }; + let sample = MetaKey::child(7, "a").encode(); + { + let engine = engine(&lsm, config.clone()); + for index in 0..100u64 { + let key = MetaKey::child(7, format!("n{index:03}")).encode(); + engine.apply_committed_put( + key, + b"v".to_vec(), + HlcTimestamp::new(2_000, index as u16), + index + 1, + ); + } + engine.flush_mvcc_to_persistent(); + for index in 0..20u64 { + let key = MetaKey::child(7, format!("n{index:03}")).encode(); + engine.apply_committed_delete(key, HlcTimestamp::new(3_000, index as u16), 200 + index); + } + engine.flush_mvcc_to_persistent(); + assert_eq!(engine.compactor().snapshot_bucket_counts(&sample), (20, 80)); + } + + let cache = Arc::new(ShardedBlockCache::new(8 * 1024 * 1024)); + let reopened = LifecycleStorageEngine::open_lsm_existing(1, &lsm, cache, config).unwrap(); + assert_eq!( + reopened.compactor().snapshot_bucket_counts(&sample), + (20, 80), + "density must be rebuilt from SSTables on open" + ); +} + +#[test] +fn compact_range_does_not_materialize_whole_ssts() { + let dir = tempdir().unwrap(); + let lsm = dir.path().join("lsm"); + let engine = engine( + &lsm, + LsmEngineConfig { + auto_locality_compaction: false, + ..LsmEngineConfig::default() + }, + ); + let mut live = Vec::new(); + for index in 0..400u64 { + let key = MetaKey::child(9, format!("k{index:04}")).encode(); + engine.apply_committed_put( + key.clone(), + vec![index as u8], + HlcTimestamp::new(4_000, index as u16), + index + 1, + ); + if index % 2 == 0 { + live.push(key); + } + } + engine.flush_mvcc_to_persistent(); + for index in 0..400u64 { + if index % 2 == 1 { + let key = MetaKey::child(9, format!("k{index:04}")).encode(); + engine.apply_committed_delete( + key, + HlcTimestamp::new(5_000, index as u16), + 1_000 + index, + ); + } + } + engine.flush_mvcc_to_persistent(); + + reset_scan_probes(); + let purged = engine.compact_range( + &MetaKey::encode_pid_prefix_start(9), + &MetaKey::encode_pid_prefix_end(9), + ); + assert!(purged >= 200, "purged {purged}"); + assert_eq!( + full_range_materializations(), + 0, + "compact_range must not call read_all_entries" + ); + + let snap = engine.acquire_linearizable_snapshot(0).unwrap(); + for key in &live { + assert!(engine.get_with_snapshot(key, &snap).unwrap().is_some()); + } + let odd = MetaKey::child(9, "k0001").encode(); + assert!(engine.get_with_snapshot(&odd, &snap).unwrap().is_none()); +} + +#[test] +fn tiny_memtable_flushes_and_immutable_tables_respect_the_cap() { + let dir = tempdir().unwrap(); + let lsm = dir.path().join("lsm"); + let engine = Arc::new(engine( + &lsm, + LsmEngineConfig { + auto_locality_compaction: false, + memtable_size_bytes: 1, + max_imm_memtables: 4, + ..LsmEngineConfig::default() + }, + )); + + engine.apply_committed_put( + b"first".to_vec(), + b"v".to_vec(), + HlcTimestamp::new(6_000, 1), + 1, + ); + assert!( + engine.durable_checkpoint().flushed_raft_index >= 1, + "a 1-byte memtable threshold must flush" + ); + assert_eq!(engine.mvcc().imm_queue_len(), 0); + + let snap = engine.acquire_linearizable_snapshot(1).unwrap(); + for index in 0..4u64 { + engine.apply_committed_put( + format!("held-{index}").into_bytes(), + b"x".to_vec(), + HlcTimestamp::new(7_000, index as u16), + 10 + index, + ); + assert!(engine.mvcc().imm_queue_len() <= 4); + } + assert_eq!(engine.mvcc().imm_queue_len(), 4); + + let blocked = Arc::clone(&engine); + let (started_tx, started_rx) = mpsc::channel(); + let handle = thread::spawn(move || { + started_tx.send(()).unwrap(); + blocked.apply_committed_put( + b"fifth".to_vec(), + b"y".to_vec(), + HlcTimestamp::new(8_000, 1), + 20, + ); + }); + started_rx.recv().unwrap(); + thread::sleep(Duration::from_millis(150)); + assert!( + !handle.is_finished(), + "the fifth immutable memtable must wait for backpressure" + ); + assert_eq!(engine.mvcc().imm_queue_len(), 4); + drop(snap); + handle.join().unwrap(); + assert!(engine.mvcc().imm_queue_len() <= 4); + + drop(engine); + let cache = Arc::new(ShardedBlockCache::new(1024 * 1024)); + let reopened = LifecycleStorageEngine::open_lsm( + 1, + &lsm, + cache, + LsmEngineConfig { + memtable_size_bytes: 32 * 1024 * 1024, + max_imm_memtables: 4, + auto_locality_compaction: false, + ..LsmEngineConfig::default() + }, + ) + .unwrap(); + let snap = reopened.acquire_linearizable_snapshot(0).unwrap(); + assert_eq!( + reopened.get_with_snapshot(b"first", &snap).unwrap(), + Some(b"v".to_vec()) + ); + assert_eq!( + reopened.get_with_snapshot(b"fifth", &snap).unwrap(), + Some(b"y".to_vec()) + ); +} diff --git a/crates/metadb-tx/src/pushdown.rs b/crates/metadb-tx/src/pushdown.rs index c070025..915eea5 100644 --- a/crates/metadb-tx/src/pushdown.rs +++ b/crates/metadb-tx/src/pushdown.rs @@ -6,7 +6,7 @@ use metadb_core::{ HlcTimestamp, InodeAttr, InodeId, MetaDbError, MetaEntry, MetaKey, Result, UserId, ATTR_ENTRY_NAME, }; -use metadb_storage::LifecycleStorageEngine; +use metadb_storage::{LifecycleStorageEngine, ScanControl}; use crate::attr_aggregator::AsyncAttrAggregator; use crate::aux_sink::AsyncAuxiliarySink; @@ -382,19 +382,48 @@ impl PushdownExecutor { pid: InodeId, read_hlc: HlcTimestamp, ) -> Result> { - let start = MetaKey::encode_pid_prefix_start(pid); - let end = MetaKey::encode_pid_prefix_end(pid); - let rows = engine.scan_range(&start, &end, read_hlc)?; + self.listdir_limited(engine, pid, None, None, usize::MAX, read_hlc) + } - let mut entries = Vec::with_capacity(rows.len()); - for (k_bytes, v_bytes) in rows { - let key = MetaKey::decode(&k_bytes)?; + /// Pinned, tombstone-aware directory page. Stops once `limit` child entries are collected + /// instead of reading the rest of the parent range. + pub fn listdir_limited( + &self, + engine: &LifecycleStorageEngine, + pid: InodeId, + after_name: Option<&str>, + name_prefix: Option<&str>, + limit: usize, + read_hlc: HlcTimestamp, + ) -> Result> { + if limit == 0 { + return Ok(Vec::new()); + } + let start = crate::two_pc::listdir_scan_start(pid, after_name, name_prefix); + let end = MetaKey::encode_pid_prefix_end(pid); + if start >= end { + return Ok(Vec::new()); + } + let snap = engine.acquire_fixed_snapshot(read_hlc)?; + let mut entries = Vec::new(); + engine.scan_live_until(&start, &end, &snap, |k_bytes, v_bytes| { + let key = MetaKey::decode(k_bytes)?; if key.is_attr_key() { - continue; + return Ok(ScanControl::Continue); + } + if let Some(prefix) = name_prefix { + if !key.name.starts_with(prefix) { + return Ok(ScanControl::Stop); + } } - let attr = InodeAttr::decode(&v_bytes)?; + let attr = InodeAttr::decode(v_bytes)?; entries.push(MetaEntry { key, attr }); - } + if entries.len() >= limit { + Ok(ScanControl::Stop) + } else { + Ok(ScanControl::Continue) + } + })?; Ok(entries) } } @@ -524,6 +553,62 @@ mod tests { use super::*; + #[test] + fn listdir_limit_stops_before_later_tombstones() { + use metadb_storage::{LsmEngineConfig, ShardedBlockCache}; + use tempfile::tempdir; + + let dir = tempdir().unwrap(); + let cache = Arc::new(ShardedBlockCache::new(4 * 1024 * 1024)); + let engine = LifecycleStorageEngine::open_lsm( + 1, + dir.path(), + cache, + LsmEngineConfig { + auto_locality_compaction: false, + tombstone_density_threshold: 100_000, + memtable_size_bytes: 32 * 1024 * 1024, + ..LsmEngineConfig::default() + }, + ) + .unwrap(); + let hlc = HlcTimestamp::new(1_000, 1); + let attr = InodeAttr { + inode_id: 40, + parent_pid: 4, + entry_type: EntryType::File, + uid: 1, + mode: 0o644, + link_count: 1, + num_children: 0, + size_bytes: 4, + create_time: hlc, + modify_time: hlc, + fid: Some(9), + applied_lsn: 1, + }; + engine.apply_committed_put(MetaKey::child(4, "a").encode(), attr.encode(), hlc, 1); + for index in 0..2048u64 { + engine.apply_committed_delete( + MetaKey::child(4, format!("t{index:04}")).encode(), + HlcTimestamp::new(1_000, 2), + index + 2, + ); + } + engine.flush_mvcc_to_persistent(); + let before = engine.metrics().tombstones_traversed_on_scans; + let push = PushdownExecutor::new( + Arc::new(AsyncAttrAggregator::new()), + Arc::new(AsyncAuxiliarySink::new()), + ); + let page = push + .listdir_limited(&engine, 4, None, None, 1, HlcTimestamp::new(2_000, 0)) + .unwrap(); + assert_eq!(page.len(), 1); + assert_eq!(page[0].key.name, "a"); + assert_eq!(engine.metrics().tombstones_traversed_on_scans - before, 0); + } + #[test] fn rename_updates_directory_attr_on_its_own_shard() { let src = LifecycleStorageEngine::new(1, 5_000, 128, false); diff --git a/crates/metadb-tx/src/two_pc.rs b/crates/metadb-tx/src/two_pc.rs index ddc48f2..ba82f52 100644 --- a/crates/metadb-tx/src/two_pc.rs +++ b/crates/metadb-tx/src/two_pc.rs @@ -18,7 +18,7 @@ use metadb_core::{ TABLE_PREFIX_SECONDARY_INDEX, }; use metadb_raft::ShardRaftGroup; -use metadb_storage::SnapshotGuard; +use metadb_storage::{ScanControl, SnapshotGuard}; use crate::recovery_worker::TxnRecoveryWorker; use crate::rename_cycle::RenameCycleWalker; @@ -1380,26 +1380,38 @@ impl DistributedTxRouter { }; Self::validate_read_epoch(&group, epoch)?; - let rows = eng.scan_with_snapshot(sub_start, sub_end, &final_snap)?; - for (k_bytes, v_bytes) in rows { - let mk = MetaKey::decode(&k_bytes)?; + let mut stop = ListDirStop::Continue; + let need = effective_limit; + eng.scan_live_until(sub_start, sub_end, &final_snap, |k_bytes, v_bytes| { + let mk = MetaKey::decode(k_bytes)?; // Strict `_attr_` isolation rule (`Resolution N2` / `G-P1`): never expose `_attr_`! if mk.is_attr_key() { - continue; + return Ok(ScanControl::Continue); } if let Some(pfx) = name_prefix { if !mk.name.starts_with(pfx) { - return Ok((collected.clone(), None)); + stop = ListDirStop::PrefixEnded; + return Ok(ScanControl::Stop); } } collected.push(MetaEntry { key: mk, - attr: InodeAttr::decode(&v_bytes)?, + attr: InodeAttr::decode(v_bytes)?, }); - if collected.len() >= effective_limit { + if collected.len() >= need { + stop = ListDirStop::PageFull; + Ok(ScanControl::Stop) + } else { + Ok(ScanControl::Continue) + } + })?; + match stop { + ListDirStop::PrefixEnded => return Ok((collected, None)), + ListDirStop::PageFull => { let next_cursor = collected.last().map(|e| e.key.name.clone()); return Ok((collected, next_cursor)); } + ListDirStop::Continue => {} } } @@ -1551,10 +1563,17 @@ impl DistributedTxRouter { #[allow(dead_code)] const _ASSERT_QUOTA_PREFIX: u8 = TABLE_PREFIX_QUOTA; +#[derive(Clone, Copy)] +enum ListDirStop { + Continue, + PageFull, + PrefixEnded, +} + /// Lower bound for a keyset `listdir`. When both a cursor and a prefix are set, the scan /// starts at the later of the two so names before the prefix are not treated as the end /// of the page. -fn listdir_scan_start( +pub(crate) fn listdir_scan_start( pid: InodeId, after_name: Option<&str>, name_prefix: Option<&str>, diff --git a/docs/designs/correctness_hardening.md b/docs/designs/correctness_hardening.md index 250dbdf..d3a58ea 100644 --- a/docs/designs/correctness_hardening.md +++ b/docs/designs/correctness_hardening.md @@ -60,4 +60,4 @@ cargo test --workspace --no-fail-fast The added tests cover both original failures and rejection boundaries, including complete-prefix recovery, malformed snapshot matrices, data corruption, ID migration/downgrade safety, transaction/split interleavings, signed quota delivery, input atomicity, address restart behavior, and actual CLI/daemon behavior. This is not a physical power-cut experiment or a proof that every distributed history is correct. -Only `all-in-one` is an implemented daemon role. Remote Cell registration, autonomous multi-process role bootstrap, replicated GM/CM state, and physical jumbo-node migration remain unimplemented. `memtable_size_bytes` remains a reserved compatibility setting rather than an active flush threshold. Directory scans and multi-shard statistics still have scalability work beyond the corrected cursor semantics. These limitations are not changed into production-scale guarantees by passing regression tests. +Only `all-in-one` is an implemented daemon role. Remote Cell registration, autonomous multi-process role bootstrap, replicated GM/CM state, and physical jumbo-node migration remain unimplemented. Directory scans and multi-shard statistics still have scalability work beyond limit pushdown and the corrected cursor semantics. These limitations are not changed into production-scale guarantees by passing regression tests. diff --git a/docs/en/architecture.md b/docs/en/architecture.md index 5ae865f..0c09cd8 100644 --- a/docs/en/architecture.md +++ b/docs/en/architecture.md @@ -82,7 +82,7 @@ Each shard's state machine opens a WAL-less LSM at `/be/shards/= 0.50`) that triggers targeted bottommost range compaction. Lower to `64` for latency-sensitive `listdir` workloads under heavy TTL deletion. | From d94ce34658961cbf38fc0fbb2dd878dd1890de2a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 17 Nov 2026 15:20:32 +0000 Subject: [PATCH 2/3] Flush memtables only after a complete Raft command. Defer size-triggered seals until the command and its side effects finish, delete acked outbox keys after releasing the outbox lock, and keep a forward SST cursor that opens later non-overlapping files only when the current file ends. Co-authored-by: RustFS --- crates/metadb-raft/src/state_machine.rs | 48 +++++- crates/metadb-storage/src/engine.rs | 116 +++++++++++-- crates/metadb-storage/src/version_set.rs | 156 ++++++++++++++---- .../tests/first_ship_storage_regressions.rs | 150 ++++++++++++++++- crates/metadb-tx/src/aux_outbox.rs | 40 +++-- 5 files changed, 434 insertions(+), 76 deletions(-) diff --git a/crates/metadb-raft/src/state_machine.rs b/crates/metadb-raft/src/state_machine.rs index d1142d5..8376e20 100644 --- a/crates/metadb-raft/src/state_machine.rs +++ b/crates/metadb-raft/src/state_machine.rs @@ -141,6 +141,26 @@ struct PersistedSnapshotMeta { file_specs: Vec<(String, u64, u32)>, } +struct CommandFlush<'a> { + engine: &'a LifecycleStorageEngine, + open: bool, +} + +impl CommandFlush<'_> { + fn commit(mut self) -> Result<()> { + self.open = false; + self.engine.end_command(true) + } +} + +impl Drop for CommandFlush<'_> { + fn drop(&mut self) { + if self.open { + let _ = self.engine.end_command(false); + } + } +} + fn attach_flush_hook(engine: &LifecycleStorageEngine, handler: &Arc) { engine.set_pre_flush_hook(Arc::new(HandlerFlushHook { handler: handler.clone(), @@ -707,21 +727,31 @@ impl RaftStateMachine for ShardStateMachine { descriptor: self.shared.handler.encode_descriptor_bytes(), aux_index: self.shared.watermarks.snapshot_values().1, }); - self.shared - .handler - .apply_command(log_id.index, commit_hlc, &req, &engine) + // Seal only after every key and side effect of this command is in the memtable. + engine.begin_command(); + let command = CommandFlush { + engine: &engine, + open: true, + }; + match self.shared.handler.apply_command( + log_id.index, + commit_hlc, + &req, + &engine, + ) { + Ok(bytes) => command.commit().map(|()| bytes), + Err(error) => { + drop(command); + Err(error) + } + } }; let apply_res = match apply_res { Ok(bytes) => self .shared .refresh_flushed_checkpoint(&engine, &log_id, commit_hlc) .map(|()| bytes), - Err(error) => { - let _ = self - .shared - .refresh_flushed_checkpoint(&engine, &log_id, commit_hlc); - Err(error) - } + Err(error) => Err(error), }; let updated_epoch = self.shared.handler.current_epoch(); diff --git a/crates/metadb-storage/src/engine.rs b/crates/metadb-storage/src/engine.rs index 3d64a1b..3968876 100644 --- a/crates/metadb-storage/src/engine.rs +++ b/crates/metadb-storage/src/engine.rs @@ -17,7 +17,7 @@ use crate::mvcc::{ MemMvccStore, MvccValue, PinnedSnapshotData, PreFlushHook, ReadCut, SnapshotGuard, }; use crate::raft_log::{LogSequenceNumber, RaftLog, RaftLogPayload}; -use crate::version_set::Version; +use crate::version_set::{MergedCursor, Version}; /// Caller of [`LifecycleStorageEngine::scan_live_until`] stops or keeps scanning. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -95,6 +95,8 @@ pub struct LifecycleStorageEngine { flush_identity: RwLock>, entries_examined: AtomicU64, last_aux_frontier: AtomicU64, + /// Non-zero while a Raft command is still applying. Size flushes wait until it returns to zero. + command_depth: AtomicU64, } impl LifecycleStorageEngine { @@ -119,6 +121,7 @@ impl LifecycleStorageEngine { flush_identity: RwLock::new(None), entries_examined: AtomicU64::new(0), last_aux_frontier: AtomicU64::new(0), + command_depth: AtomicU64::new(0), } } @@ -181,9 +184,30 @@ impl LifecycleStorageEngine { flush_identity: RwLock::new(None), entries_examined: AtomicU64::new(0), last_aux_frontier: AtomicU64::new(0), + command_depth: AtomicU64::new(0), }) } + /// Holds size-triggered flushes until [`Self::end_command`] sees the whole command. + pub fn begin_command(&self) { + self.command_depth.fetch_add(1, Ordering::AcqRel); + } + + /// Ends a command started by [`Self::begin_command`]. + /// A successful command may seal once; a failed command does not advance the checkpoint. + pub fn end_command(&self, commit: bool) -> Result<()> { + let previous = self.command_depth.fetch_sub(1, Ordering::AcqRel); + if previous == 0 { + return Err(MetaDbError::StorageIoError( + "end_command without begin_command".to_string(), + )); + } + if previous == 1 && commit { + self.maintain_memtables()?; + } + Ok(()) + } + /// Installs the Raft pre-flush hook used by size-triggered seals. pub fn set_pre_flush_hook(&self, hook: Arc) { *self.flush_hook.write().unwrap() = Some(hook); @@ -237,7 +261,7 @@ impl LifecycleStorageEngine { raft_index: LogSequenceNumber, ) { self.mvcc.put(key, value, hlc, raft_index); - self.maintain_memtables().expect("memtable size flush"); + self.maintain_after_write(); } /// Applies an externally Raft-logged `Delete` at `(raft_index, hlc)` directly into `MemMvccStore`. @@ -248,7 +272,7 @@ impl LifecycleStorageEngine { raft_index: LogSequenceNumber, ) { self.mvcc.delete(key, hlc, raft_index); - self.maintain_memtables().expect("memtable size flush"); + self.maintain_after_write(); } /// Appends a `Put` to the internal `RaftLog` and applies it to `MemMvccStore`. @@ -413,12 +437,19 @@ impl LifecycleStorageEngine { if start_key >= end_key { return Ok(()); } - let mut disk_from = start_key.to_vec(); + let version = snap.with_step_lease(|pinned| Ok(pinned.version.clone()))?; + let mut disk = if version.has_sst_files() { + DiskScan::Sst(version.lazy_merged_cursor(start_key, Some(end_key))?) + } else { + DiskScan::Inline { + from: start_key.to_vec(), + } + }; let mut mvcc_from = start_key.to_vec(); let cut = snap.cut(); loop { let pulled = snap.with_step_lease(|pinned| { - self.pull_next(pinned, cut, &mut disk_from, &mut mvcc_from, end_key) + self.pull_next(pinned, cut, &mut disk, &mut mvcc_from, end_key) })?; match pulled { Pulled::Done => break, @@ -444,14 +475,16 @@ impl LifecycleStorageEngine { &self, pinned: &PinnedSnapshotData, cut: ReadCut, - disk_from: &mut Vec, + disk: &mut DiskScan, mvcc_from: &mut Vec, end_key: &[u8], ) -> Result { - if mvcc_from.as_slice() >= end_key && disk_from.as_slice() >= end_key { + let mvcc_done = mvcc_from.as_slice() >= end_key; + let disk_done = disk.is_finished(end_key); + if mvcc_done && disk_done { return Ok(Pulled::Done); } - let mvcc_item = if mvcc_from.as_slice() >= end_key { + let mvcc_item = if mvcc_done { None } else { self.mvcc @@ -459,13 +492,10 @@ impl LifecycleStorageEngine { .into_iter() .next() }; - let disk_item = if disk_from.as_slice() >= end_key { + let disk_item = if disk_done { None } else { - let (page, _) = - self.backend - .scan_in_version_page(&pinned.version, disk_from, end_key, 1)?; - page.into_iter().next() + disk.peek(self, pinned, end_key)? }; match (mvcc_item, disk_item) { (None, None) => Ok(Pulled::Done), @@ -474,7 +504,7 @@ impl LifecycleStorageEngine { Ok(pulled_from_mvcc(key, value)) } (None, Some((key, value))) => { - *disk_from = key_successor(&key); + disk.advance(&key)?; Ok(pulled_from_disk(key, value)) } (Some((mvcc_key, mvcc_value)), Some((disk_key, disk_value))) => { @@ -482,17 +512,24 @@ impl LifecycleStorageEngine { *mvcc_from = key_successor(&mvcc_key); Ok(pulled_from_mvcc(mvcc_key, mvcc_value)) } else if disk_key < mvcc_key { - *disk_from = key_successor(&disk_key); + disk.advance(&disk_key)?; Ok(pulled_from_disk(disk_key, disk_value)) } else { *mvcc_from = key_successor(&mvcc_key); - *disk_from = key_successor(&disk_key); + disk.advance(&disk_key)?; Ok(pulled_from_mvcc(mvcc_key, mvcc_value)) } } } } + fn maintain_after_write(&self) { + if self.command_depth.load(Ordering::Acquire) > 0 { + return; + } + self.maintain_memtables().expect("memtable size flush"); + } + /// Point lookup at `read_hlc` consulting `MemMvccStore` first and falling back to persistent storage. pub fn get(&self, key: &[u8], read_hlc: HlcTimestamp) -> Result>> { if let Some(mvcc_val) = self.mvcc.get_at(key, read_hlc)? { @@ -722,6 +759,53 @@ impl LifecycleStorageEngine { } } +enum DiskScan { + Sst(MergedCursor), + Inline { from: Vec }, +} + +impl DiskScan { + fn is_finished(&self, end_key: &[u8]) -> bool { + match self { + DiskScan::Sst(_) => false, + DiskScan::Inline { from } => from.as_slice() >= end_key, + } + } + + fn peek( + &mut self, + engine: &LifecycleStorageEngine, + pinned: &PinnedSnapshotData, + end_key: &[u8], + ) -> Result, PersistentEntry)>> { + match self { + DiskScan::Sst(cursor) => Ok(cursor + .peek()? + .map(|(key, value)| (key, PersistentEntry::from(value)))), + DiskScan::Inline { from } => { + if from.as_slice() >= end_key { + return Ok(None); + } + let (page, _) = + engine + .backend + .scan_in_version_page(&pinned.version, from, end_key, 1)?; + Ok(page.into_iter().next()) + } + } + } + + fn advance(&mut self, key: &[u8]) -> Result<()> { + match self { + DiskScan::Sst(cursor) => cursor.advance(), + DiskScan::Inline { from } => { + *from = key_successor(key); + Ok(()) + } + } + } +} + enum Pulled { Done, MvccTombstone, diff --git a/crates/metadb-storage/src/version_set.rs b/crates/metadb-storage/src/version_set.rs index 3e10b99..e61a578 100644 --- a/crates/metadb-storage/src/version_set.rs +++ b/crates/metadb-storage/src/version_set.rs @@ -223,11 +223,44 @@ impl Version { } MergedCursor::open(readers, start, end) } + + /// Like [`Self::merged_cursor`], but opens only the first overlapping file of each + /// non-overlapping level. The next file is opened when the current one is exhausted. + pub(crate) fn lazy_merged_cursor( + &self, + start: &[u8], + end: Option<&[u8]>, + ) -> Result { + let l0 = self.levels[0].clone(); + let mut later = Vec::with_capacity(NUM_LSM_LEVELS - 1); + for level in &self.levels[1..] { + let first = if start.is_empty() { + 0 + } else { + level.partition_point(|sst| sst.meta().max_key.as_slice() < start) + }; + let mut files = Vec::new(); + for sst in &level[first..] { + if end.is_some_and(|bound| sst.meta().min_key.as_slice() >= bound) { + break; + } + files.push(sst.clone()); + } + later.push(files); + } + MergedCursor::open_layered(l0, later, start, end) + } + + pub(crate) fn has_sst_files(&self) -> bool { + self.levels.iter().any(|level| !level.is_empty()) + } } /// K-way merge. Earlier readers win on equal keys (`L0` newest-first, then `L1..L3`). pub(crate) struct MergedCursor { sources: Vec, + pending_levels: Vec, + reheap_ids: Vec, heap: BinaryHeap, staged: Option, exhausted: bool, @@ -237,7 +270,15 @@ struct MergeSource { cursor: SstCursor, precedence: u32, generation: u64, - needs_reheap: bool, + /// Index into `pending_levels` when this source walks a non-overlapping level. + level_slot: Option, +} + +struct PendingLevel { + files: Vec>, + next: usize, + start: Vec, + end: Option>, } struct StagedEntry { @@ -279,31 +320,63 @@ impl MergedCursor { end: Option<&[u8]>, ) -> Result { let mut sources = Vec::with_capacity(readers.len()); - let mut heap = BinaryHeap::new(); + let heap = BinaryHeap::new(); for (index, reader) in readers.into_iter().enumerate() { - let mut cursor = SstCursor::open(reader, start, end)?; + let cursor = SstCursor::open(reader, start, end)?; let cursor_id = index as u32; - if let Some((key, _)) = cursor.peek()? { - heap.push(HeapNode { - key, - precedence: cursor_id, - cursor_id, - generation: 0, - }); - } sources.push(MergeSource { cursor, precedence: cursor_id, generation: 0, - needs_reheap: false, + level_slot: None, }); } - Ok(Self { + let mut cursor = Self { sources, + pending_levels: Vec::new(), + reheap_ids: Vec::new(), heap, staged: None, exhausted: false, - }) + }; + for id in 0..cursor.sources.len() { + cursor.reheap_one(id as u32)?; + } + Ok(cursor) + } + + /// `L0` readers are opened immediately because they overlap. Each later level + /// opens its first overlapping file only; the rest wait until that file ends. + pub(crate) fn open_layered( + l0: Vec>, + levels: Vec>>, + start: &[u8], + end: Option<&[u8]>, + ) -> Result { + let mut cursor = Self::open(l0, start, end)?; + for files in levels { + if files.is_empty() { + continue; + } + let level_slot = cursor.pending_levels.len(); + let precedence = ((level_slot as u32) + 1) << 20; + cursor.pending_levels.push(PendingLevel { + files, + next: 1, + start: start.to_vec(), + end: end.map(|key| key.to_vec()), + }); + let reader = cursor.pending_levels[level_slot].files[0].clone(); + let source_id = cursor.sources.len() as u32; + cursor.sources.push(MergeSource { + cursor: SstCursor::open(reader, start, end)?, + precedence, + generation: 0, + level_slot: Some(level_slot), + }); + cursor.reheap_one(source_id)?; + } + Ok(cursor) } pub(crate) fn peek(&mut self) -> Result, BlockEntryValue)>> { @@ -321,10 +394,10 @@ impl MergedCursor { return Ok(()); }; for source_id in staged.source_ids { - let source = &mut self.sources[source_id as usize]; - source.cursor.advance(); - source.generation = source.generation.wrapping_add(1); - source.needs_reheap = true; + self.sources[source_id as usize].cursor.advance(); + self.sources[source_id as usize].generation = + self.sources[source_id as usize].generation.wrapping_add(1); + self.reheap_ids.push(source_id); } Ok(()) } @@ -389,23 +462,40 @@ impl MergedCursor { } fn reheap_pending(&mut self) -> Result<()> { - for cursor_id in 0..self.sources.len() { - if !self.sources[cursor_id].needs_reheap { - continue; + let ids = std::mem::take(&mut self.reheap_ids); + for id in ids { + self.reheap_one(id)?; + } + Ok(()) + } + + fn reheap_one(&mut self, id: u32) -> Result<()> { + loop { + if let Some((key, _)) = self.sources[id as usize].cursor.peek()? { + let generation = self.sources[id as usize].generation; + let precedence = self.sources[id as usize].precedence; + self.heap.push(HeapNode { + key, + precedence, + cursor_id: id, + generation, + }); + return Ok(()); } - self.sources[cursor_id].needs_reheap = false; - let generation = self.sources[cursor_id].generation; - let precedence = self.sources[cursor_id].precedence; - let Some((key, _)) = self.sources[cursor_id].cursor.peek()? else { - continue; + let Some(level_slot) = self.sources[id as usize].level_slot else { + return Ok(()); }; - self.heap.push(HeapNode { - key, - precedence, - cursor_id: cursor_id as u32, - generation, - }); + if self.pending_levels[level_slot].next >= self.pending_levels[level_slot].files.len() { + return Ok(()); + } + let reader = + self.pending_levels[level_slot].files[self.pending_levels[level_slot].next].clone(); + self.pending_levels[level_slot].next += 1; + let start = self.pending_levels[level_slot].start.clone(); + let end = self.pending_levels[level_slot].end.clone(); + self.sources[id as usize].generation = + self.sources[id as usize].generation.wrapping_add(1); + self.sources[id as usize].cursor = SstCursor::open(reader, &start, end.as_deref())?; } - Ok(()) } } diff --git a/crates/metadb-storage/tests/first_ship_storage_regressions.rs b/crates/metadb-storage/tests/first_ship_storage_regressions.rs index 669bee7..43fe203 100644 --- a/crates/metadb-storage/tests/first_ship_storage_regressions.rs +++ b/crates/metadb-storage/tests/first_ship_storage_regressions.rs @@ -6,10 +6,14 @@ use std::thread; use std::time::Duration; use metadb_core::{HlcTimestamp, MetaKey}; +use metadb_storage::block::BlockEntryValue; use metadb_storage::sstable::{ data_blocks_touched, full_range_materializations, reset_scan_probes, }; -use metadb_storage::{LifecycleStorageEngine, LsmEngineConfig, ScanControl, ShardedBlockCache}; +use metadb_storage::{ + DurableShardCheckpoint, LifecycleStorageEngine, LsmEngineConfig, ManifestLog, ScanControl, + ShardedBlockCache, SstableBuilder, VersionEdit, +}; use tempfile::tempdir; fn engine(dir: &std::path::Path, config: LsmEngineConfig) -> LifecycleStorageEngine { @@ -272,3 +276,147 @@ fn tiny_memtable_flushes_and_immutable_tables_respect_the_cap() { Some(b"y".to_vec()) ); } + +#[test] +fn size_flush_waits_until_the_command_finishes() { + let dir = tempdir().unwrap(); + let lsm = dir.path().join("lsm"); + let engine = engine( + &lsm, + LsmEngineConfig { + auto_locality_compaction: false, + memtable_size_bytes: 1, + ..LsmEngineConfig::default() + }, + ); + engine.begin_command(); + engine.apply_committed_put( + b"child".to_vec(), + b"a".to_vec(), + HlcTimestamp::new(9_000, 1), + 7, + ); + engine.apply_committed_put( + b"attr".to_vec(), + b"b".to_vec(), + HlcTimestamp::new(9_000, 2), + 7, + ); + assert_eq!( + engine.durable_checkpoint().flushed_raft_index, + 0, + "a threshold crossing must not checkpoint a partial command" + ); + engine.end_command(false).unwrap(); + assert_eq!(engine.durable_checkpoint().flushed_raft_index, 0); + + engine.begin_command(); + engine.apply_committed_put( + b"child".to_vec(), + b"a".to_vec(), + HlcTimestamp::new(9_000, 1), + 7, + ); + engine.apply_committed_put( + b"attr".to_vec(), + b"b".to_vec(), + HlcTimestamp::new(9_000, 2), + 7, + ); + engine.end_command(true).unwrap(); + assert!(engine.durable_checkpoint().flushed_raft_index >= 7); + + drop(engine); + let cache = Arc::new(ShardedBlockCache::new(1024 * 1024)); + let reopened = LifecycleStorageEngine::open_lsm( + 1, + &lsm, + cache, + LsmEngineConfig { + memtable_size_bytes: 32 * 1024 * 1024, + auto_locality_compaction: false, + ..LsmEngineConfig::default() + }, + ) + .unwrap(); + let snap = reopened.acquire_linearizable_snapshot(0).unwrap(); + assert_eq!( + reopened.get_with_snapshot(b"child", &snap).unwrap(), + Some(b"a".to_vec()) + ); + assert_eq!( + reopened.get_with_snapshot(b"attr", &snap).unwrap(), + Some(b"b".to_vec()) + ); +} + +#[test] +fn limited_scan_does_not_open_later_nonoverlapping_files() { + let dir = tempdir().unwrap(); + let lsm = dir.path().join("lsm"); + std::fs::create_dir_all(&lsm).unwrap(); + let live = MetaKey::child(3, "a").encode(); + let mut later = Vec::new(); + for index in 0..2048u32 { + later.push(( + MetaKey::child(3, format!("t{index:04}")).encode(), + BlockEntryValue::PhysicalTombstone, + )); + } + { + let (mut manifest, _) = ManifestLog::open_or_create(&lsm).unwrap(); + let first = SstableBuilder::new(1, 10, 1, 1, HlcTimestamp::new(1, 0)) + .write_to_dir( + &lsm, + &[(live.clone(), BlockEntryValue::Value(b"live".to_vec()))], + ) + .unwrap(); + let second = SstableBuilder::new(1, 11, 2, 2, HlcTimestamp::new(2, 0)) + .write_to_dir(&lsm, &later) + .unwrap(); + manifest + .append(&VersionEdit { + next_file_id: Some(12), + next_flush_generation: Some(3), + checkpoint: Some(DurableShardCheckpoint::default()), + deleted_files: vec![], + added_files: vec![first, second], + }) + .unwrap(); + } + + let cache = Arc::new(ShardedBlockCache::new(8 * 1024 * 1024)); + let engine = LifecycleStorageEngine::open_lsm( + 1, + &lsm, + cache, + LsmEngineConfig { + auto_locality_compaction: false, + tombstone_density_threshold: 100_000, + ..LsmEngineConfig::default() + }, + ) + .unwrap(); + let version = engine.backend().current_version(); + let first_blocks = version.levels[1][0].index_entries().len() as u64; + let later_blocks = version.levels[1][1].index_entries().len() as u64; + assert!(later_blocks > first_blocks); + + reset_scan_probes(); + let snap = engine.acquire_linearizable_snapshot(0).unwrap(); + let mut rows = Vec::new(); + engine + .scan_live_until( + &MetaKey::encode_pid_prefix_start(3), + &MetaKey::encode_pid_prefix_end(3), + &snap, + |key, value| { + rows.push((key.to_vec(), value.to_vec())); + Ok(ScanControl::Stop) + }, + ) + .unwrap(); + assert_eq!(rows, vec![(live, b"live".to_vec())]); + assert_eq!(data_blocks_touched(), first_blocks); + assert_eq!(engine.metrics().tombstones_traversed_on_scans, 0); +} diff --git a/crates/metadb-tx/src/aux_outbox.rs b/crates/metadb-tx/src/aux_outbox.rs index 42f0fce..a874d65 100644 --- a/crates/metadb-tx/src/aux_outbox.rs +++ b/crates/metadb-tx/src/aux_outbox.rs @@ -535,26 +535,32 @@ impl ShardAuxOutbox { acked_keys: &[(u64, u16)], engine: &LifecycleStorageEngine, ) -> u64 { - let mut guard = self.inner.write().unwrap(); - guard.max_observed_raft_index = guard.max_observed_raft_index.max(raft_index); - - for &(idx, ord) in acked_keys { - if let Some(removed) = guard.tasks.remove(&(idx, ord)) { - if removed.durable_spilled || removed.sealed_at_index.is_some() { - let spill_key = SpilledAuxOutboxKey::new(idx, ord).encode().to_vec(); - engine.apply_committed_delete(spill_key, hlc, raft_index); + // Collect deletions under the outbox lock, then write them after releasing it. + // A size-triggered flush takes this same lock from `spill_into_seal_batch`. + let (deletes, acked) = { + let mut guard = self.inner.write().unwrap(); + guard.max_observed_raft_index = guard.max_observed_raft_index.max(raft_index); + let mut deletes = Vec::new(); + for &(idx, ord) in acked_keys { + let delete_spill = if let Some(removed) = guard.tasks.remove(&(idx, ord)) { + removed.durable_spilled || removed.sealed_at_index.is_some() + } else { + // Idempotent delete in case `[0x07]` existed from prior snapshot/recovery. + true + }; + if delete_spill { + deletes.push(SpilledAuxOutboxKey::new(idx, ord).encode().to_vec()); } - } else { - // Idempotent delete in case `[0x07]` existed from prior snapshot/recovery. - let spill_key = SpilledAuxOutboxKey::new(idx, ord).encode().to_vec(); - engine.apply_committed_delete(spill_key, hlc, raft_index); } + guard.recompute_contiguous_acked(); + let frontier = guard.contiguous_durable_frontier(); + self.watermarks.advance_aux_outbox_durable(frontier); + (deletes, guard.contiguous_acked_index) + }; + for key in deletes { + engine.apply_committed_delete(key, hlc, raft_index); } - - guard.recompute_contiguous_acked(); - let frontier = guard.contiguous_durable_frontier(); - self.watermarks.advance_aux_outbox_durable(frontier); - guard.contiguous_acked_index + acked } /// Rehydrates spilled `[0x07]` tasks from `engine` on startup or after `install_snapshot` (`Resolution B6`). From 24b8c439c636bfbbc3c3ed2bec667ff31f37ceda Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 17 Nov 2026 15:24:25 +0000 Subject: [PATCH 3/3] Encode flush checkpoints with the shared serde codec. Main replaced bincode with wincode. Size-triggered membership bytes now use the same codec::serialize_serde path as explicit flushes, so locked release builds compile. Co-authored-by: RustFS --- crates/metadb-raft/src/state_machine.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/metadb-raft/src/state_machine.rs b/crates/metadb-raft/src/state_machine.rs index 8376e20..bab0b77 100644 --- a/crates/metadb-raft/src/state_machine.rs +++ b/crates/metadb-raft/src/state_machine.rs @@ -286,7 +286,7 @@ impl StateMachineShared { return Ok(()); } let membership = self.last_membership.read().unwrap().clone(); - let membership_bytes = bincode::serialize(&membership).unwrap_or_default(); + let membership_bytes = codec::serialize_serde(&membership).unwrap_or_default(); let descriptor = self.handler.encode_descriptor_bytes(); let corrected = DurableShardCheckpoint { flushed_raft_term: log_id.committed_leader_id().term, @@ -723,7 +723,7 @@ impl RaftStateMachine for ShardStateMachine { engine.set_flush_identity(FlushIdentity { term: log_id.committed_leader_id().term, leader: log_id.committed_leader_id().node_id, - membership: bincode::serialize(&membership).unwrap_or_default(), + membership: codec::serialize_serde(&membership).unwrap_or_default(), descriptor: self.shared.handler.encode_descriptor_bytes(), aux_index: self.shared.watermarks.snapshot_values().1, });