diff --git a/crates/metadb-raft/src/state_machine.rs b/crates/metadb-raft/src/state_machine.rs index 8aec598..bab0b77 100644 --- a/crates/metadb-raft/src/state_machine.rs +++ b/crates/metadb-raft/src/state_machine.rs @@ -24,7 +24,7 @@ use metadb_core::{ }; use metadb_storage::sstable::{sst_file_name, sync_dir}; use metadb_storage::{ - DurableShardCheckpoint, LifecycleStorageEngine, LsmEngineConfig, PreFlushHook, + DurableShardCheckpoint, FlushIdentity, LifecycleStorageEngine, LsmEngineConfig, PreFlushHook, ShardedBlockCache, }; @@ -141,6 +141,57 @@ struct PersistedSnapshotMeta { file_specs: Vec<(String, u64, u32)>, } +struct CommandFlush<'a> { + engine: &'a LifecycleStorageEngine, + open: bool, +} + +impl CommandFlush<'_> { + fn commit(mut self) -> Result<()> { + self.open = false; + self.engine.end_command(true) + } +} + +impl Drop for CommandFlush<'_> { + fn drop(&mut self) { + if self.open { + let _ = self.engine.end_command(false); + } + } +} + +fn attach_flush_hook(engine: &LifecycleStorageEngine, handler: &Arc) { + engine.set_pre_flush_hook(Arc::new(HandlerFlushHook { + handler: handler.clone(), + })); +} + +struct HandlerFlushHook { + handler: Arc, +} + +impl PreFlushHook for HandlerFlushHook { + fn on_seal_memtable( + &self, + sealed_raft_index: u64, + sealed_max_hlc: HlcTimestamp, + view_at_m: &metadb_storage::ViewAtRaftIndex<'_>, + ) -> Result, HlcTimestamp, metadb_storage::MvccValue)>> { + match self.handler.pre_flush_hook() { + Some(hook) => hook.on_seal_memtable(sealed_raft_index, sealed_max_hlc, view_at_m), + None => Ok(Vec::new()), + } + } + + fn on_flush_durable(&self, durable_raft_index: u64, durable_max_hlc: HlcTimestamp) -> u64 { + match self.handler.pre_flush_hook() { + Some(hook) => hook.on_flush_durable(durable_raft_index, durable_max_hlc), + None => 0, + } + } +} + fn checkpoint_log_id(checkpoint: &DurableShardCheckpoint) -> Option> { if ( checkpoint.flushed_raft_term, @@ -223,6 +274,41 @@ impl StateMachineShared { Ok((state_flushed, updated_cp)) } + fn refresh_flushed_checkpoint( + &self, + engine: &LifecycleStorageEngine, + log_id: &LogIdOf, + applied_hlc: HlcTimestamp, + ) -> Result<()> { + let current = engine.durable_checkpoint(); + // Only the apply that flushed this index may stamp its term and descriptor. + if current.flushed_raft_index != log_id.index { + return Ok(()); + } + let membership = self.last_membership.read().unwrap().clone(); + let membership_bytes = codec::serialize_serde(&membership).unwrap_or_default(); + let descriptor = self.handler.encode_descriptor_bytes(); + let corrected = DurableShardCheckpoint { + flushed_raft_term: log_id.committed_leader_id().term, + flushed_raft_leader: log_id.committed_leader_id().node_id, + flushed_raft_index: current.flushed_raft_index, + flushed_max_hlc: current.flushed_max_hlc.max(applied_hlc), + contiguous_aux_acked_index: current.contiguous_aux_acked_index, + membership_bytes, + descriptor_bytes: descriptor, + }; + if corrected != current { + engine.persist_checkpoint(corrected.clone())?; + } + self.watermarks + .advance_state_flushed(corrected.flushed_raft_index); + if self.handler.pre_flush_hook().is_some() { + self.watermarks + .advance_aux_outbox_durable(engine.last_aux_frontier()); + } + Ok(()) + } + fn build_snapshot_sync(&self) -> io::Result> { let _operation = self.operation_lock.lock().unwrap(); let current_idx = self @@ -454,6 +540,7 @@ impl ShardStateMachine { block_cache.clone(), lsm_config.clone(), )?); + attach_flush_hook(&engine, &handler); let cp = engine.durable_checkpoint(); // Report `MANIFEST`'s `flushed_raft_log_id` (`F`) as `applied_state` on startup so OpenRaft @@ -632,9 +719,39 @@ impl RaftStateMachine for ShardStateMachine { actual: cur_epoch, }) } else { - self.shared - .handler - .apply_command(log_id.index, commit_hlc, &req, &engine) + let membership = self.shared.last_membership.read().unwrap().clone(); + engine.set_flush_identity(FlushIdentity { + term: log_id.committed_leader_id().term, + leader: log_id.committed_leader_id().node_id, + membership: codec::serialize_serde(&membership).unwrap_or_default(), + descriptor: self.shared.handler.encode_descriptor_bytes(), + aux_index: self.shared.watermarks.snapshot_values().1, + }); + // Seal only after every key and side effect of this command is in the memtable. + engine.begin_command(); + let command = CommandFlush { + engine: &engine, + open: true, + }; + match self.shared.handler.apply_command( + log_id.index, + commit_hlc, + &req, + &engine, + ) { + Ok(bytes) => command.commit().map(|()| bytes), + Err(error) => { + drop(command); + Err(error) + } + } + }; + let apply_res = match apply_res { + Ok(bytes) => self + .shared + .refresh_flushed_checkpoint(&engine, &log_id, commit_hlc) + .map(|()| bytes), + Err(error) => Err(error), }; let updated_epoch = self.shared.handler.current_epoch(); @@ -852,6 +969,7 @@ impl RaftStateMachine for ShardStateMachine { ) .map_err(|e| io::Error::other(e.to_string()))?, ); + attach_flush_hook(&reopened, &self.shared.handler); *self.shared.engine.write().unwrap() = reopened.clone(); *self.shared.last_applied_log_id.write().unwrap() = meta.last_log_id; diff --git a/crates/metadb-server/src/config.rs b/crates/metadb-server/src/config.rs index e7f73ed..e51afe8 100644 --- a/crates/metadb-server/src/config.rs +++ b/crates/metadb-server/src/config.rs @@ -75,6 +75,9 @@ pub struct StorageSection { pub bloom_bits_per_key: usize, pub locality_tombstone_threshold: usize, pub locality_tombstone_ratio: f64, + /// Queued immutable memtables allowed before writers wait for a flush. + /// The storage design documents a backpressure limit of four. + pub max_imm_memtables: usize, } impl Default for StorageSection { @@ -86,6 +89,7 @@ impl Default for StorageSection { bloom_bits_per_key: 10, locality_tombstone_threshold: 128, locality_tombstone_ratio: 0.50, + max_imm_memtables: 4, } } } @@ -170,6 +174,8 @@ impl ServerTomlConfig { tombstone_min_ratio_permille: ratio_permille, bloom_bits_per_key: self.storage.bloom_bits_per_key, auto_locality_compaction: true, + memtable_size_bytes: self.storage.memtable_size_bytes.max(1), + max_imm_memtables: self.storage.max_imm_memtables.max(1), }) } } @@ -212,5 +218,9 @@ mod tests { .unwrap(); assert_eq!(cfg.storage.memtable_size_bytes, 1024); assert_eq!(cfg.storage.block_cache_bytes, 65536); + assert_eq!(cfg.storage.max_imm_memtables, 4); + let lsm = cfg.to_lsm_config().unwrap(); + assert_eq!(lsm.memtable_size_bytes, 1024); + assert_eq!(lsm.max_imm_memtables, 4); } } diff --git a/crates/metadb-server/src/lib.rs b/crates/metadb-server/src/lib.rs index 8fd724e..48b0a0f 100644 --- a/crates/metadb-server/src/lib.rs +++ b/crates/metadb-server/src/lib.rs @@ -367,9 +367,21 @@ impl MetaDbEngine { /// Lists all live child entries in directory `pid`. pub fn listdir(&self, pid: InodeId) -> Result> { + self.listdir_limited(pid, None, None, usize::MAX) + } + + /// Pinned directory page. `limit == usize::MAX` reads the whole directory. + pub fn listdir_limited( + &self, + pid: InodeId, + after_name: Option<&str>, + name_prefix: Option<&str>, + limit: usize, + ) -> Result> { let hlc = self.clock.now(); let (_, _, storage) = self.global_manager.route_pid(pid)?; - self.pushdown.listdir(&storage, pid, hlc) + self.pushdown + .listdir_limited(&storage, pid, after_name, name_prefix, limit, hlc) } /// Executes a lightweight metadata SQL template statement directly without full SQL optimizer overhead. @@ -411,16 +423,13 @@ impl MetaDbEngine { name_prefix, limit, } => { - let mut rows = self.listdir(pid)?; - if let Some(after) = after_name { - rows.retain(|e| e.key.name > after); - } - if let Some(pfx) = name_prefix { - rows.retain(|e| e.key.name.starts_with(&pfx)); - } - if limit > 0 && rows.len() > limit { - rows.truncate(limit); - } + let page_limit = if limit == 0 { usize::MAX } else { limit }; + let rows = self.listdir_limited( + pid, + after_name.as_deref(), + name_prefix.as_deref(), + page_limit, + )?; Ok(SqlQueryResult::Entries(rows)) } MetadataSqlCommand::Rename { diff --git a/crates/metadb-storage/src/backend.rs b/crates/metadb-storage/src/backend.rs index 9a0ce10..a906cfa 100644 --- a/crates/metadb-storage/src/backend.rs +++ b/crates/metadb-storage/src/backend.rs @@ -12,7 +12,7 @@ use crate::block::BlockEntryValue; use crate::cache::ShardedBlockCache; use crate::manifest::{DurableShardCheckpoint, ManifestLog, VersionEdit}; use crate::sstable::{sst_file_name, sync_dir, SstFileMeta, SstableBuilder, SstableReader}; -use crate::version_set::{Version, NUM_LSM_LEVELS}; +use crate::version_set::{MergedCursor, Version, NUM_LSM_LEVELS}; pub const DEFAULT_L0_COMPACTION_TRIGGER: usize = 4; pub const TARGET_L1_SST_BYTES: usize = 64 * 1024 * 1024; @@ -469,63 +469,36 @@ impl StorageBackend for LsmStorageBackend { lvl.sort_by(|a, b| a.meta().min_key.cmp(&b.meta().min_key)); } - // Step 3: Merge 100% of keys from all participating SSTables in deterministic precedence order! - let mut merged: BTreeMap, BlockEntryValue> = BTreeMap::new(); + // Step 3: Stream every key from the participating SSTables. Earlier readers + // win (`L0` newest flush generation, then `L1..L3`). One data block per + // input file is resident; the output is flushed in `TARGET_L1_SST_BYTES` chunks. let mut max_raft_idx = 0u64; let mut max_hlc = metadb_core::HlcTimestamp::ZERO; - let mut input_tombstones_in_range = 0usize; - + let mut readers = Vec::new(); for sst in &participating[0] { max_raft_idx = max_raft_idx.max(sst.meta().max_raft_index); max_hlc = max_hlc.max(sst.meta().max_hlc); - for (k, v) in sst.read_all_entries()? { - merged.entry(k).or_insert(v); - } + readers.push(sst.clone()); } for lvl in &participating[1..] { for sst in lvl { max_raft_idx = max_raft_idx.max(sst.meta().max_raft_index); max_hlc = max_hlc.max(sst.meta().max_hlc); - for (k, v) in sst.read_all_entries()? { - merged.entry(k).or_insert(v); - } + readers.push(sst.clone()); } } + let mut cursor = MergedCursor::open(readers, &[], None)?; // Step 4: Bottommost Tombstone Drop Invariant (`Resolution B4`): // Drop `PhysicalTombstone` iff zero non-participating SSTables in levels `> target_level` // overlap `K`. For `K ∈ [start_key, end_key)`, `target_level` already absorbed the deepest // overlapping SSTable so `has_overlap_above` is guaranteed `false`; for outer boundary keys, // `base_version.has_overlap_above(target_level, &k)` preserves tombstones whenever needed! - let mut output_entries: Vec<(Vec, BlockEntryValue)> = Vec::with_capacity(merged.len()); - let mut purged_tombstones = 0usize; - - for (k, v) in merged { - match v { - BlockEntryValue::Value(bytes) => { - output_entries.push((k, BlockEntryValue::Value(bytes))); - } - BlockEntryValue::PhysicalTombstone => { - let in_target_range = k.as_slice() >= start_key && k.as_slice() < end_key; - if in_target_range { - input_tombstones_in_range += 1; - } - if !base_version.has_overlap_above(target_level, &k) { - purged_tombstones += 1; - } else { - output_entries.push((k, BlockEntryValue::PhysicalTombstone)); - } - } - } - } - let _ = input_tombstones_in_range; - - // Partition output into <= TARGET_L1_SST_BYTES non-overlapping SSTables at `target_level`. let mut new_metas: Vec = Vec::new(); let mut new_readers: Vec> = Vec::new(); - let mut chunk: Vec<(Vec, BlockEntryValue)> = Vec::new(); let mut chunk_bytes = 0usize; + let mut purged_tombstones = 0usize; let write_chunk = |items: &[(Vec, BlockEntryValue)]| -> Result<(SstFileMeta, Arc)> { @@ -549,27 +522,41 @@ impl StorageBackend for LsmStorageBackend { Ok((meta, reader)) }; - for (k, v) in output_entries { + while let Some((k, v)) = cursor.next_owned()? { + let emitted = match v { + BlockEntryValue::Value(bytes) => Some(BlockEntryValue::Value(bytes)), + BlockEntryValue::PhysicalTombstone => { + if !base_version.has_overlap_above(target_level, &k) { + purged_tombstones += 1; + None + } else { + Some(BlockEntryValue::PhysicalTombstone) + } + } + }; + let Some(value) = emitted else { + continue; + }; let sz = k.len() - + match &v { - BlockEntryValue::Value(b) => b.len(), + + match &value { + BlockEntryValue::Value(bytes) => bytes.len(), BlockEntryValue::PhysicalTombstone => 0, } + 16; - chunk.push((k, v)); + chunk.push((k, value)); chunk_bytes += sz; if chunk_bytes >= TARGET_L1_SST_BYTES { - let (m, r) = write_chunk(&chunk)?; - new_metas.push(m); - new_readers.push(r); + let (meta, reader) = write_chunk(&chunk)?; + new_metas.push(meta); + new_readers.push(reader); chunk.clear(); chunk_bytes = 0; } } if !chunk.is_empty() { - let (m, r) = write_chunk(&chunk)?; - new_metas.push(m); - new_readers.push(r); + let (meta, reader) = write_chunk(&chunk)?; + new_metas.push(meta); + new_readers.push(reader); } // Step 5: Atomically publish `VersionEdit` (taking care to retain any newer L0 files flushed diff --git a/crates/metadb-storage/src/block.rs b/crates/metadb-storage/src/block.rs index 796fbbb..c6d79bd 100644 --- a/crates/metadb-storage/src/block.rs +++ b/crates/metadb-storage/src/block.rs @@ -229,6 +229,40 @@ impl DataBlock { Ok(()) } + /// Byte offset of the restart point that may contain `start_key`. + /// An empty `start_key` selects the first restart. + pub(crate) fn restart_offset_for(&self, start_key: &[u8]) -> Result { + if start_key.is_empty() || self.restarts.is_empty() { + return Ok(self.restarts.first().copied().unwrap_or(0) as usize); + } + let mut left = 0usize; + let mut right = self.restarts.len(); + while left + 1 < right { + let mid = left + (right - left) / 2; + let restart_key = self.decode_restart_key(self.restarts[mid] as usize)?; + if restart_key.as_slice() <= start_key { + left = mid; + } else { + right = mid; + } + } + Ok(self.restarts[left] as usize) + } + + /// Decodes the entry at `offset`, advancing `offset` and `key`. + /// Returns `None` at the end of the block. + pub(crate) fn read_entry( + &self, + offset: &mut usize, + key: &mut Vec, + ) -> Result> { + if *offset >= self.entries_end { + return Ok(None); + } + let value = self.decode_next_entry(offset, key)?; + Ok(Some(value)) + } + /// Decodes every entry in the block (used by compaction iterators). pub fn iter_all(&self) -> Result, BlockEntryValue)>> { let mut offset = 0usize; diff --git a/crates/metadb-storage/src/compaction.rs b/crates/metadb-storage/src/compaction.rs index 670b50f..795b3ef 100644 --- a/crates/metadb-storage/src/compaction.rs +++ b/crates/metadb-storage/src/compaction.rs @@ -54,6 +54,25 @@ impl LocalityCompactor { self.density_threshold } + /// Replaces in-memory bucket counters. Used when opening an LSM so compaction + /// decisions follow the tombstones and live keys already on disk. + pub fn restore_counts(&self, counts: BTreeMap) { + let mut guard = self.inner.write().unwrap(); + guard.buckets.clear(); + for (prefix, (tombstones, live_writes)) in counts { + if tombstones == 0 && live_writes == 0 { + continue; + } + guard.buckets.insert( + prefix, + BucketDensity { + tombstones, + live_writes, + }, + ); + } + } + /// Extracts the 9-byte `[table_prefix][group_key: 8B]` bucket prefix from a raw key. pub fn bucket_for_key(key: &[u8]) -> Option { if key.len() >= 9 { diff --git a/crates/metadb-storage/src/engine.rs b/crates/metadb-storage/src/engine.rs index 464d104..3968876 100644 --- a/crates/metadb-storage/src/engine.rs +++ b/crates/metadb-storage/src/engine.rs @@ -1,16 +1,40 @@ +use std::collections::BTreeMap; use std::path::Path; -use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::{Duration, Instant}; -use metadb_core::{HlcTimestamp, Result, ShardId}; +use metadb_core::{HlcTimestamp, MetaDbError, Result, ShardId}; pub use crate::backend::PersistentEntry; use crate::backend::{InMemoryBackend, LsmStorageBackend, StorageBackend}; +use crate::block::BlockEntryValue; use crate::bloom::DEFAULT_BLOOM_BITS_PER_KEY; use crate::cache::ShardedBlockCache; -use crate::compaction::{CompactionStats, LocalityCompactor}; +use crate::compaction::{CompactionStats, LocalityCompactor, RangeBucketPrefix}; use crate::manifest::DurableShardCheckpoint; -use crate::mvcc::{MemMvccStore, MvccValue, PreFlushHook, SnapshotGuard}; +use crate::mvcc::{ + MemMvccStore, MvccValue, PinnedSnapshotData, PreFlushHook, ReadCut, SnapshotGuard, +}; use crate::raft_log::{LogSequenceNumber, RaftLog, RaftLogPayload}; +use crate::version_set::{MergedCursor, Version}; + +/// Caller of [`LifecycleStorageEngine::scan_live_until`] stops or keeps scanning. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ScanControl { + Continue, + Stop, +} + +/// Raft identity stamped onto a size-triggered flush checkpoint. +#[derive(Debug, Clone)] +pub struct FlushIdentity { + pub term: u64, + pub leader: u64, + pub membership: Vec, + pub descriptor: Vec, + pub aux_index: u64, +} /// Operational metrics demonstrating the benefits of In-Memory MVCC and Locality-Aware Compaction. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -20,6 +44,8 @@ pub struct StorageMetrics { pub persistent_kv_writes: u64, pub active_physical_tombstones: usize, pub tombstones_traversed_on_scans: u64, + /// Keys the limited scanner actually stepped to, including physical tombstones. + pub entries_examined_on_scans: u64, pub compaction_stats: CompactionStats, } @@ -31,6 +57,10 @@ pub struct LsmEngineConfig { pub tombstone_min_ratio_permille: usize, pub bloom_bits_per_key: usize, pub auto_locality_compaction: bool, + /// Seal and flush the active memtable once its accounted bytes reach this size. + pub memtable_size_bytes: usize, + /// Immutable memtables queued above this cap block writers until one flushes. + pub max_imm_memtables: usize, } impl Default for LsmEngineConfig { @@ -41,6 +71,8 @@ impl Default for LsmEngineConfig { tombstone_min_ratio_permille: 500, bloom_bits_per_key: DEFAULT_BLOOM_BITS_PER_KEY, auto_locality_compaction: true, + memtable_size_bytes: 32 * 1024 * 1024, + max_imm_memtables: 4, } } } @@ -56,6 +88,15 @@ pub struct LifecycleStorageEngine { compactor: Arc, backend: Arc, auto_locality_compaction: bool, + memtable_size_bytes: usize, + max_imm_memtables: usize, + budget_mu: Mutex<()>, + flush_hook: RwLock>>, + flush_identity: RwLock>, + entries_examined: AtomicU64, + last_aux_frontier: AtomicU64, + /// Non-zero while a Raft command is still applying. Size flushes wait until it returns to zero. + command_depth: AtomicU64, } impl LifecycleStorageEngine { @@ -73,6 +114,14 @@ impl LifecycleStorageEngine { compactor: Arc::new(LocalityCompactor::new(tombstone_density_threshold)), backend: Arc::new(InMemoryBackend::new()), auto_locality_compaction, + memtable_size_bytes: 32 * 1024 * 1024, + max_imm_memtables: 4, + budget_mu: Mutex::new(()), + flush_hook: RwLock::new(None), + flush_identity: RwLock::new(None), + entries_examined: AtomicU64::new(0), + last_aux_frontier: AtomicU64::new(0), + command_depth: AtomicU64::new(0), } } @@ -85,7 +134,7 @@ impl LifecycleStorageEngine { ) -> Result { let lsm = LsmStorageBackend::open(shard_id, lsm_dir, block_cache, config.bloom_bits_per_key)?; - Ok(Self::with_lsm_backend(shard_id, lsm, config)) + Self::with_lsm_backend(shard_id, lsm, config) } /// Opens a complete existing checkpoint without creating or repairing its manifest. @@ -101,14 +150,14 @@ impl LifecycleStorageEngine { block_cache, config.bloom_bits_per_key, )?; - Ok(Self::with_lsm_backend(shard_id, lsm, config)) + Self::with_lsm_backend(shard_id, lsm, config) } fn with_lsm_backend( shard_id: ShardId, lsm: LsmStorageBackend, config: LsmEngineConfig, - ) -> Self { + ) -> Result { let lsm = Arc::new(lsm); let cp = lsm.current_version().checkpoint.clone(); let mvcc = Arc::new(MemMvccStore::with_recovered_watermarks( @@ -120,14 +169,63 @@ impl LifecycleStorageEngine { config.tombstone_density_threshold, config.tombstone_min_ratio_permille, )); - Self { + rebuild_density(&compactor, &lsm.current_version())?; + Ok(Self { shard_id, raft_log: Arc::new(RaftLog::new(shard_id)), mvcc, compactor, backend: lsm, auto_locality_compaction: config.auto_locality_compaction, + memtable_size_bytes: config.memtable_size_bytes.max(1), + max_imm_memtables: config.max_imm_memtables.max(1), + budget_mu: Mutex::new(()), + flush_hook: RwLock::new(None), + flush_identity: RwLock::new(None), + entries_examined: AtomicU64::new(0), + last_aux_frontier: AtomicU64::new(0), + command_depth: AtomicU64::new(0), + }) + } + + /// Holds size-triggered flushes until [`Self::end_command`] sees the whole command. + pub fn begin_command(&self) { + self.command_depth.fetch_add(1, Ordering::AcqRel); + } + + /// Ends a command started by [`Self::begin_command`]. + /// A successful command may seal once; a failed command does not advance the checkpoint. + pub fn end_command(&self, commit: bool) -> Result<()> { + let previous = self.command_depth.fetch_sub(1, Ordering::AcqRel); + if previous == 0 { + return Err(MetaDbError::StorageIoError( + "end_command without begin_command".to_string(), + )); + } + if previous == 1 && commit { + self.maintain_memtables()?; } + Ok(()) + } + + /// Installs the Raft pre-flush hook used by size-triggered seals. + pub fn set_pre_flush_hook(&self, hook: Arc) { + *self.flush_hook.write().unwrap() = Some(hook); + } + + /// Records the Raft log identity to stamp on the next size-triggered checkpoint. + pub fn set_flush_identity(&self, identity: FlushIdentity) { + *self.flush_identity.write().unwrap() = Some(identity); + } + + pub fn last_aux_frontier(&self) -> u64 { + self.last_aux_frontier.load(Ordering::Acquire) + } + + /// Rewrites only the durable checkpoint. SSTables are left in place. + pub fn persist_checkpoint(&self, checkpoint: DurableShardCheckpoint) -> Result<()> { + self.backend.persist_checkpoint_only(checkpoint)?; + Ok(()) } pub fn shard_id(&self) -> ShardId { @@ -163,6 +261,7 @@ impl LifecycleStorageEngine { raft_index: LogSequenceNumber, ) { self.mvcc.put(key, value, hlc, raft_index); + self.maintain_after_write(); } /// Applies an externally Raft-logged `Delete` at `(raft_index, hlc)` directly into `MemMvccStore`. @@ -173,6 +272,7 @@ impl LifecycleStorageEngine { raft_index: LogSequenceNumber, ) { self.mvcc.delete(key, hlc, raft_index); + self.maintain_after_write(); } /// Appends a `Put` to the internal `RaftLog` and applies it to `MemMvccStore`. @@ -185,6 +285,7 @@ impl LifecycleStorageEngine { }, ); self.mvcc.put(key, value, hlc, lsn); + self.maintain_memtables().expect("memtable size flush"); lsn } @@ -194,6 +295,7 @@ impl LifecycleStorageEngine { .raft_log .append(hlc, RaftLogPayload::Delete { key: key.clone() }); self.mvcc.delete(key, hlc, lsn); + self.maintain_memtables().expect("memtable size flush"); lsn } @@ -322,6 +424,112 @@ impl LifecycleStorageEngine { Ok(out) } + /// Visits live keys in `[start_key, end_key)` in order and stops when `visit` returns + /// [`ScanControl::Stop`]. Physical tombstones before that point are counted; keys after + /// the stop are not read. + pub fn scan_live_until( + &self, + start_key: &[u8], + end_key: &[u8], + snap: &SnapshotGuard, + mut visit: impl FnMut(&[u8], &[u8]) -> Result, + ) -> Result<()> { + if start_key >= end_key { + return Ok(()); + } + let version = snap.with_step_lease(|pinned| Ok(pinned.version.clone()))?; + let mut disk = if version.has_sst_files() { + DiskScan::Sst(version.lazy_merged_cursor(start_key, Some(end_key))?) + } else { + DiskScan::Inline { + from: start_key.to_vec(), + } + }; + let mut mvcc_from = start_key.to_vec(); + let cut = snap.cut(); + loop { + let pulled = snap.with_step_lease(|pinned| { + self.pull_next(pinned, cut, &mut disk, &mut mvcc_from, end_key) + })?; + match pulled { + Pulled::Done => break, + Pulled::MvccTombstone => { + self.entries_examined.fetch_add(1, Ordering::Relaxed); + } + Pulled::PhysicalTombstone => { + self.entries_examined.fetch_add(1, Ordering::Relaxed); + self.backend.record_scanned_tombstones(1); + } + Pulled::Live(key, value) => { + self.entries_examined.fetch_add(1, Ordering::Relaxed); + if visit(&key, &value)? == ScanControl::Stop { + break; + } + } + } + } + Ok(()) + } + + fn pull_next( + &self, + pinned: &PinnedSnapshotData, + cut: ReadCut, + disk: &mut DiskScan, + mvcc_from: &mut Vec, + end_key: &[u8], + ) -> Result { + let mvcc_done = mvcc_from.as_slice() >= end_key; + let disk_done = disk.is_finished(end_key); + if mvcc_done && disk_done { + return Ok(Pulled::Done); + } + let mvcc_item = if mvcc_done { + None + } else { + self.mvcc + .range_page_with_pinned(mvcc_from, end_key, cut, pinned, 1) + .into_iter() + .next() + }; + let disk_item = if disk_done { + None + } else { + disk.peek(self, pinned, end_key)? + }; + match (mvcc_item, disk_item) { + (None, None) => Ok(Pulled::Done), + (Some((key, value)), None) => { + *mvcc_from = key_successor(&key); + Ok(pulled_from_mvcc(key, value)) + } + (None, Some((key, value))) => { + disk.advance(&key)?; + Ok(pulled_from_disk(key, value)) + } + (Some((mvcc_key, mvcc_value)), Some((disk_key, disk_value))) => { + if mvcc_key < disk_key { + *mvcc_from = key_successor(&mvcc_key); + Ok(pulled_from_mvcc(mvcc_key, mvcc_value)) + } else if disk_key < mvcc_key { + disk.advance(&disk_key)?; + Ok(pulled_from_disk(disk_key, disk_value)) + } else { + *mvcc_from = key_successor(&mvcc_key); + disk.advance(&disk_key)?; + Ok(pulled_from_mvcc(mvcc_key, mvcc_value)) + } + } + } + } + + fn maintain_after_write(&self) { + if self.command_depth.load(Ordering::Acquire) > 0 { + return; + } + self.maintain_memtables().expect("memtable size flush"); + } + /// Point lookup at `read_hlc` consulting `MemMvccStore` first and falling back to persistent storage. pub fn get(&self, key: &[u8], read_hlc: HlcTimestamp) -> Result>> { if let Some(mvcc_val) = self.mvcc.get_at(key, read_hlc)? { @@ -380,9 +588,8 @@ impl LifecycleStorageEngine { checkpoint: DurableShardCheckpoint, hook: Option<&dyn PreFlushHook>, ) -> Result<(LogSequenceNumber, LogSequenceNumber)> { - self.mvcc - .seal_active_memtable(self.backend.as_ref(), checkpoint, hook)?; - self.flush_queued_imms_with_hook(hook) + let _budget = self.budget_mu.lock().unwrap(); + self.seal_and_flush_locked(checkpoint, hook) } /// Flushes already-sealed `ImmMemTable`s that pass the `ActiveSnapshotRegistry` Safe-Flush Gate. @@ -471,7 +678,174 @@ impl LifecycleStorageEngine { persistent_kv_writes: kv_writes, active_physical_tombstones: active_tombs, tombstones_traversed_on_scans: tombs_scanned, + entries_examined_on_scans: self.entries_examined.load(Ordering::Relaxed), compaction_stats: self.compactor.stats(), } } + + fn maintain_memtables(&self) -> Result<()> { + if self.mvcc.active_memtable_bytes() < self.memtable_size_bytes { + return Ok(()); + } + let _budget = self.budget_mu.lock().unwrap(); + if self.mvcc.active_memtable_bytes() < self.memtable_size_bytes { + return Ok(()); + } + let (lsn, hlc, _, _) = self.mvcc.latest_watermarks(); + let checkpoint = self.checkpoint_for_flush(lsn, hlc); + let hook = self.flush_hook.read().unwrap().clone(); + let hook_ref = hook.as_ref().map(|installed| installed.as_ref()); + self.seal_and_flush_locked(checkpoint, hook_ref)?; + Ok(()) + } + + fn seal_and_flush_locked( + &self, + checkpoint: DurableShardCheckpoint, + hook: Option<&dyn PreFlushHook>, + ) -> Result<(LogSequenceNumber, LogSequenceNumber)> { + self.ensure_imm_capacity(hook)?; + self.mvcc + .seal_active_memtable(self.backend.as_ref(), checkpoint, hook)?; + let flushed = self.flush_queued_imms_with_hook(hook)?; + self.last_aux_frontier.store(flushed.1, Ordering::Release); + Ok(flushed) + } + + fn ensure_imm_capacity(&self, hook: Option<&dyn PreFlushHook>) -> Result<()> { + let started = Instant::now(); + let limit = Duration::from_millis(self.mvcc.retention_ms().saturating_add(1_000)); + while self.mvcc.imm_queue_len() >= self.max_imm_memtables { + self.flush_queued_imms_with_hook(hook)?; + if self.mvcc.imm_queue_len() < self.max_imm_memtables { + return Ok(()); + } + if started.elapsed() > limit { + self.mvcc.snapshot_registry().expire_stale_and_min_bounds(); + self.flush_queued_imms_with_hook(hook)?; + if self.mvcc.imm_queue_len() >= self.max_imm_memtables { + return Err(MetaDbError::StorageIoError(format!( + "immutable memtable cap {} is pinned past the MVCC retention window", + self.max_imm_memtables + ))); + } + return Ok(()); + } + self.mvcc + .snapshot_registry() + .wait_for_flush_progress(Duration::from_millis(20)); + } + Ok(()) + } + + fn checkpoint_for_flush(&self, lsn: u64, hlc: HlcTimestamp) -> DurableShardCheckpoint { + let current = self.durable_checkpoint(); + match self.flush_identity.read().unwrap().clone() { + Some(identity) => DurableShardCheckpoint { + flushed_raft_term: identity.term, + flushed_raft_leader: identity.leader, + flushed_raft_index: lsn, + flushed_max_hlc: hlc, + contiguous_aux_acked_index: identity.aux_index, + membership_bytes: identity.membership, + descriptor_bytes: identity.descriptor, + }, + None => DurableShardCheckpoint { + flushed_raft_index: lsn, + flushed_max_hlc: hlc, + ..current + }, + } + } +} + +enum DiskScan { + Sst(MergedCursor), + Inline { from: Vec }, +} + +impl DiskScan { + fn is_finished(&self, end_key: &[u8]) -> bool { + match self { + DiskScan::Sst(_) => false, + DiskScan::Inline { from } => from.as_slice() >= end_key, + } + } + + fn peek( + &mut self, + engine: &LifecycleStorageEngine, + pinned: &PinnedSnapshotData, + end_key: &[u8], + ) -> Result, PersistentEntry)>> { + match self { + DiskScan::Sst(cursor) => Ok(cursor + .peek()? + .map(|(key, value)| (key, PersistentEntry::from(value)))), + DiskScan::Inline { from } => { + if from.as_slice() >= end_key { + return Ok(None); + } + let (page, _) = + engine + .backend + .scan_in_version_page(&pinned.version, from, end_key, 1)?; + Ok(page.into_iter().next()) + } + } + } + + fn advance(&mut self, key: &[u8]) -> Result<()> { + match self { + DiskScan::Sst(cursor) => cursor.advance(), + DiskScan::Inline { from } => { + *from = key_successor(key); + Ok(()) + } + } + } +} + +enum Pulled { + Done, + MvccTombstone, + PhysicalTombstone, + Live(Vec, Vec), +} + +fn pulled_from_mvcc(key: Vec, value: MvccValue) -> Pulled { + match value { + MvccValue::Value(bytes) => Pulled::Live(key, bytes), + MvccValue::Tombstone => Pulled::MvccTombstone, + } +} + +fn pulled_from_disk(key: Vec, value: PersistentEntry) -> Pulled { + match value { + PersistentEntry::Value(bytes) => Pulled::Live(key, bytes), + PersistentEntry::PhysicalTombstone => Pulled::PhysicalTombstone, + } +} + +fn key_successor(key: &[u8]) -> Vec { + let mut next = key.to_vec(); + next.push(0x00); + next +} + +fn rebuild_density(compactor: &LocalityCompactor, version: &Version) -> Result<()> { + let mut cursor = version.merged_cursor(&[], None)?; + let mut counts: BTreeMap = BTreeMap::new(); + while let Some((key, value)) = cursor.next_owned()? { + let Some(bucket) = LocalityCompactor::bucket_for_key(&key) else { + continue; + }; + let slot = counts.entry(bucket).or_insert((0, 0)); + match value { + BlockEntryValue::PhysicalTombstone => slot.0 += 1, + BlockEntryValue::Value(_) => slot.1 += 1, + } + } + compactor.restore_counts(counts); + Ok(()) } diff --git a/crates/metadb-storage/src/lib.rs b/crates/metadb-storage/src/lib.rs index 968c8f7..9c72b3f 100644 --- a/crates/metadb-storage/src/lib.rs +++ b/crates/metadb-storage/src/lib.rs @@ -26,7 +26,9 @@ pub mod version_set; pub use backend::{InMemoryBackend, LsmStorageBackend, PersistentEntry, StorageBackend}; pub use cache::ShardedBlockCache; pub use compaction::{CompactionStats, LocalityCompactor}; -pub use engine::{LifecycleStorageEngine, LsmEngineConfig, StorageMetrics}; +pub use engine::{ + FlushIdentity, LifecycleStorageEngine, LsmEngineConfig, ScanControl, StorageMetrics, +}; pub use manifest::{DurableShardCheckpoint, ManifestLog, VersionEdit}; pub use mvcc::{ ActiveMemTable, ActiveSnapshotRegistry, ImmMemTable, MemMvccStore, MvccValue, MvccVersion, diff --git a/crates/metadb-storage/src/mvcc.rs b/crates/metadb-storage/src/mvcc.rs index e8c068f..41a87f9 100644 --- a/crates/metadb-storage/src/mvcc.rs +++ b/crates/metadb-storage/src/mvcc.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, VecDeque}; use std::ops::Bound; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Condvar, Mutex, RwLock}; use std::time::{Duration, Instant}; use metadb_core::{HlcTimestamp, MetaDbError, Result}; @@ -168,6 +168,8 @@ pub struct ActiveSnapshotRegistry { next_id: AtomicU64, retention: Duration, slots: Mutex>>, + flush_mu: Mutex<()>, + flush_cv: Condvar, } impl ActiveSnapshotRegistry { @@ -176,6 +178,8 @@ impl ActiveSnapshotRegistry { next_id: AtomicU64::new(1), retention: Duration::from_millis(retention_ms.max(1)), slots: Mutex::new(BTreeMap::new()), + flush_mu: Mutex::new(()), + flush_cv: Condvar::new(), } } @@ -198,10 +202,23 @@ impl ActiveSnapshotRegistry { slot } + /// Wakes writers blocked on the immutable-memtable cap. + pub fn notify_flush_waiters(&self) { + let _guard = self.flush_mu.lock().unwrap(); + self.flush_cv.notify_all(); + } + + /// Parks until a snapshot is dropped or `timeout` elapses. + pub fn wait_for_flush_progress(&self, timeout: Duration) { + let guard = self.flush_mu.lock().unwrap(); + let _ = self.flush_cv.wait_timeout(guard, timeout); + } + pub fn unregister(&self, id: u64) { let removed = self.slots.lock().unwrap().remove(&id); if let Some(slot) = removed { *slot.state.write().unwrap() = None; + self.notify_flush_waiters(); } } @@ -231,9 +248,14 @@ impl ActiveSnapshotRegistry { }); } } + let expired_any = !expired_ids.is_empty(); for id in expired_ids { guard.remove(&id); } + drop(guard); + if expired_any { + self.notify_flush_waiters(); + } min_hlc.zip(min_flushed) } @@ -316,6 +338,7 @@ struct MemMvccInner { last_flushed_lsn: LogSequenceNumber, total_mvcc_writes: u64, total_pruned_versions: u64, + active_bytes: usize, } /// In-Memory MVCC store coordinating active version chains, `ImmMemTable` queue, @@ -351,6 +374,7 @@ impl MemMvccStore { last_flushed_lsn: flushed_lsn, total_mvcc_writes: 0, total_pruned_versions: 0, + active_bytes: 0, }), flush_mutex: std::sync::Mutex::new(()), } @@ -361,10 +385,12 @@ impl MemMvccStore { } pub fn put(&self, key: Vec, value: Vec, hlc: HlcTimestamp, lsn: LogSequenceNumber) { + let accounted = key.len().saturating_add(value.len()).max(1); let mut guard = self.inner.write().unwrap(); guard.latest_hlc = guard.latest_hlc.max(hlc); guard.latest_lsn = guard.latest_lsn.max(lsn); guard.total_mvcc_writes += 1; + guard.active_bytes = guard.active_bytes.saturating_add(accounted); guard.hlc_to_lsn.push_back((hlc, lsn)); guard .active_versions @@ -380,10 +406,12 @@ impl MemMvccStore { } pub fn delete(&self, key: Vec, hlc: HlcTimestamp, lsn: LogSequenceNumber) { + let accounted = key.len().max(1); let mut guard = self.inner.write().unwrap(); guard.latest_hlc = guard.latest_hlc.max(hlc); guard.latest_lsn = guard.latest_lsn.max(lsn); guard.total_mvcc_writes += 1; + guard.active_bytes = guard.active_bytes.saturating_add(accounted); guard.hlc_to_lsn.push_back((hlc, lsn)); guard .active_versions @@ -398,6 +426,14 @@ impl MemMvccStore { }); } + pub fn active_memtable_bytes(&self) -> usize { + self.inner.read().unwrap().active_bytes + } + + pub fn retention_ms(&self) -> u64 { + self.retention_ms + } + pub fn acquire_linearizable_snapshot( &self, min_read_index: LogSequenceNumber, @@ -701,10 +737,11 @@ impl MemMvccStore { } } - if guard.active_versions.read().unwrap().is_empty() - && sealed_index == guard.last_flushed_lsn - { - return Ok(None); + if guard.active_versions.read().unwrap().is_empty() { + guard.active_bytes = 0; + if sealed_index == guard.last_flushed_lsn { + return Ok(None); + } } let seal_id = guard.next_seal_id; @@ -713,6 +750,7 @@ impl MemMvccStore { &mut guard.active_versions, Arc::new(RwLock::new(BTreeMap::new())), ); + guard.active_bytes = 0; let entries: BTreeMap, Vec> = old_active .read() .unwrap() @@ -801,6 +839,7 @@ impl MemMvccStore { .unwrap_or(false) { guard.imm_queue.pop_front(); + self.registry.notify_flush_waiters(); } guard.disk_flushed_max_hlc = guard.disk_flushed_max_hlc.max(candidate.max_hlc); guard.last_flushed_lsn = guard.last_flushed_lsn.max(candidate.max_raft_index); @@ -858,6 +897,7 @@ impl MemMvccStore { g.active_versions = Arc::new(RwLock::new(BTreeMap::new())); g.hlc_to_lsn.clear(); g.imm_queue.clear(); + g.active_bytes = 0; g.latest_lsn = snapshot_lsn; g.latest_hlc = snapshot_hlc; g.last_flushed_lsn = snapshot_lsn; diff --git a/crates/metadb-storage/src/sstable.rs b/crates/metadb-storage/src/sstable.rs index 1679d38..4b96c8d 100644 --- a/crates/metadb-storage/src/sstable.rs +++ b/crates/metadb-storage/src/sstable.rs @@ -1,5 +1,6 @@ //! Immutable SSTable builder and reader (`.sst`) with 64-byte magic/CRC32C footer. +use std::cell::Cell; use std::fs::{self, File, OpenOptions}; use std::io::Write; use std::os::unix::fs::FileExt; @@ -7,6 +8,35 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +thread_local! { + static FULL_RANGE_MATERIALIZATIONS: Cell = const { Cell::new(0) }; + static DATA_BLOCKS_TOUCHED: Cell = const { Cell::new(0) }; +} + +/// Test probe: how many times `read_all_entries` ran on this thread. +pub fn full_range_materializations() -> u64 { + FULL_RANGE_MATERIALIZATIONS.with(|counter| counter.get()) +} + +/// Test probe: how many data blocks this thread has loaded or cache-hit. +pub fn data_blocks_touched() -> u64 { + DATA_BLOCKS_TOUCHED.with(|counter| counter.get()) +} + +/// Clears the thread-local compaction and scan probes. +pub fn reset_scan_probes() { + FULL_RANGE_MATERIALIZATIONS.with(|counter| counter.set(0)); + DATA_BLOCKS_TOUCHED.with(|counter| counter.set(0)); +} + +fn note_full_range_materialization() { + FULL_RANGE_MATERIALIZATIONS.with(|counter| counter.set(counter.get().saturating_add(1))); +} + +fn note_data_block_touch() { + DATA_BLOCKS_TOUCHED.with(|counter| counter.set(counter.get().saturating_add(1))); +} + use serde::{Deserialize, Serialize}; use metadb_core::{ @@ -475,8 +505,11 @@ impl SstableReader { Ok(out) } - /// Reads every entry across all data blocks in this SSTable (used during compaction). + /// Reads every entry across all data blocks in this SSTable. + /// + /// Compaction must not call this: it materializes the whole file before merge. pub fn read_all_entries(&self) -> Result, BlockEntryValue)>> { + note_full_range_materialization(); let mut out = Vec::new(); for ie in &self.index { let block = self.load_data_block(ie)?; @@ -531,6 +564,7 @@ impl SstableReader { } fn load_data_block(&self, ie: &IndexBlockEntry) -> Result> { + note_data_block_touch(); let cache_key = BlockCacheKey { shard_id: self.shard_id, sst_file_id: self.meta.file_id, @@ -666,3 +700,108 @@ impl SstableReader { }) } } + +/// Forward cursor over one SSTable. Decodes a single entry at a time and loads +/// the next data block only when the current block is exhausted. +pub(crate) struct SstCursor { + reader: Arc, + start_key: Vec, + end_key: Option>, + block_idx: usize, + block: Option>, + offset: usize, + key: Vec, + pending: Option<(Vec, BlockEntryValue)>, + needs_fill: bool, + exhausted: bool, +} + +impl SstCursor { + pub(crate) fn open( + reader: Arc, + start_key: &[u8], + end_key: Option<&[u8]>, + ) -> Result { + let block_idx = reader + .index + .partition_point(|entry| entry.last_key.as_slice() < start_key); + Ok(Self { + reader, + start_key: start_key.to_vec(), + end_key: end_key.map(|key| key.to_vec()), + block_idx, + block: None, + offset: 0, + key: Vec::new(), + pending: None, + needs_fill: true, + exhausted: false, + }) + } + + pub(crate) fn peek(&mut self) -> Result, BlockEntryValue)>> { + if self.needs_fill { + self.fill_one()?; + self.needs_fill = false; + } + Ok(self.pending.clone()) + } + + /// Drops the current entry without decoding the successor. + pub(crate) fn advance(&mut self) { + self.pending = None; + self.needs_fill = !self.exhausted; + } + + fn fill_one(&mut self) -> Result<()> { + if self.exhausted { + self.pending = None; + return Ok(()); + } + loop { + if self.block.is_none() { + if self.block_idx >= self.reader.index.len() { + self.exhausted = true; + self.pending = None; + return Ok(()); + } + let index_entry = &self.reader.index[self.block_idx]; + if index_entry.last_key.as_slice() < self.start_key.as_slice() { + self.block_idx += 1; + continue; + } + let block = self.reader.load_data_block(index_entry)?; + self.offset = block.restart_offset_for(&self.start_key)?; + self.key.clear(); + self.block = Some(block); + } + let block = self.block.as_ref().unwrap().clone(); + match block.read_entry(&mut self.offset, &mut self.key)? { + None => { + self.block = None; + self.block_idx += 1; + self.key.clear(); + self.offset = 0; + continue; + } + Some(value) => { + if self.key.as_slice() < self.start_key.as_slice() { + continue; + } + if self + .end_key + .as_ref() + .is_some_and(|end| self.key.as_slice() >= end.as_slice()) + { + self.exhausted = true; + self.pending = None; + self.block = None; + return Ok(()); + } + self.pending = Some((self.key.clone(), value)); + return Ok(()); + } + } + } + } +} diff --git a/crates/metadb-storage/src/version_set.rs b/crates/metadb-storage/src/version_set.rs index d7aabc7..e61a578 100644 --- a/crates/metadb-storage/src/version_set.rs +++ b/crates/metadb-storage/src/version_set.rs @@ -1,13 +1,14 @@ //! Immutable `Version` views and deterministic `MergingIterator` across `L0..L3` SSTables. -use std::collections::BTreeMap; +use std::cmp::Ordering; +use std::collections::{BTreeMap, BinaryHeap}; use std::sync::Arc; use metadb_core::Result; use crate::block::BlockEntryValue; use crate::manifest::DurableShardCheckpoint; -use crate::sstable::{SstFileMeta, SstableReader}; +use crate::sstable::{SstCursor, SstFileMeta, SstableReader}; pub const NUM_LSM_LEVELS: usize = 4; @@ -199,4 +200,302 @@ impl Version { } samples } + + /// Streaming merge of the newest value of each key in `[start, end)`. + /// `end == None` reads through the last key of every selected SSTable. + pub(crate) fn merged_cursor(&self, start: &[u8], end: Option<&[u8]>) -> Result { + let mut readers = Vec::new(); + for sst in &self.levels[0] { + readers.push(sst.clone()); + } + for level in &self.levels[1..] { + let first = if start.is_empty() { + 0 + } else { + level.partition_point(|sst| sst.meta().max_key.as_slice() < start) + }; + for sst in &level[first..] { + if end.is_some_and(|bound| sst.meta().min_key.as_slice() >= bound) { + break; + } + readers.push(sst.clone()); + } + } + MergedCursor::open(readers, start, end) + } + + /// Like [`Self::merged_cursor`], but opens only the first overlapping file of each + /// non-overlapping level. The next file is opened when the current one is exhausted. + pub(crate) fn lazy_merged_cursor( + &self, + start: &[u8], + end: Option<&[u8]>, + ) -> Result { + let l0 = self.levels[0].clone(); + let mut later = Vec::with_capacity(NUM_LSM_LEVELS - 1); + for level in &self.levels[1..] { + let first = if start.is_empty() { + 0 + } else { + level.partition_point(|sst| sst.meta().max_key.as_slice() < start) + }; + let mut files = Vec::new(); + for sst in &level[first..] { + if end.is_some_and(|bound| sst.meta().min_key.as_slice() >= bound) { + break; + } + files.push(sst.clone()); + } + later.push(files); + } + MergedCursor::open_layered(l0, later, start, end) + } + + pub(crate) fn has_sst_files(&self) -> bool { + self.levels.iter().any(|level| !level.is_empty()) + } +} + +/// K-way merge. Earlier readers win on equal keys (`L0` newest-first, then `L1..L3`). +pub(crate) struct MergedCursor { + sources: Vec, + pending_levels: Vec, + reheap_ids: Vec, + heap: BinaryHeap, + staged: Option, + exhausted: bool, +} + +struct MergeSource { + cursor: SstCursor, + precedence: u32, + generation: u64, + /// Index into `pending_levels` when this source walks a non-overlapping level. + level_slot: Option, +} + +struct PendingLevel { + files: Vec>, + next: usize, + start: Vec, + end: Option>, +} + +struct StagedEntry { + key: Vec, + value: BlockEntryValue, + source_ids: Vec, +} + +#[derive(Clone, Eq, PartialEq)] +struct HeapNode { + key: Vec, + precedence: u32, + cursor_id: u32, + generation: u64, +} + +impl PartialOrd for HeapNode { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for HeapNode { + fn cmp(&self, other: &Self) -> Ordering { + // `BinaryHeap` pops the greatest node. Invert key order so the smallest key wins. + other + .key + .cmp(&self.key) + .then_with(|| other.precedence.cmp(&self.precedence)) + .then_with(|| self.cursor_id.cmp(&other.cursor_id)) + .then_with(|| self.generation.cmp(&other.generation)) + } +} + +impl MergedCursor { + pub(crate) fn open( + readers: Vec>, + start: &[u8], + end: Option<&[u8]>, + ) -> Result { + let mut sources = Vec::with_capacity(readers.len()); + let heap = BinaryHeap::new(); + for (index, reader) in readers.into_iter().enumerate() { + let cursor = SstCursor::open(reader, start, end)?; + let cursor_id = index as u32; + sources.push(MergeSource { + cursor, + precedence: cursor_id, + generation: 0, + level_slot: None, + }); + } + let mut cursor = Self { + sources, + pending_levels: Vec::new(), + reheap_ids: Vec::new(), + heap, + staged: None, + exhausted: false, + }; + for id in 0..cursor.sources.len() { + cursor.reheap_one(id as u32)?; + } + Ok(cursor) + } + + /// `L0` readers are opened immediately because they overlap. Each later level + /// opens its first overlapping file only; the rest wait until that file ends. + pub(crate) fn open_layered( + l0: Vec>, + levels: Vec>>, + start: &[u8], + end: Option<&[u8]>, + ) -> Result { + let mut cursor = Self::open(l0, start, end)?; + for files in levels { + if files.is_empty() { + continue; + } + let level_slot = cursor.pending_levels.len(); + let precedence = ((level_slot as u32) + 1) << 20; + cursor.pending_levels.push(PendingLevel { + files, + next: 1, + start: start.to_vec(), + end: end.map(|key| key.to_vec()), + }); + let reader = cursor.pending_levels[level_slot].files[0].clone(); + let source_id = cursor.sources.len() as u32; + cursor.sources.push(MergeSource { + cursor: SstCursor::open(reader, start, end)?, + precedence, + generation: 0, + level_slot: Some(level_slot), + }); + cursor.reheap_one(source_id)?; + } + Ok(cursor) + } + + pub(crate) fn peek(&mut self) -> Result, BlockEntryValue)>> { + self.ensure_staged()?; + Ok(self + .staged + .as_ref() + .map(|entry| (entry.key.clone(), entry.value.clone()))) + } + + /// Drops the current winner. The successor is not decoded until the next peek. + pub(crate) fn advance(&mut self) -> Result<()> { + self.ensure_staged()?; + let Some(staged) = self.staged.take() else { + return Ok(()); + }; + for source_id in staged.source_ids { + self.sources[source_id as usize].cursor.advance(); + self.sources[source_id as usize].generation = + self.sources[source_id as usize].generation.wrapping_add(1); + self.reheap_ids.push(source_id); + } + Ok(()) + } + + pub(crate) fn next_owned(&mut self) -> Result, BlockEntryValue)>> { + let Some((key, value)) = self.peek()? else { + return Ok(None); + }; + self.advance()?; + Ok(Some((key, value))) + } + + fn ensure_staged(&mut self) -> Result<()> { + if self.staged.is_some() || self.exhausted { + return Ok(()); + } + self.reheap_pending()?; + loop { + let Some(node) = self.heap.pop() else { + self.exhausted = true; + return Ok(()); + }; + let cursor_id = node.cursor_id as usize; + if self.sources[cursor_id].generation != node.generation { + continue; + } + let Some((key, value)) = self.sources[cursor_id].cursor.peek()? else { + continue; + }; + if key != node.key { + continue; + } + let mut best_precedence = node.precedence; + let mut best_value = value; + let mut source_ids = vec![node.cursor_id]; + while self.heap.peek().is_some_and(|next| next.key == node.key) { + let other = self.heap.pop().unwrap(); + let other_index = other.cursor_id as usize; + if self.sources[other_index].generation != other.generation { + continue; + } + let Some((other_key, other_value)) = self.sources[other_index].cursor.peek()? + else { + continue; + }; + if other_key != node.key { + continue; + } + source_ids.push(other.cursor_id); + if other.precedence < best_precedence { + best_precedence = other.precedence; + best_value = other_value; + } + } + self.staged = Some(StagedEntry { + key: node.key, + value: best_value, + source_ids, + }); + return Ok(()); + } + } + + fn reheap_pending(&mut self) -> Result<()> { + let ids = std::mem::take(&mut self.reheap_ids); + for id in ids { + self.reheap_one(id)?; + } + Ok(()) + } + + fn reheap_one(&mut self, id: u32) -> Result<()> { + loop { + if let Some((key, _)) = self.sources[id as usize].cursor.peek()? { + let generation = self.sources[id as usize].generation; + let precedence = self.sources[id as usize].precedence; + self.heap.push(HeapNode { + key, + precedence, + cursor_id: id, + generation, + }); + return Ok(()); + } + let Some(level_slot) = self.sources[id as usize].level_slot else { + return Ok(()); + }; + if self.pending_levels[level_slot].next >= self.pending_levels[level_slot].files.len() { + return Ok(()); + } + let reader = + self.pending_levels[level_slot].files[self.pending_levels[level_slot].next].clone(); + self.pending_levels[level_slot].next += 1; + let start = self.pending_levels[level_slot].start.clone(); + let end = self.pending_levels[level_slot].end.clone(); + self.sources[id as usize].generation = + self.sources[id as usize].generation.wrapping_add(1); + self.sources[id as usize].cursor = SstCursor::open(reader, &start, end.as_deref())?; + } + } } diff --git a/crates/metadb-storage/tests/first_ship_storage_regressions.rs b/crates/metadb-storage/tests/first_ship_storage_regressions.rs new file mode 100644 index 0000000..43fe203 --- /dev/null +++ b/crates/metadb-storage/tests/first_ship_storage_regressions.rs @@ -0,0 +1,422 @@ +//! Regressions for the first-ship storage and listdir failures on main `f325f6a`. + +use std::sync::mpsc; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +use metadb_core::{HlcTimestamp, MetaKey}; +use metadb_storage::block::BlockEntryValue; +use metadb_storage::sstable::{ + data_blocks_touched, full_range_materializations, reset_scan_probes, +}; +use metadb_storage::{ + DurableShardCheckpoint, LifecycleStorageEngine, LsmEngineConfig, ManifestLog, ScanControl, + ShardedBlockCache, SstableBuilder, VersionEdit, +}; +use tempfile::tempdir; + +fn engine(dir: &std::path::Path, config: LsmEngineConfig) -> LifecycleStorageEngine { + let cache = Arc::new(ShardedBlockCache::new(8 * 1024 * 1024)); + LifecycleStorageEngine::open_lsm(1, dir, cache, config).unwrap() +} + +#[test] +fn listdir_limit_does_not_walk_the_remaining_tombstone_range() { + let dir = tempdir().unwrap(); + let lsm = dir.path().join("lsm"); + let engine = engine( + &lsm, + LsmEngineConfig { + auto_locality_compaction: false, + tombstone_density_threshold: 100_000, + memtable_size_bytes: 32 * 1024 * 1024, + ..LsmEngineConfig::default() + }, + ); + + let live = MetaKey::child(7, "a").encode(); + engine.apply_committed_put( + live.clone(), + b"live".to_vec(), + HlcTimestamp::new(1_000, 1), + 1, + ); + for index in 0..2048u64 { + let key = MetaKey::child(7, format!("t{index:04}")).encode(); + engine.apply_committed_delete(key, HlcTimestamp::new(1_000, 2), index + 2); + } + engine.flush_mvcc_to_persistent(); + + let version = engine.backend().current_version(); + let blocks: usize = version + .levels + .iter() + .flat_map(|level| level.iter()) + .map(|sst| sst.index_entries().len()) + .sum(); + assert!( + blocks > 4, + "the fixture must span multiple data blocks, got {blocks}" + ); + + reset_scan_probes(); + let before = engine.metrics(); + let snap = engine.acquire_linearizable_snapshot(0).unwrap(); + let mut rows = Vec::new(); + engine + .scan_live_until( + &MetaKey::encode_pid_prefix_start(7), + &MetaKey::encode_pid_prefix_end(7), + &snap, + |key, value| { + rows.push((key.to_vec(), value.to_vec())); + Ok(ScanControl::Stop) + }, + ) + .unwrap(); + drop(snap); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].0, live); + let after = engine.metrics(); + assert_eq!( + after.tombstones_traversed_on_scans - before.tombstones_traversed_on_scans, + 0, + "limit=1 must not count the 2048 tombstones that sort after the live row" + ); + assert_eq!( + after.entries_examined_on_scans - before.entries_examined_on_scans, + 1 + ); + let touched = data_blocks_touched(); + assert!( + touched < blocks as u64, + "limit=1 touched {touched} data blocks of {blocks}" + ); +} + +#[test] +fn locality_density_survives_reopen() { + let dir = tempdir().unwrap(); + let lsm = dir.path().join("lsm"); + let config = LsmEngineConfig { + auto_locality_compaction: true, + tombstone_density_threshold: 10_000, + ..LsmEngineConfig::default() + }; + let sample = MetaKey::child(7, "a").encode(); + { + let engine = engine(&lsm, config.clone()); + for index in 0..100u64 { + let key = MetaKey::child(7, format!("n{index:03}")).encode(); + engine.apply_committed_put( + key, + b"v".to_vec(), + HlcTimestamp::new(2_000, index as u16), + index + 1, + ); + } + engine.flush_mvcc_to_persistent(); + for index in 0..20u64 { + let key = MetaKey::child(7, format!("n{index:03}")).encode(); + engine.apply_committed_delete(key, HlcTimestamp::new(3_000, index as u16), 200 + index); + } + engine.flush_mvcc_to_persistent(); + assert_eq!(engine.compactor().snapshot_bucket_counts(&sample), (20, 80)); + } + + let cache = Arc::new(ShardedBlockCache::new(8 * 1024 * 1024)); + let reopened = LifecycleStorageEngine::open_lsm_existing(1, &lsm, cache, config).unwrap(); + assert_eq!( + reopened.compactor().snapshot_bucket_counts(&sample), + (20, 80), + "density must be rebuilt from SSTables on open" + ); +} + +#[test] +fn compact_range_does_not_materialize_whole_ssts() { + let dir = tempdir().unwrap(); + let lsm = dir.path().join("lsm"); + let engine = engine( + &lsm, + LsmEngineConfig { + auto_locality_compaction: false, + ..LsmEngineConfig::default() + }, + ); + let mut live = Vec::new(); + for index in 0..400u64 { + let key = MetaKey::child(9, format!("k{index:04}")).encode(); + engine.apply_committed_put( + key.clone(), + vec![index as u8], + HlcTimestamp::new(4_000, index as u16), + index + 1, + ); + if index % 2 == 0 { + live.push(key); + } + } + engine.flush_mvcc_to_persistent(); + for index in 0..400u64 { + if index % 2 == 1 { + let key = MetaKey::child(9, format!("k{index:04}")).encode(); + engine.apply_committed_delete( + key, + HlcTimestamp::new(5_000, index as u16), + 1_000 + index, + ); + } + } + engine.flush_mvcc_to_persistent(); + + reset_scan_probes(); + let purged = engine.compact_range( + &MetaKey::encode_pid_prefix_start(9), + &MetaKey::encode_pid_prefix_end(9), + ); + assert!(purged >= 200, "purged {purged}"); + assert_eq!( + full_range_materializations(), + 0, + "compact_range must not call read_all_entries" + ); + + let snap = engine.acquire_linearizable_snapshot(0).unwrap(); + for key in &live { + assert!(engine.get_with_snapshot(key, &snap).unwrap().is_some()); + } + let odd = MetaKey::child(9, "k0001").encode(); + assert!(engine.get_with_snapshot(&odd, &snap).unwrap().is_none()); +} + +#[test] +fn tiny_memtable_flushes_and_immutable_tables_respect_the_cap() { + let dir = tempdir().unwrap(); + let lsm = dir.path().join("lsm"); + let engine = Arc::new(engine( + &lsm, + LsmEngineConfig { + auto_locality_compaction: false, + memtable_size_bytes: 1, + max_imm_memtables: 4, + ..LsmEngineConfig::default() + }, + )); + + engine.apply_committed_put( + b"first".to_vec(), + b"v".to_vec(), + HlcTimestamp::new(6_000, 1), + 1, + ); + assert!( + engine.durable_checkpoint().flushed_raft_index >= 1, + "a 1-byte memtable threshold must flush" + ); + assert_eq!(engine.mvcc().imm_queue_len(), 0); + + let snap = engine.acquire_linearizable_snapshot(1).unwrap(); + for index in 0..4u64 { + engine.apply_committed_put( + format!("held-{index}").into_bytes(), + b"x".to_vec(), + HlcTimestamp::new(7_000, index as u16), + 10 + index, + ); + assert!(engine.mvcc().imm_queue_len() <= 4); + } + assert_eq!(engine.mvcc().imm_queue_len(), 4); + + let blocked = Arc::clone(&engine); + let (started_tx, started_rx) = mpsc::channel(); + let handle = thread::spawn(move || { + started_tx.send(()).unwrap(); + blocked.apply_committed_put( + b"fifth".to_vec(), + b"y".to_vec(), + HlcTimestamp::new(8_000, 1), + 20, + ); + }); + started_rx.recv().unwrap(); + thread::sleep(Duration::from_millis(150)); + assert!( + !handle.is_finished(), + "the fifth immutable memtable must wait for backpressure" + ); + assert_eq!(engine.mvcc().imm_queue_len(), 4); + drop(snap); + handle.join().unwrap(); + assert!(engine.mvcc().imm_queue_len() <= 4); + + drop(engine); + let cache = Arc::new(ShardedBlockCache::new(1024 * 1024)); + let reopened = LifecycleStorageEngine::open_lsm( + 1, + &lsm, + cache, + LsmEngineConfig { + memtable_size_bytes: 32 * 1024 * 1024, + max_imm_memtables: 4, + auto_locality_compaction: false, + ..LsmEngineConfig::default() + }, + ) + .unwrap(); + let snap = reopened.acquire_linearizable_snapshot(0).unwrap(); + assert_eq!( + reopened.get_with_snapshot(b"first", &snap).unwrap(), + Some(b"v".to_vec()) + ); + assert_eq!( + reopened.get_with_snapshot(b"fifth", &snap).unwrap(), + Some(b"y".to_vec()) + ); +} + +#[test] +fn size_flush_waits_until_the_command_finishes() { + let dir = tempdir().unwrap(); + let lsm = dir.path().join("lsm"); + let engine = engine( + &lsm, + LsmEngineConfig { + auto_locality_compaction: false, + memtable_size_bytes: 1, + ..LsmEngineConfig::default() + }, + ); + engine.begin_command(); + engine.apply_committed_put( + b"child".to_vec(), + b"a".to_vec(), + HlcTimestamp::new(9_000, 1), + 7, + ); + engine.apply_committed_put( + b"attr".to_vec(), + b"b".to_vec(), + HlcTimestamp::new(9_000, 2), + 7, + ); + assert_eq!( + engine.durable_checkpoint().flushed_raft_index, + 0, + "a threshold crossing must not checkpoint a partial command" + ); + engine.end_command(false).unwrap(); + assert_eq!(engine.durable_checkpoint().flushed_raft_index, 0); + + engine.begin_command(); + engine.apply_committed_put( + b"child".to_vec(), + b"a".to_vec(), + HlcTimestamp::new(9_000, 1), + 7, + ); + engine.apply_committed_put( + b"attr".to_vec(), + b"b".to_vec(), + HlcTimestamp::new(9_000, 2), + 7, + ); + engine.end_command(true).unwrap(); + assert!(engine.durable_checkpoint().flushed_raft_index >= 7); + + drop(engine); + let cache = Arc::new(ShardedBlockCache::new(1024 * 1024)); + let reopened = LifecycleStorageEngine::open_lsm( + 1, + &lsm, + cache, + LsmEngineConfig { + memtable_size_bytes: 32 * 1024 * 1024, + auto_locality_compaction: false, + ..LsmEngineConfig::default() + }, + ) + .unwrap(); + let snap = reopened.acquire_linearizable_snapshot(0).unwrap(); + assert_eq!( + reopened.get_with_snapshot(b"child", &snap).unwrap(), + Some(b"a".to_vec()) + ); + assert_eq!( + reopened.get_with_snapshot(b"attr", &snap).unwrap(), + Some(b"b".to_vec()) + ); +} + +#[test] +fn limited_scan_does_not_open_later_nonoverlapping_files() { + let dir = tempdir().unwrap(); + let lsm = dir.path().join("lsm"); + std::fs::create_dir_all(&lsm).unwrap(); + let live = MetaKey::child(3, "a").encode(); + let mut later = Vec::new(); + for index in 0..2048u32 { + later.push(( + MetaKey::child(3, format!("t{index:04}")).encode(), + BlockEntryValue::PhysicalTombstone, + )); + } + { + let (mut manifest, _) = ManifestLog::open_or_create(&lsm).unwrap(); + let first = SstableBuilder::new(1, 10, 1, 1, HlcTimestamp::new(1, 0)) + .write_to_dir( + &lsm, + &[(live.clone(), BlockEntryValue::Value(b"live".to_vec()))], + ) + .unwrap(); + let second = SstableBuilder::new(1, 11, 2, 2, HlcTimestamp::new(2, 0)) + .write_to_dir(&lsm, &later) + .unwrap(); + manifest + .append(&VersionEdit { + next_file_id: Some(12), + next_flush_generation: Some(3), + checkpoint: Some(DurableShardCheckpoint::default()), + deleted_files: vec![], + added_files: vec![first, second], + }) + .unwrap(); + } + + let cache = Arc::new(ShardedBlockCache::new(8 * 1024 * 1024)); + let engine = LifecycleStorageEngine::open_lsm( + 1, + &lsm, + cache, + LsmEngineConfig { + auto_locality_compaction: false, + tombstone_density_threshold: 100_000, + ..LsmEngineConfig::default() + }, + ) + .unwrap(); + let version = engine.backend().current_version(); + let first_blocks = version.levels[1][0].index_entries().len() as u64; + let later_blocks = version.levels[1][1].index_entries().len() as u64; + assert!(later_blocks > first_blocks); + + reset_scan_probes(); + let snap = engine.acquire_linearizable_snapshot(0).unwrap(); + let mut rows = Vec::new(); + engine + .scan_live_until( + &MetaKey::encode_pid_prefix_start(3), + &MetaKey::encode_pid_prefix_end(3), + &snap, + |key, value| { + rows.push((key.to_vec(), value.to_vec())); + Ok(ScanControl::Stop) + }, + ) + .unwrap(); + assert_eq!(rows, vec![(live, b"live".to_vec())]); + assert_eq!(data_blocks_touched(), first_blocks); + assert_eq!(engine.metrics().tombstones_traversed_on_scans, 0); +} diff --git a/crates/metadb-tx/src/aux_outbox.rs b/crates/metadb-tx/src/aux_outbox.rs index 42f0fce..a874d65 100644 --- a/crates/metadb-tx/src/aux_outbox.rs +++ b/crates/metadb-tx/src/aux_outbox.rs @@ -535,26 +535,32 @@ impl ShardAuxOutbox { acked_keys: &[(u64, u16)], engine: &LifecycleStorageEngine, ) -> u64 { - let mut guard = self.inner.write().unwrap(); - guard.max_observed_raft_index = guard.max_observed_raft_index.max(raft_index); - - for &(idx, ord) in acked_keys { - if let Some(removed) = guard.tasks.remove(&(idx, ord)) { - if removed.durable_spilled || removed.sealed_at_index.is_some() { - let spill_key = SpilledAuxOutboxKey::new(idx, ord).encode().to_vec(); - engine.apply_committed_delete(spill_key, hlc, raft_index); + // Collect deletions under the outbox lock, then write them after releasing it. + // A size-triggered flush takes this same lock from `spill_into_seal_batch`. + let (deletes, acked) = { + let mut guard = self.inner.write().unwrap(); + guard.max_observed_raft_index = guard.max_observed_raft_index.max(raft_index); + let mut deletes = Vec::new(); + for &(idx, ord) in acked_keys { + let delete_spill = if let Some(removed) = guard.tasks.remove(&(idx, ord)) { + removed.durable_spilled || removed.sealed_at_index.is_some() + } else { + // Idempotent delete in case `[0x07]` existed from prior snapshot/recovery. + true + }; + if delete_spill { + deletes.push(SpilledAuxOutboxKey::new(idx, ord).encode().to_vec()); } - } else { - // Idempotent delete in case `[0x07]` existed from prior snapshot/recovery. - let spill_key = SpilledAuxOutboxKey::new(idx, ord).encode().to_vec(); - engine.apply_committed_delete(spill_key, hlc, raft_index); } + guard.recompute_contiguous_acked(); + let frontier = guard.contiguous_durable_frontier(); + self.watermarks.advance_aux_outbox_durable(frontier); + (deletes, guard.contiguous_acked_index) + }; + for key in deletes { + engine.apply_committed_delete(key, hlc, raft_index); } - - guard.recompute_contiguous_acked(); - let frontier = guard.contiguous_durable_frontier(); - self.watermarks.advance_aux_outbox_durable(frontier); - guard.contiguous_acked_index + acked } /// Rehydrates spilled `[0x07]` tasks from `engine` on startup or after `install_snapshot` (`Resolution B6`). diff --git a/crates/metadb-tx/src/pushdown.rs b/crates/metadb-tx/src/pushdown.rs index c070025..915eea5 100644 --- a/crates/metadb-tx/src/pushdown.rs +++ b/crates/metadb-tx/src/pushdown.rs @@ -6,7 +6,7 @@ use metadb_core::{ HlcTimestamp, InodeAttr, InodeId, MetaDbError, MetaEntry, MetaKey, Result, UserId, ATTR_ENTRY_NAME, }; -use metadb_storage::LifecycleStorageEngine; +use metadb_storage::{LifecycleStorageEngine, ScanControl}; use crate::attr_aggregator::AsyncAttrAggregator; use crate::aux_sink::AsyncAuxiliarySink; @@ -382,19 +382,48 @@ impl PushdownExecutor { pid: InodeId, read_hlc: HlcTimestamp, ) -> Result> { - let start = MetaKey::encode_pid_prefix_start(pid); - let end = MetaKey::encode_pid_prefix_end(pid); - let rows = engine.scan_range(&start, &end, read_hlc)?; + self.listdir_limited(engine, pid, None, None, usize::MAX, read_hlc) + } - let mut entries = Vec::with_capacity(rows.len()); - for (k_bytes, v_bytes) in rows { - let key = MetaKey::decode(&k_bytes)?; + /// Pinned, tombstone-aware directory page. Stops once `limit` child entries are collected + /// instead of reading the rest of the parent range. + pub fn listdir_limited( + &self, + engine: &LifecycleStorageEngine, + pid: InodeId, + after_name: Option<&str>, + name_prefix: Option<&str>, + limit: usize, + read_hlc: HlcTimestamp, + ) -> Result> { + if limit == 0 { + return Ok(Vec::new()); + } + let start = crate::two_pc::listdir_scan_start(pid, after_name, name_prefix); + let end = MetaKey::encode_pid_prefix_end(pid); + if start >= end { + return Ok(Vec::new()); + } + let snap = engine.acquire_fixed_snapshot(read_hlc)?; + let mut entries = Vec::new(); + engine.scan_live_until(&start, &end, &snap, |k_bytes, v_bytes| { + let key = MetaKey::decode(k_bytes)?; if key.is_attr_key() { - continue; + return Ok(ScanControl::Continue); + } + if let Some(prefix) = name_prefix { + if !key.name.starts_with(prefix) { + return Ok(ScanControl::Stop); + } } - let attr = InodeAttr::decode(&v_bytes)?; + let attr = InodeAttr::decode(v_bytes)?; entries.push(MetaEntry { key, attr }); - } + if entries.len() >= limit { + Ok(ScanControl::Stop) + } else { + Ok(ScanControl::Continue) + } + })?; Ok(entries) } } @@ -524,6 +553,62 @@ mod tests { use super::*; + #[test] + fn listdir_limit_stops_before_later_tombstones() { + use metadb_storage::{LsmEngineConfig, ShardedBlockCache}; + use tempfile::tempdir; + + let dir = tempdir().unwrap(); + let cache = Arc::new(ShardedBlockCache::new(4 * 1024 * 1024)); + let engine = LifecycleStorageEngine::open_lsm( + 1, + dir.path(), + cache, + LsmEngineConfig { + auto_locality_compaction: false, + tombstone_density_threshold: 100_000, + memtable_size_bytes: 32 * 1024 * 1024, + ..LsmEngineConfig::default() + }, + ) + .unwrap(); + let hlc = HlcTimestamp::new(1_000, 1); + let attr = InodeAttr { + inode_id: 40, + parent_pid: 4, + entry_type: EntryType::File, + uid: 1, + mode: 0o644, + link_count: 1, + num_children: 0, + size_bytes: 4, + create_time: hlc, + modify_time: hlc, + fid: Some(9), + applied_lsn: 1, + }; + engine.apply_committed_put(MetaKey::child(4, "a").encode(), attr.encode(), hlc, 1); + for index in 0..2048u64 { + engine.apply_committed_delete( + MetaKey::child(4, format!("t{index:04}")).encode(), + HlcTimestamp::new(1_000, 2), + index + 2, + ); + } + engine.flush_mvcc_to_persistent(); + let before = engine.metrics().tombstones_traversed_on_scans; + let push = PushdownExecutor::new( + Arc::new(AsyncAttrAggregator::new()), + Arc::new(AsyncAuxiliarySink::new()), + ); + let page = push + .listdir_limited(&engine, 4, None, None, 1, HlcTimestamp::new(2_000, 0)) + .unwrap(); + assert_eq!(page.len(), 1); + assert_eq!(page[0].key.name, "a"); + assert_eq!(engine.metrics().tombstones_traversed_on_scans - before, 0); + } + #[test] fn rename_updates_directory_attr_on_its_own_shard() { let src = LifecycleStorageEngine::new(1, 5_000, 128, false); diff --git a/crates/metadb-tx/src/two_pc.rs b/crates/metadb-tx/src/two_pc.rs index ddc48f2..ba82f52 100644 --- a/crates/metadb-tx/src/two_pc.rs +++ b/crates/metadb-tx/src/two_pc.rs @@ -18,7 +18,7 @@ use metadb_core::{ TABLE_PREFIX_SECONDARY_INDEX, }; use metadb_raft::ShardRaftGroup; -use metadb_storage::SnapshotGuard; +use metadb_storage::{ScanControl, SnapshotGuard}; use crate::recovery_worker::TxnRecoveryWorker; use crate::rename_cycle::RenameCycleWalker; @@ -1380,26 +1380,38 @@ impl DistributedTxRouter { }; Self::validate_read_epoch(&group, epoch)?; - let rows = eng.scan_with_snapshot(sub_start, sub_end, &final_snap)?; - for (k_bytes, v_bytes) in rows { - let mk = MetaKey::decode(&k_bytes)?; + let mut stop = ListDirStop::Continue; + let need = effective_limit; + eng.scan_live_until(sub_start, sub_end, &final_snap, |k_bytes, v_bytes| { + let mk = MetaKey::decode(k_bytes)?; // Strict `_attr_` isolation rule (`Resolution N2` / `G-P1`): never expose `_attr_`! if mk.is_attr_key() { - continue; + return Ok(ScanControl::Continue); } if let Some(pfx) = name_prefix { if !mk.name.starts_with(pfx) { - return Ok((collected.clone(), None)); + stop = ListDirStop::PrefixEnded; + return Ok(ScanControl::Stop); } } collected.push(MetaEntry { key: mk, - attr: InodeAttr::decode(&v_bytes)?, + attr: InodeAttr::decode(v_bytes)?, }); - if collected.len() >= effective_limit { + if collected.len() >= need { + stop = ListDirStop::PageFull; + Ok(ScanControl::Stop) + } else { + Ok(ScanControl::Continue) + } + })?; + match stop { + ListDirStop::PrefixEnded => return Ok((collected, None)), + ListDirStop::PageFull => { let next_cursor = collected.last().map(|e| e.key.name.clone()); return Ok((collected, next_cursor)); } + ListDirStop::Continue => {} } } @@ -1551,10 +1563,17 @@ impl DistributedTxRouter { #[allow(dead_code)] const _ASSERT_QUOTA_PREFIX: u8 = TABLE_PREFIX_QUOTA; +#[derive(Clone, Copy)] +enum ListDirStop { + Continue, + PageFull, + PrefixEnded, +} + /// Lower bound for a keyset `listdir`. When both a cursor and a prefix are set, the scan /// starts at the later of the two so names before the prefix are not treated as the end /// of the page. -fn listdir_scan_start( +pub(crate) fn listdir_scan_start( pid: InodeId, after_name: Option<&str>, name_prefix: Option<&str>, diff --git a/docs/designs/correctness_hardening.md b/docs/designs/correctness_hardening.md index 250dbdf..d3a58ea 100644 --- a/docs/designs/correctness_hardening.md +++ b/docs/designs/correctness_hardening.md @@ -60,4 +60,4 @@ cargo test --workspace --no-fail-fast The added tests cover both original failures and rejection boundaries, including complete-prefix recovery, malformed snapshot matrices, data corruption, ID migration/downgrade safety, transaction/split interleavings, signed quota delivery, input atomicity, address restart behavior, and actual CLI/daemon behavior. This is not a physical power-cut experiment or a proof that every distributed history is correct. -Only `all-in-one` is an implemented daemon role. Remote Cell registration, autonomous multi-process role bootstrap, replicated GM/CM state, and physical jumbo-node migration remain unimplemented. `memtable_size_bytes` remains a reserved compatibility setting rather than an active flush threshold. Directory scans and multi-shard statistics still have scalability work beyond the corrected cursor semantics. These limitations are not changed into production-scale guarantees by passing regression tests. +Only `all-in-one` is an implemented daemon role. Remote Cell registration, autonomous multi-process role bootstrap, replicated GM/CM state, and physical jumbo-node migration remain unimplemented. Directory scans and multi-shard statistics still have scalability work beyond limit pushdown and the corrected cursor semantics. These limitations are not changed into production-scale guarantees by passing regression tests. diff --git a/docs/en/architecture.md b/docs/en/architecture.md index 5ae865f..0c09cd8 100644 --- a/docs/en/architecture.md +++ b/docs/en/architecture.md @@ -82,7 +82,7 @@ Each shard's state machine opens a WAL-less LSM at `/be/shards/= 0.50`) that triggers targeted bottommost range compaction. Lower to `64` for latency-sensitive `listdir` workloads under heavy TTL deletion. |