From 097d2963750ed04fd8b118e18587a9239d6017e6 Mon Sep 17 00:00:00 2001 From: loverustfs Date: Mon, 16 Nov 2026 19:45:28 +0800 Subject: [PATCH 1/5] Fix metadata recovery, transaction atomicity, and operator correctness Validate complete Raft and snapshot histories, serialize checkpoint boundaries, and preserve committed transactions during shard splits. Enforce durable allocator ownership and request identity, validate mutations before apply, conserve cross-origin quota deltas, and correct SQL, CLI, and metrics behavior. Add deterministic regression and real-network tests with upgrade notes. --- Cargo.lock | 26 + crates/metadb-cluster/Cargo.toml | 1 + crates/metadb-cluster/src/cell_manager.rs | 166 ++++- crates/metadb-cluster/src/global_manager.rs | 9 + crates/metadb-cluster/src/split_worker.rs | 44 +- .../tests/id_restart_regressions.rs | 68 +++ .../tests/round2_allocator_tests.rs | 185 ++++++ .../tests/round2_split_transaction_tests.rs | 192 ++++++ .../tests/split_crash_recovery_tests.rs | 2 + crates/metadb-core/src/lib.rs | 3 +- crates/metadb-core/src/schema.rs | 11 + crates/metadb-proto/Cargo.toml | 1 + crates/metadb-proto/build.rs | 1 + .../proto/metadb/v1/metadata.proto | 2 + .../proto/metadb/v1/transaction.proto | 8 + crates/metadb-proxy/src/router.rs | 9 + crates/metadb-proxy/src/sql_templates.rs | 20 +- crates/metadb-raft/src/group_host.rs | 14 +- crates/metadb-raft/src/log_store.rs | 237 +++++++- crates/metadb-raft/src/network.rs | 49 +- crates/metadb-raft/src/read_index.rs | 17 +- crates/metadb-raft/src/state_machine.rs | 176 ++++-- .../tests/checkpoint_atomicity_tests.rs | 145 +++++ .../tests/multi_raft_failover_tests.rs | 43 +- .../tests/round2_durability_tests.rs | 571 ++++++++++++++++++ crates/metadb-server/Cargo.toml | 5 + crates/metadb-server/src/bin/metadb-ctl.rs | 298 ++++++++- crates/metadb-server/src/bin/metadbd.rs | 114 +++- crates/metadb-server/src/grpc_services.rs | 20 +- crates/metadb-server/src/lib.rs | 8 +- .../metadb-server/tests/grpc_regressions.rs | 188 ++++++ .../tests/input_boundary_regressions.rs | 59 ++ .../metadb-server/tests/integration_tests.rs | 1 + .../tests/rename_concurrency_regressions.rs | 67 ++ .../metadb-server/tests/round2_api_tests.rs | 266 ++++++++ .../metadb-server/tests/round2_cli_tests.rs | 507 ++++++++++++++++ .../tests/round2_mixed_shard_tests.rs | 37 ++ .../tests/round2_name_flush_tests.rs | 60 ++ .../tests/round2_namespace_tests.rs | 56 ++ .../tests/round2_quota_order_tests.rs | 60 ++ .../tests/round2_restart_routes_tests.rs | 74 +++ crates/metadb-storage/src/backend.rs | 54 +- crates/metadb-storage/src/cache.rs | 45 +- crates/metadb-storage/src/engine.rs | 29 +- crates/metadb-storage/src/manifest.rs | 148 ++++- crates/metadb-storage/src/sstable.rs | 152 ++++- .../tests/adversarial_recovery_tests.rs | 317 ++++++++++ .../tests/lsm_recovery_tests.rs | 20 +- .../tests/round2_integrity_tests.rs | 126 ++++ crates/metadb-tx/src/attr_aggregator.rs | 21 + crates/metadb-tx/src/aux_outbox.rs | 122 +++- crates/metadb-tx/src/aux_sink.rs | 38 +- crates/metadb-tx/src/lib.rs | 1 + crates/metadb-tx/src/pushdown.rs | 20 + crates/metadb-tx/src/state_machine_impl.rs | 501 ++++++++++++++- crates/metadb-tx/src/two_pc.rs | 145 ++++- crates/metadb-tx/src/validation.rs | 30 + .../tests/attr_reconcile_race_tests.rs | 5 + .../metadb-tx/tests/aux_idempotency_tests.rs | 5 + .../tests/round2_atomic_validation_tests.rs | 421 +++++++++++++ .../tests/round2_snapshot_reconcile_tests.rs | 142 +++++ .../metadb-tx/tests/split_attr_apply_tests.rs | 390 ++++++++++++ .../metadb-tx/tests/two_pc_compare_tests.rs | 332 ++++++++++ crates/metadb-tx/tests/two_pc_crash_tests.rs | 5 + docs/designs/correctness_hardening.md | 59 ++ docs/guides/en/deployment.md | 12 +- docs/guides/en/operations-and-tuning.md | 12 +- 67 files changed, 6697 insertions(+), 275 deletions(-) create mode 100644 crates/metadb-cluster/tests/id_restart_regressions.rs create mode 100644 crates/metadb-cluster/tests/round2_allocator_tests.rs create mode 100644 crates/metadb-cluster/tests/round2_split_transaction_tests.rs create mode 100644 crates/metadb-raft/tests/checkpoint_atomicity_tests.rs create mode 100644 crates/metadb-raft/tests/round2_durability_tests.rs create mode 100644 crates/metadb-server/tests/grpc_regressions.rs create mode 100644 crates/metadb-server/tests/input_boundary_regressions.rs create mode 100644 crates/metadb-server/tests/rename_concurrency_regressions.rs create mode 100644 crates/metadb-server/tests/round2_api_tests.rs create mode 100644 crates/metadb-server/tests/round2_cli_tests.rs create mode 100644 crates/metadb-server/tests/round2_mixed_shard_tests.rs create mode 100644 crates/metadb-server/tests/round2_name_flush_tests.rs create mode 100644 crates/metadb-server/tests/round2_namespace_tests.rs create mode 100644 crates/metadb-server/tests/round2_quota_order_tests.rs create mode 100644 crates/metadb-server/tests/round2_restart_routes_tests.rs create mode 100644 crates/metadb-storage/tests/adversarial_recovery_tests.rs create mode 100644 crates/metadb-storage/tests/round2_integrity_tests.rs create mode 100644 crates/metadb-tx/src/validation.rs create mode 100644 crates/metadb-tx/tests/round2_atomic_validation_tests.rs create mode 100644 crates/metadb-tx/tests/round2_snapshot_reconcile_tests.rs create mode 100644 crates/metadb-tx/tests/split_attr_apply_tests.rs create mode 100644 crates/metadb-tx/tests/two_pc_compare_tests.rs create mode 100644 docs/designs/correctness_hardening.md diff --git a/Cargo.lock b/Cargo.lock index 9c6b91b..284a2c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -812,6 +812,7 @@ dependencies = [ "metadb-raft", "metadb-storage", "metadb-tx", + "openraft", "serde", "tempfile", "tokio", @@ -835,6 +836,7 @@ dependencies = [ "bincode", "metadb-core", "prost", + "serde", "tonic", "tonic-prost", "tonic-prost-build", @@ -880,6 +882,9 @@ version = "0.1.0" dependencies = [ "bincode", "clap", + "http-body-util", + "hyper", + "hyper-util", "metadb-cluster", "metadb-core", "metadb-proto", @@ -887,7 +892,9 @@ dependencies = [ "metadb-raft", "metadb-storage", "metadb-tx", + "openraft", "serde", + "serde_json", "tempfile", "tokio", "tokio-stream", @@ -1437,6 +1444,19 @@ dependencies = [ "syn 3.0.10", ] +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "serde_spanned" version = "0.6.9" @@ -2039,3 +2059,9 @@ checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" dependencies = [ "memchr", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/crates/metadb-cluster/Cargo.toml b/crates/metadb-cluster/Cargo.toml index dbdb268..7c06f89 100644 --- a/crates/metadb-cluster/Cargo.toml +++ b/crates/metadb-cluster/Cargo.toml @@ -18,6 +18,7 @@ serde = { workspace = true } tokio = { workspace = true } tonic = { workspace = true } tracing = { workspace = true } +openraft = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/metadb-cluster/src/cell_manager.rs b/crates/metadb-cluster/src/cell_manager.rs index cada2fb..3042738 100644 --- a/crates/metadb-cluster/src/cell_manager.rs +++ b/crates/metadb-cluster/src/cell_manager.rs @@ -9,6 +9,8 @@ use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; +use openraft::async_runtime::watch::WatchReceiver; +use openraft::{BasicNode, ChangeMembers, Precondition}; use serde::{Deserialize, Serialize}; use metadb_core::{ @@ -128,12 +130,52 @@ impl DurableCellManager { fs::create_dir_all(&state_dir)?; let state_path = state_dir.join("CELL_MANAGER_STATE"); - let is_existing = state_path.exists(); + let is_existing = state_path.try_exists()?; + if !is_existing && state_dir.join("CELL_OWNER").try_exists()? { + return Err(MetaDbError::StorageIoError( + "CELL_MANAGER_STATE is missing from an existing cell directory".to_string(), + )); + } let persisted = if is_existing { let mut raw = Vec::new(); File::open(&state_path)?.read_to_end(&mut raw)?; let body = verify_crc32c_le(&raw, "CELL_MANAGER_STATE")?; - decode_cell_state(body)? + let mut state = decode_cell_state(body)?; + let old_addr = state.nodes.get(&host.node_id()).ok_or_else(|| { + MetaDbError::ShardRoutingError( + "data directory belongs to a different node ID".to_string(), + ) + })?; + if state.routes.values().any(|route| route.cell_id != cell_id) { + return Err(MetaDbError::ShardRoutingError( + "data directory belongs to a different cell ID".to_string(), + )); + } + if old_addr != &local_addr { + for route in state + .routes + .values() + .filter(|route| route.peers.contains_key(&host.node_id())) + { + let voters: Vec<_> = route + .peers + .iter() + .filter(|(_, (_, voter))| *voter) + .map(|(&id, _)| id) + .collect(); + if voters != [host.node_id()] { + return Err(MetaDbError::ShardRoutingError( + "a replicated node must restart at its persisted address; change node addresses online before restarting".to_string())); + } + } + state.nodes.insert(host.node_id(), local_addr.clone()); + for route in state.routes.values_mut() { + if let Some((address, _)) = route.peers.get_mut(&host.node_id()) { + *address = local_addr.clone(); + } + } + } + state } else { let mut init = PersistedCellState::default(); init.nodes.insert(host.node_id(), local_addr.clone()); @@ -181,6 +223,14 @@ impl DurableCellManager { init }; + Self::verify_or_initialize_owner( + &state_dir, + host.node_id(), + cell_id, + &persisted, + is_existing, + )?; + let mgr = Arc::new(Self { cell_id, state_dir, @@ -197,8 +247,14 @@ impl DurableCellManager { crash_point: Mutex::new(None), }); - mgr.boot_local_shards().await?; - mgr.recover_incomplete_splits().await?; + if let Err(error) = mgr.boot_local_shards().await { + let _ = mgr.host.shutdown_all().await; + return Err(error); + } + if let Err(error) = mgr.recover_incomplete_splits().await { + let _ = mgr.host.shutdown_all().await; + return Err(error); + } Ok(mgr) } @@ -529,6 +585,47 @@ impl DurableCellManager { group.initialize_if_needed(voter_addrs.clone()).await?; } self.host.register_running_group(group.clone()); + let membership = group + .raft() + .metrics() + .borrow_watched() + .membership_config + .clone(); + let local_addr = &route.peers[&self.host.node_id()].0; + if membership + .membership() + .get_node(&self.host.node_id()) + .is_some_and(|node| node.addr != *local_addr) + { + let voters: Vec<_> = membership.membership().voter_ids().collect(); + if voters != [self.host.node_id()] + || membership.membership().get_joint_config().len() != 1 + { + return Err(MetaDbError::ShardRoutingError( + "persisted Raft peer address differs from the listener; a replicated address change requires an online membership update".to_string())); + } + // ReadIndex waits for a committed log from the recovered leader's + // current term before changing membership metadata. + let _ = group.linearizable_read(group.current_epoch()).await?; + group + .raft() + .change_membership_if( + ChangeMembers::SetNodes(BTreeMap::from([( + self.host.node_id(), + BasicNode::new(local_addr), + )])), + true, + [Precondition::LastMembershipLogId { + last_membership_log_id: *membership.log_id(), + }], + ) + .await + .map_err(|error| { + MetaDbError::ConsensusError(format!( + "failed to update the local Raft peer address: {error}" + )) + })?; + } self.host.record_registry_state( route.shard_id, RegistryShardState::Active { peers: voter_addrs }, @@ -540,6 +637,16 @@ impl DurableCellManager { let mut synced_route = route.clone(); synced_route.epoch = group.current_epoch(); + { + let mut state = self.state.lock().unwrap(); + let mut next = state.clone(); + next.routes + .insert(synced_route.shard_id, synced_route.clone()); + // Persist refreshed node/route addresses only after the Raft + // membership update has committed. Repeating it is safe on restart. + self.persist_sync(&next)?; + *state = next; + } self.meta_cache.upsert_route(synced_route); } else { self.meta_cache.upsert_route(route); @@ -1369,6 +1476,57 @@ impl DurableCellManager { sync_dir(dir)?; Ok(()) } + + fn verify_or_initialize_owner( + dir: &Path, + node_id: NodeId, + cell_id: CellId, + state: &PersistedCellState, + legacy: bool, + ) -> Result<()> { + let path = dir.join("CELL_OWNER"); + if path.try_exists()? { + let raw = fs::read(path)?; + let body = verify_crc32c_le(&raw, "CELL_OWNER")?; + if body.len() != 16 + || body[..8] != node_id.to_le_bytes() + || body[8..] != cell_id.to_le_bytes() + { + return Err(MetaDbError::ShardRoutingError( + "cell directory owner does not match the configured node and cell IDs" + .to_string(), + )); + } + return Ok(()); + } + // Legacy singleton routes identify the sole owner unambiguously. A + // cluster-wide nodes map does not: it also contains registered peers. + if legacy + && state + .routes + .values() + .any(|route| route.peers.len() != 1 || !route.peers.contains_key(&node_id)) + { + return Err(MetaDbError::ShardRoutingError( + "legacy replicated cell directory has no owner marker; migrate ownership offline before startup".to_string(), + )); + } + let mut raw = node_id.to_le_bytes().to_vec(); + raw.extend_from_slice(&cell_id.to_le_bytes()); + append_crc32c_le(&mut raw); + let tmp = dir.join("CELL_OWNER.tmp"); + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&tmp)?; + file.write_all(&raw)?; + file.sync_data()?; + drop(file); + fs::rename(tmp, path)?; + sync_dir(dir)?; + Ok(()) + } } /// `SplitPlanRecord` gained `dual_mode_pid` after the first on-disk layout. diff --git a/crates/metadb-cluster/src/global_manager.rs b/crates/metadb-cluster/src/global_manager.rs index 05a333b..58ed345 100644 --- a/crates/metadb-cluster/src/global_manager.rs +++ b/crates/metadb-cluster/src/global_manager.rs @@ -79,6 +79,15 @@ impl DurableGlobalManager { #[tonic::async_trait] impl ShardDirectory for DurableGlobalManager { + fn allocate_txn_id(&self, coordinator_node_id: u64) -> Result { + // Share the persisted sequence with FileId grants. Gaps in that sequence + // are harmless; resetting a process-local counter can alias live 2PC records. + let (seq, _) = + self.id_allocator + .allocate_block(0, crate::id_allocator::AllocIdKind::FileId, 1)?; + Ok(((coordinator_node_id as u128) << 64) | seq as u128) + } + fn route_user_key(&self, user_key: &[u8]) -> Result { let route = match self.meta_cache.lookup_key(user_key) { Some(r) => r, diff --git a/crates/metadb-cluster/src/split_worker.rs b/crates/metadb-cluster/src/split_worker.rs index 964f59c..08145ed 100644 --- a/crates/metadb-cluster/src/split_worker.rs +++ b/crates/metadb-cluster/src/split_worker.rs @@ -64,8 +64,21 @@ impl RaftShardSplitWorker { return Ok(None); } + // A bootstrap shard can contain several user tables. Split at a real + // table boundary before applying any table-specific row codec. + let first_table = rows[0].0.first().copied(); + if let Some((key, _)) = rows + .iter() + .find(|(key, _)| key.first().copied() != first_table) + { + return Ok(Some(SplitCandidate::SplitAt { + split_key: vec![key[0]], + dual_mode_pid: None, + })); + } + // Non-MetaTable shards (`0x02 QuotaTable`, `0x03 SecondaryIndexTable`) split at byte midpoint. - if route.range.start_key.first().copied() != Some(TABLE_PREFIX_META) { + if first_table != Some(TABLE_PREFIX_META) { let total: usize = rows.iter().map(|(k, v)| k.len() + v.len()).sum(); let half = total / 2; let mut acc = 0usize; @@ -166,28 +179,35 @@ impl RaftShardSplitWorker { Arc, )> { let old_end_key = parent_route.range.end_key.clone(); + let now_hlc = host.clock().now(); // Never publish an epoch older than the one already applied. A stale plan must // not rewind `version` and then re-fence a range that a newer split already owns. // Stage A: Scan `concat([0x05], split_key) .. concat([0x05], old_end_key)` (`Resolution B14`) // and verify no active `deleting_fence` (`Resolution B17`) before splitting! - let now_hlc = host.clock().now(); { let (_, snap) = parent_group .linearizable_read(parent_group.current_epoch()) .await?; let eng = parent_group.engine(); - let i_start = WriteIntentKey::encode_range_bound(&split_key); - let i_end = WriteIntentKey::encode_range_bound(&old_end_key); - for (_, iv) in eng.scan_with_snapshot(&i_start, &i_end, &snap)? { + // Expiration only permits recovery; it never proves that a pending + // intent is aborted. Drain every transaction before copying a shard. + if let Some((_, iv)) = eng + .scan_with_snapshot(&[0x05], &[0x06], &snap)? + .into_iter() + .next() + { let irec = WriteIntentRecord::decode(&iv)?; - if irec.expire_hlc >= now_hlc { - return Err(MetaDbError::WriteIntentConflict { - conflicting_txn_id: irec.txn_id, - primary_key: irec.primary_key, - expire_hlc: irec.expire_hlc.0, - }); - } + return Err(MetaDbError::WriteIntentConflict { + conflicting_txn_id: irec.txn_id, + primary_key: irec.primary_key, + expire_hlc: irec.expire_hlc.0, + }); + } + if !eng.scan_with_snapshot(&[0x06], &[0x07], &snap)?.is_empty() { + return Err(MetaDbError::TransactionAborted( + "split deferred until transaction decisions are acknowledged".into(), + )); } if let Some(dpid) = dual_mode_pid { let dk = DirShardDeltaKey { pid: dpid }.encode(); diff --git a/crates/metadb-cluster/tests/id_restart_regressions.rs b/crates/metadb-cluster/tests/id_restart_regressions.rs new file mode 100644 index 0000000..65282f2 --- /dev/null +++ b/crates/metadb-cluster/tests/id_restart_regressions.rs @@ -0,0 +1,68 @@ +use metadb_cluster::{AllocIdKind, DurableIdAllocator, LocalIdBuffer}; +use metadb_core::HlcTimestamp; +use std::sync::Arc; + +#[test] +fn local_id_buffer_never_reuses_grants_after_restart() { + let dir = tempfile::tempdir().unwrap(); + let alloc = Arc::new(DurableIdAllocator::open(dir.path()).unwrap()); + let buffer = LocalIdBuffer::new(alloc, 7); + let old = ( + buffer.next_file_inode_id().unwrap(), + buffer.next_dir_inode_id().unwrap(), + buffer.next_file_id().unwrap(), + ); + drop(buffer); + let alloc = Arc::new(DurableIdAllocator::open(dir.path()).unwrap()); + let restarted = LocalIdBuffer::new(alloc, 7); + let new = ( + restarted.next_file_inode_id().unwrap(), + restarted.next_dir_inode_id().unwrap(), + restarted.next_file_id().unwrap(), + ); + assert_ne!(old.0, new.0, "file inode reused after restart"); + assert_ne!(old.1, new.1, "directory inode reused after restart"); + assert_ne!(old.2, new.2, "file ID reused after restart"); +} + +#[test] +fn file_inode_grant_rejects_crossing_the_tag_bit() { + let dir = tempfile::tempdir().unwrap(); + let alloc = DurableIdAllocator::open(dir.path()).unwrap(); + assert!(alloc + .allocate_block(1, AllocIdKind::FileInode, 1u64 << 63) + .is_err()); +} + +#[test] +fn rename_lease_ttl_overflow_returns_error_without_poisoning_allocator() { + let dir = tempfile::tempdir().unwrap(); + let alloc = DurableIdAllocator::open(dir.path()).unwrap(); + assert!(alloc + .acquire_dir_rename_lease(HlcTimestamp::new(1000, 0), u64::MAX) + .is_err()); + assert!(alloc.allocate_block(2, AllocIdKind::FileId, 1).is_ok()); +} + +#[test] +fn transaction_ids_remain_unique_after_coordinator_restart() { + use metadb_cluster::{CompositeMetaCache, DurableGlobalManager}; + use metadb_core::HybridLogicalClock; + use metadb_tx::ShardDirectory; + let dir = tempfile::tempdir().unwrap(); + let new_manager = || { + DurableGlobalManager::new( + Arc::new(HybridLogicalClock::new()), + Arc::new(DurableIdAllocator::open(dir.path()).unwrap()), + Arc::new(CompositeMetaCache::new()), + ) + }; + let manager = new_manager(); + let old = manager.allocate_txn_id(7).unwrap(); + drop(manager); + let manager = new_manager(); + let new = manager.allocate_txn_id(7).unwrap(); + assert_ne!(old, new); + assert_eq!(old >> 64, 7); + assert_eq!(new >> 64, 7); +} diff --git a/crates/metadb-cluster/tests/round2_allocator_tests.rs b/crates/metadb-cluster/tests/round2_allocator_tests.rs new file mode 100644 index 0000000..d8d88db --- /dev/null +++ b/crates/metadb-cluster/tests/round2_allocator_tests.rs @@ -0,0 +1,185 @@ +use metadb_cluster::{AllocIdKind, DurableIdAllocator}; +use metadb_core::HlcTimestamp; + +#[test] +fn independently_opened_allocators_must_not_grant_overlapping_ids() { + let dir = tempfile::tempdir().unwrap(); + let first = DurableIdAllocator::open(dir.path()).unwrap(); + let second = match DurableIdAllocator::open(dir.path()) { + Ok(second) => second, + Err(_) => return, // Exclusive ownership would also satisfy the invariant. + }; + let a = first.allocate_block(101, AllocIdKind::FileId, 8).unwrap(); + let b = second.allocate_block(102, AllocIdKind::FileId, 8).unwrap(); + assert!( + a.0 + a.1 <= b.0 || b.0 + b.1 <= a.0, + "independent handles issued overlapping grants: {a:?}, {b:?}" + ); +} + +#[test] +fn independently_opened_allocators_must_share_rename_lease_exclusion() { + let dir = tempfile::tempdir().unwrap(); + let first = DurableIdAllocator::open(dir.path()).unwrap(); + let second = match DurableIdAllocator::open(dir.path()) { + Ok(second) => second, + Err(_) => return, + }; + let lease = first + .acquire_dir_rename_lease(HlcTimestamp::new(1000, 0), 3000) + .unwrap(); + let conflicting = second.acquire_dir_rename_lease(HlcTimestamp::new(1000, 0), 3000); + assert!( + conflicting.is_err(), + "two authorities granted overlapping leases: {lease:?}, {conflicting:?}" + ); +} + +#[test] +fn uuid_retry_with_different_id_kind_must_be_rejected() { + let dir = tempfile::tempdir().unwrap(); + let allocator = DurableIdAllocator::open(dir.path()).unwrap(); + let file = allocator + .allocate_block(123, AllocIdKind::FileInode, 8) + .unwrap(); + let directory = allocator.allocate_block(123, AllocIdKind::DirInode, 8); + assert!( + directory.is_err(), + "UUID replay returned a tagged file grant as directory IDs: {file:?}, {directory:?}" + ); +} + +#[test] +fn allocator_lock_is_released_when_owner_drops() { + let dir = tempfile::tempdir().unwrap(); + let first = DurableIdAllocator::open(dir.path()).unwrap(); + assert!(DurableIdAllocator::open(dir.path()).is_err()); + drop(first); + assert!(DurableIdAllocator::open(dir.path()).is_ok()); +} + +#[test] +fn request_fingerprint_survives_reopen_and_rejects_changed_count() { + let dir = tempfile::tempdir().unwrap(); + let allocator = DurableIdAllocator::open(dir.path()).unwrap(); + let grant = allocator + .allocate_block(123, AllocIdKind::DirInode, 8) + .unwrap(); + assert!(allocator + .allocate_block(123, AllocIdKind::DirInode, 9) + .is_err()); + drop(allocator); + let allocator = DurableIdAllocator::open(dir.path()).unwrap(); + assert_eq!( + allocator + .allocate_block(123, AllocIdKind::DirInode, 8) + .unwrap(), + grant + ); + assert!(allocator + .allocate_block(123, AllocIdKind::FileId, 8) + .is_err()); +} + +fn write_legacy_state(dir: &std::path::Path, file_sequence: u64) { + let state = ( + file_sequence, + 2u64, + 9u64, + 4u64, + 0u64, + None::<(u64, HlcTimestamp)>, + std::collections::BTreeMap::from([(123u128, (1u64, 8u64))]), + std::collections::VecDeque::from([123u128]), + ); + let mut raw = bincode::serialize(&state).unwrap(); + metadb_core::append_crc32c_le(&mut raw); + std::fs::write(dir.join("ID_ALLOCATOR_STATE"), raw).unwrap(); +} + +#[test] +fn legacy_allocator_migrates_without_reissuing_reserved_ids() { + let dir = tempfile::tempdir().unwrap(); + write_legacy_state(dir.path(), 1); + let allocator = DurableIdAllocator::open(dir.path()).unwrap(); + assert!(allocator + .allocate_block(123, AllocIdKind::FileId, 8) + .is_err()); + let grant = allocator + .allocate_block(456, AllocIdKind::FileId, 2) + .unwrap(); + assert_eq!(grant, (9, 2)); + drop(allocator); + let allocator = DurableIdAllocator::open(dir.path()).unwrap(); + assert_eq!( + allocator + .allocate_block(456, AllocIdKind::FileId, 2) + .unwrap(), + grant + ); +} + +#[test] +fn last_file_inode_block_does_not_overflow_local_cursor() { + let dir = tempfile::tempdir().unwrap(); + write_legacy_state( + dir.path(), + metadb_core::FILE_INODE_TAG_BIT - metadb_cluster::CELL_FILE_ID_BLOCK_SIZE, + ); + let allocator = std::sync::Arc::new(DurableIdAllocator::open(dir.path()).unwrap()); + let buffer = metadb_cluster::LocalIdBuffer::new(allocator, 1); + let mut last = 0; + for _ in 0..metadb_cluster::CELL_FILE_ID_BLOCK_SIZE { + last = buffer.next_file_inode_id().unwrap(); + } + assert_eq!(last, u64::MAX); + assert!(buffer.next_file_inode_id().is_err()); +} + +#[test] +fn missing_allocator_state_never_resets_previously_granted_ids() { + let dir = tempfile::tempdir().unwrap(); + let allocator = DurableIdAllocator::open(dir.path()).unwrap(); + allocator + .allocate_block(123, AllocIdKind::FileId, 8) + .unwrap(); + drop(allocator); + let state = dir.path().join("ID_ALLOCATOR_STATE"); + std::fs::remove_file(&state).unwrap(); + assert!(DurableIdAllocator::open(dir.path()).is_err()); + assert!( + !state.exists(), + "refusal must preserve the incomplete directory for repair" + ); +} + +#[test] +fn version_two_allocator_cannot_be_misread_as_legacy_counters() { + type LegacyState = ( + u64, + u64, + u64, + u64, + u64, + Option<(u64, HlcTimestamp)>, + std::collections::BTreeMap, + std::collections::VecDeque, + ); + let dir = tempfile::tempdir().unwrap(); + let allocator = DurableIdAllocator::open(dir.path()).unwrap(); + allocator + .allocate_block(0, AllocIdKind::FileId, 128) + .unwrap(); + drop(allocator); + let raw = std::fs::read(dir.path().join("ID_ALLOCATOR_STATE")).unwrap(); + let body = metadb_core::verify_crc32c_le(&raw, "allocator").unwrap(); + assert!( + bincode::deserialize::(body).is_err(), + "an old decoder could reinterpret the versioned counters and reissue IDs" + ); + let allocator = DurableIdAllocator::open(dir.path()).unwrap(); + assert_eq!( + allocator.allocate_block(0, AllocIdKind::FileId, 1).unwrap(), + (129, 1) + ); +} diff --git a/crates/metadb-cluster/tests/round2_split_transaction_tests.rs b/crates/metadb-cluster/tests/round2_split_transaction_tests.rs new file mode 100644 index 0000000..bdc19b2 --- /dev/null +++ b/crates/metadb-cluster/tests/round2_split_transaction_tests.rs @@ -0,0 +1,192 @@ +use metadb_cluster::{ + CompositeMetaCache, DurableCellManager, DurableGlobalManager, DurableIdAllocator, + RaftShardSplitWorker, SplitCrashPoint, +}; +use metadb_core::{CompositeKeyRange, HybridLogicalClock, MetaDbError, MetaKey}; +use metadb_raft::{RaftNodeOptions, ShardGroupHost}; +use metadb_storage::{LsmEngineConfig, ShardedBlockCache}; +use metadb_tx::{DistributedTxRouter, TwoPcCrashStage, TxnRecoveryWorker}; +use std::sync::Arc; +use std::time::Duration; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn committed_secondary_must_survive_split_copy_and_recovery_interleaving() { + let dir = tempfile::tempdir().unwrap(); + let clock = Arc::new(HybridLogicalClock::new()); + let allocator = Arc::new(DurableIdAllocator::open(dir.path().join("allocator")).unwrap()); + let cache = Arc::new(CompositeMetaCache::new()); + let host = Arc::new( + ShardGroupHost::open( + 1, + dir.path().join("be"), + Arc::new(ShardedBlockCache::new(1 << 20)), + clock.clone(), + ) + .unwrap(), + ); + let cell = DurableCellManager::open_or_bootstrap( + 1, + dir.path().join("cell"), + host.clone(), + "127.0.0.1:0".into(), + true, + usize::MAX, + LsmEngineConfig::default(), + RaftNodeOptions::default(), + allocator.clone(), + cache.clone(), + ) + .await + .unwrap(); + let directory = Arc::new(DurableGlobalManager::new(clock.clone(), allocator, cache)); + directory.register_cell(CompositeKeyRange::all_user_tables(), cell.clone()); + let tx = DistributedTxRouter::new(1, clock.clone(), directory.clone()).with_ttl_ms(500); + tx.init_dir_attr(1, 1, 7, 0o755).await.unwrap(); + tx.mkdir(1, "source_dir", 100, 7, 0o755).await.unwrap(); + tx.mkdir(1, "target_dir", 900, 7, 0o755).await.unwrap(); + + let route = cell + .list_routes() + .into_iter() + .find(|r| r.shard_id == 1) + .unwrap(); + let (_, group, handler) = cell.lookup_local_handle(1).unwrap(); + let boundary = MetaKey::encode_pid_prefix_start(500); + let secondary_id = cell + .alloc_split_plan(111, 1, boundary.clone(), 2, None) + .unwrap(); + let (left, right, _, right_handler) = RaftShardSplitWorker::execute_local_split( + &host, + &route, + &group, + &handler, + secondary_id, + boundary, + None, + LsmEngineConfig::default(), + RaftNodeOptions::default(), + 2, + None, + ) + .await + .unwrap(); + cell.activate_split(111, left, right, Some(right_handler)) + .unwrap(); + tx.create_file(100, "src", (1 << 63) + 77, 77, 7, 0o644, 100) + .await + .unwrap(); + let interrupted = tx + .rename_with_fault_injection( + 100, + "src", + 900, + "dst", + false, + TwoPcCrashStage::StopAfterPrimaryCommitBeforeSecondary, + ) + .await; + assert!(interrupted.is_err()); + tokio::time::sleep(Duration::from_millis(600)).await; + + let route = cell + .list_routes() + .into_iter() + .find(|r| r.shard_id == secondary_id) + .unwrap(); + let (_, secondary, secondary_handler) = cell.lookup_local_handle(secondary_id).unwrap(); + let boundary = MetaKey::encode_pid_prefix_start(800); + let child_id = cell + .alloc_split_plan(222, secondary_id, boundary.clone(), 3, None) + .unwrap(); + let paused = RaftShardSplitWorker::execute_local_split( + &host, + &route, + &secondary, + &secondary_handler, + child_id, + boundary.clone(), + None, + LsmEngineConfig::default(), + RaftNodeOptions::default(), + 3, + Some(SplitCrashPoint::AfterChildDirRename), + ) + .await; + assert!( + matches!(paused, Err(MetaDbError::WriteIntentConflict { .. })), + "expired committed intents must defer the split before copying" + ); + assert_eq!( + secondary_handler.key_range(), + route.range, + "rejected split must leave the original range writable for recovery" + ); + assert!( + !host.shard_dir(child_id).exists(), + "rejected split must not publish a stale child image" + ); + + // Recover the committed transaction against the original route, then retry + // the deferred split. No stale intent or decision may enter the child image. + TxnRecoveryWorker::run_recovery_pass_on_shard(&group, &handler, directory.as_ref(), &clock) + .await + .unwrap(); + let (_, primary_snapshot) = group + .linearizable_read(group.current_epoch()) + .await + .unwrap(); + let remaining = group + .engine() + .scan_with_snapshot(&[0x06], &[0x07], &primary_snapshot) + .unwrap(); + drop(primary_snapshot); + assert!( + remaining.is_empty(), + "primary decision was not GC'd, so the intended interleaving did not occur" + ); + let (_, snap) = secondary + .linearizable_read(secondary.current_epoch()) + .await + .unwrap(); + let recovered_on_parent = secondary + .engine() + .get_with_snapshot(&MetaKey::child(900, "dst").encode(), &snap) + .unwrap(); + drop(snap); + assert!( + recovered_on_parent.is_some(), + "recovery failed to materialize on old parent" + ); + + let (left, right, _, right_handler) = RaftShardSplitWorker::execute_local_split( + &host, + &route, + &secondary, + &secondary_handler, + child_id, + boundary, + None, + LsmEngineConfig::default(), + RaftNodeOptions::default(), + 3, + None, + ) + .await + .unwrap(); + cell.activate_split(222, left, right, Some(right_handler)) + .unwrap(); + let (_, child_group, child_handler) = cell.lookup_local_handle(child_id).unwrap(); + TxnRecoveryWorker::run_recovery_pass_on_shard( + &child_group, + &child_handler, + directory.as_ref(), + &clock, + ) + .await + .unwrap(); + let destination = tx.lookup(900, "dst").await.unwrap(); + let source = tx.lookup(100, "src").await.unwrap(); + host.shutdown_all().await.unwrap(); + assert!(source.is_none() && destination.as_ref().map(|e|e.attr.fid)==Some(Some(77)), + "committed rename lost its destination after split/recovery: src={source:?}, dst={destination:?}"); +} diff --git a/crates/metadb-cluster/tests/split_crash_recovery_tests.rs b/crates/metadb-cluster/tests/split_crash_recovery_tests.rs index 2c0ecdb..11f00ad 100644 --- a/crates/metadb-cluster/tests/split_crash_recovery_tests.rs +++ b/crates/metadb-cluster/tests/split_crash_recovery_tests.rs @@ -43,6 +43,8 @@ fn persist_failure_does_not_cache_id_grant_until_disk_has_it() { grant ); + assert!(DurableIdAllocator::open(dir.path()).is_err()); + drop(alloc); let reopened = DurableIdAllocator::open(dir.path()).unwrap(); assert_eq!( reopened diff --git a/crates/metadb-core/src/lib.rs b/crates/metadb-core/src/lib.rs index 9b9d5c8..de6917c 100644 --- a/crates/metadb-core/src/lib.rs +++ b/crates/metadb-core/src/lib.rs @@ -23,7 +23,8 @@ pub use schema::{ saturating_add_i64, validate_dir_inode_id, validate_file_size, validate_user_entry_name, AuxOrderingTuple, EntryType, FileId, InodeAttr, InodeId, MetaEntry, MetaKey, NodeId, QuotaKey, QuotaRecord, SecondaryIndexKey, SecondaryIndexRecord, ShardId, UserId, ATTR_ENTRY_NAME, - FILE_INODE_TAG_BIT, INODE_ATTR_ENCODED_LEN, ROOT_INODE_ID, SYSTEM_PREFIX_DIR_SHARD_DELTA, + FILE_INODE_TAG_BIT, INODE_ATTR_ENCODED_LEN, MAX_USER_NAME_BYTES, ROOT_INODE_ID, + SYSTEM_PREFIX_DIR_SHARD_DELTA, SYSTEM_PREFIX_QUOTA_ORIGIN_CURSOR, SYSTEM_PREFIX_SPILLED_AUX_OUTBOX, SYSTEM_PREFIX_TXN_RECORD, SYSTEM_PREFIX_WRITE_INTENT, TABLE_PREFIX_META, TABLE_PREFIX_QUOTA, TABLE_PREFIX_SECONDARY_INDEX, diff --git a/crates/metadb-core/src/schema.rs b/crates/metadb-core/src/schema.rs index e0d76d2..3c4d802 100644 --- a/crates/metadb-core/src/schema.rs +++ b/crates/metadb-core/src/schema.rs @@ -27,6 +27,10 @@ pub const FILE_INODE_TAG_BIT: InodeId = 1u64 << 63; /// attributes with all of its direct children in `MetaTable` (§3.1 of `arXiv:2610.32247v1`). pub const ATTR_ENTRY_NAME: &str = "_attr_"; +/// Maximum UTF-8 name length that fits an unshared SST key, including the +/// one-byte WriteIntent prefix and the nine-byte MetaKey prefix. +pub const MAX_USER_NAME_BYTES: usize = u16::MAX as usize - 10; + /// Fixed byte length of a serialized `InodeAttr` (`88` bytes including version & CRC32C). pub const INODE_ATTR_ENCODED_LEN: usize = 88; const INODE_ATTR_CODEC_V1: u8 = 1; @@ -76,6 +80,13 @@ pub fn validate_user_entry_name(name: &str) -> Result<()> { { return Err(MetaDbError::ReservedEntryName(name.to_string())); } + if name.len() > MAX_USER_NAME_BYTES { + return Err(MetaDbError::CodecError(format!( + "entry name has {} bytes; maximum is {}", + name.len(), + MAX_USER_NAME_BYTES + ))); + } Ok(()) } diff --git a/crates/metadb-proto/Cargo.toml b/crates/metadb-proto/Cargo.toml index 41bff8d..8c1e0f1 100644 --- a/crates/metadb-proto/Cargo.toml +++ b/crates/metadb-proto/Cargo.toml @@ -11,6 +11,7 @@ authors.workspace = true metadb-core = { path = "../metadb-core" } bincode = { workspace = true } prost = { workspace = true } +serde = { workspace = true } tonic = { workspace = true } tonic-prost = { workspace = true } diff --git a/crates/metadb-proto/build.rs b/crates/metadb-proto/build.rs index 80c5674..386e666 100644 --- a/crates/metadb-proto/build.rs +++ b/crates/metadb-proto/build.rs @@ -20,6 +20,7 @@ fn main() -> Result<(), Box> { } tonic_prost_build::configure() + .type_attribute(".", "#[derive(serde::Serialize)]") .build_server(true) .build_client(true) .compile_protos(&protos, &["proto"])?; diff --git a/crates/metadb-proto/proto/metadb/v1/metadata.proto b/crates/metadb-proto/proto/metadb/v1/metadata.proto index ff15a2e..fd10c82 100644 --- a/crates/metadb-proto/proto/metadb/v1/metadata.proto +++ b/crates/metadb-proto/proto/metadb/v1/metadata.proto @@ -165,6 +165,8 @@ message ExecuteSqlResponse { SecondaryIndexRecordProto single_index = 6; repeated SecondaryIndexRecordProto indices = 7; uint64 server_hlc = 8; + string next_cursor = 9; + bool has_more = 10; } message GetQuotaRequest { diff --git a/crates/metadb-proto/proto/metadb/v1/transaction.proto b/crates/metadb-proto/proto/metadb/v1/transaction.proto index 2deb69d..5a7eed1 100644 --- a/crates/metadb-proto/proto/metadb/v1/transaction.proto +++ b/crates/metadb-proto/proto/metadb/v1/transaction.proto @@ -20,6 +20,13 @@ message RenameResponse { uint64 commit_hlc = 3; } +// Compare against the exact committed bytes read by the coordinator. An absent +// value requires that the key does not exist when Prepare is applied. +message PrepareExpectedValue { + bytes user_key = 1; + optional bytes value = 2; +} + message Prepare2pcRequest { RequestRoutingHeader header = 1; bytes txn_id_be = 2; @@ -32,6 +39,7 @@ message Prepare2pcRequest { uint64 rename_fence_token = 9; repeated bytes encoded_intent_keys = 10; repeated bytes encoded_intent_records = 11; + repeated PrepareExpectedValue expected_values = 12; } message Prepare2pcResponse { diff --git a/crates/metadb-proxy/src/router.rs b/crates/metadb-proxy/src/router.rs index a4ba798..aedc36a 100644 --- a/crates/metadb-proxy/src/router.rs +++ b/crates/metadb-proxy/src/router.rs @@ -119,6 +119,15 @@ impl StatelessProxyRouter { .set_quota_limit(uid, quota_limit_bytes) .await?, ))), + MetadataSqlCommand::UpsertQuotaLimit { + uid, + insert_quota_limit_bytes, + update_quota_limit_bytes, + } => Ok(ProxySqlResult::Quota(Some( + self.tx_router + .upsert_quota_limit(uid, insert_quota_limit_bytes, update_quota_limit_bytes) + .await?, + ))), MetadataSqlCommand::ListQuotas { start_uid, limit } => Ok(ProxySqlResult::Quotas( self.tx_router.list_quotas(start_uid, limit).await?, )), diff --git a/crates/metadb-proxy/src/sql_templates.rs b/crates/metadb-proxy/src/sql_templates.rs index f849edc..878aeb2 100644 --- a/crates/metadb-proxy/src/sql_templates.rs +++ b/crates/metadb-proxy/src/sql_templates.rs @@ -59,8 +59,14 @@ pub enum MetadataSqlCommand { }, /// #16 `SELECT * FROM QuotaTable WHERE uid = ?` GetQuota { uid: UserId }, - /// #17 `INSERT INTO QuotaTable (uid, quota_limit_bytes) VALUES (?, ?) ON CONFLICT (uid) DO UPDATE SET quota_limit_bytes = ?` + /// Explicit quota setter retained for callers constructing execution commands directly. SetQuotaLimit { uid: UserId, quota_limit_bytes: u64 }, + /// #17 `INSERT INTO QuotaTable (uid, quota_limit_bytes) VALUES (?, ?) ON CONFLICT (uid) DO UPDATE SET quota_limit_bytes = ?` + UpsertQuotaLimit { + uid: UserId, + insert_quota_limit_bytes: u64, + update_quota_limit_bytes: u64, + }, /// #18 `SELECT * FROM QuotaTable WHERE uid >= ? ORDER BY uid ASC LIMIT ?` ListQuotas { start_uid: UserId, limit: usize }, /// #19 `SELECT * FROM SecondaryIndexTable WHERE fid = ?` @@ -145,6 +151,12 @@ impl<'a> TokenCursor<'a> { } } + fn parse_mode_arg(&mut self) -> Result { + let value = self.parse_u64_arg()?; + u32::try_from(value) + .map_err(|_| MetaDbError::SqlParseError(format!("mode {} exceeds u32::MAX", value))) + } + fn parse_i64_arg(&mut self) -> Result { match self.tokens.get(self.pos) { Some(SqlToken::IntLiteral(v)) if *v >= i64::MIN as i128 && *v <= i64::MAX as i128 => { @@ -314,7 +326,7 @@ impl SqlTemplateParser { c.expect_token(&SqlToken::Comma)?; let uid = c.parse_u64_arg()?; c.expect_token(&SqlToken::Comma)?; - let mode = c.parse_u64_arg()? as u32; + let mode = c.parse_mode_arg()?; c.expect_token(&SqlToken::Comma)?; let size_bytes = c.parse_i64_arg()?; c.expect_token(&SqlToken::RParen)?; @@ -354,7 +366,7 @@ impl SqlTemplateParser { c.expect_token(&SqlToken::Comma)?; let uid = c.parse_u64_arg()?; c.expect_token(&SqlToken::Comma)?; - let mode = c.parse_u64_arg()? as u32; + let mode = c.parse_mode_arg()?; c.expect_token(&SqlToken::RParen)?; Ok(MetadataSqlCommand::Mkdir { @@ -491,7 +503,7 @@ impl SqlTemplateParser { } "MODE" => { // Template #12: chmod - let mode = c.parse_u64_arg()? as u32; + let mode = c.parse_mode_arg()?; let (pid, name) = Self::parse_where_pid_and_user_name(c)?; Ok(MetadataSqlCommand::SetAttr { pid, diff --git a/crates/metadb-raft/src/group_host.rs b/crates/metadb-raft/src/group_host.rs index f28ed44..19f89ba 100644 --- a/crates/metadb-raft/src/group_host.rs +++ b/crates/metadb-raft/src/group_host.rs @@ -451,9 +451,8 @@ impl ShardRaftGroup { &self, expected_epoch: ShardEpoch, ) -> Result<(ReadToken, SnapshotGuard)> { - let engine = self.engine(); self.read_batcher - .acquire_linearizable_guard(expected_epoch, &engine) + .acquire_linearizable_guard(expected_epoch, &self.state_machine) .await } @@ -469,8 +468,11 @@ impl ShardRaftGroup { ) -> Result<(ReadToken, SnapshotGuard)> { let _ = self.clock.update_checked(barrier_hlc)?; let token = self.read_batcher.linearizable_token(expected_epoch).await?; - let engine = self.engine(); - let snap = engine.acquire_fixed_snapshot(barrier_hlc)?; + let snap = self.state_machine.acquire_snapshot( + expected_epoch, + token.read_index, + Some(barrier_hlc), + )?; Ok((token, snap)) } @@ -582,6 +584,10 @@ impl ShardGroupHost { &self.channel_pool } + pub fn set_shared_secret(&self, secret: Option) { + self.channel_pool.set_shared_secret(secret); + } + pub fn data_dir(&self) -> &Path { &self.data_dir } diff --git a/crates/metadb-raft/src/log_store.rs b/crates/metadb-raft/src/log_store.rs index 68b089d..06d413d 100644 --- a/crates/metadb-raft/src/log_store.rs +++ b/crates/metadb-raft/src/log_store.rs @@ -54,16 +54,6 @@ impl LogStoreInner { fs::create_dir_all(raft_dir)?; let hard_state = Self::load_hard_state(raft_dir)?; - // Clean up any leftover `.tmp` files from interrupted atomic renames. - for d in fs::read_dir(raft_dir)? { - let d = d?; - if let Some(name) = d.file_name().to_str() { - if name.ends_with(".tmp") { - let _ = fs::remove_file(d.path()); - } - } - } - let mut seg_paths: Vec<(u64, PathBuf)> = Vec::new(); for d in fs::read_dir(raft_dir)? { let d = d?; @@ -87,6 +77,8 @@ impl LogStoreInner { let mut entries: BTreeMap> = BTreeMap::new(); let mut segments: Vec = Vec::new(); let total_segs = seg_paths.len(); + let mut previous_index = None; + let mut tail_repairs = Vec::new(); for (seg_pos, (first_index, path)) in seg_paths.into_iter().enumerate() { let is_last_segment = seg_pos + 1 == total_segs; @@ -95,7 +87,7 @@ impl LogStoreInner { if raw.len() < 8 { if is_last_segment { - let _ = fs::remove_file(&path); + tail_repairs.push((path, None)); continue; } else { return Err(io::Error::new( @@ -174,6 +166,18 @@ impl LogStoreInner { let log_index = u64::from_le_bytes(frame_body[0..8].try_into().unwrap()); let entry: EntryOf = bincode::deserialize(&frame_body[8..]) .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?; + if entry.log_id().index != log_index + || previous_index.is_some_and(|prev| log_index <= prev) + || (entry_end_offsets.is_empty() && log_index != first_index) + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "inconsistent or duplicate raft frame index {log_index} in {path:?}" + ), + )); + } + previous_index = Some(log_index); last_idx_in_seg = log_index; entry_end_offsets.insert(log_index, frame_end as u64); @@ -188,9 +192,7 @@ impl LogStoreInner { } let final_len = if let Some(valid_len) = truncate_tail { - let wf = OpenOptions::new().write(true).open(&path)?; - wf.set_len(valid_len as u64)?; - wf.sync_data()?; + tail_repairs.push((path.clone(), Some(valid_len as u64))); valid_len as u64 } else { raw.len() as u64 @@ -205,6 +207,27 @@ impl LogStoreInner { }); } + Self::ensure_committed_recovered(&hard_state, &entries)?; + // Validate the entire history before changing any recovery evidence. + for (path, valid_len) in tail_repairs { + if let Some(valid_len) = valid_len { + let wf = OpenOptions::new().write(true).open(path)?; + wf.set_len(valid_len)?; + wf.sync_data()?; + } else { + fs::remove_file(path)?; + } + } + for d in fs::read_dir(raft_dir)? { + let d = d?; + if d.file_name() + .to_str() + .is_some_and(|name| name.ends_with(".tmp")) + { + fs::remove_file(d.path())?; + } + } + Self::persist_initialization_marker(raft_dir)?; let active_file = if let Some(last_seg) = segments.last() { Some(OpenOptions::new().append(true).open(&last_seg.path)?) } else { @@ -222,10 +245,55 @@ impl LogStoreInner { }) } + fn ensure_committed_recovered( + hard_state: &PersistedHardState, + entries: &BTreeMap>, + ) -> io::Result<()> { + let Some(committed) = hard_state.committed else { + return Ok(()); + }; + if hard_state + .last_purged_log_id + .is_some_and(|p| p.index >= committed.index) + { + return Ok(()); + } + let first = hard_state.last_purged_log_id.map_or(0, |p| p.index + 1); + let mut expected = first; + for (&index, entry) in entries.range(first..=committed.index) { + if index != expected || entry.log_id().index != index { + break; + } + if index == committed.index { + if entry.log_id() == committed { + return Ok(()); + } + break; + } + expected = index + 1; + } + Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "committed raft prefix {first}..={} is missing or corrupt at index {expected}", + committed.index, + ), + )) + } + fn load_hard_state(raft_dir: &Path) -> io::Result { let path = raft_dir.join("HARD_STATE"); - if !path.exists() { - return Ok(PersistedHardState::default()); + if !path.try_exists()? { + if fs::read_dir(raft_dir)?.next().is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "HARD_STATE is missing from an initialized or nonempty raft directory", + )); + } + let initial = PersistedHardState::default(); + Self::persist_hard_state_value(raft_dir, &initial)?; + Self::persist_initialization_marker(raft_dir)?; + return Ok(initial); } let mut raw = Vec::new(); File::open(&path)?.read_to_end(&mut raw)?; @@ -236,11 +304,29 @@ impl LogStoreInner { } fn persist_hard_state(&self) -> io::Result<()> { - let mut raw = bincode::serialize(&self.hard_state) + Self::persist_hard_state_value(&self.raft_dir, &self.hard_state) + } + + fn persist_initialization_marker(raft_dir: &Path) -> io::Result<()> { + let path = raft_dir.join("INITIALIZED"); + if !path.try_exists()? { + let mut file = OpenOptions::new().write(true).create_new(true).open(path)?; + file.write_all(b"MRAFTIN1")?; + file.sync_data()?; + sync_dir(raft_dir).map_err(|e| io::Error::other(e.to_string()))?; + } + Ok(()) + } + + fn persist_hard_state_value( + raft_dir: &Path, + hard_state: &PersistedHardState, + ) -> io::Result<()> { + let mut raw = bincode::serialize(hard_state) .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?; append_crc32c_le(&mut raw); - let tmp = self.raft_dir.join("HARD_STATE.tmp"); - let dst = self.raft_dir.join("HARD_STATE"); + let tmp = raft_dir.join("HARD_STATE.tmp"); + let dst = raft_dir.join("HARD_STATE"); { let mut f = OpenOptions::new() .create(true) @@ -251,7 +337,7 @@ impl LogStoreInner { f.sync_data()?; } fs::rename(&tmp, &dst)?; - sync_dir(&self.raft_dir).map_err(|e| io::Error::other(e.to_string()))?; + sync_dir(raft_dir).map_err(|e| io::Error::other(e.to_string()))?; Ok(()) } @@ -307,6 +393,11 @@ impl LogStoreInner { Some(seg) => seg.byte_len >= self.max_segment_bytes, }; if need_new_seg { + // One append batch can cross segment boundaries. Closing the old file + // does not make its frames durable; sync it before replacing the handle. + if let Some(active) = self.active_file.as_mut() { + active.sync_data()?; + } self.rotate_new_segment(log_idx)?; } @@ -390,7 +481,11 @@ impl LogStoreInner { self.entries.clear(); let mut removed_any = false; for seg in self.segments.drain(..) { - let _ = fs::remove_file(seg.path); + match fs::remove_file(seg.path) { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } removed_any = true; } if removed_any { @@ -406,7 +501,11 @@ impl LogStoreInner { while let Some(last_seg) = self.segments.last() { if last_seg.first_index > keep_upto { let seg = self.segments.pop().unwrap(); - let _ = fs::remove_file(&seg.path); + match fs::remove_file(&seg.path) { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } removed_trailing = true; } else { break; @@ -708,3 +807,97 @@ impl RaftLogStorage for SegmentedRaftLogStore { Ok(()) } } + +#[cfg(test)] +mod regression_tests { + use super::*; + use openraft::type_config::alias::CommittedLeaderIdOf; + use openraft::vote::RaftLeaderId; + use openraft::EntryPayload; + + fn committed_log() -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let mut inner = LogStoreInner::open_or_recover(dir.path(), 1024).unwrap(); + let log_id = + LogIdOf::::new(CommittedLeaderIdOf::::new(1, 1), 1); + inner + .append_entries_sync( + (0..=1) + .map(|index| EntryOf:: { + log_id: LogIdOf::::new( + CommittedLeaderIdOf::::new(1, 1), + index, + ), + payload: EntryPayload::Blank, + }) + .collect(), + ) + .unwrap(); + inner.hard_state.committed = Some(log_id); + inner.persist_hard_state().unwrap(); + let segment = inner.segments[0].path.clone(); + drop(inner); + (dir, segment) + } + + #[test] + fn truncation_never_acknowledges_failed_segment_unlink() { + let dir = tempfile::tempdir().unwrap(); + let mut inner = LogStoreInner::open_or_recover(dir.path(), 1024).unwrap(); + inner + .append_entries_sync(vec![EntryOf:: { + log_id: LogIdOf::::new( + CommittedLeaderIdOf::::new(1, 1), + 1, + ), + payload: EntryPayload::Blank, + }]) + .unwrap(); + let segment = inner.segments[0].path.clone(); + fs::rename(&segment, segment.with_extension("saved")).unwrap(); + fs::create_dir(&segment).unwrap(); + assert!( + inner.truncate_suffix_after(None).is_err(), + "failed unlink was reported as durable truncation" + ); + } + + #[test] + fn committed_final_frame_crc_corruption_is_fatal() { + let (dir, segment) = committed_log(); + let mut raw = fs::read(&segment).unwrap(); + let last = raw.len() - 1; + raw[last] ^= 1; + fs::write(&segment, &raw).unwrap(); + let result = LogStoreInner::open_or_recover(dir.path(), 1024); + assert!( + matches!(&result, Err(e) if e.kind() == io::ErrorKind::InvalidData), + "committed final frame must not be silently removed; remaining entries: {:?}", + result.as_ref().ok().map(|s| s.entries.len()) + ); + assert_eq!( + fs::read(segment).unwrap(), + raw, + "corruption detection must preserve evidence" + ); + } + + #[test] + fn committed_final_frame_truncation_is_fatal() { + for removed in 1..=40 { + let (dir, segment) = committed_log(); + let mut raw = fs::read(&segment).unwrap(); + if removed >= raw.len() { + break; + } + raw.truncate(raw.len() - removed); + fs::write(&segment, &raw).unwrap(); + let result = LogStoreInner::open_or_recover(dir.path(), 1024); + assert!( + matches!(&result, Err(e) if e.kind() == io::ErrorKind::InvalidData), + "truncating {removed} bytes discarded committed index 1; remaining entries: {:?}", + result.as_ref().ok().map(|s| s.entries.len()) + ); + } + } +} diff --git a/crates/metadb-raft/src/network.rs b/crates/metadb-raft/src/network.rs index b02e474..531c0ef 100644 --- a/crates/metadb-raft/src/network.rs +++ b/crates/metadb-raft/src/network.rs @@ -59,9 +59,28 @@ impl MultiRaftRouter { #[derive(Clone, Default)] pub struct ChannelPool { channels: Arc>>, + shared_secret: Arc>>, } impl ChannelPool { + /// Configures authentication for all Raft clients sharing this pool. + pub fn set_shared_secret(&self, secret: Option) { + *self.shared_secret.write().unwrap() = secret.filter(|s| !s.is_empty()); + } + + fn authenticate(&self, mut request: Request) -> Result, io::Error> { + if let Some(secret) = self.shared_secret.read().unwrap().as_ref() { + let value = format!("Bearer {secret}").parse().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "invalid Raft shared secret metadata", + ) + })?; + request.metadata_mut().insert("authorization", value); + } + Ok(request) + } + pub async fn connect(&self, addr: &str) -> Result { let url = if addr.starts_with("http://") || addr.starts_with("https://") { addr.to_string() @@ -147,10 +166,14 @@ impl RaftNetworkV2 for TonicRaftNetworkClient { let bincode_payload = bincode::serialize(&rpc).map_err(Self::unreachable)?; let reply = client - .append_entries(Request::new(RaftRpcEnvelope { - shard_id: self.shard_id, - bincode_payload, - })) + .append_entries( + self.pool + .authenticate(Request::new(RaftRpcEnvelope { + shard_id: self.shard_id, + bincode_payload, + })) + .map_err(Self::unreachable)?, + ) .await .map_err(Self::unreachable)? .into_inner(); @@ -178,10 +201,14 @@ impl RaftNetworkV2 for TonicRaftNetworkClient { let bincode_payload = bincode::serialize(&rpc).map_err(Self::unreachable)?; let reply = client - .vote(Request::new(RaftRpcEnvelope { - shard_id: self.shard_id, - bincode_payload, - })) + .vote( + self.pool + .authenticate(Request::new(RaftRpcEnvelope { + shard_id: self.shard_id, + bincode_payload, + })) + .map_err(Self::unreachable)?, + ) .await .map_err(Self::unreachable)? .into_inner(); @@ -252,7 +279,11 @@ impl RaftNetworkV2 for TonicRaftNetworkClient { let out_stream = tokio_stream::iter(chunks); let reply = client - .install_snapshot(Request::new(out_stream)) + .install_snapshot( + self.pool + .authenticate(Request::new(out_stream)) + .map_err(Self::streaming_unreachable)?, + ) .await .map_err(Self::streaming_unreachable)? .into_inner(); diff --git a/crates/metadb-raft/src/read_index.rs b/crates/metadb-raft/src/read_index.rs index 54ceb80..1206820 100644 --- a/crates/metadb-raft/src/read_index.rs +++ b/crates/metadb-raft/src/read_index.rs @@ -8,9 +8,10 @@ use openraft::ReadPolicy; use tokio::sync::{oneshot, watch, Mutex}; use metadb_core::{HlcTimestamp, MetaDbError, Result, ShardEpoch, ShardId}; -use metadb_storage::{LifecycleStorageEngine, SnapshotGuard}; +use metadb_storage::SnapshotGuard; use crate::network::MetaDbRaft; +use crate::state_machine::ShardStateMachine; use crate::types::AppliedFrontier; const MICRO_BATCH_WINDOW: Duration = Duration::from_micros(200); @@ -89,20 +90,10 @@ impl ReadIndexBatcher { pub async fn acquire_linearizable_guard( &self, expected_epoch: ShardEpoch, - engine: &LifecycleStorageEngine, + state_machine: &ShardStateMachine, ) -> Result<(ReadToken, SnapshotGuard)> { let token = self.linearizable_token(expected_epoch).await?; - let snap = engine.acquire_linearizable_snapshot(token.read_index)?; - // Re-verify epoch after acquiring `SnapshotGuard` to close the window between - // `linearizable_token()` and snapshot guard registration. - let post_guard_epoch = self.current_frontier().epoch; - if post_guard_epoch != expected_epoch { - return Err(MetaDbError::EpochMismatch { - shard_id: self.shard_id, - expected: expected_epoch, - actual: post_guard_epoch, - }); - } + let snap = state_machine.acquire_snapshot(expected_epoch, token.read_index, None)?; Ok((token, snap)) } diff --git a/crates/metadb-raft/src/state_machine.rs b/crates/metadb-raft/src/state_machine.rs index a50d35a..eb37ca4 100644 --- a/crates/metadb-raft/src/state_machine.rs +++ b/crates/metadb-raft/src/state_machine.rs @@ -1,10 +1,11 @@ //! Deterministic `RaftStateMachine` backed by WAL-less `LifecycleStorageEngine`, //! atomic SSTable snapshot build/install (`Resolution B6`), and `ShardApplyHandler`. +use std::collections::BTreeSet; use std::fs::{self, File, OpenOptions}; use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, Mutex, RwLock}; use openraft::entry::RaftEntry; use openraft::storage::{EntryResponder, RaftSnapshotBuilder, RaftStateMachine}; @@ -20,7 +21,7 @@ use metadb_core::{ append_crc32c_le, checksum_crc32c, verify_crc32c_le, HlcTimestamp, MetaDbError, Result, ShardEpoch, ShardId, }; -use metadb_storage::sstable::sync_dir; +use metadb_storage::sstable::{sst_file_name, sync_dir}; use metadb_storage::{ DurableShardCheckpoint, LifecycleStorageEngine, LsmEngineConfig, PreFlushHook, ShardedBlockCache, @@ -136,7 +137,29 @@ struct PersistedSnapshotMeta { file_specs: Vec<(String, u64, u32)>, } +fn checkpoint_log_id(checkpoint: &DurableShardCheckpoint) -> Option> { + if ( + checkpoint.flushed_raft_term, + checkpoint.flushed_raft_leader, + checkpoint.flushed_raft_index, + ) == (0, 0, 0) + { + None + } else { + Some(LogIdOf::::new( + CommittedLeaderIdOf::::new( + checkpoint.flushed_raft_term, + checkpoint.flushed_raft_leader, + ), + checkpoint.flushed_raft_index, + )) + } +} + struct StateMachineShared { + // Serializes a complete command (including metadata publication) against + // checkpoints, snapshot replacement, and read-cut acquisition. + operation_lock: Mutex<()>, shard_id: ShardId, shard_dir: PathBuf, block_cache: Arc, @@ -156,6 +179,11 @@ impl StateMachineShared { } fn flush_to_lsm(&self) -> Result<(u64, DurableShardCheckpoint)> { + let _operation = self.operation_lock.lock().unwrap(); + self.flush_to_lsm_locked() + } + + fn flush_to_lsm_locked(&self) -> Result<(u64, DurableShardCheckpoint)> { let engine = self.current_engine(); let last_log = *self.last_applied_log_id.read().unwrap(); let last_hlc = *self.last_applied_hlc.read().unwrap(); @@ -192,6 +220,7 @@ impl StateMachineShared { } fn build_snapshot_sync(&self) -> io::Result> { + let _operation = self.operation_lock.lock().unwrap(); let current_idx = self .last_applied_log_id .read() @@ -201,7 +230,7 @@ impl StateMachineShared { self.handler.prepare_for_snapshot(current_idx); let (_, _) = self - .flush_to_lsm() + .flush_to_lsm_locked() .map_err(|e| io::Error::other(e.to_string()))?; let engine = self.current_engine(); @@ -225,17 +254,7 @@ impl StateMachineShared { .create_checkpoint(&ckpt_lsm_dir) .map_err(|e| io::Error::other(e.to_string()))?; - let last_log_id = if checkpoint.flushed_raft_index == 0 { - None - } else { - Some(LogIdOf::::new( - CommittedLeaderIdOf::::new( - checkpoint.flushed_raft_term, - checkpoint.flushed_raft_leader, - ), - checkpoint.flushed_raft_index, - )) - }; + let last_log_id = checkpoint_log_id(&checkpoint); let last_membership: StoredMembershipOf = if checkpoint.membership_bytes.is_empty() { @@ -291,6 +310,8 @@ impl StateMachineShared { mf.sync_data()?; } + // Persist SNAPSHOT_META's directory entry before publishing the snapshot. + sync_dir(&tmp_dir).map_err(|e| io::Error::other(e.to_string()))?; fs::rename(&tmp_dir, &final_dir)?; sync_dir(&snaps_root).map_err(|e| io::Error::other(e.to_string()))?; // Remove older `snap_*` directories only AFTER `final_dir` is durably fsync'd in `snaps_root`. @@ -433,17 +454,7 @@ impl ShardStateMachine { // Report `MANIFEST`'s `flushed_raft_log_id` (`F`) as `applied_state` on startup so OpenRaft // automatically replays committed `.raftlog` entries strictly above `F` (`Resolution B5`)! - let initial_log_id = if cp.flushed_raft_index == 0 { - None - } else { - Some(LogIdOf::::new( - CommittedLeaderIdOf::::new( - cp.flushed_raft_term, - cp.flushed_raft_leader, - ), - cp.flushed_raft_index, - )) - }; + let initial_log_id = checkpoint_log_id(&cp); let initial_membership: StoredMembershipOf = if cp.membership_bytes.is_empty() { StoredMembership::default() @@ -467,6 +478,7 @@ impl ShardStateMachine { let (frontier_tx, frontier_rx) = watch::channel(initial_frontier); let shared = Arc::new(StateMachineShared { + operation_lock: Mutex::new(()), shard_id, shard_dir, block_cache, @@ -494,6 +506,30 @@ impl ShardStateMachine { pub fn flush_to_lsm(&self) -> Result<(u64, DurableShardCheckpoint)> { self.shared.flush_to_lsm() } + + /// Pin a read cut only between complete state-machine commands and validate + /// the epoch under the same lock as split/membership application. + pub fn acquire_snapshot( + &self, + expected_epoch: ShardEpoch, + read_index: u64, + barrier_hlc: Option, + ) -> Result { + let _operation = self.shared.operation_lock.lock().unwrap(); + let actual = self.shared.handler.current_epoch(); + if actual != expected_epoch { + return Err(MetaDbError::EpochMismatch { + shard_id: self.shared.shard_id, + expected: expected_epoch, + actual, + }); + } + let engine = self.shared.current_engine(); + match barrier_hlc { + Some(hlc) => engine.acquire_fixed_snapshot(hlc), + None => engine.acquire_linearizable_snapshot(read_index), + } + } } impl RaftStateMachine for ShardStateMachine { @@ -521,10 +557,10 @@ impl RaftStateMachine for ShardStateMachine { > + Unpin + OptionalSend, { - let engine = self.shared.current_engine(); - while let Some(item) = entries.next().await { let (entry, responder) = item?; + let _operation = self.shared.operation_lock.lock().unwrap(); + let engine = self.shared.current_engine(); let log_id = entry.log_id(); let resp = match entry.payload { @@ -635,12 +671,8 @@ impl RaftStateMachine for ShardStateMachine { meta: &SnapshotMetaOf, snapshot: Self::SnapshotData, ) -> std::result::Result<(), io::Error> { + let _operation = self.shared.operation_lock.lock().unwrap(); let snap_idx = meta.last_log_id.map(|id| id.index).unwrap_or(0); - if let Some(cur) = *self.shared.last_applied_log_id.read().unwrap() { - if snap_idx <= cur.index { - return Ok(()); - } - } if snapshot.checkpoint.flushed_raft_index != snap_idx { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -650,11 +682,37 @@ impl RaftStateMachine for ShardStateMachine { ), )); } + if meta.last_log_id != checkpoint_log_id(&snapshot.checkpoint) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "snapshot log identity differs from checkpoint", + )); + } + let membership: StoredMembershipOf = + if snapshot.checkpoint.membership_bytes.is_empty() { + StoredMembership::default() + } else { + bincode::deserialize(&snapshot.checkpoint.membership_bytes) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))? + }; + if membership != meta.last_membership { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "snapshot membership differs from checkpoint", + )); + } // Verify per-file `(expected_len, crc32c)` BEFORE staging or touching live `lsm/`. let mut file_specs = Vec::with_capacity(snapshot.files.len()); + let mut supplied_names = BTreeSet::new(); for entry in &snapshot.files { - if entry.name.is_empty() || entry.name.contains('/') || entry.name.contains("..") { + if entry.name.is_empty() + || entry.name == "." + || entry.name.contains('/') + || entry.name.contains('\\') + || entry.name.contains("..") + || !supplied_names.insert(entry.name.clone()) + { return Err(io::Error::new( io::ErrorKind::InvalidData, format!("invalid snapshot relative file name: {:?}", entry.name), @@ -683,6 +741,12 @@ impl RaftStateMachine for ShardStateMachine { } file_specs.push((entry.name.clone(), entry.expected_len, entry.crc32c)); } + if !supplied_names.contains("CURRENT") { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "snapshot is missing CURRENT", + )); + } let staging_dir = self .shared @@ -708,7 +772,7 @@ impl RaftStateMachine for ShardStateMachine { // Verify the staged LSM directory opens cleanly and passes all SSTable/MANIFEST CRC32Cs // BEFORE swapping! let verified_engine = Arc::new( - LifecycleStorageEngine::open_lsm( + LifecycleStorageEngine::open_lsm_existing( self.shared.shard_id, &staging_dir, self.shared.block_cache.clone(), @@ -716,8 +780,51 @@ impl RaftStateMachine for ShardStateMachine { ) .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?, ); + if verified_engine.durable_checkpoint() != snapshot.checkpoint { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "staged MANIFEST checkpoint differs from advertised checkpoint", + )); + } + let mut required_names = BTreeSet::from([ + "CURRENT".to_string(), + fs::read_to_string(staging_dir.join("CURRENT"))? + .trim() + .to_string(), + ]); + for reader in verified_engine + .backend() + .current_version() + .levels + .iter() + .flatten() + { + required_names.insert(sst_file_name(reader.meta().file_id)); + reader + .verify_all_data_blocks() + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?; + } + if supplied_names != required_names { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "snapshot file set differs from the MANIFEST requirements", + )); + } drop(verified_engine); + if let Some(current) = *self.shared.last_applied_log_id.read().unwrap() { + if snap_idx == current.index && meta.last_log_id != Some(current) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "snapshot conflicts with the applied log identity", + )); + } + if snap_idx <= current.index { + fs::remove_dir_all(&staging_dir)?; + return Ok(()); + } + } + let live_lsm_dir = self.shared.shard_dir.join("lsm"); let old_lsm_dir = self .shared @@ -787,6 +894,7 @@ impl RaftStateMachine for ShardStateMachine { mf.write_all(&meta_bytes)?; mf.sync_data()?; } + sync_dir(&saved_tmp).map_err(|e| io::Error::other(e.to_string()))?; fs::rename(&saved_tmp, &saved_snap_dir)?; sync_dir(&snaps_root).map_err(|e| io::Error::other(e.to_string()))?; for d in fs::read_dir(&snaps_root)? { diff --git a/crates/metadb-raft/tests/checkpoint_atomicity_tests.rs b/crates/metadb-raft/tests/checkpoint_atomicity_tests.rs new file mode 100644 index 0000000..5ec5ed0 --- /dev/null +++ b/crates/metadb-raft/tests/checkpoint_atomicity_tests.rs @@ -0,0 +1,145 @@ +use std::sync::{mpsc, Arc, Condvar, Mutex}; +use std::time::Duration; + +use metadb_core::{HlcTimestamp, Result, ShardEpoch}; +use metadb_raft::{ + MetaDbRaftRequest, MetaDbTypeConfig, ShardApplyHandler, ShardStateMachine, + ShardWatermarkTracker, +}; +use metadb_storage::{ + DurableShardCheckpoint, LifecycleStorageEngine, LsmEngineConfig, ShardedBlockCache, +}; +use openraft::storage::RaftStateMachine; +use openraft::type_config::alias::{CommittedLeaderIdOf, EntryOf, LogIdOf}; +use openraft::vote::RaftLeaderId; +use openraft::EntryPayload; + +struct PausedTwoKeyHandler { + first_written: Mutex>>, + released: (Mutex, Condvar), +} + +impl ShardApplyHandler for PausedTwoKeyHandler { + fn current_epoch(&self) -> ShardEpoch { + ShardEpoch::initial() + } + fn on_membership_changed(&self, _: ShardEpoch) {} + fn apply_command( + &self, + index: u64, + hlc: HlcTimestamp, + _: &MetaDbRaftRequest, + engine: &LifecycleStorageEngine, + ) -> Result> { + engine.apply_committed_put(b"first".to_vec(), b"present".to_vec(), hlc, index); + self.first_written + .lock() + .unwrap() + .take() + .unwrap() + .send(()) + .unwrap(); + let (released, wake) = &self.released; + let mut released = released.lock().unwrap(); + while !*released { + released = wake.wait(released).unwrap(); + } + engine.apply_committed_put(b"second".to_vec(), b"present".to_vec(), hlc, index); + Ok(Vec::new()) + } + fn on_snapshot_installed(&self, _: &DurableShardCheckpoint, _: &LifecycleStorageEngine) {} +} + +/// A checkpoint must never include half of one committed state-machine command. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn checkpoint_waits_for_entire_apply_command() { + let dir = tempfile::tempdir().unwrap(); + let (first_tx, first_rx) = mpsc::channel(); + let handler = Arc::new(PausedTwoKeyHandler { + first_written: Mutex::new(Some(first_tx)), + released: (Mutex::new(false), Condvar::new()), + }); + let cache = Arc::new(ShardedBlockCache::new(1024 * 1024)); + let (sm, _) = ShardStateMachine::open( + 1, + dir.path(), + cache.clone(), + LsmEngineConfig::default(), + handler.clone(), + Arc::new(ShardWatermarkTracker::default()), + ) + .unwrap(); + let mut applying = sm.clone(); + let entry = EntryOf:: { + log_id: LogIdOf::::new( + CommittedLeaderIdOf::::new(1, 1), + 1, + ), + payload: EntryPayload::Normal(MetaDbRaftRequest { + expected_epoch: ShardEpoch::initial(), + hlc: HlcTimestamp(1), + payload: Vec::new(), + }), + }; + let apply = tokio::spawn(async move { + applying + .apply(tokio_stream::iter([Ok((entry, None))])) + .await + .unwrap(); + }); + first_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let reading = sm.clone(); + let (read_tx, read_rx) = mpsc::channel(); + let read = tokio::task::spawn_blocking(move || { + let guard = reading + .acquire_snapshot(ShardEpoch::initial(), 0, None) + .unwrap(); + read_tx.send(()).unwrap(); + reading + .engine() + .get_with_snapshot(b"second", &guard) + .unwrap() + }); + let read_completed_early = read_rx.recv_timeout(Duration::from_millis(200)).is_ok(); + let flushing = sm.clone(); + let (flush_tx, flush_rx) = mpsc::channel(); + let flush = tokio::task::spawn_blocking(move || { + let result = flushing.flush_to_lsm(); + flush_tx.send(()).unwrap(); + result + }); + let prematurely_finished = flush_rx.recv_timeout(Duration::from_millis(200)).is_ok(); + *handler.released.0.lock().unwrap() = true; + handler.released.1.notify_all(); + apply.await.unwrap(); + assert_eq!(read.await.unwrap(), Some(b"present".to_vec())); + assert!(!read_completed_early, "read cut exposed a partial command"); + let (mut flushed, _) = flush.await.unwrap().unwrap(); + // The reader may win the operation lock and briefly pin its snapshot when + // the queued flush seals. Returning the old frontier is valid in that case. + // Its guard has now been dropped, so complete the checkpoint before reopen. + if flushed == 0 { + flushed = sm.flush_to_lsm().unwrap().0; + } + assert_eq!(flushed, 1); + drop(sm); + let recovered_handler = Arc::new(metadb_raft::BasicKvApplyHandler::new( + ShardEpoch::initial(), + Arc::new(ShardWatermarkTracker::default()), + )); + let (recovered, _) = ShardStateMachine::open( + 1, + dir.path(), + cache, + LsmEngineConfig::default(), + recovered_handler, + Arc::new(ShardWatermarkTracker::default()), + ) + .unwrap(); + let guard = recovered.engine().acquire_linearizable_snapshot(1).unwrap(); + let second = recovered + .engine() + .get_with_snapshot(b"second", &guard) + .unwrap(); + assert!(!prematurely_finished && second == Some(b"present".to_vec()), "checkpoint completed during partial apply: {prematurely_finished}; recovered second key: {second:?}; checkpoint index: {flushed}"); +} diff --git a/crates/metadb-raft/tests/multi_raft_failover_tests.rs b/crates/metadb-raft/tests/multi_raft_failover_tests.rs index d814efe..6ed7930 100644 --- a/crates/metadb-raft/tests/multi_raft_failover_tests.rs +++ b/crates/metadb-raft/tests/multi_raft_failover_tests.rs @@ -15,10 +15,27 @@ use metadb_raft::{ use metadb_storage::{LsmEngineConfig, ShardedBlockCache}; use tempfile::tempdir; -async fn spawn_raft_grpc_server(router: MultiRaftRouter) -> (String, tokio::task::JoinHandle<()>) { +async fn spawn_raft_grpc_server( + router: MultiRaftRouter, + authenticated: bool, +) -> (String, tokio::task::JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap().to_string(); - let svc = RaftTransportServiceServer::new(MultiRaftTransportServer::new(router)); + let svc = RaftTransportServiceServer::with_interceptor( + MultiRaftTransportServer::new(router), + move |req: tonic::Request<()>| { + if authenticated + && req + .metadata() + .get("authorization") + .and_then(|v| v.to_str().ok()) + != Some("Bearer raft-regression-secret") + { + return Err(tonic::Status::unauthenticated("missing shared secret")); + } + Ok(req) + }, + ); let handle = tokio::spawn(async move { let _ = Server::builder() .add_service(svc) @@ -33,6 +50,16 @@ async fn spawn_raft_grpc_server(router: MultiRaftRouter) -> (String, tokio::task /// kills original Node 1 leader, and verifies failover election + `ReadIndex` linearizable reads! #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn test_online_1_to_3_voter_snapshot_promotion_and_leader_failover() { + run_promotion_and_failover(false).await; +} + +/// Exercise authenticated Vote, AppendEntries and InstallSnapshot over real TCP. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_authenticated_raft_snapshot_promotion_and_leader_failover() { + run_promotion_and_failover(true).await; +} + +async fn run_promotion_and_failover(authenticated: bool) { let dir = tempdir().unwrap(); let cache = Arc::new(ShardedBlockCache::new(8 * 1024 * 1024)); let opts = RaftNodeOptions { @@ -47,9 +74,9 @@ async fn test_online_1_to_3_voter_snapshot_promotion_and_leader_failover() { let router1 = MultiRaftRouter::new(); let router2 = MultiRaftRouter::new(); let router3 = MultiRaftRouter::new(); - let (addr1, srv1) = spawn_raft_grpc_server(router1.clone()).await; - let (addr2, srv2) = spawn_raft_grpc_server(router2.clone()).await; - let (addr3, srv3) = spawn_raft_grpc_server(router3.clone()).await; + let (addr1, srv1) = spawn_raft_grpc_server(router1.clone(), authenticated).await; + let (addr2, srv2) = spawn_raft_grpc_server(router2.clone(), authenticated).await; + let (addr3, srv3) = spawn_raft_grpc_server(router3.clone(), authenticated).await; let start_node = |nid: u64, rtr: MultiRaftRouter| { let base = dir.path().join(format!("node_{}/shard_1", nid)); @@ -58,6 +85,10 @@ async fn test_online_1_to_3_voter_snapshot_promotion_and_leader_failover() { async move { let wm = Arc::new(ShardWatermarkTracker::new(0, 0, 0)); let handler = Arc::new(BasicKvApplyHandler::new(ShardEpoch::initial(), wm.clone())); + let pool = ChannelPool::default(); + if authenticated { + pool.set_shared_secret(Some("raft-regression-secret".to_string())); + } ShardRaftGroup::start( 1, nid, @@ -69,7 +100,7 @@ async fn test_online_1_to_3_voter_snapshot_promotion_and_leader_failover() { wm, Arc::new(HybridLogicalClock::new()), &rtr, - ChannelPool::default(), + pool, ) .await .unwrap() diff --git a/crates/metadb-raft/tests/round2_durability_tests.rs b/crates/metadb-raft/tests/round2_durability_tests.rs new file mode 100644 index 0000000..3aed682 --- /dev/null +++ b/crates/metadb-raft/tests/round2_durability_tests.rs @@ -0,0 +1,571 @@ +//! Round 2 audit regressions. These assertions describe the required recovery +//! contract under missing or inconsistent durable state. + +use std::collections::BTreeMap; +use std::fs; +use std::sync::Arc; + +use metadb_core::{HlcTimestamp, HybridLogicalClock, ShardEpoch}; +use metadb_raft::{ + BasicKvApplyHandler, BasicKvCommand, ChannelPool, MetaDbRaftRequest, MetaDbTypeConfig, + MultiRaftRouter, RaftNodeOptions, SegmentedRaftLogStore, ShardRaftGroup, ShardStateMachine, + ShardWatermarkTracker, +}; +use metadb_storage::{LsmEngineConfig, ShardedBlockCache}; +use openraft::storage::{ + IOFlushed, RaftLogReader, RaftLogStorage, RaftSnapshotBuilder, RaftStateMachine, +}; +use openraft::type_config::alias::{CommittedLeaderIdOf, EntryOf, LogIdOf, VoteOf}; +use openraft::vote::RaftLeaderId; +use openraft::EntryPayload; + +fn log_id(index: u64) -> LogIdOf { + LogIdOf::::new(CommittedLeaderIdOf::::new(1, 1), index) +} + +fn put_entry(index: u64, key: &[u8]) -> EntryOf { + EntryOf:: { + log_id: log_id(index), + payload: EntryPayload::Normal(MetaDbRaftRequest { + expected_epoch: ShardEpoch::initial(), + hlc: HlcTimestamp(index), + payload: bincode::serialize(&BasicKvCommand::Put { + key: key.to_vec(), + value: b"present".to_vec(), + }) + .unwrap(), + }), + } +} + +async fn committed_history(dir: &std::path::Path, segment_size: u64) { + let mut store = SegmentedRaftLogStore::open_with_segment_size( + dir, + segment_size, + Arc::new(ShardWatermarkTracker::default()), + ) + .unwrap(); + store + .append( + (0..=2).map(|index| put_entry(index, &[index as u8])), + IOFlushed::noop(), + ) + .await + .unwrap(); + store.save_committed(Some(log_id(2))).await.unwrap(); +} + +#[tokio::test] +async fn missing_first_committed_segment_preserves_recovery_evidence() { + let dir = tempfile::tempdir().unwrap(); + committed_history(dir.path(), 1).await; + fs::remove_file(dir.path().join("segment_00000000000000000000.raftlog")).unwrap(); + let evidence = dir.path().join("HARD_STATE.tmp"); + fs::write(&evidence, b"interrupted atomic metadata write").unwrap(); + assert!( + SegmentedRaftLogStore::open(dir.path(), Arc::new(ShardWatermarkTracker::default())) + .is_err() + ); + assert_eq!( + fs::read(evidence).unwrap(), + b"interrupted atomic metadata write" + ); +} + +#[tokio::test] +async fn committed_frame_gaps_duplicates_and_identity_mismatches_are_fatal() { + for corruption in ["missing", "duplicate", "reordered", "identity"] { + let dir = tempfile::tempdir().unwrap(); + committed_history(dir.path(), 1024 * 1024).await; + let path = dir.path().join("segment_00000000000000000000.raftlog"); + let original = fs::read(&path).unwrap(); + let mut offset = 8; + let mut frames = Vec::new(); + while offset < original.len() { + let len = u32::from_le_bytes(original[offset..offset + 4].try_into().unwrap()) as usize; + frames.push(original[offset..offset + 8 + len].to_vec()); + offset += 8 + len; + } + match corruption { + "missing" => { + frames.remove(1); + } + "duplicate" => { + frames.insert(1, frames[0].clone()); + } + "reordered" => frames.swap(1, 2), + "identity" => { + // Preserve the framing index and CRC while changing the serialized LogId. + let mut entry: EntryOf = + bincode::deserialize(&frames[1][16..]).unwrap(); + entry.log_id = log_id(9); + frames[1].splice(16.., bincode::serialize(&entry).unwrap()); + let crc = metadb_core::checksum_crc32c(&frames[1][8..]); + frames[1][4..8].copy_from_slice(&crc.to_le_bytes()); + } + _ => unreachable!(), + } + let mut corrupt = original[..8].to_vec(); + corrupt.extend(frames.into_iter().flatten()); + fs::write(&path, &corrupt).unwrap(); + assert!( + SegmentedRaftLogStore::open(dir.path(), Arc::new(ShardWatermarkTracker::default())) + .is_err(), + "accepted {corruption}" + ); + assert_eq!( + fs::read(path).unwrap(), + corrupt, + "modified {corruption} evidence" + ); + } +} + +#[tokio::test] +async fn every_legal_prefix_purge_recovers_its_remaining_committed_entries() { + for segment_size in [1, 1024 * 1024] { + for purged in 1..=4 { + let dir = tempfile::tempdir().unwrap(); + let watermarks = Arc::new(ShardWatermarkTracker::new(purged, purged, purged)); + let mut store = SegmentedRaftLogStore::open_with_segment_size( + dir.path(), + segment_size, + watermarks.clone(), + ) + .unwrap(); + store + .append((0..=4).map(|i| put_entry(i, &[i as u8])), IOFlushed::noop()) + .await + .unwrap(); + store.save_committed(Some(log_id(4))).await.unwrap(); + store.purge(log_id(purged)).await.unwrap(); + drop(store); + let mut reopened = SegmentedRaftLogStore::open(dir.path(), watermarks).unwrap(); + let indexes: Vec<_> = reopened + .try_get_log_entries(..) + .await + .unwrap() + .into_iter() + .map(|e| e.log_id.index) + .collect(); + assert_eq!(indexes, (purged + 1..=4).collect::>()); + assert_eq!( + reopened.get_log_state().await.unwrap().last_purged_log_id, + Some(log_id(purged)) + ); + } + } +} + +#[test] +fn new_store_persists_initial_metadata_and_refuses_its_subsequent_loss() { + let dir = tempfile::tempdir().unwrap(); + let watermarks = Arc::new(ShardWatermarkTracker::default()); + drop(SegmentedRaftLogStore::open(dir.path(), watermarks.clone()).unwrap()); + assert!(dir.path().join("HARD_STATE").is_file()); + assert!(SegmentedRaftLogStore::open(dir.path(), watermarks.clone()).is_ok()); + fs::remove_file(dir.path().join("HARD_STATE")).unwrap(); + assert!(SegmentedRaftLogStore::open(dir.path(), watermarks).is_err()); +} + +#[tokio::test] +async fn existing_hard_state_without_a_marker_keeps_its_durable_vote() { + let dir = tempfile::tempdir().unwrap(); + let watermarks = Arc::new(ShardWatermarkTracker::default()); + let vote = VoteOf::::new_committed(7, 1); + let mut store = SegmentedRaftLogStore::open(dir.path(), watermarks.clone()).unwrap(); + store.save_vote(&vote).await.unwrap(); + drop(store); + // Existing installations predate the initialization marker. + fs::remove_file(dir.path().join("INITIALIZED")).unwrap(); + let mut reopened = SegmentedRaftLogStore::open(dir.path(), watermarks).unwrap(); + assert_eq!(reopened.read_vote().await.unwrap(), Some(vote)); + assert!(dir.path().join("INITIALIZED").is_file()); +} + +#[tokio::test] +async fn missing_hard_state_after_purge_is_fatal() { + let dir = tempfile::tempdir().unwrap(); + let watermarks = Arc::new(ShardWatermarkTracker::new(2, 2, 2)); + let mut store = + SegmentedRaftLogStore::open_with_segment_size(dir.path(), 1, watermarks.clone()).unwrap(); + store + .append((0..=2).map(|i| put_entry(i, &[i as u8])), IOFlushed::noop()) + .await + .unwrap(); + store.save_committed(Some(log_id(2))).await.unwrap(); + store.purge(log_id(2)).await.unwrap(); + assert_eq!(store.segment_file_count(), 0); + drop(store); + fs::remove_file(dir.path().join("HARD_STATE")).unwrap(); + assert!(SegmentedRaftLogStore::open(dir.path(), watermarks).is_err()); +} + +#[tokio::test] +async fn missing_interior_committed_segment_is_fatal() { + let dir = tempfile::tempdir().unwrap(); + let watermarks = Arc::new(ShardWatermarkTracker::default()); + let mut store = + SegmentedRaftLogStore::open_with_segment_size(dir.path(), 1, watermarks.clone()).unwrap(); + store + .save_vote(&VoteOf::::new_committed(1, 1)) + .await + .unwrap(); + store + .append( + (0..=2).map(|index| put_entry(index, &[index as u8])), + IOFlushed::noop(), + ) + .await + .unwrap(); + store.save_committed(Some(log_id(2))).await.unwrap(); + assert_eq!(store.segment_file_count(), 3); + drop(store); + + fs::remove_file(dir.path().join("segment_00000000000000000001.raftlog")).unwrap(); + let reopened = SegmentedRaftLogStore::open(dir.path(), watermarks); + if let Ok(mut store) = reopened { + let retained: Vec<_> = store + .try_get_log_entries(0..=2) + .await + .unwrap() + .into_iter() + .map(|entry| entry.log_id.index) + .collect(); + panic!("recovery silently accepted missing committed index 1; retained={retained:?}; committed={:?}", store.read_committed().await.unwrap()); + } +} + +async fn start_group(dir: &std::path::Path) -> metadb_core::Result> { + let watermarks = Arc::new(ShardWatermarkTracker::default()); + ShardRaftGroup::start( + 1, + 1, + dir, + Arc::new(ShardedBlockCache::new(1024 * 1024)), + LsmEngineConfig::default(), + RaftNodeOptions { + max_segment_bytes: 1, + ..Default::default() + }, + Arc::new(BasicKvApplyHandler::new( + ShardEpoch::initial(), + watermarks.clone(), + )), + watermarks, + Arc::new(HybridLogicalClock::new()), + &MultiRaftRouter::new(), + ChannelPool::default(), + ) + .await +} + +#[tokio::test] +async fn recovery_never_reports_missing_interior_write_as_applied() { + let dir = tempfile::tempdir().unwrap(); + let group = start_group(dir.path()).await.unwrap(); + group + .initialize_if_needed(BTreeMap::from([(1, "127.0.0.1:0".into())])) + .await + .unwrap(); + let mut lost_index = 0; + for index in 0..3 { + let response = group + .propose( + group.current_epoch(), + HlcTimestamp::ZERO, + bincode::serialize(&BasicKvCommand::Put { + key: vec![index], + value: b"acknowledged".to_vec(), + }) + .unwrap(), + ) + .await + .unwrap(); + if index == 1 { + lost_index = response.applied_raft_index; + } + } + assert_eq!(group.engine().durable_checkpoint().flushed_raft_index, 0); + group.shutdown().await.unwrap(); + drop(group); + fs::remove_file( + dir.path() + .join("raft") + .join(format!("segment_{lost_index:020}.raftlog")), + ) + .unwrap(); + let recovered = match start_group(dir.path()).await { + Ok(group) => group, + Err(_) => return, // Refusing recovery is required when the committed prefix is incomplete. + }; + let (_, snapshot) = recovered + .linearizable_read(recovered.current_epoch()) + .await + .unwrap(); + let lost_value = recovered + .engine() + .get_with_snapshot(&[1], &snapshot) + .unwrap(); + let applied = recovered.read_batcher().current_frontier().raft_index; + drop(snapshot); + recovered.shutdown().await.unwrap(); + assert_eq!(lost_value, Some(b"acknowledged".to_vec()), "restart served linearizable reads after skipping committed write index {lost_index}; applied={applied}"); +} + +#[tokio::test] +async fn missing_hard_state_with_existing_logs_is_fatal() { + let dir = tempfile::tempdir().unwrap(); + let watermarks = Arc::new(ShardWatermarkTracker::default()); + let mut store = SegmentedRaftLogStore::open(dir.path(), watermarks.clone()).unwrap(); + store + .save_vote(&VoteOf::::new_committed(1, 1)) + .await + .unwrap(); + store + .append([put_entry(0, b"committed")], IOFlushed::noop()) + .await + .unwrap(); + store.save_committed(Some(log_id(0))).await.unwrap(); + drop(store); + + fs::remove_file(dir.path().join("HARD_STATE")).unwrap(); + let reopened = SegmentedRaftLogStore::open(dir.path(), watermarks); + if let Ok(mut store) = reopened { + panic!("existing raft history reopened without its durable vote/commit: vote={:?}, committed={:?}, retained={}", store.read_vote().await.unwrap(), store.read_committed().await.unwrap(), store.retained_entry_count()); + } +} + +fn state_machine(dir: &std::path::Path) -> ShardStateMachine { + let watermarks = Arc::new(ShardWatermarkTracker::default()); + ShardStateMachine::open( + 1, + dir, + Arc::new(ShardedBlockCache::new(1024 * 1024)), + LsmEngineConfig::default(), + Arc::new(BasicKvApplyHandler::new( + ShardEpoch::initial(), + watermarks.clone(), + )), + watermarks, + ) + .unwrap() + .0 +} + +#[tokio::test] +async fn snapshot_without_manifest_never_replaces_live_data() { + let source_dir = tempfile::tempdir().unwrap(); + let mut source = state_machine(source_dir.path()); + source + .apply(tokio_stream::iter([Ok((put_entry(2, b"source"), None))])) + .await + .unwrap(); + let mut builder = source.get_snapshot_builder().await; + let mut snapshot = builder.build_snapshot().await.unwrap(); + assert_eq!(snapshot.snapshot.checkpoint.flushed_raft_index, 2); + // Every supplied file has a valid CRC/length, but the required file set is absent. + snapshot.snapshot.files.clear(); + + let target_dir = tempfile::tempdir().unwrap(); + let mut target = state_machine(target_dir.path()); + target + .apply(tokio_stream::iter([Ok(( + put_entry(1, b"target-sentinel"), + None, + ))])) + .await + .unwrap(); + target.flush_to_lsm().unwrap(); + let result = target + .install_snapshot(&snapshot.meta, snapshot.snapshot) + .await; + let engine = target.engine(); + let guard = engine.acquire_linearizable_snapshot(2).unwrap(); + let sentinel = engine + .get_with_snapshot(b"target-sentinel", &guard) + .unwrap(); + let checkpoint = engine.durable_checkpoint(); + assert!(result.is_err() && sentinel == Some(b"present".to_vec()), "incomplete archive install result={result:?}; old live sentinel={sentinel:?}; actual MANIFEST index={}; applied index={:?}", checkpoint.flushed_raft_index, target.applied_state().await.unwrap().0); +} + +#[tokio::test] +async fn inconsistent_snapshot_bundles_preserve_live_data_and_applied_state() { + let source_dir = tempfile::tempdir().unwrap(); + let mut source = state_machine(source_dir.path()); + source + .apply(tokio_stream::iter([Ok((put_entry(2, b"older"), None))])) + .await + .unwrap(); + let mut builder = source.get_snapshot_builder().await; + let older = builder.build_snapshot().await.unwrap(); + source + .apply(tokio_stream::iter([Ok((put_entry(3, b"newer"), None))])) + .await + .unwrap(); + let current = builder.build_snapshot().await.unwrap(); + + for corruption in [ + "current", + "manifest", + "sst", + "duplicate", + "unexpected", + "stale-manifest", + "term", + "leader", + "membership", + "descriptor", + "data-crc", + "manifest-tail", + ] { + let mut archive = current.snapshot.clone(); + let mut meta = current.meta.clone(); + match corruption { + "current" => archive.files.retain(|f| f.name != "CURRENT"), + "manifest" => archive.files.retain(|f| !f.name.starts_with("MANIFEST-")), + "sst" => archive.files.retain(|f| !f.name.ends_with(".sst")), + "duplicate" => archive.files.push(archive.files[0].clone()), + "unexpected" => { + let mut extra = archive.files[0].clone(); + extra.name = "unexpected".into(); + archive.files.push(extra); + } + "stale-manifest" => archive.files = older.snapshot.files.clone(), + "term" => { + meta.last_log_id = Some(LogIdOf::::new( + CommittedLeaderIdOf::::new(2, 1), + 3, + )) + } + "leader" => { + meta.last_log_id = Some(LogIdOf::::new( + CommittedLeaderIdOf::::new(1, 2), + 3, + )) + } + "membership" => archive.checkpoint.membership_bytes = vec![0xff], + "descriptor" => { + archive.checkpoint.descriptor_bytes = + ShardEpoch::initial().bump_conf_ver().encode().to_vec() + } + "data-crc" => { + let sst = archive + .files + .iter_mut() + .find(|f| f.name.ends_with(".sst")) + .unwrap(); + sst.bytes[8] ^= 1; + // The archive checksum is valid, but an SST data block is corrupt. + sst.crc32c = metadb_core::checksum_crc32c(&sst.bytes); + } + "manifest-tail" => { + let manifest = archive + .files + .iter_mut() + .find(|f| f.name.starts_with("MANIFEST-")) + .unwrap(); + manifest.bytes.push(0xff); + manifest.expected_len += 1; + manifest.crc32c = metadb_core::checksum_crc32c(&manifest.bytes); + } + _ => unreachable!(), + } + + let target_dir = tempfile::tempdir().unwrap(); + let mut target = state_machine(target_dir.path()); + target + .apply(tokio_stream::iter([Ok((put_entry(1, b"sentinel"), None))])) + .await + .unwrap(); + target.flush_to_lsm().unwrap(); + let before = target.engine().durable_checkpoint(); + assert!( + target.install_snapshot(&meta, archive).await.is_err(), + "accepted {corruption}" + ); + assert_eq!( + target.applied_state().await.unwrap().0, + Some(log_id(1)), + "published {corruption}" + ); + let engine = target.engine(); + assert_eq!( + engine.durable_checkpoint(), + before, + "replaced checkpoint for {corruption}" + ); + let guard = engine.acquire_linearizable_snapshot(1).unwrap(); + assert_eq!( + engine.get_with_snapshot(b"sentinel", &guard).unwrap(), + Some(b"present".to_vec()), + "lost sentinel for {corruption}" + ); + drop(guard); + drop(engine); + // A rejected bundle must leave the receiver able to install a complete bundle. + target + .install_snapshot(¤t.meta, current.snapshot.clone()) + .await + .unwrap(); + assert_eq!( + target.engine().durable_checkpoint(), + current.snapshot.checkpoint + ); + } +} + +#[tokio::test] +async fn valid_older_snapshot_never_rolls_back_a_newer_applied_state() { + let source_dir = tempfile::tempdir().unwrap(); + let mut source = state_machine(source_dir.path()); + source + .apply(tokio_stream::iter([Ok((put_entry(2, b"source"), None))])) + .await + .unwrap(); + let mut builder = source.get_snapshot_builder().await; + let snapshot = builder.build_snapshot().await.unwrap(); + let target_dir = tempfile::tempdir().unwrap(); + let mut target = state_machine(target_dir.path()); + target + .apply(tokio_stream::iter([Ok((put_entry(3, b"sentinel"), None))])) + .await + .unwrap(); + target.flush_to_lsm().unwrap(); + let before = target.engine().durable_checkpoint(); + target + .install_snapshot(&snapshot.meta, snapshot.snapshot) + .await + .unwrap(); + assert_eq!(target.applied_state().await.unwrap().0, Some(log_id(3))); + assert_eq!(target.engine().durable_checkpoint(), before); +} + +#[tokio::test] +async fn entry_zero_snapshot_retains_its_complete_log_identity() { + let source_dir = tempfile::tempdir().unwrap(); + let mut source = state_machine(source_dir.path()); + source + .apply(tokio_stream::iter([Ok((put_entry(0, b"source"), None))])) + .await + .unwrap(); + let mut builder = source.get_snapshot_builder().await; + let snapshot = builder.build_snapshot().await.unwrap(); + assert_eq!(snapshot.meta.last_log_id, Some(log_id(0))); + let target_dir = tempfile::tempdir().unwrap(); + let mut target = state_machine(target_dir.path()); + target + .install_snapshot(&snapshot.meta, snapshot.snapshot) + .await + .unwrap(); + assert_eq!(target.applied_state().await.unwrap().0, Some(log_id(0))); + drop(target); + let mut recovered = state_machine(target_dir.path()); + assert_eq!(recovered.applied_state().await.unwrap().0, Some(log_id(0))); + let engine = recovered.engine(); + let snapshot = engine.acquire_linearizable_snapshot(0).unwrap(); + assert_eq!( + engine.get_with_snapshot(b"source", &snapshot).unwrap(), + Some(b"present".to_vec()) + ); +} diff --git a/crates/metadb-server/Cargo.toml b/crates/metadb-server/Cargo.toml index 0ba19db..84bc747 100644 --- a/crates/metadb-server/Cargo.toml +++ b/crates/metadb-server/Cargo.toml @@ -25,8 +25,12 @@ metadb-tx = { path = "../metadb-tx" } metadb-cluster = { path = "../metadb-cluster" } metadb-proxy = { path = "../metadb-proxy" } bincode = { workspace = true } +http-body-util = "0.1" +hyper = { version = "1", features = ["http1", "server"] } +hyper-util = { version = "0.1", features = ["tokio"] } clap = { workspace = true } serde = { workspace = true } +serde_json = "1" tokio = { workspace = true } tokio-stream = { workspace = true } toml = { workspace = true } @@ -36,6 +40,7 @@ tracing-subscriber = { workspace = true } [dev-dependencies] tempfile = { workspace = true } +openraft = { workspace = true } metadb-core = { path = "../metadb-core" } metadb-cluster = { path = "../metadb-cluster" } metadb-tx = { path = "../metadb-tx" } diff --git a/crates/metadb-server/src/bin/metadb-ctl.rs b/crates/metadb-server/src/bin/metadb-ctl.rs index 0e4f173..7490cec 100644 --- a/crates/metadb-server/src/bin/metadb-ctl.rs +++ b/crates/metadb-server/src/bin/metadb-ctl.rs @@ -1,15 +1,15 @@ //! `metadb-ctl` — Operator Administration, SQL Execution & Benchmark CLI for MetaDB. -use std::time::Instant; +use std::time::{Duration, Instant}; use clap::{Parser, Subcommand}; use metadb_proto::control_plane_service_client::ControlPlaneServiceClient; use metadb_proto::metadata_service_client::MetadataServiceClient; use metadb_proto::{ - CreateFileRequest, EnableDualModeSplitRequest, ExecuteSqlRequest, GetRoutesRequest, - ListDirRequest, MkdirRequest, RegisterCellRequest, ScaleShardReplicasRequest, - TriggerShardSplitRequest, UnlinkRequest, + AllocateIdBlockRequest, CreateFileRequest, EnableDualModeSplitRequest, ErrorDetailProto, + ExecuteSqlRequest, GetRoutesRequest, ListDirRequest, MkdirRequest, RegisterCellRequest, + ScaleShardReplicasRequest, TriggerShardSplitRequest, UnlinkRequest, }; #[derive(Debug, Parser)] @@ -243,7 +243,26 @@ fn reject_app_error( } #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> std::process::ExitCode { + match run().await { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(error) => { + if let Some(application) = error.downcast_ref::() { + eprintln!("{}", application); + } else { + eprintln!( + "{}", + serde_json::json!({"error": { + "operation": "client", "kind": "ClientError", "message": error.to_string() + }}) + ); + } + std::process::ExitCode::FAILURE + } + } +} + +async fn run() -> Result<(), Box> { let cli = Cli::parse(); let url = normalize_url(&cli.endpoint); let token = cli.auth_token.clone(); @@ -516,3 +535,272 @@ async fn main() -> Result<(), Box> { Ok(()) } + +type ClientError = Box; +type MetaClient = MetadataServiceClient< + tonic::service::interceptor::InterceptedService< + tonic::transport::Channel, + metadb_server::BearerAuthClient, + >, +>; + +#[derive(Debug)] +struct ApplicationError(serde_json::Value); + +impl std::fmt::Display for ApplicationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::error::Error for ApplicationError {} + +fn application_error(operation: &str, error: &ErrorDetailProto) -> ClientError { + Box::new(ApplicationError(serde_json::json!({"error": { + "operation": operation, "kind": error.error_kind, "message": error.message + }}))) +} + +fn check_error(operation: &str, error: Option<&ErrorDetailProto>) -> Result<(), ClientError> { + match error { + Some(error) => Err(application_error(operation, error)), + None => Ok(()), + } +} + +#[derive(Default)] +struct BenchStats { + attempted: usize, + succeeded: usize, + transport_failed: usize, + application_failed: usize, + errors_by_kind: std::collections::BTreeMap, + latencies: Vec, +} + +impl BenchStats { + fn record( + &mut self, + elapsed: Duration, + outcome: Result, tonic::Status>, + ) -> bool { + self.attempted += 1; + self.latencies.push(elapsed); + match outcome { + Ok(None) => { + self.succeeded += 1; + true + } + Ok(Some(error)) => { + self.application_failed += 1; + *self.errors_by_kind.entry(error.error_kind).or_default() += 1; + false + } + Err(error) => { + self.transport_failed += 1; + *self + .errors_by_kind + .entry(format!("transport:{:?}", error.code())) + .or_default() += 1; + false + } + } + } + + fn merge(&mut self, other: Self) { + self.attempted += other.attempted; + self.succeeded += other.succeeded; + self.transport_failed += other.transport_failed; + self.application_failed += other.application_failed; + self.latencies.extend(other.latencies); + for (kind, count) in other.errors_by_kind { + *self.errors_by_kind.entry(kind).or_default() += count; + } + } + + fn report(&mut self, phase: &str, elapsed: Duration) { + self.latencies.sort_unstable(); + let percentile = |permille: usize| { + if self.latencies.is_empty() { + return 0.0; + } + let rank = (self.latencies.len() * permille) + .div_ceil(1000) + .saturating_sub(1); + self.latencies[rank].as_secs_f64() * 1000.0 + }; + println!( + "{}", + serde_json::json!({ + "phase": phase, "latency_scope": "all_attempts", "attempted": self.attempted, "succeeded": self.succeeded, + "failed": self.transport_failed + self.application_failed, + "transport_failed": self.transport_failed, "application_failed": self.application_failed, + "errors_by_kind": self.errors_by_kind, "elapsed_seconds": elapsed.as_secs_f64(), + "successful_ops_per_second": self.succeeded as f64 / elapsed.as_secs_f64().max(f64::MIN_POSITIVE), + "p50_ms": percentile(500), "p99_ms": percentile(990), "p999_ms": percentile(999) + }) + ); + } +} + +async fn run_benchmark( + url: &str, + token: &str, + workload: &str, + concurrency: usize, + ops: usize, +) -> Result<(), ClientError> { + if workload != "hotspot-create" && workload != "bulk-ttl-delete" { + return Err(format!( + "unsupported --workload '{}': expected 'hotspot-create' or 'bulk-ttl-delete'", + workload + ) + .into()); + } + if concurrency == 0 { + return Err("--concurrency must be greater than zero".into()); + } + let mut meta = open_meta(url, token).await?; + let mut cp = open_cp(url, token).await?; + let mut grants = Vec::new(); + for (kind, count) in [(2, 1), (1, ops as u64), (3, ops as u64)] { + if count == 0 { + grants.push(0); + continue; + } + let grant = cp + .allocate_id_block(AllocateIdBlockRequest { + request_uuid_be: Vec::new(), + id_kind: kind, + count, + preferred_shard_id: 0, + }) + .await? + .into_inner(); + check_error("bench_allocate", grant.error.as_ref())?; + if grant.count != count || grant.start_id.checked_add(count - 1).is_none() { + return Err("allocator returned an invalid benchmark ID block".into()); + } + grants.push(grant.start_id); + } + let bench_pid = grants[0]; + let inode_start = grants[1]; + let fid_start = grants[2]; + let setup = meta + .mkdir(MkdirRequest { + header: None, + pid: 1, + name: format!("bench_{}_{}", workload.replace('-', "_"), bench_pid), + dir_id: bench_pid, + uid: 1, + mode: 0o755, + }) + .await? + .into_inner(); + check_error("bench_mkdir", setup.error.as_ref())?; + println!( + "{}", + serde_json::json!({"bench_pid": bench_pid, "workload": workload, "requested_ops": ops}) + ); + + let task_count = concurrency.min(ops); + let mut handles = Vec::new(); + let start = Instant::now(); + for tid in 0..task_count { + let mut client: MetaClient = meta.clone(); + let base = ops / task_count; + let remainder = ops % task_count; + let first = tid * base + tid.min(remainder); + let count = base + usize::from(tid < remainder); + handles.push(tokio::spawn(async move { + let mut stats = BenchStats::default(); + let mut created = Vec::new(); + for seq in first..first + count { + let name = format!("f_{}.dat", seq); + let started = Instant::now(); + let result = client + .create_file(CreateFileRequest { + header: None, + pid: bench_pid, + name: name.clone(), + inode_id: inode_start + seq as u64, + fid: fid_start + seq as u64, + uid: 1, + mode: 0o644, + size_bytes: 4096, + }) + .await + .map(|r| r.into_inner().error); + if stats.record(started.elapsed(), result) { + created.push(name); + } + } + (stats, created) + })); + } + let mut creates = BenchStats::default(); + let mut created_names = Vec::new(); + for handle in handles { + let (stats, names) = handle.await?; + creates.merge(stats); + created_names.extend(names); + } + creates.report("create", start.elapsed()); + let mut failed = creates.succeeded != ops; + + if workload == "bulk-ttl-delete" { + let mut handles = Vec::new(); + let delete_start = Instant::now(); + let chunk_size = created_names.len().div_ceil(concurrency).max(1); + for names in created_names.chunks(chunk_size) { + let names = names.to_vec(); + let mut client: MetaClient = meta.clone(); + handles.push(tokio::spawn(async move { + let mut stats = BenchStats::default(); + for name in names { + let started = Instant::now(); + let result = client + .unlink(UnlinkRequest { + header: None, + pid: bench_pid, + name, + }) + .await + .map(|r| r.into_inner().error); + stats.record(started.elapsed(), result); + } + stats + })); + } + let mut deletes = BenchStats::default(); + for handle in handles { + deletes.merge(handle.await?); + } + deletes.report("delete", delete_start.elapsed()); + failed |= deletes.succeeded != ops; + let scan_start = Instant::now(); + let listing = meta + .list_dir(ListDirRequest { + header: None, + pid: bench_pid, + after_name: String::new(), + name_prefix: String::new(), + limit: 1, + }) + .await? + .into_inner(); + check_error("bench_listdir", listing.error.as_ref())?; + println!( + "{}", + serde_json::json!({"phase": "post_delete_listdir", "empty": listing.entries.is_empty() && !listing.has_more, "elapsed_ms": scan_start.elapsed().as_secs_f64() * 1000.0}) + ); + failed |= !listing.entries.is_empty() || listing.has_more; + } + if failed { + return Err( + "benchmark did not complete every requested operation successfully; see phase reports" + .into(), + ); + } + Ok(()) +} diff --git a/crates/metadb-server/src/bin/metadbd.rs b/crates/metadb-server/src/bin/metadbd.rs index 104b3d1..5df5bbb 100644 --- a/crates/metadb-server/src/bin/metadbd.rs +++ b/crates/metadb-server/src/bin/metadbd.rs @@ -1,10 +1,19 @@ //! `metadbd` — Standalone Single-Node & Distributed Multi-Role MetaDB Server Daemon. use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; use clap::Parser; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpListener; +use http_body_util::Full; +use hyper::body::{Bytes, Incoming}; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{Method, Request, Response, StatusCode}; +use hyper_util::rt::{TokioIo, TokioTimer}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::oneshot; +use tokio::task::JoinSet; use metadb_proto::control_plane_service_client::ControlPlaneServiceClient; use metadb_proto::RegisterNodeRequest; @@ -135,27 +144,96 @@ async fn main() -> Result<(), Box> { tracing::info!(cell_manager = %cm_url, node_id, cell_id, "registered node with cell manager"); } - if let Some(m_addr) = metrics_addr_opt { - let m_cluster = cluster.clone(); + let metrics_task = if let Some(m_addr) = metrics_addr_opt { let m_listener = TcpListener::bind(&m_addr).await?; tracing::info!(metrics_addr = %m_addr, "Prometheus metrics endpoint listening"); - tokio::spawn(async move { - while let Ok((mut sock, _)) = m_listener.accept().await { - let mut req_buf = [0u8; 512]; - let _ = sock.read(&mut req_buf).await; - let body = render_prometheus_metrics(&m_cluster.metrics_snapshot()); - let resp = format!( - "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ); - let _ = sock.write_all(resp.as_bytes()).await; - } - }); - } + let (stop, stopped) = oneshot::channel(); + let m_cluster = cluster.clone(); + Some(( + stop, + tokio::spawn(serve_metrics(m_listener, m_cluster, stopped)), + )) + } else { + None + }; tokio::signal::ctrl_c().await?; tracing::info!("received shutdown signal; flushing state and stopping OpenRaft groups"); + if let Some((stop, task)) = metrics_task { + let _ = stop.send(()); + task.await?; + } cluster.shutdown().await?; Ok(()) } + +// Bound both connection count and request lifetime, including slow readers/writers. +const METRICS_MAX_CONNECTIONS: usize = 64; +const METRICS_MAX_HEADER_BYTES: usize = 8192; +const METRICS_IO_TIMEOUT: Duration = Duration::from_secs(2); + +async fn serve_metrics( + listener: TcpListener, + cluster: Arc, + mut shutdown: oneshot::Receiver<()>, +) { + let mut connections = JoinSet::new(); + loop { + tokio::select! { + _ = &mut shutdown => break, + Some(_) = connections.join_next(), if !connections.is_empty() => {}, + accepted = listener.accept(), if connections.len() < METRICS_MAX_CONNECTIONS => { + let Ok((socket, _)) = accepted else { break; }; + let cluster = cluster.clone(); + connections.spawn(async move { + let _ = tokio::time::timeout(METRICS_IO_TIMEOUT, metrics_connection(socket, cluster)).await; + }); + } + } + } + connections.abort_all(); + while connections.join_next().await.is_some() {} +} + +async fn metrics_connection( + socket: TcpStream, + cluster: Arc, +) -> Result<(), hyper::Error> { + let service = service_fn(move |request: Request| { + let cluster = cluster.clone(); + async move { + let (status, body) = if request.method() != Method::GET { + ( + StatusCode::METHOD_NOT_ALLOWED, + "method not allowed\n".to_owned(), + ) + } else if request.uri().path() != "/metrics" { + (StatusCode::NOT_FOUND, "not found\n".to_owned()) + } else { + ( + StatusCode::OK, + render_prometheus_metrics(&cluster.metrics_snapshot()), + ) + }; + let mut response = Response::new(Full::new(Bytes::from(body))); + *response.status_mut() = status; + response.headers_mut().insert( + hyper::header::CONTENT_TYPE, + hyper::header::HeaderValue::from_static("text/plain; version=0.0.4"), + ); + response.headers_mut().insert( + hyper::header::CONNECTION, + hyper::header::HeaderValue::from_static("close"), + ); + Ok::<_, std::convert::Infallible>(response) + } + }); + http1::Builder::new() + .timer(TokioTimer::new()) + .header_read_timeout(METRICS_IO_TIMEOUT) + .max_headers(64) + .max_buf_size(METRICS_MAX_HEADER_BYTES) + .keep_alive(false) + .serve_connection(TokioIo::new(socket), service) + .await +} diff --git a/crates/metadb-server/src/grpc_services.rs b/crates/metadb-server/src/grpc_services.rs index 7b94652..c19a83c 100644 --- a/crates/metadb-server/src/grpc_services.rs +++ b/crates/metadb-server/src/grpc_services.rs @@ -530,8 +530,13 @@ impl MetadataService for MetaDbGrpcService { } ProxySqlResult::OptionalEntry(None) => {} ProxySqlResult::Attr(a) => resp.single_attr = Some(InodeAttrProto::from(&a)), - ProxySqlResult::Entries { entries, .. } => { + ProxySqlResult::Entries { + entries, + next_cursor, + } => { resp.entries = entries.iter().map(MetaEntryProto::from).collect(); + resp.has_more = next_cursor.is_some(); + resp.next_cursor = next_cursor.unwrap_or_default(); } ProxySqlResult::Quota(q) => { resp.single_quota = q.as_ref().map(QuotaRecordProto::from); @@ -717,6 +722,12 @@ impl TransactionService for MetaDbGrpcService { let ep: ShardEpoch = hdr.expected_epoch.unwrap_or_default().into(); let txn_id = decode_req_txn_id(&req.txn_id_be)?; + if req.encoded_intent_keys.len() != req.encoded_intent_records.len() { + return Err(Status::invalid_argument( + "intent keys and records must have equal lengths", + )); + } + let mut intents = Vec::with_capacity(req.encoded_intent_keys.len()); for (ik, ir) in req .encoded_intent_keys @@ -730,7 +741,7 @@ impl TransactionService for MetaDbGrpcService { intents.push((wk.target_user_key, wrec.op)); } - let cmd = TxRaftCmd::Prepare2pc { + let cmd = TxRaftCmd::Prepare2pcChecked { txn_id, is_primary: req.is_primary, primary_key: req.primary_key.clone(), @@ -739,6 +750,11 @@ impl TransactionService for MetaDbGrpcService { expire_hlc: HlcTimestamp(req.expire_hlc), rename_fence_token: req.has_rename_fence_token.then_some(req.rename_fence_token), intents, + expected_values: req + .expected_values + .into_iter() + .map(|entry| (entry.user_key, entry.value)) + .collect(), }; // Prepare runs on the participant named by the routing header. `primary_key` diff --git a/crates/metadb-server/src/lib.rs b/crates/metadb-server/src/lib.rs index 257da6c..ccdcbc1 100644 --- a/crates/metadb-server/src/lib.rs +++ b/crates/metadb-server/src/lib.rs @@ -678,7 +678,7 @@ impl PersistentMetaDbCluster { _ = bg_rx.changed() => break, _ = tokio::time::sleep(Duration::from_millis(250)) => { let Some(c) = bg_cluster.upgrade() else { break; }; - let _ = c.run_maintenance_tick().await; + let _ = c.run_maintenance_tick_inner(false).await; } } } @@ -747,6 +747,10 @@ impl PersistentMetaDbCluster { /// Runs one deterministic pass of `AuxDispatcher`, `TxnRecoveryWorker`, and split checks. pub async fn run_maintenance_tick(&self) -> Result<()> { + self.run_maintenance_tick_inner(true).await + } + + async fn run_maintenance_tick_inner(&self, force_snapshot: bool) -> Result<()> { for cell in self.global_manager.all_cells() { for (_, grp, handler) in cell.local_shard_handles() { let _ = AuxDispatcher::drain_shard_outbox( @@ -779,7 +783,7 @@ impl PersistentMetaDbCluster { let (state_flushed, _, last_snap) = grp.watermarks().snapshot_values(); // First snapshot (watermark still 0) or a large unpurged tail. Skip the // full-dataset snapshot on every 250ms tick while the log is still small. - if last_snap > 0 && last_snap >= state_flushed && retained < 64 { + if !force_snapshot && last_snap > 0 && last_snap >= state_flushed && retained < 64 { continue; } grp.flush_snapshot_and_purge().await?; diff --git a/crates/metadb-server/tests/grpc_regressions.rs b/crates/metadb-server/tests/grpc_regressions.rs new file mode 100644 index 0000000..ac49845 --- /dev/null +++ b/crates/metadb-server/tests/grpc_regressions.rs @@ -0,0 +1,188 @@ +use metadb_core::{ + HlcTimestamp, IntentOp, MetaKey, QuotaKey, QuotaRecord, WriteIntentKey, WriteIntentRecord, +}; +use metadb_proto::raft_transport_service_client::RaftTransportServiceClient; +use metadb_proto::transaction_service_client::TransactionServiceClient; +use metadb_proto::{ + Prepare2pcRequest, PrepareExpectedValue, RaftRpcEnvelope, RequestRoutingHeader, +}; +use metadb_server::{PersistentMetaDbCluster, ServerTomlConfig}; + +fn config(dir: &std::path::Path) -> ServerTomlConfig { + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = dir.to_path_buf(); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + cfg +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn participant_prepare_uses_header_shard_not_primary_shard() { + let dir = tempfile::tempdir().unwrap(); + let (cluster, addr) = PersistentMetaDbCluster::open_and_serve(config(dir.path())) + .await + .unwrap(); + let (_, target, _) = cluster.cell_manager().lookup_local_handle(2).unwrap(); + let key = QuotaKey { uid: 42 }.encode().to_vec(); + let primary_key = MetaKey::dir_attr(1).encode(); + let start = cluster.tx_router().clock().now(); + let expire = HlcTimestamp::new(start.physical_ms() + 3000, 0); + let rec = WriteIntentRecord { + txn_id: 123, + primary_shard_hint: 1, + primary_key: primary_key.clone(), + start_hlc: start, + expire_hlc: expire, + rename_fence_token: None, + op: IntentOp::Put { + value: QuotaRecord::new(42, 1000).encode(), + attr_deltas: vec![], + aux_specs: vec![], + }, + }; + let mut client = TransactionServiceClient::connect(format!("http://{addr}")) + .await + .unwrap(); + let response = client + .prepare2pc(Prepare2pcRequest { + header: Some(RequestRoutingHeader { + shard_id: 2, + expected_epoch: Some(target.current_epoch().into()), + ..Default::default() + }), + txn_id_be: 123u128.to_be_bytes().to_vec(), + is_primary: false, + primary_key, + all_participant_keys: vec![key.clone()], + start_hlc: start.0, + expire_hlc: expire.0, + encoded_intent_keys: vec![WriteIntentKey::new(key.clone()).encode()], + encoded_intent_records: vec![rec.encode()], + expected_values: vec![PrepareExpectedValue { + user_key: key.clone(), + value: None, + }], + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + let (_, snap) = target + .linearizable_read(target.current_epoch()) + .await + .unwrap(); + let stored = target + .engine() + .get_with_snapshot(&WriteIntentKey::new(key).encode(), &snap) + .unwrap(); + drop(snap); + cluster.shutdown().await.unwrap(); + assert!( + response.error.is_none(), + "participant prepare failed: {:?}", + response.error + ); + assert!( + stored.is_some(), + "participant intent missing on header shard" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn mismatched_prepare_vectors_are_rejected() { + let dir = tempfile::tempdir().unwrap(); + let (cluster, addr) = PersistentMetaDbCluster::open_and_serve(config(dir.path())) + .await + .unwrap(); + let (_, target, _) = cluster.cell_manager().lookup_local_handle(1).unwrap(); + let start = cluster.tx_router().clock().now(); + let key = MetaKey::dir_attr(1).encode(); + let mut client = TransactionServiceClient::connect(format!("http://{addr}")) + .await + .unwrap(); + let result = client + .prepare2pc(Prepare2pcRequest { + header: Some(RequestRoutingHeader { + shard_id: 1, + expected_epoch: Some(target.current_epoch().into()), + ..Default::default() + }), + txn_id_be: 123u128.to_be_bytes().to_vec(), + primary_key: key.clone(), + start_hlc: start.0, + expire_hlc: HlcTimestamp::new(start.physical_ms() + 3000, 0).0, + encoded_intent_keys: vec![WriteIntentKey::new(key).encode()], + ..Default::default() + }) + .await; + cluster.shutdown().await.unwrap(); + assert_eq!(result.unwrap_err().code(), tonic::Code::InvalidArgument); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn configured_secret_protects_raft_transport() { + let dir = tempfile::tempdir().unwrap(); + let mut cfg = config(dir.path()); + cfg.security.shared_secret = "test-secret".into(); + let (cluster, addr) = PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap(); + let channel = tonic::transport::Endpoint::from_shared(format!("http://{addr}")) + .unwrap() + .connect() + .await + .unwrap(); + let mut client = RaftTransportServiceClient::new(channel.clone()); + let result = client + .vote(RaftRpcEnvelope { + shard_id: 99999, + bincode_payload: vec![], + }) + .await; + let mut authenticated = RaftTransportServiceClient::with_interceptor( + channel, + metadb_server::BearerAuthClient::new("test-secret"), + ); + let allowed = authenticated + .vote(RaftRpcEnvelope { + shard_id: 99999, + bincode_payload: vec![], + }) + .await; + cluster.shutdown().await.unwrap(); + assert!( + allowed.is_ok(), + "authenticated peer was rejected: {allowed:?}" + ); + assert_eq!(result.unwrap_err().code(), tonic::Code::Unauthenticated); +} + +#[test] +fn outbound_token_adds_authorization_header() { + use tonic::service::Interceptor; + let mut token = metadb_server::BearerAuthClient::new("test-secret"); + let request = token.call(tonic::Request::new(())).unwrap(); + assert_eq!( + request.metadata().get("authorization").unwrap(), + "Bearer test-secret" + ); + assert!(metadb_server::BearerAuth::new("test-secret") + .call(request) + .is_ok()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn unsupported_cell_registration_does_not_report_success() { + let dir = tempfile::tempdir().unwrap(); + let (cluster, addr) = PersistentMetaDbCluster::open_and_serve(config(dir.path())) + .await + .unwrap(); + let mut client = + metadb_proto::control_plane_service_client::ControlPlaneServiceClient::connect(format!( + "http://{addr}" + )) + .await + .unwrap(); + let result = client + .register_cell(metadb_proto::RegisterCellRequest::default()) + .await; + cluster.shutdown().await.unwrap(); + assert_eq!(result.unwrap_err().code(), tonic::Code::Unimplemented); +} diff --git a/crates/metadb-server/tests/input_boundary_regressions.rs b/crates/metadb-server/tests/input_boundary_regressions.rs new file mode 100644 index 0000000..a7dcad0 --- /dev/null +++ b/crates/metadb-server/tests/input_boundary_regressions.rs @@ -0,0 +1,59 @@ +use metadb_core::{ + affinity_key_for_raw_entry, validate_user_entry_name, HlcTimestamp, HybridLogicalClock, + MetaKey, ShardEpoch, +}; +use metadb_proto::convert::{decode_optional_u128_be, decode_u128_be}; +use metadb_proxy::SqlTemplateParser; + +#[test] +fn wire_identifier_lengths_and_key_prefixes_are_checked() { + for n in 0..=64 { + let bytes = vec![0; n]; + assert_eq!(decode_u128_be(&bytes).is_ok(), n == 16); + assert_eq!(decode_optional_u128_be(&bytes).is_ok(), n == 0 || n == 16); + assert_eq!(ShardEpoch::decode(&bytes).is_ok(), n == 16); + } + for prefix in [0x00, 0x08, 0xff] { + let mut raw = vec![0; 9]; + raw[0] = prefix; + assert!(MetaKey::decode(&raw).is_err()); + assert!(affinity_key_for_raw_entry(&raw, &[]).is_err()); + } +} + +#[test] +fn names_pid_max_and_future_hlc_have_defined_boundaries() { + for name in ["", "_attr_", "contains\0nul"] { + assert!(validate_user_entry_name(name).is_err()); + } + // Names must fit the largest internal SST key at every restart boundary. + assert!(validate_user_entry_name(&"a".repeat(metadb_core::MAX_USER_NAME_BYTES)).is_ok()); + assert!(validate_user_entry_name(&"a".repeat(metadb_core::MAX_USER_NAME_BYTES + 1)).is_err()); + assert_eq!(MetaKey::encode_pid_prefix_end(u64::MAX), vec![0x02]); + let clock = HybridLogicalClock::new(); + assert!(clock + .update_checked_with_wall_ms(HlcTimestamp::new(11_000, 0), 1000) + .is_err()); + assert_eq!(clock.now_with_wall_ms(1000).physical_ms(), 1000); +} + +#[test] +fn malformed_sql_and_truncated_templates_return_errors() { + for sql in [ + "", + "'", + "SELECT", + "SELECT * FROM", + "DELETE FROM MetaTable WHERE pid = -1 AND name = 'a'", + "SELECT * FROM MetaTable WHERE pid = 18446744073709551616 AND name = 'a'", + "SELECT * FROM MetaTable WHERE pid = 1 AND name = 'a'; DROP TABLE MetaTable", + "SELECT * FROM MetaTable WHERE pid = 1 AND name = 'unterminated", + ] { + assert!(SqlTemplateParser::parse(sql).is_err(), "accepted {sql:?}"); + } + let valid = "SELECT * FROM MetaTable WHERE pid = 1 AND name = 'a'"; + for end in 0..valid.len() { + assert!(SqlTemplateParser::parse(&valid[..end]).is_err()); + } + assert!(SqlTemplateParser::parse(valid).is_ok()); +} diff --git a/crates/metadb-server/tests/integration_tests.rs b/crates/metadb-server/tests/integration_tests.rs index 7c4c732..e337c65 100644 --- a/crates/metadb-server/tests/integration_tests.rs +++ b/crates/metadb-server/tests/integration_tests.rs @@ -615,5 +615,6 @@ fn prepare_req( rename_fence_token: 0, encoded_intent_keys: vec![WriteIntentKey::new(intent_user_key.clone()).encode()], encoded_intent_records: vec![record.encode()], + expected_values: vec![], } } diff --git a/crates/metadb-server/tests/rename_concurrency_regressions.rs b/crates/metadb-server/tests/rename_concurrency_regressions.rs new file mode 100644 index 0000000..0b71073 --- /dev/null +++ b/crates/metadb-server/tests/rename_concurrency_regressions.rs @@ -0,0 +1,67 @@ +use metadb_core::{EntryType, ROOT_INODE_ID}; +use metadb_server::{PersistentMetaDbCluster, ServerTomlConfig}; +use std::collections::{BTreeSet, VecDeque}; +use std::sync::Arc; +use std::time::Duration; + +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +async fn opposing_directory_renames_remain_reachable_with_eight_clients() { + let dir = tempfile::tempdir().unwrap(); + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = dir.path().to_path_buf(); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + let (cluster, _) = PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap(); + let tx = cluster.tx_router().clone(); + tx.mkdir(ROOT_INODE_ID, "A", 10, 5, 0o755).await.unwrap(); + tx.mkdir(ROOT_INODE_ID, "B", 600, 5, 0o755).await.unwrap(); + assert!(cluster + .cell_manager() + .maybe_split_or_isolate_shard(1, true) + .await + .unwrap() + .is_some()); + let barrier = Arc::new(tokio::sync::Barrier::new(8)); + let mut clients = tokio::task::JoinSet::new(); + for i in 0..8 { + let tx = tx.clone(); + let barrier = barrier.clone(); + clients.spawn(async move { + barrier.wait().await; + if i % 2 == 0 { + tx.rename(1, "A", 600, "A", false).await + } else { + tx.rename(1, "B", 10, "B", false).await + } + }); + } + tokio::time::timeout(Duration::from_secs(20), async { + while let Some(result) = clients.join_next().await { + // Contention, vanished source, or cycle rejection are valid results. + let _ = result.expect("rename task panicked"); + } + }) + .await + .expect("opposing directory renames deadlocked"); + cluster.run_maintenance_tick().await.unwrap(); + let mut seen = BTreeSet::new(); + let mut pending = VecDeque::from([ROOT_INODE_ID]); + while let Some(pid) = pending.pop_front() { + assert!(seen.insert(pid), "directory appears twice or forms a cycle"); + let (children, _) = tx.listdir_paginated(pid, None, None, 100).await.unwrap(); + for child in children { + if child.attr.entry_type == EntryType::Directory { + assert_eq!( + tx.dirstat(child.attr.inode_id).await.unwrap().parent_pid, + pid + ); + pending.push_back(child.attr.inode_id); + } + } + } + cluster.shutdown().await.unwrap(); + assert_eq!( + seen, + BTreeSet::from([ROOT_INODE_ID, 10, 600]), + "live directory became unreachable" + ); +} diff --git a/crates/metadb-server/tests/round2_api_tests.rs b/crates/metadb-server/tests/round2_api_tests.rs new file mode 100644 index 0000000..00142c1 --- /dev/null +++ b/crates/metadb-server/tests/round2_api_tests.rs @@ -0,0 +1,266 @@ +use std::sync::Arc; + +use metadb_core::{CompositeKeyRange, HlcTimestamp, MetaKey, ShardEpoch}; +use metadb_proto::metadata_service_client::MetadataServiceClient; +use metadb_proto::{ + sql_param_value, CreateFileRequest, ExecuteSqlRequest, ListDirRequest, SqlParamValue, +}; +use metadb_proxy::SqlTemplateParser; +use metadb_raft::{MetaDbRaftRequest, ShardApplyHandler, ShardWatermarkTracker}; +use metadb_server::{PersistentMetaDbCluster, ServerTomlConfig}; +use metadb_storage::{LifecycleStorageEngine, LsmEngineConfig, ShardedBlockCache}; +use metadb_tx::{TxRaftCmd, TxStateMachineHandler}; + +async fn start( + dir: &std::path::Path, +) -> ( + Arc, + MetadataServiceClient, +) { + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = dir.to_path_buf(); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + let (cluster, addr) = PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap(); + let client = MetadataServiceClient::connect(format!("http://{addr}")) + .await + .unwrap(); + (cluster, client) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn sql_upsert_must_use_insert_value_for_a_missing_uid() { + let dir = tempfile::tempdir().unwrap(); + let (cluster, mut client) = start(dir.path()).await; + let sql = "INSERT INTO QuotaTable (uid, quota_limit_bytes) VALUES (42, 100) ON CONFLICT (uid) DO UPDATE SET quota_limit_bytes = 200"; + let response = client + .execute_sql(ExecuteSqlRequest { + sql: sql.into(), + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + let stored = cluster.tx_router().get_quota(42).await.unwrap().unwrap(); + assert!(response.error.is_none(), "{response:?}"); + assert_eq!( + stored.quota_limit_bytes, 100, + "absent UID took the ON CONFLICT branch: {response:?}" + ); + let updated = client + .execute_sql(ExecuteSqlRequest { + sql: sql.into(), + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + assert!(updated.error.is_none(), "{updated:?}"); + assert_eq!( + cluster + .tx_router() + .get_quota(42) + .await + .unwrap() + .unwrap() + .quota_limit_bytes, + 200 + ); + cluster.shutdown().await.unwrap(); +} + +#[test] +fn sql_mode_must_reject_values_outside_the_wire_u32_domain() { + for query in [ + "UPDATE MetaTable SET mode = 4294967296 WHERE pid = 1 AND name = 'victim'", + "INSERT INTO MetaTable (pid, name, id, type, fid, uid, mode, size) VALUES (1, 'f', 9223372036854775809, 'FILE', 1, 7, 4294967296, 0)", + "INSERT INTO MetaTable (pid, name, id, type, uid, mode) VALUES (1, 'd', 2, 'DIR', 7, 4294967296)", + ] { + let parsed = SqlTemplateParser::parse(query); + assert!(parsed.is_err(), "mode silently wrapped modulo 2^32: {parsed:?}"); + assert!(SqlTemplateParser::parse(&query.replace("4294967296", "4294967295")).is_ok()); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn unset_sql_bind_must_not_be_removed_before_destructive_execution() { + let dir = tempfile::tempdir().unwrap(); + let (cluster, mut client) = start(dir.path()).await; + cluster + .create_file_auto_id(1, "victim", 7, 0o644, 10) + .await + .unwrap(); + // Three wire bindings for two placeholders: the missing oneof is malformed, + // and the third binding is extra. Neither may be silently discarded. + let response = client + .execute_sql(ExecuteSqlRequest { + sql: "DELETE FROM MetaTable WHERE pid = ? AND name = ?".into(), + params: vec![ + SqlParamValue { + kind: Some(sql_param_value::Kind::U64Val(1)), + }, + SqlParamValue { kind: None }, + SqlParamValue { + kind: Some(sql_param_value::Kind::StrVal("victim".into())), + }, + ], + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + let kept = cluster.tx_router().lookup(1, "victim").await.unwrap(); + cluster.shutdown().await.unwrap(); + assert!( + response.error.is_some() && kept.is_some(), + "malformed bind vector executed DELETE: response={response:?}; victim={kept:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn listdir_combined_cursor_and_prefix_must_seek_to_the_prefix() { + let dir = tempfile::tempdir().unwrap(); + let (cluster, mut client) = start(dir.path()).await; + for name in ["b", "z1", "z2"] { + cluster + .create_file_auto_id(1, name, 7, 0o644, 10) + .await + .unwrap(); + } + let response = client + .list_dir(ListDirRequest { + pid: 1, + after_name: "a".into(), + name_prefix: "z".into(), + limit: 10, + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + cluster.shutdown().await.unwrap(); + assert!(response.error.is_none(), "{response:?}"); + let names: Vec<_> = response + .entries + .iter() + .map(|entry| entry.name.as_str()) + .collect(); + assert_eq!( + names, + ["z1", "z2"], + "scan stopped at nonmatching 'b': {response:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn grpc_create_must_reject_negative_file_size_before_persisting() { + let dir = tempfile::tempdir().unwrap(); + let (cluster, mut client) = start(dir.path()).await; + let response = client + .create_file(CreateFileRequest { + pid: 1, + name: "negative".into(), + inode_id: (1 << 63) | 100, + fid: 100, + uid: 7, + mode: 0o644, + size_bytes: -1, + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + let stored = cluster.tx_router().lookup(1, "negative").await.unwrap(); + cluster.shutdown().await.unwrap(); + assert!( + response.error.is_some() && stored.is_none(), + "negative file size accepted and persisted: response={response:?}; stored={stored:?}" + ); +} + +fn apply( + handler: &TxStateMachineHandler, + engine: &LifecycleStorageEngine, + index: u64, + command: TxRaftCmd, +) -> metadb_core::Result> { + let hlc = HlcTimestamp::new(1, index as u16); + handler.apply_command( + index, + hlc, + &MetaDbRaftRequest { + expected_epoch: handler.current_epoch(), + hlc, + payload: bincode::serialize(&command).unwrap(), + }, + engine, + ) +} + +#[test] +fn minimum_file_size_must_not_panic_after_unlink_has_already_deleted_the_key() { + let dir = tempfile::tempdir().unwrap(); + let engine = LifecycleStorageEngine::open_lsm( + 1, + dir.path(), + Arc::new(ShardedBlockCache::new(1024 * 1024)), + LsmEngineConfig::default(), + ) + .unwrap(); + let handler = TxStateMachineHandler::new( + 1, + ShardEpoch::initial(), + CompositeKeyRange::all_user_tables(), + Arc::new(ShardWatermarkTracker::default()), + ); + apply( + &handler, + &engine, + 1, + TxRaftCmd::InitDirAttr { + dir_id: 1, + parent_pid: 1, + uid: 7, + mode: 0o755, + }, + ) + .unwrap(); + let creation = apply( + &handler, + &engine, + 2, + TxRaftCmd::CreateFile { + pid: 1, + name: "minimum".into(), + inode_id: (1 << 63) | 100, + fid: 100, + uid: 7, + mode: 0o644, + size_bytes: i64::MIN, + }, + ); + // Rejection at creation would satisfy the required safety property. + if creation.is_err() { + return; + } + let unlink = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + apply( + &handler, + &engine, + 3, + TxRaftCmd::Unlink { + pid: 1, + name: "minimum".into(), + }, + ) + })); + let stored = engine + .get( + &MetaKey::child(1, "minimum").encode(), + HlcTimestamp::new(1, 3), + ) + .unwrap(); + assert!( + unlink.is_ok(), + "Unlink panicked for accepted i64::MIN size; key after panic={stored:?}" + ); +} diff --git a/crates/metadb-server/tests/round2_cli_tests.rs b/crates/metadb-server/tests/round2_cli_tests.rs new file mode 100644 index 0000000..7862cfc --- /dev/null +++ b/crates/metadb-server/tests/round2_cli_tests.rs @@ -0,0 +1,507 @@ +use std::sync::Arc; + +use metadb_server::{PersistentMetaDbCluster, ServerTomlConfig}; + +async fn start(dir: &std::path::Path) -> (Arc, String) { + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = dir.to_path_buf(); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap() +} + +async fn ctl(address: &str, arguments: &[&str]) -> std::process::Output { + tokio::process::Command::new(env!("CARGO_BIN_EXE_metadb-ctl")) + .args(["--endpoint", address]) + .args(arguments) + .output() + .await + .unwrap() +} + +fn benchmark_reports(output: &std::process::Output) -> Vec { + assert!( + output.status.success(), + "stdout={:?}; stderr={:?}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout) + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} + +async fn benchmark_rows( + cluster: &PersistentMetaDbCluster, + reports: &[serde_json::Value], +) -> Vec { + let pid = reports[0]["bench_pid"].as_u64().unwrap(); + cluster + .tx_router() + .listdir_paginated(pid, None, None, 100) + .await + .unwrap() + .0 +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cli_application_errors_must_have_a_failure_exit_status() { + let dir = tempfile::tempdir().unwrap(); + let (cluster, address) = start(dir.path()).await; + let output = ctl( + &address, + &[ + "sql", + "DELETE FROM MetaTable WHERE pid = 1 AND name = 'does-not-exist'", + ], + ) + .await; + cluster.shutdown().await.unwrap(); + let error: serde_json::Value = serde_json::from_slice(&output.stderr).unwrap(); + assert_eq!(error["error"]["operation"], "sql"); + assert!(error["error"]["message"] + .as_str() + .unwrap() + .contains("not found")); + assert!( + !output.status.success(), + "CLI returned success for failed deletion: stdout={}; stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cli_sql_quota_scan_must_print_returned_rows() { + let dir = tempfile::tempdir().unwrap(); + let (cluster, address) = start(dir.path()).await; + cluster + .tx_router() + .set_quota_limit(42, 123456) + .await + .unwrap(); + let output = ctl( + &address, + &[ + "sql", + "SELECT * FROM QuotaTable WHERE uid >= 0 ORDER BY uid ASC LIMIT 10", + ], + ) + .await; + cluster.shutdown().await.unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains("42") && stdout.contains("123456"), + "CLI discarded the nonempty Quotas response: stdout={stdout:?}; stderr={:?}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn repeated_cli_benchmark_must_not_report_failed_creates_as_completed() { + let dir = tempfile::tempdir().unwrap(); + let (cluster, address) = start(dir.path()).await; + let args = [ + "bench", + "--workload", + "hotspot-create", + "--concurrency", + "1", + "--ops", + "1", + ]; + let first = ctl(&address, &args).await; + assert!(first.status.success(), "{first:?}"); + let second = ctl(&address, &args).await; + let first_reports = benchmark_reports(&first); + let second_reports = benchmark_reports(&second); + let first_rows = benchmark_rows(&cluster, &first_reports).await; + let second_rows = benchmark_rows(&cluster, &second_reports).await; + cluster.shutdown().await.unwrap(); + assert_ne!( + first_reports[0]["bench_pid"], + second_reports[0]["bench_pid"] + ); + assert_eq!(first_rows.len(), 1); + assert_eq!(second_rows.len(), 1); + assert_ne!(first_rows[0].attr.inode_id, second_rows[0].attr.inode_id); + assert_ne!(first_rows[0].attr.fid, second_rows[0].attr.fid); + for reports in [&first_reports, &second_reports] { + assert_eq!(reports[1]["attempted"], 1); + assert_eq!(reports[1]["succeeded"], 1); + assert_eq!(reports[1]["failed"], 0); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cli_benchmark_must_execute_the_requested_number_of_operations() { + let dir = tempfile::tempdir().unwrap(); + let (cluster, address) = start(dir.path()).await; + for (ops, concurrency) in [(1, 2), (5, 2), (0, 3)] { + let ops_arg = ops.to_string(); + let concurrency_arg = concurrency.to_string(); + let output = ctl( + &address, + &[ + "bench", + "--workload", + "hotspot-create", + "--concurrency", + &concurrency_arg, + "--ops", + &ops_arg, + ], + ) + .await; + let reports = benchmark_reports(&output); + let rows = benchmark_rows(&cluster, &reports).await; + assert_eq!( + rows.len(), + ops, + "--ops {ops} with --concurrency {concurrency}: {reports:?}" + ); + assert_eq!(reports[1]["attempted"], ops); + assert_eq!(reports[1]["succeeded"], ops); + assert_eq!(reports[1]["failed"], 0); + } + cluster.shutdown().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn idle_metrics_connection_must_not_block_other_scrapes() { + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let dir = tempfile::tempdir().unwrap(); + let port_reservation = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = port_reservation.local_addr().unwrap(); + let config_path = dir.path().join("server.toml"); + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = dir.path().join("data"); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + cfg.node.metrics_listen_addr = Some(address.to_string()); + std::fs::write(&config_path, toml::to_string(&cfg).unwrap()).unwrap(); + drop(port_reservation); + let mut daemon = tokio::process::Command::new(env!("CARGO_BIN_EXE_metadbd")) + .arg("--config") + .arg(&config_path) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn() + .unwrap(); + let mut idle = tokio::time::timeout(Duration::from_secs(20), async { + loop { + if let Ok(socket) = tokio::net::TcpStream::connect(address).await { + break socket; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + // Let the daemon accept the first connection and enter its request read. + tokio::time::sleep(Duration::from_millis(50)).await; + let mut scrape = tokio::net::TcpStream::connect(address).await.unwrap(); + scrape + .write_all(b"GET /metrics HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + let mut response = vec![0u8; 4096]; + let while_idle = + tokio::time::timeout(Duration::from_millis(250), scrape.read(&mut response)).await; + let blocked = while_idle.is_err(); + if blocked { + drop(idle); + let count = tokio::time::timeout(Duration::from_secs(2), scrape.read(&mut response)) + .await + .unwrap() + .unwrap(); + assert!(String::from_utf8_lossy(&response[..count]).contains("HTTP/1.1 200 OK")); + } else { + let count = while_idle.unwrap().unwrap(); + assert!(String::from_utf8_lossy(&response[..count]).contains("HTTP/1.1 200 OK")); + assert_eq!( + tokio::time::timeout(Duration::from_secs(3), idle.read(&mut response)) + .await + .unwrap() + .unwrap(), + 0, + "idle metrics connection did not expire" + ); + } + let _idle_during_shutdown = tokio::net::TcpStream::connect(address).await.unwrap(); + #[cfg(unix)] + { + let status = tokio::process::Command::new("kill") + .args(["-INT", &daemon.id().unwrap().to_string()]) + .status() + .await + .unwrap(); + assert!(status.success()); + assert!( + tokio::time::timeout(Duration::from_secs(10), daemon.wait()) + .await + .unwrap() + .unwrap() + .success(), + "daemon failed graceful shutdown with an idle metrics connection" + ); + } + #[cfg(not(unix))] + { + daemon.start_kill().unwrap(); + daemon.wait().await.unwrap(); + } + assert!( + !blocked, + "one idle TCP connection blocked every metrics request until it disconnected" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cli_benchmark_counts_application_and_transport_failures() { + use metadb_proto::control_plane_service_server::ControlPlaneServiceServer; + use metadb_proto::metadata_service_server::MetadataServiceServer; + use tokio_stream::wrappers::TcpListenerStream; + + let dir = tempfile::tempdir().unwrap(); + let (cluster, _) = start(dir.path()).await; + let inner = Arc::new(metadb_server::MetaDbGrpcService::new( + cluster.global_manager().clone(), + cluster.tx_router().clone(), + cluster.proxy_router().clone(), + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let server = tokio::spawn( + tonic::transport::Server::builder() + .add_service(MetadataServiceServer::new(FaultyCreates { + inner: inner.clone(), + calls: std::sync::atomic::AtomicUsize::new(0), + })) + .add_service(ControlPlaneServiceServer::from_arc(inner)) + .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async { + let _ = stopped.await; + }), + ); + let output = ctl(&address, &["bench", "--concurrency", "2", "--ops", "5"]).await; + let reports: Vec = String::from_utf8_lossy(&output.stdout) + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + let rows = benchmark_rows(&cluster, &reports).await; + stop.send(()).unwrap(); + server.await.unwrap().unwrap(); + cluster.shutdown().await.unwrap(); + assert!( + !output.status.success(), + "benchmark hid partial failures: {output:?}" + ); + assert_eq!(reports[1]["attempted"], 5); + assert_eq!(reports[1]["succeeded"], 3); + assert_eq!(reports[1]["failed"], 2); + assert_eq!(reports[1]["application_failed"], 1); + assert_eq!(reports[1]["transport_failed"], 1); + assert_eq!(reports[1]["errors_by_kind"]["InjectedRejection"], 1); + assert_eq!(reports[1]["errors_by_kind"]["transport:Unavailable"], 1); + assert_eq!(rows.len(), 3); +} + +struct FaultyCreates { + inner: Arc, + calls: std::sync::atomic::AtomicUsize, +} + +#[tonic::async_trait] +impl metadb_proto::metadata_service_server::MetadataService for FaultyCreates { + async fn create_file( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + match self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) { + 0 => Err(tonic::Status::unavailable("injected transport failure")), + 1 => Ok(tonic::Response::new(metadb_proto::CreateFileResponse { + error: Some(metadb_proto::ErrorDetailProto { + error_kind: "InjectedRejection".into(), + message: "injected application rejection".into(), + ..Default::default() + }), + ..Default::default() + })), + _ => self.inner.create_file(request).await, + } + } + async fn mkdir( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.mkdir(request).await + } + async fn unlink( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.unlink(request).await + } + async fn rmdir( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.rmdir(request).await + } + async fn set_attr( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.set_attr(request).await + } + async fn lookup( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.lookup(request).await + } + async fn dir_stat( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.dir_stat(request).await + } + async fn get_dir_partition_stat( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.get_dir_partition_stat(request).await + } + async fn list_dir( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.list_dir(request).await + } + async fn execute_sql( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.execute_sql(request).await + } + async fn get_quota( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.get_quota(request).await + } + async fn set_quota_limit( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.set_quota_limit(request).await + } + async fn list_quotas( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.list_quotas(request).await + } + async fn get_secondary_index( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.get_secondary_index(request).await + } + async fn scan_secondary_index( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner.scan_secondary_index(request).await + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cli_sql_prints_secondary_indices_and_directory_cursor() { + let dir = tempfile::tempdir().unwrap(); + let (cluster, address) = start(dir.path()).await; + let a = cluster + .create_file_auto_id(1, "a", 7, 0o644, 10) + .await + .unwrap(); + cluster + .create_file_auto_id(1, "b", 7, 0o644, 20) + .await + .unwrap(); + cluster.run_maintenance_tick().await.unwrap(); + let listing = ctl( + &address, + &[ + "sql", + "SELECT * FROM MetaTable WHERE pid = 1 AND name > '' ORDER BY name ASC LIMIT 1", + ], + ) + .await; + assert!(listing.status.success(), "{listing:?}"); + let listing: serde_json::Value = serde_json::from_slice(&listing.stdout).unwrap(); + assert_eq!(listing["entries"].as_array().unwrap().len(), 1); + assert_eq!(listing["entries"][0]["name"], "a"); + assert_eq!(listing["entries"][0]["attr"]["inode_id"], a.attr.inode_id); + assert_eq!(listing["next_cursor"], "a"); + assert_eq!(listing["has_more"], true); + let output = ctl( + &address, + &[ + "sql", + "SELECT * FROM SecondaryIndexTable WHERE fid > 0 ORDER BY fid ASC LIMIT 10", + ], + ) + .await; + assert!(output.status.success(), "{output:?}"); + let indices: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(indices["indices"].as_array().unwrap().len(), 2); + assert_eq!(indices["indices"][0]["fid"], a.attr.fid.unwrap()); + let single = ctl( + &address, + &[ + "sql", + &format!( + "SELECT * FROM SecondaryIndexTable WHERE fid = {}", + a.attr.fid.unwrap() + ), + ], + ) + .await; + assert!(single.status.success(), "{single:?}"); + let single: serde_json::Value = serde_json::from_slice(&single.stdout).unwrap(); + assert_eq!(single["single_index"]["name"], "a"); + cluster.shutdown().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cli_bulk_benchmark_deletes_exactly_its_successful_creates() { + let dir = tempfile::tempdir().unwrap(); + let (cluster, address) = start(dir.path()).await; + let output = ctl( + &address, + &[ + "bench", + "--workload", + "bulk-ttl-delete", + "--concurrency", + "2", + "--ops", + "5", + ], + ) + .await; + let reports = benchmark_reports(&output); + assert_eq!(reports[1]["phase"], "create"); + assert_eq!(reports[2]["phase"], "delete"); + for phase in &reports[1..3] { + assert_eq!(phase["attempted"], 5); + assert_eq!(phase["succeeded"], 5); + assert_eq!(phase["failed"], 0); + } + assert_eq!(reports[3]["empty"], true); + assert!(benchmark_rows(&cluster, &reports).await.is_empty()); + cluster.shutdown().await.unwrap(); +} diff --git a/crates/metadb-server/tests/round2_mixed_shard_tests.rs b/crates/metadb-server/tests/round2_mixed_shard_tests.rs new file mode 100644 index 0000000..4704cff --- /dev/null +++ b/crates/metadb-server/tests/round2_mixed_shard_tests.rs @@ -0,0 +1,37 @@ +use metadb_server::{PersistentMetaDbCluster, ServerTomlConfig}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn all_tables_single_shard_configuration_must_support_split() { + let dir = tempfile::tempdir().unwrap(); + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = dir.path().to_path_buf(); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + cfg.sharding.bootstrap_table_shards = false; + let (cluster, _) = PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap(); + let tx = cluster.tx_router(); + for pid in [10u64, 20] { + tx.mkdir(1, &format!("dir{pid}"), pid, 7, 0o755) + .await + .unwrap(); + for i in 0..4 { + let fid = pid * 100 + i; + tx.create_file(pid, &format!("file{i}"), (1 << 63) + fid, fid, 7, 0o644, 10) + .await + .unwrap(); + } + } + cluster.run_maintenance_tick().await.unwrap(); + let result = cluster + .cell_manager() + .maybe_split_or_isolate_shard(1, true) + .await; + cluster.shutdown().await.unwrap(); + assert!( + result.is_ok(), + "valid single-shard configuration cannot split after quota/index delivery: {result:?}" + ); + assert!( + result.unwrap().is_some(), + "balanced multi-directory data did not split" + ); +} diff --git a/crates/metadb-server/tests/round2_name_flush_tests.rs b/crates/metadb-server/tests/round2_name_flush_tests.rs new file mode 100644 index 0000000..e92e984 --- /dev/null +++ b/crates/metadb-server/tests/round2_name_flush_tests.rs @@ -0,0 +1,60 @@ +use metadb_server::{PersistentMetaDbCluster, ServerTomlConfig}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn accepted_user_name_must_be_representable_in_sstable() { + let dir = tempfile::tempdir().unwrap(); + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = dir.path().to_path_buf(); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + let (cluster, _) = PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap(); + let created = cluster + .create_file_auto_id(1, &"a".repeat(70_000), 7, 0o644, 1) + .await; + let flush = cluster.flush_all().await; + let shutdown = cluster.shutdown().await; + if shutdown.is_err() { + cluster.cell_manager().host().shutdown_all().await.unwrap(); + } + assert!( + created.is_err() || (flush.is_ok() && shutdown.is_ok()), + "accepted long name cannot flush: create_ok={}, flush={flush:?}, shutdown={shutdown:?}", + created.is_ok() + ); +} + +#[test] +fn maximum_name_fits_the_unshared_write_intent_key_on_disk() { + use metadb_core::{HlcTimestamp, MetaKey, WriteIntentKey, MAX_USER_NAME_BYTES}; + use metadb_storage::{ + DurableShardCheckpoint, LifecycleStorageEngine, LsmEngineConfig, ShardedBlockCache, + }; + use std::sync::Arc; + let dir = tempfile::tempdir().unwrap(); + let cache = Arc::new(ShardedBlockCache::new(1 << 20)); + let open = || { + LifecycleStorageEngine::open_lsm(1, dir.path(), cache.clone(), LsmEngineConfig::default()) + .unwrap() + }; + let key = + WriteIntentKey::new(MetaKey::child(1, "a".repeat(MAX_USER_NAME_BYTES)).encode()).encode(); + assert_eq!(key.len(), u16::MAX as usize); + let engine = open(); + engine.apply_committed_put(key.clone(), b"payload".to_vec(), HlcTimestamp(100), 1); + engine + .seal_and_flush_with_hook( + DurableShardCheckpoint { + flushed_raft_index: 1, + flushed_max_hlc: HlcTimestamp(100), + ..Default::default() + }, + None, + ) + .unwrap(); + drop(engine); + let reopened = open(); + let snapshot = reopened.acquire_linearizable_snapshot(1).unwrap(); + assert_eq!( + reopened.get_with_snapshot(&key, &snapshot).unwrap(), + Some(b"payload".to_vec()) + ); +} diff --git a/crates/metadb-server/tests/round2_namespace_tests.rs b/crates/metadb-server/tests/round2_namespace_tests.rs new file mode 100644 index 0000000..b261197 --- /dev/null +++ b/crates/metadb-server/tests/round2_namespace_tests.rs @@ -0,0 +1,56 @@ +use metadb_server::{PersistentMetaDbCluster, ServerTomlConfig}; + +async fn start(dir: &std::path::Path) -> std::sync::Arc { + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = dir.to_path_buf(); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + PersistentMetaDbCluster::open_and_serve(cfg) + .await + .unwrap() + .0 +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn mkdir_must_not_alias_an_existing_directory_inode() { + let dir = tempfile::tempdir().unwrap(); + let cluster = start(dir.path()).await; + let tx = cluster.tx_router(); + tx.mkdir(1, "original", 10, 7, 0o755).await.unwrap(); + tx.create_file(10, "kept", (1 << 63) + 77, 77, 7, 0o644, 100) + .await + .unwrap(); + cluster.flush_all().await.unwrap(); + let result = tx.mkdir(1, "alias", 10, 8, 0o700).await; + let stat = tx.dirstat(10).await.unwrap(); + cluster.shutdown().await.unwrap(); + assert!( + result.is_err(), + "mkdir accepted a second dentry for inode 10; stats afterward: {stat:?}" + ); + assert_eq!(stat.num_children, 1); + assert_eq!(stat.uid, 7); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn mkdir_must_not_create_a_cycle_using_root_inode() { + let dir = tempfile::tempdir().unwrap(); + let cluster = start(dir.path()).await; + let result = cluster + .tx_router() + .mkdir(1, "root_alias", 1, 7, 0o700) + .await; + cluster.shutdown().await.unwrap(); + assert!(result.is_err(), "mkdir accepted root as its own child"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn zero_voter_scale_request_must_not_report_success() { + let dir = tempfile::tempdir().unwrap(); + let cluster = start(dir.path()).await; + let result = cluster.cell_manager().scale_out_replicas(0).await; + cluster.shutdown().await.unwrap(); + assert!( + result.is_err(), + "zero-voter request was acknowledged as successful: {result:?}" + ); +} diff --git a/crates/metadb-server/tests/round2_quota_order_tests.rs b/crates/metadb-server/tests/round2_quota_order_tests.rs new file mode 100644 index 0000000..a04d1e1 --- /dev/null +++ b/crates/metadb-server/tests/round2_quota_order_tests.rs @@ -0,0 +1,60 @@ +use metadb_core::{AuxOutboxEntry, AuxTaskPayload, HlcTimestamp}; +use metadb_server::{PersistentMetaDbCluster, ServerTomlConfig}; +use metadb_tx::TxRaftCmd; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cross_origin_quota_deltas_must_conserve_totals_under_reordered_delivery() { + let dir = tempfile::tempdir().unwrap(); + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = dir.path().to_path_buf(); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + let (cluster, _) = PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap(); + let (_, quota_group, _) = cluster.cell_manager().lookup_local_handle(2).unwrap(); + // Valid causal history: create on origin 1, rename to origin 4, unlink there. + // Deliver the unlink's independent origin stream before the create stream. + for entry in [ + AuxOutboxEntry { + origin_shard_id: 4, + origin_raft_index: 7, + task_ordinal: 0, + hlc: HlcTimestamp::new(200, 0), + payload: AuxTaskPayload::QuotaDelta { + uid: 9, + delta_bytes: -100, + delta_inodes: -1, + prev_uid_cursor: (0, 0), + }, + }, + AuxOutboxEntry { + origin_shard_id: 1, + origin_raft_index: 3, + task_ordinal: 0, + hlc: HlcTimestamp::new(100, 0), + payload: AuxTaskPayload::QuotaDelta { + uid: 9, + delta_bytes: 100, + delta_inodes: 1, + prev_uid_cursor: (0, 0), + }, + }, + ] { + quota_group + .propose( + quota_group.current_epoch(), + cluster.tx_router().clock().now(), + bincode::serialize(&TxRaftCmd::ApplyAuxBatch { + entries: vec![entry], + }) + .unwrap(), + ) + .await + .unwrap(); + } + let quota = cluster.tx_router().get_quota(9).await.unwrap().unwrap(); + cluster.shutdown().await.unwrap(); + assert_eq!( + (quota.used_bytes, quota.used_inodes), + (0, 0), + "two valid per-origin streams have net zero, but delivery order changed quota" + ); +} diff --git a/crates/metadb-server/tests/round2_restart_routes_tests.rs b/crates/metadb-server/tests/round2_restart_routes_tests.rs new file mode 100644 index 0000000..bb6ce72 --- /dev/null +++ b/crates/metadb-server/tests/round2_restart_routes_tests.rs @@ -0,0 +1,74 @@ +use metadb_server::{PersistentMetaDbCluster, ServerTomlConfig}; +use openraft::async_runtime::watch::WatchReceiver; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn advertised_local_peer_address_must_follow_listener_after_restart() { + let dir = tempfile::tempdir().unwrap(); + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = dir.path().to_path_buf(); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + let (first, old_address) = PersistentMetaDbCluster::open_and_serve(cfg.clone()) + .await + .unwrap(); + first.shutdown().await.unwrap(); + drop(first); + let _occupied_old_port = std::net::TcpListener::bind(&old_address).unwrap(); + let (second, new_address) = PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap(); + let advertised: Vec<_> = second + .cell_manager() + .list_routes() + .iter() + .map(|route| (route.shard_id, route.peers.get(&1).unwrap().0.clone())) + .collect(); + for route in second.cell_manager().list_routes() { + let (_, group, _) = second + .cell_manager() + .lookup_local_handle(route.shard_id) + .unwrap(); + let membership = group + .raft() + .metrics() + .borrow_watched() + .membership_config + .clone(); + assert_eq!( + membership.membership().get_node(&1).unwrap().addr, + new_address + ); + assert_eq!(route.epoch, group.current_epoch()); + } + second.shutdown().await.unwrap(); + assert_ne!(old_address, new_address); + assert!(advertised.iter().all(|(_, address)| address == &new_address), + "restarted listener {new_address}, but advertised routes point at stopped listener {old_address}: {advertised:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn reopening_another_nodes_directory_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = dir.path().to_path_buf(); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + let (first, _) = PersistentMetaDbCluster::open_and_serve(cfg.clone()) + .await + .unwrap(); + // A registered peer is not the owner of this node's local directory. + first + .cell_manager() + .register_node(2, "127.0.0.1:1".to_string()) + .unwrap(); + first.shutdown().await.unwrap(); + drop(first); + let state_path = dir.path().join("cell_1/CELL_MANAGER_STATE"); + let before = std::fs::read(&state_path).unwrap(); + cfg.node.node_id = 2; + let result = PersistentMetaDbCluster::open_and_serve(cfg).await; + if let Ok((cluster, _)) = &result { + cluster.shutdown().await.unwrap(); + } + assert!( + result.is_err(), + "a different node ID silently adopted existing shard state" + ); + assert_eq!(std::fs::read(state_path).unwrap(), before); +} diff --git a/crates/metadb-storage/src/backend.rs b/crates/metadb-storage/src/backend.rs index 360d1c6..9a0ce10 100644 --- a/crates/metadb-storage/src/backend.rs +++ b/crates/metadb-storage/src/backend.rs @@ -125,8 +125,41 @@ impl LsmStorageBackend { block_cache: Arc, bloom_bits_per_key: usize, ) -> Result { - let lsm_dir = lsm_dir.as_ref().to_path_buf(); - fs::create_dir_all(&lsm_dir)?; + Self::open_internal( + shard_id, + lsm_dir.as_ref(), + block_cache, + bloom_bits_per_key, + true, + ) + } + + pub fn open_existing( + shard_id: ShardId, + lsm_dir: impl AsRef, + block_cache: Arc, + bloom_bits_per_key: usize, + ) -> Result { + Self::open_internal( + shard_id, + lsm_dir.as_ref(), + block_cache, + bloom_bits_per_key, + false, + ) + } + + fn open_internal( + shard_id: ShardId, + lsm_dir: &Path, + block_cache: Arc, + bloom_bits_per_key: usize, + recover: bool, + ) -> Result { + let lsm_dir = lsm_dir.to_path_buf(); + if recover { + fs::create_dir_all(&lsm_dir)?; + } let lock_path = lsm_dir.join("LOCK"); let lock_file = OpenOptions::new() .read(true) @@ -140,12 +173,24 @@ impl LsmStorageBackend { lock_path, e )) })?; - let (manifest, recovered) = ManifestLog::open_or_create(&lsm_dir)?; + let (manifest, recovered) = if recover { + ManifestLog::open_or_create(&lsm_dir)? + } else { + ManifestLog::open_existing(&lsm_dir)? + }; let mut readers = Vec::with_capacity(recovered.live_files.len()); for meta in recovered.live_files.values() { let sst_path = lsm_dir.join(sst_file_name(meta.file_id)); let reader = SstableReader::open(shard_id, sst_path, block_cache.clone())?; + if reader.meta() != meta { + return Err(MetaDbError::ManifestCorrupted(format!( + "SST {} metadata differs from its MANIFEST record: expected {:?}, found {:?}", + meta.file_id, + meta, + reader.meta() + ))); + } readers.push(Arc::new(reader)); } @@ -563,6 +608,9 @@ impl StorageBackend for LsmStorageBackend { } fn create_checkpoint(&self, dest_dir: &Path) -> Result { + // Pinned readers keep file handles alive, but compaction unlinks their + // paths. Hold the same lock until all checkpoint links are durable. + let _comp_guard = self.compaction_lock.lock().unwrap(); if dest_dir.exists() { return Err(MetaDbError::StorageIoError(format!( "checkpoint destination {:?} already exists", diff --git a/crates/metadb-storage/src/cache.rs b/crates/metadb-storage/src/cache.rs index 97f2785..0183d30 100644 --- a/crates/metadb-storage/src/cache.rs +++ b/crates/metadb-storage/src/cache.rs @@ -17,11 +17,17 @@ pub struct BlockCacheKey { pub block_offset: u64, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct ScopedCacheKey { + reader_id: u64, + block: BlockCacheKey, +} + struct LruShard { capacity_bytes: usize, used_bytes: usize, - order: VecDeque, - entries: HashMap>, + order: VecDeque, + entries: HashMap>, } impl LruShard { @@ -34,7 +40,7 @@ impl LruShard { } } - fn get(&mut self, key: &BlockCacheKey) -> Option> { + fn get(&mut self, key: &ScopedCacheKey) -> Option> { let block = self.entries.get(key)?.clone(); if self.order.back() != Some(key) { self.order.retain(|k| k != key); @@ -43,7 +49,7 @@ impl LruShard { Some(block) } - fn insert(&mut self, key: BlockCacheKey, block: Arc) { + fn insert(&mut self, key: ScopedCacheKey, block: Arc) { let block_bytes = block.byte_size().max(64); if block_bytes > self.capacity_bytes { return; @@ -80,16 +86,41 @@ impl ShardedBlockCache { } pub fn get(&self, key: &BlockCacheKey) -> Option> { - let idx = Self::shard_index(key); - self.shards[idx].lock().unwrap().get(key) + self.get_for_reader(0, key) } pub fn insert(&self, key: BlockCacheKey, block: Arc) { + self.insert_for_reader(0, key, block); + } + + pub(crate) fn get_for_reader( + &self, + reader_id: u64, + key: &BlockCacheKey, + ) -> Option> { + let key = ScopedCacheKey { + reader_id, + block: *key, + }; + let idx = Self::shard_index(&key); + self.shards[idx].lock().unwrap().get(&key) + } + + pub(crate) fn insert_for_reader( + &self, + reader_id: u64, + key: BlockCacheKey, + block: Arc, + ) { + let key = ScopedCacheKey { + reader_id, + block: key, + }; let idx = Self::shard_index(&key); self.shards[idx].lock().unwrap().insert(key, block); } - fn shard_index(key: &BlockCacheKey) -> usize { + fn shard_index(key: &ScopedCacheKey) -> usize { let mut hasher = std::collections::hash_map::DefaultHasher::new(); key.hash(&mut hasher); (hasher.finish() as usize) & (NUM_CACHE_SHARDS - 1) diff --git a/crates/metadb-storage/src/engine.rs b/crates/metadb-storage/src/engine.rs index 6fbec1f..464d104 100644 --- a/crates/metadb-storage/src/engine.rs +++ b/crates/metadb-storage/src/engine.rs @@ -83,12 +83,33 @@ impl LifecycleStorageEngine { block_cache: Arc, config: LsmEngineConfig, ) -> Result { - let lsm = Arc::new(LsmStorageBackend::open( + let lsm = + LsmStorageBackend::open(shard_id, lsm_dir, block_cache, config.bloom_bits_per_key)?; + Ok(Self::with_lsm_backend(shard_id, lsm, config)) + } + + /// Opens a complete existing checkpoint without creating or repairing its manifest. + pub fn open_lsm_existing( + shard_id: ShardId, + lsm_dir: impl AsRef, + block_cache: Arc, + config: LsmEngineConfig, + ) -> Result { + let lsm = LsmStorageBackend::open_existing( shard_id, lsm_dir, block_cache, config.bloom_bits_per_key, - )?); + )?; + Ok(Self::with_lsm_backend(shard_id, lsm, config)) + } + + fn with_lsm_backend( + shard_id: ShardId, + lsm: LsmStorageBackend, + config: LsmEngineConfig, + ) -> Self { + let lsm = Arc::new(lsm); let cp = lsm.current_version().checkpoint.clone(); let mvcc = Arc::new(MemMvccStore::with_recovered_watermarks( config.mvcc_retention_ms, @@ -99,14 +120,14 @@ impl LifecycleStorageEngine { config.tombstone_density_threshold, config.tombstone_min_ratio_permille, )); - Ok(Self { + Self { shard_id, raft_log: Arc::new(RaftLog::new(shard_id)), mvcc, compactor, backend: lsm, auto_locality_compaction: config.auto_locality_compaction, - }) + } } pub fn shard_id(&self) -> ShardId { diff --git a/crates/metadb-storage/src/manifest.rs b/crates/metadb-storage/src/manifest.rs index b4f90a4..b3abc54 100644 --- a/crates/metadb-storage/src/manifest.rs +++ b/crates/metadb-storage/src/manifest.rs @@ -98,6 +98,18 @@ impl VersionEdit { } pub fn decode_payload(raw: &[u8]) -> Result { + let (edit, consumed) = Self::decode_prefix(raw)?; + if consumed != raw.len() { + return Err(MetaDbError::ManifestCorrupted( + "trailing bytes inside VersionEdit record".to_string(), + )); + } + Ok(edit) + } + + // The payload grammar is self-delimiting. Recovery also uses this to detect + // an intact edit hidden by a corrupted (unchecksummed) frame length. + fn decode_prefix(raw: &[u8]) -> Result<(Self, usize)> { let mut pos = 0usize; let read_u8 = |p: &mut usize| -> Result { if *p >= raw.len() { @@ -110,7 +122,7 @@ impl VersionEdit { Ok(v) }; let read_u32 = |p: &mut usize| -> Result { - if *p + 4 > raw.len() { + if raw.len().saturating_sub(*p) < 4 { return Err(MetaDbError::ManifestCorrupted( "unexpected EOF reading u32 in VersionEdit".to_string(), )); @@ -120,7 +132,7 @@ impl VersionEdit { Ok(v) }; let read_u64 = |p: &mut usize| -> Result { - if *p + 8 > raw.len() { + if raw.len().saturating_sub(*p) < 8 { return Err(MetaDbError::ManifestCorrupted( "unexpected EOF reading u64 in VersionEdit".to_string(), )); @@ -131,7 +143,7 @@ impl VersionEdit { }; let read_vec = |p: &mut usize| -> Result> { let len = read_u32(p)? as usize; - if *p + len > raw.len() { + if len > raw.len().saturating_sub(*p) { return Err(MetaDbError::ManifestCorrupted( "unexpected EOF reading bytes in VersionEdit".to_string(), )); @@ -200,6 +212,11 @@ impl VersionEdit { }; let deleted_count = read_u32(&mut pos)? as usize; + if deleted_count > raw.len().saturating_sub(pos) / 9 { + return Err(MetaDbError::ManifestCorrupted( + "deleted file count exceeds VersionEdit bytes".to_string(), + )); + } let mut deleted_files = Vec::with_capacity(deleted_count); for _ in 0..deleted_count { let level = read_u8(&mut pos)?; @@ -208,6 +225,12 @@ impl VersionEdit { } let added_count = read_u32(&mut pos)? as usize; + // Each file uses 65 bytes, even with empty min/max keys. + if added_count > raw.len().saturating_sub(pos) / 65 { + return Err(MetaDbError::ManifestCorrupted( + "added file count exceeds VersionEdit bytes".to_string(), + )); + } let mut added_files = Vec::with_capacity(added_count); for _ in 0..added_count { let level = read_u8(&mut pos)?; @@ -234,19 +257,16 @@ impl VersionEdit { }); } - if pos != raw.len() { - return Err(MetaDbError::ManifestCorrupted( - "trailing bytes inside VersionEdit record".to_string(), - )); - } - - Ok(Self { - next_file_id, - next_flush_generation, - checkpoint, - deleted_files, - added_files, - }) + Ok(( + Self { + next_file_id, + next_flush_generation, + checkpoint, + deleted_files, + added_files, + }, + pos, + )) } } @@ -271,18 +291,43 @@ impl ManifestLog { /// Opens an existing `lsm_dir` (replaying `CURRENT` + cleaning `.sst.tmp` / orphan `.sst`s) /// or initializes a fresh `MANIFEST-000001`. pub fn open_or_create(lsm_dir: &Path) -> Result<(Self, RecoveredManifestState)> { - fs::create_dir_all(lsm_dir)?; + Self::open_internal(lsm_dir, true) + } + + /// Verifies an existing checkpoint without initializing, repairing, or deleting files. + pub fn open_existing(lsm_dir: &Path) -> Result<(Self, RecoveredManifestState)> { + Self::open_internal(lsm_dir, false) + } + + fn open_internal(lsm_dir: &Path, recover: bool) -> Result<(Self, RecoveredManifestState)> { + if recover { + fs::create_dir_all(lsm_dir)?; + } let current_path = lsm_dir.join("CURRENT"); - if !current_path.exists() { + if !current_path.try_exists()? { + if !recover { + return Err(MetaDbError::ManifestCorrupted( + "CURRENT is missing from an existing checkpoint".to_string(), + )); + } + // The backend creates LOCK before opening the manifest. Anything + // else may belong to an existing database or interrupted rotation; + // never overwrite it or run orphan cleanup without its CURRENT. + for entry in fs::read_dir(lsm_dir)? { + if entry?.file_name() != "LOCK" { + return Err(MetaDbError::ManifestCorrupted( + "CURRENT is missing from a nonempty database directory".to_string(), + )); + } + } let seq = 1u64; let manifest_name = format!("MANIFEST-{:06}", seq); let manifest_path = lsm_dir.join(&manifest_name); let mut f = OpenOptions::new() - .create(true) + .create_new(true) .read(true) .write(true) - .truncate(true) .open(&manifest_path)?; let init_edit = VersionEdit { next_file_id: Some(1), @@ -324,6 +369,11 @@ impl ManifestLog { let mut raw = Vec::new(); File::open(&manifest_path)?.read_to_end(&mut raw)?; + if raw.is_empty() { + return Err(MetaDbError::ManifestCorrupted( + "MANIFEST has no baseline record".to_string(), + )); + } let mut next_file_id = 1u64; let mut next_flush_generation = 1u64; @@ -335,7 +385,12 @@ impl ManifestLog { while offset < raw.len() { // Only an incomplete/torn final record at EOF may be truncated (Resolution B3). - if offset + 8 > raw.len() { + if raw.len() - offset < 8 { + if offset == 0 { + return Err(MetaDbError::ManifestCorrupted( + "incomplete MANIFEST baseline header".to_string(), + )); + } truncate_to = Some(offset); break; } @@ -345,7 +400,20 @@ impl ManifestLog { let record_end = offset.saturating_add(8).saturating_add(payload_len); if record_end > raw.len() { - // Torn final record at EOF! + if offset == 0 { + return Err(MetaDbError::ManifestCorrupted( + "incomplete MANIFEST baseline payload".to_string(), + )); + } + if VersionEdit::decode_prefix(&raw[offset + 8..]).is_ok() { + return Err(MetaDbError::ManifestCorrupted(format!( + "MANIFEST frame length exceeds complete edit at offset {}", + offset + ))); + } + // No complete edit remains: recover an incomplete append tail. + // The v1 frame length has no checksum, so simultaneous length + // and payload damage cannot always be distinguished from a tear. truncate_to = Some(offset); break; } @@ -353,12 +421,6 @@ impl ManifestLog { let payload = &raw[offset + 8..record_end]; let actual_crc = checksum_crc32c(payload); if actual_crc != expected_crc { - // A bad CRC on the final record is a torn tail only when a valid record - // already precedes it. Truncating the sole baseline would drop every SST. - if record_end == raw.len() && offset > 0 { - truncate_to = Some(offset); - break; - } return Err(MetaDbError::ManifestCorrupted(format!( "MANIFEST CRC32C mismatch at offset {} (expected 0x{:08x}, got 0x{:08x})", offset, expected_crc, actual_crc @@ -366,6 +428,16 @@ impl ManifestLog { } let edit = VersionEdit::decode_payload(payload)?; + if offset == 0 + && (edit.next_file_id.is_none() + || edit.next_flush_generation.is_none() + || edit.checkpoint.is_none() + || !edit.deleted_files.is_empty()) + { + return Err(MetaDbError::ManifestCorrupted( + "MANIFEST first record is not a complete baseline".to_string(), + )); + } if let Some(id) = edit.next_file_id { next_file_id = next_file_id.max(id); } @@ -379,8 +451,15 @@ impl ManifestLog { live_files.remove(&del); } for add in edit.added_files { - next_file_id = next_file_id.max(add.file_id + 1); - next_flush_generation = next_flush_generation.max(add.flush_generation + 1); + let following_id = add.file_id.checked_add(1).ok_or_else(|| { + MetaDbError::ManifestCorrupted("SST file id overflow".to_string()) + })?; + let following_generation = + add.flush_generation.checked_add(1).ok_or_else(|| { + MetaDbError::ManifestCorrupted("SST flush generation overflow".to_string()) + })?; + next_file_id = next_file_id.max(following_id); + next_flush_generation = next_flush_generation.max(following_generation); live_files.insert((add.level, add.file_id), add); } @@ -392,12 +471,19 @@ impl ManifestLog { .append(true) .open(&manifest_path)?; if let Some(valid_len) = truncate_to { + if !recover { + return Err(MetaDbError::ManifestCorrupted( + "incomplete MANIFEST in existing checkpoint".to_string(), + )); + } let trunc_file = OpenOptions::new().write(true).open(&manifest_path)?; trunc_file.set_len(valid_len as u64)?; trunc_file.sync_data()?; } - Self::cleanup_orphan_files(lsm_dir, &live_files, &manifest_name)?; + if recover { + Self::cleanup_orphan_files(lsm_dir, &live_files, &manifest_name)?; + } Ok(( Self { diff --git a/crates/metadb-storage/src/sstable.rs b/crates/metadb-storage/src/sstable.rs index 0d46e88..3676d76 100644 --- a/crates/metadb-storage/src/sstable.rs +++ b/crates/metadb-storage/src/sstable.rs @@ -4,6 +4,7 @@ use std::fs::{self, File, OpenOptions}; use std::io::Write; use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use serde::{Deserialize, Serialize}; @@ -20,6 +21,11 @@ pub const SSTABLE_MAGIC: u64 = 0x4D45_5441_5353_5431; // "METASST1" pub const SSTABLE_FORMAT_VERSION: u32 = 1; pub const SSTABLE_FOOTER_LEN: usize = 64; +// File IDs are local to a database history and may be reused after snapshot +// installation. Give each opened immutable file its own cache namespace, also +// isolating old readers that remain pinned during a replacement. +static NEXT_READER_ID: AtomicU64 = AtomicU64::new(1); + /// Durable metadata describing an SSTable file in `MetaBlock` and `MANIFEST`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SstFileMeta { @@ -243,6 +249,7 @@ pub struct SstableReader { bloom: BloomFilter, index: Vec, block_cache: Arc, + cache_reader_id: u64, } impl SstableReader { @@ -292,24 +299,74 @@ impl SstableReader { } let filter_offset = u64::from_le_bytes(footer[0..8].try_into().unwrap()); - let filter_len = u64::from_le_bytes(footer[8..16].try_into().unwrap()) as usize; + let filter_len = u64::from_le_bytes(footer[8..16].try_into().unwrap()); let index_offset = u64::from_le_bytes(footer[16..24].try_into().unwrap()); - let index_len = u64::from_le_bytes(footer[24..32].try_into().unwrap()) as usize; + let index_len = u64::from_le_bytes(footer[24..32].try_into().unwrap()); let meta_offset = u64::from_le_bytes(footer[32..40].try_into().unwrap()); - let meta_len = u64::from_le_bytes(footer[40..48].try_into().unwrap()) as usize; + let meta_len = u64::from_le_bytes(footer[40..48].try_into().unwrap()); + + // Validate all footer extents before converting lengths or allocating. + let data_end = file_size - SSTABLE_FOOTER_LEN as u64; + if filter_offset == 0 + || filter_len < 6 + || index_len < 8 + || meta_len < 53 + || filter_offset.checked_add(filter_len) != Some(index_offset) + || index_offset.checked_add(index_len) != Some(meta_offset) + || meta_offset.checked_add(meta_len) != Some(data_end) + { + return Err(MetaDbError::ManifestCorrupted( + "invalid SSTable section bounds".to_string(), + )); + } - let mut filter_buf = vec![0u8; filter_len]; - file.read_exact_at(&mut filter_buf, filter_offset)?; + let filter_buf = Self::read_section(&file, filter_offset, filter_len)?; let bloom = BloomFilter::from_raw_checked(&filter_buf)?; - let mut index_buf = vec![0u8; index_len]; - file.read_exact_at(&mut index_buf, index_offset)?; + let index_buf = Self::read_section(&file, index_offset, index_len)?; let index = Self::decode_index_block(&index_buf)?; + for entry in &index { + if entry.block_len < 8 + || entry + .block_offset + .checked_add(u64::from(entry.block_len)) + .is_none_or(|end| end > filter_offset) + { + return Err(MetaDbError::ManifestCorrupted( + "SSTable data block exceeds data section".to_string(), + )); + } + } - let mut meta_buf = vec![0u8; meta_len]; - file.read_exact_at(&mut meta_buf, meta_offset)?; + let meta_buf = Self::read_section(&file, meta_offset, meta_len)?; let file_id = Self::parse_file_id_from_path(&path)?; let meta = Self::decode_meta_block(&meta_buf, file_id, file_size)?; + if meta.level as usize >= crate::version_set::NUM_LSM_LEVELS + || meta.min_key > meta.max_key + || meta.num_entries == 0 + || meta.num_tombstones > meta.num_entries + || meta.num_entries > filter_offset / 9 + || index.is_empty() + { + return Err(MetaDbError::ManifestCorrupted( + "invalid SSTable metadata".to_string(), + )); + } + let mut cache_reader_id = NEXT_READER_ID.load(Ordering::Relaxed); + loop { + let next_id = cache_reader_id.checked_add(1).ok_or_else(|| { + MetaDbError::StorageIoError("SSTable cache reader IDs exhausted".to_string()) + })?; + match NEXT_READER_ID.compare_exchange_weak( + cache_reader_id, + next_id, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(observed) => cache_reader_id = observed, + } + } Ok(Self { shard_id, @@ -319,6 +376,7 @@ impl SstableReader { bloom, index, block_cache, + cache_reader_id, }) } @@ -417,7 +475,7 @@ impl SstableReader { /// Reads every entry across all data blocks in this SSTable (used during compaction). pub fn read_all_entries(&self) -> Result, BlockEntryValue)>> { - let mut out = Vec::with_capacity(self.meta.num_entries as usize); + let mut out = Vec::new(); for ie in &self.index { let block = self.load_data_block(ie)?; out.extend(block.iter_all()?); @@ -425,22 +483,83 @@ impl SstableReader { Ok(out) } + /// Verifies every data block before accepting a transferred checkpoint. + /// Work is bounded to one block at a time rather than materializing the entire SST. + pub fn verify_all_data_blocks(&self) -> Result<()> { + let mut entries = 0u64; + let mut tombstones = 0u64; + let mut first_key = None; + let mut last_key = None; + let mut next_offset = 0; + for index_entry in &self.index { + if index_entry.block_offset != next_offset { + return Err(MetaDbError::ManifestCorrupted( + "noncontiguous SST data blocks".to_string(), + )); + } + let block = self.load_data_block(index_entry)?; + for (key, value) in block.iter_all()? { + if last_key.as_ref().is_some_and(|previous| previous >= &key) { + return Err(MetaDbError::ManifestCorrupted( + "unordered SST data keys".to_string(), + )); + } + entries += 1; + tombstones += u64::from(matches!(value, BlockEntryValue::PhysicalTombstone)); + first_key.get_or_insert_with(|| key.clone()); + last_key = Some(key); + } + if last_key.as_deref() != Some(index_entry.last_key.as_slice()) { + return Err(MetaDbError::ManifestCorrupted( + "SST index key differs from data block".to_string(), + )); + } + next_offset += u64::from(index_entry.block_len); + } + if entries != self.meta.num_entries + || tombstones != self.meta.num_tombstones + || first_key.as_deref() != Some(self.meta.min_key.as_slice()) + || last_key.as_deref() != Some(self.meta.max_key.as_slice()) + { + return Err(MetaDbError::ManifestCorrupted( + "SST data differs from file metadata".to_string(), + )); + } + Ok(()) + } + fn load_data_block(&self, ie: &IndexBlockEntry) -> Result> { let cache_key = BlockCacheKey { shard_id: self.shard_id, sst_file_id: self.meta.file_id, block_offset: ie.block_offset, }; - if let Some(cached) = self.block_cache.get(&cache_key) { + if let Some(cached) = self + .block_cache + .get_for_reader(self.cache_reader_id, &cache_key) + { return Ok(cached); } - let mut raw = vec![0u8; ie.block_len as usize]; - self.file.read_exact_at(&mut raw, ie.block_offset)?; + let raw = Self::read_section(&self.file, ie.block_offset, u64::from(ie.block_len))?; let parsed = Arc::new(DataBlock::from_raw_checked(&raw)?); - self.block_cache.insert(cache_key, parsed.clone()); + self.block_cache + .insert_for_reader(self.cache_reader_id, cache_key, parsed.clone()); Ok(parsed) } + fn read_section(file: &File, offset: u64, len: u64) -> Result> { + let len = usize::try_from(len).map_err(|_| { + MetaDbError::CodecError("SSTable section length exceeds address space".to_string()) + })?; + let mut raw = Vec::new(); + raw.try_reserve_exact(len).map_err(|_| { + MetaDbError::CodecError("SSTable section allocation failed".to_string()) + })?; + raw.resize(len, 0); + file.read_exact_at(&mut raw, offset)?; + Ok(raw) + } + fn decode_index_block(raw: &[u8]) -> Result> { let body = verify_crc32c_le(raw, "SSTable IndexBlock")?; if body.len() < 4 { @@ -449,6 +568,11 @@ impl SstableReader { )); } let count = u32::from_le_bytes(body[0..4].try_into().unwrap()) as usize; + if count > (body.len() - 4) / 16 { + return Err(MetaDbError::CodecError( + "IndexBlock count exceeds available bytes".to_string(), + )); + } let mut pos = 4usize; let mut out = Vec::with_capacity(count); for _ in 0..count { diff --git a/crates/metadb-storage/tests/adversarial_recovery_tests.rs b/crates/metadb-storage/tests/adversarial_recovery_tests.rs new file mode 100644 index 0000000..4ae8d55 --- /dev/null +++ b/crates/metadb-storage/tests/adversarial_recovery_tests.rs @@ -0,0 +1,317 @@ +//! Regression tests for fail-closed recovery. Every rejected reopen must preserve the +//! original files so an operator can restore/replay rather than lose forensic data. +use metadb_core::{HlcTimestamp, MetaDbError}; +use metadb_storage::{ + DurableShardCheckpoint, LifecycleStorageEngine, LsmEngineConfig, ShardedBlockCache, +}; +use std::collections::BTreeMap; +use std::fs; +use std::sync::Arc; +use tempfile::tempdir; + +fn seed(path: &std::path::Path, compact: bool) { + let engine = reopen(path).unwrap(); + engine.apply_committed_put( + b"key".to_vec(), + b"durable".to_vec(), + HlcTimestamp::new(100, 0), + 1, + ); + engine + .seal_and_flush_with_hook( + DurableShardCheckpoint { + flushed_raft_index: 1, + flushed_max_hlc: HlcTimestamp::new(100, 0), + ..Default::default() + }, + None, + ) + .unwrap(); + if compact { + engine.try_compact_range(b"a", b"z").unwrap(); + } +} +fn reopen(path: &std::path::Path) -> metadb_core::Result { + LifecycleStorageEngine::open_lsm( + 1, + path, + Arc::new(ShardedBlockCache::new(1024 * 1024)), + LsmEngineConfig::default(), + ) +} +fn reject_without_deleting(path: &std::path::Path) { + let before: BTreeMap<_, _> = fs::read_dir(path) + .unwrap() + .map(|e| e.unwrap().path()) + .filter(|p| p.file_name().unwrap() != "LOCK") + .map(|p| { + let raw = fs::read(&p).unwrap(); + (p, raw) + }) + .collect(); + assert!(before + .keys() + .any(|p| p.extension().and_then(|x| x.to_str()) == Some("sst"))); + let result = reopen(path); + assert!( + matches!(result, Err(MetaDbError::ManifestCorrupted(_))), + "corrupt manifest must fail closed instead of accepting a partial database" + ); + for (file, raw) in before { + assert_eq!( + fs::read(&file).unwrap(), + raw, + "rejected reopen modified {}", + file.display() + ); + } +} + +#[test] +fn incomplete_baseline_header_preserves_database() { + for kept in 0..8 { + let dir = tempdir().unwrap(); + seed(dir.path(), false); + let mf = dir.path().join("MANIFEST-000001"); + let raw = fs::read(&mf).unwrap(); + fs::write(&mf, &raw[..kept]).unwrap(); + reject_without_deleting(dir.path()); + } +} + +#[test] +fn impossible_manifest_counts_are_rejected_before_allocation() { + use metadb_storage::VersionEdit; + for count_offset in [4, 8] { + let mut payload = VersionEdit::default().encode_payload(); + payload[count_offset..count_offset + 4].copy_from_slice(&u32::MAX.to_le_bytes()); + assert!(matches!( + VersionEdit::decode_payload(&payload), + Err(MetaDbError::ManifestCorrupted(_)) + )); + } +} +#[test] +fn corrupt_baseline_length_must_not_erase_ssts() { + let dir = tempdir().unwrap(); + seed(dir.path(), false); + let mf = dir.path().join("MANIFEST-000001"); + let mut raw = fs::read(&mf).unwrap(); + raw[3] ^= 0x80; + fs::write(&mf, raw).unwrap(); + reject_without_deleting(dir.path()); +} +#[test] +fn complete_crc_bad_tail_after_compaction_must_not_roll_back_manifest() { + let dir = tempdir().unwrap(); + seed(dir.path(), true); + let mf = dir.path().join("MANIFEST-000001"); + let mut raw = fs::read(&mf).unwrap(); + let last = raw.len() - 1; + raw[last] ^= 1; + fs::write(&mf, raw).unwrap(); + reject_without_deleting(dir.path()); +} +#[test] +fn missing_current_must_not_initialize_over_existing_database() { + let dir = tempdir().unwrap(); + seed(dir.path(), false); + fs::remove_file(dir.path().join("CURRENT")).unwrap(); + reject_without_deleting(dir.path()); +} +#[test] +fn sole_baseline_truncation_is_corruption_and_preserves_ssts() { + // A standalone checkpoint is exactly one MANIFEST baseline referencing all SSTs. + for removed in 1..=64 { + let dir = tempdir().unwrap(); + let source = dir.path().join("source"); + seed(&source, false); + let engine = reopen(&source).unwrap(); + let checkpoint = dir.path().join("checkpoint"); + engine.create_checkpoint(&checkpoint).unwrap(); + let mf = checkpoint.join("MANIFEST-000001"); + let raw = fs::read(&mf).unwrap(); + fs::write(&mf, &raw[..raw.len() - removed]).unwrap(); + reject_without_deleting(&checkpoint); + } +} +#[test] +fn incomplete_new_tail_recovers_last_durable_state() { + // Tail recovery remains allowed after a complete baseline/edits, including + // every truncation of an append's header and payload. + use metadb_storage::VersionEdit; + let edit = VersionEdit { + next_file_id: Some(999), + ..Default::default() + }; + let payload = edit.encode_payload(); + let mut frame = Vec::new(); + frame.extend_from_slice(&(payload.len() as u32).to_le_bytes()); + frame.extend_from_slice(&metadb_core::checksum_crc32c(&payload).to_le_bytes()); + frame.extend_from_slice(&payload); + for kept in 1..frame.len() { + let dir = tempdir().unwrap(); + seed(dir.path(), false); + let mf = dir.path().join("MANIFEST-000001"); + let mut raw = fs::read(&mf).unwrap(); + raw.extend_from_slice(&frame[..kept]); + fs::write(&mf, raw).unwrap(); + let engine = reopen(dir.path()).unwrap(); + assert_eq!(engine.durable_checkpoint().flushed_raft_index, 1); + let snap = engine.acquire_linearizable_snapshot(1).unwrap(); + assert_eq!( + engine.get_with_snapshot(b"key", &snap).unwrap(), + Some(b"durable".to_vec()) + ); + } +} +#[test] +fn corrupted_nonfinal_length_is_not_a_torn_append() { + let dir = tempdir().unwrap(); + seed(dir.path(), false); + let engine = reopen(dir.path()).unwrap(); + engine.apply_committed_put( + b"other".to_vec(), + b"value".to_vec(), + HlcTimestamp::new(101, 0), + 2, + ); + engine.flush_mvcc_to_persistent(); + drop(engine); + let mf = dir.path().join("MANIFEST-000001"); + let mut raw = fs::read(&mf).unwrap(); + let second = 8 + u32::from_le_bytes(raw[..4].try_into().unwrap()) as usize; + raw[second + 3] ^= 0x80; + fs::write(&mf, raw).unwrap(); + reject_without_deleting(dir.path()); +} +#[test] +fn corrupted_complete_final_length_is_not_a_torn_append() { + let dir = tempdir().unwrap(); + seed(dir.path(), false); + let mf = dir.path().join("MANIFEST-000001"); + let mut raw = fs::read(&mf).unwrap(); + let second = 8 + u32::from_le_bytes(raw[..4].try_into().unwrap()) as usize; + raw[second + 3] ^= 0x80; + fs::write(&mf, raw).unwrap(); + reject_without_deleting(dir.path()); +} +#[test] +fn empty_manifest_is_corruption_and_preserves_ssts() { + let dir = tempdir().unwrap(); + seed(dir.path(), false); + fs::write(dir.path().join("MANIFEST-000001"), []).unwrap(); + reject_without_deleting(dir.path()); +} +#[test] +fn replacing_lsm_files_does_not_reuse_cached_blocks() { + let dir = tempdir().unwrap(); + let cache = Arc::new(ShardedBlockCache::new(1024 * 1024)); + let old = LifecycleStorageEngine::open_lsm( + 1, + dir.path().join("old"), + cache.clone(), + LsmEngineConfig::default(), + ) + .unwrap(); + old.apply_committed_put( + b"key".to_vec(), + b"old".to_vec(), + HlcTimestamp::new(100, 0), + 1, + ); + old.flush_mvcc_to_persistent(); + assert_eq!( + old.get(b"key", HlcTimestamp::new(100, 0)).unwrap(), + Some(b"old".to_vec()) + ); + // A snapshot installed from another history can assign file 1 to different bytes. + let new = LifecycleStorageEngine::open_lsm( + 1, + dir.path().join("new"), + cache, + LsmEngineConfig::default(), + ) + .unwrap(); + new.apply_committed_put( + b"key".to_vec(), + b"new".to_vec(), + HlcTimestamp::new(200, 0), + 2, + ); + new.flush_mvcc_to_persistent(); + for _ in 0..3 { + assert_eq!( + new.get(b"key", HlcTimestamp::new(200, 0)).unwrap(), + Some(b"new".to_vec()), + "new file incarnation must not read old cached bytes" + ); + assert_eq!( + old.get(b"key", HlcTimestamp::new(100, 0)).unwrap(), + Some(b"old".to_vec()), + "old pinned reader must not populate the new reader's cache namespace" + ); + } +} +#[test] +fn checksummed_sst_footer_with_impossible_length_returns_error_without_panicking() { + use metadb_storage::block::BlockEntryValue; + use metadb_storage::{SstableBuilder, SstableReader}; + let dir = tempdir().unwrap(); + SstableBuilder::new(0, 1, 1, 1, HlcTimestamp::new(100, 0)) + .write_to_dir( + dir.path(), + &[(b"key".to_vec(), BlockEntryValue::Value(b"value".to_vec()))], + ) + .unwrap(); + let path = dir.path().join("00000000000000000001.sst"); + let mut raw = fs::read(&path).unwrap(); + let off = raw.len() - 64; + raw[off + 8..off + 16].copy_from_slice(&u64::MAX.to_le_bytes()); + let mut input = raw[off..off + 52].to_vec(); + input.extend_from_slice(&raw[off + 56..]); + raw[off + 52..off + 56].copy_from_slice(&metadb_core::checksum_crc32c(&input).to_le_bytes()); + fs::write(&path, raw).unwrap(); + let result = std::panic::catch_unwind(|| { + SstableReader::open(1, path, Arc::new(ShardedBlockCache::new(1024 * 1024))) + }); + assert!( + result.is_ok(), + "malformed SST footer caused allocation panic" + ); + assert!( + result.unwrap().is_err(), + "out-of-file section must fail validation" + ); +} + +#[test] +fn checkpoints_remain_readable_while_compaction_unlinks_inputs() { + use std::sync::Barrier; + let dir = tempdir().unwrap(); + let source = dir.path().join("source"); + seed(&source, false); + let engine = Arc::new(reopen(&source).unwrap()); + let start = Arc::new(Barrier::new(2)); + let compact_engine = engine.clone(); + let compact_start = start.clone(); + let compactor = std::thread::spawn(move || { + compact_start.wait(); + for _ in 0..100 { + compact_engine.try_compact_range(b"a", b"z").unwrap(); + } + }); + start.wait(); + for index in 0..40 { + let checkpoint = dir.path().join(format!("checkpoint_{index}")); + engine.create_checkpoint(&checkpoint).unwrap(); + let recovered = reopen(&checkpoint).unwrap(); + assert_eq!(recovered.durable_checkpoint().flushed_raft_index, 1); + let snap = recovered.acquire_linearizable_snapshot(1).unwrap(); + assert_eq!( + recovered.get_with_snapshot(b"key", &snap).unwrap(), + Some(b"durable".to_vec()) + ); + } + compactor.join().unwrap(); +} diff --git a/crates/metadb-storage/tests/lsm_recovery_tests.rs b/crates/metadb-storage/tests/lsm_recovery_tests.rs index 633f97b..d76f8d5 100644 --- a/crates/metadb-storage/tests/lsm_recovery_tests.rs +++ b/crates/metadb-storage/tests/lsm_recovery_tests.rs @@ -169,7 +169,9 @@ fn test_manifest_torn_eof_recovery_and_non_final_corruption_rejection() { ); } - // A complete but CRC-invalid record at EOF is a torn tail and must be truncated. + // A complete but CRC-invalid record at EOF is corruption. Rolling it back + // could resurrect compacted inputs that have already been unlinked. + let valid_manifest_len = fs::metadata(&manifest_path).unwrap().len(); { let mut mf = OpenOptions::new() .append(true) @@ -182,9 +184,19 @@ fn test_manifest_torn_eof_recovery_and_non_final_corruption_rejection() { mf.sync_all().unwrap(); } { - let recovered = - LifecycleStorageEngine::open_lsm(2, &lsm_dir, cache.clone(), cfg.clone()).unwrap(); - assert_eq!(recovered.durable_checkpoint().flushed_raft_index, 5); + let corrupt_len = fs::metadata(&manifest_path).unwrap().len(); + assert!(matches!( + LifecycleStorageEngine::open_lsm(2, &lsm_dir, cache.clone(), cfg.clone()), + Err(MetaDbError::ManifestCorrupted(_)) + )); + assert_eq!(fs::metadata(&manifest_path).unwrap().len(), corrupt_len); + // Restore the known-good bytes for the independent non-final case. + OpenOptions::new() + .write(true) + .open(&manifest_path) + .unwrap() + .set_len(valid_manifest_len) + .unwrap(); } // Now corrupt a NON-FINAL record inside MANIFEST-000001 and verify strict `ManifestCorrupted` error! diff --git a/crates/metadb-storage/tests/round2_integrity_tests.rs b/crates/metadb-storage/tests/round2_integrity_tests.rs new file mode 100644 index 0000000..ba4756e --- /dev/null +++ b/crates/metadb-storage/tests/round2_integrity_tests.rs @@ -0,0 +1,126 @@ +//! Round 2: the manifest is the durable authority for a live SST's identity. + +use std::fs; +use std::sync::Arc; + +use metadb_core::HlcTimestamp; +use metadb_storage::sstable::sst_file_name; +use metadb_storage::{ + LifecycleStorageEngine, LsmEngineConfig, ManifestLog, ShardedBlockCache, VersionEdit, +}; + +#[test] +fn live_sst_metadata_must_match_its_manifest_record() { + let expected = tempfile::tempdir().unwrap(); + let unrelated = tempfile::tempdir().unwrap(); + for (dir, value, index) in [ + (expected.path(), b"expected".as_slice(), 1), + (unrelated.path(), b"unrelated".as_slice(), 9), + ] { + let engine = LifecycleStorageEngine::open_lsm( + 1, + dir, + Arc::new(ShardedBlockCache::new(1024 * 1024)), + LsmEngineConfig::default(), + ) + .unwrap(); + engine.apply_committed_put(b"key".to_vec(), value.to_vec(), HlcTimestamp(index), index); + assert_eq!(engine.flush_mvcc_to_persistent(), index); + } + + // Simulate a misdirected but internally checksummed file. The manifest + // still names index 1; the substituted SST explicitly declares index 9. + fs::copy( + unrelated.path().join(sst_file_name(1)), + expected.path().join(sst_file_name(1)), + ) + .unwrap(); + let reopened = LifecycleStorageEngine::open_lsm( + 1, + expected.path(), + Arc::new(ShardedBlockCache::new(1024 * 1024)), + LsmEngineConfig::default(), + ); + if let Ok(engine) = reopened { + let checkpoint = engine.durable_checkpoint(); + let version = engine.backend().current_version(); + let declared_sst_index = version.levels[0][0].meta().max_raft_index; + let snapshot = engine.acquire_linearizable_snapshot(1).unwrap(); + let observed = engine.get_with_snapshot(b"key", &snapshot).unwrap(); + panic!("accepted SST inconsistent with manifest: manifest checkpoint={}, SST index={declared_sst_index}, returned={observed:?}", checkpoint.flushed_raft_index); + } +} + +#[test] +fn every_sst_identity_field_is_checked_against_the_manifest() { + for mismatch in [ + "level", + "generation", + "size", + "min-key", + "max-key", + "entries", + "tombstones", + "raft-index", + "hlc", + ] { + let dir = tempfile::tempdir().unwrap(); + let engine = LifecycleStorageEngine::open_lsm( + 1, + dir.path(), + Arc::new(ShardedBlockCache::new(1024 * 1024)), + LsmEngineConfig::default(), + ) + .unwrap(); + engine.apply_committed_put(b"key".to_vec(), b"value".to_vec(), HlcTimestamp(1), 1); + assert_eq!(engine.flush_mvcc_to_persistent(), 1); + let mut metadata = engine.backend().current_version().levels[0][0] + .meta() + .clone(); + drop(engine); + match mismatch { + "level" => metadata.level += 1, + "generation" => metadata.flush_generation += 1, + "size" => metadata.file_size += 1, + "min-key" => metadata.min_key = b"jey".to_vec(), + "max-key" => metadata.max_key = b"ley".to_vec(), + "entries" => metadata.num_entries += 1, + "tombstones" => metadata.num_tombstones += 1, + "raft-index" => metadata.max_raft_index += 1, + "hlc" => metadata.max_hlc = HlcTimestamp(2), + _ => unreachable!(), + } + let (mut manifest, _) = ManifestLog::open_or_create(dir.path()).unwrap(); + manifest + .append(&VersionEdit { + added_files: vec![metadata], + ..Default::default() + }) + .unwrap(); + drop(manifest); + assert!( + LifecycleStorageEngine::open_lsm( + 1, + dir.path(), + Arc::new(ShardedBlockCache::new(1024 * 1024)), + LsmEngineConfig::default() + ) + .is_err(), + "accepted SST {mismatch} mismatch" + ); + } +} + +#[test] +fn verify_existing_never_initializes_an_empty_directory() { + let dir = tempfile::tempdir().unwrap(); + assert!(LifecycleStorageEngine::open_lsm_existing( + 1, + dir.path(), + Arc::new(ShardedBlockCache::new(1024 * 1024)), + LsmEngineConfig::default() + ) + .is_err()); + assert!(!dir.path().join("CURRENT").exists()); + assert!(!dir.path().join("MANIFEST-000001").exists()); +} diff --git a/crates/metadb-tx/src/attr_aggregator.rs b/crates/metadb-tx/src/attr_aggregator.rs index f59ebdc..9193ed2 100644 --- a/crates/metadb-tx/src/attr_aggregator.rs +++ b/crates/metadb-tx/src/attr_aggregator.rs @@ -77,6 +77,27 @@ impl AsyncAttrAggregator { lsn } + pub(crate) fn validate_delta( + &self, + pid: InodeId, + base: &InodeAttr, + children: i64, + bytes: i64, + ) -> Result<()> { + let guard = self.pending_by_pid.read().unwrap(); + let mut total_children = i128::from(base.num_children) + i128::from(children); + let mut total_bytes = i128::from(base.size_bytes) + i128::from(bytes); + for delta in guard.get(&pid).into_iter().flat_map(|queue| queue.iter()) { + if delta.seq > base.applied_lsn { + total_children += i128::from(delta.delta_children); + total_bytes += i128::from(delta.delta_bytes); + } + } + crate::validation::checked_count(total_children)?; + crate::validation::checked_count(total_bytes)?; + Ok(()) + } + /// Reconciles a persisted `InodeAttr` (`_attr_`) with all un-flushed in-memory deltas /// whose `seq > persisted.applied_lsn` (§5.2 Read-Path Reconciliation). pub fn reconcile_attr(&self, pid: InodeId, mut persisted: InodeAttr) -> InodeAttr { diff --git a/crates/metadb-tx/src/aux_outbox.rs b/crates/metadb-tx/src/aux_outbox.rs index 2d6c6bd..3e7da25 100644 --- a/crates/metadb-tx/src/aux_outbox.rs +++ b/crates/metadb-tx/src/aux_outbox.rs @@ -16,12 +16,24 @@ use metadb_storage::{LifecycleStorageEngine, MvccValue, SnapshotGuard, ViewAtRaf /// Default maximum un-acknowledged tasks held in RAM before spilling to `[0x07]` on L0 flush. pub const DEFAULT_OUTBOX_SPILL_THRESHOLD: usize = 256; +/// Folded `i128` directory deltas are stored in `i64` attributes. +/// Values outside `i64` saturate; the caller then clamps a negative total to 0. +fn clamp_delta_to_i64(value: i128) -> i64 { + if value > i64::MAX as i128 { + i64::MAX + } else if value < i64::MIN as i128 { + i64::MIN + } else { + value as i64 + } +} + #[derive(Debug, Clone)] pub struct ReplicatedAttrDelta { pub raft_index: u64, pub hlc: HlcTimestamp, - pub delta_children: i64, - pub delta_bytes: i64, + pub delta_children: i128, + pub delta_bytes: i128, } /// Deterministic per-shard attribute delta aggregator integrated with `PreFlushHook` (`consensus_and_transactions.md` §7.2). @@ -51,15 +63,25 @@ impl ReplicatedAttrAggregator { delta_bytes: i64, ) { let mut guard = self.pending_by_pid.write().unwrap(); - guard - .entry(pid) - .or_default() - .push_back(ReplicatedAttrDelta { + let queue = guard.entry(pid).or_default(); + if let Some(previous) = queue + .back_mut() + .filter(|delta| delta.raft_index == raft_index) + { + // A 2PC command may modify several keys in one directory. Fold its + // deltas as one atomic step, avoiding transient counter overflow and + // ensuring an applied-index baseline includes the whole command. + previous.delta_children += i128::from(delta_children); + previous.delta_bytes += i128::from(delta_bytes); + previous.hlc = previous.hlc.max(hlc); + } else { + queue.push_back(ReplicatedAttrDelta { raft_index, hlc, - delta_children, - delta_bytes, + delta_children: i128::from(delta_children), + delta_bytes: i128::from(delta_bytes), }); + } } /// Reconciles an authoritative `(pid, "_attr_")` record at `snap.cut()` (`Resolution B1`). @@ -71,15 +93,23 @@ impl ReplicatedAttrAggregator { ) -> InodeAttr { let cut = snap.cut(); let guard = self.pending_by_pid.read().unwrap(); + let baseline = base.applied_lsn; if let Some(q) = guard.get(&pid) { for d in q { - if d.raft_index > base.applied_lsn + if d.raft_index > baseline && d.raft_index <= cut.max_raft_index && d.hlc <= cut.max_hlc { - base.num_children = - saturating_add_i64(base.num_children, d.delta_children).max(0); - base.size_bytes = saturating_add_i64(base.size_bytes, d.delta_bytes).max(0); + base.num_children = saturating_add_i64( + base.num_children, + clamp_delta_to_i64(d.delta_children), + ) + .max(0); + base.size_bytes = saturating_add_i64( + base.size_bytes, + clamp_delta_to_i64(d.delta_bytes), + ) + .max(0); if d.hlc > base.modify_time { base.modify_time = d.hlc; } @@ -99,15 +129,23 @@ impl ReplicatedAttrAggregator { ) -> DirShardDeltaRecord { let cut = snap.cut(); let guard = self.pending_by_pid.read().unwrap(); + let baseline = base.applied_raft_index; if let Some(q) = guard.get(&pid) { for d in q { - if d.raft_index > base.applied_raft_index + if d.raft_index > baseline && d.raft_index <= cut.max_raft_index && d.hlc <= cut.max_hlc { - base.num_children = - saturating_add_i64(base.num_children, d.delta_children).max(0); - base.size_bytes = saturating_add_i64(base.size_bytes, d.delta_bytes).max(0); + base.num_children = saturating_add_i64( + base.num_children, + clamp_delta_to_i64(d.delta_children), + ) + .max(0); + base.size_bytes = saturating_add_i64( + base.size_bytes, + clamp_delta_to_i64(d.delta_bytes), + ) + .max(0); if d.hlc > base.modify_time { base.modify_time = d.hlc; } @@ -118,6 +156,32 @@ impl ReplicatedAttrAggregator { base } + pub(crate) fn pending_totals_since(&self, pid: InodeId, baseline: u64) -> (i128, i128) { + let guard = self.pending_by_pid.read().unwrap(); + guard + .get(&pid) + .into_iter() + .flat_map(|q| q.iter()) + .filter(|delta| delta.raft_index > baseline) + .fold((0, 0), |(children, bytes), delta| { + (children + delta.delta_children, bytes + delta.delta_bytes) + }) + } + + /// Last child mutation represented by pending deltas at a serialized apply boundary. + pub(crate) fn latest_delta_hlc(&self, pid: InodeId, max_raft_index: u64) -> HlcTimestamp { + self.pending_by_pid + .read() + .unwrap() + .get(&pid) + .into_iter() + .flat_map(|queue| queue.iter()) + .filter(|delta| delta.raft_index <= max_raft_index) + .map(|delta| delta.hlc) + .max() + .unwrap_or(HlcTimestamp::ZERO) + } + /// Folds deltas in `(baseline.applied_index, sealed_raft_index]` reading `view_at_m` /// across `SealingMemTable + ImmMemTables + VersionSet` (`Resolution B5` / `G-S3`). pub fn fold_into_seal_batch( @@ -138,11 +202,20 @@ impl ReplicatedAttrAggregator { if let Some(raw_attr) = view_at_m.get(&attr_key)? { let mut attr = InodeAttr::decode(&raw_attr)?; let mut changed = false; + let baseline = attr.applied_lsn; for d in queue { if d.raft_index > attr.applied_lsn && d.raft_index <= sealed_raft_index { attr.num_children = - saturating_add_i64(attr.num_children, d.delta_children).max(0); - attr.size_bytes = saturating_add_i64(attr.size_bytes, d.delta_bytes).max(0); + saturating_add_i64( + attr.num_children, + clamp_delta_to_i64(d.delta_children), + ) + .max(0); + attr.size_bytes = saturating_add_i64( + attr.size_bytes, + clamp_delta_to_i64(d.delta_bytes), + ) + .max(0); if d.hlc > attr.modify_time { attr.modify_time = d.hlc; } @@ -161,11 +234,20 @@ impl ReplicatedAttrAggregator { if let Some(raw_sub) = view_at_m.get(&shard_delta_key)? { let mut sub = DirShardDeltaRecord::decode(&raw_sub)?; let mut changed = false; + let baseline = sub.applied_raft_index; for d in queue { if d.raft_index > sub.applied_raft_index && d.raft_index <= sealed_raft_index { sub.num_children = - saturating_add_i64(sub.num_children, d.delta_children).max(0); - sub.size_bytes = saturating_add_i64(sub.size_bytes, d.delta_bytes).max(0); + saturating_add_i64( + sub.num_children, + clamp_delta_to_i64(d.delta_children), + ) + .max(0); + sub.size_bytes = saturating_add_i64( + sub.size_bytes, + clamp_delta_to_i64(d.delta_bytes), + ) + .max(0); if d.hlc > sub.modify_time { sub.modify_time = d.hlc; } diff --git a/crates/metadb-tx/src/aux_sink.rs b/crates/metadb-tx/src/aux_sink.rs index 34f2593..680d518 100644 --- a/crates/metadb-tx/src/aux_sink.rs +++ b/crates/metadb-tx/src/aux_sink.rs @@ -2,8 +2,8 @@ use std::collections::{BTreeMap, VecDeque}; use std::sync::RwLock; use metadb_core::{ - saturating_add_i64, FileId, HlcTimestamp, InodeId, QuotaRecord, SecondaryIndexRecord, ShardId, - UserId, + saturating_add_i64, FileId, HlcTimestamp, InodeId, MetaDbError, QuotaRecord, Result, + SecondaryIndexRecord, ShardId, UserId, }; use metadb_storage::{LifecycleStorageEngine, LogSequenceNumber, RaftLogPayload}; @@ -180,10 +180,11 @@ impl AsyncAuxiliarySink { guard.queue.drain(..).collect() }; - let count = tasks.len(); + let mut count = 0; let mut max_lsn_by_shard: BTreeMap = BTreeMap::new(); - for task in tasks { + let mut tasks = tasks.into_iter(); + while let Some(task) = tasks.next() { let (shard_id, lsn) = match &task { AuxUpdateTask::QuotaDelta { origin_shard_id, @@ -201,11 +202,20 @@ impl AsyncAuxiliarySink { .. } => (*origin_shard_id, *lsn), }; + if self.try_apply_task_idempotent(task.clone()).is_err() { + let mut guard = self.inner.write().unwrap(); + for pending in tasks.rev() { + guard.queue.push_front(pending); + } + guard.queue.push_front(task); + // No watermark may pass a task that remains unapplied. + return count; + } + count += 1; max_lsn_by_shard .entry(shard_id) .and_modify(|cur| *cur = (*cur).max(lsn)) .or_insert(lsn); - self.apply_task_idempotent(task); } for engine in engines { @@ -222,8 +232,12 @@ impl AsyncAuxiliarySink { /// Applies an `AuxUpdateTask` with exact-once idempotency guarantees. /// Returns `true` if applied, or `false` if suppressed as a duplicate/stale replay. pub fn apply_task_idempotent(&self, task: AuxUpdateTask) -> bool { + self.try_apply_task_idempotent(task).unwrap_or(false) + } + + pub fn try_apply_task_idempotent(&self, task: AuxUpdateTask) -> Result { let mut guard = self.inner.write().unwrap(); - match task { + Ok(match task { AuxUpdateTask::QuotaDelta { origin_shard_id, lsn, @@ -242,7 +256,7 @@ impl AsyncAuxiliarySink { .copied() .unwrap_or((0, 0)); if (lsn, 0u16) <= prev_cursor { - return false; // Duplicate replay from Raft log recovery suppressed! + return Ok(false); // Duplicate replay from Raft log recovery suppressed! } record.used_bytes = saturating_add_i64(record.used_bytes, delta_bytes).max(0); record.used_inodes = saturating_add_i64(record.used_inodes, delta_inodes).max(0); @@ -262,12 +276,12 @@ impl AsyncAuxiliarySink { let incoming_ver = (hlc, origin_shard_id, lsn, 0u16); if let Some(&del_hlc) = guard.deleted_fid_hlc.get(&fid) { if hlc <= del_hlc { - return false; + return Ok(false); } } if let Some(existing) = guard.secondary_index_table.get(&fid) { if incoming_ver <= existing.last_ver.max(existing.tombstone_ver) { - return false; + return Ok(false); } } guard.secondary_index_table.insert( @@ -293,19 +307,19 @@ impl AsyncAuxiliarySink { let incoming_ver = (hlc, origin_shard_id, lsn, 0u16); if let Some(&del_hlc) = guard.deleted_fid_hlc.get(&fid) { if hlc <= del_hlc { - return false; + return Ok(false); } } if let Some(existing) = guard.secondary_index_table.get(&fid) { if incoming_ver <= existing.last_ver && hlc < existing.last_applied_hlc() { - return false; + return Ok(false); } } guard.secondary_index_table.remove(&fid); guard.deleted_fid_hlc.insert(fid, hlc); true } - } + }) } pub fn get_quota(&self, uid: UserId) -> Option { diff --git a/crates/metadb-tx/src/lib.rs b/crates/metadb-tx/src/lib.rs index f3295b2..232caea 100644 --- a/crates/metadb-tx/src/lib.rs +++ b/crates/metadb-tx/src/lib.rs @@ -13,6 +13,7 @@ pub mod recovery_worker; pub mod rename_cycle; pub mod state_machine_impl; pub mod two_pc; +mod validation; pub use attr_aggregator::{AsyncAttrAggregator, AttrDeltaItem}; pub use aux_outbox::{ReplicatedAttrAggregator, ShardAuxOutbox, DEFAULT_OUTBOX_SPILL_THRESHOLD}; diff --git a/crates/metadb-tx/src/pushdown.rs b/crates/metadb-tx/src/pushdown.rs index 21193d2..c070025 100644 --- a/crates/metadb-tx/src/pushdown.rs +++ b/crates/metadb-tx/src/pushdown.rs @@ -45,6 +45,21 @@ impl PushdownExecutor { &self.aux_sink } + fn validate_parent_delta( + &self, + engine: &LifecycleStorageEngine, + pid: InodeId, + children: i64, + bytes: i64, + hlc: HlcTimestamp, + ) -> Result<()> { + let raw = engine + .get(&MetaKey::dir_attr(pid).encode(), hlc)? + .ok_or(MetaDbError::DirectoryNotFound(pid))?; + self.attr_aggregator + .validate_delta(pid, &InodeAttr::decode(&raw)?, children, bytes) + } + /// Initializes the root directory `(ROOT_INODE_ID, "_attr_")` on the shard owning `ROOT_INODE_ID`. pub fn init_directory_attr( &self, @@ -247,6 +262,9 @@ impl PushdownExecutor { }); } + crate::validation::validate_size(attr.size_bytes)?; + self.validate_parent_delta(engine, pid, -1, -attr.size_bytes, hlc)?; + engine.delete(child_key, hlc); let released = negate_accounted_size(attr.size_bytes); @@ -309,6 +327,8 @@ impl PushdownExecutor { }); } + self.validate_parent_delta(parent_engine, pid, -1, 0, hlc)?; + parent_engine.delete(child_key, hlc); let target_attr_key = MetaKey::dir_attr(target_dir_id).encode(); dir_engine.delete(target_attr_key, hlc); diff --git a/crates/metadb-tx/src/state_machine_impl.rs b/crates/metadb-tx/src/state_machine_impl.rs index 45c4144..7a7b1db 100644 --- a/crates/metadb-tx/src/state_machine_impl.rs +++ b/crates/metadb-tx/src/state_machine_impl.rs @@ -143,6 +143,24 @@ pub enum TxRaftCmd { restored_range: CompositeKeyRange, new_epoch: ShardEpoch, }, + /// Prepare with compare-and-set predicates for the coordinator's committed reads. + /// Append new variants to preserve the bincode tags of existing Raft log entries. + Prepare2pcChecked { + txn_id: TxnId, + is_primary: bool, + primary_key: Vec, + participant_keys: Vec>, + start_hlc: HlcTimestamp, + expire_hlc: HlcTimestamp, + rename_fence_token: Option, + intents: Vec<(Vec, IntentOp)>, + expected_values: Vec<(Vec, Option>)>, + }, + UpsertQuotaLimit { + uid: UserId, + insert_quota_limit_bytes: u64, + update_quota_limit_bytes: u64, + }, } /// Typed output serialized inside `MetaDbRaftResponse.result`. @@ -340,6 +358,293 @@ impl TxStateMachineHandler { Ok(()) } + fn intent_deltas(op: &IntentOp) -> &[AttrDeltaSpec] { + match op { + IntentOp::Put { attr_deltas, .. } + | IntentOp::Insert { attr_deltas, .. } + | IntentOp::Delete { attr_deltas, .. } + | IntentOp::DeletingDir { attr_deltas, .. } => attr_deltas, + IntentOp::DirExistsFenceSet { .. } => &[], + } + } + + /// Reserve positive capacity for prepared transactions. A pending deletion + /// cannot release capacity until it commits, so other writers cannot overbook + /// the directory while its transaction is waiting for a decision. + fn validate_accounting( + &self, + engine: &LifecycleStorageEngine, + deltas: &[AttrDeltaSpec], + own_txn: Option, + ) -> Result<()> { + use crate::validation::checked_count; + let mut sums: BTreeMap = BTreeMap::new(); + for delta in deltas { + let sum = sums.entry(delta.pid).or_default(); + sum.0 += i128::from(delta.delta_children); + sum.1 += i128::from(delta.delta_bytes); + } + if sums.is_empty() { + return Ok(()); + } + let mut reserved: BTreeMap = BTreeMap::new(); + for (_, raw) in Self::scan_latest_in_apply(engine, &[0x05], &[0x06])? { + let rec = WriteIntentRecord::decode(&raw)?; + if Some(rec.txn_id) == own_txn { + continue; + } + for delta in Self::intent_deltas(&rec.op) { + let sum = reserved.entry(delta.pid).or_default(); + sum.0 += i128::from(delta.delta_children.max(0)); + sum.1 += i128::from(delta.delta_bytes.max(0)); + } + } + for (pid, delta) in sums { + let (children, bytes, baseline) = if let Some(raw) = + Self::read_latest_in_apply(engine, &MetaKey::dir_attr(pid).encode())? + { + let attr = InodeAttr::decode(&raw)?; + (attr.num_children, attr.size_bytes, attr.applied_lsn) + } else if let Some(raw) = + Self::read_latest_in_apply(engine, &DirShardDeltaKey { pid }.encode())? + { + let sub = DirShardDeltaRecord::decode(&raw)?; + (sub.num_children, sub.size_bytes, sub.applied_raft_index) + } else { + return Err(MetaDbError::DirectoryNotFound(pid)); + }; + let pending = self.attr_agg.pending_totals_since(pid, baseline); + let children = i128::from(children) + pending.0 + delta.0; + let bytes = i128::from(bytes) + pending.1 + delta.1; + checked_count(children)?; + checked_count(bytes)?; + let reserve = reserved.get(&pid).copied().unwrap_or_default(); + checked_count(children + reserve.0)?; + checked_count(bytes + reserve.1)?; + } + Ok(()) + } + + fn validate_intent_value( + &self, + engine: &LifecycleStorageEngine, + key: &[u8], + op: &IntentOp, + ) -> Result<()> { + use crate::validation::{validate_new_directory, validate_size}; + if key.first().copied() != Some(metadb_core::TABLE_PREFIX_META) { + return Ok(()); + } + let mk = MetaKey::decode(key)?; + if !mk.is_attr_key() { + validate_user_entry_name(&mk.name)?; + } + if let IntentOp::Put { value, .. } = op { + let attr = InodeAttr::decode(value)?; + validate_size(attr.size_bytes)?; + crate::validation::checked_count(i128::from(attr.num_children))?; + if attr.entry_type == EntryType::Directory { + validate_new_directory(mk.pid, attr.inode_id)?; + } else if attr.entry_type == EntryType::DirectoryAttr { + if mk.pid != attr.inode_id || !metadb_core::is_dir_inode_id(attr.inode_id) { + return Err(MetaDbError::CodecError( + "directory anchor inode mismatch".into(), + )); + } + if Self::read_latest_in_apply(engine, key)?.is_none() { + validate_new_directory(attr.parent_pid, attr.inode_id)?; + if Self::read_latest_in_apply( + engine, + &DirShardDeltaKey { pid: attr.inode_id }.encode(), + )? + .is_some() + { + return Err(MetaDbError::AlreadyExists { + pid: attr.inode_id, + name: metadb_core::ATTR_ENTRY_NAME.into(), + }); + } + } + } + } + if let Some(raw) = Self::read_latest_in_apply(engine, key)? { + validate_size(InodeAttr::decode(&raw)?.size_bytes)?; + } + Ok(()) + } + + /// Validate values and every derived counter before changing keys, deltas or + /// the outbox. This also protects deterministic replay from malformed input. + fn validate_command(&self, engine: &LifecycleStorageEngine, cmd: &TxRaftCmd) -> Result<()> { + use crate::validation::{validate_new_directory, validate_size}; + let read_child = |pid, name: &str| -> Result> { + Self::read_latest_in_apply(engine, &MetaKey::child(pid, name).encode())? + .map(|raw| InodeAttr::decode(&raw)) + .transpose() + }; + let mut deltas = Vec::new(); + let mut own_txn = None; + match cmd { + TxRaftCmd::InitDirAttr { + dir_id, parent_pid, .. + } => { + if !metadb_core::is_dir_inode_id(*dir_id) + || !metadb_core::is_dir_inode_id(*parent_pid) + { + return Err(MetaDbError::CodecError("invalid directory inode".into())); + } + } + TxRaftCmd::InitDirShardDelta { record } => { + if !metadb_core::is_dir_inode_id(record.inode_id) { + return Err(MetaDbError::CodecError("invalid directory inode".into())); + } + validate_size(record.size_bytes)?; + crate::validation::checked_count(i128::from(record.num_children))?; + } + TxRaftCmd::CreateFile { + pid, size_bytes, .. + } => { + validate_size(*size_bytes)?; + deltas.push(AttrDeltaSpec { + pid: *pid, + delta_children: 1, + delta_bytes: *size_bytes, + }); + } + TxRaftCmd::MkdirSingleShard { + pid, new_dir_id, .. + } => { + validate_new_directory(*pid, *new_dir_id)?; + let key = MetaKey::dir_attr(*new_dir_id).encode(); + if Self::read_latest_in_apply(engine, &key)?.is_some() + || Self::read_latest_in_apply( + engine, + &DirShardDeltaKey { pid: *new_dir_id }.encode(), + )? + .is_some() + { + return Err(MetaDbError::AlreadyExists { + pid: *new_dir_id, + name: metadb_core::ATTR_ENTRY_NAME.into(), + }); + } + deltas.push(AttrDeltaSpec { + pid: *pid, + delta_children: 1, + delta_bytes: 0, + }); + } + TxRaftCmd::Unlink { pid, name } => { + if let Some(attr) = read_child(*pid, name)? { + validate_size(attr.size_bytes)?; + deltas.push(AttrDeltaSpec { + pid: *pid, + delta_children: -1, + delta_bytes: -attr.size_bytes, + }); + } + } + TxRaftCmd::RmdirSingleShard { pid, name } => { + if read_child(*pid, name)?.is_some() { + deltas.push(AttrDeltaSpec { + pid: *pid, + delta_children: -1, + delta_bytes: 0, + }); + } + } + TxRaftCmd::SetAttrSingleShard { pid, name, patch } => { + if let Some(size) = patch.size_bytes { + validate_size(size)?; + } + if let Some(attr) = read_child(*pid, name)? { + validate_size(attr.size_bytes)?; + if attr.entry_type != EntryType::File + && (patch.size_bytes.is_some() || patch.fid.is_some()) + { + return Err(MetaDbError::CodecError( + "directory size and fid are derived attributes".into(), + )); + } + deltas.push(AttrDeltaSpec { + pid: *pid, + delta_children: 0, + delta_bytes: patch.size_bytes.unwrap_or(attr.size_bytes) - attr.size_bytes, + }); + } + } + TxRaftCmd::RenameSingleShard { + src_pid, + src_name, + dst_pid, + dst_name, + .. + } => { + if let Some(attr) = read_child(*src_pid, src_name)? { + validate_size(attr.size_bytes)?; + if src_pid == dst_pid && src_name == dst_name { + return Ok(()); + } + let victim = read_child(*dst_pid, dst_name)?; + if let Some(victim) = &victim { + validate_size(victim.size_bytes)?; + } + deltas.push(AttrDeltaSpec { + pid: *src_pid, + delta_children: -1, + delta_bytes: -attr.size_bytes, + }); + deltas.push(AttrDeltaSpec { + pid: *dst_pid, + delta_children: i64::from(victim.is_none()), + delta_bytes: attr.size_bytes - victim.map(|v| v.size_bytes).unwrap_or(0), + }); + } + } + TxRaftCmd::Prepare2pc { + txn_id, intents, .. + } => { + own_txn = Some(*txn_id); + for (key, op) in intents { + self.validate_intent_value(engine, key, op)?; + deltas.extend_from_slice(Self::intent_deltas(op)); + } + } + TxRaftCmd::CommitSecondary2pc { + txn_id, + target_user_keys, + .. + } => { + own_txn = Some(*txn_id); + for key in target_user_keys { + if let Some(raw) = Self::read_latest_in_apply( + engine, + &WriteIntentKey::new(key.clone()).encode(), + )? { + let rec = WriteIntentRecord::decode(&raw)?; + if rec.txn_id == *txn_id { + self.validate_intent_value(engine, key, &rec.op)?; + deltas.extend_from_slice(Self::intent_deltas(&rec.op)); + } + } + } + } + TxRaftCmd::CommitPrimary2pc { txn_id, .. } => { + own_txn = Some(*txn_id); + for (ik, raw) in Self::scan_latest_in_apply(engine, &[0x05], &[0x06])? { + let rec = WriteIntentRecord::decode(&raw)?; + if rec.txn_id == *txn_id { + let key = WriteIntentKey::decode(&ik)?.target_user_key; + self.validate_intent_value(engine, &key, &rec.op)?; + deltas.extend_from_slice(Self::intent_deltas(&rec.op)); + } + } + } + _ => {} + } + self.validate_accounting(engine, &deltas, own_txn) + } + fn apply_tx_cmd( &self, raft_index: u64, @@ -347,7 +652,50 @@ impl TxStateMachineHandler { cmd: TxRaftCmd, engine: &LifecycleStorageEngine, ) -> Result { + self.validate_command(engine, &cmd)?; match cmd { + TxRaftCmd::Prepare2pcChecked { + txn_id, + is_primary, + primary_key, + participant_keys, + start_hlc, + expire_hlc, + rename_fence_token, + intents, + expected_values, + } => { + // Check every read before installing any intent or transaction record. + // The intent then excludes mutations until commit or abort. + for (user_key, expected) in expected_values { + self.ensure_user_key_in_range(&user_key)?; + if !intents.iter().any(|(key, _)| *key == user_key) { + return Err(MetaDbError::CodecError( + "2PC expected value must have a matching local intent".to_string(), + )); + } + if Self::read_latest_in_apply(engine, &user_key)? != expected { + return Err(MetaDbError::TransactionAborted( + "2PC value changed after coordinator read".to_string(), + )); + } + } + self.apply_tx_cmd( + raft_index, + commit_hlc, + TxRaftCmd::Prepare2pc { + txn_id, + is_primary, + primary_key, + participant_keys, + start_hlc, + expire_hlc, + rename_fence_token, + intents, + }, + engine, + ) + } TxRaftCmd::InitDirAttr { dir_id, parent_pid, @@ -895,6 +1243,27 @@ impl TxStateMachineHandler { attr: src_attr, })) } + TxRaftCmd::UpsertQuotaLimit { + uid, + insert_quota_limit_bytes, + update_quota_limit_bytes, + } => { + let qk = QuotaKey { uid }.encode(); + self.ensure_user_key_in_range(&qk)?; + let mut rec = match Self::read_latest_in_apply(engine, &qk)? { + Some(raw) => { + let mut rec = QuotaRecord::decode(&raw)?; + rec.quota_limit_bytes = update_quota_limit_bytes; + rec + } + None => QuotaRecord::new(uid, insert_quota_limit_bytes), + }; + rec.last_applied_hlc = rec.last_applied_hlc.max(commit_hlc); + engine.apply_committed_put(qk, rec.encode(), commit_hlc, raft_index); + self.outbox + .record_entry_specs(raft_index, commit_hlc, vec![]); + Ok(TxRaftOutput::Quota(rec)) + } TxRaftCmd::SetQuotaLimit { uid, quota_limit_bytes, @@ -956,6 +1325,7 @@ impl TxStateMachineHandler { } } } + let mut staged: BTreeMap, Vec> = BTreeMap::new(); let mut applied_count = 0usize; for item in entries { match item.payload { @@ -967,7 +1337,10 @@ impl TxStateMachineHandler { } => { let qk = QuotaKey { uid }.encode(); self.ensure_user_key_in_range(&qk)?; - let mut rec = match Self::read_latest_in_apply(engine, &qk)? { + let mut rec = match staged.get(&qk).cloned().map_or_else( + || Self::read_latest_in_apply(engine, &qk), + |raw| Ok(Some(raw)), + )? { Some(raw) => QuotaRecord::decode(&raw)?, None => QuotaRecord::new(uid, u64::MAX), }; @@ -992,7 +1365,7 @@ impl TxStateMachineHandler { rec.last_applied_hlc = rec.last_applied_hlc.max(item.hlc); rec.applied_by_shard .insert(item.origin_shard_id, cur_cursor); - engine.apply_committed_put(qk, rec.encode(), commit_hlc, raft_index); + staged.insert(qk, rec.encode()); applied_count += 1; } AuxTaskPayload::IndexUpsert { @@ -1009,7 +1382,10 @@ impl TxStateMachineHandler { item.origin_raft_index, item.task_ordinal, ); - let existing_opt = match Self::read_latest_in_apply(engine, &ik)? { + let existing_opt = match staged.get(&ik).cloned().map_or_else( + || Self::read_latest_in_apply(engine, &ik), + |raw| Ok(Some(raw)), + )? { Some(raw) => Some(SecondaryIndexRecord::decode(&raw)?), None => None, }; @@ -1030,12 +1406,7 @@ impl TxStateMachineHandler { .unwrap_or((HlcTimestamp::ZERO, 0, 0, 0)), is_deleted: false, }; - engine.apply_committed_put( - ik, - new_rec.encode(), - commit_hlc, - raft_index, - ); + staged.insert(ik, new_rec.encode()); applied_count += 1; } AuxTaskPayload::IndexDelete { fid } => { @@ -1047,7 +1418,10 @@ impl TxStateMachineHandler { item.origin_raft_index, item.task_ordinal, ); - let mut rec = match Self::read_latest_in_apply(engine, &ik)? { + let mut rec = match staged.get(&ik).cloned().map_or_else( + || Self::read_latest_in_apply(engine, &ik), + |raw| Ok(Some(raw)), + )? { Some(raw) => SecondaryIndexRecord::decode(&raw)?, None => SecondaryIndexRecord { fid, @@ -1064,11 +1438,14 @@ impl TxStateMachineHandler { } rec.tombstone_ver = incoming_ver; rec.is_deleted = true; - engine.apply_committed_put(ik, rec.encode(), commit_hlc, raft_index); + staged.insert(ik, rec.encode()); applied_count += 1; } } } + for (key, value) in staged { + engine.apply_committed_put(key, value, commit_hlc, raft_index); + } self.outbox .record_entry_specs(raft_index, commit_hlc, vec![]); Ok(TxRaftOutput::AppliedCount(applied_count)) @@ -1375,6 +1752,9 @@ impl TxStateMachineHandler { commit_hlc: tx_commit_hlc, target_user_keys, } => { + for key in &target_user_keys { + self.ensure_user_key_in_range(key)?; + } let mut any_applied = false; let mut aux_specs = Vec::new(); for uk in &target_user_keys { @@ -1400,6 +1780,9 @@ impl TxStateMachineHandler { txn_id, target_user_keys, } => { + for key in &target_user_keys { + self.ensure_user_key_in_range(key)?; + } for uk in &target_user_keys { self.abort_intent_for_key(engine, txn_id, uk, raft_index, commit_hlc)?; } @@ -1476,6 +1859,13 @@ impl TxStateMachineHandler { new_range, new_epoch, } => { + // This check shares Raft's serialized apply boundary with the + // fence. Prepare cannot race between the worker's scan and fence. + if !Self::scan_latest_in_apply(engine, &[0x05], &[0x07])?.is_empty() { + return Err(MetaDbError::TransactionAborted( + "split deferred until all intents and decisions are drained".into(), + )); + } { let mut m = self.meta.write().unwrap(); m.range = new_range; @@ -1492,6 +1882,52 @@ impl TxStateMachineHandler { updated_anchor_attr, updated_shard_delta, } => { + // The worker's partition snapshot can precede further writes on + // the retained range. Rebuild its accounting from the serialized + // apply view, and preserve current metadata before any mutation. + let updated_anchor_attr = updated_anchor_attr + .map(|(pid, supplied)| { + let key = MetaKey::dir_attr(pid).encode(); + let mut attr = Self::read_latest_in_apply(engine, &key)? + .map(|raw| InodeAttr::decode(&raw)) + .transpose()? + .unwrap_or(supplied); + let (children, bytes, mtime) = self.retained_dir_stats( + engine, + &new_range, + pid, + attr.modify_time, + raft_index, + )?; + attr.num_children = children; + attr.size_bytes = bytes; + attr.modify_time = mtime; + Ok::<_, MetaDbError>((pid, attr)) + }) + .transpose()?; + let updated_shard_delta = updated_shard_delta + .map(|supplied| { + let key = DirShardDeltaKey { + pid: supplied.inode_id, + } + .encode(); + let mut sub = Self::read_latest_in_apply(engine, &key)? + .map(|raw| DirShardDeltaRecord::decode(&raw)) + .transpose()? + .unwrap_or(supplied); + let (children, bytes, mtime) = self.retained_dir_stats( + engine, + &new_range, + sub.inode_id, + sub.modify_time, + raft_index, + )?; + sub.num_children = children; + sub.size_bytes = bytes; + sub.modify_time = mtime; + Ok::<_, MetaDbError>(sub) + }) + .transpose()?; for k in keys_to_delete { engine.apply_committed_delete(k, commit_hlc, raft_index); } @@ -1534,6 +1970,37 @@ impl TxStateMachineHandler { } } + fn retained_dir_stats( + &self, + engine: &LifecycleStorageEngine, + range: &CompositeKeyRange, + pid: InodeId, + base_mtime: HlcTimestamp, + raft_index: u64, + ) -> Result<(i64, i64, HlcTimestamp)> { + let mut children = 0i64; + let mut bytes = 0i64; + let mut mtime = base_mtime.max(self.attr_agg.latest_delta_hlc(pid, raft_index)); + for (key, raw) in Self::scan_latest_in_apply( + engine, + &MetaKey::encode_pid_prefix_start(pid), + &MetaKey::encode_pid_prefix_end(pid), + )? { + if !range.contains_key(&key) || MetaKey::decode(&key)?.is_attr_key() { + continue; + } + let child = InodeAttr::decode(&raw)?; + children = children.checked_add(1).ok_or_else(|| { + MetaDbError::CodecError("directory child count overflow during split".into()) + })?; + bytes = bytes.checked_add(child.size_bytes).ok_or_else(|| { + MetaDbError::CodecError("directory byte count overflow during split".into()) + })?; + mtime = mtime.max(child.modify_time); + } + Ok((children, bytes, mtime)) + } + /// Materializes `[0x05][user_key]` if held by `txn_id` (`Resolution B12` exact-once check). fn materialize_intent_for_key( &self, @@ -1582,7 +2049,17 @@ impl TxStateMachineHandler { let final_val = if let Ok(mut decoded_attr) = InodeAttr::decode(&value) { decoded_attr.modify_time = commit_hlc; if decoded_attr.entry_type == EntryType::DirectoryAttr { - decoded_attr.applied_lsn = raft_index; + if let Some(raw) = Self::read_latest_in_apply(engine, user_key)? { + let current = InodeAttr::decode(&raw)?; + // Metadata-only 2PC must retain the current aggregation + // baseline, including flushes since Prepare. Pending + // deltas remain above this frontier for read/flush folding. + decoded_attr.num_children = current.num_children; + decoded_attr.size_bytes = current.size_bytes; + decoded_attr.applied_lsn = current.applied_lsn; + } else { + decoded_attr.applied_lsn = raft_index; + } } decoded_attr.encode() } else { diff --git a/crates/metadb-tx/src/two_pc.rs b/crates/metadb-tx/src/two_pc.rs index 50be4a4..6a51b70 100644 --- a/crates/metadb-tx/src/two_pc.rs +++ b/crates/metadb-tx/src/two_pc.rs @@ -3,7 +3,6 @@ //! Primary-Anchored Two-Phase Commit (`2PC`) Coordinator. use std::collections::{BTreeMap, HashMap}; -use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use metadb_core::schema::validate_user_entry_name; @@ -48,6 +47,14 @@ pub trait ShardDirectory: Send + Sync + 'static { /// Resolves `(ShardEpoch, Arc, Arc)` for `user_key`. fn route_user_key(&self, user_key: &[u8]) -> Result; + /// Allocates an identity that must never be reused, including after coordinator restart. + /// Implementations must reserve the sequence durably before returning it. + fn allocate_txn_id(&self, _coord_node_id: u64) -> Result { + Err(MetaDbError::TransactionAborted( + "ShardDirectory does not provide durable transaction IDs".to_string(), + )) + } + /// Resolves all shards intersecting `[start_user_key, end_user_key)` in ascending key order. fn route_user_range( &self, @@ -69,7 +76,6 @@ pub trait ShardDirectory: Send + Sync + 'static { /// Executes both `1-RTT` single-shard pushdown and Primary-Anchored `2PC` across `ShardRaftGroup`s. pub struct DistributedTxRouter { coord_node_id: u64, - next_txn_seq: AtomicU64, clock: Arc, directory: Arc, ttl_ms: u64, @@ -83,7 +89,6 @@ impl DistributedTxRouter { ) -> Self { Self { coord_node_id, - next_txn_seq: AtomicU64::new(1), clock, directory, ttl_ms: DEFAULT_2PC_TTL_MS, @@ -103,11 +108,6 @@ impl DistributedTxRouter { &self.clock } - fn alloc_txn_id(&self) -> TxnId { - let seq = self.next_txn_seq.fetch_add(1, Ordering::Relaxed); - ((self.coord_node_id as u128) << 64) | (seq as u128) - } - async fn propose_cmd( group: &ShardRaftGroup, epoch: ShardEpoch, @@ -335,7 +335,7 @@ impl DistributedTxRouter { }, ), ( - new_attr_bytes, + new_attr_bytes.clone(), IntentOp::Insert { value: self_attr.encode(), attr_deltas: vec![], @@ -345,7 +345,13 @@ impl DistributedTxRouter { ]; let commit_hlc = self - .execute_two_pc(&child_bytes, intents_by_key, None, TwoPcCrashStage::None) + .execute_two_pc( + &child_bytes, + intents_by_key, + vec![(child_bytes.clone(), None), (new_attr_bytes, None)], + None, + TwoPcCrashStage::None, + ) .await?; let mut committed_attr = dentry_attr; @@ -430,8 +436,14 @@ impl DistributedTxRouter { )); } - self.execute_two_pc(&child_bytes, intents_by_key, None, TwoPcCrashStage::None) - .await?; + self.execute_two_pc( + &child_bytes, + intents_by_key, + vec![(child_bytes.clone(), Some(entry.attr.encode()))], + None, + TwoPcCrashStage::None, + ) + .await?; Ok(entry.attr) } @@ -469,6 +481,12 @@ impl DistributedTxRouter { }; } + if patch.size_bytes.is_some() || patch.fid.is_some() { + return Err(MetaDbError::CodecError( + "directory size and fid are derived attributes".into(), + )); + } + // Directory `SetAttr`: check if single-shard colocated or multi-shard (`Resolution B10`). let dir_id = existing.attr.inode_id; let dir_start = MetaKey::encode_pid_prefix_start(dir_id); @@ -513,7 +531,11 @@ impl DistributedTxRouter { } let dir_attr_key = MetaKey::dir_attr(dir_id).encode(); - let mut dir_self = self.dirstat(dir_id).await?; + let dir_self_raw = self + .read_raw_user_value(&dir_attr_key) + .await? + .ok_or(MetaDbError::DirectoryNotFound(dir_id))?; + let mut dir_self = InodeAttr::decode(&dir_self_raw)?; dir_self.mode = new_dentry.mode; dir_self.uid = new_dentry.uid; @@ -527,7 +549,7 @@ impl DistributedTxRouter { }, ), ( - dir_attr_key, + dir_attr_key.clone(), IntentOp::Put { value: dir_self.encode(), attr_deltas: vec![], @@ -536,7 +558,16 @@ impl DistributedTxRouter { ), ]; let commit_hlc = self - .execute_two_pc(&child_bytes, intents, None, TwoPcCrashStage::None) + .execute_two_pc( + &child_bytes, + intents, + vec![ + (child_bytes.clone(), Some(existing.attr.encode())), + (dir_attr_key, Some(dir_self_raw)), + ], + None, + TwoPcCrashStage::None, + ) .await?; new_dentry.modify_time = commit_hlc; Ok(MetaEntry { @@ -635,6 +666,7 @@ impl DistributedTxRouter { }; } + crate::validation::validate_size(src_entry.attr.size_bytes)?; // Cross-shard 2PC `Rename`: let lease_opt = if src_entry.attr.entry_type == EntryType::Directory && src_pid != dst_pid { let lease = self @@ -658,12 +690,20 @@ impl DistributedTxRouter { // Check destination existence & victim accounting (`Resolution B11`). let dst_existing = self.lookup(dst_pid, dst_name).await?; + let mut expected_values = vec![ + (src_bytes.clone(), Some(src_entry.attr.encode())), + ( + dst_bytes.clone(), + dst_existing.as_ref().map(|entry| entry.attr.encode()), + ), + ]; let mut dst_aux_specs = Vec::new(); let src_accounted = src_entry.attr.size_bytes.max(0); let mut dst_delta_children = 1i64; let mut dst_delta_bytes = src_accounted; if let Some(victim_entry) = dst_existing { + crate::validation::validate_size(victim_entry.attr.size_bytes)?; if !overwrite { if let Some(l) = lease_opt { self.directory @@ -763,10 +803,16 @@ impl DistributedTxRouter { )); if moved_attr.entry_type == EntryType::Directory && src_pid != dst_pid { - let mut dir_self = self.dirstat(moved_attr.inode_id).await?; + let dir_self_key = MetaKey::dir_attr(moved_attr.inode_id).encode(); + let dir_self_raw = self + .read_raw_user_value(&dir_self_key) + .await? + .ok_or(MetaDbError::DirectoryNotFound(moved_attr.inode_id))?; + let mut dir_self = InodeAttr::decode(&dir_self_raw)?; dir_self.parent_pid = dst_pid; + expected_values.push((dir_self_key.clone(), Some(dir_self_raw))); intents.push(( - MetaKey::dir_attr(moved_attr.inode_id).encode(), + dir_self_key, IntentOp::Put { value: dir_self.encode(), attr_deltas: vec![], @@ -776,7 +822,7 @@ impl DistributedTxRouter { } let res = self - .execute_two_pc(&src_bytes, intents, lease_opt, crash_stage) + .execute_two_pc(&src_bytes, intents, expected_values, lease_opt, crash_stage) .await; if let Some(l) = lease_opt { @@ -797,10 +843,11 @@ impl DistributedTxRouter { &self, primary_key: &[u8], intents: Vec<(Vec, IntentOp)>, + expected_values: Vec<(Vec, Option>)>, lease_opt: Option, crash_stage: TwoPcCrashStage, ) -> Result { - let txn_id = self.alloc_txn_id(); + let txn_id = self.directory.allocate_txn_id(self.coord_node_id)?; let start_hlc = self.clock.now(); let expire_hlc = lease_opt .map(|l| l.expire_hlc) @@ -833,7 +880,12 @@ impl DistributedTxRouter { for (sid, (ep, grp, shard_intents)) in ordered_groups { let shard_keys: Vec> = shard_intents.iter().map(|(k, _)| k.clone()).collect(); - let prep_cmd = TxRaftCmd::Prepare2pc { + let shard_expected_values = expected_values + .iter() + .filter(|(key, _)| shard_keys.contains(key)) + .cloned() + .collect(); + let prep_cmd = TxRaftCmd::Prepare2pcChecked { txn_id, is_primary: sid == primary_shard_id, primary_key: primary_key.to_vec(), @@ -842,6 +894,7 @@ impl DistributedTxRouter { expire_hlc, rename_fence_token, intents: shard_intents, + expected_values: shard_expected_values, }; match Self::propose_cmd(&grp, ep, start_hlc, &prep_cmd).await { Ok((_, TxRaftOutput::Prepared { prepared_hlc })) => { @@ -988,6 +1041,38 @@ impl DistributedTxRouter { )) } + /// Reads committed bytes used as the predicate for an anchor attribute mutation. + async fn read_raw_user_value(&self, key: &[u8]) -> Result>> { + for _ in 0..12 { + let (epoch, group, _) = self.directory.route_user_key(key)?; + let (_, snap) = match group.linearizable_read(epoch).await { + Ok(pair) => pair, + Err(MetaDbError::EpochMismatch { .. } | MetaDbError::NotLeader { .. }) => continue, + Err(err) => return Err(err), + }; + let engine = group.engine(); + let intent_key = WriteIntentKey::new(key.to_vec()).encode(); + if let Some(raw) = engine.get_with_snapshot(&intent_key, &snap)? { + let intent = WriteIntentRecord::decode(&raw)?; + if !matches!(intent.op, IntentOp::DirExistsFenceSet { .. }) { + drop(snap); + TxnRecoveryWorker::resolve_single_txn( + self.directory.as_ref(), + &self.clock, + intent.txn_id, + &intent.primary_key, + ) + .await?; + continue; + } + } + return engine.get_with_snapshot(key, &snap); + } + Err(MetaDbError::ShardRoutingError( + "anchor read exceeded max intent/route retries".to_string(), + )) + } + /// Linearizable `DirStat(pid)`: /// - Fast path (`1` shard covers `[pid, pid+1)`): single `ReadIndex` + `reconcile_inode_attr`. /// - `DualModeFineGrained` (`K` sub-shards): parallel `ReadIndex` + shared `barrier_hlc` @@ -1200,6 +1285,26 @@ impl DistributedTxRouter { } } + pub async fn upsert_quota_limit( + &self, + uid: UserId, + insert_quota_limit_bytes: u64, + update_quota_limit_bytes: u64, + ) -> Result { + let qk = QuotaKey { uid }.encode(); + let cmd = TxRaftCmd::UpsertQuotaLimit { + uid, + insert_quota_limit_bytes, + update_quota_limit_bytes, + }; + match self.propose_key_with_retry(&qk, &cmd).await? { + TxRaftOutput::Quota(q) => Ok(q), + _ => Err(MetaDbError::CodecError( + "unexpected UpsertQuotaLimit output".into(), + )), + } + } + pub async fn get_quota(&self, uid: UserId) -> Result> { let qk = QuotaKey { uid }.encode(); let (epoch, group, _) = self.directory.route_user_key(&qk)?; diff --git a/crates/metadb-tx/src/validation.rs b/crates/metadb-tx/src/validation.rs new file mode 100644 index 0000000..d6026ca --- /dev/null +++ b/crates/metadb-tx/src/validation.rs @@ -0,0 +1,30 @@ +use metadb_core::{is_dir_inode_id, InodeId, MetaDbError, Result, ROOT_INODE_ID}; + +pub(crate) fn validate_size(size: i64) -> Result<()> { + if size < 0 { + return Err(MetaDbError::CodecError( + "size_bytes must be nonnegative".into(), + )); + } + Ok(()) +} + +pub(crate) fn validate_new_directory(pid: InodeId, id: InodeId) -> Result<()> { + if !is_dir_inode_id(id) || id == ROOT_INODE_ID || id == pid { + return Err(MetaDbError::CodecError( + "invalid new directory inode".into(), + )); + } + Ok(()) +} + +pub(crate) fn checked_count(value: i128) -> Result { + i64::try_from(value) + .ok() + .filter(|n| *n >= 0) + .ok_or_else(|| { + MetaDbError::CodecError( + "directory accounting is outside the nonnegative i64 domain".into(), + ) + }) +} diff --git a/crates/metadb-tx/tests/attr_reconcile_race_tests.rs b/crates/metadb-tx/tests/attr_reconcile_race_tests.rs index 096a11c..db4fae1 100644 --- a/crates/metadb-tx/tests/attr_reconcile_race_tests.rs +++ b/crates/metadb-tx/tests/attr_reconcile_race_tests.rs @@ -40,6 +40,11 @@ impl ShardDirectory for StaticTestDirectory { )) } + fn allocate_txn_id(&self, coord_node_id: u64) -> Result { + static NEXT_TXN: AtomicU64 = AtomicU64::new(1); + Ok(((coord_node_id as u128) << 64) | NEXT_TXN.fetch_add(1, Ordering::Relaxed) as u128) + } + fn route_user_range( &self, start_user_key: &[u8], diff --git a/crates/metadb-tx/tests/aux_idempotency_tests.rs b/crates/metadb-tx/tests/aux_idempotency_tests.rs index 039e0f7..71180d9 100644 --- a/crates/metadb-tx/tests/aux_idempotency_tests.rs +++ b/crates/metadb-tx/tests/aux_idempotency_tests.rs @@ -42,6 +42,11 @@ impl ShardDirectory for ThreeTableDirectory { Err(MetaDbError::ShardRoutingError("unrouted key".to_string())) } + fn allocate_txn_id(&self, coord_node_id: u64) -> Result { + static NEXT_TXN: AtomicU64 = AtomicU64::new(1); + Ok(((coord_node_id as u128) << 64) | NEXT_TXN.fetch_add(1, Ordering::Relaxed) as u128) + } + fn route_user_range( &self, start_user_key: &[u8], diff --git a/crates/metadb-tx/tests/round2_atomic_validation_tests.rs b/crates/metadb-tx/tests/round2_atomic_validation_tests.rs new file mode 100644 index 0000000..0dbd329 --- /dev/null +++ b/crates/metadb-tx/tests/round2_atomic_validation_tests.rs @@ -0,0 +1,421 @@ +use std::sync::Arc; + +use metadb_core::{ + AttrDeltaSpec, AuxOutboxEntry, AuxTaskPayload, CompositeKeyRange, HlcTimestamp, InodeAttr, + IntentOp, MetaKey, QuotaKey, QuotaRecord, ShardEpoch, WriteIntentKey, +}; +use metadb_raft::{MetaDbRaftRequest, ShardApplyHandler, ShardWatermarkTracker}; +use metadb_storage::{LifecycleStorageEngine, LsmEngineConfig, ShardedBlockCache}; +use metadb_tx::{SetAttrPatch, TxRaftCmd, TxStateMachineHandler}; + +fn apply( + handler: &TxStateMachineHandler, + engine: &LifecycleStorageEngine, + index: u64, + cmd: TxRaftCmd, +) -> metadb_core::Result> { + let hlc = HlcTimestamp::new(1, index as u16); + handler.apply_command( + index, + hlc, + &MetaDbRaftRequest { + expected_epoch: handler.current_epoch(), + hlc, + payload: bincode::serialize(&cmd).unwrap(), + }, + engine, + ) +} + +fn fixture() -> ( + tempfile::TempDir, + LifecycleStorageEngine, + TxStateMachineHandler, +) { + let dir = tempfile::tempdir().unwrap(); + let engine = LifecycleStorageEngine::open_lsm( + 1, + dir.path(), + Arc::new(ShardedBlockCache::new(1 << 20)), + LsmEngineConfig::default(), + ) + .unwrap(); + let handler = TxStateMachineHandler::new( + 1, + ShardEpoch::initial(), + CompositeKeyRange::all_user_tables(), + Arc::new(ShardWatermarkTracker::default()), + ); + apply( + &handler, + &engine, + 1, + TxRaftCmd::InitDirAttr { + dir_id: 1, + parent_pid: 1, + uid: 7, + mode: 0o755, + }, + ) + .unwrap(); + (dir, engine, handler) +} + +fn create(name: &str, size_bytes: i64) -> TxRaftCmd { + TxRaftCmd::CreateFile { + pid: 1, + name: name.into(), + inode_id: (1 << 63) + 10, + fid: 10, + uid: 7, + mode: 0o644, + size_bytes, + } +} + +fn read(engine: &LifecycleStorageEngine, key: &[u8]) -> Option> { + engine.get(key, HlcTimestamp::new(2, 0)).unwrap() +} + +#[test] +fn invalid_sizes_and_directory_capacity_fail_before_any_side_effect() { + let (_dir, engine, handler) = fixture(); + for (index, size) in [(2, -1), (3, i64::MIN)] { + assert!(apply(&handler, &engine, index, create("negative", size)).is_err()); + assert!(read(&engine, &MetaKey::child(1, "negative").encode()).is_none()); + assert_eq!(handler.attr_aggregator().pending_delta_count(), 0); + } + apply(&handler, &engine, 4, create("full", i64::MAX)).unwrap(); + assert!(apply(&handler, &engine, 5, create("overflow", 1)).is_err()); + assert!(read(&engine, &MetaKey::child(1, "overflow").encode()).is_none()); + assert_eq!(handler.attr_aggregator().pending_delta_count(), 1); + let before = read(&engine, &MetaKey::child(1, "full").encode()); + assert!(apply( + &handler, + &engine, + 6, + TxRaftCmd::SetAttrSingleShard { + pid: 1, + name: "full".into(), + patch: SetAttrPatch { + size_bytes: Some(i64::MIN), + ..Default::default() + } + } + ) + .is_err()); + assert_eq!(read(&engine, &MetaKey::child(1, "full").encode()), before); + apply( + &handler, + &engine, + 7, + TxRaftCmd::Unlink { + pid: 1, + name: "full".into(), + }, + ) + .unwrap(); + apply(&handler, &engine, 8, create("fits", 1)).unwrap(); +} + +#[test] +fn malformed_historical_size_is_rejected_before_unlink_deletes_the_key() { + let (_dir, engine, handler) = fixture(); + apply(&handler, &engine, 2, create("victim", 1)).unwrap(); + let key = MetaKey::child(1, "victim").encode(); + let mut attr = InodeAttr::decode(&read(&engine, &key).unwrap()).unwrap(); + attr.size_bytes = i64::MIN; + engine.apply_committed_put(key.clone(), attr.encode(), HlcTimestamp::new(1, 3), 3); + assert!(apply( + &handler, + &engine, + 4, + TxRaftCmd::Unlink { + pid: 1, + name: "victim".into() + } + ) + .is_err()); + assert_eq!(read(&engine, &key), Some(attr.encode())); +} + +fn quota_task( + origin: u64, + index: u64, + uid: u64, + bytes: i64, + inodes: i64, + previous: (u64, u16), +) -> AuxOutboxEntry { + AuxOutboxEntry { + origin_shard_id: origin, + origin_raft_index: index, + task_ordinal: 0, + hlc: HlcTimestamp::new(1, index as u16), + payload: AuxTaskPayload::QuotaDelta { + uid, + delta_bytes: bytes, + delta_inodes: inodes, + prev_uid_cursor: previous, + }, + } +} + +#[test] +fn quota_overflow_rejects_the_whole_batch_and_does_not_advance_frontiers() { + let (_dir, engine, handler) = fixture(); + apply( + &handler, + &engine, + 2, + TxRaftCmd::ApplyAuxBatch { + entries: vec![quota_task(1, 1, 7, i64::MAX, 1, (0, 0))], + }, + ) + .unwrap(); + let key = QuotaKey { uid: 7 }.encode(); + let before = read(&engine, &key).unwrap(); + assert!(apply( + &handler, + &engine, + 3, + TxRaftCmd::ApplyAuxBatch { + entries: vec![ + quota_task(2, 1, 8, 10, 1, (0, 0)), + quota_task(1, 2, 7, 1, 0, (1, 0)), + ] + } + ) + .is_err()); + assert!(read(&engine, &QuotaKey { uid: 8 }.encode()).is_none()); + assert_eq!(read(&engine, &key), Some(before)); + apply( + &handler, + &engine, + 4, + TxRaftCmd::ApplyAuxBatch { + entries: vec![quota_task(2, 1, 7, -i64::MAX, -1, (0, 0))], + }, + ) + .unwrap(); + let quota = QuotaRecord::decode(&read(&engine, &key).unwrap()).unwrap(); + assert_eq!((quota.used_bytes, quota.used_inodes), (0, 0)); +} + +#[test] +fn invalid_directory_ids_fail_on_single_shard_and_prepare_paths() { + let (_dir, engine, handler) = fixture(); + for id in [0, 1, 1 << 63, u64::MAX] { + assert!(apply( + &handler, + &engine, + 2, + TxRaftCmd::MkdirSingleShard { + pid: 1, + name: "alias".into(), + new_dir_id: id, + uid: 7, + mode: 0o755 + } + ) + .is_err()); + assert!(read(&engine, &MetaKey::child(1, "alias").encode()).is_none()); + let attr = InodeAttr { + inode_id: id, + parent_pid: 1, + entry_type: metadb_core::EntryType::Directory, + uid: 7, + mode: 0o755, + link_count: 1, + num_children: 0, + size_bytes: 0, + create_time: HlcTimestamp::new(1, 1), + modify_time: HlcTimestamp::new(1, 1), + fid: None, + applied_lsn: 0, + }; + let key = MetaKey::child(1, "alias").encode(); + let op = IntentOp::Put { + value: attr.encode(), + attr_deltas: vec![], + aux_specs: vec![], + }; + assert!(apply(&handler, &engine, 3, prepare(10, key.clone(), op)).is_err()); + assert!(read(&engine, &WriteIntentKey::new(key).encode()).is_none()); + } +} + +fn prepare(txn_id: u128, key: Vec, op: IntentOp) -> TxRaftCmd { + TxRaftCmd::Prepare2pc { + txn_id, + is_primary: false, + primary_key: key.clone(), + participant_keys: vec![key.clone()], + start_hlc: HlcTimestamp::new(1, 1), + expire_hlc: HlcTimestamp::new(5, 0), + rename_fence_token: None, + intents: vec![(key, op)], + } +} + +#[test] +fn prepared_positive_delta_reserves_capacity_until_commit_or_abort() { + let (_dir, engine, handler) = fixture(); + apply(&handler, &engine, 2, create("template", 0)).unwrap(); + let mut attr = + InodeAttr::decode(&read(&engine, &MetaKey::child(1, "template").encode()).unwrap()) + .unwrap(); + attr.size_bytes = i64::MAX; + let key = MetaKey::child(1, "reserved").encode(); + let op = IntentOp::Put { + value: attr.encode(), + attr_deltas: vec![AttrDeltaSpec { + pid: 1, + delta_children: 1, + delta_bytes: i64::MAX, + }], + aux_specs: vec![], + }; + apply(&handler, &engine, 3, prepare(10, key.clone(), op)).unwrap(); + assert!(apply(&handler, &engine, 4, create("overflow", 1)).is_err()); + apply( + &handler, + &engine, + 5, + TxRaftCmd::CommitSecondary2pc { + txn_id: 10, + commit_hlc: HlcTimestamp::new(1, 5), + target_user_keys: vec![key.clone()], + }, + ) + .unwrap(); + assert_eq!( + InodeAttr::decode(&read(&engine, &key).unwrap()) + .unwrap() + .size_bytes, + i64::MAX + ); + assert!(read(&engine, &WriteIntentKey::new(key).encode()).is_none()); +} + +#[test] +fn same_command_put_and_delete_preserve_maximum_directory_balance() { + let (_dir, engine, handler) = fixture(); + apply(&handler, &engine, 2, create("source", i64::MAX)).unwrap(); + let source = MetaKey::child(1, "source").encode(); + let target = MetaKey::child(1, "target").encode(); + let attr = read(&engine, &source).unwrap(); + let intents = vec![ + ( + target.clone(), + IntentOp::Put { + value: attr, + attr_deltas: vec![AttrDeltaSpec { + pid: 1, + delta_children: 1, + delta_bytes: i64::MAX, + }], + aux_specs: vec![], + }, + ), + ( + source.clone(), + IntentOp::Delete { + attr_deltas: vec![AttrDeltaSpec { + pid: 1, + delta_children: -1, + delta_bytes: -i64::MAX, + }], + aux_specs: vec![], + }, + ), + ]; + apply( + &handler, + &engine, + 3, + TxRaftCmd::Prepare2pc { + txn_id: 20, + is_primary: false, + primary_key: source.clone(), + participant_keys: vec![target.clone(), source.clone()], + start_hlc: HlcTimestamp::new(1, 3), + expire_hlc: HlcTimestamp::new(5, 0), + rename_fence_token: None, + intents, + }, + ) + .unwrap(); + // Put is materialized first, so its intermediate delta exceeds the i64 + // balance. Only the completed command is visible to readers and flushes. + apply( + &handler, + &engine, + 4, + TxRaftCmd::CommitSecondary2pc { + txn_id: 20, + commit_hlc: HlcTimestamp::new(1, 4), + target_user_keys: vec![target.clone(), source.clone()], + }, + ) + .unwrap(); + let snap = engine.acquire_linearizable_snapshot(4).unwrap(); + let base = InodeAttr::decode( + &engine + .get_with_snapshot(&MetaKey::dir_attr(1).encode(), &snap) + .unwrap() + .unwrap(), + ) + .unwrap(); + let result = handler + .attr_aggregator() + .reconcile_inode_attr(1, base, &snap); + assert_eq!((result.num_children, result.size_bytes), (1, i64::MAX)); + assert!(read(&engine, &source).is_none()); + assert!(read(&engine, &target).is_some()); +} + +#[test] +fn fence_rechecks_intents_at_the_serialized_apply_boundary() { + let (_dir, engine, handler) = fixture(); + apply(&handler, &engine, 2, create("template", 0)).unwrap(); + let key = MetaKey::child(1, "prepared").encode(); + let op = IntentOp::Put { + value: read(&engine, &MetaKey::child(1, "template").encode()).unwrap(), + attr_deltas: vec![AttrDeltaSpec { + pid: 1, + delta_children: 1, + delta_bytes: 0, + }], + aux_specs: vec![], + }; + apply(&handler, &engine, 3, prepare(30, key.clone(), op)).unwrap(); + let before = handler.key_range(); + let fence = TxRaftCmd::FenceShardRangeForSplit { + new_range: CompositeKeyRange::new( + before.start_key.clone(), + MetaKey::encode_pid_prefix_start(100), + ) + .unwrap(), + new_epoch: ShardEpoch { + conf_ver: 1, + version: 2, + }, + }; + // Simulate the lease expiring between the worker's scan and Raft apply. + let hlc = HlcTimestamp::new(6, 0); + assert!(handler + .apply_command( + 4, + hlc, + &MetaDbRaftRequest { + expected_epoch: handler.current_epoch(), + hlc, + payload: bincode::serialize(&fence).unwrap() + }, + &engine + ) + .is_err()); + assert_eq!(handler.key_range(), before); + assert!(read(&engine, &WriteIntentKey::new(key).encode()).is_some()); +} diff --git a/crates/metadb-tx/tests/round2_snapshot_reconcile_tests.rs b/crates/metadb-tx/tests/round2_snapshot_reconcile_tests.rs new file mode 100644 index 0000000..5026dab --- /dev/null +++ b/crates/metadb-tx/tests/round2_snapshot_reconcile_tests.rs @@ -0,0 +1,142 @@ +//! Deterministic interleaving of a pinned read cut and the attr-delta flush hook. +use std::sync::Arc; + +use metadb_core::{ + CompositeKeyRange, DirShardDeltaKey, DirShardDeltaRecord, HlcTimestamp, InodeAttr, MetaKey, + ShardEpoch, +}; +use metadb_raft::{MetaDbRaftRequest, ShardApplyHandler, ShardWatermarkTracker}; +use metadb_storage::{ + DurableShardCheckpoint, LifecycleStorageEngine, LsmEngineConfig, ShardedBlockCache, +}; +use metadb_tx::{TxRaftCmd, TxStateMachineHandler}; + +fn apply( + handler: &TxStateMachineHandler, + engine: &LifecycleStorageEngine, + index: u64, + cmd: TxRaftCmd, +) { + let hlc = HlcTimestamp::new(1, index as u16); + handler + .apply_command( + index, + hlc, + &MetaDbRaftRequest { + expected_epoch: handler.current_epoch(), + hlc, + payload: bincode::serialize(&cmd).unwrap(), + }, + engine, + ) + .unwrap(); +} + +fn pinned_read_must_retain_its_unfolded_deltas(anchor: bool) { + let temp = tempfile::tempdir().unwrap(); + let engine = LifecycleStorageEngine::open_lsm( + 1, + temp.path(), + Arc::new(ShardedBlockCache::new(1024 * 1024)), + LsmEngineConfig::default(), + ) + .unwrap(); + let handler = TxStateMachineHandler::new( + 1, + ShardEpoch::initial(), + CompositeKeyRange::meta_table_root(), + Arc::new(ShardWatermarkTracker::default()), + ); + let init = if anchor { + TxRaftCmd::InitDirAttr { + dir_id: 100, + parent_pid: 1, + uid: 10, + mode: 0o755, + } + } else { + TxRaftCmd::InitDirShardDelta { + record: DirShardDeltaRecord { + inode_id: 100, + parent_pid: 1, + uid: 10, + mode: 0o755, + create_time: HlcTimestamp::new(1, 1), + num_children: 0, + size_bytes: 0, + modify_time: HlcTimestamp::new(1, 1), + applied_raft_index: 1, + deleting_fence: false, + }, + } + }; + apply(&handler, &engine, 1, init); + apply( + &handler, + &engine, + 2, + TxRaftCmd::CreateFile { + pid: 100, + name: "child".into(), + inode_id: 101, + fid: 201, + uid: 10, + mode: 0o644, + size_bytes: 17, + }, + ); + + // DirStat has acquired its read cut. A flush can run immediately after this + // lock-protected cut, before DirStat reads/reconciles the pinned baseline. + let snap = engine.acquire_linearizable_snapshot(2).unwrap(); + let key = if anchor { + MetaKey::dir_attr(100).encode() + } else { + DirShardDeltaKey { pid: 100 }.encode().to_vec() + }; + // DirStat obtains owned baseline bytes before entering reconcile's queue lock. + let raw = engine.get_with_snapshot(&key, &snap).unwrap().unwrap(); + let read = || { + if anchor { + let base = InodeAttr::decode(&raw).unwrap(); + let attr = handler + .attr_aggregator() + .reconcile_inode_attr(100, base, &snap); + (attr.num_children, attr.size_bytes) + } else { + let base = DirShardDeltaRecord::decode(&raw).unwrap(); + let attr = handler + .attr_aggregator() + .reconcile_dir_shard_delta(100, base, &snap); + (attr.num_children, attr.size_bytes) + } + }; + assert_eq!(read(), (1, 17)); + engine + .seal_and_flush_with_hook( + DurableShardCheckpoint { + flushed_raft_index: 2, + flushed_max_hlc: HlcTimestamp::new(1, 2), + ..Default::default() + }, + Some(&handler), + ) + .unwrap(); + let after = read(); + eprintln!("anchor={anchor}: same pinned cut before flush=(1, 17), after flush={after:?}"); + assert_eq!( + after, + (1, 17), + "flush must preserve the already pinned read's directory totals" + ); +} + +#[test] +fn anchor_snapshot_keeps_deltas_until_reconciliation() { + pinned_read_must_retain_its_unfolded_deltas(true); +} + +#[test] +fn non_anchor_snapshot_keeps_deltas_until_reconciliation() { + pinned_read_must_retain_its_unfolded_deltas(false); +} diff --git a/crates/metadb-tx/tests/split_attr_apply_tests.rs b/crates/metadb-tx/tests/split_attr_apply_tests.rs new file mode 100644 index 0000000..a7c3ff5 --- /dev/null +++ b/crates/metadb-tx/tests/split_attr_apply_tests.rs @@ -0,0 +1,390 @@ +use std::sync::Arc; + +use metadb_core::{ + CompositeKeyRange, DirShardDeltaKey, DirShardDeltaRecord, HlcTimestamp, InodeAttr, MetaKey, + ShardEpoch, +}; +use metadb_raft::{MetaDbRaftRequest, ShardApplyHandler, ShardWatermarkTracker}; +use metadb_storage::{ + DurableShardCheckpoint, LifecycleStorageEngine, LsmEngineConfig, ShardedBlockCache, +}; +use metadb_tx::{TxRaftCmd, TxStateMachineHandler}; + +fn apply( + handler: &TxStateMachineHandler, + engine: &LifecycleStorageEngine, + index: u64, + cmd: TxRaftCmd, +) { + let hlc = HlcTimestamp::new(1, index as u16); + handler + .apply_command( + index, + hlc, + &MetaDbRaftRequest { + expected_epoch: handler.current_epoch(), + hlc, + payload: bincode::serialize(&cmd).unwrap(), + }, + engine, + ) + .unwrap(); +} + +fn flush(handler: &TxStateMachineHandler, engine: &LifecycleStorageEngine, index: u64) { + engine + .seal_and_flush_with_hook( + DurableShardCheckpoint { + flushed_raft_index: index, + flushed_max_hlc: HlcTimestamp::new(1, index as u16), + contiguous_aux_acked_index: 0, + ..Default::default() + }, + Some(handler), + ) + .unwrap(); +} + +fn run_stale_split_case(anchor: bool, flush_intervening_write: bool) { + let temp = tempfile::tempdir().unwrap(); + let cache = Arc::new(ShardedBlockCache::new(1024 * 1024)); + let engine = + LifecycleStorageEngine::open_lsm(1, temp.path(), cache.clone(), LsmEngineConfig::default()) + .unwrap(); + let start = if anchor { + MetaKey::encode_pid_prefix_start(100) + } else { + MetaKey::child(100, "a").encode() + }; + let range = CompositeKeyRange::new(start.clone(), MetaKey::encode_pid_prefix_end(100)).unwrap(); + let handler = TxStateMachineHandler::new( + 1, + ShardEpoch::initial(), + range, + Arc::new(ShardWatermarkTracker::default()), + ); + if anchor { + apply( + &handler, + &engine, + 1, + TxRaftCmd::InitDirAttr { + dir_id: 100, + parent_pid: 1, + uid: 10, + mode: 0o755, + }, + ); + } else { + apply( + &handler, + &engine, + 1, + TxRaftCmd::InitDirShardDelta { + record: DirShardDeltaRecord { + inode_id: 100, + parent_pid: 1, + uid: 10, + mode: 0o755, + create_time: HlcTimestamp::new(1, 1), + num_children: 0, + size_bytes: 0, + modify_time: HlcTimestamp::new(1, 1), + applied_raft_index: 1, + deleting_fence: false, + }, + }, + ); + } + for (index, name, size) in [(2, "a", 10), (3, "z", 20)] { + apply( + &handler, + &engine, + index, + TxRaftCmd::CreateFile { + pid: 100, + name: name.into(), + inode_id: index, + fid: index, + uid: 10, + mode: 0o644, + size_bytes: size, + }, + ); + } + flush(&handler, &engine, 3); + let attr_key = MetaKey::dir_attr(100).encode(); + let delta_key = DirShardDeltaKey { pid: 100 }.encode().to_vec(); + // The worker captured only one left child before a later left-side write. + let stale_anchor = if anchor { + let mut attr = InodeAttr::decode( + &engine + .get(&attr_key, HlcTimestamp::new(1, 3)) + .unwrap() + .unwrap(), + ) + .unwrap(); + attr.num_children = 1; + attr.size_bytes = 10; + Some((100, attr)) + } else { + None + }; + let stale_delta = if !anchor { + let mut sub = DirShardDeltaRecord::decode( + &engine + .get(&delta_key, HlcTimestamp::new(1, 3)) + .unwrap() + .unwrap(), + ) + .unwrap(); + sub.num_children = 1; + sub.size_bytes = 10; + Some(sub) + } else { + None + }; + let left = CompositeKeyRange::new(start, MetaKey::child(100, "m").encode()).unwrap(); + let epoch = ShardEpoch::initial().bump_version(); + apply( + &handler, + &engine, + 4, + TxRaftCmd::FenceShardRangeForSplit { + new_range: left.clone(), + new_epoch: epoch, + }, + ); + apply( + &handler, + &engine, + 5, + TxRaftCmd::CreateFile { + pid: 100, + name: "b".into(), + inode_id: 5, + fid: 5, + uid: 10, + mode: 0o644, + size_bytes: 30, + }, + ); + if flush_intervening_write { + flush(&handler, &engine, 5); + } + apply( + &handler, + &engine, + 6, + TxRaftCmd::UpdateShardRangeOnSplit { + new_range: left, + new_epoch: epoch, + keys_to_delete: vec![MetaKey::child(100, "z").encode()], + updated_anchor_attr: stale_anchor, + updated_shard_delta: stale_delta, + }, + ); + let snapshot = engine.acquire_linearizable_snapshot(6).unwrap(); + let (count, bytes, mtime) = if anchor { + let raw = engine + .get_with_snapshot(&attr_key, &snapshot) + .unwrap() + .unwrap(); + let attr = handler.attr_aggregator().reconcile_inode_attr( + 100, + InodeAttr::decode(&raw).unwrap(), + &snapshot, + ); + (attr.num_children, attr.size_bytes, attr.modify_time) + } else { + let raw = engine + .get_with_snapshot(&delta_key, &snapshot) + .unwrap() + .unwrap(); + let sub = handler.attr_aggregator().reconcile_dir_shard_delta( + 100, + DirShardDeltaRecord::decode(&raw).unwrap(), + &snapshot, + ); + (sub.num_children, sub.size_bytes, sub.modify_time) + }; + assert_eq!( + (count, bytes), + (2, 40), + "anchor={anchor}, flushed={flush_intervening_write}" + ); + assert!(mtime >= HlcTimestamp::new(1, 5)); + drop(snapshot); + flush(&handler, &engine, 6); + drop(engine); + let reopened = + LifecycleStorageEngine::open_lsm(1, temp.path(), cache, LsmEngineConfig::default()) + .unwrap(); + let raw = reopened + .get( + if anchor { &attr_key } else { &delta_key }, + HlcTimestamp::new(1, 6), + ) + .unwrap() + .unwrap(); + let stats = if anchor { + let attr = InodeAttr::decode(&raw).unwrap(); + (attr.num_children, attr.size_bytes) + } else { + let sub = DirShardDeltaRecord::decode(&raw).unwrap(); + (sub.num_children, sub.size_bytes) + }; + assert_eq!(stats, (2, 40), "split accounting changed after reopen"); +} + +#[test] +fn split_update_keeps_pending_anchor_writes() { + run_stale_split_case(true, false); +} +#[test] +fn split_update_keeps_flushed_anchor_writes() { + run_stale_split_case(true, true); +} +#[test] +fn split_update_keeps_pending_non_anchor_writes() { + run_stale_split_case(false, false); +} +#[test] +fn split_update_keeps_flushed_non_anchor_writes() { + run_stale_split_case(false, true); +} + +fn run_anchor_metadata_case(flush_after_prepare: bool) { + let temp = tempfile::tempdir().unwrap(); + let cache = Arc::new(ShardedBlockCache::new(1024 * 1024)); + let engine = + LifecycleStorageEngine::open_lsm(1, temp.path(), cache.clone(), LsmEngineConfig::default()) + .unwrap(); + let handler = TxStateMachineHandler::new( + 1, + ShardEpoch::initial(), + CompositeKeyRange::meta_table_root(), + Arc::new(ShardWatermarkTracker::default()), + ); + apply( + &handler, + &engine, + 1, + TxRaftCmd::InitDirAttr { + dir_id: 100, + parent_pid: 1, + uid: 10, + mode: 0o755, + }, + ); + apply( + &handler, + &engine, + 2, + TxRaftCmd::CreateFile { + pid: 100, + name: "child".into(), + inode_id: 2, + fid: 2, + uid: 10, + mode: 0o644, + size_bytes: 10, + }, + ); + let key = MetaKey::dir_attr(100).encode(); + let raw = engine.get(&key, HlcTimestamp::new(1, 2)).unwrap().unwrap(); + let mut attr = InodeAttr::decode(&raw).unwrap(); + // Model the coordinator's reconciled DirStat snapshot, followed by + // another committed child before it acquires the metadata intent. + attr.num_children = 1; + attr.size_bytes = 10; + attr.applied_lsn = 2; + attr.mode = 0o700; + apply( + &handler, + &engine, + 3, + TxRaftCmd::CreateFile { + pid: 100, + name: "later".into(), + inode_id: 3, + fid: 3, + uid: 10, + mode: 0o644, + size_bytes: 20, + }, + ); + apply( + &handler, + &engine, + 4, + TxRaftCmd::Prepare2pcChecked { + txn_id: 1, + is_primary: true, + primary_key: key.clone(), + participant_keys: vec![key.clone()], + start_hlc: HlcTimestamp::new(1, 4), + expire_hlc: HlcTimestamp::new(1000, 0), + rename_fence_token: None, + intents: vec![( + key.clone(), + metadb_core::IntentOp::Put { + value: attr.encode(), + attr_deltas: vec![], + aux_specs: vec![], + }, + )], + expected_values: vec![(key.clone(), Some(raw))], + }, + ); + if flush_after_prepare { + flush(&handler, &engine, 4); + } + apply( + &handler, + &engine, + 5, + TxRaftCmd::CommitPrimary2pc { + txn_id: 1, + commit_hlc: HlcTimestamp::new(1, 5), + rename_fence_token: None, + }, + ); + let snapshot = engine.acquire_linearizable_snapshot(5).unwrap(); + let raw = engine.get_with_snapshot(&key, &snapshot).unwrap().unwrap(); + let attr = handler.attr_aggregator().reconcile_inode_attr( + 100, + InodeAttr::decode(&raw).unwrap(), + &snapshot, + ); + assert_eq!( + (attr.num_children, attr.size_bytes, attr.mode), + (2, 30, 0o700) + ); + drop(snapshot); + flush(&handler, &engine, 5); + drop(engine); + let reopened = + LifecycleStorageEngine::open_lsm(1, temp.path(), cache, LsmEngineConfig::default()) + .unwrap(); + let attr = InodeAttr::decode( + &reopened + .get(&key, HlcTimestamp::new(1, 5)) + .unwrap() + .unwrap(), + ) + .unwrap(); + assert_eq!( + (attr.num_children, attr.size_bytes, attr.mode), + (2, 30, 0o700) + ); +} + +#[test] +fn metadata_two_pc_preserves_pending_child_accounting() { + run_anchor_metadata_case(false); +} +#[test] +fn metadata_two_pc_preserves_child_accounting_flushed_after_prepare() { + run_anchor_metadata_case(true); +} diff --git a/crates/metadb-tx/tests/two_pc_compare_tests.rs b/crates/metadb-tx/tests/two_pc_compare_tests.rs new file mode 100644 index 0000000..a59b1fe --- /dev/null +++ b/crates/metadb-tx/tests/two_pc_compare_tests.rs @@ -0,0 +1,332 @@ +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{mpsc, Arc, Condvar, Mutex}; +use std::time::Duration; + +use metadb_core::{ + CompositeKeyRange, HybridLogicalClock, InodeId, MetaDbError, MetaKey, Result, ShardEpoch, + TxnId, TxnStatusRecord, ROOT_INODE_ID, SYSTEM_PREFIX_TXN_RECORD, SYSTEM_PREFIX_WRITE_INTENT, +}; +use metadb_raft::{ + ChannelPool, MultiRaftRouter, RaftNodeOptions, ShardRaftGroup, ShardWatermarkTracker, +}; +use metadb_storage::{LsmEngineConfig, ShardedBlockCache}; +use metadb_tx::{ + AuxDispatcher, DirRenameLease, DistributedTxRouter, RoutedShardHandle, SetAttrPatch, + ShardDirectory, TwoPcCrashStage, TxStateMachineHandler, TxnRecoveryWorker, +}; + +struct TestDirectory { + shards: Vec<(CompositeKeyRange, RoutedShardHandle)>, +} + +#[tonic::async_trait] +impl ShardDirectory for TestDirectory { + fn route_user_key(&self, key: &[u8]) -> Result { + self.shards + .iter() + .find(|(range, _)| range.contains_key(key)) + .map(|(_, handle)| handle.clone()) + .ok_or_else(|| MetaDbError::ShardRoutingError("unrouted test key".into())) + } + + fn allocate_txn_id(&self, node: u64) -> Result { + static NEXT: AtomicU64 = AtomicU64::new(1); + Ok(((node as u128) << 64) | NEXT.fetch_add(1, Ordering::Relaxed) as u128) + } + + fn route_user_range(&self, start: &[u8], end: &[u8]) -> Result> { + Ok(self + .shards + .iter() + .filter(|(range, _)| range.overlaps(start, end)) + .map(|(_, handle)| handle.clone()) + .collect()) + } + + async fn acquire_dir_rename_lease(&self, _: InodeId, _: u64) -> Result { + unreachable!("file rename needs no directory lease") + } + + async fn release_dir_rename_lease(&self, _: u64) {} +} + +/// Pause immediately after both rename reads, before the first Prepare, without +/// depending on task scheduling or arbitrary sleeps to create a stale read. +struct PausedDirectory { + base: Arc, + source_key: Vec, + visits: AtomicU64, + reached: mpsc::Sender<()>, + released: Arc<(Mutex, Condvar)>, +} + +#[tonic::async_trait] +impl ShardDirectory for PausedDirectory { + fn route_user_key(&self, key: &[u8]) -> Result { + if key == self.source_key && self.visits.fetch_add(1, Ordering::Relaxed) == 2 { + self.reached.send(()).unwrap(); + let (lock, cv) = &*self.released; + let ready = lock.lock().unwrap(); + let (ready, timeout) = cv + .wait_timeout_while(ready, Duration::from_secs(10), |r| !*r) + .unwrap(); + assert!( + *ready && !timeout.timed_out(), + "concurrent edit never released rename" + ); + } + self.base.route_user_key(key) + } + + fn allocate_txn_id(&self, node: u64) -> Result { + self.base.allocate_txn_id(node) + } + + fn route_user_range(&self, start: &[u8], end: &[u8]) -> Result> { + self.base.route_user_range(start, end) + } + + async fn acquire_dir_rename_lease(&self, id: InodeId, ttl: u64) -> Result { + self.base.acquire_dir_rename_lease(id, ttl).await + } + + async fn release_dir_rename_lease(&self, token: u64) { + self.base.release_dir_rename_lease(token).await + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn stale_rename_source_destination_and_victim_never_commit() { + let temp = tempfile::tempdir().unwrap(); + let raft_router = MultiRaftRouter::new(); + let clock = Arc::new(HybridLogicalClock::new()); + let cache = Arc::new(ShardedBlockCache::new(8 * 1024 * 1024)); + let ranges = [ + CompositeKeyRange::meta_pid_range(0, 500).unwrap(), + CompositeKeyRange::meta_pid_range(500, u64::MAX).unwrap(), + CompositeKeyRange::quota_table_root(), + CompositeKeyRange::secondary_index_table_root(), + ]; + let mut shards = Vec::new(); + for (index, range) in ranges.into_iter().enumerate() { + let sid = index as u64 + 1; + let wm = Arc::new(ShardWatermarkTracker::default()); + let handler = Arc::new(TxStateMachineHandler::new( + sid, + ShardEpoch::initial(), + range.clone(), + wm.clone(), + )); + let group = ShardRaftGroup::start( + sid, + 1, + temp.path().join(format!("shard_{sid}")), + cache.clone(), + LsmEngineConfig::default(), + RaftNodeOptions::default(), + handler.clone(), + wm, + clock.clone(), + &raft_router, + ChannelPool::default(), + ) + .await + .unwrap(); + group + .initialize_if_needed(BTreeMap::from([(1, "127.0.0.1:0".into())])) + .await + .unwrap(); + shards.push((range, (ShardEpoch::initial(), group, handler))); + } + let directory = Arc::new(TestDirectory { shards }); + let tx = DistributedTxRouter::new(1, clock.clone(), directory.clone()); + tx.init_dir_attr(ROOT_INODE_ID, ROOT_INODE_ID, 0, 0o755) + .await + .unwrap(); + tx.mkdir(ROOT_INODE_ID, "src", 100, 10, 0o755) + .await + .unwrap(); + tx.mkdir(ROOT_INODE_ID, "dst", 600, 10, 0o755) + .await + .unwrap(); + + for (case, overwrite) in [("source", false), ("absent", false), ("victim", true)] { + tx.create_file(100, case, 1000, 8000, 10, 0o644, 10) + .await + .unwrap(); + if overwrite { + tx.create_file(600, case, 2000, 9000, 20, 0o644, 20) + .await + .unwrap(); + } + let (send, recv) = mpsc::channel(); + let released = Arc::new((Mutex::new(false), Condvar::new())); + let paused = Arc::new(PausedDirectory { + base: directory.clone(), + source_key: MetaKey::child(100, case).encode(), + visits: AtomicU64::new(0), + reached: send, + released: released.clone(), + }); + let stale_router = DistributedTxRouter::new(1, clock.clone(), paused); + let rename = + tokio::spawn(async move { stale_router.rename(100, case, 600, case, overwrite).await }); + // Run the blocking receive off the runtime so all Raft workers stay schedulable. + tokio::task::spawn_blocking(move || recv.recv_timeout(Duration::from_secs(10)).unwrap()) + .await + .unwrap(); + match case { + "source" => { + tx.setattr( + 100, + case, + SetAttrPatch { + size_bytes: Some(80), + uid: Some(30), + ..Default::default() + }, + ) + .await + .unwrap(); + } + "absent" => { + tx.create_file(600, case, 3000, 10000, 40, 0o600, 30) + .await + .unwrap(); + } + "victim" => { + tx.setattr( + 600, + case, + SetAttrPatch { + size_bytes: Some(90), + uid: Some(50), + ..Default::default() + }, + ) + .await + .unwrap(); + } + _ => unreachable!(), + } + *released.0.lock().unwrap() = true; + released.1.notify_all(); + let error = rename.await.unwrap().unwrap_err(); + assert!( + matches!(error, MetaDbError::TransactionAborted(_)), + "{case}: {error:?}" + ); + let source = tx.lookup(100, case).await.unwrap().unwrap(); + assert_eq!( + source.attr.size_bytes, + if case == "source" { 80 } else { 10 } + ); + let destination = tx.lookup(600, case).await.unwrap(); + match case { + "source" => assert!(destination.is_none()), + "absent" => assert_eq!(destination.as_ref().unwrap().attr.inode_id, 3000), + "victim" => assert_eq!(destination.as_ref().unwrap().attr.size_bytes, 90), + _ => unreachable!(), + } + for (_, (_, group, _)) in &directory.shards[..2] { + let (_, snapshot) = group + .linearizable_read(group.current_epoch()) + .await + .unwrap(); + let intents = group + .engine() + .scan_with_snapshot( + &[SYSTEM_PREFIX_WRITE_INTENT], + &[SYSTEM_PREFIX_WRITE_INTENT + 1], + &snapshot, + ) + .unwrap(); + assert!(intents.is_empty(), "rejected {case} prepare leaked intents"); + } + tx.unlink(100, case).await.unwrap(); + if destination.is_some() { + tx.unlink(600, case).await.unwrap(); + } + } + + // Recreating a coordinator on the same node must not reuse the first + // prepared transaction's identity or overwrite its retained primary record. + for (name, fid) in [("restart_a", 11000), ("restart_b", 11001)] { + tx.create_file(100, name, fid, fid, 60, 0o644, 1) + .await + .unwrap(); + let recreated = + DistributedTxRouter::new(1, clock.clone(), directory.clone()).with_ttl_ms(1000); + let error = recreated + .rename_with_fault_injection( + 100, + name, + 600, + name, + false, + TwoPcCrashStage::StopAfterPrepareBeforePrimaryCommit, + ) + .await + .unwrap_err(); + assert!(matches!(error, MetaDbError::RpcError(_))); + } + let primary = &directory.shards[0].1 .1; + let (_, snapshot) = primary + .linearizable_read(primary.current_epoch()) + .await + .unwrap(); + let records = primary + .engine() + .scan_with_snapshot( + &[SYSTEM_PREFIX_TXN_RECORD], + &[SYSTEM_PREFIX_TXN_RECORD + 1], + &snapshot, + ) + .unwrap(); + assert_eq!( + records.len(), + 2, + "recreated coordinator aliased an orphan transaction" + ); + let ids: std::collections::BTreeSet<_> = records + .iter() + .map(|(_, raw)| TxnStatusRecord::decode(raw).unwrap().txn_id) + .collect(); + assert_eq!(ids.len(), 2); + drop(snapshot); + tokio::time::sleep(Duration::from_millis(1100)).await; + for (_, (_, group, handler)) in &directory.shards[..2] { + TxnRecoveryWorker::run_recovery_pass_on_shard(group, handler, directory.as_ref(), &clock) + .await + .unwrap(); + } + for name in ["restart_a", "restart_b"] { + assert!(tx.lookup(100, name).await.unwrap().is_some()); + assert!(tx.lookup(600, name).await.unwrap().is_none()); + tx.unlink(100, name).await.unwrap(); + } + + for (_, (_, group, handler)) in &directory.shards[..2] { + AuxDispatcher::drain_shard_outbox(group, handler, directory.as_ref(), &clock) + .await + .unwrap(); + } + for uid in [20, 30, 40, 50, 60] { + let quota = tx.get_quota(uid).await.unwrap().unwrap(); + assert_eq!((quota.used_bytes, quota.used_inodes), (0, 0), "uid={uid}"); + } + for fid in [8000, 9000, 10000, 11000, 11001] { + assert!( + tx.get_secondary_index(fid).await.unwrap().is_none(), + "fid={fid}" + ); + } + for pid in [100, 600] { + let attr = tx.dirstat(pid).await.unwrap(); + assert_eq!((attr.num_children, attr.size_bytes), (0, 0)); + } + for (_, (_, group, _)) in &directory.shards { + group.shutdown().await.unwrap(); + } +} diff --git a/crates/metadb-tx/tests/two_pc_crash_tests.rs b/crates/metadb-tx/tests/two_pc_crash_tests.rs index 9a807fb..ca598a1 100644 --- a/crates/metadb-tx/tests/two_pc_crash_tests.rs +++ b/crates/metadb-tx/tests/two_pc_crash_tests.rs @@ -41,6 +41,11 @@ impl ShardDirectory for MultiShardTestCluster { Err(MetaDbError::ShardRoutingError("unrouted key".to_string())) } + fn allocate_txn_id(&self, coord_node_id: u64) -> Result { + static NEXT_TXN: AtomicU64 = AtomicU64::new(1); + Ok(((coord_node_id as u128) << 64) | NEXT_TXN.fetch_add(1, Ordering::Relaxed) as u128) + } + fn route_user_range( &self, start_user_key: &[u8], diff --git a/docs/designs/correctness_hardening.md b/docs/designs/correctness_hardening.md new file mode 100644 index 0000000..3c4e672 --- /dev/null +++ b/docs/designs/correctness_hardening.md @@ -0,0 +1,59 @@ +# Recovery, transaction, and operator correctness hardening + +This change hardens acknowledged-write recovery, transaction migration, input validation, and operator behavior. It includes deterministic reproductions, real TCP/gRPC tests, and real daemon/CLI tests. It does not implement disaggregated daemon roles or a replicated multi-Cell control plane. + +## Recovery and snapshots + +Recovery validates the entire unpurged committed log prefix, frame ordering and identity, and the durable vote/commit metadata before repairing files. Existing histories with missing `HARD_STATE` are rejected. A durable initialization marker also protects empty or fully purged stores from silently resetting their election identity. + +Snapshot installation requires a complete existing database image. It verifies required and duplicate filenames, the recovered checkpoint, LogId and membership, MANIFEST-to-SST metadata equality, and all data blocks before replacing live data. An empty or stale archive cannot create a new empty database and advance its advertised applied index. Snapshot publication syncs the temporary directory before its rename. + +A state-machine operation lock serializes complete apply operations, checkpoint creation, snapshot replacement, and read-cut acquisition. This favors correctness and can pause writers while a snapshot is built. A later optimization should pin immutable state under a shorter lock and stream it without weakening the atomic checkpoint boundary. + +MANIFEST recovery rejects incomplete baselines, complete CRC failures, and missing CURRENT pointers in existing databases. Valid incomplete append tails remain recoverable. The existing v1 framing does not protect its length header independently, so simultaneous structural payload and length corruption still needs a future format upgrade for complete discrimination. Full SST metadata equality detects mismatched recorded histories; identical metadata with different payload identity requires a future content digest or file UUID. + +## Transactions and accounting + +Checked prepares compare the exact committed values read by a coordinator before installing any intent. New command variants are appended to preserve existing bincode variant indices. Atomic quota UPSERT keeps both SQL branches until serialized apply chooses between them. + +Splitting waits until local intents and unacknowledged transaction decisions are drained. An expired lease is not permission to copy an unresolved intent. The replicated fence checks this condition again, closing the gap between the worker's scan and fence apply. Secondary commit/abort paths validate range ownership. The conservative drain currently covers the whole shard and may defer splits during long-running transactions or recovery. + +Directory creation rejects root/self/invalid IDs and existing authoritative directory identities. Mutations validate sizes and directory capacity before changing keys. Prepared transactions reserve positive directory capacity. Split cleanup reconstructs retained directory counters during apply, and metadata-only directory updates preserve pending and flushed child accounting. + +Quota batches validate and stage their complete effects before mutation. Internal balances remain signed so independent origin streams commute: a delete delivered before the corresponding create can temporarily make a balance negative and then converge to zero. Overflow rejects the entire batch. Callers must not interpret a temporarily negative asynchronous balance as a final quota decision. + +## IDs, names, and upgrade behavior + +The allocator holds an OS directory lock, reserves fresh local blocks after restart, and never silently recreates counters when established state is missing. File inode sequences respect the bit-63 boundary, including the final local block. + +`ID_ALLOCATOR_STATE` now uses a version-2 envelope containing request kind/count fingerprints. The envelope contains an invalid legacy Option tag, so old binaries fail to decode it instead of reinterpreting shifted counters. Legacy state can be read without reissuing reserved IDs. A cached legacy UUID has no verifiable request identity and is rejected with an explicit error; use a new UUID to request a fresh block. Matching version-2 UUID retries remain stable across restart, while different kind/count reuse is rejected. Deduplication remains bounded to the most recent 4,096 nonzero UUID grants. UUID zero explicitly disables deduplication for local allocation calls. + +Cell directories persist `CELL_OWNER` independently of the cluster-wide registered-node map. A legacy singleton directory can establish ownership automatically. A legacy replicated directory without ownership evidence fails closed and requires an offline migration using verified original node/cell identities. Do not change a node ID to adopt another node's directory. Missing established Cell state requires repair rather than bootstrap. + +Singleton listener changes are committed as guarded Raft node metadata updates before refreshed routes and epochs are persisted. Replicated nodes must restart at their persisted endpoint; online replicated endpoint migration remains a separate operation. An unsupported endpoint change fails explicitly instead of advertising a stale address. + +User names are limited to 65,525 UTF-8 bytes so a complete MetaKey plus WriteIntent prefix fits an unshared SST restart key. Overlong names are rejected before successful mutation. Historical invalid records may require offline repair; operations do not silently delete or reinterpret them. + +## SQL, CLI, benchmarks, and metrics + +Malformed or unset SQL bindings are rejected without changing parameter positions or executing a mutation. SQL mode conversions reject values outside u32. Cursor and prefix bounds are combined before scanning. + +`metadb-ctl sql` writes the complete response as JSON, including quota/index lists and pagination metadata. Application and transport errors use nonzero exits and JSON stderr. These output formats replace the former human-readable/partial success output and should be accounted for in scripts. + +Benchmarks allocate a unique directory, inode IDs, and FIDs for every run. JSONL output announces `bench_pid` and reports exact requested attempts, application and transport failures, real successful operations, and latency statistics. Zero operations are supported; zero concurrency is rejected. Repeating a benchmark no longer measures duplicate-name failures as successful creates. + +Metrics uses HTTP/1 with at most 64 concurrent connections, at most 64 headers, an 8 KiB header buffer, and a two-second connection deadline. Idle clients cannot block unrelated scrapes, and listener/connection tasks are stopped before cluster shutdown. + +## Validation and remaining implementation scope + +Run the same checks used by CI: + +```bash +cargo fmt --all -- --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace --no-fail-fast +``` + +The added tests cover both original failures and rejection boundaries, including complete-prefix recovery, malformed snapshot matrices, data corruption, ID migration/downgrade safety, transaction/split interleavings, signed quota delivery, input atomicity, address restart behavior, and actual CLI/daemon behavior. This is not a physical power-cut experiment or a proof that every distributed history is correct. + +Only `all-in-one` is an implemented daemon role. Remote Cell registration, autonomous multi-process role bootstrap, replicated GM/CM state, and physical jumbo-node migration remain unimplemented. `memtable_size_bytes` remains a reserved compatibility setting rather than an active flush threshold. Directory scans and multi-shard statistics still have scalability work beyond the corrected cursor semantics. These limitations are not changed into production-scale guarantees by passing regression tests. diff --git a/docs/guides/en/deployment.md b/docs/guides/en/deployment.md index 1ab2c79..b7a7bbc 100644 --- a/docs/guides/en/deployment.md +++ b/docs/guides/en/deployment.md @@ -2,9 +2,9 @@ * **Back to Guide Index**: [English Overview](./README.md) | [Hub](../../README.md) -MetaDB runs the **exact same WAL-less LSM storage engine and OpenRaft state machine** across all three operational stages, allowing deployments to start on a single machine and expand online to a multi-Cell cluster without offline migration. +MetaDB uses a WAL-less LSM storage engine and OpenRaft state machine for persistent all-in-one deployments. Multi-node Raft is exercised by integration tests; the disaggregated and multi-Cell daemon topology remains a design target. -> **Implementation Status**: Persistent WAL-less SSTable LSM, `openraft` Multi-Raft (`1-Voter -> 3-Voter`), Epoch-fenced `PID` & `DualModeFineGrained` shard splitting, Jumbo isolation, `metadbd`, and `metadb-ctl` are implemented and verified across the 8-crate workspace. +> **Implementation Status**: Persistent LSM, Raft replication, local shard splitting, and all-in-one gRPC/CLI operations have regression coverage. Only `all-in-one` starts a daemon. Other roles and remote Cell registration are rejected. See [correctness and compatibility notes](../../designs/correctness_hardening.md) before upgrading. --- @@ -26,7 +26,7 @@ metrics_listen_addr = "127.0.0.1:9100" max_clock_drift_ms = 500 [storage] -memtable_size_bytes = 33554432 # 32 MiB +memtable_size_bytes = 33554432 # Reserved; not an active flush threshold block_cache_bytes = 536870912 # 512 MiB (16-shard LRU) mvcc_retention_ms = 5000 # 5s In-Memory MVCC window bloom_bits_per_key = 10 @@ -50,7 +50,7 @@ metadbd --config /etc/metadb/standalone.toml `MetadataService`, `TransactionService`, `ControlPlaneService`, and `RaftTransportService` do **not** terminate TLS. With an empty `[security].shared_secret` (the default) they also perform no application authentication. The default `grpc_listen_addr` is `127.0.0.1:7800`. Do not publish that socket on an untrusted network. -Set a shared secret to require `authorization: Bearer ` on metadata, transaction, and control-plane RPCs (Raft transport on the same port stays unauthenticated). `metadb-ctl --auth-token` (or `METADB_AUTH_TOKEN`) and `metadbd --cell-manager` send that header: +Set a shared secret to require `authorization: Bearer ` on metadata, transaction, control-plane, and Raft transport RPCs. Peer Raft clients propagate the same token. `metadb-ctl --auth-token` (or `METADB_AUTH_TOKEN`) and `metadbd --cell-manager` send that header: ```toml [security] @@ -67,7 +67,7 @@ As metadata grows past `shard_split_threshold_bytes`, `CellLocalManager` and `Po --- -## Stage 3: Online Scale-Out to `3-Voter` Multi-Raft & Multi-Cell Clusters `[Implemented]` +## Stage 3: Multi-Node Raft Tests and Intended Multi-Cell Topology ### Step 3.1: Zero-Downtime Upgrade from `1-Voter` to `3-Voter` HA @@ -89,7 +89,7 @@ As metadata grows past `shard_split_threshold_bytes`, `CellLocalManager` and `Po ### Step 3.2: Disaggregated Multi-Cell Topology (`100B – 10T+` Records) -For clusters exceeding `~50,000` shards (`~200` BE nodes), deploy disaggregated roles across independent **Cells**: +The following table describes an intended topology, not runnable role-trimmed daemon deployments: | Role | `--role` Flag | Recommended Deployment | Responsibility | | :--- | :--- | :--- | :--- | diff --git a/docs/guides/en/operations-and-tuning.md b/docs/guides/en/operations-and-tuning.md index 617773c..a86a2b3 100644 --- a/docs/guides/en/operations-and-tuning.md +++ b/docs/guides/en/operations-and-tuning.md @@ -6,7 +6,7 @@ ## 1. `metadb-ctl` Administration CLI `[Implemented]` -`metadb-ctl` connects to `ControlPlaneService` and `MetadataService` over gRPC to inspect cluster topology, manage shards, execute SQL templates, and run synthetic benchmarks: +`metadb-ctl` connects to gRPC services to inspect local cluster state, manage shards, execute SQL templates, and run synthetic benchmarks. SQL results are complete JSON responses; command failures exit nonzero. Benchmark JSONL reports an allocated `bench_pid`, exact attempts, successes, failures, and latency statistics. See [upgrade and compatibility notes](../../designs/correctness_hardening.md). ```bash # Inspect Cells, BE node health, shard counts, and 3-watermark Raft log lag @@ -56,10 +56,10 @@ cargo test -p metadb-server --test topology_distributed three_voter_leader_loss_ 1. **Automatic Dedicated-Node Isolation (`JumboIsolated`)**: * When a single directory (`PID`) exceeds `shard_split_threshold_bytes` (`4 GiB` default) and represents `>80%` of shard bytes, `PolicyShardSplitter` raises `UnsplittableJumboShard`. - * `CellLocalManager` marks the shard `JumboIsolated` and migrates peer shard leaders/replicas off that BE host so the jumbo directory gets uncontended NVMe and CPU resources. + * `CellLocalManager` records `JumboIsolated` scheduling metadata. Physical migration to another daemon is not implemented. 2. **Dual-Mode `(PID, name)` Splitting (`DualModeFineGrained`)**: * Only if a single directory grows beyond a dedicated host's capacity (e.g., `>500M` files / `>100 GiB`), enable `DualModeFineGrained` via `metadb-ctl shard enable-dual-mode`. - * Non-anchor sub-shards maintain shard-local `[0x04] DirShardDelta` counters: `create_file` and `unlink` remain `1-RTT` single-shard operations, while `dirstat` switches automatically to `1-hop` parallel scatter-gather. + * Non-anchor sub-shards maintain shard-local `[0x04] DirShardDelta` counters: `create_file` and `unlink` remain `1-RTT` single-shard operations, while `dirstat` switches automatically to shared-barrier scatter-gather (currently issued serially). --- @@ -68,8 +68,8 @@ cargo test -p metadb-server --test topology_distributed three_voter_leader_loss_ | Parameter | Default | Tuning Guidance | | :--- | :--- | :--- | | `mvcc_retention_ms` | `5000` (`5s`) | Controls how long in-memory MVCC versions are retained for active `SnapshotGuard`s and write coalescing. Keep at `3000–5000ms`; never run multi-second scans in a single snapshot—use cursor-paginated `ListDir` (SQL Template #10). | -| `memtable_size_bytes` | `32 MiB` | Increase to `64 MiB` on write-heavy AI checkpoint clusters with `>= 64 GiB` RAM per BE node to maximize in-memory MVCC version pruning before L0 flush. | -| `block_cache_bytes` | `512 MiB` | Set to `25%–40%` of host RAM on dedicated BE nodes so SSTable `IndexBlock`, `FilterBlock`, and hot directory `DataBlock`s stay memory-resident across the 16 LRU cache shards. | +| `memtable_size_bytes` | `32 MiB` | Reserved compatibility setting; changing it does not currently control sealing or backpressure. | +| `block_cache_bytes` | `64 MiB` | Set to `25%–40%` of host RAM on dedicated BE nodes so SSTable `IndexBlock`, `FilterBlock`, and hot directory `DataBlock`s stay memory-resident across the 16 LRU cache shards. | | `locality_tombstone_threshold` | `128` | Number of physical LSM tombstones in a single 9-byte `[table_prefix][PID]` bucket (combined with `locality_tombstone_ratio >= 0.50`) that triggers targeted bottommost range compaction. Lower to `64` for latency-sensitive `listdir` workloads under heavy TTL deletion. | --- @@ -87,4 +87,4 @@ cargo test -p metadb-server --test topology_distributed three_voter_leader_loss_ * *Fix*: Handled automatically by `metadb-proxy` and the Rust SDK via `MetaCache` refresh + retry. Persistent occurrences indicate an unreachable `CellLocalManager`. * **Growing `.raftlog` Disk Usage**: * *Cause*: `safe_purge_index = min(state_flushed_index, aux_outbox_durable_index, last_snapshot_index)` is pinned by stalled L0 disk I/O or a blocked snapshot transfer. (Note: unreachable remote `QuotaTable` shards do **not** pin `.raftlog` indefinitely because overdue `AuxOutbox` tasks spill to local `[0x07]` SSTables on L0 flush.) - * *Fix*: Inspect NVMe write latency and `metadb_raft_watermark_lag_entries` in Prometheus metrics (`:9100/metrics`). + * *Fix*: Inspect NVMe write latency and the exposed `metadb_min_raft_safe_purge_index` metric (`:9100/metrics`). Compare per-shard logs during diagnosis; no per-shard watermark-lag metric is currently exported. From 22d5c32be6117bbded8358f33c354dbb532d8668 Mon Sep 17 00:00:00 2001 From: loverustfs Date: Mon, 16 Nov 2026 20:07:44 +0800 Subject: [PATCH 2/5] Fix deferred split recovery during startup Preserve pending pre-fence plans while transaction recovery drains their parent shards. Complete matching committed fences despite new retained-range transactions, reject malformed transaction state, and cover both restart paths with regression tests. --- crates/metadb-cluster/src/cell_manager.rs | 24 +- crates/metadb-cluster/src/split_worker.rs | 67 ++-- .../tests/round2_split_transaction_tests.rs | 285 ++++++++++------- .../tests/split_post_fence_recovery_tests.rs | 291 ++++++++++++++++++ crates/metadb-core/src/error.rs | 4 + crates/metadb-tx/src/state_machine_impl.rs | 4 +- .../tests/round2_atomic_validation_tests.rs | 23 +- docs/designs/correctness_hardening.md | 2 + 8 files changed, 562 insertions(+), 138 deletions(-) create mode 100644 crates/metadb-cluster/tests/split_post_fence_recovery_tests.rs diff --git a/crates/metadb-cluster/src/cell_manager.rs b/crates/metadb-cluster/src/cell_manager.rs index 3042738..58c307d 100644 --- a/crates/metadb-cluster/src/cell_manager.rs +++ b/crates/metadb-cluster/src/cell_manager.rs @@ -797,6 +797,18 @@ impl DurableCellManager { let _guard = self.split_lock.lock().await; // Finish or drop any plan left by a crash before discovering a new boundary. self.recover_incomplete_splits().await?; + // Recovery can defer a pre-fence plan until transaction maintenance has + // drained the original shard. Keep its durable boundary and child ID. + if self + .state + .lock() + .unwrap() + .split_plans + .values() + .any(|plan| plan.parent_shard_id == shard_id && plan.state == SplitPlanState::Planned) + { + return Ok(None); + } let route = { let st = self.state.lock().unwrap(); st.routes.get(&shard_id).cloned().ok_or_else(|| { @@ -1099,7 +1111,17 @@ impl DurableCellManager { }; let mut resumed_children: HashSet = HashSet::new(); for (uuid, plan) in plans { - self.resume_planned_split(uuid, &plan).await?; + match self.resume_planned_split(uuid, &plan).await { + Ok(()) => {} + Err(MetaDbError::WriteIntentConflict { .. } | MetaDbError::SplitDeferred(_)) => { + // Transaction recovery starts after Cell opening. A plan that + // has not fenced its parent must leave its original route + // available so that recovery can drain it, then retry on the + // next maintenance pass. Storage/codec/consensus errors still + // abort opening instead of being mistaken for deferrals. + } + Err(error) => return Err(error), + } resumed_children.insert(plan.child_shard_id); } diff --git a/crates/metadb-cluster/src/split_worker.rs b/crates/metadb-cluster/src/split_worker.rs index 08145ed..f03b25c 100644 --- a/crates/metadb-cluster/src/split_worker.rs +++ b/crates/metadb-cluster/src/split_worker.rs @@ -8,7 +8,8 @@ use std::sync::Arc; use metadb_core::{ affinity_key_for_raw_entry, saturating_add_i64, CompositeKeyRange, DirShardDeltaKey, DirShardDeltaRecord, EntryType, HlcTimestamp, InodeAttr, InodeId, MetaDbError, MetaKey, Result, - ShardEpoch, ShardId, WriteIntentKey, WriteIntentRecord, ATTR_ENTRY_NAME, TABLE_PREFIX_META, + ShardEpoch, ShardId, TxnRecordKey, TxnStatusRecord, WriteIntentKey, WriteIntentRecord, + ATTR_ENTRY_NAME, TABLE_PREFIX_META, }; use metadb_raft::{ RaftNodeOptions, RegistryShardState, ShardGroupHost, ShardRaftGroup, ShardWatermarkTracker, @@ -180,40 +181,66 @@ impl RaftShardSplitWorker { )> { let old_end_key = parent_route.range.end_key.clone(); let now_hlc = host.clock().now(); + let left_range = + CompositeKeyRange::new(parent_route.range.start_key.clone(), split_key.clone())?; + let right_range = CompositeKeyRange::new(split_key.clone(), old_end_key.clone())?; // Never publish an epoch older than the one already applied. A stale plan must // not rewind `version` and then re-fence a range that a newer split already owns. + let current_version = parent_group.current_epoch().version; + if current_version > target_version { + return Err(MetaDbError::ShardRoutingError( + "split plan is older than the current parent epoch".into(), + )); + } + let already_fenced = current_version == target_version; + if already_fenced && parent_handler.key_range() != left_range { + return Err(MetaDbError::ShardRoutingError( + "split plan does not match the committed parent fence".into(), + )); + } - // Stage A: Scan `concat([0x05], split_key) .. concat([0x05], old_end_key)` (`Resolution B14`) - // and verify no active `deleting_fence` (`Resolution B17`) before splitting! - { + // Stage A: Drain all shard-local intents and decisions (`Resolution B14`) + // and verify no active `deleting_fence` (`Resolution B17`) before fencing. + // Once the atomic fence has committed, any new transaction can touch + // only the retained range. It must not prevent finishing publication of + // the child after a crash, and its state remains on the original parent. + if !already_fenced { let (_, snap) = parent_group .linearizable_read(parent_group.current_epoch()) .await?; let eng = parent_group.engine(); // Expiration only permits recovery; it never proves that a pending // intent is aborted. Drain every transaction before copying a shard. - if let Some((_, iv)) = eng - .scan_with_snapshot(&[0x05], &[0x06], &snap)? - .into_iter() - .next() - { - let irec = WriteIntentRecord::decode(&iv)?; + let mut first_intent = None; + for (key, value) in eng.scan_with_snapshot(&[0x05], &[0x06], &snap)? { + WriteIntentKey::decode(&key)?; + let record = WriteIntentRecord::decode(&value)?; + first_intent.get_or_insert(record); + } + let decisions = eng.scan_with_snapshot(&[0x06], &[0x07], &snap)?; + // Only valid transaction state is retryable. Corrupt records must + // remain fatal even when another valid intent would defer the split. + for (key, value) in &decisions { + TxnRecordKey::decode(key)?; + TxnStatusRecord::decode(value)?; + } + if let Some(irec) = first_intent { return Err(MetaDbError::WriteIntentConflict { conflicting_txn_id: irec.txn_id, primary_key: irec.primary_key, expire_hlc: irec.expire_hlc.0, }); } - if !eng.scan_with_snapshot(&[0x06], &[0x07], &snap)?.is_empty() { - return Err(MetaDbError::TransactionAborted( - "split deferred until transaction decisions are acknowledged".into(), + if !decisions.is_empty() { + return Err(MetaDbError::SplitDeferred( + "transaction decisions must be acknowledged before fencing".into(), )); } if let Some(dpid) = dual_mode_pid { let dk = DirShardDeltaKey { pid: dpid }.encode(); if let Some(raw_sub) = eng.get_with_snapshot(&dk, &snap)? { if DirShardDeltaRecord::decode(&raw_sub)?.deleting_fence { - return Err(MetaDbError::TransactionAborted( + return Err(MetaDbError::SplitDeferred( "cannot split DualMode directory while Rmdir deleting_fence is active" .to_string(), )); @@ -221,7 +248,7 @@ impl RaftShardSplitWorker { } let anchor_ik = WriteIntentKey::new(MetaKey::dir_attr(dpid).encode()).encode(); if eng.get_with_snapshot(&anchor_ik, &snap)?.is_some() { - return Err(MetaDbError::TransactionAborted( + return Err(MetaDbError::SplitDeferred( "cannot split DualMode directory while intent is held on _attr_" .to_string(), )); @@ -229,9 +256,6 @@ impl RaftShardSplitWorker { } } - let left_range = - CompositeKeyRange::new(parent_route.range.start_key.clone(), split_key.clone())?; - let right_range = CompositeKeyRange::new(split_key.clone(), old_end_key.clone())?; let new_parent_epoch = ShardEpoch { conf_ver: parent_group.current_epoch().conf_ver, version: target_version.max(parent_group.current_epoch().version), @@ -239,7 +263,7 @@ impl RaftShardSplitWorker { // Atomically fence `S_parent` to `left_range` + `new_parent_epoch` BEFORE flushing & scanning // so zero concurrent writes or `AttrDelta`s can land in `right_range` on `S_parent` during split! - if parent_group.current_epoch().version < target_version { + if !already_fenced { let fence_cmd = TxRaftCmd::FenceShardRangeForSplit { new_range: left_range.clone(), new_epoch: new_parent_epoch, @@ -294,6 +318,11 @@ impl RaftShardSplitWorker { for (raw_k, raw_v) in &all_raw { if let Some(aff_key) = affinity_key_for_raw_entry(raw_k, raw_v)? { if aff_key >= split_key { + if matches!(raw_k.first(), Some(0x05 | 0x06)) { + return Err(MetaDbError::ShardRoutingError( + "transaction state exists outside the committed parent fence".into(), + )); + } // Reset `applied_lsn` / `applied_raft_index` to `0` on migrated directory attribute // records because pre-split deltas were just flushed on `S_parent` and `S_child`'s // brand-new OpenRaft log starts at index `1`! diff --git a/crates/metadb-cluster/tests/round2_split_transaction_tests.rs b/crates/metadb-cluster/tests/round2_split_transaction_tests.rs index bdc19b2..6b36da2 100644 --- a/crates/metadb-cluster/tests/round2_split_transaction_tests.rs +++ b/crates/metadb-cluster/tests/round2_split_transaction_tests.rs @@ -10,10 +10,167 @@ use std::sync::Arc; use std::time::Duration; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn committed_secondary_must_survive_split_copy_and_recovery_interleaving() { +async fn committed_secondary_and_deferred_split_plans_survive_restart() { let dir = tempfile::tempdir().unwrap(); let clock = Arc::new(HybridLogicalClock::new()); let allocator = Arc::new(DurableIdAllocator::open(dir.path().join("allocator")).unwrap()); + let (secondary_id, child_id, primary_child_id, primary_route, secondary_route) = { + let cache = Arc::new(CompositeMetaCache::new()); + let host = Arc::new( + ShardGroupHost::open( + 1, + dir.path().join("be"), + Arc::new(ShardedBlockCache::new(1 << 20)), + clock.clone(), + ) + .unwrap(), + ); + let cell = DurableCellManager::open_or_bootstrap( + 1, + dir.path().join("cell"), + host.clone(), + "127.0.0.1:0".into(), + true, + usize::MAX, + LsmEngineConfig::default(), + RaftNodeOptions::default(), + allocator.clone(), + cache.clone(), + ) + .await + .unwrap(); + let directory = Arc::new(DurableGlobalManager::new( + clock.clone(), + allocator.clone(), + cache, + )); + directory.register_cell(CompositeKeyRange::all_user_tables(), cell.clone()); + let tx = DistributedTxRouter::new(1, clock.clone(), directory.clone()).with_ttl_ms(500); + tx.init_dir_attr(1, 1, 7, 0o755).await.unwrap(); + tx.mkdir(1, "source_dir", 100, 7, 0o755).await.unwrap(); + tx.mkdir(1, "target_dir", 900, 7, 0o755).await.unwrap(); + + let route = cell + .list_routes() + .into_iter() + .find(|r| r.shard_id == 1) + .unwrap(); + let (_, group, handler) = cell.lookup_local_handle(1).unwrap(); + let boundary = MetaKey::encode_pid_prefix_start(500); + let secondary_id = cell + .alloc_split_plan(111, 1, boundary.clone(), 2, None) + .unwrap(); + let (left, right, _, right_handler) = RaftShardSplitWorker::execute_local_split( + &host, + &route, + &group, + &handler, + secondary_id, + boundary, + None, + LsmEngineConfig::default(), + RaftNodeOptions::default(), + 2, + None, + ) + .await + .unwrap(); + cell.activate_split(111, left, right, Some(right_handler)) + .unwrap(); + tx.create_file(100, "src", (1 << 63) + 77, 77, 7, 0o644, 100) + .await + .unwrap(); + let interrupted = tx + .rename_with_fault_injection( + 100, + "src", + 900, + "dst", + false, + TwoPcCrashStage::StopAfterPrimaryCommitBeforeSecondary, + ) + .await; + assert!(interrupted.is_err()); + tokio::time::sleep(Duration::from_millis(600)).await; + + let route = cell + .list_routes() + .into_iter() + .find(|r| r.shard_id == secondary_id) + .unwrap(); + let (_, secondary, secondary_handler) = cell.lookup_local_handle(secondary_id).unwrap(); + let boundary = MetaKey::encode_pid_prefix_start(800); + let child_id = cell + .alloc_split_plan(222, secondary_id, boundary.clone(), 3, None) + .unwrap(); + let paused = RaftShardSplitWorker::execute_local_split( + &host, + &route, + &secondary, + &secondary_handler, + child_id, + boundary.clone(), + None, + LsmEngineConfig::default(), + RaftNodeOptions::default(), + 3, + Some(SplitCrashPoint::AfterChildDirRename), + ) + .await; + assert!( + matches!(paused, Err(MetaDbError::WriteIntentConflict { .. })), + "expired committed intents must defer the split before copying" + ); + assert_eq!( + secondary_handler.key_range(), + route.range, + "rejected split must leave the original range writable for recovery" + ); + assert!( + !host.shard_dir(child_id).exists(), + "rejected split must not publish a stale child image" + ); + + // The primary has committed, so it holds an unacknowledged decision while + // the secondary still has its intents. Persist a plan on each shard and + // restart without flushing either transaction's state to the LSM. + let primary_route = cell + .list_routes() + .into_iter() + .find(|r| r.shard_id == 1) + .unwrap(); + let primary_boundary = MetaKey::encode_pid_prefix_start(50); + let primary_child_id = cell + .alloc_split_plan(333, 1, primary_boundary.clone(), 3, None) + .unwrap(); + let pending_decision = RaftShardSplitWorker::execute_local_split( + &host, + &primary_route, + &group, + &handler, + primary_child_id, + primary_boundary, + None, + LsmEngineConfig::default(), + RaftNodeOptions::default(), + 3, + None, + ) + .await; + assert!(matches!( + pending_decision, + Err(MetaDbError::SplitDeferred(_)) + )); + host.shutdown_all_no_flush().await.unwrap(); + ( + secondary_id, + child_id, + primary_child_id, + primary_route, + route, + ) + }; + let cache = Arc::new(CompositeMetaCache::new()); let host = Arc::new( ShardGroupHost::open( @@ -37,97 +194,28 @@ async fn committed_secondary_must_survive_split_copy_and_recovery_interleaving() cache.clone(), ) .await - .unwrap(); + .expect("pending pre-fence splits must not block transaction recovery startup"); let directory = Arc::new(DurableGlobalManager::new(clock.clone(), allocator, cache)); directory.register_cell(CompositeKeyRange::all_user_tables(), cell.clone()); - let tx = DistributedTxRouter::new(1, clock.clone(), directory.clone()).with_ttl_ms(500); - tx.init_dir_attr(1, 1, 7, 0o755).await.unwrap(); - tx.mkdir(1, "source_dir", 100, 7, 0o755).await.unwrap(); - tx.mkdir(1, "target_dir", 900, 7, 0o755).await.unwrap(); - - let route = cell - .list_routes() - .into_iter() - .find(|r| r.shard_id == 1) - .unwrap(); + let tx = DistributedTxRouter::new(1, clock.clone(), directory.clone()); let (_, group, handler) = cell.lookup_local_handle(1).unwrap(); - let boundary = MetaKey::encode_pid_prefix_start(500); - let secondary_id = cell - .alloc_split_plan(111, 1, boundary.clone(), 2, None) - .unwrap(); - let (left, right, _, right_handler) = RaftShardSplitWorker::execute_local_split( - &host, - &route, - &group, - &handler, - secondary_id, - boundary, - None, - LsmEngineConfig::default(), - RaftNodeOptions::default(), - 2, - None, - ) - .await - .unwrap(); - cell.activate_split(111, left, right, Some(right_handler)) - .unwrap(); - tx.create_file(100, "src", (1 << 63) + 77, 77, 7, 0o644, 100) - .await - .unwrap(); - let interrupted = tx - .rename_with_fault_injection( - 100, - "src", - 900, - "dst", - false, - TwoPcCrashStage::StopAfterPrimaryCommitBeforeSecondary, - ) - .await; - assert!(interrupted.is_err()); - tokio::time::sleep(Duration::from_millis(600)).await; - - let route = cell - .list_routes() - .into_iter() - .find(|r| r.shard_id == secondary_id) - .unwrap(); let (_, secondary, secondary_handler) = cell.lookup_local_handle(secondary_id).unwrap(); - let boundary = MetaKey::encode_pid_prefix_start(800); - let child_id = cell - .alloc_split_plan(222, secondary_id, boundary.clone(), 3, None) - .unwrap(); - let paused = RaftShardSplitWorker::execute_local_split( - &host, - &route, - &secondary, - &secondary_handler, - child_id, - boundary.clone(), - None, - LsmEngineConfig::default(), - RaftNodeOptions::default(), - 3, - Some(SplitCrashPoint::AfterChildDirRename), - ) - .await; - assert!( - matches!(paused, Err(MetaDbError::WriteIntentConflict { .. })), - "expired committed intents must defer the split before copying" - ); - assert_eq!( - secondary_handler.key_range(), - route.range, - "rejected split must leave the original range writable for recovery" - ); - assert!( - !host.shard_dir(child_id).exists(), - "rejected split must not publish a stale child image" - ); + assert_eq!(handler.key_range(), primary_route.range); + assert_eq!(secondary_handler.key_range(), secondary_route.range); + assert!(cell.lookup_local_handle(primary_child_id).is_err()); + assert!(cell.lookup_local_handle(child_id).is_err()); + assert!(cell.durable_split_plan_on_disk(222).unwrap()); + assert!(cell.durable_split_plan_on_disk(333).unwrap()); + assert!(cell + .maybe_split_or_isolate_shard(secondary_id, false) + .await + .unwrap() + .is_none()); + assert_eq!(secondary_handler.key_range(), secondary_route.range); + assert!(cell.lookup_local_handle(child_id).is_err()); - // Recover the committed transaction against the original route, then retry - // the deferred split. No stale intent or decision may enter the child image. + // Recover the committed transaction against the original routes, then retry + // both deferred splits. Neither stale intent nor decision may enter a child. TxnRecoveryWorker::run_recovery_pass_on_shard(&group, &handler, directory.as_ref(), &clock) .await .unwrap(); @@ -158,23 +246,12 @@ async fn committed_secondary_must_survive_split_copy_and_recovery_interleaving() "recovery failed to materialize on old parent" ); - let (left, right, _, right_handler) = RaftShardSplitWorker::execute_local_split( - &host, - &route, - &secondary, - &secondary_handler, - child_id, - boundary, - None, - LsmEngineConfig::default(), - RaftNodeOptions::default(), - 3, - None, - ) - .await - .unwrap(); - cell.activate_split(222, left, right, Some(right_handler)) - .unwrap(); + assert!(cell + .maybe_split_or_isolate_shard(secondary_id, false) + .await + .unwrap() + .is_none()); + assert!(cell.lookup_local_handle(primary_child_id).is_ok()); let (_, child_group, child_handler) = cell.lookup_local_handle(child_id).unwrap(); TxnRecoveryWorker::run_recovery_pass_on_shard( &child_group, diff --git a/crates/metadb-cluster/tests/split_post_fence_recovery_tests.rs b/crates/metadb-cluster/tests/split_post_fence_recovery_tests.rs new file mode 100644 index 0000000..7ea15bc --- /dev/null +++ b/crates/metadb-cluster/tests/split_post_fence_recovery_tests.rs @@ -0,0 +1,291 @@ +use std::path::Path; +use std::sync::Arc; + +use metadb_cluster::{ + CompositeMetaCache, DurableCellManager, DurableGlobalManager, DurableIdAllocator, + RaftShardSplitWorker, SplitCrashPoint, +}; +use metadb_core::{ + AttrDeltaSpec, CompositeKeyRange, HlcTimestamp, HybridLogicalClock, IntentOp, MetaDbError, + MetaKey, TxnRecordKey, TxnState, TxnStatusRecord, WriteIntentKey, WriteIntentRecord, + ROOT_INODE_ID, +}; +use metadb_raft::{RaftNodeOptions, ShardGroupHost}; +use metadb_storage::{LsmEngineConfig, ShardedBlockCache}; +use metadb_tx::{DistributedTxRouter, TxRaftCmd}; + +struct Fixture { + host: Arc, + cell: Arc, + clock: Arc, + tx: DistributedTxRouter, +} + +async fn try_open(dir: &Path) -> metadb_core::Result { + let clock = Arc::new(HybridLogicalClock::new()); + let allocator = Arc::new(DurableIdAllocator::open(dir.join("alloc"))?); + let cache = Arc::new(CompositeMetaCache::new()); + let host = Arc::new(ShardGroupHost::open( + 1, + dir.join("be"), + Arc::new(ShardedBlockCache::new(8 * 1024 * 1024)), + clock.clone(), + )?); + let cell = DurableCellManager::open_or_bootstrap( + 1, + dir.join("cell"), + host.clone(), + "127.0.0.1:0".into(), + true, + usize::MAX, + LsmEngineConfig::default(), + RaftNodeOptions::default(), + allocator.clone(), + cache.clone(), + ) + .await?; + let directory = Arc::new(DurableGlobalManager::new(clock.clone(), allocator, cache)); + directory.register_cell(CompositeKeyRange::all_user_tables(), cell.clone()); + let tx = DistributedTxRouter::new(1, clock.clone(), directory); + Ok(Fixture { + host, + cell, + clock, + tx, + }) +} + +async fn open(dir: &Path) -> Fixture { + try_open(dir) + .await + .expect("startup must complete a fenced split despite a new retained-range intent") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn fenced_split_finishes_on_restart_with_new_retained_range_transaction() { + let dir = tempfile::tempdir().unwrap(); + let split_key = MetaKey::encode_pid_prefix_start(500); + let left_key = MetaKey::child(10, "left").encode(); + let right_key = MetaKey::child(900, "right").encode(); + let intent_key = WriteIntentKey::new(left_key.clone()).encode(); + let txn_id = 0xF3CE_u128; + let txn_key = TxnRecordKey { txn_id }.encode(); + + let child_id = { + let fixture = open(dir.path()).await; + fixture + .tx + .init_dir_attr(ROOT_INODE_ID, ROOT_INODE_ID, 7, 0o755) + .await + .unwrap(); + fixture + .tx + .mkdir(ROOT_INODE_ID, "left_dir", 10, 7, 0o755) + .await + .unwrap(); + fixture + .tx + .mkdir(ROOT_INODE_ID, "right_dir", 900, 7, 0o755) + .await + .unwrap(); + fixture + .tx + .create_file(10, "left", (1 << 63) + 10, 10, 7, 0o644, 17) + .await + .unwrap(); + fixture + .tx + .create_file(900, "right", (1 << 63) + 900, 900, 7, 0o644, 29) + .await + .unwrap(); + + let route = fixture + .cell + .list_routes() + .into_iter() + .find(|route| route.shard_id == 1) + .unwrap(); + let (_, parent, handler) = fixture.cell.lookup_local_handle(1).unwrap(); + let child_id = fixture + .cell + .alloc_split_plan(0xF3CE, 1, split_key.clone(), 2, None) + .unwrap(); + let paused = RaftShardSplitWorker::execute_local_split( + &fixture.host, + &route, + &parent, + &handler, + child_id, + split_key.clone(), + None, + LsmEngineConfig::default(), + RaftNodeOptions::default(), + 2, + Some(SplitCrashPoint::AfterFence), + ) + .await + .err() + .expect("the injected crash must interrupt the split"); + assert!(paused.to_string().contains("injected split crash")); + assert_eq!(handler.key_range().end_key, split_key); + assert!(!fixture.host.shard_dir(child_id).exists()); + + // A successful fence drained the old transaction state. The parent can + // subsequently accept an unrelated transaction wholly in its left range. + let start_hlc = fixture.clock.now(); + let expire_hlc = HlcTimestamp::new(start_hlc.physical_ms() + 60_000, 0); + let prepare = TxRaftCmd::Prepare2pc { + txn_id, + is_primary: true, + primary_key: left_key.clone(), + participant_keys: vec![left_key.clone()], + start_hlc, + expire_hlc, + rename_fence_token: None, + intents: vec![( + left_key.clone(), + IntentOp::Delete { + attr_deltas: vec![AttrDeltaSpec { + pid: 10, + delta_children: -1, + delta_bytes: -17, + }], + aux_specs: vec![], + }, + )], + }; + parent + .propose( + parent.current_epoch(), + fixture.clock.now(), + bincode::serialize(&prepare).unwrap(), + ) + .await + .unwrap(); + fixture.host.shutdown_all_no_flush().await.unwrap(); + child_id + }; + + let fixture = open(dir.path()).await; + let (_, parent, handler) = fixture.cell.lookup_local_handle(1).unwrap(); + let (_, child, child_handler) = fixture.cell.lookup_local_handle(child_id).unwrap(); + assert_eq!(handler.key_range().end_key, split_key); + assert!(child_handler.key_range().contains_key(&right_key)); + assert!(!child_handler.key_range().contains_key(&left_key)); + let (_, parent_snapshot) = parent + .linearizable_read(parent.current_epoch()) + .await + .unwrap(); + let raw_intent = parent + .engine() + .get_with_snapshot(&intent_key, &parent_snapshot) + .unwrap() + .expect("new retained-range intent must stay on the parent"); + assert_eq!( + WriteIntentRecord::decode(&raw_intent).unwrap().txn_id, + txn_id + ); + let raw_txn = parent + .engine() + .get_with_snapshot(&txn_key, &parent_snapshot) + .unwrap() + .expect("new retained-range primary record must stay on the parent"); + assert_eq!( + TxnStatusRecord::decode(&raw_txn).unwrap().state, + TxnState::Pending + ); + assert!(parent + .engine() + .get_with_snapshot(&right_key, &parent_snapshot) + .unwrap() + .is_none()); + drop(parent_snapshot); + + let (_, child_snapshot) = child + .linearizable_read(child.current_epoch()) + .await + .unwrap(); + for forbidden_key in [&intent_key[..], &txn_key[..], &left_key[..]] { + assert!(child + .engine() + .get_with_snapshot(forbidden_key, &child_snapshot) + .unwrap() + .is_none()); + } + assert!(child + .engine() + .get_with_snapshot(&right_key, &child_snapshot) + .unwrap() + .is_some()); + drop(child_snapshot); + + let abort = TxRaftCmd::Abort2pc { + txn_id, + target_user_keys: vec![left_key], + }; + parent + .propose( + parent.current_epoch(), + fixture.clock.now(), + bincode::serialize(&abort).unwrap(), + ) + .await + .unwrap(); + assert_eq!( + fixture + .tx + .lookup(10, "left") + .await + .unwrap() + .unwrap() + .attr + .fid, + Some(10) + ); + assert_eq!( + fixture + .tx + .lookup(900, "right") + .await + .unwrap() + .unwrap() + .attr + .fid, + Some(900) + ); + fixture.host.shutdown_all().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn malformed_transaction_record_in_pending_split_still_aborts_startup() { + let dir = tempfile::tempdir().unwrap(); + { + let fixture = open(dir.path()).await; + fixture + .cell + .alloc_split_plan(0xBAD, 1, MetaKey::encode_pid_prefix_start(500), 2, None) + .unwrap(); + let (_, parent, _) = fixture.cell.lookup_local_handle(1).unwrap(); + // Persist an invalid application record inside a valid SST/checkpoint. + // A nonempty transaction table alone must not classify it as retryable. + parent.engine().apply_committed_put( + TxnRecordKey { txn_id: 0xBAD }.encode().to_vec(), + vec![0], + fixture.clock.now(), + 0, + ); + parent.state_machine().flush_to_lsm().unwrap(); + fixture.host.shutdown_all_no_flush().await.unwrap(); + } + let error = try_open(dir.path()) + .await + .err() + .expect("malformed transaction state must fail closed instead of deferring startup"); + assert!( + matches!( + error, + MetaDbError::CodecError(_) | MetaDbError::ChecksumMismatch { .. } + ), + "unexpected startup error: {error}" + ); +} diff --git a/crates/metadb-core/src/error.rs b/crates/metadb-core/src/error.rs index a572b8b..c9e22cf 100644 --- a/crates/metadb-core/src/error.rs +++ b/crates/metadb-core/src/error.rs @@ -92,6 +92,9 @@ pub enum MetaDbError { /// /// Appended after `InvalidFileSize` so that variant stays bincode index 23. HlcSaturated, + /// A shard split must wait for transaction recovery before fencing its range. + /// Appended after `HlcSaturated` so earlier variant tags stay stable. + SplitDeferred(String), } impl fmt::Display for MetaDbError { @@ -202,6 +205,7 @@ impl fmt::Display for MetaDbError { f, "HLC saturated at the 48-bit physical ceiling; refusing to wrap the logical counter" ), + MetaDbError::SplitDeferred(msg) => write!(f, "shard split deferred: {}", msg), } } } diff --git a/crates/metadb-tx/src/state_machine_impl.rs b/crates/metadb-tx/src/state_machine_impl.rs index 7a7b1db..d62de52 100644 --- a/crates/metadb-tx/src/state_machine_impl.rs +++ b/crates/metadb-tx/src/state_machine_impl.rs @@ -1862,8 +1862,8 @@ impl TxStateMachineHandler { // This check shares Raft's serialized apply boundary with the // fence. Prepare cannot race between the worker's scan and fence. if !Self::scan_latest_in_apply(engine, &[0x05], &[0x07])?.is_empty() { - return Err(MetaDbError::TransactionAborted( - "split deferred until all intents and decisions are drained".into(), + return Err(MetaDbError::SplitDeferred( + "all intents and decisions must be drained before fencing".into(), )); } { diff --git a/crates/metadb-tx/tests/round2_atomic_validation_tests.rs b/crates/metadb-tx/tests/round2_atomic_validation_tests.rs index 0dbd329..49a346d 100644 --- a/crates/metadb-tx/tests/round2_atomic_validation_tests.rs +++ b/crates/metadb-tx/tests/round2_atomic_validation_tests.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use metadb_core::{ AttrDeltaSpec, AuxOutboxEntry, AuxTaskPayload, CompositeKeyRange, HlcTimestamp, InodeAttr, - IntentOp, MetaKey, QuotaKey, QuotaRecord, ShardEpoch, WriteIntentKey, + IntentOp, MetaDbError, MetaKey, QuotaKey, QuotaRecord, ShardEpoch, WriteIntentKey, }; use metadb_raft::{MetaDbRaftRequest, ShardApplyHandler, ShardWatermarkTracker}; use metadb_storage::{LifecycleStorageEngine, LsmEngineConfig, ShardedBlockCache}; @@ -404,18 +404,17 @@ fn fence_rechecks_intents_at_the_serialized_apply_boundary() { }; // Simulate the lease expiring between the worker's scan and Raft apply. let hlc = HlcTimestamp::new(6, 0); - assert!(handler - .apply_command( - 4, + let deferred = handler.apply_command( + 4, + hlc, + &MetaDbRaftRequest { + expected_epoch: handler.current_epoch(), hlc, - &MetaDbRaftRequest { - expected_epoch: handler.current_epoch(), - hlc, - payload: bincode::serialize(&fence).unwrap() - }, - &engine - ) - .is_err()); + payload: bincode::serialize(&fence).unwrap(), + }, + &engine, + ); + assert!(matches!(deferred, Err(MetaDbError::SplitDeferred(_)))); assert_eq!(handler.key_range(), before); assert!(read(&engine, &WriteIntentKey::new(key).encode()).is_some()); } diff --git a/docs/designs/correctness_hardening.md b/docs/designs/correctness_hardening.md index 3c4e672..d3ab97a 100644 --- a/docs/designs/correctness_hardening.md +++ b/docs/designs/correctness_hardening.md @@ -18,6 +18,8 @@ Checked prepares compare the exact committed values read by a coordinator before Splitting waits until local intents and unacknowledged transaction decisions are drained. An expired lease is not permission to copy an unresolved intent. The replicated fence checks this condition again, closing the gap between the worker's scan and fence apply. Secondary commit/abort paths validate range ownership. The conservative drain currently covers the whole shard and may defer splits during long-running transactions or recovery. +Recovery keeps a deferred split plan pending and the original parent route available, allowing startup and transaction maintenance to proceed. Maintenance retries the plan after draining transaction state. Once the parent fence is committed, recovery completes the existing split without waiting for new transactions in the retained range. Only explicitly retryable drain conditions are deferred; corruption and storage errors still stop recovery. + Directory creation rejects root/self/invalid IDs and existing authoritative directory identities. Mutations validate sizes and directory capacity before changing keys. Prepared transactions reserve positive directory capacity. Split cleanup reconstructs retained directory counters during apply, and metadata-only directory updates preserve pending and flushed child accounting. Quota batches validate and stage their complete effects before mutation. Internal balances remain signed so independent origin streams commute: a delete delivered before the corresponding create can temporarily make a balance negative and then converge to zero. Overflow rejects the entire batch. Callers must not interpret a temporarily negative asynchronous balance as a final quota decision. From ac4c4f49fffaa794c13d004a101ba823c5cdd947 Mon Sep 17 00:00:00 2001 From: loverustfs Date: Mon, 16 Nov 2026 20:49:08 +0800 Subject: [PATCH 3/5] Fix routing across split publication windows Reject stale or incomplete shard descriptors and wait for published routes with bounded retries. Validate read ownership and epochs to prevent false misses and incomplete directory results, start transaction leases after routing, and cover delayed publication with concurrent reads and writes. --- crates/metadb-cluster/src/global_manager.rs | 118 ++++++--- .../tests/split_publication_routing_tests.rs | 205 +++++++++++++++ crates/metadb-tx/src/two_pc.rs | 233 ++++++++++++++---- docs/designs/correctness_hardening.md | 2 + 4 files changed, 471 insertions(+), 87 deletions(-) create mode 100644 crates/metadb-cluster/tests/split_publication_routing_tests.rs diff --git a/crates/metadb-cluster/src/global_manager.rs b/crates/metadb-cluster/src/global_manager.rs index 58ed345..b9ac30e 100644 --- a/crates/metadb-cluster/src/global_manager.rs +++ b/crates/metadb-cluster/src/global_manager.rs @@ -75,6 +75,76 @@ impl DurableGlobalManager { } } } + + fn try_route_user_key(&self, user_key: &[u8]) -> Result { + let route = self.meta_cache.lookup_key(user_key).ok_or_else(|| { + MetaDbError::ShardRoutingError("no published route covering key".to_string()) + })?; + let cell = self.get_cell(route.cell_id).ok_or_else(|| { + MetaDbError::ShardRoutingError(format!("unknown cell {}", route.cell_id)) + })?; + let (_, group, handler) = cell.lookup_local_handle(route.shard_id)?; + let actual = group.current_epoch(); + let actual_range = handler.key_range(); + if actual != route.epoch + || actual_range != route.range + || !actual_range.contains_key(user_key) + { + // A parent is fenced before its child's route is published. Returning + // its current epoch here would let reads observe the deleted right + // half as absent instead of waiting for the child to become routable. + return Err(MetaDbError::EpochMismatch { + shard_id: route.shard_id, + expected: route.epoch, + actual, + }); + } + // Keep this descriptor's epoch: a later fence must invalidate a read + // barrier or proposal that races after this routing check. + Ok((route.epoch, group, handler)) + } + + fn try_route_user_range( + &self, + start_user_key: &[u8], + end_user_key: &[u8], + ) -> Result> { + if start_user_key >= end_user_key { + return Ok(Vec::new()); + } + let routes = self.meta_cache.lookup_range(start_user_key, end_user_key); + let mut covered_until = start_user_key.to_vec(); + let mut out = Vec::with_capacity(routes.len()); + for route in routes { + let cell = self.get_cell(route.cell_id).ok_or_else(|| { + MetaDbError::ShardRoutingError(format!("unknown cell {}", route.cell_id)) + })?; + let (_, group, handler) = cell.lookup_local_handle(route.shard_id)?; + let actual = group.current_epoch(); + if actual != route.epoch || handler.key_range() != route.range { + return Err(MetaDbError::EpochMismatch { + shard_id: route.shard_id, + expected: route.epoch, + actual, + }); + } + let overlap_start = route.range.start_key.as_slice().max(start_user_key); + let overlap_end = route.range.end_key.as_slice().min(end_user_key); + if overlap_start != covered_until.as_slice() || overlap_end <= overlap_start { + return Err(MetaDbError::ShardRoutingError( + "published routes do not provide contiguous range coverage".to_string(), + )); + } + covered_until = overlap_end.to_vec(); + out.push((route.epoch, group, handler)); + } + if covered_until.as_slice() != end_user_key { + return Err(MetaDbError::ShardRoutingError( + "published routes do not cover the complete requested range".to_string(), + )); + } + Ok(out) + } } #[tonic::async_trait] @@ -89,34 +159,13 @@ impl ShardDirectory for DurableGlobalManager { } fn route_user_key(&self, user_key: &[u8]) -> Result { - let route = match self.meta_cache.lookup_key(user_key) { - Some(r) => r, - None => { + match self.try_route_user_key(user_key) { + Ok(handle) => Ok(handle), + Err(_) => { self.refresh_meta_cache(); - self.meta_cache.lookup_key(user_key).ok_or_else(|| { - MetaDbError::ShardRoutingError("no route covering key".to_string()) - })? + self.try_route_user_key(user_key) } - }; - let cell = self.get_cell(route.cell_id).ok_or_else(|| { - MetaDbError::ShardRoutingError(format!("unknown cell {}", route.cell_id)) - })?; - let (_, grp, handler) = cell.lookup_local_handle(route.shard_id)?; - // Return the route's cached epoch so if a caller had a stale `MetaCache` entry, - // `grp.propose(route.epoch, ...)` fences it via `EpochMismatch` (`Resolution B16`) - // and self-heals on retry! - if !handler.key_range().contains_key(user_key) || grp.current_epoch() != route.epoch { - self.refresh_meta_cache(); - let refreshed = self.meta_cache.lookup_key(user_key).ok_or_else(|| { - MetaDbError::ShardRoutingError("no route after refresh".to_string()) - })?; - let r_cell = self.get_cell(refreshed.cell_id).ok_or_else(|| { - MetaDbError::ShardRoutingError(format!("unknown cell {}", refreshed.cell_id)) - })?; - let (ep, g2, h2) = r_cell.lookup_local_handle(refreshed.shard_id)?; - return Ok((ep, g2, h2)); } - Ok((route.epoch, grp, handler)) } fn route_user_range( @@ -124,20 +173,13 @@ impl ShardDirectory for DurableGlobalManager { start_user_key: &[u8], end_user_key: &[u8], ) -> Result> { - let mut routes = self.meta_cache.lookup_range(start_user_key, end_user_key); - if routes.is_empty() { - self.refresh_meta_cache(); - routes = self.meta_cache.lookup_range(start_user_key, end_user_key); - } - let mut out = Vec::with_capacity(routes.len()); - for r in routes { - let cell = self.get_cell(r.cell_id).ok_or_else(|| { - MetaDbError::ShardRoutingError(format!("unknown cell {}", r.cell_id)) - })?; - let (ep, grp, handler) = cell.lookup_local_handle(r.shard_id)?; - out.push((ep, grp, handler)); + match self.try_route_user_range(start_user_key, end_user_key) { + Ok(handles) => Ok(handles), + Err(_) => { + self.refresh_meta_cache(); + self.try_route_user_range(start_user_key, end_user_key) + } } - Ok(out) } async fn acquire_dir_rename_lease( diff --git a/crates/metadb-cluster/tests/split_publication_routing_tests.rs b/crates/metadb-cluster/tests/split_publication_routing_tests.rs new file mode 100644 index 0000000..2d72d53 --- /dev/null +++ b/crates/metadb-cluster/tests/split_publication_routing_tests.rs @@ -0,0 +1,205 @@ +use std::sync::Arc; +use std::time::Duration; + +use metadb_cluster::{ + CompositeMetaCache, DurableCellManager, DurableGlobalManager, DurableIdAllocator, + RaftShardSplitWorker, +}; +use metadb_core::{CompositeKeyRange, HybridLogicalClock, MetaKey, ROOT_INODE_ID}; +use metadb_raft::{RaftNodeOptions, ShardGroupHost}; +use metadb_storage::{LsmEngineConfig, ShardedBlockCache}; +use metadb_tx::{DistributedTxRouter, ShardDirectory}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn reads_and_writes_wait_for_split_route_publication_without_losing_rows() { + let dir = tempfile::tempdir().unwrap(); + let clock = Arc::new(HybridLogicalClock::new()); + let allocator = Arc::new(DurableIdAllocator::open(dir.path().join("allocator")).unwrap()); + let cache = Arc::new(CompositeMetaCache::new()); + let host = Arc::new( + ShardGroupHost::open( + 1, + dir.path().join("be"), + Arc::new(ShardedBlockCache::new(8 * 1024 * 1024)), + clock.clone(), + ) + .unwrap(), + ); + let cell = DurableCellManager::open_or_bootstrap( + 1, + dir.path().join("cell"), + host.clone(), + "127.0.0.1:0".into(), + true, + usize::MAX, + LsmEngineConfig::default(), + RaftNodeOptions::default(), + allocator.clone(), + cache.clone(), + ) + .await + .unwrap(); + let directory = Arc::new(DurableGlobalManager::new(clock.clone(), allocator, cache)); + directory.register_cell(CompositeKeyRange::all_user_tables(), cell.clone()); + let tx = Arc::new(DistributedTxRouter::new(1, clock, directory.clone())); + tx.init_dir_attr(ROOT_INODE_ID, ROOT_INODE_ID, 7, 0o755) + .await + .unwrap(); + tx.mkdir(ROOT_INODE_ID, "left_dir", 10, 7, 0o755) + .await + .unwrap(); + tx.mkdir(ROOT_INODE_ID, "right_dir", 900, 7, 0o755) + .await + .unwrap(); + tx.create_file(900, "existing", (1 << 63) + 901, 901, 7, 0o644, 20) + .await + .unwrap(); + tx.create_file(900, "second", (1 << 63) + 902, 902, 7, 0o644, 40) + .await + .unwrap(); + + let original_route = cell + .list_routes() + .into_iter() + .find(|route| route.shard_id == 1) + .unwrap(); + let (_, parent, handler) = cell.lookup_local_handle(1).unwrap(); + let boundary = MetaKey::encode_pid_prefix_start(500); + let request_uuid = 0x0F3C_E900; + let child_id = cell + .alloc_split_plan(request_uuid, 1, boundary.clone(), 2, None) + .unwrap(); + // Complete the data movement but deliberately hold route publication. This + // is the exact interval after parent deletion and before activate_split. + let (left, right, child, child_handler) = RaftShardSplitWorker::execute_local_split( + &host, + &original_route, + &parent, + &handler, + child_id, + boundary, + None, + LsmEngineConfig::default(), + RaftNodeOptions::default(), + 2, + None, + ) + .await + .unwrap(); + let existing_key = MetaKey::child(900, "existing").encode(); + let (_, parent_snapshot) = parent + .linearizable_read(parent.current_epoch()) + .await + .unwrap(); + let removed_from_parent = parent + .engine() + .get_with_snapshot(&existing_key, &parent_snapshot) + .unwrap() + .is_none(); + drop(parent_snapshot); + let (_, child_snapshot) = child + .linearizable_read(child.current_epoch()) + .await + .unwrap(); + let copied_to_child = child + .engine() + .get_with_snapshot(&existing_key, &child_snapshot) + .unwrap() + .is_some(); + drop(child_snapshot); + let stale_point_was_rejected = directory.route_user_key(&existing_key).is_err(); + let stale_range_was_rejected = directory + .route_user_range( + &MetaKey::encode_pid_prefix_start(900), + &MetaKey::encode_pid_prefix_end(900), + ) + .is_err(); + + let mut lookup = tokio::spawn({ + let tx = tx.clone(); + async move { tx.lookup(900, "existing").await } + }); + let mut missing_lookup = tokio::spawn({ + let tx = tx.clone(); + async move { tx.lookup(900, "missing").await } + }); + let mut listing = tokio::spawn({ + let tx = tx.clone(); + async move { tx.listdir_paginated(900, None, None, 100).await } + }); + let mut stat = tokio::spawn({ + let tx = tx.clone(); + async move { tx.dirstat(900).await } + }); + let mut create = tokio::spawn({ + let tx = tx.clone(); + async move { + tx.create_file(900, "created", (1 << 63) + 903, 903, 7, 0o644, 11) + .await + } + }); + // The former write retry budget was about one second, while child boot can + // leave this window open for several seconds. Reads must not return absent + // or partial results simply because the old parent no longer owns the keys. + tokio::time::sleep(Duration::from_millis(1500)).await; + let completed_before_publication = [ + lookup.is_finished(), + missing_lookup.is_finished(), + listing.is_finished(), + stat.is_finished(), + create.is_finished(), + ]; + let activation = cell.activate_split(request_uuid, left, right, Some(child_handler)); + let completed = tokio::time::timeout(Duration::from_secs(10), async { + tokio::join!( + &mut lookup, + &mut missing_lookup, + &mut listing, + &mut stat, + &mut create + ) + }) + .await; + // Preserve handles across the deadline so failed regressions still cancel + // every waiter and shut down the real Raft groups before reporting errors. + lookup.abort(); + missing_lookup.abort(); + listing.abort(); + stat.abort(); + create.abort(); + tokio::task::yield_now().await; + let shutdown = host.shutdown_all().await; + + shutdown.unwrap(); + activation.unwrap(); + assert!(removed_from_parent && copied_to_child); + assert!(stale_point_was_rejected, "stale point route was served"); + assert!( + stale_range_was_rejected, + "incomplete range route was served" + ); + assert_eq!( + completed_before_publication, [false; 5], + "operations completed against a fenced, unpublished route" + ); + let (lookup, missing_lookup, listing, stat, created) = + completed.expect("operations did not resume after route publication"); + assert_eq!(lookup.unwrap().unwrap().unwrap().attr.fid, Some(901)); + assert!(missing_lookup.unwrap().unwrap().is_none()); + let (entries, cursor) = listing.unwrap().unwrap(); + let names: Vec<_> = entries + .iter() + .map(|entry| entry.key.name.as_str()) + .collect(); + assert!(names.contains(&"existing") && names.contains(&"second")); + assert!(names + .iter() + .all(|name| ["created", "existing", "second"].contains(name))); + assert!(cursor.is_none()); + let stat = stat.unwrap().unwrap(); + assert!( + [(2, 60), (3, 71)].contains(&(stat.num_children, stat.size_bytes)), + "inconsistent directory snapshot: {stat:?}" + ); + assert_eq!(created.unwrap().unwrap().attr.fid, Some(903)); +} diff --git a/crates/metadb-tx/src/two_pc.rs b/crates/metadb-tx/src/two_pc.rs index 6a51b70..5ddc62a 100644 --- a/crates/metadb-tx/src/two_pc.rs +++ b/crates/metadb-tx/src/two_pc.rs @@ -3,7 +3,9 @@ //! Primary-Anchored Two-Phase Commit (`2PC`) Coordinator. use std::collections::{BTreeMap, HashMap}; +use std::future::Future; use std::sync::Arc; +use std::time::Duration; use metadb_core::schema::validate_user_entry_name; use metadb_core::{ @@ -15,6 +17,7 @@ use metadb_core::{ TABLE_PREFIX_SECONDARY_INDEX, }; use metadb_raft::ShardRaftGroup; +use metadb_storage::SnapshotGuard; use crate::recovery_worker::TxnRecoveryWorker; use crate::rename_cycle::RenameCycleWalker; @@ -22,6 +25,46 @@ use crate::state_machine_impl::{SetAttrPatch, TxRaftCmd, TxRaftOutput, TxStateMa pub const DEFAULT_2PC_TTL_MS: u64 = 3_000; +// A split fences its parent before booting and publishing its child. That +// interval includes Raft recovery and can outlast a short count-based retry loop. +const ROUTE_RETRY_TIMEOUT: Duration = Duration::from_secs(30); +const ROUTE_RETRY_INTERVAL: Duration = Duration::from_millis(25); + +async fn retry_routing(operation: F) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let deadline = tokio::time::Instant::now() + ROUTE_RETRY_TIMEOUT; + retry_routing_until(deadline, operation).await +} + +async fn retry_routing_until( + deadline: tokio::time::Instant, + mut operation: F, +) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + loop { + match operation().await { + Err( + error @ (MetaDbError::EpochMismatch { .. } + | MetaDbError::NotLeader { .. } + | MetaDbError::ShardRoutingError(_)), + ) => { + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(error); + } + tokio::time::sleep(ROUTE_RETRY_INTERVAL.min(deadline - now)).await; + } + result => return result, + } + } +} + /// Fault-injection hook allowing tests to simulate coordinator crashes mid-2PC (`T-2PC-1`). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TwoPcCrashStage { @@ -108,6 +151,53 @@ impl DistributedTxRouter { &self.clock } + fn route_user_key_once(&self, user_key: &[u8]) -> Result { + let (epoch, group, handler) = self.directory.route_user_key(user_key)?; + if group.current_epoch() != epoch || !handler.key_range().contains_key(user_key) { + return Err(MetaDbError::EpochMismatch { + shard_id: group.shard_id(), + expected: epoch, + actual: group.current_epoch(), + }); + } + Ok((epoch, group, handler)) + } + + async fn route_user_key(&self, user_key: &[u8]) -> Result { + retry_routing(|| std::future::ready(self.route_user_key_once(user_key))).await + } + + async fn route_user_range(&self, start: &[u8], end: &[u8]) -> Result> { + retry_routing(|| std::future::ready(self.directory.route_user_range(start, end))).await + } + + fn validate_read_epoch(group: &ShardRaftGroup, epoch: ShardEpoch) -> Result<()> { + let actual = group.current_epoch(); + if actual != epoch { + return Err(MetaDbError::EpochMismatch { + shard_id: group.shard_id(), + expected: epoch, + actual, + }); + } + Ok(()) + } + + async fn read_user_key(&self, user_key: &[u8]) -> Result<(Arc, SnapshotGuard)> { + let (epoch, group, handler) = self.route_user_key_once(user_key)?; + let (_, snapshot) = group.linearizable_read(epoch).await?; + // A range change can occur after routing and before the read cut. + // Never turn a fenced parent's missing key into a successful miss. + if group.current_epoch() != epoch || !handler.key_range().contains_key(user_key) { + return Err(MetaDbError::EpochMismatch { + shard_id: group.shard_id(), + expected: epoch, + actual: group.current_epoch(), + }); + } + Ok((group, snapshot)) + } + async fn propose_cmd( group: &ShardRaftGroup, epoch: ShardEpoch, @@ -128,18 +218,15 @@ impl DistributedTxRouter { user_key: &[u8], cmd: &TxRaftCmd, ) -> Result { + let deadline = tokio::time::Instant::now() + ROUTE_RETRY_TIMEOUT; for attempt in 0..12usize { - let (epoch, group, _) = self.directory.route_user_key(user_key)?; - match Self::propose_cmd(&group, epoch, self.clock.now(), cmd).await { + let result = retry_routing_until(deadline, || async { + let (epoch, group, _) = self.route_user_key_once(user_key)?; + Self::propose_cmd(&group, epoch, self.clock.now(), cmd).await + }) + .await; + match result { Ok((_, out)) => return Ok(out), - Err( - MetaDbError::EpochMismatch { .. } - | MetaDbError::NotLeader { .. } - | MetaDbError::ShardRoutingError(_), - ) if attempt < 11 => { - tokio::time::sleep(std::time::Duration::from_millis(15 * (attempt as u64 + 1))) - .await; - } Err(MetaDbError::WriteIntentConflict { conflicting_txn_id, primary_key, @@ -263,8 +350,8 @@ impl DistributedTxRouter { let child_bytes = child_meta_key.encode(); let new_attr_bytes = MetaKey::dir_attr(new_dir_id).encode(); - let (_, g_parent, _) = self.directory.route_user_key(&child_bytes)?; - let (_, g_new_dir, _) = self.directory.route_user_key(&new_attr_bytes)?; + let (_, g_parent, _) = self.route_user_key(&child_bytes).await?; + let (_, g_new_dir, _) = self.route_user_key(&new_attr_bytes).await?; if g_parent.shard_id() == g_new_dir.shard_id() { let cmd = TxRaftCmd::MkdirSingleShard { @@ -384,8 +471,8 @@ impl DistributedTxRouter { let dir_start = MetaKey::encode_pid_prefix_start(target_dir_id); let dir_end = MetaKey::encode_pid_prefix_end(target_dir_id); - let (_, g_parent, _) = self.directory.route_user_key(&child_bytes)?; - let dir_shards = self.directory.route_user_range(&dir_start, &dir_end)?; + let (_, g_parent, _) = self.route_user_key(&child_bytes).await?; + let dir_shards = self.route_user_range(&dir_start, &dir_end).await?; if dir_shards.len() == 1 && dir_shards[0].1.shard_id() == g_parent.shard_id() { let cmd = TxRaftCmd::RmdirSingleShard { @@ -491,8 +578,8 @@ impl DistributedTxRouter { let dir_id = existing.attr.inode_id; let dir_start = MetaKey::encode_pid_prefix_start(dir_id); let dir_end = MetaKey::encode_pid_prefix_end(dir_id); - let (_, g_parent, _) = self.directory.route_user_key(&child_bytes)?; - let dir_shards = self.directory.route_user_range(&dir_start, &dir_end)?; + let (_, g_parent, _) = self.route_user_key(&child_bytes).await?; + let dir_shards = self.route_user_range(&dir_start, &dir_end).await?; if dir_shards.len() == 1 && dir_shards[0].1.shard_id() == g_parent.shard_id() { let cmd = TxRaftCmd::SetAttrSingleShard { @@ -635,13 +722,13 @@ impl DistributedTxRouter { }); }; - let (_, g_src, _) = self.directory.route_user_key(&src_bytes)?; - let (_, g_dst, _) = self.directory.route_user_key(&dst_bytes)?; + let (_, g_src, _) = self.route_user_key(&src_bytes).await?; + let (_, g_dst, _) = self.route_user_key(&dst_bytes).await?; let single_shard_eligible = if src_entry.attr.entry_type == EntryType::Directory { let d_start = MetaKey::encode_pid_prefix_start(src_entry.attr.inode_id); let d_end = MetaKey::encode_pid_prefix_end(src_entry.attr.inode_id); - let d_shards = self.directory.route_user_range(&d_start, &d_end)?; + let d_shards = self.route_user_range(&d_start, &d_end).await?; g_src.shard_id() == g_dst.shard_id() && d_shards.len() == 1 && d_shards[0].1.shard_id() == g_src.shard_id() @@ -848,21 +935,15 @@ impl DistributedTxRouter { crash_stage: TwoPcCrashStage, ) -> Result { let txn_id = self.directory.allocate_txn_id(self.coord_node_id)?; - let start_hlc = self.clock.now(); - let expire_hlc = lease_opt - .map(|l| l.expire_hlc) - .unwrap_or_else(|| HlcTimestamp::new(start_hlc.physical_ms() + self.ttl_ms, 0)); - let rename_fence_token = lease_opt.map(|l| l.rename_fence_token); - let all_participant_keys: Vec> = intents.iter().map(|(k, _)| k.clone()).collect(); // Group intents by routed `ShardId`. - let (primary_epoch, primary_group, _) = self.directory.route_user_key(primary_key)?; + let (primary_epoch, primary_group, _) = self.route_user_key(primary_key).await?; let primary_shard_id = primary_group.shard_id(); let mut grouped: HashMap = HashMap::new(); for (uk, op) in intents { - let (ep, grp, _) = self.directory.route_user_key(&uk)?; + let (ep, grp, _) = self.route_user_key(&uk).await?; grouped .entry(grp.shard_id()) .or_insert_with(|| (ep, grp, Vec::new())) @@ -870,6 +951,20 @@ impl DistributedTxRouter { .push((uk, op)); } + // Waiting for split publication must not consume a transaction's lease + // before its first prepare. An external rename lease cannot be extended + // here, so fail explicitly if that lease expired while routing. + let start_hlc = self.clock.now(); + let expire_hlc = lease_opt + .map(|l| l.expire_hlc) + .unwrap_or_else(|| HlcTimestamp::new(start_hlc.physical_ms() + self.ttl_ms, 0)); + if expire_hlc <= start_hlc { + return Err(MetaDbError::TransactionAborted( + "directory rename lease expired while waiting for shard routing".into(), + )); + } + let rename_fence_token = lease_opt.map(|l| l.rename_fence_token); + // Phase 1: Prepare on Secondary Shards then Primary Shard (or parallel). let mut prepared_shards: Vec = Vec::new(); let mut max_prepared_hlc = start_hlc; @@ -993,20 +1088,16 @@ impl DistributedTxRouter { /// Linearizable point `Lookup(pid, name)` resolving any encountered `[0x05]` `WriteIntent`s. pub async fn lookup(&self, pid: InodeId, name: &str) -> Result> { + retry_routing(|| self.lookup_once(pid, name)).await + } + + async fn lookup_once(&self, pid: InodeId, name: &str) -> Result> { validate_user_entry_name(name)?; let mk = MetaKey::child(pid, name); let key_bytes = mk.encode(); for _ in 0..8 { - let (epoch, group, _) = self.directory.route_user_key(&key_bytes)?; - let (_, snap) = match group.linearizable_read(epoch).await { - Ok(pair) => pair, - Err(MetaDbError::EpochMismatch { .. } | MetaDbError::NotLeader { .. }) => { - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - continue; - } - Err(e) => return Err(e), - }; + let (group, snap) = self.read_user_key(&key_bytes).await?; let engine = group.engine(); // Check `[0x05]` intent at `snap.cut()`. @@ -1043,13 +1134,12 @@ impl DistributedTxRouter { /// Reads committed bytes used as the predicate for an anchor attribute mutation. async fn read_raw_user_value(&self, key: &[u8]) -> Result>> { + retry_routing(|| self.read_raw_user_value_once(key)).await + } + + async fn read_raw_user_value_once(&self, key: &[u8]) -> Result>> { for _ in 0..12 { - let (epoch, group, _) = self.directory.route_user_key(key)?; - let (_, snap) = match group.linearizable_read(epoch).await { - Ok(pair) => pair, - Err(MetaDbError::EpochMismatch { .. } | MetaDbError::NotLeader { .. }) => continue, - Err(err) => return Err(err), - }; + let (group, snap) = self.read_user_key(key).await?; let engine = group.engine(); let intent_key = WriteIntentKey::new(key.to_vec()).encode(); if let Some(raw) = engine.get_with_snapshot(&intent_key, &snap)? { @@ -1078,6 +1168,10 @@ impl DistributedTxRouter { /// - `DualModeFineGrained` (`K` sub-shards): parallel `ReadIndex` + shared `barrier_hlc` /// scatter-gather across `S_1 .. S_K` (`Resolution B1` / `G-P1`). pub async fn dirstat(&self, pid: InodeId) -> Result { + retry_routing(|| self.dirstat_once(pid)).await + } + + async fn dirstat_once(&self, pid: InodeId) -> Result { let start = MetaKey::encode_pid_prefix_start(pid); let end = MetaKey::encode_pid_prefix_end(pid); @@ -1093,14 +1187,17 @@ impl DistributedTxRouter { } Err(e) => return Err(e), }; + Self::validate_read_epoch(group, *epoch)?; let attr_key = MetaKey::dir_attr(pid).encode(); let Some(raw) = group.engine().get_with_snapshot(&attr_key, &snap)? else { return Err(MetaDbError::DirectoryNotFound(pid)); }; let persisted = InodeAttr::decode(&raw)?; - return Ok(handler + let reconciled = handler .attr_aggregator() - .reconcile_inode_attr(pid, persisted, &snap)); + .reconcile_inode_attr(pid, persisted, &snap); + Self::validate_read_epoch(group, *epoch)?; + return Ok(reconciled); } // `DualModeFineGrained` multi-shard scatter-gather with shared `barrier_hlc`: @@ -1131,6 +1228,7 @@ impl DistributedTxRouter { let mut latest_mtime = HlcTimestamp::ZERO; for (epoch, group, handler) in &shards { + let range = handler.key_range(); let (_, snap) = match group.fixed_barrier_read(*epoch, barrier_hlc).await { Ok(p) => p, Err(MetaDbError::EpochMismatch { .. } | MetaDbError::NotLeader { .. }) => { @@ -1139,8 +1237,9 @@ impl DistributedTxRouter { } Err(e) => return Err(e), }; + Self::validate_read_epoch(group, *epoch)?; let eng = group.engine(); - if handler.key_range().contains_key(&attr_key) { + if range.contains_key(&attr_key) { let Some(raw) = eng.get_with_snapshot(&attr_key, &snap)? else { return Err(MetaDbError::DirectoryNotFound(pid)); }; @@ -1169,6 +1268,10 @@ impl DistributedTxRouter { continue; } + for (epoch, group, _) in &shards { + Self::validate_read_epoch(group, *epoch)?; + } + let mut canonical = anchor_attr.ok_or(MetaDbError::DirectoryNotFound(pid))?; canonical.num_children = total_children.max(0); canonical.size_bytes = total_bytes.max(0); @@ -1188,6 +1291,16 @@ impl DistributedTxRouter { after_name: Option<&str>, name_prefix: Option<&str>, limit: usize, + ) -> Result<(Vec, Option)> { + retry_routing(|| self.listdir_paginated_once(pid, after_name, name_prefix, limit)).await + } + + async fn listdir_paginated_once( + &self, + pid: InodeId, + after_name: Option<&str>, + name_prefix: Option<&str>, + limit: usize, ) -> Result<(Vec, Option)> { let effective_limit = if limit == 0 { 10_000 } else { limit }; let scan_start = listdir_scan_start(pid, after_name, name_prefix); @@ -1211,6 +1324,7 @@ impl DistributedTxRouter { } let (_, snap) = group.linearizable_read(epoch).await?; + Self::validate_read_epoch(&group, epoch)?; let eng = group.engine(); // Resolve any visible active `[0x05]` child write intents in `[sub_start, sub_end)`. @@ -1240,6 +1354,7 @@ impl DistributedTxRouter { } else { snap }; + Self::validate_read_epoch(&group, epoch)?; let rows = eng.scan_with_snapshot(sub_start, sub_end, &final_snap)?; for (k_bytes, v_bytes) in rows { @@ -1306,9 +1421,12 @@ impl DistributedTxRouter { } pub async fn get_quota(&self, uid: UserId) -> Result> { + retry_routing(|| self.get_quota_once(uid)).await + } + + async fn get_quota_once(&self, uid: UserId) -> Result> { let qk = QuotaKey { uid }.encode(); - let (epoch, group, _) = self.directory.route_user_key(&qk)?; - let (_, snap) = group.linearizable_read(epoch).await?; + let (group, snap) = self.read_user_key(&qk).await?; let Some(raw) = group.engine().get_with_snapshot(&qk, &snap)? else { return Ok(None); }; @@ -1316,6 +1434,10 @@ impl DistributedTxRouter { } pub async fn list_quotas(&self, start_uid: UserId, limit: usize) -> Result> { + retry_routing(|| self.list_quotas_once(start_uid, limit)).await + } + + async fn list_quotas_once(&self, start_uid: UserId, limit: usize) -> Result> { let lim = if limit == 0 { 1000 } else { limit }; let start = QuotaKey { uid: start_uid }.encode(); let end = vec![TABLE_PREFIX_SECONDARY_INDEX]; @@ -1329,6 +1451,7 @@ impl DistributedTxRouter { continue; } let (_, snap) = group.linearizable_read(epoch).await?; + Self::validate_read_epoch(&group, epoch)?; for (_, v) in group.engine().scan_with_snapshot(s, e, &snap)? { out.push(QuotaRecord::decode(&v)?); if out.len() >= lim { @@ -1340,9 +1463,12 @@ impl DistributedTxRouter { } pub async fn get_secondary_index(&self, fid: FileId) -> Result> { + retry_routing(|| self.get_secondary_index_once(fid)).await + } + + async fn get_secondary_index_once(&self, fid: FileId) -> Result> { let ik = SecondaryIndexKey { fid }.encode(); - let (epoch, group, _) = self.directory.route_user_key(&ik)?; - let (_, snap) = group.linearizable_read(epoch).await?; + let (group, snap) = self.read_user_key(&ik).await?; let Some(raw) = group.engine().get_with_snapshot(&ik, &snap)? else { return Ok(None); }; @@ -1358,6 +1484,14 @@ impl DistributedTxRouter { &self, after_fid: FileId, limit: usize, + ) -> Result> { + retry_routing(|| self.scan_secondary_index_once(after_fid, limit)).await + } + + async fn scan_secondary_index_once( + &self, + after_fid: FileId, + limit: usize, ) -> Result> { let lim = if limit == 0 { 1000 } else { limit }; let Some(next_fid) = after_fid.checked_add(1) else { @@ -1375,6 +1509,7 @@ impl DistributedTxRouter { continue; } let (_, snap) = group.linearizable_read(epoch).await?; + Self::validate_read_epoch(&group, epoch)?; for (_, v) in group.engine().scan_with_snapshot(s, e, &snap)? { let rec = SecondaryIndexRecord::decode(&v)?; if !rec.is_deleted { diff --git a/docs/designs/correctness_hardening.md b/docs/designs/correctness_hardening.md index d3ab97a..250dbdf 100644 --- a/docs/designs/correctness_hardening.md +++ b/docs/designs/correctness_hardening.md @@ -20,6 +20,8 @@ Splitting waits until local intents and unacknowledged transaction decisions are Recovery keeps a deferred split plan pending and the original parent route available, allowing startup and transaction maintenance to proceed. Maintenance retries the plan after draining transaction state. Once the parent fence is committed, recovery completes the existing split without waiting for new transactions in the retained range. Only explicitly retryable drain conditions are deferred; corruption and storage errors still stop recovery. +Routing validates published epochs and ranges against the live shard before returning a handle. Client operations wait for route publication with a bounded retry deadline, and reads recheck ownership at their snapshot boundary. A fenced parent cannot report a moved key as absent or silently omit an unpublished child from a directory listing. + Directory creation rejects root/self/invalid IDs and existing authoritative directory identities. Mutations validate sizes and directory capacity before changing keys. Prepared transactions reserve positive directory capacity. Split cleanup reconstructs retained directory counters during apply, and metadata-only directory updates preserve pending and flushed child accounting. Quota batches validate and stage their complete effects before mutation. Internal balances remain signed so independent origin streams commute: a delete delivered before the corresponding create can temporarily make a balance negative and then converge to zero. Overflow rejects the entire batch. Callers must not interpret a temporarily negative asynchronous balance as a final quota decision. From ceccd8d1bb3324afbf8dfa3bf3a570b870c07a40 Mon Sep 17 00:00:00 2001 From: loverustfs Date: Mon, 16 Nov 2026 21:07:08 +0800 Subject: [PATCH 4/5] Make maintenance and metrics regressions deterministic Verify the purged committed prefix, durability watermarks, and restart data instead of racing background log retention. Report the bound metrics address, discover an ephemeral listener with startup diagnostics, and validate concurrent scrapes across multiple idle connections. --- crates/metadb-server/src/bin/metadbd.rs | 2 +- .../metadb-server/tests/integration_tests.rs | 100 +++++++---- .../metadb-server/tests/round2_cli_tests.rs | 155 ++++++++++++------ 3 files changed, 175 insertions(+), 82 deletions(-) diff --git a/crates/metadb-server/src/bin/metadbd.rs b/crates/metadb-server/src/bin/metadbd.rs index 5df5bbb..a41ec75 100644 --- a/crates/metadb-server/src/bin/metadbd.rs +++ b/crates/metadb-server/src/bin/metadbd.rs @@ -146,7 +146,7 @@ async fn main() -> Result<(), Box> { let metrics_task = if let Some(m_addr) = metrics_addr_opt { let m_listener = TcpListener::bind(&m_addr).await?; - tracing::info!(metrics_addr = %m_addr, "Prometheus metrics endpoint listening"); + tracing::info!(metrics_addr = %m_listener.local_addr()?, "Prometheus metrics endpoint listening"); let (stop, stopped) = oneshot::channel(); let m_cluster = cluster.clone(); Some(( diff --git a/crates/metadb-server/tests/integration_tests.rs b/crates/metadb-server/tests/integration_tests.rs index e337c65..ffb4ff7 100644 --- a/crates/metadb-server/tests/integration_tests.rs +++ b/crates/metadb-server/tests/integration_tests.rs @@ -12,7 +12,8 @@ use metadb_server::{ MetaDbConfig, MetaDbEngine, MetaDbGrpcService, PersistentMetaDbCluster, ServerTomlConfig, SqlQueryResult, }; -use metadb_tx::AuxUpdateTask; +use metadb_tx::{AuxUpdateTask, ShardDirectory}; +use openraft::storage::{RaftLogReader, RaftLogStorage}; use tempfile::tempdir; use tonic::Request; @@ -454,7 +455,7 @@ async fn test_persistent_restart_lookup_dirstat_quota() { cluster.shutdown().await.unwrap(); } -/// After many committed writes, one maintenance tick must drop retained Raft entries. +/// Maintenance must purge a confirmed write prefix and recover it from a snapshot. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_maintenance_tick_purges_raft_log() { let dir = tempdir().unwrap(); @@ -462,51 +463,90 @@ async fn test_maintenance_tick_purges_raft_log() { let (cluster, _) = PersistentMetaDbCluster::open_and_serve(cfg.clone()) .await .unwrap(); + let (_, group, _) = cluster + .global_manager() + .route_user_key(&MetaKey::dir_attr(ROOT_INODE_ID).encode()) + .unwrap(); + let baseline = group + .linearizable_read(group.current_epoch()) + .await + .unwrap() + .0 + .read_index; for i in 0..24 { cluster .create_file_auto_id(ROOT_INODE_ID, &format!("f{i:02}.dat"), 3, 0o644, 64) .await .unwrap(); } - let before: usize = cluster - .cell_manager() - .host() - .all_groups() - .iter() - .map(|g| g.log_store().retained_entry_count()) - .sum(); + let confirmed = group + .linearizable_read(group.current_epoch()) + .await + .unwrap() + .0 + .read_index; assert!( - before > 8, - "expected a non-trivial raft log, retained {before}" + confirmed >= baseline + 24, + "the committed frontier must include all 24 creates" ); + + // The 250 ms background task may already have purged these writes. Verify + // the durable prefix instead of assuming an initial retained-entry count. cluster.run_maintenance_tick().await.unwrap(); - let after: usize = cluster - .cell_manager() - .host() - .all_groups() - .iter() - .map(|g| g.log_store().retained_entry_count()) - .sum(); + let mut log = group.log_store().clone(); + let purged = tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let purged = log + .get_log_state() + .await + .unwrap() + .last_purged_log_id + .map_or(0, |id| id.index); + if purged >= confirmed { + break purged; + } + // OpenRaft executes the requested prefix purge asynchronously. + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("maintenance did not purge the confirmed write prefix"); + let (state_flushed, aux_durable, last_snapshot) = group.watermarks().snapshot_values(); assert!( - after < before, - "maintenance must purge raft logs: before {before} after {after}" + purged <= state_flushed.min(aux_durable).min(last_snapshot), + "purging must respect all three durability watermarks" ); + assert!(log + .try_get_log_entries(..=confirmed) + .await + .unwrap() + .is_empty()); cluster.shutdown().await.unwrap(); + drop(log); + drop(group); drop(cluster); let (cluster, _) = PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap(); - let snapshot_floor: u64 = cluster - .cell_manager() - .host() - .all_groups() - .iter() - .map(|g| g.watermarks().snapshot_values().2) - .max() - .unwrap_or(0); + let (_, group, _) = cluster + .global_manager() + .route_user_key(&MetaKey::dir_attr(ROOT_INODE_ID).encode()) + .unwrap(); assert!( - snapshot_floor > 0, - "reopen must restore last_snapshot_index from the MANIFEST flushed index" + group.watermarks().snapshot_values().2 >= confirmed, + "reopen must restore the snapshot covering the purged write prefix" ); + for i in 0..24 { + let entry = cluster + .tx_router() + .lookup(ROOT_INODE_ID, &format!("f{i:02}.dat")) + .await + .unwrap() + .expect("a purged write must survive snapshot recovery"); + assert_eq!(entry.attr.size_bytes, 64); + } + let stat = cluster.tx_router().dirstat(ROOT_INODE_ID).await.unwrap(); + assert_eq!(stat.num_children, 24); + assert_eq!(stat.size_bytes, 24 * 64); cluster.shutdown().await.unwrap(); } diff --git a/crates/metadb-server/tests/round2_cli_tests.rs b/crates/metadb-server/tests/round2_cli_tests.rs index 7862cfc..b3117c3 100644 --- a/crates/metadb-server/tests/round2_cli_tests.rs +++ b/crates/metadb-server/tests/round2_cli_tests.rs @@ -173,64 +173,102 @@ async fn idle_metrics_connection_must_not_block_other_scrapes() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; let dir = tempfile::tempdir().unwrap(); - let port_reservation = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let address = port_reservation.local_addr().unwrap(); let config_path = dir.path().join("server.toml"); + let stdout_path = dir.path().join("daemon.stdout"); + let stderr_path = dir.path().join("daemon.stderr"); + let daemon_output = || { + format!( + "stdout:\n{}\nstderr:\n{}", + std::fs::read_to_string(&stdout_path).unwrap_or_default(), + std::fs::read_to_string(&stderr_path).unwrap_or_default() + ) + }; let mut cfg = ServerTomlConfig::default(); cfg.node.data_dir = dir.path().join("data"); cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); - cfg.node.metrics_listen_addr = Some(address.to_string()); + cfg.node.metrics_listen_addr = Some("127.0.0.1:0".into()); std::fs::write(&config_path, toml::to_string(&cfg).unwrap()).unwrap(); - drop(port_reservation); let mut daemon = tokio::process::Command::new(env!("CARGO_BIN_EXE_metadbd")) .arg("--config") .arg(&config_path) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) + .env("NO_COLOR", "1") + .env("RUST_LOG", "info") + .stdout(std::fs::File::create(&stdout_path).unwrap()) + .stderr(std::fs::File::create(&stderr_path).unwrap()) .kill_on_drop(true) .spawn() .unwrap(); - let mut idle = tokio::time::timeout(Duration::from_secs(20), async { + // Discover the actual ephemeral listener instead of releasing a reserved + // port before spawn. Bootstrap replays several Raft groups, so distinguish + // process exit from bounded readiness under the full parallel CLI suite. + let readiness = tokio::time::timeout(Duration::from_secs(60), async { loop { - if let Ok(socket) = tokio::net::TcpStream::connect(address).await { - break socket; + if let Some(status) = daemon.try_wait().unwrap() { + return Err(format!("metadbd exited before metrics readiness: {status}")); + } + let output = daemon_output(); + for line in output + .lines() + .filter(|line| line.contains("Prometheus metrics endpoint listening")) + { + // Locate the numeric address so diagnostics remain usable even + // if a tracing formatter includes ANSI styling around fields. + if let Some(start) = line.find("127.0.0.1:") { + let address: String = line[start..] + .chars() + .take_while(|ch| ch.is_ascii_digit() || matches!(ch, '.' | ':')) + .collect(); + if let Ok(address) = address.parse::() { + if let Ok(socket) = tokio::net::TcpStream::connect(address).await { + return Ok((address, socket)); + } + } + } } - tokio::time::sleep(Duration::from_millis(10)).await; + tokio::time::sleep(Duration::from_millis(20)).await; } }) - .await - .unwrap(); - // Let the daemon accept the first connection and enter its request read. - tokio::time::sleep(Duration::from_millis(50)).await; - let mut scrape = tokio::net::TcpStream::connect(address).await.unwrap(); - scrape - .write_all(b"GET /metrics HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") - .await - .unwrap(); - let mut response = vec![0u8; 4096]; - let while_idle = - tokio::time::timeout(Duration::from_millis(250), scrape.read(&mut response)).await; - let blocked = while_idle.is_err(); - if blocked { - drop(idle); - let count = tokio::time::timeout(Duration::from_secs(2), scrape.read(&mut response)) - .await - .unwrap() - .unwrap(); - assert!(String::from_utf8_lossy(&response[..count]).contains("HTTP/1.1 200 OK")); - } else { - let count = while_idle.unwrap().unwrap(); - assert!(String::from_utf8_lossy(&response[..count]).contains("HTTP/1.1 200 OK")); - assert_eq!( - tokio::time::timeout(Duration::from_secs(3), idle.read(&mut response)) - .await - .unwrap() - .unwrap(), - 0, - "idle metrics connection did not expire" - ); + .await; + let (address, first_idle) = match readiness { + Ok(Ok(ready)) => ready, + failed => { + let _ = daemon.start_kill(); + let _ = daemon.wait().await; + panic!("metrics readiness failed: {failed:?}\n{}", daemon_output()); + } + }; + let mut idle_connections = vec![first_idle]; + for _ in 1..8 { + idle_connections.push(tokio::net::TcpStream::connect(address).await.unwrap()); } - let _idle_during_shutdown = tokio::net::TcpStream::connect(address).await.unwrap(); + // A serial implementation with the two-second connection deadline would + // spend sixteen seconds on these clients before reaching the actual scrape. + // Five seconds tolerates scheduler load without accepting that regression. + tokio::time::sleep(Duration::from_millis(50)).await; + let scrape = tokio::time::timeout(Duration::from_secs(5), async { + let mut socket = tokio::net::TcpStream::connect(address).await?; + socket + .write_all(b"GET /metrics HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .await?; + let mut response = Vec::new(); + socket.read_to_end(&mut response).await?; + Ok::<_, std::io::Error>(response) + }) + .await; + let idle_expiration = tokio::time::timeout(Duration::from_secs(5), async { + for mut socket in idle_connections { + let mut byte = [0]; + if socket.read(&mut byte).await? != 0 { + return Err(std::io::Error::other( + "idle client received unexpected data", + )); + } + } + Ok::<_, std::io::Error>(()) + }) + .await; + let mut idle_during_shutdown = tokio::net::TcpStream::connect(address).await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; #[cfg(unix)] { let status = tokio::process::Command::new("kill") @@ -239,13 +277,15 @@ async fn idle_metrics_connection_must_not_block_other_scrapes() { .await .unwrap(); assert!(status.success()); + let shutdown = tokio::time::timeout(Duration::from_secs(10), daemon.wait()).await; + if shutdown.is_err() { + let _ = daemon.start_kill(); + let _ = daemon.wait().await; + } assert!( - tokio::time::timeout(Duration::from_secs(10), daemon.wait()) - .await - .unwrap() - .unwrap() - .success(), - "daemon failed graceful shutdown with an idle metrics connection" + shutdown.unwrap().unwrap().success(), + "daemon failed graceful shutdown with an idle metrics connection:\n{}", + daemon_output() ); } #[cfg(not(unix))] @@ -253,10 +293,23 @@ async fn idle_metrics_connection_must_not_block_other_scrapes() { daemon.start_kill().unwrap(); daemon.wait().await.unwrap(); } - assert!( - !blocked, - "one idle TCP connection blocked every metrics request until it disconnected" + let mut byte = [0]; + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), idle_during_shutdown.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0, + "shutdown left an idle metrics connection open" ); + let response = scrape + .expect("idle clients blocked the metrics scrape") + .unwrap(); + assert!(String::from_utf8_lossy(&response).contains("HTTP/1.1 200 OK")); + assert!(String::from_utf8_lossy(&response).contains("metadb_shards")); + idle_expiration + .expect("idle metrics connections did not expire concurrently") + .unwrap(); } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] From 315a264cdcda8091767378648a446424a3f156cb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 17 Nov 2026 07:12:49 +0000 Subject: [PATCH 5/5] Preserve landed join, quota, and auth behavior on the audit rebase Keep sole-voter address moves, exact quota sums that still clamp on read, fast NotLeader replies, Raft bearer auth, and the JSON operator CLI without dropping the exclusive allocator, split-range waits, or fail-closed checks. Co-authored-by: RustFS --- crates/metadb-cluster/src/cell_manager.rs | 13 +- crates/metadb-cluster/src/global_manager.rs | 20 +-- crates/metadb-cluster/src/id_allocator.rs | 151 +++++++++++++--- crates/metadb-core/src/lib.rs | 7 +- crates/metadb-proxy/src/sql_templates.rs | 32 ++-- crates/metadb-server/src/bin/metadb-ctl.rs | 163 ++---------------- crates/metadb-server/src/lib.rs | 31 ++-- .../metadb-server/tests/grpc_regressions.rs | 9 +- .../tests/input_boundary_regressions.rs | 3 +- .../tests/join_existing_group.rs | 7 + crates/metadb-tx/src/aux_outbox.rs | 66 +++---- crates/metadb-tx/src/aux_sink.rs | 4 +- crates/metadb-tx/src/state_machine_impl.rs | 21 ++- crates/metadb-tx/src/two_pc.rs | 34 +++- crates/metadb-tx/src/validation.rs | 4 +- docs/de/architecture.md | 2 +- docs/de/build-and-run.md | 2 +- docs/de/configuration.md | 4 +- docs/de/operations.md | 4 +- docs/de/testing.md | 15 +- docs/en/architecture.md | 2 +- docs/en/build-and-run.md | 2 +- docs/en/configuration.md | 8 +- docs/en/operations.md | 4 +- docs/en/testing.md | 17 +- docs/fr/architecture.md | 2 +- docs/fr/build-and-run.md | 2 +- docs/fr/configuration.md | 4 +- docs/fr/operations.md | 4 +- docs/fr/testing.md | 15 +- docs/guides/ja/deployment.md | 2 +- docs/guides/zh/deployment.md | 2 +- docs/ja/architecture.md | 2 +- docs/ja/build-and-run.md | 2 +- docs/ja/configuration.md | 4 +- docs/ja/operations.md | 4 +- docs/ja/testing.md | 15 +- docs/zh/architecture.md | 2 +- docs/zh/build-and-run.md | 2 +- docs/zh/configuration.md | 4 +- docs/zh/operations.md | 4 +- docs/zh/testing.md | 15 +- 42 files changed, 357 insertions(+), 353 deletions(-) diff --git a/crates/metadb-cluster/src/cell_manager.rs b/crates/metadb-cluster/src/cell_manager.rs index 58c307d..321ee9c 100644 --- a/crates/metadb-cluster/src/cell_manager.rs +++ b/crates/metadb-cluster/src/cell_manager.rs @@ -277,7 +277,8 @@ impl DurableCellManager { let state_dir = state_dir.as_ref().to_path_buf(); fs::create_dir_all(&state_dir)?; let state_path = state_dir.join("CELL_MANAGER_STATE"); - let persisted = if state_path.exists() { + let state_existed = state_path.exists(); + let persisted = if state_existed { let mut loaded = Self::load_state_file(&state_dir)?; loaded.nodes.insert(host.node_id(), local_addr); Self::write_state_file(&state_dir, &loaded)?; @@ -288,6 +289,16 @@ impl DurableCellManager { Self::write_state_file(&state_dir, &init)?; init }; + // Record the owner before membership adds peers. A later restart of this + // directory goes through `open_or_bootstrap`, which rejects a multi-peer + // cell directory that has no `CELL_OWNER`. + Self::verify_or_initialize_owner( + &state_dir, + host.node_id(), + cell_id, + &persisted, + state_existed, + )?; Ok(Arc::new(Self { cell_id, state_dir, diff --git a/crates/metadb-cluster/src/global_manager.rs b/crates/metadb-cluster/src/global_manager.rs index b9ac30e..b8b48bb 100644 --- a/crates/metadb-cluster/src/global_manager.rs +++ b/crates/metadb-cluster/src/global_manager.rs @@ -86,22 +86,18 @@ impl DurableGlobalManager { let (_, group, handler) = cell.lookup_local_handle(route.shard_id)?; let actual = group.current_epoch(); let actual_range = handler.key_range(); - if actual != route.epoch - || actual_range != route.range - || !actual_range.contains_key(user_key) - { - // A parent is fenced before its child's route is published. Returning - // its current epoch here would let reads observe the deleted right - // half as absent instead of waiting for the child to become routable. + // A split fence updates the handler range and epoch together, before the + // child route is published. A range mismatch, or a key that the fenced + // parent no longer contains, must wait. Membership changes only the + // epoch and leaves the range in place; those reads stay on this shard. + if actual_range != route.range || !actual_range.contains_key(user_key) { return Err(MetaDbError::EpochMismatch { shard_id: route.shard_id, expected: route.epoch, actual, }); } - // Keep this descriptor's epoch: a later fence must invalidate a read - // barrier or proposal that races after this routing check. - Ok((route.epoch, group, handler)) + Ok((actual, group, handler)) } fn try_route_user_range( @@ -121,7 +117,7 @@ impl DurableGlobalManager { })?; let (_, group, handler) = cell.lookup_local_handle(route.shard_id)?; let actual = group.current_epoch(); - if actual != route.epoch || handler.key_range() != route.range { + if handler.key_range() != route.range { return Err(MetaDbError::EpochMismatch { shard_id: route.shard_id, expected: route.epoch, @@ -136,7 +132,7 @@ impl DurableGlobalManager { )); } covered_until = overlap_end.to_vec(); - out.push((route.epoch, group, handler)); + out.push((actual, group, handler)); } if covered_until.as_slice() != end_user_key { return Err(MetaDbError::ShardRoutingError( diff --git a/crates/metadb-cluster/src/id_allocator.rs b/crates/metadb-cluster/src/id_allocator.rs index 98b4b16..bcc87cb 100644 --- a/crates/metadb-cluster/src/id_allocator.rs +++ b/crates/metadb-cluster/src/id_allocator.rs @@ -21,6 +21,8 @@ pub const CELL_FILE_ID_BLOCK_SIZE: u64 = 4_096; pub const CELL_DIR_AFFINITY_BLOCK_SIZE: u64 = 256; pub const GLOBAL_SHARD_ID_BLOCK_SIZE: u64 = 1_024; const MAX_CACHED_GRANT_UUIDS: usize = 4_096; +const ALLOCATOR_FORMAT_MAGIC: &[u8; 8] = b"MIDA0002"; +const ALLOCATOR_HEADER_LEN: usize = 41; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum AllocIdKind { @@ -70,9 +72,69 @@ impl Default for PersistedAllocatorState { } } +#[derive(Deserialize)] +struct LegacyAllocatorState { + next_file_inode_seq: u64, + next_dir_inode_id: InodeId, + next_file_id: FileId, + next_shard_id: ShardId, + rename_fence_token: u64, + active_rename_lease: Option<(u64, HlcTimestamp)>, + recent_grants: BTreeMap, + grant_fifo: VecDeque, +} + +fn decode_allocator_state(body: &[u8]) -> Result { + if body.starts_with(ALLOCATOR_FORMAT_MAGIC) { + if body.len() < ALLOCATOR_HEADER_LEN + || body[8..40].iter().any(|byte| *byte != 0) + || body[40] != 0xff + { + return Err(MetaDbError::CodecError( + "invalid allocator version 2 header".to_string(), + )); + } + return bincode::deserialize(&body[ALLOCATOR_HEADER_LEN..]) + .map_err(|e| MetaDbError::CodecError(e.to_string())); + } + if let Ok(current) = bincode::deserialize::(body) { + return Ok(current); + } + let old: LegacyAllocatorState = bincode::deserialize(body) + .map_err(|e| MetaDbError::CodecError(format!("ID_ALLOCATOR_STATE: {e}")))?; + // Legacy tuples do not record the ID kind. Keep the UUID so a replay is + // rejected, and keep the counters so new grants do not reuse those IDs. + let recent_grants = old + .recent_grants + .into_iter() + .map(|(uuid, (start, _))| { + ( + uuid, + CachedIdGrant { + kind: AllocIdKind::FileId, + start, + count: 0, + }, + ) + }) + .collect(); + Ok(PersistedAllocatorState { + next_file_inode_seq: old.next_file_inode_seq, + next_dir_inode_id: old.next_dir_inode_id, + next_file_id: old.next_file_id, + next_shard_id: old.next_shard_id, + rename_fence_token: old.rename_fence_token, + active_rename_lease: old.active_rename_lease, + recent_grants, + grant_fifo: old.grant_fifo, + next_client_seq: 1, + }) +} + /// Crash-safe persistent ID & lease allocator enforcing **Write-Before-Grant** (`Resolution B18`). pub struct DurableIdAllocator { state_dir: PathBuf, + _lock_file: File, state: Mutex, /// Test hook: the next `n` persists fail before any byte is written. fail_persists: AtomicU32, @@ -82,12 +144,30 @@ impl DurableIdAllocator { pub fn open(state_dir: impl AsRef) -> Result { let state_dir = state_dir.as_ref().to_path_buf(); fs::create_dir_all(&state_dir)?; + let had_existing_files = fs::read_dir(&state_dir)?.next().transpose()?.is_some(); + let lock_path = state_dir.join("LOCK"); + let lock_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path)?; + lock_file.try_lock().map_err(|e| { + MetaDbError::StorageIoError(format!( + "failed to acquire exclusive allocator lock on {}: {e}", + lock_path.display() + )) + })?; let path = state_dir.join("ID_ALLOCATOR_STATE"); let state = if path.exists() { let mut raw = Vec::new(); File::open(&path)?.read_to_end(&mut raw)?; let body = verify_crc32c_le(&raw, "ID_ALLOCATOR_STATE")?; - bincode::deserialize(body).map_err(|e| MetaDbError::CodecError(e.to_string()))? + decode_allocator_state(body)? + } else if had_existing_files { + return Err(MetaDbError::StorageIoError( + "ID_ALLOCATOR_STATE is missing from an existing allocator directory".to_string(), + )); } else { let init = PersistedAllocatorState::default(); Self::write_state_file(&state_dir, &init)?; @@ -95,6 +175,7 @@ impl DurableIdAllocator { }; Ok(Self { state_dir, + _lock_file: lock_file, state: Mutex::new(state), fail_persists: AtomicU32::new(0), }) @@ -114,8 +195,7 @@ impl DurableIdAllocator { let mut raw = Vec::new(); File::open(&path)?.read_to_end(&mut raw)?; let body = verify_crc32c_le(&raw, "ID_ALLOCATOR_STATE")?; - let st: PersistedAllocatorState = - bincode::deserialize(body).map_err(|e| MetaDbError::CodecError(e.to_string()))?; + let st = decode_allocator_state(body)?; Ok(st.recent_grants.contains_key(&request_uuid)) } @@ -138,6 +218,12 @@ impl DurableIdAllocator { cached.kind ))); } + if cached.count != count { + return Err(MetaDbError::StorageIoError(format!( + "request uuid {request_uuid:#x} already granted {} ids, refused for count {count}", + cached.count + ))); + } return Ok((cached.start, cached.count)); } } @@ -147,9 +233,12 @@ impl DurableIdAllocator { match kind { AllocIdKind::FileInode => { let s = next.next_file_inode_seq; - next.next_file_inode_seq = - next.next_file_inode_seq.checked_add(count).ok_or_else(|| { - MetaDbError::StorageIoError("file inode seq overflow".to_string()) + next.next_file_inode_seq = next + .next_file_inode_seq + .checked_add(count) + .filter(|end| *end <= metadb_core::FILE_INODE_TAG_BIT) + .ok_or_else(|| { + MetaDbError::StorageIoError("file inode bitspace exhausted".to_string()) })?; make_file_inode_id(s) } @@ -233,7 +322,11 @@ impl DurableIdAllocator { let mut next = guard.clone(); next.rename_fence_token = next.rename_fence_token.saturating_add(1); let token = next.rename_fence_token; - let expire_hlc = HlcTimestamp::new(now_hlc.physical_ms() + ttl_ms.max(100), 0); + let expire_ms = now_hlc + .physical_ms() + .checked_add(ttl_ms.max(100)) + .ok_or_else(|| MetaDbError::StorageIoError("rename lease TTL overflow".to_string()))?; + let expire_hlc = HlcTimestamp::new(expire_ms, 0); next.active_rename_lease = Some((token, expire_hlc)); self.persist_sync(&next)?; *guard = next; @@ -264,7 +357,12 @@ impl DurableIdAllocator { } fn write_state_file(dir: &Path, st: &PersistedAllocatorState) -> Result<()> { - let mut raw = bincode::serialize(st).map_err(|e| MetaDbError::CodecError(e.to_string()))?; + let payload = bincode::serialize(st).map_err(|e| MetaDbError::CodecError(e.to_string()))?; + let mut raw = Vec::with_capacity(ALLOCATOR_HEADER_LEN + payload.len()); + raw.extend_from_slice(ALLOCATOR_FORMAT_MAGIC); + raw.extend(std::iter::repeat_n(0u8, 32)); + raw.push(0xff); + raw.extend_from_slice(&payload); append_crc32c_le(&mut raw); let tmp = dir.join("ID_ALLOCATOR_STATE.tmp"); let dst = dir.join("ID_ALLOCATOR_STATE"); @@ -286,7 +384,7 @@ impl DurableIdAllocator { #[derive(Default)] struct LocalBlockCursor { next: u64, - end: u64, + remaining: u64, } /// Fast local consumer buffer leasing blocks from `DurableIdAllocator` with zero per-file disk I/O. @@ -313,52 +411,57 @@ impl LocalIdBuffer { self.allocator.next_client_request_uuid(self.node_id) } + fn take_next(g: &mut LocalBlockCursor) -> Result { + let id = g.next; + g.remaining -= 1; + if g.remaining > 0 { + g.next = g.next.checked_add(1).ok_or_else(|| { + MetaDbError::StorageIoError("local ID cursor overflow".to_string()) + })?; + } + Ok(id) + } + pub fn next_file_inode_id(&self) -> Result { let mut g = self.file_inodes.lock().unwrap(); - if g.next >= g.end { + if g.remaining == 0 { let (start, count) = self.allocator.allocate_block( self.next_uuid()?, AllocIdKind::FileInode, CELL_FILE_ID_BLOCK_SIZE, )?; g.next = start; - g.end = start + count; + g.remaining = count; } - let id = g.next; - g.next += 1; - Ok(id) + Self::take_next(&mut g) } pub fn next_dir_inode_id(&self) -> Result { let mut g = self.dir_inodes.lock().unwrap(); - if g.next >= g.end { + if g.remaining == 0 { let (start, count) = self.allocator.allocate_block( self.next_uuid()?, AllocIdKind::DirInode, CELL_DIR_AFFINITY_BLOCK_SIZE, )?; g.next = start; - g.end = start + count; + g.remaining = count; } - let id = g.next; - g.next += 1; - Ok(id) + Self::take_next(&mut g) } pub fn next_file_id(&self) -> Result { let mut g = self.file_ids.lock().unwrap(); - if g.next >= g.end { + if g.remaining == 0 { let (start, count) = self.allocator.allocate_block( self.next_uuid()?, AllocIdKind::FileId, CELL_FILE_ID_BLOCK_SIZE, )?; g.next = start; - g.end = start + count; + g.remaining = count; } - let id = g.next; - g.next += 1; - Ok(id) + Self::take_next(&mut g) } } diff --git a/crates/metadb-core/src/lib.rs b/crates/metadb-core/src/lib.rs index de6917c..5bd5c23 100644 --- a/crates/metadb-core/src/lib.rs +++ b/crates/metadb-core/src/lib.rs @@ -24,10 +24,9 @@ pub use schema::{ AuxOrderingTuple, EntryType, FileId, InodeAttr, InodeId, MetaEntry, MetaKey, NodeId, QuotaKey, QuotaRecord, SecondaryIndexKey, SecondaryIndexRecord, ShardId, UserId, ATTR_ENTRY_NAME, FILE_INODE_TAG_BIT, INODE_ATTR_ENCODED_LEN, MAX_USER_NAME_BYTES, ROOT_INODE_ID, - SYSTEM_PREFIX_DIR_SHARD_DELTA, - SYSTEM_PREFIX_QUOTA_ORIGIN_CURSOR, SYSTEM_PREFIX_SPILLED_AUX_OUTBOX, SYSTEM_PREFIX_TXN_RECORD, - SYSTEM_PREFIX_WRITE_INTENT, TABLE_PREFIX_META, TABLE_PREFIX_QUOTA, - TABLE_PREFIX_SECONDARY_INDEX, + SYSTEM_PREFIX_DIR_SHARD_DELTA, SYSTEM_PREFIX_QUOTA_ORIGIN_CURSOR, + SYSTEM_PREFIX_SPILLED_AUX_OUTBOX, SYSTEM_PREFIX_TXN_RECORD, SYSTEM_PREFIX_WRITE_INTENT, + TABLE_PREFIX_META, TABLE_PREFIX_QUOTA, TABLE_PREFIX_SECONDARY_INDEX, }; pub use system_keys::{ affinity_key_for_raw_entry, decode_origin_quota_cursor, encode_origin_quota_cursor, diff --git a/crates/metadb-proxy/src/sql_templates.rs b/crates/metadb-proxy/src/sql_templates.rs index 878aeb2..0e9b634 100644 --- a/crates/metadb-proxy/src/sql_templates.rs +++ b/crates/metadb-proxy/src/sql_templates.rs @@ -408,14 +408,12 @@ impl SqlTemplateParser { c.expect_word("QUOTA_LIMIT_BYTES")?; c.expect_token(&SqlToken::Eq)?; let limit_2 = c.parse_u64_arg()?; - if limit_1 != limit_2 { - return Err(MetaDbError::SqlParseError( - "Template #17 requires VALUES quota_limit_bytes to equal the ON CONFLICT UPDATE value; SetQuotaLimit stores one absolute limit".to_string(), - )); - } - Ok(MetadataSqlCommand::SetQuotaLimit { + // A missing row takes the INSERT value. An existing row takes the + // ON CONFLICT value. They are not collapsed into one absolute limit. + Ok(MetadataSqlCommand::UpsertQuotaLimit { uid, - quota_limit_bytes: limit_1, + insert_quota_limit_bytes: limit_1, + update_quota_limit_bytes: limit_2, }) } other => Err(MetaDbError::SqlParseError(format!( @@ -757,33 +755,23 @@ mod tests { ON CONFLICT (uid) DO UPDATE SET quota_limit_bytes = ?"; #[test] - fn template_17_requires_equal_insert_and_update_limits() { + fn template_17_keeps_distinct_insert_and_update_limits() { let cmd = SqlTemplateParser::parse_with_params( SET_QUOTA, &[ SqlParam::U64(77), SqlParam::U64(9_000_000), - SqlParam::U64(9_000_000), + SqlParam::U64(1_000), ], ) .unwrap(); assert_eq!( cmd, - MetadataSqlCommand::SetQuotaLimit { + MetadataSqlCommand::UpsertQuotaLimit { uid: 77, - quota_limit_bytes: 9_000_000, + insert_quota_limit_bytes: 9_000_000, + update_quota_limit_bytes: 1_000, } ); - - let err = SqlTemplateParser::parse_with_params( - SET_QUOTA, - &[ - SqlParam::U64(77), - SqlParam::U64(9_000_000), - SqlParam::U64(1_000), - ], - ) - .unwrap_err(); - assert!(matches!(err, MetaDbError::SqlParseError(_))); } } diff --git a/crates/metadb-server/src/bin/metadb-ctl.rs b/crates/metadb-server/src/bin/metadb-ctl.rs index 7490cec..1cef9fc 100644 --- a/crates/metadb-server/src/bin/metadb-ctl.rs +++ b/crates/metadb-server/src/bin/metadb-ctl.rs @@ -231,17 +231,6 @@ fn normalize_url(ep: &str) -> String { } } -fn reject_app_error( - action: &str, - error: Option, -) -> Result<(), Box> { - if let Some(error) = error { - Err(format!("{action}: {}", error.message).into()) - } else { - Ok(()) - } -} - #[tokio::main] async fn main() -> std::process::ExitCode { match run().await { @@ -276,7 +265,7 @@ async fn run() -> Result<(), Box> { .get_routes(GetRoutesRequest { cell_id: 0 }) .await? .into_inner(); - reject_app_error("cluster status", resp.error)?; + check_error("get_routes", resp.error.as_ref())?; println!("Active Shards: {}", resp.shards.len()); for s in resp.shards { let ep = s.epoch.unwrap_or_default(); @@ -301,7 +290,7 @@ async fn run() -> Result<(), Box> { .get_routes(GetRoutesRequest { cell_id }) .await? .into_inner(); - reject_app_error("shard list", resp.error)?; + check_error("get_routes", resp.error.as_ref())?; for s in resp.shards { let ep = s.epoch.unwrap_or_default(); println!( @@ -315,7 +304,9 @@ async fn run() -> Result<(), Box> { .trigger_shard_split(TriggerShardSplitRequest { shard_id }) .await? .into_inner(); - reject_app_error("shard split", resp.error)?; + if let Some(err) = resp.error { + return Err(application_error("split", &err)); + } println!( "split_executed={} child_shard_id={}", resp.split_executed, resp.child_shard_id @@ -330,7 +321,9 @@ async fn run() -> Result<(), Box> { }) .await? .into_inner(); - reject_app_error("enable-dual-mode", resp.error)?; + if let Some(err) = resp.error { + return Err(application_error("enable_dual_mode", &err)); + } println!( "DualModeFineGrained enabled on shard {} (PID {})", shard_id, pid @@ -353,7 +346,9 @@ async fn run() -> Result<(), Box> { }) .await? .into_inner(); - reject_app_error("scale-replicas", resp.error)?; + if let Some(err) = resp.error { + return Err(application_error("scale_replicas", &err)); + } println!( "Upgraded {} shard(s) to {}-Voter", resp.upgraded_shards, target_voters @@ -373,7 +368,9 @@ async fn run() -> Result<(), Box> { }) .await? .into_inner(); - reject_app_error("cell register", resp.error)?; + if let Some(err) = resp.error { + return Err(application_error("register_cell", &err)); + } println!("Registered Cell {}", cell_id); } Command::Sql { query } => { @@ -386,24 +383,8 @@ async fn run() -> Result<(), Box> { }) .await? .into_inner(); - reject_app_error("sql", resp.error)?; - if let Some(attr) = resp.single_attr { - println!( - "InodeAttr(id={}, children={}, size_bytes={}, mode=0o{:o})", - attr.inode_id, attr.num_children, attr.size_bytes, attr.mode - ); - } else if !resp.entries.is_empty() { - for e in resp.entries { - println!("Entry(pid={}, name='{}')", e.pid, e.name); - } - } else if let Some(q) = resp.single_quota { - println!( - "Quota(uid={}, used_bytes={}, used_inodes={}, limit={})", - q.uid, q.used_bytes, q.used_inodes, q.quota_limit_bytes - ); - } else { - println!("OK"); - } + check_error("sql", resp.error.as_ref())?; + println!("{}", serde_json::to_string_pretty(&resp)?); } Command::Bench { workload, @@ -417,118 +398,8 @@ async fn run() -> Result<(), Box> { } => { if embedded || workload == "mixed" { run_embedded_bench(&profile, duration_secs, concurrency, keyspace, seed).await?; - return Ok(()); - } - let concurrency = concurrency.unwrap_or(16); - if workload != "hotspot-create" && workload != "bulk-ttl-delete" { - return Err(format!( - "unsupported --workload '{}': expected 'hotspot-create', 'bulk-ttl-delete', or 'mixed'", - workload - ) - .into()); - } - let mut setup_client = open_meta(&url, &token).await?; - let bench_pid = if workload == "hotspot-create" { - 999_000u64 - } else { - 999_001u64 - }; - let _ = setup_client - .mkdir(MkdirRequest { - header: None, - pid: 1, - name: format!("bench_{}", workload.replace('-', "_")), - dir_id: bench_pid, - uid: 1, - mode: 0o755, - }) - .await?; - - let start = Instant::now(); - let ops_per_task = (ops / concurrency.max(1)).max(1); - let mut handles = Vec::new(); - - for tid in 0..concurrency.max(1) { - let u = url.clone(); - let token = token.clone(); - handles.push(tokio::spawn(async move { - let mut client = open_meta(&u, &token).await.unwrap(); - for i in 0..ops_per_task { - let seq = (tid * ops_per_task + i) as u64; - let _ = client - .create_file(CreateFileRequest { - header: None, - pid: bench_pid, - name: format!("f_{:04}_{:06}.dat", tid, i), - inode_id: (1u64 << 63) | (100_000 + seq), - fid: 500_000 + seq, - uid: 1, - mode: 0o644, - size_bytes: 4096, - }) - .await - .unwrap(); - } - })); - } - for h in handles { - h.await?; - } - let total_done = concurrency.max(1) * ops_per_task; - - if workload == "bulk-ttl-delete" { - let del_start = Instant::now(); - let mut del_handles = Vec::new(); - for tid in 0..concurrency.max(1) { - let u = url.clone(); - let token = token.clone(); - del_handles.push(tokio::spawn(async move { - let mut client = open_meta(&u, &token).await.unwrap(); - for i in 0..ops_per_task { - let _ = client - .unlink(UnlinkRequest { - header: None, - pid: bench_pid, - name: format!("f_{:04}_{:06}.dat", tid, i), - }) - .await - .unwrap(); - } - })); - } - for h in del_handles { - h.await?; - } - let scan_start = Instant::now(); - let list_resp = setup_client - .list_dir(ListDirRequest { - header: None, - pid: bench_pid, - after_name: String::new(), - name_prefix: String::new(), - limit: 1000, - }) - .await? - .into_inner(); - let scan_ms = scan_start.elapsed().as_secs_f64() * 1000.0; - let del_elapsed = del_start.elapsed(); - println!( - "Completed {} bulk TTL deletes in {:.3}s ({:.0} ops/s); post-delete listdir ({} live) took {:.3} ms", - total_done, - del_elapsed.as_secs_f64(), - total_done as f64 / del_elapsed.as_secs_f64(), - list_resp.entries.len(), - scan_ms - ); } else { - let elapsed = start.elapsed(); - let qps = total_done as f64 / elapsed.as_secs_f64(); - println!( - "Completed {} single-directory hotspot creates in {:.3}s ({:.0} ops/s)", - total_done, - elapsed.as_secs_f64(), - qps - ); + run_benchmark(&url, &token, &workload, concurrency.unwrap_or(16), ops).await?; } } } diff --git a/crates/metadb-server/src/lib.rs b/crates/metadb-server/src/lib.rs index ccdcbc1..8fd724e 100644 --- a/crates/metadb-server/src/lib.rs +++ b/crates/metadb-server/src/lib.rs @@ -534,16 +534,18 @@ impl PersistentMetaDbCluster { } else { config.node.grpc_listen_addr.clone() }; - let listener = TcpListener::bind(&listen_addr).await?; - let bound_addr = listener.local_addr()?.to_string(); - if let Some(prev) = &persisted_addr { - if &bound_addr != prev { - return Err(MetaDbError::ConsensusError(format!( - "node {} is already a raft member at {prev}; refusing to listen on {bound_addr}", - config.node.node_id - ))); + let listener = match TcpListener::bind(&listen_addr).await { + Ok(listener) => listener, + Err(error) + if error.kind() == std::io::ErrorKind::AddrInUse && persisted_addr.is_some() => + { + // The saved port is busy. A sole voter can move; several voters cannot. + // `boot_local_shards` checks that after this bind. + TcpListener::bind(&config.node.grpc_listen_addr).await? } - } + Err(error) => return Err(error.into()), + }; + let bound_addr = listener.local_addr()?.to_string(); let clock = Arc::new(HybridLogicalClock::with_max_drift_ms( config.clock.max_clock_drift_ms, @@ -563,6 +565,7 @@ impl PersistentMetaDbCluster { block_cache, clock.clone(), )?); + host.set_shared_secret(Some(config.security.shared_secret.clone())); let raft_router = host.router().clone(); let directory_role = if cell_dir.join("CELL_MANAGER_STATE").exists() { DurableCellManager::inspect_directory(&cell_dir)? @@ -637,7 +640,7 @@ impl PersistentMetaDbCluster { "gRPC metadata, transaction, and control-plane services have no authentication or TLS. \ Default bind is 127.0.0.1. Do not expose this process to untrusted networks until \ [security].shared_secret or an external TLS terminator is configured. \ - RaftTransport on the same listener is also unauthenticated." + RaftTransport on the same listener requires the same bearer when the secret is set." ); } let grpc_handle = @@ -863,9 +866,13 @@ fn spawn_grpc_server( grpc_svc.clone(), auth.clone(), )) - .add_service(ControlPlaneServiceServer::with_interceptor(grpc_svc, auth)) - .add_service(RaftTransportServiceServer::new( + .add_service(ControlPlaneServiceServer::with_interceptor( + grpc_svc, + auth.clone(), + )) + .add_service(RaftTransportServiceServer::with_interceptor( MultiRaftTransportServer::new(raft_router), + auth, )) .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async move { let _ = shutdown_rx.changed().await; diff --git a/crates/metadb-server/tests/grpc_regressions.rs b/crates/metadb-server/tests/grpc_regressions.rs index ac49845..defe5dd 100644 --- a/crates/metadb-server/tests/grpc_regressions.rs +++ b/crates/metadb-server/tests/grpc_regressions.rs @@ -182,7 +182,12 @@ async fn unsupported_cell_registration_does_not_report_success() { .unwrap(); let result = client .register_cell(metadb_proto::RegisterCellRequest::default()) - .await; + .await + .unwrap() + .into_inner(); cluster.shutdown().await.unwrap(); - assert_eq!(result.unwrap_err().code(), tonic::Code::Unimplemented); + assert!( + result.error.is_some(), + "RegisterCell must fail closed instead of reporting success" + ); } diff --git a/crates/metadb-server/tests/input_boundary_regressions.rs b/crates/metadb-server/tests/input_boundary_regressions.rs index a7dcad0..6ffb15d 100644 --- a/crates/metadb-server/tests/input_boundary_regressions.rs +++ b/crates/metadb-server/tests/input_boundary_regressions.rs @@ -13,7 +13,8 @@ fn wire_identifier_lengths_and_key_prefixes_are_checked() { assert_eq!(decode_optional_u128_be(&bytes).is_ok(), n == 0 || n == 16); assert_eq!(ShardEpoch::decode(&bytes).is_ok(), n == 16); } - for prefix in [0x00, 0x08, 0xff] { + // 0x08 is the quota-origin cursor and stays on its origin shard (`Ok(None)`). + for prefix in [0x00, 0x09, 0xff] { let mut raw = vec![0; 9]; raw[0] = prefix; assert!(MetaKey::decode(&raw).is_err()); diff --git a/crates/metadb-server/tests/join_existing_group.rs b/crates/metadb-server/tests/join_existing_group.rs index 8cf1fea..cf1faed 100644 --- a/crates/metadb-server/tests/join_existing_group.rs +++ b/crates/metadb-server/tests/join_existing_group.rs @@ -225,7 +225,14 @@ async fn second_process_joins_existing_raft_group_fails_over_and_reopens() { Arc::clone(&node2) }; leader.shutdown().await.unwrap(); + drop(survivors); drop(leader); + // The exclusive allocator lock lives until this process drops the old server. + if leader_id == 2 { + drop(node2); + } else { + drop(node3); + } let (reopened, reopened_addr) = PersistentMetaDbCluster::open_and_serve(leader_dir_cfg) .await diff --git a/crates/metadb-tx/src/aux_outbox.rs b/crates/metadb-tx/src/aux_outbox.rs index 3e7da25..42f0fce 100644 --- a/crates/metadb-tx/src/aux_outbox.rs +++ b/crates/metadb-tx/src/aux_outbox.rs @@ -100,16 +100,12 @@ impl ReplicatedAttrAggregator { && d.raft_index <= cut.max_raft_index && d.hlc <= cut.max_hlc { - base.num_children = saturating_add_i64( - base.num_children, - clamp_delta_to_i64(d.delta_children), - ) - .max(0); - base.size_bytes = saturating_add_i64( - base.size_bytes, - clamp_delta_to_i64(d.delta_bytes), - ) - .max(0); + base.num_children = + saturating_add_i64(base.num_children, clamp_delta_to_i64(d.delta_children)) + .max(0); + base.size_bytes = + saturating_add_i64(base.size_bytes, clamp_delta_to_i64(d.delta_bytes)) + .max(0); if d.hlc > base.modify_time { base.modify_time = d.hlc; } @@ -136,16 +132,12 @@ impl ReplicatedAttrAggregator { && d.raft_index <= cut.max_raft_index && d.hlc <= cut.max_hlc { - base.num_children = saturating_add_i64( - base.num_children, - clamp_delta_to_i64(d.delta_children), - ) - .max(0); - base.size_bytes = saturating_add_i64( - base.size_bytes, - clamp_delta_to_i64(d.delta_bytes), - ) - .max(0); + base.num_children = + saturating_add_i64(base.num_children, clamp_delta_to_i64(d.delta_children)) + .max(0); + base.size_bytes = + saturating_add_i64(base.size_bytes, clamp_delta_to_i64(d.delta_bytes)) + .max(0); if d.hlc > base.modify_time { base.modify_time = d.hlc; } @@ -204,18 +196,15 @@ impl ReplicatedAttrAggregator { let mut changed = false; let baseline = attr.applied_lsn; for d in queue { - if d.raft_index > attr.applied_lsn && d.raft_index <= sealed_raft_index { - attr.num_children = - saturating_add_i64( - attr.num_children, - clamp_delta_to_i64(d.delta_children), - ) - .max(0); - attr.size_bytes = saturating_add_i64( - attr.size_bytes, - clamp_delta_to_i64(d.delta_bytes), + if d.raft_index > baseline && d.raft_index <= sealed_raft_index { + attr.num_children = saturating_add_i64( + attr.num_children, + clamp_delta_to_i64(d.delta_children), ) .max(0); + attr.size_bytes = + saturating_add_i64(attr.size_bytes, clamp_delta_to_i64(d.delta_bytes)) + .max(0); if d.hlc > attr.modify_time { attr.modify_time = d.hlc; } @@ -236,18 +225,15 @@ impl ReplicatedAttrAggregator { let mut changed = false; let baseline = sub.applied_raft_index; for d in queue { - if d.raft_index > sub.applied_raft_index && d.raft_index <= sealed_raft_index { - sub.num_children = - saturating_add_i64( - sub.num_children, - clamp_delta_to_i64(d.delta_children), - ) - .max(0); - sub.size_bytes = saturating_add_i64( - sub.size_bytes, - clamp_delta_to_i64(d.delta_bytes), + if d.raft_index > baseline && d.raft_index <= sealed_raft_index { + sub.num_children = saturating_add_i64( + sub.num_children, + clamp_delta_to_i64(d.delta_children), ) .max(0); + sub.size_bytes = + saturating_add_i64(sub.size_bytes, clamp_delta_to_i64(d.delta_bytes)) + .max(0); if d.hlc > sub.modify_time { sub.modify_time = d.hlc; } diff --git a/crates/metadb-tx/src/aux_sink.rs b/crates/metadb-tx/src/aux_sink.rs index 680d518..2d88cbf 100644 --- a/crates/metadb-tx/src/aux_sink.rs +++ b/crates/metadb-tx/src/aux_sink.rs @@ -2,8 +2,8 @@ use std::collections::{BTreeMap, VecDeque}; use std::sync::RwLock; use metadb_core::{ - saturating_add_i64, FileId, HlcTimestamp, InodeId, MetaDbError, QuotaRecord, Result, - SecondaryIndexRecord, ShardId, UserId, + saturating_add_i64, FileId, HlcTimestamp, InodeId, QuotaRecord, Result, SecondaryIndexRecord, + ShardId, UserId, }; use metadb_storage::{LifecycleStorageEngine, LogSequenceNumber, RaftLogPayload}; diff --git a/crates/metadb-tx/src/state_machine_impl.rs b/crates/metadb-tx/src/state_machine_impl.rs index d62de52..0ba93f0 100644 --- a/crates/metadb-tx/src/state_machine_impl.rs +++ b/crates/metadb-tx/src/state_machine_impl.rs @@ -1359,9 +1359,24 @@ impl TxStateMachineHandler { )? { continue; // Exact-once duplicate replay suppressed. } - rec.used_bytes = saturating_add_i64(rec.used_bytes, delta_bytes).max(0); - rec.used_inodes = - saturating_add_i64(rec.used_inodes, delta_inodes).max(0); + // Keep the exact sum, including a temporary negative, so a + // later delta from another origin can cancel it. Overflow + // rejects the whole batch before anything in it is written. + // Readers clamp a negative balance to zero. + let new_bytes = + rec.used_bytes.checked_add(delta_bytes).ok_or_else(|| { + MetaDbError::CodecError(format!( + "quota byte delta overflow for uid {uid}" + )) + })?; + let new_inodes = + rec.used_inodes.checked_add(delta_inodes).ok_or_else(|| { + MetaDbError::CodecError(format!( + "quota inode delta overflow for uid {uid}" + )) + })?; + rec.used_bytes = new_bytes; + rec.used_inodes = new_inodes; rec.last_applied_hlc = rec.last_applied_hlc.max(item.hlc); rec.applied_by_shard .insert(item.origin_shard_id, cur_cursor); diff --git a/crates/metadb-tx/src/two_pc.rs b/crates/metadb-tx/src/two_pc.rs index 5ddc62a..814f579 100644 --- a/crates/metadb-tx/src/two_pc.rs +++ b/crates/metadb-tx/src/two_pc.rs @@ -25,10 +25,19 @@ use crate::state_machine_impl::{SetAttrPatch, TxRaftCmd, TxRaftOutput, TxStateMa pub const DEFAULT_2PC_TTL_MS: u64 = 3_000; +fn publish_quota(mut record: QuotaRecord) -> QuotaRecord { + record.used_bytes = record.used_bytes.max(0); + record.used_inodes = record.used_inodes.max(0); + record +} + // A split fences its parent before booting and publishing its child. That // interval includes Raft recovery and can outlast a short count-based retry loop. const ROUTE_RETRY_TIMEOUT: Duration = Duration::from_secs(30); const ROUTE_RETRY_INTERVAL: Duration = Duration::from_millis(25); +/// A follower that already knows another leader must return `NotLeader` at once. +/// Only an election with no hint is waited out, and only briefly. +const ELECTION_RETRY_TIMEOUT: Duration = Duration::from_secs(2); async fn retry_routing(operation: F) -> Result where @@ -47,12 +56,27 @@ where F: FnMut() -> Fut, Fut: Future>, { + let election_deadline = tokio::time::Instant::now() + ELECTION_RETRY_TIMEOUT; loop { match operation().await { + // Split publication is local and can take the full routing budget. + // A known leader cannot appear by retrying on this follower. + Err( + error @ MetaDbError::NotLeader { + leader_hint: Some(_), + .. + }, + ) => return Err(error), + Err(error @ MetaDbError::NotLeader { .. }) => { + let now = tokio::time::Instant::now(); + let limit = deadline.min(election_deadline); + if now >= limit { + return Err(error); + } + tokio::time::sleep(ROUTE_RETRY_INTERVAL.min(limit - now)).await; + } Err( - error @ (MetaDbError::EpochMismatch { .. } - | MetaDbError::NotLeader { .. } - | MetaDbError::ShardRoutingError(_)), + error @ (MetaDbError::EpochMismatch { .. } | MetaDbError::ShardRoutingError(_)), ) => { let now = tokio::time::Instant::now(); if now >= deadline { @@ -1430,7 +1454,7 @@ impl DistributedTxRouter { let Some(raw) = group.engine().get_with_snapshot(&qk, &snap)? else { return Ok(None); }; - Ok(Some(QuotaRecord::decode(&raw)?)) + Ok(Some(publish_quota(QuotaRecord::decode(&raw)?))) } pub async fn list_quotas(&self, start_uid: UserId, limit: usize) -> Result> { @@ -1453,7 +1477,7 @@ impl DistributedTxRouter { let (_, snap) = group.linearizable_read(epoch).await?; Self::validate_read_epoch(&group, epoch)?; for (_, v) in group.engine().scan_with_snapshot(s, e, &snap)? { - out.push(QuotaRecord::decode(&v)?); + out.push(publish_quota(QuotaRecord::decode(&v)?)); if out.len() >= lim { return Ok(out); } diff --git a/crates/metadb-tx/src/validation.rs b/crates/metadb-tx/src/validation.rs index d6026ca..2d0d5c1 100644 --- a/crates/metadb-tx/src/validation.rs +++ b/crates/metadb-tx/src/validation.rs @@ -2,9 +2,7 @@ use metadb_core::{is_dir_inode_id, InodeId, MetaDbError, Result, ROOT_INODE_ID}; pub(crate) fn validate_size(size: i64) -> Result<()> { if size < 0 { - return Err(MetaDbError::CodecError( - "size_bytes must be nonnegative".into(), - )); + return Err(MetaDbError::InvalidFileSize(size)); } Ok(()) } diff --git a/docs/de/architecture.md b/docs/de/architecture.md index 915020a..97b1962 100644 --- a/docs/de/architecture.md +++ b/docs/de/architecture.md @@ -15,7 +15,7 @@ Diese Seite folgt dem Code in `crates/`. Sie beschreibt keinen Mehrprozess-Zelle 5. Öffnet oder bootstrapt `DurableCellManager` unter `/cell_`. 6. Registriert diese eine Zelle bei `DurableGlobalManager` für den Schlüsselbereich `[0x01, 0x04)`. 7. Stellt sicher, dass Wurzel-Inode `1` (`ROOT_INODE_ID`) einen Attributsatz hat. -8. Bedient vier gRPC-Dienste auf demselben Listener. Metadaten-, Transaktions- und Control-Plane-Dienste nehmen einen optionalen Bearer-Interceptor. `RaftTransportService` nimmt keinen. +8. Bedient vier gRPC-Dienste auf demselben Listener. Alle vier nehmen denselben optionalen Bearer-Interceptor. Ein leeres Geheimnis lässt jeden Aufruf zu. 9. Startet eine Wartungsaufgabe im Abstand von 250 ms. `metadbd` ist eine dünne Hülle: Es lädt TOML, wendet CLI-Überschreibungen an, ruft `open_and_serve` auf, ruft optional `RegisterNode` an `--cell-manager` auf, liefert optional Prometheus-Text und ruft bei Strg-C `shutdown` auf. diff --git a/docs/de/build-and-run.md b/docs/de/build-and-run.md index 6b9a3a9..c035204 100644 --- a/docs/de/build-and-run.md +++ b/docs/de/build-and-run.md @@ -64,7 +64,7 @@ cargo run -p metadb-server --bin metadbd -- --config ./standalone.toml Logs verwenden den `EnvFilter` von `tracing-subscriber`. Der Standardfilter ist `info`. Überschreiben mit der üblichen Variable `RUST_LOG`, zum Beispiel `RUST_LOG=debug`. -Beim Start mit leerem `security.shared_secret` schreibt der Prozess eine Warnung, dass gRPC keine Authentifizierung und kein TLS hat. Die Standardbindung ist `127.0.0.1`. Diesen Socket nicht in ein nicht vertrauenswürdiges Netz stellen. `[security].shared_secret` authentifiziert nur Metadaten-, Transaktions- und Control-Plane-RPCs. `RaftTransportService` auf demselben Port hat keinen Interceptor, ein Bearer-Token deckt ihn also nicht ab, und TLS vor dem Listener zu beenden fügt diese Prüfung nicht hinzu. Den Listener in einem vertrauenswürdigen Netz lassen, oder jeden Dienst einschließlich Raft hinter einen Proxy setzen, der Clients authentifiziert, und den direkten Zugriff auf diesen Prozess sperren. +Beim Start mit leerem `security.shared_secret` schreibt der Prozess eine Warnung, dass gRPC keine Authentifizierung und kein TLS hat. Die Standardbindung ist `127.0.0.1`. Diesen Socket nicht in ein nicht vertrauenswürdiges Netz stellen. Ein nicht leeres `[security].shared_secret` verlangt denselben Bearer für Metadaten-, Transaktions-, Control-Plane- und Raft-Transport-RPCs. Der Prozess beendet TLS weiterhin nicht. Den Listener in einem vertrauenswürdigen Netz lassen, oder jeden Dienst hinter einen Proxy setzen, der Clients authentifiziert, und den direkten Zugriff auf diesen Prozess sperren. Strg-C schreibt den Shard-Zustand und stoppt die OpenRaft-Gruppen (`cluster.shutdown()`). diff --git a/docs/de/configuration.md b/docs/de/configuration.md index 5f583ad..5e6533e 100644 --- a/docs/de/configuration.md +++ b/docs/de/configuration.md @@ -71,9 +71,9 @@ Der Prozess beendet TLS nicht. Wenn `shared_secret` nicht leer ist, verlangen `MetadataService`, `TransactionService` und `ControlPlaneService`, dass der Metadatenwert `authorization` gleich `Bearer ` ist. `metadb-ctl --auth-token` (Umgebungsvariable `METADB_AUTH_TOKEN`) sendet diesen Header. `metadbd --cell-manager` sendet dasselbe Geheimnis, wenn es `RegisterNode` aufruft. -`RaftTransportService` wird ohne den Interceptor registriert. Ein Bearer-Token authentifiziert Raft-RPCs auf diesem Port nicht. +`RaftTransportService` verwendet denselben Interceptor. Ein nicht leeres Geheimnis verlangt auch für Raft-RPCs auf diesem Port `Bearer `. -Ein leeres Geheimnis lässt Metadaten-, Transaktions- und Control-Plane-RPCs zu. Der Code warnt beim Start davor. Ein Bearer-Geheimnis deckt `RaftTransportService` nicht ab, und ein TLS-Terminator vor dem Listener fügt diese Prüfung nicht hinzu: nach der Terminierung bleiben Raft-RPCs unauthentifiziert. Bevor ein nicht vertrauenswürdiger Client den Port erreichen kann, die Bindung auf einem vertrauenswürdigen Netz wie `127.0.0.1` lassen, oder jeden Dienst hinter einen Proxy setzen, der den Client authentifiziert (zum Beispiel gegenseitiges TLS), und den direkten Zugriff auf diesen Prozess sperren. +Ein leeres Geheimnis lässt alle vier Dienste auf dem Listener zu. Der Code warnt beim Start davor. Der Prozess beendet TLS weiterhin nicht. Bevor ein nicht vertrauenswürdiger Client den Port erreichen kann, die Bindung auf einem vertrauenswürdigen Netz wie `127.0.0.1` lassen, oder jeden Dienst hinter einen Proxy setzen, der den Client authentifiziert (zum Beispiel gegenseitiges TLS), und den direkten Zugriff auf diesen Prozess sperren. ## Was sich nicht konfigurieren lässt diff --git a/docs/de/operations.md b/docs/de/operations.md index 1edabf7..37ace8c 100644 --- a/docs/de/operations.md +++ b/docs/de/operations.md @@ -135,9 +135,9 @@ Die Varianten sind in `crates/metadb-core/src/error.rs` definiert. gRPC-Methoden - `GetRoutes` scheitert geschlossen bei einer unbekannten `cell_id` ungleich null. `StoreHeartbeat` ersetzt zwischengespeicherte Routen nicht. - Jumbo-Isolation ist ein Routenflag plus eine Knotenkennung aus der lokalen Knotentabelle. Sie verdrängt andere Shards nicht von einer Platte. - Heartbeats sind ein RPC ohne Sender in `metadbd`. `StoreHeartbeat` speichert nicht dauerhaft. -- Raft auf dem öffentlichen gRPC-Port ist unauthentifiziert. Der Prozess beendet TLS nicht. +- Ein nicht leeres gemeinsames Geheimnis authentifiziert Raft auf dem öffentlichen gRPC-Port mit demselben Bearer wie die drei anderen Dienste. Ein leeres Geheimnis lässt Raft unauthentifiziert. Der Prozess beendet TLS nicht. - `memtable_size_bytes` und alle Raft-Zeitgeberfelder ändern das Laufzeitverhalten über TOML nicht. -- `metadb-ctl bench` meldet Durchsatz für Versuche. `unwrap` fängt nur Transport- oder Statusfehler. Anwendungsfehler wie `AlreadyExists` kommen in einer OK-Antwort zurück und werden verworfen, auch bei unlink, daher kann ein zweiter Lauf weiter einen Durchsatz drucken. Es prüft nach dem Lauf weder Auflistung noch dirstat noch Quota und druckt keine Latenzperzentile. +- `metadb-ctl bench` für `hotspot-create` und `bulk-ttl-delete` druckt JSON-Zeilen: einen Kopf mit `bench_pid`, dann Phasenberichte mit attempted, succeeded, failed und Latenzperzentilen. Der Prozess endet mit einem Fehler, wenn nicht jede angeforderte Operation gelingt. `--workload mixed` oder `--embedded` startet den prozessinternen Stress-Harness. `sql` druckt die Antwort als JSON und endet bei einem Anwendungsfehler mit einem Fehlerstatus. ## Dokumentationslücken diff --git a/docs/de/testing.md b/docs/de/testing.md index bef93e6..21ae0c0 100644 --- a/docs/de/testing.md +++ b/docs/de/testing.md @@ -99,16 +99,15 @@ cargo run -p metadb-server --bin metadb-ctl -- \ bench --workload bulk-ttl-delete --concurrency 16 --ops 1000 ``` -Flags: `--workload` ist `hotspot-create` (die Vorgabe) oder `bulk-ttl-delete`. `--concurrency` ist Vorgabe 16. `--ops` ist Vorgabe 1000. Jede andere Workload-Zeichenkette ist ein Fehler. +Flags: `--workload` ist `hotspot-create` (die Vorgabe), `bulk-ttl-delete` oder `mixed`. `--concurrency` ist 16, wenn es fehlt. `--ops` ist Vorgabe 1000. `--embedded` oder `--workload mixed` startet einen temporären All-in-one-Knoten und den prozessinternen Stress-Harness, statt `--endpoint` zu wählen. -Verhalten aus `crates/metadb-server/src/bin/metadb-ctl.rs`: +Verhalten aus `crates/metadb-server/src/bin/metadb-ctl.rs` für die entfernten Workloads: -- Jede Aufgabe läuft `ops / concurrency` Iterationen, mindestens eine. Der Divisionsrest entfällt, daher kann die gedruckte Zahl kleiner als `--ops` sein. Ist `--ops` kleiner als `--concurrency`, kann die Zahl größer als `--ops` sein, weil jede Aufgabe trotzdem einmal läuft. Die gedruckte Zahl ist die Zahl der Versuche, nicht die Zahl angenommener Mutationen. -- `hotspot-create` legt per mkdir Inode `999000` namens `bench_hotspot_create` unter Eltern-Inode `1` an und ruft dann `CreateFile` für Dateien von 4096 Bytes auf. Das mkdir-`Result` wird verworfen. `unwrap` gilt nur für das tonic-`Result`, daher beendet ein Transport- oder Statusfehler den Prozess. `CreateFile` liefert Anwendungsfehler wie `AlreadyExists` in einem OK-`CreateFileResponse.error`, und der Client verwirft dieses Feld. -- `bulk-ttl-delete` tut dieselben Anlegevorgänge unter Inode `999001` (`bench_bulk_ttl_delete`), ruft `Unlink` auf dieselbe Weise auf (Transport-`unwrap`, Anwendungs-`error` ignoriert) und dann ein `ListDir` mit `limit = 1000`. -- `hotspot-create` druckt `Completed single-directory hotspot creates in s ( ops/s)` nach den Versuchen, auch nach Versuchen, deren Anwendungsfehler ignoriert wurde. -- `bulk-ttl-delete` druckt die Zahl der Unlink-Versuche, Sekunden, ops/s, die von dieser Liste gelieferte Zahl lebender Einträge und die Listendauer in Millisekunden. Der Listenaufruf wird geprüft; die Unlinks nicht. -- Der Client berechnet keine Latenzperzentile und prüft weder dirstat noch Quota. Ein zweiter Lauf gegen denselben Server kann weiter `Completed` und einen Durchsatz drucken, wenn jede Anlage mit `AlreadyExists` abgewiesen wird. +- Der Client belegt eine Verzeichnis-ID und dann Datei-Inode- und Datei-ID-Blöcke der Länge `--ops`. Eine Zuteilung mit falscher Anzahl oder Überlauf wird abgelehnt. +- Er druckt ein JSON-Objekt mit `bench_pid`, `workload` und `requested_ops` und legt danach so viele 4096-Byte-Dateien in einem neuen Verzeichnis an. +- Jede Phase druckt JSON mit `attempted`, `succeeded`, `failed`, `transport_failed`, `application_failed`, `errors_by_kind` und Latenzperzentilen. Der Prozess endet mit einem Fehler, wenn `succeeded` nicht `--ops` ist. +- `bulk-ttl-delete` entfernt die angelegten Namen und listet das Verzeichnis mit `limit = 1`. +- Ein zweiter Lauf belegt eine neue `bench_pid`. Abgewiesene Anlegen gelten nicht als Erfolg. Eine Zahl aus diesem Befehl nicht als Repository-Ergebnis behandeln. Sie hängt von der Maschine, dem Build-Profil und dem angesprochenen Server ab. Dieser Baum zeichnet keine Stichprobe auf. diff --git a/docs/en/architecture.md b/docs/en/architecture.md index bedcaa2..5ae865f 100644 --- a/docs/en/architecture.md +++ b/docs/en/architecture.md @@ -15,7 +15,7 @@ This page follows the code in `crates/`. It does not describe a multi-process Ce 5. Opens or bootstraps `DurableCellManager` at `/cell_`. 6. Registers that one cell on `DurableGlobalManager` for the key range `[0x01, 0x04)`. 7. Ensures root directory inode `1` (`ROOT_INODE_ID`) has an attribute record. -8. Serves four gRPC services on the same listener. Metadata, transaction, and control-plane services take an optional bearer interceptor. `RaftTransportService` does not. +8. Serves four gRPC services on the same listener. All four take the same optional bearer interceptor. An empty secret allows every call. 9. Spawns a maintenance task on a 250 ms interval. `metadbd` is a thin wrapper: it loads TOML, applies CLI overrides, calls `open_and_serve`, optionally calls `RegisterNode` on `--cell-manager`, optionally serves Prometheus text, and on Ctrl-C calls `shutdown`. diff --git a/docs/en/build-and-run.md b/docs/en/build-and-run.md index 431a8dc..911255d 100644 --- a/docs/en/build-and-run.md +++ b/docs/en/build-and-run.md @@ -64,7 +64,7 @@ cargo run -p metadb-server --bin metadbd -- --config ./standalone.toml Logs use `tracing-subscriber`'s `EnvFilter`. The default filter is `info`. Override it with the standard `RUST_LOG` variable, for example `RUST_LOG=debug`. -On startup with an empty `security.shared_secret`, the process logs a warning that gRPC has no authentication and no TLS. The default bind is `127.0.0.1`. Do not publish that socket on an untrusted network. `[security].shared_secret` authenticates metadata, transaction, and control-plane RPCs only. `RaftTransportService` on the same port has no interceptor, so a bearer token does not cover it, and terminating TLS in front of the listener does not add that check. Keep the listener on a trusted network, or put every service, including Raft, behind a proxy that authenticates clients and block direct access to this process. +On startup with an empty `security.shared_secret`, the process logs a warning that gRPC has no authentication and no TLS. The default bind is `127.0.0.1`. Do not publish that socket on an untrusted network. A non-empty `[security].shared_secret` requires the same bearer on metadata, transaction, control-plane, and Raft transport RPCs. The process still does not terminate TLS. Keep the listener on a trusted network, or put every service behind a proxy that authenticates clients and block direct access to this process. Ctrl-C flushes shard state and stops the OpenRaft groups (`cluster.shutdown()`). diff --git a/docs/en/configuration.md b/docs/en/configuration.md index 0a5f2e8..42f3cb9 100644 --- a/docs/en/configuration.md +++ b/docs/en/configuration.md @@ -23,7 +23,7 @@ | `storage.locality_tombstone_ratio` | `0.50` | Yes. Multiplied by 1000, clamped to 1..=1000, stored as `tombstone_min_ratio_permille` | | `sharding.bootstrap_table_shards` | `true` | Yes. `true` creates shards 1, 2, and 3. `false` creates shard 1 over `[0x01, 0x04)` | | `sharding.shard_split_threshold_bytes` | `4294967296` (4 GiB) | Yes. Split check uses approximate shard size | -| `security.shared_secret` | `""` | Yes. Empty means metadata, transaction, and control-plane RPCs are unauthenticated | +| `security.shared_secret` | `""` | Yes. Empty means all four gRPC services on this listener are unauthenticated | There is no `[raft]` table. `ServerTomlConfig.raft_options` is skipped by serde and stays at `RaftNodeOptions::default()` (heartbeat 150 ms, election 450–900 ms, snapshot log keep 0, segment 64 MiB). @@ -69,11 +69,9 @@ CLI flags override the file after it is parsed: `--node-id`, `--cell-id`, `--rol The process does not terminate TLS. -When `shared_secret` is non-empty, `MetadataService`, `TransactionService`, and `ControlPlaneService` require `authorization` to equal `Bearer ` exactly. Clients use a separate interceptor that attaches that header. `metadb-ctl --auth-token` (environment variable `METADB_AUTH_TOKEN`) and `metadbd --join` / `--cell-manager` send it. A prefix or a longer token is rejected. An empty secret still means those three services are unauthenticated. +When `shared_secret` is non-empty, `MetadataService`, `TransactionService`, `ControlPlaneService`, and `RaftTransportService` require `authorization` to equal `Bearer ` exactly. Raft peers send the same bearer. `metadb-ctl --auth-token` (environment variable `METADB_AUTH_TOKEN`) and `metadbd --join` / `--cell-manager` send it. A prefix or a longer token is rejected. An empty secret leaves all four services unauthenticated. -`RaftTransportService` is registered without the interceptor. A bearer token does not authenticate Raft RPCs on that port. - -An empty secret allows metadata, transaction, and control-plane RPCs. The code warns about this at startup. A bearer secret does not cover `RaftTransportService`, and a TLS terminator in front of the listener does not add that check: after termination, Raft RPCs are still unauthenticated. Before an untrusted client can reach the port, keep the bind on a trusted network such as `127.0.0.1`, or put every service behind a proxy that authenticates the client (for example mutual TLS) and block direct access to this process. +An empty secret allows every RPC on the listener. The code warns about this at startup. The process still does not terminate TLS. Before an untrusted client can reach the port, keep the bind on a trusted network such as `127.0.0.1`, or put every service behind a proxy that authenticates the client (for example mutual TLS) and block direct access to this process. ## What you cannot configure diff --git a/docs/en/operations.md b/docs/en/operations.md index 5760cb1..ec33a9c 100644 --- a/docs/en/operations.md +++ b/docs/en/operations.md @@ -138,9 +138,9 @@ Variants are defined in `crates/metadb-core/src/error.rs`. gRPC methods usually - Quota cursor rows are written after the outbox lock is released, so a flush that already holds the memtable lock can still take the outbox lock. - Jumbo isolation is a route flag plus a node id chosen from the local node map. It does not evict other shards from a disk. - Heartbeats are an RPC with no sender in `metadbd`. `StoreHeartbeat` does not persist. -- Raft on the public gRPC port is unauthenticated. The process does not terminate TLS. +- A non-empty shared secret authenticates Raft on the public gRPC port with the same bearer as the other three services. An empty secret leaves Raft unauthenticated. The process does not terminate TLS. - `memtable_size_bytes` and all Raft timer fields do not change runtime behavior through TOML. -- `metadb-ctl bench` reports throughput for attempts. `unwrap` catches transport or status failures only. Application errors such as `AlreadyExists` come back inside an OK response and are discarded, including on unlink, so a repeat run can still print a throughput. It does not check listing, dirstat, or quota after the run, and it does not print latency percentiles. +- `metadb-ctl bench` for `hotspot-create` and `bulk-ttl-delete` prints JSON lines: a header with `bench_pid`, then phase reports with attempted, succeeded, failed, and latency percentiles. The process exits non-zero unless every requested operation succeeds. `--workload mixed` or `--embedded` runs the in-process stress harness. `sql` prints the response as JSON and exits non-zero on an application error. ## Documentation gaps diff --git a/docs/en/testing.md b/docs/en/testing.md index 1a9477f..5f0f17d 100644 --- a/docs/en/testing.md +++ b/docs/en/testing.md @@ -99,16 +99,15 @@ cargo run -p metadb-server --bin metadb-ctl -- \ bench --workload bulk-ttl-delete --concurrency 16 --ops 1000 ``` -Flags: `--workload` is `hotspot-create` (the default) or `bulk-ttl-delete`. `--concurrency` defaults to 16. `--ops` defaults to 1000. Any other workload string is an error. +Flags: `--workload` is `hotspot-create` (the default), `bulk-ttl-delete`, or `mixed`. `--concurrency` defaults to 16 when omitted. `--ops` defaults to 1000. `--embedded` or `--workload mixed` starts a temporary all-in-one node and runs the in-process stress harness instead of dialing `--endpoint`. -Behavior from `crates/metadb-server/src/bin/metadb-ctl.rs`: +Behavior from `crates/metadb-server/src/bin/metadb-ctl.rs` for the remote workloads: -- Each task runs `ops / concurrency` iterations, and at least one. The remainder of the division is dropped, so the printed count can be smaller than `--ops`. If `--ops` is smaller than `--concurrency`, the count can be larger than `--ops` because each task still runs once. The printed count is the number of attempts, not the number of accepted mutations. -- `hotspot-create` mkdirs inode `999000` named `bench_hotspot_create` under parent inode `1`, then calls `CreateFile` for 4096-byte files. The mkdir `Result` is discarded. `unwrap` runs on the tonic `Result` only, so a transport or status failure aborts the process. `CreateFile` returns application failures such as `AlreadyExists` inside an OK `CreateFileResponse.error`, and the client drops that field. -- `bulk-ttl-delete` does the same creates under inode `999001` (`bench_bulk_ttl_delete`), then calls `Unlink` the same way (transport `unwrap`, application `error` ignored), then one `ListDir` with `limit = 1000`. -- `hotspot-create` prints `Completed single-directory hotspot creates in s ( ops/s)` after the attempts, including attempts whose application error was ignored. -- `bulk-ttl-delete` prints the unlink attempt count, seconds, ops/s, the live entry count returned by that list, and the list duration in milliseconds. The list call is checked; the unlinks are not. -- The client does not compute latency percentiles and does not check dirstat or quota. A second run against the same server can still print `Completed` and a throughput when every create is rejected with `AlreadyExists`. +- The client allocates one directory id, then file-inode and file-id blocks of length `--ops`, and refuses a grant whose count or end overflows. +- It prints one JSON object with `bench_pid`, `workload`, and `requested_ops`, then creates that many 4096-byte files under a new directory. +- Each phase prints JSON with `attempted`, `succeeded`, `failed`, `transport_failed`, `application_failed`, `errors_by_kind`, and latency percentiles (`p50_ms`, `p99_ms`, `p999_ms`). The process exits non-zero unless `succeeded` equals `--ops`. +- `bulk-ttl-delete` unlinks the names that were created, then lists the directory with `limit = 1` and prints whether that page is empty. +- A second run allocates a new `bench_pid`. It does not reuse a fixed inode, and it does not report rejected creates as successes. Do not treat a number from this command as a repository result. It depends on the machine, the build profile, and the server you pointed at. This tree does not record a sample. @@ -123,4 +122,4 @@ These commands are in this tree. CI runs them with `cargo test --workspace --loc - `cargo test -p metadb-server --test join_existing_group` starts a second `metadbd` and joins the first process's Raft group, including a shared-secret case. - `open_and_serve` still rejects every role except `all-in-one`. A second Cell stays unimplemented. `RegisterCell` returns an error. -`stress_smoke_mixed_workload` runs in that same workspace test. It prints one line for the machine that ran it. This repository does not check in a service-level target, and `metadb-ctl bench` still does not record percentiles or application errors for `hotspot-create`. +`stress_smoke_mixed_workload` runs in that same workspace test. It prints one line for the machine that ran it. This repository does not check in a service-level target. Remote `metadb-ctl bench` records attempt counts and latency percentiles in JSON and fails the process when an application or transport error occurs. diff --git a/docs/fr/architecture.md b/docs/fr/architecture.md index 3ed95f4..8526b96 100644 --- a/docs/fr/architecture.md +++ b/docs/fr/architecture.md @@ -15,7 +15,7 @@ Cette page suit le code de `crates/`. Elle ne décrit pas un déploiement de cel 5. Ouvre ou amorce `DurableCellManager` dans `/cell_`. 6. Enregistre cette unique cellule sur `DurableGlobalManager` pour l'intervalle de clés `[0x01, 0x04)`. 7. Garantit que l'inode racine `1` (`ROOT_INODE_ID`) possède un enregistrement d'attributs. -8. Sert quatre services gRPC sur le même écouteur. Les services de métadonnées, de transaction et de plan de contrôle prennent un intercepteur bearer facultatif. `RaftTransportService` n'en prend pas. +8. Sert quatre services gRPC sur le même écouteur. Les quatre prennent le même intercepteur bearer facultatif. Un secret vide autorise chaque appel. 9. Lance une tâche de maintenance toutes les 250 ms. `metadbd` est une fine enveloppe : il charge le TOML, applique les remplacements CLI, appelle `open_and_serve`, appelle éventuellement `RegisterNode` sur `--cell-manager`, sert éventuellement le texte Prometheus, et sur Ctrl-C appelle `shutdown`. diff --git a/docs/fr/build-and-run.md b/docs/fr/build-and-run.md index 2c15593..93c149f 100644 --- a/docs/fr/build-and-run.md +++ b/docs/fr/build-and-run.md @@ -64,7 +64,7 @@ cargo run -p metadb-server --bin metadbd -- --config ./standalone.toml Les journaux utilisent l'`EnvFilter` de `tracing-subscriber`. Le filtre par défaut est `info`. On le remplace avec la variable standard `RUST_LOG`, par exemple `RUST_LOG=debug`. -Au démarrage, si `security.shared_secret` est vide, le processus journalise un avertissement : gRPC n'a ni authentification ni TLS. La liaison par défaut est `127.0.0.1`. Ne pas publier cette socket sur un réseau non fiable. `[security].shared_secret` n'authentifie que les RPC de métadonnées, de transaction et de plan de contrôle. `RaftTransportService` sur le même port n'a pas d'intercepteur : un jeton bearer ne le couvre pas, et terminer TLS devant l'écouteur n'ajoute pas ce contrôle. Garder l'écouteur sur un réseau de confiance, ou placer chaque service, Raft compris, derrière un proxy qui authentifie les clients, et bloquer l'accès direct à ce processus. +Au démarrage, si `security.shared_secret` est vide, le processus journalise un avertissement : gRPC n'a ni authentification ni TLS. La liaison par défaut est `127.0.0.1`. Ne pas publier cette socket sur un réseau non fiable. Un `[security].shared_secret` non vide exige le même bearer pour les RPC de métadonnées, de transaction, de plan de contrôle et de transport Raft. Le processus ne termine toujours pas TLS. Garder l'écouteur sur un réseau de confiance, ou placer chaque service derrière un proxy qui authentifie les clients, et bloquer l'accès direct à ce processus. Ctrl-C vide l'état des shards et arrête les groupes OpenRaft (`cluster.shutdown()`). diff --git a/docs/fr/configuration.md b/docs/fr/configuration.md index 405fab7..68c6a7f 100644 --- a/docs/fr/configuration.md +++ b/docs/fr/configuration.md @@ -71,9 +71,9 @@ Le processus ne termine pas TLS. Lorsque `shared_secret` n'est pas vide, `MetadataService`, `TransactionService` et `ControlPlaneService` exigent que la valeur de métadonnée `authorization` soit égale à `Bearer `. `metadb-ctl --auth-token` (variable d'environnement `METADB_AUTH_TOKEN`) envoie cet en-tête. `metadbd --cell-manager` envoie le même secret lorsqu'il appelle `RegisterNode`. -`RaftTransportService` est enregistré sans l'intercepteur. Un jeton bearer n'authentifie pas les RPC Raft sur ce port. +`RaftTransportService` utilise le même intercepteur. Un secret non vide exige aussi `Bearer ` pour les RPC Raft sur ce port. -Un secret vide autorise les RPC de métadonnées, de transaction et de plan de contrôle. Le code avertit de cela au démarrage. Un secret bearer ne couvre pas `RaftTransportService`, et un terminateur TLS devant l'écouteur n'ajoute pas ce contrôle : après terminaison, les RPC Raft restent non authentifiés. Avant qu'un client non fiable puisse atteindre le port, garder la liaison sur un réseau de confiance tel que `127.0.0.1`, ou placer chaque service derrière un proxy qui authentifie le client (par exemple TLS mutuel) et bloquer l'accès direct à ce processus. +Un secret vide autorise les quatre services de l'écouteur. Le code avertit de cela au démarrage. Le processus ne termine toujours pas TLS. Avant qu'un client non fiable puisse atteindre le port, garder la liaison sur un réseau de confiance tel que `127.0.0.1`, ou placer chaque service derrière un proxy qui authentifie le client (par exemple TLS mutuel) et bloquer l'accès direct à ce processus. ## Ce que l'on ne peut pas configurer diff --git a/docs/fr/operations.md b/docs/fr/operations.md index 086debc..1d06dbd 100644 --- a/docs/fr/operations.md +++ b/docs/fr/operations.md @@ -135,9 +135,9 @@ Les variantes sont définies dans `crates/metadb-core/src/error.rs`. Les méthod - `GetRoutes` échoue fermé pour un `cell_id` non nul inconnu. `StoreHeartbeat` ne remplace pas les routes en cache. - L'isolation jumbo est un indicateur de route plus un identifiant de nœud choisi dans la table locale. Elle n'évince pas d'autres shards d'un disque. - Les battements de cœur sont un RPC sans émetteur dans `metadbd`. `StoreHeartbeat` ne persiste pas. -- Raft sur le port gRPC public n'est pas authentifié. Le processus ne termine pas TLS. +- Un secret partagé non vide authentifie Raft sur le port gRPC public avec le même bearer que les trois autres services. Un secret vide laisse Raft non authentifié. Le processus ne termine pas TLS. - `memtable_size_bytes` et tous les champs de temporisation Raft ne changent pas le comportement d'exécution via le TOML. -- `metadb-ctl bench` rapporte un débit sur des tentatives. `unwrap` ne prend que les échecs de transport ou de statut. Les erreurs d'application telles que `AlreadyExists` reviennent dans une réponse OK et sont abandonnées, y compris pour unlink, donc une seconde exécution peut encore imprimer un débit. Il ne vérifie ni le listage, ni dirstat, ni le quota après l'exécution, et il n'imprime pas de percentiles de latence. +- `metadb-ctl bench` pour `hotspot-create` et `bulk-ttl-delete` imprime des lignes JSON : un en-tête avec `bench_pid`, puis des rapports de phase avec attempted, succeeded, failed et des percentiles de latence. Le processus se termine en erreur si une opération demandée échoue. `--workload mixed` ou `--embedded` lance le harnais de stress dans le processus. `sql` imprime la réponse en JSON et se termine en erreur sur une erreur d'application. ## Lacunes de documentation diff --git a/docs/fr/testing.md b/docs/fr/testing.md index 0161d85..83923bd 100644 --- a/docs/fr/testing.md +++ b/docs/fr/testing.md @@ -99,16 +99,15 @@ cargo run -p metadb-server --bin metadb-ctl -- \ bench --workload bulk-ttl-delete --concurrency 16 --ops 1000 ``` -Options : `--workload` vaut `hotspot-create` (le défaut) ou `bulk-ttl-delete`. `--concurrency` vaut 16 par défaut. `--ops` vaut 1000 par défaut. Toute autre chaîne de charge est une erreur. +Options : `--workload` vaut `hotspot-create` (le défaut), `bulk-ttl-delete` ou `mixed`. `--concurrency` vaut 16 s'il est omis. `--ops` vaut 1000 par défaut. `--embedded` ou `--workload mixed` démarre un nœud all-in-one temporaire et le harnais de stress dans le processus, au lieu d'appeler `--endpoint`. -Comportement issu de `crates/metadb-server/src/bin/metadb-ctl.rs` : +Comportement issu de `crates/metadb-server/src/bin/metadb-ctl.rs` pour les charges distantes : -- Chaque tâche exécute `ops / concurrency` itérations, et au moins une. Le reste de la division est abandonné, donc le nombre imprimé peut être inférieur à `--ops`. Si `--ops` est inférieur à `--concurrency`, le nombre peut être supérieur à `--ops` car chaque tâche s'exécute encore une fois. Le nombre imprimé est celui des tentatives, pas celui des mutations acceptées. -- `hotspot-create` fait un mkdir de l'inode `999000` nommé `bench_hotspot_create` sous l'inode parent `1`, puis appelle `CreateFile` pour des fichiers de 4096 octets. Le `Result` du mkdir est abandonné. `unwrap` porte seulement sur le `Result` tonic, donc un échec de transport ou de statut interrompt le processus. `CreateFile` renvoie les échecs d'application tels que `AlreadyExists` dans un `CreateFileResponse.error` OK, et le client ignore ce champ. -- `bulk-ttl-delete` fait les mêmes créations sous l'inode `999001` (`bench_bulk_ttl_delete`), puis appelle `Unlink` de la même façon (`unwrap` de transport, `error` d'application ignorée), puis un `ListDir` avec `limit = 1000`. -- `hotspot-create` imprime `Completed single-directory hotspot creates in s ( ops/s)` après les tentatives, y compris celles dont l'erreur d'application a été ignorée. -- `bulk-ttl-delete` imprime le nombre de tentatives d'unlink, les secondes, les ops/s, le nombre d'entrées vivantes renvoyé par cette liste, et la durée de la liste en millisecondes. L'appel de liste est vérifié ; les unlink ne le sont pas. -- Le client ne calcule pas de percentiles de latence et ne vérifie ni dirstat ni quota. Une seconde exécution contre le même serveur peut encore imprimer `Completed` et un débit lorsque chaque création est rejetée avec `AlreadyExists`. +- Le client alloue un identifiant de répertoire, puis des blocs d'inodes et d'identifiants de fichier de longueur `--ops`. Une attribution au mauvais compte ou qui déborde est refusée. +- Il imprime un objet JSON avec `bench_pid`, `workload` et `requested_ops`, puis crée autant de fichiers de 4096 octets dans un nouveau répertoire. +- Chaque phase imprime du JSON avec `attempted`, `succeeded`, `failed`, `transport_failed`, `application_failed`, `errors_by_kind` et des percentiles de latence. Le processus se termine en erreur si `succeeded` n'est pas égal à `--ops`. +- `bulk-ttl-delete` supprime les noms créés, puis liste le répertoire avec `limit = 1`. +- Une seconde exécution alloue un nouveau `bench_pid`. Les créations rejetées ne comptent pas comme des succès. Ne pas traiter un nombre issu de cette commande comme un résultat du dépôt. Il dépend de la machine, du profil de compilation et du serveur visé. Cet arbre n'enregistre pas d'échantillon. diff --git a/docs/guides/ja/deployment.md b/docs/guides/ja/deployment.md index 91fed53..ecae027 100644 --- a/docs/guides/ja/deployment.md +++ b/docs/guides/ja/deployment.md @@ -34,7 +34,7 @@ bootstrap_table_shards = true shard_split_threshold_bytes = 4294967296 # 4 GiB ``` -Rust 1.100.0(`rust-toolchain.toml`)。実装されているロールは `all-in-one` のみ。gRPC は既定で認証も TLS もなく、待受は `127.0.0.1`。信頼できないネットワークに公開しないこと。任意の `[security].shared_secret` は metadata / transaction / control-plane に `authorization: Bearer ` を要求する。 +Rust 1.100.0(`rust-toolchain.toml`)。実装されているロールは `all-in-one` のみ。gRPC は既定で認証も TLS もなく、待受は `127.0.0.1`。信頼できないネットワークに公開しないこと。空でない `[security].shared_secret` は metadata、transaction、control-plane、および同じポートの Raft 転送に `authorization: Bearer ` を要求する。 ```bash metadbd --config /etc/metadb/standalone.toml diff --git a/docs/guides/zh/deployment.md b/docs/guides/zh/deployment.md index 2cfec74..693f3d6 100644 --- a/docs/guides/zh/deployment.md +++ b/docs/guides/zh/deployment.md @@ -42,7 +42,7 @@ shard_split_threshold_bytes = 4294967296 # 4 GiB 分片分裂阈值 工具链固定为 Rust **1.100.0**(`rust-toolchain.toml`)。当前只有 `all-in-one` 会真正启动;`be` / `proxy` / `cell-manager` / `global-manager` 会被拒绝,而不是静默变成全功能进程。 -gRPC 的 metadata、transaction、control-plane 与 Raft 传输**没有 TLS**。`[security].shared_secret` 为空时也没有应用层认证,默认监听 `127.0.0.1`。在补上共享密钥或外部 TLS 终结之前,不要把该端口暴露到不可信网络。非空 `shared_secret` 要求 metadata / transaction / control-plane 携带 `authorization: Bearer `(同一端口上的 Raft 传输仍不校验该密钥)。 +gRPC 的 metadata、transaction、control-plane 与 Raft 传输**没有 TLS**。`[security].shared_secret` 为空时也没有应用层认证,默认监听 `127.0.0.1`。在补上共享密钥或外部 TLS 终结之前,不要把该端口暴露到不可信网络。非空 `shared_secret` 要求 metadata、transaction、control-plane 以及同一端口上的 Raft 传输都携带 `authorization: Bearer `。对等 Raft 客户端发送同一个令牌。 ```bash metadbd --config /etc/metadb/standalone.toml diff --git a/docs/ja/architecture.md b/docs/ja/architecture.md index 20f5127..e58ca70 100644 --- a/docs/ja/architecture.md +++ b/docs/ja/architecture.md @@ -15,7 +15,7 @@ 5. `/cell_` で `DurableCellManager` を開くかブートストラップする。 6. その 1 つのセルを、キー範囲 `[0x01, 0x04)` で `DurableGlobalManager` に登録する。 7. ルートの inode `1`(`ROOT_INODE_ID`)に属性レコードがあることを保証する。 -8. 同じリスナーで 4 つの gRPC サービスを提供する。メタデータ、トランザクション、コントロールプレーンは任意のベアラインターセプタを取る。`RaftTransportService` は取らない。 +8. 同じリスナーで 4 つの gRPC サービスを提供する。4 つとも同じ任意のベアラインターセプタを取る。シークレットが空ならすべての呼び出しを通す。 9. 250 ms 間隔のメンテナンスタスクを起動する。 `metadbd` は薄いラッパです。TOML を読み、CLI の上書きを適用し、`open_and_serve` を呼び、任意で `--cell-manager` へ `RegisterNode` を呼び、任意で Prometheus テキストを提供し、Ctrl-C で `shutdown` を呼びます。 diff --git a/docs/ja/build-and-run.md b/docs/ja/build-and-run.md index 1926254..d0fd698 100644 --- a/docs/ja/build-and-run.md +++ b/docs/ja/build-and-run.md @@ -64,7 +64,7 @@ cargo run -p metadb-server --bin metadbd -- --config ./standalone.toml ログは `tracing-subscriber` の `EnvFilter` を使います。既定のフィルタは `info` です。標準の環境変数 `RUST_LOG` で上書きします。例: `RUST_LOG=debug`。 -`security.shared_secret` が空のとき、起動時に gRPC には認証も TLS も無いという警告が出ます。既定のバインドは `127.0.0.1` です。信頼できないネットワークへこのソケットを出さないでください。`[security].shared_secret` が認証するのは、メタデータ、トランザクション、コントロールプレーンの RPC だけです。同じポートの `RaftTransportService` にはインターセプタが無いので、ベアラトークンはそれを覆わず、リスナーの前で TLS を終端してもその検査は付きません。リスナーを信頼できるネットワークに留めるか、Raft を含むすべてのサービスを、クライアントを認証するプロキシの後ろに置き、このプロセスへの直接アクセスを塞いでください。 +`security.shared_secret` が空のとき、起動時に gRPC には認証も TLS も無いという警告が出ます。既定のバインドは `127.0.0.1` です。信頼できないネットワークへこのソケットを出さないでください。空でない `[security].shared_secret` は、メタデータ、トランザクション、コントロールプレーン、および Raft 転送の RPC に同じベアラを要求します。プロセスは引き続き TLS を終端しません。リスナーを信頼できるネットワークに留めるか、すべてのサービスを、クライアントを認証するプロキシの後ろに置き、このプロセスへの直接アクセスを塞いでください。 Ctrl-C はシャード状態をフラッシュし、OpenRaft グループを止めます(`cluster.shutdown()`)。 diff --git a/docs/ja/configuration.md b/docs/ja/configuration.md index be8aeab..ea3447a 100644 --- a/docs/ja/configuration.md +++ b/docs/ja/configuration.md @@ -71,9 +71,9 @@ CLI フラグはファイルを解析したあとで上書きします。`--node `shared_secret` が空でないとき、`MetadataService`、`TransactionService`、`ControlPlaneService` は、メタデータ `authorization` が `Bearer ` と等しいことを要求します。`metadb-ctl --auth-token`(環境変数 `METADB_AUTH_TOKEN`)がそのヘッダを送ります。`metadbd --cell-manager` は `RegisterNode` を呼ぶとき同じシークレットを送ります。 -`RaftTransportService` はインターセプタなしで登録されます。ベアラトークンは、そのポートの Raft RPC を認証しません。 +`RaftTransportService` も同じインターセプタを使います。シークレットが空でないとき、このポートの Raft RPC も `Bearer ` を要求します。 -シークレットが空なら、メタデータ、トランザクション、コントロールプレーンの RPC は通ります。コードは起動時にこれを警告します。ベアラシークレットは `RaftTransportService` を覆いません。リスナーの前で TLS を終端してもその検査は付きません。終端のあと、Raft RPC は無認証のままです。信頼できないクライアントがポートへ届く前に、バインドを `127.0.0.1` のような信頼できるネットワークに留めるか、すべてのサービスをクライアントを認証するプロキシ(たとえば相互 TLS)の後ろに置き、このプロセスへの直接アクセスを塞いでください。 +シークレットが空なら、このリスナーの 4 つのサービスはすべて通ります。コードは起動時にこれを警告します。プロセスは引き続き TLS を終端しません。信頼できないクライアントがポートへ届く前に、バインドを `127.0.0.1` のような信頼できるネットワークに留めるか、すべてのサービスをクライアントを認証するプロキシ(たとえば相互 TLS)の後ろに置き、このプロセスへの直接アクセスを塞いでください。 ## 設定できないもの diff --git a/docs/ja/operations.md b/docs/ja/operations.md index 87c6b1f..f46768d 100644 --- a/docs/ja/operations.md +++ b/docs/ja/operations.md @@ -135,9 +135,9 @@ metadb-ctl --endpoint 127.0.0.1:7800 sql \ - 未知で非ゼロの `cell_id` に対する `GetRoutes` は失敗して閉じる。`StoreHeartbeat` はキャッシュ済みルートを置き換えない。 - ジャンボ隔離はルートフラグと、ローカルノード表から選んだノード ID である。他のシャードをディスクから追い出すことはしない。 - ハートビートは RPC であり、`metadbd` に送信側はない。`StoreHeartbeat` は永続化しない。 -- 公開 gRPC ポート上の Raft は無認証である。プロセスは TLS を終端しない。 +- 空でない共有シークレットは、公開 gRPC ポート上の Raft を他の 3 サービスと同じベアラで認証する。空のシークレットでは Raft は無認証のままである。プロセスは TLS を終端しない。 - `memtable_size_bytes` とすべての Raft タイマ項目は、TOML 経由で実行時の挙動を変えない。 -- `metadb-ctl bench` が報告するスループットは試行に対するものである。`unwrap` が捉えるのはトランスポートまたはステータスの失敗だけである。`AlreadyExists` のようなアプリケーションエラーは OK 応答の中に戻り、unlink を含めて捨てられる。したがって再実行でもスループットを印字し得る。実行後の一覧、dirstat、クォータは確認せず、レイテンシのパーセンタイルも印字しない。 +- `hotspot-create` と `bulk-ttl-delete` の `metadb-ctl bench` は JSON 行を印字する。`bench_pid` を含むヘッダのあと、attempted、succeeded、failed とレイテンシのパーセンタイルを持つフェーズ報告が続く。要求した操作がすべて成功しない限り、プロセスは非ゼロで終了する。`--workload mixed` または `--embedded` はプロセス内のストレステストを実行する。`sql` は応答を JSON で印字し、アプリケーションエラーでは非ゼロで終了する。 ## 文書化できない欠落 diff --git a/docs/ja/testing.md b/docs/ja/testing.md index a9469e7..8f85da4 100644 --- a/docs/ja/testing.md +++ b/docs/ja/testing.md @@ -99,16 +99,15 @@ cargo run -p metadb-server --bin metadb-ctl -- \ bench --workload bulk-ttl-delete --concurrency 16 --ops 1000 ``` -フラグ: `--workload` は `hotspot-create`(既定)または `bulk-ttl-delete` です。`--concurrency` の既定は 16 です。`--ops` の既定は 1000 です。それ以外のワークロード文字列はエラーです。 +フラグ: `--workload` は `hotspot-create`(既定)、`bulk-ttl-delete`、または `mixed` です。`--concurrency` を省略すると 16 です。`--ops` の既定は 1000 です。`--embedded` または `--workload mixed` は `--endpoint` を呼ばず、一時的な all-in-one ノードとプロセス内ストレステストを起動します。 -`crates/metadb-server/src/bin/metadb-ctl.rs` の挙動: +遠隔ワークロードについて `crates/metadb-server/src/bin/metadb-ctl.rs` の挙動: -- 各タスクは `ops / concurrency` 回、少なくとも 1 回実行します。除算の余りは捨てられるので、印字される数は `--ops` より小さくなり得ます。`--ops` が `--concurrency` より小さいときは、各タスクがそれでも 1 回動くため、数は `--ops` より大きくなり得ます。印字される数は試行回数であり、受理された変異の数ではありません。 -- `hotspot-create` は親 inode `1` の下に、名前 `bench_hotspot_create` の inode `999000` を mkdir し、4096 バイトのファイルに対して `CreateFile` を呼びます。mkdir の `Result` は捨てられます。`unwrap` がかかるのは tonic の `Result` だけなので、トランスポートまたはステータスの失敗がプロセスを中断します。`CreateFile` は `AlreadyExists` のようなアプリケーション失敗を、OK な `CreateFileResponse.error` の中に返します。クライアントはそのフィールドを捨てます。 -- `bulk-ttl-delete` は inode `999001`(`bench_bulk_ttl_delete`)の下で同じ作成を行い、同じやり方で `Unlink` を呼び(トランスポートの `unwrap`、アプリケーションの `error` は無視)、`limit = 1000` の `ListDir` を 1 回行います。 -- `hotspot-create` は試行のあと、アプリケーションエラーが無視された試行も含めて、`Completed single-directory hotspot creates in s ( ops/s)` を印字します。 -- `bulk-ttl-delete` は unlink の試行数、秒、ops/s、その一覧が返した生存エントリ数、一覧の所要ミリ秒を印字します。一覧の呼び出しは確認されます。unlink は確認されません。 -- クライアントはレイテンシのパーセンタイルを計算せず、dirstat もクォータも確認しません。同じサーバへの 2 回目でも、すべての作成が `AlreadyExists` で拒否されても `Completed` とスループットを印字することがあります。 +- クライアントはディレクトリ ID を 1 つ取り、そのあと長さ `--ops` のファイル inode とファイル ID のブロックを取ります。個数や終端が不正な払い出しは拒否します。 +- `bench_pid`、`workload`、`requested_ops` を持つ JSON を 1 つ印字し、新しいディレクトリに 4096 バイトのファイルをその個数だけ作ります。 +- 各フェーズは `attempted`、`succeeded`、`failed`、`transport_failed`、`application_failed`、`errors_by_kind`、レイテンシのパーセンタイルを JSON で印字します。`succeeded` が `--ops` と等しくなければプロセスは非ゼロで終了します。 +- `bulk-ttl-delete` は作成した名前を削除し、`limit = 1` でディレクトリを一覧します。 +- 2 回目は新しい `bench_pid` を取ります。拒否された作成は成功に数えません。 このコマンドの数値をリポジトリの結果として扱わないでください。マシン、ビルドプロファイル、向けたサーバに依存します。このツリーはサンプルを記録していません。 diff --git a/docs/zh/architecture.md b/docs/zh/architecture.md index 7ae289a..da54422 100644 --- a/docs/zh/architecture.md +++ b/docs/zh/architecture.md @@ -15,7 +15,7 @@ 5. 在 `/cell_` 打开或引导 `DurableCellManager`。 6. 把这一个单元以键范围 `[0x01, 0x04)` 注册到 `DurableGlobalManager`。 7. 保证根 inode `1`(`ROOT_INODE_ID`)有属性记录。 -8. 在同一监听器上提供四个 gRPC 服务。元数据、事务和控制面服务接受可选的持有者拦截器。`RaftTransportService` 不接受。 +8. 在同一监听器上提供四个 gRPC 服务。四个服务使用同一个可选的持有者拦截器。密钥为空时,所有调用都放行。 9. 启动间隔 250 ms 的维护任务。 `metadbd` 是薄封装:加载 TOML,应用命令行覆盖,调用 `open_and_serve`,可选地对 `--cell-manager` 调用 `RegisterNode`,可选地提供 Prometheus 文本,并在 Ctrl-C 时调用 `shutdown`。 diff --git a/docs/zh/build-and-run.md b/docs/zh/build-and-run.md index 6f20bc1..5571247 100644 --- a/docs/zh/build-and-run.md +++ b/docs/zh/build-and-run.md @@ -64,7 +64,7 @@ cargo run -p metadb-server --bin metadbd -- --config ./standalone.toml 日志使用 `tracing-subscriber` 的 `EnvFilter`。默认过滤器是 `info`。用标准环境变量 `RUST_LOG` 覆盖,例如 `RUST_LOG=debug`。 -`security.shared_secret` 为空时,进程在启动时记录一条警告:gRPC 没有认证,也没有 TLS。默认绑定是 `127.0.0.1`。不要把这个套接字公布到不可信网络。`[security].shared_secret` 只认证元数据、事务和控制面 RPC。同一端口上的 `RaftTransportService` 没有拦截器,因此持有者令牌盖不住它,在监听器前面终结 TLS 也不会补上这项检查。把监听器留在可信网络上,或把包括 Raft 在内的每个服务放到会认证客户端的代理后面,并阻断对本进程的直接访问。 +`security.shared_secret` 为空时,进程在启动时记录一条警告:gRPC 没有认证,也没有 TLS。默认绑定是 `127.0.0.1`。不要把这个套接字公布到不可信网络。非空的 `[security].shared_secret` 对元数据、事务、控制面和 Raft 传输 RPC 要求同一个持有者令牌。进程仍然不终结 TLS。把监听器留在可信网络上,或把每个服务放到会认证客户端的代理后面,并阻断对本进程的直接访问。 Ctrl-C 会刷出分片状态并停止 OpenRaft 组(`cluster.shutdown()`)。 diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 545eaaa..08398e1 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -71,9 +71,9 @@ shared_secret = "replace-with-a-long-random-string" `shared_secret` 非空时,`MetadataService`、`TransactionService` 和 `ControlPlaneService` 要求元数据 `authorization` 等于 `Bearer `。`metadb-ctl --auth-token`(环境变量 `METADB_AUTH_TOKEN`)发送这个头。`metadbd --cell-manager` 在调用 `RegisterNode` 时发送同一密钥。 -`RaftTransportService` 注册时没有拦截器。持有者令牌不能认证该端口上的 Raft RPC。 +`RaftTransportService` 使用同一个拦截器。密钥非空时,该端口上的 Raft RPC 同样要求 `Bearer `。 -空密钥允许元数据、事务和控制面 RPC。代码在启动时对此发出警告。持有者密钥不覆盖 `RaftTransportService`,在监听器前面终结 TLS 也不会补上这项检查:终结之后,Raft RPC 仍然不认证。在不可信客户端能够到达该端口之前,把绑定留在 `127.0.0.1` 这类可信网络上,或把每个服务放到会认证客户端的代理后面(例如双向 TLS),并阻断对本进程的直接访问。 +空密钥允许该监听器上的四个服务。代码在启动时对此发出警告。进程仍然不终结 TLS。在不可信客户端能够到达该端口之前,把绑定留在 `127.0.0.1` 这类可信网络上,或把每个服务放到会认证客户端的代理后面(例如双向 TLS),并阻断对本进程的直接访问。 ## 不能配置的内容 diff --git a/docs/zh/operations.md b/docs/zh/operations.md index 724b6d7..7d48e0e 100644 --- a/docs/zh/operations.md +++ b/docs/zh/operations.md @@ -135,9 +135,9 @@ metadb-ctl --endpoint 127.0.0.1:7800 sql \ - 未知且非零的 `cell_id` 上,`GetRoutes` 失败关闭。`StoreHeartbeat` 不会替换缓存路由。 - 巨型隔离是路由标志加上从本地节点表选出的节点标识。它不会把其他分片从磁盘上赶走。 - 心跳是一个 RPC,`metadbd` 里没有发送方。`StoreHeartbeat` 不持久化。 -- 公开 gRPC 端口上的 Raft 不认证。进程不终结 TLS。 +- 非空共享密钥用与另外三个服务相同的持有者令牌认证公开 gRPC 端口上的 Raft。空密钥下 Raft 仍然不认证。进程不终结 TLS。 - `memtable_size_bytes` 和所有 Raft 计时字段不会通过 TOML 改变运行行为。 -- `metadb-ctl bench` 报告的是尝试的吞吐量。`unwrap` 只抓住传输或状态失败。`AlreadyExists` 这类应用错误放在 OK 响应里并被丢掉,unlink 也一样,因此再跑一次仍可能打印吞吐量。它不在运行后检查列举、dirstat 或配额,也不打印延迟分位数。 +- `hotspot-create` 和 `bulk-ttl-delete` 的 `metadb-ctl bench` 打印 JSON 行:先是带 `bench_pid` 的头,然后是包含 attempted、succeeded、failed 和延迟分位数的阶段报告。只要有一次请求的操作失败,进程就以非零状态退出。`--workload mixed` 或 `--embedded` 运行进程内的压力工具。`sql` 把响应打印成 JSON,遇到应用错误时以非零状态退出。 ## 文档缺口 diff --git a/docs/zh/testing.md b/docs/zh/testing.md index 9bb6bce..23844b2 100644 --- a/docs/zh/testing.md +++ b/docs/zh/testing.md @@ -99,16 +99,15 @@ cargo run -p metadb-server --bin metadb-ctl -- \ bench --workload bulk-ttl-delete --concurrency 16 --ops 1000 ``` -标志:`--workload` 为 `hotspot-create`(默认)或 `bulk-ttl-delete`。`--concurrency` 默认 16。`--ops` 默认 1000。其他工作负载字符串是错误。 +标志:`--workload` 为 `hotspot-create`(默认)、`bulk-ttl-delete` 或 `mixed`。省略 `--concurrency` 时为 16。`--ops` 默认 1000。`--embedded` 或 `--workload mixed` 会启动一个临时的 all-in-one 节点和进程内压力工具,而不是拨号 `--endpoint`。 -来自 `crates/metadb-server/src/bin/metadb-ctl.rs` 的行为: +来自 `crates/metadb-server/src/bin/metadb-ctl.rs` 的远程负载行为: -- 每个任务运行 `ops / concurrency` 次,至少一次。除法余数被丢掉,因此打印的数量可以小于 `--ops`。若 `--ops` 小于 `--concurrency`,因为每个任务仍会跑一次,数量可以大于 `--ops`。打印的数量是尝试次数,不是被接受的变更次数。 -- `hotspot-create` 在父 inode `1` 下 mkdir inode `999000`,名为 `bench_hotspot_create`,然后对 4096 字节的文件调用 `CreateFile`。mkdir 的 `Result` 被丢弃。`unwrap` 只作用在 tonic 的 `Result` 上,因此传输或状态失败会中止进程。`CreateFile` 把 `AlreadyExists` 这类应用错误放进 OK 的 `CreateFileResponse.error`,客户端丢掉该字段。 -- `bulk-ttl-delete` 在 inode `999001`(`bench_bulk_ttl_delete`)下做同样的创建,然后以同样方式调用 `Unlink`(传输层 `unwrap`,应用层 `error` 被忽略),再做一次 `limit = 1000` 的 `ListDir`。 -- `hotspot-create` 在这些尝试之后打印 `Completed single-directory hotspot creates in s ( ops/s)`,包括应用错误被忽略的尝试。 -- `bulk-ttl-delete` 打印 unlink 尝试次数、秒数、ops/s、该次列举返回的存活条目数,以及列举耗时的毫秒数。列举调用会被检查;unlink 不会。 -- 客户端不计算延迟分位数,也不检查 dirstat 或配额。对同一服务器再跑一次时,即使每次创建都被 `AlreadyExists` 拒绝,仍可能打印 `Completed` 和吞吐量。 +- 客户端分配一个目录 id,再分配长度为 `--ops` 的文件 inode 和文件 id 块。数量或结尾溢出的授予会被拒绝。 +- 它打印一个包含 `bench_pid`、`workload` 和 `requested_ops` 的 JSON 对象,然后在新目录下创建同样数量的 4096 字节文件。 +- 每个阶段打印包含 `attempted`、`succeeded`、`failed`、`transport_failed`、`application_failed`、`errors_by_kind` 和延迟分位数的 JSON。若 `succeeded` 不等于 `--ops`,进程以非零状态退出。 +- `bulk-ttl-delete` 删除已创建的名字,再用 `limit = 1` 列举该目录。 +- 再跑一次会分配新的 `bench_pid`。被拒绝的创建不算成功。 不要把这条命令打出的数字当作仓库结果。它取决于机器、构建配置和你指向的服务器。本树没有记录样本。