Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/metadb-cluster/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ serde = { workspace = true }
tokio = { workspace = true }
tonic = { workspace = true }
tracing = { workspace = true }
openraft = { workspace = true }

[dev-dependencies]
tempfile = { workspace = true }
203 changes: 197 additions & 6 deletions crates/metadb-cluster/src/cell_manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use std::time::Duration;

use openraft::async_runtime::watch::WatchReceiver;
use openraft::{BasicNode, ChangeMembers, Precondition};
use serde::{Deserialize, Serialize};

use metadb_core::{
Expand Down Expand Up @@ -128,12 +130,52 @@ impl DurableCellManager {
fs::create_dir_all(&state_dir)?;
let state_path = state_dir.join("CELL_MANAGER_STATE");

let is_existing = state_path.exists();
let is_existing = state_path.try_exists()?;
if !is_existing && state_dir.join("CELL_OWNER").try_exists()? {
return Err(MetaDbError::StorageIoError(
"CELL_MANAGER_STATE is missing from an existing cell directory".to_string(),
));
}
let persisted = if is_existing {
let mut raw = Vec::new();
File::open(&state_path)?.read_to_end(&mut raw)?;
let body = verify_crc32c_le(&raw, "CELL_MANAGER_STATE")?;
decode_cell_state(body)?
let mut state = decode_cell_state(body)?;
let old_addr = state.nodes.get(&host.node_id()).ok_or_else(|| {
MetaDbError::ShardRoutingError(
"data directory belongs to a different node ID".to_string(),
)
})?;
if state.routes.values().any(|route| route.cell_id != cell_id) {
return Err(MetaDbError::ShardRoutingError(
"data directory belongs to a different cell ID".to_string(),
));
}
if old_addr != &local_addr {
for route in state
.routes
.values()
.filter(|route| route.peers.contains_key(&host.node_id()))
{
let voters: Vec<_> = route
.peers
.iter()
.filter(|(_, (_, voter))| *voter)
.map(|(&id, _)| id)
.collect();
if voters != [host.node_id()] {
return Err(MetaDbError::ShardRoutingError(
"a replicated node must restart at its persisted address; change node addresses online before restarting".to_string()));
}
}
state.nodes.insert(host.node_id(), local_addr.clone());
for route in state.routes.values_mut() {
if let Some((address, _)) = route.peers.get_mut(&host.node_id()) {
*address = local_addr.clone();
}
}
}
state
} else {
let mut init = PersistedCellState::default();
init.nodes.insert(host.node_id(), local_addr.clone());
Expand Down Expand Up @@ -181,6 +223,14 @@ impl DurableCellManager {
init
};

Self::verify_or_initialize_owner(
&state_dir,
host.node_id(),
cell_id,
&persisted,
is_existing,
)?;

let mgr = Arc::new(Self {
cell_id,
state_dir,
Expand All @@ -197,8 +247,14 @@ impl DurableCellManager {
crash_point: Mutex::new(None),
});

mgr.boot_local_shards().await?;
mgr.recover_incomplete_splits().await?;
if let Err(error) = mgr.boot_local_shards().await {
let _ = mgr.host.shutdown_all().await;
return Err(error);
}
if let Err(error) = mgr.recover_incomplete_splits().await {
let _ = mgr.host.shutdown_all().await;
return Err(error);
Comment thread
loverustfs marked this conversation as resolved.
Resolved
}
Ok(mgr)
}

Expand All @@ -221,7 +277,8 @@ impl DurableCellManager {
let state_dir = state_dir.as_ref().to_path_buf();
fs::create_dir_all(&state_dir)?;
let state_path = state_dir.join("CELL_MANAGER_STATE");
let persisted = if state_path.exists() {
let state_existed = state_path.exists();
let persisted = if state_existed {
let mut loaded = Self::load_state_file(&state_dir)?;
loaded.nodes.insert(host.node_id(), local_addr);
Self::write_state_file(&state_dir, &loaded)?;
Expand All @@ -232,6 +289,16 @@ impl DurableCellManager {
Self::write_state_file(&state_dir, &init)?;
init
};
// Record the owner before membership adds peers. A later restart of this
// directory goes through `open_or_bootstrap`, which rejects a multi-peer
// cell directory that has no `CELL_OWNER`.
Self::verify_or_initialize_owner(
&state_dir,
host.node_id(),
cell_id,
&persisted,
state_existed,
)?;
Ok(Arc::new(Self {
cell_id,
state_dir,
Expand Down Expand Up @@ -529,6 +596,47 @@ impl DurableCellManager {
group.initialize_if_needed(voter_addrs.clone()).await?;
}
self.host.register_running_group(group.clone());
let membership = group
.raft()
.metrics()
.borrow_watched()
.membership_config
.clone();
let local_addr = &route.peers[&self.host.node_id()].0;
if membership
.membership()
.get_node(&self.host.node_id())
.is_some_and(|node| node.addr != *local_addr)
{
let voters: Vec<_> = membership.membership().voter_ids().collect();
if voters != [self.host.node_id()]
|| membership.membership().get_joint_config().len() != 1
{
return Err(MetaDbError::ShardRoutingError(
"persisted Raft peer address differs from the listener; a replicated address change requires an online membership update".to_string()));
}
// ReadIndex waits for a committed log from the recovered leader's
// current term before changing membership metadata.
let _ = group.linearizable_read(group.current_epoch()).await?;
group
.raft()
.change_membership_if(
ChangeMembers::SetNodes(BTreeMap::from([(
self.host.node_id(),
BasicNode::new(local_addr),
)])),
true,
[Precondition::LastMembershipLogId {
last_membership_log_id: *membership.log_id(),
}],
)
.await
.map_err(|error| {
MetaDbError::ConsensusError(format!(
"failed to update the local Raft peer address: {error}"
))
})?;
}
self.host.record_registry_state(
route.shard_id,
RegistryShardState::Active { peers: voter_addrs },
Expand All @@ -540,6 +648,16 @@ impl DurableCellManager {

let mut synced_route = route.clone();
synced_route.epoch = group.current_epoch();
{
let mut state = self.state.lock().unwrap();
let mut next = state.clone();
next.routes
.insert(synced_route.shard_id, synced_route.clone());
// Persist refreshed node/route addresses only after the Raft
// membership update has committed. Repeating it is safe on restart.
self.persist_sync(&next)?;
*state = next;
}
self.meta_cache.upsert_route(synced_route);
} else {
self.meta_cache.upsert_route(route);
Expand Down Expand Up @@ -690,6 +808,18 @@ impl DurableCellManager {
let _guard = self.split_lock.lock().await;
// Finish or drop any plan left by a crash before discovering a new boundary.
self.recover_incomplete_splits().await?;
// Recovery can defer a pre-fence plan until transaction maintenance has
// drained the original shard. Keep its durable boundary and child ID.
if self
.state
.lock()
.unwrap()
.split_plans
.values()
.any(|plan| plan.parent_shard_id == shard_id && plan.state == SplitPlanState::Planned)
{
return Ok(None);
}
let route = {
let st = self.state.lock().unwrap();
st.routes.get(&shard_id).cloned().ok_or_else(|| {
Expand Down Expand Up @@ -992,7 +1122,17 @@ impl DurableCellManager {
};
let mut resumed_children: HashSet<ShardId> = HashSet::new();
for (uuid, plan) in plans {
self.resume_planned_split(uuid, &plan).await?;
match self.resume_planned_split(uuid, &plan).await {
Ok(()) => {}
Err(MetaDbError::WriteIntentConflict { .. } | MetaDbError::SplitDeferred(_)) => {
// Transaction recovery starts after Cell opening. A plan that
// has not fenced its parent must leave its original route
// available so that recovery can drain it, then retry on the
// next maintenance pass. Storage/codec/consensus errors still
// abort opening instead of being mistaken for deferrals.
}
Err(error) => return Err(error),
}
resumed_children.insert(plan.child_shard_id);
}

Expand Down Expand Up @@ -1369,6 +1509,57 @@ impl DurableCellManager {
sync_dir(dir)?;
Ok(())
}

fn verify_or_initialize_owner(
dir: &Path,
node_id: NodeId,
cell_id: CellId,
state: &PersistedCellState,
legacy: bool,
) -> Result<()> {
let path = dir.join("CELL_OWNER");
if path.try_exists()? {
let raw = fs::read(path)?;
let body = verify_crc32c_le(&raw, "CELL_OWNER")?;
if body.len() != 16
|| body[..8] != node_id.to_le_bytes()
|| body[8..] != cell_id.to_le_bytes()
{
return Err(MetaDbError::ShardRoutingError(
"cell directory owner does not match the configured node and cell IDs"
.to_string(),
));
}
return Ok(());
}
// Legacy singleton routes identify the sole owner unambiguously. A
// cluster-wide nodes map does not: it also contains registered peers.
if legacy
&& state
.routes
.values()
.any(|route| route.peers.len() != 1 || !route.peers.contains_key(&node_id))
{
return Err(MetaDbError::ShardRoutingError(
"legacy replicated cell directory has no owner marker; migrate ownership offline before startup".to_string(),
));
}
let mut raw = node_id.to_le_bytes().to_vec();
raw.extend_from_slice(&cell_id.to_le_bytes());
append_crc32c_le(&mut raw);
let tmp = dir.join("CELL_OWNER.tmp");
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&tmp)?;
file.write_all(&raw)?;
file.sync_data()?;
drop(file);
fs::rename(tmp, path)?;
sync_dir(dir)?;
Ok(())
}
}

/// `SplitPlanRecord` gained `dual_mode_pid` after the first on-disk layout.
Expand Down
Loading
Loading