-
Notifications
You must be signed in to change notification settings - Fork 156
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feature:
Raft::trigger()::allow_next_revert()
allow to reset replic…
…ation for next detected follower log revert This method requests the RaftCore to allow to reset replication for a specific node when log revert is detected. - `allow=true`: This method instructs the RaftCore to allow the target node's log to revert to a previous state for one time. - `allow=false`: This method instructs the RaftCore to panic if the target node's log revert ### Behavior - If this node is the Leader, it will attempt to replicate logs to the target node from the beginning. - If this node is not the Leader, the request is ignored. - If the target node is not found, the request is ignored. ### Automatic Replication Reset When the `loosen-follower-log-revert` feature flag is enabled, the Leader automatically reset replication if it detects that the target node's log has reverted. This feature is primarily useful in testing environments. ### Production Considerations In production environments, state reversion is a critical issue that should not be automatically handled. However, there may be scenarios where a Follower's data is intentionally removed and needs to rejoin the cluster(without membership changes). In such cases, the Leader should reinitialize replication for that node with the following steps: - Shut down the target node. - call [`Self::allow_next_revert`] on the Leader. - Clear the target node's data directory. - Restart the target node. - Fix: #1251
- Loading branch information
1 parent
3ab0d49
commit 9fb2f5c
Showing
9 changed files
with
172 additions
and
31 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
use std::sync::Arc; | ||
use std::time::Duration; | ||
|
||
use anyhow::Result; | ||
use maplit::btreeset; | ||
use openraft::Config; | ||
|
||
use crate::fixtures::ut_harness; | ||
use crate::fixtures::RaftRouter; | ||
|
||
/// With `Trigger::allow_next_revert()` the leader allows follower to revert its log to an | ||
/// earlier state for one time. | ||
#[tracing::instrument] | ||
#[test_harness::test(harness = ut_harness)] | ||
async fn allow_follower_log_revert() -> Result<()> { | ||
let config = Arc::new( | ||
Config { | ||
enable_tick: false, | ||
enable_heartbeat: false, | ||
// Make sure the replication is done in more than one steps | ||
max_payload_entries: 1, | ||
..Default::default() | ||
} | ||
.validate()?, | ||
); | ||
|
||
let mut router = RaftRouter::new(config.clone()); | ||
|
||
tracing::info!("--- initializing cluster"); | ||
let mut log_index = router.new_cluster(btreeset! {0}, btreeset! {1}).await?; | ||
|
||
tracing::info!(log_index, "--- write 10 logs"); | ||
{ | ||
log_index += router.client_request_many(0, "0", 10).await?; | ||
for i in [0, 1] { | ||
router.wait(&i, timeout()).applied_index(Some(log_index), format!("{} writes", 10)).await?; | ||
} | ||
} | ||
tracing::info!(log_index, "--- allow next detected log revert"); | ||
{ | ||
let n0 = router.get_raft_handle(&0)?; | ||
n0.trigger().allow_next_revert(&1, true).await?; | ||
} | ||
|
||
tracing::info!(log_index, "--- erase Learner-1 and restart"); | ||
{ | ||
let (_raft, _ls, _sm) = router.remove_node(3).unwrap(); | ||
let (log, sm) = openraft_memstore::new_mem_store(); | ||
|
||
router.new_raft_node_with_sto(3, log, sm).await; | ||
router.add_learner(0, 3).await?; | ||
log_index += 1; // add learner | ||
} | ||
|
||
tracing::info!(log_index, "--- write another 10 logs, leader should not panic"); | ||
{ | ||
log_index += router.client_request_many(0, "0", 10).await?; | ||
for i in [0, 1] { | ||
router.wait(&i, timeout()).applied_index(Some(log_index), format!("{} writes", 10)).await?; | ||
} | ||
} | ||
|
||
Ok(()) | ||
} | ||
|
||
fn timeout() -> Option<Duration> { | ||
Some(Duration::from_millis(1_000)) | ||
} |