From 1a741e7ed20352232372ea3fc74a2d51edd402d4 Mon Sep 17 00:00:00 2001 From: Arul Ajmani Date: Wed, 30 Oct 2024 15:07:54 -0400 Subject: [PATCH] raft: ensure leaderMaxSupported does not regress when bumping terms Previously, it was possible for a leader to regress leaderMaxSupported by calling an election at a higher term. This was because we weren't careful to recognize this case on the leader, and followers had special case handling that allowed a leader to override `inFortifyLease`. Together, this could cause lease regressions for LeaderLeases in some rare cases. This patch fixes this issue by removing special case handling in `inFortifyLease`. We also remove a special case in the handling of `MsgHup` which allowed a leader to step down and campaign without de-fortifying itself. Fixes https://github.com/cockroachdb/cockroach/issues/133764 Release note: None --- pkg/raft/raft.go | 18 +- .../fortification_followers_dont_prevote.txt | 140 +++++++++--- .../fortification_followers_dont_vote.txt | 200 ++++++++++++------ 3 files changed, 253 insertions(+), 105 deletions(-) diff --git a/pkg/raft/raft.go b/pkg/raft/raft.go index 24ade33584ef..36d34190f7e1 100644 --- a/pkg/raft/raft.go +++ b/pkg/raft/raft.go @@ -1266,6 +1266,8 @@ func (r *raft) becomeCandidate() { r.step = stepCandidate r.reset(r.Term + 1) r.tick = r.tickElection + // We're voting for ourselves. We shouldn't be supporting a fortified leader. + assertTrue(!r.supportingFortifiedLeader(), "shouldn't become a candidate if we're supporting a fortified leader") r.setVote(r.id) r.state = pb.StateCandidate r.logger.Infof("%x became candidate at term %d", r.id, r.Term) @@ -1347,13 +1349,9 @@ func (r *raft) hup(t CampaignType) { r.logger.Infof("%x is unpromotable and can not campaign", r.id) return } - // NB: The leader is allowed to bump its term by calling an election. Note that - // we must take care to ensure the leader's support expiration doesn't regress. - // - // TODO(arul): add special handling for the r.lead == r.id case with an - // assertion to ensure the LeaderSupportExpiration is in the past before - // campaigning. - if r.supportingFortifiedLeader() && r.lead != r.id { + // NB: Even an old leader that has since stepped down needs to de-fortify + // itself before it can campaign at a higher term. + if r.supportingFortifiedLeader() { r.logger.Debugf("%x ignoring MsgHup due to leader fortification", r.id) return } @@ -1478,12 +1476,6 @@ func (r *raft) Step(m pb.Message) error { force := bytes.Equal(m.Context, []byte(campaignTransfer)) inHeartbeatLease := r.checkQuorum && r.lead != None && r.electionElapsed < r.electionTimeout inFortifyLease := r.supportingFortifiedLeader() && - // NB: A fortified leader is allowed to bump its term. It'll need to - // re-fortify once if it gets elected at the higher term though, so the - // leader must take care to not regress its supported expiration. - // However, at the follower, we grant the fortified leader our vote at - // the higher term. - r.lead != m.From && // NB: If the peer that's campaigning has an entry in its log with a // higher term than what we're aware of, then this conclusively proves // that a new leader was elected at a higher term. We never heard from diff --git a/pkg/raft/testdata/fortification_followers_dont_prevote.txt b/pkg/raft/testdata/fortification_followers_dont_prevote.txt index 39b78c69d0c6..66cac3125900 100644 --- a/pkg/raft/testdata/fortification_followers_dont_prevote.txt +++ b/pkg/raft/testdata/fortification_followers_dont_prevote.txt @@ -362,13 +362,27 @@ raft-state 2: StateLeader (Voter) Term:2 Lead:2 LeadEpoch:1 3: StateFollower (Voter) Term:2 Lead:2 LeadEpoch:1 -# Lastly, ensure that the leader is able to successfully campaign at a higher +# Lastly, ensure that the even the leader is not able to campaign at a higher # term. We'll need to step down to set this up properly, as otherwise attempts # to campaign will no-op. step-down 2 ---- INFO 2 became follower at term 2 +campaign 2 +---- +DEBUG 2 ignoring MsgHup due to leader fortification + +# Have 2 withdraw support from itself so it can move past the fortification +# for MsgHup. This allows it to campaign, but it will not be able to win the +# election as {1,3} still support the fortified leader. +withdraw-support 2 2 +---- + 1 2 3 +1 1 1 1 +2 x x 1 +3 x 1 1 + campaign 2 ---- INFO 2 is starting a new election at term 2 @@ -389,18 +403,99 @@ stabilize INFO 2 has received 1 MsgPreVoteResp votes and 0 vote rejections > 1 receiving messages 2->1 MsgPreVote Term:3 Log:2/12 - INFO 1 [logterm: 2, index: 12, vote: 0] cast MsgPreVote for 2 [logterm: 2, index: 12] at term 2 + INFO 1 [logterm: 2, index: 12, vote: 0] ignored MsgPreVote from 2 [logterm: 2, index: 12] at term 2: supporting fortified leader 2 at epoch 1 > 3 receiving messages 2->3 MsgPreVote Term:3 Log:2/12 - INFO 3 [logterm: 2, index: 12, vote: 2] cast MsgPreVote for 2 [logterm: 2, index: 12] at term 2 + INFO 3 [logterm: 2, index: 12, vote: 2] ignored MsgPreVote from 2 [logterm: 2, index: 12] at term 2: supporting fortified leader 2 at epoch 1 + +# Note that 2 wasn't able to win the pre-vote election. At this point, the +# leaderMaxSupported is in the future. This demonstrates that the leader isn't +# able to regress leaderMaxSupported by calling an election at a higher term. +raft-state +---- +1: StateFollower (Voter) Term:2 Lead:2 LeadEpoch:1 +2: StatePreCandidate (Voter) Term:2 Lead:0 LeadEpoch:0 +3: StateFollower (Voter) Term:2 Lead:2 LeadEpoch:1 + + +# Let's de-fortify {1,3}. First, sanity check that simply ticking doesn't send +# out a de-fortification request, as leaderMaxSupported isn't in the past yet. +tick-heartbeat 2 +---- +ok + +# Expired leaderMaxSupported and tick again, which should de-fortify {1,3}. +support-expired 2 +---- +ok + +tick-heartbeat 2 +---- +ok + +stabilize +---- +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgDeFortifyLeader Term:2 Log:0/0 + 2->3 MsgDeFortifyLeader Term:2 Log:0/0 +> 1 receiving messages + 2->1 MsgDeFortifyLeader Term:2 Log:0/0 +> 3 receiving messages + 2->3 MsgDeFortifyLeader Term:2 Log:0/0 > 1 handling Ready Ready MustSync=true: HardState Term:2 Commit:12 Lead:2 LeadEpoch:0 - Messages: - 1->2 MsgPreVoteResp Term:3 Log:0/0 > 3 handling Ready Ready MustSync=true: HardState Term:2 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + +# At this point, 2 can campaign and win the election. Reset store liveness by +# bumping the epoch before campaigning though. +bump-epoch 2 +---- + 1 2 3 +1 1 2 1 +2 x x 1 +3 x 2 1 + +grant-support 2 2 +---- + 1 2 3 +1 1 2 1 +2 x 3 1 +3 x 2 1 + + +campaign 2 +---- +INFO 2 is starting a new election at term 2 +INFO 2 became pre-candidate at term 2 +INFO 2 [logterm: 2, index: 12] sent MsgPreVote request to 1 at term 2 +INFO 2 [logterm: 2, index: 12] sent MsgPreVote request to 3 at term 2 + +stabilize +---- +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgPreVote Term:3 Log:2/12 + 2->3 MsgPreVote Term:3 Log:2/12 + INFO 2 received MsgPreVoteResp from 2 at term 2 + INFO 2 has received 1 MsgPreVoteResp votes and 0 vote rejections +> 1 receiving messages + 2->1 MsgPreVote Term:3 Log:2/12 + INFO 1 [logterm: 2, index: 12, vote: 0] cast MsgPreVote for 2 [logterm: 2, index: 12] at term 2 +> 3 receiving messages + 2->3 MsgPreVote Term:3 Log:2/12 + INFO 3 [logterm: 2, index: 12, vote: 2] cast MsgPreVote for 2 [logterm: 2, index: 12] at term 2 +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgPreVoteResp Term:3 Log:0/0 +> 3 handling Ready + Ready MustSync=false: Messages: 3->2 MsgPreVoteResp Term:3 Log:0/0 > 2 receiving messages @@ -445,85 +540,70 @@ stabilize INFO 2 received MsgVoteResp from 1 at term 3 INFO 2 has received 2 MsgVoteResp votes and 0 vote rejections INFO 2 became leader at term 3 + INFO 2 leader at term 3 does not support itself in the liveness fabric 3->2 MsgVoteResp Term:3 Log:0/0 > 2 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:1 + HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:0 Entries: 3/13 EntryNormal "" Messages: - 2->1 MsgFortifyLeader Term:3 Log:0/0 - 2->3 MsgFortifyLeader Term:3 Log:0/0 2->1 MsgApp Term:3 Log:2/12 Commit:12 Entries:[3/13 EntryNormal ""] 2->3 MsgApp Term:3 Log:2/12 Commit:12 Entries:[3/13 EntryNormal ""] > 1 receiving messages - 2->1 MsgFortifyLeader Term:3 Log:0/0 2->1 MsgApp Term:3 Log:2/12 Commit:12 Entries:[3/13 EntryNormal ""] > 3 receiving messages - 2->3 MsgFortifyLeader Term:3 Log:0/0 2->3 MsgApp Term:3 Log:2/12 Commit:12 Entries:[3/13 EntryNormal ""] > 1 handling Ready Ready MustSync=true: - HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:1 + HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:0 Entries: 3/13 EntryNormal "" Messages: - 1->2 MsgFortifyLeaderResp Term:3 Log:0/0 LeadEpoch:1 1->2 MsgAppResp Term:3 Log:0/13 Commit:12 > 3 handling Ready Ready MustSync=true: - HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:1 + HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:0 Entries: 3/13 EntryNormal "" Messages: - 3->2 MsgFortifyLeaderResp Term:3 Log:0/0 LeadEpoch:1 3->2 MsgAppResp Term:3 Log:0/13 Commit:12 > 2 receiving messages - 1->2 MsgFortifyLeaderResp Term:3 Log:0/0 LeadEpoch:1 1->2 MsgAppResp Term:3 Log:0/13 Commit:12 - 3->2 MsgFortifyLeaderResp Term:3 Log:0/0 LeadEpoch:1 3->2 MsgAppResp Term:3 Log:0/13 Commit:12 > 2 handling Ready Ready MustSync=true: - HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:1 + HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:0 CommittedEntries: 3/13 EntryNormal "" Messages: - 2->1 MsgApp Term:3 Log:2/12 Commit:12 Entries:[3/13 EntryNormal ""] 2->1 MsgApp Term:3 Log:3/13 Commit:13 - 2->3 MsgApp Term:3 Log:2/12 Commit:13 Entries:[3/13 EntryNormal ""] 2->3 MsgApp Term:3 Log:3/13 Commit:13 > 1 receiving messages - 2->1 MsgApp Term:3 Log:2/12 Commit:12 Entries:[3/13 EntryNormal ""] 2->1 MsgApp Term:3 Log:3/13 Commit:13 > 3 receiving messages - 2->3 MsgApp Term:3 Log:2/12 Commit:13 Entries:[3/13 EntryNormal ""] 2->3 MsgApp Term:3 Log:3/13 Commit:13 > 1 handling Ready Ready MustSync=true: - HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:1 + HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:0 CommittedEntries: 3/13 EntryNormal "" Messages: - 1->2 MsgAppResp Term:3 Log:0/13 Commit:12 1->2 MsgAppResp Term:3 Log:0/13 Commit:13 > 3 handling Ready Ready MustSync=true: - HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:1 + HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:0 CommittedEntries: 3/13 EntryNormal "" Messages: 3->2 MsgAppResp Term:3 Log:0/13 Commit:13 - 3->2 MsgAppResp Term:3 Log:0/13 Commit:13 > 2 receiving messages - 1->2 MsgAppResp Term:3 Log:0/13 Commit:12 1->2 MsgAppResp Term:3 Log:0/13 Commit:13 3->2 MsgAppResp Term:3 Log:0/13 Commit:13 - 3->2 MsgAppResp Term:3 Log:0/13 Commit:13 raft-state ---- -1: StateFollower (Voter) Term:3 Lead:2 LeadEpoch:1 -2: StateLeader (Voter) Term:3 Lead:2 LeadEpoch:1 -3: StateFollower (Voter) Term:3 Lead:2 LeadEpoch:1 +1: StateFollower (Voter) Term:3 Lead:2 LeadEpoch:0 +2: StateLeader (Voter) Term:3 Lead:2 LeadEpoch:0 +3: StateFollower (Voter) Term:3 Lead:2 LeadEpoch:0 diff --git a/pkg/raft/testdata/fortification_followers_dont_vote.txt b/pkg/raft/testdata/fortification_followers_dont_vote.txt index 6181d4fecdb7..3aaac4ee83dc 100644 --- a/pkg/raft/testdata/fortification_followers_dont_vote.txt +++ b/pkg/raft/testdata/fortification_followers_dont_vote.txt @@ -1,5 +1,7 @@ # Test to ensure that a follower will not vote for another peer # if they're supporting a fortified leader. +# TODO(arul): keeping this file around is kinda dubious, given we'll never run +# fortification without pre-vote. log-level debug ---- @@ -306,13 +308,27 @@ raft-state 2: StateLeader (Voter) Term:3 Lead:2 LeadEpoch:1 3: StateFollower (Voter) Term:3 Lead:2 LeadEpoch:1 -# Lastly, ensure that the leader is able to successfully campaign at a higher +# Lastly, ensure that the even the leader is not able to campaign at a higher # term. We'll need to step down to set this up properly, as otherwise attempts # to campaign will no-op. step-down 2 ---- INFO 2 became follower at term 3 +campaign 2 +---- +DEBUG 2 ignoring MsgHup due to leader fortification + +# Have 2 withdraw support from itself so it can move past the fortification +# for MsgHup. This allows it to campaign, but it will not be able to win the +# election as {1,3} still support the fortified leader. +withdraw-support 2 2 +---- + 1 2 3 +1 1 1 1 +2 x x 1 +3 x 1 1 + campaign 2 ---- INFO 2 is starting a new election at term 3 @@ -333,108 +349,168 @@ stabilize INFO 2 has received 1 MsgVoteResp votes and 0 vote rejections > 1 receiving messages 2->1 MsgVote Term:4 Log:3/12 - INFO 1 [term: 3] received a MsgVote message with higher term from 2 [term: 4] - INFO 1 became follower at term 4 - INFO 1 [logterm: 3, index: 12, vote: 0] cast MsgVote for 2 [logterm: 3, index: 12] at term 4 + INFO 1 [logterm: 3, index: 12, vote: 0] ignored MsgVote from 2 [logterm: 3, index: 12] at term 3: supporting fortified leader 2 at epoch 1 > 3 receiving messages 2->3 MsgVote Term:4 Log:3/12 - INFO 3 [term: 3] received a MsgVote message with higher term from 2 [term: 4] - INFO 3 became follower at term 4 - INFO 3 [logterm: 3, index: 12, vote: 0] cast MsgVote for 2 [logterm: 3, index: 12] at term 4 + INFO 3 [logterm: 3, index: 12, vote: 2] ignored MsgVote from 2 [logterm: 3, index: 12] at term 3: supporting fortified leader 2 at epoch 1 + + +# Let's de-fortify {1,3}. First, sanity check that simply ticking doesn't send +# out a de-fortification request, as leaderMaxSupported isn't in the past yet. +tick-heartbeat 2 +---- +ok + +# Expired leaderMaxSupported and tick again, which should de-fortify {1,3}. +support-expired 2 +---- +ok + +tick-heartbeat 2 +---- +DEBUG de-foritfying self at term 3 is a no-op; current term 4 + +stabilize +---- +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgDeFortifyLeader Term:3 Log:0/0 + 2->3 MsgDeFortifyLeader Term:3 Log:0/0 +> 1 receiving messages + 2->1 MsgDeFortifyLeader Term:3 Log:0/0 +> 3 receiving messages + 2->3 MsgDeFortifyLeader Term:3 Log:0/0 > 1 handling Ready Ready MustSync=true: - HardState Term:4 Vote:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:3 Commit:12 Lead:2 LeadEpoch:0 +> 3 handling Ready + Ready MustSync=true: + HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + +# At this point, 2 can campaign and win the election. Reset store liveness by +# bumping the epoch before campaigning though. +bump-epoch 2 +---- + 1 2 3 +1 1 2 1 +2 x x 1 +3 x 2 1 + +grant-support 2 2 +---- + 1 2 3 +1 1 2 1 +2 x 3 1 +3 x 2 1 + + +campaign 2 +---- +INFO 2 is starting a new election at term 4 +INFO 2 became candidate at term 5 +INFO 2 [logterm: 3, index: 12] sent MsgVote request to 1 at term 5 +INFO 2 [logterm: 3, index: 12] sent MsgVote request to 3 at term 5 + +stabilize +---- +> 2 handling Ready + Ready MustSync=true: + HardState Term:5 Vote:2 Commit:12 Lead:0 LeadEpoch:0 + Messages: + 2->1 MsgVote Term:5 Log:3/12 + 2->3 MsgVote Term:5 Log:3/12 + INFO 2 received MsgVoteResp from 2 at term 5 + INFO 2 has received 1 MsgVoteResp votes and 0 vote rejections +> 1 receiving messages + 2->1 MsgVote Term:5 Log:3/12 + INFO 1 [term: 3] received a MsgVote message with higher term from 2 [term: 5] + INFO 1 became follower at term 5 + INFO 1 [logterm: 3, index: 12, vote: 0] cast MsgVote for 2 [logterm: 3, index: 12] at term 5 +> 3 receiving messages + 2->3 MsgVote Term:5 Log:3/12 + INFO 3 [term: 3] received a MsgVote message with higher term from 2 [term: 5] + INFO 3 became follower at term 5 + INFO 3 [logterm: 3, index: 12, vote: 0] cast MsgVote for 2 [logterm: 3, index: 12] at term 5 +> 1 handling Ready + Ready MustSync=true: + HardState Term:5 Vote:2 Commit:12 Lead:0 LeadEpoch:0 Messages: - 1->2 MsgVoteResp Term:4 Log:0/0 + 1->2 MsgVoteResp Term:5 Log:0/0 > 3 handling Ready Ready MustSync=true: - HardState Term:4 Vote:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:5 Vote:2 Commit:12 Lead:0 LeadEpoch:0 Messages: - 3->2 MsgVoteResp Term:4 Log:0/0 + 3->2 MsgVoteResp Term:5 Log:0/0 > 2 receiving messages - 1->2 MsgVoteResp Term:4 Log:0/0 - INFO 2 received MsgVoteResp from 1 at term 4 + 1->2 MsgVoteResp Term:5 Log:0/0 + INFO 2 received MsgVoteResp from 1 at term 5 INFO 2 has received 2 MsgVoteResp votes and 0 vote rejections - INFO 2 became leader at term 4 - 3->2 MsgVoteResp Term:4 Log:0/0 + INFO 2 became leader at term 5 + INFO 2 leader at term 5 does not support itself in the liveness fabric + 3->2 MsgVoteResp Term:5 Log:0/0 > 2 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:4 Vote:2 Commit:12 Lead:2 LeadEpoch:1 + HardState Term:5 Vote:2 Commit:12 Lead:2 LeadEpoch:0 Entries: - 4/13 EntryNormal "" + 5/13 EntryNormal "" Messages: - 2->1 MsgFortifyLeader Term:4 Log:0/0 - 2->3 MsgFortifyLeader Term:4 Log:0/0 - 2->1 MsgApp Term:4 Log:3/12 Commit:12 Entries:[4/13 EntryNormal ""] - 2->3 MsgApp Term:4 Log:3/12 Commit:12 Entries:[4/13 EntryNormal ""] + 2->1 MsgApp Term:5 Log:3/12 Commit:12 Entries:[5/13 EntryNormal ""] + 2->3 MsgApp Term:5 Log:3/12 Commit:12 Entries:[5/13 EntryNormal ""] > 1 receiving messages - 2->1 MsgFortifyLeader Term:4 Log:0/0 - 2->1 MsgApp Term:4 Log:3/12 Commit:12 Entries:[4/13 EntryNormal ""] + 2->1 MsgApp Term:5 Log:3/12 Commit:12 Entries:[5/13 EntryNormal ""] > 3 receiving messages - 2->3 MsgFortifyLeader Term:4 Log:0/0 - 2->3 MsgApp Term:4 Log:3/12 Commit:12 Entries:[4/13 EntryNormal ""] + 2->3 MsgApp Term:5 Log:3/12 Commit:12 Entries:[5/13 EntryNormal ""] > 1 handling Ready Ready MustSync=true: - HardState Term:4 Vote:2 Commit:12 Lead:2 LeadEpoch:1 + HardState Term:5 Vote:2 Commit:12 Lead:2 LeadEpoch:0 Entries: - 4/13 EntryNormal "" + 5/13 EntryNormal "" Messages: - 1->2 MsgFortifyLeaderResp Term:4 Log:0/0 LeadEpoch:1 - 1->2 MsgAppResp Term:4 Log:0/13 Commit:12 + 1->2 MsgAppResp Term:5 Log:0/13 Commit:12 > 3 handling Ready Ready MustSync=true: - HardState Term:4 Vote:2 Commit:12 Lead:2 LeadEpoch:1 + HardState Term:5 Vote:2 Commit:12 Lead:2 LeadEpoch:0 Entries: - 4/13 EntryNormal "" + 5/13 EntryNormal "" Messages: - 3->2 MsgFortifyLeaderResp Term:4 Log:0/0 LeadEpoch:1 - 3->2 MsgAppResp Term:4 Log:0/13 Commit:12 + 3->2 MsgAppResp Term:5 Log:0/13 Commit:12 > 2 receiving messages - 1->2 MsgFortifyLeaderResp Term:4 Log:0/0 LeadEpoch:1 - 1->2 MsgAppResp Term:4 Log:0/13 Commit:12 - 3->2 MsgFortifyLeaderResp Term:4 Log:0/0 LeadEpoch:1 - 3->2 MsgAppResp Term:4 Log:0/13 Commit:12 + 1->2 MsgAppResp Term:5 Log:0/13 Commit:12 + 3->2 MsgAppResp Term:5 Log:0/13 Commit:12 > 2 handling Ready Ready MustSync=true: - HardState Term:4 Vote:2 Commit:13 Lead:2 LeadEpoch:1 + HardState Term:5 Vote:2 Commit:13 Lead:2 LeadEpoch:0 CommittedEntries: - 4/13 EntryNormal "" + 5/13 EntryNormal "" Messages: - 2->1 MsgApp Term:4 Log:3/12 Commit:12 Entries:[4/13 EntryNormal ""] - 2->1 MsgApp Term:4 Log:4/13 Commit:13 - 2->3 MsgApp Term:4 Log:3/12 Commit:13 Entries:[4/13 EntryNormal ""] - 2->3 MsgApp Term:4 Log:4/13 Commit:13 + 2->1 MsgApp Term:5 Log:5/13 Commit:13 + 2->3 MsgApp Term:5 Log:5/13 Commit:13 > 1 receiving messages - 2->1 MsgApp Term:4 Log:3/12 Commit:12 Entries:[4/13 EntryNormal ""] - 2->1 MsgApp Term:4 Log:4/13 Commit:13 + 2->1 MsgApp Term:5 Log:5/13 Commit:13 > 3 receiving messages - 2->3 MsgApp Term:4 Log:3/12 Commit:13 Entries:[4/13 EntryNormal ""] - 2->3 MsgApp Term:4 Log:4/13 Commit:13 + 2->3 MsgApp Term:5 Log:5/13 Commit:13 > 1 handling Ready Ready MustSync=true: - HardState Term:4 Vote:2 Commit:13 Lead:2 LeadEpoch:1 + HardState Term:5 Vote:2 Commit:13 Lead:2 LeadEpoch:0 CommittedEntries: - 4/13 EntryNormal "" + 5/13 EntryNormal "" Messages: - 1->2 MsgAppResp Term:4 Log:0/13 Commit:12 - 1->2 MsgAppResp Term:4 Log:0/13 Commit:13 + 1->2 MsgAppResp Term:5 Log:0/13 Commit:13 > 3 handling Ready Ready MustSync=true: - HardState Term:4 Vote:2 Commit:13 Lead:2 LeadEpoch:1 + HardState Term:5 Vote:2 Commit:13 Lead:2 LeadEpoch:0 CommittedEntries: - 4/13 EntryNormal "" + 5/13 EntryNormal "" Messages: - 3->2 MsgAppResp Term:4 Log:0/13 Commit:13 - 3->2 MsgAppResp Term:4 Log:0/13 Commit:13 + 3->2 MsgAppResp Term:5 Log:0/13 Commit:13 > 2 receiving messages - 1->2 MsgAppResp Term:4 Log:0/13 Commit:12 - 1->2 MsgAppResp Term:4 Log:0/13 Commit:13 - 3->2 MsgAppResp Term:4 Log:0/13 Commit:13 - 3->2 MsgAppResp Term:4 Log:0/13 Commit:13 + 1->2 MsgAppResp Term:5 Log:0/13 Commit:13 + 3->2 MsgAppResp Term:5 Log:0/13 Commit:13 raft-state ---- -1: StateFollower (Voter) Term:4 Lead:2 LeadEpoch:1 -2: StateLeader (Voter) Term:4 Lead:2 LeadEpoch:1 -3: StateFollower (Voter) Term:4 Lead:2 LeadEpoch:1 +1: StateFollower (Voter) Term:5 Lead:2 LeadEpoch:0 +2: StateLeader (Voter) Term:5 Lead:2 LeadEpoch:0 +3: StateFollower (Voter) Term:5 Lead:2 LeadEpoch:0