raft: proactively probe newly added followers

raft/confchange/confchange.go

@@ -257,11 +257,15 @@ func (c Changer) initProgress(cfg *tracker.Config, prs tracker.ProgressMap, id u
 		nilAwareAdd(&cfg.Learners, id)
 	}
 	prs[id] = &tracker.Progress{
-		// We initialize Progress.Next with lastIndex+1 so that the peer will be
-		// probed without an index first.
+		// Initializing the Progress with the last index means that the follower
+		// can be probed (with the last index).
 		//
-		// TODO(tbg): verify that, this is just my best guess.
-		Next:      c.LastIndex + 1,
+		// TODO(tbg): seems awfully optimistic. Using the first index would be
+		// better. The general expectation here is that the follower has no log
+		// at all (and will thus likely need a snapshot), though the app may
+		// have applied a snapshot out of band before adding the replica (thus
+		// making the first index the better choice).
+		Next:      c.LastIndex,
 		Match:     0,
 		Inflights: tracker.NewInflights(c.Tracker.MaxInflight),
 		IsLearner: isLearner,

raft/confchange/testdata/joint_autoleave.txt

@@ -5,16 +5,16 @@ simple
 v1
 ----
 voters=(1)
-1: StateProbe match=0 next=1
+1: StateProbe match=0 next=0
 
 # Autoleave is reflected in the config.
 enter-joint autoleave=true
 v2 v3
 ----
 voters=(1 2 3)&&(1) autoleave
-1: StateProbe match=0 next=1
-2: StateProbe match=0 next=2
-3: StateProbe match=0 next=2
+1: StateProbe match=0 next=0
+2: StateProbe match=0 next=1
+3: StateProbe match=0 next=1
 
 # Can't enter-joint twice, even if autoleave changes.
 enter-joint autoleave=false
@@ -24,6 +24,6 @@ config is already joint
 leave-joint
 ----
 voters=(1 2 3)
-1: StateProbe match=0 next=1
-2: StateProbe match=0 next=2
-3: StateProbe match=0 next=2
+1: StateProbe match=0 next=0
+2: StateProbe match=0 next=1
+3: StateProbe match=0 next=1

raft/confchange/testdata/joint_idempotency.txt

@@ -5,19 +5,19 @@ simple
 v1
 ----
 voters=(1)
-1: StateProbe match=0 next=1
+1: StateProbe match=0 next=0
 
 enter-joint
 r1 r2 r9 v2 v3 v4 v2 v3 v4 l2 l2 r4 r4 l1 l1
 ----
 voters=(3)&&(1) learners=(2) learners_next=(1)
-1: StateProbe match=0 next=1
-2: StateProbe match=0 next=2 learner
-3: StateProbe match=0 next=2
+1: StateProbe match=0 next=0
+2: StateProbe match=0 next=1 learner
+3: StateProbe match=0 next=1
 
 leave-joint
 ----
 voters=(3) learners=(1 2)
-1: StateProbe match=0 next=1 learner
-2: StateProbe match=0 next=2 learner
-3: StateProbe match=0 next=2
+1: StateProbe match=0 next=0 learner
+2: StateProbe match=0 next=1 learner
+3: StateProbe match=0 next=1

raft/confchange/testdata/joint_learners_next.txt

@@ -8,17 +8,17 @@ simple
 v1
 ----
 voters=(1)
-1: StateProbe match=0 next=1
+1: StateProbe match=0 next=0
 
 enter-joint
 v2 l1
 ----
 voters=(2)&&(1) learners_next=(1)
-1: StateProbe match=0 next=1
-2: StateProbe match=0 next=2
+1: StateProbe match=0 next=0
+2: StateProbe match=0 next=1
 
 leave-joint
 ----
 voters=(2) learners=(1)
-1: StateProbe match=0 next=1 learner
-2: StateProbe match=0 next=2
+1: StateProbe match=0 next=0 learner
+2: StateProbe match=0 next=1

raft/confchange/testdata/joint_safety.txt

@@ -15,7 +15,7 @@ simple
 v1
 ----
 voters=(1)
-1: StateProbe match=0 next=4
+1: StateProbe match=0 next=3
 
 leave-joint
 ----
@@ -25,7 +25,7 @@ can't leave a non-joint config
 enter-joint
 ----
 voters=(1)&&(1)
-1: StateProbe match=0 next=4
+1: StateProbe match=0 next=3
 
 enter-joint
 ----
@@ -34,7 +34,7 @@ config is already joint
 leave-joint
 ----
 voters=(1)
-1: StateProbe match=0 next=4
+1: StateProbe match=0 next=3
 
 leave-joint
 ----
@@ -45,10 +45,10 @@ enter-joint
 r1 v2 v3 l4
 ----
 voters=(2 3)&&(1) learners=(4)
-1: StateProbe match=0 next=4
-2: StateProbe match=0 next=10
-3: StateProbe match=0 next=10
-4: StateProbe match=0 next=10 learner
+1: StateProbe match=0 next=3
+2: StateProbe match=0 next=9
+3: StateProbe match=0 next=9
+4: StateProbe match=0 next=9 learner
 
 enter-joint
 ----
@@ -67,15 +67,15 @@ can't apply simple config change in joint config
 leave-joint
 ----
 voters=(2 3) learners=(4)
-2: StateProbe match=0 next=10
-3: StateProbe match=0 next=10
-4: StateProbe match=0 next=10 learner
+2: StateProbe match=0 next=9
+3: StateProbe match=0 next=9
+4: StateProbe match=0 next=9 learner
 
 simple
 l9
 ----
 voters=(2 3) learners=(4 9)
-2: StateProbe match=0 next=10
-3: StateProbe match=0 next=10
-4: StateProbe match=0 next=10 learner
-9: StateProbe match=0 next=15 learner
+2: StateProbe match=0 next=9
+3: StateProbe match=0 next=9
+4: StateProbe match=0 next=9 learner
+9: StateProbe match=0 next=14 learner

raft/confchange/testdata/simple_idempotency.txt

@@ -2,68 +2,68 @@ simple
 v1
 ----
 voters=(1)
-1: StateProbe match=0 next=1
+1: StateProbe match=0 next=0
 
 simple
 v1
 ----
 voters=(1)
-1: StateProbe match=0 next=1
+1: StateProbe match=0 next=0
 
 simple
 v2
 ----
 voters=(1 2)
-1: StateProbe match=0 next=1
-2: StateProbe match=0 next=3
+1: StateProbe match=0 next=0
+2: StateProbe match=0 next=2
 
 simple
 l1
 ----
 voters=(2) learners=(1)
-1: StateProbe match=0 next=1 learner
-2: StateProbe match=0 next=3
+1: StateProbe match=0 next=0 learner
+2: StateProbe match=0 next=2
 
 simple
 l1
 ----
 voters=(2) learners=(1)
-1: StateProbe match=0 next=1 learner
-2: StateProbe match=0 next=3
+1: StateProbe match=0 next=0 learner
+2: StateProbe match=0 next=2
 
 simple
 r1
 ----
 voters=(2)
-2: StateProbe match=0 next=3
+2: StateProbe match=0 next=2
 
 simple
 r1
 ----
 voters=(2)
-2: StateProbe match=0 next=3
+2: StateProbe match=0 next=2
 
 simple
 v3
 ----
 voters=(2 3)
-2: StateProbe match=0 next=3
-3: StateProbe match=0 next=8
+2: StateProbe match=0 next=2
+3: StateProbe match=0 next=7
 
 simple
 r3
 ----
 voters=(2)
-2: StateProbe match=0 next=3
+2: StateProbe match=0 next=2
 
 simple
 r3
 ----
 voters=(2)
-2: StateProbe match=0 next=3
+2: StateProbe match=0 next=2
 
 simple
 r4
 ----
 voters=(2)
-2: StateProbe match=0 next=3
+2: StateProbe match=0 next=2

raft/confchange/testdata/simple_promote_demote.txt

@@ -4,57 +4,57 @@ simple
 v1
 ----
 voters=(1)
-1: StateProbe match=0 next=1
+1: StateProbe match=0 next=0
 
 simple
 v2
 ----
 voters=(1 2)
-1: StateProbe match=0 next=1
-2: StateProbe match=0 next=2
+1: StateProbe match=0 next=0
+2: StateProbe match=0 next=1
 
 simple
 v3
 ----
 voters=(1 2 3)
-1: StateProbe match=0 next=1
-2: StateProbe match=0 next=2
-3: StateProbe match=0 next=3
+1: StateProbe match=0 next=0
+2: StateProbe match=0 next=1
+3: StateProbe match=0 next=2
 
 # Can atomically demote and promote without a hitch.
 # This is pointless, but possible.
 simple
 l1 v1
 ----
 voters=(1 2 3)
-1: StateProbe match=0 next=1
-2: StateProbe match=0 next=2
-3: StateProbe match=0 next=3
+1: StateProbe match=0 next=0
+2: StateProbe match=0 next=1
+3: StateProbe match=0 next=2
 
 # Can demote a voter.
 simple
 l2
 ----
 voters=(1 3) learners=(2)
-1: StateProbe match=0 next=1
-2: StateProbe match=0 next=2 learner
-3: StateProbe match=0 next=3
+1: StateProbe match=0 next=0
+2: StateProbe match=0 next=1 learner
+3: StateProbe match=0 next=2
 
 # Can atomically promote and demote the same voter.
 # This is pointless, but possible.
 simple
 v2 l2
 ----
 voters=(1 3) learners=(2)
-1: StateProbe match=0 next=1
-2: StateProbe match=0 next=2 learner
-3: StateProbe match=0 next=3
+1: StateProbe match=0 next=0
+2: StateProbe match=0 next=1 learner
+3: StateProbe match=0 next=2
 
 # Can promote a voter.
 simple
 v2
 ----
 voters=(1 2 3)
-1: StateProbe match=0 next=1
-2: StateProbe match=0 next=2
-3: StateProbe match=0 next=3
+1: StateProbe match=0 next=0
+2: StateProbe match=0 next=1
+3: StateProbe match=0 next=2

raft/confchange/testdata/simple_safety.txt

@@ -7,15 +7,15 @@ simple
 v1
 ----
 voters=(1)
-1: StateProbe match=0 next=2
+1: StateProbe match=0 next=1
 
 simple
 v2 l3
 ----
 voters=(1 2) learners=(3)
-1: StateProbe match=0 next=2
-2: StateProbe match=0 next=3
-3: StateProbe match=0 next=3 learner
+1: StateProbe match=0 next=1
+2: StateProbe match=0 next=2
+3: StateProbe match=0 next=2 learner
 
 simple
 r1 v5
@@ -46,11 +46,11 @@ simple
 l2 l3 l4 l5
 ----
 voters=(1) learners=(2 3 4 5)
-1: StateProbe match=0 next=2
-2: StateProbe match=0 next=3 learner
-3: StateProbe match=0 next=3 learner
-4: StateProbe match=0 next=9 learner
-5: StateProbe match=0 next=9 learner
+1: StateProbe match=0 next=1
+2: StateProbe match=0 next=2 learner
+3: StateProbe match=0 next=2 learner
+4: StateProbe match=0 next=8 learner
+5: StateProbe match=0 next=8 learner
 
 simple
 r1
@@ -61,4 +61,4 @@ simple
 r2 r3 r4 r5
 ----
 voters=(1)
-1: StateProbe match=0 next=2
+1: StateProbe match=0 next=1

raft/confchange/testdata/update.txt

@@ -6,18 +6,18 @@ simple
 v1
 ----
 voters=(1)
-1: StateProbe match=0 next=1
+1: StateProbe match=0 next=0
 
 simple
 v2 u1
 ----
 voters=(1 2)
-1: StateProbe match=0 next=1
-2: StateProbe match=0 next=2
+1: StateProbe match=0 next=0
+2: StateProbe match=0 next=1
 
 simple
 u1 u2 u3 u1 u2 u3
 ----
 voters=(1 2)
-1: StateProbe match=0 next=1
-2: StateProbe match=0 next=2
+1: StateProbe match=0 next=0
+2: StateProbe match=0 next=1

raft/confchange/testdata/zero.txt

@@ -3,4 +3,4 @@ simple
 v1 r0 v0 l0
 ----
 voters=(1)
-1: StateProbe match=0 next=1
+1: StateProbe match=0 next=0