Merge pull request #10218 from bdarnell/prevote

storage: Use raft PreVote instead of CheckQuorum

GLOCKFILE

@@ -46,7 +46,7 @@ github.com/cockroachdb/crlfmt f4be2332630e8f33c3262c0d5f0d13d733c8bedf
 github.com/cockroachdb/pq 44a6473ebbc26e3af09fe57bbdf761475c2c9f7c
 github.com/cockroachdb/stress 029c9348806514969d1109a6ae36e521af411ca7
 github.com/codahale/hdrhistogram 3a0bb77429bd3a61596f5e8a3172445844342120
-github.com/coreos/etcd 5c60478953b46e91074a54f5f6683b8afbef8968
+github.com/coreos/etcd 99539ff031fd2e9c590401d5943a595d191552f1
 github.com/cpuguy83/go-md2man a65d4d2de4d5f7c74868dfa9b202a3c8be315aaa
 github.com/docker/distribution d0cdc4802b80609f3064f43e680d8968daa06f2e
 github.com/docker/docker cd0e399e062b45bb0c5e11c7eb4de933b1ad0564

pkg/storage/metrics.go

@@ -148,6 +148,14 @@ var (
 		Name: "raft.rcvd.voteresp",
 		Help: "Total number of MsgVoteResp messages received by this store",
 	}
+	metaRaftRcvdPreVote = metric.Metadata{
+		Name: "raft.rcvd.prevote",
+		Help: "Total number of MsgPreVote messages received by this store",
+	}
+	metaRaftRcvdPreVoteResp = metric.Metadata{
+		Name: "raft.rcvd.prevoteresp",
+		Help: "Total number of MsgPreVoteResp messages received by this store",
+	}
 	metaRaftRcvdSnap = metric.Metadata{
 		Name: "raft.rcvd.snap",
 		Help: "Total number of MsgSnap messages received by this store",
@@ -377,6 +385,8 @@ type StoreMetrics struct {
 	RaftRcvdMsgAppResp        *metric.Counter
 	RaftRcvdMsgVote           *metric.Counter
 	RaftRcvdMsgVoteResp       *metric.Counter
+	RaftRcvdMsgPreVote        *metric.Counter
+	RaftRcvdMsgPreVoteResp    *metric.Counter
 	RaftRcvdMsgSnap           *metric.Counter
 	RaftRcvdMsgHeartbeat      *metric.Counter
 	RaftRcvdMsgHeartbeatResp  *metric.Counter
@@ -542,6 +552,8 @@ func newStoreMetrics(sampleInterval time.Duration) *StoreMetrics {
 		RaftRcvdMsgAppResp:        metric.NewCounter(metaRaftRcvdAppResp),
 		RaftRcvdMsgVote:           metric.NewCounter(metaRaftRcvdVote),
 		RaftRcvdMsgVoteResp:       metric.NewCounter(metaRaftRcvdVoteResp),
+		RaftRcvdMsgPreVote:        metric.NewCounter(metaRaftRcvdPreVote),
+		RaftRcvdMsgPreVoteResp:    metric.NewCounter(metaRaftRcvdPreVoteResp),
 		RaftRcvdMsgSnap:           metric.NewCounter(metaRaftRcvdSnap),
 		RaftRcvdMsgHeartbeat:      metric.NewCounter(metaRaftRcvdHeartbeat),
 		RaftRcvdMsgHeartbeatResp:  metric.NewCounter(metaRaftRcvdHeartbeatResp),
@@ -636,6 +648,8 @@ func newStoreMetrics(sampleInterval time.Duration) *StoreMetrics {
 	sm.raftRcvdMessages[raftpb.MsgAppResp] = sm.RaftRcvdMsgAppResp
 	sm.raftRcvdMessages[raftpb.MsgVote] = sm.RaftRcvdMsgVote
 	sm.raftRcvdMessages[raftpb.MsgVoteResp] = sm.RaftRcvdMsgVoteResp
+	sm.raftRcvdMessages[raftpb.MsgPreVote] = sm.RaftRcvdMsgPreVote
+	sm.raftRcvdMessages[raftpb.MsgPreVoteResp] = sm.RaftRcvdMsgPreVoteResp
 	sm.raftRcvdMessages[raftpb.MsgSnap] = sm.RaftRcvdMsgSnap
 	sm.raftRcvdMessages[raftpb.MsgHeartbeat] = sm.RaftRcvdMsgHeartbeat
 	sm.raftRcvdMessages[raftpb.MsgHeartbeatResp] = sm.RaftRcvdMsgHeartbeatResp

pkg/storage/replica.go

@@ -2216,26 +2216,6 @@ func (r *Replica) tickRaftMuLocked() (bool, error) {
 		return false, nil
 	}
 	if r.mu.quiescent {
-		// While a replica is quiesced we still advance its logical clock. This is
-		// necessary to avoid a scenario where the leader quiesces and a follower
-		// does not. The follower calls an election but the election fails because
-		// the leader and other follower believe that no time in the current term
-		// has passed. The Raft group is then in a state where one member has a
-		// term that is advanced which will then cause subsequent heartbeats from
-		// the existing leader to be rejected in a way that the leader will step
-		// down. This situation is caused by an interaction between quiescence and
-		// the Raft CheckQuorum feature which relies on the logical clock ticking
-		// at roughly the same rate on all members of the group.
-		//
-		// By ticking the logical clock (incrementing an integer) we avoid this
-		// situation. If one of the followers does not quiesce it will call an
-		// election but the election will succeed. Note that while we expect such
-		// elections from quiesced followers to be extremely rare, it is very
-		// difficult to completely eliminate them so we want to minimize the
-		// disruption when they do occur.
-		//
-		// For more details, see #9372.
-		r.mu.internalRaftGroup.TickQuiesced()
 		return false, nil
 	}
 	if r.maybeQuiesceLocked() {

pkg/storage/store.go

@@ -150,7 +150,7 @@ func newRaftConfig(
 		HeartbeatTick: storeCfg.RaftHeartbeatIntervalTicks,
 		Storage:       strg,
 		Logger:        logger,
-		CheckQuorum:   true,
+		PreVote:       true,
 		// TODO(bdarnell): make these configurable; evaluate defaults.
 		MaxSizePerMsg:   1024 * 1024,
 		MaxInflightMsgs: 256,

pkg/ui/app/js/protos.js

@@ -4377,6 +4377,14 @@ module.exports = require("protobufjs").newBuilder({})['import']({
                         {
                             "name": "MsgReadIndexResp",
                             "id": 16
+                        },
+                        {
+                            "name": "MsgPreVote",
+                            "id": 17
+                        },
+                        {
+                            "name": "MsgPreVoteResp",
+                            "id": 18
                         }
                     ]
                 },

pkg/ui/generated/protos.d.ts

@@ -7566,6 +7566,8 @@ declare module Proto2TypeScript.raftpb {
 		MsgTimeoutNow = 14,
 		MsgReadIndex = 15,
 		MsgReadIndexResp = 16,
+		MsgPreVote = 17,
+		MsgPreVoteResp = 18,
 
 }
 }

pkg/ui/generated/protos.json

@@ -4376,6 +4376,14 @@
                         {
                             "name": "MsgReadIndexResp",
                             "id": 16
+                        },
+                        {
+                            "name": "MsgPreVote",
+                            "id": 17
+                        },
+                        {
+                            "name": "MsgPreVoteResp",
+                            "id": 18
                         }
                     ]
                 },