Skip to content

Commit

Permalink
kvserver: never pause local replica
Browse files Browse the repository at this point in the history
As observed in cockroachdb#84884, an overloaded store that held leases could end up
"pausing" replication traffic to itself. This (likely) had no practical
effect since the leader never sends messages to itself, but it meant
reporting bogus counts of paused replicas.

This commit ensures that a raft leader will never pause itself.

Release note: None
  • Loading branch information
tbg committed Aug 8, 2022
1 parent dfac2ff commit e4ae047
Show file tree
Hide file tree
Showing 3 changed files with 11 additions and 2 deletions.
5 changes: 3 additions & 2 deletions pkg/kv/kvserver/replica_raft_overload.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ var pauseReplicationIOThreshold = settings.RegisterFloatSetting(
)

type computeExpendableOverloadedFollowersInput struct {
self roachpb.ReplicaID
replDescs roachpb.ReplicaSet
// TODO(tbg): all entries are overloaded, so consdier removing the IOThreshold here
// because it's confusing.
Expand Down Expand Up @@ -104,11 +105,10 @@ func computeExpendableOverloadedFollowers(
var nonLive map[roachpb.ReplicaID]nonLiveReason
var liveOverloadedVoterCandidates map[roachpb.ReplicaID]struct{}
var liveOverloadedNonVoterCandidates map[roachpb.ReplicaID]struct{}

var prs map[uint64]tracker.Progress

for _, replDesc := range d.replDescs.AsProto() {
if _, overloaded := d.ioOverloadMap[replDesc.StoreID]; !overloaded {
if _, overloaded := d.ioOverloadMap[replDesc.StoreID]; !overloaded || replDesc.ReplicaID == d.self {
continue
}
// There's at least one overloaded follower, so initialize
Expand Down Expand Up @@ -252,6 +252,7 @@ func (r *Replica) updatePausedFollowersLocked(
seed := int64(r.RangeID)
now := r.store.Clock().Now().GoTime()
d := computeExpendableOverloadedFollowersInput{
self: r.replicaID,
replDescs: r.descRLocked().Replicas(),
ioOverloadMap: ioOverloadMap,
getProgressMap: func(_ context.Context) map[uint64]tracker.Progress {
Expand Down
4 changes: 4 additions & 0 deletions pkg/kv/kvserver/replica_raft_overload_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ func TestReplicaRaftOverload_computeExpendableOverloadedFollowers(t *testing.T)
require.Equal(t, "run", d.Cmd)
var seed uint64
var replDescs roachpb.ReplicaSet
var self roachpb.ReplicaID
ioOverloadMap := map[roachpb.StoreID]*admissionpb.IOThreshold{}
snapshotMap := map[roachpb.ReplicaID]struct{}{}
downMap := map[roachpb.ReplicaID]struct{}{}
Expand All @@ -65,6 +66,8 @@ func TestReplicaRaftOverload_computeExpendableOverloadedFollowers(t *testing.T)
switch arg.Key {
case "min-live-match-index":
minLiveMatchIndex = id
case "self":
self = roachpb.ReplicaID(id)
case "voters", "learners":
replicaID := roachpb.ReplicaID(id)
if matchS != "" {
Expand Down Expand Up @@ -134,6 +137,7 @@ func TestReplicaRaftOverload_computeExpendableOverloadedFollowers(t *testing.T)
}

m, _ := computeExpendableOverloadedFollowers(ctx, computeExpendableOverloadedFollowersInput{
self: self,
replDescs: replDescs,
ioOverloadMap: ioOverloadMap,
getProgressMap: getProgressMap,
Expand Down
4 changes: 4 additions & 0 deletions pkg/kv/kvserver/testdata/replica_raft_overload/self.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Won't consider itself for pausing.
run voters=(1,2,3) overloaded=(1) self=1
----
[]

0 comments on commit e4ae047

Please sign in to comment.