Skip to content

Commit

Permalink
storage: improve split-snapshot warning
Browse files Browse the repository at this point in the history
It now differentiates between the case in which the follower needs a
snapshot vs the one in which it is still being probed. We're not
expecting the probing case once the bug described here is fixed:

cockroachdb#32594 (comment)

Release note: None
  • Loading branch information
tbg committed Dec 11, 2018
1 parent 64f2725 commit a94102a
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 4 deletions.
21 changes: 19 additions & 2 deletions pkg/storage/replica_command.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,8 +213,25 @@ func splitSnapshotWarningStr(rangeID roachpb.RangeID, status *raft.Status) strin
// https://github.com/etcd-io/etcd/pull/10279
continue
}
if pr.State != raft.ProgressStateReplicate {
s += fmt.Sprintf("; may cause Raft snapshot to r%d/%d: %v", rangeID, replicaID, &pr)
if pr.State == raft.ProgressStateReplicate {
// This follower is in good working order.
continue
}
s += fmt.Sprintf("; r%d/%d is ", rangeID, replicaID)
switch pr.State {
case raft.ProgressStateSnapshot:
// If the Raft snapshot queue is backed up, replicas can spend
// minutes or worse until they are caught up.
s += "waiting for a Raft snapshot"
case raft.ProgressStateProbe:
// Assuming the split has already been delayed for a little bit,
// seeing a follower that is probing hints at some problem with
// Raft or Raft message delivery. (Of course it's possible that
// the follower *just* entered probing state).
s += "being probed (may or may not need a Raft snapshot)"
default:
// Future proofing.
s += "in unknown state " + pr.State.String()
}
}
}
Expand Down
11 changes: 9 additions & 2 deletions pkg/storage/replica_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10886,8 +10886,15 @@ func TestSplitSnapshotWarningStr(t *testing.T) {

assert.Equal(
t,
"; may cause Raft snapshot to r12/2: next = 0, match = 100, state = ProgressStateProbe,"+
" waiting = false, pendingSnapshot = 0",
"; r12/2 is being probed (may or may not need a Raft snapshot)",
splitSnapshotWarningStr(12, status),
)

pr.State = raft.ProgressStateSnapshot

assert.Equal(
t,
"; r12/2 is being probed (may or may not need a Raft snapshot)",
splitSnapshotWarningStr(12, status),
)
}

0 comments on commit a94102a

Please sign in to comment.