diff --git a/pkg/kv/kvserver/raft_transport.go b/pkg/kv/kvserver/raft_transport.go index d0a5ab57a79a..3affd0f91d71 100644 --- a/pkg/kv/kvserver/raft_transport.go +++ b/pkg/kv/kvserver/raft_transport.go @@ -605,6 +605,9 @@ func (t *RaftTransport) SendAsync( } return true default: + if logRaftSendQueueFullEvery.ShouldLog() { + log.Warningf(t.AnnotateCtx(context.Background()), "raft send queue to n%d is full", toNodeID) + } releaseRaftMessageRequest(req) return false } diff --git a/pkg/kv/kvserver/store_raft.go b/pkg/kv/kvserver/store_raft.go index 3c930cddefc7..e8fd803cc4be 100644 --- a/pkg/kv/kvserver/store_raft.go +++ b/pkg/kv/kvserver/store_raft.go @@ -28,6 +28,11 @@ import ( "go.etcd.io/etcd/raft/v3/raftpb" ) +var ( + logRaftRecvQueueFullEvery = log.Every(1 * time.Second) + logRaftSendQueueFullEvery = log.Every(1 * time.Second) +) + type raftRequestInfo struct { req *kvserverpb.RaftMessageRequest respStream RaftMessageResponseStream @@ -177,6 +182,9 @@ func (s *Store) HandleRaftUncoalescedRequest( // TODO(peter): Return an error indicating the request was dropped. Note // that dropping the request is safe. Raft will retry. s.metrics.RaftRcvdMsgDropped.Inc(1) + if logRaftRecvQueueFullEvery.ShouldLog() { + log.Warningf(ctx, "raft receive queue for r%d is full", req.RangeID) + } return false } q.infos = append(q.infos, raftRequestInfo{ @@ -540,9 +548,9 @@ func (s *Store) processTick(_ context.Context, rangeID roachpb.RangeID) bool { // See the comment in shouldFollowerQuiesceOnNotify for details on how these two // functions combine to provide the guarantee that: // -// If a quorum of replica in a Raft group is alive and at least -// one of these replicas is up-to-date, the Raft group will catch -// up any of the live, lagging replicas. +// If a quorum of replica in a Raft group is alive and at least +// one of these replicas is up-to-date, the Raft group will catch +// up any of the live, lagging replicas. // // Note that this mechanism can race with concurrent invocations of processTick, // which may have a copy of the previous livenessMap where the now-live node is