diff --git a/pkg/kv/kvserver/raft_transport.go b/pkg/kv/kvserver/raft_transport.go index 566a834715e0..b3bd100ea3bd 100644 --- a/pkg/kv/kvserver/raft_transport.go +++ b/pkg/kv/kvserver/raft_transport.go @@ -505,6 +505,9 @@ func (t *RaftTransport) SendAsync( case ch <- req: return true default: + if logRaftSendQueueFullEvery.ShouldLog() { + log.Warningf(t.AnnotateCtx(context.Background()), "raft send queue to n%d is full", toNodeID) + } releaseRaftMessageRequest(req) return false } diff --git a/pkg/kv/kvserver/store_raft.go b/pkg/kv/kvserver/store_raft.go index 92164cd63f8e..eefd61f34f95 100644 --- a/pkg/kv/kvserver/store_raft.go +++ b/pkg/kv/kvserver/store_raft.go @@ -32,6 +32,11 @@ import ( "go.etcd.io/etcd/raft/v3/raftpb" ) +var ( + logRaftRecvQueueFullEvery = log.Every(1 * time.Second) + logRaftSendQueueFullEvery = log.Every(1 * time.Second) +) + type raftRequestInfo struct { req *kvserverpb.RaftMessageRequest size int64 // size of req in bytes @@ -305,6 +310,9 @@ func (s *Store) HandleRaftUncoalescedRequest( // that dropping the request is safe. Raft will retry. s.metrics.RaftRcvdDropped.Inc(1) s.metrics.RaftRcvdDroppedBytes.Inc(size) + if logRaftRecvQueueFullEvery.ShouldLog() { + log.Warningf(ctx, "raft receive queue for r%d is full", req.RangeID) + } return false } return enqueue