raft: grow the inflights buffer instead of preallocating

Grow the inflights buffer as needed instead of preallocating it to its
max size. This avoids preallocating a lot of unnecessary
space (8*MaxInflightMsgs) when using lots of raft groups while still
allowing for a reasonable MaxInflightMsgs configuration.

raft/progress.go

@@ -189,8 +189,7 @@ type inflights struct {
 
 func newInflights(size int) *inflights {
 	return &inflights{
-		size:   size,
-		buffer: make([]uint64, size),
+		size: size,
 	}
 }
 
@@ -203,10 +202,28 @@ func (in *inflights) add(inflight uint64) {
 	if next >= in.size {
 		next -= in.size
 	}
+	if next >= len(in.buffer) {
+		in.growBuf()
+	}
 	in.buffer[next] = inflight
 	in.count++
 }
 
+// grow the inflight buffer by doubling up to inflights.size. We grow on demand
+// instead of preallocating to inflights.size to handle systems which have
+// thousands of Raft groups per process.
+func (in *inflights) growBuf() {
+	newSize := len(in.buffer) * 2
+	if newSize == 0 {
+		newSize = 1
+	} else if newSize > in.size {
+		newSize = in.size
+	}
+	newBuffer := make([]uint64, newSize)
+	copy(newBuffer, in.buffer)
+	in.buffer = newBuffer
+}
+
 // freeTo frees the inflights smaller or equal to the given `to` flight.
 func (in *inflights) freeTo(to uint64) {
 	if in.count == 0 || to < in.buffer[in.start] {
@@ -228,6 +245,11 @@ func (in *inflights) freeTo(to uint64) {
 	// free i inflights and set new start index
 	in.count -= i
 	in.start = idx
+	if in.count == 0 {
+		// inflights is empty, reset the start index so that we don't grow the
+		// buffer unnecessarily.
+		in.start = 0
+	}
 }
 
 func (in *inflights) freeFirstOne() { in.freeTo(in.buffer[in.start]) }

raft/progress_test.go

@@ -155,10 +155,10 @@ func TestInflightFreeTo(t *testing.T) {
 	in.freeTo(14)
 
 	wantIn4 := &inflights{
-		start: 5,
+		start: 0,
 		count: 0,
 		size:  10,
-		//                                   ↓
+		//               ↓
 		buffer: []uint64{10, 11, 12, 13, 14, 5, 6, 7, 8, 9},
 	}