From edc9721ccc1fe4b5dd5e88fed43ce1e12c12615e Mon Sep 17 00:00:00 2001
From: Peter Mattis <petermattis@gmail.com>
Date: Sat, 5 Nov 2016 17:14:10 -0400
Subject: [PATCH] storage: preserve raft log during up-replication

Preserve the raft log, no matter how large it grows, during
up-replication.

Fixes #10409
---
 pkg/storage/raft_log_queue.go      | 27 +++++++++++++++++----------
 pkg/storage/raft_log_queue_test.go |  3 +++
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/pkg/storage/raft_log_queue.go b/pkg/storage/raft_log_queue.go
index ffeac1540680..56ae7d9b6b00 100644
--- a/pkg/storage/raft_log_queue.go
+++ b/pkg/storage/raft_log_queue.go
@@ -142,18 +142,25 @@ func computeTruncatableIndex(
 		// the target size. If the raft log is greater than the target size we
 		// always truncate to the quorum commit index.
 		truncatableIndex = getBehindIndex(raftStatus)
-		// The pending snapshot index acts as a placeholder for a replica that is
-		// about to be added to the range. We don't want to truncate the log in a
-		// way that will require that new replica to be caught up via a Raft
-		// snapshot.
-		if pendingSnapshotIndex > 0 && truncatableIndex > pendingSnapshotIndex {
-			truncatableIndex = pendingSnapshotIndex
-		}
-		if truncatableIndex < firstIndex {
-			truncatableIndex = firstIndex
-		}
 	}
 
+	// The pending snapshot index acts as a placeholder for a replica that is
+	// about to be added to the range. We don't want to truncate the log in a
+	// way that will require that new replica to be caught up via a Raft
+	// snapshot.
+	//
+	// TODO(peter): We only need to preserve the pending snapshot index in cases
+	// where adding the new replica can knock the range out of quorum. For
+	// example, if the range currently has 1 replica and we're adding a 2nd, the
+	// range will be unable to make forward progress until the 2nd replica is
+	// brought up to date. Snapshots are throttled and not always available while
+	// Raft log replays are.
+	if pendingSnapshotIndex > 0 && truncatableIndex > pendingSnapshotIndex {
+		truncatableIndex = pendingSnapshotIndex
+	}
+	if truncatableIndex < firstIndex {
+		truncatableIndex = firstIndex
+	}
 	// Never truncate past the quorum committed index.
 	if truncatableIndex > raftStatus.Commit {
 		truncatableIndex = raftStatus.Commit
diff --git a/pkg/storage/raft_log_queue_test.go b/pkg/storage/raft_log_queue_test.go
index 7cb03d2bb062..d5886cf34c89 100644
--- a/pkg/storage/raft_log_queue_test.go
+++ b/pkg/storage/raft_log_queue_test.go
@@ -87,6 +87,9 @@ func TestComputeTruncatableIndex(t *testing.T) {
 		{[]uint64{1, 2, 3, 4}, 3, 2000, 1, 0, 3},
 		{[]uint64{1, 2, 3, 4}, 3, 2000, 2, 0, 3},
 		{[]uint64{1, 2, 3, 4}, 3, 2000, 3, 0, 3},
+		// Don't truncate past the pending snapshot index
+		{[]uint64{1, 2, 3, 4}, 3, 2000, 1, 1, 1},
+		{[]uint64{1, 2, 3, 4}, 3, 2000, 1, 2, 2},
 		// Never truncate past raftStatus.Commit.
 		{[]uint64{4, 5, 6}, 3, 100, 4, 0, 3},
 	}