Merge cockroachdb#27925

27925: storage: retry failed liveness heartbeats in tests r=nvanbenschoten a=benesch Teach multiTestContext.heartbeatLiveness to retry heartbeats that fail due to ErrEpochIncremented. This error is expected when the heartbeat races with an epoch increment from another store. Fix cockroachdb#27835. Release note: None --- @nvanbenschoten you mind taking this one? Should be real quick. Co-authored-by: Nikhil Benesch <[email protected]>
a-robinson · Jul 25, 2018 · 65f7aba · 65f7aba
2 parents 299efee + 001ce1f
commit 65f7aba
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/pkg/storage/client_test.go b/pkg/storage/client_test.go
@@ -1207,7 +1207,13 @@ func (m *multiTestContext) heartbeatLiveness(ctx context.Context, store int) err
 	if err != nil {
 		return err
 	}
-	return nl.Heartbeat(ctx, l)
+
+	for r := retry.StartWithCtx(ctx, retry.Options{MaxRetries: 5}); r.Next(); {
+		if err = nl.Heartbeat(ctx, l); err != storage.ErrEpochIncremented {
+			break
+		}
+	}
+	return err
 }
 
 // advanceClock advances the mtc's manual clock such that all