backupccl: retry restore's AdminSplit on ephemeral errors

Informs cockroachdb#84635,cockroachdb#84162

Release note: none

pkg/ccl/backupccl/split_and_scatter_processor.go

@@ -18,6 +18,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/jobs/jobspb"
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/kv"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvserver"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/execinfra"
@@ -27,8 +28,10 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/sql/types"
 	"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
+	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
+	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/cockroachdb/cockroach/pkg/util/stop"
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/logtags"
@@ -95,13 +98,46 @@ func (s dbSplitAndScatterer) split(
 		newSplitKey = splitAt
 	}
 	log.VEventf(ctx, 1, "presplitting new key %+v", newSplitKey)
-	if err := s.db.AdminSplit(ctx, newSplitKey, expirationTime); err != nil {
+	if err := s.adminSplitWithRetry(ctx, newSplitKey, expirationTime); err != nil {
 		return errors.Wrapf(err, "splitting key %s", newSplitKey)
 	}
 
 	return nil
 }
 
+func (s dbSplitAndScatterer) adminSplitWithRetry(
+	ctx context.Context, newSplitKey roachpb.Key, expirationTime hlc.Timestamp,
+) error {
+
+	retryOpts := retry.Options{
+		MaxBackoff: 1 * time.Second,
+		MaxRetries: 5,
+	}
+
+	var err error
+	for r := retry.StartWithCtx(ctx, retryOpts); r.Next(); {
+
+		if err == nil {
+			break
+		}
+		err := s.db.AdminSplit(ctx, newSplitKey, expirationTime)
+
+		if !(kvserver.IsRetriableReplicationChangeError(err)) {
+			return err
+		}
+		log.Warningf(ctx, `encountered retryable error: %+v`, err)
+	}
+
+	// We have exhausted retries, but we have not seen a permanent error, so
+	// it is possible that this is a transient error that is taking longer than
+	// our configured retry to go away.
+	if err != nil {
+		return errors.Wrapf(err, "exhausted split retries with latest error")
+	}
+
+	return nil
+}
+
 // scatter implements splitAndScatterer.
 func (s dbSplitAndScatterer) scatter(
 	ctx context.Context, codec keys.SQLCodec, scatterKey roachpb.Key,