Merge pull request #100100 from erikgrinaker/backport22.2-96632-98491

release-22.2: roachtest: maybe deflake `change-replicas/mixed-version`

pkg/cmd/roachtest/tests/mixed_version_change_replicas.go

@@ -21,6 +21,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
 	"github.com/cockroachdb/cockroach/pkg/util/randutil"
+	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/stretchr/testify/require"
 )
 
@@ -119,8 +120,15 @@ func runChangeReplicasMixedVersion(ctx context.Context, t test.Test, c cluster.C
 				conn := u.c.Conn(ctx, t.L(), gateway)
 				defer conn.Close()
 
+				retryOpts := retry.Options{
+					InitialBackoff: 100 * time.Millisecond,
+					MaxBackoff:     5 * time.Second,
+					Multiplier:     2,
+					MaxRetries:     12,
+				}
 				var rangeErrors map[int]string
-				for attempt := 1; attempt <= 5; attempt++ {
+				for r := retry.StartWithCtx(ctx, retryOpts); r.Next(); {
+					setReplicateQueueEnabled(false)
 					if errCount := len(rangeErrors); errCount > 0 {
 						t.L().Printf("%d ranges failed, retrying", errCount)
 					}
@@ -145,6 +153,10 @@ func runChangeReplicasMixedVersion(ctx context.Context, t test.Test, c cluster.C
 					if len(rangeErrors) == 0 {
 						break
 					}
+					// The failure may be caused by conflicts with ongoing configuration
+					// changes by the replicate queue, so we re-enable it and let it run
+					// for a bit before the next retry.
+					setReplicateQueueEnabled(true)
 				}
 
 				if len(rangeErrors) > 0 {