diff --git a/pkg/cmd/roachtest/tests/mixed_version_change_replicas.go b/pkg/cmd/roachtest/tests/mixed_version_change_replicas.go index 7a7944641dd7..ff73271b7c75 100644 --- a/pkg/cmd/roachtest/tests/mixed_version_change_replicas.go +++ b/pkg/cmd/roachtest/tests/mixed_version_change_replicas.go @@ -21,6 +21,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/util/randutil" + "github.com/cockroachdb/cockroach/pkg/util/retry" "github.com/stretchr/testify/require" ) @@ -119,8 +120,15 @@ func runChangeReplicasMixedVersion(ctx context.Context, t test.Test, c cluster.C conn := u.c.Conn(ctx, t.L(), gateway) defer conn.Close() + retryOpts := retry.Options{ + InitialBackoff: 100 * time.Millisecond, + MaxBackoff: 5 * time.Second, + Multiplier: 2, + MaxRetries: 12, + } var rangeErrors map[int]string - for attempt := 1; attempt <= 5; attempt++ { + for r := retry.StartWithCtx(ctx, retryOpts); r.Next(); { + setReplicateQueueEnabled(false) if errCount := len(rangeErrors); errCount > 0 { t.L().Printf("%d ranges failed, retrying", errCount) } @@ -145,6 +153,10 @@ func runChangeReplicasMixedVersion(ctx context.Context, t test.Test, c cluster.C if len(rangeErrors) == 0 { break } + // The failure may be caused by conflicts with ongoing configuration + // changes by the replicate queue, so we re-enable it and let it run + // for a bit before the next retry. + setReplicateQueueEnabled(true) } if len(rangeErrors) > 0 {