cloud/amazon: add toggle for s3 storage to list prefix with additiona…

…l marker

Add toggle for s3 storage to list prefixes with a paging marker that's the
prefix with an additional `/`. This allows certain s3 clones which return
`s3://<prefix>/` as the first result of listing `s3://<prefix>` to exclude that
result.

This behavior is toggled via setting the environment variable
`COCKROACH_S3_LIST_WITH_PREFIX_SLASH_MARKER` to true.

Release note: None

pkg/cloud/amazon/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "//pkg/settings",
         "//pkg/settings/cluster",
         "//pkg/util/contextutil",
+        "//pkg/util/envutil",
         "//pkg/util/ioctx",
         "//pkg/util/log",
         "//pkg/util/syncutil",

pkg/cloud/amazon/s3_storage.go

@@ -36,6 +36,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/settings"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/util/contextutil"
+	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/ioctx"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
@@ -622,8 +623,15 @@ func (s *s3Storage) List(ctx context.Context, prefix, delim string, fn cloud.Lis
 		return true
 	}
 
+	var s3Input *s3.ListObjectsInput
+	if envutil.EnvOrDefaultBool("COCKROACH_S3_LIST_WITH_PREFIX_SLASH_MARKER", false) {
+		s3Input = &s3.ListObjectsInput{Bucket: s.bucket, Prefix: aws.String(dest), Delimiter: nilIfEmpty(delim), Marker: aws.String(dest + "/")}
+	} else {
+		s3Input = &s3.ListObjectsInput{Bucket: s.bucket, Prefix: aws.String(dest), Delimiter: nilIfEmpty(delim)}
+	}
+
 	if err := client.ListObjectsPagesWithContext(
-		ctx, &s3.ListObjectsInput{Bucket: s.bucket, Prefix: aws.String(dest), Delimiter: nilIfEmpty(delim)}, pageFn,
+		ctx, s3Input, pageFn,
 	); err != nil {
 		return errors.Wrap(err, `failed to list s3 bucket`)
 	}