Merge #85493

85493: cloud/amazon: add toggle for s3 storage to list prefix with additiona… r=rhu713 a=rhu713 …l marker Add toggle for s3 storage to list prefixes with a paging marker that's the prefix with an additional `/`. This allows certain s3 clones which return `s3://<prefix>/` as the first result of listing `s3://<prefix>` to exclude that result. This behavior is toggled via setting the environment variable `COCKROACH_S3_LIST_WITH_PREFIX_SLASH_MARKER` to true. Release note: None Co-authored-by: Rui Hu <[email protected]>
cockroachdb · Aug 15, 2022 · 91d47e8 · 91d47e8
2 parents 4561f22 + 23b3a70
commit 91d47e8
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 1 deletion.
diff --git a/pkg/cloud/amazon/BUILD.bazel b/pkg/cloud/amazon/BUILD.bazel
@@ -22,6 +22,7 @@ go_library(
         "//pkg/settings",
         "//pkg/settings/cluster",
         "//pkg/util/contextutil",
+        "//pkg/util/envutil",
         "//pkg/util/ioctx",
         "//pkg/util/log",
         "//pkg/util/syncutil",

diff --git a/pkg/cloud/amazon/s3_storage.go b/pkg/cloud/amazon/s3_storage.go
@@ -36,6 +36,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/settings"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/util/contextutil"
+	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/ioctx"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
@@ -678,8 +679,19 @@ func (s *s3Storage) List(ctx context.Context, prefix, delim string, fn cloud.Lis
 		return true
 	}
 
+	var s3Input *s3.ListObjectsInput
+	// Add an environment variable toggle for s3 storage to list prefixes with a
+	// paging marker that's the prefix with an additional /. This allows certain
+	// s3 clones which return s3://<prefix>/ as the first result of listing
+	// s3://<prefix> to exclude that result.
+	if envutil.EnvOrDefaultBool("COCKROACH_S3_LIST_WITH_PREFIX_SLASH_MARKER", false) {
+		s3Input = &s3.ListObjectsInput{Bucket: s.bucket, Prefix: aws.String(dest), Delimiter: nilIfEmpty(delim), Marker: aws.String(dest + "/")}
+	} else {
+		s3Input = &s3.ListObjectsInput{Bucket: s.bucket, Prefix: aws.String(dest), Delimiter: nilIfEmpty(delim)}
+	}
+
 	if err := client.ListObjectsPagesWithContext(
-		ctx, &s3.ListObjectsInput{Bucket: s.bucket, Prefix: aws.String(dest), Delimiter: nilIfEmpty(delim)}, pageFn,
+		ctx, s3Input, pageFn,
 	); err != nil {
 		return errors.Wrap(err, `failed to list s3 bucket`)
 	}