From 084358fc639834668eb8a190e10a3badeebd22b3 Mon Sep 17 00:00:00 2001 From: Rui Hu Date: Tue, 2 Aug 2022 17:05:51 -0400 Subject: [PATCH] cloud/amazon: add toggle for s3 storage to list prefix with additional marker Add toggle for s3 storage to list prefixes with a paging marker that's the prefix with an additional `/`. This allows certain s3 clones which return `s3:///` as the first result of listing `s3://` to exclude that result. This behavior is toggled via setting the environment variable `COCKROACH_S3_LIST_WITH_PREFIX_SLASH_MARKER` to true. Release note: None --- pkg/cloud/amazon/BUILD.bazel | 1 + pkg/cloud/amazon/s3_storage.go | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/cloud/amazon/BUILD.bazel b/pkg/cloud/amazon/BUILD.bazel index 615043d63766..b3431bc10de4 100644 --- a/pkg/cloud/amazon/BUILD.bazel +++ b/pkg/cloud/amazon/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "//pkg/settings", "//pkg/settings/cluster", "//pkg/util/contextutil", + "//pkg/util/envutil", "//pkg/util/ioctx", "//pkg/util/log", "//pkg/util/syncutil", diff --git a/pkg/cloud/amazon/s3_storage.go b/pkg/cloud/amazon/s3_storage.go index 53257103cb4b..72e6364d2a06 100644 --- a/pkg/cloud/amazon/s3_storage.go +++ b/pkg/cloud/amazon/s3_storage.go @@ -36,6 +36,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/util/contextutil" + "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/ioctx" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/syncutil" @@ -622,8 +623,15 @@ func (s *s3Storage) List(ctx context.Context, prefix, delim string, fn cloud.Lis return true } + var s3Input *s3.ListObjectsInput + if envutil.EnvOrDefaultBool("COCKROACH_S3_LIST_WITH_PREFIX_SLASH_MARKER", false) { + s3Input = &s3.ListObjectsInput{Bucket: s.bucket, Prefix: aws.String(dest), Delimiter: nilIfEmpty(delim), Marker: aws.String(dest + "/")} + } else { + s3Input = &s3.ListObjectsInput{Bucket: s.bucket, Prefix: aws.String(dest), Delimiter: nilIfEmpty(delim)} + } + if err := client.ListObjectsPagesWithContext( - ctx, &s3.ListObjectsInput{Bucket: s.bucket, Prefix: aws.String(dest), Delimiter: nilIfEmpty(delim)}, pageFn, + ctx, s3Input, pageFn, ); err != nil { return errors.Wrap(err, `failed to list s3 bucket`) }