diff --git a/s3/s3.go b/s3/s3.go index 8814412..6a490fd 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -66,9 +66,7 @@ var DefaultConfig = Config{ MaxIdleConnsPerHost: 100, MaxConnsPerHost: 0, }, - // Minimum file size after which an HTTP multipart request should be used to upload objects to storage. - // Set to 128 MiB as in the minio client. - PartSize: 1024 * 1024 * 128, + PartSize: 1024 * 1024 * 64, // 64MB. } // Config stores the configuration for s3 bucket. @@ -85,6 +83,7 @@ type Config struct { TraceConfig TraceConfig `yaml:"trace"` ListObjectsVersion string `yaml:"list_objects_version"` // PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize. + // NOTE we need to make sure this number does not produce more parts than 10 000. PartSize uint64 `yaml:"part_size"` SSEConfig SSEConfig `yaml:"sse_config"` } @@ -449,7 +448,6 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { size = -1 } - // partSize cannot be larger than object size. partSize := b.partSize if size < int64(partSize) { partSize = 0 diff --git a/s3/s3_e2e_test.go b/s3/s3_e2e_test.go new file mode 100644 index 0000000..e837b9b --- /dev/null +++ b/s3/s3_e2e_test.go @@ -0,0 +1,55 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package s3_test + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/cortexproject/cortex/integration/e2e" + e2edb "github.com/cortexproject/cortex/integration/e2e/db" + "github.com/go-kit/kit/log" + "github.com/thanos-io/thanos/pkg/objstore/s3" + "github.com/thanos-io/thanos/test/e2e/e2ethanos" + + "github.com/thanos-io/thanos/pkg/testutil" +) + +// Regression benchmark for https://github.com/thanos-io/thanos/issues/3917. +func BenchmarkUpload(b *testing.B) { + b.ReportAllocs() + ctx := context.Background() + + s, err := e2e.NewScenario("e2e_bench_mino_client") + testutil.Ok(b, err) + b.Cleanup(e2ethanos.CleanScenario(b, s)) + + const bucket = "test" + m := e2edb.NewMinio(8080, bucket) + testutil.Ok(b, s.StartAndWaitReady(m)) + + bkt, err := s3.NewBucketWithConfig(log.NewNopLogger(), s3.Config{ + Bucket: bucket, + AccessKey: e2edb.MinioAccessKey, + SecretKey: e2edb.MinioSecretKey, + Endpoint: m.HTTPEndpoint(), + Insecure: true, + }, "test-feed") + testutil.Ok(b, err) + + buf := bytes.Buffer{} + buf.Grow(1028 * 1028 * 100) // 100MB. + word := "abcdefghij" + for i := 0; i < buf.Cap()/len(word); i++ { + _, _ = buf.WriteString(word) + } + str := buf.String() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + testutil.Ok(b, bkt.Upload(ctx, "test", strings.NewReader(str))) + } +} diff --git a/s3/s3_test.go b/s3/s3_test.go index 2cac107..b084473 100644 --- a/s3/s3_test.go +++ b/s3/s3_test.go @@ -208,7 +208,7 @@ http_config: cfg, err := parseConfig(input) testutil.Ok(t, err) - testutil.Assert(t, cfg.PartSize == 1024*1024*128, "when part size not set it should default to 128MiB") + testutil.Assert(t, cfg.PartSize == 1024*1024*64, "when part size not set it should default to 128MiB") input2 := []byte(`bucket: "bucket-name" endpoint: "s3-endpoint"