Added benchmark, Moved minio-deps to fork without race fix we don't n…

…eed. (#3968) Fixes: thanos-io/thanos#3917 Long term fix: thanos-io/thanos#3967 Signed-off-by: Bartlomiej Plotka <[email protected]>
brancz · Mar 24, 2021 · 0e9931f · 0e9931f
1 parent 2a99980
commit 0e9931f
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 5 deletions.
diff --git a/s3/s3.go b/s3/s3.go
@@ -66,9 +66,7 @@ var DefaultConfig = Config{
 		MaxIdleConnsPerHost:   100,
 		MaxConnsPerHost:       0,
 	},
-	// Minimum file size after which an HTTP multipart request should be used to upload objects to storage.
-	// Set to 128 MiB as in the minio client.
-	PartSize: 1024 * 1024 * 128,
+	PartSize: 1024 * 1024 * 64, // 64MB.
 }
 
 // Config stores the configuration for s3 bucket.
@@ -85,6 +83,7 @@ type Config struct {
 	TraceConfig        TraceConfig       `yaml:"trace"`
 	ListObjectsVersion string            `yaml:"list_objects_version"`
 	// PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize.
+	// NOTE we need to make sure this number does not produce more parts than 10 000.
 	PartSize  uint64    `yaml:"part_size"`
 	SSEConfig SSEConfig `yaml:"sse_config"`
 }
@@ -449,7 +448,6 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error {
 		size = -1
 	}
 
-	// partSize cannot be larger than object size.
 	partSize := b.partSize
 	if size < int64(partSize) {
 		partSize = 0

diff --git a/s3/s3_e2e_test.go b/s3/s3_e2e_test.go
@@ -0,0 +1,55 @@
+// Copyright (c) The Thanos Authors.
+// Licensed under the Apache License 2.0.
+
+package s3_test
+
+import (
+	"bytes"
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/cortexproject/cortex/integration/e2e"
+	e2edb "github.com/cortexproject/cortex/integration/e2e/db"
+	"github.com/go-kit/kit/log"
+	"github.com/thanos-io/thanos/pkg/objstore/s3"
+	"github.com/thanos-io/thanos/test/e2e/e2ethanos"
+
+	"github.com/thanos-io/thanos/pkg/testutil"
+)
+
+// Regression benchmark for https://github.com/thanos-io/thanos/issues/3917.
+func BenchmarkUpload(b *testing.B) {
+	b.ReportAllocs()
+	ctx := context.Background()
+
+	s, err := e2e.NewScenario("e2e_bench_mino_client")
+	testutil.Ok(b, err)
+	b.Cleanup(e2ethanos.CleanScenario(b, s))
+
+	const bucket = "test"
+	m := e2edb.NewMinio(8080, bucket)
+	testutil.Ok(b, s.StartAndWaitReady(m))
+
+	bkt, err := s3.NewBucketWithConfig(log.NewNopLogger(), s3.Config{
+		Bucket:    bucket,
+		AccessKey: e2edb.MinioAccessKey,
+		SecretKey: e2edb.MinioSecretKey,
+		Endpoint:  m.HTTPEndpoint(),
+		Insecure:  true,
+	}, "test-feed")
+	testutil.Ok(b, err)
+
+	buf := bytes.Buffer{}
+	buf.Grow(1028 * 1028 * 100) // 100MB.
+	word := "abcdefghij"
+	for i := 0; i < buf.Cap()/len(word); i++ {
+		_, _ = buf.WriteString(word)
+	}
+	str := buf.String()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		testutil.Ok(b, bkt.Upload(ctx, "test", strings.NewReader(str)))
+	}
+}
diff --git a/s3/s3_test.go b/s3/s3_test.go
@@ -208,7 +208,7 @@ http_config:
 
 	cfg, err := parseConfig(input)
 	testutil.Ok(t, err)
-	testutil.Assert(t, cfg.PartSize == 1024*1024*128, "when part size not set it should default to 128MiB")
+	testutil.Assert(t, cfg.PartSize == 1024*1024*64, "when part size not set it should default to 128MiB")
 
 	input2 := []byte(`bucket: "bucket-name"
 endpoint: "s3-endpoint"