Bring 0.19.0 rc fixes to main. (#3978)

* tools: Fix partial and empty matchers in rewrite (#3891) * Fix partial and empty matchers match Signed-off-by: yeya24 <[email protected]> * add testcase for non-equal matchers Signed-off-by: yeya24 <[email protected]> * v0.19.0 patch: Added receive benchmark; Fixed Receiver excessive mem usage introduced in 0.17 (#3943) * Added receive benchmark, baseline. ``` goos: linux goarch: amd64 pkg: github.com/thanos-io/thanos/pkg/receive BenchmarkHandlerReceiveHTTP BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them. BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./OK-12 22260 1550152 ns/op 1380340 B/op 6093 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./conflict_errors-12 6619 6430408 ns/op 4522487 B/op 26118 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them. BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./OK-12 2695 17208794 ns/op 15072963 B/op 60441 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./conflict_errors-12 474 72533286 ns/op 46396932 B/op 260141 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 270 137050518 ns/op 226595379 B/op 132 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 21 1616025443 ns/op 698724321 B/op 408 allocs/op PASS Process finished with exit code 0 ``` Signed-off-by: Bartlomiej Plotka <[email protected]> * Copy labels. ``` GOROOT=/home/bwplotka/.gvm/gos/go1.15 #gosetup GOPATH=/home/bwplotka/Repos/thanosgopath #gosetup /home/bwplotka/.gvm/gos/go1.15/bin/go test -c -o /tmp/___BenchmarkHandlerReceiveHTTP_in_github_com_thanos_io_thanos_pkg_receive github.com/thanos-io/thanos/pkg/receive #gosetup /tmp/___BenchmarkHandlerReceiveHTTP_in_github_com_thanos_io_thanos_pkg_receive -test.v -test.bench ^\QBenchmarkHandlerReceiveHTTP\E$ -test.run ^$ -test.benchmem -test.benchtime=30s goos: linux goarch: amd64 pkg: github.com/thanos-io/thanos/pkg/receive BenchmarkHandlerReceiveHTTP BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 25887 1537262 ns/op 1380023 B/op 6092 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 4237 7547968 ns/op 4522583 B/op 26118 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 2205 16513380 ns/op 15071092 B/op 60420 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 525 67278233 ns/op 46396645 B/op 260141 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 285 148049189 ns/op 226596168 B/op 132 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 20 1731361499 ns/op 698722550 B/op 401 allocs/op PASS Process finished with exit code 0 ``` Signed-off-by: Bartlomiej Plotka <[email protected]> * Addded bench., Signed-off-by: Bartlomiej Plotka <[email protected]> * Fix. Signed-off-by: Bartlomiej Plotka <[email protected]> * Improved API. Signed-off-by: Bartlomiej Plotka <[email protected]> * Changelog. Signed-off-by: Bartlomiej Plotka <[email protected]> * Addressed Lucas comments. Signed-off-by: Bartlomiej Plotka <[email protected]> # Conflicts: # CHANGELOG.md * compact: clean up directories thoroughly (#3869) * compact: clean up directories properly I couldn't stop thinking about this code for some reason and I have figured that I had missed one case in #3031. We need to also clean up the directories in compaction groups. A compaction could fail leaving some new directory with a random ULID on the disk. Before attempting to do another compaction loop, we need to remove it as well because the compaction process always produces a new, unique directory. Signed-off-by: Giedrius Statkevičius <[email protected]> * Remove all non expected dirs. Signed-off-by: Bartlomiej Plotka <[email protected]> * Addressed comment. Signed-off-by: Bartlomiej Plotka <[email protected]> Co-authored-by: Bartlomiej Plotka <[email protected]> * Added benchmark, Moved minio-deps to fork without race fix we don't need. (#3968) Fixes: thanos-io/thanos#3917 Long term fix: thanos-io/thanos#3967 Signed-off-by: Bartlomiej Plotka <[email protected]> # Conflicts: # go.sum * Changelog fix. Signed-off-by: Bartlomiej Plotka <[email protected]> * Build fix. Signed-off-by: Bartlomiej Plotka <[email protected]> Co-authored-by: Ben Ye <[email protected]> Co-authored-by: Giedrius Statkevičius <[email protected]>
brancz · Mar 26, 2021 · f2b0021 · f2b0021
1 parent ba2c5e1
commit f2b0021
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 5 deletions.
diff --git a/s3/s3.go b/s3/s3.go
@@ -66,9 +66,7 @@ var DefaultConfig = Config{
 		MaxIdleConnsPerHost:   100,
 		MaxConnsPerHost:       0,
 	},
-	// Minimum file size after which an HTTP multipart request should be used to upload objects to storage.
-	// Set to 128 MiB as in the minio client.
-	PartSize: 1024 * 1024 * 128,
+	PartSize: 1024 * 1024 * 64, // 64MB.
 }
 
 // Config stores the configuration for s3 bucket.
@@ -85,6 +83,7 @@ type Config struct {
 	TraceConfig        TraceConfig       `yaml:"trace"`
 	ListObjectsVersion string            `yaml:"list_objects_version"`
 	// PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize.
+	// NOTE we need to make sure this number does not produce more parts than 10 000.
 	PartSize  uint64    `yaml:"part_size"`
 	SSEConfig SSEConfig `yaml:"sse_config"`
 }
@@ -449,7 +448,6 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error {
 		size = -1
 	}
 
-	// partSize cannot be larger than object size.
 	partSize := b.partSize
 	if size < int64(partSize) {
 		partSize = 0

diff --git a/s3/s3_e2e_test.go b/s3/s3_e2e_test.go
@@ -0,0 +1,55 @@
+// Copyright (c) The Thanos Authors.
+// Licensed under the Apache License 2.0.
+
+package s3_test
+
+import (
+	"bytes"
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/cortexproject/cortex/integration/e2e"
+	e2edb "github.com/cortexproject/cortex/integration/e2e/db"
+	"github.com/go-kit/kit/log"
+	"github.com/thanos-io/thanos/pkg/objstore/s3"
+	"github.com/thanos-io/thanos/test/e2e/e2ethanos"
+
+	"github.com/thanos-io/thanos/pkg/testutil"
+)
+
+// Regression benchmark for https://github.com/thanos-io/thanos/issues/3917.
+func BenchmarkUpload(b *testing.B) {
+	b.ReportAllocs()
+	ctx := context.Background()
+
+	s, err := e2e.NewScenario("e2e_bench_mino_client")
+	testutil.Ok(b, err)
+	b.Cleanup(e2ethanos.CleanScenario(b, s))
+
+	const bucket = "test"
+	m := e2edb.NewMinio(8080, bucket)
+	testutil.Ok(b, s.StartAndWaitReady(m))
+
+	bkt, err := s3.NewBucketWithConfig(log.NewNopLogger(), s3.Config{
+		Bucket:    bucket,
+		AccessKey: e2edb.MinioAccessKey,
+		SecretKey: e2edb.MinioSecretKey,
+		Endpoint:  m.HTTPEndpoint(),
+		Insecure:  true,
+	}, "test-feed")
+	testutil.Ok(b, err)
+
+	buf := bytes.Buffer{}
+	buf.Grow(1028 * 1028 * 100) // 100MB.
+	word := "abcdefghij"
+	for i := 0; i < buf.Cap()/len(word); i++ {
+		_, _ = buf.WriteString(word)
+	}
+	str := buf.String()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		testutil.Ok(b, bkt.Upload(ctx, "test", strings.NewReader(str)))
+	}
+}
diff --git a/s3/s3_test.go b/s3/s3_test.go
@@ -208,7 +208,7 @@ http_config:
 
 	cfg, err := parseConfig(input)
 	testutil.Ok(t, err)
-	testutil.Assert(t, cfg.PartSize == 1024*1024*128, "when part size not set it should default to 128MiB")
+	testutil.Assert(t, cfg.PartSize == 1024*1024*64, "when part size not set it should default to 128MiB")
 
 	input2 := []byte(`bucket: "bucket-name"
 endpoint: "s3-endpoint"