reduce e2e test logging and fix multipart

test/init_test.go

@@ -1,3 +1,19 @@
+/*
+ * Copyright © 2024 Clyso GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package test
 
 import (
@@ -54,12 +70,16 @@ func TestMain(m *testing.M) {
 	workerConf.RClone.LocalFileLimit.Enabled = false
 	workerConf.RClone.GlobalFileLimit.Enabled = false
 	workerConf.Features.ACL = false
+	workerConf.Features.Tagging = false
+	workerConf.Log.Level = "warn"
 
 	proxyConf, err = proxy.GetConfig()
 	if err != nil {
 		panic(err)
 	}
 	proxyConf.Features.ACL = false
+	proxyConf.Features.Tagging = false
+	proxyConf.Log.Level = "warn"
 
 	if os.Getenv("EXT_REDIS") != "true" {
 		fmt.Println("using embedded redis")

test/migration/init_test.go

@@ -78,6 +78,7 @@ func TestMain(m *testing.M) {
 	workerConf.RClone.LocalFileLimit.Enabled = false
 	workerConf.RClone.GlobalFileLimit.Enabled = false
 	workerConf.Features.ACL = false
+	workerConf.Features.Tagging = false
 	workerConf.Log.Level = "warn"
 	workerConf.Worker.SwitchRetryInterval = 500 * time.Millisecond
 
@@ -86,6 +87,7 @@ func TestMain(m *testing.M) {
 		panic(err)
 	}
 	proxyConf.Features.ACL = false
+	proxyConf.Features.Tagging = false
 	proxyConf.Log.Level = "warn"
 
 	if os.Getenv("EXT_REDIS") != "true" {

test/migration/switch_test.go

@@ -23,10 +23,10 @@ import (
 	pb "github.com/clyso/chorus/proto/gen/go/chorus"
 	mclient "github.com/minio/minio-go/v7"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/types/known/emptypb"
 	"io"
 	"math/rand"
-	"sync"
 	"testing"
 	"time"
 )
@@ -217,7 +217,7 @@ func TestApi_switch_e2e(t *testing.T) {
 	go func() {
 		defer cancel()
 		t.Log("start write")
-		for n := 0; n < 666; n++ {
+		for n := 0; n < 500; n++ {
 			select {
 			case <-writeCtx.Done():
 				return
@@ -254,34 +254,39 @@ func TestApi_switch_e2e(t *testing.T) {
 					pn++
 				}
 				parts := make([]*aws_s3.CompletedPart, pn)
-				wg := sync.WaitGroup{}
+
+				g, _ := errgroup.WithContext(tstCtx)
 				for nn := len(partBuf); nn >= len(partBuf); {
 					nn, _ = objReader.Read(partBuf)
+					if nn == 0 {
+						break
+					}
 					dd := make([]byte, nn)
 					copy(dd, partBuf[:nn])
-					wg.Add(1)
-					go func(data []byte, partId int) {
-						defer wg.Done()
-						partReader := bytes.NewReader(data)
-
+					partIDCopy := partID
+					g.Go(func() error {
+						partReader := bytes.NewReader(dd)
 						part, err := proxyAwsClient.UploadPart(&aws_s3.UploadPartInput{
 							Body:          partReader,
 							Bucket:        sPtr(bucket),
-							ContentLength: iPtr(int64(len(data))),
+							ContentLength: iPtr(int64(len(dd))),
 							Key:           sPtr(objects[i].name),
-							PartNumber:    iPtr(int64(partId)),
+							PartNumber:    iPtr(int64(partIDCopy)),
 							UploadId:      uploadID.UploadId,
 						})
 
-						r.NoError(err)
-						parts[partId-1] = &aws_s3.CompletedPart{
-							PartNumber: iPtr(int64(partId)),
+						if err != nil {
+							return err
+						}
+						parts[partIDCopy-1] = &aws_s3.CompletedPart{
+							PartNumber: iPtr(int64(partIDCopy)),
 							ETag:       part.ETag,
 						}
-					}(dd, partID)
+						return nil
+					})
 					partID++
 				}
-				wg.Wait()
+				r.NoError(g.Wait())
 				_, err = proxyAwsClient.CompleteMultipartUpload(&aws_s3.CompleteMultipartUploadInput{
 					Bucket:          sPtr(bucket),
 					Key:             sPtr(objects[i].name),
@@ -293,7 +298,7 @@ func TestApi_switch_e2e(t *testing.T) {
 		}
 		t.Log("end write")
 	}()
-	time.Sleep(time.Second)
+	time.Sleep(666 * time.Millisecond)
 	_, err = apiClient.SwitchMainBucket(tstCtx, &pb.SwitchMainBucketRequest{
 		User:    user,
 		Bucket:  bucket,