Merge pull request #67 from nabbar/bump_aws_sdk

Bump sdk-aws-go-v2 at release 0.26.0

.github/workflows/go.yml

@@ -45,6 +45,17 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_format
 
+      - name: Check Missing License
+        continue-on-error: true
+        run: |
+          for PKG in $(find . -type f -name "*.go" | grep -v 'vendor/' | grep -v 'version/license_.*.go');
+          do
+            if [ "$(head -n 25 $PKG | grep -c -P '[*\s]*MIT License')" != 1 ];
+            then
+              echo "Missing Licence: $PKG";
+            fi;
+          done;
+
       - name: Check goLinter
         continue-on-error: true
         run: ./scripts/ci_linter
@@ -62,7 +73,13 @@ jobs:
 
       - name: Check tests
         continue-on-error: true
-        run: ginkgo -cover ./...
+        run: |
+          for PKG in $(find $(pwd) -type f -name "*_suite_test.go" | sort -u );
+          do
+            cd $(dirname $PKG);
+            echo "testing >>> $(basename $(dirname $PKG))";
+            ginkgo -cover .
+          done
         env:
           GOOS: linux
           CGO_ENABLED: 0

archive/archive.go

@@ -145,7 +145,8 @@ func ExtractAll(src ioutils.FileProgress, originalName, outputPath string, defau
 
 	logger.DebugLevel.Log("prepare output...")
 	if i, e := os.Stat(outputPath); e != nil && os.IsNotExist(e) {
-		//nolint #nosec //nosec
+		//nolint #nosec
+		/* #nosec */
 		if e := os.MkdirAll(outputPath, 0775); e != nil {
 			return ErrorDirCreate.ErrorParent(e)
 		}

archive/bz2/reader.go

@@ -41,7 +41,8 @@ func GetFile(src io.ReadSeeker, dst io.WriteSeeker) errors.Error {
 
 	r := bzip2.NewReader(src)
 
-	// #nosec
+	//nolint #nosec
+	/* #nosec */
 	if _, e := io.Copy(dst, r); e != nil {
 		return ErrorIOCopy.ErrorParent(e)
 	} else if _, e = dst.Seek(0, io.SeekStart); e != nil {

archive/gzip/reader.go

@@ -48,7 +48,8 @@ func GetFile(src io.ReadSeeker, dst io.WriteSeeker) errors.Error {
 		_ = r.Close()
 	}()
 
-	// #nosec
+	//nolint #nosec
+	/* #nosec */
 	if _, e = io.Copy(dst, r); e != nil {
 		return ErrorIOCopy.ErrorParent(e)
 	} else if _, e := dst.Seek(0, io.SeekStart); e != nil {

archive/tar/reader.go

@@ -60,7 +60,8 @@ func GetFile(src, dst ioutils.FileProgress, filenameContain, filenameRegex strin
 
 		f := archive.NewFileFullPath(h.Name)
 
-		// #nosec
+		//nolint #nosec
+		/* #nosec */
 		if f.MatchingFullPath(filenameContain) || f.RegexFullPath(filenameRegex) {
 			if _, e := dst.ReadFrom(r); e != nil {
 				return ErrorIOCopy.ErrorParent(e)
@@ -89,6 +90,8 @@ func GetAll(src io.ReadSeeker, outputFolder string, defaultDirPerm os.FileMode)
 			return ErrorTarNext.ErrorParent(e)
 		}
 
+		//nolint #nosec
+		/* #nosec */
 		if err := writeContent(r, h, path.Join(outputFolder, h.Name), defaultDirPerm); err != nil {
 			return err
 		}

archive/zip/reader.go

@@ -78,7 +78,8 @@ func GetFile(src, dst ioutils.FileProgress, filenameContain, filenameRegex strin
 				_ = r.Close()
 			}()
 
-			// #nosec
+			//nolint #nosec
+			/* #nosec */
 			if _, e = dst.ReadFrom(r); e != nil {
 				//logger.ErrorLevel.LogErrorCtx(logger.DebugLevel, "copy buffer from archive reader", err)
 				return ErrorIOCopy.ErrorParent(e)
@@ -116,6 +117,8 @@ func GetAll(src ioutils.FileProgress, outputFolder string, defaultDirPerm os.Fil
 			continue
 		}
 
+		//nolint #nosec
+		/* #nosec */
 		if err := writeContent(f, path.Join(outputFolder, f.Name), defaultDirPerm); err != nil {
 			return err
 		}
@@ -164,7 +167,8 @@ func writeContent(f *zip.File, out string, defaultDirPerm os.FileMode) (err erro
 		return ErrorZipFileOpen.ErrorParent(e)
 	}
 
-	//#nosec
+	//nolint #nosec
+	/* #nosec */
 	if _, e := io.Copy(dst, r); e != nil {
 		return ErrorIOCopy.ErrorParent(e)
 	} else if e = dst.Close(); e != nil {

artifact/s3aws/error.go

@@ -41,6 +41,7 @@ const (
 	ErrorS3AWSNewVers
 	ErrorS3AWSNotFound
 	ErrorS3AWSDownloadError
+	ErrorS3AWSIOReaderError
 )
 
 var (
@@ -85,6 +86,8 @@ func getMessage(code errors.CodeError) (message string) {
 		return "requested version is not found"
 	case ErrorS3AWSDownloadError:
 		return "download object requested occurs error"
+	case ErrorS3AWSIOReaderError:
+		return "return io reader is empty"
 	}
 
 	return ""

artifact/s3aws/model.go

@@ -32,6 +32,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/hashicorp/go-version"
 	"github.com/nabbar/golib/artifact"
 	"github.com/nabbar/golib/artifact/client"
@@ -196,23 +197,15 @@ func (s *s3awsModel) Download(dst ioutils.FileProgress, containName string, rege
 
 func (s *s3awsModel) downloadObject(dst ioutils.FileProgress, object string) errors.Error {
 	var (
-		r io.ReadCloser
-		c []io.Closer
+		r *s3.GetObjectOutput
 		e errors.Error
 		i os.FileInfo
 		j int64
 	)
 
 	defer func() {
-		if len(c) > 0 {
-			for _, b := range c {
-				if b != nil {
-					_ = b.Close()
-				}
-			}
-		}
-		if r != nil {
-			_ = r.Close()
+		if r != nil && r.Body != nil {
+			_ = r.Body.Close()
 		}
 	}()
 
@@ -226,11 +219,13 @@ func (s *s3awsModel) downloadObject(dst ioutils.FileProgress, object string) err
 
 	dst.ResetMax(j)
 
-	if r, c, e = s.c.Object().Get(object); e != nil {
+	if r, e = s.c.Object().Get(object); e != nil {
 		err := ErrorS3AWSDownloadError.ErrorParent(getError(errObject, object))
 		err.AddParentError(e)
 		return err
-	} else if _, err := io.Copy(dst, r); err != nil {
+	} else if r.Body == nil {
+		return ErrorS3AWSIOReaderError.ErrorParent(getError(errObject, object))
+	} else if _, err := io.Copy(dst, r.Body); err != nil {
 		return ErrorS3AWSDownloadError.ErrorParent(getError(errObject, object), err)
 	} else if i, e = dst.FileStat(); e != nil {
 		err := ErrorS3AWSDownloadError.ErrorParent(getError(errObject, object))

artifact/s3aws/s3aws.go

@@ -46,9 +46,13 @@ func NewS3AWS(ctx context.Context, cfg aws.Config, httpcli *http.Client, forceMo
 	}
 
 	if forceModePath {
-		c.ForcePathStyle(true)
+		e = c.ForcePathStyle(true)
 	} else {
-		c.ForcePathStyle(false)
+		e = c.ForcePathStyle(false)
+	}
+
+	if e != nil {
+		return nil, e
 	}
 
 	o := &s3awsModel{

aws/aws_suite_test.go

@@ -1,11 +1,39 @@
+/*
+ *  MIT License
+ *
+ *  Copyright (c) 2020 Nicolas JUHEL
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ */
+
 package aws_test
 
 import (
+	"bytes"
 	"context"
+	"crypto/rand"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/http"
 	"net/url"
 	"os"
 	"os/exec"
@@ -18,6 +46,7 @@ import (
 	"github.com/hashicorp/go-uuid"
 	"github.com/nabbar/golib/aws"
 	"github.com/nabbar/golib/aws/configCustom"
+	"github.com/nabbar/golib/httpcli"
 	"github.com/nabbar/golib/password"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -46,6 +75,7 @@ var _ = BeforeSuite(func() {
 	var (
 		err  error
 		name string
+		htp  *http.Client
 	)
 
 	ctx, cnl = context.WithCancel(context.Background())
@@ -61,7 +91,11 @@ var _ = BeforeSuite(func() {
 			secretKey = password.Generate(64)
 		)
 
+		htp = httpcli.GetClient(uri.Hostname())
+		Expect(htp).NotTo(BeNil())
+
 		cfg = configCustom.NewConfig("", accessKey, secretKey, uri, "us-east-1")
+		Expect(cfg).NotTo(BeNil())
 
 		cfg.SetRegion("us-east-1")
 		err = cfg.RegisterRegionAws(nil)
@@ -75,10 +109,11 @@ var _ = BeforeSuite(func() {
 			time.Sleep(10 * time.Second)
 		}
 
+		time.Sleep(10 * time.Second)
 		println("Minio is waiting on : " + uri.Host)
 	}
 
-	cli, err = aws.New(ctx, cfg, nil)
+	cli, err = aws.New(ctx, cfg, htp)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cli).NotTo(BeNil())
 
@@ -201,3 +236,15 @@ func WaitMinio(host string) bool {
 
 	return err == nil
 }
+
+func randContent(size int) *bytes.Buffer {
+	p := make([]byte, size)
+
+	_, err := rand.Read(p)
+
+	if err != nil {
+		panic(err)
+	}
+
+	return bytes.NewBuffer(p)
+}