Add Features for package Archive, Artifact, AWS, OAuth, IOUtils, Semaphore, Errors

.github/workflows/go.yml

@@ -60,6 +60,13 @@ jobs:
           CGO_ENABLED: 0
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
 
+      - name: Check tests
+        continue-on-error: true
+        run: ginkgo -cover ./...
+        env:
+          GOOS: linux
+          CGO_ENABLED: 0
+
       - name: Add ming-w32 (x86 + win64)
         run: sudo apt-get install gcc-multilib gcc-mingw-w64
 

.gitignore

@@ -7,6 +7,7 @@
 
 # Test binary, built with `go test -c`
 *.test
+*.cover*
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
@@ -22,3 +23,4 @@ go.sum
 .idea/*
 .idea/**/*
 
+/test-*

archive/README.md

@@ -0,0 +1,114 @@
+# Package Archive
+This package will try to do all uncompress/unarchive and extract one file or all file.
+This package will expose 2 functions :
+- ExtractFile : for one file extracted
+- ExtractAll : to extract all file 
+
+## Example of implementation
+
+### Example of one file extracted
+
+To use `ExtractFile` function, you will need this parameters :
+- `src` : is the source file into a `ioutils.FileProgress` struct to expose an `os.File` pointer with interface `io.WriteTo`, `io.ReadFrom`, `io.ReaderAt` and progress capabilities
+- `dst` : is the source file into a `ioutils.FileProgress` struct to expose an `os.File` pointer with interface `io.WriteTo`, `io.ReadFrom`, `io.ReaderAt` and progress capabilities
+- `filenameContain` : is a `string` to search in the file name to find it and extract it. This string will be search into the `strings.Contains` function
+- `filenameRegex` : is a regex pattern `string` to search in the file name any match and extract it. This string will be search into the `regexp.MatchString` function
+
+You can implement this function as it. This example is available in [`test/test-archive`](../test/test-archive/main.go) folder.
+```go
+ import (
+	"io"
+	"io/ioutil"
+
+	"github.com/nabbar/golib/archive"
+	"github.com/nabbar/golib/ioutils"
+)
+
+const fileName = "fullpath to my archive file"
+
+func main() {
+	var (
+		src ioutils.FileProgress
+		dst ioutils.FileProgress
+		err errors.Error
+	)
+
+	// register closing function in output function callback
+  defer func() {
+		if src != nil {
+			_ = src.Close()
+		}
+		if dst != nil {
+			_ = dst.Close()
+		}
+	}()
+
+	// open archive with a ioutils NewFileProgress function
+  if src, err = ioutils.NewFileProgressPathOpen(fileName); err != nil {
+		panic(err)
+	}
+
+	// open a destination with a ioutils NewFileProgress function, as a temporary file
+	if dst, err = ioutils.NewFileProgressTemp(); err != nil {
+		panic(err)
+	}
+
+  // call the extract file function
+	if err = archive.ExtractFile(tmp, rio, "path/to/my/file/into/archive", "archive name regex"); err != nil {
+		panic(err)
+	}
+
+}
+```
+
+
+### Example of all files extracted
+
+To use `ExtractAll` function, you will need this parameters :
+- `src` : is the source file into a `ioutils.FileProgress` struct to expose an `os.File` pointer with interface `io.WriteTo`, `io.ReadFrom`, `io.ReaderAt` and progress capabilities
+- `originalName` : is a `string` to define the originalName of the archive. This params is used to create a unique file created into the outputPath if the archive is not an archive or just compressed with a not catalogued compress type like gzip or bzip2.
+- `outputPath` : is a `string` to precise the destination output directory (full path). All extracted file will be extracted with this directory as base of path.
+- `defaultDirPerm` : is a `os.FileMode` to precise the permission of directory. This parameters is usefull if the output directory is not existing.
+
+You can implement this function as it. This example is available in [`test/test-archive-all`](../test/test-archive-all/main.go) folder.
+```go
+ import (
+	"io"
+	"io/ioutil"
+
+	"github.com/nabbar/golib/archive"
+	"github.com/nabbar/golib/ioutils"
+)
+
+const fileName = "fullpath to my archive file"
+
+func main() {
+	var (
+		src ioutils.FileProgress
+		tmp ioutils.FileProgress
+		out string
+		err error
+	)
+
+  // open archive with a ioutils NewFileProgress function 
+	if src, err = ioutils.NewFileProgressPathOpen(fileName); err != nil {
+		panic(err)
+	}
+
+  // create an new temporary file to use his name as output path
+	if tmp, err = ioutils.NewFileProgressTemp(); err != nil {
+		panic(err)
+	} else {
+    // get the filename of the temporary file
+		out = tmp.FilePath()
+
+    // close the temporary file will call the delete temporary file
+		_ = tmp.Close()
+	}
+
+  if err = archive.ExtractAll(src, path.Base(src.FilePath()), out, 0775); err != nil {
+		panic(err)
+	}
+
+}
+```

archive/archive.go

@@ -26,74 +26,167 @@
 package archive
 
 import (
+	"io"
 	"os"
+	"path"
 
 	"github.com/nabbar/golib/archive/bz2"
 	"github.com/nabbar/golib/archive/gzip"
 	"github.com/nabbar/golib/archive/tar"
 	"github.com/nabbar/golib/archive/zip"
-
-	. "github.com/nabbar/golib/errors"
-	//. "github.com/nabbar/golib/logger"
-
-	iou "github.com/nabbar/golib/ioutils"
+	"github.com/nabbar/golib/errors"
+	"github.com/nabbar/golib/ioutils"
+	"github.com/nabbar/golib/logger"
 )
 
 type ArchiveType uint8
 
-func ExtractFile(src *os.File, fileNameContain, fileNameRegex string) (*os.File, Error) {
-	loc := src.Name()
+func ExtractFile(src, dst ioutils.FileProgress, fileNameContain, fileNameRegex string) errors.Error {
+	var (
+		tmp ioutils.FileProgress
+		err errors.Error
+	)
 
-	if dst, err := bz2.GetFile(src, fileNameContain, fileNameRegex); err == nil {
-		//DebugLevel.Log("try deleting source archive...")
-		if err = iou.DelTempFile(src); err != nil {
-			return nil, err
-		}
-		//DebugLevel.Log("try another archive...")
-		return ExtractFile(dst, fileNameContain, fileNameRegex)
+	if tmp, err = dst.NewFileTemp(); err != nil {
+		return err
 	}
 
-	if dst, err := gzip.GetFile(src, fileNameContain, fileNameRegex); err == nil {
-		//DebugLevel.Log("try deleting source archive...")
-		if err = iou.DelTempFile(src); err != nil {
-			return nil, err
-		}
-		//DebugLevel.Log("try another archive...")
-		return ExtractFile(dst, fileNameContain, fileNameRegex)
+	if _, e := src.Seek(0, io.SeekStart); e != nil {
+		return ErrorFileSeek.ErrorParent(e)
+		// #nosec
 	}
 
-	if dst, err := tar.GetFile(src, fileNameContain, fileNameRegex); err == nil {
-		//DebugLevel.Log("try deleting source archive...")
-		if err = iou.DelTempFile(src); err != nil {
-			return nil, err
-		}
-		//DebugLevel.Log("try another archive...")
-		return ExtractFile(dst, fileNameContain, fileNameRegex)
+	if err := bz2.GetFile(src, tmp); err == nil {
+		//logger.DebugLevel.Log("try another archive...")
+		return ExtractFile(tmp, dst, fileNameContain, fileNameRegex)
+	} else if err.IsCodeError(bz2.ErrorIOCopy) {
+		return err
 	}
 
-	if dst, err := zip.GetFile(src, fileNameContain, fileNameRegex); err == nil {
-		//DebugLevel.Log("try deleting source archive...")
-		if err = iou.DelTempFile(src); err != nil {
-			return nil, err
-		}
-		//DebugLevel.Log("try another archive...")
-		return ExtractFile(dst, fileNameContain, fileNameRegex)
+	if err := gzip.GetFile(src, tmp); err == nil {
+		//logger.DebugLevel.Log("try another archive...")
+		return ExtractFile(tmp, dst, fileNameContain, fileNameRegex)
+	} else if !err.IsCodeError(gzip.ErrorGZReader) {
+		return err
+	}
+
+	if err := tar.GetFile(src, tmp, fileNameContain, fileNameRegex); err == nil {
+		//logger.DebugLevel.Log("try another archive...")
+		return ExtractFile(tmp, dst, fileNameContain, fileNameRegex)
+	} else if !err.IsCodeError(tar.ErrorTarNext) {
+		return err
+	}
+
+	if err := zip.GetFile(src, tmp, fileNameContain, fileNameRegex); err == nil {
+		//logger.DebugLevel.Log("try another archive...")
+		return ExtractFile(tmp, dst, fileNameContain, fileNameRegex)
+	} else if !err.IsCodeError(zip.ErrorZipOpen) {
+		return err
+	}
+
+	_ = tmp.Close()
+
+	if _, e := dst.ReadFrom(src); e != nil {
+		//logger.ErrorLevel.LogErrorCtx(logger.DebugLevel, "reopening file", err)
+		return ErrorIOCopy.ErrorParent(e)
 	}
 
+	return nil
+}
+
+func ExtractAll(src ioutils.FileProgress, originalName, outputPath string, defaultDirPerm os.FileMode) errors.Error {
 	var (
-		err error
+		tmp ioutils.FileProgress
+		dst ioutils.FileProgress
+		err errors.Error
 	)
 
-	if _, err = src.Seek(0, 0); err != nil {
-		e1 := FILE_SEEK.ErrorParent(err)
-		// #nosec
-		if src, err = os.Open(loc); err != nil {
-			//ErrorLevel.LogErrorCtx(DebugLevel, "reopening file", err)
-			e2 := FILE_OPEN.ErrorParent(err)
-			e2.AddParentError(e1)
-			return nil, e2
+	defer func() {
+		if src != nil {
+			_ = src.Close()
+		}
+		if dst != nil {
+			_ = dst.Close()
 		}
+		if tmp != nil {
+			_ = tmp.Close()
+		}
+	}()
+
+	if tmp, err = src.NewFileTemp(); err != nil {
+		return ErrorFileOpen.Error(err)
+	}
+
+	logger.DebugLevel.Log("try BZ2...")
+	if err = bz2.GetFile(src, tmp); err == nil {
+		logger.DebugLevel.Log("try another archive...")
+		return ExtractAll(tmp, originalName, outputPath, defaultDirPerm)
+	} else if !err.IsCodeError(bz2.ErrorIOCopy) {
+		logger.DebugLevel.Logf("error found on BZ2 : %v", err)
+		return err
+	} else {
+		logger.DebugLevel.Logf("not a BZ2 : %v", err)
+	}
+
+	logger.DebugLevel.Log("try GZIP...")
+	if err = gzip.GetFile(src, tmp); err == nil {
+		logger.DebugLevel.Log("try another archive...")
+		return ExtractAll(tmp, originalName, outputPath, defaultDirPerm)
+	} else if !err.IsCodeError(gzip.ErrorGZReader) {
+		logger.DebugLevel.Logf("error found on GZIP : %v", err)
+		return err
+	} else {
+		logger.DebugLevel.Logf("not a GZIP : %v", err)
+	}
+
+	if tmp != nil {
+		_ = tmp.Close()
+	}
+
+	logger.DebugLevel.Log("prepare output...")
+	if i, e := os.Stat(outputPath); e != nil && os.IsNotExist(e) {
+		//nolint #nosec //nosec
+		if e := os.MkdirAll(outputPath, 0775); e != nil {
+			return ErrorDirCreate.ErrorParent(e)
+		}
+	} else if e != nil {
+		return ErrorDirStat.ErrorParent(e)
+	} else if !i.IsDir() {
+		return ErrorDirNotDir.Error(nil)
+	}
+
+	logger.DebugLevel.Log("try tar...")
+	if err = tar.GetAll(src, outputPath, defaultDirPerm); err == nil {
+		logger.DebugLevel.Log("extracting TAR finished...")
+		return nil
+	} else if !err.IsCodeError(tar.ErrorTarNext) {
+		logger.DebugLevel.Logf("error found on TAR : %v", err)
+		return err
+	} else {
+		logger.DebugLevel.Logf("not a TAR : %v", err)
+	}
+
+	logger.DebugLevel.Log("try zip...")
+	if err = zip.GetAll(src, outputPath, defaultDirPerm); err == nil {
+		logger.DebugLevel.Log("extracting ZIP finished...")
+		return nil
+	} else if !err.IsCodeError(zip.ErrorZipOpen) {
+		logger.DebugLevel.Logf("error found on ZIP : %v", err)
+		return err
+	} else {
+		logger.DebugLevel.Logf("not a ZIP : %v", err)
+	}
+
+	logger.DebugLevel.Log("writing original file...")
+	if dst, err = src.NewFilePathWrite(path.Join(outputPath, originalName), true, true, 0664); err != nil {
+		return ErrorFileOpen.Error(err)
+	}
+
+	if _, e := src.Seek(0, io.SeekStart); e != nil {
+		return ErrorFileSeek.ErrorParent(e)
+	} else if _, e := dst.ReadFrom(src); e != nil {
+		return ErrorIOCopy.ErrorParent(e)
 	}
 
-	return src, nil
+	return nil
 }

archive/bz2/error.go

@@ -29,9 +29,9 @@ package bz2
 import "github.com/nabbar/golib/errors"
 
 const (
-	EMPTY_PARAMS errors.CodeError = iota + errors.MIN_PKG_Archive + 10
-	FILE_SEEK
-	IO_COPY
+	ErrorParamsEmpty errors.CodeError = iota + errors.MIN_PKG_Archive + 10
+	ErrorFileSeek
+	ErrorIOCopy
 )
 
 var isCodeError = false
@@ -41,17 +41,19 @@ func IsCodeError() bool {
 }
 
 func init() {
-	isCodeError = errors.ExistInMapMessage(EMPTY_PARAMS)
-	errors.RegisterIdFctMessage(EMPTY_PARAMS, getMessage)
+	isCodeError = errors.ExistInMapMessage(ErrorParamsEmpty)
+	errors.RegisterIdFctMessage(ErrorParamsEmpty, getMessage)
 }
 
 func getMessage(code errors.CodeError) (message string) {
 	switch code {
-	case EMPTY_PARAMS:
+	case errors.UNK_ERROR:
+		return ""
+	case ErrorParamsEmpty:
 		return "given parameters is empty"
-	case FILE_SEEK:
+	case ErrorFileSeek:
 		return "cannot seek into file"
-	case IO_COPY:
+	case ErrorIOCopy:
 		return "io copy occurs error"
 	}