Add Aws Function & Fix Validotor

Package aws
- fix dive, use it only for slice
- add function to version delete with bypass governance policy

Package cluster
- fix dive, use it only for slice

Package config
- fix missing close channel for waitnotify function

Package errors
- fix pattern %s with uint16

Package ioutils
- fix liberr / Error type return

Package nats
- fix dive, use it only for slice

Package nutsdb
- fix dive, use it only for slice

Package request
- fix dive, use it only for slice

Other
- bump dependencies (validator v10.15.2)

aws/configAws/models.go

@@ -45,9 +45,9 @@ type Model struct {
 }
 
 type ModelStatus struct {
-	Config     Model                `json:"config" yaml:"config" toml:"config" mapstructure:"config" validate:"required,dive"`
-	HTTPClient libhtc.Options       `json:"http-client" yaml:"http-client" toml:"http-client" mapstructure:"http-client" validate:"required,dive"`
-	Monitor    libreq.OptionsHealth `json:"health" yaml:"health" toml:"health" mapstructure:"health" validate:"required,dive"`
+	Config     Model                `json:"config" yaml:"config" toml:"config" mapstructure:"config" validate:"required"`
+	HTTPClient libhtc.Options       `json:"http-client" yaml:"http-client" toml:"http-client" mapstructure:"http-client" validate:""`
+	Monitor    libreq.OptionsHealth `json:"health" yaml:"health" toml:"health" mapstructure:"health" validate:""`
 }
 
 type awsModel struct {

aws/configCustom/models.go

@@ -47,9 +47,9 @@ type Model struct {
 }
 
 type ModelStatus struct {
-	Config     Model                `json:"config" yaml:"config" toml:"config" mapstructure:"config" validate:"required,dive"`
-	HTTPClient libhtc.Options       `json:"http-client" yaml:"http-client" toml:"http-client" mapstructure:"http-client" validate:"dive"`
-	Monitor    libreq.OptionsHealth `json:"health" yaml:"health" toml:"health" mapstructure:"health" validate:"dive"`
+	Config     Model                `json:"config" yaml:"config" toml:"config" mapstructure:"config" validate:"required"`
+	HTTPClient libhtc.Options       `json:"http-client" yaml:"http-client" toml:"http-client" mapstructure:"http-client" validate:""`
+	Monitor    libreq.OptionsHealth `json:"health" yaml:"health" toml:"health" mapstructure:"health" validate:""`
 }
 
 type awsModel struct {

aws/object/interface.go

@@ -63,6 +63,7 @@ type Object interface {
 	Head(object string) (*sdksss.HeadObjectOutput, error)
 	Get(object string) (*sdksss.GetObjectOutput, error)
 	Put(object string, body io.Reader) error
+	Copy(source, destination string) error
 	Delete(check bool, object string) error
 	DeleteAll(objects *sdktps.Delete) ([]sdktps.DeletedObject, error)
 	GetAttributes(object, version string) (*sdksss.GetObjectAttributesOutput, error)
@@ -84,6 +85,8 @@ type Object interface {
 	VersionHead(object, version string) (*sdksss.HeadObjectOutput, error)
 	VersionSize(object, version string) (size int64, err error)
 	VersionDelete(check bool, object, version string) error
+	VersionCopy(source, version, destination string) error
+	VersionDeleteLock(check bool, object, version string, byPassGovernance bool) error
 
 	GetRetention(object, version string) (until time.Time, mode string, err error)
 	SetRetention(object, version string, bypass bool, until time.Time, mode string) error

aws/object/object.go

@@ -128,6 +128,10 @@ func (cli *client) Delete(check bool, object string) error {
 	return cli.VersionDelete(check, object, "")
 }
 
+func (cli *client) Copy(source, destination string) error {
+	return cli.VersionCopy(source, "", destination)
+}
+
 func (cli *client) Put(object string, body io.Reader) error {
 	var tpe *string
 

aws/object/version.go

@@ -26,6 +26,7 @@
 package object
 
 import (
+	"path"
 	"strings"
 
 	sdkaws "github.com/aws/aws-sdk-go-v2/aws"
@@ -186,6 +187,10 @@ func (cli *client) VersionSize(object, version string) (size int64, err error) {
 }
 
 func (cli *client) VersionDelete(check bool, object, version string) error {
+	return cli.VersionDeleteLock(check, object, version, false)
+}
+
+func (cli *client) VersionDeleteLock(check bool, object, version string, byPassGovernance bool) error {
 	if check {
 		if _, err := cli.VersionHead(object, version); err != nil {
 			return err
@@ -201,14 +206,38 @@ func (cli *client) VersionDelete(check bool, object, version string) error {
 		in.VersionId = sdkaws.String(version)
 	}
 
+	if byPassGovernance {
+		in.BypassGovernanceRetention = true
+	}
+
 	_, err := cli.s3.DeleteObject(cli.GetContext(), &in)
 
-	if !check && err != nil {
-		e := err.Error()
-		if strings.Contains(e, "api error NoSuchKey") {
-			return nil
-		}
+	if !check && err != nil && strings.Contains(err.Error(), "api error NoSuchKey") {
+		return nil
+	} else if err != nil {
+		return cli.GetError(err)
+	}
+
+	return nil
+}
+
+func (cli *client) VersionCopy(source, version, destination string) error {
+	in := sdksss.CopyObjectInput{
+		Bucket: cli.GetBucketAws(),
+		Key:    sdkaws.String(destination),
+	}
+
+	if version != "" {
+		in.CopySource = sdkaws.String(path.Join(*(cli.GetBucketAws()), source) + "?versionId=" + version)
+	} else {
+		in.CopySource = sdkaws.String(path.Join(*(cli.GetBucketAws()), source))
+	}
+
+	_, err := cli.s3.CopyObject(cli.GetContext(), &in)
+
+	if err != nil {
+		return cli.GetError(err)
 	}
 
-	return cli.GetError(err)
+	return nil
 }

cluster/config.go

@@ -40,8 +40,8 @@ import (
 )
 
 type Config struct {
-	Node       ConfigNode        `mapstructure:"node" json:"node" yaml:"node" toml:"node" validate:"dive"`
-	Cluster    ConfigCluster     `mapstructure:"cluster" json:"cluster" yaml:"cluster" toml:"cluster" validate:"dive"`
+	Node       ConfigNode        `mapstructure:"node" json:"node" yaml:"node" toml:"node" validate:""`
+	Cluster    ConfigCluster     `mapstructure:"cluster" json:"cluster" yaml:"cluster" toml:"cluster" validate:""`
 	InitMember map[uint64]string `mapstructure:"init_member" json:"init_member" yaml:"init_member" toml:"init_member"`
 }
 

cluster/configNode.go

@@ -254,14 +254,14 @@ type ConfigNode struct {
 	// points the local gossip service will try to talk to. The Seed field doesn't
 	// need to include all gossip end points, a few well connected nodes in the
 	// gossip network is enough.
-	Gossip ConfigGossip `mapstructure:"gossip" json:"gossip" yaml:"gossip" toml:"gossip" validate:"omitempty,dive"`
+	Gossip ConfigGossip `mapstructure:"gossip" json:"gossip" yaml:"gossip" toml:"gossip" validate:"omitempty"`
 
 	// Expert contains options for expert users who are familiar with the internals
 	// of Dragonboat. Users are recommended not to use this field unless
 	// absoloutely necessary. It is important to note that any change to this field
 	// may cause an existing instance unable to restart, it may also cause negative
 	// performance impacts.
-	Expert ConfigExpert `mapstructure:"expert" json:"expert" yaml:"expert" toml:"expert" validate:"omitempty,dive"`
+	Expert ConfigExpert `mapstructure:"expert" json:"expert" yaml:"expert" toml:"expert" validate:"omitempty"`
 }
 
 func (c ConfigNode) GetDGBConfigNodeHost() dgbcfg.NodeHostConfig {

config/interface.go

@@ -146,15 +146,19 @@ func WaitNotify() {
 	// Wait for interrupt signal to gracefully shutdown the server with
 	// a timeout of 5 seconds.
 	quit := make(chan os.Signal, 1)
+	defer func() {
+		close(quit)
+	}()
+
 	signal.Notify(quit, syscall.SIGINT)
 	signal.Notify(quit, syscall.SIGTERM)
 	signal.Notify(quit, syscall.SIGQUIT)
 
 	select {
 	case <-quit:
-		cnl()
+		Shutdown()
 	case <-ctx.Done():
-		cnl()
+		Shutdown()
 	}
 }
 

errors/errors.go

@@ -34,8 +34,8 @@ import (
 )
 
 var (
-	defaultPattern      = "[Error #%s] %s"
-	defaultPatternTrace = "[Error #%s] %s (%s)"
+	defaultPattern      = "[Error #%d] %s"
+	defaultPatternTrace = "[Error #%d] %s (%s)"
 )
 
 // GetDefaultPatternTrace define the pattern to be used for string of error with code.

go.mod

@@ -6,7 +6,7 @@ toolchain go1.21.0
 
 require (
 	github.com/aws/aws-sdk-go-v2 v1.21.0
-	github.com/aws/aws-sdk-go-v2/config v1.18.36
+	github.com/aws/aws-sdk-go-v2/config v1.18.37
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.35
 	github.com/aws/aws-sdk-go-v2/service/iam v1.22.5
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5
@@ -17,7 +17,7 @@ require (
 	github.com/fxamacker/cbor/v2 v2.5.0
 	github.com/gin-gonic/gin v1.9.1
 	github.com/go-ldap/ldap/v3 v3.4.5
-	github.com/go-playground/validator/v10 v10.15.1
+	github.com/go-playground/validator/v10 v10.15.2
 	github.com/google/go-github/v33 v33.0.0
 	github.com/hashicorp/go-hclog v1.5.0
 	github.com/hashicorp/go-retryablehttp v0.7.4
@@ -33,7 +33,7 @@ require (
 	github.com/nats-io/nats-server/v2 v2.9.21
 	github.com/nats-io/nats.go v1.28.0
 	github.com/nutsdb/nutsdb v0.13.1
-	github.com/onsi/ginkgo/v2 v2.11.0
+	github.com/onsi/ginkgo/v2 v2.12.0
 	github.com/onsi/gomega v1.27.10
 	github.com/pelletier/go-toml v1.9.5
 	github.com/prometheus/client_golang v1.16.0
@@ -47,7 +47,7 @@ require (
 	github.com/xanzy/go-gitlab v0.90.0
 	github.com/xhit/go-simple-mail v2.2.2+incompatible
 	github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235
-	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
+	golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0
 	golang.org/x/net v0.14.0
 	golang.org/x/oauth2 v0.11.0
 	golang.org/x/sync v0.3.0
@@ -65,7 +65,7 @@ require (
 require (
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/ClickHouse/ch-go v0.58.2 // indirect
-	github.com/ClickHouse/clickhouse-go/v2 v2.13.2 // indirect
+	github.com/ClickHouse/clickhouse-go/v2 v2.13.3 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
@@ -96,7 +96,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
 	github.com/chenzhuoyu/iasm v0.9.0 // indirect
-	github.com/cockroachdb/errors v1.10.0 // indirect
+	github.com/cockroachdb/errors v1.11.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/pebble v0.0.0-20210331181633-27fc006b8bfb // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
@@ -123,7 +123,7 @@ require (
 	github.com/google/btree v1.1.2 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect
+	github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect
 	github.com/google/uuid v1.3.1 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -196,14 +196,14 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xujiajun/mmap-go v1.0.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	go.opentelemetry.io/otel v1.16.0 // indirect
-	go.opentelemetry.io/otel/trace v1.16.0 // indirect
+	go.opentelemetry.io/otel v1.17.0 // indirect
+	go.opentelemetry.io/otel/trace v1.17.0 // indirect
 	golang.org/x/arch v0.4.0 // indirect
 	golang.org/x/crypto v0.12.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
+	golang.org/x/tools v0.12.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

ioutils/tempFile.go

@@ -29,10 +29,10 @@ import (
 	"os"
 	"path/filepath"
 
-	. "github.com/nabbar/golib/errors"
+	liberr "github.com/nabbar/golib/errors"
 )
 
-func NewTempFile() (*os.File, Error) {
+func NewTempFile() (*os.File, error) {
 	f, e := os.CreateTemp(os.TempDir(), "")
 	return f, ErrorIOFileTempNew.IfError(e)
 }
@@ -45,7 +45,7 @@ func GetTempFilePath(f *os.File) string {
 	return filepath.Join(os.TempDir(), filepath.Base(f.Name()))
 }
 
-func DelTempFile(f *os.File) Error {
+func DelTempFile(f *os.File) error {
 	if f == nil {
 		return nil
 	}
@@ -58,5 +58,5 @@ func DelTempFile(f *os.File) Error {
 	b := os.Remove(n)
 	e2 := ErrorIOFileTempRemove.IfError(b)
 
-	return MakeIfError(e2, e1)
+	return liberr.MakeIfError(e2, e1)
 }

nats/config.go

@@ -48,16 +48,16 @@ import (
 )
 
 type Config struct {
-	Server     ConfigSrv       `mapstructure:"server" json:"server" yaml:"server" toml:"server" validate:"dive,required"`
-	Cluster    ConfigCluster   `mapstructure:"cluster" json:"cluster" yaml:"cluster" toml:"cluster" validate:"dive,required"`
-	Gateways   ConfigGateway   `mapstructure:"gateways" json:"gateways" yaml:"gateways" toml:"gateways" validate:"dive,required"`
-	Leaf       ConfigLeaf      `mapstructure:"leaf" json:"leaf" yaml:"leaf" toml:"leaf" validate:"dive,required"`
-	Websockets ConfigWebsocket `mapstructure:"websockets" json:"websockets" yaml:"websockets" toml:"websockets" validate:"dive,required"`
-	MQTT       ConfigMQTT      `mapstructure:"mqtt" json:"mqtt" yaml:"mqtt" toml:"mqtt" validate:"dive,required"`
-	Limits     ConfigLimits    `mapstructure:"limits" json:"limits" yaml:"limits" toml:"limits" validate:"dive,required"`
-	Logs       ConfigLogger    `mapstructure:"logs" json:"logs" yaml:"logs" toml:"logs" validate:"dive,required"`
-	Auth       ConfigAuth      `mapstructure:"auth" json:"auth" yaml:"auth" toml:"auth" validate:"dive,required"`
-	Monitor    moncfg.Config   `mapstructure:"monitor" json:"monitor" yaml:"monitor" toml:"monitor" validate:"dive"`
+	Server     ConfigSrv       `mapstructure:"server" json:"server" yaml:"server" toml:"server" validate:"required"`
+	Cluster    ConfigCluster   `mapstructure:"cluster" json:"cluster" yaml:"cluster" toml:"cluster" validate:"required"`
+	Gateways   ConfigGateway   `mapstructure:"gateways" json:"gateways" yaml:"gateways" toml:"gateways" validate:"required"`
+	Leaf       ConfigLeaf      `mapstructure:"leaf" json:"leaf" yaml:"leaf" toml:"leaf" validate:"required"`
+	Websockets ConfigWebsocket `mapstructure:"websockets" json:"websockets" yaml:"websockets" toml:"websockets" validate:"required"`
+	MQTT       ConfigMQTT      `mapstructure:"mqtt" json:"mqtt" yaml:"mqtt" toml:"mqtt" validate:"required"`
+	Limits     ConfigLimits    `mapstructure:"limits" json:"limits" yaml:"limits" toml:"limits" validate:"required"`
+	Logs       ConfigLogger    `mapstructure:"logs" json:"logs" yaml:"logs" toml:"logs" validate:"required"`
+	Auth       ConfigAuth      `mapstructure:"auth" json:"auth" yaml:"auth" toml:"auth" validate:"required"`
+	Monitor    moncfg.Config   `mapstructure:"monitor" json:"monitor" yaml:"monitor" toml:"monitor" validate:""`
 
 	//function / interface are not defined in config marshall
 	Customs *ConfigCustom `mapstructure:"-" json:"-" yaml:"-" toml:"-"`

nats/configPart.go

@@ -78,22 +78,22 @@ type ConfigUser struct {
 // ConfigPermissionsUser are the allowed subjects on a per publish or subscribe basis.
 type ConfigPermissionsUser struct {
 	//Publish define the scope permission for publisher role.
-	Publish ConfigPermissionSubject `mapstructure:"publish" json:"publish" yaml:"publish" toml:"publish" validate:"dive,required"`
+	Publish ConfigPermissionSubject `mapstructure:"publish" json:"publish" yaml:"publish" toml:"publish" validate:"required"`
 
 	//Subscribe define the scope permission for subscriber role.
-	Subscribe ConfigPermissionSubject `mapstructure:"subscribe" json:"subscribe" yaml:"subscribe" toml:"subscribe" validate:"dive,required"`
+	Subscribe ConfigPermissionSubject `mapstructure:"subscribe" json:"subscribe" yaml:"subscribe" toml:"subscribe" validate:"required"`
 
 	//Response define the scope permission to allow response for a message.
-	Response ConfigPermissionResponse `mapstructure:"response" json:"response" yaml:"response" toml:"response" validate:"dive,required"`
+	Response ConfigPermissionResponse `mapstructure:"response" json:"response" yaml:"response" toml:"response" validate:"required"`
 }
 
 // ConfigPermissionsRoute are similar to user permissions but describe what a server can import/export from and to another server.
 type ConfigPermissionsRoute struct {
 	//Import define the scope permission to import data from the route.
-	Import ConfigPermissionSubject `mapstructure:"import" json:"import" yaml:"import" toml:"import" validate:"dive,required"`
+	Import ConfigPermissionSubject `mapstructure:"import" json:"import" yaml:"import" toml:"import" validate:"required"`
 
 	//Export define the scope permission to export data to the route.
-	Export ConfigPermissionSubject `mapstructure:"export" json:"export" yaml:"export" toml:"export" validate:"dive,required"`
+	Export ConfigPermissionSubject `mapstructure:"export" json:"export" yaml:"export" toml:"export" validate:"required"`
 }
 
 // ConfigPermissionResponse can be used to allow responses to any reply subject that is received on a valid subscription.
@@ -120,7 +120,7 @@ type ConfigAccount struct {
 	//Name define the name of the account.
 	Name string `mapstructure:"name" json:"name" yaml:"name" toml:"name"`
 
-	Permission ConfigPermissionsUser `mapstructure:"permission" json:"permission" yaml:"permission" toml:"permission" validate:"dive,required"`
+	Permission ConfigPermissionsUser `mapstructure:"permission" json:"permission" yaml:"permission" toml:"permission" validate:"required"`
 }
 
 type ConfigAuth struct {
@@ -311,7 +311,7 @@ type ConfigSrv struct {
 	TLSTimeout time.Duration `mapstructure:"tls_timeout" json:"tls_timeout" yaml:"tls_timeout" toml:"tls_timeout"`
 
 	//TLSConfig Configuration map for tls for client and http monitoring.
-	TLSConfig libtls.Config `mapstructure:"tls_config" json:"tls_config" yaml:"tls_config" toml:"tls_config" validate:"dive"`
+	TLSConfig libtls.Config `mapstructure:"tls_config" json:"tls_config" yaml:"tls_config" toml:"tls_config" validate:""`
 }
 
 type ConfigCluster struct {
@@ -346,7 +346,7 @@ type ConfigCluster struct {
 	AuthTimeout time.Duration `mapstructure:"auth_timeout" json:"auth_timeout" yaml:"auth_timeout" toml:"auth_timeout"`
 
 	//Permissions define the scope permission assign to route connections.
-	Permissions ConfigPermissionsRoute `mapstructure:"permissions" json:"permissions" yaml:"permissions" toml:"permissions" validate:"dive"`
+	Permissions ConfigPermissionsRoute `mapstructure:"permissions" json:"permissions" yaml:"permissions" toml:"permissions" validate:""`
 
 	//TLS Enable tls for cluster connection.
 	TLS bool `mapstructure:"tls" json:"tls" yaml:"tls" toml:"tls"`
@@ -355,7 +355,7 @@ type ConfigCluster struct {
 	TLSTimeout time.Duration `mapstructure:"tls_timeout" json:"tls_timeout" yaml:"tls_timeout" toml:"tls_timeout"`
 
 	//TLSConfig define the tls configuration for cluster connection.
-	TLSConfig libtls.Config `mapstructure:"tls_config" json:"tls_config" yaml:"tls_config" toml:"tls_config" validate:"dive"`
+	TLSConfig libtls.Config `mapstructure:"tls_config" json:"tls_config" yaml:"tls_config" toml:"tls_config" validate:""`
 }
 
 // ConfigGatewayRemote are options for connecting to a remote gateway
@@ -374,7 +374,7 @@ type ConfigGatewayRemote struct {
 	TLSTimeout time.Duration `mapstructure:"tls_timeout" json:"tls_timeout" yaml:"tls_timeout" toml:"tls_timeout"`
 
 	//TLSConfig define the tls configuration for the current gateways destination.
-	TLSConfig libtls.Config `mapstructure:"tls_config" json:"tls_config" yaml:"tls_config" toml:"tls_config" validate:"dive"`
+	TLSConfig libtls.Config `mapstructure:"tls_config" json:"tls_config" yaml:"tls_config" toml:"tls_config" validate:""`
 }
 
 // ConfigGateway are options for gateways.
@@ -417,7 +417,7 @@ type ConfigGateway struct {
 	TLSTimeout time.Duration `mapstructure:"tls_timeout" json:"tls_timeout" yaml:"tls_timeout" toml:"tls_timeout"`
 
 	//TLSConfig define the tls configuration for gateways connection.
-	TLSConfig libtls.Config `mapstructure:"tls_config" json:"tls_config" yaml:"tls_config" toml:"tls_config" validate:"dive"`
+	TLSConfig libtls.Config `mapstructure:"tls_config" json:"tls_config" yaml:"tls_config" toml:"tls_config" validate:""`
 }
 
 // ConfigLeaf are options for a given server to accept leaf node connections and/or connect to a remote cluster.