Skip to content

Commit

Permalink
Merge pull request #150 from nabbar/monitoring
Browse files Browse the repository at this point in the history
Rework Monitoring, Prometheus, Status system

Package Monitoring :
- use packag dedicated to monitor component
- each monitor work as standalone server to monitor health
- collect metrics to export them to prometheus exporter

Package Prometheus :
- review to simplify use for API and not API metrics
- optimize code

Package Status :
- Rework to use Monitor package
- Rework to use native json / text Marshaller interface

Context :
- rework context config (context var) to use sync map and sync RWMutex (WORM)
- move gin context to dedicated sub package (dependancies of logger make circular dependencies)
- optimize code

Config :
- rework to optimize sync / collect of component
- rework status to monitor
- remove monitor managment from config to each component
- add a func to set default logger to implement inherit default logger options
- optimize code

IOUtils :
- isolate logger / closer interface as a usable & public interface & instance
- this interface / instance allow to collect io.closer over a context to close all if context is done

Logger :
- rework to use context.config map
- rework to use ioutils closer
- rework to allow options to inherit a default options, or the last version of options
- optimize code

Size :
- Add package Size to calculate and manipulate size Byte or bit
- Add encoding : Text/JSON/Yaml/Toml...
- Add option to défine default unit : Byte or bit

Other :
- adjust following code
- optimize code
- limit use of atomic value
- rework to use RWMutex instead of sync.Mutex to maximize capabilities of read instead of write
- remove 32bit build for CI/CD
- add darwin/arm64 build for CI/CD

Bump Dependencies
  • Loading branch information
nabbar authored Feb 21, 2023
2 parents 771a66e + e3239db commit 443935a
Show file tree
Hide file tree
Showing 283 changed files with 20,518 additions and 10,425 deletions.
32 changes: 5 additions & 27 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,43 +93,21 @@ jobs:
continue-on-error: false
run: sudo apt-get install gcc-multilib gcc-mingw-w64

- name: Test Build Linux/amd64 with suffix
- name: Test Build Linux/amd64 with suffix/CGO
continue-on-error: false
run: |
CGO_ENABLED=0
GOOS=linux
GOARCH=amd64
GOAMD64=v4
IGNORE_BUILD=$(sed '/^[[:space:]]*$/d' "build.$(go env | grep GOARCH | cut -d'=' -f2 | tr -d '"')" | tr '\n' '|')
go build -a -v -installsuffix cgo -ldflags "-w -s -extldflags '-static' " $(go list ./... | grep -vPi ${IGNORE_BUILD::-1})
- name: Test Build Linux/386 with suffix
continue-on-error: false
run: |
CGO_ENABLED=0
GOOS=linux
GOARCH=386
IGNORE_BUILD=$(sed '/^[[:space:]]*$/d' "build.$(go env | grep GOARCH | cut -d'=' -f2 | tr -d '"')" | tr '\n' '|')
go build -a -v -installsuffix cgo -ldflags "-w -s -extldflags '-static' " $(go list ./... | grep -vPi ${IGNORE_BUILD::-1})
CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GOAMD64=v4 go build -a -v -race -installsuffix cgo -ldflags "-w -s -extldflags '-static' " $(go list ./... | grep -vPi ${IGNORE_BUILD::-1})
- name: Test Build Windows/amd64 with CGO
continue-on-error: false
run: |
CC=/usr/bin/x86_64-w64-mingw32-gcc
CGO_ENABLED=1
GOOS=windows
GOARCH=amd64
GOAMD64=v4
IGNORE_BUILD=$(sed '/^[[:space:]]*$/d' "build.$(go env | grep GOARCH | cut -d'=' -f2 | tr -d '"')" | tr '\n' '|')
go build -a -v -ldflags "-w -s -extldflags '-static' " $(go list ./... | grep -vPi ${IGNORE_BUILD::-1})
CC=/usr/bin/x86_64-w64-mingw32-gcc CGO_ENABLED=1 GOOS=windows GOARCH=amd64 GOAMD64=v4 go build -a -v -ldflags "-w -s -extldflags '-static' " $(go list ./... | grep -vPi ${IGNORE_BUILD::-1})
- name: Test Build Windows/386 with CGO
- name: Test Build Darwin/arm64 with suffix/CGO
continue-on-error: false
run: |
CC=/usr/bin/i686-w64-mingw32-gcc
CGO_ENABLED=1
GOOS=windows
GOARCH=386
IGNORE_BUILD=$(sed '/^[[:space:]]*$/d' "build.$(go env | grep GOARCH | cut -d'=' -f2 | tr -d '"')" | tr '\n' '|')
go build -a -v -ldflags "-w -s -extldflags '-static' " $(go list ./... | grep -vPi ${IGNORE_BUILD::-1})
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 GOAMD64=v1 go build -a -v -ldflags "-w -s -extldflags '-static' " $(go list ./... | grep -vPi ${IGNORE_BUILD::-1})
41 changes: 27 additions & 14 deletions aws/configAws/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,12 @@ import (
"net"
"net/url"

libsts "github.com/nabbar/golib/status/config"
moncfg "github.com/nabbar/golib/monitor/types"

sdkaws "github.com/aws/aws-sdk-go-v2/aws"
libval "github.com/go-playground/validator/v10"
"github.com/nabbar/golib/errors"
"github.com/nabbar/golib/httpcli"
liberr "github.com/nabbar/golib/errors"
libhtc "github.com/nabbar/golib/httpcli"
)

type Model struct {
Expand All @@ -47,16 +47,17 @@ type Model struct {
}

type ModelStatus struct {
Config Model `json:"config" yaml:"config" toml:"config" mapstructure:"config" validate:"required,dive"`
Status libsts.ConfigStatus `json:"status" yaml:"status" toml:"status" mapstructure:"status" validate:"required,dive"`
Config Model `json:"config" yaml:"config" toml:"config" mapstructure:"config" validate:"required,dive"`
HTTPClient libhtc.Options `json:"http-client" yaml:"http-client" toml:"http-client" mapstructure:"http-client" validate:"required,dive"`
Monitor moncfg.Config `json:"monitor" yaml:"monitor" toml:"monitor" mapstructure:"monitor" validate:"required,dive"`
}

type awsModel struct {
Model
retryer func() sdkaws.Retryer
}

func (c *awsModel) Validate() errors.Error {
func (c *awsModel) Validate() liberr.Error {
err := ErrorConfigValidator.Error(nil)

if er := libval.New().Struct(c); er != nil {
Expand Down Expand Up @@ -89,11 +90,11 @@ func (c *awsModel) SetCredentials(accessKey, secretKey string) {
func (c *awsModel) ResetRegionEndpoint() {
}

func (c *awsModel) RegisterRegionEndpoint(region string, endpoint *url.URL) errors.Error {
func (c *awsModel) RegisterRegionEndpoint(region string, endpoint *url.URL) liberr.Error {
return nil
}

func (c *awsModel) RegisterRegionAws(endpoint *url.URL) errors.Error {
func (c *awsModel) RegisterRegionAws(endpoint *url.URL) liberr.Error {
return nil
}

Expand All @@ -108,14 +109,26 @@ func (c *awsModel) GetRegion() string {
func (c *awsModel) SetEndpoint(endpoint *url.URL) {
}

func (c awsModel) GetEndpoint() *url.URL {
func (c *awsModel) GetEndpoint() *url.URL {
return nil
}

func (c *awsModel) ResolveEndpoint(service, region string) (sdkaws.Endpoint, error) {
return sdkaws.Endpoint{}, ErrorEndpointInvalid.Error(nil)
}

func (c *awsModel) ResolveEndpointWithOptions(service, region string, options ...interface{}) (sdkaws.Endpoint, error) {
return sdkaws.Endpoint{}, ErrorEndpointInvalid.Error(nil)
}

func (c *awsModel) GetDisableHTTPS() bool {
return false
}

func (c *awsModel) GetResolvedRegion() string {
return c.GetRegion()
}

func (c *awsModel) IsHTTPs() bool {
return true
}
Expand All @@ -124,14 +137,14 @@ func (c *awsModel) SetRetryer(retryer func() sdkaws.Retryer) {
c.retryer = retryer
}

func (c awsModel) Check(ctx context.Context) errors.Error {
func (c *awsModel) Check(ctx context.Context) liberr.Error {
var (
cfg *sdkaws.Config
con net.Conn
end sdkaws.Endpoint
adr *url.URL
err error
e errors.Error
e liberr.Error
)

if cfg, e = c.GetConfig(ctx, nil); e != nil {
Expand All @@ -142,7 +155,7 @@ func (c awsModel) Check(ctx context.Context) errors.Error {
ctx = context.Background()
}

if end, err = cfg.EndpointResolver.ResolveEndpoint("s3", c.GetRegion()); err != nil {
if end, err = cfg.EndpointResolverWithOptions.ResolveEndpoint("s3", c.GetRegion()); err != nil {
return ErrorEndpointInvalid.ErrorParent(err)
}

Expand All @@ -155,8 +168,8 @@ func (c awsModel) Check(ctx context.Context) errors.Error {
}

d := net.Dialer{
Timeout: httpcli.ClientTimeout5Sec,
KeepAlive: httpcli.ClientTimeout5Sec,
Timeout: libhtc.ClientTimeout5Sec,
KeepAlive: libhtc.ClientTimeout5Sec,
}

con, err = d.DialContext(ctx, "tcp", adr.Host)
Expand Down
1 change: 1 addition & 0 deletions aws/configCustom/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ func (c *awsModel) GetConfig(ctx context.Context, cli *http.Client) (*sdkaws.Con
cfg.Credentials = sdkcrd.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, "")
cfg.Retryer = c.retryer
cfg.EndpointResolver = sdkaws.EndpointResolverFunc(c.ResolveEndpoint)
cfg.EndpointResolverWithOptions = sdkaws.EndpointResolverWithOptionsFunc(c.ResolveEndpointWithOptions)
cfg.Region = c.Region

if cli != nil {
Expand Down
26 changes: 20 additions & 6 deletions aws/configCustom/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,13 @@ import (
"net/url"
"strings"

moncfg "github.com/nabbar/golib/monitor/types"

sdkaws "github.com/aws/aws-sdk-go-v2/aws"
libval "github.com/go-playground/validator/v10"
liberr "github.com/nabbar/golib/errors"
libhtc "github.com/nabbar/golib/httpcli"
liblog "github.com/nabbar/golib/logger"
libsts "github.com/nabbar/golib/status/config"
)

type Model struct {
Expand All @@ -49,8 +50,9 @@ type Model struct {
}

type ModelStatus struct {
Config Model `json:"config" yaml:"config" toml:"config" mapstructure:"config" validate:"required,dive"`
Status libsts.ConfigStatus `json:"status" yaml:"status" toml:"status" mapstructure:"status" validate:"required,dive"`
Config Model `json:"config" yaml:"config" toml:"config" mapstructure:"config" validate:"required,dive"`
HTTPClient libhtc.Options `json:"http-client" yaml:"http-client" toml:"http-client" mapstructure:"http-client" validate:"required,dive"`
Monitor moncfg.Config `json:"monitor" yaml:"monitor" toml:"monitor" mapstructure:"monitor" validate:"required,dive"`
}

type awsModel struct {
Expand Down Expand Up @@ -220,11 +222,15 @@ func (c *awsModel) SetEndpoint(endpoint *url.URL) {
c.Endpoint = strings.TrimSuffix(c.endpoint.String(), "/")
}

func (c awsModel) GetEndpoint() *url.URL {
func (c *awsModel) GetEndpoint() *url.URL {
return c.endpoint
}

func (c *awsModel) ResolveEndpoint(service, region string) (sdkaws.Endpoint, error) {
return c.ResolveEndpointWithOptions(service, region)
}

func (c *awsModel) ResolveEndpointWithOptions(service, region string, options ...interface{}) (sdkaws.Endpoint, error) {
if e, ok := c.mapRegion[region]; ok {
return sdkaws.Endpoint{
URL: strings.TrimSuffix(e.String(), "/"),
Expand All @@ -245,6 +251,14 @@ func (c *awsModel) ResolveEndpoint(service, region string) (sdkaws.Endpoint, err
return sdkaws.Endpoint{}, ErrorEndpointInvalid.Error(nil)
}

func (c *awsModel) GetDisableHTTPS() bool {
return false
}

func (c *awsModel) GetResolvedRegion() string {
return c.GetRegion()
}

func (c *awsModel) IsHTTPs() bool {
return strings.HasSuffix(strings.ToLower(c.endpoint.Scheme), "s")
}
Expand All @@ -253,7 +267,7 @@ func (c *awsModel) SetRetryer(retryer func() sdkaws.Retryer) {
c.retryer = retryer
}

func (c awsModel) Check(ctx context.Context) liberr.Error {
func (c *awsModel) Check(ctx context.Context) liberr.Error {
var (
cfg *sdkaws.Config
con net.Conn
Expand All @@ -269,7 +283,7 @@ func (c awsModel) Check(ctx context.Context) liberr.Error {
ctx = context.Background()
}

if _, err = cfg.EndpointResolver.ResolveEndpoint("s3", c.GetRegion()); err != nil {
if _, err = cfg.EndpointResolverWithOptions.ResolveEndpoint("s3", c.GetRegion()); err != nil {
return ErrorEndpointInvalid.ErrorParent(err)
}

Expand Down
3 changes: 3 additions & 0 deletions aws/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ type Config interface {

IsHTTPs() bool
ResolveEndpoint(service, region string) (sdkaws.Endpoint, error)
ResolveEndpointWithOptions(service, region string, options ...interface{}) (sdkaws.Endpoint, error)
GetDisableHTTPS() bool
GetResolvedRegion() string
SetRetryer(retryer func() sdkaws.Retryer)

GetConfig(ctx context.Context, cli *http.Client) (*sdkaws.Config, liberr.Error)
Expand Down
27 changes: 27 additions & 0 deletions cache/expiration.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
* MIT License
*
* Copyright (c) 2022 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/

package cache
73 changes: 73 additions & 0 deletions cache/interface.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
* MIT License
*
* Copyright (c) 2022 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/

package cache

import (
"context"
"sync"
"time"
)

type FuncCache[T any] func() Cache[T]

type Cache[T any] interface {
Clone(ctx context.Context, exp time.Duration) Cache[T]
Close()
Clean()

Merge(c Cache[T])
Walk(fct func(key any, val interface{}, exp time.Duration) bool)

Load(key any) (val interface{}, exp time.Duration, ok bool)
Store(key any, val interface{}) time.Duration
Delete(key any)

LoadOrStore(key any, val interface{}) (res interface{}, exp time.Duration, loaded bool)
LoadAndDelete(key any) (val interface{}, loaded bool)
}

func New[T any](ctx context.Context, exp time.Duration) Cache[T] {
if ctx == nil {
ctx = context.Background()
}

if exp < time.Microsecond {
return nil
}

n := &cache[T]{
Context: ctx,
w: sync.RWMutex{},
m: sync.Map{},
c: make(chan struct{}),
e: exp,
}

go n.ticker(exp)

return n
}
Loading

0 comments on commit 443935a

Please sign in to comment.