Merge pull request #154 from nabbar/fix_2023-03-07

Fix Package Monitor - create function to init prom metrics - add logger to log errors - add function to trigger a duration interval between 2 collect metrics - fix use unic prom metrics for all component - fix encoding info text - fix status duration - bump dependencies
nabbar · Mar 8, 2023 · 54462f0 · 54462f0
2 parents a7b7657 + 6e005a7
commit 54462f0
Show file tree

Hide file tree

Showing 15 changed files with 622 additions and 405 deletions.
diff --git a/go.mod b/go.mod
@@ -28,10 +28,10 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/nats-io/jwt/v2 v2.3.0
-	github.com/nats-io/nats-server/v2 v2.9.14
+	github.com/nats-io/nats-server/v2 v2.9.15
 	github.com/nats-io/nats.go v1.24.0
 	github.com/nutsdb/nutsdb v0.12.0
-	github.com/onsi/ginkgo/v2 v2.8.4
+	github.com/onsi/ginkgo/v2 v2.9.0
 	github.com/onsi/gomega v1.27.2
 	github.com/pelletier/go-toml v1.9.5
 	github.com/prometheus/client_golang v1.14.0
@@ -41,28 +41,28 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0
 	github.com/spf13/viper v1.15.0
 	github.com/vbauerster/mpb/v5 v5.4.0
-	github.com/xanzy/go-gitlab v0.80.2
+	github.com/xanzy/go-gitlab v0.80.3
 	github.com/xhit/go-simple-mail v2.2.2+incompatible
 	github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235
-	golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2
-	golang.org/x/net v0.7.0
-	golang.org/x/oauth2 v0.5.0
+	golang.org/x/exp v0.0.0-20230307190834-24139beb5833
+	golang.org/x/net v0.8.0
+	golang.org/x/oauth2 v0.6.0
 	golang.org/x/sync v0.1.0
-	golang.org/x/sys v0.5.0
-	golang.org/x/term v0.5.0
+	golang.org/x/sys v0.6.0
+	golang.org/x/term v0.6.0
 	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/driver/clickhouse v0.5.0
 	gorm.io/driver/mysql v1.4.7
 	gorm.io/driver/postgres v1.4.8
 	gorm.io/driver/sqlite v1.4.4
 	gorm.io/driver/sqlserver v1.4.2
-	gorm.io/gorm v1.24.5
+	gorm.io/gorm v1.24.6
 )
 
 require (
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/ClickHouse/ch-go v0.53.0 // indirect
-	github.com/ClickHouse/clickhouse-go/v2 v2.6.5 // indirect
+	github.com/ClickHouse/clickhouse-go/v2 v2.7.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
@@ -89,14 +89,14 @@ require (
 	github.com/aws/smithy-go v1.13.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0 // indirect
-	github.com/bytedance/sonic v1.8.2 // indirect
+	github.com/bytedance/sonic v1.8.3 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/cockroachdb/errors v1.9.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/pebble v0.0.0-20210331181633-27fc006b8bfb // indirect
 	github.com/cockroachdb/redact v1.1.3 // indirect
-	github.com/getsentry/sentry-go v0.18.0 // indirect
+	github.com/getsentry/sentry-go v0.19.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
 	github.com/go-faster/city v1.0.1 // indirect
@@ -167,15 +167,15 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pkg/term v1.2.0-beta.2 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.40.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
-	github.com/spf13/afero v1.9.4 // indirect
+	github.com/spf13/afero v1.9.5 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
@@ -191,12 +191,12 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.opentelemetry.io/otel v1.14.0 // indirect
 	go.opentelemetry.io/otel/trace v1.14.0 // indirect
-	golang.org/x/arch v0.2.0 // indirect
-	golang.org/x/crypto v0.6.0 // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/arch v0.3.0 // indirect
+	golang.org/x/crypto v0.7.0 // indirect
+	golang.org/x/mod v0.9.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
+	golang.org/x/tools v0.7.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

diff --git a/monitor/info/encode.go b/monitor/info/encode.go
@@ -48,11 +48,11 @@ func (e *encodingModel) stringInfo() string {
 		buf = bytes.NewBuffer(make([]byte, 0))
 	)
 
-	for _, i := range e.Info {
-		buf.WriteString(fmt.Sprintf("%v", i))
+	for n, i := range e.Info {
+		buf.WriteString(fmt.Sprintf("%s: %v,", n, i))
 	}
 
-	return buf.String()
+	return strings.Trim(strings.TrimSpace(buf.String()), ",")
 }
 
 func (e *encodingModel) stringClean(str string) string {

diff --git a/monitor/internalConfig.go b/monitor/internalConfig.go
@@ -33,8 +33,6 @@ import (
 
 	"github.com/nabbar/golib/monitor/types"
 
-	monsts "github.com/nabbar/golib/monitor/status"
-
 	liberr "github.com/nabbar/golib/errors"
 	liblog "github.com/nabbar/golib/logger"
 )
@@ -264,32 +262,14 @@ func (o *mon) getFct() types.HealthCheck {
 
 func (o *mon) getLastCheck() *lastRun {
 	if i, l := o.x.Load(keyLastRun); !l {
-		return &lastRun{
-			status:  monsts.KO,
-			runtime: time.Now(),
-			isRise:  false,
-			isFall:  false,
-		}
+		return newLastRun()
 	} else if v, k := i.(*lastRun); !k {
-		return &lastRun{
-			status:  monsts.KO,
-			runtime: time.Now(),
-			isRise:  false,
-			isFall:  false,
-		}
+		return newLastRun()
 	} else {
 		return v
 	}
 }
 
-func (o *mon) setLastCheck(m middleWare) error {
-	e := m.Next()
-	l := &lastRun{
-		status:  o.Status(),
-		runtime: time.Now(),
-		isRise:  o.IsRise(),
-		isFall:  o.IsFall(),
-	}
+func (o *mon) setLastCheck(l *lastRun) {
 	o.x.Store(keyLastRun, l)
-	return e
 }
diff --git a/monitor/last.go b/monitor/last.go
@@ -27,14 +27,211 @@
 package monitor
 
 import (
+	"fmt"
+	"sync"
 	"time"
 
 	monsts "github.com/nabbar/golib/monitor/status"
 )
 
 type lastRun struct {
+	m sync.RWMutex
+
 	status  monsts.Status
 	runtime time.Time
-	isRise  bool
-	isFall  bool
+
+	isRise bool
+	isFall bool
+
+	cntRise uint8
+	cntFall uint8
+
+	uptime   time.Duration
+	downtime time.Duration
+	riseTime time.Duration
+	fallTime time.Duration
+	latency  time.Duration
+
+	err error
+}
+
+func newLastRun() *lastRun {
+	return &lastRun{
+		m:        sync.RWMutex{},
+		status:   monsts.KO,
+		runtime:  time.Now(),
+		isRise:   false,
+		isFall:   false,
+		cntRise:  0,
+		cntFall:  0,
+		uptime:   0,
+		downtime: 0,
+		riseTime: 0,
+		fallTime: 0,
+		latency:  0,
+		err:      fmt.Errorf("no healcheck still run"),
+	}
+}
+
+func (o *lastRun) Latency() time.Duration {
+	o.m.RLock()
+	defer o.m.RUnlock()
+	return o.latency
+}
+
+func (o *lastRun) FallTime() time.Duration {
+	o.m.RLock()
+	defer o.m.RUnlock()
+	return o.fallTime
+}
+
+func (o *lastRun) RiseTime() time.Duration {
+	o.m.RLock()
+	defer o.m.RUnlock()
+	return o.riseTime
+}
+
+func (o *lastRun) UpTime() time.Duration {
+	o.m.RLock()
+	defer o.m.RUnlock()
+	return o.uptime
+}
+
+func (o *lastRun) DownTime() time.Duration {
+	o.m.RLock()
+	defer o.m.RUnlock()
+	return o.downtime
+}
+
+func (o *lastRun) Status() monsts.Status {
+	o.m.RLock()
+	defer o.m.RUnlock()
+	return o.status
+}
+
+func (o *lastRun) IsRise() bool {
+	o.m.RLock()
+	defer o.m.RUnlock()
+	return o.isRise
+}
+
+func (o *lastRun) IsFall() bool {
+	o.m.RLock()
+	defer o.m.RUnlock()
+	return o.isFall
+}
+
+func (o *lastRun) Error() error {
+	o.m.RLock()
+	defer o.m.RUnlock()
+	return o.err
+}
+
+func (o *lastRun) setStatus(err error, dur time.Duration, cfg *runCfg) {
+	o.m.Lock()
+	defer o.m.Unlock()
+
+	o.latency = dur
+
+	if err != nil {
+		o.err = err
+		o.setStatusFall(cfg)
+	} else {
+		o.err = nil
+		o.setStatusRise(cfg)
+	}
+}
+
+func (o *lastRun) setStatusFall(cfg *runCfg) {
+	if cfg == nil {
+		return
+	}
+
+	sts := o.status
+	dur := time.Since(o.runtime)
+	o.runtime = time.Now()
+
+	o.cntRise = 0
+	o.cntFall++
+
+	switch sts {
+	case monsts.OK:
+		if o.cntFall >= cfg.riseCountWarn {
+			o.cntFall = 0
+			o.status = monsts.Warn
+		} else {
+			o.status = monsts.OK
+		}
+		o.isFall = true
+		o.isRise = false
+		o.fallTime += dur
+		o.uptime += dur
+
+	case monsts.Warn:
+		if o.cntFall >= cfg.riseCountKO {
+			o.isFall = false
+			o.cntFall = 0
+			o.status = monsts.KO
+		} else {
+			o.isFall = true
+			o.status = monsts.Warn
+		}
+		o.isRise = false
+		o.fallTime += dur
+		o.downtime += dur
+
+	default:
+		o.cntFall = 0
+		o.isFall = false
+		o.isRise = false
+		o.status = monsts.KO
+		o.downtime += dur
+	}
+}
+
+func (o *lastRun) setStatusRise(cfg *runCfg) {
+	if cfg == nil {
+		return
+	}
+
+	sts := o.status
+	dur := time.Since(o.runtime)
+	o.runtime = time.Now()
+
+	o.cntFall = 0
+	o.cntRise++
+
+	switch sts {
+	case monsts.KO:
+		if o.cntRise >= cfg.riseCountKO {
+			o.cntRise = 0
+			o.status = monsts.Warn
+		} else {
+			o.status = monsts.KO
+		}
+		o.isFall = false
+		o.isRise = true
+		o.riseTime += dur
+		o.downtime += dur
+
+	case monsts.Warn:
+		if o.cntRise >= cfg.riseCountWarn {
+			o.cntRise = 0
+			o.isRise = false
+			o.status = monsts.OK
+		} else {
+			o.isRise = true
+			o.status = monsts.Warn
+		}
+		o.isFall = false
+		o.riseTime += dur
+		o.downtime += dur
+
+	default:
+		o.cntRise = 0
+		o.isFall = false
+		o.isRise = false
+		o.status = monsts.OK
+		o.uptime += dur
+	}
 }