Merge pull request #157 from nabbar/pkg_monitor_metrics

dependencies & some metrics

go.mod

@@ -4,9 +4,9 @@ go 1.20
 
 require (
 	github.com/aws/aws-sdk-go-v2 v1.17.6
-	github.com/aws/aws-sdk-go-v2/config v1.18.16
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.16
-	github.com/aws/aws-sdk-go-v2/service/iam v1.19.5
+	github.com/aws/aws-sdk-go-v2/config v1.18.17
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.17
+	github.com/aws/aws-sdk-go-v2/service/iam v1.19.6
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.30.6
 	github.com/bits-and-blooms/bitset v1.5.0
 	github.com/c-bata/go-prompt v0.2.6
@@ -32,7 +32,7 @@ require (
 	github.com/nats-io/nats.go v1.24.0
 	github.com/nutsdb/nutsdb v0.12.0
 	github.com/onsi/ginkgo/v2 v2.9.1
-	github.com/onsi/gomega v1.27.3
+	github.com/onsi/gomega v1.27.4
 	github.com/pelletier/go-toml v1.9.5
 	github.com/prometheus/client_golang v1.14.0
 	github.com/shirou/gopsutil v3.21.11+incompatible
@@ -44,7 +44,7 @@ require (
 	github.com/xanzy/go-gitlab v0.81.0
 	github.com/xhit/go-simple-mail v2.2.2+incompatible
 	github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235
-	golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0
+	golang.org/x/exp v0.0.0-20230314191032-db074128a8ec
 	golang.org/x/net v0.8.0
 	golang.org/x/oauth2 v0.6.0
 	golang.org/x/sync v0.1.0
@@ -74,7 +74,7 @@ require (
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.24 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.0 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 // indirect
@@ -89,7 +89,7 @@ require (
 	github.com/aws/smithy-go v1.13.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0 // indirect
-	github.com/bytedance/sonic v1.8.3 // indirect
+	github.com/bytedance/sonic v1.8.5 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/cockroachdb/errors v1.9.1 // indirect
@@ -107,7 +107,7 @@ require (
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-sql-driver/mysql v1.7.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
-	github.com/goccy/go-json v0.10.0 // indirect
+	github.com/goccy/go-json v0.10.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
@@ -139,7 +139,7 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/ratelimit v1.0.2 // indirect
-	github.com/klauspost/compress v1.16.0 // indirect
+	github.com/klauspost/compress v1.16.3 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
@@ -198,6 +198,6 @@ require (
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.7.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.29.0 // indirect
+	google.golang.org/protobuf v1.29.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 )

monitor/pool/metrics.go

@@ -39,17 +39,20 @@ import (
 )
 
 const (
-	metricBaseName  = "monitor"
-	metricLatency   = "latency"
-	metricUptime    = "uptime"
-	metricDowntime  = "downtime"
-	metricRiseTime  = "risetime"
-	metricFallTime  = "falltime"
-	metricStatus    = "status"
-	metricRise      = "rise"
-	metricFall      = "fall"
-	metricBoolTrue  = "true"
-	metricBoolFalse = "false"
+	metricBaseName = "monitor"
+	metricLatency  = "latency"
+	metricUptime   = "uptime"
+	metricDowntime = "downtime"
+	metricRiseTime = "risetime"
+	metricFallTime = "falltime"
+	metricStatus   = "status"
+	metricRise     = "rise"
+	metricFall     = "fall"
+	metricSLis     = "sli"
+
+	monitorMeans = "mean"
+	monitorMin   = "min"
+	monitorMax   = "max"
 )
 
 func (o *pool) normalizeName(name string) string {
@@ -400,6 +403,98 @@ func (o *pool) collectMetricFalling(ctx context.Context, m libmet.Metric) {
 	})
 }
 
+func (o *pool) createMetricsSLis() error {
+	var (
+		prm libprm.Prometheus
+		met libmet.Metric
+		mnm string
+	)
+
+	if prm = o.getProm(); prm == nil {
+		return nil
+	}
+
+	mnm = o.getMetricName(metricSLis)
+	met = libmet.NewMetrics(mnm, prmtps.Gauge)
+	met.SetDesc("the SLIs rate of each component")
+	met.AddLabel(metricBaseName)
+	met.SetCollect(o.collectMetricSLis)
+
+	return prm.AddMetric(false, met)
+}
+
+func (o *pool) collectMetricSLis(ctx context.Context, m libmet.Metric) {
+	var (
+		log         = o.getLog()
+		min float64 = 1
+		max float64 = 0
+		cur float64 = 0
+		cnt int     = 0
+		sum float64 = 0
+	)
+
+	o.MonitorWalk(func(name string, val montps.Monitor) bool {
+		cur = val.CollectDownTime().Seconds() / val.CollectUpTime().Seconds()
+		sum += cur
+		cnt++
+
+		if cur < min {
+			min = cur
+		}
+		if cur > max {
+			max = cur
+		}
+
+		if e := m.SetGaugeValue([]string{name}, cur); e != nil {
+			ent := log.Entry(liblog.ErrorLevel, "failed to collect metrics", nil)
+			ent.FieldAdd("monitor", name)
+			ent.FieldAdd("metric", val.Name())
+			ent.ErrorAdd(true, e)
+			ent.Log()
+		}
+
+		return true
+	})
+
+	mns := 1 - (sum / float64(cnt))
+	min = 1 - min
+	max = 1 - max
+
+	if mns < 0 {
+		mns = 0
+	}
+	if min < 0 {
+		min = 0
+	}
+	if max < 0 {
+		max = 0
+	}
+
+	if e := m.SetGaugeValue([]string{monitorMeans}, mns); e != nil {
+		ent := log.Entry(liblog.ErrorLevel, "failed to collect metrics", nil)
+		ent.FieldAdd("monitor", monitorMeans)
+		ent.FieldAdd("metric", metricSLis)
+		ent.ErrorAdd(true, e)
+		ent.Log()
+	}
+
+	if e := m.SetGaugeValue([]string{monitorMin}, min); e != nil {
+		ent := log.Entry(liblog.ErrorLevel, "failed to collect metrics", nil)
+		ent.FieldAdd("monitor", monitorMin)
+		ent.FieldAdd("metric", metricSLis)
+		ent.ErrorAdd(true, e)
+		ent.Log()
+	}
+
+	if e := m.SetGaugeValue([]string{monitorMax}, max); e != nil {
+		ent := log.Entry(liblog.ErrorLevel, "failed to collect metrics", nil)
+		ent.FieldAdd("monitor", monitorMax)
+		ent.FieldAdd("metric", metricSLis)
+		ent.ErrorAdd(true, e)
+		ent.Log()
+	}
+}
+
 func (o *pool) createMetrics() error {
 	if e := o.createMetricsLatency(); e != nil {
 		return e
@@ -433,5 +528,9 @@ func (o *pool) createMetrics() error {
 		return e
 	}
 
+	if e := o.createMetricsSLis(); e != nil {
+		return e
+	}
+
 	return nil
 }