fix(probe): init probe metric gauges with zero value (#1162)

init probe metric gauges with zero value

probe/cmd/probe/main.go

@@ -27,7 +27,7 @@ func main() {
 		glog.Fatal(err)
 	}
 
-	if metricsServer := metrics.NewMetricsServer(config.MetricsAddress); metricsServer != nil {
+	if metricsServer := metrics.NewMetricsServer(config.MetricsAddress, config.DataPlaneRegion); metricsServer != nil {
 		defer metrics.CloseMetricsServer(metricsServer)
 		go metrics.ListenAndServe(metricsServer)
 	} else {

probe/pkg/metrics/metrics.go

@@ -42,6 +42,13 @@ func (m *Metrics) Register(r prometheus.Registerer) {
 	r.MustRegister(m.lastFailureTimestamp)
 }
 
+// Init sets initial values for the gauge metrics.
+func (m *Metrics) Init(region string) {
+	m.lastFailureTimestamp.With(prometheus.Labels{regionLabelName: region}).Set(0)
+	m.lastStartedTimestamp.With(prometheus.Labels{regionLabelName: region}).Set(0)
+	m.lastSuccessTimestamp.With(prometheus.Labels{regionLabelName: region}).Set(0)
+}
+
 // IncStartedRuns increments the metric counter for started probe runs.
 func (m *Metrics) IncStartedRuns(region string) {
 	m.startedRuns.With(prometheus.Labels{regionLabelName: region}).Inc()

probe/pkg/metrics/metrics_test.go

@@ -6,12 +6,21 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"
+	io_prometheus_client "github.com/prometheus/client_model/go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 var regionValue = "us-east-1"
 
+func getMetricSeries(t *testing.T, registry *prometheus.Registry, name string) *io_prometheus_client.Metric {
+	metrics := serveMetrics(t, registry)
+	require.Contains(t, metrics, name)
+	targetMetric := metrics[name]
+	require.NotEmpty(t, targetMetric.Metric)
+	return targetMetric.Metric[0]
+}
+
 func TestCounterIncrements(t *testing.T) {
 	const expectedIncrement = 1.0
 
@@ -43,15 +52,12 @@ func TestCounterIncrements(t *testing.T) {
 		tc := tc
 		t.Run(tc.metricName, func(t *testing.T) {
 			m := newMetrics()
+			registry := initPrometheus(m, regionValue)
 			tc.callIncrementFunc(m)
 
-			metrics := serveMetrics(t, m)
-			require.Contains(t, metrics, tc.metricName)
-			targetMetric := metrics[tc.metricName]
+			targetSeries := getMetricSeries(t, registry, tc.metricName)
 
 			// Test that the metrics value is 1 after calling the incrementFunc.
-			require.NotEmpty(t, targetMetric.Metric)
-			targetSeries := targetMetric.Metric[0]
 			value := targetSeries.GetCounter().GetValue()
 			assert.Equalf(t, expectedIncrement, value, "metric %s has unexpected value", tc.metricName)
 			label := targetSeries.GetLabel()[0]
@@ -90,16 +96,17 @@ func TestTimestampGauges(t *testing.T) {
 		tc := tc
 		t.Run(tc.metricName, func(t *testing.T) {
 			m := newMetrics()
+			registry := initPrometheus(m, regionValue)
 			lowerBound := time.Now().Unix()
-			tc.callSetTimestampFunc(m)
-
-			metrics := serveMetrics(t, m)
-			require.Contains(t, metrics, tc.metricName)
-			targetMetric := metrics[tc.metricName]
 
-			require.NotEmpty(t, targetMetric.Metric)
-			targetSeries := targetMetric.Metric[0]
+			targetSeries := getMetricSeries(t, registry, tc.metricName)
 			value := int64(targetSeries.GetGauge().GetValue())
+			assert.Zero(t, value)
+
+			tc.callSetTimestampFunc(m)
+
+			targetSeries = getMetricSeries(t, registry, tc.metricName)
+			value = int64(targetSeries.GetGauge().GetValue())
 			assert.GreaterOrEqualf(t, value, lowerBound, "metric %s has unexpected value", tc.metricName)
 			label := targetSeries.GetLabel()[0]
 			assert.Containsf(t, label.GetName(), regionLabelName, "metric %s has unexpected label", tc.metricName)
@@ -126,16 +133,13 @@ func TestHistograms(t *testing.T) {
 		tc := tc
 		t.Run(tc.metricName, func(t *testing.T) {
 			m := newMetrics()
+			registry := initPrometheus(m, regionValue)
 			expectedCount := uint64(2)
 			expectedSum := 480.0
 			tc.callObserveFunc(m)
 
-			metrics := serveMetrics(t, m)
-			require.Contains(t, metrics, tc.metricName)
-			targetMetric := metrics[tc.metricName]
+			targetSeries := getMetricSeries(t, registry, tc.metricName)
 
-			require.NotEmpty(t, targetMetric.Metric)
-			targetSeries := targetMetric.Metric[0]
 			count := targetSeries.GetHistogram().GetSampleCount()
 			sum := targetSeries.GetHistogram().GetSampleSum()
 			assert.Equalf(t, expectedCount, count, "expected metric: %s to have a count of %v", tc.metricName, expectedCount)

probe/pkg/metrics/server.go

@@ -10,8 +10,9 @@ import (
 )
 
 // NewMetricsServer returns the metrics server.
-func NewMetricsServer(address string) *http.Server {
-	return newMetricsServer(address, MetricsInstance())
+func NewMetricsServer(address string, region string) *http.Server {
+	registry := initPrometheus(MetricsInstance(), region)
+	return newMetricsServer(address, registry)
 }
 
 // ListenAndServe listens for incoming requests and serves the metrics.
@@ -28,14 +29,18 @@ func CloseMetricsServer(server *http.Server) {
 	}
 }
 
-func newMetricsServer(address string, customMetrics *Metrics) *http.Server {
+func initPrometheus(customMetrics *Metrics, region string) *prometheus.Registry {
 	registry := prometheus.NewRegistry()
 	// Register default metrics to use a dedicated registry instead of prometheus.DefaultRegistry.
 	// This makes it easier to isolate metric state when unit testing this package.
 	registry.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
 	registry.MustRegister(prometheus.NewGoCollector())
 	customMetrics.Register(registry)
+	customMetrics.Init(region)
+	return registry
+}
 
+func newMetricsServer(address string, registry *prometheus.Registry) *http.Server {
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
 

probe/pkg/metrics/server_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	io_prometheus_client "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
 	"github.com/stretchr/testify/assert"
@@ -15,13 +16,14 @@ import (
 type metricResponse map[string]*io_prometheus_client.MetricFamily
 
 func TestMetricsServerCorrectAddress(t *testing.T) {
-	server := NewMetricsServer(":8081")
+	server := NewMetricsServer(":8081", regionValue)
 	defer server.Close()
 	assert.Equal(t, ":8081", server.Addr)
 }
 
 func TestMetricsServerServesDefaultMetrics(t *testing.T) {
-	metrics := serveMetrics(t, newMetrics())
+	registry := initPrometheus(newMetrics(), regionValue)
+	metrics := serveMetrics(t, registry)
 	assert.Contains(t, metrics, "go_memstats_alloc_bytes", "expected metrics to contain go default metrics but it did not")
 }
 
@@ -35,7 +37,8 @@ func TestMetricsServerServesCustomMetrics(t *testing.T) {
 	customMetrics.SetLastSuccessTimestamp(regionValue)
 	customMetrics.SetLastFailureTimestamp(regionValue)
 	customMetrics.ObserveTotalDuration(time.Minute, regionValue)
-	metrics := serveMetrics(t, customMetrics)
+	registry := initPrometheus(customMetrics, regionValue)
+	metrics := serveMetrics(t, registry)
 
 	expectedKeys := []string{
 		"acs_probe_runs_started_total",
@@ -52,12 +55,12 @@ func TestMetricsServerServesCustomMetrics(t *testing.T) {
 	}
 }
 
-func serveMetrics(t *testing.T, customMetrics *Metrics) metricResponse {
+func serveMetrics(t *testing.T, registry *prometheus.Registry) metricResponse {
 	rec := httptest.NewRecorder()
 	req, err := http.NewRequest(http.MethodGet, "/metrics", nil)
 	require.NoError(t, err, "failed creating metrics requests")
 
-	server := newMetricsServer(":8081", customMetrics)
+	server := newMetricsServer(":8081", registry)
 	defer server.Close()
 	server.Handler.ServeHTTP(rec, req)
 	require.Equal(t, http.StatusOK, rec.Code, "status code should be OK")