monitoring: use instrumented backoff as default for waiting (#71)

Signed-off-by: Michael Hoffmann <[email protected]>

monitoring/instrumented.go

@@ -71,7 +71,7 @@ func WithInstrumentedScheme(scheme string) InstrumentedOption {
 	}
 }
 
-// WithInstrumentedWaitBackoff allows customizing wait backoff when accessing metric endpoint.
+// WithInstrumentedWaitBackoff allows customizing wait backoff when accessing or asserting on the metric endpoint.
 func WithInstrumentedWaitBackoff(waitBackoff *backoff.Backoff) InstrumentedOption {
 	return func(o *rOpt) {
 		o.waitBackoff = waitBackoff
@@ -144,6 +144,17 @@ func (r *InstrumentedRunnable) Metrics() (_ string, err error) {
 	return string(body), err
 }
 
+func (r *InstrumentedRunnable) buildMetricsOptions(opts []MetricsOption) metricsOptions {
+	result := metricsOptions{
+		getValue:    getMetricValue,
+		waitBackoff: r.waitBackoff,
+	}
+	for _, opt := range opts {
+		opt(&result)
+	}
+	return result
+}
+
 // WaitSumMetrics waits for at least one instance of each given metric names to be present and their sums,
 // returning true when passed to given expected(...).
 func (r *InstrumentedRunnable) WaitSumMetrics(expected MetricValueExpectation, metricNames ...string) error {
@@ -154,14 +165,13 @@ func (r *InstrumentedRunnable) WaitSumMetricsWithOptions(expected MetricValueExp
 	var (
 		sums    []float64
 		err     error
-		options = buildMetricsOptions(opts)
+		options = r.buildMetricsOptions(opts)
 	)
 
-	metricsWaitBackoff := backoff.New(context.Background(), *options.waitBackoff)
-	for metricsWaitBackoff.Reset(); metricsWaitBackoff.Ongoing(); {
+	for options.waitBackoff.Reset(); options.waitBackoff.Ongoing(); {
 		sums, err = r.SumMetrics(metricNames, opts...)
 		if options.waitMissingMetrics && errors.Is(err, errMissingMetric) {
-			metricsWaitBackoff.Wait()
+			options.waitBackoff.Wait()
 			continue
 		}
 		if err != nil {
@@ -171,15 +181,14 @@ func (r *InstrumentedRunnable) WaitSumMetricsWithOptions(expected MetricValueExp
 		if expected(sums...) {
 			return nil
 		}
-
-		metricsWaitBackoff.Wait()
+		options.waitBackoff.Wait()
 	}
-	return errors.Newf("unable to find metrics %s with expected values after %d retries. Last error: %v. Last values: %v", metricNames, metricsWaitBackoff.NumRetries(), err, sums)
+	return errors.Newf("unable to find metrics %s with expected values after %d retries. Last error: %v. Last values: %v", metricNames, options.waitBackoff.NumRetries(), err, sums)
 }
 
 // SumMetrics returns the sum of the values of each given metric names.
 func (r *InstrumentedRunnable) SumMetrics(metricNames []string, opts ...MetricsOption) ([]float64, error) {
-	options := buildMetricsOptions(opts)
+	options := r.buildMetricsOptions(opts)
 	sums := make([]float64, len(metricNames))
 
 	metrics, err := r.Metrics()
@@ -224,9 +233,9 @@ func (r *InstrumentedRunnable) SumMetrics(metricNames []string, opts ...MetricsO
 
 // WaitRemovedMetric waits until a metric disappear from the list of metrics exported by the service.
 func (r *InstrumentedRunnable) WaitRemovedMetric(metricName string, opts ...MetricsOption) error {
-	options := buildMetricsOptions(opts)
+	options := r.buildMetricsOptions(opts)
 
-	for r.waitBackoff.Reset(); r.waitBackoff.Ongoing(); {
+	for options.waitBackoff.Reset(); options.waitBackoff.Ongoing(); {
 		// Fetch metrics.
 		metrics, err := r.Metrics()
 		if err != nil {
@@ -251,7 +260,7 @@ func (r *InstrumentedRunnable) WaitRemovedMetric(metricName string, opts ...Metr
 			return nil
 		}
 
-		r.waitBackoff.Wait()
+		options.waitBackoff.Wait()
 	}
 
 	return errors.Newf("the metric %s is still exported by %s", metricName, r.Name())

monitoring/instrumented_composite.go

@@ -41,6 +41,17 @@ func (r *CompositeInstrumentedRunnable) MetricTargets() (ret []Target) {
 	return ret
 }
 
+func (r *CompositeInstrumentedRunnable) buildMetricsOptions(opts []MetricsOption) metricsOptions {
+	result := metricsOptions{
+		getValue:    getMetricValue,
+		waitBackoff: r.backoff,
+	}
+	for _, opt := range opts {
+		opt(&result)
+	}
+	return result
+}
+
 // WaitSumMetrics waits for at least one instance of each given metric names to be present and their sums, returning true
 // when passed to given expected(...).
 func (r *CompositeInstrumentedRunnable) WaitSumMetrics(expected MetricValueExpectation, metricNames ...string) error {
@@ -51,13 +62,13 @@ func (r *CompositeInstrumentedRunnable) WaitSumMetricsWithOptions(expected Metri
 	var (
 		sums    []float64
 		err     error
-		options = buildMetricsOptions(opts)
+		options = r.buildMetricsOptions(opts)
 	)
 
-	for r.backoff.Reset(); r.backoff.Ongoing(); {
+	for options.waitBackoff.Reset(); options.waitBackoff.Ongoing(); {
 		sums, err = r.SumMetrics(metricNames, opts...)
 		if options.waitMissingMetrics && errors.Is(err, errMissingMetric) {
-			r.backoff.Wait()
+			options.waitBackoff.Wait()
 			continue
 		}
 		if err != nil {
@@ -68,7 +79,7 @@ func (r *CompositeInstrumentedRunnable) WaitSumMetricsWithOptions(expected Metri
 			return nil
 		}
 
-		r.backoff.Wait()
+		options.waitBackoff.Wait()
 	}
 
 	return errors.Wrapf(err, "unable to find metrics %s with expected values. Last values: %v", metricNames, sums)

monitoring/instrumented_test.go

@@ -19,6 +19,7 @@ import (
 type runnableFake struct {
 	e2e.Runnable
 
+	running   bool
 	endpoints map[string]string
 }
 
@@ -30,6 +31,22 @@ func (r *runnableFake) Endpoint(portName string) string {
 	return r.endpoints[portName]
 }
 
+func (r *runnableFake) IsRunning() bool {
+	return r.running
+}
+
+func (r *runnableFake) SetMetadata(k, v any) {
+}
+
+func (r *runnableFake) Name() string {
+	return "fake"
+}
+
+func (r *runnableFake) Start() error {
+	r.running = true
+	return nil
+}
+
 func TestWaitSumMetric(t *testing.T) {
 	// Listen on a random port before starting the HTTP server, to
 	// make sure the port is already open when we'll call WaitSumMetric()
@@ -86,6 +103,7 @@ metric_b_summary_count 1
 			Max:        600 * time.Millisecond,
 			MaxRetries: 50,
 		})))
+	testutil.Ok(t, r.Start())
 
 	testutil.Ok(t, r.WaitSumMetrics(Equals(221), "metric_a"))
 
@@ -159,6 +177,7 @@ metric_b 1000
 			Max:        600 * time.Millisecond,
 			MaxRetries: 50,
 		})))
+	testutil.Ok(t, r.Start())
 
 	testutil.Ok(t, r.WaitSumMetrics(Equals(math.NaN()), "metric_a"))
 }

monitoring/metrics.go

@@ -4,8 +4,8 @@
 package e2emon
 
 import (
+	"context"
 	"math"
-	"time"
 
 	"github.com/efficientgo/core/backoff"
 	"github.com/efficientgo/e2e/monitoring/matchers"
@@ -24,13 +24,13 @@ type metricsOptions struct {
 	labelMatchers      []*matchers.Matcher
 	waitMissingMetrics bool
 	skipMissingMetrics bool
-	waitBackoff        *backoff.Config
+	waitBackoff        *backoff.Backoff
 }
 
 // WithWaitBackoff is an option to configure a backoff when waiting on a metric value.
 func WithWaitBackoff(backoffConfig *backoff.Config) MetricsOption {
 	return func(o *metricsOptions) {
-		o.waitBackoff = backoffConfig
+		o.waitBackoff = backoff.New(context.Background(), *backoffConfig)
 	}
 }
 
@@ -63,21 +63,6 @@ func SkipMissingMetrics() MetricsOption {
 	}
 }
 
-func buildMetricsOptions(opts []MetricsOption) metricsOptions {
-	result := metricsOptions{
-		getValue: getMetricValue,
-		waitBackoff: &backoff.Config{
-			Min:        300 * time.Millisecond,
-			Max:        600 * time.Millisecond,
-			MaxRetries: 50,
-		},
-	}
-	for _, opt := range opts {
-		opt(&result)
-	}
-	return result
-}
-
 func getMetricValue(m *io_prometheus_client.Metric) float64 {
 	if m.GetGauge() != nil {
 		return m.GetGauge().GetValue()