testutil/promrated: add metric to count errors from rated api (#1739)

Rated recently added authentication and this has been failing silently as the error was just being logged

category: bug
ticket: #1738

testutil/promrated/metrics.go

@@ -59,4 +59,10 @@ var (
 		Name:      "validator_effectiveness",
 		Help:      "Effectiveness of a validation key.",
 	}, labels)
+
+	ratedErrors = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "promrated",
+		Name:      "api_error_total",
+		Help:      "Total number of rated api errors",
+	}, []string{"peer"})
 )

testutil/promrated/promrated.go

@@ -30,6 +30,7 @@ import (
 
 type Config struct {
 	RatedEndpoint  string
+	RatedAuth      string
 	PromEndpoint   string
 	PromAuth       string
 	MonitoringAddr string
@@ -89,7 +90,7 @@ func reportMetrics(ctx context.Context, config Config) {
 			z.Str("cluster_network", validator.ClusterNetwork),
 		)
 
-		stats, err := getValidationStatistics(ctx, config.RatedEndpoint, validator)
+		stats, err := getValidationStatistics(ctx, config.RatedEndpoint, config.RatedAuth, validator)
 		if err != nil {
 			log.Error(ctx, "Getting validator statistics", err, z.Str("validator", validator.PubKey))
 			continue

testutil/promrated/promrated/main.go

@@ -53,6 +53,7 @@ func newRootCmd(runFunc func(context.Context, promrated.Config) error) *cobra.Co
 
 func bindPromratedFlag(flags *pflag.FlagSet, config *promrated.Config) {
 	flags.StringVar(&config.RatedEndpoint, "rated-endpoint", "https://api.rated.network", "Rated API endpoint to poll for validator metrics.")
+	flags.StringVar(&config.RatedAuth, "rated-auth-token", "token", "[REQUIRED] Token for Rated API.")
 	flags.StringVar(&config.MonitoringAddr, "monitoring-address", "127.0.0.1:9100", "Listening address (ip and port) for the prometheus monitoring http server.")
 	flags.StringVar(&config.PromEndpoint, "prom-endpoint", "https://vm.monitoring.gcp.obol.tech/query", "Endpoint for VMetrics Prometheus API.")
 	flags.StringVar(&config.PromAuth, "prom-auth-token", "token", "[REQUIRED] Token for VMetrics Promtetheus API.")

testutil/promrated/rated.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"strconv"
 	"time"
 
 	"github.com/obolnetwork/charon/app/errors"
@@ -41,7 +42,7 @@ type validatorEffectivenessData struct {
 
 // getValidationStatistics queries rated for a pubkey and returns rated data about the pubkey
 // See https://api.rated.network/docs#/default/get_effectiveness_v0_eth_validators__validator_index_or_pubkey__effectiveness_get
-func getValidationStatistics(ctx context.Context, ratedEndpoint string, validator validator) (validatorEffectivenessData, error) {
+func getValidationStatistics(ctx context.Context, ratedEndpoint string, ratedAuth string, validator validator) (validatorEffectivenessData, error) {
 	url, err := url.Parse(ratedEndpoint)
 	if err != nil {
 		return validatorEffectivenessData{}, errors.Wrap(err, "parse rated endpoint")
@@ -74,6 +75,7 @@ func getValidationStatistics(ctx context.Context, ratedEndpoint string, validato
 			return validatorEffectivenessData{}, errors.Wrap(err, "new rated request")
 		}
 
+		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ratedAuth))
 		req.Header.Add("X-Rated-Network", clusterNetwork)
 		res, err := client.Do(req)
 		if err != nil {
@@ -91,6 +93,8 @@ func getValidationStatistics(ctx context.Context, ratedEndpoint string, validato
 
 			continue
 		} else if res.StatusCode/100 != 2 {
+			incRatedErrors(req.URL.String(), res.StatusCode)
+
 			return validatorEffectivenessData{}, errors.New("not ok http response", z.Str("body", string(body)))
 		}
 
@@ -128,3 +132,7 @@ func extractBody(res *http.Response) ([]byte, error) {
 
 	return body, nil
 }
+
+func incRatedErrors(endpoint string, statusCode int) {
+	ratedErrors.WithLabelValues(endpoint, strconv.Itoa(statusCode)).Inc()
+}

testutil/promrated/rated_internal_test.go

@@ -33,13 +33,15 @@ func TestGetValidationStatistics(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "/v0/eth/validators/0xA/effectiveness", r.URL.Path)
 		require.Equal(t, "1", r.URL.Query()["size"][0])
+		require.Equal(t, "Bearer auth", r.Header.Get("Authorization"))
+		require.Equal(t, "prater", r.Header.Get("X-Rated-Network"))
 		_, _ = w.Write([]byte(ratedFixture))
 	}))
 	defer ts.Close()
 
 	validator := validator{ClusterName: "test-cluster", ClusterHash: "hash", ClusterNetwork: "goerli", PubKey: "0xA"}
 
-	vals, err := getValidationStatistics(context.Background(), ts.URL, validator)
+	vals, err := getValidationStatistics(context.Background(), ts.URL, "auth", validator)
 	assert.NoError(t, err)
 	testutil.RequireGoldenJSON(t, vals)
 }