Add metrics for index insertion (#2015)

Add new summary vector metric `rekor_index_storage_latency_summary` to
enable tracking latency of entry insertion in the mysql or redis index
storage backend.

Since the index insertion is non-blocking, the existing API metrics are
unable to measure its latency. However, the speed of insertion affects
how fast the index is available to query, so it is relevant to rekor's
overall performance.

This new metric along with the existing API metrics will help give an
clearer picture of rekor's index storage performance.

Signed-off-by: Colleen Murphy <[email protected]>

pkg/api/entries.go

@@ -23,6 +23,8 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"strconv"
+	"time"
 
 	"github.com/cyberphone/json-canonicalization/go/src/webpki.org/jsoncanonicalizer"
 	"github.com/go-openapi/runtime"
@@ -246,6 +248,14 @@ func createLogEntry(params entries.CreateLogEntryParams) (models.LogEntry, middl
 
 	if indexStorageClient != nil {
 		go func() {
+			start := time.Now()
+			var err error
+			defer func() {
+				labels := map[string]string{
+					"success": strconv.FormatBool(err == nil),
+				}
+				metricIndexStorageLatency.With(labels).Observe(float64(time.Since(start)))
+			}()
 			keys, err := entry.IndexKeys()
 			if err != nil {
 				log.ContextLogger(ctx).Errorf("getting entry index keys: %v", err)

pkg/api/metrics.go

@@ -34,6 +34,11 @@ var (
 		Help: "The status of publishing events to Pub/Sub",
 	}, []string{"event", "content_type", "status"})
 
+	metricIndexStorageLatency = promauto.NewSummaryVec(prometheus.SummaryOpts{
+		Name: "rekor_index_storage_latency_summary",
+		Help: "Latency of backend index insertion by success/failure",
+	}, []string{"success"})
+
 	MetricLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
 		Name: "rekor_api_latency",
 		Help: "Api Latency on calls",