Merge pull request #221 from ecosia/jg-prometheus

Adds Prometheus export (requests and variant eval)

Gopkg.toml

@@ -165,3 +165,7 @@
 [prune]
   go-tests = true
   unused-packages = true
+
+[[constraint]]
+  name = "github.com/prometheus/client_golang"
+  version = "0.9.2"

pkg/config/config.go

@@ -9,13 +9,16 @@ import (
 	"github.com/evalphobia/logrus_sentry"
 	raven "github.com/getsentry/raven-go"
 	newrelic "github.com/newrelic/go-agent"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/sirupsen/logrus"
 )
 
 // Global is the global dependency we can use, such as the new relic app instance
 var Global = struct {
 	NewrelicApp  newrelic.Application
 	StatsdClient *statsd.Client
+	Prometheus   prometheusMetrics
 }{}
 
 func init() {
@@ -25,6 +28,7 @@ func init() {
 	setupLogrus()
 	setupStatsd()
 	setupNewrelic()
+	setupPrometheus()
 }
 
 func setupLogrus() {
@@ -75,3 +79,31 @@ func setupNewrelic() {
 		Global.NewrelicApp = app
 	}
 }
+
+type prometheusMetrics struct {
+	ScrapePath       string
+	EvalCounter      *prometheus.CounterVec
+	RequestCounter   *prometheus.CounterVec
+	RequestHistogram *prometheus.HistogramVec
+}
+
+func setupPrometheus() {
+	if Config.PrometheusEnabled {
+		Global.Prometheus.ScrapePath = Config.PrometheusPath
+		Global.Prometheus.EvalCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "flagr_eval_results",
+			Help: "A counter of eval results",
+		}, []string{"EntityType", "FlagID", "VariantID", "VariantKey"})
+		Global.Prometheus.RequestCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "flagr_requests_total",
+			Help: "The total http requests received",
+		}, []string{"status", "path", "method"})
+
+		if Config.PrometheusIncludeLatencyHistogram {
+			Global.Prometheus.RequestHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{
+				Name: "flagr_requests_buckets",
+				Help: "A histogram of latencies for requests received",
+			}, []string{"status", "path", "method"})
+		}
+	}
+}

pkg/config/config_test.go

@@ -3,6 +3,7 @@ package config
 import (
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -29,3 +30,27 @@ func TestSetupStatsd(t *testing.T) {
 	})
 	Config.StatsdEnabled = false
 }
+
+func TestSetupPrometheus(t *testing.T) {
+	prometheus.DefaultRegisterer = prometheus.NewRegistry()
+	Config.PrometheusEnabled = false
+	setupPrometheus()
+	assert.Nil(t, Global.Prometheus.EvalCounter)
+	Config.PrometheusEnabled = true
+	setupPrometheus()
+	assert.NotNil(t, Global.Prometheus.EvalCounter)
+	assert.NotNil(t, Global.Prometheus.RequestCounter)
+	assert.Nil(t, Global.Prometheus.RequestHistogram)
+	Config.PrometheusEnabled = false
+}
+
+func TestSetupPrometheusWithLatencies(t *testing.T) {
+	prometheus.DefaultRegisterer = prometheus.NewRegistry()
+	Config.PrometheusEnabled = true
+	Config.PrometheusIncludeLatencyHistogram = true
+	setupPrometheus()
+	assert.NotNil(t, Global.Prometheus.EvalCounter)
+	assert.NotNil(t, Global.Prometheus.RequestCounter)
+	assert.NotNil(t, Global.Prometheus.RequestHistogram)
+	Config.PrometheusEnabled = false
+}

pkg/config/env.go

@@ -67,6 +67,13 @@ var Config = struct {
 	StatsdAPMPort        string `env:"FLAGR_STATSD_APM_PORT" envDefault:"8126"`
 	StatsdAPMServiceName string `env:"FLAGR_STATSD_APM_SERVICE_NAME" envDefault:"flagr"`
 
+	// PrometheusEnabled - enable prometheus metrics export
+	PrometheusEnabled bool `env:"FLAGR_PROMETHEUS_ENABLED" envDefault:"false"`
+	// PrometheusPath - set the path on which prometheus metrics are available to scrape
+	PrometheusPath string `env:"FLAGR_PROMETHEUS_PATH" envDefault:"/metrics"`
+	// PrometheusIncludeLatencyHistogram - set whether Prometheus should also export a histogram of request latencies (this increases cardinality significantly)
+	PrometheusIncludeLatencyHistogram bool `env:"FLAGR_PROMETHEUS_INCLUDE_LATENCY_HISTOGRAM" envDefault:"false"`
+
 	// RecorderEnabled - enable data records logging
 	RecorderEnabled bool `env:"FLAGR_RECORDER_ENABLED" envDefault:"false"`
 	// RecorderType - the pipeline to log data records, e.g. Kafka

pkg/config/middleware.go

@@ -8,15 +8,17 @@ import (
 	"time"
 
 	"github.com/DataDog/datadog-go/statsd"
-	"github.com/auth0/go-jwt-middleware"
-	"github.com/dgrijalva/jwt-go"
+	jwtmiddleware "github.com/auth0/go-jwt-middleware"
+	jwt "github.com/dgrijalva/jwt-go"
 	"github.com/gohttp/pprof"
-	"github.com/meatballhat/negroni-logrus"
+	negronilogrus "github.com/meatballhat/negroni-logrus"
 	"github.com/phyber/negroni-gzip/gzip"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/cors"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/negroni"
-	"github.com/yadvendar/negroni-newrelic-go-agent"
+	negroninewrelic "github.com/yadvendar/negroni-newrelic-go-agent"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 )
 
@@ -51,6 +53,13 @@ func SetupGlobalMiddleware(handler http.Handler) http.Handler {
 		}
 	}
 
+	if Config.PrometheusEnabled {
+		n.Use(&prometheusMiddleware{
+			counter:   Global.Prometheus.RequestCounter,
+			latencies: Global.Prometheus.RequestHistogram,
+		})
+	}
+
 	if Config.NewRelicEnabled {
 		n.Use(&negroninewrelic.Newrelic{Application: &Global.NewrelicApp})
 	}
@@ -214,3 +223,28 @@ func (s *statsdMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, nex
 
 	next(w, r)
 }
+
+type prometheusMiddleware struct {
+	counter   *prometheus.CounterVec
+	latencies *prometheus.HistogramVec
+}
+
+func (p *prometheusMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
+	if r.URL.EscapedPath() == Global.Prometheus.ScrapePath {
+		handler := promhttp.Handler()
+		handler.ServeHTTP(w, r)
+	} else {
+		defer func(start time.Time) {
+			response := w.(negroni.ResponseWriter)
+			status := strconv.Itoa(response.Status())
+			duration := float64(time.Since(start)) / float64(time.Second)
+			fmt.Println(duration)
+
+			p.counter.WithLabelValues(status, r.RequestURI, r.Method).Inc()
+			if p.latencies != nil {
+				p.latencies.WithLabelValues(status, r.RequestURI, r.Method).Observe(duration)
+			}
+		}(time.Now())
+		next(w, r)
+	}
+}

pkg/handler/eval.go

@@ -174,6 +174,7 @@ var logEvalResult = func(r *models.EvalResult, dataRecordsEnabled bool) {
 	}
 
 	logEvalResultToDatadog(r)
+	logEvalResultToPrometheus(r)
 
 	if !config.Config.RecorderEnabled || !dataRecordsEnabled {
 		return
@@ -199,6 +200,19 @@ var logEvalResultToDatadog = func(r *models.EvalResult) {
 	)
 }
 
+var logEvalResultToPrometheus = func(r *models.EvalResult) {
+	if config.Global.Prometheus.EvalCounter == nil {
+		return
+	}
+	config.Global.Prometheus.EvalCounter.WithLabelValues(
+		util.SafeStringWithDefault(r.EvalContext.EntityType, "null"),
+		util.SafeStringWithDefault(r.FlagID, "null"),
+		util.SafeStringWithDefault(r.VariantID, "null"),
+		util.SafeStringWithDefault(r.VariantKey, "null"),
+	).Inc()
+
+}
+
 var evalSegment = func(
 	flagID uint,
 	evalContext models.EvalContext,