feat: Prometheus metrics support (#2204)

* support prometheus metrics

* remove redundant error checking

* load prometheus metrics config

* dynamic separator for subscopes

* removing unnecessary config file

* adding a few more metrics test cases

* remove testing handler

* fix upstream merge error

go.mod

@@ -134,6 +134,13 @@ require (
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/m3db/prometheus_client_golang v0.8.1 // indirect
+	github.com/m3db/prometheus_client_model v0.1.0 // indirect
+	github.com/m3db/prometheus_common v0.1.0 // indirect
+	github.com/m3db/prometheus_procfs v0.8.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/onsi/ginkgo v1.14.0 // indirect
 	cloud.google.com/go/compute v1.6.1 // indirect
 	cloud.google.com/go/iam v0.3.0 // indirect
 	github.com/onsi/ginkgo v1.14.0 // indirect

go.sum

@@ -100,6 +100,7 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
@@ -372,6 +373,15 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018 h1:MNApn+Z+fIT4NPZopPfCc1obT6aY3SVM6DOctz1A9ZU=
 github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
+github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
+github.com/m3db/prometheus_client_golang v0.8.1 h1:t7w/tcFws81JL1j5sqmpqcOyQOpH4RDOmIe3A3fdN3w=
+github.com/m3db/prometheus_client_golang v0.8.1/go.mod h1:8R/f1xYhXWq59KD/mbRqoBulXejss7vYtYzWmruNUwI=
+github.com/m3db/prometheus_client_model v0.1.0 h1:cg1+DiuyT6x8h9voibtarkH1KT6CmsewBSaBhe8wzLo=
+github.com/m3db/prometheus_client_model v0.1.0/go.mod h1:Qfsxn+LypxzF+lNhak7cF7k0zxK7uB/ynGYoj80zcD4=
+github.com/m3db/prometheus_common v0.1.0 h1:YJu6eCIV6MQlcwND24cRG/aRkZDX1jvYbsNNs1ZYr0w=
+github.com/m3db/prometheus_common v0.1.0/go.mod h1:EBmDQaMAy4B8i+qsg1wMXAelLNVbp49i/JOeVszQ/rs=
+github.com/m3db/prometheus_procfs v0.8.1 h1:LsxWzVELhDU9sLsZTaFLCeAwCn7bC7qecZcK4zobs/g=
+github.com/m3db/prometheus_procfs v0.8.1/go.mod h1:N8lv8fLh3U3koZx1Bnisj60GYUMDpWb09x1R+dmMOJo=
 github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
 github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@@ -389,6 +399,7 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mcdafydd/go-azuredevops v0.12.1 h1:WxwLVyGuJ8oL7uWQp1/J6GefX1wMQQZUHWRGsrm+uE8=
 github.com/mcdafydd/go-azuredevops v0.12.1/go.mod h1:B4UDyn7WEj1/97f45j3VnzEfkWKe05+/dCcAPdOET4A=

server/core/config/raw/metrics.go

@@ -7,7 +7,16 @@ import (
 )
 
 type Metrics struct {
-	Statsd *Statsd `yaml:"statsd" json:"statsd"`
+	Statsd     *Statsd     `yaml:"statsd" json:"statsd"`
+	Prometheus *Prometheus `yaml:"prometheus" json:"prometheus"`
+}
+
+type Prometheus struct {
+	Endpoint string `yaml:"endpoint" json:"endpoint"`
+}
+
+func (p *Prometheus) Validate() error {
+	return validation.ValidateStruct(p, validation.Field(&p.Endpoint, validation.Required))
 }
 
 type Statsd struct {
@@ -24,9 +33,11 @@ func (s *Statsd) Validate() error {
 }
 
 func (m Metrics) Validate() error {
-	return validation.ValidateStruct(&m,
-		validation.Field(&m.Statsd),
+	res := validation.ValidateStruct(&m,
+		validation.Field(&m.Statsd, validation.NilOrNotEmpty),
+		validation.Field(&m.Prometheus, validation.NilOrNotEmpty),
 	)
+	return res
 }
 
 func (m Metrics) ToValid() valid.Metrics {
@@ -39,6 +50,12 @@ func (m Metrics) ToValid() valid.Metrics {
 			},
 		}
 	}
-
+	if m.Prometheus != nil {
+		return valid.Metrics{
+			Prometheus: &valid.Prometheus{
+				Endpoint: m.Prometheus.Endpoint,
+			},
+		}
+	}
 	return valid.Metrics{}
 }

server/core/config/raw/metrics_test.go

@@ -16,6 +16,8 @@ func TestMetrics_Unmarshal(t *testing.T) {
 statsd:
   host: 127.0.0.1
   port: 8125
+prometheus:
+  endpoint: /metrics
 `
 
 		var result raw.Metrics
@@ -30,6 +32,9 @@ statsd:
 	"statsd": {
 		"host": "127.0.0.1",
 		"port": "8125"
+	},
+	"prometheus": {
+		"endpoint": "/metrics"
 	}
 }		
 `
@@ -47,7 +52,7 @@ func TestMetrics_Validate_Success(t *testing.T) {
 		subject     raw.Metrics
 	}{
 		{
-			description: "success",
+			description: "success with stats config",
 			subject: raw.Metrics{
 				Statsd: &raw.Statsd{
 					Host: "127.0.0.1",
@@ -58,6 +63,26 @@ func TestMetrics_Validate_Success(t *testing.T) {
 		{
 			description: "missing stats",
 		},
+		{
+			description: "success with prometheus config",
+			subject: raw.Metrics{
+				Prometheus: &raw.Prometheus{
+					Endpoint: "/metrics",
+				},
+			},
+		},
+		{
+			description: "success with both configs",
+			subject: raw.Metrics{
+				Statsd: &raw.Statsd{
+					Host: "127.0.0.1",
+					Port: "8125",
+				},
+				Prometheus: &raw.Prometheus{
+					Endpoint: "/metrics",
+				},
+			},
+		},
 	}
 
 	for _, c := range cases {
@@ -106,6 +131,14 @@ func TestMetrics_Validate_Error(t *testing.T) {
 				},
 			},
 		},
+		{
+			description: "invalid endpoint",
+			subject: raw.Metrics{
+				Prometheus: &raw.Prometheus{
+					Endpoint: "",
+				},
+			},
+		},
 	}
 
 	for _, c := range cases {

server/core/config/valid/global_cfg.go

@@ -39,14 +39,19 @@ type GlobalCfg struct {
 }
 
 type Metrics struct {
-	Statsd *Statsd
+	Statsd     *Statsd
+	Prometheus *Prometheus
 }
 
 type Statsd struct {
 	Port string
 	Host string
 }
 
+type Prometheus struct {
+	Endpoint string
+}
+
 // Repo is the final parsed version of server-side repo config.
 type Repo struct {
 	// ID is the exact match id of this config.

server/metrics/scope.go

@@ -10,44 +10,55 @@ import (
 	"github.com/runatlantis/atlantis/server/core/config/valid"
 	"github.com/runatlantis/atlantis/server/logging"
 	"github.com/uber-go/tally"
+	tallyprom "github.com/uber-go/tally/prometheus"
 	tallystatsd "github.com/uber-go/tally/statsd"
 )
 
 func NewLoggingScope(logger logging.SimpleLogging, statsNamespace string) (tally.Scope, io.Closer, error) {
-	reporter, err := newReporter(valid.Metrics{}, logger)
+	scope, _, closer, err := NewScope(valid.Metrics{}, logger, statsNamespace)
+	return scope, closer, err
+}
+
+func NewScope(cfg valid.Metrics, logger logging.SimpleLogging, statsNamespace string) (tally.Scope, tally.BaseStatsReporter, io.Closer, error) {
+	reporter, err := newReporter(cfg, logger)
 
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "initializing stats reporter")
+		return nil, nil, nil, errors.Wrap(err, "initializing stats reporter")
 	}
 
-	scope, closer := tally.NewRootScope(tally.ScopeOptions{
-		Prefix:   statsNamespace,
-		Reporter: reporter,
-	}, time.Second)
+	scopeOpts := tally.ScopeOptions{
+		Prefix: statsNamespace,
+	}
+
+	if r, ok := reporter.(tally.StatsReporter); ok {
+		scopeOpts.Reporter = r
+	} else if r, ok := reporter.(tally.CachedStatsReporter); ok {
+		scopeOpts.CachedReporter = r
+		scopeOpts.Separator = tallyprom.DefaultSeparator
+	}
 
-	return scope, closer, nil
+	scope, closer := tally.NewRootScope(scopeOpts, time.Second)
+	return scope, reporter, closer, nil
 }
 
-func NewScope(cfg valid.Metrics, logger logging.SimpleLogging, statsNamespace string) (tally.Scope, io.Closer, error) {
-	reporter, err := newReporter(cfg, logger)
+func newReporter(cfg valid.Metrics, logger logging.SimpleLogging) (tally.BaseStatsReporter, error) {
 
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "initializing stats reporter")
+	// return statsd metrics if configured
+	if cfg.Statsd != nil {
+		return newStatsReporter(cfg, logger)
 	}
 
-	scope, closer := tally.NewRootScope(tally.ScopeOptions{
-		Prefix:   statsNamespace,
-		Reporter: reporter,
-	}, time.Second)
+	// return prometheus metrics if configured
+	if cfg.Prometheus != nil {
+		return tallyprom.NewReporter(tallyprom.Options{}), nil
+	}
+
+	// return logging reporter and proceed
+	return newLoggingReporter(logger), nil
 
-	return scope, closer, nil
 }
 
-func newReporter(cfg valid.Metrics, logger logging.SimpleLogging) (tally.StatsReporter, error) {
-	if cfg.Statsd == nil {
-		// return logging reporter and proceed
-		return newLoggingReporter(logger), nil
-	}
+func newStatsReporter(cfg valid.Metrics, logger logging.SimpleLogging) (tally.StatsReporter, error) {
 
 	statsdCfg := cfg.Statsd
 

server/scheduled/runtime_stats.go

@@ -46,38 +46,43 @@ type runtimeMetrics struct {
 
 func NewRuntimeStats(scope tally.Scope) *RuntimeStatCollector {
 	runtimeScope := scope.SubScope("runtime")
+	cpuScope := runtimeScope.SubScope("cpu")
+	memoryScope := runtimeScope.SubScope("memory")
+	heapScope := memoryScope.SubScope("heap")
+	stackScope := memoryScope.SubScope("stack")
+	gcScope := memoryScope.SubScope("gc")
 	runtimeMetrics := runtimeMetrics{
 		// cpu
-		cpuGoroutines: runtimeScope.Gauge("cpu.goroutines"),
-		cpuCgoCalls:   runtimeScope.Gauge("cpu.cgo_calls"),
+		cpuGoroutines: cpuScope.Gauge("goroutines"),
+		cpuCgoCalls:   cpuScope.Gauge("cgo_calls"),
 		// memory
-		memoryAlloc:   runtimeScope.Gauge("memory.alloc"),
-		memoryTotal:   runtimeScope.Gauge("memory.total"),
-		memorySys:     runtimeScope.Gauge("memory.sys"),
-		memoryLookups: runtimeScope.Gauge("memory.lookups"),
-		memoryMalloc:  runtimeScope.Gauge("memory.malloc"),
-		memoryFrees:   runtimeScope.Gauge("memory.frees"),
+		memoryAlloc:   memoryScope.Gauge("alloc"),
+		memoryTotal:   memoryScope.Gauge("total"),
+		memorySys:     memoryScope.Gauge("sys"),
+		memoryLookups: memoryScope.Gauge("lookups"),
+		memoryMalloc:  memoryScope.Gauge("malloc"),
+		memoryFrees:   memoryScope.Gauge("frees"),
 		// heap
-		memoryHeapAlloc:    runtimeScope.Gauge("memory.heap.alloc"),
-		memoryHeapSys:      runtimeScope.Gauge("memory.heap.sys"),
-		memoryHeapIdle:     runtimeScope.Gauge("memory.heap.idle"),
-		memoryHeapInuse:    runtimeScope.Gauge("memory.heap.inuse"),
-		memoryHeapReleased: runtimeScope.Gauge("memory.heap.released"),
-		memoryHeapObjects:  runtimeScope.Gauge("memory.heap.objects"),
+		memoryHeapAlloc:    heapScope.Gauge("alloc"),
+		memoryHeapSys:      heapScope.Gauge("sys"),
+		memoryHeapIdle:     heapScope.Gauge("idle"),
+		memoryHeapInuse:    heapScope.Gauge("inuse"),
+		memoryHeapReleased: heapScope.Gauge("released"),
+		memoryHeapObjects:  heapScope.Gauge("objects"),
 		// stack
-		memoryStackInuse:       runtimeScope.Gauge("memory.stack.inuse"),
-		memoryStackSys:         runtimeScope.Gauge("memory.stack.sys"),
-		memoryStackMSpanInuse:  runtimeScope.Gauge("memory.stack.mspan_inuse"),
-		memoryStackMSpanSys:    runtimeScope.Gauge("memory.stack.sys"),
-		memoryStackMCacheInuse: runtimeScope.Gauge("memory.stack.mcache_inuse"),
-		memoryStackMCacheSys:   runtimeScope.Gauge("memory.stack.mcache_sys"),
-		memoryOtherSys:         runtimeScope.Gauge("memory.othersys"),
+		memoryStackInuse:       stackScope.Gauge("inuse"),
+		memoryStackSys:         stackScope.Gauge("sys"),
+		memoryStackMSpanInuse:  stackScope.Gauge("mspan_inuse"),
+		memoryStackMSpanSys:    stackScope.Gauge("sys"),
+		memoryStackMCacheInuse: stackScope.Gauge("mcache_inuse"),
+		memoryStackMCacheSys:   stackScope.Gauge("mcache_sys"),
+		memoryOtherSys:         memoryScope.Gauge("othersys"),
 		// GC
-		memoryGCSys:        runtimeScope.Gauge("memory.gc.sys"),
-		memoryGCNext:       runtimeScope.Gauge("memory.gc.next"),
-		memoryGCLast:       runtimeScope.Gauge("memory.gc.last"),
-		memoryGCPauseTotal: runtimeScope.Gauge("memory.gc.pause_total"),
-		memoryGCCount:      runtimeScope.Gauge("memory.gc.count"),
+		memoryGCSys:        gcScope.Gauge("sys"),
+		memoryGCNext:       gcScope.Gauge("next"),
+		memoryGCLast:       gcScope.Gauge("last"),
+		memoryGCPauseTotal: gcScope.Gauge("pause_total"),
+		memoryGCCount:      gcScope.Gauge("count"),
 	}
 
 	return &RuntimeStatCollector{

server/server.go

@@ -39,6 +39,7 @@ import (
 	"github.com/runatlantis/atlantis/server/metrics"
 	"github.com/runatlantis/atlantis/server/scheduled"
 	"github.com/uber-go/tally"
+	"github.com/uber-go/tally/prometheus"
 
 	assetfs "github.com/elazarl/go-bindata-assetfs"
 	"github.com/gorilla/mux"
@@ -94,6 +95,7 @@ type Server struct {
 	CommandRunner                  *events.DefaultCommandRunner
 	Logger                         logging.SimpleLogging
 	StatsScope                     tally.Scope
+	StatsReporter                  tally.BaseStatsReporter
 	StatsCloser                    io.Closer
 	Locker                         locking.Locker
 	ApplyLocker                    locking.ApplyLocker
@@ -188,7 +190,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) {
 		}
 	}
 
-	statsScope, closer, err := metrics.NewScope(globalCfg.Metrics, logger, userConfig.StatsNamespace)
+	statsScope, statsReporter, closer, err := metrics.NewScope(globalCfg.Metrics, logger, userConfig.StatsNamespace)
 
 	if err != nil {
 		return nil, errors.Wrapf(err, "instantiating metrics scope")
@@ -773,6 +775,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) {
 		CommandRunner:                  commandRunner,
 		Logger:                         logger,
 		StatsScope:                     statsScope,
+		StatsReporter:                  statsReporter,
 		StatsCloser:                    closer,
 		Locker:                         lockingClient,
 		ApplyLocker:                    applyLockingClient,
@@ -815,6 +818,11 @@ func (s *Server) Start() error {
 	s.Router.HandleFunc("/jobs/{job-id}", s.JobsController.GetProjectJobs).Methods("GET").Name(ProjectJobsViewRouteName)
 	s.Router.HandleFunc("/jobs/{job-id}/ws", s.JobsController.GetProjectJobsWS).Methods("GET")
 
+	r, ok := s.StatsReporter.(prometheus.Reporter)
+	if ok {
+		s.Router.Handle(s.CommandRunner.GlobalCfg.Metrics.Prometheus.Endpoint, r.HTTPHandler())
+	}
+
 	n := negroni.New(&negroni.Recovery{
 		Logger:     log.New(os.Stdout, "", log.LstdFlags),
 		PrintStack: false,