diff --git a/go.mod b/go.mod index b9980f73a5..8338e439e0 100644 --- a/go.mod +++ b/go.mod @@ -134,6 +134,13 @@ require ( ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/m3db/prometheus_client_golang v0.8.1 // indirect + github.com/m3db/prometheus_client_model v0.1.0 // indirect + github.com/m3db/prometheus_common v0.1.0 // indirect + github.com/m3db/prometheus_procfs v0.8.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/onsi/ginkgo v1.14.0 // indirect cloud.google.com/go/compute v1.6.1 // indirect cloud.google.com/go/iam v0.3.0 // indirect github.com/onsi/ginkgo v1.14.0 // indirect diff --git a/go.sum b/go.sum index d1b12efb37..7b32c7ec6e 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,7 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= @@ -372,6 +373,15 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018 h1:MNApn+Z+fIT4NPZopPfCc1obT6aY3SVM6DOctz1A9ZU= github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= +github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= +github.com/m3db/prometheus_client_golang v0.8.1 h1:t7w/tcFws81JL1j5sqmpqcOyQOpH4RDOmIe3A3fdN3w= +github.com/m3db/prometheus_client_golang v0.8.1/go.mod h1:8R/f1xYhXWq59KD/mbRqoBulXejss7vYtYzWmruNUwI= +github.com/m3db/prometheus_client_model v0.1.0 h1:cg1+DiuyT6x8h9voibtarkH1KT6CmsewBSaBhe8wzLo= +github.com/m3db/prometheus_client_model v0.1.0/go.mod h1:Qfsxn+LypxzF+lNhak7cF7k0zxK7uB/ynGYoj80zcD4= +github.com/m3db/prometheus_common v0.1.0 h1:YJu6eCIV6MQlcwND24cRG/aRkZDX1jvYbsNNs1ZYr0w= +github.com/m3db/prometheus_common v0.1.0/go.mod h1:EBmDQaMAy4B8i+qsg1wMXAelLNVbp49i/JOeVszQ/rs= +github.com/m3db/prometheus_procfs v0.8.1 h1:LsxWzVELhDU9sLsZTaFLCeAwCn7bC7qecZcK4zobs/g= +github.com/m3db/prometheus_procfs v0.8.1/go.mod h1:N8lv8fLh3U3koZx1Bnisj60GYUMDpWb09x1R+dmMOJo= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -389,6 +399,7 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mcdafydd/go-azuredevops v0.12.1 h1:WxwLVyGuJ8oL7uWQp1/J6GefX1wMQQZUHWRGsrm+uE8= github.com/mcdafydd/go-azuredevops v0.12.1/go.mod h1:B4UDyn7WEj1/97f45j3VnzEfkWKe05+/dCcAPdOET4A= diff --git a/server/core/config/raw/metrics.go b/server/core/config/raw/metrics.go index 368c6a4dbe..0844e380ad 100644 --- a/server/core/config/raw/metrics.go +++ b/server/core/config/raw/metrics.go @@ -7,7 +7,16 @@ import ( ) type Metrics struct { - Statsd *Statsd `yaml:"statsd" json:"statsd"` + Statsd *Statsd `yaml:"statsd" json:"statsd"` + Prometheus *Prometheus `yaml:"prometheus" json:"prometheus"` +} + +type Prometheus struct { + Endpoint string `yaml:"endpoint" json:"endpoint"` +} + +func (p *Prometheus) Validate() error { + return validation.ValidateStruct(p, validation.Field(&p.Endpoint, validation.Required)) } type Statsd struct { @@ -24,9 +33,11 @@ func (s *Statsd) Validate() error { } func (m Metrics) Validate() error { - return validation.ValidateStruct(&m, - validation.Field(&m.Statsd), + res := validation.ValidateStruct(&m, + validation.Field(&m.Statsd, validation.NilOrNotEmpty), + validation.Field(&m.Prometheus, validation.NilOrNotEmpty), ) + return res } func (m Metrics) ToValid() valid.Metrics { @@ -39,6 +50,12 @@ func (m Metrics) ToValid() valid.Metrics { }, } } - + if m.Prometheus != nil { + return valid.Metrics{ + Prometheus: &valid.Prometheus{ + Endpoint: m.Prometheus.Endpoint, + }, + } + } return valid.Metrics{} } diff --git a/server/core/config/raw/metrics_test.go b/server/core/config/raw/metrics_test.go index 8bf5744edd..be9c60018c 100644 --- a/server/core/config/raw/metrics_test.go +++ b/server/core/config/raw/metrics_test.go @@ -16,6 +16,8 @@ func TestMetrics_Unmarshal(t *testing.T) { statsd: host: 127.0.0.1 port: 8125 +prometheus: + endpoint: /metrics ` var result raw.Metrics @@ -30,6 +32,9 @@ statsd: "statsd": { "host": "127.0.0.1", "port": "8125" + }, + "prometheus": { + "endpoint": "/metrics" } } ` @@ -47,7 +52,7 @@ func TestMetrics_Validate_Success(t *testing.T) { subject raw.Metrics }{ { - description: "success", + description: "success with stats config", subject: raw.Metrics{ Statsd: &raw.Statsd{ Host: "127.0.0.1", @@ -58,6 +63,26 @@ func TestMetrics_Validate_Success(t *testing.T) { { description: "missing stats", }, + { + description: "success with prometheus config", + subject: raw.Metrics{ + Prometheus: &raw.Prometheus{ + Endpoint: "/metrics", + }, + }, + }, + { + description: "success with both configs", + subject: raw.Metrics{ + Statsd: &raw.Statsd{ + Host: "127.0.0.1", + Port: "8125", + }, + Prometheus: &raw.Prometheus{ + Endpoint: "/metrics", + }, + }, + }, } for _, c := range cases { @@ -106,6 +131,14 @@ func TestMetrics_Validate_Error(t *testing.T) { }, }, }, + { + description: "invalid endpoint", + subject: raw.Metrics{ + Prometheus: &raw.Prometheus{ + Endpoint: "", + }, + }, + }, } for _, c := range cases { diff --git a/server/core/config/valid/global_cfg.go b/server/core/config/valid/global_cfg.go index fe17e4b620..c52cc75e6b 100644 --- a/server/core/config/valid/global_cfg.go +++ b/server/core/config/valid/global_cfg.go @@ -39,7 +39,8 @@ type GlobalCfg struct { } type Metrics struct { - Statsd *Statsd + Statsd *Statsd + Prometheus *Prometheus } type Statsd struct { @@ -47,6 +48,10 @@ type Statsd struct { Host string } +type Prometheus struct { + Endpoint string +} + // Repo is the final parsed version of server-side repo config. type Repo struct { // ID is the exact match id of this config. diff --git a/server/metrics/scope.go b/server/metrics/scope.go index 023ca64179..02595f8940 100644 --- a/server/metrics/scope.go +++ b/server/metrics/scope.go @@ -10,44 +10,55 @@ import ( "github.com/runatlantis/atlantis/server/core/config/valid" "github.com/runatlantis/atlantis/server/logging" "github.com/uber-go/tally" + tallyprom "github.com/uber-go/tally/prometheus" tallystatsd "github.com/uber-go/tally/statsd" ) func NewLoggingScope(logger logging.SimpleLogging, statsNamespace string) (tally.Scope, io.Closer, error) { - reporter, err := newReporter(valid.Metrics{}, logger) + scope, _, closer, err := NewScope(valid.Metrics{}, logger, statsNamespace) + return scope, closer, err +} + +func NewScope(cfg valid.Metrics, logger logging.SimpleLogging, statsNamespace string) (tally.Scope, tally.BaseStatsReporter, io.Closer, error) { + reporter, err := newReporter(cfg, logger) if err != nil { - return nil, nil, errors.Wrap(err, "initializing stats reporter") + return nil, nil, nil, errors.Wrap(err, "initializing stats reporter") } - scope, closer := tally.NewRootScope(tally.ScopeOptions{ - Prefix: statsNamespace, - Reporter: reporter, - }, time.Second) + scopeOpts := tally.ScopeOptions{ + Prefix: statsNamespace, + } + + if r, ok := reporter.(tally.StatsReporter); ok { + scopeOpts.Reporter = r + } else if r, ok := reporter.(tally.CachedStatsReporter); ok { + scopeOpts.CachedReporter = r + scopeOpts.Separator = tallyprom.DefaultSeparator + } - return scope, closer, nil + scope, closer := tally.NewRootScope(scopeOpts, time.Second) + return scope, reporter, closer, nil } -func NewScope(cfg valid.Metrics, logger logging.SimpleLogging, statsNamespace string) (tally.Scope, io.Closer, error) { - reporter, err := newReporter(cfg, logger) +func newReporter(cfg valid.Metrics, logger logging.SimpleLogging) (tally.BaseStatsReporter, error) { - if err != nil { - return nil, nil, errors.Wrap(err, "initializing stats reporter") + // return statsd metrics if configured + if cfg.Statsd != nil { + return newStatsReporter(cfg, logger) } - scope, closer := tally.NewRootScope(tally.ScopeOptions{ - Prefix: statsNamespace, - Reporter: reporter, - }, time.Second) + // return prometheus metrics if configured + if cfg.Prometheus != nil { + return tallyprom.NewReporter(tallyprom.Options{}), nil + } + + // return logging reporter and proceed + return newLoggingReporter(logger), nil - return scope, closer, nil } -func newReporter(cfg valid.Metrics, logger logging.SimpleLogging) (tally.StatsReporter, error) { - if cfg.Statsd == nil { - // return logging reporter and proceed - return newLoggingReporter(logger), nil - } +func newStatsReporter(cfg valid.Metrics, logger logging.SimpleLogging) (tally.StatsReporter, error) { statsdCfg := cfg.Statsd diff --git a/server/scheduled/runtime_stats.go b/server/scheduled/runtime_stats.go index 78836267e4..c75097d36b 100644 --- a/server/scheduled/runtime_stats.go +++ b/server/scheduled/runtime_stats.go @@ -46,38 +46,43 @@ type runtimeMetrics struct { func NewRuntimeStats(scope tally.Scope) *RuntimeStatCollector { runtimeScope := scope.SubScope("runtime") + cpuScope := runtimeScope.SubScope("cpu") + memoryScope := runtimeScope.SubScope("memory") + heapScope := memoryScope.SubScope("heap") + stackScope := memoryScope.SubScope("stack") + gcScope := memoryScope.SubScope("gc") runtimeMetrics := runtimeMetrics{ // cpu - cpuGoroutines: runtimeScope.Gauge("cpu.goroutines"), - cpuCgoCalls: runtimeScope.Gauge("cpu.cgo_calls"), + cpuGoroutines: cpuScope.Gauge("goroutines"), + cpuCgoCalls: cpuScope.Gauge("cgo_calls"), // memory - memoryAlloc: runtimeScope.Gauge("memory.alloc"), - memoryTotal: runtimeScope.Gauge("memory.total"), - memorySys: runtimeScope.Gauge("memory.sys"), - memoryLookups: runtimeScope.Gauge("memory.lookups"), - memoryMalloc: runtimeScope.Gauge("memory.malloc"), - memoryFrees: runtimeScope.Gauge("memory.frees"), + memoryAlloc: memoryScope.Gauge("alloc"), + memoryTotal: memoryScope.Gauge("total"), + memorySys: memoryScope.Gauge("sys"), + memoryLookups: memoryScope.Gauge("lookups"), + memoryMalloc: memoryScope.Gauge("malloc"), + memoryFrees: memoryScope.Gauge("frees"), // heap - memoryHeapAlloc: runtimeScope.Gauge("memory.heap.alloc"), - memoryHeapSys: runtimeScope.Gauge("memory.heap.sys"), - memoryHeapIdle: runtimeScope.Gauge("memory.heap.idle"), - memoryHeapInuse: runtimeScope.Gauge("memory.heap.inuse"), - memoryHeapReleased: runtimeScope.Gauge("memory.heap.released"), - memoryHeapObjects: runtimeScope.Gauge("memory.heap.objects"), + memoryHeapAlloc: heapScope.Gauge("alloc"), + memoryHeapSys: heapScope.Gauge("sys"), + memoryHeapIdle: heapScope.Gauge("idle"), + memoryHeapInuse: heapScope.Gauge("inuse"), + memoryHeapReleased: heapScope.Gauge("released"), + memoryHeapObjects: heapScope.Gauge("objects"), // stack - memoryStackInuse: runtimeScope.Gauge("memory.stack.inuse"), - memoryStackSys: runtimeScope.Gauge("memory.stack.sys"), - memoryStackMSpanInuse: runtimeScope.Gauge("memory.stack.mspan_inuse"), - memoryStackMSpanSys: runtimeScope.Gauge("memory.stack.sys"), - memoryStackMCacheInuse: runtimeScope.Gauge("memory.stack.mcache_inuse"), - memoryStackMCacheSys: runtimeScope.Gauge("memory.stack.mcache_sys"), - memoryOtherSys: runtimeScope.Gauge("memory.othersys"), + memoryStackInuse: stackScope.Gauge("inuse"), + memoryStackSys: stackScope.Gauge("sys"), + memoryStackMSpanInuse: stackScope.Gauge("mspan_inuse"), + memoryStackMSpanSys: stackScope.Gauge("sys"), + memoryStackMCacheInuse: stackScope.Gauge("mcache_inuse"), + memoryStackMCacheSys: stackScope.Gauge("mcache_sys"), + memoryOtherSys: memoryScope.Gauge("othersys"), // GC - memoryGCSys: runtimeScope.Gauge("memory.gc.sys"), - memoryGCNext: runtimeScope.Gauge("memory.gc.next"), - memoryGCLast: runtimeScope.Gauge("memory.gc.last"), - memoryGCPauseTotal: runtimeScope.Gauge("memory.gc.pause_total"), - memoryGCCount: runtimeScope.Gauge("memory.gc.count"), + memoryGCSys: gcScope.Gauge("sys"), + memoryGCNext: gcScope.Gauge("next"), + memoryGCLast: gcScope.Gauge("last"), + memoryGCPauseTotal: gcScope.Gauge("pause_total"), + memoryGCCount: gcScope.Gauge("count"), } return &RuntimeStatCollector{ diff --git a/server/server.go b/server/server.go index f9f6f2b951..9f3979859d 100644 --- a/server/server.go +++ b/server/server.go @@ -39,6 +39,7 @@ import ( "github.com/runatlantis/atlantis/server/metrics" "github.com/runatlantis/atlantis/server/scheduled" "github.com/uber-go/tally" + "github.com/uber-go/tally/prometheus" assetfs "github.com/elazarl/go-bindata-assetfs" "github.com/gorilla/mux" @@ -94,6 +95,7 @@ type Server struct { CommandRunner *events.DefaultCommandRunner Logger logging.SimpleLogging StatsScope tally.Scope + StatsReporter tally.BaseStatsReporter StatsCloser io.Closer Locker locking.Locker ApplyLocker locking.ApplyLocker @@ -188,7 +190,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { } } - statsScope, closer, err := metrics.NewScope(globalCfg.Metrics, logger, userConfig.StatsNamespace) + statsScope, statsReporter, closer, err := metrics.NewScope(globalCfg.Metrics, logger, userConfig.StatsNamespace) if err != nil { return nil, errors.Wrapf(err, "instantiating metrics scope") @@ -773,6 +775,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { CommandRunner: commandRunner, Logger: logger, StatsScope: statsScope, + StatsReporter: statsReporter, StatsCloser: closer, Locker: lockingClient, ApplyLocker: applyLockingClient, @@ -815,6 +818,11 @@ func (s *Server) Start() error { s.Router.HandleFunc("/jobs/{job-id}", s.JobsController.GetProjectJobs).Methods("GET").Name(ProjectJobsViewRouteName) s.Router.HandleFunc("/jobs/{job-id}/ws", s.JobsController.GetProjectJobsWS).Methods("GET") + r, ok := s.StatsReporter.(prometheus.Reporter) + if ok { + s.Router.Handle(s.CommandRunner.GlobalCfg.Metrics.Prometheus.Endpoint, r.HTTPHandler()) + } + n := negroni.New(&negroni.Recovery{ Logger: log.New(os.Stdout, "", log.LstdFlags), PrintStack: false,