From 58299997352ef4a8218462c33a283ce28a7596bf Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Mon, 27 Apr 2020 21:39:59 -0400 Subject: [PATCH 1/2] Add prometheus endpoint for metrics The existing metrics package used in dkron already supports prometheus, so this just adds in the necessary pieces to hook it up to the API. There is a config value to enable serving this endpoint to allow users the choice with a default of false to keep today's functionality. I know that it was discussed in #494 that the metrics system was likely going to be updated, but this seems like an easy step in the meantime. The documentation for metrics was updated to align with current configuration keys. --- dkron/agent.go | 4 ++++ dkron/api.go | 12 ++++++++++++ dkron/config.go | 4 ++++ dkron/metrics.go | 16 ++++++++++++++++ go.mod | 1 + go.sum | 24 ++++++++++-------------- website/content/cli/dkron_agent.md | 1 + website/content/usage/metrics.md | 19 +++++++++++++++---- 8 files changed, 63 insertions(+), 18 deletions(-) diff --git a/dkron/agent.go b/dkron/agent.go index 6cb4e81d8..dd0e62e88 100644 --- a/dkron/agent.go +++ b/dkron/agent.go @@ -23,6 +23,7 @@ import ( "github.com/hashicorp/raft" raftboltdb "github.com/hashicorp/raft-boltdb" "github.com/hashicorp/serf/serf" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "github.com/soheilhy/cmux" ) @@ -101,6 +102,8 @@ type Agent struct { peers map[string][]*ServerParts localPeers map[raft.ServerAddress]*ServerParts peerLock sync.RWMutex + + promReg *prometheus.Registry } // ProcessorFactory is a function type that creates a new instance @@ -122,6 +125,7 @@ func NewAgent(config *Config, options ...AgentOption) *Agent { agent := &Agent{ config: config, retryJoinCh: make(chan error), + promReg: prometheus.NewRegistry(), } for _, option := range options { diff --git a/dkron/api.go b/dkron/api.go index f13d9cd94..0b1e0aef6 100644 --- a/dkron/api.go +++ b/dkron/api.go @@ -8,6 +8,7 @@ import ( "github.com/gin-contrib/expvar" "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "github.com/tidwall/buntdb" status "google.golang.org/grpc/status" @@ -53,6 +54,12 @@ func (h *HTTPTransport) ServeHTTP() { go h.Engine.Run(h.agent.config.HTTPAddr) } +// metricsHandler returns a prometheus style http.Handler for scraping metrics using the agent's prometheus.Registry +func (h *HTTPTransport) metricsHandler() http.Handler { + handler := promhttp.HandlerFor(h.agent.promReg, promhttp.HandlerOpts{}) + return handler +} + // APIRoutes registers the api routes on the gin RouterGroup. func (h *HTTPTransport) APIRoutes(r *gin.RouterGroup, middleware ...gin.HandlerFunc) { r.GET("/debug/vars", expvar.Handler()) @@ -63,6 +70,11 @@ func (h *HTTPTransport) APIRoutes(r *gin.RouterGroup, middleware ...gin.HandlerF }) }) + if h.agent.config.EnablePrometheus { + // Prometheus metrics scrape endpoint + r.GET("/metrics", gin.WrapH(h.metricsHandler())) + } + r.GET("/v1", h.indexHandler) v1 := r.Group("/v1") v1.Use(middleware...) diff --git a/dkron/config.go b/dkron/config.go index ebe4fe794..bb42b18d1 100644 --- a/dkron/config.go +++ b/dkron/config.go @@ -153,6 +153,9 @@ type Config struct { // StatsdAddr is the statsd standard server to be used for sending metrics. StatsdAddr string `mapstructure:"statsd-addr"` + + // EnablePrometheus enables serving of prometheus metrics at /metrics + EnablePrometheus bool `mapstructure:"enable-prometheus"` } // DefaultBindPort is the default port that dkron will use for Serf communication @@ -231,6 +234,7 @@ func ConfigFlagSet() *flag.FlagSet { cmdFlags.String("dog-statsd-addr", "", "DataDog Agent address") cmdFlags.StringSlice("dog-statsd-tags", []string{}, "Datadog tags, specified as key:value") cmdFlags.String("statsd-addr", "", "Statsd address") + cmdFlags.Bool("enable-prometheus", false, "Enable serving prometheus metrics") return cmdFlags } diff --git a/dkron/metrics.go b/dkron/metrics.go index 807dbfe80..23e0a2fc2 100644 --- a/dkron/metrics.go +++ b/dkron/metrics.go @@ -6,6 +6,7 @@ import ( "github.com/armon/go-metrics" "github.com/armon/go-metrics/datadog" + "github.com/armon/go-metrics/prometheus" ) func initMetrics(a *Agent) error { @@ -15,6 +16,21 @@ func initMetrics(a *Agent) error { var fanout metrics.FanoutSink + // Configure the prometheus sink + if a.config.EnablePrometheus { + promSink, err := prometheus.NewPrometheusSink() + if err != nil { + return err + } + // Use a custom prometheus registry instead of prometheus.DefaultRegistry to avoid the default go metrics because they are already configured + err = a.promReg.Register(promSink) + if err != nil { + return err + } + + fanout = append(fanout, promSink) + } + // Configure the statsd sink if a.config.StatsdAddr != "" { sink, err := metrics.NewStatsdSink(a.config.StatsdAddr) diff --git a/go.mod b/go.mod index d9ad68b8c..0f1ce31d2 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 github.com/mattn/go-shellwords v1.0.10 github.com/mitchellh/go-testing-interface v1.0.0 // indirect + github.com/prometheus/client_golang v1.4.0 github.com/robfig/cron/v3 v3.0.1 github.com/ryanuber/columnize v2.1.0+incompatible github.com/sirupsen/logrus v1.5.0 diff --git a/go.sum b/go.sum index 2a9942d1d..5ca87f59a 100644 --- a/go.sum +++ b/go.sum @@ -31,12 +31,15 @@ github.com/aws/aws-sdk-go v1.29.22 h1:3WmsCj3C30l6/4f50mPkDZoTPWSvaRCjcVJOWdCJoI github.com/aws/aws-sdk-go v1.29.22/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= @@ -90,6 +93,7 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= @@ -117,8 +121,6 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -158,8 +160,6 @@ github.com/hashicorp/go-discover v0.0.0-20200108194735-7698de1390a1 h1:ir6YzYlYV github.com/hashicorp/go-discover v0.0.0-20200108194735-7698de1390a1/go.mod h1:txEDH3XlSg7/Ot5HIV2caLldDa/P0zoy3lr9ijypNlU= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v0.12.1 h1:99niEVkDqsEv3/jINwoOUgGE9L41LHXM4k3jTkV+DdA= -github.com/hashicorp/go-hclog v0.12.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.12.2 h1:F1fdYblUEsxKiailtkhCCG2g4bipEgaHiDc8vffNpD4= github.com/hashicorp/go-hclog v0.12.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -170,10 +170,6 @@ github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9 github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-plugin v1.1.0 h1:HGUJ/UDgeYfIBYT1ciftFUMSzbrheczUObdtuIA63vU= -github.com/hashicorp/go-plugin v1.1.0/go.mod h1:popwcT5Xlw1syWU1GfAOQYhYt1xEf6hTOdJZbQNQcnQ= -github.com/hashicorp/go-plugin v1.2.0 h1:CUfYokW0EJNDcGecVrHZK//Cp1GFlHwoqtcUIEiU6BY= -github.com/hashicorp/go-plugin v1.2.0/go.mod h1:F9eH4LrE/ZsRdbwhfjs9k9HoDUwAHnYtXdgmf1AVNs0= github.com/hashicorp/go-plugin v1.2.2 h1:mgDpq0PkoK5gck2w4ivaMpWRHv/matdOR4xmeScmf/w= github.com/hashicorp/go-plugin v1.2.2/go.mod h1:F9eH4LrE/ZsRdbwhfjs9k9HoDUwAHnYtXdgmf1AVNs0= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= @@ -217,6 +213,7 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da h1:FjHUJJ7oBW4G/9j1KzlHaXL09LyMVM9rupS39lncbXk= github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= +github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -276,6 +273,7 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= @@ -325,20 +323,24 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0 h1:YVIb/fVcOTMSqtqZWSKnHpSLBxu8DKgxq8z6RuBZwqI= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03 h1:Wdi9nwnhFNAlseAOekn6B5G/+GMtks9UKbvRU/CMM/o= @@ -376,8 +378,6 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.6 h1:breEStsVwemnKh2/s6gMvSdMEkwW0sK8vGStnlVBMCs= -github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v0.0.7 h1:FfTH+vuMXOas8jmfb5/M7dzEYx7LpcLb7a0LPe34uOU= github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= @@ -387,8 +387,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= -github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs= github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -526,8 +524,6 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k= google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/website/content/cli/dkron_agent.md b/website/content/cli/dkron_agent.md index 430f9671d..4280b5c89 100644 --- a/website/content/cli/dkron_agent.md +++ b/website/content/cli/dkron_agent.md @@ -28,6 +28,7 @@ dkron agent [flags] --datacenter string Specifies the data center of the local agent. All members of a datacenter should share a local LAN connection. (default "dc1") --dog-statsd-addr string DataDog Agent address --dog-statsd-tags strings Datadog tags, specified as key:value + --enable-prometheus Enable serving prometheus metrics at /metrics --encrypt string Key for encrypting network traffic. Must be a base64-encoded 16-byte key -h, --help help for agent --http-addr string Address to bind the UI web server to. Only used when server. The value supports go-sockaddr/template format. (default ":8080") diff --git a/website/content/usage/metrics.md b/website/content/usage/metrics.md index 8a1f566ec..296cf24cf 100644 --- a/website/content/usage/metrics.md +++ b/website/content/usage/metrics.md @@ -2,14 +2,26 @@ title: Metrics --- -Dkron has the ability to send metrics to Statsd for dashboards and historical reporting. It sends job processing metrics and golang, serf metrics too. +Dkron has the ability to send metrics to Statsd for dashboards and historical reporting or provide prometheus format metrics via the api. It sends job processing metrics, golang, and serf metrics. ## Configuration -Add this in your yaml config file +### Statsd + +Add this in your yaml config file to enable statsd metrics. ```yaml -dog_statsd_addr: "localhost:8125" +statsd-addr: "localhost:8125" +# Or for datadog statsd +dog-statsd-addr: "localhost:8125" +``` + +### Prometheus + +Add this to your yaml config file to enable serving prometheus metrics at the endpoint `/metrics` + +```yaml +enable-prometheus: true ``` ## Metrics @@ -48,4 +60,3 @@ dog_statsd_addr: "localhost:8125" - dkron.serf.queue.Event - dkron.serf.queue.Intent - dkron.serf.queue.Query - From b6b04d190abfb29441497461ac51ca07d560b257 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Thu, 30 Apr 2020 10:31:35 -0400 Subject: [PATCH 2/2] Move to prometheus.DefaultRegistry --- dkron/agent.go | 4 ---- dkron/api.go | 8 +------- dkron/metrics.go | 5 ----- 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/dkron/agent.go b/dkron/agent.go index dd0e62e88..6cb4e81d8 100644 --- a/dkron/agent.go +++ b/dkron/agent.go @@ -23,7 +23,6 @@ import ( "github.com/hashicorp/raft" raftboltdb "github.com/hashicorp/raft-boltdb" "github.com/hashicorp/serf/serf" - "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "github.com/soheilhy/cmux" ) @@ -102,8 +101,6 @@ type Agent struct { peers map[string][]*ServerParts localPeers map[raft.ServerAddress]*ServerParts peerLock sync.RWMutex - - promReg *prometheus.Registry } // ProcessorFactory is a function type that creates a new instance @@ -125,7 +122,6 @@ func NewAgent(config *Config, options ...AgentOption) *Agent { agent := &Agent{ config: config, retryJoinCh: make(chan error), - promReg: prometheus.NewRegistry(), } for _, option := range options { diff --git a/dkron/api.go b/dkron/api.go index 0b1e0aef6..4da6f4985 100644 --- a/dkron/api.go +++ b/dkron/api.go @@ -54,12 +54,6 @@ func (h *HTTPTransport) ServeHTTP() { go h.Engine.Run(h.agent.config.HTTPAddr) } -// metricsHandler returns a prometheus style http.Handler for scraping metrics using the agent's prometheus.Registry -func (h *HTTPTransport) metricsHandler() http.Handler { - handler := promhttp.HandlerFor(h.agent.promReg, promhttp.HandlerOpts{}) - return handler -} - // APIRoutes registers the api routes on the gin RouterGroup. func (h *HTTPTransport) APIRoutes(r *gin.RouterGroup, middleware ...gin.HandlerFunc) { r.GET("/debug/vars", expvar.Handler()) @@ -72,7 +66,7 @@ func (h *HTTPTransport) APIRoutes(r *gin.RouterGroup, middleware ...gin.HandlerF if h.agent.config.EnablePrometheus { // Prometheus metrics scrape endpoint - r.GET("/metrics", gin.WrapH(h.metricsHandler())) + r.GET("/metrics", gin.WrapH(promhttp.Handler())) } r.GET("/v1", h.indexHandler) diff --git a/dkron/metrics.go b/dkron/metrics.go index 23e0a2fc2..4cfa1884e 100644 --- a/dkron/metrics.go +++ b/dkron/metrics.go @@ -22,11 +22,6 @@ func initMetrics(a *Agent) error { if err != nil { return err } - // Use a custom prometheus registry instead of prometheus.DefaultRegistry to avoid the default go metrics because they are already configured - err = a.promReg.Register(promSink) - if err != nil { - return err - } fanout = append(fanout, promSink) }