From 82e213d7e0cb318caf68ca38e97b2176471bc1fa Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Wed, 19 Feb 2020 16:17:57 -0800 Subject: [PATCH 1/5] [coordinator] Remove missing coordinator downsampling metrics --- src/query/server/query.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/query/server/query.go b/src/query/server/query.go index 08215dfb2c..8249f6c1d3 100644 --- a/src/query/server/query.go +++ b/src/query/server/query.go @@ -70,7 +70,6 @@ import ( opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" - "github.com/uber-go/tally" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/reflection" @@ -540,13 +539,12 @@ func newDownsampler( SubScope("tag-decoder-pool"))) downsampler, err := cfg.NewDownsampler(downsample.DownsamplerOptions{ - Storage: storage, - ClusterClient: clusterManagementClient, - RulesKVStore: kvStore, - AutoMappingRules: autoMappingRules, - ClockOptions: clock.NewOptions(), - // TODO: remove after https://github.com/m3db/m3/issues/992 is fixed - InstrumentOptions: instrumentOpts.SetMetricsScope(tally.NoopScope), + Storage: storage, + ClusterClient: clusterManagementClient, + RulesKVStore: kvStore, + AutoMappingRules: autoMappingRules, + ClockOptions: clock.NewOptions(), + InstrumentOptions: instrumentOpts, TagEncoderOptions: tagEncoderOptions, TagDecoderOptions: tagDecoderOptions, TagEncoderPoolOptions: tagEncoderPoolOptions, From 984006be4f4058e5711fe5acc1aa4fb473bcdd90 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Wed, 19 Feb 2020 16:57:07 -0800 Subject: [PATCH 2/5] Fix register error --- src/query/server/query.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/query/server/query.go b/src/query/server/query.go index 8249f6c1d3..90b812a726 100644 --- a/src/query/server/query.go +++ b/src/query/server/query.go @@ -146,7 +146,14 @@ func Run(runOpts RunOptions) { xconfig.WarnOnDeprecation(cfg, logger) - scope, closer, err := cfg.Metrics.NewRootScope() + scope, closer, _, err := cfg.Metrics.NewRootScopeAndReporters( + instrument.NewRootScopeAndReportersOptions{ + OnError: func(e error) { + // NB(r): Required otherwise collisions when registering metrics will + // cause a panic. + logger.Error("register metric error", zap.Error(err)) + }, + }) if err != nil { logger.Fatal("could not connect to metrics", zap.Error(err)) } From b7cf15c837a7be7400624ac0e72c330ce03f2f67 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Mon, 24 Feb 2020 01:17:00 -0500 Subject: [PATCH 3/5] Add end to end aggregator dashboard --- .../m3aggregator_end_to_end_details.json | 2011 +++++++++++++++++ 1 file changed, 2011 insertions(+) create mode 100644 integrations/grafana/m3aggregator_end_to_end_details.json diff --git a/integrations/grafana/m3aggregator_end_to_end_details.json b/integrations/grafana/m3aggregator_end_to_end_details.json new file mode 100644 index 0000000000..0cd9b59797 --- /dev/null +++ b/integrations/grafana/m3aggregator_end_to_end_details.json @@ -0,0 +1,2011 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "id": 32, + "iteration": 1582150012033, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 196, + "panels": [], + "title": "Send to Aggregator", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 198, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(m3aggregator_aggregator_client_writeForwarded_success[$step]))", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Write Timed/Forwarded Success", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 199, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(m3aggregator_aggregator_client_writeForwarded_errors[$step]))", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Write Timed/Forwarded Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 200, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(m3aggregator_aggregator_client_writer_manager_writer_queue_successes[$step]))", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Write Queue Success", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 201, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(m3aggregator_aggregator_client_writer_manager_writer_queue_dropped[$step]))", + "refId": "A" + }, + { + "expr": "sum(rate(m3aggregator_aggregator_client_writer_manager_writer_queue_errors[$step]))", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Write Queue Dropped/Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 194, + "panels": [], + "title": "Arriving at Aggregator", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(m3aggregator_aggregator_addUntimed_success{m3_cluster=~\"$cluster\"}[$step]))", + "key": 0.32458788643639513, + "legendFormat": "add untimed success", + "refId": "A" + }, + { + "expr": "sum(rate(m3aggregator_aggregator_addForwarded_success{m3_cluster=~\"$cluster\"}[$step]))", + "hide": false, + "key": 0.32458788643639513, + "legendFormat": "add forwarded success", + "refId": "C" + }, + { + "expr": "sum(rate(m3aggregator_aggregator_addTimed_success{m3_cluster=~\"$cluster\"}[$step]))", + "hide": false, + "key": 0.32458788643639513, + "legendFormat": "add timed success", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Writes Success / s", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(m3aggregator_aggregator_addUntimed_errors{m3_cluster=~\"$cluster\"}[$step])) by (reason)", + "key": 0.015375561225046352, + "legendFormat": "{{reason}}", + "refId": "D" + }, + { + "expr": "sum(rate(m3aggregator_aggregator_addForwarded_errors{m3_cluster=~\"$cluster\"}[$step])) by (reason)", + "hide": false, + "key": 0.015375561225046352, + "legendFormat": "{{reason}}", + "refId": "A" + }, + { + "expr": "sum(rate(m3aggregator_rawtcp_server_unknown_message_type_errors{m3_cluster=~\"$cluster\"}[$step]))", + "hide": false, + "key": 0.9314139646772208, + "legendFormat": "unknown message type errors", + "refId": "B" + }, + { + "expr": "sum(rate(m3aggregator_rawtcp_server_unknown_error_type_errors{m3_cluster=~\"$cluster\"}[$step]))", + "legendFormat": "unknown error type errors", + "refId": "E" + }, + { + "expr": "sum(rate(m3aggregator_rawtcp_server_decode_errors{m3_cluster=~\"$cluster\"}[$step]))", + "legendFormat": "server decode errors", + "refId": "F" + }, + { + "expr": "sum(rate(m3aggregator_aggregator_addTimed_errors{m3_cluster=~\"$cluster\"}[$step])) by (reason)", + "hide": false, + "key": 0.015375561225046352, + "legendFormat": "add timed error {{reason}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Write Errors / s", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 171, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(m3aggregator_aggregator_flush_handler_drop_oldest_async{m3_cluster=~\"$cluster\"}[$step])", + "legendFormat": "dropped {{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Metrics Dropped (All Consumers)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 191, + "panels": [], + "repeat": null, + "title": "Leaving Aggregator", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 135, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(m3aggregator_aggregator_flush_handler_message_buffered{m3_cluster=~\"$cluster\",backend=\"m3msg\",component=\"producer\"}) by (instance)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Messages Buffered", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 126, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "m3aggregator_aggregator_flush_handler_buffer_message_dropped{m3_cluster=~\"$cluster\",backend=\"m3msg\", component=\"producer\"}", + "legendFormat": "{{instance}} {{__name__}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Message Dropped", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 192, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "m3aggregator_aggregator_flush_handler_drop_oldest_async{m3_cluster=~\"$cluster\",backend=\"m3msg\",component=\"producer\"}", + "legendFormat": "{{instance}}", + "refId": "A" + }, + { + "expr": "m3aggregator_aggregator_flush_handler_drop_oldest_sync{m3_cluster=~\"$cluster\",backend=\"m3msg\",component=\"producer\"}", + "legendFormat": "{{instance}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Drop types", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 25 + }, + "id": 124, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(m3aggregator_aggregator_flush_handler_byte_buffered{m3_cluster=~\"$cluster\",backend=\"m3msg\", component=\"producer\"}) by (instance)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Bytes buffered", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 25 + }, + "id": 127, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "m3aggregator_aggregator_flush_handler_buffer_byte_dropped{m3_cluster=~\"$cluster\",backend=\"m3msg\",component=\"producer\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bytes Dropped", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 203, + "panels": [], + "title": "Arriving at Coordinator", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 33 + }, + "id": 205, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coordinator_ingest_m3msg_ack_sent{m3_cluster=~\"$cluster\"}[$step]))", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Coordinator M3Msg Ingest Ack", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 33 + }, + "id": 206, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coordinator_ingest_m3msg_errors{m3_cluster=~\"$cluster\"}[$step]))", + "refId": "A" + }, + { + "expr": "sum(rate(coordinator_ingest_m3msg_ack_encode_error{m3_cluster=~\"$cluster\"}[$step]))", + "refId": "B" + }, + { + "expr": "sum(rate(coordinator_ingest_m3msg_ack_write_error{m3_cluster=~\"$cluster\"}[$step]))", + "refId": "C" + }, + { + "expr": "sum(rate(coordinator_ingest_m3msg_errors_final{m3_cluster=~\"$cluster\"}[$step]))", + "refId": "D" + }, + { + "expr": "sum(rate(coordinator_ingest_m3msg_message_decode_error{m3_cluster=~\"$cluster\"}[$step]))", + "refId": "E" + }, + { + "expr": "sum(rate(coordinator_ingest_m3msg_message_read_error{m3_cluster=~\"$cluster\"}[$step]))", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Coordinator M3Msg Ingest Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 33 + }, + "id": 207, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coordinator_ingest_m3msg_metric_accepted{m3_cluster=~\"$cluster\"}[$step]))", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Coordinator M3Msg Ingest Accepted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 33 + }, + "id": 208, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coordinator_ingest_m3msg_metric_dropped{m3_cluster=~\"$cluster\"}[$step]))", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Coordinator M3Msg Ingest Dropped", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 40 + }, + "id": 210, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coordinator_ingest_success{m3_cluster=~\"$cluster\"}[$step]))", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Coordinator M3Msg Ingest Success", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 40 + }, + "id": 211, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coordinator_ingest_error{m3_cluster=~\"$cluster\"}[$step]))", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Coordinator M3Msg Ingest Error", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 19, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "dev", + "value": "dev" + }, + "definition": "label_values(up,m3_cluster)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(up,m3_cluster)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "glob", + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "{__name__=~\"m3aggregator_aggregator_flush_handler.*\"}", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "backend", + "options": [], + "query": "{m3_cluster=~\"$cluster\",__name__=~\"m3aggregator_aggregator_flush_handler.*\"}", + "refresh": 1, + "regex": "/backend=\\\"([^\"]+)\\\".*/", + "skipUrlSync": false, + "sort": 0, + "tag": "name", + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "glob", + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "multiFormat": "glob", + "name": "poolType", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "entry_pool", + "value": "entry_pool" + }, + { + "selected": false, + "text": "counter_elem_pool", + "value": "counter_elem_pool" + }, + { + "selected": false, + "text": "timer_elem_pool", + "value": "timer_elem_pool" + }, + { + "selected": false, + "text": "gauge_elem_pool", + "value": "gauge_elem_pool" + }, + { + "selected": false, + "text": "buffered_encoder_pool", + "value": "buffered_encoder_pool" + }, + { + "selected": false, + "text": "sample_pool", + "value": "sample_pool" + }, + { + "selected": false, + "text": "floats_pool", + "value": "floats_pool" + }, + { + "selected": false, + "text": "stream_pool", + "value": "stream_pool" + }, + { + "selected": false, + "text": "large_floats_pool", + "value": "large_floats_pool" + }, + { + "selected": false, + "text": "unaggregated_iterator_pool", + "value": "unaggregated_iterator_pool" + }, + { + "selected": false, + "text": "aggregation_types_pool", + "value": "aggregation_types_pool" + }, + { + "selected": false, + "text": "quantile_pool", + "value": "quantile_pool" + }, + { + "selected": false, + "text": "bytes_pool", + "value": "bytes_pool" + } + ], + "query": "entry_pool,counter_elem_pool,timer_elem_pool,gauge_elem_pool,buffered_encoder_pool,sample_pool,floats_pool,stream_pool,large_floats_pool,unaggregated_iterator_pool,aggregation_types_pool,quantile_pool,bytes_pool", + "refresh": 1, + "regex": "instance:(.*)", + "skipUrlSync": false, + "tag": "instance", + "type": "custom" + }, + { + "allFormat": "glob", + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "{total,free,get-on-empty,put-on-full}" + }, + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "multiFormat": "glob", + "name": "poolMetricType", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "total", + "value": "total" + }, + { + "selected": false, + "text": "free", + "value": "free" + }, + { + "selected": false, + "text": "get_on_empty", + "value": "get_on_empty" + }, + { + "selected": false, + "text": "put_on_full", + "value": "put_on_full" + } + ], + "query": "total,free,get_on_empty,put_on_full", + "refresh": 1, + "regex": "instance:(.*)", + "skipUrlSync": false, + "tag": "instance", + "type": "custom" + }, + { + "allFormat": "glob", + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "m3aggregator_aggregator_flush_handler_placement_update{m3_cluster=~\"$cluster\",backend=\"m3msg\"}", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "m3msg_consumer_service", + "options": [], + "query": "m3aggregator_aggregator_flush_handler_placement_update{m3_cluster=~\"$cluster\",backend=\"m3msg\"}", + "refresh": 1, + "regex": "/consumer_service_name=\"([^\"]+)\"/", + "skipUrlSync": false, + "sort": 0, + "tag": "name", + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "glob", + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "m3aggregator_aggregator_flush_handler_placement_update{m3_cluster=~\"$cluster\",backend=\"m3msg\"}", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "m3msg_consumer_env", + "options": [], + "query": "m3aggregator_aggregator_flush_handler_placement_update{m3_cluster=~\"$cluster\",backend=\"m3msg\"}", + "refresh": 1, + "regex": "/consumer_service_env=\"([^\"]+)\"/", + "skipUrlSync": false, + "sort": 0, + "tag": "name", + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "1m", + "value": "1m" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "step", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + } + ], + "query": "30s,1m,5m,10m", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "M3: Aggregator End-to-End Details", + "uid": "sqQiAbQZz", + "version": 9 +} \ No newline at end of file From 99b0cb7a71da9b383ab41c3aefae40d7eccaa1fb Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Mon, 24 Feb 2020 01:42:51 -0500 Subject: [PATCH 4/5] Add namespace to the metrics names --- .../grafana/m3aggregator_end_to_end_details.json | 10 +++++----- .../aggregator/docker-compose.yml | 1 + src/query/server/query.go | 4 ++++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/integrations/grafana/m3aggregator_end_to_end_details.json b/integrations/grafana/m3aggregator_end_to_end_details.json index 0cd9b59797..4c0df4f9ee 100644 --- a/integrations/grafana/m3aggregator_end_to_end_details.json +++ b/integrations/grafana/m3aggregator_end_to_end_details.json @@ -71,7 +71,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(m3aggregator_aggregator_client_writeForwarded_success[$step]))", + "expr": "sum(rate(coordinator_downsampler_remote_aggregator_client_writeForwarded_success[$step]))", "refId": "A" } ], @@ -155,7 +155,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(m3aggregator_aggregator_client_writeForwarded_errors[$step]))", + "expr": "sum(rate(coordinator_downsampler_remote_aggregator_client_writeForwarded_errors[$step]))", "refId": "A" } ], @@ -239,7 +239,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(m3aggregator_aggregator_client_writer_manager_writer_queue_successes[$step]))", + "expr": "sum(rate(coordinator_downsampler_remote_aggregator_client_writer_manager_writer_queue_successes[$step]))", "refId": "A" } ], @@ -323,11 +323,11 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(m3aggregator_aggregator_client_writer_manager_writer_queue_dropped[$step]))", + "expr": "sum(rate(coordinator_downsampler_remote_aggregator_client_writer_manager_writer_queue_dropped[$step]))", "refId": "A" }, { - "expr": "sum(rate(m3aggregator_aggregator_client_writer_manager_writer_queue_errors[$step]))", + "expr": "sum(rate(coordinator_downsampler_remote_aggregator_client_writer_manager_writer_queue_errors[$step]))", "refId": "B" } ], diff --git a/scripts/docker-integration-tests/aggregator/docker-compose.yml b/scripts/docker-integration-tests/aggregator/docker-compose.yml index 63cd26c375..c93b41ee25 100644 --- a/scripts/docker-integration-tests/aggregator/docker-compose.yml +++ b/scripts/docker-integration-tests/aggregator/docker-compose.yml @@ -19,6 +19,7 @@ services: - "7204" ports: - "0.0.0.0:7202:7202" + - "0.0.0.0:7203:7203" - "0.0.0.0:7204:7204" networks: - backend diff --git a/src/query/server/query.go b/src/query/server/query.go index 90b812a726..08729d8b76 100644 --- a/src/query/server/query.go +++ b/src/query/server/query.go @@ -523,6 +523,10 @@ func newDownsampler( tagOptions models.TagOptions, instrumentOpts instrument.Options, ) (downsample.Downsampler, error) { + // Namespace the downsampler metrics. + instrumentOpts = instrumentOpts.SetMetricsScope( + instrumentOpts.MetricsScope().SubScope("downsampler")) + if clusterManagementClient == nil { return nil, fmt.Errorf("no configured cluster management config, " + "must set this config for downsampler") From 0856cc450b8ea665cd47b34fafe762129ca50fed Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Mon, 24 Feb 2020 02:04:12 -0500 Subject: [PATCH 5/5] Fix onerror --- src/query/server/query.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/server/query.go b/src/query/server/query.go index 08729d8b76..d55f91a25f 100644 --- a/src/query/server/query.go +++ b/src/query/server/query.go @@ -148,7 +148,7 @@ func Run(runOpts RunOptions) { scope, closer, _, err := cfg.Metrics.NewRootScopeAndReporters( instrument.NewRootScopeAndReportersOptions{ - OnError: func(e error) { + OnError: func(err error) { // NB(r): Required otherwise collisions when registering metrics will // cause a panic. logger.Error("register metric error", zap.Error(err))