diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ec96d247ce16..5b59ef5cd467 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -300,6 +300,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support to load default aws config file to get credentials. {pull}12727[12727] {issue}12708[12708] - Add statistic option into cloudwatch metricset. {issue}12370[12370] {pull}12840[12840] - Add cgroup memory stats to docker/memory metricset {pull}12916[12916] +- Add AWS elb metricset. {pull}12952[12952] {issue}11701[11701] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b6393b726323..7110042794e4 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1155,6 +1155,12 @@ type: integer -- +[float] +=== elb + +`elb` contains the metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS ELB. + + [float] === rds diff --git a/metricbeat/docs/images/metricbeat-aws-elb-overview.png b/metricbeat/docs/images/metricbeat-aws-elb-overview.png new file mode 100644 index 000000000000..37eecc1bd099 Binary files /dev/null and b/metricbeat/docs/images/metricbeat-aws-elb-overview.png differ diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index cd3f8dc77485..a17096402c30 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -163,6 +163,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -175,6 +177,8 @@ include::aws/cloudwatch.asciidoc[] include::aws/ec2.asciidoc[] +include::aws/elb.asciidoc[] + include::aws/rds.asciidoc[] include::aws/s3_daily_storage.asciidoc[] diff --git a/metricbeat/docs/modules/aws/elb.asciidoc b/metricbeat/docs/modules/aws/elb.asciidoc new file mode 100644 index 000000000000..ab3d0f9d2696 --- /dev/null +++ b/metricbeat/docs/modules/aws/elb.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aws-elb]] +=== aws elb metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/elb/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/cockroachdb/status.asciidoc b/metricbeat/docs/modules/cockroachdb/status.asciidoc index 8ddc3c3d393e..3bd0aef96873 100644 --- a/metricbeat/docs/modules/cockroachdb/status.asciidoc +++ b/metricbeat/docs/modules/cockroachdb/status.asciidoc @@ -15,3 +15,9 @@ include::../../../../x-pack/metricbeat/module/cockroachdb/status/_meta/docs.asci For a description of each field in the metricset, see the <> section. +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/cockroachdb/status/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index db47ddd7842f..911e7f52ed31 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,8 +10,9 @@ This file is generated! See scripts/mage/docs_collector.go |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.6+| .6+| |<> beta[] +.7+| .7+| |<> beta[] |<> +|<> beta[] |<> beta[] |<> beta[] |<> beta[] diff --git a/x-pack/metricbeat/magefile.go b/x-pack/metricbeat/magefile.go index cce41582ca57..cfed7147c4fe 100644 --- a/x-pack/metricbeat/magefile.go +++ b/x-pack/metricbeat/magefile.go @@ -90,7 +90,7 @@ func TestPackages() error { devtools.WithModules(), // To be increased or removed when more light modules are added - devtools.MinModules(1)) + devtools.MinModules(2)) } // Fields generates a fields.yml and fields.go for each module. diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/dashboard/Metricbeat-aws-elb-overview.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/dashboard/Metricbeat-aws-elb-overview.json new file mode 100644 index 000000000000..c74a4e680f06 --- /dev/null +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/dashboard/Metricbeat-aws-elb-overview.json @@ -0,0 +1,995 @@ +{ + "objects": [ + { + "attributes": { + "description": "Overview of AWS ELB Metrics", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": {}, + "gridData": { + "h": 9, + "i": "2", + "w": 25, + "x": 23, + "y": 32 + }, + "panelIndex": "2", + "panelRefName": "panel_0", + "title": "HTTP 5XX Errors", + "version": "7.2.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 11, + "i": "3", + "w": 37, + "x": 11, + "y": 0 + }, + "panelIndex": "3", + "panelRefName": "panel_1", + "title": "Request Count", + "version": "7.2.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 8, + "i": "4", + "w": 11, + "x": 0, + "y": 15 + }, + "panelIndex": "4", + "panelRefName": "panel_2", + "title": "Unhealthy Host Count", + "version": "7.2.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 8, + "i": "5", + "w": 11, + "x": 0, + "y": 7 + }, + "panelIndex": "5", + "panelRefName": "panel_3", + "title": "Healthy Host Count", + "version": "7.2.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "6", + "w": 37, + "x": 11, + "y": 11 + }, + "panelIndex": "6", + "panelRefName": "panel_4", + "title": "Latency in Seconds", + "version": "7.2.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 9, + "i": "7", + "w": 23, + "x": 0, + "y": 32 + }, + "panelIndex": "7", + "panelRefName": "panel_5", + "title": "HTTP Backend 4XX Errors", + "version": "7.2.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 9, + "i": "8", + "w": 23, + "x": 0, + "y": 23 + }, + "panelIndex": "8", + "panelRefName": "panel_6", + "title": "Backend Connection Errors", + "version": "7.2.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 7, + "i": "9", + "w": 11, + "x": 0, + "y": 0 + }, + "panelIndex": "9", + "panelRefName": "panel_7", + "version": "7.2.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 9, + "i": "10", + "w": 25, + "x": 23, + "y": 23 + }, + "panelIndex": "10", + "panelRefName": "panel_8", + "title": "HTTP Backend 2XX", + "version": "7.2.0" + } + ], + "timeRestore": false, + "title": "[Metricbeat AWS] ELB Overview", + "version": 1 + }, + "id": "e74bf320-b3ce-11e9-87a4-078dbbae220d", + "migrationVersion": { + "dashboard": "7.0.0" + }, + "references": [ + { + "id": "b9703dd0-b3c9-11e9-87a4-078dbbae220d", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "d560de70-b3c7-11e9-87a4-078dbbae220d", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "6fc1efd0-b3c9-11e9-87a4-078dbbae220d", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "6392bc30-b3c9-11e9-87a4-078dbbae220d", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "b2ea15a0-b3c7-11e9-87a4-078dbbae220d", + "name": "panel_4", + "type": "visualization" + }, + { + "id": "21f30090-b3ca-11e9-87a4-078dbbae220d", + "name": "panel_5", + "type": "visualization" + }, + { + "id": "572d40e0-b3ca-11e9-87a4-078dbbae220d", + "name": "panel_6", + "type": "visualization" + }, + { + "id": "b5308940-7347-11e9-816b-07687310a99a", + "name": "panel_7", + "type": "visualization" + }, + { + "id": "1f528f50-b3ce-11e9-87a4-078dbbae220d", + "name": "panel_8", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2019-08-01T15:09:06.696Z", + "version": "WzYzNSw4XQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "ELB HTTP 5XX Errors [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "7e66beb0-b3c6-11e9-af6e-ef22c5680226" + } + ], + "bar_color_rules": [ + { + "id": "7db91990-b3c6-11e9-af6e-ef22c5680226" + } + ], + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": "", + "gauge_color_rules": [ + { + "id": "7d0b9b80-b3c6-11e9-af6e-ef22c5680226" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "35d3cbc0-b3c6-11e9-bf3f-29d51aa3d971", + "index_pattern": "metricbeat-*", + "interval": "5m", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#3185FC", + "fill": 0, + "formatter": "number", + "id": "35d3cbc1-b3c6-11e9-bf3f-29d51aa3d971", + "label": "HTTP 5XX Errors", + "line_width": 2, + "metrics": [ + { + "field": "aws.metrics.HTTPCode_ELB_5XX.sum", + "id": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "type": "avg" + } + ], + "point_size": "5", + "separate_axis": 0, + "split_color_mode": "rainbow", + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.cloudwatch.dimensions.LoadBalancerName", + "terms_order_by": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "ELB HTTP 5XX Errors [Metricbeat AWS]", + "type": "metrics" + } + }, + "id": "b9703dd0-b3c9-11e9-87a4-078dbbae220d", + "migrationVersion": { + "visualization": "7.2.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2019-08-02T18:37:04.544Z", + "version": "WzY0NSw5XQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "ELB Request Count [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "7e66beb0-b3c6-11e9-af6e-ef22c5680226" + } + ], + "bar_color_rules": [ + { + "id": "7db91990-b3c6-11e9-af6e-ef22c5680226" + } + ], + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": "", + "gauge_color_rules": [ + { + "id": "7d0b9b80-b3c6-11e9-af6e-ef22c5680226" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "35d3cbc0-b3c6-11e9-bf3f-29d51aa3d971", + "index_pattern": "metricbeat-*", + "interval": "5m", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#3185FC", + "fill": 0, + "formatter": "number", + "id": "35d3cbc1-b3c6-11e9-bf3f-29d51aa3d971", + "label": "Request Count", + "line_width": 2, + "metrics": [ + { + "field": "aws.metrics.RequestCount.sum", + "id": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "type": "avg" + } + ], + "point_size": "5", + "separate_axis": 0, + "split_color_mode": "rainbow", + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.cloudwatch.dimensions.LoadBalancerName", + "terms_order_by": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "ELB Request Count [Metricbeat AWS]", + "type": "metrics" + } + }, + "id": "d560de70-b3c7-11e9-87a4-078dbbae220d", + "migrationVersion": { + "visualization": "7.2.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2019-08-02T18:24:52.316Z", + "version": "WzY0MCw5XQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "ELB Unhealthy Host Count [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "background_color": "rgba(244,78,59,1)", + "color": "rgba(255,255,255,1)", + "id": "7e66beb0-b3c6-11e9-af6e-ef22c5680226", + "operator": "gt", + "value": 0 + } + ], + "bar_color_rules": [ + { + "id": "7db91990-b3c6-11e9-af6e-ef22c5680226" + } + ], + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": "", + "gauge_color_rules": [ + { + "id": "7d0b9b80-b3c6-11e9-af6e-ef22c5680226" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "35d3cbc0-b3c6-11e9-bf3f-29d51aa3d971", + "index_pattern": "metricbeat-*", + "interval": "5m", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#3185FC", + "fill": 0, + "formatter": "number", + "id": "35d3cbc1-b3c6-11e9-bf3f-29d51aa3d971", + "label": "Unhealthy Host Count", + "line_width": 2, + "metrics": [ + { + "field": "aws.metrics.UnHealthyHostCount.max", + "id": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "type": "sum" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_color_mode": "rainbow", + "split_mode": "everything", + "stacked": "none", + "terms_field": "aws.cloudwatch.dimensions.AvailabilityZone", + "terms_order_by": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "metric" + }, + "title": "ELB Unhealthy Host Count [Metricbeat AWS]", + "type": "metrics" + } + }, + "id": "6fc1efd0-b3c9-11e9-87a4-078dbbae220d", + "migrationVersion": { + "visualization": "7.2.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2019-08-02T18:24:16.077Z", + "version": "WzYzOCw5XQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "ELB Healthy Host Count [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "background_color": "rgba(104,188,0,1)", + "color": "rgba(255,255,255,1)", + "id": "7e66beb0-b3c6-11e9-af6e-ef22c5680226", + "operator": "gt", + "value": 0 + } + ], + "bar_color_rules": [ + { + "id": "7db91990-b3c6-11e9-af6e-ef22c5680226" + } + ], + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": "", + "gauge_color_rules": [ + { + "id": "7d0b9b80-b3c6-11e9-af6e-ef22c5680226" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "35d3cbc0-b3c6-11e9-bf3f-29d51aa3d971", + "index_pattern": "metricbeat-*", + "interval": "5m", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#3185FC", + "fill": 0, + "formatter": "number", + "id": "35d3cbc1-b3c6-11e9-bf3f-29d51aa3d971", + "label": "Healthy Host Count", + "line_width": 2, + "metrics": [ + { + "field": "aws.metrics.HealthyHostCount.max", + "id": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "type": "sum" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_color_mode": "rainbow", + "split_mode": "everything", + "stacked": "none", + "terms_field": "aws.cloudwatch.dimensions.AvailabilityZone", + "terms_order_by": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "metric" + }, + "title": "ELB Healthy Host Count [Metricbeat AWS]", + "type": "metrics" + } + }, + "id": "6392bc30-b3c9-11e9-87a4-078dbbae220d", + "migrationVersion": { + "visualization": "7.2.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2019-08-02T18:24:30.551Z", + "version": "WzYzOSw5XQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "ELB Latency in Seconds [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "7e66beb0-b3c6-11e9-af6e-ef22c5680226" + } + ], + "bar_color_rules": [ + { + "id": "7db91990-b3c6-11e9-af6e-ef22c5680226" + } + ], + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": "", + "gauge_color_rules": [ + { + "id": "7d0b9b80-b3c6-11e9-af6e-ef22c5680226" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "35d3cbc0-b3c6-11e9-bf3f-29d51aa3d971", + "index_pattern": "metricbeat-*", + "interval": "5m", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#3185FC", + "fill": 0, + "formatter": "s,s,3", + "id": "35d3cbc1-b3c6-11e9-bf3f-29d51aa3d971", + "label": "Latency in seconds", + "line_width": 2, + "metrics": [ + { + "field": "aws.metrics.Latency.avg", + "id": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "type": "avg" + } + ], + "point_size": "5", + "separate_axis": 0, + "split_color_mode": "rainbow", + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.cloudwatch.dimensions.LoadBalancerName", + "terms_order_by": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "ELB Latency in Seconds [Metricbeat AWS]", + "type": "metrics" + } + }, + "id": "b2ea15a0-b3c7-11e9-87a4-078dbbae220d", + "migrationVersion": { + "visualization": "7.2.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2019-08-02T18:36:49.424Z", + "version": "WzY0Myw5XQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "ELB HTTP Backend 4XX Errors [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "7e66beb0-b3c6-11e9-af6e-ef22c5680226" + } + ], + "bar_color_rules": [ + { + "id": "7db91990-b3c6-11e9-af6e-ef22c5680226" + } + ], + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": "", + "gauge_color_rules": [ + { + "id": "7d0b9b80-b3c6-11e9-af6e-ef22c5680226" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "35d3cbc0-b3c6-11e9-bf3f-29d51aa3d971", + "index_pattern": "metricbeat-*", + "interval": "5m", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#3185FC", + "fill": "0", + "formatter": "number", + "id": "35d3cbc1-b3c6-11e9-bf3f-29d51aa3d971", + "label": "HTTP Backend 4XX Errors", + "line_width": 2, + "metrics": [ + { + "field": "aws.metrics.HTTPCode_Backend_4XX.sum", + "id": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "type": "avg" + } + ], + "point_size": "5", + "separate_axis": 0, + "split_color_mode": "rainbow", + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.cloudwatch.dimensions.LoadBalancerName", + "terms_order_by": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "ELB HTTP Backend 4XX Errors [Metricbeat AWS]", + "type": "metrics" + } + }, + "id": "21f30090-b3ca-11e9-87a4-078dbbae220d", + "migrationVersion": { + "visualization": "7.2.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2019-08-02T18:28:25.630Z", + "version": "WzY0Miw5XQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "ELB Backend Connection Errors [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "7e66beb0-b3c6-11e9-af6e-ef22c5680226" + } + ], + "bar_color_rules": [ + { + "id": "7db91990-b3c6-11e9-af6e-ef22c5680226" + } + ], + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": "", + "gauge_color_rules": [ + { + "id": "7d0b9b80-b3c6-11e9-af6e-ef22c5680226" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "35d3cbc0-b3c6-11e9-bf3f-29d51aa3d971", + "index_pattern": "metricbeat-*", + "interval": "5m", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#3185FC", + "fill": "00", + "formatter": "number", + "id": "35d3cbc1-b3c6-11e9-bf3f-29d51aa3d971", + "label": "Backend Connection Errors", + "line_width": 2, + "metrics": [ + { + "field": "aws.metrics.BackendConnectionErrors.sum", + "id": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "type": "avg" + } + ], + "point_size": "5", + "separate_axis": 0, + "split_color_mode": "rainbow", + "split_mode": "terms", + "stacked": "none", + "steps": 0, + "terms_field": "aws.cloudwatch.dimensions.LoadBalancerName", + "terms_order_by": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "ELB Backend Connection Errors [Metricbeat AWS]", + "type": "metrics" + } + }, + "id": "572d40e0-b3ca-11e9-87a4-078dbbae220d", + "migrationVersion": { + "visualization": "7.2.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2019-08-02T18:28:18.932Z", + "version": "WzY0MSw5XQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "AWS Region Filter", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "cloud.region", + "id": "1549397251041", + "indexPatternRefName": "control_0_index_pattern", + "label": "region", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": true, + "useTimeFilter": false + }, + "title": "AWS Region Filter", + "type": "input_control_vis" + } + }, + "id": "b5308940-7347-11e9-816b-07687310a99a", + "migrationVersion": { + "visualization": "7.2.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "control_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-07-23T20:30:59.432Z", + "version": "WzMwMSwzXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "ELB HTTP Backend 2XX [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "7e66beb0-b3c6-11e9-af6e-ef22c5680226" + } + ], + "bar_color_rules": [ + { + "id": "7db91990-b3c6-11e9-af6e-ef22c5680226" + } + ], + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": "", + "gauge_color_rules": [ + { + "id": "7d0b9b80-b3c6-11e9-af6e-ef22c5680226" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "35d3cbc0-b3c6-11e9-bf3f-29d51aa3d971", + "index_pattern": "metricbeat-*", + "interval": "5m", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#3185FC", + "fill": 0, + "formatter": "number", + "id": "35d3cbc1-b3c6-11e9-bf3f-29d51aa3d971", + "label": "HTTP Backend 2XX", + "line_width": 2, + "metrics": [ + { + "field": "aws.metrics.HTTPCode_Backend_2XX.sum", + "id": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "type": "avg" + } + ], + "point_size": "5", + "separate_axis": 0, + "split_color_mode": "rainbow", + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.cloudwatch.dimensions.LoadBalancerName", + "terms_order_by": "35d3cbc2-b3c6-11e9-bf3f-29d51aa3d971", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "ELB HTTP Backend 2XX [Metricbeat AWS]", + "type": "metrics" + } + }, + "id": "1f528f50-b3ce-11e9-87a4-078dbbae220d", + "migrationVersion": { + "visualization": "7.2.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2019-08-02T18:36:56.883Z", + "version": "WzY0NCw5XQ==" + } + ], + "version": "7.2.0" +} diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 632ffd2615e8..70a1f14f7bbe 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -113,13 +113,11 @@ func StringInSlice(str string, list []string) bool { } // InitEvent initialize mb.Event with basic information like service.name, cloud.provider -func InitEvent(metricsetName string, regionName string) mb.Event { +func InitEvent(regionName string) mb.Event { event := mb.Event{} - event.Service = metricsetName event.MetricSetFields = common.MapStr{} event.ModuleFields = common.MapStr{} event.RootFields = common.MapStr{} - event.RootFields.Put("service.name", metricsetName) event.RootFields.Put("cloud.provider", "aws") if regionName != "" { event.RootFields.Put("cloud.region", regionName) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json index 82f70c4a894e..3ab72fe4b627 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json @@ -3,239 +3,234 @@ "aws": { "cloudwatch": { "dimensions": { - "DBClusterIdentifier": "test1-cluster", - "Role": "WRITER" - }, - "metrics": { - "ActiveTransactions": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - }, - "AuroraBinlogReplicaLag": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - }, - "AuroraReplicaLagMaximum": { - "avg": 20.067200088500975, - "count": 5, - "max": 26.357999801635742, - "min": 13.218999862670898, - "sum": 100.33600044250488 - }, - "AuroraReplicaLagMinimum": { - "avg": 20.067200088500975, - "count": 5, - "max": 26.357999801635742, - "min": 13.218999862670898, - "sum": 100.33600044250488 - }, - "BinLogDiskUsage": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - }, - "BlockedTransactions": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - }, - "BufferCacheHitRatio": { - "avg": 100, - "max": 100, - "min": 100, - "sum": 500 - }, - "CPUUtilization": { - "avg": 3.8, - "max": 5, - "min": 3, - "sum": 19 - }, - "CommitLatency": { - "avg": 7.25688, - "count": 5, - "max": 8.186233333333332, - "min": 6.463433333333333, - "sum": 36.2844 - }, - "CommitThroughput": { - "avg": 0.49994837279396603, - "count": 5, - "max": 0.5001000200040008, - "min": 0.4998000799680128, - "sum": 2.49974186396983 - }, - "DDLLatency": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - }, - "DDLThroughput": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - }, - "DMLLatency": { - "avg": 0.16545333333333337, - "count": 5, - "max": 0.17093333333333333, - "min": 0.16276666666666667, - "sum": 0.8272666666666668 - }, - "DMLThroughput": { - "avg": 0.49994837279396603, - "count": 5, - "max": 0.5001000200040008, - "min": 0.4998000799680128, - "sum": 2.49974186396983 - }, - "DatabaseConnections": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - }, - "Deadlocks": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - }, - "DeleteLatency": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - }, - "DeleteThroughput": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - }, - "EngineUptime": { - "avg": 1244563, - "count": 5, - "max": 1244683, - "min": 1244443, - "sum": 6222815 - }, - "FreeLocalStorage": { - "avg": 31932801024, - "count": 5, - "max": 31932895232, - "min": 31932706816, - "sum": 159664005120 - }, - "FreeableMemory": { - "avg": 4474304102.4, - "max": 4476133376, - "min": 4473532416, - "sum": 22371520512 - }, - "InsertLatency": { - "avg": 0.16545333333333337, - "count": 5, - "max": 0.17093333333333333, - "min": 0.16276666666666667, - "sum": 0.8272666666666668 - }, - "InsertThroughput": { - "avg": 0.49994837279396603, - "count": 5, - "max": 0.5001000200040008, - "min": 0.4998000799680128, - "sum": 2.49974186396983 - }, - "LoginFailures": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - }, - "NetworkReceiveThroughput": { - "avg": 0.8400256730059361, - "max": 1.4001633523911123, - "min": 0.6999300069993001, - "sum": 4.200128365029681 - }, - "NetworkThroughput": { - "avg": 1.6800513460118722, - "count": 5, - "max": 2.8003267047822247, - "min": 1.3998600139986002, - "sum": 8.400256730059361 - }, - "NetworkTransmitThroughput": { - "avg": 0.8400256730059361, - "count": 5, - "max": 1.4001633523911123, - "min": 0.6999300069993001, - "sum": 4.200128365029681 - }, - "Queries": { - "avg": 9.018293376391766, - "count": 5, - "max": 9.842454118509144, - "min": 8.654615795090715, - "sum": 45.09146688195883 - }, - "ResultSetCacheHitRatio": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - }, - "SelectLatency": { - "avg": 0.18611627194800234, - "count": 5, - "max": 0.20127322404371584, - "min": 0.1782340425531915, - "sum": 0.9305813597400117 - }, - "SelectThroughput": { - "avg": 3.1130005268141185, - "count": 5, - "max": 3.2153805144608825, - "min": 3.0339401213576047, - "sum": 15.565002634070593 - }, - "UpdateLatency": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - }, - "UpdateThroughput": { - "avg": 0, - "count": 5, - "max": 0, - "min": 0, - "sum": 0 - } + "DBInstanceIdentifier": "test1" }, "namespace": "AWS/RDS" + }, + "metrics": { + "ActiveTransactions": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "AuroraBinlogReplicaLag": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "AuroraReplicaLag": { + "avg": 17.945199999999996, + "count": 5, + "max": 20.299, + "min": 9.026, + "sum": 89.72599999999998 + }, + "BinLogDiskUsage": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "BlockedTransactions": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "BufferCacheHitRatio": { + "avg": 100, + "count": 5, + "max": 100, + "min": 100, + "sum": 500 + }, + "CPUUtilization": { + "avg": 3, + "count": 5, + "max": 3, + "min": 3, + "sum": 15 + }, + "CommitLatency": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "CommitThroughput": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "DDLLatency": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "DDLThroughput": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "DMLLatency": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "DMLThroughput": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "DatabaseConnections": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "Deadlocks": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "DeleteLatency": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "DeleteThroughput": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "EngineUptime": { + "avg": 2888753, + "count": 5, + "max": 2888873, + "min": 2888633, + "sum": 14443765 + }, + "FreeLocalStorage": { + "avg": 32949542912, + "count": 5, + "max": 32949567488, + "min": 32949518336, + "sum": 164747714560 + }, + "FreeableMemory": { + "avg": 4717250969.6, + "count": 5, + "max": 4717965312, + "min": 4716732416, + "sum": 23586254848 + }, + "InsertLatency": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "InsertThroughput": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "LoginFailures": { + "avg": 0, + "max": 0, + "min": 0, + "sum": 0 + }, + "NetworkReceiveThroughput": { + "avg": 0.6999860014776844, + "count": 5, + "max": 0.700023334111137, + "min": 0.6999416715273727, + "sum": 3.499930007388422 + }, + "NetworkThroughput": { + "avg": 1.3999720029553688, + "count": 5, + "max": 1.400046668222274, + "min": 1.3998833430547455, + "sum": 6.999860014776844 + }, + "NetworkTransmitThroughput": { + "avg": 0.6999860014776844, + "count": 5, + "max": 0.700023334111137, + "min": 0.6999416715273727, + "sum": 3.499930007388422 + }, + "Queries": { + "avg": 6.169305601785825, + "count": 5, + "max": 6.380462125376914, + "min": 6.031523876170482, + "sum": 30.846528008929127 + }, + "ResultSetCacheHitRatio": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "SelectLatency": { + "avg": 0.2314997785623484, + "count": 5, + "max": 0.2420805369127517, + "min": 0.2227225806451613, + "sum": 1.157498892811742 + }, + "SelectThroughput": { + "avg": 2.529737853689286, + "count": 5, + "max": 2.600086669555652, + "min": 2.4822990420658058, + "sum": 12.648689268446429 + }, + "UpdateLatency": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + }, + "UpdateThroughput": { + "avg": 0, + "count": 5, + "max": 0, + "min": 0, + "sum": 0 + } } }, "cloud": { @@ -251,7 +246,6 @@ "name": "cloudwatch" }, "service": { - "name": "cloudwatch", - "type": "cloudwatch" + "type": "aws" } } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index cd75443ef66d..42e0f206c52b 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -123,15 +123,17 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { listMetricDetailTotal, namespaceDetailTotal := m.readCloudwatchConfig() // Create events based on listMetricDetailTotal from configuration - for _, regionName := range m.MetricSet.RegionsList { - awsConfig := m.MetricSet.AwsConfig.Copy() - awsConfig.Region = regionName - svcCloudwatch := cloudwatch.New(awsConfig) - svcResourceAPI := resourcegroupstaggingapi.New(awsConfig) - - err := m.createEvents(svcCloudwatch, svcResourceAPI, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, regionName, startTime, endTime, report) - if err != nil { - return errors.Wrap(err, "createEvents failed") + if len(listMetricDetailTotal.metricsWithStats) != 0 { + for _, regionName := range m.MetricSet.RegionsList { + awsConfig := m.MetricSet.AwsConfig.Copy() + awsConfig.Region = regionName + svcCloudwatch := cloudwatch.New(awsConfig) + svcResourceAPI := resourcegroupstaggingapi.New(awsConfig) + + err := m.createEvents(svcCloudwatch, svcResourceAPI, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, regionName, startTime, endTime, report) + if err != nil { + return errors.Wrap(err, "createEvents failed") + } } } @@ -362,6 +364,10 @@ func statisticLookup(stat string) (string, bool) { "SampleCount": "count", } statMethod, ok := statisticLookupTable[stat] + if !ok { + ok = strings.HasPrefix(stat, "p") + statMethod = stat + } return statMethod, ok } @@ -369,15 +375,15 @@ func generateFieldName(labels []string) string { stat := labels[statisticIdx] // Check if statistic method is one of Sum, SampleCount, Minimum, Maximum, Average if statMethod, ok := statisticLookup(stat); ok { - return "metrics." + labels[metricNameIdx] + "." + statMethod + return "aws.metrics." + labels[metricNameIdx] + "." + statMethod } // If not, then it should be a percentile in the form of pN return "metrics." + labels[metricNameIdx] + "." + stat } -func insertMetricSetFields(event mb.Event, metricValue float64, labels []string) mb.Event { - event.MetricSetFields.Put(generateFieldName(labels), metricValue) - event.MetricSetFields.Put("namespace", labels[namespaceIdx]) +func insertRootFields(event mb.Event, metricValue float64, labels []string) mb.Event { + event.RootFields.Put(generateFieldName(labels), metricValue) + event.RootFields.Put("aws.cloudwatch.namespace", labels[namespaceIdx]) if len(labels) == 3 { return event } @@ -385,7 +391,7 @@ func insertMetricSetFields(event mb.Event, metricValue float64, labels []string) dimNames := strings.Split(labels[identifierNameIdx], ",") dimValues := strings.Split(labels[identifierValueIdx], ",") for i := 0; i < len(dimNames); i++ { - event.MetricSetFields.Put("dimensions."+dimNames[i], dimValues[i]) + event.RootFields.Put("aws.cloudwatch.dimensions."+dimNames[i], dimValues[i]) } return event } @@ -403,7 +409,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes events := map[string]mb.Event{} for _, values := range identifiers { for _, v := range values { - events[v] = aws.InitEvent(metricsetName, regionName) + events[v] = aws.InitEvent(regionName) } } // Initialize events for the ones without identifiers. @@ -434,14 +440,14 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes labels := strings.Split(*output.Label, " ") if len(labels) == 5 { identifierValue := labels[identifierValueIdx] - events[identifierValue] = insertMetricSetFields(events[identifierValue], output.Values[timestampIdx], labels) + events[identifierValue] = insertRootFields(events[identifierValue], output.Values[timestampIdx], labels) tags := resourceTagMap[identifierValue] for _, tag := range tags { - events[identifierValue].ModuleFields.Put("tags."+*tag.Key, *tag.Value) + events[identifierValue].RootFields.Put("aws.tags."+*tag.Key, *tag.Value) } } else { - eventNew := aws.InitEvent(metricsetName, regionName) - eventNew = insertMetricSetFields(eventNew, output.Values[timestampIdx], labels) + eventNew := aws.InitEvent(regionName) + eventNew = insertRootFields(eventNew, output.Values[timestampIdx], labels) eventsNoIdentifier = append(eventsNoIdentifier, eventNew) } } @@ -449,18 +455,14 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes } for _, event := range events { - if len(event.MetricSetFields) != 0 { - if reported := report.Event(event); !reported { - return nil - } + if reported := report.Event(event); !reported { + return nil } } for _, event := range eventsNoIdentifier { - if len(event.MetricSetFields) != 0 { - if reported := report.Event(event); !reported { - return nil - } + if reported := report.Event(event); !reported { + return nil } } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go index 02f48777d12b..d23e90016da7 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go @@ -38,10 +38,8 @@ func TestData(t *testing.T) { } config = addCloudwatchMetricsToConfig(config) - metricSet := mbtest.NewReportingMetricSetV2Error(t, config) - if err := mbtest.WriteEventsReporterV2Error(metricSet, t, "/"); err != nil { - t.Fatal("write", err) - } + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") } func addCloudwatchMetricsToConfig(config map[string]interface{}) map[string]interface{} { diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index ce8667d8dc8d..b3e455760461 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -378,32 +378,32 @@ func TestGenerateFieldName(t *testing.T) { { "test Average", []string{"CPUUtilization", "AWS/EC2", "Average", "InstanceId", "i-1"}, - "metrics.CPUUtilization.avg", + "aws.metrics.CPUUtilization.avg", }, { "test Maximum", []string{"CPUUtilization", "AWS/EC2", "Maximum", "InstanceId", "i-1"}, - "metrics.CPUUtilization.max", + "aws.metrics.CPUUtilization.max", }, { "test Minimum", []string{"CPUUtilization", "AWS/EC2", "Minimum", "InstanceId", "i-1"}, - "metrics.CPUUtilization.min", + "aws.metrics.CPUUtilization.min", }, { "test Sum", []string{"CPUUtilization", "AWS/EC2", "Sum", "InstanceId", "i-1"}, - "metrics.CPUUtilization.sum", + "aws.metrics.CPUUtilization.sum", }, { "test SampleCount", []string{"CPUUtilization", "AWS/EC2", "SampleCount", "InstanceId", "i-1"}, - "metrics.CPUUtilization.count", + "aws.metrics.CPUUtilization.count", }, { "test extended statistic", []string{"CPUUtilization", "AWS/EC2", "p10", "InstanceId", "i-1"}, - "metrics.CPUUtilization.p10", + "aws.metrics.CPUUtilization.p10", }, } diff --git a/x-pack/metricbeat/module/aws/ec2/_meta/data.json b/x-pack/metricbeat/module/aws/ec2/_meta/data.json index 4debf70b390c..0ad102cc4ac9 100644 --- a/x-pack/metricbeat/module/aws/ec2/_meta/data.json +++ b/x-pack/metricbeat/module/aws/ec2/_meta/data.json @@ -3,8 +3,12 @@ "aws": { "ec2": { "cpu": { + "credit_balance": 144, + "credit_usage": 0.005335, + "surplus_credit_balance": 0, + "surplus_credits_charged": 0, "total": { - "pct": 0.7000000000116418 + "pct": 0.0989071038248902 } }, "diskio": { @@ -22,32 +26,33 @@ "count": 1 }, "image": { - "id": "ami-62d35c02" + "id": "ami-05b3bcf7f311194b3" }, "monitoring": { - "state": "enabled" + "state": "disabled" }, "private": { - "dns_name": "ip-10-240-32-59.us-west-2.compute.internal", - "ip": "10.240.32.59" + "dns_name": "ip-172-31-10-23.ap-southeast-1.compute.internal", + "ip": "172.31.10.23" }, "public": { - "dns_name": "" + "dns_name": "ec2-18-136-198-93.ap-southeast-1.compute.amazonaws.com", + "ip": "18.136.198.93" }, "state": { "code": 16, "name": "running" }, - "threads_per_core": 2 + "threads_per_core": 1 }, "network": { "in": { - "bytes": 16598.6, - "packets": 127.8 + "bytes": 85.2, + "packets": 1.4 }, "out": { - "bytes": 29934.4, - "packets": 225.4 + "bytes": 108.8, + "packets": 2 } }, "status": { @@ -55,22 +60,18 @@ "check_failed_instance": 0, "check_failed_system": 0 } - }, - "tags": { - "Name": "service-node", - "aws:autoscaling:groupName": "service-asg" } }, "cloud": { - "availability_zone": "us-west-2b", + "availability_zone": "ap-southeast-1b", "instance": { - "id": "i-05cb8b44489d276f4" + "id": "i-0c68eeb552231a8d0" }, "machine": { - "type": "c4.large" + "type": "t2.micro" }, "provider": "aws", - "region": "us-west-2" + "region": "ap-southeast-1" }, "event": { "dataset": "aws.ec2", @@ -81,7 +82,6 @@ "name": "ec2" }, "service": { - "name": "ec2", - "type": "ec2" + "type": "aws" } -} +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index 1276a4c0af7d..7148b03cd87a 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -156,7 +156,7 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met events := map[string]mb.Event{} metricSetFieldResults := map[string]map[string]interface{}{} for instanceID := range instanceOutput { - events[instanceID] = aws.InitEvent(metricsetName, regionName) + events[instanceID] = aws.InitEvent(regionName) metricSetFieldResults[instanceID] = map[string]interface{}{} } diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_test.go index 1cedab665537..89049907cb19 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2_test.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2_test.go @@ -127,7 +127,6 @@ func TestGetInstanceIDs(t *testing.T) { func TestCreateCloudWatchEvents(t *testing.T) { expectedEvent := mb.Event{ RootFields: common.MapStr{ - "service": common.MapStr{"name": "ec2"}, "cloud": common.MapStr{ "region": regionName, "provider": "aws", diff --git a/x-pack/metricbeat/module/aws/elb/_meta/data.json b/x-pack/metricbeat/module/aws/elb/_meta/data.json new file mode 100644 index 000000000000..8910668edc97 --- /dev/null +++ b/x-pack/metricbeat/module/aws/elb/_meta/data.json @@ -0,0 +1,35 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "aws": { + "cloudwatch": { + "dimensions": { + "AvailabilityZone": "us-east-1b", + "LoadBalancerName": "DockerReg-EcsElast-1UFCSV3UVE46A" + }, + "namespace": "AWS/ELB" + }, + "metrics": { + "HealthyHostCount": { + "avg": 2 + }, + "UnHealthyHostCount": { + "avg": 0 + } + } + }, + "cloud": { + "provider": "aws", + "region": "us-east-1" + }, + "event": { + "dataset": "aws.elb", + "duration": 115000, + "module": "aws" + }, + "metricset": { + "name": "elb" + }, + "service": { + "type": "aws" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc new file mode 100644 index 000000000000..68961db3408c --- /dev/null +++ b/x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc @@ -0,0 +1,63 @@ +Elastic Load Balancing publishes data points to Amazon CloudWatch for your load +balancers and your back-end instances. This aws `elb` metricset collects these +Cloudwatch metrics for monitoring purposes. + +[float] +=== AWS Permissions +Some specific AWS permissions are required for IAM user to collect AWS ELB metrics. +---- +ec2:DescribeRegions +cloudwatch:GetMetricData +cloudwatch:ListMetrics +tag:getResources +---- + +[float] +=== Dashboard + +The aws elb metricset comes with a predefined dashboard. For example: + +image::./images/metricbeat-aws-elb-overview.png[] + +[float] +=== Configuration example +[source,yaml] +---- +- module: aws + period: 300s + metricsets: + - elb + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + # This module uses the aws cloudwatch metricset, all + # the options for this metricset are also available here. +---- + +[float] +=== Metrics +Elastic Load Balancing publishes data points to Amazon CloudWatch for your load +balancers and back-end instances. Please see more details for each metric in +https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-cloudwatch-metrics.html[elb-cloudwatch-metric]. + +|=== +|Metric Name|Statistic Method +|BackendConnectionErrors | Sum +|HealthyHostCount | Maximum +|HTTPCode_Backend_2XX | Sum +|HTTPCode_Backend_3XX | Sum +|HTTPCode_Backend_4XX | Sum +|HTTPCode_Backend_5XX | Sum +|HTTPCode_ELB_4XX | Sum +|HTTPCode_ELB_5XX | Sum +|Latency | Average +|RequestCount | Sum +|SpilloverCount | Sum +|SurgeQueueLength | Maximum +|UnHealthyHostCount | Maximum +|EstimatedALBActiveConnectionCount | Average +|EstimatedALBConsumedLCUs | Average +|EstimatedALBNewConnectionCount | Average +|EstimatedProcessedBytes | Average +|=== diff --git a/x-pack/metricbeat/module/aws/elb/_meta/fields.yml b/x-pack/metricbeat/module/aws/elb/_meta/fields.yml new file mode 100644 index 000000000000..f4e816791ab5 --- /dev/null +++ b/x-pack/metricbeat/module/aws/elb/_meta/fields.yml @@ -0,0 +1,6 @@ +- name: elb + type: group + description: > + `elb` contains the metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS ELB. + release: beta + fields: diff --git a/x-pack/metricbeat/module/aws/elb/elb_integration_test.go b/x-pack/metricbeat/module/aws/elb/elb_integration_test.go new file mode 100644 index 000000000000..a7e6b2a4189c --- /dev/null +++ b/x-pack/metricbeat/module/aws/elb/elb_integration_test.go @@ -0,0 +1,24 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package elb + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/x-pack/metricbeat/module/aws/mtest" +) + +func TestData(t *testing.T) { + config, info := mtest.GetConfigForTest("elb", "300s") + if info != "" { + t.Skip("Skipping TestData: " + info) + } + + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/aws/elb/elb_test.go b/x-pack/metricbeat/module/aws/elb/elb_test.go new file mode 100644 index 000000000000..eb91b04395d2 --- /dev/null +++ b/x-pack/metricbeat/module/aws/elb/elb_test.go @@ -0,0 +1,22 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package elb + +import ( + "os" + + "github.com/elastic/beats/metricbeat/mb" + xpackmb "github.com/elastic/beats/x-pack/metricbeat/mb" + + // Register input module and metricset + _ "github.com/elastic/beats/x-pack/metricbeat/module/aws" + _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/cloudwatch" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(xpackmb.NewLightModulesSource("../../../module")) +} diff --git a/x-pack/metricbeat/module/aws/elb/manifest.yml b/x-pack/metricbeat/module/aws/elb/manifest.yml new file mode 100644 index 000000000000..319ea3f90615 --- /dev/null +++ b/x-pack/metricbeat/module/aws/elb/manifest.yml @@ -0,0 +1,18 @@ +default: true +input: + module: aws + metricset: cloudwatch + defaults: + metrics: + - namespace: AWS/ELB + statistic: ["Sum"] + name: ["BackendConnectionErrors", "HTTPCode_Backend_2XX", "HTTPCode_Backend_3XX", "HTTPCode_Backend_4XX", "HTTPCode_Backend_5XX", "HTTPCode_ELB_4XX", "HTTPCode_ELB_5XX", "RequestCount", "SpilloverCount"] + tags.resource_type_filter: elasticloadbalancing + - namespace: AWS/ELB + statistic: ["Maximum"] + name: ["HealthyHostCount", "SurgeQueueLength", "UnHealthyHostCount"] + tags.resource_type_filter: elasticloadbalancing + - namespace: AWS/ELB + statistic: ["Average"] + name: ["Latency", "EstimatedALBActiveConnectionCount", "EstimatedALBConsumedLCUs", "EstimatedALBNewConnectionCount", "EstimatedProcessedBytes"] + tags.resource_type_filter: elasticloadbalancing diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index f1bb94283efe..29a1dd71dd33 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzsXE9z27YSv+dT7OSUdBL1tWnfIYc349ie18w4iWO50yMDASsKNQjQ+GNZmX74N/hDipIoS5RIuW+mObQJKWF/v93FYncB6C3c4eI9kLl5AWC5FfgeXpK5efkCgKGhmpeWK/ke/vMCAOAbmZtvUCjmBAJVQiC1Bs7+GEOhJLdKc5lDgVZzamCqVRHenQvl2JxYOhu9ANAokBh8Dzl5ATDlKJh5H0Z/C5IUWKHxf+yi9B/UypXpSQuo1UGaA1mSm9EP9eNqPDX5E6ltPI4Psvj2DhdzpVn766wgZcllnj778oeXjc+1Yot/bknuB4YHIhxCSbhO+iFzAxqNcpqiGW0woLXqNlg0tbJD+rflMN+AKmkJlwbsDGtb2RmxMEeNYKgmJbI16/3hvwvzGaez5QAtNjcoLUwWwPh0itr/w/MwJVlh13CCCVrSeL5uyaYu6pFW3lbq2DTbDqUEs8xwOSyYEimfcmQwn6GEe4d60bAAkJKPWpEl9qMfVpwNtjscrDsdU24i1ont63d7EP3UtLNG67SsbHy+QjCybqfJeIHScCXNwTzbrdQj0QabaJUm6o3phfTno+YV0p9PNKH8Fy/Pf26dQ/m+M4iWbmSVJWJUbpgpkjeUCGTZVCiy/oE9plKJmqK0JEdQUyBCKEosMg8cqCpKZxGc5Daph2gE6rQPE2IBXIIzCEoGPXJpLJEU2z3RE6EaGbeZMyRvjwhCyfyAcOCKCWqP//z6d4hCjA8N0Q5NbDBVOnzKWS74d+KH3Yl3QoT/7iCIkYRp3QQeFS2XmGfEAKFUO2RguH/CLcyJAUGcpDNkoDQYS7RFtp2McboUzmQnIJVErTKakQeECaJcWoZIcFLwgnuPq+mGSO6/dn79+3kY4UPEmpZibuA7arUvU5PRGdE5rkewnqgGLq2E/VyRyvrUgQFTc+kpb9r7DRDJUlixM2eAS+q01w1hjHsURECk0E5Zop0rfTficlQSeofWDMI0jQ0aKfIH73TSx4tKPHBpUU991rA+6Z6GrZw9Ce4QlpWzfeHmcjRZWNwb9FTpgtj30PalToTCAIOY4Tn59GUexs0dVyONhJ2Mz4dkEZJSBo+/DmfGKo3woIQr0AB5IFyQiUCwan82c80tnpiOl2lRepy98wnWUWVfE/5cFaVAv4gEG6gSdVjZzeHm8LkNaZQXJWqumHdXy4t9jDUEuzByk96BtjmEnLHEOjOiM6R32ZRw0duCeoOl0tb4dd/OUK8i9blPSYxBBhNlZ6svIyYImMIq6t+ahbFYrL7jMUMVxFgouHR2f5JZHO/EXIcgUsl5BirtFtuXTB1SqNL+P062V0J+qchRH11AKI0mFAq741n9lhckxxFvnxMHNzo+XoRJ6WH48X21xfxkjol/F3zLOnXkbdBjQ+ajZNzXjEtPYGiDxzWLY24ApY9FWyqUGmip+QOxOGLSZP5VvwpNo8PF53EQXKl3I9vYEyUv2z1x/XEHaB+vH37xqb9GY4AYoygPFfmcp/DXGaubCE6HUmgYfEOfe3plgtajFivFJRyXPrhwCh+v6zevvIJfw0Q5yaqFsatKwxQaUcXatXlwIArjruvwDfj6H37699sJt+Ck4bkM9XIQshfS/u3eihRelSiZn+5/gXZSxr+ZmbOWy/xtqIH/Aou64DL49F8+YwnNw+qvyF7vYGRnPskzWYk686F6qKUgyfHJUb0sbLYjNTNHtSM1MydsR95cjI9t6R/VkKyKktR1PLZd2bWR90/j8dSNR0YsmRCDGVVSIg1VyyB0KkHQEJT601uQTeqEeES07C86nhXku5Jwk3YI4TMp8NXZzefXwQWQ0JkPGrtBUUFMu64OgnXejDHNpbrq8vv6qcBC6QVQUhLK7QIChuqDFx92FfcN9JyhtL66XI/BfVAg3qz6rXFlKXz9Wht/KXUEtzNuGg98BupZOMnvHYad3eDv9Sf8sJ0oxlqmP3rjVI9HnGmXpbnQclMz3d6AyO4dOswYlnY2yExTznoFhGX+4xcDr/wy+WNsTWi8d2iseQ1zwv2aHzoUlPq8y5PyANuhV8X2vcgM6gfUGclR2uxPNRkmYESBMP56BeMgEM68QPACgbmwhO5VnU41oi9ssjh5TtpEJYUvhsPOXd3r0UQyVVRaT6C2Is+MVdqXrs8NO+GAsK+/Zc+ePPLCFZmvgjOriTQkBPqMsz59JImBhgRfiKdNLBP3sDyGEZyFABT6cNfK2Fzj+OtVO3glGBqbaSyFL5c9bCOUzQTJR8WkR/iC5Ll3XsO/1zE+Sa3fhTxTGRtaMKiLEOP/OLsK8aXeWujEz0eBjKuRKmNqbpC2kjpwn5o8YHCPxorPzV3s93788UuzKeoTdoNUyS05SVJG0Pw2lE/4PHNR0DFub3mBQODGo79JtmmsPd5O3s9mvOppxlSiuTw1bfNpMf569QY+Ec3JxYe4obi014qYLYmHmZMypsfPFAg8gDj3Y5MrnSlYYRxW9HREhJuwt7qMHz65WgbzdpbNmCFUbrIcJbZa85gJGByzQcVXAo1Q4gV3mllhaT391Iorese5de9Q8/3d5yB0SQbgI1Jn4/bFk6AYEiYUvRsWVi2l6mvXWekufHG/JixrzzX70uJb+etCOQ1nTiu9EpfeeGpB2BMJhVCUiGdOKyo+q+HBYlEqTfQCrH9mQpz003EXL6FyLsM2itMDO3dKS4NEINZDtrunnZ1p5fJZ6eyIqqLg7Y2ZvjBGEV3CQgMfQ4Fbth/6m4leRB0nuoBjYlBkFxdXdY3UCVcxLC4uDWpr3oArGbGYziRFPXYCGgc6AdZDrJu6/H2iqyNOtYOwFBf3q+tjOvEkgs/nfC5QcGvjVh4VHKU18WwYna3s0ftVJEXhkOL5WEyFMxb1MmQdoIEsoepRE1xSVfjS4tVNHPv1UiWaTKectqR0ngQVLrQSgraoM1YVqJdrZ/Vlr7mqs3Yxrh+HBcvH9kbj23+0UWbtrZTKMD1qRTmbq6CV2zT4/49afGk1wEReK4IsuUMZXDgknSvp5k6IBsXm4fV+o00UcUi0iaF0UHBRxCHgQnI/KLbJ+iGyYN9dEAWxKOmiYxbTZ2meEITZs5HohKhbcCF4JLEtX0wsOqQTQ1EIDR2GUy55rD6JzJ231KuLi6vXdTLSldj++chQxJ5MWTrS6Za1DMuoms0dKXSJ1j0Q6CuYV/C7RfKhLLAa7DtaoFu8H4rC6pLQkUKnVeFv5EVh3yt1L3lo7T1T36HR+lOUupLHdsqES6IXodVQpXsF8Vn8Zhc3drT1k83aBt317YR+txJaOpkNgeAFwpQL7NbPbMBfb8gODv+oRmzjy2bk/99rBdXSZ0lbwE25qevpE3olgciqPlxugVf1485ssMlmIhS96/WK1CadFRrrPdL6xlRC0gq/3q14lzHCxaLqMr5Yh9zlBNb6YGvHscK7uqE45OGs8btjz2ZNHL1D2/Npv8/prAiB8btKwlPSDf/+XG3fkOzWfZFwGYLFuw5P4Y4emqlpFi8199ngDSflmocnooQWbGH1JkLUrhbO3rT9wIB5l6W0/Vi/T8M0PD49+cfPn/CXqmaKxyAHdJbfbm+vl+3igrBwkcfH/NjxqXG+AY050UxUJ58X5ZYjFDX2HPvsm6xh/u/l7Rpu79yV73PZxmEH3tINiPf6997x9r7NsQb54vLq8vayb9Szfnt+a5h/uzy72Mufd/mCMkM6w5fxujcchLL35uQS5xLJ+PLq8vwWvgSjhwOSPtD27BWRSWYokfLEW9Tr/cRqkU9Y4qWYvdVxDPvqF1j+FvTrn4M5AX/Bh5xtNbaQaXhZ6VBygG7igvwUTqbmUijCnscy0SxLDGGy7bdkz2c+rYqn80ypZKjdqHAsnESeKLbl1KYrn5tuhSDaLKV9QOrk0WN/0z1yotZKm9Evj4/Dudsvj49pxzWKqy+kKoZ72S3OOJJ+ikRNAXm48/gvUBp+epLYr0MS+/XxEeLB6RMSq/qGU66NzbxzjIrOHnl4/7BE/bbyudA5DLVJdachnAGpXRJ9NVJvxsffntlQgVXxx2dWJmW43xL6QxOsA+/T+giJfFVdnVQlKEhpYvdki2qCrcJEXqoj3YIMR93Dm1Cu7Zq7dT16f9wVOHN/yitw469HX4FLR7kLNIbkmJEcT3yOuCy1euQFsQjpJpzXWoQFUsm3sfpgkCBW3bZwNWTb7+GFT4bChSz66wSuhqpKygqgZe8vyQ47eZunezWScCSCFwUyTiyKLStkzUUqmz1wwzd/p6+fyFvTqRlwCVPB89mWJa5GdhJU6+qzmuMDEctYsKc/eFcaFmnlr52QVeFrWGh1qj1ZACVCmCpapnNOn9IUi43tHZDN5qXXvm3OWAzo5CkdYlHaRXUKbJj7VWvqObv+WKnPzxXG4wyP2gVSEdhyOwHlMtyevL9cXaTpqOP4qt8e5fjrOMXMMO7/AgAA///XbAxn" + return "eJzsXE9z27YSv+dT7OSUdBL1tWnfIYc349ieV884iRO50yMDASsKNQjQ+GNZmX74N/hDipIoS5RIuW+mObQJKWF/v93FYncB6C3c4eI9kLl5AWC5FfgeXpK5efkCgKGhmpeWK/ke/vMCAOAbmZtvUCjmBAJVQiC1Bs7+GEOhJLdKc5lDgVZzamCqVRHenQvl2JxYOhu9ANAokBh8Dzl5ATDlKJh5H0Z/C5IUWKHxf+yi9B/UypXpSQuo1UGaA1mSm9EP9eNqPDX5E6ltPI4Psvj2DhdzpVn766wgZcllnj778oeXjc+1Yot/bknuB4YHIhxCSbhO+iFzAxqNcpqiGW0woLXqNlg0tbJD+rflMN+AKmkJlwbsDGtb2RmxMEeNYKgmJbI16/3hvwvzGaez5QAtNjcoLUwWwPh0itr/w/MwJVlh13CCCVrSeL5uyaYu6pFW3lbq2DTbDqUEs8xwOSyYEimfcmQwn6GEe4d60bAAkJKPWpEl9qMfVpwNtjscrDsdU24i1ont63d7EP3YtLNG67SsbHy+QjCybqfJeIHScCXNwTzbrdQj0QabaJUm6o3phfTno+YV0p9PNKH8Fy/Pf26dQ/m+M4iWbmSVJWJUbpgpkjeUCGTZVCiy/oE9plKJmqK0JEdQUyBCKEosMg8cqCpKZxGc5Daph2gE6rQPE2IBXIIzCEoGPXJpLJEU2z3RE6EaGbeZMyRvjwhCyfyAcOCKCWqP//zmd4hCjA8N0Q5NbDBVOnzKWS74d+KH3Yl3QoT/7iCIkYRp3QQeFS2XmGfEAKFUO2RguH/CLcyJAUGcpDNkoDQYS7RFtp2McboUzmQnIJVErTKakQeECaJcWoZIcFLwgnuPq+mGSO6/dn7z+3kY4UPEmpZibuA7arUvU5PRGdE5rkewnqgGLq2E/VyRyvrUgQFTc+kpb9r7DRDJUlixM2eAS+q01w1hjHsURECk0E5Zop0rfTficlQSeofWDMI0jQ0aKfIH73TSx4tKPHBpUU991rA+6Z6GrZw9Ce4QlpWzfeHmcjRZWNwb9FTpgtj30PalToTCAIOY4Tn59GUexs0dVyONhJ2Mz4dkEZJSBo+/DmfGKo3woIQr0AB5IFyQiUCwan82c80tnpiOl2lRepy98wnWUWVfE/5cFaVAv4gEG6gSdVjZzeHm8LkNaZQXJWqumHdXy4t9jDUEuzByk96BtjmEnLHEOjOiM6R32ZRw0duC+hVLpa3x676doV5F6nOfkhiDDCbKzlZfRkwQMIVV1L81C2OxWH3HY4YqiLFQcOns/iSzON6JuQ5BpJLzDFTaLbYvmTqkUKX9f5xsr4T8UpGjPrqAUBpNKBR2x7P6LS9IjiPePicObnRcXYRJ6WH48X21xfxkjol/F3zLOnXkbdBjQ+ZKMu5rxqUnMLTB45rFMTeA0seiLRVKDbTU/IFYHDFpMv+qX4Wm0eHi0zgIrtS7kW3siZKX7Z64/rgDtKubh1986q/RGCDGKMpDRT7nKfx1xuomgtOhFBoG39Dnnl6ZoPWoxUpxCcelDy6cwtVN/eaVV/BrmCgnWbUwdlVpmEIjqli7Ng8ORGHcdR2+AV//w0//fjvhFpw0PJehXg5C9kLav91bkcKrEiXz0/0v0E7K+Dczc9Zymb8NNfBfYFEXXAaf/stnLKF5WP0V2esdjOzMJ3kmK1FnPlQPtRQkOT45qpeFlnakmBzXjhSTU7Yjrz8c0tKvyGpmjiKrmTkh2a8X42P3L47qvlYVWGqxHtub7dq1/KfLeuouKyOWTIjBjCopkYYSbRA6lSBoCErN+C3IJnX2PyJa9rcUnBXku5LwNW2HwidS4Kuzr59eBxdAQmc+aOwGRQUx7bo6CNZ5M8Y085JqS8MXiwUWSi+AkpJQbhcQMFQfvPiwq5PRQM8ZSutL6fUFpw8KxJtVvzWuLIUv1mvjL6WO4HbGTeOBT7c9Cyf5vcOwjR38vf6EH7YTxVi49UdvnJoPEWfaUmpmFdzUTLd3W7J7hw4zhqWdDTLTlLNeASGnufps4JXPCX6MfRiN9w6NNa9hTrhPcEI7hlKfZHpSHmA79KqzcC8yg/oBdUZylDb7U02GCRhRIIy/XMM4CIQzLxC8QGAuLKF7leJTjeiruCxOnpN2jEnhK/+wTVk3tjSRTBWV1hOorcgzY5X2dfpzw044IBxi2HJAgTzywhWZL/kzq4k0JAT6jLM+fSSJgYYEuLqoduxM3LDzGEZwFgJQaDreKGNzjeMv1+3glWBobKaxFJyG5T8zQtlMkHxUTHqEL0iee+c1/Hsd45PU+l3IM5Wxod+Euggx/o+z6xBf6n2UTvx8FMi4Gqky1iEGaSupAzflyQMG92is+Nzcxeb21Y+fmx1gX50YpEpuyUmSMoLmt6F8wueZi4KOcXvLCwQCXz36r8k2jbXH28n72YxXDdyYSjSXp6ZtPi7GX67fwEeiObn4EHdPl/ZaEbMl8TBzUsb0+JkCgQcQ537s6KUDFCuMw4qezsNwEzaSl/HDJ1fLYN7OshkzhMpNlqPEVmseMwGDYzao+EqgEUq84E4zKyytp59acUXvOLfuHWq+v/schC7JAHxE6mzcq3kSFEPChKJ3w8KqpVRN/Dor3YUvbk6FZe25Zl9afCt/XSin4cxppVfi0htPLQh7IqEQihLxzGlFxWc1PFgsSqWJXoD1z0yIk3467uIlVM5l2DNyemDnTmlpkAjEesh297SzM61cPiudHVFVFLy9MdMXxiiiS1ho4GMocMteS38z0Yuo40QXcEwMiuzi4rqukTrhKobFxaVBbc0bcCUjFtMBrKjHTkDjQCfAeoh105ZGn+jqiFNtlyzFxc35+kxSPHbh8zmfCxTc2rhvSQVHaU08CEdnKwcS/CqSonBI8XwspsIZi3oZsg7QQJZQ9agJLqkqfGnx6msc+/VSJZpMp5y2pHSeBBUutBKCtqgzVhWol2tn9WWvuaqzdjGuH4cFy8f2RuPbf7RRZu2tlMowPWpFOZuroJXbNPj/j1p8aTXARF4rgiy5QxlcOCSdK+nmTogGxeZJ/X6jTRRxSLSJoXRQcFHEIeBCcj8otsn6iblg310QBbEo6aJjFtNnaZ4QhNmzkeiEqFtwIXgksS1fTCw6pBNDUQgNHYZTLnmsPonMnbfUq4uL69d1MtKV2P75yFDEnkxZOtLplrUMy6iazR0pdInWPRDoK5hX8LtF8qEssBrsO1qgW7wfisLqktCRQqdV4W/kRWHfK3UveWjtPVPfodH6U5S6ksd2yoRLoheh1VClewXxWfxmFzd2tPWTzdoG3fXthH63Elo6mQ2B4AXClAvs1s9swF9vyA4O/6hGbOPLZuT/32sF1dJnSVvATbmp6+kTeiWByKo+XG6BV/XjzmywyWYiFL3r9T7YJp0VGus90vp6WELSCr/erXiXMcLFouoyvliH3OUE1vpga8exwru6oTjk4azxu2PPZk0cvUPb89HGT+msCIHxu0rCU9IN//5cbd+Q7NZ9kXDzg8WLHU/hjh6aqWkWb3D32eANJ+WahyeihBZsYfUmQtSuFs7etP2agnmXpbT9WL9PwzQ8Pj35x8+f8JeqZorHIAd0lt9ub2+W7eKCsHBrycf82PGpcb4BjTnRTFTHvBflliMUNfYc++ybrGH+7+XtGm7v3JXvc9nGYQfe0g2I9+b33vH2vs2xBvni8vry9rJv1LN+e35rmH+7PLvYy593+YIyQzrD5/G6NxyEsvfm5BLnEsn48vry/BY+B6OHA5I+0PbsFZFJZiiR8sRb1Ov9xGqRT1jiDaC91XEM++rnZv4W9OvfvjkBf8GHnG01tpBpeFnpUHKAbuKC/BROpuZSKMKexzLRLEsMYbLtt2TPZz6tiqfzTKlkqN2ocCycRJ4otuXUpiufm26FINospX1A6uTRY3/TPXKi1kqb0S+Pj8O52y+Pj2nHNYqrb98qhnvZLc44kn53RU0Bebjg+S9QGn56ktivQxL79fER4sHpExKr+oZTro3NvHOMis4eeXj/sET9tvK50DkMtUl1pyGcAaldEn01Um/Gxx/a2VCBVfGXdlYmZbjfEvpDE6wD79P6CIl8VV2dVCUoSGli92SLaoKtwkReqiNd+QxH3cObUK7tmrt1PXp/3BU4c3/KK3DjL0dfgUtHuQs0huSYkRxPfI64LLV65AWxCOkmnNdahAVSybex+mCQIFbdtnA1ZNuP/4VPhsKFLPrrBK6GqkrKCqBl7y/JDjt5m6d7NZJwJIIXBTJOLIotK2TNRSqbPXDDN3+UsJ/IW9OpGXAJU8Hz2ZYlrkZ2ElTr6rOa4wMRy1iwpz94VxoWaeWvnZBV4WtYaHWqPVkAJUKYKlqmc04f0xSLje0dkM3mpde+bc5YDOjkKR1iUdpFdQpsmPtVa+o5u7mq1OfnCuNxhkftAqkIbLmdgHIZbk/eX64u0nTUcXzVb49y/GWcYmYY938BAAD//+OmUlg=" } diff --git a/x-pack/metricbeat/module/aws/module.yml b/x-pack/metricbeat/module/aws/module.yml new file mode 100644 index 000000000000..8ad742e3dc7b --- /dev/null +++ b/x-pack/metricbeat/module/aws/module.yml @@ -0,0 +1,3 @@ +name: aws +metricsets: + - elb diff --git a/x-pack/metricbeat/module/aws/rds/_meta/data.json b/x-pack/metricbeat/module/aws/rds/_meta/data.json index 048cbc97da24..a9937d2e31f6 100644 --- a/x-pack/metricbeat/module/aws/rds/_meta/data.json +++ b/x-pack/metricbeat/module/aws/rds/_meta/data.json @@ -18,28 +18,28 @@ "disk_usage": { "bin_log.bytes": 0 }, - "free_local_storage.bytes": 31211958272, - "freeable_memory.bytes": 4681383936, + "free_local_storage.bytes": 32025966592, + "freeable_memory.bytes": 4716777472, "latency": { - "commit": 0, + "commit": 2.810083333333333, "ddl": 0, - "dml": 0, - "insert": 0.16196666666666668, - "select": 0.22499999999999998, + "dml": 0.08358333333333333, + "insert": 0.08358333333333333, + "select": 0.24081333333333332, "update": 0 }, "login_failures": 0, - "queries": 7.510281559829625, + "queries": 8.966517224712922, "throughput": { - "commit": 0.25008753063572253, + "commit": 0, "ddl": 0, "delete": 0, - "dml": 0.25008753063572253, - "insert": 0.5001750612714451, - "network": 1.39989500797161, - "network_receive": 0.699947503985805, - "network_transmit": 0.699947503985805, - "select": 2.4832091728746897, + "dml": 0, + "insert": 0.4999333422210372, + "network": 1.4, + "network_receive": 0.6999825008749563, + "network_transmit": 0.6999650017499125, + "select": 2.498168010125908, "update": 0 }, "transactions": { @@ -62,7 +62,6 @@ "name": "rds" }, "service": { - "name": "rds", - "type": "rds" + "type": "aws" } } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index d5a44fff3eda..514c99aef0d0 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -227,7 +227,7 @@ func createCloudWatchEvents(getMetricDataResults []cloudwatch.MetricDataResult, metricSetFieldResults := map[string]map[string]interface{}{} for dbInstanceArn := range dbInstanceMap { - events[dbInstanceArn] = aws.InitEvent(metricsetName, regionName) + events[dbInstanceArn] = aws.InitEvent(regionName) metricSetFieldResults[dbInstanceArn] = map[string]interface{}{} } diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json index a8d34d27bfa8..3c5316dd961a 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json @@ -3,12 +3,12 @@ "aws": { "s3_daily_storage": { "bucket": { - "name": "test-s3-ks", + "name": "test-s3-ks-2", "size": { - "bytes": 1679631 + "bytes": 256 } }, - "number_of_objects": 3 + "number_of_objects": 8 } }, "cloud": { @@ -24,7 +24,6 @@ "name": "s3_daily_storage" }, "service": { - "name": "s3_daily_storage", - "type": "s3_daily_storage" + "type": "aws" } } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go index 29e613ed5e90..ed591deeab06 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -13,8 +13,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/pkg/errors" - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/x-pack/metricbeat/module/aws" @@ -188,12 +186,7 @@ func createMetricDataQuery(metric cloudwatch.Metric, period time.Duration, index } func createCloudWatchEvents(outputs []cloudwatch.MetricDataResult, regionName string, bucketName string) (event mb.Event, err error) { - event.Service = metricsetName - event.RootFields = common.MapStr{} - // Cloud fields in ECS - event.RootFields.Put("service.name", metricsetName) - event.RootFields.Put("cloud.region", regionName) - event.RootFields.Put("cloud.provider", "aws") + event = aws.InitEvent(regionName) // AWS s3_daily_storage metrics mapOfMetricSetFieldResults := make(map[string]interface{}) diff --git a/x-pack/metricbeat/module/aws/s3_request/_meta/data.json b/x-pack/metricbeat/module/aws/s3_request/_meta/data.json index 6b98c713d5cb..20fee5447c60 100644 --- a/x-pack/metricbeat/module/aws/s3_request/_meta/data.json +++ b/x-pack/metricbeat/module/aws/s3_request/_meta/data.json @@ -3,24 +3,30 @@ "aws": { "s3_request": { "bucket": { - "name": "test-s3-ks" + "name": "test-s3-ks-2" }, "downloaded": { - "bytes": 2606 + "bytes": 123653 }, "errors": { "4xx": 0, "5xx": 0 }, "latency": { - "first_byte.ms": 1032, - "total_request.ms": 1032 + "first_byte.ms": 7975, + "total_request.ms": 9005 }, "requests": { - "list": 2, - "total": 2 + "delete": 2, + "get": 80, + "head": 3, + "list": 37, + "put": 34, + "total": 156 }, - "uploaded": {} + "uploaded": { + "bytes": 2644 + } } }, "cloud": { @@ -36,7 +42,6 @@ "name": "s3_request" }, "service": { - "name": "s3_request", - "type": "s3_request" + "type": "aws" } } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request.go b/x-pack/metricbeat/module/aws/s3_request/s3_request.go index c81506ded913..884b8e333b47 100644 --- a/x-pack/metricbeat/module/aws/s3_request/s3_request.go +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request.go @@ -13,7 +13,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/pkg/errors" - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/x-pack/metricbeat/module/aws" @@ -189,12 +188,7 @@ func constructMetricQueries(listMetricsOutputs []cloudwatch.Metric, period time. // CreateS3Events creates s3_request and s3_daily_storage events from Cloudwatch metric data. func createS3RequestEvents(outputs []cloudwatch.MetricDataResult, regionName string, bucketName string) (event mb.Event, err error) { - event.Service = metricsetName - event.RootFields = common.MapStr{} - // Cloud fields in ECS - event.RootFields.Put("service.name", metricsetName) - event.RootFields.Put("cloud.region", regionName) - event.RootFields.Put("cloud.provider", "aws") + event = aws.InitEvent(regionName) // AWS s3_request metrics mapOfMetricSetFieldResults := make(map[string]interface{}) diff --git a/x-pack/metricbeat/module/aws/sqs/_meta/data.json b/x-pack/metricbeat/module/aws/sqs/_meta/data.json index 8611c17c945f..3a99959028d3 100644 --- a/x-pack/metricbeat/module/aws/sqs/_meta/data.json +++ b/x-pack/metricbeat/module/aws/sqs/_meta/data.json @@ -2,28 +2,27 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "aws": { "sqs": { - "empty_receives": 0, + "empty_receives": 28, "messages": { "delayed": 0, "deleted": 0, - "not_visible": 0, - "received": 0, + "not_visible": 1, + "received": 1, "sent": 0, - "visible": 80 + "visible": 0 }, "oldest_message_age": { - "sec": 83991 + "sec": 676 }, "queue": { - "name": "elb-log-queue-20171218185917658900000001" + "name": "test-s3-notification" }, - "sent_message_size": { - "bytes": 976 - } + "sent_message_size": {} } }, "cloud": { - "region": "us-east-1" + "provider": "aws", + "region": "ap-southeast-1" }, "event": { "dataset": "aws.sqs", @@ -34,7 +33,6 @@ "name": "sqs" }, "service": { - "name": "sqs", - "type": "sqs" + "type": "aws" } } \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/sqs/sqs.go b/x-pack/metricbeat/module/aws/sqs/sqs.go index d547301144e3..b92b33ca4977 100644 --- a/x-pack/metricbeat/module/aws/sqs/sqs.go +++ b/x-pack/metricbeat/module/aws/sqs/sqs.go @@ -16,7 +16,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface" "github.com/pkg/errors" - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" s "github.com/elastic/beats/libbeat/common/schema" "github.com/elastic/beats/metricbeat/mb" @@ -176,10 +175,7 @@ func createMetricDataQuery(metric cloudwatch.Metric, index int, period time.Dura } func createEventPerQueue(getMetricDataResults []cloudwatch.MetricDataResult, queueName string, metricsetName string, regionName string, schemaMetricFields s.Schema) (event mb.Event, err error) { - event.Service = metricsetName - event.RootFields = common.MapStr{} - event.RootFields.Put("service.name", metricsetName) - event.RootFields.Put("cloud.region", regionName) + event = aws.InitEvent(regionName) // AWS sqs metrics mapOfMetricSetFieldResults := make(map[string]interface{}) diff --git a/x-pack/metricbeat/module/cockroachdb/status/_meta/data.json b/x-pack/metricbeat/module/cockroachdb/status/_meta/data.json new file mode 100644 index 000000000000..e69de29bb2d1