From 7400f430128ef7aa309c5d9ec4c9e5331f046aa4 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 5 Oct 2020 07:29:25 -0600 Subject: [PATCH] [Metricbeat] Add latency config option into aws module (#20875) * Add latency config option into aws module --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/aws.asciidoc | 11 +++++++++++ x-pack/metricbeat/module/aws/_meta/config.yml | 4 ++++ x-pack/metricbeat/module/aws/_meta/docs.asciidoc | 11 +++++++++++ x-pack/metricbeat/module/aws/aws.go | 3 +++ x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go | 3 ++- .../module/aws/cloudwatch/cloudwatch_test.go | 6 +++--- x-pack/metricbeat/module/aws/ec2/ec2.go | 3 ++- x-pack/metricbeat/module/aws/rds/rds.go | 3 ++- .../module/aws/s3_daily_storage/s3_daily_storage.go | 3 ++- x-pack/metricbeat/module/aws/s3_request/s3_request.go | 3 ++- x-pack/metricbeat/module/aws/sqs/sqs.go | 3 ++- x-pack/metricbeat/module/aws/utils.go | 7 ++++++- x-pack/metricbeat/module/aws/utils_test.go | 4 ++-- x-pack/metricbeat/modules.d/aws.yml.disabled | 4 ++++ 15 files changed, 57 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2f956d70838..279eda229a7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -724,6 +724,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add billing data collection from Cost Explorer into aws billing metricset. {pull}20527[20527] {issue}20103[20103] - Migrate `compute_vm` metricset to a light one, map `cloud.instance.id` field. {pull}20889[20889] - Request prometheus endpoints to be gzipped by default {pull}20766[20766] +- Add latency config parameter into aws module. {pull}20875[20875] - Release all kubernetes `state` metricsets as GA {pull}20901[20901] - Add billing metricset into googlecloud module. {pull}20812[20812] {issue}20738[20738] - Move `compute_vm_scaleset` to light metricset. {pull}21038[21038] {issue}20985[20985] diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 42d24c65ccd..e02a7a81460 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -19,16 +19,27 @@ module. Please see <> for more details. [float] == Module-specific configuration notes +* *AWS Credentials* + The `aws` module requires AWS credentials configuration in order to make AWS API calls. Users can either use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or `AWS_SESSION_TOKEN`, or use shared AWS credentials file. Please see <> for more details. +* *regions* + This module also accepts optional configuration `regions` to specify which AWS regions to query metrics from. If the `regions` parameter is not set in the config file, then by default, the `aws` module will query metrics from all available AWS regions. +* *latency* + +Some AWS services send monitoring metrics to CloudWatch with a latency to +process larger than Metricbeat collection period. This case, please specify a +`latency` parameter so collection start time and end time will be shifted by the +given latency amount. + The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index 618ed4cd854..6f604138505 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -44,4 +44,8 @@ period: 24h metricsets: - s3_daily_storage +- module: aws + period: 1m + latency: 5m + metricsets: - s3_request diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index fe9aeea007f..e4e55e82136 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -11,16 +11,27 @@ module. Please see <> for more details. [float] == Module-specific configuration notes +* *AWS Credentials* + The `aws` module requires AWS credentials configuration in order to make AWS API calls. Users can either use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or `AWS_SESSION_TOKEN`, or use shared AWS credentials file. Please see <> for more details. +* *regions* + This module also accepts optional configuration `regions` to specify which AWS regions to query metrics from. If the `regions` parameter is not set in the config file, then by default, the `aws` module will query metrics from all available AWS regions. +* *latency* + +Some AWS services send monitoring metrics to CloudWatch with a latency to +process larger than Metricbeat collection period. This case, please specify a +`latency` parameter so collection start time and end time will be shifted by the +given latency amount. + The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index f7b744c27cb..9786c4f7b38 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -27,6 +27,7 @@ import ( type Config struct { Period time.Duration `config:"period" validate:"nonzero,required"` Regions []string `config:"regions"` + Latency time.Duration `config:"latency"` AWSConfig awscommon.ConfigAWS `config:",inline"` TagsFilter []Tag `config:"tags_filter"` } @@ -37,6 +38,7 @@ type MetricSet struct { RegionsList []string Endpoint string Period time.Duration + Latency time.Duration AwsConfig *awssdk.Config AccountName string AccountID string @@ -87,6 +89,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { metricSet := MetricSet{ BaseMetricSet: base, Period: config.Period, + Latency: config.Latency, AwsConfig: &awsConfig, TagsFilter: config.TagsFilter, } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 42d68acb3af..07e1f09acef 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -141,7 +141,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) // Check statistic method in config err := m.checkStatistics() diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 353ffd0e236..393ddecb07e 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1357,7 +1357,7 @@ func TestCreateEventsWithIdentifier(t *testing.T) { Value: "test-ec2", }, } - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) @@ -1399,7 +1399,7 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) { } resourceTypeTagFilters := map[string][]aws.Tag{} - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) @@ -1447,7 +1447,7 @@ func TestCreateEventsWithTagsFilter(t *testing.T) { Value: "foo", }, } - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index 36ad9a1ca02..4e0776072e6 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -88,7 +88,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index f8bd907b3f6..b381dcac943 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -79,7 +79,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go index 7c9d453baca..53248284d41 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -69,7 +69,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(report mb.ReporterV2) error { namespace := "AWS/S3" // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) // GetMetricData for AWS S3 from Cloudwatch for _, regionName := range m.MetricSet.RegionsList { diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request.go b/x-pack/metricbeat/module/aws/s3_request/s3_request.go index afe53ac49ba..63b93f6cdf4 100644 --- a/x-pack/metricbeat/module/aws/s3_request/s3_request.go +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request.go @@ -69,7 +69,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(report mb.ReporterV2) error { namespace := "AWS/S3" // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) // GetMetricData for AWS S3 from Cloudwatch for _, regionName := range m.MetricSet.RegionsList { diff --git a/x-pack/metricbeat/module/aws/sqs/sqs.go b/x-pack/metricbeat/module/aws/sqs/sqs.go index 7bc6a8349d0..6dc0774f66d 100644 --- a/x-pack/metricbeat/module/aws/sqs/sqs.go +++ b/x-pack/metricbeat/module/aws/sqs/sqs.go @@ -67,7 +67,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(report mb.ReporterV2) error { namespace := "AWS/SQS" // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index 67e5809bc8e..d1846083854 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -22,8 +22,13 @@ import ( ) // GetStartTimeEndTime function uses durationString to create startTime and endTime for queries. -func GetStartTimeEndTime(period time.Duration) (time.Time, time.Time) { +func GetStartTimeEndTime(period time.Duration, latency time.Duration) (time.Time, time.Time) { endTime := time.Now() + if latency != 0 { + // add latency if config is not 0 + endTime = endTime.Add(latency * -1) + } + // Set startTime double the period earlier than the endtime in order to // make sure GetMetricDataRequest gets the latest data point for each metric. return endTime.Add(period * -2), endTime diff --git a/x-pack/metricbeat/module/aws/utils_test.go b/x-pack/metricbeat/module/aws/utils_test.go index aef35f57e61..3c480d347db 100644 --- a/x-pack/metricbeat/module/aws/utils_test.go +++ b/x-pack/metricbeat/module/aws/utils_test.go @@ -176,7 +176,7 @@ func TestGetListMetricsOutputWithWildcard(t *testing.T) { } func TestGetMetricDataPerRegion(t *testing.T) { - startTime, endTime := GetStartTimeEndTime(10 * time.Minute) + startTime, endTime := GetStartTimeEndTime(10*time.Minute, 0) mockSvc := &MockCloudWatchClient{} var metricDataQueries []cloudwatch.MetricDataQuery @@ -205,7 +205,7 @@ func TestGetMetricDataPerRegion(t *testing.T) { } func TestGetMetricDataResults(t *testing.T) { - startTime, endTime := GetStartTimeEndTime(10 * time.Minute) + startTime, endTime := GetStartTimeEndTime(10*time.Minute, 0) mockSvc := &MockCloudWatchClient{} metricInfo := cloudwatch.Metric{ diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index d0053297885..cc3103643c7 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -47,4 +47,8 @@ period: 24h metricsets: - s3_daily_storage +- module: aws + period: 1m + latency: 5m + metricsets: - s3_request