From f23335cf0a7f3b2bf2a9cd140ed4ce3f6ae7b914 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 1 Jul 2021 06:00:35 +0800 Subject: [PATCH] [MetricBeat] [AWS] Fix aws metric tags with resourcegroupstaggingapi paginator (#26385) (#26443) (#26630) (cherry picked from commit 179108c9b0f75c8cf03a0d51f9e3de52b0f01542) Co-authored-by: martin --- CHANGELOG.next.asciidoc | 1 + .../module/aws/cloudwatch/cloudwatch_test.go | 8 +++++ x-pack/metricbeat/module/aws/utils.go | 30 +++++++--------- x-pack/metricbeat/module/aws/utils_test.go | 34 +++++++++++++++++-- 4 files changed, 53 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ca32a988962..b4cbd7fed51 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -385,6 +385,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix azure billing dashboard. {pull}25554[25554] - Major refactor of system/cpu and system/core metrics. {pull}25771[25771] - Fix GCP Project ID being ingested as `cloud.account.id` in `gcp.billing` module {issue}26357[26357] {pull}26412[26412] +- Fix aws metric tags with resourcegroupstaggingapi paginator. {issue}26385[26385] {pull}26443[26443] *Packetbeat* diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index ecd4bb2f9d1..25f601c0a45 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1309,7 +1309,15 @@ func (m *MockCloudWatchClientWithoutDim) GetMetricDataRequest(input *cloudwatch. func (m *MockResourceGroupsTaggingClient) GetResourcesRequest(input *resourcegroupstaggingapi.GetResourcesInput) resourcegroupstaggingapi.GetResourcesRequest { httpReq, _ := http.NewRequest("", "", nil) return resourcegroupstaggingapi.GetResourcesRequest{ + Input: input, + Copy: m.GetResourcesRequest, Request: &awssdk.Request{ + Operation: &awssdk.Operation{ + Name: "GetResources", + HTTPMethod: "POST", + HTTPPath: "/", + Paginator: nil, + }, Data: &resourcegroupstaggingapi.GetResourcesOutput{ PaginationToken: awssdk.String(""), ResourceTagMappingList: []resourcegroupstaggingapi.ResourceTagMapping{ diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index d1846083854..0788c8c29b7 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -177,27 +177,16 @@ func GetResourcesTags(svc resourcegroupstaggingapiiface.ClientAPI, resourceTypeF ResourceTypeFilters: resourceTypeFilters, } - init := true - for init || *getResourcesInput.PaginationToken != "" { - init = false - getResourcesRequest := svc.GetResourcesRequest(getResourcesInput) - output, err := getResourcesRequest.Send(context.TODO()) - if err != nil { - err = errors.Wrap(err, "error GetResources") - return nil, err - } - - getResourcesInput.PaginationToken = output.PaginationToken - if resourceTypeFilters == nil || len(output.ResourceTagMappingList) == 0 { - return nil, nil - } - - for _, resourceTag := range output.ResourceTagMappingList { + getResourcesRequest := svc.GetResourcesRequest(getResourcesInput) + paginator := resourcegroupstaggingapi.NewGetResourcesPaginator(getResourcesRequest) + for paginator.Next(context.TODO()) { + page := paginator.CurrentPage() + for _, resourceTag := range page.ResourceTagMappingList { shortIdentifier, err := FindShortIdentifierFromARN(*resourceTag.ResourceARN) if err == nil { resourceTagMap[shortIdentifier] = resourceTag.Tags } else { - err = errors.Wrap(err, "error occurs when proccessing shortIdentifier") + err = errors.Wrap(err, "error occurs when processing shortIdentifier") return nil, err } @@ -205,11 +194,16 @@ func GetResourcesTags(svc resourcegroupstaggingapiiface.ClientAPI, resourceTypeF if err == nil { resourceTagMap[wholeIdentifier] = resourceTag.Tags } else { - err = errors.Wrap(err, "error occurs when proccessing longIdentifier") + err = errors.Wrap(err, "error occurs when processing longIdentifier") return nil, err } } } + + if err := paginator.Err(); err != nil { + err = errors.Wrap(err, "error GetResources with Paginator") + return nil, err + } return resourceTagMap, nil } diff --git a/x-pack/metricbeat/module/aws/utils_test.go b/x-pack/metricbeat/module/aws/utils_test.go index 3c480d347db..985b5b4bb2d 100644 --- a/x-pack/metricbeat/module/aws/utils_test.go +++ b/x-pack/metricbeat/module/aws/utils_test.go @@ -113,10 +113,17 @@ func (m *MockCloudWatchClient) GetMetricDataRequest(input *cloudwatch.GetMetricD func (m *MockResourceGroupsTaggingClient) GetResourcesRequest(input *resourcegroupstaggingapi.GetResourcesInput) resourcegroupstaggingapi.GetResourcesRequest { httpReq, _ := http.NewRequest("", "", nil) - return resourcegroupstaggingapi.GetResourcesRequest{ + op := &awssdk.Operation{ + Name: "GetResources", + HTTPMethod: "POST", + HTTPPath: "/", + Paginator: nil, + } + firstPageResult := resourcegroupstaggingapi.GetResourcesRequest{ Request: &awssdk.Request{ + Operation: op, Data: &resourcegroupstaggingapi.GetResourcesOutput{ - PaginationToken: awssdk.String(""), + PaginationToken: awssdk.String("PaginationToken"), ResourceTagMappingList: []resourcegroupstaggingapi.ResourceTagMapping{ { ResourceARN: awssdk.String("arn:aws:rds:eu-west-1:123456789012:db:mysql-db-1"), @@ -148,6 +155,29 @@ func (m *MockResourceGroupsTaggingClient) GetResourcesRequest(input *resourcegro }, HTTPRequest: httpReq, }, + Input: input, + Copy: m.GetResourcesRequest, + } + + // aws resourcegroupstaggingapi default pagination size is 50, if resource amount is a + // multiple of 50, then last request has an empty result. + lastPageWithEmptyResult := resourcegroupstaggingapi.GetResourcesRequest{ + Request: &awssdk.Request{ + Data: &resourcegroupstaggingapi.GetResourcesOutput{ + PaginationToken: awssdk.String(""), + ResourceTagMappingList: []resourcegroupstaggingapi.ResourceTagMapping{}, + }, + HTTPRequest: httpReq, + Operation: op, + }, + Input: input, + Copy: m.GetResourcesRequest, + } + + if input.PaginationToken == nil { + return firstPageResult + } else { + return lastPageWithEmptyResult } }