From a19cb6df0b90de79ce9d7062ea223e9da36ac3be Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 14:05:32 -0400 Subject: [PATCH 01/18] tests/r/waf_byte_match_set: Add sweeper concurrency --- aws/resource_aws_waf_byte_match_set_test.go | 46 +++++++++++---------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/aws/resource_aws_waf_byte_match_set_test.go b/aws/resource_aws_waf_byte_match_set_test.go index d68ceaa8418..1ff3f417010 100644 --- a/aws/resource_aws_waf_byte_match_set_test.go +++ b/aws/resource_aws_waf_byte_match_set_test.go @@ -29,25 +29,27 @@ func init() { func testSweepWafByteMatchSet(region string) error { client, err := sharedClientForRegion(region) + if err != nil { return fmt.Errorf("error getting client: %s", err) } - conn := client.(*AWSClient).wafconn - var sweeperErrs *multierror.Error + conn := client.(*AWSClient).wafconn + sweepResources := make([]*testSweepResource, 0) + var errs *multierror.Error input := &waf.ListByteMatchSetsInput{} - err = lister.ListByteMatchSetsPages(conn, input, func(page *waf.ListByteMatchSetsOutput, lastPage bool) bool { + err = lister.ListByteMatchSetsPages(conn, input, func(page *waf.ListByteMatchSetsOutput, isLast bool) bool { if page == nil { - return !lastPage + return !isLast } for _, byteMatchSet := range page.ByteMatchSets { - id := aws.StringValue(byteMatchSet.ByteMatchSetId) - r := resourceAwsWafByteMatchSet() d := r.Data(nil) + + id := aws.StringValue(byteMatchSet.ByteMatchSetId) d.SetId(id) // Need to Read first to fill in byte_match_tuples attribute @@ -56,7 +58,7 @@ func testSweepWafByteMatchSet(region string) error { if err != nil { sweeperErr := fmt.Errorf("error reading WAF Byte Match Set (%s): %w", id, err) log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) + errs = multierror.Append(errs, sweeperErr) continue } @@ -65,29 +67,29 @@ func testSweepWafByteMatchSet(region string) error { continue } - err = r.Delete(d, client) - - if err != nil { - sweeperErr := fmt.Errorf("error deleting WAF Byte Match Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) } - return !lastPage + return !isLast }) - if testSweepSkipSweepError(err) { - log.Printf("[WARN] Skipping WAF Byte Match Set sweep for %s: %s", region, err) - return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("error listing WAF Byte Match Set for %s: %w", region, err)) } - if err != nil { - sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF Byte Match Sets: %w", err)) + if len(sweepResources) > 0 { + // Any errors didn't prevent gathering some sweeping work, so do it. + if err := testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Byte Match Set for %s: %w", region, err)) + } + } + + if testSweepSkipSweepError(errs.ErrorOrNil()) { + log.Printf("[WARN] Skipping WAF Byte Match Set sweep for %s: %s", region, errs) + return nil } - return sweeperErrs.ErrorOrNil() + return errs.ErrorOrNil() } func TestAccAWSWafByteMatchSet_basic(t *testing.T) { From b22a9d545e95821065363b5ccffc70abf7898ed8 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 14:08:47 -0400 Subject: [PATCH 02/18] tests/r/waf_geo_match_set: Add sweeper concurrency --- aws/resource_aws_waf_geo_match_set_test.go | 46 +++++++++++----------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/aws/resource_aws_waf_geo_match_set_test.go b/aws/resource_aws_waf_geo_match_set_test.go index 2646dafc162..f1bcbbc6638 100644 --- a/aws/resource_aws_waf_geo_match_set_test.go +++ b/aws/resource_aws_waf_geo_match_set_test.go @@ -29,25 +29,27 @@ func init() { func testSweepWafGeoMatchSet(region string) error { client, err := sharedClientForRegion(region) + if err != nil { return fmt.Errorf("error getting client: %s", err) } - conn := client.(*AWSClient).wafconn - var sweeperErrs *multierror.Error + conn := client.(*AWSClient).wafconn + sweepResources := make([]*testSweepResource, 0) + var errs *multierror.Error input := &waf.ListGeoMatchSetsInput{} - err = lister.ListGeoMatchSetsPages(conn, input, func(page *waf.ListGeoMatchSetsOutput, lastPage bool) bool { + err = lister.ListGeoMatchSetsPages(conn, input, func(page *waf.ListGeoMatchSetsOutput, isLast bool) bool { if page == nil { - return !lastPage + return !isLast } for _, geoMatchSet := range page.GeoMatchSets { - id := aws.StringValue(geoMatchSet.GeoMatchSetId) - r := resourceAwsWafGeoMatchSet() d := r.Data(nil) + + id := aws.StringValue(geoMatchSet.GeoMatchSetId) d.SetId(id) // Need to Read first to fill in geo_match_constraint attribute @@ -56,7 +58,7 @@ func testSweepWafGeoMatchSet(region string) error { if err != nil { sweeperErr := fmt.Errorf("error reading WAF Geo Match Set (%s): %w", id, err) log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) + errs = multierror.Append(errs, sweeperErr) continue } @@ -65,29 +67,29 @@ func testSweepWafGeoMatchSet(region string) error { continue } - err = r.Delete(d, client) - - if err != nil { - sweeperErr := fmt.Errorf("error deleting WAF Geo Match Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) } - return !lastPage + return !isLast }) - if testSweepSkipSweepError(err) { - log.Printf("[WARN] Skipping WAF Geo Match Set sweep for %s: %s", region, err) - return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("error listing WAF Geo Match Set for %s: %w", region, err)) } - if err != nil { - sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF Geo Match Sets: %w", err)) + if len(sweepResources) > 0 { + // Any errors didn't prevent gathering some sweeping work, so do it. + if err := testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Geo Match Set for %s: %w", region, err)) + } + } + + if testSweepSkipSweepError(errs.ErrorOrNil()) { + log.Printf("[WARN] Skipping WAF Geo Match Set sweep for %s: %s", region, errs) + return nil } - return sweeperErrs.ErrorOrNil() + return errs.ErrorOrNil() } func TestAccAWSWafGeoMatchSet_basic(t *testing.T) { From c27563ce43c35b0ab4ae726a8e5e6f0c1cd3669d Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 14:12:08 -0400 Subject: [PATCH 03/18] tests/r/waf_ipset: Add sweeper concurrency --- aws/resource_aws_waf_ipset_test.go | 46 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/aws/resource_aws_waf_ipset_test.go b/aws/resource_aws_waf_ipset_test.go index 2adb16a2bcc..e84601bc24e 100644 --- a/aws/resource_aws_waf_ipset_test.go +++ b/aws/resource_aws_waf_ipset_test.go @@ -32,25 +32,27 @@ func init() { func testSweepWafIPSet(region string) error { client, err := sharedClientForRegion(region) + if err != nil { return fmt.Errorf("error getting client: %s", err) } - conn := client.(*AWSClient).wafconn - var sweeperErrs *multierror.Error + conn := client.(*AWSClient).wafconn + sweepResources := make([]*testSweepResource, 0) + var errs *multierror.Error input := &waf.ListIPSetsInput{} - err = lister.ListIPSetsPages(conn, input, func(page *waf.ListIPSetsOutput, lastPage bool) bool { + err = lister.ListIPSetsPages(conn, input, func(page *waf.ListIPSetsOutput, isLast bool) bool { if page == nil { - return !lastPage + return !isLast } for _, ipSet := range page.IPSets { - id := aws.StringValue(ipSet.IPSetId) - r := resourceAwsWafIPSet() d := r.Data(nil) + + id := aws.StringValue(ipSet.IPSetId) d.SetId(id) // Need to Read first to fill in ip_set_descriptors attribute @@ -59,7 +61,7 @@ func testSweepWafIPSet(region string) error { if err != nil { sweeperErr := fmt.Errorf("error reading WAF IP Set (%s): %w", id, err) log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) + errs = multierror.Append(errs, sweeperErr) continue } @@ -68,29 +70,31 @@ func testSweepWafIPSet(region string) error { continue } - err = r.Delete(d, client) + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) - if err != nil { - sweeperErr := fmt.Errorf("error deleting WAF IP Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + err = r.Delete(d, client) } - return !lastPage + return !isLast }) - if testSweepSkipSweepError(err) { - log.Printf("[WARN] Skipping WAF IP Set sweep for %s: %s", region, err) - return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("error listing WAF IP Set for %s: %w", region, err)) } - if err != nil { - sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF IP Sets: %w", err)) + if len(sweepResources) > 0 { + // Any errors didn't prevent gathering some sweeping work, so do it. + if err := testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF IP Set for %s: %w", region, err)) + } + } + + if testSweepSkipSweepError(errs.ErrorOrNil()) { + log.Printf("[WARN] Skipping WAF IP Set sweep for %s: %s", region, errs) + return nil } - return sweeperErrs.ErrorOrNil() + return errs.ErrorOrNil() } func TestAccAWSWafIPSet_basic(t *testing.T) { From f3dbf444804704225f53d078b51d3c42520ad40d Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 14:42:09 -0400 Subject: [PATCH 04/18] tests/r/waf_byte_match_set: Add read concurrency --- aws/resource_aws_waf_byte_match_set_test.go | 40 ++++++++++++++------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/aws/resource_aws_waf_byte_match_set_test.go b/aws/resource_aws_waf_byte_match_set_test.go index 1ff3f417010..99f3295bd60 100644 --- a/aws/resource_aws_waf_byte_match_set_test.go +++ b/aws/resource_aws_waf_byte_match_set_test.go @@ -3,6 +3,7 @@ package aws import ( "fmt" "log" + "sync" "testing" "github.com/aws/aws-sdk-go/aws" @@ -40,6 +41,9 @@ func testSweepWafByteMatchSet(region string) error { input := &waf.ListByteMatchSetsInput{} + var g multierror.Group + var mutex = &sync.Mutex{} + err = lister.ListByteMatchSetsPages(conn, input, func(page *waf.ListByteMatchSetsOutput, isLast bool) bool { if page == nil { return !isLast @@ -52,22 +56,28 @@ func testSweepWafByteMatchSet(region string) error { id := aws.StringValue(byteMatchSet.ByteMatchSetId) d.SetId(id) - // Need to Read first to fill in byte_match_tuples attribute - err := r.Read(d, client) + // read concurrently and gather errors + g.Go(func() error { + // Need to Read first to fill in byte_match_tuples attribute + err := r.Read(d, client) - if err != nil { - sweeperErr := fmt.Errorf("error reading WAF Byte Match Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - errs = multierror.Append(errs, sweeperErr) - continue - } + if err != nil { + sweeperErr := fmt.Errorf("error reading WAF Byte Match Set (%s): %w", id, err) + log.Printf("[ERROR] %s", sweeperErr) + return sweeperErr + } - // In case it was already deleted - if d.Id() == "" { - continue - } + // In case it was already deleted + if d.Id() == "" { + return nil + } - sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) + mutex.Lock() + defer mutex.Unlock() + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) + + return nil + }) } return !isLast @@ -77,6 +87,10 @@ func testSweepWafByteMatchSet(region string) error { errs = multierror.Append(errs, fmt.Errorf("error listing WAF Byte Match Set for %s: %w", region, err)) } + if err = g.Wait().ErrorOrNil(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Byte Match Sets: %w", err)) + } + if len(sweepResources) > 0 { // Any errors didn't prevent gathering some sweeping work, so do it. if err := testSweepResourceOrchestrator(sweepResources); err != nil { From 6459516a55781a214148bc6ed5a20fa7e00d9d3b Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 14:46:53 -0400 Subject: [PATCH 05/18] tests/r/waf_geo_match_set: Add read concurrency --- aws/resource_aws_waf_geo_match_set_test.go | 41 +++++++++++++++------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/aws/resource_aws_waf_geo_match_set_test.go b/aws/resource_aws_waf_geo_match_set_test.go index f1bcbbc6638..0939b9c9da1 100644 --- a/aws/resource_aws_waf_geo_match_set_test.go +++ b/aws/resource_aws_waf_geo_match_set_test.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "regexp" + "sync" "testing" "github.com/aws/aws-sdk-go/aws" @@ -40,6 +41,9 @@ func testSweepWafGeoMatchSet(region string) error { input := &waf.ListGeoMatchSetsInput{} + var g multierror.Group + var mutex = &sync.Mutex{} + err = lister.ListGeoMatchSetsPages(conn, input, func(page *waf.ListGeoMatchSetsOutput, isLast bool) bool { if page == nil { return !isLast @@ -52,22 +56,29 @@ func testSweepWafGeoMatchSet(region string) error { id := aws.StringValue(geoMatchSet.GeoMatchSetId) d.SetId(id) - // Need to Read first to fill in geo_match_constraint attribute - err := r.Read(d, client) + // read concurrently and gather errors + g.Go(func() error { - if err != nil { - sweeperErr := fmt.Errorf("error reading WAF Geo Match Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - errs = multierror.Append(errs, sweeperErr) - continue - } + // Need to Read first to fill in geo_match_constraint attribute + err := r.Read(d, client) - // In case it was already deleted - if d.Id() == "" { - continue - } + if err != nil { + sweeperErr := fmt.Errorf("error reading WAF Geo Match Set (%s): %w", id, err) + log.Printf("[ERROR] %s", sweeperErr) + return sweeperErr + } + + // In case it was already deleted + if d.Id() == "" { + return nil + } + + mutex.Lock() + defer mutex.Unlock() + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) - sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) + return nil + }) } return !isLast @@ -77,6 +88,10 @@ func testSweepWafGeoMatchSet(region string) error { errs = multierror.Append(errs, fmt.Errorf("error listing WAF Geo Match Set for %s: %w", region, err)) } + if err = g.Wait().ErrorOrNil(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Geo Match Sets: %w", err)) + } + if len(sweepResources) > 0 { // Any errors didn't prevent gathering some sweeping work, so do it. if err := testSweepResourceOrchestrator(sweepResources); err != nil { From 46442951394a9c6c8956bfd4874f5018d23bfb5c Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 14:50:27 -0400 Subject: [PATCH 06/18] tests/r/waf_ipset: Add read concurrency --- aws/resource_aws_waf_ipset_test.go | 40 +++++++++++++++++++----------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/aws/resource_aws_waf_ipset_test.go b/aws/resource_aws_waf_ipset_test.go index e84601bc24e..9137c5e424b 100644 --- a/aws/resource_aws_waf_ipset_test.go +++ b/aws/resource_aws_waf_ipset_test.go @@ -7,6 +7,7 @@ import ( "reflect" "regexp" "strings" + "sync" "testing" "github.com/aws/aws-sdk-go/aws" @@ -40,6 +41,8 @@ func testSweepWafIPSet(region string) error { conn := client.(*AWSClient).wafconn sweepResources := make([]*testSweepResource, 0) var errs *multierror.Error + var g multierror.Group + var mutex = &sync.Mutex{} input := &waf.ListIPSetsInput{} @@ -55,24 +58,29 @@ func testSweepWafIPSet(region string) error { id := aws.StringValue(ipSet.IPSetId) d.SetId(id) - // Need to Read first to fill in ip_set_descriptors attribute - err := r.Read(d, client) + // read concurrently and gather errors + g.Go(func() error { - if err != nil { - sweeperErr := fmt.Errorf("error reading WAF IP Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - errs = multierror.Append(errs, sweeperErr) - continue - } + // Need to Read first to fill in ip_set_descriptors attribute + err := r.Read(d, client) - // In case it was already deleted - if d.Id() == "" { - continue - } + if err != nil { + sweeperErr := fmt.Errorf("error reading WAF IP Set (%s): %w", id, err) + log.Printf("[ERROR] %s", sweeperErr) + return sweeperErr + } + + // In case it was already deleted + if d.Id() == "" { + return nil + } - sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) + mutex.Lock() + defer mutex.Unlock() + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) - err = r.Delete(d, client) + return nil + }) } return !isLast @@ -82,6 +90,10 @@ func testSweepWafIPSet(region string) error { errs = multierror.Append(errs, fmt.Errorf("error listing WAF IP Set for %s: %w", region, err)) } + if err = g.Wait().ErrorOrNil(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF IP Sets: %w", err)) + } + if len(sweepResources) > 0 { // Any errors didn't prevent gathering some sweeping work, so do it. if err := testSweepResourceOrchestrator(sweepResources); err != nil { From 3671afd20079932c660d0b2880c241dffbf39f55 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 14:53:39 -0400 Subject: [PATCH 07/18] tests/r/waf_rate_based_rule: Add sweeper concurrency --- aws/resource_aws_waf_rate_based_rule_test.go | 80 ++++++++++++-------- 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/aws/resource_aws_waf_rate_based_rule_test.go b/aws/resource_aws_waf_rate_based_rule_test.go index 049831b9478..ec422edc45b 100644 --- a/aws/resource_aws_waf_rate_based_rule_test.go +++ b/aws/resource_aws_waf_rate_based_rule_test.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "regexp" + "sync" "testing" "github.com/aws/aws-sdk-go/aws" @@ -29,65 +30,80 @@ func init() { func testSweepWafRateBasedRules(region string) error { client, err := sharedClientForRegion(region) + if err != nil { return fmt.Errorf("error getting client: %s", err) } - conn := client.(*AWSClient).wafconn - var sweeperErrs *multierror.Error + conn := client.(*AWSClient).wafconn + sweepResources := make([]*testSweepResource, 0) + var errs *multierror.Error + var g multierror.Group + var mutex = &sync.Mutex{} input := &waf.ListRateBasedRulesInput{} - err = lister.ListRateBasedRulesPages(conn, input, func(page *waf.ListRateBasedRulesOutput, lastPage bool) bool { + err = lister.ListRateBasedRulesPages(conn, input, func(page *waf.ListRateBasedRulesOutput, isLast bool) bool { if page == nil { - return !lastPage + return !isLast } for _, rule := range page.Rules { - id := aws.StringValue(rule.RuleId) - r := resourceAwsWafRateBasedRule() d := r.Data(nil) + + id := aws.StringValue(rule.RuleId) d.SetId(id) - // Need to Read first to fill in predicates attribute - err := r.Read(d, client) + // read concurrently and gather errors + g.Go(func() error { - if err != nil { - sweeperErr := fmt.Errorf("error reading WAF Rate Based Rule (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + // Need to Read first to fill in predicates attribute + err := r.Read(d, client) - // In case it was already deleted - if d.Id() == "" { - continue - } + if err != nil { + sweeperErr := fmt.Errorf("error reading WAF Rate Based Rule (%s): %w", id, err) + log.Printf("[ERROR] %s", sweeperErr) + return sweeperErr + } - err = r.Delete(d, client) + // In case it was already deleted + if d.Id() == "" { + return nil + } - if err != nil { - sweeperErr := fmt.Errorf("error deleting WAF Rate Based Rule (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + mutex.Lock() + defer mutex.Unlock() + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) + + return nil + }) } - return !lastPage + return !isLast }) - if testSweepSkipSweepError(err) { - log.Printf("[WARN] Skipping WAF Rate Based Rule sweep for %s: %s", region, err) - return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("error listing WAF Rate Based Rule for %s: %w", region, err)) } - if err != nil { - sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF Rate Based Rules: %w", err)) + if err = g.Wait().ErrorOrNil(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Rate Based Rules: %w", err)) + } + + if len(sweepResources) > 0 { + // Any errors didn't prevent gathering some sweeping work, so do it. + if err := testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Rate Based Rule for %s: %w", region, err)) + } + } + + if testSweepSkipSweepError(errs.ErrorOrNil()) { + log.Printf("[WARN] Skipping WAF Rate Based Rule sweep for %s: %s", region, errs) + return nil } - return sweeperErrs.ErrorOrNil() + return errs.ErrorOrNil() } func TestAccAWSWafRateBasedRule_basic(t *testing.T) { From 0af45b963b7c73a7b85236311ca42e3f8bbf3d1f Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 15:03:06 -0400 Subject: [PATCH 08/18] tests/r/waf_regex_match_set: Add sweeper concurrency --- aws/resource_aws_waf_geo_match_set_test.go | 1 - aws/resource_aws_waf_ipset_test.go | 1 - aws/resource_aws_waf_rate_based_rule_test.go | 1 - aws/resource_aws_waf_regex_match_set_test.go | 79 ++++++++++++-------- 4 files changed, 47 insertions(+), 35 deletions(-) diff --git a/aws/resource_aws_waf_geo_match_set_test.go b/aws/resource_aws_waf_geo_match_set_test.go index 0939b9c9da1..2c8b7ff3fb3 100644 --- a/aws/resource_aws_waf_geo_match_set_test.go +++ b/aws/resource_aws_waf_geo_match_set_test.go @@ -58,7 +58,6 @@ func testSweepWafGeoMatchSet(region string) error { // read concurrently and gather errors g.Go(func() error { - // Need to Read first to fill in geo_match_constraint attribute err := r.Read(d, client) diff --git a/aws/resource_aws_waf_ipset_test.go b/aws/resource_aws_waf_ipset_test.go index 9137c5e424b..eba5cbd863a 100644 --- a/aws/resource_aws_waf_ipset_test.go +++ b/aws/resource_aws_waf_ipset_test.go @@ -60,7 +60,6 @@ func testSweepWafIPSet(region string) error { // read concurrently and gather errors g.Go(func() error { - // Need to Read first to fill in ip_set_descriptors attribute err := r.Read(d, client) diff --git a/aws/resource_aws_waf_rate_based_rule_test.go b/aws/resource_aws_waf_rate_based_rule_test.go index ec422edc45b..34f1bafc472 100644 --- a/aws/resource_aws_waf_rate_based_rule_test.go +++ b/aws/resource_aws_waf_rate_based_rule_test.go @@ -57,7 +57,6 @@ func testSweepWafRateBasedRules(region string) error { // read concurrently and gather errors g.Go(func() error { - // Need to Read first to fill in predicates attribute err := r.Read(d, client) diff --git a/aws/resource_aws_waf_regex_match_set_test.go b/aws/resource_aws_waf_regex_match_set_test.go index 22c813c75a2..54769e0146d 100644 --- a/aws/resource_aws_waf_regex_match_set_test.go +++ b/aws/resource_aws_waf_regex_match_set_test.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "regexp" + "sync" "testing" "github.com/aws/aws-sdk-go/aws" @@ -29,65 +30,79 @@ func init() { func testSweepWafRegexMatchSet(region string) error { client, err := sharedClientForRegion(region) + if err != nil { return fmt.Errorf("error getting client: %s", err) } - conn := client.(*AWSClient).wafconn - var sweeperErrs *multierror.Error + conn := client.(*AWSClient).wafconn + sweepResources := make([]*testSweepResource, 0) + var errs *multierror.Error + var g multierror.Group + var mutex = &sync.Mutex{} input := &waf.ListRegexMatchSetsInput{} - err = lister.ListRegexMatchSetsPages(conn, input, func(page *waf.ListRegexMatchSetsOutput, lastPage bool) bool { + err = lister.ListRegexMatchSetsPages(conn, input, func(page *waf.ListRegexMatchSetsOutput, isLast bool) bool { if page == nil { - return !lastPage + return !isLast } for _, regexMatchSet := range page.RegexMatchSets { - id := aws.StringValue(regexMatchSet.RegexMatchSetId) - r := resourceAwsWafRegexMatchSet() d := r.Data(nil) + + id := aws.StringValue(regexMatchSet.RegexMatchSetId) d.SetId(id) - // Need to Read first to fill in regex_match_tuple attribute - err := r.Read(d, client) + // read concurrently and gather errors + g.Go(func() error { + // Need to Read first to fill in regex_match_tuple attribute + err := r.Read(d, client) - if err != nil { - sweeperErr := fmt.Errorf("error reading WAF Regex Match Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + if err != nil { + sweeperErr := fmt.Errorf("error reading WAF Regex Match Set (%s): %w", id, err) + log.Printf("[ERROR] %s", sweeperErr) + return sweeperErr + } - // In case it was already deleted - if d.Id() == "" { - continue - } + // In case it was already deleted + if d.Id() == "" { + return nil + } - err = r.Delete(d, client) + mutex.Lock() + defer mutex.Unlock() + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) - if err != nil { - sweeperErr := fmt.Errorf("error deleting WAF Regex Match Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + return nil + }) } - return !lastPage + return !isLast }) - if testSweepSkipSweepError(err) { - log.Printf("[WARN] Skipping WAF Regex Match Set sweep for %s: %s", region, err) - return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("error listing WAF Regex Match Set for %s: %w", region, err)) } - if err != nil { - sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF Regex Match Sets: %w", err)) + if err = g.Wait().ErrorOrNil(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Regex Match Sets: %w", err)) + } + + if len(sweepResources) > 0 { + // Any errors didn't prevent gathering some sweeping work, so do it. + if err := testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Regex Match Set for %s: %w", region, err)) + } + } + + if testSweepSkipSweepError(errs.ErrorOrNil()) { + log.Printf("[WARN] Skipping WAF Regex Match Set sweep for %s: %s", region, errs) + return nil } - return sweeperErrs.ErrorOrNil() + return errs.ErrorOrNil() } // Serialized acceptance tests due to WAF account limits From d3f72662dc09078dbc6dada576657a403442aa08 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 15:07:52 -0400 Subject: [PATCH 09/18] tests/r/waf_regex_pattern_set: Add sweeper concurrency --- ...resource_aws_waf_regex_pattern_set_test.go | 79 +++++++++++-------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/aws/resource_aws_waf_regex_pattern_set_test.go b/aws/resource_aws_waf_regex_pattern_set_test.go index cf60a8407ed..1e7f177fba0 100644 --- a/aws/resource_aws_waf_regex_pattern_set_test.go +++ b/aws/resource_aws_waf_regex_pattern_set_test.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "regexp" + "sync" "testing" "github.com/aws/aws-sdk-go/aws" @@ -29,65 +30,79 @@ func init() { func testSweepWafRegexPatternSet(region string) error { client, err := sharedClientForRegion(region) + if err != nil { return fmt.Errorf("error getting client: %s", err) } - conn := client.(*AWSClient).wafconn - var sweeperErrs *multierror.Error + conn := client.(*AWSClient).wafconn + sweepResources := make([]*testSweepResource, 0) + var errs *multierror.Error + var g multierror.Group + var mutex = &sync.Mutex{} input := &waf.ListRegexPatternSetsInput{} - err = lister.ListRegexPatternSetsPages(conn, input, func(page *waf.ListRegexPatternSetsOutput, lastPage bool) bool { + err = lister.ListRegexPatternSetsPages(conn, input, func(page *waf.ListRegexPatternSetsOutput, isLast bool) bool { if page == nil { - return !lastPage + return !isLast } for _, regexPatternSet := range page.RegexPatternSets { - id := aws.StringValue(regexPatternSet.RegexPatternSetId) - r := resourceAwsWafRegexPatternSet() d := r.Data(nil) + + id := aws.StringValue(regexPatternSet.RegexPatternSetId) d.SetId(id) - // Need to Read first to fill in regex_pattern_strings attribute - err := r.Read(d, client) + // read concurrently and gather errors + g.Go(func() error { + // Need to Read first to fill in regex_pattern_strings attribute + err := r.Read(d, client) - if err != nil { - sweeperErr := fmt.Errorf("error reading WAF Regex Pattern Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + if err != nil { + sweeperErr := fmt.Errorf("error reading WAF Regex Pattern Set (%s): %w", id, err) + log.Printf("[ERROR] %s", sweeperErr) + return sweeperErr + } - // In case it was already deleted - if d.Id() == "" { - continue - } + // In case it was already deleted + if d.Id() == "" { + return nil + } - err = r.Delete(d, client) + mutex.Lock() + defer mutex.Unlock() + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) - if err != nil { - sweeperErr := fmt.Errorf("error deleting WAF Regex Pattern Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + return nil + }) } - return !lastPage + return !isLast }) - if testSweepSkipSweepError(err) { - log.Printf("[WARN] Skipping WAF Regex Pattern Set sweep for %s: %s", region, err) - return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("error listing WAF Regex Pattern Set for %s: %w", region, err)) } - if err != nil { - sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF Regex Pattern Sets: %w", err)) + if err = g.Wait().ErrorOrNil(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Regex Pattern Sets: %w", err)) + } + + if len(sweepResources) > 0 { + // Any errors didn't prevent gathering some sweeping work, so do it. + if err := testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Regex Pattern Set for %s: %w", region, err)) + } + } + + if testSweepSkipSweepError(errs.ErrorOrNil()) { + log.Printf("[WARN] Skipping WAF Regex Pattern Set sweep for %s: %s", region, errs) + return nil } - return sweeperErrs.ErrorOrNil() + return errs.ErrorOrNil() } // Serialized acceptance tests due to WAF account limits From c0f254f0373af60263ac81f7127e9df0a2cd6712 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 15:14:46 -0400 Subject: [PATCH 10/18] tests/r/waf_rule_group: Add sweeper concurrency --- aws/resource_aws_waf_rule_group_test.go | 79 +++++++++++++++---------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/aws/resource_aws_waf_rule_group_test.go b/aws/resource_aws_waf_rule_group_test.go index 3e76773659d..172037f9dbb 100644 --- a/aws/resource_aws_waf_rule_group_test.go +++ b/aws/resource_aws_waf_rule_group_test.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "regexp" + "sync" "testing" "github.com/aws/aws-sdk-go/aws" @@ -28,65 +29,79 @@ func init() { func testSweepWafRuleGroups(region string) error { client, err := sharedClientForRegion(region) + if err != nil { return fmt.Errorf("error getting client: %s", err) } - conn := client.(*AWSClient).wafconn - var sweeperErrs *multierror.Error + conn := client.(*AWSClient).wafconn + sweepResources := make([]*testSweepResource, 0) + var errs *multierror.Error + var g multierror.Group + var mutex = &sync.Mutex{} input := &waf.ListRuleGroupsInput{} - err = lister.ListRuleGroupsPages(conn, input, func(page *waf.ListRuleGroupsOutput, lastPage bool) bool { + err = lister.ListRuleGroupsPages(conn, input, func(page *waf.ListRuleGroupsOutput, isLast bool) bool { if page == nil { - return !lastPage + return !isLast } for _, ruleGroup := range page.RuleGroups { - id := aws.StringValue(ruleGroup.RuleGroupId) - r := resourceAwsWafRuleGroup() d := r.Data(nil) + + id := aws.StringValue(ruleGroup.RuleGroupId) d.SetId(id) - // Need to Read first to fill in activated_rule attribute - err := r.Read(d, client) + // read concurrently and gather errors + g.Go(func() error { + // Need to Read first to fill in activated_rule attribute + err := r.Read(d, client) - if err != nil { - sweeperErr := fmt.Errorf("error reading WAF Rule Group (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + if err != nil { + sweeperErr := fmt.Errorf("error reading WAF Rule Group (%s): %w", id, err) + log.Printf("[ERROR] %s", sweeperErr) + return sweeperErr + } - // In case it was already deleted - if d.Id() == "" { - continue - } + // In case it was already deleted + if d.Id() == "" { + return nil + } - err = r.Delete(d, client) + mutex.Lock() + defer mutex.Unlock() + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) - if err != nil { - sweeperErr := fmt.Errorf("error deleting WAF Rule Group (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + return nil + }) } - return !lastPage + return !isLast }) - if testSweepSkipSweepError(err) { - log.Printf("[WARN] Skipping WAF Rule Group sweep for %s: %s", region, err) - return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("error listing WAF Rule Group for %s: %w", region, err)) } - if err != nil { - sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF Rule Groups: %w", err)) + if err = g.Wait().ErrorOrNil(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Rule Groups: %w", err)) + } + + if len(sweepResources) > 0 { + // Any errors didn't prevent gathering some sweeping work, so do it. + if err := testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Rule Group for %s: %w", region, err)) + } + } + + if testSweepSkipSweepError(errs.ErrorOrNil()) { + log.Printf("[WARN] Skipping WAF Rule Group sweep for %s: %s", region, errs) + return nil } - return sweeperErrs.ErrorOrNil() + return errs.ErrorOrNil() } func TestAccAWSWafRuleGroup_basic(t *testing.T) { From 4d59d821b4026bb47fba0ea75e8d5405705a6962 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 15:18:30 -0400 Subject: [PATCH 11/18] tests/r/waf_rule: Add sweeper concurrency --- aws/resource_aws_waf_rule_test.go | 79 ++++++++++++++++++------------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/aws/resource_aws_waf_rule_test.go b/aws/resource_aws_waf_rule_test.go index 2e31b441277..be84290e4b2 100644 --- a/aws/resource_aws_waf_rule_test.go +++ b/aws/resource_aws_waf_rule_test.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "regexp" + "sync" "testing" "github.com/aws/aws-sdk-go/aws" @@ -29,18 +30,22 @@ func init() { func testSweepWafRules(region string) error { client, err := sharedClientForRegion(region) + if err != nil { return fmt.Errorf("error getting client: %s", err) } - conn := client.(*AWSClient).wafconn - var sweeperErrs *multierror.Error + conn := client.(*AWSClient).wafconn + sweepResources := make([]*testSweepResource, 0) + var errs *multierror.Error + var g multierror.Group + var mutex = &sync.Mutex{} input := &waf.ListRulesInput{} - err = lister.ListRulesPages(conn, input, func(page *waf.ListRulesOutput, lastPage bool) bool { + err = lister.ListRulesPages(conn, input, func(page *waf.ListRulesOutput, isLast bool) bool { if page == nil { - return !lastPage + return !isLast } for _, rule := range page.Rules { @@ -48,50 +53,60 @@ func testSweepWafRules(region string) error { continue } - id := aws.StringValue(rule.RuleId) - r := resourceAwsWafRule() d := r.Data(nil) + + id := aws.StringValue(rule.RuleId) d.SetId(id) - // Need to Read first to fill in predicates attribute - err := r.Read(d, client) + // read concurrently and gather errors + g.Go(func() error { + // Need to Read first to fill in predicates attribute + err := r.Read(d, client) - if err != nil { - sweeperErr := fmt.Errorf("error reading WAF Rule (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + if err != nil { + sweeperErr := fmt.Errorf("error reading WAF Rule (%s): %w", id, err) + log.Printf("[ERROR] %s", sweeperErr) + return sweeperErr + } - // In case it was already deleted - if d.Id() == "" { - continue - } + // In case it was already deleted + if d.Id() == "" { + return nil + } - err = r.Delete(d, client) + mutex.Lock() + defer mutex.Unlock() + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) - if err != nil { - sweeperErr := fmt.Errorf("error deleting WAF Rule (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + return nil + }) } - return !lastPage + return !isLast }) - if testSweepSkipSweepError(err) { - log.Printf("[WARN] Skipping WAF Rule sweep for %s: %s", region, err) - return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("error listing WAF Rules for %s: %w", region, err)) } - if err != nil { - sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF Rules: %w", err)) + if err = g.Wait().ErrorOrNil(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Rules: %w", err)) + } + + if len(sweepResources) > 0 { + // Any errors didn't prevent gathering some sweeping work, so do it. + if err := testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Rules for %s: %w", region, err)) + } + } + + if testSweepSkipSweepError(errs.ErrorOrNil()) { + log.Printf("[WARN] Skipping WAF Rule sweep for %s: %s", region, errs) + return nil } - return sweeperErrs.ErrorOrNil() + return errs.ErrorOrNil() } func TestAccAWSWafRule_basic(t *testing.T) { From c572b505a2558609610224c73a93ca2f0a947deb Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 15:22:01 -0400 Subject: [PATCH 12/18] tests/r/waf_size_constraint_set: Add sweeper concurrency --- ...source_aws_waf_size_constraint_set_test.go | 79 +++++++++++-------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/aws/resource_aws_waf_size_constraint_set_test.go b/aws/resource_aws_waf_size_constraint_set_test.go index b577de19ad6..8a5f034e988 100644 --- a/aws/resource_aws_waf_size_constraint_set_test.go +++ b/aws/resource_aws_waf_size_constraint_set_test.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "regexp" + "sync" "testing" "github.com/aws/aws-sdk-go/aws" @@ -30,65 +31,79 @@ func init() { func testSweepWafSizeConstraintSet(region string) error { client, err := sharedClientForRegion(region) + if err != nil { return fmt.Errorf("error getting client: %s", err) } - conn := client.(*AWSClient).wafconn - var sweeperErrs *multierror.Error + conn := client.(*AWSClient).wafconn + sweepResources := make([]*testSweepResource, 0) + var errs *multierror.Error + var g multierror.Group + var mutex = &sync.Mutex{} input := &waf.ListSizeConstraintSetsInput{} - err = lister.ListSizeConstraintSetsPages(conn, input, func(page *waf.ListSizeConstraintSetsOutput, lastPage bool) bool { + err = lister.ListSizeConstraintSetsPages(conn, input, func(page *waf.ListSizeConstraintSetsOutput, isLast bool) bool { if page == nil { - return !lastPage + return !isLast } for _, sizeConstraintSet := range page.SizeConstraintSets { - id := aws.StringValue(sizeConstraintSet.SizeConstraintSetId) - r := resourceAwsWafSizeConstraintSet() d := r.Data(nil) + + id := aws.StringValue(sizeConstraintSet.SizeConstraintSetId) d.SetId(id) - // Need to Read first to fill in size_constraints attribute - err := r.Read(d, client) + // read concurrently and gather errors + g.Go(func() error { + // Need to Read first to fill in size_constraints attribute + err := r.Read(d, client) - if err != nil { - sweeperErr := fmt.Errorf("error reading WAF Size Constraint Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + if err != nil { + sweeperErr := fmt.Errorf("error reading WAF Size Constraint Set (%s): %w", id, err) + log.Printf("[ERROR] %s", sweeperErr) + return sweeperErr + } - // In case it was already deleted - if d.Id() == "" { - continue - } + // In case it was already deleted + if d.Id() == "" { + return nil + } - err = r.Delete(d, client) + mutex.Lock() + defer mutex.Unlock() + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) - if err != nil { - sweeperErr := fmt.Errorf("error deleting WAF Size Constraint Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + return nil + }) } - return !lastPage + return !isLast }) - if testSweepSkipSweepError(err) { - log.Printf("[WARN] Skipping WAF Size Constraint Set sweep for %s: %s", region, err) - return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("error listing WAF Size Constraint Sets for %s: %w", region, err)) } - if err != nil { - sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF Size Constraint Sets: %w", err)) + if err = g.Wait().ErrorOrNil(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Size Constraint Sets: %w", err)) + } + + if len(sweepResources) > 0 { + // Any errors didn't prevent gathering some sweeping work, so do it. + if err := testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Size Constraint Sets for %s: %w", region, err)) + } + } + + if testSweepSkipSweepError(errs.ErrorOrNil()) { + log.Printf("[WARN] Skipping WAF Size Constraint Set sweep for %s: %s", region, errs) + return nil } - return sweeperErrs.ErrorOrNil() + return errs.ErrorOrNil() } func TestAccAWSWafSizeConstraintSet_basic(t *testing.T) { From 2404a91655f507c5b9535906f4fb75b8b57b5d84 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 15:25:24 -0400 Subject: [PATCH 13/18] tests/r/waf_sql_injection_match_set: Add sweeper concurrency --- ...ce_aws_waf_sql_injection_match_set_test.go | 79 +++++++++++-------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/aws/resource_aws_waf_sql_injection_match_set_test.go b/aws/resource_aws_waf_sql_injection_match_set_test.go index 9b007fe1384..6da0facdf8e 100644 --- a/aws/resource_aws_waf_sql_injection_match_set_test.go +++ b/aws/resource_aws_waf_sql_injection_match_set_test.go @@ -3,6 +3,7 @@ package aws import ( "fmt" "log" + "sync" "testing" "github.com/aws/aws-sdk-go/aws" @@ -28,65 +29,79 @@ func init() { func testSweepWafSqlInjectionMatchSet(region string) error { client, err := sharedClientForRegion(region) + if err != nil { return fmt.Errorf("error getting client: %s", err) } - conn := client.(*AWSClient).wafconn - var sweeperErrs *multierror.Error + conn := client.(*AWSClient).wafconn + sweepResources := make([]*testSweepResource, 0) + var errs *multierror.Error + var g multierror.Group + var mutex = &sync.Mutex{} input := &waf.ListSqlInjectionMatchSetsInput{} - err = lister.ListSqlInjectionMatchSetsPages(conn, input, func(page *waf.ListSqlInjectionMatchSetsOutput, lastPage bool) bool { + err = lister.ListSqlInjectionMatchSetsPages(conn, input, func(page *waf.ListSqlInjectionMatchSetsOutput, isLast bool) bool { if page == nil { - return !lastPage + return !isLast } for _, sqlInjectionMatchSet := range page.SqlInjectionMatchSets { - id := aws.StringValue(sqlInjectionMatchSet.SqlInjectionMatchSetId) - r := resourceAwsWafSqlInjectionMatchSet() d := r.Data(nil) + + id := aws.StringValue(sqlInjectionMatchSet.SqlInjectionMatchSetId) d.SetId(id) - // Need to Read first to fill in sql_injection_match_tuples attribute - err := r.Read(d, client) + // read concurrently and gather errors + g.Go(func() error { + // Need to Read first to fill in sql_injection_match_tuples attribute + err := r.Read(d, client) - if err != nil { - sweeperErr := fmt.Errorf("error reading WAF SQL Injection Match Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + if err != nil { + sweeperErr := fmt.Errorf("error reading WAF SQL Injection Match Set (%s): %w", id, err) + log.Printf("[ERROR] %s", sweeperErr) + return sweeperErr + } - // In case it was already deleted - if d.Id() == "" { - continue - } + // In case it was already deleted + if d.Id() == "" { + return nil + } - err = r.Delete(d, client) + mutex.Lock() + defer mutex.Unlock() + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) - if err != nil { - sweeperErr := fmt.Errorf("error deleting WAF SQL Injection Match Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + return nil + }) } - return !lastPage + return !isLast }) - if testSweepSkipSweepError(err) { - log.Printf("[WARN] Skipping WAF SQL Injection Match Set sweep for %s: %s", region, err) - return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("error listing WAF SQL Injection Matches for %s: %w", region, err)) } - if err != nil { - sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF SQL Injection Match Sets: %w", err)) + if err = g.Wait().ErrorOrNil(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF SQL Injection Matches: %w", err)) + } + + if len(sweepResources) > 0 { + // Any errors didn't prevent gathering some sweeping work, so do it. + if err := testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF SQL Injection Matches for %s: %w", region, err)) + } + } + + if testSweepSkipSweepError(errs.ErrorOrNil()) { + log.Printf("[WARN] Skipping WAF SQL Injection Match sweep for %s: %s", region, errs) + return nil } - return sweeperErrs.ErrorOrNil() + return errs.ErrorOrNil() } func TestAccAWSWafSqlInjectionMatchSet_basic(t *testing.T) { From e4f592c493c6144bdefe7c1edf5a473680dab2c8 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 15:29:18 -0400 Subject: [PATCH 14/18] tests/r/waf_web_acl: Add read/sweeper concurrency --- aws/resource_aws_waf_web_acl_test.go | 55 ++++++++++++++++++---------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/aws/resource_aws_waf_web_acl_test.go b/aws/resource_aws_waf_web_acl_test.go index 53bcd8b5a15..9f7754a8327 100644 --- a/aws/resource_aws_waf_web_acl_test.go +++ b/aws/resource_aws_waf_web_acl_test.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "regexp" + "sync" "testing" "github.com/aws/aws-sdk-go/aws" @@ -33,11 +34,14 @@ func testSweepWafWebAcls(region string) error { conn := client.(*AWSClient).wafconn sweepResources := make([]*testSweepResource, 0) var errs *multierror.Error + var g multierror.Group + var mutex = &sync.Mutex{} + input := &waf.ListWebACLsInput{} - err = lister.ListWebACLsPages(conn, input, func(page *waf.ListWebACLsOutput, lastPage bool) bool { + err = lister.ListWebACLsPages(conn, input, func(page *waf.ListWebACLsOutput, isLast bool) bool { if page == nil { - return !lastPage + return !isLast } for _, webACL := range page.WebACLs { @@ -51,33 +55,46 @@ func testSweepWafWebAcls(region string) error { id := aws.StringValue(webACL.WebACLId) d.SetId(id) - // Need to Read first to fill in rules argument - err := r.Read(d, client) + // read concurrently and gather errors + g.Go(func() error { + // Need to Read first to fill in rules argument + err := r.Read(d, client) - if err != nil { - readErr := fmt.Errorf("error reading WAF Web ACL (%s): %w", id, err) - log.Printf("[ERROR] %s", readErr) - errs = multierror.Append(errs, readErr) - continue - } + if err != nil { + readErr := fmt.Errorf("error reading WAF Web ACL (%s): %w", id, err) + log.Printf("[ERROR] %s", readErr) + return readErr + } - // In case it was already deleted - if d.Id() == "" { - continue - } + // In case it was already deleted + if d.Id() == "" { + return nil + } - sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) + mutex.Lock() + defer mutex.Unlock() + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) + + return nil + }) } - return !lastPage + return !isLast }) if err != nil { - errs = multierror.Append(errs, fmt.Errorf("error describing WAF Web ACLs: %w", err)) + errs = multierror.Append(errs, fmt.Errorf("error listing WAF Web ACLs for %s: %w", region, err)) } - if err = testSweepResourceOrchestrator(sweepResources); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Web ACL for %s: %w", region, err)) + if err = g.Wait().ErrorOrNil(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Web ACLs: %w", err)) + } + + if len(sweepResources) > 0 { + // Any errors didn't prevent gathering some sweeping work, so do it. + if err := testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Web ACLs for %s: %w", region, err)) + } } if testSweepSkipSweepError(errs.ErrorOrNil()) { From 1528e3f88dc1acd31d36e65175350f5f1a2ed1d6 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 15:32:19 -0400 Subject: [PATCH 15/18] tests/r/waf_xss_match_set: Add sweeper concurrency --- aws/resource_aws_waf_xss_match_set_test.go | 79 +++++++++++++--------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/aws/resource_aws_waf_xss_match_set_test.go b/aws/resource_aws_waf_xss_match_set_test.go index ec2a494f545..2eeda67398b 100644 --- a/aws/resource_aws_waf_xss_match_set_test.go +++ b/aws/resource_aws_waf_xss_match_set_test.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "regexp" + "sync" "testing" "github.com/aws/aws-sdk-go/aws" @@ -29,65 +30,79 @@ func init() { func testSweepWafXssMatchSet(region string) error { client, err := sharedClientForRegion(region) + if err != nil { return fmt.Errorf("error getting client: %s", err) } - conn := client.(*AWSClient).wafconn - var sweeperErrs *multierror.Error + conn := client.(*AWSClient).wafconn + sweepResources := make([]*testSweepResource, 0) + var errs *multierror.Error + var g multierror.Group + var mutex = &sync.Mutex{} input := &waf.ListXssMatchSetsInput{} - err = lister.ListXssMatchSetsPages(conn, input, func(page *waf.ListXssMatchSetsOutput, lastPage bool) bool { + err = lister.ListXssMatchSetsPages(conn, input, func(page *waf.ListXssMatchSetsOutput, isLast bool) bool { if page == nil { - return !lastPage + return !isLast } for _, xssMatchSet := range page.XssMatchSets { - id := aws.StringValue(xssMatchSet.XssMatchSetId) - r := resourceAwsWafXssMatchSet() d := r.Data(nil) + + id := aws.StringValue(xssMatchSet.XssMatchSetId) d.SetId(id) - // Need to Read first to fill in xss_match_tuples attribute - err := r.Read(d, client) + // read concurrently and gather errors + g.Go(func() error { + // Need to Read first to fill in xss_match_tuples attribute + err := r.Read(d, client) - if err != nil { - sweeperErr := fmt.Errorf("error reading WAF XSS Match Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + if err != nil { + sweeperErr := fmt.Errorf("error reading WAF XSS Match Set (%s): %w", id, err) + log.Printf("[ERROR] %s", sweeperErr) + return sweeperErr + } - // In case it was already deleted - if d.Id() == "" { - continue - } + // In case it was already deleted + if d.Id() == "" { + return nil + } - err = r.Delete(d, client) + mutex.Lock() + defer mutex.Unlock() + sweepResources = append(sweepResources, NewTestSweepResource(r, d, client)) - if err != nil { - sweeperErr := fmt.Errorf("error deleting WAF XSS Match Set (%s): %w", id, err) - log.Printf("[ERROR] %s", sweeperErr) - sweeperErrs = multierror.Append(sweeperErrs, sweeperErr) - continue - } + return nil + }) } - return !lastPage + return !isLast }) - if testSweepSkipSweepError(err) { - log.Printf("[WARN] Skipping WAF XSS Match Set sweep for %s: %s", region, err) - return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("error listing WAF XSS Match Sets for %s: %w", region, err)) } - if err != nil { - sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF XSS Match Sets: %w", err)) + if err = g.Wait().ErrorOrNil(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF XSS Match Sets: %w", err)) + } + + if len(sweepResources) > 0 { + // Any errors didn't prevent gathering some sweeping work, so do it. + if err := testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF XSS Match Sets for %s: %w", region, err)) + } + } + + if testSweepSkipSweepError(errs.ErrorOrNil()) { + log.Printf("[WARN] Skipping WAF XSS Match Set sweep for %s: %s", region, errs) + return nil } - return sweeperErrs.ErrorOrNil() + return errs.ErrorOrNil() } func TestAccAWSWafXssMatchSet_basic(t *testing.T) { From 482f2d13952a4d2133d8bdbf13a72f658b14e32b Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 15:35:19 -0400 Subject: [PATCH 16/18] tests/r/waf: Consistent var list --- aws/resource_aws_waf_byte_match_set_test.go | 5 ++--- aws/resource_aws_waf_geo_match_set_test.go | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/aws/resource_aws_waf_byte_match_set_test.go b/aws/resource_aws_waf_byte_match_set_test.go index 99f3295bd60..607c020ada5 100644 --- a/aws/resource_aws_waf_byte_match_set_test.go +++ b/aws/resource_aws_waf_byte_match_set_test.go @@ -38,12 +38,11 @@ func testSweepWafByteMatchSet(region string) error { conn := client.(*AWSClient).wafconn sweepResources := make([]*testSweepResource, 0) var errs *multierror.Error - - input := &waf.ListByteMatchSetsInput{} - var g multierror.Group var mutex = &sync.Mutex{} + input := &waf.ListByteMatchSetsInput{} + err = lister.ListByteMatchSetsPages(conn, input, func(page *waf.ListByteMatchSetsOutput, isLast bool) bool { if page == nil { return !isLast diff --git a/aws/resource_aws_waf_geo_match_set_test.go b/aws/resource_aws_waf_geo_match_set_test.go index 2c8b7ff3fb3..6a1a7d91680 100644 --- a/aws/resource_aws_waf_geo_match_set_test.go +++ b/aws/resource_aws_waf_geo_match_set_test.go @@ -38,12 +38,11 @@ func testSweepWafGeoMatchSet(region string) error { conn := client.(*AWSClient).wafconn sweepResources := make([]*testSweepResource, 0) var errs *multierror.Error - - input := &waf.ListGeoMatchSetsInput{} - var g multierror.Group var mutex = &sync.Mutex{} + input := &waf.ListGeoMatchSetsInput{} + err = lister.ListGeoMatchSetsPages(conn, input, func(page *waf.ListGeoMatchSetsOutput, isLast bool) bool { if page == nil { return !isLast From d216e99ebe41c2b187f818b6ecc15a4b509b0d8d Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Wed, 7 Apr 2021 15:52:14 -0400 Subject: [PATCH 17/18] tests/r/waf: Use consistent var name --- aws/resource_aws_waf_byte_match_set_test.go | 6 +++--- aws/resource_aws_waf_geo_match_set_test.go | 6 +++--- aws/resource_aws_waf_ipset_test.go | 6 +++--- aws/resource_aws_waf_rate_based_rule_test.go | 6 +++--- aws/resource_aws_waf_regex_match_set_test.go | 6 +++--- aws/resource_aws_waf_regex_pattern_set_test.go | 6 +++--- aws/resource_aws_waf_rule_group_test.go | 6 +++--- aws/resource_aws_waf_rule_test.go | 6 +++--- aws/resource_aws_waf_size_constraint_set_test.go | 6 +++--- aws/resource_aws_waf_sql_injection_match_set_test.go | 6 +++--- aws/resource_aws_waf_web_acl_test.go | 6 +++--- aws/resource_aws_waf_xss_match_set_test.go | 6 +++--- 12 files changed, 36 insertions(+), 36 deletions(-) diff --git a/aws/resource_aws_waf_byte_match_set_test.go b/aws/resource_aws_waf_byte_match_set_test.go index 607c020ada5..463a3e812b0 100644 --- a/aws/resource_aws_waf_byte_match_set_test.go +++ b/aws/resource_aws_waf_byte_match_set_test.go @@ -43,9 +43,9 @@ func testSweepWafByteMatchSet(region string) error { input := &waf.ListByteMatchSetsInput{} - err = lister.ListByteMatchSetsPages(conn, input, func(page *waf.ListByteMatchSetsOutput, isLast bool) bool { + err = lister.ListByteMatchSetsPages(conn, input, func(page *waf.ListByteMatchSetsOutput, lastPage bool) bool { if page == nil { - return !isLast + return !lastPage } for _, byteMatchSet := range page.ByteMatchSets { @@ -79,7 +79,7 @@ func testSweepWafByteMatchSet(region string) error { }) } - return !isLast + return !lastPage }) if err != nil { diff --git a/aws/resource_aws_waf_geo_match_set_test.go b/aws/resource_aws_waf_geo_match_set_test.go index 6a1a7d91680..ce47ef4b507 100644 --- a/aws/resource_aws_waf_geo_match_set_test.go +++ b/aws/resource_aws_waf_geo_match_set_test.go @@ -43,9 +43,9 @@ func testSweepWafGeoMatchSet(region string) error { input := &waf.ListGeoMatchSetsInput{} - err = lister.ListGeoMatchSetsPages(conn, input, func(page *waf.ListGeoMatchSetsOutput, isLast bool) bool { + err = lister.ListGeoMatchSetsPages(conn, input, func(page *waf.ListGeoMatchSetsOutput, lastPage bool) bool { if page == nil { - return !isLast + return !lastPage } for _, geoMatchSet := range page.GeoMatchSets { @@ -79,7 +79,7 @@ func testSweepWafGeoMatchSet(region string) error { }) } - return !isLast + return !lastPage }) if err != nil { diff --git a/aws/resource_aws_waf_ipset_test.go b/aws/resource_aws_waf_ipset_test.go index eba5cbd863a..c8dd7d92320 100644 --- a/aws/resource_aws_waf_ipset_test.go +++ b/aws/resource_aws_waf_ipset_test.go @@ -46,9 +46,9 @@ func testSweepWafIPSet(region string) error { input := &waf.ListIPSetsInput{} - err = lister.ListIPSetsPages(conn, input, func(page *waf.ListIPSetsOutput, isLast bool) bool { + err = lister.ListIPSetsPages(conn, input, func(page *waf.ListIPSetsOutput, lastPage bool) bool { if page == nil { - return !isLast + return !lastPage } for _, ipSet := range page.IPSets { @@ -82,7 +82,7 @@ func testSweepWafIPSet(region string) error { }) } - return !isLast + return !lastPage }) if err != nil { diff --git a/aws/resource_aws_waf_rate_based_rule_test.go b/aws/resource_aws_waf_rate_based_rule_test.go index 34f1bafc472..a298aa34745 100644 --- a/aws/resource_aws_waf_rate_based_rule_test.go +++ b/aws/resource_aws_waf_rate_based_rule_test.go @@ -43,9 +43,9 @@ func testSweepWafRateBasedRules(region string) error { input := &waf.ListRateBasedRulesInput{} - err = lister.ListRateBasedRulesPages(conn, input, func(page *waf.ListRateBasedRulesOutput, isLast bool) bool { + err = lister.ListRateBasedRulesPages(conn, input, func(page *waf.ListRateBasedRulesOutput, lastPage bool) bool { if page == nil { - return !isLast + return !lastPage } for _, rule := range page.Rules { @@ -79,7 +79,7 @@ func testSweepWafRateBasedRules(region string) error { }) } - return !isLast + return !lastPage }) if err != nil { diff --git a/aws/resource_aws_waf_regex_match_set_test.go b/aws/resource_aws_waf_regex_match_set_test.go index 54769e0146d..6dc4dfff1eb 100644 --- a/aws/resource_aws_waf_regex_match_set_test.go +++ b/aws/resource_aws_waf_regex_match_set_test.go @@ -43,9 +43,9 @@ func testSweepWafRegexMatchSet(region string) error { input := &waf.ListRegexMatchSetsInput{} - err = lister.ListRegexMatchSetsPages(conn, input, func(page *waf.ListRegexMatchSetsOutput, isLast bool) bool { + err = lister.ListRegexMatchSetsPages(conn, input, func(page *waf.ListRegexMatchSetsOutput, lastPage bool) bool { if page == nil { - return !isLast + return !lastPage } for _, regexMatchSet := range page.RegexMatchSets { @@ -79,7 +79,7 @@ func testSweepWafRegexMatchSet(region string) error { }) } - return !isLast + return !lastPage }) if err != nil { diff --git a/aws/resource_aws_waf_regex_pattern_set_test.go b/aws/resource_aws_waf_regex_pattern_set_test.go index 1e7f177fba0..a9a536f8819 100644 --- a/aws/resource_aws_waf_regex_pattern_set_test.go +++ b/aws/resource_aws_waf_regex_pattern_set_test.go @@ -43,9 +43,9 @@ func testSweepWafRegexPatternSet(region string) error { input := &waf.ListRegexPatternSetsInput{} - err = lister.ListRegexPatternSetsPages(conn, input, func(page *waf.ListRegexPatternSetsOutput, isLast bool) bool { + err = lister.ListRegexPatternSetsPages(conn, input, func(page *waf.ListRegexPatternSetsOutput, lastPage bool) bool { if page == nil { - return !isLast + return !lastPage } for _, regexPatternSet := range page.RegexPatternSets { @@ -79,7 +79,7 @@ func testSweepWafRegexPatternSet(region string) error { }) } - return !isLast + return !lastPage }) if err != nil { diff --git a/aws/resource_aws_waf_rule_group_test.go b/aws/resource_aws_waf_rule_group_test.go index 172037f9dbb..249d45006ee 100644 --- a/aws/resource_aws_waf_rule_group_test.go +++ b/aws/resource_aws_waf_rule_group_test.go @@ -42,9 +42,9 @@ func testSweepWafRuleGroups(region string) error { input := &waf.ListRuleGroupsInput{} - err = lister.ListRuleGroupsPages(conn, input, func(page *waf.ListRuleGroupsOutput, isLast bool) bool { + err = lister.ListRuleGroupsPages(conn, input, func(page *waf.ListRuleGroupsOutput, lastPage bool) bool { if page == nil { - return !isLast + return !lastPage } for _, ruleGroup := range page.RuleGroups { @@ -78,7 +78,7 @@ func testSweepWafRuleGroups(region string) error { }) } - return !isLast + return !lastPage }) if err != nil { diff --git a/aws/resource_aws_waf_rule_test.go b/aws/resource_aws_waf_rule_test.go index be84290e4b2..624f38f67c4 100644 --- a/aws/resource_aws_waf_rule_test.go +++ b/aws/resource_aws_waf_rule_test.go @@ -43,9 +43,9 @@ func testSweepWafRules(region string) error { input := &waf.ListRulesInput{} - err = lister.ListRulesPages(conn, input, func(page *waf.ListRulesOutput, isLast bool) bool { + err = lister.ListRulesPages(conn, input, func(page *waf.ListRulesOutput, lastPage bool) bool { if page == nil { - return !isLast + return !lastPage } for _, rule := range page.Rules { @@ -83,7 +83,7 @@ func testSweepWafRules(region string) error { }) } - return !isLast + return !lastPage }) if err != nil { diff --git a/aws/resource_aws_waf_size_constraint_set_test.go b/aws/resource_aws_waf_size_constraint_set_test.go index 8a5f034e988..2e8fbcbcf05 100644 --- a/aws/resource_aws_waf_size_constraint_set_test.go +++ b/aws/resource_aws_waf_size_constraint_set_test.go @@ -44,9 +44,9 @@ func testSweepWafSizeConstraintSet(region string) error { input := &waf.ListSizeConstraintSetsInput{} - err = lister.ListSizeConstraintSetsPages(conn, input, func(page *waf.ListSizeConstraintSetsOutput, isLast bool) bool { + err = lister.ListSizeConstraintSetsPages(conn, input, func(page *waf.ListSizeConstraintSetsOutput, lastPage bool) bool { if page == nil { - return !isLast + return !lastPage } for _, sizeConstraintSet := range page.SizeConstraintSets { @@ -80,7 +80,7 @@ func testSweepWafSizeConstraintSet(region string) error { }) } - return !isLast + return !lastPage }) if err != nil { diff --git a/aws/resource_aws_waf_sql_injection_match_set_test.go b/aws/resource_aws_waf_sql_injection_match_set_test.go index 6da0facdf8e..0ef056b8139 100644 --- a/aws/resource_aws_waf_sql_injection_match_set_test.go +++ b/aws/resource_aws_waf_sql_injection_match_set_test.go @@ -42,9 +42,9 @@ func testSweepWafSqlInjectionMatchSet(region string) error { input := &waf.ListSqlInjectionMatchSetsInput{} - err = lister.ListSqlInjectionMatchSetsPages(conn, input, func(page *waf.ListSqlInjectionMatchSetsOutput, isLast bool) bool { + err = lister.ListSqlInjectionMatchSetsPages(conn, input, func(page *waf.ListSqlInjectionMatchSetsOutput, lastPage bool) bool { if page == nil { - return !isLast + return !lastPage } for _, sqlInjectionMatchSet := range page.SqlInjectionMatchSets { @@ -78,7 +78,7 @@ func testSweepWafSqlInjectionMatchSet(region string) error { }) } - return !isLast + return !lastPage }) if err != nil { diff --git a/aws/resource_aws_waf_web_acl_test.go b/aws/resource_aws_waf_web_acl_test.go index 9f7754a8327..17cf50095b3 100644 --- a/aws/resource_aws_waf_web_acl_test.go +++ b/aws/resource_aws_waf_web_acl_test.go @@ -39,9 +39,9 @@ func testSweepWafWebAcls(region string) error { input := &waf.ListWebACLsInput{} - err = lister.ListWebACLsPages(conn, input, func(page *waf.ListWebACLsOutput, isLast bool) bool { + err = lister.ListWebACLsPages(conn, input, func(page *waf.ListWebACLsOutput, lastPage bool) bool { if page == nil { - return !isLast + return !lastPage } for _, webACL := range page.WebACLs { @@ -79,7 +79,7 @@ func testSweepWafWebAcls(region string) error { }) } - return !isLast + return !lastPage }) if err != nil { diff --git a/aws/resource_aws_waf_xss_match_set_test.go b/aws/resource_aws_waf_xss_match_set_test.go index 2eeda67398b..81697a3606d 100644 --- a/aws/resource_aws_waf_xss_match_set_test.go +++ b/aws/resource_aws_waf_xss_match_set_test.go @@ -43,9 +43,9 @@ func testSweepWafXssMatchSet(region string) error { input := &waf.ListXssMatchSetsInput{} - err = lister.ListXssMatchSetsPages(conn, input, func(page *waf.ListXssMatchSetsOutput, isLast bool) bool { + err = lister.ListXssMatchSetsPages(conn, input, func(page *waf.ListXssMatchSetsOutput, lastPage bool) bool { if page == nil { - return !isLast + return !lastPage } for _, xssMatchSet := range page.XssMatchSets { @@ -79,7 +79,7 @@ func testSweepWafXssMatchSet(region string) error { }) } - return !isLast + return !lastPage }) if err != nil { From ab9605e96534c55abfe1616a2b53deb5b289a9d4 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Thu, 8 Apr 2021 12:40:38 -0400 Subject: [PATCH 18/18] tests/r/waf: Remove unneeded len check --- aws/resource_aws_waf_byte_match_set_test.go | 7 ++----- aws/resource_aws_waf_geo_match_set_test.go | 7 ++----- aws/resource_aws_waf_ipset_test.go | 7 ++----- aws/resource_aws_waf_rate_based_rule_test.go | 7 ++----- aws/resource_aws_waf_regex_match_set_test.go | 7 ++----- aws/resource_aws_waf_regex_pattern_set_test.go | 7 ++----- aws/resource_aws_waf_rule_group_test.go | 7 ++----- aws/resource_aws_waf_rule_test.go | 7 ++----- aws/resource_aws_waf_size_constraint_set_test.go | 7 ++----- aws/resource_aws_waf_sql_injection_match_set_test.go | 7 ++----- aws/resource_aws_waf_web_acl_test.go | 7 ++----- aws/resource_aws_waf_xss_match_set_test.go | 7 ++----- 12 files changed, 24 insertions(+), 60 deletions(-) diff --git a/aws/resource_aws_waf_byte_match_set_test.go b/aws/resource_aws_waf_byte_match_set_test.go index 463a3e812b0..5c528bd7287 100644 --- a/aws/resource_aws_waf_byte_match_set_test.go +++ b/aws/resource_aws_waf_byte_match_set_test.go @@ -90,11 +90,8 @@ func testSweepWafByteMatchSet(region string) error { errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Byte Match Sets: %w", err)) } - if len(sweepResources) > 0 { - // Any errors didn't prevent gathering some sweeping work, so do it. - if err := testSweepResourceOrchestrator(sweepResources); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Byte Match Set for %s: %w", region, err)) - } + if err = testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Byte Match Set for %s: %w", region, err)) } if testSweepSkipSweepError(errs.ErrorOrNil()) { diff --git a/aws/resource_aws_waf_geo_match_set_test.go b/aws/resource_aws_waf_geo_match_set_test.go index ce47ef4b507..8ba529a51ba 100644 --- a/aws/resource_aws_waf_geo_match_set_test.go +++ b/aws/resource_aws_waf_geo_match_set_test.go @@ -90,11 +90,8 @@ func testSweepWafGeoMatchSet(region string) error { errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Geo Match Sets: %w", err)) } - if len(sweepResources) > 0 { - // Any errors didn't prevent gathering some sweeping work, so do it. - if err := testSweepResourceOrchestrator(sweepResources); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Geo Match Set for %s: %w", region, err)) - } + if err = testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Geo Match Set for %s: %w", region, err)) } if testSweepSkipSweepError(errs.ErrorOrNil()) { diff --git a/aws/resource_aws_waf_ipset_test.go b/aws/resource_aws_waf_ipset_test.go index c8dd7d92320..683f11a0c9c 100644 --- a/aws/resource_aws_waf_ipset_test.go +++ b/aws/resource_aws_waf_ipset_test.go @@ -93,11 +93,8 @@ func testSweepWafIPSet(region string) error { errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF IP Sets: %w", err)) } - if len(sweepResources) > 0 { - // Any errors didn't prevent gathering some sweeping work, so do it. - if err := testSweepResourceOrchestrator(sweepResources); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF IP Set for %s: %w", region, err)) - } + if err = testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF IP Set for %s: %w", region, err)) } if testSweepSkipSweepError(errs.ErrorOrNil()) { diff --git a/aws/resource_aws_waf_rate_based_rule_test.go b/aws/resource_aws_waf_rate_based_rule_test.go index a298aa34745..6f2aebe39b1 100644 --- a/aws/resource_aws_waf_rate_based_rule_test.go +++ b/aws/resource_aws_waf_rate_based_rule_test.go @@ -90,11 +90,8 @@ func testSweepWafRateBasedRules(region string) error { errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Rate Based Rules: %w", err)) } - if len(sweepResources) > 0 { - // Any errors didn't prevent gathering some sweeping work, so do it. - if err := testSweepResourceOrchestrator(sweepResources); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Rate Based Rule for %s: %w", region, err)) - } + if err = testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Rate Based Rule for %s: %w", region, err)) } if testSweepSkipSweepError(errs.ErrorOrNil()) { diff --git a/aws/resource_aws_waf_regex_match_set_test.go b/aws/resource_aws_waf_regex_match_set_test.go index 6dc4dfff1eb..e5598bf613b 100644 --- a/aws/resource_aws_waf_regex_match_set_test.go +++ b/aws/resource_aws_waf_regex_match_set_test.go @@ -90,11 +90,8 @@ func testSweepWafRegexMatchSet(region string) error { errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Regex Match Sets: %w", err)) } - if len(sweepResources) > 0 { - // Any errors didn't prevent gathering some sweeping work, so do it. - if err := testSweepResourceOrchestrator(sweepResources); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Regex Match Set for %s: %w", region, err)) - } + if err = testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Regex Match Set for %s: %w", region, err)) } if testSweepSkipSweepError(errs.ErrorOrNil()) { diff --git a/aws/resource_aws_waf_regex_pattern_set_test.go b/aws/resource_aws_waf_regex_pattern_set_test.go index a9a536f8819..0ca98ca2982 100644 --- a/aws/resource_aws_waf_regex_pattern_set_test.go +++ b/aws/resource_aws_waf_regex_pattern_set_test.go @@ -90,11 +90,8 @@ func testSweepWafRegexPatternSet(region string) error { errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Regex Pattern Sets: %w", err)) } - if len(sweepResources) > 0 { - // Any errors didn't prevent gathering some sweeping work, so do it. - if err := testSweepResourceOrchestrator(sweepResources); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Regex Pattern Set for %s: %w", region, err)) - } + if err = testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Regex Pattern Set for %s: %w", region, err)) } if testSweepSkipSweepError(errs.ErrorOrNil()) { diff --git a/aws/resource_aws_waf_rule_group_test.go b/aws/resource_aws_waf_rule_group_test.go index 249d45006ee..a61cdf80a72 100644 --- a/aws/resource_aws_waf_rule_group_test.go +++ b/aws/resource_aws_waf_rule_group_test.go @@ -89,11 +89,8 @@ func testSweepWafRuleGroups(region string) error { errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Rule Groups: %w", err)) } - if len(sweepResources) > 0 { - // Any errors didn't prevent gathering some sweeping work, so do it. - if err := testSweepResourceOrchestrator(sweepResources); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Rule Group for %s: %w", region, err)) - } + if err = testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Rule Group for %s: %w", region, err)) } if testSweepSkipSweepError(errs.ErrorOrNil()) { diff --git a/aws/resource_aws_waf_rule_test.go b/aws/resource_aws_waf_rule_test.go index 624f38f67c4..95154b35165 100644 --- a/aws/resource_aws_waf_rule_test.go +++ b/aws/resource_aws_waf_rule_test.go @@ -94,11 +94,8 @@ func testSweepWafRules(region string) error { errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Rules: %w", err)) } - if len(sweepResources) > 0 { - // Any errors didn't prevent gathering some sweeping work, so do it. - if err := testSweepResourceOrchestrator(sweepResources); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Rules for %s: %w", region, err)) - } + if err = testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Rules for %s: %w", region, err)) } if testSweepSkipSweepError(errs.ErrorOrNil()) { diff --git a/aws/resource_aws_waf_size_constraint_set_test.go b/aws/resource_aws_waf_size_constraint_set_test.go index 2e8fbcbcf05..d0a6c6b3601 100644 --- a/aws/resource_aws_waf_size_constraint_set_test.go +++ b/aws/resource_aws_waf_size_constraint_set_test.go @@ -91,11 +91,8 @@ func testSweepWafSizeConstraintSet(region string) error { errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Size Constraint Sets: %w", err)) } - if len(sweepResources) > 0 { - // Any errors didn't prevent gathering some sweeping work, so do it. - if err := testSweepResourceOrchestrator(sweepResources); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Size Constraint Sets for %s: %w", region, err)) - } + if err = testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Size Constraint Sets for %s: %w", region, err)) } if testSweepSkipSweepError(errs.ErrorOrNil()) { diff --git a/aws/resource_aws_waf_sql_injection_match_set_test.go b/aws/resource_aws_waf_sql_injection_match_set_test.go index 0ef056b8139..8c475541c0a 100644 --- a/aws/resource_aws_waf_sql_injection_match_set_test.go +++ b/aws/resource_aws_waf_sql_injection_match_set_test.go @@ -89,11 +89,8 @@ func testSweepWafSqlInjectionMatchSet(region string) error { errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF SQL Injection Matches: %w", err)) } - if len(sweepResources) > 0 { - // Any errors didn't prevent gathering some sweeping work, so do it. - if err := testSweepResourceOrchestrator(sweepResources); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF SQL Injection Matches for %s: %w", region, err)) - } + if err = testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF SQL Injection Matches for %s: %w", region, err)) } if testSweepSkipSweepError(errs.ErrorOrNil()) { diff --git a/aws/resource_aws_waf_web_acl_test.go b/aws/resource_aws_waf_web_acl_test.go index 17cf50095b3..2a151fdd23d 100644 --- a/aws/resource_aws_waf_web_acl_test.go +++ b/aws/resource_aws_waf_web_acl_test.go @@ -90,11 +90,8 @@ func testSweepWafWebAcls(region string) error { errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Web ACLs: %w", err)) } - if len(sweepResources) > 0 { - // Any errors didn't prevent gathering some sweeping work, so do it. - if err := testSweepResourceOrchestrator(sweepResources); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Web ACLs for %s: %w", region, err)) - } + if err = testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Web ACLs for %s: %w", region, err)) } if testSweepSkipSweepError(errs.ErrorOrNil()) { diff --git a/aws/resource_aws_waf_xss_match_set_test.go b/aws/resource_aws_waf_xss_match_set_test.go index 81697a3606d..e70e8fc0e7d 100644 --- a/aws/resource_aws_waf_xss_match_set_test.go +++ b/aws/resource_aws_waf_xss_match_set_test.go @@ -90,11 +90,8 @@ func testSweepWafXssMatchSet(region string) error { errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF XSS Match Sets: %w", err)) } - if len(sweepResources) > 0 { - // Any errors didn't prevent gathering some sweeping work, so do it. - if err := testSweepResourceOrchestrator(sweepResources); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF XSS Match Sets for %s: %w", region, err)) - } + if err = testSweepResourceOrchestrator(sweepResources); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF XSS Match Sets for %s: %w", region, err)) } if testSweepSkipSweepError(errs.ErrorOrNil()) {