Skip to content

Commit

Permalink
Merge pull request #18631 from hashicorp/t-concsweep-waf
Browse files Browse the repository at this point in the history 
tests/r/waf: Add sweeper concurrency
  • Loading branch information
@YakDriver
YakDriver authored Jun 17, 2021
2 parents cc10981 + ab9605e commit 87cbd86
Show file tree
Hide file tree
Showing 12 changed files with 480 additions and 334 deletions.
70 changes: 41 additions & 29 deletions aws/resource_aws_waf_byte_match_set_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package aws
import (
"fmt"
"log"
"sync"
"testing"

"github.com/aws/aws-sdk-go/aws"
Expand All @@ -29,12 +30,16 @@ func init() {

func testSweepWafByteMatchSet(region string) error {
client, err := sharedClientForRegion(region)

if err != nil {
return fmt.Errorf("error getting client: %s", err)
}
conn := client.(*AWSClient).wafconn

var sweeperErrs *multierror.Error
conn := client.(*AWSClient).wafconn
sweepResources := make([]*testSweepResource, 0)
var errs *multierror.Error
var g multierror.Group
var mutex = &sync.Mutex{}

input := &waf.ListByteMatchSetsInput{}

Expand All @@ -44,50 +49,57 @@ func testSweepWafByteMatchSet(region string) error {
}

for _, byteMatchSet := range page.ByteMatchSets {
id := aws.StringValue(byteMatchSet.ByteMatchSetId)

r := resourceAwsWafByteMatchSet()
d := r.Data(nil)

id := aws.StringValue(byteMatchSet.ByteMatchSetId)
d.SetId(id)

// Need to Read first to fill in byte_match_tuples attribute
err := r.Read(d, client)
// read concurrently and gather errors
g.Go(func() error {
// Need to Read first to fill in byte_match_tuples attribute
err := r.Read(d, client)

if err != nil {
sweeperErr := fmt.Errorf("error reading WAF Byte Match Set (%s): %w", id, err)
log.Printf("[ERROR] %s", sweeperErr)
sweeperErrs = multierror.Append(sweeperErrs, sweeperErr)
continue
}
if err != nil {
sweeperErr := fmt.Errorf("error reading WAF Byte Match Set (%s): %w", id, err)
log.Printf("[ERROR] %s", sweeperErr)
return sweeperErr
}

// In case it was already deleted
if d.Id() == "" {
continue
}
// In case it was already deleted
if d.Id() == "" {
return nil
}

err = r.Delete(d, client)
mutex.Lock()
defer mutex.Unlock()
sweepResources = append(sweepResources, NewTestSweepResource(r, d, client))

if err != nil {
sweeperErr := fmt.Errorf("error deleting WAF Byte Match Set (%s): %w", id, err)
log.Printf("[ERROR] %s", sweeperErr)
sweeperErrs = multierror.Append(sweeperErrs, sweeperErr)
continue
}
return nil
})
}

return !lastPage
})

if testSweepSkipSweepError(err) {
log.Printf("[WARN] Skipping WAF Byte Match Set sweep for %s: %s", region, err)
return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors
if err != nil {
errs = multierror.Append(errs, fmt.Errorf("error listing WAF Byte Match Set for %s: %w", region, err))
}

if err != nil {
sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF Byte Match Sets: %w", err))
if err = g.Wait().ErrorOrNil(); err != nil {
errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Byte Match Sets: %w", err))
}

if err = testSweepResourceOrchestrator(sweepResources); err != nil {
errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Byte Match Set for %s: %w", region, err))
}

if testSweepSkipSweepError(errs.ErrorOrNil()) {
log.Printf("[WARN] Skipping WAF Byte Match Set sweep for %s: %s", region, errs)
return nil
}

return sweeperErrs.ErrorOrNil()
return errs.ErrorOrNil()
}

func TestAccAWSWafByteMatchSet_basic(t *testing.T) {
Expand Down
70 changes: 41 additions & 29 deletions aws/resource_aws_waf_geo_match_set_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"log"
"regexp"
"sync"
"testing"

"github.com/aws/aws-sdk-go/aws"
Expand All @@ -29,12 +30,16 @@ func init() {

func testSweepWafGeoMatchSet(region string) error {
client, err := sharedClientForRegion(region)

if err != nil {
return fmt.Errorf("error getting client: %s", err)
}
conn := client.(*AWSClient).wafconn

var sweeperErrs *multierror.Error
conn := client.(*AWSClient).wafconn
sweepResources := make([]*testSweepResource, 0)
var errs *multierror.Error
var g multierror.Group
var mutex = &sync.Mutex{}

input := &waf.ListGeoMatchSetsInput{}

Expand All @@ -44,50 +49,57 @@ func testSweepWafGeoMatchSet(region string) error {
}

for _, geoMatchSet := range page.GeoMatchSets {
id := aws.StringValue(geoMatchSet.GeoMatchSetId)

r := resourceAwsWafGeoMatchSet()
d := r.Data(nil)

id := aws.StringValue(geoMatchSet.GeoMatchSetId)
d.SetId(id)

// Need to Read first to fill in geo_match_constraint attribute
err := r.Read(d, client)
// read concurrently and gather errors
g.Go(func() error {
// Need to Read first to fill in geo_match_constraint attribute
err := r.Read(d, client)

if err != nil {
sweeperErr := fmt.Errorf("error reading WAF Geo Match Set (%s): %w", id, err)
log.Printf("[ERROR] %s", sweeperErr)
sweeperErrs = multierror.Append(sweeperErrs, sweeperErr)
continue
}
if err != nil {
sweeperErr := fmt.Errorf("error reading WAF Geo Match Set (%s): %w", id, err)
log.Printf("[ERROR] %s", sweeperErr)
return sweeperErr
}

// In case it was already deleted
if d.Id() == "" {
continue
}
// In case it was already deleted
if d.Id() == "" {
return nil
}

err = r.Delete(d, client)
mutex.Lock()
defer mutex.Unlock()
sweepResources = append(sweepResources, NewTestSweepResource(r, d, client))

if err != nil {
sweeperErr := fmt.Errorf("error deleting WAF Geo Match Set (%s): %w", id, err)
log.Printf("[ERROR] %s", sweeperErr)
sweeperErrs = multierror.Append(sweeperErrs, sweeperErr)
continue
}
return nil
})
}

return !lastPage
})

if testSweepSkipSweepError(err) {
log.Printf("[WARN] Skipping WAF Geo Match Set sweep for %s: %s", region, err)
return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors
if err != nil {
errs = multierror.Append(errs, fmt.Errorf("error listing WAF Geo Match Set for %s: %w", region, err))
}

if err != nil {
sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF Geo Match Sets: %w", err))
if err = g.Wait().ErrorOrNil(); err != nil {
errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF Geo Match Sets: %w", err))
}

if err = testSweepResourceOrchestrator(sweepResources); err != nil {
errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF Geo Match Set for %s: %w", region, err))
}

if testSweepSkipSweepError(errs.ErrorOrNil()) {
log.Printf("[WARN] Skipping WAF Geo Match Set sweep for %s: %s", region, errs)
return nil
}

return sweeperErrs.ErrorOrNil()
return errs.ErrorOrNil()
}

func TestAccAWSWafGeoMatchSet_basic(t *testing.T) {
Expand Down
70 changes: 41 additions & 29 deletions aws/resource_aws_waf_ipset_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"reflect"
"regexp"
"strings"
"sync"
"testing"

"github.com/aws/aws-sdk-go/aws"
Expand All @@ -32,12 +33,16 @@ func init() {

func testSweepWafIPSet(region string) error {
client, err := sharedClientForRegion(region)

if err != nil {
return fmt.Errorf("error getting client: %s", err)
}
conn := client.(*AWSClient).wafconn

var sweeperErrs *multierror.Error
conn := client.(*AWSClient).wafconn
sweepResources := make([]*testSweepResource, 0)
var errs *multierror.Error
var g multierror.Group
var mutex = &sync.Mutex{}

input := &waf.ListIPSetsInput{}

Expand All @@ -47,50 +52,57 @@ func testSweepWafIPSet(region string) error {
}

for _, ipSet := range page.IPSets {
id := aws.StringValue(ipSet.IPSetId)

r := resourceAwsWafIPSet()
d := r.Data(nil)

id := aws.StringValue(ipSet.IPSetId)
d.SetId(id)

// Need to Read first to fill in ip_set_descriptors attribute
err := r.Read(d, client)
// read concurrently and gather errors
g.Go(func() error {
// Need to Read first to fill in ip_set_descriptors attribute
err := r.Read(d, client)

if err != nil {
sweeperErr := fmt.Errorf("error reading WAF IP Set (%s): %w", id, err)
log.Printf("[ERROR] %s", sweeperErr)
sweeperErrs = multierror.Append(sweeperErrs, sweeperErr)
continue
}
if err != nil {
sweeperErr := fmt.Errorf("error reading WAF IP Set (%s): %w", id, err)
log.Printf("[ERROR] %s", sweeperErr)
return sweeperErr
}

// In case it was already deleted
if d.Id() == "" {
continue
}
// In case it was already deleted
if d.Id() == "" {
return nil
}

err = r.Delete(d, client)
mutex.Lock()
defer mutex.Unlock()
sweepResources = append(sweepResources, NewTestSweepResource(r, d, client))

if err != nil {
sweeperErr := fmt.Errorf("error deleting WAF IP Set (%s): %w", id, err)
log.Printf("[ERROR] %s", sweeperErr)
sweeperErrs = multierror.Append(sweeperErrs, sweeperErr)
continue
}
return nil
})
}

return !lastPage
})

if testSweepSkipSweepError(err) {
log.Printf("[WARN] Skipping WAF IP Set sweep for %s: %s", region, err)
return sweeperErrs.ErrorOrNil() // In case we have completed some pages, but had errors
if err != nil {
errs = multierror.Append(errs, fmt.Errorf("error listing WAF IP Set for %s: %w", region, err))
}

if err != nil {
sweeperErrs = multierror.Append(sweeperErrs, fmt.Errorf("error describing WAF IP Sets: %w", err))
if err = g.Wait().ErrorOrNil(); err != nil {
errs = multierror.Append(errs, fmt.Errorf("error concurrently reading WAF IP Sets: %w", err))
}

if err = testSweepResourceOrchestrator(sweepResources); err != nil {
errs = multierror.Append(errs, fmt.Errorf("error sweeping WAF IP Set for %s: %w", region, err))
}

if testSweepSkipSweepError(errs.ErrorOrNil()) {
log.Printf("[WARN] Skipping WAF IP Set sweep for %s: %s", region, errs)
return nil
}

return sweeperErrs.ErrorOrNil()
return errs.ErrorOrNil()
}

func TestAccAWSWafIPSet_basic(t *testing.T) {
Expand Down
Loading

0 comments on commit 87cbd86

Please sign in to comment.