-
-
Notifications
You must be signed in to change notification settings - Fork 358
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Update cloudwatch nuking to support filtering and concurrent deletes
- Loading branch information
1 parent
00ccb16
commit 32bc385
Showing
8 changed files
with
208 additions
and
49 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,54 +1,134 @@ | ||
| package aws | ||
|
|
||
| import ( | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/aws/aws-sdk-go/aws" | ||
| "github.com/aws/aws-sdk-go/aws/awserr" | ||
| "github.com/aws/aws-sdk-go/aws/session" | ||
| "github.com/aws/aws-sdk-go/service/cloudwatchlogs" | ||
| "github.com/gruntwork-io/cloud-nuke/config" | ||
| "github.com/gruntwork-io/cloud-nuke/logging" | ||
| "github.com/gruntwork-io/go-commons/errors" | ||
| "github.com/hashicorp/go-multierror" | ||
| ) | ||
|
|
||
| func getAllCloudWatchLogGroups(session *session.Session, region string) ([]*string, error) { | ||
| func getAllCloudWatchLogGroups(session *session.Session, region string, excludeAfter time.Time, configObj config.Config) ([]*string, error) { | ||
| svc := cloudwatchlogs.New(session) | ||
|
|
||
| output, err := svc.DescribeLogGroups(&cloudwatchlogs.DescribeLogGroupsInput{}) | ||
| allLogGroups := []*string{} | ||
| err := svc.DescribeLogGroupsPages( | ||
| &cloudwatchlogs.DescribeLogGroupsInput{}, | ||
| func(page *cloudwatchlogs.DescribeLogGroupsOutput, lastPage bool) bool { | ||
| for _, logGroup := range page.LogGroups { | ||
| if shouldIncludeCloudWatchLogGroup(logGroup, excludeAfter, configObj) { | ||
| allLogGroups = append(allLogGroups, logGroup.LogGroupName) | ||
| } | ||
| } | ||
| return !lastPage | ||
| }, | ||
| ) | ||
| if err != nil { | ||
| return nil, errors.WithStackTrace(err) | ||
| } | ||
| return allLogGroups, nil | ||
| } | ||
|
|
||
| func shouldIncludeCloudWatchLogGroup(logGroup *cloudwatchlogs.LogGroup, excludeAfter time.Time, configObj config.Config) bool { | ||
| if logGroup == nil { | ||
| return false | ||
| } | ||
|
|
||
| var names []*string | ||
| for _, loggroup := range output.LogGroups { | ||
| names = append(names, loggroup.LogGroupName) | ||
| if logGroup.CreationTime != nil { | ||
| // Convert milliseconds since epoch to time.Time object | ||
| creationTime := time.Unix(0, aws.Int64Value(logGroup.CreationTime)*int64(time.Millisecond)) | ||
| if excludeAfter.Before(creationTime) { | ||
| return false | ||
| } | ||
| } | ||
|
|
||
| return names, nil | ||
| return config.ShouldInclude( | ||
| aws.StringValue(logGroup.LogGroupName), | ||
| configObj.CloudWatchLogGroup.IncludeRule.NamesRegExp, | ||
| configObj.CloudWatchLogGroup.ExcludeRule.NamesRegExp, | ||
| ) | ||
| } | ||
|
|
||
| func nukeAllCloudWatchLogGroups(session *session.Session, identifiers []*string) error { | ||
| region := aws.StringValue(session.Config.Region) | ||
| svc := cloudwatchlogs.New(session) | ||
|
|
||
| if len(identifiers) == 0 { | ||
| logging.Logger.Infof("No CloudWatch Log Groups to nuke in region %s", *session.Config.Region) | ||
| return nil | ||
| } | ||
|
|
||
| logging.Logger.Infof("Deleting all CloudWatch Log Groups in region %s", *session.Config.Region) | ||
| // NOTE: we don't need to do pagination here, because the pagination is handled by the caller to this function, | ||
| // based on CloudWatchLogGroup.MaxBatchSize, however we add a guard here to warn users when the batching fails and | ||
| // has a chance of throttling AWS. Since we concurrently make one call for each identifier, we pick 100 for the | ||
| // limit here because many APIs in AWS have a limit of 100 requests per second. | ||
| if len(identifiers) > 100 { | ||
| logging.Logger.Errorf("Nuking too many CloudWatch LogGroups at once (100): halting to avoid hitting AWS API rate limiting") | ||
| return TooManyLogGroupsErr{} | ||
| } | ||
|
|
||
| var deleteResources = 0 | ||
| // There is no bulk delete CloudWatch Log Group API, so we delete the batch of CloudWatch Log Groups concurrently | ||
| // using go routines. | ||
| logging.Logger.Infof("Deleting CloudWatch Log Groups in region %s", region) | ||
| wg := new(sync.WaitGroup) | ||
| wg.Add(len(identifiers)) | ||
| errChans := make([]chan error, len(identifiers)) | ||
| for i, logGroupName := range identifiers { | ||
| errChans[i] = make(chan error, 1) | ||
| go deleteCloudWatchLogGroupAsync(wg, errChans[i], svc, logGroupName, region) | ||
| } | ||
| wg.Wait() | ||
|
|
||
| for _, name := range identifiers { | ||
| _, err := svc.DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{ | ||
| LogGroupName: name, | ||
| }) | ||
| if err != nil { | ||
| logging.Logger.Errorf("[Failed] %s", err) | ||
| } else { | ||
| logging.Logger.Infof("[OK] CloudWatch Log Group %s terminated in %s", *name, *session.Config.Region) | ||
| deleteResources++ | ||
| // Collect all the errors from the async delete calls into a single error struct. | ||
| // NOTE: We ignore OperationAbortedException which is thrown when there is an eventual consistency issue, where | ||
| // cloud-nuke picks up a Log Group that is already requested to be deleted. | ||
| var allErrs *multierror.Error | ||
| for _, errChan := range errChans { | ||
| if err := <-errChan; err != nil { | ||
| if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() != "OperationAbortedException" { | ||
| allErrs = multierror.Append(allErrs, err) | ||
| } | ||
| } | ||
| } | ||
| finalErr := allErrs.ErrorOrNil() | ||
| if finalErr != nil { | ||
| return errors.WithStackTrace(finalErr) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // deleteCloudWatchLogGroupAsync deletes the provided Log Group asynchronously in a goroutine, using wait groups for | ||
| // concurrency control and a return channel for errors. | ||
| func deleteCloudWatchLogGroupAsync( | ||
| wg *sync.WaitGroup, | ||
| errChan chan error, | ||
| svc *cloudwatchlogs.CloudWatchLogs, | ||
| logGroupName *string, | ||
| region string, | ||
| ) { | ||
| defer wg.Done() | ||
| input := &cloudwatchlogs.DeleteLogGroupInput{LogGroupName: logGroupName} | ||
| _, err := svc.DeleteLogGroup(input) | ||
| errChan <- err | ||
|
|
||
| logGroupNameStr := aws.StringValue(logGroupName) | ||
| if err == nil { | ||
| logging.Logger.Infof("[OK] CloudWatch Log Group %s deleted in %s", logGroupNameStr, region) | ||
| } else { | ||
| logging.Logger.Errorf("[Failed] Error deleting CloudWatch Log Group %s in %s: %s", logGroupNameStr, region, err) | ||
| } | ||
| } | ||
|
|
||
| logging.Logger.Infof("[OK] %d CloudWatch Log Group(s) terminated in %s", deleteResources, *session.Config.Region) | ||
| // Custom errors | ||
|
|
||
| return nil | ||
| type TooManyLogGroupsErr struct{} | ||
|
|
||
| func (err TooManyLogGroupsErr) Error() string { | ||
| return "Too many LogGroups requested at once." | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters