Cloudwatch Log Group Nuking

.circleci/nuke_config.yml

@@ -39,3 +39,48 @@ OIDCProvider:
     names_regex:
       # We have an active OIDC Provider used by github actions
       - ".*token.actions.githubusercontent.com.*"
+
+CloudWatchLogGroup:
+  # Use an allow list instead of block list because we haven't done the due diligence yet to figure out what log groups
+  # we need to keep. This is hard to do in the current scenario as we have thousands of log groups in the accounts.
+  include:
+    names_regex:
+      - "/aws/containerinsights/eks-cluster-[a-zA-Z0-9]{6}/.*"
+      - "/aws/containerinsights/eks-service-catalog-[a-zA-Z0-9]{6}/.*"
+      - "/aws/ecs/containerinsights/[a-zA-Z0-9]{6}-cluster/.*"
+      - "/aws/ecs/containerinsights/[a-zA-Z0-9]{6}-ecs-cluster/.*"
+      - "/aws/ecs/containerinsights/Test-cluster[a-zA-Z0-9]{6}/.*"
+      - "/aws/eks/eks-cluster-[a-zA-Z0-9]{6}.*"
+      - "/aws/eks/eks-service-catalog-[a-zA-Z0-9]{6}.*"
+      - "/aws/lambda/Test.*"
+      - "/aws/lambda/[a-zA-Z0-9]{6}-ecs-deploy-runner.*"
+      - "/aws/lambda/ecs-deploy-runner-[a-zA-Z0-9]{6}-invoker$"
+      - "/aws/lambda/es-cluster-[a-zA-Z0-9]{6}.*"
+      - "/aws/lambda/jenkins-[a-zA-Z0-9]{6}.*"
+      - "/aws/lambda/jenkins[a-zA-Z0-9]{6}-0-backup$"
+      - "/aws/lambda/test-aurora-[a-zA-Z0-9]{6}.*"
+      - "/aws/lambda/[a-zA-Z0-9]{6}-handler$"
+      - "/aws/lambda/test[a-zA-Z0-9]{6}.*"
+      - "/aws/lambda/lambda-[a-zA-Z0-9]{6}$"
+      - "/aws/lambda/us-east-1\\.[a-zA-Z0-9]{6}cloudfront.*"
+      - "/aws/lambda/cleanup-expired-iam-certs-[a-zA-Z0-9]{6}$"
+      - "/aws/lambda/create-snapshot-example-(aurora|mysql)$"
+      - "/aws/lambda/HoustonExpress[a-zA-Z0-9]{6}.*"
+      - "/aws/rds/cluster/test[a-zA-Z0-9]{6}.*"
+      - "/aws/rds/cluster/test-aurora-[a-zA-Z0-9]{6}.*"
+      - "eks-service-catalog-[a-zA-Z0-9]{6}$"
+      - "eks-cluster-[a-zA-Z0-9]{6}-container-logs$"
+      - "es-[a-zA-Z0-9]{6}-lg$"
+      - "TestCloudWatchLogAggregation.*"
+      - "API-Gateway-Execution-Logs_.*/example$"
+      - "asg-[a-zA-Z0-9]{6}$"
+      - "^[a-zA-Z0-9]{6}-service$"
+      - "^[a-zA-Z0-9]{6}-ecs-deploy-runner-[a-zA-Z0-9]{6}$"
+      - "ecs-deploy-runner-[a-zA-Z0-9]{6}$"
+      - "^[a-zA-Z0-9]{6}(stage|prod)-ec2-syslog$"
+      - "Ubuntu1804-[a-zA-Z0-9]{6}-logs$"
+      - "bastion-host-[a-zA-Z0-9]{6}$"
+      - "ec2-instance-[a-zA-Z0-9]{6}$"
+      - "jenkins-[a-zA-Z0-9]{6}$"
+      - "openvpn-server-[a-zA-Z0-9]{6}_log_group$"
+      - "vpc-test-[a-zA-Z0-9]{6}.*"

README.md

@@ -38,6 +38,7 @@ The currently supported functionality includes:
 - Deleting all OpenSearch Domains in an AWS account
 - Deleting all IAM OpenID Connect Providers
 - Deleting all Customer managed keys from Key Management Service in an AWS account
+- Deleting all CloudWatch Log Groups in an AWS Account
 
 ### BEWARE!
 
@@ -197,6 +198,9 @@ The following resources support the Config file:
 - IAM OpenID Connect Providers
     - Resource type: `oidcprovider`
     - Config key: `OIDCProvider`
+- CloudWatch LogGroups
+    - Resource type: `cloudwatch-loggroup`
+    - Config key: `CloudWatchLogGroup`
 - KMS customer keys
     - Resource type: `kmscustomerkeys`
 
@@ -287,28 +291,29 @@ Be careful when nuking and append the `--dry-run` option if you're unsure. Even
 
 To find out what we options are supported in the config file today, consult this table. Resource types at the top level of the file that are supported are listed here.
 
-| resource type      | names | names_regex | tags | tags_regex |
-|--------------------|-------|-------------|------|------------|
-| s3                 | none  | ✅          | none | none       |
-| iam                | none  | ✅          | none | none       |
-| ecsserv            | none  | ✅          | none | none       |
-| ecscluster         | none  | ✅          | none | none       |
-| secretsmanager     | none  | ✅          | none | none       |
-| nat-gateway        | none  | ✅          | none | none       |
-| accessanalyzer     | none  | ✅          | none | none       |
-| dynamodb           | none  | ✅          | none | none       |
-| ebs                | none  | ✅          | none | none       |
-| lambda             | none  | ✅          | none | none       |
-| elbv2              | none  | ✅          | none | none       |
-| ecs                | none  | ✅          | none | none       |
-| elasticache        | none  | ✅          | none | none       |
-| vpc                | none  | ✅          | none | none       |
-| oidcprovider       | none  | ✅          | none | none       |
-| kmscustomerkeys    | none  | ✅          | none | none       |
-| acmpca             | none  | none        | none | none       |
-| ec2 instance       | none  | none        | none | none       |
-| iam role           | none  | none        | none | none       |
-| ... (more to come) | none  | none        | none | none       |
+| resource type       | names | names_regex | tags | tags_regex |
+|---------------------|-------|-------------|------|------------|
+| s3                  | none  | ✅          | none | none       |
+| iam                 | none  | ✅          | none | none       |
+| ecsserv             | none  | ✅          | none | none       |
+| ecscluster          | none  | ✅          | none | none       |
+| secretsmanager      | none  | ✅          | none | none       |
+| nat-gateway         | none  | ✅          | none | none       |
+| accessanalyzer      | none  | ✅          | none | none       |
+| dynamodb            | none  | ✅          | none | none       |
+| ebs                 | none  | ✅          | none | none       |
+| lambda              | none  | ✅          | none | none       |
+| elbv2               | none  | ✅          | none | none       |
+| ecs                 | none  | ✅          | none | none       |
+| elasticache         | none  | ✅          | none | none       |
+| vpc                 | none  | ✅          | none | none       |
+| oidcprovider        | none  | ✅          | none | none       |
+| cloudwatch-loggroup | none  | ✅          | none | none       |
+| kmscustomerkeys     | none  | ✅          | none | none       |
+| acmpca              | none  | none        | none | none       |
+| ec2 instance        | none  | none        | none | none       |
+| iam role            | none  | none        | none | none       |
+| ... (more to come)  | none  | none        | none | none       |
 
 
 

aws/aws.go

@@ -606,6 +606,20 @@ func GetAllResources(targetRegions []string, excludeAfter time.Time, resourceTyp
 		}
 		// End CloudWatchDashboard
 
+		// CloudWatchLogGroup
+		cloudwatchLogGroups := CloudWatchLogGroups{}
+		if IsNukeable(cloudwatchLogGroups.ResourceName(), resourceTypes) {
+			lgNames, err := getAllCloudWatchLogGroups(session, excludeAfter, configObj)
+			if err != nil {
+				return nil, errors.WithStackTrace(err)
+			}
+			if len(lgNames) > 0 {
+				cloudwatchLogGroups.Names = awsgo.StringValueSlice(lgNames)
+				resourcesInRegion.Resources = append(resourcesInRegion.Resources, cloudwatchLogGroups)
+			}
+		}
+		// End CloudWatchLogGroup
+
 		// S3 Buckets
 		s3Buckets := S3Buckets{}
 		if IsNukeable(s3Buckets.ResourceName(), resourceTypes) {
@@ -801,6 +815,7 @@ func ListResourceTypes() []string {
 		Elasticaches{}.ResourceName(),
 		OIDCProviders{}.ResourceName(),
 		KmsCustomerKeys{}.ResourceName(),
+		CloudWatchLogGroups{}.ResourceName(),
 	}
 	sort.Strings(resourceTypes)
 	return resourceTypes

aws/cloudwatch_loggroup.go

@@ -0,0 +1,134 @@
+package aws
+
+import (
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
+	"github.com/gruntwork-io/cloud-nuke/config"
+	"github.com/gruntwork-io/cloud-nuke/logging"
+	"github.com/gruntwork-io/go-commons/errors"
+	"github.com/hashicorp/go-multierror"
+)
+
+func getAllCloudWatchLogGroups(session *session.Session, excludeAfter time.Time, configObj config.Config) ([]*string, error) {
+	svc := cloudwatchlogs.New(session)
+
+	allLogGroups := []*string{}
+	err := svc.DescribeLogGroupsPages(
+		&cloudwatchlogs.DescribeLogGroupsInput{},
+		func(page *cloudwatchlogs.DescribeLogGroupsOutput, lastPage bool) bool {
+			for _, logGroup := range page.LogGroups {
+				if shouldIncludeCloudWatchLogGroup(logGroup, excludeAfter, configObj) {
+					allLogGroups = append(allLogGroups, logGroup.LogGroupName)
+				}
+			}
+			return !lastPage
+		},
+	)
+	if err != nil {
+		return nil, errors.WithStackTrace(err)
+	}
+	return allLogGroups, nil
+}
+
+func shouldIncludeCloudWatchLogGroup(logGroup *cloudwatchlogs.LogGroup, excludeAfter time.Time, configObj config.Config) bool {
+	if logGroup == nil {
+		return false
+	}
+
+	if logGroup.CreationTime != nil {
+		// Convert milliseconds since epoch to time.Time object
+		creationTime := time.Unix(0, aws.Int64Value(logGroup.CreationTime)*int64(time.Millisecond))
+		if excludeAfter.Before(creationTime) {
+			return false
+		}
+	}
+
+	return config.ShouldInclude(
+		aws.StringValue(logGroup.LogGroupName),
+		configObj.CloudWatchLogGroup.IncludeRule.NamesRegExp,
+		configObj.CloudWatchLogGroup.ExcludeRule.NamesRegExp,
+	)
+}
+
+func nukeAllCloudWatchLogGroups(session *session.Session, identifiers []*string) error {
+	region := aws.StringValue(session.Config.Region)
+	svc := cloudwatchlogs.New(session)
+
+	if len(identifiers) == 0 {
+		logging.Logger.Infof("No CloudWatch Log Groups to nuke in region %s", *session.Config.Region)
+		return nil
+	}
+
+	// NOTE: we don't need to do pagination here, because the pagination is handled by the caller to this function,
+	// based on CloudWatchLogGroup.MaxBatchSize, however we add a guard here to warn users when the batching fails and
+	// has a chance of throttling AWS. Since we concurrently make one call for each identifier, we pick 100 for the
+	// limit here because many APIs in AWS have a limit of 100 requests per second.
+	if len(identifiers) > 100 {
+		logging.Logger.Errorf("Nuking too many CloudWatch LogGroups at once (100): halting to avoid hitting AWS API rate limiting")
+		return TooManyLogGroupsErr{}
+	}
+
+	// There is no bulk delete CloudWatch Log Group API, so we delete the batch of CloudWatch Log Groups concurrently
+	// using go routines.
+	logging.Logger.Infof("Deleting CloudWatch Log Groups in region %s", region)
+	wg := new(sync.WaitGroup)
+	wg.Add(len(identifiers))
+	errChans := make([]chan error, len(identifiers))
+	for i, logGroupName := range identifiers {
+		errChans[i] = make(chan error, 1)
+		go deleteCloudWatchLogGroupAsync(wg, errChans[i], svc, logGroupName, region)
+	}
+	wg.Wait()
+
+	// Collect all the errors from the async delete calls into a single error struct.
+	// NOTE: We ignore OperationAbortedException which is thrown when there is an eventual consistency issue, where
+	// cloud-nuke picks up a Log Group that is already requested to be deleted.
+	var allErrs *multierror.Error
+	for _, errChan := range errChans {
+		if err := <-errChan; err != nil {
+			if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() != "OperationAbortedException" {
+				allErrs = multierror.Append(allErrs, err)
+			}
+		}
+	}
+	finalErr := allErrs.ErrorOrNil()
+	if finalErr != nil {
+		return errors.WithStackTrace(finalErr)
+	}
+	return nil
+}
+
+// deleteCloudWatchLogGroupAsync deletes the provided Log Group asynchronously in a goroutine, using wait groups for
+// concurrency control and a return channel for errors.
+func deleteCloudWatchLogGroupAsync(
+	wg *sync.WaitGroup,
+	errChan chan error,
+	svc *cloudwatchlogs.CloudWatchLogs,
+	logGroupName *string,
+	region string,
+) {
+	defer wg.Done()
+	input := &cloudwatchlogs.DeleteLogGroupInput{LogGroupName: logGroupName}
+	_, err := svc.DeleteLogGroup(input)
+	errChan <- err
+
+	logGroupNameStr := aws.StringValue(logGroupName)
+	if err == nil {
+		logging.Logger.Infof("[OK] CloudWatch Log Group %s deleted in %s", logGroupNameStr, region)
+	} else {
+		logging.Logger.Errorf("[Failed] Error deleting CloudWatch Log Group %s in %s: %s", logGroupNameStr, region, err)
+	}
+}
+
+// Custom errors
+
+type TooManyLogGroupsErr struct{}
+
+func (err TooManyLogGroupsErr) Error() string {
+	return "Too many LogGroups requested at once."
+}