From af43e9f9189313f01544ae88fdf2f186fb492b12 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Wed, 10 Oct 2018 23:22:23 -0700 Subject: [PATCH 1/4] Nuke ECS services and ECS clusters --- README.md | 2 + aws/aws.go | 22 +++ aws/ec2_test.go | 74 ++++++--- aws/ecs_cluster.go | 49 ++++++ aws/ecs_cluster_test.go | 64 ++++++++ aws/ecs_cluster_types.go | 34 ++++ aws/ecs_service.go | 157 ++++++++++++++++++ aws/ecs_service_test.go | 219 +++++++++++++++++++++++++ aws/ecs_service_types.go | 35 ++++ aws/ecs_utils_for_test.go | 325 ++++++++++++++++++++++++++++++++++++++ 10 files changed, 955 insertions(+), 26 deletions(-) create mode 100644 aws/ecs_cluster.go create mode 100644 aws/ecs_cluster_test.go create mode 100644 aws/ecs_cluster_types.go create mode 100644 aws/ecs_service.go create mode 100644 aws/ecs_service_test.go create mode 100644 aws/ecs_service_types.go create mode 100644 aws/ecs_utils_for_test.go diff --git a/README.md b/README.md index c035e172..35e92212 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ The currently supported functionality includes: * Deleting all Snapshots in an AWS account * Deleting all Elastic IPs in an AWS account * Deleting all Launch Configurations in an AWS account +* Deleting all ECS services in an AWS account +* Deleting all ECS clusters in an AWS account ## Azure diff --git a/aws/aws.go b/aws/aws.go index efdf8375..a1dc921f 100644 --- a/aws/aws.go +++ b/aws/aws.go @@ -205,6 +205,28 @@ func GetAllResources(regions []string, excludedRegions []string, excludeAfter ti resourcesInRegion.Resources = append(resourcesInRegion.Resources, snapshots) // End Snapshots + // ECS resources + clusterArns, err := getAllEcsClusters(session) + if err != nil { + return nil, errors.WithStackTrace(err) + } + serviceArns, serviceClusterMap, err := getAllEcsServices(session, clusterArns, excludeAfter) + if err != nil { + return nil, errors.WithStackTrace(err) + } + + // Must delete services before clusters + ecsServices := ECSServices{ + Services: awsgo.StringValueSlice(serviceArns), + ServiceClusterMap: serviceClusterMap, + } + resourcesInRegion.Resources = append(resourcesInRegion.Resources, ecsServices) + ecsClusters := ECSClusters{ + Clusters: awsgo.StringValueSlice(clusterArns), + } + resourcesInRegion.Resources = append(resourcesInRegion.Resources, ecsClusters) + // End ECS resources + account.Resources[region] = resourcesInRegion } diff --git a/aws/ec2_test.go b/aws/ec2_test.go index ef47a4d8..7c4fafe7 100644 --- a/aws/ec2_test.go +++ b/aws/ec2_test.go @@ -1,6 +1,7 @@ package aws import ( + "errors" "testing" "time" @@ -8,44 +9,42 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/gruntwork-io/cloud-nuke/util" - "github.com/gruntwork-io/gruntwork-cli/errors" + gruntworkerrors "github.com/gruntwork-io/gruntwork-cli/errors" "github.com/stretchr/testify/assert" ) -func createTestEC2Instance(t *testing.T, session *session.Session, name string, protected bool) ec2.Instance { - svc := ec2.New(session) - +// getAMIIdByName - Retrieves an AMI ImageId given the name of the Id. Used for +// retrieving a standard AMI across AWS regions. +func getAMIIdByName(svc *ec2.EC2, name string) (string, error) { imagesResult, err := svc.DescribeImages(&ec2.DescribeImagesInput{ Owners: []*string{awsgo.String("self"), awsgo.String("amazon")}, Filters: []*ec2.Filter{ &ec2.Filter{ Name: awsgo.String("name"), - Values: []*string{awsgo.String("amzn-ami-hvm-2017.09.1.20180115-x86_64-gp2")}, + Values: []*string{awsgo.String(name)}, }, }, }) if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) + return "", gruntworkerrors.WithStackTrace(err) } - imageID := *imagesResult.Images[0].ImageId - - params := &ec2.RunInstancesInput{ - ImageId: awsgo.String(imageID), - InstanceType: awsgo.String("t2.micro"), - MinCount: awsgo.Int64(1), - MaxCount: awsgo.Int64(1), - DisableApiTermination: awsgo.Bool(protected), - } + return *imagesResult.Images[0].ImageId, nil +} +// runAndWaitForInstance - Given a preconstructed ec2.RunInstancesInput object, +// make the API call to run the instance and then wait for the instance to be +// up and running before returning. +func runAndWaitForInstance(svc *ec2.EC2, name string, params *ec2.RunInstancesInput) (ec2.Instance, error) { runResult, err := svc.RunInstances(params) if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) + return ec2.Instance{}, gruntworkerrors.WithStackTrace(err) } if len(runResult.Instances) == 0 { - assert.Fail(t, "Could not create test EC2 instance in "+*session.Config.Region) + err := errors.New("Could not create test EC2 instance") + return ec2.Instance{}, gruntworkerrors.WithStackTrace(err) } err = svc.WaitUntilInstanceExists(&ec2.DescribeInstancesInput{ @@ -58,7 +57,7 @@ func createTestEC2Instance(t *testing.T, session *session.Session, name string, }) if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) + return ec2.Instance{}, gruntworkerrors.WithStackTrace(err) } // Add test tag to the created instance @@ -73,7 +72,7 @@ func createTestEC2Instance(t *testing.T, session *session.Session, name string, }) if err != nil { - assert.Failf(t, "Could not tag EC2 instance", errors.WithStackTrace(err).Error()) + return ec2.Instance{}, gruntworkerrors.WithStackTrace(err) } // EC2 Instance must be in a running before this function returns @@ -87,10 +86,33 @@ func createTestEC2Instance(t *testing.T, session *session.Session, name string, }) if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) + return ec2.Instance{}, gruntworkerrors.WithStackTrace(err) + } + + return *runResult.Instances[0], nil + +} + +func createTestEC2Instance(t *testing.T, session *session.Session, name string, protected bool) ec2.Instance { + svc := ec2.New(session) + + imageID, err := getAMIIdByName(svc, "amzn-ami-hvm-2017.09.1.20180115-x86_64-gp2") + if err != nil { + assert.Fail(t, err.Error()) } - return *runResult.Instances[0] + params := &ec2.RunInstancesInput{ + ImageId: awsgo.String(imageID), + InstanceType: awsgo.String("t2.micro"), + MinCount: awsgo.Int64(1), + MaxCount: awsgo.Int64(1), + DisableApiTermination: awsgo.Bool(protected), + } + instance, err := runAndWaitForInstance(svc, name, params) + if err != nil { + assert.Fail(t, err.Error()) + } + return instance } func removeEC2InstanceProtection(svc *ec2.EC2, instance *ec2.Instance) error { @@ -108,7 +130,7 @@ func removeEC2InstanceProtection(svc *ec2.EC2, instance *ec2.Instance) error { func findEC2InstancesByNameTag(t *testing.T, session *session.Session, name string) []*string { output, err := ec2.New(session).DescribeInstances(&ec2.DescribeInstancesInput{}) if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) } var instanceIds []*string @@ -140,7 +162,7 @@ func TestListInstances(t *testing.T) { ) if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) } uniqueTestID := "cloud-nuke-test-" + util.UniqueID() @@ -166,7 +188,7 @@ func TestListInstances(t *testing.T) { assert.NotContains(t, instanceIds, protectedInstance.InstanceId) if err = removeEC2InstanceProtection(ec2.New(session), &protectedInstance); err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) } } @@ -179,7 +201,7 @@ func TestNukeInstances(t *testing.T) { ) if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) } uniqueTestID := "cloud-nuke-test-" + util.UniqueID() @@ -188,7 +210,7 @@ func TestNukeInstances(t *testing.T) { instanceIds := findEC2InstancesByNameTag(t, session, uniqueTestID) if err := nukeAllEc2Instances(session, instanceIds); err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) } instances, err := getAllEc2Instances(session, region, time.Now().Add(1*time.Hour)) diff --git a/aws/ecs_cluster.go b/aws/ecs_cluster.go new file mode 100644 index 00000000..07948112 --- /dev/null +++ b/aws/ecs_cluster.go @@ -0,0 +1,49 @@ +package aws + +import ( + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/gruntwork-io/cloud-nuke/logging" + "github.com/gruntwork-io/gruntwork-cli/errors" +) + +// getAllEcsClusters - Returns a string of ECS Cluster ARNs, which uniquely identifies the cluster. +// NOTE: AWS api doesn't provide the necessary information to +// implement `excludeAfter` filter at the cluster +// level, so we will implement it at the service level. +func getAllEcsClusters(awsSession *session.Session) ([]*string, error) { + svc := ecs.New(awsSession) + result, err := svc.ListClusters(&ecs.ListClustersInput{}) + if err != nil { + return nil, errors.WithStackTrace(err) + } + return result.ClusterArns, nil +} + +// Deletes all given ECS clusters. Note that this will swallow failed deletes +// and continue along, logging the cluster ARN so that we can find it later. +func nukeAllEcsClusters(awsSession *session.Session, ecsClusterArns []*string) error { + numNuking := len(ecsClusterArns) + svc := ecs.New(awsSession) + + if numNuking == 0 { + logging.Logger.Infof("No ECS clusters to nuke in region %s", *awsSession.Config.Region) + return nil + } + + logging.Logger.Infof("Deleting %d ECS clusters in region %s", numNuking, *awsSession.Config.Region) + + numNuked := 0 + for _, ecsClusterArn := range ecsClusterArns { + params := &ecs.DeleteClusterInput{Cluster: ecsClusterArn} + _, err := svc.DeleteCluster(params) + if err != nil { + logging.Logger.Errorf("[Failed] Could not delete cluster %s: %s", *ecsClusterArn, err.Error()) + } else { + logging.Logger.Infof("Deleted cluster: %s", *ecsClusterArn) + numNuked += 1 + } + } + logging.Logger.Infof("[OK] %d of %d ECS cluster(s) deleted in %s", numNuked, numNuking, *awsSession.Config.Region) + return nil +} diff --git a/aws/ecs_cluster_test.go b/aws/ecs_cluster_test.go new file mode 100644 index 00000000..2ec44f87 --- /dev/null +++ b/aws/ecs_cluster_test.go @@ -0,0 +1,64 @@ +package aws + +import ( + "testing" + + awsgo "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/gruntwork-io/cloud-nuke/util" + "github.com/gruntwork-io/gruntwork-cli/errors" + "github.com/stretchr/testify/assert" +) + +// Test that we can find ECS clusters +func TestListECSClusters(t *testing.T) { + t.Parallel() + + region := getRandomFargateSupportedRegion() + awsSession, err := session.NewSession(&awsgo.Config{ + Region: awsgo.String(region)}, + ) + + if err != nil { + assert.Fail(t, errors.WithStackTrace(err).Error()) + } + + uniqueTestID := "cloud-nuke-test-" + util.UniqueID() + clusterName := uniqueTestID + "-cluster" + cluster := createEcsFargateCluster(t, awsSession, clusterName) + defer nukeAllEcsClusters(awsSession, []*string{cluster.ClusterArn}) + + clusterArns, err := getAllEcsClusters(awsSession) + if err != nil { + assert.Failf(t, "Unable to fetch clusters: %s", err.Error()) + } + assert.Contains(t, clusterArns, cluster.ClusterArn) +} + +// Test that we can successfully nuke ECS clusters +func TestNukeECSClusters(t *testing.T) { + t.Parallel() + + region := getRandomFargateSupportedRegion() + awsSession, err := session.NewSession(&awsgo.Config{ + Region: awsgo.String(region), + }) + + if err != nil { + assert.Fail(t, errors.WithStackTrace(err).Error()) + } + + uniqueTestID := "cloud-nuke-test-" + util.UniqueID() + clusterName := uniqueTestID + "-cluster" + + cluster := createEcsFargateCluster(t, awsSession, clusterName) + if err := nukeAllEcsClusters(awsSession, []*string{cluster.ClusterArn}); err != nil { + assert.Fail(t, errors.WithStackTrace(err).Error()) + } + + clusterArns, err := getAllEcsClusters(awsSession) + if err != nil { + assert.Failf(t, "Unable to fetch clusters: %s", err.Error()) + } + assert.NotContains(t, clusterArns, cluster.ClusterArn) +} diff --git a/aws/ecs_cluster_types.go b/aws/ecs_cluster_types.go new file mode 100644 index 00000000..cfca1ad9 --- /dev/null +++ b/aws/ecs_cluster_types.go @@ -0,0 +1,34 @@ +package aws + +import ( + awsgo "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/gruntwork-io/gruntwork-cli/errors" +) + +// ECSClusters - Represents all ECS clusters found in a region +type ECSClusters struct { + Clusters []string +} + +// ResourceName - The simple name of the aws resource +func (clusters ECSClusters) ResourceName() string { + return "ecsclust" +} + +// ResourceIdentifiers - The ARNs of the collected ECS clusters +func (clusters ECSClusters) ResourceIdentifiers() []string { + return clusters.Clusters +} + +func (clusters ECSClusters) MaxBatchSize() int { + return 200 +} + +// Nuke - nuke all ECS service resources +func (clusters ECSClusters) Nuke(awsSession *session.Session, identifiers []string) error { + if err := nukeAllEcsClusters(awsSession, awsgo.StringSlice(identifiers)); err != nil { + return errors.WithStackTrace(err) + } + return nil +} diff --git a/aws/ecs_service.go b/aws/ecs_service.go new file mode 100644 index 00000000..9be421da --- /dev/null +++ b/aws/ecs_service.go @@ -0,0 +1,157 @@ +package aws + +import ( + "time" + + awsgo "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/gruntwork-io/cloud-nuke/logging" + "github.com/gruntwork-io/gruntwork-cli/errors" +) + +// filterOutRecentServices - Given a list of services and an excludeAfter +// timestamp, filter out any services that were created after `excludeAfter`. +func filterOutRecentServices(svc *ecs.ECS, clusterArn *string, ecsServiceArns []string, excludeAfter time.Time) ([]*string, error) { + // Fetch descriptions in batches of 10, which is the max that AWS + // accepts for describe service. + var filteredEcsServiceArns []*string + batches := split(ecsServiceArns, 10) + for _, batch := range batches { + params := &ecs.DescribeServicesInput{ + Cluster: clusterArn, + Services: awsgo.StringSlice(batch), + } + describeResult, err := svc.DescribeServices(params) + if err != nil { + return nil, errors.WithStackTrace(err) + } + for _, service := range describeResult.Services { + if excludeAfter.After(*service.CreatedAt) { + filteredEcsServiceArns = append(filteredEcsServiceArns, service.ServiceArn) + } + } + } + return filteredEcsServiceArns, nil +} + +// getAllEcsServices - Returns a formatted string of ECS Service ARNs, which +// uniquely identifies the service, in addition to a mapping of services to +// clusters. For ECS, need to track ECS clusters of services as all service +// level API endpoints require providing the corresponding cluster. +// Note that this looks up services by ECS cluster ARNs. +func getAllEcsServices(awsSession *session.Session, ecsClusterArns []*string, excludeAfter time.Time) ([]*string, map[string]string, error) { + ecsServiceClusterMap := map[string]string{} + svc := ecs.New(awsSession) + + // For each cluster, fetch all services, filtering out recently created + // ones. + var ecsServiceArns []*string + for _, clusterArn := range ecsClusterArns { + result, err := svc.ListServices(&ecs.ListServicesInput{Cluster: clusterArn}) + if err != nil { + return nil, nil, errors.WithStackTrace(err) + } + filteredServiceArns, err := filterOutRecentServices(svc, clusterArn, awsgo.StringValueSlice(result.ServiceArns), excludeAfter) + if err != nil { + return nil, nil, errors.WithStackTrace(err) + } + // Update mapping to be used later in nuking + for _, serviceArn := range filteredServiceArns { + ecsServiceClusterMap[*serviceArn] = *clusterArn + } + ecsServiceArns = append(ecsServiceArns, filteredServiceArns...) + } + + return ecsServiceArns, ecsServiceClusterMap, nil +} + +// Deletes all provided ECS Services. At a high level this involves two steps: +// 1.) Drain all tasks from the service so that nothing is +// running. +// 2.) Delete service object once no tasks are running. +// Note that this will swallow failed deletes and continue along, logging the +// service ARN so that we can find it later. +func nukeAllEcsServices(awsSession *session.Session, ecsServiceClusterMap map[string]string, ecsServiceArns []*string) error { + numNuking := len(ecsServiceArns) + svc := ecs.New(awsSession) + + if numNuking == 0 { + logging.Logger.Infof("No ECS services to nuke in region %s", *awsSession.Config.Region) + return nil + } + + logging.Logger.Infof("Deleting %d ECS services in region %s", numNuking, *awsSession.Config.Region) + + // First, drain all the services to 0. You can't delete a + // service that is running tasks. + // Note that we request all the drains at once, and then + // wait for them in a separate loop because it will take a + // while to drain the services. + var requestedDrains []*string + for _, ecsServiceArn := range ecsServiceArns { + params := &ecs.UpdateServiceInput{ + Cluster: awsgo.String(ecsServiceClusterMap[*ecsServiceArn]), + Service: ecsServiceArn, + DesiredCount: awsgo.Int64(0), + } + _, err := svc.UpdateService(params) + if err != nil { + logging.Logger.Errorf("[Failed] Failed to drain service %s: %s", *ecsServiceArn, err) + } else { + requestedDrains = append(requestedDrains, ecsServiceArn) + } + } + + // Wait until service is fully drained by waiting for + // stability, which is defined as desiredCount == + // runningCount + var successfullyDrained []*string + for _, ecsServiceArn := range requestedDrains { + params := &ecs.DescribeServicesInput{ + Cluster: awsgo.String(ecsServiceClusterMap[*ecsServiceArn]), + Services: []*string{ecsServiceArn}, + } + err := svc.WaitUntilServicesStable(params) + if err != nil { + logging.Logger.Errorf("[Failed] Failed waiting for service to be stable %s: %s", *ecsServiceArn, err) + } else { + logging.Logger.Infof("Drained service: %s", *ecsServiceArn) + successfullyDrained = append(successfullyDrained, ecsServiceArn) + } + } + + // Now delete the services that were successfully drained + var requestedDeletes []*string + for _, ecsServiceArn := range successfullyDrained { + params := &ecs.DeleteServiceInput{ + Cluster: awsgo.String(ecsServiceClusterMap[*ecsServiceArn]), + Service: ecsServiceArn, + } + _, err := svc.DeleteService(params) + if err != nil { + logging.Logger.Errorf("[Failed] Failed deleting service %s: %s", *ecsServiceArn, err) + } else { + requestedDeletes = append(requestedDeletes, ecsServiceArn) + } + } + + // Wait until services are deleted + numNuked := 0 + for _, ecsServiceArn := range requestedDeletes { + params := &ecs.DescribeServicesInput{ + Cluster: awsgo.String(ecsServiceClusterMap[*ecsServiceArn]), + Services: []*string{ecsServiceArn}, + } + err := svc.WaitUntilServicesInactive(params) + if err != nil { + logging.Logger.Errorf("[Failed] Failed waiting for service to be deleted %s: %s", *ecsServiceArn, err) + } else { + logging.Logger.Infof("Deleted service: %s", *ecsServiceArn) + numNuked += 1 + } + } + + logging.Logger.Infof("[OK] %d of %d ECS service(s) deleted in %s", numNuked, numNuking, *awsSession.Config.Region) + return nil +} diff --git a/aws/ecs_service_test.go b/aws/ecs_service_test.go new file mode 100644 index 00000000..a03d3946 --- /dev/null +++ b/aws/ecs_service_test.go @@ -0,0 +1,219 @@ +package aws + +import ( + "testing" + "time" + + awsgo "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/gruntwork-io/cloud-nuke/util" + "github.com/gruntwork-io/gruntwork-cli/errors" + "github.com/stretchr/testify/assert" +) + +// Test that we can find ECS services that are running Fargate tasks +func TestListECSFargateServices(t *testing.T) { + t.Parallel() + + region := getRandomFargateSupportedRegion() + awsSession, err := session.NewSession(&awsgo.Config{ + Region: awsgo.String(region), + }) + + if err != nil { + assert.Fail(t, errors.WithStackTrace(err).Error()) + } + + ecsServiceClusterMap := map[string]string{} + uniqueTestID := "cloud-nuke-test-" + util.UniqueID() + clusterName := uniqueTestID + "-cluster" + serviceName := uniqueTestID + "-service" + taskFamilyName := uniqueTestID + "-task" + + cluster := createEcsFargateCluster(t, awsSession, clusterName) + defer nukeAllEcsClusters(awsSession, []*string{cluster.ClusterArn}) + + taskDefinition := createEcsTaskDefinition(t, awsSession, taskFamilyName, "FARGATE") + defer deleteEcsTaskDefinition(awsSession, taskDefinition) + + service := createEcsService(t, awsSession, serviceName, cluster, "FARGATE", taskDefinition) + ecsServiceClusterMap[*service.ServiceArn] = *cluster.ClusterArn + defer nukeAllEcsServices(awsSession, ecsServiceClusterMap, []*string{service.ServiceArn}) + + ecsServiceArns, newEcsServiceClusterMap, err := getAllEcsServices(awsSession, []*string{cluster.ClusterArn}, time.Now().Add(1*time.Hour*-1)) + if err != nil { + assert.Failf(t, "Unable to fetch list of services: %s", err.Error()) + } + assert.NotContains(t, awsgo.StringValueSlice(ecsServiceArns), *service.ServiceArn) + _, exists := newEcsServiceClusterMap[*service.ServiceArn] + assert.False(t, exists) + + ecsServiceArns, newEcsServiceClusterMap, err = getAllEcsServices(awsSession, []*string{cluster.ClusterArn}, time.Now().Add(1*time.Hour)) + if err != nil { + assert.Failf(t, "Unable to fetch list of services: %s", err.Error()) + } + assert.Contains(t, awsgo.StringValueSlice(ecsServiceArns), *service.ServiceArn) + _, exists = newEcsServiceClusterMap[*service.ServiceArn] + assert.True(t, exists) +} + +// Test that we can successfully nuke ECS services running Fargate tasks +func TestNukeECSFargateServices(t *testing.T) { + t.Parallel() + + region := getRandomFargateSupportedRegion() + awsSession, err := session.NewSession(&awsgo.Config{ + Region: awsgo.String(region), + }) + + if err != nil { + assert.Fail(t, errors.WithStackTrace(err).Error()) + } + + uniqueTestID := "cloud-nuke-test-" + util.UniqueID() + clusterName := uniqueTestID + "-cluster" + serviceName := uniqueTestID + "-service" + taskFamilyName := uniqueTestID + "-task" + + cluster := createEcsFargateCluster(t, awsSession, clusterName) + defer nukeAllEcsClusters(awsSession, []*string{cluster.ClusterArn}) + + taskDefinition := createEcsTaskDefinition(t, awsSession, taskFamilyName, "FARGATE") + defer deleteEcsTaskDefinition(awsSession, taskDefinition) + + service := createEcsService(t, awsSession, serviceName, cluster, "FARGATE", taskDefinition) + + ecsServiceClusterMap := map[string]string{} + ecsServiceClusterMap[*service.ServiceArn] = *cluster.ClusterArn + err = nukeAllEcsServices(awsSession, ecsServiceClusterMap, []*string{service.ServiceArn}) + if err != nil { + assert.Fail(t, err.Error()) + } + + ecsServiceArns, _, err := getAllEcsServices(awsSession, []*string{cluster.ClusterArn}, time.Now().Add(1*time.Hour)) + if err != nil { + assert.Failf(t, "Unable to fetch list of services: %s", err.Error()) + } + assert.NotContains(t, awsgo.StringValueSlice(ecsServiceArns), *service.ServiceArn) +} + +// Test that we can find ECS services running EC2 tasks +func TestListECSEC2Services(t *testing.T) { + t.Parallel() + + region := getRandomFargateSupportedRegion() + awsSession, err := session.NewSession(&awsgo.Config{ + Region: awsgo.String(region), + }) + + if err != nil { + assert.Fail(t, errors.WithStackTrace(err).Error()) + } + + ecsServiceClusterMap := map[string]string{} + uniqueTestID := "cloud-nuke-test-" + util.UniqueID() + clusterName := uniqueTestID + "-cluster" + serviceName := uniqueTestID + "-service" + taskFamilyName := uniqueTestID + "-task" + roleName := uniqueTestID + "-role" + instanceProfileName := uniqueTestID + "-instance-profile" + + // Prepare resources + // Create the IAM roles for ECS EC2 container instances + role := createEcsRole(t, awsSession, roleName) + defer deleteRole(awsSession, role) + + instanceProfile := createEcsInstanceProfile(t, awsSession, instanceProfileName, role) + defer deleteInstanceProfile(awsSession, instanceProfile) + + // IAM resources are slow to propagate, so give it some + // time + time.Sleep(15 * time.Second) + + // Provision a cluster with ec2 container instances, not + // forgetting to schedule deletion + cluster, instance := createEcsEC2Cluster(t, awsSession, clusterName, instanceProfile) + defer nukeAllEcsClusters(awsSession, []*string{cluster.ClusterArn}) + defer nukeAllEc2Instances(awsSession, []*string{instance.InstanceId}) + + // Finally, define the task and service + taskDefinition := createEcsTaskDefinition(t, awsSession, taskFamilyName, "EC2") + defer deleteEcsTaskDefinition(awsSession, taskDefinition) + + service := createEcsService(t, awsSession, serviceName, cluster, "EC2", taskDefinition) + ecsServiceClusterMap[*service.ServiceArn] = *cluster.ClusterArn + defer nukeAllEcsServices(awsSession, ecsServiceClusterMap, []*string{service.ServiceArn}) + // END prepare resources + + ecsServiceArns, newEcsServiceClusterMap, err := getAllEcsServices(awsSession, []*string{cluster.ClusterArn}, time.Now().Add(1*time.Hour*-1)) + if err != nil { + assert.Failf(t, "Unable to fetch list of services: %s", err.Error()) + } + assert.NotContains(t, awsgo.StringValueSlice(ecsServiceArns), *service.ServiceArn) + _, exists := newEcsServiceClusterMap[*service.ServiceArn] + assert.False(t, exists) + + ecsServiceArns, newEcsServiceClusterMap, err = getAllEcsServices(awsSession, []*string{cluster.ClusterArn}, time.Now().Add(1*time.Hour)) + if err != nil { + assert.Failf(t, "Unable to fetch list of services: %s", err.Error()) + } + assert.Contains(t, awsgo.StringValueSlice(ecsServiceArns), *service.ServiceArn) + _, exists = newEcsServiceClusterMap[*service.ServiceArn] + assert.True(t, exists) +} + +// Test that we can successfully nuke ECS services running EC2 tasks +func TestNukeECSEC2Services(t *testing.T) { + t.Parallel() + + region := getRandomFargateSupportedRegion() + awsSession, err := session.NewSession(&awsgo.Config{ + Region: awsgo.String(region), + }) + + if err != nil { + assert.Fail(t, errors.WithStackTrace(err).Error()) + } + + ecsServiceClusterMap := map[string]string{} + uniqueTestID := "cloud-nuke-test-" + util.UniqueID() + clusterName := uniqueTestID + "-cluster" + serviceName := uniqueTestID + "-service" + taskFamilyName := uniqueTestID + "-task" + roleName := uniqueTestID + "-role" + instanceProfileName := uniqueTestID + "-instance-profile" + + // Prepare resources + // Create the IAM roles for ECS EC2 container instances + role := createEcsRole(t, awsSession, roleName) + defer deleteRole(awsSession, role) + + instanceProfile := createEcsInstanceProfile(t, awsSession, instanceProfileName, role) + defer deleteInstanceProfile(awsSession, instanceProfile) + + // IAM resources are slow to propagate, so give it some + // time + time.Sleep(15 * time.Second) + + // Provision a cluster with ec2 container instances, not + // forgetting to schedule deletion + cluster, instance := createEcsEC2Cluster(t, awsSession, clusterName, instanceProfile) + defer nukeAllEcsClusters(awsSession, []*string{cluster.ClusterArn}) + defer nukeAllEc2Instances(awsSession, []*string{instance.InstanceId}) + + // Finally, define the task and service + taskDefinition := createEcsTaskDefinition(t, awsSession, taskFamilyName, "EC2") + defer deleteEcsTaskDefinition(awsSession, taskDefinition) + + service := createEcsService(t, awsSession, serviceName, cluster, "EC2", taskDefinition) + ecsServiceClusterMap[*service.ServiceArn] = *cluster.ClusterArn + // END prepare resources + + err = nukeAllEcsServices(awsSession, ecsServiceClusterMap, []*string{service.ServiceArn}) + + ecsServiceArns, _, err := getAllEcsServices(awsSession, []*string{cluster.ClusterArn}, time.Now().Add(1*time.Hour)) + if err != nil { + assert.Failf(t, "Unable to fetch list of services: %s", err.Error()) + } + assert.NotContains(t, awsgo.StringValueSlice(ecsServiceArns), *service.ServiceArn) +} diff --git a/aws/ecs_service_types.go b/aws/ecs_service_types.go new file mode 100644 index 00000000..7f49c510 --- /dev/null +++ b/aws/ecs_service_types.go @@ -0,0 +1,35 @@ +package aws + +import ( + awsgo "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/gruntwork-io/gruntwork-cli/errors" +) + +// ECSServices - Represents all ECS services found in a region +type ECSServices struct { + Services []string + ServiceClusterMap map[string]string +} + +// ResourceName - The simple name of the aws resource +func (services ECSServices) ResourceName() string { + return "ecsserv" +} + +// ResourceIdentifiers - The ARNs of the collected ECS services +func (services ECSServices) ResourceIdentifiers() []string { + return services.Services +} + +func (services ECSServices) MaxBatchSize() int { + return 200 +} + +// Nuke - nuke all ECS service resources +func (services ECSServices) Nuke(awsSession *session.Session, identifiers []string) error { + if err := nukeAllEcsServices(awsSession, services.ServiceClusterMap, awsgo.StringSlice(identifiers)); err != nil { + return errors.WithStackTrace(err) + } + return nil +} diff --git a/aws/ecs_utils_for_test.go b/aws/ecs_utils_for_test.go new file mode 100644 index 00000000..580b03c4 --- /dev/null +++ b/aws/ecs_utils_for_test.go @@ -0,0 +1,325 @@ +package aws + +import ( + "encoding/base64" + "errors" + "fmt" + "math/rand" + "testing" + "time" + + awsgo "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/aws/aws-sdk-go/service/iam" + gruntworkerrors "github.com/gruntwork-io/gruntwork-cli/errors" + "github.com/stretchr/testify/assert" +) + +// getRandomFargateSupportedRegion - Returns a random AWS +// region that supports Fargate. +// Refer to https://aws.amazon.com/about-aws/global-infrastructure/regional-product-services/ +func getRandomFargateSupportedRegion() string { + supportedRegions := []string{ + "us-east-1", "us-east-2", "us-west-2", + "eu-central-1", "eu-west-1", + "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", + } + rand.Seed(time.Now().UnixNano()) + randIndex := rand.Intn(len(supportedRegions)) + return supportedRegions[randIndex] +} + +func createEcsFargateCluster(t *testing.T, awsSession *session.Session, name string) ecs.Cluster { + svc := ecs.New(awsSession) + result, err := svc.CreateCluster(&ecs.CreateClusterInput{ClusterName: awsgo.String(name)}) + if err != nil { + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) + } + return *result.Cluster +} + +func createEcsEC2Cluster(t *testing.T, awsSession *session.Session, name string, instanceProfile iam.InstanceProfile) (ecs.Cluster, ec2.Instance) { + cluster := createEcsFargateCluster(t, awsSession, name) + + ec2Svc := ec2.New(awsSession) + imageID, err := getAMIIdByName(ec2Svc, "amzn-ami-2018.03.g-amazon-ecs-optimized") + if err != nil { + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) + } + + rawUserDataText := fmt.Sprintf("#!/bin/bash\necho 'ECS_CLUSTER=%s' >> /etc/ecs/ecs.config", *cluster.ClusterName) + userDataText := base64.StdEncoding.EncodeToString([]byte(rawUserDataText)) + + instanceProfileSpecification := &ec2.IamInstanceProfileSpecification{ + Arn: instanceProfile.Arn, + } + params := &ec2.RunInstancesInput{ + ImageId: awsgo.String(imageID), + InstanceType: awsgo.String("t2.micro"), + MinCount: awsgo.Int64(1), + MaxCount: awsgo.Int64(1), + DisableApiTermination: awsgo.Bool(false), + IamInstanceProfile: instanceProfileSpecification, + UserData: awsgo.String(userDataText), + } + instance, err := runAndWaitForInstance(ec2Svc, name, params) + if err != nil { + assert.Fail(t, err.Error()) + } + + // At this point we assume the instance successfully + // registered itself to the cluster + return cluster, instance +} + +func createEcsService(t *testing.T, awsSession *session.Session, serviceName string, cluster ecs.Cluster, launchType string, taskDefinition ecs.TaskDefinition) ecs.Service { + svc := ecs.New(awsSession) + createServiceParams := &ecs.CreateServiceInput{ + Cluster: cluster.ClusterArn, + DesiredCount: awsgo.Int64(1), + LaunchType: awsgo.String(launchType), + ServiceName: awsgo.String(serviceName), + TaskDefinition: taskDefinition.TaskDefinitionArn, + } + if launchType == "FARGATE" { + vpcConfiguration, err := getVpcConfiguration(awsSession) + if err != nil { + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) + } + networkConfiguration := &ecs.NetworkConfiguration{ + AwsvpcConfiguration: &vpcConfiguration, + } + createServiceParams.SetNetworkConfiguration(networkConfiguration) + } + result, err := svc.CreateService(createServiceParams) + if err != nil { + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) + } + return *result.Service +} + +func createEcsTaskDefinition(t *testing.T, awsSession *session.Session, taskFamilyName string, launchType string) ecs.TaskDefinition { + svc := ecs.New(awsSession) + containerDefinition := &ecs.ContainerDefinition{ + Image: awsgo.String("nginx:latest"), + Name: awsgo.String("nginx"), + } + registerTaskParams := &ecs.RegisterTaskDefinitionInput{ + ContainerDefinitions: []*ecs.ContainerDefinition{containerDefinition}, + Cpu: awsgo.String("256"), + Memory: awsgo.String("512"), + Family: awsgo.String(taskFamilyName), + } + if launchType == "FARGATE" { + registerTaskParams.SetNetworkMode("awsvpc") + registerTaskParams.SetRequiresCompatibilities([]*string{awsgo.String("FARGATE")}) + } + result, err := svc.RegisterTaskDefinition(registerTaskParams) + if err != nil { + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) + } + return *result.TaskDefinition +} + +func deleteEcsTaskDefinition(awsSession *session.Session, taskDefinition ecs.TaskDefinition) error { + svc := ecs.New(awsSession) + deregisterTaskDefinitionParams := &ecs.DeregisterTaskDefinitionInput{ + TaskDefinition: taskDefinition.TaskDefinitionArn, + } + _, err := svc.DeregisterTaskDefinition(deregisterTaskDefinitionParams) + if err != nil { + return gruntworkerrors.WithStackTrace(err) + } + return nil +} + +func createEcsInstanceProfile(t *testing.T, awsSession *session.Session, instanceProfileName string, role iam.Role) iam.InstanceProfile { + svc := iam.New(awsSession) + createInstanceProfileParams := &iam.CreateInstanceProfileInput{ + InstanceProfileName: awsgo.String(instanceProfileName), + } + result, err := svc.CreateInstanceProfile(createInstanceProfileParams) + if err != nil { + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) + } + instanceProfile := result.InstanceProfile + addRoleToInstanceProfileParams := &iam.AddRoleToInstanceProfileInput{ + InstanceProfileName: instanceProfile.InstanceProfileName, + RoleName: role.RoleName, + } + _, err = svc.AddRoleToInstanceProfile(addRoleToInstanceProfileParams) + if err != nil { + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) + } + return *instanceProfile +} + +func deleteInstanceProfile(awsSession *session.Session, instanceProfile iam.InstanceProfile) error { + svc := iam.New(awsSession) + getInstanceProfileParams := &iam.GetInstanceProfileInput{ + InstanceProfileName: instanceProfile.InstanceProfileName, + } + result, err := svc.GetInstanceProfile(getInstanceProfileParams) + if err != nil { + return gruntworkerrors.WithStackTrace(err) + } + refreshedInstanceProfile := result.InstanceProfile + for _, role := range refreshedInstanceProfile.Roles { + removeRoleParams := &iam.RemoveRoleFromInstanceProfileInput{ + InstanceProfileName: refreshedInstanceProfile.InstanceProfileName, + RoleName: role.RoleName, + } + _, err := svc.RemoveRoleFromInstanceProfile(removeRoleParams) + if err != nil { + return gruntworkerrors.WithStackTrace(err) + } + } + deleteInstanceProfileParams := &iam.DeleteInstanceProfileInput{ + InstanceProfileName: refreshedInstanceProfile.InstanceProfileName, + } + _, err = svc.DeleteInstanceProfile(deleteInstanceProfileParams) + if err != nil { + return gruntworkerrors.WithStackTrace(err) + } + return nil + +} + +func createEcsRole(t *testing.T, awsSession *session.Session, roleName string) iam.Role { + svc := iam.New(awsSession) + createRoleParams := &iam.CreateRoleInput{ + AssumeRolePolicyDocument: awsgo.String(ECS_ASSUME_ROLE_POLICY), + RoleName: awsgo.String(roleName), + } + result, err := svc.CreateRole(createRoleParams) + if err != nil { + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) + } + putRolePolicyParams := &iam.PutRolePolicyInput{ + RoleName: awsgo.String(roleName), + PolicyDocument: awsgo.String(ECS_ROLE_POLICY), + PolicyName: awsgo.String(roleName + "Policy"), + } + _, err = svc.PutRolePolicy(putRolePolicyParams) + if err != nil { + assert.Fail(t, gruntworkerrors.WithStackTrace(err).Error()) + } + return *result.Role +} + +func deleteRole(awsSession *session.Session, role iam.Role) error { + svc := iam.New(awsSession) + listRolePoliciesParams := &iam.ListRolePoliciesInput{ + RoleName: role.RoleName, + } + result, err := svc.ListRolePolicies(listRolePoliciesParams) + if err != nil { + return gruntworkerrors.WithStackTrace(err) + } + for _, policyName := range result.PolicyNames { + deleteRolePolicyParams := &iam.DeleteRolePolicyInput{ + RoleName: role.RoleName, + PolicyName: policyName, + } + _, err := svc.DeleteRolePolicy(deleteRolePolicyParams) + if err != nil { + return gruntworkerrors.WithStackTrace(err) + } + } + deleteRoleParams := &iam.DeleteRoleInput{ + RoleName: role.RoleName, + } + _, err = svc.DeleteRole(deleteRoleParams) + if err != nil { + return gruntworkerrors.WithStackTrace(err) + } + return nil +} + +func getVpcConfiguration(awsSession *session.Session) (ecs.AwsVpcConfiguration, error) { + ec2Svc := ec2.New(awsSession) + describeVpcsParams := &ec2.DescribeVpcsInput{ + Filters: []*ec2.Filter{ + &ec2.Filter{ + Name: awsgo.String("isDefault"), + Values: []*string{awsgo.String("true")}, + }, + }, + } + vpcs, err := ec2Svc.DescribeVpcs(describeVpcsParams) + if err != nil { + return ecs.AwsVpcConfiguration{}, gruntworkerrors.WithStackTrace(err) + } + if len(vpcs.Vpcs) == 0 { + err := errors.New(fmt.Sprintf("Could not find any default VPC in region %s", *awsSession.Config.Region)) + return ecs.AwsVpcConfiguration{}, gruntworkerrors.WithStackTrace(err) + } + defaultVpc := vpcs.Vpcs[0] + + describeSubnetsParams := &ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{ + &ec2.Filter{ + Name: awsgo.String("vpc-id"), + Values: []*string{defaultVpc.VpcId}, + }, + }, + } + subnets, err := ec2Svc.DescribeSubnets(describeSubnetsParams) + if err != nil { + return ecs.AwsVpcConfiguration{}, gruntworkerrors.WithStackTrace(err) + } + if len(subnets.Subnets) == 0 { + err := errors.New(fmt.Sprintf("Could not find any subnets for default VPC in region %s", *awsSession.Config.Region)) + return ecs.AwsVpcConfiguration{}, gruntworkerrors.WithStackTrace(err) + } + var subnetIds []*string + for _, subnet := range subnets.Subnets { + subnetIds = append(subnetIds, subnet.SubnetId) + } + return ecs.AwsVpcConfiguration{Subnets: subnetIds}, nil +} + +const ECS_ASSUME_ROLE_POLICY = `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +}` + +const ECS_ROLE_POLICY = `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:DescribeRepositories", + "ecr:GetAuthorizationToken", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:ListImages", + "ecs:CreateCluster", + "ecs:DeregisterContainerInstance", + "ecs:DiscoverPollEndpoint", + "ecs:Poll", + "ecs:RegisterContainerInstance", + "ecs:StartTask", + "ecs:StartTelemetrySession", + "ecs:SubmitContainerStateChange", + "ecs:SubmitTaskStateChange" + ], + "Resource": [ + "*" + ] + } + ] +}` From c088194eb3ed63ec04ea5def80eb8e969601af53 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Thu, 11 Oct 2018 08:23:01 -0700 Subject: [PATCH 2/4] Amend comments on cluster filtering to be more clear --- aws/ecs_cluster.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/aws/ecs_cluster.go b/aws/ecs_cluster.go index 07948112..3d5f6920 100644 --- a/aws/ecs_cluster.go +++ b/aws/ecs_cluster.go @@ -8,9 +8,15 @@ import ( ) // getAllEcsClusters - Returns a string of ECS Cluster ARNs, which uniquely identifies the cluster. -// NOTE: AWS api doesn't provide the necessary information to -// implement `excludeAfter` filter at the cluster -// level, so we will implement it at the service level. +// NOTE: +// AWS api doesn't provide the necessary information to implement +// `excludeAfter` filter at the cluster level, so we will implement it at the +// service level. Clusters can't be deleted if they have active services +// running on it. In practice this means that clusters that have recently +// been used by launching services to it will not be deleted because it will +// still have services running on it, since the services will be filtered out +// using `excludeAfter`. However, recently created clusters could still be +// deleted if it has not been used yet by deploying a service to it. func getAllEcsClusters(awsSession *session.Session) ([]*string, error) { svc := ecs.New(awsSession) result, err := svc.ListClusters(&ecs.ListClustersInput{}) From 9c2182a59ad2cad48aeda43f1adb85a3180d0c2c Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Thu, 11 Oct 2018 11:19:43 -0700 Subject: [PATCH 3/4] Refactor nukeAllEcsServices --- aws/ecs_service.go | 90 +++++++++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/aws/ecs_service.go b/aws/ecs_service.go index 9be421da..9271a444 100644 --- a/aws/ecs_service.go +++ b/aws/ecs_service.go @@ -66,28 +66,10 @@ func getAllEcsServices(awsSession *session.Session, ecsClusterArns []*string, ex return ecsServiceArns, ecsServiceClusterMap, nil } -// Deletes all provided ECS Services. At a high level this involves two steps: -// 1.) Drain all tasks from the service so that nothing is -// running. -// 2.) Delete service object once no tasks are running. -// Note that this will swallow failed deletes and continue along, logging the -// service ARN so that we can find it later. -func nukeAllEcsServices(awsSession *session.Session, ecsServiceClusterMap map[string]string, ecsServiceArns []*string) error { - numNuking := len(ecsServiceArns) - svc := ecs.New(awsSession) - - if numNuking == 0 { - logging.Logger.Infof("No ECS services to nuke in region %s", *awsSession.Config.Region) - return nil - } - - logging.Logger.Infof("Deleting %d ECS services in region %s", numNuking, *awsSession.Config.Region) - - // First, drain all the services to 0. You can't delete a - // service that is running tasks. - // Note that we request all the drains at once, and then - // wait for them in a separate loop because it will take a - // while to drain the services. +// drainEcsServices - Drain all tasks from all services requested. This will +// return a list of service ARNs that have been successfully requested to be +// drained. +func drainEcsServices(svc *ecs.ECS, ecsServiceClusterMap map[string]string, ecsServiceArns []*string) []*string { var requestedDrains []*string for _, ecsServiceArn := range ecsServiceArns { params := &ecs.UpdateServiceInput{ @@ -102,12 +84,16 @@ func nukeAllEcsServices(awsSession *session.Session, ecsServiceClusterMap map[st requestedDrains = append(requestedDrains, ecsServiceArn) } } + return requestedDrains +} - // Wait until service is fully drained by waiting for - // stability, which is defined as desiredCount == - // runningCount +// waitUntilServiceDrained - Waits until all tasks have been drained from the +// given list of services, by waiting for stability which is defined as +// desiredCount == runningCount. This will return a list of service ARNs that +// have successfully been drained. +func waitUntilServicesDrained(svc *ecs.ECS, ecsServiceClusterMap map[string]string, ecsServiceArns []*string) []*string { var successfullyDrained []*string - for _, ecsServiceArn := range requestedDrains { + for _, ecsServiceArn := range ecsServiceArns { params := &ecs.DescribeServicesInput{ Cluster: awsgo.String(ecsServiceClusterMap[*ecsServiceArn]), Services: []*string{ecsServiceArn}, @@ -120,10 +106,14 @@ func nukeAllEcsServices(awsSession *session.Session, ecsServiceClusterMap map[st successfullyDrained = append(successfullyDrained, ecsServiceArn) } } + return successfullyDrained +} - // Now delete the services that were successfully drained +// deleteEcsServices - Deletes all services requested. Returns a list of +// service ARNs that have been accepted by AWS for deletion. +func deleteEcsServices(svc *ecs.ECS, ecsServiceClusterMap map[string]string, ecsServiceArns []*string) []*string { var requestedDeletes []*string - for _, ecsServiceArn := range successfullyDrained { + for _, ecsServiceArn := range ecsServiceArns { params := &ecs.DeleteServiceInput{ Cluster: awsgo.String(ecsServiceClusterMap[*ecsServiceArn]), Service: ecsServiceArn, @@ -135,10 +125,15 @@ func nukeAllEcsServices(awsSession *session.Session, ecsServiceClusterMap map[st requestedDeletes = append(requestedDeletes, ecsServiceArn) } } + return requestedDeletes +} - // Wait until services are deleted - numNuked := 0 - for _, ecsServiceArn := range requestedDeletes { +// waitUntilServicesDeleted - Waits until the service has been actually deleted +// from AWS. Returns a list of service ARNs that have been successfully +// deleted. +func waitUntilServicesDeleted(svc *ecs.ECS, ecsServiceClusterMap map[string]string, ecsServiceArns []*string) []*string { + var successfullyDeleted []*string + for _, ecsServiceArn := range ecsServiceArns { params := &ecs.DescribeServicesInput{ Cluster: awsgo.String(ecsServiceClusterMap[*ecsServiceArn]), Services: []*string{ecsServiceArn}, @@ -148,10 +143,41 @@ func nukeAllEcsServices(awsSession *session.Session, ecsServiceClusterMap map[st logging.Logger.Errorf("[Failed] Failed waiting for service to be deleted %s: %s", *ecsServiceArn, err) } else { logging.Logger.Infof("Deleted service: %s", *ecsServiceArn) - numNuked += 1 + successfullyDeleted = append(successfullyDeleted, ecsServiceArn) } } + return successfullyDeleted +} + +// Deletes all provided ECS Services. At a high level this involves two steps: +// 1.) Drain all tasks from the service so that nothing is +// running. +// 2.) Delete service object once no tasks are running. +// Note that this will swallow failed deletes and continue along, logging the +// service ARN so that we can find it later. +func nukeAllEcsServices(awsSession *session.Session, ecsServiceClusterMap map[string]string, ecsServiceArns []*string) error { + numNuking := len(ecsServiceArns) + svc := ecs.New(awsSession) + + if numNuking == 0 { + logging.Logger.Infof("No ECS services to nuke in region %s", *awsSession.Config.Region) + return nil + } + + logging.Logger.Infof("Deleting %d ECS services in region %s", numNuking, *awsSession.Config.Region) + + // First, drain all the services to 0. You can't delete a + // service that is running tasks. + // Note that we request all the drains at once, and then + // wait for them in a separate loop because it will take a + // while to drain the services. + // Then, we delete the services that have been successfully drained. + requestedDrains := drainEcsServices(svc, ecsServiceClusterMap, ecsServiceArns) + successfullyDrained := waitUntilServicesDrained(svc, ecsServiceClusterMap, requestedDrains) + requestedDeletes := deleteEcsServices(svc, ecsServiceClusterMap, successfullyDrained) + successfullyDeleted := waitUntilServicesDeleted(svc, ecsServiceClusterMap, requestedDeletes) + numNuked := len(successfullyDeleted) logging.Logger.Infof("[OK] %d of %d ECS service(s) deleted in %s", numNuked, numNuking, *awsSession.Config.Region) return nil } From 5487ffd371f4f3546189bba1db7f1e6c176566ac Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Thu, 11 Oct 2018 16:04:38 -0700 Subject: [PATCH 4/4] Remove support for deleting ECS clusters --- README.md | 10 +++++- aws/aws.go | 5 --- aws/ecs_cluster.go | 55 --------------------------------- aws/ecs_cluster_test.go | 64 --------------------------------------- aws/ecs_cluster_types.go | 34 --------------------- aws/ecs_service.go | 11 +++++++ aws/ecs_service_test.go | 8 ++--- aws/ecs_utils_for_test.go | 10 ++++++ 8 files changed, 34 insertions(+), 163 deletions(-) delete mode 100644 aws/ecs_cluster.go delete mode 100644 aws/ecs_cluster_test.go delete mode 100644 aws/ecs_cluster_types.go diff --git a/README.md b/README.md index 35e92212..129426bd 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,15 @@ The currently supported functionality includes: * Deleting all Elastic IPs in an AWS account * Deleting all Launch Configurations in an AWS account * Deleting all ECS services in an AWS account -* Deleting all ECS clusters in an AWS account + +### Caveats + +* We currently do not support deleting ECS clusters because AWS + does not give us a good way to blacklist clusters off the list (there are not + tags and we do not know the creation timestamp). Given the destructive nature + of the tool, we have opted not to support deleting ECS clusters at the + moment. See https://github.com/gruntwork-io/cloud-nuke/pull/36 for a more + detailed discussion. ## Azure diff --git a/aws/aws.go b/aws/aws.go index a1dc921f..94abee37 100644 --- a/aws/aws.go +++ b/aws/aws.go @@ -215,16 +215,11 @@ func GetAllResources(regions []string, excludedRegions []string, excludeAfter ti return nil, errors.WithStackTrace(err) } - // Must delete services before clusters ecsServices := ECSServices{ Services: awsgo.StringValueSlice(serviceArns), ServiceClusterMap: serviceClusterMap, } resourcesInRegion.Resources = append(resourcesInRegion.Resources, ecsServices) - ecsClusters := ECSClusters{ - Clusters: awsgo.StringValueSlice(clusterArns), - } - resourcesInRegion.Resources = append(resourcesInRegion.Resources, ecsClusters) // End ECS resources account.Resources[region] = resourcesInRegion diff --git a/aws/ecs_cluster.go b/aws/ecs_cluster.go deleted file mode 100644 index 3d5f6920..00000000 --- a/aws/ecs_cluster.go +++ /dev/null @@ -1,55 +0,0 @@ -package aws - -import ( - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ecs" - "github.com/gruntwork-io/cloud-nuke/logging" - "github.com/gruntwork-io/gruntwork-cli/errors" -) - -// getAllEcsClusters - Returns a string of ECS Cluster ARNs, which uniquely identifies the cluster. -// NOTE: -// AWS api doesn't provide the necessary information to implement -// `excludeAfter` filter at the cluster level, so we will implement it at the -// service level. Clusters can't be deleted if they have active services -// running on it. In practice this means that clusters that have recently -// been used by launching services to it will not be deleted because it will -// still have services running on it, since the services will be filtered out -// using `excludeAfter`. However, recently created clusters could still be -// deleted if it has not been used yet by deploying a service to it. -func getAllEcsClusters(awsSession *session.Session) ([]*string, error) { - svc := ecs.New(awsSession) - result, err := svc.ListClusters(&ecs.ListClustersInput{}) - if err != nil { - return nil, errors.WithStackTrace(err) - } - return result.ClusterArns, nil -} - -// Deletes all given ECS clusters. Note that this will swallow failed deletes -// and continue along, logging the cluster ARN so that we can find it later. -func nukeAllEcsClusters(awsSession *session.Session, ecsClusterArns []*string) error { - numNuking := len(ecsClusterArns) - svc := ecs.New(awsSession) - - if numNuking == 0 { - logging.Logger.Infof("No ECS clusters to nuke in region %s", *awsSession.Config.Region) - return nil - } - - logging.Logger.Infof("Deleting %d ECS clusters in region %s", numNuking, *awsSession.Config.Region) - - numNuked := 0 - for _, ecsClusterArn := range ecsClusterArns { - params := &ecs.DeleteClusterInput{Cluster: ecsClusterArn} - _, err := svc.DeleteCluster(params) - if err != nil { - logging.Logger.Errorf("[Failed] Could not delete cluster %s: %s", *ecsClusterArn, err.Error()) - } else { - logging.Logger.Infof("Deleted cluster: %s", *ecsClusterArn) - numNuked += 1 - } - } - logging.Logger.Infof("[OK] %d of %d ECS cluster(s) deleted in %s", numNuked, numNuking, *awsSession.Config.Region) - return nil -} diff --git a/aws/ecs_cluster_test.go b/aws/ecs_cluster_test.go deleted file mode 100644 index 2ec44f87..00000000 --- a/aws/ecs_cluster_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package aws - -import ( - "testing" - - awsgo "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/gruntwork-io/cloud-nuke/util" - "github.com/gruntwork-io/gruntwork-cli/errors" - "github.com/stretchr/testify/assert" -) - -// Test that we can find ECS clusters -func TestListECSClusters(t *testing.T) { - t.Parallel() - - region := getRandomFargateSupportedRegion() - awsSession, err := session.NewSession(&awsgo.Config{ - Region: awsgo.String(region)}, - ) - - if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) - } - - uniqueTestID := "cloud-nuke-test-" + util.UniqueID() - clusterName := uniqueTestID + "-cluster" - cluster := createEcsFargateCluster(t, awsSession, clusterName) - defer nukeAllEcsClusters(awsSession, []*string{cluster.ClusterArn}) - - clusterArns, err := getAllEcsClusters(awsSession) - if err != nil { - assert.Failf(t, "Unable to fetch clusters: %s", err.Error()) - } - assert.Contains(t, clusterArns, cluster.ClusterArn) -} - -// Test that we can successfully nuke ECS clusters -func TestNukeECSClusters(t *testing.T) { - t.Parallel() - - region := getRandomFargateSupportedRegion() - awsSession, err := session.NewSession(&awsgo.Config{ - Region: awsgo.String(region), - }) - - if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) - } - - uniqueTestID := "cloud-nuke-test-" + util.UniqueID() - clusterName := uniqueTestID + "-cluster" - - cluster := createEcsFargateCluster(t, awsSession, clusterName) - if err := nukeAllEcsClusters(awsSession, []*string{cluster.ClusterArn}); err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) - } - - clusterArns, err := getAllEcsClusters(awsSession) - if err != nil { - assert.Failf(t, "Unable to fetch clusters: %s", err.Error()) - } - assert.NotContains(t, clusterArns, cluster.ClusterArn) -} diff --git a/aws/ecs_cluster_types.go b/aws/ecs_cluster_types.go deleted file mode 100644 index cfca1ad9..00000000 --- a/aws/ecs_cluster_types.go +++ /dev/null @@ -1,34 +0,0 @@ -package aws - -import ( - awsgo "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/gruntwork-io/gruntwork-cli/errors" -) - -// ECSClusters - Represents all ECS clusters found in a region -type ECSClusters struct { - Clusters []string -} - -// ResourceName - The simple name of the aws resource -func (clusters ECSClusters) ResourceName() string { - return "ecsclust" -} - -// ResourceIdentifiers - The ARNs of the collected ECS clusters -func (clusters ECSClusters) ResourceIdentifiers() []string { - return clusters.Clusters -} - -func (clusters ECSClusters) MaxBatchSize() int { - return 200 -} - -// Nuke - nuke all ECS service resources -func (clusters ECSClusters) Nuke(awsSession *session.Session, identifiers []string) error { - if err := nukeAllEcsClusters(awsSession, awsgo.StringSlice(identifiers)); err != nil { - return errors.WithStackTrace(err) - } - return nil -} diff --git a/aws/ecs_service.go b/aws/ecs_service.go index 9271a444..1bfbc6e9 100644 --- a/aws/ecs_service.go +++ b/aws/ecs_service.go @@ -10,6 +10,17 @@ import ( "github.com/gruntwork-io/gruntwork-cli/errors" ) +// getAllEcsClusters - Returns a string of ECS Cluster ARNs, which uniquely identifies the cluster. +// We need to get all clusters before we can get all services. +func getAllEcsClusters(awsSession *session.Session) ([]*string, error) { + svc := ecs.New(awsSession) + result, err := svc.ListClusters(&ecs.ListClustersInput{}) + if err != nil { + return nil, errors.WithStackTrace(err) + } + return result.ClusterArns, nil +} + // filterOutRecentServices - Given a list of services and an excludeAfter // timestamp, filter out any services that were created after `excludeAfter`. func filterOutRecentServices(svc *ecs.ECS, clusterArn *string, ecsServiceArns []string, excludeAfter time.Time) ([]*string, error) { diff --git a/aws/ecs_service_test.go b/aws/ecs_service_test.go index a03d3946..6293b6fa 100644 --- a/aws/ecs_service_test.go +++ b/aws/ecs_service_test.go @@ -31,7 +31,7 @@ func TestListECSFargateServices(t *testing.T) { taskFamilyName := uniqueTestID + "-task" cluster := createEcsFargateCluster(t, awsSession, clusterName) - defer nukeAllEcsClusters(awsSession, []*string{cluster.ClusterArn}) + defer deleteEcsCluster(awsSession, cluster) taskDefinition := createEcsTaskDefinition(t, awsSession, taskFamilyName, "FARGATE") defer deleteEcsTaskDefinition(awsSession, taskDefinition) @@ -76,7 +76,7 @@ func TestNukeECSFargateServices(t *testing.T) { taskFamilyName := uniqueTestID + "-task" cluster := createEcsFargateCluster(t, awsSession, clusterName) - defer nukeAllEcsClusters(awsSession, []*string{cluster.ClusterArn}) + defer deleteEcsCluster(awsSession, cluster) taskDefinition := createEcsTaskDefinition(t, awsSession, taskFamilyName, "FARGATE") defer deleteEcsTaskDefinition(awsSession, taskDefinition) @@ -133,7 +133,7 @@ func TestListECSEC2Services(t *testing.T) { // Provision a cluster with ec2 container instances, not // forgetting to schedule deletion cluster, instance := createEcsEC2Cluster(t, awsSession, clusterName, instanceProfile) - defer nukeAllEcsClusters(awsSession, []*string{cluster.ClusterArn}) + defer deleteEcsCluster(awsSession, cluster) defer nukeAllEc2Instances(awsSession, []*string{instance.InstanceId}) // Finally, define the task and service @@ -198,7 +198,7 @@ func TestNukeECSEC2Services(t *testing.T) { // Provision a cluster with ec2 container instances, not // forgetting to schedule deletion cluster, instance := createEcsEC2Cluster(t, awsSession, clusterName, instanceProfile) - defer nukeAllEcsClusters(awsSession, []*string{cluster.ClusterArn}) + defer deleteEcsCluster(awsSession, cluster) defer nukeAllEc2Instances(awsSession, []*string{instance.InstanceId}) // Finally, define the task and service diff --git a/aws/ecs_utils_for_test.go b/aws/ecs_utils_for_test.go index 580b03c4..94bfa8ee 100644 --- a/aws/ecs_utils_for_test.go +++ b/aws/ecs_utils_for_test.go @@ -74,6 +74,16 @@ func createEcsEC2Cluster(t *testing.T, awsSession *session.Session, name string, return cluster, instance } +func deleteEcsCluster(awsSession *session.Session, cluster ecs.Cluster) error { + svc := ecs.New(awsSession) + params := &ecs.DeleteClusterInput{Cluster: cluster.ClusterArn} + _, err := svc.DeleteCluster(params) + if err != nil { + return gruntworkerrors.WithStackTrace(err) + } + return nil +} + func createEcsService(t *testing.T, awsSession *session.Session, serviceName string, cluster ecs.Cluster, launchType string, taskDefinition ecs.TaskDefinition) ecs.Service { svc := ecs.New(awsSession) createServiceParams := &ecs.CreateServiceInput{