diff --git a/pkg/destroy/aws/aws.go b/pkg/destroy/aws/aws.go index d30242bf7..2a7bad571 100644 --- a/pkg/destroy/aws/aws.go +++ b/pkg/destroy/aws/aws.go @@ -2,28 +2,26 @@ package aws import ( "fmt" - "os" - "time" + "strings" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elb" "github.com/aws/aws-sdk-go/service/elbv2" "github.com/aws/aws-sdk-go/service/iam" + "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" "github.com/aws/aws-sdk-go/service/route53" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/util/wait" ) -const ( - secondsToSleep = 10 +var ( + exists = struct{}{} ) // Filter holds the key/value pairs for the tags we will be matching against. @@ -31,18 +29,6 @@ const ( // A resource matches the filter if all of the key/value pairs are in its tags. type Filter map[string]string -// awsObjectWithTags is a generic way to represent an AWS object and its tags so that -// filtering objects client-side can be done in a generic way -type awsObjectWithTags struct { - Name string - Tags map[string]string -} - -// deleteFunc type is the interface a function needs to implement to be called as a goroutine. -// The (bool, error) return type mimics wait.ExponentialBackoff where the bool indicates successful -// completion, and the error is for unrecoverable errors. -type deleteFunc func(awsClient *session.Session, filters Filter, clusterName string, logger logrus.FieldLogger) (bool, error) - // ClusterUninstaller holds the various options for the cluster we want to delete type ClusterUninstaller struct { @@ -77,22 +63,6 @@ func (o *ClusterUninstaller) validate() error { return nil } -// populateDeleteFuncs is the list of functions that will be launched as goroutines -func populateDeleteFuncs(funcs map[string]deleteFunc) { - funcs["deleteVPCs"] = deleteVPCs - funcs["deleteEIPs"] = deleteEIPs - funcs["deleteNATGateways"] = deleteNATGateways - funcs["deleteInstances"] = deleteInstances - funcs["deleteIAMresources"] = deleteIAMresources - funcs["deleteSecurityGroups"] = deleteSecurityGroups - funcs["deleteInternetGateways"] = deleteInternetGateways - funcs["deleteSubnets"] = deleteSubnets - funcs["deleteS3Buckets"] = deleteS3Buckets - funcs["deleteRoute53"] = deleteRoute53 - funcs["deletePVs"] = deletePVs - funcs["deleteUsers"] = deleteUsers -} - // Run is the entrypoint to start the uninstall process func (o *ClusterUninstaller) Run() error { err := o.validate() @@ -100,1487 +70,1202 @@ func (o *ClusterUninstaller) Run() error { return err } - deleteFuncs := map[string]deleteFunc{} - populateDeleteFuncs(deleteFuncs) - returnChannel := make(chan string) + awsConfig := &aws.Config{Region: aws.String(o.Region)} - awsSession, err := getAWSSession(o.Region) + // Relying on appropriate AWS ENV vars (eg AWS_PROFILE, AWS_ACCESS_KEY_ID, etc) + awsSession, err := session.NewSession(awsConfig) if err != nil { return err } - // launch goroutines - goroutines := 0 - for name, function := range deleteFuncs { - for _, filter := range o.Filters { - go deleteRunner(name, function, awsSession, filter, o.ClusterName, o.Logger, returnChannel) - goroutines++ - } - } + tagClients := []*resourcegroupstaggingapi.ResourceGroupsTaggingAPI{ + resourcegroupstaggingapi.New(awsSession), + } + tagClientNames := map[*resourcegroupstaggingapi.ResourceGroupsTaggingAPI]string{ + tagClients[0]: o.Region, + } + if o.Region != "us-east-1" { + tagClient := resourcegroupstaggingapi.New( + awsSession, aws.NewConfig().WithRegion("us-east-1"), + ) + tagClients = append(tagClients, tagClient) + tagClientNames[tagClient] = "us-east-1" + } + + deleted := map[string]struct{}{} + iamClient := iam.New(awsSession) + iamRoleSearch := &iamRoleSearch{ + client: iamClient, + filters: o.Filters, + logger: o.Logger, + } + iamUserSearch := &iamUserSearch{ + client: iamClient, + filters: o.Filters, + logger: o.Logger, + } + + var loopError error + for len(tagClients) > 0 || loopError != nil { + loopError = nil + nextTagClients := tagClients[:0] + for _, tagClient := range tagClients { + matched := false + for _, filter := range o.Filters { + o.Logger.Debugf("search for and delete matching resources by tag in %s matching %#+v", tagClientNames[tagClient], filter) + tagFilters := make([]*resourcegroupstaggingapi.TagFilter, 0, len(filter)) + for key, value := range filter { + tagFilters = append(tagFilters, &resourcegroupstaggingapi.TagFilter{ + Key: aws.String(key), + Values: []*string{aws.String(value)}, + }) + } + err = tagClient.GetResourcesPages( + &resourcegroupstaggingapi.GetResourcesInput{TagFilters: tagFilters}, + func(results *resourcegroupstaggingapi.GetResourcesOutput, lastPage bool) bool { + for _, resource := range results.ResourceTagMappingList { + arn := *resource.ResourceARN + if _, ok := deleted[arn]; !ok { + matched = true + err := deleteARN(awsSession, arn, o.Logger) + if err != nil { + err = errors.Wrapf(err, "deleting %s", arn) + o.Logger.Debug(err) + continue + } + deleted[arn] = exists + } + } + + return !lastPage + }, + ) + if err != nil { + err = errors.Wrapf(err, "get tagged resources") + o.Logger.Info(err) + loopError = err + } + } - // wait for them to finish - for goroutines > 0 { - select { - case res := <-returnChannel: - goroutines-- - o.Logger.Debugf("goroutine %v complete (%d left)", res, goroutines) + if matched { + nextTagClients = append(nextTagClients, tagClient) + } else { + o.Logger.Debugf("no deletions from %s, removing client", tagClientNames[tagClient]) + } } - } + tagClients = nextTagClients - return nil -} - -func deleteRunner(deleteFuncName string, dFunction deleteFunc, awsSession *session.Session, filters Filter, clusterName string, logger logrus.FieldLogger, channel chan string) { - backoffSettings := wait.Backoff{ - Duration: time.Second * 10, - Factor: 1.3, - Steps: 100, - } - - err := wait.ExponentialBackoff(backoffSettings, func() (bool, error) { - return dFunction(awsSession, filters, clusterName, logger) - }) - - if err != nil { - logger.Fatalf("Unrecoverable error/timed out: %v", err) - os.Exit(1) - } - - // record that the goroutine has run to completion - channel <- deleteFuncName - return -} + o.Logger.Debug("search for IAM roles") + arns, err := iamRoleSearch.arns() + if err != nil { + o.Logger.Info(err) + loopError = err + } -func getAWSSession(region string) (*session.Session, error) { - awsConfig := &aws.Config{Region: aws.String(region)} + o.Logger.Debug("search for IAM users") + userARNs, err := iamUserSearch.arns() + if err != nil { + o.Logger.Info(err) + loopError = err + } + arns = append(arns, userARNs...) - // Relying on appropriate AWS ENV vars (eg AWS_PROFILE, AWS_ACCESS_KEY_ID, etc) - s, err := session.NewSession(awsConfig) - if err != nil { - return nil, err + if len(arns) > 0 { + o.Logger.Debug("delete IAM roles and users") + } + for _, arn := range arns { + if _, ok := deleted[arn]; !ok { + err = deleteARN(awsSession, arn, o.Logger) + if err != nil { + err = errors.Wrapf(err, "deleting %s", arn) + o.Logger.Debug(err) + loopError = err + continue + } + deleted[arn] = exists + } + } } - return s, nil + return nil } -func createEC2Filters(filters Filter) []*ec2.Filter { - Filter := []*ec2.Filter{} - for key, val := range filters { - Filter = append(Filter, &ec2.Filter{ - Name: aws.String(fmt.Sprintf("tag:%s", key)), - Values: []*string{aws.String(val)}, - }) +func splitSlash(name string, input string) (base string, suffix string, err error) { + segments := strings.SplitN(input, "/", 2) + if len(segments) != 2 { + return "", "", errors.Errorf("%s %q does not contain the expected slash", name, input) } - - return Filter + return segments[0], segments[1], nil } -// tagsToMap takes various types of AWS-object tags and returns a map-representation -func tagsToMap(tags interface{}) (map[string]string, error) { - x := map[string]string{} - - switch v := tags.(type) { - case []*autoscaling.TagDescription: - for _, tag := range v { - x[*tag.Key] = *tag.Value - } - case *elb.TagDescription: - for _, tag := range v.Tags { - x[*tag.Key] = *tag.Value - } - case []*s3.Tag: - for _, tag := range v { - x[*tag.Key] = *tag.Value - } - case []*route53.Tag: - for _, tag := range v { - x[*tag.Key] = *tag.Value +func tagMatch(filters []Filter, tags map[string]string) bool { + for _, filter := range filters { + match := true + for filterKey, filterValue := range filter { + tagValue, ok := tags[filterKey] + if !ok { + match = false + break + } + if tagValue != filterValue { + match = false + break + } } - case []*iam.Tag: - for _, tag := range v { - x[*tag.Key] = *tag.Value + if match { + return true } - default: - return x, errors.Errorf("unable to convert type: %v", v) } - - return x, nil + return len(filters) == 0 } -// filterLBsByVPC will find all the load balancers in the provided list that are under the provided VPC -func filterLBsByVPC(lbs []*elb.LoadBalancerDescription, vpc *ec2.Vpc, logger logrus.FieldLogger) []*elb.LoadBalancerDescription { - filteredLBs := []*elb.LoadBalancerDescription{} - - for _, lb := range lbs { - if *lb.VPCId == *vpc.VpcId { - filteredLBs = append(filteredLBs, lb) - } - } - - return filteredLBs +type iamRoleSearch struct { + client *iam.IAM + filters []Filter + logger logrus.FieldLogger + unmatched map[string]struct{} } -// deleteLBs finds all load balancers under the provided VPC and attempts to delete them -// returns bool representing whether it has completed its work (ie no LBs left to delete) -func deleteLBs(vpc *ec2.Vpc, awsSession *session.Session, logger logrus.FieldLogger) bool { - logger.Debugf("Deleting load balancers (%s)", *vpc.VpcId) - defer logger.Debugf("Exiting deleting load balancers (%s)", *vpc.VpcId) - elbClient := elb.New(awsSession) - - describeLoadBalancersInput := elb.DescribeLoadBalancersInput{} - results, err := elbClient.DescribeLoadBalancers(&describeLoadBalancersInput) - if err != nil { - logger.Errorf("Error listing load balancers: %v", err) - return false +func (search *iamRoleSearch) arns() ([]string, error) { + if search.unmatched == nil { + search.unmatched = map[string]struct{}{} } - filteredLBs := filterLBsByVPC(results.LoadBalancerDescriptions, vpc, logger) - logger.Debugf("from %d total load balancers, %d scheduled for deletion", len(results.LoadBalancerDescriptions), len(filteredLBs)) + arns := []string{} + var lastError error + err := search.client.ListRolesPages( + &iam.ListRolesInput{}, + func(results *iam.ListRolesOutput, lastPage bool) bool { + for _, role := range results.Roles { + if _, ok := search.unmatched[*role.Arn]; ok { + continue + } - if len(filteredLBs) == 0 { - // no items left to delete - return true - } + // Unfortunately role.Tags is empty from ListRoles, so we need to query each one + var response *iam.GetRoleOutput + response, lastError = search.client.GetRole(&iam.GetRoleInput{RoleName: role.RoleName}) + if lastError != nil { + if lastError.(awserr.Error).Code() == iam.ErrCodeNoSuchEntityException { + search.unmatched[*role.Arn] = exists + } else { + lastError = errors.Wrapf(lastError, "get tags for %s", *role.Arn) + search.logger.Info(lastError) + } + } else { + role = response.Role + tags := make(map[string]string, len(role.Tags)) + for _, tag := range role.Tags { + tags[*tag.Key] = *tag.Value + } + if tagMatch(search.filters, tags) { + arns = append(arns, *role.Arn) + } else { + search.unmatched[*role.Arn] = exists + } + } + } - for _, lb := range filteredLBs { - logger.Debugf("Deleting load balancer: %v", *lb.LoadBalancerName) - _, err := elbClient.DeleteLoadBalancer(&elb.DeleteLoadBalancerInput{ - LoadBalancerName: lb.LoadBalancerName, - }) - if err != nil { - logger.Debugf("Error deleting load balancer %v: %v", *lb.LoadBalancerName, err) - } else { - logger.WithField("name", *lb.LoadBalancerName).Info("Deleted load balancer") - } + return !lastPage + }, + ) + + if lastError != nil { + return arns, lastError } - return false + return arns, err } -// filterV2LBsByVPC will find all the load balancers in the provided list that are under the provided VPC -func filterV2LBsByVPC(lbs []*elbv2.LoadBalancer, vpc *ec2.Vpc, logger logrus.FieldLogger) []*elbv2.LoadBalancer { - filteredLBs := []*elbv2.LoadBalancer{} +type iamUserSearch struct { + client *iam.IAM + filters []Filter + logger logrus.FieldLogger + unmatched map[string]struct{} +} - for _, lb := range lbs { - if *lb.VpcId == *vpc.VpcId { - filteredLBs = append(filteredLBs, lb) - } +func (search *iamUserSearch) arns() ([]string, error) { + if search.unmatched == nil { + search.unmatched = map[string]struct{}{} } - return filteredLBs -} + arns := []string{} + var lastError error + err := search.client.ListUsersPages( + &iam.ListUsersInput{}, + func(results *iam.ListUsersOutput, lastPage bool) bool { + for _, user := range results.Users { + if _, ok := search.unmatched[*user.Arn]; ok { + continue + } -func deleteV2LBs(vpc *ec2.Vpc, awsSession *session.Session, logger logrus.FieldLogger) bool { - logger.Debugf("Deleting V2 load balancers (%s)", *vpc.VpcId) - defer logger.Debugf("Exiting deleting V2 load balancers (%s)", *vpc.VpcId) - elbv2Client := elbv2.New(awsSession) - - total := 0 - filteredLBs := []*elbv2.LoadBalancer{} - if err := elbv2Client.DescribeLoadBalancersPages(&elbv2.DescribeLoadBalancersInput{}, func(results *elbv2.DescribeLoadBalancersOutput, lastPage bool) bool { - total += len(results.LoadBalancers) - filteredLBs = append(filteredLBs, filterV2LBsByVPC(results.LoadBalancers, vpc, logger)...) - return lastPage - }); err != nil { - logger.Errorf("Error listing V2 load balancers: %v", err) - return false - } - logger.Debugf("from %d total V2 load balancers, %d scheduled for deletion", total, len(filteredLBs)) - - if len(filteredLBs) == 0 { - // no items left to delete - // see if we can delete target groups. - return deleteTargetGroups(vpc, awsSession, logger) - } - - for _, lb := range filteredLBs { - logger.Debugf("Deleting V2 load balancer: %v", *lb.LoadBalancerName) - _, err := elbv2Client.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{ - LoadBalancerArn: lb.LoadBalancerArn, - }) - if err != nil { - logger.Debugf("Error deleting V2 load balancer %v: %v", *lb.LoadBalancerName, err) - } else { - logger.WithField("name", *lb.LoadBalancerName).Info("Deleted load balancer") - } - } - // cleanup target groups - return deleteTargetGroups(vpc, awsSession, logger) -} + // Unfortunately user.Tags is empty from ListUsers, so we need to query each one + var response *iam.GetUserOutput + response, lastError = search.client.GetUser(&iam.GetUserInput{UserName: aws.String(*user.UserName)}) + if lastError != nil { + if lastError.(awserr.Error).Code() == iam.ErrCodeNoSuchEntityException { + search.unmatched[*user.Arn] = exists + } else { + lastError = errors.Wrapf(lastError, "get tags for %s", *user.Arn) + search.logger.Info(lastError) + } + } else { + user = response.User + tags := make(map[string]string, len(user.Tags)) + for _, tag := range user.Tags { + tags[*tag.Key] = *tag.Value + } + if tagMatch(search.filters, tags) { + arns = append(arns, *user.Arn) + } else { + search.unmatched[*user.Arn] = exists + } + } + } -// filterTargetGroupsByVPC will find all the target groups in the provided list that are under the provided VPC -func filterTargetGroupsByVPC(tgs []*elbv2.TargetGroup, vpc *ec2.Vpc, logger logrus.FieldLogger) []*elbv2.TargetGroup { - filteredTGs := []*elbv2.TargetGroup{} + return !lastPage + }, + ) - for _, tg := range tgs { - if *tg.VpcId == *vpc.VpcId { - filteredTGs = append(filteredTGs, tg) - } + if lastError != nil { + return arns, lastError } - - return filteredTGs + return arns, err } -func deleteTargetGroups(vpc *ec2.Vpc, awsSession *session.Session, logger logrus.FieldLogger) bool { - logger.Debugf("Deleting target groups (%s)", *vpc.VpcId) - defer logger.Debugf("Exiting deleting target groups (%s)", *vpc.VpcId) - elbv2Client := elbv2.New(awsSession) - - total := 0 - filteredTGs := []*elbv2.TargetGroup{} - if err := elbv2Client.DescribeTargetGroupsPages(&elbv2.DescribeTargetGroupsInput{}, func(results *elbv2.DescribeTargetGroupsOutput, lastPage bool) bool { - total += len(results.TargetGroups) - filteredTGs = append(filteredTGs, filterTargetGroupsByVPC(results.TargetGroups, vpc, logger)...) - return lastPage - }); err != nil { - logger.Errorf("Error listing target groups: %v", err) - return false +// getSharedHostedZone will find the ID of the non-Terraform-managed public route53 zone given the +// Terraform-managed zone's privateID. +func getSharedHostedZone(client *route53.Route53, privateID string, logger logrus.FieldLogger) (string, error) { + response, err := client.GetHostedZone(&route53.GetHostedZoneInput{ + Id: aws.String(privateID), + }) + if err != nil { + return "", err } - logger.Debugf("from %d total target groups, %d scheduled for deletion", total, len(filteredTGs)) - if len(filteredTGs) == 0 { - // no items left to delete - return true + if !*response.HostedZone.Config.PrivateZone { + return "", errors.Errorf("getShareedHostedZone requires a private ID, but was passed the public %s", privateID) } + privateName := *response.HostedZone.Name - for _, tg := range filteredTGs { - logger.Debugf("Deleting target groups: %v", *tg.TargetGroupName) - _, err := elbv2Client.DeleteTargetGroup(&elbv2.DeleteTargetGroupInput{ - TargetGroupArn: tg.TargetGroupArn, - }) + request := &route53.ListHostedZonesByNameInput{ + DNSName: aws.String(privateName), + } + for i := 0; true; i++ { + logger.Debugf("listing AWS hosted zones (page %d)", i) + list, err := client.ListHostedZonesByName(request) if err != nil { - logger.Debugf("Error deleting target groups %v: %v", *tg.TargetGroupName, err) - } else { - logger.WithField("name", *tg.TargetGroupName).Info("Deleted target group") + return "", err } - } - return false -} -// rtHasMainAssociation will check whether a given route table has an association marked 'Main' -func rtHasMainAssociation(rt *ec2.RouteTable) bool { - for _, association := range rt.Associations { - if *association.Main == true { - return true + for _, zone := range list.HostedZones { + if *zone.Name != privateName { + return "", nil + } + if !*zone.Config.PrivateZone { + return *zone.Id, nil + } + } + + if *list.IsTruncated && *list.NextDNSName == *request.DNSName { + request.HostedZoneId = list.NextHostedZoneId + continue } + + break } - return false + + return "", nil } -// deleteVPCEndpoints will find all VPC endpoints associated with the passed in VPC and attempt to delete them -func deleteVPCEndpoints(vpc *ec2.Vpc, ec2Client *ec2.EC2, logger logrus.FieldLogger) error { - describeEndpointsInput := ec2.DescribeVpcEndpointsInput{} - describeEndpointsInput.Filters = []*ec2.Filter{ - { - Name: aws.String("vpc-id"), - Values: []*string{vpc.VpcId}, - }, - } +func deleteARN(session *session.Session, arnString string, logger logrus.FieldLogger) error { + logger = logger.WithField("arn", arnString) - results, err := ec2Client.DescribeVpcEndpoints(&describeEndpointsInput) + parsed, err := arn.Parse(arnString) if err != nil { - logger.Debugf("error describing VPC endpoints: %v", err) return err } - for _, ep := range results.VpcEndpoints { - _, err := ec2Client.DeleteVpcEndpoints(&ec2.DeleteVpcEndpointsInput{ - VpcEndpointIds: []*string{ep.VpcEndpointId}, - }) - if err != nil { - logger.Debugf("error deleting VPC endpoint: %v", err) - return err - } - logger.WithField("id", *ep.VpcEndpointId).Info("Deleted VPC endpoint") + + switch parsed.Service { + case "ec2": + return deleteEC2(session, parsed, logger) + case "elasticloadbalancing": + return deleteElasticLoadBalancing(session, parsed, logger) + case "iam": + return deleteIAM(session, parsed, logger) + case "route53": + return deleteRoute53(session, parsed, logger) + case "s3": + return deleteS3(session, parsed, logger) + default: + return errors.Errorf("unrecognized ARN service %s (%s)", parsed.Service, arnString) } - return nil } -// deleteRouteTablesWithVPC will attempt to delete all route tables associated with a given VPC -func deleteRouteTablesWithVPC(vpc *ec2.Vpc, ec2Client *ec2.EC2, logger logrus.FieldLogger) error { - var anyError error - describeRouteTablesInput := ec2.DescribeRouteTablesInput{} - describeRouteTablesInput.Filters = []*ec2.Filter{ - { - Name: aws.String("vpc-id"), - Values: []*string{vpc.VpcId}, - }, - } +func deleteEC2(session *session.Session, arn arn.ARN, logger logrus.FieldLogger) error { + client := ec2.New(session) - results, err := ec2Client.DescribeRouteTables(&describeRouteTablesInput) + resourceType, id, err := splitSlash("resource", arn.Resource) if err != nil { - logger.Debugf("error describing route tables: %v", err) return err } - for _, rt := range results.RouteTables { - err := disassociateRouteTable(rt, ec2Client, logger) - if err != nil { - logger.Debugf("error disassociating from route table: %v", err) - return err - } + logger = logger.WithField("id", id) + + switch resourceType { + case "elastic-ip": + return deleteEC2ElasticIP(client, id, logger) + case "instance": + return deleteEC2Instance(client, iam.New(session), id, logger) + case "internet-gateway": + return deleteEC2InternetGateway(client, id, logger) + case "natgateway": + return deleteEC2NATGateway(client, id, logger) + case "route-table": + return deleteEC2RouteTable(client, id, logger) + case "security-group": + return deleteEC2SecurityGroup(client, id, logger) + case "subnet": + return deleteEC2Subnet(client, id, logger) + case "volume": + return deleteEC2Volume(client, id, logger) + case "vpc": + return deleteEC2VPC(client, elb.New(session), elbv2.New(session), id, logger) + default: + return errors.Errorf("unrecognized EC2 resource type %s", resourceType) + } +} - if rtHasMainAssociation(rt) { - // can't delete route table with the 'Main' association - // it will get cleaned up as part of deleting the VPC - continue - } - // there is a certain order that route tables need to be deleted, just try to delete - // all of them and eventually they will all be deleted - logger.Debugf("deleting route table: %v", *rt.RouteTableId) - _, err = ec2Client.DeleteRouteTable(&ec2.DeleteRouteTableInput{ - RouteTableId: rt.RouteTableId, - }) - if err != nil { - logger.Debugf("error deleting route table: %v", err) - anyError = err - } else { - logger.WithField("id", *rt.RouteTableId).Info("Deleted route table") +func deleteEC2ElasticIP(client *ec2.EC2, id string, logger logrus.FieldLogger) error { + _, err := client.ReleaseAddress(&ec2.ReleaseAddressInput{ + AllocationId: aws.String(id), + }) + if err != nil { + if err.(awserr.Error).Code() == "InvalidAllocationID.NotFound" { + return nil } + return err } - return anyError + logger.Info("Released") + return nil } -// deleteVPCs will delete any VPCs that match the provided filters/tags -func deleteVPCs(awsSession *session.Session, filters Filter, clusterName string, logger logrus.FieldLogger) (bool, error) { - logger.Debugf("Deleting VPCs (%s)", filters) - defer logger.Debugf("Exiting deleting VPCs (%s)", filters) - ec2Client := getEC2Client(awsSession) - - describeVpcsInput := ec2.DescribeVpcsInput{} - describeVpcsInput.Filters = createEC2Filters(filters) - for { - results, err := ec2Client.DescribeVpcs(&describeVpcsInput) - if err != nil { - logger.Errorf("Error listing VPCs: %v", err) - return false, nil - } - - if len(results.Vpcs) == 0 { - break - } +func deleteEC2Instance(ec2Client *ec2.EC2, iamClient *iam.IAM, id string, logger logrus.FieldLogger) error { + response, err := ec2Client.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(id)}, - for _, vpc := range results.Vpcs { - // first delete any Load Balancers under this VPC (not all of them are tagged) - v1lbcomplete := deleteLBs(vpc, awsSession, logger) - v2lbcomplete := deleteV2LBs(vpc, awsSession, logger) - if !v1lbcomplete || !v2lbcomplete { - logger.Debugf("not finished deleting load balancers, will need to retry") - return false, nil - } + // only fetch instances in 'running|pending' state since 'terminated' ones take a while to really get cleaned up + Filters: []*ec2.Filter{ + { + Name: aws.String("instance-state-name"), + Values: []*string{aws.String("running"), aws.String("pending")}, + }, + }, + }) + if err != nil { + return err + } - // next delete any VPC endpoints associated with the VPC (they are not taggable) - err := deleteVPCEndpoints(vpc, ec2Client, logger) - if err != nil { - logger.Debugf("error deleting VPC endpoint: %v", err) - return false, nil - } + for _, reservation := range response.Reservations { + for _, instance := range reservation.Instances { + if instance.IamInstanceProfile != nil { + parsed, err := arn.Parse(*instance.IamInstanceProfile.Arn) + if err != nil { + return errors.Wrap(err, "parse ARN for IAM instance profile") + } - // next delete route tables associated with the VPC (not all of them are tagged) - err = deleteRouteTablesWithVPC(vpc, ec2Client, logger) - if err != nil { - logger.Debugf("error deleting route tables: %v", err) - return false, nil + err = deleteIAMInstanceProfile(iamClient, parsed, logger.WithField("IAM instance profile", parsed.String())) + if err != nil { + return errors.Wrapf(err, "deleting %s", parsed.String()) + } } - logger.Debugf("deleting VPC: %v", *vpc.VpcId) - _, err = ec2Client.DeleteVpc(&ec2.DeleteVpcInput{ - VpcId: vpc.VpcId, + _, err := ec2Client.TerminateInstances(&ec2.TerminateInstancesInput{ + InstanceIds: []*string{instance.InstanceId}, }) if err != nil { - logger.Debugf("error deleting VPC %v: %v", *vpc.VpcId, err) - return false, nil + return err } - logger.WithField("id", *vpc.VpcId).Info("Deleted VPC") + logger.Info("Deleted") } - - return false, nil } - return true, nil -} - -// getEC2Client is just a wrapper for creating an EC2 client -func getEC2Client(awsSession *session.Session) *ec2.EC2 { - return ec2.New(awsSession) -} - -// getIAMClient is a wrapper for creating an AWS IAM client. -func getIAMClient(awsSession *session.Session) *iam.IAM { - return iam.New(awsSession) + return nil } -// deleteNATGateways will attempt to delete all NAT Gateways that match the provided filters -func deleteNATGateways(awsSession *session.Session, filters Filter, clusterName string, logger logrus.FieldLogger) (bool, error) { - - logger.Debugf("Deleting NAT Gateways (%s)", filters) - defer logger.Debugf("Exiting deleting NAT Gateways (%s)", filters) - - ec2Client := getEC2Client(awsSession) - describeNatGatewaysInput := ec2.DescribeNatGatewaysInput{} - describeNatGatewaysInput.Filter = createEC2Filters(filters) - - // NAT Gateways take a while to really disappear so only find the ones not already being deleted - describeNatGatewaysInput.Filter = append(describeNatGatewaysInput.Filter, &ec2.Filter{ - Name: aws.String("state"), - Values: []*string{aws.String("available")}, +func deleteEC2InternetGateway(client *ec2.EC2, id string, logger logrus.FieldLogger) error { + response, err := client.DescribeInternetGateways(&ec2.DescribeInternetGatewaysInput{ + InternetGatewayIds: []*string{aws.String(id)}, }) + if err != nil { + return err + } - for { - results, err := ec2Client.DescribeNatGateways(&describeNatGatewaysInput) - if err != nil { - logger.Debugf("error listing NAT gateways: %v", err) - return false, nil - } - - if len(results.NatGateways) == 0 { - break - } - - for _, nat := range results.NatGateways { - logger.Debugf("deleting NAT Gateway: %v", *nat.NatGatewayId) - _, err := ec2Client.DeleteNatGateway(&ec2.DeleteNatGatewayInput{ - NatGatewayId: nat.NatGatewayId, + for _, gateway := range response.InternetGateways { + for _, vpc := range gateway.Attachments { + _, err := client.DetachInternetGateway(&ec2.DetachInternetGatewayInput{ + InternetGatewayId: gateway.InternetGatewayId, + VpcId: vpc.VpcId, }) - if err != nil { - logger.Debugf("error deleting NAT gateway: %v", err) - continue - } else { - logger.WithField("id", *nat.NatGatewayId).Info("Deleted NAT Gateway") + if err == nil { + logger.WithField("vpc", *vpc.VpcId).Debug("Detached") + } else if err.(awserr.Error).Code() != "Gateway.NotAttached" { + return errors.Wrapf(err, "detaching from %s", *vpc.VpcId) } } + } - return false, nil + _, err = client.DeleteInternetGateway(&ec2.DeleteInternetGatewayInput{ + InternetGatewayId: &id, + }) + if err != nil { + return err } - return true, nil + logger.Info("Deleted") + return nil } -// deleteNetworkIface will attempt to delete a specific network interface -func deleteNetworkIface(iface *string, ec2Client *ec2.EC2, logger logrus.FieldLogger) error { - - result, err := ec2Client.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{ - NetworkInterfaceIds: []*string{iface}, +func deleteEC2NATGateway(client *ec2.EC2, id string, logger logrus.FieldLogger) error { + _, err := client.DeleteNatGateway(&ec2.DeleteNatGatewayInput{ + NatGatewayId: aws.String(id), }) if err != nil { - logger.Debugf("error listing network interface: %v", err) return err } - if len(result.NetworkInterfaces) == 0 { - // must have already been deleted - return nil + logger.Info("Deleted") + return nil +} + +func deleteEC2RouteTable(client *ec2.EC2, id string, logger logrus.FieldLogger) error { + response, err := client.DescribeRouteTables(&ec2.DescribeRouteTablesInput{ + RouteTableIds: []*string{aws.String(id)}, + }) + if err != nil { + if err.(awserr.Error).Code() == "InvalidRouteTableID.NotFound" { + return nil + } + return err } - for _, i := range result.NetworkInterfaces { - logger.Debugf("deleting network interface: %v", *i.NetworkInterfaceId) - _, err := ec2Client.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ - NetworkInterfaceId: i.NetworkInterfaceId, - }) + for _, table := range response.RouteTables { + err = deleteEC2RouteTableObject(client, table, logger) if err != nil { - logger.Debugf("error deleting network iface: %v", err) return err } - - logger.WithField("id", *i.NetworkInterfaceId).Info("Deleted network interface") } return nil } -// deleteEIPs will attempt to delete any elastic IPs matching the provided filters -func deleteEIPs(awsSession *session.Session, filters Filter, clusterName string, logger logrus.FieldLogger) (bool, error) { - logger.Debugf("Deleting EIPs (%s)", filters) - defer logger.Debugf("Exiting deleting EIPs (%s)", filters) - ec2Client := getEC2Client(awsSession) - - describeAddressesInput := ec2.DescribeAddressesInput{} - describeAddressesInput.Filters = createEC2Filters(filters) - - for { - results, err := ec2Client.DescribeAddresses(&describeAddressesInput) - if err != nil { - logger.Debugf("error querying elastic IPs: %v", err) - return false, nil - } - - if len(results.Addresses) == 0 { - // nothing left to delete - break +func deleteEC2RouteTableObject(client *ec2.EC2, table *ec2.RouteTable, logger logrus.FieldLogger) error { + hasMain := false + for _, association := range table.Associations { + if *association.Main { + // can't remove the 'Main' association + hasMain = true + continue } - - for _, eip := range results.Addresses { - // delete any network interface associated with the EIP (they are untagged) - if eip.NetworkInterfaceId != nil { - logger.Debugf("deleting EIP: %v", *eip.NetworkInterfaceId) - err := deleteNetworkIface(eip.NetworkInterfaceId, ec2Client, logger) - if err != nil { - logger.Debugf("error deleting network iface: %v", err) - continue - } - } - - _, err := ec2Client.ReleaseAddress(&ec2.ReleaseAddressInput{ - AllocationId: eip.AllocationId, - }) - if err != nil { - logger.Debugf("error deleting EIP: %v", err) - continue - } else { - logger.WithField("ip", *eip.PublicIp).Info("Deleted Elastic IP") - } - + _, err := client.DisassociateRouteTable(&ec2.DisassociateRouteTableInput{ + AssociationId: association.RouteTableAssociationId, + }) + if err != nil { + return errors.Wrapf(err, "dissociating %s", *association.RouteTableAssociationId) } - - return false, nil + logger.WithField("id", *association.RouteTableAssociationId).Info("Disassociated") } - return true, nil -} + if hasMain { + // can't delete route table with the 'Main' association + // it will get cleaned up as part of deleting the VPC + return nil + } -// deletePoliciesFromRole will attempt to delete any role policies from a provided role -func deletePoliciesFromRole(role *string, iamClient *iam.IAM) error { - results, err := iamClient.ListRolePolicies(&iam.ListRolePoliciesInput{ - RoleName: role, + _, err := client.DeleteRouteTable(&ec2.DeleteRouteTableInput{ + RouteTableId: table.RouteTableId, }) if err != nil { return err } - for _, policy := range results.PolicyNames { - _, err := iamClient.DeleteRolePolicy(&iam.DeleteRolePolicyInput{ - RoleName: role, - PolicyName: policy, - }) - if err != nil { - return err - } - } - + logger.Info("Deleted") return nil } -// deleteRolesFromInstanceProfile will attempt to delete any roles associated with a given instance profile -func deleteRolesFromInstanceProfile(ip *iam.InstanceProfile, iamClient *iam.IAM, logger logrus.FieldLogger) error { - for _, role := range ip.Roles { - logger.Debugf("deleting role %v from instance profile %v", *role.RoleName, *ip.InstanceProfileName) +func deleteEC2RouteTablesByVPC(client *ec2.EC2, vpc string, logger logrus.FieldLogger) error { + var lastError error + err := client.DescribeRouteTablesPages( + &ec2.DescribeRouteTablesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: []*string{&vpc}, + }, + }, + }, + func(results *ec2.DescribeRouteTablesOutput, lastPage bool) bool { + for _, table := range results.RouteTables { + lastError := deleteEC2RouteTableObject(client, table, logger.WithField("table", *table.RouteTableId)) + if lastError != nil { + lastError = errors.Wrapf(lastError, "deleting EC2 route table %s", *table.RouteTableId) + logger.Info(lastError) + } + } - // empty the role - logger.Debugf("deleting policies from role: %v", *role.RoleName) - err := deletePoliciesFromRole(role.RoleName, iamClient) - if err != nil { - logger.Debugf("error deleting policies from role: %v", err) - return err - } + return !lastPage + }, + ) - logger.Infof("Deleted all policies from role: %v", *role.RoleName) + if lastError != nil { + return lastError + } + return err +} - // detach role from instance profile - _, err = iamClient.RemoveRoleFromInstanceProfile(&iam.RemoveRoleFromInstanceProfileInput{ - InstanceProfileName: ip.InstanceProfileName, - RoleName: role.RoleName, - }) - if err != nil { - logger.Debugf("error removing role from instance profile: %v", err) - return err - } +func deleteEC2SecurityGroup(client *ec2.EC2, id string, logger logrus.FieldLogger) error { + response, err := client.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ + GroupIds: []*string{aws.String(id)}, + }) + if err != nil { + return err + } - logger.Infof("Removed role %v from instance profile %v", *role.RoleName, *ip.InstanceProfileName) + for _, group := range response.SecurityGroups { + if len(group.IpPermissions) > 0 { + _, err := client.RevokeSecurityGroupIngress(&ec2.RevokeSecurityGroupIngressInput{ + GroupId: group.GroupId, + IpPermissions: group.IpPermissions, + }) + if err != nil { + return errors.Wrap(err, "revoking ingress permissions") + } + logger.Debug("Revoked ingress permissions") + } - // now delete the role - // need to loop because this is the only time we'll have the name of the role - // now that it has been detached from the instance profile - for { - _, err = iamClient.DeleteRole(&iam.DeleteRoleInput{ - RoleName: role.RoleName, + if len(group.IpPermissionsEgress) > 0 { + _, err := client.RevokeSecurityGroupEgress(&ec2.RevokeSecurityGroupEgressInput{ + GroupId: group.GroupId, + IpPermissions: group.IpPermissionsEgress, }) if err != nil { - logger.Debugf("error deleting role %v from instance profile %v: %v", *role.RoleName, ip.InstanceProfileName, err) - } else { - logger.WithField("name", *role.RoleName).Info("Deleted role") - break + return errors.Wrap(err, "revoking egress permissions") } + logger.Debug("Revoked egress permissions") + } + } - time.Sleep(time.Second * secondsToSleep) + _, err = client.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{ + GroupId: aws.String(id), + }) + if err != nil { + if err.(awserr.Error).Code() == "InvalidGroup.NotFound" { + return nil } + return err } + logger.Info("Deleted") return nil } -// deleteInstanceProfile will attempt to delete the provided instance profile -func deleteInstanceProfile(instanceProfileID *string, iamClient *iam.IAM, logger logrus.FieldLogger) error { - ipList, err := iamClient.ListInstanceProfiles(&iam.ListInstanceProfilesInput{}) +func deleteEC2Subnet(client *ec2.EC2, id string, logger logrus.FieldLogger) error { + _, err := client.DeleteSubnet(&ec2.DeleteSubnetInput{ + SubnetId: aws.String(id), + }) if err != nil { - logger.Debugf("error listing instance profiles: %v", err) + if err.(awserr.Error).Code() == "InvalidSubnetID.NotFound" { + return nil + } return err } - var matchedIP *iam.InstanceProfile - for _, ip := range ipList.InstanceProfiles { - if *ip.InstanceProfileId == *instanceProfileID { - matchedIP = ip - } - } + logger.Info("Deleted") + return nil +} - if matchedIP == nil { - // nothing found, so already deleted? - return nil - } - - // first delete any roles out of the instance profile - err = deleteRolesFromInstanceProfile(matchedIP, iamClient, logger) - if err != nil { - return errors.Errorf("error deleting roles from instance profile: %v", err) - } - - logger.Debugf("deleting instance profile: %v", *matchedIP.InstanceProfileName) - _, err = iamClient.DeleteInstanceProfile(&iam.DeleteInstanceProfileInput{ - InstanceProfileName: matchedIP.InstanceProfileName, +func deleteEC2Volume(client *ec2.EC2, id string, logger logrus.FieldLogger) error { + _, err := client.DeleteVolume(&ec2.DeleteVolumeInput{ + VolumeId: aws.String(id), }) if err != nil { - logger.Debugf("error deleting instance profile: %v", err) + if err.(awserr.Error).Code() == "InvalidVolume.NotFound" { + return nil + } return err - } else if err == nil { - logger.WithField("name", *matchedIP.InstanceProfileName).Info("Deleted instance profile") } + logger.Info("Deleted") return nil } -// tryDeleteRoleProfileByName attempts to delete roles and profiles with given name ($CLUSTER_NAME-bootstrap|master|worker-role|profile) -func tryDeleteRoleProfileByName(roleName string, profileName string, session *session.Session, logger logrus.FieldLogger) error { - logger.Debugf("deleting role: %s", roleName) - describeRoleInput := iam.GetRoleInput{} - describeRoleInput.RoleName = &roleName - iamClient := iam.New(session) - if _, err := iamClient.GetRole(&describeRoleInput); err != nil && err.(awserr.Error).Code() != iam.ErrCodeNoSuchEntityException { - return err +func deleteEC2VPC(ec2Client *ec2.EC2, elbClient *elb.ELB, elbv2Client *elbv2.ELBV2, id string, logger logrus.FieldLogger) error { + // first delete any Load Balancers under this VPC (not all of them are tagged) + v1lbError := deleteElasticLoadBalancerClassicByVPC(elbClient, id, logger) + v2lbError := deleteElasticLoadBalancerV2ByVPC(elbv2Client, id, logger) + if v1lbError != nil { + if v2lbError != nil { + logger.Info(v2lbError) + } + return v1lbError + } else if v2lbError != nil { + return v2lbError } - // empty the role - logger.Debugf("deleting policies from role: %s", roleName) - if err := deletePoliciesFromRole(&roleName, iamClient); err != nil && err.(awserr.Error).Code() != iam.ErrCodeNoSuchEntityException { - logger.Debugf("error deleting policies from role: %v", err) - return err - } - describeProfileInput := iam.GetInstanceProfileInput{} - describeProfileInput.InstanceProfileName = &profileName - if _, err := iamClient.GetInstanceProfile(&describeProfileInput); err != nil && err.(awserr.Error).Code() != iam.ErrCodeNoSuchEntityException { + // next delete any VPC endpoints associated with the VPC (they are not taggable) + err := deleteEC2VPCEndpointsByVPC(ec2Client, id, logger) + if err != nil { return err } - // detach role from profile - logger.Debugf("detaching role from profile: %s", profileName) - _, err := iamClient.RemoveRoleFromInstanceProfile(&iam.RemoveRoleFromInstanceProfileInput{ - InstanceProfileName: &profileName, - RoleName: &roleName, - }) - if err != nil && err.(awserr.Error).Code() != iam.ErrCodeNoSuchEntityException { - logger.Debugf("error removing role from instance profile: %v", err) + // next delete route tables associated with the VPC (not all of them are tagged) + err = deleteEC2RouteTablesByVPC(ec2Client, id, logger) + if err != nil { return err } - if err == nil { - logger.Infof("Removed role %v from instance profile %v", roleName, profileName) - } - // delete profile - logger.Debugf("deleting instance profile: %v", profileName) - _, err = iamClient.DeleteInstanceProfile(&iam.DeleteInstanceProfileInput{ - InstanceProfileName: &profileName, + + _, err = ec2Client.DeleteVpc(&ec2.DeleteVpcInput{ + VpcId: aws.String(id), }) - if err != nil && err.(awserr.Error).Code() != iam.ErrCodeNoSuchEntityException { - logger.Debugf("error deleting instance profile %s: %v", profileName, err) - return err - } - if err == nil { - logger.Infof("deleted profile %s", profileName) - } - // now we can delete role - logger.Debugf("deleted policies from role %s", roleName) - deleteRoleInput := iam.DeleteRoleInput{} - deleteRoleInput.RoleName = &roleName - if _, err := iamClient.DeleteRole(&deleteRoleInput); err != nil && err.(awserr.Error).Code() != iam.ErrCodeNoSuchEntityException { - logger.Debugf("error deleting role %s: %v", roleName, err) + if err != nil { return err } - if err == nil { - logger.Infof("deleted role %s", roleName) - } - return nil -} -// deleteIAMresources will delete any IAM resources created by the installer that are not associated with a running instance -// Currently openshift/installer creates 3 roles per cluster, 1 for master|worker|bootstrap and identified by the -// cluster name used to install the cluster. -func deleteIAMresources(session *session.Session, filter Filter, clusterName string, logger logrus.FieldLogger) (bool, error) { - logger.Debugf("Deleting IAM resources (%s)", filter) - defer logger.Debugf("Exiting deleting IAM resources (%s)", filter) - installerType := []string{"master", "worker", "bootstrap"} - for _, t := range installerType { - // Naming of IAM resources expected from https://github.com/openshift/installer as follows: - // $CLUSTER_NAME-master-role $CLUSTER_NAME-worker-role $CLUSTER_NAME-bootstrap-role - // $CLUSTER_NAME-master-profile $CLUSTER_NAME-worker-profile $CLUSTER_NAME-bootstrap-profile - roleName := fmt.Sprintf("%s-%s-role", clusterName, t) - instanceProfileName := fmt.Sprintf("%s-%s-profile", clusterName, t) - if err := tryDeleteRoleProfileByName(roleName, instanceProfileName, session, logger); err != nil { - logger.Debugf("error deleting instance profile %s: %v", instanceProfileName, err) - return false, nil - } - } - return true, nil + logger.Info("Deleted") + return nil } -// deleteInstances will find any running/pending instances that match the given filter and terminate them -// and any instance profiles attached to the instance(s) -func deleteInstances(session *session.Session, filter Filter, clusterName string, logger logrus.FieldLogger) (bool, error) { - logger.Debugf("Deleting instances (%s)", filter) - defer logger.Debugf("Exiting deleting instances (%s)", filter) - - ec2Client := getEC2Client(session) - iamClient := iam.New(session) - - describeInstancesInput := ec2.DescribeInstancesInput{} - describeInstancesInput.Filters = createEC2Filters(filter) - - // only fetch instances in 'running|pending' state since 'terminated' ones take a while to really get cleaned up - describeInstancesInput.Filters = append(describeInstancesInput.Filters, &ec2.Filter{ - Name: aws.String("instance-state-name"), - Values: []*string{aws.String("running"), aws.String("pending")}, - }) - - instancesFound := false - err := ec2Client.DescribeInstancesPages(&describeInstancesInput, func(results *ec2.DescribeInstancesOutput, lastPage bool) bool { - instancesFound = instancesFound || len(results.Reservations) > 0 - for _, reservation := range results.Reservations { - for _, instance := range reservation.Instances { - // first delete any instance profiles (they are not tagged) - if instance.IamInstanceProfile != nil { - err := deleteInstanceProfile(instance.IamInstanceProfile.Id, iamClient, logger) - if err != nil { - logger.Debugf("error deleting instance profile: %v", err) - continue - } - } - - // now delete the instance - logger.Debugf("deleting instance: %v", *instance.InstanceId) - _, err := ec2Client.TerminateInstances(&ec2.TerminateInstancesInput{ - InstanceIds: []*string{instance.InstanceId}, - }) - if err != nil { - logger.Debugf("error deleting instance: %v", err) - continue - } else { - logger.WithField("id", *instance.InstanceId).Info("Deleted instance") - } - } - } - - return lastPage +func deleteEC2VPCEndpoint(client *ec2.EC2, id string, logger logrus.FieldLogger) error { + _, err := client.DeleteVpcEndpoints(&ec2.DeleteVpcEndpointsInput{ + VpcEndpointIds: []*string{aws.String(id)}, }) if err != nil { - logger.Debugf("error describing instances: %v", err) - return false, nil + return errors.Wrapf(err, "cannot delete VPC endpoint %s", id) } - return !instancesFound, nil + logger.Info("Deleted") + return nil } -// deleteSecurityGroupRules will attempt to delete all the rules defined in the given security group -// since some security groups have self-referencing rules that complicate being able to delete the security group -func deleteSecurityGroupRules(sg *ec2.SecurityGroup, ec2Client *ec2.EC2, logger logrus.FieldLogger) error { +func deleteEC2VPCEndpointsByVPC(client *ec2.EC2, vpc string, logger logrus.FieldLogger) error { + response, err := client.DescribeVpcEndpoints(&ec2.DescribeVpcEndpointsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(vpc)}, + }, + }, + }) - if len(sg.IpPermissions) > 0 { - _, err := ec2Client.RevokeSecurityGroupIngress(&ec2.RevokeSecurityGroupIngressInput{ - GroupId: sg.GroupId, - IpPermissions: sg.IpPermissions, - }) - if err != nil { - logger.Debugf("error removing ingress permissions: %v", err) - } + if err != nil { + return err } - if len(sg.IpPermissionsEgress) > 0 { - _, err := ec2Client.RevokeSecurityGroupEgress(&ec2.RevokeSecurityGroupEgressInput{ - GroupId: sg.GroupId, - IpPermissions: sg.IpPermissionsEgress, - }) + for _, endpoint := range response.VpcEndpoints { + err := deleteEC2VPCEndpoint(client, *endpoint.VpcEndpointId, logger.WithField("VPC endpoint", *endpoint.VpcEndpointId)) if err != nil { - logger.Debugf("error removing egress permissions: %v", err) + if err.(awserr.Error).Code() == "InvalidVpcID.NotFound" { + return nil + } + return err } } return nil } -// deleteSecurityGroups will attempt to delete all security groups matching the given filter -func deleteSecurityGroups(session *session.Session, filter Filter, clusterName string, logger logrus.FieldLogger) (bool, error) { - logger.Debugf("Deleting security groups (%s)", filter) - defer logger.Debugf("Exiting deleting security groups (%s)", filter) - - ec2Client := getEC2Client(session) - describeSecurityGroupsInput := ec2.DescribeSecurityGroupsInput{} - describeSecurityGroupsInput.Filters = createEC2Filters(filter) - - for { - results, err := ec2Client.DescribeSecurityGroups(&describeSecurityGroupsInput) - if err != nil { - logger.Debugf("error listing security groups %v", err) - return false, nil - } - - if len(results.SecurityGroups) == 0 { - break - } - - for _, sg := range results.SecurityGroups { - // first delete rules (can get circular dependencies otherwise) - deleteSecurityGroupRules(sg, ec2Client, logger) - _, err := ec2Client.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{ - GroupId: sg.GroupId, - }) - if err != nil { - logger.Debugf("error deleting security group: %v", err) - continue - } else { - logger.WithField("id", *sg.GroupId).Info("Deleted security group") - } - } - - return false, nil +func deleteElasticLoadBalancing(session *session.Session, arn arn.ARN, logger logrus.FieldLogger) error { + resourceType, id, err := splitSlash("resource", arn.Resource) + if err != nil { + return err + } + logger = logger.WithField("id", id) + + switch resourceType { + case "loadbalancer": + segments := strings.SplitN(id, "/", 2) + if len(segments) == 1 { + return deleteElasticLoadBalancerClassic(elb.New(session), id, logger) + } else if len(segments) != 2 { + return errors.Errorf("cannot parse subresource %q into {subtype}/{id}", id) + } + subtype := segments[0] + id = segments[1] + switch subtype { + case "net": + return deleteElasticLoadBalancerV2(elbv2.New(session), arn, logger) + default: + return errors.Errorf("unrecognized elastic load balancing resource subtype %s", subtype) + } + case "targetgroup": + return deleteElasticLoadBalancerTargetGroup(elbv2.New(session), arn, logger) + default: + return errors.Errorf("unrecognized elastic load balancing resource type %s", resourceType) } - - return true, nil } -// detachInternetGateways will attempt to detach an internet gateway from the associated VPC(s) -func detachInternetGateways(gw *ec2.InternetGateway, ec2Client *ec2.EC2, logger logrus.FieldLogger) error { - for _, vpc := range gw.Attachments { - logger.Debugf("detaching Internet GW %v from VPC %v", *gw.InternetGatewayId, *vpc.VpcId) - _, err := ec2Client.DetachInternetGateway(&ec2.DetachInternetGatewayInput{ - InternetGatewayId: gw.InternetGatewayId, - VpcId: vpc.VpcId, - }) - - if err != nil { - return errors.Errorf("error detaching internet gateway: %v", err) - } else if err == nil { - logger.Infof("Detached Internet GW %v from VPC %v", *gw.InternetGatewayId, *vpc.VpcId) - } +func deleteElasticLoadBalancerClassic(client *elb.ELB, name string, logger logrus.FieldLogger) error { + _, err := client.DeleteLoadBalancer(&elb.DeleteLoadBalancerInput{ + LoadBalancerName: aws.String(name), + }) + if err != nil { + return err } + logger.Info("Deleted") return nil } -// deleteInternetGateways will attemp to delete any Internet Gateways matching the given filter -func deleteInternetGateways(session *session.Session, filter Filter, clusterName string, logger logrus.FieldLogger) (bool, error) { - logger.Debugf("Deleting internet gateways (%s)", filter) - defer logger.Debugf("Exiting deleting internet gateways (%s)", filter) - - ec2Client := getEC2Client(session) - - describeInternetGatewaysInput := ec2.DescribeInternetGatewaysInput{} - describeInternetGatewaysInput.Filters = createEC2Filters(filter) - - for { - results, err := ec2Client.DescribeInternetGateways(&describeInternetGatewaysInput) - if err != nil { - logger.Debugf("error listing internet gateways: %v", err) - return false, nil - } - - if len(results.InternetGateways) == 0 { - break - } - - for _, gw := range results.InternetGateways { - logger.Debugf("deleting internet gateway: %v", *gw.InternetGatewayId) +func deleteElasticLoadBalancerClassicByVPC(client *elb.ELB, vpc string, logger logrus.FieldLogger) error { + var lastError error + err := client.DescribeLoadBalancersPages( + &elb.DescribeLoadBalancersInput{}, + func(results *elb.DescribeLoadBalancersOutput, lastPage bool) bool { + for _, lb := range results.LoadBalancerDescriptions { + if *lb.VPCId != vpc { + continue + } - err := detachInternetGateways(gw, ec2Client, logger) - if err != nil { - logger.Debugf("error detaching igw: %v", err) - continue + lastError = deleteElasticLoadBalancerClassic(client, *lb.LoadBalancerName, logger.WithField("classic load balancer", *lb.LoadBalancerName)) + if lastError != nil { + lastError = errors.Wrapf(lastError, "deleting classic load balancer %s", *lb.LoadBalancerName) + logger.Info(lastError) + } } - _, err = ec2Client.DeleteInternetGateway(&ec2.DeleteInternetGatewayInput{ - InternetGatewayId: gw.InternetGatewayId, - }) - if err != nil { - logger.Debugf("error deleting internet gateway: %v", err) - } else { - logger.WithField("id", *gw.InternetGatewayId).Info("Deleted internet gateway") - } - } + return !lastPage + }, + ) - return false, nil + if lastError != nil { + return lastError } - - return true, nil + return err } -// disassociateRouteTable will attempt to disassociate all except the 'Main' associations defined -// for the given Route Table -func disassociateRouteTable(rt *ec2.RouteTable, ec2Client *ec2.EC2, logger logrus.FieldLogger) error { - for _, association := range rt.Associations { - if *association.Main { - // can't remove the 'Main' association - continue - } - logger.Debugf("disassociating route table association %v", *association.RouteTableAssociationId) - _, err := ec2Client.DisassociateRouteTable(&ec2.DisassociateRouteTableInput{ - AssociationId: association.RouteTableAssociationId, - }) - if err != nil { - logger.Debugf("error disassociating from route table: %v", err) - return err - } else if err == nil { - logger.WithField("id", *association.RouteTableAssociationId).Info("Disassociated route table association") - } +func deleteElasticLoadBalancerTargetGroup(client *elbv2.ELBV2, arn arn.ARN, logger logrus.FieldLogger) error { + _, err := client.DeleteTargetGroup(&elbv2.DeleteTargetGroupInput{ + TargetGroupArn: aws.String(arn.String()), + }) + if err != nil { + return err } + logger.Info("Deleted") return nil } -// deleteSubnets will attempt to delete all Subnets matching the given filter -func deleteSubnets(session *session.Session, filter Filter, clusterName string, logger logrus.FieldLogger) (bool, error) { - logger.Debugf("Deleting subnets (%s)", filter) - defer logger.Debugf("Exiting deleting subnets (%s)", filter) - - ec2Client := getEC2Client(session) - - describeSubnetsInput := ec2.DescribeSubnetsInput{} - describeSubnetsInput.Filters = createEC2Filters(filter) - - for { - results, err := ec2Client.DescribeSubnets(&describeSubnetsInput) - if err != nil { - logger.Debugf("error listing subnets: %v", err) - return false, nil - } +func deleteElasticLoadBalancerTargetGroupsByVPC(client *elbv2.ELBV2, vpc string, logger logrus.FieldLogger) error { + var lastError error + err := client.DescribeTargetGroupsPages( + &elbv2.DescribeTargetGroupsInput{}, + func(results *elbv2.DescribeTargetGroupsOutput, lastPage bool) bool { + for _, group := range results.TargetGroups { + if *group.VpcId != vpc { + continue + } - if len(results.Subnets) == 0 { - break - } + var parsed arn.ARN + parsed, lastError = arn.Parse(*group.TargetGroupArn) + if lastError != nil { + lastError = errors.Wrap(lastError, "parse ARN for target group") + logger.Info(lastError) + continue + } - for _, subnet := range results.Subnets { - _, err := ec2Client.DeleteSubnet(&ec2.DeleteSubnetInput{ - SubnetId: subnet.SubnetId, - }) - if err != nil { - logger.Debugf("error deleting subnet: %v", err) - } else { - logger.WithField("id", *subnet.SubnetId).Info("Deleted subnet") + lastError = deleteElasticLoadBalancerTargetGroup(client, parsed, logger.WithField("target group", parsed.Resource)) + if lastError != nil { + lastError = errors.Wrapf(lastError, "deleting %s", parsed.String()) + logger.Info(lastError) + } } - } - - return false, nil - } - - return true, nil -} - -// bucketsToAWSObjects will convert a list of S3 Buckets to awsObjectsWithTags (for easier filtering) -func bucketsToAWSObjects(buckets []*s3.Bucket, s3Client *s3.S3, logger logrus.FieldLogger) ([]awsObjectWithTags, error) { - bucketObjects := []awsObjectWithTags{} - - for _, bucket := range buckets { - tags, err := s3Client.GetBucketTagging(&s3.GetBucketTaggingInput{ - Bucket: bucket.Name, - }) - if err != nil { - logger.Debugf("error getting tags for bucket %s: %v, skipping...", *bucket.Name, err) - continue - } - - tagsAsMap, err := tagsToMap(tags.TagSet) - if err != nil { - return bucketObjects, err - } - bucketObjects = append(bucketObjects, awsObjectWithTags{ - Name: *bucket.Name, - Tags: tagsAsMap, - }) - } - return bucketObjects, nil -} + return !lastPage + }, + ) -// filterObjects will do client-side filtering given an appropriately filled out list of awsObjectWithTags -func filterObjects(awsObjects []awsObjectWithTags, filters Filter) []awsObjectWithTags { - objectsWithTags := []awsObjectWithTags{} - filteredObjects := []awsObjectWithTags{} - - // first find the objects that have all the desired tags - for _, object := range awsObjects { - allTagsFound := true - for key := range filters { - if _, ok := object.Tags[key]; !ok { - // doesn't have one of the tags we're looking for so skip it - allTagsFound = false - break - } - } - if allTagsFound { - objectsWithTags = append(objectsWithTags, object) - } + if lastError != nil { + return lastError } - - // now check that the values match - for _, object := range objectsWithTags { - valuesMatch := true - for key, val := range filters { - if object.Tags[key] != val { - valuesMatch = false - break - } - } - if valuesMatch { - filteredObjects = append(filteredObjects, object) - } - } - return filteredObjects + return err } -// deleteS3Buckets will attempt to delete (and empty) any S3 bucket matching the provided filter -func deleteS3Buckets(session *session.Session, filter Filter, clusterName string, logger logrus.FieldLogger) (bool, error) { - logger.Debugf("Deleting S3 buckets (%s)", filter) - defer logger.Debugf("Exiting deleting buckets (%s)", filter) - - s3Client := s3.New(session) - - listBucketsInput := s3.ListBucketsInput{} - - for { - results, err := s3Client.ListBuckets(&listBucketsInput) - if err != nil { - logger.Debugf("error listing s3 buckets: %v", err) - return false, nil - } - - awsObjects, err := bucketsToAWSObjects(results.Buckets, s3Client, logger) - if err != nil { - logger.Debugf("error converting s3 buckets to native AWS objects: %v", err) - return false, nil - } - - filteredObjects := filterObjects(awsObjects, filter) - logger.Debugf("from %d total s3 buckets, %d match filters", len(awsObjects), len(filteredObjects)) - if len(filteredObjects) == 0 { - break - } - - for _, bucket := range filteredObjects { - logger.Debugf("deleting bucket: %v", bucket.Name) - - // first empty the bucket - iter := s3manager.NewDeleteListIterator(s3Client, &s3.ListObjectsInput{ - Bucket: aws.String(bucket.Name), - }) - err := s3manager.NewBatchDeleteWithClient(s3Client).Delete(aws.BackgroundContext(), iter) - if err != nil { - logger.Debugf("error emptying bucket %v: %v", bucket.Name, err) - continue - } else { - logger.WithField("name", bucket.Name).Info("Emptied bucket") - } - - // now delete the bucket - _, err = s3Client.DeleteBucket(&s3.DeleteBucketInput{ - Bucket: aws.String(bucket.Name), - }) - if err != nil { - logger.Debugf("error deleting bucket %v: %v", bucket.Name, err) - continue - } else { - logger.WithField("name", bucket.Name).Info("Deleted bucket") - } - } - - return false, nil +func deleteElasticLoadBalancerV2(client *elbv2.ELBV2, arn arn.ARN, logger logrus.FieldLogger) error { + _, err := client.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{ + LoadBalancerArn: aws.String(arn.String()), + }) + if err != nil { + return err } - return true, nil + logger.Info("Deleted") + return nil } -// r53ZonesToAWSObjects will create a list of awsObjectsWithTags for the provided list of route53.HostedZone s -func r53ZonesToAWSObjects(zones []*route53.HostedZone, r53Client *route53.Route53, logger logrus.FieldLogger) ([]awsObjectWithTags, error) { - zonesAsAWSObjects := []awsObjectWithTags{} - - var result *route53.ListTagsForResourceOutput - var err error - for _, zone := range zones { - for { - result, err = r53Client.ListTagsForResource(&route53.ListTagsForResourceInput{ - ResourceType: aws.String("hostedzone"), - ResourceId: zone.Id, - }) - if err != nil { - if request.IsErrorThrottle(err) { - logger.Debugf("sleeping before trying to resolve tags for zone %s: %v", *zone.Id, err) - time.Sleep(time.Second) +func deleteElasticLoadBalancerV2ByVPC(client *elbv2.ELBV2, vpc string, logger logrus.FieldLogger) error { + var lastError error + err := client.DescribeLoadBalancersPages( + &elbv2.DescribeLoadBalancersInput{}, + func(results *elbv2.DescribeLoadBalancersOutput, lastPage bool) bool { + for _, lb := range results.LoadBalancers { + if *lb.VpcId != vpc { continue } - return zonesAsAWSObjects, err - } - break - } + var parsed arn.ARN + parsed, lastError = arn.Parse(*lb.LoadBalancerArn) + if lastError != nil { + lastError = errors.Wrap(lastError, "parse ARN for load balancer") + logger.Info(lastError) + continue + } - tagsToMap, err := tagsToMap(result.ResourceTagSet.Tags) - if err != nil { - return zonesAsAWSObjects, err - } + lastError = deleteElasticLoadBalancerV2(client, parsed, logger.WithField("load balancer", parsed.Resource)) + if lastError != nil { + lastError = errors.Wrapf(lastError, "deleting %s", parsed.String()) + logger.Info(lastError) + } + } - zonesAsAWSObjects = append(zonesAsAWSObjects, awsObjectWithTags{ - Name: *zone.Id, - Tags: tagsToMap, - }) + return !lastPage + }, + ) + if lastError != nil { + return lastError } - - return zonesAsAWSObjects, nil + return err } -// deleteEntriesFromSharedR53Zone will find route53 entries for the shared (ie non-terraform-managed) route53 zone -// and remove them. -// Provide the terraform-created private zone, and the manually created public/shared zone, and it will find any -// entries in the public/shared zone that match entries in the private zone, and delete them -func deleteEntriesFromSharedR53Zone(zoneID string, sharedZoneID string, r53Client *route53.Route53, logger logrus.FieldLogger) error { +func deleteIAM(session *session.Session, arn arn.ARN, logger logrus.FieldLogger) error { + client := iam.New(session) - zoneEntries, err := r53Client.ListResourceRecordSets(&route53.ListResourceRecordSetsInput{ - HostedZoneId: aws.String(zoneID), - }) - if err != nil { - return err - } - - sharedZoneEntries, err := r53Client.ListResourceRecordSets(&route53.ListResourceRecordSetsInput{ - HostedZoneId: aws.String(sharedZoneID), - }) + resourceType, id, err := splitSlash("resource", arn.Resource) if err != nil { return err } + logger = logger.WithField("id", id) - for _, entry := range zoneEntries.ResourceRecordSets { - // only interested in deleting 'A' records - if *entry.Type != "A" { - continue - } - for _, sharedEntry := range sharedZoneEntries.ResourceRecordSets { - if *sharedEntry.Name == *entry.Name && *sharedEntry.Type == *entry.Type { - _, err := r53Client.ChangeResourceRecordSets(&route53.ChangeResourceRecordSetsInput{ - HostedZoneId: aws.String(sharedZoneID), - ChangeBatch: &route53.ChangeBatch{ - Changes: []*route53.Change{ - { - Action: aws.String("DELETE"), - ResourceRecordSet: &route53.ResourceRecordSet{ - Name: sharedEntry.Name, - Type: sharedEntry.Type, - AliasTarget: sharedEntry.AliasTarget, - }, - }, - }, - }, - }) - if err != nil { - return err - } - - logger.Infof("Deleted record %v from r53 zone %v", *sharedEntry.Name, sharedZoneID) - } - } + switch resourceType { + case "instance-profile": + return deleteIAMInstanceProfile(client, arn, logger) + case "role": + return deleteIAMRole(client, arn, logger) + case "user": + return deleteIAMUser(client, id, logger) + default: + return errors.Errorf("unrecognized EC2 resource type %s", resourceType) } - - return nil } -// getSharedHostedZone will find the zoneID of the non-terraform-managed public route53 zone given the -// terraform-managed private zoneID -func getSharedHostedZone(zoneID string, allZones []*route53.HostedZone) (string, error) { - // given the ID, get the name of the zone - zoneName := "" - for _, zone := range allZones { - if *zone.Id == zoneID { - zoneName = *zone.Name - break - } +func deleteIAMInstanceProfile(client *iam.IAM, profileARN arn.ARN, logger logrus.FieldLogger) error { + resourceType, name, err := splitSlash("resource", profileARN.Resource) + if err != nil { + return err } + logger = logger.WithField("name", name) - // now find the shared zone that matches by name - for _, zone := range allZones { - // skip the actual terraform-managed zone (we're looking for the shared zone) - if *zone.Id == zoneID { - continue - } - - if *zone.Name == zoneName { - return *zone.Id, nil - } + if resourceType != "instance-profile" { + return errors.Errorf("%s ARN passed to deleteIAMInstanceProfile: %s", resourceType, profileARN.String()) } - // else we didn't find it - return "", errors.Errorf("could not find shared zone with name: %v", zoneName) -} - -// emptyAndDeleteRoute53Zone will delete all the entries in the given route53 zone and delete the zone itself -func emptyAndDeleteRoute53Zone(zoneID string, r53Client *route53.Route53, logger logrus.FieldLogger) error { - - // first need to delete all non SOA and NS records - results, err := r53Client.ListResourceRecordSets(&route53.ListResourceRecordSetsInput{ - HostedZoneId: aws.String(zoneID), + response, err := client.GetInstanceProfile(&iam.GetInstanceProfileInput{ + InstanceProfileName: &name, }) if err != nil { + if err.(awserr.Error).Code() == iam.ErrCodeNoSuchEntityException { + return nil + } return err } + profile := response.InstanceProfile - for _, entry := range results.ResourceRecordSets { - if *entry.Type == "SOA" || *entry.Type == "NS" { - // can't delete SOA and NS types - continue - } - _, err := r53Client.ChangeResourceRecordSets(&route53.ChangeResourceRecordSetsInput{ - HostedZoneId: aws.String(zoneID), - ChangeBatch: &route53.ChangeBatch{ - Changes: []*route53.Change{ - { - Action: aws.String("DELETE"), - ResourceRecordSet: &route53.ResourceRecordSet{ - Name: entry.Name, - Type: entry.Type, - TTL: entry.TTL, - ResourceRecords: entry.ResourceRecords, - AliasTarget: entry.AliasTarget, - }, - }, - }, - }, + for _, role := range profile.Roles { + _, err = client.RemoveRoleFromInstanceProfile(&iam.RemoveRoleFromInstanceProfileInput{ + InstanceProfileName: profile.InstanceProfileName, + RoleName: role.RoleName, }) if err != nil { - return err + return errors.Wrapf(err, "dissociating %s", *role.RoleName) } - logger.Infof("Deleted record %v from r53 zone %v", *entry.Name, zoneID) + logger.WithField("role", *role.RoleName).Info("Disassociated") } - // now delete zone - _, err = r53Client.DeleteHostedZone(&route53.DeleteHostedZoneInput{ - Id: aws.String(zoneID), + _, err = client.DeleteInstanceProfile(&iam.DeleteInstanceProfileInput{ + InstanceProfileName: profile.InstanceProfileName, }) if err != nil { return err } - logger.WithField("id", zoneID).Info("Deleted route53 zone") - + logger.Info("Deleted") return nil } -// deleteRoute53 will attempt to delete any route53 zone matching the given filter. -// it will also attempt to delete any entries in the shared/public route53 zone -func deleteRoute53(session *session.Session, filters Filter, clusterName string, logger logrus.FieldLogger) (bool, error) { - logger.Debugf("Deleting Route53 zones (%s)", filters) - defer logger.Debugf("Exiting deleting Route53 zones (%s)", filters) +func deleteIAMRole(client *iam.IAM, roleARN arn.ARN, logger logrus.FieldLogger) error { + resourceType, name, err := splitSlash("resource", roleARN.Resource) + if err != nil { + return err + } + logger = logger.WithField("name", name) - r53Client := route53.New(session) + if resourceType != "role" { + return errors.Errorf("%s ARN passed to deleteIAMRole: %s", resourceType, roleARN.String()) + } - listHostedZonesInput := route53.ListHostedZonesInput{} + var lastError error + err = client.ListRolePoliciesPages( + &iam.ListRolePoliciesInput{RoleName: &name}, + func(results *iam.ListRolePoliciesOutput, lastPage bool) bool { + for _, policy := range results.PolicyNames { + _, lastError = client.DeleteRolePolicy(&iam.DeleteRolePolicyInput{ + RoleName: &name, + PolicyName: policy, + }) + if lastError != nil { + lastError = errors.Wrapf(lastError, "deleting IAM role policy %s", *policy) + logger.Info(lastError) + } + logger.WithField("policy", *policy).Info("Deleted") + } - for { - allZones, err := r53Client.ListHostedZones(&listHostedZonesInput) - if err != nil { - logger.Debugf("error listing route53 zones: %v", err) - return false, nil - } + return !lastPage + }, + ) - awsZones, err := r53ZonesToAWSObjects(allZones.HostedZones, r53Client, logger) - if err != nil { - logger.Debugf("error converting r53Zones to native AWS objects: %v", err) - return false, nil - } + if lastError != nil { + return lastError + } + if err != nil { + return errors.Wrap(err, "listing IAM role policies") + } - filteredZones := filterObjects(awsZones, filters) - logger.Debugf("from %d total r53 zones, %d match filters", len(awsZones), len(filteredZones)) - if len(filteredZones) == 0 { - break - } + err = client.ListInstanceProfilesForRolePages( + &iam.ListInstanceProfilesForRoleInput{RoleName: &name}, + func(results *iam.ListInstanceProfilesForRoleOutput, lastPage bool) bool { + for _, profile := range results.InstanceProfiles { + parsed, lastError := arn.Parse(*profile.Arn) + if lastError != nil { + lastError = errors.Wrap(lastError, "parse ARN for IAM instance profile") + logger.Info(lastError) + continue + } - for _, zone := range filteredZones { - // first find the shared hostedzone (will have same name as the tagged zone) - sharedZoneID, err := getSharedHostedZone(zone.Name, allZones.HostedZones) - if err != nil { - logger.Debugf("%v", err) - return false, nil + lastError = deleteIAMInstanceProfile(client, parsed, logger.WithField("IAM instance profile", parsed.String())) + if lastError != nil { + lastError = errors.Wrapf(lastError, "deleting %s", parsed.String()) + logger.Info(lastError) + } } - // first need to delete any 'A' entries from the shared-non-private Route53 zone - // (eg. newcluster.subdomain.domain.com newcluster-api.subdomain.domain.com and - // *.newcluster.subdomain.domain.com) - err = deleteEntriesFromSharedR53Zone(zone.Name, sharedZoneID, r53Client, logger) - if err != nil { - logger.Debugf("error deleting entries from shared r53 zone: %v", err) - return false, nil - } + return !lastPage + }, + ) - // finally can delete the tagged hosted zone - err = emptyAndDeleteRoute53Zone(zone.Name, r53Client, logger) - if err != nil { - logger.Debugf("error deleting zone %v: %v", zone.Name, err) - return false, nil - } - } - return false, nil + if lastError != nil { + return lastError + } + if err != nil { + return errors.Wrap(err, "listing IAM instance profiles") } - // all done deleting r53 entries/zones - return true, nil -} -// deletePVs will find PVs based on provided filters and delete them -func deletePVs(session *session.Session, filters Filter, clusterName string, logger logrus.FieldLogger) (bool, error) { + _, err = client.DeleteRole(&iam.DeleteRoleInput{RoleName: &name}) + if err != nil { + return err + } - logger.Debugf("Deleting PVs (%s)", filters) - defer logger.Debugf("Exiting deleting PVs (%s)", filters) + logger.Info("Deleted") + return nil +} - ec2Client := getEC2Client(session) - describeVolumesInput := ec2.DescribeVolumesInput{} - describeVolumesInput.Filters = createEC2Filters(filters) +func deleteIAMUser(client *iam.IAM, id string, logger logrus.FieldLogger) error { + var lastError error + err := client.ListUserPoliciesPages( + &iam.ListUserPoliciesInput{UserName: &id}, + func(results *iam.ListUserPoliciesOutput, lastPage bool) bool { + for _, policy := range results.PolicyNames { + _, lastError = client.DeleteUserPolicy(&iam.DeleteUserPolicyInput{ + UserName: &id, + PolicyName: policy, + }) + if lastError != nil { + lastError = errors.Wrapf(lastError, "deleting IAM user policy %s", *policy) + logger.Info(lastError) + } + logger.WithField("policy", *policy).Info("Deleted") + } - results, err := ec2Client.DescribeVolumes(&describeVolumesInput) + return !lastPage + }, + ) + + if lastError != nil { + return lastError + } if err != nil { - logger.Debugf("error listing volumes: %v", err) - return false, nil + return errors.Wrap(err, "listing IAM user policies") } - if len(results.Volumes) == 0 { - // nothing to delete, we must be done - return true, nil + err = client.ListAccessKeysPages( + &iam.ListAccessKeysInput{UserName: &id}, + func(results *iam.ListAccessKeysOutput, lastPage bool) bool { + for _, key := range results.AccessKeyMetadata { + _, lastError := client.DeleteAccessKey(&iam.DeleteAccessKeyInput{ + UserName: &id, + AccessKeyId: key.AccessKeyId, + }) + if lastError != nil { + lastError = errors.Wrapf(lastError, "deleting IAM access key %s", *key.AccessKeyId) + logger.Info(lastError) + } + } + + return !lastPage + }, + ) + + if lastError != nil { + return lastError + } + if err != nil { + return errors.Wrap(err, "listing IAM access keys") } - for _, vol := range results.Volumes { - logger.Debugf("deleting volume: %v", *vol.VolumeId) - _, err := ec2Client.DeleteVolume(&ec2.DeleteVolumeInput{ - VolumeId: vol.VolumeId, - }) - if err != nil { - logger.Debugf("error deleting volume: %v", err) - } else { - logger.WithField("id", *vol.VolumeId).Info("Deleted Volume") - } + _, err = client.DeleteUser(&iam.DeleteUserInput{ + UserName: &id, + }) + if err != nil { + return err } - return false, nil + logger.Info("Deleted") + return nil } -// deleteUsers will find users created by the cloud credential operator and delete them. -func deleteUsers(session *session.Session, filters Filter, clusterName string, logger logrus.FieldLogger) (bool, error) { - - logger.Debugf("Deleting users (%s)", filters) - defer logger.Debugf("Exiting deleting users (%s)", filters) - - iamClient := getIAMClient(session) - - listUsersInput := iam.ListUsersInput{} +func deleteRoute53(session *session.Session, arn arn.ARN, logger logrus.FieldLogger) error { + resourceType, id, err := splitSlash("resource", arn.Resource) + if err != nil { + return err + } + logger = logger.WithField("id", id) - for { - allUsers, err := iamClient.ListUsers(&listUsersInput) - if err != nil { - logger.WithError(err).Debug("error listing all users") - return false, nil - } - logger.Debugf("Found %d users", len(allUsers.Users)) + if resourceType != "hostedzone" { + return errors.Errorf("unrecognized Route 53 resource type %s", resourceType) + } - awsUsers, err := usersToAWSObjects(allUsers.Users, iamClient, logger) - if err != nil { - logger.Debugf("error converting users to native AWS objects: %v", err) - return false, nil - } + client := route53.New(session) - filteredUsers := filterObjects(awsUsers, filters) - logger.Debugf("from %d total users, %d match filters", len(awsUsers), len(filteredUsers)) - if len(filteredUsers) == 0 { - break - } + sharedZoneID, err := getSharedHostedZone(client, id, logger) + if err != nil { + return err + } - for _, user := range filteredUsers { - uLog := logger.WithField("user", user.Name) + recordSetKey := func(recordSet *route53.ResourceRecordSet) string { + return fmt.Sprintf("%s %s", *recordSet.Type, *recordSet.Name) + } - // list user policies: - policiesOut, err := iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{UserName: aws.String(user.Name)}) - if err != nil { - uLog.WithError(err).Debug("error listing user policies") - return false, nil + sharedEntries := map[string]*route53.ResourceRecordSet{} + err = client.ListResourceRecordSetsPages( + &route53.ListResourceRecordSetsInput{HostedZoneId: aws.String(sharedZoneID)}, + func(results *route53.ListResourceRecordSetsOutput, lastPage bool) bool { + for _, recordSet := range results.ResourceRecordSets { + key := recordSetKey(recordSet) + sharedEntries[key] = recordSet } - // delete any user policies: - for _, policy := range policiesOut.PolicyNames { - _, err = iamClient.DeleteUserPolicy(&iam.DeleteUserPolicyInput{ - UserName: aws.String(user.Name), - PolicyName: policy, - }) - if err != nil { - uLog.WithError(err).WithField("policy", *policy).Debug("error deleting user policy") - return false, nil - } - uLog.WithField("policy", *policy).Debug("deleted user policy") - } + return !lastPage + }, + ) + if err != nil { + return err + } - // list access keys: - allUserKeys, err := iamClient.ListAccessKeys(&iam.ListAccessKeysInput{UserName: aws.String(user.Name)}) - if err != nil { - uLog.WithError(err).Error("error listing all access keys for user") - return false, err - } + var lastError error + err = client.ListResourceRecordSetsPages( + &route53.ListResourceRecordSetsInput{HostedZoneId: aws.String(id)}, + func(results *route53.ListResourceRecordSetsOutput, lastPage bool) bool { + for _, recordSet := range results.ResourceRecordSets { + if *recordSet.Type == "SOA" || *recordSet.Type == "NS" { + // can't delete SOA and NS types + continue + } + key := recordSetKey(recordSet) + if sharedEntry, ok := sharedEntries[key]; ok { + lastError = deleteRoute53RecordSet(client, sharedZoneID, sharedEntry, logger.WithField("public zone", sharedZoneID)) + if lastError != nil { + lastError = errors.Wrapf(lastError, "deleting public zone %s", sharedZoneID) + logger.Info(lastError) + } + } - // delete access keys: - for _, kmd := range allUserKeys.AccessKeyMetadata { - akLog := uLog.WithFields(logrus.Fields{ - "accessKeyID": *kmd.AccessKeyId, - }) - akLog.Info("deleting access key") - _, err := iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{AccessKeyId: kmd.AccessKeyId, UserName: aws.String(user.Name)}) - if err != nil { - akLog.WithError(err).Error("error deleting access key") - return false, err + lastError = deleteRoute53RecordSet(client, id, recordSet, logger) + if lastError != nil { + lastError = errors.Wrapf(lastError, "deleting record set %#+v from zone %s", recordSet, id) + logger.Info(lastError) } } - // delete user: - _, err = iamClient.DeleteUser(&iam.DeleteUserInput{ - UserName: aws.String(user.Name), - }) - if err != nil { - uLog.WithError(err).Debug("error deleting user") - } - uLog.Info("user deleted") - } + return !lastPage + }, + ) - return false, nil + if lastError != nil { + return lastError + } + if err != nil { + return err } - return true, nil -} + _, err = client.DeleteHostedZone(&route53.DeleteHostedZoneInput{ + Id: aws.String(id), + }) + if err != nil { + return err + } -// usersToAWSObjects will create a list of awsObjectsWithTags for the provided list of users. -func usersToAWSObjects(users []*iam.User, iamClient *iam.IAM, logger logrus.FieldLogger) ([]awsObjectWithTags, error) { - usersAsAWSObjects := []awsObjectWithTags{} + logger.Info("Deleted") + return nil +} - for _, user := range users { +func deleteRoute53RecordSet(client *route53.Route53, zoneID string, recordSet *route53.ResourceRecordSet, logger logrus.FieldLogger) error { + logger = logger.WithField("record set", fmt.Sprintf("%s %s", *recordSet.Type, *recordSet.Name)) + _, err := client.ChangeResourceRecordSets(&route53.ChangeResourceRecordSetsInput{ + HostedZoneId: aws.String(zoneID), + ChangeBatch: &route53.ChangeBatch{ + Changes: []*route53.Change{ + { + Action: aws.String("DELETE"), + ResourceRecordSet: recordSet, + }, + }, + }, + }) + if err != nil { + return err + } - // Unfortunately the ListUsers Users do not have tags populated, so we need to query each: - userOut, err := iamClient.GetUser(&iam.GetUserInput{UserName: user.UserName}) - if err != nil { - return usersAsAWSObjects, err - } + logger.Info("Deleted") + return nil +} - tagsToMap, err := tagsToMap(userOut.User.Tags) - if err != nil { - return usersAsAWSObjects, err - } +func deleteS3(session *session.Session, arn arn.ARN, logger logrus.FieldLogger) error { + client := s3.New(session) - usersAsAWSObjects = append(usersAsAWSObjects, awsObjectWithTags{ - Name: *user.UserName, - Tags: tagsToMap, - }) + iter := s3manager.NewDeleteListIterator(client, &s3.ListObjectsInput{ + Bucket: aws.String(arn.Resource), + }) + err := s3manager.NewBatchDeleteWithClient(client).Delete(aws.BackgroundContext(), iter) + if err != nil { + return err + } + logger.Debug("Emptied") + _, err = client.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(arn.Resource), + }) + if err != nil { + return err } - return usersAsAWSObjects, nil + logger.Info("Deleted") + return nil }