diff --git a/aws/resources/ec2_vpc.go b/aws/resources/ec2_vpc.go index a1ddaffe..43278ba3 100644 --- a/aws/resources/ec2_vpc.go +++ b/aws/resources/ec2_vpc.go @@ -4,16 +4,19 @@ import ( "context" cerrors "errors" "fmt" + "slices" "strconv" "strings" "time" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" "github.com/gruntwork-io/cloud-nuke/util" "github.com/pterm/pterm" + "github.com/gruntwork-io/go-commons/retry" + awsgo "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/gruntwork-io/cloud-nuke/config" @@ -64,7 +67,7 @@ func (v *EC2VPCs) getAll(_ context.Context, configObj config.Config) ([]*string, { Name: awsgo.String("is-default"), Values: []*string{ - aws.String(strconv.FormatBool(configObj.VPC.DefaultOnly)), // convert the bool status into string + awsgo.String(strconv.FormatBool(configObj.VPC.DefaultOnly)), // convert the bool status into string }, }, }, @@ -113,7 +116,7 @@ func (v *EC2VPCs) nukeAll(vpcIds []string) error { for _, id := range vpcIds { var err error - err = nuke(v.Client, id) + err = nuke(v.Client, v.ELBClient, id) // Record status of this resource e := report.Entry{ @@ -136,10 +139,30 @@ func (v *EC2VPCs) nukeAll(vpcIds []string) error { return nil } -func nuke(client ec2iface.EC2API, vpcID string) error { +func nuke(client ec2iface.EC2API, elbClient elbv2iface.ELBV2API, vpcID string) error { + var err error // Note: order is quite important, otherwise you will encounter dependency violation errors. + + err = nukeAttachedLB(client, elbClient, vpcID) + if err != nil { + logging.Debug(fmt.Sprintf("Error nuking loadbalancer for VPC %s: %s", vpcID, err.Error())) + return err + } + + err = nukeTargetGroups(elbClient, vpcID) + if err != nil { + logging.Debug(fmt.Sprintf("Error nuking target group for VPC %s: %s", vpcID, err.Error())) + return err + } + + err = nukeEc2Instances(client, vpcID) + if err != nil { + logging.Debug(fmt.Sprintf("Error nuking instances for VPC %s: %s", vpcID, err.Error())) + return err + } + logging.Debug(fmt.Sprintf("Start nuking VPC %s", vpcID)) - err := nukeDhcpOptions(client, vpcID) + err = nukeDhcpOptions(client, vpcID) if err != nil { logging.Debug(fmt.Sprintf("Error cleaning up DHCP Options for VPC %s: %s", vpcID, err.Error())) return err @@ -169,6 +192,44 @@ func nuke(client ec2iface.EC2API, vpcID string) error { return err } + // NOTE: Since the network interfaces attached to the load balancer may not be removed immediately after removing the load balancers, + // and attempting to remove the internet gateway without waiting for these network interfaces removal will result in an error. + // The actual error message states: 'has some mapped public address(es). Please unmap those public address(es) before detaching the gateway.' + // Therefore, it is recommended to wait until all the load balancer-related network interfaces are detached and deleted before proceeding. + // + // Important : The waiting should be happen before nuking the internet gateway + err = retry.DoWithRetry( + logging.Logger.WithTime(time.Now()), + "Waiting for all Network interfaces to be deleted.", + // Wait a maximum of 5 minutes: 10 seconds in between, up to 30 times + 30, 10*time.Second, + func() error { + interfaces, err := client.DescribeNetworkInterfaces( + &ec2.DescribeNetworkInterfacesInput{ + Filters: []*ec2.Filter{ + { + Name: awsgo.String("vpc-id"), + Values: []*string{awsgo.String(vpcID)}, + }, + }, + }, + ) + if err != nil { + return errors.WithStackTrace(err) + } + + if len(interfaces.NetworkInterfaces) == 0 { + return nil + } + + return fmt.Errorf("Not all Network interfaces are deleted.") + }, + ) + if err != nil { + logging.Debug(fmt.Sprintf("Error waiting up Network interfaces deletion for VPC %s: %s", vpcID, err.Error())) + return errors.WithStackTrace(err) + } + err = nukeInternetGateways(client, vpcID) if err != nil { logging.Debug(fmt.Sprintf("Error cleaning up Internet Gateway for VPC %s: %s", vpcID, err.Error())) @@ -199,6 +260,12 @@ func nuke(client ec2iface.EC2API, vpcID string) error { return err } + err = nukePeeringConnections(client, vpcID) + if err != nil { + pterm.Error.Println(fmt.Sprintf("Error deleting VPC Peer connection %s: %s ", vpcID, err)) + return err + } + err = nukeVpc(client, vpcID) if err != nil { pterm.Error.Println(fmt.Sprintf("Error deleting VPC %s: %s ", vpcID, err)) @@ -210,6 +277,79 @@ func nuke(client ec2iface.EC2API, vpcID string) error { return nil } +func nukePeeringConnections(client ec2iface.EC2API, vpcID string) error { + logging.Debug(fmt.Sprintf("Finding VPC peering connections to nuke for: %s", vpcID)) + + peerConnections := []*string{} + vpcIds := []string{vpcID} + requesterFilters := []*ec2.Filter{ + { + Name: awsgo.String("requester-vpc-info.vpc-id"), + Values: awsgo.StringSlice(vpcIds), + }, { + Name: awsgo.String("status-code"), + Values: awsgo.StringSlice([]string{"active"}), + }, + } + accepterFilters := []*ec2.Filter{ + { + Name: awsgo.String("accepter-vpc-info.vpc-id"), + Values: awsgo.StringSlice(vpcIds), + }, { + Name: awsgo.String("status-code"), + Values: awsgo.StringSlice([]string{"active"}), + }, + } + + // check the peering connection as requester + err := client.DescribeVpcPeeringConnectionsPages( + &ec2.DescribeVpcPeeringConnectionsInput{ + Filters: requesterFilters, + }, + func(page *ec2.DescribeVpcPeeringConnectionsOutput, lastPage bool) bool { + for _, connection := range page.VpcPeeringConnections { + peerConnections = append(peerConnections, connection.VpcPeeringConnectionId) + } + return !lastPage + }, + ) + if err != nil { + logging.Debug(fmt.Sprintf("Failed to describe vpc peering connections for vpc as requester: %s", vpcID)) + return errors.WithStackTrace(err) + } + + // check the peering connection as accepter + err = client.DescribeVpcPeeringConnectionsPages( + &ec2.DescribeVpcPeeringConnectionsInput{ + Filters: accepterFilters, + }, + func(page *ec2.DescribeVpcPeeringConnectionsOutput, lastPage bool) bool { + for _, connection := range page.VpcPeeringConnections { + peerConnections = append(peerConnections, connection.VpcPeeringConnectionId) + } + return !lastPage + }, + ) + if err != nil { + logging.Debug(fmt.Sprintf("Failed to describe vpc peering connections for vpc as accepter: %s", vpcID)) + return errors.WithStackTrace(err) + } + + logging.Debug(fmt.Sprintf("Found %d VPC Peering connections to Nuke.", len(peerConnections))) + + for _, connection := range peerConnections { + _, err := client.DeleteVpcPeeringConnection(&ec2.DeleteVpcPeeringConnectionInput{ + VpcPeeringConnectionId: connection, + }) + if err != nil { + logging.Debug(fmt.Sprintf("Failed to delete peering connection for vpc: %s", vpcID)) + return errors.WithStackTrace(err) + } + } + + return nil +} + // nukeVpcInternetGateways // This function is specifically for VPCs. It retrieves all the internet gateways attached to the given VPC ID and nuke them func nukeInternetGateways(client ec2iface.EC2API, vpcID string) error { @@ -678,3 +818,156 @@ func nukeVpc(client ec2iface.EC2API, vpcID string) error { logging.Debug(fmt.Sprintf("Successfully deleted VPC %s", vpcID)) return nil } + +func nukeAttachedLB(client ec2iface.EC2API, elbclient elbv2iface.ELBV2API, vpcID string) error { + logging.Debug(fmt.Sprintf("Describing load balancers for %s", vpcID)) + + // get all loadbalancers + output, err := elbclient.DescribeLoadBalancers(nil) + if err != nil { + logging.Debug(fmt.Sprintf("Failed to describe loadbalancer for %s", vpcID)) + return errors.WithStackTrace(err) + } + var attachedLoadBalancers []string + + // get a list of load balancers which was attached on the vpc + for _, lb := range output.LoadBalancers { + if awsgo.StringValue(lb.VpcId) != vpcID { + continue + } + + attachedLoadBalancers = append(attachedLoadBalancers, awsgo.StringValue(lb.LoadBalancerArn)) + } + + // check the load-balancers are attached with any vpc-endpoint-service, then nuke them first + esoutput, err := client.DescribeVpcEndpointServiceConfigurations(nil) + if err != nil { + logging.Debug(fmt.Sprintf("Failed to describe vpc endpoint services for %s", vpcID)) + return errors.WithStackTrace(err) + } + + // since we don't want duplicating the endpoint service ids, using a map here + var nukableEndpointServices = make(map[*string]struct{}) + for _, config := range esoutput.ServiceConfigurations { + // check through gateway load balancer attachments and select the service for nuking + for _, gwlb := range config.GatewayLoadBalancerArns { + if slices.Contains(attachedLoadBalancers, awsgo.StringValue(gwlb)) { + nukableEndpointServices[config.ServiceId] = struct{}{} + } + } + + // check through network load balancer attachments and select the service for nuking + for _, nwlb := range config.NetworkLoadBalancerArns { + if slices.Contains(attachedLoadBalancers, awsgo.StringValue(nwlb)) { + nukableEndpointServices[config.ServiceId] = struct{}{} + } + } + } + + logging.Debug(fmt.Sprintf("Found %d Endpoint services attached with the load balancers to nuke.", len(nukableEndpointServices))) + + // nuke the endpoint services + for endpointService := range nukableEndpointServices { + _, err := client.DeleteVpcEndpointServiceConfigurations(&ec2.DeleteVpcEndpointServiceConfigurationsInput{ + ServiceIds: []*string{endpointService}, + }) + if err != nil { + logging.Debug(fmt.Sprintf("Failed to delete endpoint service %v for %s", awsgo.StringValue(endpointService), vpcID)) + return errors.WithStackTrace(err) + } + } + logging.Debug(fmt.Sprintf("Successfully deleted he endpoints attached with load balancers for %s", vpcID)) + + // nuke the load-balancers + for _, lb := range attachedLoadBalancers { + _, err := elbclient.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{ + LoadBalancerArn: awsgo.String(lb), + }) + if err != nil { + logging.Debug(fmt.Sprintf("Failed to delete loadbalancer %v for %s", lb, vpcID)) + return errors.WithStackTrace(err) + } + } + + logging.Debug(fmt.Sprintf("Successfully deleted loadbalancers attached %s", vpcID)) + return nil +} + +func nukeTargetGroups(client elbv2iface.ELBV2API, vpcID string) error { + logging.Debug(fmt.Sprintf("Describing target groups for %s", vpcID)) + + output, err := client.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{}) + if err != nil { + logging.Debug(fmt.Sprintf("Failed to describe target groups for %s", vpcID)) + return errors.WithStackTrace(err) + } + + for _, tg := range output.TargetGroups { + // if the target group is not for this vpc, then skip + if tg.VpcId != nil && awsgo.StringValue(tg.VpcId) == vpcID { + _, err := client.DeleteTargetGroup(&elbv2.DeleteTargetGroupInput{ + TargetGroupArn: tg.TargetGroupArn, + }) + if err != nil { + logging.Debug(fmt.Sprintf("Failed to delete target group %v for %s", *tg.TargetGroupArn, vpcID)) + return errors.WithStackTrace(err) + } + } + + } + + logging.Debug(fmt.Sprintf("Successfully deleted target group attached %s", vpcID)) + + return nil +} + +func nukeEc2Instances(client ec2iface.EC2API, vpcID string) error { + logging.Debug(fmt.Sprintf("Describing instances for %s", vpcID)) + output, err := client.DescribeInstances(&ec2.DescribeInstancesInput{ + Filters: []*ec2.Filter{ + { + Name: awsgo.String("network-interface.vpc-id"), + Values: awsgo.StringSlice([]string{ + vpcID, + }), + }, + }, + }) + + if err != nil { + logging.Debug(fmt.Sprintf("Failed to describe instances for %s", vpcID)) + return errors.WithStackTrace(err) + } + + var terminateInstances []*string + for _, instance := range output.Reservations { + for _, i := range instance.Instances { + terminateInstances = append(terminateInstances, i.InstanceId) + } + } + + if len(terminateInstances) > 0 { + logging.Debug(fmt.Sprintf("Found %d VPC attached instances to Nuke.", len(terminateInstances))) + + _, err := client.TerminateInstances(&ec2.TerminateInstancesInput{ + InstanceIds: terminateInstances, + }) + if err != nil { + logging.Debug(fmt.Sprintf("Failed to terminate instances for %s", vpcID)) + return errors.WithStackTrace(err) + } + + // weight for terminate the instances + logging.Debug(fmt.Sprintf("waiting for the instance to be terminated for %s", vpcID)) + err = client.WaitUntilInstanceTerminated(&ec2.DescribeInstancesInput{ + InstanceIds: terminateInstances, + }) + if err != nil { + logging.Debug(fmt.Sprintf("Failed to wait instance termination for %s", vpcID)) + return errors.WithStackTrace(err) + } + } + + logging.Debug(fmt.Sprintf("Successfully deleted instances for %s", vpcID)) + return nil +} diff --git a/aws/resources/ec2_vpc_test.go b/aws/resources/ec2_vpc_test.go index cbee4bf3..9e1e2c79 100644 --- a/aws/resources/ec2_vpc_test.go +++ b/aws/resources/ec2_vpc_test.go @@ -6,10 +6,12 @@ import ( "testing" "time" - "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go/aws" awsgo "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" "github.com/gruntwork-io/cloud-nuke/config" "github.com/gruntwork-io/cloud-nuke/util" "github.com/stretchr/testify/require" @@ -17,8 +19,13 @@ import ( type mockedEC2VPCs struct { ec2iface.EC2API - DescribeVpcsOutput ec2.DescribeVpcsOutput - DeleteVpcOutput ec2.DeleteVpcOutput + DescribeVpcsOutput ec2.DescribeVpcsOutput + DeleteVpcOutput ec2.DeleteVpcOutput + DescribeVpcPeeringConnectionsOutput ec2.DescribeVpcPeeringConnectionsOutput + DescribeInstancesOutput ec2.DescribeInstancesOutput + TerminateInstancesOutput ec2.TerminateInstancesOutput + DescribeVpcEndpointServiceConfigurationsOutput ec2.DescribeVpcEndpointServiceConfigurationsOutput + DeleteVpcEndpointServiceConfigurationsOutput ec2.DeleteVpcEndpointServiceConfigurationsOutput } func (m mockedEC2VPCs) DescribeVpcs(input *ec2.DescribeVpcsInput) (*ec2.DescribeVpcsOutput, error) { @@ -28,7 +35,27 @@ func (m mockedEC2VPCs) DescribeVpcs(input *ec2.DescribeVpcsInput) (*ec2.Describe func (m mockedEC2VPCs) DeleteVpc(input *ec2.DeleteVpcInput) (*ec2.DeleteVpcOutput, error) { return &m.DeleteVpcOutput, nil } +func (m mockedEC2VPCs) DescribeVpcPeeringConnectionsPages(input *ec2.DescribeVpcPeeringConnectionsInput, callback func(page *ec2.DescribeVpcPeeringConnectionsOutput, lastPage bool) bool) error { + callback(&m.DescribeVpcPeeringConnectionsOutput, true) + return nil +} +func (m mockedEC2VPCs) DescribeInstances(*ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) { + return &m.DescribeInstancesOutput, nil +} + +func (m mockedEC2VPCs) TerminateInstances(*ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) { + return &m.TerminateInstancesOutput, nil +} +func (m mockedEC2VPCs) WaitUntilInstanceTerminated(*ec2.DescribeInstancesInput) error { + return nil +} +func (m mockedEC2VPCs) DescribeVpcEndpointServiceConfigurations(*ec2.DescribeVpcEndpointServiceConfigurationsInput) (*ec2.DescribeVpcEndpointServiceConfigurationsOutput, error) { + return &m.DescribeVpcEndpointServiceConfigurationsOutput, nil +} +func (m mockedEC2VPCs) DeleteVpcEndpointServiceConfigurations(*ec2.DeleteVpcEndpointServiceConfigurationsInput) (*ec2.DeleteVpcEndpointServiceConfigurationsOutput, error) { + return &m.DeleteVpcEndpointServiceConfigurationsOutput, nil +} func TestEC2VPC_GetAll(t *testing.T) { t.Parallel() @@ -128,3 +155,115 @@ func TestEC2VPC_GetAll(t *testing.T) { // err := vpc.nukeAll([]string{"test-vpc-id1", "test-vpc-id2"}) // require.NoError(t, err) //} + +func TestEC2VPCPeeringConnections_NukeAll(t *testing.T) { + t.Parallel() + vpc := EC2VPCs{ + Client: mockedEC2VPCs{ + DescribeVpcPeeringConnectionsOutput: ec2.DescribeVpcPeeringConnectionsOutput{}, + }, + } + + err := nukePeeringConnections(vpc.Client, "vpc-test-00001") + require.NoError(t, err) +} + +type mockedEC2ELB struct { + elbv2iface.ELBV2API + DescribeLoadBalancersOutput elbv2.DescribeLoadBalancersOutput + DeleteLoadBalancerOutput elbv2.DeleteLoadBalancerOutput + + DescribeTargetGroupsOutput elbv2.DescribeTargetGroupsOutput + DeleteTargetGroupOutput elbv2.DeleteTargetGroupOutput +} + +func (m mockedEC2ELB) DescribeLoadBalancers(*elbv2.DescribeLoadBalancersInput) (*elbv2.DescribeLoadBalancersOutput, error) { + return &m.DescribeLoadBalancersOutput, nil +} +func (m mockedEC2ELB) DescribeTargetGroups(*elbv2.DescribeTargetGroupsInput) (*elbv2.DescribeTargetGroupsOutput, error) { + return &m.DescribeTargetGroupsOutput, nil +} + +func (m mockedEC2ELB) DeleteLoadBalancer(*elbv2.DeleteLoadBalancerInput) (*elbv2.DeleteLoadBalancerOutput, error) { + return &m.DeleteLoadBalancerOutput, nil +} +func (m mockedEC2ELB) DeleteTargetGroup(*elbv2.DeleteTargetGroupInput) (*elbv2.DeleteTargetGroupOutput, error) { + return &m.DeleteTargetGroupOutput, nil +} + +func TestAttachedLB_Nuke(t *testing.T) { + t.Parallel() + vpcID := "vpc-0e9a3e7c72d9f3a0f" + + vpc := EC2VPCs{ + Client: mockedEC2VPCs{ + DescribeVpcEndpointServiceConfigurationsOutput: ec2.DescribeVpcEndpointServiceConfigurationsOutput{ + ServiceConfigurations: []*ec2.ServiceConfiguration{ + { + GatewayLoadBalancerArns: awsgo.StringSlice([]string{ + "load-balancer-arn-00012", + }), + }, + }, + }, + }, + ELBClient: mockedEC2ELB{ + DescribeLoadBalancersOutput: elbv2.DescribeLoadBalancersOutput{ + LoadBalancers: []*elbv2.LoadBalancer{ + { + VpcId: awsgo.String(vpcID), + LoadBalancerArn: awsgo.String("load-balancer-arn-00012"), + }, + }, + }, + }, + } + + err := nukeAttachedLB(vpc.Client, vpc.ELBClient, vpcID) + require.NoError(t, err) +} + +func TestTargetGroup_Nuke(t *testing.T) { + t.Parallel() + vpcID := "vpc-0e9a3e7c72d9f3a0f" + + vpc := EC2VPCs{ + ELBClient: mockedEC2ELB{ + DescribeTargetGroupsOutput: elbv2.DescribeTargetGroupsOutput{ + TargetGroups: []*elbv2.TargetGroup{ + { + VpcId: awsgo.String(vpcID), + TargetGroupArn: awsgo.String("arn:aws:elasticloadbalancing:us-east-1:tg-001"), + }, + }, + }, + }, + } + + err := nukeTargetGroups(vpc.ELBClient, vpcID) + require.NoError(t, err) +} + +func TestEc2Instance_Nuke(t *testing.T) { + t.Parallel() + vpcID := "vpc-0e9a3e7c72d9f3a0f" + + vpc := EC2VPCs{ + Client: mockedEC2VPCs{ + DescribeInstancesOutput: ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + { + Instances: []*ec2.Instance{ + { + InstanceId: awsgo.String("instance-001"), + }, + }, + }, + }, + }, + }, + } + + err := nukeEc2Instances(vpc.Client, vpcID) + require.NoError(t, err) +} diff --git a/aws/resources/ec2_vpc_types.go b/aws/resources/ec2_vpc_types.go index 4f3105e7..d934e40b 100644 --- a/aws/resources/ec2_vpc_types.go +++ b/aws/resources/ec2_vpc_types.go @@ -7,19 +7,23 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" "github.com/gruntwork-io/cloud-nuke/config" "github.com/gruntwork-io/go-commons/errors" ) type EC2VPCs struct { BaseAwsResource - Client ec2iface.EC2API - Region string - VPCIds []string + Client ec2iface.EC2API + ELBClient elbv2iface.ELBV2API + Region string + VPCIds []string } func (v *EC2VPCs) Init(session *session.Session) { v.Client = ec2.New(session) + v.ELBClient = elbv2.New(session) } // ResourceName - the simple name of the aws resource