From a6034a2c1ff8e8d5404f10ec82e3ae62ea9989c8 Mon Sep 17 00:00:00 2001 From: James Kwon Date: Wed, 19 Jul 2023 00:47:08 -0400 Subject: [PATCH 1/3] Refactor AMI Resource Type --- aws/ami.go | 35 ++++---- aws/ami_test.go | 210 ++++++++++++------------------------------- aws/ami_types.go | 12 +-- aws/aws.go | 2 +- commands/cli_test.go | 2 +- 5 files changed, 83 insertions(+), 178 deletions(-) diff --git a/aws/ami.go b/aws/ami.go index fe260534..9c342eec 100644 --- a/aws/ami.go +++ b/aws/ami.go @@ -3,28 +3,25 @@ package aws import ( "time" - "github.com/gruntwork-io/cloud-nuke/telemetry" - "github.com/gruntwork-io/cloud-nuke/util" + awsgo "github.com/aws/aws-sdk-go/aws" commonTelemetry "github.com/gruntwork-io/go-commons/telemetry" "github.com/aws/aws-sdk-go/aws" - awsgo "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/gruntwork-io/cloud-nuke/config" "github.com/gruntwork-io/cloud-nuke/logging" "github.com/gruntwork-io/cloud-nuke/report" + "github.com/gruntwork-io/cloud-nuke/telemetry" "github.com/gruntwork-io/go-commons/errors" ) // Returns a formatted string of AMI Image ids -func getAllAMIs(session *session.Session, region string, excludeAfter time.Time) ([]*string, error) { - svc := ec2.New(session) - +func (ami AMI) getAll(configObj config.Config) ([]*string, error) { params := &ec2.DescribeImagesInput{ Owners: []*string{awsgo.String("self")}, } - output, err := svc.DescribeImages(params) + output, err := ami.Client.DescribeImages(params) if err != nil { return nil, errors.WithStackTrace(err) } @@ -37,8 +34,10 @@ func getAllAMIs(session *session.Session, region string, excludeAfter time.Time) return nil, err } - // Test for time exclusion and check if resource is managed by AWS Backup (see note in README) - if excludeAfter.After(createdTime) && !util.HasAWSBackupTag(image.Tags) { + if configObj.AMI.ShouldInclude(config.ResourceValue{ + Name: image.Name, + Time: &createdTime, + }) { imageIds = append(imageIds, image.ImageId) } } @@ -46,16 +45,14 @@ func getAllAMIs(session *session.Session, region string, excludeAfter time.Time) return imageIds, nil } -// Deletes all AMIs -func nukeAllAMIs(session *session.Session, imageIds []*string) error { - svc := ec2.New(session) - +// Deletes all AMI +func (ami AMI) nukeAll(imageIds []*string) error { if len(imageIds) == 0 { - logging.Logger.Debugf("No AMIs to nuke in region %s", *session.Config.Region) + logging.Logger.Debugf("No AMI to nuke in region %s", ami.Region) return nil } - logging.Logger.Debugf("Deleting all AMIs in region %s", *session.Config.Region) + logging.Logger.Debugf("Deleting all AMI in region %s", ami.Region) deletedCount := 0 for _, imageID := range imageIds { @@ -63,7 +60,7 @@ func nukeAllAMIs(session *session.Session, imageIds []*string) error { ImageId: imageID, } - _, err := svc.DeregisterImage(params) + _, err := ami.Client.DeregisterImage(params) // Record status of this resource e := report.Entry{ @@ -78,7 +75,7 @@ func nukeAllAMIs(session *session.Session, imageIds []*string) error { telemetry.TrackEvent(commonTelemetry.EventContext{ EventName: "Error Nuking AMI", }, map[string]interface{}{ - "region": *session.Config.Region, + "region": ami.Region, }) } else { deletedCount++ @@ -86,6 +83,6 @@ func nukeAllAMIs(session *session.Session, imageIds []*string) error { } } - logging.Logger.Debugf("[OK] %d AMI(s) terminated in %s", deletedCount, *session.Config.Region) + logging.Logger.Debugf("[OK] %d AMI(s) terminated in %s", deletedCount, ami.Region) return nil } diff --git a/aws/ami_test.go b/aws/ami_test.go index fb448f81..81b73998 100644 --- a/aws/ami_test.go +++ b/aws/ami_test.go @@ -1,178 +1,86 @@ package aws import ( + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/gruntwork-io/cloud-nuke/config" "github.com/gruntwork-io/cloud-nuke/telemetry" + "github.com/stretchr/testify/assert" + "regexp" "testing" "time" awsgo "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/gruntwork-io/cloud-nuke/logging" - "github.com/gruntwork-io/cloud-nuke/util" - "github.com/gruntwork-io/go-commons/errors" - "github.com/stretchr/testify/assert" ) -func waitUntilImageAvailable(svc *ec2.EC2, input *ec2.DescribeImagesInput) error { - for i := 0; i < 70; i++ { - output, err := svc.DescribeImages(input) - if err != nil { - return err - } - - if *output.Images[0].State == "available" { - return nil - } - - logging.Logger.Debug("Waiting for ELB to be available") - time.Sleep(10 * time.Second) - } - - return ImageAvailableError{} +type mockedAMI struct { + ec2iface.EC2API + DescribeImagesOutput ec2.DescribeImagesOutput + DeregisterImageOutput ec2.DeregisterImageOutput } -func createTestAMI(t *testing.T, session *session.Session, name string) (*ec2.Image, error) { - svc := ec2.New(session) - instance := createTestEC2Instance(t, session, name, false) - output, err := svc.CreateImage(&ec2.CreateImageInput{ - InstanceId: instance.InstanceId, - Name: awsgo.String(name), - }) - - if err != nil { - assert.Failf(t, "Could not create test AMI", errors.WithStackTrace(err).Error()) - } - - params := &ec2.DescribeImagesInput{ - Owners: []*string{awsgo.String("self")}, - ImageIds: []*string{output.ImageId}, - } - - err = svc.WaitUntilImageExists(params) - - if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) - } - - err = svc.WaitUntilImageAvailable(params) - - if err != nil { - // clean this up since we won't use it again - defer nukeAllAMIs(session, []*string{output.ImageId}) - return nil, errors.WithStackTrace(err) - } - - images, err := svc.DescribeImages(&ec2.DescribeImagesInput{ - ImageIds: []*string{output.ImageId}, - }) - - if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) - } +func (m mockedAMI) DescribeImages(input *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) { + return &m.DescribeImagesOutput, nil +} - return images.Images[0], nil +func (m mockedAMI) DeregisterImage(input *ec2.DeregisterImageInput) (*ec2.DeregisterImageOutput, error) { + return &m.DeregisterImageOutput, nil } -func TestListAMIs(t *testing.T) { +func TestAMIGetAll(t *testing.T) { telemetry.InitTelemetry("cloud-nuke", "") t.Parallel() - region, err := getRandomRegion() - if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) - } - session, err := session.NewSession(&awsgo.Config{ - Region: awsgo.String(region)}, - ) - - if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) - } - - uniqueTestID := "cloud-nuke-test-" + util.UniqueID() - image, err := createTestAMI(t, session, uniqueTestID) - attempts := 0 - - for err != nil && attempts <= 10 { - // Image didn't become availabe in time, try again - image, err = createTestAMI(t, session, uniqueTestID) - attempts++ - } - - if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) - } - - // clean up after this test - defer nukeAllAMIs(session, []*string{image.ImageId}) - defer nukeAllEc2Instances(session, findEC2InstancesByNameTag(t, session, uniqueTestID)) - - amis, err := getAllAMIs(session, region, time.Now().Add(1*time.Hour*-1)) - if err != nil { - assert.Fail(t, "Unable to fetch list of AMIs") - } - - assert.NotContains(t, awsgo.StringValueSlice(amis), *image.ImageId) - - amis, err = getAllAMIs(session, region, time.Now().Add(1*time.Hour)) - if err != nil { - assert.Fail(t, "Unable to fetch list of AMIs") - } - - assert.Contains(t, awsgo.StringValueSlice(amis), *image.ImageId) + testName := "test-ami" + testImageId := "test-image-id" + now := time.Now() + acm := AMI{ + Client: mockedAMI{ + DescribeImagesOutput: ec2.DescribeImagesOutput{ + Images: []*ec2.Image{{ + ImageId: &testImageId, + Name: &testName, + CreationDate: awsgo.String(now.Format("2006-01-02T15:04:05.000Z")), + }}, + }, + }, + } + + // without filters + amis, err := acm.getAll(config.Config{}) + assert.NoError(t, err) + assert.Contains(t, awsgo.StringValueSlice(amis), testImageId) + + // with name filter + amis, err = acm.getAll(config.Config{ + AMI: config.ResourceType{ + ExcludeRule: config.FilterRule{ + NamesRegExp: []config.Expression{{ + RE: *regexp.MustCompile("test-ami"), + }}}}}) + assert.NoError(t, err) + assert.NotContains(t, awsgo.StringValueSlice(amis), testImageId) + + // with time filter + amis, err = acm.getAll(config.Config{ + AMI: config.ResourceType{ + ExcludeRule: config.FilterRule{ + TimeAfter: awsgo.Time(now.Add(-12 * time.Hour))}}}) + assert.NoError(t, err) + assert.NotContains(t, awsgo.StringValueSlice(amis), testImageId) } -func TestNukeAMIs(t *testing.T) { +func TestAMINukeAll(t *testing.T) { telemetry.InitTelemetry("cloud-nuke", "") t.Parallel() - region, err := getRandomRegion() - if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) - } - session, err := session.NewSession(&awsgo.Config{ - Region: awsgo.String(region)}, - ) - svc := ec2.New(session) - - if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) - } - - uniqueTestID := "cloud-nuke-test-" + util.UniqueID() - image, err := createTestAMI(t, session, uniqueTestID) - attempts := 0 - - for err != nil && attempts <= 10 { - // Image didn't become availabe in time, try again - image, err = createTestAMI(t, session, uniqueTestID) - attempts++ - } - - if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) - } - - // clean up ec2 instance created by the above call - defer nukeAllEc2Instances(session, findEC2InstancesByNameTag(t, session, uniqueTestID)) - - _, err = svc.DescribeImages(&ec2.DescribeImagesInput{ - ImageIds: []*string{image.ImageId}, - }) - - if err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) - } - - if err := nukeAllAMIs(session, []*string{image.ImageId}); err != nil { - assert.Fail(t, errors.WithStackTrace(err).Error()) - } - - amis, err := getAllAMIs(session, region, time.Now().Add(1*time.Hour)) - if err != nil { - assert.Fail(t, "Unable to fetch list of AMIs") + testName := "test-ami" + acm := AMI{ + Client: mockedAMI{ + DeregisterImageOutput: ec2.DeregisterImageOutput{}, + }, } - assert.NotContains(t, awsgo.StringValueSlice(amis), *image.ImageId) + err := acm.nukeAll([]*string{&testName}) + assert.NoError(t, err) } diff --git a/aws/ami_types.go b/aws/ami_types.go index d61717f4..a42fd379 100644 --- a/aws/ami_types.go +++ b/aws/ami_types.go @@ -15,23 +15,23 @@ type AMIs struct { } // ResourceName - the simple name of the aws resource -func (image AMIs) ResourceName() string { +func (ami AMIs) ResourceName() string { return "ami" } // ResourceIdentifiers - The AMI image ids -func (image AMIs) ResourceIdentifiers() []string { - return image.ImageIds +func (ami AMIs) ResourceIdentifiers() []string { + return ami.ImageIds } -func (image AMIs) MaxBatchSize() int { +func (ami AMIs) MaxBatchSize() int { // Tentative batch size to ensure AWS doesn't throttle return 49 } // Nuke - nuke 'em all!!! -func (image AMIs) Nuke(session *session.Session, identifiers []string) error { - if err := nukeAllAMIs(session, awsgo.StringSlice(identifiers)); err != nil { +func (ami AMIs) Nuke(session *session.Session, identifiers []string) error { + if err := ami.nukeAll(awsgo.StringSlice(identifiers)); err != nil { return errors.WithStackTrace(err) } diff --git a/aws/aws.go b/aws/aws.go index 2e9d9368..12303394 100644 --- a/aws/aws.go +++ b/aws/aws.go @@ -764,7 +764,7 @@ func GetAllResources(targetRegions []string, excludeAfter time.Time, resourceTyp } if IsNukeable(amis.ResourceName(), resourceTypes) { start := time.Now() - imageIds, err := getAllAMIs(cloudNukeSession, region, excludeAfter) + imageIds, err := amis.getAll(configObj) if err != nil { ge := report.GeneralError{ Error: err, diff --git a/commands/cli_test.go b/commands/cli_test.go index b33bbc15..17ce966b 100644 --- a/commands/cli_test.go +++ b/commands/cli_test.go @@ -50,7 +50,7 @@ func TestIsValidResourceType(t *testing.T) { func TestIsNukeable(t *testing.T) { ec2ResourceName := aws.EC2Instances{}.ResourceName() - amiResourceName := aws.AMIs{}.ResourceName() + amiResourceName := aws.AMI{}.ResourceName() assert.Equal(t, aws.IsNukeable(ec2ResourceName, []string{ec2ResourceName}), true) assert.Equal(t, aws.IsNukeable(ec2ResourceName, []string{"all"}), true) From 4f317a97c08d8c423313149518aa79be6303cd9c Mon Sep 17 00:00:00 2001 From: James Kwon Date: Fri, 21 Jul 2023 13:23:38 -0400 Subject: [PATCH 2/3] remove file --- commands/cli_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/cli_test.go b/commands/cli_test.go index 17ce966b..b33bbc15 100644 --- a/commands/cli_test.go +++ b/commands/cli_test.go @@ -50,7 +50,7 @@ func TestIsValidResourceType(t *testing.T) { func TestIsNukeable(t *testing.T) { ec2ResourceName := aws.EC2Instances{}.ResourceName() - amiResourceName := aws.AMI{}.ResourceName() + amiResourceName := aws.AMIs{}.ResourceName() assert.Equal(t, aws.IsNukeable(ec2ResourceName, []string{ec2ResourceName}), true) assert.Equal(t, aws.IsNukeable(ec2ResourceName, []string{"all"}), true) From 565e907bc4aac574d4f937fa314f7e19689042f8 Mon Sep 17 00:00:00 2001 From: James Kwon Date: Mon, 24 Jul 2023 17:15:08 -0400 Subject: [PATCH 3/3] Address merge conflict --- aws/ami.go | 4 ++-- aws/ami_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/ami.go b/aws/ami.go index 9c342eec..3563c4a7 100644 --- a/aws/ami.go +++ b/aws/ami.go @@ -16,7 +16,7 @@ import ( ) // Returns a formatted string of AMI Image ids -func (ami AMI) getAll(configObj config.Config) ([]*string, error) { +func (ami AMIs) getAll(configObj config.Config) ([]*string, error) { params := &ec2.DescribeImagesInput{ Owners: []*string{awsgo.String("self")}, } @@ -46,7 +46,7 @@ func (ami AMI) getAll(configObj config.Config) ([]*string, error) { } // Deletes all AMI -func (ami AMI) nukeAll(imageIds []*string) error { +func (ami AMIs) nukeAll(imageIds []*string) error { if len(imageIds) == 0 { logging.Logger.Debugf("No AMI to nuke in region %s", ami.Region) return nil diff --git a/aws/ami_test.go b/aws/ami_test.go index 81b73998..c4ac2bac 100644 --- a/aws/ami_test.go +++ b/aws/ami_test.go @@ -34,7 +34,7 @@ func TestAMIGetAll(t *testing.T) { testName := "test-ami" testImageId := "test-image-id" now := time.Now() - acm := AMI{ + acm := AMIs{ Client: mockedAMI{ DescribeImagesOutput: ec2.DescribeImagesOutput{ Images: []*ec2.Image{{ @@ -75,7 +75,7 @@ func TestAMINukeAll(t *testing.T) { t.Parallel() testName := "test-ami" - acm := AMI{ + acm := AMIs{ Client: mockedAMI{ DeregisterImageOutput: ec2.DeregisterImageOutput{}, },