From cf3cdeec8b3bd83cf384b8417e4b5454a5ef6db4 Mon Sep 17 00:00:00 2001 From: Derek Liang Date: Tue, 21 Nov 2023 23:50:55 -0800 Subject: [PATCH] [WIP] Feature: Support Nuking Lambda Layers/Versions (#605) * add initial resources for lambda_version * update readme for running tests in aws/resources * refactor to lambda layer, list cmd working * rm debugs * layer to version fetching * add tests for get all * add nuke lamda layers * add tests for nuke lambda * clean up * clean up logs * fix naming * address comments * add comment denoting nuances of name output --- README.md | 8 ++ aws/resource_registry.go | 4 +- aws/resources/lambda.go | 4 +- aws/resources/lambda_layer.go | 148 ++++++++++++++++++++++++++++ aws/resources/lambda_layer_test.go | 126 +++++++++++++++++++++++ aws/resources/lambda_layer_types.go | 63 ++++++++++++ aws/resources/lambda_test.go | 7 +- aws/resources/lambda_types.go | 1 + config/config.go | 1 + config/config_test.go | 1 + 10 files changed, 357 insertions(+), 6 deletions(-) create mode 100644 aws/resources/lambda_layer.go create mode 100644 aws/resources/lambda_layer_test.go create mode 100644 aws/resources/lambda_layer_types.go diff --git a/README.md b/README.md index 8ddbcb8f..c578e846 100644 --- a/README.md +++ b/README.md @@ -638,6 +638,14 @@ cd aws go test -v -run TestListAMIs ``` + +And to run a specific test, such as `TestLambdaFunction_GetAll` in package `aws/resources`: + +```bash +cd aws/resources +go test -v -run TestLambdaFunction_GetAll +``` + Use env-vars to opt-in to special tests, which are expensive to run: ```bash diff --git a/aws/resource_registry.go b/aws/resource_registry.go index d6301a60..abebf4cb 100644 --- a/aws/resource_registry.go +++ b/aws/resource_registry.go @@ -1,9 +1,10 @@ package aws import ( + "reflect" + "github.com/aws/aws-sdk-go/aws/session" "github.com/gruntwork-io/cloud-nuke/aws/resources" - "reflect" ) const Global = "global" @@ -84,6 +85,7 @@ func getRegisteredRegionalResources() []AwsResources { &resources.KinesisStreams{}, &resources.KmsCustomerKeys{}, &resources.LambdaFunctions{}, + &resources.LambdaLayers{}, &resources.LaunchConfigs{}, &resources.LaunchTemplates{}, &resources.MacieMember{}, diff --git a/aws/resources/lambda.go b/aws/resources/lambda.go index 56169fd5..a8adb5fa 100644 --- a/aws/resources/lambda.go +++ b/aws/resources/lambda.go @@ -2,8 +2,6 @@ package resources import ( "context" - "github.com/gruntwork-io/cloud-nuke/telemetry" - commonTelemetry "github.com/gruntwork-io/go-commons/telemetry" "time" "github.com/aws/aws-sdk-go/aws" @@ -12,6 +10,8 @@ import ( "github.com/gruntwork-io/cloud-nuke/config" "github.com/gruntwork-io/cloud-nuke/logging" "github.com/gruntwork-io/cloud-nuke/report" + "github.com/gruntwork-io/cloud-nuke/telemetry" + commonTelemetry "github.com/gruntwork-io/go-commons/telemetry" ) func (lf *LambdaFunctions) getAll(c context.Context, configObj config.Config) ([]*string, error) { diff --git a/aws/resources/lambda_layer.go b/aws/resources/lambda_layer.go new file mode 100644 index 00000000..af1ab74e --- /dev/null +++ b/aws/resources/lambda_layer.go @@ -0,0 +1,148 @@ +package resources + +import ( + "context" + "time" + + "github.com/aws/aws-sdk-go/aws" + awsgo "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/lambda" + "github.com/gruntwork-io/cloud-nuke/config" + "github.com/gruntwork-io/cloud-nuke/logging" + "github.com/gruntwork-io/cloud-nuke/report" + "github.com/gruntwork-io/cloud-nuke/telemetry" + commonTelemetry "github.com/gruntwork-io/go-commons/telemetry" +) + +func (ll *LambdaLayers) getAll(c context.Context, configObj config.Config) ([]*string, error) { + var layers []*lambda.LayersListItem + var names []*string + + err := ll.Client.ListLayersPages( + &lambda.ListLayersInput{}, func(page *lambda.ListLayersOutput, lastPage bool) bool { + for _, layer := range page.Layers { + logging.Logger.Debugf("Found layer! %s", layer) + + if ll.shouldInclude(layer, configObj) { + layers = append(layers, layer) + } + } + + return !lastPage + }) + + if err != nil { + return nil, err + } + + for _, layer := range layers { + err := ll.Client.ListLayerVersionsPages( + &lambda.ListLayerVersionsInput{ + LayerName: layer.LayerName, + }, func(page *lambda.ListLayerVersionsOutput, lastPage bool) bool { + for _, version := range page.LayerVersions { + logging.Logger.Debugf("Found layer version! %s", version) + + // Currently the output is just the identifier which is the layer's name. + // There could be potentially multiple rows of the same identifier or + // layer name since there can be multiple versions of it. + names = append(names, layer.LayerName) + } + + return !lastPage + }) + + if err != nil { + return nil, err + } + } + + return names, nil +} + +func (ll *LambdaLayers) shouldInclude(lambdaLayer *lambda.LayersListItem, configObj config.Config) bool { + if lambdaLayer == nil { + return false + } + + // Lambda layers are immutable, so the created date of the latest version + // is on par with last modified + fnLastModified := aws.StringValue(lambdaLayer.LatestMatchingVersion.CreatedDate) + fnName := lambdaLayer.LayerName + layout := "2006-01-02T15:04:05.000+0000" + lastModifiedDateTime, err := time.Parse(layout, fnLastModified) + if err != nil { + logging.Logger.Debugf("Could not parse last modified timestamp (%s) of Lambda layer %s. Excluding from delete.", fnLastModified, *fnName) + return false + } + + return configObj.LambdaLayer.ShouldInclude(config.ResourceValue{ + Time: &lastModifiedDateTime, + Name: fnName, + }) +} + +func (ll *LambdaLayers) nukeAll(names []*string) error { + if len(names) == 0 { + logging.Logger.Debugf("No Lambda Layers to nuke in region %s", ll.Region) + return nil + } + + logging.Logger.Debugf("Deleting all Lambda Layers in region %s", ll.Region) + deletedNames := []*string{} + deleteLayerVersions := []*lambda.DeleteLayerVersionInput{} + + for _, name := range names { + err := ll.Client.ListLayerVersionsPages( + &lambda.ListLayerVersionsInput{ + LayerName: name, + }, func(page *lambda.ListLayerVersionsOutput, lastPage bool) bool { + for _, version := range page.LayerVersions { + logging.Logger.Debugf("Found layer version! %s", version) + params := &lambda.DeleteLayerVersionInput{ + LayerName: name, + VersionNumber: version.Version, + } + deleteLayerVersions = append(deleteLayerVersions, params) + } + + return !lastPage + }) + + if err != nil { + return err + } + } + + for _, params := range deleteLayerVersions { + + _, err := ll.Client.DeleteLayerVersion(params) + + if err != nil { + return err + } + + // Record status of this resource + e := report.Entry{ + Identifier: aws.StringValue(params.LayerName), + ResourceType: "Lambda layer", + Error: err, + } + report.Record(e) + + if err != nil { + logging.Logger.Errorf("[Failed] %s: %s", *params.LayerName, err) + telemetry.TrackEvent(commonTelemetry.EventContext{ + EventName: "Error Nuking Lambda Layer", + }, map[string]interface{}{ + "region": ll.Region, + }) + } else { + deletedNames = append(deletedNames, params.LayerName) + logging.Logger.Debugf("Deleted Lambda Layer: %s", awsgo.StringValue(params.LayerName)) + } + } + + logging.Logger.Debugf("[OK] %d Lambda Layer(s) deleted in %s", len(deletedNames), ll.Region) + return nil +} diff --git a/aws/resources/lambda_layer_test.go b/aws/resources/lambda_layer_test.go new file mode 100644 index 00000000..ce56ed9f --- /dev/null +++ b/aws/resources/lambda_layer_test.go @@ -0,0 +1,126 @@ +package resources + +import ( + "context" + "regexp" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/lambda" + "github.com/aws/aws-sdk-go/service/lambda/lambdaiface" + "github.com/gruntwork-io/cloud-nuke/config" + "github.com/gruntwork-io/cloud-nuke/telemetry" + "github.com/stretchr/testify/require" +) + +type mockedLambdaLayer struct { + lambdaiface.LambdaAPI + ListLayersOutput lambda.ListLayersOutput + ListLayerVersionsOutput lambda.ListLayerVersionsOutput + DeleteLayerVersionOutput lambda.DeleteLayerVersionOutput +} + +func (m mockedLambdaLayer) ListLayersPages(input *lambda.ListLayersInput, fn func(*lambda.ListLayersOutput, bool) bool) error { + fn(&m.ListLayersOutput, true) + return nil +} + +func (m mockedLambdaLayer) ListLayerVersionsPages(input *lambda.ListLayerVersionsInput, fn func(*lambda.ListLayerVersionsOutput, bool) bool) error { + fn(&m.ListLayerVersionsOutput, true) + return nil +} + +func TestLambdaLayer_GetAll(t *testing.T) { + telemetry.InitTelemetry("cloud-nuke", "") + t.Parallel() + + testName1 := "test-lambda-layer1" + testName1Version1 := int64(1) + + testName2 := "test-lambda-layer2" + + testTime := time.Now() + + layout := "2006-01-02T15:04:05.000+0000" + testTimeStr := "2023-07-28T12:34:56.789+0000" + testTime, err := time.Parse(layout, testTimeStr) + require.NoError(t, err) + + ll := LambdaLayers{ + Client: mockedLambdaLayer{ + ListLayersOutput: lambda.ListLayersOutput{ + Layers: []*lambda.LayersListItem{ + { + LayerName: aws.String(testName1), + LatestMatchingVersion: &lambda.LayerVersionsListItem{ + CreatedDate: aws.String(testTimeStr), + }, + }, + { + LayerName: aws.String(testName2), + LatestMatchingVersion: &lambda.LayerVersionsListItem{ + CreatedDate: aws.String(testTimeStr), + }, + }, + }, + }, + ListLayerVersionsOutput: lambda.ListLayerVersionsOutput{ + LayerVersions: []*lambda.LayerVersionsListItem{ + { + Version: &testName1Version1, + }, + }, + }, + }, + } + + tests := map[string]struct { + configObj config.ResourceType + expected []string + }{ + "emptyFilter": { + configObj: config.ResourceType{}, + expected: []string{testName1, testName2}, + }, + "nameExclusionFilter": { + configObj: config.ResourceType{ + ExcludeRule: config.FilterRule{ + NamesRegExp: []config.Expression{{ + RE: *regexp.MustCompile(testName1), + }}}, + }, + expected: []string{testName2}, + }, + "timeAfterExclusionFilter": { + configObj: config.ResourceType{ + ExcludeRule: config.FilterRule{ + TimeAfter: aws.Time(testTime.Add(-2 * time.Hour)), + }}, + expected: []string{}, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + names, err := ll.getAll(context.Background(), config.Config{ + LambdaLayer: tc.configObj, + }) + require.NoError(t, err) + require.Equal(t, tc.expected, aws.StringValueSlice(names)) + }) + } +} + +func TestLambdaLayer_NukeAll(t *testing.T) { + telemetry.InitTelemetry("cloud-nuke", "") + t.Parallel() + + ll := LambdaLayers{ + Client: mockedLambdaLayer{ + DeleteLayerVersionOutput: lambda.DeleteLayerVersionOutput{}, + }, + } + + err := ll.nukeAll([]*string{aws.String("test")}) + require.NoError(t, err) +} diff --git a/aws/resources/lambda_layer_types.go b/aws/resources/lambda_layer_types.go new file mode 100644 index 00000000..8a321ca1 --- /dev/null +++ b/aws/resources/lambda_layer_types.go @@ -0,0 +1,63 @@ +package resources + +import ( + "context" + + awsgo "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/lambda" + "github.com/aws/aws-sdk-go/service/lambda/lambdaiface" + "github.com/gruntwork-io/cloud-nuke/config" + "github.com/gruntwork-io/go-commons/errors" +) + +type LambdaLayers struct { + Client lambdaiface.LambdaAPI + Region string + LambdaFunctionNames []string +} + +func (lf *LambdaLayers) Init(session *session.Session) { + lf.Client = lambda.New(session) +} + +func (lf *LambdaLayers) ResourceName() string { + return "lambda_layer" +} + +// ResourceIdentifiers - The names of the lambda functions +func (lf *LambdaLayers) ResourceIdentifiers() []string { + return lf.LambdaFunctionNames +} + +func (lf *LambdaLayers) MaxBatchSize() int { + // Tentative batch size to ensure AWS doesn't throttle + return 49 +} + +func (lf *LambdaLayers) GetAndSetIdentifiers(c context.Context, configObj config.Config) ([]string, error) { + identifiers, err := lf.getAll(c, configObj) + if err != nil { + return nil, err + } + + lf.LambdaFunctionNames = awsgo.StringValueSlice(identifiers) + return lf.LambdaFunctionNames, nil +} + +// Nuke - nuke 'em all!!! +func (lf *LambdaLayers) Nuke(identifiers []string) error { + if err := lf.nukeAll(awsgo.StringSlice(identifiers)); err != nil { + return errors.WithStackTrace(err) + } + + return nil +} + +type LambdaVersionDeleteError struct { + name string +} + +func (e LambdaVersionDeleteError) Error() string { + return "Lambda Function:" + e.name + "was not deleted" +} diff --git a/aws/resources/lambda_test.go b/aws/resources/lambda_test.go index 969f365d..d9b3d0e1 100644 --- a/aws/resources/lambda_test.go +++ b/aws/resources/lambda_test.go @@ -2,15 +2,16 @@ package resources import ( "context" + "regexp" + "testing" + "time" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/lambda" "github.com/aws/aws-sdk-go/service/lambda/lambdaiface" "github.com/gruntwork-io/cloud-nuke/config" "github.com/gruntwork-io/cloud-nuke/telemetry" "github.com/stretchr/testify/require" - "regexp" - "testing" - "time" ) type mockedLambda struct { diff --git a/aws/resources/lambda_types.go b/aws/resources/lambda_types.go index e58e7b7b..c6641c3c 100644 --- a/aws/resources/lambda_types.go +++ b/aws/resources/lambda_types.go @@ -2,6 +2,7 @@ package resources import ( "context" + awsgo "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/lambda" diff --git a/config/config.go b/config/config.go index 1456c485..e9356449 100644 --- a/config/config.go +++ b/config/config.go @@ -59,6 +59,7 @@ type Config struct { KMSCustomerKeys KMSCustomerKeyResourceType `yaml:"KMSCustomerKeys"` KinesisStream ResourceType `yaml:"KinesisStream"` LambdaFunction ResourceType `yaml:"LambdaFunction"` + LambdaLayer ResourceType `yaml:"LambdaLayer"` LaunchConfiguration ResourceType `yaml:"LaunchConfiguration"` LaunchTemplate ResourceType `yaml:"LaunchTemplate"` MacieMember ResourceType `yaml:"MacieMember"` diff --git a/config/config_test.go b/config/config_test.go index ee975ebe..9109a44c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -78,6 +78,7 @@ func emptyConfig() *Config { ResourceType{FilterRule{}, FilterRule{}}, ResourceType{FilterRule{}, FilterRule{}}, ResourceType{FilterRule{}, FilterRule{}}, + ResourceType{FilterRule{}, FilterRule{}}, } }