Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

s3 access point support #650

Merged
merged 2 commits into from
Mar 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ Cloud-nuke suppports 🔎 inspecting and 🔥💀 deleting the following AWS res
| Lambda | Functions |
| SQS | Queues |
| S3 | Buckets |
| S3 | Access Points |
| S3 | Object Lambda Access Points |
| S3 | Multi Region Access Points |
| VPC | Default VPCs |
| VPC | Default rules in the un-deletable default security group |
| VPC | NAT Gateways |
Expand Down Expand Up @@ -553,6 +556,9 @@ of the file that are supported are listed here.
| rds-parameter-group | RdsParameterGroup | ✅ (Group Name) | ❌ | ❌ |
| rds-subnet-group | DBSubnetGroups | ✅ (DB Subnet Group Name) | ❌ | ❌ |
| s3 | s3 | ✅ (Bucket Name) | ✅ (Creation Time) | ✅ |
| s3-ap | s3AccessPoint | ✅ (Access point Name) | ❌ | ❌ |
| s3-olap | S3ObjectLambdaAccessPoint | ✅ (Object Lambda Access point Name) | ❌ | ❌ |
| s3-mrap | S3MultiRegionAccessPoint | ✅ (Multi region Access point Name) | ✅ (Creation Time) | ❌ |
| ses-configuration-set | SesConfigurationset | ✅ (Configuration set name) | ❌ | ❌ |
| ses-email-template | SesEmailTemplates | ✅ (Template Name) | ✅ (Creation Time) | ❌ |
| ses-identity | SesIdentity | ✅ (Identity -Mail/Domain) | ❌ | ❌ |
Expand Down
3 changes: 3 additions & 0 deletions aws/resource_registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,9 @@ func getRegisteredRegionalResources() []AwsResource {
&resources.RdsParameterGroup{},
&resources.RedshiftClusters{},
&resources.S3Buckets{},
&resources.S3AccessPoint{},
&resources.S3ObjectLambdaAccessPoint{},
&resources.S3MultiRegionAccessPoint{},
&resources.SageMakerNotebookInstances{},
&resources.SecretsManagerSecrets{},
&resources.SecurityHub{},
Expand Down
77 changes: 77 additions & 0 deletions aws/resources/s3_access_point.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package resources

import (
"context"
"fmt"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3control"
"github.com/gruntwork-io/cloud-nuke/config"
"github.com/gruntwork-io/cloud-nuke/logging"
"github.com/gruntwork-io/cloud-nuke/report"
"github.com/gruntwork-io/cloud-nuke/util"
"github.com/gruntwork-io/go-commons/errors"
)

func (ap *S3AccessPoint) getAll(c context.Context, configObj config.Config) ([]*string, error) {
accountID, ok := c.Value(util.AccountIdKey).(string)
if !ok {
logging.Errorf("unable to read the account-id from context")
return nil, errors.WithStackTrace(fmt.Errorf("unable to lookup the account id"))
}

// set the account id in object as this is mandatory to nuke an access point
ap.AccountID = aws.String(accountID)

var accessPoints []*string
err := ap.Client.ListAccessPointsPages(&s3control.ListAccessPointsInput{
AccountId: ap.AccountID,
}, func(lapo *s3control.ListAccessPointsOutput, lastPage bool) bool {
for _, accessPoint := range lapo.AccessPointList {
if configObj.S3AccessPoint.ShouldInclude(config.ResourceValue{
Name: accessPoint.Name,
}) {
accessPoints = append(accessPoints, accessPoint.Name)
}
}
return !lastPage
})
return accessPoints, errors.WithStackTrace(err)
}

func (ap *S3AccessPoint) nukeAll(identifiers []*string) error {
if len(identifiers) == 0 {
logging.Debugf("No Access point(s) to nuke in region %s", ap.Region)
return nil
}

logging.Debugf("Deleting all Access points in region %s", ap.Region)
var deleted []*string

for _, id := range identifiers {

_, err := ap.Client.DeleteAccessPoint(&s3control.DeleteAccessPointInput{
AccountId: ap.AccountID,
Name: id,
})

// Record status of this resource
e := report.Entry{
Identifier: aws.StringValue(id),
ResourceType: "S3 Access point",
Error: err,
}
report.Record(e)

if err != nil {
logging.Debugf("[Failed] %s", err)
} else {
deleted = append(deleted, id)
logging.Debugf("Deleted S3 access point: %s", aws.StringValue(id))
}
}

logging.Debugf("[OK] %d S3 Access point(s) deleted in %s", len(deleted), ap.Region)

return nil
}
99 changes: 99 additions & 0 deletions aws/resources/s3_access_point_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package resources

import (
"context"
"regexp"
"testing"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3control"
"github.com/aws/aws-sdk-go/service/s3control/s3controliface"
"github.com/gruntwork-io/cloud-nuke/config"
"github.com/gruntwork-io/cloud-nuke/telemetry"
"github.com/gruntwork-io/cloud-nuke/util"
"github.com/stretchr/testify/require"
)

type mocks3AccessPoint struct {
s3controliface.S3ControlAPI
ListAccessPointsOutput s3control.ListAccessPointsOutput
DeleteAccessPointOutput s3control.DeleteAccessPointOutput
}

func (m mocks3AccessPoint) ListAccessPointsPages(_ *s3control.ListAccessPointsInput, fn func(*s3control.ListAccessPointsOutput, bool) bool) error {
fn(&m.ListAccessPointsOutput, true)
return nil
}
func (m mocks3AccessPoint) DeleteAccessPoint(_ *s3control.DeleteAccessPointInput) (*s3control.DeleteAccessPointOutput, error) {
return &m.DeleteAccessPointOutput, nil
}

func TestS3AccessPoint_GetAll(t *testing.T) {
telemetry.InitTelemetry("cloud-nuke", "")
t.Parallel()

testName01 := "test-access-point-01"
testName02 := "test-access-point-02"

ctx := context.Background()
ctx = context.WithValue(ctx, util.AccountIdKey, "test-account-id")

ap := S3AccessPoint{
Client: mocks3AccessPoint{
ListAccessPointsOutput: s3control.ListAccessPointsOutput{
AccessPointList: []*s3control.AccessPoint{
{
Name: aws.String(testName01),
},
{
Name: aws.String(testName02),
},
},
},
},
AccountID: aws.String("test-account-id"),
}

tests := map[string]struct {
configObj config.ResourceType
expected []string
}{
"emptyFilter": {
configObj: config.ResourceType{},
expected: []string{testName01, testName02},
},
"nameExclusionFilter": {
configObj: config.ResourceType{
ExcludeRule: config.FilterRule{
NamesRegExp: []config.Expression{{
RE: *regexp.MustCompile(testName01),
}}},
},
expected: []string{testName02},
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {

names, err := ap.getAll(ctx, config.Config{
S3AccessPoint: tc.configObj,
})
require.NoError(t, err)
require.Equal(t, tc.expected, aws.StringValueSlice(names))
})
}
}

func TestS3AccessPoint_NukeAll(t *testing.T) {
telemetry.InitTelemetry("cloud-nuke", "")
t.Parallel()

rc := S3AccessPoint{
Client: mocks3AccessPoint{
DeleteAccessPointOutput: s3control.DeleteAccessPointOutput{},
},
}

err := rc.nukeAll([]*string{aws.String("test")})
require.NoError(t, err)
}
54 changes: 54 additions & 0 deletions aws/resources/s3_access_point_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package resources

import (
"context"

awsgo "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3control"
"github.com/aws/aws-sdk-go/service/s3control/s3controliface"
"github.com/gruntwork-io/cloud-nuke/config"
"github.com/gruntwork-io/go-commons/errors"
)

type S3AccessPoint struct {
BaseAwsResource
Client s3controliface.S3ControlAPI
Region string
AccessPoints []string
AccountID *string
}

func (ap *S3AccessPoint) Init(session *session.Session) {
ap.Client = s3control.New(session)
}

func (ap *S3AccessPoint) ResourceName() string {
return "s3-ap"
}

func (ap *S3AccessPoint) ResourceIdentifiers() []string {
return ap.AccessPoints
}

func (ap *S3AccessPoint) MaxBatchSize() int {
return 5
}

func (ap *S3AccessPoint) GetAndSetIdentifiers(c context.Context, configObj config.Config) ([]string, error) {
identifiers, err := ap.getAll(c, configObj)
if err != nil {
return nil, err
}

ap.AccessPoints = awsgo.StringValueSlice(identifiers)
return ap.AccessPoints, nil
}

func (ap *S3AccessPoint) Nuke(identifiers []string) error {
if err := ap.nukeAll(awsgo.StringSlice(identifiers)); err != nil {
return errors.WithStackTrace(err)
}

return nil
}
83 changes: 83 additions & 0 deletions aws/resources/s3_multi_region_access_point.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package resources

import (
"context"
"fmt"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3control"
"github.com/gruntwork-io/cloud-nuke/config"
"github.com/gruntwork-io/cloud-nuke/logging"
"github.com/gruntwork-io/cloud-nuke/report"
"github.com/gruntwork-io/cloud-nuke/util"
"github.com/gruntwork-io/go-commons/errors"
)

func (ap *S3MultiRegionAccessPoint) getAll(c context.Context, configObj config.Config) ([]*string, error) {
accountID, ok := c.Value(util.AccountIdKey).(string)
if !ok {
logging.Errorf("unable to read the account-id from context")
return nil, errors.WithStackTrace(fmt.Errorf("unable to lookup the account id"))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was wondering why should be defined dynamic errors and not dedicated struct errors like https://github.com/gruntwork-io/cloud-nuke/blob/master/aws/resources/ec2.go#L621

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey @denis256, I'm not too sure what's the benefit of declaring a dedicated error struct in this scenario. Potential benefit would be using this error code to test things more thoroughly in unit tests -- we are not actually doing that and just checking whether certain scenario produce an error or not.

Do you have other benefits you have in mind?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, it is mostly to have more robust error checking

}

// set the account id in object as this is mandatory to nuke an access point
ap.AccountID = aws.String(accountID)

var accessPoints []*string
err := ap.Client.ListMultiRegionAccessPointsPages(&s3control.ListMultiRegionAccessPointsInput{
AccountId: ap.AccountID,
}, func(lapo *s3control.ListMultiRegionAccessPointsOutput, lastPage bool) bool {
for _, accessPoint := range lapo.AccessPoints {
if configObj.S3MultiRegionAccessPoint.ShouldInclude(config.ResourceValue{
Name: accessPoint.Name,
Time: accessPoint.CreatedAt,
}) {
accessPoints = append(accessPoints, accessPoint.Name)
}
}
return !lastPage
})
if err != nil {
logging.Errorf("[FAILED] Multi region access point listing - %v", err)
}
return accessPoints, errors.WithStackTrace(err)
}

func (ap *S3MultiRegionAccessPoint) nukeAll(identifiers []*string) error {
if len(identifiers) == 0 {
logging.Debugf("No Multi region access point(s) to nuke in region %s", ap.Region)
return nil
}

logging.Debugf("Deleting all Multi region access points in region %s", ap.Region)
var deleted []*string

for _, id := range identifiers {

_, err := ap.Client.DeleteMultiRegionAccessPoint(&s3control.DeleteMultiRegionAccessPointInput{
AccountId: ap.AccountID,
Details: &s3control.DeleteMultiRegionAccessPointInput_{
Name: id,
},
})

// Record status of this resource
e := report.Entry{
Identifier: aws.StringValue(id),
ResourceType: "S3 Multi Region Access point",
Error: err,
}
report.Record(e)

if err != nil {
logging.Debugf("[Failed] %s", err)
} else {
deleted = append(deleted, id)
logging.Debugf("Deleted S3 Multi region access point: %s", aws.StringValue(id))
}
}

logging.Debugf("[OK] %d S3 Multi region access point(s) deleted in %s", len(deleted), ap.Region)

return nil
}
Loading