From e136cbce3af4c44156789ce4309d67326c622bd3 Mon Sep 17 00:00:00 2001 From: James Kwon Date: Wed, 3 Jul 2024 17:48:30 -0400 Subject: [PATCH] fix: backup vault nuke failure with recovery point dependancy --- aws/resources/backup_vault.go | 96 ++++++++++++++++++++++++++++-- aws/resources/backup_vault_test.go | 22 ++++++- 2 files changed, 112 insertions(+), 6 deletions(-) diff --git a/aws/resources/backup_vault.go b/aws/resources/backup_vault.go index 9cceaf24..dbc2d996 100644 --- a/aws/resources/backup_vault.go +++ b/aws/resources/backup_vault.go @@ -2,8 +2,11 @@ package resources import ( "context" + "fmt" + "time" "github.com/aws/aws-sdk-go/aws" + awsgo "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/backup" "github.com/gruntwork-io/cloud-nuke/config" "github.com/gruntwork-io/cloud-nuke/logging" @@ -34,6 +37,94 @@ func (bv *BackupVault) getAll(c context.Context, configObj config.Config) ([]*st return names, nil } +func (bv *BackupVault) nuke(name *string) error { + if err := bv.nukeRecoveryPoints(name); err != nil { + return errors.WithStackTrace(err) + } + + if err := bv.nukeBackupVault(name); err != nil { + return errors.WithStackTrace(err) + } + + return nil +} + +func (bv *BackupVault) nukeBackupVault(name *string) error { + _, err := bv.Client.DeleteBackupVaultWithContext(bv.Context, &backup.DeleteBackupVaultInput{ + BackupVaultName: name, + }) + if err != nil { + logging.Debugf("[Failed] nuking the backup vault %s: %v", awsgo.StringValue(name), err) + return errors.WithStackTrace(err) + } + return nil +} + +func (bv *BackupVault) nukeRecoveryPoints(name *string) error { + logging.Debugf("Nuking the recovery points of backup vault %s", awsgo.StringValue(name)) + + output, err := bv.Client.ListRecoveryPointsByBackupVaultWithContext(bv.Context, &backup.ListRecoveryPointsByBackupVaultInput{ + BackupVaultName: name, + }) + + if err != nil { + logging.Debugf("[Failed] listing the recovery points of backup vault %s: %v", awsgo.StringValue(name), err) + return errors.WithStackTrace(err) + } + + for _, recoveryPoint := range output.RecoveryPoints { + logging.Debugf("Deleting recovery point %s from backup vault %s", awsgo.StringValue(recoveryPoint.RecoveryPointArn), awsgo.StringValue(name)) + _, err := bv.Client.DeleteRecoveryPointWithContext(bv.Context, &backup.DeleteRecoveryPointInput{ + BackupVaultName: name, + RecoveryPointArn: recoveryPoint.RecoveryPointArn, + }) + + if err != nil { + logging.Debugf("[Failed] nuking the backup vault %s: %v", awsgo.StringValue(name), err) + return errors.WithStackTrace(err) + } + } + + // wait until all the recovery points nuked successfully + err = bv.WaitUntilRecoveryPointsDeleted(name) + if err != nil { + logging.Debugf("[Failed] waiting deletion of recovery points for backup vault %s: %v", awsgo.StringValue(name), err) + return errors.WithStackTrace(err) + } + + logging.Debugf("[Ok] successfully nuked recovery points of backup vault %s", awsgo.StringValue(name)) + + return nil +} + +func (bv *BackupVault) WaitUntilRecoveryPointsDeleted(name *string) error { + timeoutCtx, cancel := context.WithTimeout(bv.Context, 1*time.Minute) + defer cancel() + + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-timeoutCtx.Done(): + return fmt.Errorf("recovery point deletion check timed out after 1 minute") + case <-ticker.C: + output, err := bv.Client.ListRecoveryPointsByBackupVaultWithContext(bv.Context, &backup.ListRecoveryPointsByBackupVaultInput{ + BackupVaultName: name, + }) + if err != nil { + logging.Debugf("recovery point(s) existance checking error : %v", err) + return err + } + + if len(output.RecoveryPoints) == 0 { + return nil + } + logging.Debugf("%v Recovery point(s) still exists, waiting...", len(output.RecoveryPoints)) + } + } +} + func (bv *BackupVault) nukeAll(names []*string) error { if len(names) == 0 { logging.Debugf("No backup vaults to nuke in region %s", bv.Region) @@ -44,10 +135,7 @@ func (bv *BackupVault) nukeAll(names []*string) error { var deletedNames []*string for _, name := range names { - _, err := bv.Client.DeleteBackupVaultWithContext(bv.Context, &backup.DeleteBackupVaultInput{ - BackupVaultName: name, - }) - + err := bv.nuke(name) // Record status of this resource e := report.Entry{ Identifier: aws.StringValue(name), diff --git a/aws/resources/backup_vault_test.go b/aws/resources/backup_vault_test.go index 512b58f2..26883ece 100644 --- a/aws/resources/backup_vault_test.go +++ b/aws/resources/backup_vault_test.go @@ -18,8 +18,10 @@ import ( type mockedBackupVault struct { backupiface.BackupAPI - ListBackupVaultsOutput backup.ListBackupVaultsOutput - DeleteBackupVaultOutput backup.DeleteBackupVaultOutput + ListBackupVaultsOutput backup.ListBackupVaultsOutput + ListRecoveryPointsByBackupVaultOutput backup.ListRecoveryPointsByBackupVaultOutput + DeleteRecoveryPointOutput backup.DeleteRecoveryPointOutput + DeleteBackupVaultOutput backup.DeleteBackupVaultOutput } func (m mockedBackupVault) ListBackupVaultsPagesWithContext(_ awsgo.Context, _ *backup.ListBackupVaultsInput, fn func(*backup.ListBackupVaultsOutput, bool) bool, _ ...request.Option) error { @@ -31,6 +33,18 @@ func (m mockedBackupVault) DeleteBackupVaultWithContext(_ awsgo.Context, _ *back return &m.DeleteBackupVaultOutput, nil } +func (m mockedBackupVault) ListRecoveryPointsByBackupVaultWithContext(aws.Context, *backup.ListRecoveryPointsByBackupVaultInput, ...request.Option) (*backup.ListRecoveryPointsByBackupVaultOutput, error) { + return &m.ListRecoveryPointsByBackupVaultOutput, nil +} + +func (m mockedBackupVault) WaitUntilRecoveryPointsDeleted(*string) error { + return nil +} + +func (m mockedBackupVault) DeleteRecoveryPointWithContext(aws.Context, *backup.DeleteRecoveryPointInput, ...request.Option) (*backup.DeleteRecoveryPointOutput, error) { + return &m.DeleteRecoveryPointOutput, nil +} + func TestBackupVaultGetAll(t *testing.T) { t.Parallel() @@ -39,6 +53,7 @@ func TestBackupVaultGetAll(t *testing.T) { testName2 := "test-backup-vault-2" now := time.Now() bv := BackupVault{ + Client: mockedBackupVault{ ListBackupVaultsOutput: backup.ListBackupVaultsOutput{ BackupVaultList: []*backup.VaultListMember{ @@ -52,6 +67,7 @@ func TestBackupVaultGetAll(t *testing.T) { }, }}}, } + bv.BaseAwsResource.Init(nil) tests := map[string]struct { configObj config.ResourceType @@ -100,6 +116,8 @@ func TestBackupVaultNuke(t *testing.T) { DeleteBackupVaultOutput: backup.DeleteBackupVaultOutput{}, }, } + bv.BaseAwsResource.Init(nil) + bv.Context = context.Background() err := bv.nukeAll([]*string{aws.String("test-backup-vault")}) require.NoError(t, err)