Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add acmpa ✨ #201

Merged
merged 12 commits into from
Aug 24, 2021
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ The currently supported functionality includes:

## AWS

- Deleting all ACM Private CA in an AWS account
- Deleting all Auto scaling groups in an AWS account
- Deleting all Elastic Load Balancers (Classic and V2) in an AWS account
- Deleting all Transit Gateways in an AWS account
Expand Down Expand Up @@ -248,6 +249,7 @@ To find out what we options are supported in the config file today, consult this
| secretsmanager | none | ✅ | none | none |
| nat-gateway | none | ✅ | none | none |
| accessanalyzer | none | ✅ | none | none |
| acmpca | none | none | none | none |
| ec2 instance | none | none | none | none |
| iam role | none | none | none | none |
| ... (more to come) | none | none | none | none |
Expand Down Expand Up @@ -335,6 +337,13 @@ cd aws
go test -v -run TestListAMIs
```

Use env-vars to opt-in to special tests, which are expensive to run:

```bash
# Run acmpca tests
TEST_ACMPCA_EXPENSIVE_ENABLE=1 go test -v ./...
```

### Formatting

Every source file in this project should be formatted with `go fmt`.
Expand Down
145 changes: 145 additions & 0 deletions aws/acmpca.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
package aws

import (
"fmt"
"sync"
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/acmpca"
"github.com/gruntwork-io/cloud-nuke/config"
"github.com/gruntwork-io/cloud-nuke/logging"
"github.com/gruntwork-io/go-commons/errors"
"github.com/hashicorp/go-multierror"
)

// getAllACMPCA returns a list of all arns of ACMPCA, which can be deleted.
func getAllACMPCA(session *session.Session, region string, excludeAfter time.Time) ([]*string, error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like this isn't implementing the config file based name filter using regexes. You need to plumb the config file through to here so that it properly filters out those that don't match the regex.

svc := acmpca.New(session)
var arns []*string
if paginationErr := svc.ListCertificateAuthoritiesPages(&acmpca.ListCertificateAuthoritiesInput{}, func(p *acmpca.ListCertificateAuthoritiesOutput, lastPage bool) bool {
for _, ca := range p.CertificateAuthorities {
if shouldIncludeACMPCA(ca, excludeAfter) {
arns = append(arns, ca.Arn)
}
}
return !lastPage
}); paginationErr != nil {
return nil, errors.WithStackTrace(paginationErr)
}
return arns, nil
}

func shouldIncludeACMPCA(ca *acmpca.CertificateAuthority, excludeAfter time.Time) bool {
if ca == nil {
return false
}

statusSafe := aws.StringValue(ca.Status)
isAlreadyDeleted := statusSafe == acmpca.CertificateAuthorityStatusDeleted
if isAlreadyDeleted {
return false
}

// reference time for excludeAfter is lastStateChangeAt time,
// unless it was never changed and createAt time is used.
var referenceTime time.Time
if ca.LastStateChangeAt == nil {
referenceTime = aws.TimeValue(ca.CreatedAt)
} else {
referenceTime = aws.TimeValue(ca.LastStateChangeAt)
}
if excludeAfter.Before(referenceTime) {
return false
}

return config.ShouldInclude(
aws.StringValue(ca.Arn),
nil,
nil,
)
}

// nukeAllACMPCA will delete all ACMPCA, which are given by a list of arns.
func nukeAllACMPCA(session *session.Session, arns []*string) error {
if len(arns) == 0 {
logging.Logger.Infof("No ACMPCA to nuke in region %s", *session.Config.Region)
return nil
}
svc := acmpca.New(session)

logging.Logger.Infof("Deleting all ACMPCA in region %s", *session.Config.Region)
// There is no bulk delete acmpca API, so we delete the batch of ARNs concurrently using go routines.
wg := new(sync.WaitGroup)
wg.Add(len(arns))
errChans := make([]chan error, len(arns))
for i, arn := range arns {
errChans[i] = make(chan error, 1)
go deleteACMPCAASync(wg, errChans[i], svc, arn, aws.StringValue(session.Config.Region))
}
wg.Wait()

// Collect all the errors from the async delete calls into a single error struct.
var allErrs *multierror.Error
for _, errChan := range errChans {
if err := <-errChan; err != nil {
allErrs = multierror.Append(allErrs, err)
logging.Logger.Errorf("[Failed] %s", err)
}
}
return errors.WithStackTrace(allErrs.ErrorOrNil())
}

// deleteACMPCAASync deletes the provided ACMPCA arn. Intended to be run in a goroutine, using wait groups
// and a return channel for errors.
func deleteACMPCAASync(wg *sync.WaitGroup, errChan chan error, svc *acmpca.ACMPCA, arn *string, region string) {
defer wg.Done()

logging.Logger.Infof("Fetching details of CA to be deleted for ACMPCA %s in region %s", *arn, region)
details, detailsErr := svc.DescribeCertificateAuthority(&acmpca.DescribeCertificateAuthorityInput{CertificateAuthorityArn: arn})
if detailsErr != nil {
errChan <- detailsErr
return
}
if details.CertificateAuthority == nil {
errChan <- fmt.Errorf("could not find CA %s", aws.StringValue(arn))
return
}
if details.CertificateAuthority.Status == nil {
errChan <- fmt.Errorf("could not fetch status for CA %s", aws.StringValue(arn))
return
}

// find out, whether we have to disable the CA first, prior to deletion.
statusSafe := aws.StringValue(details.CertificateAuthority.Status)
shouldUpdateStatus := statusSafe != acmpca.CertificateAuthorityStatusCreating &&
statusSafe != acmpca.CertificateAuthorityStatusPendingCertificate &&
statusSafe != acmpca.CertificateAuthorityStatusDisabled &&
statusSafe != acmpca.CertificateAuthorityStatusDeleted

if shouldUpdateStatus {
logging.Logger.Infof("Setting status to 'DISABLED' for ACMPCA %s in region %s", *arn, region)
if _, updateStatusErr := svc.UpdateCertificateAuthority(&acmpca.UpdateCertificateAuthorityInput{
CertificateAuthorityArn: arn,
Status: aws.String(acmpca.CertificateAuthorityStatusDisabled),
}); updateStatusErr != nil {
errChan <- updateStatusErr
return
}
logging.Logger.Infof("Did set status to 'DISABLED' for ACMPCA: %s in region %s", *arn, region)
}

if _, deleteErr := svc.DeleteCertificateAuthority(&acmpca.DeleteCertificateAuthorityInput{
CertificateAuthorityArn: arn,
// the range is 7 to 30 days.
// since cloud-nuke should not be used in production,
// we assume that the minimum (7 days) is fine.
PermanentDeletionTimeInDays: aws.Int64(7),
}); deleteErr != nil {
errChan <- deleteErr
return
}
logging.Logger.Infof("Deleted ACMPCA: %s successfully", *arn)
errChan <- nil
}
154 changes: 154 additions & 0 deletions aws/acmpca_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
package aws

import (
"fmt"
"os"
"testing"
"time"

"github.com/gruntwork-io/cloud-nuke/logging"
"github.com/gruntwork-io/go-commons/retry"

awsgo "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/acmpca"
"github.com/gruntwork-io/cloud-nuke/util"
"github.com/gruntwork-io/go-commons/errors"
"github.com/stretchr/testify/assert"
)

// enableACMPCAExpensiveEnv is used to control whether to run
// the following test or not. The idea is that the test are disabled
// per default and one has to opt-in to enable the test as creating
// and destroying a ACM PCA is expensive.
// Upper bound, worst case: $400 / month per single CA create/delete.
const enableACMPCAExpensiveEnv = "TEST_ACMPCA_EXPENSIVE_ENABLE"

// runOrSkip decides whether to run or skip the test depending
// whether the env-var `TEST_ACMPCA_EXPENSIVE_ENABLE` is set or not.
func runOrSkip(t *testing.T) {
if _, isSet := os.LookupEnv(enableACMPCAExpensiveEnv); !isSet {
t.Skipf("Skipping the integration test for acmpca. Set the env-var '%s' to enable this expensive test.", enableACMPCAExpensiveEnv)
}
}

// createTestACMPCA will create am ACMPCA and return its ARN.
func createTestACMPCA(t *testing.T, session *session.Session, name string) *string {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sanity check: Do you have a sense on how much it would cost to run this test? It appears that the cost of ACM Private CA is $400 / month, pro rated to the number of X (days or hours?) that it is active. Does that active period include the days it is waiting to be deleted (the 7 day minimum)? If so, that can rack up quickly for us if we are running cloud nuke tests for each PR.

Depending on how much it costs, can you add an environment variable flag so that we can control when we run the tests related to the ACM PCA functionality (and not on every PR)?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am currently talking to our AWS Enterprise contact as indeed their documentation and support is not clear on this and racked up a significant amount of money just for starting/stopping a CA.

I will add some environment variable and this test will be opt-in in all cases so you have to make a concious decision to run it.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We heard back from our contact. The billing is pro-rated and does reconciliation after 24-48 hours and you only pay for those hours.

Hypothesis from Support-Agent

Hi Jan
It looks like the billing is fluctuating due to the 24-48 hour delay between the billing console and reality. When a certificate is created, the full amount for the month is added to the bill and this only changes to the pro-rated amount after the certificate has been deleted. I'll keep the internal ticket open with the service team just to confirm that this is exactly what is going on. In terms of your other questions, here are the answers:

  • How does AWS want to do pricing for short-lived CAs? (i.e. can we enable a testing pipeline or not)
    ->>>> I will forward this request to the service team and I'll update you as soon as I hear back from them.
  • Is there an absolute upper service quota for CAs in an account? (I see 200 quoted right now)
    ->>>> The quota of 200 is the default, and we can request an increase should you require it, but we would need a detailed use case for this request.
  • Do deleted CAs (with their 7 - 30 day pending period) count towards the service quota?
    ->>>> Yes, deleted Private CAs count towards the quota until the end of the restoration period.

Answer from AWS PCA Service Team confirming hypothesis from Support-Agent above

Quote

Thank you for your patience. I've just heard back from the team and my original answer about the delay between the billing console and reality is correct. Because the invoice for each month is issued a few days after the end of the month, by the time the bill is issued, everything will have refreshed and the bill will reflect correctly. For the current month, the "real" bill so far is around $7.

For some more context on how we charge for short term PCAs, the team provided the following:

AWS PCA charge customer based on the time period: from the time that CA was created to the time CA was deleted.

Facts:

Customer created 22 CAs in FRA region in July. 1 of them are in free trial.
Customer create CA, and delete them in a short period of time. --> this is fine, nothing wrong.

Customer check bill of July, which is not the final statement. There are delays between customer deleted the CA and the >billing team calculate the bill based on latest data.

If customer check right now, the bill will become about $168.

When customer finally get bill in August, the total charge will be around $8. Unless they created new CAs

Considering this information, the best way to monitor the bill for PCA specifically would be to log the hours and calculate this manually on your side throughout the month.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So how do we think about the pricing here?

If we have this test case and it runs, say, nightly for a month, what sort of fee are we talking? What if we get 20 PRs in this repo, and each one does 20 test runs in a month?

I just want to be extra sure we don't end up with a massive AWS bill as a surprise from adding this new functionality!

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@brikis98 I think we can go with this for now. @weitzjdevk implemented a feature flag to disable the tests by default. We can merge with that disabled.

Separately, I'll run an experiment to enable that test locally once. Then, we can check how much our bill was for August and make our decision on whether to continually test it or not based on that bill cost. Does that seem reasonable?

Copy link
Contributor

@weitzjdevk weitzjdevk Aug 4, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we have this test case and it runs, say, nightly for a month, what sort of fee are we talking? What if we get 20 PRs in this repo, and each one does 20 test runs in a month?

tl;dr: Price per run approx: $0.60 (if you issue a certificate + $0.75 therefore approx $1.50)
So the rough calculation how this works.

Monthly charge (for CA only): $400
Month has hours: 720

Price (CA) per hour approx. 400/720 approx $ 0.6
Issue a certificate charge: $0.75

Therefore roughly:

Price per run $0.60 + $0.75 approx. $1.50

Timeline (when starting the first day of the month)

  1. Day1 (first of the month): $0
  2. Run it
  3. Delete it
  4. Day1 bill: $0
  5. Day2 bill: $400 + 0.75
  6. Day3 bill: $400 + 0.75
  7. Day4 bill: $400/720 + 0.75

When you do the run in the middle of the month

  1. Day15 (middle of the month): $0
  2. Run it
  3. Delete it
  4. Day15 bill: $0
  5. Day16 bill (middle of month. Only 15 more days for pro-rating): $400 * 15 / 30 + 0.75 = $200 + 0.75
  6. Day17 bill: $200 + 0.75
  7. Day18 bill: $400/720 + 0.75

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have confirmed this cost in our account as well. I think 0.60 per run is a tad bit expensive to be continuously running the tests, but selectively running it everytime this feature changes seems reasonable, so the env var based feature switch is satisfactory.

// As an additional safety guard, we are adding another check here
// to decide whether to run the test or not.
runOrSkip(t)

svc := acmpca.New(session)
ca, err := svc.CreateCertificateAuthority(&acmpca.CreateCertificateAuthorityInput{
CertificateAuthorityConfiguration: &acmpca.CertificateAuthorityConfiguration{
KeyAlgorithm: awsgo.String(acmpca.KeyAlgorithmRsa2048),
SigningAlgorithm: awsgo.String(acmpca.SigningAlgorithmSha256withrsa),
Subject: &acmpca.ASN1Subject{
CommonName: awsgo.String(name),
},
},
CertificateAuthorityType: awsgo.String("ROOT"),
Tags: []*acmpca.Tag{
{
Key: awsgo.String("Name"),
Value: awsgo.String(name),
},
},
})
if err != nil {
assert.Failf(t, "Could not create ACMPCA", errors.WithStackTrace(err).Error())
}

// Wait for the ACMPCA to be ready (i.e. not CREATING).
// Ready does not mean "ACTIVE".
if err := retry.DoWithRetry(
logging.Logger,
fmt.Sprintf("Waiting for ACMPCA %s to be stable", awsgo.StringValue(ca.CertificateAuthorityArn)),
10,
1*time.Second,
func() error {
details, detailsErr := svc.DescribeCertificateAuthority(&acmpca.DescribeCertificateAuthorityInput{CertificateAuthorityArn: ca.CertificateAuthorityArn})
if detailsErr != nil {
return detailsErr
}
if details.CertificateAuthority == nil {
return fmt.Errorf("no CA instance found")
}
if awsgo.StringValue(details.CertificateAuthority.Status) != acmpca.CertificateAuthorityStatusPendingCertificate {
return fmt.Errorf("CA not ready, status %s", awsgo.StringValue(details.CertificateAuthority.Status))
}
return nil
},
); err != nil {
assert.Failf(t, "WARNING: ACMPCA is in some unfinished state. Delete manually inside the test-runner.", errors.WithStackTrace(err).Error())
}

return ca.CertificateAuthorityArn
}

func TestListACMPCA(t *testing.T) {
runOrSkip(t)
t.Parallel()

region, err := getRandomRegion()
if err != nil {
assert.Fail(t, errors.WithStackTrace(err).Error())
}
session, err := session.NewSession(&awsgo.Config{
Region: awsgo.String(region)},
)

if err != nil {
assert.Fail(t, errors.WithStackTrace(err).Error())
}

uniqueTestID := "cloud-nuke-test-" + util.UniqueID()
arn := createTestACMPCA(t, session, uniqueTestID)
// clean up after this test
defer nukeAllACMPCA(session, []*string{arn})

newARNs, err := getAllACMPCA(session, region, time.Now().Add(1*time.Hour*-1))
if err != nil {
assert.Fail(t, "Unable to fetch list of ACMPCA arns")
}
assert.NotContains(t, awsgo.StringValueSlice(newARNs), awsgo.StringValue(arn))

allARNs, err := getAllACMPCA(session, region, time.Now().Add(1*time.Hour))
if err != nil {
assert.Fail(t, "Unable to fetch list of ACMPCA arns")
}

assert.Contains(t, awsgo.StringValueSlice(allARNs), awsgo.StringValue(arn))
}

func TestNukeACMPCA(t *testing.T) {
runOrSkip(t)
t.Parallel()

region, err := getRandomRegion()
if err != nil {
assert.Fail(t, errors.WithStackTrace(err).Error())
}

session, err := session.NewSession(&awsgo.Config{
Region: awsgo.String(region)},
)

if err != nil {
assert.Fail(t, errors.WithStackTrace(err).Error())
}

uniqueTestID := "cloud-nuke-test-" + util.UniqueID()
arn := createTestACMPCA(t, session, uniqueTestID)

if err := nukeAllACMPCA(session, []*string{arn}); err != nil {
assert.Fail(t, errors.WithStackTrace(err).Error())
}

arns, err := getAllACMPCA(session, region, time.Now().Add(1*time.Hour))
if err != nil {
assert.Fail(t, "Unable to fetch list of ACMPCA arns")
}

assert.NotContains(t, awsgo.StringValueSlice(arns), awsgo.StringValue(arn))
}
36 changes: 36 additions & 0 deletions aws/acmpca_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package aws

import (
awsgo "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/gruntwork-io/go-commons/errors"
)

// ACMPA - represents all ACMPA
type ACMPCA struct {
ARNs []string
}

// ResourceName - the simple name of the aws resource
func (ca ACMPCA) ResourceName() string {
return "acmpca"
}

// ResourceIdentifiers - The volume ids of the ebs volumes
func (ca ACMPCA) ResourceIdentifiers() []string {
return ca.ARNs
}

func (ca ACMPCA) MaxBatchSize() int {
// Tentative batch size to ensure AWS doesn't throttle
return 10
}

// Nuke - nuke 'em all!!!
func (ca ACMPCA) Nuke(session *session.Session, arns []string) error {
if err := nukeAllACMPCA(session, awsgo.StringSlice(arns)); err != nil {
return errors.WithStackTrace(err)
}

return nil
}
15 changes: 15 additions & 0 deletions aws/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,20 @@ func GetAllResources(targetRegions []string, excludeAfter time.Time, resourceTyp
// The order in which resources are nuked is important
// because of dependencies between resources

// ACMPCA arns
acmpca := ACMPCA{}
if IsNukeable(acmpca.ResourceName(), resourceTypes) {
arns, err := getAllACMPCA(session, region, excludeAfter)
if err != nil {
return nil, errors.WithStackTrace(err)
}
if len(arns) > 0 {
acmpca.ARNs = awsgo.StringValueSlice(arns)
resourcesInRegion.Resources = append(resourcesInRegion.Resources, acmpca)
}
}
// End ACMPCA arns

// ASG Names
asGroups := ASGroups{}
if IsNukeable(asGroups.ResourceName(), resourceTypes) {
Expand Down Expand Up @@ -634,6 +648,7 @@ func GetAllResources(targetRegions []string, excludeAfter time.Time, resourceTyp
// ListResourceTypes - Returns list of resources which can be passed to --resource-type
func ListResourceTypes() []string {
resourceTypes := []string{
ACMPCA{}.ResourceName(),
ASGroups{}.ResourceName(),
LaunchConfigs{}.ResourceName(),
LoadBalancers{}.ResourceName(),
Expand Down
Loading