From 55a13baab998d173fc6498fbbe3809a039c52ee6 Mon Sep 17 00:00:00 2001 From: Joseph Herlant Date: Fri, 15 Dec 2017 00:11:24 -0800 Subject: [PATCH] Major refactor and add s3 bucket support --- CHANGELOG.md | 6 + Makefile | 10 +- README.md | 4 + TODO => TODO.md | 4 +- glide.lock | 68 +++++++ glide.yaml | 8 +- main.go | 38 ++-- errors.go => mapper/errors.go | 2 +- mapper/interface.go | 24 +++ mapper.go => mapper/mapper.go | 15 +- mapper_test.go => mapper/mapper_test.go | 9 +- mapper/mock.go | 36 ++++ mapper/mock_test.go | 41 +++++ cloudfront.go => providers/cloudfront.go | 8 +- .../cloudfront_test.go | 14 +- cloudwatch.go => providers/cloudwatch.go | 8 +- .../cloudwatch_test.go | 2 +- ec2.go => providers/ec2.go | 8 +- ec2_test.go => providers/ec2_test.go | 2 +- .../elasticbeanstalk.go | 8 +- .../elasticbeanstalk_test.go | 7 +- .../elasticsearch.go | 8 +- .../elasticsearch_test.go | 2 +- providers/logger.go | 10 + rds.go => providers/rds.go | 10 +- rds_test.go => providers/rds_test.go | 14 +- redshift.go => providers/redshift.go | 8 +- .../redshift_test.go | 19 +- providers/s3.go | 96 ++++++++++ providers/s3_test.go | 172 ++++++++++++++++++ 30 files changed, 584 insertions(+), 77 deletions(-) rename TODO => TODO.md (89%) create mode 100644 glide.lock rename errors.go => mapper/errors.go (98%) create mode 100644 mapper/interface.go rename mapper.go => mapper/mapper.go (98%) rename mapper_test.go => mapper/mapper_test.go (99%) create mode 100644 mapper/mock.go create mode 100644 mapper/mock_test.go rename cloudfront.go => providers/cloudfront.go (95%) rename cloudfront_test.go => providers/cloudfront_test.go (82%) rename cloudwatch.go => providers/cloudwatch.go (91%) rename cloudwatch_test.go => providers/cloudwatch_test.go (97%) rename ec2.go => providers/ec2.go (90%) rename ec2_test.go => providers/ec2_test.go (97%) rename elasticbeanstalk.go => providers/elasticbeanstalk.go (94%) rename elasticbeanstalk_test.go => providers/elasticbeanstalk_test.go (97%) rename elasticsearch.go => providers/elasticsearch.go (93%) rename elasticsearch_test.go => providers/elasticsearch_test.go (97%) create mode 100644 providers/logger.go rename rds.go => providers/rds.go (92%) rename rds_test.go => providers/rds_test.go (83%) rename redshift.go => providers/redshift.go (95%) rename redshift_test.go => providers/redshift_test.go (77%) create mode 100644 providers/s3.go create mode 100644 providers/s3_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d9dac3f..118c561 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Changed +- Moved mapper and providers to separate packages for easier management + ### Added - Add more unit tests +- Add support for retagging s3 buckets +- Add an interface and a Mocking base class for the mapper for easier unit + testing ## [0.1.0] - 2017-11-22 diff --git a/Makefile b/Makefile index 81967ef..fbc3bf5 100644 --- a/Makefile +++ b/Makefile @@ -26,9 +26,13 @@ lint: fmt gocov: @go get github.com/axw/gocov/gocov \ - && go install github.com/axw/gocov/gocov - @gocov test | gocov report - # gocov test >/tmp/gocovtest.json ; gocov annotate /tmp/gocovtest.json MyFunc + && go install github.com/axw/gocov/gocov; \ + if [ -f "glide.yaml" ] ; then \ + gocov test $$(glide novendor) | gocov report; \ + else \ + gocov test ./... | gocov report; \ + fi + # gocov test $$(glide novendor) >/tmp/gocovtest.json ; gocov annotate /tmp/gocovtest.json MyFunc test: @if [ -f "glide.yaml" ] ; then \ diff --git a/README.md b/README.md index 58cd244..cc4bd86 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ The key elements of a resource are the following attribute (first come in the gi * a RDS Instance: `DBClusterIdentifier`, `DBInstanceIdentifier`, `DBName`, `MasterUsername` * a RDS Cluster: `DBClusterIdentifier`, `DatabaseName`, `MasterUsername` * a Redshift Cluster: `ClusterIdentifier`, `DBName`, `MasterUsername` +* an S3 Bucket: `BucketName` With the following configuration, an instance with a SSH KeyName set to `apple-tv-analytics-prod`, you'll end up with: @@ -240,6 +241,8 @@ Usage of ./awsRetagger: Enables the re-tagging of the RDS instances. Environment variable: RDS_INSTANCES -redshift-clusters Enables the re-tagging of the Redshift clusters. Environment variable: REDSHIFT_CLUSTERS + -s3-buckets + Enables the re-tagging of the S3 buckets. Environment variable: S3_BUCKETS ``` ### Use inside Docker @@ -261,3 +264,4 @@ you might check using the `-h` option of the command-line): * RDS Instances * RDS Clusters * Redshift Clusters +* S3 Buckets diff --git a/TODO b/TODO.md similarity index 89% rename from TODO rename to TODO.md index 53d36cb..1ef65d5 100644 --- a/TODO +++ b/TODO.md @@ -1,3 +1,6 @@ +# TODO list + +Few ideas that might be worth exploring: * Add tests * ec2.go add support for retagging: * ELB @@ -16,7 +19,6 @@ * DataPipeline * Elasticache * ElasticBeanstalk - * S3 * Lambda * Route53 * Simpledb diff --git a/glide.lock b/glide.lock new file mode 100644 index 0000000..872b32f --- /dev/null +++ b/glide.lock @@ -0,0 +1,68 @@ +hash: 8e172b940f245b2f0f427bb6b1cb210091b83e69dfa427426627604ef041e31d +updated: 2017-12-14T20:32:48.066332629-08:00 +imports: +- name: github.com/aws/aws-sdk-go + version: ed448bf2aa437d03a95b925a17f7ed5380353559 + subpackages: + - aws + - aws/arn + - aws/awserr + - aws/awsutil + - aws/client + - aws/client/metadata + - aws/corehandlers + - aws/credentials + - aws/credentials/ec2rolecreds + - aws/credentials/endpointcreds + - aws/credentials/stscreds + - aws/defaults + - aws/ec2metadata + - aws/endpoints + - aws/request + - aws/session + - aws/signer/v4 + - internal/shareddefaults + - private/protocol + - private/protocol/ec2query + - private/protocol/json/jsonutil + - private/protocol/jsonrpc + - private/protocol/query + - private/protocol/query/queryutil + - private/protocol/rest + - private/protocol/restjson + - private/protocol/restxml + - private/protocol/xml/xmlutil + - service/cloudfront + - service/cloudfront/cloudfrontiface + - service/cloudwatchlogs + - service/ec2 + - service/elasticbeanstalk + - service/elasticbeanstalk/elasticbeanstalkiface + - service/elasticsearchservice + - service/rds + - service/rds/rdsiface + - service/redshift + - service/redshift/redshiftiface + - service/s3 + - service/s3/s3iface + - service/sts +- name: github.com/go-ini/ini + version: 5b3e00af70a9484542169a976dcab8d03e601a17 +- name: github.com/gobike/envflag + version: ae3268980a293939fc31d786fe86f1453333d386 +- name: github.com/jmespath/go-jmespath + version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d +- name: github.com/sirupsen/logrus + version: d682213848ed68c0a260ca37d6dd5ace8423f5ba + subpackages: + - hooks/test +- name: golang.org/x/crypto + version: bd6f299fb381e4c3393d1c4b1f0b94f5e77650c8 + subpackages: + - ssh/terminal +- name: golang.org/x/sys + version: 95c6576299259db960f6c5b9b69ea52422860fce + subpackages: + - unix + - windows +testImports: [] diff --git a/glide.yaml b/glide.yaml index 0a09da8..799971f 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,10 +1,11 @@ package: github.com/VEVO/awsRetagger import: - package: github.com/aws/aws-sdk-go - version: ^1.12.32 + version: ^1.12.47 subpackages: - aws - aws/arn + - aws/awserr - aws/session - service/cloudfront - service/cloudfront/cloudfrontiface @@ -14,9 +15,12 @@ import: - service/elasticbeanstalk/elasticbeanstalkiface - service/elasticsearchservice - service/rds + - service/rds/rdsiface - service/redshift - service/redshift/redshiftiface + - service/s3 + - service/s3/s3iface - service/sts - package: github.com/gobike/envflag - package: github.com/sirupsen/logrus - version: ^1.0.3 + version: ^1.0.4 diff --git a/main.go b/main.go index e047eab..6df02e4 100644 --- a/main.go +++ b/main.go @@ -3,11 +3,15 @@ package main import ( "flag" "fmt" + "io" + "os" + "github.com/aws/aws-sdk-go/aws/session" "github.com/gobike/envflag" "github.com/sirupsen/logrus" - "io" - "os" + + "github.com/VEVO/awsRetagger/mapper" + "github.com/VEVO/awsRetagger/providers" ) var log *logrus.Entry @@ -38,9 +42,9 @@ func NewLogger(logLevel, format string, output io.Writer) (*logrus.Entry, error) func main() { var ( - configFilePath, logLevel, logFormat string - processEc2Instances, processRdsInstances, processRdsClusters, processCloudwatchLogGroups, processElasticSearch, processCloudFrontDist, processRedshiftClusters, processElasticBeanstalkEnv bool - err error + configFilePath, logLevel, logFormat string + processEc2Instances, processRdsInstances, processRdsClusters, processCloudwatchLogGroups, processElasticSearch, processCloudFrontDist, processRedshiftClusters, processElasticBeanstalkEnv, processS3Buckets bool + err error ) flag.StringVar(&configFilePath, "config", "config.json", "Path of the json configuration file. Environment variable: CONFIG") flag.StringVar(&logLevel, "log-level", "info", "Log level. Accepted values: debug, info, warn, error, fatal, panic. Environment variable: LOG_LEVEL") @@ -53,12 +57,15 @@ func main() { flag.BoolVar(&processCloudFrontDist, "cloudfront-distributions", false, "Enables the re-tagging of the CloudFront distributions. Environment variable: CLOUDFRONT_DISTRIBUTIONS") flag.BoolVar(&processRedshiftClusters, "redshift-clusters", false, "Enables the re-tagging of the Redshift clusters. Environment variable: REDSHIFT_CLUSTERS") flag.BoolVar(&processElasticBeanstalkEnv, "elasticbeanstalk-environments", false, "Enables the re-tagging of the ElasticBeanstalk environments. Environment variable: ELASTICBEANSTALK_ENVIRONMENTS") + flag.BoolVar(&processS3Buckets, "s3-buckets", false, "Enables the re-tagging of the S3 buckets. Environment variable: S3_BUCKETS") envflag.Parse() if log, err = NewLogger(logLevel, logFormat, os.Stdout); err != nil { fmt.Printf("Error while setting up the logger: %s\n", err) os.Exit(1) } + mapper.SetLogger(log) + providers.SetLogger(log) // Load config cfg, err := os.Open(configFilePath) defer cfg.Close() @@ -66,7 +73,7 @@ func main() { log.WithFields(logrus.Fields{"error": err}).Fatal("Unable to read config file") } - m := Mapper{} + m := mapper.Mapper{} if err = m.LoadConfig(cfg); err != nil { log.WithFields(logrus.Fields{"error": err}).Fatal("Unable to load config file") } @@ -76,12 +83,12 @@ func main() { })) if processEc2Instances { - e := NewEc2Processor(sess) + e := providers.NewEc2Processor(sess) e.RetagInstances(&m) } if processRdsInstances || processRdsClusters { - r := NewRdsProcessor(sess) + r := providers.NewRdsProcessor(sess) if processRdsInstances { r.RetagInstances(&m) } @@ -91,28 +98,33 @@ func main() { } if processCloudwatchLogGroups { - c := NewCwProcessor(sess) + c := providers.NewCwProcessor(sess) c.RetagLogGroups(&m) } if processElasticSearch { - elk := NewElkProcessor(sess) + elk := providers.NewElkProcessor(sess) elk.RetagDomains(&m) } if processCloudFrontDist { - cf := NewCloudFrontProcessor(sess) + cf := providers.NewCloudFrontProcessor(sess) cf.RetagDistributions(&m) } if processRedshiftClusters { - rs, err := NewRedshiftProcessor(sess) + rs, err := providers.NewRedshiftProcessor(sess) if err != nil { log.WithFields(logrus.Fields{"error": err}).Fatal("Unable to initialize the Redshift client") } rs.RetagClusters(&m) } if processElasticBeanstalkEnv { - eb := NewElasticBeanstalkProcessor(sess) + eb := providers.NewElasticBeanstalkProcessor(sess) eb.RetagEnvironments(&m) } + + if processS3Buckets { + sp := providers.NewS3Processor(sess) + sp.RetagBuckets(&m) + } } diff --git a/errors.go b/mapper/errors.go similarity index 98% rename from errors.go rename to mapper/errors.go index 154ac4b..d79a94e 100644 --- a/errors.go +++ b/mapper/errors.go @@ -1,4 +1,4 @@ -package main +package mapper // ErrSanityNoMapping is returned for the sanity checks in the Mapper type ErrSanityNoMapping struct { diff --git a/mapper/interface.go b/mapper/interface.go new file mode 100644 index 0000000..0445e11 --- /dev/null +++ b/mapper/interface.go @@ -0,0 +1,24 @@ +package mapper + +import ( + "io" +) + +// PutTagFn is used to specify the function structure to pass to the Retag method +type PutTagFn func(*string, []*TagItem) error + +// Iface has been created for testing purposes. It allows to create mocks +// when testing class that depend on the mapper +type Iface interface { + LoadConfig(io.Reader) error + StripDefaults(*map[string]string) + GetMissingDefaults(*map[string]string) *map[string]string + GetFromTags(*map[string]string) (*map[string]string, error) + GetFromKey(string, *map[string]string) (*map[string]string, error) + ValidateTag(string, string) (*TagItem, error) + MergeMaps(*map[string]string, *map[string]string) + + Retag(*string, *map[string]string, []string, PutTagFn) +} + +var _ Iface = (*Mapper)(nil) diff --git a/mapper.go b/mapper/mapper.go similarity index 98% rename from mapper.go rename to mapper/mapper.go index 80e8b87..62c2aab 100644 --- a/mapper.go +++ b/mapper/mapper.go @@ -1,12 +1,18 @@ -package main +package mapper import ( "encoding/json" - "github.com/sirupsen/logrus" "io" "regexp" + + "github.com/sirupsen/logrus" ) +var log *logrus.Entry + +// SetLogger is used to pass the loger from the main program +func SetLogger(logger *logrus.Entry) { log = logger } + // TagItem is a standard AWS tag structure type TagItem struct { Name string `json:"name"` @@ -48,6 +54,7 @@ type TagSanity struct { // Mapper contains the different mappings between attributes and the list of // tags that should be present on that resource type Mapper struct { + Iface CopyTag []*TagCopy `json:"copy_tags,omitempty"` TagMap []*TagMapper `json:"tags,omitempty"` KeyMap []*KeyMapper `json:"keys,omitempty"` @@ -242,10 +249,8 @@ func (m *Mapper) MergeMaps(mainMap, complementary *map[string]string) { } } -type putTagFn func(*string, []*TagItem) error - // Retag does the different re-tagging operations and calls the given setTags function -func (m *Mapper) Retag(resourceID *string, tags *map[string]string, keys []string, setTags putTagFn) { +func (m *Mapper) Retag(resourceID *string, tags *map[string]string, keys []string, setTags PutTagFn) { var ( newTags, mapFromKey, mapFromMissing *map[string]string err error diff --git a/mapper_test.go b/mapper/mapper_test.go similarity index 99% rename from mapper_test.go rename to mapper/mapper_test.go index c283231..eb452e4 100644 --- a/mapper_test.go +++ b/mapper/mapper_test.go @@ -1,13 +1,14 @@ -package main +package mapper import ( "errors" - "github.com/sirupsen/logrus" - logrus_test "github.com/sirupsen/logrus/hooks/test" "reflect" "regexp/syntax" "strings" "testing" + + "github.com/sirupsen/logrus" + logrus_test "github.com/sirupsen/logrus/hooks/test" ) func TestLoadConfig(t *testing.T) { @@ -343,7 +344,7 @@ func TestRetag(t *testing.T) { resourceID string tags map[string]string keys []string - setTags putTagFn + setTags PutTagFn logEntries int expected map[string]string config Mapper diff --git a/mapper/mock.go b/mapper/mock.go new file mode 100644 index 0000000..b198ba6 --- /dev/null +++ b/mapper/mock.go @@ -0,0 +1,36 @@ +package mapper + +// MockMapper is used to mock the calls to retag during the tests +type MockMapper struct { + Iface + // ResourceTags is used to record which tags have been pushed to the Retag + // function on which resource since the creation of the object + ResourceTags map[string]map[string]string + // ResourceKeys is used to record which keys have been pushed to the + // Retag function on which resource since the creation of the object + ResourceKeys map[string][]string +} + +// Retag just records which resource has been called with which tags +func (m *MockMapper) Retag(resourceID *string, tags *map[string]string, keys []string, setTags PutTagFn) { + if m.ResourceTags == nil { + m.ResourceTags = make(map[string]map[string]string) + } + if m.ResourceKeys == nil { + m.ResourceKeys = make(map[string][]string) + } + if val, ok := m.ResourceTags[*resourceID]; ok { + for k, v := range *tags { + if _, exists := val[k]; !exists { + m.ResourceTags[*resourceID][k] = v + } + } + } else { + m.ResourceTags[*resourceID] = *tags + } + if val, ok := m.ResourceKeys[*resourceID]; ok { + m.ResourceKeys[*resourceID] = append(val, keys...) + } else { + m.ResourceKeys[*resourceID] = keys + } +} diff --git a/mapper/mock_test.go b/mapper/mock_test.go new file mode 100644 index 0000000..bc643d1 --- /dev/null +++ b/mapper/mock_test.go @@ -0,0 +1,41 @@ +package mapper + +import ( + "reflect" + "testing" +) + +func dummyMockMapperFct(res *string, tag []*TagItem) error { + return nil +} + +func TestMockRetag(t *testing.T) { + testData := []struct { + inputResourceTags, outputResourceTags map[string]map[string]string + inputResourceKeys, outputResourceKeys map[string][]string + }{ + {map[string]map[string]string{}, nil, map[string][]string{}, nil}, + { + map[string]map[string]string{"foo": {"key1": "value1"}, "bar": {"key2": "value2", "key3": "value3"}, "joe": nil}, + map[string]map[string]string{"foo": {"key1": "value1"}, "bar": {"key2": "value2", "key3": "value3"}, "joe": nil}, + map[string][]string{"bar": {"mykey1", "mykey2"}, "joe": {"mykey3"}}, + map[string][]string{"foo": nil, "bar": {"mykey1", "mykey2"}, "joe": {"mykey3"}}, + }, + } + + for _, d := range testData { + m := MockMapper{} + for k, v := range d.inputResourceTags { + inKeys := []string{} + inKeys, _ = d.inputResourceKeys[k] + m.Retag(&k, &v, inKeys, nil) + } + if !reflect.DeepEqual(d.outputResourceTags, m.ResourceTags) { + t.Errorf("Expecting ResourceTags: %v\nGot: %v\n", d.outputResourceTags, m.ResourceTags) + } + + if !reflect.DeepEqual(d.outputResourceKeys, m.ResourceKeys) { + t.Errorf("Expecting ResourceKeys: %v\nGot: %v\n", d.outputResourceKeys, m.ResourceKeys) + } + } +} diff --git a/cloudfront.go b/providers/cloudfront.go similarity index 95% rename from cloudfront.go rename to providers/cloudfront.go index 23c8546..cad1f77 100644 --- a/cloudfront.go +++ b/providers/cloudfront.go @@ -1,4 +1,4 @@ -package main +package providers import ( "github.com/aws/aws-sdk-go/aws" @@ -6,6 +6,8 @@ import ( "github.com/aws/aws-sdk-go/service/cloudfront" "github.com/aws/aws-sdk-go/service/cloudfront/cloudfrontiface" "github.com/sirupsen/logrus" + + "github.com/VEVO/awsRetagger/mapper" ) // CloudFrontProcessor holds the cloudfront-related actions @@ -30,7 +32,7 @@ func (p *CloudFrontProcessor) TagsToMap(tagsInput []*cloudfront.Tag) map[string] } // SetTags sets tags on an cloudfront resource -func (p *CloudFrontProcessor) SetTags(resourceID *string, tags []*TagItem) error { +func (p *CloudFrontProcessor) SetTags(resourceID *string, tags []*mapper.TagItem) error { newTags := []*cloudfront.Tag{} for _, tag := range tags { newTags = append(newTags, &cloudfront.Tag{Key: aws.String((*tag).Name), Value: aws.String((*tag).Value)}) @@ -59,7 +61,7 @@ func (p *CloudFrontProcessor) GetTags(resourceID *string) ([]*cloudfront.Tag, er } // RetagDistributions parses all distributions and retags them -func (p *CloudFrontProcessor) RetagDistributions(m *Mapper) { +func (p *CloudFrontProcessor) RetagDistributions(m *mapper.Mapper) { err := p.svc.ListDistributionsPages(&cloudfront.ListDistributionsInput{}, func(page *cloudfront.ListDistributionsOutput, lastPage bool) bool { if page.DistributionList != nil { diff --git a/cloudfront_test.go b/providers/cloudfront_test.go similarity index 82% rename from cloudfront_test.go rename to providers/cloudfront_test.go index a696ff2..2d0cf76 100644 --- a/cloudfront_test.go +++ b/providers/cloudfront_test.go @@ -1,4 +1,4 @@ -package main +package providers import ( "errors" @@ -8,6 +8,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudfront" "github.com/aws/aws-sdk-go/service/cloudfront/cloudfrontiface" + + "github.com/VEVO/awsRetagger/mapper" ) type mockCloudFrontClient struct { @@ -37,14 +39,14 @@ func (m *mockCloudFrontClient) ListTagsForResource(input *cloudfront.ListTagsFor func TestCloudFrontSetTags(t *testing.T) { testData := []struct { inputResource, outputResource string - inputTag []*TagItem + inputTag []*mapper.TagItem outputTag *cloudfront.Tags inputError, outputError error }{ - {"my resource", "my resource", []*TagItem{{}}, &cloudfront.Tags{Items: []*cloudfront.Tag{{Key: aws.String(""), Value: aws.String("")}}}, nil, nil}, - {"my resource", "my resource", []*TagItem{{Name: "foo", Value: "bar"}}, &cloudfront.Tags{Items: []*cloudfront.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}}, nil, nil}, - {"my resource", "my resource", []*TagItem{{Name: "foo", Value: "bar"}, {Name: "Aerosmith", Value: "rocks"}}, &cloudfront.Tags{Items: []*cloudfront.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}, {Key: aws.String("Aerosmith"), Value: aws.String("rocks")}}}, nil, nil}, - {"my resource", "my resource", []*TagItem{{Name: "foo", Value: "bar"}}, &cloudfront.Tags{Items: []*cloudfront.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}}, errors.New("Badaboom"), errors.New("Badaboom")}, + {"my resource", "my resource", []*mapper.TagItem{{}}, &cloudfront.Tags{Items: []*cloudfront.Tag{{Key: aws.String(""), Value: aws.String("")}}}, nil, nil}, + {"my resource", "my resource", []*mapper.TagItem{{Name: "foo", Value: "bar"}}, &cloudfront.Tags{Items: []*cloudfront.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}}, nil, nil}, + {"my resource", "my resource", []*mapper.TagItem{{Name: "foo", Value: "bar"}, {Name: "Aerosmith", Value: "rocks"}}, &cloudfront.Tags{Items: []*cloudfront.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}, {Key: aws.String("Aerosmith"), Value: aws.String("rocks")}}}, nil, nil}, + {"my resource", "my resource", []*mapper.TagItem{{Name: "foo", Value: "bar"}}, &cloudfront.Tags{Items: []*cloudfront.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}}, errors.New("Badaboom"), errors.New("Badaboom")}, } for _, d := range testData { mockSvc := &mockCloudFrontClient{ReturnError: d.inputError, ResourceTags: &cloudfront.Tags{}} diff --git a/cloudwatch.go b/providers/cloudwatch.go similarity index 91% rename from cloudwatch.go rename to providers/cloudwatch.go index 94f28fe..ab1880f 100644 --- a/cloudwatch.go +++ b/providers/cloudwatch.go @@ -1,10 +1,12 @@ -package main +package providers import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" "github.com/sirupsen/logrus" + + "github.com/VEVO/awsRetagger/mapper" ) // CwProcessor holds the cloudwatch-related actions @@ -29,7 +31,7 @@ func (p *CwProcessor) TagsToMap(tagsInput map[string]*string) map[string]string } // SetTags sets tags on an cloudwatchLog resource -func (p *CwProcessor) SetTags(resourceID *string, tags []*TagItem) error { +func (p *CwProcessor) SetTags(resourceID *string, tags []*mapper.TagItem) error { newTags := map[string]*string{} for _, tag := range tags { newTags[(*tag).Name] = aws.String((*tag).Value) @@ -54,7 +56,7 @@ func (p *CwProcessor) GetTags(resourceID *string) (map[string]*string, error) { } // RetagLogGroups parses all running and stopped instances and retags them -func (p *CwProcessor) RetagLogGroups(m *Mapper) { +func (p *CwProcessor) RetagLogGroups(m *mapper.Mapper) { err := p.svc.DescribeLogGroupsPages(&cloudwatchlogs.DescribeLogGroupsInput{}, func(page *cloudwatchlogs.DescribeLogGroupsOutput, lastPage bool) bool { for _, lg := range page.LogGroups { diff --git a/cloudwatch_test.go b/providers/cloudwatch_test.go similarity index 97% rename from cloudwatch_test.go rename to providers/cloudwatch_test.go index a0c49e8..351b0da 100644 --- a/cloudwatch_test.go +++ b/providers/cloudwatch_test.go @@ -1,4 +1,4 @@ -package main +package providers import ( "reflect" diff --git a/ec2.go b/providers/ec2.go similarity index 90% rename from ec2.go rename to providers/ec2.go index beba37f..4844df9 100644 --- a/ec2.go +++ b/providers/ec2.go @@ -1,10 +1,12 @@ -package main +package providers import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/sirupsen/logrus" + + "github.com/VEVO/awsRetagger/mapper" ) // Ec2Processor holds the ec2-related actions @@ -29,7 +31,7 @@ func (e *Ec2Processor) TagsToMap(tagsInput []*ec2.Tag) map[string]string { } // SetTags sets tags on an ec2 resource -func (e *Ec2Processor) SetTags(resourceID *string, tags []*TagItem) error { +func (e *Ec2Processor) SetTags(resourceID *string, tags []*mapper.TagItem) error { newTags := []*ec2.Tag{} for _, tag := range tags { newTags = append(newTags, &ec2.Tag{Key: aws.String((*tag).Name), Value: aws.String((*tag).Value)}) @@ -43,7 +45,7 @@ func (e *Ec2Processor) SetTags(resourceID *string, tags []*TagItem) error { } // RetagInstances parses all running and stopped instances and retags them -func (e *Ec2Processor) RetagInstances(m *Mapper) { +func (e *Ec2Processor) RetagInstances(m *mapper.Mapper) { filters := []*ec2.Filter{ { Name: aws.String("instance-state-name"), diff --git a/ec2_test.go b/providers/ec2_test.go similarity index 97% rename from ec2_test.go rename to providers/ec2_test.go index aa9393e..151f6d8 100644 --- a/ec2_test.go +++ b/providers/ec2_test.go @@ -1,4 +1,4 @@ -package main +package providers import ( "reflect" diff --git a/elasticbeanstalk.go b/providers/elasticbeanstalk.go similarity index 94% rename from elasticbeanstalk.go rename to providers/elasticbeanstalk.go index ad59128..4cb4664 100644 --- a/elasticbeanstalk.go +++ b/providers/elasticbeanstalk.go @@ -1,4 +1,4 @@ -package main +package providers import ( "github.com/aws/aws-sdk-go/aws" @@ -6,6 +6,8 @@ import ( "github.com/aws/aws-sdk-go/service/elasticbeanstalk" "github.com/aws/aws-sdk-go/service/elasticbeanstalk/elasticbeanstalkiface" "github.com/sirupsen/logrus" + + "github.com/VEVO/awsRetagger/mapper" ) // ElasticBeanstalkProcessor holds the elasticbeanstalk-related actions @@ -30,7 +32,7 @@ func (p *ElasticBeanstalkProcessor) TagsToMap(tagsInput []*elasticbeanstalk.Tag) } // SetTags sets a group of tag on an elasticbeanstalk resource -func (p *ElasticBeanstalkProcessor) SetTags(resourceID *string, tags []*TagItem) error { +func (p *ElasticBeanstalkProcessor) SetTags(resourceID *string, tags []*mapper.TagItem) error { newTags := []*elasticbeanstalk.Tag{} for _, tag := range tags { if (*tag).Name != "elasticbeanstalk:environment-name" && (*tag).Name != "elasticbeanstalk:environment-id" { @@ -59,7 +61,7 @@ func (p *ElasticBeanstalkProcessor) GetTags(resourceID *string) ([]*elasticbeans } // RetagEnvironments parses all environments and retags them -func (p *ElasticBeanstalkProcessor) RetagEnvironments(m *Mapper) { +func (p *ElasticBeanstalkProcessor) RetagEnvironments(m *mapper.Mapper) { envs, err := p.svc.DescribeEnvironments(&elasticbeanstalk.DescribeEnvironmentsInput{IncludeDeleted: aws.Bool(false)}) if err != nil { log.WithFields(logrus.Fields{"error": err}).Fatal("DescribeEnvironments failed") diff --git a/elasticbeanstalk_test.go b/providers/elasticbeanstalk_test.go similarity index 97% rename from elasticbeanstalk_test.go rename to providers/elasticbeanstalk_test.go index b41f739..ba32bd3 100644 --- a/elasticbeanstalk_test.go +++ b/providers/elasticbeanstalk_test.go @@ -1,10 +1,11 @@ -package main +package providers import ( - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/elasticbeanstalk" "reflect" "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/elasticbeanstalk" ) func TestElasticBeanstalkTagsToMap(t *testing.T) { diff --git a/elasticsearch.go b/providers/elasticsearch.go similarity index 93% rename from elasticsearch.go rename to providers/elasticsearch.go index f0bb01a..4601079 100644 --- a/elasticsearch.go +++ b/providers/elasticsearch.go @@ -1,10 +1,12 @@ -package main +package providers import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/elasticsearchservice" "github.com/sirupsen/logrus" + + "github.com/VEVO/awsRetagger/mapper" ) // ElkProcessor holds the elasticsearch-related actions @@ -29,7 +31,7 @@ func (p *ElkProcessor) TagsToMap(tagsInput []*elasticsearchservice.Tag) map[stri } // SetTags sets tags on an elasticsearchservice resource -func (p *ElkProcessor) SetTags(resourceID *string, tags []*TagItem) error { +func (p *ElkProcessor) SetTags(resourceID *string, tags []*mapper.TagItem) error { newTags := []*elasticsearchservice.Tag{} for _, tag := range tags { newTags = append(newTags, &elasticsearchservice.Tag{Key: aws.String((*tag).Name), Value: aws.String((*tag).Value)}) @@ -54,7 +56,7 @@ func (p *ElkProcessor) GetTags(resourceID *string) ([]*elasticsearchservice.Tag, } // RetagDomains parses all elasticsearch domains and retags them -func (p *ElkProcessor) RetagDomains(m *Mapper) { +func (p *ElkProcessor) RetagDomains(m *mapper.Mapper) { result, err := p.svc.ListDomainNames(&elasticsearchservice.ListDomainNamesInput{}) if err != nil { log.WithFields(logrus.Fields{"error": err}).Fatal("ListDomainNames failed") diff --git a/elasticsearch_test.go b/providers/elasticsearch_test.go similarity index 97% rename from elasticsearch_test.go rename to providers/elasticsearch_test.go index a8b831f..b065770 100644 --- a/elasticsearch_test.go +++ b/providers/elasticsearch_test.go @@ -1,4 +1,4 @@ -package main +package providers import ( "reflect" diff --git a/providers/logger.go b/providers/logger.go new file mode 100644 index 0000000..855e993 --- /dev/null +++ b/providers/logger.go @@ -0,0 +1,10 @@ +package providers + +import ( + "github.com/sirupsen/logrus" +) + +var log *logrus.Entry + +// SetLogger is used to pass the loger from the main program +func SetLogger(logger *logrus.Entry) { log = logger } diff --git a/rds.go b/providers/rds.go similarity index 92% rename from rds.go rename to providers/rds.go index 260dc5f..79c9d79 100644 --- a/rds.go +++ b/providers/rds.go @@ -1,4 +1,4 @@ -package main +package providers import ( "github.com/aws/aws-sdk-go/aws" @@ -6,6 +6,8 @@ import ( "github.com/aws/aws-sdk-go/service/rds" "github.com/aws/aws-sdk-go/service/rds/rdsiface" "github.com/sirupsen/logrus" + + "github.com/VEVO/awsRetagger/mapper" ) // RdsProcessor holds the rds-related actions @@ -30,7 +32,7 @@ func (p *RdsProcessor) TagsToMap(tagsInput []*rds.Tag) map[string]string { } // SetTags sets tags on an rds resource -func (p *RdsProcessor) SetTags(resourceID *string, tags []*TagItem) error { +func (p *RdsProcessor) SetTags(resourceID *string, tags []*mapper.TagItem) error { newTags := []*rds.Tag{} for _, tag := range tags { newTags = append(newTags, &rds.Tag{Key: aws.String((*tag).Name), Value: aws.String((*tag).Value)}) @@ -55,7 +57,7 @@ func (p *RdsProcessor) GetTags(resourceID *string) ([]*rds.Tag, error) { } // RetagInstances parses all instances and retags them -func (p *RdsProcessor) RetagInstances(m *Mapper) { +func (p *RdsProcessor) RetagInstances(m *mapper.Mapper) { result, err := p.svc.DescribeDBInstances(&rds.DescribeDBInstancesInput{}) if err != nil { log.WithFields(logrus.Fields{"error": err}).Fatalf("DescribeDBInstances failed") @@ -86,7 +88,7 @@ func (p *RdsProcessor) RetagInstances(m *Mapper) { } // RetagClusters parses all clusters and retags them -func (p *RdsProcessor) RetagClusters(m *Mapper) { +func (p *RdsProcessor) RetagClusters(m *mapper.Mapper) { result, err := p.svc.DescribeDBClusters(&rds.DescribeDBClustersInput{}) if err != nil { log.WithFields(logrus.Fields{"error": err}).Fatalf("DescribeDBClusters failed") diff --git a/rds_test.go b/providers/rds_test.go similarity index 83% rename from rds_test.go rename to providers/rds_test.go index 2316659..fd31f5c 100644 --- a/rds_test.go +++ b/providers/rds_test.go @@ -1,4 +1,4 @@ -package main +package providers import ( "errors" @@ -8,6 +8,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/rds" "github.com/aws/aws-sdk-go/service/rds/rdsiface" + + "github.com/VEVO/awsRetagger/mapper" ) type mockRdsClient struct { @@ -56,14 +58,14 @@ func TestRdsTagsToMap(t *testing.T) { func TestRdsSetTags(t *testing.T) { testData := []struct { inputResource, outputResource string - inputTags []*TagItem + inputTags []*mapper.TagItem outputTags []*rds.Tag inputError, outputError error }{ - {"my resource", "my resource", []*TagItem{{}}, []*rds.Tag{{Key: aws.String(""), Value: aws.String("")}}, nil, nil}, - {"my resource", "my resource", []*TagItem{{Name: "foo", Value: "bar"}}, []*rds.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}, nil, nil}, - {"my resource", "my resource", []*TagItem{{Name: "foo", Value: "bar"}, {Name: "Aerosmith", Value: "rocks"}}, []*rds.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}, {Key: aws.String("Aerosmith"), Value: aws.String("rocks")}}, nil, nil}, - {"my resource", "my resource", []*TagItem{{Name: "foo", Value: "bar"}}, []*rds.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}, errors.New("Badaboom"), errors.New("Badaboom")}, + {"my resource", "my resource", []*mapper.TagItem{{}}, []*rds.Tag{{Key: aws.String(""), Value: aws.String("")}}, nil, nil}, + {"my resource", "my resource", []*mapper.TagItem{{Name: "foo", Value: "bar"}}, []*rds.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}, nil, nil}, + {"my resource", "my resource", []*mapper.TagItem{{Name: "foo", Value: "bar"}, {Name: "Aerosmith", Value: "rocks"}}, []*rds.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}, {Key: aws.String("Aerosmith"), Value: aws.String("rocks")}}, nil, nil}, + {"my resource", "my resource", []*mapper.TagItem{{Name: "foo", Value: "bar"}}, []*rds.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}, errors.New("Badaboom"), errors.New("Badaboom")}, } for _, d := range testData { mockSvc := &mockRdsClient{ReturnError: d.inputError, ResourceTags: []*rds.Tag{}} diff --git a/redshift.go b/providers/redshift.go similarity index 95% rename from redshift.go rename to providers/redshift.go index d285692..ea1a5f3 100644 --- a/redshift.go +++ b/providers/redshift.go @@ -1,4 +1,4 @@ -package main +package providers import ( "github.com/aws/aws-sdk-go/aws" @@ -8,6 +8,8 @@ import ( "github.com/aws/aws-sdk-go/service/redshift/redshiftiface" "github.com/aws/aws-sdk-go/service/sts" "github.com/sirupsen/logrus" + + "github.com/VEVO/awsRetagger/mapper" ) // RedshiftProcessor holds the redshift-related actions @@ -40,7 +42,7 @@ func (p *RedshiftProcessor) TagsToMap(tagsInput []*redshift.TaggedResource) map[ } // SetTags sets tags on an redshift resource -func (p *RedshiftProcessor) SetTags(resourceID *string, tags []*TagItem) error { +func (p *RedshiftProcessor) SetTags(resourceID *string, tags []*mapper.TagItem) error { newTags := []*redshift.Tag{} for _, tag := range tags { newTags = append(newTags, &redshift.Tag{Key: aws.String((*tag).Name), Value: aws.String((*tag).Value)}) @@ -72,7 +74,7 @@ func (p *RedshiftProcessor) getArn(resourceType, resourceIdentifier string) stri } // RetagClusters parses all clusters and retags them -func (p *RedshiftProcessor) RetagClusters(m *Mapper) { +func (p *RedshiftProcessor) RetagClusters(m *mapper.Mapper) { err := p.svc.DescribeClustersPages(&redshift.DescribeClustersInput{}, func(page *redshift.DescribeClustersOutput, lastPage bool) bool { for _, elt := range page.Clusters { diff --git a/redshift_test.go b/providers/redshift_test.go similarity index 77% rename from redshift_test.go rename to providers/redshift_test.go index 1a13ee9..291eeb7 100644 --- a/redshift_test.go +++ b/providers/redshift_test.go @@ -1,12 +1,15 @@ -package main +package providers import ( "errors" + "reflect" + "testing" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/redshift" "github.com/aws/aws-sdk-go/service/redshift/redshiftiface" - "reflect" - "testing" + + "github.com/VEVO/awsRetagger/mapper" ) type mockRedshiftClient struct { @@ -38,14 +41,14 @@ func (m *mockRedshiftClient) DescribeTags(input *redshift.DescribeTagsInput) (*r func TestRedshiftSetTags(t *testing.T) { testData := []struct { inputResource, outputResource string - inputTags []*TagItem + inputTags []*mapper.TagItem outputTags []*redshift.Tag inputError, outputError error }{ - {"my resource", "my resource", []*TagItem{{}}, []*redshift.Tag{{Key: aws.String(""), Value: aws.String("")}}, nil, nil}, - {"my resource", "my resource", []*TagItem{{Name: "foo", Value: "bar"}}, []*redshift.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}, nil, nil}, - {"my resource", "my resource", []*TagItem{{Name: "foo", Value: "bar"}, {Name: "Aerosmith", Value: "rocks"}}, []*redshift.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}, {Key: aws.String("Aerosmith"), Value: aws.String("rocks")}}, nil, nil}, - {"my resource", "my resource", []*TagItem{{Name: "foo", Value: "bar"}}, []*redshift.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}, errors.New("Badaboom"), errors.New("Badaboom")}, + {"my resource", "my resource", []*mapper.TagItem{{}}, []*redshift.Tag{{Key: aws.String(""), Value: aws.String("")}}, nil, nil}, + {"my resource", "my resource", []*mapper.TagItem{{Name: "foo", Value: "bar"}}, []*redshift.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}, nil, nil}, + {"my resource", "my resource", []*mapper.TagItem{{Name: "foo", Value: "bar"}, {Name: "Aerosmith", Value: "rocks"}}, []*redshift.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}, {Key: aws.String("Aerosmith"), Value: aws.String("rocks")}}, nil, nil}, + {"my resource", "my resource", []*mapper.TagItem{{Name: "foo", Value: "bar"}}, []*redshift.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}, errors.New("Badaboom"), errors.New("Badaboom")}, } for _, d := range testData { mockSvc := &mockRedshiftClient{ReturnError: d.inputError, ResourceTags: []*redshift.Tag{}} diff --git a/providers/s3.go b/providers/s3.go new file mode 100644 index 0000000..fc0a1a1 --- /dev/null +++ b/providers/s3.go @@ -0,0 +1,96 @@ +package providers + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" + "github.com/sirupsen/logrus" + + "github.com/VEVO/awsRetagger/mapper" +) + +// S3Processor holds the s3-related actions +type S3Processor struct { + svc s3iface.S3API + region *string +} + +// NewS3Processor creates a new instance of S3Processor containing an already +// initialized s3 client +func NewS3Processor(sess *session.Session) *S3Processor { + return &S3Processor{svc: s3.New(sess), region: sess.Config.Region} +} + +// TagsToMap transform the s3 tags structure into a map[string]string for +// easier manipulations +func (e *S3Processor) TagsToMap(tagsInput []*s3.Tag) map[string]string { + tagsHash := make(map[string]string) + for _, tag := range tagsInput { + tagsHash[*tag.Key] = *tag.Value + } + return tagsHash +} + +// SetTags sets tags on a s3 bucket +func (e *S3Processor) SetTags(resourceID *string, tags []*mapper.TagItem) error { + newTags := s3.Tagging{} + for _, tag := range tags { + if len((*tag).Name) > 0 { + newTags.TagSet = append(newTags.TagSet, &s3.Tag{Key: aws.String((*tag).Name), Value: aws.String((*tag).Value)}) + } + } + + if len(newTags.TagSet) == 0 { + return nil + } + _, err := e.svc.PutBucketTagging(&s3.PutBucketTaggingInput{Bucket: resourceID, Tagging: &newTags}) + return err +} + +// RetagBuckets parses all buckets and retags them +func (e *S3Processor) RetagBuckets(m mapper.Iface) { + result, err := e.svc.ListBuckets(&s3.ListBucketsInput{}) + if err != nil { + log.WithFields(logrus.Fields{"error": err}).Fatal("ListBuckets failed") + } + + for _, bucket := range result.Buckets { + // Makes sure we are in the right region and avoid stuffs like: + // AuthorizationHeaderMalformed: The authorization header is malformed; the region 'us-east-1' is wrong + if location, err := e.svc.GetBucketLocation(&s3.GetBucketLocationInput{Bucket: bucket.Name}); err != nil { + log.WithFields(logrus.Fields{"bucket": *bucket.Name, "error": err.Error()}).Fatal("GetBucketLocation failed") + } else { + loc := "" + if location.LocationConstraint != nil { + loc = *location.LocationConstraint + } + loc = s3.NormalizeBucketLocation(loc) + if loc != *e.region { + log.WithFields(logrus.Fields{"bucket": *bucket.Name, "location": loc}).Debug("Skipping bucket in different region than session") + continue // skip if the bucket is not the one specified in the session region + } + } + // Get the tags + bTags, err := e.svc.GetBucketTagging(&s3.GetBucketTaggingInput{Bucket: bucket.Name}) + if err != nil { + emptyTag := false + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + case "NoSuchTagSet": + emptyTag = true // ignore errors when not tagset associated to the bucket + } + } + if !emptyTag { + log.WithFields(logrus.Fields{"bucket": *bucket.Name, "error": err.Error()}).Fatal("GetBucketTagging failed") + } + } + tags := e.TagsToMap(bTags.TagSet) + keys := []string{} + if bucket.Name != nil { + keys = append(keys, *bucket.Name) + } + m.Retag(bucket.Name, &tags, keys, e.SetTags) + } +} diff --git a/providers/s3_test.go b/providers/s3_test.go new file mode 100644 index 0000000..c1c0661 --- /dev/null +++ b/providers/s3_test.go @@ -0,0 +1,172 @@ +package providers + +import ( + "errors" + "reflect" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" + "github.com/sirupsen/logrus" + logrus_test "github.com/sirupsen/logrus/hooks/test" + + "github.com/VEVO/awsRetagger/mapper" +) + +// mockS3Client is used to mock s3 calls +type mockS3Client struct { + s3iface.S3API + // ResourceID is the resource that has been passed to the mocked function + ResourceID *string + // ResourceTags are the tags that have been passed to the mocked function when + // setting or that is available on the mocked resource when getting + ResourceTags []*s3.Tag + // ReturnError is the error that you want your mocked function to return + ReturnError error + // BucketsNRegions is the list of buckets that ListBuckets pulls its output + // from as keys and regions as value + BucketsNRegions map[string]string + // BucketsTags is the list of tags corresponding to the buckets for + // GetBucketTagging + BucketsTags map[string][]*s3.Tag + // BucketsErrors output error of a given bucket for GetBucketTagging + BucketsErrors map[string]error +} + +func (m *mockS3Client) PutBucketTagging(input *s3.PutBucketTaggingInput) (*s3.PutBucketTaggingOutput, error) { + m.ResourceID = input.Bucket + if input.Tagging != nil { + m.ResourceTags = append(m.ResourceTags, input.Tagging.TagSet...) + } + return &s3.PutBucketTaggingOutput{}, m.ReturnError +} + +func (m *mockS3Client) ListBuckets(input *s3.ListBucketsInput) (*s3.ListBucketsOutput, error) { + buckets := []*s3.Bucket{} + for bucket := range m.BucketsNRegions { + buckets = append(buckets, &s3.Bucket{Name: aws.String(bucket)}) + } + output := s3.ListBucketsOutput{Owner: &s3.Owner{}, Buckets: buckets} + return &output, m.ReturnError +} + +func (m *mockS3Client) GetBucketLocation(input *s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error) { + region, _ := m.BucketsNRegions[*input.Bucket] + return &s3.GetBucketLocationOutput{LocationConstraint: aws.String(region)}, nil +} + +func (m *mockS3Client) GetBucketTagging(input *s3.GetBucketTaggingInput) (*s3.GetBucketTaggingOutput, error) { + var ( + tags []*s3.Tag + outputErr error + ) + tags, _ = m.BucketsTags[*input.Bucket] + outputErr, _ = m.BucketsErrors[*input.Bucket] + return &s3.GetBucketTaggingOutput{TagSet: tags}, outputErr +} + +func TestS3TagsToMap(t *testing.T) { + testData := []struct { + inputTags []*s3.Tag + outputTags map[string]string + }{ + {[]*s3.Tag{}, map[string]string{}}, + {[]*s3.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}, map[string]string{"foo": "bar"}}, + {[]*s3.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}, {Key: aws.String("Aerosmith"), Value: aws.String("rocks")}}, map[string]string{"foo": "bar", "Aerosmith": "rocks"}}, + } + for _, d := range testData { + p := S3Processor{} + + res := p.TagsToMap(d.inputTags) + if !reflect.DeepEqual(res, d.outputTags) { + t.Errorf("Expecting to get tags: %v\nGot: %v\n", d.outputTags, res) + } + } +} + +func TestS3SetTags(t *testing.T) { + testData := []struct { + inputResource, outputResource string + inputTags []*mapper.TagItem + outputTags []*s3.Tag + inputError, outputError error + }{ + {"my resource", "", []*mapper.TagItem{{}}, []*s3.Tag{}, nil, nil}, + {"my resource", "my resource", []*mapper.TagItem{{Name: "foo", Value: "bar"}}, []*s3.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}, nil, nil}, + {"my resource", "my resource", []*mapper.TagItem{{Name: "foo", Value: "bar"}, {Name: "Aerosmith", Value: "rocks"}}, []*s3.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}, {Key: aws.String("Aerosmith"), Value: aws.String("rocks")}}, nil, nil}, + {"my resource", "my resource", []*mapper.TagItem{{Name: "foo", Value: "bar"}}, []*s3.Tag{{Key: aws.String("foo"), Value: aws.String("bar")}}, errors.New("Badaboom"), errors.New("Badaboom")}, + } + for _, d := range testData { + mockSvc := &mockS3Client{ReturnError: d.inputError, ResourceTags: []*s3.Tag{}} + p := S3Processor{svc: mockSvc} + + err := p.SetTags(&d.inputResource, d.inputTags) + if !reflect.DeepEqual(err, d.outputError) { + t.Errorf("Expecting error: %v\nGot: %v\n", d.outputError, err) + } + + if mockSvc.ResourceID == nil { + if d.outputResource != "" { + t.Errorf("Expecting to update resource: %s, got no resourceID assigned\n", d.outputResource) + } + } else { + if *mockSvc.ResourceID != d.outputResource { + t.Errorf("Expecting to update resource: %s, got: %s\n", d.outputResource, *mockSvc.ResourceID) + } + } + + if !reflect.DeepEqual(mockSvc.ResourceTags, d.outputTags) { + t.Errorf("Expecting to update tag: %v\nGot: %v\n", d.outputTags, mockSvc.ResourceTags) + } + } +} + +func TestS3RetagBuckets(t *testing.T) { + testData := []struct { + sessionRegion string + inputBucketsNRegions map[string]string + inputBucketsTags map[string][]*s3.Tag + outputBucketsTags map[string]map[string]string + outputBucketsKeys map[string][]string + inputBucketsErrors map[string]error + }{ + {"us-east-1", map[string]string{}, map[string][]*s3.Tag{}, nil, nil, map[string]error{}}, + { + "us-east-1", + map[string]string{"bucket1": "us-east-1", "bucket2": "us-east-1", "homerSimpson": "us-west-2"}, + map[string][]*s3.Tag{"bucket1": {&s3.Tag{Key: aws.String("Team"), Value: aws.String("Gryffindor")}, &s3.Tag{Key: aws.String("Strength"), Value: aws.String("chivalry")}}, "homerSimpson": {&s3.Tag{Key: aws.String("Team"), Value: aws.String("Nuclear")}}}, + map[string]map[string]string{"bucket1": {"Team": "Gryffindor", "Strength": "chivalry"}, "bucket2": {}}, + map[string][]string{"bucket1": {"bucket1"}, "bucket2": {"bucket2"}}, + map[string]error{}, + }, + { + "us-west-2", + map[string]string{"bucket1": "us-east-1", "bucket2": "us-east-1", "homerSimpson": "us-west-2"}, + map[string][]*s3.Tag{"bucket1": {&s3.Tag{Key: aws.String("Team"), Value: aws.String("Gryffindor")}, &s3.Tag{Key: aws.String("Strength"), Value: aws.String("chivalry")}}, "homerSimpson": {&s3.Tag{Key: aws.String("Team"), Value: aws.String("Nuclear")}}}, + map[string]map[string]string{"homerSimpson": {"Team": "Nuclear"}}, + map[string][]string{"homerSimpson": {"homerSimpson"}}, + map[string]error{}, + }, + } + + // silence the logs + logger, _ := logrus_test.NewNullLogger() + log = logrus.NewEntry(logger) + + for _, d := range testData { + mockSvc := &mockS3Client{BucketsNRegions: d.inputBucketsNRegions, BucketsTags: d.inputBucketsTags, BucketsErrors: d.inputBucketsErrors} + m := mapper.MockMapper{} + p := S3Processor{svc: mockSvc, region: &d.sessionRegion} + p.RetagBuckets(&m) + + if !reflect.DeepEqual(d.outputBucketsTags, m.ResourceTags) { + t.Errorf("Expecting Mapper.Retag to receive tags: %v\nGot: %v\n", d.outputBucketsTags, m.ResourceTags) + } + + if !reflect.DeepEqual(d.outputBucketsKeys, m.ResourceKeys) { + t.Errorf("Expecting Mapper.Retag to receive keys: %v\nGot: %v\n", d.outputBucketsKeys, m.ResourceKeys) + } + + } +}