Major refactor and add s3 bucket support

CHANGELOG.md

@@ -6,8 +6,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+### Changed
+- Moved mapper and providers to separate packages for easier management
+
 ### Added
 - Add more unit tests
+- Add support for retagging s3 buckets
+- Add an interface and a Mocking base class for the mapper for easier unit
+  testing
 
 ## [0.1.0] - 2017-11-22
 

Makefile

@@ -26,9 +26,13 @@ lint: fmt
 
 gocov:
 	@go get github.com/axw/gocov/gocov \
-	&& go install github.com/axw/gocov/gocov
-	@gocov test | gocov report
-	# gocov test >/tmp/gocovtest.json ; gocov annotate /tmp/gocovtest.json MyFunc
+	&& go install github.com/axw/gocov/gocov; \
+	if [ -f "glide.yaml" ] ; then \
+		gocov test $$(glide novendor) | gocov report; \
+	else \
+		gocov test ./... | gocov report; \
+	fi
+	# gocov test $$(glide novendor) >/tmp/gocovtest.json ; gocov annotate /tmp/gocovtest.json MyFunc
 
 test:
 	@if [ -f "glide.yaml" ] ; then \

README.md

@@ -110,6 +110,7 @@ The key elements of a resource are the following attribute (first come in the gi
 * a RDS Instance: `DBClusterIdentifier`, `DBInstanceIdentifier`, `DBName`, `MasterUsername`
 * a RDS Cluster: `DBClusterIdentifier`, `DatabaseName`, `MasterUsername`
 * a Redshift Cluster: `ClusterIdentifier`, `DBName`, `MasterUsername`
+* an S3 Bucket: `BucketName`
 
 With the following configuration, an instance with a SSH KeyName set to
 `apple-tv-analytics-prod`, you'll end up with:
@@ -240,6 +241,8 @@ Usage of ./awsRetagger:
         Enables the re-tagging of the RDS instances. Environment variable: RDS_INSTANCES
   -redshift-clusters
         Enables the re-tagging of the Redshift clusters. Environment variable: REDSHIFT_CLUSTERS
+  -s3-buckets
+        Enables the re-tagging of the S3 buckets. Environment variable: S3_BUCKETS
 ```
 
 ### Use inside Docker
@@ -261,3 +264,4 @@ you might check using the `-h` option of the command-line):
 * RDS Instances
 * RDS Clusters
 * Redshift Clusters
+* S3 Buckets

TODO → TODO.md

@@ -1,3 +1,6 @@
+# TODO list
+
+Few ideas that might be worth exploring:
 * Add tests
 * ec2.go add support for retagging:
   * ELB
@@ -16,7 +19,6 @@
   * DataPipeline
   * Elasticache
   * ElasticBeanstalk
-  * S3
   * Lambda
   * Route53
   * Simpledb

glide.yaml

@@ -1,10 +1,11 @@
 package: github.com/VEVO/awsRetagger
 import:
 - package: github.com/aws/aws-sdk-go
-  version: ^1.12.32
+  version: ^1.12.47
   subpackages:
   - aws
   - aws/arn
+  - aws/awserr
   - aws/session
   - service/cloudfront
   - service/cloudfront/cloudfrontiface
@@ -14,9 +15,12 @@ import:
   - service/elasticbeanstalk/elasticbeanstalkiface
   - service/elasticsearchservice
   - service/rds
+  - service/rds/rdsiface
   - service/redshift
   - service/redshift/redshiftiface
+  - service/s3
+  - service/s3/s3iface
   - service/sts
 - package: github.com/gobike/envflag
 - package: github.com/sirupsen/logrus
-  version: ^1.0.3
+  version: ^1.0.4

main.go

@@ -3,11 +3,15 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
+	"os"
+
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/gobike/envflag"
 	"github.com/sirupsen/logrus"
-	"io"
-	"os"
+
+	"github.com/VEVO/awsRetagger/mapper"
+	"github.com/VEVO/awsRetagger/providers"
 )
 
 var log *logrus.Entry
@@ -38,9 +42,9 @@ func NewLogger(logLevel, format string, output io.Writer) (*logrus.Entry, error)
 
 func main() {
 	var (
-		configFilePath, logLevel, logFormat                                                                                                                                                        string
-		processEc2Instances, processRdsInstances, processRdsClusters, processCloudwatchLogGroups, processElasticSearch, processCloudFrontDist, processRedshiftClusters, processElasticBeanstalkEnv bool
-		err                                                                                                                                                                                        error
+		configFilePath, logLevel, logFormat                                                                                                                                                                          string
+		processEc2Instances, processRdsInstances, processRdsClusters, processCloudwatchLogGroups, processElasticSearch, processCloudFrontDist, processRedshiftClusters, processElasticBeanstalkEnv, processS3Buckets bool
+		err                                                                                                                                                                                                          error
 	)
 	flag.StringVar(&configFilePath, "config", "config.json", "Path of the json configuration file. Environment variable: CONFIG")
 	flag.StringVar(&logLevel, "log-level", "info", "Log level. Accepted values: debug, info, warn, error, fatal, panic. Environment variable: LOG_LEVEL")
@@ -53,20 +57,23 @@ func main() {
 	flag.BoolVar(&processCloudFrontDist, "cloudfront-distributions", false, "Enables the re-tagging of the CloudFront distributions. Environment variable: CLOUDFRONT_DISTRIBUTIONS")
 	flag.BoolVar(&processRedshiftClusters, "redshift-clusters", false, "Enables the re-tagging of the Redshift clusters. Environment variable: REDSHIFT_CLUSTERS")
 	flag.BoolVar(&processElasticBeanstalkEnv, "elasticbeanstalk-environments", false, "Enables the re-tagging of the ElasticBeanstalk environments. Environment variable: ELASTICBEANSTALK_ENVIRONMENTS")
+	flag.BoolVar(&processS3Buckets, "s3-buckets", false, "Enables the re-tagging of the S3 buckets. Environment variable: S3_BUCKETS")
 	envflag.Parse()
 
 	if log, err = NewLogger(logLevel, logFormat, os.Stdout); err != nil {
 		fmt.Printf("Error while setting up the logger: %s\n", err)
 		os.Exit(1)
 	}
+	mapper.SetLogger(log)
+	providers.SetLogger(log)
 	// Load config
 	cfg, err := os.Open(configFilePath)
 	defer cfg.Close()
 	if err != nil {
 		log.WithFields(logrus.Fields{"error": err}).Fatal("Unable to read config file")
 	}
 
-	m := Mapper{}
+	m := mapper.Mapper{}
 	if err = m.LoadConfig(cfg); err != nil {
 		log.WithFields(logrus.Fields{"error": err}).Fatal("Unable to load config file")
 	}
@@ -76,12 +83,12 @@ func main() {
 	}))
 
 	if processEc2Instances {
-		e := NewEc2Processor(sess)
+		e := providers.NewEc2Processor(sess)
 		e.RetagInstances(&m)
 	}
 
 	if processRdsInstances || processRdsClusters {
-		r := NewRdsProcessor(sess)
+		r := providers.NewRdsProcessor(sess)
 		if processRdsInstances {
 			r.RetagInstances(&m)
 		}
@@ -91,28 +98,33 @@ func main() {
 	}
 
 	if processCloudwatchLogGroups {
-		c := NewCwProcessor(sess)
+		c := providers.NewCwProcessor(sess)
 		c.RetagLogGroups(&m)
 	}
 
 	if processElasticSearch {
-		elk := NewElkProcessor(sess)
+		elk := providers.NewElkProcessor(sess)
 		elk.RetagDomains(&m)
 	}
 
 	if processCloudFrontDist {
-		cf := NewCloudFrontProcessor(sess)
+		cf := providers.NewCloudFrontProcessor(sess)
 		cf.RetagDistributions(&m)
 	}
 	if processRedshiftClusters {
-		rs, err := NewRedshiftProcessor(sess)
+		rs, err := providers.NewRedshiftProcessor(sess)
 		if err != nil {
 			log.WithFields(logrus.Fields{"error": err}).Fatal("Unable to initialize the Redshift client")
 		}
 		rs.RetagClusters(&m)
 	}
 	if processElasticBeanstalkEnv {
-		eb := NewElasticBeanstalkProcessor(sess)
+		eb := providers.NewElasticBeanstalkProcessor(sess)
 		eb.RetagEnvironments(&m)
 	}
+
+	if processS3Buckets {
+		sp := providers.NewS3Processor(sess)
+		sp.RetagBuckets(&m)
+	}
 }

errors.go → mapper/errors.go

@@ -1,4 +1,4 @@
-package main
+package mapper
 
 // ErrSanityNoMapping is returned for the sanity checks in the Mapper
 type ErrSanityNoMapping struct {

mapper/interface.go

@@ -0,0 +1,24 @@
+package mapper
+
+import (
+	"io"
+)
+
+// PutTagFn is used to specify the function structure to pass to the Retag method
+type PutTagFn func(*string, []*TagItem) error
+
+// Iface has been created for testing purposes. It allows to create mocks
+// when testing class that depend on the mapper
+type Iface interface {
+	LoadConfig(io.Reader) error
+	StripDefaults(*map[string]string)
+	GetMissingDefaults(*map[string]string) *map[string]string
+	GetFromTags(*map[string]string) (*map[string]string, error)
+	GetFromKey(string, *map[string]string) (*map[string]string, error)
+	ValidateTag(string, string) (*TagItem, error)
+	MergeMaps(*map[string]string, *map[string]string)
+
+	Retag(*string, *map[string]string, []string, PutTagFn)
+}
+
+var _ Iface = (*Mapper)(nil)

mapper.go → mapper/mapper.go

@@ -1,12 +1,18 @@
-package main
+package mapper
 
 import (
 	"encoding/json"
-	"github.com/sirupsen/logrus"
 	"io"
 	"regexp"
+
+	"github.com/sirupsen/logrus"
 )
 
+var log *logrus.Entry
+
+// SetLogger is used to pass the loger from the main program
+func SetLogger(logger *logrus.Entry) { log = logger }
+
 // TagItem is a standard AWS tag structure
 type TagItem struct {
 	Name  string `json:"name"`
@@ -48,6 +54,7 @@ type TagSanity struct {
 // Mapper contains the different mappings between attributes and the list of
 // tags that should be present on that resource
 type Mapper struct {
+	Iface
 	CopyTag          []*TagCopy        `json:"copy_tags,omitempty"`
 	TagMap           []*TagMapper      `json:"tags,omitempty"`
 	KeyMap           []*KeyMapper      `json:"keys,omitempty"`
@@ -242,10 +249,8 @@ func (m *Mapper) MergeMaps(mainMap, complementary *map[string]string) {
 	}
 }
 
-type putTagFn func(*string, []*TagItem) error
-
 // Retag does the different re-tagging operations and calls the given setTags function
-func (m *Mapper) Retag(resourceID *string, tags *map[string]string, keys []string, setTags putTagFn) {
+func (m *Mapper) Retag(resourceID *string, tags *map[string]string, keys []string, setTags PutTagFn) {
 	var (
 		newTags, mapFromKey, mapFromMissing *map[string]string
 		err                                 error

mapper_test.go → mapper/mapper_test.go

@@ -1,13 +1,14 @@
-package main
+package mapper
 
 import (
 	"errors"
-	"github.com/sirupsen/logrus"
-	logrus_test "github.com/sirupsen/logrus/hooks/test"
 	"reflect"
 	"regexp/syntax"
 	"strings"
 	"testing"
+
+	"github.com/sirupsen/logrus"
+	logrus_test "github.com/sirupsen/logrus/hooks/test"
 )
 
 func TestLoadConfig(t *testing.T) {
@@ -343,7 +344,7 @@ func TestRetag(t *testing.T) {
 		resourceID    string
 		tags          map[string]string
 		keys          []string
-		setTags       putTagFn
+		setTags       PutTagFn
 		logEntries    int
 		expected      map[string]string
 		config        Mapper