Clean flag added

go.mod

@@ -7,12 +7,13 @@ require (
 	github.com/nritholtz/stdemuxerhook v0.0.0-20181016194454-2c86ca05d211
 	github.com/rendon/testcli v0.0.0-20161027181003-6283090d169f
 	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.2.2
+	github.com/stretchr/testify v1.8.1
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,3 +1,4 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
@@ -10,8 +11,14 @@ github.com/rendon/testcli v0.0.0-20161027181003-6283090d169f h1:onGP+qmYmjKs7pkm
 github.com/rendon/testcli v0.0.0-20161027181003-6283090d169f/go.mod h1:cq57a4l475CeMvE7RRpSui1MEqCmhirIt1E7kl8BC2Q=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 h1:z8Hj/bl9cOV2grsOpEaQFUaly0JWN3i97mo3jXKJNp0=
@@ -20,3 +27,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -33,12 +33,17 @@ type module struct {
 	Source       string   `yaml:"source"`
 	Version      string   `yaml:"version"`
 	Destinations []string `yaml:"destinations"`
+	Source       string   `yaml:"source"`
+	Version      string   `yaml:"version"`
+	Destinations []string `yaml:"destinations"`
 }
 
 var opts struct {
 	ModulePath string `short:"p" long:"module_path" default:"./vendor/modules" description:"File path to install generated terraform modules, if not overridden by 'destinations:' field"`
 
 	TerrafilePath string `short:"f" long:"terrafile_file" default:"./Terrafile" description:"File path to the Terrafile file"`
+
+	Clean bool `short:"c" long:"clean" description:"Remove all artifacts from destinations and module path upon fetching module(s)"`
 }
 
 // To be set by goreleaser on build
@@ -93,6 +98,10 @@ func main() {
 		log.Fatalf("failed to parse yaml file due to error: %s", err)
 	}
 
+	if opts.Clean {
+		cleanDestinations(getUniqueDestinations(config))
+	}
+
 	// Clone modules
 	var wg sync.WaitGroup
 	_ = os.RemoveAll(opts.ModulePath)
@@ -157,3 +166,43 @@ func main() {
 
 	wg.Wait()
 }
+
+func getUniqueDestinations(config map[string]module) []string {
+
+	// Map filters duplicates
+	uniqueDestinationsM := make(map[string]string)
+
+	// Range over config and gather all uniquieDestinations
+	for _, m := range config {
+		if len(m.Destinations) == 0 {
+			uniqueDestinationsM[opts.ModulePath] = opts.ModulePath
+			continue
+		}
+
+		// range over Destinations and put them into map
+		for _, dst := range m.Destinations {
+			// Destination supposed to be conjunction of destination defined in file with module path
+			d := filepath.Join(dst, opts.ModulePath)
+			uniqueDestinationsM[d] = d
+		}
+	}
+
+	uniqueDestinations := make([]string, len(uniqueDestinationsM))
+	i := 0
+	for dst := range uniqueDestinationsM {
+		uniqueDestinations[i] = dst
+		i++
+	}
+
+	return uniqueDestinations
+}
+
+func cleanDestinations(destinations []string) {
+	for _, dst := range destinations {
+
+		log.Infof("[*] Removing artifacts from %s", dst)
+		if err := os.RemoveAll(dst); err != nil {
+			log.Errorf("Failed to remove artifacts from %s due to error: %s", dst, err)
+		}
+	}
+}

main_test.go

@@ -50,19 +50,42 @@ func TestTerraformWithTerrafilePath(t *testing.T) {
 		assert.NoError(t, os.RemoveAll("testdata/"))
 	}()
 
+	defer println(testcli.Stdout())
+	defer println(testcli.Stderr())
+
+	defer func() {
+		assert.NoError(t, os.RemoveAll("testdata/"))
+	}()
+
 	if !testcli.Success() {
 		t.Fatalf("Expected to succeed, but failed: %q with message: %q", testcli.Error(), testcli.Stderr())
 	}
 	// Assert output
 	for _, output := range []string{
+		"Checking out master of [email protected]:terraform-aws-modules/terraform-aws-vpc",
 		"Checking out master of [email protected]:terraform-aws-modules/terraform-aws-vpc",
 		"Checking out v1.46.0 of [email protected]:terraform-aws-modules/terraform-aws-vpc",
 		"Checking out v3.2.0 of [email protected]:terraform-aws-modules/terraform-aws-vpn-gateway",
 		"Checking out v3.6.1 of [email protected]:terraform-aws-modules/terraform-aws-s3-bucket",
 		"Checking out v5.11.1 of [email protected]:terraform-aws-modules/terraform-aws-iam",
+		"Checking out v3.2.0 of [email protected]:terraform-aws-modules/terraform-aws-vpn-gateway",
+		"Checking out v3.6.1 of [email protected]:terraform-aws-modules/terraform-aws-s3-bucket",
+		"Checking out v5.11.1 of [email protected]:terraform-aws-modules/terraform-aws-iam",
 	} {
 		assert.Contains(t, testcli.Stdout(), output)
 	}
+
+	// Assert output does not contain the following
+	for _, output := range []string{
+		"[*] Removing artifacts from ./vendor/modules",
+		"[*] Removing artifacts from testdata/networking/vendor/modules",
+		"[*] Removing artifacts from testdata/some-other-stack/vendor/modules",
+		"[*] Removing artifacts from testdata/iam/vendor/modules",
+		"[*] Removing artifacts from testdata/onboarding/vendor/modules",
+	} {
+		assert.NotContains(t, testcli.Stdout(), output)
+	}
+
 	// Assert folder exist with default destination
 	for _, moduleName := range []string{
 		"tf-aws-vpc",
@@ -84,6 +107,20 @@ func TestTerraformWithTerrafilePath(t *testing.T) {
 		assert.DirExists(t, path.Join(workingDirectory, moduleName))
 	}
 
+	// Assert files exist with default destination
+	// Assert folder exist with non-default destinations
+	for _, moduleName := range []string{
+		"testdata/networking/vendor/modules/tf-aws-vpn-gateway/",
+		"testdata/networking/vendor/modules/tf-aws-s3-bucket/",
+		"testdata/iam/vendor/modules/tf-aws-iam/",
+
+		// Symlinks are not Dirs. But contents will be tested later on
+		// "testdata/onboarding/vendor/modules/tf-aws-s3-bucket/",
+		// "testdata/some-other-stack/vendor/modules/tf-aws-s3-bucket/",
+	} {
+		assert.DirExists(t, path.Join(workingDirectory, moduleName))
+	}
+
 	// Assert files exist with default destination
 	for _, moduleName := range []string{
 		"tf-aws-vpc/main.tf",
@@ -106,6 +143,20 @@ func TestTerraformWithTerrafilePath(t *testing.T) {
 		assert.FileExists(t, path.Join(workingDirectory, moduleName))
 	}
 
+	// Assert files exist with non-default destinations
+	for _, moduleName := range []string{
+		"testdata/networking/vendor/modules/tf-aws-vpn-gateway/main.tf",
+		"testdata/networking/vendor/modules/tf-aws-s3-bucket/main.tf",
+
+		// terraform-aws-modules/terraform-aws-iam doesn't have main.tf, as it represents set of modules
+		// However, some terraform-aws-modules/terraform-aws-iam/modules have, e.g.:
+		"testdata/iam/vendor/modules/tf-aws-iam/modules/iam-account/main.tf",
+		"testdata/onboarding/vendor/modules/tf-aws-s3-bucket/main.tf",
+		"testdata/some-other-stack/vendor/modules/tf-aws-s3-bucket/main.tf",
+	} {
+		assert.FileExists(t, path.Join(workingDirectory, moduleName))
+	}
+
 	// Assert checked out correct version
 	for moduleName, cloneOptions := range map[string]map[string]string{
 		"tf-aws-vpc": map[string]string{
@@ -175,6 +226,7 @@ func TestTerraformWithTerrafilePath(t *testing.T) {
 }
 
 func setup(t *testing.T) (current string, back func()) {
+	folder, err := os.MkdirTemp("", "")
 	folder, err := os.MkdirTemp("", "")
 	assert.NoError(t, err)
 	createTerrafile(t, folder)
@@ -185,6 +237,7 @@ func setup(t *testing.T) (current string, back func()) {
 
 func createFile(t *testing.T, filename string, contents string) {
 	assert.NoError(t, os.WriteFile(filename, []byte(contents), 0644))
+	assert.NoError(t, os.WriteFile(filename, []byte(contents), 0644))
 }
 
 func createTerrafile(t *testing.T, folder string) {
@@ -211,6 +264,47 @@ tf-aws-s3-bucket:
     - testdata/networking
     - testdata/onboarding
     - testdata/some-other-stack
+tf-aws-vpn-gateway:
+  source:  "[email protected]:terraform-aws-modules/terraform-aws-vpn-gateway"
+  version: "v3.2.0"
+  destinations:
+    - testdata/networking
+tf-aws-iam:
+  source:  "[email protected]:terraform-aws-modules/terraform-aws-iam"
+  version: "v5.11.1"
+  destinations:
+    - testdata/iam
+tf-aws-s3-bucket:
+  source:  "[email protected]:terraform-aws-modules/terraform-aws-s3-bucket"
+  version: "v3.6.1"
+  destinations:
+    - testdata/networking
+    - testdata/onboarding
+    - testdata/some-other-stack
 `
 	createFile(t, path.Join(folder, "Terrafile"), yaml)
 }
+
+func createTerrafile2(t *testing.T, folder string) {
+	var yaml = `tf-aws-vpc:
+  source:  "[email protected]:terraform-aws-modules/terraform-aws-vpc"
+  version: "v1.46.0"
+tf-aws-vpn-gateway:
+  source:  "[email protected]:terraform-aws-modules/terraform-aws-vpn-gateway"
+  version: "v3.2.0"
+  destinations:
+    - testdata/networking
+    - testdata/some-other-stack
+tf-aws-iam:
+  source:  "[email protected]:terraform-aws-modules/terraform-aws-iam"
+  version: "v5.11.1"
+  destinations:
+    - testdata/iam
+tf-aws-s3-bucket:
+  source:  "[email protected]:terraform-aws-modules/terraform-aws-s3-bucket"
+  version: "v3.6.1"
+  destinations:
+    - testdata/onboarding
+`
+	createFile(t, path.Join(folder, "Terrafile2"), yaml)
+}