Merge 'master' to 'add-vault-support-for-sops' (#1)

* Fix dead link in multiple aws accounts docs (gruntwork-io#1563)

* Fix dead link in multiple aws accounts docs

The link to AWS docs is now 404.
The corrected link seems to most closely resemble the intended target.
Other options to consider:
https://aws.amazon.com/organizations/getting-started/best-practices/
https://docs.aws.amazon.com/controltower/latest/userguide/aws-multi-account-landing-zone.html

* Link to AWS best practices for multi account docs

* Whitespace removal (gruntwork-io#1573)

* Fix empty outputs (gruntwork-io#1568)

If stack run finished without errors, `summarizePlanAllErrors()`
receives empty buffer and outputs empty line. This change ensures that
only non-empty outputs are getting logged.

Related: gruntwork-io#1541

* doc: contributing: fix broken link to circleci (gruntwork-io#1580)

* Bump AWS SDK to version v1.37.7 to support AWS SSO (gruntwork-io#1537)

* Add TargetPrefix as config input to access bucket logging (gruntwork-io#1507)

* adding target-prefix ro access bucket logging

* Updating test & example

! Note that this needs the terratest PR (gruntwork-io/terratest#767) to be merged in to work & be tested.

* Updating Terratest dependency

* testing for target prefix

* Updating docs

* Renaming folder

* Updating to Debugf

* Adding default value

* WIP - parsing for TFstatelogs

* Updating logic & docs

* Adding a new test for default TargetPrefix in remote backend config

* Introduce validate-inputs, which can be used to check for variable alignment (gruntwork-io#1572)

* Introduce terragrunt-input-info, which can be used to check for variable alignment

* Apply suggestions from code review

Co-authored-by: Zack Proser <[email protected]>

* Tidy go modules

* Renamed input-info to validate-inputs

* Switch missing required vars to errors

* Handle -var and -var-file args

* Update cli/validate_inputs.go

Co-authored-by: Yevgeniy Brikman <[email protected]>

* Make sure to check for dynamically passed in CLI args

* Fix build

* Handle automatically loaded var files

* Remove plan args check

* Clarify difference between getTerraformInputNamesFromVarFiles and getTerraformInputNamesFromCLIArgs

* Address PR nit to move example in docs

Co-authored-by: Zack Proser <[email protected]>
Co-authored-by: Yevgeniy Brikman <[email protected]>

* Use go1.16 to build arm64 binaries (gruntwork-io#1585)

* Bump creack/pty to 1.1.11 (gruntwork-io#1582)

Co-authored-by: Andy Bohne <[email protected]>

* Add ability to specify working directory of hooks (gruntwork-io#1588)

* Add ability to specify working directory of hooks

* Fix build

* Support dynamodb_endpoint attribute of S3 backend (gruntwork-io#1586)

* Clarify non-interactive will not include external dependencies (gruntwork-io#1593)

* add getTerragruntSource helper function (gruntwork-io#1575)

* add getTerragruntSource helper function

* update docs

* update docs and tests for get_terragrunt_source_cli_flag() function

* add use cases for get_terragrunt_source_cli_flag

* Recursively extract forcedgetters until there are none (gruntwork-io#1594)

* Remove all usage of get-plugins=false which is removed in 0.15.0 (gruntwork-io#1618)

* Fix validate-inputs to support null defaults (gruntwork-io#1613)

* Clarify context of find_in_parent_folders (gruntwork-io#1623)

Co-authored-by: Paul <[email protected]>
Co-authored-by: Yoriyasu Yano <[email protected]>
Co-authored-by: amnk <[email protected]>
Co-authored-by: Marco Molteni <[email protected]>
Co-authored-by: David Wooldridge <[email protected]>
Co-authored-by: Ina Stoyanova <[email protected]>
Co-authored-by: Zack Proser <[email protected]>
Co-authored-by: Yevgeniy Brikman <[email protected]>
Co-authored-by: Andy Bohne <[email protected]>
Co-authored-by: Andy Bohne <[email protected]>
Co-authored-by: Alexey Remizov <[email protected]>
Co-authored-by: Syed Hussain <[email protected]>
Co-authored-by: David Alger <[email protected]>

.circleci/config.yml

@@ -1,6 +1,7 @@
 defaults: &defaults
   docker:
-    - image: 087285199408.dkr.ecr.us-east-1.amazonaws.com/circle-ci-test-image-base:tf14.4
+    - image: 087285199408.dkr.ecr.us-east-1.amazonaws.com/circle-ci-test-image-base:go1.16-tf14.4-tg28.8
+
 version: 2
 jobs:
   # We're running unit tests separately from integration tests - with no parallelization.
@@ -58,16 +59,12 @@ jobs:
     <<: *defaults
     steps:
       - checkout
-      # Build amd64 and 386 separately, as darwin/386 is no longer supported in later versions of go
-      - run: build-go-binaries --app-name terragrunt --dest-path bin --ld-flags "-X main.VERSION=$CIRCLE_TAG" --arch 'amd64'
-      - run: build-go-binaries --app-name terragrunt --dest-path bin --ld-flags "-X main.VERSION=$CIRCLE_TAG" --os 'linux windows' --arch 386
+      - run: build-go-binaries --app-name terragrunt --dest-path bin --ld-flags "-X main.VERSION=$CIRCLE_TAG"
   deploy:
     <<: *defaults
     steps:
       - checkout
-      # Build amd64 and 386 separately, as darwin/386 is no longer supported in later versions of go
-      - run: build-go-binaries --app-name terragrunt --dest-path bin --ld-flags "-X main.VERSION=$CIRCLE_TAG" --arch 'amd64'
-      - run: build-go-binaries --app-name terragrunt --dest-path bin --ld-flags "-X main.VERSION=$CIRCLE_TAG" --os 'linux windows' --arch 386
+      - run: build-go-binaries --app-name terragrunt --dest-path bin --ld-flags "-X main.VERSION=$CIRCLE_TAG"
       - run: cd bin && sha256sum * > SHA256SUMS
       - run: upload-github-release-assets bin/*
 workflows:

aws_helper/config.go

@@ -17,6 +17,7 @@ import (
 type AwsSessionConfig struct {
 	Region                  string
 	CustomS3Endpoint        string
+	CustomDynamoDBEndpoint  string
 	Profile                 string
 	RoleArn                 string
 	CredsFilename           string
@@ -36,6 +37,11 @@ func CreateAwsSession(config *AwsSessionConfig, terragruntOptions *options.Terra
 				URL:           config.CustomS3Endpoint,
 				SigningRegion: config.Region,
 			}, nil
+		} else if service == "dynamodb" && config.CustomDynamoDBEndpoint != "" {
+			return endpoints.ResolvedEndpoint{
+				URL:           config.CustomDynamoDBEndpoint,
+				SigningRegion: config.Region,
+			}, nil
 		}
 
 		return defaultResolver.EndpointFor(service, region, optFns...)

cli/args.go

@@ -220,23 +220,9 @@ func filterTerraformExtraArgs(terragruntOptions *options.TerragruntOptions, terr
 				}
 
 				if !skipVars {
-					// If RequiredVarFiles is specified, add -var-file=<file> for each specified files
-					if arg.RequiredVarFiles != nil {
-						for _, file := range util.RemoveDuplicatesFromListKeepLast(*arg.RequiredVarFiles) {
-							out = append(out, fmt.Sprintf("-var-file=%s", file))
-						}
-					}
-
-					// If OptionalVarFiles is specified, check for each file if it exists and if so, add -var-file=<file>
-					// It is possible that many files resolve to the same path, so we remove duplicates.
-					if arg.OptionalVarFiles != nil {
-						for _, file := range util.RemoveDuplicatesFromListKeepLast(*arg.OptionalVarFiles) {
-							if util.FileExists(file) {
-								out = append(out, fmt.Sprintf("-var-file=%s", file))
-							} else {
-								terragruntOptions.Logger.Debugf("Skipping var-file %s as it does not exist", file)
-							}
-						}
+					varFiles := arg.GetVarFiles(terragruntOptions.Logger)
+					for _, file := range varFiles {
+						out = append(out, fmt.Sprintf("-var-file=%s", file))
 					}
 				}
 			}
@@ -434,7 +420,7 @@ func toTerraformEnvVars(vars map[string]interface{}) (map[string]string, error)
 	out := map[string]string{}
 
 	for varName, varValue := range vars {
-		envVarName := fmt.Sprintf("TF_VAR_%s", varName)
+		envVarName := fmt.Sprintf("%s_%s", TFVarPrefix, varName)
 
 		envVarValue, err := asTerraformEnvVarJsonValue(varValue)
 		if err != nil {

cli/cli_app.go

@@ -81,6 +81,7 @@ const CMD_INIT_FROM_MODULE = "init-from-module"
 const CMD_PROVIDERS = "providers"
 const CMD_LOCK = "lock"
 const CMD_TERRAGRUNT_INFO = "terragrunt-info"
+const CMD_TERRAGRUNT_VALIDATE_INPUTS = "validate-inputs"
 const CMD_TERRAGRUNT_GRAPH_DEPENDENCIES = "graph-dependencies"
 const CMD_TERRAGRUNT_READ_CONFIG = "terragrunt-read-config"
 const CMD_HCLFMT = "hclfmt"
@@ -197,12 +198,13 @@ USAGE:
    {{.Usage}}
 
 COMMANDS:
-   run-all              Run a terraform command against a 'stack' by running the specified command in each subfolder. E.g., to run 'terragrunt apply' in each subfolder, use 'terragrunt run-all apply'.
-   terragrunt-info      Emits limited terragrunt state on stdout and exits
-   graph-dependencies   Prints the terragrunt dependency graph to stdout
-   hclfmt               Recursively find terragrunt.hcl files and rewrite them into a canonical format.
-   aws-provider-patch   Overwrite settings on nested AWS providers to work around a Terraform bug (issue #13018)
-   *                    Terragrunt forwards all other commands directly to Terraform
+   run-all               Run a terraform command against a 'stack' by running the specified command in each subfolder. E.g., to run 'terragrunt apply' in each subfolder, use 'terragrunt run-all apply'.
+   terragrunt-info       Emits limited terragrunt state on stdout and exits
+   validate-inputs       Checks if the terragrunt configured inputs align with the terraform defined variables.
+   graph-dependencies    Prints the terragrunt dependency graph to stdout
+   hclfmt                Recursively find terragrunt.hcl files and rewrite them into a canonical format.
+   aws-provider-patch    Overwrite settings on nested AWS providers to work around a Terraform bug (issue #13018)
+   *                     Terragrunt forwards all other commands directly to Terraform
 
 GLOBAL OPTIONS:
    terragrunt-config                            Path to the Terragrunt config file. Default is terragrunt.hcl.
@@ -244,6 +246,9 @@ const DEFAULT_TERRAFORM_VERSION_CONSTRAINT = ">= v0.12.0"
 
 const TERRAFORM_EXTENSION_GLOB = "*.tf"
 
+// Prefix to use for terraform variables set with environment variables.
+const TFVarPrefix = "TF_VAR"
+
 // The supported flags to show help of terraform commands
 var TERRAFORM_HELP_FLAGS = []string{
 	"--help",
@@ -437,6 +442,12 @@ func RunTerragrunt(terragruntOptions *options.TerragruntOptions) error {
 		}
 	}
 
+	// We do the terragrunt input validation here, after all the terragrunt generated terraform files are created so
+	// that we can ensure the necessary information is available.
+	if shouldValidateTerragruntInputs(updatedTerragruntOptions) {
+		return validateTerragruntInputs(updatedTerragruntOptions, terragruntConfig)
+	}
+
 	// We do the debug file generation here, after all the terragrunt generated terraform files are created so that we
 	// can ensure the tfvars json file only includes the vars that are defined in the module.
 	if updatedTerragruntOptions.Debug {
@@ -519,6 +530,10 @@ func shouldPrintTerragruntInfo(terragruntOptions *options.TerragruntOptions) boo
 	return util.ListContainsElement(terragruntOptions.TerraformCliArgs, CMD_TERRAGRUNT_INFO)
 }
 
+func shouldValidateTerragruntInputs(terragruntOptions *options.TerragruntOptions) bool {
+	return util.ListContainsElement(terragruntOptions.TerraformCliArgs, CMD_TERRAGRUNT_VALIDATE_INPUTS)
+}
+
 func shouldRunHCLFmt(terragruntOptions *options.TerragruntOptions) bool {
 	return util.ListContainsElement(terragruntOptions.TerraformCliArgs, CMD_HCLFMT)
 }
@@ -540,10 +555,20 @@ func processHooks(hooks []config.Hook, terragruntOptions *options.TerragruntOpti
 		allPreviousErrors := append(previousExecError, errorsOccurred...)
 		if shouldRunHook(curHook, terragruntOptions, allPreviousErrors...) {
 			terragruntOptions.Logger.Infof("Executing hook: %s", curHook.Name)
+			workingDir := ""
+			if curHook.WorkingDir != nil {
+				workingDir = *curHook.WorkingDir
+			}
+
 			actionToExecute := curHook.Execute[0]
 			actionParams := curHook.Execute[1:]
-			possibleError := shell.RunShellCommand(terragruntOptions, actionToExecute, actionParams...)
-
+			_, possibleError := shell.RunShellCommandWithOutput(
+				terragruntOptions,
+				workingDir,
+				false,
+				false,
+				actionToExecute, actionParams...,
+			)
 			if possibleError != nil {
 				terragruntOptions.Logger.Errorf("Error running hook %s with message: %s", curHook.Name, possibleError.Error())
 				errorsOccurred = append(errorsOccurred, possibleError)

cli/debug.go

@@ -27,10 +27,12 @@ func writeTerragruntDebugFile(terragruntOptions *options.TerragruntOptions, terr
 		terragruntOptions.WorkingDir,
 	)
 
-	variables, err := terraformModuleVariables(terragruntOptions)
+	required, optional, err := terraformModuleVariables(terragruntOptions)
 	if err != nil {
 		return err
 	}
+	variables := append(required, optional...)
+
 	terragruntOptions.Logger.Debugf("The following variables were detected in the terraform module:")
 	terragruntOptions.Logger.Debugf("%v", variables)
 
@@ -71,7 +73,7 @@ func terragruntDebugFileContents(
 
 	jsonValuesByKey := make(map[string]interface{})
 	for varName, varValue := range terragruntConfig.Inputs {
-		nameAsEnvVar := fmt.Sprintf("TF_VAR_%s", varName)
+		nameAsEnvVar := fmt.Sprintf("%s_%s", TFVarPrefix, varName)
 		_, varIsInEnv := envVars[nameAsEnvVar]
 		varIsDefined := util.ListContainsElement(moduleVariables, varName)
 
@@ -100,17 +102,22 @@ func terragruntDebugFileContents(
 }
 
 // terraformModuleVariables will return all the variables defined in the downloaded terraform modules, taking into
-// account all the generated sources.
-func terraformModuleVariables(terragruntOptions *options.TerragruntOptions) ([]string, error) {
+// account all the generated sources. This function will return the required and optional variables separately.
+func terraformModuleVariables(terragruntOptions *options.TerragruntOptions) ([]string, []string, error) {
 	modulePath := terragruntOptions.WorkingDir
 	module, diags := tfconfig.LoadModule(modulePath)
 	if diags.HasErrors() {
-		return nil, errors.WithStackTrace(diags)
+		return nil, nil, errors.WithStackTrace(diags)
 	}
 
-	variables := []string{}
+	required := []string{}
+	optional := []string{}
 	for _, variable := range module.Variables {
-		variables = append(variables, variable.Name)
+		if variable.Required {
+			required = append(required, variable.Name)
+		} else {
+			optional = append(optional, variable.Name)
+		}
 	}
-	return variables, nil
+	return required, optional, nil
 }

cli/tfsource/types.go

@@ -142,7 +142,15 @@ func toSourceUrl(source string, workingDir string) (*url.URL, error) {
 // Parse the given source URL into a URL struct. This method can handle source URLs that include go-getter's "forced
 // getter" prefixes, such as git::.
 func parseSourceUrl(source string) (*url.URL, error) {
+	forcedGetters := []string{}
+	// Continuously strip the forced getters until there is no more. This is to handle complex URL schemes like the
+	// git-remote-codecommit style URL.
 	forcedGetter, rawSourceUrl := getForcedGetter(source)
+	for forcedGetter != "" {
+		// Prepend like a stack, so that we prepend to the URL scheme in the right order.
+		forcedGetters = append([]string{forcedGetter}, forcedGetters...)
+		forcedGetter, rawSourceUrl = getForcedGetter(rawSourceUrl)
+	}
 
 	// Parse the URL without the getter prefix
 	canonicalSourceUrl, err := urlhelper.Parse(rawSourceUrl)
@@ -151,7 +159,7 @@ func parseSourceUrl(source string) (*url.URL, error) {
 	}
 
 	// Reattach the "getter" prefix as part of the scheme
-	if forcedGetter != "" {
+	for _, forcedGetter := range forcedGetters {
 		canonicalSourceUrl.Scheme = fmt.Sprintf("%s::%s", forcedGetter, canonicalSourceUrl.Scheme)
 	}
 

cli/tfsource/types_test.go

@@ -58,3 +58,21 @@ func TestSplitSourceUrl(t *testing.T) {
 		})
 	}
 }
+
+func TestRegressionSupportForGitRemoteCodecommit(t *testing.T) {
+	t.Parallel()
+
+	terragruntOptions, err := options.NewTerragruntOptionsForTest("testing")
+	require.NoError(t, err)
+
+	source := "git::codecommit::ap-northeast-1://my_app_modules//my-app/modules/main-module"
+	sourceURL, err := toSourceUrl(source, ".")
+	require.NoError(t, err)
+	require.Equal(t, "git::codecommit::ap-northeast-1", sourceURL.Scheme)
+
+	actualRootRepo, actualModulePath, err := splitSourceUrl(sourceURL, terragruntOptions.Logger)
+	require.NoError(t, err)
+
+	require.Equal(t, "git::codecommit::ap-northeast-1://my_app_modules", actualRootRepo.String())
+	require.Equal(t, "my-app/modules/main-module", actualModulePath)
+}