From 9bb215357bc8ec818f163e2dac420bf093a42cb1 Mon Sep 17 00:00:00 2001 From: Denis O Date: Tue, 5 Nov 2024 21:22:28 +0000 Subject: [PATCH 01/13] Terragrunt log level handling --- .circleci/config.yml | 2 +- modules/terraform/cmd.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7bf50dac5..210beefb4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ env: &env TERRAFORM_VERSION: 1.5.7 TOFU_VERSION: 1.8.0 PACKER_VERSION: 1.10.0 - TERRAGRUNT_VERSION: v0.52.0 + TERRAGRUNT_VERSION: v0.68.7 OPA_VERSION: v0.33.1 GO_VERSION: 1.21.1 GO111MODULE: auto diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index 48e1324f7..ec1e23359 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -51,6 +51,7 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.TerraformBinary == "terragrunt" { args = append(args, "--terragrunt-non-interactive") + args = append(args, "--terragrunt-log-disable") } if options.Parallelism > 0 && len(args) > 0 && collections.ListContains(commandsWithParallelism, args[0]) { From baad51e52598d905ef12e8b9c5d43c4be2406145 Mon Sep 17 00:00:00 2001 From: Denis O Date: Tue, 5 Nov 2024 22:03:09 +0000 Subject: [PATCH 02/13] Added tests for fetching tg output --- modules/terraform/cmd.go | 2 +- modules/terraform/output.go | 32 +++++++++++++++++++++++++++++++- modules/terraform/output_test.go | 26 ++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index ec1e23359..4362c35c3 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -51,7 +51,7 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.TerraformBinary == "terragrunt" { args = append(args, "--terragrunt-non-interactive") - args = append(args, "--terragrunt-log-disable") + args = append(args, "--terragrunt-forward-tf-stdout") } if options.Parallelism > 0 && len(args) > 0 && collections.ListContains(commandsWithParallelism, args[0]) { diff --git a/modules/terraform/output.go b/modules/terraform/output.go index 6e294e7a2..e60c9fdd1 100644 --- a/modules/terraform/output.go +++ b/modules/terraform/output.go @@ -5,7 +5,9 @@ import ( "errors" "fmt" "reflect" + "regexp" "strconv" + "strings" "github.com/gruntwork-io/terratest/modules/testing" "github.com/stretchr/testify/require" @@ -279,7 +281,11 @@ func OutputJsonE(t testing.TestingT, options *Options, key string) (string, erro args = append(args, key) } - return RunTerraformCommandAndGetStdoutE(t, options, args...) + rawJson, err := RunTerraformCommandAndGetStdoutE(t, options, args...) + if err != nil { + return rawJson, err + } + return cleanJson(rawJson) } // OutputStruct calls terraform output for the given variable and stores the @@ -348,3 +354,27 @@ func OutputAll(t testing.TestingT, options *Options) map[string]interface{} { func OutputAllE(t testing.TestingT, options *Options) (map[string]interface{}, error) { return OutputForKeysE(t, options, nil) } + +// clean the ANSI characters from the JSON and update formating +func cleanJson(input string) (string, error) { + ansiLineRegex := regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`) + cleaned := ansiLineRegex.ReplaceAllString(input, "") + lines := strings.Split(cleaned, "\n") + var result []string + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed != "" { + result = append(result, trimmed) + } + } + ansiClean := strings.Join(result, "\n") + var jsonObj interface{} + if err := json.Unmarshal([]byte(ansiClean), &jsonObj); err != nil { + return "", err + } + normalized, err := json.MarshalIndent(jsonObj, "", " ") + if err != nil { + return "", err + } + return string(normalized), nil +} diff --git a/modules/terraform/output_test.go b/modules/terraform/output_test.go index f3285d69f..dc26d4814 100644 --- a/modules/terraform/output_test.go +++ b/modules/terraform/output_test.go @@ -2,8 +2,11 @@ package terraform import ( "fmt" + "path/filepath" "testing" + "github.com/stretchr/testify/assert" + "github.com/gruntwork-io/terratest/modules/files" "github.com/stretchr/testify/require" ) @@ -433,3 +436,26 @@ func TestOutputsForKeysError(t *testing.T) { require.Error(t, err) } + +func TestTgOutputJsonParsing(t *testing.T) { + t.Parallel() + + testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-output-map", t.Name()) + require.NoError(t, err) + + WriteFile(t, filepath.Join(testFolder, "terragrunt.hcl"), []byte{}) + + options := &Options{ + TerraformDir: testFolder, + TerraformBinary: "terragrunt", + } + + InitAndApply(t, options) + + output, err := OutputAllE(t, options) + + require.NoError(t, err) + assert.NotNil(t, output) + assert.NotEmpty(t, output) + assert.Contains(t, output, "mogwai") +} From ddc10c31d99cc30a7df2b7554688dc1d41bef618 Mon Sep 17 00:00:00 2001 From: Denis O Date: Tue, 5 Nov 2024 22:07:15 +0000 Subject: [PATCH 03/13] Added test for unicode characters --- modules/terraform/output_test.go | 1 + test/fixtures/terraform-output-map/output.tf | 3 +++ 2 files changed, 4 insertions(+) diff --git a/modules/terraform/output_test.go b/modules/terraform/output_test.go index dc26d4814..b23f30f0d 100644 --- a/modules/terraform/output_test.go +++ b/modules/terraform/output_test.go @@ -458,4 +458,5 @@ func TestTgOutputJsonParsing(t *testing.T) { assert.NotNil(t, output) assert.NotEmpty(t, output) assert.Contains(t, output, "mogwai") + assert.Equal(t, "söme chäräcter", output["not_a_map_unicode"]) } diff --git a/test/fixtures/terraform-output-map/output.tf b/test/fixtures/terraform-output-map/output.tf index a722e8f6b..6a7aee48c 100644 --- a/test/fixtures/terraform-output-map/output.tf +++ b/test/fixtures/terraform-output-map/output.tf @@ -11,3 +11,6 @@ output "not_a_map" { value = "This is not a map." } +output "not_a_map_unicode" { + value = "söme chäräcter" +} From 57fc39ba126128d247de203f701048388c13ccef Mon Sep 17 00:00:00 2001 From: Denis O Date: Tue, 5 Nov 2024 22:16:04 +0000 Subject: [PATCH 04/13] Terragrunt output tests --- modules/terraform/output_test.go | 39 ++++++++++++++++++++++++ test/fixtures/terraform-output/output.tf | 4 +++ 2 files changed, 43 insertions(+) diff --git a/modules/terraform/output_test.go b/modules/terraform/output_test.go index b23f30f0d..f2d0d927d 100644 --- a/modules/terraform/output_test.go +++ b/modules/terraform/output_test.go @@ -34,6 +34,40 @@ func TestOutputString(t *testing.T) { num1 := Output(t, options, "number1") require.Equal(t, num1, "3", "Number %q should match %q", "3", num1) + + unicodeString := Output(t, options, "unicode_string") + require.Equal(t, "söme chäräcter", unicodeString) +} + +func TestTgOutputString(t *testing.T) { + t.Parallel() + + testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-output", t.Name()) + require.NoError(t, err) + + WriteFile(t, filepath.Join(testFolder, "terragrunt.hcl"), []byte{}) + + options := &Options{ + TerraformDir: testFolder, + TerraformBinary: "terragrunt", + } + + InitAndApply(t, options) + + b := Output(t, options, "bool") + require.Equal(t, b, "true", "Bool %q should match %q", "true", b) + + str := Output(t, options, "string") + require.Equal(t, str, "This is a string.", "String %q should match %q", "This is a string.", str) + + num := Output(t, options, "number") + require.Equal(t, num, "3.14", "Number %q should match %q", "3.14", num) + + num1 := Output(t, options, "number1") + require.Equal(t, num1, "3", "Number %q should match %q", "3", num1) + + unicodeString := Output(t, options, "unicode_string") + require.Equal(t, "söme chäräcter", unicodeString) } func TestOutputList(t *testing.T) { @@ -313,6 +347,11 @@ func TestOutputJson(t *testing.T) { "sensitive": false, "type": "string", "value": "This is a string." + }, + "unicode_string": { + "sensitive": false, + "type": "string", + "value": "söme chäräcter" } }` diff --git a/test/fixtures/terraform-output/output.tf b/test/fixtures/terraform-output/output.tf index e9d5ae693..5b92808e3 100644 --- a/test/fixtures/terraform-output/output.tf +++ b/test/fixtures/terraform-output/output.tf @@ -13,3 +13,7 @@ output "number" { output "number1" { value = 3 } + +output "unicode_string" { + value = "söme chäräcter" +} From 071224f720bd5d3c3c418a4c173eb03ae98e31b8 Mon Sep 17 00:00:00 2001 From: Denis O Date: Wed, 6 Nov 2024 14:13:42 +0000 Subject: [PATCH 05/13] Arguments cleanup --- modules/terraform/cmd.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index 4362c35c3..48e1324f7 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -51,7 +51,6 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.TerraformBinary == "terragrunt" { args = append(args, "--terragrunt-non-interactive") - args = append(args, "--terragrunt-forward-tf-stdout") } if options.Parallelism > 0 && len(args) > 0 && collections.ListContains(commandsWithParallelism, args[0]) { From 515b60c777ff79f1049e57a141434696479686f5 Mon Sep 17 00:00:00 2001 From: Denis O Date: Wed, 6 Nov 2024 19:14:25 +0000 Subject: [PATCH 06/13] Default log level update --- modules/terraform/cmd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index 48e1324f7..17eb92aa7 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -51,6 +51,7 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.TerraformBinary == "terragrunt" { args = append(args, "--terragrunt-non-interactive") + args = append(args, "--terragrunt-log-level", "warn") } if options.Parallelism > 0 && len(args) > 0 && collections.ListContains(commandsWithParallelism, args[0]) { From 4ea7ce871f53cfe512494888ea1c5a4be4dc21b7 Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 7 Nov 2024 15:37:59 +0000 Subject: [PATCH 07/13] Formatting update --- modules/terraform/cmd.go | 5 ++++- modules/terraform/output.go | 13 +++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index 17eb92aa7..d9052da9e 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -51,7 +51,10 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.TerraformBinary == "terragrunt" { args = append(args, "--terragrunt-non-interactive") - args = append(args, "--terragrunt-log-level", "warn") + _, formattingIset := options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] + if !formattingIset { + options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] = "true" + } } if options.Parallelism > 0 && len(args) > 0 && collections.ListContains(commandsWithParallelism, args[0]) { diff --git a/modules/terraform/output.go b/modules/terraform/output.go index e60c9fdd1..1fc93141a 100644 --- a/modules/terraform/output.go +++ b/modules/terraform/output.go @@ -13,6 +13,10 @@ import ( "github.com/stretchr/testify/require" ) +const skipJsonLogLine = "log=info msg=" + +var ansiLineRegex = regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`) + // Output calls terraform output for the given variable and return its string value representation. // It only designed to work with primitive terraform types: string, number and bool. // Please use OutputStruct for anything else. @@ -357,24 +361,29 @@ func OutputAllE(t testing.TestingT, options *Options) (map[string]interface{}, e // clean the ANSI characters from the JSON and update formating func cleanJson(input string) (string, error) { - ansiLineRegex := regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`) + // Remove ANSI escape codes cleaned := ansiLineRegex.ReplaceAllString(input, "") + lines := strings.Split(cleaned, "\n") var result []string for _, line := range lines { trimmed := strings.TrimSpace(line) - if trimmed != "" { + if trimmed != "" && !strings.Contains(trimmed, skipJsonLogLine) { result = append(result, trimmed) } } ansiClean := strings.Join(result, "\n") + var jsonObj interface{} if err := json.Unmarshal([]byte(ansiClean), &jsonObj); err != nil { return "", err } + + // Format JSON output with indentation normalized, err := json.MarshalIndent(jsonObj, "", " ") if err != nil { return "", err } + return string(normalized), nil } From d6ea8dbf818823ffc7d5e08d2b2784cf9d506e84 Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 7 Nov 2024 17:26:42 +0000 Subject: [PATCH 08/13] updated default path --- modules/terraform/cmd.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index d9052da9e..03a3849a7 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -39,6 +39,9 @@ const ( // TerraformDefaultPath to run terraform TerraformDefaultPath = "terraform" + + // TerragruntDefaultPath to run terragrunt + TerragruntDefaultPath = "terragrunt" ) var DefaultExecutable = defaultTerraformExecutable() @@ -49,8 +52,12 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { options.TerraformBinary = DefaultExecutable } - if options.TerraformBinary == "terragrunt" { + if options.TerraformBinary == TerragruntDefaultPath { args = append(args, "--terragrunt-non-interactive") + // for newer Terragrunt version, setting simplified log formatting + if options.EnvVars == nil { + options.EnvVars = map[string]string{} + } _, formattingIset := options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] if !formattingIset { options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] = "true" From fd220404e0bad05f1fe30319f9209b7ceea7c7cd Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 7 Nov 2024 20:12:18 +0000 Subject: [PATCH 09/13] Fixed logs cleaning --- modules/terraform/cmd.go | 4 ++-- modules/terraform/output.go | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index 03a3849a7..f5f082bbf 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -58,8 +58,8 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.EnvVars == nil { options.EnvVars = map[string]string{} } - _, formattingIset := options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] - if !formattingIset { + _, tgLogSet := options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] + if !tgLogSet { options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] = "true" } } diff --git a/modules/terraform/output.go b/modules/terraform/output.go index 1fc93141a..93a2ed299 100644 --- a/modules/terraform/output.go +++ b/modules/terraform/output.go @@ -13,9 +13,12 @@ import ( "github.com/stretchr/testify/require" ) -const skipJsonLogLine = "log=info msg=" +const skipJsonLogLine = " msg=" -var ansiLineRegex = regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`) +var ( + ansiLineRegex = regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`) + tgLogLevel = regexp.MustCompile(`.*time=\S+ level=\S+ prefix=\S+ binary=\S+ msg=.*`) +) // Output calls terraform output for the given variable and return its string value representation. // It only designed to work with primitive terraform types: string, number and bool. @@ -363,6 +366,7 @@ func OutputAllE(t testing.TestingT, options *Options) (map[string]interface{}, e func cleanJson(input string) (string, error) { // Remove ANSI escape codes cleaned := ansiLineRegex.ReplaceAllString(input, "") + cleaned = tgLogLevel.ReplaceAllString(cleaned, "") lines := strings.Split(cleaned, "\n") var result []string From a294bb8ddcf51a9d41044d3c252b0f397939cc7a Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 5 Dec 2024 17:17:35 +0000 Subject: [PATCH 10/13] Custom log format --- modules/terraform/cmd.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index f5f082bbf..c6c5d2380 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -58,9 +58,13 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.EnvVars == nil { options.EnvVars = map[string]string{} } - _, tgLogSet := options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] + _, tgLogSet := options.EnvVars["TERRAGRUNT_LOG_FORMAT"] if !tgLogSet { - options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] = "true" + options.EnvVars["TERRAGRUNT_LOG_FORMAT"] = "key-value" + } + _, tgLogFormat := options.EnvVars["TERRAGRUNT_LOG_CUSTOM_FORMAT"] + if !tgLogFormat { + options.EnvVars["TERRAGRUNT_LOG_CUSTOM_FORMAT"] = "%msg(color=disable)" } } From 44a56162e9d5b2918513b046a299b23735e5acde Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 5 Dec 2024 17:26:23 +0000 Subject: [PATCH 11/13] version update --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a499b0a8..0a0ac0cdc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ env: &env TERRAFORM_VERSION: 1.5.7 TOFU_VERSION: 1.8.0 PACKER_VERSION: 1.10.0 - TERRAGRUNT_VERSION: v0.68.7 + TERRAGRUNT_VERSION: v0.69.8 OPA_VERSION: v0.33.1 GO_VERSION: 1.21.1 GO111MODULE: auto From 6b1772c0723005fbf5e89b68d5aa3e3576614a29 Mon Sep 17 00:00:00 2001 From: Denis O Date: Mon, 9 Dec 2024 17:57:04 +0000 Subject: [PATCH 12/13] PR comments --- modules/terraform/cmd.go | 2 ++ modules/terraform/output.go | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index c6c5d2380..ea4baf511 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -60,6 +60,8 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { } _, tgLogSet := options.EnvVars["TERRAGRUNT_LOG_FORMAT"] if !tgLogSet { + // key-value format for terragrunt logs to avoid colors and have plain form + // https://terragrunt.gruntwork.io/docs/reference/cli-options/#terragrunt-log-format options.EnvVars["TERRAGRUNT_LOG_FORMAT"] = "key-value" } _, tgLogFormat := options.EnvVars["TERRAGRUNT_LOG_CUSTOM_FORMAT"] diff --git a/modules/terraform/output.go b/modules/terraform/output.go index 93a2ed299..b4b67ec7b 100644 --- a/modules/terraform/output.go +++ b/modules/terraform/output.go @@ -16,8 +16,10 @@ import ( const skipJsonLogLine = " msg=" var ( + // ansiLineRegex matches lines starting with ANSI escape codes for text formatting (e.g., colors, styles). ansiLineRegex = regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`) - tgLogLevel = regexp.MustCompile(`.*time=\S+ level=\S+ prefix=\S+ binary=\S+ msg=.*`) + // tgLogLevel matches log lines containing fields for time, level, prefix, binary, and message, each with non-whitespace values. + tgLogLevel = regexp.MustCompile(`.*time=\S+ level=\S+ prefix=\S+ binary=\S+ msg=.*`) ) // Output calls terraform output for the given variable and return its string value representation. From aa1ddcd12275b77028c8bef9d8276bb380623907 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:20:22 -0500 Subject: [PATCH 13/13] feat: Adding release instructions in PR --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6b100e40e..cebc6ff96 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -14,6 +14,7 @@ Read the [Gruntwork contribution guidelines](https://gruntwork.notion.site/Grunt - [ ] Run the relevant tests successfully, including pre-commit checks. - [ ] Ensure any 3rd party code adheres with our [license policy](https://www.notion.so/gruntwork/Gruntwork-licenses-and-open-source-usage-policy-f7dece1f780341c7b69c1763f22b1378) or delete this line if its not applicable. - [ ] Include release notes. If this PR is backward incompatible, include a migration guide. +- [ ] Make a plan for release of the functionality in this PR. If it delivers value to an end user, you are responsible for ensuring it is released promptly, and correctly. If you are not a maintainer, you are responsible for finding a maintainer to do this for you. ## Release Notes (draft)