Skip to content

Commit

Permalink
Dependency optimisation fix (#3461)
Browse files Browse the repository at this point in the history
* Dependency optimisation update

* dependency parsing improvements

* Depenedncy reading update

* improved decoding of outputs

* imports update

* Inputs handling

* updated log message

* Dep optimisation fix

* Test deps update

* Cleanup

* Add test for deps skip

* Documentation update

* Spacing update

* Cleanup

* Cleanup

* Documentation upadte

* Warning message update

* PR comments

* Errors update
  • Loading branch information
denis256 authored Oct 28, 2024
1 parent 98a8874 commit 09e9813
Show file tree
Hide file tree
Showing 12 changed files with 127 additions and 15 deletions.
12 changes: 12 additions & 0 deletions config/config_partial.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"os"
"path/filepath"

"github.com/gruntwork-io/terragrunt/internal/strict"

clone "github.com/huandu/go-clone"

"github.com/gruntwork-io/terragrunt/internal/cache"
Expand Down Expand Up @@ -333,6 +335,16 @@ func PartialParseConfig(ctx *ParsingContext, file *hclparse.File, includeFromChi
output.IamWebIdentityToken = *decoded.IamWebIdentityToken
}
case TerragruntInputs:
control, ok := strict.GetStrictControl(strict.SkipDependenciesInputs)
if ok {
_, skipInputs := control.Evaluate(ctx.TerragruntOptions)
if skipInputs != nil {
ctx.TerragruntOptions.Logger.Warnf("Skipping inputs reading from %v inputs for better performance", file.ConfigPath)

break
}
}

decoded := terragruntInputs{}

if _, ok := evalParsingContext.Variables[MetadataDependency]; !ok {
Expand Down
7 changes: 7 additions & 0 deletions docs/_docs/04_reference/strict-mode.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ The following strict mode controls are available:
- [destroy-all](#destroy-all)
- [output-all](#output-all)
- [validate-all](#validate-all)
- [skip-dependencies-inputs](#skip-dependencies-inputs)

### spin-up

Expand Down Expand Up @@ -114,3 +115,9 @@ Throw an error when using the `output-all` command.
Throw an error when using the `validate-all` command.

**Reason**: The `validate-all` command is deprecated and will be removed in a future version. Use `terragrunt run-all validate` instead.

### skip-dependencies-inputs

Disable reading of dependency inputs to enhance dependency resolution performance by preventing recursively parsing Terragrunt inputs from dependencies.

**Reason**: Enabling the `skip-dependencies-inputs` option prevents the recursive parsing of Terragrunt inputs, leading to optimized performance during dependency resolution.
38 changes: 23 additions & 15 deletions internal/strict/strict.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@
package strict

import (
"errors"
"fmt"
"strings"

"github.com/gruntwork-io/terragrunt/internal/errors"

"github.com/gruntwork-io/terragrunt/options"
"github.com/gruntwork-io/terragrunt/util"
)
Expand Down Expand Up @@ -40,6 +41,9 @@ const (
OutputAll = "output-all"
// ValidateAll is the control that prevents the deprecated `validate-all` command from being used.
ValidateAll = "validate-all"

// SkipDependenciesInputs is the control that prevents reading dependencies inputs and get performance boost.
SkipDependenciesInputs = "skip-dependencies-inputs"
)

// GetStrictControl returns the strict control with the given name.
Expand Down Expand Up @@ -76,32 +80,36 @@ type Controls map[string]Control
//nolint:lll,gochecknoglobals,stylecheck
var StrictControls = Controls{
SpinUp: {
Error: errors.New("The `spin-up` command is no longer supported. Use `terragrunt run-all apply` instead."),
Warning: "The `spin-up` command is deprecated and will be removed in a future version. Use `terragrunt run-all apply` instead.",
Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all apply` instead.", SpinUp),
Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all apply` instead.", SpinUp),
},
TearDown: {
Error: errors.New("The `tear-down` command is no longer supported. Use `terragrunt run-all destroy` instead."),
Warning: "The `tear-down` command is deprecated and will be removed in a future version. Use `terragrunt run-all destroy` instead.",
Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all destroy` instead.", TearDown),
Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all destroy` instead.", TearDown),
},
PlanAll: {
Error: errors.New("The `plan-all` command is no longer supported. Use `terragrunt run-all plan` instead."),
Warning: "The `plan-all` command is deprecated and will be removed in a future version. Use `terragrunt run-all plan` instead.",
Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all plan` instead.", PlanAll),
Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all plan` instead.", PlanAll),
},
ApplyAll: {
Error: errors.New("The `apply-all` command is no longer supported. Use `terragrunt run-all apply` instead."),
Warning: "The `apply-all` command is deprecated and will be removed in a future version. Use `terragrunt run-all apply` instead.",
Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all apply` instead.", ApplyAll),
Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all apply` instead.", ApplyAll),
},
DestroyAll: {
Error: errors.New("The `destroy-all` command is no longer supported. Use `terragrunt run-all destroy` instead."),
Warning: "The `destroy-all` command is deprecated and will be removed in a future version. Use `terragrunt run-all destroy` instead.",
Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all destroy` instead.", DestroyAll),
Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all destroy` instead.", DestroyAll),
},
OutputAll: {
Error: errors.New("The `output-all` command is no longer supported. Use `terragrunt run-all output` instead."),
Warning: "The `output-all` command is deprecated and will be removed in a future version. Use `terragrunt run-all output` instead.",
Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all output` instead.", OutputAll),
Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all output` instead.", OutputAll),
},
ValidateAll: {
Error: errors.New("The `validate-all` command is no longer supported. Use `terragrunt run-all validate` instead."),
Warning: "The `validate-all` command is deprecated and will be removed in a future version. Use `terragrunt run-all validate` instead.",
Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all validate` instead.", ValidateAll),
Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all validate` instead.", ValidateAll),
},
SkipDependenciesInputs: {
Error: errors.Errorf("The `%s` option is deprecated. Reading inputs from dependencies has been deprecated and will be removed in a future version of Terragrunt. To continue using inputs from dependencies, forward them as outputs.", SkipDependenciesInputs),
Warning: fmt.Sprintf("The `%s` option is deprecated and will be removed in a future version of Terragrunt. Reading inputs from dependencies has been deprecated. To continue using inputs from dependencies, forward them as outputs.", SkipDependenciesInputs),
},
}

Expand Down
2 changes: 2 additions & 0 deletions options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -612,6 +612,8 @@ func (opts *TerragruntOptions) Clone(terragruntConfigPath string) (*TerragruntOp
EngineLogLevel: opts.EngineLogLevel,
EngineSkipChecksumCheck: opts.EngineSkipChecksumCheck,
Engine: cloneEngineOptions(opts.Engine),
// copy array
StrictControls: util.CloneStringList(opts.StrictControls),
}, nil
}

Expand Down
3 changes: 3 additions & 0 deletions test/fixtures/dependency-optimisation/module-a/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
output "test_output" {
value = "hello from module-a"
}
7 changes: 7 additions & 0 deletions test/fixtures/dependency-optimisation/module-a/terragrunt.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
terraform {
source = ".//"
}

include "root" {
path = find_in_parent_folders()
}
7 changes: 7 additions & 0 deletions test/fixtures/dependency-optimisation/module-b/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
variable "test_var" {
type = string
}

output "test_output" {
value = var.test_var
}
15 changes: 15 additions & 0 deletions test/fixtures/dependency-optimisation/module-b/terragrunt.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
terraform {
source = ".//"
}

include "root" {
path = find_in_parent_folders()
}

dependency "module_a" {
config_path = "../module-a"
}

inputs = {
test_var = dependency.module_a.outputs.test_output
}
7 changes: 7 additions & 0 deletions test/fixtures/dependency-optimisation/module-c/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
variable "test_var" {
type = string
}

output "result" {
value = var.test_var
}
15 changes: 15 additions & 0 deletions test/fixtures/dependency-optimisation/module-c/terragrunt.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
terraform {
source = ".//"
}

include "root" {
path = find_in_parent_folders()
}

dependency "module_b" {
config_path = "../module-b"
}

inputs = {
test_var = dependency.module_b.outputs.test_output
}
10 changes: 10 additions & 0 deletions test/fixtures/dependency-optimisation/terragrunt.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
remote_state {
backend = "local"
generate = {
path = "backend.tf"
if_exists = "overwrite_terragrunt"
}
config = {
path = "terraform.tfstate"
}
}
19 changes: 19 additions & 0 deletions test/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ const (
testFixtureTfTest = "fixtures/tftest/"
testFixtureErrorPrint = "fixtures/error-print"
testFixtureBufferModuleOutput = "fixtures/buffer-module-output"
testFixtureDependenciesOptimisation = "fixtures/dependency-optimisation"

terraformFolder = ".terraform"

Expand Down Expand Up @@ -2682,6 +2683,24 @@ func TestLogFailingDependencies(t *testing.T) {
assert.Contains(t, output, fmt.Sprintf("%s invocation failed in %s", wrappedBinary(), getPathRelativeTo(t, testdataDir, path)))
}

func TestDependenciesOptimisation(t *testing.T) {
tmpEnvPath := copyEnvironment(t, testFixtureDependenciesOptimisation)
rootPath := util.JoinPath(tmpEnvPath, testFixtureDependenciesOptimisation)

_, _, err := runTerragruntCommandWithOutput(t, "terragrunt run-all apply -auto-approve --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir "+rootPath)
require.NoError(t, err)

config.ClearOutputCache()

moduleC := util.JoinPath(tmpEnvPath, testFixtureDependenciesOptimisation, "module-c")
t.Setenv("TERRAGRUNT_STRICT_CONTROL", "skip-dependencies-inputs")
_, stderr, err := runTerragruntCommandWithOutput(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir "+moduleC)
require.NoError(t, err)

// checking that dependencies optimisation is working and outputs from module-a are not retrieved
assert.NotContains(t, stderr, "Retrieved output from ../module-a/terragrunt.hcl")
}

func cleanupTerraformFolder(t *testing.T, templatesPath string) {
t.Helper()

Expand Down

0 comments on commit 09e9813

Please sign in to comment.