diff --git a/command/apply.go b/command/apply.go index 97fbe54..7030e44 100644 --- a/command/apply.go +++ b/command/apply.go @@ -12,12 +12,14 @@ import ( // ApplyCommand is a command which computes a new state and pushes it to the remote state. type ApplyCommand struct { Meta + backendConfig []string } // Run runs the procedure of this command. func (c *ApplyCommand) Run(args []string) int { cmdFlags := flag.NewFlagSet("apply", flag.ContinueOnError) cmdFlags.StringVar(&c.configFile, "config", defaultConfigFile, "A path to tfmigrate config file") + cmdFlags.StringArrayVar(&c.backendConfig, "backend-config", nil, "A backend configuration for remote state") if err := cmdFlags.Parse(args); err != nil { c.UI.Error(fmt.Sprintf("failed to parse arguments: %s", err)) @@ -32,6 +34,7 @@ func (c *ApplyCommand) Run(args []string) int { log.Printf("[DEBUG] [command] config: %#v\n", c.config) c.Option = newOption() + c.Option.BackendConfig = c.backendConfig // The option may contain sensitive values such as environment variables. // So logging the option set log level to DEBUG instead of INFO. log.Printf("[DEBUG] [command] option: %#v\n", c.Option) @@ -105,11 +108,13 @@ Apply computes a new state and pushes it to remote state. It will fail if terraform plan detects any diffs with the new state. Arguments - PATH A path of migration file - Required in non-history mode. Optional in history-mode. + PATH A path of migration file + Required in non-history mode. Optional in history-mode. Options: - --config A path to tfmigrate config file + --config A path to tfmigrate config file + --backend-config=path A backend configuration, a path to backend configuration file or key=value format backend configuraion. + This option is passed to terraform init when switching backend to remote. ` return strings.TrimSpace(helpText) } diff --git a/command/plan.go b/command/plan.go index a166320..af54288 100644 --- a/command/plan.go +++ b/command/plan.go @@ -13,13 +13,15 @@ import ( // migration operations to a temporary state. type PlanCommand struct { Meta - out string + backendConfig []string + out string } // Run runs the procedure of this command. func (c *PlanCommand) Run(args []string) int { cmdFlags := flag.NewFlagSet("plan", flag.ContinueOnError) cmdFlags.StringVar(&c.configFile, "config", defaultConfigFile, "A path to tfmigrate config file") + cmdFlags.StringArrayVar(&c.backendConfig, "backend-config", nil, "A backend configuration for remote state") cmdFlags.StringVar(&c.out, "out", "", "[Deprecated] Save a plan file after dry-run migration to the given path") if err := cmdFlags.Parse(args); err != nil { @@ -36,6 +38,7 @@ func (c *PlanCommand) Run(args []string) int { c.Option = newOption() c.Option.PlanOut = c.out + c.Option.BackendConfig = c.backendConfig // The option may contains sensitive values such as environment variables. // So logging the option set log level to DEBUG instead of INFO. log.Printf("[DEBUG] [command] option: %#v\n", c.Option) @@ -115,18 +118,20 @@ Plan computes a new state by applying state migration operations to a temporary It will fail if terraform plan detects any diffs with the new state. Arguments: - PATH A path of migration file - Required in non-history mode. Optional in history-mode. + PATH A path of migration file + Required in non-history mode. Optional in history-mode. Options: - --config A path to tfmigrate config file + --config A path to tfmigrate config file + --backend-config=path A backend configuration, a path to backend configuration file or key=value format backend configuraion. + This option is passed to terraform init when switching backend to remote. [Deprecated] --out=path - Save a plan file after dry-run migration to the given path. - Note that applying the plan file only affects a local state, - make sure to force push it to remote after terraform apply. - This option doesn't work with Terraform 1.1+ + Save a plan file after dry-run migration to the given path. + Note that applying the plan file only affects a local state, + make sure to force push it to remote after terraform apply. + This option doesn't work with Terraform 1.1+ ` return strings.TrimSpace(helpText) } diff --git a/docker-compose.yml b/docker-compose.yml index cff314a..f253348 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: - localstack localstack: - image: localstack/localstack:0.11.3 + image: localstack/localstack:0.14.4 ports: - "4566:4566" environment: @@ -29,6 +29,6 @@ services: - "./scripts/localstack:/docker-entrypoint-initaws.d" # initialize scripts on startup dockerize: - image: jwilder/dockerize + image: powerman/dockerize:0.16.0 depends_on: - localstack diff --git a/tfexec/terraform.go b/tfexec/terraform.go index 428d654..3759fa2 100644 --- a/tfexec/terraform.go +++ b/tfexec/terraform.go @@ -125,7 +125,7 @@ type TerraformCLI interface { // so we need to switch the backend to local for temporary state operations. // The filename argument must meet constraints for override file. // (e.g.) _tfexec_override.tf - OverrideBackendToLocal(ctx context.Context, filename string, workspace string, isBackendTerraformCloud bool) (func(), error) + OverrideBackendToLocal(ctx context.Context, filename string, workspace string, isBackendTerraformCloud bool, backendConfig []string) (func(), error) // PlanHasChange is a helper method which runs plan and return true if the plan has change. PlanHasChange(ctx context.Context, state *State, opts ...string) (bool, error) @@ -198,7 +198,7 @@ func (c *terraformCLI) SetExecPath(execPath string) { // The filename argument must meet constraints in order to override the file. // (e.g.) _tfexec_override.tf func (c *terraformCLI) OverrideBackendToLocal(ctx context.Context, filename string, - workspace string, isBackendTerraformCloud bool) (func(), error) { + workspace string, isBackendTerraformCloud bool, backendConfig []string) (func(), error) { // create local backend override file. path := filepath.Join(c.Dir(), filename) contents := ` @@ -254,12 +254,15 @@ terraform { } log.Printf("[INFO] [executor@%s] switch back to remote\n", c.Dir()) + var args = []string{"-input=false", "-no-color", "-reconfigure"} + for _, b := range backendConfig { + args = append(args, fmt.Sprintf("-backend-config=%s", b)) + } // Run the correct init command depending on whether the remote backend is Terraform Cloud if !isBackendTerraformCloud { - err = c.Init(ctx, "-input=false", "-no-color", "-reconfigure") - } else { - err = c.Init(ctx, "-input=false", "-no-color") + args = append(args, "-reconfigure") } + err = c.Init(ctx, args...) if err != nil { // we cannot return error here. diff --git a/tfexec/terraform_init.go b/tfexec/terraform_init.go index 523093f..45750b9 100644 --- a/tfexec/terraform_init.go +++ b/tfexec/terraform_init.go @@ -1,6 +1,8 @@ package tfexec -import "context" +import ( + "context" +) // Init initializes the current work directory. func (c *terraformCLI) Init(ctx context.Context, opts ...string) error { diff --git a/tfexec/terraform_test.go b/tfexec/terraform_test.go index 406c894..5937d58 100644 --- a/tfexec/terraform_test.go +++ b/tfexec/terraform_test.go @@ -2,6 +2,7 @@ package tfexec import ( "context" + "fmt" "os" "path/filepath" "testing" @@ -125,7 +126,155 @@ resource "aws_security_group" "bar" {} t.Fatalf("an override file already exists: %s", err) } - switchBackToRemotekFunc, err := terraformCLI.OverrideBackendToLocal(context.Background(), filename, workspace, false) + switchBackToRemotekFunc, err := terraformCLI.OverrideBackendToLocal(context.Background(), filename, workspace, false, nil) + if err != nil { + t.Fatalf("failed to run OverrideBackendToLocal: %s", err) + } + + if _, err := os.Stat(filepath.Join(terraformCLI.Dir(), filename)); os.IsNotExist(err) { + t.Fatalf("the override file does not exist: %s", err) + } + + updatedState, _, err := terraformCLI.StateMv(context.Background(), state, nil, "aws_security_group.foo", "aws_security_group.foo2") + if err != nil { + t.Fatalf("failed to run terraform state mv: %s", err) + } + + changed, err = terraformCLI.PlanHasChange(context.Background(), updatedState) + if err != nil { + t.Fatalf("failed to run PlanHasChange: %s", err) + } + if changed { + t.Fatalf("expect not to have changes") + } + + switchBackToRemotekFunc() + + if _, err := os.Stat(filepath.Join(terraformCLI.Dir(), filename)); err == nil { + t.Fatalf("the override file wasn't removed: %s", err) + } + + changed, err = terraformCLI.PlanHasChange(context.Background(), nil) + if err != nil { + t.Fatalf("failed to run PlanHasChange: %s", err) + } + if !changed { + t.Fatalf("expect to have changes") + } +} + +func TestAccTerraformCLIOverrideBackendToLocalWithBackendConfig(t *testing.T) { + SkipUnlessAcceptanceTestEnabled(t) + + endpoint := "http://localhost:4566" + localstackEndpoint := os.Getenv("LOCALSTACK_ENDPOINT") + if len(localstackEndpoint) > 0 { + endpoint = localstackEndpoint + } + + backend := fmt.Sprintf(` +terraform { + # https://www.terraform.io/docs/backends/types/s3.html + backend "s3" { + region = "ap-northeast-1" + // bucket = "tfstate-test" + key = "%s/terraform.tfstate" + + // mock s3 endpoint with localstack + endpoint = "%s" + access_key = "dummy" + secret_key = "dummy" + skip_credentials_validation = true + skip_metadata_api_check = true + force_path_style = true + } +} +# https://www.terraform.io/docs/providers/aws/index.html +# https://www.terraform.io/docs/providers/aws/guides/custom-service-endpoints.html#localstack +provider "aws" { + region = "ap-northeast-1" + + access_key = "dummy" + secret_key = "dummy" + skip_credentials_validation = true + skip_metadata_api_check = true + skip_region_validation = true + skip_requesting_account_id = true + s3_force_path_style = true + + // mock endpoints with localstack + endpoints { + s3 = "%s" + ec2 = "%s" + iam = "%s" + } +} +`, t.Name(), endpoint, endpoint, endpoint, endpoint) + source := ` +resource "aws_security_group" "foo" {} +resource "aws_security_group" "bar" {} +` + workspace := "work1" + backendConfig := []string{"bucket=tfstate-test"} + e := SetupTestAcc(t, source+backend) + terraformCLI := NewTerraformCLI(e) + ctx := context.Background() + + var args = []string{"-input=false", "-no-color"} + for _, b := range backendConfig { + args = append(args, fmt.Sprintf("-backend-config=%s", b)) + } + err := terraformCLI.Init(ctx, args...) + if err != nil { + t.Fatalf("failed to run terraform init: %s", err) + } + + //default workspace always exists so don't try to create it + if workspace != "default" { + err = terraformCLI.WorkspaceNew(ctx, workspace) + if err != nil { + t.Fatalf("failed to run terraform workspace new %s : %s", workspace, err) + } + } + + err = terraformCLI.Apply(ctx, nil, "-input=false", "-no-color", "-auto-approve") + if err != nil { + t.Fatalf("failed to run terraform apply: %s", err) + } + + // destroy resources after each test not to have any state. + t.Cleanup(func() { + err := terraformCLI.Destroy(ctx, "-input=false", "-no-color", "-auto-approve") + if err != nil { + t.Fatalf("failed to run terraform destroy: %s", err) + } + }) + + updatedSource := ` +resource "aws_security_group" "foo2" {} +resource "aws_security_group" "bar" {} +` + UpdateTestAccSource(t, terraformCLI, backend+updatedSource) + + changed, err := terraformCLI.PlanHasChange(context.Background(), nil) + if err != nil { + t.Fatalf("failed to run PlanHasChange: %s", err) + } + if !changed { + t.Fatalf("expect to have changes") + } + + state, err := terraformCLI.StatePull(context.Background()) + if err != nil { + t.Fatalf("failed to run terraform state pull: %s", err) + } + + filename := "_tfexec_override.tf" + if _, err := os.Stat(filepath.Join(terraformCLI.Dir(), filename)); err == nil { + t.Fatalf("an override file already exists: %s", err) + } + + switchBackToRemotekFunc, err := terraformCLI.OverrideBackendToLocal(context.Background(), filename, workspace, false, backendConfig) if err != nil { t.Fatalf("failed to run OverrideBackendToLocal: %s", err) } diff --git a/tfexec/test_helper.go b/tfexec/test_helper.go index ed2437b..895a72c 100644 --- a/tfexec/test_helper.go +++ b/tfexec/test_helper.go @@ -257,7 +257,6 @@ terraform { force_path_style = true } } - # https://www.terraform.io/docs/providers/aws/index.html # https://www.terraform.io/docs/providers/aws/guides/custom-service-endpoints.html#localstack provider "aws" { diff --git a/tfmigrate/config.go b/tfmigrate/config.go index 57d3936..600f34f 100644 --- a/tfmigrate/config.go +++ b/tfmigrate/config.go @@ -30,4 +30,7 @@ type MigratorOption struct { // IsBackendTerraformCloud is a boolean indicating if the remote backend is Terraform Cloud IsBackendTerraformCloud bool + + // BackendConfig is a -backend-config option for remote state + BackendConfig []string } diff --git a/tfmigrate/migrator.go b/tfmigrate/migrator.go index 00c7799..760b473 100644 --- a/tfmigrate/migrator.go +++ b/tfmigrate/migrator.go @@ -22,7 +22,7 @@ type Migrator interface { // setupWorkDir is a common helper function to set up work dir and returns the // current state and a switch back function. -func setupWorkDir(ctx context.Context, tf tfexec.TerraformCLI, workspace string, isBackendTerraformCloud bool) (*tfexec.State, func(), error) { +func setupWorkDir(ctx context.Context, tf tfexec.TerraformCLI, workspace string, isBackendTerraformCloud bool, backendConfig []string) (*tfexec.State, func(), error) { // check if terraform command is available. version, err := tf.Version(ctx) if err != nil { @@ -60,7 +60,7 @@ func setupWorkDir(ctx context.Context, tf tfexec.TerraformCLI, workspace string, } // override backend to local log.Printf("[INFO] [migrator@%s] override backend to local\n", tf.Dir()) - switchBackToRemoteFunc, err := tf.OverrideBackendToLocal(ctx, "_tfmigrate_override.tf", workspace, isBackendTerraformCloud) + switchBackToRemoteFunc, err := tf.OverrideBackendToLocal(ctx, "_tfmigrate_override.tf", workspace, isBackendTerraformCloud, backendConfig) if err != nil { return nil, nil, err } diff --git a/tfmigrate/multi_state_migrator.go b/tfmigrate/multi_state_migrator.go index ecb17f4..682bb52 100644 --- a/tfmigrate/multi_state_migrator.go +++ b/tfmigrate/multi_state_migrator.go @@ -107,14 +107,14 @@ func NewMultiStateMigrator(fromDir string, toDir string, fromWorkspace string, t // the Migrator interface between a single and multi state migrator. func (m *MultiStateMigrator) plan(ctx context.Context) (*tfexec.State, *tfexec.State, error) { // setup fromDir. - fromCurrentState, fromSwitchBackToRemoteFunc, err := setupWorkDir(ctx, m.fromTf, m.fromWorkspace, m.o.IsBackendTerraformCloud) + fromCurrentState, fromSwitchBackToRemoteFunc, err := setupWorkDir(ctx, m.fromTf, m.fromWorkspace, m.o.IsBackendTerraformCloud, m.o.BackendConfig) if err != nil { return nil, nil, err } // switch back it to remote on exit. defer fromSwitchBackToRemoteFunc() // setup toDir. - toCurrentState, toSwitchBackToRemoteFunc, err := setupWorkDir(ctx, m.toTf, m.toWorkspace, m.o.IsBackendTerraformCloud) + toCurrentState, toSwitchBackToRemoteFunc, err := setupWorkDir(ctx, m.toTf, m.toWorkspace, m.o.IsBackendTerraformCloud, m.o.BackendConfig) if err != nil { return nil, nil, err } diff --git a/tfmigrate/state_migrator.go b/tfmigrate/state_migrator.go index c5352cd..b45fb94 100644 --- a/tfmigrate/state_migrator.go +++ b/tfmigrate/state_migrator.go @@ -105,7 +105,7 @@ func NewStateMigrator(dir string, workspace string, actions []StateAction, // the Migrator interface between a single and multi state migrator. func (m *StateMigrator) plan(ctx context.Context) (*tfexec.State, error) { // setup work dir. - currentState, switchBackToRemoteFunc, err := setupWorkDir(ctx, m.tf, m.workspace, m.o.IsBackendTerraformCloud) + currentState, switchBackToRemoteFunc, err := setupWorkDir(ctx, m.tf, m.workspace, m.o.IsBackendTerraformCloud, m.o.BackendConfig) if err != nil { return nil, err }