Add --backend-config cli option to tfmigrate plan/apply

command/apply.go

@@ -12,12 +12,14 @@ import (
 // ApplyCommand is a command which computes a new state and pushes it to the remote state.
 type ApplyCommand struct {
 	Meta
+	backendConfig []string
 }
 
 // Run runs the procedure of this command.
 func (c *ApplyCommand) Run(args []string) int {
 	cmdFlags := flag.NewFlagSet("apply", flag.ContinueOnError)
 	cmdFlags.StringVar(&c.configFile, "config", defaultConfigFile, "A path to tfmigrate config file")
+	cmdFlags.StringArrayVar(&c.backendConfig, "backend-config", nil, "A backend configuration for remote state")
 
 	if err := cmdFlags.Parse(args); err != nil {
 		c.UI.Error(fmt.Sprintf("failed to parse arguments: %s", err))
@@ -32,6 +34,7 @@ func (c *ApplyCommand) Run(args []string) int {
 	log.Printf("[DEBUG] [command] config: %#v\n", c.config)
 
 	c.Option = newOption()
+	c.Option.BackendConfig = c.backendConfig
 	// The option may contain sensitive values such as environment variables.
 	// So logging the option set log level to DEBUG instead of INFO.
 	log.Printf("[DEBUG] [command] option: %#v\n", c.Option)
@@ -105,11 +108,13 @@ Apply computes a new state and pushes it to remote state.
 It will fail if terraform plan detects any diffs with the new state.
 
 Arguments
-  PATH               A path of migration file
-                     Required in non-history mode. Optional in history-mode.
+  PATH                     A path of migration file
+                           Required in non-history mode. Optional in history-mode.
 
 Options:
-  --config           A path to tfmigrate config file
+  --config                 A path to tfmigrate config file
+  --backend-config=path    A backend configuration, a path to backend configuration file or key=value format backend configuraion.
+                           This option is passed to terraform init when switching backend to remote.
 `
 	return strings.TrimSpace(helpText)
 }

command/plan.go

@@ -13,13 +13,15 @@ import (
 // migration operations to a temporary state.
 type PlanCommand struct {
 	Meta
-	out string
+	backendConfig []string
+	out           string
 }
 
 // Run runs the procedure of this command.
 func (c *PlanCommand) Run(args []string) int {
 	cmdFlags := flag.NewFlagSet("plan", flag.ContinueOnError)
 	cmdFlags.StringVar(&c.configFile, "config", defaultConfigFile, "A path to tfmigrate config file")
+	cmdFlags.StringArrayVar(&c.backendConfig, "backend-config", nil, "A backend configuration for remote state")
 	cmdFlags.StringVar(&c.out, "out", "", "[Deprecated] Save a plan file after dry-run migration to the given path")
 
 	if err := cmdFlags.Parse(args); err != nil {
@@ -36,6 +38,7 @@ func (c *PlanCommand) Run(args []string) int {
 
 	c.Option = newOption()
 	c.Option.PlanOut = c.out
+	c.Option.BackendConfig = c.backendConfig
 	// The option may contains sensitive values such as environment variables.
 	// So logging the option set log level to DEBUG instead of INFO.
 	log.Printf("[DEBUG] [command] option: %#v\n", c.Option)
@@ -115,18 +118,20 @@ Plan computes a new state by applying state migration operations to a temporary
 It will fail if terraform plan detects any diffs with the new state.
 
 Arguments:
-  PATH               A path of migration file
-                     Required in non-history mode. Optional in history-mode.
+  PATH                     A path of migration file
+                           Required in non-history mode. Optional in history-mode.
 
 Options:
-  --config           A path to tfmigrate config file
+  --config                 A path to tfmigrate config file
+  --backend-config=path    A backend configuration, a path to backend configuration file or key=value format backend configuraion.
+                           This option is passed to terraform init when switching backend to remote.
 
   [Deprecated]
   --out=path
-                     Save a plan file after dry-run migration to the given path.
-                     Note that applying the plan file only affects a local state,
-                     make sure to force push it to remote after terraform apply.
-                     This option doesn't work with Terraform 1.1+
+                           Save a plan file after dry-run migration to the given path.
+                           Note that applying the plan file only affects a local state,
+                           make sure to force push it to remote after terraform apply.
+                           This option doesn't work with Terraform 1.1+
 `
 	return strings.TrimSpace(helpText)
 }

docker-compose.yml

@@ -17,7 +17,7 @@ services:
       - localstack
 
   localstack:
-    image: localstack/localstack:0.11.3
+    image: localstack/localstack:0.14.4
     ports:
       - "4566:4566"
     environment:
@@ -29,6 +29,6 @@ services:
       - "./scripts/localstack:/docker-entrypoint-initaws.d"  # initialize scripts on startup
 
   dockerize:
-    image: jwilder/dockerize
+    image: powerman/dockerize:0.16.0
     depends_on:
       - localstack

tfexec/terraform.go

@@ -125,7 +125,7 @@ type TerraformCLI interface {
 	// so we need to switch the backend to local for temporary state operations.
 	// The filename argument must meet constraints for override file.
 	// (e.g.) _tfexec_override.tf
-	OverrideBackendToLocal(ctx context.Context, filename string, workspace string, isBackendTerraformCloud bool) (func(), error)
+	OverrideBackendToLocal(ctx context.Context, filename string, workspace string, isBackendTerraformCloud bool, backendConfig []string) (func(), error)
 
 	// PlanHasChange is a helper method which runs plan and return true if the plan has change.
 	PlanHasChange(ctx context.Context, state *State, opts ...string) (bool, error)
@@ -198,7 +198,7 @@ func (c *terraformCLI) SetExecPath(execPath string) {
 // The filename argument must meet constraints in order to override the file.
 // (e.g.) _tfexec_override.tf
 func (c *terraformCLI) OverrideBackendToLocal(ctx context.Context, filename string,
-	workspace string, isBackendTerraformCloud bool) (func(), error) {
+	workspace string, isBackendTerraformCloud bool, backendConfig []string) (func(), error) {
 	// create local backend override file.
 	path := filepath.Join(c.Dir(), filename)
 	contents := `
@@ -254,12 +254,15 @@ terraform {
 		}
 		log.Printf("[INFO] [executor@%s] switch back to remote\n", c.Dir())
 
+		var args = []string{"-input=false", "-no-color", "-reconfigure"}
+		for _, b := range backendConfig {
+			args = append(args, fmt.Sprintf("-backend-config=%s", b))
+		}
 		// Run the correct init command depending on whether the remote backend is Terraform Cloud
 		if !isBackendTerraformCloud {
-			err = c.Init(ctx, "-input=false", "-no-color", "-reconfigure")
-		} else {
-			err = c.Init(ctx, "-input=false", "-no-color")
+			args = append(args, "-reconfigure")
 		}
+		err = c.Init(ctx, args...)
 
 		if err != nil {
 			// we cannot return error here.

tfexec/terraform_init.go

@@ -1,6 +1,8 @@
 package tfexec
 
-import "context"
+import (
+	"context"
+)
 
 // Init initializes the current work directory.
 func (c *terraformCLI) Init(ctx context.Context, opts ...string) error {

tfexec/terraform_test.go

@@ -2,6 +2,7 @@ package tfexec
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
@@ -125,7 +126,155 @@ resource "aws_security_group" "bar" {}
 		t.Fatalf("an override file already exists: %s", err)
 	}
 
-	switchBackToRemotekFunc, err := terraformCLI.OverrideBackendToLocal(context.Background(), filename, workspace, false)
+	switchBackToRemotekFunc, err := terraformCLI.OverrideBackendToLocal(context.Background(), filename, workspace, false, nil)
+	if err != nil {
+		t.Fatalf("failed to run OverrideBackendToLocal: %s", err)
+	}
+
+	if _, err := os.Stat(filepath.Join(terraformCLI.Dir(), filename)); os.IsNotExist(err) {
+		t.Fatalf("the override file does not exist: %s", err)
+	}
+
+	updatedState, _, err := terraformCLI.StateMv(context.Background(), state, nil, "aws_security_group.foo", "aws_security_group.foo2")
+	if err != nil {
+		t.Fatalf("failed to run terraform state mv: %s", err)
+	}
+
+	changed, err = terraformCLI.PlanHasChange(context.Background(), updatedState)
+	if err != nil {
+		t.Fatalf("failed to run PlanHasChange: %s", err)
+	}
+	if changed {
+		t.Fatalf("expect not to have changes")
+	}
+
+	switchBackToRemotekFunc()
+
+	if _, err := os.Stat(filepath.Join(terraformCLI.Dir(), filename)); err == nil {
+		t.Fatalf("the override file wasn't removed: %s", err)
+	}
+
+	changed, err = terraformCLI.PlanHasChange(context.Background(), nil)
+	if err != nil {
+		t.Fatalf("failed to run PlanHasChange: %s", err)
+	}
+	if !changed {
+		t.Fatalf("expect to have changes")
+	}
+}
+
+func TestAccTerraformCLIOverrideBackendToLocalWithBackendConfig(t *testing.T) {
+	SkipUnlessAcceptanceTestEnabled(t)
+
+	endpoint := "http://localhost:4566"
+	localstackEndpoint := os.Getenv("LOCALSTACK_ENDPOINT")
+	if len(localstackEndpoint) > 0 {
+		endpoint = localstackEndpoint
+	}
+
+	backend := fmt.Sprintf(`
+terraform {
+  # https://www.terraform.io/docs/backends/types/s3.html
+  backend "s3" {
+    region = "ap-northeast-1"
+    // bucket = "tfstate-test"
+    key    = "%s/terraform.tfstate"
+
+    // mock s3 endpoint with localstack
+    endpoint                    = "%s"
+    access_key                  = "dummy"
+    secret_key                  = "dummy"
+    skip_credentials_validation = true
+    skip_metadata_api_check     = true
+    force_path_style            = true
+  }
+}
+# https://www.terraform.io/docs/providers/aws/index.html
+# https://www.terraform.io/docs/providers/aws/guides/custom-service-endpoints.html#localstack
+provider "aws" {
+  region = "ap-northeast-1"
+
+  access_key                  = "dummy"
+  secret_key                  = "dummy"
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_region_validation      = true
+  skip_requesting_account_id  = true
+  s3_force_path_style         = true
+
+  // mock endpoints with localstack
+  endpoints {
+    s3  = "%s"
+    ec2 = "%s"
+    iam = "%s"
+  }
+}
+`, t.Name(), endpoint, endpoint, endpoint, endpoint)
+	source := `
+resource "aws_security_group" "foo" {}
+resource "aws_security_group" "bar" {}
+`
+	workspace := "work1"
+	backendConfig := []string{"bucket=tfstate-test"}
+	e := SetupTestAcc(t, source+backend)
+	terraformCLI := NewTerraformCLI(e)
+	ctx := context.Background()
+
+	var args = []string{"-input=false", "-no-color"}
+	for _, b := range backendConfig {
+		args = append(args, fmt.Sprintf("-backend-config=%s", b))
+	}
+	err := terraformCLI.Init(ctx, args...)
+	if err != nil {
+		t.Fatalf("failed to run terraform init: %s", err)
+	}
+
+	//default workspace always exists so don't try to create it
+	if workspace != "default" {
+		err = terraformCLI.WorkspaceNew(ctx, workspace)
+		if err != nil {
+			t.Fatalf("failed to run terraform workspace new %s : %s", workspace, err)
+		}
+	}
+
+	err = terraformCLI.Apply(ctx, nil, "-input=false", "-no-color", "-auto-approve")
+	if err != nil {
+		t.Fatalf("failed to run terraform apply: %s", err)
+	}
+
+	// destroy resources after each test not to have any state.
+	t.Cleanup(func() {
+		err := terraformCLI.Destroy(ctx, "-input=false", "-no-color", "-auto-approve")
+		if err != nil {
+			t.Fatalf("failed to run terraform destroy: %s", err)
+		}
+	})
+
+	updatedSource := `
+resource "aws_security_group" "foo2" {}
+resource "aws_security_group" "bar" {}
+`
+	UpdateTestAccSource(t, terraformCLI, backend+updatedSource)
+
+	changed, err := terraformCLI.PlanHasChange(context.Background(), nil)
+	if err != nil {
+		t.Fatalf("failed to run PlanHasChange: %s", err)
+	}
+	if !changed {
+		t.Fatalf("expect to have changes")
+	}
+
+	state, err := terraformCLI.StatePull(context.Background())
+	if err != nil {
+		t.Fatalf("failed to run terraform state pull: %s", err)
+	}
+
+	filename := "_tfexec_override.tf"
+	if _, err := os.Stat(filepath.Join(terraformCLI.Dir(), filename)); err == nil {
+		t.Fatalf("an override file already exists: %s", err)
+	}
+
+	switchBackToRemotekFunc, err := terraformCLI.OverrideBackendToLocal(context.Background(), filename, workspace, false, backendConfig)
 	if err != nil {
 		t.Fatalf("failed to run OverrideBackendToLocal: %s", err)
 	}

tfexec/test_helper.go

@@ -257,7 +257,6 @@ terraform {
     force_path_style            = true
   }
 }
-
 # https://www.terraform.io/docs/providers/aws/index.html
 # https://www.terraform.io/docs/providers/aws/guides/custom-service-endpoints.html#localstack
 provider "aws" {

tfmigrate/config.go

@@ -30,4 +30,7 @@ type MigratorOption struct {
 
 	// IsBackendTerraformCloud is a boolean indicating if the remote backend is Terraform Cloud
 	IsBackendTerraformCloud bool
+
+	// BackendConfig is a -backend-config option for remote state
+	BackendConfig []string
 }

tfmigrate/migrator.go

@@ -22,7 +22,7 @@ type Migrator interface {
 
 // setupWorkDir is a common helper function to set up work dir and returns the
 // current state and a switch back function.
-func setupWorkDir(ctx context.Context, tf tfexec.TerraformCLI, workspace string, isBackendTerraformCloud bool) (*tfexec.State, func(), error) {
+func setupWorkDir(ctx context.Context, tf tfexec.TerraformCLI, workspace string, isBackendTerraformCloud bool, backendConfig []string) (*tfexec.State, func(), error) {
 	// check if terraform command is available.
 	version, err := tf.Version(ctx)
 	if err != nil {
@@ -60,7 +60,7 @@ func setupWorkDir(ctx context.Context, tf tfexec.TerraformCLI, workspace string,
 	}
 	// override backend to local
 	log.Printf("[INFO] [migrator@%s] override backend to local\n", tf.Dir())
-	switchBackToRemoteFunc, err := tf.OverrideBackendToLocal(ctx, "_tfmigrate_override.tf", workspace, isBackendTerraformCloud)
+	switchBackToRemoteFunc, err := tf.OverrideBackendToLocal(ctx, "_tfmigrate_override.tf", workspace, isBackendTerraformCloud, backendConfig)
 	if err != nil {
 		return nil, nil, err
 	}

tfmigrate/multi_state_migrator.go

@@ -107,14 +107,14 @@ func NewMultiStateMigrator(fromDir string, toDir string, fromWorkspace string, t
 // the Migrator interface between a single and multi state migrator.
 func (m *MultiStateMigrator) plan(ctx context.Context) (*tfexec.State, *tfexec.State, error) {
 	// setup fromDir.
-	fromCurrentState, fromSwitchBackToRemoteFunc, err := setupWorkDir(ctx, m.fromTf, m.fromWorkspace, m.o.IsBackendTerraformCloud)
+	fromCurrentState, fromSwitchBackToRemoteFunc, err := setupWorkDir(ctx, m.fromTf, m.fromWorkspace, m.o.IsBackendTerraformCloud, m.o.BackendConfig)
 	if err != nil {
 		return nil, nil, err
 	}
 	// switch back it to remote on exit.
 	defer fromSwitchBackToRemoteFunc()
 	// setup toDir.
-	toCurrentState, toSwitchBackToRemoteFunc, err := setupWorkDir(ctx, m.toTf, m.toWorkspace, m.o.IsBackendTerraformCloud)
+	toCurrentState, toSwitchBackToRemoteFunc, err := setupWorkDir(ctx, m.toTf, m.toWorkspace, m.o.IsBackendTerraformCloud, m.o.BackendConfig)
 	if err != nil {
 		return nil, nil, err
 	}

tfmigrate/state_migrator.go

@@ -105,7 +105,7 @@ func NewStateMigrator(dir string, workspace string, actions []StateAction,
 // the Migrator interface between a single and multi state migrator.
 func (m *StateMigrator) plan(ctx context.Context) (*tfexec.State, error) {
 	// setup work dir.
-	currentState, switchBackToRemoteFunc, err := setupWorkDir(ctx, m.tf, m.workspace, m.o.IsBackendTerraformCloud)
+	currentState, switchBackToRemoteFunc, err := setupWorkDir(ctx, m.tf, m.workspace, m.o.IsBackendTerraformCloud, m.o.BackendConfig)
 	if err != nil {
 		return nil, err
 	}