Add support for multi_state xmv (wildcard expansion)

README.md

@@ -34,6 +34,7 @@ A Terraform state migration tool for GitOps.
          * [state import](#state-import)
       * [migration block (multi_state)](#migration-block-multi_state)
          * [multi_state mv](#multi_state-mv)
+         * [multi_state xmv](#multi_state-xmv)
    * [Integrations](#integrations)
    * [License](#license)
 <!--te-->
@@ -566,6 +567,7 @@ The `state` migration updates the state in a single directory. It has the follow
 - `workspace` (optional): A terraform workspace. Defaults to "default".
 - `actions` (required): Actions is a list of state action. An action is a plain text for state operation. Valid formats are the following.
   - `"mv <source> <destination>"`
+  - `"xmv <source> <destination>"`
   - `"rm <addresses>...`
   - `"import <address> <id>"`
 - `force` (optional): Apply migrations even if plan show changes
@@ -590,12 +592,10 @@ migration "state" "test" {
 
 #### state xmv
 
-The `xmv` command works like the `mv` command but allows usage of
-wildcards `*` in the source definition. The source expressions will be 
-matched against resources defined in the terraform state. The matched value
-can be used in the destination definition via a dollar sign and their ordinal number. Note that dollar signs need to be
-escaped and therefore are placed twice: 
-`$$1`, `$$2`, ... When there is ambiguity the ordinal number can be put in curly braces (e.g. `$${1}`).
+The `xmv` command works like the `mv` command but allows usage of wildcards `*` in the source definition.
+The source expressions will be matched against resources defined in the terraform state.
+The matched value can be used in the destination definition via a dollar sign and their ordinal number (e.g. `$1`, `$2`, ...).
+When there is ambiguity, you need to put the ordinal number in curly braces, in this case, the dollar sign need to be escaped and therefore are placed twice (e.g. `$${1}`).
 
 For example if `foo` and `bar` in the `mv` command example above are the only 2 security group resources
 defined at the top level then you can rename them using:
@@ -641,6 +641,7 @@ The `multi_state` migration updates states in two different directories. It is i
 - `to_workspace` (optional): A terraform workspace in the TO directory. Defaults to "default".
 - `actions` (required): Actions is a list of multi state action. An action is a plain text for state operation. Valid formats are the following.
   - `"mv <source> <destination>"`
+  - `"xmv <source> <destination>"`
 - `force` (optional): Apply migrations even if plan show changes
 
 Note that `from_dir` and `to_dir` are relative path to the current working directory where `tfmigrate` command is invoked.
@@ -660,6 +661,34 @@ migration "multi_state" "mv_dir1_dir2" {
 }
 ```
 
+#### multi_state xmv
+
+The `xmv` command works like the `mv` command but allows usage of
+wildcards `*` in the source definition.
+The wildcard expansion rules are the same as for the single state xmv.
+
+```hcl
+migration "multi_state" "mv_dir1_dir2" {
+  from_dir = "dir1"
+  to_dir   = "dir2"
+  actions = [
+    "xmv aws_security_group.* aws_security_group.$${1}2",
+  ]
+}
+```
+
+If you want to move all resources to another dir for merging two tfstates, you can write something like this:
+
+```hcl
+migration "multi_state" "merge_dir1_to_dir2" {
+  from_dir = "dir1"
+  to_dir   = "dir2"
+  actions = [
+    "xmv * $1",
+  ]
+}
+```
+
 ## Integrations
 
 You can integrate tfmigrate with your favorite CI/CD services. Examples are as follows:

config/migration_test.go

@@ -81,6 +81,48 @@ migration "state" "test" {
 			},
 			ok: true,
 		},
+		{
+			desc: "state with a simple wildcard action",
+			source: `
+migration "state" "test" {
+	actions = [
+		"xmv null_resource.* null_resource.new_$1",
+	]
+}
+`,
+			want: &tfmigrate.MigrationConfig{
+				Type: "state",
+				Name: "test",
+				Migrator: &tfmigrate.StateMigratorConfig{
+					Dir: "",
+					Actions: []string{
+						"xmv null_resource.* null_resource.new_$1",
+					},
+				},
+			},
+			ok: true,
+		},
+		{
+			desc: "state with a escaped wildcard action",
+			source: `
+migration "state" "test" {
+	actions = [
+		"xmv null_resource.* null_resource.$${1}2",
+	]
+}
+`,
+			want: &tfmigrate.MigrationConfig{
+				Type: "state",
+				Name: "test",
+				Migrator: &tfmigrate.StateMigratorConfig{
+					Dir: "",
+					Actions: []string{
+						"xmv null_resource.* null_resource.${1}2",
+					},
+				},
+			},
+			ok: true,
+		},
 		{
 			desc: "state without actions",
 			source: `

tfmigrate/multi_state_action.go

@@ -20,6 +20,7 @@ type MultiStateAction interface {
 // This method is useful to build an action from terraform state command.
 // Valid formats are the following.
 // "mv <source> <destination>"
+// "xmv <source> <destination>"
 func NewMultiStateActionFromString(cmdStr string) (MultiStateAction, error) {
 	args, err := splitStateAction(cmdStr)
 	if err != nil {
@@ -42,6 +43,14 @@ func NewMultiStateActionFromString(cmdStr string) (MultiStateAction, error) {
 		dst := args[2]
 		action = NewMultiStateMvAction(src, dst)
 
+	case "xmv":
+		if len(args) != 3 {
+			return nil, fmt.Errorf("multi state xmv action is invalid: %s", cmdStr)
+		}
+		src := args[1]
+		dst := args[2]
+		action = NewMultiStateXmvAction(src, dst)
+
 	default:
 		return nil, fmt.Errorf("unknown multi state action type: %s", cmdStr)
 	}

tfmigrate/multi_state_action_test.go

@@ -39,6 +39,33 @@ func TestNewMultiStateActionFromString(t *testing.T) {
 			want:   nil,
 			ok:     false,
 		},
+		{
+			desc:   "xmv action (valid)",
+			cmdStr: "xmv null_resource.* null_resource.$1",
+			want: &MultiStateXmvAction{
+				source:      "null_resource.*",
+				destination: "null_resource.$1",
+			},
+			ok: true,
+		},
+		{
+			desc:   "xmv action (no args)",
+			cmdStr: "xmv",
+			want:   nil,
+			ok:     false,
+		},
+		{
+			desc:   "xmv action (1 arg)",
+			cmdStr: "xmv null_resource.foo",
+			want:   nil,
+			ok:     false,
+		},
+		{
+			desc:   "xmv action (3 args)",
+			cmdStr: "xmv null_resource.foo null_resource.foo2  null_resource.foo3",
+			want:   nil,
+			ok:     false,
+		},
 		{
 			desc:   "duplicated white spaces",
 			cmdStr: " mv  null_resource.foo    null_resource.foo2 ",

tfmigrate/multi_state_mv_action.go

@@ -12,7 +12,7 @@ import (
 type MultiStateMvAction struct {
 	// source is an address of resource or module to be moved.
 	source string
-	// // destination is a new address of resource or module to move.
+	// destination is a new address of resource or module to move.
 	destination string
 }
 

tfmigrate/multi_state_mv_action_test.go

@@ -0,0 +1,72 @@
+package tfmigrate
+
+import (
+	"context"
+	"testing"
+
+	"github.com/minamijoyo/tfmigrate/tfexec"
+)
+
+func TestAccMultiStateMvAction(t *testing.T) {
+	tfexec.SkipUnlessAcceptanceTestEnabled(t)
+	ctx := context.Background()
+
+	// setup the initial files and states
+	fromBackend := tfexec.GetTestAccBackendS3Config(t.Name() + "/fromDir")
+	fromSource := `
+resource "null_resource" "foo" {}
+resource "null_resource" "bar" {}
+resource "null_resource" "baz" {}
+`
+	fromWorkspace := "default"
+	fromTf := tfexec.SetupTestAccWithApply(t, fromWorkspace, fromBackend+fromSource)
+
+	toBackend := tfexec.GetTestAccBackendS3Config(t.Name() + "/toDir")
+	toSource := `
+resource "null_resource" "qux" {}
+`
+	toWorkspace := "default"
+	toTf := tfexec.SetupTestAccWithApply(t, toWorkspace, toBackend+toSource)
+
+	// update terraform resource files for migration
+	fromUpdatedSource := `
+resource "null_resource" "baz" {}
+`
+	tfexec.UpdateTestAccSource(t, fromTf, fromBackend+fromUpdatedSource)
+
+	toUpdatedSource := `
+resource "null_resource" "foo" {}
+resource "null_resource" "bar2" {}
+resource "null_resource" "qux" {}
+`
+	tfexec.UpdateTestAccSource(t, toTf, toBackend+toUpdatedSource)
+
+	fromChanged, err := fromTf.PlanHasChange(ctx, nil)
+	if err != nil {
+		t.Fatalf("failed to run PlanHasChange in fromDir: %s", err)
+	}
+	if !fromChanged {
+		t.Fatalf("expect to have changes in fromDir")
+	}
+
+	toChanged, err := toTf.PlanHasChange(ctx, nil)
+	if err != nil {
+		t.Fatalf("failed to run PlanHasChange in toDir: %s", err)
+	}
+	if !toChanged {
+		t.Fatalf("expect to have changes in toDir")
+	}
+
+	// perform state migration
+	actions := []MultiStateAction{
+		NewMultiStateMvAction("null_resource.foo", "null_resource.foo"),
+		NewMultiStateMvAction("null_resource.bar", "null_resource.bar2"),
+	}
+	o := &MigratorOption{}
+	force := false
+	m := NewMultiStateMigrator(fromTf.Dir(), toTf.Dir(), fromWorkspace, toWorkspace, actions, o, force)
+	err = m.Plan(ctx)
+	if err != nil {
+		t.Fatalf("failed to run migrator plan: %s", err)
+	}
+}

tfmigrate/multi_state_xmv_action.go

@@ -0,0 +1,72 @@
+package tfmigrate
+
+import (
+	"context"
+
+	"github.com/minamijoyo/tfmigrate/tfexec"
+)
+
+// MultiStateXmvAction implements the MultiStateAction interface.
+// MultiStateXmvAction is an extended version of MultiStateMvAction.
+// It allows you to move multiple resouces with wildcard matching.
+type MultiStateXmvAction struct {
+	// source is a address of resource or module to be moved which can contain wildcards.
+	source string
+	// destination is a new address of resource or module to move which can contain placeholders.
+	destination string
+}
+
+var _ MultiStateAction = (*MultiStateXmvAction)(nil)
+
+// NewMultiStateXmvAction returns a new MultiStateXmvAction instance.
+func NewMultiStateXmvAction(source string, destination string) *MultiStateXmvAction {
+	return &MultiStateXmvAction{
+		source:      source,
+		destination: destination,
+	}
+}
+
+// MultiStateUpdate updates given two states and returns new two states.
+// It moves a resource from a dir to another.
+// It also can rename an address of resource.
+func (a *MultiStateXmvAction) MultiStateUpdate(ctx context.Context, fromTf tfexec.TerraformCLI, toTf tfexec.TerraformCLI, fromState *tfexec.State, toState *tfexec.State) (*tfexec.State, *tfexec.State, error) {
+	multiStateMvActions, err := a.generateMvActions(ctx, fromTf, fromState)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for _, action := range multiStateMvActions {
+		fromState, toState, err = action.MultiStateUpdate(ctx, fromTf, toTf, fromState, toState)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+	return fromState, toState, nil
+}
+
+// generateMvActions uses an xmv and use the state to determine the corresponding mv actions.
+func (a *MultiStateXmvAction) generateMvActions(ctx context.Context, fromTf tfexec.TerraformCLI, fromState *tfexec.State) ([]*MultiStateMvAction, error) {
+	stateList, err := fromTf.StateList(ctx, fromState, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// create a temporary single state mv actions.
+	// It may look a bit strange as a type.
+	// This is only because sharing the logic while maintaining consistency.
+	stateXmv := NewStateXmvAction(a.source, a.destination)
+
+	e := newXmvExpander(stateXmv)
+	stateMvActions, err := e.expand(stateList)
+	if err != nil {
+		return nil, err
+	}
+
+	// convert StateMvAction to MultiStateMvAction.
+	multiStateMvActions := []*MultiStateMvAction{}
+	for _, action := range stateMvActions {
+		multiStateMvActions = append(multiStateMvActions, NewMultiStateMvAction(action.source, action.destination))
+	}
+
+	return multiStateMvActions, nil
+}