From b794b083c74c5c2343686f112456b7a517d3a019 Mon Sep 17 00:00:00 2001 From: magodo Date: Wed, 29 Mar 2023 10:03:52 +0800 Subject: [PATCH] Integrate with terraform-client-go for hcl only mode to eliminate `terraform` binary (#381) * Integrate with terraform-client-go for hcl only mode to eliminate `terraform` binary Integrate with terraform-client-go for hcl only mode to eliminate `terraform` binary. This improves the performance (quicker speed and lower footprint) of importing. To use it, users have to explicitly specify a new hidden flag (`--tfclient-plugin-go`). This flag is hidden since it is not supposed for CLI users to use, but is meant to be used by module users who wants to get the benefit of performance improvement. --- command_before_func.go | 5 + flag.go | 11 +- go.mod | 35 +- go.sum | 82 ++-- internal/meta/base_meta.go | 445 ++++++++++++------- internal/meta/importlist.go | 4 + internal/test/resourcegroup/hcl_only_test.go | 141 ++++++ internal/test/utils.go | 1 + internal/utils/copyfile.go | 2 +- main.go | 56 +++ pkg/config/config.go | 4 + 11 files changed, 593 insertions(+), 193 deletions(-) create mode 100644 internal/test/resourcegroup/hcl_only_test.go diff --git a/command_before_func.go b/command_before_func.go index 712d93b..0a2e017 100644 --- a/command_before_func.go +++ b/command_before_func.go @@ -45,6 +45,11 @@ func commandBeforeFunc(fset *FlagSet) func(ctx *cli.Context) error { return fmt.Errorf("`--dev-provider` conflicts with `--provider-version`") } } + if fset.hflagTFClientPluginPath != "" { + if !fset.flagHCLOnly { + return fmt.Errorf("`--tfclient-plugin-path` must be used together with `--hcl-only`") + } + } if flagLogLevel != "" { if _, err := logLevel(flagLogLevel); err != nil { diff --git a/flag.go b/flag.go index bdf0c6c..82a47c6 100644 --- a/flag.go +++ b/flag.go @@ -29,9 +29,10 @@ type FlagSet struct { flagModulePath string // common flags (hidden) - hflagMockClient bool - hflagPlainUI bool - hflagProfile string + hflagMockClient bool + hflagPlainUI bool + hflagProfile string + hflagTFClientPluginPath string // Subcommand specific flags // @@ -74,7 +75,6 @@ func (flag FlagSet) DescribeCLI(mode string) string { if flag.flagEnv != "" { args = append(args, "--env="+flag.flagEnv) } - if flag.flagOverwrite { args = append(args, "--overwrite=true") } @@ -105,6 +105,9 @@ func (flag FlagSet) DescribeCLI(mode string) string { if flag.flagHCLOnly { args = append(args, "--hcl-only=true") } + if flag.hflagTFClientPluginPath != "" { + args = append(args, "--tfclient-plugin-path=%s", flag.hflagTFClientPluginPath) + } if flag.flagModulePath != "" { args = append(args, "--module-path="+flag.flagModulePath) } diff --git a/go.mod b/go.mod index 68c7a8f..e8dd6c1 100644 --- a/go.mod +++ b/go.mod @@ -17,14 +17,17 @@ require ( github.com/hashicorp/hcl/v2 v2.13.0 github.com/hashicorp/terraform-config-inspect v0.0.0-20221020162138-81db043ad408 github.com/hashicorp/terraform-exec v0.17.2 + github.com/hashicorp/terraform-json v0.16.0 github.com/hexops/gotextdiff v1.0.3 github.com/magodo/armid v0.0.0-20220923023118-aec41eaf7370 github.com/magodo/azlist v0.0.0-20230129022211-862464772b00 github.com/magodo/aztft v0.3.1-0.20230303055806-23bd5ef27605 github.com/magodo/spinner v0.0.0-20220720073946-50f31b2dc5a6 + github.com/magodo/terraform-client-go v0.0.0-20230323074119-02ceb732dd25 github.com/magodo/textinput v0.0.0-20210913072708-7d24f2b4b0c0 - github.com/magodo/tfadd v0.10.1-0.20230303033832-99f935d1f92e + github.com/magodo/tfadd v0.10.1-0.20230323091655-a101eda67724 github.com/magodo/tfmerge v0.0.0-20221214062955-f52e46d03402 + github.com/magodo/tfstate v0.0.0-20220409052014-9b9568dda918 github.com/magodo/workerpool v0.0.0-20230119025400-40192d2716ea github.com/microsoft/ApplicationInsights-Go v0.4.4 github.com/mitchellh/go-wordwrap v1.0.0 @@ -34,7 +37,7 @@ require ( github.com/tidwall/gjson v1.14.2 github.com/tidwall/sjson v1.2.5 github.com/urfave/cli/v2 v2.24.1 - github.com/zclconf/go-cty v1.11.0 + github.com/zclconf/go-cty v1.13.0 ) require ( @@ -84,24 +87,28 @@ require ( github.com/fatih/color v1.13.0 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/golang-jwt/jwt/v4 v4.4.2 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect - github.com/google/uuid v1.1.2 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-plugin v1.4.8 // indirect github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f // indirect - github.com/hashicorp/terraform-json v0.14.0 // indirect + github.com/hashicorp/terraform-plugin-go v0.14.3 // indirect + github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magodo/tfpluginschema v0.0.0-20220905090502-2d6a05ebaefd // indirect - github.com/magodo/tfstate v0.0.0-20220409052014-9b9568dda918 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.11.1-0.20220212125758-44cd13922739 // indirect + github.com/oklog/run v1.0.0 // indirect github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect @@ -109,11 +116,19 @@ require ( github.com/sahilm/fuzzy v0.1.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect + github.com/vmihailenco/msgpack/v4 v4.3.12 // indirect + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser v0.1.1 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167 // indirect - golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect - golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/net v0.5.0 // indirect + golang.org/x/sys v0.4.0 // indirect + golang.org/x/term v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/grpc v1.53.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8d70ac2..4d8c64b 100644 --- a/go.sum +++ b/go.sum @@ -91,7 +91,6 @@ github.com/agext/levenshtein v1.2.2 h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXva github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/apparentlymart/go-dump v0.0.0-20190214190832-042adf3cf4a0 h1:MzVXffFUye+ZcSR6opIgz9Co7WcDx6ZcY+RjfFHoA0I= -github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= @@ -140,18 +139,21 @@ github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-checkpoint v0.5.0 h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3mlgcqk5mU= @@ -163,6 +165,8 @@ github.com/hashicorp/go-hclog v1.3.1 h1:vDwF1DFNZhntP4DAjuTpOw3uEgMUpXh1pB5fW9Dq github.com/hashicorp/go-hclog v1.3.1/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-plugin v1.4.8 h1:CHGwpxYDOttQOY7HOWgETU9dyVjOXzniXDqJcYJE1zM= +github.com/hashicorp/go-plugin v1.4.8/go.mod h1:viDMjcLJuDui6pXb8U4HVfb8AamCWhHGUjr2IrTF67s= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -178,8 +182,12 @@ github.com/hashicorp/terraform-config-inspect v0.0.0-20221020162138-81db043ad408 github.com/hashicorp/terraform-config-inspect v0.0.0-20221020162138-81db043ad408/go.mod h1:EAaqp5h9PsUNr6NtgLj31w+ElcCEL+1Svw1Jw+MTVKU= github.com/hashicorp/terraform-exec v0.17.2 h1:EU7i3Fh7vDUI9nNRdMATCEfnm9axzTnad8zszYZ73Go= github.com/hashicorp/terraform-exec v0.17.2/go.mod h1:tuIbsL2l4MlwwIZx9HPM+LOV9vVyEfBYu2GsO1uH3/8= -github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e17dKDpqV7s= -github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM= +github.com/hashicorp/terraform-json v0.16.0 h1:UKkeWRWb23do5LNAFlh/K3N0ymn1qTOO8c+85Albo3s= +github.com/hashicorp/terraform-json v0.16.0/go.mod h1:v0Ufk9jJnk6tcIZvScHvetlKfiNTC+WS21mnXIlc0B0= +github.com/hashicorp/terraform-plugin-go v0.14.3 h1:nlnJ1GXKdMwsC8g1Nh05tK2wsC3+3BL/DBBxFEki+j0= +github.com/hashicorp/terraform-plugin-go v0.14.3/go.mod h1:7ees7DMZ263q8wQ6E4RdIdR6nHHJtrdt4ogX5lPkX1A= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -189,6 +197,7 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -199,7 +208,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= @@ -212,10 +220,12 @@ github.com/magodo/aztft v0.3.1-0.20230303055806-23bd5ef27605 h1:93NI5zq4d0Rzhzjs github.com/magodo/aztft v0.3.1-0.20230303055806-23bd5ef27605/go.mod h1:DhhHb0CgN8ypYt5eRf4Px1QdvGaCwUUqdlDsyE9d9D0= github.com/magodo/spinner v0.0.0-20220720073946-50f31b2dc5a6 h1:CElHO4hPXC+Eivy8sUC/WrnH3jmQzdF2x0lEXBEYul8= github.com/magodo/spinner v0.0.0-20220720073946-50f31b2dc5a6/go.mod h1:Cn4fFwFH/Ddw9sjWPeSS72bNaxbM+FRXf7pkGEDReq8= +github.com/magodo/terraform-client-go v0.0.0-20230323074119-02ceb732dd25 h1:V4R1wcjD/fYQh3Qx/xUyB8xTZgJ7P+WGtHqYpjs+mTU= +github.com/magodo/terraform-client-go v0.0.0-20230323074119-02ceb732dd25/go.mod h1:L12osIvZuDH0/UzrWn3+kiBRXDFTuoYaqF7UfTsbbQA= github.com/magodo/textinput v0.0.0-20210913072708-7d24f2b4b0c0 h1:aNtr4iNv/tex2t8W1u3scAoNHEnFlTKhNNHOpYStqbs= github.com/magodo/textinput v0.0.0-20210913072708-7d24f2b4b0c0/go.mod h1:MqYhNP+PC386Bjsx5piZe7T4vDm5QIPv8b1RU0prVnU= -github.com/magodo/tfadd v0.10.1-0.20230303033832-99f935d1f92e h1:GGJFGRh59w5vnrlFQtYRfkeGYmQptTdzPkSY7sAWpCk= -github.com/magodo/tfadd v0.10.1-0.20230303033832-99f935d1f92e/go.mod h1:DbJnYhmAkyjNnVt819cytTfJMtv5DpqV9MkHuUbyv3c= +github.com/magodo/tfadd v0.10.1-0.20230323091655-a101eda67724 h1:8ElYr1s7dgUUWqIGPdT0aMLGR65zofw4gqO4PXmoXfg= +github.com/magodo/tfadd v0.10.1-0.20230323091655-a101eda67724/go.mod h1:DbJnYhmAkyjNnVt819cytTfJMtv5DpqV9MkHuUbyv3c= github.com/magodo/tfmerge v0.0.0-20221214062955-f52e46d03402 h1:RyaR4VE7hoR9AyoVH414cpM8V63H4rLe2aZyKdoDV1w= github.com/magodo/tfmerge v0.0.0-20221214062955-f52e46d03402/go.mod h1:ssV++b4DH33rsD592bvpS4Peng3ZfdGNZbFgCDkCfj8= github.com/magodo/tfpluginschema v0.0.0-20220905090502-2d6a05ebaefd h1:L0kTduNwpx60EdBPYOVF9oUY7jdfZHIncvQN490qWd4= @@ -240,12 +250,12 @@ github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U= -github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b h1:1XF24mVaiu7u+CFywTdcDo2ie1pzzhwjt6RHqzpMU34= github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= github.com/muesli/cancelreader v0.2.0/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= @@ -258,6 +268,8 @@ github.com/muesli/termenv v0.11.1-0.20220204035834-5ac8409525e0/go.mod h1:Bd5NYQ github.com/muesli/termenv v0.11.1-0.20220212125758-44cd13922739 h1:QANkGiGr39l1EESqrE0gZw0/AJNYzIvoGLhIoVYtluI= github.com/muesli/termenv v0.11.1-0.20220212125758-44cd13922739/go.mod h1:Bd5NYQ7pd+SrtBSrSNoBBmXlcY8+Xj4BMJgh8qcZrvs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -276,7 +288,6 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI= github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= -github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -285,8 +296,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= @@ -304,18 +315,20 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/urfave/cli/v2 v2.24.1 h1:/QYYr7g0EhwXEML8jO+8OYt5trPnLHS0p3mrgExJ5NU= github.com/urfave/cli/v2 v2.24.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= -github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37wVyIuWY= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= -github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= -github.com/zclconf/go-cty v1.11.0 h1:726SxLdi2SDnjY+BStqB9J1hNp4+2WlzyXLuimibIe0= -github.com/zclconf/go-cty v1.11.0/go.mod h1:s9IfD1LK5ccNMSWCVFCE2rJfHiZgi7JijgeWIMfhLvA= -github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= +github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0= +github.com/zclconf/go-cty v1.13.0/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -323,14 +336,13 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167 h1:O8uGbHCqlTp2P6QJSLmCojM4mN6UemYv8K+dCnmHmu0= golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -352,20 +364,32 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/meta/base_meta.go b/internal/meta/base_meta.go index 18303bf..e5adecf 100644 --- a/internal/meta/base_meta.go +++ b/internal/meta/base_meta.go @@ -4,11 +4,14 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "os" "path/filepath" "strings" + tfjson "github.com/hashicorp/terraform-json" + "github.com/Azure/aztfexport/pkg/config" "github.com/Azure/aztfexport/pkg/log" "github.com/zclconf/go-cty/cty" @@ -27,10 +30,15 @@ import ( "github.com/hexops/gotextdiff" "github.com/hexops/gotextdiff/myers" "github.com/hexops/gotextdiff/span" + tfclient "github.com/magodo/terraform-client-go/tfclient" + "github.com/magodo/terraform-client-go/tfclient/configschema" + "github.com/magodo/terraform-client-go/tfclient/typ" "github.com/magodo/tfadd/providers/azurerm" "github.com/magodo/tfadd/tfadd" "github.com/magodo/tfmerge/tfmerge" + "github.com/magodo/tfstate" "github.com/magodo/workerpool" + ctyjson "github.com/zclconf/go-cty/cty/json" ) const ResourceMappingFileName = "aztfexportResourceMapping.json" @@ -80,7 +88,9 @@ type baseMeta struct { providerConfig map[string]cty.Value fullConfig bool parallelism int - hclOnly bool + + hclOnly bool + tfclient tfclient.Client // The module address prefix in the resource addr. E.g. module.mod1.module.mod2.azurerm_resource_group.test. // This is an empty string if module path is not specified. @@ -110,44 +120,29 @@ func NewBaseMeta(cfg config.CommonConfig) (*baseMeta, error) { if cfg.ProviderVersion != "" && cfg.DevProvider { return nil, fmt.Errorf("ProviderVersion conflicts with DevProvider in the config") } + if cfg.TFClient != nil && !cfg.HCLOnly { + return nil, fmt.Errorf("TFClient must be used together with HCLOnly") + } // Determine the module directory and module address var ( - modulePaths []string - moduleAddr string - moduleDir = cfg.OutputDir + moduleAddr string + moduleDir = cfg.OutputDir ) if cfg.ModulePath != "" { - modulePaths = strings.Split(cfg.ModulePath, ".") + modulePaths := strings.Split(cfg.ModulePath, ".") + // Resolve the Terraform module address var segs []string for _, moduleName := range modulePaths { segs = append(segs, "module."+moduleName) } moduleAddr = strings.Join(segs, ".") - // Ensure the module path is something called by the main module - // We are following the module source and recursively call the LoadModule below. This is valid since we only support local path modules. - // (remote sources are not supported since we will end up generating config to that module, it only makes sense for local path modules) - module, err := tfconfig.LoadModule(moduleDir) + var err error + moduleDir, err = getModuleDir(modulePaths, cfg.OutputDir) if err != nil { - return nil, fmt.Errorf("loading main module: %v", err) - } - - for i, moduleName := range modulePaths { - mc := module.ModuleCalls[moduleName] - if mc == nil { - return nil, fmt.Errorf("no module %q invoked by the root module", strings.Join(modulePaths[:i+1], ".")) - } - // See https://developer.hashicorp.com/terraform/language/modules/sources#local-paths - if !strings.HasPrefix(mc.Source, "./") && !strings.HasPrefix(mc.Source, "../") { - return nil, fmt.Errorf("the source of module %q is not a local path", strings.Join(modulePaths[:i+1], ".")) - } - moduleDir = filepath.Join(moduleDir, mc.Source) - module, err = tfconfig.LoadModule(moduleDir) - if err != nil { - return nil, fmt.Errorf("loading module %q: %v", strings.Join(modulePaths[:i+1], "."), err) - } + return nil, err } } @@ -208,6 +203,7 @@ func NewBaseMeta(cfg config.CommonConfig) (*baseMeta, error) { fullConfig: cfg.FullConfig, parallelism: cfg.Parallelism, hclOnly: cfg.HCLOnly, + tfclient: cfg.TFClient, moduleAddr: moduleAddr, moduleDir: moduleDir, @@ -225,80 +221,31 @@ func (meta baseMeta) Workspace() string { func (meta *baseMeta) Init(ctx context.Context) error { meta.tc.Trace(telemetry.Info, "Init Enter") defer meta.tc.Trace(telemetry.Info, "Init Leave") - // Create the import directories per parallelism - var importBaseDirs []string - var importModuleDirs []string - modulePaths := []string{} - for i, v := range strings.Split(meta.moduleAddr, ".") { - if i%2 == 1 { - modulePaths = append(modulePaths, v) - } - } - for i := 0; i < meta.parallelism; i++ { - dir, err := os.MkdirTemp("", "aztfexport-") - if err != nil { - return fmt.Errorf("creating import directory: %v", err) - } - // Creating the module hierarchy if module path is specified. - // The hierarchy used here is not necessarily to be the same as is defined. What we need to guarantee here is the module address in TF is as specified. - mdir := dir - for _, moduleName := range modulePaths { - fpath := filepath.Join(mdir, "main.tf") - // #nosec G306 - if err := os.WriteFile(fpath, []byte(fmt.Sprintf(`module "%s" { - source = "./%s" -} -`, moduleName, moduleName)), 0644); err != nil { - return fmt.Errorf("creating %s: %v", fpath, err) - } - - mdir = filepath.Join(mdir, moduleName) - // #nosec G301 - if err := os.Mkdir(mdir, 0750); err != nil { - return fmt.Errorf("creating module dir %s: %v", mdir, err) - } - } - - importModuleDirs = append(importModuleDirs, mdir) - importBaseDirs = append(importBaseDirs, dir) - } - meta.importBaseDirs = importBaseDirs - meta.importModuleDirs = importModuleDirs - - // Init terraform - if err := meta.initTF(ctx); err != nil { - return err + if meta.tfclient != nil { + return meta.init_notf(ctx) } - // Init provider - if err := meta.initProvider(ctx); err != nil { - return err - } - - // Pull TF state - baseState, err := meta.tf.StatePull(ctx) - if err != nil { - return fmt.Errorf("failed to pull state: %v", err) - } - meta.baseState = []byte(baseState) - meta.originBaseState = []byte(baseState) - - return nil + return meta.init_tf(ctx) } -func (meta baseMeta) DeInit(_ context.Context) error { +func (meta baseMeta) DeInit(ctx context.Context) error { meta.tc.Trace(telemetry.Info, "DeInit Enter") defer meta.tc.Trace(telemetry.Info, "DeInit Leave") - // Clean up the temporary workspaces for parallel import - for _, dir := range meta.importBaseDirs { - // #nosec G104 - os.RemoveAll(dir) + + if meta.tfclient != nil { + return meta.deinit_notf(ctx) } - return nil + + return meta.deinit_tf(ctx) } func (meta *baseMeta) CleanTFState(ctx context.Context, addr string) { + // Noop if tfclient is set + if meta.tfclient != nil { + return + } + // #nosec G104 meta.tf.StateRm(ctx, addr) } @@ -314,11 +261,14 @@ func (meta *baseMeta) ParallelImport(ctx context.Context, items []*ImportItem) e wp := workerpool.NewWorkPool(meta.parallelism) - var thisBaseStateJSON map[string]interface{} - wp.Run(func(i interface{}) error { idx := i.(int) + // Noop if tfclient is set + if meta.tfclient != nil { + return nil + } + stateFile := filepath.Join(meta.importBaseDirs[idx], "terraform.tfstate") // Don't merge state file if this import dir doesn't contain state file, which can because either this import dir imported nothing, or it encountered import error @@ -328,38 +278,13 @@ func (meta *baseMeta) ParallelImport(ctx context.Context, items []*ImportItem) e // Ensure the state file is removed after this round import, preparing for the next round. defer os.Remove(stateFile) - // Performance improvement. - // In case there is no TF state in the target workspace (no matter local/remote backend), we can avoid using tfmerge (which takes care of terraform internals, like keeping the lineage, etc). - // As long as the user ensure there is no address conflicts in the import list (which is always the case as the resource names are almost unique), - // We are updating the local thisBaseStateJSON here, will update it to the meta.baseState at the end of this function. - if len(meta.originBaseState) == 0 { - log.Printf("[DEBUG] Merging terraform state file %s (simple)", stateFile) - // #nosec G304 - b, err := os.ReadFile(stateFile) - if err != nil { - return fmt.Errorf("failed to read state file: %v", err) - } - var stateJSON map[string]interface{} - if err := json.Unmarshal(b, &stateJSON); err != nil { - return fmt.Errorf("failed to unmarshal state file: %v", err) - } - // The first state file to be merged will be took as the base state. - if thisBaseStateJSON == nil { - thisBaseStateJSON = stateJSON - return nil - } - // The other merges will simply append the "resources". - thisBaseStateJSON["resources"] = append(thisBaseStateJSON["resources"].([]interface{}), stateJSON["resources"].([]interface{})...) - return nil - } - - // Otherwise, we use tfmerge to move resources from the importing state file to the target base state file. log.Printf("[DEBUG] Merging terraform state file %s (tfmerge)", stateFile) newState, err := tfmerge.Merge(ctx, meta.tf, meta.baseState, stateFile) if err != nil { return fmt.Errorf("failed to merge state file: %v", err) } meta.baseState = newState + return nil }) @@ -378,30 +303,18 @@ func (meta *baseMeta) ParallelImport(ctx context.Context, items []*ImportItem) e return err } - if thisBaseStateJSON != nil { - var baseStateJSON map[string]interface{} - if len(meta.baseState) == 0 { - baseStateJSON = thisBaseStateJSON - } else { - if err := json.Unmarshal(meta.baseState, &baseStateJSON); err != nil { - return fmt.Errorf("unmarshalling the base state at the end of import: %v", err) - } - baseStateJSON["resources"] = append(baseStateJSON["resources"].([]interface{}), thisBaseStateJSON["resources"].([]interface{})...) - } - var err error - meta.baseState, err = json.Marshal(baseStateJSON) - if err != nil { - return fmt.Errorf("marshalling the base state at the end of import: %v", err) - } - return nil - } - return nil } func (meta baseMeta) PushState(ctx context.Context) error { meta.tc.Trace(telemetry.Info, "PushState Enter") defer meta.tc.Trace(telemetry.Info, "PushState Leave") + + // Noop if tfclient is set + if meta.tfclient != nil { + return nil + } + // Don't push state if there is no state to push. This might happen when all the resources failed to import with "--continue". if len(meta.baseState) == 0 { return nil @@ -493,8 +406,9 @@ func (meta baseMeta) ExportSkippedResources(_ context.Context, l ImportList) err } func (meta baseMeta) CleanUpWorkspace(_ context.Context) error { - // Clean up everything under the output directory, except for the TF code. - if meta.hclOnly { + // For hcl only mode with using terraform binary, we will have to clean up everything under the output directory, + // except for the TF code, resource mapping file and ignore list file. + if meta.hclOnly && meta.tfclient == nil { tmpDir, err := os.MkdirTemp("", "") if err != nil { return err @@ -519,7 +433,9 @@ func (meta baseMeta) CleanUpWorkspace(_ context.Context) error { return err } if err := utils.CopyFile(filepath.Join(meta.outdir, SkippedResourcesFileName), tmpSkippedResourcesFileName); err != nil { - return err + if !errors.Is(err, os.ErrNotExist) { + return err + } } if err := utils.RemoveEverythingUnder(meta.outdir); err != nil { @@ -536,7 +452,9 @@ func (meta baseMeta) CleanUpWorkspace(_ context.Context) error { return err } if err := utils.CopyFile(tmpSkippedResourcesFileName, filepath.Join(meta.outdir, SkippedResourcesFileName)); err != nil { - return err + if !errors.Is(err, os.ErrNotExist) { + return err + } } } @@ -602,6 +520,107 @@ func (meta *baseMeta) buildProviderConfig() string { return string(f.Bytes()) } +func (meta *baseMeta) init_notf(ctx context.Context) error { + schResp, diags := meta.tfclient.GetProviderSchema() + if diags.HasErrors() { + return fmt.Errorf("getting provider schema: %v", diags) + } + + // Ensure "features" is always defined in the provider config + providerConfig := map[string]cty.Value{ + "features": cty.ListValEmpty(configschema.SchemaBlockImpliedType(schResp.Provider.Block.NestedBlocks["features"].Block)), + } + for k, v := range meta.providerConfig { + providerConfig[k] = v + } + b, err := json.Marshal(meta.providerConfig) + if err != nil { + return fmt.Errorf("marshal provider config: %v", err) + } + config, err := ctyjson.Unmarshal(b, configschema.SchemaBlockImpliedType(schResp.Provider.Block)) + if err != nil { + return fmt.Errorf("unmarshal provider config: %v", err) + } + + if _, diags = meta.tfclient.ConfigureProvider(ctx, typ.ConfigureProviderRequest{ + Config: config, + }); diags.HasErrors() { + return fmt.Errorf("configure provider: %v", diags) + } + + return nil +} + +func (meta *baseMeta) init_tf(ctx context.Context) error { + // Create the import directories per parallelism + if err := meta.initImportDirs(); err != nil { + return err + } + + // Init terraform + if err := meta.initTF(ctx); err != nil { + return err + } + + // Init provider + if err := meta.initProvider(ctx); err != nil { + return err + } + + // Pull TF state + baseState, err := meta.tf.StatePull(ctx) + if err != nil { + return fmt.Errorf("failed to pull state: %v", err) + } + meta.baseState = []byte(baseState) + meta.originBaseState = []byte(baseState) + + return nil +} + +func (meta *baseMeta) initImportDirs() error { + var importBaseDirs []string + var importModuleDirs []string + modulePaths := []string{} + for i, v := range strings.Split(meta.moduleAddr, ".") { + if i%2 == 1 { + modulePaths = append(modulePaths, v) + } + } + for i := 0; i < meta.parallelism; i++ { + dir, err := os.MkdirTemp("", "aztfexport-") + if err != nil { + return fmt.Errorf("creating import directory: %v", err) + } + + // Creating the module hierarchy if module path is specified. + // The hierarchy used here is not necessarily to be the same as is defined. What we need to guarantee here is the module address in TF is as specified. + mdir := dir + for _, moduleName := range modulePaths { + fpath := filepath.Join(mdir, "main.tf") + // #nosec G306 + if err := os.WriteFile(fpath, []byte(fmt.Sprintf(`module "%s" { + source = "./%s" +} +`, moduleName, moduleName)), 0644); err != nil { + return fmt.Errorf("creating %s: %v", fpath, err) + } + + mdir = filepath.Join(mdir, moduleName) + // #nosec G301 + if err := os.Mkdir(mdir, 0750); err != nil { + return fmt.Errorf("creating module dir %s: %v", mdir, err) + } + } + + importModuleDirs = append(importModuleDirs, mdir) + importBaseDirs = append(importBaseDirs, dir) + } + meta.importBaseDirs = importBaseDirs + meta.importModuleDirs = importModuleDirs + return nil +} + func (meta *baseMeta) initTF(ctx context.Context) error { log.Printf("[INFO] Init Terraform") execPath, err := FindTerraform(ctx) @@ -725,6 +744,15 @@ func (meta *baseMeta) importItem(ctx context.Context, item *ImportItem, importId return } + if meta.tfclient != nil { + meta.importItem_notf(ctx, item, importIdx) + return + } + + meta.importItem_tf(ctx, item, importIdx) +} + +func (meta *baseMeta) importItem_tf(ctx context.Context, item *ImportItem, importIdx int) { moduleDir := meta.importModuleDirs[importIdx] tf := meta.importTFs[importIdx] @@ -761,22 +789,100 @@ func (meta *baseMeta) importItem(ctx context.Context, item *ImportItem, importId item.Imported = err == nil } +func (meta *baseMeta) importItem_notf(ctx context.Context, item *ImportItem, importIdx int) { + // Import resources + addr := item.TFAddr.String() + log.Printf("[INFO] Importing %s as %s", item.TFResourceId, addr) + // The actual resource type names in telemetry is redacted + meta.tc.Trace(telemetry.Info, fmt.Sprintf("Importing %s as %s", item.AzureResourceID.TypeString(), addr)) + + importResp, diags := meta.tfclient.ImportResourceState(ctx, typ.ImportResourceStateRequest{ + TypeName: item.TFAddr.Type, + ID: item.TFResourceId, + }) + if diags.HasErrors() { + log.Printf("[ERROR] Importing %s: %v", item.TFAddr, diags) + meta.tc.Trace(telemetry.Error, fmt.Sprintf("Importing %s: %v", item.AzureResourceID.TypeString(), diags)) + item.ImportError = diags.Err() + item.Imported = false + return + } + if len(importResp.ImportedResources) != 1 { + err := fmt.Errorf("expect 1 resource being imported, got=%d", len(importResp.ImportedResources)) + log.Printf("[ERROR] %s", err) + meta.tc.Trace(telemetry.Error, err.Error()) + item.ImportError = err + item.Imported = false + return + } + res := importResp.ImportedResources[0] + readResp, diags := meta.tfclient.ReadResource(ctx, typ.ReadResourceRequest{ + TypeName: res.TypeName, + PriorState: res.State, + Private: res.Private, + }) + if diags.HasErrors() { + log.Printf("[ERROR] Reading %s: %v", item.TFAddr, diags) + meta.tc.Trace(telemetry.Error, fmt.Sprintf("Reading %s: %v", item.AzureResourceID.TypeString(), diags)) + item.ImportError = diags.Err() + item.Imported = false + return + } + + item.State = readResp.NewState + item.ImportError = nil + item.Imported = true + return +} + func (meta baseMeta) stateToConfig(ctx context.Context, list ImportList) (ConfigInfos, error) { var out []ConfigInfo + var bs [][]byte - var addrs []string importedList := list.Imported() - for _, item := range importedList { - addr := item.TFAddr.String() - if meta.moduleAddr != "" { - addr = meta.moduleAddr + "." + addr + + if meta.tfclient != nil { + for _, item := range importedList { + schResp, diags := meta.tfclient.GetProviderSchema() + if diags.HasErrors() { + return nil, fmt.Errorf("get provider schema: %v", diags) + } + rsch, ok := schResp.ResourceTypes[item.TFAddr.Type] + if !ok { + return nil, fmt.Errorf("no resource schema for %s found in the provider schema", item.TFAddr.Type) + } + b, err := tfadd.GenerateForOneResource( + &rsch, + tfstate.StateResource{ + Mode: tfjson.ManagedResourceMode, + Address: item.TFAddr.String(), + Type: item.TFAddr.Type, + ProviderName: "registry.terraform.io/hashicorp/azurerm", + Value: item.State, + }, + meta.fullConfig) + if err != nil { + return nil, fmt.Errorf("generating state for resource %s: %v", item.TFAddr, err) + } + bs = append(bs, b) + } + } else { + var addrs []string + for _, item := range importedList { + addr := item.TFAddr.String() + if meta.moduleAddr != "" { + addr = meta.moduleAddr + "." + addr + } + addrs = append(addrs, addr) + } + + var err error + bs, err = tfadd.StateForTargets(ctx, meta.tf, addrs, tfadd.Full(meta.fullConfig)) + if err != nil { + return nil, fmt.Errorf("converting terraform state to config: %w", err) } - addrs = append(addrs, addr) - } - bs, err := tfadd.StateForTargets(ctx, meta.tf, addrs, tfadd.Full(meta.fullConfig)) - if err != nil { - return nil, fmt.Errorf("converting terraform state to config: %w", err) } + for i, b := range bs { tpl := meta.cleanupTerraformAdd(string(b)) f, diag := hclwrite.ParseConfig([]byte(tpl), "", hcl.InitialPos) @@ -874,6 +980,47 @@ func (meta baseMeta) addDependency(configs ConfigInfos) (ConfigInfos, error) { return out, nil } +func (meta *baseMeta) deinit_notf(ctx context.Context) error { + meta.tfclient.Close() + return nil +} + +func (meta *baseMeta) deinit_tf(ctx context.Context) error { + // Clean up the temporary workspaces for parallel import + for _, dir := range meta.importBaseDirs { + // #nosec G104 + os.RemoveAll(dir) + } + return nil +} + +func getModuleDir(modulePaths []string, moduleDir string) (string, error) { + // Ensure the module path is something called by the main module + // We are following the module source and recursively call the LoadModule below. This is valid since we only support local path modules. + // (remote sources are not supported since we will end up generating config to that module, it only makes sense for local path modules) + module, err := tfconfig.LoadModule(moduleDir) + if err != nil { + return "", fmt.Errorf("loading main module: %v", err) + } + + for i, moduleName := range modulePaths { + mc := module.ModuleCalls[moduleName] + if mc == nil { + return "", fmt.Errorf("no module %q invoked by the root module", strings.Join(modulePaths[:i+1], ".")) + } + // See https://developer.hashicorp.com/terraform/language/modules/sources#local-paths + if !strings.HasPrefix(mc.Source, "./") && !strings.HasPrefix(mc.Source, "../") { + return "", fmt.Errorf("the source of module %q is not a local path", strings.Join(modulePaths[:i+1], ".")) + } + moduleDir = filepath.Join(moduleDir, mc.Source) + module, err = tfconfig.LoadModule(moduleDir) + if err != nil { + return "", fmt.Errorf("loading module %q: %v", strings.Join(modulePaths[:i+1], "."), err) + } + } + return moduleDir, nil +} + func appendToFile(path, content string) error { // #nosec G304 f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) diff --git a/internal/meta/importlist.go b/internal/meta/importlist.go index 795e58e..29fc65e 100644 --- a/internal/meta/importlist.go +++ b/internal/meta/importlist.go @@ -3,6 +3,7 @@ package meta import ( "github.com/Azure/aztfexport/internal/tfaddr" "github.com/magodo/armid" + "github.com/zclconf/go-cty/cty" ) type ImportItem struct { @@ -31,6 +32,9 @@ type ImportItem struct { IsRecommended bool Recommendations []string + + // State is what is being imported&read by terraform-plugin-go client. It is nil when importing via terraform binary. + State cty.Value } func (item ImportItem) Skip() bool { diff --git a/internal/test/resourcegroup/hcl_only_test.go b/internal/test/resourcegroup/hcl_only_test.go new file mode 100644 index 0000000..4f67374 --- /dev/null +++ b/internal/test/resourcegroup/hcl_only_test.go @@ -0,0 +1,141 @@ +package resourcegroup + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/Azure/aztfexport/internal" + internalconfig "github.com/Azure/aztfexport/internal/config" + "github.com/Azure/aztfexport/internal/test" + "github.com/Azure/aztfexport/internal/test/cases" + "github.com/Azure/aztfexport/pkg/config" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/terraform-exec/tfexec" + "github.com/hexops/gotextdiff" + "github.com/hexops/gotextdiff/myers" + "github.com/hexops/gotextdiff/span" + "github.com/magodo/terraform-client-go/tfclient" +) + +func runHCLOnly(t *testing.T, d test.Data, c cases.Case) { + if os.Getenv(test.TestPluginPathEnvVar) == "" { + t.Skipf("Skipping as %q not defined", test.TestPluginPathEnvVar) + } + tfexecPath := test.EnsureTF(t) + + // #nosec G204 + tfc, err := tfclient.New(tfclient.Option{ + Cmd: exec.Command(os.Getenv(test.TestPluginPathEnvVar)), + Logger: hclog.NewNullLogger(), + }) + if err != nil { + t.Fatalf("new tfclient: %v", err) + } + + provisionDir := t.TempDir() + if test.Keep() { + provisionDir, _ = os.MkdirTemp("", "") + t.Log(provisionDir) + } + + if err := os.WriteFile(filepath.Join(provisionDir, "main.tf"), []byte(c.Tpl(d)), 0644); err != nil { + t.Fatalf("created to create the TF config file: %v", err) + } + tf, err := tfexec.NewTerraform(provisionDir, tfexecPath) + if err != nil { + t.Fatalf("failed to new terraform: %v", err) + } + ctx := context.Background() + t.Log("Running: terraform init") + if err := tf.Init(ctx); err != nil { + t.Fatalf("terraform init failed: %v", err) + } + t.Log("Running: terraform apply") + if err := tf.Apply(ctx); err != nil { + t.Fatalf("terraform apply failed: %v", err) + } + if !test.Keep() { + defer func() { + t.Log("Running: terraform destroy") + if err := tf.Destroy(ctx); err != nil { + t.Logf("terraform destroy failed: %v", err) + } + }() + } + + const delay = time.Minute + t.Logf("Sleep for %v to wait for the just created resources be recorded in ARG\n", delay) + time.Sleep(delay) + + aztfexportDir := t.TempDir() + outDir1 := filepath.Join(aztfexportDir, "1") + outDir2 := filepath.Join(aztfexportDir, "2") + + if err := os.MkdirAll(outDir1, 0750); err != nil { + log.Fatalf("creating %s: %v", outDir1, err) + } + if err := os.MkdirAll(outDir2, 0750); err != nil { + log.Fatalf("creating %s: %v", outDir2, err) + } + + cred, clientOpt := test.BuildCredAndClientOpt(t) + + cfg := internalconfig.NonInteractiveModeConfig{ + Config: config.Config{ + CommonConfig: config.CommonConfig{ + SubscriptionId: os.Getenv("ARM_SUBSCRIPTION_ID"), + AzureSDKCredential: cred, + AzureSDKClientOption: *clientOpt, + OutputDir: outDir1, + BackendType: "local", + DevProvider: true, + Parallelism: 10, + HCLOnly: true, + }, + ResourceGroupName: d.RandomRgName(), + ResourceNamePattern: "res-", + }, + PlainUI: true, + } + t.Logf("Batch importing the resource group (with terraform) %s\n", d.RandomRgName()) + if err := internal.BatchImport(ctx, cfg); err != nil { + t.Fatalf("failed to run batch import (with terraform): %v", err) + } + + cfg.Config.CommonConfig.OutputDir = outDir2 + cfg.Config.CommonConfig.TFClient = tfc + t.Logf("Batch importing the resource group (with tfclient) %s\n", d.RandomRgName()) + if err := internal.BatchImport(ctx, cfg); err != nil { + t.Fatalf("failed to run batch import (with tfclient): %v", err) + } + + // Compare the main.tf files generated in two runs are the same + main1, main2 := filepath.Join(outDir1, "main.tf"), filepath.Join(outDir2, "main.tf") + f1, err := os.ReadFile(main1) + if err != nil { + t.Fatalf("reading %s: %v", main1, err) + } + f2, err := os.ReadFile(main2) + if err != nil { + t.Fatalf("reading %s: %v", main2, err) + } + + edits := myers.ComputeEdits(span.URIFromPath(main1), string(f1), string(f2)) + if len(edits) != 0 { + changes := fmt.Sprint(gotextdiff.ToUnified(main1, main2, string(f1), edits)) + t.Fatalf("main.tf file between two runs have diff: %s", changes) + } +} + +func TestComputeVMDiskForHCLOnly(t *testing.T) { + t.Parallel() + test.Precheck(t) + c, d := cases.CaseComputeVMDisk{}, test.NewData() + runHCLOnly(t, d, c) +} diff --git a/internal/test/utils.go b/internal/test/utils.go index 88f3c6f..c7aed70 100644 --- a/internal/test/utils.go +++ b/internal/test/utils.go @@ -26,6 +26,7 @@ import ( ) const TestToggleEnvVar = "AZTFEXPORT_E2E" +const TestPluginPathEnvVar = "AZTFEXPORT_PLUGIN_PATH" func Keep() bool { return os.Getenv("AZTFEXPORT_KEEP") != "" diff --git a/internal/utils/copyfile.go b/internal/utils/copyfile.go index 2c1afbf..66e0507 100644 --- a/internal/utils/copyfile.go +++ b/internal/utils/copyfile.go @@ -8,7 +8,7 @@ import ( func CopyFile(src, dst string) error { stat, err := os.Stat(src) if err != nil { - return fmt.Errorf("stating source file %s: %v", src, err) + return fmt.Errorf("stating source file %s: %w", src, err) } // #nosec G304 b, err := os.ReadFile(src) diff --git a/main.go b/main.go index 77d49b5..93eb9f5 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ import ( "github.com/hashicorp/go-hclog" "github.com/magodo/armid" "github.com/magodo/azlist/azlist" + "github.com/magodo/terraform-client-go/tfclient" "github.com/magodo/tfadd/providers/azurerm" "github.com/Azure/aztfexport/internal" @@ -256,6 +257,13 @@ func main() { Hidden: true, Destination: &flagset.hflagProfile, }, + &cli.StringFlag{ + Name: "tfclient-plugin-path", + EnvVars: []string{"AZTFEXPORT_TFCLIENT_PLUGIN_PATH"}, + Usage: "Replace terraform binary with terraform-client-go for importing (must be used with `--hcl-only`)", + Hidden: true, + Destination: &flagset.hflagTFClientPluginPath, + }, } resourceFlags := append([]cli.Flag{ @@ -420,6 +428,18 @@ func main() { cfg.CommonConfig.OutputFileNames = safeOutputFileNames } + if flagset.hflagTFClientPluginPath != "" { + // #nosec G204 + tfc, err := tfclient.New(tfclient.Option{ + Cmd: exec.Command(flagset.hflagTFClientPluginPath), + Logger: hclog.NewNullLogger(), + }) + if err != nil { + return err + } + cfg.CommonConfig.TFClient = tfc + } + return realMain(c.Context, cfg, flagset.flagNonInteractive, flagset.hflagMockClient, flagset.hflagPlainUI, flagset.flagGenerateMappingFile, flagset.hflagProfile, flagset.DescribeCLI(ModeResource)) }, }, @@ -472,6 +492,18 @@ func main() { cfg.CommonConfig.OutputFileNames = safeOutputFileNames } + if flagset.hflagTFClientPluginPath != "" { + // #nosec G204 + tfc, err := tfclient.New(tfclient.Option{ + Cmd: exec.Command(flagset.hflagTFClientPluginPath), + Logger: hclog.NewNullLogger(), + }) + if err != nil { + return err + } + cfg.CommonConfig.TFClient = tfc + } + return realMain(c.Context, cfg, flagset.flagNonInteractive, flagset.hflagMockClient, flagset.hflagPlainUI, flagset.flagGenerateMappingFile, flagset.hflagProfile, flagset.DescribeCLI(ModeResourceGroup)) }, }, @@ -523,6 +555,18 @@ func main() { cfg.CommonConfig.OutputFileNames = safeOutputFileNames } + if flagset.hflagTFClientPluginPath != "" { + // #nosec G204 + tfc, err := tfclient.New(tfclient.Option{ + Cmd: exec.Command(flagset.hflagTFClientPluginPath), + Logger: hclog.NewNullLogger(), + }) + if err != nil { + return err + } + cfg.CommonConfig.TFClient = tfc + } + return realMain(c.Context, cfg, flagset.flagNonInteractive, flagset.hflagMockClient, flagset.hflagPlainUI, flagset.flagGenerateMappingFile, flagset.hflagProfile, flagset.DescribeCLI(ModeQuery)) }, }, @@ -573,6 +617,18 @@ func main() { cfg.CommonConfig.OutputFileNames = safeOutputFileNames } + if flagset.hflagTFClientPluginPath != "" { + // #nosec G204 + tfc, err := tfclient.New(tfclient.Option{ + Cmd: exec.Command(flagset.hflagTFClientPluginPath), + Logger: hclog.NewNullLogger(), + }) + if err != nil { + return err + } + cfg.CommonConfig.TFClient = tfc + } + return realMain(c.Context, cfg, flagset.flagNonInteractive, flagset.hflagMockClient, flagset.hflagPlainUI, flagset.flagGenerateMappingFile, flagset.hflagProfile, flagset.DescribeCLI(ModeMappingFile)) }, }, diff --git a/pkg/config/config.go b/pkg/config/config.go index 6575686..2ceeed1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -4,6 +4,7 @@ import ( "github.com/Azure/aztfexport/pkg/telemetry" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/magodo/terraform-client-go/tfclient" "github.com/zclconf/go-cty/cty" ) @@ -53,6 +54,9 @@ type CommonConfig struct { // HCLOnly is a strange field, which is only used internally by aztfexport to indicate whether to remove other files other than TF config at the end. // External Go modules shoudl just ignore it. HCLOnly bool + // TFClient is the terraform-client-go client used to replace terraform binary for importing resources. + // This can only be used together with HCLOnly as tfclient can't replace terraform for state file management. + TFClient tfclient.Client // TelemetryClient is a client to send telemetry TelemetryClient telemetry.Client }