From c0ac45f440e07c093e468ec39b4cfd24964dca69 Mon Sep 17 00:00:00 2001 From: magodo Date: Fri, 28 Jun 2024 16:27:20 +0800 Subject: [PATCH 1/6] Cfg: Add `ImportCallback` to allow caller inspect import status --- internal/meta/base_meta.go | 14 +++++++++++++- pkg/config/config.go | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/internal/meta/base_meta.go b/internal/meta/base_meta.go index 2b17b27..67d16c4 100644 --- a/internal/meta/base_meta.go +++ b/internal/meta/base_meta.go @@ -92,6 +92,7 @@ type baseMeta struct { providerConfig map[string]cty.Value fullConfig bool parallelism int + importCallback config.ImportCallback generateImportFile bool hclOnly bool @@ -216,6 +217,7 @@ func NewBaseMeta(cfg config.CommonConfig) (*baseMeta, error) { providerName: cfg.ProviderName, fullConfig: cfg.FullConfig, parallelism: cfg.Parallelism, + importCallback: cfg.ImportCallback, generateImportFile: cfg.GenerateImportBlock, hclOnly: cfg.HCLOnly, tfclient: cfg.TFClient, @@ -273,7 +275,8 @@ func (meta *baseMeta) ParallelImport(ctx context.Context, items []*ImportItem) e meta.tc.Trace(telemetry.Info, "ParallelImport Enter") defer meta.tc.Trace(telemetry.Info, "ParallelImport Leave") - itemsCh := make(chan *ImportItem, len(items)) + total := len(items) + itemsCh := make(chan *ImportItem, total) for _, item := range items { itemsCh <- item } @@ -313,6 +316,15 @@ func (meta *baseMeta) ParallelImport(ctx context.Context, items []*ImportItem) e wp.AddTask(func() (interface{}, error) { for item := range itemsCh { meta.importItem(ctx, item, i) + if meta.importCallback != nil { + item := config.ImportItem{ + AzureResourceID: item.AzureResourceID, + TFResourceId: item.TFResourceId, + ImportError: item.ImportError, + TFAddr: item.TFAddr, + } + meta.importCallback(total, item) + } } return i, nil }) diff --git a/pkg/config/config.go b/pkg/config/config.go index 678bd14..d4260e4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,13 +1,31 @@ package config import ( + "github.com/Azure/aztfexport/internal/tfaddr" "github.com/Azure/aztfexport/pkg/telemetry" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/magodo/armid" "github.com/magodo/terraform-client-go/tfclient" "github.com/zclconf/go-cty/cty" ) +type ImportItem struct { + // Azure resource Id + AzureResourceID armid.ResourceId + + // The TF resource id + TFResourceId string + + // Whether this azure resource failed to import into terraform (this might due to the TFResourceType doesn't match the resource) + ImportError error + + // The terraform resource + TFAddr tfaddr.TFAddr +} + +type ImportCallback func(total int, item ImportItem) + type OutputFileNames struct { // The filename for the generated "terraform.tf" (default) TerraformFileName string @@ -52,6 +70,8 @@ type CommonConfig struct { FullConfig bool // Parallelism specifies the parallelism for the process Parallelism int + // ImportCallback is a way to inspect each resource after being imported during ParallelImport + ImportCallback ImportCallback // ModulePath specifies the path of the module (e.g. "module1.module2") where the resources will be imported and config generated. // Note that only modules whose "source" is local path is supported. By default, it is the root module. ModulePath string From 670bd7e5af75d79d508cab815067f09e56abbdf8 Mon Sep 17 00:00:00 2001 From: magodo Date: Fri, 28 Jun 2024 17:27:21 +0800 Subject: [PATCH 2/6] workflow: revert gosec to v2.18.2 --- .github/workflows/pr.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b1dd764..5887802 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,7 +23,11 @@ jobs: - name: Checkout Source uses: actions/checkout@v3 - name: Run Gosec Security Scanner - uses: securego/gosec@2ae137abcf405533ad6e549e9363e58e4f6e8b7d + # This is due to https://github.com/securego/gosec/issues/1105 + # Per https://github.com/securego/gosec/issues/1105#issuecomment-1948225619, the issue occurs since 2.19.0. + # The commit that updates the GH action to 2.19.0 is d13d7dac9b7e2b40e86be5b830d297816376f1db + # It's parent commit is 26e57d6b340778c2983cd61775bc7e8bb41d002a + uses: securego/gosec@26e57d6b340778c2983cd61775bc7e8bb41d002a with: args: './...' From 12a04d0d03c5418dca9a60be85926b2914d8034b Mon Sep 17 00:00:00 2001 From: magodo Date: Fri, 28 Jun 2024 17:45:41 +0800 Subject: [PATCH 3/6] setup go --- .github/workflows/pr.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5887802..e4053a2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -22,6 +22,12 @@ jobs: steps: - name: Checkout Source uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.22 + - name: Run Gosec Security Scanner # This is due to https://github.com/securego/gosec/issues/1105 # Per https://github.com/securego/gosec/issues/1105#issuecomment-1948225619, the issue occurs since 2.19.0. From 3a6c1178527ddeaee15acb64bdb374c1cb8bc759 Mon Sep 17 00:00:00 2001 From: magodo Date: Fri, 28 Jun 2024 17:49:24 +0800 Subject: [PATCH 4/6] gh action runs on ubuntu-24.04 --- .github/workflows/pr.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e4053a2..de54a61 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -18,16 +18,11 @@ jobs: gosec: name: gosec - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout Source uses: actions/checkout@v3 - - name: Set up Go - uses: actions/setup-go@v1 - with: - go-version: 1.22 - - name: Run Gosec Security Scanner # This is due to https://github.com/securego/gosec/issues/1105 # Per https://github.com/securego/gosec/issues/1105#issuecomment-1948225619, the issue occurs since 2.19.0. From 2d7954fd62b0d544a91b827c7ddeedda31487827 Mon Sep 17 00:00:00 2001 From: magodo Date: Fri, 28 Jun 2024 17:55:37 +0800 Subject: [PATCH 5/6] using tags --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index de54a61..80b65bd 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -18,7 +18,7 @@ jobs: gosec: name: gosec - runs-on: ubuntu-24.04 + runs-on: ubuntu-latest steps: - name: Checkout Source uses: actions/checkout@v3 @@ -30,7 +30,7 @@ jobs: # It's parent commit is 26e57d6b340778c2983cd61775bc7e8bb41d002a uses: securego/gosec@26e57d6b340778c2983cd61775bc7e8bb41d002a with: - args: './...' + args: '-tags -buildvcs=false ./...' depscheck: name: depscheck From 07e509d12e979eef7eeedfa399c3251203ea3f99 Mon Sep 17 00:00:00 2001 From: magodo Date: Fri, 28 Jun 2024 17:58:08 +0800 Subject: [PATCH 6/6] revert --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 80b65bd..796c41c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -30,7 +30,7 @@ jobs: # It's parent commit is 26e57d6b340778c2983cd61775bc7e8bb41d002a uses: securego/gosec@26e57d6b340778c2983cd61775bc7e8bb41d002a with: - args: '-tags -buildvcs=false ./...' + args: './...' depscheck: name: depscheck