Update azlist & move global logger to meta wise (#537)

command_before_func.go

@@ -50,11 +50,6 @@ func commandBeforeFunc(fset *FlagSet) func(ctx *cli.Context) error {
 				return fmt.Errorf("`--tfclient-plugin-path` must be used together with `--hcl-only`")
 			}
 		}
-		if flagLogLevel != "" {
-			if _, err := logLevel(flagLogLevel); err != nil {
-				return err
-			}
-		}
 		occur := 0
 		for _, ok := range []bool{
 			fset.flagUseEnvironmentCred,

flag.go

@@ -1,16 +1,21 @@
 package main
 
 import (
+	"context"
 	"fmt"
+	"io"
+	"log/slog"
 	"os"
 	"strings"
 
 	"github.com/Azure/aztfexport/internal/cfgfile"
+	"github.com/Azure/aztfexport/internal/log"
 	"github.com/Azure/aztfexport/pkg/config"
 	"github.com/Azure/aztfexport/pkg/telemetry"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
+	azlog "github.com/Azure/azure-sdk-for-go/sdk/azcore/log"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/gofrs/uuid"
@@ -40,6 +45,8 @@ type FlagSet struct {
 	flagHCLOnly             bool
 	flagModulePath          string
 	flagGenerateImportBlock bool
+	flagLogPath             string
+	flagLogLevel            string
 
 	// common flags (auth)
 	flagUseEnvironmentCred     bool
@@ -340,6 +347,7 @@ func (f FlagSet) BuildCommonConfig() (config.CommonConfig, error) {
 	}
 
 	cfg := config.CommonConfig{
+		Logger:               slog.New(slog.NewTextHandler(io.Discard, nil)),
 		SubscriptionId:       f.flagSubscriptionId,
 		AzureSDKCredential:   cred,
 		AzureSDKClientOption: *clientOpt,
@@ -367,5 +375,48 @@ func (f FlagSet) BuildCommonConfig() (config.CommonConfig, error) {
 		}
 	}
 
+	// Logger is only enabled when the log path is specified.
+	// This is because either interactive/non-interactive mode controls the terminal rendering,
+	// logging to stdout/stderr will impact the rendering.
+	if path := f.flagLogPath; path != "" {
+		level, err := logLevel(f.flagLogLevel)
+		if err != nil {
+			return config.CommonConfig{}, err
+		}
+
+		// #nosec G304
+		f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
+		if err != nil {
+			return config.CommonConfig{}, fmt.Errorf("creating log file %s: %v", path, err)
+		}
+
+		logger := slog.New(slog.NewTextHandler(f, &slog.HandlerOptions{Level: level}))
+
+		// Enable log for azure sdk
+		os.Setenv("AZURE_SDK_GO_LOGGING", "all") // #nosec G104
+		azlog.SetListener(func(cls azlog.Event, msg string) {
+			logger.Log(context.Background(), log.LevelTrace, msg, "event", cls)
+		})
+
+		cfg.Logger = logger
+	}
+
 	return cfg, nil
 }
+
+func logLevel(level string) (slog.Level, error) {
+	switch strings.ToUpper(level) {
+	case "ERROR":
+		return slog.LevelError, nil
+	case "WARN":
+		return slog.LevelWarn, nil
+	case "INFO":
+		return slog.LevelInfo, nil
+	case "DEBUG":
+		return slog.LevelDebug, nil
+	case "TRACE":
+		return log.LevelTrace, nil
+	default:
+		return slog.Level(0), fmt.Errorf("unknown log level: %s", level)
+	}
+}

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/hashicorp/terraform-json v0.19.0
 	github.com/hexops/gotextdiff v1.0.3
 	github.com/magodo/armid v0.0.0-20230511151020-27880e5961c3
-	github.com/magodo/azlist v0.0.0-20240613024003-b4529218cc6a
+	github.com/magodo/azlist v0.0.0-20240702100525-5ac55e2823fa
 	github.com/magodo/aztft v0.3.1-0.20240429022627-002cdc06267a
 	github.com/magodo/slog2hclog v0.0.0-20240614031327-090ebd72a033
 	github.com/magodo/spinner v0.0.0-20220720073946-50f31b2dc5a6

go.sum

@@ -228,8 +228,8 @@ github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69
 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/magodo/armid v0.0.0-20230511151020-27880e5961c3 h1:ob6vk6PlChZvutcxcLnmPH/VNmJEuwz+TmCYCVtJqeA=
 github.com/magodo/armid v0.0.0-20230511151020-27880e5961c3/go.mod h1:rR8E7zfGMbmfnSQvrkFiWYdhrfTqsVSltelnZB09BwA=
-github.com/magodo/azlist v0.0.0-20240613024003-b4529218cc6a h1:CFX3fvA3ajq81F7fkSBF+RgrZEgyuNP8x2QWt4Zhw5k=
-github.com/magodo/azlist v0.0.0-20240613024003-b4529218cc6a/go.mod h1:xefFDOxzRssOEjGoxvrO8jeTWlzHXbY6sCJYOt+Jh5k=
+github.com/magodo/azlist v0.0.0-20240702100525-5ac55e2823fa h1:CP3OIkp1AMEolym+8tFCyDdG3NM1T95MFcdYvnWxPns=
+github.com/magodo/azlist v0.0.0-20240702100525-5ac55e2823fa/go.mod h1:xefFDOxzRssOEjGoxvrO8jeTWlzHXbY6sCJYOt+Jh5k=
 github.com/magodo/aztft v0.3.1-0.20240429022627-002cdc06267a h1:HYp5h1HOwwrFI6CVSeJTVfnXyO7Es6Kx8cyw3UlomDo=
 github.com/magodo/aztft v0.3.1-0.20240429022627-002cdc06267a/go.mod h1:hqk4M4qig7+LTNKeHCNgi+hZfJBndnx4oSNAvP5KT0Y=
 github.com/magodo/slog2hclog v0.0.0-20240614031327-090ebd72a033 h1:K2seYsMAzoICCLdDe7uU2WyaACLW+tvdTWG3QB+pyec=

internal/log/log.go

@@ -0,0 +1,5 @@
+package log
+
+import "log/slog"
+
+const LevelTrace = slog.LevelDebug - 4

internal/meta/base_meta.go

@@ -6,14 +6,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log/slog"
 	"os"
 	"path/filepath"
 	"strings"
 
 	tfjson "github.com/hashicorp/terraform-json"
 
 	"github.com/Azure/aztfexport/pkg/config"
-	"github.com/Azure/aztfexport/pkg/log"
 	"github.com/zclconf/go-cty/cty"
 
 	"github.com/Azure/aztfexport/internal/client"
@@ -48,6 +48,8 @@ const SkippedResourcesFileName = "aztfexportSkippedResources.txt"
 type TFConfigTransformer func(configs ConfigInfos) (ConfigInfos, error)
 
 type BaseMeta interface {
+	// Logger returns a slog.Logger
+	Logger() *slog.Logger
 	// ProviderName returns the target provider name, which is either azurerm or azapi.
 	ProviderName() string
 	// Init initializes the base meta, including initialize terraform, provider and soem runtime temporary resources.
@@ -77,6 +79,7 @@ type BaseMeta interface {
 var _ BaseMeta = &baseMeta{}
 
 type baseMeta struct {
+	logger             *slog.Logger
 	subscriptionId     string
 	azureSDKCred       azcore.TokenCredential
 	azureSDKClientOpt  arm.ClientOptions
@@ -203,6 +206,7 @@ func NewBaseMeta(cfg config.CommonConfig) (*baseMeta, error) {
 	}
 
 	meta := &baseMeta{
+		logger:             cfg.Logger,
 		subscriptionId:     cfg.SubscriptionId,
 		azureSDKCred:       cfg.AzureSDKCredential,
 		azureSDKClientOpt:  cfg.AzureSDKClientOption,
@@ -231,6 +235,10 @@ func NewBaseMeta(cfg config.CommonConfig) (*baseMeta, error) {
 	return meta, nil
 }
 
+func (meta baseMeta) Logger() *slog.Logger {
+	return meta.logger
+}
+
 func (meta baseMeta) ProviderName() string {
 	return meta.providerName
 }
@@ -301,7 +309,7 @@ func (meta *baseMeta) ParallelImport(ctx context.Context, items []*ImportItem) e
 		// Ensure the state file is removed after this round import, preparing for the next round.
 		defer os.Remove(stateFile)
 
-		log.Debug("Merging terraform state file (tfmerge)", "file", stateFile)
+		meta.Logger().Debug("Merging terraform state file (tfmerge)", "file", stateFile)
 		newState, err := tfmerge.Merge(ctx, meta.tf, meta.baseState, stateFile)
 		if err != nil {
 			return fmt.Errorf("failed to merge state file: %v", err)
@@ -679,12 +687,12 @@ func (meta *baseMeta) initImportDirs() error {
 }
 
 func (meta *baseMeta) initTF(ctx context.Context) error {
-	log.Info("Init Terraform")
+	meta.Logger().Info("Init Terraform")
 	execPath, err := FindTerraform(ctx)
 	if err != nil {
 		return fmt.Errorf("error finding a terraform exectuable: %w", err)
 	}
-	log.Info("Found terraform binary", "path", execPath)
+	meta.Logger().Info("Found terraform binary", "path", execPath)
 
 	newTF := func(dir string) (*tfexec.Terraform, error) {
 		tf, err := tfexec.NewTerraform(dir, execPath)
@@ -720,7 +728,7 @@ func (meta *baseMeta) initTF(ctx context.Context) error {
 }
 
 func (meta *baseMeta) initProvider(ctx context.Context) error {
-	log.Info("Init provider")
+	meta.Logger().Info("Init provider")
 
 	module, diags := tfconfig.LoadModule(meta.outdir)
 	if diags.HasErrors() {
@@ -733,7 +741,7 @@ func (meta *baseMeta) initProvider(ctx context.Context) error {
 	}
 
 	if module.ProviderConfigs[meta.providerName] == nil {
-		log.Info("Output directory doesn't contain provider setting, create one then")
+		meta.Logger().Info("Output directory doesn't contain provider setting, create one then")
 		cfgFile := filepath.Join(meta.outdir, meta.outputFileNames.ProviderFileName)
 		// #nosec G306
 		if err := os.WriteFile(cfgFile, []byte(meta.buildProviderConfig()), 0644); err != nil {
@@ -742,7 +750,7 @@ func (meta *baseMeta) initProvider(ctx context.Context) error {
 	}
 
 	if tfblock == nil {
-		log.Info("Output directory doesn't contain terraform block, create one then")
+		meta.Logger().Info("Output directory doesn't contain terraform block, create one then")
 		cfgFile := filepath.Join(meta.outdir, meta.outputFileNames.TerraformFileName)
 		// #nosec G306
 		if err := os.WriteFile(cfgFile, []byte(meta.buildTerraformConfig(meta.backendType)), 0644); err != nil {
@@ -756,7 +764,7 @@ func (meta *baseMeta) initProvider(ctx context.Context) error {
 		opts = append(opts, tfexec.BackendConfig(opt))
 	}
 
-	log.Debug(`Run "terraform init" for the output directory`, "dir", meta.outdir)
+	meta.Logger().Debug(`Run "terraform init" for the output directory`, "dir", meta.outdir)
 	if err := meta.tf.Init(ctx, opts...); err != nil {
 		return fmt.Errorf("error running terraform init for the output directory: %s", err)
 	}
@@ -778,9 +786,9 @@ func (meta *baseMeta) initProvider(ctx context.Context) error {
 				return nil, fmt.Errorf("error creating terraform config: %w", err)
 			}
 			if meta.devProvider {
-				log.Debug(`Skip running "terraform init" for the import directory (dev provider)`, "dir", meta.importBaseDirs[i])
+				meta.Logger().Debug(`Skip running "terraform init" for the import directory (dev provider)`, "dir", meta.importBaseDirs[i])
 			} else {
-				log.Debug(`Run "terraform init" for the import directory`, "dir", meta.importBaseDirs[i])
+				meta.Logger().Debug(`Run "terraform init" for the import directory`, "dir", meta.importBaseDirs[i])
 				if err := meta.importTFs[i].Init(ctx); err != nil {
 					return nil, fmt.Errorf("error running terraform init: %s", err)
 				}
@@ -797,7 +805,7 @@ func (meta *baseMeta) initProvider(ctx context.Context) error {
 
 func (meta *baseMeta) importItem(ctx context.Context, item *ImportItem, importIdx int) {
 	if item.Skip() {
-		log.Info("Skipping resource", "tf_id", item.TFResourceId)
+		meta.Logger().Info("Skipping resource", "tf_id", item.TFResourceId)
 		return
 	}
 
@@ -819,7 +827,7 @@ func (meta *baseMeta) importItem_tf(ctx context.Context, item *ImportItem, impor
 	// #nosec G306
 	if err := os.WriteFile(cfgFile, []byte(tpl), 0644); err != nil {
 		err := fmt.Errorf("generating resource template file for %s: %w", item.TFAddr, err)
-		log.Error("Failed to generate resource template file", "error", err, "tf_addr", item.TFAddr)
+		meta.Logger().Error("Failed to generate resource template file", "error", err, "tf_addr", item.TFAddr)
 		item.ImportError = err
 		return
 	}
@@ -831,13 +839,13 @@ func (meta *baseMeta) importItem_tf(ctx context.Context, item *ImportItem, impor
 		addr = meta.moduleAddr + "." + addr
 	}
 
-	log.Info("Importing a resource", "tf_id", item.TFResourceId, "tf_addr", addr)
+	meta.Logger().Info("Importing a resource", "tf_id", item.TFResourceId, "tf_addr", addr)
 	// The actual resource type names in telemetry is redacted
 	meta.tc.Trace(telemetry.Info, fmt.Sprintf("Importing %s as %s", item.AzureResourceID.TypeString(), addr))
 
 	err := tf.Import(ctx, addr, item.TFResourceId)
 	if err != nil {
-		log.Error("Terraform import failed", "tf_addr", item.TFAddr, "error", err)
+		meta.Logger().Error("Terraform import failed", "tf_addr", item.TFAddr, "error", err)
 		meta.tc.Trace(telemetry.Error, fmt.Sprintf("Importing %s failed", item.AzureResourceID.TypeString()))
 		meta.tc.Trace(telemetry.Error, fmt.Sprintf("Error detail: %v", err))
 	} else {
@@ -850,7 +858,7 @@ func (meta *baseMeta) importItem_tf(ctx context.Context, item *ImportItem, impor
 func (meta *baseMeta) importItem_notf(ctx context.Context, item *ImportItem, importIdx int) {
 	// Import resources
 	addr := item.TFAddr.String()
-	log.Info("Importing a resource", "tf_id", item.TFResourceId, "tf_addr", addr)
+	meta.Logger().Info("Importing a resource", "tf_id", item.TFResourceId, "tf_addr", addr)
 	// The actual resource type names in telemetry is redacted
 	meta.tc.Trace(telemetry.Info, fmt.Sprintf("Importing %s as %s", item.AzureResourceID.TypeString(), addr))
 
@@ -859,7 +867,7 @@ func (meta *baseMeta) importItem_notf(ctx context.Context, item *ImportItem, imp
 		ID:       item.TFResourceId,
 	})
 	if diags.HasErrors() {
-		log.Error("Terraform import failed", "tf_addr", item.TFAddr, "error", diags.Err())
+		meta.Logger().Error("Terraform import failed", "tf_addr", item.TFAddr, "error", diags.Err())
 		meta.tc.Trace(telemetry.Error, fmt.Sprintf("Importing %s failed", item.AzureResourceID.TypeString()))
 		meta.tc.Trace(telemetry.Error, fmt.Sprintf("Error detail: %v", diags.Err()))
 		item.ImportError = diags.Err()
@@ -868,7 +876,7 @@ func (meta *baseMeta) importItem_notf(ctx context.Context, item *ImportItem, imp
 	}
 	if len(importResp.ImportedResources) != 1 {
 		err := fmt.Errorf("expect 1 resource being imported, got=%d", len(importResp.ImportedResources))
-		log.Error(err.Error())
+		meta.Logger().Error(err.Error())
 		meta.tc.Trace(telemetry.Error, err.Error())
 		item.ImportError = err
 		item.Imported = false
@@ -881,7 +889,7 @@ func (meta *baseMeta) importItem_notf(ctx context.Context, item *ImportItem, imp
 		Private:    res.Private,
 	})
 	if diags.HasErrors() {
-		log.Error("Terraform read a resource failed", "tf_addr", item.TFAddr, "error", diags.Err())
+		meta.Logger().Error("Terraform read a resource failed", "tf_addr", item.TFAddr, "error", diags.Err())
 		meta.tc.Trace(telemetry.Error, fmt.Sprintf("Reading %s failed", item.AzureResourceID.TypeString()))
 		meta.tc.Trace(telemetry.Error, fmt.Sprintf("Error detail: %v", diags.Err()))
 		item.ImportError = diags.Err()

internal/meta/meta_dummy.go

@@ -2,6 +2,8 @@ package meta
 
 import (
 	"context"
+	"io"
+	"log/slog"
 	"time"
 )
 
@@ -14,6 +16,10 @@ func NewGroupMetaDummy(rg string, providerName string) MetaGroupDummy {
 	return MetaGroupDummy{rg: rg, providerName: providerName}
 }
 
+func (m MetaGroupDummy) Logger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, nil))
+}
+
 func (m MetaGroupDummy) ProviderName() string {
 	return m.providerName
 }

internal/meta/meta_map.go

@@ -8,7 +8,6 @@ import (
 	"sort"
 
 	"github.com/Azure/aztfexport/pkg/config"
-	"github.com/Azure/aztfexport/pkg/log"
 
 	"github.com/Azure/aztfexport/internal/resmap"
 	"github.com/Azure/aztfexport/internal/tfaddr"
@@ -21,7 +20,7 @@ type MetaMap struct {
 }
 
 func NewMetaMap(cfg config.Config) (*MetaMap, error) {
-	log.Info("New map meta")
+	cfg.Logger.Info("New map meta")
 	baseMeta, err := NewBaseMeta(cfg.CommonConfig)
 	if err != nil {
 		return nil, err
@@ -42,7 +41,7 @@ func (meta MetaMap) ScopeName() string {
 func (meta *MetaMap) ListResource(_ context.Context) (ImportList, error) {
 	var m resmap.ResourceMapping
 
-	log.Debug("Read resource set from mapping file")
+	meta.Logger().Debug("Read resource set from mapping file")
 	b, err := os.ReadFile(meta.mappingFile)
 	if err != nil {
 		return nil, fmt.Errorf("reading mapping file %s: %v", meta.mappingFile, err)