diff --git a/command_before_func.go b/command_before_func.go index 73143a4..3571a0b 100644 --- a/command_before_func.go +++ b/command_before_func.go @@ -50,11 +50,6 @@ func commandBeforeFunc(fset *FlagSet) func(ctx *cli.Context) error { return fmt.Errorf("`--tfclient-plugin-path` must be used together with `--hcl-only`") } } - if flagLogLevel != "" { - if _, err := logLevel(flagLogLevel); err != nil { - return err - } - } occur := 0 for _, ok := range []bool{ fset.flagUseEnvironmentCred, diff --git a/flag.go b/flag.go index 17d2e29..9ddac5f 100644 --- a/flag.go +++ b/flag.go @@ -1,16 +1,21 @@ package main import ( + "context" "fmt" + "io" + "log/slog" "os" "strings" "github.com/Azure/aztfexport/internal/cfgfile" + "github.com/Azure/aztfexport/internal/log" "github.com/Azure/aztfexport/pkg/config" "github.com/Azure/aztfexport/pkg/telemetry" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + azlog "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/gofrs/uuid" @@ -40,6 +45,8 @@ type FlagSet struct { flagHCLOnly bool flagModulePath string flagGenerateImportBlock bool + flagLogPath string + flagLogLevel string // common flags (auth) flagUseEnvironmentCred bool @@ -340,6 +347,7 @@ func (f FlagSet) BuildCommonConfig() (config.CommonConfig, error) { } cfg := config.CommonConfig{ + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), SubscriptionId: f.flagSubscriptionId, AzureSDKCredential: cred, AzureSDKClientOption: *clientOpt, @@ -367,5 +375,48 @@ func (f FlagSet) BuildCommonConfig() (config.CommonConfig, error) { } } + // Logger is only enabled when the log path is specified. + // This is because either interactive/non-interactive mode controls the terminal rendering, + // logging to stdout/stderr will impact the rendering. + if path := f.flagLogPath; path != "" { + level, err := logLevel(f.flagLogLevel) + if err != nil { + return config.CommonConfig{}, err + } + + // #nosec G304 + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return config.CommonConfig{}, fmt.Errorf("creating log file %s: %v", path, err) + } + + logger := slog.New(slog.NewTextHandler(f, &slog.HandlerOptions{Level: level})) + + // Enable log for azure sdk + os.Setenv("AZURE_SDK_GO_LOGGING", "all") // #nosec G104 + azlog.SetListener(func(cls azlog.Event, msg string) { + logger.Log(context.Background(), log.LevelTrace, msg, "event", cls) + }) + + cfg.Logger = logger + } + return cfg, nil } + +func logLevel(level string) (slog.Level, error) { + switch strings.ToUpper(level) { + case "ERROR": + return slog.LevelError, nil + case "WARN": + return slog.LevelWarn, nil + case "INFO": + return slog.LevelInfo, nil + case "DEBUG": + return slog.LevelDebug, nil + case "TRACE": + return log.LevelTrace, nil + default: + return slog.Level(0), fmt.Errorf("unknown log level: %s", level) + } +} diff --git a/go.mod b/go.mod index d60a372..7553623 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/hashicorp/terraform-json v0.19.0 github.com/hexops/gotextdiff v1.0.3 github.com/magodo/armid v0.0.0-20230511151020-27880e5961c3 - github.com/magodo/azlist v0.0.0-20240613024003-b4529218cc6a + github.com/magodo/azlist v0.0.0-20240702100525-5ac55e2823fa github.com/magodo/aztft v0.3.1-0.20240429022627-002cdc06267a github.com/magodo/slog2hclog v0.0.0-20240614031327-090ebd72a033 github.com/magodo/spinner v0.0.0-20220720073946-50f31b2dc5a6 diff --git a/go.sum b/go.sum index e84b6ef..49802b2 100644 --- a/go.sum +++ b/go.sum @@ -228,8 +228,8 @@ github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magodo/armid v0.0.0-20230511151020-27880e5961c3 h1:ob6vk6PlChZvutcxcLnmPH/VNmJEuwz+TmCYCVtJqeA= github.com/magodo/armid v0.0.0-20230511151020-27880e5961c3/go.mod h1:rR8E7zfGMbmfnSQvrkFiWYdhrfTqsVSltelnZB09BwA= -github.com/magodo/azlist v0.0.0-20240613024003-b4529218cc6a h1:CFX3fvA3ajq81F7fkSBF+RgrZEgyuNP8x2QWt4Zhw5k= -github.com/magodo/azlist v0.0.0-20240613024003-b4529218cc6a/go.mod h1:xefFDOxzRssOEjGoxvrO8jeTWlzHXbY6sCJYOt+Jh5k= +github.com/magodo/azlist v0.0.0-20240702100525-5ac55e2823fa h1:CP3OIkp1AMEolym+8tFCyDdG3NM1T95MFcdYvnWxPns= +github.com/magodo/azlist v0.0.0-20240702100525-5ac55e2823fa/go.mod h1:xefFDOxzRssOEjGoxvrO8jeTWlzHXbY6sCJYOt+Jh5k= github.com/magodo/aztft v0.3.1-0.20240429022627-002cdc06267a h1:HYp5h1HOwwrFI6CVSeJTVfnXyO7Es6Kx8cyw3UlomDo= github.com/magodo/aztft v0.3.1-0.20240429022627-002cdc06267a/go.mod h1:hqk4M4qig7+LTNKeHCNgi+hZfJBndnx4oSNAvP5KT0Y= github.com/magodo/slog2hclog v0.0.0-20240614031327-090ebd72a033 h1:K2seYsMAzoICCLdDe7uU2WyaACLW+tvdTWG3QB+pyec= diff --git a/internal/armschema/armschema.go b/internal/armschema/armschema.go deleted file mode 100644 index 90d6cd9..0000000 --- a/internal/armschema/armschema.go +++ /dev/null @@ -1,28 +0,0 @@ -package armschema - -import ( - "encoding/json" - "strings" - "sync" - - "github.com/magodo/azlist/azlist" -) - -var once sync.Once -var armSchemas = map[string][]string{} - -func GetARMSchemas() (map[string][]string, error) { - var err error - once.Do(func() { - var m map[string][]string - err = json.Unmarshal(azlist.ARMSchemaFile, &m) - if err != nil { - return - } - for k, v := range m { - armSchemas[strings.ToUpper(k)] = v - } - }) - return armSchemas, nil - -} diff --git a/internal/log/log.go b/internal/log/log.go new file mode 100644 index 0000000..e7e7216 --- /dev/null +++ b/internal/log/log.go @@ -0,0 +1,5 @@ +package log + +import "log/slog" + +const LevelTrace = slog.LevelDebug - 4 diff --git a/internal/meta/base_meta.go b/internal/meta/base_meta.go index 67d16c4..3c9c348 100644 --- a/internal/meta/base_meta.go +++ b/internal/meta/base_meta.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" "os" "path/filepath" "strings" @@ -13,7 +14,6 @@ import ( tfjson "github.com/hashicorp/terraform-json" "github.com/Azure/aztfexport/pkg/config" - "github.com/Azure/aztfexport/pkg/log" "github.com/zclconf/go-cty/cty" "github.com/Azure/aztfexport/internal/client" @@ -48,6 +48,8 @@ const SkippedResourcesFileName = "aztfexportSkippedResources.txt" type TFConfigTransformer func(configs ConfigInfos) (ConfigInfos, error) type BaseMeta interface { + // Logger returns a slog.Logger + Logger() *slog.Logger // ProviderName returns the target provider name, which is either azurerm or azapi. ProviderName() string // Init initializes the base meta, including initialize terraform, provider and soem runtime temporary resources. @@ -77,6 +79,7 @@ type BaseMeta interface { var _ BaseMeta = &baseMeta{} type baseMeta struct { + logger *slog.Logger subscriptionId string azureSDKCred azcore.TokenCredential azureSDKClientOpt arm.ClientOptions @@ -203,6 +206,7 @@ func NewBaseMeta(cfg config.CommonConfig) (*baseMeta, error) { } meta := &baseMeta{ + logger: cfg.Logger, subscriptionId: cfg.SubscriptionId, azureSDKCred: cfg.AzureSDKCredential, azureSDKClientOpt: cfg.AzureSDKClientOption, @@ -231,6 +235,10 @@ func NewBaseMeta(cfg config.CommonConfig) (*baseMeta, error) { return meta, nil } +func (meta baseMeta) Logger() *slog.Logger { + return meta.logger +} + func (meta baseMeta) ProviderName() string { return meta.providerName } @@ -301,7 +309,7 @@ func (meta *baseMeta) ParallelImport(ctx context.Context, items []*ImportItem) e // Ensure the state file is removed after this round import, preparing for the next round. defer os.Remove(stateFile) - log.Debug("Merging terraform state file (tfmerge)", "file", stateFile) + meta.Logger().Debug("Merging terraform state file (tfmerge)", "file", stateFile) newState, err := tfmerge.Merge(ctx, meta.tf, meta.baseState, stateFile) if err != nil { return fmt.Errorf("failed to merge state file: %v", err) @@ -679,12 +687,12 @@ func (meta *baseMeta) initImportDirs() error { } func (meta *baseMeta) initTF(ctx context.Context) error { - log.Info("Init Terraform") + meta.Logger().Info("Init Terraform") execPath, err := FindTerraform(ctx) if err != nil { return fmt.Errorf("error finding a terraform exectuable: %w", err) } - log.Info("Found terraform binary", "path", execPath) + meta.Logger().Info("Found terraform binary", "path", execPath) newTF := func(dir string) (*tfexec.Terraform, error) { tf, err := tfexec.NewTerraform(dir, execPath) @@ -720,7 +728,7 @@ func (meta *baseMeta) initTF(ctx context.Context) error { } func (meta *baseMeta) initProvider(ctx context.Context) error { - log.Info("Init provider") + meta.Logger().Info("Init provider") module, diags := tfconfig.LoadModule(meta.outdir) if diags.HasErrors() { @@ -733,7 +741,7 @@ func (meta *baseMeta) initProvider(ctx context.Context) error { } if module.ProviderConfigs[meta.providerName] == nil { - log.Info("Output directory doesn't contain provider setting, create one then") + meta.Logger().Info("Output directory doesn't contain provider setting, create one then") cfgFile := filepath.Join(meta.outdir, meta.outputFileNames.ProviderFileName) // #nosec G306 if err := os.WriteFile(cfgFile, []byte(meta.buildProviderConfig()), 0644); err != nil { @@ -742,7 +750,7 @@ func (meta *baseMeta) initProvider(ctx context.Context) error { } if tfblock == nil { - log.Info("Output directory doesn't contain terraform block, create one then") + meta.Logger().Info("Output directory doesn't contain terraform block, create one then") cfgFile := filepath.Join(meta.outdir, meta.outputFileNames.TerraformFileName) // #nosec G306 if err := os.WriteFile(cfgFile, []byte(meta.buildTerraformConfig(meta.backendType)), 0644); err != nil { @@ -756,7 +764,7 @@ func (meta *baseMeta) initProvider(ctx context.Context) error { opts = append(opts, tfexec.BackendConfig(opt)) } - log.Debug(`Run "terraform init" for the output directory`, "dir", meta.outdir) + meta.Logger().Debug(`Run "terraform init" for the output directory`, "dir", meta.outdir) if err := meta.tf.Init(ctx, opts...); err != nil { return fmt.Errorf("error running terraform init for the output directory: %s", err) } @@ -778,9 +786,9 @@ func (meta *baseMeta) initProvider(ctx context.Context) error { return nil, fmt.Errorf("error creating terraform config: %w", err) } if meta.devProvider { - log.Debug(`Skip running "terraform init" for the import directory (dev provider)`, "dir", meta.importBaseDirs[i]) + meta.Logger().Debug(`Skip running "terraform init" for the import directory (dev provider)`, "dir", meta.importBaseDirs[i]) } else { - log.Debug(`Run "terraform init" for the import directory`, "dir", meta.importBaseDirs[i]) + meta.Logger().Debug(`Run "terraform init" for the import directory`, "dir", meta.importBaseDirs[i]) if err := meta.importTFs[i].Init(ctx); err != nil { return nil, fmt.Errorf("error running terraform init: %s", err) } @@ -797,7 +805,7 @@ func (meta *baseMeta) initProvider(ctx context.Context) error { func (meta *baseMeta) importItem(ctx context.Context, item *ImportItem, importIdx int) { if item.Skip() { - log.Info("Skipping resource", "tf_id", item.TFResourceId) + meta.Logger().Info("Skipping resource", "tf_id", item.TFResourceId) return } @@ -819,7 +827,7 @@ func (meta *baseMeta) importItem_tf(ctx context.Context, item *ImportItem, impor // #nosec G306 if err := os.WriteFile(cfgFile, []byte(tpl), 0644); err != nil { err := fmt.Errorf("generating resource template file for %s: %w", item.TFAddr, err) - log.Error("Failed to generate resource template file", "error", err, "tf_addr", item.TFAddr) + meta.Logger().Error("Failed to generate resource template file", "error", err, "tf_addr", item.TFAddr) item.ImportError = err return } @@ -831,13 +839,13 @@ func (meta *baseMeta) importItem_tf(ctx context.Context, item *ImportItem, impor addr = meta.moduleAddr + "." + addr } - log.Info("Importing a resource", "tf_id", item.TFResourceId, "tf_addr", addr) + meta.Logger().Info("Importing a resource", "tf_id", item.TFResourceId, "tf_addr", addr) // The actual resource type names in telemetry is redacted meta.tc.Trace(telemetry.Info, fmt.Sprintf("Importing %s as %s", item.AzureResourceID.TypeString(), addr)) err := tf.Import(ctx, addr, item.TFResourceId) if err != nil { - log.Error("Terraform import failed", "tf_addr", item.TFAddr, "error", err) + meta.Logger().Error("Terraform import failed", "tf_addr", item.TFAddr, "error", err) meta.tc.Trace(telemetry.Error, fmt.Sprintf("Importing %s failed", item.AzureResourceID.TypeString())) meta.tc.Trace(telemetry.Error, fmt.Sprintf("Error detail: %v", err)) } else { @@ -850,7 +858,7 @@ func (meta *baseMeta) importItem_tf(ctx context.Context, item *ImportItem, impor func (meta *baseMeta) importItem_notf(ctx context.Context, item *ImportItem, importIdx int) { // Import resources addr := item.TFAddr.String() - log.Info("Importing a resource", "tf_id", item.TFResourceId, "tf_addr", addr) + meta.Logger().Info("Importing a resource", "tf_id", item.TFResourceId, "tf_addr", addr) // The actual resource type names in telemetry is redacted meta.tc.Trace(telemetry.Info, fmt.Sprintf("Importing %s as %s", item.AzureResourceID.TypeString(), addr)) @@ -859,7 +867,7 @@ func (meta *baseMeta) importItem_notf(ctx context.Context, item *ImportItem, imp ID: item.TFResourceId, }) if diags.HasErrors() { - log.Error("Terraform import failed", "tf_addr", item.TFAddr, "error", diags.Err()) + meta.Logger().Error("Terraform import failed", "tf_addr", item.TFAddr, "error", diags.Err()) meta.tc.Trace(telemetry.Error, fmt.Sprintf("Importing %s failed", item.AzureResourceID.TypeString())) meta.tc.Trace(telemetry.Error, fmt.Sprintf("Error detail: %v", diags.Err())) item.ImportError = diags.Err() @@ -868,7 +876,7 @@ func (meta *baseMeta) importItem_notf(ctx context.Context, item *ImportItem, imp } if len(importResp.ImportedResources) != 1 { err := fmt.Errorf("expect 1 resource being imported, got=%d", len(importResp.ImportedResources)) - log.Error(err.Error()) + meta.Logger().Error(err.Error()) meta.tc.Trace(telemetry.Error, err.Error()) item.ImportError = err item.Imported = false @@ -881,7 +889,7 @@ func (meta *baseMeta) importItem_notf(ctx context.Context, item *ImportItem, imp Private: res.Private, }) if diags.HasErrors() { - log.Error("Terraform read a resource failed", "tf_addr", item.TFAddr, "error", diags.Err()) + meta.Logger().Error("Terraform read a resource failed", "tf_addr", item.TFAddr, "error", diags.Err()) meta.tc.Trace(telemetry.Error, fmt.Sprintf("Reading %s failed", item.AzureResourceID.TypeString())) meta.tc.Trace(telemetry.Error, fmt.Sprintf("Error detail: %v", diags.Err())) item.ImportError = diags.Err() diff --git a/internal/meta/meta_dummy.go b/internal/meta/meta_dummy.go index 4405be4..7c138b2 100644 --- a/internal/meta/meta_dummy.go +++ b/internal/meta/meta_dummy.go @@ -2,6 +2,8 @@ package meta import ( "context" + "io" + "log/slog" "time" ) @@ -14,6 +16,10 @@ func NewGroupMetaDummy(rg string, providerName string) MetaGroupDummy { return MetaGroupDummy{rg: rg, providerName: providerName} } +func (m MetaGroupDummy) Logger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + func (m MetaGroupDummy) ProviderName() string { return m.providerName } diff --git a/internal/meta/meta_map.go b/internal/meta/meta_map.go index a98c1cf..6ea513a 100644 --- a/internal/meta/meta_map.go +++ b/internal/meta/meta_map.go @@ -8,7 +8,6 @@ import ( "sort" "github.com/Azure/aztfexport/pkg/config" - "github.com/Azure/aztfexport/pkg/log" "github.com/Azure/aztfexport/internal/resmap" "github.com/Azure/aztfexport/internal/tfaddr" @@ -21,7 +20,7 @@ type MetaMap struct { } func NewMetaMap(cfg config.Config) (*MetaMap, error) { - log.Info("New map meta") + cfg.Logger.Info("New map meta") baseMeta, err := NewBaseMeta(cfg.CommonConfig) if err != nil { return nil, err @@ -42,7 +41,7 @@ func (meta MetaMap) ScopeName() string { func (meta *MetaMap) ListResource(_ context.Context) (ImportList, error) { var m resmap.ResourceMapping - log.Debug("Read resource set from mapping file") + meta.Logger().Debug("Read resource set from mapping file") b, err := os.ReadFile(meta.mappingFile) if err != nil { return nil, fmt.Errorf("reading mapping file %s: %v", meta.mappingFile, err) diff --git a/internal/meta/meta_query.go b/internal/meta/meta_query.go index a789bd1..eedb892 100644 --- a/internal/meta/meta_query.go +++ b/internal/meta/meta_query.go @@ -7,7 +7,6 @@ import ( "github.com/Azure/aztfexport/internal/resourceset" "github.com/Azure/aztfexport/internal/tfaddr" "github.com/Azure/aztfexport/pkg/config" - "github.com/Azure/aztfexport/pkg/log" "github.com/magodo/azlist/azlist" ) @@ -22,7 +21,7 @@ type MetaQuery struct { } func NewMetaQuery(cfg config.Config) (*MetaQuery, error) { - log.Info("New query meta") + cfg.Logger.Info("New query meta") baseMeta, err := NewBaseMeta(cfg.CommonConfig) if err != nil { return nil, err @@ -49,27 +48,27 @@ func (meta MetaQuery) ScopeName() string { } func (meta *MetaQuery) ListResource(ctx context.Context) (ImportList, error) { - log.Debug("Query resource set") + meta.Logger().Debug("Query resource set") rset, err := meta.queryResourceSet(ctx, meta.argPredicate, meta.recursiveQuery) if err != nil { return nil, err } var rl []resourceset.TFResource if meta.useAzAPI() { - log.Debug("Azure Resource set map to TF resource set") + meta.Logger().Debug("Azure Resource set map to TF resource set") rl = rset.ToTFAzAPIResources() } else { - log.Debug("Populate resource set") + meta.Logger().Debug("Populate resource set") if err := rset.PopulateResource(); err != nil { return nil, fmt.Errorf("tweaking single resources in the azure resource set: %v", err) } - log.Debug("Reduce resource set") + meta.Logger().Debug("Reduce resource set") if err := rset.ReduceResource(); err != nil { return nil, fmt.Errorf("tweaking across resources in the azure resource set: %v", err) } - log.Debug("Azure Resource set map to TF resource set") - rl = rset.ToTFAzureRMResources(meta.parallelism, meta.azureSDKCred, meta.azureSDKClientOpt) + meta.Logger().Debug("Azure Resource set map to TF resource set") + rl = rset.ToTFAzureRMResources(meta.Logger(), meta.parallelism, meta.azureSDKCred, meta.azureSDKClientOpt) } var l ImportList @@ -99,16 +98,21 @@ func (meta *MetaQuery) ListResource(ctx context.Context) (ImportList, error) { } func (meta MetaQuery) queryResourceSet(ctx context.Context, predicate string, recursive bool) (*resourceset.AzureResourceSet, error) { - result, err := azlist.List(ctx, predicate, - azlist.Option{ - SubscriptionId: meta.subscriptionId, - Cred: meta.azureSDKCred, - ClientOpt: meta.azureSDKClientOpt, - Parallelism: meta.parallelism, - Recursive: recursive, - ExtensionResourceTypes: extBuilder{includeRoleAssignment: meta.includeRoleAssignment}.Build(), - IncludeResourceGroup: meta.includeResourceGroup, - }) + opt := azlist.Option{ + Logger: meta.logger.WithGroup("azlist"), + SubscriptionId: meta.subscriptionId, + Cred: meta.azureSDKCred, + ClientOpt: meta.azureSDKClientOpt, + Parallelism: meta.parallelism, + Recursive: recursive, + ExtensionResourceTypes: extBuilder{includeRoleAssignment: meta.includeRoleAssignment}.Build(), + IncludeResourceGroup: meta.includeResourceGroup, + } + lister, err := azlist.NewLister(opt) + if err != nil { + return nil, fmt.Errorf("building azlister: %v", err) + } + result, err := lister.List(ctx, predicate) if err != nil { return nil, fmt.Errorf("listing resource set: %v", err) } diff --git a/internal/meta/meta_res.go b/internal/meta/meta_res.go index e128c80..83cb539 100644 --- a/internal/meta/meta_res.go +++ b/internal/meta/meta_res.go @@ -7,7 +7,6 @@ import ( "github.com/Azure/aztfexport/internal/resourceset" "github.com/Azure/aztfexport/internal/tfaddr" "github.com/Azure/aztfexport/pkg/config" - "github.com/Azure/aztfexport/pkg/log" "github.com/magodo/armid" "github.com/magodo/aztft/aztft" ) @@ -20,7 +19,7 @@ type MetaResource struct { } func NewMetaResource(cfg config.Config) (*MetaResource, error) { - log.Info("New resource meta") + cfg.Logger.Info("New resource meta") baseMeta, err := NewBaseMeta(cfg.CommonConfig) if err != nil { return nil, err @@ -51,13 +50,13 @@ func (meta *MetaResource) ListResource(_ context.Context) (ImportList, error) { }, }, } - log.Debug("Azure Resource set map to TF resource set") + meta.Logger().Debug("Azure Resource set map to TF resource set") var rl []resourceset.TFResource if meta.useAzAPI() { rl = resourceSet.ToTFAzAPIResources() } else { - rl = resourceSet.ToTFAzureRMResources(meta.parallelism, meta.azureSDKCred, meta.azureSDKClientOpt) + rl = resourceSet.ToTFAzureRMResources(meta.Logger(), meta.parallelism, meta.azureSDKCred, meta.azureSDKClientOpt) } // This is to record known resource types. In case there is a known resource type and there comes another same typed resource, diff --git a/internal/meta/meta_rg.go b/internal/meta/meta_rg.go index e8dfd91..bc84705 100644 --- a/internal/meta/meta_rg.go +++ b/internal/meta/meta_rg.go @@ -7,7 +7,6 @@ import ( "github.com/Azure/aztfexport/internal/resourceset" "github.com/Azure/aztfexport/internal/tfaddr" "github.com/Azure/aztfexport/pkg/config" - "github.com/Azure/aztfexport/pkg/log" "github.com/magodo/armid" "github.com/magodo/azlist/azlist" ) @@ -21,7 +20,7 @@ type MetaResourceGroup struct { } func NewMetaResourceGroup(cfg config.Config) (*MetaResourceGroup, error) { - log.Info("New resource group meta") + cfg.Logger.Info("New resource group meta") baseMeta, err := NewBaseMeta(cfg.CommonConfig) if err != nil { return nil, err @@ -42,7 +41,7 @@ func (meta MetaResourceGroup) ScopeName() string { } func (meta *MetaResourceGroup) ListResource(ctx context.Context) (ImportList, error) { - log.Debug("Query resource set") + meta.Logger().Debug("Query resource set") rset, err := meta.queryResourceSet(ctx, meta.resourceGroup) if err != nil { return nil, err @@ -52,17 +51,17 @@ func (meta *MetaResourceGroup) ListResource(ctx context.Context) (ImportList, er if meta.useAzAPI() { rl = rset.ToTFAzAPIResources() } else { - log.Debug("Populate resource set") + meta.Logger().Debug("Populate resource set") if err := rset.PopulateResource(); err != nil { return nil, fmt.Errorf("tweaking single resources in the azure resource set: %v", err) } - log.Debug("Reduce resource set") + meta.Logger().Debug("Reduce resource set") if err := rset.ReduceResource(); err != nil { return nil, fmt.Errorf("tweaking across resources in the azure resource set: %v", err) } - log.Debug("Azure Resource set map to TF resource set") - rl = rset.ToTFAzureRMResources(meta.parallelism, meta.azureSDKCred, meta.azureSDKClientOpt) + meta.Logger().Debug("Azure Resource set map to TF resource set") + rl = rset.ToTFAzureRMResources(meta.Logger(), meta.parallelism, meta.azureSDKCred, meta.azureSDKClientOpt) } var l ImportList @@ -90,16 +89,21 @@ func (meta *MetaResourceGroup) ListResource(ctx context.Context) (ImportList, er } func (meta MetaResourceGroup) queryResourceSet(ctx context.Context, rg string) (*resourceset.AzureResourceSet, error) { - result, err := azlist.List(ctx, fmt.Sprintf("resourceGroup =~ %q", rg), - azlist.Option{ - SubscriptionId: meta.subscriptionId, - Cred: meta.azureSDKCred, - ClientOpt: meta.azureSDKClientOpt, - Parallelism: meta.parallelism, - Recursive: true, - IncludeResourceGroup: false, - ExtensionResourceTypes: extBuilder{includeRoleAssignment: meta.includeRoleAssignment}.Build(), - }) + opt := azlist.Option{ + Logger: meta.logger.WithGroup("azlist"), + SubscriptionId: meta.subscriptionId, + Cred: meta.azureSDKCred, + ClientOpt: meta.azureSDKClientOpt, + Parallelism: meta.parallelism, + Recursive: true, + IncludeResourceGroup: false, + ExtensionResourceTypes: extBuilder{includeRoleAssignment: meta.includeRoleAssignment}.Build(), + } + lister, err := azlist.NewLister(opt) + if err != nil { + return nil, fmt.Errorf("building azlister: %v", err) + } + result, err := lister.List(ctx, fmt.Sprintf("resourceGroup =~ %q", rg)) if err != nil { return nil, fmt.Errorf("listing resource set: %v", err) } diff --git a/internal/resourceset/azure_resource_set.go b/internal/resourceset/azure_resource_set.go index 8d8047c..e3d4c69 100644 --- a/internal/resourceset/azure_resource_set.go +++ b/internal/resourceset/azure_resource_set.go @@ -1,9 +1,9 @@ package resourceset import ( + "log/slog" "sort" - "github.com/Azure/aztfexport/pkg/log" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" @@ -26,7 +26,7 @@ type PesudoResourceInfo struct { TFId string } -func (rset AzureResourceSet) ToTFAzureRMResources(parallelism int, cred azcore.TokenCredential, clientOpt arm.ClientOptions) []TFResource { +func (rset AzureResourceSet) ToTFAzureRMResources(logger *slog.Logger, parallelism int, cred azcore.TokenCredential, clientOpt arm.ClientOptions) []TFResource { tfresources := []TFResource{} wp := workerpool.NewWorkPool(parallelism) @@ -42,7 +42,7 @@ func (rset AzureResourceSet) ToTFAzureRMResources(parallelism int, cred azcore.T wp.Run(func(v interface{}) error { res := v.(result) if res.err != nil { - log.Warn("Failed to query resource type", "id", res.resid, "error", res.err) + logger.Warn("Failed to query resource type", "id", res.resid, "error", res.err) // Still put this unresolved resource in the resource set, so that users can later specify the expected TF resource type. tfresources = append(tfresources, TFResource{ AzureId: res.resid, @@ -52,7 +52,7 @@ func (rset AzureResourceSet) ToTFAzureRMResources(parallelism int, cred azcore.T } else { if !res.exact { // It is not possible to return multiple result when API is used. - log.Warn("No query result for resource type and TF id", "id", res.resid) + logger.Warn("No query result for resource type and TF id", "id", res.resid) // Still put this unresolved resource in the resource set, so that users can later specify the expected TF resource type. tfresources = append(tfresources, TFResource{ AzureId: res.resid, diff --git a/internal/test/query/query_test.go b/internal/test/query/query_test.go index afd5545..20c66a4 100644 --- a/internal/test/query/query_test.go +++ b/internal/test/query/query_test.go @@ -3,6 +3,8 @@ package query import ( "context" "fmt" + "io" + "log/slog" "os" "path/filepath" "testing" @@ -90,6 +92,7 @@ resource "azurerm_subnet" "test" { cfg := internalconfig.NonInteractiveModeConfig{ Config: config.Config{ CommonConfig: config.CommonConfig{ + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), SubscriptionId: os.Getenv("ARM_SUBSCRIPTION_ID"), AzureSDKCredential: cred, AzureSDKClientOption: *clientOpt, diff --git a/internal/test/resmap/e2e_cases_test.go b/internal/test/resmap/e2e_cases_test.go index 1f87670..8f2d8e3 100644 --- a/internal/test/resmap/e2e_cases_test.go +++ b/internal/test/resmap/e2e_cases_test.go @@ -3,6 +3,8 @@ package resmap import ( "context" "encoding/json" + "io" + "log/slog" "os" "path/filepath" "testing" @@ -73,6 +75,7 @@ func runCase(t *testing.T, d test.Data, c cases.Case) { cfg := internalconfig.NonInteractiveModeConfig{ Config: config.Config{ CommonConfig: config.CommonConfig{ + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), SubscriptionId: os.Getenv("ARM_SUBSCRIPTION_ID"), AzureSDKCredential: cred, AzureSDKClientOption: *clientOpt, diff --git a/internal/test/resource/e2e_cases_test.go b/internal/test/resource/e2e_cases_test.go index d4d29ed..2a5d9fe 100644 --- a/internal/test/resource/e2e_cases_test.go +++ b/internal/test/resource/e2e_cases_test.go @@ -3,6 +3,8 @@ package resource import ( "context" "fmt" + "io" + "log/slog" "os" "path/filepath" "testing" @@ -71,6 +73,7 @@ func runCase(t *testing.T, d test.Data, c cases.Case) { cfg := internalconfig.NonInteractiveModeConfig{ Config: config.Config{ CommonConfig: config.CommonConfig{ + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), SubscriptionId: os.Getenv("ARM_SUBSCRIPTION_ID"), AzureSDKCredential: cred, AzureSDKClientOption: *clientOpt, diff --git a/internal/test/resourcegroup/append_test.go b/internal/test/resourcegroup/append_test.go index a0a415b..164d99a 100644 --- a/internal/test/resourcegroup/append_test.go +++ b/internal/test/resourcegroup/append_test.go @@ -3,6 +3,8 @@ package resourcegroup import ( "context" "fmt" + "io" + "log/slog" "os" "path/filepath" "testing" @@ -84,6 +86,7 @@ resource "azurerm_resource_group" "test3" { cfg := internalconfig.NonInteractiveModeConfig{ Config: config.Config{ CommonConfig: config.CommonConfig{ + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), SubscriptionId: os.Getenv("ARM_SUBSCRIPTION_ID"), AzureSDKCredential: cred, AzureSDKClientOption: *clientOpt, diff --git a/internal/test/resourcegroup/e2e_cases_test.go b/internal/test/resourcegroup/e2e_cases_test.go index 26c6bc2..6b25127 100644 --- a/internal/test/resourcegroup/e2e_cases_test.go +++ b/internal/test/resourcegroup/e2e_cases_test.go @@ -2,6 +2,8 @@ package resourcegroup import ( "context" + "io" + "log/slog" "os" "path/filepath" "testing" @@ -63,6 +65,7 @@ func runCase(t *testing.T, d test.Data, c cases.Case) { cfg := internalconfig.NonInteractiveModeConfig{ Config: config.Config{ CommonConfig: config.CommonConfig{ + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), SubscriptionId: os.Getenv("ARM_SUBSCRIPTION_ID"), AzureSDKCredential: cred, AzureSDKClientOption: *clientOpt, diff --git a/internal/test/resourcegroup/hcl_only_test.go b/internal/test/resourcegroup/hcl_only_test.go index 0643549..726fe22 100644 --- a/internal/test/resourcegroup/hcl_only_test.go +++ b/internal/test/resourcegroup/hcl_only_test.go @@ -3,7 +3,9 @@ package resourcegroup import ( "context" "fmt" + "io" "log" + "log/slog" "os" "os/exec" "path/filepath" @@ -89,6 +91,7 @@ func runHCLOnly(t *testing.T, d test.Data, c cases.Case) { cfg := internalconfig.NonInteractiveModeConfig{ Config: config.Config{ CommonConfig: config.CommonConfig{ + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), SubscriptionId: os.Getenv("ARM_SUBSCRIPTION_ID"), AzureSDKCredential: cred, AzureSDKClientOption: *clientOpt, diff --git a/internal/test/resourcegroup/module_test.go b/internal/test/resourcegroup/module_test.go index 0d347d4..937cd45 100644 --- a/internal/test/resourcegroup/module_test.go +++ b/internal/test/resourcegroup/module_test.go @@ -3,6 +3,8 @@ package resourcegroup import ( "context" "fmt" + "io" + "log/slog" "os" "path/filepath" "testing" @@ -111,6 +113,7 @@ module "sub-module" { cfg := internalconfig.NonInteractiveModeConfig{ Config: config.Config{ CommonConfig: config.CommonConfig{ + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), SubscriptionId: os.Getenv("ARM_SUBSCRIPTION_ID"), AzureSDKCredential: cred, AzureSDKClientOption: *clientOpt, diff --git a/internal/ui/ui.go b/internal/ui/ui.go index d64c2da..3a235be 100644 --- a/internal/ui/ui.go +++ b/internal/ui/ui.go @@ -5,8 +5,8 @@ import ( "fmt" "github.com/Azure/aztfexport/internal/config" + "github.com/Azure/aztfexport/internal/log" internalmeta "github.com/Azure/aztfexport/internal/meta" - "github.com/Azure/aztfexport/pkg/log" "github.com/Azure/aztfexport/pkg/meta" "github.com/Azure/aztfexport/internal/ui/aztfexportclient" @@ -118,7 +118,7 @@ func (m model) Init() tea.Cmd { func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { if _, ok := msg.(spinner.TickMsg); !ok { - log.Trace("UI update", "status", m.status, "msg", fmt.Sprintf("%#v", msg)) + m.meta.Logger().Log(context.Background(), log.LevelTrace, "UI update", "status", m.status, "msg", fmt.Sprintf("%#v", msg)) } switch msg := msg.(type) { diff --git a/main.go b/main.go index 08295df..5abe984 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "log/slog" "os" "os/exec" "path/filepath" @@ -20,10 +19,8 @@ import ( "github.com/pkg/profile" "github.com/Azure/aztfexport/pkg/config" - "github.com/Azure/aztfexport/pkg/log" "github.com/magodo/armid" - "github.com/magodo/azlist/azlist" "github.com/magodo/slog2hclog" "github.com/magodo/terraform-client-go/tfclient" "github.com/magodo/tfadd/providers/azapi" @@ -31,15 +28,9 @@ import ( "github.com/Azure/aztfexport/internal" "github.com/Azure/aztfexport/internal/ui" - azlog "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" "github.com/urfave/cli/v2" ) -var ( - flagLogPath string - flagLogLevel string -) - func prepareConfigFile(ctx *cli.Context) error { // Prepare the config directory at $HOME/.aztfexport homeDir, err := os.UserHomeDir() @@ -70,13 +61,9 @@ func prepareConfigFile(ctx *cli.Context) error { if id, err := cfgfile.GetInstallationIdFromCLI(); err == nil { return id, nil } - log.Debug("Installation ID not found from Azure CLI", "error", err) - if id, err := cfgfile.GetInstallationIdFromPWSH(); err == nil { return id, nil } - log.Debug("Installation ID not found from Azure PWSH", "error", err) - uuid, err := uuid.NewV4() if err != nil { return "", fmt.Errorf("generating installation id: %w", err) @@ -239,13 +226,13 @@ func main() { Name: "log-path", EnvVars: []string{"AZTFEXPORT_LOG_PATH"}, Usage: "The file path to store the log", - Destination: &flagLogPath, + Destination: &flagset.flagLogPath, }, &cli.StringFlag{ Name: "log-level", EnvVars: []string{"AZTFEXPORT_LOG_LEVEL"}, Usage: `Log level, can be one of "ERROR", "WARN", "INFO", "DEBUG" and "TRACE"`, - Destination: &flagLogLevel, + Destination: &flagset.flagLogLevel, Value: "INFO", }, @@ -585,59 +572,6 @@ func main() { } } -func logLevel(level string) (slog.Level, error) { - switch strings.ToUpper(level) { - case "ERROR": - return slog.LevelError, nil - case "WARN": - return slog.LevelWarn, nil - case "INFO": - return slog.LevelInfo, nil - case "DEBUG": - return slog.LevelDebug, nil - case "TRACE": - return log.LevelTrace, nil - default: - return slog.Level(0), fmt.Errorf("unknown log level: %s", level) - } -} - -func initLog(path string, flagLevel string) error { - //golog.SetOutput(io.Discard) - - // Logger is only enabled when the log path is specified. - // This is because either interactive/non-interactive mode controls the terminal rendering, - // logging to stdout/stderr will impact the rendering. - if path != "" { - level, err := logLevel(flagLevel) - if err != nil { - return err - } - - // #nosec G304 - f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return fmt.Errorf("creating log file %s: %v", path, err) - } - - logger := slog.New(slog.NewTextHandler(f, &slog.HandlerOptions{Level: level})) - - // Enable log for aztfexport - log.SetLogger(logger) - - // Enable log for azlist - azlist.SetLogger(logger) - - // Enable log for azure sdk - os.Setenv("AZURE_SDK_GO_LOGGING", "all") // #nosec G104 - azlog.SetListener(func(cls azlog.Event, msg string) { - logger.Log(context.Background(), log.LevelTrace, msg, "event", cls) - }) - } - - return nil -} - func subscriptionIdFromCLI() (string, error) { var stderr bytes.Buffer var stdout bytes.Buffer @@ -665,18 +599,12 @@ func realMain(ctx context.Context, cfg config.Config, batch, mockMeta, plainUI, defer profile.Start(profile.MemProfile, profile.ProfilePath("."), profile.NoShutdownHook).Stop() } - // Initialize log - if err := initLog(flagLogPath, flagLogLevel); err != nil { - result = err - return - } - // Initialize the TFClient if tfClientPluginPath != "" { // #nosec G204 tfc, err := tfclient.New(tfclient.Option{ Cmd: exec.Command(flagset.hflagTFClientPluginPath), - Logger: slog2hclog.New(log.GetLogger(), nil), + Logger: slog2hclog.New(cfg.Logger.WithGroup("provider"), nil), }) if err != nil { return err @@ -688,17 +616,17 @@ func realMain(ctx context.Context, cfg config.Config, batch, mockMeta, plainUI, defer func() { if result == nil { - log.Info("aztfexport ends") + cfg.Logger.Info("aztfexport ends") tc.Trace(telemetry.Info, "aztfexport ends") } else { - log.Error("aztfexport ends with error", "error", result) + cfg.Logger.Error("aztfexport ends with error", "error", result) tc.Trace(telemetry.Error, fmt.Sprintf("aztfexport ends with error")) tc.Trace(telemetry.Error, fmt.Sprintf("Error detail: %v", result)) } tc.Close() }() - log.Info("aztfexport starts", "config", fmt.Sprintf("%#v", cfg)) + cfg.Logger.Info("aztfexport starts", "config", fmt.Sprintf("%#v", cfg)) tc.Trace(telemetry.Info, "aztfexport starts") tc.Trace(telemetry.Info, "Effective CLI: "+effectiveCLI) diff --git a/pkg/config/config.go b/pkg/config/config.go index d4260e4..b42a8a2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,6 +1,8 @@ package config import ( + "log/slog" + "github.com/Azure/aztfexport/internal/tfaddr" "github.com/Azure/aztfexport/pkg/telemetry" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -38,6 +40,7 @@ type OutputFileNames struct { } type CommonConfig struct { + Logger *slog.Logger // SubscriptionId specifies the user's Azure subscription id. SubscriptionId string // AzureSDKCredential specifies the Azure SDK token credential diff --git a/pkg/log/log.go b/pkg/log/log.go deleted file mode 100644 index 6ed5a89..0000000 --- a/pkg/log/log.go +++ /dev/null @@ -1,69 +0,0 @@ -package log - -import ( - "context" - "io" - "log/slog" - - "github.com/magodo/slog2hclog" -) - -var logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - -const LevelTrace = slog2hclog.SlogLevelTrace - -func SetLogger(l *slog.Logger) { - logger = l -} - -func GetLogger() *slog.Logger { - return logger -} - -func Trace(msg string, args ...any) { - logger.Log(context.Background(), LevelTrace, msg, args...) -} - -func TraceContext(ctx context.Context, msg string, args ...any) { - logger.Log(ctx, LevelTrace, msg, args...) -} - -func Debug(msg string, args ...any) { - logger.Debug(msg, args...) -} - -func DebugContext(ctx context.Context, msg string, args ...any) { - logger.DebugContext(ctx, msg, args...) -} - -func Info(msg string, args ...any) { - logger.Info(msg, args...) -} - -func InfoContext(ctx context.Context, msg string, args ...any) { - logger.InfoContext(ctx, msg, args...) -} - -func Warn(msg string, args ...any) { - logger.Warn(msg, args...) -} - -func WarnContext(ctx context.Context, msg string, args ...any) { - logger.WarnContext(ctx, msg, args...) -} - -func Error(msg string, args ...any) { - logger.Error(msg, args...) -} - -func ErrorContext(ctx context.Context, msg string, args ...any) { - logger.ErrorContext(ctx, msg, args...) -} - -func Log(ctx context.Context, level slog.Level, msg string, args ...any) { - logger.Log(ctx, level, msg, args...) -} - -func LogAttrs(ctx context.Context, level slog.Level, msg string, attrs ...slog.Attr) { - logger.LogAttrs(ctx, level, msg, attrs...) -}