Support for v2 auth tokens (i.e. MSAL)

- This is opt-in behaviour via the provider property `use_msal`, and
  environment variables `ARM_USE_MSAL` / `ARM_USE_MSGRAPH` (the latter
  for compatibility with the related backend property

- When `use_msal` is true, do not make any API calls to a graph API
  (legacy or current). There are only 2 uses of this at present:

  - `data.azurerm_client_config`, which doesn't actually do anything
    with the result so this appears to be a vestige anyway
  - `azurerm_hdinsight_kafka_cluster`, the API for which requires
    both an AAD group ID and name to be specified (?) so currently
    this resource looks up the group name from the supplied ID. In
    future we'll require that both are specified (e.g. using
    `data.azuread_group` for any necessary lookup)

- In v3.0, we'll remove support for graph clients in order to delegate
  any required usage to the AzureAD provider.

- Also removes support for Azure Germany, which is now offline

.gitignore

@@ -24,6 +24,7 @@ website/node_modules
 .idea
 *.iml
 *.test
+.python-version
 .terraform.tfstate.lock.info
 .terraform.lock.hcl
 

internal/clients/auth.go

@@ -16,6 +16,9 @@ type ResourceManagerAccount struct {
 	SkipResourceProviderRegistration bool
 	SubscriptionId                   string
 	TenantId                         string
+
+	// TODO: remove in v3.0
+	UseMSAL bool
 }
 
 func NewResourceManagerAccount(ctx context.Context, config authentication.Config, env azure.Environment, skipResourceProviderRegistration bool) (*ResourceManagerAccount, error) {
@@ -38,6 +41,9 @@ func NewResourceManagerAccount(ctx context.Context, config authentication.Config
 		TenantId:                         config.TenantID,
 		SkipResourceProviderRegistration: skipResourceProviderRegistration,
 		SubscriptionId:                   config.SubscriptionID,
+
+		// TODO: remove in v3.0
+		UseMSAL: config.UseMicrosoftGraph,
 	}
 	return &account, nil
 }

internal/clients/builder.go

@@ -11,6 +11,8 @@ import (
 	"github.com/hashicorp/go-azure-helpers/authentication"
 	"github.com/hashicorp/go-azure-helpers/resourcemanager/location"
 	"github.com/hashicorp/go-azure-helpers/sender"
+	"github.com/manicminer/hamilton/environments"
+
 	"github.com/hashicorp/terraform-provider-azurerm/internal/common"
 	"github.com/hashicorp/terraform-provider-azurerm/internal/features"
 	"github.com/hashicorp/terraform-provider-azurerm/internal/resourceproviders"
@@ -26,15 +28,16 @@ type ClientBuilder struct {
 	StorageUseAzureAD           bool
 	TerraformVersion            string
 	Features                    features.UserFeatures
+	UseMSAL                     bool
 }
 
 const azureStackEnvironmentError = `
-The AzureRM Provider supports the different Azure Public Clouds - including China, Germany,
-Public and US Government - however it does not support Azure Stack due to differences in
-API and feature availability.
+The AzureRM Provider supports the different Azure Public Clouds - including China, Public,
+and US Government - however it does not support Azure Stack due to differences in API and
+feature availability.
 
 Terraform instead offers a separate "azurestack" provider which supports the functionality
-and API's available in Azure Stack via Azure Stack Profiles.
+and APIs available in Azure Stack via Azure Stack Profiles.
 `
 
 func Build(ctx context.Context, builder ClientBuilder) (*Client, error) {
@@ -51,11 +54,18 @@ func Build(ctx context.Context, builder ClientBuilder) (*Client, error) {
 		return nil, fmt.Errorf(azureStackEnvironmentError)
 	}
 
+	// Autorest environment configuration
 	env, err := authentication.AzureEnvironmentByNameFromEndpoint(ctx, builder.AuthConfig.MetadataHost, builder.AuthConfig.Environment)
 	if err != nil {
 		return nil, fmt.Errorf("unable to find environment %q from endpoint %q: %+v", builder.AuthConfig.Environment, builder.AuthConfig.MetadataHost, err)
 	}
 
+	// Hamilton environment configuration
+	environment, err := environments.EnvironmentFromString(builder.AuthConfig.Environment)
+	if err != nil {
+		return nil, fmt.Errorf("unable to find environment %q from endpoint %q: %+v", builder.AuthConfig.Environment, builder.AuthConfig.MetadataHost, err)
+	}
+
 	// client declarations:
 	account, err := NewResourceManagerAccount(ctx, *builder.AuthConfig, *env, builder.SkipProviderRegistration)
 	if err != nil {
@@ -78,56 +88,99 @@ func Build(ctx context.Context, builder ClientBuilder) (*Client, error) {
 
 	sender := sender.BuildSender("AzureRM")
 
-	// Resource Manager endpoints
-	endpoint := env.ResourceManagerEndpoint
-	auth, err := builder.AuthConfig.GetADALToken(ctx, sender, oauthConfig, env.TokenAudience)
-	if err != nil {
-		return nil, fmt.Errorf("unable to get authorization token for resource manager: %+v", err)
-	}
+	// Authorizers, via autorest or hamilton/auth
+	var auth, storageAuth, synapseAuth, batchManagementAuth autorest.Authorizer
+	var keyVaultAuth *autorest.BearerAuthorizerCallback
+	var tokenFunc common.EndpointTokenFunc
+	var graphAuth autorest.Authorizer // TODO: remove in v3.0
 
-	// Graph Endpoints
-	graphEndpoint := env.GraphEndpoint
-	graphAuth, err := builder.AuthConfig.GetADALToken(ctx, sender, oauthConfig, graphEndpoint)
-	if err != nil {
-		return nil, fmt.Errorf("unable to get authorization token for graph endpoints: %+v", err)
-	}
+	if builder.UseMSAL {
+		// TODO: remove UseMSAL toggle and make this the default behaviour in v3.0
+		auth, err = builder.AuthConfig.GetMSALToken(ctx, environment.ResourceManager, sender, oauthConfig, string(environment.ResourceManager.Endpoint))
+		if err != nil {
+			return nil, fmt.Errorf("unable to get MSAL authorization token for resource manager API: %+v", err)
+		}
 
-	// Storage Endpoints
-	storageAuth, err := builder.AuthConfig.GetADALToken(ctx, sender, oauthConfig, env.ResourceIdentifiers.Storage)
-	if err != nil {
-		return nil, fmt.Errorf("unable to get authorization token for storage endpoints: %+v", err)
-	}
+		storageAuth, err = builder.AuthConfig.GetMSALToken(ctx, environment.Storage, sender, oauthConfig, string(environment.Storage.Endpoint))
+		if err != nil {
+			return nil, fmt.Errorf("unable to get MSAL authorization token for storage API: %+v", err)
+		}
 
-	// Synapse Endpoints
-	var synapseAuth autorest.Authorizer = nil
-	if env.ResourceIdentifiers.Synapse != azure.NotAvailable {
-		synapseAuth, err = builder.AuthConfig.GetADALToken(ctx, sender, oauthConfig, env.ResourceIdentifiers.Synapse)
+		if environment.Synapse.IsAvailable() {
+			synapseAuth, err = builder.AuthConfig.GetMSALToken(ctx, environment.Synapse, sender, oauthConfig, string(environment.Synapse.Endpoint))
+			if err != nil {
+				return nil, fmt.Errorf("unable to get MSAL authorization token for synapse API: %+v", err)
+			}
+		} else {
+			log.Printf("[DEBUG] Skipping building the Synapse MSAL Authorizer since this is not supported in the current Azure Environment")
+		}
+
+		batchManagementAuth, err = builder.AuthConfig.GetMSALToken(ctx, environment.BatchManagement, sender, oauthConfig, string(environment.BatchManagement.Endpoint))
 		if err != nil {
-			return nil, fmt.Errorf("unable to get authorization token for synapse endpoints: %+v", err)
+			return nil, fmt.Errorf("unable to get MSAL authorization token for batch management API: %+v", err)
+		}
+
+		keyVaultAuth = builder.AuthConfig.MSALBearerAuthorizerCallback(ctx, environment.KeyVault, sender, oauthConfig, string(environment.KeyVault.Endpoint))
+
+		// Helper for obtaining endpoint-specific tokens
+		tokenFunc = func(endpoint string) (autorest.Authorizer, error) {
+			api := environments.Api{Endpoint: environments.ApiEndpoint(endpoint)}
+			authorizer, err := builder.AuthConfig.GetMSALToken(ctx, api, sender, oauthConfig, endpoint)
+			if err != nil {
+				return nil, fmt.Errorf("getting MSAL authorization token for endpoint %s: %+v", endpoint, err)
+			}
+			return authorizer, nil
 		}
 	} else {
-		log.Printf("[DEBUG] Skipping building the Synapse Authorizer since this is not supported in the current Azure Environment")
-	}
+		auth, err = builder.AuthConfig.GetADALToken(ctx, sender, oauthConfig, env.TokenAudience)
+		if err != nil {
+			return nil, fmt.Errorf("unable to get ADAL authorization token for resource manager API: %+v", err)
+		}
 
-	// Key Vault Endpoints
-	keyVaultAuth := builder.AuthConfig.ADALBearerAuthorizerCallback(ctx, sender, oauthConfig)
+		storageAuth, err = builder.AuthConfig.GetADALToken(ctx, sender, oauthConfig, env.ResourceIdentifiers.Storage)
+		if err != nil {
+			return nil, fmt.Errorf("unable to get ADAL authorization token for storage API: %+v", err)
+		}
 
-	// Batch Management Endpoints
-	batchManagementAuth, err := builder.AuthConfig.GetADALToken(ctx, sender, oauthConfig, env.BatchManagementEndpoint)
-	if err != nil {
-		return nil, fmt.Errorf("unable to get authorization token for batch management endpoint: %+v", err)
+		if env.ResourceIdentifiers.Synapse != azure.NotAvailable {
+			synapseAuth, err = builder.AuthConfig.GetADALToken(ctx, sender, oauthConfig, env.ResourceIdentifiers.Synapse)
+			if err != nil {
+				return nil, fmt.Errorf("unable to get ADAL authorization token for synapse API: %+v", err)
+			}
+		} else {
+			log.Printf("[DEBUG] Skipping building the Synapse ADAL Authorizer since this is not supported in the current Azure Environment")
+		}
+
+		batchManagementAuth, err = builder.AuthConfig.GetADALToken(ctx, sender, oauthConfig, env.BatchManagementEndpoint)
+		if err != nil {
+			return nil, fmt.Errorf("unable to get ADAL authorization token for batch management API: %+v", err)
+		}
+
+		keyVaultAuth = builder.AuthConfig.ADALBearerAuthorizerCallback(ctx, sender, oauthConfig)
+
+		// Helper for obtaining endpoint-specific tokens
+		tokenFunc = func(endpoint string) (autorest.Authorizer, error) {
+			authorizer, err := builder.AuthConfig.GetADALToken(ctx, sender, oauthConfig, endpoint)
+			if err != nil {
+				return nil, fmt.Errorf("getting ADAL authorization token for endpoint %s: %+v", endpoint, err)
+			}
+			return authorizer, nil
+		}
+
+		graphAuth, err = builder.AuthConfig.GetADALToken(ctx, sender, oauthConfig, env.GraphEndpoint)
+		if err != nil {
+			return nil, fmt.Errorf("unable to get ADAL authorization token for aadgraph API: %+v", err)
+		}
 	}
 
 	o := &common.ClientOptions{
 		SubscriptionId:              builder.AuthConfig.SubscriptionID,
 		TenantID:                    builder.AuthConfig.TenantID,
 		PartnerId:                   builder.PartnerId,
 		TerraformVersion:            builder.TerraformVersion,
-		GraphAuthorizer:             graphAuth,
-		GraphEndpoint:               graphEndpoint,
 		KeyVaultAuthorizer:          keyVaultAuth,
 		ResourceManagerAuthorizer:   auth,
-		ResourceManagerEndpoint:     endpoint,
+		ResourceManagerEndpoint:     env.ResourceManagerEndpoint,
 		StorageAuthorizer:           storageAuth,
 		SynapseAuthorizer:           synapseAuth,
 		BatchManagementAuthorizer:   batchManagementAuth,
@@ -138,13 +191,13 @@ func Build(ctx context.Context, builder ClientBuilder) (*Client, error) {
 		Environment:                 *env,
 		Features:                    builder.Features,
 		StorageUseAzureAD:           builder.StorageUseAzureAD,
-		TokenFunc: func(endpoint string) (autorest.Authorizer, error) {
-			authorizer, err := builder.AuthConfig.GetADALToken(ctx, sender, oauthConfig, endpoint)
-			if err != nil {
-				return nil, fmt.Errorf("getting authorization token for endpoint %s: %+v", endpoint, err)
-			}
-			return authorizer, nil
-		},
+		TokenFunc:                   tokenFunc,
+	}
+
+	// TODO: remove in v3.0
+	if !builder.UseMSAL {
+		o.GraphEndpoint = env.GraphEndpoint
+		o.GraphAuthorizer = graphAuth
 	}
 
 	if err := client.Build(ctx, o); err != nil {

internal/common/client_options.go

@@ -13,14 +13,14 @@ import (
 	"github.com/hashicorp/terraform-provider-azurerm/version"
 )
 
+type EndpointTokenFunc func(endpoint string) (autorest.Authorizer, error)
+
 type ClientOptions struct {
 	SubscriptionId   string
 	TenantID         string
 	PartnerId        string
 	TerraformVersion string
 
-	GraphAuthorizer           autorest.Authorizer
-	GraphEndpoint             string
 	KeyVaultAuthorizer        autorest.Authorizer
 	ResourceManagerAuthorizer autorest.Authorizer
 	ResourceManagerEndpoint   string
@@ -37,7 +37,11 @@ type ClientOptions struct {
 	StorageUseAzureAD           bool
 
 	// Some Dataplane APIs require a token scoped for a specific endpoint
-	TokenFunc func(endpoint string) (autorest.Authorizer, error)
+	TokenFunc EndpointTokenFunc
+
+	// TODO: remove graph configuration in v3.0
+	GraphAuthorizer autorest.Authorizer
+	GraphEndpoint   string
 }
 
 func (o ClientOptions) ConfigureClient(c *autorest.Client, authorizer autorest.Authorizer) {

internal/provider/provider.go

@@ -134,7 +134,7 @@ func azureProvider(supportLegacyTestSuite bool) *schema.Provider {
 				Type:        schema.TypeString,
 				Required:    true,
 				DefaultFunc: schema.EnvDefaultFunc("ARM_ENVIRONMENT", "public"),
-				Description: "The Cloud Environment which should be used. Possible values are public, usgovernment, german, and china. Defaults to public.",
+				Description: "The Cloud Environment which should be used. Possible values are public, usgovernment, and china. Defaults to public.",
 			},
 
 			"metadata_host": {
@@ -228,6 +228,13 @@ func azureProvider(supportLegacyTestSuite bool) *schema.Provider {
 				DefaultFunc: schema.EnvDefaultFunc("ARM_STORAGE_USE_AZUREAD", false),
 				Description: "Should the AzureRM Provider use AzureAD to access the Storage Data Plane API's?",
 			},
+
+			"use_msal": {
+				Type:        schema.TypeBool,
+				Optional:    true,
+				Description: "Should Terraform obtain MSAL auth tokens and no longer use Azure Active Directory Graph?",
+				DefaultFunc: schema.MultiEnvDefaultFunc([]string{"ARM_USE_MSAL", "ARM_USE_MSGRAPH"}, false),
+			},
 		},
 
 		DataSourcesMap: dataSources,
@@ -258,7 +265,7 @@ func providerConfigure(p *schema.Provider) schema.ConfigureContextFunc {
 		}
 
 		if len(auxTenants) > 3 {
-			return nil, diag.FromErr(fmt.Errorf("The provider only supports 3 auxiliary tenant IDs"))
+			return nil, diag.Errorf("The provider only supports 3 auxiliary tenant IDs")
 		}
 
 		metadataHost := d.Get("metadata_host").(string)
@@ -291,11 +298,14 @@ func providerConfigure(p *schema.Provider) schema.ConfigureContextFunc {
 
 			// Doc Links
 			ClientSecretDocsLink: "https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/service_principal_client_secret",
+
+			// MSAL opt-in
+			UseMicrosoftGraph: d.Get("use_msal").(bool),
 		}
 
 		config, err := builder.Build()
 		if err != nil {
-			return nil, diag.FromErr(fmt.Errorf("building AzureRM Client: %s", err))
+			return nil, diag.Errorf("building AzureRM Client: %s", err)
 		}
 
 		terraformVersion := p.TerraformVersion
@@ -315,6 +325,7 @@ func providerConfigure(p *schema.Provider) schema.ConfigureContextFunc {
 			DisableTerraformPartnerID:   d.Get("disable_terraform_partner_id").(bool),
 			Features:                    expandFeatures(d.Get("features").([]interface{})),
 			StorageUseAzureAD:           d.Get("storage_use_azuread").(bool),
+			UseMSAL:                     d.Get("use_msal").(bool),
 
 			// this field is intentionally not exposed in the provider block, since it's only used for
 			// platform level tracing
@@ -339,16 +350,16 @@ func providerConfigure(p *schema.Provider) schema.ConfigureContextFunc {
 			// requests. This also lets us check if the provider credentials are correct.
 			providerList, err := client.Resource.ProvidersClient.List(ctx, nil, "")
 			if err != nil {
-				return nil, diag.FromErr(fmt.Errorf("Unable to list provider registration status, it is possible that this is due to invalid "+
+				return nil, diag.Errorf("Unable to list provider registration status, it is possible that this is due to invalid "+
 					"credentials or the service principal does not have permission to use the Resource Manager API, Azure "+
-					"error: %s", err))
+					"error: %s", err)
 			}
 
 			availableResourceProviders := providerList.Values()
 			requiredResourceProviders := resourceproviders.Required()
 
 			if err := resourceproviders.EnsureRegistered(ctx, *client.Resource.ProvidersClient, availableResourceProviders, requiredResourceProviders); err != nil {
-				return nil, diag.FromErr(fmt.Errorf(resourceProviderRegistrationErrorFmt, err))
+				return nil, diag.Errorf(resourceProviderRegistrationErrorFmt, err)
 			}
 		}
 

internal/services/authorization/client/client.go

@@ -1,35 +1,24 @@
 package client
 
 import (
-	"github.com/Azure/azure-sdk-for-go/services/graphrbac/1.6/graphrbac"
 	"github.com/Azure/azure-sdk-for-go/services/preview/authorization/mgmt/2020-04-01-preview/authorization"
 	"github.com/hashicorp/terraform-provider-azurerm/internal/common"
 )
 
 type Client struct {
-	GroupsClient            *graphrbac.GroupsClient
-	RoleAssignmentsClient   *authorization.RoleAssignmentsClient
-	RoleDefinitionsClient   *authorization.RoleDefinitionsClient
-	ServicePrincipalsClient *graphrbac.ServicePrincipalsClient
+	RoleAssignmentsClient *authorization.RoleAssignmentsClient
+	RoleDefinitionsClient *authorization.RoleDefinitionsClient
 }
 
 func NewClient(o *common.ClientOptions) *Client {
-	groupsClient := graphrbac.NewGroupsClientWithBaseURI(o.GraphEndpoint, o.TenantID)
-	o.ConfigureClient(&groupsClient.Client, o.GraphAuthorizer)
-
 	roleAssignmentsClient := authorization.NewRoleAssignmentsClientWithBaseURI(o.ResourceManagerEndpoint, o.SubscriptionId)
 	o.ConfigureClient(&roleAssignmentsClient.Client, o.ResourceManagerAuthorizer)
 
 	roleDefinitionsClient := authorization.NewRoleDefinitionsClientWithBaseURI(o.ResourceManagerEndpoint, o.SubscriptionId)
 	o.ConfigureClient(&roleDefinitionsClient.Client, o.ResourceManagerAuthorizer)
 
-	servicePrincipalsClient := graphrbac.NewServicePrincipalsClientWithBaseURI(o.GraphEndpoint, o.TenantID)
-	o.ConfigureClient(&servicePrincipalsClient.Client, o.GraphAuthorizer)
-
 	return &Client{
-		GroupsClient:            &groupsClient,
-		RoleAssignmentsClient:   &roleAssignmentsClient,
-		RoleDefinitionsClient:   &roleDefinitionsClient,
-		ServicePrincipalsClient: &servicePrincipalsClient,
+		RoleAssignmentsClient: &roleAssignmentsClient,
+		RoleDefinitionsClient: &roleDefinitionsClient,
 	}
 }