r/kubernetes_cluster: making the aci_connector_linux & `http_applic…

…ation_routing` blocks cloud specific The blocks `aci_connector_linux` and `http_application_routing` are not supported in Azure China or Azure US Government Clouds. As such this commit filters them out if they're disabled (since the API does not allow them to exist in the payload, even if disabled) and returns an error if attempting to use these add-ons in this region.
hashicorp · Apr 6, 2020 · babcc62 · babcc62
1 parent 3c77cc3
commit babcc62
Show file tree

Hide file tree

Showing 4 changed files with 83 additions and 19 deletions.
diff --git a/azurerm/internal/services/containers/client/client.go b/azurerm/internal/services/containers/client/client.go
@@ -4,6 +4,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2018-10-01/containerinstance"
 	"github.com/Azure/azure-sdk-for-go/services/containerregistry/mgmt/2018-09-01/containerregistry"
 	"github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2019-11-01/containerservice"
+	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/common"
 )
 
@@ -15,6 +16,8 @@ type Client struct {
 	ReplicationsClient       *containerregistry.ReplicationsClient
 	ServicesClient           *containerservice.ContainerServicesClient
 	WebhooksClient           *containerregistry.WebhooksClient
+
+	Environment azure.Environment
 }
 
 func NewClient(o *common.ClientOptions) *Client {
@@ -48,5 +51,6 @@ func NewClient(o *common.ClientOptions) *Client {
 		WebhooksClient:           &webhooksClient,
 		ReplicationsClient:       &replicationsClient,
 		ServicesClient:           &servicesClient,
+		Environment:              o.Environment,
 	}
 }
diff --git a/azurerm/internal/services/containers/kubernetes_addons.go b/azurerm/internal/services/containers/kubernetes_addons.go
@@ -1,24 +1,42 @@
 package containers
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2019-11-01/containerservice"
+	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
 	"github.com/hashicorp/terraform-plugin-sdk/helper/validation"
-	"github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure"
+	azureHelpers "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure"
 	"github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils"
 )
 
 const (
+	// note: the casing on these keys is important
 	aciConnectorKey           = "aciConnectorLinux"
 	azurePolicyKey            = "azurepolicy"
 	kubernetesDashboardKey    = "kubeDashboard"
 	httpApplicationRoutingKey = "httpApplicationRouting"
 	omsAgentKey               = "omsagent"
 )
 
-func SchemaKubernetesAddOnProfiles() *schema.Schema {
+// The AKS API hard-codes which add-ons are supported in which environment
+// as such unfortunately we can't just send "disabled" - we need to strip
+// the unsupported addons from the HTTP response. As such this defines
+// the list of unsupported addons in the defined region - e.g. by being
+// omited from this list an addon/environment combination will be supported
+var unsupportedAddonsForEnvironment = map[string][]string{
+	azure.ChinaCloud.Name: {
+		aciConnectorKey,           // https://github.com/terraform-providers/terraform-provider-azurerm/issues/5510
+		httpApplicationRoutingKey, // https://github.com/terraform-providers/terraform-provider-azurerm/issues/5960
+	},
+	azure.USGovernmentCloud.Name: {
+		httpApplicationRoutingKey, // https://github.com/terraform-providers/terraform-provider-azurerm/issues/5960
+	},
+}
+
+func schemaKubernetesAddOnProfiles() *schema.Schema {
 	return &schema.Schema{
 		Type:     schema.TypeList,
 		MaxItems: 1,
@@ -107,7 +125,7 @@ func SchemaKubernetesAddOnProfiles() *schema.Schema {
 							"log_analytics_workspace_id": {
 								Type:         schema.TypeString,
 								Optional:     true,
-								ValidateFunc: azure.ValidateResourceID,
+								ValidateFunc: azureHelpers.ValidateResourceID,
 							},
 						},
 					},
@@ -117,21 +135,21 @@ func SchemaKubernetesAddOnProfiles() *schema.Schema {
 	}
 }
 
-func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerservice.ManagedClusterAddonProfile {
+func expandKubernetesAddOnProfiles(input []interface{}, env azure.Environment) (*map[string]*containerservice.ManagedClusterAddonProfile, error) {
 	disabled := containerservice.ManagedClusterAddonProfile{
 		Enabled: utils.Bool(false),
 	}
 
 	profiles := map[string]*containerservice.ManagedClusterAddonProfile{
-		// note: the casing on these keys is important
 		aciConnectorKey:           &disabled,
 		azurePolicyKey:            &disabled,
 		kubernetesDashboardKey:    &disabled,
 		httpApplicationRoutingKey: &disabled,
 		omsAgentKey:               &disabled,
 	}
+
 	if len(input) == 0 {
-		return profiles
+		return filterUnsupportedKubernetesAddOns(profiles, env)
 	}
 
 	profile := input[0].(map[string]interface{})
@@ -141,7 +159,7 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser
 	if len(httpApplicationRouting) > 0 && httpApplicationRouting[0] != nil {
 		value := httpApplicationRouting[0].(map[string]interface{})
 		enabled := value["enabled"].(bool)
-		addonProfiles["httpApplicationRouting"] = &containerservice.ManagedClusterAddonProfile{
+		addonProfiles[httpApplicationRoutingKey] = &containerservice.ManagedClusterAddonProfile{
 			Enabled: utils.Bool(enabled),
 		}
 	}
@@ -156,7 +174,7 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser
 			config["logAnalyticsWorkspaceResourceID"] = utils.String(workspaceId.(string))
 		}
 
-		addonProfiles["omsagent"] = &containerservice.ManagedClusterAddonProfile{
+		addonProfiles[omsAgentKey] = &containerservice.ManagedClusterAddonProfile{
 			Enabled: utils.Bool(enabled),
 			Config:  config,
 		}
@@ -172,7 +190,7 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser
 			config["SubnetName"] = utils.String(subnetName.(string))
 		}
 
-		addonProfiles["aciConnectorLinux"] = &containerservice.ManagedClusterAddonProfile{
+		addonProfiles[aciConnectorKey] = &containerservice.ManagedClusterAddonProfile{
 			Enabled: utils.Bool(enabled),
 			Config:  config,
 		}
@@ -183,7 +201,7 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser
 		value := kubeDashboard[0].(map[string]interface{})
 		enabled := value["enabled"].(bool)
 
-		addonProfiles["kubeDashboard"] = &containerservice.ManagedClusterAddonProfile{
+		addonProfiles[kubernetesDashboardKey] = &containerservice.ManagedClusterAddonProfile{
 			Enabled: utils.Bool(enabled),
 			Config:  nil,
 		}
@@ -194,16 +212,45 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser
 		value := azurePolicy[0].(map[string]interface{})
 		enabled := value["enabled"].(bool)
 
-		addonProfiles["azurepolicy"] = &containerservice.ManagedClusterAddonProfile{
+		addonProfiles[azurePolicyKey] = &containerservice.ManagedClusterAddonProfile{
 			Enabled: utils.Bool(enabled),
 			Config:  nil,
 		}
 	}
 
-	return addonProfiles
+	return filterUnsupportedKubernetesAddOns(addonProfiles, env)
+}
+
+func filterUnsupportedKubernetesAddOns(input map[string]*containerservice.ManagedClusterAddonProfile, env azure.Environment) (*map[string]*containerservice.ManagedClusterAddonProfile, error) {
+	var filter = func(input map[string]*containerservice.ManagedClusterAddonProfile, key string) (*map[string]*containerservice.ManagedClusterAddonProfile, error) {
+		output := input
+		if v, ok := output[key]; ok {
+			if v.Enabled != nil && *v.Enabled {
+				return nil, fmt.Errorf("The addon %q is not supported for a Kubernetes Cluster located in %q", key, env.Name)
+			}
+
+			// otherwise it's disabled by default, so just remove it
+			delete(output, key)
+		}
+
+		return &output, nil
+	}
+
+	output := input
+	if unsupportedAddons, ok := unsupportedAddonsForEnvironment[env.Name]; ok {
+		for _, key := range unsupportedAddons {
+			out, err := filter(output, key)
+			if err != nil {
+				return nil, err
+			}
+
+			output = *out
+		}
+	}
+	return &output, nil
 }
 
-func FlattenKubernetesAddOnProfiles(profile map[string]*containerservice.ManagedClusterAddonProfile) []interface{} {
+func flattenKubernetesAddOnProfiles(profile map[string]*containerservice.ManagedClusterAddonProfile) []interface{} {
 	// when the Kubernetes Cluster is updated in the Portal - Azure updates the casing on the keys
 	// meaning what's submitted could be different to what's returned..
 	var locateInProfile = func(key string) *containerservice.ManagedClusterAddonProfile {

diff --git a/azurerm/internal/services/containers/resource_arm_kubernetes_cluster.go b/azurerm/internal/services/containers/resource_arm_kubernetes_cluster.go
@@ -133,7 +133,7 @@ func resourceArmKubernetesCluster() *schema.Resource {
 			},
 
 			// Optional
-			"addon_profile": SchemaKubernetesAddOnProfiles(),
+			"addon_profile": schemaKubernetesAddOnProfiles(),
 
 			"api_server_authorized_ip_ranges": {
 				Type:     schema.TypeSet,
@@ -519,6 +519,7 @@ func resourceArmKubernetesCluster() *schema.Resource {
 
 func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{}) error {
 	client := meta.(*clients.Client).Containers.KubernetesClustersClient
+	env := meta.(*clients.Client).Containers.Environment
 	ctx, cancel := timeouts.ForCreate(meta.(*clients.Client).StopContext, d)
 	defer cancel()
 	tenantId := meta.(*clients.Client).Account.TenantId
@@ -554,7 +555,10 @@ func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{}
 	}
 
 	addOnProfilesRaw := d.Get("addon_profile").([]interface{})
-	addonProfiles := ExpandKubernetesAddOnProfiles(addOnProfilesRaw)
+	addonProfiles, err := expandKubernetesAddOnProfiles(addOnProfilesRaw, env)
+	if err != nil {
+		return err
+	}
 
 	networkProfileRaw := d.Get("network_profile").([]interface{})
 	networkProfile, err := expandKubernetesClusterNetworkProfile(networkProfileRaw)
@@ -597,7 +601,7 @@ func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{}
 		ManagedClusterProperties: &containerservice.ManagedClusterProperties{
 			APIServerAccessProfile:  &apiAccessProfile,
 			AadProfile:              azureADProfile,
-			AddonProfiles:           addonProfiles,
+			AddonProfiles:           *addonProfiles,
 			AgentPoolProfiles:       agentProfiles,
 			DNSPrefix:               utils.String(dnsPrefix),
 			EnableRBAC:              utils.Bool(rbacEnabled),
@@ -642,6 +646,7 @@ func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{}
 func resourceArmKubernetesClusterUpdate(d *schema.ResourceData, meta interface{}) error {
 	nodePoolsClient := meta.(*clients.Client).Containers.AgentPoolsClient
 	clusterClient := meta.(*clients.Client).Containers.KubernetesClustersClient
+	env := meta.(*clients.Client).Containers.Environment
 	ctx, cancel := timeouts.ForUpdate(meta.(*clients.Client).StopContext, d)
 	defer cancel()
 	tenantId := meta.(*clients.Client).Account.TenantId
@@ -721,8 +726,12 @@ func resourceArmKubernetesClusterUpdate(d *schema.ResourceData, meta interface{}
 	if d.HasChange("addon_profile") {
 		updateCluster = true
 		addOnProfilesRaw := d.Get("addon_profile").([]interface{})
-		addonProfiles := ExpandKubernetesAddOnProfiles(addOnProfilesRaw)
-		existing.ManagedClusterProperties.AddonProfiles = addonProfiles
+		addonProfiles, err := expandKubernetesAddOnProfiles(addOnProfilesRaw, env)
+		if err != nil {
+			return err
+		}
+
+		existing.ManagedClusterProperties.AddonProfiles = *addonProfiles
 	}
 
 	if d.HasChange("api_server_authorized_ip_ranges") {
@@ -893,7 +902,7 @@ func resourceArmKubernetesClusterRead(d *schema.ResourceData, meta interface{})
 			d.Set("private_link_enabled", accessProfile.EnablePrivateCluster)
 		}
 
-		addonProfiles := FlattenKubernetesAddOnProfiles(props.AddonProfiles)
+		addonProfiles := flattenKubernetesAddOnProfiles(props.AddonProfiles)
 		if err := d.Set("addon_profile", addonProfiles); err != nil {
 			return fmt.Errorf("Error setting `addon_profile`: %+v", err)
 		}

diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown
@@ -138,12 +138,16 @@ A `addon_profile` block supports the following:
 
 * `aci_connector_linux` - (Optional) A `aci_connector_linux` block. For more details, please visit [Create and configure an AKS cluster to use virtual nodes](https://docs.microsoft.com/en-us/azure/aks/virtual-nodes-portal).
 
+-> **NOTE:** At this time ACI Connector's are not supported in Azure China.
+
 * `azure_policy` - (Optional) A `azure_policy` block as defined below. For more details please visit [Understand Azure Policy for Azure Kubernetes Service](https://docs.microsoft.com/en-ie/azure/governance/policy/concepts/rego-for-aks)
 
 -> **NOTE**: Azure Policy for Azure Kubernetes Service is currently in preview and not available to subscriptions that have not [opted-in](https://docs.microsoft.com/en-us/azure/governance/policy/concepts/rego-for-aks?toc=/azure/aks/toc.json) to join `Azure Policy` preview.
 
 * `http_application_routing` - (Optional) A `http_application_routing` block as defined below.
 
+-> **NOTE:** At this time HTTP Application Routing is not supported in Azure China or Azure US Government.
+
 * `kube_dashboard` - (Optional) A `kube_dashboard` block as defined below.
 
 * `oms_agent` - (Optional) A `oms_agent` block as defined below. For more details, please visit [How to onboard Azure Monitor for containers](https://docs.microsoft.com/en-us/azure/monitoring/monitoring-container-insights-onboard).