diff --git a/azurerm/internal/services/containers/client/client.go b/azurerm/internal/services/containers/client/client.go index b560bc983b4a1..1ac54642f4b10 100644 --- a/azurerm/internal/services/containers/client/client.go +++ b/azurerm/internal/services/containers/client/client.go @@ -4,6 +4,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2018-10-01/containerinstance" "github.com/Azure/azure-sdk-for-go/services/containerregistry/mgmt/2018-09-01/containerregistry" "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2019-11-01/containerservice" + "github.com/Azure/go-autorest/autorest/azure" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/common" ) @@ -15,6 +16,8 @@ type Client struct { ReplicationsClient *containerregistry.ReplicationsClient ServicesClient *containerservice.ContainerServicesClient WebhooksClient *containerregistry.WebhooksClient + + Environment azure.Environment } func NewClient(o *common.ClientOptions) *Client { @@ -48,5 +51,6 @@ func NewClient(o *common.ClientOptions) *Client { WebhooksClient: &webhooksClient, ReplicationsClient: &replicationsClient, ServicesClient: &servicesClient, + Environment: o.Environment, } } diff --git a/azurerm/internal/services/containers/kubernetes_addons.go b/azurerm/internal/services/containers/kubernetes_addons.go index 1f4579c58bb6b..c3c7b31fcc2d3 100644 --- a/azurerm/internal/services/containers/kubernetes_addons.go +++ b/azurerm/internal/services/containers/kubernetes_addons.go @@ -1,16 +1,19 @@ package containers import ( + "fmt" "strings" "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2019-11-01/containerservice" + "github.com/Azure/go-autorest/autorest/azure" "github.com/hashicorp/terraform-plugin-sdk/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/helper/validation" - "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure" + azureHelpers "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils" ) const ( + // note: the casing on these keys is important aciConnectorKey = "aciConnectorLinux" azurePolicyKey = "azurepolicy" kubernetesDashboardKey = "kubeDashboard" @@ -18,7 +21,22 @@ const ( omsAgentKey = "omsagent" ) -func SchemaKubernetesAddOnProfiles() *schema.Schema { +// The AKS API hard-codes which add-ons are supported in which environment +// as such unfortunately we can't just send "disabled" - we need to strip +// the unsupported addons from the HTTP response. As such this defines +// the list of unsupported addons in the defined region - e.g. by being +// omited from this list an addon/environment combination will be supported +var unsupportedAddonsForEnvironment = map[string][]string{ + azure.ChinaCloud.Name: { + aciConnectorKey, // https://github.com/terraform-providers/terraform-provider-azurerm/issues/5510 + httpApplicationRoutingKey, // https://github.com/terraform-providers/terraform-provider-azurerm/issues/5960 + }, + azure.USGovernmentCloud.Name: { + httpApplicationRoutingKey, // https://github.com/terraform-providers/terraform-provider-azurerm/issues/5960 + }, +} + +func schemaKubernetesAddOnProfiles() *schema.Schema { return &schema.Schema{ Type: schema.TypeList, MaxItems: 1, @@ -107,7 +125,7 @@ func SchemaKubernetesAddOnProfiles() *schema.Schema { "log_analytics_workspace_id": { Type: schema.TypeString, Optional: true, - ValidateFunc: azure.ValidateResourceID, + ValidateFunc: azureHelpers.ValidateResourceID, }, }, }, @@ -117,21 +135,21 @@ func SchemaKubernetesAddOnProfiles() *schema.Schema { } } -func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerservice.ManagedClusterAddonProfile { +func expandKubernetesAddOnProfiles(input []interface{}, env azure.Environment) (*map[string]*containerservice.ManagedClusterAddonProfile, error) { disabled := containerservice.ManagedClusterAddonProfile{ Enabled: utils.Bool(false), } profiles := map[string]*containerservice.ManagedClusterAddonProfile{ - // note: the casing on these keys is important aciConnectorKey: &disabled, azurePolicyKey: &disabled, kubernetesDashboardKey: &disabled, httpApplicationRoutingKey: &disabled, omsAgentKey: &disabled, } + if len(input) == 0 { - return profiles + return filterUnsupportedKubernetesAddOns(profiles, env) } profile := input[0].(map[string]interface{}) @@ -141,7 +159,7 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser if len(httpApplicationRouting) > 0 && httpApplicationRouting[0] != nil { value := httpApplicationRouting[0].(map[string]interface{}) enabled := value["enabled"].(bool) - addonProfiles["httpApplicationRouting"] = &containerservice.ManagedClusterAddonProfile{ + addonProfiles[httpApplicationRoutingKey] = &containerservice.ManagedClusterAddonProfile{ Enabled: utils.Bool(enabled), } } @@ -156,7 +174,7 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser config["logAnalyticsWorkspaceResourceID"] = utils.String(workspaceId.(string)) } - addonProfiles["omsagent"] = &containerservice.ManagedClusterAddonProfile{ + addonProfiles[omsAgentKey] = &containerservice.ManagedClusterAddonProfile{ Enabled: utils.Bool(enabled), Config: config, } @@ -172,7 +190,7 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser config["SubnetName"] = utils.String(subnetName.(string)) } - addonProfiles["aciConnectorLinux"] = &containerservice.ManagedClusterAddonProfile{ + addonProfiles[aciConnectorKey] = &containerservice.ManagedClusterAddonProfile{ Enabled: utils.Bool(enabled), Config: config, } @@ -183,7 +201,7 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser value := kubeDashboard[0].(map[string]interface{}) enabled := value["enabled"].(bool) - addonProfiles["kubeDashboard"] = &containerservice.ManagedClusterAddonProfile{ + addonProfiles[kubernetesDashboardKey] = &containerservice.ManagedClusterAddonProfile{ Enabled: utils.Bool(enabled), Config: nil, } @@ -194,16 +212,45 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser value := azurePolicy[0].(map[string]interface{}) enabled := value["enabled"].(bool) - addonProfiles["azurepolicy"] = &containerservice.ManagedClusterAddonProfile{ + addonProfiles[azurePolicyKey] = &containerservice.ManagedClusterAddonProfile{ Enabled: utils.Bool(enabled), Config: nil, } } - return addonProfiles + return filterUnsupportedKubernetesAddOns(addonProfiles, env) +} + +func filterUnsupportedKubernetesAddOns(input map[string]*containerservice.ManagedClusterAddonProfile, env azure.Environment) (*map[string]*containerservice.ManagedClusterAddonProfile, error) { + var filter = func(input map[string]*containerservice.ManagedClusterAddonProfile, key string) (*map[string]*containerservice.ManagedClusterAddonProfile, error) { + output := input + if v, ok := output[key]; ok { + if v.Enabled != nil && *v.Enabled { + return nil, fmt.Errorf("The addon %q is not supported for a Kubernetes Cluster located in %q", key, env.Name) + } + + // otherwise it's disabled by default, so just remove it + delete(output, key) + } + + return &output, nil + } + + output := input + if unsupportedAddons, ok := unsupportedAddonsForEnvironment[env.Name]; ok { + for _, key := range unsupportedAddons { + out, err := filter(output, key) + if err != nil { + return nil, err + } + + output = *out + } + } + return &output, nil } -func FlattenKubernetesAddOnProfiles(profile map[string]*containerservice.ManagedClusterAddonProfile) []interface{} { +func flattenKubernetesAddOnProfiles(profile map[string]*containerservice.ManagedClusterAddonProfile) []interface{} { // when the Kubernetes Cluster is updated in the Portal - Azure updates the casing on the keys // meaning what's submitted could be different to what's returned.. var locateInProfile = func(key string) *containerservice.ManagedClusterAddonProfile { diff --git a/azurerm/internal/services/containers/resource_arm_kubernetes_cluster.go b/azurerm/internal/services/containers/resource_arm_kubernetes_cluster.go index 4ffd5332901b0..9b573caa2a2a9 100644 --- a/azurerm/internal/services/containers/resource_arm_kubernetes_cluster.go +++ b/azurerm/internal/services/containers/resource_arm_kubernetes_cluster.go @@ -133,7 +133,7 @@ func resourceArmKubernetesCluster() *schema.Resource { }, // Optional - "addon_profile": SchemaKubernetesAddOnProfiles(), + "addon_profile": schemaKubernetesAddOnProfiles(), "api_server_authorized_ip_ranges": { Type: schema.TypeSet, @@ -519,6 +519,7 @@ func resourceArmKubernetesCluster() *schema.Resource { func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{}) error { client := meta.(*clients.Client).Containers.KubernetesClustersClient + env := meta.(*clients.Client).Containers.Environment ctx, cancel := timeouts.ForCreate(meta.(*clients.Client).StopContext, d) defer cancel() tenantId := meta.(*clients.Client).Account.TenantId @@ -554,7 +555,10 @@ func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{} } addOnProfilesRaw := d.Get("addon_profile").([]interface{}) - addonProfiles := ExpandKubernetesAddOnProfiles(addOnProfilesRaw) + addonProfiles, err := expandKubernetesAddOnProfiles(addOnProfilesRaw, env) + if err != nil { + return err + } networkProfileRaw := d.Get("network_profile").([]interface{}) networkProfile, err := expandKubernetesClusterNetworkProfile(networkProfileRaw) @@ -597,7 +601,7 @@ func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{} ManagedClusterProperties: &containerservice.ManagedClusterProperties{ APIServerAccessProfile: &apiAccessProfile, AadProfile: azureADProfile, - AddonProfiles: addonProfiles, + AddonProfiles: *addonProfiles, AgentPoolProfiles: agentProfiles, DNSPrefix: utils.String(dnsPrefix), EnableRBAC: utils.Bool(rbacEnabled), @@ -642,6 +646,7 @@ func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{} func resourceArmKubernetesClusterUpdate(d *schema.ResourceData, meta interface{}) error { nodePoolsClient := meta.(*clients.Client).Containers.AgentPoolsClient clusterClient := meta.(*clients.Client).Containers.KubernetesClustersClient + env := meta.(*clients.Client).Containers.Environment ctx, cancel := timeouts.ForUpdate(meta.(*clients.Client).StopContext, d) defer cancel() tenantId := meta.(*clients.Client).Account.TenantId @@ -721,8 +726,12 @@ func resourceArmKubernetesClusterUpdate(d *schema.ResourceData, meta interface{} if d.HasChange("addon_profile") { updateCluster = true addOnProfilesRaw := d.Get("addon_profile").([]interface{}) - addonProfiles := ExpandKubernetesAddOnProfiles(addOnProfilesRaw) - existing.ManagedClusterProperties.AddonProfiles = addonProfiles + addonProfiles, err := expandKubernetesAddOnProfiles(addOnProfilesRaw, env) + if err != nil { + return err + } + + existing.ManagedClusterProperties.AddonProfiles = *addonProfiles } if d.HasChange("api_server_authorized_ip_ranges") { @@ -893,7 +902,7 @@ func resourceArmKubernetesClusterRead(d *schema.ResourceData, meta interface{}) d.Set("private_link_enabled", accessProfile.EnablePrivateCluster) } - addonProfiles := FlattenKubernetesAddOnProfiles(props.AddonProfiles) + addonProfiles := flattenKubernetesAddOnProfiles(props.AddonProfiles) if err := d.Set("addon_profile", addonProfiles); err != nil { return fmt.Errorf("Error setting `addon_profile`: %+v", err) } diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown index 5f00319e000a5..e4f61e8da2a89 100644 --- a/website/docs/r/kubernetes_cluster.html.markdown +++ b/website/docs/r/kubernetes_cluster.html.markdown @@ -138,12 +138,16 @@ A `addon_profile` block supports the following: * `aci_connector_linux` - (Optional) A `aci_connector_linux` block. For more details, please visit [Create and configure an AKS cluster to use virtual nodes](https://docs.microsoft.com/en-us/azure/aks/virtual-nodes-portal). +-> **NOTE:** At this time ACI Connector's are not supported in Azure China. + * `azure_policy` - (Optional) A `azure_policy` block as defined below. For more details please visit [Understand Azure Policy for Azure Kubernetes Service](https://docs.microsoft.com/en-ie/azure/governance/policy/concepts/rego-for-aks) -> **NOTE**: Azure Policy for Azure Kubernetes Service is currently in preview and not available to subscriptions that have not [opted-in](https://docs.microsoft.com/en-us/azure/governance/policy/concepts/rego-for-aks?toc=/azure/aks/toc.json) to join `Azure Policy` preview. * `http_application_routing` - (Optional) A `http_application_routing` block as defined below. +-> **NOTE:** At this time HTTP Application Routing is not supported in Azure China or Azure US Government. + * `kube_dashboard` - (Optional) A `kube_dashboard` block as defined below. * `oms_agent` - (Optional) A `oms_agent` block as defined below. For more details, please visit [How to onboard Azure Monitor for containers](https://docs.microsoft.com/en-us/azure/monitoring/monitoring-container-insights-onboard).