diff --git a/internal/services/containers/kubernetes_cluster_auth_resource_test.go b/internal/services/containers/kubernetes_cluster_auth_resource_test.go index 6f9907496cce..a417e901a027 100644 --- a/internal/services/containers/kubernetes_cluster_auth_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_auth_resource_test.go @@ -26,7 +26,7 @@ func TestAccKubernetesCluster_apiServerAuthorizedIPRanges(t *testing.T) { check.That(data.ResourceName).Key("kube_admin_config.#").HasValue("0"), check.That(data.ResourceName).Key("kube_admin_config_raw").HasValue(""), check.That(data.ResourceName).Key("default_node_pool.0.max_pods").Exists(), - check.That(data.ResourceName).Key("api_server_authorized_ip_ranges.#").HasValue("3"), + check.That(data.ResourceName).Key("api_server_access_profile.0.authorized_ip_ranges.#").HasValue("3"), ), }, data.ImportStep(), @@ -466,11 +466,13 @@ resource "azurerm_kubernetes_cluster" "test" { load_balancer_sku = "standard" } - api_server_authorized_ip_ranges = [ - "8.8.8.8/32", - "8.8.4.4/32", - "8.8.2.0/24", - ] + api_server_access_profile { + authorized_ip_ranges = [ + "8.8.8.8/32", + "8.8.4.4/32", + "8.8.2.0/24", + ] + } } `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger) } diff --git a/internal/services/containers/kubernetes_cluster_network_resource_test.go b/internal/services/containers/kubernetes_cluster_network_resource_test.go index cb6f82b45e5c..14ea312743e0 100644 --- a/internal/services/containers/kubernetes_cluster_network_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_network_resource_test.go @@ -764,6 +764,177 @@ func TestAccKubernetesCluster_ebpfDataPlane(t *testing.T) { }) } +func TestAccKubernetesCluster_apiServerInManagedSubnet(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") + r := KubernetesClusterResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.apiServerInManagedSubnet(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + }) +} + +func TestAccKubernetesCluster_apiServerInBYOSubnet(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") + r := KubernetesClusterResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.apiServerInBYOSubnet(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + }) +} + +func (KubernetesClusterResource) apiServerInBYOSubnet(data acceptance.TestData) string { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} + +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%d" + location = "%s" +} + +resource "azurerm_virtual_network" "test" { + name = "acctestvirtnet%d" + address_space = ["10.0.0.0/8"] + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name +} + +resource "azurerm_subnet" "test" { + name = "acctestsubnet%d" + resource_group_name = azurerm_resource_group.test.name + virtual_network_name = azurerm_virtual_network.test.name + address_prefixes = ["10.1.0.0/16"] + + delegation { + name = "aks-delegation" + + service_delegation { + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + name = "Microsoft.ContainerService/managedClusters" + } + } +} + +resource "azurerm_subnet" "test1" { + name = "acctestsubnet1%d" + resource_group_name = azurerm_resource_group.test.name + virtual_network_name = azurerm_virtual_network.test.name + address_prefixes = ["10.2.0.0/16"] +} + +resource "azurerm_user_assigned_identity" "test" { + name = "acctestRG-aks-%d" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location +} + +resource "azurerm_role_assignment" "test" { + scope = azurerm_subnet.test.id + role_definition_name = "Network Contributor" + principal_id = azurerm_user_assigned_identity.test.principal_id +} + +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%d" + + api_server_access_profile { + vnet_integration_enabled = true + subnet_id = azurerm_subnet.test.id + } + + linux_profile { + admin_username = "acctestuser%d" + + ssh_key { + key_data = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCqaZoyiz1qbdOQ8xEf6uEu1cCwYowo5FHtsBhqLoDnnp7KUTEBN+L2NxRIfQ781rxV6Iq5jSav6b2Q8z5KiseOlvKA/RF2wqU0UPYqQviQhLmW6THTpmrv/YkUCuzxDpsH7DUDhZcwySLKVVe0Qm3+5N2Ta6UYH3lsDf9R9wTP2K/+vAnflKebuypNlmocIvakFWoZda18FOmsOoIVXQ8HWFNCuw9ZCunMSN62QGamCe3dL5cXlkgHYv7ekJE15IA9aOJcM7e90oeTqo+7HTcWfdu0qQqPWY5ujyMw/llas8tsXY85LFqRnr3gJ02bAscjc477+X+j/gkpFoN1QEmt terraform@demo.tld" + } + } + + default_node_pool { + name = "default" + node_count = 2 + vm_size = "Standard_DS2_v2" + vnet_subnet_id = azurerm_subnet.test1.id + } + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.test.id] + } + + network_profile { + network_plugin = "azure" + } + + depends_on = [ + azurerm_role_assignment.test, + ] +} +`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger) +} + +func (KubernetesClusterResource) apiServerInManagedSubnet(data acceptance.TestData) string { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} + +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%d" + location = "%s" +} + +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%d" + + api_server_access_profile { + vnet_integration_enabled = true + } + + linux_profile { + admin_username = "acctestuser%d" + + ssh_key { + key_data = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCqaZoyiz1qbdOQ8xEf6uEu1cCwYowo5FHtsBhqLoDnnp7KUTEBN+L2NxRIfQ781rxV6Iq5jSav6b2Q8z5KiseOlvKA/RF2wqU0UPYqQviQhLmW6THTpmrv/YkUCuzxDpsH7DUDhZcwySLKVVe0Qm3+5N2Ta6UYH3lsDf9R9wTP2K/+vAnflKebuypNlmocIvakFWoZda18FOmsOoIVXQ8HWFNCuw9ZCunMSN62QGamCe3dL5cXlkgHYv7ekJE15IA9aOJcM7e90oeTqo+7HTcWfdu0qQqPWY5ujyMw/llas8tsXY85LFqRnr3gJ02bAscjc477+X+j/gkpFoN1QEmt terraform@demo.tld" + } + } + + default_node_pool { + name = "default" + node_count = 2 + vm_size = "Standard_DS2_v2" + } + + identity { + type = "SystemAssigned" + } + + network_profile { + network_plugin = "azure" + } +} +`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, data.RandomInteger) +} + func (KubernetesClusterResource) advancedNetworkingConfig(data acceptance.TestData, networkPlugin string) string { return fmt.Sprintf(` provider "azurerm" { @@ -3089,7 +3260,9 @@ resource "azurerm_kubernetes_cluster" "test" { func (KubernetesClusterResource) publicNetworkAccess(data acceptance.TestData, enabled bool) string { authorizedIPConfig := "" if !enabled { - authorizedIPConfig = `api_server_authorized_ip_ranges = ["0.0.0.0/32"]` + authorizedIPConfig = `api_server_access_profile { + authorized_ip_ranges = ["0.0.0.0/32"] + }` } return fmt.Sprintf(` provider "azurerm" { diff --git a/internal/services/containers/kubernetes_cluster_resource.go b/internal/services/containers/kubernetes_cluster_resource.go index 8a9fae71ccca..f650eeb1faf8 100644 --- a/internal/services/containers/kubernetes_cluster_resource.go +++ b/internal/services/containers/kubernetes_cluster_resource.go @@ -26,10 +26,12 @@ import ( "github.com/hashicorp/terraform-provider-azurerm/helpers/tf" "github.com/hashicorp/terraform-provider-azurerm/helpers/validate" "github.com/hashicorp/terraform-provider-azurerm/internal/clients" + "github.com/hashicorp/terraform-provider-azurerm/internal/features" computeValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/compute/validate" "github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/kubernetes" "github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/migration" containerValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/validate" + networkValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/network/validate" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/pluginsdk" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/suppress" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/validation" @@ -63,6 +65,9 @@ func resourceKubernetesCluster() *pluginsdk.Resource { pluginsdk.ForceNewIfChange("windows_profile.0.gmsa.0.root_domain", func(ctx context.Context, old, new, meta interface{}) bool { return old != "" && new == "" }), + pluginsdk.ForceNewIfChange("api_server_access_profile.0.subnet_id", func(ctx context.Context, old, new, meta interface{}) bool { + return old != "" && new == "" + }), ), Timeouts: &pluginsdk.ResourceTimeout{ @@ -90,12 +95,41 @@ func resourceKubernetesCluster() *pluginsdk.Resource { "resource_group_name": commonschema.ResourceGroupName(), - "api_server_authorized_ip_ranges": { - Type: pluginsdk.TypeSet, + "api_server_access_profile": { + Type: pluginsdk.TypeList, Optional: true, - Elem: &pluginsdk.Schema{ - Type: pluginsdk.TypeString, - ValidateFunc: validate.CIDR, + Computed: !features.FourPointOhBeta(), + MaxItems: 1, + Elem: &pluginsdk.Resource{ + Schema: map[string]*pluginsdk.Schema{ + "vnet_integration_enabled": { + Type: pluginsdk.TypeBool, + Optional: true, + Default: false, + }, + + "subnet_id": { + Type: pluginsdk.TypeString, + Optional: true, + ValidateFunc: networkValidate.SubnetID, + }, + + "authorized_ip_ranges": { + Type: pluginsdk.TypeSet, + Optional: true, + Computed: !features.FourPointOhBeta(), + ConflictsWith: func() []string { + if !features.FourPointOhBeta() { + return []string{"api_server_authorized_ip_ranges"} + } + return []string{} + }(), + Elem: &pluginsdk.Schema{ + Type: pluginsdk.TypeString, + ValidateFunc: validate.CIDR, + }, + }, + }, }, }, @@ -1170,6 +1204,20 @@ func resourceKubernetesCluster() *pluginsdk.Resource { resource.Schema[k] = v } + if !features.FourPointOhBeta() { + resource.Schema["api_server_authorized_ip_ranges"] = &pluginsdk.Schema{ + Type: pluginsdk.TypeSet, + Optional: true, + Computed: true, + Elem: &pluginsdk.Schema{ + Type: pluginsdk.TypeString, + ValidateFunc: validate.CIDR, + }, + Deprecated: "This property has been renamed to `authorized_ip_ranges` within the `api_server_access_profile` block and will be removed in v4.0 of the provider", + ConflictsWith: []string{"api_server_access_profile.0.authorized_ip_ranges"}, + } + } + return resource } @@ -1252,25 +1300,11 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} workloadAutoscalerProfileRaw := d.Get("workload_autoscaler_profile").([]interface{}) workloadAutoscalerProfile := expandKubernetesClusterWorkloadAutoscalerProfile(workloadAutoscalerProfileRaw) - apiServerAuthorizedIPRangesRaw := d.Get("api_server_authorized_ip_ranges").(*pluginsdk.Set).List() - apiServerAuthorizedIPRanges := utils.ExpandStringSlice(apiServerAuthorizedIPRangesRaw) - - enablePrivateCluster := false - if v, ok := d.GetOk("private_cluster_enabled"); ok { - enablePrivateCluster = v.(bool) - } - - if !enablePrivateCluster && dnsPrefix == "" { + apiAccessProfile := expandKubernetesClusterAPIAccessProfile(d) + if !(*apiAccessProfile.EnablePrivateCluster) && dnsPrefix == "" { return fmt.Errorf("`dns_prefix` should be set if it is not a private cluster") } - apiAccessProfile := managedclusters.ManagedClusterAPIServerAccessProfile{ - EnablePrivateCluster: &enablePrivateCluster, - AuthorizedIPRanges: apiServerAuthorizedIPRanges, - EnablePrivateClusterPublicFQDN: utils.Bool(d.Get("private_cluster_public_fqdn_enabled").(bool)), - DisableRunCommand: utils.Bool(!d.Get("run_command_enabled").(bool)), - } - nodeResourceGroup := d.Get("node_resource_group").(string) if d.Get("enable_pod_security_policy").(bool) { @@ -1338,7 +1372,7 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} Tier: utils.ToPtr(managedclusters.ManagedClusterSKUTier(d.Get("sku_tier").(string))), }, Properties: &managedclusters.ManagedClusterProperties{ - ApiServerAccessProfile: &apiAccessProfile, + ApiServerAccessProfile: apiAccessProfile, AadProfile: azureADProfile, AddonProfiles: addonProfiles, AgentPoolProfiles: agentProfiles, @@ -1412,7 +1446,7 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} } if v, ok := d.GetOk("dns_prefix_private_cluster"); ok { - if !enablePrivateCluster || apiAccessProfile.PrivateDNSZone == nil || *apiAccessProfile.PrivateDNSZone == "System" || *apiAccessProfile.PrivateDNSZone == "None" { + if !(*apiAccessProfile.EnablePrivateCluster) || apiAccessProfile.PrivateDNSZone == nil || *apiAccessProfile.PrivateDNSZone == "System" || *apiAccessProfile.PrivateDNSZone == "None" { return fmt.Errorf("`dns_prefix_private_cluster` should only be set for private cluster with custom private dns zone") } parameters.Properties.FqdnSubdomain = utils.String(v.(string)) @@ -1575,34 +1609,11 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{} existing.Model.Properties.AddonProfiles = addonProfiles } - if d.HasChange("api_server_authorized_ip_ranges") { + if d.HasChange("api_server_authorized_ip_ranges") || d.HasChange("run_command_enabled") || d.HasChange("private_cluster_public_fqdn_enabled") || d.HasChange("api_server_access_profile") { updateCluster = true - apiServerAuthorizedIPRangesRaw := d.Get("api_server_authorized_ip_ranges").(*pluginsdk.Set).List() - - enablePrivateCluster := false - if v, ok := d.GetOk("private_cluster_enabled"); ok { - enablePrivateCluster = v.(bool) - } - existing.Model.Properties.ApiServerAccessProfile = &managedclusters.ManagedClusterAPIServerAccessProfile{ - AuthorizedIPRanges: utils.ExpandStringSlice(apiServerAuthorizedIPRangesRaw), - EnablePrivateCluster: &enablePrivateCluster, - } - if v, ok := d.GetOk("private_dns_zone_id"); ok { - existing.Model.Properties.ApiServerAccessProfile.PrivateDNSZone = utils.String(v.(string)) - } - } - if d.HasChange("private_cluster_public_fqdn_enabled") { - updateCluster = true - existing.Model.Properties.ApiServerAccessProfile.EnablePrivateClusterPublicFQDN = utils.Bool(d.Get("private_cluster_public_fqdn_enabled").(bool)) - } - - if d.HasChange("run_command_enabled") { - updateCluster = true - if existing.Model.Properties.ApiServerAccessProfile == nil { - existing.Model.Properties.ApiServerAccessProfile = &managedclusters.ManagedClusterAPIServerAccessProfile{} - } - existing.Model.Properties.ApiServerAccessProfile.DisableRunCommand = utils.Bool(!d.Get("run_command_enabled").(bool)) + apiServerProfile := expandKubernetesClusterAPIAccessProfile(d) + existing.Model.Properties.ApiServerAccessProfile = apiServerProfile } if d.HasChange("auto_scaler_profile") { @@ -2054,10 +2065,17 @@ func resourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{}) enablePrivateCluster := false enablePrivateClusterPublicFQDN := false runCommandEnabled := true + + apiServerAccessProfile := flattenKubernetesClusterAPIAccessProfile(props.ApiServerAccessProfile) + if err := d.Set("api_server_access_profile", apiServerAccessProfile); err != nil { + return fmt.Errorf("setting `api_server_access_profile`: %+v", err) + } if accessProfile := props.ApiServerAccessProfile; accessProfile != nil { - apiServerAuthorizedIPRanges := utils.FlattenStringSlice(accessProfile.AuthorizedIPRanges) - if err := d.Set("api_server_authorized_ip_ranges", apiServerAuthorizedIPRanges); err != nil { - return fmt.Errorf("setting `api_server_authorized_ip_ranges`: %+v", err) + if !features.FourPointOhBeta() { + apiServerAuthorizedIPRanges := utils.FlattenStringSlice(accessProfile.AuthorizedIPRanges) + if err := d.Set("api_server_authorized_ip_ranges", apiServerAuthorizedIPRanges); err != nil { + return fmt.Errorf("setting `api_server_authorized_ip_ranges`: %+v", err) + } } if accessProfile.EnablePrivateCluster != nil { enablePrivateCluster = *accessProfile.EnablePrivateCluster @@ -2077,7 +2095,6 @@ func resourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{}) d.Set("private_dns_zone_id", accessProfile.PrivateDNSZone) } } - d.Set("private_cluster_enabled", enablePrivateCluster) d.Set("private_cluster_public_fqdn_enabled", enablePrivateClusterPublicFQDN) d.Set("run_command_enabled", runCommandEnabled) @@ -2451,6 +2468,79 @@ func expandKubernetesClusterWindowsProfile(input []interface{}) *managedclusters } } +func expandKubernetesClusterAPIAccessProfile(d *pluginsdk.ResourceData) *managedclusters.ManagedClusterAPIServerAccessProfile { + apiServerAuthorizedIPRangesRaw := []interface{}{} + if !features.FourPointOhBeta() { + apiServerAuthorizedIPRangesRaw = d.Get("api_server_authorized_ip_ranges").(*pluginsdk.Set).List() + } + apiServerAuthorizedIPRanges := utils.ExpandStringSlice(apiServerAuthorizedIPRangesRaw) + + enablePrivateCluster := false + if v, ok := d.GetOk("private_cluster_enabled"); ok { + enablePrivateCluster = v.(bool) + } + + apiAccessProfile := &managedclusters.ManagedClusterAPIServerAccessProfile{ + EnablePrivateCluster: &enablePrivateCluster, + AuthorizedIPRanges: apiServerAuthorizedIPRanges, + EnablePrivateClusterPublicFQDN: utils.Bool(d.Get("private_cluster_public_fqdn_enabled").(bool)), + DisableRunCommand: utils.Bool(!d.Get("run_command_enabled").(bool)), + } + + apiServerAccessProfileRaw := d.Get("api_server_access_profile").([]interface{}) + if len(apiServerAccessProfileRaw) == 0 { + return apiAccessProfile + } + + config := apiServerAccessProfileRaw[0].(map[string]interface{}) + + if v := config["authorized_ip_ranges"]; v != nil { + apiServerAuthorizedIPRangesRaw := v.(*pluginsdk.Set).List() + if apiServerAuthorizedIPRanges := utils.ExpandStringSlice(apiServerAuthorizedIPRangesRaw); len(*apiServerAuthorizedIPRanges) > 0 { + apiAccessProfile.AuthorizedIPRanges = apiServerAuthorizedIPRanges + } + } + + enableVnetIntegration := false + if v := config["vnet_integration_enabled"]; v != nil { + enableVnetIntegration = v.(bool) + } + apiAccessProfile.EnableVnetIntegration = utils.Bool(enableVnetIntegration) + + subnetId := "" + if v := config["subnet_id"]; v != nil { + subnetId = v.(string) + } + apiAccessProfile.SubnetId = utils.String(subnetId) + + return apiAccessProfile +} + +func flattenKubernetesClusterAPIAccessProfile(profile *managedclusters.ManagedClusterAPIServerAccessProfile) []interface{} { + if profile == nil { + return []interface{}{} + } + + apiServerAuthorizedIPRanges := utils.FlattenStringSlice(profile.AuthorizedIPRanges) + + enableVnetIntegration := false + if profile.EnableVnetIntegration != nil { + enableVnetIntegration = *profile.EnableVnetIntegration + } + subnetId := "" + if profile.SubnetId != nil && *profile.SubnetId != "" { + subnetId = *profile.SubnetId + } + + return []interface{}{ + map[string]interface{}{ + "authorized_ip_ranges": apiServerAuthorizedIPRanges, + "subnet_id": subnetId, + "vnet_integration_enabled": enableVnetIntegration, + }, + } +} + func expandKubernetesClusterWorkloadAutoscalerProfile(input []interface{}) *managedclusters.ManagedClusterWorkloadAutoScalerProfile { if len(input) == 0 { return nil diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown index e18e46b6b0df..7c23d86be24e 100644 --- a/website/docs/r/kubernetes_cluster.html.markdown +++ b/website/docs/r/kubernetes_cluster.html.markdown @@ -91,7 +91,7 @@ In addition, one of either `identity` or `service_principal` blocks must be spec -> **Note:** Cluster Auto-Upgrade only updates to GA versions of Kubernetes and will not update to Preview versions. -* `api_server_authorized_ip_ranges` - (Optional) Set of authorized IP ranges to allow access to API server, e.g. ["198.51.100.0/24"]. +* `api_server_access_profile` - (Optional) An `api_server_access_profile` block as defined below. * `auto_scaler_profile` - (Optional) A `auto_scaler_profile` block as defined below. @@ -221,7 +221,7 @@ resource "azurerm_kubernetes_cluster" "example" { `public_network_access_enabled` - (Optional) Whether public network access is allowed for this Kubernetes Cluster. Defaults to `true`. Changing this forces a new resource to be created. --> **Note:** When `public_network_access_enabled` is set to `true`, `0.0.0.0/32` must be added to `api_server_authorized_ip_ranges`. +-> **Note:** When `public_network_access_enabled` is set to `true`, `0.0.0.0/32` must be added to `authorized_ip_ranges` in the `api_server_access_profile` block. * `role_based_access_control_enabled` - (Optional) Whether Role Based Access Control for the Kubernetes Cluster should be enabled. Defaults to `true`. Changing this forces a new resource to be created. @@ -268,6 +268,18 @@ resource "azurerm_subnet" "virtual" { --- +An `api_server_access_profile` block supports the following: + +* `authorized_ip_ranges` - (Optional) Set of authorized IP ranges to allow access to API server, e.g. ["198.51.100.0/24"]. + +* `subnet_id` - (Optional) The ID of the Subnet where the API server endpoint is delegated to. + +* `vnet_integration_enabled` - (Optional) Should API Server VNet Integration be enabled? For more details please visit [Use API Server VNet Integration](https://learn.microsoft.com/en-us/azure/aks/api-server-vnet-integration). + +-> **Note:** This requires that the Preview Feature `Microsoft.ContainerService/EnableAPIServerVnetIntegrationPreview` is enabled and the Resource Provider is re-registered, see [the documentation](https://learn.microsoft.com/en-us/azure/aks/api-server-vnet-integration#register-the-enableapiservervnetintegrationpreview-preview-feature) for more information. + +--- + An `auto_scaler_profile` block supports the following: * `balance_similar_node_groups` - (Optional) Detect similar node groups and balance the number of nodes between them. Defaults to `false`.