diff --git a/internal/services/containers/kubernetes_cluster_node_pool_resource.go b/internal/services/containers/kubernetes_cluster_node_pool_resource.go index a4bdb1de57ef1..2729a2f118313 100644 --- a/internal/services/containers/kubernetes_cluster_node_pool_resource.go +++ b/internal/services/containers/kubernetes_cluster_node_pool_resource.go @@ -133,9 +133,9 @@ func resourceKubernetesClusterNodePool() *pluginsdk.Resource { }, false), }, - "kubelet_config": schemaNodePoolKubeletConfig(), + "kubelet_config": schemaNodePoolKubeletConfig(true), - "linux_os_config": schemaNodePoolLinuxOSConfig(), + "linux_os_config": schemaNodePoolLinuxOSConfig(true), "fips_enabled": { Type: pluginsdk.TypeBool, diff --git a/internal/services/containers/kubernetes_cluster_resource.go b/internal/services/containers/kubernetes_cluster_resource.go index 534ab1681c236..a300c9fa13892 100644 --- a/internal/services/containers/kubernetes_cluster_resource.go +++ b/internal/services/containers/kubernetes_cluster_resource.go @@ -2062,8 +2062,18 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{} } } + cycleNodePool := false + cycleNodePoolProperties := []string{"name", "enable_host_encryption", "enable_node_public_ip", "kubelet_config", "linux_os_config", "max_pods", + "node_taints", "only_critical_addons_enabled", "os_disk_size_gb", "os_disk_type", "os_sku", "vm_size", "zones"} + + for _, property := range cycleNodePoolProperties { + if d.HasChange("default_node_pool.0." + property) { + cycleNodePool = true + } + } + // if the default node pool name has changed, it means the initial attempt at resizing failed - if d.HasChange("default_node_pool.0.vm_size") || d.HasChange("default_node_pool.0.name") || d.HasChange("default_node_pool.0.os_sku") || d.HasChange("default_node_pool.0.os_disk_type") || d.HasChange("default_node_pool.0.os_disk_size_gb") || d.HasChange("default_node_pool.0.zones") { + if cycleNodePool { log.Printf("[DEBUG] Cycling Default Node Pool..") // to provide a seamless updating experience for the vm size of the default node pool we need to cycle the default // node pool by provisioning a temporary system node pool, tearing down the former default node pool and then diff --git a/internal/services/containers/kubernetes_cluster_scaling_resource_test.go b/internal/services/containers/kubernetes_cluster_scaling_resource_test.go index 25ccc0e1f97ff..2aa67e962b1f9 100644 --- a/internal/services/containers/kubernetes_cluster_scaling_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_scaling_resource_test.go @@ -587,7 +587,7 @@ resource "azurerm_kubernetes_cluster" "test" { temporary_name_for_rotation = "temp" node_count = 1 vm_size = "%s" - enable_host_encryption = true + enable_host_encryption = false } identity { @@ -620,11 +620,65 @@ resource "azurerm_kubernetes_cluster" "test" { dns_prefix = "acctestaks%d" default_node_pool { - name = "default" - temporary_name_for_rotation = "temp" - node_count = 1 - vm_size = "%s" - zones = %s + name = "default" + temporary_name_for_rotation = "temp" + node_count = 1 + vm_size = "%s" + zones = %s + enable_node_public_ip = true + max_pods = 60 + only_critical_addons_enabled = true + + kubelet_config { + cpu_manager_policy = "static" + cpu_cfs_quota_enabled = true + cpu_cfs_quota_period = "10ms" + image_gc_high_threshold = 90 + image_gc_low_threshold = 70 + topology_manager_policy = "best-effort" + allowed_unsafe_sysctls = ["kernel.msg*", "net.core.somaxconn"] + container_log_max_size_mb = 100 + container_log_max_line = 100000 + pod_max_pid = 12345 + } + + linux_os_config { + transparent_huge_page_enabled = "always" + transparent_huge_page_defrag = "always" + swap_file_size_mb = 300 + + sysctl_config { + fs_aio_max_nr = 65536 + fs_file_max = 100000 + fs_inotify_max_user_watches = 1000000 + fs_nr_open = 1048576 + kernel_threads_max = 200000 + net_core_netdev_max_backlog = 1800 + net_core_optmem_max = 30000 + net_core_rmem_max = 300000 + net_core_rmem_default = 300000 + net_core_somaxconn = 5000 + net_core_wmem_default = 300000 + net_core_wmem_max = 300000 + net_ipv4_ip_local_port_range_min = 32768 + net_ipv4_ip_local_port_range_max = 60000 + net_ipv4_neigh_default_gc_thresh1 = 128 + net_ipv4_neigh_default_gc_thresh2 = 512 + net_ipv4_neigh_default_gc_thresh3 = 1024 + net_ipv4_tcp_fin_timeout = 60 + net_ipv4_tcp_keepalive_probes = 9 + net_ipv4_tcp_keepalive_time = 6000 + net_ipv4_tcp_max_syn_backlog = 2048 + net_ipv4_tcp_max_tw_buckets = 100000 + net_ipv4_tcp_tw_reuse = true + net_ipv4_tcp_keepalive_intvl = 70 + net_netfilter_nf_conntrack_buckets = 65536 + net_netfilter_nf_conntrack_max = 200000 + vm_max_map_count = 65536 + vm_swappiness = 45 + vm_vfs_cache_pressure = 80 + } + } } identity { @@ -700,6 +754,57 @@ resource "azurerm_kubernetes_cluster" "test" { node_count = 1 os_sku = "%s" vm_size = "Standard_D2ads_v5" + + kubelet_config { + cpu_manager_policy = "static" + cpu_cfs_quota_enabled = true + cpu_cfs_quota_period = "10ms" + image_gc_high_threshold = 91 + image_gc_low_threshold = 71 + topology_manager_policy = "best-effort" + allowed_unsafe_sysctls = ["net.core.somaxconn"] + container_log_max_size_mb = 100 + container_log_max_line = 100000 + pod_max_pid = 12346 + } + + linux_os_config { + transparent_huge_page_enabled = "always" + transparent_huge_page_defrag = "always" + swap_file_size_mb = 301 + + sysctl_config { + fs_aio_max_nr = 65535 + fs_file_max = 100001 + fs_inotify_max_user_watches = 1000001 + fs_nr_open = 1048575 + kernel_threads_max = 200000 + net_core_netdev_max_backlog = 1800 + net_core_optmem_max = 30000 + net_core_rmem_max = 300000 + net_core_rmem_default = 300000 + net_core_somaxconn = 5000 + net_core_wmem_default = 300000 + net_core_wmem_max = 300000 + net_ipv4_ip_local_port_range_min = 32768 + net_ipv4_ip_local_port_range_max = 60000 + net_ipv4_neigh_default_gc_thresh1 = 128 + net_ipv4_neigh_default_gc_thresh2 = 512 + net_ipv4_neigh_default_gc_thresh3 = 1024 + net_ipv4_tcp_fin_timeout = 50 + net_ipv4_tcp_keepalive_probes = 9 + net_ipv4_tcp_keepalive_time = 6000 + net_ipv4_tcp_max_syn_backlog = 2048 + net_ipv4_tcp_max_tw_buckets = 100000 + net_ipv4_tcp_tw_reuse = true + net_ipv4_tcp_keepalive_intvl = 70 + net_netfilter_nf_conntrack_buckets = 65536 + net_netfilter_nf_conntrack_max = 200000 + vm_max_map_count = 65536 + vm_swappiness = 40 + vm_vfs_cache_pressure = 80 + } + } } identity { diff --git a/internal/services/containers/kubernetes_nodepool.go b/internal/services/containers/kubernetes_nodepool.go index 8a328b4d9c144..a6271eda4e32a 100644 --- a/internal/services/containers/kubernetes_nodepool.go +++ b/internal/services/containers/kubernetes_nodepool.go @@ -85,19 +85,17 @@ func SchemaDefaultNodePool() *pluginsdk.Schema { "enable_node_public_ip": { Type: pluginsdk.TypeBool, Optional: true, - ForceNew: true, }, // TODO 4.0: change this from enable_* to *_enabled "enable_host_encryption": { Type: pluginsdk.TypeBool, Optional: true, - ForceNew: true, }, - "kubelet_config": schemaNodePoolKubeletConfig(), + "kubelet_config": schemaNodePoolKubeletConfig(false), - "linux_os_config": schemaNodePoolLinuxOSConfig(), + "linux_os_config": schemaNodePoolLinuxOSConfig(false), "fips_enabled": { Type: pluginsdk.TypeBool, @@ -126,7 +124,6 @@ func SchemaDefaultNodePool() *pluginsdk.Schema { Type: pluginsdk.TypeInt, Optional: true, Computed: true, - ForceNew: true, }, "message_of_the_day": { @@ -171,7 +168,6 @@ func SchemaDefaultNodePool() *pluginsdk.Schema { "node_taints": { Type: pluginsdk.TypeList, - ForceNew: true, Optional: true, Elem: &pluginsdk.Schema{ Type: pluginsdk.TypeString, @@ -244,7 +240,6 @@ func SchemaDefaultNodePool() *pluginsdk.Schema { "only_critical_addons_enabled": { Type: pluginsdk.TypeBool, Optional: true, - ForceNew: true, }, "scale_down_mode": { @@ -285,18 +280,18 @@ func SchemaDefaultNodePool() *pluginsdk.Schema { } } -func schemaNodePoolKubeletConfig() *pluginsdk.Schema { +func schemaNodePoolKubeletConfig(forceNew bool) *pluginsdk.Schema { return &pluginsdk.Schema{ Type: pluginsdk.TypeList, Optional: true, - ForceNew: true, + ForceNew: forceNew, MaxItems: 1, Elem: &pluginsdk.Resource{ Schema: map[string]*pluginsdk.Schema{ "cpu_manager_policy": { Type: pluginsdk.TypeString, Optional: true, - ForceNew: true, + ForceNew: forceNew, ValidateFunc: validation.StringInSlice([]string{ "none", "static", @@ -306,33 +301,33 @@ func schemaNodePoolKubeletConfig() *pluginsdk.Schema { "cpu_cfs_quota_enabled": { Type: pluginsdk.TypeBool, Optional: true, - ForceNew: true, + ForceNew: forceNew, }, "cpu_cfs_quota_period": { Type: pluginsdk.TypeString, Optional: true, - ForceNew: true, + ForceNew: forceNew, }, "image_gc_high_threshold": { Type: pluginsdk.TypeInt, Optional: true, - ForceNew: true, + ForceNew: forceNew, ValidateFunc: validation.IntBetween(0, 100), }, "image_gc_low_threshold": { Type: pluginsdk.TypeInt, Optional: true, - ForceNew: true, + ForceNew: forceNew, ValidateFunc: validation.IntBetween(0, 100), }, "topology_manager_policy": { Type: pluginsdk.TypeString, Optional: true, - ForceNew: true, + ForceNew: forceNew, ValidateFunc: validation.StringInSlice([]string{ "none", "best-effort", @@ -344,7 +339,7 @@ func schemaNodePoolKubeletConfig() *pluginsdk.Schema { "allowed_unsafe_sysctls": { Type: pluginsdk.TypeSet, Optional: true, - ForceNew: true, + ForceNew: forceNew, Elem: &pluginsdk.Schema{ Type: pluginsdk.TypeString, }, @@ -353,32 +348,32 @@ func schemaNodePoolKubeletConfig() *pluginsdk.Schema { "container_log_max_size_mb": { Type: pluginsdk.TypeInt, Optional: true, - ForceNew: true, + ForceNew: forceNew, }, // TODO 4.0: change this to `container_log_max_files` "container_log_max_line": { Type: pluginsdk.TypeInt, Optional: true, - ForceNew: true, + ForceNew: forceNew, ValidateFunc: validation.IntAtLeast(2), }, "pod_max_pid": { Type: pluginsdk.TypeInt, Optional: true, - ForceNew: true, + ForceNew: forceNew, }, }, }, } } -func schemaNodePoolLinuxOSConfig() *pluginsdk.Schema { +func schemaNodePoolLinuxOSConfig(forceNew bool) *pluginsdk.Schema { return &pluginsdk.Schema{ Type: pluginsdk.TypeList, Optional: true, - ForceNew: true, + ForceNew: forceNew, MaxItems: 1, Elem: &pluginsdk.Resource{ Schema: map[string]*pluginsdk.Schema{ @@ -387,7 +382,7 @@ func schemaNodePoolLinuxOSConfig() *pluginsdk.Schema { "transparent_huge_page_enabled": { Type: pluginsdk.TypeString, Optional: true, - ForceNew: true, + ForceNew: forceNew, ValidateFunc: validation.StringInSlice([]string{ "always", "madvise", @@ -398,7 +393,7 @@ func schemaNodePoolLinuxOSConfig() *pluginsdk.Schema { "transparent_huge_page_defrag": { Type: pluginsdk.TypeString, Optional: true, - ForceNew: true, + ForceNew: forceNew, ValidateFunc: validation.StringInSlice([]string{ "always", "defer", @@ -411,7 +406,7 @@ func schemaNodePoolLinuxOSConfig() *pluginsdk.Schema { "swap_file_size_mb": { Type: pluginsdk.TypeInt, Optional: true, - ForceNew: true, + ForceNew: forceNew, }, }, }, diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown index 161d820f9d12c..097f7b67d2d8c 100644 --- a/website/docs/r/kubernetes_cluster.html.markdown +++ b/website/docs/r/kubernetes_cluster.html.markdown @@ -376,17 +376,21 @@ A `default_node_pool` block supports the following: -> **Note:** If you're using AutoScaling, you may wish to use [Terraform's `ignore_changes` functionality](https://www.terraform.io/docs/language/meta-arguments/lifecycle.html#ignore_changes) to ignore changes to the `node_count` field. -* `enable_host_encryption` - (Optional) Should the nodes in the Default Node Pool have host encryption enabled? Changing this forces a new resource to be created. +* `enable_host_encryption` - (Optional) Should the nodes in the Default Node Pool have host encryption enabled? -> **Note:** This requires that the Preview Feature `Microsoft.ContainerService/EnableEncryptionAtHostPreview` is enabled and the Resource Provider is re-registered. -* `enable_node_public_ip` - (Optional) Should nodes in this Node Pool have a Public IP Address? Changing this forces a new resource to be created. +* `enable_node_public_ip` - (Optional) Should nodes in this Node Pool have a Public IP Address? + +-> **Note:** Changing the `enable_host_encryption` or `enable_node_public_ip` Virtual Machine is done by cycling the system node pool of the cluster. `temporary_name_for_rotation` must be specified when attempting a resize. * `host_group_id` - (Optional) Specifies the ID of the Host Group within which this AKS Cluster should be created. Changing this forces a new resource to be created. -* `kubelet_config` - (Optional) A `kubelet_config` block as defined below. Changing this forces a new resource to be created. +* `kubelet_config` - (Optional) A `kubelet_config` block as defined below. + +* `linux_os_config` - (Optional) A `linux_os_config` block as defined below. -* `linux_os_config` - (Optional) A `linux_os_config` block as defined below. Changing this forces a new resource to be created. +-> **Note:** Changing the `kubelet_config` or `linux_os_config` Virtual Machine is done by cycling the system node pool of the cluster. `temporary_name_for_rotation` must be specified when attempting a resize. * `fips_enabled` - (Optional) Should the nodes in this Node Pool have Federal Information Processing Standard enabled? Changing this forces a new resource to be created. @@ -394,6 +398,8 @@ A `default_node_pool` block supports the following: * `max_pods` - (Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created. +-> **Note:** Changing the `max_pods` Virtual Machine is done by cycling the system node pool of the cluster. `temporary_name_for_rotation` must be specified when attempting a resize. + * `message_of_the_day` - (Optional) A base64-encoded string which will be written to /etc/motd after decoding. This allows customization of the message of the day for Linux nodes. It cannot be specified for Windows nodes and must be a static string (i.e. will be printed raw and not executed as a script). Changing this forces a new resource to be created. * `node_network_profile` - (Optional) A `node_network_profile` block as documented below. @@ -402,9 +408,11 @@ A `default_node_pool` block supports the following: * `node_labels` - (Optional) A map of Kubernetes labels which should be applied to nodes in the Default Node Pool. -* `node_taints` - (Optional) A list of the taints added to new nodes during node pool create and scale. Changing this forces a new resource to be created. +* `node_taints` - (Optional) A list of the taints added to new nodes during node pool create and scale. + +* `only_critical_addons_enabled` - (Optional) Enabling this option will taint default node pool with `CriticalAddonsOnly=true:NoSchedule` taint. -* `only_critical_addons_enabled` - (Optional) Enabling this option will taint default node pool with `CriticalAddonsOnly=true:NoSchedule` taint. Changing this forces a new resource to be created. +-> **Note:** Changing `node_taints` or `only_critical_addons_enabled` of the `default_node_pool` is done by cycling the system node pool of the cluster. `temporary_name_for_rotation` must be specified when attempting a resize. * `orchestrator_version` - (Optional) Version of Kubernetes used for the Agents. If not specified, the default node pool will be created with the version specified by `kubernetes_version`. If both are unspecified, the latest recommended version will be used at provisioning time (but won't auto-upgrade). AKS does not require an exact patch version to be specified, minor version aliases such as `1.22` are also supported. - The minor version's latest GA patch is automatically chosen in that case. More details can be found in [the documentation](https://docs.microsoft.com/en-us/azure/aks/supported-kubernetes-versions?tabs=azure-cli#alias-minor-version).