Skip to content

Commit

Permalink
Add cycle support for more properties
Browse files Browse the repository at this point in the history
  • Loading branch information
@jkroepke
jkroepke committed May 13, 2023
1 parent d6a7c0c commit f5c05a0
Show file tree
Hide file tree
Showing 5 changed files with 157 additions and 39 deletions.
4 changes: 2 additions & 2 deletions internal/services/containers/kubernetes_cluster_node_pool_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,9 @@ func resourceKubernetesClusterNodePool() *pluginsdk.Resource {
}, false),
},

"kubelet_config": schemaNodePoolKubeletConfig(),
"kubelet_config": schemaNodePoolKubeletConfig(true),

"linux_os_config": schemaNodePoolLinuxOSConfig(),
"linux_os_config": schemaNodePoolLinuxOSConfig(true),

"fips_enabled": {
Type: pluginsdk.TypeBool,
Expand Down
12 changes: 11 additions & 1 deletion internal/services/containers/kubernetes_cluster_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2062,8 +2062,18 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{}
}
}

cycleNodePool := false
cycleNodePoolProperties := []string{"name", "enable_host_encryption", "enable_node_public_ip", "kubelet_config", "linux_os_config", "max_pods",
"node_taints", "only_critical_addons_enabled", "os_disk_size_gb", "os_disk_type", "os_sku", "vm_size", "zones"}

for _, property := range cycleNodePoolProperties {
if d.HasChange("default_node_pool.0." + property) {
cycleNodePool = true
}
}

// if the default node pool name has changed, it means the initial attempt at resizing failed
if d.HasChange("default_node_pool.0.vm_size") || d.HasChange("default_node_pool.0.name") || d.HasChange("default_node_pool.0.os_sku") || d.HasChange("default_node_pool.0.os_disk_type") || d.HasChange("default_node_pool.0.os_disk_size_gb") || d.HasChange("default_node_pool.0.zones") {
if cycleNodePool {
log.Printf("[DEBUG] Cycling Default Node Pool..")
// to provide a seamless updating experience for the vm size of the default node pool we need to cycle the default
// node pool by provisioning a temporary system node pool, tearing down the former default node pool and then
Expand Down
117 changes: 111 additions & 6 deletions internal/services/containers/kubernetes_cluster_scaling_resource_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -587,7 +587,7 @@ resource "azurerm_kubernetes_cluster" "test" {
temporary_name_for_rotation = "temp"
node_count = 1
vm_size = "%s"
enable_host_encryption = true
enable_host_encryption = false
}
identity {
Expand Down Expand Up @@ -620,11 +620,65 @@ resource "azurerm_kubernetes_cluster" "test" {
dns_prefix = "acctestaks%d"
default_node_pool {
name = "default"
temporary_name_for_rotation = "temp"
node_count = 1
vm_size = "%s"
zones = %s
name = "default"
temporary_name_for_rotation = "temp"
node_count = 1
vm_size = "%s"
zones = %s
enable_node_public_ip = true
max_pods = 60
only_critical_addons_enabled = true
kubelet_config {
cpu_manager_policy = "static"
cpu_cfs_quota_enabled = true
cpu_cfs_quota_period = "10ms"
image_gc_high_threshold = 90
image_gc_low_threshold = 70
topology_manager_policy = "best-effort"
allowed_unsafe_sysctls = ["kernel.msg*", "net.core.somaxconn"]
container_log_max_size_mb = 100
container_log_max_line = 100000
pod_max_pid = 12345
}
linux_os_config {
transparent_huge_page_enabled = "always"
transparent_huge_page_defrag = "always"
swap_file_size_mb = 300
sysctl_config {
fs_aio_max_nr = 65536
fs_file_max = 100000
fs_inotify_max_user_watches = 1000000
fs_nr_open = 1048576
kernel_threads_max = 200000
net_core_netdev_max_backlog = 1800
net_core_optmem_max = 30000
net_core_rmem_max = 300000
net_core_rmem_default = 300000
net_core_somaxconn = 5000
net_core_wmem_default = 300000
net_core_wmem_max = 300000
net_ipv4_ip_local_port_range_min = 32768
net_ipv4_ip_local_port_range_max = 60000
net_ipv4_neigh_default_gc_thresh1 = 128
net_ipv4_neigh_default_gc_thresh2 = 512
net_ipv4_neigh_default_gc_thresh3 = 1024
net_ipv4_tcp_fin_timeout = 60
net_ipv4_tcp_keepalive_probes = 9
net_ipv4_tcp_keepalive_time = 6000
net_ipv4_tcp_max_syn_backlog = 2048
net_ipv4_tcp_max_tw_buckets = 100000
net_ipv4_tcp_tw_reuse = true
net_ipv4_tcp_keepalive_intvl = 70
net_netfilter_nf_conntrack_buckets = 65536
net_netfilter_nf_conntrack_max = 200000
vm_max_map_count = 65536
vm_swappiness = 45
vm_vfs_cache_pressure = 80
}
}
}
identity {
Expand Down Expand Up @@ -700,6 +754,57 @@ resource "azurerm_kubernetes_cluster" "test" {
node_count = 1
os_sku = "%s"
vm_size = "Standard_D2ads_v5"
kubelet_config {
cpu_manager_policy = "static"
cpu_cfs_quota_enabled = true
cpu_cfs_quota_period = "10ms"
image_gc_high_threshold = 91
image_gc_low_threshold = 71
topology_manager_policy = "best-effort"
allowed_unsafe_sysctls = ["net.core.somaxconn"]
container_log_max_size_mb = 100
container_log_max_line = 100000
pod_max_pid = 12346
}
linux_os_config {
transparent_huge_page_enabled = "always"
transparent_huge_page_defrag = "always"
swap_file_size_mb = 301
sysctl_config {
fs_aio_max_nr = 65535
fs_file_max = 100001
fs_inotify_max_user_watches = 1000001
fs_nr_open = 1048575
kernel_threads_max = 200000
net_core_netdev_max_backlog = 1800
net_core_optmem_max = 30000
net_core_rmem_max = 300000
net_core_rmem_default = 300000
net_core_somaxconn = 5000
net_core_wmem_default = 300000
net_core_wmem_max = 300000
net_ipv4_ip_local_port_range_min = 32768
net_ipv4_ip_local_port_range_max = 60000
net_ipv4_neigh_default_gc_thresh1 = 128
net_ipv4_neigh_default_gc_thresh2 = 512
net_ipv4_neigh_default_gc_thresh3 = 1024
net_ipv4_tcp_fin_timeout = 50
net_ipv4_tcp_keepalive_probes = 9
net_ipv4_tcp_keepalive_time = 6000
net_ipv4_tcp_max_syn_backlog = 2048
net_ipv4_tcp_max_tw_buckets = 100000
net_ipv4_tcp_tw_reuse = true
net_ipv4_tcp_keepalive_intvl = 70
net_netfilter_nf_conntrack_buckets = 65536
net_netfilter_nf_conntrack_max = 200000
vm_max_map_count = 65536
vm_swappiness = 40
vm_vfs_cache_pressure = 80
}
}
}
identity {
Expand Down
43 changes: 19 additions & 24 deletions internal/services/containers/kubernetes_nodepool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,19 +85,17 @@ func SchemaDefaultNodePool() *pluginsdk.Schema {
"enable_node_public_ip": {
Type: pluginsdk.TypeBool,
Optional: true,
ForceNew: true,
},

// TODO 4.0: change this from enable_* to *_enabled
"enable_host_encryption": {
Type: pluginsdk.TypeBool,
Optional: true,
ForceNew: true,
},

"kubelet_config": schemaNodePoolKubeletConfig(),
"kubelet_config": schemaNodePoolKubeletConfig(false),

"linux_os_config": schemaNodePoolLinuxOSConfig(),
"linux_os_config": schemaNodePoolLinuxOSConfig(false),

"fips_enabled": {
Type: pluginsdk.TypeBool,
Expand Down Expand Up @@ -126,7 +124,6 @@ func SchemaDefaultNodePool() *pluginsdk.Schema {
Type: pluginsdk.TypeInt,
Optional: true,
Computed: true,
ForceNew: true,
},

"message_of_the_day": {
Expand Down Expand Up @@ -171,7 +168,6 @@ func SchemaDefaultNodePool() *pluginsdk.Schema {

"node_taints": {
Type: pluginsdk.TypeList,
ForceNew: true,
Optional: true,
Elem: &pluginsdk.Schema{
Type: pluginsdk.TypeString,
Expand Down Expand Up @@ -244,7 +240,6 @@ func SchemaDefaultNodePool() *pluginsdk.Schema {
"only_critical_addons_enabled": {
Type: pluginsdk.TypeBool,
Optional: true,
ForceNew: true,
},

"scale_down_mode": {
Expand Down Expand Up @@ -285,18 +280,18 @@ func SchemaDefaultNodePool() *pluginsdk.Schema {
}
}

func schemaNodePoolKubeletConfig() *pluginsdk.Schema {
func schemaNodePoolKubeletConfig(forceNew bool) *pluginsdk.Schema {
return &pluginsdk.Schema{
Type: pluginsdk.TypeList,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
MaxItems: 1,
Elem: &pluginsdk.Resource{
Schema: map[string]*pluginsdk.Schema{
"cpu_manager_policy": {
Type: pluginsdk.TypeString,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
ValidateFunc: validation.StringInSlice([]string{
"none",
"static",
Expand All @@ -306,33 +301,33 @@ func schemaNodePoolKubeletConfig() *pluginsdk.Schema {
"cpu_cfs_quota_enabled": {
Type: pluginsdk.TypeBool,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
},

"cpu_cfs_quota_period": {
Type: pluginsdk.TypeString,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
},

"image_gc_high_threshold": {
Type: pluginsdk.TypeInt,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
ValidateFunc: validation.IntBetween(0, 100),
},

"image_gc_low_threshold": {
Type: pluginsdk.TypeInt,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
ValidateFunc: validation.IntBetween(0, 100),
},

"topology_manager_policy": {
Type: pluginsdk.TypeString,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
ValidateFunc: validation.StringInSlice([]string{
"none",
"best-effort",
Expand All @@ -344,7 +339,7 @@ func schemaNodePoolKubeletConfig() *pluginsdk.Schema {
"allowed_unsafe_sysctls": {
Type: pluginsdk.TypeSet,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
Elem: &pluginsdk.Schema{
Type: pluginsdk.TypeString,
},
Expand All @@ -353,32 +348,32 @@ func schemaNodePoolKubeletConfig() *pluginsdk.Schema {
"container_log_max_size_mb": {
Type: pluginsdk.TypeInt,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
},

// TODO 4.0: change this to `container_log_max_files`
"container_log_max_line": {
Type: pluginsdk.TypeInt,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
ValidateFunc: validation.IntAtLeast(2),
},

"pod_max_pid": {
Type: pluginsdk.TypeInt,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
},
},
},
}
}

func schemaNodePoolLinuxOSConfig() *pluginsdk.Schema {
func schemaNodePoolLinuxOSConfig(forceNew bool) *pluginsdk.Schema {
return &pluginsdk.Schema{
Type: pluginsdk.TypeList,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
MaxItems: 1,
Elem: &pluginsdk.Resource{
Schema: map[string]*pluginsdk.Schema{
Expand All @@ -387,7 +382,7 @@ func schemaNodePoolLinuxOSConfig() *pluginsdk.Schema {
"transparent_huge_page_enabled": {
Type: pluginsdk.TypeString,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
ValidateFunc: validation.StringInSlice([]string{
"always",
"madvise",
Expand All @@ -398,7 +393,7 @@ func schemaNodePoolLinuxOSConfig() *pluginsdk.Schema {
"transparent_huge_page_defrag": {
Type: pluginsdk.TypeString,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
ValidateFunc: validation.StringInSlice([]string{
"always",
"defer",
Expand All @@ -411,7 +406,7 @@ func schemaNodePoolLinuxOSConfig() *pluginsdk.Schema {
"swap_file_size_mb": {
Type: pluginsdk.TypeInt,
Optional: true,
ForceNew: true,
ForceNew: forceNew,
},
},
},
Expand Down
20 changes: 14 additions & 6 deletions website/docs/r/kubernetes_cluster.html.markdown
View file
Original file line number Diff line number Diff line change
Expand Up @@ -376,24 +376,30 @@ A `default_node_pool` block supports the following:

-> **Note:** If you're using AutoScaling, you may wish to use [Terraform's `ignore_changes` functionality](https://www.terraform.io/docs/language/meta-arguments/lifecycle.html#ignore_changes) to ignore changes to the `node_count` field.

* `enable_host_encryption` - (Optional) Should the nodes in the Default Node Pool have host encryption enabled? Changing this forces a new resource to be created.
* `enable_host_encryption` - (Optional) Should the nodes in the Default Node Pool have host encryption enabled?

-> **Note:** This requires that the Preview Feature `Microsoft.ContainerService/EnableEncryptionAtHostPreview` is enabled and the Resource Provider is re-registered.

* `enable_node_public_ip` - (Optional) Should nodes in this Node Pool have a Public IP Address? Changing this forces a new resource to be created.
* `enable_node_public_ip` - (Optional) Should nodes in this Node Pool have a Public IP Address?

-> **Note:** Changing the `enable_host_encryption` or `enable_node_public_ip` Virtual Machine is done by cycling the system node pool of the cluster. `temporary_name_for_rotation` must be specified when attempting a resize.

* `host_group_id` - (Optional) Specifies the ID of the Host Group within which this AKS Cluster should be created. Changing this forces a new resource to be created.

* `kubelet_config` - (Optional) A `kubelet_config` block as defined below. Changing this forces a new resource to be created.
* `kubelet_config` - (Optional) A `kubelet_config` block as defined below.

* `linux_os_config` - (Optional) A `linux_os_config` block as defined below.

* `linux_os_config` - (Optional) A `linux_os_config` block as defined below. Changing this forces a new resource to be created.
-> **Note:** Changing the `kubelet_config` or `linux_os_config` Virtual Machine is done by cycling the system node pool of the cluster. `temporary_name_for_rotation` must be specified when attempting a resize.

* `fips_enabled` - (Optional) Should the nodes in this Node Pool have Federal Information Processing Standard enabled? Changing this forces a new resource to be created.

* `kubelet_disk_type` - (Optional) The type of disk used by kubelet. Possible values are `OS` and `Temporary`.

* `max_pods` - (Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created.

-> **Note:** Changing the `max_pods` Virtual Machine is done by cycling the system node pool of the cluster. `temporary_name_for_rotation` must be specified when attempting a resize.

* `message_of_the_day` - (Optional) A base64-encoded string which will be written to /etc/motd after decoding. This allows customization of the message of the day for Linux nodes. It cannot be specified for Windows nodes and must be a static string (i.e. will be printed raw and not executed as a script). Changing this forces a new resource to be created.

* `node_network_profile` - (Optional) A `node_network_profile` block as documented below.
Expand All @@ -402,9 +408,11 @@ A `default_node_pool` block supports the following:

* `node_labels` - (Optional) A map of Kubernetes labels which should be applied to nodes in the Default Node Pool.

* `node_taints` - (Optional) A list of the taints added to new nodes during node pool create and scale. Changing this forces a new resource to be created.
* `node_taints` - (Optional) A list of the taints added to new nodes during node pool create and scale.

* `only_critical_addons_enabled` - (Optional) Enabling this option will taint default node pool with `CriticalAddonsOnly=true:NoSchedule` taint.

* `only_critical_addons_enabled` - (Optional) Enabling this option will taint default node pool with `CriticalAddonsOnly=true:NoSchedule` taint. Changing this forces a new resource to be created.
-> **Note:** Changing `node_taints` or `only_critical_addons_enabled` of the `default_node_pool` is done by cycling the system node pool of the cluster. `temporary_name_for_rotation` must be specified when attempting a resize.

* `orchestrator_version` - (Optional) Version of Kubernetes used for the Agents. If not specified, the default node pool will be created with the version specified by `kubernetes_version`. If both are unspecified, the latest recommended version will be used at provisioning time (but won't auto-upgrade). AKS does not require an exact patch version to be specified, minor version aliases such as `1.22` are also supported. - The minor version's latest GA patch is automatically chosen in that case. More details can be found in [the documentation](https://docs.microsoft.com/en-us/azure/aks/supported-kubernetes-versions?tabs=azure-cli#alias-minor-version).

Expand Down

0 comments on commit f5c05a0

Please sign in to comment.