From ca117857df100c14c01a625b73a3ab69ec756568 Mon Sep 17 00:00:00 2001 From: Saverio Proto Date: Fri, 26 Jan 2024 14:16:42 +0100 Subject: [PATCH 1/2] Extend multiple_node_pools example to test AKS upgrades --- examples/multiple_node_pools/README.md | 74 +++++++++++++++++++++++ examples/multiple_node_pools/main.tf | 25 ++++---- examples/multiple_node_pools/variables.tf | 10 +++ 3 files changed, 98 insertions(+), 11 deletions(-) create mode 100644 examples/multiple_node_pools/README.md diff --git a/examples/multiple_node_pools/README.md b/examples/multiple_node_pools/README.md new file mode 100644 index 00000000..1cd82c9b --- /dev/null +++ b/examples/multiple_node_pools/README.md @@ -0,0 +1,74 @@ +# Testing the upgrade scenario + +You can use this example to manually test the upgrade scenario. + +See existing AKS versions: + +``` +% az aks get-versions --location centralus +KubernetesVersion Upgrades +------------------- ------------------------ +1.28.3 None available +1.28.0 1.28.3 +1.27.7 1.28.0, 1.28.3 +1.27.3 1.27.7, 1.28.0, 1.28.3 +1.26.10 1.27.3, 1.27.7 +1.26.6 1.26.10, 1.27.3, 1.27.7 +1.25.15 1.26.6, 1.26.10 +1.25.11 1.25.15, 1.26.6, 1.26.10 +``` + +In this example we test an upgrade from 1.26.10 to 1.27.7. + +## Create the AKS cluster at version 1.26.10: + +``` +terraform init -upgrade +terraform apply -var="kubernetes_version=1.26.10" -var="orchestrator_version=1.26.10" +``` + +Verify the AKS cluster version: + +``` +az aks list -o table # check AKS version +az aks get-credentials --resource-group --name +kubectl version # check api server version +kubectl get nodes # check nodes version +``` + +In the `az aks list` output you will have `KubernetesVersion` and `CurrentKubernetesVersion` both at 1.26.10 + +## Upgrade the AKS cluster control plane only to version 1.27.7 + +``` +terraform apply -var="kubernetes_version=1.27.7" -var="orchestrator_version=1.26.10" +``` + +Check the new versions: + + +``` +az aks list -o table # check AKS version +kubectl version # check api server version +kubectl get nodes # check nodes version +``` + +In the `az aks list` output you will have `KubernetesVersion` and `CurrentKubernetesVersion` both at 1.27.7 +The control plane version will be 1.27.7 and the nodes will be 1.26.10. + +## Upgrade the AKS cluster node pools to version 1.27.7 + +``` +terraform apply -var="kubernetes_version=1.27.7" -var="orchestrator_version=1.27.7" +``` + +Check the new versions: + +``` +az aks list -o table # check AKS version +kubectl version # check api server version +kubectl get nodes # check nodes version +``` + +In the `az aks list` output you will have `KubernetesVersion` and `CurrentKubernetesVersion` both at 1.27.7 +The control plane version will be 1.27.7 and the nodes will be 1.27.7. diff --git a/examples/multiple_node_pools/main.tf b/examples/multiple_node_pools/main.tf index a9ee476a..3e63afd8 100644 --- a/examples/multiple_node_pools/main.tf +++ b/examples/multiple_node_pools/main.tf @@ -34,10 +34,11 @@ resource "azurerm_subnet" "test" { locals { nodes = { for i in range(3) : "worker${i}" => { - name = substr("worker${i}${random_id.prefix.hex}", 0, 8) - vm_size = "Standard_D2s_v3" - node_count = 1 - vnet_subnet_id = azurerm_subnet.test.id + name = substr("worker${i}${random_id.prefix.hex}", 0, 8) + vm_size = "Standard_D2s_v3" + node_count = 1 + vnet_subnet_id = azurerm_subnet.test.id + orchestrator_version = var.orchestrator_version } } } @@ -45,11 +46,13 @@ locals { module "aks" { source = "../.." - prefix = "prefix-${random_id.prefix.hex}" - resource_group_name = local.resource_group.name - os_disk_size_gb = 60 - sku_tier = "Standard" - rbac_aad = false - vnet_subnet_id = azurerm_subnet.test.id - node_pools = local.nodes + prefix = "prefix-${random_id.prefix.hex}" + resource_group_name = local.resource_group.name + os_disk_size_gb = 60 + sku_tier = "Standard" + rbac_aad = false + vnet_subnet_id = azurerm_subnet.test.id + node_pools = local.nodes + kubernetes_version = var.kubernetes_version + orchestrator_version = var.orchestrator_version } diff --git a/examples/multiple_node_pools/variables.tf b/examples/multiple_node_pools/variables.tf index 753f24dd..1cdb7fe2 100644 --- a/examples/multiple_node_pools/variables.tf +++ b/examples/multiple_node_pools/variables.tf @@ -12,3 +12,13 @@ variable "resource_group_name" { type = string default = null } + +variable "kubernetes_version" { + type = string + default = null +} + +variable "orchestrator_version" { + type = string + default = null +} From c32d9ff43de97d488d559587ce68fecf10904afb Mon Sep 17 00:00:00 2001 From: Saverio Proto Date: Fri, 26 Jan 2024 15:26:36 +0100 Subject: [PATCH 2/2] explain in Readme why can't upgrade `var.kubernetes_version` and `var.orchestrator_version` at the same time --- examples/multiple_node_pools/README.md | 39 ++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/examples/multiple_node_pools/README.md b/examples/multiple_node_pools/README.md index 1cd82c9b..9e6971a2 100644 --- a/examples/multiple_node_pools/README.md +++ b/examples/multiple_node_pools/README.md @@ -72,3 +72,42 @@ kubectl get nodes # check nodes version In the `az aks list` output you will have `KubernetesVersion` and `CurrentKubernetesVersion` both at 1.27.7 The control plane version will be 1.27.7 and the nodes will be 1.27.7. + +## Note on Issue #465 + +The current implementation does not allow to upgrade `var.kubernetes_version` and `var.orchestrator_version` at the same time. + +We can test at this point a simultaneous upgrade to 1.28.3: + +``` +terraform apply -var="kubernetes_version=1.28.3" -var="orchestrator_version=1.28.3" +``` +This will generate a plan where the azure_kubernetes_cluster resource is updated in place and the system node pool is updated. + +``` + # module.aks.azurerm_kubernetes_cluster.main will be updated in-place + ~ resource "azurerm_kubernetes_cluster" "main" { + id = "/subscriptions//resourceGroups/4c273d71bc7898d6-rg/providers/Microsoft.ContainerService/managedClusters/prefix-4c273d71bc7898d6-aks" + name = "prefix-4c273d71bc7898d6-aks" + tags = {} + # (29 unchanged attributes hidden) + + ~ default_node_pool { + name = "nodepool" + ~ orchestrator_version = "1.27.7" -> "1.28.3" + tags = {} + # (22 unchanged attributes hidden) + } + + # (4 unchanged blocks hidden) + } +``` + +that will fail with the following error: + +``` +│ Error: updating Default Node Pool Agent Pool (Subscription: "" +│ Resource Group Name: "4c273d71bc7898d6-rg" +│ Managed Cluster Name: "prefix-4c273d71bc7898d6-aks" +│ Agent Pool Name: "nodepool") performing CreateOrUpdate: agentpools.AgentPoolsClient#CreateOrUpdate: Failure sending request: StatusCode=0 -- Original Error: Code="NodePoolMcVersionIncompatible" Message="Node pool version 1.28.3 and control plane version 1.27.7 are incompatible. Minor version of node pool version 28 is bigger than control plane version 27. For more information, please check https://aka.ms/aks/UpgradeVersionRules" +```